From 5ba3f43ea354af8ad55bea84372a2bc834d8757c Mon Sep 17 00:00:00 2001 From: Apple Date: Tue, 26 Sep 2017 16:45:51 +0000 Subject: [PATCH] xnu-4570.1.46.tar.gz --- .gitignore | 5 +- .upstream_base_commits | 2 + EXTERNAL_HEADERS/AvailabilityMacros.h | 2 + EXTERNAL_HEADERS/architecture/arm/Makefile | 21 + EXTERNAL_HEADERS/architecture/arm/arm_neon.h | 74267 ++++++++++++++++ EXTERNAL_HEADERS/corecrypto/cc.h | 26 +- EXTERNAL_HEADERS/corecrypto/cc_config.h | 137 +- EXTERNAL_HEADERS/corecrypto/cc_debug.h | 21 +- EXTERNAL_HEADERS/corecrypto/cc_priv.h | 35 +- .../corecrypto/cc_runtime_config.h | 48 + EXTERNAL_HEADERS/corecrypto/ccaes.h | 7 + EXTERNAL_HEADERS/corecrypto/ccasn1.h | 2 +- .../corecrypto/ccchacha20poly1305.h | 295 + EXTERNAL_HEADERS/corecrypto/cccmac.h | 92 - EXTERNAL_HEADERS/corecrypto/ccder.h | 122 +- EXTERNAL_HEADERS/corecrypto/ccdrbg.h | 6 +- EXTERNAL_HEADERS/corecrypto/cchmac.h | 23 + EXTERNAL_HEADERS/corecrypto/ccmode.h | 45 +- EXTERNAL_HEADERS/corecrypto/ccmode_factory.h | 13 +- EXTERNAL_HEADERS/corecrypto/ccmode_impl.h | 18 +- EXTERNAL_HEADERS/corecrypto/ccmode_siv.h | 6 +- EXTERNAL_HEADERS/corecrypto/ccn.h | 32 +- EXTERNAL_HEADERS/corecrypto/ccrng.h | 53 +- EXTERNAL_HEADERS/corecrypto/ccrsa.h | 2 +- EXTERNAL_HEADERS/corecrypto/cczp.h | 27 +- EXTERNAL_HEADERS/corecrypto/fipspost_trace.h | 45 + EXTERNAL_HEADERS/mach-o/arm/reloc.h | 60 + EXTERNAL_HEADERS/mach-o/arm64/reloc.h | 41 + EXTERNAL_HEADERS/mach-o/loader.h | 50 +- Makefile | 9 +- README.md | 7 + SETUP/kextsymboltool/kextsymboltool.c | 5 - bsd/arm/Makefile | 34 + bsd/arm/_limits.h | 9 + bsd/arm/_mcontext.h | 91 + bsd/arm/_param.h | 22 + bsd/arm/_types.h | 98 + bsd/arm/disklabel.h | 21 + bsd/arm/endian.h | 78 + bsd/arm/exec.h | 96 + bsd/arm/fasttrap_isa.h | 230 + bsd/arm/limits.h | 110 + bsd/arm/param.h | 147 + bsd/arm/profile.h | 32 + bsd/arm/psl.h | 16 + bsd/arm/ptrace.h | 43 + bsd/arm/reboot.h | 32 + bsd/arm/reg.h | 16 + bsd/arm/signal.h | 19 + bsd/arm/types.h | 151 + bsd/arm/vmparam.h | 35 + bsd/bsm/audit_kevents.h | 5 +- bsd/bsm/audit_record.h | 1 + bsd/conf/Makefile.arm | 13 + bsd/conf/Makefile.arm64 | 10 + bsd/conf/Makefile.template | 10 +- bsd/conf/files | 41 +- bsd/conf/files.arm | 20 + bsd/conf/files.arm64 | 21 + bsd/conf/files.x86_64 | 2 - bsd/conf/param.c | 9 + bsd/dev/arm/conf.c | 306 + bsd/dev/arm/cons.c | 111 + bsd/dev/arm/cpu_in_cksum.s | 444 + bsd/dev/arm/disassembler.c | 1097 + bsd/dev/arm/dtrace_isa.c | 631 + bsd/dev/arm/dtrace_subr_arm.c | 186 + bsd/dev/arm/fasttrap_isa.c | 1297 + bsd/dev/arm/fbt_arm.c | 681 + bsd/dev/arm/kern_machdep.c | 188 + bsd/dev/arm/km.c | 403 + bsd/dev/arm/munge.c | 767 + bsd/dev/arm/pci_device.h | 106 + bsd/dev/arm/pio.h | 224 + bsd/dev/arm/sdt_arm.c | 166 + bsd/dev/arm/stubs.c | 83 + bsd/dev/arm/sysctl.c | 60 + bsd/dev/arm/systemcalls.c | 651 + bsd/dev/arm/table_inline.h | 36 + bsd/dev/arm/unix_signal.c | 737 + bsd/dev/arm64/conf.c | 306 + bsd/dev/arm64/cpu_in_cksum.s | 404 + bsd/dev/arm64/disassembler.c | 1146 + bsd/dev/arm64/dtrace_isa.c | 696 + bsd/dev/arm64/dtrace_subr_arm.c | 219 + bsd/dev/arm64/fasttrap_isa.c | 2127 + bsd/dev/arm64/fbt_arm.c | 608 + bsd/dev/arm64/sdt_arm.c | 162 + bsd/dev/arm64/sysctl.c | 55 + bsd/dev/dtrace/dtrace.c | 325 +- bsd/dev/dtrace/dtrace_glue.c | 73 +- bsd/dev/dtrace/dtrace_ptss.c | 54 +- bsd/dev/dtrace/fasttrap.c | 459 +- bsd/dev/dtrace/fbt.c | 545 +- bsd/dev/dtrace/lockstat.c | 185 +- bsd/dev/dtrace/profile_prvd.c | 131 +- bsd/dev/dtrace/scripts/Makefile | 3 +- bsd/dev/dtrace/scripts/mptcp.d | 22 +- .../dtrace/scripts/vm_map_delete_permanent.d | 14 + bsd/dev/dtrace/sdt.c | 25 +- bsd/dev/dtrace/sdt_subr.c | 14 +- bsd/dev/dtrace/systrace.c | 180 +- bsd/dev/dtrace/systrace.h | 5 - bsd/dev/i386/conf.c | 171 +- bsd/dev/i386/dis_tables.c | 1081 +- bsd/dev/i386/dtrace_isa.c | 15 +- bsd/dev/i386/fasttrap_isa.c | 139 +- bsd/dev/i386/fbt_x86.c | 479 +- bsd/dev/i386/km.c | 9 +- bsd/dev/i386/sdt_x86.c | 2 +- bsd/dev/i386/sysctl.c | 7 + bsd/dev/i386/systemcalls.c | 18 + bsd/dev/i386/unix_signal.c | 113 +- bsd/dev/monotonic.c | 459 + bsd/dev/munge.c | 16 + bsd/dev/unix_startup.c | 14 +- bsd/i386/_mcontext.h | 51 +- bsd/i386/dis_tables.h | 10 +- bsd/i386/fasttrap_isa.h | 5 + bsd/kern/ast.h | 6 + bsd/kern/bsd_init.c | 62 +- bsd/kern/bsd_stubs.c | 17 +- bsd/kern/decmpfs.c | 19 +- bsd/kern/imageboot.c | 152 +- bsd/kern/kdebug.c | 496 +- bsd/kern/kern_acct.c | 1 - bsd/kern/kern_aio.c | 36 +- bsd/kern/kern_clock.c | 21 +- bsd/kern/kern_control.c | 20 +- bsd/kern/kern_core.c | 32 +- bsd/kern/kern_credential.c | 65 +- bsd/kern/kern_cs.c | 114 +- bsd/kern/kern_csr.c | 10 +- bsd/kern/kern_descrip.c | 114 +- bsd/kern/kern_event.c | 6462 +- bsd/kern/kern_exec.c | 293 +- bsd/kern/kern_exit.c | 259 +- bsd/kern/kern_fork.c | 47 +- bsd/kern/kern_guarded.c | 750 +- bsd/kern/kern_kpc.c | 192 +- bsd/kern/kern_ktrace.c | 117 +- bsd/kern/kern_lockf.c | 6 + bsd/kern/kern_malloc.c | 72 +- bsd/kern/kern_memorystatus.c | 1493 +- bsd/kern/kern_mib.c | 91 + bsd/kern/kern_mman.c | 216 +- bsd/kern/kern_newsysctl.c | 4 + bsd/kern/kern_ntptime.c | 782 + bsd/kern/kern_overrides.c | 1 - bsd/kern/kern_priv.c | 23 +- bsd/kern/kern_proc.c | 98 +- bsd/kern/kern_resource.c | 205 +- bsd/kern/kern_shutdown.c | 99 +- bsd/kern/kern_sig.c | 26 +- bsd/kern/kern_symfile.c | 4 +- bsd/kern/kern_synch.c | 2 - bsd/kern/kern_sysctl.c | 514 +- bsd/kern/kern_time.c | 72 +- bsd/kern/kern_xxx.c | 27 +- bsd/kern/kpi_mbuf.c | 23 +- bsd/kern/kpi_socket.c | 136 +- bsd/kern/kpi_socketfilter.c | 39 +- bsd/kern/mach_loader.c | 512 +- bsd/kern/mach_loader.h | 1 + bsd/kern/mach_process.c | 14 + bsd/kern/makesyscalls.sh | 83 +- bsd/kern/mcache.c | 133 +- bsd/kern/policy_check.c | 21 +- bsd/kern/posix_sem.c | 31 +- bsd/kern/posix_shm.c | 37 +- bsd/kern/proc_info.c | 279 +- bsd/kern/process_policy.c | 158 +- bsd/kern/pthread_shims.c | 79 +- bsd/kern/stackshot.c | 3 +- bsd/kern/subr_eventhandler.c | 359 + bsd/kern/subr_log.c | 19 +- bsd/kern/subr_prf.c | 9 +- bsd/kern/subr_prof.c | 1 - bsd/kern/sys_coalition.c | 34 +- bsd/kern/sys_generic.c | 22 +- bsd/kern/sys_pipe.c | 46 +- bsd/kern/sys_socket.c | 8 +- bsd/kern/sys_ulock.c | 24 +- bsd/kern/sys_work_interval.c | 135 +- bsd/kern/syscalls.master | 37 +- bsd/kern/sysv_msg.c | 6 +- bsd/kern/sysv_shm.c | 9 +- bsd/kern/trace_codes | 244 +- bsd/kern/tty.c | 455 +- bsd/kern/tty_ptmx.c | 354 +- bsd/kern/tty_pty.c | 36 +- bsd/kern/ubc_subr.c | 103 +- bsd/kern/uipc_domain.c | 50 +- bsd/kern/uipc_mbuf.c | 205 +- bsd/kern/uipc_mbuf2.c | 86 +- bsd/kern/uipc_socket.c | 333 +- bsd/kern/uipc_socket2.c | 150 +- bsd/kern/uipc_syscalls.c | 80 +- bsd/kern/uipc_usrreq.c | 36 +- bsd/libkern/libkern.h | 25 +- bsd/libkern/url_encode.c | 3 - bsd/machine/Makefile | 2 +- bsd/machine/_limits.h | 2 + bsd/machine/_mcontext.h | 2 + bsd/machine/_param.h | 2 + bsd/machine/_types.h | 2 + bsd/machine/disklabel.h | 2 + bsd/machine/endian.h | 2 + bsd/machine/exec.h | 2 + bsd/machine/fasttrap_isa.h | 2 + bsd/machine/limits.h | 2 + bsd/machine/param.h | 2 + bsd/machine/profile.h | 2 + bsd/machine/psl.h | 2 + bsd/machine/ptrace.h | 2 + bsd/machine/reboot.h | 2 + bsd/machine/reg.h | 2 + bsd/machine/signal.h | 2 + bsd/machine/smp.h | 39 + bsd/machine/types.h | 2 + bsd/machine/vmparam.h | 2 + bsd/man/man2/Makefile | 9 +- bsd/man/man2/clonefile.2 | 15 +- bsd/man/man2/connectx.2 | 15 - bsd/man/man2/exchangedata.2 | 4 +- bsd/man/man2/fcntl.2 | 8 +- bsd/man/man2/fs_snapshot_create.2 | 201 + bsd/man/man2/fs_snapshot_delete.2 | 1 + bsd/man/man2/fs_snapshot_list.2 | 1 + bsd/man/man2/fs_snapshot_rename.2 | 1 + bsd/man/man2/fsgetpath.2 | 126 + bsd/man/man2/futimens.2 | 1 + bsd/man/man2/getattrlist.2 | 83 +- bsd/man/man2/kqueue.2 | 70 +- bsd/man/man2/mount.2 | 23 +- bsd/man/man2/peeloff.2 | 99 - bsd/man/man2/posix_spawn.2 | 3 + bsd/man/man2/profil.2 | 144 - bsd/man/man2/readlink.2 | 1 + bsd/man/man2/setattrlist.2 | 60 +- bsd/man/man2/setattrlistat.2 | 1 + bsd/man/man2/utimensat.2 | 256 + bsd/man/man9/monotonic.9 | 81 + bsd/miscfs/devfs/devfs.h | 1 + bsd/miscfs/devfs/devfs_fdesc_support.c | 8 +- bsd/miscfs/devfs/devfs_tree.c | 2 +- bsd/miscfs/devfs/devfs_vnops.c | 52 +- bsd/miscfs/devfs/devfsdefs.h | 2 - bsd/miscfs/fifofs/fifo_vnops.c | 2 +- bsd/miscfs/nullfs/nullfs.h | 2 +- bsd/miscfs/routefs/routefs_ops.c | 2 +- bsd/miscfs/specfs/spec_vnops.c | 421 +- bsd/miscfs/specfs/specdev.h | 2 +- bsd/net/Makefile | 2 + bsd/net/altq/altq.h | 9 +- bsd/net/altq/altq_cbq.c | 272 - bsd/net/altq/altq_cbq.h | 19 +- bsd/net/altq/altq_fairq.c | 304 - bsd/net/altq/altq_fairq.h | 18 +- bsd/net/altq/altq_hfsc.c | 290 - bsd/net/altq/altq_hfsc.h | 18 +- bsd/net/altq/altq_priq.c | 266 - bsd/net/altq/altq_priq.h | 18 +- bsd/net/altq/altq_qfq.c | 240 - bsd/net/altq/altq_qfq.h | 18 +- bsd/net/altq/altq_subr.c | 487 - bsd/net/altq/altq_var.h | 95 - bsd/net/altq/if_altq.h | 168 - bsd/net/bpf.c | 346 +- bsd/net/bpf.h | 29 +- bsd/net/bpf_filter.c | 296 +- bsd/net/bridgestp.c | 2 +- bsd/net/classq/classq.c | 271 +- bsd/net/classq/classq.h | 44 +- bsd/net/classq/classq_blue.c | 385 - bsd/net/classq/classq_blue.h | 47 +- bsd/net/classq/classq_fq_codel.c | 257 +- bsd/net/classq/classq_fq_codel.h | 31 +- bsd/net/classq/classq_red.c | 630 - bsd/net/classq/classq_red.h | 78 +- bsd/net/classq/classq_rio.c | 554 - bsd/net/classq/classq_rio.h | 61 +- bsd/net/classq/classq_sfb.c | 267 +- bsd/net/classq/classq_sfb.h | 8 +- bsd/net/classq/classq_subr.c | 447 +- bsd/net/classq/if_classq.h | 140 +- bsd/net/content_filter.c | 265 +- bsd/net/content_filter.h | 81 +- bsd/net/dlil.c | 1067 +- bsd/net/dlil.h | 85 +- bsd/net/ether_if_module.c | 13 +- bsd/net/ethernet.h | 12 + bsd/net/flowadv.c | 7 +- bsd/net/flowadv.h | 5 +- bsd/net/flowhash.c | 63 +- bsd/net/if.c | 357 +- bsd/net/if.h | 67 +- bsd/net/if_bond.c | 4 +- bsd/net/if_bridge.c | 603 +- bsd/net/if_dl.h | 6 + bsd/net/if_fake.c | 1029 + bsd/net/if_fake_var.h | 76 + bsd/net/if_gif.c | 9 +- bsd/net/if_gif.h | 2 +- bsd/net/if_ipsec.c | 2491 +- bsd/net/if_ipsec.h | 14 +- bsd/net/if_llatbl.c | 860 + bsd/net/if_llatbl.h | 302 + bsd/net/if_llreach.c | 2 +- bsd/net/if_llreach.h | 4 +- bsd/net/if_loop.c | 27 +- bsd/net/if_pflog.c | 19 +- bsd/net/if_stf.c | 9 +- bsd/net/if_utun.c | 2198 +- bsd/net/if_utun.h | 15 +- bsd/net/if_var.h | 342 +- bsd/net/if_vlan.c | 92 +- bsd/net/iptap.c | 42 +- bsd/net/kpi_interface.c | 312 +- bsd/net/kpi_interface.h | 135 +- bsd/net/kpi_interfacefilter.c | 22 +- bsd/net/kpi_interfacefilter.h | 10 +- bsd/net/kpi_protocol.c | 2 +- bsd/net/ndrv.c | 10 +- bsd/net/necp.c | 1525 +- bsd/net/necp.h | 373 +- bsd/net/necp_client.c | 3371 +- bsd/net/net_api_stats.h | 160 + bsd/net/net_kev.h | 17 +- bsd/net/net_stubs.c | 12 +- bsd/net/netsrc.c | 381 +- bsd/net/netsrc.h | 58 +- bsd/net/network_agent.c | 304 +- bsd/net/network_agent.h | 65 +- bsd/net/ntstat.c | 1686 +- bsd/net/ntstat.h | 450 +- bsd/net/nwk_wq.c | 137 + bsd/net/nwk_wq.h | 45 + bsd/net/packet_mangler.c | 2 +- bsd/net/pf.c | 1482 +- bsd/net/pf_if.c | 18 +- bsd/net/pf_ioctl.c | 619 +- bsd/net/pf_norm.c | 435 +- bsd/net/pf_osfp.c | 8 +- bsd/net/pf_pbuf.c | 410 + bsd/net/pf_pbuf.h | 106 + bsd/net/pf_ruleset.c | 18 +- bsd/net/pf_table.c | 66 +- bsd/net/pfkeyv2.h | 3 + bsd/net/pfvar.h | 78 +- bsd/net/pktap.c | 59 +- bsd/net/pktap.h | 6 +- bsd/net/pktsched/pktsched.c | 224 +- bsd/net/pktsched/pktsched.h | 42 +- bsd/net/pktsched/pktsched_cbq.c | 705 - bsd/net/pktsched/pktsched_cbq.h | 47 +- bsd/net/pktsched/pktsched_fairq.c | 1300 - bsd/net/pktsched/pktsched_fairq.h | 93 +- bsd/net/pktsched/pktsched_fq_codel.c | 577 +- bsd/net/pktsched/pktsched_fq_codel.h | 35 +- bsd/net/pktsched/pktsched_hfsc.c | 2065 - bsd/net/pktsched/pktsched_hfsc.h | 177 +- bsd/net/pktsched/pktsched_priq.c | 1309 - bsd/net/pktsched/pktsched_priq.h | 69 +- bsd/net/pktsched/pktsched_qfq.c | 528 +- bsd/net/pktsched/pktsched_qfq.h | 22 +- bsd/net/pktsched/pktsched_rmclass.c | 1852 - bsd/net/pktsched/pktsched_rmclass.h | 221 +- bsd/net/pktsched/pktsched_rmclass_debug.h | 140 - bsd/net/pktsched/pktsched_tcq.c | 512 +- bsd/net/pktsched/pktsched_tcq.h | 25 +- bsd/net/radix.h | 8 + bsd/net/raw_cb.h | 2 +- bsd/net/raw_usrreq.c | 11 +- bsd/net/route.c | 388 +- bsd/net/route.h | 107 +- bsd/net/rtsock.c | 113 +- bsd/net/skmem_sysctl.c | 30 + bsd/netinet/Makefile | 3 +- .../{cpu_in_cksum.c => cpu_in_cksum_gen.c} | 176 +- bsd/netinet/flow_divert.c | 41 +- bsd/netinet/flow_divert.h | 3 +- bsd/netinet/flow_divert_proto.h | 4 +- bsd/netinet/icmp6.h | 2 +- bsd/netinet/igmp.c | 6 +- bsd/netinet/igmp_var.h | 4 +- bsd/netinet/in.c | 390 +- bsd/netinet/in.h | 35 +- bsd/netinet/in_arp.c | 158 +- bsd/netinet/in_arp.h | 1 + bsd/netinet/in_cksum.c | 644 +- bsd/netinet/in_mcast.c | 198 +- bsd/netinet/in_pcb.c | 158 +- bsd/netinet/in_pcb.h | 22 +- bsd/netinet/in_pcblist.c | 28 +- bsd/netinet/in_rmx.c | 19 +- bsd/netinet/in_stat.c | 99 + bsd/netinet/in_stat.h | 49 + bsd/netinet/in_tclass.c | 89 +- bsd/netinet/in_tclass.h | 25 +- bsd/netinet/in_var.h | 12 +- bsd/netinet/ip_compat.h | 6 +- bsd/netinet/ip_divert.c | 10 +- bsd/netinet/ip_dummynet.c | 255 +- bsd/netinet/ip_dummynet.h | 66 +- bsd/netinet/ip_fw2.c | 1 - bsd/netinet/ip_icmp.c | 8 +- bsd/netinet/ip_id.c | 8 +- bsd/netinet/ip_input.c | 192 +- bsd/netinet/ip_output.c | 41 +- bsd/netinet/ip_var.h | 14 +- bsd/netinet/kpi_ipfilter.c | 187 +- bsd/netinet/kpi_ipfilter.h | 18 +- bsd/netinet/mp_pcb.c | 134 +- bsd/netinet/mp_pcb.h | 49 +- bsd/netinet/mp_proto.c | 25 +- bsd/netinet/mptcp.c | 1374 +- bsd/netinet/mptcp.h | 6 +- bsd/netinet/mptcp_opt.c | 934 +- bsd/netinet/mptcp_opt.h | 20 +- bsd/netinet/mptcp_seq.h | 2 +- bsd/netinet/mptcp_subr.c | 5903 +- bsd/netinet/mptcp_timer.c | 27 +- bsd/netinet/mptcp_timer.h | 2 +- bsd/netinet/mptcp_usrreq.c | 1229 +- bsd/netinet/mptcp_var.h | 518 +- bsd/netinet/raw_ip.c | 38 +- bsd/netinet/tcp.h | 66 +- bsd/netinet/tcp_cache.c | 711 +- bsd/netinet/tcp_cache.h | 15 +- bsd/netinet/tcp_cc.c | 24 +- bsd/netinet/tcp_cc.h | 2 + bsd/netinet/tcp_cubic.c | 28 +- bsd/netinet/tcp_fsm.h | 4 + bsd/netinet/tcp_input.c | 510 +- bsd/netinet/tcp_ledbat.c | 21 +- bsd/netinet/tcp_lro.c | 1 + bsd/netinet/tcp_output.c | 269 +- bsd/netinet/tcp_sack.c | 15 +- bsd/netinet/tcp_seq.h | 6 +- bsd/netinet/tcp_subr.c | 827 +- bsd/netinet/tcp_timer.c | 371 +- bsd/netinet/tcp_timer.h | 3 + bsd/netinet/tcp_usrreq.c | 288 +- bsd/netinet/tcp_var.h | 150 +- bsd/netinet/udp_usrreq.c | 187 +- bsd/netinet/udp_var.h | 2 +- bsd/netinet6/Makefile | 6 +- bsd/netinet6/ah_input.c | 2 +- bsd/netinet6/esp.h | 1 + bsd/netinet6/esp6.h | 3 +- bsd/netinet6/esp_chachapoly.c | 481 + bsd/netinet6/esp_chachapoly.h | 53 + bsd/netinet6/esp_core.c | 32 +- bsd/netinet6/esp_input.c | 92 +- bsd/netinet6/esp_output.c | 4 +- bsd/netinet6/esp_rijndael.c | 66 +- bsd/netinet6/frag6.c | 47 +- bsd/netinet6/icmp6.c | 17 +- bsd/netinet6/in6.c | 591 +- bsd/netinet6/in6.h | 63 +- bsd/netinet6/in6_cga.c | 5 +- bsd/netinet6/in6_cksum.c | 44 + bsd/netinet6/in6_ifattach.c | 14 +- bsd/netinet6/in6_mcast.c | 224 +- bsd/netinet6/in6_pcb.c | 70 +- bsd/netinet6/in6_pcb.h | 3 +- bsd/netinet6/in6_proto.c | 9 +- bsd/netinet6/in6_rmx.c | 21 +- bsd/netinet6/in6_src.c | 475 +- bsd/netinet6/in6_var.h | 25 +- bsd/netinet6/ip6_fw.c | 22 - bsd/netinet6/ip6_input.c | 130 +- bsd/netinet6/ip6_output.c | 84 +- bsd/netinet6/ip6_var.h | 24 +- bsd/netinet6/ip6protosw.h | 8 +- bsd/netinet6/ipsec.c | 58 +- bsd/netinet6/ipsec.h | 5 + bsd/netinet6/mld6.c | 108 +- bsd/netinet6/mld6_var.h | 7 +- bsd/netinet6/nd6.c | 191 +- bsd/netinet6/nd6.h | 32 +- bsd/netinet6/nd6_nbr.c | 122 +- bsd/netinet6/nd6_prproxy.c | 12 +- bsd/netinet6/nd6_rtr.c | 58 +- bsd/netinet6/nd6_send.c | 8 +- bsd/netinet6/raw_ip6.c | 15 +- bsd/netinet6/tcp6_var.h | 2 +- bsd/netinet6/udp6_output.c | 14 +- bsd/netinet6/udp6_usrreq.c | 70 +- bsd/netinet6/udp6_var.h | 2 +- bsd/netkey/key.c | 182 +- bsd/netkey/key.h | 3 + bsd/netkey/keydb.c | 12 +- bsd/nfs/gss/gss_krb5_mech.c | 8 +- bsd/nfs/nfs.h | 20 +- bsd/nfs/nfs4_subs.c | 862 +- bsd/nfs/nfs4_vnops.c | 4 +- bsd/nfs/nfs_bio.c | 4 +- bsd/nfs/nfs_gss.c | 31 +- bsd/nfs/nfs_ioctl.h | 18 +- bsd/nfs/nfs_lock.c | 2 +- bsd/nfs/nfs_socket.c | 10 +- bsd/nfs/nfs_subs.c | 2 +- bsd/nfs/nfs_syscalls.c | 29 +- bsd/nfs/nfs_vfsops.c | 220 +- bsd/nfs/nfs_vnops.c | 53 +- bsd/nfs/nfsmount.h | 2 + bsd/pgo/profile_runtime.c | 24 +- bsd/security/audit/audit.h | 5 - bsd/security/audit/audit_bsm.c | 15 + bsd/sys/Makefile | 18 +- bsd/sys/_types/Makefile | 8 + bsd/sys/_types/_blkcnt_t.h | 1 + bsd/sys/_types/_blksize_t.h | 1 + bsd/sys/_types/_caddr_t.h | 31 + bsd/sys/_types/_clock_t.h | 1 + bsd/sys/_types/_ct_rune_t.h | 1 + bsd/sys/_types/_dev_t.h | 1 + bsd/sys/_types/_fd_def.h | 3 + bsd/sys/_types/_fsblkcnt_t.h | 1 + bsd/sys/_types/_fsfilcnt_t.h | 1 + bsd/sys/_types/_fsid_t.h | 1 + bsd/sys/_types/_fsobj_id_t.h | 2 + bsd/sys/_types/_gid_t.h | 1 + bsd/sys/_types/_id_t.h | 1 + bsd/sys/_types/_in_addr_t.h | 1 + bsd/sys/_types/_in_port_t.h | 1 + bsd/sys/_types/_ino64_t.h | 1 + bsd/sys/_types/_ino_t.h | 1 + bsd/sys/_types/_intptr_t.h | 2 + bsd/sys/_types/_iovec_t.h | 1 + bsd/sys/_types/_key_t.h | 1 + bsd/sys/_types/_mach_port_t.h | 1 + bsd/sys/_types/_mbstate_t.h | 1 + bsd/sys/_types/_mode_t.h | 1 + bsd/sys/_types/_nlink_t.h | 1 + bsd/sys/_types/_null.h | 3 +- bsd/sys/_types/_off_t.h | 1 + bsd/sys/_types/_pid_t.h | 1 + bsd/sys/_types/_ptrdiff_t.h | 1 + bsd/sys/_types/_rsize_t.h | 1 + bsd/sys/_types/_rune_t.h | 1 + bsd/sys/_types/_sa_family_t.h | 1 + bsd/sys/_types/_seek_set.h | 4 + bsd/sys/_types/_sigaltstack.h | 6 + bsd/sys/_types/_sigset_t.h | 1 + bsd/sys/_types/_size_t.h | 1 + bsd/sys/_types/_socklen_t.h | 1 + bsd/sys/_types/_ssize_t.h | 1 + bsd/sys/_types/_suseconds_t.h | 1 + bsd/sys/_types/_time_t.h | 1 + bsd/sys/_types/_timespec.h | 3 + bsd/sys/_types/_timeval.h | 4 + bsd/sys/_types/_timeval32.h | 3 + bsd/sys/_types/_timeval64.h | 3 + bsd/sys/_types/_u_char.h | 31 + bsd/sys/_types/_u_int.h | 31 + bsd/sys/_types/_u_short.h | 31 + bsd/sys/_types/_ucontext.h | 8 + bsd/sys/_types/_ucontext64.h | 8 + bsd/sys/_types/_uid_t.h | 1 + bsd/sys/_types/_useconds_t.h | 1 + bsd/sys/_types/_user32_ntptimeval.h | 41 + bsd/sys/_types/_user32_timex.h | 54 + bsd/sys/_types/_user64_ntptimeval.h | 41 + bsd/sys/_types/_user64_timex.h | 54 + bsd/sys/_types/_uuid_t.h | 1 + bsd/sys/_types/_va_list.h | 1 + bsd/sys/_types/_wchar_t.h | 1 + bsd/sys/_types/_wint_t.h | 1 + bsd/sys/acct.h | 4 + bsd/sys/attr.h | 9 +- bsd/sys/bitstring.h | 46 +- bsd/sys/buf_internal.h | 1 + bsd/sys/cdefs.h | 8 + bsd/sys/clonefile.h | 5 +- bsd/sys/coalition.h | 2 + bsd/sys/codesign.h | 150 +- bsd/sys/commpage.h | 41 + bsd/sys/conf.h | 6 +- bsd/sys/csr.h | 4 +- bsd/sys/decmpfs.h | 2 +- bsd/sys/disk.h | 11 + bsd/sys/domain.h | 3 + bsd/sys/dtrace.h | 21 + bsd/sys/dtrace_glue.h | 52 +- bsd/sys/dtrace_impl.h | 5 +- bsd/sys/dtrace_ptss.h | 3 + bsd/sys/event.h | 348 +- bsd/sys/eventhandler.h | 221 + bsd/sys/eventvar.h | 198 +- bsd/sys/fasttrap.h | 7 + bsd/sys/fasttrap_impl.h | 13 + bsd/sys/fbt.h | 11 + bsd/sys/fcntl.h | 2 +- bsd/sys/file_internal.h | 7 +- bsd/sys/filedesc.h | 12 +- bsd/sys/fsctl.h | 24 +- bsd/sys/fsevents.h | 4 +- bsd/sys/fsgetpath.h | 40 +- bsd/sys/guarded.h | 68 +- bsd/sys/imgact.h | 4 + bsd/sys/kauth.h | 2 + bsd/sys/kdebug.h | 267 +- bsd/sys/kern_control.h | 3 + bsd/sys/kern_memorystatus.h | 73 +- bsd/sys/kpi_mbuf.h | 8 +- bsd/sys/kpi_socket.h | 36 +- bsd/sys/kpi_socketfilter.h | 10 +- bsd/sys/ktrace.h | 8 +- bsd/sys/lctx.h | 2 + bsd/sys/linker_set.h | 10 +- bsd/sys/malloc.h | 30 +- bsd/sys/mbuf.h | 47 +- bsd/sys/mcache.h | 15 +- bsd/sys/mman.h | 3 + bsd/sys/monotonic.h | 149 + bsd/sys/mount.h | 23 +- bsd/sys/mount_internal.h | 2 + bsd/sys/munge.h | 53 + bsd/sys/netport.h | 2 + bsd/sys/persona.h | 2 +- bsd/sys/pgo.h | 5 + bsd/sys/pipe.h | 4 +- bsd/sys/priv.h | 19 +- bsd/sys/proc.h | 11 +- bsd/sys/proc_info.h | 58 +- bsd/sys/proc_internal.h | 16 +- bsd/sys/process_policy.h | 18 + bsd/sys/protosw.h | 107 +- bsd/sys/pthread_internal.h | 1 + bsd/sys/pthread_shims.h | 104 +- bsd/sys/quota.h | 1 + bsd/sys/reason.h | 26 +- bsd/sys/resource.h | 52 +- bsd/sys/resourcevar.h | 1 + bsd/sys/sdt_impl.h | 4 + bsd/sys/sem.h | 1 + bsd/sys/signal.h | 4 +- bsd/sys/snapshot.h | 2 + bsd/sys/socket.h | 13 +- bsd/sys/socketvar.h | 84 +- bsd/sys/sockio.h | 26 +- bsd/sys/spawn.h | 3 + bsd/sys/stat.h | 22 +- bsd/sys/subr_prf.h | 2 +- bsd/sys/sysctl.h | 28 + bsd/sys/sysent.h | 2 + bsd/sys/syslog.h | 6 +- bsd/sys/systm.h | 5 +- bsd/sys/systrace_args.h | 36 + bsd/sys/time.h | 4 + bsd/sys/timex.h | 212 + bsd/sys/tprintf.h | 2 +- bsd/sys/tty.h | 12 +- bsd/sys/types.h | 9 +- bsd/sys/ubc.h | 7 + bsd/sys/ubc_internal.h | 5 + bsd/sys/unistd.h | 6 +- bsd/sys/unpcb.h | 3 + bsd/sys/user.h | 12 +- bsd/sys/vm.h | 4 + bsd/sys/vnode.h | 76 +- bsd/sys/vnode_if.h | 6 +- bsd/sys/vnode_internal.h | 4 +- bsd/sys/work_interval.h | 145 +- bsd/vfs/Makefile | 4 +- bsd/vfs/kpi_vfs.c | 92 +- bsd/vfs/vfs_attrlist.c | 103 +- bsd/vfs/vfs_bio.c | 35 +- bsd/vfs/vfs_cache.c | 56 +- bsd/vfs/vfs_cluster.c | 83 +- bsd/vfs/vfs_disk_conditioner.c | 235 + bsd/vfs/vfs_disk_conditioner.h | 42 + bsd/vfs/vfs_fsevents.c | 79 +- bsd/vfs/vfs_lookup.c | 32 +- bsd/vfs/vfs_subr.c | 201 +- bsd/vfs/vfs_syscalls.c | 279 +- bsd/vfs/vfs_vnops.c | 30 +- bsd/vfs/vfs_xattr.c | 24 +- bsd/vm/vm_compressor_backing_file.c | 18 +- bsd/vm/vm_unix.c | 314 +- bsd/vm/vnode_pager.c | 14 +- config/BSDKernel.arm.exports | 16 + config/BSDKernel.arm64.exports | 16 + config/BSDKernel.exports | 3 +- config/IOKit.arm.exports | 309 + config/IOKit.arm64.exports | 230 + config/IOKit.exports | 10 +- config/IOKit.x86_64.exports | 7 +- config/Libkern.arm.exports | 4 + config/Libkern.arm64.exports | 5 + config/Libkern.exports | 10 + config/MACFramework.arm.exports | 0 config/MACFramework.arm64.exports | 0 config/MASTER | 50 +- config/MASTER.arm | 86 + config/MASTER.arm64 | 92 + config/MASTER.x86_64 | 26 +- config/Mach.arm.exports | 2 + config/Mach.arm64.exports | 1 + config/Mach.exports | 5 + config/Makefile | 4 + config/MasterVersion | 2 +- config/Private.arm.exports | 21 + config/Private.arm64.exports | 34 + config/Private.exports | 51 +- config/Private.x86_64.exports | 12 + .../System.kext/PlugIns/Kasan.kext/Info.plist | 34 + config/Unsupported.arm.exports | 24 + config/Unsupported.arm64.exports | 40 + config/Unsupported.exports | 13 +- config/Unsupported.x86_64.exports | 2 + iokit/IOKit/IOCPU.h | 14 +- iokit/IOKit/IODeviceTreeSupport.h | 5 + iokit/IOKit/IOEventSource.h | 19 +- iokit/IOKit/IOHibernatePrivate.h | 18 +- iokit/IOKit/IOInterruptAccounting.h | 2 + iokit/IOKit/IOInterruptController.h | 10 + iokit/IOKit/IOKernelReportStructs.h | 212 +- iokit/IOKit/IOKernelReporters.h | 18 +- iokit/IOKit/IOKitDebug.h | 35 +- iokit/IOKit/IOKitKeys.h | 1 + iokit/IOKit/IOLib.h | 11 +- iokit/IOKit/IOMapper.h | 4 +- iokit/IOKit/IOMemoryDescriptor.h | 54 +- iokit/IOKit/IOPlatformExpert.h | 5 +- iokit/IOKit/IOPolledInterface.h | 5 +- iokit/IOKit/IOReportTypes.h | 219 +- iokit/IOKit/IOReturn.h | 11 +- iokit/IOKit/IOService.h | 20 +- iokit/IOKit/IOTimeStamp.h | 1 + iokit/IOKit/IOTimerEventSource.h | 98 +- iokit/IOKit/IOTypes.h | 4 +- iokit/IOKit/Makefile | 19 +- iokit/IOKit/pwr_mgt/IOPM.h | 20 + iokit/IOKit/pwr_mgt/IOPMPrivate.h | 24 +- iokit/IOKit/pwr_mgt/IOPMlog.h | 2 + iokit/IOKit/pwr_mgt/RootDomain.h | 14 +- iokit/Kernel/IOBufferMemoryDescriptor.cpp | 21 +- iokit/Kernel/IOCPU.cpp | 241 +- iokit/Kernel/IOCommandGate.cpp | 8 +- iokit/Kernel/IOCommandQueue.cpp | 4 +- iokit/Kernel/IODMACommand.cpp | 136 +- iokit/Kernel/IODeviceTreeSupport.cpp | 340 +- iokit/Kernel/IOFilterInterruptEventSource.cpp | 13 +- iokit/Kernel/IOHibernateIO.cpp | 77 +- iokit/Kernel/IOHibernateRestoreKernel.c | 18 +- iokit/Kernel/IOHistogramReporter.cpp | 53 +- iokit/Kernel/IOInterruptAccounting.cpp | 2 + iokit/Kernel/IOInterruptController.cpp | 85 +- iokit/Kernel/IOInterruptEventSource.cpp | 20 +- iokit/Kernel/IOKitDebug.cpp | 99 +- iokit/Kernel/IOKitKernelInternal.h | 11 +- iokit/Kernel/IOLib.cpp | 42 +- iokit/Kernel/IOMapper.cpp | 38 +- iokit/Kernel/IOMemoryDescriptor.cpp | 550 +- iokit/Kernel/IONVRAM.cpp | 51 +- iokit/Kernel/IOPMrootDomain.cpp | 527 +- iokit/Kernel/IOPlatformExpert.cpp | 119 +- iokit/Kernel/IOPolledInterface.cpp | 110 +- iokit/Kernel/IOReporter.cpp | 8 +- iokit/Kernel/IOService.cpp | 371 +- iokit/Kernel/IOServicePM.cpp | 162 +- iokit/Kernel/IOServicePMPrivate.h | 8 +- iokit/Kernel/IOServicePrivate.h | 2 + iokit/Kernel/IOSimpleReporter.cpp | 4 +- iokit/Kernel/IOStartIOKit.cpp | 9 +- iokit/Kernel/IOStateReporter.cpp | 4 +- iokit/Kernel/IOStatistics.cpp | 7 +- iokit/Kernel/IOStringFuncs.c | 14 - iokit/Kernel/IOTimerEventSource.cpp | 221 +- iokit/Kernel/IOUserClient.cpp | 143 +- iokit/Kernel/IOWorkLoop.cpp | 17 +- iokit/Kernel/RootDomainUserClient.cpp | 9 +- iokit/Kernel/RootDomainUserClient.h | 2 +- iokit/Kernel/i386/IOKeyStoreHelper.cpp | 56 + iokit/Tests/TestIOMemoryDescriptor.cpp | 238 +- iokit/Tests/Tests.cpp | 50 +- iokit/bsddev/DINetBootHook.cpp | 65 +- iokit/bsddev/DINetBootHook.h | 3 +- iokit/bsddev/IOKitBSDInit.cpp | 76 +- iokit/conf/Makefile.arm | 18 + iokit/conf/Makefile.arm64 | 18 + iokit/conf/Makefile.template | 1 + iokit/conf/files.arm | 4 + iokit/conf/files.arm64 | 4 + libkdd/kcdata.h | 1140 +- libkdd/kcdtypes.c | 63 + libkdd/kdd.xcodeproj/project.pbxproj | 42 +- libkdd/tests/Tests.swift | 643 +- libkdd/tests/stackshot-sample-coalitions | Bin 0 -> 14448 bytes .../stackshot-sample-coalitions.plist.gz | Bin 0 -> 6794 bytes libkdd/tests/stackshot-sample-instrs-cycles | Bin 0 -> 625 bytes .../stackshot-sample-instrs-cycles.plist.gz | Bin 0 -> 1630 bytes libkdd/tests/stackshot-sample-thread-groups | Bin 0 -> 6784 bytes .../stackshot-sample-thread-groups.plist.gz | Bin 0 -> 4312 bytes libkdd/tests/stackshot-sample-thread-policy | Bin 0 -> 1274 bytes .../stackshot-sample-thread-policy.plist.gz | Bin 0 -> 2631 bytes libkern/OSKextLib.cpp | 10 +- libkern/c++/OSData.cpp | 4 +- libkern/c++/OSKext.cpp | 209 +- libkern/c++/OSMetaClass.cpp | 83 +- libkern/c++/OSUnserializeXML.cpp | 10 +- .../TestSerialization/test1/test1_main.cpp | 0 libkern/conf/Makefile.arm | 20 + libkern/conf/Makefile.arm64 | 20 + libkern/conf/Makefile.template | 1 + libkern/conf/files | 2 +- libkern/conf/files.arm | 3 + libkern/crypto/corecrypto_chacha20poly1305.c | 86 + libkern/firehose/chunk_private.h | 3 +- libkern/firehose/firehose_types_private.h | 42 +- libkern/gen/OSDebug.cpp | 43 +- libkern/kmod/libkmodtest/libkmodtest.h | 2 +- libkern/kxld/kxld_object.c | 32 +- libkern/kxld/kxld_util.c | 3 +- libkern/kxld/kxld_util.h | 2 +- libkern/kxld/kxld_versionmin.c | 31 +- libkern/kxld/kxld_versionmin.h | 5 +- libkern/kxld/tests/loadtest.py | 0 libkern/libkern/OSAtomic.h | 10 +- libkern/libkern/OSByteOrder.h | 2 + libkern/libkern/OSKextLib.h | 12 +- libkern/libkern/OSKextLibPrivate.h | 2 +- libkern/libkern/OSMalloc.h | 6 +- libkern/libkern/_OSByteOrder.h | 3 + libkern/libkern/arm/Makefile | 21 + libkern/libkern/arm/OSByteOrder.h | 147 + libkern/libkern/c++/OSData.h | 2 +- libkern/libkern/c++/OSKext.h | 14 +- libkern/libkern/c++/OSMetaClass.h | 92 +- libkern/libkern/crypto/Makefile | 2 +- libkern/libkern/crypto/chacha20poly1305.h | 55 + libkern/libkern/crypto/register_crypto.h | 18 +- libkern/net/inet_aton.c | 2 +- libkern/os/Makefile | 6 +- libkern/os/log.c | 19 +- libkern/os/log.h | 14 +- libkern/os/overflow.h | 11 + libkern/os/reason_private.h | 59 + libkern/zlib/gzio.c | 1031 - libsa/bootstrap.cpp | 29 + libsa/conf/Makefile.arm | 10 + libsa/conf/Makefile.arm64 | 10 + libsa/conf/Makefile.template | 1 + libsa/conf/files.arm | 0 libsa/conf/files.arm64 | 0 libsa/lastkerneldataconst.c | 5 + libsyscall/Libsyscall.xcconfig | 6 +- .../Libsyscall.xcodeproj/project.pbxproj | 83 +- libsyscall/Platforms/iPhoneOS/arm/syscall.map | 77 + .../Platforms/iPhoneOS/arm64/syscall.map | 67 + libsyscall/custom/SYS.h | 311 + libsyscall/custom/__fork.s | 53 + libsyscall/custom/__getpid.s | 71 + libsyscall/custom/__gettimeofday.s | 24 + libsyscall/custom/__kdebug_trace_string.s | 8 + libsyscall/custom/__lseek.s | 8 + libsyscall/custom/__pipe.s | 19 + libsyscall/custom/__ptrace.s | 19 + libsyscall/custom/__sigaltstack.s | 8 + libsyscall/custom/__sigreturn.s | 8 + libsyscall/custom/__syscall.s | 18 + libsyscall/custom/__thread_selfid.s | 8 + libsyscall/custom/__thread_selfusage.s | 8 + libsyscall/custom/__vfork.s | 101 + libsyscall/custom/custom.s | 20 + libsyscall/mach/err_libkern.sub | 1 + libsyscall/mach/host.c | 17 + libsyscall/mach/mach_init.c | 12 + libsyscall/mach/string.h | 4 +- libsyscall/os/tsd.h | 46 +- libsyscall/wrappers/__commpage_gettimeofday.c | 119 +- libsyscall/wrappers/__commpage_gettimeofday.s | 131 - libsyscall/wrappers/__get_cpu_capabilities.s | 25 + libsyscall/wrappers/_libkernel_init.c | 10 + libsyscall/wrappers/coalition.c | 11 + libsyscall/wrappers/init_cpu_capabilities.c | 12 + libsyscall/wrappers/libproc/libproc.c | 223 +- libsyscall/wrappers/libproc/libproc.h | 4 + .../wrappers/libproc/libproc_internal.h | 47 + libsyscall/wrappers/mach_absolute_time.s | 124 + libsyscall/wrappers/mach_approximate_time.s | 40 +- libsyscall/wrappers/mach_continuous_time.c | 41 +- libsyscall/wrappers/mach_get_times.c | 4 + libsyscall/wrappers/pid_shutdown_networking.c | 33 + libsyscall/wrappers/quota_obsolete.c | 2 + libsyscall/wrappers/remove-counter.c | 11 + libsyscall/wrappers/spawn/posix_spawn.c | 35 + libsyscall/wrappers/spawn/spawn_private.h | 4 + libsyscall/wrappers/thread_register_state.c | 38 +- libsyscall/wrappers/utimensat.c | 134 + libsyscall/wrappers/varargs_wrappers.s | 124 + libsyscall/wrappers/work_interval.c | 177 +- libsyscall/xcodescripts/create-syscalls.pl | 4 + libsyscall/xcodescripts/mach_install_mig.sh | 8 +- makedefs/MakeInc.cmd | 4 +- makedefs/MakeInc.def | 147 +- makedefs/MakeInc.top | 16 +- osfmk/arm/Makefile | 49 + osfmk/arm/WKdmCompress_new.s | 710 + osfmk/arm/WKdmData_new.s | 289 + osfmk/arm/WKdmDecompress_new.s | 427 + osfmk/arm/arch.h | 67 + osfmk/arm/arm_init.c | 531 + osfmk/arm/arm_timer.c | 279 + osfmk/arm/arm_vm_init.c | 537 + osfmk/arm/asm.h | 320 + osfmk/arm/atomic.h | 261 + osfmk/arm/bcopy.s | 402 + osfmk/arm/bsd_arm.c | 71 + osfmk/arm/bzero.s | 173 + osfmk/arm/caches.c | 753 + osfmk/arm/caches_asm.s | 362 + osfmk/arm/caches_internal.h | 101 + osfmk/arm/commpage/commpage.c | 432 + osfmk/arm/commpage/commpage.h | 49 + osfmk/arm/commpage/commpage_sigs.h | 111 + osfmk/arm/conf.c | 83 + osfmk/arm/cpu.c | 604 + osfmk/arm/cpu_affinity.h | 45 + osfmk/arm/cpu_capabilities.h | 212 + osfmk/arm/cpu_common.c | 577 + osfmk/arm/cpu_data.h | 92 + osfmk/arm/cpu_data_internal.h | 319 + osfmk/arm/cpu_internal.h | 79 + osfmk/arm/cpu_number.h | 80 + osfmk/arm/cpuid.c | 314 + osfmk/arm/cpuid.h | 224 + osfmk/arm/cpuid_internal.h | 55 + osfmk/arm/cswitch.s | 290 + osfmk/arm/data.s | 117 + osfmk/arm/dbgwrap.c | 53 + osfmk/arm/dbgwrap.h | 63 + osfmk/arm/exception.h | 77 + osfmk/arm/genassym.c | 368 + osfmk/arm/globals_asm.h | 40 + osfmk/arm/hw_lock_types.h | 73 + osfmk/arm/io_map.c | 112 + osfmk/arm/io_map_entries.h | 49 + osfmk/arm/kpc_arm.c | 986 + osfmk/arm/lock.h | 71 + osfmk/arm/locks.h | 332 + osfmk/arm/locks_arm.c | 2882 + osfmk/arm/locore.s | 2041 + osfmk/arm/loose_ends.c | 665 + osfmk/arm/lowglobals.h | 78 + osfmk/arm/lowmem_vectors.c | 91 + osfmk/arm/lz4_decode_armv7NEON.s | 348 + osfmk/arm/lz4_encode_armv7.s | 429 + osfmk/arm/machdep_call.c | 92 + osfmk/arm/machdep_call.h | 65 + osfmk/arm/machine_cpu.h | 60 + osfmk/arm/machine_cpuid.c | 163 + osfmk/arm/machine_cpuid.h | 107 + osfmk/arm/machine_kpc.h | 60 + osfmk/arm/machine_routines.c | 1176 + osfmk/arm/machine_routines.h | 884 + osfmk/arm/machine_routines_asm.s | 1131 + osfmk/arm/machine_routines_common.c | 614 + osfmk/arm/machine_task.c | 179 + osfmk/arm/machlimits.h | 98 + osfmk/arm/machparam.h | 64 + osfmk/arm/misc_protos.h | 98 + osfmk/arm/model_dep.c | 868 + osfmk/arm/monotonic.h | 37 + osfmk/arm/monotonic_arm.c | 51 + osfmk/arm/pal_routines.c | 62 + osfmk/arm/pal_routines.h | 69 + osfmk/arm/pcb.c | 406 + osfmk/arm/pmap.c | 10555 +++ osfmk/arm/pmap.h | 516 + osfmk/arm/proc_reg.h | 1084 + osfmk/arm/rtclock.c | 495 + osfmk/arm/rtclock.h | 97 + osfmk/arm/sched_param.h | 67 + osfmk/arm/setjmp.h | 69 + osfmk/arm/simple_lock.h | 194 + osfmk/arm/smp.h | 37 + osfmk/arm/start.s | 434 + osfmk/arm/status.c | 873 + osfmk/arm/status_shared.c | 81 + osfmk/arm/strlcpy.c | 42 + osfmk/arm/strlen.s | 119 + osfmk/arm/strncmp.s | 159 + osfmk/arm/strncpy.c | 42 + osfmk/arm/strnlen.s | 154 + osfmk/arm/task.h | 65 + osfmk/arm/thread.h | 219 + osfmk/arm/trap.c | 897 + osfmk/arm/trap.h | 284 + osfmk/arm/vm_tuning.h | 67 + osfmk/arm/xpr.h | 36 + osfmk/arm64/Makefile | 31 + osfmk/arm64/WKdmCompress_16k.s | 634 + osfmk/arm64/WKdmCompress_4k.s | 632 + osfmk/arm64/WKdmData.s | 330 + osfmk/arm64/WKdmDecompress_16k.s | 428 + osfmk/arm64/WKdmDecompress_4k.s | 428 + osfmk/arm64/alternate_debugger.c | 175 + osfmk/arm64/alternate_debugger.h | 45 + osfmk/arm64/alternate_debugger_asm.s | 65 + osfmk/arm64/arm_vm_init.c | 1203 + osfmk/arm64/asm.h | 189 + osfmk/arm64/bcopy.s | 296 + osfmk/arm64/bsd_arm64.c | 227 + osfmk/arm64/bzero.s | 153 + osfmk/arm64/caches_asm.s | 369 + osfmk/arm64/copyio.c | 311 + osfmk/arm64/cpu.c | 864 + osfmk/arm64/cswitch.s | 239 + osfmk/arm64/dbgwrap.c | 283 + osfmk/arm64/genassym.c | 430 + osfmk/arm64/kpc.c | 1135 + osfmk/arm64/locore.s | 868 + osfmk/arm64/loose_ends.c | 699 + osfmk/arm64/lowglobals.h | 87 + osfmk/arm64/lowmem_vectors.c | 98 + osfmk/arm64/lz4_decode_arm64.s | 333 + osfmk/arm64/lz4_encode_arm64.s | 406 + osfmk/arm64/machine_cpuid.h | 70 + osfmk/arm64/machine_kpc.h | 58 + osfmk/arm64/machine_machdep.h | 42 + osfmk/arm64/machine_routines.c | 2048 + osfmk/arm64/machine_routines_asm.s | 970 + osfmk/arm64/machine_task.c | 251 + osfmk/arm64/monotonic.h | 58 + osfmk/arm64/monotonic_arm64.c | 391 + osfmk/arm64/pcb.c | 878 + osfmk/arm64/pgtrace.c | 594 + osfmk/arm64/pgtrace.h | 163 + osfmk/arm64/pgtrace_decoder.c | 1551 + osfmk/arm64/pgtrace_decoder.h | 40 + osfmk/arm64/pinst.s | 127 + osfmk/arm64/platform_tests.c | 1087 + osfmk/arm64/proc_reg.h | 1401 + osfmk/arm64/sleh.c | 1456 + osfmk/arm64/start.s | 898 + osfmk/arm64/status.c | 1455 + osfmk/arm64/strncmp.s | 187 + osfmk/arm64/strnlen.s | 198 + osfmk/atm/atm.c | 6 +- osfmk/bank/bank.c | 265 +- osfmk/bank/bank_internal.h | 13 +- osfmk/chud/chud_thread.c | 2 + osfmk/chud/chud_xnu_glue.h | 2 + osfmk/chud/chud_xnu_private.h | 2 + osfmk/chud/i386/chud_osfmk_callback_i386.c | 2 + osfmk/conf/Makefile.arm | 10 + osfmk/conf/Makefile.arm64 | 13 + osfmk/conf/Makefile.template | 5 + osfmk/conf/files | 25 +- osfmk/conf/files.arm | 80 + osfmk/conf/files.arm64 | 92 + osfmk/conf/files.x86_64 | 2 + osfmk/console/Makefile | 3 +- osfmk/console/art/scalegear.c | 27 + osfmk/console/serial_console.c | 87 +- osfmk/console/serial_general.c | 57 +- osfmk/console/serial_protos.h | 42 + osfmk/console/video_console.c | 69 +- osfmk/console/video_console.h | 2 + osfmk/console/video_scroll.c | 2 - osfmk/corecrypto/cc/src/cc_abort.c | 36 - osfmk/corecrypto/cc/src/cc_clear.c | 27 + osfmk/corecrypto/cc/src/cc_cmp_safe.c | 24 + osfmk/corecrypto/cc/src/cc_try_abort.c | 60 + osfmk/corecrypto/ccaes/src/aes_tab.c | 1061 + .../ccaes/src/ccaes_ltc_ecb_encrypt_mode.c | 421 + .../ccaes/src/ccaes_private_types.h | 48 + osfmk/corecrypto/ccdbrg/src/ccdrbg_nisthmac.c | 112 +- osfmk/corecrypto/ccdigest/src/ccdigest_init.c | 24 + .../corecrypto/ccdigest/src/ccdigest_update.c | 24 + osfmk/corecrypto/cchmac/src/cchmac.c | 28 + osfmk/corecrypto/cchmac/src/cchmac_final.c | 24 + osfmk/corecrypto/cchmac/src/cchmac_init.c | 24 + osfmk/corecrypto/cchmac/src/cchmac_update.c | 24 + .../corecrypto/ccmode/src/ccmode_ctr_crypt.c | 72 + osfmk/corecrypto/ccmode/src/ccmode_ctr_init.c | 49 + .../corecrypto/ccmode/src/ccmode_ctr_setctr.c | 43 + .../ccmode/src/ccmode_factory_ctr_crypt.c | 41 + osfmk/corecrypto/ccmode/src/ccmode_internal.h | 297 + osfmk/corecrypto/ccn/src/ccn_set.c | 24 + .../ccsha1/src/ccdigest_final_64be.c | 24 + osfmk/corecrypto/ccsha1/src/ccsha1_eay.c | 24 + .../ccsha1/src/ccsha1_initial_state.c | 24 + osfmk/corecrypto/ccsha2/src/ccsha256_K.c | 53 + osfmk/corecrypto/ccsha2/src/ccsha256_di.c | 59 + .../ccsha2/src/ccsha256_initial_state.c | 46 + .../ccsha2/src/ccsha256_ltc_compress.c | 152 + osfmk/corecrypto/ccsha2/src/ccsha256_ltc_di.c | 48 + osfmk/corecrypto/ccsha2/src/ccsha2_internal.h | 63 + osfmk/corpses/corpse.c | 249 +- osfmk/corpses/task_corpse.h | 32 +- osfmk/device/device_port.h | 1 + osfmk/device/iokit_rpc.c | 22 +- osfmk/device/subrs.c | 154 +- osfmk/i386/AT386/conf.c | 5 +- osfmk/i386/AT386/model_dep.c | 385 +- osfmk/i386/Diagnostics.c | 27 +- osfmk/i386/Makefile | 1 + osfmk/i386/acpi.c | 54 +- osfmk/i386/bsd_i386.c | 25 +- osfmk/i386/bsd_i386_native.c | 1 - osfmk/i386/commpage/commpage.c | 116 +- osfmk/i386/commpage/commpage.h | 5 +- osfmk/i386/cpu.c | 4 +- osfmk/i386/cpu_capabilities.h | 11 + osfmk/i386/cpu_data.h | 30 +- osfmk/i386/cpu_threads.c | 32 +- osfmk/i386/cpu_threads.h | 2 + osfmk/i386/cpuid.c | 34 +- osfmk/i386/cpuid.h | 18 + osfmk/i386/fpu.c | 882 +- osfmk/i386/fpu.h | 27 +- osfmk/i386/genassym.c | 25 +- osfmk/i386/hibernate_i386.c | 53 +- osfmk/i386/hibernate_restore.c | 2 +- osfmk/i386/i386_init.c | 111 +- osfmk/i386/i386_lock.s | 517 +- osfmk/i386/i386_timer.c | 44 +- osfmk/i386/i386_vm_init.c | 131 +- osfmk/i386/io_map.c | 4 + osfmk/i386/lapic_native.c | 22 +- osfmk/i386/locks.h | 63 +- osfmk/i386/locks_i386.c | 475 +- osfmk/i386/machine_check.c | 18 +- osfmk/i386/machine_routines.c | 70 +- osfmk/i386/machine_routines.h | 23 +- osfmk/i386/machine_task.c | 22 + osfmk/i386/misc_protos.h | 2 + osfmk/i386/mp.c | 278 +- osfmk/i386/mp.h | 11 +- osfmk/i386/mp_desc.c | 48 +- osfmk/i386/mp_desc.h | 4 +- osfmk/i386/pcb.c | 145 +- osfmk/i386/pcb_native.c | 3 +- osfmk/i386/phys.c | 1 - osfmk/i386/pmCPU.c | 29 +- osfmk/i386/pmap.h | 14 +- osfmk/i386/pmap_common.c | 14 +- osfmk/i386/pmap_internal.h | 23 +- osfmk/i386/pmap_x86_common.c | 108 +- osfmk/i386/postcode.h | 45 +- osfmk/i386/proc_reg.h | 24 +- osfmk/i386/rtclock.c | 16 +- osfmk/i386/rtclock_asm.h | 4 +- osfmk/i386/seg.h | 6 +- osfmk/i386/task.h | 14 +- osfmk/i386/thread.h | 21 +- osfmk/i386/trap.c | 81 +- osfmk/i386/trap.h | 5 +- osfmk/i386/tsc.c | 1 + osfmk/i386/ucode.c | 29 +- osfmk/i386/ucode.h | 27 + osfmk/ipc/flipc.c | 2 +- osfmk/ipc/ipc_importance.c | 102 +- osfmk/ipc/ipc_init.c | 13 +- osfmk/ipc/ipc_kmsg.c | 130 +- osfmk/ipc/ipc_kmsg.h | 14 +- osfmk/ipc/ipc_notify.c | 2 + osfmk/ipc/ipc_object.c | 4 +- osfmk/ipc/ipc_object.h | 2 - osfmk/ipc/ipc_port.c | 488 +- osfmk/ipc/ipc_port.h | 81 +- osfmk/ipc/ipc_pset.c | 51 +- osfmk/ipc/ipc_right.c | 57 +- osfmk/ipc/ipc_types.h | 1 + osfmk/ipc/mach_kernelrpc.c | 16 +- osfmk/ipc/mach_msg.c | 105 +- osfmk/ipc/mach_port.c | 78 +- osfmk/kdp/Makefile | 4 +- osfmk/kdp/kdp.c | 3 +- osfmk/kdp/kdp_core.c | 1033 +- osfmk/kdp/kdp_core.h | 38 +- osfmk/kdp/kdp_dyld.h | 25 +- osfmk/kdp/kdp_internal.h | 10 +- osfmk/kdp/kdp_udp.c | 246 +- osfmk/kdp/ml/arm/kdp_machdep.c | 727 + osfmk/kdp/ml/arm/kdp_vm.c | 355 + osfmk/kdp/ml/i386/kdp_x86_common.c | 13 +- osfmk/kdp/ml/x86_64/kdp_machdep.c | 94 +- osfmk/kdp/ml/x86_64/kdp_vm.c | 9 +- osfmk/kdp/processor_core.c | 738 + osfmk/kdp/processor_core.h | 191 + osfmk/kern/Makefile | 12 +- osfmk/kern/affinity.c | 5 + osfmk/kern/arithmetic_128.h | 102 + osfmk/kern/assert.h | 5 - osfmk/kern/ast.c | 424 +- osfmk/kern/ast.h | 48 +- osfmk/kern/backtrace.c | 104 +- osfmk/kern/bits.h | 24 + osfmk/kern/block_hint.h | 20 + osfmk/kern/bsd_kern.c | 104 +- osfmk/kern/build_config.h | 5 + osfmk/kern/call_entry.h | 8 +- osfmk/kern/clock.c | 1095 +- osfmk/kern/clock.h | 23 +- osfmk/kern/clock_oldops.c | 5 +- osfmk/kern/coalition.c | 248 +- osfmk/kern/coalition.h | 38 +- osfmk/kern/copyout_shim.c | 99 + osfmk/kern/copyout_shim.h | 90 + osfmk/kern/cpu_number.h | 4 - osfmk/kern/cs_blobs.h | 213 + osfmk/kern/debug.c | 1231 +- osfmk/kern/debug.h | 393 +- osfmk/kern/exc_guard.h | 142 + osfmk/kern/exception.c | 104 +- osfmk/kern/gzalloc.c | 175 +- osfmk/kern/host.c | 53 +- osfmk/kern/host.h | 4 + osfmk/kern/ipc_host.c | 75 +- osfmk/kern/ipc_kobject.c | 16 +- osfmk/kern/ipc_kobject.h | 4 +- osfmk/kern/ipc_mig.c | 6 +- osfmk/kern/ipc_tt.c | 266 +- osfmk/kern/kalloc.c | 129 +- osfmk/kern/kalloc.h | 67 +- osfmk/kern/kcdata.h | 63 + osfmk/kern/kern_cdata.c | 15 +- osfmk/kern/kern_cdata.h | 4 +- osfmk/kern/kern_monotonic.c | 523 + osfmk/kern/kern_stackshot.c | 365 +- osfmk/kern/kern_types.h | 60 + osfmk/kern/kext_alloc.c | 14 +- osfmk/kern/kpc.h | 11 + osfmk/kern/kpc_common.c | 54 +- osfmk/kern/kpc_thread.c | 5 + osfmk/kern/ledger.c | 103 +- osfmk/kern/ledger.h | 9 +- osfmk/kern/locks.c | 110 +- osfmk/kern/locks.h | 15 +- osfmk/kern/ltable.c | 6 +- osfmk/kern/machine.c | 18 +- osfmk/kern/machine.h | 29 +- osfmk/kern/memset_s.c | 63 + osfmk/kern/misc_protos.h | 34 +- osfmk/kern/monotonic.h | 116 + osfmk/kern/policy_internal.h | 36 +- osfmk/kern/printf.c | 108 +- osfmk/kern/priority.c | 108 +- osfmk/kern/processor.c | 95 +- osfmk/kern/processor.h | 56 +- osfmk/kern/processor_data.c | 3 + osfmk/kern/processor_data.h | 12 + osfmk/kern/queue.h | 11 + osfmk/kern/sched.h | 17 +- osfmk/kern/sched_dualq.c | 17 +- osfmk/kern/sched_grrr.c | 14 + osfmk/kern/sched_multiq.c | 19 +- osfmk/kern/sched_prim.c | 1478 +- osfmk/kern/sched_prim.h | 140 +- osfmk/kern/sched_proto.c | 20 +- osfmk/kern/sched_traditional.c | 34 +- osfmk/kern/sfi.c | 17 +- osfmk/kern/stack.c | 52 +- osfmk/kern/startup.c | 88 +- osfmk/kern/sync_sema.c | 6 - osfmk/kern/syscall_subr.c | 24 +- osfmk/kern/syscall_sw.c | 4 +- osfmk/kern/syscall_sw.h | 2 + osfmk/kern/task.c | 606 +- osfmk/kern/task.h | 71 +- osfmk/kern/task_policy.c | 445 +- osfmk/kern/telemetry.c | 17 +- osfmk/kern/telemetry.h | 2 +- osfmk/kern/thread.c | 311 +- osfmk/kern/thread.h | 135 +- osfmk/kern/thread_act.c | 49 +- osfmk/kern/thread_call.c | 1259 +- osfmk/kern/thread_call.h | 104 +- osfmk/kern/thread_group.c | 51 + osfmk/kern/thread_group.h | 42 + osfmk/kern/thread_kernel_state.h | 46 + osfmk/kern/thread_policy.c | 284 +- osfmk/kern/timer.c | 4 + osfmk/kern/timer_call.c | 228 +- osfmk/kern/timer_call.h | 9 + osfmk/kern/timer_queue.h | 5 + osfmk/kern/waitq.c | 162 +- osfmk/kern/waitq.h | 33 +- osfmk/kern/work_interval.c | 459 + osfmk/kern/work_interval.h | 86 + osfmk/kern/zalloc.c | 1617 +- osfmk/kern/zalloc.h | 100 +- osfmk/kperf/action.c | 53 +- osfmk/kperf/action.h | 12 +- .../spl.c => osfmk/kperf/arm/kperf_meminfo.c | 122 +- osfmk/kperf/arm/kperf_mp.c | 100 + osfmk/kperf/buffer.h | 43 +- osfmk/kperf/callstack.c | 95 +- osfmk/kperf/kperf.c | 87 +- osfmk/kperf/kperf.h | 5 + osfmk/kperf/kperf_arch.h | 4 +- osfmk/kperf/kperf_timer.c | 71 +- osfmk/kperf/kperf_timer.h | 25 + osfmk/kperf/kperfbsd.c | 141 +- osfmk/kperf/pet.c | 3 +- osfmk/kperf/sample.h | 2 +- osfmk/kperf/thread_samplers.c | 91 +- osfmk/kperf/thread_samplers.h | 13 +- osfmk/kperf/x86_64/kperf_mp.c | 50 +- osfmk/libsa/arm/types.h | 75 + osfmk/libsa/machine/types.h | 2 + osfmk/libsa/string.h | 50 + osfmk/mach/Makefile | 2 + osfmk/mach/arm/Makefile | 32 + osfmk/mach/arm/_structs.h | 323 + osfmk/mach/arm/boolean.h | 70 + osfmk/mach/arm/exception.h | 66 + osfmk/mach/arm/kern_return.h | 74 + osfmk/mach/arm/ndr_def.h | 45 + osfmk/mach/arm/processor_info.h | 57 + osfmk/mach/arm/rpc.h | 35 + osfmk/mach/arm/sdt_isa.h | 440 + osfmk/mach/arm/syscall_sw.h | 123 + .../spl.h => osfmk/mach/arm/thread_state.h | 53 +- osfmk/mach/arm/thread_status.h | 708 + osfmk/mach/arm/vm_param.h | 219 + osfmk/mach/arm/vm_types.h | 161 + osfmk/mach/arm64/Makefile | 28 + osfmk/mach/coalition.h | 52 +- osfmk/mach/error.h | 2 +- osfmk/mach/exc.defs | 16 - osfmk/mach/exception_types.h | 6 +- osfmk/mach/host_priv.defs | 30 +- osfmk/mach/host_special_ports.h | 9 +- osfmk/mach/i386/_structs.h | 465 +- osfmk/mach/i386/fp_reg.h | 84 +- osfmk/mach/i386/thread_state.h | 4 + osfmk/mach/i386/thread_status.h | 71 +- osfmk/mach/i386/vm_param.h | 20 +- osfmk/mach/mach_exc.defs | 16 - osfmk/mach/mach_host.defs | 16 +- osfmk/mach/mach_traps.h | 12 + osfmk/mach/mach_types.defs | 6 +- osfmk/mach/mach_types.h | 1 + osfmk/mach/mach_vm.defs | 32 +- osfmk/mach/machine.h | 4 +- osfmk/mach/machine/Makefile | 2 +- osfmk/mach/machine/_structs.h | 40 + osfmk/mach/machine/asm.h | 4 + osfmk/mach/machine/boolean.h | 2 + osfmk/mach/machine/exception.h | 2 + osfmk/mach/machine/kern_return.h | 2 + osfmk/mach/machine/ndr_def.h | 2 + osfmk/mach/machine/processor_info.h | 2 + osfmk/mach/machine/rpc.h | 2 + osfmk/mach/machine/sdt_isa.h | 2 + osfmk/mach/machine/syscall_sw.h | 2 + osfmk/mach/machine/thread_state.h | 2 + osfmk/mach/machine/thread_status.h | 2 + osfmk/mach/machine/vm_param.h | 2 + osfmk/mach/machine/vm_types.h | 2 + osfmk/mach/memory_object_control.defs | 6 +- osfmk/mach/memory_object_types.h | 47 +- osfmk/mach/message.h | 77 +- osfmk/mach/mig.h | 2 +- osfmk/mach/mig_strncpy_zerofill_support.h | 27 + osfmk/mach/mig_voucher_support.h | 27 + osfmk/mach/port.h | 4 +- osfmk/mach/shared_memory_server.h | 14 + osfmk/mach/shared_region.h | 22 +- osfmk/mach/syscall_sw.h | 1 + osfmk/mach/task.defs | 12 + osfmk/mach/task_info.h | 74 +- osfmk/mach/task_inspect.h | 54 + osfmk/mach/thread_act.defs | 1 - osfmk/mach/thread_policy.h | 3 +- osfmk/mach/thread_status.h | 1 + osfmk/mach/vm_map.defs | 17 +- osfmk/mach/vm_param.h | 127 +- osfmk/mach/vm_prot.h | 6 - osfmk/mach/vm_purgable.h | 23 + osfmk/mach/vm_statistics.h | 83 +- osfmk/mach/vm_types.h | 60 +- osfmk/mach_debug/hash_info.h | 2 + osfmk/mach_debug/mach_debug_types.defs | 2 +- osfmk/mach_debug/mach_debug_types.h | 17 +- osfmk/mach_debug/zone_info.h | 24 +- osfmk/machine/Makefile | 6 + osfmk/machine/asm.h | 4 + osfmk/machine/atomic.h | 2 + osfmk/machine/commpage.h | 3 + osfmk/machine/config.h | 40 + osfmk/machine/cpu_affinity.h | 2 + osfmk/machine/cpu_capabilities.h | 4 + osfmk/machine/cpu_data.h | 2 + osfmk/machine/cpu_number.h | 2 + osfmk/machine/endian.h | 2 + osfmk/machine/io_map_entries.h | 2 + osfmk/machine/lock.h | 2 + osfmk/machine/locks.h | 2 + osfmk/machine/lowglobals.h | 4 + osfmk/machine/machine_cpu.h | 2 + osfmk/machine/machine_cpuid.h | 6 + osfmk/machine/machine_kpc.h | 4 + osfmk/machine/machine_routines.h | 2 + osfmk/machine/machine_rpc.h | 2 + osfmk/machine/machlimits.h | 2 + osfmk/machine/machparam.h | 2 + osfmk/machine/monotonic.h | 70 + osfmk/machine/pal_hibernate.h | 2 + osfmk/machine/pal_routines.h | 2 + osfmk/machine/pmap.h | 2 + osfmk/machine/sched_param.h | 2 + osfmk/machine/setjmp.h | 2 + osfmk/machine/simple_lock.h | 2 + osfmk/machine/smp.h | 2 + osfmk/machine/task.h | 4 + osfmk/machine/thread.h | 2 + osfmk/machine/trap.h | 2 + osfmk/machine/vm_tuning.h | 2 + osfmk/machine/xpr.h | 2 + osfmk/prng/random.c | 388 +- osfmk/prng/random.h | 33 +- osfmk/profiling/machine/profile-md.h | 2 + osfmk/vm/WKdm_new.h | 25 + osfmk/vm/bsd_vm.c | 70 +- osfmk/vm/device_vm.c | 179 +- osfmk/vm/lz4.c | 1 + osfmk/vm/lz4.h | 2 + osfmk/vm/lz4_assembly_select.h | 5 +- osfmk/vm/memory_object.c | 196 +- osfmk/vm/memory_object.h | 8 +- osfmk/vm/pmap.h | 97 +- osfmk/vm/vm32_user.c | 6 +- osfmk/vm/vm_apple_protect.c | 89 +- osfmk/vm/vm_compressor.c | 468 +- osfmk/vm/vm_compressor.h | 143 +- osfmk/vm/vm_compressor_algorithms.c | 130 +- osfmk/vm/vm_compressor_algorithms.h | 19 +- osfmk/vm/vm_compressor_backing_store.c | 396 +- osfmk/vm/vm_compressor_backing_store.h | 12 +- osfmk/vm/vm_compressor_pager.c | 56 +- osfmk/vm/vm_debug.c | 23 +- osfmk/vm/vm_fault.c | 719 +- osfmk/vm/vm_fault.h | 15 + osfmk/vm/vm_fourk_pager.c | 95 +- osfmk/vm/vm_init.c | 31 +- osfmk/vm/vm_init.h | 1 - osfmk/vm/vm_kern.c | 356 +- osfmk/vm/vm_kern.h | 211 +- osfmk/vm/vm_map.c | 3197 +- osfmk/vm/vm_map.h | 160 +- osfmk/vm/vm_map_store_rb.c | 9 + osfmk/vm/vm_object.c | 1288 +- osfmk/vm/vm_object.h | 148 +- osfmk/vm/vm_page.h | 235 +- osfmk/vm/vm_pageout.c | 3166 +- osfmk/vm/vm_pageout.h | 84 +- osfmk/vm/vm_phantom_cache.c | 12 + osfmk/vm/vm_protos.h | 75 +- osfmk/vm/vm_purgeable.c | 6 + osfmk/vm/vm_resident.c | 1432 +- osfmk/vm/vm_shared_region.c | 140 +- osfmk/vm/vm_shared_region.h | 2 + osfmk/vm/vm_swapfile_pager.c | 85 +- osfmk/vm/vm_user.c | 768 +- osfmk/x86_64/Makefile | 14 +- osfmk/x86_64/bzero.s | 7 + osfmk/x86_64/copyio.c | 22 +- osfmk/x86_64/idt64.s | 10 +- osfmk/x86_64/kpc_x86.c | 3 - osfmk/x86_64/locore.s | 12 +- osfmk/x86_64/loose_ends.c | 48 +- osfmk/x86_64/lz4_decode_x86_64.s | 27 + osfmk/x86_64/monotonic.h | 42 + osfmk/x86_64/monotonic_x86_64.c | 272 + osfmk/x86_64/pmap.c | 138 +- pexpert/arm/pe_bootargs.c | 12 + pexpert/arm/pe_consistent_debug.c | 68 + pexpert/arm/pe_identify_machine.c | 670 + pexpert/arm/pe_init.c | 661 + pexpert/arm/pe_kprintf.c | 141 + pexpert/arm/pe_serial.c | 831 + pexpert/conf/Makefile.arm | 7 + pexpert/conf/Makefile.arm64 | 7 + pexpert/conf/Makefile.template | 1 + pexpert/conf/files.arm | 8 + pexpert/conf/files.arm64 | 8 + pexpert/gen/bootargs.c | 13 + pexpert/gen/device_tree.c | 99 +- pexpert/gen/pe_gen.c | 7 + pexpert/i386/pe_init.c | 47 +- pexpert/pexpert/arm/AIC.h | 88 + pexpert/pexpert/arm/Makefile | 29 + pexpert/pexpert/arm/PL192_VIC.h | 33 + pexpert/pexpert/arm/S3cUART.h | 64 + pexpert/pexpert/arm/S7002.h | 41 + pexpert/pexpert/arm/T8002.h | 45 + pexpert/pexpert/arm/board_config.h | 40 + pexpert/pexpert/arm/boot.h | 67 + pexpert/pexpert/arm/consistent_debug.h | 117 + pexpert/pexpert/arm/protos.h | 33 + pexpert/pexpert/arm64/AIC.h | 32 + pexpert/pexpert/arm64/AMCC.h | 21 + pexpert/pexpert/arm64/Makefile | 34 + pexpert/pexpert/arm64/S3c2410x.h | 544 + pexpert/pexpert/arm64/S5L8960X.h | 19 + pexpert/pexpert/arm64/S8000.h | 17 + pexpert/pexpert/arm64/T7000.h | 19 + pexpert/pexpert/arm64/T8010.h | 42 + pexpert/pexpert/arm64/arm64_common.h | 157 + pexpert/pexpert/arm64/board_config.h | 110 + pexpert/pexpert/arm64/boot.h | 78 + pexpert/pexpert/arm64/cyclone.h | 25 + pexpert/pexpert/arm64/hurricane.h | 26 + pexpert/pexpert/arm64/twister.h | 16 + pexpert/pexpert/arm64/typhoon.h | 16 + pexpert/pexpert/device_tree.h | 81 +- pexpert/pexpert/i386/boot.h | 6 +- pexpert/pexpert/machine/boot.h | 4 + pexpert/pexpert/machine/protos.h | 2 + pexpert/pexpert/pexpert.h | 43 + san/Kasan.exports | 2 + san/Kasan_kasan.exports | 112 + san/Makefile | 93 + san/conf/Makefile | 43 + san/conf/Makefile.template | 91 + san/conf/Makefile.x86_64 | 7 + san/conf/files | 5 + san/conf/files.x86_64 | 5 + san/kasan-arm64.c | 322 + san/kasan-blacklist | 24 + san/kasan-blacklist-arm64 | 10 + san/kasan-blacklist-x86_64 | 71 + san/kasan-fakestack.c | 339 + san/kasan-memintrinsics.c | 167 + san/kasan-test-arm64.s | 58 + san/kasan-test-x86_64.s | 117 + san/kasan-test.c | 624 + san/kasan-x86_64.c | 342 + san/kasan.c | 1241 + san/kasan.h | 244 + san/kasan_dynamic_blacklist.c | 473 + san/kasan_internal.h | 148 + san/memintrinsics.h | 82 + san/tools/generate_dynamic_blacklist.py | 46 + san/tools/kasan_install | 159 + security/Makefile | 40 +- security/_label.h | 8 + security/conf/Makefile.arm | 7 + security/conf/Makefile.arm64 | 7 + security/conf/Makefile.template | 1 + security/conf/files | 1 + security/conf/files.arm | 0 security/conf/files.arm64 | 0 security/mac.h | 21 - security/mac_alloc.c | 4 +- security/mac_audit.c | 38 +- security/mac_base.c | 44 +- security/mac_framework.h | 11 +- security/mac_internal.h | 44 +- security/mac_kext.c | 28 + security/mac_mach.c | 115 +- security/mac_mach_internal.h | 22 +- security/mac_policy.h | 205 +- security/mac_priv.c | 6 + security/mac_process.c | 59 +- security/mac_pty.c | 28 + security/mac_skywalk.c | 52 + security/mac_socket.c | 61 +- security/mac_vfs.c | 390 +- tools/lldbmacros/Makefile | 4 + tools/lldbmacros/README.md | 2 + tools/lldbmacros/apic.py | 0 tools/lldbmacros/atm.py | 0 tools/lldbmacros/bank.py | 0 tools/lldbmacros/core/__init__.py | 0 tools/lldbmacros/core/caching.py | 0 tools/lldbmacros/core/configuration.py | 0 tools/lldbmacros/core/cvalue.py | 28 +- tools/lldbmacros/core/kernelcore.py | 54 +- tools/lldbmacros/core/lazytarget.py | 0 tools/lldbmacros/core/operating_system.py | 17 +- tools/lldbmacros/core/standard.py | 0 tools/lldbmacros/core/xnu_lldb_init.py | 3 + tools/lldbmacros/ioreg.py | 17 +- tools/lldbmacros/ipc.py | 31 +- tools/lldbmacros/ipcimportancedetail.py | 0 tools/lldbmacros/kasan.py | 345 + tools/lldbmacros/kauth.py | 0 tools/lldbmacros/kcdata.py | 236 +- tools/lldbmacros/kdp.py | 2 +- tools/lldbmacros/kevent.py | 381 + tools/lldbmacros/ktrace.py | 6 +- tools/lldbmacros/macho.py | 0 tools/lldbmacros/mbufdefines.py | 2 +- tools/lldbmacros/mbufs.py | 2 +- tools/lldbmacros/memory.py | 485 +- tools/lldbmacros/misc.py | 420 +- tools/lldbmacros/net.py | 2 - tools/lldbmacros/netdefines.py | 0 tools/lldbmacros/ntstat.py | 175 + tools/lldbmacros/pci.py | 0 tools/lldbmacros/pgtrace.py | 0 tools/lldbmacros/plugins/__init__.py | 0 tools/lldbmacros/plugins/iosspeedtracer.py | 0 tools/lldbmacros/plugins/speedtracer.py | 0 tools/lldbmacros/plugins/zprint_perf_log.py | 0 tools/lldbmacros/pmap.py | 33 +- tools/lldbmacros/process.py | 272 +- tools/lldbmacros/routedefines.py | 0 tools/lldbmacros/scheduler.py | 368 +- tools/lldbmacros/structanalyze.py | 0 tools/lldbmacros/userspace.py | 44 +- .../lldbmacros/usertaskdebugging/__init__.py | 0 .../lldbmacros/usertaskdebugging/gdbserver.py | 2 +- .../lldbmacros/usertaskdebugging/interface.py | 0 .../usertaskdebugging/rsprotocol.py | 0 tools/lldbmacros/usertaskdebugging/target.py | 2 +- .../usertaskdebugging/userprocess.py | 18 +- tools/lldbmacros/usertaskgdbserver.py | 0 tools/lldbmacros/utils.py | 0 tools/lldbmacros/waitq.py | 0 tools/lldbmacros/xnu.py | 161 +- tools/lldbmacros/xnudefines.py | 100 +- tools/lldbmacros/xnutriage.py | 2 +- tools/stackshot/Makefile | 24 - tools/stackshot/stackshot.c | 192 - tools/tests/MPMMTest/MPMMtest_run.sh | 24 +- tools/tests/Makefile | 4 +- tools/tests/Makefile.common | 4 +- tools/tests/darwintests/Makefile | 83 +- tools/tests/darwintests/avx.c | 736 + tools/tests/darwintests/contextswitch.c | 285 + tools/tests/darwintests/cpucount.c | 266 + tools/tests/darwintests/data_protection.c | 2 +- .../disk_mount_conditioner-entitlements.plist | 8 + .../darwintests/disk_mount_conditioner.c | 388 + .../tests/darwintests/gettimeofday_29192647.c | 47 + tools/tests/darwintests/ioperf.c | 256 + tools/tests/darwintests/kdebug.c | 139 +- tools/tests/darwintests/kevent_pty.c | 259 + tools/tests/darwintests/kevent_qos.c | 908 + tools/tests/darwintests/kpc.c | 68 + tools/tests/darwintests/kperf.c | 707 +- tools/tests/darwintests/kperf_backtracing.c | 35 +- .../darwintests/kqueue_add_and_trigger.c | 37 + tools/tests/darwintests/kqueue_close.c | 77 + .../tests/darwintests/kqueue_fifo_18776047.c | 4 +- .../kqueue_file_tests.c | 488 +- tools/tests/darwintests/kqueue_timer_tests.c | 437 + .../com.apple.xnu.test.kevent_qos.plist | 24 + .../tests/darwintests/mach_continuous_time.c | 209 +- .../mach_port_deallocate_21692215.c | 38 + tools/tests/darwintests/mach_port_mod_refs.c | 92 + .../darwintests/memorystatus_zone_test.c | 393 + tools/tests/darwintests/monotonic_core.c | 236 + tools/tests/darwintests/netbsd_utimensat.c | 191 + .../tests/darwintests/ntp_adjtime_29192647.c | 371 + tools/tests/darwintests/perf_compressor.c | 9 +- tools/tests/darwintests/perf_exit.c | 97 +- tools/tests/darwintests/perf_kdebug.c | 6 +- .../poll_select_kevent_paired_fds.c | 158 +- .../darwintests/private_entitlement.plist | 8 + tools/tests/darwintests/proc_info.c | 322 + .../tests/darwintests/settimeofday_29193041.c | 229 + .../settimeofday_29193041.entitlements | 8 + .../settimeofday_29193041_entitled.c | 214 + tools/tests/darwintests/sigchld_return.c | 50 + tools/tests/darwintests/sigcont_return.c | 28 + tools/tests/darwintests/stackshot.m | 422 + .../task_for_pid_entitlement.plist | 10 + tools/tests/darwintests/task_info.c | 907 + tools/tests/darwintests/task_info_28439149.c | 81 + tools/tests/darwintests/task_inspect.c | 19 +- .../darwintests/thread_group_set_32261625.c | 62 + tools/tests/darwintests/utimensat.c | 77 + tools/tests/darwintests/work_interval_test.c | 122 + .../work_interval_test.entitlements | 8 + tools/tests/darwintests/xnu_quick_test.c | 74 +- .../darwintests/xnu_quick_test_helpers.c | 114 + .../darwintests/xnu_quick_test_helpers.h | 16 + tools/tests/execperf/exit-asm.S | 18 + tools/tests/execperf/exit.c | 2 + tools/tests/jitter/Makefile | 12 +- tools/tests/jitter/cpu_number.s | 33 - tools/tests/jitter/timer_jitter.c | 8 +- tools/tests/kqueue_tests/Makefile | 31 - tools/tests/kqueue_tests/kqueue_timer_tests.c | 255 - tools/tests/libMicro/Makefile.com.Darwin | 0 .../perf_index/PerfIndex_COPS_Module/PITest.h | 2 +- .../PerfIndex.xcodeproj/project.pbxproj | 4 - tools/tests/perf_index/test_controller.py | 0 tools/tests/perf_index/test_fault_helper.c | 4 + tools/tests/zero-to-n/Makefile | 2 +- tools/tests/zero-to-n/zero-to-n.c | 43 +- tools/trace/kqtrace.lua | 339 + tools/trace/parse_ipc_trace.py | 28 +- 1695 files changed, 278930 insertions(+), 53551 deletions(-) create mode 100644 EXTERNAL_HEADERS/architecture/arm/Makefile create mode 100644 EXTERNAL_HEADERS/architecture/arm/arm_neon.h create mode 100644 EXTERNAL_HEADERS/corecrypto/cc_runtime_config.h create mode 100644 EXTERNAL_HEADERS/corecrypto/ccchacha20poly1305.h create mode 100644 EXTERNAL_HEADERS/corecrypto/fipspost_trace.h create mode 100644 EXTERNAL_HEADERS/mach-o/arm/reloc.h create mode 100644 EXTERNAL_HEADERS/mach-o/arm64/reloc.h create mode 100644 bsd/arm/Makefile create mode 100644 bsd/arm/_limits.h create mode 100644 bsd/arm/_mcontext.h create mode 100644 bsd/arm/_param.h create mode 100644 bsd/arm/_types.h create mode 100644 bsd/arm/disklabel.h create mode 100644 bsd/arm/endian.h create mode 100644 bsd/arm/exec.h create mode 100644 bsd/arm/fasttrap_isa.h create mode 100644 bsd/arm/limits.h create mode 100644 bsd/arm/param.h create mode 100644 bsd/arm/profile.h create mode 100644 bsd/arm/psl.h create mode 100644 bsd/arm/ptrace.h create mode 100644 bsd/arm/reboot.h create mode 100644 bsd/arm/reg.h create mode 100644 bsd/arm/signal.h create mode 100644 bsd/arm/types.h create mode 100644 bsd/arm/vmparam.h create mode 100644 bsd/conf/Makefile.arm create mode 100644 bsd/conf/Makefile.arm64 create mode 100644 bsd/conf/files.arm create mode 100644 bsd/conf/files.arm64 create mode 100644 bsd/dev/arm/conf.c create mode 100644 bsd/dev/arm/cons.c create mode 100644 bsd/dev/arm/cpu_in_cksum.s create mode 100644 bsd/dev/arm/disassembler.c create mode 100644 bsd/dev/arm/dtrace_isa.c create mode 100644 bsd/dev/arm/dtrace_subr_arm.c create mode 100644 bsd/dev/arm/fasttrap_isa.c create mode 100644 bsd/dev/arm/fbt_arm.c create mode 100644 bsd/dev/arm/kern_machdep.c create mode 100644 bsd/dev/arm/km.c create mode 100644 bsd/dev/arm/munge.c create mode 100644 bsd/dev/arm/pci_device.h create mode 100644 bsd/dev/arm/pio.h create mode 100644 bsd/dev/arm/sdt_arm.c create mode 100644 bsd/dev/arm/stubs.c create mode 100644 bsd/dev/arm/sysctl.c create mode 100644 bsd/dev/arm/systemcalls.c create mode 100644 bsd/dev/arm/table_inline.h create mode 100644 bsd/dev/arm/unix_signal.c create mode 100644 bsd/dev/arm64/conf.c create mode 100644 bsd/dev/arm64/cpu_in_cksum.s create mode 100644 bsd/dev/arm64/disassembler.c create mode 100644 bsd/dev/arm64/dtrace_isa.c create mode 100644 bsd/dev/arm64/dtrace_subr_arm.c create mode 100644 bsd/dev/arm64/fasttrap_isa.c create mode 100644 bsd/dev/arm64/fbt_arm.c create mode 100644 bsd/dev/arm64/sdt_arm.c create mode 100644 bsd/dev/arm64/sysctl.c create mode 100644 bsd/dev/dtrace/scripts/vm_map_delete_permanent.d create mode 100644 bsd/dev/monotonic.c create mode 100644 bsd/kern/kern_ntptime.c create mode 100644 bsd/kern/subr_eventhandler.c create mode 100644 bsd/machine/smp.h create mode 100644 bsd/man/man2/fs_snapshot_create.2 create mode 100644 bsd/man/man2/fs_snapshot_delete.2 create mode 100644 bsd/man/man2/fs_snapshot_list.2 create mode 100644 bsd/man/man2/fs_snapshot_rename.2 create mode 100644 bsd/man/man2/fsgetpath.2 create mode 100644 bsd/man/man2/futimens.2 delete mode 100644 bsd/man/man2/peeloff.2 delete mode 100644 bsd/man/man2/profil.2 create mode 100644 bsd/man/man2/setattrlistat.2 create mode 100644 bsd/man/man2/utimensat.2 create mode 100644 bsd/man/man9/monotonic.9 delete mode 100644 bsd/net/altq/altq_cbq.c delete mode 100644 bsd/net/altq/altq_fairq.c delete mode 100644 bsd/net/altq/altq_hfsc.c delete mode 100644 bsd/net/altq/altq_priq.c delete mode 100644 bsd/net/altq/altq_qfq.c delete mode 100644 bsd/net/altq/altq_subr.c delete mode 100644 bsd/net/altq/altq_var.h delete mode 100644 bsd/net/altq/if_altq.h delete mode 100644 bsd/net/classq/classq_blue.c delete mode 100644 bsd/net/classq/classq_red.c delete mode 100644 bsd/net/classq/classq_rio.c create mode 100644 bsd/net/if_fake.c create mode 100644 bsd/net/if_fake_var.h create mode 100644 bsd/net/if_llatbl.c create mode 100644 bsd/net/if_llatbl.h create mode 100644 bsd/net/net_api_stats.h create mode 100644 bsd/net/nwk_wq.c create mode 100644 bsd/net/nwk_wq.h create mode 100644 bsd/net/pf_pbuf.c create mode 100644 bsd/net/pf_pbuf.h delete mode 100644 bsd/net/pktsched/pktsched_cbq.c delete mode 100644 bsd/net/pktsched/pktsched_fairq.c delete mode 100644 bsd/net/pktsched/pktsched_hfsc.c delete mode 100644 bsd/net/pktsched/pktsched_priq.c delete mode 100644 bsd/net/pktsched/pktsched_rmclass.c delete mode 100644 bsd/net/pktsched/pktsched_rmclass_debug.h create mode 100644 bsd/net/skmem_sysctl.c rename bsd/netinet/{cpu_in_cksum.c => cpu_in_cksum_gen.c} (71%) create mode 100644 bsd/netinet/in_stat.c create mode 100644 bsd/netinet/in_stat.h create mode 100644 bsd/netinet6/esp_chachapoly.c create mode 100644 bsd/netinet6/esp_chachapoly.h create mode 100644 bsd/sys/_types/_caddr_t.h create mode 100644 bsd/sys/_types/_u_char.h create mode 100644 bsd/sys/_types/_u_int.h create mode 100644 bsd/sys/_types/_u_short.h create mode 100644 bsd/sys/_types/_user32_ntptimeval.h create mode 100644 bsd/sys/_types/_user32_timex.h create mode 100644 bsd/sys/_types/_user64_ntptimeval.h create mode 100644 bsd/sys/_types/_user64_timex.h create mode 100644 bsd/sys/commpage.h create mode 100644 bsd/sys/eventhandler.h create mode 100644 bsd/sys/monotonic.h create mode 100644 bsd/sys/systrace_args.h create mode 100644 bsd/sys/timex.h create mode 100644 bsd/vfs/vfs_disk_conditioner.c create mode 100644 bsd/vfs/vfs_disk_conditioner.h create mode 100644 config/BSDKernel.arm.exports create mode 100644 config/BSDKernel.arm64.exports create mode 100644 config/IOKit.arm.exports create mode 100644 config/IOKit.arm64.exports create mode 100644 config/Libkern.arm.exports create mode 100644 config/Libkern.arm64.exports create mode 100644 config/MACFramework.arm.exports create mode 100644 config/MACFramework.arm64.exports create mode 100644 config/MASTER.arm create mode 100644 config/MASTER.arm64 create mode 100644 config/Mach.arm.exports create mode 100644 config/Mach.arm64.exports create mode 100644 config/Private.arm.exports create mode 100644 config/Private.arm64.exports create mode 100644 config/System.kext/PlugIns/Kasan.kext/Info.plist create mode 100644 config/Unsupported.arm.exports create mode 100644 config/Unsupported.arm64.exports create mode 100644 iokit/conf/Makefile.arm create mode 100644 iokit/conf/Makefile.arm64 create mode 100644 iokit/conf/files.arm create mode 100644 iokit/conf/files.arm64 mode change 120000 => 100644 libkdd/kcdata.h create mode 100644 libkdd/tests/stackshot-sample-coalitions create mode 100644 libkdd/tests/stackshot-sample-coalitions.plist.gz create mode 100644 libkdd/tests/stackshot-sample-instrs-cycles create mode 100644 libkdd/tests/stackshot-sample-instrs-cycles.plist.gz create mode 100644 libkdd/tests/stackshot-sample-thread-groups create mode 100644 libkdd/tests/stackshot-sample-thread-groups.plist.gz create mode 100644 libkdd/tests/stackshot-sample-thread-policy create mode 100644 libkdd/tests/stackshot-sample-thread-policy.plist.gz mode change 100755 => 100644 libkern/c++/Tests/TestSerialization/test1/test1_main.cpp create mode 100644 libkern/conf/Makefile.arm create mode 100644 libkern/conf/Makefile.arm64 create mode 100644 libkern/conf/files.arm create mode 100644 libkern/crypto/corecrypto_chacha20poly1305.c mode change 100644 => 100755 libkern/kxld/tests/loadtest.py create mode 100644 libkern/libkern/arm/Makefile create mode 100644 libkern/libkern/arm/OSByteOrder.h create mode 100644 libkern/libkern/crypto/chacha20poly1305.h create mode 100644 libkern/os/reason_private.h delete mode 100644 libkern/zlib/gzio.c create mode 100644 libsa/conf/Makefile.arm create mode 100644 libsa/conf/Makefile.arm64 create mode 100644 libsa/conf/files.arm create mode 100644 libsa/conf/files.arm64 create mode 100644 libsyscall/Platforms/iPhoneOS/arm/syscall.map create mode 100644 libsyscall/Platforms/iPhoneOS/arm64/syscall.map delete mode 100644 libsyscall/wrappers/__commpage_gettimeofday.s create mode 100644 libsyscall/wrappers/pid_shutdown_networking.c create mode 100644 libsyscall/wrappers/utimensat.c create mode 100644 osfmk/arm/Makefile create mode 100644 osfmk/arm/WKdmCompress_new.s create mode 100644 osfmk/arm/WKdmData_new.s create mode 100644 osfmk/arm/WKdmDecompress_new.s create mode 100644 osfmk/arm/arch.h create mode 100644 osfmk/arm/arm_init.c create mode 100644 osfmk/arm/arm_timer.c create mode 100644 osfmk/arm/arm_vm_init.c create mode 100644 osfmk/arm/asm.h create mode 100644 osfmk/arm/atomic.h create mode 100644 osfmk/arm/bcopy.s create mode 100644 osfmk/arm/bsd_arm.c create mode 100644 osfmk/arm/bzero.s create mode 100644 osfmk/arm/caches.c create mode 100644 osfmk/arm/caches_asm.s create mode 100644 osfmk/arm/caches_internal.h create mode 100644 osfmk/arm/commpage/commpage.c create mode 100644 osfmk/arm/commpage/commpage.h create mode 100644 osfmk/arm/commpage/commpage_sigs.h create mode 100644 osfmk/arm/conf.c create mode 100644 osfmk/arm/cpu.c create mode 100644 osfmk/arm/cpu_affinity.h create mode 100644 osfmk/arm/cpu_capabilities.h create mode 100644 osfmk/arm/cpu_common.c create mode 100644 osfmk/arm/cpu_data.h create mode 100644 osfmk/arm/cpu_data_internal.h create mode 100644 osfmk/arm/cpu_internal.h create mode 100644 osfmk/arm/cpu_number.h create mode 100644 osfmk/arm/cpuid.c create mode 100644 osfmk/arm/cpuid.h create mode 100644 osfmk/arm/cpuid_internal.h create mode 100644 osfmk/arm/cswitch.s create mode 100644 osfmk/arm/data.s create mode 100644 osfmk/arm/dbgwrap.c create mode 100644 osfmk/arm/dbgwrap.h create mode 100644 osfmk/arm/exception.h create mode 100644 osfmk/arm/genassym.c create mode 100644 osfmk/arm/globals_asm.h create mode 100644 osfmk/arm/hw_lock_types.h create mode 100644 osfmk/arm/io_map.c create mode 100644 osfmk/arm/io_map_entries.h create mode 100644 osfmk/arm/kpc_arm.c create mode 100644 osfmk/arm/lock.h create mode 100644 osfmk/arm/locks.h create mode 100644 osfmk/arm/locks_arm.c create mode 100644 osfmk/arm/locore.s create mode 100644 osfmk/arm/loose_ends.c create mode 100644 osfmk/arm/lowglobals.h create mode 100644 osfmk/arm/lowmem_vectors.c create mode 100644 osfmk/arm/lz4_decode_armv7NEON.s create mode 100644 osfmk/arm/lz4_encode_armv7.s create mode 100644 osfmk/arm/machdep_call.c create mode 100644 osfmk/arm/machdep_call.h create mode 100644 osfmk/arm/machine_cpu.h create mode 100644 osfmk/arm/machine_cpuid.c create mode 100644 osfmk/arm/machine_cpuid.h create mode 100644 osfmk/arm/machine_kpc.h create mode 100644 osfmk/arm/machine_routines.c create mode 100644 osfmk/arm/machine_routines.h create mode 100644 osfmk/arm/machine_routines_asm.s create mode 100644 osfmk/arm/machine_routines_common.c create mode 100644 osfmk/arm/machine_task.c create mode 100644 osfmk/arm/machlimits.h create mode 100644 osfmk/arm/machparam.h create mode 100644 osfmk/arm/misc_protos.h create mode 100644 osfmk/arm/model_dep.c create mode 100644 osfmk/arm/monotonic.h create mode 100644 osfmk/arm/monotonic_arm.c create mode 100644 osfmk/arm/pal_routines.c create mode 100644 osfmk/arm/pal_routines.h create mode 100644 osfmk/arm/pcb.c create mode 100644 osfmk/arm/pmap.c create mode 100644 osfmk/arm/pmap.h create mode 100644 osfmk/arm/proc_reg.h create mode 100644 osfmk/arm/rtclock.c create mode 100644 osfmk/arm/rtclock.h create mode 100644 osfmk/arm/sched_param.h create mode 100644 osfmk/arm/setjmp.h create mode 100644 osfmk/arm/simple_lock.h create mode 100644 osfmk/arm/smp.h create mode 100644 osfmk/arm/start.s create mode 100644 osfmk/arm/status.c create mode 100644 osfmk/arm/status_shared.c create mode 100644 osfmk/arm/strlcpy.c create mode 100644 osfmk/arm/strlen.s create mode 100644 osfmk/arm/strncmp.s create mode 100644 osfmk/arm/strncpy.c create mode 100644 osfmk/arm/strnlen.s create mode 100644 osfmk/arm/task.h create mode 100644 osfmk/arm/thread.h create mode 100644 osfmk/arm/trap.c create mode 100644 osfmk/arm/trap.h create mode 100644 osfmk/arm/vm_tuning.h create mode 100644 osfmk/arm/xpr.h create mode 100644 osfmk/arm64/Makefile create mode 100644 osfmk/arm64/WKdmCompress_16k.s create mode 100644 osfmk/arm64/WKdmCompress_4k.s create mode 100644 osfmk/arm64/WKdmData.s create mode 100644 osfmk/arm64/WKdmDecompress_16k.s create mode 100644 osfmk/arm64/WKdmDecompress_4k.s create mode 100644 osfmk/arm64/alternate_debugger.c create mode 100644 osfmk/arm64/alternate_debugger.h create mode 100644 osfmk/arm64/alternate_debugger_asm.s create mode 100644 osfmk/arm64/arm_vm_init.c create mode 100644 osfmk/arm64/asm.h create mode 100644 osfmk/arm64/bcopy.s create mode 100644 osfmk/arm64/bsd_arm64.c create mode 100644 osfmk/arm64/bzero.s create mode 100644 osfmk/arm64/caches_asm.s create mode 100644 osfmk/arm64/copyio.c create mode 100644 osfmk/arm64/cpu.c create mode 100644 osfmk/arm64/cswitch.s create mode 100644 osfmk/arm64/dbgwrap.c create mode 100644 osfmk/arm64/genassym.c create mode 100644 osfmk/arm64/kpc.c create mode 100644 osfmk/arm64/locore.s create mode 100644 osfmk/arm64/loose_ends.c create mode 100644 osfmk/arm64/lowglobals.h create mode 100644 osfmk/arm64/lowmem_vectors.c create mode 100644 osfmk/arm64/lz4_decode_arm64.s create mode 100644 osfmk/arm64/lz4_encode_arm64.s create mode 100644 osfmk/arm64/machine_cpuid.h create mode 100644 osfmk/arm64/machine_kpc.h create mode 100644 osfmk/arm64/machine_machdep.h create mode 100644 osfmk/arm64/machine_routines.c create mode 100644 osfmk/arm64/machine_routines_asm.s create mode 100644 osfmk/arm64/machine_task.c create mode 100644 osfmk/arm64/monotonic.h create mode 100644 osfmk/arm64/monotonic_arm64.c create mode 100644 osfmk/arm64/pcb.c create mode 100644 osfmk/arm64/pgtrace.c create mode 100644 osfmk/arm64/pgtrace.h create mode 100644 osfmk/arm64/pgtrace_decoder.c create mode 100644 osfmk/arm64/pgtrace_decoder.h create mode 100644 osfmk/arm64/pinst.s create mode 100644 osfmk/arm64/platform_tests.c create mode 100644 osfmk/arm64/proc_reg.h create mode 100644 osfmk/arm64/sleh.c create mode 100644 osfmk/arm64/start.s create mode 100644 osfmk/arm64/status.c create mode 100644 osfmk/arm64/strncmp.s create mode 100644 osfmk/arm64/strnlen.s create mode 100644 osfmk/conf/Makefile.arm create mode 100644 osfmk/conf/Makefile.arm64 create mode 100644 osfmk/conf/files.arm create mode 100644 osfmk/conf/files.arm64 delete mode 100644 osfmk/corecrypto/cc/src/cc_abort.c create mode 100644 osfmk/corecrypto/cc/src/cc_try_abort.c create mode 100644 osfmk/corecrypto/ccaes/src/aes_tab.c create mode 100644 osfmk/corecrypto/ccaes/src/ccaes_ltc_ecb_encrypt_mode.c create mode 100644 osfmk/corecrypto/ccaes/src/ccaes_private_types.h create mode 100644 osfmk/corecrypto/ccmode/src/ccmode_ctr_crypt.c create mode 100644 osfmk/corecrypto/ccmode/src/ccmode_ctr_init.c create mode 100644 osfmk/corecrypto/ccmode/src/ccmode_ctr_setctr.c create mode 100644 osfmk/corecrypto/ccmode/src/ccmode_factory_ctr_crypt.c create mode 100644 osfmk/corecrypto/ccmode/src/ccmode_internal.h create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha256_K.c create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha256_di.c create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha256_initial_state.c create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha256_ltc_compress.c create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha256_ltc_di.c create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha2_internal.h create mode 100644 osfmk/kdp/ml/arm/kdp_machdep.c create mode 100644 osfmk/kdp/ml/arm/kdp_vm.c create mode 100644 osfmk/kdp/processor_core.c create mode 100644 osfmk/kdp/processor_core.h create mode 100644 osfmk/kern/arithmetic_128.h create mode 100644 osfmk/kern/copyout_shim.c create mode 100644 osfmk/kern/copyout_shim.h create mode 100644 osfmk/kern/cs_blobs.h create mode 100644 osfmk/kern/exc_guard.h create mode 100644 osfmk/kern/kern_monotonic.c create mode 100644 osfmk/kern/memset_s.c create mode 100644 osfmk/kern/monotonic.h create mode 100644 osfmk/kern/thread_group.c create mode 100644 osfmk/kern/thread_group.h create mode 100644 osfmk/kern/thread_kernel_state.h create mode 100644 osfmk/kern/work_interval.c create mode 100644 osfmk/kern/work_interval.h rename bsd/kern/spl.c => osfmk/kperf/arm/kperf_meminfo.c (59%) create mode 100644 osfmk/kperf/arm/kperf_mp.c create mode 100644 osfmk/libsa/arm/types.h create mode 100644 osfmk/mach/arm/Makefile create mode 100644 osfmk/mach/arm/_structs.h create mode 100644 osfmk/mach/arm/boolean.h create mode 100644 osfmk/mach/arm/exception.h create mode 100644 osfmk/mach/arm/kern_return.h create mode 100644 osfmk/mach/arm/ndr_def.h create mode 100644 osfmk/mach/arm/processor_info.h create mode 100644 osfmk/mach/arm/rpc.h create mode 100644 osfmk/mach/arm/sdt_isa.h create mode 100644 osfmk/mach/arm/syscall_sw.h rename bsd/machine/spl.h => osfmk/mach/arm/thread_state.h (58%) create mode 100644 osfmk/mach/arm/thread_status.h create mode 100644 osfmk/mach/arm/vm_param.h create mode 100644 osfmk/mach/arm/vm_types.h create mode 100644 osfmk/mach/arm64/Makefile create mode 100644 osfmk/mach/machine/_structs.h create mode 100644 osfmk/mach/task_inspect.h create mode 100644 osfmk/machine/config.h create mode 100644 osfmk/machine/monotonic.h create mode 100644 osfmk/x86_64/monotonic.h create mode 100644 osfmk/x86_64/monotonic_x86_64.c create mode 100644 pexpert/arm/pe_bootargs.c create mode 100644 pexpert/arm/pe_consistent_debug.c create mode 100644 pexpert/arm/pe_identify_machine.c create mode 100644 pexpert/arm/pe_init.c create mode 100644 pexpert/arm/pe_kprintf.c create mode 100644 pexpert/arm/pe_serial.c create mode 100644 pexpert/conf/Makefile.arm create mode 100644 pexpert/conf/Makefile.arm64 create mode 100644 pexpert/conf/files.arm create mode 100644 pexpert/conf/files.arm64 create mode 100644 pexpert/pexpert/arm/AIC.h create mode 100644 pexpert/pexpert/arm/Makefile create mode 100644 pexpert/pexpert/arm/PL192_VIC.h create mode 100644 pexpert/pexpert/arm/S3cUART.h create mode 100644 pexpert/pexpert/arm/S7002.h create mode 100644 pexpert/pexpert/arm/T8002.h create mode 100644 pexpert/pexpert/arm/board_config.h create mode 100644 pexpert/pexpert/arm/boot.h create mode 100644 pexpert/pexpert/arm/consistent_debug.h create mode 100644 pexpert/pexpert/arm/protos.h create mode 100644 pexpert/pexpert/arm64/AIC.h create mode 100644 pexpert/pexpert/arm64/AMCC.h create mode 100644 pexpert/pexpert/arm64/Makefile create mode 100644 pexpert/pexpert/arm64/S3c2410x.h create mode 100644 pexpert/pexpert/arm64/S5L8960X.h create mode 100644 pexpert/pexpert/arm64/S8000.h create mode 100644 pexpert/pexpert/arm64/T7000.h create mode 100644 pexpert/pexpert/arm64/T8010.h create mode 100644 pexpert/pexpert/arm64/arm64_common.h create mode 100644 pexpert/pexpert/arm64/board_config.h create mode 100644 pexpert/pexpert/arm64/boot.h create mode 100644 pexpert/pexpert/arm64/cyclone.h create mode 100644 pexpert/pexpert/arm64/hurricane.h create mode 100644 pexpert/pexpert/arm64/twister.h create mode 100644 pexpert/pexpert/arm64/typhoon.h create mode 100644 san/Kasan.exports create mode 100644 san/Kasan_kasan.exports create mode 100644 san/Makefile create mode 100644 san/conf/Makefile create mode 100644 san/conf/Makefile.template create mode 100644 san/conf/Makefile.x86_64 create mode 100644 san/conf/files create mode 100644 san/conf/files.x86_64 create mode 100644 san/kasan-arm64.c create mode 100644 san/kasan-blacklist create mode 100644 san/kasan-blacklist-arm64 create mode 100644 san/kasan-blacklist-x86_64 create mode 100644 san/kasan-fakestack.c create mode 100644 san/kasan-memintrinsics.c create mode 100644 san/kasan-test-arm64.s create mode 100644 san/kasan-test-x86_64.s create mode 100644 san/kasan-test.c create mode 100644 san/kasan-x86_64.c create mode 100644 san/kasan.c create mode 100644 san/kasan.h create mode 100644 san/kasan_dynamic_blacklist.c create mode 100644 san/kasan_internal.h create mode 100644 san/memintrinsics.h create mode 100755 san/tools/generate_dynamic_blacklist.py create mode 100755 san/tools/kasan_install create mode 100644 security/conf/Makefile.arm create mode 100644 security/conf/Makefile.arm64 create mode 100644 security/conf/files.arm create mode 100644 security/conf/files.arm64 create mode 100644 security/mac_skywalk.c mode change 100644 => 100755 tools/lldbmacros/apic.py mode change 100644 => 100755 tools/lldbmacros/atm.py mode change 100644 => 100755 tools/lldbmacros/bank.py mode change 100644 => 100755 tools/lldbmacros/core/__init__.py mode change 100644 => 100755 tools/lldbmacros/core/caching.py mode change 100644 => 100755 tools/lldbmacros/core/configuration.py mode change 100644 => 100755 tools/lldbmacros/core/cvalue.py mode change 100644 => 100755 tools/lldbmacros/core/kernelcore.py mode change 100644 => 100755 tools/lldbmacros/core/lazytarget.py mode change 100644 => 100755 tools/lldbmacros/core/operating_system.py mode change 100644 => 100755 tools/lldbmacros/core/standard.py mode change 100644 => 100755 tools/lldbmacros/core/xnu_lldb_init.py mode change 100644 => 100755 tools/lldbmacros/ioreg.py mode change 100644 => 100755 tools/lldbmacros/ipc.py mode change 100644 => 100755 tools/lldbmacros/ipcimportancedetail.py create mode 100755 tools/lldbmacros/kasan.py mode change 100644 => 100755 tools/lldbmacros/kauth.py mode change 100644 => 100755 tools/lldbmacros/kdp.py create mode 100755 tools/lldbmacros/kevent.py mode change 100644 => 100755 tools/lldbmacros/ktrace.py mode change 100644 => 100755 tools/lldbmacros/macho.py mode change 100644 => 100755 tools/lldbmacros/mbufdefines.py mode change 100644 => 100755 tools/lldbmacros/mbufs.py mode change 100644 => 100755 tools/lldbmacros/memory.py mode change 100644 => 100755 tools/lldbmacros/misc.py mode change 100644 => 100755 tools/lldbmacros/net.py mode change 100644 => 100755 tools/lldbmacros/netdefines.py create mode 100755 tools/lldbmacros/ntstat.py mode change 100644 => 100755 tools/lldbmacros/pci.py mode change 100644 => 100755 tools/lldbmacros/pgtrace.py mode change 100644 => 100755 tools/lldbmacros/plugins/__init__.py mode change 100644 => 100755 tools/lldbmacros/plugins/iosspeedtracer.py mode change 100644 => 100755 tools/lldbmacros/plugins/speedtracer.py mode change 100644 => 100755 tools/lldbmacros/plugins/zprint_perf_log.py mode change 100644 => 100755 tools/lldbmacros/pmap.py mode change 100644 => 100755 tools/lldbmacros/process.py mode change 100644 => 100755 tools/lldbmacros/routedefines.py mode change 100644 => 100755 tools/lldbmacros/scheduler.py mode change 100644 => 100755 tools/lldbmacros/structanalyze.py mode change 100644 => 100755 tools/lldbmacros/userspace.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/__init__.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/gdbserver.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/interface.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/rsprotocol.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/target.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/userprocess.py mode change 100644 => 100755 tools/lldbmacros/usertaskgdbserver.py mode change 100644 => 100755 tools/lldbmacros/utils.py mode change 100644 => 100755 tools/lldbmacros/waitq.py mode change 100644 => 100755 tools/lldbmacros/xnu.py mode change 100644 => 100755 tools/lldbmacros/xnudefines.py mode change 100644 => 100755 tools/lldbmacros/xnutriage.py delete mode 100644 tools/stackshot/Makefile delete mode 100644 tools/stackshot/stackshot.c create mode 100644 tools/tests/darwintests/avx.c create mode 100644 tools/tests/darwintests/contextswitch.c create mode 100644 tools/tests/darwintests/cpucount.c create mode 100644 tools/tests/darwintests/disk_mount_conditioner-entitlements.plist create mode 100644 tools/tests/darwintests/disk_mount_conditioner.c create mode 100644 tools/tests/darwintests/gettimeofday_29192647.c create mode 100644 tools/tests/darwintests/ioperf.c create mode 100644 tools/tests/darwintests/kevent_pty.c create mode 100644 tools/tests/darwintests/kevent_qos.c create mode 100644 tools/tests/darwintests/kpc.c create mode 100644 tools/tests/darwintests/kqueue_add_and_trigger.c create mode 100644 tools/tests/darwintests/kqueue_close.c rename tools/tests/{kqueue_tests => darwintests}/kqueue_file_tests.c (84%) create mode 100644 tools/tests/darwintests/kqueue_timer_tests.c create mode 100644 tools/tests/darwintests/launchd_plists/com.apple.xnu.test.kevent_qos.plist create mode 100644 tools/tests/darwintests/mach_port_deallocate_21692215.c create mode 100644 tools/tests/darwintests/mach_port_mod_refs.c create mode 100644 tools/tests/darwintests/memorystatus_zone_test.c create mode 100644 tools/tests/darwintests/monotonic_core.c create mode 100644 tools/tests/darwintests/netbsd_utimensat.c create mode 100644 tools/tests/darwintests/ntp_adjtime_29192647.c create mode 100644 tools/tests/darwintests/private_entitlement.plist create mode 100644 tools/tests/darwintests/proc_info.c create mode 100644 tools/tests/darwintests/settimeofday_29193041.c create mode 100644 tools/tests/darwintests/settimeofday_29193041.entitlements create mode 100644 tools/tests/darwintests/settimeofday_29193041_entitled.c create mode 100644 tools/tests/darwintests/sigchld_return.c create mode 100644 tools/tests/darwintests/sigcont_return.c create mode 100644 tools/tests/darwintests/stackshot.m create mode 100644 tools/tests/darwintests/task_for_pid_entitlement.plist create mode 100644 tools/tests/darwintests/task_info.c create mode 100644 tools/tests/darwintests/task_info_28439149.c create mode 100644 tools/tests/darwintests/thread_group_set_32261625.c create mode 100644 tools/tests/darwintests/utimensat.c create mode 100644 tools/tests/darwintests/work_interval_test.c create mode 100644 tools/tests/darwintests/work_interval_test.entitlements create mode 100644 tools/tests/darwintests/xnu_quick_test_helpers.c create mode 100644 tools/tests/darwintests/xnu_quick_test_helpers.h delete mode 100644 tools/tests/jitter/cpu_number.s delete mode 100755 tools/tests/kqueue_tests/Makefile delete mode 100644 tools/tests/kqueue_tests/kqueue_timer_tests.c mode change 100755 => 100644 tools/tests/libMicro/Makefile.com.Darwin mode change 100644 => 100755 tools/tests/perf_index/test_controller.py create mode 100755 tools/trace/kqtrace.lua mode change 100644 => 100755 tools/trace/parse_ipc_trace.py diff --git a/.gitignore b/.gitignore index b502c3239..f5ad2c6fd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,10 +3,13 @@ BUILD/ build/ .DS_Store -# vim turds +# vim swap files *~ *.swp +# JSON compilation definitions (for the YouCompleteMe vim plugin) +compile_commands.json + # / /.remotebuild_credential /cscope.* diff --git a/.upstream_base_commits b/.upstream_base_commits index d9ee6e9d6..0343ee6c1 100644 --- a/.upstream_base_commits +++ b/.upstream_base_commits @@ -1,3 +1,5 @@ #freebsd = https://github.com/freebsd/freebsd.git bsd/man/man2/access.2 freebsd lib/libc/sys/access.2 5b882020081a138285227631c46a406c08e17bc8 bsd/man/man7/sticky.7 freebsd share/man/man7/sticky.7 5b882020081a138285227631c46a406c08e17bc8 +bsd/man/man2/utimensat.2 freebsd lib/libc/sys/utimensat.2 89c1fcc0d088065021703b658ef547f46b5481f0 +tools/tests/darwintests/netbsd_utimensat.c freebsd contrib/netbsd-tests/lib/libc/c063/t_utimensat.c 89c1fcc0d088065021703b658ef547f46b5481f0 diff --git a/EXTERNAL_HEADERS/AvailabilityMacros.h b/EXTERNAL_HEADERS/AvailabilityMacros.h index a9027a58a..6728fc446 100644 --- a/EXTERNAL_HEADERS/AvailabilityMacros.h +++ b/EXTERNAL_HEADERS/AvailabilityMacros.h @@ -124,6 +124,8 @@ #else #if __i386__ || __x86_64__ #define MAC_OS_X_VERSION_MIN_REQUIRED MAC_OS_X_VERSION_10_4 + #elif __arm__ || __arm64__ + #define MAC_OS_X_VERSION_MIN_REQUIRED MAC_OS_X_VERSION_10_5 #else #define MAC_OS_X_VERSION_MIN_REQUIRED MAC_OS_X_VERSION_10_1 #endif diff --git a/EXTERNAL_HEADERS/architecture/arm/Makefile b/EXTERNAL_HEADERS/architecture/arm/Makefile new file mode 100644 index 000000000..08f41e365 --- /dev/null +++ b/EXTERNAL_HEADERS/architecture/arm/Makefile @@ -0,0 +1,21 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +EXPORT_FILES = \ + arm_neon.h \ + +INSTALL_MD_LIST = + +INSTALL_MD_DIR = + +EXPORT_MD_LIST = ${EXPORT_FILES} + +EXPORT_MD_DIR = architecture/arm + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/EXTERNAL_HEADERS/architecture/arm/arm_neon.h b/EXTERNAL_HEADERS/architecture/arm/arm_neon.h new file mode 100644 index 000000000..e294bd96c --- /dev/null +++ b/EXTERNAL_HEADERS/architecture/arm/arm_neon.h @@ -0,0 +1,74267 @@ +/*===---- arm_neon.h - ARM Neon intrinsics ---------------------------------=== + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + *===-----------------------------------------------------------------------=== + */ + +#ifndef __ARM_NEON_H +#define __ARM_NEON_H + +#if !defined(__ARM_NEON) +#error "NEON support not enabled" +#endif + +#include + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wfloat-equal" +#pragma clang diagnostic ignored "-Wvector-conversion" + +typedef float float32_t; +typedef __fp16 float16_t; +#ifdef __aarch64__ +typedef double float64_t; +#endif + +#ifdef __aarch64__ +typedef uint8_t poly8_t; +typedef uint16_t poly16_t; +typedef uint64_t poly64_t; +typedef __uint128_t poly128_t; +#else +typedef int8_t poly8_t; +typedef int16_t poly16_t; +#endif +typedef __attribute__((neon_vector_type(8))) int8_t int8x8_t; +typedef __attribute__((neon_vector_type(16))) int8_t int8x16_t; +typedef __attribute__((neon_vector_type(4))) int16_t int16x4_t; +typedef __attribute__((neon_vector_type(8))) int16_t int16x8_t; +typedef __attribute__((neon_vector_type(2))) int32_t int32x2_t; +typedef __attribute__((neon_vector_type(4))) int32_t int32x4_t; +typedef __attribute__((neon_vector_type(1))) int64_t int64x1_t; +typedef __attribute__((neon_vector_type(2))) int64_t int64x2_t; +typedef __attribute__((neon_vector_type(8))) uint8_t uint8x8_t; +typedef __attribute__((neon_vector_type(16))) uint8_t uint8x16_t; +typedef __attribute__((neon_vector_type(4))) uint16_t uint16x4_t; +typedef __attribute__((neon_vector_type(8))) uint16_t uint16x8_t; +typedef __attribute__((neon_vector_type(2))) uint32_t uint32x2_t; +typedef __attribute__((neon_vector_type(4))) uint32_t uint32x4_t; +typedef __attribute__((neon_vector_type(1))) uint64_t uint64x1_t; +typedef __attribute__((neon_vector_type(2))) uint64_t uint64x2_t; +typedef __attribute__((neon_vector_type(4))) float16_t float16x4_t; +typedef __attribute__((neon_vector_type(8))) float16_t float16x8_t; +typedef __attribute__((neon_vector_type(2))) float32_t float32x2_t; +typedef __attribute__((neon_vector_type(4))) float32_t float32x4_t; +#ifdef __aarch64__ +typedef __attribute__((neon_vector_type(1))) float64_t float64x1_t; +typedef __attribute__((neon_vector_type(2))) float64_t float64x2_t; +#endif +typedef __attribute__((neon_polyvector_type(8))) poly8_t poly8x8_t; +typedef __attribute__((neon_polyvector_type(16))) poly8_t poly8x16_t; +typedef __attribute__((neon_polyvector_type(4))) poly16_t poly16x4_t; +typedef __attribute__((neon_polyvector_type(8))) poly16_t poly16x8_t; +#ifdef __aarch64__ +typedef __attribute__((neon_polyvector_type(1))) poly64_t poly64x1_t; +typedef __attribute__((neon_polyvector_type(2))) poly64_t poly64x2_t; +#endif + +typedef struct int8x8x2_t { + int8x8_t val[2]; +} int8x8x2_t; + +typedef struct int8x16x2_t { + int8x16_t val[2]; +} int8x16x2_t; + +typedef struct int16x4x2_t { + int16x4_t val[2]; +} int16x4x2_t; + +typedef struct int16x8x2_t { + int16x8_t val[2]; +} int16x8x2_t; + +typedef struct int32x2x2_t { + int32x2_t val[2]; +} int32x2x2_t; + +typedef struct int32x4x2_t { + int32x4_t val[2]; +} int32x4x2_t; + +typedef struct int64x1x2_t { + int64x1_t val[2]; +} int64x1x2_t; + +typedef struct int64x2x2_t { + int64x2_t val[2]; +} int64x2x2_t; + +typedef struct uint8x8x2_t { + uint8x8_t val[2]; +} uint8x8x2_t; + +typedef struct uint8x16x2_t { + uint8x16_t val[2]; +} uint8x16x2_t; + +typedef struct uint16x4x2_t { + uint16x4_t val[2]; +} uint16x4x2_t; + +typedef struct uint16x8x2_t { + uint16x8_t val[2]; +} uint16x8x2_t; + +typedef struct uint32x2x2_t { + uint32x2_t val[2]; +} uint32x2x2_t; + +typedef struct uint32x4x2_t { + uint32x4_t val[2]; +} uint32x4x2_t; + +typedef struct uint64x1x2_t { + uint64x1_t val[2]; +} uint64x1x2_t; + +typedef struct uint64x2x2_t { + uint64x2_t val[2]; +} uint64x2x2_t; + +typedef struct float16x4x2_t { + float16x4_t val[2]; +} float16x4x2_t; + +typedef struct float16x8x2_t { + float16x8_t val[2]; +} float16x8x2_t; + +typedef struct float32x2x2_t { + float32x2_t val[2]; +} float32x2x2_t; + +typedef struct float32x4x2_t { + float32x4_t val[2]; +} float32x4x2_t; + +#ifdef __aarch64__ +typedef struct float64x1x2_t { + float64x1_t val[2]; +} float64x1x2_t; + +typedef struct float64x2x2_t { + float64x2_t val[2]; +} float64x2x2_t; + +#endif +typedef struct poly8x8x2_t { + poly8x8_t val[2]; +} poly8x8x2_t; + +typedef struct poly8x16x2_t { + poly8x16_t val[2]; +} poly8x16x2_t; + +typedef struct poly16x4x2_t { + poly16x4_t val[2]; +} poly16x4x2_t; + +typedef struct poly16x8x2_t { + poly16x8_t val[2]; +} poly16x8x2_t; + +#ifdef __aarch64__ +typedef struct poly64x1x2_t { + poly64x1_t val[2]; +} poly64x1x2_t; + +typedef struct poly64x2x2_t { + poly64x2_t val[2]; +} poly64x2x2_t; + +#endif +typedef struct int8x8x3_t { + int8x8_t val[3]; +} int8x8x3_t; + +typedef struct int8x16x3_t { + int8x16_t val[3]; +} int8x16x3_t; + +typedef struct int16x4x3_t { + int16x4_t val[3]; +} int16x4x3_t; + +typedef struct int16x8x3_t { + int16x8_t val[3]; +} int16x8x3_t; + +typedef struct int32x2x3_t { + int32x2_t val[3]; +} int32x2x3_t; + +typedef struct int32x4x3_t { + int32x4_t val[3]; +} int32x4x3_t; + +typedef struct int64x1x3_t { + int64x1_t val[3]; +} int64x1x3_t; + +typedef struct int64x2x3_t { + int64x2_t val[3]; +} int64x2x3_t; + +typedef struct uint8x8x3_t { + uint8x8_t val[3]; +} uint8x8x3_t; + +typedef struct uint8x16x3_t { + uint8x16_t val[3]; +} uint8x16x3_t; + +typedef struct uint16x4x3_t { + uint16x4_t val[3]; +} uint16x4x3_t; + +typedef struct uint16x8x3_t { + uint16x8_t val[3]; +} uint16x8x3_t; + +typedef struct uint32x2x3_t { + uint32x2_t val[3]; +} uint32x2x3_t; + +typedef struct uint32x4x3_t { + uint32x4_t val[3]; +} uint32x4x3_t; + +typedef struct uint64x1x3_t { + uint64x1_t val[3]; +} uint64x1x3_t; + +typedef struct uint64x2x3_t { + uint64x2_t val[3]; +} uint64x2x3_t; + +typedef struct float16x4x3_t { + float16x4_t val[3]; +} float16x4x3_t; + +typedef struct float16x8x3_t { + float16x8_t val[3]; +} float16x8x3_t; + +typedef struct float32x2x3_t { + float32x2_t val[3]; +} float32x2x3_t; + +typedef struct float32x4x3_t { + float32x4_t val[3]; +} float32x4x3_t; + +#ifdef __aarch64__ +typedef struct float64x1x3_t { + float64x1_t val[3]; +} float64x1x3_t; + +typedef struct float64x2x3_t { + float64x2_t val[3]; +} float64x2x3_t; + +#endif +typedef struct poly8x8x3_t { + poly8x8_t val[3]; +} poly8x8x3_t; + +typedef struct poly8x16x3_t { + poly8x16_t val[3]; +} poly8x16x3_t; + +typedef struct poly16x4x3_t { + poly16x4_t val[3]; +} poly16x4x3_t; + +typedef struct poly16x8x3_t { + poly16x8_t val[3]; +} poly16x8x3_t; + +#ifdef __aarch64__ +typedef struct poly64x1x3_t { + poly64x1_t val[3]; +} poly64x1x3_t; + +typedef struct poly64x2x3_t { + poly64x2_t val[3]; +} poly64x2x3_t; + +#endif +typedef struct int8x8x4_t { + int8x8_t val[4]; +} int8x8x4_t; + +typedef struct int8x16x4_t { + int8x16_t val[4]; +} int8x16x4_t; + +typedef struct int16x4x4_t { + int16x4_t val[4]; +} int16x4x4_t; + +typedef struct int16x8x4_t { + int16x8_t val[4]; +} int16x8x4_t; + +typedef struct int32x2x4_t { + int32x2_t val[4]; +} int32x2x4_t; + +typedef struct int32x4x4_t { + int32x4_t val[4]; +} int32x4x4_t; + +typedef struct int64x1x4_t { + int64x1_t val[4]; +} int64x1x4_t; + +typedef struct int64x2x4_t { + int64x2_t val[4]; +} int64x2x4_t; + +typedef struct uint8x8x4_t { + uint8x8_t val[4]; +} uint8x8x4_t; + +typedef struct uint8x16x4_t { + uint8x16_t val[4]; +} uint8x16x4_t; + +typedef struct uint16x4x4_t { + uint16x4_t val[4]; +} uint16x4x4_t; + +typedef struct uint16x8x4_t { + uint16x8_t val[4]; +} uint16x8x4_t; + +typedef struct uint32x2x4_t { + uint32x2_t val[4]; +} uint32x2x4_t; + +typedef struct uint32x4x4_t { + uint32x4_t val[4]; +} uint32x4x4_t; + +typedef struct uint64x1x4_t { + uint64x1_t val[4]; +} uint64x1x4_t; + +typedef struct uint64x2x4_t { + uint64x2_t val[4]; +} uint64x2x4_t; + +typedef struct float16x4x4_t { + float16x4_t val[4]; +} float16x4x4_t; + +typedef struct float16x8x4_t { + float16x8_t val[4]; +} float16x8x4_t; + +typedef struct float32x2x4_t { + float32x2_t val[4]; +} float32x2x4_t; + +typedef struct float32x4x4_t { + float32x4_t val[4]; +} float32x4x4_t; + +#ifdef __aarch64__ +typedef struct float64x1x4_t { + float64x1_t val[4]; +} float64x1x4_t; + +typedef struct float64x2x4_t { + float64x2_t val[4]; +} float64x2x4_t; + +#endif +typedef struct poly8x8x4_t { + poly8x8_t val[4]; +} poly8x8x4_t; + +typedef struct poly8x16x4_t { + poly8x16_t val[4]; +} poly8x16x4_t; + +typedef struct poly16x4x4_t { + poly16x4_t val[4]; +} poly16x4x4_t; + +typedef struct poly16x8x4_t { + poly16x8_t val[4]; +} poly16x8x4_t; + +#ifdef __aarch64__ +typedef struct poly64x1x4_t { + poly64x1_t val[4]; +} poly64x1x4_t; + +typedef struct poly64x2x4_t { + poly64x2_t val[4]; +} poly64x2x4_t; + +#endif + +#define __ai static inline __attribute__((__always_inline__, __nodebug__)) + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vabdq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vabdq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x16_t __noswap_vabdq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabdq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vabdq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vabdq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabdq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vabdq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vabdq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vabdq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vabdq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x16_t __noswap_vabdq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vabdq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vabdq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabdq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vabdq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vabdq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabdq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vabdq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vabdq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vabd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vabd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vabd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vabd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vabd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vabd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vabd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vabd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vabd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vabd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vabd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vabd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vabd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vabd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vabd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vabd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vabd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vabd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vabd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vabd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vabsq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vabsq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vabsq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vabsq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabsq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vabsq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabsq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vabsq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vabs_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vabs_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vabs_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vabs_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vabs_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vabs_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vabs_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vabs_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vabs_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vabs_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vabs_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vabs_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vabs_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vabs_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vabs_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vabs_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint8x16_t vaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint32x4_t vaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint64x2_t vaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint16x8_t vaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int8x16_t vaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vaddq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float32x4_t vaddq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int32x4_t vaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int64x2_t vaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int16x8_t vaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint8x8_t vadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint32x2_t vadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint64x1_t vadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint16x4_t vadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int8x8_t vadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vadd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float32x2_t vadd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int32x2_t vadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int64x1_t vadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int16x4_t vadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vaddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vaddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vaddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vaddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vaddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vaddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vaddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vaddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vaddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vaddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vaddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vaddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vaddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vaddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vaddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vaddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vaddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vaddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vandq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint8x16_t vandq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vandq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint32x4_t vandq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vandq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint64x2_t vandq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vandq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint16x8_t vandq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vandq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int8x16_t vandq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vandq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int32x4_t vandq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vandq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int64x2_t vandq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vandq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int16x8_t vandq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vand_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint8x8_t vand_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vand_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint32x2_t vand_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vand_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint64x1_t vand_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vand_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint16x4_t vand_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vand_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int8x8_t vand_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vand_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int32x2_t vand_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vand_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int64x1_t vand_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vand_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int16x4_t vand_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vbicq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint8x16_t vbicq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vbicq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint32x4_t vbicq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vbicq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint64x2_t vbicq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vbicq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint16x8_t vbicq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vbicq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int8x16_t vbicq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vbicq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int32x4_t vbicq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vbicq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int64x2_t vbicq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vbicq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int16x8_t vbicq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vbic_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint8x8_t vbic_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vbic_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint32x2_t vbic_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vbic_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint64x1_t vbic_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vbic_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint16x4_t vbic_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vbic_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int8x8_t vbic_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vbic_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int32x2_t vbic_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vbic_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int64x1_t vbic_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vbic_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int16x4_t vbic_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vbsl_p8(uint8x8_t __p0, poly8x8_t __p1, poly8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vbsl_p8(uint8x8_t __p0, poly8x8_t __p1, poly8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vbsl_p16(uint16x4_t __p0, poly16x4_t __p1, poly16x4_t __p2) { + poly16x4_t __ret; + __ret = (poly16x4_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 5); + return __ret; +} +#else +__ai poly16x4_t vbsl_p16(uint16x4_t __p0, poly16x4_t __p1, poly16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = (poly16x4_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vbslq_p8(uint8x16_t __p0, poly8x16_t __p1, poly8x16_t __p2) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 36); + return __ret; +} +#else +__ai poly8x16_t vbslq_p8(uint8x16_t __p0, poly8x16_t __p1, poly8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vbslq_p16(uint16x8_t __p0, poly16x8_t __p1, poly16x8_t __p2) { + poly16x8_t __ret; + __ret = (poly16x8_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 37); + return __ret; +} +#else +__ai poly16x8_t vbslq_p16(uint16x8_t __p0, poly16x8_t __p1, poly16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = (poly16x8_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 37); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vbslq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 48); + return __ret; +} +#else +__ai uint8x16_t vbslq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vbslq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 50); + return __ret; +} +#else +__ai uint32x4_t vbslq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vbslq_u64(uint64x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 51); + return __ret; +} +#else +__ai uint64x2_t vbslq_u64(uint64x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vbslq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 49); + return __ret; +} +#else +__ai uint16x8_t vbslq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vbslq_s8(uint8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 32); + return __ret; +} +#else +__ai int8x16_t vbslq_s8(uint8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vbslq_f32(uint32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); + return __ret; +} +#else +__ai float32x4_t vbslq_f32(uint32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vbslq_s32(uint32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 34); + return __ret; +} +#else +__ai int32x4_t vbslq_s32(uint32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vbslq_s64(uint64x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 35); + return __ret; +} +#else +__ai int64x2_t vbslq_s64(uint64x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vbslq_s16(uint16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 33); + return __ret; +} +#else +__ai int16x8_t vbslq_s16(uint16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vbsl_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vbsl_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vbsl_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 18); + return __ret; +} +#else +__ai uint32x2_t vbsl_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vbsl_u64(uint64x1_t __p0, uint64x1_t __p1, uint64x1_t __p2) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 19); + return __ret; +} +#else +__ai uint64x1_t vbsl_u64(uint64x1_t __p0, uint64x1_t __p1, uint64x1_t __p2) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vbsl_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 17); + return __ret; +} +#else +__ai uint16x4_t vbsl_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vbsl_s8(uint8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vbsl_s8(uint8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vbsl_f32(uint32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); + return __ret; +} +#else +__ai float32x2_t vbsl_f32(uint32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vbsl_s32(uint32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 2); + return __ret; +} +#else +__ai int32x2_t vbsl_s32(uint32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vbsl_s64(uint64x1_t __p0, int64x1_t __p1, int64x1_t __p2) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 3); + return __ret; +} +#else +__ai int64x1_t vbsl_s64(uint64x1_t __p0, int64x1_t __p1, int64x1_t __p2) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vbsl_s16(uint16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 1); + return __ret; +} +#else +__ai int16x4_t vbsl_s16(uint16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcageq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcageq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vcageq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcageq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcage_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcage_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vcage_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcage_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcagtq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcagtq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vcagtq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcagtq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcagt_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcagt_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vcagt_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcagt_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcaleq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcaleq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vcaleq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcaleq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcale_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcale_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vcale_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcale_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcaltq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcaltq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vcaltq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcaltq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcalt_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcalt_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vcalt_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcalt_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceq_p8(poly8x8_t __p0, poly8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x8_t vceq_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqq_p8(poly8x16_t __p0, poly8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x16_t vceqq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x16_t vceqq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x4_t vceqq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x8_t vceqq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x16_t vceqq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x4_t vceqq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x4_t vceqq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x8_t vceqq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceq_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x8_t vceq_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceq_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x2_t vceq_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceq_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x4_t vceq_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceq_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x8_t vceq_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceq_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x2_t vceq_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceq_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x2_t vceq_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceq_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x4_t vceq_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgeq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint8x16_t vcgeq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgeq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x4_t vcgeq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgeq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x8_t vcgeq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgeq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint8x16_t vcgeq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgeq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x4_t vcgeq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgeq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x4_t vcgeq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgeq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x8_t vcgeq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcge_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint8x8_t vcge_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcge_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x2_t vcge_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcge_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x4_t vcge_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcge_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint8x8_t vcge_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcge_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x2_t vcge_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcge_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x2_t vcge_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcge_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x4_t vcge_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgtq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint8x16_t vcgtq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgtq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x4_t vcgtq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgtq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x8_t vcgtq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgtq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint8x16_t vcgtq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgtq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x4_t vcgtq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgtq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x4_t vcgtq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgtq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x8_t vcgtq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcgt_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint8x8_t vcgt_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgt_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x2_t vcgt_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgt_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x4_t vcgt_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcgt_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint8x8_t vcgt_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgt_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x2_t vcgt_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgt_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x2_t vcgt_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgt_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x4_t vcgt_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcleq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint8x16_t vcleq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcleq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x4_t vcleq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcleq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x8_t vcleq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcleq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint8x16_t vcleq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcleq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x4_t vcleq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcleq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x4_t vcleq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcleq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x8_t vcleq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcle_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint8x8_t vcle_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcle_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x2_t vcle_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcle_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x4_t vcle_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcle_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint8x8_t vcle_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcle_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x2_t vcle_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcle_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x2_t vcle_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcle_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x4_t vcle_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vclsq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vclsq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vclsq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vclsq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vclsq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vclsq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vclsq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vclsq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vclsq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vclsq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vclsq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vclsq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vcls_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vcls_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vcls_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vcls_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcls_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcls_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcls_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcls_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcls_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcls_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcls_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcls_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcltq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint8x16_t vcltq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcltq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x4_t vcltq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcltq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x8_t vcltq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcltq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint8x16_t vcltq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcltq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x4_t vcltq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcltq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x4_t vcltq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcltq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x8_t vcltq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vclt_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint8x8_t vclt_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclt_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x2_t vclt_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclt_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x4_t vclt_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vclt_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint8x8_t vclt_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclt_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x2_t vclt_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclt_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x2_t vclt_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclt_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x4_t vclt_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vclzq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vclzq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vclzq_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vclzq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vclzq_u16(uint16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vclzq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vclzq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vclzq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vclzq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vclzq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vclzq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vclzq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vclz_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vclz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vclz_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclz_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vclz_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclz_u16(uint16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vclz_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vclz_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vclz_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vclz_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vclz_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vclz_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vclz_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vclz_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vclz_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vclz_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vcnt_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vcnt_v((int8x8_t)__p0, 4); + return __ret; +} +#else +__ai poly8x8_t vcnt_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vcnt_v((int8x8_t)__rev0, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vcntq_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vcntq_v((int8x16_t)__p0, 36); + return __ret; +} +#else +__ai poly8x16_t vcntq_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vcntq_v((int8x16_t)__rev0, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcntq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcntq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vcntq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcntq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vcntq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vcntq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vcntq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vcntq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcnt_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcnt_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vcnt_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcnt_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vcnt_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vcnt_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vcnt_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vcnt_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vcombine_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai poly8x16_t vcombine_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vcombine_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai poly16x8_t vcombine_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcombine_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai uint8x16_t vcombine_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x16_t __noswap_vcombine_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcombine_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#else +__ai uint32x4_t vcombine_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vcombine_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcombine_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + return __ret; +} +#else +__ai uint64x2_t vcombine_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcombine_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai uint16x8_t vcombine_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vcombine_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vcombine_s8(int8x8_t __p0, int8x8_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai int8x16_t vcombine_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x16_t __noswap_vcombine_s8(int8x8_t __p0, int8x8_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcombine_f32(float32x2_t __p0, float32x2_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#else +__ai float32x4_t vcombine_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float32x4_t __noswap_vcombine_f32(float32x2_t __p0, float32x2_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vcombine_f16(float16x4_t __p0, float16x4_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai float16x8_t vcombine_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai float16x8_t __noswap_vcombine_f16(float16x4_t __p0, float16x4_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcombine_s32(int32x2_t __p0, int32x2_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#else +__ai int32x4_t vcombine_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vcombine_s32(int32x2_t __p0, int32x2_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcombine_s64(int64x1_t __p0, int64x1_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + return __ret; +} +#else +__ai int64x2_t vcombine_s64(int64x1_t __p0, int64x1_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcombine_s16(int16x4_t __p0, int16x4_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai int16x8_t vcombine_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vcombine_s16(int16x4_t __p0, int16x4_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vcreate_p8(uint64_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vcreate_p8(uint64_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vcreate_p16(uint64_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vcreate_p16(uint64_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcreate_u8(uint64_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vcreate_u8(uint64_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcreate_u32(uint64_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vcreate_u32(uint64_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcreate_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vcreate_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcreate_u16(uint64_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vcreate_u16(uint64_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vcreate_s8(uint64_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vcreate_s8(uint64_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vcreate_f32(uint64_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vcreate_f32(uint64_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vcreate_f16(uint64_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vcreate_f16(uint64_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcreate_s32(uint64_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vcreate_s32(uint64_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcreate_s64(uint64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vcreate_s64(uint64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcreate_s16(uint64_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vcreate_s16(uint64_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvtq_f32_u32(uint32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvtq_f32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai float32x4_t vcvtq_f32_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvtq_f32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvtq_f32_s32(int32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvtq_f32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai float32x4_t vcvtq_f32_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvtq_f32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vcvt_f32_u32(uint32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai float32x2_t vcvt_f32_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vcvt_f32_s32(int32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai float32x2_t vcvt_f32_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vcvtq_n_f32_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vcvtq_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vcvtq_n_f32_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vcvtq_n_f32_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vcvtq_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vcvtq_n_f32_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vcvt_n_f32_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vcvt_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vcvt_n_f32_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vcvt_n_f32_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vcvt_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vcvt_n_f32_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vcvtq_n_s32_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vcvtq_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vcvtq_n_s32_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vcvt_n_s32_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vcvt_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vcvt_n_s32_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vcvtq_n_u32_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vcvtq_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vcvtq_n_u32_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vcvt_n_u32_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vcvt_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vcvt_n_u32_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcvtq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtq_s32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vcvtq_s32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtq_s32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcvt_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvt_s32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcvt_s32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvt_s32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcvtq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtq_u32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcvtq_u32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtq_u32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcvt_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvt_u32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcvt_u32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvt_u32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vdup_n_p8(poly8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly8x8_t vdup_n_p8(poly8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vdup_n_p16(poly16_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly16x4_t vdup_n_p16(poly16_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vdupq_n_p8(poly8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly8x16_t vdupq_n_p8(poly8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vdupq_n_p16(poly16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly16x8_t vdupq_n_p16(poly16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vdupq_n_u8(uint8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint8x16_t vdupq_n_u8(uint8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vdupq_n_u32(uint32_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint32x4_t vdupq_n_u32(uint32_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vdupq_n_u64(uint64_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai uint64x2_t vdupq_n_u64(uint64_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vdupq_n_u16(uint16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint16x8_t vdupq_n_u16(uint16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vdupq_n_s8(int8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int8x16_t vdupq_n_s8(int8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vdupq_n_f32(float32_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai float32x4_t vdupq_n_f32(float32_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) {__s0, __s0, __s0, __s0, __s0, __s0, __s0, __s0}; \ + __ret; \ +}) +#else +#define vdupq_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) {__s0, __s0, __s0, __s0, __s0, __s0, __s0, __s0}; \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vdupq_n_s32(int32_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int32x4_t vdupq_n_s32(int32_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vdupq_n_s64(int64_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai int64x2_t vdupq_n_s64(int64_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vdupq_n_s16(int16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int16x8_t vdupq_n_s16(int16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vdup_n_u8(uint8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint8x8_t vdup_n_u8(uint8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vdup_n_u32(uint32_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai uint32x2_t vdup_n_u32(uint32_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vdup_n_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) {__p0}; + return __ret; +} +#else +__ai uint64x1_t vdup_n_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vdup_n_u16(uint16_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint16x4_t vdup_n_u16(uint16_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vdup_n_s8(int8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int8x8_t vdup_n_s8(int8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vdup_n_f32(float32_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai float32x2_t vdup_n_f32(float32_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) {__s0, __s0, __s0, __s0}; \ + __ret; \ +}) +#else +#define vdup_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) {__s0, __s0, __s0, __s0}; \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vdup_n_s32(int32_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai int32x2_t vdup_n_s32(int32_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vdup_n_s64(int64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) {__p0}; + return __ret; +} +#else +__ai int64x1_t vdup_n_s64(int64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vdup_n_s16(int16_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int16x4_t vdup_n_s16(int16_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t veorq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint8x16_t veorq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t veorq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint32x4_t veorq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t veorq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint64x2_t veorq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t veorq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint16x8_t veorq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t veorq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int8x16_t veorq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t veorq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int32x4_t veorq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t veorq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int64x2_t veorq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t veorq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int16x8_t veorq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t veor_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint8x8_t veor_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t veor_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint32x2_t veor_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t veor_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint64x1_t veor_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t veor_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint16x4_t veor_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t veor_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int8x8_t veor_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t veor_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int32x2_t veor_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t veor_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int64x1_t veor_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t veor_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int16x4_t veor_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 4); \ + __ret; \ +}) +#else +#define vext_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 5); \ + __ret; \ +}) +#else +#define vext_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 36); \ + __ret; \ +}) +#else +#define vextq_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 37); \ + __ret; \ +}) +#else +#define vextq_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vextq_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vextq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vextq_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vextq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vextq_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 41); \ + __ret; \ +}) +#else +#define vextq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vextq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vextq_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vextq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vext_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vext_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vext_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vext_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vext_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 9); \ + __ret; \ +}) +#else +#define vext_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vext_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vext_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vext_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vget_high_p8(poly8x16_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai poly8x8_t vget_high_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai poly8x8_t __noswap_vget_high_p8(poly8x16_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vget_high_p16(poly16x8_t __p0) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#else +__ai poly16x4_t vget_high_p16(poly16x8_t __p0) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vget_high_u8(uint8x16_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai uint8x8_t vget_high_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vget_high_u8(uint8x16_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vget_high_u32(uint32x4_t __p0) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#else +__ai uint32x2_t vget_high_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vget_high_u32(uint32x4_t __p0) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vget_high_u64(uint64x2_t __p0) { + uint64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1); + return __ret; +} +#else +__ai uint64x1_t vget_high_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vget_high_u16(uint16x8_t __p0) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#else +__ai uint16x4_t vget_high_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vget_high_u16(uint16x8_t __p0) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vget_high_s8(int8x16_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai int8x8_t vget_high_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vget_high_s8(int8x16_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vget_high_f32(float32x4_t __p0) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#else +__ai float32x2_t vget_high_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float32x2_t __noswap_vget_high_f32(float32x4_t __p0) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vget_high_f16(float16x8_t __p0) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#else +__ai float16x4_t vget_high_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float16x4_t __noswap_vget_high_f16(float16x8_t __p0) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vget_high_s32(int32x4_t __p0) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#else +__ai int32x2_t vget_high_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vget_high_s32(int32x4_t __p0) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vget_high_s64(int64x2_t __p0) { + int64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1); + return __ret; +} +#else +__ai int64x1_t vget_high_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vget_high_s16(int16x8_t __p0) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#else +__ai int16x4_t vget_high_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vget_high_s16(int16x8_t __p0) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vget_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vget_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vgetq_lane_f32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vgetq_lane_f32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vgetq_lane_f32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vgetq_lane_f16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vgetq_lane_f16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vgetq_lane_f16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vget_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vget_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vget_lane_i32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vget_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vget_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vget_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vget_lane_f32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vget_lane_f32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vget_lane_f32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vget_lane_f16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vget_lane_f16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vget_lane_f16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vget_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vget_lane_i32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vget_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vget_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vget_low_p8(poly8x16_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai poly8x8_t vget_low_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vget_low_p16(poly16x8_t __p0) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); + return __ret; +} +#else +__ai poly16x4_t vget_low_p16(poly16x8_t __p0) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vget_low_u8(uint8x16_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai uint8x8_t vget_low_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vget_low_u32(uint32x4_t __p0) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1); + return __ret; +} +#else +__ai uint32x2_t vget_low_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vget_low_u64(uint64x2_t __p0) { + uint64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0); + return __ret; +} +#else +__ai uint64x1_t vget_low_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vget_low_u16(uint16x8_t __p0) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); + return __ret; +} +#else +__ai uint16x4_t vget_low_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vget_low_s8(int8x16_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai int8x8_t vget_low_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vget_low_f32(float32x4_t __p0) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1); + return __ret; +} +#else +__ai float32x2_t vget_low_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vget_low_f16(float16x8_t __p0) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); + return __ret; +} +#else +__ai float16x4_t vget_low_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vget_low_s32(int32x4_t __p0) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1); + return __ret; +} +#else +__ai int32x2_t vget_low_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vget_low_s64(int64x2_t __p0) { + int64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0); + return __ret; +} +#else +__ai int64x1_t vget_low_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vget_low_s16(int16x8_t __p0) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); + return __ret; +} +#else +__ai int16x4_t vget_low_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vhaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vhaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vhaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vhaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vhaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vhaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vhaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vhaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vhaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vhaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vhaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vhaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vhadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vhadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vhadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vhadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vhadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vhadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vhadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vhadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vhadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vhadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vhadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vhadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vhsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vhsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vhsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vhsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vhsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vhsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vhsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vhsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vhsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vhsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vhsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vhsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vhsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vhsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vhsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vhsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vhsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vhsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vhsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vhsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vhsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vhsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p8(__p0) __extension__ ({ \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_v(__p0, 4); \ + __ret; \ +}) +#else +#define vld1_p8(__p0) __extension__ ({ \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_v(__p0, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p16(__p0) __extension__ ({ \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_v(__p0, 5); \ + __ret; \ +}) +#else +#define vld1_p16(__p0) __extension__ ({ \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_v(__p0, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p8(__p0) __extension__ ({ \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_v(__p0, 36); \ + __ret; \ +}) +#else +#define vld1q_p8(__p0) __extension__ ({ \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_v(__p0, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p16(__p0) __extension__ ({ \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_v(__p0, 37); \ + __ret; \ +}) +#else +#define vld1q_p16(__p0) __extension__ ({ \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_v(__p0, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u8(__p0) __extension__ ({ \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_v(__p0, 48); \ + __ret; \ +}) +#else +#define vld1q_u8(__p0) __extension__ ({ \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_v(__p0, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u32(__p0) __extension__ ({ \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_v(__p0, 50); \ + __ret; \ +}) +#else +#define vld1q_u32(__p0) __extension__ ({ \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_v(__p0, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u64(__p0) __extension__ ({ \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_v(__p0, 51); \ + __ret; \ +}) +#else +#define vld1q_u64(__p0) __extension__ ({ \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_v(__p0, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u16(__p0) __extension__ ({ \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_v(__p0, 49); \ + __ret; \ +}) +#else +#define vld1q_u16(__p0) __extension__ ({ \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_v(__p0, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s8(__p0) __extension__ ({ \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_v(__p0, 32); \ + __ret; \ +}) +#else +#define vld1q_s8(__p0) __extension__ ({ \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_v(__p0, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f32(__p0) __extension__ ({ \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_v(__p0, 41); \ + __ret; \ +}) +#else +#define vld1q_f32(__p0) __extension__ ({ \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_v(__p0, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f16(__p0) __extension__ ({ \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_v(__p0, 40); \ + __ret; \ +}) +#else +#define vld1q_f16(__p0) __extension__ ({ \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_v(__p0, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s32(__p0) __extension__ ({ \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_v(__p0, 34); \ + __ret; \ +}) +#else +#define vld1q_s32(__p0) __extension__ ({ \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_v(__p0, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s64(__p0) __extension__ ({ \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_v(__p0, 35); \ + __ret; \ +}) +#else +#define vld1q_s64(__p0) __extension__ ({ \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_v(__p0, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s16(__p0) __extension__ ({ \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_v(__p0, 33); \ + __ret; \ +}) +#else +#define vld1q_s16(__p0) __extension__ ({ \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_v(__p0, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u8(__p0) __extension__ ({ \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_v(__p0, 16); \ + __ret; \ +}) +#else +#define vld1_u8(__p0) __extension__ ({ \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_v(__p0, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u32(__p0) __extension__ ({ \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_v(__p0, 18); \ + __ret; \ +}) +#else +#define vld1_u32(__p0) __extension__ ({ \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_v(__p0, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u64(__p0) __extension__ ({ \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_v(__p0, 19); \ + __ret; \ +}) +#else +#define vld1_u64(__p0) __extension__ ({ \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_v(__p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u16(__p0) __extension__ ({ \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_v(__p0, 17); \ + __ret; \ +}) +#else +#define vld1_u16(__p0) __extension__ ({ \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_v(__p0, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s8(__p0) __extension__ ({ \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_v(__p0, 0); \ + __ret; \ +}) +#else +#define vld1_s8(__p0) __extension__ ({ \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_v(__p0, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f32(__p0) __extension__ ({ \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_v(__p0, 9); \ + __ret; \ +}) +#else +#define vld1_f32(__p0) __extension__ ({ \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_v(__p0, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f16(__p0) __extension__ ({ \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_v(__p0, 8); \ + __ret; \ +}) +#else +#define vld1_f16(__p0) __extension__ ({ \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_v(__p0, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s32(__p0) __extension__ ({ \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_v(__p0, 2); \ + __ret; \ +}) +#else +#define vld1_s32(__p0) __extension__ ({ \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_v(__p0, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s64(__p0) __extension__ ({ \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_v(__p0, 3); \ + __ret; \ +}) +#else +#define vld1_s64(__p0) __extension__ ({ \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_v(__p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s16(__p0) __extension__ ({ \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_v(__p0, 1); \ + __ret; \ +}) +#else +#define vld1_s16(__p0) __extension__ ({ \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_v(__p0, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_p8(__p0) __extension__ ({ \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_dup_v(__p0, 4); \ + __ret; \ +}) +#else +#define vld1_dup_p8(__p0) __extension__ ({ \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_dup_v(__p0, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_p16(__p0) __extension__ ({ \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_dup_v(__p0, 5); \ + __ret; \ +}) +#else +#define vld1_dup_p16(__p0) __extension__ ({ \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_dup_v(__p0, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_p8(__p0) __extension__ ({ \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_dup_v(__p0, 36); \ + __ret; \ +}) +#else +#define vld1q_dup_p8(__p0) __extension__ ({ \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_dup_v(__p0, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_p16(__p0) __extension__ ({ \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_dup_v(__p0, 37); \ + __ret; \ +}) +#else +#define vld1q_dup_p16(__p0) __extension__ ({ \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_dup_v(__p0, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_u8(__p0) __extension__ ({ \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_dup_v(__p0, 48); \ + __ret; \ +}) +#else +#define vld1q_dup_u8(__p0) __extension__ ({ \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_dup_v(__p0, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_u32(__p0) __extension__ ({ \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_dup_v(__p0, 50); \ + __ret; \ +}) +#else +#define vld1q_dup_u32(__p0) __extension__ ({ \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_dup_v(__p0, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_u64(__p0) __extension__ ({ \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_dup_v(__p0, 51); \ + __ret; \ +}) +#else +#define vld1q_dup_u64(__p0) __extension__ ({ \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_dup_v(__p0, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_u16(__p0) __extension__ ({ \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_dup_v(__p0, 49); \ + __ret; \ +}) +#else +#define vld1q_dup_u16(__p0) __extension__ ({ \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_dup_v(__p0, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_s8(__p0) __extension__ ({ \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_dup_v(__p0, 32); \ + __ret; \ +}) +#else +#define vld1q_dup_s8(__p0) __extension__ ({ \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_dup_v(__p0, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_f32(__p0) __extension__ ({ \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_dup_v(__p0, 41); \ + __ret; \ +}) +#else +#define vld1q_dup_f32(__p0) __extension__ ({ \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_dup_v(__p0, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_f16(__p0) __extension__ ({ \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_dup_v(__p0, 40); \ + __ret; \ +}) +#else +#define vld1q_dup_f16(__p0) __extension__ ({ \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_dup_v(__p0, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_s32(__p0) __extension__ ({ \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_dup_v(__p0, 34); \ + __ret; \ +}) +#else +#define vld1q_dup_s32(__p0) __extension__ ({ \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_dup_v(__p0, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_s64(__p0) __extension__ ({ \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_dup_v(__p0, 35); \ + __ret; \ +}) +#else +#define vld1q_dup_s64(__p0) __extension__ ({ \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_dup_v(__p0, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_s16(__p0) __extension__ ({ \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_dup_v(__p0, 33); \ + __ret; \ +}) +#else +#define vld1q_dup_s16(__p0) __extension__ ({ \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_dup_v(__p0, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_u8(__p0) __extension__ ({ \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_dup_v(__p0, 16); \ + __ret; \ +}) +#else +#define vld1_dup_u8(__p0) __extension__ ({ \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_dup_v(__p0, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_u32(__p0) __extension__ ({ \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_dup_v(__p0, 18); \ + __ret; \ +}) +#else +#define vld1_dup_u32(__p0) __extension__ ({ \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_dup_v(__p0, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_u64(__p0) __extension__ ({ \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_dup_v(__p0, 19); \ + __ret; \ +}) +#else +#define vld1_dup_u64(__p0) __extension__ ({ \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_dup_v(__p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_u16(__p0) __extension__ ({ \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_dup_v(__p0, 17); \ + __ret; \ +}) +#else +#define vld1_dup_u16(__p0) __extension__ ({ \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_dup_v(__p0, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_s8(__p0) __extension__ ({ \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_dup_v(__p0, 0); \ + __ret; \ +}) +#else +#define vld1_dup_s8(__p0) __extension__ ({ \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_dup_v(__p0, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_f32(__p0) __extension__ ({ \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_dup_v(__p0, 9); \ + __ret; \ +}) +#else +#define vld1_dup_f32(__p0) __extension__ ({ \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_dup_v(__p0, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_f16(__p0) __extension__ ({ \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_dup_v(__p0, 8); \ + __ret; \ +}) +#else +#define vld1_dup_f16(__p0) __extension__ ({ \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_dup_v(__p0, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_s32(__p0) __extension__ ({ \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_dup_v(__p0, 2); \ + __ret; \ +}) +#else +#define vld1_dup_s32(__p0) __extension__ ({ \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_dup_v(__p0, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_s64(__p0) __extension__ ({ \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_dup_v(__p0, 3); \ + __ret; \ +}) +#else +#define vld1_dup_s64(__p0) __extension__ ({ \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_dup_v(__p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_s16(__p0) __extension__ ({ \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_dup_v(__p0, 1); \ + __ret; \ +}) +#else +#define vld1_dup_s16(__p0) __extension__ ({ \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_dup_v(__p0, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 4); \ + __ret; \ +}) +#else +#define vld1_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 5); \ + __ret; \ +}) +#else +#define vld1_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 36); \ + __ret; \ +}) +#else +#define vld1q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 37); \ + __ret; \ +}) +#else +#define vld1q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vld1q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vld1q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vld1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vld1q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vld1q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 41); \ + __ret; \ +}) +#else +#define vld1q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 40); \ + __ret; \ +}) +#else +#define vld1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vld1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vld1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vld1q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vld1_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vld1_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vld1_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vld1_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vld1_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 9); \ + __ret; \ +}) +#else +#define vld1_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 8); \ + __ret; \ +}) +#else +#define vld1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vld1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vld1_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vld1_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_p8(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld2_p8(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_p16(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld2_p16(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_p8(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld2q_p8(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_p16(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld2q_p16(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_u8(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld2q_u8(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_u32(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld2q_u32(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_u16(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld2q_u16(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_s8(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld2q_s8(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_f32(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld2q_f32(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_f16(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld2q_f16(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_s32(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld2q_s32(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_s16(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld2q_s16(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_u8(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld2_u8(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_u32(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld2_u32(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_u64(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld2_u64(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_u16(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld2_u16(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_s8(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld2_s8(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_f32(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld2_f32(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_f16(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld2_f16(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_s32(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld2_s32(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_s64(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld2_s64(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_s16(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld2_s16(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_p8(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld2_dup_p8(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_p16(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld2_dup_p16(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_u8(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld2_dup_u8(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_u32(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld2_dup_u32(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_u64(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld2_dup_u64(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_u16(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld2_dup_u16(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_s8(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld2_dup_s8(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_f32(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld2_dup_f32(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_f16(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld2_dup_f16(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_s32(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld2_dup_s32(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_s64(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld2_dup_s64(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_s16(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld2_dup_s16(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 4); \ + __ret; \ +}) +#else +#define vld2_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + poly8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 5); \ + __ret; \ +}) +#else +#define vld2_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + poly16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 37); \ + __ret; \ +}) +#else +#define vld2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + poly16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 50); \ + __ret; \ +}) +#else +#define vld2q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + uint32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 49); \ + __ret; \ +}) +#else +#define vld2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + uint16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 41); \ + __ret; \ +}) +#else +#define vld2q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + float32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 40); \ + __ret; \ +}) +#else +#define vld2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + float16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 34); \ + __ret; \ +}) +#else +#define vld2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + int32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 33); \ + __ret; \ +}) +#else +#define vld2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + int16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 16); \ + __ret; \ +}) +#else +#define vld2_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + uint8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 18); \ + __ret; \ +}) +#else +#define vld2_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + uint32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 17); \ + __ret; \ +}) +#else +#define vld2_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + uint16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 0); \ + __ret; \ +}) +#else +#define vld2_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + int8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 9); \ + __ret; \ +}) +#else +#define vld2_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + float32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 8); \ + __ret; \ +}) +#else +#define vld2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + float16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 2); \ + __ret; \ +}) +#else +#define vld2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + int32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 1); \ + __ret; \ +}) +#else +#define vld2_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + int16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_p8(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld3_p8(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_p16(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld3_p16(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_p8(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld3q_p8(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_p16(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld3q_p16(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_u8(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld3q_u8(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_u32(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld3q_u32(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_u16(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld3q_u16(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_s8(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld3q_s8(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_f32(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld3q_f32(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_f16(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld3q_f16(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_s32(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld3q_s32(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_s16(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld3q_s16(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_u8(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld3_u8(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_u32(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld3_u32(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_u64(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld3_u64(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_u16(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld3_u16(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_s8(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld3_s8(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_f32(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld3_f32(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_f16(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld3_f16(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_s32(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld3_s32(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_s64(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld3_s64(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_s16(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld3_s16(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_p8(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld3_dup_p8(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_p16(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld3_dup_p16(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_u8(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld3_dup_u8(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_u32(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld3_dup_u32(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_u64(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld3_dup_u64(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_u16(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld3_dup_u16(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_s8(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld3_dup_s8(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_f32(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld3_dup_f32(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_f16(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld3_dup_f16(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_s32(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld3_dup_s32(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_s64(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld3_dup_s64(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_s16(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld3_dup_s16(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 4); \ + __ret; \ +}) +#else +#define vld3_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + poly8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 5); \ + __ret; \ +}) +#else +#define vld3_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + poly16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 37); \ + __ret; \ +}) +#else +#define vld3q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + poly16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 50); \ + __ret; \ +}) +#else +#define vld3q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + uint32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 49); \ + __ret; \ +}) +#else +#define vld3q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + uint16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 41); \ + __ret; \ +}) +#else +#define vld3q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + float32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 40); \ + __ret; \ +}) +#else +#define vld3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + float16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 34); \ + __ret; \ +}) +#else +#define vld3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + int32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 33); \ + __ret; \ +}) +#else +#define vld3q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + int16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 16); \ + __ret; \ +}) +#else +#define vld3_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + uint8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 18); \ + __ret; \ +}) +#else +#define vld3_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + uint32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 17); \ + __ret; \ +}) +#else +#define vld3_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + uint16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 0); \ + __ret; \ +}) +#else +#define vld3_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + int8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 9); \ + __ret; \ +}) +#else +#define vld3_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + float32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 8); \ + __ret; \ +}) +#else +#define vld3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + float16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 2); \ + __ret; \ +}) +#else +#define vld3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + int32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 1); \ + __ret; \ +}) +#else +#define vld3_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + int16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_p8(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld4_p8(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_p16(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld4_p16(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_p8(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld4q_p8(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_p16(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld4q_p16(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_u8(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld4q_u8(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_u32(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld4q_u32(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_u16(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld4q_u16(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_s8(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld4q_s8(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_f32(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld4q_f32(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_f16(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld4q_f16(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_s32(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld4q_s32(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_s16(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld4q_s16(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_u8(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld4_u8(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_u32(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld4_u32(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_u64(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld4_u64(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_u16(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld4_u16(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_s8(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld4_s8(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_f32(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld4_f32(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_f16(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld4_f16(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_s32(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld4_s32(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_s64(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld4_s64(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_s16(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld4_s16(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_p8(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld4_dup_p8(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_p16(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld4_dup_p16(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_u8(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld4_dup_u8(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_u32(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld4_dup_u32(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_u64(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld4_dup_u64(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_u16(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld4_dup_u16(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_s8(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld4_dup_s8(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_f32(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld4_dup_f32(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_f16(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld4_dup_f16(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_s32(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld4_dup_s32(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_s64(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld4_dup_s64(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_s16(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld4_dup_s16(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 4); \ + __ret; \ +}) +#else +#define vld4_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + poly8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 5); \ + __ret; \ +}) +#else +#define vld4_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + poly16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 37); \ + __ret; \ +}) +#else +#define vld4q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + poly16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 50); \ + __ret; \ +}) +#else +#define vld4q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + uint32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 49); \ + __ret; \ +}) +#else +#define vld4q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + uint16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 41); \ + __ret; \ +}) +#else +#define vld4q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + float32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 40); \ + __ret; \ +}) +#else +#define vld4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + float16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 34); \ + __ret; \ +}) +#else +#define vld4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + int32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 33); \ + __ret; \ +}) +#else +#define vld4q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + int16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 16); \ + __ret; \ +}) +#else +#define vld4_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + uint8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 18); \ + __ret; \ +}) +#else +#define vld4_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + uint32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 17); \ + __ret; \ +}) +#else +#define vld4_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + uint16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 0); \ + __ret; \ +}) +#else +#define vld4_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + int8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 9); \ + __ret; \ +}) +#else +#define vld4_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + float32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 8); \ + __ret; \ +}) +#else +#define vld4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + float16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 2); \ + __ret; \ +}) +#else +#define vld4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + int32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 1); \ + __ret; \ +}) +#else +#define vld4_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + int16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmaxq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vmaxq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmaxq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vmaxq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmaxq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vmaxq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmaxq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vmaxq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmaxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vmaxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmaxq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vmaxq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmaxq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vmaxq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmax_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vmax_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmax_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vmax_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmax_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vmax_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmax_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vmax_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmax_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vmax_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmax_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vmax_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmax_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vmax_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vminq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vminq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vminq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vminq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vminq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vminq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vminq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vminq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vminq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vminq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vminq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vminq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vminq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vminq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmin_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vmin_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmin_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vmin_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmin_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vmin_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmin_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vmin_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmin_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vmin_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmin_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vmin_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmin_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vmin_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmlaq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint8x16_t vmlaq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlaq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint32x4_t vmlaq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlaq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint16x8_t vmlaq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmlaq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int8x16_t vmlaq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmlaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai float32x4_t vmlaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlaq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int32x4_t vmlaq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlaq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int16x8_t vmlaq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmla_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint8x8_t vmla_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmla_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint32x2_t vmla_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x2_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmla_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint16x4_t vmla_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmla_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int8x8_t vmla_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmla_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai float32x2_t vmla_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float32x2_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmla_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int32x2_t vmla_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmla_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int16x4_t vmla_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x8_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlaq_n_u32(uint32x4_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint32x4_t __ret; + __ret = __p0 + __p1 * (uint32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint32x4_t vmlaq_n_u32(uint32x4_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __rev1 * (uint32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlaq_n_u16(uint16x8_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint16x8_t __ret; + __ret = __p0 + __p1 * (uint16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint16x8_t vmlaq_n_u16(uint16x8_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __rev1 * (uint16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmlaq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __ret; + __ret = __p0 + __p1 * (float32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai float32x4_t vmlaq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 + __rev1 * (float32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlaq_n_s32(int32x4_t __p0, int32x4_t __p1, int32_t __p2) { + int32x4_t __ret; + __ret = __p0 + __p1 * (int32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int32x4_t vmlaq_n_s32(int32x4_t __p0, int32x4_t __p1, int32_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __rev1 * (int32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlaq_n_s16(int16x8_t __p0, int16x8_t __p1, int16_t __p2) { + int16x8_t __ret; + __ret = __p0 + __p1 * (int16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int16x8_t vmlaq_n_s16(int16x8_t __p0, int16x8_t __p1, int16_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __rev1 * (int16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmla_n_u32(uint32x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint32x2_t __ret; + __ret = __p0 + __p1 * (uint32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai uint32x2_t vmla_n_u32(uint32x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 + __rev1 * (uint32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmla_n_u16(uint16x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint16x4_t __ret; + __ret = __p0 + __p1 * (uint16x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint16x4_t vmla_n_u16(uint16x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 + __rev1 * (uint16x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmla_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __ret; + __ret = __p0 + __p1 * (float32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai float32x2_t vmla_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 + __rev1 * (float32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmla_n_s32(int32x2_t __p0, int32x2_t __p1, int32_t __p2) { + int32x2_t __ret; + __ret = __p0 + __p1 * (int32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai int32x2_t vmla_n_s32(int32x2_t __p0, int32x2_t __p1, int32_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 + __rev1 * (int32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmla_n_s16(int16x4_t __p0, int16x4_t __p1, int16_t __p2) { + int16x4_t __ret; + __ret = __p0 + __p1 * (int16x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int16x4_t vmla_n_s16(int16x4_t __p0, int16x4_t __p1, int16_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 + __rev1 * (int16x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmlsq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint8x16_t vmlsq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint32x4_t vmlsq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlsq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint16x8_t vmlsq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmlsq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int8x16_t vmlsq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmlsq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai float32x4_t vmlsq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int32x4_t vmlsq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlsq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int16x8_t vmlsq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmls_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint8x8_t vmls_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmls_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint32x2_t vmls_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x2_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmls_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint16x4_t vmls_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmls_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int8x8_t vmls_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmls_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai float32x2_t vmls_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float32x2_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmls_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int32x2_t vmls_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmls_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int16x4_t vmls_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x8_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsq_n_u32(uint32x4_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint32x4_t __ret; + __ret = __p0 - __p1 * (uint32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint32x4_t vmlsq_n_u32(uint32x4_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __rev1 * (uint32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlsq_n_u16(uint16x8_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint16x8_t __ret; + __ret = __p0 - __p1 * (uint16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint16x8_t vmlsq_n_u16(uint16x8_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __rev1 * (uint16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmlsq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __ret; + __ret = __p0 - __p1 * (float32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai float32x4_t vmlsq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 - __rev1 * (float32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsq_n_s32(int32x4_t __p0, int32x4_t __p1, int32_t __p2) { + int32x4_t __ret; + __ret = __p0 - __p1 * (int32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int32x4_t vmlsq_n_s32(int32x4_t __p0, int32x4_t __p1, int32_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __rev1 * (int32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlsq_n_s16(int16x8_t __p0, int16x8_t __p1, int16_t __p2) { + int16x8_t __ret; + __ret = __p0 - __p1 * (int16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int16x8_t vmlsq_n_s16(int16x8_t __p0, int16x8_t __p1, int16_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __rev1 * (int16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmls_n_u32(uint32x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint32x2_t __ret; + __ret = __p0 - __p1 * (uint32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai uint32x2_t vmls_n_u32(uint32x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 - __rev1 * (uint32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmls_n_u16(uint16x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint16x4_t __ret; + __ret = __p0 - __p1 * (uint16x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint16x4_t vmls_n_u16(uint16x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 - __rev1 * (uint16x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmls_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __ret; + __ret = __p0 - __p1 * (float32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai float32x2_t vmls_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 - __rev1 * (float32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmls_n_s32(int32x2_t __p0, int32x2_t __p1, int32_t __p2) { + int32x2_t __ret; + __ret = __p0 - __p1 * (int32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai int32x2_t vmls_n_s32(int32x2_t __p0, int32x2_t __p1, int32_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 - __rev1 * (int32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmls_n_s16(int16x4_t __p0, int16x4_t __p1, int16_t __p2) { + int16x4_t __ret; + __ret = __p0 - __p1 * (int16x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int16x4_t vmls_n_s16(int16x4_t __p0, int16x4_t __p1, int16_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 - __rev1 * (int16x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vmov_n_p8(poly8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly8x8_t vmov_n_p8(poly8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vmov_n_p16(poly16_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly16x4_t vmov_n_p16(poly16_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vmovq_n_p8(poly8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly8x16_t vmovq_n_p8(poly8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vmovq_n_p16(poly16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly16x8_t vmovq_n_p16(poly16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmovq_n_u8(uint8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint8x16_t vmovq_n_u8(uint8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmovq_n_u32(uint32_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint32x4_t vmovq_n_u32(uint32_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmovq_n_u64(uint64_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai uint64x2_t vmovq_n_u64(uint64_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmovq_n_u16(uint16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint16x8_t vmovq_n_u16(uint16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmovq_n_s8(int8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int8x16_t vmovq_n_s8(int8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmovq_n_f32(float32_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai float32x4_t vmovq_n_f32(float32_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmovq_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) {__s0, __s0, __s0, __s0, __s0, __s0, __s0, __s0}; \ + __ret; \ +}) +#else +#define vmovq_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) {__s0, __s0, __s0, __s0, __s0, __s0, __s0, __s0}; \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmovq_n_s32(int32_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int32x4_t vmovq_n_s32(int32_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmovq_n_s64(int64_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai int64x2_t vmovq_n_s64(int64_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmovq_n_s16(int16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int16x8_t vmovq_n_s16(int16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmov_n_u8(uint8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint8x8_t vmov_n_u8(uint8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmov_n_u32(uint32_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai uint32x2_t vmov_n_u32(uint32_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vmov_n_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) {__p0}; + return __ret; +} +#else +__ai uint64x1_t vmov_n_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmov_n_u16(uint16_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint16x4_t vmov_n_u16(uint16_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmov_n_s8(int8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int8x8_t vmov_n_s8(int8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmov_n_f32(float32_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai float32x2_t vmov_n_f32(float32_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmov_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) {__s0, __s0, __s0, __s0}; \ + __ret; \ +}) +#else +#define vmov_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) {__s0, __s0, __s0, __s0}; \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmov_n_s32(int32_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai int32x2_t vmov_n_s32(int32_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vmov_n_s64(int64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) {__p0}; + return __ret; +} +#else +__ai int64x1_t vmov_n_s64(int64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmov_n_s16(int16_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int16x4_t vmov_n_s16(int16_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmovl_u8(uint8x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vmovl_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vmovl_u8(uint8x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 49); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmovl_u32(uint32x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vmovl_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmovl_u32(uint32x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 51); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmovl_u16(uint16x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vmovl_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmovl_u16(uint16x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 50); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmovl_s8(int8x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vmovl_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vmovl_s8(int8x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmovl_s32(int32x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vmovl_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmovl_s32(int32x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmovl_s16(int16x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vmovl_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmovl_s16(int16x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmovn_u32(uint32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vmovn_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vmovn_u32(uint32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmovn_u64(uint64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vmovn_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vmovn_u64(uint64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmovn_u16(uint16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vmovn_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vmovn_u16(uint16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmovn_s32(int32x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vmovn_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vmovn_s32(int32x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmovn_s64(int64x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vmovn_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vmovn_s64(int64x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmovn_s16(int16x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vmovn_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vmovn_s16(int16x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmulq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint8x16_t vmulq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmulq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint32x4_t vmulq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmulq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint16x8_t vmulq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmulq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int8x16_t vmulq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmulq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float32x4_t vmulq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmulq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int32x4_t vmulq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmulq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int16x8_t vmulq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmul_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint8x8_t vmul_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmul_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint32x2_t vmul_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmul_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint16x4_t vmul_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmul_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int8x8_t vmul_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmul_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float32x2_t vmul_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmul_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int32x2_t vmul_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmul_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int16x4_t vmul_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vmul_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vmul_v((int8x8_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vmul_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vmul_v((int8x8_t)__rev0, (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vmulq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vmulq_v((int8x16_t)__p0, (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16_t vmulq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vmulq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmulq_n_u32(uint32x4_t __p0, uint32_t __p1) { + uint32x4_t __ret; + __ret = __p0 * (uint32x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai uint32x4_t vmulq_n_u32(uint32x4_t __p0, uint32_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 * (uint32x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmulq_n_u16(uint16x8_t __p0, uint16_t __p1) { + uint16x8_t __ret; + __ret = __p0 * (uint16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai uint16x8_t vmulq_n_u16(uint16x8_t __p0, uint16_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 * (uint16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmulq_n_f32(float32x4_t __p0, float32_t __p1) { + float32x4_t __ret; + __ret = __p0 * (float32x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai float32x4_t vmulq_n_f32(float32x4_t __p0, float32_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 * (float32x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmulq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __ret; + __ret = __p0 * (int32x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai int32x4_t vmulq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 * (int32x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmulq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __ret; + __ret = __p0 * (int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai int16x8_t vmulq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 * (int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmul_n_u32(uint32x2_t __p0, uint32_t __p1) { + uint32x2_t __ret; + __ret = __p0 * (uint32x2_t) {__p1, __p1}; + return __ret; +} +#else +__ai uint32x2_t vmul_n_u32(uint32x2_t __p0, uint32_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = __rev0 * (uint32x2_t) {__p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmul_n_u16(uint16x4_t __p0, uint16_t __p1) { + uint16x4_t __ret; + __ret = __p0 * (uint16x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai uint16x4_t vmul_n_u16(uint16x4_t __p0, uint16_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 * (uint16x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmul_n_f32(float32x2_t __p0, float32_t __p1) { + float32x2_t __ret; + __ret = __p0 * (float32x2_t) {__p1, __p1}; + return __ret; +} +#else +__ai float32x2_t vmul_n_f32(float32x2_t __p0, float32_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = __rev0 * (float32x2_t) {__p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmul_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __ret; + __ret = __p0 * (int32x2_t) {__p1, __p1}; + return __ret; +} +#else +__ai int32x2_t vmul_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = __rev0 * (int32x2_t) {__p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmul_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __ret; + __ret = __p0 * (int16x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai int16x4_t vmul_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 * (int16x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vmull_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly16x8_t __ret; + __ret = (poly16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 37); + return __ret; +} +#else +__ai poly16x8_t vmull_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = (poly16x8_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 37); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai poly16x8_t __noswap_vmull_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly16x8_t __ret; + __ret = (poly16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 37); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmull_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vmull_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vmull_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 49); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmull_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vmull_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmull_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 51); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmull_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vmull_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmull_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 50); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmull_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vmull_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vmull_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmull_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vmull_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmull_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmull_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vmull_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmull_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = vmull_u32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = __noswap_vmull_u32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = vmull_u16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __noswap_vmull_u16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vmull_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vmull_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vmull_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vmull_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmull_n_u32(uint32x2_t __p0, uint32_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(uint32x2_t) {__p1, __p1}, 51); + return __ret; +} +#else +__ai uint64x2_t vmull_n_u32(uint32x2_t __p0, uint32_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)(uint32x2_t) {__p1, __p1}, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmull_n_u32(uint32x2_t __p0, uint32_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(uint32x2_t) {__p1, __p1}, 51); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmull_n_u16(uint16x4_t __p0, uint16_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(uint16x4_t) {__p1, __p1, __p1, __p1}, 50); + return __ret; +} +#else +__ai uint32x4_t vmull_n_u16(uint16x4_t __p0, uint16_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)(uint16x4_t) {__p1, __p1, __p1, __p1}, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmull_n_u16(uint16x4_t __p0, uint16_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(uint16x4_t) {__p1, __p1, __p1, __p1}, 50); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmull_n_s32(int32x2_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + return __ret; +} +#else +__ai int64x2_t vmull_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmull_n_s32(int32x2_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmull_n_s16(int16x4_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#else +__ai int32x4_t vmull_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmull_n_s16(int16x4_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vmvn_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai poly8x8_t vmvn_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vmvnq_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai poly8x16_t vmvnq_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmvnq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint8x16_t vmvnq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmvnq_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint32x4_t vmvnq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmvnq_u16(uint16x8_t __p0) { + uint16x8_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint16x8_t vmvnq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmvnq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int8x16_t vmvnq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmvnq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int32x4_t vmvnq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmvnq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int16x8_t vmvnq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmvn_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint8x8_t vmvn_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmvn_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint32x2_t vmvn_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmvn_u16(uint16x4_t __p0) { + uint16x4_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint16x4_t vmvn_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmvn_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int8x8_t vmvn_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmvn_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int32x2_t vmvn_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmvn_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int16x4_t vmvn_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vnegq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int8x16_t vnegq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vnegq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float32x4_t vnegq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vnegq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int32x4_t vnegq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vnegq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int16x8_t vnegq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vneg_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int8x8_t vneg_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vneg_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float32x2_t vneg_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vneg_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int32x2_t vneg_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vneg_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int16x4_t vneg_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vornq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint8x16_t vornq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vornq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint32x4_t vornq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vornq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint64x2_t vornq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vornq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint16x8_t vornq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vornq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int8x16_t vornq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vornq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int32x4_t vornq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vornq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int64x2_t vornq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vornq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int16x8_t vornq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vorn_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint8x8_t vorn_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vorn_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint32x2_t vorn_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vorn_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint64x1_t vorn_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vorn_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint16x4_t vorn_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vorn_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int8x8_t vorn_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vorn_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int32x2_t vorn_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vorn_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int64x1_t vorn_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vorn_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int16x4_t vorn_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vorrq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint8x16_t vorrq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vorrq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint32x4_t vorrq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vorrq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint64x2_t vorrq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vorrq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint16x8_t vorrq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vorrq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int8x16_t vorrq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vorrq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int32x4_t vorrq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vorrq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int64x2_t vorrq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vorrq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int16x8_t vorrq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vorr_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint8x8_t vorr_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vorr_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint32x2_t vorr_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vorr_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint64x1_t vorr_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vorr_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint16x4_t vorr_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vorr_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int8x8_t vorr_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vorr_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int32x2_t vorr_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vorr_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int64x1_t vorr_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vorr_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int16x4_t vorr_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vpadalq_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vpadalq_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vpadalq_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vpadalq_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vpadalq_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vpadalq_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vpadalq_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vpadalq_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vpadalq_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vpadalq_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vpadalq_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vpadalq_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vpadal_u8(uint16x4_t __p0, uint8x8_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vpadal_u8(uint16x4_t __p0, uint8x8_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpadal_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vpadal_u32(uint64x1_t __p0, uint32x2_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vpadal_u32(uint64x1_t __p0, uint32x2_t __p1) { + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__rev1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vpadal_u16(uint32x2_t __p0, uint16x4_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vpadal_u16(uint32x2_t __p0, uint16x4_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpadal_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vpadal_s8(int16x4_t __p0, int8x8_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vpadal_s8(int16x4_t __p0, int8x8_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpadal_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vpadal_s32(int64x1_t __p0, int32x2_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vpadal_s32(int64x1_t __p0, int32x2_t __p1) { + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__rev1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vpadal_s16(int32x2_t __p0, int16x4_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vpadal_s16(int32x2_t __p0, int16x4_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpadal_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vpadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vpadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vpadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vpadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vpadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vpadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vpadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vpadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vpadd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vpadd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vpadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vpadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vpadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vpadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vpaddlq_u8(uint8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vpaddlq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vpaddlq_u32(uint32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vpaddlq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vpaddlq_u16(uint16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vpaddlq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vpaddlq_s8(int8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vpaddlq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vpaddlq_s32(int32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vpaddlq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vpaddlq_s16(int16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vpaddlq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vpaddl_u8(uint8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vpaddl_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vpaddl_u32(uint32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vpaddl_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vpaddl_u16(uint16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vpaddl_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vpaddl_s8(int8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vpaddl_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vpaddl_s32(int32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vpaddl_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vpaddl_s16(int16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vpaddl_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vpmax_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vpmax_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vpmax_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vpmax_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vpmax_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vpmax_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vpmax_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vpmax_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vpmax_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vpmax_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vpmax_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vpmax_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vpmax_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vpmax_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vpmin_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vpmin_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vpmin_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vpmin_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vpmin_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vpmin_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vpmin_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vpmin_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vpmin_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vpmin_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vpmin_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vpmin_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vpmin_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vpmin_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqabsq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqabsq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vqabsq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqabsq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqabsq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqabsq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vqabsq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqabsq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqabsq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqabsq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vqabsq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqabsq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqabs_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqabs_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vqabs_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqabs_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqabs_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqabs_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vqabs_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqabs_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqabs_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqabs_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vqabs_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqabs_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vqaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vqaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vqadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vqadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vqadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vqadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vqadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vqadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vqadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 35); + return __ret; +} +#else +__ai int64x2_t vqdmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 34); + return __ret; +} +#else +__ai int32x4_t vqdmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlal_s32(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlal_s32(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlal_s16(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlal_s16(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + return __ret; +} +#else +__ai int64x2_t vqdmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + return __ret; +} +#else +__ai int32x4_t vqdmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 35); + return __ret; +} +#else +__ai int64x2_t vqdmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 34); + return __ret; +} +#else +__ai int32x4_t vqdmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlsl_s32(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlsl_s32(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlsl_s16(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlsl_s16(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + return __ret; +} +#else +__ai int64x2_t vqdmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + return __ret; +} +#else +__ai int32x4_t vqdmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmulhq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqdmulhq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vqdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqdmulh_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vqdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqdmulh_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vqdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmulhq_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulhq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmulhq_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = vqdmulhq_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulhq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqdmulhq_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulh_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = vqdmulh_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulh_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqdmulh_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulh_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = vqdmulh_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulh_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqdmulh_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmulhq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)(int32x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#else +__ai int32x4_t vqdmulhq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmulhq_v((int8x16_t)__rev0, (int8x16_t)(int32x4_t) {__p1, __p1, __p1, __p1}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqdmulhq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)(int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}, 33); + return __ret; +} +#else +__ai int16x8_t vqdmulhq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqdmulhq_v((int8x16_t)__rev0, (int8x16_t)(int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqdmulh_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 2); + return __ret; +} +#else +__ai int32x2_t vqdmulh_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqdmulh_v((int8x8_t)__rev0, (int8x8_t)(int32x2_t) {__p1, __p1}, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqdmulh_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 1); + return __ret; +} +#else +__ai int16x4_t vqdmulh_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqdmulh_v((int8x8_t)__rev0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmull_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vqdmull_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmull_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)__p1, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmull_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqdmull_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmull_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vqdmull_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmull_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmull_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmull_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmull_n_s32(int32x2_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + return __ret; +} +#else +__ai int64x2_t vqdmull_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__rev0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmull_n_s32(int32x2_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmull_n_s16(int16x4_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#else +__ai int32x4_t vqdmull_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__rev0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmull_n_s16(int16x4_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqmovn_u32(uint32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vqmovn_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vqmovn_u32(uint32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqmovn_u64(uint64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vqmovn_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vqmovn_u64(uint64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqmovn_u16(uint16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vqmovn_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vqmovn_u16(uint16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqmovn_s32(int32x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vqmovn_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vqmovn_s32(int32x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqmovn_s64(int64x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vqmovn_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vqmovn_s64(int64x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqmovn_s16(int16x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vqmovn_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vqmovn_s16(int16x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqmovun_s32(int32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vqmovun_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovun_v((int8x16_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vqmovun_s32(int32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqmovun_s64(int64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vqmovun_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovun_v((int8x16_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vqmovun_s64(int64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqmovun_s16(int16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vqmovun_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovun_v((int8x16_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vqmovun_s16(int16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqnegq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqnegq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vqnegq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqnegq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqnegq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqnegq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vqnegq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqnegq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqnegq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqnegq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vqnegq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqnegq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqneg_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqneg_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vqneg_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqneg_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqneg_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqneg_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vqneg_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqneg_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqneg_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqneg_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vqneg_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqneg_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqrdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqrdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqrdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqrdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqrdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vqrdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqrdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqrdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrdmulh_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vqrdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqrdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqrdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrdmulh_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vqrdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqrdmulhq_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulhq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqrdmulhq_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = vqrdmulhq_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulhq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqrdmulhq_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulh_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = vqrdmulh_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulh_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqrdmulh_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulh_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = vqrdmulh_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulh_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqrdmulh_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqrdmulhq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)(int32x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#else +__ai int32x4_t vqrdmulhq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__rev0, (int8x16_t)(int32x4_t) {__p1, __p1, __p1, __p1}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqrdmulhq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)(int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}, 33); + return __ret; +} +#else +__ai int16x8_t vqrdmulhq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__rev0, (int8x16_t)(int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqrdmulh_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 2); + return __ret; +} +#else +__ai int32x2_t vqrdmulh_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrdmulh_v((int8x8_t)__rev0, (int8x8_t)(int32x2_t) {__p1, __p1}, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqrdmulh_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 1); + return __ret; +} +#else +__ai int16x4_t vqrdmulh_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrdmulh_v((int8x8_t)__rev0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqrshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqrshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqrshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vqrshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vqrshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vqrshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqrshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vqrshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqrshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqrshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqrshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqrshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqrshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vqrshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqrshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqrshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqrshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqrshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqrshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vqrshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vqrshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vqrshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqrshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vqrshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqrshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqrshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqrshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqrshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqrshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vqrshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqrshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqrshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vqrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vqrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vqrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqrshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqrshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqrshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vqshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vqshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vqshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vqshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vqshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vqshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vqshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vqshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vqshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vqshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 48); \ + __ret; \ +}) +#else +#define vqshlq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vqshlq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vqshlq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vqshlq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 32); \ + __ret; \ +}) +#else +#define vqshlq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vqshlq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vqshlq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vqshlq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqshl_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqshl_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vqshl_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqshl_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vqshl_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vqshl_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vqshl_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vqshl_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshluq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vqshluq_n_v((int8x16_t)__s0, __p1, 48); \ + __ret; \ +}) +#else +#define vqshluq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vqshluq_n_v((int8x16_t)__rev0, __p1, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshluq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vqshluq_n_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vqshluq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vqshluq_n_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshluq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vqshluq_n_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vqshluq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vqshluq_n_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshluq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vqshluq_n_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vqshluq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vqshluq_n_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlu_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshlu_n_v((int8x8_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqshlu_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshlu_n_v((int8x8_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlu_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshlu_n_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqshlu_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshlu_n_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlu_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vqshlu_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vqshlu_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vqshlu_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlu_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshlu_n_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqshlu_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshlu_n_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vqshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vqshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vqshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrun_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrun_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrun_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vqsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vqsubq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vqsubq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vqsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqsubq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vqsubq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vqsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vqsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vqsub_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vqsub_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vqsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vqsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqsub_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vqsub_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vqsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vraddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vraddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vraddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vraddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vraddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vraddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vraddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vraddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vraddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vraddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vraddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vraddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vraddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vraddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vraddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vraddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vraddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vraddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrecpeq_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrecpeq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vrecpeq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrecpeq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrecpeq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrecpeq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrecpeq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrecpeq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrecpe_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrecpe_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vrecpe_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrecpe_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrecpe_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrecpe_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrecpe_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrecpe_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrecpsq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrecpsq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vrecpsq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrecpsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrecps_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrecps_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vrecps_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrecps_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vrev16_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai poly8x8_t vrev16_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vrev16q_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + return __ret; +} +#else +__ai poly8x16_t vrev16q_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrev16q_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + return __ret; +} +#else +__ai uint8x16_t vrev16q_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrev16q_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + return __ret; +} +#else +__ai int8x16_t vrev16q_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrev16_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai uint8x8_t vrev16_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrev16_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai int8x8_t vrev16_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vrev32_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai poly8x8_t vrev32_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vrev32_p16(poly16x4_t __p0) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai poly16x4_t vrev32_p16(poly16x4_t __p0) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vrev32q_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + return __ret; +} +#else +__ai poly8x16_t vrev32q_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vrev32q_p16(poly16x8_t __p0) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai poly16x8_t vrev32q_p16(poly16x8_t __p0) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrev32q_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + return __ret; +} +#else +__ai uint8x16_t vrev32q_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vrev32q_u16(uint16x8_t __p0) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai uint16x8_t vrev32q_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrev32q_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + return __ret; +} +#else +__ai int8x16_t vrev32q_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vrev32q_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai int16x8_t vrev32q_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrev32_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai uint8x8_t vrev32_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vrev32_u16(uint16x4_t __p0) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai uint16x4_t vrev32_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrev32_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai int8x8_t vrev32_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vrev32_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai int16x4_t vrev32_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vrev64_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#else +__ai poly8x8_t vrev64_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vrev64_p16(poly16x4_t __p0) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + return __ret; +} +#else +__ai poly16x4_t vrev64_p16(poly16x4_t __p0) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vrev64q_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + return __ret; +} +#else +__ai poly8x16_t vrev64q_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vrev64q_p16(poly16x8_t __p0) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai poly16x8_t vrev64q_p16(poly16x8_t __p0) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrev64q_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + return __ret; +} +#else +__ai uint8x16_t vrev64q_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrev64q_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai uint32x4_t vrev64q_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vrev64q_u16(uint16x8_t __p0) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai uint16x8_t vrev64q_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrev64q_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + return __ret; +} +#else +__ai int8x16_t vrev64q_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrev64q_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai float32x4_t vrev64q_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vrev64q_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai int32x4_t vrev64q_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vrev64q_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai int16x8_t vrev64q_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrev64_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#else +__ai uint8x8_t vrev64_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrev64_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0); + return __ret; +} +#else +__ai uint32x2_t vrev64_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vrev64_u16(uint16x4_t __p0) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + return __ret; +} +#else +__ai uint16x4_t vrev64_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrev64_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#else +__ai int8x8_t vrev64_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrev64_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0); + return __ret; +} +#else +__ai float32x2_t vrev64_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vrev64_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0); + return __ret; +} +#else +__ai int32x2_t vrev64_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vrev64_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + return __ret; +} +#else +__ai int16x4_t vrev64_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrhaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vrhaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrhaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vrhaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vrhaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vrhaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrhaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vrhaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vrhaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vrhaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vrhaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vrhaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrhadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vrhadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrhadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vrhadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vrhadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vrhadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrhadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vrhadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vrhadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vrhadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vrhadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vrhadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vrshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vrshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vrshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vrshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vrshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vrshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vrshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vrshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vrshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vrshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vrshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vrshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vrshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vrshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vrshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vrshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vrshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vrshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vrshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vrshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vrshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vrshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vrshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vrshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vrshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vrshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 48); \ + __ret; \ +}) +#else +#define vrshrq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vrshrq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vrshrq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vrshrq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 32); \ + __ret; \ +}) +#else +#define vrshrq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vrshrq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vrshrq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vrshrq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vrshr_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vrshr_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vrshr_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vrshr_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vrshr_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vrshr_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vrshr_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vrshr_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrsqrteq_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrsqrteq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vrsqrteq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrsqrteq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrsqrteq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrsqrteq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrsqrteq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrsqrteq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrsqrte_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrsqrte_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vrsqrte_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrsqrte_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrsqrte_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrsqrte_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrsqrte_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrsqrte_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrsqrtsq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vrsqrtsq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrsqrts_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrsqrts_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vrsqrts_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrsqrts_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vrsraq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vrsraq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vrsraq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vrsraq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vrsraq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vrsraq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vrsraq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vrsraq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vrsra_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vrsra_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vrsra_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vrsra_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vrsra_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vrsra_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vrsra_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vrsra_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vrsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vrsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vrsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vrsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vrsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vrsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vrsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vrsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vrsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vrsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vrsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vrsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vrsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vrsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vrsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vsetq_lane_f32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vsetq_lane_f32(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vsetq_lane_f32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vsetq_lane_f16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vsetq_lane_f16(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#define __noswap_vset_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vset_lane_f32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vset_lane_f32(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vset_lane_f32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vset_lane_f16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vset_lane_f16(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#define __noswap_vset_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 48); \ + __ret; \ +}) +#else +#define vshlq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vshlq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vshlq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vshlq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 32); \ + __ret; \ +}) +#else +#define vshlq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vshlq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vshlq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vshlq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vshl_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vshl_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vshl_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vshl_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vshl_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vshl_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vshl_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vshl_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vshll_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 49); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vshll_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 51); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vshll_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 50); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vshll_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 33); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vshll_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 35); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vshll_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 34); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 48); \ + __ret; \ +}) +#else +#define vshrq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vshrq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vshrq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vshrq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 32); \ + __ret; \ +}) +#else +#define vshrq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vshrq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vshrq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vshrq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vshr_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vshr_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vshr_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vshr_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vshr_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vshr_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vshr_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vshr_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 4); \ + __ret; \ +}) +#else +#define vsli_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 5); \ + __ret; \ +}) +#else +#define vsli_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 36); \ + __ret; \ +}) +#else +#define vsliq_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 37); \ + __ret; \ +}) +#else +#define vsliq_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vsliq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vsliq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vsliq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vsliq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vsliq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vsliq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vsliq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vsliq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vsli_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vsli_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vsli_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vsli_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vsli_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vsli_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vsli_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vsli_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vsraq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vsraq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vsraq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vsraq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vsraq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vsraq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vsraq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vsraq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vsra_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vsra_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vsra_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vsra_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vsra_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vsra_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vsra_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vsra_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 4); \ + __ret; \ +}) +#else +#define vsri_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 5); \ + __ret; \ +}) +#else +#define vsri_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 36); \ + __ret; \ +}) +#else +#define vsriq_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 37); \ + __ret; \ +}) +#else +#define vsriq_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vsriq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vsriq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vsriq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vsriq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vsriq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vsriq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vsriq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vsriq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vsri_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vsri_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vsri_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vsri_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vsri_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vsri_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vsri_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vsri_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 4); \ +}) +#else +#define vst1_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 5); \ +}) +#else +#define vst1_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 36); \ +}) +#else +#define vst1q_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 37); \ +}) +#else +#define vst1q_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 48); \ +}) +#else +#define vst1q_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 50); \ +}) +#else +#define vst1q_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 51); \ +}) +#else +#define vst1q_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 49); \ +}) +#else +#define vst1q_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 32); \ +}) +#else +#define vst1q_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 41); \ +}) +#else +#define vst1q_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 40); \ +}) +#else +#define vst1q_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 34); \ +}) +#else +#define vst1q_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 35); \ +}) +#else +#define vst1q_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 33); \ +}) +#else +#define vst1q_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 16); \ +}) +#else +#define vst1_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 18); \ +}) +#else +#define vst1_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 19); \ +}) +#else +#define vst1_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 17); \ +}) +#else +#define vst1_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 0); \ +}) +#else +#define vst1_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 9); \ +}) +#else +#define vst1_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 8); \ +}) +#else +#define vst1_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 2); \ +}) +#else +#define vst1_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 3); \ +}) +#else +#define vst1_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 1); \ +}) +#else +#define vst1_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 4); \ +}) +#else +#define vst1_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 5); \ +}) +#else +#define vst1_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 36); \ +}) +#else +#define vst1q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 37); \ +}) +#else +#define vst1q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 48); \ +}) +#else +#define vst1q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 50); \ +}) +#else +#define vst1q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 51); \ +}) +#else +#define vst1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 49); \ +}) +#else +#define vst1q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 32); \ +}) +#else +#define vst1q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 41); \ +}) +#else +#define vst1q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 40); \ +}) +#else +#define vst1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 34); \ +}) +#else +#define vst1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 35); \ +}) +#else +#define vst1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 33); \ +}) +#else +#define vst1q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 16); \ +}) +#else +#define vst1_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 18); \ +}) +#else +#define vst1_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 19); \ +}) +#else +#define vst1_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 17); \ +}) +#else +#define vst1_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 0); \ +}) +#else +#define vst1_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 9); \ +}) +#else +#define vst1_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 8); \ +}) +#else +#define vst1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 2); \ +}) +#else +#define vst1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 3); \ +}) +#else +#define vst1_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 1); \ +}) +#else +#define vst1_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_p8(__p0, __p1) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 4); \ +}) +#else +#define vst2_p8(__p0, __p1) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + poly8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_p16(__p0, __p1) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 5); \ +}) +#else +#define vst2_p16(__p0, __p1) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + poly16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 36); \ +}) +#else +#define vst2q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + poly8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 37); \ +}) +#else +#define vst2q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + poly16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 48); \ +}) +#else +#define vst2q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + uint8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 50); \ +}) +#else +#define vst2q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + uint32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 49); \ +}) +#else +#define vst2q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + uint16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_s8(__p0, __p1) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 32); \ +}) +#else +#define vst2q_s8(__p0, __p1) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + int8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_f32(__p0, __p1) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 41); \ +}) +#else +#define vst2q_f32(__p0, __p1) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + float32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_f16(__p0, __p1) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 40); \ +}) +#else +#define vst2q_f16(__p0, __p1) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + float16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_s32(__p0, __p1) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 34); \ +}) +#else +#define vst2q_s32(__p0, __p1) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + int32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_s16(__p0, __p1) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 33); \ +}) +#else +#define vst2q_s16(__p0, __p1) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + int16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_u8(__p0, __p1) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 16); \ +}) +#else +#define vst2_u8(__p0, __p1) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + uint8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_u32(__p0, __p1) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 18); \ +}) +#else +#define vst2_u32(__p0, __p1) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + uint32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_u64(__p0, __p1) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 19); \ +}) +#else +#define vst2_u64(__p0, __p1) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_u16(__p0, __p1) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 17); \ +}) +#else +#define vst2_u16(__p0, __p1) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + uint16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_s8(__p0, __p1) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 0); \ +}) +#else +#define vst2_s8(__p0, __p1) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + int8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_f32(__p0, __p1) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 9); \ +}) +#else +#define vst2_f32(__p0, __p1) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + float32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_v(__p0, __rev1.val[0], __rev1.val[1], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_f16(__p0, __p1) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 8); \ +}) +#else +#define vst2_f16(__p0, __p1) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + float16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, __rev1.val[0], __rev1.val[1], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_s32(__p0, __p1) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 2); \ +}) +#else +#define vst2_s32(__p0, __p1) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + int32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_v(__p0, __rev1.val[0], __rev1.val[1], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_s64(__p0, __p1) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 3); \ +}) +#else +#define vst2_s64(__p0, __p1) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_s16(__p0, __p1) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 1); \ +}) +#else +#define vst2_s16(__p0, __p1) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + int16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, __rev1.val[0], __rev1.val[1], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 4); \ +}) +#else +#define vst2_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + poly8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 5); \ +}) +#else +#define vst2_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + poly16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 37); \ +}) +#else +#define vst2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + poly16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 50); \ +}) +#else +#define vst2q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + uint32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 49); \ +}) +#else +#define vst2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + uint16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 41); \ +}) +#else +#define vst2q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + float32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 40); \ +}) +#else +#define vst2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + float16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 34); \ +}) +#else +#define vst2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + int32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 33); \ +}) +#else +#define vst2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + int16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 16); \ +}) +#else +#define vst2_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + uint8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 18); \ +}) +#else +#define vst2_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + uint32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 17); \ +}) +#else +#define vst2_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + uint16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 0); \ +}) +#else +#define vst2_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + int8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 9); \ +}) +#else +#define vst2_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + float32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 8); \ +}) +#else +#define vst2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + float16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 2); \ +}) +#else +#define vst2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + int32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 1); \ +}) +#else +#define vst2_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + int16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_p8(__p0, __p1) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 4); \ +}) +#else +#define vst3_p8(__p0, __p1) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + poly8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_p16(__p0, __p1) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 5); \ +}) +#else +#define vst3_p16(__p0, __p1) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + poly16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 36); \ +}) +#else +#define vst3q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + poly8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 37); \ +}) +#else +#define vst3q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + poly16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 48); \ +}) +#else +#define vst3q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + uint8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 50); \ +}) +#else +#define vst3q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + uint32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 49); \ +}) +#else +#define vst3q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + uint16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_s8(__p0, __p1) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 32); \ +}) +#else +#define vst3q_s8(__p0, __p1) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + int8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_f32(__p0, __p1) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 41); \ +}) +#else +#define vst3q_f32(__p0, __p1) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + float32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_f16(__p0, __p1) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 40); \ +}) +#else +#define vst3q_f16(__p0, __p1) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + float16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_s32(__p0, __p1) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 34); \ +}) +#else +#define vst3q_s32(__p0, __p1) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + int32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_s16(__p0, __p1) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 33); \ +}) +#else +#define vst3q_s16(__p0, __p1) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + int16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_u8(__p0, __p1) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 16); \ +}) +#else +#define vst3_u8(__p0, __p1) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + uint8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_u32(__p0, __p1) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 18); \ +}) +#else +#define vst3_u32(__p0, __p1) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + uint32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_u64(__p0, __p1) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 19); \ +}) +#else +#define vst3_u64(__p0, __p1) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_u16(__p0, __p1) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 17); \ +}) +#else +#define vst3_u16(__p0, __p1) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + uint16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_s8(__p0, __p1) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 0); \ +}) +#else +#define vst3_s8(__p0, __p1) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + int8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_f32(__p0, __p1) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 9); \ +}) +#else +#define vst3_f32(__p0, __p1) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + float32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_f16(__p0, __p1) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 8); \ +}) +#else +#define vst3_f16(__p0, __p1) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + float16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_s32(__p0, __p1) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 2); \ +}) +#else +#define vst3_s32(__p0, __p1) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + int32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_s64(__p0, __p1) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 3); \ +}) +#else +#define vst3_s64(__p0, __p1) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_s16(__p0, __p1) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 1); \ +}) +#else +#define vst3_s16(__p0, __p1) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + int16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 4); \ +}) +#else +#define vst3_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + poly8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 5); \ +}) +#else +#define vst3_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + poly16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 37); \ +}) +#else +#define vst3q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + poly16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 50); \ +}) +#else +#define vst3q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + uint32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 49); \ +}) +#else +#define vst3q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + uint16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 41); \ +}) +#else +#define vst3q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + float32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 40); \ +}) +#else +#define vst3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + float16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 34); \ +}) +#else +#define vst3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + int32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 33); \ +}) +#else +#define vst3q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + int16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 16); \ +}) +#else +#define vst3_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + uint8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 18); \ +}) +#else +#define vst3_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + uint32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 17); \ +}) +#else +#define vst3_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + uint16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 0); \ +}) +#else +#define vst3_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + int8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 9); \ +}) +#else +#define vst3_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + float32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 8); \ +}) +#else +#define vst3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + float16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 2); \ +}) +#else +#define vst3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + int32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 1); \ +}) +#else +#define vst3_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + int16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_p8(__p0, __p1) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 4); \ +}) +#else +#define vst4_p8(__p0, __p1) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + poly8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_p16(__p0, __p1) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 5); \ +}) +#else +#define vst4_p16(__p0, __p1) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + poly16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 36); \ +}) +#else +#define vst4q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + poly8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 37); \ +}) +#else +#define vst4q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + poly16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 48); \ +}) +#else +#define vst4q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + uint8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 50); \ +}) +#else +#define vst4q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + uint32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 49); \ +}) +#else +#define vst4q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + uint16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_s8(__p0, __p1) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 32); \ +}) +#else +#define vst4q_s8(__p0, __p1) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + int8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_f32(__p0, __p1) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 41); \ +}) +#else +#define vst4q_f32(__p0, __p1) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + float32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_f16(__p0, __p1) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 40); \ +}) +#else +#define vst4q_f16(__p0, __p1) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + float16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_s32(__p0, __p1) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 34); \ +}) +#else +#define vst4q_s32(__p0, __p1) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + int32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_s16(__p0, __p1) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 33); \ +}) +#else +#define vst4q_s16(__p0, __p1) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + int16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_u8(__p0, __p1) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 16); \ +}) +#else +#define vst4_u8(__p0, __p1) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + uint8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_u32(__p0, __p1) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 18); \ +}) +#else +#define vst4_u32(__p0, __p1) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + uint32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_u64(__p0, __p1) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 19); \ +}) +#else +#define vst4_u64(__p0, __p1) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_u16(__p0, __p1) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 17); \ +}) +#else +#define vst4_u16(__p0, __p1) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + uint16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_s8(__p0, __p1) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 0); \ +}) +#else +#define vst4_s8(__p0, __p1) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + int8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_f32(__p0, __p1) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 9); \ +}) +#else +#define vst4_f32(__p0, __p1) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + float32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_f16(__p0, __p1) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 8); \ +}) +#else +#define vst4_f16(__p0, __p1) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + float16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_s32(__p0, __p1) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 2); \ +}) +#else +#define vst4_s32(__p0, __p1) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + int32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_s64(__p0, __p1) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 3); \ +}) +#else +#define vst4_s64(__p0, __p1) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_s16(__p0, __p1) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 1); \ +}) +#else +#define vst4_s16(__p0, __p1) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + int16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 4); \ +}) +#else +#define vst4_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + poly8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 5); \ +}) +#else +#define vst4_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + poly16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 37); \ +}) +#else +#define vst4q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + poly16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 50); \ +}) +#else +#define vst4q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + uint32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 49); \ +}) +#else +#define vst4q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + uint16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 41); \ +}) +#else +#define vst4q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + float32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 40); \ +}) +#else +#define vst4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + float16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 34); \ +}) +#else +#define vst4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + int32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 33); \ +}) +#else +#define vst4q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + int16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 16); \ +}) +#else +#define vst4_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + uint8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 18); \ +}) +#else +#define vst4_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + uint32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 17); \ +}) +#else +#define vst4_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + uint16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 0); \ +}) +#else +#define vst4_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + int8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 9); \ +}) +#else +#define vst4_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + float32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 8); \ +}) +#else +#define vst4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + float16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 2); \ +}) +#else +#define vst4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + int32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 1); \ +}) +#else +#define vst4_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + int16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint8x16_t vsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint32x4_t vsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsubq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint64x2_t vsubq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint16x8_t vsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int8x16_t vsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vsubq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float32x4_t vsubq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int32x4_t vsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vsubq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int64x2_t vsubq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int16x8_t vsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint8x8_t vsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint32x2_t vsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vsub_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint64x1_t vsub_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint16x4_t vsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int8x8_t vsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vsub_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float32x2_t vsub_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int32x2_t vsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vsub_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int64x1_t vsub_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int16x4_t vsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = vmovl_u8(__p0) - vmovl_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vsubl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmovl_u8(__rev0) - __noswap_vmovl_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsubl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = vmovl_u32(__p0) - vmovl_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vsubl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmovl_u32(__rev0) - __noswap_vmovl_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = vmovl_u16(__p0) - vmovl_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vsubl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmovl_u16(__rev0) - __noswap_vmovl_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubl_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = vmovl_s8(__p0) - vmovl_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vsubl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmovl_s8(__rev0) - __noswap_vmovl_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vsubl_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = vmovl_s32(__p0) - vmovl_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vsubl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmovl_s32(__rev0) - __noswap_vmovl_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubl_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = vmovl_s16(__p0) - vmovl_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vsubl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmovl_s16(__rev0) - __noswap_vmovl_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubw_u8(uint16x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 - vmovl_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vsubw_u8(uint16x8_t __p0, uint8x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __noswap_vmovl_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsubw_u32(uint64x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 - vmovl_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vsubw_u32(uint64x2_t __p0, uint32x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 - __noswap_vmovl_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubw_u16(uint32x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 - vmovl_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vsubw_u16(uint32x4_t __p0, uint16x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __noswap_vmovl_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubw_s8(int16x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = __p0 - vmovl_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vsubw_s8(int16x8_t __p0, int8x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __noswap_vmovl_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vsubw_s32(int64x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = __p0 - vmovl_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vsubw_s32(int64x2_t __p0, int32x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 - __noswap_vmovl_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubw_s16(int32x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = __p0 - vmovl_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vsubw_s16(int32x4_t __p0, int16x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __noswap_vmovl_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbl1_p8(poly8x8_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vtbl1_p8(poly8x8_t __p0, uint8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__rev0, (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbl1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtbl1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbl1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vtbl1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbl2_p8(poly8x8x2_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vtbl2_p8(poly8x8x2_t __p0, uint8x8_t __p1) { + poly8x8x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbl2_u8(uint8x8x2_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtbl2_u8(uint8x8x2_t __p0, uint8x8_t __p1) { + uint8x8x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbl2_s8(int8x8x2_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vtbl2_s8(int8x8x2_t __p0, int8x8_t __p1) { + int8x8x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbl3_p8(poly8x8x3_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vtbl3_p8(poly8x8x3_t __p0, uint8x8_t __p1) { + poly8x8x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbl3_u8(uint8x8x3_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtbl3_u8(uint8x8x3_t __p0, uint8x8_t __p1) { + uint8x8x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbl3_s8(int8x8x3_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vtbl3_s8(int8x8x3_t __p0, int8x8_t __p1) { + int8x8x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbl4_p8(poly8x8x4_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p0.val[3], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vtbl4_p8(poly8x8x4_t __p0, uint8x8_t __p1) { + poly8x8x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev0.val[3], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbl4_u8(uint8x8x4_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p0.val[3], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtbl4_u8(uint8x8x4_t __p0, uint8x8_t __p1) { + uint8x8x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev0.val[3], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbl4_s8(int8x8x4_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p0.val[3], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vtbl4_s8(int8x8x4_t __p0, int8x8_t __p1) { + int8x8x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev0.val[3], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbx1_p8(poly8x8_t __p0, poly8x8_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vtbx1_p8(poly8x8_t __p0, poly8x8_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbx1_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vtbx1_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbx1_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vtbx1_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbx2_p8(poly8x8_t __p0, poly8x8x2_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vtbx2_p8(poly8x8_t __p0, poly8x8x2_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbx2_u8(uint8x8_t __p0, uint8x8x2_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vtbx2_u8(uint8x8_t __p0, uint8x8x2_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbx2_s8(int8x8_t __p0, int8x8x2_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vtbx2_s8(int8x8_t __p0, int8x8x2_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbx3_p8(poly8x8_t __p0, poly8x8x3_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vtbx3_p8(poly8x8_t __p0, poly8x8x3_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbx3_u8(uint8x8_t __p0, uint8x8x3_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vtbx3_u8(uint8x8_t __p0, uint8x8x3_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbx3_s8(int8x8_t __p0, int8x8x3_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vtbx3_s8(int8x8_t __p0, int8x8x3_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbx4_p8(poly8x8_t __p0, poly8x8x4_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p1.val[3], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vtbx4_p8(poly8x8_t __p0, poly8x8x4_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbx4_u8(uint8x8_t __p0, uint8x8x4_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p1.val[3], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vtbx4_u8(uint8x8_t __p0, uint8x8x4_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbx4_s8(int8x8_t __p0, int8x8x4_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p1.val[3], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vtbx4_s8(int8x8_t __p0, int8x8x4_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8x2_t vtrn_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8x2_t vtrn_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 4); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4x2_t vtrn_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 5); + return __ret; +} +#else +__ai poly16x4x2_t vtrn_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 5); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16x2_t vtrnq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16x2_t vtrnq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 36); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8x2_t vtrnq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 37); + return __ret; +} +#else +__ai poly16x8x2_t vtrnq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 37); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16x2_t vtrnq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16x2_t vtrnq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 48); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4x2_t vtrnq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4x2_t vtrnq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 50); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8x2_t vtrnq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8x2_t vtrnq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 49); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16x2_t vtrnq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16x2_t vtrnq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 32); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4x2_t vtrnq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4x2_t vtrnq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 41); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4x2_t vtrnq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4x2_t vtrnq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 34); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8x2_t vtrnq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8x2_t vtrnq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 33); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8x2_t vtrn_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8x2_t vtrn_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 16); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2x2_t vtrn_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2x2_t vtrn_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 18); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4x2_t vtrn_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4x2_t vtrn_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 17); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8x2_t vtrn_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8x2_t vtrn_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 0); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2x2_t vtrn_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2x2_t vtrn_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 9); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2x2_t vtrn_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2x2_t vtrn_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 2); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4x2_t vtrn_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4x2_t vtrn_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 1); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtst_p8(poly8x8_t __p0, poly8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtst_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vtst_p16(poly16x4_t __p0, poly16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vtst_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vtstq_p8(poly8x16_t __p0, poly8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vtstq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vtstq_p16(poly16x8_t __p0, poly16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vtstq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vtstq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vtstq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vtstq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vtstq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vtstq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vtstq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vtstq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vtstq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vtstq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vtstq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vtstq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vtstq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtst_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtst_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vtst_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vtst_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vtst_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vtst_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtst_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtst_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vtst_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vtst_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vtst_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vtst_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8x2_t vuzp_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8x2_t vuzp_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 4); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4x2_t vuzp_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 5); + return __ret; +} +#else +__ai poly16x4x2_t vuzp_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 5); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16x2_t vuzpq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16x2_t vuzpq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 36); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8x2_t vuzpq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 37); + return __ret; +} +#else +__ai poly16x8x2_t vuzpq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 37); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16x2_t vuzpq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16x2_t vuzpq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 48); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4x2_t vuzpq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4x2_t vuzpq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 50); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8x2_t vuzpq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8x2_t vuzpq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 49); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16x2_t vuzpq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16x2_t vuzpq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 32); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4x2_t vuzpq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4x2_t vuzpq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 41); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4x2_t vuzpq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4x2_t vuzpq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 34); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8x2_t vuzpq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8x2_t vuzpq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 33); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8x2_t vuzp_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8x2_t vuzp_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 16); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2x2_t vuzp_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2x2_t vuzp_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 18); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4x2_t vuzp_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4x2_t vuzp_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 17); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8x2_t vuzp_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8x2_t vuzp_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 0); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2x2_t vuzp_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2x2_t vuzp_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 9); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2x2_t vuzp_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2x2_t vuzp_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 2); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4x2_t vuzp_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4x2_t vuzp_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 1); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8x2_t vzip_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8x2_t vzip_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 4); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4x2_t vzip_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 5); + return __ret; +} +#else +__ai poly16x4x2_t vzip_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 5); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16x2_t vzipq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16x2_t vzipq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 36); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8x2_t vzipq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 37); + return __ret; +} +#else +__ai poly16x8x2_t vzipq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 37); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16x2_t vzipq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16x2_t vzipq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 48); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4x2_t vzipq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4x2_t vzipq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 50); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8x2_t vzipq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8x2_t vzipq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 49); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16x2_t vzipq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16x2_t vzipq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 32); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4x2_t vzipq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4x2_t vzipq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 41); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4x2_t vzipq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4x2_t vzipq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 34); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8x2_t vzipq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8x2_t vzipq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 33); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8x2_t vzip_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8x2_t vzip_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 16); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2x2_t vzip_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2x2_t vzip_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 18); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4x2_t vzip_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4x2_t vzip_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 17); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8x2_t vzip_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8x2_t vzip_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 0); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2x2_t vzip_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2x2_t vzip_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 9); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2x2_t vzip_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2x2_t vzip_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 2); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4x2_t vzip_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4x2_t vzip_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 1); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#if !defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_p16(poly16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_p16(poly16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u8(uint8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u8(uint8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u32(uint32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u32(uint32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u64(uint64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u64(uint64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u16(uint16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u16(uint16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s8(int8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s8(int8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_f32(float32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_f32(float32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_f16(float16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_f16(float16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s32(int32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s32(int32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s64(int64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s64(int64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_p8(poly8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_p8(poly8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u8(uint8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u8(uint8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u32(uint32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u32(uint32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u64(uint64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u64(uint64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u16(uint16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u16(uint16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s8(int8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s8(int8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_f32(float32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_f32(float32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_f16(float16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_f16(float16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s32(int32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s32(int32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s64(int64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s64(int64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_p16(poly16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_p16(poly16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u8(uint8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u8(uint8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u32(uint32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u32(uint32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u64(uint64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u64(uint64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u16(uint16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u16(uint16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s8(int8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s8(int8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_f32(float32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_f32(float32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_f16(float16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_f16(float16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s32(int32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s32(int32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s64(int64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s64(int64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_p8(poly8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_p8(poly8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u8(uint8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u8(uint8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u32(uint32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u32(uint32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u64(uint64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u64(uint64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u16(uint16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u16(uint16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s8(int8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s8(int8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_f32(float32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_f32(float32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_f16(float16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_f16(float16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s32(int32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s32(int32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s64(int64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s64(int64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p8(poly8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p8(poly8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p16(poly16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p16(poly16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u32(uint32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u32(uint32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u64(uint64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u64(uint64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u16(uint16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u16(uint16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_f32(float32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_f32(float32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_f16(float16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_f16(float16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s32(int32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s32(int32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s64(int64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s64(int64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p8(poly8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p8(poly8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p16(poly16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p16(poly16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u8(uint8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u8(uint8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u64(uint64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u64(uint64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u16(uint16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u16(uint16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s8(int8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s8(int8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_f16(float16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_f16(float16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s64(int64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s64(int64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p8(poly8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p8(poly8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p16(poly16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p16(poly16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u8(uint8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u8(uint8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u32(uint32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u32(uint32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u16(uint16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u16(uint16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s8(int8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s8(int8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_f32(float32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_f32(float32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_f16(float16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_f16(float16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s32(int32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s32(int32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p8(poly8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p8(poly8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p16(poly16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p16(poly16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u8(uint8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u8(uint8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u32(uint32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u32(uint32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u64(uint64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u64(uint64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s8(int8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s8(int8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_f32(float32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_f32(float32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s32(int32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s32(int32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s64(int64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s64(int64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p8(poly8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p8(poly8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p16(poly16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p16(poly16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u8(uint8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u8(uint8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u32(uint32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u32(uint32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u64(uint64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u64(uint64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u16(uint16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u16(uint16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_f32(float32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_f32(float32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_f16(float16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_f16(float16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s32(int32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s32(int32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s64(int64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s64(int64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p8(poly8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p8(poly8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p16(poly16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p16(poly16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u8(uint8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u8(uint8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u32(uint32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u32(uint32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u64(uint64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u64(uint64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u16(uint16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u16(uint16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s8(int8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s8(int8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_f16(float16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_f16(float16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s32(int32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s32(int32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s64(int64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s64(int64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p8(poly8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p8(poly8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p16(poly16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p16(poly16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u8(uint8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u8(uint8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u32(uint32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u32(uint32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u64(uint64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u64(uint64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u16(uint16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u16(uint16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s8(int8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s8(int8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_f32(float32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_f32(float32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s32(int32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s32(int32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s64(int64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s64(int64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p8(poly8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p8(poly8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p16(poly16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p16(poly16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u8(uint8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u8(uint8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u32(uint32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u32(uint32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u64(uint64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u64(uint64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u16(uint16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u16(uint16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s8(int8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s8(int8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_f16(float16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_f16(float16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s64(int64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s64(int64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p8(poly8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p8(poly8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p16(poly16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p16(poly16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u8(uint8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u8(uint8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u32(uint32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u32(uint32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u64(uint64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u64(uint64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u16(uint16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u16(uint16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s8(int8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s8(int8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_f32(float32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_f32(float32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_f16(float16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_f16(float16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s32(int32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s32(int32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p16(poly16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p16(poly16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u8(uint8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u8(uint8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u32(uint32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u32(uint32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u64(uint64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u64(uint64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u16(uint16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u16(uint16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s8(int8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s8(int8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_f32(float32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_f32(float32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s32(int32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s32(int32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s64(int64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s64(int64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_p16(poly16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_p16(poly16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u32(uint32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u32(uint32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u64(uint64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u64(uint64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u16(uint16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u16(uint16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_f32(float32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_f32(float32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_f16(float16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_f16(float16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s32(int32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s32(int32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s64(int64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s64(int64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_p8(poly8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_p8(poly8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_p16(poly16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_p16(poly16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u8(uint8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u8(uint8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u64(uint64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u64(uint64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u16(uint16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u16(uint16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s8(int8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s8(int8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_f16(float16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_f16(float16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s64(int64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s64(int64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_p8(poly8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_p8(poly8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_p16(poly16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_p16(poly16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u8(uint8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u8(uint8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u32(uint32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u32(uint32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u16(uint16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u16(uint16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s8(int8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s8(int8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_f32(float32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_f32(float32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_f16(float16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_f16(float16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s32(int32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s32(int32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_p8(poly8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_p8(poly8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_p16(poly16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_p16(poly16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u8(uint8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u8(uint8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u32(uint32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u32(uint32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u64(uint64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u64(uint64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s8(int8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s8(int8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_f32(float32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_f32(float32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s32(int32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s32(int32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s64(int64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s64(int64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_p8(poly8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_p8(poly8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_p16(poly16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_p16(poly16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u8(uint8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u8(uint8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u32(uint32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u32(uint32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u64(uint64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u64(uint64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u16(uint16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u16(uint16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_f32(float32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_f32(float32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_f16(float16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_f16(float16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s32(int32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s32(int32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s64(int64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s64(int64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s16(int16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s16(int16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_p8(poly8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_p8(poly8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_p16(poly16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_p16(poly16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u8(uint8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u8(uint8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u32(uint32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u32(uint32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u64(uint64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u64(uint64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u16(uint16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u16(uint16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s8(int8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s8(int8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_f16(float16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_f16(float16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s32(int32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s32(int32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s64(int64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s64(int64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s16(int16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s16(int16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_p8(poly8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_p8(poly8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_p16(poly16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_p16(poly16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u8(uint8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u8(uint8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u32(uint32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u32(uint32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u64(uint64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u64(uint64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u16(uint16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u16(uint16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s8(int8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s8(int8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_f32(float32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_f32(float32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s32(int32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s32(int32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s64(int64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s64(int64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s16(int16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s16(int16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_p8(poly8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_p8(poly8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_p16(poly16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_p16(poly16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u8(uint8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u8(uint8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u32(uint32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u32(uint32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u64(uint64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u64(uint64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u16(uint16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u16(uint16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s8(int8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s8(int8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_f16(float16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_f16(float16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s64(int64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s64(int64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s16(int16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s16(int16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_p8(poly8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_p8(poly8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_p16(poly16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_p16(poly16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u8(uint8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u8(uint8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u32(uint32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u32(uint32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u64(uint64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u64(uint64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u16(uint16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u16(uint16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s8(int8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s8(int8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_f32(float32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_f32(float32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_f16(float16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_f16(float16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s32(int32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s32(int32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s16(int16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s16(int16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_p16(poly16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_p16(poly16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u8(uint8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u8(uint8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u32(uint32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u32(uint32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u64(uint64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u64(uint64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u16(uint16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u16(uint16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s8(int8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s8(int8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_f32(float32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_f32(float32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s32(int32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s32(int32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s64(int64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s64(int64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#endif +#if (__ARM_FP & 2) +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vcvt_f16_f32(float32x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_f32((int8x16_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vcvt_f16_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_f32((int8x16_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float16x4_t __noswap_vcvt_f16_f32(float32x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_f32((int8x16_t)__p0, 8); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvt_f32_f16(float16x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvt_f32_f16((int8x8_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vcvt_f32_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvt_f32_f16((int8x8_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float32x4_t __noswap_vcvt_f32_f16(float16x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvt_f32_f16((int8x8_t)__p0, 41); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcvtaq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtaq_s32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vcvtaq_s32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtaq_s32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcvta_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvta_s32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcvta_s32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvta_s32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcvtaq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtaq_u32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcvtaq_u32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtaq_u32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcvta_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvta_u32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcvta_u32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvta_u32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcvtmq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtmq_s32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vcvtmq_s32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtmq_s32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcvtm_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtm_s32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcvtm_s32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtm_s32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcvtmq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtmq_u32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcvtmq_u32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtmq_u32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcvtm_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtm_u32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcvtm_u32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtm_u32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcvtnq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtnq_s32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vcvtnq_s32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtnq_s32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcvtn_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtn_s32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcvtn_s32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtn_s32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcvtnq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtnq_u32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcvtnq_u32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtnq_u32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcvtn_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtn_u32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcvtn_u32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtn_u32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcvtpq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtpq_s32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vcvtpq_s32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtpq_s32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcvtp_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtp_s32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcvtp_s32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtp_s32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcvtpq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtpq_u32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcvtpq_u32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtpq_u32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcvtp_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtp_u32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcvtp_u32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtp_u32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING) +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrnd_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrnd_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrnd_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrnd_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndaq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndaq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndaq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndaq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrnda_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrnda_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrnda_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrnda_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndmq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndmq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndmq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndmq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrndm_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndm_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrndm_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndm_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndnq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndnq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndnq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndnq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrndn_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndn_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrndn_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndn_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndpq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndpq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndpq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndpq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrndp_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndp_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrndp_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndp_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndxq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndxq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndxq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndxq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrndx_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndx_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrndx_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndx_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN) +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmaxnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vmaxnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmaxnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vmaxnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmaxnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vminnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vminnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vminnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vminnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vminnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 && defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcvtaq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtaq_s64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vcvtaq_s64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtaq_s64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcvta_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvta_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vcvta_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvta_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcvtaq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtaq_u64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcvtaq_u64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtaq_u64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcvta_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvta_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcvta_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvta_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcvtmq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtmq_s64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vcvtmq_s64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtmq_s64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcvtm_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtm_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vcvtm_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtm_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcvtmq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtmq_u64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcvtmq_u64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtmq_u64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcvtm_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtm_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcvtm_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtm_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcvtnq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtnq_s64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vcvtnq_s64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtnq_s64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcvtn_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtn_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vcvtn_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtn_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcvtnq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtnq_u64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcvtnq_u64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtnq_u64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcvtn_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtn_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcvtn_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtn_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcvtpq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtpq_s64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vcvtpq_s64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtpq_s64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcvtp_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtp_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vcvtp_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtp_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcvtpq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtpq_u64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcvtpq_u64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtpq_u64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcvtp_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtp_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcvtp_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtp_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_p64(poly64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_p64(poly64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_p16(poly16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_p16(poly16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u8(uint8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u8(uint8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u32(uint32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u32(uint32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u64(uint64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u64(uint64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u16(uint16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u16(uint16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s8(int8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s8(int8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_f64(float64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_f64(float64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_f32(float32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_f32(float32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_f16(float16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_f16(float16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s32(int32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s32(int32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s64(int64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s64(int64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_p8(poly8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_p8(poly8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_p16(poly16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_p16(poly16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_u8(uint8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_u8(uint8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_u32(uint32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_u32(uint32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_u64(uint64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_u64(uint64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_u16(uint16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_u16(uint16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_s8(int8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_s8(int8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_f64(float64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_f64(float64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_f32(float32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_f32(float32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_f16(float16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_f16(float16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_s32(int32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_s32(int32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_s64(int64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_s64(int64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_s16(int16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_s16(int16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_p8(poly8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_p8(poly8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_p64(poly64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_p64(poly64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u8(uint8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u8(uint8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u32(uint32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u32(uint32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u64(uint64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u64(uint64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u16(uint16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u16(uint16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s8(int8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s8(int8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_f64(float64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_f64(float64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_f32(float32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_f32(float32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_f16(float16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_f16(float16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s32(int32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s32(int32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s64(int64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s64(int64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_p128(poly128_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_p128(poly128_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_p64(poly64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_p64(poly64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_p16(poly16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_p16(poly16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u8(uint8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u8(uint8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u32(uint32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u32(uint32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u64(uint64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u64(uint64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u16(uint16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u16(uint16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s8(int8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s8(int8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_f64(float64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_f64(float64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_f32(float32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_f32(float32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_f16(float16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_f16(float16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s32(int32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s32(int32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s64(int64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s64(int64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_p8(poly8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_p8(poly8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_p64(poly64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_p64(poly64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_p16(poly16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_p16(poly16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_u8(uint8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_u8(uint8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_u32(uint32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_u32(uint32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_u64(uint64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_u64(uint64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_u16(uint16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_u16(uint16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_s8(int8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_s8(int8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_f64(float64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_f64(float64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_f32(float32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_f32(float32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_f16(float16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_f16(float16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_s32(int32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_s32(int32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_s64(int64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_s64(int64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_s16(int16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_s16(int16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_p8(poly8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_p8(poly8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_p128(poly128_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_p128(poly128_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_p16(poly16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_p16(poly16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_u8(uint8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_u8(uint8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_u32(uint32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_u32(uint32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_u64(uint64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_u64(uint64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_u16(uint16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_u16(uint16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_s8(int8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_s8(int8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_f64(float64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_f64(float64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_f32(float32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_f32(float32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_f16(float16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_f16(float16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_s32(int32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_s32(int32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_s64(int64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_s64(int64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_s16(int16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_s16(int16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_p8(poly8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_p8(poly8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_p128(poly128_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_p128(poly128_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_p64(poly64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_p64(poly64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u8(uint8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u8(uint8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u32(uint32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u32(uint32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u64(uint64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u64(uint64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u16(uint16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u16(uint16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s8(int8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s8(int8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_f64(float64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_f64(float64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_f32(float32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_f32(float32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_f16(float16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_f16(float16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s32(int32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s32(int32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s64(int64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s64(int64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p8(poly8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p8(poly8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p128(poly128_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p128(poly128_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p64(poly64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p64(poly64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p16(poly16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p16(poly16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u32(uint32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u32(uint32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u64(uint64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u64(uint64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u16(uint16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u16(uint16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_f64(float64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_f64(float64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_f32(float32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_f32(float32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_f16(float16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_f16(float16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s32(int32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s32(int32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s64(int64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s64(int64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p8(poly8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p8(poly8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p128(poly128_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p128(poly128_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p64(poly64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p64(poly64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p16(poly16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p16(poly16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u8(uint8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u8(uint8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u64(uint64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u64(uint64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u16(uint16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u16(uint16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s8(int8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s8(int8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_f64(float64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_f64(float64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_f16(float16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_f16(float16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s64(int64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s64(int64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p8(poly8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p8(poly8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p128(poly128_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p128(poly128_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p64(poly64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p64(poly64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p16(poly16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p16(poly16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u8(uint8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u8(uint8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u32(uint32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u32(uint32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u16(uint16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u16(uint16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s8(int8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s8(int8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_f32(float32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_f32(float32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_f16(float16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_f16(float16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s32(int32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s32(int32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p8(poly8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p8(poly8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p128(poly128_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p128(poly128_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p64(poly64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p64(poly64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p16(poly16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p16(poly16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u8(uint8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u8(uint8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u32(uint32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u32(uint32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u64(uint64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u64(uint64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s8(int8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s8(int8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_f64(float64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_f64(float64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_f32(float32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_f32(float32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s32(int32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s32(int32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s64(int64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s64(int64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p8(poly8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p8(poly8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p128(poly128_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p128(poly128_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p64(poly64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p64(poly64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p16(poly16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p16(poly16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u8(uint8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u8(uint8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u32(uint32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u32(uint32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u64(uint64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u64(uint64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u16(uint16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u16(uint16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_f64(float64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_f64(float64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_f32(float32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_f32(float32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_f16(float16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_f16(float16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s32(int32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s32(int32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s64(int64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s64(int64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_p8(poly8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_p8(poly8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_p128(poly128_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_p128(poly128_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_p64(poly64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_p64(poly64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_p16(poly16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_p16(poly16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_u8(uint8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_u8(uint8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_u32(uint32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_u32(uint32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_u64(uint64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_u64(uint64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_u16(uint16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_u16(uint16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_s8(int8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_s8(int8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_f32(float32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_f32(float32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_f16(float16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_f16(float16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_s32(int32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_s32(int32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_s64(int64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_s64(int64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_s16(int16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_s16(int16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p8(poly8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p8(poly8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p128(poly128_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p128(poly128_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p64(poly64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p64(poly64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p16(poly16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p16(poly16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u8(uint8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u8(uint8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u32(uint32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u32(uint32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u64(uint64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u64(uint64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u16(uint16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u16(uint16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s8(int8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s8(int8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_f64(float64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_f64(float64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_f16(float16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_f16(float16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s32(int32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s32(int32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s64(int64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s64(int64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p8(poly8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p8(poly8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p128(poly128_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p128(poly128_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p64(poly64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p64(poly64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p16(poly16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p16(poly16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u8(uint8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u8(uint8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u32(uint32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u32(uint32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u64(uint64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u64(uint64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u16(uint16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u16(uint16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s8(int8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s8(int8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_f64(float64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_f64(float64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_f32(float32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_f32(float32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s32(int32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s32(int32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s64(int64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s64(int64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p8(poly8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p8(poly8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p128(poly128_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p128(poly128_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p64(poly64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p64(poly64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p16(poly16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p16(poly16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u8(uint8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u8(uint8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u32(uint32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u32(uint32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u64(uint64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u64(uint64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u16(uint16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u16(uint16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s8(int8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s8(int8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_f64(float64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_f64(float64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_f16(float16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_f16(float16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s64(int64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s64(int64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p8(poly8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p8(poly8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p128(poly128_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p128(poly128_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p64(poly64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p64(poly64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p16(poly16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p16(poly16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u8(uint8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u8(uint8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u32(uint32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u32(uint32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u64(uint64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u64(uint64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u16(uint16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u16(uint16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s8(int8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s8(int8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_f32(float32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_f32(float32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_f16(float16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_f16(float16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s32(int32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s32(int32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p128(poly128_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p128(poly128_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p64(poly64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p64(poly64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p16(poly16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p16(poly16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u8(uint8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u8(uint8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u32(uint32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u32(uint32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u64(uint64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u64(uint64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u16(uint16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u16(uint16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s8(int8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s8(int8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_f64(float64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_f64(float64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_f32(float32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_f32(float32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s32(int32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s32(int32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s64(int64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s64(int64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_p64(poly64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_p64(poly64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_p16(poly16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_p16(poly16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u32(uint32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u32(uint32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u64(uint64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u64(uint64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u16(uint16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u16(uint16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_f64(float64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_f64(float64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_f32(float32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_f32(float32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_f16(float16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_f16(float16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s32(int32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s32(int32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s64(int64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s64(int64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_p8(poly8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_p8(poly8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_p64(poly64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_p64(poly64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_p16(poly16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_p16(poly16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u8(uint8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u8(uint8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u64(uint64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u64(uint64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u16(uint16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u16(uint16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s8(int8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s8(int8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_f64(float64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_f64(float64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_f16(float16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_f16(float16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s64(int64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s64(int64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_p8(poly8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_p8(poly8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_p64(poly64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_p64(poly64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_p16(poly16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_p16(poly16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u8(uint8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u8(uint8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u32(uint32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u32(uint32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u16(uint16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u16(uint16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s8(int8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s8(int8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_f32(float32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_f32(float32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_f16(float16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_f16(float16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s32(int32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s32(int32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_p8(poly8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_p8(poly8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_p64(poly64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_p64(poly64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_p16(poly16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_p16(poly16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u8(uint8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u8(uint8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u32(uint32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u32(uint32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u64(uint64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u64(uint64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s8(int8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s8(int8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_f64(float64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_f64(float64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_f32(float32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_f32(float32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s32(int32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s32(int32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s64(int64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s64(int64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_p8(poly8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_p8(poly8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_p64(poly64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_p64(poly64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_p16(poly16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_p16(poly16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u8(uint8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u8(uint8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u32(uint32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u32(uint32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u64(uint64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u64(uint64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u16(uint16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u16(uint16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_f64(float64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_f64(float64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_f32(float32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_f32(float32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_f16(float16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_f16(float16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s32(int32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s32(int32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s64(int64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s64(int64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s16(int16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s16(int16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_p8(poly8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_p8(poly8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_p64(poly64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_p64(poly64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_p16(poly16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_p16(poly16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_u8(uint8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_u8(uint8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_u32(uint32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_u32(uint32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_u64(uint64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_u64(uint64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_u16(uint16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_u16(uint16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_s8(int8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_s8(int8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_f32(float32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_f32(float32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_f16(float16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_f16(float16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_s32(int32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_s32(int32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_s64(int64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_s64(int64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_s16(int16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_s16(int16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_p8(poly8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_p8(poly8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_p64(poly64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_p64(poly64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_p16(poly16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_p16(poly16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u8(uint8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u8(uint8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u32(uint32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u32(uint32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u64(uint64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u64(uint64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u16(uint16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u16(uint16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s8(int8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s8(int8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_f64(float64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_f64(float64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_f16(float16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_f16(float16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s32(int32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s32(int32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s64(int64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s64(int64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s16(int16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s16(int16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_p8(poly8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_p8(poly8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_p64(poly64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_p64(poly64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_p16(poly16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_p16(poly16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u8(uint8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u8(uint8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u32(uint32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u32(uint32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u64(uint64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u64(uint64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u16(uint16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u16(uint16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s8(int8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s8(int8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_f64(float64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_f64(float64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_f32(float32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_f32(float32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s32(int32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s32(int32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s64(int64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s64(int64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s16(int16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s16(int16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_p8(poly8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_p8(poly8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_p64(poly64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_p64(poly64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_p16(poly16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_p16(poly16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u8(uint8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u8(uint8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u32(uint32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u32(uint32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u64(uint64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u64(uint64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u16(uint16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u16(uint16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s8(int8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s8(int8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_f64(float64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_f64(float64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_f16(float16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_f16(float16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s64(int64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s64(int64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s16(int16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s16(int16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_p8(poly8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_p8(poly8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_p64(poly64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_p64(poly64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_p16(poly16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_p16(poly16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u8(uint8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u8(uint8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u32(uint32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u32(uint32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u64(uint64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u64(uint64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u16(uint16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u16(uint16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s8(int8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s8(int8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_f32(float32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_f32(float32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_f16(float16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_f16(float16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s32(int32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s32(int32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s16(int16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s16(int16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_p64(poly64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_p64(poly64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_p16(poly16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_p16(poly16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u8(uint8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u8(uint8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u32(uint32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u32(uint32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u64(uint64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u64(uint64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u16(uint16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u16(uint16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s8(int8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s8(int8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_f64(float64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_f64(float64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_f32(float32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_f32(float32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s32(int32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s32(int32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s64(int64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s64(int64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_DIRECTED_ROUNDING) +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrnd_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrnd_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrnd_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrnd_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndaq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndaq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndaq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndaq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrnda_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrnda_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrnda_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrnda_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndiq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndiq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndiq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndiq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndiq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndiq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndiq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndiq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrndi_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndi_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrndi_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndi_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrndi_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndi_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrndi_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndi_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndmq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndmq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndmq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndmq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrndm_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndm_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrndm_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndm_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndnq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndnq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndnq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndnq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrndn_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndn_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrndn_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndn_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndpq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndpq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndpq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndpq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrndp_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndp_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrndp_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndp_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndxq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndxq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndxq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndxq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrndx_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndx_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrndx_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndx_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_NUMERIC_MAXMIN) +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmaxnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vmaxnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmaxnm_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vmaxnm_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vminnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vminnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vminnm_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vminnm_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#endif +#if __ARM_FEATURE_CRYPTO +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaesdq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesdq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vaesdq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaeseq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaeseq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vaeseq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaeseq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaesimcq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesimcq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vaesimcq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesimcq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaesmcq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesmcq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vaesmcq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesmcq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha1cq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1cq_u32((int8x16_t)__p0, __p1, (int8x16_t)__p2); + return __ret; +} +#else +__ai uint32x4_t vsha1cq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1cq_u32((int8x16_t)__rev0, __p1, (int8x16_t)__rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vsha1h_u32(uint32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vsha1h_u32(__p0); + return __ret; +} +#else +__ai uint32_t vsha1h_u32(uint32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vsha1h_u32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha1mq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1mq_u32((int8x16_t)__p0, __p1, (int8x16_t)__p2); + return __ret; +} +#else +__ai uint32x4_t vsha1mq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1mq_u32((int8x16_t)__rev0, __p1, (int8x16_t)__rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha1pq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1pq_u32((int8x16_t)__p0, __p1, (int8x16_t)__p2); + return __ret; +} +#else +__ai uint32x4_t vsha1pq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1pq_u32((int8x16_t)__rev0, __p1, (int8x16_t)__rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha1su0q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1su0q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 50); + return __ret; +} +#else +__ai uint32x4_t vsha1su0q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1su0q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha1su1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1su1q_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vsha1su1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1su1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha256hq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256hq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 50); + return __ret; +} +#else +__ai uint32x4_t vsha256hq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256hq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha256h2q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256h2q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 50); + return __ret; +} +#else +__ai uint32x4_t vsha256h2q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256h2q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha256su0q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256su0q_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vsha256su0q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256su0q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha256su1q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256su1q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 50); + return __ret; +} +#else +__ai uint32x4_t vsha256su1q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256su1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_FMA) +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vfmaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); + return __ret; +} +#else +__ai float32x4_t vfmaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vfmaq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float32x4_t __noswap_vfmaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vfma_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); + return __ret; +} +#else +__ai float32x2_t vfma_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vfma_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float32x2_t __noswap_vfma_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vfmsq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = vfmaq_f32(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float32x4_t vfmsq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __noswap_vfmaq_f32(__rev0, -__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vfms_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = vfma_f32(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float32x2_t vfms_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float32x2_t __ret; + __ret = __noswap_vfma_f32(__rev0, -__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vabdh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vabdh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vabdh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vabdh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vabsh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vabsh_f16(__p0); + return __ret; +} +#else +__ai float16_t vabsh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vabsh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vaddh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vaddh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vaddh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vaddh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcageh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcageh_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcageh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcageh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcagth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcagth_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcagth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcagth_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcaleh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcaleh_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcaleh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcaleh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcalth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcalth_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcalth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcalth_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vceqh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqh_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vceqh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vceqzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqzh_f16(__p0); + return __ret; +} +#else +__ai uint32_t vceqzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqzh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgeh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgeh_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcgeh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgeh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgezh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgezh_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcgezh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgezh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgth_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcgth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgth_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgtzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgtzh_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcgtzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgtzh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcleh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcleh_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcleh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcleh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vclezh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclezh_f16(__p0); + return __ret; +} +#else +__ai uint32_t vclezh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclezh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vclth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclth_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vclth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclth_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcltzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcltzh_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcltzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcltzh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvth_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s16_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvth_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvth_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s32_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvth_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvth_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s64_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvth_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvth_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u16_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvth_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvth_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u32_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvth_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvth_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u64_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvth_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtah_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s16_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtah_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtah_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s32_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtah_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtah_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s64_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtah_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtah_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u16_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtah_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtah_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u32_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtah_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtah_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u64_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtah_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_s32(int32_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s32(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_s32(int32_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_s64(int64_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s64(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_s64(int64_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_s16(int16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s16(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_s16(int16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_u32(uint32_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u32(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_u32(uint32_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_u64(uint64_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u64(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_u64(uint64_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_u16(uint16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u16(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_u16(uint16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtmh_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s16_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtmh_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtmh_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s32_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtmh_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtmh_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s64_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtmh_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtmh_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u16_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtmh_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtmh_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u32_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtmh_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtmh_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u64_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtmh_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtnh_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s16_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtnh_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtnh_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s32_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtnh_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtnh_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s64_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtnh_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtnh_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u16_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtnh_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtnh_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u32_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtnh_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtnh_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u64_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtnh_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtph_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s16_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtph_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtph_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s32_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtph_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtph_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s64_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtph_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtph_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u16_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtph_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtph_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u32_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtph_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtph_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u64_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtph_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vfmah_f16(float16_t __p0, float16_t __p1, float16_t __p2) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vfmah_f16(__p0, __p1, __p2); + return __ret; +} +#else +__ai float16_t vfmah_f16(float16_t __p0, float16_t __p1, float16_t __p2) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vfmah_f16(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmah_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vfmah_lane_f16(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmah_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vfmah_lane_f16(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmah_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vfmah_laneq_f16(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmah_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vfmah_laneq_f16(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vmaxh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxnmh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vmaxnmh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vminh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminnmh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vminnmh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmulh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vmulh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulh_f16(__p0, __p1); + return __ret; +} +__ai float16_t __noswap_vmulh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulh_lane_f16(__p0_0, __p1_0, __p2_0) __extension__ ({ \ + float16_t __s0_0 = __p0_0; \ + float16x4_t __s1_0 = __p1_0; \ + float16_t __ret_0; \ + __ret_0 = vmulh_f16(__s0_0, vget_lane_f16(__s1_0, __p2_0)); \ + __ret_0; \ +}) +#else +#define vmulh_lane_f16(__p0_1, __p1_1, __p2_1) __extension__ ({ \ + float16_t __s0_1 = __p0_1; \ + float16x4_t __s1_1 = __p1_1; \ + float16x4_t __rev1_1; __rev1_1 = __builtin_shufflevector(__s1_1, __s1_1, 3, 2, 1, 0); \ + float16_t __ret_1; \ + __ret_1 = __noswap_vmulh_f16(__s0_1, __noswap_vget_lane_f16(__rev1_1, __p2_1)); \ + __ret_1; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulh_laneq_f16(__p0_2, __p1_2, __p2_2) __extension__ ({ \ + float16_t __s0_2 = __p0_2; \ + float16x8_t __s1_2 = __p1_2; \ + float16_t __ret_2; \ + __ret_2 = vmulh_f16(__s0_2, vgetq_lane_f16(__s1_2, __p2_2)); \ + __ret_2; \ +}) +#else +#define vmulh_laneq_f16(__p0_3, __p1_3, __p2_3) __extension__ ({ \ + float16_t __s0_3 = __p0_3; \ + float16x8_t __s1_3 = __p1_3; \ + float16x8_t __rev1_3; __rev1_3 = __builtin_shufflevector(__s1_3, __s1_3, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret_3; \ + __ret_3 = __noswap_vmulh_f16(__s0_3, __noswap_vgetq_lane_f16(__rev1_3, __p2_3)); \ + __ret_3; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmulxh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulxh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vmulxh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulxh_f16(__p0, __p1); + return __ret; +} +__ai float16_t __noswap_vmulxh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulxh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxh_lane_f16(__p0_4, __p1_4, __p2_4) __extension__ ({ \ + float16_t __s0_4 = __p0_4; \ + float16x4_t __s1_4 = __p1_4; \ + float16_t __ret_4; \ + __ret_4 = vmulxh_f16(__s0_4, vget_lane_f16(__s1_4, __p2_4)); \ + __ret_4; \ +}) +#else +#define vmulxh_lane_f16(__p0_5, __p1_5, __p2_5) __extension__ ({ \ + float16_t __s0_5 = __p0_5; \ + float16x4_t __s1_5 = __p1_5; \ + float16x4_t __rev1_5; __rev1_5 = __builtin_shufflevector(__s1_5, __s1_5, 3, 2, 1, 0); \ + float16_t __ret_5; \ + __ret_5 = __noswap_vmulxh_f16(__s0_5, __noswap_vget_lane_f16(__rev1_5, __p2_5)); \ + __ret_5; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxh_laneq_f16(__p0_6, __p1_6, __p2_6) __extension__ ({ \ + float16_t __s0_6 = __p0_6; \ + float16x8_t __s1_6 = __p1_6; \ + float16_t __ret_6; \ + __ret_6 = vmulxh_f16(__s0_6, vgetq_lane_f16(__s1_6, __p2_6)); \ + __ret_6; \ +}) +#else +#define vmulxh_laneq_f16(__p0_7, __p1_7, __p2_7) __extension__ ({ \ + float16_t __s0_7 = __p0_7; \ + float16x8_t __s1_7 = __p1_7; \ + float16x8_t __rev1_7; __rev1_7 = __builtin_shufflevector(__s1_7, __s1_7, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret_7; \ + __ret_7 = __noswap_vmulxh_f16(__s0_7, __noswap_vgetq_lane_f16(__rev1_7, __p2_7)); \ + __ret_7; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vnegh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vnegh_f16(__p0); + return __ret; +} +#else +__ai float16_t vnegh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vnegh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrecpeh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpeh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrecpeh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpeh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrecpsh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpsh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vrecpsh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpsh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrecpxh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpxh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrecpxh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpxh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndah_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndah_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndah_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndah_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndih_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndih_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndih_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndih_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndmh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndmh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndmh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndmh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndnh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndnh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndnh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndnh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndph_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndph_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndph_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndph_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndxh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndxh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndxh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndxh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrsqrteh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrsqrteh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrsqrteh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrsqrteh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrsqrtsh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrsqrtsh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vrsqrtsh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrsqrtsh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vsubh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vsubh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vsubh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vsubh_f16(__p0, __p1); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) && defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vdivh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vdivh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vdivh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vdivh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vfmsh_f16(float16_t __p0, float16_t __p1, float16_t __p2) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vfmsh_f16(__p0, __p1, __p2); + return __ret; +} +#else +__ai float16_t vfmsh_f16(float16_t __p0, float16_t __p1, float16_t __p2) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vfmsh_f16(__p0, __p1, __p2); + return __ret; +} +__ai float16_t __noswap_vfmsh_f16(float16_t __p0, float16_t __p1, float16_t __p2) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vfmsh_f16(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vsqrth_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vsqrth_f16(__p0); + return __ret; +} +#else +__ai float16_t vsqrth_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vsqrth_f16(__p0); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vabdq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vabdq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vabd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vabd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vabsq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vabsq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vabs_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vabs_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vabs_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vabs_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vaddq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float16x8_t vaddq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vadd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float16x4_t vadd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vbslq_f16(uint16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 40); + return __ret; +} +#else +__ai float16x8_t vbslq_f16(uint16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vbsl_f16(uint16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 8); + return __ret; +} +#else +__ai float16x4_t vbsl_f16(uint16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcageq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcageq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vcageq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcageq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcage_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcage_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vcage_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcage_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcagtq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcagtq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vcagtq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcagtq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcagt_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcagt_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vcagt_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcagt_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcaleq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcaleq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vcaleq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcaleq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcale_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcale_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vcale_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcale_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcaltq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcaltq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vcaltq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcaltq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcalt_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcalt_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vcalt_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcalt_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x8_t vceqq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceq_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x4_t vceq_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgeq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x8_t vcgeq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcge_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x4_t vcge_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgtq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x8_t vcgtq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgt_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x4_t vcgt_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcleq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x8_t vcleq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcle_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x4_t vcle_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcltq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x8_t vcltq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclt_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x4_t vclt_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vcvtq_f16_u16(uint16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vcvtq_f16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai float16x8_t vcvtq_f16_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vcvtq_f16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vcvtq_f16_s16(int16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vcvtq_f16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai float16x8_t vcvtq_f16_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vcvtq_f16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vcvt_f16_u16(uint16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai float16x4_t vcvt_f16_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vcvt_f16_s16(int16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai float16x4_t vcvt_f16_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vcvtq_n_f16_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vcvtq_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vcvtq_n_f16_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vcvtq_n_f16_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vcvtq_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vcvtq_n_f16_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vcvt_n_f16_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vcvt_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vcvt_n_f16_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vcvt_n_f16_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vcvt_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vcvt_n_f16_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vcvtq_n_s16_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vcvtq_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vcvtq_n_s16_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vcvt_n_s16_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vcvt_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vcvt_n_s16_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vcvtq_n_u16_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vcvtq_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vcvtq_n_u16_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vcvt_n_u16_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vcvt_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vcvt_n_u16_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcvtq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtq_s16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vcvtq_s16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtq_s16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcvt_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvt_s16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcvt_s16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvt_s16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcvtq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtq_u16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcvtq_u16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtq_u16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcvt_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvt_u16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcvt_u16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvt_u16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcvtaq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtaq_s16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vcvtaq_s16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtaq_s16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcvta_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvta_s16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcvta_s16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvta_s16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcvtaq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtaq_u16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcvtaq_u16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtaq_u16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcvta_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvta_u16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcvta_u16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvta_u16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcvtmq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtmq_s16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vcvtmq_s16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtmq_s16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcvtm_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtm_s16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcvtm_s16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtm_s16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcvtmq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtmq_u16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcvtmq_u16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtmq_u16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcvtm_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtm_u16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcvtm_u16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtm_u16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcvtnq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtnq_s16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vcvtnq_s16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtnq_s16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcvtn_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtn_s16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcvtn_s16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtn_s16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcvtnq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtnq_u16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcvtnq_u16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtnq_u16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcvtn_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtn_u16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcvtn_u16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtn_u16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcvtpq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtpq_s16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vcvtpq_s16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtpq_s16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcvtp_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtp_s16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcvtp_s16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtp_s16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcvtpq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtpq_u16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcvtpq_u16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtpq_u16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcvtp_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtp_u16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcvtp_u16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtp_u16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vduph_lane_f16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vduph_lane_f16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vduph_laneq_f16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vduph_laneq_f16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 40); \ + __ret; \ +}) +#else +#define vextq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 8); \ + __ret; \ +}) +#else +#define vext_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vfmaq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 40); + return __ret; +} +#else +__ai float16x8_t vfmaq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vfmaq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai float16x8_t __noswap_vfmaq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 40); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vfma_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 8); + return __ret; +} +#else +__ai float16x4_t vfma_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vfma_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float16x4_t __noswap_vfma_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 8); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmaxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vmaxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmax_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vmax_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmaxnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vmaxnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmaxnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vmaxnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmaxnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vminq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vminq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmin_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vmin_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vminnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vminnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vminnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vminnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vminnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmulq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float16x8_t vmulq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmul_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float16x4_t vmul_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmulq_n_f16(float16x8_t __p0, float16_t __p1) { + float16x8_t __ret; + __ret = __p0 * (float16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai float16x8_t vmulq_n_f16(float16x8_t __p0, float16_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __rev0 * (float16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmul_n_f16(float16x4_t __p0, float16_t __p1) { + float16x4_t __ret; + __ret = __p0 * (float16x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai float16x4_t vmul_n_f16(float16x4_t __p0, float16_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __rev0 * (float16x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vnegq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float16x8_t vnegq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vneg_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float16x4_t vneg_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vpadd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vpadd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vpmax_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vpmax_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vpmin_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vpmin_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrecpeq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrecpeq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrecpeq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrecpeq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrecpe_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrecpe_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrecpe_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrecpe_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrecpsq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrecpsq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vrecpsq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrecpsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrecps_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrecps_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vrecps_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrecps_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrev64q_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai float16x8_t vrev64q_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrev64_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + return __ret; +} +#else +__ai float16x4_t vrev64_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrnd_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrnd_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrnd_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrnd_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndaq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndaq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndaq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndaq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrnda_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrnda_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrnda_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrnda_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndiq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndiq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndiq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndiq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrndi_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndi_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrndi_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndi_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndmq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndmq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndmq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndmq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrndm_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndm_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrndm_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndm_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndnq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndnq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndnq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndnq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrndn_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndn_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrndn_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndn_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndpq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndpq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndpq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndpq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrndp_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndp_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrndp_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndp_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndxq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndxq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndxq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndxq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrndx_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndx_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrndx_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndx_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrsqrteq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrsqrteq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrsqrteq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrsqrteq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrsqrte_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrsqrte_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrsqrte_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrsqrte_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrsqrtsq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vrsqrtsq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrsqrts_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrsqrts_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vrsqrts_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrsqrts_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vsubq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float16x8_t vsubq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vsub_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float16x4_t vsub_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8x2_t vtrnq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8x2_t vtrnq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 40); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4x2_t vtrn_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4x2_t vtrn_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 8); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vtrn1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai float16x8_t vtrn1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vtrn1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai float16x4_t vtrn1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vtrn2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai float16x8_t vtrn2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vtrn2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai float16x4_t vtrn2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8x2_t vuzpq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8x2_t vuzpq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 40); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4x2_t vuzp_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4x2_t vuzp_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 8); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vuzp1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai float16x8_t vuzp1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vuzp1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai float16x4_t vuzp1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vuzp2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai float16x8_t vuzp2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vuzp2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai float16x4_t vuzp2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8x2_t vzipq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8x2_t vzipq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 40); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4x2_t vzip_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4x2_t vzip_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 8); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vzip1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai float16x8_t vzip1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vzip1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai float16x4_t vzip1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vzip2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai float16x8_t vzip2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vzip2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai float16x4_t vzip2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqzq_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vceqzq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceqz_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vceqz_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgezq_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcgezq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgez_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcgez_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgez_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgtzq_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcgtzq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgtz_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcgtz_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgtz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vclezq_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vclezq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclez_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclez_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vclez_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclez_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcltzq_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcltzq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcltz_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcltz_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcltz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vdivq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float16x8_t vdivq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __rev0 / __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vdiv_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float16x4_t vdiv_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __rev0 / __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 40); \ + __ret; \ +}) +#else +#define vfmaq_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, __p3, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 40); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 8); \ + __ret; \ +}) +#else +#define vfma_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_lane_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, __p3, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfma_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 8); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 40); \ + __ret; \ +}) +#else +#define vfmaq_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, __p3, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 40); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 8); \ + __ret; \ +}) +#else +#define vfma_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_laneq_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x16_t)__rev2, __p3, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfma_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 8); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vfmaq_n_f16(float16x8_t __p0, float16x8_t __p1, float16_t __p2) { + float16x8_t __ret; + __ret = vfmaq_f16(__p0, __p1, (float16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float16x8_t vfmaq_n_f16(float16x8_t __p0, float16x8_t __p1, float16_t __p2) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __noswap_vfmaq_f16(__rev0, __rev1, (float16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vfma_n_f16(float16x4_t __p0, float16x4_t __p1, float16_t __p2) { + float16x4_t __ret; + __ret = vfma_f16(__p0, __p1, (float16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float16x4_t vfma_n_f16(float16x4_t __p0, float16x4_t __p1, float16_t __p2) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __noswap_vfma_f16(__rev0, __rev1, (float16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vfmsq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __ret; + __ret = vfmaq_f16(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float16x8_t vfmsq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __noswap_vfmaq_f16(__rev0, -__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vfms_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __ret; + __ret = vfma_f16(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float16x4_t vfms_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __noswap_vfma_f16(__rev0, -__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_lane_f16(__p0_8, __p1_8, __p2_8, __p3_8) __extension__ ({ \ + float16x8_t __s0_8 = __p0_8; \ + float16x8_t __s1_8 = __p1_8; \ + float16x4_t __s2_8 = __p2_8; \ + float16x8_t __ret_8; \ + __ret_8 = vfmaq_lane_f16(__s0_8, -__s1_8, __s2_8, __p3_8); \ + __ret_8; \ +}) +#else +#define vfmsq_lane_f16(__p0_9, __p1_9, __p2_9, __p3_9) __extension__ ({ \ + float16x8_t __s0_9 = __p0_9; \ + float16x8_t __s1_9 = __p1_9; \ + float16x4_t __s2_9 = __p2_9; \ + float16x8_t __rev0_9; __rev0_9 = __builtin_shufflevector(__s0_9, __s0_9, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1_9; __rev1_9 = __builtin_shufflevector(__s1_9, __s1_9, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __rev2_9; __rev2_9 = __builtin_shufflevector(__s2_9, __s2_9, 3, 2, 1, 0); \ + float16x8_t __ret_9; \ + __ret_9 = __noswap_vfmaq_lane_f16(__rev0_9, -__rev1_9, __rev2_9, __p3_9); \ + __ret_9 = __builtin_shufflevector(__ret_9, __ret_9, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_9; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_lane_f16(__p0_10, __p1_10, __p2_10, __p3_10) __extension__ ({ \ + float16x4_t __s0_10 = __p0_10; \ + float16x4_t __s1_10 = __p1_10; \ + float16x4_t __s2_10 = __p2_10; \ + float16x4_t __ret_10; \ + __ret_10 = vfma_lane_f16(__s0_10, -__s1_10, __s2_10, __p3_10); \ + __ret_10; \ +}) +#else +#define vfms_lane_f16(__p0_11, __p1_11, __p2_11, __p3_11) __extension__ ({ \ + float16x4_t __s0_11 = __p0_11; \ + float16x4_t __s1_11 = __p1_11; \ + float16x4_t __s2_11 = __p2_11; \ + float16x4_t __rev0_11; __rev0_11 = __builtin_shufflevector(__s0_11, __s0_11, 3, 2, 1, 0); \ + float16x4_t __rev1_11; __rev1_11 = __builtin_shufflevector(__s1_11, __s1_11, 3, 2, 1, 0); \ + float16x4_t __rev2_11; __rev2_11 = __builtin_shufflevector(__s2_11, __s2_11, 3, 2, 1, 0); \ + float16x4_t __ret_11; \ + __ret_11 = __noswap_vfma_lane_f16(__rev0_11, -__rev1_11, __rev2_11, __p3_11); \ + __ret_11 = __builtin_shufflevector(__ret_11, __ret_11, 3, 2, 1, 0); \ + __ret_11; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_laneq_f16(__p0_12, __p1_12, __p2_12, __p3_12) __extension__ ({ \ + float16x8_t __s0_12 = __p0_12; \ + float16x8_t __s1_12 = __p1_12; \ + float16x8_t __s2_12 = __p2_12; \ + float16x8_t __ret_12; \ + __ret_12 = vfmaq_laneq_f16(__s0_12, -__s1_12, __s2_12, __p3_12); \ + __ret_12; \ +}) +#else +#define vfmsq_laneq_f16(__p0_13, __p1_13, __p2_13, __p3_13) __extension__ ({ \ + float16x8_t __s0_13 = __p0_13; \ + float16x8_t __s1_13 = __p1_13; \ + float16x8_t __s2_13 = __p2_13; \ + float16x8_t __rev0_13; __rev0_13 = __builtin_shufflevector(__s0_13, __s0_13, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1_13; __rev1_13 = __builtin_shufflevector(__s1_13, __s1_13, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev2_13; __rev2_13 = __builtin_shufflevector(__s2_13, __s2_13, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret_13; \ + __ret_13 = __noswap_vfmaq_laneq_f16(__rev0_13, -__rev1_13, __rev2_13, __p3_13); \ + __ret_13 = __builtin_shufflevector(__ret_13, __ret_13, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_13; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_laneq_f16(__p0_14, __p1_14, __p2_14, __p3_14) __extension__ ({ \ + float16x4_t __s0_14 = __p0_14; \ + float16x4_t __s1_14 = __p1_14; \ + float16x8_t __s2_14 = __p2_14; \ + float16x4_t __ret_14; \ + __ret_14 = vfma_laneq_f16(__s0_14, -__s1_14, __s2_14, __p3_14); \ + __ret_14; \ +}) +#else +#define vfms_laneq_f16(__p0_15, __p1_15, __p2_15, __p3_15) __extension__ ({ \ + float16x4_t __s0_15 = __p0_15; \ + float16x4_t __s1_15 = __p1_15; \ + float16x8_t __s2_15 = __p2_15; \ + float16x4_t __rev0_15; __rev0_15 = __builtin_shufflevector(__s0_15, __s0_15, 3, 2, 1, 0); \ + float16x4_t __rev1_15; __rev1_15 = __builtin_shufflevector(__s1_15, __s1_15, 3, 2, 1, 0); \ + float16x8_t __rev2_15; __rev2_15 = __builtin_shufflevector(__s2_15, __s2_15, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __ret_15; \ + __ret_15 = __noswap_vfma_laneq_f16(__rev0_15, -__rev1_15, __rev2_15, __p3_15); \ + __ret_15 = __builtin_shufflevector(__ret_15, __ret_15, 3, 2, 1, 0); \ + __ret_15; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vfmsq_n_f16(float16x8_t __p0, float16x8_t __p1, float16_t __p2) { + float16x8_t __ret; + __ret = vfmaq_f16(__p0, -__p1, (float16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float16x8_t vfmsq_n_f16(float16x8_t __p0, float16x8_t __p1, float16_t __p2) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __noswap_vfmaq_f16(__rev0, -__rev1, (float16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vfms_n_f16(float16x4_t __p0, float16x4_t __p1, float16_t __p2) { + float16x4_t __ret; + __ret = vfma_f16(__p0, -__p1, (float16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float16x4_t vfms_n_f16(float16x4_t __p0, float16x4_t __p1, float16_t __p2) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __noswap_vfma_f16(__rev0, -__rev1, (float16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxnmvq_f16(float16x8_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmvq_f16((int8x16_t)__p0); + return __ret; +} +#else +__ai float16_t vmaxnmvq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmvq_f16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxnmv_f16(float16x4_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmv_f16((int8x8_t)__p0); + return __ret; +} +#else +__ai float16_t vmaxnmv_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmv_f16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxvq_f16(float16x8_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxvq_f16((int8x16_t)__p0); + return __ret; +} +#else +__ai float16_t vmaxvq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxvq_f16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxv_f16(float16x4_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxv_f16((int8x8_t)__p0); + return __ret; +} +#else +__ai float16_t vmaxv_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxv_f16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminnmvq_f16(float16x8_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmvq_f16((int8x16_t)__p0); + return __ret; +} +#else +__ai float16_t vminnmvq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmvq_f16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminnmv_f16(float16x4_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmv_f16((int8x8_t)__p0); + return __ret; +} +#else +__ai float16_t vminnmv_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmv_f16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminvq_f16(float16x8_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminvq_f16((int8x16_t)__p0); + return __ret; +} +#else +__ai float16_t vminvq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminvq_f16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminv_f16(float16x4_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminv_f16((int8x8_t)__p0); + return __ret; +} +#else +__ai float16_t vminv_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminv_f16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmulxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai float16x8_t __noswap_vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmulx_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vmulx_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmulx_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float16x4_t __noswap_vmulx_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = vmulxq_f16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __noswap_vmulxq_f16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = vmulx_f16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulx_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __noswap_vmulx_f16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = vmulxq_f16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __noswap_vmulxq_f16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = vmulx_f16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulx_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __noswap_vmulx_f16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmulxq_n_f16(float16x8_t __p0, float16_t __p1) { + float16x8_t __ret; + __ret = vmulxq_f16(__p0, (float16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}); + return __ret; +} +#else +__ai float16x8_t vmulxq_n_f16(float16x8_t __p0, float16_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __noswap_vmulxq_f16(__rev0, (float16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmulx_n_f16(float16x4_t __p0, float16_t __p1) { + float16x4_t __ret; + __ret = vmulx_f16(__p0, (float16x4_t) {__p1, __p1, __p1, __p1}); + return __ret; +} +#else +__ai float16x4_t vmulx_n_f16(float16x4_t __p0, float16_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __noswap_vmulx_f16(__rev0, (float16x4_t) {__p1, __p1, __p1, __p1}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vpmaxnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vpmaxnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vpmaxnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vpmaxnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmaxnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vpminnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vpminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vpminnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vpminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vpminnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vpminnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpminnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vsqrtq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vsqrtq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vsqrtq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vsqrtq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vsqrt_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vsqrt_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vsqrt_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vsqrt_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_QRDMX) +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqrdmlahq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = vqaddq_s32(__p0, vqrdmulhq_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vqrdmlahq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqaddq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqrdmlahq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = vqaddq_s16(__p0, vqrdmulhq_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vqrdmlahq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vqaddq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqrdmlah_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = vqadd_s32(__p0, vqrdmulh_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32x2_t vqrdmlah_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = __noswap_vqadd_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqrdmlah_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = vqadd_s16(__p0, vqrdmulh_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16x4_t vqrdmlah_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __noswap_vqadd_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqaddq_s32(__s0, vqrdmulhq_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlahq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqaddq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = vqaddq_s16(__s0, vqrdmulhq_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlahq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqaddq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlah_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = vqadd_s32(__s0, vqrdmulh_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlah_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqadd_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlah_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = vqadd_s16(__s0, vqrdmulh_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlah_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqadd_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqrdmlshq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = vqsubq_s32(__p0, vqrdmulhq_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vqrdmlshq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqsubq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqrdmlshq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = vqsubq_s16(__p0, vqrdmulhq_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vqrdmlshq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vqsubq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqrdmlsh_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = vqsub_s32(__p0, vqrdmulh_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32x2_t vqrdmlsh_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = __noswap_vqsub_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqrdmlsh_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = vqsub_s16(__p0, vqrdmulh_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16x4_t vqrdmlsh_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __noswap_vqsub_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqsubq_s32(__s0, vqrdmulhq_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlshq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqsubq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = vqsubq_s16(__s0, vqrdmulhq_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlshq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqsubq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlsh_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = vqsub_s32(__s0, vqrdmulh_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlsh_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqsub_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlsh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = vqsub_s16(__s0, vqrdmulh_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlsh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqsub_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#endif +#if defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqaddq_s32(__s0, vqrdmulhq_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlahq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqaddq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = vqaddq_s16(__s0, vqrdmulhq_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlahq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqaddq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlah_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = vqadd_s32(__s0, vqrdmulh_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlah_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqadd_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlah_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = vqadd_s16(__s0, vqrdmulh_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlah_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqadd_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqsubq_s32(__s0, vqrdmulhq_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlshq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqsubq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = vqsubq_s16(__s0, vqrdmulhq_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlshq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqsubq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlsh_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = vqsub_s32(__s0, vqrdmulh_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlsh_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqsub_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlsh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = vqsub_s16(__s0, vqrdmulh_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlsh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqsub_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#endif +#if defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vabdq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vabdq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vabd_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vabd_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vabdd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vabdd_f64(__p0, __p1); + return __ret; +} +#else +__ai float64_t vabdd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vabdd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vabds_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vabds_f32(__p0, __p1); + return __ret; +} +#else +__ai float32_t vabds_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vabds_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vabsq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vabsq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vabsq_s64(int64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vabsq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vabs_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vabs_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vabs_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vabs_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vabs_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vabs_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vabs_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vabs_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vabsd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vabsd_s64(__p0); + return __ret; +} +#else +__ai int64_t vabsd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vabsd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vaddq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float64x2_t vaddq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vadd_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float64x1_t vadd_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vaddhn_u32(__p1, __p2)); + return __ret; +} +#else +__ai uint16x8_t vaddhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vaddhn_u32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vaddhn_u64(__p1, __p2)); + return __ret; +} +#else +__ai uint32x4_t vaddhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vaddhn_u64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaddhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vaddhn_u16(__p1, __p2)); + return __ret; +} +#else +__ai uint8x16_t vaddhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vaddhn_u16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vaddhn_s32(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vaddhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vaddhn_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vaddhn_s64(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vaddhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vaddhn_s64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vaddhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vaddhn_s16(__p1, __p2)); + return __ret; +} +#else +__ai int8x16_t vaddhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vaddhn_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vaddlvq_u8(uint8x16_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddlvq_u8((int8x16_t)__p0); + return __ret; +} +#else +__ai uint16_t vaddlvq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddlvq_u8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vaddlvq_u32(uint32x4_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddlvq_u32((int8x16_t)__p0); + return __ret; +} +#else +__ai uint64_t vaddlvq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddlvq_u32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vaddlvq_u16(uint16x8_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddlvq_u16((int8x16_t)__p0); + return __ret; +} +#else +__ai uint32_t vaddlvq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddlvq_u16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vaddlvq_s8(int8x16_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddlvq_s8((int8x16_t)__p0); + return __ret; +} +#else +__ai int16_t vaddlvq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddlvq_s8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vaddlvq_s32(int32x4_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddlvq_s32((int8x16_t)__p0); + return __ret; +} +#else +__ai int64_t vaddlvq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddlvq_s32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vaddlvq_s16(int16x8_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddlvq_s16((int8x16_t)__p0); + return __ret; +} +#else +__ai int32_t vaddlvq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddlvq_s16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vaddlv_u8(uint8x8_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddlv_u8((int8x8_t)__p0); + return __ret; +} +#else +__ai uint16_t vaddlv_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddlv_u8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vaddlv_u32(uint32x2_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddlv_u32((int8x8_t)__p0); + return __ret; +} +#else +__ai uint64_t vaddlv_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddlv_u32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vaddlv_u16(uint16x4_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddlv_u16((int8x8_t)__p0); + return __ret; +} +#else +__ai uint32_t vaddlv_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddlv_u16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vaddlv_s8(int8x8_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddlv_s8((int8x8_t)__p0); + return __ret; +} +#else +__ai int16_t vaddlv_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddlv_s8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vaddlv_s32(int32x2_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddlv_s32((int8x8_t)__p0); + return __ret; +} +#else +__ai int64_t vaddlv_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddlv_s32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vaddlv_s16(int16x4_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddlv_s16((int8x8_t)__p0); + return __ret; +} +#else +__ai int32_t vaddlv_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddlv_s16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vaddvq_u8(uint8x16_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vaddvq_u8((int8x16_t)__p0); + return __ret; +} +#else +__ai uint8_t vaddvq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vaddvq_u8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vaddvq_u32(uint32x4_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddvq_u32((int8x16_t)__p0); + return __ret; +} +#else +__ai uint32_t vaddvq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddvq_u32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vaddvq_u64(uint64x2_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddvq_u64((int8x16_t)__p0); + return __ret; +} +#else +__ai uint64_t vaddvq_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddvq_u64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vaddvq_u16(uint16x8_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddvq_u16((int8x16_t)__p0); + return __ret; +} +#else +__ai uint16_t vaddvq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddvq_u16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vaddvq_s8(int8x16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vaddvq_s8((int8x16_t)__p0); + return __ret; +} +#else +__ai int8_t vaddvq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vaddvq_s8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vaddvq_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vaddvq_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vaddvq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vaddvq_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vaddvq_f32(float32x4_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vaddvq_f32((int8x16_t)__p0); + return __ret; +} +#else +__ai float32_t vaddvq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vaddvq_f32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vaddvq_s32(int32x4_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddvq_s32((int8x16_t)__p0); + return __ret; +} +#else +__ai int32_t vaddvq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddvq_s32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vaddvq_s64(int64x2_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddvq_s64((int8x16_t)__p0); + return __ret; +} +#else +__ai int64_t vaddvq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddvq_s64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vaddvq_s16(int16x8_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddvq_s16((int8x16_t)__p0); + return __ret; +} +#else +__ai int16_t vaddvq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddvq_s16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vaddv_u8(uint8x8_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vaddv_u8((int8x8_t)__p0); + return __ret; +} +#else +__ai uint8_t vaddv_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vaddv_u8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vaddv_u32(uint32x2_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddv_u32((int8x8_t)__p0); + return __ret; +} +#else +__ai uint32_t vaddv_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddv_u32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vaddv_u16(uint16x4_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddv_u16((int8x8_t)__p0); + return __ret; +} +#else +__ai uint16_t vaddv_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddv_u16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vaddv_s8(int8x8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vaddv_s8((int8x8_t)__p0); + return __ret; +} +#else +__ai int8_t vaddv_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vaddv_s8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vaddv_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vaddv_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vaddv_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vaddv_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vaddv_s32(int32x2_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddv_s32((int8x8_t)__p0); + return __ret; +} +#else +__ai int32_t vaddv_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddv_s32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vaddv_s16(int16x4_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddv_s16((int8x8_t)__p0); + return __ret; +} +#else +__ai int16_t vaddv_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddv_s16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vbsl_p64(uint64x1_t __p0, poly64x1_t __p1, poly64x1_t __p2) { + poly64x1_t __ret; + __ret = (poly64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 6); + return __ret; +} +#else +__ai poly64x1_t vbsl_p64(uint64x1_t __p0, poly64x1_t __p1, poly64x1_t __p2) { + poly64x1_t __ret; + __ret = (poly64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 6); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vbslq_p64(uint64x2_t __p0, poly64x2_t __p1, poly64x2_t __p2) { + poly64x2_t __ret; + __ret = (poly64x2_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 38); + return __ret; +} +#else +__ai poly64x2_t vbslq_p64(uint64x2_t __p0, poly64x2_t __p1, poly64x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + poly64x2_t __ret; + __ret = (poly64x2_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 38); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vbslq_f64(uint64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42); + return __ret; +} +#else +__ai float64x2_t vbslq_f64(uint64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vbsl_f64(uint64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10); + return __ret; +} +#else +__ai float64x1_t vbsl_f64(uint64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcageq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcageq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vcageq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcageq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcage_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcage_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vcage_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcage_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcaged_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaged_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcaged_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaged_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcages_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcages_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcages_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcages_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcagtq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcagtq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vcagtq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcagtq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcagt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcagt_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vcagt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcagt_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcagtd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcagtd_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcagtd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcagtd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcagts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcagts_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcagts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcagts_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcaleq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcaleq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vcaleq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcaleq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcale_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcale_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vcale_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcale_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcaled_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaled_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcaled_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaled_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcales_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcales_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcales_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcales_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcaltq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcaltq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vcaltq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcaltq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcalt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcalt_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vcalt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcalt_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcaltd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaltd_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcaltd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaltd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcalts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcalts_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcalts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcalts_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceq_p64(poly64x1_t __p0, poly64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x1_t vceq_p64(poly64x1_t __p0, poly64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqq_p64(poly64x2_t __p0, poly64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x2_t vceqq_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x2_t vceqq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x2_t vceqq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x2_t vceqq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceq_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x1_t vceq_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceq_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x1_t vceq_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceq_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x1_t vceq_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vceqd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vceqd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vceqd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vceqd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vceqd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vceqd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vceqd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqd_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vceqd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vceqs_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqs_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vceqs_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqs_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceqz_p8(poly8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vceqz_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceqz_p64(poly64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vceqz_p64(poly64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceqz_p16(poly16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vceqz_p16(poly16x4_t __p0) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqzq_p8(poly8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vceqzq_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqzq_p64(poly64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vceqzq_p64(poly64x2_t __p0) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqzq_p16(poly16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vceqzq_p16(poly16x8_t __p0) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqzq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vceqzq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqzq_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vceqzq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqzq_u64(uint64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vceqzq_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqzq_u16(uint16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vceqzq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqzq_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vceqzq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqzq_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vceqzq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqzq_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vceqzq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqzq_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vceqzq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqzq_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vceqzq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqzq_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vceqzq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceqz_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vceqz_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceqz_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vceqz_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceqz_u64(uint64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vceqz_u64(uint64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceqz_u16(uint16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vceqz_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceqz_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vceqz_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceqz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vceqz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceqz_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vceqz_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceqz_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vceqz_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceqz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vceqz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceqz_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vceqz_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vceqzd_u64(uint64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqzd_u64(__p0); + return __ret; +} +#else +__ai uint64_t vceqzd_u64(uint64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqzd_u64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vceqzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vceqzd_s64(__p0); + return __ret; +} +#else +__ai int64_t vceqzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vceqzd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vceqzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqzd_f64(__p0); + return __ret; +} +#else +__ai uint64_t vceqzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqzd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vceqzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqzs_f32(__p0); + return __ret; +} +#else +__ai uint32_t vceqzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqzs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgeq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x2_t vcgeq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgeq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x2_t vcgeq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgeq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x2_t vcgeq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcge_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x1_t vcge_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcge_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x1_t vcge_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcge_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x1_t vcge_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcged_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcged_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vcged_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcged_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcged_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcged_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcged_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcged_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcged_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcged_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcged_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcged_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcges_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcges_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcges_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcges_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgezq_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vcgezq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgezq_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcgezq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgezq_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcgezq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgezq_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcgezq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgezq_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcgezq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgezq_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcgezq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcgez_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vcgez_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcgez_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgez_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcgez_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgez_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcgez_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgez_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgez_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcgez_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgez_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgez_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcgez_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgez_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcgez_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgez_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcgezd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgezd_s64(__p0); + return __ret; +} +#else +__ai int64_t vcgezd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgezd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcgezd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgezd_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcgezd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgezd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgezs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgezs_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcgezs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgezs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgtq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x2_t vcgtq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgtq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x2_t vcgtq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgtq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x2_t vcgtq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgt_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x1_t vcgt_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x1_t vcgt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgt_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x1_t vcgt_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcgtd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgtd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vcgtd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgtd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcgtd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcgtd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcgtd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtd_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcgtd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgts_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcgts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgts_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgtzq_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vcgtzq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgtzq_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcgtzq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgtzq_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcgtzq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgtzq_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcgtzq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgtzq_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcgtzq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgtzq_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcgtzq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcgtz_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vcgtz_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcgtz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgtz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcgtz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgtz_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcgtz_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgtz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgtz_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcgtz_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgtz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgtz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcgtz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgtz_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcgtz_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgtz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcgtzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgtzd_s64(__p0); + return __ret; +} +#else +__ai int64_t vcgtzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgtzd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcgtzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtzd_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcgtzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtzd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgtzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgtzs_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcgtzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgtzs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcleq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x2_t vcleq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcleq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x2_t vcleq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcleq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x2_t vcleq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcle_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x1_t vcle_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcle_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x1_t vcle_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcle_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x1_t vcle_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcled_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcled_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcled_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcled_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcled_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcled_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vcled_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcled_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcled_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcled_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcled_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcled_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcles_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcles_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcles_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcles_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vclezq_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vclezq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vclezq_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vclezq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vclezq_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vclezq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vclezq_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vclezq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vclezq_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vclezq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vclezq_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vclezq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vclez_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vclez_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vclez_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vclez_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vclez_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vclez_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vclez_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vclez_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclez_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclez_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vclez_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclez_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclez_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclez_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vclez_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclez_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vclez_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vclez_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vclez_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vclez_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclez_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclez_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vclez_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclez_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vclezd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vclezd_s64(__p0); + return __ret; +} +#else +__ai int64_t vclezd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vclezd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vclezd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vclezd_f64(__p0); + return __ret; +} +#else +__ai uint64_t vclezd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vclezd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vclezs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclezs_f32(__p0); + return __ret; +} +#else +__ai uint32_t vclezs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclezs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcltq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x2_t vcltq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcltq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x2_t vcltq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcltq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x2_t vcltq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vclt_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x1_t vclt_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vclt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x1_t vclt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vclt_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x1_t vclt_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcltd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcltd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcltd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcltd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vcltd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcltd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcltd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltd_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcltd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vclts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclts_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vclts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclts_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcltzq_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vcltzq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcltzq_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcltzq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcltzq_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcltzq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcltzq_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcltzq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcltzq_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcltzq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcltzq_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcltzq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcltz_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vcltz_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcltz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcltz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcltz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcltz_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcltz_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcltz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcltz_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcltz_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcltz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcltz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcltz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcltz_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcltz_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcltz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcltzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcltzd_s64(__p0); + return __ret; +} +#else +__ai int64_t vcltzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcltzd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcltzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltzd_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcltzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltzd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcltzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcltzs_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcltzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcltzs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vcombine_p64(poly64x1_t __p0, poly64x1_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + return __ret; +} +#else +__ai poly64x2_t vcombine_p64(poly64x1_t __p0, poly64x1_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vcombine_f64(float64x1_t __p0, float64x1_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + return __ret; +} +#else +__ai float64x2_t vcombine_f64(float64x1_t __p0, float64x1_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_p8(__p0_16, __p1_16, __p2_16, __p3_16) __extension__ ({ \ + poly8x16_t __s0_16 = __p0_16; \ + poly8x8_t __s2_16 = __p2_16; \ + poly8x16_t __ret_16; \ + __ret_16 = vsetq_lane_p8(vget_lane_p8(__s2_16, __p3_16), __s0_16, __p1_16); \ + __ret_16; \ +}) +#else +#define vcopyq_lane_p8(__p0_17, __p1_17, __p2_17, __p3_17) __extension__ ({ \ + poly8x16_t __s0_17 = __p0_17; \ + poly8x8_t __s2_17 = __p2_17; \ + poly8x16_t __rev0_17; __rev0_17 = __builtin_shufflevector(__s0_17, __s0_17, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __rev2_17; __rev2_17 = __builtin_shufflevector(__s2_17, __s2_17, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret_17; \ + __ret_17 = __noswap_vsetq_lane_p8(__noswap_vget_lane_p8(__rev2_17, __p3_17), __rev0_17, __p1_17); \ + __ret_17 = __builtin_shufflevector(__ret_17, __ret_17, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_17; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_p16(__p0_18, __p1_18, __p2_18, __p3_18) __extension__ ({ \ + poly16x8_t __s0_18 = __p0_18; \ + poly16x4_t __s2_18 = __p2_18; \ + poly16x8_t __ret_18; \ + __ret_18 = vsetq_lane_p16(vget_lane_p16(__s2_18, __p3_18), __s0_18, __p1_18); \ + __ret_18; \ +}) +#else +#define vcopyq_lane_p16(__p0_19, __p1_19, __p2_19, __p3_19) __extension__ ({ \ + poly16x8_t __s0_19 = __p0_19; \ + poly16x4_t __s2_19 = __p2_19; \ + poly16x8_t __rev0_19; __rev0_19 = __builtin_shufflevector(__s0_19, __s0_19, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x4_t __rev2_19; __rev2_19 = __builtin_shufflevector(__s2_19, __s2_19, 3, 2, 1, 0); \ + poly16x8_t __ret_19; \ + __ret_19 = __noswap_vsetq_lane_p16(__noswap_vget_lane_p16(__rev2_19, __p3_19), __rev0_19, __p1_19); \ + __ret_19 = __builtin_shufflevector(__ret_19, __ret_19, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_19; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u8(__p0_20, __p1_20, __p2_20, __p3_20) __extension__ ({ \ + uint8x16_t __s0_20 = __p0_20; \ + uint8x8_t __s2_20 = __p2_20; \ + uint8x16_t __ret_20; \ + __ret_20 = vsetq_lane_u8(vget_lane_u8(__s2_20, __p3_20), __s0_20, __p1_20); \ + __ret_20; \ +}) +#else +#define vcopyq_lane_u8(__p0_21, __p1_21, __p2_21, __p3_21) __extension__ ({ \ + uint8x16_t __s0_21 = __p0_21; \ + uint8x8_t __s2_21 = __p2_21; \ + uint8x16_t __rev0_21; __rev0_21 = __builtin_shufflevector(__s0_21, __s0_21, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev2_21; __rev2_21 = __builtin_shufflevector(__s2_21, __s2_21, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_21; \ + __ret_21 = __noswap_vsetq_lane_u8(__noswap_vget_lane_u8(__rev2_21, __p3_21), __rev0_21, __p1_21); \ + __ret_21 = __builtin_shufflevector(__ret_21, __ret_21, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_21; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u32(__p0_22, __p1_22, __p2_22, __p3_22) __extension__ ({ \ + uint32x4_t __s0_22 = __p0_22; \ + uint32x2_t __s2_22 = __p2_22; \ + uint32x4_t __ret_22; \ + __ret_22 = vsetq_lane_u32(vget_lane_u32(__s2_22, __p3_22), __s0_22, __p1_22); \ + __ret_22; \ +}) +#else +#define vcopyq_lane_u32(__p0_23, __p1_23, __p2_23, __p3_23) __extension__ ({ \ + uint32x4_t __s0_23 = __p0_23; \ + uint32x2_t __s2_23 = __p2_23; \ + uint32x4_t __rev0_23; __rev0_23 = __builtin_shufflevector(__s0_23, __s0_23, 3, 2, 1, 0); \ + uint32x2_t __rev2_23; __rev2_23 = __builtin_shufflevector(__s2_23, __s2_23, 1, 0); \ + uint32x4_t __ret_23; \ + __ret_23 = __noswap_vsetq_lane_u32(__noswap_vget_lane_u32(__rev2_23, __p3_23), __rev0_23, __p1_23); \ + __ret_23 = __builtin_shufflevector(__ret_23, __ret_23, 3, 2, 1, 0); \ + __ret_23; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u64(__p0_24, __p1_24, __p2_24, __p3_24) __extension__ ({ \ + uint64x2_t __s0_24 = __p0_24; \ + uint64x1_t __s2_24 = __p2_24; \ + uint64x2_t __ret_24; \ + __ret_24 = vsetq_lane_u64(vget_lane_u64(__s2_24, __p3_24), __s0_24, __p1_24); \ + __ret_24; \ +}) +#else +#define vcopyq_lane_u64(__p0_25, __p1_25, __p2_25, __p3_25) __extension__ ({ \ + uint64x2_t __s0_25 = __p0_25; \ + uint64x1_t __s2_25 = __p2_25; \ + uint64x2_t __rev0_25; __rev0_25 = __builtin_shufflevector(__s0_25, __s0_25, 1, 0); \ + uint64x2_t __ret_25; \ + __ret_25 = __noswap_vsetq_lane_u64(__noswap_vget_lane_u64(__s2_25, __p3_25), __rev0_25, __p1_25); \ + __ret_25 = __builtin_shufflevector(__ret_25, __ret_25, 1, 0); \ + __ret_25; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u16(__p0_26, __p1_26, __p2_26, __p3_26) __extension__ ({ \ + uint16x8_t __s0_26 = __p0_26; \ + uint16x4_t __s2_26 = __p2_26; \ + uint16x8_t __ret_26; \ + __ret_26 = vsetq_lane_u16(vget_lane_u16(__s2_26, __p3_26), __s0_26, __p1_26); \ + __ret_26; \ +}) +#else +#define vcopyq_lane_u16(__p0_27, __p1_27, __p2_27, __p3_27) __extension__ ({ \ + uint16x8_t __s0_27 = __p0_27; \ + uint16x4_t __s2_27 = __p2_27; \ + uint16x8_t __rev0_27; __rev0_27 = __builtin_shufflevector(__s0_27, __s0_27, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev2_27; __rev2_27 = __builtin_shufflevector(__s2_27, __s2_27, 3, 2, 1, 0); \ + uint16x8_t __ret_27; \ + __ret_27 = __noswap_vsetq_lane_u16(__noswap_vget_lane_u16(__rev2_27, __p3_27), __rev0_27, __p1_27); \ + __ret_27 = __builtin_shufflevector(__ret_27, __ret_27, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_27; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_s8(__p0_28, __p1_28, __p2_28, __p3_28) __extension__ ({ \ + int8x16_t __s0_28 = __p0_28; \ + int8x8_t __s2_28 = __p2_28; \ + int8x16_t __ret_28; \ + __ret_28 = vsetq_lane_s8(vget_lane_s8(__s2_28, __p3_28), __s0_28, __p1_28); \ + __ret_28; \ +}) +#else +#define vcopyq_lane_s8(__p0_29, __p1_29, __p2_29, __p3_29) __extension__ ({ \ + int8x16_t __s0_29 = __p0_29; \ + int8x8_t __s2_29 = __p2_29; \ + int8x16_t __rev0_29; __rev0_29 = __builtin_shufflevector(__s0_29, __s0_29, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev2_29; __rev2_29 = __builtin_shufflevector(__s2_29, __s2_29, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_29; \ + __ret_29 = __noswap_vsetq_lane_s8(__noswap_vget_lane_s8(__rev2_29, __p3_29), __rev0_29, __p1_29); \ + __ret_29 = __builtin_shufflevector(__ret_29, __ret_29, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_29; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_f32(__p0_30, __p1_30, __p2_30, __p3_30) __extension__ ({ \ + float32x4_t __s0_30 = __p0_30; \ + float32x2_t __s2_30 = __p2_30; \ + float32x4_t __ret_30; \ + __ret_30 = vsetq_lane_f32(vget_lane_f32(__s2_30, __p3_30), __s0_30, __p1_30); \ + __ret_30; \ +}) +#else +#define vcopyq_lane_f32(__p0_31, __p1_31, __p2_31, __p3_31) __extension__ ({ \ + float32x4_t __s0_31 = __p0_31; \ + float32x2_t __s2_31 = __p2_31; \ + float32x4_t __rev0_31; __rev0_31 = __builtin_shufflevector(__s0_31, __s0_31, 3, 2, 1, 0); \ + float32x2_t __rev2_31; __rev2_31 = __builtin_shufflevector(__s2_31, __s2_31, 1, 0); \ + float32x4_t __ret_31; \ + __ret_31 = __noswap_vsetq_lane_f32(__noswap_vget_lane_f32(__rev2_31, __p3_31), __rev0_31, __p1_31); \ + __ret_31 = __builtin_shufflevector(__ret_31, __ret_31, 3, 2, 1, 0); \ + __ret_31; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_s32(__p0_32, __p1_32, __p2_32, __p3_32) __extension__ ({ \ + int32x4_t __s0_32 = __p0_32; \ + int32x2_t __s2_32 = __p2_32; \ + int32x4_t __ret_32; \ + __ret_32 = vsetq_lane_s32(vget_lane_s32(__s2_32, __p3_32), __s0_32, __p1_32); \ + __ret_32; \ +}) +#else +#define vcopyq_lane_s32(__p0_33, __p1_33, __p2_33, __p3_33) __extension__ ({ \ + int32x4_t __s0_33 = __p0_33; \ + int32x2_t __s2_33 = __p2_33; \ + int32x4_t __rev0_33; __rev0_33 = __builtin_shufflevector(__s0_33, __s0_33, 3, 2, 1, 0); \ + int32x2_t __rev2_33; __rev2_33 = __builtin_shufflevector(__s2_33, __s2_33, 1, 0); \ + int32x4_t __ret_33; \ + __ret_33 = __noswap_vsetq_lane_s32(__noswap_vget_lane_s32(__rev2_33, __p3_33), __rev0_33, __p1_33); \ + __ret_33 = __builtin_shufflevector(__ret_33, __ret_33, 3, 2, 1, 0); \ + __ret_33; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_s64(__p0_34, __p1_34, __p2_34, __p3_34) __extension__ ({ \ + int64x2_t __s0_34 = __p0_34; \ + int64x1_t __s2_34 = __p2_34; \ + int64x2_t __ret_34; \ + __ret_34 = vsetq_lane_s64(vget_lane_s64(__s2_34, __p3_34), __s0_34, __p1_34); \ + __ret_34; \ +}) +#else +#define vcopyq_lane_s64(__p0_35, __p1_35, __p2_35, __p3_35) __extension__ ({ \ + int64x2_t __s0_35 = __p0_35; \ + int64x1_t __s2_35 = __p2_35; \ + int64x2_t __rev0_35; __rev0_35 = __builtin_shufflevector(__s0_35, __s0_35, 1, 0); \ + int64x2_t __ret_35; \ + __ret_35 = __noswap_vsetq_lane_s64(__noswap_vget_lane_s64(__s2_35, __p3_35), __rev0_35, __p1_35); \ + __ret_35 = __builtin_shufflevector(__ret_35, __ret_35, 1, 0); \ + __ret_35; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_s16(__p0_36, __p1_36, __p2_36, __p3_36) __extension__ ({ \ + int16x8_t __s0_36 = __p0_36; \ + int16x4_t __s2_36 = __p2_36; \ + int16x8_t __ret_36; \ + __ret_36 = vsetq_lane_s16(vget_lane_s16(__s2_36, __p3_36), __s0_36, __p1_36); \ + __ret_36; \ +}) +#else +#define vcopyq_lane_s16(__p0_37, __p1_37, __p2_37, __p3_37) __extension__ ({ \ + int16x8_t __s0_37 = __p0_37; \ + int16x4_t __s2_37 = __p2_37; \ + int16x8_t __rev0_37; __rev0_37 = __builtin_shufflevector(__s0_37, __s0_37, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2_37; __rev2_37 = __builtin_shufflevector(__s2_37, __s2_37, 3, 2, 1, 0); \ + int16x8_t __ret_37; \ + __ret_37 = __noswap_vsetq_lane_s16(__noswap_vget_lane_s16(__rev2_37, __p3_37), __rev0_37, __p1_37); \ + __ret_37 = __builtin_shufflevector(__ret_37, __ret_37, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_37; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_p8(__p0_38, __p1_38, __p2_38, __p3_38) __extension__ ({ \ + poly8x8_t __s0_38 = __p0_38; \ + poly8x8_t __s2_38 = __p2_38; \ + poly8x8_t __ret_38; \ + __ret_38 = vset_lane_p8(vget_lane_p8(__s2_38, __p3_38), __s0_38, __p1_38); \ + __ret_38; \ +}) +#else +#define vcopy_lane_p8(__p0_39, __p1_39, __p2_39, __p3_39) __extension__ ({ \ + poly8x8_t __s0_39 = __p0_39; \ + poly8x8_t __s2_39 = __p2_39; \ + poly8x8_t __rev0_39; __rev0_39 = __builtin_shufflevector(__s0_39, __s0_39, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __rev2_39; __rev2_39 = __builtin_shufflevector(__s2_39, __s2_39, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret_39; \ + __ret_39 = __noswap_vset_lane_p8(__noswap_vget_lane_p8(__rev2_39, __p3_39), __rev0_39, __p1_39); \ + __ret_39 = __builtin_shufflevector(__ret_39, __ret_39, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_39; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_p16(__p0_40, __p1_40, __p2_40, __p3_40) __extension__ ({ \ + poly16x4_t __s0_40 = __p0_40; \ + poly16x4_t __s2_40 = __p2_40; \ + poly16x4_t __ret_40; \ + __ret_40 = vset_lane_p16(vget_lane_p16(__s2_40, __p3_40), __s0_40, __p1_40); \ + __ret_40; \ +}) +#else +#define vcopy_lane_p16(__p0_41, __p1_41, __p2_41, __p3_41) __extension__ ({ \ + poly16x4_t __s0_41 = __p0_41; \ + poly16x4_t __s2_41 = __p2_41; \ + poly16x4_t __rev0_41; __rev0_41 = __builtin_shufflevector(__s0_41, __s0_41, 3, 2, 1, 0); \ + poly16x4_t __rev2_41; __rev2_41 = __builtin_shufflevector(__s2_41, __s2_41, 3, 2, 1, 0); \ + poly16x4_t __ret_41; \ + __ret_41 = __noswap_vset_lane_p16(__noswap_vget_lane_p16(__rev2_41, __p3_41), __rev0_41, __p1_41); \ + __ret_41 = __builtin_shufflevector(__ret_41, __ret_41, 3, 2, 1, 0); \ + __ret_41; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_u8(__p0_42, __p1_42, __p2_42, __p3_42) __extension__ ({ \ + uint8x8_t __s0_42 = __p0_42; \ + uint8x8_t __s2_42 = __p2_42; \ + uint8x8_t __ret_42; \ + __ret_42 = vset_lane_u8(vget_lane_u8(__s2_42, __p3_42), __s0_42, __p1_42); \ + __ret_42; \ +}) +#else +#define vcopy_lane_u8(__p0_43, __p1_43, __p2_43, __p3_43) __extension__ ({ \ + uint8x8_t __s0_43 = __p0_43; \ + uint8x8_t __s2_43 = __p2_43; \ + uint8x8_t __rev0_43; __rev0_43 = __builtin_shufflevector(__s0_43, __s0_43, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev2_43; __rev2_43 = __builtin_shufflevector(__s2_43, __s2_43, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret_43; \ + __ret_43 = __noswap_vset_lane_u8(__noswap_vget_lane_u8(__rev2_43, __p3_43), __rev0_43, __p1_43); \ + __ret_43 = __builtin_shufflevector(__ret_43, __ret_43, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_43; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_u32(__p0_44, __p1_44, __p2_44, __p3_44) __extension__ ({ \ + uint32x2_t __s0_44 = __p0_44; \ + uint32x2_t __s2_44 = __p2_44; \ + uint32x2_t __ret_44; \ + __ret_44 = vset_lane_u32(vget_lane_u32(__s2_44, __p3_44), __s0_44, __p1_44); \ + __ret_44; \ +}) +#else +#define vcopy_lane_u32(__p0_45, __p1_45, __p2_45, __p3_45) __extension__ ({ \ + uint32x2_t __s0_45 = __p0_45; \ + uint32x2_t __s2_45 = __p2_45; \ + uint32x2_t __rev0_45; __rev0_45 = __builtin_shufflevector(__s0_45, __s0_45, 1, 0); \ + uint32x2_t __rev2_45; __rev2_45 = __builtin_shufflevector(__s2_45, __s2_45, 1, 0); \ + uint32x2_t __ret_45; \ + __ret_45 = __noswap_vset_lane_u32(__noswap_vget_lane_u32(__rev2_45, __p3_45), __rev0_45, __p1_45); \ + __ret_45 = __builtin_shufflevector(__ret_45, __ret_45, 1, 0); \ + __ret_45; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_u64(__p0_46, __p1_46, __p2_46, __p3_46) __extension__ ({ \ + uint64x1_t __s0_46 = __p0_46; \ + uint64x1_t __s2_46 = __p2_46; \ + uint64x1_t __ret_46; \ + __ret_46 = vset_lane_u64(vget_lane_u64(__s2_46, __p3_46), __s0_46, __p1_46); \ + __ret_46; \ +}) +#else +#define vcopy_lane_u64(__p0_47, __p1_47, __p2_47, __p3_47) __extension__ ({ \ + uint64x1_t __s0_47 = __p0_47; \ + uint64x1_t __s2_47 = __p2_47; \ + uint64x1_t __ret_47; \ + __ret_47 = __noswap_vset_lane_u64(__noswap_vget_lane_u64(__s2_47, __p3_47), __s0_47, __p1_47); \ + __ret_47; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_u16(__p0_48, __p1_48, __p2_48, __p3_48) __extension__ ({ \ + uint16x4_t __s0_48 = __p0_48; \ + uint16x4_t __s2_48 = __p2_48; \ + uint16x4_t __ret_48; \ + __ret_48 = vset_lane_u16(vget_lane_u16(__s2_48, __p3_48), __s0_48, __p1_48); \ + __ret_48; \ +}) +#else +#define vcopy_lane_u16(__p0_49, __p1_49, __p2_49, __p3_49) __extension__ ({ \ + uint16x4_t __s0_49 = __p0_49; \ + uint16x4_t __s2_49 = __p2_49; \ + uint16x4_t __rev0_49; __rev0_49 = __builtin_shufflevector(__s0_49, __s0_49, 3, 2, 1, 0); \ + uint16x4_t __rev2_49; __rev2_49 = __builtin_shufflevector(__s2_49, __s2_49, 3, 2, 1, 0); \ + uint16x4_t __ret_49; \ + __ret_49 = __noswap_vset_lane_u16(__noswap_vget_lane_u16(__rev2_49, __p3_49), __rev0_49, __p1_49); \ + __ret_49 = __builtin_shufflevector(__ret_49, __ret_49, 3, 2, 1, 0); \ + __ret_49; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_s8(__p0_50, __p1_50, __p2_50, __p3_50) __extension__ ({ \ + int8x8_t __s0_50 = __p0_50; \ + int8x8_t __s2_50 = __p2_50; \ + int8x8_t __ret_50; \ + __ret_50 = vset_lane_s8(vget_lane_s8(__s2_50, __p3_50), __s0_50, __p1_50); \ + __ret_50; \ +}) +#else +#define vcopy_lane_s8(__p0_51, __p1_51, __p2_51, __p3_51) __extension__ ({ \ + int8x8_t __s0_51 = __p0_51; \ + int8x8_t __s2_51 = __p2_51; \ + int8x8_t __rev0_51; __rev0_51 = __builtin_shufflevector(__s0_51, __s0_51, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev2_51; __rev2_51 = __builtin_shufflevector(__s2_51, __s2_51, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret_51; \ + __ret_51 = __noswap_vset_lane_s8(__noswap_vget_lane_s8(__rev2_51, __p3_51), __rev0_51, __p1_51); \ + __ret_51 = __builtin_shufflevector(__ret_51, __ret_51, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_51; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_f32(__p0_52, __p1_52, __p2_52, __p3_52) __extension__ ({ \ + float32x2_t __s0_52 = __p0_52; \ + float32x2_t __s2_52 = __p2_52; \ + float32x2_t __ret_52; \ + __ret_52 = vset_lane_f32(vget_lane_f32(__s2_52, __p3_52), __s0_52, __p1_52); \ + __ret_52; \ +}) +#else +#define vcopy_lane_f32(__p0_53, __p1_53, __p2_53, __p3_53) __extension__ ({ \ + float32x2_t __s0_53 = __p0_53; \ + float32x2_t __s2_53 = __p2_53; \ + float32x2_t __rev0_53; __rev0_53 = __builtin_shufflevector(__s0_53, __s0_53, 1, 0); \ + float32x2_t __rev2_53; __rev2_53 = __builtin_shufflevector(__s2_53, __s2_53, 1, 0); \ + float32x2_t __ret_53; \ + __ret_53 = __noswap_vset_lane_f32(__noswap_vget_lane_f32(__rev2_53, __p3_53), __rev0_53, __p1_53); \ + __ret_53 = __builtin_shufflevector(__ret_53, __ret_53, 1, 0); \ + __ret_53; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_s32(__p0_54, __p1_54, __p2_54, __p3_54) __extension__ ({ \ + int32x2_t __s0_54 = __p0_54; \ + int32x2_t __s2_54 = __p2_54; \ + int32x2_t __ret_54; \ + __ret_54 = vset_lane_s32(vget_lane_s32(__s2_54, __p3_54), __s0_54, __p1_54); \ + __ret_54; \ +}) +#else +#define vcopy_lane_s32(__p0_55, __p1_55, __p2_55, __p3_55) __extension__ ({ \ + int32x2_t __s0_55 = __p0_55; \ + int32x2_t __s2_55 = __p2_55; \ + int32x2_t __rev0_55; __rev0_55 = __builtin_shufflevector(__s0_55, __s0_55, 1, 0); \ + int32x2_t __rev2_55; __rev2_55 = __builtin_shufflevector(__s2_55, __s2_55, 1, 0); \ + int32x2_t __ret_55; \ + __ret_55 = __noswap_vset_lane_s32(__noswap_vget_lane_s32(__rev2_55, __p3_55), __rev0_55, __p1_55); \ + __ret_55 = __builtin_shufflevector(__ret_55, __ret_55, 1, 0); \ + __ret_55; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_s64(__p0_56, __p1_56, __p2_56, __p3_56) __extension__ ({ \ + int64x1_t __s0_56 = __p0_56; \ + int64x1_t __s2_56 = __p2_56; \ + int64x1_t __ret_56; \ + __ret_56 = vset_lane_s64(vget_lane_s64(__s2_56, __p3_56), __s0_56, __p1_56); \ + __ret_56; \ +}) +#else +#define vcopy_lane_s64(__p0_57, __p1_57, __p2_57, __p3_57) __extension__ ({ \ + int64x1_t __s0_57 = __p0_57; \ + int64x1_t __s2_57 = __p2_57; \ + int64x1_t __ret_57; \ + __ret_57 = __noswap_vset_lane_s64(__noswap_vget_lane_s64(__s2_57, __p3_57), __s0_57, __p1_57); \ + __ret_57; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_s16(__p0_58, __p1_58, __p2_58, __p3_58) __extension__ ({ \ + int16x4_t __s0_58 = __p0_58; \ + int16x4_t __s2_58 = __p2_58; \ + int16x4_t __ret_58; \ + __ret_58 = vset_lane_s16(vget_lane_s16(__s2_58, __p3_58), __s0_58, __p1_58); \ + __ret_58; \ +}) +#else +#define vcopy_lane_s16(__p0_59, __p1_59, __p2_59, __p3_59) __extension__ ({ \ + int16x4_t __s0_59 = __p0_59; \ + int16x4_t __s2_59 = __p2_59; \ + int16x4_t __rev0_59; __rev0_59 = __builtin_shufflevector(__s0_59, __s0_59, 3, 2, 1, 0); \ + int16x4_t __rev2_59; __rev2_59 = __builtin_shufflevector(__s2_59, __s2_59, 3, 2, 1, 0); \ + int16x4_t __ret_59; \ + __ret_59 = __noswap_vset_lane_s16(__noswap_vget_lane_s16(__rev2_59, __p3_59), __rev0_59, __p1_59); \ + __ret_59 = __builtin_shufflevector(__ret_59, __ret_59, 3, 2, 1, 0); \ + __ret_59; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_p8(__p0_60, __p1_60, __p2_60, __p3_60) __extension__ ({ \ + poly8x16_t __s0_60 = __p0_60; \ + poly8x16_t __s2_60 = __p2_60; \ + poly8x16_t __ret_60; \ + __ret_60 = vsetq_lane_p8(vgetq_lane_p8(__s2_60, __p3_60), __s0_60, __p1_60); \ + __ret_60; \ +}) +#else +#define vcopyq_laneq_p8(__p0_61, __p1_61, __p2_61, __p3_61) __extension__ ({ \ + poly8x16_t __s0_61 = __p0_61; \ + poly8x16_t __s2_61 = __p2_61; \ + poly8x16_t __rev0_61; __rev0_61 = __builtin_shufflevector(__s0_61, __s0_61, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __rev2_61; __rev2_61 = __builtin_shufflevector(__s2_61, __s2_61, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret_61; \ + __ret_61 = __noswap_vsetq_lane_p8(__noswap_vgetq_lane_p8(__rev2_61, __p3_61), __rev0_61, __p1_61); \ + __ret_61 = __builtin_shufflevector(__ret_61, __ret_61, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_61; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_p16(__p0_62, __p1_62, __p2_62, __p3_62) __extension__ ({ \ + poly16x8_t __s0_62 = __p0_62; \ + poly16x8_t __s2_62 = __p2_62; \ + poly16x8_t __ret_62; \ + __ret_62 = vsetq_lane_p16(vgetq_lane_p16(__s2_62, __p3_62), __s0_62, __p1_62); \ + __ret_62; \ +}) +#else +#define vcopyq_laneq_p16(__p0_63, __p1_63, __p2_63, __p3_63) __extension__ ({ \ + poly16x8_t __s0_63 = __p0_63; \ + poly16x8_t __s2_63 = __p2_63; \ + poly16x8_t __rev0_63; __rev0_63 = __builtin_shufflevector(__s0_63, __s0_63, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __rev2_63; __rev2_63 = __builtin_shufflevector(__s2_63, __s2_63, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret_63; \ + __ret_63 = __noswap_vsetq_lane_p16(__noswap_vgetq_lane_p16(__rev2_63, __p3_63), __rev0_63, __p1_63); \ + __ret_63 = __builtin_shufflevector(__ret_63, __ret_63, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_63; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_u8(__p0_64, __p1_64, __p2_64, __p3_64) __extension__ ({ \ + uint8x16_t __s0_64 = __p0_64; \ + uint8x16_t __s2_64 = __p2_64; \ + uint8x16_t __ret_64; \ + __ret_64 = vsetq_lane_u8(vgetq_lane_u8(__s2_64, __p3_64), __s0_64, __p1_64); \ + __ret_64; \ +}) +#else +#define vcopyq_laneq_u8(__p0_65, __p1_65, __p2_65, __p3_65) __extension__ ({ \ + uint8x16_t __s0_65 = __p0_65; \ + uint8x16_t __s2_65 = __p2_65; \ + uint8x16_t __rev0_65; __rev0_65 = __builtin_shufflevector(__s0_65, __s0_65, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev2_65; __rev2_65 = __builtin_shufflevector(__s2_65, __s2_65, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_65; \ + __ret_65 = __noswap_vsetq_lane_u8(__noswap_vgetq_lane_u8(__rev2_65, __p3_65), __rev0_65, __p1_65); \ + __ret_65 = __builtin_shufflevector(__ret_65, __ret_65, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_65; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_u32(__p0_66, __p1_66, __p2_66, __p3_66) __extension__ ({ \ + uint32x4_t __s0_66 = __p0_66; \ + uint32x4_t __s2_66 = __p2_66; \ + uint32x4_t __ret_66; \ + __ret_66 = vsetq_lane_u32(vgetq_lane_u32(__s2_66, __p3_66), __s0_66, __p1_66); \ + __ret_66; \ +}) +#else +#define vcopyq_laneq_u32(__p0_67, __p1_67, __p2_67, __p3_67) __extension__ ({ \ + uint32x4_t __s0_67 = __p0_67; \ + uint32x4_t __s2_67 = __p2_67; \ + uint32x4_t __rev0_67; __rev0_67 = __builtin_shufflevector(__s0_67, __s0_67, 3, 2, 1, 0); \ + uint32x4_t __rev2_67; __rev2_67 = __builtin_shufflevector(__s2_67, __s2_67, 3, 2, 1, 0); \ + uint32x4_t __ret_67; \ + __ret_67 = __noswap_vsetq_lane_u32(__noswap_vgetq_lane_u32(__rev2_67, __p3_67), __rev0_67, __p1_67); \ + __ret_67 = __builtin_shufflevector(__ret_67, __ret_67, 3, 2, 1, 0); \ + __ret_67; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_u64(__p0_68, __p1_68, __p2_68, __p3_68) __extension__ ({ \ + uint64x2_t __s0_68 = __p0_68; \ + uint64x2_t __s2_68 = __p2_68; \ + uint64x2_t __ret_68; \ + __ret_68 = vsetq_lane_u64(vgetq_lane_u64(__s2_68, __p3_68), __s0_68, __p1_68); \ + __ret_68; \ +}) +#else +#define vcopyq_laneq_u64(__p0_69, __p1_69, __p2_69, __p3_69) __extension__ ({ \ + uint64x2_t __s0_69 = __p0_69; \ + uint64x2_t __s2_69 = __p2_69; \ + uint64x2_t __rev0_69; __rev0_69 = __builtin_shufflevector(__s0_69, __s0_69, 1, 0); \ + uint64x2_t __rev2_69; __rev2_69 = __builtin_shufflevector(__s2_69, __s2_69, 1, 0); \ + uint64x2_t __ret_69; \ + __ret_69 = __noswap_vsetq_lane_u64(__noswap_vgetq_lane_u64(__rev2_69, __p3_69), __rev0_69, __p1_69); \ + __ret_69 = __builtin_shufflevector(__ret_69, __ret_69, 1, 0); \ + __ret_69; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_u16(__p0_70, __p1_70, __p2_70, __p3_70) __extension__ ({ \ + uint16x8_t __s0_70 = __p0_70; \ + uint16x8_t __s2_70 = __p2_70; \ + uint16x8_t __ret_70; \ + __ret_70 = vsetq_lane_u16(vgetq_lane_u16(__s2_70, __p3_70), __s0_70, __p1_70); \ + __ret_70; \ +}) +#else +#define vcopyq_laneq_u16(__p0_71, __p1_71, __p2_71, __p3_71) __extension__ ({ \ + uint16x8_t __s0_71 = __p0_71; \ + uint16x8_t __s2_71 = __p2_71; \ + uint16x8_t __rev0_71; __rev0_71 = __builtin_shufflevector(__s0_71, __s0_71, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev2_71; __rev2_71 = __builtin_shufflevector(__s2_71, __s2_71, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret_71; \ + __ret_71 = __noswap_vsetq_lane_u16(__noswap_vgetq_lane_u16(__rev2_71, __p3_71), __rev0_71, __p1_71); \ + __ret_71 = __builtin_shufflevector(__ret_71, __ret_71, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_71; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_s8(__p0_72, __p1_72, __p2_72, __p3_72) __extension__ ({ \ + int8x16_t __s0_72 = __p0_72; \ + int8x16_t __s2_72 = __p2_72; \ + int8x16_t __ret_72; \ + __ret_72 = vsetq_lane_s8(vgetq_lane_s8(__s2_72, __p3_72), __s0_72, __p1_72); \ + __ret_72; \ +}) +#else +#define vcopyq_laneq_s8(__p0_73, __p1_73, __p2_73, __p3_73) __extension__ ({ \ + int8x16_t __s0_73 = __p0_73; \ + int8x16_t __s2_73 = __p2_73; \ + int8x16_t __rev0_73; __rev0_73 = __builtin_shufflevector(__s0_73, __s0_73, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev2_73; __rev2_73 = __builtin_shufflevector(__s2_73, __s2_73, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_73; \ + __ret_73 = __noswap_vsetq_lane_s8(__noswap_vgetq_lane_s8(__rev2_73, __p3_73), __rev0_73, __p1_73); \ + __ret_73 = __builtin_shufflevector(__ret_73, __ret_73, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_73; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_f32(__p0_74, __p1_74, __p2_74, __p3_74) __extension__ ({ \ + float32x4_t __s0_74 = __p0_74; \ + float32x4_t __s2_74 = __p2_74; \ + float32x4_t __ret_74; \ + __ret_74 = vsetq_lane_f32(vgetq_lane_f32(__s2_74, __p3_74), __s0_74, __p1_74); \ + __ret_74; \ +}) +#else +#define vcopyq_laneq_f32(__p0_75, __p1_75, __p2_75, __p3_75) __extension__ ({ \ + float32x4_t __s0_75 = __p0_75; \ + float32x4_t __s2_75 = __p2_75; \ + float32x4_t __rev0_75; __rev0_75 = __builtin_shufflevector(__s0_75, __s0_75, 3, 2, 1, 0); \ + float32x4_t __rev2_75; __rev2_75 = __builtin_shufflevector(__s2_75, __s2_75, 3, 2, 1, 0); \ + float32x4_t __ret_75; \ + __ret_75 = __noswap_vsetq_lane_f32(__noswap_vgetq_lane_f32(__rev2_75, __p3_75), __rev0_75, __p1_75); \ + __ret_75 = __builtin_shufflevector(__ret_75, __ret_75, 3, 2, 1, 0); \ + __ret_75; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_s32(__p0_76, __p1_76, __p2_76, __p3_76) __extension__ ({ \ + int32x4_t __s0_76 = __p0_76; \ + int32x4_t __s2_76 = __p2_76; \ + int32x4_t __ret_76; \ + __ret_76 = vsetq_lane_s32(vgetq_lane_s32(__s2_76, __p3_76), __s0_76, __p1_76); \ + __ret_76; \ +}) +#else +#define vcopyq_laneq_s32(__p0_77, __p1_77, __p2_77, __p3_77) __extension__ ({ \ + int32x4_t __s0_77 = __p0_77; \ + int32x4_t __s2_77 = __p2_77; \ + int32x4_t __rev0_77; __rev0_77 = __builtin_shufflevector(__s0_77, __s0_77, 3, 2, 1, 0); \ + int32x4_t __rev2_77; __rev2_77 = __builtin_shufflevector(__s2_77, __s2_77, 3, 2, 1, 0); \ + int32x4_t __ret_77; \ + __ret_77 = __noswap_vsetq_lane_s32(__noswap_vgetq_lane_s32(__rev2_77, __p3_77), __rev0_77, __p1_77); \ + __ret_77 = __builtin_shufflevector(__ret_77, __ret_77, 3, 2, 1, 0); \ + __ret_77; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_s64(__p0_78, __p1_78, __p2_78, __p3_78) __extension__ ({ \ + int64x2_t __s0_78 = __p0_78; \ + int64x2_t __s2_78 = __p2_78; \ + int64x2_t __ret_78; \ + __ret_78 = vsetq_lane_s64(vgetq_lane_s64(__s2_78, __p3_78), __s0_78, __p1_78); \ + __ret_78; \ +}) +#else +#define vcopyq_laneq_s64(__p0_79, __p1_79, __p2_79, __p3_79) __extension__ ({ \ + int64x2_t __s0_79 = __p0_79; \ + int64x2_t __s2_79 = __p2_79; \ + int64x2_t __rev0_79; __rev0_79 = __builtin_shufflevector(__s0_79, __s0_79, 1, 0); \ + int64x2_t __rev2_79; __rev2_79 = __builtin_shufflevector(__s2_79, __s2_79, 1, 0); \ + int64x2_t __ret_79; \ + __ret_79 = __noswap_vsetq_lane_s64(__noswap_vgetq_lane_s64(__rev2_79, __p3_79), __rev0_79, __p1_79); \ + __ret_79 = __builtin_shufflevector(__ret_79, __ret_79, 1, 0); \ + __ret_79; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_s16(__p0_80, __p1_80, __p2_80, __p3_80) __extension__ ({ \ + int16x8_t __s0_80 = __p0_80; \ + int16x8_t __s2_80 = __p2_80; \ + int16x8_t __ret_80; \ + __ret_80 = vsetq_lane_s16(vgetq_lane_s16(__s2_80, __p3_80), __s0_80, __p1_80); \ + __ret_80; \ +}) +#else +#define vcopyq_laneq_s16(__p0_81, __p1_81, __p2_81, __p3_81) __extension__ ({ \ + int16x8_t __s0_81 = __p0_81; \ + int16x8_t __s2_81 = __p2_81; \ + int16x8_t __rev0_81; __rev0_81 = __builtin_shufflevector(__s0_81, __s0_81, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2_81; __rev2_81 = __builtin_shufflevector(__s2_81, __s2_81, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret_81; \ + __ret_81 = __noswap_vsetq_lane_s16(__noswap_vgetq_lane_s16(__rev2_81, __p3_81), __rev0_81, __p1_81); \ + __ret_81 = __builtin_shufflevector(__ret_81, __ret_81, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_81; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_p8(__p0_82, __p1_82, __p2_82, __p3_82) __extension__ ({ \ + poly8x8_t __s0_82 = __p0_82; \ + poly8x16_t __s2_82 = __p2_82; \ + poly8x8_t __ret_82; \ + __ret_82 = vset_lane_p8(vgetq_lane_p8(__s2_82, __p3_82), __s0_82, __p1_82); \ + __ret_82; \ +}) +#else +#define vcopy_laneq_p8(__p0_83, __p1_83, __p2_83, __p3_83) __extension__ ({ \ + poly8x8_t __s0_83 = __p0_83; \ + poly8x16_t __s2_83 = __p2_83; \ + poly8x8_t __rev0_83; __rev0_83 = __builtin_shufflevector(__s0_83, __s0_83, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __rev2_83; __rev2_83 = __builtin_shufflevector(__s2_83, __s2_83, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret_83; \ + __ret_83 = __noswap_vset_lane_p8(__noswap_vgetq_lane_p8(__rev2_83, __p3_83), __rev0_83, __p1_83); \ + __ret_83 = __builtin_shufflevector(__ret_83, __ret_83, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_83; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_p16(__p0_84, __p1_84, __p2_84, __p3_84) __extension__ ({ \ + poly16x4_t __s0_84 = __p0_84; \ + poly16x8_t __s2_84 = __p2_84; \ + poly16x4_t __ret_84; \ + __ret_84 = vset_lane_p16(vgetq_lane_p16(__s2_84, __p3_84), __s0_84, __p1_84); \ + __ret_84; \ +}) +#else +#define vcopy_laneq_p16(__p0_85, __p1_85, __p2_85, __p3_85) __extension__ ({ \ + poly16x4_t __s0_85 = __p0_85; \ + poly16x8_t __s2_85 = __p2_85; \ + poly16x4_t __rev0_85; __rev0_85 = __builtin_shufflevector(__s0_85, __s0_85, 3, 2, 1, 0); \ + poly16x8_t __rev2_85; __rev2_85 = __builtin_shufflevector(__s2_85, __s2_85, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x4_t __ret_85; \ + __ret_85 = __noswap_vset_lane_p16(__noswap_vgetq_lane_p16(__rev2_85, __p3_85), __rev0_85, __p1_85); \ + __ret_85 = __builtin_shufflevector(__ret_85, __ret_85, 3, 2, 1, 0); \ + __ret_85; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_u8(__p0_86, __p1_86, __p2_86, __p3_86) __extension__ ({ \ + uint8x8_t __s0_86 = __p0_86; \ + uint8x16_t __s2_86 = __p2_86; \ + uint8x8_t __ret_86; \ + __ret_86 = vset_lane_u8(vgetq_lane_u8(__s2_86, __p3_86), __s0_86, __p1_86); \ + __ret_86; \ +}) +#else +#define vcopy_laneq_u8(__p0_87, __p1_87, __p2_87, __p3_87) __extension__ ({ \ + uint8x8_t __s0_87 = __p0_87; \ + uint8x16_t __s2_87 = __p2_87; \ + uint8x8_t __rev0_87; __rev0_87 = __builtin_shufflevector(__s0_87, __s0_87, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev2_87; __rev2_87 = __builtin_shufflevector(__s2_87, __s2_87, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret_87; \ + __ret_87 = __noswap_vset_lane_u8(__noswap_vgetq_lane_u8(__rev2_87, __p3_87), __rev0_87, __p1_87); \ + __ret_87 = __builtin_shufflevector(__ret_87, __ret_87, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_87; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_u32(__p0_88, __p1_88, __p2_88, __p3_88) __extension__ ({ \ + uint32x2_t __s0_88 = __p0_88; \ + uint32x4_t __s2_88 = __p2_88; \ + uint32x2_t __ret_88; \ + __ret_88 = vset_lane_u32(vgetq_lane_u32(__s2_88, __p3_88), __s0_88, __p1_88); \ + __ret_88; \ +}) +#else +#define vcopy_laneq_u32(__p0_89, __p1_89, __p2_89, __p3_89) __extension__ ({ \ + uint32x2_t __s0_89 = __p0_89; \ + uint32x4_t __s2_89 = __p2_89; \ + uint32x2_t __rev0_89; __rev0_89 = __builtin_shufflevector(__s0_89, __s0_89, 1, 0); \ + uint32x4_t __rev2_89; __rev2_89 = __builtin_shufflevector(__s2_89, __s2_89, 3, 2, 1, 0); \ + uint32x2_t __ret_89; \ + __ret_89 = __noswap_vset_lane_u32(__noswap_vgetq_lane_u32(__rev2_89, __p3_89), __rev0_89, __p1_89); \ + __ret_89 = __builtin_shufflevector(__ret_89, __ret_89, 1, 0); \ + __ret_89; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_u64(__p0_90, __p1_90, __p2_90, __p3_90) __extension__ ({ \ + uint64x1_t __s0_90 = __p0_90; \ + uint64x2_t __s2_90 = __p2_90; \ + uint64x1_t __ret_90; \ + __ret_90 = vset_lane_u64(vgetq_lane_u64(__s2_90, __p3_90), __s0_90, __p1_90); \ + __ret_90; \ +}) +#else +#define vcopy_laneq_u64(__p0_91, __p1_91, __p2_91, __p3_91) __extension__ ({ \ + uint64x1_t __s0_91 = __p0_91; \ + uint64x2_t __s2_91 = __p2_91; \ + uint64x2_t __rev2_91; __rev2_91 = __builtin_shufflevector(__s2_91, __s2_91, 1, 0); \ + uint64x1_t __ret_91; \ + __ret_91 = __noswap_vset_lane_u64(__noswap_vgetq_lane_u64(__rev2_91, __p3_91), __s0_91, __p1_91); \ + __ret_91; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_u16(__p0_92, __p1_92, __p2_92, __p3_92) __extension__ ({ \ + uint16x4_t __s0_92 = __p0_92; \ + uint16x8_t __s2_92 = __p2_92; \ + uint16x4_t __ret_92; \ + __ret_92 = vset_lane_u16(vgetq_lane_u16(__s2_92, __p3_92), __s0_92, __p1_92); \ + __ret_92; \ +}) +#else +#define vcopy_laneq_u16(__p0_93, __p1_93, __p2_93, __p3_93) __extension__ ({ \ + uint16x4_t __s0_93 = __p0_93; \ + uint16x8_t __s2_93 = __p2_93; \ + uint16x4_t __rev0_93; __rev0_93 = __builtin_shufflevector(__s0_93, __s0_93, 3, 2, 1, 0); \ + uint16x8_t __rev2_93; __rev2_93 = __builtin_shufflevector(__s2_93, __s2_93, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __ret_93; \ + __ret_93 = __noswap_vset_lane_u16(__noswap_vgetq_lane_u16(__rev2_93, __p3_93), __rev0_93, __p1_93); \ + __ret_93 = __builtin_shufflevector(__ret_93, __ret_93, 3, 2, 1, 0); \ + __ret_93; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_s8(__p0_94, __p1_94, __p2_94, __p3_94) __extension__ ({ \ + int8x8_t __s0_94 = __p0_94; \ + int8x16_t __s2_94 = __p2_94; \ + int8x8_t __ret_94; \ + __ret_94 = vset_lane_s8(vgetq_lane_s8(__s2_94, __p3_94), __s0_94, __p1_94); \ + __ret_94; \ +}) +#else +#define vcopy_laneq_s8(__p0_95, __p1_95, __p2_95, __p3_95) __extension__ ({ \ + int8x8_t __s0_95 = __p0_95; \ + int8x16_t __s2_95 = __p2_95; \ + int8x8_t __rev0_95; __rev0_95 = __builtin_shufflevector(__s0_95, __s0_95, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev2_95; __rev2_95 = __builtin_shufflevector(__s2_95, __s2_95, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret_95; \ + __ret_95 = __noswap_vset_lane_s8(__noswap_vgetq_lane_s8(__rev2_95, __p3_95), __rev0_95, __p1_95); \ + __ret_95 = __builtin_shufflevector(__ret_95, __ret_95, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_95; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_f32(__p0_96, __p1_96, __p2_96, __p3_96) __extension__ ({ \ + float32x2_t __s0_96 = __p0_96; \ + float32x4_t __s2_96 = __p2_96; \ + float32x2_t __ret_96; \ + __ret_96 = vset_lane_f32(vgetq_lane_f32(__s2_96, __p3_96), __s0_96, __p1_96); \ + __ret_96; \ +}) +#else +#define vcopy_laneq_f32(__p0_97, __p1_97, __p2_97, __p3_97) __extension__ ({ \ + float32x2_t __s0_97 = __p0_97; \ + float32x4_t __s2_97 = __p2_97; \ + float32x2_t __rev0_97; __rev0_97 = __builtin_shufflevector(__s0_97, __s0_97, 1, 0); \ + float32x4_t __rev2_97; __rev2_97 = __builtin_shufflevector(__s2_97, __s2_97, 3, 2, 1, 0); \ + float32x2_t __ret_97; \ + __ret_97 = __noswap_vset_lane_f32(__noswap_vgetq_lane_f32(__rev2_97, __p3_97), __rev0_97, __p1_97); \ + __ret_97 = __builtin_shufflevector(__ret_97, __ret_97, 1, 0); \ + __ret_97; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_s32(__p0_98, __p1_98, __p2_98, __p3_98) __extension__ ({ \ + int32x2_t __s0_98 = __p0_98; \ + int32x4_t __s2_98 = __p2_98; \ + int32x2_t __ret_98; \ + __ret_98 = vset_lane_s32(vgetq_lane_s32(__s2_98, __p3_98), __s0_98, __p1_98); \ + __ret_98; \ +}) +#else +#define vcopy_laneq_s32(__p0_99, __p1_99, __p2_99, __p3_99) __extension__ ({ \ + int32x2_t __s0_99 = __p0_99; \ + int32x4_t __s2_99 = __p2_99; \ + int32x2_t __rev0_99; __rev0_99 = __builtin_shufflevector(__s0_99, __s0_99, 1, 0); \ + int32x4_t __rev2_99; __rev2_99 = __builtin_shufflevector(__s2_99, __s2_99, 3, 2, 1, 0); \ + int32x2_t __ret_99; \ + __ret_99 = __noswap_vset_lane_s32(__noswap_vgetq_lane_s32(__rev2_99, __p3_99), __rev0_99, __p1_99); \ + __ret_99 = __builtin_shufflevector(__ret_99, __ret_99, 1, 0); \ + __ret_99; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_s64(__p0_100, __p1_100, __p2_100, __p3_100) __extension__ ({ \ + int64x1_t __s0_100 = __p0_100; \ + int64x2_t __s2_100 = __p2_100; \ + int64x1_t __ret_100; \ + __ret_100 = vset_lane_s64(vgetq_lane_s64(__s2_100, __p3_100), __s0_100, __p1_100); \ + __ret_100; \ +}) +#else +#define vcopy_laneq_s64(__p0_101, __p1_101, __p2_101, __p3_101) __extension__ ({ \ + int64x1_t __s0_101 = __p0_101; \ + int64x2_t __s2_101 = __p2_101; \ + int64x2_t __rev2_101; __rev2_101 = __builtin_shufflevector(__s2_101, __s2_101, 1, 0); \ + int64x1_t __ret_101; \ + __ret_101 = __noswap_vset_lane_s64(__noswap_vgetq_lane_s64(__rev2_101, __p3_101), __s0_101, __p1_101); \ + __ret_101; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_s16(__p0_102, __p1_102, __p2_102, __p3_102) __extension__ ({ \ + int16x4_t __s0_102 = __p0_102; \ + int16x8_t __s2_102 = __p2_102; \ + int16x4_t __ret_102; \ + __ret_102 = vset_lane_s16(vgetq_lane_s16(__s2_102, __p3_102), __s0_102, __p1_102); \ + __ret_102; \ +}) +#else +#define vcopy_laneq_s16(__p0_103, __p1_103, __p2_103, __p3_103) __extension__ ({ \ + int16x4_t __s0_103 = __p0_103; \ + int16x8_t __s2_103 = __p2_103; \ + int16x4_t __rev0_103; __rev0_103 = __builtin_shufflevector(__s0_103, __s0_103, 3, 2, 1, 0); \ + int16x8_t __rev2_103; __rev2_103 = __builtin_shufflevector(__s2_103, __s2_103, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret_103; \ + __ret_103 = __noswap_vset_lane_s16(__noswap_vgetq_lane_s16(__rev2_103, __p3_103), __rev0_103, __p1_103); \ + __ret_103 = __builtin_shufflevector(__ret_103, __ret_103, 3, 2, 1, 0); \ + __ret_103; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vcreate_p64(uint64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vcreate_p64(uint64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vcreate_f64(uint64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vcreate_f64(uint64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vcvts_f32_s32(int32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvts_f32_s32(__p0); + return __ret; +} +#else +__ai float32_t vcvts_f32_s32(int32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvts_f32_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vcvts_f32_u32(uint32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvts_f32_u32(__p0); + return __ret; +} +#else +__ai float32_t vcvts_f32_u32(uint32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvts_f32_u32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vcvt_f32_f64(float64x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_f64((int8x16_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vcvt_f32_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_f64((int8x16_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float32x2_t __noswap_vcvt_f32_f64(float64x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_f64((int8x16_t)__p0, 9); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vcvtd_f64_s64(int64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vcvtd_f64_s64(__p0); + return __ret; +} +#else +__ai float64_t vcvtd_f64_s64(int64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vcvtd_f64_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vcvtd_f64_u64(uint64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vcvtd_f64_u64(__p0); + return __ret; +} +#else +__ai float64_t vcvtd_f64_u64(uint64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vcvtd_f64_u64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vcvtq_f64_u64(uint64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvtq_f64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai float64x2_t vcvtq_f64_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvtq_f64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vcvtq_f64_s64(int64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvtq_f64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai float64x2_t vcvtq_f64_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvtq_f64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vcvt_f64_u64(uint64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vcvt_f64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai float64x1_t vcvt_f64_u64(uint64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vcvt_f64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vcvt_f64_s64(int64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vcvt_f64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai float64x1_t vcvt_f64_s64(int64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vcvt_f64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vcvt_f64_f32(float32x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvt_f64_f32((int8x8_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vcvt_f64_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvt_f64_f32((int8x8_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float64x2_t __noswap_vcvt_f64_f32(float32x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvt_f64_f32((int8x8_t)__p0, 42); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vcvt_high_f16_f32(float16x4_t __p0, float32x4_t __p1) { + float16x8_t __ret; + __ret = vcombine_f16(__p0, vcvt_f16_f32(__p1)); + return __ret; +} +#else +__ai float16x8_t vcvt_high_f16_f32(float16x4_t __p0, float32x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __noswap_vcombine_f16(__rev0, __noswap_vcvt_f16_f32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvt_high_f32_f16(float16x8_t __p0) { + float32x4_t __ret; + __ret = vcvt_f32_f16(vget_high_f16(__p0)); + return __ret; +} +#else +__ai float32x4_t vcvt_high_f32_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __noswap_vcvt_f32_f16(__noswap_vget_high_f16(__rev0)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvt_high_f32_f64(float32x2_t __p0, float64x2_t __p1) { + float32x4_t __ret; + __ret = vcombine_f32(__p0, vcvt_f32_f64(__p1)); + return __ret; +} +#else +__ai float32x4_t vcvt_high_f32_f64(float32x2_t __p0, float64x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x4_t __ret; + __ret = __noswap_vcombine_f32(__rev0, __noswap_vcvt_f32_f64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vcvt_high_f64_f32(float32x4_t __p0) { + float64x2_t __ret; + __ret = vcvt_f64_f32(vget_high_f32(__p0)); + return __ret; +} +#else +__ai float64x2_t vcvt_high_f64_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float64x2_t __ret; + __ret = __noswap_vcvt_f64_f32(__noswap_vget_high_f32(__rev0)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvts_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vcvts_n_f32_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvts_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vcvts_n_f32_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvts_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vcvts_n_f32_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvts_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vcvts_n_f32_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vcvtq_n_f64_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vcvtq_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vcvtq_n_f64_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vcvtq_n_f64_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vcvtq_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vcvtq_n_f64_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vcvt_n_f64_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vcvt_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vcvt_n_f64_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vcvt_n_f64_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vcvt_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vcvt_n_f64_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtd_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vcvtd_n_f64_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvtd_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vcvtd_n_f64_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtd_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vcvtd_n_f64_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvtd_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vcvtd_n_f64_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvts_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvts_n_s32_f32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvts_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvts_n_s32_f32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vcvtq_n_s64_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vcvtq_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vcvtq_n_s64_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vcvt_n_s64_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vcvt_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vcvt_n_s64_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtd_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvtd_n_s64_f64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvtd_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvtd_n_s64_f64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvts_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvts_n_u32_f32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvts_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvts_n_u32_f32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vcvtq_n_u64_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vcvtq_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vcvtq_n_u64_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vcvt_n_u64_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vcvt_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vcvt_n_u64_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtd_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvtd_n_u64_f64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvtd_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvtd_n_u64_f64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvts_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvts_s32_f32(__p0); + return __ret; +} +#else +__ai int32_t vcvts_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvts_s32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcvtd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtd_s64_f64(__p0); + return __ret; +} +#else +__ai int64_t vcvtd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtd_s64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcvtq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtq_s64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vcvtq_s64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtq_s64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcvt_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvt_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vcvt_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvt_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvts_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvts_u32_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcvts_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvts_u32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcvtd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtd_u64_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcvtd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtd_u64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcvtq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtq_u64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcvtq_u64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtq_u64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcvt_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvt_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcvt_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvt_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtas_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtas_s32_f32(__p0); + return __ret; +} +#else +__ai int32_t vcvtas_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtas_s32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcvtad_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtad_s64_f64(__p0); + return __ret; +} +#else +__ai int64_t vcvtad_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtad_s64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtas_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtas_u32_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcvtas_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtas_u32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcvtad_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtad_u64_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcvtad_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtad_u64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtms_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtms_s32_f32(__p0); + return __ret; +} +#else +__ai int32_t vcvtms_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtms_s32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcvtmd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtmd_s64_f64(__p0); + return __ret; +} +#else +__ai int64_t vcvtmd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtmd_s64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtms_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtms_u32_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcvtms_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtms_u32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcvtmd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtmd_u64_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcvtmd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtmd_u64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtns_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtns_s32_f32(__p0); + return __ret; +} +#else +__ai int32_t vcvtns_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtns_s32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcvtnd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtnd_s64_f64(__p0); + return __ret; +} +#else +__ai int64_t vcvtnd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtnd_s64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtns_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtns_u32_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcvtns_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtns_u32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcvtnd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtnd_u64_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcvtnd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtnd_u64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtps_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtps_s32_f32(__p0); + return __ret; +} +#else +__ai int32_t vcvtps_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtps_s32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcvtpd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtpd_s64_f64(__p0); + return __ret; +} +#else +__ai int64_t vcvtpd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtpd_s64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtps_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtps_u32_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcvtps_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtps_u32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcvtpd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtpd_u64_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcvtpd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtpd_u64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vcvtxd_f32_f64(float64_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvtxd_f32_f64(__p0); + return __ret; +} +#else +__ai float32_t vcvtxd_f32_f64(float64_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvtxd_f32_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vcvtx_f32_f64(float64x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvtx_f32_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float32x2_t vcvtx_f32_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvtx_f32_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float32x2_t __noswap_vcvtx_f32_f64(float64x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvtx_f32_v((int8x16_t)__p0, 42); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvtx_high_f32_f64(float32x2_t __p0, float64x2_t __p1) { + float32x4_t __ret; + __ret = vcombine_f32(__p0, vcvtx_f32_f64(__p1)); + return __ret; +} +#else +__ai float32x4_t vcvtx_high_f32_f64(float32x2_t __p0, float64x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x4_t __ret; + __ret = __noswap_vcombine_f32(__rev0, __noswap_vcvtx_f32_f64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vdivq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float64x2_t vdivq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 / __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vdivq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float32x4_t vdivq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 / __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vdiv_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float64x1_t vdiv_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vdiv_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float32x2_t vdiv_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 / __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vdups_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vdups_lane_i32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vdupd_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vdupd_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vdupd_lane_f64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vdupd_lane_f64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vdups_lane_f32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vdups_lane_f32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vdups_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vdups_lane_i32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vdupd_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vdupd_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vdups_laneq_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vdups_laneq_i32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vdupd_laneq_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vdupd_laneq_i64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vdupd_laneq_f64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vdupd_laneq_f64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vdups_laneq_f32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vdups_laneq_f32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vdups_laneq_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vdups_laneq_i32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vdupd_laneq_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vdupd_laneq_i64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64x1_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x1_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x1_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x1_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vdup_n_p64(poly64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t) {__p0}; + return __ret; +} +#else +__ai poly64x1_t vdup_n_p64(poly64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vdupq_n_p64(poly64_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai poly64x2_t vdupq_n_p64(poly64_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vdupq_n_f64(float64_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai float64x2_t vdupq_n_f64(float64_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vdup_n_f64(float64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) {__p0}; + return __ret; +} +#else +__ai float64x1_t vdup_n_f64(float64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#else +#define vext_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 38); \ + __ret; \ +}) +#else +#define vextq_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 42); \ + __ret; \ +}) +#else +#define vextq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#else +#define vext_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vfmaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42); + return __ret; +} +#else +__ai float64x2_t vfmaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vfmaq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float64x2_t __noswap_vfmaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vfma_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10); + return __ret; +} +#else +__ai float64x1_t vfma_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10); + return __ret; +} +__ai float64x1_t __noswap_vfma_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmad_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_lane_f64(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmad_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_lane_f64(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#define __noswap_vfmad_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_lane_f64(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmas_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_lane_f32(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmas_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_lane_f32(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#define __noswap_vfmas_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_lane_f32(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 42); \ + __ret; \ +}) +#else +#define vfmaq_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__s2, __p3, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 42); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 41); \ + __ret; \ +}) +#else +#define vfmaq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, __p3, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 41); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 10); \ + __ret; \ +}) +#else +#define vfma_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 10); \ + __ret; \ +}) +#define __noswap_vfma_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 9); \ + __ret; \ +}) +#else +#define vfma_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_lane_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, __p3, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vfma_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 9); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmad_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_laneq_f64(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmad_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_laneq_f64(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#define __noswap_vfmad_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_laneq_f64(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmas_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_laneq_f32(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmas_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_laneq_f32(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#define __noswap_vfmas_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_laneq_f32(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 42); \ + __ret; \ +}) +#else +#define vfmaq_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, __p3, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 42); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 41); \ + __ret; \ +}) +#else +#define vfmaq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, __p3, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 41); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 10); \ + __ret; \ +}) +#else +#define vfma_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__rev2, __p3, 10); \ + __ret; \ +}) +#define __noswap_vfma_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 9); \ + __ret; \ +}) +#else +#define vfma_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_laneq_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x16_t)__rev2, __p3, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vfma_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 9); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vfmaq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __ret; + __ret = vfmaq_f64(__p0, __p1, (float64x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai float64x2_t vfmaq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __noswap_vfmaq_f64(__rev0, __rev1, (float64x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vfmaq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __ret; + __ret = vfmaq_f32(__p0, __p1, (float32x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float32x4_t vfmaq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __noswap_vfmaq_f32(__rev0, __rev1, (float32x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vfma_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __ret; + __ret = vfma_f32(__p0, __p1, (float32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai float32x2_t vfma_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __noswap_vfma_f32(__rev0, __rev1, (float32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vfmsq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = vfmaq_f64(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float64x2_t vfmsq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float64x2_t __ret; + __ret = __noswap_vfmaq_f64(__rev0, -__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vfms_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = vfma_f64(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float64x1_t vfms_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = __noswap_vfma_f64(__p0, -__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsd_lane_f64(__p0_104, __p1_104, __p2_104, __p3_104) __extension__ ({ \ + float64_t __s0_104 = __p0_104; \ + float64_t __s1_104 = __p1_104; \ + float64x1_t __s2_104 = __p2_104; \ + float64_t __ret_104; \ + __ret_104 = vfmad_lane_f64(__s0_104, -__s1_104, __s2_104, __p3_104); \ + __ret_104; \ +}) +#else +#define vfmsd_lane_f64(__p0_105, __p1_105, __p2_105, __p3_105) __extension__ ({ \ + float64_t __s0_105 = __p0_105; \ + float64_t __s1_105 = __p1_105; \ + float64x1_t __s2_105 = __p2_105; \ + float64_t __ret_105; \ + __ret_105 = __noswap_vfmad_lane_f64(__s0_105, -__s1_105, __s2_105, __p3_105); \ + __ret_105; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmss_lane_f32(__p0_106, __p1_106, __p2_106, __p3_106) __extension__ ({ \ + float32_t __s0_106 = __p0_106; \ + float32_t __s1_106 = __p1_106; \ + float32x2_t __s2_106 = __p2_106; \ + float32_t __ret_106; \ + __ret_106 = vfmas_lane_f32(__s0_106, -__s1_106, __s2_106, __p3_106); \ + __ret_106; \ +}) +#else +#define vfmss_lane_f32(__p0_107, __p1_107, __p2_107, __p3_107) __extension__ ({ \ + float32_t __s0_107 = __p0_107; \ + float32_t __s1_107 = __p1_107; \ + float32x2_t __s2_107 = __p2_107; \ + float32x2_t __rev2_107; __rev2_107 = __builtin_shufflevector(__s2_107, __s2_107, 1, 0); \ + float32_t __ret_107; \ + __ret_107 = __noswap_vfmas_lane_f32(__s0_107, -__s1_107, __rev2_107, __p3_107); \ + __ret_107; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_lane_f64(__p0_108, __p1_108, __p2_108, __p3_108) __extension__ ({ \ + float64x2_t __s0_108 = __p0_108; \ + float64x2_t __s1_108 = __p1_108; \ + float64x1_t __s2_108 = __p2_108; \ + float64x2_t __ret_108; \ + __ret_108 = vfmaq_lane_f64(__s0_108, -__s1_108, __s2_108, __p3_108); \ + __ret_108; \ +}) +#else +#define vfmsq_lane_f64(__p0_109, __p1_109, __p2_109, __p3_109) __extension__ ({ \ + float64x2_t __s0_109 = __p0_109; \ + float64x2_t __s1_109 = __p1_109; \ + float64x1_t __s2_109 = __p2_109; \ + float64x2_t __rev0_109; __rev0_109 = __builtin_shufflevector(__s0_109, __s0_109, 1, 0); \ + float64x2_t __rev1_109; __rev1_109 = __builtin_shufflevector(__s1_109, __s1_109, 1, 0); \ + float64x2_t __ret_109; \ + __ret_109 = __noswap_vfmaq_lane_f64(__rev0_109, -__rev1_109, __s2_109, __p3_109); \ + __ret_109 = __builtin_shufflevector(__ret_109, __ret_109, 1, 0); \ + __ret_109; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_lane_f32(__p0_110, __p1_110, __p2_110, __p3_110) __extension__ ({ \ + float32x4_t __s0_110 = __p0_110; \ + float32x4_t __s1_110 = __p1_110; \ + float32x2_t __s2_110 = __p2_110; \ + float32x4_t __ret_110; \ + __ret_110 = vfmaq_lane_f32(__s0_110, -__s1_110, __s2_110, __p3_110); \ + __ret_110; \ +}) +#else +#define vfmsq_lane_f32(__p0_111, __p1_111, __p2_111, __p3_111) __extension__ ({ \ + float32x4_t __s0_111 = __p0_111; \ + float32x4_t __s1_111 = __p1_111; \ + float32x2_t __s2_111 = __p2_111; \ + float32x4_t __rev0_111; __rev0_111 = __builtin_shufflevector(__s0_111, __s0_111, 3, 2, 1, 0); \ + float32x4_t __rev1_111; __rev1_111 = __builtin_shufflevector(__s1_111, __s1_111, 3, 2, 1, 0); \ + float32x2_t __rev2_111; __rev2_111 = __builtin_shufflevector(__s2_111, __s2_111, 1, 0); \ + float32x4_t __ret_111; \ + __ret_111 = __noswap_vfmaq_lane_f32(__rev0_111, -__rev1_111, __rev2_111, __p3_111); \ + __ret_111 = __builtin_shufflevector(__ret_111, __ret_111, 3, 2, 1, 0); \ + __ret_111; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_lane_f64(__p0_112, __p1_112, __p2_112, __p3_112) __extension__ ({ \ + float64x1_t __s0_112 = __p0_112; \ + float64x1_t __s1_112 = __p1_112; \ + float64x1_t __s2_112 = __p2_112; \ + float64x1_t __ret_112; \ + __ret_112 = vfma_lane_f64(__s0_112, -__s1_112, __s2_112, __p3_112); \ + __ret_112; \ +}) +#else +#define vfms_lane_f64(__p0_113, __p1_113, __p2_113, __p3_113) __extension__ ({ \ + float64x1_t __s0_113 = __p0_113; \ + float64x1_t __s1_113 = __p1_113; \ + float64x1_t __s2_113 = __p2_113; \ + float64x1_t __ret_113; \ + __ret_113 = __noswap_vfma_lane_f64(__s0_113, -__s1_113, __s2_113, __p3_113); \ + __ret_113; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_lane_f32(__p0_114, __p1_114, __p2_114, __p3_114) __extension__ ({ \ + float32x2_t __s0_114 = __p0_114; \ + float32x2_t __s1_114 = __p1_114; \ + float32x2_t __s2_114 = __p2_114; \ + float32x2_t __ret_114; \ + __ret_114 = vfma_lane_f32(__s0_114, -__s1_114, __s2_114, __p3_114); \ + __ret_114; \ +}) +#else +#define vfms_lane_f32(__p0_115, __p1_115, __p2_115, __p3_115) __extension__ ({ \ + float32x2_t __s0_115 = __p0_115; \ + float32x2_t __s1_115 = __p1_115; \ + float32x2_t __s2_115 = __p2_115; \ + float32x2_t __rev0_115; __rev0_115 = __builtin_shufflevector(__s0_115, __s0_115, 1, 0); \ + float32x2_t __rev1_115; __rev1_115 = __builtin_shufflevector(__s1_115, __s1_115, 1, 0); \ + float32x2_t __rev2_115; __rev2_115 = __builtin_shufflevector(__s2_115, __s2_115, 1, 0); \ + float32x2_t __ret_115; \ + __ret_115 = __noswap_vfma_lane_f32(__rev0_115, -__rev1_115, __rev2_115, __p3_115); \ + __ret_115 = __builtin_shufflevector(__ret_115, __ret_115, 1, 0); \ + __ret_115; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsd_laneq_f64(__p0_116, __p1_116, __p2_116, __p3_116) __extension__ ({ \ + float64_t __s0_116 = __p0_116; \ + float64_t __s1_116 = __p1_116; \ + float64x2_t __s2_116 = __p2_116; \ + float64_t __ret_116; \ + __ret_116 = vfmad_laneq_f64(__s0_116, -__s1_116, __s2_116, __p3_116); \ + __ret_116; \ +}) +#else +#define vfmsd_laneq_f64(__p0_117, __p1_117, __p2_117, __p3_117) __extension__ ({ \ + float64_t __s0_117 = __p0_117; \ + float64_t __s1_117 = __p1_117; \ + float64x2_t __s2_117 = __p2_117; \ + float64x2_t __rev2_117; __rev2_117 = __builtin_shufflevector(__s2_117, __s2_117, 1, 0); \ + float64_t __ret_117; \ + __ret_117 = __noswap_vfmad_laneq_f64(__s0_117, -__s1_117, __rev2_117, __p3_117); \ + __ret_117; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmss_laneq_f32(__p0_118, __p1_118, __p2_118, __p3_118) __extension__ ({ \ + float32_t __s0_118 = __p0_118; \ + float32_t __s1_118 = __p1_118; \ + float32x4_t __s2_118 = __p2_118; \ + float32_t __ret_118; \ + __ret_118 = vfmas_laneq_f32(__s0_118, -__s1_118, __s2_118, __p3_118); \ + __ret_118; \ +}) +#else +#define vfmss_laneq_f32(__p0_119, __p1_119, __p2_119, __p3_119) __extension__ ({ \ + float32_t __s0_119 = __p0_119; \ + float32_t __s1_119 = __p1_119; \ + float32x4_t __s2_119 = __p2_119; \ + float32x4_t __rev2_119; __rev2_119 = __builtin_shufflevector(__s2_119, __s2_119, 3, 2, 1, 0); \ + float32_t __ret_119; \ + __ret_119 = __noswap_vfmas_laneq_f32(__s0_119, -__s1_119, __rev2_119, __p3_119); \ + __ret_119; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_laneq_f64(__p0_120, __p1_120, __p2_120, __p3_120) __extension__ ({ \ + float64x2_t __s0_120 = __p0_120; \ + float64x2_t __s1_120 = __p1_120; \ + float64x2_t __s2_120 = __p2_120; \ + float64x2_t __ret_120; \ + __ret_120 = vfmaq_laneq_f64(__s0_120, -__s1_120, __s2_120, __p3_120); \ + __ret_120; \ +}) +#else +#define vfmsq_laneq_f64(__p0_121, __p1_121, __p2_121, __p3_121) __extension__ ({ \ + float64x2_t __s0_121 = __p0_121; \ + float64x2_t __s1_121 = __p1_121; \ + float64x2_t __s2_121 = __p2_121; \ + float64x2_t __rev0_121; __rev0_121 = __builtin_shufflevector(__s0_121, __s0_121, 1, 0); \ + float64x2_t __rev1_121; __rev1_121 = __builtin_shufflevector(__s1_121, __s1_121, 1, 0); \ + float64x2_t __rev2_121; __rev2_121 = __builtin_shufflevector(__s2_121, __s2_121, 1, 0); \ + float64x2_t __ret_121; \ + __ret_121 = __noswap_vfmaq_laneq_f64(__rev0_121, -__rev1_121, __rev2_121, __p3_121); \ + __ret_121 = __builtin_shufflevector(__ret_121, __ret_121, 1, 0); \ + __ret_121; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_laneq_f32(__p0_122, __p1_122, __p2_122, __p3_122) __extension__ ({ \ + float32x4_t __s0_122 = __p0_122; \ + float32x4_t __s1_122 = __p1_122; \ + float32x4_t __s2_122 = __p2_122; \ + float32x4_t __ret_122; \ + __ret_122 = vfmaq_laneq_f32(__s0_122, -__s1_122, __s2_122, __p3_122); \ + __ret_122; \ +}) +#else +#define vfmsq_laneq_f32(__p0_123, __p1_123, __p2_123, __p3_123) __extension__ ({ \ + float32x4_t __s0_123 = __p0_123; \ + float32x4_t __s1_123 = __p1_123; \ + float32x4_t __s2_123 = __p2_123; \ + float32x4_t __rev0_123; __rev0_123 = __builtin_shufflevector(__s0_123, __s0_123, 3, 2, 1, 0); \ + float32x4_t __rev1_123; __rev1_123 = __builtin_shufflevector(__s1_123, __s1_123, 3, 2, 1, 0); \ + float32x4_t __rev2_123; __rev2_123 = __builtin_shufflevector(__s2_123, __s2_123, 3, 2, 1, 0); \ + float32x4_t __ret_123; \ + __ret_123 = __noswap_vfmaq_laneq_f32(__rev0_123, -__rev1_123, __rev2_123, __p3_123); \ + __ret_123 = __builtin_shufflevector(__ret_123, __ret_123, 3, 2, 1, 0); \ + __ret_123; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_laneq_f64(__p0_124, __p1_124, __p2_124, __p3_124) __extension__ ({ \ + float64x1_t __s0_124 = __p0_124; \ + float64x1_t __s1_124 = __p1_124; \ + float64x2_t __s2_124 = __p2_124; \ + float64x1_t __ret_124; \ + __ret_124 = vfma_laneq_f64(__s0_124, -__s1_124, __s2_124, __p3_124); \ + __ret_124; \ +}) +#else +#define vfms_laneq_f64(__p0_125, __p1_125, __p2_125, __p3_125) __extension__ ({ \ + float64x1_t __s0_125 = __p0_125; \ + float64x1_t __s1_125 = __p1_125; \ + float64x2_t __s2_125 = __p2_125; \ + float64x2_t __rev2_125; __rev2_125 = __builtin_shufflevector(__s2_125, __s2_125, 1, 0); \ + float64x1_t __ret_125; \ + __ret_125 = __noswap_vfma_laneq_f64(__s0_125, -__s1_125, __rev2_125, __p3_125); \ + __ret_125; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_laneq_f32(__p0_126, __p1_126, __p2_126, __p3_126) __extension__ ({ \ + float32x2_t __s0_126 = __p0_126; \ + float32x2_t __s1_126 = __p1_126; \ + float32x4_t __s2_126 = __p2_126; \ + float32x2_t __ret_126; \ + __ret_126 = vfma_laneq_f32(__s0_126, -__s1_126, __s2_126, __p3_126); \ + __ret_126; \ +}) +#else +#define vfms_laneq_f32(__p0_127, __p1_127, __p2_127, __p3_127) __extension__ ({ \ + float32x2_t __s0_127 = __p0_127; \ + float32x2_t __s1_127 = __p1_127; \ + float32x4_t __s2_127 = __p2_127; \ + float32x2_t __rev0_127; __rev0_127 = __builtin_shufflevector(__s0_127, __s0_127, 1, 0); \ + float32x2_t __rev1_127; __rev1_127 = __builtin_shufflevector(__s1_127, __s1_127, 1, 0); \ + float32x4_t __rev2_127; __rev2_127 = __builtin_shufflevector(__s2_127, __s2_127, 3, 2, 1, 0); \ + float32x2_t __ret_127; \ + __ret_127 = __noswap_vfma_laneq_f32(__rev0_127, -__rev1_127, __rev2_127, __p3_127); \ + __ret_127 = __builtin_shufflevector(__ret_127, __ret_127, 1, 0); \ + __ret_127; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vfmsq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __ret; + __ret = vfmaq_f64(__p0, -__p1, (float64x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai float64x2_t vfmsq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __noswap_vfmaq_f64(__rev0, -__rev1, (float64x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vfmsq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __ret; + __ret = vfmaq_f32(__p0, -__p1, (float32x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float32x4_t vfmsq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __noswap_vfmaq_f32(__rev0, -__rev1, (float32x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vfms_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __ret; + __ret = vfma_f32(__p0, -__p1, (float32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai float32x2_t vfms_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __noswap_vfma_f32(__rev0, -__rev1, (float32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vget_high_p64(poly64x2_t __p0) { + poly64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1); + return __ret; +} +#else +__ai poly64x1_t vget_high_p64(poly64x2_t __p0) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1); + return __ret; +} +__ai poly64x1_t __noswap_vget_high_p64(poly64x2_t __p0) { + poly64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vget_high_f64(float64x2_t __p0) { + float64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1); + return __ret; +} +#else +__ai float64x1_t vget_high_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vgetq_lane_f64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vgetq_lane_f64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vgetq_lane_f64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vget_lane_f64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vget_lane_f64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vget_lane_f64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vget_low_p64(poly64x2_t __p0) { + poly64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0); + return __ret; +} +#else +__ai poly64x1_t vget_low_p64(poly64x2_t __p0) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vget_low_f64(float64x2_t __p0) { + float64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0); + return __ret; +} +#else +__ai float64x1_t vget_low_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p64(__p0) __extension__ ({ \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_v(__p0, 6); \ + __ret; \ +}) +#else +#define vld1_p64(__p0) __extension__ ({ \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_v(__p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p64(__p0) __extension__ ({ \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_v(__p0, 38); \ + __ret; \ +}) +#else +#define vld1q_p64(__p0) __extension__ ({ \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_v(__p0, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f64(__p0) __extension__ ({ \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_v(__p0, 42); \ + __ret; \ +}) +#else +#define vld1q_f64(__p0) __extension__ ({ \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_v(__p0, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f64(__p0) __extension__ ({ \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_v(__p0, 10); \ + __ret; \ +}) +#else +#define vld1_f64(__p0) __extension__ ({ \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_v(__p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_p64(__p0) __extension__ ({ \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_dup_v(__p0, 6); \ + __ret; \ +}) +#else +#define vld1_dup_p64(__p0) __extension__ ({ \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_dup_v(__p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_p64(__p0) __extension__ ({ \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_dup_v(__p0, 38); \ + __ret; \ +}) +#else +#define vld1q_dup_p64(__p0) __extension__ ({ \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_dup_v(__p0, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_f64(__p0) __extension__ ({ \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_dup_v(__p0, 42); \ + __ret; \ +}) +#else +#define vld1q_dup_f64(__p0) __extension__ ({ \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_dup_v(__p0, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_f64(__p0) __extension__ ({ \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_dup_v(__p0, 10); \ + __ret; \ +}) +#else +#define vld1_dup_f64(__p0) __extension__ ({ \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_dup_v(__p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#else +#define vld1_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 38); \ + __ret; \ +}) +#else +#define vld1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 42); \ + __ret; \ +}) +#else +#define vld1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#else +#define vld1_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p8_x2(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld1_p8_x2(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p64_x2(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld1_p64_x2(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p16_x2(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld1_p16_x2(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p8_x2(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld1q_p8_x2(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p64_x2(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld1q_p64_x2(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p16_x2(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld1q_p16_x2(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u8_x2(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld1q_u8_x2(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u32_x2(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld1q_u32_x2(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u64_x2(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld1q_u64_x2(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u16_x2(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld1q_u16_x2(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s8_x2(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld1q_s8_x2(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f64_x2(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld1q_f64_x2(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f32_x2(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld1q_f32_x2(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f16_x2(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld1q_f16_x2(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s32_x2(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld1q_s32_x2(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s64_x2(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld1q_s64_x2(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s16_x2(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld1q_s16_x2(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u8_x2(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld1_u8_x2(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u32_x2(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld1_u32_x2(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u64_x2(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld1_u64_x2(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u16_x2(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld1_u16_x2(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s8_x2(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld1_s8_x2(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f64_x2(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld1_f64_x2(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f32_x2(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld1_f32_x2(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f16_x2(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld1_f16_x2(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s32_x2(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld1_s32_x2(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s64_x2(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld1_s64_x2(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s16_x2(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld1_s16_x2(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p8_x3(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld1_p8_x3(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p64_x3(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld1_p64_x3(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p16_x3(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld1_p16_x3(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p8_x3(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld1q_p8_x3(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p64_x3(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld1q_p64_x3(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p16_x3(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld1q_p16_x3(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u8_x3(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld1q_u8_x3(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u32_x3(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld1q_u32_x3(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u64_x3(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld1q_u64_x3(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u16_x3(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld1q_u16_x3(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s8_x3(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld1q_s8_x3(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f64_x3(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld1q_f64_x3(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f32_x3(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld1q_f32_x3(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f16_x3(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld1q_f16_x3(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s32_x3(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld1q_s32_x3(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s64_x3(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld1q_s64_x3(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s16_x3(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld1q_s16_x3(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u8_x3(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld1_u8_x3(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u32_x3(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld1_u32_x3(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u64_x3(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld1_u64_x3(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u16_x3(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld1_u16_x3(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s8_x3(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld1_s8_x3(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f64_x3(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld1_f64_x3(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f32_x3(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld1_f32_x3(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f16_x3(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld1_f16_x3(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s32_x3(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld1_s32_x3(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s64_x3(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld1_s64_x3(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s16_x3(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld1_s16_x3(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p8_x4(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld1_p8_x4(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p64_x4(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld1_p64_x4(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p16_x4(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld1_p16_x4(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p8_x4(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld1q_p8_x4(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p64_x4(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld1q_p64_x4(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p16_x4(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld1q_p16_x4(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u8_x4(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld1q_u8_x4(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u32_x4(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld1q_u32_x4(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u64_x4(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld1q_u64_x4(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u16_x4(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld1q_u16_x4(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s8_x4(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld1q_s8_x4(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f64_x4(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld1q_f64_x4(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f32_x4(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld1q_f32_x4(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f16_x4(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld1q_f16_x4(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s32_x4(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld1q_s32_x4(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s64_x4(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld1q_s64_x4(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s16_x4(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld1q_s16_x4(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u8_x4(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld1_u8_x4(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u32_x4(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld1_u32_x4(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u64_x4(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld1_u64_x4(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u16_x4(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld1_u16_x4(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s8_x4(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld1_s8_x4(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f64_x4(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld1_f64_x4(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f32_x4(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld1_f32_x4(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f16_x4(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld1_f16_x4(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s32_x4(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld1_s32_x4(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s64_x4(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld1_s64_x4(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s16_x4(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld1_s16_x4(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_p64(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld2_p64(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_p64(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld2q_p64(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_u64(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld2q_u64(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_f64(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld2q_f64(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_s64(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld2q_s64(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_f64(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld2_f64(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_p64(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld2_dup_p64(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_p8(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld2q_dup_p8(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_p64(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld2q_dup_p64(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_p16(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld2q_dup_p16(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_u8(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld2q_dup_u8(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_u32(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld2q_dup_u32(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_u64(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld2q_dup_u64(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_u16(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld2q_dup_u16(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_s8(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld2q_dup_s8(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_f64(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld2q_dup_f64(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_f32(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld2q_dup_f32(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_f16(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld2q_dup_f16(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_s32(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld2q_dup_s32(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_s64(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld2q_dup_s64(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_s16(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld2q_dup_s16(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_f64(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld2_dup_f64(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 6); \ + __ret; \ +}) +#else +#define vld2_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 36); \ + __ret; \ +}) +#else +#define vld2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + poly8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 38); \ + __ret; \ +}) +#else +#define vld2q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + poly64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 48); \ + __ret; \ +}) +#else +#define vld2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + uint8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 51); \ + __ret; \ +}) +#else +#define vld2q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + uint64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 32); \ + __ret; \ +}) +#else +#define vld2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + int8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 42); \ + __ret; \ +}) +#else +#define vld2q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + float64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 35); \ + __ret; \ +}) +#else +#define vld2q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + int64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 19); \ + __ret; \ +}) +#else +#define vld2_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 10); \ + __ret; \ +}) +#else +#define vld2_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 3); \ + __ret; \ +}) +#else +#define vld2_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_p64(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld3_p64(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_p64(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld3q_p64(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_u64(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld3q_u64(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_f64(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld3q_f64(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_s64(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld3q_s64(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_f64(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld3_f64(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_p64(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld3_dup_p64(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_p8(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld3q_dup_p8(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_p64(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld3q_dup_p64(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_p16(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld3q_dup_p16(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_u8(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld3q_dup_u8(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_u32(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld3q_dup_u32(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_u64(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld3q_dup_u64(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_u16(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld3q_dup_u16(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_s8(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld3q_dup_s8(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_f64(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld3q_dup_f64(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_f32(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld3q_dup_f32(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_f16(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld3q_dup_f16(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_s32(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld3q_dup_s32(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_s64(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld3q_dup_s64(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_s16(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld3q_dup_s16(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_f64(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld3_dup_f64(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 6); \ + __ret; \ +}) +#else +#define vld3_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 36); \ + __ret; \ +}) +#else +#define vld3q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + poly8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 38); \ + __ret; \ +}) +#else +#define vld3q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + poly64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 48); \ + __ret; \ +}) +#else +#define vld3q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + uint8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 51); \ + __ret; \ +}) +#else +#define vld3q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + uint64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 32); \ + __ret; \ +}) +#else +#define vld3q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + int8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 42); \ + __ret; \ +}) +#else +#define vld3q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + float64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 35); \ + __ret; \ +}) +#else +#define vld3q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + int64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 19); \ + __ret; \ +}) +#else +#define vld3_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 10); \ + __ret; \ +}) +#else +#define vld3_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 3); \ + __ret; \ +}) +#else +#define vld3_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_p64(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld4_p64(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_p64(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld4q_p64(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_u64(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld4q_u64(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_f64(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld4q_f64(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_s64(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld4q_s64(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_f64(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld4_f64(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_p64(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld4_dup_p64(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_p8(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld4q_dup_p8(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_p64(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld4q_dup_p64(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_p16(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld4q_dup_p16(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_u8(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld4q_dup_u8(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_u32(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld4q_dup_u32(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_u64(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld4q_dup_u64(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_u16(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld4q_dup_u16(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_s8(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld4q_dup_s8(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_f64(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld4q_dup_f64(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_f32(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld4q_dup_f32(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_f16(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld4q_dup_f16(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_s32(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld4q_dup_s32(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_s64(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld4q_dup_s64(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_s16(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld4q_dup_s16(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_f64(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld4_dup_f64(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 6); \ + __ret; \ +}) +#else +#define vld4_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 36); \ + __ret; \ +}) +#else +#define vld4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + poly8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 38); \ + __ret; \ +}) +#else +#define vld4q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + poly64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 48); \ + __ret; \ +}) +#else +#define vld4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + uint8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 51); \ + __ret; \ +}) +#else +#define vld4q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + uint64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 32); \ + __ret; \ +}) +#else +#define vld4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + int8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 42); \ + __ret; \ +}) +#else +#define vld4q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + float64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 35); \ + __ret; \ +}) +#else +#define vld4q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + int64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 19); \ + __ret; \ +}) +#else +#define vld4_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 10); \ + __ret; \ +}) +#else +#define vld4_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 3); \ + __ret; \ +}) +#else +#define vld4_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vldrq_p128(__p0) __extension__ ({ \ + poly128_t __ret; \ + __ret = (poly128_t) __builtin_neon_vldrq_p128(__p0); \ + __ret; \ +}) +#else +#define vldrq_p128(__p0) __extension__ ({ \ + poly128_t __ret; \ + __ret = (poly128_t) __builtin_neon_vldrq_p128(__p0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmaxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vmaxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmax_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vmax_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vmaxnmvq_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmaxnmvq_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vmaxnmvq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmaxnmvq_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vmaxnmvq_f32(float32x4_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxnmvq_f32((int8x16_t)__p0); + return __ret; +} +#else +__ai float32_t vmaxnmvq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxnmvq_f32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vmaxnmv_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxnmv_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vmaxnmv_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxnmv_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vmaxvq_u8(uint8x16_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vmaxvq_u8((int8x16_t)__p0); + return __ret; +} +#else +__ai uint8_t vmaxvq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vmaxvq_u8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vmaxvq_u32(uint32x4_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vmaxvq_u32((int8x16_t)__p0); + return __ret; +} +#else +__ai uint32_t vmaxvq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vmaxvq_u32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vmaxvq_u16(uint16x8_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vmaxvq_u16((int8x16_t)__p0); + return __ret; +} +#else +__ai uint16_t vmaxvq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vmaxvq_u16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vmaxvq_s8(int8x16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vmaxvq_s8((int8x16_t)__p0); + return __ret; +} +#else +__ai int8_t vmaxvq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vmaxvq_s8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vmaxvq_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmaxvq_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vmaxvq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmaxvq_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vmaxvq_f32(float32x4_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxvq_f32((int8x16_t)__p0); + return __ret; +} +#else +__ai float32_t vmaxvq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxvq_f32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vmaxvq_s32(int32x4_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vmaxvq_s32((int8x16_t)__p0); + return __ret; +} +#else +__ai int32_t vmaxvq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vmaxvq_s32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vmaxvq_s16(int16x8_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vmaxvq_s16((int8x16_t)__p0); + return __ret; +} +#else +__ai int16_t vmaxvq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vmaxvq_s16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vmaxv_u8(uint8x8_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vmaxv_u8((int8x8_t)__p0); + return __ret; +} +#else +__ai uint8_t vmaxv_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vmaxv_u8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vmaxv_u32(uint32x2_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vmaxv_u32((int8x8_t)__p0); + return __ret; +} +#else +__ai uint32_t vmaxv_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vmaxv_u32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vmaxv_u16(uint16x4_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vmaxv_u16((int8x8_t)__p0); + return __ret; +} +#else +__ai uint16_t vmaxv_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vmaxv_u16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vmaxv_s8(int8x8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vmaxv_s8((int8x8_t)__p0); + return __ret; +} +#else +__ai int8_t vmaxv_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vmaxv_s8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vmaxv_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxv_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vmaxv_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxv_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vmaxv_s32(int32x2_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vmaxv_s32((int8x8_t)__p0); + return __ret; +} +#else +__ai int32_t vmaxv_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vmaxv_s32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vmaxv_s16(int16x4_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vmaxv_s16((int8x8_t)__p0); + return __ret; +} +#else +__ai int16_t vmaxv_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vmaxv_s16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vminq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vminq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmin_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vmin_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vminnmvq_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vminnmvq_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vminnmvq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vminnmvq_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vminnmvq_f32(float32x4_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminnmvq_f32((int8x16_t)__p0); + return __ret; +} +#else +__ai float32_t vminnmvq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminnmvq_f32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vminnmv_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminnmv_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vminnmv_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminnmv_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vminvq_u8(uint8x16_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vminvq_u8((int8x16_t)__p0); + return __ret; +} +#else +__ai uint8_t vminvq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vminvq_u8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vminvq_u32(uint32x4_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vminvq_u32((int8x16_t)__p0); + return __ret; +} +#else +__ai uint32_t vminvq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vminvq_u32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vminvq_u16(uint16x8_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vminvq_u16((int8x16_t)__p0); + return __ret; +} +#else +__ai uint16_t vminvq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vminvq_u16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vminvq_s8(int8x16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vminvq_s8((int8x16_t)__p0); + return __ret; +} +#else +__ai int8_t vminvq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vminvq_s8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vminvq_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vminvq_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vminvq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vminvq_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vminvq_f32(float32x4_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminvq_f32((int8x16_t)__p0); + return __ret; +} +#else +__ai float32_t vminvq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminvq_f32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vminvq_s32(int32x4_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vminvq_s32((int8x16_t)__p0); + return __ret; +} +#else +__ai int32_t vminvq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vminvq_s32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vminvq_s16(int16x8_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vminvq_s16((int8x16_t)__p0); + return __ret; +} +#else +__ai int16_t vminvq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vminvq_s16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vminv_u8(uint8x8_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vminv_u8((int8x8_t)__p0); + return __ret; +} +#else +__ai uint8_t vminv_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vminv_u8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vminv_u32(uint32x2_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vminv_u32((int8x8_t)__p0); + return __ret; +} +#else +__ai uint32_t vminv_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vminv_u32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vminv_u16(uint16x4_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vminv_u16((int8x8_t)__p0); + return __ret; +} +#else +__ai uint16_t vminv_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vminv_u16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vminv_s8(int8x8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vminv_s8((int8x8_t)__p0); + return __ret; +} +#else +__ai int8_t vminv_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vminv_s8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vminv_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminv_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vminv_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminv_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vminv_s32(int32x2_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vminv_s32((int8x8_t)__p0); + return __ret; +} +#else +__ai int32_t vminv_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vminv_s32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vminv_s16(int16x4_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vminv_s16((int8x8_t)__p0); + return __ret; +} +#else +__ai int16_t vminv_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vminv_s16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmlaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai float64x2_t vmlaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float64x2_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmla_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai float64x1_t vmla_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x8_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmlaq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __ret; + __ret = __p0 + __p1 * (float64x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai float64x2_t vmlaq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 + __rev1 * (float64x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 + vmull_u32(vget_high_u32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_u32(__noswap_vget_high_u32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + vmull_u16(vget_high_u16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_u16(__noswap_vget_high_u16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 + vmull_s32(vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_s32(__noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + vmull_s16(vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_s16(__noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 + vmull_u32(vget_high_u32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_u32(__noswap_vget_high_u32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + vmull_u16(vget_high_u16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_u16(__noswap_vget_high_u16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 + vmull_s32(vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_s32(__noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + vmull_s16(vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_s16(__noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 + vmull_u32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_u32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + vmull_u16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_u16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 + vmull_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + vmull_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmlsq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai float64x2_t vmlsq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float64x2_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmls_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai float64x1_t vmls_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x8_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmlsq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __ret; + __ret = __p0 - __p1 * (float64x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai float64x2_t vmlsq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 - __rev1 * (float64x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 - vmull_u32(vget_high_u32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_u32(__noswap_vget_high_u32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - vmull_u16(vget_high_u16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_u16(__noswap_vget_high_u16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 - vmull_s32(vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_s32(__noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - vmull_s16(vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_s16(__noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 - vmull_u32(vget_high_u32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_u32(__noswap_vget_high_u32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - vmull_u16(vget_high_u16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_u16(__noswap_vget_high_u16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 - vmull_s32(vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_s32(__noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - vmull_s16(vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_s16(__noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 - vmull_u32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_u32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - vmull_u16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_u16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 - vmull_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - vmull_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vmov_n_p64(poly64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t) {__p0}; + return __ret; +} +#else +__ai poly64x1_t vmov_n_p64(poly64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vmovq_n_p64(poly64_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai poly64x2_t vmovq_n_p64(poly64_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmovq_n_f64(float64_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai float64x2_t vmovq_n_f64(float64_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmov_n_f64(float64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) {__p0}; + return __ret; +} +#else +__ai float64x1_t vmov_n_f64(float64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmovl_high_u8(uint8x16_t __p0_128) { + uint16x8_t __ret_128; + uint8x8_t __a1_128 = vget_high_u8(__p0_128); + __ret_128 = (uint16x8_t)(vshll_n_u8(__a1_128, 0)); + return __ret_128; +} +#else +__ai uint16x8_t vmovl_high_u8(uint8x16_t __p0_129) { + uint8x16_t __rev0_129; __rev0_129 = __builtin_shufflevector(__p0_129, __p0_129, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret_129; + uint8x8_t __a1_129 = __noswap_vget_high_u8(__rev0_129); + __ret_129 = (uint16x8_t)(__noswap_vshll_n_u8(__a1_129, 0)); + __ret_129 = __builtin_shufflevector(__ret_129, __ret_129, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret_129; +} +__ai uint16x8_t __noswap_vmovl_high_u8(uint8x16_t __p0_130) { + uint16x8_t __ret_130; + uint8x8_t __a1_130 = __noswap_vget_high_u8(__p0_130); + __ret_130 = (uint16x8_t)(__noswap_vshll_n_u8(__a1_130, 0)); + return __ret_130; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmovl_high_u32(uint32x4_t __p0_131) { + uint64x2_t __ret_131; + uint32x2_t __a1_131 = vget_high_u32(__p0_131); + __ret_131 = (uint64x2_t)(vshll_n_u32(__a1_131, 0)); + return __ret_131; +} +#else +__ai uint64x2_t vmovl_high_u32(uint32x4_t __p0_132) { + uint32x4_t __rev0_132; __rev0_132 = __builtin_shufflevector(__p0_132, __p0_132, 3, 2, 1, 0); + uint64x2_t __ret_132; + uint32x2_t __a1_132 = __noswap_vget_high_u32(__rev0_132); + __ret_132 = (uint64x2_t)(__noswap_vshll_n_u32(__a1_132, 0)); + __ret_132 = __builtin_shufflevector(__ret_132, __ret_132, 1, 0); + return __ret_132; +} +__ai uint64x2_t __noswap_vmovl_high_u32(uint32x4_t __p0_133) { + uint64x2_t __ret_133; + uint32x2_t __a1_133 = __noswap_vget_high_u32(__p0_133); + __ret_133 = (uint64x2_t)(__noswap_vshll_n_u32(__a1_133, 0)); + return __ret_133; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmovl_high_u16(uint16x8_t __p0_134) { + uint32x4_t __ret_134; + uint16x4_t __a1_134 = vget_high_u16(__p0_134); + __ret_134 = (uint32x4_t)(vshll_n_u16(__a1_134, 0)); + return __ret_134; +} +#else +__ai uint32x4_t vmovl_high_u16(uint16x8_t __p0_135) { + uint16x8_t __rev0_135; __rev0_135 = __builtin_shufflevector(__p0_135, __p0_135, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret_135; + uint16x4_t __a1_135 = __noswap_vget_high_u16(__rev0_135); + __ret_135 = (uint32x4_t)(__noswap_vshll_n_u16(__a1_135, 0)); + __ret_135 = __builtin_shufflevector(__ret_135, __ret_135, 3, 2, 1, 0); + return __ret_135; +} +__ai uint32x4_t __noswap_vmovl_high_u16(uint16x8_t __p0_136) { + uint32x4_t __ret_136; + uint16x4_t __a1_136 = __noswap_vget_high_u16(__p0_136); + __ret_136 = (uint32x4_t)(__noswap_vshll_n_u16(__a1_136, 0)); + return __ret_136; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmovl_high_s8(int8x16_t __p0_137) { + int16x8_t __ret_137; + int8x8_t __a1_137 = vget_high_s8(__p0_137); + __ret_137 = (int16x8_t)(vshll_n_s8(__a1_137, 0)); + return __ret_137; +} +#else +__ai int16x8_t vmovl_high_s8(int8x16_t __p0_138) { + int8x16_t __rev0_138; __rev0_138 = __builtin_shufflevector(__p0_138, __p0_138, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret_138; + int8x8_t __a1_138 = __noswap_vget_high_s8(__rev0_138); + __ret_138 = (int16x8_t)(__noswap_vshll_n_s8(__a1_138, 0)); + __ret_138 = __builtin_shufflevector(__ret_138, __ret_138, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret_138; +} +__ai int16x8_t __noswap_vmovl_high_s8(int8x16_t __p0_139) { + int16x8_t __ret_139; + int8x8_t __a1_139 = __noswap_vget_high_s8(__p0_139); + __ret_139 = (int16x8_t)(__noswap_vshll_n_s8(__a1_139, 0)); + return __ret_139; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmovl_high_s32(int32x4_t __p0_140) { + int64x2_t __ret_140; + int32x2_t __a1_140 = vget_high_s32(__p0_140); + __ret_140 = (int64x2_t)(vshll_n_s32(__a1_140, 0)); + return __ret_140; +} +#else +__ai int64x2_t vmovl_high_s32(int32x4_t __p0_141) { + int32x4_t __rev0_141; __rev0_141 = __builtin_shufflevector(__p0_141, __p0_141, 3, 2, 1, 0); + int64x2_t __ret_141; + int32x2_t __a1_141 = __noswap_vget_high_s32(__rev0_141); + __ret_141 = (int64x2_t)(__noswap_vshll_n_s32(__a1_141, 0)); + __ret_141 = __builtin_shufflevector(__ret_141, __ret_141, 1, 0); + return __ret_141; +} +__ai int64x2_t __noswap_vmovl_high_s32(int32x4_t __p0_142) { + int64x2_t __ret_142; + int32x2_t __a1_142 = __noswap_vget_high_s32(__p0_142); + __ret_142 = (int64x2_t)(__noswap_vshll_n_s32(__a1_142, 0)); + return __ret_142; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmovl_high_s16(int16x8_t __p0_143) { + int32x4_t __ret_143; + int16x4_t __a1_143 = vget_high_s16(__p0_143); + __ret_143 = (int32x4_t)(vshll_n_s16(__a1_143, 0)); + return __ret_143; +} +#else +__ai int32x4_t vmovl_high_s16(int16x8_t __p0_144) { + int16x8_t __rev0_144; __rev0_144 = __builtin_shufflevector(__p0_144, __p0_144, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret_144; + int16x4_t __a1_144 = __noswap_vget_high_s16(__rev0_144); + __ret_144 = (int32x4_t)(__noswap_vshll_n_s16(__a1_144, 0)); + __ret_144 = __builtin_shufflevector(__ret_144, __ret_144, 3, 2, 1, 0); + return __ret_144; +} +__ai int32x4_t __noswap_vmovl_high_s16(int16x8_t __p0_145) { + int32x4_t __ret_145; + int16x4_t __a1_145 = __noswap_vget_high_s16(__p0_145); + __ret_145 = (int32x4_t)(__noswap_vshll_n_s16(__a1_145, 0)); + return __ret_145; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmovn_high_u32(uint16x4_t __p0, uint32x4_t __p1) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vmovn_u32(__p1)); + return __ret; +} +#else +__ai uint16x8_t vmovn_high_u32(uint16x4_t __p0, uint32x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vmovn_u32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmovn_high_u64(uint32x2_t __p0, uint64x2_t __p1) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vmovn_u64(__p1)); + return __ret; +} +#else +__ai uint32x4_t vmovn_high_u64(uint32x2_t __p0, uint64x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vmovn_u64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmovn_high_u16(uint8x8_t __p0, uint16x8_t __p1) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vmovn_u16(__p1)); + return __ret; +} +#else +__ai uint8x16_t vmovn_high_u16(uint8x8_t __p0, uint16x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vmovn_u16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmovn_high_s32(int16x4_t __p0, int32x4_t __p1) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vmovn_s32(__p1)); + return __ret; +} +#else +__ai int16x8_t vmovn_high_s32(int16x4_t __p0, int32x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vmovn_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmovn_high_s64(int32x2_t __p0, int64x2_t __p1) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vmovn_s64(__p1)); + return __ret; +} +#else +__ai int32x4_t vmovn_high_s64(int32x2_t __p0, int64x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vmovn_s64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmovn_high_s16(int8x8_t __p0, int16x8_t __p1) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vmovn_s16(__p1)); + return __ret; +} +#else +__ai int8x16_t vmovn_high_s16(int8x8_t __p0, int16x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vmovn_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmulq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float64x2_t vmulq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmul_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float64x1_t vmul_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmuld_lane_f64(__p0_146, __p1_146, __p2_146) __extension__ ({ \ + float64_t __s0_146 = __p0_146; \ + float64x1_t __s1_146 = __p1_146; \ + float64_t __ret_146; \ + __ret_146 = __s0_146 * vget_lane_f64(__s1_146, __p2_146); \ + __ret_146; \ +}) +#else +#define vmuld_lane_f64(__p0_147, __p1_147, __p2_147) __extension__ ({ \ + float64_t __s0_147 = __p0_147; \ + float64x1_t __s1_147 = __p1_147; \ + float64_t __ret_147; \ + __ret_147 = __s0_147 * __noswap_vget_lane_f64(__s1_147, __p2_147); \ + __ret_147; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmuls_lane_f32(__p0_148, __p1_148, __p2_148) __extension__ ({ \ + float32_t __s0_148 = __p0_148; \ + float32x2_t __s1_148 = __p1_148; \ + float32_t __ret_148; \ + __ret_148 = __s0_148 * vget_lane_f32(__s1_148, __p2_148); \ + __ret_148; \ +}) +#else +#define vmuls_lane_f32(__p0_149, __p1_149, __p2_149) __extension__ ({ \ + float32_t __s0_149 = __p0_149; \ + float32x2_t __s1_149 = __p1_149; \ + float32x2_t __rev1_149; __rev1_149 = __builtin_shufflevector(__s1_149, __s1_149, 1, 0); \ + float32_t __ret_149; \ + __ret_149 = __s0_149 * __noswap_vget_lane_f32(__rev1_149, __p2_149); \ + __ret_149; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vmul_lane_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#else +#define vmul_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vmul_lane_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmuld_laneq_f64(__p0_150, __p1_150, __p2_150) __extension__ ({ \ + float64_t __s0_150 = __p0_150; \ + float64x2_t __s1_150 = __p1_150; \ + float64_t __ret_150; \ + __ret_150 = __s0_150 * vgetq_lane_f64(__s1_150, __p2_150); \ + __ret_150; \ +}) +#else +#define vmuld_laneq_f64(__p0_151, __p1_151, __p2_151) __extension__ ({ \ + float64_t __s0_151 = __p0_151; \ + float64x2_t __s1_151 = __p1_151; \ + float64x2_t __rev1_151; __rev1_151 = __builtin_shufflevector(__s1_151, __s1_151, 1, 0); \ + float64_t __ret_151; \ + __ret_151 = __s0_151 * __noswap_vgetq_lane_f64(__rev1_151, __p2_151); \ + __ret_151; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmuls_laneq_f32(__p0_152, __p1_152, __p2_152) __extension__ ({ \ + float32_t __s0_152 = __p0_152; \ + float32x4_t __s1_152 = __p1_152; \ + float32_t __ret_152; \ + __ret_152 = __s0_152 * vgetq_lane_f32(__s1_152, __p2_152); \ + __ret_152; \ +}) +#else +#define vmuls_laneq_f32(__p0_153, __p1_153, __p2_153) __extension__ ({ \ + float32_t __s0_153 = __p0_153; \ + float32x4_t __s1_153 = __p1_153; \ + float32x4_t __rev1_153; __rev1_153 = __builtin_shufflevector(__s1_153, __s1_153, 3, 2, 1, 0); \ + float32_t __ret_153; \ + __ret_153 = __s0_153 * __noswap_vgetq_lane_f32(__rev1_153, __p2_153); \ + __ret_153; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vmul_laneq_v((int8x8_t)__s0, (int8x16_t)__s1, __p2, 10); \ + __ret; \ +}) +#else +#define vmul_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vmul_laneq_v((int8x8_t)__s0, (int8x16_t)__rev1, __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmul_n_f64(float64x1_t __p0, float64_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmul_n_f64((int8x8_t)__p0, __p1); + return __ret; +} +#else +__ai float64x1_t vmul_n_f64(float64x1_t __p0, float64_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmul_n_f64((int8x8_t)__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmulq_n_f64(float64x2_t __p0, float64_t __p1) { + float64x2_t __ret; + __ret = __p0 * (float64x2_t) {__p1, __p1}; + return __ret; +} +#else +__ai float64x2_t vmulq_n_f64(float64x2_t __p0, float64_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = __rev0 * (float64x2_t) {__p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vmull_p64(poly64_t __p0, poly64_t __p1) { + poly128_t __ret; + __ret = (poly128_t) __builtin_neon_vmull_p64(__p0, __p1); + return __ret; +} +#else +__ai poly128_t vmull_p64(poly64_t __p0, poly64_t __p1) { + poly128_t __ret; + __ret = (poly128_t) __builtin_neon_vmull_p64(__p0, __p1); + return __ret; +} +__ai poly128_t __noswap_vmull_p64(poly64_t __p0, poly64_t __p1) { + poly128_t __ret; + __ret = (poly128_t) __builtin_neon_vmull_p64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vmull_high_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly16x8_t __ret; + __ret = vmull_p8(vget_high_p8(__p0), vget_high_p8(__p1)); + return __ret; +} +#else +__ai poly16x8_t vmull_high_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __noswap_vmull_p8(__noswap_vget_high_p8(__rev0), __noswap_vget_high_p8(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmull_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = vmull_u8(vget_high_u8(__p0), vget_high_u8(__p1)); + return __ret; +} +#else +__ai uint16x8_t vmull_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmull_u8(__noswap_vget_high_u8(__rev0), __noswap_vget_high_u8(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmull_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = vmull_u32(vget_high_u32(__p0), vget_high_u32(__p1)); + return __ret; +} +#else +__ai uint64x2_t vmull_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmull_u32(__noswap_vget_high_u32(__rev0), __noswap_vget_high_u32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmull_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = vmull_u16(vget_high_u16(__p0), vget_high_u16(__p1)); + return __ret; +} +#else +__ai uint32x4_t vmull_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmull_u16(__noswap_vget_high_u16(__rev0), __noswap_vget_high_u16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmull_high_s8(int8x16_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = vmull_s8(vget_high_s8(__p0), vget_high_s8(__p1)); + return __ret; +} +#else +__ai int16x8_t vmull_high_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmull_s8(__noswap_vget_high_s8(__rev0), __noswap_vget_high_s8(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmull_high_s32(int32x4_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = vmull_s32(vget_high_s32(__p0), vget_high_s32(__p1)); + return __ret; +} +#else +__ai int64x2_t vmull_high_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmull_s32(__noswap_vget_high_s32(__rev0), __noswap_vget_high_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmull_high_s16(int16x8_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = vmull_s16(vget_high_s16(__p0), vget_high_s16(__p1)); + return __ret; +} +#else +__ai int32x4_t vmull_high_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmull_s16(__noswap_vget_high_s16(__rev0), __noswap_vget_high_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vmull_high_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly128_t __ret; + __ret = vmull_p64((poly64_t)(vget_high_p64(__p0)), (poly64_t)(vget_high_p64(__p1))); + return __ret; +} +#else +__ai poly128_t vmull_high_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly128_t __ret; + __ret = __noswap_vmull_p64((poly64_t)(__noswap_vget_high_p64(__rev0)), (poly64_t)(__noswap_vget_high_p64(__rev1))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = vmull_u32(vget_high_u32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = __noswap_vmull_u32(__noswap_vget_high_u32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = vmull_u16(vget_high_u16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __noswap_vmull_u16(__noswap_vget_high_u16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vmull_s32(vget_high_s32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vmull_s32(__noswap_vget_high_s32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vmull_s16(vget_high_s16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vmull_s16(__noswap_vget_high_s16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = vmull_u32(vget_high_u32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __noswap_vmull_u32(__noswap_vget_high_u32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = vmull_u16(vget_high_u16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __noswap_vmull_u16(__noswap_vget_high_u16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vmull_s32(vget_high_s32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vmull_s32(__noswap_vget_high_s32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vmull_s16(vget_high_s16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vmull_s16(__noswap_vget_high_s16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmull_high_n_u32(uint32x4_t __p0, uint32_t __p1) { + uint64x2_t __ret; + __ret = vmull_n_u32(vget_high_u32(__p0), __p1); + return __ret; +} +#else +__ai uint64x2_t vmull_high_n_u32(uint32x4_t __p0, uint32_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmull_n_u32(__noswap_vget_high_u32(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmull_high_n_u16(uint16x8_t __p0, uint16_t __p1) { + uint32x4_t __ret; + __ret = vmull_n_u16(vget_high_u16(__p0), __p1); + return __ret; +} +#else +__ai uint32x4_t vmull_high_n_u16(uint16x8_t __p0, uint16_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmull_n_u16(__noswap_vget_high_u16(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmull_high_n_s32(int32x4_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = vmull_n_s32(vget_high_s32(__p0), __p1); + return __ret; +} +#else +__ai int64x2_t vmull_high_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmull_n_s32(__noswap_vget_high_s32(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmull_high_n_s16(int16x8_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = vmull_n_s16(vget_high_s16(__p0), __p1); + return __ret; +} +#else +__ai int32x4_t vmull_high_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmull_n_s16(__noswap_vget_high_s16(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = vmull_u32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __noswap_vmull_u32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = vmull_u16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __noswap_vmull_u16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vmull_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vmull_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vmull_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vmull_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmulxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vmulxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmulxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float64x2_t __noswap_vmulxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmulxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vmulxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmulxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float32x4_t __noswap_vmulxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmulx_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vmulx_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmulx_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vmulx_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmulx_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float32x2_t __noswap_vmulx_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vmulxd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmulxd_f64(__p0, __p1); + return __ret; +} +#else +__ai float64_t vmulxd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmulxd_f64(__p0, __p1); + return __ret; +} +__ai float64_t __noswap_vmulxd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmulxd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vmulxs_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmulxs_f32(__p0, __p1); + return __ret; +} +#else +__ai float32_t vmulxs_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmulxs_f32(__p0, __p1); + return __ret; +} +__ai float32_t __noswap_vmulxs_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmulxs_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxd_lane_f64(__p0_154, __p1_154, __p2_154) __extension__ ({ \ + float64_t __s0_154 = __p0_154; \ + float64x1_t __s1_154 = __p1_154; \ + float64_t __ret_154; \ + __ret_154 = vmulxd_f64(__s0_154, vget_lane_f64(__s1_154, __p2_154)); \ + __ret_154; \ +}) +#else +#define vmulxd_lane_f64(__p0_155, __p1_155, __p2_155) __extension__ ({ \ + float64_t __s0_155 = __p0_155; \ + float64x1_t __s1_155 = __p1_155; \ + float64_t __ret_155; \ + __ret_155 = __noswap_vmulxd_f64(__s0_155, __noswap_vget_lane_f64(__s1_155, __p2_155)); \ + __ret_155; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxs_lane_f32(__p0_156, __p1_156, __p2_156) __extension__ ({ \ + float32_t __s0_156 = __p0_156; \ + float32x2_t __s1_156 = __p1_156; \ + float32_t __ret_156; \ + __ret_156 = vmulxs_f32(__s0_156, vget_lane_f32(__s1_156, __p2_156)); \ + __ret_156; \ +}) +#else +#define vmulxs_lane_f32(__p0_157, __p1_157, __p2_157) __extension__ ({ \ + float32_t __s0_157 = __p0_157; \ + float32x2_t __s1_157 = __p1_157; \ + float32x2_t __rev1_157; __rev1_157 = __builtin_shufflevector(__s1_157, __s1_157, 1, 0); \ + float32_t __ret_157; \ + __ret_157 = __noswap_vmulxs_f32(__s0_157, __noswap_vget_lane_f32(__rev1_157, __p2_157)); \ + __ret_157; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = vmulxq_f64(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __ret; \ + __ret = __noswap_vmulxq_f64(__rev0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = vmulxq_f32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x4_t __ret; \ + __ret = __noswap_vmulxq_f32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = vmulx_f32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulx_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __ret; \ + __ret = __noswap_vmulx_f32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxd_laneq_f64(__p0_158, __p1_158, __p2_158) __extension__ ({ \ + float64_t __s0_158 = __p0_158; \ + float64x2_t __s1_158 = __p1_158; \ + float64_t __ret_158; \ + __ret_158 = vmulxd_f64(__s0_158, vgetq_lane_f64(__s1_158, __p2_158)); \ + __ret_158; \ +}) +#else +#define vmulxd_laneq_f64(__p0_159, __p1_159, __p2_159) __extension__ ({ \ + float64_t __s0_159 = __p0_159; \ + float64x2_t __s1_159 = __p1_159; \ + float64x2_t __rev1_159; __rev1_159 = __builtin_shufflevector(__s1_159, __s1_159, 1, 0); \ + float64_t __ret_159; \ + __ret_159 = __noswap_vmulxd_f64(__s0_159, __noswap_vgetq_lane_f64(__rev1_159, __p2_159)); \ + __ret_159; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxs_laneq_f32(__p0_160, __p1_160, __p2_160) __extension__ ({ \ + float32_t __s0_160 = __p0_160; \ + float32x4_t __s1_160 = __p1_160; \ + float32_t __ret_160; \ + __ret_160 = vmulxs_f32(__s0_160, vgetq_lane_f32(__s1_160, __p2_160)); \ + __ret_160; \ +}) +#else +#define vmulxs_laneq_f32(__p0_161, __p1_161, __p2_161) __extension__ ({ \ + float32_t __s0_161 = __p0_161; \ + float32x4_t __s1_161 = __p1_161; \ + float32x4_t __rev1_161; __rev1_161 = __builtin_shufflevector(__s1_161, __s1_161, 3, 2, 1, 0); \ + float32_t __ret_161; \ + __ret_161 = __noswap_vmulxs_f32(__s0_161, __noswap_vgetq_lane_f32(__rev1_161, __p2_161)); \ + __ret_161; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = vmulxq_f64(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = __noswap_vmulxq_f64(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = vmulxq_f32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = __noswap_vmulxq_f32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = vmulx_f32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulx_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = __noswap_vmulx_f32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vnegq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float64x2_t vnegq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vnegq_s64(int64x2_t __p0) { + int64x2_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int64x2_t vnegq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vneg_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float64x1_t vneg_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = -__p0; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vneg_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int64x1_t vneg_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = -__p0; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vnegd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vnegd_s64(__p0); + return __ret; +} +#else +__ai int64_t vnegd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vnegd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vpaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vpaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vpaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vpaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vpaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vpaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vpaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vpaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vpaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vpaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vpaddq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vpaddq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vpaddq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vpaddq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vpaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vpaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vpaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vpaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vpaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vpaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vpaddd_u64(uint64x2_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vpaddd_u64((int8x16_t)__p0); + return __ret; +} +#else +__ai uint64_t vpaddd_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vpaddd_u64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vpaddd_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpaddd_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vpaddd_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpaddd_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vpaddd_s64(int64x2_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vpaddd_s64((int8x16_t)__p0); + return __ret; +} +#else +__ai int64_t vpaddd_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64_t __ret; + __ret = (int64_t) __builtin_neon_vpaddd_s64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vpadds_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpadds_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vpadds_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpadds_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vpmaxq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vpmaxq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vpmaxq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vpmaxq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vpmaxq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vpmaxq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vpmaxq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vpmaxq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vpmaxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vpmaxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vpmaxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vpmaxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vpmaxq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vpmaxq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vpmaxq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vpmaxq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vpmaxqd_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpmaxqd_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vpmaxqd_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpmaxqd_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vpmaxs_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmaxs_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vpmaxs_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmaxs_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vpmaxnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vpmaxnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vpmaxnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vpmaxnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vpmaxnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vpmaxnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmaxnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vpmaxnmqd_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpmaxnmqd_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vpmaxnmqd_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpmaxnmqd_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vpmaxnms_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmaxnms_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vpmaxnms_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmaxnms_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vpminq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vpminq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vpminq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vpminq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vpminq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vpminq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vpminq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vpminq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vpminq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vpminq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vpminq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vpminq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vpminq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vpminq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vpminq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vpminq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vpminqd_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpminqd_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vpminqd_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpminqd_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vpmins_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmins_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vpmins_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmins_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vpminnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vpminnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vpminnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vpminnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vpminnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vpminnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpminnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vpminnmqd_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpminnmqd_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vpminnmqd_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpminnmqd_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vpminnms_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpminnms_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vpminnms_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpminnms_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqabsq_s64(int64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqabsq_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vqabsq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqabsq_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqabs_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqabs_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vqabs_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqabs_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqabsb_s8(int8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqabsb_s8(__p0); + return __ret; +} +#else +__ai int8_t vqabsb_s8(int8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqabsb_s8(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqabss_s32(int32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqabss_s32(__p0); + return __ret; +} +#else +__ai int32_t vqabss_s32(int32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqabss_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqabsd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqabsd_s64(__p0); + return __ret; +} +#else +__ai int64_t vqabsd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqabsd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqabsh_s16(int16_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqabsh_s16(__p0); + return __ret; +} +#else +__ai int16_t vqabsh_s16(int16_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqabsh_s16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vqaddb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqaddb_u8(__p0, __p1); + return __ret; +} +#else +__ai uint8_t vqaddb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqaddb_u8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vqadds_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqadds_u32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vqadds_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqadds_u32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vqaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqaddd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vqaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqaddd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vqaddh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqaddh_u16(__p0, __p1); + return __ret; +} +#else +__ai uint16_t vqaddh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqaddh_u16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqaddb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqaddb_s8(__p0, __p1); + return __ret; +} +#else +__ai int8_t vqaddb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqaddb_s8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqadds_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqadds_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqadds_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqadds_s32(__p0, __p1); + return __ret; +} +__ai int32_t __noswap_vqadds_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqadds_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqaddd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vqaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqaddd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqaddh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqaddh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqaddh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqaddh_s16(__p0, __p1); + return __ret; +} +__ai int16_t __noswap_vqaddh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqaddh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqdmlals_s32(int64_t __p0, int32_t __p1, int32_t __p2) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmlals_s32(__p0, __p1, __p2); + return __ret; +} +#else +__ai int64_t vqdmlals_s32(int64_t __p0, int32_t __p1, int32_t __p2) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmlals_s32(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqdmlalh_s16(int32_t __p0, int16_t __p1, int16_t __p2) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmlalh_s16(__p0, __p1, __p2); + return __ret; +} +#else +__ai int32_t vqdmlalh_s16(int32_t __p0, int16_t __p1, int16_t __p2) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmlalh_s16(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __ret; + __ret = vqdmlal_s32(__p0, vget_high_s32(__p1), vget_high_s32(__p2)); + return __ret; +} +#else +__ai int64x2_t vqdmlal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmlal_s32(__rev0, __noswap_vget_high_s32(__rev1), __noswap_vget_high_s32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __ret; + __ret = vqdmlal_s16(__p0, vget_high_s16(__p1), vget_high_s16(__p2)); + return __ret; +} +#else +__ai int32x4_t vqdmlal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmlal_s16(__rev0, __noswap_vget_high_s16(__rev1), __noswap_vget_high_s16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlal_s32(__s0, vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlal_s32(__rev0, __noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlal_s16(__s0, vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlal_s16(__rev0, __noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlal_s32(__s0, vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlal_s32(__rev0, __noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlal_s16(__s0, vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlal_s16(__rev0, __noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlal_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = vqdmlal_n_s32(__p0, vget_high_s32(__p1), __p2); + return __ret; +} +#else +__ai int64x2_t vqdmlal_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmlal_n_s32(__rev0, __noswap_vget_high_s32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlal_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = vqdmlal_n_s16(__p0, vget_high_s16(__p1), __p2); + return __ret; +} +#else +__ai int32x4_t vqdmlal_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmlal_n_s16(__rev0, __noswap_vget_high_s16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlals_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlals_lane_s32(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlals_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlals_lane_s32(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlalh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlalh_lane_s16(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlalh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlalh_lane_s16(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlals_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlals_laneq_s32(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlals_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlals_laneq_s32(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlalh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlalh_laneq_s16(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlalh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlalh_laneq_s16(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlal_s32(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlal_s32(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlal_s16(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlal_s16(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqdmlsls_s32(int64_t __p0, int32_t __p1, int32_t __p2) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmlsls_s32(__p0, __p1, __p2); + return __ret; +} +#else +__ai int64_t vqdmlsls_s32(int64_t __p0, int32_t __p1, int32_t __p2) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmlsls_s32(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqdmlslh_s16(int32_t __p0, int16_t __p1, int16_t __p2) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmlslh_s16(__p0, __p1, __p2); + return __ret; +} +#else +__ai int32_t vqdmlslh_s16(int32_t __p0, int16_t __p1, int16_t __p2) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmlslh_s16(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlsl_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __ret; + __ret = vqdmlsl_s32(__p0, vget_high_s32(__p1), vget_high_s32(__p2)); + return __ret; +} +#else +__ai int64x2_t vqdmlsl_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmlsl_s32(__rev0, __noswap_vget_high_s32(__rev1), __noswap_vget_high_s32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlsl_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __ret; + __ret = vqdmlsl_s16(__p0, vget_high_s16(__p1), vget_high_s16(__p2)); + return __ret; +} +#else +__ai int32x4_t vqdmlsl_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmlsl_s16(__rev0, __noswap_vget_high_s16(__rev1), __noswap_vget_high_s16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlsl_s32(__s0, vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlsl_s32(__rev0, __noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlsl_s16(__s0, vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlsl_s16(__rev0, __noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlsl_s32(__s0, vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlsl_s32(__rev0, __noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlsl_s16(__s0, vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlsl_s16(__rev0, __noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlsl_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = vqdmlsl_n_s32(__p0, vget_high_s32(__p1), __p2); + return __ret; +} +#else +__ai int64x2_t vqdmlsl_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmlsl_n_s32(__rev0, __noswap_vget_high_s32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlsl_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = vqdmlsl_n_s16(__p0, vget_high_s16(__p1), __p2); + return __ret; +} +#else +__ai int32x4_t vqdmlsl_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmlsl_n_s16(__rev0, __noswap_vget_high_s16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsls_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlsls_lane_s32(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlsls_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlsls_lane_s32(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlslh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlslh_lane_s16(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlslh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlslh_lane_s16(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsls_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlsls_laneq_s32(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlsls_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlsls_laneq_s32(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlslh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlslh_laneq_s16(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlslh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlslh_laneq_s16(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlsl_s32(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlsl_s32(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlsl_s16(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlsl_s16(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmulhs_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmulhs_s32(__p0, __p1); + return __ret; +} +__ai int32_t __noswap_vqdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmulhs_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqdmulhh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqdmulhh_s16(__p0, __p1); + return __ret; +} +__ai int16_t __noswap_vqdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqdmulhh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhs_lane_s32(__p0_162, __p1_162, __p2_162) __extension__ ({ \ + int32_t __s0_162 = __p0_162; \ + int32x2_t __s1_162 = __p1_162; \ + int32_t __ret_162; \ + __ret_162 = vqdmulhs_s32(__s0_162, vget_lane_s32(__s1_162, __p2_162)); \ + __ret_162; \ +}) +#else +#define vqdmulhs_lane_s32(__p0_163, __p1_163, __p2_163) __extension__ ({ \ + int32_t __s0_163 = __p0_163; \ + int32x2_t __s1_163 = __p1_163; \ + int32x2_t __rev1_163; __rev1_163 = __builtin_shufflevector(__s1_163, __s1_163, 1, 0); \ + int32_t __ret_163; \ + __ret_163 = __noswap_vqdmulhs_s32(__s0_163, __noswap_vget_lane_s32(__rev1_163, __p2_163)); \ + __ret_163; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhh_lane_s16(__p0_164, __p1_164, __p2_164) __extension__ ({ \ + int16_t __s0_164 = __p0_164; \ + int16x4_t __s1_164 = __p1_164; \ + int16_t __ret_164; \ + __ret_164 = vqdmulhh_s16(__s0_164, vget_lane_s16(__s1_164, __p2_164)); \ + __ret_164; \ +}) +#else +#define vqdmulhh_lane_s16(__p0_165, __p1_165, __p2_165) __extension__ ({ \ + int16_t __s0_165 = __p0_165; \ + int16x4_t __s1_165 = __p1_165; \ + int16x4_t __rev1_165; __rev1_165 = __builtin_shufflevector(__s1_165, __s1_165, 3, 2, 1, 0); \ + int16_t __ret_165; \ + __ret_165 = __noswap_vqdmulhh_s16(__s0_165, __noswap_vget_lane_s16(__rev1_165, __p2_165)); \ + __ret_165; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhs_laneq_s32(__p0_166, __p1_166, __p2_166) __extension__ ({ \ + int32_t __s0_166 = __p0_166; \ + int32x4_t __s1_166 = __p1_166; \ + int32_t __ret_166; \ + __ret_166 = vqdmulhs_s32(__s0_166, vgetq_lane_s32(__s1_166, __p2_166)); \ + __ret_166; \ +}) +#else +#define vqdmulhs_laneq_s32(__p0_167, __p1_167, __p2_167) __extension__ ({ \ + int32_t __s0_167 = __p0_167; \ + int32x4_t __s1_167 = __p1_167; \ + int32x4_t __rev1_167; __rev1_167 = __builtin_shufflevector(__s1_167, __s1_167, 3, 2, 1, 0); \ + int32_t __ret_167; \ + __ret_167 = __noswap_vqdmulhs_s32(__s0_167, __noswap_vgetq_lane_s32(__rev1_167, __p2_167)); \ + __ret_167; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhh_laneq_s16(__p0_168, __p1_168, __p2_168) __extension__ ({ \ + int16_t __s0_168 = __p0_168; \ + int16x8_t __s1_168 = __p1_168; \ + int16_t __ret_168; \ + __ret_168 = vqdmulhh_s16(__s0_168, vgetq_lane_s16(__s1_168, __p2_168)); \ + __ret_168; \ +}) +#else +#define vqdmulhh_laneq_s16(__p0_169, __p1_169, __p2_169) __extension__ ({ \ + int16_t __s0_169 = __p0_169; \ + int16x8_t __s1_169 = __p1_169; \ + int16x8_t __rev1_169; __rev1_169 = __builtin_shufflevector(__s1_169, __s1_169, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret_169; \ + __ret_169 = __noswap_vqdmulhh_s16(__s0_169, __noswap_vgetq_lane_s16(__rev1_169, __p2_169)); \ + __ret_169; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmulhq_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulhq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmulhq_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = vqdmulhq_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulhq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqdmulhq_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulh_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = vqdmulh_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulh_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqdmulh_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulh_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = vqdmulh_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulh_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqdmulh_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqdmulls_s32(int32_t __p0, int32_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmulls_s32(__p0, __p1); + return __ret; +} +#else +__ai int64_t vqdmulls_s32(int32_t __p0, int32_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmulls_s32(__p0, __p1); + return __ret; +} +__ai int64_t __noswap_vqdmulls_s32(int32_t __p0, int32_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmulls_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqdmullh_s16(int16_t __p0, int16_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmullh_s16(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqdmullh_s16(int16_t __p0, int16_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmullh_s16(__p0, __p1); + return __ret; +} +__ai int32_t __noswap_vqdmullh_s16(int16_t __p0, int16_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmullh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmull_high_s32(int32x4_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = vqdmull_s32(vget_high_s32(__p0), vget_high_s32(__p1)); + return __ret; +} +#else +__ai int64x2_t vqdmull_high_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmull_s32(__noswap_vget_high_s32(__rev0), __noswap_vget_high_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmull_high_s16(int16x8_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = vqdmull_s16(vget_high_s16(__p0), vget_high_s16(__p1)); + return __ret; +} +#else +__ai int32x4_t vqdmull_high_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmull_s16(__noswap_vget_high_s16(__rev0), __noswap_vget_high_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_high_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vqdmull_s32(vget_high_s32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_high_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmull_s32(__noswap_vget_high_s32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_high_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmull_s16(vget_high_s16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_high_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmull_s16(__noswap_vget_high_s16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_high_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vqdmull_s32(vget_high_s32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_high_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmull_s32(__noswap_vget_high_s32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_high_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmull_s16(vget_high_s16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_high_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmull_s16(__noswap_vget_high_s16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmull_high_n_s32(int32x4_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = vqdmull_n_s32(vget_high_s32(__p0), __p1); + return __ret; +} +#else +__ai int64x2_t vqdmull_high_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmull_n_s32(__noswap_vget_high_s32(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmull_high_n_s16(int16x8_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = vqdmull_n_s16(vget_high_s16(__p0), __p1); + return __ret; +} +#else +__ai int32x4_t vqdmull_high_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmull_n_s16(__noswap_vget_high_s16(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulls_lane_s32(__p0_170, __p1_170, __p2_170) __extension__ ({ \ + int32_t __s0_170 = __p0_170; \ + int32x2_t __s1_170 = __p1_170; \ + int64_t __ret_170; \ + __ret_170 = vqdmulls_s32(__s0_170, vget_lane_s32(__s1_170, __p2_170)); \ + __ret_170; \ +}) +#else +#define vqdmulls_lane_s32(__p0_171, __p1_171, __p2_171) __extension__ ({ \ + int32_t __s0_171 = __p0_171; \ + int32x2_t __s1_171 = __p1_171; \ + int32x2_t __rev1_171; __rev1_171 = __builtin_shufflevector(__s1_171, __s1_171, 1, 0); \ + int64_t __ret_171; \ + __ret_171 = __noswap_vqdmulls_s32(__s0_171, __noswap_vget_lane_s32(__rev1_171, __p2_171)); \ + __ret_171; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmullh_lane_s16(__p0_172, __p1_172, __p2_172) __extension__ ({ \ + int16_t __s0_172 = __p0_172; \ + int16x4_t __s1_172 = __p1_172; \ + int32_t __ret_172; \ + __ret_172 = vqdmullh_s16(__s0_172, vget_lane_s16(__s1_172, __p2_172)); \ + __ret_172; \ +}) +#else +#define vqdmullh_lane_s16(__p0_173, __p1_173, __p2_173) __extension__ ({ \ + int16_t __s0_173 = __p0_173; \ + int16x4_t __s1_173 = __p1_173; \ + int16x4_t __rev1_173; __rev1_173 = __builtin_shufflevector(__s1_173, __s1_173, 3, 2, 1, 0); \ + int32_t __ret_173; \ + __ret_173 = __noswap_vqdmullh_s16(__s0_173, __noswap_vget_lane_s16(__rev1_173, __p2_173)); \ + __ret_173; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulls_laneq_s32(__p0_174, __p1_174, __p2_174) __extension__ ({ \ + int32_t __s0_174 = __p0_174; \ + int32x4_t __s1_174 = __p1_174; \ + int64_t __ret_174; \ + __ret_174 = vqdmulls_s32(__s0_174, vgetq_lane_s32(__s1_174, __p2_174)); \ + __ret_174; \ +}) +#else +#define vqdmulls_laneq_s32(__p0_175, __p1_175, __p2_175) __extension__ ({ \ + int32_t __s0_175 = __p0_175; \ + int32x4_t __s1_175 = __p1_175; \ + int32x4_t __rev1_175; __rev1_175 = __builtin_shufflevector(__s1_175, __s1_175, 3, 2, 1, 0); \ + int64_t __ret_175; \ + __ret_175 = __noswap_vqdmulls_s32(__s0_175, __noswap_vgetq_lane_s32(__rev1_175, __p2_175)); \ + __ret_175; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmullh_laneq_s16(__p0_176, __p1_176, __p2_176) __extension__ ({ \ + int16_t __s0_176 = __p0_176; \ + int16x8_t __s1_176 = __p1_176; \ + int32_t __ret_176; \ + __ret_176 = vqdmullh_s16(__s0_176, vgetq_lane_s16(__s1_176, __p2_176)); \ + __ret_176; \ +}) +#else +#define vqdmullh_laneq_s16(__p0_177, __p1_177, __p2_177) __extension__ ({ \ + int16_t __s0_177 = __p0_177; \ + int16x8_t __s1_177 = __p1_177; \ + int16x8_t __rev1_177; __rev1_177 = __builtin_shufflevector(__s1_177, __s1_177, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32_t __ret_177; \ + __ret_177 = __noswap_vqdmullh_s16(__s0_177, __noswap_vgetq_lane_s16(__rev1_177, __p2_177)); \ + __ret_177; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vqdmull_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmull_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmull_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmull_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqmovns_s32(int32_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqmovns_s32(__p0); + return __ret; +} +#else +__ai int16_t vqmovns_s32(int32_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqmovns_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqmovnd_s64(int64_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqmovnd_s64(__p0); + return __ret; +} +#else +__ai int32_t vqmovnd_s64(int64_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqmovnd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqmovnh_s16(int16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqmovnh_s16(__p0); + return __ret; +} +#else +__ai int8_t vqmovnh_s16(int16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqmovnh_s16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vqmovns_u32(uint32_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqmovns_u32(__p0); + return __ret; +} +#else +__ai uint16_t vqmovns_u32(uint32_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqmovns_u32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vqmovnd_u64(uint64_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqmovnd_u64(__p0); + return __ret; +} +#else +__ai uint32_t vqmovnd_u64(uint64_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqmovnd_u64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vqmovnh_u16(uint16_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqmovnh_u16(__p0); + return __ret; +} +#else +__ai uint8_t vqmovnh_u16(uint16_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqmovnh_u16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqmovn_high_u32(uint16x4_t __p0, uint32x4_t __p1) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vqmovn_u32(__p1)); + return __ret; +} +#else +__ai uint16x8_t vqmovn_high_u32(uint16x4_t __p0, uint32x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vqmovn_u32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqmovn_high_u64(uint32x2_t __p0, uint64x2_t __p1) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vqmovn_u64(__p1)); + return __ret; +} +#else +__ai uint32x4_t vqmovn_high_u64(uint32x2_t __p0, uint64x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vqmovn_u64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqmovn_high_u16(uint8x8_t __p0, uint16x8_t __p1) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vqmovn_u16(__p1)); + return __ret; +} +#else +__ai uint8x16_t vqmovn_high_u16(uint8x8_t __p0, uint16x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vqmovn_u16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqmovn_high_s32(int16x4_t __p0, int32x4_t __p1) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vqmovn_s32(__p1)); + return __ret; +} +#else +__ai int16x8_t vqmovn_high_s32(int16x4_t __p0, int32x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vqmovn_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqmovn_high_s64(int32x2_t __p0, int64x2_t __p1) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vqmovn_s64(__p1)); + return __ret; +} +#else +__ai int32x4_t vqmovn_high_s64(int32x2_t __p0, int64x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vqmovn_s64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqmovn_high_s16(int8x8_t __p0, int16x8_t __p1) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vqmovn_s16(__p1)); + return __ret; +} +#else +__ai int8x16_t vqmovn_high_s16(int8x8_t __p0, int16x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vqmovn_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqmovuns_s32(int32_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqmovuns_s32(__p0); + return __ret; +} +#else +__ai int16_t vqmovuns_s32(int32_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqmovuns_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqmovund_s64(int64_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqmovund_s64(__p0); + return __ret; +} +#else +__ai int32_t vqmovund_s64(int64_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqmovund_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqmovunh_s16(int16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqmovunh_s16(__p0); + return __ret; +} +#else +__ai int8_t vqmovunh_s16(int16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqmovunh_s16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqmovun_high_s32(int16x4_t __p0, int32x4_t __p1) { + uint16x8_t __ret; + __ret = vcombine_u16((uint16x4_t)(__p0), vqmovun_s32(__p1)); + return __ret; +} +#else +__ai uint16x8_t vqmovun_high_s32(int16x4_t __p0, int32x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16((uint16x4_t)(__rev0), __noswap_vqmovun_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqmovun_high_s64(int32x2_t __p0, int64x2_t __p1) { + uint32x4_t __ret; + __ret = vcombine_u32((uint32x2_t)(__p0), vqmovun_s64(__p1)); + return __ret; +} +#else +__ai uint32x4_t vqmovun_high_s64(int32x2_t __p0, int64x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32((uint32x2_t)(__rev0), __noswap_vqmovun_s64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqmovun_high_s16(int8x8_t __p0, int16x8_t __p1) { + uint8x16_t __ret; + __ret = vcombine_u8((uint8x8_t)(__p0), vqmovun_s16(__p1)); + return __ret; +} +#else +__ai uint8x16_t vqmovun_high_s16(int8x8_t __p0, int16x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8((uint8x8_t)(__rev0), __noswap_vqmovun_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqnegq_s64(int64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqnegq_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vqnegq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqnegq_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqneg_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqneg_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vqneg_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqneg_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqnegb_s8(int8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqnegb_s8(__p0); + return __ret; +} +#else +__ai int8_t vqnegb_s8(int8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqnegb_s8(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqnegs_s32(int32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqnegs_s32(__p0); + return __ret; +} +#else +__ai int32_t vqnegs_s32(int32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqnegs_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqnegd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqnegd_s64(__p0); + return __ret; +} +#else +__ai int64_t vqnegd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqnegd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqnegh_s16(int16_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqnegh_s16(__p0); + return __ret; +} +#else +__ai int16_t vqnegh_s16(int16_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqnegh_s16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqrdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqrdmulhs_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqrdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqrdmulhs_s32(__p0, __p1); + return __ret; +} +__ai int32_t __noswap_vqrdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqrdmulhs_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqrdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqrdmulhh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqrdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqrdmulhh_s16(__p0, __p1); + return __ret; +} +__ai int16_t __noswap_vqrdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqrdmulhh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhs_lane_s32(__p0_178, __p1_178, __p2_178) __extension__ ({ \ + int32_t __s0_178 = __p0_178; \ + int32x2_t __s1_178 = __p1_178; \ + int32_t __ret_178; \ + __ret_178 = vqrdmulhs_s32(__s0_178, vget_lane_s32(__s1_178, __p2_178)); \ + __ret_178; \ +}) +#else +#define vqrdmulhs_lane_s32(__p0_179, __p1_179, __p2_179) __extension__ ({ \ + int32_t __s0_179 = __p0_179; \ + int32x2_t __s1_179 = __p1_179; \ + int32x2_t __rev1_179; __rev1_179 = __builtin_shufflevector(__s1_179, __s1_179, 1, 0); \ + int32_t __ret_179; \ + __ret_179 = __noswap_vqrdmulhs_s32(__s0_179, __noswap_vget_lane_s32(__rev1_179, __p2_179)); \ + __ret_179; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhh_lane_s16(__p0_180, __p1_180, __p2_180) __extension__ ({ \ + int16_t __s0_180 = __p0_180; \ + int16x4_t __s1_180 = __p1_180; \ + int16_t __ret_180; \ + __ret_180 = vqrdmulhh_s16(__s0_180, vget_lane_s16(__s1_180, __p2_180)); \ + __ret_180; \ +}) +#else +#define vqrdmulhh_lane_s16(__p0_181, __p1_181, __p2_181) __extension__ ({ \ + int16_t __s0_181 = __p0_181; \ + int16x4_t __s1_181 = __p1_181; \ + int16x4_t __rev1_181; __rev1_181 = __builtin_shufflevector(__s1_181, __s1_181, 3, 2, 1, 0); \ + int16_t __ret_181; \ + __ret_181 = __noswap_vqrdmulhh_s16(__s0_181, __noswap_vget_lane_s16(__rev1_181, __p2_181)); \ + __ret_181; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhs_laneq_s32(__p0_182, __p1_182, __p2_182) __extension__ ({ \ + int32_t __s0_182 = __p0_182; \ + int32x4_t __s1_182 = __p1_182; \ + int32_t __ret_182; \ + __ret_182 = vqrdmulhs_s32(__s0_182, vgetq_lane_s32(__s1_182, __p2_182)); \ + __ret_182; \ +}) +#else +#define vqrdmulhs_laneq_s32(__p0_183, __p1_183, __p2_183) __extension__ ({ \ + int32_t __s0_183 = __p0_183; \ + int32x4_t __s1_183 = __p1_183; \ + int32x4_t __rev1_183; __rev1_183 = __builtin_shufflevector(__s1_183, __s1_183, 3, 2, 1, 0); \ + int32_t __ret_183; \ + __ret_183 = __noswap_vqrdmulhs_s32(__s0_183, __noswap_vgetq_lane_s32(__rev1_183, __p2_183)); \ + __ret_183; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhh_laneq_s16(__p0_184, __p1_184, __p2_184) __extension__ ({ \ + int16_t __s0_184 = __p0_184; \ + int16x8_t __s1_184 = __p1_184; \ + int16_t __ret_184; \ + __ret_184 = vqrdmulhh_s16(__s0_184, vgetq_lane_s16(__s1_184, __p2_184)); \ + __ret_184; \ +}) +#else +#define vqrdmulhh_laneq_s16(__p0_185, __p1_185, __p2_185) __extension__ ({ \ + int16_t __s0_185 = __p0_185; \ + int16x8_t __s1_185 = __p1_185; \ + int16x8_t __rev1_185; __rev1_185 = __builtin_shufflevector(__s1_185, __s1_185, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret_185; \ + __ret_185 = __noswap_vqrdmulhh_s16(__s0_185, __noswap_vgetq_lane_s16(__rev1_185, __p2_185)); \ + __ret_185; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqrdmulhq_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulhq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqrdmulhq_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = vqrdmulhq_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulhq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqrdmulhq_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulh_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = vqrdmulh_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulh_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqrdmulh_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulh_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = vqrdmulh_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulh_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqrdmulh_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vqrshlb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqrshlb_u8(__p0, __p1); + return __ret; +} +#else +__ai uint8_t vqrshlb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqrshlb_u8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vqrshls_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqrshls_u32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vqrshls_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqrshls_u32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vqrshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqrshld_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vqrshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqrshld_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vqrshlh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqrshlh_u16(__p0, __p1); + return __ret; +} +#else +__ai uint16_t vqrshlh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqrshlh_u16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqrshlb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqrshlb_s8(__p0, __p1); + return __ret; +} +#else +__ai int8_t vqrshlb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqrshlb_s8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqrshls_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqrshls_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqrshls_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqrshls_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqrshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqrshld_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vqrshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqrshld_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqrshlh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqrshlh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqrshlh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqrshlh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_u32(__p0_186, __p1_186, __p2_186) __extension__ ({ \ + uint16x4_t __s0_186 = __p0_186; \ + uint32x4_t __s1_186 = __p1_186; \ + uint16x8_t __ret_186; \ + __ret_186 = (uint16x8_t)(vcombine_u16((uint16x4_t)(__s0_186), (uint16x4_t)(vqrshrn_n_u32(__s1_186, __p2_186)))); \ + __ret_186; \ +}) +#else +#define vqrshrn_high_n_u32(__p0_187, __p1_187, __p2_187) __extension__ ({ \ + uint16x4_t __s0_187 = __p0_187; \ + uint32x4_t __s1_187 = __p1_187; \ + uint16x4_t __rev0_187; __rev0_187 = __builtin_shufflevector(__s0_187, __s0_187, 3, 2, 1, 0); \ + uint32x4_t __rev1_187; __rev1_187 = __builtin_shufflevector(__s1_187, __s1_187, 3, 2, 1, 0); \ + uint16x8_t __ret_187; \ + __ret_187 = (uint16x8_t)(__noswap_vcombine_u16((uint16x4_t)(__rev0_187), (uint16x4_t)(__noswap_vqrshrn_n_u32(__rev1_187, __p2_187)))); \ + __ret_187 = __builtin_shufflevector(__ret_187, __ret_187, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_187; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_u64(__p0_188, __p1_188, __p2_188) __extension__ ({ \ + uint32x2_t __s0_188 = __p0_188; \ + uint64x2_t __s1_188 = __p1_188; \ + uint32x4_t __ret_188; \ + __ret_188 = (uint32x4_t)(vcombine_u32((uint32x2_t)(__s0_188), (uint32x2_t)(vqrshrn_n_u64(__s1_188, __p2_188)))); \ + __ret_188; \ +}) +#else +#define vqrshrn_high_n_u64(__p0_189, __p1_189, __p2_189) __extension__ ({ \ + uint32x2_t __s0_189 = __p0_189; \ + uint64x2_t __s1_189 = __p1_189; \ + uint32x2_t __rev0_189; __rev0_189 = __builtin_shufflevector(__s0_189, __s0_189, 1, 0); \ + uint64x2_t __rev1_189; __rev1_189 = __builtin_shufflevector(__s1_189, __s1_189, 1, 0); \ + uint32x4_t __ret_189; \ + __ret_189 = (uint32x4_t)(__noswap_vcombine_u32((uint32x2_t)(__rev0_189), (uint32x2_t)(__noswap_vqrshrn_n_u64(__rev1_189, __p2_189)))); \ + __ret_189 = __builtin_shufflevector(__ret_189, __ret_189, 3, 2, 1, 0); \ + __ret_189; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_u16(__p0_190, __p1_190, __p2_190) __extension__ ({ \ + uint8x8_t __s0_190 = __p0_190; \ + uint16x8_t __s1_190 = __p1_190; \ + uint8x16_t __ret_190; \ + __ret_190 = (uint8x16_t)(vcombine_u8((uint8x8_t)(__s0_190), (uint8x8_t)(vqrshrn_n_u16(__s1_190, __p2_190)))); \ + __ret_190; \ +}) +#else +#define vqrshrn_high_n_u16(__p0_191, __p1_191, __p2_191) __extension__ ({ \ + uint8x8_t __s0_191 = __p0_191; \ + uint16x8_t __s1_191 = __p1_191; \ + uint8x8_t __rev0_191; __rev0_191 = __builtin_shufflevector(__s0_191, __s0_191, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1_191; __rev1_191 = __builtin_shufflevector(__s1_191, __s1_191, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_191; \ + __ret_191 = (uint8x16_t)(__noswap_vcombine_u8((uint8x8_t)(__rev0_191), (uint8x8_t)(__noswap_vqrshrn_n_u16(__rev1_191, __p2_191)))); \ + __ret_191 = __builtin_shufflevector(__ret_191, __ret_191, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_191; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_s32(__p0_192, __p1_192, __p2_192) __extension__ ({ \ + int16x4_t __s0_192 = __p0_192; \ + int32x4_t __s1_192 = __p1_192; \ + int16x8_t __ret_192; \ + __ret_192 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_192), (int16x4_t)(vqrshrn_n_s32(__s1_192, __p2_192)))); \ + __ret_192; \ +}) +#else +#define vqrshrn_high_n_s32(__p0_193, __p1_193, __p2_193) __extension__ ({ \ + int16x4_t __s0_193 = __p0_193; \ + int32x4_t __s1_193 = __p1_193; \ + int16x4_t __rev0_193; __rev0_193 = __builtin_shufflevector(__s0_193, __s0_193, 3, 2, 1, 0); \ + int32x4_t __rev1_193; __rev1_193 = __builtin_shufflevector(__s1_193, __s1_193, 3, 2, 1, 0); \ + int16x8_t __ret_193; \ + __ret_193 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_193), (int16x4_t)(__noswap_vqrshrn_n_s32(__rev1_193, __p2_193)))); \ + __ret_193 = __builtin_shufflevector(__ret_193, __ret_193, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_193; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_s64(__p0_194, __p1_194, __p2_194) __extension__ ({ \ + int32x2_t __s0_194 = __p0_194; \ + int64x2_t __s1_194 = __p1_194; \ + int32x4_t __ret_194; \ + __ret_194 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_194), (int32x2_t)(vqrshrn_n_s64(__s1_194, __p2_194)))); \ + __ret_194; \ +}) +#else +#define vqrshrn_high_n_s64(__p0_195, __p1_195, __p2_195) __extension__ ({ \ + int32x2_t __s0_195 = __p0_195; \ + int64x2_t __s1_195 = __p1_195; \ + int32x2_t __rev0_195; __rev0_195 = __builtin_shufflevector(__s0_195, __s0_195, 1, 0); \ + int64x2_t __rev1_195; __rev1_195 = __builtin_shufflevector(__s1_195, __s1_195, 1, 0); \ + int32x4_t __ret_195; \ + __ret_195 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_195), (int32x2_t)(__noswap_vqrshrn_n_s64(__rev1_195, __p2_195)))); \ + __ret_195 = __builtin_shufflevector(__ret_195, __ret_195, 3, 2, 1, 0); \ + __ret_195; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_s16(__p0_196, __p1_196, __p2_196) __extension__ ({ \ + int8x8_t __s0_196 = __p0_196; \ + int16x8_t __s1_196 = __p1_196; \ + int8x16_t __ret_196; \ + __ret_196 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_196), (int8x8_t)(vqrshrn_n_s16(__s1_196, __p2_196)))); \ + __ret_196; \ +}) +#else +#define vqrshrn_high_n_s16(__p0_197, __p1_197, __p2_197) __extension__ ({ \ + int8x8_t __s0_197 = __p0_197; \ + int16x8_t __s1_197 = __p1_197; \ + int8x8_t __rev0_197; __rev0_197 = __builtin_shufflevector(__s0_197, __s0_197, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_197; __rev1_197 = __builtin_shufflevector(__s1_197, __s1_197, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_197; \ + __ret_197 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_197), (int8x8_t)(__noswap_vqrshrn_n_s16(__rev1_197, __p2_197)))); \ + __ret_197 = __builtin_shufflevector(__ret_197, __ret_197, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_197; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrns_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqrshrns_n_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrns_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqrshrns_n_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrnd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqrshrnd_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrnd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqrshrnd_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrnh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqrshrnh_n_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrnh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqrshrnh_n_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqrshrns_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqrshrns_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrnd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqrshrnd_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrnd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqrshrnd_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrnh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqrshrnh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrnh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqrshrnh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_high_n_s32(__p0_198, __p1_198, __p2_198) __extension__ ({ \ + int16x4_t __s0_198 = __p0_198; \ + int32x4_t __s1_198 = __p1_198; \ + int16x8_t __ret_198; \ + __ret_198 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_198), (int16x4_t)(vqrshrun_n_s32(__s1_198, __p2_198)))); \ + __ret_198; \ +}) +#else +#define vqrshrun_high_n_s32(__p0_199, __p1_199, __p2_199) __extension__ ({ \ + int16x4_t __s0_199 = __p0_199; \ + int32x4_t __s1_199 = __p1_199; \ + int16x4_t __rev0_199; __rev0_199 = __builtin_shufflevector(__s0_199, __s0_199, 3, 2, 1, 0); \ + int32x4_t __rev1_199; __rev1_199 = __builtin_shufflevector(__s1_199, __s1_199, 3, 2, 1, 0); \ + int16x8_t __ret_199; \ + __ret_199 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_199), (int16x4_t)(__noswap_vqrshrun_n_s32(__rev1_199, __p2_199)))); \ + __ret_199 = __builtin_shufflevector(__ret_199, __ret_199, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_199; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_high_n_s64(__p0_200, __p1_200, __p2_200) __extension__ ({ \ + int32x2_t __s0_200 = __p0_200; \ + int64x2_t __s1_200 = __p1_200; \ + int32x4_t __ret_200; \ + __ret_200 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_200), (int32x2_t)(vqrshrun_n_s64(__s1_200, __p2_200)))); \ + __ret_200; \ +}) +#else +#define vqrshrun_high_n_s64(__p0_201, __p1_201, __p2_201) __extension__ ({ \ + int32x2_t __s0_201 = __p0_201; \ + int64x2_t __s1_201 = __p1_201; \ + int32x2_t __rev0_201; __rev0_201 = __builtin_shufflevector(__s0_201, __s0_201, 1, 0); \ + int64x2_t __rev1_201; __rev1_201 = __builtin_shufflevector(__s1_201, __s1_201, 1, 0); \ + int32x4_t __ret_201; \ + __ret_201 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_201), (int32x2_t)(__noswap_vqrshrun_n_s64(__rev1_201, __p2_201)))); \ + __ret_201 = __builtin_shufflevector(__ret_201, __ret_201, 3, 2, 1, 0); \ + __ret_201; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_high_n_s16(__p0_202, __p1_202, __p2_202) __extension__ ({ \ + int8x8_t __s0_202 = __p0_202; \ + int16x8_t __s1_202 = __p1_202; \ + int8x16_t __ret_202; \ + __ret_202 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_202), (int8x8_t)(vqrshrun_n_s16(__s1_202, __p2_202)))); \ + __ret_202; \ +}) +#else +#define vqrshrun_high_n_s16(__p0_203, __p1_203, __p2_203) __extension__ ({ \ + int8x8_t __s0_203 = __p0_203; \ + int16x8_t __s1_203 = __p1_203; \ + int8x8_t __rev0_203; __rev0_203 = __builtin_shufflevector(__s0_203, __s0_203, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_203; __rev1_203 = __builtin_shufflevector(__s1_203, __s1_203, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_203; \ + __ret_203 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_203), (int8x8_t)(__noswap_vqrshrun_n_s16(__rev1_203, __p2_203)))); \ + __ret_203 = __builtin_shufflevector(__ret_203, __ret_203, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_203; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshruns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqrshruns_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshruns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqrshruns_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrund_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqrshrund_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrund_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqrshrund_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrunh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqrshrunh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrunh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqrshrunh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vqshlb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqshlb_u8(__p0, __p1); + return __ret; +} +#else +__ai uint8_t vqshlb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqshlb_u8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vqshls_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqshls_u32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vqshls_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqshls_u32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vqshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqshld_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vqshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqshld_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vqshlh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqshlh_u16(__p0, __p1); + return __ret; +} +#else +__ai uint16_t vqshlh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqshlh_u16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqshlb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqshlb_s8(__p0, __p1); + return __ret; +} +#else +__ai int8_t vqshlb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqshlb_s8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqshls_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqshls_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqshls_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqshls_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqshld_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vqshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqshld_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqshlh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqshlh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqshlh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqshlh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlb_n_u8(__p0, __p1) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqshlb_n_u8(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlb_n_u8(__p0, __p1) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqshlb_n_u8(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshls_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqshls_n_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshls_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqshls_n_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshld_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vqshld_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshld_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vqshld_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqshlh_n_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqshlh_n_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlb_n_s8(__p0, __p1) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshlb_n_s8(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlb_n_s8(__p0, __p1) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshlb_n_s8(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshls_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshls_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshls_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshls_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshld_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqshld_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshld_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqshld_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshlh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshlh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlub_n_s8(__p0, __p1) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshlub_n_s8(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlub_n_s8(__p0, __p1) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshlub_n_s8(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlus_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshlus_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlus_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshlus_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlud_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqshlud_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlud_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqshlud_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshluh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshluh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshluh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshluh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_u32(__p0_204, __p1_204, __p2_204) __extension__ ({ \ + uint16x4_t __s0_204 = __p0_204; \ + uint32x4_t __s1_204 = __p1_204; \ + uint16x8_t __ret_204; \ + __ret_204 = (uint16x8_t)(vcombine_u16((uint16x4_t)(__s0_204), (uint16x4_t)(vqshrn_n_u32(__s1_204, __p2_204)))); \ + __ret_204; \ +}) +#else +#define vqshrn_high_n_u32(__p0_205, __p1_205, __p2_205) __extension__ ({ \ + uint16x4_t __s0_205 = __p0_205; \ + uint32x4_t __s1_205 = __p1_205; \ + uint16x4_t __rev0_205; __rev0_205 = __builtin_shufflevector(__s0_205, __s0_205, 3, 2, 1, 0); \ + uint32x4_t __rev1_205; __rev1_205 = __builtin_shufflevector(__s1_205, __s1_205, 3, 2, 1, 0); \ + uint16x8_t __ret_205; \ + __ret_205 = (uint16x8_t)(__noswap_vcombine_u16((uint16x4_t)(__rev0_205), (uint16x4_t)(__noswap_vqshrn_n_u32(__rev1_205, __p2_205)))); \ + __ret_205 = __builtin_shufflevector(__ret_205, __ret_205, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_205; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_u64(__p0_206, __p1_206, __p2_206) __extension__ ({ \ + uint32x2_t __s0_206 = __p0_206; \ + uint64x2_t __s1_206 = __p1_206; \ + uint32x4_t __ret_206; \ + __ret_206 = (uint32x4_t)(vcombine_u32((uint32x2_t)(__s0_206), (uint32x2_t)(vqshrn_n_u64(__s1_206, __p2_206)))); \ + __ret_206; \ +}) +#else +#define vqshrn_high_n_u64(__p0_207, __p1_207, __p2_207) __extension__ ({ \ + uint32x2_t __s0_207 = __p0_207; \ + uint64x2_t __s1_207 = __p1_207; \ + uint32x2_t __rev0_207; __rev0_207 = __builtin_shufflevector(__s0_207, __s0_207, 1, 0); \ + uint64x2_t __rev1_207; __rev1_207 = __builtin_shufflevector(__s1_207, __s1_207, 1, 0); \ + uint32x4_t __ret_207; \ + __ret_207 = (uint32x4_t)(__noswap_vcombine_u32((uint32x2_t)(__rev0_207), (uint32x2_t)(__noswap_vqshrn_n_u64(__rev1_207, __p2_207)))); \ + __ret_207 = __builtin_shufflevector(__ret_207, __ret_207, 3, 2, 1, 0); \ + __ret_207; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_u16(__p0_208, __p1_208, __p2_208) __extension__ ({ \ + uint8x8_t __s0_208 = __p0_208; \ + uint16x8_t __s1_208 = __p1_208; \ + uint8x16_t __ret_208; \ + __ret_208 = (uint8x16_t)(vcombine_u8((uint8x8_t)(__s0_208), (uint8x8_t)(vqshrn_n_u16(__s1_208, __p2_208)))); \ + __ret_208; \ +}) +#else +#define vqshrn_high_n_u16(__p0_209, __p1_209, __p2_209) __extension__ ({ \ + uint8x8_t __s0_209 = __p0_209; \ + uint16x8_t __s1_209 = __p1_209; \ + uint8x8_t __rev0_209; __rev0_209 = __builtin_shufflevector(__s0_209, __s0_209, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1_209; __rev1_209 = __builtin_shufflevector(__s1_209, __s1_209, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_209; \ + __ret_209 = (uint8x16_t)(__noswap_vcombine_u8((uint8x8_t)(__rev0_209), (uint8x8_t)(__noswap_vqshrn_n_u16(__rev1_209, __p2_209)))); \ + __ret_209 = __builtin_shufflevector(__ret_209, __ret_209, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_209; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_s32(__p0_210, __p1_210, __p2_210) __extension__ ({ \ + int16x4_t __s0_210 = __p0_210; \ + int32x4_t __s1_210 = __p1_210; \ + int16x8_t __ret_210; \ + __ret_210 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_210), (int16x4_t)(vqshrn_n_s32(__s1_210, __p2_210)))); \ + __ret_210; \ +}) +#else +#define vqshrn_high_n_s32(__p0_211, __p1_211, __p2_211) __extension__ ({ \ + int16x4_t __s0_211 = __p0_211; \ + int32x4_t __s1_211 = __p1_211; \ + int16x4_t __rev0_211; __rev0_211 = __builtin_shufflevector(__s0_211, __s0_211, 3, 2, 1, 0); \ + int32x4_t __rev1_211; __rev1_211 = __builtin_shufflevector(__s1_211, __s1_211, 3, 2, 1, 0); \ + int16x8_t __ret_211; \ + __ret_211 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_211), (int16x4_t)(__noswap_vqshrn_n_s32(__rev1_211, __p2_211)))); \ + __ret_211 = __builtin_shufflevector(__ret_211, __ret_211, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_211; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_s64(__p0_212, __p1_212, __p2_212) __extension__ ({ \ + int32x2_t __s0_212 = __p0_212; \ + int64x2_t __s1_212 = __p1_212; \ + int32x4_t __ret_212; \ + __ret_212 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_212), (int32x2_t)(vqshrn_n_s64(__s1_212, __p2_212)))); \ + __ret_212; \ +}) +#else +#define vqshrn_high_n_s64(__p0_213, __p1_213, __p2_213) __extension__ ({ \ + int32x2_t __s0_213 = __p0_213; \ + int64x2_t __s1_213 = __p1_213; \ + int32x2_t __rev0_213; __rev0_213 = __builtin_shufflevector(__s0_213, __s0_213, 1, 0); \ + int64x2_t __rev1_213; __rev1_213 = __builtin_shufflevector(__s1_213, __s1_213, 1, 0); \ + int32x4_t __ret_213; \ + __ret_213 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_213), (int32x2_t)(__noswap_vqshrn_n_s64(__rev1_213, __p2_213)))); \ + __ret_213 = __builtin_shufflevector(__ret_213, __ret_213, 3, 2, 1, 0); \ + __ret_213; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_s16(__p0_214, __p1_214, __p2_214) __extension__ ({ \ + int8x8_t __s0_214 = __p0_214; \ + int16x8_t __s1_214 = __p1_214; \ + int8x16_t __ret_214; \ + __ret_214 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_214), (int8x8_t)(vqshrn_n_s16(__s1_214, __p2_214)))); \ + __ret_214; \ +}) +#else +#define vqshrn_high_n_s16(__p0_215, __p1_215, __p2_215) __extension__ ({ \ + int8x8_t __s0_215 = __p0_215; \ + int16x8_t __s1_215 = __p1_215; \ + int8x8_t __rev0_215; __rev0_215 = __builtin_shufflevector(__s0_215, __s0_215, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_215; __rev1_215 = __builtin_shufflevector(__s1_215, __s1_215, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_215; \ + __ret_215 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_215), (int8x8_t)(__noswap_vqshrn_n_s16(__rev1_215, __p2_215)))); \ + __ret_215 = __builtin_shufflevector(__ret_215, __ret_215, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_215; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrns_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqshrns_n_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrns_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqshrns_n_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrnd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqshrnd_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrnd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqshrnd_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrnh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqshrnh_n_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrnh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqshrnh_n_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshrns_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshrns_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrnd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshrnd_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrnd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshrnd_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrnh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshrnh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrnh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshrnh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_high_n_s32(__p0_216, __p1_216, __p2_216) __extension__ ({ \ + int16x4_t __s0_216 = __p0_216; \ + int32x4_t __s1_216 = __p1_216; \ + int16x8_t __ret_216; \ + __ret_216 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_216), (int16x4_t)(vqshrun_n_s32(__s1_216, __p2_216)))); \ + __ret_216; \ +}) +#else +#define vqshrun_high_n_s32(__p0_217, __p1_217, __p2_217) __extension__ ({ \ + int16x4_t __s0_217 = __p0_217; \ + int32x4_t __s1_217 = __p1_217; \ + int16x4_t __rev0_217; __rev0_217 = __builtin_shufflevector(__s0_217, __s0_217, 3, 2, 1, 0); \ + int32x4_t __rev1_217; __rev1_217 = __builtin_shufflevector(__s1_217, __s1_217, 3, 2, 1, 0); \ + int16x8_t __ret_217; \ + __ret_217 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_217), (int16x4_t)(__noswap_vqshrun_n_s32(__rev1_217, __p2_217)))); \ + __ret_217 = __builtin_shufflevector(__ret_217, __ret_217, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_217; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_high_n_s64(__p0_218, __p1_218, __p2_218) __extension__ ({ \ + int32x2_t __s0_218 = __p0_218; \ + int64x2_t __s1_218 = __p1_218; \ + int32x4_t __ret_218; \ + __ret_218 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_218), (int32x2_t)(vqshrun_n_s64(__s1_218, __p2_218)))); \ + __ret_218; \ +}) +#else +#define vqshrun_high_n_s64(__p0_219, __p1_219, __p2_219) __extension__ ({ \ + int32x2_t __s0_219 = __p0_219; \ + int64x2_t __s1_219 = __p1_219; \ + int32x2_t __rev0_219; __rev0_219 = __builtin_shufflevector(__s0_219, __s0_219, 1, 0); \ + int64x2_t __rev1_219; __rev1_219 = __builtin_shufflevector(__s1_219, __s1_219, 1, 0); \ + int32x4_t __ret_219; \ + __ret_219 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_219), (int32x2_t)(__noswap_vqshrun_n_s64(__rev1_219, __p2_219)))); \ + __ret_219 = __builtin_shufflevector(__ret_219, __ret_219, 3, 2, 1, 0); \ + __ret_219; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_high_n_s16(__p0_220, __p1_220, __p2_220) __extension__ ({ \ + int8x8_t __s0_220 = __p0_220; \ + int16x8_t __s1_220 = __p1_220; \ + int8x16_t __ret_220; \ + __ret_220 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_220), (int8x8_t)(vqshrun_n_s16(__s1_220, __p2_220)))); \ + __ret_220; \ +}) +#else +#define vqshrun_high_n_s16(__p0_221, __p1_221, __p2_221) __extension__ ({ \ + int8x8_t __s0_221 = __p0_221; \ + int16x8_t __s1_221 = __p1_221; \ + int8x8_t __rev0_221; __rev0_221 = __builtin_shufflevector(__s0_221, __s0_221, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_221; __rev1_221 = __builtin_shufflevector(__s1_221, __s1_221, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_221; \ + __ret_221 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_221), (int8x8_t)(__noswap_vqshrun_n_s16(__rev1_221, __p2_221)))); \ + __ret_221 = __builtin_shufflevector(__ret_221, __ret_221, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_221; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshruns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshruns_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshruns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshruns_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrund_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshrund_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrund_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshrund_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrunh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshrunh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrunh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshrunh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vqsubb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqsubb_u8(__p0, __p1); + return __ret; +} +#else +__ai uint8_t vqsubb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqsubb_u8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vqsubs_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqsubs_u32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vqsubs_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqsubs_u32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vqsubd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqsubd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vqsubd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqsubd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vqsubh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqsubh_u16(__p0, __p1); + return __ret; +} +#else +__ai uint16_t vqsubh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqsubh_u16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqsubb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqsubb_s8(__p0, __p1); + return __ret; +} +#else +__ai int8_t vqsubb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqsubb_s8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqsubs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqsubs_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqsubs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqsubs_s32(__p0, __p1); + return __ret; +} +__ai int32_t __noswap_vqsubs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqsubs_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqsubd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqsubd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vqsubd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqsubd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqsubh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqsubh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqsubh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqsubh_s16(__p0, __p1); + return __ret; +} +__ai int16_t __noswap_vqsubh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqsubh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbl1_p8(poly8x16_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbl1_p8(poly8x16_t __p0, uint8x8_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__rev0, (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbl1q_p8(poly8x16_t __p0, uint8x16_t __p1) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__p0, (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbl1q_p8(poly8x16_t __p0, uint8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbl1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbl1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbl1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqtbl1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbl1_u8(uint8x16_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbl1_u8(uint8x16_t __p0, uint8x8_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbl1_s8(int8x16_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqtbl1_s8(int8x16_t __p0, int8x8_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbl2_p8(poly8x16x2_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbl2_p8(poly8x16x2_t __p0, uint8x8_t __p1) { + poly8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbl2q_p8(poly8x16x2_t __p0, uint8x16_t __p1) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbl2q_p8(poly8x16x2_t __p0, uint8x16_t __p1) { + poly8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbl2q_u8(uint8x16x2_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbl2q_u8(uint8x16x2_t __p0, uint8x16_t __p1) { + uint8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbl2q_s8(int8x16x2_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqtbl2q_s8(int8x16x2_t __p0, int8x16_t __p1) { + int8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbl2_u8(uint8x16x2_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbl2_u8(uint8x16x2_t __p0, uint8x8_t __p1) { + uint8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbl2_s8(int8x16x2_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqtbl2_s8(int8x16x2_t __p0, int8x8_t __p1) { + int8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbl3_p8(poly8x16x3_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbl3_p8(poly8x16x3_t __p0, uint8x8_t __p1) { + poly8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbl3q_p8(poly8x16x3_t __p0, uint8x16_t __p1) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbl3q_p8(poly8x16x3_t __p0, uint8x16_t __p1) { + poly8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev1, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbl3q_u8(uint8x16x3_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbl3q_u8(uint8x16x3_t __p0, uint8x16_t __p1) { + uint8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbl3q_s8(int8x16x3_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqtbl3q_s8(int8x16x3_t __p0, int8x16_t __p1) { + int8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbl3_u8(uint8x16x3_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbl3_u8(uint8x16x3_t __p0, uint8x8_t __p1) { + uint8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbl3_s8(int8x16x3_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqtbl3_s8(int8x16x3_t __p0, int8x8_t __p1) { + int8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbl4_p8(poly8x16x4_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbl4_p8(poly8x16x4_t __p0, uint8x8_t __p1) { + poly8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbl4q_p8(poly8x16x4_t __p0, uint8x16_t __p1) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbl4q_p8(poly8x16x4_t __p0, uint8x16_t __p1) { + poly8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x16_t)__rev1, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbl4q_u8(uint8x16x4_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbl4q_u8(uint8x16x4_t __p0, uint8x16_t __p1) { + uint8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbl4q_s8(int8x16x4_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqtbl4q_s8(int8x16x4_t __p0, int8x16_t __p1) { + int8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbl4_u8(uint8x16x4_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbl4_u8(uint8x16x4_t __p0, uint8x8_t __p1) { + uint8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbl4_s8(int8x16x4_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqtbl4_s8(int8x16x4_t __p0, int8x8_t __p1) { + int8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbx1_p8(poly8x8_t __p0, poly8x16_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__p0, (int8x16_t)__p1, (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbx1_p8(poly8x8_t __p0, poly8x16_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbx1q_p8(poly8x16_t __p0, poly8x16_t __p1, uint8x16_t __p2) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbx1q_p8(poly8x16_t __p0, poly8x16_t __p1, uint8x16_t __p2) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbx1q_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbx1q_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbx1q_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 32); + return __ret; +} +#else +__ai int8x16_t vqtbx1q_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbx1_u8(uint8x8_t __p0, uint8x16_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__p0, (int8x16_t)__p1, (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbx1_u8(uint8x8_t __p0, uint8x16_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbx1_s8(int8x8_t __p0, int8x16_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__p0, (int8x16_t)__p1, (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vqtbx1_s8(int8x8_t __p0, int8x16_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbx2_p8(poly8x8_t __p0, poly8x16x2_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbx2_p8(poly8x8_t __p0, poly8x16x2_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbx2q_p8(poly8x16_t __p0, poly8x16x2_t __p1, uint8x16_t __p2) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p2, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbx2q_p8(poly8x16_t __p0, poly8x16x2_t __p1, uint8x16_t __p2) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev2, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbx2q_u8(uint8x16_t __p0, uint8x16x2_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p2, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbx2q_u8(uint8x16_t __p0, uint8x16x2_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev2, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbx2q_s8(int8x16_t __p0, int8x16x2_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p2, 32); + return __ret; +} +#else +__ai int8x16_t vqtbx2q_s8(int8x16_t __p0, int8x16x2_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev2, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbx2_u8(uint8x8_t __p0, uint8x16x2_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbx2_u8(uint8x8_t __p0, uint8x16x2_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbx2_s8(int8x8_t __p0, int8x16x2_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vqtbx2_s8(int8x8_t __p0, int8x16x2_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbx3_p8(poly8x8_t __p0, poly8x16x3_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbx3_p8(poly8x8_t __p0, poly8x16x3_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbx3q_p8(poly8x16_t __p0, poly8x16x3_t __p1, uint8x16_t __p2) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p2, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbx3q_p8(poly8x16_t __p0, poly8x16x3_t __p1, uint8x16_t __p2) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev2, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbx3q_u8(uint8x16_t __p0, uint8x16x3_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p2, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbx3q_u8(uint8x16_t __p0, uint8x16x3_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev2, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbx3q_s8(int8x16_t __p0, int8x16x3_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p2, 32); + return __ret; +} +#else +__ai int8x16_t vqtbx3q_s8(int8x16_t __p0, int8x16x3_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev2, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbx3_u8(uint8x8_t __p0, uint8x16x3_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbx3_u8(uint8x8_t __p0, uint8x16x3_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbx3_s8(int8x8_t __p0, int8x16x3_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vqtbx3_s8(int8x8_t __p0, int8x16x3_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbx4_p8(poly8x8_t __p0, poly8x16x4_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbx4_p8(poly8x8_t __p0, poly8x16x4_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbx4q_p8(poly8x16_t __p0, poly8x16x4_t __p1, uint8x16_t __p2) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x16_t)__p2, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbx4q_p8(poly8x16_t __p0, poly8x16x4_t __p1, uint8x16_t __p2) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x16_t)__rev2, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbx4q_u8(uint8x16_t __p0, uint8x16x4_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x16_t)__p2, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbx4q_u8(uint8x16_t __p0, uint8x16x4_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x16_t)__rev2, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbx4q_s8(int8x16_t __p0, int8x16x4_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x16_t)__p2, 32); + return __ret; +} +#else +__ai int8x16_t vqtbx4q_s8(int8x16_t __p0, int8x16x4_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x16_t)__rev2, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbx4_u8(uint8x8_t __p0, uint8x16x4_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbx4_u8(uint8x8_t __p0, uint8x16x4_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbx4_s8(int8x8_t __p0, int8x16x4_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vqtbx4_s8(int8x8_t __p0, int8x16x4_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vraddhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vraddhn_u32(__p1, __p2)); + return __ret; +} +#else +__ai uint16x8_t vraddhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vraddhn_u32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vraddhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vraddhn_u64(__p1, __p2)); + return __ret; +} +#else +__ai uint32x4_t vraddhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vraddhn_u64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vraddhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vraddhn_u16(__p1, __p2)); + return __ret; +} +#else +__ai uint8x16_t vraddhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vraddhn_u16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vraddhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vraddhn_s32(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vraddhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vraddhn_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vraddhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vraddhn_s64(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vraddhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vraddhn_s64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vraddhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vraddhn_s16(__p1, __p2)); + return __ret; +} +#else +__ai int8x16_t vraddhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vraddhn_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vrbit_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vrbit_v((int8x8_t)__p0, 4); + return __ret; +} +#else +__ai poly8x8_t vrbit_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vrbit_v((int8x8_t)__rev0, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vrbitq_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__p0, 36); + return __ret; +} +#else +__ai poly8x16_t vrbitq_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__rev0, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrbitq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vrbitq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrbitq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vrbitq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrbit_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrbit_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vrbit_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrbit_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrbit_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrbit_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vrbit_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrbit_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrecpeq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrecpeq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrecpeq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrecpeq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrecpe_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrecpe_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrecpe_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrecpe_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vrecped_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecped_f64(__p0); + return __ret; +} +#else +__ai float64_t vrecped_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecped_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vrecpes_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpes_f32(__p0); + return __ret; +} +#else +__ai float32_t vrecpes_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpes_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrecpsq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrecpsq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vrecpsq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrecpsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrecps_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrecps_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vrecps_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrecps_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vrecpsd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecpsd_f64(__p0, __p1); + return __ret; +} +#else +__ai float64_t vrecpsd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecpsd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vrecpss_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpss_f32(__p0, __p1); + return __ret; +} +#else +__ai float32_t vrecpss_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpss_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vrecpxd_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecpxd_f64(__p0); + return __ret; +} +#else +__ai float64_t vrecpxd_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecpxd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vrecpxs_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpxs_f32(__p0); + return __ret; +} +#else +__ai float32_t vrecpxs_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpxs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vrshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vrshld_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vrshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vrshld_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vrshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vrshld_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vrshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vrshld_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vrshrd_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vrshrd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vrshrd_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vrshrd_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vrshrd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vrshrd_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_u32(__p0_222, __p1_222, __p2_222) __extension__ ({ \ + uint16x4_t __s0_222 = __p0_222; \ + uint32x4_t __s1_222 = __p1_222; \ + uint16x8_t __ret_222; \ + __ret_222 = (uint16x8_t)(vcombine_u16((uint16x4_t)(__s0_222), (uint16x4_t)(vrshrn_n_u32(__s1_222, __p2_222)))); \ + __ret_222; \ +}) +#else +#define vrshrn_high_n_u32(__p0_223, __p1_223, __p2_223) __extension__ ({ \ + uint16x4_t __s0_223 = __p0_223; \ + uint32x4_t __s1_223 = __p1_223; \ + uint16x4_t __rev0_223; __rev0_223 = __builtin_shufflevector(__s0_223, __s0_223, 3, 2, 1, 0); \ + uint32x4_t __rev1_223; __rev1_223 = __builtin_shufflevector(__s1_223, __s1_223, 3, 2, 1, 0); \ + uint16x8_t __ret_223; \ + __ret_223 = (uint16x8_t)(__noswap_vcombine_u16((uint16x4_t)(__rev0_223), (uint16x4_t)(__noswap_vrshrn_n_u32(__rev1_223, __p2_223)))); \ + __ret_223 = __builtin_shufflevector(__ret_223, __ret_223, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_223; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_u64(__p0_224, __p1_224, __p2_224) __extension__ ({ \ + uint32x2_t __s0_224 = __p0_224; \ + uint64x2_t __s1_224 = __p1_224; \ + uint32x4_t __ret_224; \ + __ret_224 = (uint32x4_t)(vcombine_u32((uint32x2_t)(__s0_224), (uint32x2_t)(vrshrn_n_u64(__s1_224, __p2_224)))); \ + __ret_224; \ +}) +#else +#define vrshrn_high_n_u64(__p0_225, __p1_225, __p2_225) __extension__ ({ \ + uint32x2_t __s0_225 = __p0_225; \ + uint64x2_t __s1_225 = __p1_225; \ + uint32x2_t __rev0_225; __rev0_225 = __builtin_shufflevector(__s0_225, __s0_225, 1, 0); \ + uint64x2_t __rev1_225; __rev1_225 = __builtin_shufflevector(__s1_225, __s1_225, 1, 0); \ + uint32x4_t __ret_225; \ + __ret_225 = (uint32x4_t)(__noswap_vcombine_u32((uint32x2_t)(__rev0_225), (uint32x2_t)(__noswap_vrshrn_n_u64(__rev1_225, __p2_225)))); \ + __ret_225 = __builtin_shufflevector(__ret_225, __ret_225, 3, 2, 1, 0); \ + __ret_225; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_u16(__p0_226, __p1_226, __p2_226) __extension__ ({ \ + uint8x8_t __s0_226 = __p0_226; \ + uint16x8_t __s1_226 = __p1_226; \ + uint8x16_t __ret_226; \ + __ret_226 = (uint8x16_t)(vcombine_u8((uint8x8_t)(__s0_226), (uint8x8_t)(vrshrn_n_u16(__s1_226, __p2_226)))); \ + __ret_226; \ +}) +#else +#define vrshrn_high_n_u16(__p0_227, __p1_227, __p2_227) __extension__ ({ \ + uint8x8_t __s0_227 = __p0_227; \ + uint16x8_t __s1_227 = __p1_227; \ + uint8x8_t __rev0_227; __rev0_227 = __builtin_shufflevector(__s0_227, __s0_227, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1_227; __rev1_227 = __builtin_shufflevector(__s1_227, __s1_227, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_227; \ + __ret_227 = (uint8x16_t)(__noswap_vcombine_u8((uint8x8_t)(__rev0_227), (uint8x8_t)(__noswap_vrshrn_n_u16(__rev1_227, __p2_227)))); \ + __ret_227 = __builtin_shufflevector(__ret_227, __ret_227, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_227; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_s32(__p0_228, __p1_228, __p2_228) __extension__ ({ \ + int16x4_t __s0_228 = __p0_228; \ + int32x4_t __s1_228 = __p1_228; \ + int16x8_t __ret_228; \ + __ret_228 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_228), (int16x4_t)(vrshrn_n_s32(__s1_228, __p2_228)))); \ + __ret_228; \ +}) +#else +#define vrshrn_high_n_s32(__p0_229, __p1_229, __p2_229) __extension__ ({ \ + int16x4_t __s0_229 = __p0_229; \ + int32x4_t __s1_229 = __p1_229; \ + int16x4_t __rev0_229; __rev0_229 = __builtin_shufflevector(__s0_229, __s0_229, 3, 2, 1, 0); \ + int32x4_t __rev1_229; __rev1_229 = __builtin_shufflevector(__s1_229, __s1_229, 3, 2, 1, 0); \ + int16x8_t __ret_229; \ + __ret_229 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_229), (int16x4_t)(__noswap_vrshrn_n_s32(__rev1_229, __p2_229)))); \ + __ret_229 = __builtin_shufflevector(__ret_229, __ret_229, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_229; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_s64(__p0_230, __p1_230, __p2_230) __extension__ ({ \ + int32x2_t __s0_230 = __p0_230; \ + int64x2_t __s1_230 = __p1_230; \ + int32x4_t __ret_230; \ + __ret_230 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_230), (int32x2_t)(vrshrn_n_s64(__s1_230, __p2_230)))); \ + __ret_230; \ +}) +#else +#define vrshrn_high_n_s64(__p0_231, __p1_231, __p2_231) __extension__ ({ \ + int32x2_t __s0_231 = __p0_231; \ + int64x2_t __s1_231 = __p1_231; \ + int32x2_t __rev0_231; __rev0_231 = __builtin_shufflevector(__s0_231, __s0_231, 1, 0); \ + int64x2_t __rev1_231; __rev1_231 = __builtin_shufflevector(__s1_231, __s1_231, 1, 0); \ + int32x4_t __ret_231; \ + __ret_231 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_231), (int32x2_t)(__noswap_vrshrn_n_s64(__rev1_231, __p2_231)))); \ + __ret_231 = __builtin_shufflevector(__ret_231, __ret_231, 3, 2, 1, 0); \ + __ret_231; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_s16(__p0_232, __p1_232, __p2_232) __extension__ ({ \ + int8x8_t __s0_232 = __p0_232; \ + int16x8_t __s1_232 = __p1_232; \ + int8x16_t __ret_232; \ + __ret_232 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_232), (int8x8_t)(vrshrn_n_s16(__s1_232, __p2_232)))); \ + __ret_232; \ +}) +#else +#define vrshrn_high_n_s16(__p0_233, __p1_233, __p2_233) __extension__ ({ \ + int8x8_t __s0_233 = __p0_233; \ + int16x8_t __s1_233 = __p1_233; \ + int8x8_t __rev0_233; __rev0_233 = __builtin_shufflevector(__s0_233, __s0_233, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_233; __rev1_233 = __builtin_shufflevector(__s1_233, __s1_233, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_233; \ + __ret_233 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_233), (int8x8_t)(__noswap_vrshrn_n_s16(__rev1_233, __p2_233)))); \ + __ret_233 = __builtin_shufflevector(__ret_233, __ret_233, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_233; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrsqrteq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrsqrteq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrsqrteq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrsqrteq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrsqrte_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrsqrte_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrsqrte_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrsqrte_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vrsqrted_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrsqrted_f64(__p0); + return __ret; +} +#else +__ai float64_t vrsqrted_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrsqrted_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vrsqrtes_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrsqrtes_f32(__p0); + return __ret; +} +#else +__ai float32_t vrsqrtes_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrsqrtes_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrsqrtsq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vrsqrtsq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrsqrts_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrsqrts_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vrsqrts_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrsqrts_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vrsqrtsd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrsqrtsd_f64(__p0, __p1); + return __ret; +} +#else +__ai float64_t vrsqrtsd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrsqrtsd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vrsqrtss_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrsqrtss_f32(__p0, __p1); + return __ret; +} +#else +__ai float32_t vrsqrtss_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrsqrtss_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsrad_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vrsrad_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vrsrad_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vrsrad_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsrad_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vrsrad_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vrsrad_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vrsrad_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vrsubhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vrsubhn_u32(__p1, __p2)); + return __ret; +} +#else +__ai uint16x8_t vrsubhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vrsubhn_u32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrsubhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vrsubhn_u64(__p1, __p2)); + return __ret; +} +#else +__ai uint32x4_t vrsubhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vrsubhn_u64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrsubhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vrsubhn_u16(__p1, __p2)); + return __ret; +} +#else +__ai uint8x16_t vrsubhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vrsubhn_u16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vrsubhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vrsubhn_s32(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vrsubhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vrsubhn_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vrsubhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vrsubhn_s64(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vrsubhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vrsubhn_s64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrsubhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vrsubhn_s16(__p1, __p2)); + return __ret; +} +#else +__ai int8x16_t vrsubhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vrsubhn_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#define __noswap_vset_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vsetq_lane_f64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vsetq_lane_f64(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vsetq_lane_f64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vset_lane_f64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vset_lane_f64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#define __noswap_vset_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vset_lane_f64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vshld_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vshld_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vshld_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vshld_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshld_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vshld_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vshld_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vshld_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshld_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vshld_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vshld_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vshld_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_u8(__p0_234, __p1_234) __extension__ ({ \ + uint8x16_t __s0_234 = __p0_234; \ + uint16x8_t __ret_234; \ + __ret_234 = (uint16x8_t)(vshll_n_u8(vget_high_u8(__s0_234), __p1_234)); \ + __ret_234; \ +}) +#else +#define vshll_high_n_u8(__p0_235, __p1_235) __extension__ ({ \ + uint8x16_t __s0_235 = __p0_235; \ + uint8x16_t __rev0_235; __rev0_235 = __builtin_shufflevector(__s0_235, __s0_235, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret_235; \ + __ret_235 = (uint16x8_t)(__noswap_vshll_n_u8(__noswap_vget_high_u8(__rev0_235), __p1_235)); \ + __ret_235 = __builtin_shufflevector(__ret_235, __ret_235, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_235; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_u32(__p0_236, __p1_236) __extension__ ({ \ + uint32x4_t __s0_236 = __p0_236; \ + uint64x2_t __ret_236; \ + __ret_236 = (uint64x2_t)(vshll_n_u32(vget_high_u32(__s0_236), __p1_236)); \ + __ret_236; \ +}) +#else +#define vshll_high_n_u32(__p0_237, __p1_237) __extension__ ({ \ + uint32x4_t __s0_237 = __p0_237; \ + uint32x4_t __rev0_237; __rev0_237 = __builtin_shufflevector(__s0_237, __s0_237, 3, 2, 1, 0); \ + uint64x2_t __ret_237; \ + __ret_237 = (uint64x2_t)(__noswap_vshll_n_u32(__noswap_vget_high_u32(__rev0_237), __p1_237)); \ + __ret_237 = __builtin_shufflevector(__ret_237, __ret_237, 1, 0); \ + __ret_237; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_u16(__p0_238, __p1_238) __extension__ ({ \ + uint16x8_t __s0_238 = __p0_238; \ + uint32x4_t __ret_238; \ + __ret_238 = (uint32x4_t)(vshll_n_u16(vget_high_u16(__s0_238), __p1_238)); \ + __ret_238; \ +}) +#else +#define vshll_high_n_u16(__p0_239, __p1_239) __extension__ ({ \ + uint16x8_t __s0_239 = __p0_239; \ + uint16x8_t __rev0_239; __rev0_239 = __builtin_shufflevector(__s0_239, __s0_239, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret_239; \ + __ret_239 = (uint32x4_t)(__noswap_vshll_n_u16(__noswap_vget_high_u16(__rev0_239), __p1_239)); \ + __ret_239 = __builtin_shufflevector(__ret_239, __ret_239, 3, 2, 1, 0); \ + __ret_239; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_s8(__p0_240, __p1_240) __extension__ ({ \ + int8x16_t __s0_240 = __p0_240; \ + int16x8_t __ret_240; \ + __ret_240 = (int16x8_t)(vshll_n_s8(vget_high_s8(__s0_240), __p1_240)); \ + __ret_240; \ +}) +#else +#define vshll_high_n_s8(__p0_241, __p1_241) __extension__ ({ \ + int8x16_t __s0_241 = __p0_241; \ + int8x16_t __rev0_241; __rev0_241 = __builtin_shufflevector(__s0_241, __s0_241, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret_241; \ + __ret_241 = (int16x8_t)(__noswap_vshll_n_s8(__noswap_vget_high_s8(__rev0_241), __p1_241)); \ + __ret_241 = __builtin_shufflevector(__ret_241, __ret_241, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_241; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_s32(__p0_242, __p1_242) __extension__ ({ \ + int32x4_t __s0_242 = __p0_242; \ + int64x2_t __ret_242; \ + __ret_242 = (int64x2_t)(vshll_n_s32(vget_high_s32(__s0_242), __p1_242)); \ + __ret_242; \ +}) +#else +#define vshll_high_n_s32(__p0_243, __p1_243) __extension__ ({ \ + int32x4_t __s0_243 = __p0_243; \ + int32x4_t __rev0_243; __rev0_243 = __builtin_shufflevector(__s0_243, __s0_243, 3, 2, 1, 0); \ + int64x2_t __ret_243; \ + __ret_243 = (int64x2_t)(__noswap_vshll_n_s32(__noswap_vget_high_s32(__rev0_243), __p1_243)); \ + __ret_243 = __builtin_shufflevector(__ret_243, __ret_243, 1, 0); \ + __ret_243; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_s16(__p0_244, __p1_244) __extension__ ({ \ + int16x8_t __s0_244 = __p0_244; \ + int32x4_t __ret_244; \ + __ret_244 = (int32x4_t)(vshll_n_s16(vget_high_s16(__s0_244), __p1_244)); \ + __ret_244; \ +}) +#else +#define vshll_high_n_s16(__p0_245, __p1_245) __extension__ ({ \ + int16x8_t __s0_245 = __p0_245; \ + int16x8_t __rev0_245; __rev0_245 = __builtin_shufflevector(__s0_245, __s0_245, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret_245; \ + __ret_245 = (int32x4_t)(__noswap_vshll_n_s16(__noswap_vget_high_s16(__rev0_245), __p1_245)); \ + __ret_245 = __builtin_shufflevector(__ret_245, __ret_245, 3, 2, 1, 0); \ + __ret_245; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vshrd_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vshrd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vshrd_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vshrd_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vshrd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vshrd_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_u32(__p0_246, __p1_246, __p2_246) __extension__ ({ \ + uint16x4_t __s0_246 = __p0_246; \ + uint32x4_t __s1_246 = __p1_246; \ + uint16x8_t __ret_246; \ + __ret_246 = (uint16x8_t)(vcombine_u16((uint16x4_t)(__s0_246), (uint16x4_t)(vshrn_n_u32(__s1_246, __p2_246)))); \ + __ret_246; \ +}) +#else +#define vshrn_high_n_u32(__p0_247, __p1_247, __p2_247) __extension__ ({ \ + uint16x4_t __s0_247 = __p0_247; \ + uint32x4_t __s1_247 = __p1_247; \ + uint16x4_t __rev0_247; __rev0_247 = __builtin_shufflevector(__s0_247, __s0_247, 3, 2, 1, 0); \ + uint32x4_t __rev1_247; __rev1_247 = __builtin_shufflevector(__s1_247, __s1_247, 3, 2, 1, 0); \ + uint16x8_t __ret_247; \ + __ret_247 = (uint16x8_t)(__noswap_vcombine_u16((uint16x4_t)(__rev0_247), (uint16x4_t)(__noswap_vshrn_n_u32(__rev1_247, __p2_247)))); \ + __ret_247 = __builtin_shufflevector(__ret_247, __ret_247, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_247; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_u64(__p0_248, __p1_248, __p2_248) __extension__ ({ \ + uint32x2_t __s0_248 = __p0_248; \ + uint64x2_t __s1_248 = __p1_248; \ + uint32x4_t __ret_248; \ + __ret_248 = (uint32x4_t)(vcombine_u32((uint32x2_t)(__s0_248), (uint32x2_t)(vshrn_n_u64(__s1_248, __p2_248)))); \ + __ret_248; \ +}) +#else +#define vshrn_high_n_u64(__p0_249, __p1_249, __p2_249) __extension__ ({ \ + uint32x2_t __s0_249 = __p0_249; \ + uint64x2_t __s1_249 = __p1_249; \ + uint32x2_t __rev0_249; __rev0_249 = __builtin_shufflevector(__s0_249, __s0_249, 1, 0); \ + uint64x2_t __rev1_249; __rev1_249 = __builtin_shufflevector(__s1_249, __s1_249, 1, 0); \ + uint32x4_t __ret_249; \ + __ret_249 = (uint32x4_t)(__noswap_vcombine_u32((uint32x2_t)(__rev0_249), (uint32x2_t)(__noswap_vshrn_n_u64(__rev1_249, __p2_249)))); \ + __ret_249 = __builtin_shufflevector(__ret_249, __ret_249, 3, 2, 1, 0); \ + __ret_249; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_u16(__p0_250, __p1_250, __p2_250) __extension__ ({ \ + uint8x8_t __s0_250 = __p0_250; \ + uint16x8_t __s1_250 = __p1_250; \ + uint8x16_t __ret_250; \ + __ret_250 = (uint8x16_t)(vcombine_u8((uint8x8_t)(__s0_250), (uint8x8_t)(vshrn_n_u16(__s1_250, __p2_250)))); \ + __ret_250; \ +}) +#else +#define vshrn_high_n_u16(__p0_251, __p1_251, __p2_251) __extension__ ({ \ + uint8x8_t __s0_251 = __p0_251; \ + uint16x8_t __s1_251 = __p1_251; \ + uint8x8_t __rev0_251; __rev0_251 = __builtin_shufflevector(__s0_251, __s0_251, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1_251; __rev1_251 = __builtin_shufflevector(__s1_251, __s1_251, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_251; \ + __ret_251 = (uint8x16_t)(__noswap_vcombine_u8((uint8x8_t)(__rev0_251), (uint8x8_t)(__noswap_vshrn_n_u16(__rev1_251, __p2_251)))); \ + __ret_251 = __builtin_shufflevector(__ret_251, __ret_251, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_251; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_s32(__p0_252, __p1_252, __p2_252) __extension__ ({ \ + int16x4_t __s0_252 = __p0_252; \ + int32x4_t __s1_252 = __p1_252; \ + int16x8_t __ret_252; \ + __ret_252 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_252), (int16x4_t)(vshrn_n_s32(__s1_252, __p2_252)))); \ + __ret_252; \ +}) +#else +#define vshrn_high_n_s32(__p0_253, __p1_253, __p2_253) __extension__ ({ \ + int16x4_t __s0_253 = __p0_253; \ + int32x4_t __s1_253 = __p1_253; \ + int16x4_t __rev0_253; __rev0_253 = __builtin_shufflevector(__s0_253, __s0_253, 3, 2, 1, 0); \ + int32x4_t __rev1_253; __rev1_253 = __builtin_shufflevector(__s1_253, __s1_253, 3, 2, 1, 0); \ + int16x8_t __ret_253; \ + __ret_253 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_253), (int16x4_t)(__noswap_vshrn_n_s32(__rev1_253, __p2_253)))); \ + __ret_253 = __builtin_shufflevector(__ret_253, __ret_253, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_253; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_s64(__p0_254, __p1_254, __p2_254) __extension__ ({ \ + int32x2_t __s0_254 = __p0_254; \ + int64x2_t __s1_254 = __p1_254; \ + int32x4_t __ret_254; \ + __ret_254 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_254), (int32x2_t)(vshrn_n_s64(__s1_254, __p2_254)))); \ + __ret_254; \ +}) +#else +#define vshrn_high_n_s64(__p0_255, __p1_255, __p2_255) __extension__ ({ \ + int32x2_t __s0_255 = __p0_255; \ + int64x2_t __s1_255 = __p1_255; \ + int32x2_t __rev0_255; __rev0_255 = __builtin_shufflevector(__s0_255, __s0_255, 1, 0); \ + int64x2_t __rev1_255; __rev1_255 = __builtin_shufflevector(__s1_255, __s1_255, 1, 0); \ + int32x4_t __ret_255; \ + __ret_255 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_255), (int32x2_t)(__noswap_vshrn_n_s64(__rev1_255, __p2_255)))); \ + __ret_255 = __builtin_shufflevector(__ret_255, __ret_255, 3, 2, 1, 0); \ + __ret_255; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_s16(__p0_256, __p1_256, __p2_256) __extension__ ({ \ + int8x8_t __s0_256 = __p0_256; \ + int16x8_t __s1_256 = __p1_256; \ + int8x16_t __ret_256; \ + __ret_256 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_256), (int8x8_t)(vshrn_n_s16(__s1_256, __p2_256)))); \ + __ret_256; \ +}) +#else +#define vshrn_high_n_s16(__p0_257, __p1_257, __p2_257) __extension__ ({ \ + int8x8_t __s0_257 = __p0_257; \ + int16x8_t __s1_257 = __p1_257; \ + int8x8_t __rev0_257; __rev0_257 = __builtin_shufflevector(__s0_257, __s0_257, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_257; __rev1_257 = __builtin_shufflevector(__s1_257, __s1_257, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_257; \ + __ret_257 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_257), (int8x8_t)(__noswap_vshrn_n_s16(__rev1_257, __p2_257)))); \ + __ret_257 = __builtin_shufflevector(__ret_257, __ret_257, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_257; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vslid_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vslid_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vslid_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vslid_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vslid_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vslid_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vslid_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vslid_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#else +#define vsli_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 38); \ + __ret; \ +}) +#else +#define vsliq_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vsqaddb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vsqaddb_u8(__p0, __p1); + return __ret; +} +#else +__ai uint8_t vsqaddb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vsqaddb_u8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vsqadds_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vsqadds_u32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vsqadds_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vsqadds_u32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vsqaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vsqaddd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vsqaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vsqaddd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vsqaddh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vsqaddh_u16(__p0, __p1); + return __ret; +} +#else +__ai uint16_t vsqaddh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vsqaddh_u16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vsqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vsqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vsqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vsqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vsqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vsqadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vsqadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vsqadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vsqadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vsqadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vsqadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vsqadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vsqadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vsqrtq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vsqrtq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vsqrtq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vsqrtq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vsqrtq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vsqrtq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vsqrtq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vsqrtq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vsqrt_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vsqrt_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vsqrt_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vsqrt_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vsqrt_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vsqrt_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vsqrt_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vsqrt_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsrad_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vsrad_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vsrad_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vsrad_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsrad_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vsrad_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vsrad_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vsrad_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsrid_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vsrid_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vsrid_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vsrid_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsrid_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vsrid_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vsrid_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vsrid_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#else +#define vsri_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 38); \ + __ret; \ +}) +#else +#define vsriq_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 6); \ +}) +#else +#define vst1_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 38); \ +}) +#else +#define vst1q_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 42); \ +}) +#else +#define vst1q_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 10); \ +}) +#else +#define vst1_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 6); \ +}) +#else +#define vst1_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 38); \ +}) +#else +#define vst1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 42); \ +}) +#else +#define vst1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 10); \ +}) +#else +#define vst1_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p8_x2(__p0, __p1) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 4); \ +}) +#else +#define vst1_p8_x2(__p0, __p1) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + poly8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p64_x2(__p0, __p1) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 6); \ +}) +#else +#define vst1_p64_x2(__p0, __p1) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p16_x2(__p0, __p1) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 5); \ +}) +#else +#define vst1_p16_x2(__p0, __p1) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + poly16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p8_x2(__p0, __p1) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 36); \ +}) +#else +#define vst1q_p8_x2(__p0, __p1) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + poly8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p64_x2(__p0, __p1) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 38); \ +}) +#else +#define vst1q_p64_x2(__p0, __p1) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + poly64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p16_x2(__p0, __p1) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 37); \ +}) +#else +#define vst1q_p16_x2(__p0, __p1) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + poly16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u8_x2(__p0, __p1) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 48); \ +}) +#else +#define vst1q_u8_x2(__p0, __p1) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + uint8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u32_x2(__p0, __p1) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 50); \ +}) +#else +#define vst1q_u32_x2(__p0, __p1) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + uint32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u64_x2(__p0, __p1) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 51); \ +}) +#else +#define vst1q_u64_x2(__p0, __p1) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + uint64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u16_x2(__p0, __p1) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 49); \ +}) +#else +#define vst1q_u16_x2(__p0, __p1) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + uint16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s8_x2(__p0, __p1) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 32); \ +}) +#else +#define vst1q_s8_x2(__p0, __p1) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + int8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f64_x2(__p0, __p1) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 42); \ +}) +#else +#define vst1q_f64_x2(__p0, __p1) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + float64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f32_x2(__p0, __p1) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 41); \ +}) +#else +#define vst1q_f32_x2(__p0, __p1) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + float32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f16_x2(__p0, __p1) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 40); \ +}) +#else +#define vst1q_f16_x2(__p0, __p1) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + float16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s32_x2(__p0, __p1) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 34); \ +}) +#else +#define vst1q_s32_x2(__p0, __p1) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + int32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s64_x2(__p0, __p1) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 35); \ +}) +#else +#define vst1q_s64_x2(__p0, __p1) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + int64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s16_x2(__p0, __p1) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 33); \ +}) +#else +#define vst1q_s16_x2(__p0, __p1) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + int16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u8_x2(__p0, __p1) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 16); \ +}) +#else +#define vst1_u8_x2(__p0, __p1) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + uint8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u32_x2(__p0, __p1) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 18); \ +}) +#else +#define vst1_u32_x2(__p0, __p1) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + uint32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u64_x2(__p0, __p1) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 19); \ +}) +#else +#define vst1_u64_x2(__p0, __p1) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u16_x2(__p0, __p1) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 17); \ +}) +#else +#define vst1_u16_x2(__p0, __p1) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + uint16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s8_x2(__p0, __p1) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 0); \ +}) +#else +#define vst1_s8_x2(__p0, __p1) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + int8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f64_x2(__p0, __p1) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 10); \ +}) +#else +#define vst1_f64_x2(__p0, __p1) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f32_x2(__p0, __p1) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 9); \ +}) +#else +#define vst1_f32_x2(__p0, __p1) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + float32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, __rev1.val[0], __rev1.val[1], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f16_x2(__p0, __p1) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 8); \ +}) +#else +#define vst1_f16_x2(__p0, __p1) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + float16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, __rev1.val[0], __rev1.val[1], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s32_x2(__p0, __p1) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 2); \ +}) +#else +#define vst1_s32_x2(__p0, __p1) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + int32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, __rev1.val[0], __rev1.val[1], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s64_x2(__p0, __p1) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 3); \ +}) +#else +#define vst1_s64_x2(__p0, __p1) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s16_x2(__p0, __p1) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 1); \ +}) +#else +#define vst1_s16_x2(__p0, __p1) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + int16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, __rev1.val[0], __rev1.val[1], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p8_x3(__p0, __p1) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 4); \ +}) +#else +#define vst1_p8_x3(__p0, __p1) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + poly8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p64_x3(__p0, __p1) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 6); \ +}) +#else +#define vst1_p64_x3(__p0, __p1) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p16_x3(__p0, __p1) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 5); \ +}) +#else +#define vst1_p16_x3(__p0, __p1) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + poly16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p8_x3(__p0, __p1) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 36); \ +}) +#else +#define vst1q_p8_x3(__p0, __p1) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + poly8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p64_x3(__p0, __p1) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 38); \ +}) +#else +#define vst1q_p64_x3(__p0, __p1) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + poly64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p16_x3(__p0, __p1) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 37); \ +}) +#else +#define vst1q_p16_x3(__p0, __p1) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + poly16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u8_x3(__p0, __p1) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 48); \ +}) +#else +#define vst1q_u8_x3(__p0, __p1) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + uint8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u32_x3(__p0, __p1) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 50); \ +}) +#else +#define vst1q_u32_x3(__p0, __p1) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + uint32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u64_x3(__p0, __p1) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 51); \ +}) +#else +#define vst1q_u64_x3(__p0, __p1) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + uint64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u16_x3(__p0, __p1) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 49); \ +}) +#else +#define vst1q_u16_x3(__p0, __p1) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + uint16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s8_x3(__p0, __p1) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 32); \ +}) +#else +#define vst1q_s8_x3(__p0, __p1) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + int8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f64_x3(__p0, __p1) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 42); \ +}) +#else +#define vst1q_f64_x3(__p0, __p1) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + float64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f32_x3(__p0, __p1) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 41); \ +}) +#else +#define vst1q_f32_x3(__p0, __p1) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + float32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f16_x3(__p0, __p1) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 40); \ +}) +#else +#define vst1q_f16_x3(__p0, __p1) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + float16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s32_x3(__p0, __p1) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 34); \ +}) +#else +#define vst1q_s32_x3(__p0, __p1) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + int32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s64_x3(__p0, __p1) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 35); \ +}) +#else +#define vst1q_s64_x3(__p0, __p1) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + int64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s16_x3(__p0, __p1) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 33); \ +}) +#else +#define vst1q_s16_x3(__p0, __p1) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + int16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u8_x3(__p0, __p1) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 16); \ +}) +#else +#define vst1_u8_x3(__p0, __p1) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + uint8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u32_x3(__p0, __p1) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 18); \ +}) +#else +#define vst1_u32_x3(__p0, __p1) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + uint32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u64_x3(__p0, __p1) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 19); \ +}) +#else +#define vst1_u64_x3(__p0, __p1) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u16_x3(__p0, __p1) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 17); \ +}) +#else +#define vst1_u16_x3(__p0, __p1) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + uint16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s8_x3(__p0, __p1) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 0); \ +}) +#else +#define vst1_s8_x3(__p0, __p1) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + int8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f64_x3(__p0, __p1) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 10); \ +}) +#else +#define vst1_f64_x3(__p0, __p1) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f32_x3(__p0, __p1) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 9); \ +}) +#else +#define vst1_f32_x3(__p0, __p1) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + float32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f16_x3(__p0, __p1) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 8); \ +}) +#else +#define vst1_f16_x3(__p0, __p1) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + float16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s32_x3(__p0, __p1) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 2); \ +}) +#else +#define vst1_s32_x3(__p0, __p1) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + int32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s64_x3(__p0, __p1) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 3); \ +}) +#else +#define vst1_s64_x3(__p0, __p1) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s16_x3(__p0, __p1) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 1); \ +}) +#else +#define vst1_s16_x3(__p0, __p1) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + int16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p8_x4(__p0, __p1) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 4); \ +}) +#else +#define vst1_p8_x4(__p0, __p1) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + poly8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p64_x4(__p0, __p1) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 6); \ +}) +#else +#define vst1_p64_x4(__p0, __p1) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p16_x4(__p0, __p1) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 5); \ +}) +#else +#define vst1_p16_x4(__p0, __p1) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + poly16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p8_x4(__p0, __p1) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 36); \ +}) +#else +#define vst1q_p8_x4(__p0, __p1) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + poly8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p64_x4(__p0, __p1) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 38); \ +}) +#else +#define vst1q_p64_x4(__p0, __p1) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + poly64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p16_x4(__p0, __p1) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 37); \ +}) +#else +#define vst1q_p16_x4(__p0, __p1) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + poly16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u8_x4(__p0, __p1) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 48); \ +}) +#else +#define vst1q_u8_x4(__p0, __p1) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + uint8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u32_x4(__p0, __p1) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 50); \ +}) +#else +#define vst1q_u32_x4(__p0, __p1) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + uint32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u64_x4(__p0, __p1) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 51); \ +}) +#else +#define vst1q_u64_x4(__p0, __p1) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + uint64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u16_x4(__p0, __p1) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 49); \ +}) +#else +#define vst1q_u16_x4(__p0, __p1) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + uint16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s8_x4(__p0, __p1) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 32); \ +}) +#else +#define vst1q_s8_x4(__p0, __p1) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + int8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f64_x4(__p0, __p1) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 42); \ +}) +#else +#define vst1q_f64_x4(__p0, __p1) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + float64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f32_x4(__p0, __p1) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 41); \ +}) +#else +#define vst1q_f32_x4(__p0, __p1) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + float32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f16_x4(__p0, __p1) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 40); \ +}) +#else +#define vst1q_f16_x4(__p0, __p1) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + float16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s32_x4(__p0, __p1) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 34); \ +}) +#else +#define vst1q_s32_x4(__p0, __p1) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + int32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s64_x4(__p0, __p1) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 35); \ +}) +#else +#define vst1q_s64_x4(__p0, __p1) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + int64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s16_x4(__p0, __p1) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 33); \ +}) +#else +#define vst1q_s16_x4(__p0, __p1) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + int16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u8_x4(__p0, __p1) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 16); \ +}) +#else +#define vst1_u8_x4(__p0, __p1) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + uint8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u32_x4(__p0, __p1) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 18); \ +}) +#else +#define vst1_u32_x4(__p0, __p1) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + uint32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u64_x4(__p0, __p1) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 19); \ +}) +#else +#define vst1_u64_x4(__p0, __p1) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u16_x4(__p0, __p1) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 17); \ +}) +#else +#define vst1_u16_x4(__p0, __p1) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + uint16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s8_x4(__p0, __p1) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 0); \ +}) +#else +#define vst1_s8_x4(__p0, __p1) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + int8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f64_x4(__p0, __p1) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 10); \ +}) +#else +#define vst1_f64_x4(__p0, __p1) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f32_x4(__p0, __p1) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 9); \ +}) +#else +#define vst1_f32_x4(__p0, __p1) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + float32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f16_x4(__p0, __p1) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 8); \ +}) +#else +#define vst1_f16_x4(__p0, __p1) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + float16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s32_x4(__p0, __p1) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 2); \ +}) +#else +#define vst1_s32_x4(__p0, __p1) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + int32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s64_x4(__p0, __p1) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 3); \ +}) +#else +#define vst1_s64_x4(__p0, __p1) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s16_x4(__p0, __p1) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 1); \ +}) +#else +#define vst1_s16_x4(__p0, __p1) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + int16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_p64(__p0, __p1) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 6); \ +}) +#else +#define vst2_p64(__p0, __p1) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 38); \ +}) +#else +#define vst2q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + poly64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 51); \ +}) +#else +#define vst2q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + uint64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_f64(__p0, __p1) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 42); \ +}) +#else +#define vst2q_f64(__p0, __p1) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + float64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_s64(__p0, __p1) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 35); \ +}) +#else +#define vst2q_s64(__p0, __p1) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + int64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_f64(__p0, __p1) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 10); \ +}) +#else +#define vst2_f64(__p0, __p1) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 6); \ +}) +#else +#define vst2_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 36); \ +}) +#else +#define vst2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + poly8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 38); \ +}) +#else +#define vst2q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + poly64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 48); \ +}) +#else +#define vst2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + uint8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 51); \ +}) +#else +#define vst2q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + uint64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 32); \ +}) +#else +#define vst2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + int8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 42); \ +}) +#else +#define vst2q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + float64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 35); \ +}) +#else +#define vst2q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + int64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 19); \ +}) +#else +#define vst2_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 10); \ +}) +#else +#define vst2_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 3); \ +}) +#else +#define vst2_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_p64(__p0, __p1) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 6); \ +}) +#else +#define vst3_p64(__p0, __p1) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 38); \ +}) +#else +#define vst3q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + poly64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 51); \ +}) +#else +#define vst3q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + uint64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_f64(__p0, __p1) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 42); \ +}) +#else +#define vst3q_f64(__p0, __p1) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + float64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_s64(__p0, __p1) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 35); \ +}) +#else +#define vst3q_s64(__p0, __p1) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + int64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_f64(__p0, __p1) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 10); \ +}) +#else +#define vst3_f64(__p0, __p1) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 6); \ +}) +#else +#define vst3_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 36); \ +}) +#else +#define vst3q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + poly8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 38); \ +}) +#else +#define vst3q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + poly64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 48); \ +}) +#else +#define vst3q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + uint8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 51); \ +}) +#else +#define vst3q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + uint64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 32); \ +}) +#else +#define vst3q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + int8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 42); \ +}) +#else +#define vst3q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + float64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 35); \ +}) +#else +#define vst3q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + int64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 19); \ +}) +#else +#define vst3_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 10); \ +}) +#else +#define vst3_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 3); \ +}) +#else +#define vst3_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_p64(__p0, __p1) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 6); \ +}) +#else +#define vst4_p64(__p0, __p1) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 38); \ +}) +#else +#define vst4q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + poly64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 51); \ +}) +#else +#define vst4q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + uint64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_f64(__p0, __p1) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 42); \ +}) +#else +#define vst4q_f64(__p0, __p1) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + float64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_s64(__p0, __p1) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 35); \ +}) +#else +#define vst4q_s64(__p0, __p1) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + int64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_f64(__p0, __p1) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 10); \ +}) +#else +#define vst4_f64(__p0, __p1) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 6); \ +}) +#else +#define vst4_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 36); \ +}) +#else +#define vst4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + poly8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 38); \ +}) +#else +#define vst4q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + poly64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 48); \ +}) +#else +#define vst4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + uint8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 51); \ +}) +#else +#define vst4q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + uint64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 32); \ +}) +#else +#define vst4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + int8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 42); \ +}) +#else +#define vst4q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + float64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 35); \ +}) +#else +#define vst4q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + int64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 19); \ +}) +#else +#define vst4_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 10); \ +}) +#else +#define vst4_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 3); \ +}) +#else +#define vst4_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vstrq_p128(__p0, __p1) __extension__ ({ \ + poly128_t __s1 = __p1; \ + __builtin_neon_vstrq_p128(__p0, __s1); \ +}) +#else +#define vstrq_p128(__p0, __p1) __extension__ ({ \ + poly128_t __s1 = __p1; \ + __builtin_neon_vstrq_p128(__p0, __s1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vsubd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vsubd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vsubd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vsubd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vsubd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vsubd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vsubd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vsubd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vsubq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float64x2_t vsubq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vsub_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float64x1_t vsub_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vsubhn_u32(__p1, __p2)); + return __ret; +} +#else +__ai uint16x8_t vsubhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vsubhn_u32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vsubhn_u64(__p1, __p2)); + return __ret; +} +#else +__ai uint32x4_t vsubhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vsubhn_u64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vsubhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vsubhn_u16(__p1, __p2)); + return __ret; +} +#else +__ai uint8x16_t vsubhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vsubhn_u16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vsubhn_s32(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vsubhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vsubhn_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vsubhn_s64(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vsubhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vsubhn_s64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vsubhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vsubhn_s16(__p1, __p2)); + return __ret; +} +#else +__ai int8x16_t vsubhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vsubhn_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = vmovl_high_u8(__p0) - vmovl_high_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vsubl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmovl_high_u8(__rev0) - __noswap_vmovl_high_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsubl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = vmovl_high_u32(__p0) - vmovl_high_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vsubl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmovl_high_u32(__rev0) - __noswap_vmovl_high_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = vmovl_high_u16(__p0) - vmovl_high_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vsubl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmovl_high_u16(__rev0) - __noswap_vmovl_high_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = vmovl_high_s8(__p0) - vmovl_high_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vsubl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmovl_high_s8(__rev0) - __noswap_vmovl_high_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vsubl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = vmovl_high_s32(__p0) - vmovl_high_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vsubl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmovl_high_s32(__rev0) - __noswap_vmovl_high_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = vmovl_high_s16(__p0) - vmovl_high_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vsubl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmovl_high_s16(__rev0) - __noswap_vmovl_high_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubw_high_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = __p0 - vmovl_high_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vsubw_high_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __noswap_vmovl_high_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsubw_high_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = __p0 - vmovl_high_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vsubw_high_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __rev0 - __noswap_vmovl_high_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubw_high_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = __p0 - vmovl_high_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vsubw_high_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __noswap_vmovl_high_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubw_high_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = __p0 - vmovl_high_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vsubw_high_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __noswap_vmovl_high_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vsubw_high_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = __p0 - vmovl_high_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vsubw_high_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __rev0 - __noswap_vmovl_high_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubw_high_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = __p0 - vmovl_high_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vsubw_high_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __noswap_vmovl_high_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtrn1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai poly8x8_t vtrn1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vtrn1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai poly16x4_t vtrn1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vtrn1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + return __ret; +} +#else +__ai poly8x16_t vtrn1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vtrn1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai poly64x2_t vtrn1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vtrn1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai poly16x8_t vtrn1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vtrn1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + return __ret; +} +#else +__ai uint8x16_t vtrn1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vtrn1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai uint32x4_t vtrn1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vtrn1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint64x2_t vtrn1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vtrn1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai uint16x8_t vtrn1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vtrn1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + return __ret; +} +#else +__ai int8x16_t vtrn1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vtrn1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float64x2_t vtrn1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vtrn1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai float32x4_t vtrn1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vtrn1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai int32x4_t vtrn1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vtrn1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int64x2_t vtrn1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vtrn1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai int16x8_t vtrn1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtrn1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai uint8x8_t vtrn1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vtrn1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint32x2_t vtrn1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vtrn1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai uint16x4_t vtrn1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtrn1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai int8x8_t vtrn1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vtrn1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float32x2_t vtrn1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vtrn1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int32x2_t vtrn1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vtrn1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai int16x4_t vtrn1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtrn2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai poly8x8_t vtrn2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vtrn2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai poly16x4_t vtrn2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vtrn2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + return __ret; +} +#else +__ai poly8x16_t vtrn2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vtrn2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai poly64x2_t vtrn2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vtrn2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai poly16x8_t vtrn2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vtrn2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + return __ret; +} +#else +__ai uint8x16_t vtrn2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vtrn2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai uint32x4_t vtrn2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vtrn2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint64x2_t vtrn2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vtrn2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai uint16x8_t vtrn2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vtrn2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + return __ret; +} +#else +__ai int8x16_t vtrn2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vtrn2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float64x2_t vtrn2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vtrn2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai float32x4_t vtrn2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vtrn2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai int32x4_t vtrn2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vtrn2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int64x2_t vtrn2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vtrn2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai int16x8_t vtrn2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtrn2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai uint8x8_t vtrn2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vtrn2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint32x2_t vtrn2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vtrn2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai uint16x4_t vtrn2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtrn2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai int8x8_t vtrn2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vtrn2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float32x2_t vtrn2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vtrn2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int32x2_t vtrn2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vtrn2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai int16x4_t vtrn2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vtst_p64(poly64x1_t __p0, poly64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vtst_p64(poly64x1_t __p0, poly64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vtstq_p64(poly64x2_t __p0, poly64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vtstq_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vtstq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vtstq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vtstq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vtstq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vtst_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vtst_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vtst_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vtst_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vtstd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vtstd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vtstd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vtstd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vtstd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vtstd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vtstd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vtstd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vuqaddb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vuqaddb_s8(__p0, __p1); + return __ret; +} +#else +__ai int8_t vuqaddb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vuqaddb_s8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vuqadds_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vuqadds_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vuqadds_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vuqadds_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vuqaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vuqaddd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vuqaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vuqaddd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vuqaddh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vuqaddh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vuqaddh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vuqaddh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vuqaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vuqaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vuqaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vuqaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vuqaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vuqaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vuqaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vuqaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vuqadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vuqadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vuqadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vuqadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vuqadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vuqadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vuqadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vuqadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vuzp1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai poly8x8_t vuzp1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vuzp1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai poly16x4_t vuzp1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vuzp1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + return __ret; +} +#else +__ai poly8x16_t vuzp1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vuzp1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai poly64x2_t vuzp1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vuzp1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai poly16x8_t vuzp1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vuzp1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + return __ret; +} +#else +__ai uint8x16_t vuzp1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vuzp1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai uint32x4_t vuzp1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vuzp1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint64x2_t vuzp1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vuzp1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai uint16x8_t vuzp1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vuzp1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + return __ret; +} +#else +__ai int8x16_t vuzp1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vuzp1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float64x2_t vuzp1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vuzp1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai float32x4_t vuzp1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vuzp1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai int32x4_t vuzp1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vuzp1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int64x2_t vuzp1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vuzp1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai int16x8_t vuzp1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vuzp1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai uint8x8_t vuzp1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vuzp1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint32x2_t vuzp1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vuzp1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai uint16x4_t vuzp1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vuzp1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai int8x8_t vuzp1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vuzp1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float32x2_t vuzp1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vuzp1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int32x2_t vuzp1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vuzp1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai int16x4_t vuzp1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vuzp2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai poly8x8_t vuzp2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vuzp2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai poly16x4_t vuzp2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vuzp2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + return __ret; +} +#else +__ai poly8x16_t vuzp2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vuzp2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai poly64x2_t vuzp2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vuzp2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai poly16x8_t vuzp2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vuzp2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + return __ret; +} +#else +__ai uint8x16_t vuzp2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vuzp2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai uint32x4_t vuzp2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vuzp2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint64x2_t vuzp2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vuzp2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai uint16x8_t vuzp2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vuzp2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + return __ret; +} +#else +__ai int8x16_t vuzp2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vuzp2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float64x2_t vuzp2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vuzp2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai float32x4_t vuzp2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vuzp2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai int32x4_t vuzp2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vuzp2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int64x2_t vuzp2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vuzp2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai int16x8_t vuzp2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vuzp2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai uint8x8_t vuzp2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vuzp2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint32x2_t vuzp2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vuzp2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai uint16x4_t vuzp2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vuzp2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai int8x8_t vuzp2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vuzp2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float32x2_t vuzp2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vuzp2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int32x2_t vuzp2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vuzp2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai int16x4_t vuzp2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vzip1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai poly8x8_t vzip1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vzip1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai poly16x4_t vzip1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vzip1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + return __ret; +} +#else +__ai poly8x16_t vzip1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vzip1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai poly64x2_t vzip1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vzip1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai poly16x8_t vzip1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vzip1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + return __ret; +} +#else +__ai uint8x16_t vzip1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vzip1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai uint32x4_t vzip1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vzip1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint64x2_t vzip1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vzip1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai uint16x8_t vzip1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vzip1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + return __ret; +} +#else +__ai int8x16_t vzip1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vzip1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float64x2_t vzip1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vzip1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai float32x4_t vzip1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vzip1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai int32x4_t vzip1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vzip1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int64x2_t vzip1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vzip1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai int16x8_t vzip1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vzip1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai uint8x8_t vzip1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vzip1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint32x2_t vzip1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vzip1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai uint16x4_t vzip1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vzip1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai int8x8_t vzip1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vzip1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float32x2_t vzip1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vzip1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int32x2_t vzip1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vzip1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai int16x4_t vzip1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vzip2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai poly8x8_t vzip2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vzip2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai poly16x4_t vzip2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vzip2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + return __ret; +} +#else +__ai poly8x16_t vzip2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vzip2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai poly64x2_t vzip2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vzip2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai poly16x8_t vzip2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vzip2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + return __ret; +} +#else +__ai uint8x16_t vzip2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vzip2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai uint32x4_t vzip2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vzip2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint64x2_t vzip2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vzip2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai uint16x8_t vzip2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vzip2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + return __ret; +} +#else +__ai int8x16_t vzip2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vzip2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float64x2_t vzip2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vzip2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai float32x4_t vzip2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vzip2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai int32x4_t vzip2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vzip2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int64x2_t vzip2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vzip2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai int16x8_t vzip2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vzip2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai uint8x8_t vzip2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vzip2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint32x2_t vzip2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vzip2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai uint16x4_t vzip2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vzip2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai int8x8_t vzip2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vzip2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float32x2_t vzip2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vzip2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int32x2_t vzip2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vzip2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai int16x4_t vzip2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#endif +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vabaq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = __p0 + vabdq_u8(__p1, __p2); + return __ret; +} +#else +__ai uint8x16_t vabaq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 + __noswap_vabdq_u8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabaq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + vabdq_u32(__p1, __p2); + return __ret; +} +#else +__ai uint32x4_t vabaq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vabdq_u32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabaq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + vabdq_u16(__p1, __p2); + return __ret; +} +#else +__ai uint16x8_t vabaq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __noswap_vabdq_u16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vabaq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = __p0 + vabdq_s8(__p1, __p2); + return __ret; +} +#else +__ai int8x16_t vabaq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 + __noswap_vabdq_s8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabaq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + vabdq_s32(__p1, __p2); + return __ret; +} +#else +__ai int32x4_t vabaq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vabdq_s32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabaq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + vabdq_s16(__p1, __p2); + return __ret; +} +#else +__ai int16x8_t vabaq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __noswap_vabdq_s16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vaba_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = __p0 + vabd_u8(__p1, __p2); + return __ret; +} +#else +__ai uint8x8_t vaba_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 + __noswap_vabd_u8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vaba_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __ret; + __ret = __p0 + vabd_u32(__p1, __p2); + return __ret; +} +#else +__ai uint32x2_t vaba_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x2_t __ret; + __ret = __rev0 + __noswap_vabd_u32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vaba_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __ret; + __ret = __p0 + vabd_u16(__p1, __p2); + return __ret; +} +#else +__ai uint16x4_t vaba_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 + __noswap_vabd_u16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vaba_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = __p0 + vabd_s8(__p1, __p2); + return __ret; +} +#else +__ai int8x8_t vaba_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 + __noswap_vabd_s8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vaba_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = __p0 + vabd_s32(__p1, __p2); + return __ret; +} +#else +__ai int32x2_t vaba_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = __rev0 + __noswap_vabd_s32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vaba_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = __p0 + vabd_s16(__p1, __p2); + return __ret; +} +#else +__ai int16x4_t vaba_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 + __noswap_vabd_s16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabdl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(vmovl_u8((uint8x8_t)(vabd_u8(__p0, __p1)))); + return __ret; +} +#else +__ai uint16x8_t vabdl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__noswap_vmovl_u8((uint8x8_t)(__noswap_vabd_u8(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vabdl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__noswap_vmovl_u8((uint8x8_t)(__noswap_vabd_u8(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vabdl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(vmovl_u32((uint32x2_t)(vabd_u32(__p0, __p1)))); + return __ret; +} +#else +__ai uint64x2_t vabdl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__noswap_vmovl_u32((uint32x2_t)(__noswap_vabd_u32(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vabdl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__noswap_vmovl_u32((uint32x2_t)(__noswap_vabd_u32(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabdl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(vmovl_u16((uint16x4_t)(vabd_u16(__p0, __p1)))); + return __ret; +} +#else +__ai uint32x4_t vabdl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__noswap_vmovl_u16((uint16x4_t)(__noswap_vabd_u16(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vabdl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__noswap_vmovl_u16((uint16x4_t)(__noswap_vabd_u16(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabdl_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t)(vmovl_u8((uint8x8_t)(vabd_s8(__p0, __p1)))); + return __ret; +} +#else +__ai int16x8_t vabdl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t)(__noswap_vmovl_u8((uint8x8_t)(__noswap_vabd_s8(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vabdl_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t)(__noswap_vmovl_u8((uint8x8_t)(__noswap_vabd_s8(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vabdl_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t)(vmovl_u32((uint32x2_t)(vabd_s32(__p0, __p1)))); + return __ret; +} +#else +__ai int64x2_t vabdl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t)(__noswap_vmovl_u32((uint32x2_t)(__noswap_vabd_s32(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vabdl_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t)(__noswap_vmovl_u32((uint32x2_t)(__noswap_vabd_s32(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabdl_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t)(vmovl_u16((uint16x4_t)(vabd_s16(__p0, __p1)))); + return __ret; +} +#else +__ai int32x4_t vabdl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t)(__noswap_vmovl_u16((uint16x4_t)(__noswap_vabd_s16(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vabdl_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t)(__noswap_vmovl_u16((uint16x4_t)(__noswap_vabd_s16(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = vmovl_u8(__p0) + vmovl_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vaddl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmovl_u8(__rev0) + __noswap_vmovl_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vaddl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = vmovl_u32(__p0) + vmovl_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vaddl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmovl_u32(__rev0) + __noswap_vmovl_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = vmovl_u16(__p0) + vmovl_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vaddl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmovl_u16(__rev0) + __noswap_vmovl_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddl_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = vmovl_s8(__p0) + vmovl_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vaddl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmovl_s8(__rev0) + __noswap_vmovl_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vaddl_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = vmovl_s32(__p0) + vmovl_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vaddl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmovl_s32(__rev0) + __noswap_vmovl_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddl_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = vmovl_s16(__p0) + vmovl_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vaddl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmovl_s16(__rev0) + __noswap_vmovl_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddw_u8(uint16x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 + vmovl_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vaddw_u8(uint16x8_t __p0, uint8x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __noswap_vmovl_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vaddw_u32(uint64x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 + vmovl_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vaddw_u32(uint64x2_t __p0, uint32x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __noswap_vmovl_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddw_u16(uint32x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 + vmovl_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vaddw_u16(uint32x4_t __p0, uint16x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vmovl_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddw_s8(int16x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = __p0 + vmovl_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vaddw_s8(int16x8_t __p0, int8x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __noswap_vmovl_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vaddw_s32(int64x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = __p0 + vmovl_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vaddw_s32(int64x2_t __p0, int32x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __noswap_vmovl_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddw_s16(int32x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = __p0 + vmovl_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vaddw_s16(int32x4_t __p0, int16x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vmovl_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + vmull_u8(__p1, __p2); + return __ret; +} +#else +__ai uint16x8_t vmlal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __noswap_vmull_u8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vmlal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + __noswap_vmull_u8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 + vmull_u32(__p1, __p2); + return __ret; +} +#else +__ai uint64x2_t vmlal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __noswap_vmull_u32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmlal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 + __noswap_vmull_u32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + vmull_u16(__p1, __p2); + return __ret; +} +#else +__ai uint32x4_t vmlal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vmull_u16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmlal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + __noswap_vmull_u16(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + vmull_s8(__p1, __p2); + return __ret; +} +#else +__ai int16x8_t vmlal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __noswap_vmull_s8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vmlal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + __noswap_vmull_s8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 + vmull_s32(__p1, __p2); + return __ret; +} +#else +__ai int64x2_t vmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __noswap_vmull_s32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 + __noswap_vmull_s32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + vmull_s16(__p1, __p2); + return __ret; +} +#else +__ai int32x4_t vmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vmull_s16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + __noswap_vmull_s16(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 + vmull_u32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_u32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + vmull_u16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_u16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 + vmull_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + vmull_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlal_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = __p0 + vmull_u32(__p1, (uint32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai uint64x2_t vmlal_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __noswap_vmull_u32(__rev1, (uint32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmlal_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = __p0 + __noswap_vmull_u32(__p1, (uint32x2_t) {__p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlal_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = __p0 + vmull_u16(__p1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai uint32x4_t vmlal_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vmull_u16(__rev1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmlal_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = __p0 + __noswap_vmull_u16(__p1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = __p0 + vmull_s32(__p1, (int32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai int64x2_t vmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __noswap_vmull_s32(__rev1, (int32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = __p0 + __noswap_vmull_s32(__p1, (int32x2_t) {__p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = __p0 + vmull_s16(__p1, (int16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai int32x4_t vmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vmull_s16(__rev1, (int16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = __p0 + __noswap_vmull_s16(__p1, (int16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlsl_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 - vmull_u8(__p1, __p2); + return __ret; +} +#else +__ai uint16x8_t vmlsl_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __noswap_vmull_u8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vmlsl_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 - __noswap_vmull_u8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlsl_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 - vmull_u32(__p1, __p2); + return __ret; +} +#else +__ai uint64x2_t vmlsl_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint64x2_t __ret; + __ret = __rev0 - __noswap_vmull_u32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmlsl_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 - __noswap_vmull_u32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsl_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 - vmull_u16(__p1, __p2); + return __ret; +} +#else +__ai uint32x4_t vmlsl_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __noswap_vmull_u16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmlsl_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 - __noswap_vmull_u16(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlsl_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 - vmull_s8(__p1, __p2); + return __ret; +} +#else +__ai int16x8_t vmlsl_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __noswap_vmull_s8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vmlsl_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 - __noswap_vmull_s8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 - vmull_s32(__p1, __p2); + return __ret; +} +#else +__ai int64x2_t vmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = __rev0 - __noswap_vmull_s32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 - __noswap_vmull_s32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 - vmull_s16(__p1, __p2); + return __ret; +} +#else +__ai int32x4_t vmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __noswap_vmull_s16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 - __noswap_vmull_s16(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 - vmull_u32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_u32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - vmull_u16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_u16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 - vmull_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - vmull_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlsl_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = __p0 - vmull_u32(__p1, (uint32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai uint64x2_t vmlsl_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 - __noswap_vmull_u32(__rev1, (uint32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmlsl_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = __p0 - __noswap_vmull_u32(__p1, (uint32x2_t) {__p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsl_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = __p0 - vmull_u16(__p1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai uint32x4_t vmlsl_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __noswap_vmull_u16(__rev1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmlsl_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = __p0 - __noswap_vmull_u16(__p1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = __p0 - vmull_s32(__p1, (int32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai int64x2_t vmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 - __noswap_vmull_s32(__rev1, (int32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = __p0 - __noswap_vmull_s32(__p1, (int32x2_t) {__p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = __p0 - vmull_s16(__p1, (int16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai int32x4_t vmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __noswap_vmull_s16(__rev1, (int16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = __p0 - __noswap_vmull_s16(__p1, (int16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#endif + +#if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) +#ifdef __LITTLE_ENDIAN__ +#define vfmsh_lane_f16(__p0_258, __p1_258, __p2_258, __p3_258) __extension__ ({ \ + float16_t __s0_258 = __p0_258; \ + float16_t __s1_258 = __p1_258; \ + float16x4_t __s2_258 = __p2_258; \ + float16_t __ret_258; \ + __ret_258 = vfmsh_f16(__s0_258, __s1_258, vget_lane_f16(__s2_258, __p3_258)); \ + __ret_258; \ +}) +#else +#define vfmsh_lane_f16(__p0_259, __p1_259, __p2_259, __p3_259) __extension__ ({ \ + float16_t __s0_259 = __p0_259; \ + float16_t __s1_259 = __p1_259; \ + float16x4_t __s2_259 = __p2_259; \ + float16x4_t __rev2_259; __rev2_259 = __builtin_shufflevector(__s2_259, __s2_259, 3, 2, 1, 0); \ + float16_t __ret_259; \ + __ret_259 = __noswap_vfmsh_f16(__s0_259, __s1_259, __noswap_vget_lane_f16(__rev2_259, __p3_259)); \ + __ret_259; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsh_laneq_f16(__p0_260, __p1_260, __p2_260, __p3_260) __extension__ ({ \ + float16_t __s0_260 = __p0_260; \ + float16_t __s1_260 = __p1_260; \ + float16x8_t __s2_260 = __p2_260; \ + float16_t __ret_260; \ + __ret_260 = vfmsh_f16(__s0_260, __s1_260, vgetq_lane_f16(__s2_260, __p3_260)); \ + __ret_260; \ +}) +#else +#define vfmsh_laneq_f16(__p0_261, __p1_261, __p2_261, __p3_261) __extension__ ({ \ + float16_t __s0_261 = __p0_261; \ + float16_t __s1_261 = __p1_261; \ + float16x8_t __s2_261 = __p2_261; \ + float16x8_t __rev2_261; __rev2_261 = __builtin_shufflevector(__s2_261, __s2_261, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret_261; \ + __ret_261 = __noswap_vfmsh_f16(__s0_261, __s1_261, __noswap_vgetq_lane_f16(__rev2_261, __p3_261)); \ + __ret_261; \ +}) +#endif + +#endif +#if defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqrdmlahs_s32(int32_t __p0, int32_t __p1, int32_t __p2) { + int32_t __ret; + __ret = vqadds_s32(__p0, vqrdmulhs_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32_t vqrdmlahs_s32(int32_t __p0, int32_t __p1, int32_t __p2) { + int32_t __ret; + __ret = __noswap_vqadds_s32(__p0, __noswap_vqrdmulhs_s32(__p1, __p2)); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqrdmlahh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { + int16_t __ret; + __ret = vqaddh_s16(__p0, vqrdmulhh_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16_t vqrdmlahh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { + int16_t __ret; + __ret = __noswap_vqaddh_s16(__p0, __noswap_vqrdmulhh_s16(__p1, __p2)); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahs_lane_s32(__p0_262, __p1_262, __p2_262, __p3_262) __extension__ ({ \ + int32_t __s0_262 = __p0_262; \ + int32_t __s1_262 = __p1_262; \ + int32x2_t __s2_262 = __p2_262; \ + int32_t __ret_262; \ + __ret_262 = vqadds_s32(__s0_262, vqrdmulhs_s32(__s1_262, vget_lane_s32(__s2_262, __p3_262))); \ + __ret_262; \ +}) +#else +#define vqrdmlahs_lane_s32(__p0_263, __p1_263, __p2_263, __p3_263) __extension__ ({ \ + int32_t __s0_263 = __p0_263; \ + int32_t __s1_263 = __p1_263; \ + int32x2_t __s2_263 = __p2_263; \ + int32x2_t __rev2_263; __rev2_263 = __builtin_shufflevector(__s2_263, __s2_263, 1, 0); \ + int32_t __ret_263; \ + __ret_263 = __noswap_vqadds_s32(__s0_263, __noswap_vqrdmulhs_s32(__s1_263, __noswap_vget_lane_s32(__rev2_263, __p3_263))); \ + __ret_263; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahh_lane_s16(__p0_264, __p1_264, __p2_264, __p3_264) __extension__ ({ \ + int16_t __s0_264 = __p0_264; \ + int16_t __s1_264 = __p1_264; \ + int16x4_t __s2_264 = __p2_264; \ + int16_t __ret_264; \ + __ret_264 = vqaddh_s16(__s0_264, vqrdmulhh_s16(__s1_264, vget_lane_s16(__s2_264, __p3_264))); \ + __ret_264; \ +}) +#else +#define vqrdmlahh_lane_s16(__p0_265, __p1_265, __p2_265, __p3_265) __extension__ ({ \ + int16_t __s0_265 = __p0_265; \ + int16_t __s1_265 = __p1_265; \ + int16x4_t __s2_265 = __p2_265; \ + int16x4_t __rev2_265; __rev2_265 = __builtin_shufflevector(__s2_265, __s2_265, 3, 2, 1, 0); \ + int16_t __ret_265; \ + __ret_265 = __noswap_vqaddh_s16(__s0_265, __noswap_vqrdmulhh_s16(__s1_265, __noswap_vget_lane_s16(__rev2_265, __p3_265))); \ + __ret_265; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahs_laneq_s32(__p0_266, __p1_266, __p2_266, __p3_266) __extension__ ({ \ + int32_t __s0_266 = __p0_266; \ + int32_t __s1_266 = __p1_266; \ + int32x4_t __s2_266 = __p2_266; \ + int32_t __ret_266; \ + __ret_266 = vqadds_s32(__s0_266, vqrdmulhs_s32(__s1_266, vgetq_lane_s32(__s2_266, __p3_266))); \ + __ret_266; \ +}) +#else +#define vqrdmlahs_laneq_s32(__p0_267, __p1_267, __p2_267, __p3_267) __extension__ ({ \ + int32_t __s0_267 = __p0_267; \ + int32_t __s1_267 = __p1_267; \ + int32x4_t __s2_267 = __p2_267; \ + int32x4_t __rev2_267; __rev2_267 = __builtin_shufflevector(__s2_267, __s2_267, 3, 2, 1, 0); \ + int32_t __ret_267; \ + __ret_267 = __noswap_vqadds_s32(__s0_267, __noswap_vqrdmulhs_s32(__s1_267, __noswap_vgetq_lane_s32(__rev2_267, __p3_267))); \ + __ret_267; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahh_laneq_s16(__p0_268, __p1_268, __p2_268, __p3_268) __extension__ ({ \ + int16_t __s0_268 = __p0_268; \ + int16_t __s1_268 = __p1_268; \ + int16x8_t __s2_268 = __p2_268; \ + int16_t __ret_268; \ + __ret_268 = vqaddh_s16(__s0_268, vqrdmulhh_s16(__s1_268, vgetq_lane_s16(__s2_268, __p3_268))); \ + __ret_268; \ +}) +#else +#define vqrdmlahh_laneq_s16(__p0_269, __p1_269, __p2_269, __p3_269) __extension__ ({ \ + int16_t __s0_269 = __p0_269; \ + int16_t __s1_269 = __p1_269; \ + int16x8_t __s2_269 = __p2_269; \ + int16x8_t __rev2_269; __rev2_269 = __builtin_shufflevector(__s2_269, __s2_269, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret_269; \ + __ret_269 = __noswap_vqaddh_s16(__s0_269, __noswap_vqrdmulhh_s16(__s1_269, __noswap_vgetq_lane_s16(__rev2_269, __p3_269))); \ + __ret_269; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqrdmlshs_s32(int32_t __p0, int32_t __p1, int32_t __p2) { + int32_t __ret; + __ret = vqsubs_s32(__p0, vqrdmulhs_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32_t vqrdmlshs_s32(int32_t __p0, int32_t __p1, int32_t __p2) { + int32_t __ret; + __ret = __noswap_vqsubs_s32(__p0, __noswap_vqrdmulhs_s32(__p1, __p2)); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqrdmlshh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { + int16_t __ret; + __ret = vqsubh_s16(__p0, vqrdmulhh_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16_t vqrdmlshh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { + int16_t __ret; + __ret = __noswap_vqsubh_s16(__p0, __noswap_vqrdmulhh_s16(__p1, __p2)); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshs_lane_s32(__p0_270, __p1_270, __p2_270, __p3_270) __extension__ ({ \ + int32_t __s0_270 = __p0_270; \ + int32_t __s1_270 = __p1_270; \ + int32x2_t __s2_270 = __p2_270; \ + int32_t __ret_270; \ + __ret_270 = vqsubs_s32(__s0_270, vqrdmulhs_s32(__s1_270, vget_lane_s32(__s2_270, __p3_270))); \ + __ret_270; \ +}) +#else +#define vqrdmlshs_lane_s32(__p0_271, __p1_271, __p2_271, __p3_271) __extension__ ({ \ + int32_t __s0_271 = __p0_271; \ + int32_t __s1_271 = __p1_271; \ + int32x2_t __s2_271 = __p2_271; \ + int32x2_t __rev2_271; __rev2_271 = __builtin_shufflevector(__s2_271, __s2_271, 1, 0); \ + int32_t __ret_271; \ + __ret_271 = __noswap_vqsubs_s32(__s0_271, __noswap_vqrdmulhs_s32(__s1_271, __noswap_vget_lane_s32(__rev2_271, __p3_271))); \ + __ret_271; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshh_lane_s16(__p0_272, __p1_272, __p2_272, __p3_272) __extension__ ({ \ + int16_t __s0_272 = __p0_272; \ + int16_t __s1_272 = __p1_272; \ + int16x4_t __s2_272 = __p2_272; \ + int16_t __ret_272; \ + __ret_272 = vqsubh_s16(__s0_272, vqrdmulhh_s16(__s1_272, vget_lane_s16(__s2_272, __p3_272))); \ + __ret_272; \ +}) +#else +#define vqrdmlshh_lane_s16(__p0_273, __p1_273, __p2_273, __p3_273) __extension__ ({ \ + int16_t __s0_273 = __p0_273; \ + int16_t __s1_273 = __p1_273; \ + int16x4_t __s2_273 = __p2_273; \ + int16x4_t __rev2_273; __rev2_273 = __builtin_shufflevector(__s2_273, __s2_273, 3, 2, 1, 0); \ + int16_t __ret_273; \ + __ret_273 = __noswap_vqsubh_s16(__s0_273, __noswap_vqrdmulhh_s16(__s1_273, __noswap_vget_lane_s16(__rev2_273, __p3_273))); \ + __ret_273; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshs_laneq_s32(__p0_274, __p1_274, __p2_274, __p3_274) __extension__ ({ \ + int32_t __s0_274 = __p0_274; \ + int32_t __s1_274 = __p1_274; \ + int32x4_t __s2_274 = __p2_274; \ + int32_t __ret_274; \ + __ret_274 = vqsubs_s32(__s0_274, vqrdmulhs_s32(__s1_274, vgetq_lane_s32(__s2_274, __p3_274))); \ + __ret_274; \ +}) +#else +#define vqrdmlshs_laneq_s32(__p0_275, __p1_275, __p2_275, __p3_275) __extension__ ({ \ + int32_t __s0_275 = __p0_275; \ + int32_t __s1_275 = __p1_275; \ + int32x4_t __s2_275 = __p2_275; \ + int32x4_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, 3, 2, 1, 0); \ + int32_t __ret_275; \ + __ret_275 = __noswap_vqsubs_s32(__s0_275, __noswap_vqrdmulhs_s32(__s1_275, __noswap_vgetq_lane_s32(__rev2_275, __p3_275))); \ + __ret_275; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshh_laneq_s16(__p0_276, __p1_276, __p2_276, __p3_276) __extension__ ({ \ + int16_t __s0_276 = __p0_276; \ + int16_t __s1_276 = __p1_276; \ + int16x8_t __s2_276 = __p2_276; \ + int16_t __ret_276; \ + __ret_276 = vqsubh_s16(__s0_276, vqrdmulhh_s16(__s1_276, vgetq_lane_s16(__s2_276, __p3_276))); \ + __ret_276; \ +}) +#else +#define vqrdmlshh_laneq_s16(__p0_277, __p1_277, __p2_277, __p3_277) __extension__ ({ \ + int16_t __s0_277 = __p0_277; \ + int16_t __s1_277 = __p1_277; \ + int16x8_t __s2_277 = __p2_277; \ + int16x8_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret_277; \ + __ret_277 = __noswap_vqsubh_s16(__s0_277, __noswap_vqrdmulhh_s16(__s1_277, __noswap_vgetq_lane_s16(__rev2_277, __p3_277))); \ + __ret_277; \ +}) +#endif + +#endif +#if defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabdl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = vabdl_u8(vget_high_u8(__p0), vget_high_u8(__p1)); + return __ret; +} +#else +__ai uint16x8_t vabdl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vabdl_u8(__noswap_vget_high_u8(__rev0), __noswap_vget_high_u8(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vabdl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = vabdl_u32(vget_high_u32(__p0), vget_high_u32(__p1)); + return __ret; +} +#else +__ai uint64x2_t vabdl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vabdl_u32(__noswap_vget_high_u32(__rev0), __noswap_vget_high_u32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabdl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = vabdl_u16(vget_high_u16(__p0), vget_high_u16(__p1)); + return __ret; +} +#else +__ai uint32x4_t vabdl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vabdl_u16(__noswap_vget_high_u16(__rev0), __noswap_vget_high_u16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabdl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = vabdl_s8(vget_high_s8(__p0), vget_high_s8(__p1)); + return __ret; +} +#else +__ai int16x8_t vabdl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vabdl_s8(__noswap_vget_high_s8(__rev0), __noswap_vget_high_s8(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vabdl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = vabdl_s32(vget_high_s32(__p0), vget_high_s32(__p1)); + return __ret; +} +#else +__ai int64x2_t vabdl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vabdl_s32(__noswap_vget_high_s32(__rev0), __noswap_vget_high_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabdl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = vabdl_s16(vget_high_s16(__p0), vget_high_s16(__p1)); + return __ret; +} +#else +__ai int32x4_t vabdl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vabdl_s16(__noswap_vget_high_s16(__rev0), __noswap_vget_high_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = vmovl_high_u8(__p0) + vmovl_high_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vaddl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmovl_high_u8(__rev0) + __noswap_vmovl_high_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vaddl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = vmovl_high_u32(__p0) + vmovl_high_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vaddl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmovl_high_u32(__rev0) + __noswap_vmovl_high_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = vmovl_high_u16(__p0) + vmovl_high_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vaddl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmovl_high_u16(__rev0) + __noswap_vmovl_high_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = vmovl_high_s8(__p0) + vmovl_high_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vaddl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmovl_high_s8(__rev0) + __noswap_vmovl_high_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vaddl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = vmovl_high_s32(__p0) + vmovl_high_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vaddl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmovl_high_s32(__rev0) + __noswap_vmovl_high_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = vmovl_high_s16(__p0) + vmovl_high_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vaddl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmovl_high_s16(__rev0) + __noswap_vmovl_high_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddw_high_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = __p0 + vmovl_high_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vaddw_high_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __noswap_vmovl_high_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vaddw_high_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = __p0 + vmovl_high_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vaddw_high_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __noswap_vmovl_high_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddw_high_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = __p0 + vmovl_high_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vaddw_high_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vmovl_high_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddw_high_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = __p0 + vmovl_high_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vaddw_high_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __noswap_vmovl_high_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vaddw_high_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = __p0 + vmovl_high_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vaddw_high_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __noswap_vmovl_high_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddw_high_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = __p0 + vmovl_high_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vaddw_high_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vmovl_high_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_p64(__p0_278, __p1_278, __p2_278, __p3_278) __extension__ ({ \ + poly64x2_t __s0_278 = __p0_278; \ + poly64x1_t __s2_278 = __p2_278; \ + poly64x2_t __ret_278; \ + __ret_278 = vsetq_lane_p64(vget_lane_p64(__s2_278, __p3_278), __s0_278, __p1_278); \ + __ret_278; \ +}) +#else +#define vcopyq_lane_p64(__p0_279, __p1_279, __p2_279, __p3_279) __extension__ ({ \ + poly64x2_t __s0_279 = __p0_279; \ + poly64x1_t __s2_279 = __p2_279; \ + poly64x2_t __rev0_279; __rev0_279 = __builtin_shufflevector(__s0_279, __s0_279, 1, 0); \ + poly64x2_t __ret_279; \ + __ret_279 = __noswap_vsetq_lane_p64(__noswap_vget_lane_p64(__s2_279, __p3_279), __rev0_279, __p1_279); \ + __ret_279 = __builtin_shufflevector(__ret_279, __ret_279, 1, 0); \ + __ret_279; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_f64(__p0_280, __p1_280, __p2_280, __p3_280) __extension__ ({ \ + float64x2_t __s0_280 = __p0_280; \ + float64x1_t __s2_280 = __p2_280; \ + float64x2_t __ret_280; \ + __ret_280 = vsetq_lane_f64(vget_lane_f64(__s2_280, __p3_280), __s0_280, __p1_280); \ + __ret_280; \ +}) +#else +#define vcopyq_lane_f64(__p0_281, __p1_281, __p2_281, __p3_281) __extension__ ({ \ + float64x2_t __s0_281 = __p0_281; \ + float64x1_t __s2_281 = __p2_281; \ + float64x2_t __rev0_281; __rev0_281 = __builtin_shufflevector(__s0_281, __s0_281, 1, 0); \ + float64x2_t __ret_281; \ + __ret_281 = __noswap_vsetq_lane_f64(__noswap_vget_lane_f64(__s2_281, __p3_281), __rev0_281, __p1_281); \ + __ret_281 = __builtin_shufflevector(__ret_281, __ret_281, 1, 0); \ + __ret_281; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_p64(__p0_282, __p1_282, __p2_282, __p3_282) __extension__ ({ \ + poly64x1_t __s0_282 = __p0_282; \ + poly64x1_t __s2_282 = __p2_282; \ + poly64x1_t __ret_282; \ + __ret_282 = vset_lane_p64(vget_lane_p64(__s2_282, __p3_282), __s0_282, __p1_282); \ + __ret_282; \ +}) +#else +#define vcopy_lane_p64(__p0_283, __p1_283, __p2_283, __p3_283) __extension__ ({ \ + poly64x1_t __s0_283 = __p0_283; \ + poly64x1_t __s2_283 = __p2_283; \ + poly64x1_t __ret_283; \ + __ret_283 = __noswap_vset_lane_p64(__noswap_vget_lane_p64(__s2_283, __p3_283), __s0_283, __p1_283); \ + __ret_283; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_f64(__p0_284, __p1_284, __p2_284, __p3_284) __extension__ ({ \ + float64x1_t __s0_284 = __p0_284; \ + float64x1_t __s2_284 = __p2_284; \ + float64x1_t __ret_284; \ + __ret_284 = vset_lane_f64(vget_lane_f64(__s2_284, __p3_284), __s0_284, __p1_284); \ + __ret_284; \ +}) +#else +#define vcopy_lane_f64(__p0_285, __p1_285, __p2_285, __p3_285) __extension__ ({ \ + float64x1_t __s0_285 = __p0_285; \ + float64x1_t __s2_285 = __p2_285; \ + float64x1_t __ret_285; \ + __ret_285 = __noswap_vset_lane_f64(__noswap_vget_lane_f64(__s2_285, __p3_285), __s0_285, __p1_285); \ + __ret_285; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_p64(__p0_286, __p1_286, __p2_286, __p3_286) __extension__ ({ \ + poly64x2_t __s0_286 = __p0_286; \ + poly64x2_t __s2_286 = __p2_286; \ + poly64x2_t __ret_286; \ + __ret_286 = vsetq_lane_p64(vgetq_lane_p64(__s2_286, __p3_286), __s0_286, __p1_286); \ + __ret_286; \ +}) +#else +#define vcopyq_laneq_p64(__p0_287, __p1_287, __p2_287, __p3_287) __extension__ ({ \ + poly64x2_t __s0_287 = __p0_287; \ + poly64x2_t __s2_287 = __p2_287; \ + poly64x2_t __rev0_287; __rev0_287 = __builtin_shufflevector(__s0_287, __s0_287, 1, 0); \ + poly64x2_t __rev2_287; __rev2_287 = __builtin_shufflevector(__s2_287, __s2_287, 1, 0); \ + poly64x2_t __ret_287; \ + __ret_287 = __noswap_vsetq_lane_p64(__noswap_vgetq_lane_p64(__rev2_287, __p3_287), __rev0_287, __p1_287); \ + __ret_287 = __builtin_shufflevector(__ret_287, __ret_287, 1, 0); \ + __ret_287; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_f64(__p0_288, __p1_288, __p2_288, __p3_288) __extension__ ({ \ + float64x2_t __s0_288 = __p0_288; \ + float64x2_t __s2_288 = __p2_288; \ + float64x2_t __ret_288; \ + __ret_288 = vsetq_lane_f64(vgetq_lane_f64(__s2_288, __p3_288), __s0_288, __p1_288); \ + __ret_288; \ +}) +#else +#define vcopyq_laneq_f64(__p0_289, __p1_289, __p2_289, __p3_289) __extension__ ({ \ + float64x2_t __s0_289 = __p0_289; \ + float64x2_t __s2_289 = __p2_289; \ + float64x2_t __rev0_289; __rev0_289 = __builtin_shufflevector(__s0_289, __s0_289, 1, 0); \ + float64x2_t __rev2_289; __rev2_289 = __builtin_shufflevector(__s2_289, __s2_289, 1, 0); \ + float64x2_t __ret_289; \ + __ret_289 = __noswap_vsetq_lane_f64(__noswap_vgetq_lane_f64(__rev2_289, __p3_289), __rev0_289, __p1_289); \ + __ret_289 = __builtin_shufflevector(__ret_289, __ret_289, 1, 0); \ + __ret_289; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_p64(__p0_290, __p1_290, __p2_290, __p3_290) __extension__ ({ \ + poly64x1_t __s0_290 = __p0_290; \ + poly64x2_t __s2_290 = __p2_290; \ + poly64x1_t __ret_290; \ + __ret_290 = vset_lane_p64(vgetq_lane_p64(__s2_290, __p3_290), __s0_290, __p1_290); \ + __ret_290; \ +}) +#else +#define vcopy_laneq_p64(__p0_291, __p1_291, __p2_291, __p3_291) __extension__ ({ \ + poly64x1_t __s0_291 = __p0_291; \ + poly64x2_t __s2_291 = __p2_291; \ + poly64x2_t __rev2_291; __rev2_291 = __builtin_shufflevector(__s2_291, __s2_291, 1, 0); \ + poly64x1_t __ret_291; \ + __ret_291 = __noswap_vset_lane_p64(__noswap_vgetq_lane_p64(__rev2_291, __p3_291), __s0_291, __p1_291); \ + __ret_291; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_f64(__p0_292, __p1_292, __p2_292, __p3_292) __extension__ ({ \ + float64x1_t __s0_292 = __p0_292; \ + float64x2_t __s2_292 = __p2_292; \ + float64x1_t __ret_292; \ + __ret_292 = vset_lane_f64(vgetq_lane_f64(__s2_292, __p3_292), __s0_292, __p1_292); \ + __ret_292; \ +}) +#else +#define vcopy_laneq_f64(__p0_293, __p1_293, __p2_293, __p3_293) __extension__ ({ \ + float64x1_t __s0_293 = __p0_293; \ + float64x2_t __s2_293 = __p2_293; \ + float64x2_t __rev2_293; __rev2_293 = __builtin_shufflevector(__s2_293, __s2_293, 1, 0); \ + float64x1_t __ret_293; \ + __ret_293 = __noswap_vset_lane_f64(__noswap_vgetq_lane_f64(__rev2_293, __p3_293), __s0_293, __p1_293); \ + __ret_293; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlal_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __ret; + __ret = vmlal_u8(__p0, vget_high_u8(__p1), vget_high_u8(__p2)); + return __ret; +} +#else +__ai uint16x8_t vmlal_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmlal_u8(__rev0, __noswap_vget_high_u8(__rev1), __noswap_vget_high_u8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlal_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __ret; + __ret = vmlal_u32(__p0, vget_high_u32(__p1), vget_high_u32(__p2)); + return __ret; +} +#else +__ai uint64x2_t vmlal_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmlal_u32(__rev0, __noswap_vget_high_u32(__rev1), __noswap_vget_high_u32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlal_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __ret; + __ret = vmlal_u16(__p0, vget_high_u16(__p1), vget_high_u16(__p2)); + return __ret; +} +#else +__ai uint32x4_t vmlal_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmlal_u16(__rev0, __noswap_vget_high_u16(__rev1), __noswap_vget_high_u16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlal_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __ret; + __ret = vmlal_s8(__p0, vget_high_s8(__p1), vget_high_s8(__p2)); + return __ret; +} +#else +__ai int16x8_t vmlal_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmlal_s8(__rev0, __noswap_vget_high_s8(__rev1), __noswap_vget_high_s8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __ret; + __ret = vmlal_s32(__p0, vget_high_s32(__p1), vget_high_s32(__p2)); + return __ret; +} +#else +__ai int64x2_t vmlal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmlal_s32(__rev0, __noswap_vget_high_s32(__rev1), __noswap_vget_high_s32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __ret; + __ret = vmlal_s16(__p0, vget_high_s16(__p1), vget_high_s16(__p2)); + return __ret; +} +#else +__ai int32x4_t vmlal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmlal_s16(__rev0, __noswap_vget_high_s16(__rev1), __noswap_vget_high_s16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlal_high_n_u32(uint64x2_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = vmlal_n_u32(__p0, vget_high_u32(__p1), __p2); + return __ret; +} +#else +__ai uint64x2_t vmlal_high_n_u32(uint64x2_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmlal_n_u32(__rev0, __noswap_vget_high_u32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlal_high_n_u16(uint32x4_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = vmlal_n_u16(__p0, vget_high_u16(__p1), __p2); + return __ret; +} +#else +__ai uint32x4_t vmlal_high_n_u16(uint32x4_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmlal_n_u16(__rev0, __noswap_vget_high_u16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlal_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = vmlal_n_s32(__p0, vget_high_s32(__p1), __p2); + return __ret; +} +#else +__ai int64x2_t vmlal_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmlal_n_s32(__rev0, __noswap_vget_high_s32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlal_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = vmlal_n_s16(__p0, vget_high_s16(__p1), __p2); + return __ret; +} +#else +__ai int32x4_t vmlal_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmlal_n_s16(__rev0, __noswap_vget_high_s16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlsl_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __ret; + __ret = vmlsl_u8(__p0, vget_high_u8(__p1), vget_high_u8(__p2)); + return __ret; +} +#else +__ai uint16x8_t vmlsl_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmlsl_u8(__rev0, __noswap_vget_high_u8(__rev1), __noswap_vget_high_u8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlsl_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __ret; + __ret = vmlsl_u32(__p0, vget_high_u32(__p1), vget_high_u32(__p2)); + return __ret; +} +#else +__ai uint64x2_t vmlsl_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmlsl_u32(__rev0, __noswap_vget_high_u32(__rev1), __noswap_vget_high_u32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsl_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __ret; + __ret = vmlsl_u16(__p0, vget_high_u16(__p1), vget_high_u16(__p2)); + return __ret; +} +#else +__ai uint32x4_t vmlsl_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmlsl_u16(__rev0, __noswap_vget_high_u16(__rev1), __noswap_vget_high_u16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlsl_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __ret; + __ret = vmlsl_s8(__p0, vget_high_s8(__p1), vget_high_s8(__p2)); + return __ret; +} +#else +__ai int16x8_t vmlsl_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmlsl_s8(__rev0, __noswap_vget_high_s8(__rev1), __noswap_vget_high_s8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlsl_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __ret; + __ret = vmlsl_s32(__p0, vget_high_s32(__p1), vget_high_s32(__p2)); + return __ret; +} +#else +__ai int64x2_t vmlsl_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmlsl_s32(__rev0, __noswap_vget_high_s32(__rev1), __noswap_vget_high_s32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsl_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __ret; + __ret = vmlsl_s16(__p0, vget_high_s16(__p1), vget_high_s16(__p2)); + return __ret; +} +#else +__ai int32x4_t vmlsl_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmlsl_s16(__rev0, __noswap_vget_high_s16(__rev1), __noswap_vget_high_s16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlsl_high_n_u32(uint64x2_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = vmlsl_n_u32(__p0, vget_high_u32(__p1), __p2); + return __ret; +} +#else +__ai uint64x2_t vmlsl_high_n_u32(uint64x2_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmlsl_n_u32(__rev0, __noswap_vget_high_u32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsl_high_n_u16(uint32x4_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = vmlsl_n_u16(__p0, vget_high_u16(__p1), __p2); + return __ret; +} +#else +__ai uint32x4_t vmlsl_high_n_u16(uint32x4_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmlsl_n_u16(__rev0, __noswap_vget_high_u16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlsl_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = vmlsl_n_s32(__p0, vget_high_s32(__p1), __p2); + return __ret; +} +#else +__ai int64x2_t vmlsl_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmlsl_n_s32(__rev0, __noswap_vget_high_s32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsl_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = vmlsl_n_s16(__p0, vget_high_s16(__p1), __p2); + return __ret; +} +#else +__ai int32x4_t vmlsl_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmlsl_n_s16(__rev0, __noswap_vget_high_s16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_lane_f64(__p0_294, __p1_294, __p2_294) __extension__ ({ \ + float64x1_t __s0_294 = __p0_294; \ + float64x1_t __s1_294 = __p1_294; \ + float64x1_t __ret_294; \ + float64_t __x_294 = vget_lane_f64(__s0_294, 0); \ + float64_t __y_294 = vget_lane_f64(__s1_294, __p2_294); \ + float64_t __z_294 = vmulxd_f64(__x_294, __y_294); \ + __ret_294 = vset_lane_f64(__z_294, __s0_294, __p2_294); \ + __ret_294; \ +}) +#else +#define vmulx_lane_f64(__p0_295, __p1_295, __p2_295) __extension__ ({ \ + float64x1_t __s0_295 = __p0_295; \ + float64x1_t __s1_295 = __p1_295; \ + float64x1_t __ret_295; \ + float64_t __x_295 = __noswap_vget_lane_f64(__s0_295, 0); \ + float64_t __y_295 = __noswap_vget_lane_f64(__s1_295, __p2_295); \ + float64_t __z_295 = __noswap_vmulxd_f64(__x_295, __y_295); \ + __ret_295 = __noswap_vset_lane_f64(__z_295, __s0_295, __p2_295); \ + __ret_295; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_laneq_f64(__p0_296, __p1_296, __p2_296) __extension__ ({ \ + float64x1_t __s0_296 = __p0_296; \ + float64x2_t __s1_296 = __p1_296; \ + float64x1_t __ret_296; \ + float64_t __x_296 = vget_lane_f64(__s0_296, 0); \ + float64_t __y_296 = vgetq_lane_f64(__s1_296, __p2_296); \ + float64_t __z_296 = vmulxd_f64(__x_296, __y_296); \ + __ret_296 = vset_lane_f64(__z_296, __s0_296, 0); \ + __ret_296; \ +}) +#else +#define vmulx_laneq_f64(__p0_297, __p1_297, __p2_297) __extension__ ({ \ + float64x1_t __s0_297 = __p0_297; \ + float64x2_t __s1_297 = __p1_297; \ + float64x2_t __rev1_297; __rev1_297 = __builtin_shufflevector(__s1_297, __s1_297, 1, 0); \ + float64x1_t __ret_297; \ + float64_t __x_297 = __noswap_vget_lane_f64(__s0_297, 0); \ + float64_t __y_297 = __noswap_vgetq_lane_f64(__rev1_297, __p2_297); \ + float64_t __z_297 = __noswap_vmulxd_f64(__x_297, __y_297); \ + __ret_297 = __noswap_vset_lane_f64(__z_297, __s0_297, 0); \ + __ret_297; \ +}) +#endif + +#endif +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + vabdl_u8(__p1, __p2); + return __ret; +} +#else +__ai uint16x8_t vabal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __noswap_vabdl_u8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vabal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + __noswap_vabdl_u8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vabal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 + vabdl_u32(__p1, __p2); + return __ret; +} +#else +__ai uint64x2_t vabal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __noswap_vabdl_u32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vabal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 + __noswap_vabdl_u32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + vabdl_u16(__p1, __p2); + return __ret; +} +#else +__ai uint32x4_t vabal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vabdl_u16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vabal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + __noswap_vabdl_u16(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + vabdl_s8(__p1, __p2); + return __ret; +} +#else +__ai int16x8_t vabal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __noswap_vabdl_s8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vabal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + __noswap_vabdl_s8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vabal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 + vabdl_s32(__p1, __p2); + return __ret; +} +#else +__ai int64x2_t vabal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __noswap_vabdl_s32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vabal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 + __noswap_vabdl_s32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + vabdl_s16(__p1, __p2); + return __ret; +} +#else +__ai int32x4_t vabal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vabdl_s16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vabal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + __noswap_vabdl_s16(__p1, __p2); + return __ret; +} +#endif + +#if defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabal_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __ret; + __ret = vabal_u8(__p0, vget_high_u8(__p1), vget_high_u8(__p2)); + return __ret; +} +#else +__ai uint16x8_t vabal_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vabal_u8(__rev0, __noswap_vget_high_u8(__rev1), __noswap_vget_high_u8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vabal_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __ret; + __ret = vabal_u32(__p0, vget_high_u32(__p1), vget_high_u32(__p2)); + return __ret; +} +#else +__ai uint64x2_t vabal_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vabal_u32(__rev0, __noswap_vget_high_u32(__rev1), __noswap_vget_high_u32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabal_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __ret; + __ret = vabal_u16(__p0, vget_high_u16(__p1), vget_high_u16(__p2)); + return __ret; +} +#else +__ai uint32x4_t vabal_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vabal_u16(__rev0, __noswap_vget_high_u16(__rev1), __noswap_vget_high_u16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabal_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __ret; + __ret = vabal_s8(__p0, vget_high_s8(__p1), vget_high_s8(__p2)); + return __ret; +} +#else +__ai int16x8_t vabal_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vabal_s8(__rev0, __noswap_vget_high_s8(__rev1), __noswap_vget_high_s8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vabal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __ret; + __ret = vabal_s32(__p0, vget_high_s32(__p1), vget_high_s32(__p2)); + return __ret; +} +#else +__ai int64x2_t vabal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vabal_s32(__rev0, __noswap_vget_high_s32(__rev1), __noswap_vget_high_s32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __ret; + __ret = vabal_s16(__p0, vget_high_s16(__p1), vget_high_s16(__p2)); + return __ret; +} +#else +__ai int32x4_t vabal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vabal_s16(__rev0, __noswap_vget_high_s16(__rev1), __noswap_vget_high_s16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#endif + +#undef __ai + +#pragma clang diagnostic pop + +#endif /* __ARM_NEON_H */ diff --git a/EXTERNAL_HEADERS/corecrypto/cc.h b/EXTERNAL_HEADERS/corecrypto/cc.h index 6b01e33c1..7790a4faa 100644 --- a/EXTERNAL_HEADERS/corecrypto/cc.h +++ b/EXTERNAL_HEADERS/corecrypto/cc.h @@ -39,26 +39,28 @@ uint8_t b[_alignment_]; \ /* sizeof of a context declared with cc_ctx_decl */ #define cc_ctx_sizeof(_type_, _size_) sizeof(_type_[cc_ctx_n(_type_, _size_)]) -//- WARNING: The _MSC_VER version of cc_ctx_decl() is not compatible with the way *_decl macros are used in CommonCrypto, AppleKeyStore and SecurityFrameworks -// to observe the incompatibilities and errors, use below definition. Corecrypto itself, accepts both deinitions -// #define cc_ctx_decl(_type_, _size_, _name_) _type_ _name_ ## _array[cc_ctx_n(_type_, (_size_))]; _type_ *_name_ = _name_ ## _array -//- Never use sizeof() operator for the variables declared with cc_ctx_decl(), because it is not be compatible with the _MSC_VER version of cc_ctx_decl(). +/* + 1. _alloca cannot be removed becasue this header file is compiled with both MSVC++ and with clang. + 2. The _MSC_VER version of cc_ctx_decl() is not compatible with the way *_decl macros as used in CommonCrypto, AppleKeyStore and SecurityFrameworks. To observe the incompatibilities and errors, use below definition. Corecrypto itself, accepts both deinitions + #define cc_ctx_decl(_type_, _size_, _name_) _type_ _name_ ## _array[cc_ctx_n(_type_, (_size_))]; _type_ *_name_ = _name_ ## _array + 3. Never use sizeof() operator for the variables declared with cc_ctx_decl(), because it is not be compatible with the _MSC_VER version of cc_ctx_decl(). + */ #if defined(_MSC_VER) - #define UNIQUE_ARRAY(data_type, _var_, total_count) data_type* _var_ = (data_type*)_alloca(sizeof(data_type)*(total_count)); - #define cc_ctx_decl(_type_, _size_, _name_) UNIQUE_ARRAY(_type_, _name_,cc_ctx_n(_type_, (_size_))) +#define cc_ctx_decl(_type_, _size_, _name_) _type_ * _name_ = (_type_ *) _alloca(sizeof(_type_) * cc_ctx_n(_type_, _size_) ) #else - #define cc_ctx_decl(_type_, _size_, _name_) _type_ _name_ [cc_ctx_n(_type_, _size_)] +#define cc_ctx_decl(_type_, _size_, _name_) _type_ _name_ [cc_ctx_n(_type_, _size_)] #endif /* bzero is deprecated. memset is the way to go */ /* FWIW, L4, HEXAGON and ARMCC even with gnu compatibility mode don't have bzero */ #define cc_zero(_size_,_data_) memset((_data_),0 ,(_size_)) -/* cc_clear: - Set "len" bytes of memory to zero at address "dst". - cc_clear has been developed so that it won't be optimized out. - To be used to clear key buffers or sensitive data. -*/ +/*! + @brief cc_clear(len, dst) zeroizes array dst and it will not be optimized out. + @discussion It is used to clear sensitive data, particularly when the are defined in the stack + @param len number of bytes to be cleared in dst + @param dst input array + */ CC_NONNULL2 void cc_clear(size_t len, void *dst); diff --git a/EXTERNAL_HEADERS/corecrypto/cc_config.h b/EXTERNAL_HEADERS/corecrypto/cc_config.h index 464f32b18..044c8e168 100644 --- a/EXTERNAL_HEADERS/corecrypto/cc_config.h +++ b/EXTERNAL_HEADERS/corecrypto/cc_config.h @@ -45,15 +45,20 @@ */ -//Do not set these macros to 1, unless you are developing/testing for Windows +//Do not set this macros to 1, unless you are developing/testing for Linux under macOS +#define CORECRYPTO_SIMULATE_POSIX_ENVIRONMENT 0 + +//Do not set these macros to 1, unless you are developing/testing for Windows under macOS #define CORECRYPTO_SIMULATE_WINDOWS_ENVIRONMENT 0 -#define CORECRYPTO_HACK_FOR_WINDOWS_DEVELOPMENT 0 //to be removed after port corecrypto to Windows +#define CORECRYPTO_HACK_FOR_WINDOWS_DEVELOPMENT 0 //to be removed after port corecrypto to Windows //this macro is used to turn on/off usage of transparent union in corecrypto //it should be commented out in corecrypto and be used only by the software that use corecrypto //#define CORECRYPTO_DONOT_USE_TRANSPARENT_UNION -#ifdef CORECRYPTO_DONOT_USE_TRANSPARENT_UNION - #define CORECRYPTO_USE_TRANSPARENT_UNION 0 +#if defined(__cplusplus) +#define CORECRYPTO_USE_TRANSPARENT_UNION 0 +#elif defined(CORECRYPTO_DONOT_USE_TRANSPARENT_UNION) + #define CORECRYPTO_USE_TRANSPARENT_UNION !CORECRYPTO_DONOT_USE_TRANSPARENT_UNION #else #define CORECRYPTO_USE_TRANSPARENT_UNION 1 #endif @@ -76,9 +81,7 @@ #define CC_KERNEL 0 #endif -// LINUX_BUILD_TEST is for sanity check of the configuration -// > xcodebuild -scheme "corecrypto_test" OTHER_CFLAGS="$(values) -DLINUX_BUILD_TEST" -#if defined(__linux__) || defined(LINUX_BUILD_TEST) +#if defined(__linux__) || CORECRYPTO_SIMULATE_POSIX_ENVIRONMENT #define CC_LINUX 1 #else #define CC_LINUX 0 @@ -90,6 +93,12 @@ #define CC_USE_L4 0 #endif +#if defined(RTKIT) && (RTKIT) + #define CC_RTKIT 1 +#else + #define CC_RTKIT 0 +#endif + #if defined(USE_SEPROM) && (USE_SEPROM) #define CC_USE_SEPROM 1 #else @@ -120,20 +129,32 @@ #define CC_IBOOT 0 #endif -// BB configuration +// Defined by the XNU build scripts +// Applies to code embedded in XNU but NOT to the kext +#if defined(XNU_KERNEL_PRIVATE) + #define CC_XNU_KERNEL_PRIVATE 1 +#else + #define CC_XNU_KERNEL_PRIVATE 0 +#endif + +// handle unaligned data, if the cpu cannot. Currently for gladman AES and the C version of the SHA256 +#define CC_HANDLE_UNALIGNED_DATA CC_BASEBAND + +// BaseBand configuration #if CC_BASEBAND // -- ENDIANESS +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) #if defined(ENDIAN_LITTLE) || (defined(__arm__) && !defined(__BIG_ENDIAN)) #define __LITTLE_ENDIAN__ #elif !defined(ENDIAN_BIG) && !defined(__BIG_ENDIAN) #error Baseband endianess not defined. #endif #define AESOPT_ENDIAN_NO_FILE +#endif // -- Architecture #define CCN_UNIT_SIZE 4 // 32 bits - #define SAFE_IO // AES support for unaligned Input/Output // -- External function #define assert ASSERT // sanity @@ -143,21 +164,27 @@ // #1254-D: arithmetic on pointer to void or function type // #186-D: pointless comparison of unsigned integer with zero // #546-D: transfer of control bypasses initialization of - #if defined(__GNUC__) + #ifdef __arm__ + #pragma diag_suppress 186, 1254,546 + #elif defined(__GNUC__) // warning: pointer of type 'void *' used in arithmetic #pragma GCC diagnostic ignored "-Wpointer-arith" - #endif // arm or gnuc - + #endif // __arm__ #endif // CC_BASEBAND //CC_XNU_KERNEL_AVAILABLE indicates the availibity of XNU kernel functions, //like what we have on OSX, iOS, tvOS, Watch OS -#if defined(__APPLE__) && defined(__MACH__) +#if defined(__APPLE__) && defined(__MACH__) #define CC_XNU_KERNEL_AVAILABLE 1 #else #define CC_XNU_KERNEL_AVAILABLE 0 #endif +//arm arch64 definition for gcc +#if defined(__GNUC__) && defined(__aarch64__) && !defined(__arm64__) + #define __arm64__ +#endif + #if !defined(CCN_UNIT_SIZE) #if defined(__arm64__) || defined(__x86_64__) || defined(_WIN64) #define CCN_UNIT_SIZE 8 @@ -192,16 +219,35 @@ #endif #endif -#if __clang__ || CCN_UNIT_SIZE==8 - #define CC_ALIGNED(x) __attribute__ ((aligned(x))) -#elif _MSC_VER - #define CC_ALIGNED(x) __declspec(align(x)) +#if defined(_MSC_VER) + #if defined(__clang__) + #define CC_ALIGNED(x) __attribute__ ((aligned(x))) //clang compiler + #else + #define CC_ALIGNED(x) __declspec(align(x)) //MS complier + #endif #else - #define CC_ALIGNED(x) __attribute__ ((aligned((x)>8?8:(x)))) + #if __clang__ || CCN_UNIT_SIZE==8 + #define CC_ALIGNED(x) __attribute__ ((aligned(x))) + #else + #define CC_ALIGNED(x) __attribute__ ((aligned((x)>8?8:(x)))) + #endif #endif +#if defined(__arm__) +//this is copied from , because is not available on SEPROM environment + #if defined (__ARM_ARCH_7A__) || defined (__ARM_ARCH_7S__) || defined (__ARM_ARCH_7F__) || defined (__ARM_ARCH_7K__) + #define _ARM_ARCH_7 + #endif + + #if defined(__ARM_ARCH_6M__) || defined(__TARGET_ARCH_6S_M) || defined (__armv6m__) + #define _ARM_ARCH_6M + #endif +#endif -#if defined(__x86_64__) || defined(__i386__) +#if defined(__arm64__) || defined(__arm__) + #define CCN_IOS 1 + #define CCN_OSX 0 +#elif defined(__x86_64__) || defined(__i386__) #define CCN_IOS 0 #define CCN_OSX 1 #endif @@ -213,17 +259,15 @@ #endif #if !defined(CC_USE_HEAP_FOR_WORKSPACE) - #if CC_USE_L4 || CC_IBOOT || defined(_MSC_VER) - /* For L4, stack is too short, need to use HEAP for some computations */ - /* CC_USE_HEAP_FOR_WORKSPACE not supported for KERNEL! */ - #define CC_USE_HEAP_FOR_WORKSPACE 1 - #else + #if CC_USE_S3 || CC_USE_SEPROM || CC_RTKIT #define CC_USE_HEAP_FOR_WORKSPACE 0 + #else + #define CC_USE_HEAP_FOR_WORKSPACE 1 #endif #endif /* memset_s is only available in few target */ -#if CC_KERNEL || CC_USE_SEPROM || defined(__CC_ARM) \ +#if CC_USE_SEPROM || defined(__CC_ARM) \ || defined(__hexagon__) || CC_EFI #define CC_HAS_MEMSET_S 0 #else @@ -237,8 +281,7 @@ #endif /* __has_include() */ #endif /* defined(__has_include) */ -// Disable FIPS key gen algorithm on userland and kext so that related POST -// is skipped and boot time is reduced +// Disable RSA Keygen on iBridge #if defined(TARGET_OS_BRIDGE) && TARGET_OS_BRIDGE && CC_KERNEL #define CC_DISABLE_RSAKEYGEN 1 /* for iBridge */ #else @@ -247,13 +290,14 @@ //- functions implemented in assembly ------------------------------------------ //this the list of corecrypto clients that use assembly and the clang compiler -#if !(CC_XNU_KERNEL_AVAILABLE || CC_KERNEL || CC_USE_L4 || CC_IBOOT || CC_USE_SEPROM || CC_USE_S3) && !defined(_WIN32) && CORECRYPTO_DEBUG +#if !(CC_XNU_KERNEL_AVAILABLE || CC_KERNEL || CC_USE_L4 || CC_IBOOT || CC_RTKIT || CC_USE_SEPROM || CC_USE_S3) && !defined(_WIN32) && CORECRYPTO_DEBUG #warning "You are using the default corecrypto configuration, assembly optimizations may not be available for your platform" #endif -// use this macro to strictly disable assembly regardless of cpu/os/compiler/etc +// Use this macro to strictly disable assembly regardless of cpu/os/compiler/etc. +// Our assembly code is not gcc compatible. Clang defines the __GNUC__ macro as well. #if !defined(CC_USE_ASM) - #if defined(_MSC_VER) || CC_LINUX || CC_EFI || CC_BASEBAND + #if defined(_WIN32) || CC_EFI || CC_BASEBAND || CC_XNU_KERNEL_PRIVATE || (defined(__GNUC__) && !defined(__clang__)) || defined(__ANDROID_API__) #define CC_USE_ASM 0 #else #define CC_USE_ASM 1 @@ -277,7 +321,7 @@ #define CCN_SHIFT_RIGHT_ASM 1 #define CCAES_ARM_ASM 1 #define CCAES_INTEL_ASM 0 - #if CC_KERNEL || CC_USE_L4 || CC_IBOOT || CC_USE_SEPROM || CC_USE_S3 + #if CC_KERNEL || CC_USE_L4 || CC_IBOOT || CC_RTKIT || CC_USE_SEPROM || CC_USE_S3 #define CCAES_MUX 0 #else #define CCAES_MUX 1 @@ -296,6 +340,31 @@ #define CCSHA256_ARMV6M_ASM 0 //-(2) ARM 64 +#elif defined(__arm64__) && __clang__ && CC_USE_ASM + #define CCN_DEDICATED_SQR 1 + #define CCN_MUL_KARATSUBA 1 // 4*n CCN_UNIT extra memory required. + #define CCN_ADD_ASM 1 + #define CCN_SUB_ASM 1 + #define CCN_MUL_ASM 1 + #define CCN_ADDMUL1_ASM 0 + #define CCN_MUL1_ASM 0 + #define CCN_CMP_ASM 1 + #define CCN_ADD1_ASM 0 + #define CCN_SUB1_ASM 0 + #define CCN_N_ASM 1 + #define CCN_SET_ASM 0 + #define CCN_SHIFT_RIGHT_ASM 1 + #define CCAES_ARM_ASM 1 + #define CCAES_INTEL_ASM 0 + #define CCAES_MUX 0 // On 64bit SoC, asm is much faster than HW + #define CCN_USE_BUILTIN_CLZ 1 + #define CCSHA1_VNG_INTEL 0 + #define CCSHA2_VNG_INTEL 0 + #define CCSHA1_VNG_ARMV7NEON 1 // reused this to avoid making change to xcode project, put arm64 assembly code with armv7 code + #define CCSHA2_VNG_ARMV7NEON 1 + #define CCSHA256_ARMV6M_ASM 0 + +//-(3) Intel 32/64 #elif (defined(__x86_64__) || defined(__i386__)) && __clang__ && CC_USE_ASM #define CCN_DEDICATED_SQR 1 #define CCN_MUL_KARATSUBA 1 // 4*n CCN_UNIT extra memory required. @@ -431,5 +500,11 @@ #define CC_MALLOC #endif /* !__GNUC__ */ +// Enable FIPSPOST function tracing only when supported. */ +#ifdef CORECRYPTO_POST_TRACE +#define CC_FIPSPOST_TRACE 1 +#else +#define CC_FIPSPOST_TRACE 0 +#endif #endif /* _CORECRYPTO_CC_CONFIG_H_ */ diff --git a/EXTERNAL_HEADERS/corecrypto/cc_debug.h b/EXTERNAL_HEADERS/corecrypto/cc_debug.h index 5c8ebbdc7..80e61a7b3 100644 --- a/EXTERNAL_HEADERS/corecrypto/cc_debug.h +++ b/EXTERNAL_HEADERS/corecrypto/cc_debug.h @@ -21,15 +21,20 @@ // Printf for corecrypto // ======================== #if CC_KERNEL -#include -#define cc_printf(x...) kprintf(x) -extern int printf(const char *format, ...) __printflike(1,2); -#elif CC_USE_S3 -#include -#define cc_printf(x...) printf(x) + #include + #define cc_printf(x...) kprintf(x) + #if !CONFIG_EMBEDDED + extern int printf(const char *format, ...) __printflike(1,2); + #endif +#elif CC_USE_S3 || CC_IBOOT || CC_RTKIT + #include + #define cc_printf(x...) printf(x) +#elif defined(__ANDROID_API__) + #include + #define cc_printf(x...) __android_log_print(ANDROID_LOG_DEBUG, "corecrypto", x); #else -#include -#define cc_printf(x...) fprintf(stderr, x) + #include + #define cc_printf(x...) fprintf(stderr, x) #endif // ======================== diff --git a/EXTERNAL_HEADERS/corecrypto/cc_priv.h b/EXTERNAL_HEADERS/corecrypto/cc_priv.h index 417d45c5c..55e0eb2b8 100644 --- a/EXTERNAL_HEADERS/corecrypto/cc_priv.h +++ b/EXTERNAL_HEADERS/corecrypto/cc_priv.h @@ -53,14 +53,9 @@ The following are not defined yet... define them if needed. CC_BSWAP64c : byte swap a 64 bits constant CC_READ_LE32 : read a 32 bits little endian value - CC_READ_LE64 : read a 64 bits little endian value - CC_READ_BE32 : read a 32 bits big endian value - CC_READ_BE64 : read a 64 bits big endian value CC_WRITE_LE32 : write a 32 bits little endian value CC_WRITE_LE64 : write a 64 bits little endian value - CC_WRITE_BE32 : write a 32 bits big endian value - CC_WRITE_BE64 : write a 64 bits big endian value CC_H2BE64 : convert a 64 bits value between host and big endian order CC_H2LE64 : convert a 64 bits value between host and little endian order @@ -360,6 +355,32 @@ CC_INLINE uint32_t CC_BSWAP(uint32_t x) #define CC_H2LE32(x) CC_BSWAP(x) #endif +#define CC_READ_LE32(ptr) \ +( (uint32_t)( \ +((uint32_t)((const uint8_t *)(ptr))[0]) | \ +(((uint32_t)((const uint8_t *)(ptr))[1]) << 8) | \ +(((uint32_t)((const uint8_t *)(ptr))[2]) << 16) | \ +(((uint32_t)((const uint8_t *)(ptr))[3]) << 24))) + +#define CC_WRITE_LE32(ptr, x) \ +do { \ +((uint8_t *)(ptr))[0] = (uint8_t)( (x) & 0xFF); \ +((uint8_t *)(ptr))[1] = (uint8_t)(((x) >> 8) & 0xFF); \ +((uint8_t *)(ptr))[2] = (uint8_t)(((x) >> 16) & 0xFF); \ +((uint8_t *)(ptr))[3] = (uint8_t)(((x) >> 24) & 0xFF); \ +} while(0) + +#define CC_WRITE_LE64(ptr, x) \ +do { \ +((uint8_t *)(ptr))[0] = (uint8_t)( (x) & 0xFF); \ +((uint8_t *)(ptr))[1] = (uint8_t)(((x) >> 8) & 0xFF); \ +((uint8_t *)(ptr))[2] = (uint8_t)(((x) >> 16) & 0xFF); \ +((uint8_t *)(ptr))[3] = (uint8_t)(((x) >> 24) & 0xFF); \ +((uint8_t *)(ptr))[4] = (uint8_t)(((x) >> 32) & 0xFF); \ +((uint8_t *)(ptr))[5] = (uint8_t)(((x) >> 40) & 0xFF); \ +((uint8_t *)(ptr))[6] = (uint8_t)(((x) >> 48) & 0xFF); \ +((uint8_t *)(ptr))[7] = (uint8_t)(((x) >> 56) & 0xFF); \ +} while(0) /* extract a byte portably */ #ifdef _MSC_VER @@ -413,8 +434,8 @@ CC_INLINE uint32_t CC_BSWAP(uint32_t x) #define cc_ceiling(a,b) (((a)+((b)-1))/(b)) #define CC_BITLEN_TO_BYTELEN(x) cc_ceiling((x), 8) -//cc_abort() is implemented to comply with FIPS 140-2. See radar 19129408 -void cc_abort(const char * msg , ...); +//cc_try_abort() is implemented to comply with FIPS 140-2. See radar 19129408 +void cc_try_abort(const char * msg , ...); /*! @brief cc_muxp(s, a, b) is equivalent to z = s ? a : b, but it executes in constant time diff --git a/EXTERNAL_HEADERS/corecrypto/cc_runtime_config.h b/EXTERNAL_HEADERS/corecrypto/cc_runtime_config.h new file mode 100644 index 000000000..0064c6ca6 --- /dev/null +++ b/EXTERNAL_HEADERS/corecrypto/cc_runtime_config.h @@ -0,0 +1,48 @@ +/* + * cc_runtime_config.h + * corecrypto + * + * Created on 09/18/2012 + * + * Copyright (c) 2012,2014,2015 Apple Inc. All rights reserved. + * + */ + +#ifndef CORECRYPTO_CC_RUNTIME_CONFIG_H_ +#define CORECRYPTO_CC_RUNTIME_CONFIG_H_ + +#include + +/* Only intel systems have these runtime switches today. */ +#if (CCSHA1_VNG_INTEL || CCSHA2_VNG_INTEL || CCAES_INTEL_ASM) \ + && (defined(__x86_64__) || defined(__i386__)) + +#if CC_KERNEL + #include + #define CC_HAS_AESNI() ((cpuid_features() & CPUID_FEATURE_AES) != 0) + #define CC_HAS_SupplementalSSE3() ((cpuid_features() & CPUID_FEATURE_SSSE3) != 0) + #define CC_HAS_AVX1() ((cpuid_features() & CPUID_FEATURE_AVX1_0) != 0) + #define CC_HAS_AVX2() ((cpuid_info()->cpuid_leaf7_features & CPUID_LEAF7_FEATURE_AVX2) != 0) + +#elif CC_XNU_KERNEL_AVAILABLE + # include + + #ifndef kHasAVX2_0 /* 10.8 doesn't have kHasAVX2_0 defined */ + #define kHasAVX2_0 0 + #endif + + extern int _cpu_capabilities; + #define CC_HAS_AESNI() (_cpu_capabilities & kHasAES) + #define CC_HAS_SupplementalSSE3() (_cpu_capabilities & kHasSupplementalSSE3) + #define CC_HAS_AVX1() (_cpu_capabilities & kHasAVX1_0) + #define CC_HAS_AVX2() (_cpu_capabilities & kHasAVX2_0) +#else + #define CC_HAS_AESNI() 0 + #define CC_HAS_SupplementalSSE3() 0 + #define CC_HAS_AVX1() 0 + #define CC_HAS_AVX2() 0 +#endif + +#endif /* !(defined(__x86_64__) || defined(__i386__)) */ + +#endif /* CORECRYPTO_CC_RUNTIME_CONFIG_H_ */ diff --git a/EXTERNAL_HEADERS/corecrypto/ccaes.h b/EXTERNAL_HEADERS/corecrypto/ccaes.h index 630cdd282..ec119b9b6 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccaes.h +++ b/EXTERNAL_HEADERS/corecrypto/ccaes.h @@ -19,6 +19,8 @@ #define CCAES_KEY_SIZE_192 24 #define CCAES_KEY_SIZE_256 32 +#define CCAES_CTR_MAX_PARALLEL_NBLOCKS 8 + extern const struct ccmode_ecb ccaes_ltc_ecb_decrypt_mode; extern const struct ccmode_ecb ccaes_ltc_ecb_encrypt_mode; @@ -46,8 +48,13 @@ extern const struct ccmode_ofb ccaes_arm_ofb_crypt_mode; extern const struct ccmode_cbc ccaes_ios_hardware_cbc_encrypt_mode; extern const struct ccmode_cbc ccaes_ios_hardware_cbc_decrypt_mode; +extern const struct ccmode_ctr ccaes_ios_hardware_ctr_crypt_mode; + extern const struct ccmode_cbc *ccaes_ios_mux_cbc_encrypt_mode(void); extern const struct ccmode_cbc *ccaes_ios_mux_cbc_decrypt_mode(void); + +extern const struct ccmode_ctr *ccaes_ios_mux_ctr_crypt_mode(void); + #endif #if CCAES_INTEL_ASM diff --git a/EXTERNAL_HEADERS/corecrypto/ccasn1.h b/EXTERNAL_HEADERS/corecrypto/ccasn1.h index 7eb1182e6..28fba4eef 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccasn1.h +++ b/EXTERNAL_HEADERS/corecrypto/ccasn1.h @@ -88,7 +88,7 @@ size_t ccoid_size(ccoid_t oid) { return 2 + CCOID(oid)[1]; } -CC_INLINE CC_PURE CC_NONNULL((1)) CC_NONNULL((2)) +CC_INLINE CC_PURE CC_NONNULL_TU((1)) CC_NONNULL_TU((2)) bool ccoid_equal(ccoid_t oid1, ccoid_t oid2) { return (ccoid_size(oid1) == ccoid_size(oid2) && memcmp(CCOID(oid1), CCOID(oid2), ccoid_size(oid1))== 0); diff --git a/EXTERNAL_HEADERS/corecrypto/ccchacha20poly1305.h b/EXTERNAL_HEADERS/corecrypto/ccchacha20poly1305.h new file mode 100644 index 000000000..3e76b81b4 --- /dev/null +++ b/EXTERNAL_HEADERS/corecrypto/ccchacha20poly1305.h @@ -0,0 +1,295 @@ +/* + ccchacha20poly1305.h + corecrypto + + Copyright 2014 Apple Inc. All rights reserved. +*/ + +#ifndef _CORECRYPTO_CCCHACHA20POLY1305_H_ +#define _CORECRYPTO_CCCHACHA20POLY1305_H_ + +#include +#include +#include + +#define CCCHACHA20_KEY_NBYTES 32 +#define CCCHACHA20_BLOCK_NBYTES 64 +#define CCCHACHA20_BLOCK_NBITS (CCCHACHA20_BLOCK_NBYTES * 8) +#define CCCHACHA20_NONCE_NBYTES 12 + +typedef struct { + uint32_t state[16]; + uint8_t buffer[CCCHACHA20_BLOCK_NBYTES]; + size_t leftover; +} ccchacha20_ctx; + +#define CCPOLY1305_TAG_NBYTES 16 + +typedef struct { + uint32_t r0, r1, r2, r3, r4; + uint32_t s1, s2, s3, s4; + uint32_t h0, h1, h2, h3, h4; + uint8_t buf[16]; + size_t buf_used; + uint8_t key[16]; +} ccpoly1305_ctx; + + +/*! + @group ccchacha20poly1305 + @abstract Encrypts and authenticates or decrypts and verifies data. + @discussion See RFC 7539 for details. + + @warning The key-nonce pair must be unique per encryption. + + @warning A single message can be at most (2^38 - 64) bytes in length. + + The correct sequence of calls to encrypt is: + + @code ccchacha20poly1305_init(...) + ccchacha20poly1305_setnonce(...) + ccchacha20poly1305_aad(...) (may be called zero or more times) + ccchacha20poly1305_encrypt(...) (may be called zero or more times) + ccchacha20poly1305_finalize(...) + + To reuse the context for additional encryptions, follow this sequence: + + @code ccchacha20poly1305_reset(...) + ccchacha20poly1305_setnonce(...) + ccchacha20poly1305_aad(...) (may be called zero or more times) + ccchacha20poly1305_encrypt(...) (may be called zero or more times) + ccchacha20poly1305_finalize(...) + + To decrypt, follow this call sequence: + + @code ccchacha20poly1305_init(...) + ccchacha20poly1305_setnonce(...) + ccchacha20poly1305_aad(...) (may be called zero or more times) + ccchacha20poly1305_decrypt(...) (may be called zero or more times) + ccchacha20poly1305_verify(...) (returns zero on successful decryption) + + To reuse the context for additional encryptions, follow this sequence: + + @code ccchacha20poly1305_reset(...) + ccchacha20poly1305_setnonce(...) + ccchacha20poly1305_aad(...) (may be called zero or more times) + ccchacha20poly1305_decrypt(...) (may be called zero or more times) + ccchacha20poly1305_verify(...) (returns zero on successful decryption) +*/ + +#define CCCHACHA20POLY1305_KEY_NBYTES (CCCHACHA20_KEY_NBYTES) +#define CCCHACHA20POLY1305_NONCE_NBYTES (CCCHACHA20_NONCE_NBYTES) +#define CCCHACHA20POLY1305_TAG_NBYTES (CCPOLY1305_TAG_NBYTES) + +/* (2^32 - 1) blocks */ +/* (2^38 - 64) bytes */ +/* (2^41 - 512) bits */ +/* Exceeding this figure breaks confidentiality and authenticity. */ +#define CCCHACHA20POLY1305_TEXT_MAX_NBYTES ((1ULL << 38) - 64ULL) + +#define CCCHACHA20POLY1305_STATE_SETNONCE 1 +#define CCCHACHA20POLY1305_STATE_AAD 2 +#define CCCHACHA20POLY1305_STATE_ENCRYPT 3 +#define CCCHACHA20POLY1305_STATE_DECRYPT 4 +#define CCCHACHA20POLY1305_STATE_FINAL 5 + +typedef struct { + ccchacha20_ctx chacha20_ctx; + ccpoly1305_ctx poly1305_ctx; + uint64_t aad_nbytes; + uint64_t text_nbytes; + uint8_t state; +} ccchacha20poly1305_ctx; + +// This is just a stub right now. +// Eventually we will optimize by platform. +struct ccchacha20poly1305_info { + +}; + +extern const struct ccchacha20poly1305_info ccchacha20poly1305_info_default; + +const struct ccchacha20poly1305_info *ccchacha20poly1305_info(void); + +/*! + @function ccchacha20poly1305_init + @abstract Initialize a chacha20poly1305 context. + + @param info Implementation descriptor + @param ctx Context for this instance + @param key Secret chacha20 key + + @result 0 iff successful. + + @discussion The key is 32 bytes in length. + + @warning The key-nonce pair must be unique per encryption. + */ +int ccchacha20poly1305_init(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *key); + +/*! + @function ccchacha20poly1305_reset + @abstract Reset a chacha20poly1305 context for reuse. + + @param info Implementation descriptor + @param ctx Context for this instance + + @result 0 iff successful. + */ +int ccchacha20poly1305_reset(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx); + +/*! + @function ccchacha20poly1305_setnonce + @abstract Set the nonce for encryption or decryption. + + @param info Implementation descriptor + @param ctx Context for this instance + @param nonce Unique nonce per encryption + + @result 0 iff successful. + + @discussion The nonce is 12 bytes in length. + + @warning The key-nonce pair must be unique per encryption. + */ +int ccchacha20poly1305_setnonce(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *nonce); +int ccchacha20poly1305_incnonce(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, uint8_t *nonce); + +/*! + @function ccchacha20poly1305_aad + @abstract Authenticate additional data. + + @param info Descriptor for the mode + @param ctx Context for this instance + @param nbytes Length of the additional data in bytes + @param aad Additional data to authenticate + + @result 0 iff successful. + + @discussion This is typically used to authenticate data that cannot be encrypted (e.g. packet headers). + + This function may be called zero or more times. + */ +int ccchacha20poly1305_aad(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *aad); + +/*! + @function ccchacha20poly1305_encrypt + @abstract Encrypt data. + + @param info Descriptor for the mode + @param ctx Context for this instance + @param nbytes Length of the plaintext in bytes + @param ptext Input plaintext + @param ctext Output ciphertext + + @result 0 iff successful. + + @discussion In-place processing is supported. + + This function may be called zero or more times. + */ +int ccchacha20poly1305_encrypt(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *ptext, void *ctext); + +/*! + @function ccchacha20poly1305_finalize + @abstract Finalize encryption. + + @param info Descriptor for the mode + @param ctx Context for this instance + @param tag Generated authentication tag + + @result 0 iff successful. + + @discussion The generated tag is 16 bytes in length. + */ +int ccchacha20poly1305_finalize(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, uint8_t *tag); + +/*! + @function ccchacha20poly1305_decrypt + @abstract Decrypt data. + + @param info Descriptor for the mode + @param ctx Context for this instance + @param nbytes Length of the ciphertext in bytes + @param ctext Input ciphertext + @param ptext Output plaintext + + @result 0 iff successful. + + @discussion In-place processing is supported. + + This function may be called zero or more times. + */ +int ccchacha20poly1305_decrypt(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *ctext, void *ptext); + +/*! + @function ccchacha20poly1305_verify + @abstract Verify authenticity. + + @param info Descriptor for the mode + @param ctx Context for this instance + @param tag Expected authentication tag + + @result 0 iff authentic and otherwise successful. + + @discussion The expected tag is 16 bytes in length. + */ +int ccchacha20poly1305_verify(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *tag); + +/*! + @function ccchacha20poly1305_encrypt_oneshot + @abstract Encrypt with chacha20poly1305. + + @param info Descriptor for the mode + @param key Secret chacha20 key + @param nonce Unique nonce per encryption + @param aad_nbytes Length of the additional data in bytes + @param aad Additional data to authenticate + @param ptext_nbytes Length of the plaintext in bytes + @param ptext Input plaintext + @param ctext Output ciphertext + @param tag Generated authentication tag + + @discussion See RFC 7539 for details. + + The key is 32 bytes in length. + + The nonce is 12 bytes in length. + + The generated tag is 16 bytes in length. + + In-place processing is supported. + + @warning The key-nonce pair must be unique per encryption. + + @warning A single message can be at most (2^38 - 64) bytes in length. + */ +int ccchacha20poly1305_encrypt_oneshot(const struct ccchacha20poly1305_info *info, const uint8_t *key, const uint8_t *nonce, size_t aad_nbytes, const void *aad, size_t ptext_nbytes, const void *ptext, void *ctext, uint8_t *tag); + +/*! + @function ccchacha20poly1305_decrypt_oneshot + @abstract Decrypt with chacha20poly1305. + + @param info Descriptor for the mode + @param key Secret chacha20 key + @param nonce Unique nonce per encryption + @param aad_nbytes Length of the additional data in bytes + @param aad Additional data to authenticate + @param ctext_nbytes Length of the ciphertext in bytes + @param ctext Input ciphertext + @param ptext Output plaintext + @param tag Expected authentication tag + + @discussion See RFC 7539 for details. + + The key is 32 bytes in length. + + The nonce is 12 bytes in length. + + The generated tag is 16 bytes in length. + + In-place processing is supported. + */ +int ccchacha20poly1305_decrypt_oneshot(const struct ccchacha20poly1305_info *info, const uint8_t *key, const uint8_t *nonce, size_t aad_nbytes, const void *aad, size_t ctext_nbytes, const void *ctext, void *ptext, const uint8_t *tag); + +#endif diff --git a/EXTERNAL_HEADERS/corecrypto/cccmac.h b/EXTERNAL_HEADERS/corecrypto/cccmac.h index 75918fe84..63a892fd6 100644 --- a/EXTERNAL_HEADERS/corecrypto/cccmac.h +++ b/EXTERNAL_HEADERS/corecrypto/cccmac.h @@ -89,16 +89,6 @@ typedef struct cccmac_ctx* cccmac_ctx_t; /* CMAC as defined in NIST SP800-38B - 2005 */ -/* HACK: - To change the prototype of cccmac_init (and preserve the name) we need to - proceed in steps: - 1) Make corecrypto change (23557380) - 2) Have all clients define "CC_CHANGEFUNCTION_28544056_cccmac_init" - 3) Remove CC_CHANGEFUNCTION_28544056_cccmac_init logic and old functions of corecrypto - 4) Clients can remove CC_CHANGEFUNCTION_28544056_cccmac_init at their leisure - - */ - /* ============================================================================= ONE SHOT @@ -169,25 +159,9 @@ int cccmac_one_shot_verify(const struct ccmode_cbc *cbc, @discussion Only supports CMAC_BLOCKSIZE block ciphers */ - - -#ifndef CC_CHANGEFUNCTION_28544056_cccmac_init -int cccmac_init(const struct ccmode_cbc *cbc, - cccmac_ctx_t ctx, - size_t key_nbytes, const void *key) -__attribute__((deprecated("see guidelines in corecrypto/cccmac.h for migration", "define 'CC_CHANGEFUNCTION_28544056_cccmac_init' and use new cccmac_init with parameter key_nbytes"))); -// If you see this deprecate warning -// Define CC_CHANGEFUNCTION_28544056_cccmac_init and use "cccmac_init(...,...,16,...)" -// This will be removed with 28544056 -#define cccmac_init(cbc,ctx,key) cccmac_init(cbc,ctx,16,key) - -#else - -// This is the authoritative prototype, which will be left after 28544056 int cccmac_init(const struct ccmode_cbc *cbc, cccmac_ctx_t ctx, size_t key_nbytes, const void *key); -#endif /*! @function cccmac_update @@ -235,70 +209,4 @@ int cccmac_final_generate(cccmac_ctx_t ctx, int cccmac_final_verify(cccmac_ctx_t ctx, size_t expected_mac_nbytes, const void *expected_mac); - -/* ============================================================================= - - Legacy - Please migrate to new functions above - - ==============================================================================*/ - -/* - Guidelines for switching to new CMAC functions - - Legacy New functions - cccmac_init -> cccmac_init w/ key kength in bytes - cccmac_block_update -> cccmac_update w/ size in bytes instead of blocks - cccmac_final -> cccmac_final_generate or cccmac_final_verify - depending the use case preceeded - by cccmac_update if any leftover bytes. - cccmac -> cccmac_one_shot_generate or cccmac_one_shot_verify - depending the use case - - */ - -/*! - @function cccmac_block_update - @abstract Process data - */ - -CC_INLINE void cccmac_block_update(CC_UNUSED const struct ccmode_cbc *cbc, cccmac_ctx_t ctx, - size_t nblocks, const void *data) -__attribute__((deprecated("see guidelines in corecrypto/cccmac.h for migration", "cccmac_update"))); - -CC_INLINE void cccmac_block_update(CC_UNUSED const struct ccmode_cbc *cbc, cccmac_ctx_t ctx, - size_t nblocks, const void *data) { - cccmac_update(ctx,(nblocks)*CMAC_BLOCKSIZE,data); -} - -/*! - @function cccmac_final - @abstract Finalize CMAC generation - */ -CC_INLINE void cccmac_final(CC_UNUSED const struct ccmode_cbc *cbc, cccmac_ctx_t ctx, - size_t nbytes, const void *in, void *out) -__attribute__((deprecated("see guidelines in corecrypto/cccmac.h for migration", "cccmac_final_generate or cccmac_final_verify"))); - -CC_INLINE void cccmac_final(CC_UNUSED const struct ccmode_cbc *cbc, cccmac_ctx_t ctx, - size_t nbytes, const void *in, void *out) { - cccmac_update(ctx, nbytes, in); - cccmac_final_generate(ctx,CMAC_BLOCKSIZE,out); -} - -/*! - @function cccmac - @abstract One shot CMAC generation with 128bit key - */ -CC_INLINE void cccmac(const struct ccmode_cbc *cbc, - const void *key, - size_t data_len, const void *data, void *mac) -__attribute__((deprecated("see guidelines in corecrypto/cccmac.h for migration", "cccmac_one_shot_generate or cccmac_one_shot_verify"))); - -CC_INLINE void cccmac(const struct ccmode_cbc *cbc, - const void *key, - size_t data_len, const void *data, void *mac) { - cccmac_one_shot_generate(cbc,16,key,data_len,data,16,mac); -} - - - #endif /* _CORECRYPTO_cccmac_H_ */ diff --git a/EXTERNAL_HEADERS/corecrypto/ccder.h b/EXTERNAL_HEADERS/corecrypto/ccder.h index f29140edf..6e2c504be 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccder.h +++ b/EXTERNAL_HEADERS/corecrypto/ccder.h @@ -17,82 +17,76 @@ #define CCDER_MULTIBYTE_TAGS 1 #ifdef CCDER_MULTIBYTE_TAGS - #if defined(_MSC_VER) - //TODO related to rdar://problem/24868013 - typedef int ccder_tag; //MSVC forces enums to be ints - #else - typedef unsigned long ccder_tag; - #endif +typedef unsigned long ccder_tag; #else typedef uint8_t ccder_tag; #endif /* DER types to be used with ccder_decode and ccder_encode functions. */ -enum { - CCDER_EOL = CCASN1_EOL, - CCDER_BOOLEAN = CCASN1_BOOLEAN, - CCDER_INTEGER = CCASN1_INTEGER, - CCDER_BIT_STRING = CCASN1_BIT_STRING, - CCDER_OCTET_STRING = CCASN1_OCTET_STRING, - CCDER_NULL = CCASN1_NULL, - CCDER_OBJECT_IDENTIFIER = CCASN1_OBJECT_IDENTIFIER, - CCDER_OBJECT_DESCRIPTOR = CCASN1_OBJECT_DESCRIPTOR, +#define CCDER_EOL CCASN1_EOL +#define CCDER_BOOLEAN CCASN1_BOOLEAN +#define CCDER_INTEGER CCASN1_INTEGER +#define CCDER_BIT_STRING CCASN1_BIT_STRING +#define CCDER_OCTET_STRING CCASN1_OCTET_STRING +#define CCDER_NULL CCASN1_NULL +#define CCDER_OBJECT_IDENTIFIER CCASN1_OBJECT_IDENTIFIER +#define CCDER_OBJECT_DESCRIPTOR CCASN1_OBJECT_DESCRIPTOR /* External or instance-of 0x08 */ - CCDER_REAL = CCASN1_REAL, - CCDER_ENUMERATED = CCASN1_ENUMERATED, - CCDER_EMBEDDED_PDV = CCASN1_EMBEDDED_PDV, - CCDER_UTF8_STRING = CCASN1_UTF8_STRING, +#define CCDER_REAL CCASN1_REAL +#define CCDER_ENUMERATED CCASN1_ENUMERATED +#define CCDER_EMBEDDED_PDV CCASN1_EMBEDDED_PDV +#define CCDER_UTF8_STRING CCASN1_UTF8_STRING /* 0x0d */ /* 0x0e */ /* 0x0f */ - CCDER_SEQUENCE = CCASN1_SEQUENCE, - CCDER_SET = CCASN1_SET, - CCDER_NUMERIC_STRING = CCASN1_NUMERIC_STRING, - CCDER_PRINTABLE_STRING = CCASN1_PRINTABLE_STRING, - CCDER_T61_STRING = CCASN1_T61_STRING, - CCDER_VIDEOTEX_STRING = CCASN1_VIDEOTEX_STRING, - CCDER_IA5_STRING = CCASN1_IA5_STRING, - CCDER_UTC_TIME = CCASN1_UTC_TIME, - CCDER_GENERALIZED_TIME = CCASN1_GENERALIZED_TIME, - CCDER_GRAPHIC_STRING = CCASN1_GRAPHIC_STRING, - CCDER_VISIBLE_STRING = CCASN1_VISIBLE_STRING, - CCDER_GENERAL_STRING = CCASN1_GENERAL_STRING, - CCDER_UNIVERSAL_STRING = CCASN1_UNIVERSAL_STRING, +#define CCDER_SEQUENCE CCASN1_SEQUENCE +#define CCDER_SET CCASN1_SET +#define CCDER_NUMERIC_STRING CCASN1_NUMERIC_STRING +#define CCDER_PRINTABLE_STRING CCASN1_PRINTABLE_STRING +#define CCDER_T61_STRING CCASN1_T61_STRING +#define CCDER_VIDEOTEX_STRING CCASN1_VIDEOTEX_STRING +#define CCDER_IA5_STRING CCASN1_IA5_STRING +#define CCDER_UTC_TIME CCASN1_UTC_TIME +#define CCDER_GENERALIZED_TIME CCASN1_GENERALIZED_TIME +#define CCDER_GRAPHIC_STRING CCASN1_GRAPHIC_STRING +#define CCDER_VISIBLE_STRING CCASN1_VISIBLE_STRING +#define CCDER_GENERAL_STRING CCASN1_GENERAL_STRING +#define CCDER_UNIVERSAL_STRING CCASN1_UNIVERSAL_STRING /* 0x1d */ - CCDER_BMP_STRING = CCASN1_BMP_STRING, - CCDER_HIGH_TAG_NUMBER = CCASN1_HIGH_TAG_NUMBER, - CCDER_TELETEX_STRING = CCDER_T61_STRING, +#define CCDER_BMP_STRING CCASN1_BMP_STRING +#define CCDER_HIGH_TAG_NUMBER CCASN1_HIGH_TAG_NUMBER +#define CCDER_TELETEX_STRING CCDER_T61_STRING #ifdef CCDER_MULTIBYTE_TAGS - CCDER_TAG_MASK = ((ccder_tag)~0), - CCDER_TAGNUM_MASK = ((ccder_tag)~((ccder_tag)7 << (sizeof(ccder_tag) * 8 - 3))), - - CCDER_METHOD_MASK = ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 3)), - CCDER_PRIMITIVE = ((ccder_tag)0 << (sizeof(ccder_tag) * 8 - 3)), - CCDER_CONSTRUCTED = ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 3)), - - CCDER_CLASS_MASK = ((ccder_tag)3 << (sizeof(ccder_tag) * 8 - 2)), - CCDER_UNIVERSAL = ((ccder_tag)0 << (sizeof(ccder_tag) * 8 - 2)), - CCDER_APPLICATION = ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 2)), - CCDER_CONTEXT_SPECIFIC = ((ccder_tag)2 << (sizeof(ccder_tag) * 8 - 2)), - CCDER_PRIVATE = ((ccder_tag)3 << (sizeof(ccder_tag) * 8 - 2)), -#else - CCDER_TAG_MASK = CCASN1_TAG_MASK, - CCDER_TAGNUM_MASK = CCASN1_TAGNUM_MASK, - - CCDER_METHOD_MASK = CCASN1_METHOD_MASK, - CCDER_PRIMITIVE = CCASN1_PRIMITIVE, - CCDER_CONSTRUCTED = CCASN1_CONSTRUCTED, - - CCDER_CLASS_MASK = CCASN1_CLASS_MASK, - CCDER_UNIVERSAL = CCASN1_UNIVERSAL, - CCDER_APPLICATION = CCASN1_APPLICATION, - CCDER_CONTEXT_SPECIFIC = CCASN1_CONTEXT_SPECIFIC, - CCDER_PRIVATE = CCASN1_PRIVATE, -#endif - CCDER_CONSTRUCTED_SET = CCDER_SET | CCDER_CONSTRUCTED, - CCDER_CONSTRUCTED_SEQUENCE = CCDER_SEQUENCE | CCDER_CONSTRUCTED, -}; +#define CCDER_TAG_MASK ((ccder_tag)~0) +#define CCDER_TAGNUM_MASK ((ccder_tag)~((ccder_tag)7 << (sizeof(ccder_tag) * 8 - 3))) + +#define CCDER_METHOD_MASK ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 3)) +#define CCDER_PRIMITIVE ((ccder_tag)0 << (sizeof(ccder_tag) * 8 - 3)) +#define CCDER_CONSTRUCTED ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 3)) + +#define CCDER_CLASS_MASK ((ccder_tag)3 << (sizeof(ccder_tag) * 8 - 2)) +#define CCDER_UNIVERSAL ((ccder_tag)0 << (sizeof(ccder_tag) * 8 - 2)) +#define CCDER_APPLICATION ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 2)) +#define CCDER_CONTEXT_SPECIFIC ((ccder_tag)2 << (sizeof(ccder_tag) * 8 - 2)) +#define CCDER_PRIVATE ((ccder_tag)3 << (sizeof(ccder_tag) * 8 - 2)) +#else /* !CCDER_MULTIBYTE_TAGS */ +#define CCDER_TAG_MASK CCASN1_TAG_MASK +#define CCDER_TAGNUM_MASK CCASN1_TAGNUM_MASK + +#define CCDER_METHOD_MASK CCASN1_METHOD_MASK +#define CCDER_PRIMITIVE CCASN1_PRIMITIVE +#define CCDER_CONSTRUCTED CCASN1_CONSTRUCTED + +#define CCDER_CLASS_MASK CCASN1_CLASS_MASK +#define CCDER_UNIVERSAL CCASN1_UNIVERSAL +#define CCDER_APPLICATION CCASN1_APPLICATION +#define CCDER_CONTEXT_SPECIFIC CCASN1_CONTEXT_SPECIFIC +#define CCDER_PRIVATE CCASN1_PRIVATE +#endif /* !CCDER_MULTIBYTE_TAGS */ +#define CCDER_CONSTRUCTED_SET (CCDER_SET | CCDER_CONSTRUCTED) +#define CCDER_CONSTRUCTED_SEQUENCE (CCDER_SEQUENCE | CCDER_CONSTRUCTED) + // MARK: ccder_sizeof_ functions diff --git a/EXTERNAL_HEADERS/corecrypto/ccdrbg.h b/EXTERNAL_HEADERS/corecrypto/ccdrbg.h index 7ab4f491d..af5b010a9 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccdrbg.h +++ b/EXTERNAL_HEADERS/corecrypto/ccdrbg.h @@ -41,7 +41,7 @@ #define CCDRBG_MAX_ENTROPY_SIZE ((uint32_t)1<<16) #define CCDRBG_MAX_ADDITIONALINPUT_SIZE ((uint32_t)1<<16) #define CCDRBG_MAX_PSINPUT_SIZE ((uint32_t)1<<16) -#define CCDRBG_MAX_REQUEST_SIZE ((uint32_t)1<<16) //this is the the absolute maximum in NIST 800-90A +#define CCDRBG_MAX_REQUEST_SIZE ((uint32_t)1<<16) //this is the absolute maximum in NIST 800-90A #define CCDRBG_RESEED_INTERVAL ((uint64_t)1<<30) // must be able to fit the NIST maximum of 2^48 @@ -95,10 +95,10 @@ CC_INLINE size_t ccdrbg_context_size(const struct ccdrbg_info *drbg) /* * NIST SP 800-90 CTR_DRBG - * the mximum security strengh of drbg equals to the block size of the corresponding ECB. + * the maximum security strengh of drbg equals to the block size of the corresponding ECB. */ struct ccdrbg_nistctr_custom { - const struct ccmode_ecb *ecb; + const struct ccmode_ctr *ctr_info; size_t keylen; int strictFIPS; int use_df; diff --git a/EXTERNAL_HEADERS/corecrypto/cchmac.h b/EXTERNAL_HEADERS/corecrypto/cchmac.h index c3427eaab..81c1ab835 100644 --- a/EXTERNAL_HEADERS/corecrypto/cchmac.h +++ b/EXTERNAL_HEADERS/corecrypto/cchmac.h @@ -19,10 +19,14 @@ struct cchmac_ctx { uint8_t b[8]; } CC_ALIGNED(8); +#if CORECRYPTO_USE_TRANSPARENT_UNION typedef union { struct cchmac_ctx *hdr; ccdigest_ctx_t digest; } cchmac_ctx_t __attribute__((transparent_union)); +#else +typedef struct cchmac_ctx* cchmac_ctx_t; +#endif #define cchmac_ctx_size(STATE_SIZE, BLOCK_SIZE) (ccdigest_ctx_size(STATE_SIZE, BLOCK_SIZE) + (STATE_SIZE)) #define cchmac_di_size(_di_) (cchmac_ctx_size((_di_)->state_size, (_di_)->block_size)) @@ -35,24 +39,43 @@ typedef union { #define cchmac_di_clear(_di_, _name_) cchmac_ctx_clear((_di_)->state_size, (_di_)->block_size, _name_) /* Return a ccdigest_ctx_t which can be accesed with the macros in ccdigest.h */ +#if CORECRYPTO_USE_TRANSPARENT_UNION #define cchmac_digest_ctx(_di_, HC) (((cchmac_ctx_t)(HC)).digest) +#else +#define cchmac_digest_ctx(_di_, HC) ((ccdigest_ctx_t)(HC)) +#endif /* Accesors for ostate fields, this is all cchmac_ctx_t adds to the ccdigest_ctx_t. */ +#if CORECRYPTO_USE_TRANSPARENT_UNION #define cchmac_ostate(_di_, HC) ((struct ccdigest_state *)(((cchmac_ctx_t)(HC)).hdr->b + ccdigest_di_size(_di_))) +#else +#define cchmac_ostate(_di_, HC) ((struct ccdigest_state *)(((cchmac_ctx_t)(HC))->b + ccdigest_di_size(_di_))) +#endif #define cchmac_ostate8(_di_, HC) (ccdigest_u8(cchmac_ostate(_di_, HC))) #define cchmac_ostate32(_di_, HC) (ccdigest_u32(cchmac_ostate(_di_, HC))) #define cchmac_ostate64(_di_, HC) (ccdigest_u64(cchmac_ostate(_di_, HC))) #define cchmac_ostateccn(_di_, HC) (ccdigest_ccn(cchmac_ostate(_di_, HC))) /* Convenience accessors for ccdigest_ctx_t fields. */ +#if CORECRYPTO_USE_TRANSPARENT_UNION #define cchmac_istate(_di_, HC) ccdigest_state(_di_, ((cchmac_ctx_t)(HC)).digest) +#else +#define cchmac_istate(_di_, HC) ccdigest_state(_di_, ((ccdigest_ctx_t)(HC))) +#endif #define cchmac_istate8(_di_, HC) ccdigest_u8(cchmac_istate(_di_, HC)) #define cchmac_istate32(_di_, HC) ccdigest_u32(cchmac_istate(_di_, HC)) #define cchmac_istate64(_di_, HC) ccdigest_u64(cchmac_istate(_di_, HC)) #define cchmac_istateccn(_di_, HC) ccdigest_ccn(cchmac_istate(_di_, HC)) + +#if CORECRYPTO_USE_TRANSPARENT_UNION #define cchmac_data(_di_, HC) ccdigest_data(_di_, ((cchmac_ctx_t)(HC)).digest) #define cchmac_num(_di_, HC) ccdigest_num(_di_, ((cchmac_ctx_t)(HC)).digest) #define cchmac_nbits(_di_, HC) ccdigest_nbits(_di_, ((cchmac_ctx_t)(HC)).digest) +#else +#define cchmac_data(_di_, HC) ccdigest_data(_di_, ((ccdigest_ctx_t)(HC))) +#define cchmac_num(_di_, HC) ccdigest_num(_di_, ((ccdigest_ctx_t)(HC))) +#define cchmac_nbits(_di_, HC) ccdigest_nbits(_di_, ((ccdigest_ctx_t)(HC))) +#endif void cchmac_init(const struct ccdigest_info *di, cchmac_ctx_t ctx, size_t key_len, const void *key); diff --git a/EXTERNAL_HEADERS/corecrypto/ccmode.h b/EXTERNAL_HEADERS/corecrypto/ccmode.h index eda253b4d..191460b9b 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccmode.h +++ b/EXTERNAL_HEADERS/corecrypto/ccmode.h @@ -115,23 +115,10 @@ CC_INLINE int cccbc_update(const struct ccmode_cbc *mode, cccbc_ctx *ctx, return mode->cbc(ctx, iv, nblocks, in, out); } -CC_INLINE int cccbc_one_shot(const struct ccmode_cbc *mode, - size_t key_len, const void *key, - const void *iv, size_t nblocks, - const void *in, void *out) -{ - int rc; - cccbc_ctx_decl(mode->size, ctx); - cccbc_iv_decl(mode->block_size, iv_ctx); - rc = mode->init(mode, ctx, key_len, key); - if (iv) - cccbc_set_iv(mode, iv_ctx, iv); - else - cc_zero(mode->block_size, iv_ctx); - mode->cbc(ctx, iv_ctx, nblocks, in, out); - cccbc_ctx_clear(mode->size, ctx); - return rc; -} +int cccbc_one_shot(const struct ccmode_cbc *mode, + size_t key_len, const void *key, + const void *iv, size_t nblocks, + const void *in, void *out); /* CFB mode. */ @@ -256,7 +243,8 @@ CC_INLINE int ccctr_one_shot(const struct ccmode_ctr *mode, int rc; ccctr_ctx_decl(mode->size, ctx); rc = mode->init(mode, ctx, key_len, key, iv); - mode->ctr(ctx, nbytes, in, out); + if (rc) return rc; + rc = mode->ctr(ctx, nbytes, in, out); ccctr_ctx_clear(mode->size, ctx); return rc; } @@ -429,6 +417,12 @@ int ccxts_one_shot(const struct ccmode_xts *mode, #define CCGCM_IV_NBYTES 12 #define CCGCM_BLOCK_NBYTES 16 +/* (2^32 - 2) blocks */ +/* (2^36 - 32) bytes */ +/* (2^39 - 256) bits */ +/* Exceeding this figure breaks confidentiality and authenticity. */ +#define CCGCM_TEXT_MAX_NBYTES ((1ULL << 36) - 32ULL) + CC_INLINE size_t ccgcm_context_size(const struct ccmode_gcm *mode) { return mode->size; @@ -470,6 +464,7 @@ CC_INLINE size_t ccgcm_block_size(const struct ccmode_gcm *mode) @warning It is not permitted to call @p ccgcm_inc_iv after initializing the cipher via the @p ccgcm_init interface. Nonzero is returned in the event of an improper call sequence. + @warning This function is not FIPS-compliant. Use @p ccgcm_init_with_iv instead. */ CC_INLINE int ccgcm_init(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, size_t key_nbytes, const void *key) @@ -536,6 +531,8 @@ int ccgcm_init_with_iv(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, In stateless protocols, it is recommended to choose a 16-byte value using a cryptographically-secure pseudorandom number generator (e.g. @p ccrng). @warning This function may not be used after initializing the cipher via @p ccgcm_init_with_iv. Nonzero is returned in the event of an improper call sequence. + + @warning This function is not FIPS-compliant. Use @p ccgcm_init_with_iv instead. */ CC_INLINE int ccgcm_set_iv(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, size_t iv_nbytes, const void *iv) @@ -653,9 +650,9 @@ CC_INLINE int ccgcm_update(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, On encryption, @p tag is purely an output parameter. The generated tag is written to @p tag. - On decryption, @p tag is primarily an input parameter. The caller should provide the authentication tag generated during encryption. The function will return nonzero if the input tag does not match the generated tag. + On decryption, @p tag is both an input and an output parameter. Well-behaved callers should provide the authentication tag generated during encryption. The function will return nonzero if the input tag does not match the generated tag. The generated tag will be written into the @p tag buffer whether authentication succeeds or fails. - @warning To support legacy applications, @p tag is also an output parameter during decryption. The generated tag is written to @p tag. Legacy callers may choose to compare this to the tag generated during encryption. Do not follow this usage pattern in new applications. + @warning The generated tag is written to @p tag to support legacy applications that perform authentication manually. Do not follow this usage pattern in new applications. Rely on the function's error code to verify authenticity. */ CC_INLINE int ccgcm_finalize(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, size_t tag_nbytes, void *tag) @@ -815,10 +812,10 @@ CC_INLINE int ccccm_reset(const struct ccmode_ccm *mode, ccccm_ctx *ctx, ccccm_n CC_INLINE int ccccm_one_shot(const struct ccmode_ccm *mode, size_t key_len, const void *key, - unsigned nonce_len, const void *nonce, + size_t nonce_len, const void *nonce, size_t nbytes, const void *in, void *out, - unsigned adata_len, const void* adata, - unsigned mac_size, void *mac) + size_t adata_len, const void* adata, + size_t mac_size, void *mac) { int rc=0; ccccm_ctx_decl(mode->size, ctx); @@ -829,7 +826,7 @@ CC_INLINE int ccccm_one_shot(const struct ccmode_ccm *mode, if(rc==0) rc=mode->ccm(ctx, nonce_ctx, nbytes, in, out); if(rc==0) rc=mode->finalize(ctx, nonce_ctx, mac); ccccm_ctx_clear(mode->size, ctx); - ccccm_nonce_clear(mode->size, nonce_ctx); + ccccm_nonce_clear(mode->nonce_size, nonce_ctx); return rc; } diff --git a/EXTERNAL_HEADERS/corecrypto/ccmode_factory.h b/EXTERNAL_HEADERS/corecrypto/ccmode_factory.h index c05518e27..668ea9d59 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccmode_factory.h +++ b/EXTERNAL_HEADERS/corecrypto/ccmode_factory.h @@ -251,12 +251,13 @@ void ccmode_factory_cfb8_encrypt(struct ccmode_cfb8 *cfb8, int ccmode_ctr_init(const struct ccmode_ctr *ctr, ccctr_ctx *ctx, size_t rawkey_len, const void *rawkey, const void *iv); +int ccmode_ctr_setctr(const struct ccmode_ctr *mode, ccctr_ctx *ctx, const void *ctr); int ccmode_ctr_crypt(ccctr_ctx *ctx, size_t nbytes, const void *in, void *out); struct _ccmode_ctr_key { const struct ccmode_ecb *ecb; - size_t pad_len; + size_t pad_offset; cc_unit u[]; }; @@ -264,7 +265,9 @@ struct _ccmode_ctr_key { #define CCMODE_FACTORY_CTR_CRYPT(ECB_ENCRYPT) { \ .size = ccn_sizeof_size(sizeof(struct _ccmode_ctr_key)) + 2 * ccn_sizeof_size((ECB_ENCRYPT)->block_size) + ccn_sizeof_size((ECB_ENCRYPT)->size), \ .block_size = 1, \ +.ecb_block_size = (ECB_ENCRYPT)->block_size, \ .init = ccmode_ctr_init, \ +.setctr = ccmode_ctr_setctr, \ .ctr = ccmode_ctr_crypt, \ .custom = (ECB_ENCRYPT) \ } @@ -292,13 +295,13 @@ int ccmode_gcm_encrypt(ccgcm_ctx *ctx, size_t nbytes, const void *in, /*! @function ccmode_gcm_finalize() finalizes AES-GCM call sequence @param key encryption or decryption key - @param tag_size - @param tag + @param tag_nbytes length of tag in bytes + @param tag authentication tag @result 0=success or non zero= error @discussion For decryption, the tag parameter must be the expected-tag. A secure compare is performed between the provided expected-tag and the computed-tag. If they are the same, 0 is returned. Otherwise, non zero is returned. For encryption, tag is output and provides the authentication tag. */ -int ccmode_gcm_finalize(ccgcm_ctx *key, size_t tag_size, void *tag); +int ccmode_gcm_finalize(ccgcm_ctx *key, size_t tag_nbytes, void *tag); int ccmode_gcm_reset(ccgcm_ctx *key); #define CCGCM_FLAGS_INIT_WITH_IV 1 @@ -331,7 +334,7 @@ struct _ccmode_gcm_key { int encdec; //is it an encrypt or decrypt object // Buffer with ECB key and H table if applicable - unsigned char u[] __attribute__ ((aligned (16))); // ecb key + tables + CC_ALIGNED(16) unsigned char u[]; // ecb key + tables }; #define GCM_ECB_KEY_SIZE(ECB_ENCRYPT) \ diff --git a/EXTERNAL_HEADERS/corecrypto/ccmode_impl.h b/EXTERNAL_HEADERS/corecrypto/ccmode_impl.h index 1337e1467..795054161 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccmode_impl.h +++ b/EXTERNAL_HEADERS/corecrypto/ccmode_impl.h @@ -101,9 +101,11 @@ cc_aligned_struct(16) ccctr_ctx; struct ccmode_ctr { size_t size; /* first argument to ccctr_ctx_decl(). */ - size_t block_size; - int (*init)(const struct ccmode_ctr *ctr, ccctr_ctx *ctx, + size_t block_size; /* for historical reasons, this is set to 1 */ + size_t ecb_block_size; /* the actual block size of the underlying cipher */ + int (*init)(const struct ccmode_ctr *mode, ccctr_ctx *ctx, size_t key_len, const void *key, const void *iv); + int (*setctr)(const struct ccmode_ctr *mode, ccctr_ctx *ctx, const void *ctr); int (*ctr)(ccctr_ctx *ctx, size_t nbytes, const void *in, void *out); const void *custom; }; @@ -125,15 +127,13 @@ cc_aligned_struct(16) ccxts_ctx; cc_aligned_struct(16) ccxts_tweak; struct ccmode_xts { - size_t size; /* first argument to ccxts_ctx_decl(). */ - size_t tweak_size; /* first argument to ccxts_tweak_decl(). */ + size_t size; /* first argument to ccxts_ctx_decl(). Size of the ctx data structure */ + size_t tweak_size; /* first argument to ccxts_tweak_decl(). Size of the tweak structure, not the expected tweak size */ size_t block_size; - /* Create a xts key from a xts mode object. The tweak_len here - determines how long the tweak is in bytes, for each subsequent call to - ccmode_xts->xts(). - key must point to at least 'size' cc_units of free storage. - tweak_key must point to at least 'tweak_size' cc_units of free storage. + /* Create a xts key from a xts mode object. + key must point to at least 'size' bytes of free storage. + tweak_key must point to at least 'tweak_size' bytes of free storage. key and tweak_key must differ. Returns nonzero on failure. */ diff --git a/EXTERNAL_HEADERS/corecrypto/ccmode_siv.h b/EXTERNAL_HEADERS/corecrypto/ccmode_siv.h index 69069bb3a..5186e1227 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccmode_siv.h +++ b/EXTERNAL_HEADERS/corecrypto/ccmode_siv.h @@ -68,8 +68,7 @@ CC_INLINE size_t ccsiv_plaintext_size(const struct ccmode_siv *mode, return ciphertext_size-mode->cbc->block_size; } -// In theory, supported key sizes are 32, 48, 64 bytes -// In practice, we only support key size 32 bytes due to cmac limitation +// Supported key sizes are 32, 48, 64 bytes CC_INLINE int ccsiv_init(const struct ccmode_siv *mode, ccsiv_ctx *ctx, size_t key_byte_len, const uint8_t *key) { @@ -115,7 +114,8 @@ CC_INLINE int ccsiv_one_shot(const struct ccmode_siv *mode, { int rc; ccsiv_ctx_decl(mode->size, ctx); - ccsiv_init(mode, ctx, key_len, key); + rc=mode->init(mode, ctx, key_len, key); + if (rc) {return rc;} rc=mode->set_nonce(ctx, nonce_nbytes, nonce); if (rc) {return rc;} rc=mode->auth(ctx, adata_nbytes, adata); diff --git a/EXTERNAL_HEADERS/corecrypto/ccn.h b/EXTERNAL_HEADERS/corecrypto/ccn.h index 53c152c88..afaed41ae 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccn.h +++ b/EXTERNAL_HEADERS/corecrypto/ccn.h @@ -402,8 +402,8 @@ void ccn_mul(cc_size n, cc_unit *r_2n, const cc_unit *s, const cc_unit *t); { n bit, n bit -> 2 * n bit } n = count * sizeof(cc_unit) * 8 { N bit, N bit -> 2N bit } N = ccn_bitsof(n) Provide a workspace for potential speedup */ -CC_NONNULL((2, 3, 4, 5)) -void ccn_mul_ws(cc_size count, cc_unit *r, const cc_unit *s, const cc_unit *t, cc_ws_t ws); +CC_NONNULL((1, 3, 4, 5)) +void ccn_mul_ws(cc_ws_t ws, cc_size count, cc_unit *r, const cc_unit *s, const cc_unit *t); /* s[0..n) * v -> r[0..n)+return value { N bit, sizeof(cc_unit) * 8 bit -> N + sizeof(cc_unit) * 8 bit } N = n * sizeof(cc_unit) * 8 */ @@ -500,8 +500,8 @@ void ccn_sqr(cc_size n, cc_unit *r, const cc_unit *s); /* s^2 -> r { n bit -> 2 * n bit } */ -CC_NONNULL((2, 3, 4)) -void ccn_sqr_ws(cc_size n, cc_unit *r, const cc_unit *s, cc_ws_t ws); +CC_NONNULL((1, 3, 4)) +void ccn_sqr_ws(cc_ws_t ws, cc_size n, cc_unit *r, const cc_unit *s); #else @@ -515,8 +515,8 @@ void ccn_sqr(cc_size n, cc_unit *r, const cc_unit *s) { /* s^2 -> r { n bit -> 2 * n bit } */ CC_INLINE CC_NONNULL((2, 3, 4)) -void ccn_sqr_ws(cc_size n, cc_unit *r, const cc_unit *s, cc_ws_t ws) { - ccn_mul_ws(n, r, s, s, ws); +void ccn_sqr_ws(cc_ws_t ws, cc_size n, cc_unit *r, const cc_unit *s) { + ccn_mul_ws(ws, n, r, s, s); } #endif @@ -639,7 +639,7 @@ int ccn_random_bits(cc_size nbits, cc_unit *r, struct ccrng_state *rng); @param d input number d */ CC_NONNULL((2, 3)) -void ccn_make_recip(cc_size nd, cc_unit *recip, const cc_unit *d); +int ccn_make_recip(cc_size nd, cc_unit *recip, const cc_unit *d); CC_NONNULL((6, 8)) int ccn_div_euclid(cc_size nq, cc_unit *q, cc_size nr, cc_unit *r, cc_size na, const cc_unit *a, cc_size nd, const cc_unit *d); @@ -647,22 +647,4 @@ int ccn_div_euclid(cc_size nq, cc_unit *q, cc_size nr, cc_unit *r, cc_size na, c #define ccn_div(nq, q, na, a, nd, d) ccn_div_euclid(nq, q, 0, NULL, na, a, nd, d) #define ccn_mod(nr, r, na, a, nd, d) ccn_div_euclid(0 , NULL, nr, r, na, a, nd, d) -/*! - @brief ccn_div_use_recip(nq, q, nr, r, na, a, nd, d) comutes q=a/d and r=a%d - @discussion q and rcan be NULL. Reads na from a and nd from d. Writes nq in q and nr in r. nq and nr must be large enough to accomodate results, otherwise error is retuned. Execution time depends on the size of a. Computation is perfomed on of fixedsize and the leadig zeros of a of q are are also used in the computation. - @param nq length of array q that hold the quotients. The maximum length of quotient is the actual length of dividend a - @param q returned quotient. If nq is larger than needed, it is filled with leading zeros. If it is smaller, error is returned. q can be set to NULL, if not needed. - @param nr length of array r that hold the remainder. The maximum length of remainder is the actual length of divisor d - @param r returned remainder. If nr is larger than needed, it is filled with leading zeros. Ifi is smaller error is returned. r can be set to NULL if not required. - @param na length of dividend. Dividend may have leading zeros. - @param a input Dividend - @param nd length of input divisor. Divisor may have leading zeros. - @param d input Divisor - @param recip_d The reciprocal of d, of length nd+1. - - @return returns 0 if successful, negative of error. - */ -CC_NONNULL((6, 8, 9)) -int ccn_div_use_recip(cc_size nq, cc_unit *q, cc_size nr, cc_unit *r, cc_size na, const cc_unit *a, cc_size nd, const cc_unit *d, const cc_unit *recip_d); - #endif /* _CORECRYPTO_CCN_H_ */ diff --git a/EXTERNAL_HEADERS/corecrypto/ccrng.h b/EXTERNAL_HEADERS/corecrypto/ccrng.h index f32922276..698f412ca 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccrng.h +++ b/EXTERNAL_HEADERS/corecrypto/ccrng.h @@ -11,33 +11,50 @@ #ifndef _CORECRYPTO_CCRNG_H_ #define _CORECRYPTO_CCRNG_H_ -#include - #include -#define CC_ERR_DEVICE -100 -#define CC_ERR_INTERUPTS -101 -#define CC_ERR_CRYPTO_CONFIG -102 -#define CC_ERR_PERMS -103 -#define CC_ERR_PARAMETER -104 -#define CC_ERR_MEMORY -105 -#define CC_ERR_FILEDESC -106 -#define CC_ERR_OUT_OF_ENTROPY -107 -#define CC_ERR_INTERNAL -108 -#define CC_ERR_ATFORK -109 -#define CC_ERR_OVERFLOW -110 +#define CCERR_DEVICE -100 +#define CCERR_INTERUPTS -101 +#define CCERR_CRYPTO_CONFIG -102 +#define CCERR_PERMS -103 +#define CCERR_PARAMETER -104 +#define CCERR_MEMORY -105 +#define CCERR_FILEDESC -106 +#define CCERR_OUT_OF_ENTROPY -107 +#define CCERR_INTERNAL -108 +#define CCERR_ATFORK -109 +#define CCERR_OVERFLOW -110 #define CCRNG_STATE_COMMON \ int (*generate)(struct ccrng_state *rng, size_t outlen, void *out); -/* Get a pointer to a ccrng has never been simpler! Just call this */ -struct ccrng_state *ccrng(int *error); - -/* default state structure - do not instantiate, instead use the specific one you need */ +/* default state structure. Do not instantiate, ccrng() returns a reference to this structure */ struct ccrng_state { CCRNG_STATE_COMMON }; -#define ccrng_generate(ctx, outlen, out) ((ctx)->generate((ctx), (outlen), (out))) +/*! + @function ccrng + @abstract initializes a AES-CTR mode cryptographic random number generator and returns the statically alocated rng object. + Getting a pointer to a ccrng has never been simpler! + Call this function, get an rng object and then pass the object to ccrng_generate() to generate randoms. + ccrng() may be called more than once. It returns pointer to the same object on all calls. + + @result a cryptographically secure random number generator or NULL if fails + + @discussion + - It is significantly faster than using the system /dev/random + - FIPS Compliant: NIST SP800-80A + FIPS 140-2 + - Seeded from the system entropy. + - Provides at least 128bit security if the system provide 2bit of entropy / byte. + - Entropy accumulation + - Backtracing resistance + - Prediction break with frequent (asynchronous) reseed + */ + +struct ccrng_state *ccrng(int *error); + +//call this macro with the rng argument set to output of the call to the ccrng() function +#define ccrng_generate(rng, outlen, out) ((rng)->generate((rng), (outlen), (out))) #endif /* _CORECRYPTO_CCRNG_H_ */ diff --git a/EXTERNAL_HEADERS/corecrypto/ccrsa.h b/EXTERNAL_HEADERS/corecrypto/ccrsa.h index 97a88529a..c821efc40 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccrsa.h +++ b/EXTERNAL_HEADERS/corecrypto/ccrsa.h @@ -182,7 +182,7 @@ ccrsa_pubkeylength(ccrsa_pub_ctx_t pubk) { /* Initialize key based on modulus and e as cc_unit. key->zp.n must already be set. */ CC_NONNULL_TU((1)) CC_NONNULL((2, 3)) -void ccrsa_init_pub(ccrsa_pub_ctx_t key, const cc_unit *modulus, +int ccrsa_init_pub(ccrsa_pub_ctx_t key, const cc_unit *modulus, const cc_unit *e); /* Initialize key based on modulus and e as big endian byte array diff --git a/EXTERNAL_HEADERS/corecrypto/cczp.h b/EXTERNAL_HEADERS/corecrypto/cczp.h index f19891bd8..f06b96a9d 100644 --- a/EXTERNAL_HEADERS/corecrypto/cczp.h +++ b/EXTERNAL_HEADERS/corecrypto/cczp.h @@ -41,7 +41,7 @@ typedef union { typedef struct cczp* cczp_t; typedef const struct cczp* cczp_const_t; #endif -typedef void (*ccmod_func_t)(cczp_const_t zp, cc_unit *r, const cc_unit *s, cc_ws_t ws); +typedef void (*ccmod_func_t)(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *s); // keep cczp_hd and cczp structures consistent // cczp_hd is typecasted to cczp to read EC curve params @@ -168,7 +168,7 @@ CC_INLINE size_t cczp_bitlen(cczp_const_t zp) { /* Ensure both cczp_mod_prime(zp) and cczp_recip(zp) are valid. cczp_n and cczp_prime must have been previously initialized. */ CC_NONNULL_TU((1)) -void cczp_init(cczp_t zp); +int cczp_init(cczp_t zp); /* Compute r = s2n mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads 2 * cczp_n(zp) units units from s2n. If r and s2n are not @@ -176,7 +176,7 @@ void cczp_init(cczp_t zp); cczp_init(zp) must have been called or both CCZP_MOD_PRIME((cc_unit *)zp) and CCZP_RECIP((cc_unit *)zp) must be initialized some other way. */ CC_NONNULL_TU((1)) CC_NONNULL((2, 3)) -void cczp_mod(cczp_const_t zp, cc_unit *r, const cc_unit *s2n, cc_ws_t ws); +void cczp_mod(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *s2n); /* Compute r = sn mod cczp_prime(zp), Will write cczp_n(zp) units to r and reads sn units units from s. If r and s are not @@ -184,7 +184,6 @@ void cczp_mod(cczp_const_t zp, cc_unit *r, const cc_unit *s2n, cc_ws_t ws); cczp_init(zp) must have been called or both CCZP_MOD_PRIME((cc_unit *)zp) and CCZP_RECIP((cc_unit *)zp) must be initialized some other way. */ CC_NONNULL_TU((1)) CC_NONNULL((2, 4)) - int cczp_modn(cczp_const_t zp, cc_unit *r, cc_size ns, const cc_unit *s); /* Compute r = x * y mod cczp_prime(zp). Will write cczp_n(zp) units to r @@ -197,7 +196,7 @@ CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4)) void cczp_mul(cczp_const_t zp, cc_unit *t, const cc_unit *x, const cc_unit *y); CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4, 5)) -void cczp_mul_ws(cczp_const_t zp, cc_unit *t, const cc_unit *x, const cc_unit *y, cc_ws_t ws); +void cczp_mul_ws(cc_ws_t ws, cczp_const_t zp, cc_unit *t, const cc_unit *x, const cc_unit *y); /* Compute r = x * x mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads cczp_n(zp) units from x. If r and x are not identical they must @@ -208,7 +207,7 @@ CC_NONNULL_TU((1)) CC_NONNULL((2, 3)) void cczp_sqr(cczp_const_t zp, cc_unit *r, const cc_unit *x); CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4)) -void cczp_sqr_ws(cczp_const_t zp, cc_unit *r, const cc_unit *x, cc_ws_t ws); +void cczp_sqr_ws(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *x); /* Compute r = x^(1/2) mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads cczp_n(zp) units from x. If r and x are not identical they must @@ -229,8 +228,8 @@ int cczp_sqrt(cczp_const_t zp, cc_unit *r, const cc_unit *x); be initialized some other way. */ CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4)) -void cczp_power(cczp_const_t zp, cc_unit *r, const cc_unit *m, - const cc_unit *e); +int cczp_power(cczp_const_t zp, cc_unit *r, const cc_unit *m, + const cc_unit *e); /* Compute r = m ^ e mod cczp_prime(zp), using Square Square Multiply Always. - writes cczp_n(zp) units to r @@ -258,8 +257,8 @@ int cczp_power_ssma_ws(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *s or both CCZP_MOD_PRIME((cc_unit *)zp) and CCZP_RECIP((cc_unit *)zp) must be initialized some other way. */ CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 5)) -void cczp_powern(cczp_const_t zp, cc_unit *r, const cc_unit *s, - size_t ebitlen, const cc_unit *e); +int cczp_powern(cczp_const_t zp, cc_unit *r, const cc_unit *s, + size_t ebitlen, const cc_unit *e); /* Compute r = x + y mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads cczp_n(zp) units units from x and y. If r and x are not identical @@ -270,8 +269,8 @@ void cczp_add(cczp_const_t zp, cc_unit *r, const cc_unit *x, const cc_unit *y); CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4, 5)) -void cczp_add_ws(cczp_const_t zp, cc_unit *r, const cc_unit *x, - const cc_unit *y, cc_ws_t ws); +void cczp_add_ws(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *x, + const cc_unit *y); /* Compute r = x - y mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads cczp_n(zp) units units from x and y. If r and x are not identical @@ -281,8 +280,8 @@ CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4)) void cczp_sub(cczp_const_t zp, cc_unit *r, const cc_unit *x, const cc_unit *y); CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4, 5)) -void cczp_sub_ws(cczp_const_t zp, cc_unit *r, const cc_unit *x, - const cc_unit *y, cc_ws_t ws); +void cczp_sub_ws(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *x, + const cc_unit *y); /* Compute r = x / 2 mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads cczp_n(zp) units units from x. If r and x are not identical diff --git a/EXTERNAL_HEADERS/corecrypto/fipspost_trace.h b/EXTERNAL_HEADERS/corecrypto/fipspost_trace.h new file mode 100644 index 000000000..c236bebd7 --- /dev/null +++ b/EXTERNAL_HEADERS/corecrypto/fipspost_trace.h @@ -0,0 +1,45 @@ +/* + * fipspost_trace.h + * corecrypto + * + * Created on 01/25/2017 + * + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + */ + +#ifndef _CORECRYPTO_FIPSPOST_TRACE_H_ +#define _CORECRYPTO_FIPSPOST_TRACE_H_ + +#if CC_FIPSPOST_TRACE + +/* + * Use this string to separate out tests. + */ +#define FIPSPOST_TRACE_TEST_STR "?" + +int fipspost_trace_is_active(void); +void fipspost_trace_call(const char *fname); + +/* Only trace when VERBOSE is set to avoid impacting normal boots. */ +#define FIPSPOST_TRACE_EVENT do { \ + if (fipspost_trace_is_active()) { \ + fipspost_trace_call(__FUNCTION__); \ + } \ +} while (0); + +#define FIPSPOST_TRACE_MESSAGE(MSG) do { \ + if (fipspost_trace_is_active()) { \ + fipspost_trace_call(MSG); \ + } \ +} while (0); + +#else + +/* Not building a CC_FIPSPOST_TRACE-enabled, no TRACE operations. */ +#define FIPSPOST_TRACE_EVENT +#define FIPSPOST_TRACE_MESSAGE(X) + +#endif + +#endif diff --git a/EXTERNAL_HEADERS/mach-o/arm/reloc.h b/EXTERNAL_HEADERS/mach-o/arm/reloc.h new file mode 100644 index 000000000..2447814fd --- /dev/null +++ b/EXTERNAL_HEADERS/mach-o/arm/reloc.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 1999 Apple Computer, Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ +/* + * Relocation types used in the arm implementation. Relocation entries for + * things other than instructions use the same generic relocation as discribed + * in and their r_type is ARM_RELOC_VANILLA, one of the + * *_SECTDIFF or the *_PB_LA_PTR types. The rest of the relocation types are + * for instructions. Since they are for instructions the r_address field + * indicates the 32 bit instruction that the relocation is to be preformed on. + */ +enum reloc_type_arm +{ + ARM_RELOC_VANILLA, /* generic relocation as discribed above */ + ARM_RELOC_PAIR, /* the second relocation entry of a pair */ + ARM_RELOC_SECTDIFF, /* a PAIR follows with subtract symbol value */ + ARM_RELOC_LOCAL_SECTDIFF, /* like ARM_RELOC_SECTDIFF, but the symbol + referenced was local. */ + ARM_RELOC_PB_LA_PTR,/* prebound lazy pointer */ + ARM_RELOC_BR24, /* 24 bit branch displacement (to a word address) */ + ARM_THUMB_RELOC_BR22, /* 22 bit branch displacement (to a half-word + address) */ + ARM_THUMB_32BIT_BRANCH, /* obsolete - a thumb 32-bit branch instruction + possibly needing page-spanning branch workaround */ + + /* + * For these two r_type relocations they always have a pair following them + * and the r_length bits are used differently. The encoding of the + * r_length is as follows: + * low bit of r_length: + * 0 - :lower16: for movw instructions + * 1 - :upper16: for movt instructions + * high bit of r_length: + * 0 - arm instructions + * 1 - thumb instructions + * the other half of the relocated expression is in the following pair + * relocation entry in the the low 16 bits of r_address field. + */ + ARM_RELOC_HALF, + ARM_RELOC_HALF_SECTDIFF +}; diff --git a/EXTERNAL_HEADERS/mach-o/arm64/reloc.h b/EXTERNAL_HEADERS/mach-o/arm64/reloc.h new file mode 100644 index 000000000..0a98f18e7 --- /dev/null +++ b/EXTERNAL_HEADERS/mach-o/arm64/reloc.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2010 Apple Computer, Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ +/* + * Relocation types used in the arm64 implementation. + */ +enum reloc_type_arm64 +{ + ARM64_RELOC_UNSIGNED, // for pointers + ARM64_RELOC_SUBTRACTOR, // must be followed by a ARM64_RELOC_UNSIGNED + ARM64_RELOC_BRANCH26, // a B/BL instruction with 26-bit displacement + ARM64_RELOC_PAGE21, // pc-rel distance to page of target + ARM64_RELOC_PAGEOFF12, // offset within page, scaled by r_length + ARM64_RELOC_GOT_LOAD_PAGE21, // pc-rel distance to page of GOT slot + ARM64_RELOC_GOT_LOAD_PAGEOFF12, // offset within page of GOT slot, + // scaled by r_length + ARM64_RELOC_POINTER_TO_GOT, // for pointers to GOT slots + ARM64_RELOC_TLVP_LOAD_PAGE21, // pc-rel distance to page of TLVP slot + ARM64_RELOC_TLVP_LOAD_PAGEOFF12, // offset within page of TLVP slot, + // scaled by r_length + ARM64_RELOC_ADDEND // must be followed by PAGE21 or PAGEOFF12 +}; diff --git a/EXTERNAL_HEADERS/mach-o/loader.h b/EXTERNAL_HEADERS/mach-o/loader.h index ffaf873d8..d6bc7e0cd 100644 --- a/EXTERNAL_HEADERS/mach-o/loader.h +++ b/EXTERNAL_HEADERS/mach-o/loader.h @@ -302,6 +302,8 @@ struct load_command { #define LC_LINKER_OPTIMIZATION_HINT 0x2E /* optimization hints in MH_OBJECT files */ #define LC_VERSION_MIN_TVOS 0x2F /* build for AppleTV min OS version */ #define LC_VERSION_MIN_WATCHOS 0x30 /* build for Watch min OS version */ +#define LC_NOTE 0x31 /* arbitrary data included within a Mach-O file */ +#define LC_BUILD_VERSION 0x32 /* build for platform min OS version */ /* * A variable length string in a load command is represented by an lc_str @@ -1203,13 +1205,45 @@ struct encryption_info_command_64 { */ struct version_min_command { uint32_t cmd; /* LC_VERSION_MIN_MACOSX or - LC_VERSION_MIN_IPHONEOS - LC_VERSION_MIN_WATCHOS */ + LC_VERSION_MIN_IPHONEOS or + LC_VERSION_MIN_WATCHOS or + LC_VERSION_MIN_TVOS */ uint32_t cmdsize; /* sizeof(struct min_version_command) */ uint32_t version; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ }; +/* + * The build_version_command contains the min OS version on which this + * binary was built to run for its platform. The list of known platforms and + * tool values following it. + */ +struct build_version_command { + uint32_t cmd; /* LC_BUILD_VERSION */ + uint32_t cmdsize; /* sizeof(struct build_version_command) plus */ + /* ntools * sizeof(struct build_tool_version) */ + uint32_t platform; /* platform */ + uint32_t minos; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ + uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ + uint32_t ntools; /* number of tool entries following this */ +}; + +struct build_tool_version { + uint32_t tool; /* enum for the tool */ + uint32_t version; /* version number of the tool */ +}; + +/* Known values for the platform field above. */ +#define PLATFORM_MACOS 1 +#define PLATFORM_IOS 2 +#define PLATFORM_TVOS 3 +#define PLATFORM_WATCHOS 4 + +/* Known values for the tool field above. */ +#define TOOL_CLANG 1 +#define TOOL_SWIFT 2 +#define TOOL_LD 3 + /* * The dyld_info_command contains the file offsets and sizes of * the new compressed form of the information dyld needs to @@ -1489,4 +1523,16 @@ struct tlv_descriptor unsigned long offset; }; +/* + * LC_NOTE commands describe a region of arbitrary data included in a Mach-O + * file. Its initial use is to record extra data in MH_CORE files. + */ +struct note_command { + uint32_t cmd; /* LC_NOTE */ + uint32_t cmdsize; /* sizeof(struct note_command) */ + char data_owner[16]; /* owner name for this LC_NOTE */ + uint64_t offset; /* file offset of this data */ + uint64_t size; /* length of data region */ +}; + #endif /* _MACHO_LOADER_H_ */ diff --git a/Makefile b/Makefile index c4c0663ef..fa9f39132 100644 --- a/Makefile +++ b/Makefile @@ -196,16 +196,17 @@ include $(MakeInc_cmd) include $(MakeInc_def) ALL_SUBDIRS = \ + security \ bsd \ iokit \ osfmk \ pexpert \ libkern \ libsa \ - security \ - config + config \ + san -CONFIG_SUBDIRS = config tools +CONFIG_SUBDIRS = config tools san INSTINC_SUBDIRS = $(ALL_SUBDIRS) EXTERNAL_HEADERS INSTINC_SUBDIRS_X86_64 = $(INSTINC_SUBDIRS) @@ -219,7 +220,7 @@ EXPINC_SUBDIRS_X86_64H = $(EXPINC_SUBDIRS) EXPINC_SUBDIRS_ARM = $(EXPINC_SUBDIRS) EXPINC_SUBDIRS_ARM64 = $(EXPINC_SUBDIRS) -SETUP_SUBDIRS = SETUP +SETUP_SUBDIRS = SETUP san COMP_SUBDIRS_X86_64 = $(ALL_SUBDIRS) COMP_SUBDIRS_X86_64H = $(ALL_SUBDIRS) diff --git a/README.md b/README.md index 2d3ba49a8..b285c1a6a 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,8 @@ Other makefile options * $ make REMOTEBUILD=user@remotehost # perform build on remote host * $ make BUILD_JSON_COMPILATION_DATABASE=1 # Build Clang JSON Compilation Database +The XNU build system can optionally output color-formatted build output. To enable this, you can either +set the `XNU_LOGCOLORS` environment variable to `y`, or you can pass `LOGCOLORS=y` to the make command. Debug information formats @@ -244,6 +246,11 @@ member file lists and their default location are described below - Definition - INSTALL_KF_MI_LCL_LIST = ${KERNELFILES} ${PRIVATE_KERNELFILES} + e. `EXPORT_MI_LIST` : Exports header file to all of xnu (bsd/, osfmk/, etc.) + for compilation only. Does not install anything into the SDK. + Definition - + EXPORT_MI_LIST = ${KERNELFILES} ${PRIVATE_KERNELFILES} + If you want to install the header file in a sub-directory of the paths described in (1), specify the directory name using two variables `INSTALL_MI_DIR` and `EXPORT_MI_DIR` as follows - diff --git a/SETUP/kextsymboltool/kextsymboltool.c b/SETUP/kextsymboltool/kextsymboltool.c index 8bd2c293c..edb6dfaea 100644 --- a/SETUP/kextsymboltool/kextsymboltool.c +++ b/SETUP/kextsymboltool/kextsymboltool.c @@ -597,11 +597,6 @@ int main(int argc, char * argv[]) else num_export_syms += files[filenum].nsyms; } - if (!num_export_syms) - { - fprintf(stderr, "no export names\n"); - exit(1); - } import_symbols = calloc(num_import_syms, sizeof(struct symbol)); export_symbols = calloc(num_export_syms, sizeof(struct symbol)); diff --git a/bsd/arm/Makefile b/bsd/arm/Makefile new file mode 100644 index 000000000..f9ab9a989 --- /dev/null +++ b/bsd/arm/Makefile @@ -0,0 +1,34 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +DATAFILES = \ + endian.h fasttrap_isa.h param.h \ + profile.h signal.h limits.h _limits.h \ + types.h vmparam.h _types.h _param.h \ + _mcontext.h + +PRIVATE_DATAFILES = \ + disklabel.h + +KERNELFILES = \ + endian.h param.h \ + profile.h signal.h limits.h _limits.h \ + types.h vmparam.h _types.h _param.h \ + _mcontext.h + +INSTALL_MD_LIST = ${DATAFILES} +INSTALL_MD_LCL_LIST = ${PRIVATE_DATAFILES} + +INSTALL_MD_DIR = arm + +EXPORT_MD_LIST = ${KERNELFILES} + +EXPORT_MD_DIR = arm + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/bsd/arm/_limits.h b/bsd/arm/_limits.h new file mode 100644 index 000000000..f3d3fcb2c --- /dev/null +++ b/bsd/arm/_limits.h @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2004-2007 Apple Inc. All rights reserved. + */ +#ifndef _ARM__LIMITS_H_ +#define _ARM__LIMITS_H_ + +#define __DARWIN_CLK_TCK 100 /* ticks per second */ + +#endif /* _ARM__LIMITS_H_ */ diff --git a/bsd/arm/_mcontext.h b/bsd/arm/_mcontext.h new file mode 100644 index 000000000..5a2e735cf --- /dev/null +++ b/bsd/arm/_mcontext.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2003-2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef __ARM_MCONTEXT_H_ +#define __ARM_MCONTEXT_H_ + +#include /* __DARWIN_UNIX03 */ +#include +#include + +#ifndef _STRUCT_MCONTEXT32 +#if __DARWIN_UNIX03 +#define _STRUCT_MCONTEXT32 struct __darwin_mcontext32 +_STRUCT_MCONTEXT32 +{ + _STRUCT_ARM_EXCEPTION_STATE __es; + _STRUCT_ARM_THREAD_STATE __ss; + _STRUCT_ARM_VFP_STATE __fs; +}; + +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_MCONTEXT32 struct mcontext32 +_STRUCT_MCONTEXT32 +{ + _STRUCT_ARM_EXCEPTION_STATE es; + _STRUCT_ARM_THREAD_STATE ss; + _STRUCT_ARM_VFP_STATE fs; +}; + +#endif /* __DARWIN_UNIX03 */ +#endif /* _STRUCT_MCONTEXT32 */ + + +#ifndef _STRUCT_MCONTEXT64 +#if __DARWIN_UNIX03 +#define _STRUCT_MCONTEXT64 struct __darwin_mcontext64 +_STRUCT_MCONTEXT64 +{ + _STRUCT_ARM_EXCEPTION_STATE64 __es; + _STRUCT_ARM_THREAD_STATE64 __ss; + _STRUCT_ARM_NEON_STATE64 __ns; +}; + +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_MCONTEXT64 struct mcontext64 +_STRUCT_MCONTEXT64 +{ + _STRUCT_ARM_EXCEPTION_STATE64 es; + _STRUCT_ARM_THREAD_STATE64 ss; + _STRUCT_ARM_NEON_STATE64 ns; +}; +#endif /* __DARWIN_UNIX03 */ +#endif /* _STRUCT_MCONTEXT32 */ + +#ifndef _MCONTEXT_T +#define _MCONTEXT_T +#if defined(__LP64__) +typedef _STRUCT_MCONTEXT64 *mcontext_t; +#define _STRUCT_MCONTEXT _STRUCT_MCONTEXT64 +#else +typedef _STRUCT_MCONTEXT32 *mcontext_t; +#define _STRUCT_MCONTEXT _STRUCT_MCONTEXT32 +#endif +#endif /* _MCONTEXT_T */ + +#endif /* __ARM_MCONTEXT_H_ */ diff --git a/bsd/arm/_param.h b/bsd/arm/_param.h new file mode 100644 index 000000000..2d1e03a96 --- /dev/null +++ b/bsd/arm/_param.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2006-2007 Apple Inc. All rights reserved. + */ + +#ifndef _ARM__PARAM_H_ +#define _ARM__PARAM_H_ + +#include + +/* + * Round p (pointer or byte index) up to a correctly-aligned value for all + * data types (int, long, ...). The result is unsigned int and must be + * cast to any desired pointer type. + */ +#define __DARWIN_ALIGNBYTES (sizeof(__darwin_size_t) - 1) +#define __DARWIN_ALIGN(p) ((__darwin_size_t)((char *)(__darwin_size_t)(p) + __DARWIN_ALIGNBYTES) &~ __DARWIN_ALIGNBYTES) + +#define __DARWIN_ALIGNBYTES32 (sizeof(__uint32_t) - 1) +#define __DARWIN_ALIGN32(p) ((__darwin_size_t)((char *)(__darwin_size_t)(p) + __DARWIN_ALIGNBYTES32) &~ __DARWIN_ALIGNBYTES32) + + +#endif /* _ARM__PARAM_H_ */ diff --git a/bsd/arm/_types.h b/bsd/arm/_types.h new file mode 100644 index 000000000..d76d8a64b --- /dev/null +++ b/bsd/arm/_types.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +#ifndef _BSD_ARM__TYPES_H_ +#define _BSD_ARM__TYPES_H_ + +/* + * This header file contains integer types. It's intended to also contain + * flotaing point and other arithmetic types, as needed, later. + */ + +#ifdef __GNUC__ +typedef __signed char __int8_t; +#else /* !__GNUC__ */ +typedef char __int8_t; +#endif /* !__GNUC__ */ +typedef unsigned char __uint8_t; +typedef short __int16_t; +typedef unsigned short __uint16_t; +typedef int __int32_t; +typedef unsigned int __uint32_t; +typedef long long __int64_t; +typedef unsigned long long __uint64_t; + +typedef long __darwin_intptr_t; +typedef unsigned int __darwin_natural_t; + +/* + * The rune type below is declared to be an ``int'' instead of the more natural + * ``unsigned long'' or ``long''. Two things are happening here. It is not + * unsigned so that EOF (-1) can be naturally assigned to it and used. Also, + * it looks like 10646 will be a 31 bit standard. This means that if your + * ints cannot hold 32 bits, you will be in trouble. The reason an int was + * chosen over a long is that the is*() and to*() routines take ints (says + * ANSI C), but they use __darwin_ct_rune_t instead of int. By changing it + * here, you lose a bit of ANSI conformance, but your programs will still + * work. + * + * NOTE: rune_t is not covered by ANSI nor other standards, and should not + * be instantiated outside of lib/libc/locale. Use wchar_t. wchar_t and + * rune_t must be the same type. Also wint_t must be no narrower than + * wchar_t, and should also be able to hold all members of the largest + * character set plus one extra value (WEOF). wint_t must be at least 16 bits. + */ + +typedef int __darwin_ct_rune_t; /* ct_rune_t */ + +/* + * mbstate_t is an opaque object to keep conversion state, during multibyte + * stream conversions. The content must not be referenced by user programs. + */ +typedef union { + char __mbstate8[128]; + long long _mbstateL; /* for alignment */ +} __mbstate_t; + +typedef __mbstate_t __darwin_mbstate_t; /* mbstate_t */ + +#if defined(__PTRDIFF_TYPE__) +typedef __PTRDIFF_TYPE__ __darwin_ptrdiff_t; /* ptr1 - ptr2 */ +#elif defined(__LP64__) +typedef long __darwin_ptrdiff_t; /* ptr1 - ptr2 */ +#else +typedef int __darwin_ptrdiff_t; /* ptr1 - ptr2 */ +#endif /* __GNUC__ */ + +#if defined(__SIZE_TYPE__) +typedef __SIZE_TYPE__ __darwin_size_t; /* sizeof() */ +#else +typedef unsigned long __darwin_size_t; /* sizeof() */ +#endif + +#if (__GNUC__ > 2) +typedef __builtin_va_list __darwin_va_list; /* va_list */ +#else +typedef void * __darwin_va_list; /* va_list */ +#endif + +#if defined(__WCHAR_TYPE__) +typedef __WCHAR_TYPE__ __darwin_wchar_t; /* wchar_t */ +#else +typedef __darwin_ct_rune_t __darwin_wchar_t; /* wchar_t */ +#endif + +typedef __darwin_wchar_t __darwin_rune_t; /* rune_t */ + +#if defined(__WINT_TYPE__) +typedef __WINT_TYPE__ __darwin_wint_t; /* wint_t */ +#else +typedef __darwin_ct_rune_t __darwin_wint_t; /* wint_t */ +#endif + +typedef unsigned long __darwin_clock_t; /* clock() */ +typedef __uint32_t __darwin_socklen_t; /* socklen_t (duh) */ +typedef long __darwin_ssize_t; /* byte count or error */ +typedef long __darwin_time_t; /* time() */ + +#endif /* _BSD_ARM__TYPES_H_ */ diff --git a/bsd/arm/disklabel.h b/bsd/arm/disklabel.h new file mode 100644 index 000000000..966f66d50 --- /dev/null +++ b/bsd/arm/disklabel.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +#ifndef _MACHINE_DISKLABEL_H_ +#define _MACHINE_DISKLABEL_H_ + +#include + +#ifdef __APPLE_API_OBSOLETE +#define LABELSECTOR (1024 / DEV_BSIZE) /* sector containing label */ +#define LABELOFFSET 0 /* offset of label in sector */ +#define MAXPARTITIONS 8 /* number of partitions */ +#define RAW_PART 2 /* raw partition: xx?c */ + +/* Just a dummy */ +struct cpu_disklabel { + int cd_dummy; /* must have one element. */ +}; +#endif /* __APPLE_API_OBSOLETE */ + +#endif /* _MACHINE_DISKLABEL_H_ */ diff --git a/bsd/arm/endian.h b/bsd/arm/endian.h new file mode 100644 index 000000000..6cd67268d --- /dev/null +++ b/bsd/arm/endian.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright 1995 NeXT Computer, Inc. All rights reserved. + */ +/* + * Copyright (c) 1987, 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)endian.h 8.1 (Berkeley) 6/11/93 + */ + +#ifndef _ARM__ENDIAN_H_ +#define _ARM__ENDIAN_H_ + +#include +/* + * Define _NOQUAD if the compiler does NOT support 64-bit integers. + */ +/* #define _NOQUAD */ + +/* + * Define the order of 32-bit words in 64-bit words. + */ +#define _QUAD_HIGHWORD 1 +#define _QUAD_LOWWORD 0 + +/* + * Definitions for byte order, according to byte significance from low + * address to high. + */ +#define __DARWIN_LITTLE_ENDIAN 1234 /* LSB first: i386, vax */ +#define __DARWIN_BIG_ENDIAN 4321 /* MSB first: 68000, ibm, net */ +#define __DARWIN_PDP_ENDIAN 3412 /* LSB first in word, MSW first in long */ + +#define __DARWIN_BYTE_ORDER __DARWIN_LITTLE_ENDIAN + +#if defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) + +#define LITTLE_ENDIAN __DARWIN_LITTLE_ENDIAN +#define BIG_ENDIAN __DARWIN_BIG_ENDIAN +#define PDP_ENDIAN __DARWIN_PDP_ENDIAN + +#define BYTE_ORDER __DARWIN_BYTE_ORDER + +#include + +#endif /* defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) */ +#endif /* !_ARM__ENDIAN_H_ */ diff --git a/bsd/arm/exec.h b/bsd/arm/exec.h new file mode 100644 index 000000000..e1266aacd --- /dev/null +++ b/bsd/arm/exec.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/*- + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)exec.h 8.1 (Berkeley) 6/11/93 + */ + +#ifndef _BSD_ARM_EXEC_H_ +#define _BSD_ARM_EXEC_H_ + + +#ifdef BSD_KERNEL_PRIVATE +/* Size of a page in an object file. */ +#define __LDPGSZ 4096 + +/* Valid magic number check. */ +#define N_BADMAG(ex) \ + ((ex).a_magic != NMAGIC && (ex).a_magic != OMAGIC && \ + (ex).a_magic != ZMAGIC) + +/* Address of the bottom of the text segment. */ +#define N_TXTADDR(X) 0 + +/* Address of the bottom of the data segment. */ +#define N_DATADDR(ex) \ + (N_TXTADDR(ex) + ((ex).a_magic == OMAGIC ? (ex).a_text \ + : __LDPGSZ + ((ex).a_text - 1 & ~(__LDPGSZ - 1)))) + +/* Text segment offset. */ +#define N_TXTOFF(ex) \ + ((ex).a_magic == ZMAGIC ? __LDPGSZ : sizeof(struct exec)) + +/* Data segment offset. */ +#define N_DATOFF(ex) \ + (N_TXTOFF(ex) + ((ex).a_magic != ZMAGIC ? (ex).a_text : \ + __LDPGSZ + ((ex).a_text - 1 & ~(__LDPGSZ - 1)))) + +/* Symbol table offset. */ +#define N_SYMOFF(ex) \ + (N_TXTOFF(ex) + (ex).a_text + (ex).a_data + (ex).a_trsize + \ + (ex).a_drsize) + +/* String table offset. */ +#define N_STROFF(ex) (N_SYMOFF(ex) + (ex).a_syms) + +/* Description of the object file header (a.out format). */ +struct exec { +#define OMAGIC 0407 /* old impure format */ +#define NMAGIC 0410 /* read-only text */ +#define ZMAGIC 0413 /* demand load format */ +#define QMAGIC 0314 /* demand load format. Header in text. */ + unsigned int a_magic; /* magic number */ + + unsigned int a_text; /* text segment size */ + unsigned int a_data; /* initialized data size */ + unsigned int a_bss; /* uninitialized data size */ + unsigned int a_syms; /* symbol table size */ + unsigned int a_entry; /* entry point */ + unsigned int a_trsize; /* text relocation size */ + unsigned int a_drsize; /* data relocation size */ +}; + +#endif /* BSD_KERNEL_PRIVATE */ + +#endif /* _BSD_ARM_EXEC_H_ */ diff --git a/bsd/arm/fasttrap_isa.h b/bsd/arm/fasttrap_isa.h new file mode 100644 index 000000000..eb577a43f --- /dev/null +++ b/bsd/arm/fasttrap_isa.h @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FASTTRAP_ISA_H +#define _FASTTRAP_ISA_H + +/* #pragma ident "@(#)fasttrap_isa.h 1.4 05/06/08 SMI" */ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef union { + uint32_t instr32; + struct { + uint16_t instr1; + uint16_t instr2; + } instr16; +} fasttrap_instr_t; + +typedef struct fasttrap_machtp { + fasttrap_instr_t ftmt_instr; /* original instruction */ + + uint8_t ftmt_fntype; /* One of the FASTTRAP_FN* constants defined below */ + /* Once the tracepoint is initialized, fntype will be FN_DONE_INIT and thumb will be 0 for ARM, 1 for Thumb */ + uint8_t ftmt_thumb; + + uint8_t ftmt_type; + uint8_t ftmt_installed:1; + uint8_t ftmt_retired:1; +} fasttrap_machtp_t; + +#define ftt_instr ftt_mtp.ftmt_instr.instr32 +#define ftt_instr1 ftt_mtp.ftmt_instr.instr16.instr1 +#define ftt_instr2 ftt_mtp.ftmt_instr.instr16.instr2 +#define ftt_fntype ftt_mtp.ftmt_fntype +#define ftt_thumb ftt_mtp.ftmt_thumb +#define ftt_type ftt_mtp.ftmt_type +#define ftt_installed ftt_mtp.ftmt_installed +#define ftt_retired ftt_mtp.ftmt_retired + +#define FASTTRAP_T_INV 1 +#define FASTTRAP_T_COMMON 2 +#define FASTTRAP_T_BLX 3 +#define FASTTRAP_T_B_COND 4 +#define FASTTRAP_T_B_UNCOND 5 +#define FASTTRAP_T_BX_REG 6 +#define FASTTRAP_T_PUSH_LR 7 +#define FASTTRAP_T_POP_PC 8 +#define FASTTRAP_T_STM_LR 9 +#define FASTTRAP_T_LDM_PC 10 +#define FASTTRAP_T_CPY_PC 11 +#define FASTTRAP_T_MOV_PC_REG 12 +#define FASTTRAP_T_LDR_PC_IMMED 13 +#define FASTTRAP_T_VLDR_PC_IMMED 14 +#define FASTTRAP_T_CB_N_Z 15 +#if defined(__arm64__) +#define FASTTRAP_T_ARM64_STANDARD_FUNCTION_ENTRY 16 /* stp fp, lr, [sp, #-16]! */ +#define FASTTRAP_T_ARM64_LDR_S_PC_REL 17 +#define FASTTRAP_T_ARM64_LDR_W_PC_REL 18 +#define FASTTRAP_T_ARM64_LDR_D_PC_REL 19 +#define FASTTRAP_T_ARM64_LDR_X_PC_REL 20 +#define FASTTRAP_T_ARM64_LDR_Q_PC_REL 21 +#define FASTTRAP_T_ARM64_LDRSW_PC_REL 22 +#define FASTTRAP_T_ARM64_B_COND 23 +#define FASTTRAP_T_ARM64_CBNZ_W 24 +#define FASTTRAP_T_ARM64_CBNZ_X 25 +#define FASTTRAP_T_ARM64_CBZ_W 26 +#define FASTTRAP_T_ARM64_CBZ_X 27 +#define FASTTRAP_T_ARM64_TBNZ 28 +#define FASTTRAP_T_ARM64_TBZ 29 +#define FASTTRAP_T_ARM64_B 30 +#define FASTTRAP_T_ARM64_BL 31 +#define FASTTRAP_T_ARM64_BLR 32 +#define FASTTRAP_T_ARM64_BR 33 +#define FASTTRAP_T_ARM64_RET 34 +#define FASTTRAP_T_ARM64_ADRP 35 +#define FASTTRAP_T_ARM64_ADR 36 +#define FASTTRAP_T_ARM64_PRFM 37 +#define FASTTRAP_T_ARM64_EXCLUSIVE_MEM 38 +#endif + +#if defined (__arm__) +#define FASTTRAP_ARM_INSTR 0xe7ffdefc +#define FASTTRAP_THUMB_INSTR 0xdefc +#define FASTTRAP_ARM_RET_INSTR 0xe7ffdefb +#define FASTTRAP_THUMB_RET_INSTR 0xdefb + +#elif defined (__arm64__) +#define FASTTRAP_ARM32_INSTR 0xe7ffdefc +#define FASTTRAP_THUMB32_INSTR 0xdefc +#define FASTTRAP_ARM64_INSTR 0xe7eeee7e + +#define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb +#define FASTTRAP_THUMB32_RET_INSTR 0xdefb +#define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d +#endif + +#define FASTTRAP_FN_DONE_INIT 255 +#define FASTTRAP_FN_UNKNOWN 0 +#define FASTTRAP_FN_ARM 1 +#define FASTTRAP_FN_THUMB 2 +#define FASTTRAP_FN_USDT 3 + +#define ARM_RM(x) ((x) & 0xF) +#define ARM_RS(x) (((x) >> 8) & 0xF) +#define ARM_RD(x) (((x) >> 12) & 0xF) +#define ARM_RN(x) (((x) >> 16) & 0xF) +#define ARM_CONDCODE(x) ((x) >> 28) + +#define THUMB16_HRM(x) (((x) >> 3) & 0xF) +#define THUMB16_HRD(x) (((x) & 0x7) | ((((x) >> 4) & 0x8))) + +#define THUMB32_RM(x,y) ((y) & 0xF) +#define THUMB32_RD(x,y) (((y) >> 8) & 0xF) +#define THUMB32_RT(x,y) (((y) >> 12) & 0xF) +#define THUMB32_RN(x,y) ((x) & 0xF) + +#define REG_SP 13 +#define REG_LR 14 +#define REG_PC 15 + +#define FASTTRAP_RETURN_AFRAMES 6 +#define FASTTRAP_ENTRY_AFRAMES 5 +#define FASTTRAP_OFFSET_AFRAMES 5 + +#if defined(__arm64__) +#define FASTTRAP_ARM64_OP_VALUE_FUNC_ENTRY 0xa9bf7bfd /* stp fp, lr, [sp, #-16]! */ + +#define FASTTRAP_ARM64_OP_MASK_LDR_S_PC_REL 0xff000000 /* Bits to check for ldr St, label */ +#define FASTTRAP_ARM64_OP_VALUE_LDR_S_PC_REL 0x1c000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_LDR_W_PC_REL 0xff000000 /* Bits to check for ldr Wt, label */ +#define FASTTRAP_ARM64_OP_VALUE_LDR_W_PC_REL 0x18000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_LDR_D_PC_REL 0xff000000 /* Bits to check for ldr Dt, label */ +#define FASTTRAP_ARM64_OP_VALUE_LDR_D_PC_REL 0x5c000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_LDR_X_PC_REL 0xff000000 /* Bits to check for ldr Xt, label */ +#define FASTTRAP_ARM64_OP_VALUE_LDR_X_PC_REL 0x58000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_LDR_Q_PC_REL 0xff000000 /* Bits to check for ldr Qt, label */ +#define FASTTRAP_ARM64_OP_VALUE_LDR_Q_PC_REL 0x9c000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_LRDSW_PC_REL 0xff000000 /* Bits to check for ldrsw , label */ +#define FASTTRAP_ARM64_OP_VALUE_LRDSW_PC_REL 0x98000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_B_COND_PC_REL 0xff000010 /* Bits to check for b.cond label */ +#define FASTTRAP_ARM64_OP_VALUE_B_COND_PC_REL 0x54000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_CBNZ_W_PC_REL 0xff000000 /* Bits to check for cbnz Wt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_CBNZ_W_PC_REL 0x35000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_CBNZ_X_PC_REL 0xff000000 /* Bits to check for cbnz Xt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_CBNZ_X_PC_REL 0xb5000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_CBZ_W_PC_REL 0xff000000 /* Bits to check for cbz Wt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_CBZ_W_PC_REL 0x34000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_CBZ_X_PC_REL 0xff000000 /* Bits to check for cbz Xt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_CBZ_X_PC_REL 0xb4000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_TBNZ_PC_REL 0x7f000000 /* Bits to check for tbnz Xt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_TBNZ_PC_REL 0x37000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_TBZ_PC_REL 0x7f000000 /* Bits to check for tbz Xt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_TBZ_PC_REL 0x36000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_B_PC_REL 0xfc000000 /* Bits to check for b _label */ +#define FASTTRAP_ARM64_OP_VALUE_B_PC_REL 0x14000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_BL_PC_REL 0xfc000000 /* Bits to check for bl _label */ +#define FASTTRAP_ARM64_OP_VALUE_BL_PC_REL 0x94000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_BLR 0xfffffe1f /* Bits to check for blr Xt */ +#define FASTTRAP_ARM64_OP_VALUE_BLR 0xd63f0000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_BR 0xfffffe1f /* Bits to check for br Xt */ +#define FASTTRAP_ARM64_OP_VALUE_BR 0xd61f0000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_RET 0xfffffc1f /* Bits to check for ret Rt */ +#define FASTTRAP_ARM64_OP_VALUE_RET 0xd65f0000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_ADRP 0x9f000000 /* Bits to check for adrp Xt, label*/ +#define FASTTRAP_ARM64_OP_VALUE_ADRP 0x90000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_ADR 0x9f000000 /* Bits to check for adr Xt, label*/ +#define FASTTRAP_ARM64_OP_VALUE_ADR 0x10000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_PRFM 0xff000000 /* Bits to check for adr Xt, label*/ +#define FASTTRAP_ARM64_OP_VALUE_PRFM 0xd8000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_EXCL_MEM 0x3f000000 /* Bits to check for exclusive memory operation */ +#define FASTTRAP_ARM64_OP_VALUE_EXCL_MEM 0x08000000 /* Value to find */ +#endif /* defined(__arm64__) */ + +#ifdef __cplusplus +} +#endif + +#endif /* _FASTTRAP_ISA_H */ diff --git a/bsd/arm/limits.h b/bsd/arm/limits.h new file mode 100644 index 000000000..32c8033b9 --- /dev/null +++ b/bsd/arm/limits.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1988, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)limits.h 8.3 (Berkeley) 1/4/94 + */ + +#ifndef _ARM_LIMITS_H_ +#define _ARM_LIMITS_H_ + +#include +#include + +#define CHAR_BIT 8 /* number of bits in a char */ +#define MB_LEN_MAX 6 /* Allow 31 bit UTF2 */ + +#if !defined(_ANSI_SOURCE) && (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) +#define CLK_TCK __DARWIN_CLK_TCK /* ticks per second */ +#endif /* !_ANSI_SOURCE && (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ + +/* + * According to ANSI (section 2.2.4.2), the values below must be usable by + * #if preprocessing directives. Additionally, the expression must have the + * same type as would an expression that is an object of the corresponding + * type converted according to the integral promotions. The subtraction for + * INT_MIN and LONG_MIN is so the value is not unsigned; 2147483648 is an + * unsigned int for 32-bit two's complement ANSI compilers (section 3.1.3.2). + * These numbers work for pcc as well. The UINT_MAX and ULONG_MAX values + * are written as hex so that GCC will be quiet about large integer constants. + */ +#define SCHAR_MAX 127 /* min value for a signed char */ +#define SCHAR_MIN (-128) /* max value for a signed char */ + +#define UCHAR_MAX 255 /* max value for an unsigned char */ +#define CHAR_MAX 127 /* max value for a char */ +#define CHAR_MIN (-128) /* min value for a char */ + +#define USHRT_MAX 65535 /* max value for an unsigned short */ +#define SHRT_MAX 32767 /* max value for a short */ +#define SHRT_MIN (-32768) /* min value for a short */ + +#define UINT_MAX 0xffffffff /* max value for an unsigned int */ +#define INT_MAX 2147483647 /* max value for an int */ +#define INT_MIN (-2147483647-1) /* min value for an int */ + +#ifdef __LP64__ +#define ULONG_MAX 0xffffffffffffffffUL /* max unsigned long */ +#define LONG_MAX 0x7fffffffffffffffL /* max signed long */ +#define LONG_MIN (-0x7fffffffffffffffL-1) /* min signed long */ +#else /* !__LP64__ */ +#define ULONG_MAX 0xffffffffUL /* max unsigned long */ +#define LONG_MAX 2147483647L /* max signed long */ +#define LONG_MIN (-2147483647L-1) /* min signed long */ +#endif /* __LP64__ */ + +#define ULLONG_MAX 0xffffffffffffffffULL /* max unsigned long long */ +#define LLONG_MAX 0x7fffffffffffffffLL /* max signed long long */ +#define LLONG_MIN (-0x7fffffffffffffffLL-1) /* min signed long long */ + +#if !defined(_ANSI_SOURCE) +#ifdef __LP64__ +#define LONG_BIT 64 +#else /* !__LP64__ */ +#define LONG_BIT 32 +#endif /* __LP64__ */ +#define SSIZE_MAX LONG_MAX /* max value for a ssize_t */ +#define WORD_BIT 32 + +#if (!defined(_POSIX_C_SOURCE) && !defined(_XOPEN_SOURCE)) || defined(_DARWIN_C_SOURCE) +#define SIZE_T_MAX ULONG_MAX /* max value for a size_t */ + +#define UQUAD_MAX ULLONG_MAX +#define QUAD_MAX LLONG_MAX +#define QUAD_MIN LLONG_MIN + +#endif /* (!_POSIX_C_SOURCE && !_XOPEN_SOURCE) || _DARWIN_C_SOURCE */ +#endif /* !_ANSI_SOURCE */ + +#endif /* _ARM_LIMITS_H_ */ diff --git a/bsd/arm/param.h b/bsd/arm/param.h new file mode 100644 index 000000000..538e53418 --- /dev/null +++ b/bsd/arm/param.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2000-2010 Apple Inc. All rights reserved. + */ +/*- + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * (c) UNIX System Laboratories, Inc. + * All or some portions of this file are derived from material licensed + * to the University of California by American Telephone and Telegraph + * Co. or Unix System Laboratories, Inc. and are reproduced herein with + * the permission of UNIX System Laboratories, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)param.h 8.1 (Berkeley) 4/4/95 + */ + +/* + * Machine dependent constants for ARM + */ + +#ifndef _ARM_PARAM_H_ +#define _ARM_PARAM_H_ + +#include + +/* + * Round p (pointer or byte index) up to a correctly-aligned value for all + * data types (int, long, ...). The result is unsigned int and must be + * cast to any desired pointer type. + */ +#define ALIGNBYTES __DARWIN_ALIGNBYTES +#define ALIGN(p) __DARWIN_ALIGN(p) + +#define NBPG 4096 /* bytes/page */ +#define PGOFSET (NBPG-1) /* byte offset into page */ +#define PGSHIFT 12 /* LOG2(NBPG) */ + +#define DEV_BSIZE 512 +#define DEV_BSHIFT 9 /* log2(DEV_BSIZE) */ +#define BLKDEV_IOSIZE 2048 +#define MAXPHYS (64 * 1024) /* max raw I/O transfer size */ + +#define CLSIZE 1 +#define CLSIZELOG2 0 + +/* + * Constants related to network buffer management. + * MCLBYTES must be no larger than CLBYTES (the software page size), and, + * on machines that exchange pages of input or output buffers with mbuf + * clusters (MAPPED_MBUFS), MCLBYTES must also be an integral multiple + * of the hardware page size. + */ +#define MSIZESHIFT 8 /* 256 */ +#define MSIZE (1 << MSIZESHIFT) /* size of an mbuf */ +#define MCLSHIFT 11 /* 2048 */ +#define MCLBYTES (1 << MCLSHIFT) /* size of an mbuf cluster */ +#define MBIGCLSHIFT 12 /* 4096 */ +#define MBIGCLBYTES (1 << MBIGCLSHIFT) /* size of a big cluster */ +#define M16KCLSHIFT 14 /* 16384 */ +#define M16KCLBYTES (1 << M16KCLSHIFT) /* size of a jumbo cluster */ + +#define MCLOFSET (MCLBYTES - 1) +#ifndef NMBCLUSTERS +#define NMBCLUSTERS CONFIG_NMBCLUSTERS /* cl map size */ +#endif + +/* + * Some macros for units conversion + */ +/* Core clicks (NeXT_page_size bytes) to segments and vice versa */ +#define ctos(x) (x) +#define stoc(x) (x) + +/* Core clicks (4096 bytes) to disk blocks */ +#define ctod(x) ((x)<<(PGSHIFT-DEV_BSHIFT)) +#define dtoc(x) ((x)>>(PGSHIFT-DEV_BSHIFT)) +#define dtob(x) ((x)<>PGSHIFT) + +#ifdef __APPLE__ +#define btodb(bytes, devBlockSize) \ + ((unsigned)(bytes) / devBlockSize) +#define dbtob(db, devBlockSize) \ + ((unsigned)(db) * devBlockSize) +#else +#define btodb(bytes) /* calculates (bytes / DEV_BSIZE) */ \ + ((unsigned)(bytes) >> DEV_BSHIFT) +#define dbtob(db) /* calculates (db * DEV_BSIZE) */ \ + ((unsigned)(db) << DEV_BSHIFT) +#endif + +/* + * Map a ``block device block'' to a file system block. + * This should be device dependent, and will be if we + * add an entry to cdevsw/bdevsw for that purpose. + * For now though just use DEV_BSIZE. + */ +#define bdbtofsb(bn) ((bn) / (BLKDEV_IOSIZE/DEV_BSIZE)) + +/* + * Macros to decode (and encode) processor status word. + */ +#define STATUS_WORD(rpl, ipl) (((ipl) << 8) | (rpl)) +#define USERMODE(x) (((x) & 3) == 3) +#define BASEPRI(x) (((x) & (255 << 8)) == 0) + + +#if defined(KERNEL) || defined(STANDALONE) +#define DELAY(n) delay(n) + +#else /* defined(KERNEL) || defined(STANDALONE) */ +#define DELAY(n) { int N = (n); while (--N > 0); } +#endif /* defined(KERNEL) || defined(STANDALONE) */ + +#endif /* _ARM_PARAM_H_ */ diff --git a/bsd/arm/profile.h b/bsd/arm/profile.h new file mode 100644 index 000000000..728d3f99b --- /dev/null +++ b/bsd/arm/profile.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1997, Apple Computer, Inc. All rights reserved. + * + */ + +#ifndef _BSD_ARM_PROFILE_H_ +#define _BSD_ARM_PROFILE_H_ + +#include + +#ifdef KERNEL +#ifdef __APPLE_API_UNSTABLE + +/* + * Block interrupts during mcount so that those interrupts can also be + * counted (as soon as we get done with the current counting). On the + * arm platfom, can't do splhigh/splx as those are C routines and can + * recursively invoke mcount. + */ +#warning MCOUNT_* not implemented yet. + +#define MCOUNT_INIT +#define MCOUNT_ENTER /* s = splhigh(); */ /* XXX TODO */ +#define MCOUNT_EXIT /* (void) splx(s); */ /* XXX TODO */ + +#endif /* __APPLE_API_UNSTABLE */ +#endif /* KERNEL */ + +#endif /* _BSD_ARM_PROFILE_H_ */ diff --git a/bsd/arm/psl.h b/bsd/arm/psl.h new file mode 100644 index 000000000..313ba2d20 --- /dev/null +++ b/bsd/arm/psl.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + */ + +#if KERNEL_PRIVATE + +#ifndef _BSD_ARM_PSL_H_ +#define _BSD_ARM_PSL_H_ + +#endif /* _BSD_ARM_PSL_H_ */ + +#endif /* KERNEL_PRIVATE */ diff --git a/bsd/arm/ptrace.h b/bsd/arm/ptrace.h new file mode 100644 index 000000000..4bac00b86 --- /dev/null +++ b/bsd/arm/ptrace.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)ptrace.h 8.1 (Berkeley) 6/11/93 + */ + +/* + * Machine dependent trace commands. + * + * None for the ARM at this time. + */ diff --git a/bsd/arm/reboot.h b/bsd/arm/reboot.h new file mode 100644 index 000000000..5d47728b4 --- /dev/null +++ b/bsd/arm/reboot.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ + +#ifndef _BSD_ARM_REBOOT_H_ +#define _BSD_ARM_REBOOT_H_ + +/* + * Empty file (publicly) + */ + +#include + +#ifdef BSD_KERNEL_PRIVATE + +/* + * Use most significant 16 bits to avoid collisions with + * machine independent flags. + */ +#define RB_POWERDOWN 0x00010000 /* power down on halt */ +#define RB_NOBOOTRC 0x00020000 /* don't run '/etc/rc.boot' */ +#define RB_DEBUG 0x00040000 /* drop into mini monitor on panic */ +#define RB_EJECT 0x00080000 /* eject disks on halt */ +#define RB_COMMAND 0x00100000 /* new boot command specified */ +#define RB_NOFP 0x00200000 /* don't use floating point */ +#define RB_BOOTNEXT 0x00400000 /* reboot into NeXT */ +#define RB_BOOTDOS 0x00800000 /* reboot into DOS */ +#define RB_PRETTY 0x01000000 /* shutdown with pretty graphics */ + +#endif /* BSD_KERNEL_PRIVATE */ + +#endif /* _BSD_ARM_REBOOT_H_ */ diff --git a/bsd/arm/reg.h b/bsd/arm/reg.h new file mode 100644 index 000000000..bffce0700 --- /dev/null +++ b/bsd/arm/reg.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + */ + +#ifdef KERNEL_PRIVATE + +#ifndef _BSD_ARM_REG_H_ +#define _BSD_ARM_REG_H_ + +#endif /* _BSD_ARM_REG_H_ */ + +#endif /* KERNEL_PRIVATE */ diff --git a/bsd/arm/signal.h b/bsd/arm/signal.h new file mode 100644 index 000000000..e6ed0e24c --- /dev/null +++ b/bsd/arm/signal.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2000-2009 Apple, Inc. All rights reserved. + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + */ + +#ifndef _ARM_SIGNAL_ +#define _ARM_SIGNAL_ 1 + +#include + +#ifndef _ANSI_SOURCE +typedef int sig_atomic_t; +#endif /* ! _ANSI_SOURCE */ + +#endif /* _ARM_SIGNAL_ */ + diff --git a/bsd/arm/types.h b/bsd/arm/types.h new file mode 100644 index 000000000..18906141c --- /dev/null +++ b/bsd/arm/types.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2000-2008 Apple Inc. All rights reserved. + */ +/* + * Copyright 1995 NeXT Computer, Inc. All rights reserved. + */ +/* + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)types.h 8.3 (Berkeley) 1/5/94 + */ + +#ifndef _MACHTYPES_H_ +#define _MACHTYPES_H_ + +#ifndef __ASSEMBLER__ +#include +#include +/* + * Basic integral types. Omit the typedef if + * not possible for a machine/compiler combination. + */ +#include +#include +#include +#include + +#include +#include +#include +#include + +#if __LP64__ +typedef int64_t register_t; +#else +typedef int32_t register_t; +#endif + +#include +#include + +#if !defined(_ANSI_SOURCE) && (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) +/* These types are used for reserving the largest possible size. */ +#ifdef __arm64__ +typedef u_int64_t user_addr_t; +typedef u_int64_t user_size_t; +typedef int64_t user_ssize_t; +typedef int64_t user_long_t; +typedef u_int64_t user_ulong_t; +typedef int64_t user_time_t; +typedef int64_t user_off_t; +#else +typedef u_int32_t user_addr_t; +typedef u_int32_t user_size_t; +typedef int32_t user_ssize_t; +typedef int32_t user_long_t; +typedef u_int32_t user_ulong_t; +typedef int32_t user_time_t; +typedef int64_t user_off_t; +#endif + +#define USER_ADDR_NULL ((user_addr_t) 0) +#define CAST_USER_ADDR_T(a_ptr) ((user_addr_t)((uintptr_t)(a_ptr))) + +#ifdef KERNEL + +/* + * These types are used when you know the word size of the target + * user process. They can be used to create struct layouts independent + * of the types and alignment requirements of the current running + * kernel. + */ + +/* + * The user64_ types are not used on the ARM platform, but exist + * so that APIs that conditionalize their behavior based on the + * size of an input structure (like many ioctl(2) implementations) + * can differentiate those structures without a duplicate case + * value. + */ + +/* + * The default ABI for the ARM platform aligns fundamental integral + * data types to their natural boundaries, with a maximum alignment + * of 4, even for 8-byte quantites. + */ + +typedef __uint64_t user64_addr_t; +typedef __uint64_t user64_size_t; +typedef __int64_t user64_ssize_t; +typedef __int64_t user64_long_t; +typedef __uint64_t user64_ulong_t; +typedef __int64_t user64_time_t; +typedef __int64_t user64_off_t; + +typedef __uint32_t user32_addr_t; +typedef __uint32_t user32_size_t; +typedef __int32_t user32_ssize_t; +typedef __int32_t user32_long_t; +typedef __uint32_t user32_ulong_t; +typedef __int32_t user32_time_t; +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +typedef __int64_t user32_off_t; +#else +typedef __int64_t user32_off_t __attribute__((aligned(4))); +#endif + +#endif /* KERNEL */ + +#endif /* !_ANSI_SOURCE && (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ + +/* This defines the size of syscall arguments after copying into the kernel: */ +#if defined(__arm__) +typedef u_int32_t syscall_arg_t; +#elif defined(__arm64__) +typedef u_int64_t syscall_arg_t; +#else +#error Unknown architecture. +#endif + +#endif /* __ASSEMBLER__ */ +#endif /* _MACHTYPES_H_ */ diff --git a/bsd/arm/vmparam.h b/bsd/arm/vmparam.h new file mode 100644 index 000000000..dbee6526d --- /dev/null +++ b/bsd/arm/vmparam.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ + +#ifndef _BSD_ARM_VMPARAM_H_ +#define _BSD_ARM_VMPARAM_H_ 1 + +#include + +#define USRSTACK (0x27E00000) /* ASLR slides stack down by up to 1MB */ +#define USRSTACK64 (0x000000016FE00000ULL) + +/* + * Virtual memory related constants, all in bytes + */ +#ifndef DFLDSIZ +#define DFLDSIZ (RLIM_INFINITY) /* initial data size limit */ +#endif +#ifndef MAXDSIZ +#define MAXDSIZ (RLIM_INFINITY) /* max data size */ +#endif +#ifndef DFLSSIZ +#define DFLSSIZ (1024*1024 - 16*1024) /* initial stack size limit */ +#endif +#ifndef MAXSSIZ +#define MAXSSIZ (1024*1024) /* max stack size */ +#endif +#ifndef DFLCSIZ +#define DFLCSIZ (0) /* initial core size limit */ +#endif +#ifndef MAXCSIZ +#define MAXCSIZ (RLIM_INFINITY) /* max core size */ +#endif /* MAXCSIZ */ + +#endif /* _BSD_ARM_VMPARAM_H_ */ diff --git a/bsd/bsm/audit_kevents.h b/bsd/bsm/audit_kevents.h index fff152e65..cd7142d60 100644 --- a/bsd/bsm/audit_kevents.h +++ b/bsd/bsm/audit_kevents.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2005-2016 Apple Inc. + * Copyright (c) 2005-2017 Apple Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -612,6 +612,8 @@ #define AUE_OPENBYID_RWT 43209 /* Darwin. */ #define AUE_CLONEFILEAT 43210 /* Darwin. */ #define AUE_FCLONEFILEAT 43211 /* Darwin. */ +#define AUE_SETATTRLISTAT 43212 /* Darwin. */ +#define AUE_FMOUNT 43213 /* Darwin. */ #define AUE_SESSION_START 44901 /* Darwin. */ #define AUE_SESSION_UPDATE 44902 /* Darwin. */ @@ -817,5 +819,6 @@ #define AUE_USRCTL AUE_NULL #define AUE_NEXUS AUE_NULL #define AUE_CHANNEL AUE_NULL +#define AUE_NET AUE_NULL #endif /* !_BSM_AUDIT_KEVENTS_H_ */ diff --git a/bsd/bsm/audit_record.h b/bsd/bsm/audit_record.h index f8dced869..2b6ae891a 100644 --- a/bsd/bsm/audit_record.h +++ b/bsd/bsm/audit_record.h @@ -32,6 +32,7 @@ #ifndef _BSM_AUDIT_RECORD_H_ #define _BSM_AUDIT_RECORD_H_ +#include /* token_t */ #include /* struct timeval */ /* diff --git a/bsd/conf/Makefile.arm b/bsd/conf/Makefile.arm new file mode 100644 index 000000000..9141afe61 --- /dev/null +++ b/bsd/conf/Makefile.arm @@ -0,0 +1,13 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# Files that currently violate cast alignment checks at build time +fbt_arm.o_CFLAGS_ADD += -Wno-cast-qual + +# Inline assembly doesn't interact well with LTO +fbt_arm.o_CFLAGS_ADD += $(CFLAGS_NOLTO_FLAG) + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/bsd/conf/Makefile.arm64 b/bsd/conf/Makefile.arm64 new file mode 100644 index 000000000..c22cdd613 --- /dev/null +++ b/bsd/conf/Makefile.arm64 @@ -0,0 +1,10 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# Inline assembly doesn't interact well with LTO +fbt_arm.o_CFLAGS_ADD += $(CFLAGS_NOLTO_FLAG) + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/bsd/conf/Makefile.template b/bsd/conf/Makefile.template index edfd0c767..afe23cf34 100644 --- a/bsd/conf/Makefile.template +++ b/bsd/conf/Makefile.template @@ -41,6 +41,7 @@ include $(MakeInc_def) CFLAGS+= -include meta_features.h -DDRIVER_PRIVATE \ -D_KERNEL_BUILD -DKERNEL_BUILD -DMACH_KERNEL -DBSD_BUILD \ -DBSD_KERNEL_PRIVATE -DLP64_DEBUG=0 +SFLAGS+= -include meta_features.h # # Directories for mig generated files @@ -82,8 +83,6 @@ vm_unix.o_CFLAGS_ADD += -Wshorten-64-to-32 pthread_synch.o_CFLAGS_ADD += -Wno-unused-parameter -Wno-missing-prototypes pthread_support.o_CFLAGS_ADD += -Wno-unused-parameter -Wno-missing-prototypes -ip_icmp.o_CFLFAGS_ADD += -O0 - # Objects that don't want -Wsign-compare OBJS_NO_SIGN_COMPARE = \ radix.o \ @@ -115,6 +114,7 @@ OBJS_NO_SIGN_COMPARE = \ esp_input.o \ esp_output.o \ esp_rijndael.o \ + esp_chachapoly.o \ ipsec.o \ dest6.o \ frag6.o \ @@ -238,6 +238,7 @@ OBJS_NO_PACKED_ADDRESS = \ ip6_forward.o \ ip6_input.o \ ip6_output.o \ + iptap.o \ ipsec.o \ mld6.o \ mptcp_opt.o \ @@ -256,7 +257,6 @@ OBJS_NO_PACKED_ADDRESS = \ udp6_usrreq.o \ udp_usrreq.o -$(foreach file,$(OBJS_NO_PACKED_ADDRESS),$(eval $(call add_perfile_cflags,$(file),-Wno-unknown-warning-option))) $(foreach file,$(OBJS_NO_PACKED_ADDRESS),$(eval $(call add_perfile_cflags,$(file),-Wno-address-of-packed-member))) # @@ -300,6 +300,10 @@ audit_kevents.c: $(SRCROOT)/bsd/kern/syscalls.master $(MAKESYSCALLS) @echo "[$(CMD_MC)] $(ColorH)GENERATING$(Color0) $(ColorLF)$@$(Color0) from $(ColorF)$<$(Color0)"; $(_v)$(MAKESYSCALLS) $< audit > /dev/null +systrace_args.c: $(SRCROOT)/bsd/kern/syscalls.master $(MAKESYSCALLS) + @echo "[$(CMD_MC)] $(ColorH)GENERATING$(Color0) $(ColorLF)$@$(Color0) from $(ColorF)$<$(Color0)"; + $(_v)$(MAKESYSCALLS) $< systrace > /dev/null + do_all: $(COMPONENT).filelist do_build_all:: do_all diff --git a/bsd/conf/files b/bsd/conf/files index 3eaa5a6a3..104c577ef 100644 --- a/bsd/conf/files +++ b/bsd/conf/files @@ -59,6 +59,7 @@ OPTIONS/ipv6send optional ipv6send OPTIONS/ether optional ether OPTIONS/vlan optional vlan OPTIONS/bond optional bond +OPTIONS/if_fake optional if_fake OPTIONS/bpfilter optional bpfilter OPTIONS/multipath optional multipath OPTIONS/mptcp optional mptcp @@ -70,14 +71,6 @@ OPTIONS/gif optional gif OPTIONS/sendfile optional sendfile OPTIONS/pf optional pf OPTIONS/pflog optional pflog pf -OPTIONS/pf_altq optional pf_altq pf -OPTIONS/classq_blue optional classq_blue -OPTIONS/classq_red optional classq_red -OPTIONS/classq_rio optional classq_rio -OPTIONS/pktsched_cbq optional pktsched_cbq -OPTIONS/pktsched_fairq optional pktsched_fairq -OPTIONS/pktsched_hfsc optional pktsched_hfsc -OPTIONS/pktsched_priq optional pktsched_priq OPTIONS/zlib optional zlib @@ -132,6 +125,7 @@ bsd/dev/dtrace/sdt_subr.c optional config_dtrace bsd/dev/dtrace/systrace.c optional config_dtrace bsd/dev/dtrace/profile_prvd.c optional config_dtrace bsd/dev/dtrace/fasttrap.c optional config_dtrace +./systrace_args.c optional config_dtrace bsd/dev/random/randomdev.c standard @@ -143,6 +137,8 @@ bsd/dev/unix_startup.c standard bsd/dev/vn/vn.c optional vndevice bsd/dev/vn/shadow.c optional vndevice +bsd/dev/monotonic.c optional monotonic + bsd/libkern/crc16.c standard bsd/libkern/crc32.c standard bsd/libkern/random.c standard @@ -173,6 +169,7 @@ bsd/vfs/kpi_vfs.c standard bsd/vfs/vfs_fsevents.c standard bsd/vfs/vfs_cprotect.c standard bsd/vfs/doc_tombstone.c standard +bsd/vfs/vfs_disk_conditioner.c standard bsd/miscfs/deadfs/dead_vnops.c standard bsd/miscfs/devfs/devfs_fdesc_support.c optional fdesc @@ -205,6 +202,7 @@ bsd/net/ether_inet6_pr_module.c optional ether inet6 bsd/net/if_loop.c optional loop bsd/net/if_mib.c optional networking bsd/net/if_vlan.c optional vlan +bsd/net/if_fake.c optional if_fake bsd/net/multicast_list.c optional networking bsd/net/if_bond.c optional bond bsd/net/devtimer.c optional bond @@ -234,6 +232,7 @@ bsd/net/pf_if.c optional pf bsd/net/pf_ioctl.c optional pf bsd/net/pf_norm.c optional pf bsd/net/pf_osfp.c optional pf +bsd/net/pf_pbuf.c optional pf bsd/net/pf_ruleset.c optional pf bsd/net/pf_table.c optional pf bsd/net/iptap.c optional networking @@ -243,33 +242,23 @@ bsd/net/flowhash.c optional networking bsd/net/flowadv.c optional networking bsd/net/content_filter.c optional content_filter bsd/net/packet_mangler.c optional packet_mangler +bsd/net/if_llatbl.c optional networking +bsd/net/nwk_wq.c optional networking +bsd/net/skmem_sysctl.c optional skywalk bsd/net/classq/classq.c optional networking -bsd/net/classq/classq_blue.c optional classq_blue -bsd/net/classq/classq_red.c optional classq_red -bsd/net/classq/classq_rio.c optional classq_rio bsd/net/classq/classq_sfb.c optional networking bsd/net/classq/classq_subr.c optional networking bsd/net/classq/classq_util.c optional networking bsd/net/classq/classq_fq_codel.c optional networking bsd/net/pktsched/pktsched.c optional networking -bsd/net/pktsched/pktsched_cbq.c optional pktsched_cbq -bsd/net/pktsched/pktsched_fairq.c optional pktsched_fairq -bsd/net/pktsched/pktsched_hfsc.c optional pktsched_hfsc -bsd/net/pktsched/pktsched_priq.c optional pktsched_priq bsd/net/pktsched/pktsched_qfq.c optional networking -bsd/net/pktsched/pktsched_rmclass.c optional pktsched_cbq bsd/net/pktsched/pktsched_tcq.c optional networking bsd/net/pktsched/pktsched_fq_codel.c optional networking -bsd/net/altq/altq_cbq.c optional pktsched_cbq pf_altq -bsd/net/altq/altq_fairq.c optional pktsched_fairq pf_altq -bsd/net/altq/altq_hfsc.c optional pktsched_hfsc pf_altq -bsd/net/altq/altq_priq.c optional pktsched_priq pf_altq -bsd/net/altq/altq_qfq.c optional pf_altq -bsd/net/altq/altq_subr.c optional pf_altq - +bsd/netinet/cpu_in_cksum_gen.c standard +bsd/netinet/in_cksum.c optional inet bsd/netinet/igmp.c optional inet bsd/netinet/in.c optional inet bsd/netinet/dhcp_options.c optional inet @@ -279,6 +268,7 @@ bsd/netinet/in_pcb.c optional inet bsd/netinet/in_pcblist.c optional inet bsd/netinet/in_proto.c optional inet bsd/netinet/in_rmx.c optional inet +bsd/netinet/in_stat.c optional inet bsd/netinet/in_tclass.c optional inet bsd/netinet/ip_dummynet.c optional dummynet bsd/netinet/ip_icmp.c optional inet @@ -320,6 +310,7 @@ bsd/netinet6/esp_core.c optional ipsec ipsec_esp bsd/netinet6/esp_input.c optional ipsec ipsec_esp bsd/netinet6/esp_output.c optional ipsec ipsec_esp bsd/netinet6/esp_rijndael.c optional ipsec ipsec_esp +bsd/netinet6/esp_chachapoly.c optional ipsec ipsec_esp bsd/netinet6/ipsec.c optional ipsec bsd/netinet6/dest6.c optional inet6 bsd/netinet6/frag6.c optional inet6 @@ -433,7 +424,6 @@ bsd/kern/kern_time.c standard bsd/kern/kern_xxx.c standard bsd/kern/mach_process.c standard bsd/kern/mcache.c optional sockets -bsd/kern/spl.c standard bsd/kern/stackshot.c standard bsd/kern/subr_log.c standard bsd/kern/subr_prf.c standard @@ -483,14 +473,15 @@ bsd/kern/proc_info.c standard bsd/kern/process_policy.c standard bsd/kern/kern_overrides.c standard bsd/kern/socket_info.c optional sockets +bsd/kern/subr_eventhandler.c standard bsd/kern/sys_reason.c standard bsd/vm/vnode_pager.c standard bsd/vm/vm_unix.c standard bsd/vm/dp_backing_file.c standard bsd/vm/vm_compressor_backing_file.c standard - bsd/kern/kern_ecc.c optional config_ecc_logging +bsd/kern/kern_ntptime.c standard bsd/uxkern/ux_exception.c standard diff --git a/bsd/conf/files.arm b/bsd/conf/files.arm new file mode 100644 index 000000000..0b1712ed9 --- /dev/null +++ b/bsd/conf/files.arm @@ -0,0 +1,20 @@ +bsd/dev/arm/conf.c standard +bsd/dev/arm/cons.c standard +bsd/dev/arm/km.c standard +bsd/dev/arm/kern_machdep.c standard +bsd/dev/arm/stubs.c standard +bsd/dev/arm/systemcalls.c standard +bsd/dev/arm/sysctl.c standard +bsd/dev/arm/unix_signal.c standard +bsd/dev/arm/cpu_in_cksum.s standard + +bsd/dev/arm/dtrace_isa.c optional config_dtrace +bsd/dev/arm/dtrace_subr_arm.c optional config_dtrace +bsd/dev/arm/fbt_arm.c optional config_dtrace +bsd/dev/arm/fasttrap_isa.c optional config_dtrace +bsd/dev/arm/disassembler.c optional config_dtrace +bsd/dev/arm/sdt_arm.c optional config_dtrace + +bsd/dev/arm/munge.c standard + +bsd/kern/bsd_stubs.c standard diff --git a/bsd/conf/files.arm64 b/bsd/conf/files.arm64 new file mode 100644 index 000000000..64009971c --- /dev/null +++ b/bsd/conf/files.arm64 @@ -0,0 +1,21 @@ +bsd/dev/arm64/conf.c standard +bsd/dev/arm/cons.c standard +bsd/dev/arm/km.c standard +bsd/dev/arm/kern_machdep.c standard +bsd/dev/arm/stubs.c standard +bsd/dev/arm/systemcalls.c standard +bsd/dev/arm64/sysctl.c standard +bsd/dev/arm/unix_signal.c standard + +bsd/dev/arm64/cpu_in_cksum.s standard + +bsd/dev/arm64/dtrace_isa.c optional config_dtrace +bsd/dev/arm64/dtrace_subr_arm.c optional config_dtrace +bsd/dev/arm64/fbt_arm.c optional config_dtrace +bsd/dev/arm64/fasttrap_isa.c optional config_dtrace +bsd/dev/arm64/disassembler.c optional config_dtrace +bsd/dev/arm64/sdt_arm.c optional config_dtrace + +bsd/dev/pgtrace/pgtrace_dev.c optional config_pgtrace_nonkext + +bsd/kern/bsd_stubs.c standard diff --git a/bsd/conf/files.x86_64 b/bsd/conf/files.x86_64 index 2fba68035..4eb06372f 100644 --- a/bsd/conf/files.x86_64 +++ b/bsd/conf/files.x86_64 @@ -21,5 +21,3 @@ bsd/dev/i386/dis_tables.c optional config_dtrace bsd/kern/policy_check.c optional config_macf bsd/kern/bsd_stubs.c standard -bsd/netinet/cpu_in_cksum.c standard -bsd/netinet/in_cksum.c optional inet diff --git a/bsd/conf/param.c b/bsd/conf/param.c index 00da0c590..d78d06c4a 100644 --- a/bsd/conf/param.c +++ b/bsd/conf/param.c @@ -82,15 +82,24 @@ struct timezone tz = { 0, 0 }; +#if CONFIG_EMBEDDED +#define NPROC 1000 /* Account for TOTAL_CORPSES_ALLOWED by making this slightly lower than we can. */ +#define NPROC_PER_UID 950 +#else #define NPROC (20 + 16 * 32) #define NPROC_PER_UID (NPROC/2) +#endif /* NOTE: maxproc and hard_maxproc values are subject to device specific scaling in bsd_scale_setup */ #define HNPROC 2500 /* based on thread_max */ int maxproc = NPROC; int maxprocperuid = NPROC_PER_UID; +#if CONFIG_EMBEDDED +int hard_maxproc = NPROC; /* hardcoded limit -- for embedded the number of processes is limited by the ASID space */ +#else int hard_maxproc = HNPROC; /* hardcoded limit */ +#endif int nprocs = 0; /* XXX */ diff --git a/bsd/dev/arm/conf.c b/bsd/dev/arm/conf.c new file mode 100644 index 000000000..ef78baad1 --- /dev/null +++ b/bsd/dev/arm/conf.c @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1997 by Apple Computer, Inc., all rights reserved + * Copyright (c) 1993 NeXT Computer, Inc. + * + * UNIX Device switch tables. + * + * HISTORY + * + * 30 July 1997 Umesh Vaishampayan (umeshv@apple.com) + * enabled file descriptor pseudo-device. + * 18 June 1993 ? at NeXT + * Cleaned up a lot of stuff in this file. + */ + +#include +#include +#include +#include +#include + +/* Prototypes that should be elsewhere: */ +extern dev_t chrtoblk(dev_t dev); +extern int chrtoblk_set(int cdev, int bdev); + +struct bdevsw bdevsw[] = +{ + /* + * For block devices, every other block of 8 slots is + * reserved to NeXT. The other slots are available for + * the user. This way we can both add new entries without + * running into each other. Be sure to fill in NeXT's + * 8 reserved slots when you jump over us -- we'll do the + * same for you. + */ + + /* 0 - 7 are reserved to NeXT */ + + NO_BDEVICE, /* 0 */ + NO_BDEVICE, /* 1 */ + NO_BDEVICE, /* 2 */ + NO_BDEVICE, /* 3 */ + NO_BDEVICE, /* 4 */ + NO_BDEVICE, /* 5 */ + NO_BDEVICE, /* 6 */ + NO_BDEVICE, /* 7 */ + + /* 8 - 15 are reserved to the user */ + NO_BDEVICE, /* 8 */ + NO_BDEVICE, /* 9 */ + NO_BDEVICE, /* 10 */ + NO_BDEVICE, /* 11 */ + NO_BDEVICE, /* 12 */ + NO_BDEVICE, /* 13 */ + NO_BDEVICE, /* 14 */ + NO_BDEVICE, /* 15 */ + + /* 16 - 23 are reserved to NeXT */ + NO_BDEVICE, /* 16 */ + NO_BDEVICE, /* 17 */ + NO_BDEVICE, /* 18 */ + NO_BDEVICE, /* 18 */ + NO_BDEVICE, /* 20 */ + NO_BDEVICE, /* 21 */ + NO_BDEVICE, /* 22 */ + NO_BDEVICE, /* 23 */ +}; + +const int nblkdev = sizeof(bdevsw) / sizeof(bdevsw[0]); + +extern struct tty *km_tty[]; +extern d_open_t cnopen; +extern d_close_t cnclose; +extern d_read_t cnread; +extern d_write_t cnwrite; +extern d_ioctl_t cnioctl; +extern d_select_t cnselect; +extern d_open_t kmopen; +extern d_close_t kmclose; +extern d_read_t kmread; +extern d_write_t kmwrite; +extern d_ioctl_t kmioctl; +extern d_open_t sgopen; +extern d_close_t sgclose; +extern d_ioctl_t sgioctl; + +#if NVOL > 0 +extern d_open_t volopen; +extern d_close_t volclose; +extern d_ioctl_t volioctl; +#else +#define volopen eno_opcl +#define volclose eno_opcl +#define volioctl eno_ioctl +#endif + +extern d_open_t cttyopen; +extern d_read_t cttyread; +extern d_write_t cttywrite; +extern d_ioctl_t cttyioctl; +extern d_select_t cttyselect; + +extern d_read_t mmread; +extern d_write_t mmwrite; +extern d_ioctl_t mmioctl; +#define mmselect (select_fcn_t *)seltrue +#define mmmmap eno_mmap + +#include +#if NPTY > 0 +extern d_open_t ptsopen; +extern d_close_t ptsclose; +extern d_read_t ptsread; +extern d_write_t ptswrite; +extern d_stop_t ptsstop; +extern d_select_t ptsselect; +extern d_open_t ptcopen; +extern d_close_t ptcclose; +extern d_read_t ptcread; +extern d_write_t ptcwrite; +extern d_select_t ptcselect; +extern d_ioctl_t ptyioctl; +#else +#define ptsopen eno_opcl +#define ptsclose eno_opcl +#define ptsread eno_rdwrt +#define ptswrite eno_rdwrt +#define ptsstop nulldev + +#define ptcopen eno_opcl +#define ptcclose eno_opcl +#define ptcread eno_rdwrt +#define ptcwrite eno_rdwrt +#define ptcselect eno_select +#define ptyioctl eno_ioctl +#endif + +extern d_open_t logopen; +extern d_close_t logclose; +extern d_read_t logread; +extern d_ioctl_t logioctl; +extern d_select_t logselect; + +extern d_open_t oslog_streamopen; +extern d_close_t oslog_streamclose; +extern d_read_t oslog_streamread; +extern d_ioctl_t oslog_streamioctl; +extern d_select_t oslog_streamselect; + +extern d_open_t oslogopen; +extern d_close_t oslogclose; +extern d_select_t oslogselect; +extern d_ioctl_t oslogioctl; + +#define nullopen (d_open_t *)&nulldev +#define nullclose (d_close_t *)&nulldev +#define nullread (d_read_t *)&nulldev +#define nullwrite (d_write_t *)&nulldev +#define nullioctl (d_ioctl_t *)&nulldev +#define nullselect (d_select_t *)&nulldev +#define nullstop (d_stop_t *)&nulldev +#define nullreset (d_reset_t *)&nulldev + +struct cdevsw cdevsw[] = { + /* + * To add character devices to this table dynamically, use cdevsw_add. + */ + + [0] = { + cnopen, cnclose, cnread, cnwrite, + cnioctl, nullstop, nullreset, 0, cnselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [1] = NO_CDEVICE, + [2] = { + cttyopen, nullclose, cttyread, cttywrite, + cttyioctl, nullstop, nullreset, 0, cttyselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [3] = { + nullopen, nullclose, mmread, mmwrite, + mmioctl, nullstop, nullreset, 0, mmselect, + mmmmap, eno_strat, eno_getc, eno_putc, D_DISK + }, + [PTC_MAJOR] = { + ptsopen, ptsclose, ptsread, ptswrite, + ptyioctl, ptsstop, nullreset, 0, ptsselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [PTS_MAJOR] = { + ptcopen, ptcclose, ptcread, ptcwrite, + ptyioctl, nullstop, nullreset, 0, ptcselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [6] = { + logopen, logclose, logread, eno_rdwrt, + logioctl, eno_stop, nullreset, 0, logselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [7] = { + oslogopen, oslogclose, eno_rdwrt, eno_rdwrt, + oslogioctl, eno_stop, nullreset, 0, oslogselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [8] = { + oslog_streamopen, oslog_streamclose, oslog_streamread, eno_rdwrt, + oslog_streamioctl, eno_stop, nullreset, 0, oslog_streamselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [9 ... 11] = NO_CDEVICE, + [12] = { + kmopen, kmclose, kmread, kmwrite, + kmioctl, nullstop, nullreset, km_tty, ttselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [13 ... 41] = NO_CDEVICE, + [42] = { + volopen, volclose, eno_rdwrt, eno_rdwrt, + volioctl, eno_stop, eno_reset, 0, (select_fcn_t *) seltrue, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + } +}; +const int nchrdev = sizeof(cdevsw) / sizeof(cdevsw[0]); + +uint64_t cdevsw_flags[sizeof(cdevsw) / sizeof(cdevsw[0])]; + +#include /* for VCHR and VBLK */ +/* + * return true if a disk + */ +int +isdisk(dev_t dev, int type) +{ + dev_t maj = major(dev); + + switch (type) { + case VCHR: + maj = chrtoblk(maj); + if (maj == NODEV) { + break; + } + /* FALL THROUGH */ + case VBLK: + if (bdevsw[maj].d_type == D_DISK) { + return (1); + } + break; + } + return (0); +} + +static int chrtoblktab[] = { + /* CHR *//* BLK *//* CHR *//* BLK */ + /* 0 */ NODEV, /* 1 */ NODEV, + /* 2 */ NODEV, /* 3 */ NODEV, + /* 4 */ NODEV, /* 5 */ NODEV, + /* 6 */ NODEV, /* 7 */ NODEV, + /* 8 */ NODEV, /* 9 */ NODEV, + /* 10 */ NODEV, /* 11 */ NODEV, + /* 12 */ NODEV, /* 13 */ NODEV, + /* 14 */ NODEV, /* 15 */ NODEV, + /* 16 */ NODEV, /* 17 */ NODEV, + /* 18 */ NODEV, /* 19 */ NODEV, + /* 20 */ NODEV, /* 21 */ NODEV, + /* 22 */ NODEV, /* 23 */ NODEV, + /* 24 */ NODEV, /* 25 */ NODEV, + /* 26 */ NODEV, /* 27 */ NODEV, + /* 28 */ NODEV, /* 29 */ NODEV, + /* 30 */ NODEV, /* 31 */ NODEV, + /* 32 */ NODEV, /* 33 */ NODEV, + /* 34 */ NODEV, /* 35 */ NODEV, + /* 36 */ NODEV, /* 37 */ NODEV, + /* 38 */ NODEV, /* 39 */ NODEV, + /* 40 */ NODEV, /* 41 */ NODEV, + /* 42 */ NODEV, /* 43 */ NODEV, + /* 44 */ NODEV, +}; + +/* + * convert chr dev to blk dev + */ +dev_t +chrtoblk(dev_t dev) +{ + int blkmaj; + + if (major(dev) >= nchrdev) + return (NODEV); + blkmaj = chrtoblktab[major(dev)]; + if (blkmaj == NODEV) + return (NODEV); + return (makedev(blkmaj, minor(dev))); +} + +int +chrtoblk_set(int cdev, int bdev) +{ + if (cdev >= nchrdev) + return (-1); + if (bdev != NODEV && bdev >= nblkdev) + return (-1); + chrtoblktab[cdev] = bdev; + return 0; +} diff --git a/bsd/dev/arm/cons.c b/bsd/dev/arm/cons.c new file mode 100644 index 000000000..910c15257 --- /dev/null +++ b/bsd/dev/arm/cons.c @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1987, 1988 NeXT, Inc. + * + * HISTORY 7-Jan-93 Mac Gillon (mgillon) at NeXT Integrated POSIX support + * + * 12-Aug-87 John Seamons (jks) at NeXT Ported to NeXT. + */ + +/* + * Indirect driver for console. + */ +#include +#include +#include +#include +#include +#include +#include + +struct tty *constty; /* current console device */ + +/* + * The km driver supplied the default console device for the systems + * (usually a raw frame buffer driver, but potentially a serial driver). + */ +extern struct tty *km_tty[1]; + +/* + * cdevsw[] entries for the console device driver + */ +int cnopen(__unused dev_t dev, int flag, int devtype, proc_t pp); +int cnclose(__unused dev_t dev, int flag, int mode, proc_t pp); +int cnread(__unused dev_t dev, struct uio *uio, int ioflag); +int cnwrite(__unused dev_t dev, struct uio *uio, int ioflag); +int cnioctl(__unused dev_t dev, u_long cmd, caddr_t addr, int flg, proc_t p); +int cnselect(__unused dev_t dev, int flag, void * wql, proc_t p); + +static dev_t +cndev(void) +{ + if (constty) + return constty->t_dev; + else + return km_tty[0]->t_dev; +} + +int +cnopen(__unused dev_t dev, int flag, int devtype, struct proc *pp) +{ + dev = cndev(); + return ((*cdevsw[major(dev)].d_open)(dev, flag, devtype, pp)); +} + + +int +cnclose(__unused dev_t dev, int flag, int mode, struct proc *pp) +{ + dev = cndev(); + return ((*cdevsw[major(dev)].d_close)(dev, flag, mode, pp)); +} + + +int +cnread(__unused dev_t dev, struct uio *uio, int ioflag) +{ + dev = cndev(); + return ((*cdevsw[major(dev)].d_read)(dev, uio, ioflag)); +} + + +int +cnwrite(__unused dev_t dev, struct uio *uio, int ioflag) +{ + dev = cndev(); + return ((*cdevsw[major(dev)].d_write)(dev, uio, ioflag)); +} + + +int +cnioctl(__unused dev_t dev, u_long cmd, caddr_t addr, int flag, struct proc *p) +{ + dev = cndev(); + + /* + * XXX This check prevents the cons.c code from being shared between + * XXX all architectures; it is probably not needed on ARM, either, + * XXX but I have no test platforms or ability to run a kernel. + * + * Superuser can always use this to wrest control of console + * output from the "virtual" console. + */ + if ((unsigned) cmd == TIOCCONS && constty) { + int error = proc_suser(p); + if (error) + return (error); + constty = NULL; + return (0); + } + return ((*cdevsw[major(dev)].d_ioctl)(dev, cmd, addr, flag, p)); +} + + +int +cnselect(__unused dev_t dev, int flag, void *wql, struct proc *p) +{ + dev = cndev(); + return ((*cdevsw[major(dev)].d_select)(dev, flag, wql, p)); +} diff --git a/bsd/dev/arm/cpu_in_cksum.s b/bsd/dev/arm/cpu_in_cksum.s new file mode 100644 index 000000000..28f648183 --- /dev/null +++ b/bsd/dev/arm/cpu_in_cksum.s @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2009-2017 Apple Inc. All rights reserved. + * + * This document is the property of Apple Inc. + * It is considered confidential and proprietary. + * + * This document may not be reproduced or transmitted in any form, + * in whole or in part, without the express written permission of + * Apple Inc. + */ + +/* $NetBSD: cpu_in_cksum.S,v 1.2 2008/01/27 16:58:05 chris Exp $ */ + +/* + * Copyright 2003 Wasabi Systems, Inc. + * All rights reserved. + * + * Written by Steve C. Woodford for Wasabi Systems, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed for the NetBSD Project by + * Wasabi Systems, Inc. + * 4. The name of Wasabi Systems, Inc. may not be used to endorse + * or promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef KERNEL +#include "../../../osfmk/arm/arch.h" +#include "../../../osfmk/arm/proc_reg.h" + +#if __ARM_VFP__ < 3 +#error "Unsupported: __ARM_VFP__ < 3" +#endif /* __ARM_VFP__ < 3 */ +#define CKSUM_ERR _kprintf +#else /* !KERNEL */ +#ifndef LIBSYSCALL_INTERFACE +#error "LIBSYSCALL_INTERFACE not defined" +#endif /* !LIBSYSCALL_INTERFACE */ +#define CKSUM_ERR _fprintf_stderr +#define __ARM_VFP__ 3 +#endif /* !KERNEL */ + +/* + * The following default the implementation to little-endian architectures. + */ +#define LITTLE_ENDIAN 1 +#define BYTE_ORDER LITTLE_ENDIAN + +.syntax unified + +/* + * XXX: adi@apple.com: + * + * Ugly, but we have little choice, since relying on genassym and + * is not possible unless this code lives in osfmk. Note also that this + * routine expects "mbuf-like" argument, and it does not expect the mbuf to be + * authentic; it only cares about 3 fields. + */ +#define M_NEXT 0 +#define M_DATA 8 +#define M_LEN 12 + +/* + * APPLE MODIFICATION + * + * The use of R7 in this code as data register prevents + * the use of debugging or instrumentation tools, which is an acceptable + * tradeoff considering the potential gain in performance. + */ + +/* + * Hand-optimised implementations for ARM/Xscale + */ + + .macro EnableVFP +#ifdef KERNEL + push {r0, r1, r2, r12} + bl _enable_kernel_vfp_context + pop {r0, r1, r2, r12} +#endif /* KERNEL */ + .endm + + +/* + * uint32_t os_cpu_in_cksum_mbuf(struct mbuf *m, int len, int off, + * uint32_t initial_sum); + * + * Entry: + * r0 m + * r1 len + * r2 off + * r3 initial_sum + * + * Function wide register usage + * r8 accumulated sum + * r9 remaining length to parse + * ip pointer to next mbuf + * + * This function returns the partial 16-bit checksum accumulated in + * a 32-bit variable (withouth 1's complement); caller is responsible + * for folding the 32-bit sum into 16-bit and performinng the 1's + * complement if applicable + */ + .globl _os_cpu_in_cksum_mbuf + .text + .align 4 +_os_cpu_in_cksum_mbuf: + stmfd sp!, {r4-r11,lr} + + mov r8, r3 /* Accumulate sum in r8 */ + mov r9, r1 /* save len in r9 */ + mov ip, r0 /* set ip to the current mbuf */ + + cmp r9, #0 /* length is 0? */ + bne .Lin_cksum_skip_loop /* if not, proceed further */ + mov r0, r8 /* otherwise, return initial sum */ + + ldmfd sp!, {r4-r11, pc} + +.Lin_cksum_skip_loop: + ldr r1, [ip, #(M_LEN)] + ldr r0, [ip, #(M_DATA)] + ldr ip, [ip, #(M_NEXT)] +.Lin_cksum_skip_entry: + subs r2, r2, r1 /* offset = offset - mbuf length */ + blt .Lin_cksum_skip_done /* if offset has gone negative start with this mbuf */ + cmp ip, #0x00 + bne .Lin_cksum_skip_loop + b .Lin_cksum_whoops + +.Lin_cksum_skip_done: + add r0, r2, r0 /* data += offset (offset is < 0) */ + add r0, r0, r1 /* data += length of mbuf */ + /* data == start of data to cksum */ + rsb r1, r2, #0x00 /* length = remainder of mbuf to read */ + mov r10, #0x00 + b .Lin_cksum_entry + +.Lin_cksum_loop: + ldr r1, [ip, #(M_LEN)] + ldr r0, [ip, #(M_DATA)] + ldr ip, [ip, #(M_NEXT)] +.Lin_cksum_entry: + cmp r9, r1 + movlt r1, r9 + sub r9, r9, r1 + eor r11, r10, r0 + add r10, r10, r1 + adds r2, r1, #0x00 + + beq .Lin_cksum_next + +/* + * APPLE MODIFICATION + * + * Replace the 'blne _ASM_LABEL(L_cksumdata)' by bringing the called function + * inline. This results in slightly faster code, and also permits the whole + * function to be included in kernel profiling data. + */ + +/* + * The main in*_cksum() workhorse... + * + * Entry parameters: + * r0 Pointer to buffer + * r1 Buffer length + * lr Return address + * + * Returns: + * r2 Accumulated 32-bit sum + * + * Clobbers: + * r0-r7 + */ + mov r2, #0 + + /* We first have to word-align the buffer. */ + ands r7, r0, #0x03 + beq .Lcksumdata_wordaligned + rsb r7, r7, #0x04 + cmp r1, r7 /* Enough bytes left to make it? */ + blt .Lcksumdata_endgame + cmp r7, #0x02 + ldrb r4, [r0], #0x01 /* Fetch 1st byte */ + ldrbge r5, [r0], #0x01 /* Fetch 2nd byte */ + movlt r5, #0x00 + ldrbgt r6, [r0], #0x01 /* Fetch 3rd byte */ + movle r6, #0x00 + /* Combine the three bytes depending on endianness and alignment */ +#if BYTE_ORDER != LITTLE_ENDIAN + orreq r2, r5, r4, lsl #8 + orreq r2, r2, r6, lsl #24 + orrne r2, r4, r5, lsl #8 + orrne r2, r2, r6, lsl #16 +#else + orreq r2, r4, r5, lsl #8 + orreq r2, r2, r6, lsl #16 + orrne r2, r5, r4, lsl #8 + orrne r2, r2, r6, lsl #24 +#endif + subs r1, r1, r7 /* Update length */ + beq .Lin_cksum_next /* All done? */ + + /* Buffer is now word aligned */ +.Lcksumdata_wordaligned: + +#if __ARM_VFP__ >= 3 + + cmp r1, #512 // do this if r1 is at least 512 + blt 9f + + EnableVFP + + and r3, r1, #~0x3f + + vpush {q0-q7} + + // move r2 to s16 (q4) for neon computation + veor q4, q4, q4 + vld1.32 {q0-q1}, [r0]! + vmov s16, r2 + vld1.32 {q2-q3}, [r0]! + + // pre-decrement size by 64 + subs r3, r3, #0x80 + + vpadal.u32 q4, q0 + vld1.32 {q0}, [r0]! + vpaddl.u32 q5, q1 + vld1.32 {q1}, [r0]! + vpaddl.u32 q6, q2 + vld1.32 {q2}, [r0]! + vpaddl.u32 q7, q3 + vld1.32 {q3}, [r0]! + +0: + subs r3, r3, #0x40 // decrement size by 64 + + vpadal.u32 q4, q0 + vld1.32 {q0}, [r0]! + vpadal.u32 q5, q1 + vld1.32 {q1}, [r0]! + vpadal.u32 q6, q2 + vld1.32 {q2}, [r0]! + vpadal.u32 q7, q3 + vld1.32 {q3}, [r0]! + + bgt 0b + + vpadal.u32 q4, q0 + vpadal.u32 q5, q1 + vpadal.u32 q6, q2 + vpadal.u32 q7, q3 + + vpadal.u32 q4, q5 + vpadal.u32 q6, q7 + vpadal.u32 q4, q6 + vadd.i64 d8, d9 + + vpaddl.u32 d8, d8 + vpaddl.u32 d8, d8 + vpaddl.u32 d8, d8 + + vmov r2, s16 + + vpop {q0-q7} + + ands r1, r1, #0x3f // residual bytes + beq .Lin_cksum_next + +9: + +#endif /* __ARM_VFP__ >= 3 */ + + subs r1, r1, #0x40 + blt .Lcksumdata_bigloop_end + +.Lcksumdata_bigloop: + ldmia r0!, {r3, r4, r5, r6} + adds r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r7} + adcs r2, r2, r6 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r6} + adcs r2, r2, r7 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r7} + adcs r2, r2, r6 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + adcs r2, r2, r7 + adc r2, r2, #0x00 + subs r1, r1, #0x40 + bge .Lcksumdata_bigloop +.Lcksumdata_bigloop_end: + + adds r1, r1, #0x40 + beq .Lin_cksum_next + + cmp r1, #0x20 + + blt .Lcksumdata_less_than_32 + ldmia r0!, {r3, r4, r5, r6} + adds r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r7} + adcs r2, r2, r6 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + adcs r2, r2, r7 + adc r2, r2, #0x00 + subs r1, r1, #0x20 + beq .Lin_cksum_next + +.Lcksumdata_less_than_32: + /* There are less than 32 bytes left */ + and r3, r1, #0x18 + rsb r4, r3, #0x18 + sub r1, r1, r3 + adds r4, r4, r4, lsr #1 /* Side effect: Clear carry flag */ + addne pc, pc, r4 + +/* + * Note: We use ldm here, even on Xscale, since the combined issue/result + * latencies for ldm and ldrd are the same. Using ldm avoids needless #ifdefs. + */ + /* At least 24 bytes remaining... */ + ldmia r0!, {r4, r5} + nop + adcs r2, r2, r4 + adcs r2, r2, r5 + + /* At least 16 bytes remaining... */ + ldmia r0!, {r4, r5} + adcs r2, r2, r4 + adcs r2, r2, r5 + + /* At least 8 bytes remaining... */ + ldmia r0!, {r4, r5} + adcs r2, r2, r4 + adcs r2, r2, r5 + + /* Less than 8 bytes remaining... */ + adc r2, r2, #0x00 + subs r1, r1, #0x04 + blt .Lcksumdata_lessthan4 + + ldr r4, [r0], #0x04 + sub r1, r1, #0x04 + adds r2, r2, r4 + adc r2, r2, #0x00 + + /* Deal with < 4 bytes remaining */ +.Lcksumdata_lessthan4: + adds r1, r1, #0x04 + beq .Lin_cksum_next + + /* Deal with 1 to 3 remaining bytes, possibly misaligned */ +.Lcksumdata_endgame: + ldrb r3, [r0] /* Fetch first byte */ + cmp r1, #0x02 + ldrbge r4, [r0, #0x01] /* Fetch 2nd and 3rd as necessary */ + movlt r4, #0x00 + ldrbgt r5, [r0, #0x02] + movle r5, #0x00 + /* Combine the three bytes depending on endianness and alignment */ + tst r0, #0x01 +#if BYTE_ORDER != LITTLE_ENDIAN + orreq r3, r4, r3, lsl #8 + orreq r3, r3, r5, lsl #24 + orrne r3, r3, r4, lsl #8 + orrne r3, r3, r5, lsl #16 +#else + orreq r3, r3, r4, lsl #8 + orreq r3, r3, r5, lsl #16 + orrne r3, r4, r3, lsl #8 + orrne r3, r3, r5, lsl #24 +#endif + adds r2, r2, r3 + adc r2, r2, #0x00 + +.Lin_cksum_next: + tst r11, #0x01 + movne r2, r2, ror #8 + adds r8, r8, r2 + adc r8, r8, #0x00 + cmp ip, #00 + bne .Lin_cksum_loop + + mov r1, #0xff + orr r1, r1, #0xff00 + and r0, r8, r1 + add r0, r0, r8, lsr #16 + add r0, r0, r0, lsr #16 + and r0, r0, r1 + /* + * If we were to 1's complement it (XOR with 0xffff): + * + * eor r0, r0, r1 + */ + + ldmfd sp!, {r4-r11, pc} + +.Lin_cksum_whoops: + adr r0, .Lin_cksum_whoops_str + bl #CKSUM_ERR + mov r0, #-1 + + ldmfd sp!, {r4-r11, pc} + +.Lin_cksum_whoops_str: + .asciz "os_cpu_in_cksum_mbuf: out of data\n" + .align 5 diff --git a/bsd/dev/arm/disassembler.c b/bsd/dev/arm/disassembler.c new file mode 100644 index 000000000..a5db2033e --- /dev/null +++ b/bsd/dev/arm/disassembler.c @@ -0,0 +1,1097 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * Disassemblers for ARM (arm), Thumb (thumb16), and Thumb2 (thumb32). + * + * Each disassembly begins with a call to dtrace_decode_arm or dtrace_decode_thumb. The thumb + * decoder will then call dtrace_decode_thumb16 or dtrace_decode_thumb32 as appropriate. + * + * The respective disassembly functions are all of the form {arm,thumb16,thumb32}_type. They + * follow the ordering and breakdown in the ARMv7 Architecture Reference Manual. + */ + +#include + +#define BITS(x,n,mask) (((x) >> (n)) & (mask)) + +static uint32_t thumb32_instword_to_arm(uint16_t hw1, uint16_t hw2) +{ + return (hw1 << 16) | hw2; +} + +int dtrace_decode_arm(uint32_t instr); +int dtrace_decode_thumb(uint32_t instr); + +/* + * VFP decoder - shared between ARM and THUMB32 mode + */ + +static +int vfp_struct_loadstore(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_64transfer(uint32_t instr) +{ + /* These instructions all use RD and RN */ + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_transfer(uint32_t instr) +{ + /* These instructions all use RD only */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_loadstore(uint32_t instr) +{ + int opcode = BITS(instr,20,0x1F); + + /* Instrument VLDR */ + if ((opcode & 0x13) == 0x11 && ARM_RN(instr) == REG_PC) + return FASTTRAP_T_VLDR_PC_IMMED; + + /* These instructions all use RN only */ + if (ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +/* + * ARM decoder + */ + +static +int arm_unconditional_misc(uint32_t instr) +{ + int op = BITS(instr,20,0x7F); + + if ((op & 0x60) == 0x20) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } + + if ((op & 0x71) == 0x40) { + return vfp_struct_loadstore(instr); + } + + return FASTTRAP_T_INV; +} + +static +int arm_unconditional(uint32_t instr) +{ + if (BITS(instr,27,0x1) == 0) + return arm_unconditional_misc(instr); + + /* The rest are privileged or BL/BLX, do not instrument */ + + /* Do not need to instrument BL/BLX either, see comment in arm_misc(uint32_t) */ + + return FASTTRAP_T_INV; +} + +static +int arm_syscall_coproc(uint32_t instr) +{ + /* Instrument any VFP data processing instructions, ignore the rest */ + + int op1 = BITS(instr,20,0x3F), coproc = BITS(instr,8,0xF), op = BITS(instr,4,0x1); + + if ((op1 & 0x3E) == 0 || (op1 & 0x30) == 0x30) { + /* Undefined or swi */ + return FASTTRAP_T_INV; + } + + if ((coproc & 0xE) == 0xA) { + /* VFP instruction */ + + if ((op1 & 0x20) == 0 && (op1 & 0x3A) != 0) + return vfp_loadstore(instr); + + if ((op1 & 0x3E) == 0x04) + return vfp_64transfer(instr); + + if ((op1 & 0x30) == 0x20) { + /* VFP data processing or 8, 16, or 32 bit move between ARM reg and VFP reg */ + if (op == 0) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } else { + return vfp_transfer(instr); + } + } + } + + return FASTTRAP_T_INV; +} + +static +int arm_branch_link_blockdata(uint32_t instr) +{ + int branch = BITS(instr,25,0x1), link = BITS(instr,24,0x1), op = BITS(instr,20,0x1F), uses_pc = BITS(instr,15,0x1), uses_lr = BITS(instr,14,0x1); + + if (branch == 1) { + if (link == 0) + return FASTTRAP_T_B_COND; + return FASTTRAP_T_INV; + } else { + /* Only emulate a use of the pc if it's a return from function: ldmia sp!, { ... pc } */ + if (op == 0x0B && ARM_RN(instr) == REG_SP && uses_pc == 1) + return FASTTRAP_T_LDM_PC; + + /* stmia sp!, { ... lr } doesn't touch the pc, but it is very common, so special case it */ + if (op == 0x12 && ARM_RN(instr) == REG_SP && uses_lr == 1) + return FASTTRAP_T_STM_LR; + + if (ARM_RN(instr) != REG_PC && uses_pc == 0) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_signed_multiplies(uint32_t instr) +{ + int op1 = BITS(instr,20,0x7), op2 = BITS(instr,5,0x7); + + /* smlald, smlsld, smmls use RD in addition to RM, RS, and RN */ + if ((op1 == 0x4 && (op2 & 0x4) == 0) || (op1 == 0x5 && (op2 & 0x6) == 0x6)) { + if (ARM_RD(instr) == REG_PC) + return FASTTRAP_T_INV; + } + + if (ARM_RM(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_pack_unpack_sat_reversal(uint32_t instr) +{ + int op1 = BITS(instr,20,0x7), op2 = BITS(instr,5,0x7); + + /* pkh, sel use RN in addition to RD and RM */ + if ((op1 == 0 && (op2 & 0x1) == 0) || (op1 == 0 && op2 == 0x5)) { + if (ARM_RN(instr) == REG_PC) + return FASTTRAP_T_INV; + } + + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_parallel_addsub_unsigned(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_parallel_addsub_signed(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_media(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F), op2 = BITS(instr,5,0x7); + + if ((op1 & 0x1C) == 0) + return arm_parallel_addsub_signed(instr); + + if ((op1 & 0x1C) == 0x04) + return arm_parallel_addsub_unsigned(instr); + + if ((op1 & 0x18) == 0x08) + return arm_pack_unpack_sat_reversal(instr); + + if ((op1 & 0x18) == 0x10) + return arm_signed_multiplies(instr); + + if (op1 == 0x1F && op2 == 0x7) { + /* Undefined instruction */ + return FASTTRAP_T_INV; + } + + if (op1 == 0x18 && op2 == 0) { + /* usad8 usada8 */ + /* The registers are named differently in the reference manual for this instruction + * but the following positions are correct */ + + if (ARM_RM(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if ((op1 & 0x1E) == 0x1C && (op2 & 0x3) == 0) { + /* bfc bfi */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if (((op1 & 0x1E) == 0x1A || (op1 & 0x1E) == 0x1E) && ((op2 & 0x3) == 0x2)) { + /* sbfx ubfx */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + return FASTTRAP_T_INV; +} + +static +int arm_loadstore_wordbyte(uint32_t instr) +{ + /* Instrument PC relative load with immediate, ignore any other uses of the PC */ + int R = BITS(instr,25,0x1), L = BITS(instr,20,0x1); + + if (R == 1) { + /* Three register load/store */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Immediate load/store, but still do not support ldr pc, [pc...] */ + if (L == 1 && ARM_RN(instr) == REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_LDR_PC_IMMED; + + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_saturating(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_misc(uint32_t instr) +{ + int op = BITS(instr,21,0x3), __unused op1 = BITS(instr,16,0xF), op2 = BITS(instr,4,0x7); + + if (op2 == 1 && op == 1) + return FASTTRAP_T_BX_REG; + + /* We do not need to emulate BLX for entry/return probes; if we eventually support full offset + * tracing, then we will. This is because BLX overwrites the link register, so a function that + * can execute this as its first instruction is a special function indeed. + */ + + if (op2 == 0x5) + return arm_saturating(instr); + + return FASTTRAP_T_INV; +} + +static +int arm_msr_hints(__unused uint32_t instr) +{ + /* These deal with the psr, not instrumented */ + + return FASTTRAP_T_INV; +} + +static +int arm_sync_primitive(__unused uint32_t instr) +{ + /* TODO will instrumenting these interfere with any kernel usage of these instructions? */ + /* Don't instrument for now */ + + return FASTTRAP_T_INV; +} + +static +int arm_extra_loadstore_unpriv(uint32_t instr) +{ + int op = BITS(instr,20,0x1), __unused op2 = BITS(instr,5,0x3), immed = BITS(instr,22,0x1); + + if (op == 0 && (op2 & 0x2) == 0x2) { + /* Unpredictable or undefined */ + return FASTTRAP_T_INV; + } + + if (immed == 1) { + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_extra_loadstore(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F); + + /* There are two variants, and we do not instrument either of them that use the PC */ + + if ((op1 & 0x4) == 0) { + /* Variant 1, register */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Variant 2, immediate */ + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_halfword_multiply(uint32_t instr) +{ + /* Not all multiply instructions use all four registers. The ones that don't should have those + * register locations set to 0, so we can test them anyway. + */ + + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_multiply(uint32_t instr) +{ + /* Not all multiply instructions use all four registers. The ones that don't should have those + * register locations set to 0, so we can test them anyway. + */ + + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_immed(uint32_t instr) +{ + /* All these instructions are either two registers, or one register and have 0 where the other reg would be used */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_regshift(uint32_t instr) +{ + /* All these instructions are either four registers, or three registers and have 0 where there last reg would be used */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_reg(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F), op2 = BITS(instr,7,0x1F), op3 = BITS(instr,5,0x3); + + if (op1 == 0x11 || op1 == 0x13 || op1 == 0x15 || op1 == 0x17) { + /* These are comparison flag setting instructions and do not have RD */ + if (ARM_RN(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + /* The rest can, in theory, write or use the PC. The only one we instrument is mov pc, reg. + * movs pc, reg is a privileged instruction so we don't instrument that variant. The s bit + * is bit 0 of op1 and should be zero. + */ + if (op1 == 0x1A && op2 == 0 && op3 == 0 && ARM_RD(instr) == REG_PC) + return FASTTRAP_T_MOV_PC_REG; + + /* Any instruction at this point is a three register instruction or two register instruction with RN = 0 */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_misc(uint32_t instr) +{ + int op = BITS(instr,25,0x1), op1 = BITS(instr,20,0x1F), op2 = BITS(instr,4,0xF); + + if (op == 0) { + if ((op1 & 0x19) != 0x10 && (op2 & 0x1) == 0) + return arm_dataproc_reg(instr); + + if ((op1 & 0x19) != 0x10 && (op2 & 0x9) == 0x1) + return arm_dataproc_regshift(instr); + + if ((op1 & 0x19) == 0x10 && (op2 & 0x8) == 0) + return arm_misc(instr); + + if ((op1 & 0x19) == 0x19 && (op2 & 0x9) == 0x8) + return arm_halfword_multiply(instr); + + if ((op1 & 0x10) == 0 && op2 == 0x9) + return arm_multiply(instr); + + if ((op1 & 0x10) == 0x10 && op2 == 0x9) + return arm_sync_primitive(instr); + + if ((op1 & 0x12) != 0x02 && (op2 == 0xB || (op2 & 0xD) == 0xD)) + return arm_extra_loadstore(instr); + + if ((op1 & 0x12) == 0x02 && (op2 == 0xB || (op2 & 0xD) == 0xD)) + return arm_extra_loadstore_unpriv(instr); + } else { + if ((op1 & 0x19) != 0x10) + return arm_dataproc_immed(instr); + + if (op1 == 0x10) { + /* 16 bit immediate load (mov (immed)) [encoding A2] */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if (op1 == 0x14) { + /* high halfword 16 bit immediate load (movt) [encoding A1] */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if ((op1 & 0x1B) == 0x12) + return arm_msr_hints(instr); + } + + return FASTTRAP_T_INV; +} + +int dtrace_decode_arm(uint32_t instr) +{ + int cond = BITS(instr,28,0xF), op1 = BITS(instr,25,0x7), op = BITS(instr,4,0x1); + + if (cond == 0xF) + return arm_unconditional(instr); + + if ((op1 & 0x6) == 0) + return arm_dataproc_misc(instr); + + if (op1 == 0x2) + return arm_loadstore_wordbyte(instr); + + if (op1 == 0x3 && op == 0) + return arm_loadstore_wordbyte(instr); + + if (op1 == 0x3 && op == 1) + return arm_media(instr); + + if ((op1 & 0x6) == 0x4) + return arm_branch_link_blockdata(instr); + + if ((op1 & 0x6) == 0x6) + return arm_syscall_coproc(instr); + + return FASTTRAP_T_INV; +} + + +/* + * Thumb 16-bit decoder + */ + +static +int thumb16_cond_supervisor(uint16_t instr) +{ + int opcode = BITS(instr,8,0xF); + + if ((opcode & 0xE) != 0xE) + return FASTTRAP_T_B_COND; + + return FASTTRAP_T_INV; +} + +static +int thumb16_misc(uint16_t instr) +{ + int opcode = BITS(instr,5,0x7F); + + if ((opcode & 0x70) == 0x30 || (opcode & 0x70) == 0x70) { + /* setend, cps, breakpoint, or if-then, not instrumentable */ + return FASTTRAP_T_INV; + } else if ((opcode & 0x78) == 0x28) { + /* Doesn't modify pc, but this happens a lot so make this a special case for emulation */ + return FASTTRAP_T_PUSH_LR; + } else if ((opcode & 0x78) == 0x68) { + return FASTTRAP_T_POP_PC; + } else if ((opcode & 0x28) == 0x08) { + return FASTTRAP_T_CB_N_Z; + } + + /* All other instructions work on low regs only and are instrumentable */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_loadstore_single(__unused uint16_t instr) +{ + /* These all access the low registers or SP only */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_data_special_and_branch(uint16_t instr) +{ + int opcode = BITS(instr,6,0xF); + + if (opcode == 0x4) { + /* Unpredictable */ + return FASTTRAP_T_INV; + } else if ((opcode & 0xC) == 0xC) { + /* bx or blx */ + /* Only instrument the bx */ + if ((opcode & 0x2) == 0) + return FASTTRAP_T_BX_REG; + return FASTTRAP_T_INV; + } else { + /* Data processing on high registers, only instrument mov pc, reg */ + if ((opcode & 0xC) == 0x8 && THUMB16_HRD(instr) == REG_PC) + return FASTTRAP_T_CPY_PC; + + if (THUMB16_HRM(instr) != REG_PC && THUMB16_HRD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb16_data_proc(__unused uint16_t instr) +{ + /* These all access the low registers only */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_shift_addsub_move_compare(__unused uint16_t instr) +{ + /* These all access the low registers only */ + return FASTTRAP_T_COMMON; +} + +static +int dtrace_decode_thumb16(uint16_t instr) +{ + int opcode = BITS(instr,10,0x3F); + + if ((opcode & 0x30) == 0) + return thumb16_shift_addsub_move_compare(instr); + + if (opcode == 0x10) + return thumb16_data_proc(instr); + + if (opcode == 0x11) + return thumb16_data_special_and_branch(instr); + + if ((opcode & 0x3E) == 0x12) { + /* ldr (literal) */ + return FASTTRAP_T_LDR_PC_IMMED; + } + + if ((opcode & 0x3C) == 0x14 || (opcode & 0x38) == 0x18 || (opcode & 0x38) == 0x20) + return thumb16_loadstore_single(instr); + + if ((opcode & 0x3E) == 0x28) { + /* adr, uses the pc */ + return FASTTRAP_T_INV; + } + + if ((opcode & 0x3E) == 0x2A) { + /* add (sp plus immediate) */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3C) == 0x2C) + return thumb16_misc(instr); + + if ((opcode & 0x3E) == 0x30) { + /* stm - can't access high registers */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3E) == 0x32) { + /* ldm - can't access high registers */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3C) == 0x34) { + return thumb16_cond_supervisor(instr); + } + + if ((opcode & 0x3E) == 0x38) { + /* b unconditional */ + return FASTTRAP_T_B_UNCOND; + } + + return FASTTRAP_T_INV; +} + +/* + * Thumb 32-bit decoder + */ + +static +int thumb32_coproc(uint16_t instr1, uint16_t instr2) +{ + /* Instrument any VFP data processing instructions, ignore the rest */ + + int op1 = BITS(instr1,4,0x3F), coproc = BITS(instr2,8,0xF), op = BITS(instr2,4,0x1); + + if ((op1 & 0x3E) == 0) { + /* Undefined */ + return FASTTRAP_T_INV; + } + + if ((coproc & 0xE) == 0xA || (op1 & 0x30) == 0x30) { + /* VFP instruction */ + uint32_t instr = thumb32_instword_to_arm(instr1,instr2); + + if ((op1 & 0x30) == 0x30) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x3A) == 0x02 || (op1 & 0x38) == 0x08 || (op1 & 0x30) == 0x10) + return vfp_loadstore(instr); + + if ((op1 & 0x3E) == 0x04) + return vfp_64transfer(instr); + + if ((op1 & 0x30) == 0x20) { + /* VFP data processing or 8, 16, or 32 bit move between ARM reg and VFP reg */ + if (op == 0) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } else { + return vfp_transfer(instr); + } + } + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_longmultiply(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0x7), op2 = BITS(instr2,4,0xF); + + if ((op1 == 1 && op2 == 0xF) || (op1 == 0x3 && op2 == 0xF)) { + /* Three register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Four register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && + THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_multiply(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0x7), op2 = BITS(instr2,4,0x3); + + if ((op1 == 0 && op2 == 1) || (op1 == 0x6 && (op2 & 0x2) == 0)) { + if (THUMB32_RT(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_misc(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_parallel_addsub_unsigned(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_parallel_addsub_signed(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_reg(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0xF), op2 = BITS(instr2,4,0xF); + + if (((0 <= op1) && (op1 <= 5)) && (op2 & 0x8) == 0x8) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x8) == 0 && op2 == 0) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x8) == 0x8 && (op2 & 0xC) == 0) + return thumb32_parallel_addsub_signed(instr1,instr2); + + if ((op1 & 0x8) == 0x8 && (op2 & 0xC) == 0x4) + return thumb32_parallel_addsub_unsigned(instr1,instr2); + + if ((op1 & 0xC) == 0x8 && (op2 & 0xC) == 0x8) + return thumb32_misc(instr1,instr2); + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_regshift(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,5,0xF), S = BITS(instr1,4,0x1); + + if (op == 0 || op == 0x4 || op == 0x8 || op == 0xD) { + /* These become test instructions if S is 1 and Rd is PC, otherwise they are data instructions. */ + if (S == 1) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && + THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + } else if (op == 0x2 || op == 0x3) { + /* These become moves if RN is PC, otherwise they are data insts. We don't instrument mov pc, reg here */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Normal three register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_store_single(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,5,0x7), op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the pc yet */ + if ((op1 == 0 || op1 == 1 || op1 == 2) && (op2 & 0x20) == 0) { + /* str (register) uses RM */ + if (THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadbyte_memhint(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), __unused op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the pc yet */ + if ((op1 == 0 || op1 == 0x2) && THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadhalfword_memhint(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the PC yet */ + if (op1 == 0 && op2 == 0 && THUMB32_RM(inst1,instr2) == REG_PC) + return FASTTRAP_T_INV; + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadword(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), op2 = BITS(instr2,6,0x3F); + + if ((op1 & 0x2) == 0 && THUMB32_RN(instr1,instr2) == REG_PC && THUMB32_RT(instr1,instr2) != REG_PC) + return FASTTRAP_T_LDR_PC_IMMED; + + if (op1 == 0 && op2 == 0) { + /* ldr (register) uses an additional reg */ + if (THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadstore_double_exclusive_table(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Don't instrument any of these */ + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadstore_multiple(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,7,0x3), L = BITS(instr1,4,0x1), uses_pc = BITS(instr2,15,0x1), uses_lr = BITS(instr2,14,0x1); + + if (op == 0 || op == 0x3) { + /* Privileged instructions: srs, rfe */ + return FASTTRAP_T_INV; + } + + /* Only emulate a use of the pc if it's a return from function: ldmia sp!, { ... pc }, aka pop { ... pc } */ + if (op == 0x1 && L == 1 && THUMB32_RN(instr1,instr2) == REG_SP && uses_pc == 1) + return FASTTRAP_T_LDM_PC; + + /* stmia sp!, { ... lr }, aka push { ... lr } doesn't touch the pc, but it is very common, so special case it */ + if (op == 0x2 && L == 0 && THUMB32_RN(instr1,instr2) == REG_SP && uses_lr == 1) + return FASTTRAP_T_STM_LR; + + if (THUMB32_RN(instr1,instr2) != REG_PC && uses_pc == 0) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_misc_control(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Privileged, and instructions dealing with ThumbEE */ + return FASTTRAP_T_INV; +} + +static +int thumb32_cps_hints(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Privileged */ + return FASTTRAP_T_INV; +} + +static +int thumb32_b_misc_control(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,4,0x7F), op1 = BITS(instr2,12,0x7), __unused op2 = BITS(instr2,8,0xF); + + if ((op1 & 0x5) == 0) { + if ((op & 0x38) != 0x38) + return FASTTRAP_T_B_COND; + + if (op == 0x3A) + return thumb32_cps_hints(instr1,instr2); + + if (op == 0x3B) + return thumb32_misc_control(instr1,instr2); + } + + if ((op1 & 0x5) == 1) + return FASTTRAP_T_B_UNCOND; + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_plain_immed(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,4,0x1F); + + if (op == 0x04 || op == 0x0C || op == 0x16) { + /* mov, movt, bfi, bfc */ + /* These use only RD */ + if (THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_mod_immed(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,5,0xF), S = BITS(instr1,4,0x1); + + if (op == 0x2 || op == 0x3) { + /* These allow REG_PC in RN, but it doesn't mean use the PC! */ + if (THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if (op == 0 || op == 0x4 || op == 0x8 || op == 0xD) { + /* These are test instructions, if the sign bit is set and RD is the PC. */ + if (S && THUMB32_RD(instr1,instr2) == REG_PC) + return FASTTRAP_T_COMMON; + } + + if (THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int dtrace_decode_thumb32(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,11,0x3), op2 = BITS(instr1,4,0x7F), op = BITS(instr2,15,0x1); + + if (op1 == 0x1) { + if ((op2 & 0x64) == 0) + return thumb32_loadstore_multiple(instr1,instr2); + + if ((op2 & 0x64) == 0x04) + return thumb32_loadstore_double_exclusive_table(instr1,instr2); + + if ((op2 & 0x60) == 0x20) + return thumb32_dataproc_regshift(instr1,instr2); + + if ((op2 & 0x40) == 0x40) + return thumb32_coproc(instr1,instr2); + } + + if (op1 == 0x2) { + if ((op2 & 0x20) == 0 && op == 0) + return thumb32_dataproc_mod_immed(instr1,instr2); + + if ((op2 & 0x20) == 0x20 && op == 0) + return thumb32_dataproc_plain_immed(instr1,instr2); + + if (op == 1) + return thumb32_b_misc_control(instr1,instr2); + } + + if (op1 == 0x3) { + if ((op2 & 0x71) == 0) + return thumb32_store_single(instr1,instr2); + + if ((op2 & 0x71) == 0x10) { + return vfp_struct_loadstore(thumb32_instword_to_arm(instr1,instr2)); + } + + if ((op2 & 0x67) == 0x01) + return thumb32_loadbyte_memhint(instr1,instr2); + + if ((op2 & 0x67) == 0x03) + return thumb32_loadhalfword_memhint(instr1,instr2); + + if ((op2 & 0x67) == 0x05) + return thumb32_loadword(instr1,instr2); + + if ((op2 & 0x67) == 0x07) { + /* Undefined instruction */ + return FASTTRAP_T_INV; + } + + if ((op2 & 0x70) == 0x20) + return thumb32_dataproc_reg(instr1,instr2); + + if ((op2 & 0x78) == 0x30) + return thumb32_multiply(instr1,instr2); + + if ((op2 & 0x78) == 0x38) + return thumb32_longmultiply(instr1,instr2); + + if ((op2 & 0x40) == 0x40) + return thumb32_coproc(instr1,instr2); + } + + return FASTTRAP_T_INV; +} + +int dtrace_decode_thumb(uint32_t instr) +{ + uint16_t* pInstr = (uint16_t*) &instr; + uint16_t hw1 = pInstr[0], hw2 = pInstr[1]; + + int size = BITS(hw1,11,0x1F); + + if (size == 0x1D || size == 0x1E || size == 0x1F) + return dtrace_decode_thumb32(hw1,hw2); + else + return dtrace_decode_thumb16(hw1); +} + diff --git a/bsd/dev/arm/dtrace_isa.c b/bsd/dev/arm/dtrace_isa.c new file mode 100644 index 000000000..d38831ba3 --- /dev/null +++ b/bsd/dev/arm/dtrace_isa.c @@ -0,0 +1,631 @@ +/* + * Copyright (c) 2005-2008 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from + * mach/ppc/thread_status.h */ +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for thread_wakeup() */ +#include +#include +#include +#include + +extern struct arm_saved_state *find_kern_regs(thread_t); + +extern dtrace_id_t dtrace_probeid_error; /* special ERROR probe */ +typedef arm_saved_state_t savearea_t; + +extern lck_attr_t *dtrace_lck_attr; +extern lck_grp_t *dtrace_lck_grp; + +int dtrace_arm_condition_true(int condition, int cpsr); + +/* + * Atomicity and synchronization + */ +inline void +dtrace_membar_producer(void) +{ +#if __ARM_SMP__ + __asm__ volatile("dmb ish" : : : "memory"); +#else + __asm__ volatile("nop" : : : "memory"); +#endif +} + +inline void +dtrace_membar_consumer(void) +{ +#if __ARM_SMP__ + __asm__ volatile("dmb ish" : : : "memory"); +#else + __asm__ volatile("nop" : : : "memory"); +#endif +} + +/* + * Interrupt manipulation + * XXX dtrace_getipl() can be called from probe context. + */ +int +dtrace_getipl(void) +{ + /* + * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE + * in osfmk/kern/cpu_data.h + */ + /* return get_interrupt_level(); */ + return (ml_at_interrupt_context() ? 1 : 0); +} + +#if __ARM_SMP__ +/* + * MP coordination + */ + +decl_lck_mtx_data(static, dt_xc_lock); +static uint32_t dt_xc_sync; + +typedef struct xcArg { + processorid_t cpu; + dtrace_xcall_t f; + void *arg; +} xcArg_t; + +static void +xcRemote(void *foo) +{ + xcArg_t *pArg = (xcArg_t *) foo; + + if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) + (pArg->f) (pArg->arg); + + if (hw_atomic_sub(&dt_xc_sync, 1) == 0) + thread_wakeup((event_t) &dt_xc_sync); +} +#endif + +/* + * dtrace_xcall() is not called from probe context. + */ +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg) +{ +#if __ARM_SMP__ + /* Only one dtrace_xcall in flight allowed */ + lck_mtx_lock(&dt_xc_lock); + + xcArg_t xcArg; + + xcArg.cpu = cpu; + xcArg.f = f; + xcArg.arg = arg; + + cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg); + + lck_mtx_unlock(&dt_xc_lock); + return; +#else +#pragma unused(cpu) + /* On uniprocessor systems, the cpu should always be either ourselves or all */ + ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL); + + (*f)(arg); + return; +#endif +} + +/* + * Initialization + */ +void +dtrace_isa_init(void) +{ +#if __ARM_SMP__ + lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr); +#endif + return; +} + +/* + * Runtime and ABI + */ +uint64_t +dtrace_getreg(struct regs * savearea, uint_t reg) +{ + struct arm_saved_state *regs = (struct arm_saved_state *) savearea; + + /* beyond register limit? */ + if (reg > ARM_SAVED_STATE32_COUNT - 1) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + return (uint64_t) ((unsigned int *) (&(regs->r)))[reg]; +} + +#define RETURN_OFFSET 4 + +static int +dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc, + user_addr_t sp) +{ + int ret = 0; + + ASSERT(pcstack == NULL || pcstack_limit > 0); + + while (pc != 0) { + ret++; + if (pcstack != NULL) { + *pcstack++ = (uint64_t) pc; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + } + + if (sp == 0) + break; + + pc = dtrace_fuword32((sp + RETURN_OFFSET)); + sp = dtrace_fuword32(sp); + } + + return (ret); +} + +void +dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit) +{ + thread_t thread = current_thread(); + savearea_t *regs; + user_addr_t pc, sp; + volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags; + int n; + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (thread == NULL) + goto zero; + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + goto zero; + + *pcstack++ = (uint64_t)dtrace_proc_selfpid(); + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = regs->pc; + sp = regs->sp; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t) pc; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = regs->lr; + } + + n = dtrace_getustack_common(pcstack, pcstack_limit, pc, regs->r[7]); + + ASSERT(n >= 0); + ASSERT(n <= pcstack_limit); + + pcstack += n; + pcstack_limit -= n; + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0ULL; +} + +int +dtrace_getustackdepth(void) +{ + thread_t thread = current_thread(); + savearea_t *regs; + user_addr_t pc, sp; + int n = 0; + + if (thread == NULL) + return 0; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) + return (-1); + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + return 0; + + pc = regs->pc; + sp = regs->sp; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + n++; + pc = regs->lr; + } + + /* + * Note that unlike ppc, the arm code does not use + * CPU_DTRACE_USTACK_FP. This is because arm always + * traces from the sp, even in syscall/profile/fbt + * providers. + */ + + n += dtrace_getustack_common(NULL, 0, pc, regs->r[7]); + + return (n); +} + +void +dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit) +{ + /* XXX ARMTODO 64vs32 */ + thread_t thread = current_thread(); + savearea_t *regs; + user_addr_t pc, sp; + + volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags; + +#if 0 + uintptr_t oldcontext; + size_t s1, s2; +#endif + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (thread == NULL) + goto zero; + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + goto zero; + + *pcstack++ = (uint64_t)dtrace_proc_selfpid(); + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = regs->pc; + sp = regs->sp; + +#if 0 /* XXX signal stack crawl */ + oldcontext = lwp->lwp_oldcontext; + + if (p->p_model == DATAMODEL_NATIVE) { + s1 = sizeof(struct frame) + 2 * sizeof(long); + s2 = s1 + sizeof(siginfo_t); + } else { + s1 = sizeof(struct frame32) + 3 * sizeof(int); + s2 = s1 + sizeof(siginfo32_t); + } +#endif + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t) pc; + *fpstack++ = 0; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = dtrace_fuword32(sp); + } + while (pc != 0 && sp != 0) { + *pcstack++ = (uint64_t) pc; + *fpstack++ = sp; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + +#if 0 /* XXX signal stack crawl */ + if (oldcontext == sp + s1 || oldcontext == sp + s2) { + if (p->p_model == DATAMODEL_NATIVE) { + ucontext_t *ucp = (ucontext_t *) oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fulword(&gregs[REG_FP]); + pc = dtrace_fulword(&gregs[REG_PC]); + + oldcontext = dtrace_fulword(&ucp->uc_link); + } else { + ucontext_t *ucp = (ucontext_t *) oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fuword32(&gregs[EBP]); + pc = dtrace_fuword32(&gregs[EIP]); + + oldcontext = dtrace_fuword32(&ucp->uc_link); + } + } else +#endif + { + pc = dtrace_fuword32((sp + RETURN_OFFSET)); + sp = dtrace_fuword32(sp); + } + +#if 0 + /* XXX ARMTODO*/ + /* + * This is totally bogus: if we faulted, we're going to clear + * the fault and break. This is to deal with the apparently + * broken Java stacks on x86. + */ + if (*flags & CPU_DTRACE_FAULT) { + *flags &= ~CPU_DTRACE_FAULT; + break; + } +#endif + } + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0ULL; +} + +void +dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes, + uint32_t * intrpc) +{ + struct frame *fp = (struct frame *) __builtin_frame_address(0); + struct frame *nextfp, *minfp, *stacktop; + int depth = 0; + int on_intr; + int last = 0; + uintptr_t pc; + uintptr_t caller = CPU->cpu_dtrace_caller; + + if ((on_intr = CPU_ON_INTR(CPU)) != 0) + stacktop = (struct frame *) dtrace_get_cpu_int_stack_top(); + else + stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size); + + minfp = fp; + + aframes++; + + if (intrpc != NULL && depth < pcstack_limit) + pcstack[depth++] = (pc_t) intrpc; + + while (depth < pcstack_limit) { + nextfp = *(struct frame **) fp; + pc = *(uintptr_t *) (((uint32_t) fp) + RETURN_OFFSET); + + if (nextfp <= minfp || nextfp >= stacktop) { + if (on_intr) { + /* + * Hop from interrupt stack to thread stack. + */ + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + if (arm_kern_regs) { + nextfp = (struct frame *)arm_kern_regs->r[7]; + + vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread()); + + minfp = (struct frame *)kstack_base; + stacktop = (struct frame *)(kstack_base + kernel_stack_size); + + on_intr = 0; + + if (nextfp <= minfp || nextfp >= stacktop) { + last = 1; + } + } else { + /* + * If this thread was on the interrupt stack, but did not + * take an interrupt (i.e, the idle thread), there is no + * explicit saved state for us to use. + */ + last = 1; + } + } else { + /* + * This is the last frame we can process; indicate + * that we should return after processing this frame. + */ + last = 1; + } + } + if (aframes > 0) { + if (--aframes == 0 && caller != (uintptr_t)NULL) { + /* + * We've just run out of artificial frames, + * and we have a valid caller -- fill it in + * now. + */ + ASSERT(depth < pcstack_limit); + pcstack[depth++] = (pc_t) caller; + caller = (uintptr_t)NULL; + } + } else { + if (depth < pcstack_limit) + pcstack[depth++] = (pc_t) pc; + } + + if (last) { + while (depth < pcstack_limit) + pcstack[depth++] = (pc_t) NULL; + return; + } + fp = nextfp; + minfp = fp; + } +} + +int +dtrace_instr_size(uint32_t instr, int thumb_mode) +{ + if (thumb_mode) { + uint16_t instr16 = *(uint16_t*) &instr; + if (((instr16 >> 11) & 0x1F) > 0x1C) + return 4; + else + return 2; + } else { + return 4; + } +} + +uint64_t +dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) +{ +#pragma unused(arg, aframes, mstate, vstate) +#if 0 + /* XXX ARMTODO */ + uint64_t val; + uintptr_t *fp = (uintptr_t *)__builtin_frame_address(0); + uintptr_t *stack; + uintptr_t pc; + int i; + + for (i = 1; i <= aframes; i++) { + fp = fp[0]; + pc = fp[1]; + + if (dtrace_invop_callsite_pre != NULL + && pc > (uintptr_t)dtrace_invop_callsite_pre + && pc <= (uintptr_t)dtrace_invop_callsite_post) { + /* + * If we pass through the invalid op handler, we will + * use the pointer that it passed to the stack as the + * second argument to dtrace_invop() as the pointer to + * the frame we're hunting for. + */ + + stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ + fp = (struct frame *)stack[1]; /* Grab *second* argument */ + stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + val = (uint64_t)(stack[arg]); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + return val; + } + } + + /* + * Arrive here when provider has called dtrace_probe directly. + */ + stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ + stack++; /* Advance past probeID */ + + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + val = *(((uint64_t *)stack) + arg); /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */ + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + return (val); +#endif + return 0xfeedfacedeafbeadLL; +} + +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fltoffs, int fault, uint64_t illval) +{ + /* XXX ARMTODO */ + /* + * For the case of the error probe firing lets + * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG. + */ + state->dts_arg_error_illval = illval; + dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault ); +} + +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + /* XXX ARMTODO check copied from ppc/x86*/ + /* + * "base" is the smallest toxic address in the range, "limit" is the first + * VALID address greater than "base". + */ + func(0x0, VM_MIN_KERNEL_ADDRESS); + if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) + func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0); +} + +int +dtrace_arm_condition_true(int cond, int cpsr) +{ + int taken = 0; + int zf = (cpsr & PSR_ZF) ? 1 : 0, + nf = (cpsr & PSR_NF) ? 1 : 0, + cf = (cpsr & PSR_CF) ? 1 : 0, + vf = (cpsr & PSR_VF) ? 1 : 0; + + switch(cond) { + case 0: taken = zf; break; + case 1: taken = !zf; break; + case 2: taken = cf; break; + case 3: taken = !cf; break; + case 4: taken = nf; break; + case 5: taken = !nf; break; + case 6: taken = vf; break; + case 7: taken = !vf; break; + case 8: taken = (cf && !zf); break; + case 9: taken = (!cf || zf); break; + case 10: taken = (nf == vf); break; + case 11: taken = (nf != vf); break; + case 12: taken = (!zf && (nf == vf)); break; + case 13: taken = (zf || (nf != vf)); break; + case 14: taken = 1; break; + case 15: taken = 1; break; /* always "true" for ARM, unpredictable for THUMB. */ + } + + return taken; +} diff --git a/bsd/dev/arm/dtrace_subr_arm.c b/bsd/dev/arm/dtrace_subr_arm.c new file mode 100644 index 000000000..ab0bc4820 --- /dev/null +++ b/bsd/dev/arm/dtrace_subr_arm.c @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * #pragma ident "@(#)dtrace_subr.c 1.12 05/06/08 SMI" + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int (*dtrace_pid_probe_ptr) (arm_saved_state_t *); +int (*dtrace_return_probe_ptr) (arm_saved_state_t *); + +kern_return_t +dtrace_user_probe(arm_saved_state_t *, unsigned int); + +kern_return_t +dtrace_user_probe(arm_saved_state_t *regs, unsigned int instr) +{ + /* + * FIXME + * + * The only call path into this method is always a user trap. + * We don't need to test for user trap, but should assert it. + */ + + lck_rw_t *rwp; + struct proc *p = current_proc(); + + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + + kauth_cred_uthread_update(uthread, p); + + if (((regs->cpsr & PSR_TF) && ((uint16_t) instr) == FASTTRAP_THUMB_RET_INSTR) || + ((uint32_t) instr == FASTTRAP_ARM_RET_INSTR)) { + uint8_t step = uthread->t_dtrace_step; + uint8_t ret = uthread->t_dtrace_ret; + user_addr_t npc = uthread->t_dtrace_npc; + + if (uthread->t_dtrace_ast) { + printf("dtrace_user_probe() should be calling aston()\n"); + // aston(thread); + // uthread->t_sig_check = 1; + } + + /* + * Clear all user tracing flags. + */ + uthread->t_dtrace_ft = 0; + + /* + * If we weren't expecting to take a return probe trap, kill + * the process as though it had just executed an unassigned + * trap instruction. + */ + if (step == 0) { + /* + * APPLE NOTE: We're returning KERN_FAILURE, which causes + * the generic signal handling code to take over, which will effectively + * deliver a EXC_BAD_INSTRUCTION to the user process. + */ + return KERN_FAILURE; + } + + /* + * If we hit this trap unrelated to a return probe, we're + * just here to reset the AST flag since we deferred a signal + * until after we logically single-stepped the instruction we + * copied out. + */ + if (ret == 0) { + regs->pc = npc; + return KERN_SUCCESS; + } + + /* + * We need to wait until after we've called the + * dtrace_return_probe_ptr function pointer to step the pc. + */ + rwp = &CPU->cpu_ft_lock; + lck_rw_lock_shared(rwp); + + if (dtrace_return_probe_ptr != NULL) + (void) (*dtrace_return_probe_ptr)(regs); + lck_rw_unlock_shared(rwp); + + regs->pc = npc; + + return KERN_SUCCESS; + } else { + rwp = &CPU->cpu_ft_lock; + + /* + * The DTrace fasttrap provider uses a trap, + * FASTTRAP_{ARM,THUMB}_INSTR. We let + * DTrace take the first crack at handling + * this trap; if it's not a probe that DTrace knows about, + * we call into the trap() routine to handle it like a + * breakpoint placed by a conventional debugger. + */ + + /* + * APPLE NOTE: I believe the purpose of the reader/writers lock + * is thus: There are times which dtrace needs to prevent calling + * dtrace_pid_probe_ptr(). Sun's original impl grabbed a plain + * mutex here. However, that serialized all probe calls, and + * destroyed MP behavior. So now they use a RW lock, with probes + * as readers, and the top level synchronization as a writer. + */ + lck_rw_lock_shared(rwp); + if (dtrace_pid_probe_ptr != NULL && + (*dtrace_pid_probe_ptr)(regs) == 0) { + lck_rw_unlock_shared(rwp); + return KERN_SUCCESS; + } + lck_rw_unlock_shared(rwp); + + /* + * If the instruction that caused the breakpoint trap doesn't + * look like our trap anymore, it may be that this tracepoint + * was removed just after the user thread executed it. In + * that case, return to user land to retry the instuction. + * + * Note that the PC points to the instruction that caused the fault. + */ + if (regs->cpsr & PSR_TF) { + uint16_t instr_check; + if (fuword16(regs->pc, &instr_check) == 0 && instr_check != FASTTRAP_THUMB_INSTR) { + return KERN_SUCCESS; + } + } else { + uint32_t instr_check; + if (fuword32(regs->pc, &instr_check) == 0 && instr_check != FASTTRAP_ARM_INSTR) { + return KERN_SUCCESS; + } + } + } + + return KERN_FAILURE; +} + +void +dtrace_safe_synchronous_signal(void) +{ + /* Not implemented */ +} + +int +dtrace_safe_defer_signal(void) +{ + /* Not implemented */ + return 0; +} diff --git a/bsd/dev/arm/fasttrap_isa.c b/bsd/dev/arm/fasttrap_isa.c new file mode 100644 index 000000000..d48b48a71 --- /dev/null +++ b/bsd/dev/arm/fasttrap_isa.c @@ -0,0 +1,1297 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * #pragma ident "@(#)fasttrap_isa.c 1.19 05/09/14 SMI" + */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +extern dtrace_id_t dtrace_probeid_error; + +/* Solaris proc_t is the struct. Darwin's proc_t is a pointer to it. */ +#define proc_t struct proc /* Steer clear of the Darwin typedef for proc_t */ + +extern int dtrace_decode_arm(uint32_t instr); +extern int dtrace_decode_thumb(uint32_t instr); + +/* + * Lossless User-Land Tracing on ARM + * --------------------------------- + * + * The details here will be fleshed out as more of this is implemented. The + * basic design will be the same way as tracing works in x86. + * + * Some ARM specific issues: + * + * We need to patch differently for ARM instructions and Thumb instructions. + * When we hit a probe, we check to see if the mode we're currently in is the + * same as the mode we're patching for. If not, we remove the tracepoint and + * abort. This ARM/Thumb information is pulled in from the arch specific + * information in the fasttrap probe. + * + * On ARM, any instruction that uses registers can also use the pc as a + * register. This presents problems during emulation because we have copied + * the instruction and thus the pc can be different. Currently we've emulated + * any instructions that use the pc if they can be used in a return probe. + * Eventually we will want to support all instructions that use the pc, but + * to do so requires disassembling the instruction and reconstituting it by + * substituting a different register. + * + */ + +#define THUMB_INSTR(x) (*(uint16_t*) &(x)) + +#define SIGNEXTEND(x,v) ((((int) (x)) << (32-(v))) >> (32-(v))) +#define ALIGNADDR(x,v) (((x) >> (v)) << (v)) +#define GETITSTATE(x) ((((x) >> 8) & 0xFC) | (((x) >> 25) & 0x3)) +#define ISLASTINIT(x) (((x) & 0xF) == 8) + +#define SET16(x,w) *((uint16_t*) (x)) = (w) +#define SET32(x,w) *((uint32_t*) (x)) = (w) + +#define IS_ARM_NOP(x) ((x) == 0xE1A00000) +/* Marker for is-enabled probes */ +#define IS_ARM_IS_ENABLED(x) ((x) == 0xE0200000) + +#define IS_THUMB_NOP(x) ((x) == 0x46C0) +/* Marker for is-enabled probes */ +#define IS_THUMB_IS_ENABLED(x) ((x) == 0x4040) + +#define ARM_LDM_UF (1 << 23) +#define ARM_LDM_PF (1 << 24) +#define ARM_LDM_WF (1 << 21) + +#define ARM_LDR_UF (1 << 23) +#define ARM_LDR_BF (1 << 22) + +extern int dtrace_arm_condition_true(int cond, int cpsr); + +static +void flush_caches(void) +{ + /* TODO There were some problems with flushing just the cache line that had been modified. + * For now, we'll flush the entire cache, until we figure out how to flush just the patched block. + */ + FlushPoU_Dcache(); + InvalidatePoU_Icache(); +} + +int +fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, + user_addr_t pc, fasttrap_probe_type_t type) +{ +#pragma unused(type) + uint32_t instr; + + /* + * Read the instruction at the given address out of the process's + * address space. We don't have to worry about a debugger + * changing this instruction before we overwrite it with our trap + * instruction since P_PR_LOCK is set. Since instructions can span + * pages, we potentially read the instruction in two parts. If the + * second part fails, we just zero out that part of the instruction. + */ + /* + * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey... + */ + + if (uread(p, &instr, 4, pc) != 0) + return (-1); + + /* We want &instr to always point to the saved instruction, so just copy the + * whole thing When cast to a pointer to a uint16_t, that will give us a + * pointer to the first two bytes, which is the thumb instruction. + */ + tp->ftt_instr = instr; + + if (tp->ftt_fntype != FASTTRAP_FN_DONE_INIT) { + switch(tp->ftt_fntype) { + case FASTTRAP_FN_UNKNOWN: + /* Can't instrument without any information. We can add some heuristics later if necessary. */ + return (-1); + + case FASTTRAP_FN_USDT: + if (IS_ARM_NOP(instr) || IS_ARM_IS_ENABLED(instr)) { + tp->ftt_thumb = 0; + } else if (IS_THUMB_NOP(THUMB_INSTR(instr)) || IS_THUMB_IS_ENABLED(THUMB_INSTR(instr))) { + tp->ftt_thumb = 1; + } else { + /* Shouldn't reach here - this means we don't recognize + * the instruction at one of the USDT probe locations + */ + return (-1); + } + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + case FASTTRAP_FN_ARM: + tp->ftt_thumb = 0; + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + case FASTTRAP_FN_THUMB: + tp->ftt_thumb = 1; + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + default: + return (-1); + } + } + + if (tp->ftt_thumb) { + tp->ftt_type = dtrace_decode_thumb(instr); + } else { + tp->ftt_type = dtrace_decode_arm(instr); + } + + if (tp->ftt_type == FASTTRAP_T_INV) { + /* This is an instruction we either don't recognize or can't instrument */ + printf("dtrace: fasttrap: Unrecognized instruction: %08x at %08x\n", + (tp->ftt_thumb && dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb) == 2) ? tp->ftt_instr1 : instr, pc); + return (-1); + } + + return (0); +} + +// These are not exported from vm_map.h. +extern kern_return_t vm_map_write_user(vm_map_t map, void *src_p, vm_map_address_t dst_addr, vm_size_t size); + +/* Patches the instructions. Almost like uwrite, but need special instructions on ARM to flush the caches. */ +static +int patchInst(proc_t *p, void *buf, user_size_t len, user_addr_t a) +{ + kern_return_t ret; + + ASSERT(p != NULL); + ASSERT(p->task != NULL); + + task_t task = p->task; + + /* + * Grab a reference to the task vm_map_t to make sure + * the map isn't pulled out from under us. + * + * Because the proc_lock is not held at all times on all code + * paths leading here, it is possible for the proc to have + * exited. If the map is null, fail. + */ + vm_map_t map = get_task_map_reference(task); + if (map) { + /* Find the memory permissions. */ + uint32_t nestingDepth=999999; + vm_region_submap_short_info_data_64_t info; + mach_msg_type_number_t count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64; + mach_vm_address_t address = (mach_vm_address_t)a; + mach_vm_size_t sizeOfRegion = (mach_vm_size_t)len; + + ret = mach_vm_region_recurse(map, &address, &sizeOfRegion, &nestingDepth, (vm_region_recurse_info_t)&info, &count); + if (ret != KERN_SUCCESS) + goto done; + + vm_prot_t reprotect; + + if (!(info.protection & VM_PROT_WRITE)) { + /* Save the original protection values for restoration later */ + reprotect = info.protection; + if (info.max_protection & VM_PROT_WRITE) { + /* The memory is not currently writable, but can be made writable. */ + /* Making it both writable and executable at the same time causes warning on embedded */ + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, (reprotect & ~VM_PROT_EXECUTE) | VM_PROT_WRITE); + } else { + /* + * The memory is not currently writable, and cannot be made writable. We need to COW this memory. + * + * Strange, we can't just say "reprotect | VM_PROT_COPY", that fails. + */ + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, VM_PROT_COPY | VM_PROT_READ | VM_PROT_WRITE); + } + + if (ret != KERN_SUCCESS) + goto done; + + } else { + /* The memory was already writable. */ + reprotect = VM_PROT_NONE; + } + + ret = vm_map_write_user( map, + buf, + (vm_map_address_t)a, + (vm_size_t)len); + + flush_caches(); + + if (ret != KERN_SUCCESS) + goto done; + + if (reprotect != VM_PROT_NONE) { + ASSERT(reprotect & VM_PROT_EXECUTE); + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, reprotect); + } + +done: + vm_map_deallocate(map); + } else + ret = KERN_TERMINATED; + + return (int)ret; +} + +int +fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp) +{ + /* The thumb patch is a 2 byte instruction regardless of the size of the original instruction */ + uint32_t instr; + int size = tp->ftt_thumb ? 2 : 4; + + if (tp->ftt_thumb) { + *((uint16_t*) &instr) = FASTTRAP_THUMB_INSTR; + } else { + instr = FASTTRAP_ARM_INSTR; + } + + if (patchInst(p, &instr, size, tp->ftt_pc) != 0) + return (-1); + + tp->ftt_installed = 1; + + return (0); +} + +int +fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp) +{ + /* The thumb patch is a 2 byte instruction regardless of the size of the original instruction */ + uint32_t instr; + int size = tp->ftt_thumb ? 2 : 4; + + /* + * Distinguish between read or write failures and a changed + * instruction. + */ + if (uread(p, &instr, size, tp->ftt_pc) != 0) + goto end; + if (tp->ftt_thumb) { + if (*((uint16_t*) &instr) != FASTTRAP_THUMB_INSTR) + goto end; + } else { + if (instr != FASTTRAP_ARM_INSTR) + goto end; + } + if (patchInst(p, &tp->ftt_instr, size, tp->ftt_pc) != 0) + return (-1); + +end: + tp->ftt_installed = 0; + + return (0); +} + +static void +fasttrap_return_common(proc_t *p, arm_saved_state_t *regs, user_addr_t pc, user_addr_t new_pc) +{ + pid_t pid = p->p_pid; + fasttrap_tracepoint_t *tp; + fasttrap_bucket_t *bucket; + fasttrap_id_t *id; + lck_mtx_t *pid_mtx; + int retire_tp = 1; + + pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; + lck_mtx_lock(pid_mtx); + bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; + + for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + if (pid == tp->ftt_pid && pc == tp->ftt_pc && + tp->ftt_proc->ftpc_acount != 0) + break; + } + + /* + * Don't sweat it if we can't find the tracepoint again; unlike + * when we're in fasttrap_pid_probe(), finding the tracepoint here + * is not essential to the correct execution of the process. + */ + if (tp == NULL) { + lck_mtx_unlock(pid_mtx); + return; + } + + for (id = tp->ftt_retids; id != NULL; id = id->fti_next) { + fasttrap_probe_t *probe = id->fti_probe; + /* + * If there's a branch that could act as a return site, we + * need to trace it, and check here if the program counter is + * external to the function. + */ + if (tp->ftt_type != FASTTRAP_T_LDM_PC && + tp->ftt_type != FASTTRAP_T_POP_PC && + new_pc - probe->ftp_faddr < probe->ftp_fsize) + continue; + + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least one probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } +#ifndef CONFIG_EMBEDDED + if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { + dtrace_probe(dtrace_probeid_error, 0 /* state */, id->fti_probe->ftp_id, + 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); +#else + if (FALSE) { +#endif + } else { + dtrace_probe(id->fti_probe->ftp_id, + pc - id->fti_probe->ftp_faddr, + regs->r[0], 0, 0, 0); + } + } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); + } + + lck_mtx_unlock(pid_mtx); +} + +static void +fasttrap_sigsegv(proc_t *p, uthread_t t, user_addr_t addr, arm_saved_state_t *regs) +{ + /* TODO: This function isn't implemented yet. In debug mode, panic the system to + * find out why we're hitting this point. In other modes, kill the process. + */ +#if DEBUG +#pragma unused(p,t,addr,arm_saved_state) + panic("fasttrap: sigsegv not yet implemented"); +#else +#pragma unused(p,t,addr) + /* Kill the process */ + regs->pc = 0; +#endif + +#if 0 + proc_lock(p); + + /* Set fault address and mark signal */ + t->uu_code = addr; + t->uu_siglist |= sigmask(SIGSEGV); + + /* + * XXX These two line may be redundant; if not, then we need + * XXX to potentially set the data address in the machine + * XXX specific thread state structure to indicate the address. + */ + t->uu_exception = KERN_INVALID_ADDRESS; /* SIGSEGV */ + t->uu_subcode = 0; /* XXX pad */ + + proc_unlock(p); + + /* raise signal */ + signal_setast(t->uu_context.vc_thread); +#endif +} + +static void +fasttrap_usdt_args(fasttrap_probe_t *probe, arm_saved_state_t *regs, int argc, + uint32_t *argv) +{ + int i, x, cap = MIN(argc, probe->ftp_nargs); + + for (i = 0; i < cap; i++) { + x = probe->ftp_argmap[i]; + + if (x < 4) { + argv[i] = regs->r[x]; + } else { + fasttrap_fuword32_noerr(regs->sp + (x - 4) * sizeof(uint32_t), &argv[i]); + } + } + + for (; i < argc; i++) { + argv[i] = 0; + } +} + +static void set_thumb_flag(arm_saved_state_t *regs, user_addr_t pc) +{ + if (pc & 1) { + regs->cpsr |= PSR_TF; + } else { + regs->cpsr &= ~PSR_TF; + } +} + +int +fasttrap_pid_probe(arm_saved_state_t *regs) +{ + proc_t *p = current_proc(); + user_addr_t new_pc = 0; + fasttrap_bucket_t *bucket; + lck_mtx_t *pid_mtx; + fasttrap_tracepoint_t *tp, tp_local; + pid_t pid; + dtrace_icookie_t cookie; + uint_t is_enabled = 0; + int instr_size; + int was_simulated = 1, retire_tp = 1; + + user_addr_t pc = regs->pc; + + uthread_t uthread = (uthread_t) get_bsdthread_info(current_thread()); + + /* + * It's possible that a user (in a veritable orgy of bad planning) + * could redirect this thread's flow of control before it reached the + * return probe fasttrap. In this case we need to kill the process + * since it's in a unrecoverable state. + */ + if (uthread->t_dtrace_step) { + ASSERT(uthread->t_dtrace_on); + fasttrap_sigtrap(p, uthread, pc); + return (0); + } + + /* + * Clear all user tracing flags. + */ + uthread->t_dtrace_ft = 0; + uthread->t_dtrace_pc = 0; + uthread->t_dtrace_npc = 0; + uthread->t_dtrace_scrpc = 0; + uthread->t_dtrace_astpc = 0; + + /* + * Treat a child created by a call to vfork(2) as if it were its + * parent. We know that there's only one thread of control in such a + * process: this one. + */ + if (p->p_lflag & P_LINVFORK) { + proc_list_lock(); + while (p->p_lflag & P_LINVFORK) + p = p->p_pptr; + proc_list_unlock(); + } + + pid = p->p_pid; + pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; + lck_mtx_lock(pid_mtx); + bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid,pc)]; + + /* + * Lookup the tracepoint that the process just hit. + */ + for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + if (pid == tp->ftt_pid && pc == tp->ftt_pc && + tp->ftt_proc->ftpc_acount != 0) + break; + } + + /* + * If we couldn't find a matching tracepoint, either a tracepoint has + * been inserted without using the pid ioctl interface (see + * fasttrap_ioctl), or somehow we have mislaid this tracepoint. + */ + if (tp == NULL) { + lck_mtx_unlock(pid_mtx); + return (-1); + } + + /* Default to always execute */ + int condition_code = 0xE; + if (tp->ftt_thumb) { + uint32_t itstate = GETITSTATE(regs->cpsr); + if (itstate != 0) { + /* In IT block, make sure it's the last statement in the block */ + if (ISLASTINIT(itstate)) { + condition_code = itstate >> 4; + } else { + printf("dtrace: fasttrap: Tried to trace instruction %08x at %08x but not at end of IT block\n", + (tp->ftt_thumb && dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb) == 2) ? tp->ftt_instr1 : tp->ftt_instr, pc); + + fasttrap_tracepoint_remove(p, tp); + lck_mtx_unlock(pid_mtx); + return (-1); + } + } + } else { + condition_code = ARM_CONDCODE(tp->ftt_instr); + } + + if (!tp->ftt_thumb != !(regs->cpsr & PSR_TF)) { + /* The ARM/Thumb mode does not match what we expected for this probe. + * Remove this probe and bail. + */ + fasttrap_tracepoint_remove(p, tp); + lck_mtx_unlock(pid_mtx); + return (-1); + } + + if (tp->ftt_ids != NULL) { + fasttrap_id_t *id; + + uint32_t s4; + uint32_t *stack = (uint32_t *)regs->sp; + + /* First four parameters are passed in registers */ + fasttrap_fuword32_noerr((user_addr_t)(uint32_t)stack, &s4); + + for (id = tp->ftt_ids; id != NULL; id = id->fti_next) { + fasttrap_probe_t *probe = id->fti_probe; + +#ifndef CONFIG_EMBEDDED + if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { + dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id, + 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); +#else + if (FALSE) { +#endif + } else { + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } + if (id->fti_ptype == DTFTP_ENTRY) { + /* + * We note that this was an entry + * probe to help ustack() find the + * first caller. + */ + cookie = dtrace_interrupt_disable(); + DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY); + dtrace_probe(probe->ftp_id, regs->r[0], regs->r[1], regs->r[2], regs->r[3], s4); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY); + dtrace_interrupt_enable(cookie); + } else if (id->fti_ptype == DTFTP_IS_ENABLED) { + /* + * Note that in this case, we don't + * call dtrace_probe() since it's only + * an artificial probe meant to change + * the flow of control so that it + * encounters the true probe. + */ + is_enabled = 1; + } else if (probe->ftp_argmap == NULL) { + dtrace_probe(probe->ftp_id, regs->r[0], regs->r[1], regs->r[2], regs->r[3], s4); + } else { + uint32_t t[5]; + + fasttrap_usdt_args(probe, regs, 5, t); + dtrace_probe(probe->ftp_id, t[0], t[1], t[2], t[3], t[4]); + } + } + } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); + } + } + /* + * We're about to do a bunch of work so we cache a local copy of + * the tracepoint to emulate the instruction, and then find the + * tracepoint again later if we need to light up any return probes. + */ + tp_local = *tp; + lck_mtx_unlock(pid_mtx); + tp = &tp_local; + + /* + * If there's an is-enabled probe connected to this tracepoint it + * means that there was a 'eor r0,r0,r0' + * instruction that was placed there by DTrace when the binary was + * linked. As this probe is, in fact, enabled, we need to stuff 1 + * into R0. Accordingly, we can bypass all the instruction + * emulation logic since we know the inevitable result. It's possible + * that a user could construct a scenario where the 'is-enabled' + * probe was on some other instruction, but that would be a rather + * exotic way to shoot oneself in the foot. + */ + + if (is_enabled) { + regs->r[0] = 1; + new_pc = regs->pc + (tp->ftt_thumb ? 2 : 4); + goto done; + } + + /* For USDT probes, bypass all the emulation logic for the nop instruction */ + if ((tp->ftt_thumb && IS_THUMB_NOP(THUMB_INSTR(tp->ftt_instr))) || + (!tp->ftt_thumb && IS_ARM_NOP(tp->ftt_instr))) { + new_pc = regs->pc + (tp->ftt_thumb ? 2 : 4); + goto done; + } + + instr_size = dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb); + + switch (tp->ftt_type) { + case FASTTRAP_T_MOV_PC_REG: + case FASTTRAP_T_CPY_PC: + { + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + int rm; + if (tp->ftt_thumb) { + rm = THUMB16_HRM(tp->ftt_instr1); + } else { + rm = tp->ftt_instr & 0xF; + } + new_pc = regs->r[rm]; + + /* This instruction does not change the Thumb state */ + + break; + } + + case FASTTRAP_T_STM_LR: + case FASTTRAP_T_PUSH_LR: + { + /* + * This is a very common case, so we want to emulate this instruction if + * possible. However, on a push, it is possible that we might reach the end + * of a page and have to allocate a new page. Most of the time this will not + * happen, and we know that the push instruction can store at most 16 words, + * so check to see if we are far from the boundary, and if so, emulate. This + * can be made more aggressive by checking the actual number of words being + * pushed, but we won't do that for now. + * + * Some of the same issues that apply to POP_PC probably apply here also. + */ + + int reglist; + int ret; + uintptr_t* base; + + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + base = (uintptr_t*) regs->sp; + if (((((uintptr_t) base)-16*4) >> PAGE_SHIFT) != (((uintptr_t) base) >> PAGE_SHIFT)) { + /* Crosses the page boundary, go to emulation */ + goto instr_emulate; + } + + if (tp->ftt_thumb) { + if (instr_size == 4) { + /* We know we have to push lr, never push sp or pc */ + reglist = tp->ftt_instr2 & 0x1FFF; + } else { + reglist = tp->ftt_instr1 & 0xFF; + } + } else { + /* We know we have to push lr, never push sp or pc */ + reglist = tp->ftt_instr & 0x1FFF; + } + + /* Push the link register */ + base--; + ret = fasttrap_suword32((uint32_t) base, regs->lr); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, regs); + new_pc = regs->pc; + break; + } + + /* Start pushing from $r12 */ + int regmask = 1 << 12; + int regnum = 12; + + while (regmask) { + if (reglist & regmask) { + base--; + ret = fasttrap_suword32((uint32_t) base, regs->r[regnum]); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, regs); + new_pc = regs->pc; + break; + } + } + regmask >>= 1; + regnum--; + } + + regs->sp = (uintptr_t) base; + + new_pc = pc + instr_size; + + break; + } + + + case FASTTRAP_T_LDM_PC: + case FASTTRAP_T_POP_PC: + { + /* TODO Two issues that will eventually need to be resolved: + * + * 1. Understand what the hardware does if we have to segfault (data abort) in + * the middle of a load multiple. We currently don't have a working segfault + * handler anyway, and with no swapfile we should never segfault on this load. + * If we do, we'll just kill the process by setting the pc to 0. + * + * 2. The emulation is no longer atomic. We currently only emulate pop for + * function epilogues, and so we should never have a race here because one + * thread should never be trying to manipulate another thread's stack frames. + * That is almost certainly a bug in the program. + * + * This will need to be fixed if we ever: + * a. Ship dtrace externally, as this could be a potential attack vector + * b. Support instruction level tracing, as we might then pop/ldm non epilogues. + * + */ + + /* Assume ldmia! sp/pop ... pc */ + + int regnum = 0, reglist; + int ret; + uintptr_t* base; + + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (tp->ftt_thumb) { + if (instr_size == 4) { + /* We know we have to load the pc, don't do it twice */ + reglist = tp->ftt_instr2 & 0x7FFF; + } else { + reglist = tp->ftt_instr1 & 0xFF; + } + } else { + /* We know we have to load the pc, don't do it twice */ + reglist = tp->ftt_instr & 0x7FFF; + } + + base = (uintptr_t*) regs->sp; + while (reglist) { + if (reglist & 1) { + ret = fasttrap_fuword32((uint32_t) base, ®s->r[regnum]); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, regs); + new_pc = regs->pc; + break; + } + base++; + } + reglist >>= 1; + regnum++; + } + + ret = fasttrap_fuword32((uint32_t) base, &new_pc); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, regs); + new_pc = regs->pc; + break; + } + base++; + + regs->sp = (uintptr_t) base; + + set_thumb_flag(regs, new_pc); + + break; + } + + case FASTTRAP_T_CB_N_Z: + { + /* Thumb mode instruction, and not permitted in IT block, so skip the condition code check */ + int rn = tp->ftt_instr1 & 0x7; + int offset = (((tp->ftt_instr1 & 0x00F8) >> 2) | ((tp->ftt_instr1 & 0x0200) >> 3)) + 4; + int nonzero = tp->ftt_instr1 & 0x0800; + if (!nonzero != !(regs->r[rn] == 0)) { + new_pc = pc + offset; + } else { + new_pc = pc + instr_size; + } + break; + } + + case FASTTRAP_T_B_COND: + { + /* Use the condition code in the instruction and ignore the ITSTATE */ + + int code, offset; + if (tp->ftt_thumb) { + if (instr_size == 4) { + code = (tp->ftt_instr1 >> 6) & 0xF; + if (code == 14 || code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + int S = (tp->ftt_instr1 >> 10) & 1, + J1 = (tp->ftt_instr2 >> 13) & 1, + J2 = (tp->ftt_instr2 >> 11) & 1; + offset = 4 + SIGNEXTEND( + (S << 20) | (J2 << 19) | (J1 << 18) | + ((tp->ftt_instr1 & 0x003F) << 12) | + ((tp->ftt_instr2 & 0x07FF) << 1), + 21); + } else { + code = (tp->ftt_instr1 >> 8) & 0xF; + if (code == 14 || code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + offset = 4 + (SIGNEXTEND(tp->ftt_instr1 & 0xFF, 8) << 1); + } + } else { + code = ARM_CONDCODE(tp->ftt_instr); + if (code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + offset = 8 + (SIGNEXTEND(tp->ftt_instr & 0x00FFFFFF, 24) << 2); + } + + if (dtrace_arm_condition_true(code, regs->cpsr)) { + new_pc = pc + offset; + } else { + new_pc = pc + instr_size; + } + + break; + } + + case FASTTRAP_T_B_UNCOND: + { + int offset; + + /* Unconditional branches can only be taken from Thumb mode */ + /* (This is different from an ARM branch with condition code "always") */ + ASSERT(tp->ftt_thumb == 1); + + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (instr_size == 4) { + int S = (tp->ftt_instr1 >> 10) & 1, + J1 = (tp->ftt_instr2 >> 13) & 1, + J2 = (tp->ftt_instr2 >> 11) & 1; + int I1 = (J1 != S) ? 0 : 1, I2 = (J2 != S) ? 0 : 1; + offset = 4 + SIGNEXTEND( + (S << 24) | (I1 << 23) | (I2 << 22) | + ((tp->ftt_instr1 & 0x03FF) << 12) | + ((tp->ftt_instr2 & 0x07FF) << 1), + 25); + } else { + uint32_t instr1 = tp->ftt_instr1; + offset = 4 + (SIGNEXTEND(instr1 & 0x7FF, 11) << 1); + } + + new_pc = pc + offset; + + break; + } + + case FASTTRAP_T_BX_REG: + { + int reg; + + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (tp->ftt_thumb) { + reg = THUMB16_HRM(tp->ftt_instr1); + } else { + reg = ARM_RM(tp->ftt_instr); + } + new_pc = regs->r[reg]; + set_thumb_flag(regs, new_pc); + + break; + } + + case FASTTRAP_T_LDR_PC_IMMED: + case FASTTRAP_T_VLDR_PC_IMMED: + /* Handle these instructions by replacing the PC in the instruction with another + * register. They are common, so we'd like to support them, and this way we do so + * without any risk of having to simulate a segfault. + */ + + /* Fall through */ + + instr_emulate: + case FASTTRAP_T_COMMON: + { + user_addr_t addr; + uint8_t scratch[32]; + uint_t i = 0; + fasttrap_instr_t emul_instr; + emul_instr.instr32 = tp->ftt_instr; + int emul_instr_size; + + /* + * Unfortunately sometimes when we emulate the instruction and have to replace the + * PC, there is no longer a thumb mode equivalent. We end up having to run the + * modified instruction in ARM mode. We use this variable to keep track of which + * mode we should emulate in. We still use the original variable to determine + * what mode to return to. + */ + uint8_t emul_thumb = tp->ftt_thumb; + int save_reg = -1; + uint32_t save_val = 0; + + /* + * Dealing with condition codes and emulation: + * We can't just uniformly do a condition code check here because not all instructions + * have condition codes. We currently do not support an instruction by instruction trace, + * so we can assume that either: 1. We are executing a Thumb instruction, in which case + * we either are not in an IT block and should execute always, or we are last in an IT + * block. Either way, the traced instruction will run correctly, and we won't have any + * problems when we return to the original code, because we will no longer be in the IT + * block. 2. We are executing an ARM instruction, in which case we are ok as long as + * we don't attempt to change the condition code. + */ + if (tp->ftt_type == FASTTRAP_T_LDR_PC_IMMED) { + /* We know we always have a free register (the one we plan to write the + * result value to!). So we'll replace the pc with that one. + */ + int new_reg; + if (tp->ftt_thumb) { + /* Check to see if thumb or thumb2 */ + if (instr_size == 2) { + /* + * Sadness. We need to emulate this instruction in ARM mode + * because it has an 8 bit immediate offset. Instead of having + * to deal with condition codes in the ARM instruction, we'll + * just check the condition and abort if the condition is false. + */ + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + new_reg = (tp->ftt_instr1 >> 8) & 0x7; + regs->r[new_reg] = ALIGNADDR(regs->pc + 4, 2); + emul_thumb = 0; + emul_instr.instr32 = 0xE5900000 | (new_reg << 16) | (new_reg << 12) | ((tp->ftt_instr1 & 0xFF) << 2); + } else { + /* Thumb2. Just replace the register. */ + new_reg = (tp->ftt_instr2 >> 12) & 0xF; + regs->r[new_reg] = ALIGNADDR(regs->pc + 4, 2); + emul_instr.instr16.instr1 &= ~0x000F; + emul_instr.instr16.instr1 |= new_reg; + } + } else { + /* ARM. Just replace the register. */ + new_reg = (tp->ftt_instr >> 12) & 0xF; + regs->r[new_reg] = ALIGNADDR(regs->pc + 8,2); + emul_instr.instr32 &= ~0x000F0000; + emul_instr.instr32 |= new_reg << 16; + } + } else if (tp->ftt_type == FASTTRAP_T_VLDR_PC_IMMED) { + /* This instruction only uses one register, and if we're here, we know + * it must be the pc. So we'll just replace it with R0. + */ + save_reg = 0; + save_val = regs->r[0]; + regs->r[save_reg] = ALIGNADDR(regs->pc + (tp->ftt_thumb ? 4 : 8), 2); + if (tp->ftt_thumb) { + emul_instr.instr16.instr1 &= ~0x000F; + } else { + emul_instr.instr32 &= ~0x000F0000; + } + } + + emul_instr_size = dtrace_instr_size(emul_instr.instr32, emul_thumb); + + /* + * At this point: + * tp->ftt_thumb = thumb mode of original instruction + * emul_thumb = thumb mode for emulation + * emul_instr = instruction we are using to emulate original instruction + * emul_instr_size = size of emulating instruction + */ + + addr = uthread->t_dtrace_scratch->addr; + + if (addr == 0LL) { + fasttrap_sigtrap(p, uthread, pc); // Should be killing target proc + new_pc = pc; + break; + } + + uthread->t_dtrace_scrpc = addr; + if (emul_thumb) { + /* + * No way to do an unconditional branch in Thumb mode, shove the address + * onto the user stack and go to the next location with a pop. This can + * segfault if this push happens to cross a stack page, but that's ok, since + * we are running in userland, and the kernel knows how to handle userland + * stack expansions correctly. + * + * Layout of scratch space for Thumb mode: + * Emulated instruction + * ldr save_reg, [pc, #16] (if necessary, restore any register we clobbered) + * push { r0, r1 } + * ldr r0, [pc, #4] + * str r0, [sp, #4] + * pop { r0, pc } + * Location we should return to in original program + * Saved value of clobbered register (if necessary) + */ + + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + + if (save_reg != -1) { + uint16_t restore_inst = 0x4803; + restore_inst |= (save_reg & 0x7) << 8; + SET16(scratch+i, restore_inst); i += 2; // ldr reg, [pc , #16] + } + + SET16(scratch+i, 0xB403); i += 2; // push { r0, r1 } + SET16(scratch+i, 0x4801); i += 2; // ldr r0, [pc, #4] + SET16(scratch+i, 0x9001); i += 2; // str r0, [sp, #4] + SET16(scratch+i, 0xBD01); i += 2; // pop { r0, pc } + + if (i % 4) { + SET16(scratch+i, 0); i += 2; // padding - saved 32 bit words must be aligned + } + SET32(scratch+i, pc + instr_size + (tp->ftt_thumb ? 1 : 0)); i += 4; // Return address + if (save_reg != -1) { + SET32(scratch+i, save_val); i += 4; // saved value of clobbered register + } + + uthread->t_dtrace_astpc = addr + i; + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + SET16(scratch+i, FASTTRAP_THUMB_RET_INSTR); i += 2; + } else { + /* + * Layout of scratch space for ARM mode: + * Emulated instruction + * ldr save_reg, [pc, #12] (if necessary, restore any register we clobbered) + * ldr pc, [pc, #4] + * Location we should return to in original program + * Saved value of clobbered register (if necessary) + */ + + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + + if (save_reg != -1) { + uint32_t restore_inst = 0xE59F0004; + restore_inst |= save_reg << 12; + SET32(scratch+i, restore_inst); i += 4; // ldr reg, [pc, #12] + } + SET32(scratch+i, 0xE51FF004); i += 4; // ldr pc, [pc, #4] + + SET32(scratch+i, pc + instr_size + (tp->ftt_thumb ? 1 : 0)); i += 4; // Return address + if (save_reg != -1) { + SET32(scratch+i, save_val); i += 4; // Saved value of clobbered register + } + + uthread->t_dtrace_astpc = addr + i; + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + SET32(scratch+i, FASTTRAP_ARM_RET_INSTR); i += 4; + } + + if (patchInst(p, scratch, i, uthread->t_dtrace_scratch->write_addr) != KERN_SUCCESS) { + fasttrap_sigtrap(p, uthread, pc); + new_pc = pc; + break; + } + + if (tp->ftt_retids != NULL) { + uthread->t_dtrace_step = 1; + uthread->t_dtrace_ret = 1; + new_pc = uthread->t_dtrace_astpc + (emul_thumb ? 1 : 0); + } else { + new_pc = uthread->t_dtrace_scrpc + (emul_thumb ? 1 : 0); + } + + uthread->t_dtrace_pc = pc; + uthread->t_dtrace_npc = pc + instr_size; + uthread->t_dtrace_on = 1; + was_simulated = 0; + set_thumb_flag(regs, new_pc); + break; + } + + default: + panic("fasttrap: mishandled an instruction"); + } + +done: + /* + * APPLE NOTE: + * + * We're setting this earlier than Solaris does, to get a "correct" + * ustack() output. In the Sun code, a() -> b() -> c() -> d() is + * reported at: d, b, a. The new way gives c, b, a, which is closer + * to correct, as the return instruction has already exectued. + */ + regs->pc = new_pc; + + /* + * If there were no return probes when we first found the tracepoint, + * we should feel no obligation to honor any return probes that were + * subsequently enabled -- they'll just have to wait until the next + * time around. + */ + if (tp->ftt_retids != NULL) { + /* + * We need to wait until the results of the instruction are + * apparent before invoking any return probes. If this + * instruction was emulated we can just call + * fasttrap_return_common(); if it needs to be executed, we + * need to wait until the user thread returns to the kernel. + */ + /* + * It used to be that only common instructions were simulated. + * For performance reasons, we now simulate some instructions + * when safe and go back to userland otherwise. The was_simulated + * flag means we don't need to go back to userland. + */ + if (was_simulated) { + fasttrap_return_common(p, regs, pc, new_pc); + } else { + ASSERT(uthread->t_dtrace_ret != 0); + ASSERT(uthread->t_dtrace_pc == pc); + ASSERT(uthread->t_dtrace_scrpc != 0); + ASSERT(new_pc == uthread->t_dtrace_astpc); + } + } + + return (0); +} + +int +fasttrap_return_probe(arm_saved_state_t *regs) +{ + proc_t *p = current_proc(); + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + user_addr_t pc = uthread->t_dtrace_pc; + user_addr_t npc = uthread->t_dtrace_npc; + + uthread->t_dtrace_pc = 0; + uthread->t_dtrace_npc = 0; + uthread->t_dtrace_scrpc = 0; + uthread->t_dtrace_astpc = 0; + + /* + * Treat a child created by a call to vfork(2) as if it were its + * parent. We know that there's only one thread of control in such a + * process: this one. + */ + if (p->p_lflag & P_LINVFORK) { + proc_list_lock(); + while (p->p_lflag & P_LINVFORK) + p = p->p_pptr; + proc_list_unlock(); + } + + /* + * We set rp->r_pc to the address of the traced instruction so + * that it appears to dtrace_probe() that we're on the original + * instruction, and so that the user can't easily detect our + * complex web of lies. dtrace_return_probe() (our caller) + * will correctly set %pc after we return. + */ + regs->pc = pc; + + fasttrap_return_common(p, regs, pc, npc); + + return (0); +} + +uint64_t +fasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno, + int aframes) +{ +#pragma unused(arg, id, parg, aframes) + arm_saved_state_t* regs = find_user_regs(current_thread()); + + /* First four arguments are in registers */ + if (argno < 4) + return regs->r[argno]; + + /* Look on the stack for the rest */ + uint32_t value; + uint32_t* sp = (uint32_t*) regs->sp; + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + value = dtrace_fuword32((user_addr_t) (sp+argno-4)); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); + + return value; +} + +uint64_t +fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ +#pragma unused(arg, id, parg, argno, aframes) +#if 0 + return (fasttrap_anarg(ttolwp(curthread)->lwp_regs, 0, argno)); +#endif + + return 0; +} + diff --git a/bsd/dev/arm/fbt_arm.c b/bsd/dev/arm/fbt_arm.c new file mode 100644 index 000000000..c594f9c92 --- /dev/null +++ b/bsd/dev/arm/fbt_arm.c @@ -0,0 +1,681 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* #pragma ident "@(#)fbt.c 1.15 05/09/19 SMI" */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from + * mach/ppc/thread_status.h */ +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#define DTRACE_INVOP_PUSH_LR 8 +#define DTRACE_INVOP_BL 9 +#define DTRACE_INVOP_POP_PC 10 + +#define DTRACE_INVOP_THUMB_NOP_SKIP 2 +#define DTRACE_INVOP_POP_PC_SKIP 2 +#define DTRACE_INVOP_THUMB_SET_R7_SKIP 2 +#define DTRACE_INVOP_THUMB_MOV_SP_TO_R7_SKIP 2 + +#define FBT_IS_THUMB_PUSH_LR(x) (((x) & 0x0000ff00) == 0x0000b500) +#define FBT_IS_THUMB_POP_R7(x) (((x) & 0x0000ff80) == 0x0000bc80) +#define FBT_IS_THUMB32_POP_R7LR(x,y) (((x) == 0x0000e8bd) && (((y) & 0x00004080) == 0x00004080)) +#define FBT_IS_THUMB_POP_PC(x) (((x) & 0x0000ff00) == 0x0000bd00) +#define FBT_IS_THUMB_SET_R7(x) (((x) & 0x0000ff00) == 0x0000af00) +#define FBT_IS_THUMB_MOV_SP_TO_R7(x) (((x) & 0x0000ffff) == 0x0000466f) +#define FBT_THUMB_SET_R7_OFFSET(x) (((x) & 0x000000ff) << 2) +#define FBT_IS_THUMB_LDR_PC(x) (((x) & 0x0000f800) == 0x00004800) +#define FBT_IS_THUMB32_LDR_PC(x,y) ((x) == 0x0000f8df) /* Only for positive offset PC relative loads */ +#define FBT_THUMB_STACK_REGS(x) ((x) & 0x00FF) +#define FBT_IS_THUMB_BX_REG(x) (((x) & 0x0000ff87) == 0x00004700) + +#define FBT_PATCHVAL 0xdefc +#define FBT_AFRAMES_ENTRY 8 +#define FBT_AFRAMES_RETURN 6 + +#define FBT_ENTRY "entry" +#define FBT_RETURN "return" +#define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask) + +#define VFPSAVE_ALIGN_DTRACE 16 /* This value should come from VFPSAVE_ALIGN */ + +extern dtrace_provider_id_t fbt_id; +extern fbt_probe_t **fbt_probetab; +extern int fbt_probetab_mask; + +kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int); + +static int fbt_uninstrumented_arm = 0; +static const int fbt_log_uninstrumented = 0; + +extern int dtrace_arm_condition_true(int cond, int cpsr); + + +/* Calculate the address of the ldr. (From the ARM Architecture reference) */ +/* Does not check to see if it's really a load instruction, caller must do that */ + +static uint32_t thumb_ldr_pc_address(uint32_t address) +{ + return (address & 0xFFFFFFFC) + (*(uint16_t*) address & 0xFF) * 4 + 4; +} + +static uint32_t thumb32_ldr_pc_address(uint32_t address) +{ + return (address & 0xFFFFFFFC) + (*(uint16_t*) (address+2) & 0xFFF) + 4; +} + +/* Extract the current ITSTATE from the CPSR */ +static uint32_t get_itstate(uint32_t cpsr) +{ + return + ((cpsr & 0x06000000) >> 25) | + ((cpsr & 0x0000FC00) >> 8); +} + +static void clear_itstate(uint32_t* cpsr) +{ + *cpsr &= ~0x0600FC00; +} + +int +fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval) +{ + fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; + + for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { + if ((uintptr_t) fbt->fbtp_patchpoint == addr) { + if (0 == CPU->cpu_dtrace_invop_underway) { + CPU->cpu_dtrace_invop_underway = 1; /* Race not possible on + * this per-cpu state */ + + struct arm_saved_state* regs = (struct arm_saved_state*) stack; + uintptr_t stack4 = *((uintptr_t*) regs->sp); + + if ((regs->cpsr & PSR_MODE_MASK) == PSR_FIQ_MODE) { + /* + * We do not handle probes firing from FIQ context. We used to + * try to undo the patch and rerun the instruction, but + * most of the time we can't do that successfully anyway. + * Instead, we just panic now so we fail fast. + */ + panic("dtrace: fbt: The probe at %08x was called from FIQ_MODE",(unsigned) addr); + } + + /* + * If we are not outside an IT block, and are not executing the last instruction of an IT block, + * then that is an instrumentation error or a code gen error. Either way, we panic. + */ + uint32_t itstate = get_itstate(regs->cpsr); + if ((itstate & 0x7) != 0) { + panic("dtrace: fbt: Instruction stream error: Middle of IT block at %08x",(unsigned) addr); + } + + if (fbt->fbtp_roffset == 0) { + /* + We need the frames to set up the backtrace, but we won't have the frame pointers + until after the instruction is emulated. So here we calculate the address of the + frame pointer from the saved instruction and put it in the stack. Yes, we end up + repeating this work again when we emulate the instruction. + + This assumes that the frame area is immediately after the saved reg storage! + */ + uint32_t offset = ((uint32_t) regs) + sizeof(struct arm_saved_state); +#if __ARM_VFP__ + /* Match the stack alignment required for arm_vfpsaved_state */ + offset &= ~(VFPSAVE_ALIGN_DTRACE - 1); + offset += VFPSAVE_ALIGN_DTRACE + sizeof(struct arm_vfpsaved_state); +#endif /* __ARM_VFP__ */ + if (FBT_IS_THUMB_SET_R7(fbt->fbtp_savedval)) + *((uint32_t*) offset) = regs->sp + FBT_THUMB_SET_R7_OFFSET(fbt->fbtp_savedval); + else + *((uint32_t*) offset) = regs->sp; + + CPU->cpu_dtrace_caller = regs->lr; + dtrace_probe(fbt->fbtp_id, regs->r[0], regs->r[1], regs->r[2], regs->r[3], stack4); + CPU->cpu_dtrace_caller = 0; + } else { + /* Check to see if we're in the middle of an IT block. */ + if (itstate != 0) { + /* + * We've already checked previously to see how far we are in the IT block. + * Here we must be getting ready to execute the last instruction. + */ + int condition_it = (itstate & 0xF0) >> 4; + + if (dtrace_arm_condition_true(condition_it, regs->cpsr) == 0) { + /* Condition wasn't true, so becomes a nop. */ + clear_itstate(®s->cpsr); + CPU->cpu_dtrace_invop_underway = 0; + return DTRACE_INVOP_NOP; + } + } + + dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0); + CPU->cpu_dtrace_caller = 0; + + /* The dtrace script may access cpsr, so make sure to clear only after probe fired. */ + clear_itstate(®s->cpsr); + } + CPU->cpu_dtrace_invop_underway = 0; + } + + /* + On other architectures, we return a DTRACE constant to let the callback function + know what was replaced. On the ARM, since the function prologue/epilogue machine code + can vary, we need the actual bytes of the instruction, so return the savedval instead. + */ + return (fbt->fbtp_savedval); + } + } + + return (0); +} + +#define IS_USER_TRAP(regs) (((regs)->cpsr & PSR_MODE_MASK) == PSR_USER_MODE) +#define T_INVALID_OPCODE EXC_BAD_INSTRUCTION +#define FBT_EXCEPTION_CODE T_INVALID_OPCODE + +kern_return_t +fbt_perfCallback( + int trapno, + struct arm_saved_state * regs, + __unused int unused1, + __unused int unused2) +{ +#pragma unused (unused1) +#pragma unused (unused2) + kern_return_t retval = KERN_FAILURE; + + if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) { + boolean_t oldlevel = 0; + machine_inst_t emul = 0; + + oldlevel = ml_set_interrupts_enabled(FALSE); + + __asm__ volatile( + "Ldtrace_invop_callsite_pre_label:\n" + ".data\n" + ".private_extern _dtrace_invop_callsite_pre\n" + "_dtrace_invop_callsite_pre:\n" + " .long Ldtrace_invop_callsite_pre_label\n" + ".text\n" + ); + + emul = dtrace_invop(regs->pc, (uintptr_t*) regs, regs->r[0]); + + __asm__ volatile( + "Ldtrace_invop_callsite_post_label:\n" + ".data\n" + ".private_extern _dtrace_invop_callsite_post\n" + "_dtrace_invop_callsite_post:\n" + " .long Ldtrace_invop_callsite_post_label\n" + ".text\n" + ); + + /* + * The following emulation code does not execute properly if we are in the middle of + * an IT block. IT blocks need to be handled in the dtrace_invop function. If we do + * manage to get here and we are inside an IT block, then we missed a case somewhere + * prior to this point. + */ + uint32_t itstate = get_itstate(regs->cpsr); + if (itstate != 0) { + panic("dtrace: fbt: Not emulated: Middle of IT block at %08x",(unsigned) regs->pc); + } + + if (emul == DTRACE_INVOP_NOP) { + regs->pc += DTRACE_INVOP_THUMB_NOP_SKIP; + retval = KERN_SUCCESS; + } else if (FBT_IS_THUMB_SET_R7(emul)) { + regs->r[7] = regs->sp + FBT_THUMB_SET_R7_OFFSET(emul); + regs->pc += DTRACE_INVOP_THUMB_SET_R7_SKIP; + retval = KERN_SUCCESS; + } else if (FBT_IS_THUMB_MOV_SP_TO_R7(emul)) { + regs->r[7] = regs->sp; + regs->pc += DTRACE_INVOP_THUMB_MOV_SP_TO_R7_SKIP; + retval = KERN_SUCCESS; + } else if (FBT_IS_THUMB_POP_PC(emul)) { + uintptr_t* sp = (uintptr_t*) regs->sp; + + machine_inst_t mask = 0x0001; + int regnum = 0; + while (mask & 0x00ff) { + if (emul & mask) { + /* Pop this register */ + regs->r[regnum] = *sp++; + } + mask <<= 1; + regnum++; + } + + regs->pc = *sp++; + regs->sp = (uintptr_t) sp; + if (regs->pc & 1) { + regs->cpsr |= PSR_TF; + } else { + regs->cpsr &= ~PSR_TF; + } + + retval = KERN_SUCCESS; + } else if (FBT_IS_THUMB_BX_REG(emul)) { + regs->pc = regs->r[(emul >> 3) & 0xF]; + + if (regs->pc & 1) { + regs->cpsr |= PSR_TF; + } else { + regs->cpsr &= ~PSR_TF; + } + + retval = KERN_SUCCESS; + } else if (emul == FBT_PATCHVAL) { + /* Means we encountered an error but handled it, try same inst again */ + retval = KERN_SUCCESS; + } else { + retval = KERN_FAILURE; + } + + ml_set_interrupts_enabled(oldlevel); + } + + return retval; +} + +void +fbt_provide_probe(struct modctl *ctl, uintptr_t instrLow, uintptr_t instrHigh, char *modname, char* symbolName, machine_inst_t* symbolStart) +{ + unsigned int j; + int doenable = 0; + dtrace_id_t thisid; + + fbt_probe_t *newfbt, *retfbt, *entryfbt; + machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr; + int foundPushLR, savedRegs; + + /* + * Guard against null symbols + */ + if (!symbolStart || !instrLow || !instrHigh) { + kprintf("dtrace: %s has an invalid address\n", symbolName); + return; + } + + /* + * Assume the compiler doesn't schedule instructions in the prologue. + */ + foundPushLR = 0; + savedRegs = -1; + limit = (machine_inst_t *)instrHigh; + for (j = 0, instr = symbolStart, theInstr = 0; + (j < 8) && ((uintptr_t)instr >= instrLow) && (instrHigh > (uintptr_t)(instr)); j++, instr++) + { + theInstr = *instr; + if (FBT_IS_THUMB_PUSH_LR(theInstr)) { + foundPushLR = 1; + /* Keep track of what registers we pushed. Compare this against the pop later. */ + savedRegs = FBT_THUMB_STACK_REGS(theInstr); + pushinstr = instr; + } + if (foundPushLR && (FBT_IS_THUMB_SET_R7(theInstr) || FBT_IS_THUMB_MOV_SP_TO_R7(theInstr))) + /* Guard against a random setting of r7 from sp, we make sure we found the push first */ + break; + if (FBT_IS_THUMB_BX_REG(theInstr)) /* We've gone too far, bail. */ + break; + if (FBT_IS_THUMB_POP_PC(theInstr)) /* We've gone too far, bail. */ + break; + + /* Check for 4 byte thumb instruction */ + if (dtrace_instr_size(theInstr,1) == 4) + instr++; + } + + if (!(foundPushLR && (FBT_IS_THUMB_SET_R7(theInstr) || FBT_IS_THUMB_MOV_SP_TO_R7(theInstr)))) { + return; + } + + thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY); + newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); + newfbt->fbtp_next = NULL; + strlcpy( (char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS ); + + if (thisid != 0) { + /* + * The dtrace_probe previously existed, so we have to hook + * the newfbt entry onto the end of the existing fbt's + * chain. + * If we find an fbt entry that was previously patched to + * fire, (as indicated by the current patched value), then + * we want to enable this newfbt on the spot. + */ + entryfbt = dtrace_probe_arg (fbt_id, thisid); + ASSERT (entryfbt != NULL); + for(; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) { + if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval) + doenable++; + + if (entryfbt->fbtp_next == NULL) { + entryfbt->fbtp_next = newfbt; + newfbt->fbtp_id = entryfbt->fbtp_id; + break; + } + } + } + else { + /* + * The dtrace_probe did not previously exist, so we + * create it and hook in the newfbt. Since the probe is + * new, we obviously do not need to enable it on the spot. + */ + newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt); + doenable = 0; + } + + newfbt->fbtp_patchpoint = instr; + newfbt->fbtp_ctl = ctl; + newfbt->fbtp_loadcnt = ctl->mod_loadcnt; + newfbt->fbtp_rval = DTRACE_INVOP_PUSH_LR; + newfbt->fbtp_savedval = theInstr; + newfbt->fbtp_patchval = FBT_PATCHVAL; + newfbt->fbtp_currentval = 0; + newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; + + if (doenable) + fbt_enable(NULL, newfbt->fbtp_id, newfbt); + + /* + * The fbt entry chain is in place, one entry point per symbol. + * The fbt return chain can have multiple return points per + * symbol. + * Here we find the end of the fbt return chain. + */ + + doenable=0; + + thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN); + + if (thisid != 0) { + /* The dtrace_probe previously existed, so we have to + * find the end of the existing fbt chain. If we find + * an fbt return that was previously patched to fire, + * (as indicated by the currrent patched value), then + * we want to enable any new fbts on the spot. + */ + retfbt = dtrace_probe_arg (fbt_id, thisid); + ASSERT(retfbt != NULL); + for (; retfbt != NULL; retfbt = retfbt->fbtp_next) { + if (retfbt->fbtp_currentval == retfbt->fbtp_patchval) + doenable++; + if(retfbt->fbtp_next == NULL) + break; + } + } + else { + doenable = 0; + retfbt = NULL; + } + + /* + * Go back to the start of the function, in case + * the compiler emitted pcrel data loads + * before R7 was adjusted. + */ + instr = pushinstr + 1; +again: + if (instr >= limit) + return; + + /* + * We (desperately) want to avoid erroneously instrumenting a + * jump table. To determine if we're looking at a true instruction + * or an inline jump table that happens to contain the same + * byte sequences, we resort to some heuristic sleeze: we + * treat this instruction as being contained within a pointer, + * and see if that pointer points to within the body of the + * function. If it does, we refuse to instrument it. + */ + if (((uintptr_t)instr & 0x3) == 0) { + machine_inst_t *ptr = *(machine_inst_t **)(void *)instr; + + if (ptr >= (machine_inst_t *)symbolStart && ptr < limit) { + /* kprintf("dtrace: fbt: Found jump table in %s, at %08x\n",symbolName,(unsigned)instr); */ + instr++; + goto again; + } + } + + /* + * OK, it's an instruction. + */ + theInstr = *instr; + + /* Walked onto the start of the next routine? If so, bail out from this function */ + if (FBT_IS_THUMB_PUSH_LR(theInstr)) { + if (!retfbt) + kprintf("dtrace: fbt: No return probe for %s, walked to next routine at %08x\n",symbolName,(unsigned)instr); + return; + } + + /* The PC relative data should be stored after the end of the function. If + * we see a PC relative load, assume the address to load from is the new end + * of the function. */ + if (FBT_IS_THUMB_LDR_PC(theInstr)) { + uint32_t newlimit = thumb_ldr_pc_address((uint32_t) instr); + if (newlimit < (uint32_t) limit) + limit = (machine_inst_t*) newlimit; + } + if ((instr+1) < limit && FBT_IS_THUMB32_LDR_PC(*instr,*(instr+1))) { + uint32_t newlimit = thumb32_ldr_pc_address((uint32_t) instr); + if (newlimit < (uint32_t) limit) + limit = (machine_inst_t*) newlimit; + } + + /* Look for the 1. pop { ..., pc } or 2. pop { ..., r7 } ... bx reg or 3. ldmia.w sp!, { ..., r7, lr } ... bx reg */ + if (!FBT_IS_THUMB_POP_PC(theInstr) && + !FBT_IS_THUMB_POP_R7(theInstr) && + !FBT_IS_THUMB32_POP_R7LR(theInstr,*(instr+1))) { + instr++; + if (dtrace_instr_size(theInstr,1) == 4) + instr++; + goto again; + } + + if (FBT_IS_THUMB_POP_PC(theInstr)) { + if (savedRegs != FBT_THUMB_STACK_REGS(theInstr)) { + /* What we're popping doesn't match what we're pushing, assume that we've + * gone too far in the function. Bail. + */ + kprintf("dtrace: fbt: No return probe for %s, popped regs don't match at %08x\n",symbolName,(unsigned)instr); + return; + } + } else { + /* Scan ahead for the bx */ + for (j = 0; (j < 4) && (instr < limit); j++, instr++) { + theInstr = *instr; + if (FBT_IS_THUMB_BX_REG(theInstr)) + break; + if (dtrace_instr_size(theInstr,1) == 4) + instr++; + } + + if (!FBT_IS_THUMB_BX_REG(theInstr)) + return; + } + + /* + * pop { ..., pc}, bx reg -- We have a winner! + */ + + newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); + newfbt->fbtp_next = NULL; + strlcpy( (char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS ); + + if (retfbt == NULL) { + newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, + symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt); + } else { + retfbt->fbtp_next = newfbt; + newfbt->fbtp_id = retfbt->fbtp_id; + } + + retfbt = newfbt; + newfbt->fbtp_patchpoint = instr; + newfbt->fbtp_ctl = ctl; + newfbt->fbtp_loadcnt = ctl->mod_loadcnt; + + ASSERT(FBT_IS_THUMB_POP_PC(theInstr) || FBT_IS_THUMB_BX_REG(theInstr)); + newfbt->fbtp_rval = DTRACE_INVOP_POP_PC; + newfbt->fbtp_roffset = + (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart); + newfbt->fbtp_savedval = theInstr; + newfbt->fbtp_patchval = FBT_PATCHVAL; + newfbt->fbtp_currentval = 0; + newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; + + if (doenable) + fbt_enable(NULL, newfbt->fbtp_id, newfbt); + + instr++; + goto again; +} + +void +fbt_provide_module_kernel_syms(struct modctl *ctl) +{ + kernel_mach_header_t *mh; + struct load_command *cmd; + kernel_segment_command_t *orig_ts = NULL, *orig_le = NULL; + struct symtab_command *orig_st = NULL; + kernel_nlist_t *sym = NULL; + char *strings; + uintptr_t instrLow, instrHigh; + char *modname; + unsigned int i; + + mh = (kernel_mach_header_t *)(ctl->mod_address); + modname = ctl->mod_modname; + + /* + * Employees of dtrace and their families are ineligible. Void + * where prohibited. + */ + + if (mh->magic != MH_MAGIC_KERNEL) + return; + + cmd = (struct load_command *) & mh[1]; + for (i = 0; i < mh->ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_KERNEL) { + kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd; + + if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT)) + orig_ts = orig_sg; + else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT)) + orig_le = orig_sg; + else if (LIT_STRNEQL(orig_sg->segname, "")) + orig_ts = orig_sg; /* kexts have a single + * unnamed segment */ + } else if (cmd->cmd == LC_SYMTAB) + orig_st = (struct symtab_command *) cmd; + + cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize); + } + + if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL)) + return; + + sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff); + strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff); + + /* Find extent of the TEXT section */ + instrLow = (uintptr_t) orig_ts->vmaddr; + instrHigh = (uintptr_t) (orig_ts->vmaddr + orig_ts->vmsize); + + for (i = 0; i < orig_st->nsyms; i++) { + uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT); + char *name = strings + sym[i].n_un.n_strx; + + /* Check that the symbol is a global and that it has a name. */ + if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type)) + continue; + + if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */ + continue; + + /* Lop off omnipresent leading underscore. */ + if (*name == '_') + name += 1; + + + if (sym[i].n_sect == 1 && !(sym[i].n_desc & N_ARM_THUMB_DEF)) { + /* A function but not a Thumb function */ + fbt_uninstrumented_arm++; + if (fbt_log_uninstrumented) + kprintf("dtrace: fbt: Skipping ARM mode function %s at %08x\n",name,(unsigned)sym[i].n_value); + + continue; + } + + /* + * We're only blacklisting functions in the kernel for now. + */ + if (MOD_IS_MACH_KERNEL(ctl) && fbt_excluded(name)) + continue; + + fbt_provide_probe(ctl, instrLow, instrHigh, modname, name, (machine_inst_t*)sym[i].n_value); + } +} diff --git a/bsd/dev/arm/kern_machdep.c b/bsd/dev/arm/kern_machdep.c new file mode 100644 index 000000000..312952ac9 --- /dev/null +++ b/bsd/dev/arm/kern_machdep.c @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (C) 1990, NeXT, Inc. + * + * File: next/kern_machdep.c + * Author: John Seamons + * + * Machine-specific kernel routines. + */ + +#include +#include +#include +#include + +#if __arm64__ +extern int bootarg_no64exec; /* bsd_init.c */ +static cpu_subtype_t cpu_subtype32(void); +#endif /* __arm64__ */ + +#if __arm64__ +/* + * When an arm64 CPU is executing an arm32 binary, we need to map from the + * host's 64-bit subtype to the appropriate 32-bit subtype. + */ +static cpu_subtype_t +cpu_subtype32() +{ + switch (cpu_subtype()) { + case CPU_SUBTYPE_ARM64_V8: + return CPU_SUBTYPE_ARM_V8; + default: + return 0; + } +} +#endif /* __arm64__*/ + +/********************************************************************** + * Routine: grade_binary() + * + * Function: Return a relative preference for exectypes and + * execsubtypes in fat executable files. The higher the + * grade, the higher the preference. A grade of 0 means + * not acceptable. + **********************************************************************/ +int +grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype) +{ +#if __arm64__ + cpu_subtype_t hostsubtype = (exectype & CPU_ARCH_ABI64) ? cpu_subtype() : cpu_subtype32(); +#else + cpu_subtype_t hostsubtype = cpu_subtype(); +#endif /* __arm64__ */ + + switch (exectype) { +#if __arm64__ + case CPU_TYPE_ARM64: + if (bootarg_no64exec) return 0; + + switch (hostsubtype) { + case CPU_SUBTYPE_ARM64_V8: + switch (execsubtype) { + case CPU_SUBTYPE_ARM64_V8: + return 9; + case CPU_SUBTYPE_ARM64_ALL: + return 8; + } + break; + + + break; +#else /* __arm64__ */ + + case CPU_TYPE_ARM: + switch (hostsubtype) { + /* + * For 32-bit ARMv8, try the ARMv8 slice before falling back to Swift. + */ + case CPU_SUBTYPE_ARM_V8: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V8: + return 7; + } + goto v7s; + + /* + * For Swift and later, we prefer to run a swift slice, but fall back + * to v7 as Cortex A9 errata should not apply + */ +v7s: + case CPU_SUBTYPE_ARM_V7S: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V7S: + return 6; + } + goto v7; + + /* + * For Cortex A7, accept v7k only due to differing ABI + */ + case CPU_SUBTYPE_ARM_V7K: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V7K: + return 6; + } + break; + + /* + * For Cortex A9, we prefer the A9 slice, but will run v7 albeit + * under the risk of hitting the NEON load/store errata + */ + case CPU_SUBTYPE_ARM_V7F: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V7F: + return 6; + } + goto v7; + +v7: + case CPU_SUBTYPE_ARM_V7: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V7: + return 5; + } + // fall through... + + case CPU_SUBTYPE_ARM_V6: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V6: + return 4; + } + // fall through... + + case CPU_SUBTYPE_ARM_V5TEJ: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V5TEJ: + return 3; + } + // fall through + + case CPU_SUBTYPE_ARM_V4T: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V4T: + return 2; + case CPU_SUBTYPE_ARM_ALL: + return 1; + } + break; + + case CPU_SUBTYPE_ARM_XSCALE: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_XSCALE: + return 4; + case CPU_SUBTYPE_ARM_V5TEJ: + return 3; + case CPU_SUBTYPE_ARM_V4T: + return 2; + case CPU_SUBTYPE_ARM_ALL: + return 1; + } + break; + } +#endif /* __arm64__ */ + } + + return 0; +} + +boolean_t +pie_required(cpu_type_t exectype, cpu_subtype_t execsubtype) +{ + switch (exectype) { +#if __arm64__ + case CPU_TYPE_ARM64: + return TRUE; +#endif /* __arm64__ */ + + case CPU_TYPE_ARM: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V7K: + return TRUE; + } + break; + } + return FALSE; +} diff --git a/bsd/dev/arm/km.c b/bsd/dev/arm/km.c new file mode 100644 index 000000000..cd77c9fb1 --- /dev/null +++ b/bsd/dev/arm/km.c @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. All rights reserved. + * + * km.m - kernel keyboard/monitor module, procedural interface. + * + * HISTORY + */ +#include +#include + +#include +#include +#include +#include +#include /* for kmopen */ +#include +#include /* for kmopen */ +#include +#include +#include +#include +#include + +extern int hz; + +extern void cnputcusr(char); +extern void cnputsusr(char *, int); +extern int cngetc(void); + + +void kminit(void); +void cons_cinput(char ch); + +/* + * 'Global' variables, shared only by this file and conf.c. + */ +struct tty *km_tty[1] = { 0 }; + +/* + * this works early on, after initialize_screen() but before autoconf (and thus + * before we have a kmDevice). + */ +int disableConsoleOutput; + +/* + * 'Global' variables, shared only by this file and kmDevice.m. + */ +int initialized = 0; + +static int kmoutput(struct tty * tp); +static void kmstart(struct tty * tp); + +extern void KeyboardOpen(void); + +void +kminit(void) +{ + km_tty[0] = ttymalloc(); + km_tty[0]->t_dev = makedev(12, 0); + initialized = 1; +} + +/* + * cdevsw interface to km driver. + */ +int +kmopen(dev_t dev, int flag, __unused int devtype, proc_t pp) +{ + int unit; + struct tty *tp; + struct winsize *wp; + int ret; + + unit = minor(dev); + if (unit >= 1) + return (ENXIO); + + tp = km_tty[unit]; + + tty_lock(tp); + + tp->t_oproc = kmstart; + tp->t_param = NULL; + tp->t_dev = dev; + + if (!(tp->t_state & TS_ISOPEN)) { + tp->t_iflag = TTYDEF_IFLAG; + tp->t_oflag = TTYDEF_OFLAG; + tp->t_cflag = (CREAD | CS8 | CLOCAL); + tp->t_lflag = TTYDEF_LFLAG; + tp->t_ispeed = tp->t_ospeed = TTYDEF_SPEED; + termioschars(&tp->t_termios); + ttsetwater(tp); + } else if ((tp->t_state & TS_XCLUDE) && proc_suser(pp)) { + ret = EBUSY; + goto out; + } + + tp->t_state |= TS_CARR_ON; /* lie and say carrier exists and is + * on. */ + ret = ((*linesw[tp->t_line].l_open) (dev, tp)); + { + PE_Video video; + wp = &tp->t_winsize; + /* + * Magic numbers. These are CHARWIDTH and CHARHEIGHT from + * pexpert/i386/video_console.c + */ + wp->ws_xpixel = 8; + wp->ws_ypixel = 16; + + tty_unlock(tp); /* XXX race window */ + + if (flag & O_POPUP) + PE_initialize_console(0, kPETextScreen); + + bzero(&video, sizeof(video)); + PE_current_console(&video); + + tty_lock(tp); + + if (serialmode & SERIALMODE_OUTPUT) { + wp->ws_col = 80; + wp->ws_row = 24; + } else if (video.v_width != 0 && video.v_height != 0) { + wp->ws_col = video.v_width / wp->ws_xpixel; + wp->ws_row = video.v_height / wp->ws_ypixel; + } else { + wp->ws_col = 100; + wp->ws_row = 36; + } + } + +out: + tty_unlock(tp); + + return ret; +} + +int +kmclose(dev_t dev, int flag, __unused int mode, __unused proc_t p) +{ + int ret; + struct tty *tp = km_tty[minor(dev)]; + + tty_lock(tp); + ret = (*linesw[tp->t_line].l_close)(tp, flag); + ttyclose(tp); + tty_unlock(tp); + + return (ret); +} + +int +kmread(dev_t dev, struct uio * uio, int ioflag) +{ + int ret; + struct tty *tp = km_tty[minor(dev)]; + + tty_lock(tp); + ret = (*linesw[tp->t_line].l_read)(tp, uio, ioflag); + tty_unlock(tp); + + return (ret); +} + +int +kmwrite(dev_t dev, struct uio * uio, int ioflag) +{ + int ret; + struct tty *tp = km_tty[minor(dev)]; + + tty_lock(tp); + ret = (*linesw[tp->t_line].l_write)(tp, uio, ioflag); + tty_unlock(tp); + + return (ret); +} + +int +kmioctl(dev_t dev, u_long cmd, caddr_t data, int flag, proc_t p) +{ + int error = 0; + struct tty *tp = km_tty[minor(dev)]; + struct winsize *wp; + + tty_lock(tp); + + switch (cmd) { + case KMIOCSIZE: + wp = (struct winsize *) data; + *wp = tp->t_winsize; + break; + + case TIOCSWINSZ: + /* + * Prevent changing of console size -- this ensures that + * login doesn't revert to the termcap-defined size + */ + error = EINVAL; + break; + + /* Bodge in the CLOCAL flag as the km device is always local */ + case TIOCSETA_32: + case TIOCSETAW_32: + case TIOCSETAF_32: + { + struct termios32 *t = (struct termios32 *)data; + t->c_cflag |= CLOCAL; + /* No Break */ + } + goto fallthrough; + case TIOCSETA_64: + case TIOCSETAW_64: + case TIOCSETAF_64: + { + struct user_termios *t = (struct user_termios *)data; + t->c_cflag |= CLOCAL; + /* No Break */ + } +fallthrough: + default: + error = (*linesw[tp->t_line].l_ioctl) (tp, cmd, data, flag, p); + if (ENOTTY != error) + break; + error = ttioctl_locked(tp, cmd, data, flag, p); + break; + } + + tty_unlock(tp); + + return (error); +} + + +/* + * kmputc + * + * Output a character to the serial console driver via cnputcusr(), + * which is exported by that driver. + * + * Locks: Assumes tp in the calling tty driver code is locked on + * entry, remains locked on exit + * + * Notes: Called from kmoutput(); giving the locking output + * assumptions here, this routine should be static (and + * inlined, given there is only one call site). + */ +int +kmputc(__unused dev_t dev, char c) +{ + if(!disableConsoleOutput && initialized) { + /* OCRNL */ + if(c == '\n') + cnputcusr('\r'); + cnputcusr(c); + } + + return (0); +} + + +/* + * Callouts from linesw. + */ + +#define KM_LOWAT_DELAY ((ns_time_t)1000) + +/* + * t_oproc for this driver; called from within the line discipline + * + * Locks: Assumes tp is locked on entry, remains locked on exit + */ +static void +kmstart(struct tty *tp) +{ + if (tp->t_state & (TS_TIMEOUT | TS_BUSY | TS_TTSTOP)) + goto out; + if (tp->t_outq.c_cc == 0) + goto out; + tp->t_state |= TS_BUSY; + if (tp->t_outq.c_cc > tp->t_lowat) { + /* + * Start immediately. + */ + kmoutput(tp); + } else { + /* + * Wait a bit... + */ +#if 0 + /* FIXME */ + timeout(kmtimeout, tp, hz); +#else + kmoutput(tp); +#endif + } + return; + +out: + (*linesw[tp->t_line].l_start) (tp); + return; +} + +/* + * One-shot output retry timeout from kmoutput(); re-calls kmoutput() at + * intervals until the output queue for the tty is empty, at which point + * the timeout is not rescheduled by kmoutput() + * + * This function must take the tty_lock() around the kmoutput() call; it + * ignores the return value. + */ +static void +kmtimeout(void *arg) +{ + struct tty *tp = (struct tty *)arg; + + tty_lock(tp); + (void)kmoutput(tp); + tty_unlock(tp); +} + +/* + * kmoutput + * + * Locks: Assumes tp is locked on entry, remains locked on exit + * + * Notes: Called from kmstart() and kmtimeout(); kmtimeout() is a + * timer initiated by this routine to deal with pending + * output not yet flushed (output is flushed at a maximum + * of sizeof(buf) charatcers at a time before dropping into + * the timeout code). + */ +static int +kmoutput(struct tty * tp) +{ + unsigned char buf[80]; /* buffer; limits output per call */ + unsigned char *cp; + int cc = -1; + + /* While there is data available to be output... */ + while (tp->t_outq.c_cc > 0) { + cc = ndqb(&tp->t_outq, 0); + if (cc == 0) + break; + /* + * attempt to output as many characters as are available, + * up to the available transfer buffer size. + */ + cc = min(cc, sizeof(buf)); + /* copy the output queue contents to the buffer */ + (void) q_to_b(&tp->t_outq, buf, cc); + for (cp = buf; cp < &buf[cc]; cp++) { + /* output the buffer one charatcer at a time */ + *cp = *cp & 0x7f; + } + if (cc > 1) { + cnputsusr((char *)buf, cc); + } else { + kmputc(tp->t_dev, *buf); + } + } + /* + * XXX This is likely not necessary, as the tty output queue is not + * XXX writeable while we hold the tty_lock(). + */ + if (tp->t_outq.c_cc > 0) { + timeout(kmtimeout, tp, hz); + } + tp->t_state &= ~TS_BUSY; + /* Start the output processing for the line discipline */ + (*linesw[tp->t_line].l_start) (tp); + + return 0; +} + + +/* + * cons_cinput + * + * Driver character input from the polled mode serial console driver calls + * this routine to input a character from the serial driver into the tty + * line discipline specific input processing receiv interrupt routine, + * l_rint(). + * + * Locks: Assumes that the tty_lock() is NOT held on the tp, so a + * serial driver should NOT call this function as a result + * of being called from a function which already holds the + * lock; ECHOE will be handled at the line discipline, if + * output echo processing is going to occur. + */ +void +cons_cinput(char ch) +{ + struct tty *tp = km_tty[0]; /* XXX */ + + tty_lock(tp); + (*linesw[tp->t_line].l_rint) (ch, tp); + tty_unlock(tp); +} diff --git a/bsd/dev/arm/munge.c b/bsd/dev/arm/munge.c new file mode 100644 index 000000000..d98953ad2 --- /dev/null +++ b/bsd/dev/arm/munge.c @@ -0,0 +1,767 @@ +/* + * Coyright (c) 2005-2015 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + * For arm32 ABI where 64-bit types are aligned to even registers and + * 64-bits on stack, we need to unpack registers differently. So + * we use the mungers for that. Currently this is just ARMv7k. + * + * Since arm32 has no need for munging otherwise, we don't include + * any of this for other arm32 ABIs + */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) + +#include +#include +#include +#include +#include + + +/* + * Userspace args are in r0-r6, then r8, then stack unless this is an + * indirect call in which case the syscall number is in r0 then args + * are in registers r1-r6, then r8, then stack. This is for mach and + * BSD style syscalls. + */ + + +#define SS_TO_STYLE(ss) ((ss->r[12] != 0) ? kDirect : kIndirect) +#define REGS_TO_STYLE(regs) (SS_TO_STYLE(((const arm_saved_state_t *)regs))) + +typedef enum { + kIndirect = 0, + kDirect +} style_t; + +#define DECLARE_AND_CAST(regs, args, ss, uu_args) const arm_saved_state_t *ss = (const arm_saved_state_t *)regs; \ + uint32_t *uu_args = (uint32_t *)args; + +/* + * We start 32 bytes after sp since 4 registers are pushed onto the stack + * in the userspace syscall handler, and the first 4 stack argumnets are moved + * into registers already + */ +#define ARG_SP_BYTE_OFFSET 32 + + +/* + * Marshal in arguments from userspace where no padding exists + */ + +static int +marshal_no_pad(const arm_saved_state_t *ss, uint32_t *args, const uint32_t word_count) +{ + int error = 0; + /* init assuming kDirect style */ + uint32_t copy_count, contiguous_reg_count = 7, contiguous_reg_start = 0; + style_t style = SS_TO_STYLE(ss); + + if (style == kIndirect) { + contiguous_reg_count--; + contiguous_reg_start++; + } + + /* r0 through r6 */ + copy_count = MIN(word_count, contiguous_reg_count); + memcpy(args, &(ss->r[contiguous_reg_start]), copy_count * sizeof(uint32_t)); + args += copy_count; + + if (word_count > copy_count) { + /* r8 */ + *args = ss->r[8]; + args++; + copy_count++; + + /* stack */ + if (word_count > copy_count) { + error = copyin(ss->sp + ARG_SP_BYTE_OFFSET, + args, (word_count - copy_count) * sizeof(uint32_t)); + if (error) + return error; + } + } + return error; +} + +/* + * Define mungers to marshal userspace data into argument structs + */ + +int +munge_w(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 1); +} + +int +munge_ww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 2); +} + +int +munge_www(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 3); +} + +int +munge_wwww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 4); +} + +int +munge_wwwww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 5); +} + +int +munge_wwwwww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 6); +} + +int +munge_wwwwwww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 7); +} + +int +munge_wwwwwwww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 8); +} + +int +munge_wwl(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 3); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[4]; // l (longs are aligned to even registers for armv7k, so skip r3) + uu_args[3] = ss->r[5]; // + return 0; + } +} + +int +munge_wwlw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 5); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwl(regs, args); // wwl + uu_args[4] = ss->r[6]; // w + return error; + } +} + +int +munge_wwlww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + // the long-long here is aligned on an even register + // so there shouldn't be any padding + return marshal_no_pad(regs, args, 6); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwlw(regs, args); // wwlw + uu_args[5] = ss->r[8]; // w + return error; + } +} + +int +munge_wwlll(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 8); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwl(regs, args); // wwl + if (error) + return error; + uu_args[4] = ss->r[6]; // l + uu_args[5] = ss->r[8]; // + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + &(uu_args[6]), 2 * sizeof(uint32_t)); + } +} + +int +munge_wwllww(const void *regs, void *args) +{ + return munge_wwlll(regs, args); +} + +int +munge_wl(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + memcpy(args, regs, 4 * sizeof(uint32_t)); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + } + return 0; +} + +int +munge_wlw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + memcpy(args, regs, 5 * sizeof(uint32_t)); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + } + return 0; +} + +int +munge_wlww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + memcpy(args, regs, 6 * sizeof(uint32_t)); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + uu_args[5] = ss->r[5]; // w + } + return 0; +} + +int +munge_wlwwwll(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + if (REGS_TO_STYLE(regs) == kDirect) { + memcpy(args, regs, 7 * sizeof(uint32_t)); // wlwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + uu_args + 8, 4 * sizeof(uint32_t)); + } + else { + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + uu_args[5] = ss->r[5]; // w + uu_args[6] = ss->r[6]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + uu_args + 8, 4 * sizeof(uint32_t)); + } +} + +int +munge_wlwwwllw(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + if (REGS_TO_STYLE(regs) == kDirect) { + memcpy(args, regs, 7 * sizeof(uint32_t)); // wlwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, + uu_args + 8, 5 * sizeof(uint32_t)); // ll + } + else { + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + uu_args[5] = ss->r[5]; // w + uu_args[6] = ss->r[6]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // llw + uu_args + 8, 5 * sizeof(uint32_t)); + } +} + +int +munge_wlwwlwlw(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + if (REGS_TO_STYLE(regs) == kDirect) + uu_args[0] = ss->r[0]; // w + else + uu_args[0] = ss->r[1]; // w + + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + uu_args[5] = ss->r[5]; // w + uu_args[6] = ss->r[6]; // l + uu_args[7] = ss->r[8]; // + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // wlw + uu_args + 8, 5 * sizeof(uint32_t)); +} + +int +munge_wll(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + memcpy(args, regs, 6 * sizeof(uint32_t)); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // l + uu_args[5] = ss->r[5]; // + } + return 0; +} + +int +munge_wlll(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wll(regs, args); // wll + uu_args[6] = ss->r[6]; // l + uu_args[7] = ss->r[8]; // + return error; +} + +int +munge_wllll(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + munge_wlll(regs, args); // wlll + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + uu_args + 8, 2 * sizeof(uint32_t)); +} + +int +munge_wllww(const void *regs, void *args) +{ + return munge_wlll(regs, args); +} + +int +munge_wllwwll(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wlll(regs, args); // wllww + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + uu_args + 8, 4 * sizeof(uint32_t)); +} + +int +munge_wwwlw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + memcpy(args, regs, 7 * sizeof(uint32_t)); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[3]; // w + uu_args[4] = ss->r[4]; // l + uu_args[5] = ss->r[5]; // + uu_args[6] = ss->r[6]; // w + } + return 0; +} + +int +munge_wwwlww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return munge_wlll(regs, args); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[3]; // w + uu_args[4] = ss->r[4]; // l + uu_args[5] = ss->r[5]; // + uu_args[6] = ss->r[6]; // w + uu_args[7] = ss->r[8]; // w + return 0; + } +} + +int +munge_wwwl(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return munge_wll(regs, args); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[3]; // w + uu_args[4] = ss->r[4]; // l + uu_args[5] = ss->r[5]; // + return 0; + } +} + +int +munge_wwwwl(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 6); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[3]; // w + uu_args[3] = ss->r[4]; // w + uu_args[4] = ss->r[6]; // l + uu_args[5] = ss->r[8]; // + return 0; + } +} + +int +munge_wwwwlw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 7); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwwwl(regs, args); // wwwwl + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // w + uu_args + 6, sizeof(uint32_t)); + } +} + +int +munge_wwwwwl(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return munge_wlll(regs, args); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[3]; // w + uu_args[3] = ss->r[4]; // w + uu_args[4] = ss->r[5]; // w + uu_args[6] = ss->r[6]; // l + uu_args[7] = ss->r[8]; // + return 0; + } +} + +int +munge_wwwwwlww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return munge_wllll(regs, args); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwwwwl(regs, args); // wwwwwl + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ww + uu_args + 8, 2 * sizeof(uint32_t)); + } +} + +int +munge_wwwwwllw(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwwwwl(regs, args); // wwwwwl + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // lw + uu_args + 8, 3 * sizeof(uint32_t)); +} + +int +munge_wwwwwlll(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + int error; + + if (REGS_TO_STYLE(regs) == kDirect) { + error = munge_wlll(regs, args); // wlll + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + uu_args + 8, 4 * sizeof(uint32_t)); + } + else { + error = munge_wwwwwl(regs, args); // wwwwwl + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + uu_args + 8, 4 * sizeof(uint32_t)); + } +} + +int +munge_wwwwwwl(const void *regs, void *args) +{ + munge_wwlll(regs, args); + + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 8); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + memcpy(args, &(ss->r[1]), 6 * sizeof(uint32_t)); // wwwwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + &(uu_args[6]), 2 * sizeof(uint32_t)); + } +} + +int +munge_wwwwwwlw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 9); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + memcpy(args, &(ss->r[1]), 6 * sizeof(uint32_t)); // wwwwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // lw + &(uu_args[6]), 3 * sizeof(uint32_t)); + } +} + +int +munge_wwwwwwll(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 10); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + memcpy(args, &(ss->r[1]), 6 * sizeof(uint32_t)); // wwwwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + &(uu_args[6]), 4 * sizeof(uint32_t)); + } +} + +int +munge_wsw(const void *regs, void *args) +{ + return munge_wlw(regs, args); +} + +int +munge_wws(const void *regs, void *args) +{ + return munge_wwl(regs, args); +} + +int +munge_wwws(const void *regs, void *args) +{ + return munge_wwwl(regs, args); +} + +int +munge_wwwsw(const void *regs, void *args) +{ + return munge_wwwlw(regs, args); +} + +int +munge_llllll(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 12); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[2]; // l + uu_args[1] = ss->r[3]; // + uu_args[2] = ss->r[4]; // l + uu_args[3] = ss->r[5]; // + uu_args[4] = ss->r[6]; // l + uu_args[5] = ss->r[8]; // + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // lll + uu_args + 6, 6 * sizeof(uint32_t)); + } +} + +int +munge_ll(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 4); + else + memcpy(args, (const uint32_t*)regs + 2, 4 * sizeof(uint32_t)); + return 0; +} + +int +munge_l(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 2); + else + memcpy(args, (const uint32_t*)regs + 2, 2 * sizeof(uint32_t)); + return 0; +} + +int +munge_lw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 3); + else + memcpy(args, (const uint32_t*)regs + 2, 3 * sizeof(uint32_t)); + return 0; +} + +int +munge_lwww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 5); + else + memcpy(args, (const uint32_t*)regs + 2, 5 * sizeof(uint32_t)); + return 0; +} + +int +munge_lwwwwwww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 9); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[2]; // l + uu_args[1] = ss->r[3]; // + uu_args[2] = ss->r[4]; // w + uu_args[3] = ss->r[5]; // w + uu_args[4] = ss->r[6]; // w + uu_args[5] = ss->r[8]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // www + uu_args + 6, 3 * sizeof(uint32_t)); + } +} + +int +munge_wwlwww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 7); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[4]; // l + uu_args[3] = ss->r[5]; // + uu_args[4] = ss->r[6]; // w + uu_args[5] = ss->r[8]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // w + uu_args + 6, sizeof(uint32_t)); + } + +} + +int +munge_wlwwwl(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + if (REGS_TO_STYLE(regs) == kDirect) { + memcpy(args, regs, 7 * sizeof(uint32_t)); // wlwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + uu_args + 8, 2 * sizeof(uint32_t)); + } else { + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + uu_args[5] = ss->r[5]; // w + uu_args[6] = ss->r[6]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + uu_args + 8, 2 * sizeof(uint32_t)); + } +} + +int +munge_wwlwwwl(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + if (REGS_TO_STYLE(regs) == kDirect) { + memcpy(args, regs, 7 * sizeof(uint32_t)); // wwlwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + uu_args + 8, 2 * sizeof(uint32_t)); + } else { + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[4]; // l + uu_args[3] = ss->r[5]; // + uu_args[4] = ss->r[6]; // w + uu_args[5] = ss->r[8]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // wl + uu_args + 6, 4 * sizeof(uint32_t)); + } +} + +#endif // __arm__ && (__BIGGEST_ALIGNMENT__ > 4) diff --git a/bsd/dev/arm/pci_device.h b/bsd/dev/arm/pci_device.h new file mode 100644 index 000000000..32844c3ce --- /dev/null +++ b/bsd/dev/arm/pci_device.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * @OSF_FREE_COPYRIGHT@ + * + */ +/* + * HISTORY + * + * Revision 1.2 1998/09/30 21:20:44 wsanchez + * Merged in IntelMerge1 (mburg: Intel support) + * + * Revision 1.1.2.1 1998/09/30 18:18:50 mburg + * Changes for Intel port + * + * Revision 1.1.1.1 1998/03/07 02:25:45 wsanchez + * Import of OSF Mach kernel (~mburg) + * + * Revision 1.1.6.2 1995/12/15 10:52:14 bernadat + * Split dev and vendor ids. + * [95/11/15 bernadat] + * + * Revision 1.1.6.1 1995/02/23 17:22:27 alanl + * Taken from DIPC2_SHARED + * [1995/01/03 19:09:31 alanl] + * + * Revision 1.1.2.1 1994/10/11 18:24:42 rwd + * Created. + * [1994/10/11 18:15:31 rwd] + * + * $EndLog$ + */ +/* + * Taken from + * + * Copyright (c) 1994 Wolfgang Stanglmeier, Koeln, Germany + * + */ + +#ifndef __PCI_DEVICE_H__ +#define __PCI_DEVICE_H__ + +/*------------------------------------------------------------ + * + * Per driver structure. + * + *------------------------------------------------------------ +*/ + +typedef unsigned short pci_vendor_id_t; +typedef unsigned short pci_dev_id_t; + +typedef union { + unsigned long cfg1; + struct { + unsigned char enable; + unsigned char forward; + unsigned short port; + } cfg2; + } pcici_t; + +struct pci_driver { + int (*probe )(pcici_t pci_ident); /* test whether device + is present */ + int (*attach)(pcici_t pci_ident); /* setup driver for a + device */ + pci_vendor_id_t vendor_id; /* vendor pci id */ + pci_dev_id_t device_id; /* device pci id */ + char *name; /* device name */ + char *vendor; /* device long name */ + void (*intr)(int); /* interupt handler */ +}; + +/*----------------------------------------------------------- + * + * Per device structure. + * + * It is initialized by the config utility and should live in + * "ioconf.c". At the moment there is only one field. + * + * This is a first attempt to include the pci bus to 386bsd. + * So this structure may grow .. + * + *----------------------------------------------------------- +*/ + +struct pci_device { + struct pci_driver * pd_driver; +}; + +/*----------------------------------------------------------- + * + * This functions may be used by drivers to map devices + * to virtual and physical addresses. The va and pa + * addresses are "in/out" parameters. If they are 0 + * on entry, the mapping function assigns an address. + * + *----------------------------------------------------------- +*/ + +int pci_map_mem(pcici_t tag, + unsigned long entry, + vm_offset_t *va, + vm_offset_t *pa); +#endif /*__PCI_DEVICE_H__*/ diff --git a/bsd/dev/arm/pio.h b/bsd/dev/arm/pio.h new file mode 100644 index 000000000..fd9c1ecca --- /dev/null +++ b/bsd/dev/arm/pio.h @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2000-2007 AppleInc. All rights reserved. + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * HISTORY + * + * Revision 1.2 1998/09/30 21:20:45 wsanchez + * Merged in IntelMerge1 (mburg: Intel support) + * + * Revision 1.1.2.1 1998/09/30 18:18:50 mburg + * Changes for Intel port + * + * Revision 1.1.1.1 1998/03/07 02:25:38 wsanchez + * Import of OSF Mach kernel (~mburg) + * + * Revision 1.1.8.2 1996/07/31 09:46:36 paire + * Merged with nmk20b7_shared (1.1.11.2 -> 1.1.11.1) + * [96/06/10 paire] + * + * Revision 1.1.11.2 1996/06/13 12:38:25 bernadat + * Do not use inline macros when MACH_ASSERT is configured. + * [96/05/24 bernadat] + * + * Revision 1.1.11.1 1996/05/14 13:50:23 paire + * Added new linl and loutl __inline__. + * Added conditional compilation for [l]{in|oub}[bwl]() __inline__. + * [95/11/24 paire] + * + * Revision 1.1.8.1 1994/09/23 02:00:28 ezf + * change marker to not FREE + * [1994/09/22 21:25:52 ezf] + * + * Revision 1.1.4.5 1993/08/09 19:40:41 dswartz + * Add ANSI prototypes - CR#9523 + * [1993/08/06 17:45:57 dswartz] + * + * Revision 1.1.4.4 1993/06/11 15:17:37 jeffc + * CR9176 - ANSI C violations: inb/outb macros must be changed from + * ({ ... }) to inline functions, with proper type definitions. Callers + * must pass proper types to these functions: 386 I/O port addresses + * are unsigned shorts (not pointers). + * [1993/06/10 14:26:10 jeffc] + * + * Revision 1.1.4.3 1993/06/07 22:09:28 jeffc + * CR9176 - ANSI C violations: trailing tokens on CPP + * directives, extra semicolons after decl_ ..., asm keywords + * [1993/06/07 19:00:26 jeffc] + * + * Revision 1.1.4.2 1993/06/04 15:28:45 jeffc + * CR9176 - ANSI problems - + * Added casts to get macros to take caddr_t as an I/O space address. + * [1993/06/04 13:45:55 jeffc] + * + * Revision 1.1 1992/09/30 02:25:51 robert + * Initial revision + * + * $EndLog$ + */ +/* CMU_HIST */ +/* + * Revision 2.5 91/05/14 16:14:20 mrt + * Correcting copyright + * + * Revision 2.4 91/02/05 17:13:56 mrt + * Changed to new Mach copyright + * [91/02/01 17:37:08 mrt] + * + * Revision 2.3 90/12/20 16:36:37 jeffreyh + * changes for __STDC__ + * [90/12/07 jeffreyh] + * + * Revision 2.2 90/11/26 14:48:41 rvb + * Pulled from 2.5 + * [90/11/22 10:09:38 rvb] + * + * [90/08/14 mg32] + * + * Now we know how types are factor in. + * Cleaned up a bunch: eliminated ({ for output and flushed unused + * output variables. + * [90/08/14 rvb] + * + * This is how its done in gcc: + * Created. + * [90/03/26 rvb] + * + */ +/* CMU_ENDHIST */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ +#ifndef ARM_PIO_H +#define ARM_PIO_H + +typedef unsigned short i386_ioport_t; + +/* read a longword */ +extern unsigned long inl( + i386_ioport_t port); +/* read a shortword */ +extern unsigned short inw( + i386_ioport_t port); +/* read a byte */ +extern unsigned char inb( + i386_ioport_t port); +/* write a longword */ +extern void outl( + i386_ioport_t port, + unsigned long datum); +/* write a word */ +extern void outw( + i386_ioport_t port, + unsigned short datum); +/* write a longword */ +extern void outb( + i386_ioport_t port, + unsigned char datum); + +/* input an array of longwords */ +extern void linl( + i386_ioport_t port, + int * data, + int count); +/* output an array of longwords */ +extern void loutl( + i386_ioport_t port, + int * data, + int count); + +/* input an array of words */ +extern void linw( + i386_ioport_t port, + int * data, + int count); +/* output an array of words */ +extern void loutw( + i386_ioport_t port, + int * data, + int count); + +/* input an array of bytes */ +extern void linb( + i386_ioport_t port, + char * data, + int count); +/* output an array of bytes */ +extern void loutb( + i386_ioport_t port, + char * data, + int count); + +extern __inline__ unsigned long inl( + i386_ioport_t port) +{ + unsigned long datum; + __asm__ volatile("inl %1, %0" : "=a" (datum) : "d" (port)); + return(datum); +} + +extern __inline__ unsigned short inw( + i386_ioport_t port) +{ + unsigned short datum; + __asm__ volatile(".byte 0x66; inl %1, %0" : "=a" (datum) : "d" (port)); + return(datum); +} + +extern __inline__ unsigned char inb( + i386_ioport_t port) +{ + unsigned char datum; + __asm__ volatile("inb %1, %0" : "=a" (datum) : "d" (port)); + return(datum); +} + +extern __inline__ void outl( + i386_ioport_t port, + unsigned long datum) +{ + __asm__ volatile("outl %0, %1" : : "a" (datum), "d" (port)); +} + +extern __inline__ void outw( + i386_ioport_t port, + unsigned short datum) +{ + __asm__ volatile(".byte 0x66; outl %0, %1" : : "a" (datum), "d" (port)); +} + +extern __inline__ void outb( + i386_ioport_t port, + unsigned char datum) +{ + __asm__ volatile("outb %0, %1" : : "a" (datum), "d" (port)); +} + +#endif /* ARM_PIO_H */ diff --git a/bsd/dev/arm/sdt_arm.c b/bsd/dev/arm/sdt_arm.c new file mode 100644 index 000000000..b8db51b52 --- /dev/null +++ b/bsd/dev/arm/sdt_arm.c @@ -0,0 +1,166 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* #pragma ident "@(#)sdt.c 1.6 06/03/24 SMI" */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */ +#include +#include +#include +#include + +#include +#include + +#include + +#include + +extern sdt_probe_t **sdt_probetab; + +int +sdt_invop(__unused uintptr_t addr, __unused uintptr_t *stack, __unused uintptr_t eax) +{ +#pragma unused(eax) + sdt_probe_t *sdt = sdt_probetab[SDT_ADDR2NDX(addr)]; + + for (; sdt != NULL; sdt = sdt->sdp_hashnext) { + if ((uintptr_t) sdt->sdp_patchpoint == addr) { + struct arm_saved_state* regs = (struct arm_saved_state*) stack; + uintptr_t stack4 = *((uintptr_t*) regs->sp); + + dtrace_probe(sdt->sdp_id, regs->r[0], regs->r[1], regs->r[2], regs->r[3], stack4); + + return (DTRACE_INVOP_NOP); + } + } + + return (0); +} + +struct frame { + struct frame *backchain; + uintptr_t retaddr; +}; + +/*ARGSUSED*/ +uint64_t +sdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ +#pragma unused(arg,id,parg) /* __APPLE__ */ + uint64_t val = 0; + struct frame *fp = (struct frame *)__builtin_frame_address(0); + uintptr_t *stack; + uintptr_t pc; + int i; + + /* + * On ARM, up to four args are passed via registers; r0,r1,r2,r3 + * So coming into this function, arg >= 4 should be on the stack. + * e.g. arg==5 refers to the 6th arg passed to the probed function. + */ + int inreg = 4; + + for (i = 1; i <= aframes; i++) { + fp = fp->backchain; + pc = fp->retaddr; + + if (dtrace_invop_callsite_pre != NULL + && pc > (uintptr_t)dtrace_invop_callsite_pre + && pc <= (uintptr_t)dtrace_invop_callsite_post) { + + /* + * When we pass through the invalid op handler, + * we expect to find the save area structure, + * pushed on the stack where we took the trap. + * If the argument we seek is passed in a register, then + * we can load it directly from this saved area. + * If the argument we seek is passed on the stack, then + * we increment the frame pointer further, to find the + * pushed args + */ + + /* fp points to the dtrace_invop activation */ + fp = fp->backchain; /* to the fbt_perfCallback activation */ + fp = fp->backchain; /* to the sleh_undef activation */ + +#if __BIGGEST_ALIGNMENT__ > 4 + /** + * rdar://problem/24228656: On armv7k, the stack is realigned in sleh_undef2 to + * be 16-bytes aligned and the old value is pushed to + * the stack, so we retrieve it from here + */ + arm_saved_state_t *saved_state = (arm_saved_state_t *)(uintptr_t*)*((uintptr_t *)&fp[1]); +#else + arm_saved_state_t *saved_state = (arm_saved_state_t *)((uintptr_t *)&fp[1]); +#endif + if (argno <= inreg) { + /* For clarity only... should not get here */ + stack = (uintptr_t *)&saved_state->r[0]; + } else { + fp = (struct frame *)(saved_state->sp); + stack = (uintptr_t *)&fp[0]; /* Find marshalled arguments */ + argno -= inreg; + } + goto load; + } + } + + /* + * We know that we did not come through a trap to get into + * dtrace_probe() -- We arrive here when the provider has + * called dtrace_probe() directly. + * The probe ID is the first argument to dtrace_probe(). + * We must advance beyond that to get the argX. + */ + argno++; /* Advance past probeID */ + + if (argno <= inreg) { + /* + * This shouldn't happen. If the argument is passed in a + * register then it should have been, well, passed in a + * register... + */ + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + argno -= (inreg + 1); + stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ + +load: + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */ + val = (uint64_t)(*(((uintptr_t *)stack) + argno)); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + return (val); + +} diff --git a/bsd/dev/arm/stubs.c b/bsd/dev/arm/stubs.c new file mode 100644 index 000000000..644dae630 --- /dev/null +++ b/bsd/dev/arm/stubs.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1997 by Apple Computer, Inc., all rights reserved + * Copyright (c) 1993 NeXT Computer, Inc. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * copy a null terminated string from the kernel address space into the user + * address space. - if the user is denied write access, return EFAULT. - if + * the end of string isn't found before maxlen bytes are copied, return + * ENAMETOOLONG, indicating an incomplete copy. - otherwise, return 0, + * indicating success. the number of bytes copied is always returned in + * lencopied. + */ +int +copyoutstr(const void *from, user_addr_t to, size_t maxlen, size_t * lencopied) +{ + size_t slen; + size_t len; + int error = 0; + + slen = strlen(from) + 1; + if (slen > maxlen) + error = ENAMETOOLONG; + + len = min(maxlen, slen); + if (copyout(from, to, len)) + error = EFAULT; + *lencopied = len; + + return error; +} + + +/* + * copy a null terminated string from one point to another in the kernel + * address space. - no access checks are performed. - if the end of string + * isn't found before maxlen bytes are copied, return ENAMETOOLONG, + * indicating an incomplete copy. - otherwise, return 0, indicating success. + * the number of bytes copied is always returned in lencopied. + */ +/* from ppc/fault_copy.c -Titan1T4 VERSION */ +int +copystr(const void *vfrom, void *vto, size_t maxlen, size_t * lencopied) +{ + size_t l; + char const *from = (char const *) vfrom; + char *to = (char *) vto; + + for (l = 0; l < maxlen; l++) { + if ((*to++ = *from++) == '\0') { + if (lencopied) + *lencopied = l + 1; + return 0; + } + } + if (lencopied) + *lencopied = maxlen; + return ENAMETOOLONG; +} + +int +copywithin(void *src, void *dst, size_t count) +{ + bcopy(src, dst, count); + return 0; +} diff --git a/bsd/dev/arm/sysctl.c b/bsd/dev/arm/sysctl.c new file mode 100644 index 000000000..a1ee66f16 --- /dev/null +++ b/bsd/dev/arm/sysctl.c @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2003-2007 Apple Inc. All rights reserved. + */ +#include +#include +#include + +#include + +extern int trap_on_alignment_fault; +extern uint64_t wake_abstime; + +static +SYSCTL_INT(_machdep, OID_AUTO, alignmenttrap, + CTLFLAG_RW, &trap_on_alignment_fault, 0, + "trap on alignment faults (number of alignment faults per trap)"); + +static +SYSCTL_QUAD(_machdep, OID_AUTO, wake_abstime, + CTLFLAG_RD | CTLFLAG_KERN, &wake_abstime, + "Absolute Time at the last wakeup"); + +static int +sysctl_time_since_reset SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2, oidp) + int error = 0; + uint64_t return_value = 0; + + return_value = ml_get_time_since_reset(); + + SYSCTL_OUT(req, &return_value, sizeof(return_value)); + + return error; +} + +SYSCTL_PROC(_machdep, OID_AUTO, time_since_reset, + CTLFLAG_RD | CTLTYPE_QUAD | CTLFLAG_LOCKED, + 0, 0, sysctl_time_since_reset, "I", + "Continuous time since last SOC boot/wake started"); + +static int +sysctl_wake_conttime SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2, oidp) + int error = 0; + uint64_t return_value = 0; + + return_value = ml_get_conttime_wake_time(); + + SYSCTL_OUT(req, &return_value, sizeof(return_value)); + + return error; +} + +SYSCTL_PROC(_machdep, OID_AUTO, wake_conttime, + CTLFLAG_RD | CTLTYPE_QUAD | CTLFLAG_LOCKED, + 0, 0, sysctl_wake_conttime, "I", + "Continuous Time at the last wakeup"); + diff --git a/bsd/dev/arm/systemcalls.c b/bsd/dev/arm/systemcalls.c new file mode 100644 index 000000000..df9f22d09 --- /dev/null +++ b/bsd/dev/arm/systemcalls.c @@ -0,0 +1,651 @@ +/* + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if CONFIG_DTRACE +extern int32_t dtrace_systrace_syscall(struct proc *, void *, int *); +extern void dtrace_systrace_syscall_return(unsigned short, int, int *); +#endif /* CONFIG_DTRACE */ + +extern void +unix_syscall(struct arm_saved_state * regs, thread_t thread_act, + struct uthread * uthread, struct proc * proc); + +static int arm_get_syscall_args(uthread_t, struct arm_saved_state *, struct sysent *); +static int arm_get_u32_syscall_args(uthread_t, arm_saved_state32_t *, struct sysent *); +static void arm_prepare_u32_syscall_return(struct sysent *, arm_saved_state32_t *, uthread_t, int); +static void arm_prepare_syscall_return(struct sysent *, struct arm_saved_state *, uthread_t, int); +static int arm_get_syscall_number(struct arm_saved_state *); +static void arm_trace_unix_syscall(int, struct arm_saved_state *); +static void arm_clear_syscall_error(struct arm_saved_state *); +#define save_r0 r[0] +#define save_r1 r[1] +#define save_r2 r[2] +#define save_r3 r[3] +#define save_r4 r[4] +#define save_r5 r[5] +#define save_r6 r[6] +#define save_r7 r[7] +#define save_r8 r[8] +#define save_r9 r[9] +#define save_r10 r[10] +#define save_r11 r[11] +#define save_r12 r[12] +#define save_r13 r[13] + +#if COUNT_SYSCALLS +__XNU_PRIVATE_EXTERN int do_count_syscalls = 1; +__XNU_PRIVATE_EXTERN int syscalls_log[SYS_MAXSYSCALL]; +#endif + +#define code_is_kdebug_trace(code) (((code) == SYS_kdebug_trace) || \ + ((code) == SYS_kdebug_trace64) || \ + ((code) == SYS_kdebug_trace_string)) + +/* + * Function: unix_syscall + * + * Inputs: regs - pointer to Process Control Block + * + * Outputs: none + */ +#ifdef __arm__ +__attribute__((noreturn)) +#endif +void +unix_syscall( + struct arm_saved_state * state, + __unused thread_t thread_act, + struct uthread * uthread, + struct proc * proc) +{ + struct sysent *callp; + int error; + unsigned short code; + pid_t pid; + +#if defined(__arm__) + assert(is_saved_state32(state)); +#endif + + uthread_reset_proc_refcount(uthread); + + code = arm_get_syscall_number(state); + +#define unix_syscall_kprintf(x...) /* kprintf("unix_syscall: " x) */ + +#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_IST) + if (kdebug_enable && !code_is_kdebug_trace(code)) { + arm_trace_unix_syscall(code, state); + } +#endif + + if ((uthread->uu_flag & UT_VFORK)) + proc = current_proc(); + + callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code]; + + /* + * sy_narg is inaccurate on ARM if a 64 bit parameter is specified. Since user_addr_t + * is currently a 32 bit type, this is really a long word count. See rdar://problem/6104668. + */ + if (callp->sy_narg != 0) { + if (arm_get_syscall_args(uthread, state, callp) != 0) { + /* Too many arguments, or something failed */ + unix_syscall_kprintf("arm_get_syscall_args failed.\n"); + callp = &sysent[SYS_invalid]; + } + } + + uthread->uu_flag |= UT_NOTCANCELPT; + uthread->syscall_code = code; + + uthread->uu_rval[0] = 0; + + /* + * r4 is volatile, if we set it to regs->save_r4 here the child + * will have parents r4 after execve + */ + uthread->uu_rval[1] = 0; + + error = 0; + + /* + * ARM runtime will call cerror if the carry bit is set after a + * system call, so clear it here for the common case of success. + */ + arm_clear_syscall_error(state); + +#if COUNT_SYSCALLS + if (do_count_syscalls > 0) { + syscalls_log[code]++; + } +#endif + pid = proc_pid(proc); + +#ifdef JOE_DEBUG + uthread->uu_iocount = 0; + uthread->uu_vpindex = 0; +#endif + unix_syscall_kprintf("code %d (pid %d - %s, tid %lld)\n", code, + pid, proc->p_comm, thread_tid(current_thread())); + + AUDIT_SYSCALL_ENTER(code, proc, uthread); + error = (*(callp->sy_call)) (proc, &uthread->uu_arg[0], &(uthread->uu_rval[0])); + AUDIT_SYSCALL_EXIT(code, proc, uthread, error); + + unix_syscall_kprintf("code %d, error %d, results %x, %x (pid %d - %s, tid %lld)\n", code, error, + uthread->uu_rval[0], uthread->uu_rval[1], + pid, get_bsdtask_info(current_task()) ? proc->p_comm : "unknown" , thread_tid(current_thread())); + +#ifdef JOE_DEBUG + if (uthread->uu_iocount) { + printf("system call returned with uu_iocount != 0"); + } +#endif +#if CONFIG_DTRACE + uthread->t_dtrace_errno = error; +#endif /* CONFIG_DTRACE */ +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_set_allocation_name(NULL); + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + + arm_prepare_syscall_return(callp, state, uthread, error); + + uthread->uu_flag &= ~UT_NOTCANCELPT; + + if (uthread->uu_lowpri_window) { + /* + * task is marked as a low priority I/O type + * and the I/O we issued while in this system call + * collided with normal I/O operations... we'll + * delay in order to mitigate the impact of this + * task on the normal operation of the system + */ + throttle_lowpri_io(1); + } +#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_IST) + if (kdebug_enable && !code_is_kdebug_trace(code)) { + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END, + error, uthread->uu_rval[0], uthread->uu_rval[1], pid, 0); + } +#endif + +#if PROC_REF_DEBUG + if (__improbable(uthread_get_proc_refcount(uthread) != 0)) { + panic("system call returned with uu_proc_refcount != 0"); + } +#endif + +#ifdef __arm__ + thread_exception_return(); +#endif +} + +void +unix_syscall_return(int error) +{ + thread_t thread_act; + struct uthread *uthread; + struct proc *proc; + struct arm_saved_state *regs; + unsigned short code; + struct sysent *callp; + +#define unix_syscall_return_kprintf(x...) /* kprintf("unix_syscall_retur + * n: " x) */ + + thread_act = current_thread(); + proc = current_proc(); + uthread = get_bsdthread_info(thread_act); + + regs = find_user_regs(thread_act); + code = uthread->syscall_code; + callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code]; + +#if CONFIG_DTRACE + if (callp->sy_call == dtrace_systrace_syscall) + dtrace_systrace_syscall_return( code, error, uthread->uu_rval ); +#endif /* CONFIG_DTRACE */ +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_set_allocation_name(NULL); + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + + AUDIT_SYSCALL_EXIT(code, proc, uthread, error); + + /* + * Get index into sysent table + */ + arm_prepare_syscall_return(callp, regs, uthread, error); + + uthread->uu_flag &= ~UT_NOTCANCELPT; + + if (uthread->uu_lowpri_window) { + /* + * task is marked as a low priority I/O type + * and the I/O we issued while in this system call + * collided with normal I/O operations... we'll + * delay in order to mitigate the impact of this + * task on the normal operation of the system + */ + throttle_lowpri_io(1); + } +#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_IST) + if (kdebug_enable && !code_is_kdebug_trace(code)) { + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END, + error, uthread->uu_rval[0], uthread->uu_rval[1], proc->p_pid, 0); + } +#endif + + thread_exception_return(); + /* NOTREACHED */ +} + +static void +arm_prepare_u32_syscall_return(struct sysent *callp, arm_saved_state32_t *regs, uthread_t uthread, int error) +{ + if (error == ERESTART) { + regs->pc -= 4; + } else if (error != EJUSTRETURN) { + if (error) { + regs->save_r0 = error; + regs->save_r1 = 0; + /* set the carry bit to execute cerror routine */ + regs->cpsr |= PSR_CF; + unix_syscall_return_kprintf("error: setting carry to trigger cerror call\n"); + } else { /* (not error) */ + switch (callp->sy_return_type) { + case _SYSCALL_RET_INT_T: + case _SYSCALL_RET_UINT_T: + case _SYSCALL_RET_OFF_T: + case _SYSCALL_RET_ADDR_T: + case _SYSCALL_RET_SIZE_T: + case _SYSCALL_RET_SSIZE_T: + case _SYSCALL_RET_UINT64_T: + regs->save_r0 = uthread->uu_rval[0]; + regs->save_r1 = uthread->uu_rval[1]; + break; + case _SYSCALL_RET_NONE: + regs->save_r0 = 0; + regs->save_r1 = 0; + break; + default: + panic("unix_syscall: unknown return type"); + break; + } + } + } + /* else (error == EJUSTRETURN) { nothing } */ + +} + +static void +arm_trace_u32_unix_syscall(int code, arm_saved_state32_t *regs) +{ + boolean_t indirect = (regs->save_r12 == 0); + if (indirect) + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START, + regs->save_r1, regs->save_r2, regs->save_r3, regs->save_r4, 0); + else + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START, + regs->save_r0, regs->save_r1, regs->save_r2, regs->save_r3, 0); +} + +static void +arm_clear_u32_syscall_error(arm_saved_state32_t *regs) +{ + regs->cpsr &= ~PSR_CF; +} + +#if defined(__arm__) + +static int +arm_get_syscall_args(uthread_t uthread, struct arm_saved_state *state, struct sysent *callp) +{ + assert(is_saved_state32(state)); + return arm_get_u32_syscall_args(uthread, saved_state32(state), callp); +} + +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* + * For armv7k, the alignment constraints of the ABI mean we don't know how the userspace + * arguments are arranged without knowing the the prototype of the syscall. So we use mungers + * to marshal the userspace data into the uu_arg. This also means we need the same convention + * as mach syscalls. That means we use r8 to pass arguments in the BSD case as well. + */ +static int +arm_get_u32_syscall_args(uthread_t uthread, arm_saved_state32_t *regs, struct sysent *callp) +{ + sy_munge_t *munger; + + /* This check is probably not very useful since these both come from build-time */ + if (callp->sy_arg_bytes > sizeof(uthread->uu_arg)) + return -1; + + /* get the munger and use it to marshal in the data from userspace */ + munger = callp->sy_arg_munge32; + if (munger == NULL || (callp->sy_arg_bytes == 0)) + return 0; + + return munger(regs, uthread->uu_arg); +} +#else +/* + * For an AArch32 kernel, where we know that we have only AArch32 userland, + * we do not do any munging (which is a little confusing, as it is a contrast + * to the i386 kernel, where, like the x86_64 kernel, we always munge + * arguments from a 32-bit userland out to 64-bit. + */ +static int +arm_get_u32_syscall_args(uthread_t uthread, arm_saved_state32_t *regs, struct sysent *callp) +{ + int regparams; + int flavor = (regs->save_r12 == 0 ? 1 : 0); + + regparams = (7 - flavor); /* Indirect value consumes a register */ + + assert((unsigned) callp->sy_arg_bytes <= sizeof (uthread->uu_arg)); + + if (callp->sy_arg_bytes <= (sizeof(uint32_t) * regparams)) { + /* + * Seven arguments or less are passed in registers. + */ + memcpy(&uthread->uu_arg[0], ®s->r[flavor], callp->sy_arg_bytes); + } else if (callp->sy_arg_bytes <= sizeof(uthread->uu_arg)) { + /* + * In this case, we composite - take the first args from registers, + * the remainder from the stack (offset by the 7 regs therein). + */ + unix_syscall_kprintf("%s: spillover...\n", __FUNCTION__); + memcpy(&uthread->uu_arg[0] , ®s->r[flavor], regparams * sizeof(int)); + if (copyin((user_addr_t)regs->sp + 7 * sizeof(int), (int *)&uthread->uu_arg[0] + regparams, + (callp->sy_arg_bytes - (sizeof(uint32_t) * regparams))) != 0) { + return -1; + } + } else { + return -1; + } + + return 0; +} +#endif + +static int +arm_get_syscall_number(struct arm_saved_state *regs) +{ + if (regs->save_r12 != 0) { + return regs->save_r12; + } else { + return regs->save_r0; + } +} + +static void +arm_prepare_syscall_return(struct sysent *callp, struct arm_saved_state *state, uthread_t uthread, int error) +{ + assert(is_saved_state32(state)); + arm_prepare_u32_syscall_return(callp, state, uthread, error); +} + +static void +arm_trace_unix_syscall(int code, struct arm_saved_state *state) +{ + assert(is_saved_state32(state)); + arm_trace_u32_unix_syscall(code, saved_state32(state)); +} + +static void +arm_clear_syscall_error(struct arm_saved_state * state) +{ + assert(is_saved_state32(state)); + arm_clear_u32_syscall_error(saved_state32(state)); +} + +#elif defined(__arm64__) +static void arm_prepare_u64_syscall_return(struct sysent *, arm_saved_state64_t *, uthread_t, int); +static int arm_get_u64_syscall_args(uthread_t, arm_saved_state64_t *, struct sysent *); + +static int +arm_get_syscall_args(uthread_t uthread, struct arm_saved_state *state, struct sysent *callp) +{ + if (is_saved_state32(state)) { + return arm_get_u32_syscall_args(uthread, saved_state32(state), callp); + } else { + return arm_get_u64_syscall_args(uthread, saved_state64(state), callp); + } +} + +/* + * 64-bit: all arguments in registers. We're willing to use x9, a temporary + * register per the ABI, to pass an argument to the kernel for one case, + * an indirect syscall with 8 arguments. No munging required, as all arguments + * are in 64-bit wide registers already. + */ +static int +arm_get_u64_syscall_args(uthread_t uthread, arm_saved_state64_t *regs, struct sysent *callp) +{ + int indirect_offset, regparams; + + indirect_offset = (regs->x[ARM64_SYSCALL_CODE_REG_NUM] == 0) ? 1 : 0; + regparams = 9 - indirect_offset; + + /* + * Everything should fit in registers for now. + */ + assert(callp->sy_narg <= 8); + if (callp->sy_narg > regparams) { + return -1; + } + + memcpy(&uthread->uu_arg[0], ®s->x[indirect_offset], callp->sy_narg * sizeof(uint64_t)); + return 0; +} +/* + * When the kernel is running AArch64, munge arguments from 32-bit + * userland out to 64-bit. + * + * flavor == 1 indicates an indirect syscall. + */ +static int +arm_get_u32_syscall_args(uthread_t uthread, arm_saved_state32_t *regs, struct sysent *callp) +{ + int regparams; +#if CONFIG_REQUIRES_U32_MUNGING + sy_munge_t *mungerp; +#else +#error U32 syscalls on ARM64 kernel requires munging +#endif + int flavor = (regs->save_r12 == 0 ? 1 : 0); + + regparams = (7 - flavor); /* Indirect value consumes a register */ + + assert((unsigned) callp->sy_arg_bytes <= sizeof (uthread->uu_arg)); + + if (callp->sy_arg_bytes <= (sizeof(uint32_t) * regparams)) { + /* + * Seven arguments or less are passed in registers. + */ + memcpy(&uthread->uu_arg[0], ®s->r[flavor], callp->sy_arg_bytes); + } else if (callp->sy_arg_bytes <= sizeof(uthread->uu_arg)) { + /* + * In this case, we composite - take the first args from registers, + * the remainder from the stack (offset by the 7 regs therein). + */ + unix_syscall_kprintf("%s: spillover...\n", __FUNCTION__); + memcpy(&uthread->uu_arg[0] , ®s->r[flavor], regparams * sizeof(int)); + if (copyin((user_addr_t)regs->sp + 7 * sizeof(int), (int *)&uthread->uu_arg[0] + regparams, + (callp->sy_arg_bytes - (sizeof(uint32_t) * regparams))) != 0) { + return -1; + } + } else { + return -1; + } + +#if CONFIG_REQUIRES_U32_MUNGING + /* Munge here */ + mungerp = callp->sy_arg_munge32; + if (mungerp != NULL) { + (*mungerp)(&uthread->uu_arg[0]); + } +#endif + + return 0; + +} + +static int +arm_get_syscall_number(struct arm_saved_state *state) +{ + if (is_saved_state32(state)) { + if (saved_state32(state)->save_r12 != 0) { + return saved_state32(state)->save_r12; + } else { + return saved_state32(state)->save_r0; + } + } else { + if (saved_state64(state)->x[ARM64_SYSCALL_CODE_REG_NUM] != 0) { + return saved_state64(state)->x[ARM64_SYSCALL_CODE_REG_NUM]; + } else { + return saved_state64(state)->x[0]; + } + } + +} + +static void +arm_prepare_syscall_return(struct sysent *callp, struct arm_saved_state *state, uthread_t uthread, int error) +{ + if (is_saved_state32(state)) { + arm_prepare_u32_syscall_return(callp, saved_state32(state), uthread, error); + } else { + arm_prepare_u64_syscall_return(callp, saved_state64(state), uthread, error); + } +} + +static void +arm_prepare_u64_syscall_return(struct sysent *callp, arm_saved_state64_t *regs, uthread_t uthread, int error) +{ + if (error == ERESTART) { + regs->pc -= 4; + } else if (error != EJUSTRETURN) { + if (error) { + regs->x[0] = error; + regs->x[1] = 0; + /* + * Set the carry bit to execute cerror routine. + * ARM64_TODO: should we have a separate definition? + * The bits are the same. + */ + regs->cpsr |= PSR_CF; + unix_syscall_return_kprintf("error: setting carry to trigger cerror call\n"); + } else { /* (not error) */ + switch (callp->sy_return_type) { + case _SYSCALL_RET_INT_T: + regs->x[0] = uthread->uu_rval[0]; + regs->x[1] = uthread->uu_rval[1]; + break; + case _SYSCALL_RET_UINT_T: + regs->x[0] = (u_int)uthread->uu_rval[0]; + regs->x[1] = (u_int)uthread->uu_rval[1]; + break; + case _SYSCALL_RET_OFF_T: + case _SYSCALL_RET_ADDR_T: + case _SYSCALL_RET_SIZE_T: + case _SYSCALL_RET_SSIZE_T: + case _SYSCALL_RET_UINT64_T: + regs->x[0] = *((uint64_t *)(&uthread->uu_rval[0])); + regs->x[1] = 0; + break; + case _SYSCALL_RET_NONE: + break; + default: + panic("unix_syscall: unknown return type"); + break; + } + } + } + /* else (error == EJUSTRETURN) { nothing } */ + + +} +static void +arm_trace_u64_unix_syscall(int code, arm_saved_state64_t *regs) +{ + boolean_t indirect = (regs->x[ARM64_SYSCALL_CODE_REG_NUM] == 0); + if (indirect) + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START, + regs->x[1], regs->x[2], regs->x[3], regs->x[4], 0); + else + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START, + regs->x[0], regs->x[1], regs->x[2], regs->x[3], 0); +} + +static void +arm_trace_unix_syscall(int code, struct arm_saved_state *state) +{ + if (is_saved_state32(state)) { + arm_trace_u32_unix_syscall(code, saved_state32(state)); + } else { + arm_trace_u64_unix_syscall(code, saved_state64(state)); + } +} + +static void +arm_clear_u64_syscall_error(arm_saved_state64_t *regs) +{ + /* + * ARM64_TODO: should we have a separate definition? + * The bits are the same. + */ + regs->cpsr &= ~PSR_CF; +} + +static void +arm_clear_syscall_error(struct arm_saved_state * state) +{ + if (is_saved_state32(state)) { + arm_clear_u32_syscall_error(saved_state32(state)); + } else { + arm_clear_u64_syscall_error(saved_state64(state)); + } +} + +#else +#error Unknown architecture. +#endif diff --git a/bsd/dev/arm/table_inline.h b/bsd/dev/arm/table_inline.h new file mode 100644 index 000000000..8f358e423 --- /dev/null +++ b/bsd/dev/arm/table_inline.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + * Intel386 Family: Selector based access to descriptor tables. + * + * HISTORY + * + * 2 April 1992 ? at NeXT + * Created. + */ + +#include + +#include +#include + +static inline gdt_entry_t * +sel_to_gdt_entry(sel_t sel) +{ + return (&gdt[sel.index]); +} + +static inline idt_entry_t * +sel_to_idt_entry(sel_t sel) +{ + return (&idt[sel.index]); +} + +static inline ldt_entry_t * +sel_to_ldt_entry(ldt_t *tbl, sel_t sel) +{ + return (&tbl[sel.index]); +} diff --git a/bsd/dev/arm/unix_signal.c b/bsd/dev/arm/unix_signal.c new file mode 100644 index 000000000..51c4d7e48 --- /dev/null +++ b/bsd/dev/arm/unix_signal.c @@ -0,0 +1,737 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +extern struct arm_saved_state *get_user_regs(thread_t); +extern user_addr_t thread_get_cthread_self(void); +extern kern_return_t thread_getstatus(thread_t act, int flavor, + thread_state_t tstate, mach_msg_type_number_t *count); +extern kern_return_t thread_setstatus(thread_t thread, int flavor, + thread_state_t tstate, mach_msg_type_number_t count); +/* XXX Put these someplace smarter... */ +typedef struct mcontext32 mcontext32_t; +typedef struct mcontext64 mcontext64_t; + +/* Signal handler flavors supported */ +/* These defns should match the Libc implmn */ +#define UC_TRAD 1 +#define UC_FLAVOR 30 + +/* The following are valid mcontext sizes */ +#define UC_FLAVOR_SIZE32 ((ARM_THREAD_STATE_COUNT + ARM_EXCEPTION_STATE_COUNT + ARM_VFP_STATE_COUNT) * sizeof(int)) +#define UC_FLAVOR_SIZE64 ((ARM_THREAD_STATE64_COUNT + ARM_EXCEPTION_STATE64_COUNT + ARM_NEON_STATE64_COUNT) * sizeof(int)) + +#if __arm64__ +#define C_64_REDZONE_LEN 128 +#endif + +static int +sendsig_get_state32(thread_t th_act, mcontext32_t *mcp) +{ + void *tstate; + mach_msg_type_number_t state_count; + + assert(!proc_is64bit(current_proc())); + + tstate = (void *) &mcp->ss; + state_count = ARM_THREAD_STATE_COUNT; + if (thread_getstatus(th_act, ARM_THREAD_STATE, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + tstate = (void *) &mcp->es; + state_count = ARM_EXCEPTION_STATE_COUNT; + if (thread_getstatus(th_act, ARM_EXCEPTION_STATE, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + tstate = (void *) &mcp->fs; + state_count = ARM_VFP_STATE_COUNT; + if (thread_getstatus(th_act, ARM_VFP_STATE, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + return 0; +} + +#if defined(__arm64__) +struct user_sigframe64 { + /* We can pass the last arg in a register for ARM64 */ + user64_siginfo_t sinfo; + struct user_ucontext64 uctx; + mcontext64_t mctx; +}; + +static int +sendsig_get_state64(thread_t th_act, mcontext64_t *mcp) +{ + void *tstate; + mach_msg_type_number_t state_count; + + assert(proc_is64bit(current_proc())); + + tstate = (void *) &mcp->ss; + state_count = ARM_THREAD_STATE64_COUNT; + if (thread_getstatus(th_act, ARM_THREAD_STATE64, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + tstate = (void *) &mcp->es; + state_count = ARM_EXCEPTION_STATE64_COUNT; + if (thread_getstatus(th_act, ARM_EXCEPTION_STATE64, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + tstate = (void *) &mcp->ns; + state_count = ARM_NEON_STATE64_COUNT; + if (thread_getstatus(th_act, ARM_NEON_STATE64, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + return 0; +} + +static void +sendsig_fill_uctx64(user_ucontext64_t *uctx, int oonstack, int mask, user64_addr_t sp, user64_size_t stack_size, user64_addr_t p_mctx) +{ + bzero(uctx, sizeof(*uctx)); + uctx->uc_onstack = oonstack; + uctx->uc_sigmask = mask; + uctx->uc_stack.ss_sp = sp; + uctx->uc_stack.ss_size = stack_size; + if (oonstack) + uctx->uc_stack.ss_flags |= SS_ONSTACK; + uctx->uc_link = (user64_addr_t)0; + uctx->uc_mcsize = (user64_size_t) UC_FLAVOR_SIZE64; + uctx->uc_mcontext64 = (user64_addr_t) p_mctx; +} + +static kern_return_t +sendsig_set_thread_state64(arm_thread_state64_t *regs, + user64_addr_t catcher, int infostyle, int sig, user64_addr_t p_sinfo, + user64_addr_t p_uctx, user64_addr_t trampact, user64_addr_t sp, thread_t th_act) +{ + assert(proc_is64bit(current_proc())); + + regs->x[0] = catcher; + regs->x[1] = infostyle; + regs->x[2] = sig; + regs->x[3] = p_sinfo; + regs->x[4] = p_uctx; + regs->pc = trampact; + regs->cpsr = PSR64_USER64_DEFAULT; + regs->sp = sp; + + return thread_setstatus(th_act, ARM_THREAD_STATE64, (void *)regs, ARM_THREAD_STATE64_COUNT); +} +#endif /* defined(__arm64__) */ + +static void +sendsig_fill_uctx32(user_ucontext32_t *uctx, int oonstack, int mask, user_addr_t sp, user_size_t stack_size, user_addr_t p_mctx) +{ + bzero(uctx, sizeof(*uctx)); + uctx->uc_onstack = oonstack; + uctx->uc_sigmask = mask; + uctx->uc_stack.ss_sp = (user32_addr_t) sp; + uctx->uc_stack.ss_size = (user32_size_t) stack_size; + if (oonstack) + uctx->uc_stack.ss_flags |= SS_ONSTACK; + uctx->uc_link = (user32_addr_t)0; + uctx->uc_mcsize = (user32_size_t) UC_FLAVOR_SIZE32; + uctx->uc_mcontext = (user32_addr_t) p_mctx; +} + +static kern_return_t +sendsig_set_thread_state32(arm_thread_state_t *regs, + user32_addr_t catcher, int infostyle, int sig, user32_addr_t p_sinfo, + user32_addr_t trampact, user32_addr_t sp, thread_t th_act) +{ + + assert(!proc_is64bit(current_proc())); + + regs->r[0] = catcher; + regs->r[1] = infostyle; + regs->r[2] = sig; + regs->r[3] = p_sinfo; + if (trampact & 1) { + regs->pc = trampact & ~1; +#if defined(__arm64__) + regs->cpsr = PSR64_USER32_DEFAULT | PSR64_MODE_USER32_THUMB; +#elif defined(__arm__) + regs->cpsr = PSR_USERDFLT | PSR_TF; +#else +#error Unknown architeture. +#endif + } else { + regs->pc = trampact; + regs->cpsr = PSR_USERDFLT; + } + regs->sp = sp; + + return thread_setstatus(th_act, ARM_THREAD_STATE, (void *)regs, ARM_THREAD_STATE_COUNT); +} + +#if CONFIG_DTRACE +static void +sendsig_do_dtrace(uthread_t ut, user_siginfo_t *sinfo, int sig, user_addr_t catcher) +{ + bzero((caddr_t)&(ut->t_dtrace_siginfo), sizeof(ut->t_dtrace_siginfo)); + + ut->t_dtrace_siginfo.si_signo = sinfo->si_signo; + ut->t_dtrace_siginfo.si_code = sinfo->si_code; + ut->t_dtrace_siginfo.si_pid = sinfo->si_pid; + ut->t_dtrace_siginfo.si_uid = sinfo->si_uid; + ut->t_dtrace_siginfo.si_status = sinfo->si_status; + /* XXX truncates faulting address to void * */ + ut->t_dtrace_siginfo.si_addr = CAST_DOWN_EXPLICIT(void *, sinfo->si_addr); + + /* Fire DTrace proc:::fault probe when signal is generated by hardware. */ + switch (sig) { + case SIGILL: case SIGBUS: case SIGSEGV: case SIGFPE: case SIGTRAP: + DTRACE_PROC2(fault, int, (int)(ut->uu_code), siginfo_t *, &(ut->t_dtrace_siginfo)); + break; + default: + break; + } + + /* XXX truncates faulting address to uintptr_t */ + DTRACE_PROC3(signal__handle, int, sig, siginfo_t *, &(ut->t_dtrace_siginfo), + void (*)(void), CAST_DOWN(sig_t, catcher)); +} +#endif + +struct user_sigframe32 { + user32_addr_t puctx; + user32_siginfo_t sinfo; + struct user_ucontext32 uctx; + mcontext32_t mctx; +}; + +/* + * Send an interrupt to process. + * + */ +void +sendsig( + struct proc * p, + user_addr_t catcher, + int sig, + int mask, + __unused uint32_t code +) +{ + union { + struct user_sigframe32 uf32; +#if defined(__arm64__) + struct user_sigframe64 uf64; +#endif + } user_frame; + + user_siginfo_t sinfo; + user_addr_t sp = 0, trampact; + struct sigacts *ps = p->p_sigacts; + int oonstack, infostyle; + thread_t th_act; + struct uthread *ut; + user_size_t stack_size = 0; + + th_act = current_thread(); + ut = get_bsdthread_info(th_act); + + bzero(&user_frame, sizeof(user_frame)); + + if (p->p_sigacts->ps_siginfo & sigmask(sig)) + infostyle = UC_FLAVOR; + else + infostyle = UC_TRAD; + + trampact = ps->ps_trampact[sig]; + oonstack = ps->ps_sigstk.ss_flags & SA_ONSTACK; + + /* + * Get sundry thread state. + */ + if (proc_is64bit(p)) { +#ifdef __arm64__ + if (sendsig_get_state64(th_act, &user_frame.uf64.mctx) != 0) { + goto bad2; + } +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + if (sendsig_get_state32(th_act, &user_frame.uf32.mctx) != 0) { + goto bad2; + } + } + + /* + * Figure out where our new stack lives. + */ + if ((ps->ps_flags & SAS_ALTSTACK) && !oonstack && + (ps->ps_sigonstack & sigmask(sig))) { + sp = ps->ps_sigstk.ss_sp; + sp += ps->ps_sigstk.ss_size; + stack_size = ps->ps_sigstk.ss_size; + ps->ps_sigstk.ss_flags |= SA_ONSTACK; + } else { + /* + * Get stack pointer, and allocate enough space + * for signal handler data. + */ + if (proc_is64bit(p)) { +#if defined(__arm64__) + sp = CAST_USER_ADDR_T(user_frame.uf64.mctx.ss.sp); + sp = (sp - sizeof(user_frame.uf64) - C_64_REDZONE_LEN) & ~0xf; /* Make sure to align to 16 bytes and respect red zone */ +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + sp = CAST_USER_ADDR_T(user_frame.uf32.mctx.ss.sp); + sp -= sizeof(user_frame.uf32); +#if defined(__arm__) && (__BIGGEST_ALIGNMENT__ > 4) + sp &= ~0xf; /* Make sure to align to 16 bytes for armv7k */ +#endif + } + } + + proc_unlock(p); + + /* + * Fill in ucontext (points to mcontext, i.e. thread states). + */ + if (proc_is64bit(p)) { +#if defined(__arm64__) + sendsig_fill_uctx64(&user_frame.uf64.uctx, oonstack, mask, sp, (user64_size_t)stack_size, + (user64_addr_t)&((struct user_sigframe64*)sp)->mctx); +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + sendsig_fill_uctx32(&user_frame.uf32.uctx, oonstack, mask, sp, (user32_size_t)stack_size, + (user32_addr_t)&((struct user_sigframe32*)sp)->mctx); + } + + /* + * Setup siginfo. + */ + bzero((caddr_t) & sinfo, sizeof(sinfo)); + sinfo.si_signo = sig; + + if (proc_is64bit(p)) { +#if defined(__arm64__) + sinfo.si_addr = user_frame.uf64.mctx.ss.pc; + sinfo.pad[0] = user_frame.uf64.mctx.ss.sp; +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + sinfo.si_addr = user_frame.uf32.mctx.ss.pc; + sinfo.pad[0] = user_frame.uf32.mctx.ss.sp; + } + + switch (sig) { + case SIGILL: +#ifdef BER_XXX + if (mctx.ss.srr1 & (1 << (31 - SRR1_PRG_ILL_INS_BIT))) + sinfo.si_code = ILL_ILLOPC; + else if (mctx.ss.srr1 & (1 << (31 - SRR1_PRG_PRV_INS_BIT))) + sinfo.si_code = ILL_PRVOPC; + else if (mctx.ss.srr1 & (1 << (31 - SRR1_PRG_TRAP_BIT))) + sinfo.si_code = ILL_ILLTRP; + else + sinfo.si_code = ILL_NOOP; +#else + sinfo.si_code = ILL_ILLTRP; +#endif + break; + + case SIGFPE: + break; + + case SIGBUS: + if (proc_is64bit(p)) { +#if defined(__arm64__) + sinfo.si_addr = user_frame.uf64.mctx.es.far; +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + sinfo.si_addr = user_frame.uf32.mctx.es.far; + } + + sinfo.si_code = BUS_ADRALN; + break; + + case SIGSEGV: + if (proc_is64bit(p)) { +#if defined(__arm64__) + sinfo.si_addr = user_frame.uf64.mctx.es.far; +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + sinfo.si_addr = user_frame.uf32.mctx.es.far; + } + +#ifdef BER_XXX + /* First check in srr1 and then in dsisr */ + if (mctx.ss.srr1 & (1 << (31 - DSISR_PROT_BIT))) + sinfo.si_code = SEGV_ACCERR; + else if (mctx.es.dsisr & (1 << (31 - DSISR_PROT_BIT))) + sinfo.si_code = SEGV_ACCERR; + else + sinfo.si_code = SEGV_MAPERR; +#else + sinfo.si_code = SEGV_ACCERR; +#endif + break; + + default: + { + int status_and_exitcode; + + /* + * All other signals need to fill out a minimum set of + * information for the siginfo structure passed into + * the signal handler, if SA_SIGINFO was specified. + * + * p->si_status actually contains both the status and + * the exit code; we save it off in its own variable + * for later breakdown. + */ + proc_lock(p); + sinfo.si_pid = p->si_pid; + p->si_pid = 0; + status_and_exitcode = p->si_status; + p->si_status = 0; + sinfo.si_uid = p->si_uid; + p->si_uid = 0; + sinfo.si_code = p->si_code; + p->si_code = 0; + proc_unlock(p); + if (sinfo.si_code == CLD_EXITED) { + if (WIFEXITED(status_and_exitcode)) + sinfo.si_code = CLD_EXITED; + else if (WIFSIGNALED(status_and_exitcode)) { + if (WCOREDUMP(status_and_exitcode)) { + sinfo.si_code = CLD_DUMPED; + status_and_exitcode = W_EXITCODE(status_and_exitcode,status_and_exitcode); + } else { + sinfo.si_code = CLD_KILLED; + status_and_exitcode = W_EXITCODE(status_and_exitcode,status_and_exitcode); + } + } + } + /* + * The recorded status contains the exit code and the + * signal information, but the information to be passed + * in the siginfo to the handler is supposed to only + * contain the status, so we have to shift it out. + */ + sinfo.si_status = (WEXITSTATUS(status_and_exitcode) & 0x00FFFFFF) | (((uint32_t)(p->p_xhighbits) << 24) & 0xFF000000); + p->p_xhighbits = 0; + break; + } + } + +#if CONFIG_DTRACE + sendsig_do_dtrace(ut, &sinfo, sig, catcher); +#endif /* CONFIG_DTRACE */ + + /* + * Copy signal-handling frame out to user space, set thread state. + */ + if (proc_is64bit(p)) { +#if defined(__arm64__) + /* + * mctx filled in when we get state. uctx filled in by + * sendsig_fill_uctx64(). We fill in the sinfo now. + */ + siginfo_user_to_user64(&sinfo, &user_frame.uf64.sinfo); + + if (copyout(&user_frame.uf64, sp, sizeof(user_frame.uf64)) != 0) { + goto bad; + } + + if (sendsig_set_thread_state64(&user_frame.uf64.mctx.ss, + catcher, infostyle, sig, (user64_addr_t)&((struct user_sigframe64*)sp)->sinfo, + (user64_addr_t)&((struct user_sigframe64*)sp)->uctx, trampact, sp, th_act) != KERN_SUCCESS) + goto bad; + +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + /* + * mctx filled in when we get state. uctx filled in by + * sendsig_fill_uctx32(). We fill in the sinfo and *pointer* + * to uctx now. + */ + siginfo_user_to_user32(&sinfo, &user_frame.uf32.sinfo); + user_frame.uf32.puctx = (user32_addr_t) &((struct user_sigframe32*)sp)->uctx; + + if (copyout(&user_frame.uf32, sp, sizeof(user_frame.uf32)) != 0) { + goto bad; + } + + if (sendsig_set_thread_state32(&user_frame.uf32.mctx.ss, + CAST_DOWN_EXPLICIT(user32_addr_t, catcher), infostyle, sig, (user32_addr_t)&((struct user_sigframe32*)sp)->sinfo, + CAST_DOWN_EXPLICIT(user32_addr_t, trampact), CAST_DOWN_EXPLICIT(user32_addr_t, sp), th_act) != KERN_SUCCESS) + goto bad; + } + + proc_lock(p); + return; + +bad: + proc_lock(p); +bad2: + SIGACTION(p, SIGILL) = SIG_DFL; + sig = sigmask(SIGILL); + p->p_sigignore &= ~sig; + p->p_sigcatch &= ~sig; + ut->uu_sigmask &= ~sig; + /* sendsig is called with signal lock held */ + proc_unlock(p); + psignal_locked(p, SIGILL); + proc_lock(p); +} + +/* + * System call to cleanup state after a signal + * has been taken. Reset signal mask and + * stack state from context left by sendsig (above). + * Return to previous * context left by sendsig. + * Check carefully to * make sure that the user has not + * modified the * spr to gain improper priviledges. + */ + +static int +sigreturn_copyin_ctx32(struct user_ucontext32 *uctx, mcontext32_t *mctx, user_addr_t uctx_addr) +{ + int error; + + assert(!proc_is64bit(current_proc())); + + error = copyin(uctx_addr, uctx, sizeof(*uctx)); + if (error) { + return (error); + } + + /* validate the machine context size */ + switch (uctx->uc_mcsize) { + case UC_FLAVOR_SIZE32: + break; + default: + return (EINVAL); + } + + assert(uctx->uc_mcsize == sizeof(*mctx)); + error = copyin((user_addr_t)uctx->uc_mcontext, mctx, uctx->uc_mcsize); + if (error) { + return (error); + } + + return 0; +} + +static int +sigreturn_set_state32(thread_t th_act, mcontext32_t *mctx) +{ + assert(!proc_is64bit(current_proc())); + + /* validate the thread state, set/reset appropriate mode bits in cpsr */ +#if defined(__arm__) + mctx->ss.cpsr = (mctx->ss.cpsr & ~PSR_MODE_MASK) | PSR_USERDFLT; +#elif defined(__arm64__) + mctx->ss.cpsr = (mctx->ss.cpsr & ~PSR64_MODE_MASK) | PSR64_USER32_DEFAULT; +#else +#error Unknown architecture. +#endif + + if (thread_setstatus(th_act, ARM_THREAD_STATE, (void *)&mctx->ss, ARM_THREAD_STATE_COUNT) != KERN_SUCCESS) { + return (EINVAL); + } + if (thread_setstatus(th_act, ARM_VFP_STATE, (void *)&mctx->fs, ARM_VFP_STATE_COUNT) != KERN_SUCCESS) { + return (EINVAL); + } + + return 0; +} + +#if defined(__arm64__) +static int +sigreturn_copyin_ctx64(struct user_ucontext64 *uctx, mcontext64_t *mctx, user_addr_t uctx_addr) +{ + int error; + + assert(proc_is64bit(current_proc())); + + error = copyin(uctx_addr, uctx, sizeof(*uctx)); + if (error) { + return (error); + } + + /* validate the machine context size */ + switch (uctx->uc_mcsize) { + case UC_FLAVOR_SIZE64: + break; + default: + return (EINVAL); + } + + assert(uctx->uc_mcsize == sizeof(*mctx)); + error = copyin((user_addr_t)uctx->uc_mcontext64, mctx, uctx->uc_mcsize); + if (error) { + return (error); + } + + return 0; +} + +static int +sigreturn_set_state64(thread_t th_act, mcontext64_t *mctx) +{ + assert(proc_is64bit(current_proc())); + + /* validate the thread state, set/reset appropriate mode bits in cpsr */ + mctx->ss.cpsr = (mctx->ss.cpsr & ~PSR64_MODE_MASK) | PSR64_USER64_DEFAULT; + + if (thread_setstatus(th_act, ARM_THREAD_STATE64, (void *)&mctx->ss, ARM_THREAD_STATE64_COUNT) != KERN_SUCCESS) { + return (EINVAL); + } + if (thread_setstatus(th_act, ARM_NEON_STATE64, (void *)&mctx->ns, ARM_NEON_STATE64_COUNT) != KERN_SUCCESS) { + return (EINVAL); + } + + return 0; +} +#endif /* defined(__arm64__) */ + +/* ARGSUSED */ +int +sigreturn( + struct proc * p, + struct sigreturn_args * uap, + __unused int *retval) +{ + union { + user_ucontext32_t uc32; +#if defined(__arm64__) + user_ucontext64_t uc64; +#endif + } uctx; + + union { + mcontext32_t mc32; +#if defined(__arm64__) + mcontext64_t mc64; +#endif + } mctx; + + int error, sigmask = 0, onstack = 0; + thread_t th_act; + struct uthread *ut; + + th_act = current_thread(); + ut = (struct uthread *) get_bsdthread_info(th_act); + + if (proc_is64bit(p)) { +#if defined(__arm64__) + error = sigreturn_copyin_ctx64(&uctx.uc64, &mctx.mc64, uap->uctx); + if (error != 0) { + return error; + } + + onstack = uctx.uc64.uc_onstack; + sigmask = uctx.uc64.uc_sigmask; +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + error = sigreturn_copyin_ctx32(&uctx.uc32, &mctx.mc32, uap->uctx); + if (error != 0) { + return error; + } + + onstack = uctx.uc32.uc_onstack; + sigmask = uctx.uc32.uc_sigmask; + } + + if ((onstack & 01)) + p->p_sigacts->ps_sigstk.ss_flags |= SA_ONSTACK; + else + p->p_sigacts->ps_sigstk.ss_flags &= ~SA_ONSTACK; + + ut->uu_sigmask = sigmask & ~sigcantmask; + if (ut->uu_siglist & ~ut->uu_sigmask) + signal_setast(current_thread()); + + if (proc_is64bit(p)) { +#if defined(__arm64__) + error = sigreturn_set_state64(th_act, &mctx.mc64); + if (error != 0) { + return error; + } +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + error = sigreturn_set_state32(th_act, &mctx.mc32); + if (error != 0) { + return error; + } + } + + return (EJUSTRETURN); +} + +/* + * machine_exception() performs MD translation + * of a mach exception to a unix signal and code. + */ + +boolean_t +machine_exception( + int exception, + mach_exception_subcode_t code, + __unused mach_exception_subcode_t subcode, + int *unix_signal, + mach_exception_subcode_t * unix_code +) +{ + switch (exception) { + case EXC_BAD_INSTRUCTION: + *unix_signal = SIGILL; + *unix_code = code; + break; + + case EXC_ARITHMETIC: + *unix_signal = SIGFPE; + *unix_code = code; + break; + + default: + return (FALSE); + } + return (TRUE); +} diff --git a/bsd/dev/arm64/conf.c b/bsd/dev/arm64/conf.c new file mode 100644 index 000000000..761484db1 --- /dev/null +++ b/bsd/dev/arm64/conf.c @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1997 by Apple Computer, Inc., all rights reserved + * Copyright (c) 1993 NeXT Computer, Inc. + * + * UNIX Device switch tables. + * + * HISTORY + * + * 30 July 1997 Umesh Vaishampayan (umeshv@apple.com) + * enabled file descriptor pseudo-device. + * 18 June 1993 ? at NeXT + * Cleaned up a lot of stuff in this file. + */ + +#include +#include +#include +#include +#include + +/* Prototypes that should be elsewhere: */ +extern dev_t chrtoblk(dev_t dev); +extern int chrtoblk_set(int cdev, int bdev); + +struct bdevsw bdevsw[] = +{ + /* + * For block devices, every other block of 8 slots is reserved to Apple. + * The other slots are available for the user. This way we can both + * add new entries without running into each other. Be sure to fill in + * Apple's 8 reserved slots when you jump over us -- we'll do the same + * for you. + */ + + /* 0 - 7 are reserved to Apple */ + + NO_BDEVICE, /* 0 */ + NO_BDEVICE, /* 1 */ + NO_BDEVICE, /* 2 */ + NO_BDEVICE, /* 3 */ + NO_BDEVICE, /* 4 */ + NO_BDEVICE, /* 5 */ + NO_BDEVICE, /* 6 */ + NO_BDEVICE, /* 7 */ + + /* 8 - 15 are reserved to the user */ + NO_BDEVICE, /* 8 */ + NO_BDEVICE, /* 9 */ + NO_BDEVICE, /* 10 */ + NO_BDEVICE, /* 11 */ + NO_BDEVICE, /* 12 */ + NO_BDEVICE, /* 13 */ + NO_BDEVICE, /* 14 */ + NO_BDEVICE, /* 15 */ + + /* 16 - 23 are reserved to Apple */ + NO_BDEVICE, /* 16 */ + NO_BDEVICE, /* 17 */ + NO_BDEVICE, /* 18 */ + NO_BDEVICE, /* 18 */ + NO_BDEVICE, /* 20 */ + NO_BDEVICE, /* 21 */ + NO_BDEVICE, /* 22 */ + NO_BDEVICE, /* 23 */ +}; + +const int nblkdev = sizeof(bdevsw) / sizeof(bdevsw[0]); + +extern struct tty *km_tty[]; +extern d_open_t cnopen; +extern d_close_t cnclose; +extern d_read_t cnread; +extern d_write_t cnwrite; +extern d_ioctl_t cnioctl; +extern d_select_t cnselect; +extern d_open_t kmopen; +extern d_close_t kmclose; +extern d_read_t kmread; +extern d_write_t kmwrite; +extern d_ioctl_t kmioctl; +extern d_open_t sgopen; +extern d_close_t sgclose; +extern d_ioctl_t sgioctl; + +#if NVOL > 0 +extern d_open_t volopen; +extern d_close_t volclose; +extern d_ioctl_t volioctl; +#else +#define volopen eno_opcl +#define volclose eno_opcl +#define volioctl eno_ioctl +#endif + +extern d_open_t cttyopen; +extern d_read_t cttyread; +extern d_write_t cttywrite; +extern d_ioctl_t cttyioctl; +extern d_select_t cttyselect; + +extern d_read_t mmread; +extern d_write_t mmwrite; +extern d_ioctl_t mmioctl; +#define mmselect (select_fcn_t *)seltrue +#define mmmmap eno_mmap + +#include +#if NPTY > 0 +extern d_open_t ptsopen; +extern d_close_t ptsclose; +extern d_read_t ptsread; +extern d_write_t ptswrite; +extern d_select_t ptsselect; +extern d_stop_t ptsstop; +extern d_open_t ptcopen; +extern d_close_t ptcclose; +extern d_read_t ptcread; +extern d_write_t ptcwrite; +extern d_select_t ptcselect; +extern d_ioctl_t ptyioctl; +#else +#define ptsopen eno_opcl +#define ptsclose eno_opcl +#define ptsread eno_rdwrt +#define ptswrite eno_rdwrt +#define ptsstop nulldev + +#define ptcopen eno_opcl +#define ptcclose eno_opcl +#define ptcread eno_rdwrt +#define ptcwrite eno_rdwrt +#define ptcselect eno_select +#define ptyioctl eno_ioctl +#endif + +extern d_open_t logopen; +extern d_close_t logclose; +extern d_read_t logread; +extern d_ioctl_t logioctl; +extern d_select_t logselect; + +extern d_open_t oslog_streamopen; +extern d_close_t oslog_streamclose; +extern d_read_t oslog_streamread; +extern d_ioctl_t oslog_streamioctl; +extern d_select_t oslog_streamselect; + + +extern d_open_t oslogopen; +extern d_close_t oslogclose; +extern d_ioctl_t oslogioctl; +extern d_select_t oslogselect; + +#define nullopen (d_open_t *)&nulldev +#define nullclose (d_close_t *)&nulldev +#define nullread (d_read_t *)&nulldev +#define nullwrite (d_write_t *)&nulldev +#define nullioctl (d_ioctl_t *)&nulldev +#define nullselect (d_select_t *)&nulldev +#define nullstop (d_stop_t *)&nulldev +#define nullreset (d_reset_t *)&nulldev + +struct cdevsw cdevsw[] = { + /* + * To add character devices to this table dynamically, use cdevsw_add. + */ + + [0] = { + cnopen, cnclose, cnread, cnwrite, + cnioctl, nullstop, nullreset, 0, cnselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [1] = NO_CDEVICE, + [2] = { + cttyopen, nullclose, cttyread, cttywrite, + cttyioctl, nullstop, nullreset, 0, cttyselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [3] = { + nullopen, nullclose, mmread, mmwrite, + mmioctl, nullstop, nullreset, 0, mmselect, + mmmmap, eno_strat, eno_getc, eno_putc, D_DISK + }, + [PTC_MAJOR] = { + ptsopen, ptsclose, ptsread, ptswrite, + ptyioctl, ptsstop, nullreset, 0, ptsselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [PTS_MAJOR] = { + ptcopen, ptcclose, ptcread, ptcwrite, + ptyioctl, nullstop, nullreset, 0, ptcselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [6] = { + logopen, logclose, logread, eno_rdwrt, + logioctl, eno_stop, nullreset, 0, logselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [7] = { + oslogopen, oslogclose, eno_rdwrt, eno_rdwrt, + oslogioctl, eno_stop, nullreset, 0, oslogselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [8] = { + oslog_streamopen, oslog_streamclose, oslog_streamread, eno_rdwrt, + oslog_streamioctl, eno_stop, nullreset, 0, oslog_streamselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [9 ... 11] = NO_CDEVICE, + [12] = { + kmopen, kmclose, kmread, kmwrite, + kmioctl, nullstop, nullreset, km_tty, ttselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [13 ... 41] = NO_CDEVICE, + [42] = { + volopen, volclose, eno_rdwrt, eno_rdwrt, + volioctl, eno_stop, eno_reset, 0, (select_fcn_t *) seltrue, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + } +}; +const int nchrdev = sizeof(cdevsw) / sizeof(cdevsw[0]); + +uint64_t cdevsw_flags[sizeof(cdevsw) / sizeof(cdevsw[0])]; + +#include /* for VCHR and VBLK */ +/* + * return true if a disk + */ +int +isdisk(dev_t dev, int type) +{ + dev_t maj = major(dev); + + switch (type) { + case VCHR: + maj = chrtoblk(maj); + if (maj == NODEV) { + break; + } + /* FALL THROUGH */ + case VBLK: + if (bdevsw[maj].d_type == D_DISK) { + return (1); + } + break; + } + return (0); +} + +static int chrtoblktab[] = { + /* CHR *//* BLK *//* CHR *//* BLK */ + /* 0 */ NODEV, /* 1 */ NODEV, + /* 2 */ NODEV, /* 3 */ NODEV, + /* 4 */ NODEV, /* 5 */ NODEV, + /* 6 */ NODEV, /* 7 */ NODEV, + /* 8 */ NODEV, /* 9 */ NODEV, + /* 10 */ NODEV, /* 11 */ NODEV, + /* 12 */ NODEV, /* 13 */ NODEV, + /* 14 */ NODEV, /* 15 */ NODEV, + /* 16 */ NODEV, /* 17 */ NODEV, + /* 18 */ NODEV, /* 19 */ NODEV, + /* 20 */ NODEV, /* 21 */ NODEV, + /* 22 */ NODEV, /* 23 */ NODEV, + /* 24 */ NODEV, /* 25 */ NODEV, + /* 26 */ NODEV, /* 27 */ NODEV, + /* 28 */ NODEV, /* 29 */ NODEV, + /* 30 */ NODEV, /* 31 */ NODEV, + /* 32 */ NODEV, /* 33 */ NODEV, + /* 34 */ NODEV, /* 35 */ NODEV, + /* 36 */ NODEV, /* 37 */ NODEV, + /* 38 */ NODEV, /* 39 */ NODEV, + /* 40 */ NODEV, /* 41 */ NODEV, + /* 42 */ NODEV, /* 43 */ NODEV, + /* 44 */ NODEV, +}; + +/* + * convert chr dev to blk dev + */ +dev_t +chrtoblk(dev_t dev) +{ + int blkmaj; + + if (major(dev) >= nchrdev) + return (NODEV); + blkmaj = chrtoblktab[major(dev)]; + if (blkmaj == NODEV) + return (NODEV); + return (makedev(blkmaj, minor(dev))); +} + +int +chrtoblk_set(int cdev, int bdev) +{ + if (cdev >= nchrdev) + return (-1); + if (bdev != NODEV && bdev >= nblkdev) + return (-1); + chrtoblktab[cdev] = bdev; + return 0; +} diff --git a/bsd/dev/arm64/cpu_in_cksum.s b/bsd/dev/arm64/cpu_in_cksum.s new file mode 100644 index 000000000..b01b27172 --- /dev/null +++ b/bsd/dev/arm64/cpu_in_cksum.s @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. + * + * This document is the property of Apple Inc. + * It is considered confidential and proprietary. + * + * This document may not be reproduced or transmitted in any form, + * in whole or in part, without the express written permission of + * Apple Inc. + */ + +/* + * This assembly was previously cloned from ../arm/cpu_in_cksum.s (__arm__) + * with __arm64__ tagged ARM64_TODO . This code revision is optimized based + * on the 64-bit part in netinet/cpu_in_cksum.c + * + * cclee - CoreOS - Vector & Numerics. 06/20/2012. + */ + +#ifdef KERNEL +#define CKSUM_ERR _kprintf +#else +#ifndef LIBSYSCALL_INTERFACE +#error "LIBSYSCALL_INTERFACE not defined" +#endif /* !LIBSYSCALL_INTERFACE */ +#define CKSUM_ERR _fprintf_stderr +#endif /* !KERNEL */ + +/* + * XXX: adi@apple.com: + * + * Ugly, but we have little choice, since relying on genassym and + * is not possible unless this code lives in osfmk. Note also that this + * routine expects "mbuf-like" argument, and it does not expect the mbuf to be + * authentic; it only cares about 3 fields. + */ +#define M_NEXT 0 +#define M_DATA 16 // 8-byte address, would be aligned to 8-byte boundary +#define M_LEN 24 + + .globl _os_cpu_in_cksum_mbuf + .text + .align 4 +_os_cpu_in_cksum_mbuf: + + +/* + * 64-bit version. + * + * This function returns the partial 16-bit checksum accumulated in + * a 32-bit variable (withouth 1's complement); caller is responsible + * for folding the 32-bit sum into 16-bit and performinng the 1's + * complement if applicable + */ + +/* + * uint32_t + * os_cpu_in_cksum_mbuf(struct mbuf *m, int len, int off, uint32_t initial_sum) + * { + * int mlen; + * uint64_t sum, partial; + * unsigned int final_acc; + * uint8_t *data; + * boolean_t needs_swap, started_on_odd; + * + * VERIFY(len >= 0); + * VERIFY(off >= 0); + * + * needs_swap = FALSE; + * started_on_odd = FALSE; + * sum = initial_sum; + */ + + #define m x0 + #define len x1 + #define off x2 + #define sum x3 + #define needs_swap x4 + #define started_on_odd x5 + #define mlen x6 + #define Wmlen w6 + #define t x7 + #define data x8 + + mov needs_swap, #0 // needs_swap = FALSE; + mov started_on_odd, #0 // started_on_odd = FALSE; + mov w3, w3 // clear higher half + + +/* + * for (;;) { + * if (PREDICT_FALSE(m == NULL)) { + * CKSUM_ERR("%s: out of data\n", __func__); + * return (-1); + * } + * mlen = m->m_len; + * if (mlen > off) { + * mlen -= off; + * data = mtod(m, uint8_t *) + off; + * goto post_initial_offset; + * } + * off -= mlen; + * if (len == 0) + * break; + * m = m->m_next; + * } + */ + +0: + cbz m, Lin_cksum_whoops // if (m == NULL) return -1; + ldr Wmlen, [m, #M_LEN] // mlen = m->m_len; + cmp mlen, off + b.le 1f + ldr data, [m, #M_DATA] // mtod(m, uint8_t *) + sub mlen, mlen, off // mlen -= off; + add data, data, off // data = mtod(m, uint8_t *) + off; + b L_post_initial_offset +1: + sub off, off, mlen + cbnz len, 2f + mov x0, x3 + ret lr +2: + ldr m, [m, #M_NEXT] + b 0b + +L_loop: // for (; len > 0; m = m->m_next) { +/* + * if (PREDICT_FALSE(m == NULL)) { + * CKSUM_ERR("%s: out of data\n", __func__); + * return (-1); + * } + * mlen = m->m_len; + * data = mtod(m, uint8_t *); + */ + cbz m, Lin_cksum_whoops // if (m == NULL) return -1; + ldr Wmlen, [m, #M_LEN] // mlen = m->m_len; + ldr data, [m, #M_DATA] // mtod(m, uint8_t *) + +L_post_initial_offset: +/* + * if (mlen == 0) continue; + * if (mlen > len) mlen = len; + * len -= mlen; + */ + + cbz mlen, L_continue + cmp mlen, len + csel mlen, mlen, len, le + sub len, len, mlen + +/* + * partial = 0; + * if ((uintptr_t)data & 1) { + * started_on_odd = !started_on_odd; + * partial = *data << 8; + * ++data; + * --mlen; + * } + * needs_swap = started_on_odd; + */ + + tst data, #1 + mov x7, #0 + mov x10, #0 + b.eq 1f + ldrb w7, [data], #1 + eor started_on_odd, started_on_odd, #1 + sub mlen, mlen, #1 + lsl w7, w7, #8 +1: + + +/* + * if ((uintptr_t)data & 2) { + * if (mlen < 2) + * goto trailing_bytes; + * partial += *(uint16_t *)(void *)data; + * data += 2; + * mlen -= 2; + * } + */ + tst data, #2 + mov needs_swap, started_on_odd + b.eq 1f + cmp mlen, #2 + b.lt L_trailing_bytes + ldrh w9, [data], #2 + sub mlen, mlen, #2 + add w7, w7, w9 +1: + +/* + * while (mlen >= 64) { + * __builtin_prefetch(data + 32); + * __builtin_prefetch(data + 64); + * partial += *(uint32_t *)(void *)data; + * partial += *(uint32_t *)(void *)(data + 4); + * partial += *(uint32_t *)(void *)(data + 8); + * partial += *(uint32_t *)(void *)(data + 12); + * partial += *(uint32_t *)(void *)(data + 16); + * partial += *(uint32_t *)(void *)(data + 20); + * partial += *(uint32_t *)(void *)(data + 24); + * partial += *(uint32_t *)(void *)(data + 28); + * partial += *(uint32_t *)(void *)(data + 32); + * partial += *(uint32_t *)(void *)(data + 36); + * partial += *(uint32_t *)(void *)(data + 40); + * partial += *(uint32_t *)(void *)(data + 44); + * partial += *(uint32_t *)(void *)(data + 48); + * partial += *(uint32_t *)(void *)(data + 52); + * partial += *(uint32_t *)(void *)(data + 56); + * partial += *(uint32_t *)(void *)(data + 60); + * data += 64; + * mlen -= 64; + * // if (PREDICT_FALSE(partial & (3ULL << 62))) { + * // if (needs_swap) + * // partial = (partial << 8) + + * // (partial >> 56); + * // sum += (partial >> 32); + * // sum += (partial & 0xffffffff); + * // partial = 0; + * // } + * } +*/ + + // pre-decrement mlen by 64, and if < 64 bytes, try 32 bytes next + subs mlen, mlen, #64 + b.lt L32_bytes + + // save used vector registers + sub sp, sp, #8*16 + mov x11, sp + st1.4s {v0, v1, v2, v3}, [x11], #4*16 + st1.4s {v4, v5, v6, v7}, [x11], #4*16 + + // spread partial into 8 8-byte registers in v0-v3 + fmov s3, w7 + eor.16b v0, v0, v0 + eor.16b v1, v1, v1 + eor.16b v2, v2, v2 + + // load the 1st 64 bytes (16 32-bit words) + ld1.4s {v4,v5,v6,v7},[data],#64 + + // branch to finish off if mlen<64 + subs mlen, mlen, #64 + b.lt L64_finishup + + /* + * loop for loading and accumulating 16 32-bit words into + * 8 8-byte accumulators per iteration. + */ +L64_loop: + subs mlen, mlen, #64 // mlen -= 64 + + uadalp.2d v0, v4 + ld1.4s {v4},[data], #16 + + uadalp.2d v1, v5 + ld1.4s {v5},[data], #16 + + uadalp.2d v2, v6 + ld1.4s {v6},[data], #16 + + uadalp.2d v3, v7 + ld1.4s {v7},[data], #16 + + b.ge L64_loop + +L64_finishup: + uadalp.2d v0, v4 + uadalp.2d v1, v5 + uadalp.2d v2, v6 + uadalp.2d v3, v7 + + add.2d v0, v0, v1 + add.2d v2, v2, v3 + addp.2d d0, v0 + addp.2d d2, v2 + add.2d v0, v0, v2 + fmov x7, d0 // partial in x7 now + + // restore used vector registers + ld1.4s {v0, v1, v2, v3}, [sp], #4*16 + ld1.4s {v4, v5, v6, v7}, [sp], #4*16 + +L32_bytes: + tst mlen, #32 + b.eq L16_bytes + ldp x9, x10, [data], #16 + ldp x11, x12, [data], #16 + adds x7, x7, x9 + mov x9, #0 + adcs x7, x7, x10 + adcs x7, x7, x11 + adcs x7, x7, x12 + adc x7, x7, x9 + +L16_bytes: + tst mlen, #16 + b.eq L8_bytes + ldp x9, x10, [data], #16 + adds x7, x7, x9 + mov x9, #0 + adcs x7, x7, x10 + adc x7, x7, x9 + +L8_bytes: + tst mlen, #8 + mov x10, #0 + b.eq L4_bytes + ldr x9,[data],#8 + adds x7, x7, x9 + adc x7, x7, x10 + +L4_bytes: + tst mlen, #4 + b.eq L2_bytes + ldr w9,[data],#4 + adds x7, x7, x9 + adc x7, x7, x10 + +L2_bytes: + tst mlen, #2 + b.eq L_trailing_bytes + ldrh w9,[data],#2 + adds x7, x7, x9 + adc x7, x7, x10 + +L_trailing_bytes: + tst mlen, #1 + b.eq L0_bytes + ldrb w9,[data],#1 + adds x7, x7, x9 + adc x7, x7, x10 + eor started_on_odd, started_on_odd, #1 + +L0_bytes: +/* + * if (needs_swap) + * partial = (partial << 8) + (partial >> 56); + */ + cbz needs_swap, 1f + ror x7, x7, #56 +1: +/* + * sum += (partial >> 32) + (partial & 0xffffffff); + * sum = (sum >> 32) + (sum & 0xffffffff); + * } + */ + + add x3, x3, x7, lsr #32 + mov w7, w7 + add x3, x3, x7 + mov w7, w3 + add x3, x7, x3, lsr #32 + +L_continue: + cmp len, #0 + ldr m, [m, #M_NEXT] // m = m->m_next + b.gt L_loop + +/* + * final_acc = (sum >> 48) + ((sum >> 32) & 0xffff) + + * ((sum >> 16) & 0xffff) + (sum & 0xffff); + * final_acc = (final_acc >> 16) + (final_acc & 0xffff); + * final_acc = (final_acc >> 16) + (final_acc & 0xffff); + * return (final_acc & 0xffff); + * } + */ + + mov w4, #0x00ffff + and x0, x4, x3, lsr #48 + and x1, x4, x3, lsr #32 + and x2, x4, x3, lsr #16 + and x3, x4, x3 + add w0, w0, w1 + add w2, w2, w3 + add w0, w0, w2 + and w1, w4, w0, lsr #16 + and w0, w4, w0 + add w0, w0, w1 + and w1, w4, w0, lsr #16 + and w0, w4, w0 + add w0, w0, w1 + /* + * If we were to 1's complement it (XOR with 0xffff): + * + * eor w0, w0, w4 + */ + and w0, w0, w4 + + ret lr + +Lin_cksum_whoops: + adrp x0, Lin_cksum_whoops_str@page + add x0, x0, Lin_cksum_whoops_str@pageoff + bl #CKSUM_ERR + mov x0, #-1 + ret lr + +Lin_cksum_whoops_str: + .asciz "os_cpu_in_cksum_mbuf: out of data\n" + .align 5 diff --git a/bsd/dev/arm64/disassembler.c b/bsd/dev/arm64/disassembler.c new file mode 100644 index 000000000..a00f8d0eb --- /dev/null +++ b/bsd/dev/arm64/disassembler.c @@ -0,0 +1,1146 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * Disassemblers for ARM (arm), Thumb (thumb16), and Thumb2 (thumb32). + * + * Each disassembly begins with a call to dtrace_decode_arm or dtrace_decode_thumb. The thumb + * decoder will then call dtrace_decode_thumb16 or dtrace_decode_thumb32 as appropriate. + * + * The respective disassembly functions are all of the form {arm,thumb16,thumb32}_type. They + * follow the ordering and breakdown in the ARMv7 Architecture Reference Manual. + */ + +#include + +#define BITS(x,n,mask) (((x) >> (n)) & (mask)) + +static uint32_t thumb32_instword_to_arm(uint16_t hw1, uint16_t hw2) +{ + return (hw1 << 16) | hw2; +} + +int dtrace_decode_arm(uint32_t instr); +int dtrace_decode_arm64(uint32_t instr); +int dtrace_decode_thumb(uint32_t instr); + +/* + * VFP decoder - shared between ARM and THUMB32 mode + */ + +static +int vfp_struct_loadstore(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_64transfer(uint32_t instr) +{ + /* These instructions all use RD and RN */ + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_transfer(uint32_t instr) +{ + /* These instructions all use RD only */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_loadstore(uint32_t instr) +{ + int opcode = BITS(instr,20,0x1F); + + /* Instrument VLDR */ + if ((opcode & 0x13) == 0x11 && ARM_RN(instr) == REG_PC) + return FASTTRAP_T_VLDR_PC_IMMED; + + /* These instructions all use RN only */ + if (ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +/* + * ARM decoder + */ + +static +int arm_unconditional_misc(uint32_t instr) +{ + int op = BITS(instr,20,0x7F); + + if ((op & 0x60) == 0x20) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } + + if ((op & 0x71) == 0x40) { + return vfp_struct_loadstore(instr); + } + + return FASTTRAP_T_INV; +} + +static +int arm_unconditional(uint32_t instr) +{ + if (BITS(instr,27,0x1) == 0) + return arm_unconditional_misc(instr); + + /* The rest are privileged or BL/BLX, do not instrument */ + + /* Do not need to instrument BL/BLX either, see comment in arm_misc(uint32_t) */ + + return FASTTRAP_T_INV; +} + +static +int arm_syscall_coproc(uint32_t instr) +{ + /* Instrument any VFP data processing instructions, ignore the rest */ + + int op1 = BITS(instr,20,0x3F), coproc = BITS(instr,8,0xF), op = BITS(instr,4,0x1); + + if ((op1 & 0x3E) == 0 || (op1 & 0x30) == 0x30) { + /* Undefined or swi */ + return FASTTRAP_T_INV; + } + + if ((coproc & 0xE) == 0xA) { + /* VFP instruction */ + + if ((op1 & 0x20) == 0 && (op1 & 0x3A) != 0) + return vfp_loadstore(instr); + + if ((op1 & 0x3E) == 0x04) + return vfp_64transfer(instr); + + if ((op1 & 0x30) == 0x20) { + /* VFP data processing or 8, 16, or 32 bit move between ARM reg and VFP reg */ + if (op == 0) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } else { + return vfp_transfer(instr); + } + } + } + + return FASTTRAP_T_INV; +} + +static +int arm_branch_link_blockdata(uint32_t instr) +{ + int branch = BITS(instr,25,0x1), link = BITS(instr,24,0x1), op = BITS(instr,20,0x1F), uses_pc = BITS(instr,15,0x1), uses_lr = BITS(instr,14,0x1); + + if (branch == 1) { + if (link == 0) + return FASTTRAP_T_B_COND; + return FASTTRAP_T_INV; + } else { + /* Only emulate a use of the pc if it's a return from function: ldmia sp!, { ... pc } */ + if (op == 0x0B && ARM_RN(instr) == REG_SP && uses_pc == 1) + return FASTTRAP_T_LDM_PC; + + /* stmia sp!, { ... lr } doesn't touch the pc, but it is very common, so special case it */ + if (op == 0x12 && ARM_RN(instr) == REG_SP && uses_lr == 1) + return FASTTRAP_T_STM_LR; + + if (ARM_RN(instr) != REG_PC && uses_pc == 0) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_signed_multiplies(uint32_t instr) +{ + int op1 = BITS(instr,20,0x7), op2 = BITS(instr,5,0x7); + + /* smlald, smlsld, smmls use RD in addition to RM, RS, and RN */ + if ((op1 == 0x4 && (op2 & 0x4) == 0) || (op1 == 0x5 && (op2 & 0x6) == 0x6)) { + if (ARM_RD(instr) == REG_PC) + return FASTTRAP_T_INV; + } + + if (ARM_RM(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_pack_unpack_sat_reversal(uint32_t instr) +{ + int op1 = BITS(instr,20,0x7), op2 = BITS(instr,5,0x7); + + /* pkh, sel use RN in addition to RD and RM */ + if ((op1 == 0 && (op2 & 0x1) == 0) || (op1 == 0 && op2 == 0x5)) { + if (ARM_RN(instr) == REG_PC) + return FASTTRAP_T_INV; + } + + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_parallel_addsub_unsigned(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_parallel_addsub_signed(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_media(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F), op2 = BITS(instr,5,0x7); + + if ((op1 & 0x1C) == 0) + return arm_parallel_addsub_signed(instr); + + if ((op1 & 0x1C) == 0x04) + return arm_parallel_addsub_unsigned(instr); + + if ((op1 & 0x18) == 0x08) + return arm_pack_unpack_sat_reversal(instr); + + if ((op1 & 0x18) == 0x10) + return arm_signed_multiplies(instr); + + if (op1 == 0x1F && op2 == 0x7) { + /* Undefined instruction */ + return FASTTRAP_T_INV; + } + + if (op1 == 0x18 && op2 == 0) { + /* usad8 usada8 */ + /* The registers are named differently in the reference manual for this instruction + * but the following positions are correct */ + + if (ARM_RM(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if ((op1 & 0x1E) == 0x1C && (op2 & 0x3) == 0) { + /* bfc bfi */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if (((op1 & 0x1E) == 0x1A || (op1 & 0x1E) == 0x1E) && ((op2 & 0x3) == 0x2)) { + /* sbfx ubfx */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + return FASTTRAP_T_INV; +} + +static +int arm_loadstore_wordbyte(uint32_t instr) +{ + /* Instrument PC relative load with immediate, ignore any other uses of the PC */ + int R = BITS(instr,25,0x1), L = BITS(instr,20,0x1); + + if (R == 1) { + /* Three register load/store */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Immediate load/store, but still do not support ldr pc, [pc...] */ + if (L == 1 && ARM_RN(instr) == REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_LDR_PC_IMMED; + + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_saturating(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_misc(uint32_t instr) +{ + int op = BITS(instr,21,0x3), __unused op1 = BITS(instr,16,0xF), op2 = BITS(instr,4,0x7); + + if (op2 == 1 && op == 1) + return FASTTRAP_T_BX_REG; + + /* We do not need to emulate BLX for entry/return probes; if we eventually support full offset + * tracing, then we will. This is because BLX overwrites the link register, so a function that + * can execute this as its first instruction is a special function indeed. + */ + + if (op2 == 0x5) + return arm_saturating(instr); + + return FASTTRAP_T_INV; +} + +static +int arm_msr_hints(__unused uint32_t instr) +{ + /* These deal with the psr, not instrumented */ + + return FASTTRAP_T_INV; +} + +static +int arm_sync_primitive(__unused uint32_t instr) +{ + /* TODO will instrumenting these interfere with any kernel usage of these instructions? */ + /* Don't instrument for now */ + + return FASTTRAP_T_INV; +} + +static +int arm_extra_loadstore_unpriv(uint32_t instr) +{ + int op = BITS(instr,20,0x1), __unused op2 = BITS(instr,5,0x3), immed = BITS(instr,22,0x1); + + if (op == 0 && (op2 & 0x2) == 0x2) { + /* Unpredictable or undefined */ + return FASTTRAP_T_INV; + } + + if (immed == 1) { + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_extra_loadstore(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F); + + /* There are two variants, and we do not instrument either of them that use the PC */ + + if ((op1 & 0x4) == 0) { + /* Variant 1, register */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Variant 2, immediate */ + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_halfword_multiply(uint32_t instr) +{ + /* Not all multiply instructions use all four registers. The ones that don't should have those + * register locations set to 0, so we can test them anyway. + */ + + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_multiply(uint32_t instr) +{ + /* Not all multiply instructions use all four registers. The ones that don't should have those + * register locations set to 0, so we can test them anyway. + */ + + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_immed(uint32_t instr) +{ + /* All these instructions are either two registers, or one register and have 0 where the other reg would be used */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_regshift(uint32_t instr) +{ + /* All these instructions are either four registers, or three registers and have 0 where there last reg would be used */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_reg(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F), op2 = BITS(instr,7,0x1F), op3 = BITS(instr,5,0x3); + + if (op1 == 0x11 || op1 == 0x13 || op1 == 0x15 || op1 == 0x17) { + /* These are comparison flag setting instructions and do not have RD */ + if (ARM_RN(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + /* The rest can, in theory, write or use the PC. The only one we instrument is mov pc, reg. + * movs pc, reg is a privileged instruction so we don't instrument that variant. The s bit + * is bit 0 of op1 and should be zero. + */ + if (op1 == 0x1A && op2 == 0 && op3 == 0 && ARM_RD(instr) == REG_PC) + return FASTTRAP_T_MOV_PC_REG; + + /* Any instruction at this point is a three register instruction or two register instruction with RN = 0 */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_misc(uint32_t instr) +{ + int op = BITS(instr,25,0x1), op1 = BITS(instr,20,0x1F), op2 = BITS(instr,4,0xF); + + if (op == 0) { + if ((op1 & 0x19) != 0x10 && (op2 & 0x1) == 0) + return arm_dataproc_reg(instr); + + if ((op1 & 0x19) != 0x10 && (op2 & 0x9) == 0x1) + return arm_dataproc_regshift(instr); + + if ((op1 & 0x19) == 0x10 && (op2 & 0x8) == 0) + return arm_misc(instr); + + if ((op1 & 0x19) == 0x19 && (op2 & 0x9) == 0x8) + return arm_halfword_multiply(instr); + + if ((op1 & 0x10) == 0 && op2 == 0x9) + return arm_multiply(instr); + + if ((op1 & 0x10) == 0x10 && op2 == 0x9) + return arm_sync_primitive(instr); + + if ((op1 & 0x12) != 0x02 && (op2 == 0xB || (op2 & 0xD) == 0xD)) + return arm_extra_loadstore(instr); + + if ((op1 & 0x12) == 0x02 && (op2 == 0xB || (op2 & 0xD) == 0xD)) + return arm_extra_loadstore_unpriv(instr); + } else { + if ((op1 & 0x19) != 0x10) + return arm_dataproc_immed(instr); + + if (op1 == 0x10) { + /* 16 bit immediate load (mov (immed)) [encoding A2] */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if (op1 == 0x14) { + /* high halfword 16 bit immediate load (movt) [encoding A1] */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if ((op1 & 0x1B) == 0x12) + return arm_msr_hints(instr); + } + + return FASTTRAP_T_INV; +} + +int dtrace_decode_arm(uint32_t instr) +{ + int cond = BITS(instr,28,0xF), op1 = BITS(instr,25,0x7), op = BITS(instr,4,0x1); + + if (cond == 0xF) + return arm_unconditional(instr); + + if ((op1 & 0x6) == 0) + return arm_dataproc_misc(instr); + + if (op1 == 0x2) + return arm_loadstore_wordbyte(instr); + + if (op1 == 0x3 && op == 0) + return arm_loadstore_wordbyte(instr); + + if (op1 == 0x3 && op == 1) + return arm_media(instr); + + if ((op1 & 0x6) == 0x4) + return arm_branch_link_blockdata(instr); + + if ((op1 & 0x6) == 0x6) + return arm_syscall_coproc(instr); + + return FASTTRAP_T_INV; +} + +/* + * Thumb 16-bit decoder + */ + +static +int thumb16_cond_supervisor(uint16_t instr) +{ + int opcode = BITS(instr,8,0xF); + + if ((opcode & 0xE) != 0xE) + return FASTTRAP_T_B_COND; + + return FASTTRAP_T_INV; +} + +static +int thumb16_misc(uint16_t instr) +{ + int opcode = BITS(instr,5,0x7F); + + if ((opcode & 0x70) == 0x30 || (opcode & 0x70) == 0x70) { + /* setend, cps, breakpoint, or if-then, not instrumentable */ + return FASTTRAP_T_INV; + } else if ((opcode & 0x78) == 0x28) { + /* Doesn't modify pc, but this happens a lot so make this a special case for emulation */ + return FASTTRAP_T_PUSH_LR; + } else if ((opcode & 0x78) == 0x68) { + return FASTTRAP_T_POP_PC; + } else if ((opcode & 0x28) == 0x08) { + return FASTTRAP_T_CB_N_Z; + } + + /* All other instructions work on low regs only and are instrumentable */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_loadstore_single(__unused uint16_t instr) +{ + /* These all access the low registers or SP only */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_data_special_and_branch(uint16_t instr) +{ + int opcode = BITS(instr,6,0xF); + + if (opcode == 0x4) { + /* Unpredictable */ + return FASTTRAP_T_INV; + } else if ((opcode & 0xC) == 0xC) { + /* bx or blx */ + /* Only instrument the bx */ + if ((opcode & 0x2) == 0) + return FASTTRAP_T_BX_REG; + return FASTTRAP_T_INV; + } else { + /* Data processing on high registers, only instrument mov pc, reg */ + if ((opcode & 0xC) == 0x8 && THUMB16_HRD(instr) == REG_PC) + return FASTTRAP_T_CPY_PC; + + if (THUMB16_HRM(instr) != REG_PC && THUMB16_HRD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb16_data_proc(__unused uint16_t instr) +{ + /* These all access the low registers only */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_shift_addsub_move_compare(__unused uint16_t instr) +{ + /* These all access the low registers only */ + return FASTTRAP_T_COMMON; +} + +static +int dtrace_decode_thumb16(uint16_t instr) +{ + int opcode = BITS(instr,10,0x3F); + + if ((opcode & 0x30) == 0) + return thumb16_shift_addsub_move_compare(instr); + + if (opcode == 0x10) + return thumb16_data_proc(instr); + + if (opcode == 0x11) + return thumb16_data_special_and_branch(instr); + + if ((opcode & 0x3E) == 0x12) { + /* ldr (literal) */ + return FASTTRAP_T_LDR_PC_IMMED; + } + + if ((opcode & 0x3C) == 0x14 || (opcode & 0x38) == 0x18 || (opcode & 0x38) == 0x20) + return thumb16_loadstore_single(instr); + + if ((opcode & 0x3E) == 0x28) { + /* adr, uses the pc */ + return FASTTRAP_T_INV; + } + + if ((opcode & 0x3E) == 0x2A) { + /* add (sp plus immediate) */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3C) == 0x2C) + return thumb16_misc(instr); + + if ((opcode & 0x3E) == 0x30) { + /* stm - can't access high registers */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3E) == 0x32) { + /* ldm - can't access high registers */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3C) == 0x34) { + return thumb16_cond_supervisor(instr); + } + + if ((opcode & 0x3E) == 0x38) { + /* b unconditional */ + return FASTTRAP_T_B_UNCOND; + } + + return FASTTRAP_T_INV; +} + +/* + * Thumb 32-bit decoder + */ + +static +int thumb32_coproc(uint16_t instr1, uint16_t instr2) +{ + /* Instrument any VFP data processing instructions, ignore the rest */ + + int op1 = BITS(instr1,4,0x3F), coproc = BITS(instr2,8,0xF), op = BITS(instr2,4,0x1); + + if ((op1 & 0x3E) == 0) { + /* Undefined */ + return FASTTRAP_T_INV; + } + + if ((coproc & 0xE) == 0xA || (op1 & 0x30) == 0x30) { + /* VFP instruction */ + uint32_t instr = thumb32_instword_to_arm(instr1,instr2); + + if ((op1 & 0x30) == 0x30) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x3A) == 0x02 || (op1 & 0x38) == 0x08 || (op1 & 0x30) == 0x10) + return vfp_loadstore(instr); + + if ((op1 & 0x3E) == 0x04) + return vfp_64transfer(instr); + + if ((op1 & 0x30) == 0x20) { + /* VFP data processing or 8, 16, or 32 bit move between ARM reg and VFP reg */ + if (op == 0) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } else { + return vfp_transfer(instr); + } + } + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_longmultiply(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0x7), op2 = BITS(instr2,4,0xF); + + if ((op1 == 1 && op2 == 0xF) || (op1 == 0x3 && op2 == 0xF)) { + /* Three register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Four register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && + THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_multiply(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0x7), op2 = BITS(instr2,4,0x3); + + if ((op1 == 0 && op2 == 1) || (op1 == 0x6 && (op2 & 0x2) == 0)) { + if (THUMB32_RT(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_misc(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_parallel_addsub_unsigned(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_parallel_addsub_signed(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_reg(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0xF), op2 = BITS(instr2,4,0xF); + + if (((0 <= op1) && (op1 <= 5)) && (op2 & 0x8) == 0x8) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x8) == 0 && op2 == 0) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x8) == 0x8 && (op2 & 0xC) == 0) + return thumb32_parallel_addsub_signed(instr1,instr2); + + if ((op1 & 0x8) == 0x8 && (op2 & 0xC) == 0x4) + return thumb32_parallel_addsub_unsigned(instr1,instr2); + + if ((op1 & 0xC) == 0x8 && (op2 & 0xC) == 0x8) + return thumb32_misc(instr1,instr2); + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_regshift(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,5,0xF), S = BITS(instr1,4,0x1); + + if (op == 0 || op == 0x4 || op == 0x8 || op == 0xD) { + /* These become test instructions if S is 1 and Rd is PC, otherwise they are data instructions. */ + if (S == 1) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && + THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + } else if (op == 0x2 || op == 0x3) { + /* These become moves if RN is PC, otherwise they are data insts. We don't instrument mov pc, reg here */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Normal three register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_store_single(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,5,0x7), op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the pc yet */ + if ((op1 == 0 || op1 == 1 || op1 == 2) && (op2 & 0x20) == 0) { + /* str (register) uses RM */ + if (THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadbyte_memhint(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), __unused op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the pc yet */ + if ((op1 == 0 || op1 == 0x2) && THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadhalfword_memhint(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the PC yet */ + if (op1 == 0 && op2 == 0 && THUMB32_RM(inst1,instr2) == REG_PC) + return FASTTRAP_T_INV; + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadword(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), op2 = BITS(instr2,6,0x3F); + + if ((op1 & 0x2) == 0 && THUMB32_RN(instr1,instr2) == REG_PC && THUMB32_RT(instr1,instr2) != REG_PC) + return FASTTRAP_T_LDR_PC_IMMED; + + if (op1 == 0 && op2 == 0) { + /* ldr (register) uses an additional reg */ + if (THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadstore_double_exclusive_table(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Don't instrument any of these */ + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadstore_multiple(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,7,0x3), L = BITS(instr1,4,0x1), uses_pc = BITS(instr2,15,0x1), uses_lr = BITS(instr2,14,0x1); + + if (op == 0 || op == 0x3) { + /* Privileged instructions: srs, rfe */ + return FASTTRAP_T_INV; + } + + /* Only emulate a use of the pc if it's a return from function: ldmia sp!, { ... pc }, aka pop { ... pc } */ + if (op == 0x1 && L == 1 && THUMB32_RN(instr1,instr2) == REG_SP && uses_pc == 1) + return FASTTRAP_T_LDM_PC; + + /* stmia sp!, { ... lr }, aka push { ... lr } doesn't touch the pc, but it is very common, so special case it */ + if (op == 0x2 && L == 0 && THUMB32_RN(instr1,instr2) == REG_SP && uses_lr == 1) + return FASTTRAP_T_STM_LR; + + if (THUMB32_RN(instr1,instr2) != REG_PC && uses_pc == 0) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_misc_control(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Privileged, and instructions dealing with ThumbEE */ + return FASTTRAP_T_INV; +} + +static +int thumb32_cps_hints(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Privileged */ + return FASTTRAP_T_INV; +} + +static +int thumb32_b_misc_control(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,4,0x7F), op1 = BITS(instr2,12,0x7), __unused op2 = BITS(instr2,8,0xF); + + if ((op1 & 0x5) == 0) { + if ((op & 0x38) != 0x38) + return FASTTRAP_T_B_COND; + + if (op == 0x3A) + return thumb32_cps_hints(instr1,instr2); + + if (op == 0x3B) + return thumb32_misc_control(instr1,instr2); + } + + if ((op1 & 0x5) == 1) + return FASTTRAP_T_B_UNCOND; + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_plain_immed(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,4,0x1F); + + if (op == 0x04 || op == 0x0C || op == 0x16) { + /* mov, movt, bfi, bfc */ + /* These use only RD */ + if (THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_mod_immed(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,5,0xF), S = BITS(instr1,4,0x1); + + if (op == 0x2 || op == 0x3) { + /* These allow REG_PC in RN, but it doesn't mean use the PC! */ + if (THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if (op == 0 || op == 0x4 || op == 0x8 || op == 0xD) { + /* These are test instructions, if the sign bit is set and RD is the PC. */ + if (S && THUMB32_RD(instr1,instr2) == REG_PC) + return FASTTRAP_T_COMMON; + } + + if (THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int dtrace_decode_thumb32(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,11,0x3), op2 = BITS(instr1,4,0x7F), op = BITS(instr2,15,0x1); + + if (op1 == 0x1) { + if ((op2 & 0x64) == 0) + return thumb32_loadstore_multiple(instr1,instr2); + + if ((op2 & 0x64) == 0x04) + return thumb32_loadstore_double_exclusive_table(instr1,instr2); + + if ((op2 & 0x60) == 0x20) + return thumb32_dataproc_regshift(instr1,instr2); + + if ((op2 & 0x40) == 0x40) + return thumb32_coproc(instr1,instr2); + } + + if (op1 == 0x2) { + if ((op2 & 0x20) == 0 && op == 0) + return thumb32_dataproc_mod_immed(instr1,instr2); + + if ((op2 & 0x20) == 0x20 && op == 0) + return thumb32_dataproc_plain_immed(instr1,instr2); + + if (op == 1) + return thumb32_b_misc_control(instr1,instr2); + } + + if (op1 == 0x3) { + if ((op2 & 0x71) == 0) + return thumb32_store_single(instr1,instr2); + + if ((op2 & 0x71) == 0x10) { + return vfp_struct_loadstore(thumb32_instword_to_arm(instr1,instr2)); + } + + if ((op2 & 0x67) == 0x01) + return thumb32_loadbyte_memhint(instr1,instr2); + + if ((op2 & 0x67) == 0x03) + return thumb32_loadhalfword_memhint(instr1,instr2); + + if ((op2 & 0x67) == 0x05) + return thumb32_loadword(instr1,instr2); + + if ((op2 & 0x67) == 0x07) { + /* Undefined instruction */ + return FASTTRAP_T_INV; + } + + if ((op2 & 0x70) == 0x20) + return thumb32_dataproc_reg(instr1,instr2); + + if ((op2 & 0x78) == 0x30) + return thumb32_multiply(instr1,instr2); + + if ((op2 & 0x78) == 0x38) + return thumb32_longmultiply(instr1,instr2); + + if ((op2 & 0x40) == 0x40) + return thumb32_coproc(instr1,instr2); + } + + return FASTTRAP_T_INV; +} + +int dtrace_decode_thumb(uint32_t instr) +{ + uint16_t* pInstr = (uint16_t*) &instr; + uint16_t hw1 = pInstr[0], hw2 = pInstr[1]; + + int size = BITS(hw1,11,0x1F); + + if (size == 0x1D || size == 0x1E || size == 0x1F) + return dtrace_decode_thumb32(hw1,hw2); + else + return dtrace_decode_thumb16(hw1); +} + +struct arm64_decode_entry { + uint32_t mask; + uint32_t value; + uint32_t type; +}; + +struct arm64_decode_entry arm64_decode_table[] = { + { .mask = 0xFFFFFFFF, .value = FASTTRAP_ARM64_OP_VALUE_FUNC_ENTRY, .type = FASTTRAP_T_ARM64_STANDARD_FUNCTION_ENTRY }, + { .mask = FASTTRAP_ARM64_OP_MASK_LDR_S_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LDR_S_PC_REL, .type = FASTTRAP_T_ARM64_LDR_S_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_LDR_W_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LDR_W_PC_REL, .type = FASTTRAP_T_ARM64_LDR_W_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_LDR_D_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LDR_D_PC_REL, .type = FASTTRAP_T_ARM64_LDR_D_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_LDR_X_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LDR_X_PC_REL, .type = FASTTRAP_T_ARM64_LDR_X_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_LDR_Q_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LDR_Q_PC_REL, .type = FASTTRAP_T_ARM64_LDR_Q_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_LRDSW_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LRDSW_PC_REL, .type = FASTTRAP_T_ARM64_LDRSW_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_B_COND_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_B_COND_PC_REL, .type = FASTTRAP_T_ARM64_B_COND }, + { .mask = FASTTRAP_ARM64_OP_MASK_CBNZ_W_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_CBNZ_W_PC_REL, .type = FASTTRAP_T_ARM64_CBNZ_W }, + { .mask = FASTTRAP_ARM64_OP_MASK_CBNZ_X_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_CBNZ_X_PC_REL, .type = FASTTRAP_T_ARM64_CBNZ_X }, + { .mask = FASTTRAP_ARM64_OP_MASK_CBZ_W_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_CBZ_W_PC_REL, .type = FASTTRAP_T_ARM64_CBZ_W }, + { .mask = FASTTRAP_ARM64_OP_MASK_CBZ_X_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_CBZ_X_PC_REL, .type = FASTTRAP_T_ARM64_CBZ_X }, + { .mask = FASTTRAP_ARM64_OP_MASK_TBNZ_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_TBNZ_PC_REL, .type = FASTTRAP_T_ARM64_TBNZ }, + { .mask = FASTTRAP_ARM64_OP_MASK_TBZ_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_TBZ_PC_REL, .type = FASTTRAP_T_ARM64_TBZ }, + { .mask = FASTTRAP_ARM64_OP_MASK_B_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_B_PC_REL, .type = FASTTRAP_T_ARM64_B }, + { .mask = FASTTRAP_ARM64_OP_MASK_BL_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_BL_PC_REL, .type = FASTTRAP_T_ARM64_BL }, + { .mask = FASTTRAP_ARM64_OP_MASK_BLR, .value = FASTTRAP_ARM64_OP_VALUE_BLR, .type = FASTTRAP_T_ARM64_BLR }, + { .mask = FASTTRAP_ARM64_OP_MASK_BR, .value = FASTTRAP_ARM64_OP_VALUE_BR, .type = FASTTRAP_T_ARM64_BR }, + { .mask = FASTTRAP_ARM64_OP_MASK_RET, .value = FASTTRAP_ARM64_OP_VALUE_RET, .type = FASTTRAP_T_ARM64_RET }, + { .mask = FASTTRAP_ARM64_OP_MASK_ADRP, .value = FASTTRAP_ARM64_OP_VALUE_ADRP, .type = FASTTRAP_T_ARM64_ADRP }, + { .mask = FASTTRAP_ARM64_OP_MASK_ADR, .value = FASTTRAP_ARM64_OP_VALUE_ADR, .type = FASTTRAP_T_ARM64_ADR }, + { .mask = FASTTRAP_ARM64_OP_MASK_PRFM, .value = FASTTRAP_ARM64_OP_VALUE_PRFM, .type = FASTTRAP_T_ARM64_PRFM }, + { .mask = FASTTRAP_ARM64_OP_MASK_EXCL_MEM, .value = FASTTRAP_ARM64_OP_VALUE_EXCL_MEM, .type = FASTTRAP_T_ARM64_EXCLUSIVE_MEM }}; + +#define NUM_DECODE_ENTRIES (sizeof(arm64_decode_table) / sizeof(struct arm64_decode_entry)) + + + +int dtrace_decode_arm64(uint32_t instr) +{ + unsigned i; + + for (i = 0; i < NUM_DECODE_ENTRIES; i++) { + if ((instr & arm64_decode_table[i].mask) == arm64_decode_table[i].value) { + return arm64_decode_table[i].type; + } + } + + return FASTTRAP_T_COMMON; +} + + diff --git a/bsd/dev/arm64/dtrace_isa.c b/bsd/dev/arm64/dtrace_isa.c new file mode 100644 index 000000000..3f81cb706 --- /dev/null +++ b/bsd/dev/arm64/dtrace_isa.c @@ -0,0 +1,696 @@ +/* + * Copyright (c) 2005-2008 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from + * mach/ppc/thread_status.h */ +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for thread_wakeup() */ +#include +#include +#include +#include + +extern struct arm_saved_state *find_kern_regs(thread_t); + +extern dtrace_id_t dtrace_probeid_error; /* special ERROR probe */ +typedef arm_saved_state_t savearea_t; + +extern lck_attr_t *dtrace_lck_attr; +extern lck_grp_t *dtrace_lck_grp; + + +struct frame { + struct frame *backchain; + uintptr_t retaddr; +}; + +/* + * Atomicity and synchronization + */ +inline void +dtrace_membar_producer(void) +{ +#if __ARM_SMP__ + __asm__ volatile("dmb ish" : : : "memory"); +#else + __asm__ volatile("nop" : : : "memory"); +#endif +} + +inline void +dtrace_membar_consumer(void) +{ +#if __ARM_SMP__ + __asm__ volatile("dmb ish" : : : "memory"); +#else + __asm__ volatile("nop" : : : "memory"); +#endif +} + +/* + * Interrupt manipulation + * XXX dtrace_getipl() can be called from probe context. + */ +int +dtrace_getipl(void) +{ + /* + * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE + * in osfmk/kern/cpu_data.h + */ + /* return get_interrupt_level(); */ + return (ml_at_interrupt_context() ? 1 : 0); +} + +#if __ARM_SMP__ +/* + * MP coordination + */ + +decl_lck_mtx_data(static, dt_xc_lock); +static uint32_t dt_xc_sync; + +typedef struct xcArg { + processorid_t cpu; + dtrace_xcall_t f; + void *arg; +} xcArg_t; + +static void +xcRemote(void *foo) +{ + xcArg_t *pArg = (xcArg_t *) foo; + + if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) + (pArg->f) (pArg->arg); + + if (hw_atomic_sub(&dt_xc_sync, 1) == 0) + thread_wakeup((event_t) &dt_xc_sync); +} +#endif + +/* + * dtrace_xcall() is not called from probe context. + */ +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg) +{ +#if __ARM_SMP__ + /* Only one dtrace_xcall in flight allowed */ + lck_mtx_lock(&dt_xc_lock); + + xcArg_t xcArg; + + xcArg.cpu = cpu; + xcArg.f = f; + xcArg.arg = arg; + + cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg); + + lck_mtx_unlock(&dt_xc_lock); + return; +#else +#pragma unused(cpu) + /* On uniprocessor systems, the cpu should always be either ourselves or all */ + ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL); + + (*f)(arg); + return; +#endif +} + +/* + * Initialization + */ +void +dtrace_isa_init(void) +{ + lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr); + return; +} + + +/** + * Register definitions + */ +#define ARM_FP 7 +#define ARM_SP 13 +#define ARM_LR 14 +#define ARM_PC 15 +#define ARM_CPSR 16 + +#define ARM64_FP 29 +#define ARM64_LR 30 +#define ARM64_SP 31 +#define ARM64_PC 32 +#define ARM64_CPSR 33 + +/* + * Runtime and ABI + */ +uint64_t +dtrace_getreg(struct regs * savearea, uint_t reg) +{ + struct arm_saved_state *regs = (struct arm_saved_state *) savearea; + + if (is_saved_state32(regs)) { + // Fix special registers if user is 32 bits + switch (reg) { + case ARM64_FP: + reg = ARM_FP; + break; + case ARM64_SP: + reg = ARM_SP; + break; + case ARM64_LR: + reg = ARM_LR; + break; + case ARM64_PC: + reg = ARM_PC; + break; + case ARM64_CPSR: + reg = ARM_CPSR; + break; + } + } + + if (!check_saved_state_reglimit(regs, reg)) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + return ((uint64_t)get_saved_state_reg(regs, reg)); +} + +#define RETURN_OFFSET 4 +#define RETURN_OFFSET64 8 + +static int +dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc, + user_addr_t sp) +{ + int ret = 0; + boolean_t is64bit = proc_is64bit(current_proc()); + + ASSERT(pcstack == NULL || pcstack_limit > 0); + + while (pc != 0) { + ret++; + if (pcstack != NULL) { + *pcstack++ = (uint64_t) pc; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + } + + if (sp == 0) + break; + + if (is64bit) { + pc = dtrace_fuword64((sp + RETURN_OFFSET64)); + sp = dtrace_fuword64(sp); + } else { + pc = dtrace_fuword32((sp + RETURN_OFFSET)); + sp = dtrace_fuword32(sp); + } + } + + return (ret); +} + +void +dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit) +{ + thread_t thread = current_thread(); + savearea_t *regs; + user_addr_t pc, sp, fp; + volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags; + int n; + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (thread == NULL) + goto zero; + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + goto zero; + + *pcstack++ = (uint64_t)dtrace_proc_selfpid(); + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = get_saved_state_pc(regs); + sp = get_saved_state_sp(regs); + fp = get_saved_state_fp(regs); + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t) pc; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = get_saved_state_lr(regs); + } + + n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp); + + ASSERT(n >= 0); + ASSERT(n <= pcstack_limit); + + pcstack += n; + pcstack_limit -= n; + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0ULL; +} + +int +dtrace_getustackdepth(void) +{ + thread_t thread = current_thread(); + savearea_t *regs; + user_addr_t pc, sp, fp; + int n = 0; + + if (thread == NULL) + return 0; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) + return (-1); + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + return 0; + + pc = get_saved_state_pc(regs); + sp = get_saved_state_sp(regs); + fp = get_saved_state_fp(regs); + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + n++; + pc = get_saved_state_lr(regs); + } + + /* + * Note that unlike ppc, the arm code does not use + * CPU_DTRACE_USTACK_FP. This is because arm always + * traces from the sp, even in syscall/profile/fbt + * providers. + */ + + n += dtrace_getustack_common(NULL, 0, pc, fp); + + return (n); +} + +void +dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit) +{ + thread_t thread = current_thread(); + boolean_t is64bit = proc_is64bit(current_proc()); + savearea_t *regs; + user_addr_t pc, sp; + volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags; + +#if 0 + uintptr_t oldcontext; + size_t s1, s2; +#endif + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (thread == NULL) + goto zero; + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + goto zero; + + *pcstack++ = (uint64_t)dtrace_proc_selfpid(); + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = get_saved_state_pc(regs); + sp = get_saved_state_lr(regs); + +#if 0 /* XXX signal stack crawl */ + oldcontext = lwp->lwp_oldcontext; + + if (p->p_model == DATAMODEL_NATIVE) { + s1 = sizeof(struct frame) + 2 * sizeof(long); + s2 = s1 + sizeof(siginfo_t); + } else { + s1 = sizeof(struct frame32) + 3 * sizeof(int); + s2 = s1 + sizeof(siginfo32_t); + } +#endif + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t) pc; + *fpstack++ = 0; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + if (is64bit) + pc = dtrace_fuword64(sp); + else + pc = dtrace_fuword32(sp); + } + while (pc != 0 && sp != 0) { + *pcstack++ = (uint64_t) pc; + *fpstack++ = sp; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + +#if 0 /* XXX signal stack crawl */ + if (oldcontext == sp + s1 || oldcontext == sp + s2) { + if (p->p_model == DATAMODEL_NATIVE) { + ucontext_t *ucp = (ucontext_t *) oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fulword(&gregs[REG_FP]); + pc = dtrace_fulword(&gregs[REG_PC]); + + oldcontext = dtrace_fulword(&ucp->uc_link); + } else { + ucontext_t *ucp = (ucontext_t *) oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fuword32(&gregs[EBP]); + pc = dtrace_fuword32(&gregs[EIP]); + + oldcontext = dtrace_fuword32(&ucp->uc_link); + } + } else +#endif + { + if (is64bit) { + pc = dtrace_fuword64((sp + RETURN_OFFSET64)); + sp = dtrace_fuword64(sp); + } else { + pc = dtrace_fuword32((sp + RETURN_OFFSET)); + sp = dtrace_fuword32(sp); + } + } + +#if 0 + /* XXX ARMTODO*/ + /* + * This is totally bogus: if we faulted, we're going to clear + * the fault and break. This is to deal with the apparently + * broken Java stacks on x86. + */ + if (*flags & CPU_DTRACE_FAULT) { + *flags &= ~CPU_DTRACE_FAULT; + break; + } +#endif + } + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0ULL; +} + + +void +dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes, + uint32_t * intrpc) +{ + struct frame *fp = (struct frame *) __builtin_frame_address(0); + struct frame *nextfp, *minfp, *stacktop; + int depth = 0; + int on_intr; + int last = 0; + uintptr_t pc; + uintptr_t caller = CPU->cpu_dtrace_caller; + + if ((on_intr = CPU_ON_INTR(CPU)) != 0) + stacktop = (struct frame *) dtrace_get_cpu_int_stack_top(); + else + stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size); + + minfp = fp; + + aframes++; + + if (intrpc != NULL && depth < pcstack_limit) + pcstack[depth++] = (pc_t) intrpc; + + while (depth < pcstack_limit) { + nextfp = *(struct frame **) fp; + pc = *(uintptr_t *) (((uintptr_t) fp) + RETURN_OFFSET64); + + if (nextfp <= minfp || nextfp >= stacktop) { + if (on_intr) { + /* + * Hop from interrupt stack to thread stack. + */ + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + if (arm_kern_regs) { + nextfp = (struct frame *)(saved_state64(arm_kern_regs)->fp); + + { + vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread()); + + minfp = (struct frame *)kstack_base; + stacktop = (struct frame *)(kstack_base + kernel_stack_size); + } + + on_intr = 0; + + if (nextfp <= minfp || nextfp >= stacktop) { + last = 1; + } + } else { + /* + * If this thread was on the interrupt stack, but did not + * take an interrupt (i.e, the idle thread), there is no + * explicit saved state for us to use. + */ + last = 1; + } + } else { + { + /* + * This is the last frame we can process; indicate + * that we should return after processing this frame. + */ + last = 1; + } + } + } + if (aframes > 0) { + if (--aframes == 0 && caller != (uintptr_t)NULL) { + /* + * We've just run out of artificial frames, + * and we have a valid caller -- fill it in + * now. + */ + ASSERT(depth < pcstack_limit); + pcstack[depth++] = (pc_t) caller; + caller = (uintptr_t)NULL; + } + } else { + if (depth < pcstack_limit) + pcstack[depth++] = (pc_t) pc; + } + + if (last) { + while (depth < pcstack_limit) + pcstack[depth++] = (pc_t) NULL; + return; + } + fp = nextfp; + minfp = fp; + } +} + +/* + * On arm64, we support both 32bit and 64bit user processes. + * This routine is only called when handling 32bit processes + * where thumb_mode is pertinent. + * If this routine is called when handling 64bit processes + * thumb_mode should always be zero. + */ +int +dtrace_instr_size(uint32_t instr, int thumb_mode) +{ + if (thumb_mode) { + uint16_t instr16 = *(uint16_t*) &instr; + if (((instr16 >> 11) & 0x1F) > 0x1C) + return 4; + else + return 2; + } else { + return 4; + } +} + +uint64_t +dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) +{ +#pragma unused(arg, aframes) + uint64_t val = 0; + struct frame *fp = (struct frame *)__builtin_frame_address(0); + uintptr_t *stack; + uintptr_t pc; + int i; + + /* + * A total of 8 arguments are passed via registers; any argument with + * index of 7 or lower is therefore in a register. + */ + int inreg = 7; + + for (i = 1; i <= aframes; ++i) { + fp = fp->backchain; + pc = fp->retaddr; + + if (dtrace_invop_callsite_pre != NULL + && pc > (uintptr_t) dtrace_invop_callsite_pre + && pc <= (uintptr_t) dtrace_invop_callsite_post) + { + /* fp points to frame of dtrace_invop() activation */ + fp = fp->backchain; /* to fbt_perfCallback activation */ + fp = fp->backchain; /* to sleh_synchronous activation */ + fp = fp->backchain; /* to fleh_synchronous activation */ + + arm_saved_state_t *tagged_regs = (arm_saved_state_t*) ((void*) &fp[1]); + arm_saved_state64_t *saved_state = saved_state64(tagged_regs); + + if (arg <= inreg) { + /* the argument will be found in a register */ + stack = (uintptr_t*) &saved_state->x[0]; + } else { + /* the argument will be found in the stack */ + fp = (struct frame*) saved_state->sp; + stack = (uintptr_t*) &fp[1]; + arg -= (inreg + 1); + } + + goto load; + } + } + + /* + * We know that we did not come through a trap to get into + * dtrace_probe() -- We arrive here when the provider has + * called dtrace_probe() directly. + * The probe ID is the first argument to dtrace_probe(). + * We must advance beyond that to get the argX. + */ + arg++; /* Advance past probeID */ + + if (arg <= inreg) { + /* + * This shouldn't happen. If the argument is passed in a + * register then it should have been, well, passed in a + * register... + */ + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + arg -= (inreg + 1); + stack = (uintptr_t*) &fp[1]; /* Find marshalled arguments */ + +load: + if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t), + mstate, vstate)) { + /* dtrace_probe arguments arg0 ... arg4 are 64bits wide */ + val = dtrace_load64((uint64_t)(stack + arg)); + } + + return (val); +} + +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fltoffs, int fault, uint64_t illval) +{ + /* XXX ARMTODO */ + /* + * For the case of the error probe firing lets + * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG. + */ + state->dts_arg_error_illval = illval; + dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault ); +} + +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + /* XXX ARMTODO check copied from ppc/x86*/ + /* + * "base" is the smallest toxic address in the range, "limit" is the first + * VALID address greater than "base". + */ + func(0x0, VM_MIN_KERNEL_ADDRESS); + if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) + func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0); +} + diff --git a/bsd/dev/arm64/dtrace_subr_arm.c b/bsd/dev/arm64/dtrace_subr_arm.c new file mode 100644 index 000000000..efdbb54f5 --- /dev/null +++ b/bsd/dev/arm64/dtrace_subr_arm.c @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * #pragma ident "@(#)dtrace_subr.c 1.12 05/06/08 SMI" + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int (*dtrace_pid_probe_ptr) (arm_saved_state_t *); +int (*dtrace_return_probe_ptr) (arm_saved_state_t *); + +kern_return_t +dtrace_user_probe(arm_saved_state_t *); + +kern_return_t +dtrace_user_probe(arm_saved_state_t *regs) +{ + /* + * FIXME + * + * The only call path into this method is always a user trap. + * We don't need to test for user trap, but should assert it. + */ + + lck_rw_t *rwp; + struct proc *p = current_proc(); + int is_fasttrap = 0; + + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + + kauth_cred_uthread_update(uthread, p); + + if (is_saved_state32(regs)) { + if (saved_state32(regs)->cpsr & PSR_TF) { + uint16_t pc; + if (copyin((user_addr_t)saved_state32(regs)->pc, &pc, sizeof(uint16_t))) { + return KERN_FAILURE; + } + is_fasttrap = (pc == FASTTRAP_THUMB32_RET_INSTR); + } else { + uint32_t pc; + if (copyin((user_addr_t)saved_state32(regs)->pc, &pc, sizeof(uint32_t))) { + return KERN_FAILURE; + } + is_fasttrap = (pc == FASTTRAP_ARM32_RET_INSTR); + } + } else { + uint32_t pc; + if (copyin((user_addr_t)saved_state64(regs)->pc, &pc, sizeof(uint32_t))) { + return KERN_FAILURE; + } + is_fasttrap = (pc == FASTTRAP_ARM64_RET_INSTR); + } + + if (is_fasttrap) { + uint8_t step = uthread->t_dtrace_step; + uint8_t ret = uthread->t_dtrace_ret; + user_addr_t npc = uthread->t_dtrace_npc; + + if (uthread->t_dtrace_ast) { + printf("dtrace_user_probe() should be calling aston()\n"); + // aston(thread); + // uthread->t_sig_check = 1; + } + + /* + * Clear all user tracing flags. + */ + uthread->t_dtrace_ft = 0; + + /* + * If we weren't expecting a quick return to the kernel, just kill + * the process as though it had just executed an unassigned + * trap instruction. + */ + if (step == 0) { + /* + * APPLE NOTE: We're returning KERN_FAILURE, which causes + * the generic signal handling code to take over, which will effectively + * deliver a EXC_BAD_INSTRUCTION to the user process. + */ + return KERN_FAILURE; + } + + /* + * If we hit this trap unrelated to a return probe, we're + * here to either: + * + * 1. Reset the AST flag, since we deferred a signal + * until after we logically single-stepped the instruction we + * copied out. + * + * 2. Just return to normal execution (required for U64). + */ + if (ret == 0) { + set_saved_state_pc(regs, npc); + return KERN_SUCCESS; + } + + /* + * We need to wait until after we've called the + * dtrace_return_probe_ptr function pointer to step the pc. + */ + rwp = &CPU->cpu_ft_lock; + lck_rw_lock_shared(rwp); + + if (dtrace_return_probe_ptr != NULL) + (void) (*dtrace_return_probe_ptr)(regs); + lck_rw_unlock_shared(rwp); + + set_saved_state_pc(regs, npc); + + return KERN_SUCCESS; + } else { + rwp = &CPU->cpu_ft_lock; + + /* + * The DTrace fasttrap provider uses a trap, + * FASTTRAP_{ARM,THUMB}_INSTR. We let + * DTrace take the first crack at handling + * this trap; if it's not a probe that DTrace knows about, + * we call into the trap() routine to handle it like a + * breakpoint placed by a conventional debugger. + */ + + /* + * APPLE NOTE: I believe the purpose of the reader/writers lock + * is thus: There are times which dtrace needs to prevent calling + * dtrace_pid_probe_ptr(). Sun's original impl grabbed a plain + * mutex here. However, that serialized all probe calls, and + * destroyed MP behavior. So now they use a RW lock, with probes + * as readers, and the top level synchronization as a writer. + */ + lck_rw_lock_shared(rwp); + if (dtrace_pid_probe_ptr != NULL && + (*dtrace_pid_probe_ptr)(regs) == 0) { + lck_rw_unlock_shared(rwp); + return KERN_SUCCESS; + } + lck_rw_unlock_shared(rwp); + + /* + * If the instruction that caused the breakpoint trap doesn't + * look like our trap anymore, it may be that this tracepoint + * was removed just after the user thread executed it. In + * that case, return to user land to retry the instuction. + * + * Note that the PC points to the instruction that caused the fault. + */ + if (is_saved_state32(regs)) { + if (saved_state32(regs)->cpsr & PSR_TF) { + uint16_t instr; + if (fuword16(saved_state32(regs)->pc, &instr) == 0 && instr != FASTTRAP_THUMB32_INSTR) { + return KERN_SUCCESS; + } + } else { + uint32_t instr; + if (fuword32(saved_state32(regs)->pc, &instr) == 0 && instr != FASTTRAP_ARM32_INSTR) { + return KERN_SUCCESS; + } + } + } else { + uint32_t instr; + if (fuword32(saved_state64(regs)->pc, &instr) == 0 && instr != FASTTRAP_ARM64_INSTR) { + return KERN_SUCCESS; + } + } + } + + return KERN_FAILURE; +} + +void +dtrace_safe_synchronous_signal(void) +{ + /* Not implemented */ +} + +int +dtrace_safe_defer_signal(void) +{ + /* Not implemented */ + return 0; +} diff --git a/bsd/dev/arm64/fasttrap_isa.c b/bsd/dev/arm64/fasttrap_isa.c new file mode 100644 index 000000000..c0af6a9e2 --- /dev/null +++ b/bsd/dev/arm64/fasttrap_isa.c @@ -0,0 +1,2127 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * #pragma ident "@(#)fasttrap_isa.c 1.19 05/09/14 SMI" + */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +extern dtrace_id_t dtrace_probeid_error; + +/* Solaris proc_t is the struct. Darwin's proc_t is a pointer to it. */ +#define proc_t struct proc /* Steer clear of the Darwin typedef for proc_t */ + +extern int dtrace_decode_arm64(uint32_t instr); +extern int dtrace_decode_arm(uint32_t instr); +extern int dtrace_decode_thumb(uint32_t instr); + +/* + * Lossless User-Land Tracing on ARM + * --------------------------------- + * + * The details here will be fleshed out as more of this is implemented. The + * basic design will be the same way as tracing works in x86. + * + * Some ARM specific issues: + * + * We need to patch differently for ARM instructions and Thumb instructions. + * When we hit a probe, we check to see if the mode we're currently in is the + * same as the mode we're patching for. If not, we remove the tracepoint and + * abort. This ARM/Thumb information is pulled in from the arch specific + * information in the fasttrap probe. + * + * On ARM, any instruction that uses registers can also use the pc as a + * register. This presents problems during emulation because we have copied + * the instruction and thus the pc can be different. Currently we've emulated + * any instructions that use the pc if they can be used in a return probe. + * Eventually we will want to support all instructions that use the pc, but + * to do so requires disassembling the instruction and reconstituting it by + * substituting a different register. + * + */ + +#define THUMB_INSTR(x) (*(uint16_t*) &(x)) + +#define SIGNEXTEND(x,v) ((((int) (x)) << (32-(v))) >> (32-(v))) +#define ALIGNADDR(x,v) (((x) >> (v)) << (v)) +#define GETITSTATE(x) ((((x) >> 8) & 0xFC) | (((x) >> 25) & 0x3)) +#define ISLASTINIT(x) (((x) & 0xF) == 8) + +#define SET16(x,w) *((uint16_t*) (x)) = (w) +#define SET32(x,w) *((uint32_t*) (x)) = (w) + +#define IS_ARM32_NOP(x) ((x) == 0xE1A00000) +/* Marker for is-enabled probes */ +#define IS_ARM32_IS_ENABLED(x) ((x) == 0xE0200000) + +#define IS_ARM64_NOP(x) ((x) == 0xD503201F) +/* Marker for is-enabled probes */ +#define IS_ARM64_IS_ENABLED(x) ((x) == 0xD2800000) + +#define IS_THUMB32_NOP(x) ((x) == 0x46C0) +/* Marker for is-enabled probes */ +#define IS_THUMB32_IS_ENABLED(x) ((x) == 0x4040) + +#define ARM_LDM_UF (1 << 23) +#define ARM_LDM_PF (1 << 24) +#define ARM_LDM_WF (1 << 21) + +#define ARM_LDR_UF (1 << 23) +#define ARM_LDR_BF (1 << 22) + +static void +flush_caches(void) +{ + /* TODO There were some problems with flushing just the cache line that had been modified. + * For now, we'll flush the entire cache, until we figure out how to flush just the patched block. + */ + FlushPoU_Dcache(); + InvalidatePoU_Icache(); +} + + +static int fasttrap_tracepoint_init32 (proc_t *, fasttrap_tracepoint_t *, user_addr_t, fasttrap_probe_type_t); +static int fasttrap_tracepoint_init64 (proc_t *, fasttrap_tracepoint_t *, user_addr_t, fasttrap_probe_type_t); + +int +fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, + user_addr_t pc, fasttrap_probe_type_t type) +{ + if (proc_is64bit(p)) { + return fasttrap_tracepoint_init64(p, tp, pc, type); + } else { + return fasttrap_tracepoint_init32(p, tp, pc, type); + } +} + +static int +fasttrap_tracepoint_init32(proc_t *p, fasttrap_tracepoint_t *tp, + user_addr_t pc, fasttrap_probe_type_t type) +{ +#pragma unused(type) + uint32_t instr; + + /* + * Read the instruction at the given address out of the process's + * address space. We don't have to worry about a debugger + * changing this instruction before we overwrite it with our trap + * instruction since P_PR_LOCK is set. Since instructions can span + * pages, we potentially read the instruction in two parts. If the + * second part fails, we just zero out that part of the instruction. + */ + /* + * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey... + */ + + if (uread(p, &instr, 4, pc) != 0) + return (-1); + + /* We want &instr to always point to the saved instruction, so just copy the + * whole thing When cast to a pointer to a uint16_t, that will give us a + * pointer to the first two bytes, which is the thumb instruction. + */ + tp->ftt_instr = instr; + + if (tp->ftt_fntype != FASTTRAP_FN_DONE_INIT) { + switch(tp->ftt_fntype) { + case FASTTRAP_FN_UNKNOWN: + /* Can't instrument without any information. We can add some heuristics later if necessary. */ + return (-1); + + case FASTTRAP_FN_USDT: + if (IS_ARM32_NOP(instr) || IS_ARM32_IS_ENABLED(instr)) { + tp->ftt_thumb = 0; + } else if (IS_THUMB32_NOP(THUMB_INSTR(instr)) || IS_THUMB32_IS_ENABLED(THUMB_INSTR(instr))) { + tp->ftt_thumb = 1; + } else { + /* Shouldn't reach here - this means we don't recognize + * the instruction at one of the USDT probe locations + */ + return (-1); + } + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + case FASTTRAP_FN_ARM: + tp->ftt_thumb = 0; + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + case FASTTRAP_FN_THUMB: + tp->ftt_thumb = 1; + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + default: + return (-1); + } + } + + if (tp->ftt_thumb) { + tp->ftt_type = dtrace_decode_thumb(instr); + } else { + tp->ftt_type = dtrace_decode_arm(instr); + } + + if (tp->ftt_type == FASTTRAP_T_INV) { + /* This is an instruction we either don't recognize or can't instrument */ + printf("dtrace: fasttrap init32: Unrecognized instruction: %08x at %08llx\n", + (tp->ftt_thumb && dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb) == 2) ? tp->ftt_instr1 : instr, pc); + return (-1); + } + + return (0); +} + + +static int +fasttrap_tracepoint_init64(proc_t *p, fasttrap_tracepoint_t *tp, + user_addr_t pc, fasttrap_probe_type_t type) +{ +#pragma unused(type) + uint32_t instr = 0; + + /* + * Read the instruction at the given address out of the process's + * address space. We don't have to worry about a debugger + * changing this instruction before we overwrite it with our trap + * instruction since P_PR_LOCK is set. Since instructions can span + * pages, we potentially read the instruction in two parts. If the + * second part fails, we just zero out that part of the instruction. + */ + /* + * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey... + */ + + if (uread(p, &instr, 4, pc) != 0) + return (-1); + + tp->ftt_instr = instr; + tp->ftt_thumb = 0; /* Always zero on 64bit */ + + if (tp->ftt_fntype != FASTTRAP_FN_DONE_INIT) { + switch(tp->ftt_fntype) { + case FASTTRAP_FN_UNKNOWN: + /* + * On arm64 there is no distinction between + * arm vs. thumb mode instruction types. + */ + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + case FASTTRAP_FN_USDT: + if (IS_ARM64_NOP(instr) || IS_ARM64_IS_ENABLED(instr)) { + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + } else { + /* + * Shouldn't reach here - this means we don't + * recognize the instruction at one of the + * USDT probe locations + */ + return (-1); + } + + break; + + case FASTTRAP_FN_ARM: + case FASTTRAP_FN_THUMB: + default: + /* + * If we get an arm or thumb mode type + * then we are clearly in the wrong path. + */ + return (-1); + } + } + + tp->ftt_type = dtrace_decode_arm64(instr); + + if (tp->ftt_type == FASTTRAP_T_ARM64_EXCLUSIVE_MEM) { + kprintf("Detected attempt to place DTrace probe on exclusive memory instruction (pc = 0x%llx); refusing to trace (or exclusive operation could never succeed).\n", pc); + tp->ftt_type = FASTTRAP_T_INV; + return (-1); + } + + if (tp->ftt_type == FASTTRAP_T_INV) { + /* This is an instruction we either don't recognize or can't instrument */ + printf("dtrace: fasttrap init64: Unrecognized instruction: %08x at %08llx\n", instr, pc); + return (-1); + } + + return (0); +} + +// These are not exported from vm_map.h. +extern kern_return_t vm_map_write_user(vm_map_t map, void *src_p, vm_map_address_t dst_addr, vm_size_t size); + +/* Patches the instructions. Almost like uwrite, but need special instructions on ARM to flush the caches. */ +static +int patchInst(proc_t *p, void *buf, user_size_t len, user_addr_t a) +{ + kern_return_t ret; + + ASSERT(p != NULL); + ASSERT(p->task != NULL); + + task_t task = p->task; + + /* + * Grab a reference to the task vm_map_t to make sure + * the map isn't pulled out from under us. + * + * Because the proc_lock is not held at all times on all code + * paths leading here, it is possible for the proc to have + * exited. If the map is null, fail. + */ + vm_map_t map = get_task_map_reference(task); + if (map) { + /* Find the memory permissions. */ + uint32_t nestingDepth=999999; + vm_region_submap_short_info_data_64_t info; + mach_msg_type_number_t count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64; + mach_vm_address_t address = (mach_vm_address_t)a; + mach_vm_size_t sizeOfRegion = (mach_vm_size_t)len; + + ret = mach_vm_region_recurse(map, &address, &sizeOfRegion, &nestingDepth, (vm_region_recurse_info_t)&info, &count); + if (ret != KERN_SUCCESS) + goto done; + + vm_prot_t reprotect; + + if (!(info.protection & VM_PROT_WRITE)) { + /* Save the original protection values for restoration later */ + reprotect = info.protection; + if (info.max_protection & VM_PROT_WRITE) { + /* The memory is not currently writable, but can be made writable. */ + /* Making it both writable and executable at the same time causes warning on embedded */ + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, (reprotect & ~VM_PROT_EXECUTE) | VM_PROT_WRITE); + } else { + /* + * The memory is not currently writable, and cannot be made writable. We need to COW this memory. + * + * Strange, we can't just say "reprotect | VM_PROT_COPY", that fails. + */ + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, VM_PROT_COPY | VM_PROT_READ | VM_PROT_WRITE); + } + + if (ret != KERN_SUCCESS) + goto done; + + } else { + /* The memory was already writable. */ + reprotect = VM_PROT_NONE; + } + + ret = vm_map_write_user( map, + buf, + (vm_map_address_t)a, + (vm_size_t)len); + + flush_caches(); + + if (ret != KERN_SUCCESS) + goto done; + + if (reprotect != VM_PROT_NONE) { + ASSERT(reprotect & VM_PROT_EXECUTE); + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, reprotect); + } + +done: + vm_map_deallocate(map); + } else + ret = KERN_TERMINATED; + + return (int)ret; +} + +int +fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp) +{ + /* The thumb patch is a 2 byte instruction regardless of the size of the original instruction */ + uint32_t instr; + int size; + + if (proc_is64bit(p)) { + size = 4; + instr = FASTTRAP_ARM64_INSTR; + } + else { + size = tp->ftt_thumb ? 2 : 4; + if (tp->ftt_thumb) { + *((uint16_t*) &instr) = FASTTRAP_THUMB32_INSTR; + } else { + instr = FASTTRAP_ARM32_INSTR; + } + } + + if (patchInst(p, &instr, size, tp->ftt_pc) != 0) + return (-1); + + tp->ftt_installed = 1; + + return (0); +} + +int +fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp) +{ + /* The thumb patch is a 2 byte instruction regardless of the size of the original instruction */ + uint32_t instr; + int size; + + if (proc_is64bit(p)) { + /* + * Distinguish between read or write failures and a changed + * instruction. + */ + size = 4; + if (uread(p, &instr, size, tp->ftt_pc) != 0) + goto end; + + if (instr != FASTTRAP_ARM64_INSTR) + goto end; + } else { + /* + * Distinguish between read or write failures and a changed + * instruction. + */ + size = tp->ftt_thumb ? 2 : 4; + if (uread(p, &instr, size, tp->ftt_pc) != 0) + goto end; + + if (tp->ftt_thumb) { + if (*((uint16_t*) &instr) != FASTTRAP_THUMB32_INSTR) + goto end; + } else { + if (instr != FASTTRAP_ARM32_INSTR) + goto end; + } + } + + if (patchInst(p, &tp->ftt_instr, size, tp->ftt_pc) != 0) + return (-1); + +end: + tp->ftt_installed = 0; + + return (0); +} + +static void +fasttrap_return_common(proc_t *p, arm_saved_state_t *regs, user_addr_t pc, user_addr_t new_pc) +{ + pid_t pid = p->p_pid; + fasttrap_tracepoint_t *tp; + fasttrap_bucket_t *bucket; + fasttrap_id_t *id; + lck_mtx_t *pid_mtx; + int retire_tp = 1; + pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; + lck_mtx_lock(pid_mtx); + bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; + + for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + if (pid == tp->ftt_pid && pc == tp->ftt_pc && + tp->ftt_proc->ftpc_acount != 0) + break; + } + + /* + * Don't sweat it if we can't find the tracepoint again; unlike + * when we're in fasttrap_pid_probe(), finding the tracepoint here + * is not essential to the correct execution of the process. + */ + if (tp == NULL) { + lck_mtx_unlock(pid_mtx); + return; + } + + for (id = tp->ftt_retids; id != NULL; id = id->fti_next) { + fasttrap_probe_t *probe = id->fti_probe; + /* + * If there's a branch that could act as a return site, we + * need to trace it, and check here if the program counter is + * external to the function. + */ + if (is_saved_state32(regs)) + { + if (tp->ftt_type != FASTTRAP_T_LDM_PC && + tp->ftt_type != FASTTRAP_T_POP_PC && + new_pc - probe->ftp_faddr < probe->ftp_fsize) + continue; + } + else { + /* ARM64_TODO - check for FASTTRAP_T_RET */ + if ((tp->ftt_type != FASTTRAP_T_ARM64_RET) && + new_pc - probe->ftp_faddr < probe->ftp_fsize) + continue; + } + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least one probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } + +#ifndef CONFIG_EMBEDDED + if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { + dtrace_probe(dtrace_probeid_error, 0 /* state */, id->fti_probe->ftp_id, + 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); +#else + if (FALSE) { +#endif + } else { + if (is_saved_state32(regs)) { + dtrace_probe(probe->ftp_id, + pc - id->fti_probe->ftp_faddr, + saved_state32(regs)->r[0], 0, 0, 0); + } else { + dtrace_probe(probe->ftp_id, + pc - id->fti_probe->ftp_faddr, + saved_state64(regs)->x[0], 0, 0, 0); + } + } + } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); + } + + lck_mtx_unlock(pid_mtx); +} + +static void +fasttrap_sigsegv(proc_t *p, uthread_t t, user_addr_t addr, arm_saved_state_t *regs) +{ + /* TODO: This function isn't implemented yet. In debug mode, panic the system to + * find out why we're hitting this point. In other modes, kill the process. + */ +#if DEBUG +#pragma unused(p,t,addr,arm_saved_state) + panic("fasttrap: sigsegv not yet implemented"); +#else +#pragma unused(p,t,addr) + /* Kill the process */ + set_saved_state_pc(regs, 0); +#endif + +#if 0 + proc_lock(p); + + /* Set fault address and mark signal */ + t->uu_code = addr; + t->uu_siglist |= sigmask(SIGSEGV); + + /* + * XXX These two line may be redundant; if not, then we need + * XXX to potentially set the data address in the machine + * XXX specific thread state structure to indicate the address. + */ + t->uu_exception = KERN_INVALID_ADDRESS; /* SIGSEGV */ + t->uu_subcode = 0; /* XXX pad */ + + proc_unlock(p); + + /* raise signal */ + signal_setast(t->uu_context.vc_thread); +#endif +} + +static void +fasttrap_usdt_args32(fasttrap_probe_t *probe, arm_saved_state32_t *regs32, int argc, + uint64_t *argv) +{ + int i, x, cap = MIN(argc, probe->ftp_nargs); + + for (i = 0; i < cap; i++) { + x = probe->ftp_argmap[i]; + + /* Up to 4 args are passed in registers on arm */ + if (x < 4) { + argv[i] = regs32->r[x]; + } else { + uint32_t arg; + fasttrap_fuword32_noerr(regs32->sp + (x - 4) * sizeof(uint32_t), &arg); + + argv[i] = arg; + } + } + + for (; i < argc; i++) { + argv[i] = 0; + } +} + +static void +fasttrap_usdt_args64(fasttrap_probe_t *probe, arm_saved_state64_t *regs64, int argc, + uint64_t *argv) +{ + int i, x, cap = MIN(argc, probe->ftp_nargs); + + for (i = 0; i < cap; i++) { + x = probe->ftp_argmap[i]; + + /* Up to 8 args are passed in registers on arm64 */ + if (x < 8) { + argv[i] = regs64->x[x]; + } else { + fasttrap_fuword64_noerr(regs64->sp + (x - 8) * sizeof(uint64_t), &argv[i]); + } + } + + for (; i < argc; i++) { + argv[i] = 0; + } +} + +static int condition_true(int cond, int cpsr) +{ + int taken = 0; + int zf = (cpsr & PSR_ZF) ? 1 : 0, + nf = (cpsr & PSR_NF) ? 1 : 0, + cf = (cpsr & PSR_CF) ? 1 : 0, + vf = (cpsr & PSR_VF) ? 1 : 0; + + switch(cond) { + case 0: taken = zf; break; + case 1: taken = !zf; break; + case 2: taken = cf; break; + case 3: taken = !cf; break; + case 4: taken = nf; break; + case 5: taken = !nf; break; + case 6: taken = vf; break; + case 7: taken = !vf; break; + case 8: taken = (cf && !zf); break; + case 9: taken = (!cf || zf); break; + case 10: taken = (nf == vf); break; + case 11: taken = (nf != vf); break; + case 12: taken = (!zf && (nf == vf)); break; + case 13: taken = (zf || (nf != vf)); break; + case 14: taken = 1; break; + case 15: taken = 1; break; /* always "true" for ARM, unpredictable for THUMB. */ + } + + return taken; +} + +static void set_thumb_flag(arm_saved_state32_t *regs32, user_addr_t pc) +{ + if (pc & 1) { + regs32->cpsr |= PSR_TF; + } else { + regs32->cpsr &= ~PSR_TF; + } +} + +static int +fasttrap_pid_probe_thumb_state_valid(arm_saved_state32_t *state32, fasttrap_tracepoint_t *tp) +{ + uint32_t cpsr = state32->cpsr; + uint32_t itstate = GETITSTATE(cpsr); + + /* If in IT block, make sure it's the last statement in the block */ + if ((itstate != 0) && !ISLASTINIT(itstate)) { + printf("dtrace: fasttrap: Tried to trace instruction %08x at %08x but not at end of IT block\n", + (tp->ftt_thumb && dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb) == 2) ? tp->ftt_instr1 : tp->ftt_instr, state32->pc); + return 0; + } + + if (!(cpsr & PSR_TF)) { + return 0; + } + + return 1; +} + +static int +fasttrap_get_condition_code(arm_saved_state32_t *regs32, fasttrap_tracepoint_t *tp) +{ + /* Default to always execute */ + int condition_code = 0xE; + if (tp->ftt_thumb) { + uint32_t itstate = GETITSTATE(regs32->cpsr); + if (itstate != 0) { + /* In IT block, make sure it's the last statement in the block */ + assert(ISLASTINIT(itstate)); + condition_code = itstate >> 4; + } + } else { + condition_code = ARM_CONDCODE(tp->ftt_instr); + } + + return condition_code; +} + +static void +fasttrap_pid_probe_handle_patched_instr32(arm_saved_state_t *state, fasttrap_tracepoint_t *tp, uthread_t uthread, + proc_t *p, uint_t is_enabled, int *was_simulated) +{ + arm_saved_state32_t *regs32 = saved_state32(state); + uint32_t new_pc = 0; + uint32_t pc = regs32->pc; + int instr_size; + int condition_code; + + *was_simulated = 1; + + /* + * If there's an is-enabled probe connected to this tracepoint it + * means that there was a 'eor r0,r0,r0' + * instruction that was placed there by DTrace when the binary was + * linked. As this probe is, in fact, enabled, we need to stuff 1 + * into R0. Accordingly, we can bypass all the instruction + * emulation logic since we know the inevitable result. It's possible + * that a user could construct a scenario where the 'is-enabled' + * probe was on some other instruction, but that would be a rather + * exotic way to shoot oneself in the foot. + */ + + if (is_enabled) { + regs32->r[0] = 1; + new_pc = regs32->pc + (tp->ftt_thumb ? 2 : 4); + goto done; + } + + /* For USDT probes, bypass all the emulation logic for the nop instruction */ + if ((tp->ftt_thumb && IS_THUMB32_NOP(THUMB_INSTR(tp->ftt_instr))) || + (!tp->ftt_thumb && IS_ARM32_NOP(tp->ftt_instr))) { + new_pc = regs32->pc + (tp->ftt_thumb ? 2 : 4); + goto done; + } + + condition_code = fasttrap_get_condition_code(regs32, tp); + instr_size = dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb); + + switch (tp->ftt_type) { + case FASTTRAP_T_MOV_PC_REG: + case FASTTRAP_T_CPY_PC: + { + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + int rm; + if (tp->ftt_thumb) { + rm = THUMB16_HRM(tp->ftt_instr1); + } else { + rm = tp->ftt_instr & 0xF; + } + new_pc = regs32->r[rm]; + + /* This instruction does not change the Thumb state */ + + break; + } + + case FASTTRAP_T_STM_LR: + case FASTTRAP_T_PUSH_LR: + { + /* + * This is a very common case, so we want to emulate this instruction if + * possible. However, on a push, it is possible that we might reach the end + * of a page and have to allocate a new page. Most of the time this will not + * happen, and we know that the push instruction can store at most 16 words, + * so check to see if we are far from the boundary, and if so, emulate. This + * can be made more aggressive by checking the actual number of words being + * pushed, but we won't do that for now. + * + * Some of the same issues that apply to POP_PC probably apply here also. + */ + + int reglist; + int ret; + uint32_t base; + + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + base = regs32->sp; + if (((base-16*4) >> PAGE_SHIFT) != (base >> PAGE_SHIFT)) { + /* Crosses the page boundary, go to emulation */ + goto instr_emulate; + } + + if (tp->ftt_thumb) { + if (instr_size == 4) { + /* We know we have to push lr, never push sp or pc */ + reglist = tp->ftt_instr2 & 0x1FFF; + } else { + reglist = tp->ftt_instr1 & 0xFF; + } + } else { + /* We know we have to push lr, never push sp or pc */ + reglist = tp->ftt_instr & 0x1FFF; + } + + /* Push the link register */ + base -= 4; + ret = fasttrap_suword32(base, regs32->lr); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, state); + new_pc = regs32->pc; + break; + } + + /* Start pushing from $r12 */ + int regmask = 1 << 12; + int regnum = 12; + + while (regmask) { + if (reglist & regmask) { + base -= 4; + ret = fasttrap_suword32(base, regs32->r[regnum]); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, state); + new_pc = regs32->pc; + break; + } + } + regmask >>= 1; + regnum--; + } + + regs32->sp = base; + + new_pc = pc + instr_size; + + break; + } + + + case FASTTRAP_T_LDM_PC: + case FASTTRAP_T_POP_PC: + { + /* TODO Two issues that will eventually need to be resolved: + * + * 1. Understand what the hardware does if we have to segfault (data abort) in + * the middle of a load multiple. We currently don't have a working segfault + * handler anyway, and with no swapfile we should never segfault on this load. + * If we do, we'll just kill the process by setting the pc to 0. + * + * 2. The emulation is no longer atomic. We currently only emulate pop for + * function epilogues, and so we should never have a race here because one + * thread should never be trying to manipulate another thread's stack frames. + * That is almost certainly a bug in the program. + * + * This will need to be fixed if we ever: + * a. Ship dtrace externally, as this could be a potential attack vector + * b. Support instruction level tracing, as we might then pop/ldm non epilogues. + * + */ + + /* Assume ldmia! sp/pop ... pc */ + + int regnum = 0, reglist; + int ret; + uint32_t base; + + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (tp->ftt_thumb) { + if (instr_size == 4) { + /* We know we have to load the pc, don't do it twice */ + reglist = tp->ftt_instr2 & 0x7FFF; + } else { + reglist = tp->ftt_instr1 & 0xFF; + } + } else { + /* We know we have to load the pc, don't do it twice */ + reglist = tp->ftt_instr & 0x7FFF; + } + + base = regs32->sp; + while (reglist) { + if (reglist & 1) { + ret = fasttrap_fuword32((user_addr_t)base, ®s32->r[regnum]); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, state); + new_pc = regs32->pc; + break; + } + base += 4; + } + reglist >>= 1; + regnum++; + } + + ret = fasttrap_fuword32((user_addr_t)base, &new_pc); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, state); + new_pc = regs32->pc; + break; + } + base += 4; + + regs32->sp = base; + + set_thumb_flag(regs32, new_pc); + + break; + } + + case FASTTRAP_T_CB_N_Z: + { + /* Thumb mode instruction, and not permitted in IT block, so skip the condition code check */ + int rn = tp->ftt_instr1 & 0x7; + int offset = (((tp->ftt_instr1 & 0x00F8) >> 2) | ((tp->ftt_instr1 & 0x0200) >> 3)) + 4; + int nonzero = tp->ftt_instr1 & 0x0800; + if (!nonzero != !(regs32->r[rn] == 0)) { + new_pc = pc + offset; + } else { + new_pc = pc + instr_size; + } + break; + } + + case FASTTRAP_T_B_COND: + { + /* Use the condition code in the instruction and ignore the ITSTATE */ + + int code, offset; + if (tp->ftt_thumb) { + if (instr_size == 4) { + code = (tp->ftt_instr1 >> 6) & 0xF; + if (code == 14 || code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + int S = (tp->ftt_instr1 >> 10) & 1, + J1 = (tp->ftt_instr2 >> 13) & 1, + J2 = (tp->ftt_instr2 >> 11) & 1; + offset = 4 + SIGNEXTEND( + (S << 20) | (J2 << 19) | (J1 << 18) | + ((tp->ftt_instr1 & 0x003F) << 12) | + ((tp->ftt_instr2 & 0x07FF) << 1), + 21); + } else { + code = (tp->ftt_instr1 >> 8) & 0xF; + if (code == 14 || code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + offset = 4 + (SIGNEXTEND(tp->ftt_instr1 & 0xFF, 8) << 1); + } + } else { + code = ARM_CONDCODE(tp->ftt_instr); + if (code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + offset = 8 + (SIGNEXTEND(tp->ftt_instr & 0x00FFFFFF, 24) << 2); + } + + if (condition_true(code, regs32->cpsr)) { + new_pc = pc + offset; + } else { + new_pc = pc + instr_size; + } + + break; + } + + case FASTTRAP_T_B_UNCOND: + { + int offset; + + /* Unconditional branches can only be taken from Thumb mode */ + /* (This is different from an ARM branch with condition code "always") */ + ASSERT(tp->ftt_thumb == 1); + + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (instr_size == 4) { + int S = (tp->ftt_instr1 >> 10) & 1, + J1 = (tp->ftt_instr2 >> 13) & 1, + J2 = (tp->ftt_instr2 >> 11) & 1; + int I1 = (J1 != S) ? 0 : 1, I2 = (J2 != S) ? 0 : 1; + offset = 4 + SIGNEXTEND( + (S << 24) | (I1 << 23) | (I2 << 22) | + ((tp->ftt_instr1 & 0x03FF) << 12) | + ((tp->ftt_instr2 & 0x07FF) << 1), + 25); + } else { + uint32_t instr1 = tp->ftt_instr1; + offset = 4 + (SIGNEXTEND(instr1 & 0x7FF, 11) << 1); + } + + new_pc = pc + offset; + + break; + } + + case FASTTRAP_T_BX_REG: + { + int reg; + + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (tp->ftt_thumb) { + reg = THUMB16_HRM(tp->ftt_instr1); + } else { + reg = ARM_RM(tp->ftt_instr); + } + new_pc = regs32->r[reg]; + set_thumb_flag(regs32, new_pc); + + break; + } + + case FASTTRAP_T_LDR_PC_IMMED: + case FASTTRAP_T_VLDR_PC_IMMED: + /* Handle these instructions by replacing the PC in the instruction with another + * register. They are common, so we'd like to support them, and this way we do so + * without any risk of having to simulate a segfault. + */ + + /* Fall through */ + + instr_emulate: + case FASTTRAP_T_COMMON: + { + user_addr_t addr; + uint8_t scratch[32]; + uint_t i = 0; + fasttrap_instr_t emul_instr; + emul_instr.instr32 = tp->ftt_instr; + int emul_instr_size; + + /* + * Unfortunately sometimes when we emulate the instruction and have to replace the + * PC, there is no longer a thumb mode equivalent. We end up having to run the + * modified instruction in ARM mode. We use this variable to keep track of which + * mode we should emulate in. We still use the original variable to determine + * what mode to return to. + */ + uint8_t emul_thumb = tp->ftt_thumb; + int save_reg = -1; + uint32_t save_val = 0; + + /* + * Dealing with condition codes and emulation: + * We can't just uniformly do a condition code check here because not all instructions + * have condition codes. We currently do not support an instruction by instruction trace, + * so we can assume that either: 1. We are executing a Thumb instruction, in which case + * we either are not in an IT block and should execute always, or we are last in an IT + * block. Either way, the traced instruction will run correctly, and we won't have any + * problems when we return to the original code, because we will no longer be in the IT + * block. 2. We are executing an ARM instruction, in which case we are ok as long as + * we don't attempt to change the condition code. + */ + if (tp->ftt_type == FASTTRAP_T_LDR_PC_IMMED) { + /* We know we always have a free register (the one we plan to write the + * result value to!). So we'll replace the pc with that one. + */ + int new_reg; + if (tp->ftt_thumb) { + /* Check to see if thumb or thumb2 */ + if (instr_size == 2) { + /* + * Sadness. We need to emulate this instruction in ARM mode + * because it has an 8 bit immediate offset. Instead of having + * to deal with condition codes in the ARM instruction, we'll + * just check the condition and abort if the condition is false. + */ + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + new_reg = (tp->ftt_instr1 >> 8) & 0x7; + regs32->r[new_reg] = ALIGNADDR(regs32->pc + 4, 2); + emul_thumb = 0; + emul_instr.instr32 = 0xE5900000 | (new_reg << 16) | (new_reg << 12) | ((tp->ftt_instr1 & 0xFF) << 2); + } else { + /* Thumb2. Just replace the register. */ + new_reg = (tp->ftt_instr2 >> 12) & 0xF; + regs32->r[new_reg] = ALIGNADDR(regs32->pc + 4, 2); + emul_instr.instr16.instr1 &= ~0x000F; + emul_instr.instr16.instr1 |= new_reg; + } + } else { + /* ARM. Just replace the register. */ + new_reg = (tp->ftt_instr >> 12) & 0xF; + regs32->r[new_reg] = ALIGNADDR(regs32->pc + 8,2); + emul_instr.instr32 &= ~0x000F0000; + emul_instr.instr32 |= new_reg << 16; + } + } else if (tp->ftt_type == FASTTRAP_T_VLDR_PC_IMMED) { + /* This instruction only uses one register, and if we're here, we know + * it must be the pc. So we'll just replace it with R0. + */ + save_reg = 0; + save_val = regs32->r[0]; + regs32->r[save_reg] = ALIGNADDR(regs32->pc + (tp->ftt_thumb ? 4 : 8), 2); + if (tp->ftt_thumb) { + emul_instr.instr16.instr1 &= ~0x000F; + } else { + emul_instr.instr32 &= ~0x000F0000; + } + } + + emul_instr_size = dtrace_instr_size(emul_instr.instr32, emul_thumb); + + /* + * At this point: + * tp->ftt_thumb = thumb mode of original instruction + * emul_thumb = thumb mode for emulation + * emul_instr = instruction we are using to emulate original instruction + * emul_instr_size = size of emulating instruction + */ + + addr = uthread->t_dtrace_scratch->addr; + + if (addr == 0LL) { + fasttrap_sigtrap(p, uthread, pc); // Should be killing target proc + new_pc = pc; + break; + } + + uthread->t_dtrace_scrpc = addr; + if (emul_thumb) { + /* + * No way to do an unconditional branch in Thumb mode, shove the address + * onto the user stack and go to the next location with a pop. This can + * segfault if this push happens to cross a stack page, but that's ok, since + * we are running in userland, and the kernel knows how to handle userland + * stack expansions correctly. + * + * Layout of scratch space for Thumb mode: + * Emulated instruction + * ldr save_reg, [pc, #16] (if necessary, restore any register we clobbered) + * push { r0, r1 } + * ldr r0, [pc, #4] + * str r0, [sp, #4] + * pop { r0, pc } + * Location we should return to in original program + * Saved value of clobbered register (if necessary) + */ + + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + + if (save_reg != -1) { + uint16_t restore_inst = 0x4803; + restore_inst |= (save_reg & 0x7) << 8; + SET16(scratch+i, restore_inst); i += 2; // ldr reg, [pc , #16] + } + + SET16(scratch+i, 0xB403); i += 2; // push { r0, r1 } + SET16(scratch+i, 0x4801); i += 2; // ldr r0, [pc, #4] + SET16(scratch+i, 0x9001); i += 2; // str r0, [sp, #4] + SET16(scratch+i, 0xBD01); i += 2; // pop { r0, pc } + + if (i % 4) { + SET16(scratch+i, 0); i += 2; // padding - saved 32 bit words must be aligned + } + SET32(scratch+i, pc + instr_size + (tp->ftt_thumb ? 1 : 0)); i += 4; // Return address + if (save_reg != -1) { + SET32(scratch+i, save_val); i += 4; // saved value of clobbered register + } + + uthread->t_dtrace_astpc = addr + i; + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + SET16(scratch+i, FASTTRAP_THUMB32_RET_INSTR); i += 2; + } else { + /* + * Layout of scratch space for ARM mode: + * Emulated instruction + * ldr save_reg, [pc, #12] (if necessary, restore any register we clobbered) + * ldr pc, [pc, #4] + * Location we should return to in original program + * Saved value of clobbered register (if necessary) + */ + + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + + if (save_reg != -1) { + uint32_t restore_inst = 0xE59F0004; + restore_inst |= save_reg << 12; + SET32(scratch+i, restore_inst); i += 4; // ldr reg, [pc, #12] + } + SET32(scratch+i, 0xE51FF004); i += 4; // ldr pc, [pc, #4] + + SET32(scratch+i, pc + instr_size + (tp->ftt_thumb ? 1 : 0)); i += 4; // Return address + if (save_reg != -1) { + SET32(scratch+i, save_val); i += 4; // Saved value of clobbered register + } + + uthread->t_dtrace_astpc = addr + i; + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + SET32(scratch+i, FASTTRAP_ARM32_RET_INSTR); i += 4; + } + + if (patchInst(p, scratch, i, uthread->t_dtrace_scratch->write_addr) != KERN_SUCCESS) { + fasttrap_sigtrap(p, uthread, pc); + new_pc = pc; + break; + } + + if (tp->ftt_retids != NULL) { + uthread->t_dtrace_step = 1; + uthread->t_dtrace_ret = 1; + new_pc = uthread->t_dtrace_astpc + (emul_thumb ? 1 : 0); + } else { + new_pc = uthread->t_dtrace_scrpc + (emul_thumb ? 1 : 0); + } + + uthread->t_dtrace_pc = pc; + uthread->t_dtrace_npc = pc + instr_size; + uthread->t_dtrace_on = 1; + *was_simulated = 0; + set_thumb_flag(regs32, new_pc); + break; + } + + default: + panic("fasttrap: mishandled an instruction"); + } +done: + set_saved_state_pc(state, new_pc); + return; +} + +/* + * Copy out an instruction for execution in userland. + * Trap back to kernel to handle return to original flow of execution, because + * direct branches don't have sufficient range (+/- 128MB) and we + * cannot clobber a GPR. Note that we have to specially handle PC-rel loads/stores + * as well, which have range +/- 1MB (convert to an indirect load). Instruction buffer + * layout: + * + * [ Thunked instruction sequence ] + * [ Trap for return to original code and return probe handling ] + * + * This *does* make it impossible for an ldxr/stxr pair to succeed if we trace on or between + * them... may need to get fancy at some point. + */ +static void +fasttrap_pid_probe_thunk_instr64(arm_saved_state_t *state, fasttrap_tracepoint_t *tp, proc_t *p, uthread_t uthread, + const uint32_t *instructions, uint32_t num_instrs, user_addr_t *pc_out) +{ + uint32_t local_scratch[8]; + user_addr_t pc = get_saved_state_pc(state); + user_addr_t user_scratch_area; + + assert(num_instrs < 8); + + bcopy(instructions, local_scratch, num_instrs * sizeof(uint32_t)); + local_scratch[num_instrs] = FASTTRAP_ARM64_RET_INSTR; + + uthread->t_dtrace_astpc = uthread->t_dtrace_scrpc = uthread->t_dtrace_scratch->addr; + user_scratch_area = uthread->t_dtrace_scratch->write_addr; + + if (user_scratch_area == (user_addr_t)0) { + fasttrap_sigtrap(p, uthread, pc); // Should be killing target proc + *pc_out = pc; + return; + } + + if (patchInst(p, local_scratch, (num_instrs + 1) * sizeof(uint32_t), user_scratch_area) != KERN_SUCCESS) { + fasttrap_sigtrap(p, uthread, pc); + *pc_out = pc; + return; + } + + /* We're stepping (come back to kernel to adjust PC for return to regular code). */ + uthread->t_dtrace_step = 1; + + /* We may or may not be about to run a return probe (but we wouldn't thunk ret lr)*/ + uthread->t_dtrace_ret = (tp->ftt_retids != NULL); + assert(tp->ftt_type != FASTTRAP_T_ARM64_RET); + + /* Set address of instruction we've patched */ + uthread->t_dtrace_pc = pc; + + /* Any branch would be emulated, next instruction should be one ahead */ + uthread->t_dtrace_npc = pc + 4; + + /* We are certainly handling a probe */ + uthread->t_dtrace_on = 1; + + /* Let's jump to the scratch area */ + *pc_out = uthread->t_dtrace_scratch->addr; +} + +/* + * Sign-extend bit "sign_bit_index" out to bit 64. + */ +static int64_t +sign_extend(int64_t input, uint32_t sign_bit_index) +{ + assert(sign_bit_index < 63); + if (input & (1ULL << sign_bit_index)) { + /* All 1's & ~[1's from 0 to sign bit] */ + input |= ((~0ULL) & ~((1ULL << (sign_bit_index + 1)) - 1ULL)); + } + + return input; +} + +/* + * Handle xzr vs. sp, fp, lr, etc. Will *not* read the SP. + */ +static uint64_t +get_saved_state64_regno(arm_saved_state64_t *regs64, uint32_t regno, int use_xzr) +{ + /* Set PC to register value */ + switch (regno) { + case 29: + return regs64->fp; + case 30: + return regs64->lr; + case 31: + /* xzr */ + if (use_xzr) { + return 0; + } else { + return regs64->sp; + } + default: + return regs64->x[regno]; + } +} + +static void +set_saved_state64_regno(arm_saved_state64_t *regs64, uint32_t regno, int use_xzr, register_t value) +{ + /* Set PC to register value */ + switch (regno) { + case 29: + regs64->fp = value; + break; + case 30: + regs64->lr = value; + break; + case 31: + if (!use_xzr) { + regs64->sp = value; + } + break; + default: + regs64->x[regno] = value; + break; + } +} + +/* + * Common operation: extract sign-extended PC offset from instruction + * Left-shifts result by two bits. + */ +static uint64_t +extract_address_literal_sign_extended(uint32_t instr, uint32_t base, uint32_t numbits) +{ + uint64_t offset; + + offset = (instr >> base) & ((1 << numbits) - 1); + offset = sign_extend(offset, numbits - 1); + offset = offset << 2; + + return offset; +} + +static void +do_cbz_cnbz(arm_saved_state64_t *regs64, uint32_t regwidth, uint32_t instr, int is_cbz, user_addr_t *pc_out) +{ + uint32_t regno; + uint64_t regval; + uint64_t offset; + + /* Extract register */ + regno = (instr & 0x1f); + assert(regno <= 31); + regval = get_saved_state64_regno(regs64, regno, 1); + + /* Control for size */ + if (regwidth == 32) { + regval &= 0xFFFFFFFFULL; + } + + /* Extract offset */ + offset = extract_address_literal_sign_extended(instr, 5, 19); + + /* Do test */ + if ((is_cbz && regval == 0) || ((!is_cbz) && regval != 0)) { + /* Set PC from label */ + *pc_out = regs64->pc + offset; + } else { + /* Advance PC */ + *pc_out = regs64->pc + 4; + } +} + +static void +do_tbz_tbnz(arm_saved_state64_t *regs64, uint32_t instr, int is_tbz, user_addr_t *pc_out) +{ + uint64_t offset, regval; + uint32_t bit_index, b5, b40, regno, bit_set; + + /* Compute offset */ + offset = extract_address_literal_sign_extended(instr, 5, 14); + + /* Extract bit index */ + b5 = (instr >> 31); + b40 = ((instr >> 19) & 0x1f); + bit_index = (b5 << 5) | b40; + assert(bit_index <= 63); + + /* Extract register */ + regno = (instr & 0x1f); + assert(regno <= 31); + regval = get_saved_state64_regno(regs64, regno, 1); + + /* Test bit */ + bit_set = ((regval & (1 << bit_index)) != 0); + + if ((is_tbz && (!bit_set)) || ((!is_tbz) && bit_set)) { + /* Branch: unsigned addition so overflow defined */ + *pc_out = regs64->pc + offset; + } else { + /* Advance PC */ + *pc_out = regs64->pc + 4; + } +} + + +static void +fasttrap_pid_probe_handle_patched_instr64(arm_saved_state_t *state, fasttrap_tracepoint_t *tp __unused, uthread_t uthread, + proc_t *p, uint_t is_enabled, int *was_simulated) +{ + int res1, res2; + arm_saved_state64_t *regs64 = saved_state64(state); + uint32_t instr = tp->ftt_instr; + user_addr_t new_pc = 0; + + /* Neon state should be threaded throw, but hack it until we have better arm/arm64 integration */ + arm_neon_saved_state64_t *ns64 = &(get_user_neon_regs(uthread->uu_thread)->ns_64); + + /* is-enabled probe: set x0 to 1 and step forwards */ + if (is_enabled) { + regs64->x[0] = 1; + set_saved_state_pc(state, regs64->pc + 4); + return; + } + + /* For USDT probes, bypass all the emulation logic for the nop instruction */ + if (IS_ARM64_NOP(tp->ftt_instr)) { + set_saved_state_pc(state, regs64->pc + 4); + return; + } + + + /* Only one of many cases in the switch doesn't simulate */ + switch(tp->ftt_type) { + /* + * Function entry: emulate for speed. + * stp fp, lr, [sp, #-16]! + */ + case FASTTRAP_T_ARM64_STANDARD_FUNCTION_ENTRY: + { + /* Store values to stack */ + res1 = fasttrap_suword64(regs64->sp - 16, regs64->fp); + res2 = fasttrap_suword64(regs64->sp - 8, regs64->lr); + if (res1 != 0 || res2 != 0) { + fasttrap_sigsegv(p, uthread, regs64->sp - (res1 ? 16 : 8), state); + new_pc = regs64->pc; /* Bit of a hack */ + break; + } + + /* Move stack pointer */ + regs64->sp -= 16; + + /* Move PC forward */ + new_pc = regs64->pc + 4; + *was_simulated = 1; + break; + } + + /* + * PC-relative loads/stores: emulate for correctness. + * All loads are 32bits or greater (no need to handle byte or halfword accesses). + * LDR Wt, addr + * LDR Xt, addr + * LDRSW Xt, addr + * + * LDR St, addr + * LDR Dt, addr + * LDR Qt, addr + * PRFM label -> becomes a NOP + */ + case FASTTRAP_T_ARM64_LDR_S_PC_REL: + case FASTTRAP_T_ARM64_LDR_W_PC_REL: + case FASTTRAP_T_ARM64_LDR_D_PC_REL: + case FASTTRAP_T_ARM64_LDR_X_PC_REL: + case FASTTRAP_T_ARM64_LDR_Q_PC_REL: + case FASTTRAP_T_ARM64_LDRSW_PC_REL: + { + uint64_t offset; + uint32_t valsize, regno; + user_addr_t address; + union { + uint32_t val32; + uint64_t val64; + uint128_t val128; + } value; + + /* Extract 19-bit offset, add to pc */ + offset = extract_address_literal_sign_extended(instr, 5, 19); + address = regs64->pc + offset; + + /* Extract destination register */ + regno = (instr & 0x1f); + assert(regno <= 31); + + /* Read value of desired size from memory */ + switch (tp->ftt_type) { + case FASTTRAP_T_ARM64_LDR_S_PC_REL: + case FASTTRAP_T_ARM64_LDR_W_PC_REL: + case FASTTRAP_T_ARM64_LDRSW_PC_REL: + valsize = 4; + break; + case FASTTRAP_T_ARM64_LDR_D_PC_REL: + case FASTTRAP_T_ARM64_LDR_X_PC_REL: + valsize = 8; + break; + case FASTTRAP_T_ARM64_LDR_Q_PC_REL: + valsize = 16; + break; + default: + panic("Should never get here!"); + valsize = -1; + break; + } + + if (copyin(address, &value, valsize) != 0) { + fasttrap_sigsegv(p, uthread, address, state); + new_pc = regs64->pc; /* Bit of a hack, we know about update in fasttrap_sigsegv() */ + break; + } + + /* Stash in correct register slot */ + switch (tp->ftt_type) { + case FASTTRAP_T_ARM64_LDR_W_PC_REL: + set_saved_state64_regno(regs64, regno, 1, value.val32); + break; + case FASTTRAP_T_ARM64_LDRSW_PC_REL: + set_saved_state64_regno(regs64, regno, 1, sign_extend(value.val32, 31)); + break; + case FASTTRAP_T_ARM64_LDR_X_PC_REL: + set_saved_state64_regno(regs64, regno, 1, value.val64); + break; + case FASTTRAP_T_ARM64_LDR_S_PC_REL: + ns64->v.s[regno][0] = value.val32; + break; + case FASTTRAP_T_ARM64_LDR_D_PC_REL: + ns64->v.d[regno][0] = value.val64; + break; + case FASTTRAP_T_ARM64_LDR_Q_PC_REL: + ns64->v.q[regno] = value.val128; + break; + default: + panic("Should never get here!"); + } + + + /* Move PC forward */ + new_pc = regs64->pc + 4; + *was_simulated = 1; + break; + + } + + case FASTTRAP_T_ARM64_PRFM: + { + /* Becomes a NOP (architecturally permitted). Just move PC forward */ + new_pc = regs64->pc + 4; + *was_simulated = 1; + break; + } + + /* + * End explicit memory accesses. + */ + + /* + * Branches: parse condition codes if needed, emulate for correctness and + * in the case of the indirect branches, convenience + * B.cond + * CBNZ Wn, label + * CBNZ Xn, label + * CBZ Wn, label + * CBZ Xn, label + * TBNZ, Xn|Wn, #uimm16, label + * TBZ, Xn|Wn, #uimm16, label + * + * B label + * BL label + * + * BLR Xm + * BR Xm + * RET Xm + */ + case FASTTRAP_T_ARM64_B_COND: + { + int cond; + + /* Extract condition code */ + cond = (instr & 0xf); + + /* Determine if it passes */ + if (condition_true(cond, regs64->cpsr)) { + uint64_t offset; + + /* Extract 19-bit target offset, add to PC */ + offset = extract_address_literal_sign_extended(instr, 5, 19); + new_pc = regs64->pc + offset; + } else { + /* Move forwards */ + new_pc = regs64->pc + 4; + } + + *was_simulated = 1; + break; + } + + case FASTTRAP_T_ARM64_CBNZ_W: + { + do_cbz_cnbz(regs64, 32, instr, 0, &new_pc); + *was_simulated = 1; + break; + } + case FASTTRAP_T_ARM64_CBNZ_X: + { + do_cbz_cnbz(regs64, 64, instr, 0, &new_pc); + *was_simulated = 1; + break; + } + case FASTTRAP_T_ARM64_CBZ_W: + { + do_cbz_cnbz(regs64, 32, instr, 1, &new_pc); + *was_simulated = 1; + break; + } + case FASTTRAP_T_ARM64_CBZ_X: + { + do_cbz_cnbz(regs64, 64, instr, 1, &new_pc); + *was_simulated = 1; + break; + } + + case FASTTRAP_T_ARM64_TBNZ: + { + do_tbz_tbnz(regs64, instr, 0, &new_pc); + *was_simulated = 1; + break; + } + case FASTTRAP_T_ARM64_TBZ: + { + do_tbz_tbnz(regs64, instr, 1, &new_pc); + *was_simulated = 1; + break; + } + case FASTTRAP_T_ARM64_B: + case FASTTRAP_T_ARM64_BL: + { + uint64_t offset; + + /* Extract offset from instruction */ + offset = extract_address_literal_sign_extended(instr, 0, 26); + + /* Update LR if appropriate */ + if (tp->ftt_type == FASTTRAP_T_ARM64_BL) { + regs64->lr = regs64->pc + 4; + } + + /* Compute PC (unsigned addition for defined overflow) */ + new_pc = regs64->pc + offset; + *was_simulated = 1; + break; + } + + case FASTTRAP_T_ARM64_BLR: + case FASTTRAP_T_ARM64_BR: + { + uint32_t regno; + + /* Extract register from instruction */ + regno = ((instr >> 5) & 0x1f); + assert(regno <= 31); + + /* Update LR if appropriate */ + if (tp->ftt_type == FASTTRAP_T_ARM64_BLR) { + regs64->lr = regs64->pc + 4; + } + + /* Update PC in saved state */ + new_pc = get_saved_state64_regno(regs64, regno, 1); + *was_simulated = 1; + break; + } + + case FASTTRAP_T_ARM64_RET: + { + /* Extract register */ + unsigned regno = ((instr >> 5) & 0x1f); + assert(regno <= 31); + + /* Set PC to register value (xzr, not sp) */ + new_pc = get_saved_state64_regno(regs64, regno, 1); + *was_simulated = 1; + break; + } + + /* + * End branches. + */ + + /* + * Address calculations: emulate for correctness. + * + * ADRP Xd, label + * ADR Xd, label + */ + case FASTTRAP_T_ARM64_ADRP: + case FASTTRAP_T_ARM64_ADR: + { + uint64_t immhi, immlo, offset, result; + uint32_t regno; + + /* Extract destination register */ + regno = (instr & 0x1f); + assert(regno <= 31); + + /* Extract offset */ + immhi = ((instr & 0x00ffffe0) >> 5); /* bits [23,5]: 19 bits */ + immlo = ((instr & 0x60000000) >> 29); /* bits [30,29]: 2 bits */ + + /* Add to PC. Use unsigned addition so that overflow wraps (rather than being undefined). */ + if (tp->ftt_type == FASTTRAP_T_ARM64_ADRP) { + offset = (immhi << 14) | (immlo << 12); /* Concatenate bits into [32,12]*/ + offset = sign_extend(offset, 32); /* Sign extend from bit 32 */ + result = (regs64->pc & ~0xfffULL) + offset; /* And add to page of current pc */ + } else { + assert(tp->ftt_type == FASTTRAP_T_ARM64_ADR); + offset = (immhi << 2) | immlo; /* Concatenate bits into [20,0] */ + offset = sign_extend(offset, 20); /* Sign-extend */ + result = regs64->pc + offset; /* And add to page of current pc */ + } + + /* xzr, not sp */ + set_saved_state64_regno(regs64, regno, 1, result); + + /* Move PC forward */ + new_pc = regs64->pc + 4; + *was_simulated = 1; + break; + } + + /* + * End address calculations. + */ + + /* + * Everything else: thunk to userland + */ + case FASTTRAP_T_COMMON: + { + fasttrap_pid_probe_thunk_instr64(state, tp, p, uthread, &tp->ftt_instr, 1, &new_pc); + *was_simulated = 0; + break; + } + default: + { + panic("An instruction DTrace doesn't expect: %d\n", tp->ftt_type); + break; + } + } + + set_saved_state_pc(state, new_pc); + return; +} + +int +fasttrap_pid_probe(arm_saved_state_t *state) +{ + proc_t *p = current_proc(); + fasttrap_bucket_t *bucket; + lck_mtx_t *pid_mtx; + fasttrap_tracepoint_t *tp, tp_local; + pid_t pid; + dtrace_icookie_t cookie; + uint_t is_enabled = 0; + int was_simulated, retire_tp = 1; + int is_64_bit = is_saved_state64(state); + + uint64_t pc = get_saved_state_pc(state); + + assert(is_64_bit || (pc <= UINT32_MAX)); + + uthread_t uthread = (uthread_t) get_bsdthread_info(current_thread()); + + /* + * It's possible that a user (in a veritable orgy of bad planning) + * could redirect this thread's flow of control before it reached the + * return probe fasttrap. In this case we need to kill the process + * since it's in a unrecoverable state. + */ + if (uthread->t_dtrace_step) { + ASSERT(uthread->t_dtrace_on); + fasttrap_sigtrap(p, uthread, (user_addr_t)pc); + return (0); + } + + /* + * Clear all user tracing flags. + */ + uthread->t_dtrace_ft = 0; + uthread->t_dtrace_pc = 0; + uthread->t_dtrace_npc = 0; + uthread->t_dtrace_scrpc = 0; + uthread->t_dtrace_astpc = 0; + uthread->t_dtrace_reg = 0; + + /* + * Treat a child created by a call to vfork(2) as if it were its + * parent. We know that there's only one thread of control in such a + * process: this one. + */ + if (p->p_lflag & P_LINVFORK) { + proc_list_lock(); + while (p->p_lflag & P_LINVFORK) + p = p->p_pptr; + proc_list_unlock(); + } + + pid = p->p_pid; + pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; + lck_mtx_lock(pid_mtx); + bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid,pc)]; + + /* + * Lookup the tracepoint that the process just hit. + */ + for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + if (pid == tp->ftt_pid && pc == tp->ftt_pc && + tp->ftt_proc->ftpc_acount != 0) + break; + } + + /* + * If we couldn't find a matching tracepoint, either a tracepoint has + * been inserted without using the pid ioctl interface (see + * fasttrap_ioctl), or somehow we have mislaid this tracepoint. + */ + if (tp == NULL) { + lck_mtx_unlock(pid_mtx); + return (-1); + } + + /* Validation of THUMB-related state */ + if (tp->ftt_thumb) { + if (!fasttrap_pid_probe_thumb_state_valid(saved_state32(state), tp)) { + fasttrap_tracepoint_remove(p, tp); + lck_mtx_unlock(pid_mtx); + return (-1); + } + } + + /* Execute the actual probe */ + if (tp->ftt_ids != NULL) { + fasttrap_id_t *id; + uint64_t arg4; + + if (is_saved_state64(state)) { + arg4 = get_saved_state_reg(state, 4); + } else { + uint32_t arg; + user_addr_t stack = (user_addr_t)get_saved_state_sp(state); + + fasttrap_fuword32_noerr(stack, &arg); + arg4 = arg; + } + + + /* First four parameters are passed in registers */ + + for (id = tp->ftt_ids; id != NULL; id = id->fti_next) { + fasttrap_probe_t *probe = id->fti_probe; + +#ifndef CONFIG_EMBEDDED + if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { + dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id, + 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); +#else + if (FALSE) { +#endif + } else { + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least one probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } + if (id->fti_ptype == DTFTP_ENTRY) { + /* + * We note that this was an entry + * probe to help ustack() find the + * first caller. + */ + cookie = dtrace_interrupt_disable(); + DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY); + dtrace_probe(probe->ftp_id, + get_saved_state_reg(state, 0), + get_saved_state_reg(state, 1), + get_saved_state_reg(state, 2), + get_saved_state_reg(state, 3), + arg4); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY); + dtrace_interrupt_enable(cookie); + } else if (id->fti_ptype == DTFTP_IS_ENABLED) { + /* + * Note that in this case, we don't + * call dtrace_probe() since it's only + * an artificial probe meant to change + * the flow of control so that it + * encounters the true probe. + */ + is_enabled = 1; + } else if (probe->ftp_argmap == NULL) { + dtrace_probe(probe->ftp_id, + get_saved_state_reg(state, 0), + get_saved_state_reg(state, 1), + get_saved_state_reg(state, 2), + get_saved_state_reg(state, 3), + arg4); + + } else { + uint64_t t[5]; + + if (is_64_bit) { + fasttrap_usdt_args64(probe, saved_state64(state), 5, t); + } else { + fasttrap_usdt_args32(probe, saved_state32(state), 5, t); + } + dtrace_probe(probe->ftp_id, t[0], t[1], t[2], t[3], t[4]); + } + } + } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); + } + } + /* + * We're about to do a bunch of work so we cache a local copy of + * the tracepoint to emulate the instruction, and then find the + * tracepoint again later if we need to light up any return probes. + */ + tp_local = *tp; + lck_mtx_unlock(pid_mtx); + tp = &tp_local; + + /* + * APPLE NOTE: + * + * Subroutines should update PC. + * We're setting this earlier than Solaris does, to get a "correct" + * ustack() output. In the Sun code, a() -> b() -> c() -> d() is + * reported at: d, b, a. The new way gives c, b, a, which is closer + * to correct, as the return instruction has already exectued. + */ + if (is_64_bit) { + fasttrap_pid_probe_handle_patched_instr64(state, tp, uthread, p, is_enabled, &was_simulated); + } else { + fasttrap_pid_probe_handle_patched_instr32(state, tp, uthread, p, is_enabled, &was_simulated); + } + + /* + * If there were no return probes when we first found the tracepoint, + * we should feel no obligation to honor any return probes that were + * subsequently enabled -- they'll just have to wait until the next + * time around. + */ + if (tp->ftt_retids != NULL) { + /* + * We need to wait until the results of the instruction are + * apparent before invoking any return probes. If this + * instruction was emulated we can just call + * fasttrap_return_common(); if it needs to be executed, we + * need to wait until the user thread returns to the kernel. + */ + /* + * It used to be that only common instructions were simulated. + * For performance reasons, we now simulate some instructions + * when safe and go back to userland otherwise. The was_simulated + * flag means we don't need to go back to userland. + */ + if (was_simulated) { + fasttrap_return_common(p, state, (user_addr_t)pc, (user_addr_t)get_saved_state_pc(state)); + } else { + ASSERT(uthread->t_dtrace_ret != 0); + ASSERT(uthread->t_dtrace_pc == pc); + ASSERT(uthread->t_dtrace_scrpc != 0); + ASSERT(((user_addr_t)get_saved_state_pc(state)) == uthread->t_dtrace_astpc); + } + } + + return (0); +} + +int +fasttrap_return_probe(arm_saved_state_t *regs) +{ + proc_t *p = current_proc(); + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + user_addr_t pc = uthread->t_dtrace_pc; + user_addr_t npc = uthread->t_dtrace_npc; + + uthread->t_dtrace_pc = 0; + uthread->t_dtrace_npc = 0; + uthread->t_dtrace_scrpc = 0; + uthread->t_dtrace_astpc = 0; + + /* + * Treat a child created by a call to vfork(2) as if it were its + * parent. We know that there's only one thread of control in such a + * process: this one. + */ + if (p->p_lflag & P_LINVFORK) { + proc_list_lock(); + while (p->p_lflag & P_LINVFORK) + p = p->p_pptr; + proc_list_unlock(); + } + + /* + * We set rp->r_pc to the address of the traced instruction so + * that it appears to dtrace_probe() that we're on the original + * instruction, and so that the user can't easily detect our + * complex web of lies. dtrace_return_probe() (our caller) + * will correctly set %pc after we return. + */ + set_saved_state_pc(regs, pc); + + fasttrap_return_common(p, regs, pc, npc); + + return (0); +} + +uint64_t +fasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno, + int aframes) +{ +#pragma unused(arg, id, parg, aframes) + arm_saved_state_t* regs = find_user_regs(current_thread()); + + if (is_saved_state32(regs)) { + /* First four arguments are in registers */ + if (argno < 4) + return saved_state32(regs)->r[argno]; + + /* Look on the stack for the rest */ + uint32_t value; + uint32_t* sp = (uint32_t*)(uintptr_t) saved_state32(regs)->sp; + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + value = dtrace_fuword32((user_addr_t) (sp+argno-4)); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); + + return value; + } + else { + /* First eight arguments are in registers */ + if (argno < 8) + return saved_state64(regs)->x[argno]; + + /* Look on the stack for the rest */ + uint64_t value; + uint64_t* sp = (uint64_t*) saved_state64(regs)->sp; + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + value = dtrace_fuword64((user_addr_t) (sp+argno-8)); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); + + return value; + } + +} + +uint64_t +fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ +#pragma unused(arg, id, parg, argno, aframes) +#if 0 + return (fasttrap_anarg(ttolwp(curthread)->lwp_regs, 0, argno)); +#endif + + return 0; +} + diff --git a/bsd/dev/arm64/fbt_arm.c b/bsd/dev/arm64/fbt_arm.c new file mode 100644 index 000000000..c2f348f9a --- /dev/null +++ b/bsd/dev/arm64/fbt_arm.c @@ -0,0 +1,608 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* #pragma ident "@(#)fbt.c 1.15 05/09/19 SMI" */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from + * mach/ppc/thread_status.h */ +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#define DTRACE_INVOP_PUSH_FRAME 11 + +#define DTRACE_INVOP_NOP_SKIP 4 +#define DTRACE_INVOP_ADD_FP_SP_SKIP 4 + +#define DTRACE_INVOP_POP_PC_SKIP 2 + +/* + * stp fp, lr, [sp, #val] + * stp fp, lr, [sp, #val]! + */ +#define FBT_IS_ARM64_FRAME_PUSH(x) \ + (((x) & 0xffc07fff) == 0xa9007bfd || ((x) & 0xffc07fff) == 0xa9807bfd) + +/* + * stp Xt1, Xt2, [sp, #val] + * stp Xt1, Xt2, [sp, #val]! + */ +#define FBT_IS_ARM64_PUSH(x) \ + (((x) & 0xffc003e0) == 0xa90003e0 || ((x) & 0xffc003e0) == 0xa98003e0) + +/* + * ldp fp, lr, [sp, #val] + * ldp fp, lr, [sp], #val + */ +#define FBT_IS_ARM64_FRAME_POP(x) \ + (((x) & 0xffc07fff) == 0xa9407bfd || ((x) & 0xffc07fff) == 0xa8c07bfd) + +#define FBT_IS_ARM64_ADD_FP_SP(x) (((x) & 0xffc003ff) == 0x910003fd) /* add fp, sp, #val (add fp, sp, #0 == mov fp, sp) */ +#define FBT_IS_ARM64_RET(x) ((x) == 0xd65f03c0) /* ret */ + + +#define FBT_B_MASK 0xff000000 +#define FBT_B_IMM_MASK 0x00ffffff +#define FBT_B_INSTR 0x14000000 + +#define FBT_IS_ARM64_B_INSTR(x) ((x & FBT_B_MASK) == FBT_B_INSTR) +#define FBT_GET_ARM64_B_IMM(x) ((x & FBT_B_IMM_MASK) << 2) + +#define FBT_PATCHVAL 0xe7eeee7e +#define FBT_AFRAMES_ENTRY 7 +#define FBT_AFRAMES_RETURN 7 + +#define FBT_ENTRY "entry" +#define FBT_RETURN "return" +#define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask) + +extern dtrace_provider_id_t fbt_id; +extern fbt_probe_t **fbt_probetab; +extern int fbt_probetab_mask; + +kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int); + +int +fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval) +{ + fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; + + for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { + if ((uintptr_t) fbt->fbtp_patchpoint == addr) { + if (0 == CPU->cpu_dtrace_invop_underway) { + CPU->cpu_dtrace_invop_underway = 1; /* Race not possible on + * this per-cpu state */ + + if (fbt->fbtp_roffset == 0) { + /* + * Stack looks like this: + * + * [Higher addresses] + * + * Frame of caller + * Extra args for callee + * ------------------------ + * Frame from traced function: + * ------------------------ + * arm_context_t + * ------------------------ + * Frame from trap handler: + * The traced function never got to mov fp, sp, + * so there is no frame in the backtrace pointing + * to the frame on the stack containing the LR in the + * caller. + * ------------------------ + * | + * | + * | stack grows this way + * | + * | + * v + * [Lower addresses] + */ + + arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss); + + /* + * cpu_dtrace_caller compensates for fact that the traced function never got to update its fp. + * When walking the stack, when we reach the frame where we extract a PC in the patched + * function, we put the cpu_dtrace_caller in the backtrace instead. The next frame we extract + * will be in the caller's caller, so we output a backtrace starting at the caller and going + * sequentially up the stack. + */ + CPU->cpu_dtrace_caller = get_saved_state_lr(regs); + dtrace_probe(fbt->fbtp_id, get_saved_state_reg(regs, 0), get_saved_state_reg(regs, 1), + get_saved_state_reg(regs, 2), get_saved_state_reg(regs, 3),get_saved_state_reg(regs, 4)); + CPU->cpu_dtrace_caller = 0; + } else { + /* + * When fbtp_roffset is non-zero, we know we are handling a return probe point. + * + * + * Stack looks like this, as we've already popped the frame in the traced callee, and + * we trap with lr set to the return address in the caller. + * [Higher addresses] + * + * Frame of caller + * Extra args for callee + * ------------------------ + * arm_context_t + * ------------------------ + * Frame from trap handler: + * ------------------------ + * | + * | + * | stack grows this way + * | + * | + * v + * [Lower addresses] + */ + arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss); + + CPU->cpu_dtrace_caller = get_saved_state_lr(regs); + dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0); + CPU->cpu_dtrace_caller = 0; + } + CPU->cpu_dtrace_invop_underway = 0; + } + + /* + On other architectures, we return a DTRACE constant to let the callback function + know what was replaced. On the ARM, since the function prologue/epilogue machine code + can vary, we need the actual bytes of the instruction, so return the savedval instead. + */ + return (fbt->fbtp_savedval); + } + } + + return (0); +} + +#define IS_USER_TRAP(regs) (PSR64_IS_USER(get_saved_state_cpsr(regs))) +#define T_INVALID_OPCODE EXC_BAD_INSTRUCTION +#define FBT_EXCEPTION_CODE T_INVALID_OPCODE + +kern_return_t +fbt_perfCallback( + int trapno, + struct arm_saved_state * regs, + __unused int unused1, + __unused int unused2) +{ + kern_return_t retval = KERN_FAILURE; + + if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) { + boolean_t oldlevel = 0; + machine_inst_t emul = 0; + uint64_t sp, pc, lr, imm; + + oldlevel = ml_set_interrupts_enabled(FALSE); + + __asm__ volatile( + "Ldtrace_invop_callsite_pre_label:\n" + ".data\n" + ".private_extern _dtrace_invop_callsite_pre\n" + "_dtrace_invop_callsite_pre:\n" + " .quad Ldtrace_invop_callsite_pre_label\n" + ".text\n" + ); + + emul = dtrace_invop(get_saved_state_pc(regs), (uintptr_t*) regs, get_saved_state_reg(regs,0)); + + __asm__ volatile( + "Ldtrace_invop_callsite_post_label:\n" + ".data\n" + ".private_extern _dtrace_invop_callsite_post\n" + "_dtrace_invop_callsite_post:\n" + " .quad Ldtrace_invop_callsite_post_label\n" + ".text\n" + ); + + if (emul == DTRACE_INVOP_NOP) { + /* + * Skip over the patched NOP planted by sdt + */ + pc = get_saved_state_pc(regs); + set_saved_state_pc(regs, pc + DTRACE_INVOP_NOP_SKIP); + retval = KERN_SUCCESS; + } else if (FBT_IS_ARM64_ADD_FP_SP(emul)) { + /* retrieve the value to add */ + uint64_t val = (emul >> 10) & 0xfff; + assert(val < 4096); + + /* retrieve sp */ + sp = get_saved_state_sp(regs); + + /* + * emulate the instruction: + * add fp, sp, #val + */ + assert(sp < (UINT64_MAX - val)); + set_saved_state_fp(regs, sp + val); + + /* skip over the bytes of the patched instruction */ + pc = get_saved_state_pc(regs); + set_saved_state_pc(regs, pc + DTRACE_INVOP_ADD_FP_SP_SKIP); + + retval = KERN_SUCCESS; + } else if (FBT_IS_ARM64_RET(emul)) { + lr = get_saved_state_lr(regs); + set_saved_state_pc(regs, lr); + retval = KERN_SUCCESS; + } else if (FBT_IS_ARM64_B_INSTR(emul)) { + pc = get_saved_state_pc(regs); + imm = FBT_GET_ARM64_B_IMM(emul); + set_saved_state_pc(regs, pc + imm); + retval = KERN_SUCCESS; + } else if (emul == FBT_PATCHVAL) { + /* Means we encountered an error but handled it, try same inst again */ + retval = KERN_SUCCESS; + } else { + retval = KERN_FAILURE; + } + + ml_set_interrupts_enabled(oldlevel); + } + + return retval; +} + +void +fbt_provide_probe(struct modctl *ctl, uintptr_t instrLow, uintptr_t instrHigh, char *modname, char* symbolName, machine_inst_t* symbolStart) +{ + unsigned int j; + int doenable = 0; + dtrace_id_t thisid; + + fbt_probe_t *newfbt, *retfbt, *entryfbt; + machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr; + int foundPushLR, savedRegs; + + /* + * Guard against null symbols + */ + if (!symbolStart || !instrLow || !instrHigh) { + kprintf("dtrace: %s has an invalid address\n", symbolName); + return; + } + + /* + * Assume the compiler doesn't schedule instructions in the prologue. + */ + + foundPushLR = 0; + savedRegs = -1; + limit = (machine_inst_t *)instrHigh; + + assert(sizeof(*instr) == 4); + + for (j = 0, instr = symbolStart, theInstr = 0; + (j < 8) && ((uintptr_t)instr >= instrLow) && (instrHigh > (uintptr_t)(instr)); j++, instr++) + { + /* + * Count the number of time we pushed something onto the stack + * before hitting a frame push. That will give us an estimation + * of how many stack pops we should expect when looking for the + * RET instruction. + */ + theInstr = *instr; + if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) { + foundPushLR = 1; + pushinstr = instr; + } + + if (foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr))) + /* Guard against a random setting of fp from sp, we make sure we found the push first */ + break; + if (FBT_IS_ARM64_RET(theInstr)) /* We've gone too far, bail. */ + break; + if (FBT_IS_ARM64_FRAME_POP(theInstr)) /* We've gone too far, bail. */ + break; + } + + if (!(foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr)))) { + return; + } + + thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY); + newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); + newfbt->fbtp_next = NULL; + strlcpy( (char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS ); + + if (thisid != 0) { + /* + * The dtrace_probe previously existed, so we have to hook + * the newfbt entry onto the end of the existing fbt's + * chain. + * If we find an fbt entry that was previously patched to + * fire, (as indicated by the current patched value), then + * we want to enable this newfbt on the spot. + */ + entryfbt = dtrace_probe_arg (fbt_id, thisid); + ASSERT (entryfbt != NULL); + for(; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) { + if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval) + doenable++; + + if (entryfbt->fbtp_next == NULL) { + entryfbt->fbtp_next = newfbt; + newfbt->fbtp_id = entryfbt->fbtp_id; + break; + } + } + } + else { + /* + * The dtrace_probe did not previously exist, so we + * create it and hook in the newfbt. Since the probe is + * new, we obviously do not need to enable it on the spot. + */ + newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt); + doenable = 0; + } + + newfbt->fbtp_patchpoint = instr; + newfbt->fbtp_ctl = ctl; + newfbt->fbtp_loadcnt = ctl->mod_loadcnt; + newfbt->fbtp_rval = DTRACE_INVOP_PUSH_FRAME; + newfbt->fbtp_savedval = theInstr; + newfbt->fbtp_patchval = FBT_PATCHVAL; + newfbt->fbtp_currentval = 0; + newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; + + if (doenable) + fbt_enable(NULL, newfbt->fbtp_id, newfbt); + + /* + * The fbt entry chain is in place, one entry point per symbol. + * The fbt return chain can have multiple return points per + * symbol. + * Here we find the end of the fbt return chain. + */ + + doenable=0; + + thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN); + + if (thisid != 0) { + /* The dtrace_probe previously existed, so we have to + * find the end of the existing fbt chain. If we find + * an fbt return that was previously patched to fire, + * (as indicated by the currrent patched value), then + * we want to enable any new fbts on the spot. + */ + retfbt = dtrace_probe_arg (fbt_id, thisid); + ASSERT(retfbt != NULL); + for (; retfbt != NULL; retfbt = retfbt->fbtp_next) { + if (retfbt->fbtp_currentval == retfbt->fbtp_patchval) + doenable++; + if(retfbt->fbtp_next == NULL) + break; + } + } + else { + doenable = 0; + retfbt = NULL; + } + + /* + * Go back to the start of the function, in case + * the compiler emitted pcrel data loads + * before FP was adjusted. + */ + instr = pushinstr + 1; +again: + if (instr >= limit) + return; + + /* XXX FIXME ... extra jump table detection? */ + + /* + * OK, it's an instruction. + */ + theInstr = *instr; + + /* Walked onto the start of the next routine? If so, bail out from this function */ + if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) { + if (!retfbt) + kprintf("dtrace: fbt: No return probe for %s, walked to next routine at 0x%016llx\n",symbolName,(uint64_t)instr); + return; + } + + /* XXX fancy detection of end of function using PC-relative loads */ + + /* + * Look for: + * ldp fp, lr, [sp], #val + * ldp fp, lr, [sp, #val] + */ + if (!FBT_IS_ARM64_FRAME_POP(theInstr)) { + instr++; + goto again; + } + + /* go to the next instruction */ + instr++; + + /* Scan ahead for a ret or a branch outside the function */ + for (; instr < limit; instr++) { + theInstr = *instr; + if (FBT_IS_ARM64_RET(theInstr)) + break; + if (FBT_IS_ARM64_B_INSTR(theInstr)) { + machine_inst_t *dest = instr + FBT_GET_ARM64_B_IMM(theInstr); + /* + * Check whether the destination of the branch + * is outside of the function + */ + if (dest >= limit || dest < symbolStart) + break; + } + } + + if (!FBT_IS_ARM64_RET(theInstr) && !FBT_IS_ARM64_B_INSTR(theInstr)) + return; + + newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); + newfbt->fbtp_next = NULL; + strlcpy( (char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS ); + + if (retfbt == NULL) { + newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, + symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt); + } else { + retfbt->fbtp_next = newfbt; + newfbt->fbtp_id = retfbt->fbtp_id; + } + + retfbt = newfbt; + newfbt->fbtp_patchpoint = instr; + newfbt->fbtp_ctl = ctl; + newfbt->fbtp_loadcnt = ctl->mod_loadcnt; + + ASSERT(FBT_IS_ARM64_RET(theInstr)); + newfbt->fbtp_rval = DTRACE_INVOP_RET; + newfbt->fbtp_roffset = (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart); + newfbt->fbtp_savedval = theInstr; + newfbt->fbtp_patchval = FBT_PATCHVAL; + newfbt->fbtp_currentval = 0; + newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; + + if (doenable) + fbt_enable(NULL, newfbt->fbtp_id, newfbt); + + instr++; + goto again; +} + +void +fbt_provide_module_kernel_syms(struct modctl *ctl) +{ + kernel_mach_header_t *mh; + struct load_command *cmd; + kernel_segment_command_t *orig_ts = NULL, *orig_le = NULL; + struct symtab_command *orig_st = NULL; + kernel_nlist_t *sym = NULL; + char *strings; + uintptr_t instrLow, instrHigh; + char *modname; + unsigned int i; + + mh = (kernel_mach_header_t *)(ctl->mod_address); + modname = ctl->mod_modname; + + /* + * Employees of dtrace and their families are ineligible. Void + * where prohibited. + */ + + if (mh->magic != MH_MAGIC_KERNEL) + return; + + cmd = (struct load_command *) & mh[1]; + for (i = 0; i < mh->ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_KERNEL) { + kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd; + + if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT)) + orig_ts = orig_sg; + else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT)) + orig_le = orig_sg; + else if (LIT_STRNEQL(orig_sg->segname, "")) + orig_ts = orig_sg; /* kexts have a single + * unnamed segment */ + } else if (cmd->cmd == LC_SYMTAB) + orig_st = (struct symtab_command *) cmd; + + cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize); + } + + if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL)) + return; + + sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff); + strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff); + + /* Find extent of the TEXT section */ + instrLow = (uintptr_t) orig_ts->vmaddr; + instrHigh = (uintptr_t) (orig_ts->vmaddr + orig_ts->vmsize); + + for (i = 0; i < orig_st->nsyms; i++) { + uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT); + char *name = strings + sym[i].n_un.n_strx; + + /* Check that the symbol is a global and that it has a name. */ + if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type)) + continue; + + if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */ + continue; + + /* Lop off omnipresent leading underscore. */ + if (*name == '_') + name += 1; + + /* + * We're only blacklisting functions in the kernel for now. + */ + if (MOD_IS_MACH_KERNEL(ctl) && fbt_excluded(name)) + continue; + + fbt_provide_probe(ctl, instrLow, instrHigh, modname, name, (machine_inst_t*)sym[i].n_value); + } +} diff --git a/bsd/dev/arm64/sdt_arm.c b/bsd/dev/arm64/sdt_arm.c new file mode 100644 index 000000000..17bb69327 --- /dev/null +++ b/bsd/dev/arm64/sdt_arm.c @@ -0,0 +1,162 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* #pragma ident "@(#)sdt.c 1.6 06/03/24 SMI" */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include + +extern sdt_probe_t **sdt_probetab; + +int +sdt_invop(__unused uintptr_t addr, __unused uintptr_t *stack, __unused uintptr_t eax) +{ +#pragma unused(eax) + sdt_probe_t *sdt = sdt_probetab[SDT_ADDR2NDX(addr)]; + + for (; sdt != NULL; sdt = sdt->sdp_hashnext) { + if ((uintptr_t) sdt->sdp_patchpoint == addr) { + struct arm_saved_state* regs = (struct arm_saved_state*) stack; + + dtrace_probe(sdt->sdp_id, get_saved_state_reg(regs, 0), get_saved_state_reg(regs, 1), + get_saved_state_reg(regs, 2), get_saved_state_reg(regs, 3),get_saved_state_reg(regs, 4)); + + return (DTRACE_INVOP_NOP); + } + } + + return (0); +} + +struct frame { + struct frame *backchain; + uintptr_t retaddr; +}; + +/*ARGSUSED*/ +uint64_t +sdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ + +#pragma unused(arg,id,parg) /* __APPLE__ */ + + uint64_t val = 0; + struct frame *fp = (struct frame *)__builtin_frame_address(0); + uintptr_t *stack; + uintptr_t pc; + int i; + + /* + * A total of eight arguments are passed via registers; any argument + * with an index of 7 or lower is therefore in a register. + */ + + int inreg = 7; + + for (i = 1; i <= aframes; i++) { + fp = fp->backchain; + pc = fp->retaddr; + + if (dtrace_invop_callsite_pre != NULL + && pc > (uintptr_t)dtrace_invop_callsite_pre + && pc <= (uintptr_t)dtrace_invop_callsite_post) { + + /* + * When we pass through the invalid op handler, + * we expect to find the save area structure, + * pushed on the stack where we took the trap. + * If the argument we seek is passed in a register, then + * we can load it directly from this saved area. + * If the argument we seek is passed on the stack, then + * we increment the frame pointer further, to find the + * pushed args + */ + + /* fp points to the dtrace_invop activation */ + fp = fp->backchain; /* fbt_perfCallback */ + fp = fp->backchain; /* sleh_synchronous */ + fp = fp->backchain; /* fleh_synchronous */ + + arm_saved_state_t *tagged_regs = (arm_saved_state_t *)((uintptr_t *)&fp[1]); + arm_saved_state64_t *saved_state = saved_state64(tagged_regs); + + if (argno <= inreg) { + /* The argument will be in a register */ + stack = (uintptr_t *)&saved_state->x[0]; + } else { + /* The argument will be found on the stack */ + fp = (struct frame *)(saved_state->sp); + stack = (uintptr_t *)&fp[0]; /* Find marshalled arguments */ + argno -= (inreg + 1); + } + goto load; + } + } + + /* + * We know that we did not come through a trap to get into + * dtrace_probe() -- We arrive here when the provider has + * called dtrace_probe() directly. + * The probe ID is the first argument to dtrace_probe(). + * We must advance beyond that to get the argX. + */ + argno++; /* Advance past probeID */ + + if (argno <= inreg) { + /* + * This shouldn't happen. If the argument is passed in a + * register then it should have been, well, passed in a + * register... + */ + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + argno -= (inreg + 1); + stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ + +load: + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */ + val = (uint64_t)(*(((uintptr_t *)stack) + argno)); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + return (val); +} diff --git a/bsd/dev/arm64/sysctl.c b/bsd/dev/arm64/sysctl.c new file mode 100644 index 000000000..22dcc12d7 --- /dev/null +++ b/bsd/dev/arm64/sysctl.c @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2003-2007 Apple Inc. All rights reserved. + */ +#include +#include +#include + +#include + +extern uint64_t wake_abstime; + +static +SYSCTL_QUAD(_machdep, OID_AUTO, wake_abstime, + CTLFLAG_RD, &wake_abstime, + "Absolute Time at the last wakeup"); + +static int +sysctl_time_since_reset SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2, oidp) + int error = 0; + uint64_t return_value = 0; + + return_value = ml_get_time_since_reset(); + + SYSCTL_OUT(req, &return_value, sizeof(return_value)); + + return error; +} + +SYSCTL_PROC(_machdep, OID_AUTO, time_since_reset, + CTLFLAG_RD | CTLTYPE_QUAD | CTLFLAG_LOCKED, + 0, 0, sysctl_time_since_reset, "I", + "Continuous time since last SOC boot/wake started"); + +static int +sysctl_wake_conttime SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2, oidp) + int error = 0; + uint64_t return_value = 0; + + return_value = ml_get_conttime_wake_time(); + + SYSCTL_OUT(req, &return_value, sizeof(return_value)); + + return error; +} + +SYSCTL_PROC(_machdep, OID_AUTO, wake_conttime, + CTLFLAG_RD | CTLTYPE_QUAD | CTLFLAG_LOCKED, + 0, 0, sysctl_wake_conttime, "I", + "Continuous Time at the last wakeup"); + + diff --git a/bsd/dev/dtrace/dtrace.c b/bsd/dev/dtrace/dtrace.c index f1f32ea70..75fc4d38d 100644 --- a/bsd/dev/dtrace/dtrace.c +++ b/bsd/dev/dtrace/dtrace.c @@ -100,6 +100,11 @@ #include #include +#if MONOTONIC +#include +#include +#endif /* MONOTONIC */ + #include extern uint32_t pmap_find_phys(void *, uint64_t); extern boolean_t pmap_valid_page(uint32_t); @@ -145,7 +150,7 @@ uint64_t dtrace_buffer_memory_inuse = 0; int dtrace_destructive_disallow = 0; dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024); size_t dtrace_difo_maxsize = (256 * 1024); -dtrace_optval_t dtrace_dof_maxsize = (384 * 1024); +dtrace_optval_t dtrace_dof_maxsize = (512 * 1024); dtrace_optval_t dtrace_statvar_maxsize = (16 * 1024); dtrace_optval_t dtrace_statvar_maxsize_max = (16 * 10 * 1024); size_t dtrace_actions_max = (16 * 1024); @@ -289,7 +294,7 @@ static int dtrace_module_unloaded(struct kmod_info *kmod); * * ASSERT(MUTEX_HELD(&cpu_lock)); * becomes: - * lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + * LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); * */ static lck_mtx_t dtrace_lock; /* probe state lock */ @@ -352,6 +357,15 @@ int dtrace_helptrace_enabled = 1; int dtrace_helptrace_enabled = 0; #endif +#if defined (__arm64__) +/* + * The ioctl for adding helper DOF is based on the + * size of a user_addr_t. We need to recognize both + * U32 and U64 as the same action. + */ +#define DTRACEHIOC_ADDDOF_U32 _IOW('h', 4, user32_addr_t) +#define DTRACEHIOC_ADDDOF_U64 _IOW('h', 4, user64_addr_t) +#endif /* __arm64__ */ /* * DTrace Error Hashing @@ -418,6 +432,25 @@ static lck_mtx_t dtrace_errlock; (where) = ((thr + DIF_VARIABLE_MAX) & \ (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ } +#elif defined(__arm__) +/* FIXME: three function calls!!! */ +#define DTRACE_TLS_THRKEY(where) { \ + uint_t intr = ml_at_interrupt_context(); /* Note: just one measly bit */ \ + uint64_t thr = (uintptr_t)current_thread(); \ + uint_t pid = (uint_t)dtrace_proc_selfpid(); \ + ASSERT(intr < (1 << 3)); \ + (where) = (((thr << 32 | pid) + DIF_VARIABLE_MAX) & \ + (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ +} +#elif defined (__arm64__) +/* FIXME: two function calls!! */ +#define DTRACE_TLS_THRKEY(where) { \ + uint_t intr = ml_at_interrupt_context(); /* Note: just one measly bit */ \ + uint64_t thr = (uintptr_t)current_thread(); \ + ASSERT(intr < (1 << 3)); \ + (where) = ((thr + DIF_VARIABLE_MAX) & \ + (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ +} #else #error Unknown architecture #endif @@ -786,13 +819,21 @@ sysctl_dtrace_provide_private_probes SYSCTL_HANDLER_ARGS if (error) return (error); - if (value != 0 && value != 1) - return (ERANGE); + if (req->newptr) { + if (value != 0 && value != 1) + return (ERANGE); - lck_mtx_lock(&dtrace_lock); - dtrace_provide_private_probes = value; - lck_mtx_unlock(&dtrace_lock); + /* + * We do not allow changing this back to zero, as private probes + * would still be left registered + */ + if (value != 1) + return (EPERM); + lck_mtx_lock(&dtrace_lock); + dtrace_provide_private_probes = value; + lck_mtx_unlock(&dtrace_lock); + } return (0); } @@ -1052,7 +1093,7 @@ dtrace_canstore_remains(uint64_t addr, size_t sz, size_t *remain, * DTrace subroutines (DIF_SUBR_*) should use this helper to implement * appropriate memory access protection. */ -static int +int dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) { @@ -3180,6 +3221,7 @@ dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v, * APPLE NOTE: Account for introduction of __dtrace_probe() */ int aframes = mstate->dtms_probe->dtpr_aframes + 3; + dtrace_vstate_t *vstate = &state->dts_vstate; dtrace_provider_t *pv; uint64_t val; @@ -3194,7 +3236,7 @@ dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v, } else - val = dtrace_getarg(ndx, aframes); + val = dtrace_getarg(ndx, aframes, mstate, vstate); /* * This is regrettably required to keep the compiler @@ -3467,27 +3509,47 @@ dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v, case DIF_VAR_ZONENAME: - { - /* scratch_size is equal to length('global') + 1 for the null-terminator. */ - char *zname = (char *)mstate->dtms_scratch_ptr; - size_t scratch_size = 6 + 1; + { + /* scratch_size is equal to length('global') + 1 for the null-terminator. */ + char *zname = (char *)mstate->dtms_scratch_ptr; + size_t scratch_size = 6 + 1; if (!dtrace_priv_proc(state)) return (0); - /* The scratch allocation's lifetime is that of the clause. */ - if (!DTRACE_INSCRATCH(mstate, scratch_size)) { - DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); - return 0; - } + /* The scratch allocation's lifetime is that of the clause. */ + if (!DTRACE_INSCRATCH(mstate, scratch_size)) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); + return 0; + } + + mstate->dtms_scratch_ptr += scratch_size; - mstate->dtms_scratch_ptr += scratch_size; + /* The kernel does not provide zonename, it will always return 'global'. */ + strlcpy(zname, "global", scratch_size); - /* The kernel does not provide zonename, it will always return 'global'. */ - strlcpy(zname, "global", scratch_size); + return ((uint64_t)(uintptr_t)zname); + } - return ((uint64_t)(uintptr_t)zname); - } +#if MONOTONIC + case DIF_VAR_CPUINSTRS: + return mt_cur_cpu_instrs(); + + case DIF_VAR_CPUCYCLES: + return mt_cur_cpu_cycles(); + + case DIF_VAR_VINSTRS: + return mt_cur_thread_instrs(); + + case DIF_VAR_VCYCLES: + return mt_cur_thread_cycles(); +#else /* MONOTONIC */ + case DIF_VAR_CPUINSTRS: /* FALLTHROUGH */ + case DIF_VAR_CPUCYCLES: /* FALLTHROUGH */ + case DIF_VAR_VINSTRS: /* FALLTHROUGH */ + case DIF_VAR_VCYCLES: /* FALLTHROUGH */ + return 0; +#endif /* !MONOTONIC */ case DIF_VAR_UID: if (!dtrace_priv_proc_relaxed(state)) @@ -3896,7 +3958,7 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, char c, target = (char)tupregs[1].dttk_value; if (!dtrace_strcanload(addr, size, &lim, mstate, vstate)) { - regs[rd] = NULL; + regs[rd] = 0; break; } addr_limit = addr + lim; @@ -4156,7 +4218,7 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, */ regs[rd] = 0; mstate->dtms_strtok = 0; - mstate->dtms_strtok_limit = NULL; + mstate->dtms_strtok_limit = 0; break; } @@ -4305,9 +4367,20 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, case DIF_SUBR_LLTOSTR: { int64_t i = (int64_t)tupregs[0].dttk_value; - int64_t val = i < 0 ? i * -1 : i; - uint64_t size = 22; /* enough room for 2^64 in decimal */ + uint64_t val, digit; + uint64_t size = 65; /* enough room for 2^64 in binary */ char *end = (char *)mstate->dtms_scratch_ptr + size - 1; + int base = 10; + + if (nargs > 1) { + if ((base = tupregs[1].dttk_value) <= 1 || + base > ('z' - 'a' + 1) + ('9' - '0' + 1)) { + *flags |= CPU_DTRACE_ILLOP; + break; + } + } + + val = (base == 10 && i < 0) ? i * -1 : i; if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); @@ -4315,13 +4388,24 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, break; } - for (*end-- = '\0'; val; val /= 10) - *end-- = '0' + (val % 10); + for (*end-- = '\0'; val; val /= base) { + if ((digit = val % base) <= '9' - '0') { + *end-- = '0' + digit; + } else { + *end-- = 'a' + (digit - ('9' - '0') - 1); + } + } - if (i == 0) + if (i == 0 && base == 16) *end-- = '0'; - if (i < 0) + if (base == 16) + *end-- = 'x'; + + if (i == 0 || base == 8 || base == 16) + *end-- = '0'; + + if (i < 0 && base == 10) *end-- = '-'; regs[rd] = (uintptr_t)end + 1; @@ -6458,7 +6542,7 @@ __dtrace_probe(dtrace_id_t id, uint64_t arg0, uint64_t arg1, if (pred != NULL) { dtrace_difo_t *dp = pred->dtp_difo; - int rval; + uint64_t rval; rval = dtrace_dif_emulate(dp, &mstate, vstate, state); @@ -7179,11 +7263,13 @@ dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp) if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) { if (dtrace_is_restricted() && !dtrace_are_restrictions_relaxed()) { - priv = DTRACE_PRIV_USER | DTRACE_PRIV_PROC; + priv = DTRACE_PRIV_USER | DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER; } else { priv = DTRACE_PRIV_ALL; } + *uidp = 0; + *zoneidp = 0; } else { *uidp = crgetuid(cr); *zoneidp = crgetzoneid(cr); @@ -7471,7 +7557,7 @@ dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid, int len, rc, best = INT_MAX, nmatched = 0; dtrace_id_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); /* * If the probe ID is specified in the key, just lookup by ID and @@ -7730,8 +7816,8 @@ dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv, *idp = (dtrace_provider_id_t)provider; if (pops == &dtrace_provider_ops) { - lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dtrace_anon.dta_enabling == NULL); /* @@ -7801,8 +7887,8 @@ dtrace_unregister(dtrace_provider_id_t id) */ ASSERT(old == dtrace_provider); ASSERT(dtrace_devi != NULL); - lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); self = 1; if (dtrace_provider->dtpv_next != NULL) { @@ -8035,7 +8121,7 @@ dtrace_probe_create(dtrace_provider_id_t prov, const char *mod, dtrace_id_t id; if (provider == dtrace_provider) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); } else { lck_mtx_lock(&dtrace_lock); } @@ -8108,7 +8194,7 @@ dtrace_probe_create(dtrace_provider_id_t prov, const char *mod, static dtrace_probe_t * dtrace_probe_lookup_id(dtrace_id_t id) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (id == 0 || id > (dtrace_id_t)dtrace_nprobes) return (NULL); @@ -8215,7 +8301,7 @@ dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv) struct modctl *ctl; int all = 0; - lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); if (prv == NULL) { all = 1; @@ -8295,7 +8381,7 @@ dtrace_probe_enable(const dtrace_probedesc_t *desc, dtrace_enabling_t *enab, dtr uid_t uid; zoneid_t zoneid; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); dtrace_ecb_create_cache = NULL; @@ -8451,7 +8537,7 @@ dtrace_helper_provide(dof_helper_t *dhp, proc_t *p) dof_hdr_t *dof = (dof_hdr_t *)daddr; uint32_t i; - lck_mtx_assert(&dtrace_meta_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_meta_lock, LCK_MTX_ASSERT_OWNED); for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + @@ -8499,7 +8585,7 @@ dtrace_helper_provider_remove(dof_helper_t *dhp, proc_t *p) dof_hdr_t *dof = (dof_hdr_t *)daddr; uint32_t i; - lck_mtx_assert(&dtrace_meta_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_meta_lock, LCK_MTX_ASSERT_OWNED); for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + @@ -9318,7 +9404,7 @@ dtrace_difo_hold(dtrace_difo_t *dp) { uint_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); dp->dtdo_refcnt++; ASSERT(dp->dtdo_refcnt != 0); @@ -9502,7 +9588,7 @@ dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate) int oldsvars, osz, nsz, otlocals, ntlocals; uint_t i, id; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0); for (i = 0; i < dp->dtdo_varlen; i++) { @@ -9724,7 +9810,7 @@ dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate) { uint_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dp->dtdo_refcnt != 0); for (i = 0; i < dp->dtdo_varlen; i++) { @@ -9837,7 +9923,7 @@ dtrace_predicate_create(dtrace_difo_t *dp) { dtrace_predicate_t *pred; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dp->dtdo_refcnt != 0); pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP); @@ -9867,7 +9953,7 @@ dtrace_predicate_create(dtrace_difo_t *dp) static void dtrace_predicate_hold(dtrace_predicate_t *pred) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0); ASSERT(pred->dtp_refcnt > 0); @@ -9880,7 +9966,7 @@ dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate) dtrace_difo_t *dp = pred->dtp_difo; #pragma unused(dp) /* __APPLE__ */ - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dp != NULL && dp->dtdo_refcnt != 0); ASSERT(pred->dtp_refcnt > 0); @@ -9955,7 +10041,7 @@ dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe) dtrace_ecb_t *ecb; dtrace_epid_t epid; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP); ecb->dte_predicate = NULL; @@ -10021,8 +10107,8 @@ dtrace_ecb_enable(dtrace_ecb_t *ecb) { dtrace_probe_t *probe = ecb->dte_probe; - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(ecb->dte_next == NULL); if (probe == NULL) { @@ -10367,7 +10453,7 @@ dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc) dtrace_optval_t nframes=0, strsize; uint64_t arg = desc->dtad_arg; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1); if (DTRACEACT_ISAGG(desc->dtad_kind)) { @@ -10646,7 +10732,7 @@ dtrace_ecb_disable(dtrace_ecb_t *ecb) dtrace_ecb_t *pecb, *prev = NULL; dtrace_probe_t *probe = ecb->dte_probe; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (probe == NULL) { /* @@ -10725,7 +10811,7 @@ dtrace_ecb_destroy(dtrace_ecb_t *ecb) dtrace_predicate_t *pred; dtrace_epid_t epid = ecb->dte_epid; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(ecb->dte_next == NULL); ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb); @@ -10750,7 +10836,7 @@ dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe, dtrace_provider_t *prov; dtrace_ecbdesc_t *desc = enab->dten_current; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(state != NULL); ecb = dtrace_ecb_add(state, probe); @@ -10862,7 +10948,7 @@ dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id) dtrace_ecb_t *ecb; #pragma unused(ecb) /* __APPLE__ */ - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (id == 0 || id > (dtrace_epid_t)state->dts_necbs) return (NULL); @@ -10879,7 +10965,7 @@ dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id) dtrace_aggregation_t *agg; #pragma unused(agg) /* __APPLE__ */ - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (id == 0 || id > (dtrace_aggid_t)state->dts_naggregations) return (NULL); @@ -10982,8 +11068,8 @@ dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t limit, size_t size, int flags, dtrace_buffer_t *buf; size_t size_before_alloc = dtrace_buffer_memory_inuse; - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (size > (size_t)dtrace_nonroot_maxsize && !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE)) @@ -11009,7 +11095,6 @@ dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t limit, size_t size, int flags, ASSERT(buf->dtb_xamot == NULL); - /* DTrace, please do not eat all the memory. */ if (dtrace_buffer_canalloc(size) == B_FALSE) goto err; @@ -11346,7 +11431,7 @@ static void dtrace_buffer_polish(dtrace_buffer_t *buf) { ASSERT(buf->dtb_flags & DTRACEBUF_RING); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (!(buf->dtb_flags & DTRACEBUF_WRAPPED)) return; @@ -11523,7 +11608,7 @@ dtrace_enabling_destroy(dtrace_enabling_t *enab) dtrace_ecbdesc_t *ep; dtrace_vstate_t *vstate = enab->dten_vstate; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); for (i = 0; i < enab->dten_ndesc; i++) { dtrace_actdesc_t *act, *next; @@ -11583,7 +11668,7 @@ dtrace_enabling_retain(dtrace_enabling_t *enab) { dtrace_state_t *state; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL); ASSERT(enab->dten_vstate != NULL); @@ -11618,7 +11703,7 @@ dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match, dtrace_enabling_t *new, *enab; int found = 0, err = ENOENT; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN); ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN); ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN); @@ -11685,7 +11770,7 @@ dtrace_enabling_retract(dtrace_state_t *state) { dtrace_enabling_t *enab, *next; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); /* * Iterate over all retained enablings, destroy the enablings retained @@ -11715,8 +11800,8 @@ dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched, dtrace_match_cond_ int i = 0; int total_matched = 0, matched = 0; - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); for (i = 0; i < enab->dten_ndesc; i++) { dtrace_ecbdesc_t *ep = enab->dten_desc[i]; @@ -11867,8 +11952,8 @@ dtrace_enabling_provide(dtrace_provider_t *prv) dtrace_probedesc_t desc; dtrace_genid_t gen; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); if (prv == NULL) { all = 1; @@ -11935,7 +12020,7 @@ dtrace_dof_create(dtrace_state_t *state) roundup(sizeof (dof_sec_t), sizeof (uint64_t)) + sizeof (dof_optdesc_t) * DTRACEOPT_MAX; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); dof = dt_kmem_zalloc_aligned(len, 8, KM_SLEEP); dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0; @@ -11988,7 +12073,7 @@ dtrace_dof_copyin(user_addr_t uarg, int *errp) { dof_hdr_t hdr, *dof; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); /* * First, we're going to copyin() the sizeof (dof_hdr_t). @@ -12032,7 +12117,7 @@ dtrace_dof_copyin_from_proc(proc_t* p, user_addr_t uarg, int *errp) { dof_hdr_t hdr, *dof; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); /* * First, we're going to copyin() the sizeof (dof_hdr_t). @@ -12638,7 +12723,7 @@ dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr, dtrace_enabling_t *enab; uint_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t)); /* @@ -12880,7 +12965,7 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size) dtrace_dynvar_t *dvar, *next, *start; size_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL); bzero(dstate, sizeof (dtrace_dstate_t)); @@ -12979,7 +13064,7 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size) static void dtrace_dstate_fini(dtrace_dstate_t *dstate) { - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); if (dstate->dtds_base == NULL) return; @@ -13061,8 +13146,8 @@ dtrace_state_create(dev_t *devp, cred_t *cr, dtrace_state_t **new_state) dtrace_optval_t *opt; int bufsize = (int)NCPU * sizeof (dtrace_buffer_t), i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); /* Cause restart */ *new_state = NULL; @@ -13291,8 +13376,8 @@ dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which) size_t limit = buf->dtb_size; int flags = 0, rval; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); ASSERT(which < DTRACEOPT_MAX); ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE || (state == dtrace_anon.dta_state && @@ -13690,7 +13775,7 @@ dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu) { dtrace_icookie_t cookie; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE && state->dts_activity != DTRACE_ACTIVITY_DRAINING) @@ -13741,7 +13826,7 @@ static int dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option, dtrace_optval_t val) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) return (EBUSY); @@ -13808,8 +13893,8 @@ dtrace_state_destroy(dtrace_state_t *state) int nspec = state->dts_nspeculations; uint32_t match; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); /* * First, retract any retained enablings for this state. @@ -13920,7 +14005,7 @@ dtrace_anon_grab(void) { dtrace_state_t *state; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if ((state = dtrace_anon.dta_state) == NULL) { ASSERT(dtrace_anon.dta_enabling == NULL); @@ -13945,8 +14030,8 @@ dtrace_anon_property(void) dof_hdr_t *dof; char c[32]; /* enough for "dof-data-" + digits */ - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); for (i = 0; ; i++) { (void) snprintf(c, sizeof (c), "dof-data-%d", i); @@ -14217,7 +14302,7 @@ dtrace_helper_destroygen(proc_t* p, int gen) dtrace_vstate_t *vstate; uint_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (help == NULL || gen > help->dthps_generation) return (EINVAL); @@ -14394,7 +14479,7 @@ static void dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help, dof_helper_t *dofhp) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(&dtrace_meta_lock); lck_mtx_lock(&dtrace_lock); @@ -14456,7 +14541,7 @@ dtrace_helper_provider_add(proc_t* p, dof_helper_t *dofhp, int gen) dtrace_helper_provider_t *hprov, **tmp_provs; uint_t tmp_maxprovs, i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); help = p->p_dtrace_helpers; ASSERT(help != NULL); @@ -14752,7 +14837,7 @@ dtrace_helper_slurp(proc_t* p, dof_hdr_t *dof, dof_helper_t *dhp) int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1; uintptr_t daddr = (uintptr_t)dof; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if ((help = p->p_dtrace_helpers) == NULL) help = dtrace_helpers_create(p); @@ -15199,9 +15284,9 @@ dtrace_lazy_dofs_proc_iterate_doit(proc_t *p, void* ignored) static int dtrace_lazy_dofs_duplicate(proc_t *parent, proc_t *child) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); - lck_mtx_assert(&parent->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); - lck_mtx_assert(&child->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&parent->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&child->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); lck_rw_lock_shared(&dtrace_dof_mode_lock); lck_mtx_lock(&parent->p_dtrace_sprlock); @@ -15254,7 +15339,7 @@ dtrace_helpers_create(proc_t *p) { dtrace_helpers_t *help; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(p->p_dtrace_helpers == NULL); help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP); @@ -15600,7 +15685,7 @@ dtrace_modctl_add(struct modctl * newctl) struct modctl *nextp, *prevp; ASSERT(newctl != NULL); - lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); // Insert new module at the front of the list, @@ -15646,7 +15731,7 @@ dtrace_modctl_add(struct modctl * newctl) static modctl_t * dtrace_modctl_lookup(struct kmod_info * kmod) { - lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); struct modctl * ctl; @@ -15666,7 +15751,7 @@ static void dtrace_modctl_remove(struct modctl * ctl) { ASSERT(ctl != NULL); - lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); modctl_t *prevp, *nextp, *curp; // Remove stale chain first @@ -16047,7 +16132,7 @@ dtrace_resume(void) static int dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu) { - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); lck_mtx_lock(&dtrace_lock); switch (what) { @@ -16179,7 +16264,7 @@ dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1, NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); @@ -16190,7 +16275,7 @@ dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) sizeof (dtrace_dstate_percpu_t) * (int)NCPU, DTRACE_STATE_ALIGN, NULL, NULL, NULL, NULL, NULL, 0); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod), offsetof(dtrace_probe_t, dtpr_nextmod), @@ -16235,6 +16320,13 @@ dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) dtrace_provider, NULL, NULL, "END", 0, NULL); dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t) dtrace_provider, NULL, NULL, "ERROR", 3, NULL); +#elif (defined(__arm__) || defined(__arm64__)) + dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t) + dtrace_provider, NULL, NULL, "BEGIN", 2, NULL); + dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t) + dtrace_provider, NULL, NULL, "END", 1, NULL); + dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t) + dtrace_provider, NULL, NULL, "ERROR", 4, NULL); #else #error Unknown Architecture #endif @@ -16515,7 +16607,12 @@ dtrace_ioctl_helper(u_long cmd, caddr_t arg, int *rv) return KERN_SUCCESS; switch (cmd) { +#if defined (__arm64__) + case DTRACEHIOC_ADDDOF_U32: + case DTRACEHIOC_ADDDOF_U64: +#else case DTRACEHIOC_ADDDOF: +#endif /* __arm64__*/ { dof_helper_t *dhp = NULL; size_t dof_ioctl_data_size; @@ -16527,6 +16624,16 @@ dtrace_ioctl_helper(u_long cmd, caddr_t arg, int *rv) int multi_dof_claimed = 0; proc_t* p = current_proc(); + /* + * If this is a restricted process and dtrace is restricted, + * do not allow DOFs to be registered + */ + if (dtrace_is_restricted() && + !dtrace_are_restrictions_relaxed() && + !dtrace_can_attach_to_proc(current_proc())) { + return (EACCES); + } + /* * Read the number of DOF sections being passed in. */ @@ -16536,7 +16643,7 @@ dtrace_ioctl_helper(u_long cmd, caddr_t arg, int *rv) dtrace_dof_error(NULL, "failed to copyin dofiod_count"); return (EFAULT); } - + /* * Range check the count. */ @@ -17309,7 +17416,8 @@ dtrace_ioctl(dev_t dev, u_long cmd, user_addr_t arg, int md, cred_t *cr, int *rv /* * We have our snapshot; now copy it out. */ - if (copyout(buf->dtb_xamot, (user_addr_t)desc.dtbd_data, + if (dtrace_buffer_copyout(buf->dtb_xamot, + (user_addr_t)desc.dtbd_data, buf->dtb_xamot_offset) != 0) { lck_mtx_unlock(&dtrace_lock); return (EFAULT); @@ -18242,7 +18350,12 @@ dtrace_init( void ) * makes no sense... */ if (!PE_parse_boot_argn("dtrace_dof_mode", &dtrace_dof_mode, sizeof (dtrace_dof_mode))) { +#if CONFIG_EMBEDDED + /* Disable DOF mode by default for performance reasons */ + dtrace_dof_mode = DTRACE_DOF_MODE_NEVER; +#else dtrace_dof_mode = DTRACE_DOF_MODE_LAZY_ON; +#endif } /* @@ -18307,6 +18420,10 @@ dtrace_postinit(void) if (dtrace_module_loaded(&fake_kernel_kmod, 0) != 0) { printf("dtrace_postinit: Could not register mach_kernel modctl\n"); } + + if (!PE_parse_boot_argn("dtrace_provide_private_probes", &dtrace_provide_private_probes, sizeof (dtrace_provide_private_probes))) { + dtrace_provide_private_probes = 0; + } (void)OSKextRegisterKextsWithDTrace(); } diff --git a/bsd/dev/dtrace/dtrace_glue.c b/bsd/dev/dtrace/dtrace_glue.c index 7bd5500b0..57f0f3207 100644 --- a/bsd/dev/dtrace/dtrace_glue.c +++ b/bsd/dev/dtrace/dtrace_glue.c @@ -301,7 +301,11 @@ typedef struct wrap_timer_call { typedef struct cyc_list { cyc_omni_handler_t cyl_omni; wrap_timer_call_t cyl_wrap_by_cpus[]; +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +} __attribute__ ((aligned (8))) cyc_list_t; +#else } cyc_list_t; +#endif /* CPU going online/offline notifications */ void (*dtrace_cpu_state_changed_hook)(int, boolean_t) = NULL; @@ -573,29 +577,6 @@ cyclic_remove(cyclic_id_t cyclic) } } -/* - * timeout / untimeout (converted to dtrace_timeout / dtrace_untimeout due to name collision) - */ - -thread_call_t -dtrace_timeout(void (*func)(void *, void *), void* arg, uint64_t nanos) -{ -#pragma unused(arg) - thread_call_t call = thread_call_allocate(func, NULL); - - nanoseconds_to_absolutetime(nanos, &nanos); - - /* - * This method does not use clock_deadline_for_periodic_event() because it is a one-shot, - * and clock drift on later invocations is not a worry. - */ - uint64_t deadline = mach_absolute_time() + nanos; - /* DRK: consider using a lower priority callout here */ - thread_call_enter_delayed(call, deadline); - - return call; -} - /* * ddi */ @@ -1249,6 +1230,30 @@ dtrace_copyoutstr(uintptr_t src, user_addr_t dst, size_t len, volatile uint16_t } } +extern const int copysize_limit_panic; + +int +dtrace_buffer_copyout(const void *kaddr, user_addr_t uaddr, vm_size_t nbytes) +{ + /* + * Partition the copyout in copysize_limit_panic-sized chunks + */ + while (nbytes >= (vm_size_t)copysize_limit_panic) { + if (copyout(kaddr, uaddr, copysize_limit_panic) != 0) + return (EFAULT); + + nbytes -= copysize_limit_panic; + uaddr += copysize_limit_panic; + kaddr += copysize_limit_panic; + } + if (nbytes > 0) { + if (copyout(kaddr, uaddr, nbytes) != 0) + return (EFAULT); + } + + return (0); +} + uint8_t dtrace_fuword8(user_addr_t uaddr) { @@ -1483,6 +1488,8 @@ strstr(const char *in, const char *str) { char c; size_t len; + if (!in || !str) + return in; c = *str++; if (!c) @@ -1502,6 +1509,26 @@ strstr(const char *in, const char *str) return (const char *) (in - 1); } +const void* +bsearch(const void *key, const void *base0, size_t nmemb, size_t size, int (*compar)(const void *, const void *)) +{ + const char *base = base0; + size_t lim; + int cmp; + const void *p; + for (lim = nmemb; lim != 0; lim >>= 1) { + p = base + (lim >> 1) * size; + cmp = (*compar)(key, p); + if (cmp == 0) + return p; + if (cmp > 0) { /* key > p: move right */ + base = (const char *)p + size; + lim--; + } /* else move left */ + } + return (NULL); +} + /* * Runtime and ABI */ diff --git a/bsd/dev/dtrace/dtrace_ptss.c b/bsd/dev/dtrace/dtrace_ptss.c index c09b8f32e..1ce9c28d8 100644 --- a/bsd/dev/dtrace/dtrace_ptss.c +++ b/bsd/dev/dtrace/dtrace_ptss.c @@ -49,7 +49,7 @@ */ struct dtrace_ptss_page_entry* dtrace_ptss_claim_entry_locked(struct proc* p) { - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); struct dtrace_ptss_page_entry* entry = NULL; @@ -98,8 +98,8 @@ dtrace_ptss_claim_entry_locked(struct proc* p) { struct dtrace_ptss_page_entry* dtrace_ptss_claim_entry(struct proc* p) { // Verify no locks held on entry - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); - lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED); struct dtrace_ptss_page_entry* entry = NULL; @@ -167,17 +167,43 @@ dtrace_ptss_allocate_page(struct proc* p) mach_vm_size_t size = PAGE_MAX_SIZE; mach_vm_offset_t addr = 0; +#if CONFIG_EMBEDDED + mach_vm_offset_t write_addr = 0; + /* + * The embedded OS has extra permissions for writable and executable pages. + * To ensure correct permissions, we must set the page protections separately. + */ + vm_prot_t cur_protection = VM_PROT_READ|VM_PROT_EXECUTE; + vm_prot_t max_protection = VM_PROT_READ|VM_PROT_EXECUTE|VM_PROT_WRITE; +#else vm_prot_t cur_protection = VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE; vm_prot_t max_protection = VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE; +#endif /* CONFIG_EMBEDDED */ - kern_return_t kr = mach_vm_map(map, &addr, size, 0, VM_FLAGS_ANYWHERE, IPC_PORT_NULL, 0, FALSE, cur_protection, max_protection, VM_INHERIT_DEFAULT); + kern_return_t kr = mach_vm_map_kernel(map, &addr, size, 0, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_NONE, IPC_PORT_NULL, 0, FALSE, cur_protection, max_protection, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) { goto err; } +#if CONFIG_EMBEDDED + /* + * If on embedded, remap the scratch space as writable at another + * virtual address + */ + kr = mach_vm_remap_kernel(map, &write_addr, size, 0, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_NONE, map, addr, FALSE, &cur_protection, &max_protection, VM_INHERIT_DEFAULT); + if (kr != KERN_SUCCESS || !(max_protection & VM_PROT_WRITE)) + goto err; + + kr = mach_vm_protect (map, (mach_vm_offset_t)write_addr, (mach_vm_size_t)size, 0, VM_PROT_READ | VM_PROT_WRITE); + if (kr != KERN_SUCCESS) + goto err; +#endif // Chain the page entries. int i; for (i=0; ientries[i].addr = addr + (i * DTRACE_PTSS_SCRATCH_SPACE_PER_THREAD); +#if CONFIG_EMBEDDED + ptss_page->entries[i].write_addr = write_addr + (i * DTRACE_PTSS_SCRATCH_SPACE_PER_THREAD); +#endif ptss_page->entries[i].next = &ptss_page->entries[i+1]; } @@ -217,6 +243,10 @@ dtrace_ptss_free_page(struct proc* p, struct dtrace_ptss_page* ptss_page) // Silent failures, no point in checking return code. mach_vm_deallocate(map, addr, size); +#ifdef CONFIG_EMBEDDED + mach_vm_address_t write_addr = ptss_page->entries[0].write_addr; + mach_vm_deallocate(map, write_addr, size); +#endif vm_map_deallocate(map); } @@ -227,8 +257,8 @@ dtrace_ptss_free_page(struct proc* p, struct dtrace_ptss_page* ptss_page) */ void dtrace_ptss_enable(struct proc* p) { - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_OWNED); struct uthread* uth; /* @@ -252,8 +282,8 @@ dtrace_ptss_exec_exit(struct proc* p) { * Should hold sprlock to touch the pages list. Must not * hold the proc lock to avoid deadlock. */ - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED); p->p_dtrace_ptss_free_list = NULL; @@ -295,10 +325,10 @@ dtrace_ptss_fork(struct proc* parent, struct proc* child) { * Finally, to prevent a deadlock with the fasttrap cleanup code, * neither the parent or child proc_lock should be held. */ - lck_mtx_assert(&parent->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&parent->p_mlock, LCK_MTX_ASSERT_NOTOWNED); - lck_mtx_assert(&child->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&child->p_mlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&parent->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&parent->p_mlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&child->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&child->p_mlock, LCK_MTX_ASSERT_NOTOWNED); // Get page list from *PARENT* struct dtrace_ptss_page* temp = parent->p_dtrace_ptss_pages; diff --git a/bsd/dev/dtrace/fasttrap.c b/bsd/dev/dtrace/fasttrap.c index d25f82dba..8425b98af 100644 --- a/bsd/dev/dtrace/fasttrap.c +++ b/bsd/dev/dtrace/fasttrap.c @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -50,8 +51,14 @@ #include #include +#include +#include #include +#include + +extern kern_return_t kernel_thread_start_priority(thread_continue_t continuation, void *parameter, integer_t priority, thread_t *new_thread); + /* Solaris proc_t is the struct. Darwin's proc_t is a pointer to it. */ #define proc_t struct proc /* Steer clear of the Darwin typedef for proc_t */ @@ -139,9 +146,15 @@ qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *)); static dev_info_t *fasttrap_devi; static dtrace_meta_provider_id_t fasttrap_meta_id; -static thread_call_t fasttrap_timeout; +static thread_t fasttrap_cleanup_thread; + static lck_mtx_t fasttrap_cleanup_mtx; -static uint_t fasttrap_cleanup_work; + + +#define FASTTRAP_CLEANUP_PROVIDER 0x1 +#define FASTTRAP_CLEANUP_TRACEPOINT 0x2 + +static uint32_t fasttrap_cleanup_work = 0; /* * Generation count on modifications to the global tracepoint lookup table. @@ -156,6 +169,7 @@ static volatile uint64_t fasttrap_mod_gen; */ static uint32_t fasttrap_max; +static uint32_t fasttrap_retired; static uint32_t fasttrap_total; @@ -312,139 +326,257 @@ fasttrap_mod_barrier(uint64_t gen) } } -/* - * This is the timeout's callback for cleaning up the providers and their - * probes. - */ -/*ARGSUSED*/ -static void -fasttrap_pid_cleanup_cb(void *ignored, void* ignored2) +static void fasttrap_pid_cleanup(uint32_t); + +static unsigned int +fasttrap_pid_cleanup_providers(void) { -#pragma unused(ignored, ignored2) fasttrap_provider_t **fpp, *fp; fasttrap_bucket_t *bucket; dtrace_provider_id_t provid; - unsigned int i, later = 0; + unsigned int later = 0, i; - static volatile int in = 0; - ASSERT(in == 0); - in = 1; + /* + * Iterate over all the providers trying to remove the marked + * ones. If a provider is marked but not retired, we just + * have to take a crack at removing it -- it's no big deal if + * we can't. + */ + for (i = 0; i < fasttrap_provs.fth_nent; i++) { + bucket = &fasttrap_provs.fth_table[i]; + lck_mtx_lock(&bucket->ftb_mtx); + fpp = (fasttrap_provider_t **)&bucket->ftb_data; - lck_mtx_lock(&fasttrap_cleanup_mtx); - while (fasttrap_cleanup_work) { - fasttrap_cleanup_work = 0; - lck_mtx_unlock(&fasttrap_cleanup_mtx); + while ((fp = *fpp) != NULL) { + if (!fp->ftp_marked) { + fpp = &fp->ftp_next; + continue; + } - later = 0; + lck_mtx_lock(&fp->ftp_mtx); - /* - * Iterate over all the providers trying to remove the marked - * ones. If a provider is marked but not retired, we just - * have to take a crack at removing it -- it's no big deal if - * we can't. - */ - for (i = 0; i < fasttrap_provs.fth_nent; i++) { - bucket = &fasttrap_provs.fth_table[i]; - lck_mtx_lock(&bucket->ftb_mtx); - fpp = (fasttrap_provider_t **)&bucket->ftb_data; + /* + * If this provider has consumers actively + * creating probes (ftp_ccount) or is a USDT + * provider (ftp_mcount), we can't unregister + * or even condense. + */ + if (fp->ftp_ccount != 0 || + fp->ftp_mcount != 0) { + fp->ftp_marked = 0; + lck_mtx_unlock(&fp->ftp_mtx); + continue; + } - while ((fp = *fpp) != NULL) { - if (!fp->ftp_marked) { - fpp = &fp->ftp_next; - continue; - } + if (!fp->ftp_retired || fp->ftp_rcount != 0) + fp->ftp_marked = 0; - lck_mtx_lock(&fp->ftp_mtx); + lck_mtx_unlock(&fp->ftp_mtx); - /* - * If this provider has consumers actively - * creating probes (ftp_ccount) or is a USDT - * provider (ftp_mcount), we can't unregister - * or even condense. - */ - if (fp->ftp_ccount != 0 || - fp->ftp_mcount != 0) { - fp->ftp_marked = 0; - lck_mtx_unlock(&fp->ftp_mtx); - continue; - } + /* + * If we successfully unregister this + * provider we can remove it from the hash + * chain and free the memory. If our attempt + * to unregister fails and this is a retired + * provider, increment our flag to try again + * pretty soon. If we've consumed more than + * half of our total permitted number of + * probes call dtrace_condense() to try to + * clean out the unenabled probes. + */ + provid = fp->ftp_provid; + if (dtrace_unregister(provid) != 0) { + if (fasttrap_total > fasttrap_max / 2) + (void) dtrace_condense(provid); + later += fp->ftp_marked; + fpp = &fp->ftp_next; + } else { + *fpp = fp->ftp_next; + fasttrap_provider_free(fp); + } + } + lck_mtx_unlock(&bucket->ftb_mtx); + } - if (!fp->ftp_retired || fp->ftp_rcount != 0) - fp->ftp_marked = 0; + return later; +} - lck_mtx_unlock(&fp->ftp_mtx); +#ifdef FASTTRAP_ASYNC_REMOVE +typedef struct fasttrap_tracepoint_spec { + pid_t fttps_pid; + user_addr_t fttps_pc; +} fasttrap_tracepoint_spec_t; - /* - * If we successfully unregister this - * provider we can remove it from the hash - * chain and free the memory. If our attempt - * to unregister fails and this is a retired - * provider, increment our flag to try again - * pretty soon. If we've consumed more than - * half of our total permitted number of - * probes call dtrace_condense() to try to - * clean out the unenabled probes. - */ - provid = fp->ftp_provid; - if (dtrace_unregister(provid) != 0) { - if (fasttrap_total > fasttrap_max / 2) - (void) dtrace_condense(provid); - later += fp->ftp_marked; - fpp = &fp->ftp_next; - } else { - *fpp = fp->ftp_next; - fasttrap_provider_free(fp); - } +static fasttrap_tracepoint_spec_t *fasttrap_retired_spec; +static size_t fasttrap_cur_retired = 0, fasttrap_retired_size; +static lck_mtx_t fasttrap_retired_mtx; + +#define DEFAULT_RETIRED_SIZE 256 + +static void +fasttrap_tracepoint_cleanup(void) +{ + size_t i; + pid_t pid = 0; + user_addr_t pc; + proc_t *p = PROC_NULL; + fasttrap_tracepoint_t *tp = NULL; + lck_mtx_lock(&fasttrap_retired_mtx); + fasttrap_bucket_t *bucket; + for (i = 0; i < fasttrap_cur_retired; i++) { + pc = fasttrap_retired_spec[i].fttps_pc; + if (fasttrap_retired_spec[i].fttps_pid != pid) { + pid = fasttrap_retired_spec[i].fttps_pid; + if (p != PROC_NULL) { + sprunlock(p); + } + if ((p = sprlock(pid)) == PROC_NULL) { + pid = 0; + continue; } + } + bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; + lck_mtx_lock(&bucket->ftb_mtx); + for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + if (pid == tp->ftt_pid && pc == tp->ftt_pc && + tp->ftt_proc->ftpc_acount != 0) + break; + } + /* + * Check that the tracepoint is not gone or has not been + * re-activated for another probe + */ + if (tp == NULL || tp->ftt_retired == 0) { lck_mtx_unlock(&bucket->ftb_mtx); + continue; } + fasttrap_tracepoint_remove(p, tp); + lck_mtx_unlock(&bucket->ftb_mtx); + } + if (p != PROC_NULL) { + sprunlock(p); + } - lck_mtx_lock(&fasttrap_cleanup_mtx); + fasttrap_cur_retired = 0; + + lck_mtx_unlock(&fasttrap_retired_mtx); +} + +void +fasttrap_tracepoint_retire(proc_t *p, fasttrap_tracepoint_t *tp) +{ + if (tp->ftt_retired) + return; + lck_mtx_lock(&fasttrap_retired_mtx); + fasttrap_tracepoint_spec_t *s = &fasttrap_retired_spec[fasttrap_cur_retired++]; + s->fttps_pid = p->p_pid; + s->fttps_pc = tp->ftt_pc; + + if (fasttrap_cur_retired == fasttrap_retired_size) { + fasttrap_retired_size *= 2; + fasttrap_tracepoint_spec_t *new_retired = kmem_zalloc( + fasttrap_retired_size * + sizeof(fasttrap_tracepoint_t*), + KM_SLEEP); + memcpy(new_retired, fasttrap_retired_spec, sizeof(fasttrap_tracepoint_t*) * fasttrap_retired_size); + kmem_free(fasttrap_retired_spec, sizeof(fasttrap_tracepoint_t*) * (fasttrap_retired_size / 2)); + fasttrap_retired_spec = new_retired; } - ASSERT(fasttrap_timeout != 0); + lck_mtx_unlock(&fasttrap_retired_mtx); - /* - * APPLE NOTE: You must hold the fasttrap_cleanup_mtx to do this! - */ - if (fasttrap_timeout != (thread_call_t)1) - thread_call_free(fasttrap_timeout); + tp->ftt_retired = 1; - /* - * If we were unable to remove a retired provider, try again after - * a second. This situation can occur in certain circumstances where - * providers cannot be unregistered even though they have no probes - * enabled because of an execution of dtrace -l or something similar. - * If the timeout has been disabled (set to 1 because we're trying - * to detach), we set fasttrap_cleanup_work to ensure that we'll - * get a chance to do that work if and when the timeout is reenabled - * (if detach fails). - */ - if (later > 0 && fasttrap_timeout != (thread_call_t)1) - /* The time value passed to dtrace_timeout is in nanos */ - fasttrap_timeout = dtrace_timeout(&fasttrap_pid_cleanup_cb, NULL, NANOSEC / SEC); - else if (later > 0) - fasttrap_cleanup_work = 1; - else - fasttrap_timeout = 0; + fasttrap_pid_cleanup(FASTTRAP_CLEANUP_TRACEPOINT); +} +#else +void fasttrap_tracepoint_retire(proc_t *p, fasttrap_tracepoint_t *tp) +{ + if (tp->ftt_retired) + return; + + fasttrap_tracepoint_remove(p, tp); +} +#endif + +static void +fasttrap_pid_cleanup_compute_priority(void) +{ + if (fasttrap_total > (fasttrap_max / 100 * 90) || fasttrap_retired > fasttrap_max / 2) { + thread_precedence_policy_data_t precedence = {12 /* BASEPRI_PREEMPT_HIGH */}; + thread_policy_set(fasttrap_cleanup_thread, THREAD_PRECEDENCE_POLICY, (thread_policy_t) &precedence, THREAD_PRECEDENCE_POLICY_COUNT); + } + else { + thread_precedence_policy_data_t precedence = {-39 /* BASEPRI_USER_INITIATED */}; + thread_policy_set(fasttrap_cleanup_thread, THREAD_PRECEDENCE_POLICY, (thread_policy_t) &precedence, THREAD_PRECEDENCE_POLICY_COUNT); + + } +} + +/* + * This is the timeout's callback for cleaning up the providers and their + * probes. + */ +/*ARGSUSED*/ +__attribute__((noreturn)) +static void +fasttrap_pid_cleanup_cb(void) +{ + uint32_t work = 0; + lck_mtx_lock(&fasttrap_cleanup_mtx); + msleep(&fasttrap_pid_cleanup_cb, &fasttrap_cleanup_mtx, PRIBIO, "fasttrap_pid_cleanup_cb", NULL); + while (1) { + unsigned int later = 0; + + work = atomic_and_32(&fasttrap_cleanup_work, 0); + lck_mtx_unlock(&fasttrap_cleanup_mtx); + if (work & FASTTRAP_CLEANUP_PROVIDER) { + later = fasttrap_pid_cleanup_providers(); + } +#ifdef FASTTRAP_ASYNC_REMOVE + if (work & FASTTRAP_CLEANUP_TRACEPOINT) { + fasttrap_tracepoint_cleanup(); + } +#endif + lck_mtx_lock(&fasttrap_cleanup_mtx); + + fasttrap_pid_cleanup_compute_priority(); + if (!fasttrap_cleanup_work) { + /* + * If we were unable to remove a retired provider, try again after + * a second. This situation can occur in certain circumstances where + * providers cannot be unregistered even though they have no probes + * enabled because of an execution of dtrace -l or something similar. + * If the timeout has been disabled (set to 1 because we're trying + * to detach), we set fasttrap_cleanup_work to ensure that we'll + * get a chance to do that work if and when the timeout is reenabled + * (if detach fails). + */ + if (later > 0) { + struct timespec t = {1, 0}; + msleep(&fasttrap_pid_cleanup_cb, &fasttrap_cleanup_mtx, PRIBIO, "fasttrap_pid_cleanup_cb", &t); + } + else + msleep(&fasttrap_pid_cleanup_cb, &fasttrap_cleanup_mtx, PRIBIO, "fasttrap_pid_cleanup_cb", NULL); + } + } - lck_mtx_unlock(&fasttrap_cleanup_mtx); - in = 0; } /* * Activates the asynchronous cleanup mechanism. */ static void -fasttrap_pid_cleanup(void) +fasttrap_pid_cleanup(uint32_t work) { lck_mtx_lock(&fasttrap_cleanup_mtx); - fasttrap_cleanup_work = 1; - if (fasttrap_timeout == 0) - fasttrap_timeout = dtrace_timeout(&fasttrap_pid_cleanup_cb, NULL, NANOSEC / MILLISEC); + atomic_or_32(&fasttrap_cleanup_work, work); + fasttrap_pid_cleanup_compute_priority(); + wakeup(&fasttrap_pid_cleanup_cb); lck_mtx_unlock(&fasttrap_cleanup_mtx); } + /* * This is called from cfork() via dtrace_fasttrap_fork(). The child * process's address space is a (roughly) a copy of the parent process's so @@ -458,7 +590,7 @@ fasttrap_fork(proc_t *p, proc_t *cp) unsigned int i; ASSERT(current_proc() == p); - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); ASSERT(p->p_dtrace_count > 0); ASSERT(cp->p_dtrace_count == 0); @@ -477,7 +609,7 @@ fasttrap_fork(proc_t *p, proc_t *cp) * because we're in fork(). */ if (cp != sprlock(cp->p_pid)) { - printf("fasttrap_fork: sprlock(%d) returned a differt proc\n", cp->p_pid); + printf("fasttrap_fork: sprlock(%d) returned a different proc\n", cp->p_pid); return; } proc_unlock(cp); @@ -527,8 +659,8 @@ static void fasttrap_exec_exit(proc_t *p) { ASSERT(p == current_proc()); - lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); /* APPLE NOTE: Okay, the locking here is really odd and needs some @@ -618,6 +750,7 @@ fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) again: lck_mtx_lock(&bucket->ftb_mtx); for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + int rc = 0; /* * Note that it's safe to access the active count on the * associated proc structure because we know that at least one @@ -634,7 +767,10 @@ again: * enabled and the trap instruction hasn't been overwritten. * Since this is a little hairy, we'll punt for now. */ - + if (!tp->ftt_installed) { + if (fasttrap_tracepoint_install(p, tp) != 0) + rc = FASTTRAP_ENABLE_PARTIAL; + } /* * This can't be the first interested probe. We don't have * to worry about another thread being in the midst of @@ -666,6 +802,8 @@ again: ASSERT(0); } + tp->ftt_retired = 0; + lck_mtx_unlock(&bucket->ftb_mtx); if (new_tp != NULL) { @@ -673,7 +811,7 @@ again: new_tp->ftt_retids = NULL; } - return (0); + return rc; } /* @@ -697,7 +835,6 @@ again: */ if (fasttrap_tracepoint_install(p, new_tp) != 0) rc = FASTTRAP_ENABLE_PARTIAL; - /* * Increment the count of the number of tracepoints active in * the victim process. @@ -705,6 +842,7 @@ again: //ASSERT(p->p_proc_flag & P_PR_LOCK); p->p_dtrace_count++; + return (rc); } @@ -714,6 +852,7 @@ again: * Initialize the tracepoint that's been preallocated with the probe. */ new_tp = probe->ftp_tps[index].fit_tp; + new_tp->ftt_retired = 0; ASSERT(new_tp->ftt_pid == pid); ASSERT(new_tp->ftt_pc == pc); @@ -1148,8 +1287,9 @@ fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) lck_mtx_unlock(&provider->ftp_mtx); } - if (whack) - fasttrap_pid_cleanup(); + if (whack) { + fasttrap_pid_cleanup(FASTTRAP_CLEANUP_PROVIDER); + } if (!probe->ftp_enabled) return; @@ -1213,6 +1353,7 @@ fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) ASSERT(fasttrap_total >= probe->ftp_ntps); atomic_add_32(&fasttrap_total, -probe->ftp_ntps); + atomic_add_32(&fasttrap_retired, -probe->ftp_ntps); if (probe->ftp_gen + 1 >= fasttrap_mod_gen) fasttrap_mod_barrier(probe->ftp_gen); @@ -1579,7 +1720,6 @@ fasttrap_provider_retire(proc_t *p, const char *name, int mprov) fasttrap_provider_t *fp; fasttrap_bucket_t *bucket; dtrace_provider_id_t provid; - ASSERT(strlen(name) < sizeof (fp->ftp_name)); bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(p->p_pid, name)]; @@ -1622,6 +1762,13 @@ fasttrap_provider_retire(proc_t *p, const char *name, int mprov) atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); + /* + * Add this provider probes to the retired count and + * make sure we don't add them twice + */ + atomic_add_32(&fasttrap_retired, fp->ftp_pcount); + fp->ftp_pcount = 0; + fp->ftp_retired = 1; fp->ftp_marked = 1; provid = fp->ftp_provid; @@ -1636,7 +1783,7 @@ fasttrap_provider_retire(proc_t *p, const char *name, int mprov) lck_mtx_unlock(&bucket->ftb_mtx); - fasttrap_pid_cleanup(); + fasttrap_pid_cleanup(FASTTRAP_CLEANUP_PROVIDER); } static int @@ -1703,9 +1850,21 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) if (p == PROC_NULL) return (ESRCH); + /* + * Set that the process is allowed to run modified code and + * bail if it is not allowed to + */ +#if CONFIG_EMBEDDED + if ((p->p_csflags & (CS_KILL|CS_HARD)) && !cs_allow_invalid(p)) { + proc_rele(p); + return (EPERM); + } +#endif if ((provider = fasttrap_provider_lookup(p, pdata->ftps_provider_type, - provider_name, &pid_attr)) == NULL) + provider_name, &pid_attr)) == NULL) { + proc_rele(p); return (ESRCH); + } proc_rele(p); /* @@ -1738,11 +1897,11 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) continue; atomic_add_32(&fasttrap_total, 1); - if (fasttrap_total > fasttrap_max) { atomic_add_32(&fasttrap_total, -1); goto no_mem; } + provider->ftp_pcount++; pp = zalloc(fasttrap_probe_t_zones[1]); bzero(pp, sizeof (fasttrap_probe_t)); @@ -1760,6 +1919,15 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; tp->ftt_pid = pdata->ftps_pid; +#if defined(__arm__) || defined(__arm64__) + /* + * On arm the subinfo is used to distinguish between arm + * and thumb modes. On arm64 there is no thumb mode, so + * this field is simply initialized to 0 on its way + * into the kernel. + */ + tp->ftt_fntype = pdata->ftps_arch_subinfo; +#endif pp->ftp_tps[0].fit_tp = tp; pp->ftp_tps[0].fit_id.fti_probe = pp; @@ -1792,7 +1960,7 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); goto no_mem; } - + provider->ftp_pcount += pdata->ftps_noffs; ASSERT(pdata->ftps_noffs > 0); if (pdata->ftps_noffs < FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS) { pp = zalloc(fasttrap_probe_t_zones[pdata->ftps_noffs]); @@ -1814,6 +1982,16 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; tp->ftt_pid = pdata->ftps_pid; +#if defined(__arm__) || defined (__arm64__) + /* + * On arm the subinfo is used to distinguish between arm + * and thumb modes. On arm64 there is no thumb mode, so + * this field is simply initialized to 0 on its way + * into the kernel. + */ + + tp->ftt_fntype = pdata->ftps_arch_subinfo; +#endif pp->ftp_tps[i].fit_tp = tp; pp->ftp_tps[i].fit_id.fti_probe = pp; pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_probe_type; @@ -1837,7 +2015,7 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) lck_mtx_unlock(&provider->ftp_mtx); if (whack) - fasttrap_pid_cleanup(); + fasttrap_pid_cleanup(FASTTRAP_CLEANUP_PROVIDER); return (0); @@ -1854,7 +2032,7 @@ no_mem: provider->ftp_marked = 1; lck_mtx_unlock(&provider->ftp_mtx); - fasttrap_pid_cleanup(); + fasttrap_pid_cleanup(FASTTRAP_CLEANUP_PROVIDER); return (ENOMEM); } @@ -2025,6 +2203,8 @@ fasttrap_meta_create_probe(void *arg, void *parg, return; } + provider->ftp_pcount += ntps; + if (ntps < FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS) { pp = zalloc(fasttrap_probe_t_zones[ntps]); bzero(pp, offsetof(fasttrap_probe_t, ftp_tps[ntps])); @@ -2058,6 +2238,14 @@ fasttrap_meta_create_probe(void *arg, void *parg, * Both 32 & 64 bit want to go back one byte, to point at the first NOP */ tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_offs[i] - 1; +#elif defined(__arm__) || defined(__arm64__) + /* + * All ARM and ARM64 probes are zero offset. We need to zero out the + * thumb bit because we still support 32bit user processes. + * On 64bit user processes, bit zero won't be set anyway. + */ + tp->ftt_pc = (dhpb->dthpb_base + (int64_t)dhpb->dthpb_offs[i]) & ~0x1UL; + tp->ftt_fntype = FASTTRAP_FN_USDT; #else #error "Architecture not supported" #endif @@ -2088,6 +2276,14 @@ fasttrap_meta_create_probe(void *arg, void *parg, * Both 32 & 64 bit want to go forward two bytes, to point at a single byte nop. */ tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_enoffs[j] + 2; +#elif defined(__arm__) || defined(__arm64__) + /* + * All ARM and ARM64 probes are zero offset. We need to zero out the + * thumb bit because we still support 32bit user processes. + * On 64bit user processes, bit zero won't be set anyway. + */ + tp->ftt_pc = (dhpb->dthpb_base + (int64_t)dhpb->dthpb_enoffs[j]) & ~0x1UL; + tp->ftt_fntype = FASTTRAP_FN_USDT; #else #error "Architecture not supported" #endif @@ -2356,10 +2552,23 @@ fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) * Yes, this is a WAG. */ fasttrap_max = (sane_size >> 28) * 100000; + +#if CONFIG_EMBEDDED +#if defined(__LP64__) + /* + * On embedded, the zone map does not grow with the memory size over 1GB + * (see osfmk/vm/vm_init.c) + */ + if (fasttrap_max > 400000) { + fasttrap_max = 400000; + } +#endif +#endif if (fasttrap_max == 0) fasttrap_max = 50000; fasttrap_total = 0; + fasttrap_retired = 0; /* * Conjure up the tracepoints hashtable... @@ -2561,6 +2770,22 @@ fasttrap_init( void ) return; } + /* + * Start the fasttrap cleanup thread + */ + kern_return_t res = kernel_thread_start_priority((thread_continue_t)fasttrap_pid_cleanup_cb, NULL, 46 /* BASEPRI_BACKGROUND */, &fasttrap_cleanup_thread); + if (res != KERN_SUCCESS) { + panic("Could not create fasttrap_cleanup_thread"); + } + thread_set_thread_name(fasttrap_cleanup_thread, "dtrace_fasttrap_cleanup_thread"); + +#ifdef FASTTRAP_ASYNC_REMOVE + fasttrap_retired_size = DEFAULT_RETIRED_SIZE; + fasttrap_retired_spec = kmem_zalloc(fasttrap_retired_size * sizeof(fasttrap_tracepoint_t*), + KM_SLEEP); + lck_mtx_init(&fasttrap_retired_mtx, fasttrap_lck_grp, fasttrap_lck_attr); +#endif + gFasttrapInited = 1; } } diff --git a/bsd/dev/dtrace/fbt.c b/bsd/dev/dtrace/fbt.c index e05d5a922..25f052f1a 100644 --- a/bsd/dev/dtrace/fbt.c +++ b/bsd/dev/dtrace/fbt.c @@ -31,11 +31,12 @@ #endif #endif -#include +#include #include #include #include +#include #include #include #include @@ -53,7 +54,11 @@ /* #include */ struct savearea_t; /* Used anonymously */ -#if defined(__x86_64__) +#if defined(__arm__) || defined(__arm64__) +typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int); +extern perfCallback tempDTraceTrapHook; +extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int); +#elif defined(__x86_64__) typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, __unused int); extern perfCallback tempDTraceTrapHook; extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, __unused int); @@ -71,8 +76,407 @@ fbt_probe_t **fbt_probetab; int fbt_probetab_mask; static int fbt_verbose = 0; +int ignore_fbt_blacklist = 0; + +extern int dtrace_kernel_symbol_mode; + + void fbt_init( void ); +/* + * Critical routines that must not be probed. PR_5221096, PR_5379018. + * The blacklist must be kept in alphabetic order for purposes of bsearch(). + */ +static const char * critical_blacklist[] = +{ + "Call_DebuggerC", + "SysChoked", + "_ZN9IOService14newTemperatureElPS_", /* IOService::newTemperature */ + "_ZN9IOService26temperatureCriticalForZoneEPS_", /* IOService::temperatureCriticalForZone */ + "_ZNK6OSData14getBytesNoCopyEv", /* Data::getBytesNoCopy, IOHibernateSystemWake path */ + "_disable_preemption", + "_enable_preemption", + "bcopy_phys", + "console_cpu_alloc", + "console_cpu_free", + "cpu_IA32e_disable", + "cpu_IA32e_enable", + "cpu_NMI_interrupt", + "cpu_control", + "cpu_data_alloc", + "cpu_desc_init", + "cpu_desc_init64", + "cpu_desc_load", + "cpu_desc_load64", + "cpu_exit_wait", + "cpu_info", + "cpu_info_count", + "cpu_init", + "cpu_interrupt", + "cpu_machine_init", + "cpu_mode_init", + "cpu_processor_alloc", + "cpu_processor_free", + "cpu_signal_handler", + "cpu_sleep", + "cpu_start", + "cpu_subtype", + "cpu_thread_alloc", + "cpu_thread_halt", + "cpu_thread_init", + "cpu_threadtype", + "cpu_to_processor", + "cpu_topology_sort", + "cpu_topology_start_cpu", + "cpu_type", + "cpuid_cpu_display", + "cpuid_extfeatures", + "dtrace_invop", + "enter_lohandler", + "fbt_invop", + "fbt_perfCallback", + "get_threadtask", + "handle_pending_TLB_flushes", + "hw_compare_and_store", + "interrupt", + "kernel_trap", + "kprintf", + "lo_alltraps", + "lock_debugger", + "machine_idle_cstate", + "machine_thread_get_kern_state", + "mca_cpu_alloc", + "mca_cpu_init", + "ml_nofault_copy", + "nanoseconds_to_absolutetime", + "nanotime_to_absolutetime", + "packA", + "panic", + "pmKextRegister", + "pmMarkAllCPUsOff", + "pmSafeMode", + "pmTimerRestore", + "pmTimerSave", + "pmUnRegister", + "pmap_cpu_alloc", + "pmap_cpu_free", + "pmap_cpu_high_map_vaddr", + "pmap_cpu_high_shared_remap", + "pmap_cpu_init", + "power_management_init", + "preemption_underflow_panic", + "register_cpu_setup_func", + "sdt_invop", + "sprlock", + "sprunlock", + "t_invop", + "tmrCvt", + "uread", + "uwrite", + "unlock_debugger", + "unpackA", + "unregister_cpu_setup_func", + "vstart" +}; +#define CRITICAL_BLACKLIST_COUNT (sizeof(critical_blacklist)/sizeof(critical_blacklist[0])) + +/* + * The transitive closure of entry points that can be reached from probe context. + * (Apart from routines whose names begin with dtrace_). + */ +static const char * probe_ctx_closure[] = +{ + "ClearIdlePop", + "Debugger", + "IS_64BIT_PROCESS", + "OSCompareAndSwap", + "SetIdlePop", + "absolutetime_to_microtime", + "act_set_astbsd", + "arm_init_idle_cpu", + "ast_dtrace_on", + "ast_pending", + "clean_dcache", + "clean_mmu_dcache", + "clock_get_calendar_nanotime_nowait", + "copyin", + "copyin_kern", + "copyin_user", + "copyinstr", + "copyout", + "copyoutstr", + "cpu_number", + "current_proc", + "current_processor", + "current_task", + "current_thread", + "debug_enter", + "drain_write_buffer", + "find_user_regs", + "flush_dcache", + "flush_tlb64", + "get_bsdtask_info", + "get_bsdthread_info", + "hertz_tick", + "hw_atomic_and", + "invalidate_mmu_icache", + "kauth_cred_get", + "kauth_getgid", + "kauth_getuid", + "kernel_preempt_check", + "kvtophys", + "mach_absolute_time", + "max_valid_stack_address", + "memcpy", + "memmove", + "ml_at_interrupt_context", + "ml_phys_write_byte_64", + "ml_phys_write_half_64", + "ml_phys_write_word_64", + "ml_set_interrupts_enabled", + "mt_core_snap", + "mt_cur_cpu_cycles", + "mt_cur_cpu_instrs", + "mt_cur_thread_cycles", + "mt_cur_thread_instrs", + "mt_fixed_counts", + "mt_fixed_counts_internal", + "mt_mtc_update_count", + "mt_update_thread", + "ovbcopy", + "panic", + "pmap64_pde", + "pmap64_pdpt", + "pmap_find_phys", + "pmap_get_mapwindow", + "pmap_pde", + "pmap_pte", + "pmap_put_mapwindow", + "pmap_valid_page", + "prf", + "proc_is64bit", + "proc_selfname", + "psignal_lock", + "rtc_nanotime_load", + "rtc_nanotime_read", + "sdt_getargdesc", + "setPop", + "strlcpy", + "sync_iss_to_iks_unconditionally", + "systrace_stub", + "timer_grab" +}; +#define PROBE_CTX_CLOSURE_COUNT (sizeof(probe_ctx_closure)/sizeof(probe_ctx_closure[0])) + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +static int _cmp(const void *a, const void *b) +{ + return strncmp((const char *)a, *(const char **)b, strlen((const char *)a) + 1); +} +#pragma clang diagnostic pop +/* + * Module validation + */ +int +fbt_module_excluded(struct modctl* ctl) +{ + ASSERT(!MOD_FBT_DONE(ctl)); + + if (ctl->mod_address == 0 || ctl->mod_size == 0) { + return TRUE; + } + + if (ctl->mod_loaded == 0) { + return TRUE; + } + + /* + * If the user sets this, trust they know what they are doing. + */ + if (ignore_fbt_blacklist) + return FALSE; + + /* + * These drivers control low level functions that when traced + * cause problems often in the sleep/wake paths as well as + * critical debug and panic paths. + * If somebody really wants to drill in on one of these kexts, then + * they can override blacklisting using the boot-arg above. + */ + +#ifdef __x86_64__ + if (strstr(ctl->mod_modname, "AppleACPIEC") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleACPIPlatform") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleRTC") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "IOACPIFamily") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleIntelCPUPowerManagement") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleProfile") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleIntelProfile") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleEFI") != NULL) + return TRUE; + +#elif __arm__ || __arm64__ + if (LIT_STRNEQL(ctl->mod_modname, "com.apple.driver.AppleARMPlatform") || + LIT_STRNEQL(ctl->mod_modname, "com.apple.driver.AppleARMPL192VIC") || + LIT_STRNEQL(ctl->mod_modname, "com.apple.driver.AppleInterruptController")) + return TRUE; +#endif + + return FALSE; +} + +/* + * FBT probe name validation + */ +int +fbt_excluded(const char* name) +{ + /* + * If the user set this, trust they know what they are doing. + */ + if (ignore_fbt_blacklist) + return FALSE; + + if (LIT_STRNSTART(name, "dtrace_") && !LIT_STRNSTART(name, "dtrace_safe_")) { + /* + * Anything beginning with "dtrace_" may be called + * from probe context unless it explitly indicates + * that it won't be called from probe context by + * using the prefix "dtrace_safe_". + */ + return TRUE; + } + + /* + * Place no probes on critical routines (5221096) + */ + if (bsearch( name, critical_blacklist, CRITICAL_BLACKLIST_COUNT, sizeof(name), _cmp ) != NULL) + return TRUE; + + /* + * Place no probes that could be hit in probe context. + */ + if (bsearch( name, probe_ctx_closure, PROBE_CTX_CLOSURE_COUNT, sizeof(name), _cmp ) != NULL) { + return TRUE; + } + + /* + * Place no probes that could be hit in probe context. + * In the interests of safety, some of these may be overly cautious. + * Also exclude very low-level "firmware" class calls. + */ + if (LIT_STRNSTART(name, "cpu_") || /* Coarse */ + LIT_STRNSTART(name, "platform_") || /* Coarse */ + LIT_STRNSTART(name, "machine_") || /* Coarse */ + LIT_STRNSTART(name, "ml_") || /* Coarse */ + LIT_STRNSTART(name, "PE_") || /* Coarse */ + LIT_STRNSTART(name, "rtc_") || /* Coarse */ + LIT_STRNSTART(name, "_rtc_") || + LIT_STRNSTART(name, "rtclock_") || + LIT_STRNSTART(name, "clock_") || + LIT_STRNSTART(name, "bcopy") || + LIT_STRNSTART(name, "pmap_") || + LIT_STRNSTART(name, "hw_") || /* Coarse */ + LIT_STRNSTART(name, "lapic_") || /* Coarse */ + LIT_STRNSTART(name, "OSAdd") || + LIT_STRNSTART(name, "OSBit") || + LIT_STRNSTART(name, "OSDecrement") || + LIT_STRNSTART(name, "OSIncrement") || + LIT_STRNSTART(name, "OSCompareAndSwap") || + LIT_STRNSTART(name, "etimer_") || + LIT_STRNSTART(name, "dtxnu_kern_") || + LIT_STRNSTART(name, "flush_mmu_tlb_")) + return TRUE; + /* + * Fasttrap inner-workings we can't instrument + * on Intel (6230149) + */ + if (LIT_STRNSTART(name, "fasttrap_") || + LIT_STRNSTART(name, "fuword") || + LIT_STRNSTART(name, "suword")) + return TRUE; + + if (LIT_STRNSTART(name, "_dtrace")) + return TRUE; /* Shims in dtrace.c */ + + if (LIT_STRNSTART(name, "hibernate_")) + return TRUE; + + /* + * Place no probes in the exception handling path + */ +#if __arm__ || __arm64__ + if (LIT_STRNSTART(name, "fleh_") || + LIT_STRNSTART(name, "sleh_") || + LIT_STRNSTART(name, "timer_state_event") || + LIT_STRNEQL(name, "get_vfp_enabled")) + return TRUE; + + if (LIT_STRNSTART(name, "_ZNK15OSMetaClassBase8metaCastEPK11OSMetaClass") || + LIT_STRNSTART(name, "_ZN15OSMetaClassBase12safeMetaCastEPKS_PK11OSMetaClass") || + LIT_STRNSTART(name, "_ZNK11OSMetaClass13checkMetaCastEPK15OSMetaClassBase")) + return TRUE; +#endif + + +#ifdef __x86_64__ + if (LIT_STRNSTART(name, "machine_") || + LIT_STRNSTART(name, "mapping_") || + LIT_STRNSTART(name, "tsc_") || + LIT_STRNSTART(name, "pmCPU") || + LIT_STRNSTART(name, "pms") || + LIT_STRNSTART(name, "usimple_") || + LIT_STRNSTART(name, "lck_spin_lock") || + LIT_STRNSTART(name, "lck_spin_unlock") || + LIT_STRNSTART(name, "absolutetime_to_") || + LIT_STRNSTART(name, "commpage_") || + LIT_STRNSTART(name, "ml_") || + LIT_STRNSTART(name, "PE_") || + LIT_STRNSTART(name, "act_machine") || + LIT_STRNSTART(name, "acpi_") || + LIT_STRNSTART(name, "pal_")) { + return TRUE; + } + // Don't Steal Mac OS X + if (LIT_STRNSTART(name, "dsmos_")) + return TRUE; + +#endif + + /* + * Place no probes that could be hit on the way to the debugger. + */ + if (LIT_STRNSTART(name, "kdp_") || + LIT_STRNSTART(name, "kdb_") || + LIT_STRNSTART(name, "debug_")) { + return TRUE; + } + + /* + * Place no probes that could be hit on the way to a panic. + */ + if (NULL != strstr(name, "panic_")) + return TRUE; + + return FALSE; +} + + /*ARGSUSED*/ static void fbt_destroy(void *arg, dtrace_id_t id, void *parg) @@ -267,6 +671,13 @@ fbt_resume(void *arg, dtrace_id_t id, void *parg) (void)ml_nofault_copy( (vm_offset_t)&fbt->fbtp_patchval, (vm_offset_t)fbt->fbtp_patchpoint, sizeof(fbt->fbtp_patchval)); +#if CONFIG_EMBEDDED + /* + * Make the patched instruction visible via a data + instruction cache flush. + */ + flush_dcache((vm_offset_t)fbt->fbtp_patchpoint,(vm_size_t)sizeof(fbt->fbtp_patchval), 0); + invalidate_icache((vm_offset_t)fbt->fbtp_patchpoint,(vm_size_t)sizeof(fbt->fbtp_patchval), 0); +#endif fbt->fbtp_currentval = fbt->fbtp_patchval; } @@ -374,6 +785,85 @@ err: } #endif /* __APPLE__ */ +static void +fbt_provide_module_user_syms(struct modctl *ctl) +{ + unsigned int i; + char *modname = ctl->mod_modname; + + dtrace_module_symbols_t* module_symbols = ctl->mod_user_symbols; + if (module_symbols) { + for (i=0; idtmodsyms_count; i++) { + + /* + * symbol->dtsym_addr (the symbol address) passed in from + * user space, is already slid for both kexts and kernel. + */ + dtrace_symbol_t* symbol = &module_symbols->dtmodsyms_symbols[i]; + + char* name = symbol->dtsym_name; + + /* Lop off omnipresent leading underscore. */ + if (*name == '_') + name += 1; + + /* + * We're only blacklisting functions in the kernel for now. + */ + if (MOD_IS_MACH_KERNEL(ctl) && fbt_excluded(name)) + continue; + + /* + * Ignore symbols with a null address + */ + if (!symbol->dtsym_addr) + continue; + + fbt_provide_probe(ctl, (uintptr_t)symbol->dtsym_addr, (uintptr_t)(symbol->dtsym_addr + symbol->dtsym_size), modname, name, (machine_inst_t*)(uintptr_t)symbol->dtsym_addr); + } + } +} + + +void +fbt_provide_module(void *arg, struct modctl *ctl) +{ +#pragma unused(arg) + ASSERT(ctl != NULL); + ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); + + // Update the "ignore blacklist" bit + if (ignore_fbt_blacklist) + ctl->mod_flags |= MODCTL_FBT_PROVIDE_BLACKLISTED_PROBES; + + if (MOD_FBT_DONE(ctl)) + return; + + if (fbt_module_excluded(ctl)) { + ctl->mod_flags |= MODCTL_FBT_INVALID; + return; + } + + if (MOD_HAS_KERNEL_SYMBOLS(ctl)) { + fbt_provide_module_kernel_syms(ctl); + ctl->mod_flags |= MODCTL_FBT_PROBES_PROVIDED; + if (MOD_FBT_PROVIDE_BLACKLISTED_PROBES(ctl)) + ctl->mod_flags |= MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED; + return; + } + + if (MOD_HAS_USERSPACE_SYMBOLS(ctl)) { + fbt_provide_module_user_syms(ctl); + ctl->mod_flags |= MODCTL_FBT_PROBES_PROVIDED; + if (MOD_FBT_PROVIDE_PRIVATE_PROBES(ctl)) + ctl->mod_flags |= MODCTL_FBT_PRIVATE_PROBES_PROVIDED; + if (MOD_FBT_PROVIDE_BLACKLISTED_PROBES(ctl)) + ctl->mod_flags |= MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED; + return; + } +} + static dtrace_pattr_t fbt_attr = { { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, @@ -451,6 +941,47 @@ _fbt_open(dev_t dev, int flags, int devtype, struct proc *p) #define FBT_MAJOR -24 /* let the kernel pick the device number */ +SYSCTL_DECL(_kern_dtrace); + +static int +sysctl_dtrace_ignore_fbt_blacklist SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg2) + int err; + int value = *(int*)arg1; + + err = sysctl_io_number(req, value, sizeof(value), &value, NULL); + if (err) + return (err); + if (req->newptr) { + if (!(value == 0 || value == 1)) + return (ERANGE); + + /* + * We do not allow setting the blacklist back to on, as we have no way + * of knowing if those unsafe probes are still used. + * + * If we are using kernel symbols, we also do not allow any change, + * since the symbols are jettison'd after the first pass. + * + * We do not need to take any locks here because those symbol modes + * are permanent and do not change after boot. + */ + if (value != 1 || dtrace_kernel_symbol_mode == DTRACE_KERNEL_SYMBOLS_NEVER || + dtrace_kernel_symbol_mode == DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL) + return (EPERM); + + ignore_fbt_blacklist = 1; + } + + return (0); +} + +SYSCTL_PROC(_kern_dtrace, OID_AUTO, ignore_fbt_blacklist, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &ignore_fbt_blacklist, 0, + sysctl_dtrace_ignore_fbt_blacklist, "I", "fbt provider ignore blacklist"); + /* * A struct describing which functions will get invoked for certain * actions. @@ -473,8 +1004,8 @@ static struct cdevsw fbt_cdevsw = 0 /* type */ }; -int gIgnoreFBTBlacklist = 0; -static int gFBTInited = 0; +static int fbt_inited = 0; + #undef kmem_alloc /* from its binding to dt_kmem_alloc glue */ #undef kmem_free /* from its binding to dt_kmem_free glue */ #include @@ -482,7 +1013,7 @@ static int gFBTInited = 0; void fbt_init( void ) { - if (0 == gFBTInited) + if (0 == fbt_inited) { int majdevno = cdevsw_add(FBT_MAJOR, &fbt_cdevsw); @@ -491,11 +1022,11 @@ fbt_init( void ) return; } - PE_parse_boot_argn("IgnoreFBTBlacklist", &gIgnoreFBTBlacklist, sizeof (gIgnoreFBTBlacklist)); + PE_parse_boot_argn("IgnoreFBTBlacklist", &ignore_fbt_blacklist, sizeof (ignore_fbt_blacklist)); fbt_attach( (dev_info_t *)(uintptr_t)majdevno, DDI_ATTACH ); - gFBTInited = 1; /* Ensure this initialization occurs just one time. */ + fbt_inited = 1; /* Ensure this initialization occurs just one time. */ } else panic("fbt_init: called twice!\n"); diff --git a/bsd/dev/dtrace/lockstat.c b/bsd/dev/dtrace/lockstat.c index ef1d9f1e7..2aebd0a1e 100644 --- a/bsd/dev/dtrace/lockstat.c +++ b/bsd/dev/dtrace/lockstat.c @@ -52,6 +52,18 @@ #define membar_producer dtrace_membar_producer +#define PROBE_ARGS0(a, b, c, d, e) "\000" +#define PROBE_ARGS1(a, b, c, d, e) a "\000" +#define PROBE_ARGS2(a, b, c, d, e) a "\000" b "\000" +#define PROBE_ARGS3(a, b, c, d, e) a "\000" b "\000" c "\000" +#define PROBE_ARGS4(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" +#define PROBE_ARGS5(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" e "\000" +#define PROBE_ARGS_(a, b, c, d, e, n, ...) PROBE_ARGS##n(a, b, c, d, e) +#define PROBE_ARGS(...) PROBE_ARGS_(__VA_ARGS__, 5, 4, 3, 2, 1, 0) + +#define LOCKSTAT_PROBE(func, name, probe, ...) \ + {func, name, probe, DTRACE_IDNONE, PROBE_ARGS(__VA_ARGS__)} + /* * Hot patch values, x86 */ @@ -59,6 +71,14 @@ #define NOP 0x90 #define RET 0xc3 #define LOCKSTAT_AFRAMES 1 +#elif defined(__arm__) +#define NOP 0xE1A00000 +#define BXLR 0xE12FFF1E +#define LOCKSTAT_AFRAMES 2 +#elif defined(__arm64__) +#define NOP 0xD503201F +#define RET 0xD65f03c0 +#define LOCKSTAT_AFRAMES 2 #else #error "not ported to this architecture" #endif @@ -68,52 +88,90 @@ typedef struct lockstat_probe { const char *lsp_name; int lsp_probe; dtrace_id_t lsp_id; + const char *lsp_args; } lockstat_probe_t; lockstat_probe_t lockstat_probes[] = { #if defined(__x86_64__) /* Only provide implemented probes for each architecture */ - { LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, DTRACE_IDNONE }, - { LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, DTRACE_IDNONE }, - { LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, DTRACE_IDNONE }, - { LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, DTRACE_IDNONE }, - { LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, DTRACE_IDNONE }, -// { LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, DTRACE_IDNONE }, - { LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, DTRACE_IDNONE }, - { LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, DTRACE_IDNONE }, - { LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, DTRACE_IDNONE }, + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE) + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"), + // TODO: This should not be a uint64_t ! + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + // TODO: This should NOT be a uint64_t + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"), + // TODO : This should NOT be a uint64_t + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"), + // See above + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"), + //TODO : Separate the probes for the hw_bit from the probe for the normal hw locks + LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"), + LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"), +#elif defined(__arm__) || defined(__arm64__) + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE) +// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"), + // TODO : This should NOT be a uint64_t + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"), + // See above + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"), + //TODO : Separate the probes for the hw_bit from the probe for the normal hw locks + LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"), + LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"), #endif + /* Interlock measurements would be nice, but later */ + #ifdef LATER - /* Interlock and spinlock measurements would be nice, but later */ - { LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, DTRACE_IDNONE }, - { LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, DTRACE_IDNONE }, - - { LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSA_ILK_SPIN, LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN, DTRACE_IDNONE }, - { LS_LCK_MTX_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_LOCK_ILK_SPIN, DTRACE_IDNONE }, - { LS_LCK_MTX_EXT_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_EXT_LOCK_ILK_SPIN, DTRACE_IDNONE }, - { LS_LCK_RW_TRY_LOCK_EXCL, LSA_ILK_SPIN, LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN, DTRACE_IDNONE }, - { LS_LCK_RW_TRY_LOCK_SHARED, LSA_SPIN, LS_LCK_RW_TRY_LOCK_SHARED_SPIN, DTRACE_IDNONE }, + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSA_ILK_SPIN, LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN), + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_LOCK_ILK_SPIN), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_EXT_LOCK_ILK_SPIN), + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSA_ILK_SPIN, LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN), + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSA_SPIN, LS_LCK_RW_TRY_LOCK_SHARED_SPIN) #endif - { NULL, NULL, 0, 0 } + { NULL, NULL, 0, 0, NULL} }; dtrace_id_t lockstat_probemap[LS_NPROBES]; @@ -126,14 +184,15 @@ extern void lck_mtx_try_lock_spin_lockstat_patch_point(void); extern void lck_mtx_unlock_lockstat_patch_point(void); extern void lck_mtx_lock_ext_lockstat_patch_point(void); extern void lck_mtx_ext_unlock_lockstat_patch_point(void); -extern void lck_rw_lock_shared_lockstat_patch_point(void); -extern void lck_rw_lock_exclusive_lockstat_patch_point(void); -extern void lck_rw_lock_shared_to_exclusive_lockstat_patch_point(void); -extern void lck_rw_try_lock_shared_lockstat_patch_point(void); -extern void lck_rw_try_lock_exclusive_lockstat_patch_point(void); extern void lck_mtx_lock_spin_lockstat_patch_point(void); +#endif +#if defined (__arm__) + #endif +#if defined (__arm64__) + +#endif #endif /* CONFIG_DTRACE */ typedef struct lockstat_assembly_probe { @@ -158,11 +217,6 @@ typedef struct lockstat_assembly_probe { { LS_LCK_MTX_UNLOCK_RELEASE, (vm_offset_t *) lck_mtx_unlock_lockstat_patch_point }, { LS_LCK_MTX_EXT_LOCK_ACQUIRE, (vm_offset_t *) lck_mtx_lock_ext_lockstat_patch_point }, { LS_LCK_MTX_EXT_UNLOCK_RELEASE, (vm_offset_t *) lck_mtx_ext_unlock_lockstat_patch_point }, - { LS_LCK_RW_LOCK_SHARED_ACQUIRE, (vm_offset_t *) lck_rw_lock_shared_lockstat_patch_point }, - { LS_LCK_RW_LOCK_EXCL_ACQUIRE, (vm_offset_t *) lck_rw_lock_exclusive_lockstat_patch_point }, - { LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE,(vm_offset_t *) lck_rw_lock_shared_to_exclusive_lockstat_patch_point }, - { LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, (vm_offset_t *) lck_rw_try_lock_shared_lockstat_patch_point }, - { LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, (vm_offset_t *) lck_rw_try_lock_exclusive_lockstat_patch_point }, { LS_LCK_MTX_LOCK_SPIN_ACQUIRE, (vm_offset_t *) lck_mtx_lock_spin_lockstat_patch_point }, #endif /* No assembly patch points for ARM */ @@ -199,6 +253,20 @@ void lockstat_hot_patch(boolean_t active, int ls_probe) (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point), sizeof(instr)); } +#elif defined (__arm__) + { + uint32_t instr; + instr = (active ? NOP : BXLR ); + (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point), + sizeof(instr)); + } +#elif defined (__arm64__) + { + uint32_t instr; + instr = (active ? NOP : RET ); + (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point), + sizeof(instr)); + } #endif } /* for */ } @@ -310,6 +378,29 @@ lockstat_destroy(void *arg, dtrace_id_t id, void *parg) probe->lsp_id = 0; } +static void +lockstat_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) +{ +#pragma unused(arg, id) + lockstat_probe_t *probe = parg; + const char* argdesc = probe->lsp_args; + int narg = 0; + + desc->dtargd_native[0] = '\0'; + desc->dtargd_xlate[0] = '\0'; + + while(argdesc[0] != '\0') { + if (narg == desc->dtargd_ndx) { + strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN); + return; + } + argdesc += strlen(argdesc) + 1; + narg++; + } + + desc->dtargd_ndx = DTRACE_ARGNONE; +} + static dtrace_pattr_t lockstat_attr = { { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, @@ -325,7 +416,7 @@ static dtrace_pops_t lockstat_pops = { lockstat_disable, NULL, NULL, - NULL, + lockstat_getargdesc, NULL, NULL, lockstat_destroy diff --git a/bsd/dev/dtrace/profile_prvd.c b/bsd/dev/dtrace/profile_prvd.c index eb5ada1bb..259fab8bc 100644 --- a/bsd/dev/dtrace/profile_prvd.c +++ b/bsd/dev/dtrace/profile_prvd.c @@ -54,6 +54,8 @@ #if defined(__x86_64__) extern x86_saved_state_t *find_kern_regs(thread_t); +#elif defined (__arm__) || defined(__arm64__) +extern struct arm_saved_state *find_kern_regs(thread_t); #else #error Unknown architecture #endif @@ -98,6 +100,8 @@ static dtrace_provider_id_t profile_id; #if defined(__x86_64__) #define PROF_ARTIFICIAL_FRAMES 9 +#elif defined(__arm__) || defined(__arm64__) +#define PROF_ARTIFICIAL_FRAMES 8 #else #error Unknown architecture #endif @@ -186,6 +190,50 @@ profile_fire(void *arg) dtrace_probe(prof->prof_id, 0x0, regs->eip, late, 0, 0); } } +#elif defined(__arm__) + { + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + + // We should only come in here from interrupt context, so we should always have valid kernel regs + assert(NULL != arm_kern_regs); + + if (arm_kern_regs->cpsr & 0xF) { + /* Kernel was interrupted. */ + dtrace_probe(prof->prof_id, arm_kern_regs->pc, 0x0, late, 0, 0); + } else { + /* Possibly a user interrupt */ + arm_saved_state_t *arm_user_regs = (arm_saved_state_t *)find_user_regs(current_thread()); + + if (NULL == arm_user_regs) { + /* Too bad, so sad, no useful interrupt state. */ + dtrace_probe(prof->prof_id, 0xcafebabe, 0x0, late, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */ + } else { + dtrace_probe(prof->prof_id, 0x0, arm_user_regs->pc, late, 0, 0); + } + } + } +#elif defined(__arm64__) + { + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + + // We should only come in here from interrupt context, so we should always have valid kernel regs + assert(NULL != arm_kern_regs); + + if (saved_state64(arm_kern_regs)->cpsr & 0xF) { + /* Kernel was interrupted. */ + dtrace_probe(prof->prof_id, saved_state64(arm_kern_regs)->pc, 0x0, late, 0, 0); + } else { + /* Possibly a user interrupt */ + arm_saved_state_t *arm_user_regs = (arm_saved_state_t *)find_user_regs(current_thread()); + + if (NULL == arm_user_regs) { + /* Too bad, so sad, no useful interrupt state. */ + dtrace_probe(prof->prof_id, 0xcafebabe, 0x0, late, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */ + } else { + dtrace_probe(prof->prof_id, 0x0, get_saved_state_pc(arm_user_regs), late, 0, 0); + } + } + } #else #error Unknown architecture #endif @@ -221,6 +269,45 @@ profile_tick(void *arg) dtrace_probe(prof->prof_id, 0x0, regs->eip, 0, 0, 0); } } +#elif defined(__arm__) + { + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + + if (NULL != arm_kern_regs) { + /* Kernel was interrupted. */ + dtrace_probe(prof->prof_id, arm_kern_regs->pc, 0x0, 0, 0, 0); + } else { + /* Possibly a user interrupt */ + arm_saved_state_t *arm_user_regs = (arm_saved_state_t *)find_user_regs(current_thread()); + + if (NULL == arm_user_regs) { + /* Too bad, so sad, no useful interrupt state. */ + dtrace_probe(prof->prof_id, 0xcafebabe, 0x0, 0, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */ + } else { + dtrace_probe(prof->prof_id, 0x0, arm_user_regs->pc, 0, 0, 0); + } + } + } +#elif defined(__arm64__) + { + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + + if (NULL != arm_kern_regs) { + /* Kernel was interrupted. */ + dtrace_probe(prof->prof_id, saved_state64(arm_kern_regs)->pc, 0x0, 0, 0, 0); + } else { + /* Possibly a user interrupt */ + arm_saved_state_t *arm_user_regs = (arm_saved_state_t *)find_user_regs(current_thread()); + + if (NULL == arm_user_regs) { + /* Too bad, so sad, no useful interrupt state. */ + dtrace_probe(prof->prof_id, 0xcafebabe, 0x0, 0, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */ + } else { + dtrace_probe(prof->prof_id, 0x0, get_saved_state_pc(arm_user_regs), 0, 0, 0); + } + } + } + #else #error Unknown architecture #endif @@ -499,6 +586,46 @@ profile_disable(void *arg, dtrace_id_t id, void *parg) prof->prof_cyclic = CYCLIC_NONE; } +static uint64_t +profile_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ +#pragma unused(arg, id, parg, argno, aframes) + /* + * All the required arguments for the profile probe are passed directly + * to dtrace_probe, and we do not go through dtrace_getarg which doesn't + * know how to hop to the kernel stack from the interrupt stack like + * dtrace_getpcstack + */ + return 0; +} + +static void +profile_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) +{ +#pragma unused(arg, id) + profile_probe_t *prof = parg; + const char *argdesc = NULL; + switch (desc->dtargd_ndx) { + case 0: + argdesc = "void*"; + break; + case 1: + argdesc = "user_addr_t"; + break; + case 2: + if (prof->prof_kind == PROF_PROFILE) { + argdesc = "hrtime_t"; + } + break; + } + if (argdesc) { + strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN); + } + else { + desc->dtargd_ndx = DTRACE_ARGNONE; + } +} + /* * APPLE NOTE: profile_usermode call not supported. */ @@ -524,8 +651,8 @@ static dtrace_pops_t profile_pops = { profile_disable, NULL, NULL, - NULL, - NULL, + profile_getargdesc, + profile_getarg, profile_usermode, profile_destroy }; diff --git a/bsd/dev/dtrace/scripts/Makefile b/bsd/dev/dtrace/scripts/Makefile index a6f2527cd..1957fb2b0 100644 --- a/bsd/dev/dtrace/scripts/Makefile +++ b/bsd/dev/dtrace/scripts/Makefile @@ -18,7 +18,8 @@ INSTALL_DTRACE_SCRIPTS_LIST = \ unistd.d INSTALL_DTRACE_LIBEXEC_LIST = \ - log_unnest_badness.d + log_unnest_badness.d \ + vm_map_delete_permanent.d ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) INSTALL_DTRACE_SCRIPTS_LIST += mptcp.d diff --git a/bsd/dev/dtrace/scripts/mptcp.d b/bsd/dev/dtrace/scripts/mptcp.d index 331f82928..479b531dc 100644 --- a/bsd/dev/dtrace/scripts/mptcp.d +++ b/bsd/dev/dtrace/scripts/mptcp.d @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 Apple Computer, Inc. All Rights Reserved. + * Copyright (c) 2013-2017 Apple Computer, Inc. All Rights Reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -48,14 +48,9 @@ inline int MPTCPS_FIN_WAIT_2 = 7; #pragma D binding "1.0" MPTCPS_FIN_WAIT_2 inline int MPTCPS_TIME_WAIT = 8; #pragma D binding "1.0" MPTCPS_TIME_WAIT -inline int MPTCPS_FASTCLOSE_WAIT = 9; -#pragma D binding "1.0" MPTCPS_FASTCLOSE_WAIT inline int MPTCPS_TERMINATE = 10; #pragma D binding "1.0" MPTCPS_TERMINATE -typedef uint64_t mptcp_key_t; -typedef uint32_t mptcp_token_t; - typedef struct mptsinfo { string state; uint32_t flags; @@ -76,7 +71,6 @@ typedef struct mptsinfo { uint64_t local_idsn; uint32_t sndwnd; uint64_t rcvnxt; - uint64_t rcvatmark; uint64_t remote_idsn; uint32_t rcvwnd; struct mptcb *mptcb; @@ -94,15 +88,13 @@ translator mptsinfo_t < struct mptcb *T > { T->mpt_state == MPTCPS_LAST_ACK ? "state-last-ack" : T->mpt_state == MPTCPS_FIN_WAIT_2 ? "state-fin-wait-2" : T->mpt_state == MPTCPS_TIME_WAIT ? "state-time-wait" : - T->mpt_state == MPTCPS_FASTCLOSE_WAIT ? - "state-fastclose-wait" : T->mpt_state == MPTCPS_TERMINATE ? "state-terminate" : ""; flags = T->mpt_flags; vers = T->mpt_version; error = T->mpt_softerror; - localkey = T->mpt_localkey ? *T->mpt_localkey : 0; + localkey = T->mpt_localkey; remotekey = T->mpt_remotekey; localtoken = T->mpt_localtoken; remotetoken = T->mpt_remotetoken; @@ -117,7 +109,6 @@ translator mptsinfo_t < struct mptcb *T > { local_idsn = T->mpt_local_idsn; sndwnd = T->mpt_sndwnd; rcvnxt = T->mpt_rcvnxt; - rcvatmark = T->mpt_rcvatmark; remote_idsn = T->mpt_remote_idsn; rcvwnd = T->mpt_rcvwnd; mptcb = T; @@ -210,17 +201,12 @@ inline int MPTSF_ACTIVE = 0x40000; #pragma D binding "1.0" MPTSF_ACTIVE inline int MPTSF_MPCAP_CTRSET = 0x80000; #pragma D binding "1.0" MPTSF_MPCAP_CTRSET -inline int MPTSF_FASTJ_SEND = 0x100000; -#pragma D binding "1.0" MPTSF_FASTJ_SEND typedef struct mptsubinfo { uint32_t flags; uint32_t evctl; - uint32_t family; sae_connid_t connid; uint32_t rank; - int32_t error; - uint64_t sndnxt; struct mptsub *mptsub; } mptsubinfo_t; @@ -228,10 +214,6 @@ typedef struct mptsubinfo { translator mptsubinfo_t < struct mptsub *T > { flags = T->mpts_flags; evctl = T->mpts_evctl; - family = T->mpts_family; connid = T->mpts_connid; - rank = T->mpts_rank; - error = T->mpts_soerror; - sndnxt = T->mpts_sndnxt; mptsub = T; }; diff --git a/bsd/dev/dtrace/scripts/vm_map_delete_permanent.d b/bsd/dev/dtrace/scripts/vm_map_delete_permanent.d new file mode 100644 index 000000000..9adb1c65f --- /dev/null +++ b/bsd/dev/dtrace/scripts/vm_map_delete_permanent.d @@ -0,0 +1,14 @@ +#!/usr/sbin/dtrace -s + +vminfo::vm_map_delete_permanent: +{ + printf("%d[%s]: attempt to delete permanent mapping (0x%llx, 0x%llx) prot 0x%x/0x%x", + $pid, + execname, + (uint64_t) arg0, + (uint64_t) arg1, + arg2, + arg3); + stack(); + ustack(); +} diff --git a/bsd/dev/dtrace/sdt.c b/bsd/dev/dtrace/sdt.c index a157923ee..2923bf644 100644 --- a/bsd/dev/dtrace/sdt.c +++ b/bsd/dev/dtrace/sdt.c @@ -40,6 +40,9 @@ #include #include +#if CONFIG_EMBEDDED +#include +#endif #include #include @@ -52,7 +55,19 @@ extern int dtrace_kernel_symbol_mode; /* #include sdpd_func, prev_name, len); /* NUL termination is ensured. */ sdpd->sdpd_offset = *(unsigned long *)sym[i].n_value; +#if defined(__arm__) + /* PR8353094 - mask off thumb-bit */ + sdpd->sdpd_offset &= ~0x1U; +#elif defined(__arm64__) + sdpd->sdpd_offset &= ~0x1LU; +#endif /* __arm__ */ #if 0 printf("sdt_init: sdpd_offset=0x%lx, n_value=0x%lx, name=%s\n", @@ -594,7 +615,7 @@ sdt_provide_module(void *arg, struct modctl *ctl) #pragma unused(arg) ASSERT(ctl != NULL); ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER); - lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); if (MOD_SDT_DONE(ctl)) return; diff --git a/bsd/dev/dtrace/sdt_subr.c b/bsd/dev/dtrace/sdt_subr.c index 18c16e074..ad71d1ffe 100644 --- a/bsd/dev/dtrace/sdt_subr.c +++ b/bsd/dev/dtrace/sdt_subr.c @@ -887,11 +887,7 @@ sdt_argdesc_t sdt_args[] = { { "mptcp", "checksum-result", 0, 0, "struct tcpcb *", "tcpsinfo_t *" }, { "mptcp", "checksum-result", 1, 1, "struct mbuf *", "pktinfo_t *" }, { "mptcp", "checksum-result", 2, 2, "uint32_t", "uint32_t" }, - { "mptcp", "session-create", 0, 0, "struct socket *", "socketinfo_t *"}, - { "mptcp", "session-create", 1, 1, "struct sockbuf *", "socketbuf_t *"}, - { "mptcp", "session-create", 2, 2, "struct sockbuf *", "socketbuf_t *"}, - { "mptcp", "session-create", 3, 3, "struct mppcb *", "mppsinfo_t *" }, - { "mptcp", "session-create", 4, 4, "int", "int" }, + { "mptcp", "session-create", 0, 0, "struct mppcb *", "mppsinfo_t *" }, { "mptcp", "session-destroy", 0, 0, "struct mptses *", "mptsesinfo_t *" }, { "mptcp", "session-destroy", 1, 1, "struct mptcb *", "mptsinfo_t *" }, { "mptcp", "subflow-create", 0, 0, "struct mptses *", "mptsesinfo_t *"}, @@ -909,11 +905,6 @@ sdt_argdesc_t sdt_args[] = { { "mptcp", "subflow-receive", 0, 0, "struct socket *", "socketinfo_t *" }, { "mptcp", "subflow-receive", 1, 1, "struct sockbuf *", "socketbuf_t *" }, { "mptcp", "subflow-receive", 2, 2, "struct sockbuf *", "socketbuf_t *" }, - { "mptcp", "subflow-peeloff", 0, 0, "struct mptses *", "mptsesinfo_t *",}, - { "mptcp", "subflow-peeloff", 1, 1, "struct mptsub *", "mptsubinfo_t *",}, - { "mptcp", "subflow-peeloff", 2, 2, "struct socket *", "socketinfo_t *",}, - { "mptcp", "subflow-peeloff", 3, 3, "struct sockbuf *", "socketbuf_t *" }, - { "mptcp", "subflow-peeloff", 4, 4, "struct sockbuf *", "socketbuf_t *" }, { "mptcp", "subflow-input", 0, 0, "struct mptses *", "mptsesinfo_t *" }, { "mptcp", "subflow-input", 1, 1, "struct mptsub *", "mptsubinfo_t *" }, { "mptcp", "subflow-output", 0, 0, "struct mptses *", "mptsesinfo_t *"}, @@ -955,9 +946,6 @@ sdt_argdesc_t sdt_args[] = { { "mptcp", "disconnectx", 2, 2, "sae_connid_t", "sae_connid_t" }, { "mptcp", "disconnectx", 3, 3, "struct socket *", "sockinfo_t *" }, { "mptcp", "disconnectx", 4, 4, "struct mptcb *", "mptsinfo_t *" }, - { "mptcp", "peeloff", 0, 0, "struct mptses *", "mptsesinfo_t *" }, - { "mptcp", "peeloff", 1, 1, "sae_associd_t", "sae_associd_t" }, - { "mptcp", "peeloff", 2, 2, "struct socket *", "sockinfo_t *" }, { NULL, NULL, 0, 0, NULL, NULL } }; diff --git a/bsd/dev/dtrace/systrace.c b/bsd/dev/dtrace/systrace.c index 00ee62d29..9c0c34f63 100644 --- a/bsd/dev/dtrace/systrace.c +++ b/bsd/dev/dtrace/systrace.c @@ -54,6 +54,7 @@ #include #include +#include #include "systrace.h" #include #include @@ -65,10 +66,15 @@ #if defined (__x86_64__) #define SYSTRACE_ARTIFICIAL_FRAMES 2 #define MACHTRACE_ARTIFICIAL_FRAMES 3 +#elif defined(__arm__) || defined(__arm64__) +#define SYSTRACE_ARTIFICIAL_FRAMES 2 +#define MACHTRACE_ARTIFICIAL_FRAMES 3 #else #error Unknown Architecture #endif +#define SYSTRACE_NARGS (int)(sizeof(((uthread_t)NULL)->uu_arg) / sizeof(((uthread_t)NULL)->uu_arg[0])) + #include #define sy_callc sy_call /* Map Solaris slot name to Darwin's */ #define NSYSCALL nsysent /* and is less than 500 or so */ @@ -89,7 +95,8 @@ static lck_mtx_t dtrace_systrace_lock; /* probe state lock */ systrace_sysent_t *systrace_sysent = NULL; void (*systrace_probe)(dtrace_id_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); -static uint64_t systrace_getarg(void *, dtrace_id_t, void *, int, int); +static uint64_t systrace_getargval(void *, dtrace_id_t, void *, int, int); +static void systrace_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); void systrace_stub(dtrace_id_t id, uint64_t arg0, uint64_t arg1, @@ -106,10 +113,8 @@ dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv) systrace_sysent_t *sy; dtrace_id_t id; int32_t rval; -#if 0 /* XXX */ - proc_t *p; -#endif syscall_arg_t *ip = (syscall_arg_t *)uap; + uint64_t uargs[SYSTRACE_NARGS] = {0}; #if defined (__x86_64__) { @@ -135,6 +140,51 @@ dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv) } } } +#elif defined(__arm__) + { + /* + * On arm, syscall numbers depend on a flavor (indirect or not) + * and can be in either r0 or r12 (always u32) + */ + + /* See bsd/dev/arm/systemcalls.c:arm_get_syscall_number */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + + /* Check for indirect system call */ + if (arm_regs->r[12] != 0) + code = arm_regs->r[12]; + else + code = arm_regs->r[0]; + } +#elif defined(__arm64__) + { + /* + * On arm64, syscall numbers depend on a flavor (indirect or not) + * ... and for u32 can be in either r0 or r12 + * ... and for u64 can be in either x0 or x16 + */ + + /* see bsd/dev/arm/systemcalls.c:arm_get_syscall_number */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + + if (is_saved_state32(arm_regs)) { + /* Check for indirect system call */ + if (saved_state32(arm_regs)->r[12] != 0) { + code = saved_state32(arm_regs)->r[12]; + } + else { + code = saved_state32(arm_regs)->r[0]; + } + } else { + /* Check for indirect system call */ + if (saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM] != 0 ) { + code = saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM]; + } + else { + code = saved_state64(arm_regs)->x[0]; + } + } + } #else #error Unknown Architecture #endif @@ -142,20 +192,22 @@ dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv) // Bounds "check" the value of code a la unix_syscall sy = (code >= nsysent) ? &systrace_sysent[SYS_invalid] : &systrace_sysent[code]; + systrace_args(code, ip, uargs); + if ((id = sy->stsy_entry) != DTRACE_IDNONE) { uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); if (uthread) - uthread->t_dtrace_syscall_args = (void *)ip; + uthread->t_dtrace_syscall_args = uargs; - if (ip) - (*systrace_probe)(id, *ip, *(ip+1), *(ip+2), *(ip+3), *(ip+4)); - else - (*systrace_probe)(id, 0, 0, 0, 0, 0); + static_assert(SYSTRACE_NARGS >= 5, "not enough system call arguments"); + (*systrace_probe)(id, uargs[0], uargs[1], uargs[2], uargs[3], uargs[4]); if (uthread) - uthread->t_dtrace_syscall_args = (void *)0; + uthread->t_dtrace_syscall_args = NULL; } + + #if 0 /* XXX */ /* * APPLE NOTE: Not implemented. @@ -356,11 +408,6 @@ systrace_init(struct sysent *actual, systrace_sysent_t **interposed) if (a->sy_callc == dtrace_systrace_syscall) continue; -#ifdef _SYSCALL32_IMPL - if (a->sy_callc == dtrace_systrace_syscall32) - continue; -#endif - s->stsy_underlying = a->sy_callc; s->stsy_return_type = a->sy_return_type; } @@ -379,9 +426,6 @@ systrace_provide(void *arg, const dtrace_probedesc_t *desc) return; systrace_init(sysent, &systrace_sysent); -#ifdef _SYSCALL32_IMPL - systrace_init(sysent32, &systrace_sysent32); -#endif for (i = 0; i < NSYSCALL; i++) { if (systrace_sysent[i].stsy_underlying == NULL) @@ -400,10 +444,6 @@ systrace_provide(void *arg, const dtrace_probedesc_t *desc) systrace_sysent[i].stsy_entry = DTRACE_IDNONE; systrace_sysent[i].stsy_return = DTRACE_IDNONE; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[i].stsy_entry = DTRACE_IDNONE; - systrace_sysent32[i].stsy_return = DTRACE_IDNONE; -#endif } } #undef systrace_init @@ -423,14 +463,8 @@ systrace_destroy(void *arg, dtrace_id_t id, void *parg) */ if (SYSTRACE_ISENTRY((uintptr_t)parg)) { ASSERT(systrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE); -#ifdef _SYSCALL32_IMPL - ASSERT(systrace_sysent32[sysnum].stsy_entry == DTRACE_IDNONE); -#endif } else { ASSERT(systrace_sysent[sysnum].stsy_return == DTRACE_IDNONE); -#ifdef _SYSCALL32_IMPL - ASSERT(systrace_sysent32[sysnum].stsy_return == DTRACE_IDNONE); -#endif } } @@ -446,25 +480,14 @@ systrace_enable(void *arg, dtrace_id_t id, void *parg) if (SYSTRACE_ISENTRY((uintptr_t)parg)) { systrace_sysent[sysnum].stsy_entry = id; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_entry = id; -#endif } else { systrace_sysent[sysnum].stsy_return = id; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_return = id; -#endif } if (enabled) { ASSERT(sysent[sysnum].sy_callc == dtrace_systrace_syscall); return(0); } -#ifdef _SYSCALL32_IMPL - (void) casptr(&sysent32[sysnum].sy_callc, - (void *)systrace_sysent32[sysnum].stsy_underlying, - (void *)dtrace_systrace_syscall32); -#endif lck_mtx_lock(&dtrace_systrace_lock); if (sysent[sysnum].sy_callc == systrace_sysent[sysnum].stsy_underlying) { @@ -491,23 +514,12 @@ systrace_disable(void *arg, dtrace_id_t id, void *parg) ml_nofault_copy((vm_offset_t)&systrace_sysent[sysnum].stsy_underlying, (vm_offset_t)&sysent[sysnum].sy_callc, sizeof(systrace_sysent[sysnum].stsy_underlying)); lck_mtx_unlock(&dtrace_systrace_lock); -#ifdef _SYSCALL32_IMPL - (void) casptr(&sysent32[sysnum].sy_callc, - (void *)dtrace_systrace_syscall32, - (void *)systrace_sysent32[sysnum].stsy_underlying); -#endif } if (SYSTRACE_ISENTRY((uintptr_t)parg)) { systrace_sysent[sysnum].stsy_entry = DTRACE_IDNONE; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_entry = DTRACE_IDNONE; -#endif } else { systrace_sysent[sysnum].stsy_return = DTRACE_IDNONE; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_return = DTRACE_IDNONE; -#endif } } @@ -526,8 +538,8 @@ static dtrace_pops_t systrace_pops = { systrace_disable, NULL, NULL, - NULL, - systrace_getarg, + systrace_getargdesc, + systrace_getargval, NULL, systrace_destroy }; @@ -673,6 +685,28 @@ dtrace_machtrace_syscall(struct mach_call_args *args) code = -saved_state32(tagged_regs)->eax; } } +#elif defined(__arm__) + { + /* r12 has the machcall number, but it is -ve */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + code = (int)arm_regs->r[12]; + ASSERT(code < 0); /* Otherwise it would be a Unix syscall */ + code = -code; + } +#elif defined(__arm64__) + { + /* From arm/thread_status.h:get_saved_state_svc_number */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + if (is_saved_state32(arm_regs)) { + code = (int)saved_state32(arm_regs)->r[12]; + } else { + code = (int)saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM]; + } + + /* From bsd/arm64.c:mach_syscall */ + ASSERT(code < 0); /* Otherwise it would be a Unix syscall */ + code = -code; + } #else #error Unknown Architecture #endif @@ -966,27 +1000,55 @@ void systrace_init( void ) #undef SYSTRACE_MAJOR static uint64_t -systrace_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +systrace_getargval(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) { #pragma unused(arg,id,parg,aframes) /* __APPLE__ */ uint64_t val = 0; - syscall_arg_t *stack = (syscall_arg_t *)NULL; + uint64_t *uargs = NULL; uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); if (uthread) - stack = (syscall_arg_t *)uthread->t_dtrace_syscall_args; - - if (!stack) + uargs = uthread->t_dtrace_syscall_args; + if (!uargs) + return(0); + if (argno < 0 || argno > SYSTRACE_NARGS) return(0); DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); - /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */ - val = (uint64_t)*(stack+argno); + val = uargs[argno]; DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); return (val); } +static void +systrace_getargdesc(void *arg, dtrace_id_t id, void *parg, + dtrace_argdesc_t *desc) +{ +#pragma unused(arg, id) + int sysnum = SYSTRACE_SYSNUM(parg); + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + uint64_t *uargs = NULL; + + if (!uthread) { + desc->dtargd_ndx = DTRACE_ARGNONE; + return; + } + + uargs = uthread->t_dtrace_syscall_args; + + if (SYSTRACE_ISENTRY((uintptr_t)parg)) { + systrace_entry_setargdesc(sysnum, desc->dtargd_ndx, + desc->dtargd_native, sizeof(desc->dtargd_native)); + } + else { + systrace_return_setargdesc(sysnum, desc->dtargd_ndx, + desc->dtargd_native, sizeof(desc->dtargd_native)); + } + + if (desc->dtargd_native[0] == '\0') + desc->dtargd_ndx = DTRACE_ARGNONE; +} static uint64_t machtrace_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) diff --git a/bsd/dev/dtrace/systrace.h b/bsd/dev/dtrace/systrace.h index e9af96363..c86f324a9 100644 --- a/bsd/dev/dtrace/systrace.h +++ b/bsd/dev/dtrace/systrace.h @@ -62,11 +62,6 @@ extern int32_t dtrace_systrace_syscall(struct proc *, void *, int *); extern void dtrace_systrace_syscall_return(unsigned short, int, int *); -#ifdef _SYSCALL32_IMPL -extern int64_t dtrace_systrace_syscall32(uintptr_t arg0, uintptr_t arg1, - uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5); -#endif - #endif /* _KERNEL */ #ifdef __cplusplus diff --git a/bsd/dev/i386/conf.c b/bsd/dev/i386/conf.c index f76413818..36de4f945 100644 --- a/bsd/dev/i386/conf.c +++ b/bsd/dev/i386/conf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997-2012 Apple Computer, Inc. All rights reserved. + * Copyright (c) 1997-2017 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -165,11 +165,6 @@ extern d_close_t logclose; extern d_read_t logread; extern d_ioctl_t logioctl; extern d_select_t logselect; -extern d_open_t fdesc_open; -extern d_read_t fdesc_read; -extern d_write_t fdesc_write; -extern d_ioctl_t fdesc_ioctl; -extern d_select_t fdesc_select; extern d_open_t oslog_streamopen; extern d_close_t oslog_streamclose; @@ -191,118 +186,64 @@ extern d_ioctl_t oslogioctl; #define nullstop (d_stop_t *)&nulldev #define nullreset (d_reset_t *)&nulldev -struct cdevsw cdevsw[] = -{ +struct cdevsw cdevsw[] = { /* - * For character devices, every other block of 16 slots is - * reserved for Apple. The other slots are available for - * the user. This way we can both add new entries without - * running into each other. Be sure to fill in Apple's - * 16 reserved slots when you jump over us -- we'll do the - * same for you. + * To add character devices to this table dynamically, use cdevsw_add. */ - /* 0 - 15 are reserved for Apple */ - - { - cnopen, cnclose, cnread, cnwrite, /* 0*/ - cnioctl, nullstop, nullreset, 0, cnselect, - eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY - }, - NO_CDEVICE, /* 1*/ - { - cttyopen, nullclose, cttyread, cttywrite, /* 2*/ - cttyioctl, nullstop, nullreset, 0, cttyselect, - eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY - }, - { - nullopen, nullclose, mmread, mmwrite, /* 3*/ - mmioctl, nullstop, nullreset, 0, mmselect, - mmmmap, eno_strat, eno_getc, eno_putc, D_DISK - }, - { - ptsopen, ptsclose, ptsread, ptswrite, /* 4*/ - ptyioctl, ptsstop, nullreset, 0, ptsselect, - eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY - }, - { - ptcopen, ptcclose, ptcread, ptcwrite, /* 5*/ - ptyioctl, nullstop, nullreset, 0, ptcselect, - eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY - }, - { - logopen, logclose, logread, eno_rdwrt, /* 6*/ - logioctl, eno_stop, nullreset, 0, logselect, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, - { - oslogopen, oslogclose, eno_rdwrt, eno_rdwrt, /* 7*/ - oslogioctl, eno_stop, nullreset, 0, oslogselect, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, - { - oslog_streamopen, oslog_streamclose, oslog_streamread, eno_rdwrt, /* 8*/ - oslog_streamioctl, eno_stop, nullreset, 0, oslog_streamselect, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, - NO_CDEVICE, /* 9*/ - NO_CDEVICE, /*10*/ - NO_CDEVICE, /*11*/ - { - kmopen, kmclose, kmread, kmwrite, /*12*/ - kmioctl, nullstop, nullreset, km_tty, ttselect, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, - NO_CDEVICE, /*13*/ - NO_CDEVICE, /*14*/ - NO_CDEVICE, /*15*/ - - /* 16 - 31 are reserved to the user */ - NO_CDEVICE, /*16*/ - NO_CDEVICE, /*17*/ - NO_CDEVICE, /*18*/ - NO_CDEVICE, /*19*/ - NO_CDEVICE, /*20*/ - NO_CDEVICE, /*21*/ - NO_CDEVICE, /*22*/ - NO_CDEVICE, /*23*/ - NO_CDEVICE, /*24*/ - NO_CDEVICE, /*25*/ - NO_CDEVICE, /*26*/ - NO_CDEVICE, /*27*/ - NO_CDEVICE, /*28*/ - NO_CDEVICE, /*29*/ - NO_CDEVICE, /*30*/ - NO_CDEVICE, /*31*/ - - /* 32 - 47 are reserved to NeXT */ - { - fdesc_open, eno_opcl, fdesc_read, fdesc_write, /*32*/ - fdesc_ioctl, eno_stop, eno_reset, 0, fdesc_select, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, -#if 1 - NO_CDEVICE, -#else - { - sgopen, sgclose, eno_rdwrt, eno_rdwrt, /*33*/ - sgioctl, eno_stop, eno_reset, 0, eno_select, - eno_mmap, eno_strat, eno_getc, eno_putc, D_TAPE - }, -#endif - NO_CDEVICE, /*34*/ - NO_CDEVICE, /*35*/ - NO_CDEVICE, /*36*/ - NO_CDEVICE, /*37*/ - NO_CDEVICE, /*38*/ - NO_CDEVICE, /*39*/ - NO_CDEVICE, /*40*/ - NO_CDEVICE, /*41*/ - { - volopen, volclose, eno_rdwrt, eno_rdwrt, /*42*/ - volioctl, eno_stop, eno_reset, 0, (select_fcn_t *)seltrue, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, + [0] = { + cnopen, cnclose, cnread, cnwrite, + cnioctl, nullstop, nullreset, 0, cnselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [1] = NO_CDEVICE, + [2] = { + cttyopen, nullclose, cttyread, cttywrite, + cttyioctl, nullstop, nullreset, 0, cttyselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [3] = { + nullopen, nullclose, mmread, mmwrite, + mmioctl, nullstop, nullreset, 0, mmselect, + mmmmap, eno_strat, eno_getc, eno_putc, D_DISK + }, + [PTC_MAJOR] = { + ptsopen, ptsclose, ptsread, ptswrite, + ptyioctl, ptsstop, nullreset, 0, ptsselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [PTS_MAJOR] = { + ptcopen, ptcclose, ptcread, ptcwrite, + ptyioctl, nullstop, nullreset, 0, ptcselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [6] = { + logopen, logclose, logread, eno_rdwrt, + logioctl, eno_stop, nullreset, 0, logselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [7] = { + oslogopen, oslogclose, eno_rdwrt, eno_rdwrt, + oslogioctl, eno_stop, nullreset, 0, oslogselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [8] = { + oslog_streamopen, oslog_streamclose, oslog_streamread, eno_rdwrt, + oslog_streamioctl, eno_stop, nullreset, 0, oslog_streamselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [9 ... 11] = NO_CDEVICE, + [12] = { + kmopen, kmclose, kmread, kmwrite, + kmioctl, nullstop, nullreset, km_tty, ttselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [13 ... 41] = NO_CDEVICE, + [42] = { + volopen, volclose, eno_rdwrt, eno_rdwrt, + volioctl, eno_stop, eno_reset, 0, (select_fcn_t *) seltrue, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + } }; const int nchrdev = sizeof(cdevsw) / sizeof(cdevsw[0]); diff --git a/bsd/dev/i386/dis_tables.c b/bsd/dev/i386/dis_tables.c index 03f3c6197..c67273b79 100644 --- a/bsd/dev/i386/dis_tables.c +++ b/bsd/dev/i386/dis_tables.c @@ -23,8 +23,6 @@ /* * Copyright (c) 2015, Joyent, Inc. * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. - * - * Use is subject to license terms. */ /* @@ -35,9 +33,15 @@ /* Copyright (c) 1988 AT&T */ /* All Rights Reserved */ +/* + * APPLE NOTE: There is a copy of this file in userspace in + * dtrace:/disassembler/dis_tables.c + * + * It needs to be in sync with this file. + */ /* - * #pragma ident "@(#)dis_tables.c 1.18 08/05/24 SMI" + * #pragma ident "@(#)dis_tables.c 1.18 08/05/24 SMI" */ #include #include @@ -68,6 +72,7 @@ #ifdef DIS_TEXT extern char *strncpy(char *, const char *, size_t); extern size_t strlen(const char *); +extern int strcmp(const char *, const char *); extern int strncmp(const char *, const char *, size_t); extern size_t strlcat(char *, const char *, size_t); #endif @@ -91,6 +96,8 @@ typedef struct instable { uint_t it_always64:1; /* 64 bit when in 64 bit mode */ uint_t it_invalid32:1; /* invalid in IA32 */ uint_t it_stackop:1; /* push/pop stack operation */ + uint_t it_vexwoxmm:1; /* VEX instructions that don't use XMM/YMM */ + uint_t it_avxsuf:1; /* AVX suffix required */ } instable_t; /* @@ -109,16 +116,18 @@ enum { Mv, Mw, M, /* register or memory */ + MG9, /* register or memory in group 9 (prefix optional) */ Mb, /* register or memory, always byte sized */ MO, /* memory only (no registers) */ PREF, - SWAPGS, + SWAPGS_RDTSCP, MONITOR_MWAIT, R, RA, SEG, MR, RM, + RM_66r, /* RM, but with a required 0x66 prefix */ IA, MA, SD, @@ -221,6 +230,7 @@ enum { VEX_NONE, /* VEX no operand */ VEX_MO, /* VEX mod_rm -> implicit reg */ VEX_RMrX, /* VEX VEX.vvvv, mod_rm -> mod_reg */ + VEX_VRMrX, /* VEX mod_rm, VEX.vvvv -> mod_rm */ VEX_RRX, /* VEX VEX.vvvv, mod_reg -> mod_rm */ VEX_RMRX, /* VEX VEX.vvvv, mod_rm, imm8[7:4] -> mod_reg */ VEX_MX, /* VEX mod_rm -> mod_reg */ @@ -232,9 +242,16 @@ enum { VEX_RR, /* VEX mod_rm -> mod_reg */ VEX_RRi, /* VEX mod_rm, imm8 -> mod_reg */ VEX_RM, /* VEX mod_reg -> mod_rm */ - VEX_RIM, /* VEX mod_reg, imm8 -> mod_rm */ + VEX_RIM, /* VEX mod_reg, imm8 -> mod_rm */ VEX_RRM, /* VEX VEX.vvvv, mod_reg -> mod_rm */ VEX_RMX, /* VEX VEX.vvvv, mod_rm -> mod_reg */ + VEX_SbVM, /* VEX SIB, VEX.vvvv -> mod_rm */ + VMx, /* vmcall/vmlaunch/vmresume/vmxoff */ + VMxo, /* VMx instruction with optional prefix */ + SVM, /* AMD SVM instructions */ + BLS, /* BLSR, BLSMSK, BLSI */ + FMA, /* FMA instructions, all VEX_RMrX */ + ADX /* ADX instructions, support REX.w, mod_rm->mod_reg */ }; /* @@ -272,32 +289,36 @@ enum { * IND - indirect to another to another table * "T" - means to Terminate indirections (this is the final opcode) * "S" - means "operand length suffix required" + * "Sa" - means AVX2 suffix (d/q) required * "NS" - means "no suffix" which is the operand length suffix of the opcode * "Z" - means instruction size arg required * "u" - means the opcode is invalid in IA32 but valid in amd64 * "x" - means the opcode is invalid in amd64, but not IA32 * "y" - means the operand size is always 64 bits in 64 bit mode * "p" - means push/pop stack operation + * "vr" - means VEX instruction that operates on normal registers, not fpu */ #if defined(DIS_TEXT) && defined(DIS_MEM) -#define IND(table) {(instable_t *)table, 0, "", 0, 0, 0, 0, 0, 0} -#define INDx(table) {(instable_t *)table, 0, "", 0, 0, 1, 0, 0, 0} -#define TNS(name, amode) {TERM, amode, name, 0, 0, 0, 0, 0, 0} -#define TNSu(name, amode) {TERM, amode, name, 0, 0, 0, 0, 1, 0} -#define TNSx(name, amode) {TERM, amode, name, 0, 0, 1, 0, 0, 0} -#define TNSy(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0, 0} -#define TNSyp(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0, 1} -#define TNSZ(name, amode, sz) {TERM, amode, name, 0, sz, 0, 0, 0, 0} -#define TNSZy(name, amode, sz) {TERM, amode, name, 0, sz, 0, 1, 0, 0} -#define TS(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 0} -#define TSx(name, amode) {TERM, amode, name, 1, 0, 1, 0, 0, 0} -#define TSy(name, amode) {TERM, amode, name, 1, 0, 0, 1, 0, 0} -#define TSp(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 1} -#define TSZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0} -#define TSZx(name, amode, sz) {TERM, amode, name, 1, sz, 1, 0, 0, 0} -#define TSZy(name, amode, sz) {TERM, amode, name, 1, sz, 0, 1, 0, 0} -#define INVALID {TERM, UNKNOWN, "", 0, 0, 0, 0, 0} +#define IND(table) {(instable_t *)table, 0, "", 0, 0, 0, 0, 0, 0, 0, 0} +#define INDx(table) {(instable_t *)table, 0, "", 0, 0, 1, 0, 0, 0, 0, 0} +#define TNS(name, amode) {TERM, amode, name, 0, 0, 0, 0, 0, 0, 0, 0} +#define TNSu(name, amode) {TERM, amode, name, 0, 0, 0, 0, 1, 0, 0, 0} +#define TNSx(name, amode) {TERM, amode, name, 0, 0, 1, 0, 0, 0, 0, 0} +#define TNSy(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0, 0, 0, 0} +#define TNSyp(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0, 1, 0, 0} +#define TNSZ(name, amode, sz) {TERM, amode, name, 0, sz, 0, 0, 0, 0, 0, 0} +#define TNSZy(name, amode, sz) {TERM, amode, name, 0, sz, 0, 1, 0, 0, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, name, 0, sz, 0, 0, 0, 0, 1, 0} +#define TS(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 0, 0, 0} +#define TSx(name, amode) {TERM, amode, name, 1, 0, 1, 0, 0, 0, 0, 0} +#define TSy(name, amode) {TERM, amode, name, 1, 0, 0, 1, 0, 0, 0, 0} +#define TSp(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 1, 0, 0} +#define TSZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0, 0, 1} +#define TSZx(name, amode, sz) {TERM, amode, name, 1, sz, 1, 0, 0, 0, 0, 0} +#define TSZy(name, amode, sz) {TERM, amode, name, 1, sz, 0, 1, 0, 0, 0, 0} +#define INVALID {TERM, UNKNOWN, "", 0, 0, 0, 0, 0, 0, 0, 0} #elif defined(DIS_TEXT) #define IND(table) {(instable_t *)table, 0, "", 0, 0, 0, 0, 0} #define INDx(table) {(instable_t *)table, 0, "", 0, 1, 0, 0, 0} @@ -308,50 +329,56 @@ enum { #define TNSyp(name, amode) {TERM, amode, name, 0, 0, 1, 0, 1} #define TNSZ(name, amode, sz) {TERM, amode, name, 0, 0, 0, 0, 0} #define TNSZy(name, amode, sz) {TERM, amode, name, 0, 0, 1, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, name, 0, 0, 0, 0, 0, 1} #define TS(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0} #define TSx(name, amode) {TERM, amode, name, 1, 1, 0, 0, 0} #define TSy(name, amode) {TERM, amode, name, 1, 0, 1, 0, 0} #define TSp(name, amode) {TERM, amode, name, 1, 0, 0, 0, 1} #define TSZ(name, amode, sz) {TERM, amode, name, 1, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, name, 1, 0, 0, 0, 0, 0, 1} #define TSZx(name, amode, sz) {TERM, amode, name, 1, 1, 0, 0, 0} #define TSZy(name, amode, sz) {TERM, amode, name, 1, 0, 1, 0, 0} #define INVALID {TERM, UNKNOWN, "", 0, 0, 0, 0, 0} #elif defined(DIS_MEM) #define IND(table) {(instable_t *)table, 0, 0, 0, 0, 0, 0} #define INDx(table) {(instable_t *)table, 0, 0, 1, 0, 0, 0} -#define TNS(name, amode) {TERM, amode, 0, 0, 0, 0, 0} -#define TNSu(name, amode) {TERM, amode, 0, 0, 0, 1, 0} -#define TNSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0} -#define TNSyp(name, amode) {TERM, amode, 0, 0, 1, 0, 1} -#define TNSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0} -#define TNSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0} -#define TNSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0} -#define TS(name, amode) {TERM, amode, 0, 0, 0, 0, 0} -#define TSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0} -#define TSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0} -#define TSp(name, amode) {TERM, amode, 0, 0, 0, 0, 1} -#define TSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0} -#define TSZx(name, amode, sz) {TERM, amode, sz, 1, 0, 0, 0} -#define TSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0} -#define INVALID {TERM, UNKNOWN, 0, 0, 0, 0, 0} +#define TNS(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0, 0} +#define TNSu(name, amode) {TERM, amode, 0, 0, 0, 1, 0, 0, 0} +#define TNSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0, 0, 0} +#define TNSyp(name, amode) {TERM, amode, 0, 0, 1, 0, 1, 0, 0} +#define TNSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0, 0, 0} +#define TNSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 0, 0} +#define TNSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 1, 0} +#define TS(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0, 0} +#define TSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0, 0, 0} +#define TSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0, 0, 0} +#define TSp(name, amode) {TERM, amode, 0, 0, 0, 0, 1, 0, 0} +#define TSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 0, 1} +#define TSZx(name, amode, sz) {TERM, amode, sz, 1, 0, 0, 0, 0 ,0} +#define TSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0, 0, 0} +#define INVALID {TERM, UNKNOWN, 0, 0, 0, 0, 0, 0, 0} #else -#define IND(table) {(instable_t *)table, 0, 0, 0, 0, 0} -#define INDx(table) {(instable_t *)table, 0, 1, 0, 0, 0} -#define TNS(name, amode) {TERM, amode, 0, 0, 0, 0} -#define TNSu(name, amode) {TERM, amode, 0, 0, 1, 0} -#define TNSy(name, amode) {TERM, amode, 0, 1, 0, 0} -#define TNSyp(name, amode) {TERM, amode, 0, 1, 0, 1} -#define TNSx(name, amode) {TERM, amode, 1, 0, 0, 0} -#define TNSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0} -#define TNSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0} -#define TS(name, amode) {TERM, amode, 0, 0, 0, 0} -#define TSx(name, amode) {TERM, amode, 1, 0, 0, 0} -#define TSy(name, amode) {TERM, amode, 0, 1, 0, 0} -#define TSp(name, amode) {TERM, amode, 0, 0, 0, 1} -#define TSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0} -#define TSZx(name, amode, sz) {TERM, amode, 1, 0, 0, 0} -#define TSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0} -#define INVALID {TERM, UNKNOWN, 0, 0, 0, 0} +#define IND(table) {(instable_t *)table, 0, 0, 0, 0, 0, 0, 0} +#define INDx(table) {(instable_t *)table, 0, 1, 0, 0, 0, 0, 0} +#define TNS(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0} +#define TNSu(name, amode) {TERM, amode, 0, 0, 1, 0, 0, 0} +#define TNSy(name, amode) {TERM, amode, 0, 1, 0, 0, 0, 0} +#define TNSyp(name, amode) {TERM, amode, 0, 1, 0, 1, 0, 0} +#define TNSx(name, amode) {TERM, amode, 1, 0, 0, 0, 0, 0} +#define TNSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 0, 0} +#define TNSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 1, 0} +#define TS(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0} +#define TSx(name, amode) {TERM, amode, 1, 0, 0, 0, 0, 0} +#define TSy(name, amode) {TERM, amode, 0, 1, 0, 0, 0, 0} +#define TSp(name, amode) {TERM, amode, 0, 0, 0, 1, 0, 0} +#define TSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 0, 1} +#define TSZx(name, amode, sz) {TERM, amode, 1, 0, 0, 0, 0, 0} +#define TSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0, 0, 0} +#define INVALID {TERM, UNKNOWN, 0, 0, 0, 0, 0, 0} #endif #ifdef DIS_TEXT @@ -398,6 +425,12 @@ const char *const dis_addr64_mode12[16] = { */ const char *const dis_scale_factor[4] = { ")", ",2)", ",4)", ",8)" }; +/* + * decode for scale from VSIB byte, note that we always include the scale factor + * to match gas. + */ +const char *const dis_vscale_factor[4] = { ",1)", ",2)", ",4)", ",8)" }; + /* * register decoding for normal references to registers (ie. not addressing) */ @@ -477,9 +510,6 @@ const char *const dis_AVXvgrp7[3][8] = { #endif /* DIS_TEXT */ - - - /* * "decode table" for 64 bit mode MOVSXD instruction (opcode 0x63) */ @@ -505,8 +535,8 @@ const instable_t dis_op0F00[8] = { */ const instable_t dis_op0F01[8] = { -/* [0] */ TNSZ("sgdt",MO,6), TNSZ("sidt",MONITOR_MWAIT,6), TNSZ("lgdt",XGETBV_XSETBV,6), TNSZ("lidt",MO,6), -/* [4] */ TNSZ("smsw",M,2), INVALID, TNSZ("lmsw",M,2), TNS("invlpg",SWAPGS), +/* [0] */ TNSZ("sgdt",VMx,6), TNSZ("sidt",MONITOR_MWAIT,6), TNSZ("lgdt",XGETBV_XSETBV,6), TNSZ("lidt",SVM,6), +/* [4] */ TNSZ("smsw",M,2), INVALID, TNSZ("lmsw",M,2), TNS("invlpg",SWAPGS_RDTSCP), }; /* @@ -515,7 +545,7 @@ const instable_t dis_op0F01[8] = { const instable_t dis_op0F18[8] = { /* [0] */ TNS("prefetchnta",PREF),TNS("prefetcht0",PREF), TNS("prefetcht1",PREF), TNS("prefetcht2",PREF), -/* [4] */ TNSZ("xsave",M,512), TNS("lfence",XMMFENCE), TNS("mfence",XMMFENCE), TNS("sfence",XMMSFNC), +/* [4] */ INVALID, INVALID, INVALID, INVALID, }; /* @@ -523,7 +553,7 @@ const instable_t dis_op0F18[8] = { */ const instable_t dis_op0FAE[8] = { /* [0] */ TNSZ("fxsave",M,512), TNSZ("fxrstor",M,512), TNS("ldmxcsr",M), TNS("stmxcsr",M), -/* [4] */ INVALID, TNS("lfence",XMMFENCE), TNS("mfence",XMMFENCE), TNS("sfence",XMMSFNC), +/* [4] */ TNSZ("xsave",M,512), TNS("lfence",XMMFENCE), TNS("mfence",XMMFENCE), TNS("sfence",XMMSFNC), }; /* @@ -537,15 +567,44 @@ const instable_t dis_op0FBA[8] = { }; /* - * Decode table for 0x0FC7 opcode + * Decode table for 0x0FC7 opcode (group 9) */ const instable_t dis_op0FC7[8] = { /* [0] */ INVALID, TNS("cmpxchg8b",M), INVALID, INVALID, -/* [4] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("vmptrld",MG9), TNS("vmptrst",MG9), +}; + +/* + * Decode table for 0x0FC7 opcode (group 9) mode 3 + */ + +const instable_t dis_op0FC7m3[8] = { + +/* [0] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("rdrand",MG9), TNS("rdseed", MG9), }; +/* + * Decode table for 0x0FC7 opcode with 0x66 prefix + */ + +const instable_t dis_op660FC7[8] = { + +/* [0] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("vmclear",M), INVALID, +}; + +/* + * Decode table for 0x0FC7 opcode with 0xF3 prefix + */ + +const instable_t dis_opF30FC7[8] = { + +/* [0] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("vmxon",M), INVALID, +}; /* * Decode table for 0x0FC8 opcode -- 486 bswap instruction @@ -643,7 +702,7 @@ const instable_t dis_opSIMDdata16[256] = { /* [70] */ TNSZ("pshufd",XMMP,16), INVALID, INVALID, INVALID, /* [74] */ TNSZ("pcmpeqb",XMM,16), TNSZ("pcmpeqw",XMM,16), TNSZ("pcmpeqd",XMM,16), INVALID, /* [78] */ TNSZ("extrq",XMM2I,16), TNSZ("extrq",XMM,16), INVALID, INVALID, -/* [7C] */ INVALID, INVALID, TNSZ("movd",XMM3MXS,4), TNSZ("movdqa",XMMS,16), +/* [7C] */ TNSZ("haddpd",XMM,16), TNSZ("hsubpd",XMM,16), TNSZ("movd",XMM3MXS,4), TNSZ("movdqa",XMMS,16), /* [80] */ INVALID, INVALID, INVALID, INVALID, /* [84] */ INVALID, INVALID, INVALID, INVALID, @@ -670,7 +729,7 @@ const instable_t dis_opSIMDdata16[256] = { /* [C8] */ INVALID, INVALID, INVALID, INVALID, /* [CC] */ INVALID, INVALID, INVALID, INVALID, -/* [D0] */ INVALID, TNSZ("psrlw",XMM,16), TNSZ("psrld",XMM,16), TNSZ("psrlq",XMM,16), +/* [D0] */ TNSZ("addsubpd",XMM,16),TNSZ("psrlw",XMM,16), TNSZ("psrld",XMM,16), TNSZ("psrlq",XMM,16), /* [D4] */ TNSZ("paddq",XMM,16), TNSZ("pmullw",XMM,16), TNSZ("movq",XMMS,8), TNS("pmovmskb",XMMX3), /* [D8] */ TNSZ("psubusb",XMM,16), TNSZ("psubusw",XMM,16), TNSZ("pminub",XMM,16), TNSZ("pand",XMM,16), /* [DC] */ TNSZ("paddusb",XMM,16), TNSZ("paddusw",XMM,16), TNSZ("pmaxub",XMM,16), TNSZ("pandn",XMM,16), @@ -777,7 +836,7 @@ const instable_t dis_opSIMDrepnz[256] = { /* [08] */ INVALID, INVALID, INVALID, INVALID, /* [0C] */ INVALID, INVALID, INVALID, INVALID, -/* [10] */ TNSZ("movsd",XMM,8), TNSZ("movsd",XMMS,8), INVALID, INVALID, +/* [10] */ TNSZ("movsd",XMM,8), TNSZ("movsd",XMMS,8), TNSZ("movddup",XMM,8), INVALID, /* [14] */ INVALID, INVALID, INVALID, INVALID, /* [18] */ INVALID, INVALID, INVALID, INVALID, /* [1C] */ INVALID, INVALID, INVALID, INVALID, @@ -810,7 +869,7 @@ const instable_t dis_opSIMDrepnz[256] = { /* [70] */ TNSZ("pshuflw",XMMP,16),INVALID, INVALID, INVALID, /* [74] */ INVALID, INVALID, INVALID, INVALID, /* [78] */ TNSZ("insertq",XMMX2I,16),TNSZ("insertq",XMM,8),INVALID, INVALID, -/* [7C] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ TNSZ("haddps",XMM,16), TNSZ("hsubps",XMM,16), INVALID, INVALID, /* [80] */ INVALID, INVALID, INVALID, INVALID, /* [84] */ INVALID, INVALID, INVALID, INVALID, @@ -837,7 +896,7 @@ const instable_t dis_opSIMDrepnz[256] = { /* [C8] */ INVALID, INVALID, INVALID, INVALID, /* [CC] */ INVALID, INVALID, INVALID, INVALID, -/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D0] */ TNSZ("addsubps",XMM,16),INVALID, INVALID, INVALID, /* [D4] */ INVALID, INVALID, TNS("movdq2q",XMMXM), INVALID, /* [D8] */ INVALID, INVALID, INVALID, INVALID, /* [DC] */ INVALID, INVALID, INVALID, INVALID, @@ -847,7 +906,7 @@ const instable_t dis_opSIMDrepnz[256] = { /* [E8] */ INVALID, INVALID, INVALID, INVALID, /* [EC] */ INVALID, INVALID, INVALID, INVALID, -/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F0] */ TNS("lddqu",XMMM), INVALID, INVALID, INVALID, /* [F4] */ INVALID, INVALID, INVALID, INVALID, /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, @@ -935,6 +994,251 @@ const instable_t dis_opAVXF20F[256] = { /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; +const instable_t dis_opAVXF20F3A[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ TNSZvr("rorx",VEX_MXI,6),INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVXF20F38[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, TNSZvr("pdep",VEX_RMrX,5),TNSZvr("mulx",VEX_RMrX,5),TNSZvr("shrx",VEX_VRMrX,5), +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVXF30F38[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, TNSZvr("pext",VEX_RMrX,5),INVALID, TNSZvr("sarx",VEX_VRMrX,5), +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; /* * Decode table for SIMD instructions with the repz (0xf3) prefix. */ @@ -944,8 +1248,8 @@ const instable_t dis_opSIMDrepz[256] = { /* [08] */ INVALID, INVALID, INVALID, INVALID, /* [0C] */ INVALID, INVALID, INVALID, INVALID, -/* [10] */ TNSZ("movss",XMM,4), TNSZ("movss",XMMS,4), INVALID, INVALID, -/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [10] */ TNSZ("movss",XMM,4), TNSZ("movss",XMMS,4), TNSZ("movsldup",XMM,16),INVALID, +/* [14] */ INVALID, INVALID, TNSZ("movshdup",XMM,16),INVALID, /* [18] */ INVALID, INVALID, INVALID, INVALID, /* [1C] */ INVALID, INVALID, INVALID, INVALID, @@ -997,7 +1301,7 @@ const instable_t dis_opSIMDrepz[256] = { /* [B0] */ INVALID, INVALID, INVALID, INVALID, /* [B4] */ INVALID, INVALID, INVALID, INVALID, /* [B8] */ TS("popcnt",MRw), INVALID, INVALID, INVALID, -/* [BC] */ INVALID, TS("lzcnt",MRw), INVALID, INVALID, +/* [BC] */ TNSZ("tzcnt",MRw,5), TS("lzcnt",MRw), INVALID, INVALID, /* [C0] */ INVALID, INVALID, TNSZ("cmpss",XMMP,4), INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, @@ -1101,7 +1405,6 @@ const instable_t dis_opAVXF30F[256] = { /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; - /* * The following two tables are used to encode crc32 and movbe * since they share the same opcodes. @@ -1116,6 +1419,14 @@ const instable_t dis_op0F38F1[2] = { TS("movbe",MOVBE), }; +/* + * The following table is used to distinguish between adox and adcx which share + * the same opcodes. + */ +const instable_t dis_op0F38F6[2] = { +/* [00] */ TNS("adcx",ADX), + TNS("adox",ADX), +}; const instable_t dis_op0F38[256] = { /* [00] */ TNSZ("pshufb",XMM_66o,16),TNSZ("phaddw",XMM_66o,16),TNSZ("phaddd",XMM_66o,16),TNSZ("phaddsw",XMM_66o,16), @@ -1158,7 +1469,7 @@ const instable_t dis_op0F38[256] = { /* [78] */ INVALID, INVALID, INVALID, INVALID, /* [7C] */ INVALID, INVALID, INVALID, INVALID, -/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [80] */ TNSy("invept", RM_66r), TNSy("invvpid", RM_66r),TNSy("invpcid", RM_66r),INVALID, /* [84] */ INVALID, INVALID, INVALID, INVALID, /* [88] */ INVALID, INVALID, INVALID, INVALID, /* [8C] */ INVALID, INVALID, INVALID, INVALID, @@ -1180,21 +1491,20 @@ const instable_t dis_op0F38[256] = { /* [C0] */ INVALID, INVALID, INVALID, INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, -/* [C8] */ INVALID, INVALID, INVALID, INVALID, -/* [CC] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ TNSZ("sha1nexte",XMM,16),TNSZ("sha1msg1",XMM,16),TNSZ("sha1msg2",XMM,16),TNSZ("sha256rnds2",XMM,16), +/* [CC] */ TNSZ("sha256msg1",XMM,16),TNSZ("sha256msg2",XMM,16),INVALID, INVALID, /* [D0] */ INVALID, INVALID, INVALID, INVALID, /* [D4] */ INVALID, INVALID, INVALID, INVALID, -/* [D8] */ INVALID, INVALID, INVALID, INVALID, -/* [DC] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, TNSZ("aesimc",XMM_66r,16), +/* [DC] */ TNSZ("aesenc",XMM_66r,16),TNSZ("aesenclast",XMM_66r,16),TNSZ("aesdec",XMM_66r,16),TNSZ("aesdeclast",XMM_66r,16), /* [E0] */ INVALID, INVALID, INVALID, INVALID, /* [E4] */ INVALID, INVALID, INVALID, INVALID, /* [E8] */ INVALID, INVALID, INVALID, INVALID, /* [EC] */ INVALID, INVALID, INVALID, INVALID, - -/* [F0] */ TNS("crc32b",CRC32), TS("crc32",CRC32), INVALID, INVALID, -/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F0] */ IND(dis_op0F38F0), IND(dis_op0F38F1), INVALID, INVALID, +/* [F4] */ INVALID, INVALID, IND(dis_op0F38F6), INVALID, /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; @@ -1205,8 +1515,8 @@ const instable_t dis_opAVX660F38[256] = { /* [08] */ TNSZ("vpsignb",VEX_RMrX,16),TNSZ("vpsignw",VEX_RMrX,16),TNSZ("vpsignd",VEX_RMrX,16),TNSZ("vpmulhrsw",VEX_RMrX,16), /* [0C] */ TNSZ("vpermilps",VEX_RMrX,8),TNSZ("vpermilpd",VEX_RMrX,16),TNSZ("vtestps",VEX_RRI,8), TNSZ("vtestpd",VEX_RRI,16), -/* [10] */ INVALID, INVALID, INVALID, INVALID, -/* [14] */ INVALID, INVALID, INVALID, TNSZ("vptest",VEX_RRI,16), +/* [10] */ INVALID, INVALID, INVALID, TNSZ("vcvtph2ps",VEX_MX,16), +/* [14] */ INVALID, INVALID, TNSZ("vpermps",VEX_RMrX,16),TNSZ("vptest",VEX_RRI,16), /* [18] */ TNSZ("vbroadcastss",VEX_MX,4),TNSZ("vbroadcastsd",VEX_MX,8),TNSZ("vbroadcastf128",VEX_MX,16),INVALID, /* [1C] */ TNSZ("vpabsb",VEX_MX,16),TNSZ("vpabsw",VEX_MX,16),TNSZ("vpabsd",VEX_MX,16),INVALID, @@ -1216,12 +1526,12 @@ const instable_t dis_opAVX660F38[256] = { /* [2C] */ TNSZ("vmaskmovps",VEX_RMrX,8),TNSZ("vmaskmovpd",VEX_RMrX,16),TNSZ("vmaskmovps",VEX_RRM,8),TNSZ("vmaskmovpd",VEX_RRM,16), /* [30] */ TNSZ("vpmovzxbw",VEX_MX,16),TNSZ("vpmovzxbd",VEX_MX,16),TNSZ("vpmovzxbq",VEX_MX,16),TNSZ("vpmovzxwd",VEX_MX,16), -/* [34] */ TNSZ("vpmovzxwq",VEX_MX,16),TNSZ("vpmovzxdq",VEX_MX,16),TNSZ("vpermd",VEX_RMrX,16),TNSZ("vpcmpgtq",VEX_RMrX,16), +/* [34] */ TNSZ("vpmovzxwq",VEX_MX,16),TNSZ("vpmovzxdq",VEX_MX,16),TNSZ("vpermd",VEX_RMrX,16),TNSZ("vpcmpgtq",VEX_RMrX,16), /* [38] */ TNSZ("vpminsb",VEX_RMrX,16),TNSZ("vpminsd",VEX_RMrX,16),TNSZ("vpminuw",VEX_RMrX,16),TNSZ("vpminud",VEX_RMrX,16), /* [3C] */ TNSZ("vpmaxsb",VEX_RMrX,16),TNSZ("vpmaxsd",VEX_RMrX,16),TNSZ("vpmaxuw",VEX_RMrX,16),TNSZ("vpmaxud",VEX_RMrX,16), /* [40] */ TNSZ("vpmulld",VEX_RMrX,16),TNSZ("vphminposuw",VEX_MX,16),INVALID, INVALID, -/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, TSaZ("vpsrlv",VEX_RMrX,16),TNSZ("vpsravd",VEX_RMrX,16),TSaZ("vpsllv",VEX_RMrX,16), /* [48] */ INVALID, INVALID, INVALID, INVALID, /* [4C] */ INVALID, INVALID, INVALID, INVALID, @@ -1242,23 +1552,23 @@ const instable_t dis_opAVX660F38[256] = { /* [80] */ INVALID, INVALID, INVALID, INVALID, /* [84] */ INVALID, INVALID, INVALID, INVALID, -/* [88] */ INVALID, INVALID, INVALID, INVALID, -/* [8C] */ INVALID, INVALID, INVALID, INVALID, - -/* [90] */ INVALID, INVALID, INVALID, INVALID, -/* [94] */ INVALID, INVALID, INVALID, INVALID, -/* [98] */ INVALID, INVALID, INVALID, INVALID, -/* [9C] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ TSaZ("vpmaskmov",VEX_RMrX,16),INVALID, TSaZ("vpmaskmov",VEX_RRM,16),INVALID, + +/* [90] */ TNSZ("vpgatherd",VEX_SbVM,16),TNSZ("vpgatherq",VEX_SbVM,16),TNSZ("vgatherdp",VEX_SbVM,16),TNSZ("vgatherqp",VEX_SbVM,16), +/* [94] */ INVALID, INVALID, TNSZ("vfmaddsub132p",FMA,16),TNSZ("vfmsubadd132p",FMA,16), +/* [98] */ TNSZ("vfmadd132p",FMA,16),TNSZ("vfmadd132s",FMA,16),TNSZ("vfmsub132p",FMA,16),TNSZ("vfmsub132s",FMA,16), +/* [9C] */ TNSZ("vfnmadd132p",FMA,16),TNSZ("vfnmadd132s",FMA,16),TNSZ("vfnmsub132p",FMA,16),TNSZ("vfnmsub132s",FMA,16), /* [A0] */ INVALID, INVALID, INVALID, INVALID, -/* [A4] */ INVALID, INVALID, INVALID, INVALID, -/* [A8] */ INVALID, INVALID, INVALID, INVALID, -/* [AC] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, TNSZ("vfmaddsub213p",FMA,16),TNSZ("vfmsubadd213p",FMA,16), +/* [A8] */ TNSZ("vfmadd213p",FMA,16),TNSZ("vfmadd213s",FMA,16),TNSZ("vfmsub213p",FMA,16),TNSZ("vfmsub213s",FMA,16), +/* [AC] */ TNSZ("vfnmadd213p",FMA,16),TNSZ("vfnmadd213s",FMA,16),TNSZ("vfnmsub213p",FMA,16),TNSZ("vfnmsub213s",FMA,16), /* [B0] */ INVALID, INVALID, INVALID, INVALID, -/* [B4] */ INVALID, INVALID, INVALID, INVALID, -/* [B8] */ INVALID, INVALID, INVALID, INVALID, -/* [BC] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, TNSZ("vfmaddsub231p",FMA,16),TNSZ("vfmsubadd231p",FMA,16), +/* [B8] */ TNSZ("vfmadd231p",FMA,16),TNSZ("vfmadd231s",FMA,16),TNSZ("vfmsub231p",FMA,16),TNSZ("vfmsub231s",FMA,16), +/* [BC] */ TNSZ("vfnmadd231p",FMA,16),TNSZ("vfnmadd231s",FMA,16),TNSZ("vfnmsub231p",FMA,16),TNSZ("vfnmsub231s",FMA,16), /* [C0] */ INVALID, INVALID, INVALID, INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, @@ -1275,7 +1585,7 @@ const instable_t dis_opAVX660F38[256] = { /* [E8] */ INVALID, INVALID, INVALID, INVALID, /* [EC] */ INVALID, INVALID, INVALID, INVALID, /* [F0] */ IND(dis_op0F38F0), IND(dis_op0F38F1), INVALID, INVALID, -/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, TNSZvr("shlx",VEX_VRMrX,5), /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; @@ -1302,7 +1612,7 @@ const instable_t dis_op0F3A[256] = { /* [3C] */ INVALID, INVALID, INVALID, INVALID, /* [40] */ TNSZ("dpps",XMMP_66r,16),TNSZ("dppd",XMMP_66r,16),TNSZ("mpsadbw",XMMP_66r,16),INVALID, -/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ TNSZ("pclmulqdq",XMMP_66r,16),INVALID, INVALID, INVALID, /* [48] */ INVALID, INVALID, INVALID, INVALID, /* [4C] */ INVALID, INVALID, INVALID, INVALID, @@ -1344,12 +1654,12 @@ const instable_t dis_op0F3A[256] = { /* [C0] */ INVALID, INVALID, INVALID, INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, /* [C8] */ INVALID, INVALID, INVALID, INVALID, -/* [CC] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ TNSZ("sha1rnds4",XMMP,16),INVALID, INVALID, INVALID, /* [D0] */ INVALID, INVALID, INVALID, INVALID, /* [D4] */ INVALID, INVALID, INVALID, INVALID, /* [D8] */ INVALID, INVALID, INVALID, INVALID, -/* [DC] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, TNSZ("aeskeygenassist",XMMP_66r,16), /* [E0] */ INVALID, INVALID, INVALID, INVALID, /* [E4] */ INVALID, INVALID, INVALID, INVALID, @@ -1371,7 +1681,7 @@ const instable_t dis_opAVX660F3A[256] = { /* [10] */ INVALID, INVALID, INVALID, INVALID, /* [14] */ TNSZ("vpextrb",VEX_RRi,8),TNSZ("vpextrw",VEX_RRi,16),TNSZ("vpextrd",VEX_RRi,16),TNSZ("vextractps",VEX_RM,16), /* [18] */ TNSZ("vinsertf128",VEX_RMRX,16),TNSZ("vextractf128",VEX_RX,16),INVALID, INVALID, -/* [1C] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, TNSZ("vcvtps2ph",VEX_RX,16), INVALID, INVALID, /* [20] */ TNSZ("vpinsrb",VEX_RMRX,8),TNSZ("vinsertps",VEX_RMRX,16),TNSZ("vpinsrd",VEX_RMRX,16),INVALID, /* [24] */ INVALID, INVALID, INVALID, INVALID, @@ -1384,7 +1694,7 @@ const instable_t dis_opAVX660F3A[256] = { /* [3C] */ INVALID, INVALID, INVALID, INVALID, /* [40] */ TNSZ("vdpps",VEX_RMRX,16),TNSZ("vdppd",VEX_RMRX,16),TNSZ("vmpsadbw",VEX_RMRX,16),INVALID, -/* [44] */ TNSZ("vpclmulqdq",VEX_RMRX,16),INVALID, INVALID, INVALID, +/* [44] */ TNSZ("vpclmulqdq",VEX_RMRX,16),INVALID, TNSZ("vperm2i128",VEX_RMRX,16),INVALID, /* [48] */ INVALID, INVALID, TNSZ("vblendvps",VEX_RMRX,8), TNSZ("vblendvpd",VEX_RMRX,16), /* [4C] */ TNSZ("vpblendvb",VEX_RMRX,16),INVALID, INVALID, INVALID, @@ -1444,6 +1754,15 @@ const instable_t dis_opAVX660F3A[256] = { /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; +/* + * Decode table for 0x0F0D which uses the first byte of the mod_rm to + * indicate a sub-code. + */ +const instable_t dis_op0F0D[8] = { +/* [00] */ INVALID, TNS("prefetchw",PREF), TNS("prefetchwt1",PREF),INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +}; + /* * Decode table for 0x0F opcodes */ @@ -1453,12 +1772,11 @@ const instable_t dis_op0F[16][16] = { /* [00] */ IND(dis_op0F00), IND(dis_op0F01), TNS("lar",MR), TNS("lsl",MR), /* [04] */ INVALID, TNS("syscall",NORM), TNS("clts",NORM), TNS("sysret",NORM), /* [08] */ TNS("invd",NORM), TNS("wbinvd",NORM), INVALID, TNS("ud2",NORM), -/* [0C] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, IND(dis_op0F0D), INVALID, INVALID, }, { /* [10] */ TNSZ("movups",XMMO,16), TNSZ("movups",XMMOS,16),TNSZ("movlps",XMMO,8), TNSZ("movlps",XMMOS,8), /* [14] */ TNSZ("unpcklps",XMMO,16),TNSZ("unpckhps",XMMO,16),TNSZ("movhps",XMMOM,8),TNSZ("movhps",XMMOMS,8), /* [18] */ IND(dis_op0F18), INVALID, INVALID, INVALID, -/* APPLE NOTE: Need to handle multi-byte NOP */ /* [1C] */ INVALID, INVALID, INVALID, TS("nop",Mw), }, { /* [20] */ TSy("mov",SREG), TSy("mov",SREG), TSy("mov",SREG), TSy("mov",SREG), @@ -1488,7 +1806,7 @@ const instable_t dis_op0F[16][16] = { }, { /* [70] */ TNSZ("pshufw",MMOPM,8), TNS("psrXXX",MR), TNS("psrXXX",MR), TNS("psrXXX",MR), /* [74] */ TNSZ("pcmpeqb",MMO,8), TNSZ("pcmpeqw",MMO,8), TNSZ("pcmpeqd",MMO,8), TNS("emms",NORM), -/* [78] */ TNS("INVALID",XMMO), TNS("INVALID",XMMO), INVALID, INVALID, +/* [78] */ TNSy("vmread",RM), TNSy("vmwrite",MR), INVALID, INVALID, /* [7C] */ INVALID, INVALID, TNSZ("movd",MMOS,4), TNSZ("movq",MMOS,8), }, { /* [80] */ TNS("jo",D), TNS("jno",D), TNS("jb",D), TNS("jae",D), @@ -1609,8 +1927,8 @@ const instable_t dis_opAVX0F[16][16] = { /* [E8] */ INVALID, INVALID, INVALID, INVALID, /* [EC] */ INVALID, INVALID, INVALID, INVALID, }, { -/* [F0] */ INVALID, INVALID, INVALID, INVALID, -/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F0] */ INVALID, INVALID, TNSZvr("andn",VEX_RMrX,5),TNSZvr("bls",BLS,5), +/* [F4] */ INVALID, TNSZvr("bzhi",VEX_VRMrX,5),INVALID, TNSZvr("bextr",VEX_VRMrX,5), /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, } }; @@ -1773,19 +2091,19 @@ const instable_t dis_opFP1n2[8][8] = { /* [2,0] */ TNS("fiaddl",M), TNS("fimull",M), TNS("ficoml",M), TNS("ficompl",M), /* [2,4] */ TNS("fisubl",M), TNS("fisubrl",M), TNS("fidivl",M), TNS("fidivrl",M), }, { -/* [3,0] */ TNS("fildl",M), INVALID, TNS("fistl",M), TNS("fistpl",M), +/* [3,0] */ TNS("fildl",M), TNSZ("tisttpl",M,4), TNS("fistl",M), TNS("fistpl",M), /* [3,4] */ INVALID, TNSZ("fldt",M,10), INVALID, TNSZ("fstpt",M,10), }, { /* [4,0] */ TNSZ("faddl",M,8), TNSZ("fmull",M,8), TNSZ("fcoml",M,8), TNSZ("fcompl",M,8), /* [4,1] */ TNSZ("fsubl",M,8), TNSZ("fsubrl",M,8), TNSZ("fdivl",M,8), TNSZ("fdivrl",M,8), }, { -/* [5,0] */ TNSZ("fldl",M,8), INVALID, TNSZ("fstl",M,8), TNSZ("fstpl",M,8), +/* [5,0] */ TNSZ("fldl",M,8), TNSZ("fisttpll",M,8), TNSZ("fstl",M,8), TNSZ("fstpl",M,8), /* [5,4] */ TNSZ("frstor",M,108), INVALID, TNSZ("fnsave",M,108), TNSZ("fnstsw",M,2), }, { /* [6,0] */ TNSZ("fiadd",M,2), TNSZ("fimul",M,2), TNSZ("ficom",M,2), TNSZ("ficomp",M,2), /* [6,4] */ TNSZ("fisub",M,2), TNSZ("fisubr",M,2), TNSZ("fidiv",M,2), TNSZ("fidivr",M,2), }, { -/* [7,0] */ TNSZ("fild",M,2), INVALID, TNSZ("fist",M,2), TNSZ("fistp",M,2), +/* [7,0] */ TNSZ("fild",M,2), TNSZ("fisttp",M,2), TNSZ("fist",M,2), TNSZ("fistp",M,2), /* [7,4] */ TNSZ("fbld",M,10), TNSZ("fildll",M,8), TNSZ("fbstp",M,10), TNSZ("fistpll",M,8), } }; @@ -1909,7 +2227,7 @@ const instable_t dis_distable[16][16] = { /* [9,0] */ TNS("nop",NORM), TS("xchg",RA), TS("xchg",RA), TS("xchg",RA), /* [9,4] */ TS("xchg",RA), TS("xchg",RA), TS("xchg",RA), TS("xchg",RA), /* [9,8] */ TNS("cXtX",CBW), TNS("cXtX",CWD), TNSx("lcall",SO), TNS("fwait",NORM), -/* [9,C] */ TSZy("pushf",IMPLMEM,4),TSZy("popf",IMPLMEM,4), TNSx("sahf",NORM), TNSx("lahf",NORM), +/* [9,C] */ TSZy("pushf",IMPLMEM,4),TSZy("popf",IMPLMEM,4), TNS("sahf",NORM), TNS("lahf",NORM), }, { /* [A,0] */ TNS("movb",OA), TS("mov",OA), TNS("movb",AO), TS("mov",AO), /* [A,4] */ TNSZ("movsb",SD,1), TS("movs",SD), TNSZ("cmpsb",SD,1), TS("cmps",SD), @@ -2016,6 +2334,80 @@ static int isize64[] = {1, 2, 4, 8}; #define WORD_OPND 8 /* w-bit value indicating word size reg */ #define YMM_OPND 9 /* "value" used to indicate a ymm reg */ +/* + * The AVX2 gather instructions are a bit of a mess. While there's a pattern, + * there's not really a consistent scheme that we can use to know what the mode + * is supposed to be for a given type. Various instructions, like VPGATHERDD, + * always match the value of VEX_L. Other instructions like VPGATHERDQ, have + * some registers match VEX_L, but the VSIB is always XMM. + * + * The simplest way to deal with this is to just define a table based on the + * instruction opcodes, which are 0x90-0x93, so we subtract 0x90 to index into + * them. + * + * We further have to subdivide this based on the value of VEX_W and the value + * of VEX_L. The array is constructed to be indexed as: + * [opcode - 0x90][VEX_W][VEX_L]. + */ +/* w = 0, 0x90 */ +typedef struct dis_gather_regs { + uint_t dgr_arg0; /* src reg */ + uint_t dgr_arg1; /* vsib reg */ + uint_t dgr_arg2; /* dst reg */ + const char *dgr_suffix; /* suffix to append */ +} dis_gather_regs_t; + +static dis_gather_regs_t dis_vgather[4][2][2] = { + { + /* op 0x90, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "d" } + }, + /* op 0x90, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "q" }, + { YMM_OPND, XMM_OPND, YMM_OPND, "q" } + } + }, + { + /* op 0x91, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { XMM_OPND, YMM_OPND, XMM_OPND, "d" }, + }, + /* op 0x91, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "q" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "q" }, + } + }, + { + /* op 0x92, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "s" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "s" } + }, + /* op 0x92, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { YMM_OPND, XMM_OPND, YMM_OPND, "d" } + } + }, + { + /* op 0x93, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "s" }, + { XMM_OPND, YMM_OPND, XMM_OPND, "s" } + }, + /* op 0x93, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "d" } + } + } +}; + /* * Get the next byte and separate the op code into the high and low nibbles. */ @@ -2118,6 +2510,7 @@ dtrace_vex_adjust(uint_t vex_byte1, uint_t mode, uint_t *reg, uint_t *r_m) *r_m += 8; } } + /* * Get an immediate operand of the given size, with sign extension. */ @@ -2392,16 +2785,29 @@ dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) } else { uint_t need_paren = 0; char **regs; + char **bregs; + const char *const *sf; if (x->d86_mode == SIZE32) /* NOTE this is not addr_size! */ regs = (char **)dis_REG32; else regs = (char **)dis_REG64; + if (x->d86_vsib != 0) { + if (wbit == YMM_OPND) /* NOTE this is not addr_size! */ + bregs = (char **)dis_YMMREG; + else + bregs = (char **)dis_XMMREG; + sf = dis_vscale_factor; + } else { + bregs = regs; + sf = dis_scale_factor; + } + /* * print the base (if any) */ if (base == EBP_REGNO && mode == 0) { - if (index != ESP_REGNO) { + if (index != ESP_REGNO || x->d86_vsib != 0) { (void) strlcat(opnd, "(", OPLEN); need_paren = 1; } @@ -2414,10 +2820,10 @@ dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) /* * print the index (if any) */ - if (index != ESP_REGNO) { + if (index != ESP_REGNO || x->d86_vsib) { (void) strlcat(opnd, ",", OPLEN); - (void) strlcat(opnd, regs[index], OPLEN); - (void) strlcat(opnd, dis_scale_factor[ss], OPLEN); + (void) strlcat(opnd, bregs[index], OPLEN); + (void) strlcat(opnd, sf[ss], OPLEN); } else if (need_paren) (void) strlcat(opnd, ")", OPLEN); @@ -2515,16 +2921,16 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) uint_t w2; /* wbit value for second operand */ uint_t vbit; uint_t mode = 0; /* mode value from ModRM byte */ - uint_t reg; /* reg value from ModRM byte */ - uint_t r_m; /* r_m value from ModRM byte */ - - uint_t opcode1; /* high nibble of 1st byte */ - uint_t opcode2; /* low nibble of 1st byte */ - uint_t opcode3; /* extra opcode bits usually from ModRM byte */ - uint_t opcode4; /* high nibble of 2nd byte */ - uint_t opcode5; /* low nibble of 2nd byte */ - uint_t opcode6; /* high nibble of 3rd byte */ - uint_t opcode7; /* low nibble of 3rd byte */ + uint_t reg = 0; /* reg value from ModRM byte */ + uint_t r_m = 0; /* r_m value from ModRM byte */ + + uint_t opcode1 = 0; /* high nibble of 1st byte */ + uint_t opcode2 = 0; /* low nibble of 1st byte */ + uint_t opcode3 = 0; /* extra opcode bits usually from ModRM byte */ + uint_t opcode4 = 0; /* high nibble of 2nd byte */ + uint_t opcode5 = 0; /* low nibble of 2nd byte */ + uint_t opcode6 = 0; /* high nibble of 3rd byte */ + uint_t opcode7 = 0; /* low nibble of 3rd byte */ uint_t opcode_bytes = 1; /* @@ -2563,7 +2969,13 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) uint_t vex_X = 1; uint_t vex_B = 1; uint_t vex_W = 0; - uint_t vex_L; + uint_t vex_L = 0; + dis_gather_regs_t *vreg; + +#ifdef DIS_TEXT + /* Instruction name for BLS* family of instructions */ + char *blsinstr; +#endif size_t off; @@ -2571,8 +2983,7 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) x->d86_len = 0; x->d86_rmindex = -1; - x->d86_rex_prefix = 0; - x->d86_got_modrm = 0; + x->d86_error = 0; #ifdef DIS_TEXT x->d86_numopnds = 0; x->d86_seg_prefix = NULL; @@ -2585,8 +2996,10 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) x->d86_opnd[i].d86_mode = MODE_NONE; } #endif - x->d86_error = 0; + x->d86_rex_prefix = 0; + x->d86_got_modrm = 0; x->d86_memsize = 0; + x->d86_vsib = 0; if (cpu_mode == SIZE16) { opnd_size = SIZE16; @@ -2610,7 +3023,6 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) x->d86_check_func != NULL && x->d86_check_func(x->d86_data)) { #ifdef DIS_TEXT (void) strncpy(x->d86_mnem, ".byte\t0", OPLEN); - x->d86_mnem[OPLEN - 1] = '\0'; #endif goto done; } @@ -2785,6 +3197,10 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) dp = (instable_t *) &dis_opAVXF30F [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F38) { + dp = (instable_t *) + &dis_opAVXF30F38 + [(opcode1 << 4) | opcode2]; } else { goto error; } @@ -2794,6 +3210,14 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) dp = (instable_t *) &dis_opAVXF20F [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F3A) { + dp = (instable_t *) + &dis_opAVXF20F3A + [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F38) { + dp = (instable_t *) + &dis_opAVXF20F38 + [(opcode1 << 4) | opcode2]; } else { goto error; } @@ -2802,14 +3226,17 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) dp = (instable_t *) &dis_opAVX0F[opcode1][opcode2]; - } - } - + } + } if (vex_prefix) { - if (vex_L) - wbit = YMM_OPND; - else - wbit = XMM_OPND; + if (dp->it_vexwoxmm) { + wbit = LONG_OPND; + } else { + if (vex_L) + wbit = YMM_OPND; + else + wbit = XMM_OPND; + } } /* @@ -2836,7 +3263,6 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) if (addr_size_prefix) addr_size = SIZE32; } - /* * The pause instruction - a repz'd nop. This doesn't fit * with any of the other prefix goop added for SSE, so we'll @@ -2879,6 +3305,8 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) goto error; #endif switch (dp->it_adrmode) { + case XMMP: + break; case XMMP_66r: case XMMPRM_66r: case XMM3PM_66r: @@ -2905,11 +3333,66 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) if (dtrace_get_opcode(x, &opcode6, &opcode7) != 0) goto error; dp = (instable_t *)&dis_op0F38[(opcode6<<4)|opcode7]; + + /* + * Both crc32 and movbe have the same 3rd opcode + * byte of either 0xF0 or 0xF1, so we use another + * indirection to distinguish between the two. + */ + if (dp->it_indirect == (instable_t *)dis_op0F38F0 || + dp->it_indirect == (instable_t *)dis_op0F38F1) { + + dp = dp->it_indirect; + if (rep_prefix != 0xF2) { + /* It is movbe */ + dp++; + } + } + + /* + * The adx family of instructions (adcx and adox) + * continue the classic Intel tradition of abusing + * arbitrary prefixes without actually meaning the + * prefix bit. Therefore, if we find either the + * opnd_size_prefix or rep_prefix we end up zeroing it + * out after making our determination so as to ensure + * that we don't get confused and accidentally print + * repz prefixes and the like on these instructions. + * + * In addition, these instructions are actually much + * closer to AVX instructions in semantics. Importantly, + * they always default to having 32-bit operands. + * However, if the CPU is in 64-bit mode, then and only + * then, does it use REX.w promotes things to 64-bits + * and REX.r allows 64-bit mode to use register r8-r15. + */ + if (dp->it_indirect == (instable_t *)dis_op0F38F6) { + dp = dp->it_indirect; + if (opnd_size_prefix == 0 && + rep_prefix == 0xf3) { + /* It is adox */ + dp++; + } else if (opnd_size_prefix != 0x66 && + rep_prefix != 0) { + /* It isn't adcx */ + goto error; + } + opnd_size_prefix = 0; + rep_prefix = 0; + opnd_size = SIZE32; + if (rex_prefix & REX_W) + opnd_size = SIZE64; + } + #ifdef DIS_TEXT if (LIT_STRNEQL(dp->it_name, "INVALID")) goto error; #endif switch (dp->it_adrmode) { + case ADX: + case XMM: + break; + case RM_66r: case XMM_66r: case XMMM_66r: if (opnd_size_prefix == 0) { @@ -2933,6 +3416,11 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) } rep_prefix = 0; break; + case MOVBE: + if (rep_prefix != 0x0) { + goto error; + } + break; default: goto error; } @@ -2995,9 +3483,12 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) goto error; /* - * deal with MMX/SSE opcodes which are changed by prefixes + * Deal with MMX/SSE opcodes which are changed by prefixes. Note, we do + * need to include UNKNOWN below, as we may have instructions that + * actually have a prefix, but don't exist in any other form. */ switch (dp->it_adrmode) { + case UNKNOWN: case MMO: case MMOIMPL: case MMO3P: @@ -3056,6 +3547,59 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) } break; + case MG9: + /* + * More horribleness: the group 9 (0xF0 0xC7) instructions are + * allowed an optional prefix of 0x66 or 0xF3. This is similar + * to the SIMD business described above, but with a different + * addressing mode (and an indirect table), so we deal with it + * separately (if similarly). + * + * Intel further complicated this with the release of Ivy Bridge + * where they overloaded these instructions based on the ModR/M + * bytes. The VMX instructions have a mode of 0 since they are + * memory instructions but rdrand instructions have a mode of + * 0b11 (REG_ONLY) because they only operate on registers. While + * there are different prefix formats, for now it is sufficient + * to use a single different table. + */ + + /* + * Calculate our offset in dis_op0FC7 (the group 9 table) + */ + if ((uintptr_t)dp - (uintptr_t)dis_op0FC7 > sizeof (dis_op0FC7)) + goto error; + + off = ((uintptr_t)dp - (uintptr_t)dis_op0FC7) / + sizeof (instable_t); + + /* + * If we have a mode of 0b11 then we have to rewrite this. + */ + dtrace_get_modrm(x, &mode, ®, &r_m); + if (mode == REG_ONLY) { + dp = (instable_t *)&dis_op0FC7m3[off]; + break; + } + + /* + * Rewrite if this instruction used one of the magic prefixes. + */ + if (rep_prefix) { + if (rep_prefix == 0xf3) + dp = (instable_t *)&dis_opF30FC7[off]; + else + goto error; + rep_prefix = 0; + } else if (opnd_size_prefix) { + dp = (instable_t *)&dis_op660FC7[off]; + opnd_size_prefix = 0; + if (opnd_size == SIZE16) + opnd_size = SIZE32; + } + break; + + case MMOSH: /* * As with the "normal" SIMD instructions, the MMX @@ -3133,7 +3677,10 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) if (LIT_STRNEQL(dp->it_name, "INVALID")) goto error; (void) strlcat(x->d86_mnem, dp->it_name, OPLEN); - if (dp->it_suffix) { + if (dp->it_avxsuf && dp->it_suffix) { + (void) strlcat(x->d86_mnem, vex_W != 0 ? "q" : "d", + OPLEN); + } else if (dp->it_suffix) { char *types[] = {"", "w", "l", "q"}; if (opcode_bytes == 2 && opcode4 == 4) { /* It's a cmovx.yy. Replace the suffix x */ @@ -3222,6 +3769,27 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) x->d86_opnd_size = opnd_size = SIZE16; dtrace_get_operand(x, mode, r_m, wbit, 0); break; + case MOVBE: + opnd_size = SIZE32; + if (rex_prefix & REX_W) + opnd_size = SIZE64; + x->d86_opnd_size = opnd_size; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + wbit = WBIT(opcode7); + if (opnd_size_prefix) + x->d86_opnd_size = opnd_size = SIZE16; + if (wbit) { + /* reg -> mem */ + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 0); + dtrace_get_operand(x, mode, r_m, wbit, 1); + } else { + /* mem -> reg */ + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1); + dtrace_get_operand(x, mode, r_m, wbit, 0); + } + break; /* * imul instruction, with either 8-bit or longer immediate @@ -3235,6 +3803,7 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) /* memory or register operand to register, with 'w' bit */ case MRw: + case ADX: wbit = WBIT(opcode2); STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 0); break; @@ -3417,15 +3986,22 @@ just_mem: dtrace_get_operand(x, mode, r_m, wbit, 0); break; - case SWAPGS: + case SWAPGS_RDTSCP: if (cpu_mode == SIZE64 && mode == 3 && r_m == 0) { #ifdef DIS_TEXT (void) strncpy(x->d86_mnem, "swapgs", OPLEN); x->d86_mnem[OPLEN - 1] = '\0'; +#endif + NOMEM; + break; + } else if (mode == 3 && r_m == 1) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "rdtscp", OPLEN); #endif NOMEM; break; } + /*FALLTHROUGH*/ /* prefetch instruction - memory operand, but no memory acess */ @@ -3435,6 +4011,7 @@ just_mem: /* single memory or register operand */ case M: + case MG9: wbit = LONG_OPND; goto just_mem; @@ -3443,6 +4020,76 @@ just_mem: wbit = BYTE_OPND; goto just_mem; + case VMx: + if (mode == 3) { +#ifdef DIS_TEXT + char *vminstr; + + switch (r_m) { + case 1: + vminstr = "vmcall"; + break; + case 2: + vminstr = "vmlaunch"; + break; + case 3: + vminstr = "vmresume"; + break; + case 4: + vminstr = "vmxoff"; + break; + default: + goto error; + } + + (void) strncpy(x->d86_mnem, vminstr, OPLEN); +#else + if (r_m < 1 || r_m > 4) + goto error; +#endif + + NOMEM; + break; + } + /*FALLTHROUGH*/ + case SVM: + if (mode == 3) { +#if DIS_TEXT + char *vinstr; + + switch (r_m) { + case 0: + vinstr = "vmrun"; + break; + case 1: + vinstr = "vmmcall"; + break; + case 2: + vinstr = "vmload"; + break; + case 3: + vinstr = "vmsave"; + break; + case 4: + vinstr = "stgi"; + break; + case 5: + vinstr = "clgi"; + break; + case 6: + vinstr = "skinit"; + break; + case 7: + vinstr = "invlpga"; + break; + } + + (void) strncpy(x->d86_mnem, vinstr, OPLEN); +#endif + NOMEM; + break; + } + /*FALLTHROUGH*/ case MONITOR_MWAIT: if (mode == 3) { if (r_m == 0) { @@ -3456,6 +4103,18 @@ just_mem: #ifdef DIS_TEXT (void) strncpy(x->d86_mnem, "mwait", OPLEN); x->d86_mnem[OPLEN - 1] = '\0'; +#endif + NOMEM; + break; + } else if (r_m == 2) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "clac", OPLEN); +#endif + NOMEM; + break; + } else if (r_m == 3) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "stac", OPLEN); #endif NOMEM; break; @@ -3576,14 +4235,14 @@ just_mem: /* memory or register operand to register */ case MR: - if (vex_prefetch) { + if (vex_prefetch) x->d86_got_modrm = 1; - } wbit = LONG_OPND; STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 0); break; case RM: + case RM_66r: wbit = LONG_OPND; STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1); break; @@ -3684,13 +4343,13 @@ xmmprm: * movhps and movlhps behave similarly. */ if (mode == REG_ONLY) { - if (LIT_STRNEQL(dp->it_name, "movlps")) + if (LIT_STRNEQL(dp->it_name, "movlps")) { (void) strncpy(x->d86_mnem, "movhlps", OPLEN); x->d86_mnem[OPLEN - 1] = '\0'; - } else if (LIT_STRNEQL(dp->it_name, "movhps")) { + } else if (strcmp(dp->it_name, "movhps") == 0) { (void) strncpy(x->d86_mnem, "movlhps", OPLEN); x->d86_mnem[OPLEN - 1] = '\0'; - } + } } #endif if (dp->it_adrmode == XMMXIMPL) @@ -3705,8 +4364,8 @@ xmmprm: dtrace_get_modrm(x, &mode, ®, &r_m); #ifdef DIS_TEXT if ((LIT_STRNEQL(dp->it_name, "movlps") || - LIT_STRNEQL(dp->it_name, "movhps") || - LIT_STRNEQL(dp->it_name, "movntps")) && + LIT_STRNEQL(dp->it_name, "movhps") || + LIT_STRNEQL(dp->it_name, "movntps")) && mode == REG_ONLY) goto error; #endif @@ -4124,7 +4783,6 @@ xmmprm: } break; - /* float reg */ case F: #ifdef DIS_TEXT @@ -4162,11 +4820,31 @@ xmmprm: dtrace_get_operand(x, mode, r_m, wbit, 0); break; case VEX_RMrX: + case FMA: /* ModR/M.reg := op(VEX.vvvv, ModR/M.r/m) */ x->d86_numopnds = 3; dtrace_get_modrm(x, &mode, ®, &r_m); dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + /* + * In classic Intel fashion, the opcodes for all of the FMA + * instructions all have two possible mnemonics which vary by + * one letter, which is selected based on the value of the wbit. + * When wbit is one, they have the 'd' suffix and when 'wbit' is + * 0, they have the 's' suffix. Otherwise, the FMA instructions + * are all a standard VEX_RMrX. + */ +#ifdef DIS_TEXT + if (dp->it_adrmode == FMA) { + size_t len = strlen(dp->it_name); + (void) strncpy(x->d86_mnem, dp->it_name, OPLEN); + if (len + 1 < OPLEN) { + (void) strncpy(x->d86_mnem + len, + vex_W != 0 ? "d" : "s", OPLEN - len); + } + } +#endif + if (mode != REG_ONLY) { if ((dp == &dis_opAVXF20F[0x10]) || (dp == &dis_opAVXF30F[0x10])) { @@ -4205,6 +4883,53 @@ xmmprm: break; + case VEX_VRMrX: + /* ModR/M.reg := op(MODR/M.r/m, VEX.vvvv) */ + x->d86_numopnds = 3; + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, REG_ONLY, reg, wbit, 2); + /* + * VEX prefix uses the 1's complement form to encode the + * XMM/YMM regs + */ + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 0); + + dtrace_get_operand(x, mode, r_m, wbit, 1); + break; + + case VEX_SbVM: + /* ModR/M.reg := op(MODR/M.r/m, VSIB, VEX.vvvv) */ + x->d86_numopnds = 3; + x->d86_vsib = 1; + + /* + * All instructions that use VSIB are currently a mess. See the + * comment around the dis_gather_regs_t structure definition. + */ + + vreg = &dis_vgather[opcode2][vex_W][vex_L]; + +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, dp->it_name, OPLEN); + (void) strlcat(x->d86_mnem + strlen(dp->it_name), + vreg->dgr_suffix, OPLEN - strlen(dp->it_name)); +#endif + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, REG_ONLY, reg, vreg->dgr_arg2, 2); + /* + * VEX prefix uses the 1's complement form to encode the + * XMM/YMM regs + */ + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), vreg->dgr_arg0, + 0); + dtrace_get_operand(x, mode, r_m, vreg->dgr_arg1, 1); + break; + case VEX_RRX: /* ModR/M.rm := op(VEX.vvvv, ModR/M.reg) */ x->d86_numopnds = 3; @@ -4294,12 +5019,16 @@ L_VEX_MX: dtrace_get_operand(x, mode, r_m, wbit, 0); } else if ((dp == &dis_opAVXF30F[0xE6]) || (dp == &dis_opAVX0F[0x5][0xA]) || + (dp == &dis_opAVX660F38[0x13]) || + (dp == &dis_opAVX660F38[0x18]) || + (dp == &dis_opAVX660F38[0x19]) || (dp == &dis_opAVX660F38[0x58]) || - (dp == &dis_opAVX660F38[0x59]) || (dp == &dis_opAVX660F38[0x78]) || - (dp == &dis_opAVX660F38[0x79])) { + (dp == &dis_opAVX660F38[0x79]) || + (dp == &dis_opAVX660F38[0x59])) { /* vcvtdq2pd , */ /* or vcvtps2pd , */ + /* or vcvtph2ps , */ /* or vbroadcasts* , */ dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); dtrace_get_operand(x, mode, r_m, XMM_OPND, 0); @@ -4383,7 +5112,9 @@ L_VEX_MX: case VEX_RX: /* ModR/M.rm := op(ModR/M.reg) */ - if (dp == &dis_opAVX660F3A[0x19]) { /* vextractf128 */ + /* vextractf128 || vcvtps2ph */ + if (dp == &dis_opAVX660F3A[0x19] || + dp == &dis_opAVX660F3A[0x1d]) { x->d86_numopnds = 3; dtrace_get_modrm(x, &mode, ®, &r_m); @@ -4445,7 +5176,6 @@ L_VEX_MX: /* one byte immediate number */ dtrace_imm_opnd(x, wbit, 1, 0); break; - case VEX_RIM: /* ModR/M.rm := op(ModR/M.reg, imm) */ x->d86_numopnds = 3; @@ -4513,7 +5243,48 @@ L_VEX_RM: (void) strncpy(x->d86_mnem, "vzeroall", OPLEN); #endif break; + case BLS: { + + /* + * The BLS instructions are VEX instructions that are based on + * VEX.0F38.F3; however, they are considered special group 17 + * and like everything else, they use the bits in 3-5 of the + * MOD R/M to determine the sub instruction. Unlike many others + * like the VMX instructions, these are valid both for memory + * and register forms. + */ + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + switch (reg) { + case 1: +#ifdef DIS_TEXT + blsinstr = "blsr"; +#endif + break; + case 2: +#ifdef DIS_TEXT + blsinstr = "blsmsk"; +#endif + break; + case 3: +#ifdef DIS_TEXT + blsinstr = "blsi"; +#endif + break; + default: + goto error; + } + x->d86_numopnds = 2; +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, blsinstr, OPLEN); +#endif + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 1); + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + } /* an invalid op code */ case AM: case DM: diff --git a/bsd/dev/i386/dtrace_isa.c b/bsd/dev/i386/dtrace_isa.c index dfdeaad82..cb6795580 100644 --- a/bsd/dev/i386/dtrace_isa.c +++ b/bsd/dev/i386/dtrace_isa.c @@ -734,9 +734,9 @@ struct frame { }; uint64_t -dtrace_getarg(int arg, int aframes) +dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) { - uint64_t val; + uint64_t val = 0; struct frame *fp = (struct frame *)__builtin_frame_address(0); uintptr_t *stack; uintptr_t pc; @@ -778,7 +778,7 @@ dtrace_getarg(int arg, int aframes) x86_saved_state64_t *saved_state = saved_state64(tagged_regs); if (arg <= inreg) { - stack = (uintptr_t *)&saved_state->rdi; + stack = (uintptr_t *)(void*)&saved_state->rdi; } else { fp = (struct frame *)(saved_state->isf.rsp); stack = (uintptr_t *)&fp[1]; /* Find marshalled @@ -812,10 +812,11 @@ dtrace_getarg(int arg, int aframes) stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ load: - DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); - /* dtrace_probe arguments arg0 ... arg4 are 64bits wide */ - val = (uint64_t)(*(((uintptr_t *)stack) + arg)); - DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t), + mstate, vstate)) { + /* dtrace_probe arguments arg0 ... arg4 are 64bits wide */ + val = dtrace_load64((uint64_t)(stack + arg)); + } return (val); } diff --git a/bsd/dev/i386/fasttrap_isa.c b/bsd/dev/i386/fasttrap_isa.c index 4eeb6d140..a70039322 100644 --- a/bsd/dev/i386/fasttrap_isa.c +++ b/bsd/dev/i386/fasttrap_isa.c @@ -640,6 +640,8 @@ fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp) if (uwrite(p, &instr, 1, tp->ftt_pc) != 0) return (-1); + tp->ftt_installed = 1; + return (0); } @@ -653,11 +655,13 @@ fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp) * instruction. */ if (uread(p, &instr, 1, tp->ftt_pc) != 0) - return (0); + goto end; if (instr != FASTTRAP_INSTR) - return (0); + goto end; if (uwrite(p, &tp->ftt_instr[0], 1, tp->ftt_pc) != 0) return (-1); +end: + tp->ftt_installed = 0; return (0); } @@ -669,6 +673,7 @@ fasttrap_return_common(x86_saved_state_t *regs, user_addr_t pc, pid_t pid, x86_saved_state64_t *regs64; x86_saved_state32_t *regs32; unsigned int p_model; + int retire_tp = 1; dtrace_icookie_t cookie; @@ -708,6 +713,7 @@ fasttrap_return_common(x86_saved_state_t *regs, user_addr_t pc, pid_t pid, } for (id = tp->ftt_retids; id != NULL; id = id->fti_next) { + fasttrap_probe_t *probe = id->fti_probe; /* * If there's a branch that could act as a return site, we * need to trace it, and check here if the program counter is @@ -715,10 +721,23 @@ fasttrap_return_common(x86_saved_state_t *regs, user_addr_t pc, pid_t pid, */ if (tp->ftt_type != FASTTRAP_T_RET && tp->ftt_type != FASTTRAP_T_RET16 && - new_pc - id->fti_probe->ftp_faddr < - id->fti_probe->ftp_fsize) + new_pc - probe->ftp_faddr < probe->ftp_fsize) continue; + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least one probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } /* * Provide a hint to the stack trace functions to add the * following pc to the top of the stack since it's missing @@ -727,14 +746,14 @@ fasttrap_return_common(x86_saved_state_t *regs, user_addr_t pc, pid_t pid, cookie = dtrace_interrupt_disable(); cpu_core[CPU->cpu_id].cpuc_missing_tos = pc; if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { - dtrace_probe(dtrace_probeid_error, 0 /* state */, id->fti_probe->ftp_id, + dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id, 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); } else if (p_model == DATAMODEL_LP64) { - dtrace_probe(id->fti_probe->ftp_id, + dtrace_probe(probe->ftp_id, pc - id->fti_probe->ftp_faddr, regs64->rax, regs64->rdx, 0, 0); } else { - dtrace_probe(id->fti_probe->ftp_id, + dtrace_probe(probe->ftp_id, pc - id->fti_probe->ftp_faddr, regs32->eax, regs32->edx, 0, 0); } @@ -953,7 +972,7 @@ fasttrap_pid_probe32(x86_saved_state_t *regs) fasttrap_tracepoint_t *tp, tp_local; pid_t pid; dtrace_icookie_t cookie; - uint_t is_enabled = 0; + uint_t is_enabled = 0, retire_tp = 1; uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); @@ -1046,45 +1065,59 @@ fasttrap_pid_probe32(x86_saved_state_t *regs) if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id, 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); - } else if (id->fti_ptype == DTFTP_ENTRY) { - /* - * We note that this was an entry - * probe to help ustack() find the - * first caller. - */ - cookie = dtrace_interrupt_disable(); - DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY); - dtrace_probe(probe->ftp_id, s1, s2, - s3, s4, s5); - DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY); - dtrace_interrupt_enable(cookie); - } else if (id->fti_ptype == DTFTP_IS_ENABLED) { + } else { + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } /* - * Note that in this case, we don't - * call dtrace_probe() since it's only - * an artificial probe meant to change - * the flow of control so that it - * encounters the true probe. + * If we have at least one probe associated that + * is not a oneshot probe, don't remove the + * tracepoint */ - is_enabled = 1; - } else if (probe->ftp_argmap == NULL) { - dtrace_probe(probe->ftp_id, s0, s1, - s2, s3, s4); - } else { - uint32_t t[5]; - - fasttrap_usdt_args32(probe, regs32, - sizeof (t) / sizeof (t[0]), t); - - dtrace_probe(probe->ftp_id, t[0], t[1], - t[2], t[3], t[4]); - } + else { + retire_tp = 0; + } + if (id->fti_ptype == DTFTP_ENTRY) { + /* + * We note that this was an entry + * probe to help ustack() find the + * first caller. + */ + cookie = dtrace_interrupt_disable(); + DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY); + dtrace_probe(probe->ftp_id, s1, s2, + s3, s4, s5); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY); + dtrace_interrupt_enable(cookie); + } else if (id->fti_ptype == DTFTP_IS_ENABLED) { + /* + * Note that in this case, we don't + * call dtrace_probe() since it's only + * an artificial probe meant to change + * the flow of control so that it + * encounters the true probe. + */ + is_enabled = 1; + } else if (probe->ftp_argmap == NULL) { + dtrace_probe(probe->ftp_id, s0, s1, + s2, s3, s4); + } else { + uint32_t t[5]; - /* APPLE NOTE: Oneshot probes get one and only one chance... */ - if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { - fasttrap_tracepoint_remove(p, tp); + fasttrap_usdt_args32(probe, regs32, + sizeof (t) / sizeof (t[0]), t); + + dtrace_probe(probe->ftp_id, t[0], t[1], + t[2], t[3], t[4]); + } } } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); + } } /* @@ -1512,6 +1545,7 @@ fasttrap_pid_probe64(x86_saved_state_t *regs) pid_t pid; dtrace_icookie_t cookie; uint_t is_enabled = 0; + int retire_tp = 1; uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); @@ -1585,6 +1619,20 @@ fasttrap_pid_probe64(x86_saved_state_t *regs) for (id = tp->ftt_ids; id != NULL; id = id->fti_next) { fasttrap_probe_t *probe = id->fti_probe; + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id, 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); @@ -1624,10 +1672,9 @@ fasttrap_pid_probe64(x86_saved_state_t *regs) t[2], t[3], t[4]); } - /* APPLE NOTE: Oneshot probes get one and only one chance... */ - if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { - fasttrap_tracepoint_remove(p, tp); - } + } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); } } diff --git a/bsd/dev/i386/fbt_x86.c b/bsd/dev/i386/fbt_x86.c index 54f00f290..6553c2412 100644 --- a/bsd/dev/i386/fbt_x86.c +++ b/bsd/dev/i386/fbt_x86.c @@ -102,400 +102,8 @@ extern dtrace_provider_id_t fbt_id; extern fbt_probe_t **fbt_probetab; extern int fbt_probetab_mask; -extern int gIgnoreFBTBlacklist; /* From fbt_init */ - kern_return_t fbt_perfCallback(int, x86_saved_state_t *, uintptr_t *, __unused int); -/* - * Critical routines that must not be probed. PR_5221096, PR_5379018. - * The blacklist must be kept in alphabetic order for purposes of bsearch(). - */ - -static const char * critical_blacklist[] = -{ - "bcopy_phys", - "console_cpu_alloc", - "console_cpu_free", - "cpu_IA32e_disable", - "cpu_IA32e_enable", - "cpu_NMI_interrupt", - "cpu_control", - "cpu_data_alloc", - "cpu_desc_init", - "cpu_desc_init64", - "cpu_desc_load", - "cpu_desc_load64", - "cpu_exit_wait", - "cpu_info", - "cpu_info_count", - "cpu_init", - "cpu_interrupt", - "cpu_machine_init", - "cpu_mode_init", - "cpu_processor_alloc", - "cpu_processor_free", - "cpu_signal_handler", - "cpu_sleep", - "cpu_start", - "cpu_subtype", - "cpu_thread_alloc", - "cpu_thread_halt", - "cpu_thread_init", - "cpu_threadtype", - "cpu_to_processor", - "cpu_topology_sort", - "cpu_topology_start_cpu", - "cpu_type", - "cpuid_cpu_display", - "cpuid_extfeatures", - "handle_pending_TLB_flushes", - "hw_compare_and_store", - "machine_idle_cstate", - "mca_cpu_alloc", - "mca_cpu_init", - "ml_nofault_copy", - "pmap_cpu_alloc", - "pmap_cpu_free", - "pmap_cpu_high_map_vaddr", - "pmap_cpu_high_shared_remap", - "pmap_cpu_init", - "register_cpu_setup_func", - "unregister_cpu_setup_func", - "vstart" -}; -#define CRITICAL_BLACKLIST_COUNT (sizeof(critical_blacklist)/sizeof(critical_blacklist[0])) - -/* - * The transitive closure of entry points that can be reached from probe context. - * (Apart from routines whose names begin with dtrace_). - */ -static const char * probe_ctx_closure[] = -{ - "Debugger", - "IS_64BIT_PROCESS", - "OSCompareAndSwap", - "_disable_preemption", - "_enable_preemption", - "absolutetime_to_microtime", - "act_set_astbsd", - "ast_dtrace_on", - "ast_pending", - "clock_get_calendar_nanotime_nowait", - "copyin", - "copyin_user", - "copyinstr", - "copyout", - "copyoutstr", - "cpu_number", - "current_proc", - "current_processor", - "current_task", - "current_thread", - "debug_enter", - "find_user_regs", - "flush_tlb64", - "get_bsdtask_info", - "get_bsdthread_info", - "hw_atomic_and", - "kauth_cred_get", - "kauth_getgid", - "kauth_getuid", - "kernel_preempt_check", - "mach_absolute_time", - "max_valid_stack_address", - "ml_at_interrupt_context", - "ml_phys_write_byte_64", - "ml_phys_write_half_64", - "ml_phys_write_word_64", - "ml_set_interrupts_enabled", - "panic", - "pmap64_pde", - "pmap64_pdpt", - "pmap_find_phys", - "pmap_get_mapwindow", - "pmap_pde", - "pmap_pte", - "pmap_put_mapwindow", - "pmap_valid_page", - "prf", - "proc_is64bit", - "proc_selfname", - "psignal_lock", - "rtc_nanotime_load", - "rtc_nanotime_read", - "sdt_getargdesc", - "strlcpy", - "sync_iss_to_iks_unconditionally", - "systrace_stub", - "timer_grab" -}; -#define PROBE_CTX_CLOSURE_COUNT (sizeof(probe_ctx_closure)/sizeof(probe_ctx_closure[0])) - - -static int _cmp(const void *a, const void *b) -{ - return strncmp((const char *)a, *(const char **)b, strlen((const char *)a) + 1); -} - -static const void * bsearch( - const void *key, - const void *base0, - size_t nmemb, - size_t size, - int (*compar)(const void *, const void *)) { - - const char *base = base0; - size_t lim; - int cmp; - const void *p; - - for (lim = nmemb; lim != 0; lim >>= 1) { - p = base + (lim >> 1) * size; - cmp = (*compar)(key, p); - if (cmp == 0) - return p; - if (cmp > 0) { /* key > p: move right */ - base = (const char *)p + size; - lim--; - } /* else move left */ - } - return (NULL); -} - -/* - * Module validation - */ -static int -is_module_valid(struct modctl* ctl) -{ - ASSERT(!MOD_FBT_PROBES_PROVIDED(ctl)); - ASSERT(!MOD_FBT_INVALID(ctl)); - - if (0 == ctl->mod_address || 0 == ctl->mod_size) { - return FALSE; - } - - if (0 == ctl->mod_loaded) { - return FALSE; - } - - if (strstr(ctl->mod_modname, "CHUD") != NULL) - return FALSE; - - /* - * If the user sets this, trust they know what they are doing. - */ - if (gIgnoreFBTBlacklist) /* per boot-arg set in fbt_init() */ - return TRUE; - - /* - * These drivers control low level functions that when traced - * cause problems often in the sleep/wake paths as well as - * critical debug and panic paths. - * If somebody really wants to drill in on one of these kexts, then - * they can override blacklisting using the boot-arg above. - */ - - if (strstr(ctl->mod_modname, "AppleACPIEC") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleACPIPlatform") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleRTC") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "IOACPIFamily") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleIntelCPUPowerManagement") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleProfile") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleIntelProfile") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleEFI") != NULL) - return FALSE; - - return TRUE; -} - -/* - * FBT probe name validation - */ -static int -is_symbol_valid(const char* name) -{ - /* - * If the user set this, trust they know what they are doing. - */ - if (gIgnoreFBTBlacklist) - return TRUE; - - if (LIT_STRNSTART(name, "dtrace_") && !LIT_STRNSTART(name, "dtrace_safe_")) { - /* - * Anything beginning with "dtrace_" may be called - * from probe context unless it explitly indicates - * that it won't be called from probe context by - * using the prefix "dtrace_safe_". - */ - return FALSE; - } - - if (LIT_STRNSTART(name, "fasttrap_") || - LIT_STRNSTART(name, "fuword") || - LIT_STRNSTART(name, "suword") || - LIT_STRNEQL(name, "sprlock") || - LIT_STRNEQL(name, "sprunlock") || - LIT_STRNEQL(name, "uread") || - LIT_STRNEQL(name, "uwrite")) { - return FALSE; /* Fasttrap inner-workings. */ - } - - if (LIT_STRNSTART(name, "dsmos_")) - return FALSE; /* Don't Steal Mac OS X! */ - - if (LIT_STRNSTART(name, "_dtrace")) - return FALSE; /* Shims in dtrace.c */ - - if (LIT_STRNSTART(name, "chud")) - return FALSE; /* Professional courtesy. */ - - if (LIT_STRNSTART(name, "hibernate_")) - return FALSE; /* Let sleeping dogs lie. */ - - if (LIT_STRNEQL(name, "_ZNK6OSData14getBytesNoCopyEv")) - return FALSE; /* Data::getBytesNoCopy, IOHibernateSystemWake path */ - - if (LIT_STRNEQL(name, "_ZN9IOService14newTemperatureElPS_") || /* IOService::newTemperature */ - LIT_STRNEQL(name, "_ZN9IOService26temperatureCriticalForZoneEPS_")) { /* IOService::temperatureCriticalForZone */ - return FALSE; /* Per the fire code */ - } - - /* - * Place no probes (illegal instructions) in the exception handling path! - */ - if (LIT_STRNEQL(name, "t_invop") || - LIT_STRNEQL(name, "enter_lohandler") || - LIT_STRNEQL(name, "lo_alltraps") || - LIT_STRNEQL(name, "kernel_trap") || - LIT_STRNEQL(name, "interrupt") || - LIT_STRNEQL(name, "i386_astintr")) { - return FALSE; - } - - if (LIT_STRNEQL(name, "current_thread") || - LIT_STRNEQL(name, "ast_pending") || - LIT_STRNEQL(name, "fbt_perfCallback") || - LIT_STRNEQL(name, "machine_thread_get_kern_state") || - LIT_STRNEQL(name, "get_threadtask") || - LIT_STRNEQL(name, "ml_set_interrupts_enabled") || - LIT_STRNEQL(name, "dtrace_invop") || - LIT_STRNEQL(name, "fbt_invop") || - LIT_STRNEQL(name, "sdt_invop") || - LIT_STRNEQL(name, "max_valid_stack_address")) { - return FALSE; - } - - /* - * Voodoo. - */ - if (LIT_STRNSTART(name, "machine_stack_") || - LIT_STRNSTART(name, "mapping_") || - LIT_STRNEQL(name, "tmrCvt") || - - LIT_STRNSTART(name, "tsc_") || - - LIT_STRNSTART(name, "pmCPU") || - LIT_STRNEQL(name, "pmKextRegister") || - LIT_STRNEQL(name, "pmMarkAllCPUsOff") || - LIT_STRNEQL(name, "pmSafeMode") || - LIT_STRNEQL(name, "pmTimerSave") || - LIT_STRNEQL(name, "pmTimerRestore") || - LIT_STRNEQL(name, "pmUnRegister") || - LIT_STRNSTART(name, "pms") || - LIT_STRNEQL(name, "power_management_init") || - LIT_STRNSTART(name, "usimple_") || - LIT_STRNSTART(name, "lck_spin_lock") || - LIT_STRNSTART(name, "lck_spin_unlock") || - - LIT_STRNSTART(name, "rtc_") || - LIT_STRNSTART(name, "_rtc_") || - LIT_STRNSTART(name, "rtclock_") || - LIT_STRNSTART(name, "clock_") || - LIT_STRNSTART(name, "absolutetime_to_") || - LIT_STRNEQL(name, "setPop") || - LIT_STRNEQL(name, "nanoseconds_to_absolutetime") || - LIT_STRNEQL(name, "nanotime_to_absolutetime") || - - LIT_STRNSTART(name, "etimer_") || - - LIT_STRNSTART(name, "commpage_") || - LIT_STRNSTART(name, "pmap_") || - LIT_STRNSTART(name, "ml_") || - LIT_STRNSTART(name, "PE_") || - LIT_STRNEQL(name, "kprintf") || - LIT_STRNSTART(name, "lapic_") || - LIT_STRNSTART(name, "act_machine") || - LIT_STRNSTART(name, "acpi_") || - LIT_STRNSTART(name, "pal_")){ - return FALSE; - } - - /* - * Avoid machine_ routines. PR_5346750. - */ - if (LIT_STRNSTART(name, "machine_")) - return FALSE; - - if (LIT_STRNEQL(name, "handle_pending_TLB_flushes")) - return FALSE; - - /* - * Place no probes on critical routines. PR_5221096 - */ - if (bsearch( name, critical_blacklist, CRITICAL_BLACKLIST_COUNT, sizeof(name), _cmp ) != NULL) - return FALSE; - - /* - * Place no probes that could be hit in probe context. - */ - if (bsearch( name, probe_ctx_closure, PROBE_CTX_CLOSURE_COUNT, sizeof(name), _cmp ) != NULL) { - return FALSE; - } - - /* - * Place no probes that could be hit on the way to the debugger. - */ - if (LIT_STRNSTART(name, "kdp_") || - LIT_STRNSTART(name, "kdb_") || - LIT_STRNSTART(name, "debug_") || - LIT_STRNEQL(name, "Debugger") || - LIT_STRNEQL(name, "Call_DebuggerC") || - LIT_STRNEQL(name, "lock_debugger") || - LIT_STRNEQL(name, "unlock_debugger") || - LIT_STRNEQL(name, "packA") || - LIT_STRNEQL(name, "unpackA") || - LIT_STRNEQL(name, "SysChoked")) { - return FALSE; - } - - - /* - * Place no probes that could be hit on the way to a panic. - */ - if (NULL != strstr(name, "panic_") || - LIT_STRNEQL(name, "panic") || - LIT_STRNEQL(name, "preemption_underflow_panic")) { - return FALSE; - } - - return TRUE; -} - int fbt_invop(uintptr_t addr, uintptr_t *state, uintptr_t rval) { @@ -630,7 +238,9 @@ fbt_perfCallback( retval = KERN_FAILURE; break; } - saved_state->isf.trapno = T_PREEMPT; /* Avoid call to i386_astintr()! */ + + /* Trick trap_from_kernel into not attempting to handle pending AST_URGENT */ + saved_state->isf.trapno = T_PREEMPT; ml_set_interrupts_enabled(oldlevel); } @@ -638,9 +248,8 @@ fbt_perfCallback( return retval; } -/*ARGSUSED*/ -static void -__provide_probe_64(struct modctl *ctl, uintptr_t instrLow, uintptr_t instrHigh, char *modname, char* symbolName, machine_inst_t* symbolStart) +void +fbt_provide_probe(struct modctl *ctl, uintptr_t instrLow, uintptr_t instrHigh, char *modname, char* symbolName, machine_inst_t* symbolStart) { unsigned int j; unsigned int doenable = 0; @@ -918,10 +527,9 @@ again: goto again; } -static void -__kernel_syms_provide_module(void *arg, struct modctl *ctl) +void +fbt_provide_module_kernel_syms(struct modctl *ctl) { -#pragma unused(arg) kernel_mach_header_t *mh; struct load_command *cmd; kernel_segment_command_t *orig_ts = NULL, *orig_le = NULL; @@ -984,78 +592,9 @@ __kernel_syms_provide_module(void *arg, struct modctl *ctl) /* * We're only blacklisting functions in the kernel for now. */ - if (MOD_IS_MACH_KERNEL(ctl) && !is_symbol_valid(name)) + if (MOD_IS_MACH_KERNEL(ctl) && fbt_excluded(name)) continue; - __provide_probe_64(ctl, instrLow, instrHigh, modname, name, (machine_inst_t*)sym[i].n_value); + fbt_provide_probe(ctl, instrLow, instrHigh, modname, name, (machine_inst_t*)sym[i].n_value); } } - -static void -__user_syms_provide_module(void *arg, struct modctl *ctl) -{ -#pragma unused(arg) - char *modname; - unsigned int i; - - modname = ctl->mod_modname; - - dtrace_module_symbols_t* module_symbols = ctl->mod_user_symbols; - if (module_symbols) { - for (i=0; idtmodsyms_count; i++) { - - /* - * symbol->dtsym_addr (the symbol address) passed in from - * user space, is already slid for both kexts and kernel. - */ - dtrace_symbol_t* symbol = &module_symbols->dtmodsyms_symbols[i]; - - char* name = symbol->dtsym_name; - - /* Lop off omnipresent leading underscore. */ - if (*name == '_') - name += 1; - - /* - * We're only blacklisting functions in the kernel for now. - */ - if (MOD_IS_MACH_KERNEL(ctl) && !is_symbol_valid(name)) - continue; - - __provide_probe_64(ctl, (uintptr_t)symbol->dtsym_addr, (uintptr_t)(symbol->dtsym_addr + symbol->dtsym_size), modname, name, (machine_inst_t*)(uintptr_t)symbol->dtsym_addr); - } - } -} - -extern int dtrace_kernel_symbol_mode; - -/*ARGSUSED*/ -void -fbt_provide_module(void *arg, struct modctl *ctl) -{ - ASSERT(ctl != NULL); - ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER); - lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED); - - if (MOD_FBT_DONE(ctl)) - return; - - if (!is_module_valid(ctl)) { - ctl->mod_flags |= MODCTL_FBT_INVALID; - return; - } - - if (MOD_HAS_KERNEL_SYMBOLS(ctl)) { - __kernel_syms_provide_module(arg, ctl); - ctl->mod_flags |= MODCTL_FBT_PROBES_PROVIDED; - return; - } - - if (MOD_HAS_USERSPACE_SYMBOLS(ctl)) { - __user_syms_provide_module(arg, ctl); - ctl->mod_flags |= MODCTL_FBT_PROBES_PROVIDED; - if (MOD_FBT_PROVIDE_PRIVATE_PROBES(ctl)) - ctl->mod_flags |= MODCTL_FBT_PRIVATE_PROBES_PROVIDED; - return; - } -} diff --git a/bsd/dev/i386/km.c b/bsd/dev/i386/km.c index d276b6d95..19923cdc2 100644 --- a/bsd/dev/i386/km.c +++ b/bsd/dev/i386/km.c @@ -51,6 +51,7 @@ extern int hz; extern void cnputcusr(char); +extern void cnputsusr(char *, int); extern int cngetc(void); void kminit(void); @@ -359,7 +360,13 @@ kmoutput(struct tty *tp) (void) q_to_b(&tp->t_outq, buf, cc); for (cp = buf; cp < &buf[cc]; cp++) { /* output the buffer one charatcer at a time */ - kmputc(tp->t_dev, *cp & 0x7f); + *cp = *cp & 0x7f; + } + + if (cc > 1) { + cnputsusr((char *)buf, cc); + } else { + kmputc(tp->t_dev, *buf); } } /* diff --git a/bsd/dev/i386/sdt_x86.c b/bsd/dev/i386/sdt_x86.c index aeb7b3410..9bd151891 100644 --- a/bsd/dev/i386/sdt_x86.c +++ b/bsd/dev/i386/sdt_x86.c @@ -118,7 +118,7 @@ sdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) x86_saved_state64_t *saved_state = saved_state64(tagged_regs); if (argno <= inreg) { - stack = (uintptr_t *)&saved_state->rdi; + stack = (uintptr_t *)(void*)&saved_state->rdi; } else { fp = (struct frame *)(saved_state->isf.rsp); stack = (uintptr_t *)&fp[0]; /* Find marshalled diff --git a/bsd/dev/i386/sysctl.c b/bsd/dev/i386/sysctl.c index 1dc26017a..588f53d58 100644 --- a/bsd/dev/i386/sysctl.c +++ b/bsd/dev/i386/sysctl.c @@ -812,6 +812,13 @@ SYSCTL_UINT(_machdep_tsc_nanotime, OID_AUTO, generation, SYSCTL_NODE(_machdep, OID_AUTO, misc, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Miscellaneous x86 kernel parameters"); +#if (DEVELOPMENT || DEBUG) +extern uint32_t mp_interrupt_watchdog_events; +SYSCTL_UINT(_machdep_misc, OID_AUTO, interrupt_watchdog_events, + CTLFLAG_RW|CTLFLAG_LOCKED, &mp_interrupt_watchdog_events, 0, ""); +#endif + + SYSCTL_PROC(_machdep_misc, OID_AUTO, panic_restart_timeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0, diff --git a/bsd/dev/i386/systemcalls.c b/bsd/dev/i386/systemcalls.c index 9d7d476f9..3a8d5dffa 100644 --- a/bsd/dev/i386/systemcalls.c +++ b/bsd/dev/i386/systemcalls.c @@ -229,6 +229,12 @@ unix_syscall(x86_saved_state_t *state) uthread->uu_flag &= ~UT_NOTCANCELPT; +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_set_allocation_name(NULL); + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + if (__improbable(uthread->uu_lowpri_window)) { /* * task is marked as a low priority I/O type @@ -432,6 +438,12 @@ unix_syscall64(x86_saved_state_t *state) uthread->uu_flag &= ~UT_NOTCANCELPT; +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_set_allocation_name(NULL); + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + if (__improbable(uthread->uu_lowpri_window)) { /* * task is marked as a low priority I/O type @@ -565,6 +577,12 @@ unix_syscall_return(int error) uthread->uu_flag &= ~UT_NOTCANCELPT; +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_set_allocation_name(NULL); + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + if (uthread->uu_lowpri_window) { /* * task is marked as a low priority I/O type diff --git a/bsd/dev/i386/unix_signal.c b/bsd/dev/i386/unix_signal.c index bb073b026..1c271607f 100644 --- a/bsd/dev/i386/unix_signal.c +++ b/bsd/dev/i386/unix_signal.c @@ -53,6 +53,7 @@ #include #include #include +#include #include @@ -100,6 +101,28 @@ struct sigframe32 { user32_addr_t uctx; /* struct ucontext32 */ }; +/* + * Declare table of structure flavors and sizes for 64-bit and 32-bit processes + * for the cases of extended states (plain FP, or AVX): + */ +typedef struct { + int flavor; natural_t state_count; size_t mcontext_size; +} xstate_info_t; +static const xstate_info_t thread_state64[] = { + [FP] = { x86_FLOAT_STATE64, x86_FLOAT_STATE64_COUNT, sizeof(struct mcontext64) }, + [AVX] = { x86_AVX_STATE64, x86_AVX_STATE64_COUNT, sizeof(struct mcontext_avx64) }, +#if !defined(RC_HIDE_XNU_J137) + [AVX512] = { x86_AVX512_STATE64, x86_AVX512_STATE64_COUNT, sizeof(struct mcontext_avx512_64) } +#endif +}; +static const xstate_info_t thread_state32[] = { + [FP] = { x86_FLOAT_STATE32, x86_FLOAT_STATE32_COUNT, sizeof(struct mcontext32) }, + [AVX] = { x86_AVX_STATE32, x86_AVX_STATE32_COUNT, sizeof(struct mcontext_avx32) }, +#if !defined(RC_HIDE_XNU_J137) + [AVX512] = { x86_AVX512_STATE32, x86_AVX512_STATE32_COUNT, sizeof(struct mcontext_avx512_32) } +#endif +}; + /* * NOTE: Source and target may *NOT* overlap! * XXX: Unify with bsd/kern/kern_exit.c @@ -139,8 +162,12 @@ void sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint32_t code) { union { - struct mcontext_avx32 mctx_avx32; - struct mcontext_avx64 mctx_avx64; + struct mcontext_avx32 mctx_avx32; + struct mcontext_avx64 mctx_avx64; +#if !defined(RC_HIDE_XNU_J137) + struct mcontext_avx512_32 mctx_avx512_32; + struct mcontext_avx512_64 mctx_avx512_64; +#endif } mctx_store, *mctxp = &mctx_store; user_addr_t ua_sp; @@ -162,7 +189,7 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint struct uthread * ut; int stack_size = 0; int infostyle = UC_TRAD; - boolean_t sig_avx; + xstate_t sig_xstate; thread = current_thread(); ut = get_bsdthread_info(thread); @@ -183,7 +210,8 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint sinfo64.si_signo = sig; bzero(mctxp, sizeof(*mctxp)); - sig_avx = ml_fpu_avx_enabled(); + + sig_xstate = current_xstate(); if (proc_is64bit(p)) { x86_thread_state64_t *tstate64; @@ -195,14 +223,8 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS) goto bad; - if (sig_avx) { - flavor = x86_AVX_STATE64; - state_count = x86_AVX_STATE64_COUNT; - } - else { - flavor = x86_FLOAT_STATE64; - state_count = x86_FLOAT_STATE64_COUNT; - } + flavor = thread_state64[sig_xstate].flavor; + state_count = thread_state64[sig_xstate].state_count; state = (void *)&mctxp->mctx_avx64.fs; if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS) goto bad; @@ -236,7 +258,7 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint ua_sp -= sizeof (user64_siginfo_t); ua_sip = ua_sp; - ua_sp -= sizeof (struct mcontext_avx64); + ua_sp -= thread_state64[sig_xstate].mcontext_size; ua_mctxp = ua_sp; /* @@ -265,13 +287,13 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint uctx64.uc_stack.ss_flags |= SS_ONSTACK; uctx64.uc_link = 0; - uctx64.uc_mcsize = sig_avx ? sizeof(struct mcontext_avx64) : sizeof(struct mcontext64); + uctx64.uc_mcsize = thread_state64[sig_xstate].mcontext_size; uctx64.uc_mcontext64 = ua_mctxp; if (copyout((caddr_t)&uctx64, ua_uctxp, sizeof (uctx64))) goto bad; - if (copyout((caddr_t)&mctxp->mctx_avx64, ua_mctxp, sizeof (struct mcontext_avx64))) + if (copyout((caddr_t)&mctx_store, ua_mctxp, thread_state64[sig_xstate].mcontext_size)) goto bad; sinfo64.pad[0] = tstate64->rsp; @@ -308,15 +330,8 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS) goto bad; - if (sig_avx) { - flavor = x86_AVX_STATE32; - state_count = x86_AVX_STATE32_COUNT; - } - else { - flavor = x86_FLOAT_STATE32; - state_count = x86_FLOAT_STATE32_COUNT; - } - + flavor = thread_state32[sig_xstate].flavor; + state_count = thread_state32[sig_xstate].state_count; state = (void *)&mctxp->mctx_avx32.fs; if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS) goto bad; @@ -347,7 +362,7 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint ua_sp -= sizeof (user32_siginfo_t); ua_sip = ua_sp; - ua_sp -= sizeof (struct mcontext_avx32); + ua_sp -= thread_state32[sig_xstate].mcontext_size; ua_mctxp = ua_sp; ua_sp -= sizeof (struct sigframe32); @@ -393,14 +408,14 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint uctx32.uc_stack.ss_flags |= SS_ONSTACK; uctx32.uc_link = 0; - uctx32.uc_mcsize = sig_avx ? sizeof(struct mcontext_avx32) : sizeof(struct mcontext32); + uctx32.uc_mcsize = thread_state64[sig_xstate].mcontext_size; uctx32.uc_mcontext = CAST_DOWN_EXPLICIT(user32_addr_t, ua_mctxp); if (copyout((caddr_t)&uctx32, ua_uctxp, sizeof (uctx32))) goto bad; - if (copyout((caddr_t)&mctxp->mctx_avx32, ua_mctxp, sizeof (struct mcontext_avx32))) + if (copyout((caddr_t)&mctx_store, ua_mctxp, thread_state32[sig_xstate].mcontext_size)) goto bad; sinfo64.pad[0] = tstate32->esp; @@ -513,7 +528,8 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint * in the siginfo to the handler is supposed to only * contain the status, so we have to shift it out. */ - sinfo64.si_status = WEXITSTATUS(status_and_exitcode); + sinfo64.si_status = (WEXITSTATUS(status_and_exitcode) & 0x00FFFFFF) | (((uint32_t)(p->p_xhighbits) << 24) & 0xFF000000); + p->p_xhighbits = 0; break; } } @@ -648,8 +664,12 @@ int sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) { union { - struct mcontext_avx32 mctx_avx32; - struct mcontext_avx64 mctx_avx64; + struct mcontext_avx32 mctx_avx32; + struct mcontext_avx64 mctx_avx64; +#if !defined(RC_HIDE_XNU_J137) + struct mcontext_avx512_32 mctx_avx512_32; + struct mcontext_avx512_64 mctx_avx512_64; +#endif } mctx_store, *mctxp = &mctx_store; thread_t thread = current_thread(); @@ -663,8 +683,8 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) mach_msg_type_number_t fs_count; unsigned int fs_flavor; void * fs; - int rval = EJUSTRETURN; - boolean_t sig_avx; + int rval = EJUSTRETURN; + xstate_t sig_xstate; ut = (struct uthread *)get_bsdthread_info(thread); @@ -681,7 +701,8 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) } bzero(mctxp, sizeof(*mctxp)); - sig_avx = ml_fpu_avx_enabled(); + + sig_xstate = current_xstate(); if (proc_is64bit(p)) { struct user_ucontext64 uctx64; @@ -689,7 +710,7 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) if ((error = copyin(uap->uctx, (void *)&uctx64, sizeof (uctx64)))) return(error); - if ((error = copyin(uctx64.uc_mcontext64, (void *)&mctxp->mctx_avx64, sizeof (struct mcontext_avx64)))) + if ((error = copyin(uctx64.uc_mcontext64, (void *)mctxp, thread_state64[sig_xstate].mcontext_size))) return(error); onstack = uctx64.uc_onstack & 01; @@ -699,15 +720,8 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) ts_count = x86_THREAD_STATE64_COUNT; ts = (void *)&mctxp->mctx_avx64.ss; - if (sig_avx) { - fs_flavor = x86_AVX_STATE64; - fs_count = x86_AVX_STATE64_COUNT; - } - else { - fs_flavor = x86_FLOAT_STATE64; - fs_count = x86_FLOAT_STATE64_COUNT; - } - + fs_flavor = thread_state64[sig_xstate].flavor; + fs_count = thread_state64[sig_xstate].state_count; fs = (void *)&mctxp->mctx_avx64.fs; } else { @@ -716,7 +730,7 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) if ((error = copyin(uap->uctx, (void *)&uctx32, sizeof (uctx32)))) return(error); - if ((error = copyin(CAST_USER_ADDR_T(uctx32.uc_mcontext), (void *)&mctxp->mctx_avx32, sizeof (struct mcontext_avx32)))) + if ((error = copyin(CAST_USER_ADDR_T(uctx32.uc_mcontext), (void *)mctxp, thread_state32[sig_xstate].mcontext_size))) return(error); onstack = uctx32.uc_onstack & 01; @@ -726,15 +740,8 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) ts_count = x86_THREAD_STATE32_COUNT; ts = (void *)&mctxp->mctx_avx32.ss; - if (sig_avx) { - fs_flavor = x86_AVX_STATE32; - fs_count = x86_AVX_STATE32_COUNT; - } - else { - fs_flavor = x86_FLOAT_STATE32; - fs_count = x86_FLOAT_STATE32_COUNT; - } - + fs_flavor = thread_state32[sig_xstate].flavor; + fs_count = thread_state32[sig_xstate].state_count; fs = (void *)&mctxp->mctx_avx32.fs; } diff --git a/bsd/dev/monotonic.c b/bsd/dev/monotonic.c new file mode 100644 index 000000000..91ef1f2bd --- /dev/null +++ b/bsd/dev/monotonic.c @@ -0,0 +1,459 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include /* NULL */ +#include /* dev_t */ +#include /* must come after sys/stat.h */ +#include /* must come after sys/stat.h */ +#include +#include +#include +#include +#include + +static int mt_dev_open(dev_t dev, int flags, int devtype, struct proc *p); +static int mt_dev_close(dev_t dev, int flags, int devtype, struct proc *p); +static int mt_dev_ioctl(dev_t dev, unsigned long cmd, char *uptr, int fflag, + struct proc *p); + +static struct cdevsw mt_cdevsw = { + .d_open = mt_dev_open, + .d_close = mt_dev_close, + .d_read = eno_rdwrt, + .d_write = eno_rdwrt, + .d_ioctl = mt_dev_ioctl, + .d_stop = eno_stop, + .d_reset = eno_reset, + .d_ttys = NULL, + .d_select = eno_select, + .d_mmap = eno_mmap, + .d_strategy = eno_strat, + .d_type = 0 +}; + +/* + * Written at initialization, read-only thereafter. + */ +lck_grp_t *mt_lock_grp = NULL; + +static int mt_dev_major; +decl_lck_mtx_data(static, mt_dev_mtxs[MT_NDEVS]); +static bool mt_dev_owned[MT_NDEVS]; + +static void +mt_dev_lock(dev_t dev) +{ + lck_mtx_lock(&mt_dev_mtxs[minor(dev)]); +} + +static void +mt_dev_unlock(dev_t dev) +{ + lck_mtx_unlock(&mt_dev_mtxs[minor(dev)]); +} + +static void +mt_dev_assert_lock_held(__assert_only dev_t dev) +{ + LCK_MTX_ASSERT(&mt_dev_mtxs[minor(dev)], LCK_MTX_ASSERT_OWNED); +} + +int +mt_dev_init(void) +{ + lck_grp_attr_t *lock_grp_attr = NULL; + int devices = 0; + + lock_grp_attr = lck_grp_attr_alloc_init(); + mt_lock_grp = lck_grp_alloc_init("monotonic", lock_grp_attr); + lck_grp_attr_free(lock_grp_attr); + + mt_dev_major = cdevsw_add(-1 /* allocate a major number */, &mt_cdevsw); + if (mt_dev_major < 0) { + panic("monotonic: cdevsw_add failed: %d", mt_dev_major); + __builtin_trap(); + } + + for (int i = 0; i < MT_NDEVS; i++) { + dev_t dev; + void *dn; + int error; + + error = monotonic_devs[i].mtd_init(); + if (error) { + continue; + } + + dev = makedev(mt_dev_major, i); + dn = devfs_make_node(dev, + DEVFS_CHAR, UID_ROOT, GID_WINDOWSERVER, 0666, + monotonic_devs[i].mtd_name); + if (dn == NULL) { + panic("monotonic: devfs_make_node failed for '%s'", + monotonic_devs[i].mtd_name); + __builtin_trap(); + } + + lck_mtx_init(&mt_dev_mtxs[i], mt_lock_grp, LCK_ATTR_NULL); + + devices++; + } + + return 0; +} + +static int +mt_dev_open(dev_t dev, __unused int flags, __unused int devtype, + __unused struct proc *p) +{ + int error = 0; + + mt_dev_lock(dev); + + if (mt_dev_owned[minor(dev)]) { + error = EBUSY; + goto out; + } + + mt_dev_owned[minor(dev)] = true; + +out: + mt_dev_unlock(dev); + return error; +} + +static int +mt_dev_close(dev_t dev, __unused int flags, __unused int devtype, + __unused struct proc *p) +{ + mt_dev_lock(dev); + + assert(mt_dev_owned[minor(dev)]); + mt_dev_owned[minor(dev)] = false; + + monotonic_devs[minor(dev)].mtd_reset(); + + mt_dev_unlock(dev); + + return 0; +} + +static int +mt_ctl_add(dev_t dev, user_addr_t uptr, __unused int flags, + __unused struct proc *p) +{ + int error; + uint32_t ctr; + union monotonic_ctl_add ctl; + + mt_dev_assert_lock_held(dev); + + error = copyin(uptr, &ctl, sizeof(ctl.in)); + if (error) { + return error; + } + + error = monotonic_devs[minor(dev)].mtd_add(&ctl.in.config, &ctr); + if (error) { + return error; + } + + ctl.out.ctr = ctr; + + error = copyout(&ctl, uptr, sizeof(ctl.out)); + if (error) { + return error; + } + + return 0; +} + +static int +mt_ctl_counts(dev_t dev, user_addr_t uptr, __unused int flags, + __unused struct proc *p) +{ + int error; + uint64_t ctrs; + union monotonic_ctl_counts ctl; + + mt_dev_assert_lock_held(dev); + + error = copyin(uptr, &ctl, sizeof(ctl.in)); + if (error) { + return error; + } + + if (ctl.in.ctr_mask == 0) { + return EINVAL; + } + ctrs = __builtin_popcountll(ctl.in.ctr_mask); + + { + uint64_t counts[ctrs]; + error = monotonic_devs[minor(dev)].mtd_read(ctl.in.ctr_mask, counts); + if (error) { + return error; + } + + error = copyout(&counts, uptr, sizeof(counts)); + if (error) { + return error; + } + } + + return 0; +} + +static int +mt_ctl_enable(dev_t dev, user_addr_t uptr) +{ + int error; + union monotonic_ctl_enable ctl; + + mt_dev_assert_lock_held(dev); + + error = copyin(uptr, &ctl, sizeof(ctl)); + if (error) { + return error; + } + + monotonic_devs[minor(dev)].mtd_enable(ctl.in.enable); + + return 0; +} + +static int +mt_ctl_reset(dev_t dev) +{ + mt_dev_assert_lock_held(dev); + monotonic_devs[minor(dev)].mtd_reset(); + return 0; +} + +static int +mt_dev_ioctl(dev_t dev, unsigned long cmd, char *arg, int flags, + struct proc *p) +{ + int error; + user_addr_t uptr = *(user_addr_t *)(void *)arg; + + mt_dev_lock(dev); + + switch (cmd) { + case MT_IOC_RESET: + error = mt_ctl_reset(dev); + break; + + case MT_IOC_ADD: + error = mt_ctl_add(dev, uptr, flags, p); + break; + + case MT_IOC_ENABLE: + error = mt_ctl_enable(dev, uptr); + break; + + case MT_IOC_COUNTS: + error = mt_ctl_counts(dev, uptr, flags, p); + break; + + default: + error = ENODEV; + break; + } + + mt_dev_unlock(dev); + + return error; +} + +int thread_selfcounts(__unused struct proc *p, + struct thread_selfcounts_args *uap, __unused int *ret_out) +{ + switch (uap->type) { + case 1: { + uint64_t counts[2] = {}; + uint64_t thread_counts[MT_CORE_NFIXED]; + + mt_cur_thread_fixed_counts(thread_counts); + +#ifdef MT_CORE_INSTRS + counts[0] = thread_counts[MT_CORE_INSTRS]; +#endif /* defined(MT_CORE_INSTRS) */ + counts[1] = thread_counts[MT_CORE_CYCLES]; + + return copyout(counts, uap->buf, MIN(sizeof(counts), uap->nbytes)); + } + default: + return EINVAL; + } +} + +enum mt_sysctl { + MT_SUPPORTED, + MT_PMIS, + MT_RETROGRADE, + MT_TASK_THREAD, + MT_DEBUG, + MT_KDBG_TEST, + MT_FIX_CPU_PERF, + MT_FIX_THREAD_PERF, + MT_FIX_TASK_PERF, +}; + +static int +mt_sysctl SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg2) + uint64_t start[MT_CORE_NFIXED], end[MT_CORE_NFIXED]; + uint64_t counts[2] = {}; + + switch ((enum mt_sysctl)arg1) { + case MT_SUPPORTED: + return sysctl_io_number(req, mt_core_supported, sizeof(mt_core_supported), NULL, NULL); + case MT_PMIS: + return sysctl_io_number(req, mt_pmis, sizeof(mt_pmis), NULL, NULL); + case MT_RETROGRADE: + return sysctl_io_number(req, mt_retrograde, sizeof(mt_retrograde), NULL, NULL); + case MT_TASK_THREAD: + return sysctl_io_number(req, mt_core_supported, sizeof(mt_core_supported), NULL, NULL); + case MT_DEBUG: { + int value = mt_debug; + + int r = sysctl_io_number(req, value, sizeof(value), &value, NULL); + if (r) { + return r; + } + mt_debug = value; + + return 0; + } + case MT_KDBG_TEST: { + if (req->newptr == USER_ADDR_NULL) { + return EINVAL; + } + + int intrs_en = ml_set_interrupts_enabled(FALSE); + MT_KDBG_TMPCPU_START(0x3fff); + MT_KDBG_TMPCPU_END(0x3fff); + + MT_KDBG_TMPTH_START(0x3fff); + MT_KDBG_TMPTH_END(0x3fff); + ml_set_interrupts_enabled(intrs_en); + + return 0; + } + case MT_FIX_CPU_PERF: { + int intrs_en = ml_set_interrupts_enabled(FALSE); + mt_fixed_counts(start); + mt_fixed_counts(end); + ml_set_interrupts_enabled(intrs_en); + + goto copyout_counts; + } + case MT_FIX_THREAD_PERF: { + int intrs_en = ml_set_interrupts_enabled(FALSE); + mt_cur_thread_fixed_counts(start); + mt_cur_thread_fixed_counts(end); + ml_set_interrupts_enabled(intrs_en); + + goto copyout_counts; + } + case MT_FIX_TASK_PERF: { + int intrs_en = ml_set_interrupts_enabled(FALSE); + mt_cur_task_fixed_counts(start); + mt_cur_task_fixed_counts(end); + ml_set_interrupts_enabled(intrs_en); + + goto copyout_counts; + } + default: + return ENOENT; + } + +copyout_counts: + +#ifdef MT_CORE_INSTRS + counts[0] = end[MT_CORE_INSTRS] - start[MT_CORE_INSTRS]; +#endif /* defined(MT_CORE_INSTRS) */ + counts[1] = end[MT_CORE_CYCLES] - start[MT_CORE_CYCLES]; + + return copyout(counts, req->oldptr, MIN(req->oldlen, sizeof(counts))); +} + +SYSCTL_DECL(_kern_monotonic); +SYSCTL_NODE(_kern, OID_AUTO, monotonic, CTLFLAG_RW | CTLFLAG_LOCKED, 0, + "monotonic"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, supported, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_SUPPORTED, sizeof(int), mt_sysctl, "I", + "whether monotonic is supported"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, debug, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED, + (void *)MT_DEBUG, sizeof(int), mt_sysctl, "I", + "whether monotonic is printing debug messages"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, pmis, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_PMIS, sizeof(uint64_t), mt_sysctl, "Q", + "how many PMIs have been seen"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, retrograde_updates, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_RETROGRADE, sizeof(uint64_t), mt_sysctl, "Q", + "how many times a counter appeared to go backwards"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, task_thread_counting, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED, + (void *)MT_TASK_THREAD, sizeof(int), mt_sysctl, "I", + "task and thread counting enabled"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, kdebug_test, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_KDBG_TEST, sizeof(int), mt_sysctl, "O", + "test that kdebug integration works"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_cpu_perf, + CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_FIX_CPU_PERF, sizeof(uint64_t) * 2, mt_sysctl, "O", + "overhead of accessing the current CPU's counters"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_thread_perf, + CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_FIX_THREAD_PERF, sizeof(uint64_t) * 2, mt_sysctl, "O", + "overhead of accessing the current thread's counters"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_task_perf, + CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_FIX_TASK_PERF, sizeof(uint64_t) * 2, mt_sysctl, "O", + "overhead of accessing the current task's counters"); diff --git a/bsd/dev/munge.c b/bsd/dev/munge.c index 227720935..e44a638d6 100644 --- a/bsd/dev/munge.c +++ b/bsd/dev/munge.c @@ -536,6 +536,22 @@ munge_lwww(void *args) out_args[0] = *(volatile uint64_t*)&in_args[0]; } +void +munge_lwwwwwww(void *args) +{ + volatile uint64_t *out_args = (volatile uint64_t*)args; + volatile uint32_t *in_args = (volatile uint32_t*)args; + + out_args[7] = in_args[8]; + out_args[6] = in_args[7]; + out_args[5] = in_args[6]; + out_args[4] = in_args[5]; + out_args[3] = in_args[4]; + out_args[2] = in_args[3]; + out_args[1] = in_args[2]; + out_args[0] = *(volatile uint64_t*)&in_args[0]; +} + void munge_wwlww(void *args) { diff --git a/bsd/dev/unix_startup.c b/bsd/dev/unix_startup.c index 98cbcce99..a63db5b5f 100644 --- a/bsd/dev/unix_startup.c +++ b/bsd/dev/unix_startup.c @@ -85,7 +85,7 @@ int nbuf_headers = 0; #endif SYSCTL_INT (_kern, OID_AUTO, nbuf, CTLFLAG_RD | CTLFLAG_LOCKED, &nbuf_headers, 0, ""); -SYSCTL_INT (_kern, OID_AUTO, maxnbuf, CTLFLAG_RW | CTLFLAG_LOCKED, &max_nbuf_headers, 0, ""); +SYSCTL_INT (_kern, OID_AUTO, maxnbuf, CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_KERN, &max_nbuf_headers, 0, ""); __private_extern__ int customnbuf = 0; int serverperfmode = 0; /* Flag indicates a server boot when set */ @@ -140,7 +140,9 @@ bsd_startupearly(void) &firstaddr, size, FALSE, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_FILE), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_FILE, &bufferhdr_map); if (ret != KERN_SUCCESS) @@ -219,7 +221,9 @@ bsd_bufferinit(void) (vm_offset_t *) &mbutl, (vm_size_t) (nmbclusters * MCLBYTES), FALSE, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_MBUF), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_MBUF, &mb_map); if (ret != KERN_SUCCESS) @@ -317,6 +321,10 @@ bsd_scale_setup(int scale) if ((scale > 0) && (serverperfmode == 0)) { maxproc *= scale; maxprocperuid = (maxproc * 2) / 3; + if (scale > 2) { + maxfiles *= scale; + maxfilesperproc = maxfiles/2; + } } /* Apply server scaling rules */ if ((scale > 0) && (serverperfmode !=0)) { diff --git a/bsd/i386/_mcontext.h b/bsd/i386/_mcontext.h index 0abb4c87a..e2544d076 100644 --- a/bsd/i386/_mcontext.h +++ b/bsd/i386/_mcontext.h @@ -29,8 +29,9 @@ #ifndef __I386_MCONTEXT_H_ #define __I386_MCONTEXT_H_ +#include /* __DARWIN_UNIX03 */ #include -#include +#include #ifndef _STRUCT_MCONTEXT32 #if __DARWIN_UNIX03 @@ -50,6 +51,18 @@ _STRUCT_MCONTEXT_AVX32 _STRUCT_X86_AVX_STATE32 __fs; }; +#if !defined(RC_HIDE_XNU_J137) +#if defined(_STRUCT_X86_AVX512_STATE32) +#define _STRUCT_MCONTEXT_AVX512_32 struct __darwin_mcontext_avx512_32 +_STRUCT_MCONTEXT_AVX512_32 +{ + _STRUCT_X86_EXCEPTION_STATE32 __es; + _STRUCT_X86_THREAD_STATE32 __ss; + _STRUCT_X86_AVX512_STATE32 __fs; +}; +#endif /* _STRUCT_X86_AVX512_STATE32 */ +#endif /* RC_HIDE_XNU_J137 */ + #else /* !__DARWIN_UNIX03 */ #define _STRUCT_MCONTEXT32 struct mcontext32 _STRUCT_MCONTEXT32 @@ -67,6 +80,18 @@ _STRUCT_MCONTEXT_AVX32 _STRUCT_X86_AVX_STATE32 fs; }; +#if !defined(RC_HIDE_XNU_J137) +#if defined(_STRUCT_X86_AVX512_STATE32) +#define _STRUCT_MCONTEXT_AVX512_32 struct mcontext_avx512_32 +_STRUCT_MCONTEXT_AVX512_32 +{ + _STRUCT_X86_EXCEPTION_STATE32 es; + _STRUCT_X86_THREAD_STATE32 ss; + _STRUCT_X86_AVX512_STATE32 fs; +}; +#endif /* _STRUCT_X86_AVX512_STATE32 */ +#endif /* RC_HIDE_XNU_J137 */ + #endif /* __DARWIN_UNIX03 */ #endif /* _STRUCT_MCONTEXT32 */ @@ -88,6 +113,18 @@ _STRUCT_MCONTEXT_AVX64 _STRUCT_X86_AVX_STATE64 __fs; }; +#if !defined(RC_HIDE_XNU_J137) +#if defined(_STRUCT_X86_AVX512_STATE64) +#define _STRUCT_MCONTEXT_AVX512_64 struct __darwin_mcontext_avx512_64 +_STRUCT_MCONTEXT_AVX512_64 +{ + _STRUCT_X86_EXCEPTION_STATE64 __es; + _STRUCT_X86_THREAD_STATE64 __ss; + _STRUCT_X86_AVX512_STATE64 __fs; +}; +#endif /* _STRUCT_X86_AVX512_STATE64 */ +#endif /* RC_HIDE_XNU_J137 */ + #else /* !__DARWIN_UNIX03 */ #define _STRUCT_MCONTEXT64 struct mcontext64 _STRUCT_MCONTEXT64 @@ -105,6 +142,18 @@ _STRUCT_MCONTEXT_AVX64 _STRUCT_X86_AVX_STATE64 fs; }; +#if !defined(RC_HIDE_XNU_J137) +#if defined(_STRUCT_X86_AVX512_STATE64) +#define _STRUCT_MCONTEXT_AVX512_64 struct mcontext_avx512_64 +_STRUCT_MCONTEXT_AVX512_64 +{ + _STRUCT_X86_EXCEPTION_STATE64 es; + _STRUCT_X86_THREAD_STATE64 ss; + _STRUCT_X86_AVX512_STATE64 fs; +}; +#endif /* _STRUCT_X86_AVX512_STATE64 */ +#endif /* RC_HIDE_XNU_J137 */ + #endif /* __DARWIN_UNIX03 */ #endif /* _STRUCT_MCONTEXT64 */ diff --git a/bsd/i386/dis_tables.h b/bsd/i386/dis_tables.h index 5367b5277..6e2ec7f54 100644 --- a/bsd/i386/dis_tables.h +++ b/bsd/i386/dis_tables.h @@ -37,6 +37,13 @@ * for usage information and documentation. */ +/* + * APPLE NOTE: There is a copy of this file for userspace in + * dtrace:sys_dis_tables.h + * + * It needs to be in sync with this file. + */ + #ifdef __cplusplus extern "C" { #endif @@ -54,7 +61,7 @@ extern "C" { #define OPLEN 256 #define PFIXLEN 8 -#define NCPS 12 /* number of chars per symbol */ +#define NCPS 20 /* number of chars per symbol */ /* * data structures that must be provided to dtrace_dis86() @@ -81,6 +88,7 @@ typedef struct dis86 { uint_t d86_opnd_size; uint_t d86_addr_size; uint_t d86_got_modrm; + uint_t d86_vsib; /* Has a VSIB */ struct d86opnd d86_opnd[4]; /* up to 4 operands */ int (*d86_check_func)(void *); int (*d86_get_byte)(void *); diff --git a/bsd/i386/fasttrap_isa.h b/bsd/i386/fasttrap_isa.h index a71101a2d..c2392eb0c 100644 --- a/bsd/i386/fasttrap_isa.h +++ b/bsd/i386/fasttrap_isa.h @@ -57,6 +57,8 @@ typedef struct fasttrap_machtp { uint8_t ftmt_scale; /* branch scale */ uint8_t ftmt_segment; /* segment for memory accesses */ user_addr_t ftmt_dest; /* destination of control flow */ + uint8_t ftmt_installed:1; + uint8_t ftmt_retired:1; } fasttrap_machtp_t; #define ftt_instr ftt_mtp.ftmt_instr @@ -70,6 +72,9 @@ typedef struct fasttrap_machtp { #define ftt_scale ftt_mtp.ftmt_scale #define ftt_segment ftt_mtp.ftmt_segment #define ftt_dest ftt_mtp.ftmt_dest +#define ftt_installed ftt_mtp.ftmt_installed +#define ftt_retired ftt_mtp.ftmt_retired + #define FASTTRAP_T_COMMON 0x00 /* common case -- no emulation */ #define FASTTRAP_T_JCC 0x01 /* near and far conditional jumps */ diff --git a/bsd/kern/ast.h b/bsd/kern/ast.h index 00a8fa199..43d896fd9 100644 --- a/bsd/kern/ast.h +++ b/bsd/kern/ast.h @@ -39,6 +39,12 @@ extern void act_set_astbsd(thread_t); extern void bsd_ast(thread_t); +#define AST_KEVENT_RETURN_TO_KERNEL 0x0001 +#define AST_KEVENT_REDRIVE_THREADREQ 0x0002 + +extern void kevent_ast(thread_t thread, uint16_t bits); +extern void act_set_astkevent(thread_t thread, uint16_t bits); + #if CONFIG_DTRACE extern void ast_dtrace_on(void); #endif diff --git a/bsd/kern/bsd_init.c b/bsd/kern/bsd_init.c index 232e966ee..25395d352 100644 --- a/bsd/kern/bsd_init.c +++ b/bsd/kern/bsd_init.c @@ -119,6 +119,7 @@ #include #include /* for pseudo_inits */ #include +#include #include #include @@ -134,6 +135,7 @@ #include /* for mcache_init() */ #include /* for mbinit() */ #include /* for knote_init() */ +#include /* for eventhandler_init() */ #include /* for memorystatus_init() */ #include /* for aio_init() */ #include /* for psem_cache_init() */ @@ -167,6 +169,7 @@ #include /* for nstat_init() */ #include /* for tcp_cc_init() */ #include /* for mptcp_control_register() */ +#include /* for nwk_wq_init */ #include /* for assert() */ #include /* for init_system_override() */ @@ -240,7 +243,7 @@ int hostnamelen; char domainname[MAXDOMNAMELEN]; int domainnamelen; -char rootdevice[16]; /* device names have at least 9 chars */ +char rootdevice[DEVMAXNAMESIZE]; #if KMEMSTATS struct kmemstats kmemstats[M_LAST]; @@ -249,6 +252,9 @@ struct kmemstats kmemstats[M_LAST]; struct vnode *rootvp; int boothowto = RB_DEBUG; int minimalboot = 0; +#if CONFIG_EMBEDDED +int darkboot = 0; +#endif #if PROC_REF_DEBUG __private_extern__ int proc_ref_tracking_disabled = 0; /* disable panics on leaked proc refs across syscall boundary */ @@ -283,6 +289,9 @@ __private_extern__ vm_offset_t * execargs_cache = NULL; void bsd_exec_setup(int); +#if __arm64__ +__private_extern__ int bootarg_no64exec = 0; +#endif __private_extern__ int bootarg_vnode_cache_defeat = 0; #if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) @@ -381,6 +390,8 @@ extern int (*mountroot)(void); lck_grp_t * proc_lck_grp; lck_grp_t * proc_slock_grp; lck_grp_t * proc_fdmlock_grp; +lck_grp_t * proc_kqhashlock_grp; +lck_grp_t * proc_knhashlock_grp; lck_grp_t * proc_ucred_mlock_grp; lck_grp_t * proc_mlock_grp; lck_grp_attr_t * proc_lck_grp_attr; @@ -476,13 +487,15 @@ bsd_init(void) proc_lck_grp_attr= lck_grp_attr_alloc_init(); proc_lck_grp = lck_grp_alloc_init("proc", proc_lck_grp_attr); + #if CONFIG_FINE_LOCK_GROUPS proc_slock_grp = lck_grp_alloc_init("proc-slock", proc_lck_grp_attr); - proc_fdmlock_grp = lck_grp_alloc_init("proc-fdmlock", proc_lck_grp_attr); proc_ucred_mlock_grp = lck_grp_alloc_init("proc-ucred-mlock", proc_lck_grp_attr); proc_mlock_grp = lck_grp_alloc_init("proc-mlock", proc_lck_grp_attr); + proc_fdmlock_grp = lck_grp_alloc_init("proc-fdmlock", proc_lck_grp_attr); #endif - + proc_kqhashlock_grp = lck_grp_alloc_init("proc-kqhashlock", proc_lck_grp_attr); + proc_knhashlock_grp = lck_grp_alloc_init("proc-knhashlock", proc_lck_grp_attr); /* Allocate proc lock attribute */ proc_lck_attr = lck_attr_alloc_init(); #if 0 @@ -526,7 +539,6 @@ bsd_init(void) * Initialize the MAC Framework */ mac_policy_initbsd(); - kernproc->p_mac_enforce = 0; #if defined (__i386__) || defined (__x86_64__) /* @@ -653,6 +665,8 @@ bsd_init(void) filedesc0.fd_knlist = NULL; filedesc0.fd_knhash = NULL; filedesc0.fd_knhashmask = 0; + lck_mtx_init(&filedesc0.fd_kqhashlock, proc_kqhashlock_grp, proc_lck_attr); + lck_mtx_init(&filedesc0.fd_knhashlock, proc_knhashlock_grp, proc_lck_attr); /* Create the limits structures. */ kernproc->p_limit = &limit0; @@ -689,7 +703,9 @@ bsd_init(void) &minimum, (vm_size_t)bsd_pageable_map_size, TRUE, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_BSD), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_BSD, &bsd_pageable_map); if (ret != KERN_SUCCESS) panic("bsd_init: Failed to allocate bsd pageable map"); @@ -705,12 +721,6 @@ bsd_init(void) bsd_init_kprintf("calling bsd_bufferinit\n"); bsd_bufferinit(); - /* Initialize the execve() semaphore */ - bsd_init_kprintf("calling semaphore_create\n"); - - if (ret != KERN_SUCCESS) - panic("bsd_init: Failed to create execve semaphore"); - /* * Initialize the calendar. */ @@ -753,6 +763,10 @@ bsd_init(void) bsd_init_kprintf("calling knote_init\n"); knote_init(); + /* Initialize event handler */ + bsd_init_kprintf("calling eventhandler_init\n"); + eventhandler_init(); + /* Initialize for async IO */ bsd_init_kprintf("calling aio_init\n"); aio_init(); @@ -797,6 +811,8 @@ bsd_init(void) * until everything is ready. */ #if NETWORKING + bsd_init_kprintf("calling nwk_wq_init\n"); + nwk_wq_init(); bsd_init_kprintf("calling dlil_init\n"); dlil_init(); bsd_init_kprintf("calling proto_kpi_init\n"); @@ -812,7 +828,6 @@ bsd_init(void) flow_divert_init(); #endif /* FLOW_DIVERT */ #endif /* SOCKETS */ - kernproc->p_fd->fd_cdir = NULL; kernproc->p_fd->fd_rdir = NULL; @@ -1026,7 +1041,7 @@ bsd_init(void) mountroot_post_hook(); #if 0 /* not yet */ - consider_zone_gc(); + consider_zone_gc(FALSE); #endif bsd_init_kprintf("done\n"); @@ -1056,6 +1071,10 @@ bsdinit_task(void) mac_cred_label_associate_user(p->p_ucred); #endif + vm_init_before_launchd(); + + + bsd_init_kprintf("bsd_do_post - done"); load_init_program(p); lock_trace = 1; @@ -1167,6 +1186,11 @@ parse_bsd_args(void) minimalboot = 1; } +#if __arm64__ + /* disable 64 bit grading */ + if (PE_parse_boot_argn("-no64exec", namep, sizeof (namep))) + bootarg_no64exec = 1; +#endif /* disable vnode_cache_is_authorized() by setting vnode_cache_defeat */ if (PE_parse_boot_argn("-vnode_cache_defeat", namep, sizeof (namep))) @@ -1204,6 +1228,18 @@ parse_bsd_args(void) #endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ +#if CONFIG_EMBEDDED + /* + * The darkboot flag is specified by the bootloader and is stored in + * boot_args->bootFlags. This flag is available starting revision 2. + */ + boot_args *args = (boot_args *) PE_state.bootArgs; + if ((args != NULL) && (args->Revision >= kBootArgsRevision2)) { + darkboot = (args->bootFlags & kBootFlagsDarkBoot) ? 1 : 0; + } else { + darkboot = 0; + } +#endif #if PROC_REF_DEBUG if (PE_parse_boot_argn("-disable_procref_tracking", namep, sizeof(namep))) { diff --git a/bsd/kern/bsd_stubs.c b/bsd/kern/bsd_stubs.c index 85931c3e9..7883a1b75 100644 --- a/bsd/kern/bsd_stubs.c +++ b/bsd/kern/bsd_stubs.c @@ -301,24 +301,23 @@ cdevsw_add_with_bdev(int index, struct cdevsw * csw, int bdev) } int -cdevsw_setkqueueok(int index, struct cdevsw * csw, int use_offset) +cdevsw_setkqueueok(int maj, struct cdevsw * csw, int extra_flags) { struct cdevsw * devsw; uint64_t flags = CDEVSW_SELECT_KQUEUE; - if (index < 0 || index >= nchrdev) - return (-1); + if (maj < 0 || maj >= nchrdev) { + return -1; + } - devsw = &cdevsw[index]; + devsw = &cdevsw[maj]; if ((memcmp((char *)devsw, (char *)csw, sizeof(struct cdevsw)) != 0)) { - return (-1); + return -1; } - if (use_offset) { - flags |= CDEVSW_USE_OFFSET; - } + flags |= extra_flags; - cdevsw_flags[index] = flags; + cdevsw_flags[maj] = flags; return 0; } diff --git a/bsd/kern/decmpfs.c b/bsd/kern/decmpfs.c index bb4b8c2ff..ebca25271 100644 --- a/bsd/kern/decmpfs.c +++ b/bsd/kern/decmpfs.c @@ -68,6 +68,7 @@ UNUSED_SYMBOL(decmpfs_validate_compressed_file) #include #include #include +#include #pragma mark --- debugging --- @@ -196,7 +197,7 @@ _free(char *ret, __unused int type, const char *file, int line) static lck_grp_t *decmpfs_lockgrp; -static decmpfs_registration * decompressors[CMP_MAX]; /* the registered compressors */ +SECURITY_READ_ONLY_EARLY(static decmpfs_registration *) decompressors[CMP_MAX]; /* the registered compressors */ static lck_rw_t * decompressorsLock; static int decompress_channel; /* channel used by decompress_file to wake up waiters */ static lck_mtx_t *decompress_channel_mtx; @@ -211,10 +212,10 @@ static void * _func_from_offset(uint32_t type, uintptr_t offset) { /* get the function at the given offset in the registration for the given type */ - decmpfs_registration *reg = decompressors[type]; - char *regChar = (char*)reg; - char *func = ®Char[offset]; - void **funcPtr = (void**)func; + const decmpfs_registration *reg = decompressors[type]; + const char *regChar = (const char*)reg; + const char *func = ®Char[offset]; + void * const * funcPtr = (void * const *) func; switch (reg->decmpfs_registration) { case DECMPFS_REGISTRATION_VERSION_V1: @@ -948,13 +949,13 @@ decmpfs_hides_xattr(vfs_context_t ctx, decmpfs_cnode *cp, const char *xattr) #pragma mark --- registration/validation routines --- -static inline int registration_valid(decmpfs_registration *registration) +static inline int registration_valid(const decmpfs_registration *registration) { return registration && ((registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V1) || (registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V3)); } errno_t -register_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration) +register_decmpfs_decompressor(uint32_t compression_type, const decmpfs_registration *registration) { /* called by kexts to register decompressors */ @@ -1375,7 +1376,7 @@ decmpfs_read_compressed(struct vnop_read_args *ap, int *is_compressed, decmpfs_c } /* create the upl */ - kr = ubc_create_upl(vp, curUplPos, curUplSize, &upl, &pli, UPL_SET_LITE); + kr = ubc_create_upl_kernel(vp, curUplPos, curUplSize, &upl, &pli, UPL_SET_LITE, VM_KERN_MEMORY_FILE); if (kr != KERN_SUCCESS) { ErrorLogWithPath("ubc_create_upl error %d\n", (int)kr); err = EINVAL; @@ -1842,7 +1843,7 @@ out: return err; } -static decmpfs_registration Type1Reg = +SECURITY_READ_ONLY_EARLY(static decmpfs_registration) Type1Reg = { .decmpfs_registration = DECMPFS_REGISTRATION_VERSION, .validate = decmpfs_validate_compressed_file_Type1, diff --git a/bsd/kern/imageboot.c b/bsd/kern/imageboot.c index 6ba54a69f..493d740f8 100644 --- a/bsd/kern/imageboot.c +++ b/bsd/kern/imageboot.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -54,7 +55,7 @@ extern struct filedesc filedesc0; extern int (*mountroot)(void); -extern char rootdevice[]; +extern char rootdevice[DEVMAXNAMESIZE]; #define DEBUG_IMAGEBOOT 0 @@ -64,7 +65,9 @@ extern char rootdevice[]; #define DBG_TRACE(...) do {} while(0) #endif -extern int di_root_image(const char *path, char devname[], dev_t *dev_p); +extern int di_root_image(const char *path, char *devname, size_t devsz, dev_t *dev_p); +extern int di_root_ramfile_buf(void *buf, size_t bufsz, char *devname, size_t devsz, dev_t *dev_p); + static boolean_t imageboot_setup_new(void); #define kIBFilePrefix "file://" @@ -150,7 +153,7 @@ imageboot_mount_image(const char *root_path, int height) vnode_t newdp; mount_t new_rootfs; - error = di_root_image(root_path, rootdevice, &dev); + error = di_root_image(root_path, rootdevice, DEVMAXNAMESIZE, &dev); if (error) { panic("%s: di_root_image failed: %d\n", __FUNCTION__, error); } @@ -259,7 +262,7 @@ read_file(const char *path, void **bufp, size_t *bufszp) proc_t p = vfs_context_proc(ctx); kauth_cred_t kerncred = vfs_context_ucred(ctx); - NDINIT(&ndp, LOOKUP, OP_OPEN, LOCKLEAF, UIO_SYSSPACE, path, ctx); + NDINIT(&ndp, LOOKUP, OP_OPEN, LOCKLEAF, UIO_SYSSPACE, CAST_USER_ADDR_T(path), ctx); if ((err = namei(&ndp)) != 0) { AUTHPRNT("namei failed (%s)", path); goto out; @@ -493,7 +496,7 @@ validate_root_image(const char *root_path, void *chunklist) /* * Open the DMG */ - NDINIT(&ndp, LOOKUP, OP_OPEN, LOCKLEAF, UIO_SYSSPACE, root_path, ctx); + NDINIT(&ndp, LOOKUP, OP_OPEN, LOCKLEAF, UIO_SYSSPACE, CAST_USER_ADDR_T(root_path), ctx); if ((err = namei(&ndp)) != 0) { AUTHPRNT("namei failed (%s)", root_path); goto out; @@ -831,6 +834,96 @@ auth_imgboot_test(proc_t __unused ap, struct auth_imgboot_test_args *uap, int32_ } #endif +/* + * Attach the image at 'path' as a ramdisk and mount it as our new rootfs. + * All existing mounts are first umounted. + */ +static int +imageboot_mount_ramdisk(const char *path) +{ + int err = 0; + size_t bufsz = 0; + void *buf = NULL; + dev_t dev; + vnode_t newdp; + mount_t new_rootfs; + + /* Read our target image from disk */ + err = read_file(path, &buf, &bufsz); + if (err) { + printf("%s: failed: read_file() = %d\n", __func__, err); + goto out; + } + DBG_TRACE("%s: read '%s' sz = %lu\n", __func__, path, bufsz); + +#if CONFIG_IMGSRC_ACCESS + /* Re-add all root mounts to the mount list in the correct order... */ + mount_list_remove(rootvnode->v_mount); + for (int i = 0; i < MAX_IMAGEBOOT_NESTING; i++) { + struct vnode *vn = imgsrc_rootvnodes[i]; + if (vn) { + vnode_getalways(vn); + imgsrc_rootvnodes[i] = NULLVP; + + mount_t mnt = vn->v_mount; + mount_lock(mnt); + mnt->mnt_flag |= MNT_ROOTFS; + mount_list_add(mnt); + mount_unlock(mnt); + + vnode_rele(vn); + vnode_put(vn); + } + } + mount_list_add(rootvnode->v_mount); +#endif + + /* ... and unmount everything */ + vnode_get_and_drop_always(rootvnode); + filedesc0.fd_cdir = NULL; + rootvnode = NULL; + vfs_unmountall(); + + /* Attach the ramfs image ... */ + err = di_root_ramfile_buf(buf, bufsz, rootdevice, DEVMAXNAMESIZE, &dev); + if (err) { + printf("%s: failed: di_root_ramfile_buf() = %d\n", __func__, err); + goto out; + } + + /* ... and mount it */ + rootdev = dev; + mountroot = NULL; + err = vfs_mountroot(); + if (err) { + printf("%s: failed: vfs_mountroot() = %d\n", __func__, err); + goto out; + } + + /* Switch to new root vnode */ + if (VFS_ROOT(TAILQ_LAST(&mountlist,mntlist), &newdp, vfs_context_kernel())) { + panic("%s: cannot find root vnode", __func__); + } + rootvnode = newdp; + rootvnode->v_flag |= VROOT; + new_rootfs = rootvnode->v_mount; + mount_lock(new_rootfs); + new_rootfs->mnt_flag |= MNT_ROOTFS; + mount_unlock(new_rootfs); + + vnode_ref(newdp); + vnode_put(newdp); + filedesc0.fd_cdir = newdp; + + DBG_TRACE("%s: root switched\n", __func__); + +out: + if (err) { + kfree_safe(buf); + } + return err; +} + static boolean_t imageboot_setup_new() { @@ -839,10 +932,16 @@ imageboot_setup_new() int height = 0; boolean_t done = FALSE; boolean_t auth_root = FALSE; + boolean_t ramdisk_root = FALSE; MALLOC_ZONE(root_path, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK); assert(root_path != NULL); + unsigned imgboot_arg; + if (PE_parse_boot_argn("-rootdmg-ramdisk", &imgboot_arg, sizeof(imgboot_arg))) { + ramdisk_root = TRUE; + } + if (PE_parse_boot_argn(IMAGEBOOT_CONTAINER_ARG, root_path, MAXPATHLEN) == TRUE) { printf("%s: container image url is %s\n", __FUNCTION__, root_path); error = imageboot_mount_image(root_path, height); @@ -871,36 +970,41 @@ imageboot_setup_new() } #endif - if (auth_root) { - /* Copy the path to use locally */ - char *path_alloc = kalloc(MAXPATHLEN); - if (path_alloc == NULL) { - panic("imageboot path allocation failed\n"); - } - - char *path = path_alloc; - strlcpy(path, root_path, MAXPATHLEN); + /* Make a copy of the path to URL-decode */ + char *path_alloc = kalloc(MAXPATHLEN); + if (path_alloc == NULL) { + panic("imageboot path allocation failed\n"); + } + char *path = path_alloc; - size_t len = strlen(kIBFilePrefix); - if (strncmp(kIBFilePrefix, path, len) == 0) { - /* its a URL - remove the file:// prefix and percent-decode */ - path += len; - url_decode(path); - } + size_t len = strlen(kIBFilePrefix); + strlcpy(path, root_path, MAXPATHLEN); + if (strncmp(kIBFilePrefix, path, len) == 0) { + /* its a URL - remove the file:// prefix and percent-decode */ + path += len; + url_decode(path); + } + if (auth_root) { AUTHDBG("authenticating root image at %s", path); error = authenticate_root(path); if (error) { panic("root image authentication failed (err = %d)\n", error); } AUTHDBG("successfully authenticated %s", path); + } - kfree_safe(path_alloc); + if (ramdisk_root) { + error = imageboot_mount_ramdisk(path); + } else { + error = imageboot_mount_image(root_path, height); } - error = imageboot_mount_image(root_path, height); - if (error != 0) { - panic("Failed to mount root image."); + kfree_safe(path_alloc); + + if (error) { + panic("Failed to mount root image (err=%d, auth=%d, ramdisk=%d)\n", + error, auth_root, ramdisk_root); } if (auth_root) { diff --git a/bsd/kern/kdebug.c b/bsd/kern/kdebug.c index d1a1b023b..978b02c49 100644 --- a/bsd/kern/kdebug.c +++ b/bsd/kern/kdebug.c @@ -20,8 +20,6 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#include - #include #include #include @@ -48,6 +46,7 @@ #include #include #include +#include #endif #include @@ -165,46 +164,46 @@ static typefilter_t typefilter_create(void) static void typefilter_deallocate(typefilter_t tf) { - assert(tf); + assert(tf != NULL); assert(tf != kdbg_typefilter); kmem_free(kernel_map, (vm_offset_t)tf, TYPEFILTER_ALLOC_SIZE); } static void typefilter_copy(typefilter_t dst, typefilter_t src) { - assert(src); - assert(dst); + assert(src != NULL); + assert(dst != NULL); memcpy(dst, src, KDBG_TYPEFILTER_BITMAP_SIZE); } static void typefilter_reject_all(typefilter_t tf) { - assert(tf); + assert(tf != NULL); memset(tf, 0, KDBG_TYPEFILTER_BITMAP_SIZE); } static void typefilter_allow_class(typefilter_t tf, uint8_t class) { - assert(tf); + assert(tf != NULL); const uint32_t BYTES_PER_CLASS = 256 / 8; // 256 subclasses, 1 bit each memset(&tf[class * BYTES_PER_CLASS], 0xFF, BYTES_PER_CLASS); } static void typefilter_allow_csc(typefilter_t tf, uint16_t csc) { - assert(tf); + assert(tf != NULL); setbit(tf, csc); } -static boolean_t typefilter_is_debugid_allowed(typefilter_t tf, uint32_t id) +static bool typefilter_is_debugid_allowed(typefilter_t tf, uint32_t id) { - assert(tf); + assert(tf != NULL); return isset(tf, KDBG_EXTRACT_CSC(id)); } static mach_port_t typefilter_create_memory_entry(typefilter_t tf) { - assert(tf); + assert(tf != NULL); mach_port_t memory_entry = MACH_PORT_NULL; memory_object_size_t size = TYPEFILTER_ALLOC_SIZE; @@ -232,7 +231,22 @@ int cpu_number(void); /* XXX include path broken */ void commpage_update_kdebug_state(void); /* XXX sign */ extern int log_leaks; -extern boolean_t kdebug_serial; + +/* + * This flag is for testing purposes only -- it's highly experimental and tools + * have not been updated to support it. + */ +static bool kdbg_continuous_time = false; + +static inline uint64_t +kdbg_timestamp(void) +{ + if (kdbg_continuous_time) { + return mach_continuous_time(); + } else { + return mach_absolute_time(); + } +} #if KDEBUG_MOJO_TRACE #include @@ -253,7 +267,7 @@ static int kdbg_setpid(kd_regtype *); static void kdbg_thrmap_init(void); static int kdbg_reinit(boolean_t); static int kdbg_bootstrap(boolean_t); -static int kdbg_test(void); +static int kdbg_test(size_t flavor); static int kdbg_write_v1_header(boolean_t write_thread_map, vnode_t vp, vfs_context_t ctx); static int kdbg_write_thread_map(vnode_t vp, vfs_context_t ctx); @@ -271,7 +285,6 @@ static kd_threadmap *kdbg_thrmap_init_internal(unsigned int count, unsigned int *mapcount); static boolean_t kdebug_current_proc_enabled(uint32_t debugid); -boolean_t kdebug_debugid_enabled(uint32_t debugid); static errno_t kdebug_check_trace_string(uint32_t debugid, uint64_t str_id); int kdbg_write_v3_header(user_addr_t, size_t *, int); @@ -297,10 +310,23 @@ extern void IOSleep(int); unsigned int kdebug_enable = 0; /* A static buffer to record events prior to the start of regular logging */ -#define KD_EARLY_BUFFER_MAX 64 -static kd_buf kd_early_buffer[KD_EARLY_BUFFER_MAX]; -static int kd_early_index = 0; -static boolean_t kd_early_overflow = FALSE; + +#define KD_EARLY_BUFFER_SIZE (16 * 1024) +#define KD_EARLY_BUFFER_NBUFS (KD_EARLY_BUFFER_SIZE / sizeof(kd_buf)) +#if CONFIG_EMBEDDED +/* + * On embedded, the space for this is carved out by osfmk/arm/data.s -- clang + * has problems aligning to greater than 4K. + */ +extern kd_buf kd_early_buffer[KD_EARLY_BUFFER_NBUFS]; +#else /* CONFIG_EMBEDDED */ +__attribute__((aligned(KD_EARLY_BUFFER_SIZE))) +static kd_buf kd_early_buffer[KD_EARLY_BUFFER_NBUFS]; +#endif /* !CONFIG_EMBEDDED */ + +static unsigned int kd_early_index = 0; +static bool kd_early_overflow = false; +static bool kd_early_done = false; #define SLOW_NOLOG 0x01 #define SLOW_CHECKS 0x02 @@ -329,8 +355,10 @@ struct kd_storage { kd_buf kds_records[EVENTS_PER_STORAGE_UNIT]; }; -#define MAX_BUFFER_SIZE (1024 * 1024 * 128) -#define N_STORAGE_UNITS_PER_BUFFER (MAX_BUFFER_SIZE / sizeof(struct kd_storage)) +#define MAX_BUFFER_SIZE (1024 * 1024 * 128) +#define N_STORAGE_UNITS_PER_BUFFER (MAX_BUFFER_SIZE / sizeof(struct kd_storage)) +static_assert(N_STORAGE_UNITS_PER_BUFFER <= 0x7ff, + "shoudn't overflow kds_ptr.offset"); struct kd_storage_buffers { struct kd_storage *kdsb_addr; @@ -339,10 +367,10 @@ struct kd_storage_buffers { #define KDS_PTR_NULL 0xffffffff struct kd_storage_buffers *kd_bufs = NULL; -int n_storage_units = 0; -int n_storage_buffers = 0; -int n_storage_threshold = 0; -int kds_waiter = 0; +int n_storage_units = 0; +unsigned int n_storage_buffers = 0; +int n_storage_threshold = 0; +int kds_waiter = 0; #pragma pack(0) struct kd_bufinfo { @@ -460,7 +488,11 @@ static uint32_t kdbg_cpu_count(boolean_t early_trace) { if (early_trace) { +#if CONFIG_EMBEDDED + return ml_get_cpu_count(); +#else return max_ncpus; +#endif } host_basic_info_data_t hinfo; @@ -471,6 +503,41 @@ kdbg_cpu_count(boolean_t early_trace) } #if MACH_ASSERT +#if CONFIG_EMBEDDED +static boolean_t +kdbg_iop_list_is_valid(kd_iop_t* iop) +{ + if (iop) { + /* Is list sorted by cpu_id? */ + kd_iop_t* temp = iop; + do { + assert(!temp->next || temp->next->cpu_id == temp->cpu_id - 1); + assert(temp->next || (temp->cpu_id == kdbg_cpu_count(FALSE) || temp->cpu_id == kdbg_cpu_count(TRUE))); + } while ((temp = temp->next)); + + /* Does each entry have a function and a name? */ + temp = iop; + do { + assert(temp->callback.func); + assert(strlen(temp->callback.iop_name) < sizeof(temp->callback.iop_name)); + } while ((temp = temp->next)); + } + + return TRUE; +} + +static boolean_t +kdbg_iop_list_contains_cpu_id(kd_iop_t* list, uint32_t cpu_id) +{ + while (list) { + if (list->cpu_id == cpu_id) + return TRUE; + list = list->next; + } + + return FALSE; +} +#endif /* CONFIG_EMBEDDED */ #endif /* MACH_ASSERT */ static void @@ -488,6 +555,10 @@ kdbg_set_tracing_enabled(boolean_t enabled, uint32_t trace_type) int s = ml_set_interrupts_enabled(FALSE); lck_spin_lock(kds_spin_lock); if (enabled) { + /* + * The oldest valid time is now; reject old events from IOPs. + */ + kd_ctrl_page.oldest_time = kdbg_timestamp(); kdebug_enable |= trace_type; kd_ctrl_page.kdebug_slowcheck &= ~SLOW_NOLOG; kd_ctrl_page.enabled = 1; @@ -576,10 +647,10 @@ enable_wrap(uint32_t old_slowcheck, boolean_t lostevents) static int create_buffers(boolean_t early_trace) { - int i; - int p_buffer_size; - int f_buffer_size; - int f_buffers; + unsigned int i; + unsigned int p_buffer_size; + unsigned int f_buffer_size; + unsigned int f_buffers; int error = 0; /* @@ -591,6 +662,9 @@ create_buffers(boolean_t early_trace) */ kd_ctrl_page.kdebug_iops = kd_iops; +#if CONFIG_EMBEDDED + assert(kdbg_iop_list_is_valid(kd_ctrl_page.kdebug_iops)); +#endif /* * If the list is valid, it is sorted, newest -> oldest. Each iop entry @@ -675,7 +749,7 @@ create_buffers(boolean_t early_trace) bzero((char *)kdbip, sizeof(struct kd_bufinfo) * kd_ctrl_page.kdebug_cpus); - for (i = 0; i < (int)kd_ctrl_page.kdebug_cpus; i++) { + for (i = 0; i < kd_ctrl_page.kdebug_cpus; i++) { kdbip[i].kd_list_head.raw = KDS_PTR_NULL; kdbip[i].kd_list_tail.raw = KDS_PTR_NULL; kdbip[i].kd_lostevents = FALSE; @@ -696,7 +770,7 @@ out: static void delete_buffers(void) { - int i; + unsigned int i; if (kd_bufs) { for (i = 0; i < n_storage_buffers; i++) { @@ -858,7 +932,7 @@ allocate_storage_unit(int cpu) kd_ctrl_page.oldest_time = oldest_ts; kd_ctrl_page.kdebug_flags |= KDBG_WRAPPED; } - kdsp_actual->kds_timestamp = mach_absolute_time(); + kdsp_actual->kds_timestamp = kdbg_timestamp(); kdsp_actual->kds_next.raw = KDS_PTR_NULL; kdsp_actual->kds_bufcnt = 0; kdsp_actual->kds_readlast = 0; @@ -976,13 +1050,23 @@ kernel_debug_enter( } } - if (kd_ctrl_page.kdebug_flags & KDBG_WRAPPED) { - if (timestamp < kd_ctrl_page.oldest_time) { - goto out1; - } +record_event: + if (timestamp < kd_ctrl_page.oldest_time) { + goto out1; } -record_event: +#if CONFIG_EMBEDDED + /* + * When start_kern_tracing is called by the kernel to trace very + * early kernel events, it saves data to a secondary buffer until + * it is possible to initialize ktrace, and then dumps the events + * into the ktrace buffer using this method. In this case, iops will + * be NULL, and the coreid will be zero. It is not possible to have + * a valid IOP coreid of zero, so pass if both iops is NULL and coreid + * is zero. + */ + assert(kdbg_iop_list_contains_cpu_id(kd_ctrl_page.kdebug_iops, coreid) || (kd_ctrl_page.kdebug_iops == NULL && coreid == 0)); +#endif disable_preemption(); @@ -1004,8 +1088,10 @@ retry_q: if (kds_raw.raw != KDS_PTR_NULL) { kdsp_actual = POINTER_FROM_KDS_PTR(kds_raw); bindx = kdsp_actual->kds_bufindx; - } else + } else { kdsp_actual = NULL; + bindx = EVENTS_PER_STORAGE_UNIT; + } if (kdsp_actual == NULL || bindx >= EVENTS_PER_STORAGE_UNIT) { if (allocate_storage_unit(coreid) == FALSE) { @@ -1138,7 +1224,7 @@ record_event: #if KDEBUG_MOJO_TRACE if (kdebug_enable & KDEBUG_ENABLE_SERIAL) kdebug_serial_print(cpu, debugid, - mach_absolute_time() & KDBG_TIMESTAMP_MASK, + kdbg_timestamp() & KDBG_TIMESTAMP_MASK, arg1, arg2, arg3, arg4, arg5); #endif @@ -1148,9 +1234,11 @@ retry_q: if (kds_raw.raw != KDS_PTR_NULL) { kdsp_actual = POINTER_FROM_KDS_PTR(kds_raw); bindx = kdsp_actual->kds_bufindx; - } else + } else { kdsp_actual = NULL; - + bindx = EVENTS_PER_STORAGE_UNIT; + } + if (kdsp_actual == NULL || bindx >= EVENTS_PER_STORAGE_UNIT) { if (allocate_storage_unit(cpu) == FALSE) { /* @@ -1161,7 +1249,7 @@ retry_q: } goto retry_q; } - now = mach_absolute_time() & KDBG_TIMESTAMP_MASK; + now = kdbg_timestamp() & KDBG_TIMESTAMP_MASK; if ( !OSCompareAndSwap(bindx, bindx + 1, &kdsp_actual->kds_bufindx)) goto retry_q; @@ -1300,17 +1388,17 @@ kernel_debug_early( uintptr_t arg3, uintptr_t arg4) { - /* If tracing is already initialized, use it */ - if (nkdbufs) { + /* If early tracing is over, use the normal path. */ + if (kd_early_done) { KERNEL_DEBUG_CONSTANT(debugid, arg1, arg2, arg3, arg4, 0); return; } - /* Do nothing if the buffer is full or we're not on the boot cpu */ - kd_early_overflow = kd_early_index >= KD_EARLY_BUFFER_MAX; - if (kd_early_overflow || - cpu_number() != master_cpu) + /* Do nothing if the buffer is full or we're not on the boot cpu. */ + kd_early_overflow = kd_early_index >= KD_EARLY_BUFFER_NBUFS; + if (kd_early_overflow || cpu_number() != master_cpu) { return; + } kd_early_buffer[kd_early_index].debugid = debugid; kd_early_buffer[kd_early_index].timestamp = mach_absolute_time(); @@ -1323,31 +1411,33 @@ kernel_debug_early( } /* - * Transfen the contents of the temporary buffer into the trace buffers. + * Transfer the contents of the temporary buffer into the trace buffers. * Precede that by logging the rebase time (offset) - the TSC-based time (in ns) * when mach_absolute_time is set to 0. */ static void kernel_debug_early_end(void) { - int i; - - if (cpu_number() != master_cpu) + if (cpu_number() != master_cpu) { panic("kernel_debug_early_end() not call on boot processor"); + } + /* reset the current oldest time to allow early events */ + kd_ctrl_page.oldest_time = 0; + +#if !CONFIG_EMBEDDED /* Fake sentinel marking the start of kernel time relative to TSC */ - kernel_debug_enter( - 0, - TRACE_TIMESTAMPS, - 0, - (uint32_t)(tsc_rebase_abs_time >> 32), - (uint32_t)tsc_rebase_abs_time, - 0, - 0, - 0); - for (i = 0; i < kd_early_index; i++) { - kernel_debug_enter( + kernel_debug_enter(0, + TRACE_TIMESTAMPS, + 0, + (uint32_t)(tsc_rebase_abs_time >> 32), + (uint32_t)tsc_rebase_abs_time, + tsc_at_boot, 0, + 0); +#endif + for (unsigned int i = 0; i < kd_early_index; i++) { + kernel_debug_enter(0, kd_early_buffer[i].debugid, kd_early_buffer[i].timestamp, kd_early_buffer[i].arg1, @@ -1358,9 +1448,11 @@ kernel_debug_early_end(void) } /* Cut events-lost event on overflow */ - if (kd_early_overflow) - KERNEL_DEBUG_CONSTANT( - TRACE_LOST_EVENTS, 0, 0, 0, 0, 0); + if (kd_early_overflow) { + KDBG_RELEASE(TRACE_LOST_EVENTS, 1); + } + + kd_early_done = true; /* This trace marks the start of kernel tracing */ kernel_debug_string_early("early trace done"); @@ -1444,11 +1536,12 @@ kdebug_typefilter(__unused struct proc* p, vm_map_t user_map = current_map(); ret = mach_to_bsd_errno( - mach_vm_map(user_map, // target map + mach_vm_map_kernel(user_map, // target map &user_addr, // [in, out] target address TYPEFILTER_ALLOC_SIZE, // initial size 0, // mask (alignment?) VM_FLAGS_ANYWHERE, // flags + VM_KERN_MEMORY_NONE, kdbg_typefilter_memory_entry, // port (memory entry!) 0, // offset (in memory entry) FALSE, // should copy @@ -1632,13 +1725,6 @@ kdebug_current_proc_enabled(uint32_t debugid) return TRUE; } -/* - * Returns false if the debugid is disabled by filters, and true if the - * debugid is allowed to be traced. A debugid may not be traced if the - * typefilter disables its class and subclass, it's outside a range - * check, or if it's not an allowed debugid in a value check. Trace - * system events bypass this check. - */ boolean_t kdebug_debugid_enabled(uint32_t debugid) { @@ -1647,13 +1733,17 @@ kdebug_debugid_enabled(uint32_t debugid) return TRUE; } + return kdebug_debugid_explicitly_enabled(debugid); +} + +boolean_t +kdebug_debugid_explicitly_enabled(uint32_t debugid) +{ if (kd_ctrl_page.kdebug_flags & KDBG_TYPEFILTER_CHECK) { return typefilter_is_debugid_allowed(kdbg_typefilter, debugid); } else if (KDBG_EXTRACT_CLASS(debugid) == DBG_TRACE) { return TRUE; - } - - if (kd_ctrl_page.kdebug_flags & KDBG_RANGECHECK) { + } else if (kd_ctrl_page.kdebug_flags & KDBG_RANGECHECK) { if (debugid < kdlog_beg || debugid > kdlog_end) { return FALSE; } @@ -1861,12 +1951,18 @@ kdbg_reinit(boolean_t early_trace) } void -kdbg_trace_data(struct proc *proc, long *arg_pid) +kdbg_trace_data(struct proc *proc, long *arg_pid, long *arg_uniqueid) { - if (!proc) + if (!proc) { *arg_pid = 0; - else + *arg_uniqueid = 0; + } else { *arg_pid = proc->p_pid; + *arg_uniqueid = proc->p_uniqueid; + if ((uint64_t) *arg_uniqueid != proc->p_uniqueid) { + *arg_uniqueid = 0; + } + } } @@ -2007,7 +2103,7 @@ kdbg_cpumap_init_internal(kd_iop_t* iops, uint32_t cpu_count, uint8_t** cpumap, void kdbg_thrmap_init(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if (kd_ctrl_page.kdebug_flags & KDBG_MAPINIT) { return; @@ -2150,7 +2246,7 @@ kdbg_clear(void) kd_ctrl_page.oldest_time = 0; delete_buffers(); - nkdbufs = 0; + nkdbufs = 0; /* Clean up the thread map buffer */ kdbg_clear_thread_map(); @@ -2162,7 +2258,7 @@ kdbg_clear(void) void kdebug_reset(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); kdbg_lock_init(); @@ -2173,6 +2269,13 @@ kdebug_reset(void) } } +void +kdebug_free_early_buf(void) +{ + /* Must be done with the buffer, so release it back to the VM. */ + ml_static_mfree((vm_offset_t)&kd_early_buffer, sizeof(kd_early_buffer)); +} + int kdbg_setpid(kd_regtype *kdr) { @@ -2267,7 +2370,7 @@ kdbg_setpidex(kd_regtype *kdr) static int kdbg_initialize_typefilter(typefilter_t tf) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); assert(!kdbg_typefilter); assert(!kdbg_typefilter_memory_entry); typefilter_t deallocate_tf = NULL; @@ -2300,7 +2403,7 @@ kdbg_copyin_typefilter(user_addr_t addr, size_t size) int ret = ENOMEM; typefilter_t tf; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if (size != KDBG_TYPEFILTER_BITMAP_SIZE) { return EINVAL; @@ -2938,7 +3041,7 @@ write_error: static void kdbg_clear_thread_map(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if (kd_ctrl_page.kdebug_flags & KDBG_MAPINIT) { assert(kd_mapptr != NULL); @@ -2964,7 +3067,7 @@ kdbg_write_thread_map(vnode_t vp, vfs_context_t ctx) int ret = 0; boolean_t map_initialized; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); assert(ctx != NULL); map_initialized = (kd_ctrl_page.kdebug_flags & KDBG_MAPINIT); @@ -2995,7 +3098,7 @@ kdbg_copyout_thread_map(user_addr_t buffer, size_t *buffer_size) size_t map_size; int ret = 0; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); assert(buffer_size != NULL); map_initialized = (kd_ctrl_page.kdebug_flags & KDBG_MAPINIT); @@ -3023,7 +3126,7 @@ kdbg_readthrmap_v3(user_addr_t buffer, size_t buffer_size, int fd) boolean_t map_initialized; size_t map_size; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if ((!fd && !buffer) || (fd && buffer)) { return EINVAL; @@ -3079,6 +3182,8 @@ kdbg_wait(uint64_t timeout_ms, boolean_t locked_wait) int wait_result = THREAD_AWAKENED; uint64_t abstime = 0; + ktrace_assert_lock_held(); + if (timeout_ms != 0) { uint64_t ns = timeout_ms * NSEC_PER_MSEC; nanoseconds_to_absolutetime(ns, &abstime); @@ -3093,7 +3198,7 @@ kdbg_wait(uint64_t timeout_ms, boolean_t locked_wait) if (!locked_wait) { /* drop the mutex to allow others to access trace */ - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); } while (wait_result == THREAD_AWAKENED && @@ -3118,7 +3223,7 @@ kdbg_wait(uint64_t timeout_ms, boolean_t locked_wait) if (!locked_wait) { /* pick the mutex back up again */ - lck_mtx_lock(ktrace_lock); + ktrace_lock(); } /* write out whether we've exceeded the threshold */ @@ -3189,7 +3294,7 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep) kdbg_lock_init(); assert(kd_ctrl_page.kdebug_flags & KDBG_LOCKINIT); - lck_mtx_lock(ktrace_lock); + ktrace_lock(); /* * Some requests only require "read" access to kdebug trace. Regardless, @@ -3372,12 +3477,12 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep) if (name[0] == KERN_KDWRITETR || name[0] == KERN_KDWRITETR_V3) { number = nkdbufs * sizeof(kd_buf); - KERNEL_DEBUG_CONSTANT(TRACE_WRITING_EVENTS | DBG_FUNC_START, 0, 0, 0, 0, 0); + KDBG(TRACE_WRITING_EVENTS | DBG_FUNC_START); if (name[0] == KERN_KDWRITETR_V3) ret = kdbg_read(0, &number, vp, &context, RAW_VERSION3); else ret = kdbg_read(0, &number, vp, &context, RAW_VERSION1); - KERNEL_DEBUG_CONSTANT(TRACE_WRITING_EVENTS | DBG_FUNC_END, number, 0, 0, 0, 0); + KDBG(TRACE_WRITING_EVENTS | DBG_FUNC_END, number); *sizep = number; } else { @@ -3439,7 +3544,7 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep) } case KERN_KDTEST: - ret = kdbg_test(); + ret = kdbg_test(size); break; default: @@ -3447,9 +3552,9 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep) break; } out: - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); - return(ret); + return ret; } @@ -3470,6 +3575,7 @@ kdbg_read(user_addr_t buffer, size_t *number, vnode_t vp, vfs_context_t ctx, uin uint32_t rcursor; kd_buf lostevent; union kds_ptr kdsp; + bool traced_retrograde = false; struct kd_storage *kdsp_actual; struct kd_bufinfo *kdbp; struct kd_bufinfo *min_kdbp; @@ -3485,6 +3591,8 @@ kdbg_read(user_addr_t buffer, size_t *number, vnode_t vp, vfs_context_t ctx, uin count = *number/sizeof(kd_buf); *number = 0; + ktrace_assert_lock_held(); + if (count == 0 || !(kd_ctrl_page.kdebug_flags & KDBG_BUFINIT) || kdcopybuf == 0) return EINVAL; @@ -3500,7 +3608,7 @@ kdbg_read(user_addr_t buffer, size_t *number, vnode_t vp, vfs_context_t ctx, uin * disabled, no new events should occur on the AP. */ if (kd_ctrl_page.enabled) { - barrier_max = mach_absolute_time() & KDBG_TIMESTAMP_MASK; + barrier_max = kdbg_timestamp() & KDBG_TIMESTAMP_MASK; } /* @@ -3666,12 +3774,22 @@ next_cpu: */ if (earliest_time < min_kdbp->kd_prev_timebase) { /* - * if so, use the previous timestamp + 1 cycle + * If we haven't already, emit a retrograde events event. */ - min_kdbp->kd_prev_timebase++; + if (traced_retrograde) { + continue; + } + kdbg_set_timestamp_and_cpu(tempbuf, min_kdbp->kd_prev_timebase, kdbg_get_cpu(tempbuf)); - } else + tempbuf->arg1 = tempbuf->debugid; + tempbuf->arg2 = earliest_time; + tempbuf->arg3 = 0; + tempbuf->arg4 = 0; + tempbuf->debugid = TRACE_RETROGRADE_EVENTS; + traced_retrograde = true; + } else { min_kdbp->kd_prev_timebase = earliest_time; + } nextevent: tempbuf_count--; tempbuf_number++; @@ -3734,45 +3852,70 @@ check_error: } static int -kdbg_test(void) +kdbg_test(size_t flavor) { -#define KDEBUG_TEST_CODE(code) BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, (code)) int code = 0; + int dummy_iop = 0; - KDBG(KDEBUG_TEST_CODE(code)); code++; - KDBG(KDEBUG_TEST_CODE(code), 1); code++; - KDBG(KDEBUG_TEST_CODE(code), 1, 2); code++; - KDBG(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; - KDBG(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; - - KDBG_RELEASE(KDEBUG_TEST_CODE(code)); code++; - KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1); code++; - KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2); code++; - KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; - KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; - - KDBG_FILTERED(KDEBUG_TEST_CODE(code)); code++; - KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1); code++; - KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2); code++; - KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; - KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; - - KDBG_DEBUG(KDEBUG_TEST_CODE(code)); code++; - KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1); code++; - KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2); code++; - KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; - KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; +#define KDEBUG_TEST_CODE(code) BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, (code)) + switch (flavor) { + case 1: + /* try each macro */ + KDBG(KDEBUG_TEST_CODE(code)); code++; + KDBG(KDEBUG_TEST_CODE(code), 1); code++; + KDBG(KDEBUG_TEST_CODE(code), 1, 2); code++; + KDBG(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; + KDBG(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; + + KDBG_RELEASE(KDEBUG_TEST_CODE(code)); code++; + KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1); code++; + KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2); code++; + KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; + KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; + + KDBG_FILTERED(KDEBUG_TEST_CODE(code)); code++; + KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1); code++; + KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2); code++; + KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; + KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; + + KDBG_DEBUG(KDEBUG_TEST_CODE(code)); code++; + KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1); code++; + KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2); code++; + KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; + KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; + break; - return 0; + case 2: + if (kd_ctrl_page.kdebug_iops) { + /* avoid the assertion in kernel_debug_enter for a valid IOP */ + dummy_iop = kd_ctrl_page.kdebug_iops[0].cpu_id; + } + + /* ensure old timestamps are not emitted from kernel_debug_enter */ + kernel_debug_enter(dummy_iop, KDEBUG_TEST_CODE(code), + 100 /* very old timestamp */, 0, 0, 0, + 0, (uintptr_t)thread_tid(current_thread())); + code++; + kernel_debug_enter(dummy_iop, KDEBUG_TEST_CODE(code), + kdbg_timestamp(), 0, 0, 0, 0, + (uintptr_t)thread_tid(current_thread())); + code++; + break; + default: + return ENOTSUP; + } #undef KDEBUG_TEST_CODE + + return 0; } void -kdebug_boot_trace(unsigned int n_events, char *filter_desc) +kdebug_init(unsigned int n_events, char *filter_desc) { assert(filter_desc != NULL); -#if (defined(__i386__) || defined(__x86_64__)) +#if defined(__x86_64__) /* only trace MACH events when outputting kdebug to serial */ if (kdebug_serial) { n_events = 1; @@ -3782,7 +3925,7 @@ kdebug_boot_trace(unsigned int n_events, char *filter_desc) filter_desc[2] = '\0'; } } -#endif +#endif /* defined(__x86_64__) */ if (log_leaks && n_events == 0) { n_events = 200000; @@ -3796,7 +3939,7 @@ kdbg_set_typefilter_string(const char *filter_desc) { char *end = NULL; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); assert(filter_desc != NULL); @@ -3862,15 +4005,16 @@ kdbg_set_typefilter_string(const char *filter_desc) */ void kdebug_trace_start(unsigned int n_events, const char *filter_desc, - boolean_t need_map) + boolean_t at_wake) { uint32_t old1, old2; if (!n_events) { + kd_early_done = true; return; } - lck_mtx_lock(ktrace_lock); + ktrace_start_single_threaded(); kdbg_lock_init(); @@ -3904,19 +4048,20 @@ kdebug_trace_start(unsigned int n_events, const char *filter_desc, */ boolean_t s = ml_set_interrupts_enabled(FALSE); - if (need_map == TRUE) { + if (at_wake) { kdbg_thrmap_init(); } - kdbg_set_tracing_enabled(TRUE, kdebug_serial ? - (KDEBUG_ENABLE_TRACE | KDEBUG_ENABLE_SERIAL) : - KDEBUG_ENABLE_TRACE); + kdbg_set_tracing_enabled(TRUE, KDEBUG_ENABLE_TRACE | (kdebug_serial ? + KDEBUG_ENABLE_SERIAL : 0)); - /* - * Transfer all very early events from the static buffer into the real - * buffers. - */ - kernel_debug_early_end(); + if (!at_wake) { + /* + * Transfer all very early events from the static buffer into the real + * buffers. + */ + kernel_debug_early_end(); + } ml_set_interrupts_enabled(s); @@ -3927,10 +4072,10 @@ kdebug_trace_start(unsigned int n_events, const char *filter_desc, printf("serial output enabled with %lu named events\n", sizeof(kd_events)/sizeof(kd_event_t)); } -#endif +#endif /* KDEBUG_MOJO_TRACE */ out: - lck_mtx_unlock(ktrace_lock); + ktrace_end_single_threaded(); } void @@ -3939,8 +4084,9 @@ kdbg_dump_trace_to_file(const char *filename) vfs_context_t ctx; vnode_t vp; size_t write_size; + int ret; - lck_mtx_lock(ktrace_lock); + ktrace_lock(); if (!(kdebug_enable & KDEBUG_ENABLE_TRACE)) { goto out; @@ -3949,7 +4095,7 @@ kdbg_dump_trace_to_file(const char *filename) if (ktrace_get_owning_pid() != 0) { /* * Another process owns ktrace and is still active, disable tracing to - * capture whatever was being recorded. + * prevent wrapping. */ kdebug_enable = 0; kd_ctrl_page.enabled = 0; @@ -3957,7 +4103,7 @@ kdbg_dump_trace_to_file(const char *filename) goto out; } - KERNEL_DEBUG_CONSTANT(TRACE_PANIC | DBG_FUNC_NONE, 0, 0, 0, 0, 0); + KDBG(TRACE_WRITING_EVENTS | DBG_FUNC_START); kdebug_enable = 0; kd_ctrl_page.enabled = 0; @@ -3972,13 +4118,39 @@ kdbg_dump_trace_to_file(const char *filename) kdbg_write_thread_map(vp, ctx); write_size = nkdbufs * sizeof(kd_buf); - kdbg_read(0, &write_size, vp, ctx, RAW_VERSION1); + ret = kdbg_read(0, &write_size, vp, ctx, RAW_VERSION1); + if (ret) { + goto out_close; + } + /* + * Wait to synchronize the file to capture the I/O in the + * TRACE_WRITING_EVENTS interval. + */ + ret = VNOP_FSYNC(vp, MNT_WAIT, ctx); + + /* + * Balance the starting TRACE_WRITING_EVENTS tracepoint manually. + */ + kd_buf end_event = { + .debugid = TRACE_WRITING_EVENTS | DBG_FUNC_END, + .arg1 = write_size, + .arg2 = ret, + .arg5 = thread_tid(current_thread()), + }; + kdbg_set_timestamp_and_cpu(&end_event, kdbg_timestamp(), + cpu_number()); + + /* this is best effort -- ignore any errors */ + (void)kdbg_write_to_vnode((caddr_t)&end_event, sizeof(kd_buf), vp, ctx, + RAW_file_offset); + +out_close: vnode_close(vp, FWRITE, ctx); sync(current_proc(), (void *)NULL, (int *)NULL); out: - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); } /* Helper function for filling in the BSD name for an address space @@ -4002,6 +4174,34 @@ void kdbg_get_task_name(char* name_buf, int len, task_t task) snprintf(name_buf, len, "%p [!bsd]", task); } +static int +kdbg_sysctl_continuous SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int value = kdbg_continuous_time; + int ret = sysctl_io_number(req, value, sizeof(value), &value, NULL); + + if (ret || !req->newptr) { + return ret; + } + + kdbg_continuous_time = value; + return 0; +} + +SYSCTL_NODE(_kern, OID_AUTO, kdbg, CTLFLAG_RD | CTLFLAG_LOCKED, 0, + "kdbg"); + +SYSCTL_PROC(_kern_kdbg, OID_AUTO, experimental_continuous, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, + sizeof(int), kdbg_sysctl_continuous, "I", + "Set kdebug to use mach_continuous_time"); + +SYSCTL_QUAD(_kern_kdbg, OID_AUTO, oldest_time, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, + &kd_ctrl_page.oldest_time, + "Find the oldest timestamp still in trace"); + #if KDEBUG_MOJO_TRACE static kd_event_t * binary_search(uint32_t id) diff --git a/bsd/kern/kern_acct.c b/bsd/kern/kern_acct.c index aec90c9e0..8bca8fa41 100644 --- a/bsd/kern/kern_acct.c +++ b/bsd/kern/kern_acct.c @@ -93,7 +93,6 @@ #include #include #include -#include #if CONFIG_MACF #include #endif diff --git a/bsd/kern/kern_aio.c b/bsd/kern/kern_aio.c index 3869ad669..08f8d135f 100644 --- a/bsd/kern/kern_aio.c +++ b/bsd/kern/kern_aio.c @@ -1465,6 +1465,8 @@ lio_listio(proc_t p, struct lio_listio_args *uap, int *retval ) struct user_sigevent aiosigev; aio_lio_context *lio_context; boolean_t free_context = FALSE; + uint32_t *paio_offset; + uint32_t *paio_nbytes; KERNEL_DEBUG( (BSDDBG_CODE(DBG_BSD_AIO, AIO_listio)) | DBG_FUNC_START, (int)p, uap->nent, uap->mode, 0, 0 ); @@ -1596,9 +1598,15 @@ lio_listio(proc_t p, struct lio_listio_args *uap, int *retval ) aio_enqueue_work(p, entryp, 1); aio_proc_unlock(p); - KERNEL_DEBUG( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_NONE, - (int)p, (int)entryp->uaiocbp, 0, 0, 0 ); - } + KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_START, + (int)p, (int)entryp->uaiocbp, entryp->flags, entryp->aiocb.aio_fildes, 0 ); + paio_offset = (uint32_t*) &entryp->aiocb.aio_offset; + paio_nbytes = (uint32_t*) &entryp->aiocb.aio_nbytes; + KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_END, + paio_offset[0], (sizeof(entryp->aiocb.aio_offset) == sizeof(uint64_t) ? paio_offset[1] : 0), + paio_nbytes[0], (sizeof(entryp->aiocb.aio_nbytes) == sizeof(uint64_t) ? paio_nbytes[1] : 0), + 0 ); + } switch(uap->mode) { case LIO_WAIT: @@ -1930,9 +1938,11 @@ static int aio_queue_async_request(proc_t procp, user_addr_t aiocbp, int kindOfIO ) { aio_workq_entry *entryp; - int result; - int old_count; - + int result; + int old_count; + uint32_t *paio_offset; + uint32_t *paio_nbytes; + old_count = aio_increment_total_count(); if (old_count >= aio_max_requests) { result = EAGAIN; @@ -1965,10 +1975,16 @@ aio_queue_async_request(proc_t procp, user_addr_t aiocbp, int kindOfIO ) aio_enqueue_work(procp, entryp, 1); aio_proc_unlock(procp); - - KERNEL_DEBUG( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_NONE, - (int)procp, (int)aiocbp, 0, 0, 0 ); - + + paio_offset = (uint32_t*) &entryp->aiocb.aio_offset; + paio_nbytes = (uint32_t*) &entryp->aiocb.aio_nbytes; + KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_START, + (int)procp, (int)aiocbp, entryp->flags, entryp->aiocb.aio_fildes, 0 ); + KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_END, + paio_offset[0], (sizeof(entryp->aiocb.aio_offset) == sizeof(uint64_t) ? paio_offset[1] : 0), + paio_nbytes[0], (sizeof(entryp->aiocb.aio_nbytes) == sizeof(uint64_t) ? paio_nbytes[1] : 0), + 0 ); + return( 0 ); error_exit: diff --git a/bsd/kern/kern_clock.c b/bsd/kern/kern_clock.c index f0e051345..08507cdc5 100644 --- a/bsd/kern/kern_clock.c +++ b/bsd/kern/kern_clock.c @@ -69,8 +69,6 @@ * HISTORY */ -#include - #include #include #include @@ -167,6 +165,8 @@ timeout_with_leeway( /* * Cancel a timeout. + * Deprecated because it's very inefficient. + * Switch to an allocated thread call instead. */ void untimeout( @@ -201,6 +201,8 @@ bsd_timeout( /* * Cancel a timeout. + * Deprecated because it's very inefficient. + * Switch to an allocated thread call instead. */ void bsd_untimeout( @@ -253,15 +255,14 @@ static int sysctl_clockrate (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, __unused struct sysctl_req *req) { - struct clockinfo clkinfo; + struct clockinfo clkinfo = { + .hz = hz, + .tick = tick, + .tickadj = 0, + .stathz = hz, + .profhz = hz, + }; - /* - * Construct clockinfo structure. - */ - clkinfo.hz = hz; - clkinfo.tick = tick; - clkinfo.profhz = hz; - clkinfo.stathz = hz; return sysctl_io_opaque(req, &clkinfo, sizeof(clkinfo), NULL); } diff --git a/bsd/kern/kern_control.c b/bsd/kern/kern_control.c index f7ca33348..cda6f1046 100644 --- a/bsd/kern/kern_control.c +++ b/bsd/kern/kern_control.c @@ -839,7 +839,7 @@ ctl_enqueuembuf_list(void *kctlref, u_int32_t unit, struct mbuf *m_list, errno_t error = 0; struct mbuf *m, *nextpkt; int needwakeup = 0; - int len; + int len = 0; u_int32_t kctlflags; /* @@ -1820,9 +1820,9 @@ ctl_unlock(struct socket *so, int refcount, void *lr) } static lck_mtx_t * -ctl_getlock(struct socket *so, int locktype) +ctl_getlock(struct socket *so, int flags) { -#pragma unused(locktype) +#pragma unused(flags) struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb; if (so->so_pcb) { @@ -1903,15 +1903,15 @@ kctl_reg_list SYSCTL_HANDLER_ARGS xkr->xkr_sendbufsize = kctl->sendbufsize; xkr->xkr_lastunit = kctl->lastunit; xkr->xkr_pcbcount = pcbcount; - xkr->xkr_connect = (uint64_t)VM_KERNEL_ADDRPERM(kctl->connect); + xkr->xkr_connect = (uint64_t)VM_KERNEL_UNSLIDE(kctl->connect); xkr->xkr_disconnect = - (uint64_t)VM_KERNEL_ADDRPERM(kctl->disconnect); - xkr->xkr_send = (uint64_t)VM_KERNEL_ADDRPERM(kctl->send); + (uint64_t)VM_KERNEL_UNSLIDE(kctl->disconnect); + xkr->xkr_send = (uint64_t)VM_KERNEL_UNSLIDE(kctl->send); xkr->xkr_send_list = - (uint64_t)VM_KERNEL_ADDRPERM(kctl->send_list); - xkr->xkr_setopt = (uint64_t)VM_KERNEL_ADDRPERM(kctl->setopt); - xkr->xkr_getopt = (uint64_t)VM_KERNEL_ADDRPERM(kctl->getopt); - xkr->xkr_rcvd = (uint64_t)VM_KERNEL_ADDRPERM(kctl->rcvd); + (uint64_t)VM_KERNEL_UNSLIDE(kctl->send_list); + xkr->xkr_setopt = (uint64_t)VM_KERNEL_UNSLIDE(kctl->setopt); + xkr->xkr_getopt = (uint64_t)VM_KERNEL_UNSLIDE(kctl->getopt); + xkr->xkr_rcvd = (uint64_t)VM_KERNEL_UNSLIDE(kctl->rcvd); strlcpy(xkr->xkr_name, kctl->name, sizeof(xkr->xkr_name)); error = SYSCTL_OUT(req, buf, item_size); diff --git a/bsd/kern/kern_core.c b/bsd/kern/kern_core.c index 5cb6e4fa2..73a9a454b 100644 --- a/bsd/kern/kern_core.c +++ b/bsd/kern/kern_core.c @@ -36,7 +36,7 @@ #include #include - +#include #include #include #include @@ -83,6 +83,21 @@ mythread_state_flavor_t thread_flavor_array [] = { {x86_EXCEPTION_STATE, x86_EXCEPTION_STATE_COUNT}, }; int mynum_flavors=3; +#elif defined (__arm__) +mythread_state_flavor_t thread_flavor_array[]={ + {ARM_THREAD_STATE , ARM_THREAD_STATE_COUNT}, + {ARM_VFP_STATE, ARM_VFP_STATE_COUNT}, + {ARM_EXCEPTION_STATE, ARM_EXCEPTION_STATE_COUNT} + }; +int mynum_flavors=3; + +#elif defined (__arm64__) +mythread_state_flavor_t thread_flavor_array[]={ + {ARM_THREAD_STATE64 , ARM_THREAD_STATE64_COUNT}, + /* ARM64_TODO: VFP */ + {ARM_EXCEPTION_STATE64, ARM_EXCEPTION_STATE64_COUNT} + }; +int mynum_flavors=2; #else #error architecture not supported #endif @@ -124,6 +139,12 @@ process_cpu_type(proc_t core_proc) } else { what_we_think = CPU_TYPE_I386; } +#elif defined (__arm__) || defined(__arm64__) + if (IS_64BIT_PROCESS(core_proc)) { + what_we_think = CPU_TYPE_ARM64; + } else { + what_we_think = CPU_TYPE_ARM; + } #endif return what_we_think; } @@ -138,6 +159,12 @@ process_cpu_subtype(proc_t core_proc) } else { what_we_think = CPU_SUBTYPE_I386_ALL; } +#elif defined (__arm__) || defined(__arm64__) + if (IS_64BIT_PROCESS(core_proc)) { + what_we_think = CPU_SUBTYPE_ARM64_ALL; + } else { + what_we_think = CPU_SUBTYPE_ARM_ALL; + } #endif return what_we_think; } @@ -310,6 +337,9 @@ coredump(proc_t core_proc, uint32_t reserve_mb, int coredump_flags) VATTR_INIT(&va); /* better to do it here than waste more stack in vnode_setsize */ VATTR_SET(&va, va_data_size, 0); + if (core_proc == initproc) { + VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D); + } vnode_setattr(vp, &va, ctx); core_proc->p_acflag |= ACORE; diff --git a/bsd/kern/kern_credential.c b/bsd/kern/kern_credential.c index 1376ff3c5..bac6af26f 100644 --- a/bsd/kern/kern_credential.c +++ b/bsd/kern/kern_credential.c @@ -74,6 +74,8 @@ #include #endif +#include + void mach_kauth_cred_uthread_update( void ); #define CRED_DIAGNOSTIC 0 @@ -186,6 +188,22 @@ TAILQ_HEAD(kauth_resolver_done_head, kauth_resolver_work) kauth_resolver_done; #define KAUTH_COMPLAINT_INTERVAL 1000 int kauth_resolver_timeout_cnt = 0; +#if DEVELOPMENT || DEBUG +/* Internal builds get different (less ambiguous) breadcrumbs. */ +#define KAUTH_RESOLVER_FAILED_ERRCODE EOWNERDEAD +#else +/* But non-Internal builds get errors that are allowed by standards. */ +#define KAUTH_RESOLVER_FAILED_ERRCODE EIO +#endif /* DEVELOPMENT || DEBUG */ + +int kauth_resolver_failed_cnt = 0; +#define RESOLVER_FAILED_MESSAGE(fmt, args...) \ +do { \ + if (!(kauth_resolver_failed_cnt++ % 100)) { \ + printf("%s: " fmt "\n", __PRETTY_FUNCTION__, ##args); \ + } \ +} while (0) + static int kauth_resolver_submit(struct kauth_identity_extlookup *lkp, uint64_t extend_data); static int kauth_resolver_complete(user_addr_t message); static int kauth_resolver_getwork(user_addr_t message); @@ -318,7 +336,8 @@ __KERNEL_IS_WAITING_ON_EXTERNAL_CREDENTIAL_RESOLVER__( break; /* woken because the resolver has died? */ if (kauth_resolver_identity == 0) { - error = EIO; + RESOLVER_FAILED_MESSAGE("kauth external resolver died while while waiting for work to complete"); + error = KAUTH_RESOLVER_FAILED_ERRCODE; break; } /* an error? */ @@ -578,6 +597,11 @@ identitysvc(__unused struct proc *p, struct identitysvc_args *uap, __unused int3 int error; pid_t new_id; + if (!IOTaskHasEntitlement(current_task(), IDENTITYSVC_ENTITLEMENT)) { + KAUTH_DEBUG("RESOLVER - pid %d not entitled to call identitysvc", current_proc()->p_pid); + return(EPERM); + } + /* * New server registering itself. */ @@ -755,8 +779,10 @@ kauth_resolver_getwork_continue(int result) * If this is a wakeup from another thread in the resolver * deregistering it, error out the request-for-work thread */ - if (!kauth_resolver_identity) - error = EIO; + if (!kauth_resolver_identity) { + RESOLVER_FAILED_MESSAGE("external resolver died"); + error = KAUTH_RESOLVER_FAILED_ERRCODE; + } KAUTH_RESOLVER_UNLOCK(); return(error); } @@ -897,8 +923,10 @@ kauth_resolver_getwork(user_addr_t message) * If this is a wakeup from another thread in the resolver * deregistering it, error out the request-for-work thread */ - if (!kauth_resolver_identity) - error = EIO; + if (!kauth_resolver_identity) { + printf("external resolver died"); + error = KAUTH_RESOLVER_FAILED_ERRCODE; + } return(error); } return kauth_resolver_getwork2(message); @@ -922,7 +950,7 @@ kauth_resolver_complete(user_addr_t message) struct kauth_identity_extlookup extl; struct kauth_resolver_work *workp; struct kauth_resolver_work *killp; - int error, result, request_flags; + int error, result, want_extend_data; /* * Copy in the mesage, including the extension field, since we are @@ -956,6 +984,7 @@ kauth_resolver_complete(user_addr_t message) case KAUTH_EXTLOOKUP_FATAL: /* fatal error means the resolver is dead */ KAUTH_DEBUG("RESOLVER - resolver %d died, waiting for a new one", kauth_resolver_identity); + RESOLVER_FAILED_MESSAGE("resolver %d died, waiting for a new one", kauth_resolver_identity); /* * Terminate outstanding requests; without an authoritative * resolver, we are now back on our own authority. Tag the @@ -973,7 +1002,7 @@ kauth_resolver_complete(user_addr_t message) /* Cause all waiting-for-work threads to return EIO */ wakeup((caddr_t)&kauth_resolver_unsubmitted); /* and return EIO to the caller */ - error = EIO; + error = KAUTH_RESOLVER_FAILED_ERRCODE; break; case KAUTH_EXTLOOKUP_BADRQ: @@ -983,12 +1012,14 @@ kauth_resolver_complete(user_addr_t message) case KAUTH_EXTLOOKUP_FAILURE: KAUTH_DEBUG("RESOLVER - resolver reported transient failure for request %d", extl.el_seqno); - result = EIO; + RESOLVER_FAILED_MESSAGE("resolver reported transient failure for request %d", extl.el_seqno); + result = KAUTH_RESOLVER_FAILED_ERRCODE; break; default: KAUTH_DEBUG("RESOLVER - resolver returned unexpected status %d", extl.el_result); - result = EIO; + RESOLVER_FAILED_MESSAGE("resolver returned unexpected status %d", extl.el_result); + result = KAUTH_RESOLVER_FAILED_ERRCODE; break; } @@ -1004,9 +1035,9 @@ kauth_resolver_complete(user_addr_t message) /* found it? */ if (workp->kr_seqno == extl.el_seqno) { /* - * Take a snapshot of the original request flags. + * Do we want extend_data? */ - request_flags = workp->kr_work.el_flags; + want_extend_data = (workp->kr_work.el_flags & (KAUTH_EXTLOOKUP_WANT_PWNAM|KAUTH_EXTLOOKUP_WANT_GRNAM)); /* * Get the request of the submitted queue so @@ -1049,11 +1080,13 @@ kauth_resolver_complete(user_addr_t message) * part of a user's address space if they return * flags that mismatch the original request's flags. */ - if ((extl.el_flags & request_flags) & (KAUTH_EXTLOOKUP_VALID_PWNAM|KAUTH_EXTLOOKUP_VALID_GRNAM)) { + if (want_extend_data && (extl.el_flags & (KAUTH_EXTLOOKUP_VALID_PWNAM|KAUTH_EXTLOOKUP_VALID_GRNAM))) { size_t actual; /* notused */ KAUTH_RESOLVER_UNLOCK(); error = copyinstr(extl.el_extend, CAST_DOWN(void *, workp->kr_extend), MAXPATHLEN, &actual); + KAUTH_DEBUG("RESOLVER - resolver got name :%*s: len = %d\n", (int)actual, + actual ? "null" : (char *)extl.el_extend, actual); KAUTH_RESOLVER_LOCK(); } else if (extl.el_flags & (KAUTH_EXTLOOKUP_VALID_PWNAM|KAUTH_EXTLOOKUP_VALID_GRNAM)) { error = EFAULT; @@ -2608,7 +2641,10 @@ kauth_cred_cache_lookup(int from, int to, void *src, void *dst) * atomically. */ if (to == KI_VALID_PWNAM || to == KI_VALID_GRNAM) { + if (dst == NULL) + return (EINVAL); namebuf = dst; + *namebuf = '\0'; } ki.ki_valid = 0; switch(from) { @@ -2632,6 +2668,9 @@ kauth_cred_cache_lookup(int from, int to, void *src, void *dst) default: return(EINVAL); } + /* If we didn't get what we're asking for. Call the resolver */ + if (!error && !(to & ki.ki_valid)) + error = ENOENT; /* lookup failure or error */ if (error != 0) { /* any other error is fatal */ @@ -4574,7 +4613,6 @@ int kauth_proc_label_update(struct proc *p, struct label *label) /* update cred on proc */ PROC_UPDATE_CREDS_ONPROC(p); - mac_proc_set_enforce(p, MAC_ALL_ENFORCE); proc_ucred_unlock(p); } break; @@ -4653,7 +4691,6 @@ kauth_proc_label_update_execve(struct proc *p, vfs_context_t ctx, p->p_ucred = my_new_cred; /* update cred on proc */ PROC_UPDATE_CREDS_ONPROC(p); - mac_proc_set_enforce(p, MAC_ALL_ENFORCE); proc_ucred_unlock(p); } break; diff --git a/bsd/kern/kern_cs.c b/bsd/kern/kern_cs.c index 214b041b7..7a3d22baf 100644 --- a/bsd/kern/kern_cs.c +++ b/bsd/kern/kern_cs.c @@ -84,6 +84,7 @@ const int cs_enforcement_enable = 1; const int cs_library_val_enable = 1; #else /* !SECURE_KERNEL */ int cs_enforcement_panic=0; +int cs_relax_platform_task_ports = 0; #if CONFIG_ENFORCE_SIGNED_CODE #define DEFAULT_CS_ENFORCEMENT_ENABLE 1 @@ -120,6 +121,7 @@ SYSCTL_INT(_vm, OID_AUTO, cs_library_validation, CTLFLAG_RW | CTLFLAG_LOCKED, &c #endif /* !SECURE_KERNEL */ int panic_on_cs_killed = 0; + void cs_init(void) { @@ -140,6 +142,10 @@ cs_init(void) cs_enforcement_panic = (panic != 0); } + PE_parse_boot_argn("cs_relax_platform_task_ports", + &cs_relax_platform_task_ports, + sizeof(cs_relax_platform_task_ports)); + PE_parse_boot_argn("cs_debug", &cs_debug, sizeof (cs_debug)); #if !CONFIG_ENFORCE_LIBRARY_VALIDATION @@ -180,7 +186,9 @@ cs_allow_invalid(struct proc *p) { p->p_csflags |= CS_DEBUGGED; } + proc_unlock(p); + vm_map_switch_protect(get_task_map(p->task), FALSE); #endif return (p->p_csflags & (CS_KILL | CS_HARD)) == 0; @@ -378,6 +386,18 @@ csblob_get_flags(struct cs_blob *blob) return blob->csb_flags; } +/* + * Function: csblob_get_hashtype + * + * Description: This function returns the hash type for a given blob +*/ + +uint8_t +csblob_get_hashtype(struct cs_blob const * const blob) +{ + return blob->csb_hashtype != NULL ? cs_hash_type(blob->csb_hashtype) : 0; +} + /* * Function: csproc_get_blob * @@ -453,6 +473,18 @@ csblob_get_cdhash(struct cs_blob *csblob) return csblob->csb_cdhash; } +/* + * Function: csblob_get_signer_type + * + * Description: This function returns the signer type + * as an integer + */ +unsigned int +csblob_get_signer_type(struct cs_blob *csblob) +{ + return csblob->csb_signer_type; +} + void * csblob_entitlements_dictionary_copy(struct cs_blob *csblob) { @@ -487,6 +519,24 @@ csproc_get_teamid(struct proc *p) return csblob_get_teamid(csblob); } +/* + * Function: csproc_get_signer_type + * + * Description: This function returns the signer type + * of the process p +*/ +unsigned int +csproc_get_signer_type(struct proc *p) +{ + struct cs_blob *csblob; + + csblob = csproc_get_blob(p); + if (csblob == NULL) + return CS_SIGNER_TYPE_UNKNOWN; + + return csblob_get_signer_type(csblob); +} + /* * Function: csvnode_get_teamid * @@ -534,6 +584,26 @@ csproc_get_platform_path(struct proc *p) return (csblob == NULL) ? 0 : csblob->csb_platform_path; } +#if DEVELOPMENT || DEBUG +void +csproc_clear_platform_binary(struct proc *p) +{ + struct cs_blob *csblob = csproc_get_blob(p); + + if (csblob == NULL) { + return; + } + + if (cs_debug) { + printf("clearing platform binary on proc/task: pid = %d\n", p->p_pid); + } + + csblob->csb_platform_binary = 0; + csblob->csb_platform_path = 0; + task_set_platform_binary(proc_task(p), FALSE); +} +#endif + /* * Function: csproc_get_prod_signed * @@ -611,6 +681,46 @@ csfg_get_cdhash(struct fileglob *fg, uint64_t offset, size_t *cdhash_size) return csblob->csb_cdhash; } +/* + * Function: csfg_get_signer_type + * + * Description: This returns the signer type + * for the fileglob fg + */ +unsigned int +csfg_get_signer_type(struct fileglob *fg) +{ + struct ubc_info *uip; + unsigned int signer_type = CS_SIGNER_TYPE_UNKNOWN; + vnode_t vp; + + if (FILEGLOB_DTYPE(fg) != DTYPE_VNODE) + return CS_SIGNER_TYPE_UNKNOWN; + + vp = (struct vnode *)fg->fg_data; + if (vp == NULL) + return CS_SIGNER_TYPE_UNKNOWN; + + vnode_lock(vp); + if (!UBCINFOEXISTS(vp)) + goto out; + + uip = vp->v_ubcinfo; + if (uip == NULL) + goto out; + + if (uip->cs_blobs == NULL) + goto out; + + /* It is OK to extract the signer type from the first blob, + because all blobs of a vnode must have the same signer type. */ + signer_type = uip->cs_blobs->csb_signer_type; +out: + vnode_unlock(vp); + + return signer_type; +} + /* * Function: csfg_get_teamid * @@ -666,11 +776,11 @@ csfg_get_prod_signed(struct fileglob *fg) int prod_signed = 0; if (FILEGLOB_DTYPE(fg) != DTYPE_VNODE) - return NULL; + return 0; vp = (struct vnode *)fg->fg_data; if (vp == NULL) - return NULL; + return 0; vnode_lock(vp); if (!UBCINFOEXISTS(vp)) diff --git a/bsd/kern/kern_csr.c b/bsd/kern/kern_csr.c index bc5e03661..15a5ede70 100644 --- a/bsd/kern/kern_csr.c +++ b/bsd/kern/kern_csr.c @@ -72,7 +72,15 @@ csr_check(csr_config_t mask) return ret; } - ret = (config & mask) ? 0 : EPERM; + // CSR_ALLOW_KERNEL_DEBUGGER needs to be allowed when SIP is disabled + // to allow 3rd-party developers to debug their kexts. Use + // CSR_ALLOW_UNTRUSTED_KEXTS as a proxy for "SIP is disabled" on the + // grounds that you can do the same damage with a kernel debugger as + // you can with an untrusted kext. + if ((config & (CSR_ALLOW_UNTRUSTED_KEXTS|CSR_ALLOW_APPLE_INTERNAL)) != 0) + config |= CSR_ALLOW_KERNEL_DEBUGGER; + + ret = ((config & mask) == mask) ? 0 : EPERM; if (ret == EPERM) { // Override the return value if booted from the BaseSystem and the mask does not contain any flag that should always be enforced. if (csr_allow_all && (mask & CSR_ALWAYS_ENFORCED_FLAGS) == 0) diff --git a/bsd/kern/kern_descrip.c b/bsd/kern/kern_descrip.c index b93ab3bbf..06f9b82a6 100644 --- a/bsd/kern/kern_descrip.c +++ b/bsd/kern/kern_descrip.c @@ -120,6 +120,10 @@ #include #include +#if CONFIG_MACF +#include +#endif + kern_return_t ipc_object_copyin(ipc_space_t, mach_port_name_t, mach_msg_type_name_t, ipc_port_t *); void ipc_port_release_send(ipc_port_t); @@ -418,6 +422,7 @@ _fdrelse(struct proc * p, int fd) while ((nfd = fdp->fd_lastfile) > 0 && fdp->fd_ofiles[nfd] == NULL && !(fdp->fd_ofileflags[nfd] & UF_RESERVED)) + /* JMM - What about files with lingering EV_VANISHED knotes? */ fdp->fd_lastfile--; } @@ -1121,6 +1126,10 @@ fcntl_nocancel(proc_t p, struct fcntl_nocancel_args *uap, int32_t *retval) case F_GETLK: case F_OFD_GETLK: +#if CONFIG_EMBEDDED + case F_GETLKPID: + case F_OFD_GETLKPID: +#endif if (fp->f_type != DTYPE_VNODE) { error = EBADF; goto out; @@ -1966,7 +1975,7 @@ fcntl_nocancel(proc_t p, struct fcntl_nocancel_args *uap, int32_t *retval) error = copyin(argp, &lv32, sizeof (lv32)); lv.lv_file_start = lv32.lv_file_start; - lv.lv_error_message = CAST_USER_ADDR_T(lv32.lv_error_message); + lv.lv_error_message = (void *)(uintptr_t)lv32.lv_error_message; lv.lv_error_message_size = lv32.lv_error_message; } if (error) @@ -1974,7 +1983,7 @@ fcntl_nocancel(proc_t p, struct fcntl_nocancel_args *uap, int32_t *retval) #if CONFIG_MACF error = mac_file_check_library_validation(p, fg, lv.lv_file_start, - lv.lv_error_message, lv.lv_error_message_size); + (user_long_t)lv.lv_error_message, lv.lv_error_message_size); #endif break; @@ -4631,6 +4640,7 @@ fg_free(struct fileglob *fg) } + /* * fdexec * @@ -4648,16 +4658,39 @@ fg_free(struct fileglob *fg) * Returns: void * * Locks: This function internally takes and drops proc_fdlock() + * But assumes tables don't grow/change while unlocked. * */ void -fdexec(proc_t p, short flags) +fdexec(proc_t p, short flags, int self_exec) { struct filedesc *fdp = p->p_fd; int i; boolean_t cloexec_default = (flags & POSIX_SPAWN_CLOEXEC_DEFAULT) != 0; + thread_t self = current_thread(); + struct uthread *ut = get_bsdthread_info(self); + struct kqueue *dealloc_kq = NULL; + + /* + * If the current thread is bound as a workq/workloop + * servicing thread, we need to unbind it first. + */ + if (ut->uu_kqueue_bound && self_exec) { + kevent_qos_internal_unbind(p, 0, self, + ut->uu_kqueue_flags); + } proc_fdlock(p); + + /* + * Deallocate the knotes for this process + * and mark the tables non-existent so + * subsequent kqueue closes go faster. + */ + knotes_dealloc(p); + assert(fdp->fd_knlistsize == -1); + assert(fdp->fd_knhashmask == 0); + for (i = fdp->fd_lastfile; i >= 0; i--) { struct fileproc *fp = fdp->fd_ofiles[i]; @@ -4681,8 +4714,6 @@ fdexec(proc_t p, short flags) || (fp && mac_file_check_inherit(proc_ucred(p), fp->f_fglob)) #endif ) { - if (i < fdp->fd_knlistsize) - knote_fdclose(p, i, TRUE); procfdtbl_clearfd(p, i); if (i == fdp->fd_lastfile && i > 0) fdp->fd_lastfile--; @@ -4704,7 +4735,18 @@ fdexec(proc_t p, short flags) fileproc_free(fp); } } + + /* release the per-process workq kq */ + if (fdp->fd_wqkqueue) { + dealloc_kq = fdp->fd_wqkqueue; + fdp->fd_wqkqueue = NULL; + } + proc_fdunlock(p); + + /* Anything to free? */ + if (dealloc_kq) + kqueue_dealloc(dealloc_kq); } @@ -4892,10 +4934,6 @@ fdcopy(proc_t p, vnode_t uth_cdir) if (*fpp == NULL && i == newfdp->fd_lastfile && i > 0) newfdp->fd_lastfile--; } - newfdp->fd_knlist = NULL; - newfdp->fd_knlistsize = -1; - newfdp->fd_knhash = NULL; - newfdp->fd_knhashmask = 0; } fpp = newfdp->fd_ofiles; flags = newfdp->fd_ofileflags; @@ -4931,6 +4969,20 @@ fdcopy(proc_t p, vnode_t uth_cdir) } proc_fdunlock(p); + + /* + * Initialize knote and kqueue tracking structs + */ + newfdp->fd_knlist = NULL; + newfdp->fd_knlistsize = -1; + newfdp->fd_knhash = NULL; + newfdp->fd_knhashmask = 0; + newfdp->fd_kqhash = NULL; + newfdp->fd_kqhashmask = 0; + newfdp->fd_wqkqueue = NULL; + lck_mtx_init(&newfdp->fd_kqhashlock, proc_kqhashlock_grp, proc_lck_attr); + lck_mtx_init(&newfdp->fd_knhashlock, proc_knhashlock_grp, proc_lck_attr); + return (newfdp); } @@ -4954,6 +5006,7 @@ fdfree(proc_t p) { struct filedesc *fdp; struct fileproc *fp; + struct kqueue *dealloc_kq = NULL; int i; proc_fdlock(p); @@ -4968,13 +5021,17 @@ fdfree(proc_t p) if (&filedesc0 == fdp) panic("filedesc0"); + /* + * deallocate all the knotes up front and claim empty + * tables to make any subsequent kqueue closes faster. + */ + knotes_dealloc(p); + assert(fdp->fd_knlistsize == -1); + assert(fdp->fd_knhashmask == 0); + + /* close file descriptors */ if (fdp->fd_nfiles > 0 && fdp->fd_ofiles) { for (i = fdp->fd_lastfile; i >= 0; i--) { - - /* May still have knotes for fd without open file */ - if (i < fdp->fd_knlistsize) - knote_fdclose(p, i, TRUE); - if ((fp = fdp->fd_ofiles[i]) != NULL) { if (fdp->fd_ofileflags[i] & UF_RESERVED) @@ -4993,10 +5050,18 @@ fdfree(proc_t p) fdp->fd_nfiles = 0; } + if (fdp->fd_wqkqueue) { + dealloc_kq = fdp->fd_wqkqueue; + fdp->fd_wqkqueue = NULL; + } + proc_fdunlock(p); + if (dealloc_kq) + kqueue_dealloc(dealloc_kq); + if (fdp->fd_cdir) - vnode_rele(fdp->fd_cdir); + vnode_rele(fdp->fd_cdir); if (fdp->fd_rdir) vnode_rele(fdp->fd_rdir); @@ -5004,10 +5069,14 @@ fdfree(proc_t p) p->p_fd = NULL; proc_fdunlock(p); - if (fdp->fd_knlist) - FREE(fdp->fd_knlist, M_KQUEUE); - if (fdp->fd_knhash) - FREE(fdp->fd_knhash, M_KQUEUE); + if (fdp->fd_kqhash) { + for (uint32_t j = 0; j <= fdp->fd_kqhashmask; j++) + assert(SLIST_EMPTY(&fdp->fd_kqhash[j])); + FREE(fdp->fd_kqhash, M_KQUEUE); + } + + lck_mtx_destroy(&fdp->fd_kqhashlock, proc_kqhashlock_grp); + lck_mtx_destroy(&fdp->fd_knhashlock, proc_knhashlock_grp); FREE_ZONE(fdp, sizeof(*fdp), M_FILEDESC); } @@ -5437,6 +5506,7 @@ fileport_makefd(proc_t p, struct fileport_makefd_args *uap, int32_t *retval) err = fdalloc(p, 0, &fd); if (err != 0) { proc_fdunlock(p); + fg_drop(fp); goto out; } *fdflags(p, fd) |= UF_EXCLOSE; @@ -5885,9 +5955,10 @@ fo_close(struct fileglob *fg, vfs_context_t ctx) * !0 Filter is active */ int -fo_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) +fo_kqfilter(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx) { - return ((*fp->f_ops->fo_kqfilter)(fp, kn, ctx)); + return ((*fp->f_ops->fo_kqfilter)(fp, kn, kev, ctx)); } /* @@ -5904,6 +5975,7 @@ file_issendable(proc_t p, struct fileproc *fp) case DTYPE_SOCKET: case DTYPE_PIPE: case DTYPE_PSXSHM: + case DTYPE_NETPOLICY: return (0 == (fp->f_fglob->fg_lflags & FG_CONFINED)); default: /* DTYPE_KQUEUE, DTYPE_FSEVENTS, DTYPE_PSXSEM */ diff --git a/bsd/kern/kern_event.c b/bsd/kern/kern_event.c index 66cd6e2a5..f64bef436 100644 --- a/bsd/kern/kern_event.c +++ b/bsd/kern/kern_event.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -55,6 +55,7 @@ * @(#)kern_event.c 1.0 (3/31/2000) */ #include +#include #include #include @@ -83,9 +84,13 @@ #include #include #include +#include +#include +#include #include #include +#include #include #include #include @@ -93,16 +98,27 @@ #include #include #include +#include +#include +#include #include +#include + #include "net/net_str_id.h" #include +#include #if CONFIG_MEMORYSTATUS #include #endif +extern thread_t port_name_to_thread(mach_port_name_t port_name); /* osfmk/kern/ipc_tt.h */ +extern mach_port_name_t ipc_entry_name_mask(mach_port_name_t name); /* osfmk/ipc/ipc_entry.h */ + +#define KEV_EVTID(code) BSDDBG_CODE(DBG_BSD_KEVENT, (code)) + /* * JMM - this typedef needs to be unified with pthread_priority_t * and mach_msg_priority_t. It also needs to be the same type @@ -114,25 +130,25 @@ MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system"); #define KQ_EVENT NO_EVENT64 -static inline void kqlock(struct kqueue *kq); -static inline void kqunlock(struct kqueue *kq); - -static int kqlock2knoteuse(struct kqueue *kq, struct knote *kn); +#define KNUSE_NONE 0x0 +#define KNUSE_STEAL_DROP 0x1 +#define KNUSE_BOOST 0x2 +static int kqlock2knoteuse(struct kqueue *kq, struct knote *kn, int flags); static int kqlock2knotedrop(struct kqueue *kq, struct knote *kn); -static int kqlock2knotedetach(struct kqueue *kq, struct knote *kn); -static int knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int defer_drop); +static int kqlock2knotedetach(struct kqueue *kq, struct knote *kn, int flags); +static int knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int flags); static int kqueue_read(struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int kqueue_write(struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int kqueue_ioctl(struct fileproc *fp, u_long com, caddr_t data, - vfs_context_t ctx); + vfs_context_t ctx); static int kqueue_select(struct fileproc *fp, int which, void *wq_link_id, - vfs_context_t ctx); + vfs_context_t ctx); static int kqueue_close(struct fileglob *fg, vfs_context_t ctx); static int kqueue_kqfilter(struct fileproc *fp, struct knote *kn, - vfs_context_t ctx); + struct kevent_internal_s *kev, vfs_context_t ctx); static int kqueue_drain(struct fileproc *fp, vfs_context_t ctx); static const struct fileops kqueueops = { @@ -146,7 +162,9 @@ static const struct fileops kqueueops = { .fo_drain = kqueue_drain, }; -static int kevent_internal(struct proc *p, int fd, +static void kevent_put_kq(struct proc *p, kqueue_id_t id, struct fileproc *fp, struct kqueue *kq); +static int kevent_internal(struct proc *p, + kqueue_id_t id, kqueue_id_t *id_out, user_addr_t changelist, int nchanges, user_addr_t eventlist, int nevents, user_addr_t data_out, uint64_t data_available, @@ -165,10 +183,7 @@ static int kevent_callback(struct kqueue *kq, struct kevent_internal_s *kevp, static void kevent_continue(struct kqueue *kq, void *data, int error); static void kqueue_scan_continue(void *contp, wait_result_t wait_result); static int kqueue_process(struct kqueue *kq, kevent_callback_t callback, void *callback_data, - struct filt_process_s *process_data, kq_index_t servicer_qos_index, - int *countp, struct proc *p); -static int kqueue_begin_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int flags); -static void kqueue_end_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int flags); + struct filt_process_s *process_data, int *countp, struct proc *p); static struct kqtailq *kqueue_get_base_queue(struct kqueue *kq, kq_index_t qos_index); static struct kqtailq *kqueue_get_high_queue(struct kqueue *kq, kq_index_t qos_index); static int kqueue_queue_empty(struct kqueue *kq, kq_index_t qos_index); @@ -176,12 +191,62 @@ static int kqueue_queue_empty(struct kqueue *kq, kq_index_t qos_index); static struct kqtailq *kqueue_get_suppressed_queue(struct kqueue *kq, kq_index_t qos_index); static void kqworkq_request_thread(struct kqworkq *kqwq, kq_index_t qos_index); -static void kqworkq_request_help(struct kqworkq *kqwq, kq_index_t qos_index, uint32_t type); +static void kqworkq_request_help(struct kqworkq *kqwq, kq_index_t qos_index); static void kqworkq_update_override(struct kqworkq *kqwq, kq_index_t qos_index, kq_index_t override_index); -static void kqworkq_bind_thread(struct kqworkq *kqwq, kq_index_t qos_index, thread_t thread, unsigned int flags); +static void kqworkq_bind_thread_impl(struct kqworkq *kqwq, kq_index_t qos_index, thread_t thread, unsigned int flags); static void kqworkq_unbind_thread(struct kqworkq *kqwq, kq_index_t qos_index, thread_t thread, unsigned int flags); static struct kqrequest *kqworkq_get_request(struct kqworkq *kqwq, kq_index_t qos_index); +enum { + KQWL_UO_NONE = 0, + KQWL_UO_OLD_OVERRIDE_IS_SYNC_UI = 0x1, + KQWL_UO_NEW_OVERRIDE_IS_SYNC_UI = 0x2, + KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS = 0x4, + KQWL_UO_UPDATE_OVERRIDE_LAZY = 0x8 +}; + +static void kqworkloop_update_override(struct kqworkloop *kqwl, kq_index_t qos_index, kq_index_t override_index, uint32_t flags); +static void kqworkloop_bind_thread_impl(struct kqworkloop *kqwl, thread_t thread, unsigned int flags); +static void kqworkloop_unbind_thread(struct kqworkloop *kqwl, thread_t thread, unsigned int flags); +static inline kq_index_t kqworkloop_combined_qos(struct kqworkloop *kqwl, boolean_t *); +static void kqworkloop_update_suppress_sync_count(struct kqrequest *kqr, uint32_t flags); +enum { + KQWL_UTQ_NONE, + /* + * The wakeup qos is the qos of QUEUED knotes. + * + * This QoS is accounted for with the events override in the + * kqr_override_index field. It is raised each time a new knote is queued at + * a given QoS. The kqr_wakeup_indexes field is a superset of the non empty + * knote buckets and is recomputed after each event delivery. + */ + KQWL_UTQ_UPDATE_WAKEUP_QOS, + KQWL_UTQ_UPDATE_STAYACTIVE_QOS, + KQWL_UTQ_RECOMPUTE_WAKEUP_QOS, + /* + * The wakeup override is for suppressed knotes that have fired again at + * a higher QoS than the one for which they are suppressed already. + * This override is cleared when the knote suppressed list becomes empty. + */ + KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE, + KQWL_UTQ_RESET_WAKEUP_OVERRIDE, + /* + * The async QoS is the maximum QoS of an event enqueued on this workloop in + * userland. It is copied from the only EVFILT_WORKLOOP knote with + * a NOTE_WL_THREAD_REQUEST bit set allowed on this workloop. If there is no + * such knote, this QoS is 0. + */ + KQWL_UTQ_SET_ASYNC_QOS, + /* + * The sync waiters QoS is the maximum QoS of any thread blocked on an + * EVFILT_WORKLOOP knote marked with the NOTE_WL_SYNC_WAIT bit. + * If there is no such knote, this QoS is 0. + */ + KQWL_UTQ_SET_SYNC_WAITERS_QOS, + KQWL_UTQ_REDRIVE_EVENTS, +}; +static void kqworkloop_update_threads_qos(struct kqworkloop *kqwl, int op, kq_index_t qos); +static void kqworkloop_request_help(struct kqworkloop *kqwl, kq_index_t qos_index); static int knote_process(struct knote *kn, kevent_callback_t callback, void *callback_data, struct filt_process_s *process_data, struct proc *p); @@ -189,9 +254,10 @@ static int knote_process(struct knote *kn, kevent_callback_t callback, void *cal static void knote_put(struct knote *kn); #endif -static int knote_fdadd(struct knote *kn, struct proc *p); -static void knote_fdremove(struct knote *kn, struct proc *p); -static struct knote *knote_fdfind(struct kqueue *kq, struct kevent_internal_s *kev, struct proc *p); +static int kq_add_knote(struct kqueue *kq, struct knote *kn, + struct kevent_internal_s *kev, struct proc *p, int *knoteuse_flags); +static struct knote *kq_find_knote_and_kq_lock(struct kqueue *kq, struct kevent_internal_s *kev, bool is_fd, struct proc *p); +static void kq_remove_knote(struct kqueue *kq, struct knote *kn, struct proc *p, kn_status_t *kn_status, uint16_t *kq_state); static void knote_drop(struct knote *kn, struct proc *p); static struct knote *knote_alloc(void); @@ -212,15 +278,16 @@ static void knote_wakeup(struct knote *kn); static kq_index_t knote_get_queue_index(struct knote *kn); static struct kqtailq *knote_get_queue(struct knote *kn); -static struct kqtailq *knote_get_suppressed_queue(struct knote *kn); static kq_index_t knote_get_req_index(struct knote *kn); static kq_index_t knote_get_qos_index(struct knote *kn); static void knote_set_qos_index(struct knote *kn, kq_index_t qos_index); static kq_index_t knote_get_qos_override_index(struct knote *kn); -static void knote_set_qos_override_index(struct knote *kn, kq_index_t qos_index); +static kq_index_t knote_get_sync_qos_override_index(struct knote *kn); +static void knote_set_qos_override_index(struct knote *kn, kq_index_t qos_index, boolean_t override_is_sync); +static void knote_set_qos_overcommit(struct knote *kn); -static int filt_fileattach(struct knote *kn); -static struct filterops file_filtops = { +static int filt_fileattach(struct knote *kn, struct kevent_internal_s *kev); +SECURITY_READ_ONLY_EARLY(static struct filterops) file_filtops = { .f_isfd = 1, .f_attach = filt_fileattach, }; @@ -229,7 +296,7 @@ static void filt_kqdetach(struct knote *kn); static int filt_kqueue(struct knote *kn, long hint); static int filt_kqtouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_kqprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static struct filterops kqread_filtops = { +SECURITY_READ_ONLY_EARLY(static struct filterops) kqread_filtops = { .f_isfd = 1, .f_detach = filt_kqdetach, .f_event = filt_kqueue, @@ -238,17 +305,17 @@ static struct filterops kqread_filtops = { }; /* placeholder for not-yet-implemented filters */ -static int filt_badattach(struct knote *kn); -static struct filterops bad_filtops = { +static int filt_badattach(struct knote *kn, struct kevent_internal_s *kev); +SECURITY_READ_ONLY_EARLY(static struct filterops) bad_filtops = { .f_attach = filt_badattach, }; -static int filt_procattach(struct knote *kn); +static int filt_procattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_procdetach(struct knote *kn); static int filt_proc(struct knote *kn, long hint); static int filt_proctouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_procprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static struct filterops proc_filtops = { +SECURITY_READ_ONLY_EARLY(static struct filterops) proc_filtops = { .f_attach = filt_procattach, .f_detach = filt_procdetach, .f_event = filt_proc, @@ -257,60 +324,30 @@ static struct filterops proc_filtops = { }; #if CONFIG_MEMORYSTATUS -extern struct filterops memorystatus_filtops; +extern const struct filterops memorystatus_filtops; #endif /* CONFIG_MEMORYSTATUS */ -extern struct filterops fs_filtops; - -extern struct filterops sig_filtops; - -/* Timer filter */ -static int filt_timerattach(struct knote *kn); -static void filt_timerdetach(struct knote *kn); -static int filt_timer(struct knote *kn, long hint); -static int filt_timertouch(struct knote *kn, struct kevent_internal_s *kev); -static int filt_timerprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static struct filterops timer_filtops = { - .f_attach = filt_timerattach, - .f_detach = filt_timerdetach, - .f_event = filt_timer, - .f_touch = filt_timertouch, - .f_process = filt_timerprocess, -}; - -/* Helpers */ -static void filt_timerexpire(void *knx, void *param1); -static int filt_timervalidate(struct knote *kn); -static void filt_timerupdate(struct knote *kn, int num_fired); -static void filt_timercancel(struct knote *kn); +extern const struct filterops fs_filtops; -#define TIMER_RUNNING 0x1 -#define TIMER_CANCELWAIT 0x2 - -static lck_mtx_t _filt_timerlock; -static void filt_timerlock(void); -static void filt_timerunlock(void); +extern const struct filterops sig_filtops; static zone_t knote_zone; static zone_t kqfile_zone; static zone_t kqworkq_zone; +static zone_t kqworkloop_zone; #define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask)) -#if 0 -extern struct filterops aio_filtops; -#endif - /* Mach portset filter */ -extern struct filterops machport_filtops; +extern const struct filterops machport_filtops; /* User filter */ -static int filt_userattach(struct knote *kn); +static int filt_userattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_userdetach(struct knote *kn); static int filt_user(struct knote *kn, long hint); static int filt_usertouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_userprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static struct filterops user_filtops = { +SECURITY_READ_ONLY_EARLY(static struct filterops) user_filtops = { .f_attach = filt_userattach, .f_detach = filt_userdetach, .f_event = filt_user, @@ -322,20 +359,41 @@ static lck_spin_t _filt_userlock; static void filt_userlock(void); static void filt_userunlock(void); -extern struct filterops pipe_rfiltops; -extern struct filterops pipe_wfiltops; -extern struct filterops ptsd_kqops; -extern struct filterops soread_filtops; -extern struct filterops sowrite_filtops; -extern struct filterops sock_filtops; -extern struct filterops soexcept_filtops; -extern struct filterops spec_filtops; -extern struct filterops bpfread_filtops; -extern struct filterops necp_fd_rfiltops; -extern struct filterops skywalk_channel_rfiltops; -extern struct filterops skywalk_channel_wfiltops; -extern struct filterops fsevent_filtops; -extern struct filterops vnode_filtops; +/* Workloop filter */ +static bool filt_wlneeds_boost(struct kevent_internal_s *kev); +static int filt_wlattach(struct knote *kn, struct kevent_internal_s *kev); +static int filt_wlpost_attach(struct knote *kn, struct kevent_internal_s *kev); +static void filt_wldetach(struct knote *kn); +static int filt_wlevent(struct knote *kn, long hint); +static int filt_wltouch(struct knote *kn, struct kevent_internal_s *kev); +static int filt_wldrop_and_unlock(struct knote *kn, struct kevent_internal_s *kev); +static int filt_wlprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); +SECURITY_READ_ONLY_EARLY(static struct filterops) workloop_filtops = { + .f_needs_boost = filt_wlneeds_boost, + .f_attach = filt_wlattach, + .f_post_attach = filt_wlpost_attach, + .f_detach = filt_wldetach, + .f_event = filt_wlevent, + .f_touch = filt_wltouch, + .f_drop_and_unlock = filt_wldrop_and_unlock, + .f_process = filt_wlprocess, +}; + +extern const struct filterops pipe_rfiltops; +extern const struct filterops pipe_wfiltops; +extern const struct filterops ptsd_kqops; +extern const struct filterops soread_filtops; +extern const struct filterops sowrite_filtops; +extern const struct filterops sock_filtops; +extern const struct filterops soexcept_filtops; +extern const struct filterops spec_filtops; +extern const struct filterops bpfread_filtops; +extern const struct filterops necp_fd_rfiltops; +extern const struct filterops fsevent_filtops; +extern const struct filterops vnode_filtops; +extern const struct filterops tty_filtops; + +const static struct filterops timer_filtops; /* * @@ -353,7 +411,7 @@ extern struct filterops vnode_filtops; * - Add a filterops to the sysfilt_ops. Private filters should be added at the end of * the Private filters section of the array. */ -static struct filterops *sysfilt_ops[EVFILTID_MAX] = { +SECURITY_READ_ONLY_EARLY(static struct filterops *) sysfilt_ops[EVFILTID_MAX] = { /* Public Filters */ [~EVFILT_READ] = &file_filtops, [~EVFILT_WRITE] = &file_filtops, @@ -375,6 +433,8 @@ static struct filterops *sysfilt_ops[EVFILTID_MAX] = { #endif [~EVFILT_EXCEPT] = &file_filtops, + [~EVFILT_WORKLOOP] = &workloop_filtops, + /* Private filters */ [EVFILTID_KQREAD] = &kqread_filtops, [EVFILTID_PIPE_R] = &pipe_rfiltops, @@ -388,7 +448,8 @@ static struct filterops *sysfilt_ops[EVFILTID_MAX] = { [EVFILTID_BPFREAD] = &bpfread_filtops, [EVFILTID_NECP_FD] = &necp_fd_rfiltops, [EVFILTID_FSEVENT] = &fsevent_filtops, - [EVFILTID_VN] = &vnode_filtops + [EVFILTID_VN] = &vnode_filtops, + [EVFILTID_TTY] = &tty_filtops }; /* waitq prepost callback */ @@ -407,27 +468,51 @@ void waitq_set__CALLING_PREPOST_HOOK__(void *kq_hook, void *knote_hook, int qos) #define _PTHREAD_PRIORITY_QOS_CLASS_SHIFT_32 8 #endif +static inline __kdebug_only +uintptr_t +kqr_thread_id(struct kqrequest *kqr) +{ + return (uintptr_t)thread_tid(kqr->kqr_thread); +} + +static inline +boolean_t is_workqueue_thread(thread_t thread) +{ + return (thread_get_tag(thread) & THREAD_TAG_WORKQUEUE); +} + static inline -qos_t canonicalize_kevent_qos(qos_t qos) +void knote_canonicalize_kevent_qos(struct knote *kn) { + struct kqueue *kq = knote_get_kq(kn); unsigned long canonical; + if ((kq->kq_state & (KQ_WORKQ | KQ_WORKLOOP)) == 0) + return; + /* preserve manager and overcommit flags in this case */ - canonical = pthread_priority_canonicalize(qos, FALSE); - return (qos_t)canonical; + canonical = pthread_priority_canonicalize(kn->kn_qos, FALSE); + kn->kn_qos = (qos_t)canonical; } static inline -kq_index_t qos_index_from_qos(qos_t qos, boolean_t propagation) +kq_index_t qos_index_from_qos(struct knote *kn, qos_t qos, boolean_t propagation) { + struct kqueue *kq = knote_get_kq(kn); kq_index_t qos_index; unsigned long flags = 0; + if ((kq->kq_state & (KQ_WORKQ | KQ_WORKLOOP)) == 0) + return QOS_INDEX_KQFILE; + qos_index = (kq_index_t)thread_qos_from_pthread_priority( (unsigned long)qos, &flags); - if (!propagation && (flags & _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) - return KQWQ_QOS_MANAGER; + if (kq->kq_state & KQ_WORKQ) { + /* workq kqueues support requesting a manager thread (non-propagation) */ + if (!propagation && (flags & _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) + return KQWQ_QOS_MANAGER; + } return qos_index; } @@ -435,38 +520,42 @@ kq_index_t qos_index_from_qos(qos_t qos, boolean_t propagation) static inline qos_t qos_from_qos_index(kq_index_t qos_index) { - if (qos_index == KQWQ_QOS_MANAGER) + /* should only happen for KQ_WORKQ */ + if (qos_index == KQWQ_QOS_MANAGER) return _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG; if (qos_index == 0) - return 0; /* Unspecified */ + return THREAD_QOS_UNSPECIFIED; /* Should have support from pthread kext support */ return (1 << (qos_index - 1 + _PTHREAD_PRIORITY_QOS_CLASS_SHIFT_32)); } +/* kqr lock must be held */ +static inline +unsigned long pthread_priority_for_kqrequest( + struct kqrequest *kqr, + kq_index_t qos_index) +{ + unsigned long priority = qos_from_qos_index(qos_index); + if (kqr->kqr_state & KQR_THOVERCOMMIT) { + priority |= _PTHREAD_PRIORITY_OVERCOMMIT_FLAG; + } + return priority; +} + static inline kq_index_t qos_index_for_servicer(int qos_class, thread_t thread, int flags) { +#pragma unused(thread) kq_index_t qos_index; if (flags & KEVENT_FLAG_WORKQ_MANAGER) return KQWQ_QOS_MANAGER; - /* - * If the caller didn't pass in a class (legacy pthread kext) - * the we use the thread policy QoS of the current thread. - */ - assert(qos_class != -1); - if (qos_class == -1) - qos_index = proc_get_thread_policy(thread, - TASK_POLICY_ATTRIBUTE, - TASK_POLICY_QOS); - else - qos_index = (kq_index_t)qos_class; - - assert(qos_index > 0 && qos_index < KQWQ_NQOS); + qos_index = (kq_index_t)qos_class; + assert(qos_index > 0 && qos_index < KQWQ_QOS_MANAGER); return qos_index; } @@ -499,12 +588,30 @@ kqlock(struct kqueue *kq) lck_spin_lock(&kq->kq_lock); } +static inline void +kqlock_held(__assert_only struct kqueue *kq) +{ + LCK_SPIN_ASSERT(&kq->kq_lock, LCK_ASSERT_OWNED); +} + static inline void kqunlock(struct kqueue *kq) { lck_spin_unlock(&kq->kq_lock); } +static inline void +knhash_lock(proc_t p) +{ + lck_mtx_lock(&p->p_fd->fd_knhashlock); +} + +static inline void +knhash_unlock(proc_t p) +{ + lck_mtx_unlock(&p->p_fd->fd_knhashlock); +} + /* * Convert a kq lock to a knote use referece. @@ -517,17 +624,44 @@ kqunlock(struct kqueue *kq) * - unlock on exit if we get the use reference */ static int -kqlock2knoteuse(struct kqueue *kq, struct knote *kn) +kqlock2knoteuse(struct kqueue *kq, struct knote *kn, int flags) { if (kn->kn_status & (KN_DROPPING | KN_VANISHED)) return (0); assert(kn->kn_status & KN_ATTACHED); kn->kn_inuse++; + if (flags & KNUSE_BOOST) { + set_thread_rwlock_boost(); + } kqunlock(kq); return (1); } +/* + * - kq locked at entry + * - kq unlocked at exit + */ +__disable_tail_calls +static wait_result_t +knoteusewait(struct kqueue *kq, struct knote *kn) +{ + kn->kn_status |= KN_USEWAIT; + waitq_assert_wait64((struct waitq *)&kq->kq_wqs, + CAST_EVENT64_T(&kn->kn_status), + THREAD_UNINT, TIMEOUT_WAIT_FOREVER); + kqunlock(kq); + return thread_block(THREAD_CONTINUE_NULL); +} + +static bool +knoteuse_needs_boost(struct knote *kn, struct kevent_internal_s *kev) +{ + if (knote_fops(kn)->f_needs_boost) { + return knote_fops(kn)->f_needs_boost(kev); + } + return false; +} /* * Convert from a knote use reference back to kq lock. @@ -536,7 +670,7 @@ kqlock2knoteuse(struct kqueue *kq, struct knote *kn) * this is the last one. * * If someone is trying to drop the knote, but the - * caller has events they must deliver, take + * caller has events they must deliver, take * responsibility for the drop later - and wake the * other attempted dropper in a manner that informs * him of the transfer of responsibility. @@ -548,11 +682,16 @@ kqlock2knoteuse(struct kqueue *kq, struct knote *kn) * The kqueue lock is re-taken unconditionally. */ static int -knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int steal_drop) +knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int flags) { int dropped = 0; + int steal_drop = (flags & KNUSE_STEAL_DROP); kqlock(kq); + if (flags & KNUSE_BOOST) { + clear_thread_rwlock_boost(); + } + if (--kn->kn_inuse == 0) { if ((kn->kn_status & KN_ATTACHING) != 0) { @@ -591,12 +730,7 @@ knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int steal_drop) kn->kn_status |= KN_STOLENDROP; /* but we now have to wait to be the last ref */ - kn->kn_status |= KN_USEWAIT; - waitq_assert_wait64((struct waitq *)&kq->kq_wqs, - CAST_EVENT64_T(&kn->kn_status), - THREAD_UNINT, TIMEOUT_WAIT_FOREVER); - kqunlock(kq); - thread_block(THREAD_CONTINUE_NULL); + knoteusewait(kq, kn); kqlock(kq); } else { dropped = 1; @@ -620,25 +754,23 @@ knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int steal_drop) * (caller will start over at lookup). * * - kq locked at entry - * - unlocked on exit + * - unlocked on exit */ static int -kqlock2knotedetach(struct kqueue *kq, struct knote *kn) +kqlock2knotedetach(struct kqueue *kq, struct knote *kn, int flags) { if ((kn->kn_status & KN_DROPPING) || kn->kn_inuse) { /* have to wait for dropper or current uses to go away */ - kn->kn_status |= KN_USEWAIT; - waitq_assert_wait64((struct waitq *)&kq->kq_wqs, - CAST_EVENT64_T(&kn->kn_status), - THREAD_UNINT, TIMEOUT_WAIT_FOREVER); - kqunlock(kq); - thread_block(THREAD_CONTINUE_NULL); + knoteusewait(kq, kn); return (0); } assert((kn->kn_status & KN_VANISHED) == 0); assert(kn->kn_status & KN_ATTACHED); kn->kn_status &= ~KN_ATTACHED; kn->kn_status |= KN_VANISHED; + if (flags & KNUSE_BOOST) { + clear_thread_rwlock_boost(); + } kn->kn_inuse++; kqunlock(kq); return (1); @@ -675,12 +807,7 @@ kqlock2knotedrop(struct kqueue *kq, struct knote *kn) return (oktodrop); } } - kn->kn_status |= KN_USEWAIT; - waitq_assert_wait64((struct waitq *)&kq->kq_wqs, - CAST_EVENT64_T(&kn->kn_status), - THREAD_UNINT, TIMEOUT_WAIT_FOREVER); - kqunlock(kq); - result = thread_block(THREAD_CONTINUE_NULL); + result = knoteusewait(kq, kn); /* THREAD_RESTART == another thread stole the knote drop */ return (result == THREAD_AWAKENED); } @@ -709,9 +836,9 @@ knote_put(struct knote *kn) #endif static int -filt_fileattach(struct knote *kn) +filt_fileattach(struct knote *kn, struct kevent_internal_s *kev) { - return (fo_kqfilter(kn->kn_fp, kn, vfs_context_current())); + return (fo_kqfilter(kn->kn_fp, kn, kev, vfs_context_current())); } #define f_flag f_fglob->fg_flag @@ -781,8 +908,10 @@ filt_kqprocess(struct knote *kn, struct filt_process_s *data, struct kevent_inte return res; } +#pragma mark EVFILT_PROC + static int -filt_procattach(struct knote *kn) +filt_procattach(struct knote *kn, __unused struct kevent_internal_s *kev) { struct proc *p; @@ -1007,32 +1136,95 @@ filt_procprocess(struct knote *kn, struct filt_process_s *data, struct kevent_in return res; } + +#pragma mark EVFILT_TIMER + + /* - * filt_timervalidate - process data from user + * Values stored in the knote at rest (using Mach absolute time units) * - * Converts to either interval or deadline format. + * kn->kn_hook where the thread_call object is stored + * kn->kn_ext[0] next deadline or 0 if immediate expiration + * kn->kn_ext[1] leeway value + * kn->kn_sdata interval timer: the interval + * absolute/deadline timer: 0 + * kn->kn_data fire count + */ + +static lck_mtx_t _filt_timerlock; + +static void filt_timerlock(void) { lck_mtx_lock(&_filt_timerlock); } +static void filt_timerunlock(void) { lck_mtx_unlock(&_filt_timerlock); } + +static inline void filt_timer_assert_locked(void) +{ + LCK_MTX_ASSERT(&_filt_timerlock, LCK_MTX_ASSERT_OWNED); +} + +/* state flags stored in kn_hookid */ +#define TIMER_RUNNING 0x1 +#define TIMER_CANCELWAIT 0x2 + +/* + * filt_timervalidate - process data from user * - * The saved-data field in the knote contains the - * time value. The saved filter-flags indicates - * the unit of measurement. + * Sets up the deadline, interval, and leeway from the provided user data * - * After validation, either the saved-data field - * contains the interval in absolute time, or ext[0] - * contains the expected deadline. If that deadline - * is in the past, ext[0] is 0. + * Input: + * kn_sdata timer deadline or interval time + * kn_sfflags style of timer, unit of measurement * - * Returns EINVAL for unrecognized units of time. + * Output: + * kn_sdata either interval in abstime or 0 if non-repeating timer + * ext[0] fire deadline in abs/cont time + * (or 0 if NOTE_ABSOLUTE and deadline is in past) * - * Timer filter lock is held. + * Returns: + * EINVAL Invalid user data parameters * + * Called with timer filter lock held. */ static int filt_timervalidate(struct knote *kn) { + /* + * There are 4 knobs that need to be chosen for a timer registration: + * + * A) Units of time (what is the time duration of the specified number) + * Absolute and interval take: + * NOTE_SECONDS, NOTE_USECONDS, NOTE_NSECONDS, NOTE_MACHTIME + * Defaults to milliseconds if not specified + * + * B) Clock epoch (what is the zero point of the specified number) + * For interval, there is none + * For absolute, defaults to the gettimeofday/calendar epoch + * With NOTE_MACHTIME, uses mach_absolute_time() + * With NOTE_MACHTIME and NOTE_MACH_CONTINUOUS_TIME, uses mach_continuous_time() + * + * C) The knote's behavior on delivery + * Interval timer causes the knote to arm for the next interval unless one-shot is set + * Absolute is a forced one-shot timer which deletes on delivery + * TODO: Add a way for absolute to be not forced one-shot + * + * D) Whether the time duration is relative to now or absolute + * Interval fires at now + duration when it is set up + * Absolute fires at now + difference between now walltime and passed in walltime + * With NOTE_MACHTIME it fires at an absolute MAT or MCT. + * + * E) Whether the timer continues to tick across sleep + * By default all three do not. + * For interval and absolute, NOTE_MACH_CONTINUOUS_TIME causes them to tick across sleep + * With NOTE_ABSOLUTE | NOTE_MACHTIME | NOTE_MACH_CONTINUOUS_TIME: + * expires when mach_continuous_time() is > the passed in value. + */ + + filt_timer_assert_locked(); + uint64_t multiplier; - uint64_t raw = 0; - switch (kn->kn_sfflags & (NOTE_SECONDS|NOTE_USECONDS|NOTE_NSECONDS)) { + boolean_t use_abstime = FALSE; + + switch (kn->kn_sfflags & (NOTE_SECONDS|NOTE_USECONDS|NOTE_NSECONDS|NOTE_MACHTIME)) { case NOTE_SECONDS: multiplier = NSEC_PER_SEC; break; @@ -1042,6 +1234,10 @@ filt_timervalidate(struct knote *kn) case NOTE_NSECONDS: multiplier = 1; break; + case NOTE_MACHTIME: + multiplier = 0; + use_abstime = TRUE; + break; case 0: /* milliseconds (default) */ multiplier = NSEC_PER_SEC / 1000; break; @@ -1049,89 +1245,123 @@ filt_timervalidate(struct knote *kn) return (EINVAL); } - /* transform the slop delta(leeway) in kn_ext[1] if passed to same time scale */ - if(kn->kn_sfflags & NOTE_LEEWAY){ - nanoseconds_to_absolutetime((uint64_t)kn->kn_ext[1] * multiplier, &raw); - kn->kn_ext[1] = raw; - } + /* transform the leeway in kn_ext[1] to same time scale */ + if (kn->kn_sfflags & NOTE_LEEWAY) { + uint64_t leeway_abs; + + if (use_abstime) { + leeway_abs = (uint64_t)kn->kn_ext[1]; + } else { + uint64_t leeway_ns; + if (os_mul_overflow((uint64_t)kn->kn_ext[1], multiplier, &leeway_ns)) + return (ERANGE); - nanoseconds_to_absolutetime((uint64_t)kn->kn_sdata * multiplier, &raw); + nanoseconds_to_absolutetime(leeway_ns, &leeway_abs); + } - kn->kn_ext[0] = 0; - kn->kn_sdata = 0; + kn->kn_ext[1] = leeway_abs; + } if (kn->kn_sfflags & NOTE_ABSOLUTE) { - clock_sec_t seconds; - clock_nsec_t nanoseconds; - uint64_t now; + uint64_t deadline_abs; + + if (use_abstime) { + deadline_abs = (uint64_t)kn->kn_sdata; + } else { + uint64_t calendar_deadline_ns; + + if (os_mul_overflow((uint64_t)kn->kn_sdata, multiplier, &calendar_deadline_ns)) + return (ERANGE); + + /* calendar_deadline_ns is in nanoseconds since the epoch */ + + clock_sec_t seconds; + clock_nsec_t nanoseconds; + + /* + * Note that the conversion through wall-time is only done once. + * + * If the relationship between MAT and gettimeofday changes, + * the underlying timer does not update. + * + * TODO: build a wall-time denominated timer_call queue + * and a flag to request DTRTing with wall-time timers + */ + clock_get_calendar_nanotime(&seconds, &nanoseconds); + + uint64_t calendar_now_ns = (uint64_t)seconds * NSEC_PER_SEC + nanoseconds; - clock_get_calendar_nanotime(&seconds, &nanoseconds); - nanoseconds_to_absolutetime((uint64_t)seconds * NSEC_PER_SEC + - nanoseconds, &now); + /* if deadline is in the future */ + if (calendar_now_ns < calendar_deadline_ns) { + uint64_t interval_ns = calendar_deadline_ns - calendar_now_ns; + uint64_t interval_abs; - /* if time is in the future */ - if (now < raw) { - raw -= now; + nanoseconds_to_absolutetime(interval_ns, &interval_abs); + + /* + * Note that the NOTE_MACH_CONTINUOUS_TIME flag here only + * causes the timer to keep ticking across sleep, but + * it does not change the calendar timebase. + */ - if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) { - clock_continuoustime_interval_to_deadline(raw, - &kn->kn_ext[0]); + if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) + clock_continuoustime_interval_to_deadline(interval_abs, + &deadline_abs); + else + clock_absolutetime_interval_to_deadline(interval_abs, + &deadline_abs); } else { - clock_absolutetime_interval_to_deadline(raw, - &kn->kn_ext[0]); + deadline_abs = 0; /* cause immediate expiration */ } } + + kn->kn_ext[0] = deadline_abs; + kn->kn_sdata = 0; /* NOTE_ABSOLUTE is non-repeating */ + } else if (kn->kn_sdata < 0) { + /* + * Negative interval timers fire immediately, once. + * + * Ideally a negative interval would be an error, but certain clients + * pass negative values on accident, and expect an event back. + * + * In the old implementation the timer would repeat with no delay + * N times until mach_absolute_time() + (N * interval) underflowed, + * then it would wait ~forever by accidentally arming a timer for the far future. + * + * We now skip the power-wasting hot spin phase and go straight to the idle phase. + */ + + kn->kn_sdata = 0; /* non-repeating */ + kn->kn_ext[0] = 0; /* expire immediately */ } else { - kn->kn_sdata = raw; + uint64_t interval_abs = 0; + + if (use_abstime) { + interval_abs = (uint64_t)kn->kn_sdata; + } else { + uint64_t interval_ns; + if (os_mul_overflow((uint64_t)kn->kn_sdata, multiplier, &interval_ns)) + return (ERANGE); + + nanoseconds_to_absolutetime(interval_ns, &interval_abs); + } + + uint64_t deadline = 0; + + if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) + clock_continuoustime_interval_to_deadline(interval_abs, &deadline); + else + clock_absolutetime_interval_to_deadline(interval_abs, &deadline); + + kn->kn_sdata = interval_abs; /* default to a repeating timer */ + kn->kn_ext[0] = deadline; } return (0); } -/* - * filt_timerupdate - compute the next deadline - * - * Repeating timers store their interval in kn_sdata. Absolute - * timers have already calculated the deadline, stored in ext[0]. - * - * On return, the next deadline (or zero if no deadline is needed) - * is stored in kn_ext[0]. - * - * Timer filter lock is held. - */ -static void -filt_timerupdate(struct knote *kn, int num_fired) -{ - assert(num_fired > 0); - /* if there's no interval, deadline is just in kn_ext[0] */ - if (kn->kn_sdata == 0) - return; - /* if timer hasn't fired before, fire in interval nsecs */ - if (kn->kn_ext[0] == 0) { - assert(num_fired == 1); - if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) { - clock_continuoustime_interval_to_deadline(kn->kn_sdata, - &kn->kn_ext[0]); - } else { - clock_absolutetime_interval_to_deadline(kn->kn_sdata, - &kn->kn_ext[0]); - } - } else { - /* - * If timer has fired before, schedule the next pop - * relative to the last intended deadline. - * - * We could check for whether the deadline has expired, - * but the thread call layer can handle that. - * - * Go forward an additional number of periods, in the case the - * timer fired multiple times while the system was asleep. - */ - kn->kn_ext[0] += (kn->kn_sdata * num_fired); - } -} /* * filt_timerexpire - the timer callout routine @@ -1155,6 +1385,7 @@ filt_timerexpire(void *knx, __unused void *spare) /* no "object" for timers, so fake a list */ SLIST_INIT(&timer_list); SLIST_INSERT_HEAD(&timer_list, kn, kn_selnext); + KNOTE(&timer_list, 1); /* if someone is waiting for timer to pop */ @@ -1164,6 +1395,8 @@ filt_timerexpire(void *knx, __unused void *spare) CAST_EVENT64_T(&kn->kn_hook), THREAD_AWAKENED, WAITQ_ALL_PRIORITIES); + + kn->kn_hookid &= ~TIMER_CANCELWAIT; } filt_timerunlock(); @@ -1172,44 +1405,114 @@ filt_timerexpire(void *knx, __unused void *spare) /* * Cancel a running timer (or wait for the pop). * Timer filter lock is held. + * May drop and retake the timer filter lock. */ static void filt_timercancel(struct knote *kn) { - struct kqueue *kq = knote_get_kq(kn); - thread_call_t callout = kn->kn_hook; - boolean_t cancelled; - - if (kn->kn_hookid & TIMER_RUNNING) { - /* cancel the callout if we can */ - cancelled = thread_call_cancel(callout); - if (cancelled) { - kn->kn_hookid &= ~TIMER_RUNNING; - } else { - /* we have to wait for the expire routine. */ - kn->kn_hookid |= TIMER_CANCELWAIT; - waitq_assert_wait64((struct waitq *)&kq->kq_wqs, - CAST_EVENT64_T(&kn->kn_hook), - THREAD_UNINT, TIMEOUT_WAIT_FOREVER); - filt_timerunlock(); - thread_block(THREAD_CONTINUE_NULL); - filt_timerlock(); - assert((kn->kn_hookid & TIMER_RUNNING) == 0); - } + filt_timer_assert_locked(); + + assert((kn->kn_hookid & TIMER_CANCELWAIT) == 0); + + /* if no timer, then we're good */ + if ((kn->kn_hookid & TIMER_RUNNING) == 0) + return; + + thread_call_t callout = (thread_call_t)kn->kn_hook; + + /* cancel the callout if we can */ + if (thread_call_cancel(callout)) { + kn->kn_hookid &= ~TIMER_RUNNING; + return; } + + /* cancel failed, we have to wait for the in-flight expire routine */ + + kn->kn_hookid |= TIMER_CANCELWAIT; + + struct kqueue *kq = knote_get_kq(kn); + + waitq_assert_wait64((struct waitq *)&kq->kq_wqs, + CAST_EVENT64_T(&kn->kn_hook), + THREAD_UNINT, TIMEOUT_WAIT_FOREVER); + + filt_timerunlock(); + thread_block(THREAD_CONTINUE_NULL); + filt_timerlock(); + + assert((kn->kn_hookid & TIMER_CANCELWAIT) == 0); + assert((kn->kn_hookid & TIMER_RUNNING) == 0); +} + +static void +filt_timerarm(struct knote *kn) +{ + filt_timer_assert_locked(); + + assert((kn->kn_hookid & TIMER_RUNNING) == 0); + + thread_call_t callout = (thread_call_t)kn->kn_hook; + + uint64_t deadline = kn->kn_ext[0]; + uint64_t leeway = kn->kn_ext[1]; + + int filter_flags = kn->kn_sfflags; + unsigned int timer_flags = 0; + + if (filter_flags & NOTE_CRITICAL) + timer_flags |= THREAD_CALL_DELAY_USER_CRITICAL; + else if (filter_flags & NOTE_BACKGROUND) + timer_flags |= THREAD_CALL_DELAY_USER_BACKGROUND; + else + timer_flags |= THREAD_CALL_DELAY_USER_NORMAL; + + if (filter_flags & NOTE_LEEWAY) + timer_flags |= THREAD_CALL_DELAY_LEEWAY; + + if (filter_flags & NOTE_MACH_CONTINUOUS_TIME) + timer_flags |= THREAD_CALL_CONTINUOUS; + + thread_call_enter_delayed_with_leeway(callout, NULL, + deadline, leeway, + timer_flags); + + kn->kn_hookid |= TIMER_RUNNING; +} + +/* + * Does this knote need a timer armed for it, or should it be ready immediately? + */ +static boolean_t +filt_timer_is_ready(struct knote *kn) +{ + uint64_t now; + + if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) + now = mach_continuous_time(); + else + now = mach_absolute_time(); + + uint64_t deadline = kn->kn_ext[0]; + + if (deadline < now) + return TRUE; + else + return FALSE; } /* * Allocate a thread call for the knote's lifetime, and kick off the timer. */ static int -filt_timerattach(struct knote *kn) +filt_timerattach(struct knote *kn, __unused struct kevent_internal_s *kev) { thread_call_t callout; int error; - int res; - callout = thread_call_allocate(filt_timerexpire, kn); + callout = thread_call_allocate_with_options(filt_timerexpire, + (thread_call_param_t)kn, THREAD_CALL_PRIORITY_HIGH, + THREAD_CALL_OPTIONS_ONCE); + if (NULL == callout) { kn->kn_flags = EV_ERROR; kn->kn_data = ENOMEM; @@ -1217,52 +1520,37 @@ filt_timerattach(struct knote *kn) } filt_timerlock(); - error = filt_timervalidate(kn); - if (error != 0) { - filt_timerunlock(); - thread_call_free(callout); + + if ((error = filt_timervalidate(kn)) != 0) { kn->kn_flags = EV_ERROR; - kn->kn_data = error; + kn->kn_data = error; + filt_timerunlock(); + + __assert_only boolean_t freed = thread_call_free(callout); + assert(freed); return 0; } kn->kn_hook = (void*)callout; kn->kn_hookid = 0; + kn->kn_flags |= EV_CLEAR; - /* absolute=EV_ONESHOT */ + /* NOTE_ABSOLUTE implies EV_ONESHOT */ if (kn->kn_sfflags & NOTE_ABSOLUTE) kn->kn_flags |= EV_ONESHOT; - filt_timerupdate(kn, 1); - if (kn->kn_ext[0]) { - kn->kn_flags |= EV_CLEAR; - unsigned int timer_flags = 0; - if (kn->kn_sfflags & NOTE_CRITICAL) - timer_flags |= THREAD_CALL_DELAY_USER_CRITICAL; - else if (kn->kn_sfflags & NOTE_BACKGROUND) - timer_flags |= THREAD_CALL_DELAY_USER_BACKGROUND; - else - timer_flags |= THREAD_CALL_DELAY_USER_NORMAL; - - if (kn->kn_sfflags & NOTE_LEEWAY) - timer_flags |= THREAD_CALL_DELAY_LEEWAY; - if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) - timer_flags |= THREAD_CALL_CONTINUOUS; - - thread_call_enter_delayed_with_leeway(callout, NULL, - kn->kn_ext[0], kn->kn_ext[1], timer_flags); + boolean_t timer_ready = FALSE; - kn->kn_hookid |= TIMER_RUNNING; - } else { - /* fake immediate */ + if ((timer_ready = filt_timer_is_ready(kn))) { + /* cause immediate expiration */ kn->kn_data = 1; + } else { + filt_timerarm(kn); } - res = (kn->kn_data > 0); - filt_timerunlock(); - return res; + return timer_ready; } /* @@ -1280,93 +1568,25 @@ filt_timerdetach(struct knote *kn) filt_timerunlock(); - thread_call_free(callout); -} - - -static int filt_timer_num_fired(struct knote *kn) -{ - /* by default we fire a timer once */ - int num_fired = 1; - - /* - * When the time base is mach_continuous_time, we have to calculate - * the number of times the timer fired while we were asleep. - */ - if ((kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) && - (kn->kn_sdata != 0) && - (kn->kn_ext[0] != 0)) - { - const uint64_t now = mach_continuous_time(); - // time for timer to fire (right now) is kn_ext[0] - // kn_sdata is period for timer to fire - assert(now >= kn->kn_ext[0]); - assert(kn->kn_sdata > 0); - - const uint64_t overrun_ticks = now - kn->kn_ext[0]; - const uint64_t kn_sdata = kn->kn_sdata; - - if (overrun_ticks < kn_sdata) { - num_fired = 1; - } else if (overrun_ticks < (kn_sdata << 1)) { - num_fired = 2; - } else { - num_fired = (overrun_ticks / kn_sdata) + 1; - } - } - - return num_fired; + __assert_only boolean_t freed = thread_call_free(callout); + assert(freed); } /* - * filt_timer - post events to a timer knote + * filt_timerevent - post events to a timer knote * - * Count the timer fire and re-arm as requested. - * This always crosses the threshold of interest, - * so always return an indication that the knote - * should be activated (if not already). + * Called in the context of filt_timerexpire with + * the filt_timerlock held */ static int -filt_timer( - struct knote *kn, - long hint) +filt_timerevent(struct knote *kn, __unused long hint) { -#pragma unused(hint) - - /* real timer pop -- timer lock held by filt_timerexpire */ - int num_fired = filt_timer_num_fired(kn); - kn->kn_data += num_fired; - - if (((kn->kn_hookid & TIMER_CANCELWAIT) == 0) && - ((kn->kn_flags & EV_ONESHOT) == 0)) { - /* evaluate next time to fire */ - filt_timerupdate(kn, num_fired); - - if (kn->kn_ext[0]) { - unsigned int timer_flags = 0; - - /* keep the callout and re-arm */ - if (kn->kn_sfflags & NOTE_CRITICAL) - timer_flags |= THREAD_CALL_DELAY_USER_CRITICAL; - else if (kn->kn_sfflags & NOTE_BACKGROUND) - timer_flags |= THREAD_CALL_DELAY_USER_BACKGROUND; - else - timer_flags |= THREAD_CALL_DELAY_USER_NORMAL; - - if (kn->kn_sfflags & NOTE_LEEWAY) - timer_flags |= THREAD_CALL_DELAY_LEEWAY; + filt_timer_assert_locked(); - thread_call_enter_delayed_with_leeway(kn->kn_hook, NULL, - kn->kn_ext[0], kn->kn_ext[1], timer_flags); - - kn->kn_hookid |= TIMER_RUNNING; - } - } + kn->kn_data = 1; return (1); } - - /* * filt_timertouch - update timer knote with new user input * @@ -1380,13 +1600,18 @@ filt_timertouch( struct kevent_internal_s *kev) { int error; - int res; filt_timerlock(); - /* cancel current call */ + /* + * cancel current call - drops and retakes lock + * TODO: not safe against concurrent touches? + */ filt_timercancel(kn); + /* clear if the timer had previously fired, the user no longer wants to see it */ + kn->kn_data = 0; + /* capture the new values used to compute deadline */ kn->kn_sdata = kev->data; kn->kn_sfflags = kev->fflags; @@ -1400,42 +1625,24 @@ filt_timertouch( error = filt_timervalidate(kn); if (error) { /* no way to report error, so mark it in the knote */ - filt_timerunlock(); kn->kn_flags |= EV_ERROR; kn->kn_data = error; + filt_timerunlock(); return 1; } - /* start timer if necessary */ - filt_timerupdate(kn, 1); - - if (kn->kn_ext[0]) { - unsigned int timer_flags = 0; - if (kn->kn_sfflags & NOTE_CRITICAL) - timer_flags |= THREAD_CALL_DELAY_USER_CRITICAL; - else if (kn->kn_sfflags & NOTE_BACKGROUND) - timer_flags |= THREAD_CALL_DELAY_USER_BACKGROUND; - else - timer_flags |= THREAD_CALL_DELAY_USER_NORMAL; - - if (kn->kn_sfflags & NOTE_LEEWAY) - timer_flags |= THREAD_CALL_DELAY_LEEWAY; - - thread_call_enter_delayed_with_leeway(kn->kn_hook, NULL, - kn->kn_ext[0], kn->kn_ext[1], timer_flags); + boolean_t timer_ready = FALSE; - kn->kn_hookid |= TIMER_RUNNING; - } else { - /* pretend the timer has fired */ + if ((timer_ready = filt_timer_is_ready(kn))) { + /* cause immediate expiration */ kn->kn_data = 1; + } else { + filt_timerarm(kn); } - /* capture if already fired */ - res = (kn->kn_data > 0); - filt_timerunlock(); - return res; + return timer_ready; } /* @@ -1453,43 +1660,114 @@ filt_timerprocess( { filt_timerlock(); - /* user-query */ - if (kn->kn_data == 0) { + if (kn->kn_data == 0 || (kn->kn_hookid & TIMER_CANCELWAIT)) { + /* + * kn_data = 0: + * The timer hasn't yet fired, so there's nothing to deliver + * TIMER_CANCELWAIT: + * touch is in the middle of canceling the timer, + * so don't deliver or re-arm anything + * + * This can happen if a touch resets a timer that had fired + * without being processed + */ filt_timerunlock(); return 0; } + if (kn->kn_sdata != 0 && ((kn->kn_flags & EV_ERROR) == 0)) { + /* + * This is a 'repeating' timer, so we have to emit + * how many intervals expired between the arm + * and the process. + * + * A very strange style of interface, because + * this could easily be done in the client... + */ + + /* The timer better have had expired... */ + assert((kn->kn_hookid & TIMER_RUNNING) == 0); + + uint64_t now; + + if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) + now = mach_continuous_time(); + else + now = mach_absolute_time(); + + uint64_t first_deadline = kn->kn_ext[0]; + uint64_t interval_abs = kn->kn_sdata; + uint64_t orig_arm_time = first_deadline - interval_abs; + + assert(now > orig_arm_time); + assert(now > first_deadline); + + uint64_t elapsed = now - orig_arm_time; + + uint64_t num_fired = elapsed / interval_abs; + + /* + * To reach this code, we must have seen the timer pop + * and be in repeating mode, so therefore it must have been + * more than 'interval' time since the attach or last + * successful touch. + * + * An unsuccessful touch would: + * disarm the timer + * clear kn_data + * clear kn_sdata + * set EV_ERROR + * all of which will prevent this code from running. + */ + assert(num_fired > 0); + + /* report how many intervals have elapsed to the user */ + kn->kn_data = (int64_t) num_fired; + + /* We only need to re-arm the timer if it's not about to be destroyed */ + if ((kn->kn_flags & EV_ONESHOT) == 0) { + /* fire at the end of the next interval */ + uint64_t new_deadline = first_deadline + num_fired * interval_abs; + + assert(new_deadline > now); + + kn->kn_ext[0] = new_deadline; + + filt_timerarm(kn); + } + } + /* * Copy out the interesting kevent state, * but don't leak out the raw time calculations. + * + * TODO: potential enhancements - tell the user about: + * - deadline to which this timer thought it was expiring + * - return kn_sfflags in the fflags field so the client can know + * under what flags the timer fired */ *kev = kn->kn_kevent; kev->ext[0] = 0; /* kev->ext[1] = 0; JMM - shouldn't we hide this too? */ - /* - * reset the timer pop count in kn_data - * and (optionally) clear the fflags. - */ + /* we have delivered the event, reset the timer pop count */ kn->kn_data = 0; - if (kn->kn_flags & EV_CLEAR) - kn->kn_fflags = 0; filt_timerunlock(); return 1; } -static void -filt_timerlock(void) -{ - lck_mtx_lock(&_filt_timerlock); -} +SECURITY_READ_ONLY_EARLY(static struct filterops) timer_filtops = { + .f_attach = filt_timerattach, + .f_detach = filt_timerdetach, + .f_event = filt_timerevent, + .f_touch = filt_timertouch, + .f_process = filt_timerprocess, +}; + + +#pragma mark EVFILT_USER -static void -filt_timerunlock(void) -{ - lck_mtx_unlock(&_filt_timerlock); -} static void filt_userlock(void) @@ -1504,12 +1782,12 @@ filt_userunlock(void) } static int -filt_userattach(struct knote *kn) +filt_userattach(struct knote *kn, __unused struct kevent_internal_s *kev) { /* EVFILT_USER knotes are not attached to anything in the kernel */ /* Cant discover this knote until after attach - so no lock needed */ kn->kn_hook = NULL; - if (kn->kn_fflags & NOTE_TRIGGER) { + if (kn->kn_sfflags & NOTE_TRIGGER) { kn->kn_hookid = 1; } else { kn->kn_hookid = 0; @@ -1599,203 +1877,1334 @@ filt_userprocess( return 1; } +#pragma mark EVFILT_WORKLOOP + +#if DEBUG || DEVELOPMENT /* - * JMM - placeholder for not-yet-implemented filters + * see src/queue_internal.h in libdispatch */ -static int -filt_badattach(__unused struct knote *kn) +#define DISPATCH_QUEUE_ENQUEUED 0x1ull +#endif + +static inline void +filt_wllock(struct kqworkloop *kqwl) { - kn->kn_flags |= EV_ERROR; - kn->kn_data = ENOTSUP; - return 0; + lck_mtx_lock(&kqwl->kqwl_statelock); } -struct kqueue * -kqueue_alloc(struct proc *p, unsigned int flags) +static inline void +filt_wlunlock(struct kqworkloop *kqwl) { - struct filedesc *fdp = p->p_fd; - struct kqueue *kq = NULL; - int policy; - void *hook; - uint64_t kq_addr_offset; + lck_mtx_unlock(&kqwl->kqwl_statelock); +} - if (flags & KEVENT_FLAG_WORKQ) { - struct kqworkq *kqwq; - int i; +static inline void +filt_wlheld(__assert_only struct kqworkloop *kqwl) +{ + LCK_MTX_ASSERT(&kqwl->kqwl_statelock, LCK_MTX_ASSERT_OWNED); +} - kqwq = (struct kqworkq *)zalloc(kqworkq_zone); - if (kqwq == NULL) - return NULL; +#define WL_OWNER_SUSPENDED ((thread_t)(~0ull)) /* special owner when suspended */ - kq = &kqwq->kqwq_kqueue; - bzero(kqwq, sizeof (struct kqworkq)); +static inline bool +filt_wlowner_is_valid(thread_t owner) +{ + return owner != THREAD_NULL && owner != WL_OWNER_SUSPENDED; +} - kqwq->kqwq_state = KQ_WORKQ; +static inline bool +filt_wlshould_end_ownership(struct kqworkloop *kqwl, + struct kevent_internal_s *kev, int error) +{ + thread_t owner = kqwl->kqwl_owner; + return (error == 0 || error == ESTALE) && + (kev->fflags & NOTE_WL_END_OWNERSHIP) && + (owner == current_thread() || owner == WL_OWNER_SUSPENDED); +} - for (i = 0; i < KQWQ_NBUCKETS; i++) { - TAILQ_INIT(&kq->kq_queue[i]); - } - for (i = 0; i < KQWQ_NQOS; i++) { - TAILQ_INIT(&kqwq->kqwq_request[i].kqr_suppressed); - } +static inline bool +filt_wlshould_update_ownership(struct kevent_internal_s *kev, int error) +{ + return error == 0 && (kev->fflags & NOTE_WL_DISCOVER_OWNER) && + kev->ext[EV_EXTIDX_WL_ADDR]; +} - lck_spin_init(&kqwq->kqwq_reqlock, kq_lck_grp, kq_lck_attr); - policy = SYNC_POLICY_FIFO; - hook = (void *)kqwq; - - } else { - struct kqfile *kqf; - - kqf = (struct kqfile *)zalloc(kqfile_zone); - if (kqf == NULL) - return NULL; +static inline bool +filt_wlshould_set_async_qos(struct kevent_internal_s *kev, int error, + kq_index_t async_qos) +{ + if (error != 0) { + return false; + } + if (async_qos != THREAD_QOS_UNSPECIFIED) { + return true; + } + if ((kev->fflags & NOTE_WL_THREAD_REQUEST) && (kev->flags & EV_DELETE)) { + /* see filt_wlprocess() */ + return true; + } + return false; +} - kq = &kqf->kqf_kqueue; - bzero(kqf, sizeof (struct kqfile)); - TAILQ_INIT(&kq->kq_queue[0]); - TAILQ_INIT(&kqf->kqf_suppressed); - - policy = SYNC_POLICY_FIFO | SYNC_POLICY_PREPOST; - hook = NULL; +__result_use_check +static int +filt_wlupdateowner(struct kqworkloop *kqwl, struct kevent_internal_s *kev, + int error, kq_index_t async_qos) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + thread_t cur_owner, new_owner, extra_thread_ref = THREAD_NULL; + kq_index_t cur_override = THREAD_QOS_UNSPECIFIED; + kq_index_t old_owner_override = THREAD_QOS_UNSPECIFIED; + boolean_t ipc_override_is_sync = false; + boolean_t old_owner_override_is_sync = false; + int action = KQWL_UTQ_NONE; - } + filt_wlheld(kqwl); - waitq_set_init(&kq->kq_wqs, policy, NULL, hook); - lck_spin_init(&kq->kq_lock, kq_lck_grp, kq_lck_attr); - kq->kq_p = p; + /* + * The owner is only changed under both the filt_wllock and the + * kqwl_req_lock. Looking at it with either one held is fine. + */ + cur_owner = kqwl->kqwl_owner; + if (filt_wlshould_end_ownership(kqwl, kev, error)) { + new_owner = THREAD_NULL; + } else if (filt_wlshould_update_ownership(kev, error)) { + /* + * Decipher the owner port name, and translate accordingly. + * The low 2 bits were borrowed for other flags, so mask them off. + */ + uint64_t udata = kev->ext[EV_EXTIDX_WL_VALUE]; + mach_port_name_t new_owner_name = (mach_port_name_t)udata & ~0x3; + if (new_owner_name != MACH_PORT_NULL) { + new_owner_name = ipc_entry_name_mask(new_owner_name); + } - if (fdp->fd_knlistsize < 0) { - proc_fdlock(p); - if (fdp->fd_knlistsize < 0) - fdp->fd_knlistsize = 0; /* this process has had a kq */ - proc_fdunlock(p); + if (MACH_PORT_VALID(new_owner_name)) { + new_owner = port_name_to_thread(new_owner_name); + if (new_owner == THREAD_NULL) + return EOWNERDEAD; + extra_thread_ref = new_owner; + } else if (new_owner_name == MACH_PORT_DEAD) { + new_owner = WL_OWNER_SUSPENDED; + } else { + /* + * We never want to learn a new owner that is NULL. + * Ownership should be ended with END_OWNERSHIP. + */ + new_owner = cur_owner; + } + } else { + new_owner = cur_owner; } - kq_addr_offset = ((uintptr_t)kq - (uintptr_t)VM_MIN_KERNEL_AND_KEXT_ADDRESS); - /* Assert that the address can be pointer compacted for use with knote */ - assert(kq_addr_offset < (uint64_t)(1ull << KNOTE_KQ_BITSIZE)); - return (kq); -} - -/* - * kqueue_dealloc - detach all knotes from a kqueue and free it - * - * We walk each list looking for knotes referencing this - * this kqueue. If we find one, we try to drop it. But - * if we fail to get a drop reference, that will wait - * until it is dropped. So, we can just restart again - * safe in the assumption that the list will eventually - * not contain any more references to this kqueue (either - * we dropped them all, or someone else did). - * - * Assumes no new events are being added to the kqueue. - * Nothing locked on entry or exit. - */ -void -kqueue_dealloc(struct kqueue *kq) -{ - struct proc *p; - struct filedesc *fdp; - struct knote *kn; - int i; + if (filt_wlshould_set_async_qos(kev, error, async_qos)) { + action = KQWL_UTQ_SET_ASYNC_QOS; + } + if (cur_owner == new_owner && action == KQWL_UTQ_NONE) { + goto out; + } - if (kq == NULL) - return; + kqwl_req_lock(kqwl); - p = kq->kq_p; - fdp = p->p_fd; + /* If already tracked as servicer, don't track as owner */ + if ((kqr->kqr_state & KQR_BOUND) && new_owner == kqr->kqr_thread) { + kqwl->kqwl_owner = new_owner = THREAD_NULL; + } - proc_fdlock(p); - for (i = 0; i < fdp->fd_knlistsize; i++) { - kn = SLIST_FIRST(&fdp->fd_knlist[i]); - while (kn != NULL) { - if (kq == knote_get_kq(kn)) { - kqlock(kq); - proc_fdunlock(p); - /* drop it ourselves or wait */ - if (kqlock2knotedrop(kq, kn)) { - knote_drop(kn, p); + if (cur_owner != new_owner) { + kqwl->kqwl_owner = new_owner; + if (new_owner == extra_thread_ref) { + /* we just transfered this ref to kqwl_owner */ + extra_thread_ref = THREAD_NULL; + } + cur_override = kqworkloop_combined_qos(kqwl, &ipc_override_is_sync); + old_owner_override = kqr->kqr_dsync_owner_qos; + old_owner_override_is_sync = kqr->kqr_owner_override_is_sync; + + if (filt_wlowner_is_valid(new_owner)) { + /* override it before we drop the old */ + if (cur_override != THREAD_QOS_UNSPECIFIED) { + thread_add_ipc_override(new_owner, cur_override); + } + if (ipc_override_is_sync) { + thread_add_sync_ipc_override(new_owner); + } + /* Update the kqr to indicate that owner has sync ipc override */ + kqr->kqr_dsync_owner_qos = cur_override; + kqr->kqr_owner_override_is_sync = ipc_override_is_sync; + thread_starts_owning_workloop(new_owner); + if ((kqr->kqr_state & (KQR_THREQUESTED | KQR_BOUND)) == KQR_THREQUESTED) { + if (action == KQWL_UTQ_NONE) { + action = KQWL_UTQ_REDRIVE_EVENTS; } - proc_fdlock(p); - /* start over at beginning of list */ - kn = SLIST_FIRST(&fdp->fd_knlist[i]); - continue; } - kn = SLIST_NEXT(kn, kn_link); - } - } - if (fdp->fd_knhashmask != 0) { - for (i = 0; i < (int)fdp->fd_knhashmask + 1; i++) { - kn = SLIST_FIRST(&fdp->fd_knhash[i]); - while (kn != NULL) { - if (kq == knote_get_kq(kn)) { - kqlock(kq); - proc_fdunlock(p); - /* drop it ourselves or wait */ - if (kqlock2knotedrop(kq, kn)) { - knote_drop(kn, p); - } - proc_fdlock(p); - /* start over at beginning of list */ - kn = SLIST_FIRST(&fdp->fd_knhash[i]); - continue; + } else if (new_owner == THREAD_NULL) { + kqr->kqr_dsync_owner_qos = THREAD_QOS_UNSPECIFIED; + kqr->kqr_owner_override_is_sync = false; + if ((kqr->kqr_state & (KQR_THREQUESTED | KQR_WAKEUP)) == KQR_WAKEUP) { + if (action == KQWL_UTQ_NONE) { + action = KQWL_UTQ_REDRIVE_EVENTS; } - kn = SLIST_NEXT(kn, kn_link); } } } - proc_fdunlock(p); - /* - * waitq_set_deinit() remove the KQ's waitq set from - * any select sets to which it may belong. - */ - waitq_set_deinit(&kq->kq_wqs); - lck_spin_destroy(&kq->kq_lock, kq_lck_grp); + if (action != KQWL_UTQ_NONE) { + kqworkloop_update_threads_qos(kqwl, action, async_qos); + } - if (kq->kq_state & KQ_WORKQ) { - struct kqworkq *kqwq = (struct kqworkq *)kq; + kqwl_req_unlock(kqwl); - lck_spin_destroy(&kqwq->kqwq_reqlock, kq_lck_grp); - zfree(kqworkq_zone, kqwq); - } else { - struct kqfile *kqf = (struct kqfile *)kq; + /* Now that we are unlocked, drop the override and ref on old owner */ + if (new_owner != cur_owner && filt_wlowner_is_valid(cur_owner)) { + if (old_owner_override != THREAD_QOS_UNSPECIFIED) { + thread_drop_ipc_override(cur_owner); + } + if (old_owner_override_is_sync) { + thread_drop_sync_ipc_override(cur_owner); + } + thread_ends_owning_workloop(cur_owner); + thread_deallocate(cur_owner); + } - zfree(kqfile_zone, kqf); +out: + if (extra_thread_ref) { + thread_deallocate(extra_thread_ref); } + return error; } -int -kqueue_body(struct proc *p, fp_allocfn_t fp_zalloc, void *cra, int32_t *retval) +static int +filt_wldebounce( + struct kqworkloop *kqwl, + struct kevent_internal_s *kev, + int default_result) { - struct kqueue *kq; - struct fileproc *fp; - int fd, error; + user_addr_t addr = CAST_USER_ADDR_T(kev->ext[EV_EXTIDX_WL_ADDR]); + uint64_t udata; + int error; - error = falloc_withalloc(p, - &fp, &fd, vfs_context_current(), fp_zalloc, cra); - if (error) { - return (error); - } + /* we must have the workloop state mutex held */ + filt_wlheld(kqwl); - kq = kqueue_alloc(p, 0); - if (kq == NULL) { - fp_free(p, fd, fp); - return (ENOMEM); - } + /* Do we have a debounce address to work with? */ + if (addr) { + uint64_t kdata = kev->ext[EV_EXTIDX_WL_VALUE]; + uint64_t mask = kev->ext[EV_EXTIDX_WL_MASK]; - fp->f_flag = FREAD | FWRITE; - fp->f_ops = &kqueueops; - fp->f_data = kq; + error = copyin_word(addr, &udata, sizeof(udata)); + if (error) { + return error; + } - proc_fdlock(p); - *fdflags(p, fd) |= UF_EXCLOSE; - procfdtbl_releasefd(p, fd, NULL); - fp_drop(p, fd, fp, 1); - proc_fdunlock(p); + /* update state as copied in */ + kev->ext[EV_EXTIDX_WL_VALUE] = udata; - *retval = fd; - return (error); + /* If the masked bits don't match, reject it as stale */ + if ((udata & mask) != (kdata & mask)) { + return ESTALE; + } + +#if DEBUG || DEVELOPMENT + if ((kev->fflags & NOTE_WL_THREAD_REQUEST) && !(kev->flags & EV_DELETE)) { + if ((udata & DISPATCH_QUEUE_ENQUEUED) == 0) { + panic("kevent: workloop %#016llx is not enqueued " + "(kev:%p dq_state:%#016llx)", kev->udata, kev, udata); + } + } +#endif + } + + return default_result; +} + +/* + * Remembers the last updated that came in from userspace for debugging reasons. + * - fflags is mirrored from the userspace kevent + * - ext[i, i != VALUE] is mirrored from the userspace kevent + * - ext[VALUE] is set to what the kernel loaded atomically + * - data is set to the error if any + */ +static inline void +filt_wlremember_last_update( + __assert_only struct kqworkloop *kqwl, + struct knote *kn, + struct kevent_internal_s *kev, + int error) +{ + filt_wlheld(kqwl); + kn->kn_fflags = kev->fflags; + kn->kn_data = error; + memcpy(kn->kn_ext, kev->ext, sizeof(kev->ext)); +} + +/* + * Return which operations on EVFILT_WORKLOOP need to be protected against + * knoteusewait() causing priority inversions. + */ +static bool +filt_wlneeds_boost(struct kevent_internal_s *kev) +{ + if (kev == NULL) { + /* + * this is an f_process() usecount, and it can cause a drop to wait + */ + return true; + } + if (kev->fflags & NOTE_WL_THREAD_REQUEST) { + /* + * All operations on thread requests may starve drops or re-attach of + * the same knote, all of them need boosts. None of what we do under + * thread-request usecount holds blocks anyway. + */ + return true; + } + if (kev->fflags & NOTE_WL_SYNC_WAIT) { + /* + * this may call filt_wlwait() and we don't want to hold any boost when + * woken up, this would cause background threads contending on + * dispatch_sync() to wake up at 64 and be preempted immediately when + * this drops. + */ + return false; + } + + /* + * SYNC_WAIT knotes when deleted don't need to be rushed, there's no + * detach/reattach race with these ever. In addition to this, when the + * SYNC_WAIT knote is dropped, the caller is no longer receiving the + * workloop overrides if any, and we'd rather schedule other threads than + * him, he's not possibly stalling anything anymore. + */ + return (kev->flags & EV_DELETE) == 0; +} + +static int +filt_wlattach(struct knote *kn, struct kevent_internal_s *kev) +{ + struct kqueue *kq = knote_get_kq(kn); + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + int error = 0; + kq_index_t qos_index = 0; + + if ((kq->kq_state & KQ_WORKLOOP) == 0) { + error = ENOTSUP; + goto out; + } + +#if DEVELOPMENT || DEBUG + if (kev->ident == 0 && kev->udata == 0 && kev->fflags == 0) { + struct kqrequest *kqr = &kqwl->kqwl_request; + + kqwl_req_lock(kqwl); + kev->fflags = 0; + if (kqr->kqr_dsync_waiters) { + kev->fflags |= NOTE_WL_SYNC_WAIT; + } + if (kqr->kqr_qos_index) { + kev->fflags |= NOTE_WL_THREAD_REQUEST; + } + if (kqwl->kqwl_owner == WL_OWNER_SUSPENDED) { + kev->ext[0] = ~0ull; + } else { + kev->ext[0] = thread_tid(kqwl->kqwl_owner); + } + kev->ext[1] = thread_tid(kqwl->kqwl_request.kqr_thread); + kev->ext[2] = thread_owned_workloops_count(current_thread()); + kev->ext[3] = kn->kn_kevent.ext[3]; + kqwl_req_unlock(kqwl); + error = EBUSY; + goto out; + } +#endif + + /* Some simple validation */ + int command = (kn->kn_sfflags & NOTE_WL_COMMANDS_MASK); + switch (command) { + case NOTE_WL_THREAD_REQUEST: + if (kn->kn_id != kqwl->kqwl_dynamicid) { + error = EINVAL; + goto out; + } + qos_index = qos_index_from_qos(kn, kn->kn_qos, FALSE); + if (qos_index < THREAD_QOS_MAINTENANCE || + qos_index > THREAD_QOS_USER_INTERACTIVE) { + error = ERANGE; + goto out; + } + break; + case NOTE_WL_SYNC_WAIT: + case NOTE_WL_SYNC_WAKE: + if (kq->kq_state & KQ_NO_WQ_THREAD) { + error = ENOTSUP; + goto out; + } + if (kn->kn_id == kqwl->kqwl_dynamicid) { + error = EINVAL; + goto out; + } + if ((kn->kn_flags & EV_DISABLE) == 0) { + error = EINVAL; + goto out; + } + if (kn->kn_sfflags & NOTE_WL_END_OWNERSHIP) { + error = EINVAL; + goto out; + } + break; + default: + error = EINVAL; + goto out; + } + + filt_wllock(kqwl); + kn->kn_hook = NULL; + + if (command == NOTE_WL_THREAD_REQUEST && kqwl->kqwl_request.kqr_qos_index) { + /* + * There already is a thread request, and well, you're only allowed + * one per workloop, so fail the attach. + * + * Note: kqr_qos_index is always set with the wllock held, so we + * don't need to take the kqr lock. + */ + error = EALREADY; + } else { + /* Make sure user and kernel are in agreement on important state */ + error = filt_wldebounce(kqwl, kev, 0); + } + + error = filt_wlupdateowner(kqwl, kev, error, qos_index); + filt_wlunlock(kqwl); +out: + if (error) { + kn->kn_flags |= EV_ERROR; + /* If userland wants ESTALE to be hidden, fail the attach anyway */ + if (error == ESTALE && (kn->kn_sfflags & NOTE_WL_IGNORE_ESTALE)) { + error = 0; + } + kn->kn_data = error; + return 0; + } + + /* Just attaching the thread request successfully will fire it */ + return command == NOTE_WL_THREAD_REQUEST; +} + +__attribute__((noinline,not_tail_called)) +static int +filt_wlwait(struct kqworkloop *kqwl, + struct knote *kn, + struct kevent_internal_s *kev) +{ + filt_wlheld(kqwl); + assert((kn->kn_sfflags & NOTE_WL_SYNC_WAKE) == 0); + + /* + * Hint to the wakeup side that this thread is waiting. Also used by + * stackshot for waitinfo. + */ + kn->kn_hook = current_thread(); + + thread_set_pending_block_hint(current_thread(), kThreadWaitWorkloopSyncWait); + + wait_result_t wr = assert_wait(kn, THREAD_ABORTSAFE); + + if (wr == THREAD_WAITING) { + kq_index_t qos_index = qos_index_from_qos(kn, kev->qos, TRUE); + struct kqrequest *kqr = &kqwl->kqwl_request; + + thread_t thread_to_handoff = THREAD_NULL; /* holds +1 thread ref */ + + thread_t kqwl_owner = kqwl->kqwl_owner; + if (filt_wlowner_is_valid(kqwl_owner)) { + thread_reference(kqwl_owner); + thread_to_handoff = kqwl_owner; + } + + kqwl_req_lock(kqwl); + + if (qos_index) { + assert(kqr->kqr_dsync_waiters < UINT16_MAX); + kqr->kqr_dsync_waiters++; + if (qos_index > kqr->kqr_dsync_waiters_qos) { + kqworkloop_update_threads_qos(kqwl, + KQWL_UTQ_SET_SYNC_WAITERS_QOS, qos_index); + } + } + + if ((kqr->kqr_state & KQR_BOUND) && thread_to_handoff == THREAD_NULL) { + assert(kqr->kqr_thread != THREAD_NULL); + thread_t servicer = kqr->kqr_thread; + + thread_reference(servicer); + thread_to_handoff = servicer; + } + + kqwl_req_unlock(kqwl); + + filt_wlunlock(kqwl); + + /* TODO: use continuation based blocking */ + + /* consume a refcount on thread_to_handoff, then thread_block() */ + wr = thread_handoff(thread_to_handoff); + thread_to_handoff = THREAD_NULL; + + filt_wllock(kqwl); + + /* clear waiting state (only one waiting thread - so no race) */ + assert(kn->kn_hook == current_thread()); + + if (qos_index) { + kqwl_req_lock(kqwl); + assert(kqr->kqr_dsync_waiters > 0); + if (--kqr->kqr_dsync_waiters == 0) { + assert(kqr->kqr_dsync_waiters_qos); + kqworkloop_update_threads_qos(kqwl, + KQWL_UTQ_SET_SYNC_WAITERS_QOS, 0); + } + kqwl_req_unlock(kqwl); + } + } + + kn->kn_hook = NULL; + + switch (wr) { + case THREAD_AWAKENED: + return 0; + case THREAD_INTERRUPTED: + return EINTR; + case THREAD_RESTART: + return ECANCELED; + default: + panic("filt_wlattach: unexpected wait result %d", wr); + return EINVAL; + } +} + +/* called in stackshot context to report the thread responsible for blocking this thread */ +void +kdp_workloop_sync_wait_find_owner(__assert_only thread_t thread, + event64_t event, + thread_waitinfo_t *waitinfo) +{ + struct knote *kn = (struct knote*) event; + assert(kdp_is_in_zone(kn, "knote zone")); + + assert(kn->kn_hook == thread); + + struct kqueue *kq = knote_get_kq(kn); + assert(kdp_is_in_zone(kq, "kqueue workloop zone")); + assert(kq->kq_state & KQ_WORKLOOP); + + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct kqrequest *kqr = &kqwl->kqwl_request; + + thread_t kqwl_owner = kqwl->kqwl_owner; + thread_t servicer = kqr->kqr_thread; + + if (kqwl_owner == WL_OWNER_SUSPENDED) { + waitinfo->owner = STACKSHOT_WAITOWNER_SUSPENDED; + } else if (kqwl_owner != THREAD_NULL) { + assert(kdp_is_in_zone(kqwl_owner, "threads")); + + waitinfo->owner = thread_tid(kqwl->kqwl_owner); + } else if (servicer != THREAD_NULL) { + assert(kdp_is_in_zone(servicer, "threads")); + + waitinfo->owner = thread_tid(servicer); + } else if (kqr->kqr_state & KQR_THREQUESTED) { + waitinfo->owner = STACKSHOT_WAITOWNER_THREQUESTED; + } else { + waitinfo->owner = 0; + } + + waitinfo->context = kqwl->kqwl_dynamicid; + + return; +} + +/* + * Takes kqueue locked, returns locked, may drop in the middle and/or block for a while + */ +static int +filt_wlpost_attach(struct knote *kn, struct kevent_internal_s *kev) +{ + struct kqueue *kq = knote_get_kq(kn); + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + int error = 0; + + if (kev->fflags & NOTE_WL_SYNC_WAIT) { + if (kqlock2knoteuse(kq, kn, KNUSE_NONE)) { + filt_wllock(kqwl); + /* if the wake has already preposted, don't wait */ + if ((kn->kn_sfflags & NOTE_WL_SYNC_WAKE) == 0) + error = filt_wlwait(kqwl, kn, kev); + filt_wlunlock(kqwl); + knoteuse2kqlock(kq, kn, KNUSE_NONE); + } + } + return error; +} + +static void +filt_wldetach(__assert_only struct knote *kn) +{ + assert(knote_get_kq(kn)->kq_state & KQ_WORKLOOP); + + /* + * Thread requests have nothing to detach. + * Sync waiters should have been aborted out + * and drop their refs before we could drop/ + * detach their knotes. + */ + assert(kn->kn_hook == NULL); +} + +static int +filt_wlevent( + __unused struct knote *kn, + __unused long hint) +{ + panic("filt_wlevent"); + return 0; +} + +static int +filt_wlvalidate_kev_flags(struct knote *kn, struct kevent_internal_s *kev) +{ + int new_commands = kev->fflags & NOTE_WL_COMMANDS_MASK; + int sav_commands = kn->kn_sfflags & NOTE_WL_COMMANDS_MASK; + int error = 0; + + switch (new_commands) { + case NOTE_WL_THREAD_REQUEST: + /* thread requests can only update themselves */ + if (sav_commands != new_commands) + error = EINVAL; + break; + + case NOTE_WL_SYNC_WAIT: + if (kev->fflags & NOTE_WL_END_OWNERSHIP) + error = EINVAL; + /* FALLTHROUGH */ + case NOTE_WL_SYNC_WAKE: + /* waits and wakes can update themselves or their counterparts */ + if (!(sav_commands & (NOTE_WL_SYNC_WAIT | NOTE_WL_SYNC_WAKE))) + error = EINVAL; + if (kev->fflags & NOTE_WL_UPDATE_QOS) + error = EINVAL; + if ((kev->flags & (EV_ENABLE | EV_DELETE)) == EV_ENABLE) + error = EINVAL; + if (kev->flags & EV_DELETE) { + /* + * Really this is not supported: there is absolutely no reason + * whatsoever to want to fail the drop of a NOTE_WL_SYNC_WAIT knote. + */ + if (kev->ext[EV_EXTIDX_WL_ADDR] && kev->ext[EV_EXTIDX_WL_MASK]) { + error = EINVAL; + } + } + break; + + default: + error = EINVAL; + } + if ((kev->flags & EV_DELETE) && (kev->fflags & NOTE_WL_DISCOVER_OWNER)) { + error = EINVAL; + } + return error; +} + +static int +filt_wltouch( + struct knote *kn, + struct kevent_internal_s *kev) +{ + struct kqueue *kq = knote_get_kq(kn); + int error = 0; + struct kqworkloop *kqwl; + + assert(kq->kq_state & KQ_WORKLOOP); + kqwl = (struct kqworkloop *)kq; + + error = filt_wlvalidate_kev_flags(kn, kev); + if (error) { + goto out; + } + + filt_wllock(kqwl); + + /* Make sure user and kernel are in agreement on important state */ + error = filt_wldebounce(kqwl, kev, 0); + if (error) { + error = filt_wlupdateowner(kqwl, kev, error, 0); + goto out_unlock; + } + + int new_command = kev->fflags & NOTE_WL_COMMANDS_MASK; + switch (new_command) { + case NOTE_WL_THREAD_REQUEST: + assert(kqwl->kqwl_request.kqr_qos_index != THREAD_QOS_UNSPECIFIED); + break; + + case NOTE_WL_SYNC_WAIT: + /* + * we need to allow waiting several times on the same knote because + * of EINTR. If it's already woken though, it won't block. + */ + break; + + case NOTE_WL_SYNC_WAKE: + if (kn->kn_sfflags & NOTE_WL_SYNC_WAKE) { + /* disallow waking the same knote twice */ + error = EALREADY; + goto out_unlock; + } + if (kn->kn_hook) { + thread_wakeup_thread((event_t)kn, (thread_t)kn->kn_hook); + } + break; + + default: + error = EINVAL; + goto out_unlock; + } + + /* + * Save off any additional fflags/data we just accepted + * But only keep the last round of "update" bits we acted on which helps + * debugging a lot. + */ + kn->kn_sfflags &= ~NOTE_WL_UPDATES_MASK; + kn->kn_sfflags |= kev->fflags; + kn->kn_sdata = kev->data; + + kq_index_t qos_index = THREAD_QOS_UNSPECIFIED; + + if (kev->fflags & NOTE_WL_UPDATE_QOS) { + qos_t qos = pthread_priority_canonicalize(kev->qos, FALSE); + + if (kn->kn_qos != qos) { + qos_index = qos_index_from_qos(kn, qos, FALSE); + if (qos_index == THREAD_QOS_UNSPECIFIED) { + error = ERANGE; + goto out_unlock; + } + kqlock(kq); + if (kn->kn_status & KN_QUEUED) { + knote_dequeue(kn); + knote_set_qos_index(kn, qos_index); + knote_enqueue(kn); + knote_wakeup(kn); + } else { + knote_set_qos_index(kn, qos_index); + } + kn->kn_qos = qos; + kqunlock(kq); + } + } + + error = filt_wlupdateowner(kqwl, kev, 0, qos_index); + if (error) { + goto out_unlock; + } + + if (new_command == NOTE_WL_SYNC_WAIT) { + /* if the wake has already preposted, don't wait */ + if ((kn->kn_sfflags & NOTE_WL_SYNC_WAKE) == 0) + error = filt_wlwait(kqwl, kn, kev); + } + +out_unlock: + filt_wlremember_last_update(kqwl, kn, kev, error); + filt_wlunlock(kqwl); +out: + if (error) { + if (error == ESTALE && (kev->fflags & NOTE_WL_IGNORE_ESTALE)) { + /* If userland wants ESTALE to be hidden, do not activate */ + return 0; + } + kev->flags |= EV_ERROR; + kev->data = error; + return 0; + } + /* Just touching the thread request successfully will fire it */ + return new_command == NOTE_WL_THREAD_REQUEST; +} + +static int +filt_wldrop_and_unlock( + struct knote *kn, + struct kevent_internal_s *kev) +{ + struct kqueue *kq = knote_get_kq(kn); + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + int error = 0, knoteuse_flags = KNUSE_NONE; + + kqlock_held(kq); + + assert(kev->flags & EV_DELETE); + assert(kq->kq_state & KQ_WORKLOOP); + + error = filt_wlvalidate_kev_flags(kn, kev); + if (error) { + goto out; + } + + if (kn->kn_sfflags & NOTE_WL_THREAD_REQUEST) { + knoteuse_flags |= KNUSE_BOOST; + } + + /* take a usecount to allow taking the filt_wllock */ + if (!kqlock2knoteuse(kq, kn, knoteuse_flags)) { + /* knote is being dropped already */ + error = EINPROGRESS; + goto out; + } + + filt_wllock(kqwl); + + /* + * Make sure user and kernel are in agreement on important state + * + * Userland will modify bits to cause this to fail for the touch / drop + * race case (when a drop for a thread request quiescing comes in late after + * the workloop has been woken up again). + */ + error = filt_wldebounce(kqwl, kev, 0); + + if (!knoteuse2kqlock(kq, kn, knoteuse_flags)) { + /* knote is no longer alive */ + error = EINPROGRESS; + goto out_unlock; + } + + if (!error && (kn->kn_sfflags & NOTE_WL_THREAD_REQUEST) && kn->kn_inuse) { + /* + * There is a concurrent drop or touch happening, we can't resolve this, + * userland has to redrive. + * + * The race we're worried about here is the following: + * + * f_touch | f_drop_and_unlock + * ------------------------+-------------------------------------------- + * | kqlock() + * | kqlock2knoteuse() + * | filt_wllock() + * | debounces successfully + * kqlock() | + * kqlock2knoteuse | + * filt_wllock() | + * | knoteuse2kqlock() + * | filt_wlunlock() + * | kqlock2knotedrop() + * debounces successfully | + * filt_wlunlock() | + * caller WAKES f_drop | + * | performs drop, but f_touch should have won + * + * So if the usecount is not 0 here, we need to wait for it to drop and + * redrive the whole logic (including looking up the knote again). + */ + filt_wlunlock(kqwl); + knoteusewait(kq, kn); + return ERESTART; + } + + /* + * If error is 0 this will set kqr_qos_index to THREAD_QOS_UNSPECIFIED + * + * If error is 0 or ESTALE this may drop ownership and cause a thread + * request redrive, however the kqlock is held which prevents f_process() to + * run until we did the drop for real. + */ + error = filt_wlupdateowner(kqwl, kev, error, 0); + if (error) { + goto out_unlock; + } + + if ((kn->kn_sfflags & (NOTE_WL_SYNC_WAIT | NOTE_WL_SYNC_WAKE)) == + NOTE_WL_SYNC_WAIT) { + /* + * When deleting a SYNC_WAIT knote that hasn't been woken up + * explicitly, issue a wake up. + */ + kn->kn_sfflags |= NOTE_WL_SYNC_WAKE; + if (kn->kn_hook) { + thread_wakeup_thread((event_t)kn, (thread_t)kn->kn_hook); + } + } + +out_unlock: + filt_wlremember_last_update(kqwl, kn, kev, error); + filt_wlunlock(kqwl); + +out: + if (error == 0) { + /* If nothing failed, do the regular knote drop. */ + if (kqlock2knotedrop(kq, kn)) { + knote_drop(kn, current_proc()); + } else { + error = EINPROGRESS; + } + } else { + kqunlock(kq); + } + if (error == ESTALE && (kev->fflags & NOTE_WL_IGNORE_ESTALE)) { + error = 0; + } + if (error == EINPROGRESS) { + /* + * filt_wlprocess() makes sure that no event can be delivered for + * NOTE_WL_THREAD_REQUEST knotes once a drop is happening, and + * NOTE_WL_SYNC_* knotes are never fired. + * + * It means that EINPROGRESS is about a state that userland cannot + * observe for this filter (an event being delivered concurrently from + * a drop), so silence the error. + */ + error = 0; + } + return error; +} + +static int +filt_wlprocess( + struct knote *kn, + __unused struct filt_process_s *data, + struct kevent_internal_s *kev) +{ + struct kqueue *kq = knote_get_kq(kn); + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct kqrequest *kqr = &kqwl->kqwl_request; + int rc = 0; + + assert(kq->kq_state & KQ_WORKLOOP); + + /* only thread requests should get here */ + assert(kn->kn_sfflags & NOTE_WL_THREAD_REQUEST); + if (kn->kn_sfflags & NOTE_WL_THREAD_REQUEST) { + filt_wllock(kqwl); + assert(kqr->kqr_qos_index != THREAD_QOS_UNSPECIFIED); + if (kqwl->kqwl_owner) { + /* + * userspace sometimes due to events being + * delivered but not triggering a drain session can cause a process + * of the thread request knote. + * + * When that happens, the automatic deactivation due to process + * would swallow the event, so we have to activate the knote again. + */ + kqlock(kq); + knote_activate(kn); + kqunlock(kq); + } else if (kqr->kqr_qos_index) { +#if DEBUG || DEVELOPMENT + user_addr_t addr = CAST_USER_ADDR_T(kn->kn_ext[EV_EXTIDX_WL_ADDR]); + task_t t = current_task(); + uint64_t val; + if (addr && task_is_active(t) && !task_is_halting(t) && + copyin_word(addr, &val, sizeof(val)) == 0 && + val && (val & DISPATCH_QUEUE_ENQUEUED) == 0) { + panic("kevent: workloop %#016llx is not enqueued " + "(kn:%p dq_state:%#016llx kev.dq_state:%#016llx)", + kn->kn_udata, kn, val, + kn->kn_ext[EV_EXTIDX_WL_VALUE]); + } +#endif + *kev = kn->kn_kevent; + kev->fflags = kn->kn_sfflags; + kev->data = kn->kn_sdata; + kev->qos = kn->kn_qos; + rc = 1; + } + filt_wlunlock(kqwl); + } + return rc; +} + +#pragma mark kevent / knotes + +/* + * JMM - placeholder for not-yet-implemented filters + */ +static int +filt_badattach(__unused struct knote *kn, __unused struct kevent_internal_s *kev) +{ + kn->kn_flags |= EV_ERROR; + kn->kn_data = ENOTSUP; + return 0; +} + +struct kqueue * +kqueue_alloc(struct proc *p, unsigned int flags) +{ + struct filedesc *fdp = p->p_fd; + struct kqueue *kq = NULL; + int policy; + void *hook = NULL; + uint64_t kq_addr_offset; + + if (flags & KEVENT_FLAG_WORKQ) { + struct kqworkq *kqwq; + int i; + + kqwq = (struct kqworkq *)zalloc(kqworkq_zone); + if (kqwq == NULL) + return NULL; + + kq = &kqwq->kqwq_kqueue; + bzero(kqwq, sizeof (struct kqworkq)); + + kqwq->kqwq_state = KQ_WORKQ; + + for (i = 0; i < KQWQ_NBUCKETS; i++) { + TAILQ_INIT(&kq->kq_queue[i]); + } + for (i = 0; i < KQWQ_NQOS; i++) { + kqwq->kqwq_request[i].kqr_qos_index = i; + } + + lck_spin_init(&kqwq->kqwq_reqlock, kq_lck_grp, kq_lck_attr); + policy = SYNC_POLICY_FIFO; + hook = (void *)kqwq; + + } else if (flags & KEVENT_FLAG_WORKLOOP) { + struct kqworkloop *kqwl; + int i; + + kqwl = (struct kqworkloop *)zalloc(kqworkloop_zone); + if (kqwl == NULL) + return NULL; + + bzero(kqwl, sizeof (struct kqworkloop)); + + kqwl->kqwl_state = KQ_WORKLOOP | KQ_DYNAMIC; + kqwl->kqwl_retains = 1; /* donate a retain to creator */ + + kq = &kqwl->kqwl_kqueue; + for (i = 0; i < KQWL_NBUCKETS; i++) { + TAILQ_INIT(&kq->kq_queue[i]); + } + TAILQ_INIT(&kqwl->kqwl_request.kqr_suppressed); + + lck_spin_init(&kqwl->kqwl_reqlock, kq_lck_grp, kq_lck_attr); + lck_mtx_init(&kqwl->kqwl_statelock, kq_lck_grp, kq_lck_attr); + + policy = SYNC_POLICY_FIFO; + if (flags & KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD) { + policy |= SYNC_POLICY_PREPOST; + kq->kq_state |= KQ_NO_WQ_THREAD; + } else { + hook = (void *)kqwl; + } + + } else { + struct kqfile *kqf; + + kqf = (struct kqfile *)zalloc(kqfile_zone); + if (kqf == NULL) + return NULL; + + kq = &kqf->kqf_kqueue; + bzero(kqf, sizeof (struct kqfile)); + TAILQ_INIT(&kq->kq_queue[0]); + TAILQ_INIT(&kqf->kqf_suppressed); + + policy = SYNC_POLICY_FIFO | SYNC_POLICY_PREPOST; + } + + waitq_set_init(&kq->kq_wqs, policy, NULL, hook); + lck_spin_init(&kq->kq_lock, kq_lck_grp, kq_lck_attr); + kq->kq_p = p; + + if (fdp->fd_knlistsize < 0) { + proc_fdlock(p); + if (fdp->fd_knlistsize < 0) + fdp->fd_knlistsize = 0; /* this process has had a kq */ + proc_fdunlock(p); + } + + kq_addr_offset = ((uintptr_t)kq - (uintptr_t)VM_MIN_KERNEL_AND_KEXT_ADDRESS); + /* Assert that the address can be pointer compacted for use with knote */ + assert(kq_addr_offset < (uint64_t)(1ull << KNOTE_KQ_BITSIZE)); + return (kq); +} + +/* + * knotes_dealloc - detach all knotes for the process and drop them + * + * Called with proc_fdlock held. + * Returns with it locked. + * May drop it temporarily. + * Process is in such a state that it will not try to allocate + * any more knotes during this process (stopped for exit or exec). + */ +void +knotes_dealloc(proc_t p) +{ + struct filedesc *fdp = p->p_fd; + struct kqueue *kq; + struct knote *kn; + struct klist *kn_hash = NULL; + int i; + + /* Close all the fd-indexed knotes up front */ + if (fdp->fd_knlistsize > 0) { + for (i = 0; i < fdp->fd_knlistsize; i++) { + while ((kn = SLIST_FIRST(&fdp->fd_knlist[i])) != NULL) { + kq = knote_get_kq(kn); + kqlock(kq); + proc_fdunlock(p); + /* drop it ourselves or wait */ + if (kqlock2knotedrop(kq, kn)) { + knote_drop(kn, p); + } + proc_fdlock(p); + } + } + /* free the table */ + FREE(fdp->fd_knlist, M_KQUEUE); + fdp->fd_knlist = NULL; + } + fdp->fd_knlistsize = -1; + + knhash_lock(p); + proc_fdunlock(p); + + /* Clean out all the hashed knotes as well */ + if (fdp->fd_knhashmask != 0) { + for (i = 0; i <= (int)fdp->fd_knhashmask; i++) { + while ((kn = SLIST_FIRST(&fdp->fd_knhash[i])) != NULL) { + kq = knote_get_kq(kn); + kqlock(kq); + knhash_unlock(p); + /* drop it ourselves or wait */ + if (kqlock2knotedrop(kq, kn)) { + knote_drop(kn, p); + } + knhash_lock(p); + } + } + kn_hash = fdp->fd_knhash; + fdp->fd_knhashmask = 0; + fdp->fd_knhash = NULL; + } + + knhash_unlock(p); + + /* free the kn_hash table */ + if (kn_hash) + FREE(kn_hash, M_KQUEUE); + + proc_fdlock(p); +} + + +/* + * kqueue_dealloc - detach all knotes from a kqueue and free it + * + * We walk each list looking for knotes referencing this + * this kqueue. If we find one, we try to drop it. But + * if we fail to get a drop reference, that will wait + * until it is dropped. So, we can just restart again + * safe in the assumption that the list will eventually + * not contain any more references to this kqueue (either + * we dropped them all, or someone else did). + * + * Assumes no new events are being added to the kqueue. + * Nothing locked on entry or exit. + * + * Workloop kqueues cant get here unless all the knotes + * are already gone and all requested threads have come + * and gone (cancelled or arrived). + */ +void +kqueue_dealloc(struct kqueue *kq) +{ + struct proc *p; + struct filedesc *fdp; + struct knote *kn; + int i; + + if (kq == NULL) + return; + + p = kq->kq_p; + fdp = p->p_fd; + + proc_fdlock(p); + for (i = 0; i < fdp->fd_knlistsize; i++) { + kn = SLIST_FIRST(&fdp->fd_knlist[i]); + while (kn != NULL) { + if (kq == knote_get_kq(kn)) { + assert((kq->kq_state & KQ_WORKLOOP) == 0); + kqlock(kq); + proc_fdunlock(p); + /* drop it ourselves or wait */ + if (kqlock2knotedrop(kq, kn)) { + knote_drop(kn, p); + } + proc_fdlock(p); + /* start over at beginning of list */ + kn = SLIST_FIRST(&fdp->fd_knlist[i]); + continue; + } + kn = SLIST_NEXT(kn, kn_link); + } + } + knhash_lock(p); + proc_fdunlock(p); + + if (fdp->fd_knhashmask != 0) { + for (i = 0; i < (int)fdp->fd_knhashmask + 1; i++) { + kn = SLIST_FIRST(&fdp->fd_knhash[i]); + while (kn != NULL) { + if (kq == knote_get_kq(kn)) { + assert((kq->kq_state & KQ_WORKLOOP) == 0); + kqlock(kq); + knhash_unlock(p); + /* drop it ourselves or wait */ + if (kqlock2knotedrop(kq, kn)) { + knote_drop(kn, p); + } + knhash_lock(p); + /* start over at beginning of list */ + kn = SLIST_FIRST(&fdp->fd_knhash[i]); + continue; + } + kn = SLIST_NEXT(kn, kn_link); + } + } + } + knhash_unlock(p); + + if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct kqrequest *kqr = &kqwl->kqwl_request; + thread_t cur_owner = kqwl->kqwl_owner; + + assert(TAILQ_EMPTY(&kqwl->kqwl_request.kqr_suppressed)); + if (filt_wlowner_is_valid(cur_owner)) { + /* + * If the kqueue had an owner that prevented the thread request to + * go through, then no unbind happened, and we may have lingering + * overrides to drop. + */ + if (kqr->kqr_dsync_owner_qos != THREAD_QOS_UNSPECIFIED) { + thread_drop_ipc_override(cur_owner); + kqr->kqr_dsync_owner_qos = THREAD_QOS_UNSPECIFIED; + } + + if (kqr->kqr_owner_override_is_sync) { + thread_drop_sync_ipc_override(cur_owner); + kqr->kqr_owner_override_is_sync = 0; + } + thread_ends_owning_workloop(cur_owner); + thread_deallocate(cur_owner); + kqwl->kqwl_owner = THREAD_NULL; + } + } + + /* + * waitq_set_deinit() remove the KQ's waitq set from + * any select sets to which it may belong. + */ + waitq_set_deinit(&kq->kq_wqs); + lck_spin_destroy(&kq->kq_lock, kq_lck_grp); + + if (kq->kq_state & KQ_WORKQ) { + struct kqworkq *kqwq = (struct kqworkq *)kq; + + lck_spin_destroy(&kqwq->kqwq_reqlock, kq_lck_grp); + zfree(kqworkq_zone, kqwq); + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + assert(kqwl->kqwl_retains == 0); + lck_spin_destroy(&kqwl->kqwl_reqlock, kq_lck_grp); + lck_mtx_destroy(&kqwl->kqwl_statelock, kq_lck_grp); + zfree(kqworkloop_zone, kqwl); + } else { + struct kqfile *kqf = (struct kqfile *)kq; + + zfree(kqfile_zone, kqf); + } +} + +static inline void +kqueue_retain(struct kqueue *kq) +{ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + uint32_t previous; + + if ((kq->kq_state & KQ_DYNAMIC) == 0) + return; + + previous = OSIncrementAtomic(&kqwl->kqwl_retains); + if (previous == KQ_WORKLOOP_RETAINS_MAX) + panic("kq(%p) retain overflow", kq); + + if (previous == 0) + panic("kq(%p) resurrection", kq); +} + +#define KQUEUE_CANT_BE_LAST_REF 0 +#define KQUEUE_MIGHT_BE_LAST_REF 1 + +static inline int +kqueue_release(struct kqueue *kq, __assert_only int possibly_last) +{ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + if ((kq->kq_state & KQ_DYNAMIC) == 0) { + return 0; + } + + assert(kq->kq_state & KQ_WORKLOOP); /* for now */ + uint32_t refs = OSDecrementAtomic(&kqwl->kqwl_retains); + if (__improbable(refs == 0)) { + panic("kq(%p) over-release", kq); + } + if (refs == 1) { + assert(possibly_last); + } + return refs == 1; +} + +int +kqueue_body(struct proc *p, fp_allocfn_t fp_zalloc, void *cra, int32_t *retval) +{ + struct kqueue *kq; + struct fileproc *fp; + int fd, error; + + error = falloc_withalloc(p, + &fp, &fd, vfs_context_current(), fp_zalloc, cra); + if (error) { + return (error); + } + + kq = kqueue_alloc(p, 0); + if (kq == NULL) { + fp_free(p, fd, fp); + return (ENOMEM); + } + + fp->f_flag = FREAD | FWRITE; + fp->f_ops = &kqueueops; + fp->f_data = kq; + + proc_fdlock(p); + *fdflags(p, fd) |= UF_EXCLOSE; + procfdtbl_releasefd(p, fd, NULL); + fp_drop(p, fd, fp, 1); + proc_fdunlock(p); + + *retval = fd; + return (error); } int @@ -2066,8 +3475,7 @@ kevent_continue(__unused struct kqueue *kq, void *data, int error) fd = cont_args->fd; fp = cont_args->fp; - if (fp != NULL) - fp_drop(p, fd, fp, 0); + kevent_put_kq(p, fd, fp, kq); /* don't abandon other output just because of residual copyout failures */ if (error == 0 && data_available && data_resid != data_size) { @@ -2094,7 +3502,7 @@ kevent(struct proc *p, struct kevent_args *uap, int32_t *retval) unsigned int flags = KEVENT_FLAG_LEGACY32; return kevent_internal(p, - uap->fd, + (kqueue_id_t)uap->fd, NULL, uap->changelist, uap->nchanges, uap->eventlist, uap->nevents, 0ULL, 0ULL, @@ -2114,35 +3522,71 @@ kevent64(struct proc *p, struct kevent64_args *uap, int32_t *retval) flags |= KEVENT_FLAG_LEGACY64; return kevent_internal(p, - uap->fd, - uap->changelist, uap->nchanges, - uap->eventlist, uap->nevents, - 0ULL, 0ULL, - flags, - uap->timeout, - kevent_continue, + (kqueue_id_t)uap->fd, NULL, + uap->changelist, uap->nchanges, + uap->eventlist, uap->nevents, + 0ULL, 0ULL, + flags, + uap->timeout, + kevent_continue, + retval); +} + +int +kevent_qos(struct proc *p, struct kevent_qos_args *uap, int32_t *retval) +{ + /* restrict to user flags */ + uap->flags &= KEVENT_FLAG_USER; + + return kevent_internal(p, + (kqueue_id_t)uap->fd, NULL, + uap->changelist, uap->nchanges, + uap->eventlist, uap->nevents, + uap->data_out, (uint64_t)uap->data_available, + uap->flags, + 0ULL, + kevent_continue, + retval); +} + +int +kevent_qos_internal(struct proc *p, int fd, + user_addr_t changelist, int nchanges, + user_addr_t eventlist, int nevents, + user_addr_t data_out, user_size_t *data_available, + unsigned int flags, + int32_t *retval) +{ + return kevent_internal(p, + (kqueue_id_t)fd, NULL, + changelist, nchanges, + eventlist, nevents, + data_out, (uint64_t)data_available, + (flags | KEVENT_FLAG_KERNEL), + 0ULL, + NULL, retval); } int -kevent_qos(struct proc *p, struct kevent_qos_args *uap, int32_t *retval) +kevent_id(struct proc *p, struct kevent_id_args *uap, int32_t *retval) { /* restrict to user flags */ uap->flags &= KEVENT_FLAG_USER; return kevent_internal(p, - uap->fd, + (kqueue_id_t)uap->id, NULL, uap->changelist, uap->nchanges, uap->eventlist, uap->nevents, uap->data_out, (uint64_t)uap->data_available, - uap->flags, + (uap->flags | KEVENT_FLAG_DYNAMIC_KQUEUE), 0ULL, kevent_continue, retval); } -int -kevent_qos_internal(struct proc *p, int fd, +int +kevent_id_internal(struct proc *p, kqueue_id_t *id, user_addr_t changelist, int nchanges, user_addr_t eventlist, int nevents, user_addr_t data_out, user_size_t *data_available, @@ -2150,11 +3594,11 @@ kevent_qos_internal(struct proc *p, int fd, int32_t *retval) { return kevent_internal(p, - fd, + *id, id, changelist, nchanges, eventlist, nevents, data_out, (uint64_t)data_available, - (flags | KEVENT_FLAG_KERNEL), + (flags | KEVENT_FLAG_KERNEL | KEVENT_FLAG_DYNAMIC_KQUEUE), 0ULL, NULL, retval); @@ -2203,82 +3647,600 @@ kevent_get_timeout(struct proc *p, return 0; } -static int -kevent_set_kq_mode(struct kqueue *kq, unsigned int flags) +static int +kevent_set_kq_mode(struct kqueue *kq, unsigned int flags) +{ + /* each kq should only be used for events of one type */ + kqlock(kq); + if (kq->kq_state & (KQ_KEV32 | KQ_KEV64 | KQ_KEV_QOS)) { + if (flags & KEVENT_FLAG_LEGACY32) { + if ((kq->kq_state & KQ_KEV32) == 0) { + kqunlock(kq); + return EINVAL; + } + } else if (kq->kq_state & KQ_KEV32) { + kqunlock(kq); + return EINVAL; + } + } else if (flags & KEVENT_FLAG_LEGACY32) { + kq->kq_state |= KQ_KEV32; + } else if (flags & KEVENT_FLAG_LEGACY64) { + kq->kq_state |= KQ_KEV64; + } else { + kq->kq_state |= KQ_KEV_QOS; + } + kqunlock(kq); + return 0; +} + +#define KQ_HASH(val, mask) (((val) ^ (val >> 8)) & (mask)) +#define CONFIG_KQ_HASHSIZE CONFIG_KN_HASHSIZE + +static inline void +kqhash_lock(proc_t p) +{ + lck_mtx_lock_spin_always(&p->p_fd->fd_kqhashlock); +} + +static inline void +kqhash_lock_held(__assert_only proc_t p) +{ + LCK_MTX_ASSERT(&p->p_fd->fd_kqhashlock, LCK_MTX_ASSERT_OWNED); +} + +static inline void +kqhash_unlock(proc_t p) +{ + lck_mtx_unlock(&p->p_fd->fd_kqhashlock); +} + +static void +kqueue_hash_init_if_needed(proc_t p) +{ + struct filedesc *fdp = p->p_fd; + + kqhash_lock_held(p); + + if (__improbable(fdp->fd_kqhash == NULL)) { + struct kqlist *alloc_hash; + u_long alloc_mask; + + kqhash_unlock(p); + alloc_hash = hashinit(CONFIG_KQ_HASHSIZE, M_KQUEUE, &alloc_mask); + kqhash_lock(p); + + /* See if we won the race */ + if (fdp->fd_kqhashmask == 0) { + fdp->fd_kqhash = alloc_hash; + fdp->fd_kqhashmask = alloc_mask; + } else { + kqhash_unlock(p); + FREE(alloc_hash, M_KQUEUE); + kqhash_lock(p); + } + } +} + +/* + * Called with the kqhash_lock() held + */ +static void +kqueue_hash_insert( + struct proc *p, + kqueue_id_t id, + struct kqueue *kq) +{ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct filedesc *fdp = p->p_fd; + struct kqlist *list; + + /* should hold the kq hash lock */ + kqhash_lock_held(p); + + if ((kq->kq_state & KQ_DYNAMIC) == 0) { + assert(kq->kq_state & KQ_DYNAMIC); + return; + } + + /* only dynamically allocate workloop kqs for now */ + assert(kq->kq_state & KQ_WORKLOOP); + assert(fdp->fd_kqhash); + + kqwl->kqwl_dynamicid = id; + + list = &fdp->fd_kqhash[KQ_HASH(id, fdp->fd_kqhashmask)]; + SLIST_INSERT_HEAD(list, kqwl, kqwl_hashlink); +} + +/* Called with kqhash_lock held */ +static void +kqueue_hash_remove( + struct proc *p, + struct kqueue *kq) +{ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct filedesc *fdp = p->p_fd; + struct kqlist *list; + + /* should hold the kq hash lock */ + kqhash_lock_held(p); + + if ((kq->kq_state & KQ_DYNAMIC) == 0) { + assert(kq->kq_state & KQ_DYNAMIC); + return; + } + assert(kq->kq_state & KQ_WORKLOOP); /* for now */ + list = &fdp->fd_kqhash[KQ_HASH(kqwl->kqwl_dynamicid, fdp->fd_kqhashmask)]; + SLIST_REMOVE(list, kqwl, kqworkloop, kqwl_hashlink); +} + +/* Called with kqhash_lock held */ +static struct kqueue * +kqueue_hash_lookup(struct proc *p, kqueue_id_t id) +{ + struct filedesc *fdp = p->p_fd; + struct kqlist *list; + struct kqworkloop *kqwl; + + /* should hold the kq hash lock */ + kqhash_lock_held(p); + + if (fdp->fd_kqhashmask == 0) return NULL; + + list = &fdp->fd_kqhash[KQ_HASH(id, fdp->fd_kqhashmask)]; + SLIST_FOREACH(kqwl, list, kqwl_hashlink) { + if (kqwl->kqwl_dynamicid == id) { + struct kqueue *kq = (struct kqueue *)kqwl; + + assert(kq->kq_state & KQ_DYNAMIC); + assert(kq->kq_state & KQ_WORKLOOP); /* for now */ + return kq; + } + } + return NULL; +} + +static inline void +kqueue_release_last(struct proc *p, struct kqueue *kq) +{ + if (kq->kq_state & KQ_DYNAMIC) { + kqhash_lock(p); + if (kqueue_release(kq, KQUEUE_MIGHT_BE_LAST_REF)) { + kqueue_hash_remove(p, kq); + kqhash_unlock(p); + kqueue_dealloc(kq); + } else { + kqhash_unlock(p); + } + } +} + +static struct kqueue * +kevent_get_bound_kq(__assert_only struct proc *p, thread_t thread, + unsigned int kev_flags, unsigned int kq_flags) +{ + struct kqueue *kq; + struct uthread *ut = get_bsdthread_info(thread); + + assert(p == get_bsdthreadtask_info(thread)); + + if (!(ut->uu_kqueue_flags & kev_flags)) + return NULL; + + kq = ut->uu_kqueue_bound; + if (!kq) + return NULL; + + if (!(kq->kq_state & kq_flags)) + return NULL; + + return kq; +} + +static int +kevent_get_kq(struct proc *p, kqueue_id_t id, unsigned int flags, struct fileproc **fpp, int *fdp, struct kqueue **kqp) +{ + struct filedesc *descp = p->p_fd; + struct fileproc *fp = NULL; + struct kqueue *kq; + int fd = 0; + int error = 0; + + /* Was the workloop flag passed? Then it is for sure only a workloop */ + if (flags & KEVENT_FLAG_DYNAMIC_KQUEUE) { + assert(flags & KEVENT_FLAG_WORKLOOP); + if (id == (kqueue_id_t)-1 && + (flags & KEVENT_FLAG_KERNEL) && + (flags & KEVENT_FLAG_WORKLOOP)) { + + assert(is_workqueue_thread(current_thread())); + + /* + * when kevent_id_internal is called from within the + * kernel, and the passed 'id' value is '-1' then we + * look for the currently bound workloop kq. + * + * Until pthread kext avoids calling in to kevent_id_internal + * for threads whose fulfill is canceled, calling in unbound + * can't be fatal. + */ + kq = kevent_get_bound_kq(p, current_thread(), + KEVENT_FLAG_WORKLOOP, KQ_WORKLOOP); + if (kq) { + kqueue_retain(kq); + } else { + struct uthread *ut = get_bsdthread_info(current_thread()); + + /* If thread is unbound due to cancel, just return an error */ + if (ut->uu_kqueue_flags == KEVENT_FLAG_WORKLOOP_CANCELED) { + ut->uu_kqueue_flags = 0; + error = ECANCELED; + } else { + panic("Unbound thread called kevent_internal with id=-1" + " uu_kqueue_flags:0x%x, uu_kqueue_bound:%p", + ut->uu_kqueue_flags, ut->uu_kqueue_bound); + } + } + + *fpp = NULL; + *fdp = 0; + *kqp = kq; + return error; + } + + /* try shortcut on kq lookup for bound threads */ + kq = kevent_get_bound_kq(p, current_thread(), KEVENT_FLAG_WORKLOOP, KQ_WORKLOOP); + if (kq != NULL && ((struct kqworkloop *)kq)->kqwl_dynamicid == id) { + + if (flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST) { + error = EEXIST; + kq = NULL; + goto out; + } + + /* retain a reference while working with this kq. */ + assert(kq->kq_state & KQ_DYNAMIC); + kqueue_retain(kq); + error = 0; + goto out; + } + + /* look for the kq on the hash table */ + kqhash_lock(p); + kq = kqueue_hash_lookup(p, id); + if (kq == NULL) { + kqhash_unlock(p); + + if (flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST) { + error = ENOENT; + goto out; + } + + struct kqueue *alloc_kq; + alloc_kq = kqueue_alloc(p, flags); + if (alloc_kq) { + kqhash_lock(p); + kqueue_hash_init_if_needed(p); + kq = kqueue_hash_lookup(p, id); + if (kq == NULL) { + /* insert our new one */ + kq = alloc_kq; + kqueue_hash_insert(p, id, kq); + kqhash_unlock(p); + } else { + /* lost race, retain existing workloop */ + kqueue_retain(kq); + kqhash_unlock(p); + kqueue_release(alloc_kq, KQUEUE_MIGHT_BE_LAST_REF); + kqueue_dealloc(alloc_kq); + } + } else { + error = ENOMEM; + goto out; + } + } else { + + if (flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST) { + kqhash_unlock(p); + kq = NULL; + error = EEXIST; + goto out; + } + + /* retain a reference while working with this kq. */ + assert(kq->kq_state & KQ_DYNAMIC); + kqueue_retain(kq); + kqhash_unlock(p); + } + + } else if (flags & KEVENT_FLAG_WORKQ) { + /* must already exist for bound threads. */ + if (flags & KEVENT_FLAG_KERNEL) { + assert(descp->fd_wqkqueue != NULL); + } + + /* + * use the private kq associated with the proc workq. + * Just being a thread within the process (and not + * being the exit/exec thread) is enough to hold a + * reference on this special kq. + */ + kq = descp->fd_wqkqueue; + if (kq == NULL) { + struct kqueue *alloc_kq = kqueue_alloc(p, KEVENT_FLAG_WORKQ); + if (alloc_kq == NULL) + return ENOMEM; + + knhash_lock(p); + if (descp->fd_wqkqueue == NULL) { + kq = descp->fd_wqkqueue = alloc_kq; + knhash_unlock(p); + } else { + knhash_unlock(p); + kq = descp->fd_wqkqueue; + kqueue_dealloc(alloc_kq); + } + } + } else { + /* get a usecount for the kq itself */ + fd = (int)id; + if ((error = fp_getfkq(p, fd, &fp, &kq)) != 0) + return (error); + } + if ((error = kevent_set_kq_mode(kq, flags)) != 0) { + /* drop the usecount */ + if (fp != NULL) + fp_drop(p, fd, fp, 0); + return error; + } + +out: + *fpp = fp; + *fdp = fd; + *kqp = kq; + + return error; +} + +static void +kevent_put_kq( + struct proc *p, + kqueue_id_t id, + struct fileproc *fp, + struct kqueue *kq) +{ + kqueue_release_last(p, kq); + if (fp != NULL) { + assert((kq->kq_state & KQ_WORKQ) == 0); + fp_drop(p, (int)id, fp, 0); + } +} + +static uint64_t +kevent_workloop_serial_no_copyin(proc_t p, uint64_t workloop_id) +{ + uint64_t serial_no = 0; + user_addr_t addr; + int rc; + + if (workloop_id == 0 || p->p_dispatchqueue_serialno_offset == 0) { + return 0; + } + addr = (user_addr_t)(workloop_id + p->p_dispatchqueue_serialno_offset); + + if (proc_is64bit(p)) { + rc = copyin(addr, (caddr_t)&serial_no, sizeof(serial_no)); + } else { + uint32_t serial_no32 = 0; + rc = copyin(addr, (caddr_t)&serial_no32, sizeof(serial_no32)); + serial_no = serial_no32; + } + return rc == 0 ? serial_no : 0; +} + +int +kevent_exit_on_workloop_ownership_leak(thread_t thread) +{ + proc_t p = current_proc(); + struct filedesc *fdp = p->p_fd; + kqueue_id_t workloop_id = 0; + os_reason_t reason; + mach_vm_address_t addr; + uint32_t reason_size; + + kqhash_lock(p); + if (fdp->fd_kqhashmask > 0) { + for (uint32_t i = 0; i < fdp->fd_kqhashmask + 1; i++) { + struct kqworkloop *kqwl; + + SLIST_FOREACH(kqwl, &fdp->fd_kqhash[i], kqwl_hashlink) { + struct kqueue *kq = &kqwl->kqwl_kqueue; + if ((kq->kq_state & KQ_DYNAMIC) && kqwl->kqwl_owner == thread) { + workloop_id = kqwl->kqwl_dynamicid; + break; + } + } + } + } + kqhash_unlock(p); + assert(workloop_id); + + reason = os_reason_create(OS_REASON_LIBSYSTEM, + OS_REASON_LIBSYSTEM_CODE_WORKLOOP_OWNERSHIP_LEAK); + if (reason == OS_REASON_NULL) { + goto out; + } + + reason->osr_flags |= OS_REASON_FLAG_GENERATE_CRASH_REPORT; + reason_size = 2 * sizeof(uint64_t); + reason_size = kcdata_estimate_required_buffer_size(2, reason_size); + if (os_reason_alloc_buffer(reason, reason_size) != 0) { + goto out; + } + + struct kcdata_descriptor *kcd = &reason->osr_kcd_descriptor; + + if (kcdata_get_memory_addr(kcd, EXIT_REASON_WORKLOOP_ID, + sizeof(workloop_id), &addr) == KERN_SUCCESS) { + kcdata_memcpy(kcd, addr, &workloop_id, sizeof(workloop_id)); + } + + uint64_t serial_no = kevent_workloop_serial_no_copyin(p, workloop_id); + if (serial_no && kcdata_get_memory_addr(kcd, EXIT_REASON_DISPATCH_QUEUE_NO, + sizeof(serial_no), &addr) == KERN_SUCCESS) { + kcdata_memcpy(kcd, addr, &serial_no, sizeof(serial_no)); + } + +out: +#if DEVELOPMENT || DEBUG + psignal_try_thread_with_reason(p, thread, SIGABRT, reason); + return 0; +#else + return exit_with_reason(p, W_EXITCODE(0, SIGKILL), (int *)NULL, + FALSE, FALSE, 0, reason); +#endif +} + + +static int +kevent_servicer_detach_preflight(thread_t thread, unsigned int flags, struct kqueue *kq) +{ + int error = 0; + struct kqworkloop *kqwl; + struct uthread *ut; + struct kqrequest *kqr; + + if (!(flags & KEVENT_FLAG_WORKLOOP) || !(kq->kq_state & KQ_WORKLOOP)) + return EINVAL; + + /* only kq created with KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD from userspace can have attached threads */ + if (!(kq->kq_state & KQ_NO_WQ_THREAD)) + return EINVAL; + + /* allow detach only on not wq threads */ + if (is_workqueue_thread(thread)) + return EINVAL; + + /* check that the current thread is bound to the requested wq */ + ut = get_bsdthread_info(thread); + if (ut->uu_kqueue_bound != kq) + return EINVAL; + + kqwl = (struct kqworkloop *)kq; + kqwl_req_lock(kqwl); + kqr = &kqwl->kqwl_request; + + /* check that the wq is bound to the thread */ + if ((kqr->kqr_state & KQR_BOUND) == 0 || (kqr->kqr_thread != thread)) + error = EINVAL; + + kqwl_req_unlock(kqwl); + + return error; +} + +static void +kevent_servicer_detach_thread(struct proc *p, kqueue_id_t id, thread_t thread, + unsigned int flags, struct kqueue *kq) { - /* each kq should only be used for events of one type */ + struct kqworkloop *kqwl; + struct uthread *ut; + + assert((flags & KEVENT_FLAG_WORKLOOP) && (kq->kq_state & KQ_WORKLOOP)); + + /* allow detach only on not wqthreads threads */ + assert(!is_workqueue_thread(thread)); + + /* only kq created with KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD from userspace can have attached threads */ + assert(kq->kq_state & KQ_NO_WQ_THREAD); + + /* check that the current thread is bound to the requested kq */ + ut = get_bsdthread_info(thread); + assert(ut->uu_kqueue_bound == kq); + + kqwl = (struct kqworkloop *)kq; + kqlock(kq); - if (kq->kq_state & (KQ_KEV32 | KQ_KEV64 | KQ_KEV_QOS)) { - if (flags & KEVENT_FLAG_LEGACY32) { - if ((kq->kq_state & KQ_KEV32) == 0) { - kqunlock(kq); - return EINVAL; - } - } else if (kq->kq_state & KQ_KEV32) { - kqunlock(kq); - return EINVAL; - } - } else if (flags & KEVENT_FLAG_LEGACY32) { - kq->kq_state |= KQ_KEV32; - } else { - /* JMM - set KQ_KEVQOS when we are ready for exclusive */ - kq->kq_state |= KQ_KEV64; - } + + /* unbind the thread. + * unbind itself checks if still processing and ends it. + */ + kqworkloop_unbind_thread(kqwl, thread, flags); + kqunlock(kq); - return 0; + + kevent_put_kq(p, id, NULL, kq); + + return; } static int -kevent_get_kq(struct proc *p, int fd, unsigned int flags, struct fileproc **fpp, struct kqueue **kqp) +kevent_servicer_attach_thread(thread_t thread, unsigned int flags, struct kqueue *kq) { - struct fileproc *fp = NULL; - struct kqueue *kq; - int error; + int error = 0; + struct kqworkloop *kqwl; + struct uthread *ut; + struct kqrequest *kqr; - if (flags & KEVENT_FLAG_WORKQ) { - /* - * use the private kq associated with the proc workq. - * Just being a thread within the process (and not - * being the exit/exec thread) is enough to hold a - * reference on this special kq. - */ - kq = p->p_wqkqueue; - if (kq == NULL) { - struct kqueue *alloc_kq = kqueue_alloc(p, KEVENT_FLAG_WORKQ); - if (alloc_kq == NULL) - return ENOMEM; + if (!(flags & KEVENT_FLAG_WORKLOOP) || !(kq->kq_state & KQ_WORKLOOP)) + return EINVAL; - proc_fdlock(p); - if (p->p_wqkqueue == NULL) { - kq = p->p_wqkqueue = alloc_kq; - proc_fdunlock(p); - } else { - proc_fdunlock(p); - kq = p->p_wqkqueue; - kqueue_dealloc(alloc_kq); - } - } - } else { - /* get a usecount for the kq itself */ - if ((error = fp_getfkq(p, fd, &fp, &kq)) != 0) - return (error); + /* only kq created with KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD from userspace can have attached threads*/ + if (!(kq->kq_state & KQ_NO_WQ_THREAD)) + return EINVAL; + + /* allow attach only on not wqthreads */ + if (is_workqueue_thread(thread)) + return EINVAL; + + /* check that the thread is not already bound */ + ut = get_bsdthread_info(thread); + if (ut->uu_kqueue_bound != NULL) + return EINVAL; + + assert(ut->uu_kqueue_flags == 0); + + kqlock(kq); + kqwl = (struct kqworkloop *)kq; + kqwl_req_lock(kqwl); + kqr = &kqwl->kqwl_request; + + /* check that the kqueue is not already bound */ + if (kqr->kqr_state & (KQR_BOUND | KQR_THREQUESTED | KQR_DRAIN)) { + error = EINVAL; + goto out; } - if ((error = kevent_set_kq_mode(kq, flags)) != 0) { - /* drop the usecount */ - if (fp != NULL) - fp_drop(p, fd, fp, 0); - return error; - } - *fpp = fp; - *kqp = kq; - return 0; + assert(kqr->kqr_thread == NULL); + assert((kqr->kqr_state & KQR_PROCESSING) == 0); + + kqr->kqr_state |= KQR_THREQUESTED; + kqr->kqr_qos_index = THREAD_QOS_UNSPECIFIED; + kqr->kqr_override_index = THREAD_QOS_UNSPECIFIED; + kqr->kqr_dsync_owner_qos = THREAD_QOS_UNSPECIFIED; + kqr->kqr_owner_override_is_sync = 0; + + kqworkloop_bind_thread_impl(kqwl, thread, KEVENT_FLAG_WORKLOOP); + + /* get a ref on the wlkq on behalf of the attached thread */ + kqueue_retain(kq); + +out: + kqwl_req_unlock(kqwl); + kqunlock(kq); + + return error; } +static inline +boolean_t kevent_args_requesting_events(unsigned int flags, int nevents) +{ + return (!(flags & KEVENT_FLAG_ERROR_EVENTS) && nevents > 0); +} static int -kevent_internal(struct proc *p, - int fd, +kevent_internal(struct proc *p, + kqueue_id_t id, kqueue_id_t *id_out, user_addr_t changelist, int nchanges, user_addr_t ueventlist, int nevents, user_addr_t data_out, uint64_t data_available, @@ -2291,17 +4253,40 @@ kevent_internal(struct proc *p, uthread_t ut; struct kqueue *kq; struct fileproc *fp = NULL; + int fd = 0; struct kevent_internal_s kev; int error, noutputs; struct timeval atv; user_size_t data_size; user_size_t data_resid; + thread_t thread = current_thread(); - /* Don't allow user-space threads to process output events from the workq kq */ - if ((flags & (KEVENT_FLAG_WORKQ | KEVENT_FLAG_KERNEL)) == KEVENT_FLAG_WORKQ && - !(flags & KEVENT_FLAG_ERROR_EVENTS) && nevents > 0) + /* Don't allow user-space threads to process output events from the workq kqs */ + if (((flags & (KEVENT_FLAG_WORKQ | KEVENT_FLAG_KERNEL)) == KEVENT_FLAG_WORKQ) && + kevent_args_requesting_events(flags, nevents)) return EINVAL; + /* restrict dynamic kqueue allocation to workloops (for now) */ + if ((flags & (KEVENT_FLAG_DYNAMIC_KQUEUE | KEVENT_FLAG_WORKLOOP)) == KEVENT_FLAG_DYNAMIC_KQUEUE) + return EINVAL; + + if (flags & (KEVENT_FLAG_WORKLOOP_SERVICER_ATTACH | KEVENT_FLAG_WORKLOOP_SERVICER_DETACH | + KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST | KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST | KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD)) { + + /* allowed only on workloops when calling kevent_id from user-space */ + if (!(flags & KEVENT_FLAG_WORKLOOP) || (flags & KEVENT_FLAG_KERNEL) || !(flags & KEVENT_FLAG_DYNAMIC_KQUEUE)) + return EINVAL; + + /* cannot attach and detach simultaneously*/ + if ((flags & KEVENT_FLAG_WORKLOOP_SERVICER_ATTACH) && (flags & KEVENT_FLAG_WORKLOOP_SERVICER_DETACH)) + return EINVAL; + + /* cannot ask for events and detach */ + if ((flags & KEVENT_FLAG_WORKLOOP_SERVICER_DETACH) && kevent_args_requesting_events(flags, nevents)) + return EINVAL; + + } + /* prepare to deal with stack-wise allocation of out events */ if (flags & KEVENT_FLAG_STACK_EVENTS) { int scale = ((flags & KEVENT_FLAG_LEGACY32) ? @@ -2323,10 +4308,42 @@ kevent_internal(struct proc *p, return error; /* get the kq we are going to be working on */ - error = kevent_get_kq(p, fd, flags, &fp, &kq); + error = kevent_get_kq(p, id, flags, &fp, &fd, &kq); if (error) return error; + /* only bound threads can receive events on workloops */ + if ((flags & KEVENT_FLAG_WORKLOOP) && kevent_args_requesting_events(flags, nevents)) { + ut = (uthread_t)get_bsdthread_info(thread); + if (ut->uu_kqueue_bound != kq) { + error = EXDEV; + goto out; + } + + } + + /* attach the current thread if necessary */ + if (flags & KEVENT_FLAG_WORKLOOP_SERVICER_ATTACH) { + error = kevent_servicer_attach_thread(thread, flags, kq); + if (error) + goto out; + } + else { + /* before processing events and committing to the system call, return an error if the thread cannot be detached when requested */ + if (flags & KEVENT_FLAG_WORKLOOP_SERVICER_DETACH) { + error = kevent_servicer_detach_preflight(thread, flags, kq); + if (error) + goto out; + } + } + + if (id_out && kq && (flags & KEVENT_FLAG_WORKLOOP)) { + assert(kq->kq_state & KQ_WORKLOOP); + struct kqworkloop *kqwl; + kqwl = (struct kqworkloop *)kq; + *id_out = kqwl->kqwl_dynamicid; + } + /* register all the change requests the user provided... */ noutputs = 0; while (nchanges > 0 && error == 0) { @@ -2362,9 +4379,8 @@ kevent_internal(struct proc *p, /* process pending events */ if (nevents > 0 && noutputs == 0 && error == 0) { - /* store the continuation/completion data in the uthread */ - ut = (uthread_t)get_bsdthread_info(current_thread()); + ut = (uthread_t)get_bsdthread_info(thread); cont_args = &ut->uu_kevent.ss_kevent; cont_args->fp = fp; cont_args->fd = fd; @@ -2373,7 +4389,7 @@ kevent_internal(struct proc *p, cont_args->eventcount = nevents; cont_args->eventout = noutputs; cont_args->data_available = data_available; - cont_args->process_data.fp_fd = fd; + cont_args->process_data.fp_fd = (int)id; cont_args->process_data.fp_flags = flags; cont_args->process_data.fp_data_out = data_out; cont_args->process_data.fp_data_size = data_size; @@ -2395,6 +4411,15 @@ kevent_internal(struct proc *p, } } + /* detach the current thread if necessary */ + if (flags & KEVENT_FLAG_WORKLOOP_SERVICER_DETACH) { + assert(fp == NULL); + kevent_servicer_detach_thread(p, id, thread, flags, kq); + } + +out: + kevent_put_kq(p, id, fp, kq); + /* don't restart after signals... */ if (error == ERESTART) error = EINTR; @@ -2402,8 +4427,6 @@ kevent_internal(struct proc *p, error = 0; if (error == 0) *retval = noutputs; - if (fp != NULL) - fp_drop(p, fd, fp, 0); return (error); } @@ -2482,10 +4505,12 @@ kevent_register(struct kqueue *kq, struct kevent_internal_s *kev, __unused struct proc *ctxp) { struct proc *p = kq->kq_p; - struct filterops *fops; + const struct filterops *fops; struct knote *kn = NULL; int result = 0; int error = 0; + unsigned short kev_flags = kev->flags; + int knoteuse_flags = KNUSE_NONE; if (kev->filter < 0) { if (kev->filter + EVFILT_SYSCOUNT < 0) { @@ -2511,36 +4536,46 @@ kevent_register(struct kqueue *kq, struct kevent_internal_s *kev, if (kev->flags & EV_DISABLE) kev->flags &= ~EV_ENABLE; -restart: + if (kq->kq_state & KQ_WORKLOOP) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_REGISTER), + ((struct kqworkloop *)kq)->kqwl_dynamicid, + kev->udata, kev->flags, kev->filter); + } else if (kq->kq_state & KQ_WORKQ) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_REGISTER), + 0, kev->udata, kev->flags, kev->filter); + } else { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_REGISTER), + VM_KERNEL_UNSLIDE_OR_PERM(kq), + kev->udata, kev->flags, kev->filter); + } - proc_fdlock(p); +restart: /* find the matching knote from the fd tables/hashes */ - kn = knote_fdfind(kq, kev, p); + kn = kq_find_knote_and_kq_lock(kq, kev, fops->f_isfd, p); if (kn == NULL) { if (kev->flags & EV_ADD) { - struct fileproc *fp = NULL; + struct fileproc *knote_fp = NULL; /* grab a file reference for the new knote */ if (fops->f_isfd) { - if ((error = fp_lookup(p, kev->ident, &fp, 1)) != 0) { - proc_fdunlock(p); + if ((error = fp_lookup(p, kev->ident, &knote_fp, 0)) != 0) { goto out; } } kn = knote_alloc(); if (kn == NULL) { - proc_fdunlock(p); error = ENOMEM; - if (fp != NULL) - fp_drop(p, kev->ident, fp, 0); + if (knote_fp != NULL) + fp_drop(p, kev->ident, knote_fp, 0); goto out; } - kn->kn_fp = fp; - knote_set_kq(kn,kq); + kn->kn_fp = knote_fp; + knote_set_kq(kn, kq); + kqueue_retain(kq); /* retain a kq ref */ kn->kn_filtid = ~kev->filter; kn->kn_inuse = 1; /* for f_attach() */ kn->kn_status = KN_ATTACHING | KN_ATTACHED; @@ -2570,42 +4605,46 @@ restart: kn->kn_data = 0; /* invoke pthread kext to convert kevent qos to thread qos */ - if (kq->kq_state & KQ_WORKQ) { - kn->kn_qos = canonicalize_kevent_qos(kn->kn_qos); - knote_set_qos_index(kn, qos_index_from_qos(kn->kn_qos, FALSE)); - knote_set_qos_override_index(kn, QOS_INDEX_KQFILE); - assert(knote_get_qos_index(kn) < KQWQ_NQOS); - } else { - knote_set_qos_index(kn, QOS_INDEX_KQFILE); - knote_set_qos_override_index(kn, QOS_INDEX_KQFILE); - } + knote_canonicalize_kevent_qos(kn); + knote_set_qos_index(kn, qos_index_from_qos(kn, kn->kn_qos, FALSE)); /* before anyone can find it */ - if (kev->flags & EV_DISABLE) - knote_disable(kn); + if (kev->flags & EV_DISABLE) { + /* + * do this before anyone can find it, + * this can't call knote_disable() because it expects having + * the kqlock held + */ + kn->kn_status |= KN_DISABLED; + } /* Add the knote for lookup thru the fd table */ - error = knote_fdadd(kn, p); - proc_fdunlock(p); - + error = kq_add_knote(kq, kn, kev, p, &knoteuse_flags); if (error) { + (void)kqueue_release(kq, KQUEUE_CANT_BE_LAST_REF); knote_free(kn); - if (fp != NULL) - fp_drop(p, kev->ident, fp, 0); + if (knote_fp != NULL) + fp_drop(p, kev->ident, knote_fp, 0); + + if (error == ERESTART) { + error = 0; + goto restart; + } goto out; } /* fp reference count now applies to knote */ + /* rwlock boost is now held */ /* call filter attach routine */ - result = fops->f_attach(kn); + result = fops->f_attach(kn, kev); /* * Trade knote use count for kq lock. * Cannot be dropped because we held * KN_ATTACHING throughout. */ - knoteuse2kqlock(kq, kn, 1); + knoteuse2kqlock(kq, kn, KNUSE_STEAL_DROP | knoteuse_flags); if (kn->kn_flags & EV_ERROR) { /* @@ -2636,6 +4675,9 @@ restart: goto out; } + /* Mark the thread request overcommit - if appropos */ + knote_set_qos_overcommit(kn); + /* * If the attach routine indicated that an * event is already fired, activate the knote. @@ -2643,28 +4685,37 @@ restart: if (result) knote_activate(kn); + if (knote_fops(kn)->f_post_attach) { + error = knote_fops(kn)->f_post_attach(kn, kev); + if (error) { + kqunlock(kq); + goto out; + } + } + } else { - proc_fdunlock(p); - error = ENOENT; + if ((kev_flags & (EV_ADD | EV_DELETE)) == (EV_ADD | EV_DELETE) && + (kq->kq_state & KQ_WORKLOOP)) { + /* + * For workloops, understand EV_ADD|EV_DELETE as a "soft" delete + * that doesn't care about ENOENT, so just pretend the deletion + * happened. + */ + } else { + error = ENOENT; + } goto out; } } else { - /* existing knote - get kqueue lock */ - kqlock(kq); - proc_fdunlock(p); + /* existing knote: kqueue lock already taken by kq_find_knote_and_kq_lock */ if ((kn->kn_status & (KN_DROPPING | KN_ATTACHING)) != 0) { /* * The knote is not in a stable state, wait for that * transition to complete and then redrive the lookup. */ - kn->kn_status |= KN_USEWAIT; - waitq_assert_wait64((struct waitq *)&kq->kq_wqs, - CAST_EVENT64_T(&kn->kn_status), - THREAD_UNINT, TIMEOUT_WAIT_FOREVER); - kqunlock(kq); - thread_block(THREAD_CONTINUE_NULL); + knoteusewait(kq, kn); goto restart; } @@ -2681,7 +4732,19 @@ restart: kn->kn_status |= KN_DEFERDELETE; kqunlock(kq); error = EINPROGRESS; + } else if (knote_fops(kn)->f_drop_and_unlock) { + /* + * The filter has requested to handle EV_DELETE events + * + * ERESTART means the kevent has to be re-evaluated + */ + error = knote_fops(kn)->f_drop_and_unlock(kn, kev); + if (error == ERESTART) { + error = 0; + goto restart; + } } else if (kqlock2knotedrop(kq, kn)) { + /* standard/default EV_DELETE path */ knote_drop(kn, p); } else { /* @@ -2723,8 +4786,10 @@ restart: * Convert the kqlock to a use reference on the * knote so we can call the filter touch routine. */ - if (kqlock2knoteuse(kq, kn)) { - + if (knoteuse_needs_boost(kn, kev)) { + knoteuse_flags |= KNUSE_BOOST; + } + if (kqlock2knoteuse(kq, kn, knoteuse_flags)) { /* * Call touch routine to notify filter of changes * in filter values (and to re-determine if any @@ -2733,7 +4798,14 @@ restart: result = knote_fops(kn)->f_touch(kn, kev); /* Get the kq lock back (don't defer droppers). */ - if (!knoteuse2kqlock(kq, kn, 0)) { + if (!knoteuse2kqlock(kq, kn, knoteuse_flags)) { + kqunlock(kq); + goto out; + } + + /* Handle errors during touch routine */ + if (kev->flags & EV_ERROR) { + error = kev->data; kqunlock(kq); goto out; } @@ -2752,7 +4824,7 @@ restart: /* still have kqlock held and knote is valid */ kqunlock(kq); - out: +out: /* output local errors through the kevent */ if (error) { kev->flags |= EV_ERROR; @@ -2803,6 +4875,21 @@ knote_process(struct knote *kn, assert(kn->kn_status & (KN_ACTIVE|KN_STAYACTIVE)); assert(!(kn->kn_status & (KN_DISABLED|KN_SUPPRESSED|KN_DROPPING))); + if (kq->kq_state & KQ_WORKLOOP) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS), + ((struct kqworkloop *)kq)->kqwl_dynamicid, + kn->kn_udata, kn->kn_status | (kn->kn_id << 32), + kn->kn_filtid); + } else if (kq->kq_state & KQ_WORKQ) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS), + 0, kn->kn_udata, kn->kn_status | (kn->kn_id << 32), + kn->kn_filtid); + } else { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS), + VM_KERNEL_UNSLIDE_OR_PERM(kq), kn->kn_udata, + kn->kn_status | (kn->kn_id << 32), kn->kn_filtid); + } + /* * For deferred-drop or vanished events, we just create a fake * event to acknowledge end-of-life. Otherwise, we call the @@ -2822,26 +4909,30 @@ knote_process(struct knote *kn, knote_suppress(kn); } else { - + int flags = KNUSE_NONE; /* deactivate - so new activations indicate a wakeup */ knote_deactivate(kn); /* suppress knotes to avoid returning the same event multiple times in a single call. */ knote_suppress(kn); + if (knoteuse_needs_boost(kn, NULL)) { + flags |= KNUSE_BOOST; + } /* convert lock to a knote use reference */ - if (!kqlock2knoteuse(kq, kn)) + if (!kqlock2knoteuse(kq, kn, flags)) panic("dropping knote found on queue\n"); /* call out to the filter to process with just a ref */ result = knote_fops(kn)->f_process(kn, process_data, &kev); + if (result) flags |= KNUSE_STEAL_DROP; /* * convert our reference back to a lock. accept drop * responsibility from others if we've committed to * delivering event data. */ - if (!knoteuse2kqlock(kq, kn, result)) { + if (!knoteuse2kqlock(kq, kn, flags)) { /* knote dropped */ kn = NULL; } @@ -2932,54 +5023,139 @@ kqworkq_begin_processing(struct kqworkq *kqwq, kq_index_t qos_index, int flags) struct kqrequest *kqr; thread_t self = current_thread(); __assert_only struct uthread *ut = get_bsdthread_info(self); - thread_t thread; assert(kqwq->kqwq_state & KQ_WORKQ); assert(qos_index < KQWQ_NQOS); + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN) | DBG_FUNC_START, + flags, qos_index); + kqwq_req_lock(kqwq); - kqr = kqworkq_get_request(kqwq, qos_index); - thread = kqr->kqr_thread; + kqr = kqworkq_get_request(kqwq, qos_index); - /* manager skips buckets that haven't ask for its help */ + /* manager skips buckets that haven't asked for its help */ if (flags & KEVENT_FLAG_WORKQ_MANAGER) { /* If nothing for manager to do, just return */ if ((kqr->kqr_state & KQWQ_THMANAGER) == 0) { - assert(kqr->kqr_thread != self); + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN) | DBG_FUNC_END, + 0, kqr->kqr_state); kqwq_req_unlock(kqwq); return -1; } - /* bind manager thread from this time on */ - kqworkq_bind_thread(kqwq, qos_index, self, flags); + kqworkq_bind_thread_impl(kqwq, qos_index, self, flags); } else { - /* must have been bound by now */ - assert(thread == self); - assert(ut->uu_kqueue_bound == qos_index); + /* We should already be bound to this kqueue */ + assert(kqr->kqr_state & KQR_BOUND); + assert(kqr->kqr_thread == self); + assert(ut->uu_kqueue_bound == (struct kqueue *)kqwq); + assert(ut->uu_kqueue_qos_index == qos_index); assert((ut->uu_kqueue_flags & flags) == ut->uu_kqueue_flags); } - /* nobody else should still be processing */ - assert(kqr->kqr_state & KQWQ_THREQUESTED); - assert((kqr->kqr_state & KQWQ_PROCESSING) == 0); - - /* anything left to process? */ - if (kqueue_queue_empty(&kqwq->kqwq_kqueue, qos_index)) { - kqwq_req_unlock(kqwq); - return -1; - } + /* + * we should have been requested to be here + * and nobody else should still be processing + */ + assert(kqr->kqr_state & KQR_WAKEUP); + assert(kqr->kqr_state & KQR_THREQUESTED); + assert((kqr->kqr_state & KQR_PROCESSING) == 0); + + /* reset wakeup trigger to catch new events after we start processing */ + kqr->kqr_state &= ~KQR_WAKEUP; /* convert to processing mode */ - /* reset workq triggers and thread requests - maybe processing */ - kqr->kqr_state &= ~(KQWQ_HOOKCALLED | KQWQ_WAKEUP); - kqr->kqr_state |= KQWQ_PROCESSING; + kqr->kqr_state |= KQR_PROCESSING; + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN) | DBG_FUNC_END, + kqr_thread_id(kqr), kqr->kqr_state); + kqwq_req_unlock(kqwq); return 0; } +static inline bool +kqworkloop_is_processing_on_current_thread(struct kqworkloop *kqwl) +{ + struct kqueue *kq = &kqwl->kqwl_kqueue; + + kqlock_held(kq); + + if (kq->kq_state & KQ_PROCESSING) { + /* + * KQ_PROCESSING is unset with the kqlock held, and the kqr thread is + * never modified while KQ_PROCESSING is set, meaning that peeking at + * its value is safe from this context. + */ + return kqwl->kqwl_request.kqr_thread == current_thread(); + } + return false; +} + +static void +kqworkloop_acknowledge_events(struct kqworkloop *kqwl, boolean_t clear_ipc_override) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + struct knote *kn, *tmp; + + kqlock_held(&kqwl->kqwl_kqueue); + + TAILQ_FOREACH_SAFE(kn, &kqr->kqr_suppressed, kn_tqe, tmp) { + /* + * If a knote that can adjust QoS is disabled because of the automatic + * behavior of EV_DISPATCH, the knotes should stay suppressed so that + * further overrides keep pushing. + */ + if (knote_fops(kn)->f_adjusts_qos && (kn->kn_status & KN_DISABLED) && + (kn->kn_status & (KN_STAYACTIVE | KN_DROPPING)) == 0 && + (kn->kn_flags & (EV_DISPATCH | EV_DISABLE)) == EV_DISPATCH) { + /* + * When called from unbind, clear the sync ipc override on the knote + * for events which are delivered. + */ + if (clear_ipc_override) { + knote_adjust_sync_qos(kn, THREAD_QOS_UNSPECIFIED, FALSE); + } + continue; + } + knote_unsuppress(kn); + } +} + +static int +kqworkloop_begin_processing(struct kqworkloop *kqwl, + __assert_only unsigned int flags) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + struct kqueue *kq = &kqwl->kqwl_kqueue; + + kqlock_held(kq); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_BEGIN) | DBG_FUNC_START, + kqwl->kqwl_dynamicid, flags, 0); + + kqwl_req_lock(kqwl); + + /* nobody else should still be processing */ + assert((kqr->kqr_state & KQR_PROCESSING) == 0); + assert((kq->kq_state & KQ_PROCESSING) == 0); + + kqr->kqr_state |= KQR_PROCESSING | KQR_R2K_NOTIF_ARMED; + kq->kq_state |= KQ_PROCESSING; + + kqwl_req_unlock(kqwl); + + kqworkloop_acknowledge_events(kqwl, FALSE); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_BEGIN) | DBG_FUNC_END, + kqwl->kqwl_dynamicid, flags, 0); + + return 0; +} + /* * Return 0 to indicate that processing should proceed, * -1 if there is nothing to process. @@ -2993,15 +5169,26 @@ kqueue_begin_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int fl { struct kqtailq *suppressq; - if (kq->kq_state & KQ_WORKQ) + kqlock_held(kq); + + if (kq->kq_state & KQ_WORKQ) { return kqworkq_begin_processing((struct kqworkq *)kq, qos_index, flags); + } else if (kq->kq_state & KQ_WORKLOOP) { + return kqworkloop_begin_processing((struct kqworkloop*)kq, flags); + } + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_START, + VM_KERNEL_UNSLIDE_OR_PERM(kq), flags); assert(qos_index == QOS_INDEX_KQFILE); /* wait to become the exclusive processing thread */ for (;;) { - if (kq->kq_state & KQ_DRAIN) + if (kq->kq_state & KQ_DRAIN) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END, + VM_KERNEL_UNSLIDE_OR_PERM(kq), 2); return -1; + } if ((kq->kq_state & KQ_PROCESSING) == 0) break; @@ -3023,14 +5210,20 @@ kqueue_begin_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int fl /* clear pre-posts and KQ_WAKEUP now, in case we bail early */ waitq_set_clear_preposts(&kq->kq_wqs); kq->kq_state &= ~KQ_WAKEUP; - + /* anything left to process? */ - if (kqueue_queue_empty(kq, qos_index)) + if (kqueue_queue_empty(kq, qos_index)) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END, + VM_KERNEL_UNSLIDE_OR_PERM(kq), 1); return -1; + } /* convert to processing mode */ kq->kq_state |= KQ_PROCESSING; + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END, + VM_KERNEL_UNSLIDE_OR_PERM(kq)); + return 0; } @@ -3054,52 +5247,63 @@ kqworkq_end_processing(struct kqworkq *kqwq, kq_index_t qos_index, int flags) struct kqtailq *suppressq = kqueue_get_suppressed_queue(kq, qos_index); thread_t self = current_thread(); - __assert_only struct uthread *ut = get_bsdthread_info(self); + struct uthread *ut = get_bsdthread_info(self); struct knote *kn; struct kqrequest *kqr; - int queued_events; - uint16_t pended; thread_t thread; assert(kqwq->kqwq_state & KQ_WORKQ); assert(qos_index < KQWQ_NQOS); - /* leave early if we are not even processing */ - kqwq_req_lock(kqwq); + /* Are we really bound to this kqueue? */ + if (ut->uu_kqueue_bound != kq) { + assert(ut->uu_kqueue_bound == kq); + return; + } + kqr = kqworkq_get_request(kqwq, qos_index); - thread = kqr->kqr_thread; + kqwq_req_lock(kqwq); + + /* Do we claim to be manager? */ if (flags & KEVENT_FLAG_WORKQ_MANAGER) { - assert(ut->uu_kqueue_bound == KQWQ_QOS_MANAGER); - assert(ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER); - /* if this bucket didn't need manager help, bail */ - if ((kqr->kqr_state & KQWQ_THMANAGER) == 0) { - assert(thread != self); + /* bail if not bound that way */ + if (ut->uu_kqueue_qos_index != KQWQ_QOS_MANAGER || + (ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER) == 0) { + assert(ut->uu_kqueue_qos_index == KQWQ_QOS_MANAGER); + assert(ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER); kqwq_req_unlock(kqwq); return; } - assert(kqr->kqr_state & KQWQ_THREQUESTED); - - /* unbound bucket - see if still needs servicing */ - if (thread == THREAD_NULL) { - assert((kqr->kqr_state & KQWQ_PROCESSING) == 0); - assert(TAILQ_EMPTY(suppressq)); - } else { - assert(thread == self); + /* bail if this request wasn't already getting manager help */ + if ((kqr->kqr_state & KQWQ_THMANAGER) == 0 || + (kqr->kqr_state & KQR_PROCESSING) == 0) { + kqwq_req_unlock(kqwq); + return; } - } else { - assert(thread == self); - assert(ut->uu_kqueue_bound == qos_index); - assert((ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER) == 0); + if (ut->uu_kqueue_qos_index != qos_index || + (ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER)) { + assert(ut->uu_kqueue_qos_index == qos_index); + assert((ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER) == 0); + kqwq_req_unlock(kqwq); + return; + } } - kqwq_req_unlock(kqwq); + assert(kqr->kqr_state & KQR_BOUND); + thread = kqr->kqr_thread; + assert(thread == self); - /* Any events queued before we put suppressed ones back? */ - queued_events = !kqueue_queue_empty(kq, qos_index); + assert(kqr->kqr_state & KQR_PROCESSING); + + /* If we didn't drain the whole queue, re-mark a wakeup being needed */ + if (!kqueue_queue_empty(kq, qos_index)) + kqr->kqr_state |= KQR_WAKEUP; + + kqwq_req_unlock(kqwq); /* * Return suppressed knotes to their original state. @@ -3115,51 +5319,95 @@ kqworkq_end_processing(struct kqworkq *kqwq, kq_index_t qos_index, int flags) kqwq_req_lock(kqwq); - /* Determine if wakeup-type events were pended during servicing */ - pended = (kqr->kqr_state & (KQWQ_HOOKCALLED | KQWQ_WAKEUP)); - - /* unbind thread thread */ - kqworkq_unbind_thread(kqwq, qos_index, self, flags); + /* Indicate that we are done processing this request */ + kqr->kqr_state &= ~KQR_PROCESSING; - /* Indicate that we are done processing */ - kqr->kqr_state &= ~(KQWQ_PROCESSING | \ - KQWQ_THREQUESTED | KQWQ_THMANAGER); + /* + * Drop our association with this one request and its + * override on us. + */ + kqworkq_unbind_thread(kqwq, qos_index, thread, flags); /* - * request a new thread if events have happened - * (not just putting stay-active events back). + * request a new thread if we didn't process the whole + * queue or real events have happened (not just putting + * stay-active events back). */ - if ((queued_events || pended) && - !kqueue_queue_empty(kq, qos_index)) { - kqworkq_request_thread(kqwq, qos_index); + if (kqr->kqr_state & KQR_WAKEUP) { + if (kqueue_queue_empty(kq, qos_index)) { + kqr->kqr_state &= ~KQR_WAKEUP; + } else { + kqworkq_request_thread(kqwq, qos_index); + } } - kqwq_req_unlock(kqwq); } +static void +kqworkloop_end_processing(struct kqworkloop *kqwl, int nevents, + unsigned int flags) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + struct kqueue *kq = &kqwl->kqwl_kqueue; + + kqlock_held(kq); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_END) | DBG_FUNC_START, + kqwl->kqwl_dynamicid, flags, 0); + + if ((kq->kq_state & KQ_NO_WQ_THREAD) && nevents == 0 && + (flags & KEVENT_FLAG_IMMEDIATE) == 0) { + /* + * We may soon block, but have returned no + * kevents that need to be kept supressed for overriding purposes. + * + * It is hence safe to acknowledge events and unsuppress everything, so + * that if we block we can observe all events firing. + */ + kqworkloop_acknowledge_events(kqwl, TRUE); + } + + kqwl_req_lock(kqwl); + + assert(kqr->kqr_state & KQR_PROCESSING); + assert(kq->kq_state & KQ_PROCESSING); + + kq->kq_state &= ~KQ_PROCESSING; + kqr->kqr_state &= ~KQR_PROCESSING; + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_RECOMPUTE_WAKEUP_QOS, 0); + + kqwl_req_unlock(kqwl); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_END) | DBG_FUNC_END, + kqwl->kqwl_dynamicid, flags, 0); +} + /* * Called with kqueue lock held. */ static void -kqueue_end_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int flags) +kqueue_end_processing(struct kqueue *kq, kq_index_t qos_index, + int nevents, unsigned int flags) { struct knote *kn; struct kqtailq *suppressq; int procwait; - if (kq->kq_state & KQ_WORKQ) { - kqworkq_end_processing((struct kqworkq *)kq, qos_index, flags); - return; + kqlock_held(kq); + + assert((kq->kq_state & KQ_WORKQ) == 0); + + if (kq->kq_state & KQ_WORKLOOP) { + return kqworkloop_end_processing((struct kqworkloop *)kq, nevents, flags); } + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_END), + VM_KERNEL_UNSLIDE_OR_PERM(kq), flags); + assert(qos_index == QOS_INDEX_KQFILE); /* * Return suppressed knotes to their original state. - * For workq kqueues, suppressed ones that are still - * truly active (not just forced into the queue) will - * set flags we check below to see if anything got - * woken up. */ suppressq = kqueue_get_suppressed_queue(kq, qos_index); while ((kn = TAILQ_FIRST(suppressq)) != NULL) { @@ -3176,8 +5424,226 @@ kqueue_end_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int flag CAST_EVENT64_T(suppressq), THREAD_AWAKENED, WAITQ_ALL_PRIORITIES); - } + } +} + +/* + * kqwq_internal_bind - bind thread to processing workq kqueue + * + * Determines if the provided thread will be responsible for + * servicing the particular QoS class index specified in the + * parameters. Once the binding is done, any overrides that may + * be associated with the cooresponding events can be applied. + * + * This should be called as soon as the thread identity is known, + * preferably while still at high priority during creation. + * + * - caller holds a reference on the process (and workq kq) + * - the thread MUST call kevent_qos_internal after being bound + * or the bucket of events may never be delivered. + * - Nothing locked + * (unless this is a synchronous bind, then the request is locked) + */ +static int +kqworkq_internal_bind( + struct proc *p, + kq_index_t qos_index, + thread_t thread, + unsigned int flags) +{ + struct kqueue *kq; + struct kqworkq *kqwq; + struct kqrequest *kqr; + struct uthread *ut = get_bsdthread_info(thread); + + /* If no process workq, can't be our thread. */ + kq = p->p_fd->fd_wqkqueue; + + if (kq == NULL) + return 0; + + assert(kq->kq_state & KQ_WORKQ); + kqwq = (struct kqworkq *)kq; + + /* + * No need to bind the manager thread to any specific + * bucket, but still claim the thread. + */ + if (qos_index == KQWQ_QOS_MANAGER) { + assert(ut->uu_kqueue_bound == NULL); + assert(flags & KEVENT_FLAG_WORKQ_MANAGER); + ut->uu_kqueue_bound = kq; + ut->uu_kqueue_qos_index = qos_index; + ut->uu_kqueue_flags = flags; + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_BIND), + thread_tid(thread), flags, qos_index); + + return 1; + } + + /* + * If this is a synchronous bind callback, the request + * lock is already held, so just do the bind. + */ + if (flags & KEVENT_FLAG_SYNCHRONOUS_BIND) { + kqwq_req_held(kqwq); + /* strip out synchronout bind flag */ + flags &= ~KEVENT_FLAG_SYNCHRONOUS_BIND; + kqworkq_bind_thread_impl(kqwq, qos_index, thread, flags); + return 1; + } + + /* + * check the request that corresponds to our qos_index + * to see if there is an outstanding request. + */ + kqr = kqworkq_get_request(kqwq, qos_index); + assert(kqr->kqr_qos_index == qos_index); + kqwq_req_lock(kqwq); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_BIND), + thread_tid(thread), flags, qos_index, kqr->kqr_state); + + if ((kqr->kqr_state & KQR_THREQUESTED) && + (kqr->kqr_state & KQR_PROCESSING) == 0) { + + if ((kqr->kqr_state & KQR_BOUND) && + thread == kqr->kqr_thread) { + /* duplicate bind - claim the thread */ + assert(ut->uu_kqueue_bound == kq); + assert(ut->uu_kqueue_qos_index == qos_index); + kqwq_req_unlock(kqwq); + return 1; + } + if ((kqr->kqr_state & (KQR_BOUND | KQWQ_THMANAGER)) == 0) { + /* ours to bind to */ + kqworkq_bind_thread_impl(kqwq, qos_index, thread, flags); + kqwq_req_unlock(kqwq); + return 1; + } + } + kqwq_req_unlock(kqwq); + return 0; +} + +static void +kqworkloop_bind_thread_impl(struct kqworkloop *kqwl, + thread_t thread, + __assert_only unsigned int flags) +{ + assert(flags & KEVENT_FLAG_WORKLOOP); + + /* the request object must be locked */ + kqwl_req_held(kqwl); + + struct kqrequest *kqr = &kqwl->kqwl_request; + struct uthread *ut = get_bsdthread_info(thread); + boolean_t ipc_override_is_sync; + kq_index_t qos_index = kqworkloop_combined_qos(kqwl, &ipc_override_is_sync); + + /* nobody else bound so finally bind (as a workloop) */ + assert(kqr->kqr_state & KQR_THREQUESTED); + assert((kqr->kqr_state & (KQR_BOUND | KQR_PROCESSING)) == 0); + assert(thread != kqwl->kqwl_owner); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_BIND), + kqwl->kqwl_dynamicid, (uintptr_t)thread_tid(thread), + qos_index, + (uintptr_t)(((uintptr_t)kqr->kqr_override_index << 16) | + (((uintptr_t)kqr->kqr_state) << 8) | + ((uintptr_t)ipc_override_is_sync))); + + kqr->kqr_state |= KQR_BOUND | KQR_R2K_NOTIF_ARMED; + kqr->kqr_thread = thread; + + /* bind the workloop to the uthread */ + ut->uu_kqueue_bound = (struct kqueue *)kqwl; + ut->uu_kqueue_flags = flags; + ut->uu_kqueue_qos_index = qos_index; + assert(ut->uu_kqueue_override_is_sync == 0); + ut->uu_kqueue_override_is_sync = ipc_override_is_sync; + if (qos_index) { + thread_add_ipc_override(thread, qos_index); + } + if (ipc_override_is_sync) { + thread_add_sync_ipc_override(thread); + } +} + +/* + * workloop_fulfill_threadreq - bind thread to processing workloop + * + * The provided thread will be responsible for delivering events + * associated with the given kqrequest. Bind it and get ready for + * the thread to eventually arrive. + * + * If WORKLOOP_FULFILL_THREADREQ_SYNC is specified, the callback + * within the context of the pthread_functions->workq_threadreq + * callout. In this case, the request structure is already locked. + */ +int +workloop_fulfill_threadreq(struct proc *p, + workq_threadreq_t req, + thread_t thread, + int flags) +{ + int sync = (flags & WORKLOOP_FULFILL_THREADREQ_SYNC); + int cancel = (flags & WORKLOOP_FULFILL_THREADREQ_CANCEL); + struct kqrequest *kqr; + struct kqworkloop *kqwl; + + kqwl = (struct kqworkloop *)((uintptr_t)req - + offsetof(struct kqworkloop, kqwl_request) - + offsetof(struct kqrequest, kqr_req)); + kqr = &kqwl->kqwl_request; + + /* validate we're looking at something valid */ + if (kqwl->kqwl_p != p || + (kqwl->kqwl_state & KQ_WORKLOOP) == 0) { + assert(kqwl->kqwl_p == p); + assert(kqwl->kqwl_state & KQ_WORKLOOP); + return EINVAL; + } + + if (!sync) + kqwl_req_lock(kqwl); + + /* Should be a pending request */ + if ((kqr->kqr_state & KQR_BOUND) || + (kqr->kqr_state & KQR_THREQUESTED) == 0) { + + assert((kqr->kqr_state & KQR_BOUND) == 0); + assert(kqr->kqr_state & KQR_THREQUESTED); + if (!sync) + kqwl_req_unlock(kqwl); + return EINPROGRESS; + } + + assert((kqr->kqr_state & KQR_DRAIN) == 0); + + /* + * Is it a cancel indication from pthread. + * If so, we must be exiting/exec'ing. Forget + * our pending request. + */ + if (cancel) { + kqr->kqr_state &= ~KQR_THREQUESTED; + kqr->kqr_state |= KQR_DRAIN; + } else { + /* do the actual bind? */ + kqworkloop_bind_thread_impl(kqwl, thread, KEVENT_FLAG_WORKLOOP); + } + + if (!sync) + kqwl_req_unlock(kqwl); + + if (cancel) + kqueue_release_last(p, &kqwl->kqwl_kqueue); /* may dealloc kq */ + + return 0; } + /* * kevent_qos_internal_bind - bind thread to processing kqueue @@ -3203,90 +5669,96 @@ kevent_qos_internal_bind( thread_t thread, unsigned int flags) { - struct fileproc *fp = NULL; - struct kqueue *kq = NULL; - struct kqworkq *kqwq; - struct kqrequest *kqr; - struct uthread *ut; kq_index_t qos_index; - int res = 0; - assert(thread != THREAD_NULL); assert(flags & KEVENT_FLAG_WORKQ); - if (thread == THREAD_NULL || - (flags & KEVENT_FLAG_WORKQ) == 0) { + if (thread == THREAD_NULL || (flags & KEVENT_FLAG_WORKQ) == 0) { return EINVAL; } - ut = get_bsdthread_info(thread); - - /* find the kqueue */ - res = kevent_get_kq(p, -1, flags, &fp, &kq); - assert(fp == NULL); - if (res) - return res; - /* get the qos index we're going to service */ qos_index = qos_index_for_servicer(qos_class, thread, flags); - - /* No need to bind the manager thread to any bucket */ - if (qos_index == KQWQ_QOS_MANAGER) { - assert(ut->uu_kqueue_bound == 0); - ut->uu_kqueue_bound = qos_index; - ut->uu_kqueue_flags = flags; + + if (kqworkq_internal_bind(p, qos_index, thread, flags)) return 0; - } - kqlock(kq); - assert(kq->kq_state & KQ_WORKQ); - - kqwq = (struct kqworkq *)kq; - kqr = kqworkq_get_request(kqwq, qos_index); + return EINPROGRESS; +} - kqwq_req_lock(kqwq); - /* - * A (non-emergency) request should have been made - * and nobody should already be servicing this bucket. - */ - assert(kqr->kqr_state & KQWQ_THREQUESTED); - assert((kqr->kqr_state & KQWQ_THMANAGER) == 0); - assert((kqr->kqr_state & KQWQ_PROCESSING) == 0); +static void +kqworkloop_internal_unbind( + struct proc *p, + thread_t thread, + unsigned int flags) +{ + struct kqueue *kq; + struct kqworkloop *kqwl; + struct uthread *ut = get_bsdthread_info(thread); - /* Is this is an extraneous bind? */ - if (thread == kqr->kqr_thread) { - assert(ut->uu_kqueue_bound == qos_index); - goto out; - } + assert(ut->uu_kqueue_bound != NULL); + kq = ut->uu_kqueue_bound; + assert(kq->kq_state & KQ_WORKLOOP); + kqwl = (struct kqworkloop *)kq; - /* nobody else bound and we're not bound elsewhere */ - assert(ut->uu_kqueue_bound == 0); - assert(ut->uu_kqueue_flags == 0); - assert(kqr->kqr_thread == THREAD_NULL); + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_UNBIND), + kqwl->kqwl_dynamicid, (uintptr_t)thread_tid(thread), + flags, 0); - /* Don't bind if there is a conflict */ - if (kqr->kqr_thread != THREAD_NULL || - (kqr->kqr_state & KQWQ_THMANAGER)) { - res = EINPROGRESS; - goto out; - } + if (!(kq->kq_state & KQ_NO_WQ_THREAD)) { + assert(is_workqueue_thread(thread)); - /* finally bind the thread */ - kqr->kqr_thread = thread; - ut->uu_kqueue_bound = qos_index; - ut->uu_kqueue_flags = flags; + kqlock(kq); + kqworkloop_unbind_thread(kqwl, thread, flags); + kqunlock(kq); - /* add any pending overrides to the thread */ - if (kqr->kqr_override_delta) { - thread_add_ipc_override(thread, qos_index + kqr->kqr_override_delta); + /* If last reference, dealloc the workloop kq */ + kqueue_release_last(p, kq); + } else { + assert(!is_workqueue_thread(thread)); + kevent_servicer_detach_thread(p, kqwl->kqwl_dynamicid, thread, flags, kq); } +} -out: - kqwq_req_unlock(kqwq); - kqunlock(kq); +static void +kqworkq_internal_unbind( + struct proc *p, + kq_index_t qos_index, + thread_t thread, + unsigned int flags) +{ + struct kqueue *kq; + struct kqworkq *kqwq; + struct uthread *ut; + kq_index_t end_index; - return res; + assert(thread == current_thread()); + ut = get_bsdthread_info(thread); + + kq = p->p_fd->fd_wqkqueue; + assert(kq->kq_state & KQ_WORKQ); + assert(ut->uu_kqueue_bound == kq); + + kqwq = (struct kqworkq *)kq; + + /* end servicing any requests we might own */ + end_index = (qos_index == KQWQ_QOS_MANAGER) ? + 0 : qos_index; + kqlock(kq); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_UNBIND), + (uintptr_t)thread_tid(thread), flags, qos_index); + + do { + kqworkq_end_processing(kqwq, qos_index, flags); + } while (qos_index-- > end_index); + + ut->uu_kqueue_bound = NULL; + ut->uu_kqueue_qos_index = 0; + ut->uu_kqueue_flags = 0; + + kqunlock(kq); } /* @@ -3306,56 +5778,49 @@ kevent_qos_internal_unbind( thread_t thread, unsigned int flags) { - struct kqueue *kq; - struct uthread *ut; - struct fileproc *fp = NULL; - kq_index_t qos_index; - kq_index_t end_index; - int res; +#pragma unused(qos_class) - assert(flags & KEVENT_FLAG_WORKQ); - assert(thread == current_thread()); + struct uthread *ut; + struct kqueue *kq; + unsigned int bound_flags; + bool check_flags; - if (thread == THREAD_NULL || - (flags & KEVENT_FLAG_WORKQ) == 0) - return EINVAL; - - /* get the kq */ - res = kevent_get_kq(p, -1, flags, &fp, &kq); - assert(fp == NULL); - if (res) - return res; + ut = get_bsdthread_info(thread); + if (ut->uu_kqueue_bound == NULL) { + /* early out if we are already unbound */ + assert(ut->uu_kqueue_flags == 0); + assert(ut->uu_kqueue_qos_index == 0); + assert(ut->uu_kqueue_override_is_sync == 0); + return EALREADY; + } - assert(kq->kq_state & KQ_WORKQ); + assert(flags & (KEVENT_FLAG_WORKQ | KEVENT_FLAG_WORKLOOP)); + assert(thread == current_thread()); - /* get the index we have been servicing */ - qos_index = qos_index_for_servicer(qos_class, thread, flags); + check_flags = flags & KEVENT_FLAG_UNBIND_CHECK_FLAGS; - ut = get_bsdthread_info(thread); + /* Get the kqueue we started with */ + kq = ut->uu_kqueue_bound; + assert(kq != NULL); + assert(kq->kq_state & (KQ_WORKQ | KQ_WORKLOOP)); - /* early out if we were already unbound - or never bound */ - if (ut->uu_kqueue_bound != qos_index) { - __assert_only struct kqworkq *kqwq = (struct kqworkq *)kq; - __assert_only struct kqrequest *kqr = kqworkq_get_request(kqwq, qos_index); + /* get flags and QoS parameters we started with */ + bound_flags = ut->uu_kqueue_flags; - assert(ut->uu_kqueue_bound == 0); - assert(ut->uu_kqueue_flags == 0); - assert(kqr->kqr_thread != thread); - return EALREADY; - } + /* Unbind from the class of workq */ + if (kq->kq_state & KQ_WORKQ) { + if (check_flags && !(flags & KEVENT_FLAG_WORKQ)) { + return EINVAL; + } - /* unbind from all the buckets we might own */ - end_index = (qos_index == KQWQ_QOS_MANAGER) ? - 0 : qos_index; - kqlock(kq); - do { - kqueue_end_processing(kq, qos_index, flags); - } while (qos_index-- > end_index); - kqunlock(kq); + kqworkq_internal_unbind(p, ut->uu_kqueue_qos_index, thread, bound_flags); + } else { + if (check_flags && !(flags & KEVENT_FLAG_WORKLOOP)) { + return EINVAL; + } - /* indicate that we are done processing in the uthread */ - ut->uu_kqueue_bound = 0; - ut->uu_kqueue_flags = 0; + kqworkloop_internal_unbind(p, thread, bound_flags); + } return 0; } @@ -3380,22 +5845,45 @@ kqueue_process(struct kqueue *kq, kevent_callback_t callback, void *callback_data, struct filt_process_s *process_data, - kq_index_t servicer_qos_index, int *countp, struct proc *p) { unsigned int flags = process_data ? process_data->fp_flags : 0; + struct uthread *ut = get_bsdthread_info(current_thread()); kq_index_t start_index, end_index, i; struct knote *kn; int nevents = 0; int error = 0; /* - * Based on the native QoS of the servicer, - * determine the range of QoSes that need checking + * Based on the mode of the kqueue and the bound QoS of the servicer, + * determine the range of thread requests that need checking */ - start_index = servicer_qos_index; - end_index = (start_index == KQWQ_QOS_MANAGER) ? 0 : start_index; + if (kq->kq_state & KQ_WORKQ) { + if (flags & KEVENT_FLAG_WORKQ_MANAGER) { + start_index = KQWQ_QOS_MANAGER; + } else if (ut->uu_kqueue_bound != kq) { + return EJUSTRETURN; + } else { + start_index = ut->uu_kqueue_qos_index; + } + + /* manager services every request in a workq kqueue */ + assert(start_index > 0 && start_index <= KQWQ_QOS_MANAGER); + end_index = (start_index == KQWQ_QOS_MANAGER) ? 0 : start_index; + + } else if (kq->kq_state & KQ_WORKLOOP) { + if (ut->uu_kqueue_bound != kq) + return EJUSTRETURN; + + /* + * Single request servicing + * we want to deliver all events, regardless of the QOS + */ + start_index = end_index = THREAD_QOS_UNSPECIFIED; + } else { + start_index = end_index = QOS_INDEX_KQFILE; + } i = start_index; @@ -3407,41 +5895,39 @@ kqueue_process(struct kqueue *kq, } /* - * loop through the enqueued knotes, processing each one and - * revalidating those that need it. As they are processed, - * they get moved to the inprocess queue (so the loop can end). + * loop through the enqueued knotes associated with this request, + * processing each one. Each request may have several queues + * of knotes to process (depending on the type of kqueue) so we + * have to loop through all the queues as long as we have additional + * space. */ error = 0; struct kqtailq *base_queue = kqueue_get_base_queue(kq, i); struct kqtailq *queue = kqueue_get_high_queue(kq, i); do { - while (error == 0 && - (kn = TAILQ_FIRST(queue)) != NULL) { - /* Process the knote */ + while (error == 0 && (kn = TAILQ_FIRST(queue)) != NULL) { error = knote_process(kn, callback, callback_data, process_data, p); - if (error == EJUSTRETURN) + if (error == EJUSTRETURN) { error = 0; - else + } else { nevents++; - - /* break out if no more space for additional events */ - if (error == EWOULDBLOCK) { - if ((kq->kq_state & KQ_WORKQ) == 0) - kqueue_end_processing(kq, i, flags); - error = 0; - goto out; } + /* error is EWOULDBLOCK when the out event array is full */ } } while (error == 0 && queue-- > base_queue); - /* let somebody else process events if we're not in workq mode */ - if ((kq->kq_state & KQ_WORKQ) == 0) - kqueue_end_processing(kq, i, flags); + if ((kq->kq_state & KQ_WORKQ) == 0) { + kqueue_end_processing(kq, i, nevents, flags); + } + if (error == EWOULDBLOCK) { + /* break out if no more space for additional events */ + error = 0; + break; + } } while (i-- > end_index); -out: *countp = nevents; return (error); } @@ -3463,11 +5949,16 @@ kqueue_scan_continue(void *data, wait_result_t wait_result) kqlock(kq); retry: error = kqueue_process(kq, cont_args->call, cont_args->data, - process_data, cont_args->servicer_qos_index, - &count, current_proc()); + process_data, &count, current_proc()); if (error == 0 && count == 0) { + if (kq->kq_state & KQ_DRAIN) { + kqunlock(kq); + goto drain; + } + if (kq->kq_state & KQ_WAKEUP) goto retry; + waitq_assert_wait64((struct waitq *)&kq->kq_wqs, KQ_EVENT, THREAD_ABORTSAFE, cont_args->deadline); @@ -3485,6 +5976,7 @@ kqueue_scan_continue(void *data, wait_result_t wait_result) error = EINTR; break; case THREAD_RESTART: + drain: error = EBADF; break; default: @@ -3523,7 +6015,6 @@ kqueue_scan(struct kqueue *kq, struct proc *p) { thread_continue_t cont = THREAD_CONTINUE_NULL; - kq_index_t servicer_qos_index; unsigned int flags; uint64_t deadline; int error; @@ -3537,9 +6028,6 @@ kqueue_scan(struct kqueue *kq, */ flags = (process_data) ? process_data->fp_flags : 0; fd = (process_data) ? process_data->fp_fd : -1; - servicer_qos_index = (kq->kq_state & KQ_WORKQ) ? - qos_index_for_servicer(fd, current_thread(), flags) : - QOS_INDEX_KQFILE; first = 1; for (;;) { @@ -3552,8 +6040,7 @@ kqueue_scan(struct kqueue *kq, */ kqlock(kq); error = kqueue_process(kq, callback, callback_data, - process_data, servicer_qos_index, - &count, p); + process_data, &count, p); if (error || count) break; /* lock still held */ @@ -3588,11 +6075,15 @@ kqueue_scan(struct kqueue *kq, cont_args->deadline = deadline; cont_args->data = callback_data; cont_args->process_data = process_data; - cont_args->servicer_qos_index = servicer_qos_index; cont = kqueue_scan_continue; } } + if (kq->kq_state & KQ_DRAIN) { + kqunlock(kq); + return EBADF; + } + /* If awakened during processing, try again */ if (kq->kq_state & KQ_WAKEUP) { kqunlock(kq); @@ -3745,13 +6236,14 @@ kqueue_select(struct fileproc *fp, int which, void *wq_link_id, while ((kn = (struct knote *)TAILQ_FIRST(suppressq)) != NULL) { unsigned peek = 1; - /* If didn't vanish while suppressed - peek at it */ - if (kqlock2knoteuse(kq, kn)) { + assert(!knoteuse_needs_boost(kn, NULL)); + /* If didn't vanish while suppressed - peek at it */ + if (kqlock2knoteuse(kq, kn, KNUSE_NONE)) { peek = knote_fops(kn)->f_peek(kn); /* if it dropped while getting lock - move on */ - if (!knoteuse2kqlock(kq, kn, 0)) + if (!knoteuse2kqlock(kq, kn, KNUSE_NONE)) continue; } @@ -3767,7 +6259,7 @@ kqueue_select(struct fileproc *fp, int which, void *wq_link_id, } out: - kqueue_end_processing(kq, QOS_INDEX_KQFILE, 0); + kqueue_end_processing(kq, QOS_INDEX_KQFILE, retnum, 0); kqunlock(kq); return (retnum); } @@ -3794,7 +6286,8 @@ kqueue_close(struct fileglob *fg, __unused vfs_context_t ctx) * that relationship is torn down. */ static int -kqueue_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused vfs_context_t ctx) +kqueue_kqfilter(__unused struct fileproc *fp, struct knote *kn, + __unused struct kevent_internal_s *kev, __unused vfs_context_t ctx) { struct kqfile *kqf = (struct kqfile *)kn->kn_fp->f_data; struct kqueue *kq = &kqf->kqf_kqueue; @@ -3877,155 +6370,671 @@ kqueue_stat(struct kqueue *kq, void *ub, int isstat64, proc_t p) if (isstat64 != 0) { struct stat64 *sb64 = (struct stat64 *)ub; - bzero((void *)sb64, sizeof(*sb64)); - sb64->st_size = kq->kq_count; - if (kq->kq_state & KQ_KEV_QOS) - sb64->st_blksize = sizeof(struct kevent_qos_s); - else if (kq->kq_state & KQ_KEV64) - sb64->st_blksize = sizeof(struct kevent64_s); - else if (IS_64BIT_PROCESS(p)) - sb64->st_blksize = sizeof(struct user64_kevent); - else - sb64->st_blksize = sizeof(struct user32_kevent); - sb64->st_mode = S_IFIFO; - } else { - struct stat *sb = (struct stat *)ub; + bzero((void *)sb64, sizeof(*sb64)); + sb64->st_size = kq->kq_count; + if (kq->kq_state & KQ_KEV_QOS) + sb64->st_blksize = sizeof(struct kevent_qos_s); + else if (kq->kq_state & KQ_KEV64) + sb64->st_blksize = sizeof(struct kevent64_s); + else if (IS_64BIT_PROCESS(p)) + sb64->st_blksize = sizeof(struct user64_kevent); + else + sb64->st_blksize = sizeof(struct user32_kevent); + sb64->st_mode = S_IFIFO; + } else { + struct stat *sb = (struct stat *)ub; + + bzero((void *)sb, sizeof(*sb)); + sb->st_size = kq->kq_count; + if (kq->kq_state & KQ_KEV_QOS) + sb->st_blksize = sizeof(struct kevent_qos_s); + else if (kq->kq_state & KQ_KEV64) + sb->st_blksize = sizeof(struct kevent64_s); + else if (IS_64BIT_PROCESS(p)) + sb->st_blksize = sizeof(struct user64_kevent); + else + sb->st_blksize = sizeof(struct user32_kevent); + sb->st_mode = S_IFIFO; + } + kqunlock(kq); + return (0); +} + +/* + * Interact with the pthread kext to request a servicing there. + * Eventually, this will request threads at specific QoS levels. + * For now, it only requests a dispatch-manager-QoS thread, and + * only one-at-a-time. + * + * - Caller holds the workq request lock + * + * - May be called with the kqueue's wait queue set locked, + * so cannot do anything that could recurse on that. + */ +static void +kqworkq_request_thread( + struct kqworkq *kqwq, + kq_index_t qos_index) +{ + struct kqrequest *kqr; + + assert(kqwq->kqwq_state & KQ_WORKQ); + assert(qos_index < KQWQ_NQOS); + + kqr = kqworkq_get_request(kqwq, qos_index); + + assert(kqr->kqr_state & KQR_WAKEUP); + + /* + * If we have already requested a thread, and it hasn't + * started processing yet, there's no use hammering away + * on the pthread kext. + */ + if (kqr->kqr_state & KQR_THREQUESTED) + return; + + assert((kqr->kqr_state & KQR_BOUND) == 0); + + /* request additional workq threads if appropriate */ + if (pthread_functions != NULL && + pthread_functions->workq_reqthreads != NULL) { + unsigned int flags = KEVENT_FLAG_WORKQ; + unsigned long priority; + thread_t wqthread; + + /* Compute the appropriate pthread priority */ + priority = qos_from_qos_index(qos_index); + +#if 0 + /* JMM - for now remain compatible with old invocations */ + /* set the over-commit flag on the request if needed */ + if (kqr->kqr_state & KQR_THOVERCOMMIT) + priority |= _PTHREAD_PRIORITY_OVERCOMMIT_FLAG; +#endif /* 0 */ + + /* Compute a priority based on qos_index. */ + struct workq_reqthreads_req_s request = { + .priority = priority, + .count = 1 + }; + + /* mark that we are making a request */ + kqr->kqr_state |= KQR_THREQUESTED; + if (qos_index == KQWQ_QOS_MANAGER) + kqr->kqr_state |= KQWQ_THMANAGER; + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_THREQUEST), + 0, qos_index, + (((uintptr_t)kqr->kqr_override_index << 8) | + (uintptr_t)kqr->kqr_state)); + wqthread = (*pthread_functions->workq_reqthreads)(kqwq->kqwq_p, 1, &request); + + /* We've been switched to the emergency/manager thread */ + if (wqthread == (thread_t)-1) { + assert(qos_index != KQWQ_QOS_MANAGER); + kqr->kqr_state |= KQWQ_THMANAGER; + return; + } + + /* + * bind the returned thread identity + * This goes away when we switch to synchronous callback + * binding from the pthread kext. + */ + if (wqthread != NULL) { + kqworkq_bind_thread_impl(kqwq, qos_index, wqthread, flags); + } + } +} + +/* + * If we aren't already busy processing events [for this QoS], + * request workq thread support as appropriate. + * + * TBD - for now, we don't segregate out processing by QoS. + * + * - May be called with the kqueue's wait queue set locked, + * so cannot do anything that could recurse on that. + */ +static void +kqworkq_request_help( + struct kqworkq *kqwq, + kq_index_t qos_index) +{ + struct kqrequest *kqr; + + /* convert to thread qos value */ + assert(qos_index < KQWQ_NQOS); + + kqwq_req_lock(kqwq); + kqr = kqworkq_get_request(kqwq, qos_index); + + if ((kqr->kqr_state & KQR_WAKEUP) == 0) { + /* Indicate that we needed help from this request */ + kqr->kqr_state |= KQR_WAKEUP; + + /* Go assure a thread request has been made */ + kqworkq_request_thread(kqwq, qos_index); + } + kqwq_req_unlock(kqwq); +} + +static void +kqworkloop_threadreq_impl(struct kqworkloop *kqwl, kq_index_t qos_index) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + unsigned long pri = pthread_priority_for_kqrequest(kqr, qos_index); + int op, ret; + + assert((kqr->kqr_state & (KQR_THREQUESTED | KQR_BOUND)) == KQR_THREQUESTED); + + /* + * New-style thread request supported. Provide + * the pthread kext a pointer to a workq_threadreq_s + * structure for its use until a corresponding + * workloop_fulfill_threqreq callback. + */ + if (current_proc() == kqwl->kqwl_kqueue.kq_p) { + op = WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL; + } else { + op = WORKQ_THREADREQ_WORKLOOP; + } +again: + ret = (*pthread_functions->workq_threadreq)(kqwl->kqwl_p, &kqr->kqr_req, + WORKQ_THREADREQ_WORKLOOP, pri, 0); + switch (ret) { + case ENOTSUP: + assert(op == WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL); + op = WORKQ_THREADREQ_WORKLOOP; + goto again; + + case ECANCELED: + case EINVAL: + /* + * Process is shutting down or exec'ing. + * All the kqueues are going to be cleaned up + * soon. Forget we even asked for a thread - + * and make sure we don't ask for more. + */ + kqueue_release((struct kqueue *)kqwl, KQUEUE_CANT_BE_LAST_REF); + kqr->kqr_state &= ~KQR_THREQUESTED; + kqr->kqr_state |= KQR_DRAIN; + break; + + case EAGAIN: + assert(op == WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL); + act_set_astkevent(current_thread(), AST_KEVENT_REDRIVE_THREADREQ); + break; + + default: + assert(ret == 0); + } +} + +static void +kqworkloop_threadreq_modify(struct kqworkloop *kqwl, kq_index_t qos_index) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + unsigned long pri = pthread_priority_for_kqrequest(kqr, qos_index); + int ret, op = WORKQ_THREADREQ_CHANGE_PRI_NO_THREAD_CALL; + + assert((kqr->kqr_state & (KQR_THREQUESTED | KQR_BOUND)) == KQR_THREQUESTED); + + if (current_proc() == kqwl->kqwl_kqueue.kq_p) { + op = WORKQ_THREADREQ_CHANGE_PRI_NO_THREAD_CALL; + } else { + op = WORKQ_THREADREQ_CHANGE_PRI; + } +again: + ret = (*pthread_functions->workq_threadreq_modify)(kqwl->kqwl_p, + &kqr->kqr_req, op, pri, 0); + switch (ret) { + case ENOTSUP: + assert(op == WORKQ_THREADREQ_CHANGE_PRI_NO_THREAD_CALL); + op = WORKQ_THREADREQ_CHANGE_PRI; + goto again; + + case EAGAIN: + assert(op == WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL); + act_set_astkevent(current_thread(), AST_KEVENT_REDRIVE_THREADREQ); + break; + + case ECANCELED: + case EINVAL: + case 0: + break; + + default: + assert(ret == 0); + } +} + +/* + * Interact with the pthread kext to request a servicing thread. + * This will request a single thread at the highest QoS level + * for which there is work (whether that was the requested QoS + * for an event or an override applied to a lower-QoS request). + * + * - Caller holds the workloop request lock + * + * - May be called with the kqueue's wait queue set locked, + * so cannot do anything that could recurse on that. + */ +static void +kqworkloop_request_thread(struct kqworkloop *kqwl, kq_index_t qos_index) +{ + struct kqrequest *kqr; + + assert(kqwl->kqwl_state & KQ_WORKLOOP); + + kqr = &kqwl->kqwl_request; + + assert(kqwl->kqwl_owner == THREAD_NULL); + assert((kqr->kqr_state & KQR_BOUND) == 0); + assert((kqr->kqr_state & KQR_THREQUESTED) == 0); + assert(!(kqwl->kqwl_kqueue.kq_state & KQ_NO_WQ_THREAD)); + + /* If we're draining thread requests, just bail */ + if (kqr->kqr_state & KQR_DRAIN) + return; + + if (pthread_functions != NULL && + pthread_functions->workq_threadreq != NULL) { + /* + * set request state flags, etc... before calling pthread + * This assures they are set before a possible synchronous + * callback to workloop_fulfill_threadreq(). + */ + kqr->kqr_state |= KQR_THREQUESTED; + + /* Add a thread request reference on the kqueue. */ + kqueue_retain((struct kqueue *)kqwl); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_THREQUEST), + kqwl->kqwl_dynamicid, + 0, qos_index, kqr->kqr_state); + kqworkloop_threadreq_impl(kqwl, qos_index); + } else { + panic("kqworkloop_request_thread"); + return; + } +} + +static void +kqworkloop_update_sync_override_state(struct kqworkloop *kqwl, boolean_t sync_ipc_override) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + kqwl_req_lock(kqwl); + kqr->kqr_has_sync_override = sync_ipc_override; + kqwl_req_unlock(kqwl); + +} + +static inline kq_index_t +kqworkloop_combined_qos(struct kqworkloop *kqwl, boolean_t *ipc_override_is_sync) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + kq_index_t override; + + *ipc_override_is_sync = FALSE; + override = MAX(MAX(kqr->kqr_qos_index, kqr->kqr_override_index), + kqr->kqr_dsync_waiters_qos); + + if (kqr->kqr_sync_suppress_count > 0 || kqr->kqr_has_sync_override) { + *ipc_override_is_sync = TRUE; + override = THREAD_QOS_USER_INTERACTIVE; + } + return override; +} + +static inline void +kqworkloop_request_fire_r2k_notification(struct kqworkloop *kqwl) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + + kqwl_req_held(kqwl); + + if (kqr->kqr_state & KQR_R2K_NOTIF_ARMED) { + assert(kqr->kqr_state & KQR_BOUND); + assert(kqr->kqr_thread); + + kqr->kqr_state &= ~KQR_R2K_NOTIF_ARMED; + act_set_astkevent(kqr->kqr_thread, AST_KEVENT_RETURN_TO_KERNEL); + } +} + +static void +kqworkloop_update_threads_qos(struct kqworkloop *kqwl, int op, kq_index_t qos) +{ + const uint8_t KQWL_STAYACTIVE_FIRED_BIT = (1 << 0); + + struct kqrequest *kqr = &kqwl->kqwl_request; + boolean_t old_ipc_override_is_sync = FALSE; + kq_index_t old_qos = kqworkloop_combined_qos(kqwl, &old_ipc_override_is_sync); + struct kqueue *kq = &kqwl->kqwl_kqueue; + bool static_thread = (kq->kq_state & KQ_NO_WQ_THREAD); + kq_index_t i; + + /* must hold the kqr lock */ + kqwl_req_held(kqwl); + + switch (op) { + case KQWL_UTQ_UPDATE_WAKEUP_QOS: + if (qos == KQWL_BUCKET_STAYACTIVE) { + /* + * the KQWL_BUCKET_STAYACTIVE is not a QoS bucket, we only remember + * a high watermark (kqr_stayactive_qos) of any stay active knote + * that was ever registered with this workloop. + * + * When waitq_set__CALLING_PREPOST_HOOK__() wakes up any stay active + * knote, we use this high-watermark as a wakeup-index, and also set + * the magic KQWL_BUCKET_STAYACTIVE bit to make sure we remember + * there is at least one stay active knote fired until the next full + * processing of this bucket. + */ + kqr->kqr_wakeup_indexes |= KQWL_STAYACTIVE_FIRED_BIT; + qos = kqr->kqr_stayactive_qos; + assert(qos); + assert(!static_thread); + } + if (kqr->kqr_wakeup_indexes & (1 << qos)) { + assert(kqr->kqr_state & KQR_WAKEUP); + break; + } + + kqr->kqr_wakeup_indexes |= (1 << qos); + kqr->kqr_state |= KQR_WAKEUP; + kqworkloop_request_fire_r2k_notification(kqwl); + goto recompute_async; + + case KQWL_UTQ_UPDATE_STAYACTIVE_QOS: + assert(qos); + if (kqr->kqr_stayactive_qos < qos) { + kqr->kqr_stayactive_qos = qos; + if (kqr->kqr_wakeup_indexes & KQWL_STAYACTIVE_FIRED_BIT) { + assert(kqr->kqr_state & KQR_WAKEUP); + kqr->kqr_wakeup_indexes |= (1 << qos); + goto recompute_async; + } + } + break; + + case KQWL_UTQ_RECOMPUTE_WAKEUP_QOS: + kqlock_held(kq); // to look at kq_queues + kqr->kqr_has_sync_override = FALSE; + i = KQWL_BUCKET_STAYACTIVE; + if (TAILQ_EMPTY(&kqr->kqr_suppressed)) { + kqr->kqr_override_index = THREAD_QOS_UNSPECIFIED; + } + if (!TAILQ_EMPTY(&kq->kq_queue[i]) && + (kqr->kqr_wakeup_indexes & KQWL_STAYACTIVE_FIRED_BIT)) { + /* + * If the KQWL_STAYACTIVE_FIRED_BIT is set, it means a stay active + * knote may have fired, so we need to merge in kqr_stayactive_qos. + * + * Unlike other buckets, this one is never empty but could be idle. + */ + kqr->kqr_wakeup_indexes &= KQWL_STAYACTIVE_FIRED_BIT; + kqr->kqr_wakeup_indexes |= (1 << kqr->kqr_stayactive_qos); + } else { + kqr->kqr_wakeup_indexes = 0; + } + for (i = THREAD_QOS_UNSPECIFIED + 1; i < KQWL_BUCKET_STAYACTIVE; i++) { + if (!TAILQ_EMPTY(&kq->kq_queue[i])) { + kqr->kqr_wakeup_indexes |= (1 << i); + struct knote *kn = TAILQ_FIRST(&kqwl->kqwl_kqueue.kq_queue[i]); + if (i == THREAD_QOS_USER_INTERACTIVE && + kn->kn_qos_override_is_sync) { + kqr->kqr_has_sync_override = TRUE; + } + } + } + if (kqr->kqr_wakeup_indexes) { + kqr->kqr_state |= KQR_WAKEUP; + kqworkloop_request_fire_r2k_notification(kqwl); + } else { + kqr->kqr_state &= ~KQR_WAKEUP; + } + assert(qos == THREAD_QOS_UNSPECIFIED); + goto recompute_async; + + case KQWL_UTQ_RESET_WAKEUP_OVERRIDE: + kqr->kqr_override_index = THREAD_QOS_UNSPECIFIED; + assert(qos == THREAD_QOS_UNSPECIFIED); + goto recompute_async; + + case KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE: + recompute_async: + /* + * When modifying the wakeup QoS or the async override QoS, we always + * need to maintain our invariant that kqr_override_index is at least as + * large as the highest QoS for which an event is fired. + * + * However this override index can be larger when there is an overriden + * suppressed knote pushing on the kqueue. + */ + if (kqr->kqr_wakeup_indexes > (1 << qos)) { + qos = fls(kqr->kqr_wakeup_indexes) - 1; /* fls is 1-based */ + } + if (kqr->kqr_override_index < qos) { + kqr->kqr_override_index = qos; + } + break; + + case KQWL_UTQ_REDRIVE_EVENTS: + break; - bzero((void *)sb, sizeof(*sb)); - sb->st_size = kq->kq_count; - if (kq->kq_state & KQ_KEV_QOS) - sb->st_blksize = sizeof(struct kevent_qos_s); - else if (kq->kq_state & KQ_KEV64) - sb->st_blksize = sizeof(struct kevent64_s); - else if (IS_64BIT_PROCESS(p)) - sb->st_blksize = sizeof(struct user64_kevent); - else - sb->st_blksize = sizeof(struct user32_kevent); - sb->st_mode = S_IFIFO; - } - kqunlock(kq); - return (0); -} + case KQWL_UTQ_SET_ASYNC_QOS: + filt_wlheld(kqwl); + kqr->kqr_qos_index = qos; + break; + case KQWL_UTQ_SET_SYNC_WAITERS_QOS: + filt_wlheld(kqwl); + kqr->kqr_dsync_waiters_qos = qos; + break; -/* - * Interact with the pthread kext to request a servicing there. - * Eventually, this will request threads at specific QoS levels. - * For now, it only requests a dispatch-manager-QoS thread, and - * only one-at-a-time. - * - * - Caller holds the workq request lock - * - * - May be called with the kqueue's wait queue set locked, - * so cannot do anything that could recurse on that. - */ -static void -kqworkq_request_thread( - struct kqworkq *kqwq, - kq_index_t qos_index) -{ - struct kqrequest *kqr; + default: + panic("unknown kqwl thread qos update operation: %d", op); + } - assert(kqwq->kqwq_state & KQ_WORKQ); - assert(qos_index < KQWQ_NQOS); + boolean_t new_ipc_override_is_sync = FALSE; + kq_index_t new_qos = kqworkloop_combined_qos(kqwl, &new_ipc_override_is_sync); + thread_t kqwl_owner = kqwl->kqwl_owner; + thread_t servicer = kqr->kqr_thread; + __assert_only int ret; - kqr = kqworkq_get_request(kqwq, qos_index); + /* + * Apply the diffs to the owner if applicable + */ + if (filt_wlowner_is_valid(kqwl_owner)) { +#if 0 + /* JMM - need new trace hooks for owner overrides */ + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_THADJUST), + kqwl->kqwl_dynamicid, + (kqr->kqr_state & KQR_BOUND) ? thread_tid(kqwl_owner) : 0, + (kqr->kqr_qos_index << 8) | new_qos, + (kqr->kqr_override_index << 8) | kqr->kqr_state); +#endif + if (new_qos == kqr->kqr_dsync_owner_qos) { + // nothing to do + } else if (kqr->kqr_dsync_owner_qos == THREAD_QOS_UNSPECIFIED) { + thread_add_ipc_override(kqwl_owner, new_qos); + } else if (new_qos == THREAD_QOS_UNSPECIFIED) { + thread_drop_ipc_override(kqwl_owner); + } else /* kqr->kqr_dsync_owner_qos != new_qos */ { + thread_update_ipc_override(kqwl_owner, new_qos); + } + kqr->kqr_dsync_owner_qos = new_qos; + + if (new_ipc_override_is_sync && + !kqr->kqr_owner_override_is_sync) { + thread_add_sync_ipc_override(kqwl_owner); + } else if (!new_ipc_override_is_sync && + kqr->kqr_owner_override_is_sync) { + thread_drop_sync_ipc_override(kqwl_owner); + } + kqr->kqr_owner_override_is_sync = new_ipc_override_is_sync; + } - /* - * If we have already requested a thread, and it hasn't - * started processing yet, there's no use hammering away - * on the pthread kext. + /* + * apply the diffs to the servicer */ - if (kqr->kqr_state & KQWQ_THREQUESTED) - return; + if (static_thread) { + /* + * Statically bound thread + * + * These threads don't participates in QoS overrides today, just wakeup + * the thread blocked on this kqueue if a new event arrived. + */ - assert(kqr->kqr_thread == THREAD_NULL); + switch (op) { + case KQWL_UTQ_UPDATE_WAKEUP_QOS: + case KQWL_UTQ_UPDATE_STAYACTIVE_QOS: + case KQWL_UTQ_RECOMPUTE_WAKEUP_QOS: + break; - /* request additional workq threads if appropriate */ - if (pthread_functions != NULL && - pthread_functions->workq_reqthreads != NULL) { - unsigned int flags = KEVENT_FLAG_WORKQ; + case KQWL_UTQ_RESET_WAKEUP_OVERRIDE: + case KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE: + case KQWL_UTQ_REDRIVE_EVENTS: + case KQWL_UTQ_SET_ASYNC_QOS: + case KQWL_UTQ_SET_SYNC_WAITERS_QOS: + panic("should never be called"); + break; + } - /* Compute a priority based on qos_index. */ - struct workq_reqthreads_req_s request = { - .priority = qos_from_qos_index(qos_index), - .count = 1 - }; + kqlock_held(kq); - thread_t wqthread; - wqthread = (*pthread_functions->workq_reqthreads)(kqwq->kqwq_p, 1, &request); - kqr->kqr_state |= KQWQ_THREQUESTED; + if ((kqr->kqr_state & KQR_BOUND) && (kqr->kqr_state & KQR_WAKEUP)) { + assert(servicer && !is_workqueue_thread(servicer)); + if (kq->kq_state & (KQ_SLEEP | KQ_SEL)) { + kq->kq_state &= ~(KQ_SLEEP | KQ_SEL); + waitq_wakeup64_all((struct waitq *)&kq->kq_wqs, KQ_EVENT, + THREAD_AWAKENED, WAITQ_ALL_PRIORITIES); + } + } + } else if ((kqr->kqr_state & KQR_THREQUESTED) == 0) { + /* + * No servicer, nor thread-request + * + * Make a new thread request, unless there is an owner (or the workloop + * is suspended in userland) or if there is no asynchronous work in the + * first place. + */ - /* Have we been switched to the emergency/manager thread? */ - if (wqthread == (thread_t)-1) { - flags |= KEVENT_FLAG_WORKQ_MANAGER; - wqthread = THREAD_NULL; - } else if (qos_index == KQWQ_QOS_MANAGER) - flags |= KEVENT_FLAG_WORKQ_MANAGER; + if (kqwl_owner == THREAD_NULL && (kqr->kqr_state & KQR_WAKEUP)) { + kqworkloop_request_thread(kqwl, new_qos); + } + } else if ((kqr->kqr_state & KQR_BOUND) == 0 && + (kqwl_owner || (kqr->kqr_state & KQR_WAKEUP) == 0)) { + /* + * No servicer, thread request in flight we want to cancel + * + * We just got rid of the last knote of the kqueue or noticed an owner + * with a thread request still in flight, take it back. + */ + ret = (*pthread_functions->workq_threadreq_modify)(kqwl->kqwl_p, + &kqr->kqr_req, WORKQ_THREADREQ_CANCEL, 0, 0); + if (ret == 0) { + kqr->kqr_state &= ~KQR_THREQUESTED; + kqueue_release(kq, KQUEUE_CANT_BE_LAST_REF); + } + } else { + boolean_t qos_changed = FALSE; + + /* + * Servicer or request is in flight + * + * Just apply the diff to the servicer or the thread request + */ + if (kqr->kqr_state & KQR_BOUND) { + servicer = kqr->kqr_thread; + struct uthread *ut = get_bsdthread_info(servicer); + if (ut->uu_kqueue_qos_index != new_qos) { + if (ut->uu_kqueue_qos_index == THREAD_QOS_UNSPECIFIED) { + thread_add_ipc_override(servicer, new_qos); + } else if (new_qos == THREAD_QOS_UNSPECIFIED) { + thread_drop_ipc_override(servicer); + } else /* ut->uu_kqueue_qos_index != new_qos */ { + thread_update_ipc_override(servicer, new_qos); + } + ut->uu_kqueue_qos_index = new_qos; + qos_changed = TRUE; + } - /* bind the thread */ - kqworkq_bind_thread(kqwq, qos_index, wqthread, flags); + if (new_ipc_override_is_sync != ut->uu_kqueue_override_is_sync) { + if (new_ipc_override_is_sync && + !ut->uu_kqueue_override_is_sync) { + thread_add_sync_ipc_override(servicer); + } else if (!new_ipc_override_is_sync && + ut->uu_kqueue_override_is_sync) { + thread_drop_sync_ipc_override(servicer); + } + ut->uu_kqueue_override_is_sync = new_ipc_override_is_sync; + qos_changed = TRUE; + } + } else if (old_qos != new_qos) { + assert(new_qos); + kqworkloop_threadreq_modify(kqwl, new_qos); + qos_changed = TRUE; + } + if (qos_changed) { + servicer = kqr->kqr_thread; + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_THADJUST), + kqwl->kqwl_dynamicid, + (kqr->kqr_state & KQR_BOUND) ? thread_tid(servicer) : 0, + (kqr->kqr_qos_index << 16) | (new_qos << 8) | new_ipc_override_is_sync, + (kqr->kqr_override_index << 8) | kqr->kqr_state); + } } } -/* - * If we aren't already busy processing events [for this QoS], - * request workq thread support as appropriate. - * - * TBD - for now, we don't segregate out processing by QoS. - * - * - May be called with the kqueue's wait queue set locked, - * so cannot do anything that could recurse on that. - */ static void -kqworkq_request_help( - struct kqworkq *kqwq, - kq_index_t qos_index, - uint32_t type) +kqworkloop_request_help(struct kqworkloop *kqwl, kq_index_t qos_index) { - struct kqrequest *kqr; - /* convert to thread qos value */ - assert(qos_index < KQWQ_NQOS); - - kqwq_req_lock(kqwq); - kqr = kqworkq_get_request(kqwq, qos_index); - - /* - * If someone is processing the queue, just mark what type - * of attempt this was (from a kq wakeup or from a waitq hook). - * They'll be noticed at the end of servicing and a new thread - * will be requested at that point. - */ - if (kqr->kqr_state & KQWQ_PROCESSING) { - kqr->kqr_state |= type; - kqwq_req_unlock(kqwq); - return; - } + assert(qos_index < KQWL_NBUCKETS); - kqworkq_request_thread(kqwq, qos_index); - kqwq_req_unlock(kqwq); + kqwl_req_lock(kqwl); + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_UPDATE_WAKEUP_QOS, qos_index); + kqwl_req_unlock(kqwl); } /* * These arrays described the low and high qindexes for a given qos_index. * The values come from the chart in (must stay in sync). */ -static kq_index_t _kq_base_index[KQWQ_NQOS] = {0, 0, 6, 11, 15, 18, 20, 21}; -static kq_index_t _kq_high_index[KQWQ_NQOS] = {0, 5, 10, 14, 17, 19, 20, 21}; +static kq_index_t _kqwq_base_index[KQWQ_NQOS] = {0, 0, 6, 11, 15, 18, 20, 21}; +static kq_index_t _kqwq_high_index[KQWQ_NQOS] = {0, 5, 10, 14, 17, 19, 20, 21}; static struct kqtailq * kqueue_get_base_queue(struct kqueue *kq, kq_index_t qos_index) { - assert(qos_index < KQWQ_NQOS); - return &kq->kq_queue[_kq_base_index[qos_index]]; + if (kq->kq_state & KQ_WORKQ) { + assert(qos_index < KQWQ_NQOS); + return &kq->kq_queue[_kqwq_base_index[qos_index]]; + } else if (kq->kq_state & KQ_WORKLOOP) { + assert(qos_index < KQWL_NBUCKETS); + return &kq->kq_queue[qos_index]; + } else { + assert(qos_index == QOS_INDEX_KQFILE); + return &kq->kq_queue[QOS_INDEX_KQFILE]; + } } static struct kqtailq * kqueue_get_high_queue(struct kqueue *kq, kq_index_t qos_index) { - assert(qos_index < KQWQ_NQOS); - return &kq->kq_queue[_kq_high_index[qos_index]]; + if (kq->kq_state & KQ_WORKQ) { + assert(qos_index < KQWQ_NQOS); + return &kq->kq_queue[_kqwq_high_index[qos_index]]; + } else if (kq->kq_state & KQ_WORKLOOP) { + assert(qos_index < KQWL_NBUCKETS); + return &kq->kq_queue[KQWL_BUCKET_STAYACTIVE]; + } else { + assert(qos_index == QOS_INDEX_KQFILE); + return &kq->kq_queue[QOS_INDEX_KQFILE]; + } } static int @@ -4044,16 +7053,24 @@ kqueue_queue_empty(struct kqueue *kq, kq_index_t qos_index) static struct kqtailq * kqueue_get_suppressed_queue(struct kqueue *kq, kq_index_t qos_index) { + struct kqtailq *res; + struct kqrequest *kqr; + if (kq->kq_state & KQ_WORKQ) { struct kqworkq *kqwq = (struct kqworkq *)kq; - struct kqrequest *kqr; kqr = kqworkq_get_request(kqwq, qos_index); - return &kqr->kqr_suppressed; + res = &kqr->kqr_suppressed; + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + kqr = &kqwl->kqwl_request; + res = &kqr->kqr_suppressed; } else { struct kqfile *kqf = (struct kqfile *)kq; - return &kqf->kqf_suppressed; + res = &kqf->kqf_suppressed; } + return res; } static kq_index_t @@ -4064,15 +7081,19 @@ knote_get_queue_index(struct knote *kn) struct kqueue *kq = knote_get_kq(kn); kq_index_t res; - if ((kq->kq_state & KQ_WORKQ) == 0) { - assert(qos_index == 0); - assert(override_index == 0); + if (kq->kq_state & KQ_WORKQ) { + res = _kqwq_base_index[qos_index]; + if (override_index > qos_index) + res += override_index - qos_index; + assert(res <= _kqwq_high_index[qos_index]); + } else if (kq->kq_state & KQ_WORKLOOP) { + res = MAX(override_index, qos_index); + assert(res < KQWL_NBUCKETS); + } else { + assert(qos_index == QOS_INDEX_KQFILE); + assert(override_index == QOS_INDEX_KQFILE); + res = QOS_INDEX_KQFILE; } - res = _kq_base_index[qos_index]; - if (override_index > qos_index) - res += override_index - qos_index; - - assert(res <= _kq_high_index[qos_index]); return res; } @@ -4084,15 +7105,6 @@ knote_get_queue(struct knote *kn) return &(knote_get_kq(kn))->kq_queue[qindex]; } -static struct kqtailq * -knote_get_suppressed_queue(struct knote *kn) -{ - kq_index_t qos_index = knote_get_qos_index(kn); - struct kqueue *kq = knote_get_kq(kn); - - return kqueue_get_suppressed_queue(kq, qos_index); -} - static kq_index_t knote_get_req_index(struct knote *kn) { @@ -4113,10 +7125,14 @@ knote_set_qos_index(struct knote *kn, kq_index_t qos_index) assert(qos_index < KQWQ_NQOS); assert((kn->kn_status & KN_QUEUED) == 0); - if (kq->kq_state & KQ_WORKQ) - assert(qos_index > QOS_INDEX_KQFILE); - else - assert(qos_index == QOS_INDEX_KQFILE); + if (kq->kq_state & KQ_WORKQ) { + assert(qos_index > THREAD_QOS_UNSPECIFIED); + } else if (kq->kq_state & KQ_WORKLOOP) { + /* XXX this policy decision shouldn't be here */ + if (qos_index == THREAD_QOS_UNSPECIFIED) + qos_index = THREAD_QOS_LEGACY; + } else + qos_index = QOS_INDEX_KQFILE; /* always set requested */ kn->kn_req_index = qos_index; @@ -4126,6 +7142,35 @@ knote_set_qos_index(struct knote *kn, kq_index_t qos_index) kn->kn_qos_index = qos_index; } +static void +knote_set_qos_overcommit(struct knote *kn) +{ + struct kqueue *kq = knote_get_kq(kn); + struct kqrequest *kqr; + + /* turn overcommit on for the appropriate thread request? */ + if (kn->kn_qos & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG) { + if (kq->kq_state & KQ_WORKQ) { + kq_index_t qos_index = knote_get_qos_index(kn); + struct kqworkq *kqwq = (struct kqworkq *)kq; + + kqr = kqworkq_get_request(kqwq, qos_index); + + kqwq_req_lock(kqwq); + kqr->kqr_state |= KQR_THOVERCOMMIT; + kqwq_req_unlock(kqwq); + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + kqr = &kqwl->kqwl_request; + + kqwl_req_lock(kqwl); + kqr->kqr_state |= KQR_THOVERCOMMIT; + kqwl_req_unlock(kqwl); + } + } +} + static kq_index_t knote_get_qos_override_index(struct knote *kn) { @@ -4133,58 +7178,88 @@ knote_get_qos_override_index(struct knote *kn) } static void -knote_set_qos_override_index(struct knote *kn, kq_index_t override_index) +knote_set_qos_override_index(struct knote *kn, kq_index_t override_index, + boolean_t override_is_sync) { struct kqueue *kq = knote_get_kq(kn); kq_index_t qos_index = knote_get_qos_index(kn); + kq_index_t old_override_index = knote_get_qos_override_index(kn); + boolean_t old_override_is_sync = kn->kn_qos_override_is_sync; + uint32_t flags = 0; assert((kn->kn_status & KN_QUEUED) == 0); - if (override_index == KQWQ_QOS_MANAGER) + if (override_index == KQWQ_QOS_MANAGER) { assert(qos_index == KQWQ_QOS_MANAGER); - else + } else { assert(override_index < KQWQ_QOS_MANAGER); + } kn->kn_qos_override = override_index; + kn->kn_qos_override_is_sync = override_is_sync; - /* - * If this is a workq kqueue, apply the override to the - * workq servicing thread. + /* + * If this is a workq/workloop kqueue, apply the override to the + * servicing thread. */ if (kq->kq_state & KQ_WORKQ) { struct kqworkq *kqwq = (struct kqworkq *)kq; - assert(qos_index > QOS_INDEX_KQFILE); + assert(qos_index > THREAD_QOS_UNSPECIFIED); kqworkq_update_override(kqwq, qos_index, override_index); + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + if ((kn->kn_status & KN_SUPPRESSED) == KN_SUPPRESSED) { + flags = flags | KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS; + + if (override_index == THREAD_QOS_USER_INTERACTIVE + && override_is_sync) { + flags = flags | KQWL_UO_NEW_OVERRIDE_IS_SYNC_UI; + } + + if (old_override_index == THREAD_QOS_USER_INTERACTIVE + && old_override_is_sync) { + flags = flags | KQWL_UO_OLD_OVERRIDE_IS_SYNC_UI; + } + } + + assert(qos_index > THREAD_QOS_UNSPECIFIED); + kqworkloop_update_override(kqwl, qos_index, override_index, flags); } } +static kq_index_t +knote_get_sync_qos_override_index(struct knote *kn) +{ + return kn->kn_qos_sync_override; +} + static void kqworkq_update_override(struct kqworkq *kqwq, kq_index_t qos_index, kq_index_t override_index) { struct kqrequest *kqr; - kq_index_t new_delta; - kq_index_t old_delta; + kq_index_t old_override_index; - new_delta = (override_index > qos_index) ? - override_index - qos_index : 0; + if (override_index <= qos_index) { + return; + } kqr = kqworkq_get_request(kqwq, qos_index); kqwq_req_lock(kqwq); - old_delta = kqr->kqr_override_delta; - - if (new_delta > old_delta) { - thread_t wqthread = kqr->kqr_thread; - - /* store the new override delta */ - kqr->kqr_override_delta = new_delta; + old_override_index = kqr->kqr_override_index; + if (override_index > MAX(kqr->kqr_qos_index, old_override_index)) { + kqr->kqr_override_index = override_index; /* apply the override to [incoming?] servicing thread */ - if (wqthread) { + if (kqr->kqr_state & KQR_BOUND) { + thread_t wqthread = kqr->kqr_thread; + /* only apply if non-manager */ + assert(wqthread); if ((kqr->kqr_state & KQWQ_THMANAGER) == 0) { - if (old_delta) + if (old_override_index) thread_update_ipc_override(wqthread, override_index); else thread_add_ipc_override(wqthread, override_index); @@ -4194,70 +7269,212 @@ kqworkq_update_override(struct kqworkq *kqwq, kq_index_t qos_index, kq_index_t o kqwq_req_unlock(kqwq); } -/* called with the kqworkq lock held */ +/* called with the kqworkq lock held */ +static void +kqworkq_bind_thread_impl( + struct kqworkq *kqwq, + kq_index_t qos_index, + thread_t thread, + unsigned int flags) +{ + /* request lock must be held */ + kqwq_req_held(kqwq); + + struct kqrequest *kqr = kqworkq_get_request(kqwq, qos_index); + assert(kqr->kqr_state & KQR_THREQUESTED); + + if (qos_index == KQWQ_QOS_MANAGER) + flags |= KEVENT_FLAG_WORKQ_MANAGER; + + struct uthread *ut = get_bsdthread_info(thread); + + /* + * If this is a manager, and the manager request bit is + * not set, assure no other thread is bound. If the bit + * is set, make sure the old thread is us (or not set). + */ + if (flags & KEVENT_FLAG_WORKQ_MANAGER) { + if ((kqr->kqr_state & KQR_BOUND) == 0) { + kqr->kqr_state |= (KQR_BOUND | KQWQ_THMANAGER); + TAILQ_INIT(&kqr->kqr_suppressed); + kqr->kqr_thread = thread; + ut->uu_kqueue_bound = (struct kqueue *)kqwq; + ut->uu_kqueue_qos_index = KQWQ_QOS_MANAGER; + ut->uu_kqueue_flags = (KEVENT_FLAG_WORKQ | + KEVENT_FLAG_WORKQ_MANAGER); + } else { + assert(kqr->kqr_state & KQR_BOUND); + assert(thread == kqr->kqr_thread); + assert(ut->uu_kqueue_bound == (struct kqueue *)kqwq); + assert(ut->uu_kqueue_qos_index == KQWQ_QOS_MANAGER); + assert(ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER); + } + return; + } + + /* Just a normal one-queue servicing thread */ + assert(kqr->kqr_state & KQR_THREQUESTED); + assert(kqr->kqr_qos_index == qos_index); + + if ((kqr->kqr_state & KQR_BOUND) == 0) { + kqr->kqr_state |= KQR_BOUND; + TAILQ_INIT(&kqr->kqr_suppressed); + kqr->kqr_thread = thread; + + /* apply an ipc QoS override if one is needed */ + if (kqr->kqr_override_index) { + assert(kqr->kqr_qos_index); + assert(kqr->kqr_override_index > kqr->kqr_qos_index); + assert(thread_get_ipc_override(thread) == THREAD_QOS_UNSPECIFIED); + thread_add_ipc_override(thread, kqr->kqr_override_index); + } + + /* indicate that we are processing in the uthread */ + ut->uu_kqueue_bound = (struct kqueue *)kqwq; + ut->uu_kqueue_qos_index = qos_index; + ut->uu_kqueue_flags = flags; + } else { + /* + * probably syncronously bound AND post-request bound + * this logic can go away when we get rid of post-request bind + */ + assert(kqr->kqr_state & KQR_BOUND); + assert(thread == kqr->kqr_thread); + assert(ut->uu_kqueue_bound == (struct kqueue *)kqwq); + assert(ut->uu_kqueue_qos_index == qos_index); + assert((ut->uu_kqueue_flags & flags) == flags); + } +} + +static void +kqworkloop_update_override( + struct kqworkloop *kqwl, + kq_index_t qos_index, + kq_index_t override_index, + uint32_t flags) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + + kqwl_req_lock(kqwl); + + /* Do not override on attached threads */ + if (kqr->kqr_state & KQR_BOUND) { + assert(kqr->kqr_thread); + + if (kqwl->kqwl_kqueue.kq_state & KQ_NO_WQ_THREAD) { + kqwl_req_unlock(kqwl); + assert(!is_workqueue_thread(kqr->kqr_thread)); + return; + } + } + + /* Update sync ipc counts on kqr for suppressed knotes */ + if (flags & KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS) { + kqworkloop_update_suppress_sync_count(kqr, flags); + } + + if ((flags & KQWL_UO_UPDATE_OVERRIDE_LAZY) == 0) { + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE, + MAX(qos_index, override_index)); + } + kqwl_req_unlock(kqwl); +} + +static void +kqworkloop_update_suppress_sync_count( + struct kqrequest *kqr, + uint32_t flags) +{ + if (flags & KQWL_UO_NEW_OVERRIDE_IS_SYNC_UI) { + kqr->kqr_sync_suppress_count++; + } + + if (flags & KQWL_UO_OLD_OVERRIDE_IS_SYNC_UI) { + assert(kqr->kqr_sync_suppress_count > 0); + kqr->kqr_sync_suppress_count--; + } +} + +/* + * kqworkloop_unbind_thread - Unbind the servicer thread of a workloop kqueue + * + * It will end the processing phase in case it was still processing: + * + * We may have to request a new thread for not KQ_NO_WQ_THREAD workloop. + * This can happen if : + * - there were active events at or above our QoS we never got to (count > 0) + * - we pended waitq hook callouts during processing + * - we pended wakeups while processing (or unsuppressing) + * + * Called with kqueue lock held. + */ + static void -kqworkq_bind_thread( - struct kqworkq *kqwq, - kq_index_t qos_index, +kqworkloop_unbind_thread( + struct kqworkloop *kqwl, thread_t thread, - unsigned int flags) + __unused unsigned int flags) { - struct kqrequest *kqr = kqworkq_get_request(kqwq, qos_index); - thread_t old_thread = kqr->kqr_thread; - struct uthread *ut; + struct kqueue *kq = &kqwl->kqwl_kqueue; + struct kqrequest *kqr = &kqwl->kqwl_request; - assert(kqr->kqr_state & KQWQ_THREQUESTED); + kqlock_held(kq); - /* If no identity yet, just set flags as needed */ - if (thread == THREAD_NULL) { - assert(old_thread == THREAD_NULL); - - /* emergency or unindetified */ - if (flags & KEVENT_FLAG_WORKQ_MANAGER) { - assert((kqr->kqr_state & KQWQ_THMANAGER) == 0); - kqr->kqr_state |= KQWQ_THMANAGER; - } + assert((kq->kq_state & KQ_PROCESSING) == 0); + if (kq->kq_state & KQ_PROCESSING) { return; } - /* Known thread identity */ - ut = get_bsdthread_info(thread); - - /* - * If this is a manager, and the manager request bit is - * not set, assure no other thread is bound. If the bit - * is set, make sure the old thread is us (or not set). + /* + * Forcing the KQ_PROCESSING flag allows for QoS updates because of + * unsuppressing knotes not to be applied until the eventual call to + * kqworkloop_update_threads_qos() below. */ - if (flags & KEVENT_FLAG_WORKQ_MANAGER) { - if ((kqr->kqr_state & KQWQ_THMANAGER) == 0) { - assert(old_thread == THREAD_NULL); - kqr->kqr_state |= KQWQ_THMANAGER; - } else if (old_thread == THREAD_NULL) { - kqr->kqr_thread = thread; - ut->uu_kqueue_bound = KQWQ_QOS_MANAGER; - ut->uu_kqueue_flags = (KEVENT_FLAG_WORKQ | - KEVENT_FLAG_WORKQ_MANAGER); - } else { - assert(thread == old_thread); - assert(ut->uu_kqueue_bound == KQWQ_QOS_MANAGER); - assert(ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER); - } + kq->kq_state |= KQ_PROCESSING; + kqworkloop_acknowledge_events(kqwl, TRUE); + kq->kq_state &= ~KQ_PROCESSING; + + kqwl_req_lock(kqwl); + + /* deal with extraneous unbinds in release kernels */ + assert((kqr->kqr_state & (KQR_BOUND | KQR_PROCESSING)) == KQR_BOUND); + if ((kqr->kqr_state & (KQR_BOUND | KQR_PROCESSING)) != KQR_BOUND) { + kqwl_req_unlock(kqwl); return; } - /* Just a normal one-queue servicing thread */ - assert(old_thread == THREAD_NULL); - assert((kqr->kqr_state & KQWQ_THMANAGER) == 0); + assert(thread == current_thread()); + assert(kqr->kqr_thread == thread); + if (kqr->kqr_thread != thread) { + kqwl_req_unlock(kqwl); + return; + } + + struct uthread *ut = get_bsdthread_info(thread); + kq_index_t old_qos_index = ut->uu_kqueue_qos_index; + boolean_t ipc_override_is_sync = ut->uu_kqueue_override_is_sync; + ut->uu_kqueue_bound = NULL; + ut->uu_kqueue_qos_index = 0; + ut->uu_kqueue_override_is_sync = 0; + ut->uu_kqueue_flags = 0; - kqr->kqr_thread = thread; - - /* apply an ipc QoS override if one is needed */ - if (kqr->kqr_override_delta) - thread_add_ipc_override(thread, qos_index + kqr->kqr_override_delta); + /* unbind the servicer thread, drop overrides */ + kqr->kqr_thread = NULL; + kqr->kqr_state &= ~(KQR_BOUND | KQR_THREQUESTED | KQR_R2K_NOTIF_ARMED); + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_RECOMPUTE_WAKEUP_QOS, 0); - /* indicate that we are processing in the uthread */ - ut->uu_kqueue_bound = qos_index; - ut->uu_kqueue_flags = flags; + kqwl_req_unlock(kqwl); + + /* + * Drop the override on the current thread last, after the call to + * kqworkloop_update_threads_qos above. + */ + if (old_qos_index) { + thread_drop_ipc_override(thread); + } + if (ipc_override_is_sync) { + thread_drop_sync_ipc_override(thread); + } } /* called with the kqworkq lock held */ @@ -4269,37 +7486,54 @@ kqworkq_unbind_thread( __unused unsigned int flags) { struct kqrequest *kqr = kqworkq_get_request(kqwq, qos_index); - kq_index_t override = 0; + kq_index_t override_index = 0; + + /* request lock must be held */ + kqwq_req_held(kqwq); assert(thread == current_thread()); + if ((kqr->kqr_state & KQR_BOUND) == 0) { + assert(kqr->kqr_state & KQR_BOUND); + return; + } + + assert(kqr->kqr_thread == thread); + assert(TAILQ_EMPTY(&kqr->kqr_suppressed)); + /* * If there is an override, drop it from the current thread * and then we are free to recompute (a potentially lower) * minimum override to apply to the next thread request. */ - if (kqr->kqr_override_delta) { + if (kqr->kqr_override_index) { struct kqtailq *base_queue = kqueue_get_base_queue(&kqwq->kqwq_kqueue, qos_index); struct kqtailq *queue = kqueue_get_high_queue(&kqwq->kqwq_kqueue, qos_index); /* if not bound to a manager thread, drop the current ipc override */ if ((kqr->kqr_state & KQWQ_THMANAGER) == 0) { - assert(thread == kqr->kqr_thread); thread_drop_ipc_override(thread); } /* recompute the new override */ do { if (!TAILQ_EMPTY(queue)) { - override = queue - base_queue; + override_index = queue - base_queue + qos_index; break; } } while (queue-- > base_queue); } - /* unbind the thread and apply the new override */ - kqr->kqr_thread = THREAD_NULL; - kqr->kqr_override_delta = override; + /* Mark it unbound */ + kqr->kqr_thread = NULL; + kqr->kqr_state &= ~(KQR_BOUND | KQR_THREQUESTED | KQWQ_THMANAGER); + + /* apply the new override */ + if (override_index > kqr->kqr_qos_index) { + kqr->kqr_override_index = override_index; + } else { + kqr->kqr_override_index = THREAD_QOS_UNSPECIFIED; + } } struct kqrequest * @@ -4310,53 +7544,124 @@ kqworkq_get_request(struct kqworkq *kqwq, kq_index_t qos_index) } void -knote_adjust_qos(struct knote *kn, qos_t new_qos, qos_t new_override) +knote_adjust_qos(struct knote *kn, qos_t new_qos, qos_t new_override, kq_index_t sync_override_index) { - if (knote_get_kq(kn)->kq_state & KQ_WORKQ) { + struct kqueue *kq = knote_get_kq(kn); + boolean_t override_is_sync = FALSE; + + if (kq->kq_state & (KQ_WORKQ | KQ_WORKLOOP)) { kq_index_t new_qos_index; kq_index_t new_override_index; kq_index_t servicer_qos_index; - new_qos_index = qos_index_from_qos(new_qos, FALSE); - new_override_index = qos_index_from_qos(new_override, TRUE); + new_qos_index = qos_index_from_qos(kn, new_qos, FALSE); + new_override_index = qos_index_from_qos(kn, new_override, TRUE); /* make sure the servicer qos acts as a floor */ - servicer_qos_index = qos_index_from_qos(kn->kn_qos, FALSE); + servicer_qos_index = qos_index_from_qos(kn, kn->kn_qos, FALSE); if (servicer_qos_index > new_qos_index) new_qos_index = servicer_qos_index; if (servicer_qos_index > new_override_index) new_override_index = servicer_qos_index; + if (sync_override_index >= new_override_index) { + new_override_index = sync_override_index; + override_is_sync = TRUE; + } - kqlock(knote_get_kq(kn)); + kqlock(kq); if (new_qos_index != knote_get_req_index(kn) || - new_override_index != knote_get_qos_override_index(kn)) { + new_override_index != knote_get_qos_override_index(kn) || + override_is_sync != kn->kn_qos_override_is_sync) { if (kn->kn_status & KN_QUEUED) { knote_dequeue(kn); knote_set_qos_index(kn, new_qos_index); - knote_set_qos_override_index(kn, new_override_index); + knote_set_qos_override_index(kn, new_override_index, override_is_sync); knote_enqueue(kn); knote_wakeup(kn); } else { knote_set_qos_index(kn, new_qos_index); - knote_set_qos_override_index(kn, new_override_index); + knote_set_qos_override_index(kn, new_override_index, override_is_sync); + } + } + kqunlock(kq); + } +} + +void +knote_adjust_sync_qos(struct knote *kn, kq_index_t sync_qos, boolean_t lock_kq) +{ + struct kqueue *kq = knote_get_kq(kn); + kq_index_t old_sync_override; + kq_index_t qos_index = knote_get_qos_index(kn); + uint32_t flags = 0; + + /* Tracking only happens for UI qos */ + if (sync_qos != THREAD_QOS_USER_INTERACTIVE && + sync_qos != THREAD_QOS_UNSPECIFIED) { + return; + } + + if (lock_kq) + kqlock(kq); + + if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + old_sync_override = knote_get_sync_qos_override_index(kn); + if (old_sync_override != sync_qos) { + kn->kn_qos_sync_override = sync_qos; + + /* update sync ipc counters for suppressed knotes */ + if ((kn->kn_status & KN_SUPPRESSED) == KN_SUPPRESSED) { + flags = flags | KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS; + + /* Do not recalculate kqwl override, it would be done later */ + flags = flags | KQWL_UO_UPDATE_OVERRIDE_LAZY; + + if (sync_qos == THREAD_QOS_USER_INTERACTIVE) { + flags = flags | KQWL_UO_NEW_OVERRIDE_IS_SYNC_UI; + } + + if (old_sync_override == THREAD_QOS_USER_INTERACTIVE) { + flags = flags | KQWL_UO_OLD_OVERRIDE_IS_SYNC_UI; + } + + kqworkloop_update_override(kqwl, qos_index, sync_qos, + flags); } + } - kqunlock(knote_get_kq(kn)); } + if (lock_kq) + kqunlock(kq); } static void knote_wakeup(struct knote *kn) { struct kqueue *kq = knote_get_kq(kn); + kq_index_t qos_index = knote_get_qos_index(kn); + + kqlock_held(kq); if (kq->kq_state & KQ_WORKQ) { /* request a servicing thread */ struct kqworkq *kqwq = (struct kqworkq *)kq; - kq_index_t qos_index = knote_get_qos_index(kn); - kqworkq_request_help(kqwq, qos_index, KQWQ_WAKEUP); + kqworkq_request_help(kqwq, qos_index); + + } else if (kq->kq_state & KQ_WORKLOOP) { + /* request a servicing thread */ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + if (kqworkloop_is_processing_on_current_thread(kqwl)) { + /* + * kqworkloop_end_processing() will perform the required QoS + * computations when it unsets the processing mode. + */ + return; + } + kqworkloop_request_help(kqwl, qos_index); } else { struct kqfile *kqf = (struct kqfile *)kq; @@ -4377,7 +7682,7 @@ knote_wakeup(struct knote *kn) KNOTE(&kqf->kqf_sel.si_note, 0); } } - + /* * Called with the kqueue locked */ @@ -4426,10 +7731,18 @@ waitq_set__CALLING_PREPOST_HOOK__(void *kq_hook, void *knote_hook, int qos) { #pragma unused(knote_hook, qos) - struct kqworkq *kqwq = (struct kqworkq *)kq_hook; + struct kqueue *kq = (struct kqueue *)kq_hook; - assert(kqwq->kqwq_state & KQ_WORKQ); - kqworkq_request_help(kqwq, KQWQ_QOS_MANAGER, KQWQ_HOOKCALLED); + if (kq->kq_state & KQ_WORKQ) { + struct kqworkq *kqwq = (struct kqworkq *)kq; + + kqworkq_request_help(kqwq, KQWQ_QOS_MANAGER); + + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + kqworkloop_request_help(kqwl, KQWL_BUCKET_STAYACTIVE); + } } void @@ -4461,15 +7774,17 @@ knote(struct klist *list, long hint) kqlock(kq); + assert(!knoteuse_needs_boost(kn, NULL)); + /* If we can get a use reference - deliver event */ - if (kqlock2knoteuse(kq, kn)) { + if (kqlock2knoteuse(kq, kn, KNUSE_NONE)) { int result; /* call the event with only a use count */ result = knote_fops(kn)->f_event(kn, hint); /* if its not going away and triggered */ - if (knoteuse2kqlock(kq, kn, 0) && result) + if (knoteuse2kqlock(kq, kn, KNUSE_NONE) && result) knote_activate(kn); /* kq lock held */ } @@ -4526,19 +7841,21 @@ knote_vanish(struct klist *list) int result; kqlock(kq); - if ((kn->kn_status & KN_DROPPING) == 0) { + assert(!knoteuse_needs_boost(kn, NULL)); + + if ((kn->kn_status & KN_DROPPING) == 0) { /* If EV_VANISH supported - prepare to deliver one */ if (kn->kn_status & KN_REQVANISH) { kn->kn_status |= KN_VANISHED; knote_activate(kn); - } else if (kqlock2knoteuse(kq, kn)) { + } else if (kqlock2knoteuse(kq, kn, KNUSE_NONE)) { /* call the event with only a use count */ result = knote_fops(kn)->f_event(kn, NOTE_REVOKE); - + /* if its not going away and triggered */ - if (knoteuse2kqlock(kq, kn, 0) && result) + if (knoteuse2kqlock(kq, kn, KNUSE_NONE) && result) knote_activate(kn); /* lock held again */ } @@ -4629,16 +7946,17 @@ restart: if ((kn->kn_status & KN_VANISHED) == 0) { proc_fdunlock(p); - /* get detach reference (also marks vanished) */ - if (kqlock2knotedetach(kq, kn)) { + assert(!knoteuse_needs_boost(kn, NULL)); + /* get detach reference (also marks vanished) */ + if (kqlock2knotedetach(kq, kn, KNUSE_NONE)) { /* detach knote and drop fp use reference */ knote_fops(kn)->f_detach(kn); if (knote_fops(kn)->f_isfd) fp_drop(p, kn->kn_id, kn->kn_fp, 0); /* activate it if it's still in existence */ - if (knoteuse2kqlock(kq, kn, 0)) { + if (knoteuse2kqlock(kq, kn, KNUSE_NONE)) { knote_activate(kn); } kqunlock(kq); @@ -4670,7 +7988,66 @@ restart: } /* - * knote_fdadd - Add knote to the fd table for process + * knote_fdfind - lookup a knote in the fd table for process + * + * If the filter is file-based, lookup based on fd index. + * Otherwise use a hash based on the ident. + * + * Matching is based on kq, filter, and ident. Optionally, + * it may also be based on the udata field in the kevent - + * allowing multiple event registration for the file object + * per kqueue. + * + * fd_knhashlock or fdlock held on entry (and exit) + */ +static struct knote * +knote_fdfind(struct kqueue *kq, + struct kevent_internal_s *kev, + bool is_fd, + struct proc *p) +{ + struct filedesc *fdp = p->p_fd; + struct klist *list = NULL; + struct knote *kn = NULL; + + /* + * determine where to look for the knote + */ + if (is_fd) { + /* fd-based knotes are linked off the fd table */ + if (kev->ident < (u_int)fdp->fd_knlistsize) { + list = &fdp->fd_knlist[kev->ident]; + } + } else if (fdp->fd_knhashmask != 0) { + /* hash non-fd knotes here too */ + list = &fdp->fd_knhash[KN_HASH((u_long)kev->ident, fdp->fd_knhashmask)]; + } + + /* + * scan the selected list looking for a match + */ + if (list != NULL) { + SLIST_FOREACH(kn, list, kn_link) { + if (kq == knote_get_kq(kn) && + kev->ident == kn->kn_id && + kev->filter == kn->kn_filter) { + if (kev->flags & EV_UDATA_SPECIFIC) { + if ((kn->kn_status & KN_UDATA_SPECIFIC) && + kev->udata == kn->kn_udata) { + break; /* matching udata-specific knote */ + } + } else if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) { + break; /* matching non-udata-specific knote */ + } + } + } + } + return kn; +} + +/* + * kq_add_knote- Add knote to the fd table for process + * while checking for duplicates. * * All file-based filters associate a list of knotes by file * descriptor index. All other filters hash the knote by ident. @@ -4678,39 +8055,79 @@ restart: * May have to grow the table of knote lists to cover the * file descriptor index presented. * - * proc_fdlock held on entry (and exit) + * fd_knhashlock and fdlock unheld on entry (and exit). + * + * Takes a rwlock boost if inserting the knote is successful. */ static int -knote_fdadd(struct knote *kn, struct proc *p) +kq_add_knote(struct kqueue *kq, struct knote *kn, + struct kevent_internal_s *kev, + struct proc *p, int *knoteuse_flags) { struct filedesc *fdp = p->p_fd; struct klist *list = NULL; + int ret = 0; + bool is_fd = knote_fops(kn)->f_isfd; + + if (is_fd) + proc_fdlock(p); + else + knhash_lock(p); + + if (knote_fdfind(kq, kev, is_fd, p) != NULL) { + /* found an existing knote: we can't add this one */ + ret = ERESTART; + goto out_locked; + } + + /* knote was not found: add it now */ + if (!is_fd) { + if (fdp->fd_knhashmask == 0) { + u_long size = 0; + + list = hashinit(CONFIG_KN_HASHSIZE, M_KQUEUE, + &size); + if (list == NULL) { + ret = ENOMEM; + goto out_locked; + } + + fdp->fd_knhash = list; + fdp->fd_knhashmask = size; + } - if (! knote_fops(kn)->f_isfd) { - if (fdp->fd_knhashmask == 0) - fdp->fd_knhash = hashinit(CONFIG_KN_HASHSIZE, M_KQUEUE, - &fdp->fd_knhashmask); list = &fdp->fd_knhash[KN_HASH(kn->kn_id, fdp->fd_knhashmask)]; + SLIST_INSERT_HEAD(list, kn, kn_link); + ret = 0; + goto out_locked; + } else { + /* knote is fd based */ + if ((u_int)fdp->fd_knlistsize <= kn->kn_id) { u_int size = 0; if (kn->kn_id >= (uint64_t)p->p_rlimit[RLIMIT_NOFILE].rlim_cur - || kn->kn_id >= (uint64_t)maxfiles) - return (EINVAL); - + || kn->kn_id >= (uint64_t)maxfiles) { + ret = EINVAL; + goto out_locked; + } /* have to grow the fd_knlist */ size = fdp->fd_knlistsize; while (size <= kn->kn_id) size += KQEXTENT; - if (size >= (UINT_MAX/sizeof(struct klist *))) - return (EINVAL); + if (size >= (UINT_MAX/sizeof(struct klist *))) { + ret = EINVAL; + goto out_locked; + } MALLOC(list, struct klist *, size * sizeof(struct klist *), M_KQUEUE, M_WAITOK); - if (list == NULL) - return (ENOMEM); + if (list == NULL) { + ret = ENOMEM; + goto out_locked; + } bcopy((caddr_t)fdp->fd_knlist, (caddr_t)list, fdp->fd_knlistsize * sizeof(struct klist *)); @@ -4721,95 +8138,106 @@ knote_fdadd(struct knote *kn, struct proc *p) fdp->fd_knlist = list; fdp->fd_knlistsize = size; } + list = &fdp->fd_knlist[kn->kn_id]; + SLIST_INSERT_HEAD(list, kn, kn_link); + ret = 0; + goto out_locked; + } - SLIST_INSERT_HEAD(list, kn, kn_link); - return (0); + +out_locked: + if (ret == 0 && knoteuse_needs_boost(kn, kev)) { + set_thread_rwlock_boost(); + *knoteuse_flags = KNUSE_BOOST; + } else { + *knoteuse_flags = KNUSE_NONE; + } + if (is_fd) + proc_fdunlock(p); + else + knhash_unlock(p); + + return ret; } -/* - * knote_fdremove - remove a knote from the fd table for process +/* + * kq_remove_knote - remove a knote from the fd table for process + * and copy kn_status an kq_state while holding kqlock and + * fd table locks. * * If the filter is file-based, remove based on fd index. * Otherwise remove from the hash based on the ident. * - * proc_fdlock held on entry (and exit) + * fd_knhashlock and fdlock unheld on entry (and exit). */ static void -knote_fdremove(struct knote *kn, struct proc *p) +kq_remove_knote(struct kqueue *kq, struct knote *kn, struct proc *p, + kn_status_t *kn_status, uint16_t *kq_state) { struct filedesc *fdp = p->p_fd; struct klist *list = NULL; + bool is_fd; + + is_fd = knote_fops(kn)->f_isfd; - if (knote_fops(kn)->f_isfd) { + if (is_fd) + proc_fdlock(p); + else + knhash_lock(p); + + if (is_fd) { assert ((u_int)fdp->fd_knlistsize > kn->kn_id); list = &fdp->fd_knlist[kn->kn_id]; } else { list = &fdp->fd_knhash[KN_HASH(kn->kn_id, fdp->fd_knhashmask)]; } SLIST_REMOVE(list, kn, knote, kn_link); + + kqlock(kq); + *kn_status = kn->kn_status; + *kq_state = kq->kq_state; + kqunlock(kq); + + if (is_fd) + proc_fdunlock(p); + else + knhash_unlock(p); } -/* - * knote_fdfind - lookup a knote in the fd table for process - * - * If the filter is file-based, lookup based on fd index. - * Otherwise use a hash based on the ident. - * - * Matching is based on kq, filter, and ident. Optionally, - * it may also be based on the udata field in the kevent - - * allowing multiple event registration for the file object - * per kqueue. +/* + * kq_find_knote_and_kq_lock - lookup a knote in the fd table for process + * and, if the knote is found, acquires the kqlock while holding the fd table lock/spinlock. * - * proc_fdlock held on entry (and exit) + * fd_knhashlock or fdlock unheld on entry (and exit) */ -static struct knote * -knote_fdfind(struct kqueue *kq, - struct kevent_internal_s *kev, - struct proc *p) -{ - struct filedesc *fdp = p->p_fd; - struct klist *list = NULL; - struct knote *kn = NULL; - struct filterops *fops; - - fops = sysfilt_ops[~kev->filter]; /* to 0-base index */ - - /* - * determine where to look for the knote - */ - if (fops->f_isfd) { - /* fd-based knotes are linked off the fd table */ - if (kev->ident < (u_int)fdp->fd_knlistsize) { - list = &fdp->fd_knlist[kev->ident]; - } - } else if (fdp->fd_knhashmask != 0) { - /* hash non-fd knotes here too */ - list = &fdp->fd_knhash[KN_HASH((u_long)kev->ident, fdp->fd_knhashmask)]; - } - /* - * scan the selected list looking for a match - */ - if (list != NULL) { - SLIST_FOREACH(kn, list, kn_link) { - if (kq == knote_get_kq(kn) && - kev->ident == kn->kn_id && - kev->filter == kn->kn_filter) { - if (kev->flags & EV_UDATA_SPECIFIC) { - if ((kn->kn_status & KN_UDATA_SPECIFIC) && - kev->udata == kn->kn_udata) { - break; /* matching udata-specific knote */ - } - } else if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) { - break; /* matching non-udata-specific knote */ - } - } - } +static struct knote * +kq_find_knote_and_kq_lock(struct kqueue *kq, + struct kevent_internal_s *kev, + bool is_fd, + struct proc *p) +{ + struct knote * ret; + + if (is_fd) + proc_fdlock(p); + else + knhash_lock(p); + + ret = knote_fdfind(kq, kev, is_fd, p); + + if (ret) { + kqlock(kq); } - return kn; -} + if (is_fd) + proc_fdunlock(p); + else + knhash_unlock(p); + + return ret; +} /* * knote_drop - disconnect and drop the knote * @@ -4829,31 +8257,38 @@ knote_drop(struct knote *kn, __unused struct proc *ctxp) { struct kqueue *kq = knote_get_kq(kn); struct proc *p = kq->kq_p; - int needswakeup; - - /* We have to have a dropping reference on the knote */ - assert(kn->kn_status & KN_DROPPING); + kn_status_t kn_status; + uint16_t kq_state; /* If we are attached, disconnect from the source first */ if (kn->kn_status & KN_ATTACHED) { knote_fops(kn)->f_detach(kn); } - proc_fdlock(p); - /* Remove the source from the appropriate hash */ - knote_fdremove(kn, p); + kq_remove_knote(kq, kn, p, &kn_status, &kq_state); - /* trade fdlock for kq lock */ - kqlock(kq); - proc_fdunlock(p); + /* + * If a kqueue_dealloc is happening in parallel for the kq + * pointed by the knote the kq could be aready deallocated + * at this point. + * Do not access the kq after the kq_remove_knote if it is + * not a KQ_DYNAMIC. + */ /* determine if anyone needs to know about the drop */ - assert((kn->kn_status & (KN_SUPPRESSED | KN_QUEUED)) == 0); - needswakeup = (kn->kn_status & KN_USEWAIT); - kqunlock(kq); + assert((kn_status & (KN_DROPPING | KN_SUPPRESSED | KN_QUEUED)) == KN_DROPPING); - if (needswakeup) + /* + * If KN_USEWAIT is set, some other thread was trying to drop the kn. + * Or it was in kqueue_dealloc, so the kqueue_dealloc did not happen + * because that thread was waiting on this wake, or it was a drop happening + * because of a kevent_register that takes a reference on the kq, and therefore + * the kq cannot be deallocated in parallel. + * + * It is safe to access kq->kq_wqs if needswakeup is set. + */ + if (kn_status & KN_USEWAIT) waitq_wakeup64_all((struct waitq *)&kq->kq_wqs, CAST_EVENT64_T(&kn->kn_status), THREAD_RESTART, @@ -4863,6 +8298,14 @@ knote_drop(struct knote *kn, __unused struct proc *ctxp) fp_drop(p, kn->kn_id, kn->kn_fp, 0); knote_free(kn); + + /* + * release reference on dynamic kq (and free if last). + * Will only be last if this is from fdfree, etc... + * because otherwise processing thread has reference. + */ + if (kq_state & KQ_DYNAMIC) + kqueue_release_last(p, kq); } /* called with kqueue lock held */ @@ -4872,6 +8315,10 @@ knote_activate(struct knote *kn) if (kn->kn_status & KN_ACTIVE) return; + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KNOTE_ACTIVATE), + kn->kn_udata, kn->kn_status | (kn->kn_id << 32), + kn->kn_filtid); + kn->kn_status |= KN_ACTIVE; if (knote_enqueue(kn)) knote_wakeup(kn); @@ -4894,8 +8341,32 @@ knote_enable(struct knote *kn) return; kn->kn_status &= ~KN_DISABLED; - if (knote_enqueue(kn)) + + if (kn->kn_status & KN_SUPPRESSED) { + /* Clear the sync qos on the knote */ + knote_adjust_sync_qos(kn, THREAD_QOS_UNSPECIFIED, FALSE); + + /* + * it is possible for userland to have knotes registered for a given + * workloop `wl_orig` but really handled on another workloop `wl_new`. + * + * In that case, rearming will happen from the servicer thread of + * `wl_new` which if `wl_orig` is no longer being serviced, would cause + * this knote to stay suppressed forever if we only relied on + * kqworkloop_acknowledge_events to be called by `wl_orig`. + * + * However if we see the KQ_PROCESSING bit on `wl_orig` set, we can't + * unsuppress because that would mess with the processing phase of + * `wl_orig`, however it also means kqworkloop_acknowledge_events() + * will be called. + */ + struct kqueue *kq = knote_get_kq(kn); + if ((kq->kq_state & KQ_PROCESSING) == 0) { + knote_unsuppress(kn); + } + } else if (knote_enqueue(kn)) { knote_wakeup(kn); + } } /* called with kqueue lock held */ @@ -4914,14 +8385,27 @@ static void knote_suppress(struct knote *kn) { struct kqtailq *suppressq; + struct kqueue *kq = knote_get_kq(kn); + + kqlock_held(kq); if (kn->kn_status & KN_SUPPRESSED) return; knote_dequeue(kn); kn->kn_status |= KN_SUPPRESSED; - suppressq = knote_get_suppressed_queue(kn); + suppressq = kqueue_get_suppressed_queue(kq, knote_get_qos_index(kn)); TAILQ_INSERT_TAIL(suppressq, kn, kn_tqe); + + if ((kq->kq_state & KQ_WORKLOOP) && + knote_get_qos_override_index(kn) == THREAD_QOS_USER_INTERACTIVE && + kn->kn_qos_override_is_sync) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + /* update the sync qos override counter for suppressed knotes */ + kqworkloop_update_override(kqwl, knote_get_qos_index(kn), + knote_get_qos_override_index(kn), + (KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS | KQWL_UO_NEW_OVERRIDE_IS_SYNC_UI)); + } } /* called with kqueue lock held */ @@ -4929,21 +8413,75 @@ static void knote_unsuppress(struct knote *kn) { struct kqtailq *suppressq; + struct kqueue *kq = knote_get_kq(kn); + + kqlock_held(kq); if ((kn->kn_status & KN_SUPPRESSED) == 0) return; + /* Clear the sync qos on the knote */ + knote_adjust_sync_qos(kn, THREAD_QOS_UNSPECIFIED, FALSE); + kn->kn_status &= ~KN_SUPPRESSED; - suppressq = knote_get_suppressed_queue(kn); + suppressq = kqueue_get_suppressed_queue(kq, knote_get_qos_index(kn)); TAILQ_REMOVE(suppressq, kn, kn_tqe); /* udate in-use qos to equal requested qos */ kn->kn_qos_index = kn->kn_req_index; /* don't wakeup if unsuppressing just a stay-active knote */ - if (knote_enqueue(kn) && - (kn->kn_status & KN_ACTIVE)) + if (knote_enqueue(kn) && (kn->kn_status & KN_ACTIVE)) { knote_wakeup(kn); + } + + if ((kq->kq_state & KQ_WORKLOOP) && !(kq->kq_state & KQ_NO_WQ_THREAD) && + knote_get_qos_override_index(kn) == THREAD_QOS_USER_INTERACTIVE && + kn->kn_qos_override_is_sync) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + /* update the sync qos override counter for suppressed knotes */ + kqworkloop_update_override(kqwl, knote_get_qos_index(kn), + knote_get_qos_override_index(kn), + (KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS | KQWL_UO_OLD_OVERRIDE_IS_SYNC_UI)); + } + + if (TAILQ_EMPTY(suppressq) && (kq->kq_state & KQ_WORKLOOP) && + !(kq->kq_state & KQ_NO_WQ_THREAD)) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + if (kqworkloop_is_processing_on_current_thread(kqwl)) { + /* + * kqworkloop_end_processing() will perform the required QoS + * computations when it unsets the processing mode. + */ + } else { + kqwl_req_lock(kqwl); + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_RESET_WAKEUP_OVERRIDE, 0); + kqwl_req_unlock(kqwl); + } + } +} + +/* called with kqueue lock held */ +static void +knote_update_sync_override_state(struct knote *kn) +{ + struct kqtailq *queue = knote_get_queue(kn); + struct kqueue *kq = knote_get_kq(kn); + + if (!(kq->kq_state & KQ_WORKLOOP) || + knote_get_queue_index(kn) != THREAD_QOS_USER_INTERACTIVE) + return; + + /* Update the sync ipc state on workloop */ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + boolean_t sync_ipc_override = FALSE; + if (!TAILQ_EMPTY(queue)) { + struct knote *kn_head = TAILQ_FIRST(queue); + if (kn_head->kn_qos_override_is_sync) + sync_ipc_override = TRUE; + } + kqworkloop_update_sync_override_state(kqwl, sync_ipc_override); } /* called with kqueue lock held */ @@ -4958,9 +8496,16 @@ knote_enqueue(struct knote *kn) struct kqtailq *queue = knote_get_queue(kn); struct kqueue *kq = knote_get_kq(kn); - TAILQ_INSERT_TAIL(queue, kn, kn_tqe); + kqlock_held(kq); + /* insert at head for sync ipc waiters */ + if (kn->kn_qos_override_is_sync) { + TAILQ_INSERT_HEAD(queue, kn, kn_tqe); + } else { + TAILQ_INSERT_TAIL(queue, kn, kn_tqe); + } kn->kn_status |= KN_QUEUED; kq->kq_count++; + knote_update_sync_override_state(kn); return 1; } return ((kn->kn_status & KN_STAYACTIVE) != 0); @@ -4974,6 +8519,8 @@ knote_dequeue(struct knote *kn) struct kqueue *kq = knote_get_kq(kn); struct kqtailq *queue; + kqlock_held(kq); + if ((kn->kn_status & KN_QUEUED) == 0) return; @@ -4981,6 +8528,7 @@ knote_dequeue(struct knote *kn) TAILQ_REMOVE(queue, kn, kn_tqe); kn->kn_status &= ~KN_QUEUED; kq->kq_count--; + knote_update_sync_override_state(kn); } void @@ -4995,6 +8543,9 @@ knote_init(void) kqworkq_zone = zinit(sizeof(struct kqworkq), 8192*sizeof(struct kqworkq), 8192, "kqueue workq zone"); + kqworkloop_zone = zinit(sizeof(struct kqworkloop), 8192*sizeof(struct kqworkloop), + 8192, "kqueue workloop zone"); + /* allocate kq lock group attribute and group */ kq_lck_grp_attr = lck_grp_attr_alloc_init(); @@ -5016,7 +8567,7 @@ knote_init(void) } SYSINIT(knote, SI_SUB_PSEUDO, SI_ORDER_ANY, knote_init, NULL) -struct filterops * +const struct filterops * knote_fops(struct knote *kn) { return sysfilt_ops[kn->kn_filtid]; @@ -5025,7 +8576,10 @@ knote_fops(struct knote *kn) static struct knote * knote_alloc(void) { - return ((struct knote *)zalloc(knote_zone)); + struct knote *kn; + kn = ((struct knote *)zalloc(knote_zone)); + *kn = (struct knote) { .kn_qos_override = 0, .kn_qos_sync_override = 0, .kn_qos_override_is_sync = 0 }; + return kn; } static void @@ -5105,9 +8659,9 @@ SYSCTL_PROC(_net_systm_kevt, OID_AUTO, pcblist, kevt_pcblist, "S,xkevtpcb", ""); static lck_mtx_t * -event_getlock(struct socket *so, int locktype) +event_getlock(struct socket *so, int flags) { -#pragma unused(locktype) +#pragma unused(flags) struct kern_event_pcb *ev_pcb = (struct kern_event_pcb *)so->so_pcb; if (so->so_pcb != NULL) { @@ -5168,7 +8722,6 @@ event_unlock(struct socket *so, int refcount, void *lr) lr_saved = lr; if (refcount) { - VERIFY(so->so_usecount > 0); so->so_usecount--; } if (so->so_usecount < 0) { @@ -5184,7 +8737,7 @@ event_unlock(struct socket *so, int refcount, void *lr) } mutex_held = (&((struct kern_event_pcb *)so->so_pcb)->evp_mtx); - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; @@ -5203,7 +8756,7 @@ event_sofreelastref(struct socket *so) { struct kern_event_pcb *ev_pcb = (struct kern_event_pcb *)so->so_pcb; - lck_mtx_assert(&(ev_pcb->evp_mtx), LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&(ev_pcb->evp_mtx), LCK_MTX_ASSERT_OWNED); so->so_pcb = NULL; @@ -5217,7 +8770,7 @@ event_sofreelastref(struct socket *so) so->so_event = sonullevent; lck_mtx_unlock(&(ev_pcb->evp_mtx)); - lck_mtx_assert(&(ev_pcb->evp_mtx), LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&(ev_pcb->evp_mtx), LCK_MTX_ASSERT_NOTOWNED); lck_rw_lock_exclusive(kev_rwlock); LIST_REMOVE(ev_pcb, evp_link); kevtstat.kes_pcbcount--; @@ -5401,7 +8954,7 @@ kev_post_msg(struct kev_msg *event_msg) return (EMSGSIZE); } - m = m_get(M_DONTWAIT, MT_DATA); + m = m_get(M_WAIT, MT_DATA); if (m == 0) { OSIncrementAtomic64((SInt64 *)&kevtstat.kes_nomem); return (ENOMEM); @@ -5459,7 +9012,7 @@ kev_post_msg(struct kev_msg *event_msg) } } - m2 = m_copym(m, 0, m->m_len, M_NOWAIT); + m2 = m_copym(m, 0, m->m_len, M_WAIT); if (m2 == 0) { OSIncrementAtomic64((SInt64 *)&kevtstat.kes_nomem); m_free(m); @@ -5680,27 +9233,84 @@ fill_kqueueinfo(struct kqueue *kq, struct kqueue_info * kinfo) else st->vst_blksize = sizeof(struct kevent); st->vst_mode = S_IFIFO; + st->vst_ino = (kq->kq_state & KQ_DYNAMIC) ? + ((struct kqworkloop *)kq)->kqwl_dynamicid : 0; /* flags exported to libproc as PROC_KQUEUE_* (sys/proc_info.h) */ -#define PROC_KQUEUE_MASK (KQ_SEL|KQ_SLEEP|KQ_KEV32|KQ_KEV64|KQ_KEV_QOS) +#define PROC_KQUEUE_MASK (KQ_SEL|KQ_SLEEP|KQ_KEV32|KQ_KEV64|KQ_KEV_QOS|KQ_WORKQ|KQ_WORKLOOP) kinfo->kq_state = kq->kq_state & PROC_KQUEUE_MASK; return (0); } +static int +fill_kqueue_dyninfo(struct kqueue *kq, struct kqueue_dyninfo *kqdi) +{ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct kqrequest *kqr = &kqwl->kqwl_request; + int err; + + if ((kq->kq_state & KQ_WORKLOOP) == 0) { + return EINVAL; + } + + if ((err = fill_kqueueinfo(kq, &kqdi->kqdi_info))) { + return err; + } + + kqwl_req_lock(kqwl); + + if (kqr->kqr_thread) { + kqdi->kqdi_servicer = thread_tid(kqr->kqr_thread); + } + + if (kqwl->kqwl_owner == WL_OWNER_SUSPENDED) { + kqdi->kqdi_owner = ~0ull; + } else { + kqdi->kqdi_owner = thread_tid(kqwl->kqwl_owner); + } + + kqdi->kqdi_request_state = kqr->kqr_state; + kqdi->kqdi_async_qos = kqr->kqr_qos_index; + kqdi->kqdi_events_qos = kqr->kqr_override_index; + kqdi->kqdi_sync_waiters = kqr->kqr_dsync_waiters; + kqdi->kqdi_sync_waiter_qos = kqr->kqr_dsync_waiters_qos; + + kqwl_req_unlock(kqwl); + + return 0; +} + void knote_markstayactive(struct knote *kn) { - kqlock(knote_get_kq(kn)); + struct kqueue *kq = knote_get_kq(kn); + + kqlock(kq); kn->kn_status |= KN_STAYACTIVE; - /* handle all stayactive knotes on the manager */ - if (knote_get_kq(kn)->kq_state & KQ_WORKQ) + /* + * Making a knote stay active is a property of the knote that must be + * established before it is fully attached. + */ + assert(kn->kn_status & KN_ATTACHING); + + /* handle all stayactive knotes on the (appropriate) manager */ + if (kq->kq_state & KQ_WORKQ) { knote_set_qos_index(kn, KQWQ_QOS_MANAGER); + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + kqwl_req_lock(kqwl); + assert(kn->kn_req_index && kn->kn_req_index < THREAD_QOS_LAST); + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_UPDATE_STAYACTIVE_QOS, + kn->kn_req_index); + kqwl_req_unlock(kqwl); + knote_set_qos_index(kn, KQWL_BUCKET_STAYACTIVE); + } knote_activate(kn); - kqunlock(knote_get_kq(kn)); + kqunlock(kq); } void @@ -5716,27 +9326,27 @@ static unsigned long kevent_extinfo_emit(struct kqueue *kq, struct knote *kn, struct kevent_extinfo *buf, unsigned long buflen, unsigned long nknotes) { - struct kevent_internal_s *kevp; for (; kn; kn = SLIST_NEXT(kn, kn_link)) { if (kq == knote_get_kq(kn)) { if (nknotes < buflen) { struct kevent_extinfo *info = &buf[nknotes]; - struct kevent_qos_s kevqos; + struct kevent_internal_s *kevp = &kn->kn_kevent; kqlock(kq); - kevp = &(kn->kn_kevent); - - bzero(&kevqos, sizeof(kevqos)); - kevqos.ident = kevp->ident; - kevqos.filter = kevp->filter; - kevqos.flags = kevp->flags; - kevqos.fflags = kevp->fflags; - kevqos.data = (int64_t) kevp->data; - kevqos.udata = kevp->udata; - kevqos.ext[0] = kevp->ext[0]; - kevqos.ext[1] = kevp->ext[1]; - - memcpy(&info->kqext_kev, &kevqos, sizeof(info->kqext_kev)); + + info->kqext_kev = (struct kevent_qos_s){ + .ident = kevp->ident, + .filter = kevp->filter, + .flags = kevp->flags, + .fflags = kevp->fflags, + .data = (int64_t)kevp->data, + .udata = kevp->udata, + .ext[0] = kevp->ext[0], + .ext[1] = kevp->ext[1], + .ext[2] = kevp->ext[2], + .ext[3] = kevp->ext[3], + .qos = kn->kn_req_index, + }; info->kqext_sdata = kn->kn_sdata; info->kqext_status = kn->kn_status; info->kqext_sfflags = kn->kn_sfflags; @@ -5752,6 +9362,142 @@ kevent_extinfo_emit(struct kqueue *kq, struct knote *kn, struct kevent_extinfo * return nknotes; } +int +kevent_copyout_proc_dynkqids(void *proc, user_addr_t ubuf, uint32_t ubufsize, + int32_t *nkqueues_out) +{ + proc_t p = (proc_t)proc; + struct filedesc *fdp = p->p_fd; + unsigned int nkqueues = 0; + unsigned long ubuflen = ubufsize / sizeof(kqueue_id_t); + size_t buflen, bufsize; + kqueue_id_t *kq_ids = NULL; + int err = 0; + + assert(p != NULL); + + if (ubuf == USER_ADDR_NULL && ubufsize != 0) { + err = EINVAL; + goto out; + } + + buflen = min(ubuflen, PROC_PIDDYNKQUEUES_MAX); + + if (ubuflen != 0) { + if (os_mul_overflow(sizeof(kqueue_id_t), buflen, &bufsize)) { + err = ERANGE; + goto out; + } + kq_ids = kalloc(bufsize); + assert(kq_ids != NULL); + } + + kqhash_lock(p); + + if (fdp->fd_kqhashmask > 0) { + for (uint32_t i = 0; i < fdp->fd_kqhashmask + 1; i++) { + struct kqworkloop *kqwl; + + SLIST_FOREACH(kqwl, &fdp->fd_kqhash[i], kqwl_hashlink) { + /* report the number of kqueues, even if they don't all fit */ + if (nkqueues < buflen) { + kq_ids[nkqueues] = kqwl->kqwl_dynamicid; + } + nkqueues++; + } + } + } + + kqhash_unlock(p); + + if (kq_ids) { + size_t copysize; + if (os_mul_overflow(sizeof(kqueue_id_t), min(ubuflen, nkqueues), ©size)) { + err = ERANGE; + goto out; + } + + assert(ubufsize >= copysize); + err = copyout(kq_ids, ubuf, copysize); + } + +out: + if (kq_ids) { + kfree(kq_ids, bufsize); + } + + if (!err) { + *nkqueues_out = (int)min(nkqueues, PROC_PIDDYNKQUEUES_MAX); + } + return err; +} + +int +kevent_copyout_dynkqinfo(void *proc, kqueue_id_t kq_id, user_addr_t ubuf, + uint32_t ubufsize, int32_t *size_out) +{ + proc_t p = (proc_t)proc; + struct kqueue *kq; + int err = 0; + struct kqueue_dyninfo kqdi = { }; + + assert(p != NULL); + + if (ubufsize < sizeof(struct kqueue_info)) { + return ENOBUFS; + } + + kqhash_lock(p); + kq = kqueue_hash_lookup(p, kq_id); + if (!kq) { + kqhash_unlock(p); + return ESRCH; + } + kqueue_retain(kq); + kqhash_unlock(p); + + /* + * backward compatibility: allow the argument to this call to only be + * a struct kqueue_info + */ + if (ubufsize >= sizeof(struct kqueue_dyninfo)) { + ubufsize = sizeof(struct kqueue_dyninfo); + err = fill_kqueue_dyninfo(kq, &kqdi); + } else { + ubufsize = sizeof(struct kqueue_info); + err = fill_kqueueinfo(kq, &kqdi.kqdi_info); + } + if (err == 0 && (err = copyout(&kqdi, ubuf, ubufsize)) == 0) { + *size_out = ubufsize; + } + kqueue_release_last(p, kq); + return err; +} + +int +kevent_copyout_dynkqextinfo(void *proc, kqueue_id_t kq_id, user_addr_t ubuf, + uint32_t ubufsize, int32_t *nknotes_out) +{ + proc_t p = (proc_t)proc; + struct kqueue *kq; + int err; + + assert(p != NULL); + + kqhash_lock(p); + kq = kqueue_hash_lookup(p, kq_id); + if (!kq) { + kqhash_unlock(p); + return ESRCH; + } + kqueue_retain(kq); + kqhash_unlock(p); + + err = pid_kqueue_extinfo(p, kq, ubuf, ubufsize, nknotes_out); + kqueue_release_last(p, kq); + return err; +} + int pid_kqueue_extinfo(proc_t p, struct kqueue *kq, user_addr_t ubuf, uint32_t bufsize, int32_t *retval) @@ -5775,21 +9521,21 @@ pid_kqueue_extinfo(proc_t p, struct kqueue *kq, user_addr_t ubuf, bzero(kqext, buflen * sizeof(struct kevent_extinfo)); proc_fdlock(p); - for (i = 0; i < fdp->fd_knlistsize; i++) { kn = SLIST_FIRST(&fdp->fd_knlist[i]); nknotes = kevent_extinfo_emit(kq, kn, kqext, buflen, nknotes); } + proc_fdunlock(p); if (fdp->fd_knhashmask != 0) { for (i = 0; i < (int)fdp->fd_knhashmask + 1; i++) { + kqhash_lock(p); kn = SLIST_FIRST(&fdp->fd_knhash[i]); nknotes = kevent_extinfo_emit(kq, kn, kqext, buflen, nknotes); + kqhash_unlock(p); } } - proc_fdunlock(p); - assert(bufsize >= sizeof(struct kevent_extinfo) * min(buflen, nknotes)); err = copyout(kqext, ubuf, sizeof(struct kevent_extinfo) * min(buflen, nknotes)); @@ -5805,53 +9551,185 @@ pid_kqueue_extinfo(proc_t p, struct kqueue *kq, user_addr_t ubuf, return err; } -static unsigned long -kevent_udatainfo_emit(struct kqueue *kq, struct knote *kn, uint64_t *buf, - unsigned long buflen, unsigned long nknotes) +static unsigned int +klist_copy_udata(struct klist *list, uint64_t *buf, + unsigned int buflen, unsigned int nknotes) { - struct kevent_internal_s *kevp; - for (; kn; kn = SLIST_NEXT(kn, kn_link)) { - if (kq == knote_get_kq(kn)) { - if (nknotes < buflen) { - kqlock(kq); - kevp = &(kn->kn_kevent); - buf[nknotes] = kevp->udata; - kqunlock(kq); - } - - /* we return total number of knotes, which may be more than requested */ - nknotes++; + struct kevent_internal_s *kev; + struct knote *kn; + SLIST_FOREACH(kn, list, kn_link) { + if (nknotes < buflen) { + struct kqueue *kq = knote_get_kq(kn); + kqlock(kq); + kev = &(kn->kn_kevent); + buf[nknotes] = kev->udata; + kqunlock(kq); } + /* we return total number of knotes, which may be more than requested */ + nknotes++; } return nknotes; } +static unsigned int +kqlist_copy_dynamicids(__assert_only proc_t p, struct kqlist *list, + uint64_t *buf, unsigned int buflen, unsigned int nids) +{ + kqhash_lock_held(p); + struct kqworkloop *kqwl; + SLIST_FOREACH(kqwl, list, kqwl_hashlink) { + if (nids < buflen) { + buf[nids] = kqwl->kqwl_dynamicid; + } + nids++; + } + return nids; +} + int -pid_kqueue_udatainfo(proc_t p, struct kqueue *kq, uint64_t *buf, - uint32_t bufsize) +kevent_proc_copy_uptrs(void *proc, uint64_t *buf, int bufsize) { - struct knote *kn; - int i; + proc_t p = (proc_t)proc; struct filedesc *fdp = p->p_fd; - unsigned long nknotes = 0; + unsigned int nuptrs = 0; unsigned long buflen = bufsize / sizeof(uint64_t); + if (buflen > 0) { + assert(buf != NULL); + } + proc_fdlock(p); + for (int i = 0; i < fdp->fd_knlistsize; i++) { + nuptrs = klist_copy_udata(&fdp->fd_knlist[i], buf, buflen, nuptrs); + } + knhash_lock(p); + proc_fdunlock(p); + if (fdp->fd_knhashmask != 0) { + for (int i = 0; i < (int)fdp->fd_knhashmask + 1; i++) { + nuptrs = klist_copy_udata(&fdp->fd_knhash[i], buf, buflen, nuptrs); + } + } + knhash_unlock(p); - for (i = 0; i < fdp->fd_knlistsize; i++) { - kn = SLIST_FIRST(&fdp->fd_knlist[i]); - nknotes = kevent_udatainfo_emit(kq, kn, buf, buflen, nknotes); + kqhash_lock(p); + if (fdp->fd_kqhashmask != 0) { + for (int i = 0; i < (int)fdp->fd_kqhashmask + 1; i++) { + nuptrs = kqlist_copy_dynamicids(p, &fdp->fd_kqhash[i], buf, buflen, + nuptrs); + } } + kqhash_unlock(p); - if (fdp->fd_knhashmask != 0) { - for (i = 0; i < (int)fdp->fd_knhashmask + 1; i++) { - kn = SLIST_FIRST(&fdp->fd_knhash[i]); - nknotes = kevent_udatainfo_emit(kq, kn, buf, buflen, nknotes); + return (int)nuptrs; +} + +static void +kevent_redrive_proc_thread_request(proc_t p) +{ + __assert_only int ret; + ret = (*pthread_functions->workq_threadreq)(p, NULL, WORKQ_THREADREQ_REDRIVE, 0, 0); + assert(ret == 0 || ret == ECANCELED); +} + +static void +kevent_set_return_to_kernel_user_tsd(proc_t p, thread_t thread) +{ + uint64_t ast_addr; + bool proc_is_64bit = !!(p->p_flag & P_LP64); + size_t user_addr_size = proc_is_64bit ? 8 : 4; + uint32_t ast_flags32 = 0; + uint64_t ast_flags64 = 0; + struct uthread *ut = get_bsdthread_info(thread); + + if (ut->uu_kqueue_bound != NULL) { + if (ut->uu_kqueue_flags & KEVENT_FLAG_WORKLOOP) { + ast_flags64 |= R2K_WORKLOOP_PENDING_EVENTS; + } else if (ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ) { + ast_flags64 |= R2K_WORKQ_PENDING_EVENTS; } } - proc_fdunlock(p); - return (int)nknotes; + if (ast_flags64 == 0) { + return; + } + + if (!(p->p_flag & P_LP64)) { + ast_flags32 = (uint32_t)ast_flags64; + assert(ast_flags64 < 0x100000000ull); + } + + ast_addr = thread_rettokern_addr(thread); + if (ast_addr == 0) { + return; + } + + if (copyout((proc_is_64bit ? (void *)&ast_flags64 : (void *)&ast_flags32), + (user_addr_t)ast_addr, + user_addr_size) != 0) { + printf("pid %d (tid:%llu): copyout of return_to_kernel ast flags failed with " + "ast_addr = %llu\n", p->p_pid, thread_tid(current_thread()), ast_addr); + } +} + +void +kevent_ast(thread_t thread, uint16_t bits) +{ + proc_t p = current_proc(); + + if (bits & AST_KEVENT_REDRIVE_THREADREQ) { + kevent_redrive_proc_thread_request(p); + } + if (bits & AST_KEVENT_RETURN_TO_KERNEL) { + kevent_set_return_to_kernel_user_tsd(p, thread); + } +} + +#if DEVELOPMENT || DEBUG + +#define KEVENT_SYSCTL_BOUND_ID 1 + +static int +kevent_sysctl SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg2) + uintptr_t type = (uintptr_t)arg1; + uint64_t bound_id = 0; + struct uthread *ut; + struct kqueue *kq; + + if (type != KEVENT_SYSCTL_BOUND_ID) { + return EINVAL; + } + + if (req->newptr) { + return EINVAL; + } + + ut = get_bsdthread_info(current_thread()); + if (!ut) { + return EFAULT; + } + + kq = ut->uu_kqueue_bound; + if (kq) { + if (kq->kq_state & KQ_WORKLOOP) { + bound_id = ((struct kqworkloop *)kq)->kqwl_dynamicid; + } else if (kq->kq_state & KQ_WORKQ) { + bound_id = -1; + } + } + + return sysctl_io_number(req, bound_id, sizeof(bound_id), NULL, NULL); } +SYSCTL_NODE(_kern, OID_AUTO, kevent, CTLFLAG_RW | CTLFLAG_LOCKED, 0, + "kevent information"); + +SYSCTL_PROC(_kern_kevent, OID_AUTO, bound_id, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED | CTLFLAG_MASKED, + (void *)KEVENT_SYSCTL_BOUND_ID, + sizeof(kqueue_id_t), kevent_sysctl, "Q", + "get the ID of the bound kqueue"); + +#endif /* DEVELOPMENT || DEBUG */ diff --git a/bsd/kern/kern_exec.c b/bsd/kern/kern_exec.c index 604da6996..9412a4472 100644 --- a/bsd/kern/kern_exec.c +++ b/bsd/kern/kern_exec.c @@ -138,8 +138,10 @@ #include #include +#include + #if CONFIG_MACF -#include +#include #include #endif @@ -242,6 +244,16 @@ extern const struct fileops vnops; ( ( (user_addr_t)(addr) + (val) - 1) \ & ~((val) - 1) ) + /* Platform Code Exec Logging */ +static int platform_exec_logging = 0; + +SYSCTL_DECL(_security_mac); + +SYSCTL_INT(_security_mac, OID_AUTO, platform_exec_logging, CTLFLAG_RW, &platform_exec_logging, 0, + "log cdhashes for all platform binary executions"); + +static os_log_t peLog = OS_LOG_DEFAULT; + struct image_params; /* Forward */ static int exec_activate_image(struct image_params *imgp); static int exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp); @@ -454,10 +466,6 @@ exec_shell_imgact(struct image_params *imgp) char *ihp; char *line_startp, *line_endp; char *interp; - proc_t p; - struct fileproc *fp; - int fd; - int error; /* * Make sure it's a shell script. If we've already redirected @@ -552,12 +560,18 @@ exec_shell_imgact(struct image_params *imgp) *interp++ = *ihp; *interp = '\0'; +#if !SECURE_KERNEL /* - * If we have a SUID oder SGID script, create a file descriptor + * If we have an SUID or SGID script, create a file descriptor * from the vnode and pass /dev/fd/%d instead of the actual * path name so that the script does not get opened twice */ if (imgp->ip_origvattr->va_mode & (VSUID | VSGID)) { + proc_t p; + struct fileproc *fp; + int fd; + int error; + p = vfs_context_proc(imgp->ip_vfs_context); error = falloc(p, &fp, &fd, imgp->ip_vfs_context); if (error) @@ -575,6 +589,7 @@ exec_shell_imgact(struct image_params *imgp) imgp->ip_interp_sugid_fd = fd; } +#endif return (-3); } @@ -773,6 +788,15 @@ set_proc_name(struct image_params *imgp, proc_t p) p->p_comm[imgp->ip_ndp->ni_cnd.cn_namelen] = '\0'; } +static uint64_t get_va_fsid(struct vnode_attr *vap) +{ + if (VATTR_IS_SUPPORTED(vap, va_fsid64)) { + return *(uint64_t *)&vap->va_fsid64; + } else { + return vap->va_fsid; + } +} + /* * exec_mach_imgact * @@ -984,8 +1008,14 @@ grade: imgp->ip_csflags |= CS_KILL; if (p->p_csflags & CS_EXEC_SET_ENFORCEMENT) imgp->ip_csflags |= CS_ENFORCEMENT; - if (p->p_csflags & CS_EXEC_SET_INSTALLER) - imgp->ip_csflags |= CS_INSTALLER; + if (p->p_csflags & CS_EXEC_INHERIT_SIP) { + if (p->p_csflags & CS_INSTALLER) + imgp->ip_csflags |= CS_INSTALLER; + if (p->p_csflags & CS_DATAVAULT_CONTROLLER) + imgp->ip_csflags |= CS_DATAVAULT_CONTROLLER; + if (p->p_csflags & CS_NVRAM_UNRESTRICTED) + imgp->ip_csflags |= CS_NVRAM_UNRESTRICTED; + } /* * Set up the system reserved areas in the new address space. @@ -995,7 +1025,7 @@ grade: /* * Close file descriptors which specify close-on-exec. */ - fdexec(p, psa != NULL ? psa->psa_flags : 0); + fdexec(p, psa != NULL ? psa->psa_flags : 0, exec); /* * deal with set[ug]id. @@ -1147,17 +1177,10 @@ grade: set_proc_name(imgp, p); #if CONFIG_SECLUDED_MEMORY - if (secluded_for_apps) { + if (secluded_for_apps && + load_result.platform_binary) { if (strncmp(p->p_name, "Camera", - sizeof (p->p_name)) == 0 || -#if 00 - strncmp(p->p_name, - "camerad", - sizeof (p->p_name)) == 0 || -#endif - strncmp(p->p_name, - "testCamera", sizeof (p->p_name)) == 0) { task_set_could_use_secluded_mem(task, TRUE); } else { @@ -1171,10 +1194,34 @@ grade: } #endif /* CONFIG_SECLUDED_MEMORY */ - pal_dbg_set_task_name( task ); + pal_dbg_set_task_name(task); + + /* + * The load result will have already been munged by AMFI to include the + * platform binary flag if boot-args dictated it (AMFI will mark anything + * that doesn't go through the upcall path as a platform binary if its + * enforcement is disabled). + */ + if (load_result.platform_binary) { + if (cs_debug) { + printf("setting platform binary on task: pid = %d\n", p->p_pid); + } + + /* + * We must use 'task' here because the proc's task has not yet been + * switched to the new one. + */ + task_set_platform_binary(task, TRUE); + } else { + if (cs_debug) { + printf("clearing platform binary on task: pid = %d\n", p->p_pid); + } + + task_set_platform_binary(task, FALSE); + } #if DEVELOPMENT || DEBUG - /* + /* * Update the pid an proc name for importance base if any */ task_importance_update_owner_info(task); @@ -1194,8 +1241,18 @@ grade: */ kdbg_trace_string(p, &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4); + uintptr_t fsid = 0, fileid = 0; + if (imgp->ip_vattr) { + uint64_t fsid64 = get_va_fsid(imgp->ip_vattr); + fsid = fsid64; + fileid = imgp->ip_vattr->va_fileid; + // check for (unexpected) overflow and trace zero in that case + if (fsid != fsid64 || fileid != imgp->ip_vattr->va_fileid) { + fsid = fileid = 0; + } + } KERNEL_DEBUG_CONSTANT1(TRACE_DATA_EXEC | DBG_FUNC_NONE, - p->p_pid ,0,0,0, (uintptr_t)thread_tid(thread)); + p->p_pid , fsid, fileid, 0, (uintptr_t)thread_tid(thread)); KERNEL_DEBUG_CONSTANT1(TRACE_STRING_EXEC | DBG_FUNC_NONE, dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, (uintptr_t)thread_tid(thread)); } @@ -1306,7 +1363,8 @@ struct execsw { * * Description: Iterate through the available image activators, and activate * the image associated with the imgp structure. We start with - * the + * the activator for Mach-o binaries followed by that for Fat binaries + * for Interpreter scripts. * * Parameters: struct image_params * Image parameter block * @@ -1527,9 +1585,11 @@ exec_handle_spawnattr_policy(proc_t p, int psa_apptype, uint64_t psa_qos_clamp, case POSIX_SPAWN_PROC_TYPE_APP_DEFAULT: apptype = TASK_APPTYPE_APP_DEFAULT; break; +#if !CONFIG_EMBEDDED case POSIX_SPAWN_PROC_TYPE_APP_TAL: apptype = TASK_APPTYPE_APP_TAL; break; +#endif /* !CONFIG_EMBEDDED */ default: apptype = TASK_APPTYPE_NONE; /* TODO: Should an invalid value here fail the spawn? */ @@ -2443,9 +2503,25 @@ do_fork1: #if CONFIG_COALITIONS /* set the roles of this task within each given coalition */ if (error == 0) { - kr = coalitions_set_roles(coal, get_threadtask(imgp->ip_new_thread), coal_role); + kr = coalitions_set_roles(coal, new_task, coal_role); if (kr != KERN_SUCCESS) error = EINVAL; + if (kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_COALITION, + MACH_COALITION_ADOPT))) { + for (i = 0; i < COALITION_NUM_TYPES; i++) { + if (coal[i] != COALITION_NULL) { + /* + * On 32-bit targets, uniqueid + * will get truncated to 32 bits + */ + KDBG_RELEASE(MACHDBG_CODE( + DBG_MACH_COALITION, + MACH_COALITION_ADOPT), + coalition_id(coal[i]), + get_task_uniqueid(new_task)); + } + } + } } /* drop our references and activations - fork1() now holds them */ @@ -2671,6 +2747,10 @@ do_fork1: OSBitOrAtomic(P_DISABLE_ASLR, &p->p_flag); #endif /* !SECURE_KERNEL */ + /* Randomize high bits of ASLR slide */ + if (px_sa.psa_flags & _POSIX_SPAWN_HIGH_BITS_ASLR) + imgp->ip_flags |= IMGPF_HIGH_BITS_ASLR; + /* * Forcibly disallow execution from data pages for the spawned process * even if it would otherwise be permitted by the architecture default. @@ -2788,8 +2868,10 @@ bad: if (error == 0) { /* reset delay idle sleep status if set */ +#if !CONFIG_EMBEDDED if ((p->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP) OSBitAndAtomic(~((uint32_t)P_DELAYIDLESLEEP), &p->p_flag); +#endif /* !CONFIG_EMBEDDED */ /* upon successful spawn, re/set the proc control state */ if (imgp->ip_px_sa != NULL) { switch (px_sa.psa_pcontrol) { @@ -2816,20 +2898,17 @@ bad: /* * With 2-level high-water-mark support, POSIX_SPAWN_JETSAM_HIWATER_BACKGROUND is no * longer relevant, as background limits are described via the inactive limit slots. - * At the kernel layer, the flag is ignored. * * That said, however, if the POSIX_SPAWN_JETSAM_HIWATER_BACKGROUND is passed in, * we attempt to mimic previous behavior by forcing the BG limit data into the * inactive/non-fatal mode and force the active slots to hold system_wide/fatal mode. - * The kernel layer will flag this mapping. */ if (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_HIWATER_BACKGROUND) { memorystatus_update(p, px_sa.psa_priority, 0, (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_USE_EFFECTIVE_PRIORITY), TRUE, -1, TRUE, - px_sa.psa_memlimit_inactive, FALSE, - (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_HIWATER_BACKGROUND)); + px_sa.psa_memlimit_inactive, FALSE); } else { memorystatus_update(p, px_sa.psa_priority, 0, (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_USE_EFFECTIVE_PRIORITY), @@ -2837,8 +2916,7 @@ bad: px_sa.psa_memlimit_active, (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_MEMLIMIT_ACTIVE_FATAL), px_sa.psa_memlimit_inactive, - (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_MEMLIMIT_INACTIVE_FATAL), - (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_HIWATER_BACKGROUND)); + (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_MEMLIMIT_INACTIVE_FATAL)); } } @@ -3163,8 +3241,12 @@ proc_exec_switch_task(proc_t p, task_t old_task, task_t new_task, thread_t new_t * Switch the task pointer of proc to new task. * Before switching the task, wait for proc_refdrain. * After the switch happens, the proc can disappear, - * take a ref before it disappears. + * take a ref before it disappears. Waiting for + * proc_refdrain in exec will block all other threads + * trying to take a proc ref, boost the current thread + * to avoid priority inversion. */ + thread_set_exec_promotion(old_thread); p = proc_refdrain_with_refwait(p, TRUE); /* extra proc ref returned to the caller */ @@ -3194,6 +3276,11 @@ proc_exec_switch_task(proc_t p, task_t old_task, task_t new_task, thread_t new_t */ p->task = new_task; + /* Clear dispatchqueue and workloop ast offset */ + p->p_dispatchqueue_offset = 0; + p->p_dispatchqueue_serialno_offset = 0; + p->p_return_to_kernel_offset = 0; + /* Copy the signal state, dtrace state and set bsd ast on new thread */ act_set_astbsd(new_thread); new_uthread->uu_siglist = old_uthread->uu_siglist; @@ -3228,12 +3315,15 @@ proc_exec_switch_task(proc_t p, task_t old_task, task_t new_task, thread_t new_t task_set_did_exec_flag(old_task); task_clear_exec_copy_flag(new_task); + task_copy_fields_for_exec(new_task, old_task); + proc_transend(p, 1); } } proc_unlock(p); proc_refwake(p); + thread_clear_exec_promotion(old_thread); if (error != 0 || !task_active || !proc_active || !thread_active) { task_terminate_internal(new_task); @@ -4132,6 +4222,12 @@ extern user64_addr_t commpage_text64_location; #define MAIN_STACK_VALUES 4 #define MAIN_STACK_KEY "main_stack=" +#define FSID_KEY "executable_file=" +#define DYLD_FSID_KEY "dyld_file=" +#define CDHASH_KEY "executable_cdhash=" + +#define FSID_MAX_STRING "0x1234567890abcdef,0x1234567890abcdef" + #define HEX_STR_LEN 18 // 64-bit hex value "0x0123456701234567" static int @@ -4251,6 +4347,48 @@ exec_add_apple_strings(struct image_params *imgp, imgp->ip_applec++; } + if (imgp->ip_vattr) { + uint64_t fsid = get_va_fsid(imgp->ip_vattr); + uint64_t fsobjid = imgp->ip_vattr->va_fileid; + + char fsid_string[strlen(FSID_KEY) + strlen(FSID_MAX_STRING) + 1]; + snprintf(fsid_string, sizeof(fsid_string), + FSID_KEY "0x%llx,0x%llx", fsid, fsobjid); + error = exec_add_user_string(imgp, CAST_USER_ADDR_T(fsid_string), UIO_SYSSPACE, FALSE); + if (error) { + goto bad; + } + imgp->ip_applec++; + } + + if (imgp->ip_dyld_fsid || imgp->ip_dyld_fsobjid ) { + char fsid_string[strlen(DYLD_FSID_KEY) + strlen(FSID_MAX_STRING) + 1]; + snprintf(fsid_string, sizeof(fsid_string), + DYLD_FSID_KEY "0x%llx,0x%llx", imgp->ip_dyld_fsid, imgp->ip_dyld_fsobjid); + error = exec_add_user_string(imgp, CAST_USER_ADDR_T(fsid_string), UIO_SYSSPACE, FALSE); + if (error) { + goto bad; + } + imgp->ip_applec++; + } + + uint8_t cdhash[SHA1_RESULTLEN]; + int cdhash_errror = ubc_cs_getcdhash(imgp->ip_vp, imgp->ip_arch_offset, cdhash); + if (cdhash_errror == 0) { + char hash_string[strlen(CDHASH_KEY) + 2*SHA1_RESULTLEN + 1]; + strncpy(hash_string, CDHASH_KEY, sizeof(hash_string)); + char *p = hash_string + sizeof(CDHASH_KEY) - 1; + for (int i = 0; i < SHA1_RESULTLEN; i++) { + snprintf(p, 3, "%02x", (int) cdhash[i]); + p += 2; + } + error = exec_add_user_string(imgp, CAST_USER_ADDR_T(hash_string), UIO_SYSSPACE, FALSE); + if (error) { + goto bad; + } + imgp->ip_applec++; + } + /* Align the tail of the combined applev area */ while (imgp->ip_strspace % img_ptr_size != 0) { *imgp->ip_strendp++ = '\0'; @@ -4298,6 +4436,7 @@ exec_check_permissions(struct image_params *imgp) VATTR_WANTED(vap, va_gid); VATTR_WANTED(vap, va_mode); VATTR_WANTED(vap, va_fsid); + VATTR_WANTED(vap, va_fsid64); VATTR_WANTED(vap, va_fileid); VATTR_WANTED(vap, va_data_size); if ((error = vnode_getattr(vp, vap, imgp->ip_vfs_context)) != 0) @@ -4449,6 +4588,7 @@ exec_handle_sugid(struct image_params *imgp) handle_mac_transition: #endif +#if !SECURE_KERNEL /* * Replace the credential with a copy of itself if euid or * egid change. @@ -4526,6 +4666,7 @@ handle_mac_transition: break; } +#endif /* !SECURE_KERNEL */ #if CONFIG_MACF /* @@ -4785,15 +4926,13 @@ create_unix_stack(vm_map_t map, load_result_t* load_result, return KERN_INVALID_ARGUMENT; } addr = mach_vm_trunc_page(load_result->user_stack - size); - kr = mach_vm_allocate(map, &addr, size, - VM_MAKE_TAG(VM_MEMORY_STACK) | - VM_FLAGS_FIXED); + kr = mach_vm_allocate_kernel(map, &addr, size, + VM_FLAGS_FIXED, VM_MEMORY_STACK); if (kr != KERN_SUCCESS) { // Can't allocate at default location, try anywhere addr = 0; - kr = mach_vm_allocate(map, &addr, size, - VM_MAKE_TAG(VM_MEMORY_STACK) | - VM_FLAGS_ANYWHERE); + kr = mach_vm_allocate_kernel(map, &addr, size, + VM_FLAGS_ANYWHERE, VM_MEMORY_STACK); if (kr != KERN_SUCCESS) { return kr; } @@ -4985,10 +5124,10 @@ load_init_program(proc_t p) mach_vm_offset_t scratch_addr = 0; mach_vm_size_t map_page_size = vm_map_page_size(map); - (void) mach_vm_allocate(map, &scratch_addr, map_page_size, VM_FLAGS_ANYWHERE); -#if CONFIG_MEMORYSTATUS && CONFIG_JETSAM + (void) mach_vm_allocate_kernel(map, &scratch_addr, map_page_size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_NONE); +#if CONFIG_MEMORYSTATUS (void) memorystatus_init_at_boot_snapshot(); -#endif /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */ +#endif /* CONFIG_MEMORYSTATUS */ #if DEBUG || DEVELOPMENT /* Check for boot-arg suffix first */ @@ -4999,6 +5138,7 @@ load_init_program(proc_t p) (strcmp(launchd_suffix, "release") == 0)); if (is_release_suffix) { + printf("load_init_program: attempting to load /sbin/launchd\n"); error = load_init_program_at_path(p, (user_addr_t)scratch_addr, "/sbin/launchd"); if (!error) return; @@ -5008,10 +5148,12 @@ load_init_program(proc_t p) strlcpy(launchd_path, "/usr/local/sbin/launchd.", sizeof(launchd_path)); strlcat(launchd_path, launchd_suffix, sizeof(launchd_path)); - /* All the error data is lost in the loop below, don't - * attempt to save it. */ - if (!load_init_program_at_path(p, (user_addr_t)scratch_addr, launchd_path)) { + printf("load_init_program: attempting to load %s\n", launchd_path); + error = load_init_program_at_path(p, (user_addr_t)scratch_addr, launchd_path); + if (!error) { return; + } else { + printf("load_init_program: failed loading %s: errno %d\n", launchd_path, error); } } } @@ -5019,9 +5161,13 @@ load_init_program(proc_t p) error = ENOENT; for (i = 0; i < sizeof(init_programs)/sizeof(init_programs[0]); i++) { + printf("load_init_program: attempting to load %s\n", init_programs[i]); error = load_init_program_at_path(p, (user_addr_t)scratch_addr, init_programs[i]); - if (!error) + if (!error) { return; + } else { + printf("load_init_program: failed loading %s: errno %d\n", init_programs[i], error); + } } panic("Process 1 exec of %s failed, errno %d", ((i == 0) ? "" : init_programs[i-1]), error); @@ -5136,7 +5282,7 @@ execargs_lock_sleep(void) { static kern_return_t execargs_purgeable_allocate(char **execarg_address) { - kern_return_t kr = vm_allocate(bsd_pageable_map, (vm_offset_t *)execarg_address, BSD_PAGEABLE_SIZE_PER_EXEC, VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE); + kern_return_t kr = vm_allocate_kernel(bsd_pageable_map, (vm_offset_t *)execarg_address, BSD_PAGEABLE_SIZE_PER_EXEC, VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE, VM_KERN_MEMORY_NONE); assert(kr == KERN_SUCCESS); return kr; } @@ -5302,6 +5448,24 @@ exec_resettextvp(proc_t p, struct image_params *imgp) } +// Includes the 0-byte (therefore "SIZE" instead of "LEN"). +static const size_t CS_CDHASH_STRING_SIZE = CS_CDHASH_LEN * 2 + 1; + +static void cdhash_to_string(char str[CS_CDHASH_STRING_SIZE], uint8_t const * const cdhash) { + static char const nibble[] = "0123456789abcdef"; + + /* Apparently still the safest way to get a hex representation + * of binary data. + * xnu's printf routines have %*D/%20D in theory, but "not really", see: + * confusion around %*D/%nD in printf + */ + for (int i = 0; i < CS_CDHASH_LEN; ++i) { + str[i*2] = nibble[(cdhash[i] & 0xf0) >> 4]; + str[i*2+1] = nibble[cdhash[i] & 0x0f]; + } + str[CS_CDHASH_STRING_SIZE - 1] = 0; +} + /* * If the process is not signed or if it contains entitlements, we * need to communicate through the task_access_port to taskgated. @@ -5320,7 +5484,11 @@ taskgated_required(proc_t p, boolean_t *require_success) if (cs_debug > 2) csvnode_print_debug(p->p_textvp); +#if !CONFIG_EMBEDDED const int can_skip_taskgated = csproc_get_platform_binary(p) && !csproc_get_platform_path(p); +#else + const int can_skip_taskgated = csproc_get_platform_binary(p); +#endif if (can_skip_taskgated) { if (cs_debug) printf("taskgated not required for: %s\n", p->p_name); *require_success = FALSE; @@ -5334,6 +5502,7 @@ taskgated_required(proc_t p, boolean_t *require_success) error = cs_entitlements_blob_get(p, &blob, &length); if (error == 0 && blob != NULL) { +#if !CONFIG_EMBEDDED /* * fatal on the desktop when entitlements are present, * unless we started in single-user mode @@ -5348,6 +5517,7 @@ taskgated_required(proc_t p, boolean_t *require_success) return FALSE; } +#endif if (cs_debug) printf("taskgated required for: %s\n", p->p_name); return TRUE; @@ -5377,7 +5547,7 @@ check_for_signature(proc_t p, struct image_params *imgp) kern_return_t kr = KERN_FAILURE; int error = EACCES; boolean_t unexpected_failure = FALSE; - unsigned char hash[SHA1_RESULTLEN]; + unsigned char hash[CS_CDHASH_LEN]; boolean_t require_success = FALSE; int spawn = (imgp->ip_flags & IMGPF_SPAWN); int vfexec = (imgp->ip_flags & IMGPF_VFORK_EXEC); @@ -5396,7 +5566,7 @@ check_for_signature(proc_t p, struct image_params *imgp) if(p->p_csflags & (CS_HARD|CS_KILL)) { vm_map_switch_protect(get_task_map(p->task), TRUE); } - + /* * image activation may be failed due to policy * which is unexpected but security framework does not @@ -5481,7 +5651,20 @@ check_for_signature(proc_t p, struct image_params *imgp) } done: - if (0 != error) { + if (0 == error) { + /* The process's code signature related properties are + * fully set up, so this is an opportune moment to log + * platform binary execution, if desired. */ + if (platform_exec_logging != 0 && csproc_get_platform_binary(p)) { + uint8_t cdhash[CS_CDHASH_LEN]; + char cdhash_string[CS_CDHASH_STRING_SIZE]; + proc_getcdhash(p, cdhash); + cdhash_to_string(cdhash_string, cdhash); + + os_log(peLog, "CS Platform Exec Logging: Executing platform signed binary " + "'%s' with cdhash %s\n", p->p_name, cdhash_string); + } + } else { if (!unexpected_failure) p->p_csflags |= CS_KILLED; /* make very sure execution fails */ @@ -5525,7 +5708,7 @@ static void exec_prefault_data(proc_t p __unused, struct image_params *imgp, loa vm_map_trunc_page(load_result->entry_point, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_EXECUTE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); if (imgp->ip_flags & IMGPF_IS_64BIT) { @@ -5551,7 +5734,7 @@ static void exec_prefault_data(proc_t p __unused, struct image_params *imgp, loa vm_map_trunc_page(load_result->all_image_info_addr, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_WRITE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); if ((load_result->all_image_info_addr & PAGE_MASK) + expected_all_image_infos_size > PAGE_SIZE) { /* all_image_infos straddles a page */ @@ -5559,14 +5742,14 @@ static void exec_prefault_data(proc_t p __unused, struct image_params *imgp, loa vm_map_trunc_page(load_result->all_image_info_addr + expected_all_image_infos_size - 1, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_WRITE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); } ret = copyin(load_result->all_image_info_addr, &all_image_infos, expected_all_image_infos_size); - if (ret == 0 && all_image_infos.infos32.version >= 9) { + if (ret == 0 && all_image_infos.infos32.version >= DYLD_ALL_IMAGE_INFOS_ADDRESS_MINIMUM_VERSION) { user_addr_t notification_address; user_addr_t dyld_image_address; @@ -5615,25 +5798,25 @@ static void exec_prefault_data(proc_t p __unused, struct image_params *imgp, loa vm_map_trunc_page(notification_address + dyld_slide_amount, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_EXECUTE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); vm_fault(current_map(), vm_map_trunc_page(dyld_image_address + dyld_slide_amount, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_EXECUTE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); vm_fault(current_map(), vm_map_trunc_page(dyld_version_address + dyld_slide_amount, vm_map_page_mask(current_map())), VM_PROT_READ, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); vm_fault(current_map(), vm_map_trunc_page(dyld_all_image_infos_address + dyld_slide_amount, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_WRITE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); } } diff --git a/bsd/kern/kern_exit.c b/bsd/kern/kern_exit.c index 4442c7ff4..b8f5def79 100644 --- a/bsd/kern/kern_exit.c +++ b/bsd/kern/kern_exit.c @@ -74,6 +74,7 @@ #include #include +#include #include "compat_43.h" @@ -102,21 +103,20 @@ #include #include #include -#include /* fdfree */ -#if SYSV_SHM -#include /* shmexit */ -#endif -#include /* acct_process */ -#if CONFIG_PERSONAS -#include -#endif +#include /* fdfree */ +#include /* acct_process */ +#include +#include /* kevent_proc_copy_uptrs */ +#include #include #include #include -#include +#include +#include +#include #include #include #include @@ -126,53 +126,53 @@ #include #include #include +#include -#include +#include + +#include +#if SYSV_SHM +#include /* shmexit */ +#endif /* SYSV_SHM */ +#if CONFIG_PERSONAS +#include +#endif /* CONFIG_PERSONAS */ #if CONFIG_MEMORYSTATUS #include -#endif - +#endif /* CONFIG_MEMORYSTATUS */ #if CONFIG_DTRACE /* Do not include dtrace.h, it redefines kmem_[alloc/free] */ void dtrace_proc_exit(proc_t p); - #include -#endif - +#endif /* CONFIG_DTRACE */ #if CONFIG_MACF -#include +#include #include #include -#endif - -#include -#include -#include - -#include - -#include +#endif /* CONFIG_MACF */ void proc_prepareexit(proc_t p, int rv, boolean_t perf_notify); -void gather_populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, mach_exception_data_type_t code, mach_exception_data_type_t subcode, uint64_t *udata_buffer, int num_udata); +void gather_populate_corpse_crashinfo(proc_t p, task_t corpse_task, + mach_exception_data_type_t code, mach_exception_data_type_t subcode, + uint64_t *udata_buffer, int num_udata, void *reason); mach_exception_data_type_t proc_encode_exit_exception_code(proc_t p); void vfork_exit(proc_t p, int rv); -void vproc_exit(proc_t p); __private_extern__ void munge_user64_rusage(struct rusage *a_rusage_p, struct user64_rusage *a_user_rusage_p); __private_extern__ void munge_user32_rusage(struct rusage *a_rusage_p, struct user32_rusage *a_user_rusage_p); static int reap_child_locked(proc_t parent, proc_t child, int deadparent, int reparentedtoinit, int locked, int droplock); -static void populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, struct rusage_superset *rup, mach_exception_data_type_t code, mach_exception_data_type_t subcode, uint64_t *udata_buffer, int num_udata); +static void populate_corpse_crashinfo(proc_t p, task_t corpse_task, + struct rusage_superset *rup, mach_exception_data_type_t code, + mach_exception_data_type_t subcode, uint64_t *udata_buffer, + int num_udata, os_reason_t reason); static void proc_update_corpse_exception_codes(proc_t p, mach_exception_data_type_t *code, mach_exception_data_type_t *subcode); extern int proc_pidpathinfo_internal(proc_t p, uint64_t arg, char *buffer, uint32_t buffersize, int32_t *retval); -static void abort_with_payload_internal(proc_t p, uint32_t reason_namespace, uint64_t reason_code, user_addr_t payload, - uint32_t payload_size, user_addr_t reason_string, uint64_t reason_flags); - static __attribute__((noinline)) void launchd_crashed_panic(proc_t p, int rv); extern void proc_piduniqidentifierinfo(proc_t p, struct proc_uniqidentifierinfo *p_uniqidinfo); extern void task_coalition_ids(task_t task, uint64_t ids[COALITION_NUM_TYPES]); extern uint64_t get_task_phys_footprint_limit(task_t); int proc_list_uptrs(void *p, uint64_t *udata_buffer, int size); +extern uint64_t task_corpse_get_crashed_thread_id(task_t corpse_task); /* @@ -184,6 +184,7 @@ int waitidcontinue(int result); kern_return_t sys_perf_notify(thread_t thread, int pid); kern_return_t task_exception_notify(exception_type_t exception, mach_exception_data_type_t code, mach_exception_data_type_t subcode); +kern_return_t task_violated_guard(mach_exception_code_t, mach_exception_subcode_t, void *); void delay(int); void gather_rusage_info(proc_t p, rusage_info_current *ru, int flavor); @@ -226,7 +227,7 @@ copyoutsiginfo(user_siginfo_t *native, boolean_t is64, user_addr_t uaddr) { if (is64) { user64_siginfo_t sinfo64; - + bzero(&sinfo64, sizeof (sinfo64)); siginfo_user_to_user64(native, &sinfo64); return (copyout(&sinfo64, uaddr, sizeof (sinfo64))); @@ -239,13 +240,16 @@ copyoutsiginfo(user_siginfo_t *native, boolean_t is64, user_addr_t uaddr) } } -void gather_populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, mach_exception_data_type_t code, mach_exception_data_type_t subcode, uint64_t *udata_buffer, int num_udata) +void gather_populate_corpse_crashinfo(proc_t p, task_t corpse_task, + mach_exception_data_type_t code, mach_exception_data_type_t subcode, + uint64_t *udata_buffer, int num_udata, void *reason) { struct rusage_superset rup; gather_rusage_info(p, &rup.ri, RUSAGE_INFO_CURRENT); rup.ri.ri_phys_footprint = 0; - populate_corpse_crashinfo(p, crash_info_ptr, &rup, code, subcode, udata_buffer, num_udata); + populate_corpse_crashinfo(p, corpse_task, &rup, code, subcode, + udata_buffer, num_udata, reason); } static void proc_update_corpse_exception_codes(proc_t p, mach_exception_data_type_t *code, mach_exception_data_type_t *subcode) @@ -291,7 +295,10 @@ mach_exception_data_type_t proc_encode_exit_exception_code(proc_t p) return (mach_exception_data_type_t)subcode; } -static void populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, struct rusage_superset *rup, mach_exception_data_type_t code, mach_exception_data_type_t subcode, uint64_t *udata_buffer, int num_udata) +static void +populate_corpse_crashinfo(proc_t p, task_t corpse_task, struct rusage_superset *rup, + mach_exception_data_type_t code, mach_exception_data_type_t subcode, + uint64_t *udata_buffer, int num_udata, os_reason_t reason) { mach_vm_address_t uaddr = 0; mach_exception_data_type_t exc_codes[EXCEPTION_CODE_MAX]; @@ -301,10 +308,11 @@ static void populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, struct rus struct proc_uniqidentifierinfo p_uniqidinfo; struct proc_workqueueinfo pwqinfo; int retval = 0; - uint64_t crashed_threadid = thread_tid(current_thread()); + uint64_t crashed_threadid = task_corpse_get_crashed_thread_id(corpse_task); unsigned int pflags = 0; uint64_t max_footprint_mb; uint64_t max_footprint; + void *crash_info_ptr = task_get_corpseinfo(corpse_task); #if CONFIG_MEMORYSTATUS int memstat_dirty_flags = 0; @@ -322,8 +330,11 @@ static void populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, struct rus kcdata_memcpy(crash_info_ptr, uaddr, &p->p_ppid, sizeof(p->p_ppid)); } - if (KERN_SUCCESS == kcdata_get_memory_addr(crash_info_ptr, TASK_CRASHINFO_CRASHED_THREADID, sizeof(uint64_t), &uaddr)) { - kcdata_memcpy(crash_info_ptr, uaddr, &crashed_threadid, sizeof(uint64_t)); + /* Don't include the crashed thread ID if there's an exit reason that indicates it's irrelevant */ + if ((p->p_exit_reason == OS_REASON_NULL) || !(p->p_exit_reason->osr_flags & OS_REASON_FLAG_NO_CRASHED_TID)) { + if (KERN_SUCCESS == kcdata_get_memory_addr(crash_info_ptr, TASK_CRASHINFO_CRASHED_THREADID, sizeof(uint64_t), &uaddr)) { + kcdata_memcpy(crash_info_ptr, uaddr, &crashed_threadid, sizeof(uint64_t)); + } } if (KERN_SUCCESS == @@ -427,23 +438,26 @@ static void populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, struct rus } #endif - if (p->p_exit_reason != OS_REASON_NULL) { + if (p->p_exit_reason != OS_REASON_NULL && reason == OS_REASON_NULL) { + reason = p->p_exit_reason; + } + if (reason != OS_REASON_NULL) { if (KERN_SUCCESS == kcdata_get_memory_addr(crash_info_ptr, EXIT_REASON_SNAPSHOT, sizeof(struct exit_reason_snapshot), &uaddr)) { struct exit_reason_snapshot ers = { - .ers_namespace = p->p_exit_reason->osr_namespace, - .ers_code = p->p_exit_reason->osr_code, - .ers_flags = p->p_exit_reason->osr_flags + .ers_namespace = reason->osr_namespace, + .ers_code = reason->osr_code, + .ers_flags = reason->osr_flags }; kcdata_memcpy(crash_info_ptr, uaddr, &ers, sizeof(ers)); } - if (p->p_exit_reason->osr_kcd_buf != 0) { - uint32_t reason_buf_size = kcdata_memory_get_used_bytes(&p->p_exit_reason->osr_kcd_descriptor); + if (reason->osr_kcd_buf != 0) { + uint32_t reason_buf_size = kcdata_memory_get_used_bytes(&reason->osr_kcd_descriptor); assert(reason_buf_size != 0); if (KERN_SUCCESS == kcdata_get_memory_addr(crash_info_ptr, KCDATA_TYPE_NESTED_KCDATA, reason_buf_size, &uaddr)) { - kcdata_memcpy(crash_info_ptr, uaddr, p->p_exit_reason->osr_kcd_buf, reason_buf_size); + kcdata_memcpy(crash_info_ptr, uaddr, reason->osr_kcd_buf, reason_buf_size); } } } @@ -510,6 +524,14 @@ launchd_crashed_panic(proc_t p, int rv) launchd_exit_reason_desc : "none"); } + const char *launchd_crashed_prefix_str; + + if (strnstr(p->p_name, "preinit", sizeof(p->p_name))) { + launchd_crashed_prefix_str = "LTE preinit process exited"; + } else { + launchd_crashed_prefix_str = "initproc exited"; + } + #if (DEVELOPMENT || DEBUG) && CONFIG_COREDUMP /* * For debugging purposes, generate a core file of initproc before @@ -525,6 +547,7 @@ launchd_crashed_panic(proc_t p, int rv) clock_usec_t tv_usec; uint32_t tv_msec; + err = coredump(p, 300, COREDUMP_IGNORE_ULIMIT | COREDUMP_FULLFSYNC); coredump_end = mach_absolute_time(); @@ -545,48 +568,107 @@ launchd_crashed_panic(proc_t p, int rv) sync(p, (void *)NULL, (int *)NULL); if (p->p_exit_reason == OS_REASON_NULL) { - panic_plain(LAUNCHD_CRASHED_PREFIX " -- no exit reason available -- (signal %d, exit status %d %s)", - WTERMSIG(rv), WEXITSTATUS(rv), ((p->p_csflags & CS_KILLED) ? "CS_KILLED" : "")); + panic_with_options(0, NULL, DEBUGGER_OPTION_INITPROC_PANIC, "%s -- no exit reason available -- (signal %d, exit status %d %s)", + launchd_crashed_prefix_str, WTERMSIG(rv), WEXITSTATUS(rv), ((p->p_csflags & CS_KILLED) ? "CS_KILLED" : "")); } else { - panic_plain(LAUNCHD_CRASHED_PREFIX " %s -- exit reason namespace %d subcode 0x%llx description: %." LAUNCHD_PANIC_REASON_STRING_MAXLEN "s", + panic_with_options(0, NULL, DEBUGGER_OPTION_INITPROC_PANIC, "%s %s -- exit reason namespace %d subcode 0x%llx description: %." LAUNCHD_PANIC_REASON_STRING_MAXLEN "s", ((p->p_csflags & CS_KILLED) ? "CS_KILLED" : ""), - p->p_exit_reason->osr_namespace, p->p_exit_reason->osr_code, + launchd_crashed_prefix_str, p->p_exit_reason->osr_namespace, p->p_exit_reason->osr_code, launchd_exit_reason_desc ? launchd_exit_reason_desc : "none"); } } -static void -abort_with_payload_internal(proc_t p, uint32_t reason_namespace, uint64_t reason_code, user_addr_t payload, uint32_t payload_size, - user_addr_t reason_string, uint64_t reason_flags) +#define OS_REASON_IFLAG_USER_FAULT 0x1 + +#define OS_REASON_TOTAL_USER_FAULTS_PER_PROC 5 + +static int +abort_with_payload_internal(proc_t p, + uint32_t reason_namespace, uint64_t reason_code, + user_addr_t payload, uint32_t payload_size, + user_addr_t reason_string, uint64_t reason_flags, + uint32_t internal_flags) { os_reason_t exit_reason = OS_REASON_NULL; + kern_return_t kr = KERN_SUCCESS; + + if (internal_flags & OS_REASON_IFLAG_USER_FAULT) { + uint32_t old_value = atomic_load_explicit(&p->p_user_faults, + memory_order_relaxed); + for (;;) { + if (old_value >= OS_REASON_TOTAL_USER_FAULTS_PER_PROC) { + return EQFULL; + } + // this reloads the value in old_value + if (atomic_compare_exchange_strong_explicit(&p->p_user_faults, + &old_value, old_value + 1, memory_order_relaxed, + memory_order_relaxed)) { + break; + } + } + } KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXITREASON_CREATE) | DBG_FUNC_NONE, p->p_pid, reason_namespace, reason_code, 0, 0); - exit_reason = build_userspace_exit_reason(reason_namespace, reason_code, payload, payload_size, reason_string, - reason_flags); + exit_reason = build_userspace_exit_reason(reason_namespace, reason_code, + payload, payload_size, reason_string, reason_flags); - /* - * We use SIGABRT (rather than calling exit directly from here) so that - * the debugger can catch abort_with_{reason,payload} calls. - */ - psignal_try_thread_with_reason(p, current_thread(), SIGABRT, exit_reason); + if (internal_flags & OS_REASON_IFLAG_USER_FAULT) { + mach_exception_code_t code = 0; - return; + EXC_GUARD_ENCODE_TYPE(code, GUARD_TYPE_USER); /* simulated EXC_GUARD */ + EXC_GUARD_ENCODE_FLAVOR(code, 0); + EXC_GUARD_ENCODE_TARGET(code, reason_namespace); + + if (exit_reason == OS_REASON_NULL) { + kr = KERN_RESOURCE_SHORTAGE; + } else { + kr = task_violated_guard(code, reason_code, exit_reason); + } + os_reason_free(exit_reason); + } else { + /* + * We use SIGABRT (rather than calling exit directly from here) so that + * the debugger can catch abort_with_{reason,payload} calls. + */ + psignal_try_thread_with_reason(p, current_thread(), SIGABRT, exit_reason); + } + + switch (kr) { + case KERN_SUCCESS: + return 0; + case KERN_NOT_SUPPORTED: + return ENOTSUP; + case KERN_INVALID_ARGUMENT: + return EINVAL; + case KERN_RESOURCE_SHORTAGE: + default: + return EBUSY; + } } int abort_with_payload(struct proc *cur_proc, struct abort_with_payload_args *args, __unused void *retval) { - abort_with_payload_internal(cur_proc, args->reason_namespace, args->reason_code, args->payload, args->payload_size, - args->reason_string, args->reason_flags); + abort_with_payload_internal(cur_proc, args->reason_namespace, + args->reason_code, args->payload, args->payload_size, + args->reason_string, args->reason_flags, 0); return 0; } +int +os_fault_with_payload(struct proc *cur_proc, + struct os_fault_with_payload_args *args, __unused int *retval) +{ + return abort_with_payload_internal(cur_proc, args->reason_namespace, + args->reason_code, args->payload, args->payload_size, + args->reason_string, args->reason_flags, OS_REASON_IFLAG_USER_FAULT); +} + /* * exit -- @@ -596,6 +678,7 @@ __attribute__((noreturn)) void exit(proc_t p, struct exit_args *uap, int *retval) { + p->p_xhighbits = ((uint32_t)(uap->rval) & 0xFF000000) >> 24; exit1(p, W_EXITCODE(uap->rval, 0), retval); thread_exception_return(); @@ -858,11 +941,11 @@ skipcheck: int buf_size = 0; /* Get all the udata pointers from kqueue */ - est_knotes = proc_list_uptrs(p, NULL, 0); + est_knotes = kevent_proc_copy_uptrs(p, NULL, 0); if (est_knotes > 0) { buf_size = (est_knotes + 32) * sizeof(uint64_t); buffer = (uint64_t *) kalloc(buf_size); - num_knotes = proc_list_uptrs(p, buffer, buf_size); + num_knotes = kevent_proc_copy_uptrs(p, buffer, buf_size); if (num_knotes > est_knotes + 32) { num_knotes = est_knotes + 32; } @@ -870,7 +953,8 @@ skipcheck: /* Update the code, subcode based on exit reason */ proc_update_corpse_exception_codes(p, &code, &subcode); - populate_corpse_crashinfo(p, task_get_corpseinfo(p->task), rup, code, subcode, buffer, num_knotes); + populate_corpse_crashinfo(p, p->task, rup, + code, subcode, buffer, num_knotes, NULL); if (buffer != NULL) { kfree(buffer, buf_size); } @@ -981,9 +1065,6 @@ proc_exit(proc_t p) task_clear_cpuusage(p->task, TRUE); workqueue_mark_exiting(p); - workqueue_exit(p); - kqueue_dealloc(p->p_wqkqueue); - p->p_wqkqueue = NULL; _aio_exit( p ); @@ -993,6 +1074,12 @@ proc_exit(proc_t p) */ fdfree(p); + /* + * Once all the knotes, kqueues & workloops are destroyed, get rid of the + * workqueue. + */ + workqueue_exit(p); + if (uth->uu_lowpri_window) { /* * task is marked as a low priority I/O type @@ -1213,6 +1300,15 @@ proc_exit(proc_t p) proc_childdrainend(p); proc_list_unlock(); +#if CONFIG_MACF + /* + * Notify MAC policies that proc is dead. + * This should be replaced with proper label management + * (rdar://problem/32126399). + */ + mac_proc_notify_exit(p); +#endif + /* * Release reference to text vnode */ @@ -1318,6 +1414,8 @@ proc_exit(proc_t p) if (pp != initproc) { proc_lock(pp); pp->si_pid = p->p_pid; + pp->p_xhighbits = p->p_xhighbits; + p->p_xhighbits = 0; pp->si_status = p->p_xstat; pp->si_code = CLD_EXITED; /* @@ -1576,16 +1674,16 @@ reap_child_locked(proc_t parent, proc_t child, int deadparent, int reparentedtoi #if CONFIG_FINE_LOCK_GROUPS lck_mtx_destroy(&child->p_mlock, proc_mlock_grp); - lck_mtx_destroy(&child->p_fdmlock, proc_fdmlock_grp); lck_mtx_destroy(&child->p_ucred_mlock, proc_ucred_mlock_grp); + lck_mtx_destroy(&child->p_fdmlock, proc_fdmlock_grp); #if CONFIG_DTRACE lck_mtx_destroy(&child->p_dtrace_sprlock, proc_lck_grp); #endif lck_spin_destroy(&child->p_slock, proc_slock_grp); #else /* CONFIG_FINE_LOCK_GROUPS */ lck_mtx_destroy(&child->p_mlock, proc_lck_grp); - lck_mtx_destroy(&child->p_fdmlock, proc_lck_grp); lck_mtx_destroy(&child->p_ucred_mlock, proc_lck_grp); + lck_mtx_destroy(&child->p_fdmlock, proc_lck_grp); #if CONFIG_DTRACE lck_mtx_destroy(&child->p_dtrace_sprlock, proc_lck_grp); #endif @@ -1937,7 +2035,8 @@ loop1: #endif siginfo.si_signo = SIGCHLD; siginfo.si_pid = p->p_pid; - siginfo.si_status = WEXITSTATUS(p->p_xstat); + siginfo.si_status = (WEXITSTATUS(p->p_xstat) & 0x00FFFFFF) | (((uint32_t)(p->p_xhighbits) << 24) & 0xFF000000); + p->p_xhighbits = 0; if (WIFSIGNALED(p->p_xstat)) { siginfo.si_code = WCOREDUMP(p->p_xstat) ? CLD_DUMPED : CLD_KILLED; @@ -2201,23 +2300,15 @@ vfork_exit_internal(proc_t p, int rv, int forceexit) p->p_sigignore = ~0; proc_unlock(p); - proc_free_realitimer(p); - ut->uu_siglist = 0; - vproc_exit(p); -} + /* begin vproc_exit */ -void -vproc_exit(proc_t p) -{ proc_t q; proc_t pp; - + vnode_t tvp; -#ifdef FIXME - struct task *task = p->task; -#endif + struct pgrp * pg; struct session *sessp; struct rusage_superset *rup; @@ -2488,6 +2579,8 @@ vproc_exit(proc_t p) pth_proc_hashdelete(p); #endif /* PSYNCH */ + proc_free_realitimer(p); + /* * Other substructures are freed from wait(). */ @@ -2515,6 +2608,8 @@ vproc_exit(proc_t p) if (pp != initproc) { proc_lock(pp); pp->si_pid = p->p_pid; + pp->p_xhighbits = p->p_xhighbits; + p->p_xhighbits = 0; pp->si_status = p->p_xstat; pp->si_code = CLD_EXITED; /* diff --git a/bsd/kern/kern_fork.c b/bsd/kern/kern_fork.c index a42891ae7..952b6f8fb 100644 --- a/bsd/kern/kern_fork.c +++ b/bsd/kern/kern_fork.c @@ -124,10 +124,12 @@ static void (*dtrace_proc_waitfor_hook)(proc_t) = NULL; #include #include -#include +#include + +#include #if CONFIG_MACF -#include +#include #include #endif @@ -385,6 +387,14 @@ fork1(proc_t parent_proc, thread_t *child_threadp, int kind, coalition_t *coalit uid = kauth_getruid(); proc_list_lock(); if ((nprocs >= maxproc - 1 && uid != 0) || nprocs >= maxproc) { +#if (DEVELOPMENT || DEBUG) && CONFIG_EMBEDDED + /* + * On the development kernel, panic so that the fact that we hit + * the process limit is obvious, as this may very well wedge the + * system. + */ + panic("The process table is full; parent pid=%d", parent_proc->p_pid); +#endif proc_list_unlock(); tablefull("proc"); return (EAGAIN); @@ -400,6 +410,15 @@ fork1(proc_t parent_proc, thread_t *child_threadp, int kind, coalition_t *coalit count = chgproccnt(uid, 1); if (uid != 0 && (rlim_t)count > parent_proc->p_rlimit[RLIMIT_NPROC].rlim_cur) { +#if (DEVELOPMENT || DEBUG) && CONFIG_EMBEDDED + /* + * On the development kernel, panic so that the fact that we hit + * the per user process limit is obvious. This may be less dire + * than hitting the global process limit, but we cannot rely on + * that. + */ + panic("The per-user process limit has been hit; parent pid=%d, uid=%d", parent_proc->p_pid, uid); +#endif err = EAGAIN; goto bad; } @@ -1210,6 +1229,13 @@ retry: LIST_INSERT_HEAD(PIDHASH(child_proc->p_pid), child_proc, p_hash); proc_list_unlock(); + if (child_proc->p_uniqueid == startup_serial_num_procs) { + /* + * Turn off startup serial logging now that we have reached + * the defined number of startup processes. + */ + startup_serial_logging_active = false; + } /* * We've identified the PID we are going to use; initialize the new @@ -1232,7 +1258,11 @@ retry: * Increase reference counts on shared objects. * The p_stats and p_sigacts substructs are set in vm_fork. */ +#if !CONFIG_EMBEDDED child_proc->p_flag = (parent_proc->p_flag & (P_LP64 | P_DISABLE_ASLR | P_DELAYIDLESLEEP | P_SUGID)); +#else /* !CONFIG_EMBEDDED */ + child_proc->p_flag = (parent_proc->p_flag & (P_LP64 | P_DISABLE_ASLR | P_SUGID)); +#endif /* !CONFIG_EMBEDDED */ if (parent_proc->p_flag & P_PROFIL) startprofclock(child_proc); @@ -1354,9 +1384,11 @@ retry: if ((parent_proc->p_lflag & P_LREGISTER) != 0) { child_proc->p_lflag |= P_LREGISTER; } - child_proc->p_wqkqueue = NULL; child_proc->p_dispatchqueue_offset = parent_proc->p_dispatchqueue_offset; child_proc->p_dispatchqueue_serialno_offset = parent_proc->p_dispatchqueue_serialno_offset; + child_proc->p_return_to_kernel_offset = parent_proc->p_return_to_kernel_offset; + child_proc->p_mach_thread_self_offset = parent_proc->p_mach_thread_self_offset; + child_proc->p_pth_tsd_offset = parent_proc->p_pth_tsd_offset; #if PSYNCH pth_proc_hashinit(child_proc); #endif /* PSYNCH */ @@ -1397,7 +1429,7 @@ bad: void proc_lock(proc_t p) { - lck_mtx_assert(proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(&p->p_mlock); } @@ -1615,12 +1647,11 @@ uthread_cleanup(task_t task, void *uthread, void * bsd_info) assert(uth->uu_ar == NULL); if (uth->uu_kqueue_bound) { - kevent_qos_internal_unbind(p, - uth->uu_kqueue_bound, + kevent_qos_internal_unbind(p, + 0, /* didn't save qos_class */ uth->uu_thread, uth->uu_kqueue_flags); - uth->uu_kqueue_flags = 0; - uth->uu_kqueue_bound = 0; + assert(uth->uu_kqueue_override_is_sync == 0); } sel = &uth->uu_select; diff --git a/bsd/kern/kern_guarded.c b/bsd/kern/kern_guarded.c index a24987fc0..ea583e9cf 100644 --- a/bsd/kern/kern_guarded.c +++ b/bsd/kern/kern_guarded.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,14 @@ #include #include #include +#include +#if CONFIG_MACF && CONFIG_VNGUARD +#include +#include +#include +#include +#include +#endif #define f_flag f_fglob->fg_flag @@ -60,6 +69,7 @@ extern int wr_uio(struct proc *p, struct fileproc *fp, uio_t uio, user_ssize_t * kern_return_t task_exception_notify(exception_type_t exception, mach_exception_data_type_t code, mach_exception_data_type_t subcode); +kern_return_t task_violated_guard(mach_exception_code_t, mach_exception_subcode_t, void *); /* * Most fd's have an underlying fileproc struct; but some may be @@ -78,10 +88,7 @@ struct guarded_fileproc { struct fileproc gf_fileproc; u_int gf_magic; u_int gf_attrs; - thread_t gf_thread; guardid_t gf_guard; - int gf_exc_fd; - u_int gf_exc_code; }; const size_t sizeof_guarded_fileproc = sizeof (struct guarded_fileproc); @@ -180,48 +187,23 @@ fp_isguarded(struct fileproc *fp, u_int attrs) extern char *proc_name_address(void *p); int -fp_guard_exception(proc_t p, int fd, struct fileproc *fp, u_int code) +fp_guard_exception(proc_t p, int fd, struct fileproc *fp, u_int flavor) { if (FILEPROC_TYPE(fp) != FTYPE_GUARDED) panic("%s corrupt fp %p flags %x", __func__, fp, fp->f_flags); struct guarded_fileproc *gfp = FP_TO_GFP(fp); - /* all gfd fields protected via proc_fdlock() */ proc_fdlock_assert(p, LCK_MTX_ASSERT_OWNED); - if (NULL == gfp->gf_thread) { - thread_t t = current_thread(); - gfp->gf_thread = t; - gfp->gf_exc_fd = fd; - gfp->gf_exc_code = code; - - /* - * This thread was the first to attempt the - * operation that violated the guard on this fd; - * generate an exception. - */ - printf("%s: guarded fd exception: " - "fd %d code 0x%x guard 0x%llx\n", - proc_name_address(p), gfp->gf_exc_fd, - gfp->gf_exc_code, gfp->gf_guard); - - thread_guard_violation(t, GUARD_TYPE_FD); - } else { - /* - * We already recorded a violation on this fd for a - * different thread, so posting an exception is - * already in progress. We could pause for a bit - * and check again, or we could panic (though that seems - * heavy handed), or we could just press on with the - * error return alone. For now, resort to printf. - */ - printf("%s: guarded fd exception+: " - "fd %d code 0x%x guard 0x%llx\n", - proc_name_address(p), gfp->gf_exc_fd, - gfp->gf_exc_code, gfp->gf_guard); - } + mach_exception_code_t code = 0; + EXC_GUARD_ENCODE_TYPE(code, GUARD_TYPE_FD); + EXC_GUARD_ENCODE_FLAVOR(code, flavor); + EXC_GUARD_ENCODE_TARGET(code, fd); + mach_exception_subcode_t subcode = gfp->gf_guard; + thread_t t = current_thread(); + thread_guard_violation(t, code, subcode); return (EPERM); } @@ -229,73 +211,14 @@ fp_guard_exception(proc_t p, int fd, struct fileproc *fp, u_int code) * (Invoked before returning to userland from the syscall handler.) */ void -fd_guard_ast(thread_t t) +fd_guard_ast( + thread_t __unused t, + mach_exception_code_t code, + mach_exception_subcode_t subcode) { + task_exception_notify(EXC_GUARD, code, subcode); proc_t p = current_proc(); - struct filedesc *fdp = p->p_fd; - int i; - - proc_fdlock(p); - for (i = fdp->fd_lastfile; i >= 0; i--) { - struct fileproc *fp = fdp->fd_ofiles[i]; - - if (fp == NULL || - FILEPROC_TYPE(fp) != FTYPE_GUARDED) - continue; - - struct guarded_fileproc *gfp = FP_TO_GFP(fp); - - if (GUARDED_FILEPROC_MAGIC != gfp->gf_magic) - panic("%s: corrupt gfp %p flags %x", - __func__, gfp, fp->f_flags); - - if (gfp->gf_thread == t) { - mach_exception_data_type_t code, subcode; - - gfp->gf_thread = NULL; - - /* - * EXC_GUARD exception code namespace. - * - * code: - * +-------------------------------------------------+ - * | [63:61] guard type | [60:0] guard-specific data | - * +-------------------------------------------------+ - * - * subcode: - * +-------------------------------------------------+ - * | [63:0] guard-specific data | - * +-------------------------------------------------+ - * - * At the moment, we have just one guard type: file - * descriptor guards. - * - * File descriptor guards use the exception codes like - * so: - * - * code: - * +--------------------------------------------------+ - * |[63:61] GUARD_TYPE_FD | [60:32] flavor | [31:0] fd| - * +--------------------------------------------------+ - * - * subcode: - * +--------------------------------------------------+ - * | [63:0] guard value | - * +--------------------------------------------------+ - */ - code = (((uint64_t)GUARD_TYPE_FD) << 61) | - (((uint64_t)gfp->gf_exc_code) << 32) | - ((uint64_t)gfp->gf_exc_fd); - subcode = gfp->gf_guard; - proc_fdunlock(p); - - (void) task_exception_notify(EXC_GUARD, code, subcode); - psignal(p, SIGKILL); - - return; - } - } - proc_fdunlock(p); + psignal(p, SIGKILL); } /* @@ -665,6 +588,7 @@ restart: case DTYPE_PIPE: case DTYPE_SOCKET: case DTYPE_KQUEUE: + case DTYPE_NETPOLICY: break; default: error = ENOTSUP; @@ -981,3 +905,627 @@ falloc_guarded(struct proc *p, struct fileproc **fp, int *fd, return (falloc_withalloc(p, fp, fd, ctx, guarded_fileproc_alloc_init, &crarg)); } + +#if CONFIG_MACF && CONFIG_VNGUARD + +/* + * Guarded vnodes + * + * Uses MAC hooks to guard operations on vnodes in the system. Given an fd, + * add data to the label on the fileglob and the vnode it points at. + * The data contains a pointer to the fileglob, the set of attributes to + * guard, a guard value for uniquification, and the pid of the process + * who set the guard up in the first place. + * + * The fd must have been opened read/write, and the underlying + * fileglob is FG_CONFINED so that there's no ambiguity about the + * owning process. + * + * When there's a callback for a vnode operation of interest (rename, unlink, + * etc.) check to see if the guard permits that operation, and if not + * take an action e.g. log a message or generate a crash report. + * + * The label is removed from the vnode and the fileglob when the fileglob + * is closed. + * + * The initial action to be taken can be specified by a boot arg (vnguard=0x42) + * and change via the "kern.vnguard.flags" sysctl. + */ + +struct vng_owner; + +struct vng_info { /* lives on the vnode label */ + guardid_t vgi_guard; + unsigned vgi_attrs; + TAILQ_HEAD(, vng_owner) vgi_owners; +}; + +struct vng_owner { /* lives on the fileglob label */ + proc_t vgo_p; + struct fileglob *vgo_fg; + struct vng_info *vgo_vgi; + TAILQ_ENTRY(vng_owner) vgo_link; +}; + +static struct vng_info * +new_vgi(unsigned attrs, guardid_t guard) +{ + struct vng_info *vgi = kalloc(sizeof (*vgi)); + vgi->vgi_guard = guard; + vgi->vgi_attrs = attrs; + TAILQ_INIT(&vgi->vgi_owners); + return vgi; +} + +static struct vng_owner * +new_vgo(proc_t p, struct fileglob *fg) +{ + struct vng_owner *vgo = kalloc(sizeof (*vgo)); + memset(vgo, 0, sizeof (*vgo)); + vgo->vgo_p = p; + vgo->vgo_fg = fg; + return vgo; +} + +static void +vgi_add_vgo(struct vng_info *vgi, struct vng_owner *vgo) +{ + vgo->vgo_vgi = vgi; + TAILQ_INSERT_HEAD(&vgi->vgi_owners, vgo, vgo_link); +} + +static boolean_t +vgi_remove_vgo(struct vng_info *vgi, struct vng_owner *vgo) +{ + TAILQ_REMOVE(&vgi->vgi_owners, vgo, vgo_link); + vgo->vgo_vgi = NULL; + return TAILQ_EMPTY(&vgi->vgi_owners); +} + +static void +free_vgi(struct vng_info *vgi) +{ + assert(TAILQ_EMPTY(&vgi->vgi_owners)); +#if DEVELOP || DEBUG + memset(vgi, 0xbeadfade, sizeof (*vgi)); +#endif + kfree(vgi, sizeof (*vgi)); +} + +static void +free_vgo(struct vng_owner *vgo) +{ +#if DEVELOP || DEBUG + memset(vgo, 0x2bedf1d0, sizeof (*vgo)); +#endif + kfree(vgo, sizeof (*vgo)); +} + +static int label_slot; +static lck_rw_t llock; +static lck_grp_t *llock_grp; + +static __inline void * +vng_lbl_get(struct label *label) +{ + lck_rw_assert(&llock, LCK_RW_ASSERT_HELD); + void *data; + if (NULL == label) + data = NULL; + else + data = (void *)mac_label_get(label, label_slot); + return data; +} + +static __inline struct vng_info * +vng_lbl_get_withattr(struct label *label, unsigned attrmask) +{ + struct vng_info *vgi = vng_lbl_get(label); + assert(NULL == vgi || (vgi->vgi_attrs & ~VNG_ALL) == 0); + if (NULL != vgi && 0 == (vgi->vgi_attrs & attrmask)) + vgi = NULL; + return vgi; +} + +static __inline void +vng_lbl_set(struct label *label, void *data) +{ + assert(NULL != label); + lck_rw_assert(&llock, LCK_RW_ASSERT_EXCLUSIVE); + mac_label_set(label, label_slot, (intptr_t)data); +} + +static int +vnguard_sysc_setguard(proc_t p, const struct vnguard_set *vns) +{ + const int fd = vns->vns_fd; + + if ((vns->vns_attrs & ~VNG_ALL) != 0 || + 0 == vns->vns_attrs || 0 == vns->vns_guard) + return EINVAL; + + int error; + struct fileproc *fp; + if (0 != (error = fp_lookup(p, fd, &fp, 0))) + return error; + do { + /* + * To avoid trivial DoS, insist that the caller + * has read/write access to the file. + */ + if ((FREAD|FWRITE) != (fp->f_flag & (FREAD|FWRITE))) { + error = EBADF; + break; + } + struct fileglob *fg = fp->f_fglob; + if (FILEGLOB_DTYPE(fg) != DTYPE_VNODE) { + error = EBADF; + break; + } + /* + * Confinement means there's only one fd pointing at + * this fileglob, and will always be associated with + * this pid. + */ + if (0 == (FG_CONFINED & fg->fg_lflags)) { + error = EBADF; + break; + } + struct vnode *vp = fg->fg_data; + if (!vnode_isreg(vp) || NULL == vp->v_mount) { + error = EBADF; + break; + } + error = vnode_getwithref(vp); + if (0 != error) { + fp_drop(p, fd, fp, 0); + break; + } + /* Ensure the target vnode -has- a label */ + struct vfs_context *ctx = vfs_context_current(); + mac_vnode_label_update(ctx, vp, NULL); + + struct vng_info *nvgi = new_vgi(vns->vns_attrs, vns->vns_guard); + struct vng_owner *nvgo = new_vgo(p, fg); + + lck_rw_lock_exclusive(&llock); + + do { + /* + * A vnode guard is associated with one or more + * fileglobs in one or more processes. + */ + struct vng_info *vgi = vng_lbl_get(vp->v_label); + struct vng_owner *vgo = vng_lbl_get(fg->fg_label); + + if (NULL == vgi) { + /* vnode unguarded, add the first guard */ + if (NULL != vgo) + panic("vnguard label on fileglob " + "but not vnode"); + /* add a kusecount so we can unlabel later */ + error = vnode_ref_ext(vp, O_EVTONLY, 0); + if (0 == error) { + /* add the guard */ + vgi_add_vgo(nvgi, nvgo); + vng_lbl_set(vp->v_label, nvgi); + vng_lbl_set(fg->fg_label, nvgo); + } else { + free_vgo(nvgo); + free_vgi(nvgi); + } + } else { + /* vnode already guarded */ + free_vgi(nvgi); + if (vgi->vgi_guard != vns->vns_guard) + error = EPERM; /* guard mismatch */ + else if (vgi->vgi_attrs != vns->vns_attrs) + error = EACCES; /* attr mismatch */ + if (0 != error || NULL != vgo) { + free_vgo(nvgo); + break; + } + /* record shared ownership */ + vgi_add_vgo(vgi, nvgo); + vng_lbl_set(fg->fg_label, nvgo); + } + } while (0); + + lck_rw_unlock_exclusive(&llock); + vnode_put(vp); + } while (0); + + fp_drop(p, fd, fp, 0); + return error; +} + +static int +vng_policy_syscall(proc_t p, int cmd, user_addr_t arg) +{ + int error = EINVAL; + + switch (cmd) { + case VNG_SYSC_PING: + if (0 == arg) + error = 0; + break; + case VNG_SYSC_SET_GUARD: { + struct vnguard_set vns; + error = copyin(arg, (void *)&vns, sizeof (vns)); + if (error) + break; + error = vnguard_sysc_setguard(p, &vns); + break; + } + default: + break; + } + return (error); +} + +/* + * This is called just before the fileglob disappears in fg_free(). + * Take the exclusive lock: no other thread can add or remove + * a vng_info to any vnode in the system. + */ +static void +vng_file_label_destroy(struct label *label) +{ + lck_rw_lock_exclusive(&llock); + struct vng_owner *lvgo = vng_lbl_get(label); + if (lvgo) { + vng_lbl_set(label, 0); + struct vng_info *vgi = lvgo->vgo_vgi; + assert(vgi); + if (vgi_remove_vgo(vgi, lvgo)) { + /* that was the last reference */ + vgi->vgi_attrs = 0; + struct fileglob *fg = lvgo->vgo_fg; + assert(fg); + if (DTYPE_VNODE == FILEGLOB_DTYPE(fg)) { + struct vnode *vp = fg->fg_data; + int error = vnode_getwithref(vp); + if (0 == error) { + vng_lbl_set(vp->v_label, 0); + lck_rw_unlock_exclusive(&llock); + /* may trigger VNOP_INACTIVE */ + vnode_rele_ext(vp, O_EVTONLY, 0); + vnode_put(vp); + free_vgi(vgi); + free_vgo(lvgo); + return; + } + } + } + free_vgo(lvgo); + } + lck_rw_unlock_exclusive(&llock); +} + +static int vng_policy_flags; + +static int +vng_guard_violation(const struct vng_info *vgi, + unsigned opval, const char *nm) +{ + int retval = 0; + + if (vng_policy_flags & kVNG_POLICY_EPERM) { + /* deny the operation */ + retval = EPERM; + } + + if (vng_policy_flags & kVNG_POLICY_LOGMSG) { + /* log a message */ + const char *op; + switch (opval) { + case VNG_RENAME_FROM: + op = "rename-from"; + break; + case VNG_RENAME_TO: + op = "rename-to"; + break; + case VNG_UNLINK: + op = "unlink"; + break; + case VNG_LINK: + op = "link"; + break; + case VNG_EXCHDATA: + op = "exchdata"; + break; + case VNG_WRITE_OTHER: + op = "write"; + break; + case VNG_TRUNC_OTHER: + op = "truncate"; + break; + default: + op = "(unknown)"; + break; + } + proc_t p = current_proc(); + const struct vng_owner *vgo; + TAILQ_FOREACH(vgo, &vgi->vgi_owners, vgo_link) { + printf("%s[%d]: %s%s: '%s' guarded by %s[%d] (0x%llx)\n", + proc_name_address(p), proc_pid(p), op, + 0 != retval ? " denied" : "", + NULL != nm ? nm : "(unknown)", + proc_name_address(vgo->vgo_p), proc_pid(vgo->vgo_p), + vgi->vgi_guard); + } + } + + if (vng_policy_flags & (kVNG_POLICY_EXC|kVNG_POLICY_EXC_CORPSE)) { + /* EXC_GUARD exception */ + const struct vng_owner *vgo = TAILQ_FIRST(&vgi->vgi_owners); + pid_t pid = vgo ? proc_pid(vgo->vgo_p) : 0; + mach_exception_code_t code; + mach_exception_subcode_t subcode; + + code = 0; + EXC_GUARD_ENCODE_TYPE(code, GUARD_TYPE_VN); + EXC_GUARD_ENCODE_FLAVOR(code, opval); + EXC_GUARD_ENCODE_TARGET(code, pid); + subcode = vgi->vgi_guard; + + if (vng_policy_flags & kVNG_POLICY_EXC_CORPSE) { + task_violated_guard(code, subcode, NULL); + /* not fatal */ + } else { + thread_t t = current_thread(); + thread_guard_violation(t, code, subcode); + } + } else if (vng_policy_flags & kVNG_POLICY_SIGKILL) { + proc_t p = current_proc(); + psignal(p, SIGKILL); + } + + return retval; +} + +/* + * A vnode guard was tripped on this thread. + * + * (Invoked before returning to userland from the syscall handler.) + */ +void +vn_guard_ast(thread_t __unused t, + mach_exception_data_type_t code, mach_exception_data_type_t subcode) +{ + task_exception_notify(EXC_GUARD, code, subcode); + proc_t p = current_proc(); + psignal(p, SIGKILL); +} + +/* + * vnode callbacks + */ + +static int +vng_vnode_check_rename(kauth_cred_t __unused cred, + struct vnode *__unused dvp, struct label *__unused dlabel, + struct vnode *__unused vp, struct label *label, + struct componentname *cnp, + struct vnode *__unused tdvp, struct label *__unused tdlabel, + struct vnode *__unused tvp, struct label *tlabel, + struct componentname *tcnp) +{ + int error = 0; + if (NULL != label || NULL != tlabel) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(label, VNG_RENAME_FROM); + if (NULL != vgi) + error = vng_guard_violation(vgi, + VNG_RENAME_FROM, cnp->cn_nameptr); + if (0 == error) { + vgi = vng_lbl_get_withattr(tlabel, VNG_RENAME_TO); + if (NULL != vgi) + error = vng_guard_violation(vgi, + VNG_RENAME_TO, tcnp->cn_nameptr); + } + lck_rw_unlock_shared(&llock); + } + return error; +} + +static int +vng_vnode_check_link(kauth_cred_t __unused cred, + struct vnode *__unused dvp, struct label *__unused dlabel, + struct vnode *vp, struct label *label, struct componentname *__unused cnp) +{ + int error = 0; + if (NULL != label) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(label, VNG_LINK); + if (vgi) { + const char *nm = vnode_getname(vp); + error = vng_guard_violation(vgi, VNG_LINK, nm); + if (nm) + vnode_putname(nm); + } + lck_rw_unlock_shared(&llock); + } + return error; +} + +static int +vng_vnode_check_unlink(kauth_cred_t __unused cred, + struct vnode *__unused dvp, struct label *__unused dlabel, + struct vnode *__unused vp, struct label *label, struct componentname *cnp) +{ + int error = 0; + if (NULL != label) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(label, VNG_UNLINK); + if (vgi) + error = vng_guard_violation(vgi, VNG_UNLINK, + cnp->cn_nameptr); + lck_rw_unlock_shared(&llock); + } + return error; +} + +/* + * Only check violations for writes performed by "other processes" + */ +static int +vng_vnode_check_write(kauth_cred_t __unused actv_cred, + kauth_cred_t __unused file_cred, struct vnode *vp, struct label *label) +{ + int error = 0; + if (NULL != label) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(label, VNG_WRITE_OTHER); + if (vgi) { + proc_t p = current_proc(); + const struct vng_owner *vgo; + TAILQ_FOREACH(vgo, &vgi->vgi_owners, vgo_link) { + if (vgo->vgo_p == p) + goto done; + } + const char *nm = vnode_getname(vp); + error = vng_guard_violation(vgi, + VNG_WRITE_OTHER, nm); + if (nm) + vnode_putname(nm); + } + done: + lck_rw_unlock_shared(&llock); + } + return error; +} + +/* + * Only check violations for truncates performed by "other processes" + */ +static int +vng_vnode_check_truncate(kauth_cred_t __unused actv_cred, + kauth_cred_t __unused file_cred, struct vnode *vp, + struct label *label) +{ + int error = 0; + if (NULL != label) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(label, VNG_TRUNC_OTHER); + if (vgi) { + proc_t p = current_proc(); + const struct vng_owner *vgo; + TAILQ_FOREACH(vgo, &vgi->vgi_owners, vgo_link) { + if (vgo->vgo_p == p) + goto done; + } + const char *nm = vnode_getname(vp); + error = vng_guard_violation(vgi, + VNG_TRUNC_OTHER, nm); + if (nm) + vnode_putname(nm); + } + done: + lck_rw_unlock_shared(&llock); + } + return error; +} + +static int +vng_vnode_check_exchangedata(kauth_cred_t __unused cred, + struct vnode *fvp, struct label *flabel, + struct vnode *svp, struct label *slabel) +{ + int error = 0; + if (NULL != flabel || NULL != slabel) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(flabel, VNG_EXCHDATA); + if (NULL != vgi) { + const char *nm = vnode_getname(fvp); + error = vng_guard_violation(vgi, + VNG_EXCHDATA, nm); + if (nm) + vnode_putname(nm); + } + if (0 == error) { + vgi = vng_lbl_get_withattr(slabel, VNG_EXCHDATA); + if (NULL != vgi) { + const char *nm = vnode_getname(svp); + error = vng_guard_violation(vgi, + VNG_EXCHDATA, nm); + if (nm) + vnode_putname(nm); + } + } + lck_rw_unlock_shared(&llock); + } + return error; +} + +/* + * Configuration gorp + */ + +static void +vng_init(struct mac_policy_conf *mpc) +{ + llock_grp = lck_grp_alloc_init(mpc->mpc_name, LCK_GRP_ATTR_NULL); + lck_rw_init(&llock, llock_grp, LCK_ATTR_NULL); +} + +SECURITY_READ_ONLY_EARLY(static struct mac_policy_ops) vng_policy_ops = { + .mpo_file_label_destroy = vng_file_label_destroy, + + .mpo_vnode_check_link = vng_vnode_check_link, + .mpo_vnode_check_unlink = vng_vnode_check_unlink, + .mpo_vnode_check_rename = vng_vnode_check_rename, + .mpo_vnode_check_write = vng_vnode_check_write, + .mpo_vnode_check_truncate = vng_vnode_check_truncate, + .mpo_vnode_check_exchangedata = vng_vnode_check_exchangedata, + + .mpo_policy_syscall = vng_policy_syscall, + .mpo_policy_init = vng_init, +}; + +static const char *vng_labelnames[] = { + "vnguard", +}; + +#define ACOUNT(arr) ((unsigned)(sizeof (arr) / sizeof (arr[0]))) + +SECURITY_READ_ONLY_LATE(static struct mac_policy_conf) vng_policy_conf = { + .mpc_name = VNG_POLICY_NAME, + .mpc_fullname = "Guarded vnode policy", + .mpc_field_off = &label_slot, + .mpc_labelnames = vng_labelnames, + .mpc_labelname_count = ACOUNT(vng_labelnames), + .mpc_ops = &vng_policy_ops, + .mpc_loadtime_flags = 0, + .mpc_runtime_flags = 0 +}; + +static mac_policy_handle_t vng_policy_handle; + +void +vnguard_policy_init(void) +{ + if (0 == PE_i_can_has_debugger(NULL)) + return; + vng_policy_flags = kVNG_POLICY_LOGMSG | kVNG_POLICY_EXC_CORPSE; + PE_parse_boot_argn("vnguard", &vng_policy_flags, sizeof (vng_policy_flags)); + if (vng_policy_flags) + mac_policy_register(&vng_policy_conf, &vng_policy_handle, NULL); +} + +#if DEBUG || DEVELOPMENT +#include + +SYSCTL_DECL(_kern_vnguard); +SYSCTL_NODE(_kern, OID_AUTO, vnguard, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "vnguard"); +SYSCTL_INT(_kern_vnguard, OID_AUTO, flags, CTLFLAG_RW | CTLFLAG_LOCKED, + &vng_policy_flags, 0, "vnguard policy flags"); +#endif + +#endif /* CONFIG_MACF && CONFIG_VNGUARD */ diff --git a/bsd/kern/kern_kpc.c b/bsd/kern/kern_kpc.c index 38bc2abbd..098b7349f 100644 --- a/bsd/kern/kern_kpc.c +++ b/bsd/kern/kern_kpc.c @@ -60,25 +60,39 @@ typedef int (*getint_t)(void); typedef int (*setint_t)(int); -/* safety */ static int kpc_initted = 0; -/* locking and buffer for large data requests */ -#define SYSCTL_BUFFER_SIZE (33 * sizeof(uint64_t)) -static lck_grp_attr_t *sysctl_buffer_lckgrp_attr = NULL; -static lck_grp_t *sysctl_buffer_lckgrp = NULL; -static lck_mtx_t sysctl_buffer_lock; -static void *sysctl_buffer = NULL; +static lck_grp_attr_t *sysctl_lckgrp_attr = NULL; +static lck_grp_t *sysctl_lckgrp = NULL; +static lck_mtx_t sysctl_lock; + +#if defined(__x86_64__) +/* 18 cores, 7 counters each */ +#define KPC_MAX_COUNTERS_COPIED (18 * 7) +#elif defined(__arm64__) +#include +#if defined(CPU_COUNT) +#define KPC_MAX_COUNTERS_COPIED (CPU_COUNT * 10) +#else /* defined(CPU_COUNT) */ +#define KPC_MAX_COUNTERS_COPIED (2 * 10) +#endif /* !defined(CPU_COUNT) */ +#elif defined(__arm__) +#define KPC_MAX_COUNTERS_COPIED (16) +#else /* !defined(__arm__) && !defined(__arm64__) && !defined(__x86_64__) */ +#error "unknown architecture for kpc buffer sizes" +#endif /* !defined(__arm__) && !defined(__arm64__) && !defined(__x86_64__) */ + +static_assert((KPC_MAX_COUNTERS_COPIED * sizeof(uint64_t)) < 1024, + "kpc's stack could grow too large"); typedef int (*setget_func_t)(int); void kpc_init(void) { - sysctl_buffer_lckgrp_attr = lck_grp_attr_alloc_init(); - sysctl_buffer_lckgrp = lck_grp_alloc_init("kpc", - sysctl_buffer_lckgrp_attr); - lck_mtx_init(&sysctl_buffer_lock, sysctl_buffer_lckgrp, LCK_ATTR_NULL); + sysctl_lckgrp_attr = lck_grp_attr_alloc_init(); + sysctl_lckgrp = lck_grp_alloc_init("kpc", sysctl_lckgrp_attr); + lck_mtx_init(&sysctl_lock, sysctl_lckgrp, LCK_ATTR_NULL); kpc_arch_init(); kpc_common_init(); @@ -156,26 +170,6 @@ sysctl_setget_int( struct sysctl_req *req, } static int -kpc_sysctl_acquire_buffer(void) -{ - if( sysctl_buffer == NULL ) - { - sysctl_buffer = kalloc(SYSCTL_BUFFER_SIZE); - if( sysctl_buffer ) - { - bzero( sysctl_buffer, SYSCTL_BUFFER_SIZE ); - } - } - - if( !sysctl_buffer ) - { - return ENOMEM; - } - - return 0; -} - -static int sysctl_kpc_get_counters(uint32_t counters, uint32_t *size, void *buf) { @@ -218,7 +212,7 @@ sysctl_kpc_get_shadow_counters(uint32_t counters, return 0; } -static int +static int sysctl_kpc_get_thread_counters(uint32_t tid, uint32_t *size, void *buf) { @@ -233,7 +227,7 @@ sysctl_kpc_get_thread_counters(uint32_t tid, *size = count * sizeof(uint64_t); return r; -} +} static int sysctl_kpc_get_config(uint32_t classes, void* buf) @@ -279,35 +273,23 @@ sysctl_kpc_set_actionid(uint32_t classes, void* buf) static int -sysctl_get_bigarray( struct sysctl_req *req, - int (*get_fn)(uint32_t, uint32_t*, void*) ) +sysctl_get_bigarray(struct sysctl_req *req, + int (*get_fn)(uint32_t, uint32_t*, void*)) { - int error = 0; - uint32_t bufsize = SYSCTL_BUFFER_SIZE; + uint64_t buf[KPC_MAX_COUNTERS_COPIED] = {}; + uint32_t bufsize = sizeof(buf); uint32_t arg = 0; /* get the argument */ - error = SYSCTL_IN( req, &arg, sizeof(arg) ); - if(error) - { - printf( "kpc: no arg?\n" ); + int error = SYSCTL_IN(req, &arg, sizeof(arg)); + if (error) { return error; } - /* get the wired buffer */ - error = kpc_sysctl_acquire_buffer(); - if (error) - return error; - - /* atomically get the array into the wired buffer. We have a double - * copy, but this is better than page faulting / interrupting during - * a copy. - */ - error = get_fn( arg, &bufsize, sysctl_buffer ); - - /* do the copy out */ - if( !error ) - error = SYSCTL_OUT( req, sysctl_buffer, bufsize ); + error = get_fn(arg, &bufsize, &buf); + if (!error) { + error = SYSCTL_OUT(req, &buf, bufsize); + } return error; } @@ -332,79 +314,53 @@ sysctl_actionid_size( uint32_t classes ) } static int -sysctl_getset_bigarray( struct sysctl_req *req, - int (*size_fn)(uint32_t arg), - int (*get_fn)(uint32_t, void*), - int (*set_fn)(uint32_t, void*) ) +sysctl_getset_bigarray(struct sysctl_req *req, int (*size_fn)(uint32_t arg), + int (*get_fn)(uint32_t, void*), int (*set_fn)(uint32_t, void*)) { int error = 0; - uint32_t bufsize = SYSCTL_BUFFER_SIZE; - uint32_t regsize = 0; + uint64_t buf[KPC_MAX_COUNTERS_COPIED] = {}; + uint32_t bufsize = sizeof(buf); uint64_t arg; /* get the config word */ - error = SYSCTL_IN( req, &arg, sizeof(arg) ); - if(error) - { - printf( "kpc: no arg?\n" ); + error = SYSCTL_IN(req, &arg, sizeof(arg)); + if (error) { return error; } - /* Work out size of registers */ - regsize = size_fn((uint32_t)arg); - - /* Ignore NULL requests */ - if(regsize == 0) - return EINVAL; - - /* ensure not too big */ - if( regsize > bufsize ) + /* Determine the size of registers to modify. */ + uint32_t regsize = size_fn((uint32_t)arg); + if (regsize == 0 || regsize > bufsize) { return EINVAL; + } - /* get the wired buffer */ - error = kpc_sysctl_acquire_buffer(); - if (error) - return error; - - // if writing... - if(req->newptr) - { - // copy in the rest in -- sysctl remembers we did one already - error = SYSCTL_IN( req, sysctl_buffer, - regsize ); - - // if SYSCTL_IN fails it means we are only doing a read - if(!error) { - // set it - error = set_fn( (uint32_t)arg, sysctl_buffer ); - if( error ) - goto fail; + /* if writing */ + if (req->newptr) { + /* copy the rest -- SYSCTL_IN knows the copyin should be shifted */ + error = SYSCTL_IN(req, &buf, regsize); + + /* SYSCTL_IN failure means only need to read */ + if (!error) { + error = set_fn((uint32_t)arg, &buf); + if (error) { + return error; + } } } - // if reading - if(req->oldptr) - { - // read it - error = get_fn( (uint32_t)arg, sysctl_buffer ); - if( error ) - goto fail; + /* if reading */ + if (req->oldptr) { + error = get_fn((uint32_t)arg, &buf); + if (error) { + return error; + } - // copy out the full set - error = SYSCTL_OUT( req, sysctl_buffer, regsize ); + error = SYSCTL_OUT(req, &buf, regsize); } - -fail: + return error; } - - -/* - * #define SYSCTL_HANDLER_ARGS (struct sysctl_oid *oidp, \ - * void *arg1, int arg2, \ - * struct sysctl_req *req ) - */ static int kpc_sysctl SYSCTL_HANDLER_ARGS { @@ -412,11 +368,11 @@ kpc_sysctl SYSCTL_HANDLER_ARGS // __unused struct sysctl_oid *unused_oidp = oidp; (void)arg2; - + if( !kpc_initted ) panic("kpc_init not called"); - lck_mtx_lock(ktrace_lock); + ktrace_lock(); // Most sysctls require an access check, but a few are public. switch( (uintptr_t) arg1 ) { @@ -430,15 +386,15 @@ kpc_sysctl SYSCTL_HANDLER_ARGS // Require kperf access to read or write anything else. // This is either root or the blessed pid. if ((ret = ktrace_read_check())) { - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); return ret; } break; } - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); - lck_mtx_lock(&sysctl_buffer_lock); + lck_mtx_lock(&sysctl_lock); /* which request */ switch( (uintptr_t) arg1 ) @@ -505,7 +461,7 @@ kpc_sysctl SYSCTL_HANDLER_ARGS case REQ_SW_INC: ret = sysctl_set_int( req, (setget_func_t)kpc_set_sw_inc ); - break; + break; case REQ_PMU_VERSION: ret = sysctl_get_int(oidp, req, kpc_get_pmu_version()); @@ -516,8 +472,8 @@ kpc_sysctl SYSCTL_HANDLER_ARGS break; } - lck_mtx_unlock(&sysctl_buffer_lock); - + lck_mtx_unlock(&sysctl_lock); + return ret; } diff --git a/bsd/kern/kern_ktrace.c b/bsd/kern/kern_ktrace.c index af4573ef8..628a19a55 100644 --- a/bsd/kern/kern_ktrace.c +++ b/bsd/kern/kern_ktrace.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -70,7 +70,7 @@ char *proc_name_address(void *p); kern_return_t ktrace_background_available_notify_user(void); -lck_mtx_t *ktrace_lock; +static lck_mtx_t *ktrace_mtx; /* * The overall state of ktrace, whether it is unconfigured, in foreground mode, @@ -105,7 +105,7 @@ static uint32_t ktrace_active_mask = 0; * * Background tools must be RunAtLoad daemons. */ -static boolean_t should_notify_on_init = TRUE; +static bool should_notify_on_init = true; /* Set the owning process of ktrace. */ static void ktrace_set_owning_proc(proc_t p); @@ -124,7 +124,10 @@ static void ktrace_promote_background(void); * This is managed by the user space-oriented function ktrace_set_owning_pid * and ktrace_unset_owning_pid. */ -boolean_t ktrace_keep_ownership_on_reset = FALSE; +bool ktrace_keep_ownership_on_reset = false; + +/* Whether the kernel is the owner of ktrace. */ +bool ktrace_owner_kernel = false; /* Allow user space to unset the owning pid and potentially reset ktrace. */ static void ktrace_set_invalid_owning_pid(void); @@ -135,6 +138,48 @@ static void ktrace_set_invalid_owning_pid(void); */ int ktrace_root_set_owner_allowed = 0; +/* + * If ktrace is guaranteed that it's the only thread running on the system + * (e.g., during boot or wake) this flag disables locking requirements. + */ +static bool ktrace_single_threaded = false; + +void +ktrace_lock(void) +{ + if (!ktrace_single_threaded) { + lck_mtx_lock(ktrace_mtx); + } +} + +void +ktrace_unlock(void) +{ + if (!ktrace_single_threaded) { + lck_mtx_unlock(ktrace_mtx); + } +} + +void +ktrace_assert_lock_held(void) +{ + if (!ktrace_single_threaded) { + lck_mtx_assert(ktrace_mtx, LCK_MTX_ASSERT_OWNED); + } +} + +void +ktrace_start_single_threaded(void) +{ + ktrace_single_threaded = true; +} + +void +ktrace_end_single_threaded(void) +{ + ktrace_single_threaded = false; +} + static void ktrace_reset_internal(uint32_t reset_mask) { @@ -155,7 +200,7 @@ ktrace_reset_internal(uint32_t reset_mask) ktrace_promote_background(); } else if (ktrace_state == KTRACE_STATE_BG) { /* background tool is resetting ktrace */ - should_notify_on_init = TRUE; + should_notify_on_init = true; ktrace_release_ownership(); ktrace_state = KTRACE_STATE_OFF; } @@ -165,7 +210,7 @@ ktrace_reset_internal(uint32_t reset_mask) void ktrace_reset(uint32_t reset_mask) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if (ktrace_active_mask == 0) { if (!ktrace_keep_ownership_on_reset) { @@ -180,7 +225,6 @@ ktrace_reset(uint32_t reset_mask) static void ktrace_promote_background(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); assert(ktrace_state != KTRACE_STATE_BG); /* @@ -189,9 +233,9 @@ ktrace_promote_background(void) * for the host special port). */ if (ktrace_background_available_notify_user() == KERN_FAILURE) { - should_notify_on_init = TRUE; + should_notify_on_init = true; } else { - should_notify_on_init = FALSE; + should_notify_on_init = false; } ktrace_release_ownership(); @@ -201,14 +245,13 @@ ktrace_promote_background(void) bool ktrace_background_active(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); return (ktrace_state == KTRACE_STATE_BG); } int ktrace_read_check(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if (proc_uniqueid(current_proc()) == ktrace_owning_unique_id) { @@ -222,7 +265,7 @@ ktrace_read_check(void) static void ktrace_ownership_maintenance(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); /* do nothing if ktrace is not owned */ if (ktrace_owning_unique_id == 0) { @@ -248,7 +291,7 @@ ktrace_ownership_maintenance(void) int ktrace_configure(uint32_t config_mask) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); assert(config_mask != 0); proc_t p = current_proc(); @@ -274,6 +317,7 @@ ktrace_configure(uint32_t config_mask) return EPERM; } + ktrace_owner_kernel = false; ktrace_set_owning_proc(p); ktrace_active_mask |= config_mask; return 0; @@ -295,7 +339,7 @@ ktrace_disable(enum ktrace_state state_to_match) int ktrace_get_owning_pid(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); ktrace_ownership_maintenance(); return ktrace_owning_pid; @@ -304,18 +348,24 @@ ktrace_get_owning_pid(void) void ktrace_kernel_configure(uint32_t config_mask) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + assert(ktrace_single_threaded == true); + + if (ktrace_owner_kernel) { + ktrace_active_mask |= config_mask; + return; + } if (ktrace_state != KTRACE_STATE_OFF) { - if (ktrace_active_mask & KTRACE_KPERF) { + if (ktrace_active_mask & config_mask & KTRACE_KPERF) { kperf_reset(); } - if (ktrace_active_mask & KTRACE_KDEBUG) { + if (ktrace_active_mask & config_mask & KTRACE_KDEBUG) { kdebug_reset(); } } - ktrace_active_mask = config_mask; + ktrace_owner_kernel = true; + ktrace_active_mask |= config_mask; ktrace_state = KTRACE_STATE_FG; ktrace_release_ownership(); @@ -328,7 +378,7 @@ ktrace_init_background(void) { int err = 0; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if ((err = priv_check_cred(kauth_cred_get(), PRIV_KTRACE_BACKGROUND, 0))) { return err; @@ -350,7 +400,7 @@ ktrace_init_background(void) return EINVAL; } } - should_notify_on_init = FALSE; + should_notify_on_init = false; } proc_t p = current_proc(); @@ -369,7 +419,7 @@ void ktrace_set_invalid_owning_pid(void) { if (ktrace_keep_ownership_on_reset) { - ktrace_keep_ownership_on_reset = FALSE; + ktrace_keep_ownership_on_reset = false; ktrace_reset_internal(ktrace_active_mask); } } @@ -377,7 +427,7 @@ ktrace_set_invalid_owning_pid(void) int ktrace_set_owning_pid(int pid) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); /* allow user space to successfully unset owning pid */ if (pid == -1) { @@ -397,7 +447,7 @@ ktrace_set_owning_pid(int pid) return ESRCH; } - ktrace_keep_ownership_on_reset = TRUE; + ktrace_keep_ownership_on_reset = true; ktrace_set_owning_proc(p); proc_rele(p); @@ -407,8 +457,8 @@ ktrace_set_owning_pid(int pid) static void ktrace_set_owning_proc(proc_t p) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); - assert(p); + ktrace_assert_lock_held(); + assert(p != NULL); if (ktrace_state != KTRACE_STATE_FG) { if (proc_uniqueid(p) == ktrace_bg_unique_id) { @@ -425,10 +475,11 @@ ktrace_set_owning_proc(proc_t p) ktrace_active_mask = 0; } ktrace_state = KTRACE_STATE_FG; - should_notify_on_init = FALSE; + should_notify_on_init = false; } } + ktrace_owner_kernel = false; ktrace_owning_unique_id = proc_uniqueid(p); ktrace_owning_pid = proc_pid(p); strlcpy(ktrace_last_owner_execname, proc_name_address(p), @@ -438,8 +489,6 @@ ktrace_set_owning_proc(proc_t p) static void ktrace_release_ownership(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); - ktrace_owning_unique_id = 0; ktrace_owning_pid = 0; } @@ -477,7 +526,7 @@ ktrace_sysctl SYSCTL_HANDLER_ARGS int ret = 0; uintptr_t type = (uintptr_t)arg1; - lck_mtx_lock(ktrace_lock); + ktrace_lock(); if (!kauth_cred_issuser(kauth_cred_get())) { ret = EPERM; @@ -498,7 +547,7 @@ ktrace_sysctl SYSCTL_HANDLER_ARGS } out: - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); return ret; } @@ -508,7 +557,7 @@ ktrace_init(void) { static lck_grp_attr_t *lock_grp_attr = NULL; static lck_grp_t *lock_grp = NULL; - static boolean_t initialized = FALSE; + static bool initialized = false; if (initialized) { return; @@ -518,7 +567,7 @@ ktrace_init(void) lock_grp = lck_grp_alloc_init("ktrace", lock_grp_attr); lck_grp_attr_free(lock_grp_attr); - ktrace_lock = lck_mtx_alloc_init(lock_grp, LCK_ATTR_NULL); - assert(ktrace_lock); - initialized = TRUE; + ktrace_mtx = lck_mtx_alloc_init(lock_grp, LCK_ATTR_NULL); + assert(ktrace_mtx != NULL);; + initialized = true; } diff --git a/bsd/kern/kern_lockf.c b/bsd/kern/kern_lockf.c index f0bfa129f..5284f060c 100644 --- a/bsd/kern/kern_lockf.c +++ b/bsd/kern/kern_lockf.c @@ -330,6 +330,12 @@ lf_advlock(struct vnop_advlock_args *ap) FREE(lock, M_LOCKF); break; +#if CONFIG_EMBEDDED + case F_GETLKPID: + error = lf_getlock(lock, fl, fl->l_pid); + FREE(lock, M_LOCKF); + break; +#endif default: FREE(lock, M_LOCKF); diff --git a/bsd/kern/kern_malloc.c b/bsd/kern/kern_malloc.c index d33382ec3..e5ae62f3f 100644 --- a/bsd/kern/kern_malloc.c +++ b/bsd/kern/kern_malloc.c @@ -300,6 +300,9 @@ const char *memname[] = { "fdvnodedata" /* 122 M_FD_VN_DATA */ "fddirbuf", /* 123 M_FD_DIRBUF */ "netagent", /* 124 M_NETAGENT */ + "Event Handler",/* 125 M_EVENTHANDLER */ + "Link Layer Table", /* 126 M_LLTABLE */ + "Network Work Queue", /* 127 M_NWKWQ */ "" }; @@ -485,6 +488,9 @@ struct kmzones { { 0, KMZ_MALLOC, FALSE }, /* 122 M_FD_VN_DATA */ { 0, KMZ_MALLOC, FALSE }, /* 123 M_FD_DIRBUF */ { 0, KMZ_MALLOC, FALSE }, /* 124 M_NETAGENT */ + { 0, KMZ_MALLOC, FALSE }, /* 125 M_EVENTHANDLER */ + { 0, KMZ_MALLOC, FALSE }, /* 126 M_LLTABLE */ + { 0, KMZ_MALLOC, FALSE }, /* 127 M_NWKWQ */ #undef SOS #undef SOX }; @@ -557,7 +563,7 @@ _MALLOC_external( int type, int flags) { - static vm_allocation_site_t site = { VM_KERN_MEMORY_KALLOC, VM_TAG_BT }; + static vm_allocation_site_t site = { .tag = VM_KERN_MEMORY_KALLOC, .flags = VM_TAG_BT }; return (__MALLOC(size, type, flags, &site)); } @@ -650,9 +656,9 @@ __REALLOC( if (kalloc_bucket_size(size) == alloc) { if (flags & M_ZERO) { if (alloc < size) - bzero((uintptr_t)addr + alloc, (size - alloc)); + bzero(addr + alloc, (size - alloc)); else - bzero((uintptr_t)addr + size, (alloc - size)); + bzero(addr + size, (alloc - size)); } return addr; } @@ -751,6 +757,51 @@ _FREE_ZONE( kfree(elem, size); } +#if DEBUG || DEVELOPMENT + +extern unsigned int zone_map_jetsam_limit; + +static int +sysctl_zone_map_jetsam_limit SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int oldval = 0, val = 0, error = 0; + + oldval = zone_map_jetsam_limit; + error = sysctl_io_number(req, oldval, sizeof(int), &val, NULL); + if (error || !req->newptr) { + return (error); + } + + if (val <= 0 || val > 100) { + printf("sysctl_zone_map_jetsam_limit: new jetsam limit value is invalid.\n"); + return EINVAL; + } + + zone_map_jetsam_limit = val; + return (0); +} + +SYSCTL_PROC(_kern, OID_AUTO, zone_map_jetsam_limit, CTLTYPE_INT|CTLFLAG_RW, 0, 0, + sysctl_zone_map_jetsam_limit, "I", "Zone map jetsam limit"); + +extern boolean_t run_zone_test(void); + +static int +sysctl_run_zone_test SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int ret_val = run_zone_test(); + + return SYSCTL_OUT(req, &ret_val, sizeof(ret_val)); +} + +SYSCTL_PROC(_kern, OID_AUTO, run_zone_test, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, + 0, 0, &sysctl_run_zone_test, "I", "Test zone allocator KPI"); + +#endif /* DEBUG || DEVELOPMENT */ + #if CONFIG_ZLEAKS SYSCTL_DECL(_kern_zleak); @@ -863,3 +914,18 @@ SYSCTL_PROC(_kern_zleak, OID_AUTO, zone_threshold, sysctl_zleak_threshold, "Q", "zleak per-zone threshold"); #endif /* CONFIG_ZLEAKS */ + +extern uint64_t get_zones_collectable_bytes(void); + +static int +sysctl_zones_collectable_bytes SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + uint64_t zones_free_mem = get_zones_collectable_bytes(); + + return SYSCTL_OUT(req, &zones_free_mem, sizeof(zones_free_mem)); +} + +SYSCTL_PROC(_kern, OID_AUTO, zones_collectable_bytes, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, + 0, 0, &sysctl_zones_collectable_bytes, "Q", "Collectable memory in zones"); diff --git a/bsd/kern/kern_memorystatus.c b/bsd/kern/kern_memorystatus.c index 46b037aba..74a80dc55 100644 --- a/bsd/kern/kern_memorystatus.c +++ b/bsd/kern/kern_memorystatus.c @@ -36,6 +36,7 @@ #include #include #include +#include #include @@ -45,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -68,9 +70,10 @@ #include #include +#include /* For logging clarity */ -static const char *jetsam_kill_cause_name[] = { +static const char *memorystatus_kill_cause_name[] = { "" , "jettisoned" , /* kMemorystatusKilled */ "highwater" , /* kMemorystatusKilledHiwat */ @@ -81,12 +84,35 @@ static const char *jetsam_kill_cause_name[] = { "per-process-limit" , /* kMemorystatusKilledPerProcessLimit */ "diagnostic" , /* kMemorystatusKilledDiagnostic */ "idle-exit" , /* kMemorystatusKilledIdleExit */ + "zone-map-exhaustion" , /* kMemorystatusKilledZoneMapExhaustion */ }; -#if CONFIG_JETSAM +static const char * +memorystatus_priority_band_name(int32_t priority) +{ + switch (priority) { + case JETSAM_PRIORITY_FOREGROUND: + return "FOREGROUND"; + case JETSAM_PRIORITY_AUDIO_AND_ACCESSORY: + return "AUDIO_AND_ACCESSORY"; + case JETSAM_PRIORITY_CONDUCTOR: + return "CONDUCTOR"; + case JETSAM_PRIORITY_HOME: + return "HOME"; + case JETSAM_PRIORITY_EXECUTIVE: + return "EXECUTIVE"; + case JETSAM_PRIORITY_IMPORTANT: + return "IMPORTANT"; + case JETSAM_PRIORITY_CRITICAL: + return "CRITICAL"; + } + + return ("?"); +} + /* Does cause indicate vm or fc thrashing? */ static boolean_t -is_thrashing(unsigned cause) +is_reason_thrashing(unsigned cause) { switch (cause) { case kMemorystatusKilledVMThrashing: @@ -97,9 +123,26 @@ is_thrashing(unsigned cause) } } -/* Callback into vm_compressor.c to signal that thrashing has been mitigated. */ -extern void vm_thrashing_jetsam_done(void); -#endif /* CONFIG_JETSAM */ +/* Is the zone map almost full? */ +static boolean_t +is_reason_zone_map_exhaustion(unsigned cause) +{ + if (cause == kMemorystatusKilledZoneMapExhaustion) + return TRUE; + return FALSE; +} + +/* + * Returns the current zone map size and capacity to include in the jetsam snapshot. + * Defined in zalloc.c + */ +extern void get_zone_map_size(uint64_t *current_size, uint64_t *capacity); + +/* + * Returns the name of the largest zone and its size to include in the jetsam snapshot. + * Defined in zalloc.c + */ +extern void get_largest_zone_info(char *zone_name, size_t zone_name_len, uint64_t *zone_size); /* These are very verbose printfs(), enable with * MEMORYSTATUS_DEBUG_LOG @@ -193,13 +236,13 @@ static uint64_t memorystatus_apps_idle_delay_time = 0; * Memorystatus kevents */ -static int filt_memorystatusattach(struct knote *kn); +static int filt_memorystatusattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_memorystatusdetach(struct knote *kn); static int filt_memorystatus(struct knote *kn, long hint); static int filt_memorystatustouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_memorystatusprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops memorystatus_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) memorystatus_filtops = { .f_attach = filt_memorystatusattach, .f_detach = filt_memorystatusdetach, .f_event = filt_memorystatus, @@ -229,6 +272,8 @@ static void memorystatus_reschedule_idle_demotion_locked(void); static void memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_insert, boolean_t skip_demotion_check); +int memorystatus_update_priority_for_appnap(proc_t p, boolean_t is_appnap); + vm_pressure_level_t convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t); boolean_t is_knote_registered_modify_task_pressure_bits(struct knote*, int, task_t, vm_pressure_level_t, vm_pressure_level_t); @@ -250,6 +295,8 @@ typedef struct memstat_bucket { memstat_bucket_t memstat_bucket[MEMSTAT_BUCKET_COUNT]; +int memorystatus_get_proccnt_upto_priority(int32_t max_bucket_index); + uint64_t memstat_idle_demotion_deadline = 0; int system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1; @@ -268,8 +315,9 @@ int applications_aging_band = JETSAM_PRIORITY_IDLE; unsigned int jetsam_aging_policy = kJetsamAgingPolicyLegacy; extern int corpse_for_fatal_memkill; -extern unsigned long total_corpses_count; +extern unsigned long total_corpses_count(void) __attribute__((pure)); extern void task_purge_all_corpses(void); +boolean_t memorystatus_allowed_vm_map_fork(__unused task_t); #if 0 @@ -459,6 +507,11 @@ static unsigned int memorystatus_dirty_count = 0; SYSCTL_INT(_kern, OID_AUTO, max_task_pmem, CTLFLAG_RD|CTLFLAG_LOCKED|CTLFLAG_MASKED, &max_task_footprint_mb, 0, ""); +#if CONFIG_EMBEDDED + +SYSCTL_INT(_kern, OID_AUTO, memorystatus_level, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_level, 0, ""); + +#endif /* CONFIG_EMBEDDED */ int memorystatus_get_level(__unused struct proc *p, struct memorystatus_get_level_args *args, __unused int *ret) @@ -488,12 +541,6 @@ static boolean_t memorystatus_kill_specific_process(pid_t victim_pid, uint32_t c static boolean_t memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason); -/* Jetsam */ - -#if CONFIG_JETSAM - -static int memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit); - static int memorystatus_cmd_set_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval); static int memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry); @@ -508,12 +555,8 @@ static boolean_t memorystatus_idle_snapshot = 0; unsigned int memorystatus_delta = 0; -static unsigned int memorystatus_available_pages_critical_base = 0; -//static unsigned int memorystatus_last_foreground_pressure_pages = (unsigned int)-1; -static unsigned int memorystatus_available_pages_critical_idle_offset = 0; - /* Jetsam Loop Detection */ -static boolean_t memorystatus_jld_enabled = TRUE; /* Enables jetsam loop detection on all devices */ +static boolean_t memorystatus_jld_enabled = FALSE; /* Enable jetsam loop detection */ static uint32_t memorystatus_jld_eval_period_msecs = 0; /* Init pass sets this based on device memory size */ static int memorystatus_jld_eval_aggressive_count = 3; /* Raise the priority max after 'n' aggressive loops */ static int memorystatus_jld_eval_aggressive_priority_band_max = 15; /* Kill aggressively up through this band */ @@ -545,14 +588,6 @@ SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_count, CTLFLAG_RW| SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_priority_band_max, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_jld_eval_aggressive_priority_band_max, 0, ""); #endif /* DEVELOPMENT || DEBUG */ -#if DEVELOPMENT || DEBUG -static unsigned int memorystatus_jetsam_panic_debug = 0; -static unsigned int memorystatus_jetsam_policy_offset_pages_diagnostic = 0; -#endif - -static unsigned int memorystatus_jetsam_policy = kPolicyDefault; -static unsigned int memorystatus_thread_wasted_wakeup = 0; - static uint32_t kill_under_pressure_cause = 0; /* @@ -586,11 +621,10 @@ static void memorystatus_get_task_phys_footprint_page_counts(task_t task, static void memorystatus_get_task_memory_region_count(task_t task, uint64_t *count); static uint32_t memorystatus_build_state(proc_t p); -static void memorystatus_update_levels_locked(boolean_t critical_only); //static boolean_t memorystatus_issue_pressure_kevent(boolean_t pressured); static boolean_t memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause, os_reason_t jetsam_reason, int32_t *priority, uint32_t *errors); -static boolean_t memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reason_t jetsam_reason, int aggr_count, int32_t priority_max, uint32_t *errors); +static boolean_t memorystatus_kill_top_process_aggressive(uint32_t cause, int aggr_count, int32_t priority_max, uint32_t *errors); static boolean_t memorystatus_kill_elevated_process(uint32_t cause, os_reason_t jetsam_reason, int aggr_count, uint32_t *errors); static boolean_t memorystatus_kill_hiwat_proc(uint32_t *errors); @@ -607,8 +641,6 @@ typedef int (*cmpfunc_t)(const void *a, const void *b); extern void qsort(void *a, size_t n, size_t es, cmpfunc_t cmp); static int memstat_asc_cmp(const void *a, const void *b); -#endif /* CONFIG_JETSAM */ - /* VM pressure */ extern unsigned int vm_page_free_count; @@ -621,28 +653,48 @@ extern unsigned int vm_page_wire_count; extern unsigned int vm_page_secluded_count; #endif /* CONFIG_SECLUDED_MEMORY */ -#if VM_PRESSURE_EVENTS - -boolean_t memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused boolean_t is_fatal, boolean_t exceeded); - -vm_pressure_level_t memorystatus_vm_pressure_level = kVMPressureNormal; - -#if CONFIG_MEMORYSTATUS +#if CONFIG_JETSAM unsigned int memorystatus_available_pages = (unsigned int)-1; unsigned int memorystatus_available_pages_pressure = 0; unsigned int memorystatus_available_pages_critical = 0; -unsigned int memorystatus_frozen_count = 0; -unsigned int memorystatus_suspended_count = 0; -unsigned int memorystatus_policy_more_free_offset_pages = 0; +static unsigned int memorystatus_available_pages_critical_base = 0; +static unsigned int memorystatus_available_pages_critical_idle_offset = 0; -#if CONFIG_JETSAM #if DEVELOPMENT || DEBUG SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_available_pages, 0, ""); #else -SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD| CTLFLAG_MASKED | CTLFLAG_LOCKED, &memorystatus_available_pages, 0, ""); +SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, &memorystatus_available_pages, 0, ""); #endif /* DEVELOPMENT || DEBUG */ + +static unsigned int memorystatus_jetsam_policy = kPolicyDefault; +unsigned int memorystatus_policy_more_free_offset_pages = 0; +static void memorystatus_update_levels_locked(boolean_t critical_only); +static unsigned int memorystatus_thread_wasted_wakeup = 0; + +/* Callback into vm_compressor.c to signal that thrashing has been mitigated. */ +extern void vm_thrashing_jetsam_done(void); +static int memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit); + +int32_t max_kill_priority = JETSAM_PRIORITY_MAX; + +#else /* CONFIG_JETSAM */ + +uint64_t memorystatus_available_pages = (uint64_t)-1; +uint64_t memorystatus_available_pages_pressure = (uint64_t)-1; +uint64_t memorystatus_available_pages_critical = (uint64_t)-1; + +int32_t max_kill_priority = JETSAM_PRIORITY_IDLE; #endif /* CONFIG_JETSAM */ +unsigned int memorystatus_frozen_count = 0; +unsigned int memorystatus_suspended_count = 0; + +#if VM_PRESSURE_EVENTS + +boolean_t memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused boolean_t is_fatal, boolean_t exceeded); + +vm_pressure_level_t memorystatus_vm_pressure_level = kVMPressureNormal; + /* * We use this flag to signal if we have any HWM offenders * on the system. This way we can reduce the number of wakeups @@ -658,7 +710,6 @@ SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD| CTLFLAG_M boolean_t memorystatus_hwm_candidates = 0; static int memorystatus_send_note(int event_code, void *data, size_t data_length); -#endif /* CONFIG_MEMORYSTATUS */ #endif /* VM_PRESSURE_EVENTS */ @@ -669,7 +720,8 @@ lck_grp_attr_t *disconnect_page_mappings_lck_grp_attr; lck_grp_t *disconnect_page_mappings_lck_grp; static lck_mtx_t disconnect_page_mappings_mutex; -#endif +extern boolean_t kill_on_no_paging_space; +#endif /* DEVELOPMENT || DEBUG */ /* Freeze */ @@ -714,7 +766,7 @@ static unsigned int memorystatus_suspended_footprint_total = 0; /* pages */ extern uint64_t vm_swap_get_free_space(void); -static boolean_t memorystatus_freeze_update_throttle(); +static boolean_t memorystatus_freeze_update_throttle(void); #endif /* CONFIG_FREEZE */ @@ -851,10 +903,6 @@ sysctl_memorystatus_highwater_enable SYSCTL_HANDLER_ARGS use_active = proc_jetsam_state_is_active_locked(p); if (enable) { - /* - * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore. - * Background limits are described via the inactive limit slots. - */ if (use_active == TRUE) { CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal); @@ -937,12 +985,14 @@ sysctl_memorystatus_vm_pressure_send SYSCTL_HANDLER_ARGS /* * See event.h ... fflags for EVFILT_MEMORYSTATUS */ - if (!((fflags == NOTE_MEMORYSTATUS_PRESSURE_NORMAL)|| - (fflags == NOTE_MEMORYSTATUS_PRESSURE_WARN) || - (fflags == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) || - (fflags == NOTE_MEMORYSTATUS_LOW_SWAP) || - (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) || - (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL))) { + if (!((fflags == NOTE_MEMORYSTATUS_PRESSURE_NORMAL)|| + (fflags == NOTE_MEMORYSTATUS_PRESSURE_WARN) || + (fflags == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) || + (fflags == NOTE_MEMORYSTATUS_LOW_SWAP) || + (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) || + (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) || + (((fflags & NOTE_MEMORYSTATUS_MSL_STATUS) != 0 && + ((fflags & ~NOTE_MEMORYSTATUS_MSL_STATUS) == 0))))) { printf("memorystatus_vm_pressure_send: notification [0x%x] not supported \n", fflags); error = 1; @@ -987,15 +1037,17 @@ SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_send, CTLTYPE_QUAD|CTLFLAG #endif /* VM_PRESSURE_EVENTS */ -#if CONFIG_JETSAM - SYSCTL_INT(_kern, OID_AUTO, memorystatus_idle_snapshot, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_idle_snapshot, 0, ""); +#if CONFIG_JETSAM SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_available_pages_critical, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical_base, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_available_pages_critical_base, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical_idle_offset, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_available_pages_critical_idle_offset, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, memorystatus_policy_more_free_offset_pages, CTLFLAG_RW, &memorystatus_policy_more_free_offset_pages, 0, ""); +static unsigned int memorystatus_jetsam_panic_debug = 0; +static unsigned int memorystatus_jetsam_policy_offset_pages_diagnostic = 0; + /* Diagnostic code */ enum { @@ -1250,8 +1302,6 @@ SYSCTL_PROC(_kern, OID_AUTO, memorystatus_disconnect_page_mappings, CTLTYPE_INT| #endif /* DEVELOPMENT || DEBUG */ - -#if CONFIG_JETSAM /* * Picks the sorting routine for a given jetsam priority band. * @@ -1292,6 +1342,12 @@ static int memorystatus_sort_bucket(unsigned int bucket_index, int sort_order) #endif proc_list_lock(); + + if (memstat_bucket[bucket_index].count == 0) { + proc_list_unlock(); + return (0); + } + switch (bucket_index) { case JETSAM_PRIORITY_FOREGROUND: if (memorystatus_sort_by_largest_coalition_locked(bucket_index, coal_sort_order) == 0) { @@ -1362,8 +1418,6 @@ static void memorystatus_sort_by_largest_process_locked(unsigned int bucket_inde } } -#endif /* CONFIG_JETSAM */ - static proc_t memorystatus_get_first_proc_locked(unsigned int *bucket_index, boolean_t search) { memstat_bucket_t *current_bucket; proc_t next_p; @@ -1418,19 +1472,24 @@ memorystatus_init(void) disconnect_page_mappings_lck_grp = lck_grp_alloc_init("disconnect_page_mappings", disconnect_page_mappings_lck_grp_attr); lck_mtx_init(&disconnect_page_mappings_mutex, disconnect_page_mappings_lck_grp, NULL); + + if (kill_on_no_paging_space == TRUE) { + max_kill_priority = JETSAM_PRIORITY_MAX; + } #endif - nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_sysprocs_idle_delay_time); - nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_apps_idle_delay_time); /* Init buckets */ for (i = 0; i < MEMSTAT_BUCKET_COUNT; i++) { TAILQ_INIT(&memstat_bucket[i].list); memstat_bucket[i].count = 0; } - memorystatus_idle_demotion_call = thread_call_allocate((thread_call_func_t)memorystatus_perform_idle_demotion, NULL); +#if CONFIG_JETSAM + nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_sysprocs_idle_delay_time); + nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_apps_idle_delay_time); + /* Apply overrides */ PE_get_default("kern.jetsam_delta", &delta_percentage, sizeof(delta_percentage)); if (delta_percentage == 0) { @@ -1500,7 +1559,6 @@ memorystatus_init(void) assert(JETSAM_PRIORITY_ELEVATED_INACTIVE > system_procs_aging_band); assert(JETSAM_PRIORITY_ELEVATED_INACTIVE > applications_aging_band); -#if CONFIG_JETSAM /* Take snapshots for idle-exit kills by default? First check the boot-arg... */ if (!PE_parse_boot_argn("jetsam_idle_snapshot", &memorystatus_idle_snapshot, sizeof (memorystatus_idle_snapshot))) { /* ...no boot-arg, so check the device tree */ @@ -1512,6 +1570,22 @@ memorystatus_init(void) memorystatus_available_pages_critical_base = (critical_threshold_percentage / delta_percentage) * memorystatus_delta; memorystatus_policy_more_free_offset_pages = (policy_more_free_offset_percentage / delta_percentage) * memorystatus_delta; + /* Jetsam Loop Detection */ + if (max_mem <= (512 * 1024 * 1024)) { + /* 512 MB devices */ + memorystatus_jld_eval_period_msecs = 8000; /* 8000 msecs == 8 second window */ + } else { + /* 1GB and larger devices */ + memorystatus_jld_eval_period_msecs = 6000; /* 6000 msecs == 6 second window */ + } + + memorystatus_jld_enabled = TRUE; + + /* No contention at this point */ + memorystatus_update_levels_locked(FALSE); + +#endif /* CONFIG_JETSAM */ + memorystatus_jetsam_snapshot_max = maxproc; memorystatus_jetsam_snapshot = (memorystatus_jetsam_snapshot_t*)kalloc(sizeof(memorystatus_jetsam_snapshot_t) + @@ -1524,19 +1598,6 @@ memorystatus_init(void) memset(&memorystatus_at_boot_snapshot, 0, sizeof(memorystatus_jetsam_snapshot_t)); - /* No contention at this point */ - memorystatus_update_levels_locked(FALSE); - - /* Jetsam Loop Detection */ - if (max_mem <= (512 * 1024 * 1024)) { - /* 512 MB devices */ - memorystatus_jld_eval_period_msecs = 8000; /* 8000 msecs == 8 second window */ - } else { - /* 1GB and larger devices */ - memorystatus_jld_eval_period_msecs = 6000; /* 6000 msecs == 6 second window */ - } -#endif - #if CONFIG_FREEZE memorystatus_freeze_threshold = (freeze_threshold_percentage / delta_percentage) * memorystatus_delta; #endif @@ -1585,6 +1646,14 @@ memorystatus_do_kill(proc_t p, uint32_t cause, os_reason_t jetsam_reason) { #else #pragma unused(cause) #endif + + if (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND) { + printf("memorystatus: killing process %d [%s] in high band %s (%d) - memorystatus_available_pages: %llu\n", p->p_pid, + (*p->p_name ? p->p_name : "unknown"), + memorystatus_priority_band_name(p->p_memstat_effectivepriority), p->p_memstat_effectivepriority, + (uint64_t)memorystatus_available_pages); + } + int jetsam_flags = P_LTERM_JETSAM; switch (cause) { case kMemorystatusKilledHiwat: jetsam_flags |= P_JETSAM_HIWAT; break; @@ -1614,7 +1683,12 @@ memorystatus_check_levels_locked(void) { #if CONFIG_JETSAM /* Update levels */ memorystatus_update_levels_locked(TRUE); -#endif +#else /* CONFIG_JETSAM */ + /* + * Nothing to do here currently since we update + * memorystatus_available_pages in vm_pressure_response. + */ +#endif /* CONFIG_JETSAM */ } /* @@ -2101,9 +2175,6 @@ memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_inser boolean_t ledger_update_needed = TRUE; /* - * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore. - * Background limits are described via the inactive limit slots. - * * Here, we must update the cached memory limit if the task * is transitioning between: * active <--> inactive @@ -2231,11 +2302,6 @@ memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_inser * memlimit_inactive_is_fatal When a process is inactive and exceeds its memory footprint, * this describes whether or not it should be immediatly fatal. * - * memlimit_background This process has a high-water-mark while in the background. - * No longer meaningful. Background limits are described via - * the inactive slots. Flag is ignored. - * - * * Returns: 0 Success * non-0 Failure */ @@ -2243,8 +2309,7 @@ memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_inser int memorystatus_update(proc_t p, int priority, uint64_t user_data, boolean_t effective, boolean_t update_memlimit, int32_t memlimit_active, boolean_t memlimit_active_is_fatal, - int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal, - __unused boolean_t memlimit_background) + int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal) { int ret; boolean_t head_insert = false; @@ -2308,23 +2373,6 @@ memorystatus_update(proc_t p, int priority, uint64_t user_data, boolean_t effect memlimit_active, (memlimit_active_is_fatal ? "F " : "NF"), memlimit_inactive, (memlimit_inactive_is_fatal ? "F " : "NF")); - if (memlimit_background) { - - /* - * With 2-level HWM support, we no longer honor P_MEMSTAT_MEMLIMIT_BACKGROUND. - * Background limits are described via the inactive limit slots. - */ - - // p->p_memstat_state |= P_MEMSTAT_MEMLIMIT_BACKGROUND; - -#if DEVELOPMENT || DEBUG - printf("memorystatus_update: WARNING %s[%d] set unused flag P_MEMSTAT_MEMLIMIT_BACKGROUND [A==%dMB %s] [IA==%dMB %s]\n", - (*p->p_name ? p->p_name : "unknown"), p->p_pid, - memlimit_active, (memlimit_active_is_fatal ? "F " : "NF"), - memlimit_inactive, (memlimit_inactive_is_fatal ? "F " : "NF")); -#endif /* DEVELOPMENT || DEBUG */ - } - if (memlimit_active <= 0) { /* * This process will have a system_wide task limit when active. @@ -2572,6 +2620,19 @@ memorystatus_update_idle_priority_locked(proc_t p) { * explicitly because it won't be going through the demotion paths * that take care to apply the limits appropriately. */ + + if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) { + + /* + * This process has the 'elevated inactive jetsam band' attribute. + * So, there will be no trip to IDLE after all. + * Instead, we pin the process in the elevated band, + * where its ACTIVE limits will apply. + */ + + priority = JETSAM_PRIORITY_ELEVATED_INACTIVE; + } + memorystatus_update_priority_locked(p, priority, false, true); } else { @@ -2908,6 +2969,8 @@ memorystatus_dirty_set(proc_t p, boolean_t self, uint32_t pcontrol) { if (proc_jetsam_state_is_active_locked(p) == TRUE) { /* + * process is pinned in elevated band + * or * process is dirty */ CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal); @@ -3178,8 +3241,6 @@ memorystatus_build_state(proc_t p) { return snapshot_state; } -#if !CONFIG_JETSAM - static boolean_t kill_idle_exit_proc(void) { @@ -3220,7 +3281,7 @@ kill_idle_exit_proc(void) proc_list_unlock(); if (victim_p) { - printf("memorystatus_thread: idle exiting pid %d [%s]\n", victim_p->p_pid, (*victim_p->p_name ? victim_p->p_name : "(unknown)")); + printf("memorystatus: killing_idle_process pid %d [%s]\n", victim_p->p_pid, (*victim_p->p_name ? victim_p->p_name : "unknown")); killed = memorystatus_do_kill(victim_p, kMemorystatusKilledIdleExit, jetsam_reason); proc_rele(victim_p); } else { @@ -3229,14 +3290,11 @@ kill_idle_exit_proc(void) return killed; } -#endif -#if CONFIG_JETSAM static void memorystatus_thread_wake(void) { thread_wakeup((event_t)&memorystatus_wakeup); } -#endif /* CONFIG_JETSAM */ extern void vm_pressure_response(void); @@ -3252,32 +3310,305 @@ memorystatus_thread_block(uint32_t interval_ms, thread_continue_t continuation) return thread_block(continuation); } +static boolean_t +memorystatus_avail_pages_below_pressure(void) +{ +#if CONFIG_EMBEDDED +/* + * Instead of CONFIG_EMBEDDED for these *avail_pages* routines, we should + * key off of the system having dynamic swap support. With full swap support, + * the system shouldn't really need to worry about various page thresholds. + */ + return (memorystatus_available_pages <= memorystatus_available_pages_pressure); +#else /* CONFIG_EMBEDDED */ + return FALSE; +#endif /* CONFIG_EMBEDDED */ +} + +static boolean_t +memorystatus_avail_pages_below_critical(void) +{ +#if CONFIG_EMBEDDED + return (memorystatus_available_pages <= memorystatus_available_pages_critical); +#else /* CONFIG_EMBEDDED */ + return FALSE; +#endif /* CONFIG_EMBEDDED */ +} + +static boolean_t +memorystatus_post_snapshot(int32_t priority, uint32_t cause) +{ +#if CONFIG_EMBEDDED +#pragma unused(cause) + /* + * Don't generate logs for steady-state idle-exit kills, + * unless it is overridden for debug or by the device + * tree. + */ + + return ((priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot); + +#else /* CONFIG_EMBEDDED */ + /* + * Don't generate logs for steady-state idle-exit kills, + * unless + * - it is overridden for debug or by the device + * tree. + * OR + * - the kill causes are important i.e. not kMemorystatusKilledIdleExit + */ + + boolean_t snapshot_eligible_kill_cause = (is_reason_thrashing(cause) || is_reason_zone_map_exhaustion(cause)); + return ((priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot || snapshot_eligible_kill_cause); +#endif /* CONFIG_EMBEDDED */ +} + +static boolean_t +memorystatus_action_needed(void) +{ +#if CONFIG_EMBEDDED + return (is_reason_thrashing(kill_under_pressure_cause) || + is_reason_zone_map_exhaustion(kill_under_pressure_cause) || + memorystatus_available_pages <= memorystatus_available_pages_pressure); +#else /* CONFIG_EMBEDDED */ + return (is_reason_thrashing(kill_under_pressure_cause) || + is_reason_zone_map_exhaustion(kill_under_pressure_cause)); +#endif /* CONFIG_EMBEDDED */ +} + +static boolean_t +memorystatus_act_on_hiwat_processes(uint32_t *errors, uint32_t *hwm_kill, boolean_t *post_snapshot, __unused boolean_t *is_critical) +{ + boolean_t killed = memorystatus_kill_hiwat_proc(errors); + + if (killed) { + *hwm_kill = *hwm_kill + 1; + *post_snapshot = TRUE; + return TRUE; + } else { + memorystatus_hwm_candidates = FALSE; + } + +#if CONFIG_JETSAM + /* No highwater processes to kill. Continue or stop for now? */ + if (!is_reason_thrashing(kill_under_pressure_cause) && + !is_reason_zone_map_exhaustion(kill_under_pressure_cause) && + (memorystatus_available_pages > memorystatus_available_pages_critical)) { + /* + * We are _not_ out of pressure but we are above the critical threshold and there's: + * - no compressor thrashing + * - enough zone memory + * - no more HWM processes left. + * For now, don't kill any other processes. + */ + + if (*hwm_kill == 0) { + memorystatus_thread_wasted_wakeup++; + } + + *is_critical = FALSE; + + return TRUE; + } +#endif /* CONFIG_JETSAM */ + + return FALSE; +} + +static boolean_t +memorystatus_act_aggressive(uint32_t cause, os_reason_t jetsam_reason, int *jld_idle_kills, boolean_t *corpse_list_purged, boolean_t *post_snapshot) +{ + if (memorystatus_jld_enabled == TRUE) { + + boolean_t killed; + uint32_t errors = 0; + + /* Jetsam Loop Detection - locals */ + memstat_bucket_t *bucket; + int jld_bucket_count = 0; + struct timeval jld_now_tstamp = {0,0}; + uint64_t jld_now_msecs = 0; + int elevated_bucket_count = 0; + + /* Jetsam Loop Detection - statics */ + static uint64_t jld_timestamp_msecs = 0; + static int jld_idle_kill_candidates = 0; /* Number of available processes in band 0,1 at start */ + static int jld_eval_aggressive_count = 0; /* Bumps the max priority in aggressive loop */ + static int32_t jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT; + /* + * Jetsam Loop Detection: attempt to detect + * rapid daemon relaunches in the lower bands. + */ + + microuptime(&jld_now_tstamp); + + /* + * Ignore usecs in this calculation. + * msecs granularity is close enough. + */ + jld_now_msecs = (jld_now_tstamp.tv_sec * 1000); + + proc_list_lock(); + switch (jetsam_aging_policy) { + case kJetsamAgingPolicyLegacy: + bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; + jld_bucket_count = bucket->count; + bucket = &memstat_bucket[JETSAM_PRIORITY_AGING_BAND1]; + jld_bucket_count += bucket->count; + break; + case kJetsamAgingPolicySysProcsReclaimedFirst: + case kJetsamAgingPolicyAppsReclaimedFirst: + bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; + jld_bucket_count = bucket->count; + bucket = &memstat_bucket[system_procs_aging_band]; + jld_bucket_count += bucket->count; + bucket = &memstat_bucket[applications_aging_band]; + jld_bucket_count += bucket->count; + break; + case kJetsamAgingPolicyNone: + default: + bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; + jld_bucket_count = bucket->count; + break; + } + + bucket = &memstat_bucket[JETSAM_PRIORITY_ELEVATED_INACTIVE]; + elevated_bucket_count = bucket->count; + + proc_list_unlock(); + + /* + * memorystatus_jld_eval_period_msecs is a tunable + * memorystatus_jld_eval_aggressive_count is a tunable + * memorystatus_jld_eval_aggressive_priority_band_max is a tunable + */ + if ( (jld_bucket_count == 0) || + (jld_now_msecs > (jld_timestamp_msecs + memorystatus_jld_eval_period_msecs))) { + + /* + * Refresh evaluation parameters + */ + jld_timestamp_msecs = jld_now_msecs; + jld_idle_kill_candidates = jld_bucket_count; + *jld_idle_kills = 0; + jld_eval_aggressive_count = 0; + jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT; + } + + if (*jld_idle_kills > jld_idle_kill_candidates) { + jld_eval_aggressive_count++; + +#if DEVELOPMENT || DEBUG + printf("memorystatus: aggressive%d: beginning of window: %lld ms, : timestamp now: %lld ms\n", + jld_eval_aggressive_count, + jld_timestamp_msecs, + jld_now_msecs); + printf("memorystatus: aggressive%d: idle candidates: %d, idle kills: %d\n", + jld_eval_aggressive_count, + jld_idle_kill_candidates, + *jld_idle_kills); +#endif /* DEVELOPMENT || DEBUG */ + + if ((jld_eval_aggressive_count == memorystatus_jld_eval_aggressive_count) && + (total_corpses_count() > 0) && (*corpse_list_purged == FALSE)) { + /* + * If we reach this aggressive cycle, corpses might be causing memory pressure. + * So, in an effort to avoid jetsams in the FG band, we will attempt to purge + * corpse memory prior to this final march through JETSAM_PRIORITY_UI_SUPPORT. + */ + task_purge_all_corpses(); + *corpse_list_purged = TRUE; + } + else if (jld_eval_aggressive_count > memorystatus_jld_eval_aggressive_count) { + /* + * Bump up the jetsam priority limit (eg: the bucket index) + * Enforce bucket index sanity. + */ + if ((memorystatus_jld_eval_aggressive_priority_band_max < 0) || + (memorystatus_jld_eval_aggressive_priority_band_max >= MEMSTAT_BUCKET_COUNT)) { + /* + * Do nothing. Stick with the default level. + */ + } else { + jld_priority_band_max = memorystatus_jld_eval_aggressive_priority_band_max; + } + } + + /* Visit elevated processes first */ + while (elevated_bucket_count) { + + elevated_bucket_count--; + + /* + * memorystatus_kill_elevated_process() drops a reference, + * so take another one so we can continue to use this exit reason + * even after it returns. + */ + + os_reason_ref(jetsam_reason); + killed = memorystatus_kill_elevated_process( + cause, + jetsam_reason, + jld_eval_aggressive_count, + &errors); + + if (killed) { + *post_snapshot = TRUE; + if (memorystatus_avail_pages_below_pressure()) { + /* + * Still under pressure. + * Find another pinned processes. + */ + continue; + } else { + return TRUE; + } + } else { + /* + * No pinned processes left to kill. + * Abandon elevated band. + */ + break; + } + } + + /* + * memorystatus_kill_top_process_aggressive() allocates its own + * jetsam_reason so the kMemorystatusKilledVMThrashing cause + * is consistent throughout the aggressive march. + */ + killed = memorystatus_kill_top_process_aggressive( + kMemorystatusKilledVMThrashing, + jld_eval_aggressive_count, + jld_priority_band_max, + &errors); + + if (killed) { + /* Always generate logs after aggressive kill */ + *post_snapshot = TRUE; + *jld_idle_kills = 0; + return TRUE; + } + } + + return FALSE; + } + + return FALSE; +} + + static void memorystatus_thread(void *param __unused, wait_result_t wr __unused) { static boolean_t is_vm_privileged = FALSE; -#if CONFIG_JETSAM boolean_t post_snapshot = FALSE; uint32_t errors = 0; uint32_t hwm_kill = 0; boolean_t sort_flag = TRUE; boolean_t corpse_list_purged = FALSE; - - /* Jetsam Loop Detection - locals */ - memstat_bucket_t *bucket; - int jld_bucket_count = 0; - struct timeval jld_now_tstamp = {0,0}; - uint64_t jld_now_msecs = 0; - int elevated_bucket_count = 0; - - /* Jetsam Loop Detection - statics */ - static uint64_t jld_timestamp_msecs = 0; - static int jld_idle_kill_candidates = 0; /* Number of available processes in band 0,1 at start */ - static int jld_idle_kills = 0; /* Number of procs killed during eval period */ - static int jld_eval_aggressive_count = 0; /* Bumps the max priority in aggressive loop */ - static int32_t jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT; -#endif + int jld_idle_kills = 0; if (is_vm_privileged == FALSE) { /* @@ -3289,12 +3620,10 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) if (vm_restricted_to_single_processor == TRUE) thread_vm_bind_group_add(); - + thread_set_thread_name(current_thread(), "VM_memorystatus"); memorystatus_thread_block(0, memorystatus_thread); } -#if CONFIG_JETSAM - KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_SCAN) | DBG_FUNC_START, memorystatus_available_pages, memorystatus_jld_enabled, memorystatus_jld_eval_period_msecs, memorystatus_jld_eval_aggressive_count,0); @@ -3310,8 +3639,7 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) * If we run out of HWM processes and our available pages drops below the critical threshold, then, * we target the least recently used process in order of increasing jetsam priority (exception: the FG band). */ - while (is_thrashing(kill_under_pressure_cause) || - memorystatus_available_pages <= memorystatus_available_pages_pressure) { + while (memorystatus_action_needed()) { boolean_t killed; int32_t priority; uint32_t cause; @@ -3326,6 +3654,9 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) case kMemorystatusKilledVMThrashing: jetsam_reason_code = JETSAM_REASON_MEMORY_VMTHRASHING; break; + case kMemorystatusKilledZoneMapExhaustion: + jetsam_reason_code = JETSAM_REASON_ZONE_MAP_EXHAUSTION; + break; case kMemorystatusKilledVMPageShortage: /* falls through */ default: @@ -3335,30 +3666,16 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) } /* Highwater */ - killed = memorystatus_kill_hiwat_proc(&errors); - if (killed) { - hwm_kill++; - post_snapshot = TRUE; - goto done; - } else { - memorystatus_hwm_candidates = FALSE; - } - - /* No highwater processes to kill. Continue or stop for now? */ - if (!is_thrashing(kill_under_pressure_cause) && - (memorystatus_available_pages > memorystatus_available_pages_critical)) { - /* - * We are _not_ out of pressure but we are above the critical threshold and there's: - * - no compressor thrashing - * - no more HWM processes left. - * For now, don't kill any other processes. - */ - - if (hwm_kill == 0) { - memorystatus_thread_wasted_wakeup++; + boolean_t is_critical = TRUE; + if (memorystatus_act_on_hiwat_processes(&errors, &hwm_kill, &post_snapshot, &is_critical)) { + if (is_critical == FALSE) { + /* + * For now, don't kill any other processes. + */ + break; + } else { + goto done; } - - break; } jetsam_reason = os_reason_create(OS_REASON_JETSAM, jetsam_reason_code); @@ -3366,166 +3683,8 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) printf("memorystatus_thread: failed to allocate jetsam reason\n"); } - if (memorystatus_jld_enabled == TRUE) { - - /* - * Jetsam Loop Detection: attempt to detect - * rapid daemon relaunches in the lower bands. - */ - - microuptime(&jld_now_tstamp); - - /* - * Ignore usecs in this calculation. - * msecs granularity is close enough. - */ - jld_now_msecs = (jld_now_tstamp.tv_sec * 1000); - - proc_list_lock(); - switch (jetsam_aging_policy) { - case kJetsamAgingPolicyLegacy: - bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; - jld_bucket_count = bucket->count; - bucket = &memstat_bucket[JETSAM_PRIORITY_AGING_BAND1]; - jld_bucket_count += bucket->count; - break; - case kJetsamAgingPolicySysProcsReclaimedFirst: - case kJetsamAgingPolicyAppsReclaimedFirst: - bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; - jld_bucket_count = bucket->count; - bucket = &memstat_bucket[system_procs_aging_band]; - jld_bucket_count += bucket->count; - bucket = &memstat_bucket[applications_aging_band]; - jld_bucket_count += bucket->count; - break; - case kJetsamAgingPolicyNone: - default: - bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; - jld_bucket_count = bucket->count; - break; - } - - bucket = &memstat_bucket[JETSAM_PRIORITY_ELEVATED_INACTIVE]; - elevated_bucket_count = bucket->count; - - proc_list_unlock(); - - /* - * memorystatus_jld_eval_period_msecs is a tunable - * memorystatus_jld_eval_aggressive_count is a tunable - * memorystatus_jld_eval_aggressive_priority_band_max is a tunable - */ - if ( (jld_bucket_count == 0) || - (jld_now_msecs > (jld_timestamp_msecs + memorystatus_jld_eval_period_msecs))) { - - /* - * Refresh evaluation parameters - */ - jld_timestamp_msecs = jld_now_msecs; - jld_idle_kill_candidates = jld_bucket_count; - jld_idle_kills = 0; - jld_eval_aggressive_count = 0; - jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT; - } - - if (jld_idle_kills > jld_idle_kill_candidates) { - jld_eval_aggressive_count++; - -#if DEVELOPMENT || DEBUG - printf("memorystatus: aggressive%d: beginning of window: %lld ms, : timestamp now: %lld ms\n", - jld_eval_aggressive_count, - jld_timestamp_msecs, - jld_now_msecs); - printf("memorystatus: aggressive%d: idle candidates: %d, idle kills: %d\n", - jld_eval_aggressive_count, - jld_idle_kill_candidates, - jld_idle_kills); -#endif /* DEVELOPMENT || DEBUG */ - - if ((jld_eval_aggressive_count == memorystatus_jld_eval_aggressive_count) && - (total_corpses_count > 0) && (corpse_list_purged == FALSE)) { - /* - * If we reach this aggressive cycle, corpses might be causing memory pressure. - * So, in an effort to avoid jetsams in the FG band, we will attempt to purge - * corpse memory prior to this final march through JETSAM_PRIORITY_UI_SUPPORT. - */ - task_purge_all_corpses(); - corpse_list_purged = TRUE; - } - else if (jld_eval_aggressive_count > memorystatus_jld_eval_aggressive_count) { - /* - * Bump up the jetsam priority limit (eg: the bucket index) - * Enforce bucket index sanity. - */ - if ((memorystatus_jld_eval_aggressive_priority_band_max < 0) || - (memorystatus_jld_eval_aggressive_priority_band_max >= MEMSTAT_BUCKET_COUNT)) { - /* - * Do nothing. Stick with the default level. - */ - } else { - jld_priority_band_max = memorystatus_jld_eval_aggressive_priority_band_max; - } - } - - /* Visit elevated processes first */ - while (elevated_bucket_count) { - - elevated_bucket_count--; - - /* - * memorystatus_kill_elevated_process() drops a reference, - * so take another one so we can continue to use this exit reason - * even after it returns. - */ - - os_reason_ref(jetsam_reason); - killed = memorystatus_kill_elevated_process( - kMemorystatusKilledVMThrashing, - jetsam_reason, - jld_eval_aggressive_count, - &errors); - - if (killed) { - post_snapshot = TRUE; - if (memorystatus_available_pages <= memorystatus_available_pages_pressure) { - /* - * Still under pressure. - * Find another pinned processes. - */ - continue; - } else { - goto done; - } - } else { - /* - * No pinned processes left to kill. - * Abandon elevated band. - */ - break; - } - } - - /* - * memorystatus_kill_top_process_aggressive() drops a reference, - * so take another one so we can continue to use this exit reason - * even after it returns - */ - os_reason_ref(jetsam_reason); - killed = memorystatus_kill_top_process_aggressive( - TRUE, - kMemorystatusKilledVMThrashing, - jetsam_reason, - jld_eval_aggressive_count, - jld_priority_band_max, - &errors); - - if (killed) { - /* Always generate logs after aggressive kill */ - post_snapshot = TRUE; - jld_idle_kills = 0; - goto done; - } - } + if (memorystatus_act_aggressive(cause, jetsam_reason, &jld_idle_kills, &corpse_list_purged, &post_snapshot)) { + goto done; } /* @@ -3540,12 +3699,8 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) sort_flag = FALSE; if (killed) { - /* - * Don't generate logs for steady-state idle-exit kills, - * unless it is overridden for debug or by the device - * tree. - */ - if ((priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot) { + if (memorystatus_post_snapshot(priority, cause) == TRUE) { + post_snapshot = TRUE; } @@ -3561,7 +3716,7 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) } } - if ((priority >= JETSAM_PRIORITY_UI_SUPPORT) && (total_corpses_count > 0) && (corpse_list_purged == FALSE)) { + if ((priority >= JETSAM_PRIORITY_UI_SUPPORT) && (total_corpses_count() > 0) && (corpse_list_purged == FALSE)) { /* * If we have jetsammed a process in or above JETSAM_PRIORITY_UI_SUPPORT * then we attempt to relieve pressure by purging corpse memory. @@ -3572,20 +3727,20 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) goto done; } - if (memorystatus_available_pages <= memorystatus_available_pages_critical) { + if (memorystatus_avail_pages_below_critical()) { /* * Still under pressure and unable to kill a process - purge corpse memory */ - if (total_corpses_count > 0) { + if (total_corpses_count() > 0) { task_purge_all_corpses(); corpse_list_purged = TRUE; } - if (memorystatus_available_pages <= memorystatus_available_pages_critical) { + if (memorystatus_avail_pages_below_critical()) { /* * Still under pressure and unable to kill a process - panic */ - panic("memorystatus_jetsam_thread: no victim! available pages:%d\n", memorystatus_available_pages); + panic("memorystatus_jetsam_thread: no victim! available pages:%llu\n", (uint64_t)memorystatus_available_pages); } } @@ -3596,9 +3751,13 @@ done: * To avoid that, we reset the flag here and notify the * compressor. */ - if (is_thrashing(kill_under_pressure_cause)) { + if (is_reason_thrashing(kill_under_pressure_cause)) { kill_under_pressure_cause = 0; +#if CONFIG_JETSAM vm_thrashing_jetsam_done(); +#endif /* CONFIG_JETSAM */ + } else if (is_reason_zone_map_exhaustion(kill_under_pressure_cause)) { + kill_under_pressure_cause = 0; } os_reason_free(jetsam_reason); @@ -3610,17 +3769,6 @@ done: memorystatus_clear_errors(); } -#if VM_PRESSURE_EVENTS - /* - * LD: We used to target the foreground process first and foremost here. - * Now, we target all processes, starting from the non-suspended, background - * processes first. We will target foreground too. - * - * memorystatus_update_vm_pressure(TRUE); - */ - //vm_pressure_response(); -#endif - if (post_snapshot) { proc_list_lock(); size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) + @@ -3645,18 +3793,9 @@ done: KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_SCAN) | DBG_FUNC_END, memorystatus_available_pages, 0, 0, 0, 0); -#else /* CONFIG_JETSAM */ - - /* - * Jetsam not enabled - */ - -#endif /* CONFIG_JETSAM */ - memorystatus_thread_block(0, memorystatus_thread); } -#if !CONFIG_JETSAM /* * Returns TRUE: * when an idle-exitable proc was killed @@ -3665,9 +3804,20 @@ done: * when the attempt to kill an idle-exitable proc failed */ boolean_t memorystatus_idle_exit_from_VM(void) { + + /* + * This routine should no longer be needed since we are + * now using jetsam bands on all platforms and so will deal + * with IDLE processes within the memorystatus thread itself. + * + * But we still use it because we observed that macos systems + * started heavy compression/swapping with a bunch of + * idle-exitable processes alive and doing nothing. We decided + * to rather kill those processes than start swapping earlier. + */ + return(kill_idle_exit_proc()); } -#endif /* !CONFIG_JETSAM */ /* * Callback invoked when allowable physical memory footprint exceeded @@ -3691,7 +3841,7 @@ memorystatus_on_ledger_footprint_exceeded(boolean_t warning, boolean_t memlimit_ */ if (memorystatus_warn_process(p->p_pid, memlimit_is_active, memlimit_is_fatal, FALSE /* not exceeded */) != TRUE) { /* Print warning, since it's possible that task has not registered for pressure notifications */ - printf("task_exceeded_footprint: failed to warn the current task (%d exiting, or no handler registered?).\n", p->p_pid); + os_log(OS_LOG_DEFAULT, "memorystatus_on_ledger_footprint_exceeded: failed to warn the current task (%d exiting, or no handler registered?).\n", p->p_pid); } return; } @@ -3743,9 +3893,10 @@ memorystatus_log_exception(const int max_footprint_mb, boolean_t memlimit_is_act * Hard memory limit is a fatal custom-task-limit or system-wide per-task memory limit. */ - printf("process %d (%s) exceeded physical memory footprint, the %s%sMemoryLimit of %d MB\n", - p->p_pid, (*p->p_name ? p->p_name : "unknown"), (memlimit_is_active ? "Active" : "Inactive"), - (memlimit_is_fatal ? "Hard" : "Soft"), max_footprint_mb); + os_log_with_startup_serial(OS_LOG_DEFAULT, "EXC_RESOURCE -> %s[%d] exceeded mem limit: %s%s %d MB (%s)\n", + (*p->p_name ? p->p_name : "unknown"), p->p_pid, (memlimit_is_active ? "Active" : "Inactive"), + (memlimit_is_fatal ? "Hard" : "Soft"), max_footprint_mb, + (memlimit_is_fatal ? "fatal" : "non-fatal")); return; } @@ -3753,7 +3904,13 @@ memorystatus_log_exception(const int max_footprint_mb, boolean_t memlimit_is_act /* * Description: - * Evaluates active vs. inactive process state. + * Evaluates process state to determine which limit + * should be applied (active vs. inactive limit). + * + * Processes that have the 'elevated inactive jetsam band' attribute + * are first evaluated based on their current priority band. + * presently elevated ==> active + * * Processes that opt into dirty tracking are evaluated * based on clean vs dirty state. * dirty ==> active @@ -3771,7 +3928,15 @@ memorystatus_log_exception(const int max_footprint_mb, boolean_t memlimit_is_act static boolean_t proc_jetsam_state_is_active_locked(proc_t p) { - if (p->p_memstat_dirty & P_DIRTY_TRACK) { + if ((p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) && + (p->p_memstat_effectivepriority == JETSAM_PRIORITY_ELEVATED_INACTIVE)) { + /* + * process has the 'elevated inactive jetsam band' attribute + * and process is present in the elevated band + * implies active state + */ + return TRUE; + } else if (p->p_memstat_dirty & P_DIRTY_TRACK) { /* * process has opted into dirty tracking * active state is based on dirty vs. clean @@ -3808,7 +3973,6 @@ static boolean_t memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason) { boolean_t res; -#if CONFIG_JETSAM uint32_t errors = 0; if (victim_pid == -1) { @@ -3842,12 +4006,7 @@ memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jet proc_list_unlock(); } } -#else /* !CONFIG_JETSAM */ - - res = memorystatus_kill_specific_process(victim_pid, cause, jetsam_reason); -#endif /* CONFIG_JETSAM */ - return res; } @@ -3873,7 +4032,6 @@ memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause, os_reason_t proc_list_lock(); -#if CONFIG_JETSAM if (memorystatus_jetsam_snapshot_count == 0) { memorystatus_init_jetsam_snapshot_locked(NULL,0); } @@ -3886,19 +4044,9 @@ memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause, os_reason_t proc_list_unlock(); - printf("%lu.%02d memorystatus: specifically killing pid %d [%s] (%s %d) - memorystatus_available_pages: %d\n", - (unsigned long)tv_sec, tv_msec, victim_pid, (*p->p_name ? p->p_name : "(unknown)"), - jetsam_kill_cause_name[cause], p->p_memstat_effectivepriority, memorystatus_available_pages); -#else /* !CONFIG_JETSAM */ - proc_list_unlock(); - - killtime = mach_absolute_time(); - absolutetime_to_microtime(killtime, &tv_sec, &tv_usec); - tv_msec = tv_usec / 1000; - printf("%lu.%02d memorystatus: specifically killing pid %d [%s] (%s %d)\n", - (unsigned long)tv_sec, tv_msec, victim_pid, (*p->p_name ? p->p_name : "(unknown)"), - jetsam_kill_cause_name[cause], p->p_memstat_effectivepriority); -#endif /* CONFIG_JETSAM */ + os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: killing_specific_process pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n", + (unsigned long)tv_sec, tv_msec, victim_pid, (*p->p_name ? p->p_name : "unknown"), + memorystatus_kill_cause_name[cause], p->p_memstat_effectivepriority, (uint64_t)memorystatus_available_pages); killed = memorystatus_do_kill(p, cause, jetsam_reason); proc_rele(p); @@ -3964,6 +4112,8 @@ jetsam_on_ledger_cpulimit_exceeded(void) } } +#endif /* CONFIG_JETSAM */ + static void memorystatus_get_task_memory_region_count(task_t task, uint64_t *count) { @@ -3973,6 +4123,69 @@ memorystatus_get_task_memory_region_count(task_t task, uint64_t *count) *count = get_task_memory_region_count(task); } +/* + * Called during EXC_RESOURCE handling when a process exceeds a soft + * memory limit. This is the corpse fork path and here we decide if + * vm_map_fork will be allowed when creating the corpse. + * The current task is suspended. + * + * By default, a vm_map_fork is allowed to proceed. + * + * A few simple policy assumptions: + * Desktop platform is not considered in this path. + * The vm_map_fork is always allowed. + * + * If the device has a zero system-wide task limit, + * then the vm_map_fork is allowed. + * + * And if a process's memory footprint calculates less + * than or equal to half of the system-wide task limit, + * then the vm_map_fork is allowed. This calculation + * is based on the assumption that a process can + * munch memory up to the system-wide task limit. + */ +boolean_t +memorystatus_allowed_vm_map_fork(__unused task_t task) +{ + boolean_t is_allowed = TRUE; /* default */ + +#if CONFIG_EMBEDDED + + uint64_t footprint_in_bytes = 0; + uint64_t purgeable_in_bytes = 0; + uint64_t max_allowed_bytes = 0; + + if (max_task_footprint_mb == 0) { + return (is_allowed); + } + + purgeable_in_bytes = get_task_purgeable_size(task); + footprint_in_bytes = get_task_phys_footprint(task); + + /* + * Maximum is half the system-wide task limit. + */ + max_allowed_bytes = ((((uint64_t)max_task_footprint_mb) * 1024ULL * 1024ULL) >> 1); + + if (footprint_in_bytes > purgeable_in_bytes) { + footprint_in_bytes -= purgeable_in_bytes; + } + + if (footprint_in_bytes <= max_allowed_bytes) { + return (is_allowed); + } else { + printf("memorystatus disallowed vm_map_fork %lld %lld\n", footprint_in_bytes, max_allowed_bytes); + return (!is_allowed); + } + +#else /* CONFIG_EMBEDDED */ + + return (is_allowed); + +#endif /* CONFIG_EMBEDDED */ + +} + static void memorystatus_get_task_page_counts(task_t task, uint32_t *footprint, uint32_t *max_footprint, uint32_t *max_footprint_lifetime, uint32_t *purgeable_pages) { @@ -3986,7 +4199,7 @@ memorystatus_get_task_page_counts(task_t task, uint32_t *footprint, uint32_t *ma *footprint = (uint32_t)pages; if (max_footprint) { - pages = (get_task_phys_footprint_max(task) / PAGE_SIZE_64); + pages = (get_task_phys_footprint_recent_max(task) / PAGE_SIZE_64); assert(((uint32_t)pages) == pages); *max_footprint = (uint32_t)pages; } @@ -4213,6 +4426,7 @@ exit: return; } +#if CONFIG_JETSAM void memorystatus_pages_update(unsigned int pages_avail) { memorystatus_available_pages = pages_avail; @@ -4256,6 +4470,7 @@ void memorystatus_pages_update(unsigned int pages_avail) } #endif /* VM_PRESSURE_EVENTS */ } +#endif /* CONFIG_JETSAM */ static boolean_t memorystatus_init_jetsam_snapshot_entry_locked(proc_t p, memorystatus_jetsam_snapshot_entry_t *entry, uint64_t gencount) @@ -4355,6 +4570,10 @@ memorystatus_init_snapshot_vmstats(memorystatus_jetsam_snapshot_t *snapshot) snapshot->stats.compressor_pages = vm_stat.compressor_page_count; snapshot->stats.total_uncompressed_pages_in_compressor = vm_stat.total_uncompressed_pages_in_compressor; } + + get_zone_map_size(&snapshot->stats.zone_map_size, &snapshot->stats.zone_map_capacity); + get_largest_zone_info(snapshot->stats.largest_zone_name, sizeof(snapshot->stats.largest_zone_name), + &snapshot->stats.largest_zone_size); } /* @@ -4433,6 +4652,7 @@ memorystatus_init_jetsam_snapshot_locked(memorystatus_jetsam_snapshot_t *od_snap #if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM static int memorystatus_cmd_set_panic_bits(user_addr_t buffer, uint32_t buffer_size) { int ret; @@ -4456,6 +4676,7 @@ memorystatus_cmd_set_panic_bits(user_addr_t buffer, uint32_t buffer_size) { return ret; } +#endif /* CONFIG_JETSAM */ /* * Triggers a sort_order on a specified jetsam priority band. @@ -4492,7 +4713,7 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause { pid_t aPid; proc_t p = PROC_NULL, next_p = PROC_NULL; - boolean_t new_snapshot = FALSE, killed = FALSE; + boolean_t new_snapshot = FALSE, force_new_snapshot = FALSE, killed = FALSE; int kill_count = 0; unsigned int i = 0; uint32_t aPid_ep; @@ -4500,6 +4721,7 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause clock_sec_t tv_sec; clock_usec_t tv_usec; uint32_t tv_msec; + int32_t local_max_kill_prio = JETSAM_PRIORITY_IDLE; #ifndef CONFIG_FREEZE #pragma unused(any) @@ -4509,14 +4731,53 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause memorystatus_available_pages, 0, 0, 0, 0); +#if CONFIG_JETSAM + if (sort_flag == TRUE) { + (void)memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT); + } + + local_max_kill_prio = max_kill_priority; + + force_new_snapshot = FALSE; + +#else /* CONFIG_JETSAM */ + if (sort_flag == TRUE) { - (void)memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT); + (void)memorystatus_sort_bucket(JETSAM_PRIORITY_IDLE, JETSAM_SORT_DEFAULT); + } + + /* + * On macos, we currently only have 2 reasons to be here: + * + * kMemorystatusKilledZoneMapExhaustion + * AND + * kMemorystatusKilledVMThrashing + * + * If we are here because of kMemorystatusKilledZoneMapExhaustion, we will consider + * any and all processes as eligible kill candidates since we need to avoid a panic. + * + * Since this function can be called async. it is harder to toggle the max_kill_priority + * value before and after a call. And so we use this local variable to set the upper band + * on the eligible kill bands. + */ + if (cause == kMemorystatusKilledZoneMapExhaustion) { + local_max_kill_prio = JETSAM_PRIORITY_MAX; + } else { + local_max_kill_prio = max_kill_priority; } + /* + * And, because we are here under extreme circumstances, we force a snapshot even for + * IDLE kills. + */ + force_new_snapshot = TRUE; + +#endif /* CONFIG_JETSAM */ + proc_list_lock(); next_p = memorystatus_get_first_proc_locked(&i, TRUE); - while (next_p) { + while (next_p && (next_p->p_memstat_effectivepriority <= local_max_kill_prio)) { #if DEVELOPMENT || DEBUG int activeProcess; int procSuspendedForDiagnosis; @@ -4537,12 +4798,12 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause continue; /* with lock held */ } -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && procSuspendedForDiagnosis) { printf("jetsam: continuing after ignoring proc suspended already for diagnosis - %d\n", aPid); continue; } -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ if (cause == kMemorystatusKilledVnodes) { @@ -4550,8 +4811,8 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause * If the system runs out of vnodes, we systematically jetsam * processes in hopes of stumbling onto a vnode gain that helps * the system recover. The process that happens to trigger - * this path has no known relationship to the vnode consumption. - * We attempt to safeguard that process e.g: do not jetsam it. + * this path has no known relationship to the vnode shortage. + * Deadlock avoidance: attempt to safeguard the caller. */ if (p == current_proc()) { @@ -4576,11 +4837,14 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause { /* * Capture a snapshot if none exists and: + * - we are forcing a new snapshot creation, either because: + * - on a particular platform we need these snapshots every time, OR + * - a boot-arg/embedded device tree property has been set. * - priority was not requested (this is something other than an ambient kill) * - the priority was requested *and* the targeted process is not at idle priority */ if ((memorystatus_jetsam_snapshot_count == 0) && - (memorystatus_idle_snapshot || ((!priority) || (priority && (aPid_ep != JETSAM_PRIORITY_IDLE))))) { + (force_new_snapshot || memorystatus_idle_snapshot || ((!priority) || (priority && (aPid_ep != JETSAM_PRIORITY_IDLE))))) { memorystatus_init_jetsam_snapshot_locked(NULL,0); new_snapshot = TRUE; } @@ -4597,7 +4861,7 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause absolutetime_to_microtime(killtime, &tv_sec, &tv_usec); tv_msec = tv_usec / 1000; -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && activeProcess) { MEMORYSTATUS_DEBUG(1, "jetsam: suspending pid %d [%s] (active) for diagnosis - memory_status_level: %d\n", aPid, (*p->p_name ? p->p_name: "(unknown)"), memorystatus_level); @@ -4621,18 +4885,18 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause goto exit; } else -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ { /* Shift queue, update stats */ memorystatus_update_jetsam_snapshot_entry_locked(p, cause, killtime); if (proc_ref_locked(p) == p) { proc_list_unlock(); - printf("%lu.%02d memorystatus: %s %d [%s] (%s %d) - memorystatus_available_pages: %d\n", + os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: %s pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n", (unsigned long)tv_sec, tv_msec, - ((aPid_ep == JETSAM_PRIORITY_IDLE) ? "idle exiting pid" : "jetsam killing top process pid"), - aPid, (*p->p_name ? p->p_name : "(unknown)"), - jetsam_kill_cause_name[cause], aPid_ep, memorystatus_available_pages); + ((aPid_ep == JETSAM_PRIORITY_IDLE) ? "killing_idle_process" : "killing_top_process"), + aPid, (*p->p_name ? p->p_name : "unknown"), + memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages); /* * memorystatus_do_kill() drops a reference, so take another one so we can @@ -4704,7 +4968,7 @@ exit: * Jetsam aggressively */ static boolean_t -memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reason_t jetsam_reason, int aggr_count, +memorystatus_kill_top_process_aggressive(uint32_t cause, int aggr_count, int32_t priority_max, uint32_t *errors) { pid_t aPid; @@ -4718,14 +4982,18 @@ memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reaso clock_sec_t tv_sec; clock_usec_t tv_usec; uint32_t tv_msec; - -#pragma unused(any) + os_reason_t jetsam_reason = OS_REASON_NULL; KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_START, memorystatus_available_pages, priority_max, 0, 0, 0); memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT); + jetsam_reason = os_reason_create(OS_REASON_JETSAM, cause); + if (jetsam_reason == OS_REASON_NULL) { + printf("memorystatus_kill_top_process_aggressive: failed to allocate exit reason\n"); + } + proc_list_lock(); next_p = memorystatus_get_first_proc_locked(&i, TRUE); @@ -4735,20 +5003,22 @@ memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reaso int procSuspendedForDiagnosis; #endif /* DEVELOPMENT || DEBUG */ - if ((unsigned int)(next_p->p_memstat_effectivepriority) != i) { + if (((next_p->p_listflag & P_LIST_EXITED) != 0) || + ((unsigned int)(next_p->p_memstat_effectivepriority) != i)) { /* - * We have raced with next_p running on another core, as it has - * moved to a different jetsam priority band. This means we have - * lost our place in line while traversing the jetsam list. We + * We have raced with next_p running on another core. + * It may be exiting or it may have moved to a different + * jetsam priority band. This means we have lost our + * place in line while traversing the jetsam list. We * attempt to recover by rewinding to the beginning of the band * we were already traversing. By doing this, we do not guarantee * that no process escapes this aggressive march, but we can make * skipping an entire range of processes less likely. (PR-21069019) */ - MEMORYSTATUS_DEBUG(1, "memorystatus: aggressive%d: rewinding %s moved from band %d --> %d\n", - aggr_count, (*next_p->p_name ? next_p->p_name : "unknown"), i, next_p->p_memstat_effectivepriority); + MEMORYSTATUS_DEBUG(1, "memorystatus: aggressive%d: rewinding band %d, %s(%d) moved or exiting.\n", + aggr_count, i, (*next_p->p_name ? next_p->p_name : "unknown"), next_p->p_pid); next_p = memorystatus_get_first_proc_locked(&i, TRUE); continue; @@ -4780,12 +5050,12 @@ memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reaso continue; } -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && procSuspendedForDiagnosis) { printf("jetsam: continuing after ignoring proc suspended already for diagnosis - %d\n", aPid); continue; } -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ /* * Capture a snapshot if none exists. @@ -4836,11 +5106,11 @@ memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reaso } proc_list_unlock(); - printf("%lu.%01d memorystatus: aggressive%d: %s %d [%s] (%s %d) - memorystatus_available_pages: %d\n", - (unsigned long)tv_sec, tv_msec, aggr_count, - ((aPid_ep == JETSAM_PRIORITY_IDLE) ? "idle exiting pid" : "jetsam killing pid"), - aPid, (*p->p_name ? p->p_name : "(unknown)"), - jetsam_kill_cause_name[cause], aPid_ep, memorystatus_available_pages); + printf("%lu.%03d memorystatus: %s%d pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n", + (unsigned long)tv_sec, tv_msec, + ((aPid_ep == JETSAM_PRIORITY_IDLE) ? "killing_idle_process_aggressive" : "killing_top_process_aggressive"), + aggr_count, aPid, (*p->p_name ? p->p_name : "unknown"), + memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages); memorystatus_level_snapshot = memorystatus_level; @@ -4977,30 +5247,17 @@ memorystatus_kill_hiwat_proc(uint32_t *errors) continue; } -#if 0 - /* - * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore. - * Background limits are described via the inactive limit slots. - * Their fatal/non-fatal setting will drive whether or not to be - * considered in this kill path. - */ - - /* skip if a currently inapplicable limit is encountered */ - if ((p->p_memstat_state & P_MEMSTAT_MEMLIMIT_BACKGROUND) && (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND)) { - continue; - } -#endif footprint_in_bytes = get_task_phys_footprint(p->task); memlimit_in_bytes = (((uint64_t)p->p_memstat_memlimit) * 1024ULL * 1024ULL); /* convert MB to bytes */ skip = (footprint_in_bytes <= memlimit_in_bytes); -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) if (!skip && (memorystatus_jetsam_policy & kPolicyDiagnoseActive)) { if (p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED) { continue; } } -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ #if CONFIG_FREEZE if (!skip) { @@ -5015,13 +5272,13 @@ memorystatus_kill_hiwat_proc(uint32_t *errors) if (skip) { continue; } else { -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) MEMORYSTATUS_DEBUG(1, "jetsam: %s pid %d [%s] - %lld Mb > 1 (%d Mb)\n", (memorystatus_jetsam_policy & kPolicyDiagnoseActive) ? "suspending": "killing", aPid, (*p->p_name ? p->p_name : "unknown"), (footprint_in_bytes / (1024ULL * 1024ULL)), /* converted bytes to MB */ p->p_memstat_memlimit); -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ if (memorystatus_jetsam_snapshot_count == 0) { memorystatus_init_jetsam_snapshot_locked(NULL,0); @@ -5034,7 +5291,7 @@ memorystatus_kill_hiwat_proc(uint32_t *errors) absolutetime_to_microtime(killtime, &tv_sec, &tv_usec); tv_msec = tv_usec / 1000; -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) { MEMORYSTATUS_DEBUG(1, "jetsam: pid %d suspended for diagnosis - memorystatus_available_pages: %d\n", aPid, memorystatus_available_pages); memorystatus_update_jetsam_snapshot_entry_locked(p, kMemorystatusKilledDiagnostic, killtime); @@ -5050,15 +5307,15 @@ memorystatus_kill_hiwat_proc(uint32_t *errors) goto exit; } else -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ { memorystatus_update_jetsam_snapshot_entry_locked(p, kMemorystatusKilledHiwat, killtime); if (proc_ref_locked(p) == p) { proc_list_unlock(); - printf("%lu.%02d memorystatus: jetsam killing pid %d [%s] (highwater %d) - memorystatus_available_pages: %d\n", - (unsigned long)tv_sec, tv_msec, aPid, (*p->p_name ? p->p_name : "(unknown)"), aPid_ep, memorystatus_available_pages); + os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: killing_highwater_process pid %d [%s] (highwater %d) - memorystatus_available_pages: %llu\n", + (unsigned long)tv_sec, tv_msec, aPid, (*p->p_name ? p->p_name : "unknown"), aPid_ep, (uint64_t)memorystatus_available_pages); /* * memorystatus_do_kill drops a reference, so take another one so we can @@ -5199,11 +5456,11 @@ memorystatus_kill_elevated_process(uint32_t cause, os_reason_t jetsam_reason, in proc_list_unlock(); - printf("%lu.%01d memorystatus: elevated%d: jetsam killing pid %d [%s] (%s %d) - memorystatus_available_pages: %d\n", + os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: killing_top_process_elevated%d pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n", (unsigned long)tv_sec, tv_msec, aggr_count, - aPid, (*p->p_name ? p->p_name : "(unknown)"), - jetsam_kill_cause_name[cause], aPid_ep, memorystatus_available_pages); + aPid, (*p->p_name ? p->p_name : "unknown"), + memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages); /* * memorystatus_do_kill drops a reference, so take another one so we can @@ -5273,7 +5530,7 @@ memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause) { * add the appropriate exit reason code mapping. */ if ((victim_pid != -1) || (cause != kMemorystatusKilledVMPageShortage && cause != kMemorystatusKilledVMThrashing && - cause != kMemorystatusKilledFCThrashing)) { + cause != kMemorystatusKilledFCThrashing && cause != kMemorystatusKilledZoneMapExhaustion)) { return FALSE; } @@ -5282,31 +5539,32 @@ memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause) { return TRUE; } -boolean_t -memorystatus_kill_on_VM_page_shortage(boolean_t async) { +boolean_t +memorystatus_kill_on_VM_thrashing(boolean_t async) { if (async) { - return memorystatus_kill_process_async(-1, kMemorystatusKilledVMPageShortage); + return memorystatus_kill_process_async(-1, kMemorystatusKilledVMThrashing); } else { - os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMPAGESHORTAGE); + os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMTHRASHING); if (jetsam_reason == OS_REASON_NULL) { - printf("memorystatus_kill_on_VM_page_shortage -- sync: failed to allocate jetsam reason\n"); + printf("memorystatus_kill_on_VM_thrashing -- sync: failed to allocate jetsam reason\n"); } - return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMPageShortage, jetsam_reason); + return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMThrashing, jetsam_reason); } } -boolean_t -memorystatus_kill_on_VM_thrashing(boolean_t async) { +#if CONFIG_JETSAM +boolean_t +memorystatus_kill_on_VM_page_shortage(boolean_t async) { if (async) { - return memorystatus_kill_process_async(-1, kMemorystatusKilledVMThrashing); + return memorystatus_kill_process_async(-1, kMemorystatusKilledVMPageShortage); } else { - os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMTHRASHING); + os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMPAGESHORTAGE); if (jetsam_reason == OS_REASON_NULL) { - printf("memorystatus_kill_on_VM_thrashing -- sync: failed to allocate jetsam reason\n"); + printf("memorystatus_kill_on_VM_page_shortage -- sync: failed to allocate jetsam reason\n"); } - return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMThrashing, jetsam_reason); + return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMPageShortage, jetsam_reason); } } @@ -5338,6 +5596,22 @@ memorystatus_kill_on_vnode_limit(void) { #endif /* CONFIG_JETSAM */ +boolean_t +memorystatus_kill_on_zone_map_exhaustion(pid_t pid) { + boolean_t res = FALSE; + if (pid == -1) { + res = memorystatus_kill_process_async(-1, kMemorystatusKilledZoneMapExhaustion); + } else { + os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_ZONE_MAP_EXHAUSTION); + if (jetsam_reason == OS_REASON_NULL) { + printf("memorystatus_kill_on_zone_map_exhaustion: failed to allocate jetsam reason\n"); + } + + res = memorystatus_kill_process_sync(pid, kMemorystatusKilledZoneMapExhaustion, jetsam_reason); + } + return res; +} + #if CONFIG_FREEZE __private_extern__ void @@ -5873,6 +6147,38 @@ memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused bool * filt_memorystatus(). */ +#if CONFIG_EMBEDDED + if (!limit_exceeded) { + /* + * Intentionally set either the unambiguous limit warning, + * the system-wide critical or the system-wide warning + * notification bit. + */ + + if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) { + kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN; + found_knote = TRUE; + send_knote_count++; + } else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) { + kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL; + found_knote = TRUE; + send_knote_count++; + } else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) { + kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN; + found_knote = TRUE; + send_knote_count++; + } + } else { + /* + * Send this notification when a process has exceeded a soft limit. + */ + if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) { + kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL; + found_knote = TRUE; + send_knote_count++; + } + } +#else /* CONFIG_EMBEDDED */ if (!limit_exceeded) { /* @@ -5969,6 +6275,7 @@ memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused bool } } } +#endif /* CONFIG_EMBEDDED */ } } @@ -6091,7 +6398,11 @@ vm_pressure_select_optimal_candidate_to_notify(struct klist *, int, boolean_t); /* * This value is the threshold that a process must meet to be considered for scavenging. */ +#if CONFIG_EMBEDDED +#define VM_PRESSURE_MINIMUM_RSIZE 1 /* MB */ +#else /* CONFIG_EMBEDDED */ #define VM_PRESSURE_MINIMUM_RSIZE 10 /* MB */ +#endif /* CONFIG_EMBEDDED */ #define VM_PRESSURE_NOTIFY_WAIT_PERIOD 10000 /* milliseconds */ @@ -6213,7 +6524,7 @@ vm_pressure_select_optimal_candidate_to_notify(struct klist *candidate_list, int pressure_increase = TRUE; } else { - if (level >= pressure_snapshot) { + if (level && (level >= pressure_snapshot)) { pressure_increase = TRUE; } else { pressure_increase = FALSE; @@ -6286,7 +6597,11 @@ vm_pressure_select_optimal_candidate_to_notify(struct klist *candidate_list, int } #endif /* CONFIG_MEMORYSTATUS */ +#if CONFIG_EMBEDDED + curr_task_importance = p->p_memstat_effectivepriority; +#else /* CONFIG_EMBEDDED */ curr_task_importance = task_importance_estimate(t); +#endif /* CONFIG_EMBEDDED */ /* * Privileged listeners are only considered in the multi-level pressure scheme @@ -6462,15 +6777,24 @@ memorystatus_update_vm_pressure(boolean_t target_foreground_process) if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) { - if (curr_ts < next_warning_notification_sent_at_ts) { - delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */); - return KERN_SUCCESS; + if (next_warning_notification_sent_at_ts) { + if (curr_ts < next_warning_notification_sent_at_ts) { + delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */); + return KERN_SUCCESS; + } + + next_warning_notification_sent_at_ts = 0; + memorystatus_klist_reset_all_for_level(kVMPressureWarning); } } else if (level_snapshot == kVMPressureCritical) { - if (curr_ts < next_critical_notification_sent_at_ts) { - delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */); - return KERN_SUCCESS; + if (next_critical_notification_sent_at_ts) { + if (curr_ts < next_critical_notification_sent_at_ts) { + delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */); + return KERN_SUCCESS; + } + next_critical_notification_sent_at_ts = 0; + memorystatus_klist_reset_all_for_level(kVMPressureCritical); } } } @@ -6523,16 +6847,16 @@ memorystatus_update_vm_pressure(boolean_t target_foreground_process) if (level_snapshot != kVMPressureNormal) { if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) { nanoseconds_to_absolutetime(WARNING_NOTIFICATION_RESTING_PERIOD * NSEC_PER_SEC, &curr_ts); - next_warning_notification_sent_at_ts = mach_absolute_time() + curr_ts; - memorystatus_klist_reset_all_for_level(kVMPressureWarning); + /* Next warning notification (if nothing changes) won't be sent before...*/ + next_warning_notification_sent_at_ts = mach_absolute_time() + curr_ts; } if (level_snapshot == kVMPressureCritical) { nanoseconds_to_absolutetime(CRITICAL_NOTIFICATION_RESTING_PERIOD * NSEC_PER_SEC, &curr_ts); - next_critical_notification_sent_at_ts = mach_absolute_time() + curr_ts; - memorystatus_klist_reset_all_for_level(kVMPressureCritical); + /* Next critical notification (if nothing changes) won't be sent before...*/ + next_critical_notification_sent_at_ts = mach_absolute_time() + curr_ts; } } return KERN_FAILURE; @@ -6686,6 +7010,14 @@ static int sysctl_memorystatus_vm_pressure_level SYSCTL_HANDLER_ARGS { #pragma unused(arg1, arg2, oidp) +#if CONFIG_EMBEDDED + int error = 0; + + error = priv_check_cred(kauth_cred_get(), PRIV_VM_PRESSURE, 0); + if (error) + return (error); + +#endif /* CONFIG_EMBEDDED */ vm_pressure_level_t dispatch_level = convert_internal_pressure_level_to_dispatch_level(memorystatus_vm_pressure_level); return SYSCTL_OUT(req, &dispatch_level, sizeof(dispatch_level)); @@ -6860,12 +7192,6 @@ memorystatus_get_priority_list(memorystatus_priority_entry_t **list_ptr, size_t list_entry->priority = p->p_memstat_effectivepriority; list_entry->user_data = p->p_memstat_userdata; - /* - * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore. - * Background limits are described via the inactive limit slots. - * So, here, the cached limit should always be valid. - */ - if (p->p_memstat_memlimit <= 0) { task_get_phys_footprint_limit(p->task, &list_entry->limit); } else { @@ -6888,37 +7214,78 @@ memorystatus_get_priority_list(memorystatus_priority_entry_t **list_ptr, size_t } static int -memorystatus_cmd_get_priority_list(user_addr_t buffer, size_t buffer_size, int32_t *retval) { - int error = EINVAL; +memorystatus_get_priority_pid(pid_t pid, user_addr_t buffer, size_t buffer_size) { + int error = 0; + memorystatus_priority_entry_t mp_entry; + + /* Validate inputs */ + if ((pid == 0) || (buffer == USER_ADDR_NULL) || (buffer_size != sizeof(memorystatus_priority_entry_t))) { + return EINVAL; + } + + proc_t p = proc_find(pid); + if (!p) { + return ESRCH; + } + + memset (&mp_entry, 0, sizeof(memorystatus_priority_entry_t)); + + mp_entry.pid = p->p_pid; + mp_entry.priority = p->p_memstat_effectivepriority; + mp_entry.user_data = p->p_memstat_userdata; + if (p->p_memstat_memlimit <= 0) { + task_get_phys_footprint_limit(p->task, &mp_entry.limit); + } else { + mp_entry.limit = p->p_memstat_memlimit; + } + mp_entry.state = memorystatus_build_state(p); + + proc_rele(p); + + error = copyout(&mp_entry, buffer, buffer_size); + + return (error); +} + +static int +memorystatus_cmd_get_priority_list(pid_t pid, user_addr_t buffer, size_t buffer_size, int32_t *retval) { + int error = 0; boolean_t size_only; - memorystatus_priority_entry_t *list = NULL; size_t list_size; + + /* + * When a non-zero pid is provided, the 'list' has only one entry. + */ size_only = ((buffer == USER_ADDR_NULL) ? TRUE: FALSE); - - error = memorystatus_get_priority_list(&list, &buffer_size, &list_size, size_only); - if (error) { - goto out; - } - if (!size_only) { - error = copyout(list, buffer, list_size); + if (pid != 0) { + list_size = sizeof(memorystatus_priority_entry_t) * 1; + if (!size_only) { + error = memorystatus_get_priority_pid(pid, buffer, buffer_size); + } + } else { + memorystatus_priority_entry_t *list = NULL; + error = memorystatus_get_priority_list(&list, &buffer_size, &list_size, size_only); + + if (error == 0) { + if (!size_only) { + error = copyout(list, buffer, list_size); + } + } + + if (list) { + kfree(list, buffer_size); + } } - + if (error == 0) { *retval = list_size; } -out: - - if (list) { - kfree(list, buffer_size); - } - return error; + return (error); } -#if CONFIG_JETSAM - static void memorystatus_clear_errors(void) { @@ -6942,6 +7309,7 @@ memorystatus_clear_errors(void) KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CLEAR_ERRORS) | DBG_FUNC_END, 0, 0, 0, 0, 0); } +#if CONFIG_JETSAM static void memorystatus_update_levels_locked(boolean_t critical_only) { @@ -6974,7 +7342,7 @@ memorystatus_update_levels_locked(boolean_t critical_only) { memorystatus_available_pages_critical = memorystatus_available_pages_pressure; } } -#endif +#endif /* DEBUG || DEVELOPMENT */ if (memorystatus_jetsam_policy & kPolicyMoreFree) { memorystatus_available_pages_critical += memorystatus_policy_more_free_offset_pages; @@ -6994,6 +7362,7 @@ memorystatus_update_levels_locked(boolean_t critical_only) { #endif } + static int sysctl_kern_memorystatus_policy_more_free SYSCTL_HANDLER_ARGS { @@ -7038,6 +7407,8 @@ sysctl_kern_memorystatus_policy_more_free SYSCTL_HANDLER_ARGS SYSCTL_PROC(_kern, OID_AUTO, memorystatus_policy_more_free, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED, 0, 0, &sysctl_kern_memorystatus_policy_more_free, "I", ""); +#endif /* CONFIG_JETSAM */ + /* * Get the at_boot snapshot */ @@ -7466,7 +7837,7 @@ memorystatus_cmd_set_priority_properties(pid_t pid, user_addr_t buffer, size_t b return EPERM; } - error = memorystatus_update(p, mpp_entry.priority, mpp_entry.user_data, FALSE, FALSE, 0, 0, FALSE, FALSE, FALSE); + error = memorystatus_update(p, mpp_entry.priority, mpp_entry.user_data, FALSE, FALSE, 0, 0, FALSE, FALSE); proc_rele(p); } @@ -7643,6 +8014,7 @@ memorystatus_get_pressure_status_kdp() { * - so mapping is (active/fatal, inactive/fatal) */ +#if CONFIG_JETSAM static int memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit) { int error = 0; @@ -7661,6 +8033,7 @@ memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __u error = memorystatus_set_memlimit_properties(pid, &entry); return (error); } +#endif /* CONFIG_JETSAM */ static int memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry) { @@ -7754,10 +8127,6 @@ memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties if (memorystatus_highwater_enabled) { boolean_t is_fatal; boolean_t use_active; - /* - * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore. - * Background limits are described via the inactive limit slots. - */ if (proc_jetsam_state_is_active_locked(p) == TRUE) { CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal); @@ -7800,8 +8169,6 @@ proc_get_memstat_priority(proc_t p, boolean_t effective_priority) return 0; } -#endif /* CONFIG_JETSAM */ - int memorystatus_control(struct proc *p __unused, struct memorystatus_control_args *args, int *ret) { int error = EINVAL; @@ -7831,9 +8198,8 @@ memorystatus_control(struct proc *p __unused, struct memorystatus_control_args * switch (args->command) { case MEMORYSTATUS_CMD_GET_PRIORITY_LIST: - error = memorystatus_cmd_get_priority_list(args->buffer, args->buffersize, ret); + error = memorystatus_cmd_get_priority_list(args->pid, args->buffer, args->buffersize, ret); break; -#if CONFIG_JETSAM case MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES: error = memorystatus_cmd_set_priority_properties(args->pid, args->buffer, args->buffersize, ret); break; @@ -7855,6 +8221,7 @@ memorystatus_control(struct proc *p __unused, struct memorystatus_control_args * case MEMORYSTATUS_CMD_GET_PRESSURE_STATUS: error = memorystatus_cmd_get_pressure_status(ret); break; +#if CONFIG_JETSAM case MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK: /* * This call does not distinguish between active and inactive limits. @@ -7871,6 +8238,7 @@ memorystatus_control(struct proc *p __unused, struct memorystatus_control_args * */ error = memorystatus_cmd_set_jetsam_memory_limit(args->pid, (int32_t)args->flags, ret, TRUE); break; +#endif /* CONFIG_JETSAM */ /* Test commands */ #if DEVELOPMENT || DEBUG case MEMORYSTATUS_CMD_TEST_JETSAM: @@ -7884,9 +8252,11 @@ memorystatus_control(struct proc *p __unused, struct memorystatus_control_args * case MEMORYSTATUS_CMD_TEST_JETSAM_SORT: error = memorystatus_cmd_test_jetsam_sort(args->pid, (int32_t)args->flags); break; +#if CONFIG_JETSAM case MEMORYSTATUS_CMD_SET_JETSAM_PANIC_BITS: error = memorystatus_cmd_set_panic_bits(args->buffer, args->buffersize); break; +#endif /* CONFIG_JETSAM */ #else /* DEVELOPMENT || DEBUG */ #pragma unused(jetsam_reason) #endif /* DEVELOPMENT || DEBUG */ @@ -7909,18 +8279,15 @@ memorystatus_control(struct proc *p __unused, struct memorystatus_control_args * memorystatus_aggressive_jetsam_lenient = FALSE; error = 0; break; -#endif /* CONFIG_JETSAM */ case MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_ENABLE: case MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_DISABLE: error = memorystatus_low_mem_privileged_listener(args->command); break; -#if CONFIG_JETSAM case MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE: case MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_DISABLE: error = memorystatus_update_inactive_jetsam_priority_band(args->pid, args->command, args->flags ? TRUE : FALSE); break; -#endif /* CONFIG_JETSAM */ default: break; @@ -7932,7 +8299,7 @@ out: static int -filt_memorystatusattach(struct knote *kn) +filt_memorystatusattach(struct knote *kn, __unused struct kevent_internal_s *kev) { int error; @@ -8025,6 +8392,7 @@ filt_memorystatustouch(struct knote *kn, struct kevent_internal_s *kev) prev_kn_sfflags = kn->kn_sfflags; kn->kn_sfflags = (kev->fflags & EVFILT_MEMORYSTATUS_ALL_MASK); +#if !CONFIG_EMBEDDED /* * Only on desktop do we restrict notifications to * one per active/inactive state (soft limits only). @@ -8078,6 +8446,7 @@ filt_memorystatustouch(struct knote *kn, struct kevent_internal_s *kev) kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE; } } +#endif /* !CONFIG_EMBEDDED */ if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) kn->kn_udata = kev->udata; @@ -8140,8 +8509,9 @@ memorystatus_knote_register(struct knote *kn) { /* * Support only userspace visible flags. */ - if ((kn->kn_sfflags & EVFILT_MEMORYSTATUS_ALL_MASK) == kn->kn_sfflags) { + if ((kn->kn_sfflags & EVFILT_MEMORYSTATUS_ALL_MASK) == (unsigned int) kn->kn_sfflags) { +#if !CONFIG_EMBEDDED if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) { kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE; kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE; @@ -8151,6 +8521,7 @@ memorystatus_knote_register(struct knote *kn) { kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE; kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE; } +#endif /* !CONFIG_EMBEDDED */ KNOTE_ATTACH(&memorystatus_klist, kn); @@ -8183,7 +8554,6 @@ memorystatus_issue_pressure_kevent(boolean_t pressured) { #endif /* CONFIG_JETSAM && VM_PRESSURE_EVENTS */ #endif /* 0 */ -#if CONFIG_JETSAM /* Coalition support */ /* sorting info for a particular priority bucket */ @@ -8445,4 +8815,131 @@ memorystatus_move_list_locked(unsigned int bucket_index, pid_t *pid_list, int li } return(found_pids); } -#endif /* CONFIG_JETSAM */ + +int +memorystatus_get_proccnt_upto_priority(int32_t max_bucket_index) +{ + int32_t i = JETSAM_PRIORITY_IDLE; + int count = 0; + + if (max_bucket_index >= MEMSTAT_BUCKET_COUNT) { + return(-1); + } + + while(i <= max_bucket_index) { + count += memstat_bucket[i++].count; + } + + return count; +} + +int +memorystatus_update_priority_for_appnap(proc_t p, boolean_t is_appnap) +{ +#if !CONFIG_JETSAM + if (!p || (!isApp(p)) || (p->p_memstat_state & P_MEMSTAT_INTERNAL)) { + /* + * Ineligible processes OR system processes e.g. launchd. + */ + return -1; + } + + /* + * For macOS only: + * We would like to use memorystatus_update() here to move the processes + * within the bands. Unfortunately memorystatus_update() calls + * memorystatus_update_priority_locked() which uses any band transitions + * as an indication to modify ledgers. For that it needs the task lock + * and since we came into this function with the task lock held, we'll deadlock. + * + * Unfortunately we can't completely disable ledger updates because we still + * need the ledger updates for a subset of processes i.e. daemons. + * When all processes on all platforms support memory limits, we can simply call + * memorystatus_update(). + + * It also has some logic to deal with 'aging' which, currently, is only applicable + * on CONFIG_JETSAM configs. So, till every platform has CONFIG_JETSAM we'll need + * to do this explicit band transition. + */ + + memstat_bucket_t *current_bucket, *new_bucket; + int32_t priority = 0; + + proc_list_lock(); + + if (((p->p_listflag & P_LIST_EXITED) != 0) || + (p->p_memstat_state & (P_MEMSTAT_ERROR | P_MEMSTAT_TERMINATED))) { + /* + * If the process is on its way out OR + * jetsam has alread tried and failed to kill this process, + * let's skip the whole jetsam band transition. + */ + proc_list_unlock(); + return(0); + } + + if (is_appnap) { + current_bucket = &memstat_bucket[p->p_memstat_effectivepriority]; + new_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; + priority = JETSAM_PRIORITY_IDLE; + } else { + if (p->p_memstat_effectivepriority != JETSAM_PRIORITY_IDLE) { + /* + * It is possible that someone pulled this process + * out of the IDLE band without updating its app-nap + * parameters. + */ + proc_list_unlock(); + return (0); + } + + current_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; + new_bucket = &memstat_bucket[p->p_memstat_requestedpriority]; + priority = p->p_memstat_requestedpriority; + } + + TAILQ_REMOVE(¤t_bucket->list, p, p_memstat_list); + current_bucket->count--; + + TAILQ_INSERT_TAIL(&new_bucket->list, p, p_memstat_list); + new_bucket->count++; + + /* + * Record idle start or idle delta. + */ + if (p->p_memstat_effectivepriority == priority) { + /* + * This process is not transitioning between + * jetsam priority buckets. Do nothing. + */ + } else if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) { + uint64_t now; + /* + * Transitioning out of the idle priority bucket. + * Record idle delta. + */ + assert(p->p_memstat_idle_start != 0); + now = mach_absolute_time(); + if (now > p->p_memstat_idle_start) { + p->p_memstat_idle_delta = now - p->p_memstat_idle_start; + } + } else if (priority == JETSAM_PRIORITY_IDLE) { + /* + * Transitioning into the idle priority bucket. + * Record idle start. + */ + p->p_memstat_idle_start = mach_absolute_time(); + } + + p->p_memstat_effectivepriority = priority; + + proc_list_unlock(); + + return (0); + +#else /* !CONFIG_JETSAM */ + #pragma unused(p) + #pragma unused(is_appnap) + return -1; +#endif /* !CONFIG_JETSAM */ +} diff --git a/bsd/kern/kern_mib.c b/bsd/kern/kern_mib.c index 9ef3a2479..dd57cd722 100644 --- a/bsd/kern/kern_mib.c +++ b/bsd/kern/kern_mib.c @@ -119,6 +119,9 @@ extern vm_map_t bsd_pageable_map; #include /* for cpuid_info() */ #endif +#if defined(__arm__) || defined(__arm64__) +#include /* for cpuid_info() & cache_info() */ +#endif #ifndef MAX @@ -337,7 +340,11 @@ sysctl_pagesize32 (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { long long l; +#if __arm64__ + l = (long long) (1 << page_shift_user32); +#else /* __arm64__ */ l = (long long) PAGE_SIZE; +#endif /* __arm64__ */ return sysctl_io_number(req, l, sizeof(l), NULL, NULL); } @@ -367,17 +374,28 @@ SYSCTL_OPAQUE (_hw, OID_AUTO, cacheconfig, CTLFLAG_RD | CTLFLAG_LOCKED, &cachec SYSCTL_OPAQUE (_hw, OID_AUTO, cachesize, CTLFLAG_RD | CTLFLAG_LOCKED, &cachesize, sizeof(cachesize), "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, pagesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, 0, sysctl_pagesize, "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, pagesize32, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, 0, sysctl_pagesize32, "Q", ""); +#if DEBUG || DEVELOPMENT || (!defined(__arm__) && !defined(__arm64__)) SYSCTL_QUAD (_hw, OID_AUTO, busfrequency, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.bus_frequency_hz, ""); SYSCTL_QUAD (_hw, OID_AUTO, busfrequency_min, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.bus_frequency_min_hz, ""); SYSCTL_QUAD (_hw, OID_AUTO, busfrequency_max, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.bus_frequency_max_hz, ""); SYSCTL_QUAD (_hw, OID_AUTO, cpufrequency, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.cpu_frequency_hz, ""); SYSCTL_QUAD (_hw, OID_AUTO, cpufrequency_min, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.cpu_frequency_min_hz, ""); SYSCTL_QUAD (_hw, OID_AUTO, cpufrequency_max, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.cpu_frequency_max_hz, ""); +#endif SYSCTL_PROC (_hw, OID_AUTO, cachelinesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, HW_CACHELINE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, l1icachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, HW_L1ICACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, l1dcachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, HW_L1DCACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, l2cachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, HW_L2CACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, l3cachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, HW_L3CACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", ""); +#if (defined(__arm__) || defined(__arm64__)) && (DEBUG || DEVELOPMENT) +SYSCTL_QUAD (_hw, OID_AUTO, memfrequency, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.mem_frequency_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, memfrequency_min, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.mem_frequency_min_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, memfrequency_max, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.mem_frequency_max_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, prffrequency, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.prf_frequency_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, prffrequency_min, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.prf_frequency_min_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, prffrequency_max, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.prf_frequency_max_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, fixfrequency, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.fix_frequency_hz, ""); +#endif /* __arm__ || __arm64__ */ SYSCTL_PROC(_hw, OID_AUTO, tbfrequency, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, 0, sysctl_tbfrequency, "Q", ""); SYSCTL_QUAD (_hw, HW_MEMSIZE, memsize, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &max_mem, ""); SYSCTL_INT (_hw, OID_AUTO, packages, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &packages, 0, ""); @@ -412,8 +430,10 @@ SYSCTL_NODE(_hw, OID_AUTO, features, CTLFLAG_RD | CTLFLAG_LOCKED, NULL, "hardwar * The *_compat nodes are *NOT* visible within the kernel. */ SYSCTL_PROC(_hw, HW_PAGESIZE, pagesize_compat, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, 0, HW_PAGESIZE, sysctl_hw_generic, "I", ""); +#if DEBUG || DEVELOPMENT || (!defined(__arm__) && !defined(__arm64__)) SYSCTL_COMPAT_INT (_hw, HW_BUS_FREQ, busfrequency_compat, CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.bus_clock_rate_hz, 0, ""); SYSCTL_COMPAT_INT (_hw, HW_CPU_FREQ, cpufrequency_compat, CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.cpu_clock_rate_hz, 0, ""); +#endif SYSCTL_PROC(_hw, HW_CACHELINE, cachelinesize_compat, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, 0, HW_CACHELINE, sysctl_hw_generic, "I", ""); SYSCTL_PROC(_hw, HW_L1ICACHESIZE, l1icachesize_compat, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, 0, HW_L1ICACHESIZE, sysctl_hw_generic, "I", ""); SYSCTL_PROC(_hw, HW_L1DCACHESIZE, l1dcachesize_compat, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, 0, HW_L1DCACHESIZE, sysctl_hw_generic, "I", ""); @@ -466,6 +486,48 @@ SYSCTL_PROC(_hw_optional, OID_AUTO, hle, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN SYSCTL_PROC(_hw_optional, OID_AUTO, adx, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasADX, 0, sysctl_cpu_capability, "I", ""); SYSCTL_PROC(_hw_optional, OID_AUTO, mpx, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasMPX, 0, sysctl_cpu_capability, "I", ""); SYSCTL_PROC(_hw_optional, OID_AUTO, sgx, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasSGX, 0, sysctl_cpu_capability, "I", ""); +#if !defined(RC_HIDE_XNU_J137) +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512f, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512F, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512cd, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512CD, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512dq, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512DQ, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512bw, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512BW, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512vl, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512VL, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512ifma, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512IFMA, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512vbmi, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512VBMI, 0, sysctl_cpu_capability, "I", ""); +#endif /* not RC_HIDE_XNU_J137 */ +#elif defined (__arm__) || defined (__arm64__) +int watchpoint_flag = -1; +int breakpoint_flag = -1; +int gNeon = -1; +int gNeonHpfp = -1; +int gARMv81Atomics = 0; + +#if defined (__arm__) +int arm64_flag = 0; +#elif defined (__arm64__) /* end __arm__*/ +int arm64_flag = 1; +#else /* end __arm64__*/ +int arm64_flag = -1; +#endif + +SYSCTL_INT(_hw_optional, OID_AUTO, watchpoint, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &watchpoint_flag, 0, ""); +SYSCTL_INT(_hw_optional, OID_AUTO, breakpoint, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &breakpoint_flag, 0, ""); +SYSCTL_INT(_hw_optional, OID_AUTO, neon, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gNeon, 0, ""); +SYSCTL_INT(_hw_optional, OID_AUTO, neon_hpfp, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gNeonHpfp, 0, ""); +SYSCTL_INT(_hw_optional, OID_AUTO, armv8_1_atomics, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gARMv81Atomics, 0, ""); + +/* + * Without this little ifdef dance, the preprocessor replaces "arm64" with "1", + * leaving us with a less-than-helpful sysctl.hwoptional.1. + */ +#ifdef arm64 +#undef arm64 +SYSCTL_INT(_hw_optional, OID_AUTO, arm64, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &arm64_flag, 0, ""); +#define arm64 1 +#else +SYSCTL_INT(_hw_optional, OID_AUTO, arm64, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &arm64_flag, 0, ""); +#endif + #else #error Unsupported arch #endif /* !__i386__ && !__x86_64 && !__arm__ && ! __arm64__ */ @@ -485,6 +547,8 @@ sysctl_mib_init(void) cputhreadtype = cpu_threadtype(); #if defined(__i386__) || defined (__x86_64__) cpu64bit = (_get_cpu_capabilities() & k64Bit) == k64Bit; +#elif defined(__arm__) || defined (__arm64__) + cpu64bit = (cputype & CPU_ARCH_ABI64) == CPU_ARCH_ABI64; #else #error Unsupported arch #endif @@ -523,6 +587,33 @@ sysctl_mib_init(void) packages = roundup(ml_cpu_cache_sharing(0), cpuid_info()->thread_count) / cpuid_info()->thread_count; +#elif defined(__arm__) || defined(__arm64__) /* end __i386 */ + + cpufamily = cpuid_get_cpufamily(); + + watchpoint_flag = arm_debug_info()->num_watchpoint_pairs; + breakpoint_flag = arm_debug_info()->num_breakpoint_pairs; + + arm_mvfp_info_t *mvfp_info; + mvfp_info = arm_mvfp_info(); + gNeon = mvfp_info->neon; + gNeonHpfp = mvfp_info->neon_hpfp; + + cacheconfig[0] = ml_get_max_cpus(); + cacheconfig[1] = 1; + cacheconfig[2] = cache_info()->c_l2size ? 1:0; + cacheconfig[3] = 0; + cacheconfig[4] = 0; + cacheconfig[5] = 0; + cacheconfig[6] = 0; + + cachesize[0] = ml_get_machine_mem(); + cachesize[1] = cache_info()->c_dsize; /* Using the DCache */ + cachesize[2] = cache_info()->c_l2size; + cachesize[3] = 0; + cachesize[4] = 0; + + packages = 1; #else #error unknown architecture #endif /* !__i386__ && !__x86_64 && !__arm__ && !__arm64__ */ diff --git a/bsd/kern/kern_mman.c b/bsd/kern/kern_mman.c index 318400dc9..adb144567 100644 --- a/bsd/kern/kern_mman.c +++ b/bsd/kern/kern_mman.c @@ -128,6 +128,10 @@ #include #include +#if CONFIG_MACF +#include +#endif + /* * XXX Internally, we use VM_PROT_* somewhat interchangeably, but the correct * XXX usage is PROT_* from an interface perspective. Thus the values of @@ -150,7 +154,9 @@ mmap(proc_t p, struct mmap_args *uap, user_addr_t *retval) vm_map_size_t user_size; vm_object_offset_t pageoff; vm_object_offset_t file_pos; - int alloc_flags=0; + int alloc_flags = 0; + vm_tag_t tag = VM_KERN_MEMORY_NONE; + vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; boolean_t docow; vm_prot_t maxprot; void *handle; @@ -293,12 +299,15 @@ mmap(proc_t p, struct mmap_args *uap, user_addr_t *retval) * Use "fd" to pass (some) Mach VM allocation flags, * (see the VM_FLAGS_* definitions). */ - alloc_flags = fd & (VM_FLAGS_ALIAS_MASK | VM_FLAGS_SUPERPAGE_MASK | + alloc_flags = fd & (VM_FLAGS_ALIAS_MASK | + VM_FLAGS_SUPERPAGE_MASK | VM_FLAGS_PURGABLE); if (alloc_flags != fd) { /* reject if there are any extra flags */ return EINVAL; } + VM_GET_FLAGS_ALIAS(alloc_flags, tag); + alloc_flags &= ~VM_FLAGS_ALIAS_MASK; } handle = NULL; @@ -487,7 +496,7 @@ mmap(proc_t p, struct mmap_args *uap, user_addr_t *retval) alloc_flags |= VM_FLAGS_NO_CACHE; if (flags & MAP_JIT) { - alloc_flags |= VM_FLAGS_MAP_JIT; + vmk_flags.vmkf_map_jit = TRUE; } if (flags & MAP_RESILIENT_CODESIGN) { @@ -518,7 +527,8 @@ mmap(proc_t p, struct mmap_args *uap, user_addr_t *retval) map_anon_retry: result = vm_map_enter_mem_object(user_map, &user_addr, user_size, - 0, alloc_flags, + 0, alloc_flags, vmk_flags, + tag, IPC_PORT_NULL, 0, FALSE, prot, maxprot, (flags & MAP_SHARED) ? @@ -599,7 +609,8 @@ map_file_retry: } result = vm_map_enter_mem_object_control(user_map, &user_addr, user_size, - 0, alloc_flags, + 0, alloc_flags, vmk_flags, + tag, control, file_pos, docow, prot, maxprot, (flags & MAP_SHARED) ? @@ -650,8 +661,10 @@ bad: fp_drop(p, fd, fp, 0); KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO, SYS_mmap) | DBG_FUNC_NONE), fd, (uint32_t)(*retval), (uint32_t)user_size, error, 0); +#ifndef CONFIG_EMBEDDED KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO2, SYS_mmap) | DBG_FUNC_NONE), (uint32_t)(*retval >> 32), (uint32_t)(user_size >> 32), (uint32_t)(file_pos >> 32), (uint32_t)file_pos, 0); +#endif return(error); } @@ -675,7 +688,9 @@ msync_nocancel(__unused proc_t p, struct msync_nocancel_args *uap, __unused int3 user_map = current_map(); addr = (mach_vm_offset_t) uap->addr; size = (mach_vm_size_t)uap->len; +#ifndef CONFIG_EMBEDDED KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO, SYS_msync) | DBG_FUNC_NONE), (uint32_t)(addr >> 32), (uint32_t)(size >> 32), 0, 0, 0); +#endif if (addr & vm_map_page_mask(user_map)) { /* UNIX SPEC: user address is not page-aligned, return EINVAL */ return EINVAL; @@ -802,6 +817,10 @@ mprotect(__unused proc_t p, struct mprotect_args *uap, __unused int32_t *retval) prot |= VM_PROT_READ; #endif /* 3936456 */ +#if defined(__arm64__) + if (prot & VM_PROT_STRIP_READ) + prot &= ~(VM_PROT_READ | VM_PROT_STRIP_READ); +#endif #if CONFIG_MACF /* @@ -952,6 +971,24 @@ madvise(__unused proc_t p, struct madvise_args *uap, __unused int32_t *retval) start = (mach_vm_offset_t) uap->addr; size = (mach_vm_size_t) uap->len; +#if __arm64__ + if (start == 0 && + size != 0 && + (uap->behav == MADV_FREE || + uap->behav == MADV_FREE_REUSABLE)) { + printf("** FOURK_COMPAT: %d[%s] " + "failing madvise(0x%llx,0x%llx,%s)\n", + p->p_pid, p->p_comm, start, size, + ((uap->behav == MADV_FREE_REUSABLE) + ? "MADV_FREE_REUSABLE" + : "MADV_FREE")); + DTRACE_VM3(fourk_compat_madvise, + uint64_t, start, + uint64_t, size, + int, uap->behav); + return EINVAL; + } +#endif /* __arm64__ */ user_map = current_map(); @@ -971,17 +1008,19 @@ madvise(__unused proc_t p, struct madvise_args *uap, __unused int32_t *retval) int mincore(__unused proc_t p, struct mincore_args *uap, __unused int32_t *retval) { - mach_vm_offset_t addr, first_addr, end; - vm_map_t map; - user_addr_t vec; - int error; - int vecindex, lastvecindex; + mach_vm_offset_t addr = 0, first_addr = 0, end = 0, cur_end = 0; + vm_map_t map = VM_MAP_NULL; + user_addr_t vec = 0; + int error = 0; + int vecindex = 0, lastvecindex = 0; int mincoreinfo=0; - int pqueryinfo; - kern_return_t ret; - int numref; - - char c; + int pqueryinfo = 0; + unsigned int pqueryinfo_vec_size = 0; + vm_page_info_basic_t info = NULL; + mach_msg_type_number_t count = 0; + char *kernel_vec = NULL; + int req_vec_size_pages = 0, cur_vec_size_pages = 0; + kern_return_t kr = KERN_SUCCESS; map = current_map(); @@ -991,82 +1030,117 @@ mincore(__unused proc_t p, struct mincore_args *uap, __unused int32_t *retval) */ first_addr = addr = vm_map_trunc_page(uap->addr, vm_map_page_mask(map)); - end = addr + vm_map_round_page(uap->len, + end = vm_map_round_page(uap->addr + uap->len, vm_map_page_mask(map)); if (end < addr) return (EINVAL); + if (end == addr) + return (0); + /* - * Address of byte vector + * We are going to loop through the whole 'req_vec_size' pages + * range in chunks of 'cur_vec_size'. */ - vec = uap->vec; - map = current_map(); + req_vec_size_pages = (end - addr) >> PAGE_SHIFT; + cur_vec_size_pages = MIN(req_vec_size_pages, (int)(MAX_PAGE_RANGE_QUERY >> PAGE_SHIFT)); + + kernel_vec = (void*) _MALLOC(cur_vec_size_pages * sizeof(char), M_TEMP, M_WAITOK); + + if (kernel_vec == NULL) { + return (ENOMEM); + } /* - * Do this on a map entry basis so that if the pages are not - * in the current processes address space, we can easily look - * up the pages elsewhere. + * Address of byte vector */ - lastvecindex = -1; - for( ; addr < end; addr += PAGE_SIZE ) { - pqueryinfo = 0; - ret = mach_vm_page_query(map, addr, &pqueryinfo, &numref); - if (ret != KERN_SUCCESS) - pqueryinfo = 0; - mincoreinfo = 0; - if (pqueryinfo & VM_PAGE_QUERY_PAGE_PRESENT) - mincoreinfo |= MINCORE_INCORE; - if (pqueryinfo & VM_PAGE_QUERY_PAGE_REF) - mincoreinfo |= MINCORE_REFERENCED; - if (pqueryinfo & VM_PAGE_QUERY_PAGE_DIRTY) - mincoreinfo |= MINCORE_MODIFIED; - - - /* - * calculate index into user supplied byte vector - */ - vecindex = (addr - first_addr)>> PAGE_SHIFT; + vec = uap->vec; + + pqueryinfo_vec_size = cur_vec_size_pages * sizeof(struct vm_page_info_basic); + info = (void*) _MALLOC(pqueryinfo_vec_size, M_TEMP, M_WAITOK); + + if (info == NULL) { + FREE(kernel_vec, M_TEMP); + return (ENOMEM); + } + + while (addr < end) { + + cur_end = addr + (cur_vec_size_pages * PAGE_SIZE_64); + + count = VM_PAGE_INFO_BASIC_COUNT; + kr = vm_map_page_range_info_internal(map, + addr, + cur_end, + VM_PAGE_INFO_BASIC, + (vm_page_info_t) info, + &count); + + assert(kr == KERN_SUCCESS); /* - * If we have skipped map entries, we need to make sure that - * the byte vector is zeroed for those skipped entries. + * Do this on a map entry basis so that if the pages are not + * in the current processes address space, we can easily look + * up the pages elsewhere. */ - while((lastvecindex + 1) < vecindex) { - c = 0; - error = copyout(&c, vec + lastvecindex, 1); - if (error) { - return (EFAULT); - } - ++lastvecindex; + lastvecindex = -1; + for( ; addr < cur_end; addr += PAGE_SIZE ) { + + pqueryinfo = info[lastvecindex + 1].disposition; + + mincoreinfo = 0; + + if (pqueryinfo & VM_PAGE_QUERY_PAGE_PRESENT) + mincoreinfo |= MINCORE_INCORE; + if (pqueryinfo & VM_PAGE_QUERY_PAGE_REF) + mincoreinfo |= MINCORE_REFERENCED; + if (pqueryinfo & VM_PAGE_QUERY_PAGE_DIRTY) + mincoreinfo |= MINCORE_MODIFIED; + if (pqueryinfo & VM_PAGE_QUERY_PAGE_PAGED_OUT) + mincoreinfo |= MINCORE_PAGED_OUT; + if (pqueryinfo & VM_PAGE_QUERY_PAGE_COPIED) + mincoreinfo |= MINCORE_COPIED; + if ((pqueryinfo & VM_PAGE_QUERY_PAGE_EXTERNAL) == 0) + mincoreinfo |= MINCORE_ANONYMOUS; + /* + * calculate index into user supplied byte vector + */ + vecindex = (addr - first_addr)>> PAGE_SHIFT; + kernel_vec[vecindex] = (char)mincoreinfo; + lastvecindex = vecindex; } - /* - * Pass the page information to the user - */ - c = (char)mincoreinfo; - error = copyout(&c, vec + vecindex, 1); + + assert(vecindex == (cur_vec_size_pages - 1)); + + error = copyout(kernel_vec, vec, cur_vec_size_pages * sizeof(char) /* a char per page */); + if (error) { - return (EFAULT); + break; } - lastvecindex = vecindex; + + /* + * For the next chunk, we'll need: + * - bump the location in the user buffer for our next disposition. + * - new length + * - starting address + */ + vec += cur_vec_size_pages * sizeof(char); + req_vec_size_pages = (end - addr) >> PAGE_SHIFT; + cur_vec_size_pages = MIN(req_vec_size_pages, (int)(MAX_PAGE_RANGE_QUERY >> PAGE_SHIFT)); + + first_addr = addr; } + FREE(kernel_vec, M_TEMP); + FREE(info, M_TEMP); - /* - * Zero the last entries in the byte vector. - */ - vecindex = (end - first_addr) >> PAGE_SHIFT; - while((lastvecindex + 1) < vecindex) { - c = 0; - error = copyout(&c, vec + lastvecindex, 1); - if (error) { - return (EFAULT); - } - ++lastvecindex; + if (error) { + return (EFAULT); } - + return (0); } @@ -1097,7 +1171,7 @@ mlock(__unused proc_t p, struct mlock_args *uap, __unused int32_t *retvalval) size = vm_map_round_page(size+pageoff, vm_map_page_mask(user_map)); /* have to call vm_map_wire directly to pass "I don't know" protections */ - result = vm_map_wire(user_map, addr, addr+size, VM_PROT_NONE | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK), TRUE); + result = vm_map_wire_kernel(user_map, addr, addr+size, VM_PROT_NONE, VM_KERN_MEMORY_MLOCK, TRUE); if (result == KERN_RESOURCE_SHORTAGE) return EAGAIN; @@ -1125,7 +1199,7 @@ munlock(__unused proc_t p, struct munlock_args *uap, __unused int32_t *retval) user_map = current_map(); /* JMM - need to remove all wirings by spec - this just removes one */ - result = mach_vm_wire(host_priv_self(), user_map, addr, size, VM_PROT_NONE); + result = mach_vm_wire_kernel(host_priv_self(), user_map, addr, size, VM_PROT_NONE, VM_KERN_MEMORY_MLOCK); return (result == KERN_SUCCESS ? 0 : ENOMEM); } diff --git a/bsd/kern/kern_newsysctl.c b/bsd/kern/kern_newsysctl.c index 6895674f9..4103009fe 100644 --- a/bsd/kern/kern_newsysctl.c +++ b/bsd/kern/kern_newsysctl.c @@ -80,6 +80,10 @@ #include #include +#if CONFIG_MACF +#include +#endif + lck_grp_t * sysctl_lock_group = NULL; lck_rw_t * sysctl_geometry_lock = NULL; lck_mtx_t * sysctl_unlocked_node_lock = NULL; diff --git a/bsd/kern/kern_ntptime.c b/bsd/kern/kern_ntptime.c new file mode 100644 index 000000000..a922c3676 --- /dev/null +++ b/bsd/kern/kern_ntptime.c @@ -0,0 +1,782 @@ +/*- + *********************************************************************** + * * + * Copyright (c) David L. Mills 1993-2001 * + * * + * Permission to use, copy, modify, and distribute this software and * + * its documentation for any purpose and without fee is hereby * + * granted, provided that the above copyright notice appears in all * + * copies and that both the copyright notice and this permission * + * notice appear in supporting documentation, and that the name * + * University of Delaware not be used in advertising or publicity * + * pertaining to distribution of the software without specific, * + * written prior permission. The University of Delaware makes no * + * representations about the suitability this software for any * + * purpose. It is provided "as is" without express or implied * + * warranty. * + * * + **********************************************************************/ + + +/* + * Adapted from the original sources for FreeBSD and timecounters by: + * Poul-Henning Kamp . + * + * The 32bit version of the "LP" macros seems a bit past its "sell by" + * date so I have retained only the 64bit version and included it directly + * in this file. + * + * Only minor changes done to interface with the timecounters over in + * sys/kern/kern_clock.c. Some of the comments below may be (even more) + * confusing and/or plain wrong in that context. + */ + +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if CONFIG_MACF +#include +#endif +#include + +typedef int64_t l_fp; +#define L_ADD(v, u) ((v) += (u)) +#define L_SUB(v, u) ((v) -= (u)) +#define L_ADDHI(v, a) ((v) += (int64_t)(a) << 32) +#define L_NEG(v) ((v) = -(v)) +#define L_RSHIFT(v, n) \ + do { \ + if ((v) < 0) \ + (v) = -(-(v) >> (n)); \ + else \ + (v) = (v) >> (n); \ + } while (0) +#define L_MPY(v, a) ((v) *= (a)) +#define L_CLR(v) ((v) = 0) +#define L_ISNEG(v) ((v) < 0) +#define L_LINT(v, a) \ + do { \ + if ((a) > 0) \ + ((v) = (int64_t)(a) << 32); \ + else \ + ((v) = -((int64_t)(-(a)) << 32)); \ + } while (0) +#define L_GINT(v) ((v) < 0 ? -(-(v) >> 32) : (v) >> 32) + +/* + * Generic NTP kernel interface + * + * These routines constitute the Network Time Protocol (NTP) interfaces + * for user and daemon application programs. The ntp_gettime() routine + * provides the time, maximum error (synch distance) and estimated error + * (dispersion) to client user application programs. The ntp_adjtime() + * routine is used by the NTP daemon to adjust the calendar clock to an + * externally derived time. The time offset and related variables set by + * this routine are used by other routines in this module to adjust the + * phase and frequency of the clock discipline loop which controls the + * system clock. + * + * When the kernel time is reckoned directly in nanoseconds (NTP_NANO + * defined), the time at each tick interrupt is derived directly from + * the kernel time variable. When the kernel time is reckoned in + * microseconds, (NTP_NANO undefined), the time is derived from the + * kernel time variable together with a variable representing the + * leftover nanoseconds at the last tick interrupt. In either case, the + * current nanosecond time is reckoned from these values plus an + * interpolated value derived by the clock routines in another + * architecture-specific module. The interpolation can use either a + * dedicated counter or a processor cycle counter (PCC) implemented in + * some architectures. + * + */ +/* + * Phase/frequency-lock loop (PLL/FLL) definitions + * + * The nanosecond clock discipline uses two variable types, time + * variables and frequency variables. Both types are represented as 64- + * bit fixed-point quantities with the decimal point between two 32-bit + * halves. On a 32-bit machine, each half is represented as a single + * word and mathematical operations are done using multiple-precision + * arithmetic. On a 64-bit machine, ordinary computer arithmetic is + * used. + * + * A time variable is a signed 64-bit fixed-point number in ns and + * fraction. It represents the remaining time offset to be amortized + * over succeeding tick interrupts. The maximum time offset is about + * 0.5 s and the resolution is about 2.3e-10 ns. + * + * 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |s s s| ns | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | fraction | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * A frequency variable is a signed 64-bit fixed-point number in ns/s + * and fraction. It represents the ns and fraction to be added to the + * kernel time variable at each second. The maximum frequency offset is + * about +-500000 ns/s and the resolution is about 2.3e-10 ns/s. + * + * 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |s s s s s s s s s s s s s| ns/s | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | fraction | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + +#define SHIFT_PLL 4 +#define SHIFT_FLL 2 + +static int time_state = TIME_OK; +int time_status = STA_UNSYNC; +static long time_tai; +static long time_constant; +static long time_precision = 1; +static long time_maxerror = MAXPHASE / 1000; +static unsigned long last_time_maxerror_update; +long time_esterror = MAXPHASE / 1000; +static long time_reftime; +static l_fp time_offset; +static l_fp time_freq; +static int64_t time_adjtime; +static int updated; + +static lck_spin_t * ntp_lock; +static lck_grp_t * ntp_lock_grp; +static lck_attr_t * ntp_lock_attr; +static lck_grp_attr_t *ntp_lock_grp_attr; + +#define NTP_LOCK(enable) \ + enable = ml_set_interrupts_enabled(FALSE); \ + lck_spin_lock(ntp_lock); + +#define NTP_UNLOCK(enable) \ + lck_spin_unlock(ntp_lock);\ + ml_set_interrupts_enabled(enable); + +#define NTP_ASSERT_LOCKED() LCK_SPIN_ASSERT(ntp_lock, LCK_ASSERT_OWNED) + +static timer_call_data_t ntp_loop_update; +static uint64_t ntp_loop_deadline; +static uint32_t ntp_loop_active; +static uint32_t ntp_loop_period; +#define NTP_LOOP_PERIOD_INTERVAL (NSEC_PER_SEC) /*1 second interval*/ + +void ntp_init(void); +static void hardupdate(long offset); +static void ntp_gettime1(struct ntptimeval *ntvp); +static bool ntp_is_time_error(int tsl); + +static void ntp_loop_update_call(void); +static void refresh_ntp_loop(void); +static void start_ntp_loop(void); + +static bool +ntp_is_time_error(int tsl) +{ + + if (tsl & (STA_UNSYNC | STA_CLOCKERR)) + return (true); + + return (false); +} + +static void +ntp_gettime1(struct ntptimeval *ntvp) +{ + struct timespec atv; + + NTP_ASSERT_LOCKED(); + + nanotime(&atv); + ntvp->time.tv_sec = atv.tv_sec; + ntvp->time.tv_nsec = atv.tv_nsec; + if ((unsigned long)atv.tv_sec > last_time_maxerror_update) { + time_maxerror += (MAXFREQ / 1000)*(atv.tv_sec-last_time_maxerror_update); + last_time_maxerror_update = atv.tv_sec; + } + ntvp->maxerror = time_maxerror; + ntvp->esterror = time_esterror; + ntvp->tai = time_tai; + ntvp->time_state = time_state; + + if (ntp_is_time_error(time_status)) + ntvp->time_state = TIME_ERROR; +} + +int +ntp_gettime(struct proc *p, struct ntp_gettime_args *uap, __unused int32_t *retval) +{ + struct ntptimeval ntv; + int error; + boolean_t enable; + + NTP_LOCK(enable); + ntp_gettime1(&ntv); + NTP_UNLOCK(enable); + + if (IS_64BIT_PROCESS(p)) { + struct user64_ntptimeval user_ntv; + user_ntv.time.tv_sec = ntv.time.tv_sec; + user_ntv.time.tv_nsec = ntv.time.tv_nsec; + user_ntv.maxerror = ntv.maxerror; + user_ntv.esterror = ntv.esterror; + user_ntv.tai = ntv.tai; + user_ntv.time_state = ntv.time_state; + error = copyout(&user_ntv, uap->ntvp, sizeof(user_ntv)); + } else { + struct user32_ntptimeval user_ntv; + user_ntv.time.tv_sec = ntv.time.tv_sec; + user_ntv.time.tv_nsec = ntv.time.tv_nsec; + user_ntv.maxerror = ntv.maxerror; + user_ntv.esterror = ntv.esterror; + user_ntv.tai = ntv.tai; + user_ntv.time_state = ntv.time_state; + error = copyout(&user_ntv, uap->ntvp, sizeof(user_ntv)); + } + + if (error) + return error; + + return ntv.time_state; +} + +int +ntp_adjtime(struct proc *p, struct ntp_adjtime_args *uap, __unused int32_t *retval) +{ + struct timex ntv; + long freq; + int modes; + int error, ret = 0; + clock_sec_t sec; + clock_usec_t microsecs; + boolean_t enable; + + if (IS_64BIT_PROCESS(p)) { + struct user64_timex user_ntv; + error = copyin(uap->tp, &user_ntv, sizeof(user_ntv)); + ntv.modes = user_ntv.modes; + ntv.offset = user_ntv.offset; + ntv.freq = user_ntv.freq; + ntv.maxerror = user_ntv.maxerror; + ntv.esterror = user_ntv.esterror; + ntv.status = user_ntv.status; + ntv.constant = user_ntv.constant; + ntv.precision = user_ntv.precision; + ntv.tolerance = user_ntv.tolerance; + + } else { + struct user32_timex user_ntv; + error = copyin(uap->tp, &user_ntv, sizeof(user_ntv)); + ntv.modes = user_ntv.modes; + ntv.offset = user_ntv.offset; + ntv.freq = user_ntv.freq; + ntv.maxerror = user_ntv.maxerror; + ntv.esterror = user_ntv.esterror; + ntv.status = user_ntv.status; + ntv.constant = user_ntv.constant; + ntv.precision = user_ntv.precision; + ntv.tolerance = user_ntv.tolerance; + } + if (error) + return (error); + + /* + * Update selected clock variables - only the superuser can + * change anything. Note that there is no error checking here on + * the assumption the superuser should know what it is doing. + * Note that either the time constant or TAI offset are loaded + * from the ntv.constant member, depending on the mode bits. If + * the STA_PLL bit in the status word is cleared, the state and + * status words are reset to the initial values at boot. + */ + modes = ntv.modes; + if (modes) { + /* Check that this task is entitled to set the time or it is root */ + if (!IOTaskHasEntitlement(current_task(), SETTIME_ENTITLEMENT)) { +#if CONFIG_MACF + error = mac_system_check_settime(kauth_cred_get()); + if (error) + return (error); +#endif + if ((error = priv_check_cred(kauth_cred_get(), PRIV_ADJTIME, 0))) + return (error); + + } + } + + NTP_LOCK(enable); + + if (modes & MOD_MAXERROR) { + clock_gettimeofday(&sec, µsecs); + time_maxerror = ntv.maxerror; + last_time_maxerror_update = sec; + } + if (modes & MOD_ESTERROR) + time_esterror = ntv.esterror; + if (modes & MOD_STATUS) { + if (time_status & STA_PLL && !(ntv.status & STA_PLL)) { + time_state = TIME_OK; + time_status = STA_UNSYNC; + } + time_status &= STA_RONLY; + time_status |= ntv.status & ~STA_RONLY; + /* + * Nor PPS or leaps seconds are supported. + * Filter out unsupported bits. + */ + time_status &= STA_SUPPORTED; + } + if (modes & MOD_TIMECONST) { + if (ntv.constant < 0) + time_constant = 0; + else if (ntv.constant > MAXTC) + time_constant = MAXTC; + else + time_constant = ntv.constant; + } + if (modes & MOD_TAI) { + if (ntv.constant > 0) + time_tai = ntv.constant; + } + if (modes & MOD_NANO) + time_status |= STA_NANO; + if (modes & MOD_MICRO) + time_status &= ~STA_NANO; + if (modes & MOD_CLKB) + time_status |= STA_CLK; + if (modes & MOD_CLKA) + time_status &= ~STA_CLK; + if (modes & MOD_FREQUENCY) { + freq = (ntv.freq * 1000LL) >> 16; + if (freq > MAXFREQ) + L_LINT(time_freq, MAXFREQ); + else if (freq < -MAXFREQ) + L_LINT(time_freq, -MAXFREQ); + else { + /* + * ntv.freq is [PPM * 2^16] = [us/s * 2^16] + * time_freq is [ns/s * 2^32] + */ + time_freq = ntv.freq * 1000LL * 65536LL; + } + } + if (modes & MOD_OFFSET) { + if (time_status & STA_NANO) + hardupdate(ntv.offset); + else + hardupdate(ntv.offset * 1000); + } + + ret = ntp_is_time_error(time_status) ? TIME_ERROR : time_state; + + /* + * Retrieve all clock variables. Note that the TAI offset is + * returned only by ntp_gettime(); + */ + if (IS_64BIT_PROCESS(p)) { + struct user64_timex user_ntv; + + if (time_status & STA_NANO) + user_ntv.offset = L_GINT(time_offset); + else + user_ntv.offset = L_GINT(time_offset) / 1000; + user_ntv.freq = L_GINT((time_freq / 1000LL) << 16); + user_ntv.maxerror = time_maxerror; + user_ntv.esterror = time_esterror; + user_ntv.status = time_status; + user_ntv.constant = time_constant; + if (time_status & STA_NANO) + user_ntv.precision = time_precision; + else + user_ntv.precision = time_precision / 1000; + user_ntv.tolerance = MAXFREQ * SCALE_PPM; + + /* unlock before copyout */ + NTP_UNLOCK(enable); + + error = copyout(&user_ntv, uap->tp, sizeof(user_ntv)); + + } + else{ + struct user32_timex user_ntv; + + if (time_status & STA_NANO) + user_ntv.offset = L_GINT(time_offset); + else + user_ntv.offset = L_GINT(time_offset) / 1000; + user_ntv.freq = L_GINT((time_freq / 1000LL) << 16); + user_ntv.maxerror = time_maxerror; + user_ntv.esterror = time_esterror; + user_ntv.status = time_status; + user_ntv.constant = time_constant; + if (time_status & STA_NANO) + user_ntv.precision = time_precision; + else + user_ntv.precision = time_precision / 1000; + user_ntv.tolerance = MAXFREQ * SCALE_PPM; + + /* unlock before copyout */ + NTP_UNLOCK(enable); + + error = copyout(&user_ntv, uap->tp, sizeof(user_ntv)); + } + + if (modes) + start_ntp_loop(); + + if (error == 0) + *retval = ret; + + return (error); +} + +int64_t +ntp_get_freq(void){ + return time_freq; +} + +/* + * Compute the adjustment to add to the next second. + */ +void +ntp_update_second(int64_t *adjustment, clock_sec_t secs) +{ + int tickrate; + l_fp time_adj; + l_fp ftemp, old_time_adjtime, old_offset; + + NTP_ASSERT_LOCKED(); + + if (secs > last_time_maxerror_update) { + time_maxerror += (MAXFREQ / 1000)*(secs-last_time_maxerror_update); + last_time_maxerror_update = secs; + } + + old_offset = time_offset; + old_time_adjtime = time_adjtime; + + ftemp = time_offset; + L_RSHIFT(ftemp, SHIFT_PLL + time_constant); + time_adj = ftemp; + L_SUB(time_offset, ftemp); + L_ADD(time_adj, time_freq); + + /* + * Apply any correction from adjtime. If more than one second + * off we slew at a rate of 5ms/s (5000 PPM) else 500us/s (500PPM) + * until the last second is slewed the final < 500 usecs. + */ + if (time_adjtime != 0) { + if (time_adjtime > 1000000) + tickrate = 5000; + else if (time_adjtime < -1000000) + tickrate = -5000; + else if (time_adjtime > 500) + tickrate = 500; + else if (time_adjtime < -500) + tickrate = -500; + else + tickrate = time_adjtime; + time_adjtime -= tickrate; + L_LINT(ftemp, tickrate * 1000); + L_ADD(time_adj, ftemp); + } + + if (old_time_adjtime || ((time_offset || old_offset) && (time_offset != old_offset))) { + updated = 1; + } + else{ + updated = 0; + } + + *adjustment = time_adj; +} + +/* + * hardupdate() - local clock update + * + * This routine is called by ntp_adjtime() when an offset is provided + * to update the local clock phase and frequency. + * The implementation is of an adaptive-parameter, hybrid + * phase/frequency-lock loop (PLL/FLL). The routine computes new + * time and frequency offset estimates for each call. + * Presumably, calls to ntp_adjtime() occur only when the caller + * believes the local clock is valid within some bound (+-128 ms with + * NTP). + * + * For uncompensated quartz crystal oscillators and nominal update + * intervals less than 256 s, operation should be in phase-lock mode, + * where the loop is disciplined to phase. For update intervals greater + * than 1024 s, operation should be in frequency-lock mode, where the + * loop is disciplined to frequency. Between 256 s and 1024 s, the mode + * is selected by the STA_MODE status bit. + */ +static void +hardupdate(offset) + long offset; +{ + long mtemp = 0; + long time_monitor; + clock_sec_t time_uptime; + l_fp ftemp; + + NTP_ASSERT_LOCKED(); + + if (!(time_status & STA_PLL)) + return; + + if (offset > MAXPHASE) + time_monitor = MAXPHASE; + else if (offset < -MAXPHASE) + time_monitor = -MAXPHASE; + else + time_monitor = offset; + L_LINT(time_offset, time_monitor); + + clock_get_calendar_uptime(&time_uptime); + + if (time_status & STA_FREQHOLD || time_reftime == 0) { + time_reftime = time_uptime; + } + + mtemp = time_uptime - time_reftime; + L_LINT(ftemp, time_monitor); + L_RSHIFT(ftemp, (SHIFT_PLL + 2 + time_constant) << 1); + L_MPY(ftemp, mtemp); + L_ADD(time_freq, ftemp); + time_status &= ~STA_MODE; + if (mtemp >= MINSEC && (time_status & STA_FLL || mtemp > + MAXSEC)) { + L_LINT(ftemp, (time_monitor << 4) / mtemp); + L_RSHIFT(ftemp, SHIFT_FLL + 4); + L_ADD(time_freq, ftemp); + time_status |= STA_MODE; + } + time_reftime = time_uptime; + + if (L_GINT(time_freq) > MAXFREQ) + L_LINT(time_freq, MAXFREQ); + else if (L_GINT(time_freq) < -MAXFREQ) + L_LINT(time_freq, -MAXFREQ); +} + + +static int +kern_adjtime(struct timeval *delta) +{ + struct timeval atv; + int64_t ltr, ltw; + boolean_t enable; + + if (delta == NULL) + return (EINVAL); + + ltw = (int64_t)delta->tv_sec * (int64_t)USEC_PER_SEC + delta->tv_usec; + + NTP_LOCK(enable); + ltr = time_adjtime; + time_adjtime = ltw; + NTP_UNLOCK(enable); + + atv.tv_sec = ltr / (int64_t)USEC_PER_SEC; + atv.tv_usec = ltr % (int64_t)USEC_PER_SEC; + if (atv.tv_usec < 0) { + atv.tv_usec += (suseconds_t)USEC_PER_SEC; + atv.tv_sec--; + } + + *delta = atv; + + start_ntp_loop(); + + return (0); +} + +int +adjtime(struct proc *p, struct adjtime_args *uap, __unused int32_t *retval) +{ + + struct timeval atv; + int error; + + /* Check that this task is entitled to set the time or it is root */ + if (!IOTaskHasEntitlement(current_task(), SETTIME_ENTITLEMENT)) { + +#if CONFIG_MACF + error = mac_system_check_settime(kauth_cred_get()); + if (error) + return (error); +#endif + if ((error = priv_check_cred(kauth_cred_get(), PRIV_ADJTIME, 0))) + return (error); + } + + if (IS_64BIT_PROCESS(p)) { + struct user64_timeval user_atv; + error = copyin(uap->delta, &user_atv, sizeof(user_atv)); + atv.tv_sec = user_atv.tv_sec; + atv.tv_usec = user_atv.tv_usec; + } else { + struct user32_timeval user_atv; + error = copyin(uap->delta, &user_atv, sizeof(user_atv)); + atv.tv_sec = user_atv.tv_sec; + atv.tv_usec = user_atv.tv_usec; + } + if (error) + return (error); + + kern_adjtime(&atv); + + if (uap->olddelta) { + if (IS_64BIT_PROCESS(p)) { + struct user64_timeval user_atv; + user_atv.tv_sec = atv.tv_sec; + user_atv.tv_usec = atv.tv_usec; + error = copyout(&user_atv, uap->olddelta, sizeof(user_atv)); + } else { + struct user32_timeval user_atv; + user_atv.tv_sec = atv.tv_sec; + user_atv.tv_usec = atv.tv_usec; + error = copyout(&user_atv, uap->olddelta, sizeof(user_atv)); + } + } + + return (error); + +} + +static void +ntp_loop_update_call(void) +{ + boolean_t enable; + + NTP_LOCK(enable); + + /* + * Update the scale factor used by clock_calend. + * NOTE: clock_update_calendar will call ntp_update_second to compute the next adjustment. + */ + clock_update_calendar(); + + refresh_ntp_loop(); + + NTP_UNLOCK(enable); +} + +static void +refresh_ntp_loop(void) +{ + + NTP_ASSERT_LOCKED(); + if (--ntp_loop_active == 0) { + /* + * Activate the timer only if the next second adjustment might change. + * ntp_update_second checks it and sets updated accordingly. + */ + if (updated) { + clock_deadline_for_periodic_event(ntp_loop_period, mach_absolute_time(), &ntp_loop_deadline); + + if (!timer_call_enter(&ntp_loop_update, ntp_loop_deadline, TIMER_CALL_SYS_CRITICAL)) + ntp_loop_active++; + } + } + +} + +/* + * This function triggers a timer that each second will calculate the adjustment to + * provide to clock_calendar to scale the time (used by gettimeofday-family syscalls). + * The periodic timer will stop when the adjustment will reach a stable value. + */ +static void +start_ntp_loop(void) +{ + boolean_t enable; + + NTP_LOCK(enable); + + ntp_loop_deadline = mach_absolute_time() + ntp_loop_period; + + if (!timer_call_enter(&ntp_loop_update, ntp_loop_deadline, TIMER_CALL_SYS_CRITICAL)) { + ntp_loop_active++; + } + + NTP_UNLOCK(enable); +} + + +static void +init_ntp_loop(void) +{ + uint64_t abstime; + + ntp_loop_active = 0; + nanoseconds_to_absolutetime(NTP_LOOP_PERIOD_INTERVAL, &abstime); + ntp_loop_period = (uint32_t)abstime; + timer_call_setup(&ntp_loop_update, (timer_call_func_t)ntp_loop_update_call, NULL); +} + +void +ntp_init(void) +{ + + L_CLR(time_offset); + L_CLR(time_freq); + + ntp_lock_grp_attr = lck_grp_attr_alloc_init(); + ntp_lock_grp = lck_grp_alloc_init("ntp_lock", ntp_lock_grp_attr); + ntp_lock_attr = lck_attr_alloc_init(); + ntp_lock = lck_spin_alloc_init(ntp_lock_grp, ntp_lock_attr); + + updated = 0; + + init_ntp_loop(); +} + +SYSINIT(ntpclocks, SI_SUB_CLOCKS, SI_ORDER_MIDDLE, ntp_init, NULL); diff --git a/bsd/kern/kern_overrides.c b/bsd/kern/kern_overrides.c index 7b2a4622c..8e70d80f0 100644 --- a/bsd/kern/kern_overrides.c +++ b/bsd/kern/kern_overrides.c @@ -39,7 +39,6 @@ #include #include #include -#include #include #include diff --git a/bsd/kern/kern_priv.c b/bsd/kern/kern_priv.c index ee17dd2a8..462c3fbd2 100644 --- a/bsd/kern/kern_priv.c +++ b/bsd/kern/kern_priv.c @@ -75,7 +75,7 @@ int proc_check_footprint_priv(void); * only a few to grant it. */ int -priv_check_cred(kauth_cred_t cred, int priv, __unused int flags) +priv_check_cred(kauth_cred_t cred, int priv, int flags) { #if !CONFIG_MACF #pragma unused(priv) @@ -92,15 +92,18 @@ priv_check_cred(kauth_cred_t cred, int priv, __unused int flags) goto out; #endif - /* - * Having determined if privilege is restricted by various policies, - * now determine if privilege is granted. At this point, any policy - * may grant privilege. For now, we allow short-circuit boolean - * evaluation, so may not call all policies. Perhaps we should. - */ - if (kauth_cred_getuid(cred) == 0) { - error = 0; - goto out; + /* Only grant all privileges to root if DEFAULT_UNPRIVELEGED flag is NOT set. */ + if (!(flags & PRIVCHECK_DEFAULT_UNPRIVILEGED_FLAG)) { + /* + * Having determined if privilege is restricted by various policies, + * now determine if privilege is granted. At this point, any policy + * may grant privilege. For now, we allow short-circuit boolean + * evaluation, so may not call all policies. Perhaps we should. + */ + if (kauth_cred_getuid(cred) == 0) { + error = 0; + goto out; + } } /* diff --git a/bsd/kern/kern_proc.c b/bsd/kern/kern_proc.c index 8628e301f..c599a4bc7 100644 --- a/bsd/kern/kern_proc.c +++ b/bsd/kern/kern_proc.c @@ -112,6 +112,10 @@ #include #include +#if CONFIG_CSR +#include +#endif + #if CONFIG_MEMORYSTATUS #include #endif @@ -150,13 +154,21 @@ extern struct tty cons; extern int cs_debug; +#if DEVELOPMENT || DEBUG +extern int cs_enforcement_enable; +#endif + #if DEBUG #define __PROC_INTERNAL_DEBUG 1 #endif #if CONFIG_COREDUMP /* Name to give to core files */ +#if CONFIG_EMBEDDED +__XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/private/var/cores/%N.core"}; +#else __XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/cores/core.%P"}; #endif +#endif #if PROC_REF_DEBUG #include @@ -488,11 +500,16 @@ proc_t proc_ref_locked(proc_t p) { proc_t p1 = p; + int pid = proc_pid(p); - /* if process still in creation return failure */ - if ((p == PROC_NULL) || ((p->p_listflag & P_LIST_INCREATE) != 0)) - return (PROC_NULL); retry: + /* + * if process still in creation or proc got recycled + * during msleep then return failure. + */ + if ((p == PROC_NULL) || (p1 != p) || ((p->p_listflag & P_LIST_INCREATE) != 0)) + return (PROC_NULL); + /* * Do not return process marked for termination * or proc_refdrain called without ref wait. @@ -508,6 +525,11 @@ retry: ((p->p_listflag & P_LIST_REFWAIT) != 0))) { if ((p->p_listflag & P_LIST_REFWAIT) != 0 && uthread_needs_to_wait_in_proc_refwait()) { msleep(&p->p_listflag, proc_list_mlock, 0, "proc_refwait", 0) ; + /* + * the proc might have been recycled since we dropped + * the proc list lock, get the proc again. + */ + p = pfind_locked(pid); goto retry; } p->p_refcount++; @@ -1148,6 +1170,7 @@ bsd_set_dependency_capable(task_t task) } +#ifndef __arm__ int IS_64BIT_PROCESS(proc_t p) { @@ -1156,6 +1179,7 @@ IS_64BIT_PROCESS(proc_t p) else return(0); } +#endif /* * Locate a process by number @@ -1297,6 +1321,7 @@ pinsertchild(proc_t parent, proc_t child) child->p_pptr = parent; child->p_ppid = parent->p_pid; child->p_puniqueid = parent->p_uniqueid; + child->p_xhighbits = 0; pg = proc_pgrp(parent); pgrp_add(pg, parent, child); @@ -1949,6 +1974,7 @@ csops_internal(pid_t pid, int ops, user_addr_t uaddr, user_size_t usersize, user case CS_OPS_MARKRESTRICT: case CS_OPS_SET_STATUS: case CS_OPS_CLEARINSTALLER: + case CS_OPS_CLEARPLATFORM: if ((error = mac_proc_check_set_cs_info(current_proc(), pt, ops))) goto out; break; @@ -2149,7 +2175,7 @@ csops_internal(pid_t pid, int ops, user_addr_t uaddr, user_size_t usersize, user error = ENOENT; break; } - + length = strlen(identity) + 1; /* include NUL */ idlen = htonl(length + sizeof(fakeheader)); memcpy(&fakeheader[4], &idlen, sizeof(idlen)); @@ -2168,10 +2194,34 @@ csops_internal(pid_t pid, int ops, user_addr_t uaddr, user_size_t usersize, user case CS_OPS_CLEARINSTALLER: proc_lock(pt); - pt->p_csflags &= ~(CS_INSTALLER | CS_EXEC_SET_INSTALLER); + pt->p_csflags &= ~(CS_INSTALLER | CS_DATAVAULT_CONTROLLER | CS_EXEC_INHERIT_SIP); proc_unlock(pt); break; + case CS_OPS_CLEARPLATFORM: +#if DEVELOPMENT || DEBUG + if (cs_enforcement_enable) { + error = ENOTSUP; + break; + } + +#if CONFIG_CSR + if (csr_check(CSR_ALLOW_APPLE_INTERNAL) != 0) { + error = ENOTSUP; + break; + } +#endif + + proc_lock(pt); + pt->p_csflags &= ~(CS_PLATFORM_BINARY|CS_PLATFORM_PATH); + csproc_clear_platform_binary(pt); + proc_unlock(pt); + break; +#else + error = ENOTSUP; + break; +#endif /* !DEVELOPMENT || DEBUG */ + default: error = EINVAL; break; @@ -2201,7 +2251,7 @@ proc_iterate( for (;;) { proc_list_lock(); - pid_count_available = nprocs; + pid_count_available = nprocs + 1; //kernel_task is not counted in nprocs assert(pid_count_available > 0); pid_list_size_needed = pid_count_available * sizeof(pid_t); @@ -3170,6 +3220,10 @@ extern boolean_t kill_on_no_paging_space; #endif /* DEVELOPMENT || DEBUG */ #define MB_SIZE (1024 * 1024ULL) +boolean_t memorystatus_kill_on_VM_thrashing(boolean_t); + +extern int32_t max_kill_priority; +extern int memorystatus_get_proccnt_upto_priority(int32_t max_bucket_index); int no_paging_space_action() @@ -3235,6 +3289,22 @@ no_paging_space_action() } } + /* + * We have some processes within our jetsam bands of consideration and hence can be killed. + * So we will invoke the memorystatus thread to go ahead and kill something. + */ + if (memorystatus_get_proccnt_upto_priority(max_kill_priority) > 0) { + + last_no_space_action = now; + memorystatus_kill_on_VM_thrashing(TRUE /* async */); + return (1); + } + + /* + * No eligible processes to kill. So let's suspend/kill the largest + * process depending on its policy control specifications. + */ + if (nps.pcs_max_size > 0) { if ((p = proc_find(nps.pcs_pid)) != PROC_NULL) { @@ -3246,22 +3316,6 @@ no_paging_space_action() */ last_no_space_action = now; -#if DEVELOPMENT || DEBUG - if (kill_on_no_paging_space == TRUE) { - /* - * We found the largest process that has a process policy i.e. one of - * PC_KILL, PC_SUSP, PC_THROTTLE. - * But we are in a mode where we will kill it regardless of its policy. - */ - printf("low swap: killing largest process with pid %d (%s) and size %llu MB\n", p->p_pid, p->p_comm, (nps.pcs_max_size/MB_SIZE)); - psignal(p, SIGKILL); - - proc_rele(p); - - return 1; - } -#endif /* DEVELOPMENT || DEBUG */ - proc_dopcontrol(p); proc_rele(p); diff --git a/bsd/kern/kern_resource.c b/bsd/kern/kern_resource.c index 625916715..7bc8466bc 100644 --- a/bsd/kern/kern_resource.c +++ b/bsd/kern/kern_resource.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2008 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -81,8 +81,6 @@ #include #include #include -#include - #include #include @@ -104,6 +102,9 @@ #include /* for absolutetime_to_microtime() */ #include /* for TRAFFIC_MGT_SO_* */ #include /* for struct socket */ +#if NECP +#include +#endif /* NECP */ #include @@ -112,6 +113,10 @@ #include #include +#if CONFIG_MACF +#include +#endif + int donice(struct proc *curp, struct proc *chgp, int n); int dosetrlimit(struct proc *p, u_int which, struct rlimit *limp); int uthread_get_background_state(uthread_t); @@ -128,7 +133,8 @@ int fill_task_rusage(task_t task, rusage_info_current *ri); void fill_task_billed_usage(task_t task, rusage_info_current *ri); int fill_task_io_rusage(task_t task, rusage_info_current *ri); int fill_task_qos_rusage(task_t task, rusage_info_current *ri); -static void rusage_info_conversion(rusage_info_t ri_info, rusage_info_current *ri_current, int flavor); +uint64_t get_task_logical_writes(task_t task); +void fill_task_monotonic_rusage(task_t task, rusage_info_current *ri); int proc_get_rusage(proc_t p, int flavor, user_addr_t buffer, __unused int is_zombie); @@ -583,7 +589,7 @@ static int proc_set_darwin_role(proc_t curp, proc_t targetp, int priority) { int error = 0; - uint32_t flagsp; + uint32_t flagsp = 0; kauth_cred_t ucred, target_cred; @@ -762,16 +768,20 @@ do_background_socket(struct proc *p, thread_t thread) fdp = p->p_fd; for (i = 0; i < fdp->fd_nfiles; i++) { - struct socket *sockp; - fp = fdp->fd_ofiles[i]; - if (fp == NULL || (fdp->fd_ofileflags[i] & UF_RESERVED) != 0 || - FILEGLOB_DTYPE(fp->f_fglob) != DTYPE_SOCKET) { + if (fp == NULL || (fdp->fd_ofileflags[i] & UF_RESERVED) != 0) { continue; } - sockp = (struct socket *)fp->f_fglob->fg_data; - socket_set_traffic_mgt_flags(sockp, TRAFFIC_MGT_SO_BACKGROUND); - sockp->so_background_thread = NULL; + if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_SOCKET) { + struct socket *sockp = (struct socket *)fp->f_fglob->fg_data; + socket_set_traffic_mgt_flags(sockp, TRAFFIC_MGT_SO_BACKGROUND); + sockp->so_background_thread = NULL; + } +#if NECP + else if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_NETPOLICY) { + necp_set_client_as_background(p, fp, background); + } +#endif /* NECP */ } } } else { @@ -785,17 +795,23 @@ do_background_socket(struct proc *p, thread_t thread) struct socket *sockp; fp = fdp->fd_ofiles[ i ]; - if ( fp == NULL || (fdp->fd_ofileflags[ i ] & UF_RESERVED) != 0 || - FILEGLOB_DTYPE(fp->f_fglob) != DTYPE_SOCKET ) { + if (fp == NULL || (fdp->fd_ofileflags[ i ] & UF_RESERVED) != 0) { continue; } - sockp = (struct socket *)fp->f_fglob->fg_data; - /* skip if only clearing this thread's sockets */ - if ((thread) && (sockp->so_background_thread != thread)) { - continue; + if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_SOCKET) { + sockp = (struct socket *)fp->f_fglob->fg_data; + /* skip if only clearing this thread's sockets */ + if ((thread) && (sockp->so_background_thread != thread)) { + continue; + } + socket_clear_traffic_mgt_flags(sockp, TRAFFIC_MGT_SO_BACKGROUND); + sockp->so_background_thread = NULL; } - socket_clear_traffic_mgt_flags(sockp, TRAFFIC_MGT_SO_BACKGROUND); - sockp->so_background_thread = NULL; +#if NECP + else if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_NETPOLICY) { + necp_set_client_as_background(p, fp, background); + } +#endif /* NECP */ } } @@ -1509,9 +1525,15 @@ iopolicysys_disk(struct proc *p __unused, int cmd, int scope, int policy, struct break; case IOPOL_SCOPE_DARWIN_BG: +#if CONFIG_EMBEDDED + /* Embedded doesn't want this as BG is always IOPOL_THROTTLE */ + error = ENOTSUP; + goto out; +#else /* CONFIG_EMBEDDED */ thread = THREAD_NULL; policy_flavor = TASK_POLICY_DARWIN_BG_IOPOL; break; +#endif /* CONFIG_EMBEDDED */ default: error = EINVAL; @@ -1670,7 +1692,13 @@ gather_rusage_info(proc_t p, rusage_info_current *ru, int flavor) struct rusage_info_child *ri_child; assert(p->p_stats != NULL); + memset(ru, 0, sizeof(*ru)); switch(flavor) { + case RUSAGE_INFO_V4: + ru->ri_logical_writes = get_task_logical_writes(p->task); + ru->ri_lifetime_max_phys_footprint = get_task_phys_footprint_lifetime_max(p->task); + fill_task_monotonic_rusage(p->task, ru); + /* fall through */ case RUSAGE_INFO_V3: fill_task_qos_rusage(p->task, ru); @@ -1705,131 +1733,57 @@ gather_rusage_info(proc_t p, rusage_info_current *ru, int flavor) } } -static void -rusage_info_conversion(rusage_info_t ri_info, rusage_info_current *ri_current, int flavor) -{ - struct rusage_info_v0 *ri_v0; - struct rusage_info_v1 *ri_v1; - struct rusage_info_v2 *ri_v2; - - switch (flavor) { - - case RUSAGE_INFO_V2: - ri_v2 = (struct rusage_info_v2 *)ri_info; - ri_v2->ri_diskio_bytesread = ri_current->ri_diskio_bytesread; - ri_v2->ri_diskio_byteswritten = ri_current->ri_diskio_byteswritten; - /* fall through */ - - case RUSAGE_INFO_V1: - ri_v1 = (struct rusage_info_v1 *)ri_info; - ri_v1->ri_child_user_time = ri_current->ri_child_user_time; - ri_v1->ri_child_system_time = ri_current->ri_child_system_time; - ri_v1->ri_child_pkg_idle_wkups = ri_current->ri_child_pkg_idle_wkups; - ri_v1->ri_child_interrupt_wkups = ri_current->ri_child_interrupt_wkups; - ri_v1->ri_child_pageins = ri_current->ri_child_pageins; - ri_v1->ri_child_elapsed_abstime = ri_current->ri_child_elapsed_abstime; - /* fall through */ - - case RUSAGE_INFO_V0: - ri_v0 = (struct rusage_info_v0 *)ri_info; - memcpy(&ri_v0->ri_uuid[0], &ri_current->ri_uuid[0], sizeof(ri_v0->ri_uuid)); - ri_v0->ri_user_time = ri_current->ri_user_time; - ri_v0->ri_system_time = ri_current->ri_system_time; - ri_v0->ri_pkg_idle_wkups = ri_current->ri_pkg_idle_wkups; - ri_v0->ri_interrupt_wkups = ri_current->ri_interrupt_wkups; - ri_v0->ri_pageins = ri_current->ri_pageins; - ri_v0->ri_wired_size = ri_current->ri_wired_size; - ri_v0->ri_resident_size = ri_current->ri_resident_size; - ri_v0->ri_phys_footprint = ri_current->ri_phys_footprint; - ri_v0->ri_proc_start_abstime = ri_current->ri_proc_start_abstime; - ri_v0->ri_proc_exit_abstime = ri_current->ri_proc_exit_abstime; - - break; - - default: - break; - } -} - - int proc_get_rusage(proc_t p, int flavor, user_addr_t buffer, __unused int is_zombie) { - struct rusage_info_v0 ri_v0; - struct rusage_info_v1 ri_v1; - struct rusage_info_v2 ri_v2; - struct rusage_info_v3 ri_v3; - rusage_info_current ri_current; int error = 0; + size_t size = 0; switch (flavor) { case RUSAGE_INFO_V0: - /* - * If task is still alive, collect info from the live task itself. - * Otherwise, look to the cached info in the zombie proc. - */ - if (p->p_ru == NULL) { - gather_rusage_info(p, &ri_current, flavor); - ri_current.ri_proc_exit_abstime = 0; - rusage_info_conversion(&ri_v0, &ri_current, flavor); - } else { - rusage_info_conversion(&ri_v0, &p->p_ru->ri, flavor); - } - error = copyout(&ri_v0, buffer, sizeof (ri_v0)); + size = sizeof(struct rusage_info_v0); break; case RUSAGE_INFO_V1: - /* - * If task is still alive, collect info from the live task itself. - * Otherwise, look to the cached info in the zombie proc. - */ - if (p->p_ru == NULL) { - gather_rusage_info(p, &ri_current, flavor); - ri_current.ri_proc_exit_abstime = 0; - rusage_info_conversion(&ri_v1, &ri_current, flavor); - } else { - rusage_info_conversion(&ri_v1, &p->p_ru->ri, flavor); - } - error = copyout(&ri_v1, buffer, sizeof (ri_v1)); + size = sizeof(struct rusage_info_v1); break; case RUSAGE_INFO_V2: - /* - * If task is still alive, collect info from the live task itself. - * Otherwise, look to the cached info in the zombie proc. - */ - if (p->p_ru == NULL) { - gather_rusage_info(p, &ri_current, flavor); - ri_current.ri_proc_exit_abstime = 0; - rusage_info_conversion(&ri_v2, &ri_current, flavor); - } else { - rusage_info_conversion(&ri_v2, &p->p_ru->ri, flavor); - } - error = copyout(&ri_v2, buffer, sizeof (ri_v2)); + size = sizeof(struct rusage_info_v2); break; case RUSAGE_INFO_V3: - /* - * If task is still alive, collect info from the live task itself. - * Otherwise, look to the cached info in the zombie proc. - */ - if (p->p_ru == NULL) { - gather_rusage_info(p, &ri_v3, flavor); - ri_v3.ri_proc_exit_abstime = 0; - } else { - ri_v3 = p->p_ru->ri; - } - error = copyout(&ri_v3, buffer, sizeof (ri_v3)); + size = sizeof(struct rusage_info_v3); break; - default: - error = EINVAL; + case RUSAGE_INFO_V4: + size = sizeof(struct rusage_info_v4); break; + + default: + return EINVAL; + } + + if(size == 0) { + return EINVAL; } - return (error); + /* + * If task is still alive, collect info from the live task itself. + * Otherwise, look to the cached info in the zombie proc. + */ + if (p->p_ru == NULL) { + gather_rusage_info(p, &ri_current, flavor); + ri_current.ri_proc_exit_abstime = 0; + error = copyout(&ri_current, buffer, size); + } else { + ri_current = p->p_ru->ri; + error = copyout(&p->p_ru->ri, buffer, size); + } + + return (error); } static int @@ -1955,3 +1909,10 @@ int thread_selfusage(struct proc *p __unused, struct thread_selfusage_args *uap return (0); } + +#if !MONOTONIC +int thread_selfcounts(__unused struct proc *p, __unused struct thread_selfcounts_args *uap, __unused int *ret_out) +{ + return ENOTSUP; +} +#endif /* !MONOTONIC */ diff --git a/bsd/kern/kern_shutdown.c b/bsd/kern/kern_shutdown.c index 216a32baf..2be1a1310 100644 --- a/bsd/kern/kern_shutdown.c +++ b/bsd/kern/kern_shutdown.c @@ -84,6 +84,8 @@ static void sd_log(vfs_context_t, const char *, ...); static void proc_shutdown(void); static void kernel_hwm_panic_info(void); extern void IOSystemShutdownNotification(void); +extern void halt_log_enter(const char * what, const void * pc, uint64_t time); + #if DEVELOPMENT || DEBUG extern boolean_t kdp_has_polled_corefile(void); #endif /* DEVELOPMENT || DEBUG */ @@ -112,31 +114,29 @@ static int sd_callback2(proc_t p, void * arg); static int sd_callback3(proc_t p, void * arg); extern boolean_t panic_include_zprint; -extern vm_offset_t panic_kext_memory_info; -extern vm_size_t panic_kext_memory_size; +extern mach_memory_info_t *panic_kext_memory_info; +extern vm_size_t panic_kext_memory_size; static void kernel_hwm_panic_info(void) { - mach_memory_info_t *memory_info; - unsigned int num_sites; - kern_return_t kr; + unsigned int num_sites; + kern_return_t kr; panic_include_zprint = TRUE; - panic_kext_memory_info = 0; + panic_kext_memory_info = NULL; panic_kext_memory_size = 0; - num_sites = VM_KERN_MEMORY_COUNT + VM_KERN_COUNTER_COUNT; - panic_kext_memory_size = round_page(num_sites * sizeof(mach_zone_info_t)); - - kr = kmem_alloc(kernel_map, (vm_offset_t *) &panic_kext_memory_info, panic_kext_memory_size, VM_KERN_MEMORY_OSFMK); + num_sites = vm_page_diagnose_estimate(); + panic_kext_memory_size = num_sites * sizeof(panic_kext_memory_info[0]); + + kr = kmem_alloc(kernel_map, (vm_offset_t *)&panic_kext_memory_info, round_page(panic_kext_memory_size), VM_KERN_MEMORY_OSFMK); if (kr != KERN_SUCCESS) { - panic_kext_memory_info = 0; + panic_kext_memory_info = NULL; return; } - memory_info = (mach_memory_info_t *)panic_kext_memory_info; - vm_page_diagnose(memory_info, num_sites, 0); - return; + + vm_page_diagnose(panic_kext_memory_info, num_sites, 0); } int @@ -149,6 +149,7 @@ int reboot_kernel(int howto, char *message) { int hostboot_option=0; + uint64_t startTime; if (!OSCompareAndSwap(0, 1, &system_inshutdown)) { if ( (howto&RB_QUICK) == RB_QUICK) @@ -177,19 +178,28 @@ reboot_kernel(int howto, char *message) */ /* handle live procs (deallocate their root and current directories), suspend initproc */ + + startTime = mach_absolute_time(); proc_shutdown(); + halt_log_enter("proc_shutdown", 0, mach_absolute_time() - startTime); #if CONFIG_AUDIT + startTime = mach_absolute_time(); audit_shutdown(); + halt_log_enter("audit_shutdown", 0, mach_absolute_time() - startTime); #endif if (unmountroot_pre_hook != NULL) unmountroot_pre_hook(); + startTime = mach_absolute_time(); sync((proc_t)NULL, (void *)NULL, (int *)NULL); - if (kdebug_enable) + if (kdebug_enable) { + startTime = mach_absolute_time(); kdbg_dump_trace_to_file("/var/log/shutdown/shutdown.trace"); + halt_log_enter("shutdown.trace", 0, mach_absolute_time() - startTime); + } /* * Unmount filesystems @@ -199,10 +209,13 @@ reboot_kernel(int howto, char *message) if (!(howto & RB_PANIC) || !kdp_has_polled_corefile()) #endif /* DEVELOPMENT || DEBUG */ { + startTime = mach_absolute_time(); vfs_unmountall(); + halt_log_enter("vfs_unmountall", 0, mach_absolute_time() - startTime); } /* Wait for the buffer cache to clean remaining dirty buffers */ + startTime = mach_absolute_time(); for (iter = 0; iter < 100; iter++) { nbusy = count_busy_buffers(); if (nbusy == 0) @@ -214,6 +227,7 @@ reboot_kernel(int howto, char *message) printf("giving up\n"); else printf("done\n"); + halt_log_enter("bufferclean", 0, mach_absolute_time() - startTime); } #if NETWORKING /* @@ -221,7 +235,9 @@ reboot_kernel(int howto, char *message) * because that will lock out softints which the disk * drivers depend on to finish DMAs. */ + startTime = mach_absolute_time(); if_down_all(); + halt_log_enter("if_down_all", 0, mach_absolute_time() - startTime); #endif /* NETWORKING */ force_reboot: @@ -455,7 +471,7 @@ sd_callback3(proc_t p, void * args) * * POSIX modifications: * - * For POSIX fcntl() file locking call vno_lockrelease() on + * For POSIX fcntl() file locking call vno_lockrelease() on * the file to release all of its record locks, if any. */ @@ -474,10 +490,10 @@ proc_shutdown(void) * Kill as many procs as we can. (Except ourself...) */ self = (struct proc *)current_proc(); - + /* * Signal the init with SIGTERM so that he does not launch - * new processes + * new processes */ p = proc_find(1); if (p && p != self) { @@ -506,11 +522,11 @@ sigterm_loop: proc_list_lock(); if (proc_shutdown_exitcount != 0) { /* - * now wait for up to 30 seconds to allow those procs catching SIGTERM - * to digest it - * as soon as these procs have exited, we'll continue on to the next step - */ - ts.tv_sec = 30; + * now wait for up to 3 seconds to allow those procs catching SIGTERM + * to digest it + * as soon as these procs have exited, we'll continue on to the next step + */ + ts.tv_sec = 3; ts.tv_nsec = 0; error = msleep(&proc_shutdown_exitcount, proc_list_mlock, PWAIT, "shutdownwait", &ts); if (error != 0) { @@ -523,7 +539,6 @@ sigterm_loop: p->p_listflag &= ~P_LIST_EXITCOUNT; } } - } proc_list_unlock(); } @@ -532,7 +547,6 @@ sigterm_loop: * log the names of the unresponsive tasks */ - proc_list_lock(); for (p = allproc.lh_first; p; p = p->p_list.le_next) { @@ -543,8 +557,6 @@ sigterm_loop: } proc_list_unlock(); - - delay_for_interval(1000 * 5, 1000 * 1000); } /* @@ -560,16 +572,18 @@ sigterm_loop: /* post a SIGKILL to all that catch SIGTERM and not marked for delay */ proc_rebootscan(sd_callback2, (void *)&sdargs, sd_filt2, (void *)&sfargs); + error = 0; + if (sdargs.activecount != 0 && proc_shutdown_exitcount!= 0) { proc_list_lock(); if (proc_shutdown_exitcount != 0) { /* - * wait for up to 60 seconds to allow these procs to exit normally - * - * History: The delay interval was changed from 100 to 200 - * for NFS requests in particular. - */ - ts.tv_sec = 60; + * wait for up to 60 seconds to allow these procs to exit normally + * + * History: The delay interval was changed from 100 to 200 + * for NFS requests in particular. + */ + ts.tv_sec = 10; ts.tv_nsec = 0; error = msleep(&proc_shutdown_exitcount, proc_list_mlock, PWAIT, "shutdownwait", &ts); if (error != 0) { @@ -586,6 +600,23 @@ sigterm_loop: proc_list_unlock(); } + if (error == ETIMEDOUT) { + /* + * log the names of the unresponsive tasks + */ + + proc_list_lock(); + + for (p = allproc.lh_first; p; p = p->p_list.le_next) { + if (p->p_shutdownstate == 2) { + printf("%s[%d]: didn't act on SIGKILL\n", p->p_comm, p->p_pid); + sd_log(ctx, "%s[%d]: didn't act on SIGKILL\n", p->p_comm, p->p_pid); + } + } + + proc_list_unlock(); + } + /* * if we still have procs that haven't exited, then brute force 'em */ @@ -596,6 +627,8 @@ sigterm_loop: sdargs.countproc = 0; sdargs.activecount = 0; + + /* post a SIGTERM to all that catch SIGTERM and not marked for delay */ proc_rebootscan(sd_callback3, (void *)&sdargs, sd_filt2, (void *)&sfargs); printf("\n"); diff --git a/bsd/kern/kern_sig.c b/bsd/kern/kern_sig.c index 5d38f292e..ca7b6c584 100644 --- a/bsd/kern/kern_sig.c +++ b/bsd/kern/kern_sig.c @@ -95,8 +95,6 @@ #include -#include - #include #include @@ -116,6 +114,11 @@ #include #include +#include + +#if CONFIG_MACF +#include +#endif /* * Missing prototypes that Mach should export @@ -149,13 +152,13 @@ kern_return_t semaphore_timedwait_trap_internal(mach_port_name_t, unsigned int, kern_return_t semaphore_wait_signal_trap_internal(mach_port_name_t, mach_port_name_t, void (*)(kern_return_t)); kern_return_t semaphore_wait_trap_internal(mach_port_name_t, void (*)(kern_return_t)); -static int filt_sigattach(struct knote *kn); +static int filt_sigattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_sigdetach(struct knote *kn); static int filt_signal(struct knote *kn, long hint); static int filt_signaltouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_signalprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops sig_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) sig_filtops = { .f_attach = filt_sigattach, .f_detach = filt_sigdetach, .f_event = filt_signal, @@ -481,11 +484,11 @@ sigaction(proc_t p, struct sigaction_args *uap, __unused int32_t *retval) sa->sa_flags |= SA_NOCLDWAIT; if (IS_64BIT_PROCESS(p)) { - struct user64_sigaction vec64; + struct user64_sigaction vec64 = {}; sigaction_kern_to_user64(sa, &vec64); error = copyout(&vec64, uap->osa, sizeof(vec64)); } else { - struct user32_sigaction vec32; + struct user32_sigaction vec32 = {}; sigaction_kern_to_user32(sa, &vec32); error = copyout(&vec32, uap->osa, sizeof(vec32)); } @@ -1405,11 +1408,11 @@ sigaltstack(__unused proc_t p, struct sigaltstack_args *uap, __unused int32_t *r onstack = pstk->ss_flags & SA_ONSTACK; if (uap->oss) { if (IS_64BIT_PROCESS(p)) { - struct user64_sigaltstack ss64; + struct user64_sigaltstack ss64 = {}; sigaltstack_kern_to_user64(pstk, &ss64); error = copyout(&ss64, uap->oss, sizeof(ss64)); } else { - struct user32_sigaltstack ss32; + struct user32_sigaltstack ss32 = {}; sigaltstack_kern_to_user32(pstk, &ss32); error = copyout(&ss32, uap->oss, sizeof(ss32)); } @@ -1668,7 +1671,7 @@ terminate_with_payload_internal(struct proc *cur_proc, int target_pid, uint32_t reason_code, 0, 0); signal_reason = build_userspace_exit_reason(reason_namespace, reason_code, payload, payload_size, - reason_string, reason_flags); + reason_string, (reason_flags | OS_REASON_FLAG_NO_CRASHED_TID)); if (target_pid == cur_proc->p_pid) { /* @@ -2507,6 +2510,7 @@ psignal_internal(proc_t p, task_t task, thread_t thread, int flavor, int signum, assert(signal_reason == NULL); OSBitOrAtomic(P_CONTINUED, &sig_proc->p_flag); sig_proc->p_contproc = sig_proc->p_pid; + sig_proc->p_xstat = signum; (void) task_resume_internal(sig_task); @@ -2760,6 +2764,8 @@ issignal_locked(proc_t p) proc_lock(pp); pp->si_pid = p->p_pid; + pp->p_xhighbits = p->p_xhighbits; + p->p_xhighbits = 0; pp->si_status = p->p_xstat; pp->si_code = CLD_TRAPPED; pp->si_uid = r_uid; @@ -3211,7 +3217,7 @@ postsig_locked(int signum) */ static int -filt_sigattach(struct knote *kn) +filt_sigattach(struct knote *kn, __unused struct kevent_internal_s *kev) { proc_t p = current_proc(); /* can attach only to oneself */ diff --git a/bsd/kern/kern_symfile.c b/bsd/kern/kern_symfile.c index f1586ef1f..46018b2de 100644 --- a/bsd/kern/kern_symfile.c +++ b/bsd/kern/kern_symfile.c @@ -148,7 +148,7 @@ kern_ioctl_file_extents(struct kern_direct_file_io_ref_t * ref, u_long theIoctl, if (filechunk > (size_t)(end - offset)) filechunk = (size_t)(end - offset); error = VNOP_BLOCKMAP(ref->vp, offset, filechunk, &blkno, - &filechunk, NULL, VNODE_WRITE, NULL); + &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) break; if (-1LL == blkno) continue; fileblk = blkno * ref->blksize; @@ -382,7 +382,7 @@ kern_open_file_for_direct_io(const char * name, daddr64_t blkno; error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno, - &filechunk, NULL, VNODE_WRITE, NULL); + &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) goto out; if (-1LL == blkno) continue; fileblk = blkno * ref->blksize; diff --git a/bsd/kern/kern_synch.c b/bsd/kern/kern_synch.c index 5f3b8546b..841cdeba9 100644 --- a/bsd/kern/kern_synch.c +++ b/bsd/kern/kern_synch.c @@ -40,8 +40,6 @@ #include #include -#include - #include #include #include diff --git a/bsd/kern/kern_sysctl.c b/bsd/kern/kern_sysctl.c index 29b714684..af90ce9ff 100644 --- a/bsd/kern/kern_sysctl.c +++ b/bsd/kern/kern_sysctl.c @@ -104,18 +104,24 @@ #include #include #include +#include #include #include +#include #include #include #include +#include #include +#include #include #include #include +#include #include +#include #include #include #include @@ -166,7 +172,6 @@ extern int lowpri_IO_window_msecs; extern int lowpri_IO_delay_msecs; extern int nx_enabled; extern int speculative_reads_disabled; -extern int ignore_is_ssd; extern unsigned int speculative_prefetch_max; extern unsigned int speculative_prefetch_max_iosize; extern unsigned int preheat_max_bytes; @@ -227,8 +232,6 @@ netboot_root(void); int pcsamples_ops(int *name, u_int namelen, user_addr_t where, size_t *sizep, proc_t p); -__private_extern__ kern_return_t -reset_vmobjectcache(unsigned int val1, unsigned int val2); int sysctl_procargs(int *name, u_int namelen, user_addr_t where, size_t *sizep, proc_t cur_proc); @@ -257,7 +260,9 @@ STATIC int sysctl_kdebug_ops SYSCTL_HANDLER_ARGS; #if COUNT_SYSCALLS STATIC int sysctl_docountsyscalls SYSCTL_HANDLER_ARGS; #endif /* COUNT_SYSCALLS */ +#if !CONFIG_EMBEDDED STATIC int sysctl_doprocargs SYSCTL_HANDLER_ARGS; +#endif /* !CONFIG_EMBEDDED */ STATIC int sysctl_doprocargs2 SYSCTL_HANDLER_ARGS; STATIC int sysctl_prochandle SYSCTL_HANDLER_ARGS; STATIC int sysctl_aiomax(struct sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req); @@ -752,6 +757,7 @@ sysctl_prochandle SYSCTL_HANDLER_ARGS int uidcheck = 0; int ruidcheck = 0; int ttycheck = 0; + int success = 0; if (namelen != 1 && !(namelen == 0 && cmd == KERN_PROC_ALL)) return (EINVAL); @@ -808,8 +814,16 @@ sysctl_prochandle SYSCTL_HANDLER_ARGS if (namelen) args.uidval = name[0]; - proc_iterate((PROC_ALLPROCLIST | PROC_ZOMBPROCLIST), - sysdoproc_callback, &args, filterfn, name); + success = proc_iterate((PROC_ALLPROCLIST | PROC_ZOMBPROCLIST), + sysdoproc_callback, &args, filterfn, name); + + /* + * rdar://problem/28433391: if we can't iterate over the processes, + * make sure to return an error. + */ + + if (success != 0) + return (ENOMEM); if (error) return (error); @@ -1180,6 +1194,7 @@ SYSCTL_PROC(_kern, KERN_KDEBUG, kdebug, CTLTYPE_NODE|CTLFLAG_RD | CTLFLAG_LOCKED ""); +#if !CONFIG_EMBEDDED /* * Return the top *sizep bytes of the user stack, or the entire area of the * user stack down through the saved exec_path, whichever is smaller. @@ -1210,6 +1225,7 @@ SYSCTL_PROC(_kern, KERN_PROCARGS, procargs, CTLTYPE_NODE|CTLFLAG_RD | CTLFLAG_LO sysctl_doprocargs, /* Handler function */ NULL, /* Data pointer */ ""); +#endif /* !CONFIG_EMBEDDED */ STATIC int sysctl_doprocargs2 SYSCTL_HANDLER_ARGS @@ -1258,6 +1274,7 @@ sysctl_procargsx(int *name, u_int namelen, user_addr_t where, int pid; kauth_cred_t my_cred; uid_t uid; + int argc = -1; if ( namelen < 1 ) return(EINVAL); @@ -1304,24 +1321,23 @@ sysctl_procargsx(int *name, u_int namelen, user_addr_t where, proc_rele(p); return(EFAULT); } - - size = p->p_argslen; + + size = p->p_argslen; proc_rele(p); - if (argc_yes) { - size += sizeof(int); - } - else { + if (argc_yes) { + size += sizeof(int); + } else { /* * old PROCARGS will return the executable's path and plus some * extra space for work alignment and data tags */ - size += PATH_MAX + (6 * sizeof(int)); - } + size += PATH_MAX + (6 * sizeof(int)); + } size += (size & (sizeof(int) - 1)) ? (sizeof(int) - (size & (sizeof(int) - 1))) : 0; *sizep = size; return (0); } - + my_cred = kauth_cred_proc_ref(p); uid = kauth_cred_getuid(my_cred); kauth_cred_unref(&my_cred); @@ -1337,7 +1353,6 @@ sysctl_procargsx(int *name, u_int namelen, user_addr_t where, arg_addr = p->user_stack - arg_size; - /* * Before we can block (any VM code), make another * reference to the map to keep it alive. We do @@ -1348,7 +1363,10 @@ sysctl_procargsx(int *name, u_int namelen, user_addr_t where, proc_rele(p); return(EINVAL); } - + + /* save off argc before releasing the proc */ + argc = p->p_argc; + argslen = p->p_argslen; /* * Once we have a task reference we can convert that into a @@ -1421,7 +1439,7 @@ sysctl_procargsx(int *name, u_int namelen, user_addr_t where, if (argc_yes) { /* Put processes argc as the first word in the copyout buffer */ - suword(where, p->p_argc); + suword(where, argc); error = copyout(data, (where + sizeof(int)), size); size += sizeof(int); } else { @@ -1668,15 +1686,43 @@ SYSCTL_PROC(_kern, KERN_OSVERSION, osversion, osversion, 256 /* OSVERSIZE*/, sysctl_osversion, "A", ""); +static uint64_t osvariant_status = 0; + +STATIC int +sysctl_osvariant_status(__unused struct sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req) +{ + if (req->newptr != 0) { + /* + * Can only ever be set by launchd, and only once at boot. + */ + if (req->p->p_pid != 1 || osvariant_status != 0) { + return EPERM; + } + } + + return sysctl_handle_quad(oidp, arg1, arg2, req); +} + +SYSCTL_PROC(_kern, OID_AUTO, osvariant_status, + CTLFLAG_RW | CTLTYPE_QUAD | CTLFLAG_LOCKED | CTLFLAG_MASKED, + &osvariant_status, sizeof(osvariant_status), + sysctl_osvariant_status, "Q", "Opaque flags used to cache OS variant information"); + STATIC int sysctl_sysctl_bootargs (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { int error; - char buf[256]; + /* BOOT_LINE_LENGTH */ +#if CONFIG_EMBEDDED + size_t boot_args_len = 256; +#else + size_t boot_args_len = 1024; +#endif + char buf[boot_args_len]; - strlcpy(buf, PE_boot_args(), 256); - error = sysctl_io_string(req, buf, 256, 0, NULL); + strlcpy(buf, PE_boot_args(), boot_args_len); + error = sysctl_io_string(req, buf, boot_args_len, 0, NULL); return(error); } @@ -1732,7 +1778,6 @@ sysctl_maxvnodes (__unused struct sysctl_oid *oidp, __unused void *arg1, __unuse int error = sysctl_io_number(req, desiredvnodes, sizeof(int), &desiredvnodes, NULL); if (oldval != desiredvnodes) { - reset_vmobjectcache(oldval, desiredvnodes); resize_namecache(desiredvnodes); } @@ -1768,7 +1813,71 @@ extern int sched_smt_balance; SYSCTL_INT(_kern, OID_AUTO, sched_smt_balance, CTLFLAG_KERN| CTLFLAG_RW| CTLFLAG_LOCKED, &sched_smt_balance, 0, ""); -#endif +#if __arm__ || __arm64__ +extern uint32_t perfcontrol_requested_recommended_cores; +SYSCTL_UINT(_kern, OID_AUTO, sched_recommended_cores, + CTLFLAG_KERN | CTLFLAG_RD | CTLFLAG_LOCKED, + &perfcontrol_requested_recommended_cores, 0, ""); + +/* Scheduler perfcontrol callouts sysctls */ +SYSCTL_DECL(_kern_perfcontrol_callout); +SYSCTL_NODE(_kern, OID_AUTO, perfcontrol_callout, CTLFLAG_RW | CTLFLAG_LOCKED, 0, + "scheduler perfcontrol callouts"); + +extern int perfcontrol_callout_stats_enabled; +SYSCTL_INT(_kern_perfcontrol_callout, OID_AUTO, stats_enabled, + CTLFLAG_KERN| CTLFLAG_RW| CTLFLAG_LOCKED, + &perfcontrol_callout_stats_enabled, 0, ""); + +extern uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type, + perfcontrol_callout_stat_t stat); + +/* On-Core Callout */ +STATIC int +sysctl_perfcontrol_callout_stat +(__unused struct sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req) +{ + perfcontrol_callout_stat_t stat = (perfcontrol_callout_stat_t)arg1; + perfcontrol_callout_type_t type = (perfcontrol_callout_type_t)arg2; + return sysctl_io_number(req, (int)perfcontrol_callout_stat_avg(type, stat), + sizeof(int), NULL, NULL); +} + +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, oncore_instr, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_INSTRS, PERFCONTROL_CALLOUT_ON_CORE, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, oncore_cycles, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_CYCLES, PERFCONTROL_CALLOUT_ON_CORE, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, offcore_instr, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_INSTRS, PERFCONTROL_CALLOUT_OFF_CORE, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, offcore_cycles, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_CYCLES, PERFCONTROL_CALLOUT_OFF_CORE, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, context_instr, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_INSTRS, PERFCONTROL_CALLOUT_CONTEXT, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, context_cycles, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_CYCLES, PERFCONTROL_CALLOUT_CONTEXT, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, update_instr, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_INSTRS, PERFCONTROL_CALLOUT_STATE_UPDATE, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, update_cycles, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_CYCLES, PERFCONTROL_CALLOUT_STATE_UPDATE, + sysctl_perfcontrol_callout_stat, "I", ""); + +#endif /* __arm__ || __arm64__ */ +#endif /* (DEVELOPMENT || DEBUG) */ STATIC int sysctl_securelvl @@ -1847,10 +1956,6 @@ SYSCTL_INT(_kern, KERN_SPECULATIVE_READS, speculative_reads_disabled, CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED, &speculative_reads_disabled, 0, ""); -SYSCTL_INT(_kern, OID_AUTO, ignore_is_ssd, - CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED, - &ignore_is_ssd, 0, ""); - SYSCTL_UINT(_kern, OID_AUTO, preheat_max_bytes, CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED, &preheat_max_bytes, 0, ""); @@ -1908,12 +2013,12 @@ sysctl_boottime struct proc *p = req->p; if (proc_is64bit(p)) { - struct user64_timeval t; + struct user64_timeval t = {}; t.tv_sec = tv.tv_sec; t.tv_usec = tv.tv_usec; return sysctl_io_opaque(req, &t, sizeof(t), NULL); } else { - struct user32_timeval t; + struct user32_timeval t = {}; t.tv_sec = tv.tv_sec; t.tv_usec = tv.tv_usec; return sysctl_io_opaque(req, &t, sizeof(t), NULL); @@ -2001,7 +2106,7 @@ sysctl_imgsrcinfo (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { int error; - struct imgsrc_info info[MAX_IMAGEBOOT_NESTING]; /* 2 for now, no problem */ + struct imgsrc_info info[MAX_IMAGEBOOT_NESTING] = {}; /* 2 for now, no problem */ uint32_t i; vnode_t rvp, devvp; @@ -2083,7 +2188,7 @@ SYSCTL_NODE(_kern_timer, OID_AUTO, longterm, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "lo enum { THRESHOLD, QCOUNT, ENQUEUES, DEQUEUES, ESCALATES, SCANS, PREEMPTS, - LATENCY, LATENCY_MIN, LATENCY_MAX + LATENCY, LATENCY_MIN, LATENCY_MAX, SCAN_LIMIT, PAUSES }; extern uint64_t timer_sysctl_get(int); extern int timer_sysctl_set(int, uint64_t); @@ -2108,10 +2213,13 @@ sysctl_timer SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, threshold, CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, (void *) THRESHOLD, 0, sysctl_timer, "Q", ""); +SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, scan_limit, + CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, + (void *) SCAN_LIMIT, 0, sysctl_timer, "Q", ""); SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, qlen, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, (void *) QCOUNT, 0, sysctl_timer, "Q", ""); -#if DEBUG +#if DEBUG SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, enqueues, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, (void *) ENQUEUES, 0, sysctl_timer, "Q", ""); @@ -2136,6 +2244,9 @@ SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, latency_min, SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, latency_max, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, (void *) LATENCY_MAX, 0, sysctl_timer, "Q", ""); +SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, scan_pauses, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *) PAUSES, 0, sysctl_timer, "Q", ""); #endif /* DEBUG */ STATIC int @@ -2404,11 +2515,11 @@ sysctl_loadavg (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { if (proc_is64bit(req->p)) { - struct user64_loadavg loadinfo64; + struct user64_loadavg loadinfo64 = {}; fill_loadavg64(&averunnable, &loadinfo64); return sysctl_io_opaque(req, &loadinfo64, sizeof(loadinfo64), NULL); } else { - struct user32_loadavg loadinfo32; + struct user32_loadavg loadinfo32 = {}; fill_loadavg32(&averunnable, &loadinfo32); return sysctl_io_opaque(req, &loadinfo32, sizeof(loadinfo32), NULL); } @@ -2448,7 +2559,7 @@ sysctl_swapusage uint64_t swap_avail; vm_size_t swap_pagesize; boolean_t swap_encrypted; - struct xsw_usage xsu; + struct xsw_usage xsu = {}; error = macx_swapinfo(&swap_total, &swap_avail, @@ -2657,9 +2768,15 @@ vm_map_size_t vm_user_wire_limit; /* * There needs to be a more automatic/elegant way to do this */ +#if defined(__ARM__) +SYSCTL_INT(_vm, OID_AUTO, global_no_user_wire_amount, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_global_no_user_wire_amount, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, global_user_wire_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_global_user_wire_limit, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, user_wire_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_user_wire_limit, 0, ""); +#else SYSCTL_QUAD(_vm, OID_AUTO, global_no_user_wire_amount, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_global_no_user_wire_amount, ""); SYSCTL_QUAD(_vm, OID_AUTO, global_user_wire_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_global_user_wire_limit, ""); SYSCTL_QUAD(_vm, OID_AUTO, user_wire_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_user_wire_limit, ""); +#endif extern int vm_map_copy_overwrite_aligned_src_not_internal; extern int vm_map_copy_overwrite_aligned_src_not_symmetric; @@ -2693,7 +2810,9 @@ extern uint32_t vm_compressor_majorcompact_threshold_divisor; extern uint32_t vm_compressor_unthrottle_threshold_divisor; extern uint32_t vm_compressor_catchup_threshold_divisor; extern uint32_t vm_compressor_time_thread; -extern uint64_t vm_compressor_thread_runtime; +#if DEVELOPMENT || DEBUG +extern vmct_stats_t vmct_stats; +#endif SYSCTL_QUAD(_vm, OID_AUTO, compressor_input_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &c_segment_input_bytes, ""); SYSCTL_QUAD(_vm, OID_AUTO, compressor_compressed_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &c_segment_compressed_bytes, ""); @@ -2719,7 +2838,26 @@ SYSCTL_INT(_vm, OID_AUTO, compressor_catchup_threshold_divisor, CTLFLAG_RW | CTL SYSCTL_STRING(_vm, OID_AUTO, swapfileprefix, CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED, swapfilename, sizeof(swapfilename) - SWAPFILENAME_INDEX_LEN, ""); SYSCTL_INT(_vm, OID_AUTO, compressor_timing_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_compressor_time_thread, 0, ""); -SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_runtime, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_compressor_thread_runtime, ""); + +#if DEVELOPMENT || DEBUG +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_runtime0, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_runtimes[0], ""); +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_runtime1, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_runtimes[1], ""); + +SYSCTL_QUAD(_vm, OID_AUTO, compressor_threads_total, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_cthreads_total, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_pages0, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_pages[0], ""); +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_pages1, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_pages[1], ""); + +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_iterations0, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_iterations[0], ""); +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_iterations1, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_iterations[1], ""); + +SYSCTL_INT(_vm, OID_AUTO, compressor_thread_minpages0, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_minpages[0], 0, ""); +SYSCTL_INT(_vm, OID_AUTO, compressor_thread_minpages1, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_minpages[1], 0, ""); + +SYSCTL_INT(_vm, OID_AUTO, compressor_thread_maxpages0, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_maxpages[0], 0, ""); +SYSCTL_INT(_vm, OID_AUTO, compressor_thread_maxpages1, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_maxpages[1], 0, ""); + +#endif SYSCTL_QUAD(_vm, OID_AUTO, lz4_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.lz4_compressions, ""); SYSCTL_QUAD(_vm, OID_AUTO, lz4_compression_failures, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.lz4_compression_failures, ""); @@ -2733,6 +2871,15 @@ SYSCTL_QUAD(_vm, OID_AUTO, lz4_decompressed_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, SYSCTL_QUAD(_vm, OID_AUTO, uc_decompressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.uc_decompressions, ""); SYSCTL_QUAD(_vm, OID_AUTO, wk_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_compressions, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wk_catime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_cabstime, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wkh_catime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wkh_cabstime, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wkh_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wkh_compressions, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wks_catime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_cabstime, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wks_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_compressions, ""); + SYSCTL_QUAD(_vm, OID_AUTO, wk_compressions_exclusive, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_compressions_exclusive, ""); SYSCTL_QUAD(_vm, OID_AUTO, wk_sv_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_sv_compressions, ""); SYSCTL_QUAD(_vm, OID_AUTO, wk_mzv_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_mzv_compressions, ""); @@ -2740,7 +2887,21 @@ SYSCTL_QUAD(_vm, OID_AUTO, wk_compression_failures, CTLFLAG_RD | CTLFLAG_LOCKED, SYSCTL_QUAD(_vm, OID_AUTO, wk_compressed_bytes_exclusive, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_compressed_bytes_exclusive, ""); SYSCTL_QUAD(_vm, OID_AUTO, wk_compressed_bytes_total, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_compressed_bytes_total, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wks_compressed_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_compressed_bytes, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wks_compression_failures, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_compression_failures, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wks_sv_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_sv_compressions, ""); + + SYSCTL_QUAD(_vm, OID_AUTO, wk_decompressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_decompressions, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wk_datime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_dabstime, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wkh_datime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wkh_dabstime, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wkh_decompressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wkh_decompressions, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wks_datime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_dabstime, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wks_decompressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_decompressions, ""); + SYSCTL_QUAD(_vm, OID_AUTO, wk_decompressed_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_decompressed_bytes, ""); SYSCTL_QUAD(_vm, OID_AUTO, wk_sv_decompressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_sv_decompressions, ""); @@ -2752,6 +2913,21 @@ SYSCTL_INT(_vm, OID_AUTO, lz4_max_preselects, CTLFLAG_RW | CTLFLAG_LOCKED, &vmct SYSCTL_INT(_vm, OID_AUTO, lz4_run_preselection_threshold, CTLFLAG_RW | CTLFLAG_LOCKED, &vmctune.lz4_run_preselection_threshold, 0, ""); SYSCTL_INT(_vm, OID_AUTO, lz4_run_continue_bytes, CTLFLAG_RW | CTLFLAG_LOCKED, &vmctune.lz4_run_continue_bytes, 0, ""); SYSCTL_INT(_vm, OID_AUTO, lz4_profitable_bytes, CTLFLAG_RW | CTLFLAG_LOCKED, &vmctune.lz4_profitable_bytes, 0, ""); +#if DEVELOPMENT || DEBUG +extern int vm_compressor_current_codec; +extern int vm_compressor_test_seg_wp; +extern boolean_t vm_compressor_force_sw_wkdm; +SYSCTL_INT(_vm, OID_AUTO, compressor_codec, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_compressor_current_codec, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, compressor_test_wp, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_compressor_test_seg_wp, 0, ""); + +SYSCTL_INT(_vm, OID_AUTO, wksw_force, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_compressor_force_sw_wkdm, 0, ""); +extern int precompy, wkswhw; + +SYSCTL_INT(_vm, OID_AUTO, precompy, CTLFLAG_RW | CTLFLAG_LOCKED, &precompy, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, wkswhw, CTLFLAG_RW | CTLFLAG_LOCKED, &wkswhw, 0, ""); +extern unsigned int vm_ktrace_enabled; +SYSCTL_INT(_vm, OID_AUTO, vm_ktrace, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_ktrace_enabled, 0, ""); +#endif #if CONFIG_PHANTOM_CACHE extern uint32_t phantom_cache_thrashing_threshold; @@ -2767,7 +2943,6 @@ SYSCTL_INT(_vm, OID_AUTO, phantom_cache_thrashing_threshold_ssd, CTLFLAG_RW | CT #if CONFIG_BACKGROUND_QUEUE extern uint32_t vm_page_background_count; -extern uint32_t vm_page_background_limit; extern uint32_t vm_page_background_target; extern uint32_t vm_page_background_internal_count; extern uint32_t vm_page_background_external_count; @@ -2781,7 +2956,6 @@ extern uint64_t vm_pageout_rejected_bq_external; SYSCTL_INT(_vm, OID_AUTO, vm_page_background_mode, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_page_background_mode, 0, ""); SYSCTL_INT(_vm, OID_AUTO, vm_page_background_exclude_external, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_page_background_exclude_external, 0, ""); -SYSCTL_INT(_vm, OID_AUTO, vm_page_background_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_page_background_limit, 0, ""); SYSCTL_INT(_vm, OID_AUTO, vm_page_background_target, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_page_background_target, 0, ""); SYSCTL_INT(_vm, OID_AUTO, vm_page_background_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_background_count, 0, ""); SYSCTL_INT(_vm, OID_AUTO, vm_page_background_internal_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_background_internal_count, 0, ""); @@ -2865,6 +3039,10 @@ SYSCTL_INT (_kern, OID_AUTO, stack_size, SYSCTL_INT (_kern, OID_AUTO, stack_depth_max, CTLFLAG_RD | CTLFLAG_LOCKED, (int *) &kernel_stack_depth_max, 0, "Max kernel stack depth at interrupt or context switch"); +extern unsigned int kern_feature_overrides; +SYSCTL_INT (_kern, OID_AUTO, kern_feature_overrides, + CTLFLAG_RD | CTLFLAG_LOCKED, &kern_feature_overrides, 0, "Kernel feature override mask"); + /* * enable back trace for port allocations */ @@ -2887,6 +3065,14 @@ SYSCTL_STRING(_kern, OID_AUTO, sched, * Only support runtime modification on embedded platforms * with development config enabled */ +#if CONFIG_EMBEDDED +#if !SECURE_KERNEL +extern int precise_user_kernel_time; +SYSCTL_INT(_kern, OID_AUTO, precise_user_kernel_time, + CTLFLAG_RW | CTLFLAG_LOCKED, + &precise_user_kernel_time, 0, "Precise accounting of kernel vs. user time"); +#endif +#endif /* Parameters related to timer coalescing tuning, to be replaced @@ -3069,6 +3255,77 @@ SYSCTL_INT(_kern, OID_AUTO, hv_support, &hv_support_available, 0, ""); #endif +#if CONFIG_EMBEDDED +STATIC int +sysctl_darkboot SYSCTL_HANDLER_ARGS +{ + int err = 0, value = 0; +#pragma unused(oidp, arg1, arg2, err, value, req) + + /* + * Handle the sysctl request. + * + * If this is a read, the function will set the value to the current darkboot value. Otherwise, + * we'll get the request identifier into "value" and then we can honor it. + */ + if ((err = sysctl_io_number(req, darkboot, sizeof(int), &value, NULL)) != 0) { + goto exit; + } + + /* writing requested, let's process the request */ + if (req->newptr) { + /* writing is protected by an entitlement */ + if (priv_check_cred(kauth_cred_get(), PRIV_DARKBOOT, 0) != 0) { + err = EPERM; + goto exit; + } + + switch (value) { + case MEMORY_MAINTENANCE_DARK_BOOT_UNSET: + /* + * If the darkboot sysctl is unset, the NVRAM variable + * must be unset too. If that's not the case, it means + * someone is doing something crazy and not supported. + */ + if (darkboot != 0) { + int ret = PERemoveNVRAMProperty(MEMORY_MAINTENANCE_DARK_BOOT_NVRAM_NAME); + if (ret) { + darkboot = 0; + } else { + err = EINVAL; + } + } + break; + case MEMORY_MAINTENANCE_DARK_BOOT_SET: + darkboot = 1; + break; + case MEMORY_MAINTENANCE_DARK_BOOT_SET_PERSISTENT: { + /* + * Set the NVRAM and update 'darkboot' in case + * of success. Otherwise, do not update + * 'darkboot' and report the failure. + */ + if (PEWriteNVRAMBooleanProperty(MEMORY_MAINTENANCE_DARK_BOOT_NVRAM_NAME, TRUE)) { + darkboot = 1; + } else { + err = EINVAL; + } + + break; + } + default: + err = EINVAL; + } + } + +exit: + return err; +} + +SYSCTL_PROC(_kern, OID_AUTO, darkboot, + CTLFLAG_KERN | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, + 0, 0, sysctl_darkboot, "I", ""); +#endif /* * This is set by core audio to tell tailspin (ie background tracing) how long @@ -3132,3 +3389,184 @@ SYSCTL_COMPAT_INT(_kern, OID_AUTO, development, CTLFLAG_RD | CTLFLAG_MASKED, NUL #else SYSCTL_COMPAT_INT(_kern, OID_AUTO, development, CTLFLAG_RD | CTLFLAG_MASKED, NULL, 0, ""); #endif + + +#if DEVELOPMENT || DEBUG + +static int +sysctl_panic_test SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int rval = 0; + char str[32] = "entry prelog postlog postcore"; + + rval = sysctl_handle_string(oidp, str, sizeof(str), req); + + if (rval == 0 && req->newptr) { + if (strncmp("entry", str, strlen("entry")) == 0) { + panic_with_options(0, NULL, DEBUGGER_OPTION_RECURPANIC_ENTRY, "test recursive panic at entry"); + } else if (strncmp("prelog", str, strlen("prelog")) == 0) { + panic_with_options(0, NULL, DEBUGGER_OPTION_RECURPANIC_PRELOG, "test recursive panic prior to writing a paniclog"); + } else if (strncmp("postlog", str, strlen("postlog")) == 0) { + panic_with_options(0, NULL, DEBUGGER_OPTION_RECURPANIC_POSTLOG, "test recursive panic subsequent to paniclog"); + } else if (strncmp("postcore", str, strlen("postcore")) == 0) { + panic_with_options(0, NULL, DEBUGGER_OPTION_RECURPANIC_POSTCORE, "test recursive panic subsequent to on-device core"); + } + } + + return rval; +} + +static int +sysctl_debugger_test SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int rval = 0; + char str[32] = "entry prelog postlog postcore"; + + rval = sysctl_handle_string(oidp, str, sizeof(str), req); + + if (rval == 0 && req->newptr) { + if (strncmp("entry", str, strlen("entry")) == 0) { + DebuggerWithContext(0, NULL, "test recursive panic via debugger at entry", DEBUGGER_OPTION_RECURPANIC_ENTRY); + } else if (strncmp("prelog", str, strlen("prelog")) == 0) { + DebuggerWithContext(0, NULL, "test recursive panic via debugger prior to writing a paniclog", DEBUGGER_OPTION_RECURPANIC_PRELOG); + } else if (strncmp("postlog", str, strlen("postlog")) == 0) { + DebuggerWithContext(0, NULL, "test recursive panic via debugger subsequent to paniclog", DEBUGGER_OPTION_RECURPANIC_POSTLOG); + } else if (strncmp("postcore", str, strlen("postcore")) == 0) { + DebuggerWithContext(0, NULL, "test recursive panic via debugger subsequent to on-device core", DEBUGGER_OPTION_RECURPANIC_POSTCORE); + } + } + + return rval; +} + +decl_lck_spin_data(, spinlock_panic_test_lock) + +__attribute__((noreturn)) +static void +spinlock_panic_test_acquire_spinlock(void * arg __unused, wait_result_t wres __unused) +{ + lck_spin_lock(&spinlock_panic_test_lock); + while (1) { ; } +} + +static int +sysctl_spinlock_panic_test SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + if (req->newlen == 0) + return EINVAL; + + thread_t panic_spinlock_thread; + /* Initialize panic spinlock */ + lck_grp_t * panic_spinlock_grp; + lck_grp_attr_t * panic_spinlock_grp_attr; + lck_attr_t * panic_spinlock_attr; + + panic_spinlock_grp_attr = lck_grp_attr_alloc_init(); + panic_spinlock_grp = lck_grp_alloc_init("panic_spinlock", panic_spinlock_grp_attr); + panic_spinlock_attr = lck_attr_alloc_init(); + + lck_spin_init(&spinlock_panic_test_lock, panic_spinlock_grp, panic_spinlock_attr); + + + /* Create thread to acquire spinlock */ + if (kernel_thread_start(spinlock_panic_test_acquire_spinlock, NULL, &panic_spinlock_thread) != KERN_SUCCESS) { + return EBUSY; + } + + /* Try to acquire spinlock -- should panic eventually */ + lck_spin_lock(&spinlock_panic_test_lock); + while(1) { ; } +} + +__attribute__((noreturn)) +static void +simultaneous_panic_worker +(void * arg, wait_result_t wres __unused) +{ + atomic_int *start_panic = (atomic_int *)arg; + + while (!atomic_load(start_panic)) { ; } + panic("SIMULTANEOUS PANIC TEST: INITIATING PANIC FROM CPU %d", cpu_number()); + __builtin_unreachable(); +} + +static int +sysctl_simultaneous_panic_test SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + if (req->newlen == 0) + return EINVAL; + + int i = 0, threads_to_create = 2 * processor_count; + atomic_int start_panic = 0; + unsigned int threads_created = 0; + thread_t new_panic_thread; + + for (i = threads_to_create; i > 0; i--) { + if (kernel_thread_start(simultaneous_panic_worker, (void *) &start_panic, &new_panic_thread) == KERN_SUCCESS) { + threads_created++; + } + } + + /* FAIL if we couldn't create at least processor_count threads */ + if (threads_created < processor_count) { + panic("SIMULTANEOUS PANIC TEST: FAILED TO CREATE ENOUGH THREADS, ONLY CREATED %d (of %d)", + threads_created, threads_to_create); + } + + atomic_exchange(&start_panic, 1); + while (1) { ; } +} + +SYSCTL_PROC(_debug, OID_AUTO, panic_test, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_MASKED, 0, 0, sysctl_panic_test, "A", "panic test"); +SYSCTL_PROC(_debug, OID_AUTO, debugger_test, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_MASKED, 0, 0, sysctl_debugger_test, "A", "debugger test"); +SYSCTL_PROC(_debug, OID_AUTO, spinlock_panic_test, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_MASKED, 0, 0, sysctl_spinlock_panic_test, "A", "spinlock panic test"); +SYSCTL_PROC(_debug, OID_AUTO, simultaneous_panic_test, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_MASKED, 0, 0, sysctl_simultaneous_panic_test, "A", "simultaneous panic test"); + + +#endif /* DEVELOPMENT || DEBUG */ + +const uint32_t thread_groups_supported = 0; + +STATIC int +sysctl_thread_groups_supported (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) +{ + int value = thread_groups_supported; + return sysctl_io_number(req, value, sizeof(value), NULL, NULL); +} + +SYSCTL_PROC(_kern, OID_AUTO, thread_groups_supported, CTLFLAG_RD | CTLFLAG_LOCKED | CTLFLAG_KERN, + 0, 0, &sysctl_thread_groups_supported, "I", "thread groups supported"); + +static int +sysctl_grade_cputype SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2, oidp) + int error = 0; + int type_tuple[2] = {}; + int return_value = 0; + + error = SYSCTL_IN(req, &type_tuple, sizeof(type_tuple)); + + if (error) { + return error; + } + + return_value = grade_binary(type_tuple[0], type_tuple[1]); + + error = SYSCTL_OUT(req, &return_value, sizeof(return_value)); + + if (error) { + return error; + } + + return error; +} + +SYSCTL_PROC(_kern, OID_AUTO, grade_cputype, + CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED|CTLTYPE_OPAQUE, + 0, 0, &sysctl_grade_cputype, "S", + "grade value of cpu_type_t+cpu_sub_type_t"); diff --git a/bsd/kern/kern_time.c b/bsd/kern/kern_time.c index 7f94f9b50..92da86a67 100644 --- a/bsd/kern/kern_time.c +++ b/bsd/kern/kern_time.c @@ -88,6 +88,8 @@ #if CONFIG_MACF #include #endif +#include +#include #define HZ 100 /* XXX */ @@ -174,13 +176,20 @@ settimeofday(__unused struct proc *p, struct settimeofday_args *uap, __unused i bzero(&atv, sizeof(atv)); + /* Check that this task is entitled to set the time or it is root */ + if (!IOTaskHasEntitlement(current_task(), SETTIME_ENTITLEMENT)) { + #if CONFIG_MACF - error = mac_system_check_settime(kauth_cred_get()); - if (error) - return (error); + error = mac_system_check_settime(kauth_cred_get()); + if (error) + return (error); #endif - if ((error = suser(kauth_cred_get(), &p->p_acflag))) - return (error); +#ifndef CONFIG_EMBEDDED + if ((error = suser(kauth_cred_get(), &p->p_acflag))) + return (error); +#endif + } + /* Verify all parameters before changing time */ if (uap->tv) { if (IS_64BIT_PROCESS(p)) { @@ -220,59 +229,6 @@ setthetime( clock_set_calendar_microtime(tv->tv_sec, tv->tv_usec); } -/* - * XXX Y2038 bug because of clock_adjtime() first argument - */ -/* ARGSUSED */ -int -adjtime(struct proc *p, struct adjtime_args *uap, __unused int32_t *retval) -{ - struct timeval atv; - int error; - -#if CONFIG_MACF - error = mac_system_check_settime(kauth_cred_get()); - if (error) - return (error); -#endif - if ((error = priv_check_cred(kauth_cred_get(), PRIV_ADJTIME, 0))) - return (error); - if (IS_64BIT_PROCESS(p)) { - struct user64_timeval user_atv; - error = copyin(uap->delta, &user_atv, sizeof(user_atv)); - atv.tv_sec = user_atv.tv_sec; - atv.tv_usec = user_atv.tv_usec; - } else { - struct user32_timeval user_atv; - error = copyin(uap->delta, &user_atv, sizeof(user_atv)); - atv.tv_sec = user_atv.tv_sec; - atv.tv_usec = user_atv.tv_usec; - } - if (error) - return (error); - - /* - * Compute the total correction and the rate at which to apply it. - */ - clock_adjtime(&atv.tv_sec, &atv.tv_usec); - - if (uap->olddelta) { - if (IS_64BIT_PROCESS(p)) { - struct user64_timeval user_atv; - user_atv.tv_sec = atv.tv_sec; - user_atv.tv_usec = atv.tv_usec; - error = copyout(&user_atv, uap->olddelta, sizeof(user_atv)); - } else { - struct user32_timeval user_atv; - user_atv.tv_sec = atv.tv_sec; - user_atv.tv_usec = atv.tv_usec; - error = copyout(&user_atv, uap->olddelta, sizeof(user_atv)); - } - } - - return (0); -} - /* * Verify the calendar value. If negative, * reset to zero (the epoch). diff --git a/bsd/kern/kern_xxx.c b/bsd/kern/kern_xxx.c index a4e96162e..c4674fc33 100644 --- a/bsd/kern/kern_xxx.c +++ b/bsd/kern/kern_xxx.c @@ -102,27 +102,38 @@ reboot(struct proc *p, struct reboot_args *uap, __unused int32_t *retval) message[0] = '\0'; - if ((error = suser(kauth_cred_get(), &p->p_acflag))) - return(error); - + if ((error = suser(kauth_cred_get(), &p->p_acflag))) { +#if (DEVELOPMENT || DEBUG) + /* allow non-root user to call panic on dev/debug kernels */ + if (!(uap->opt & RB_PANIC)) + return error; +#else + return error; +#endif + } + if (uap->opt & RB_COMMAND) return ENOSYS; if (uap->opt & RB_PANIC) { -#if !(DEVELOPMENT || DEBUG) - if (p != initproc) { - return EPERM; - } -#endif error = copyinstr(uap->command, (void *)message, sizeof(message), (size_t *)&dummy); } #if CONFIG_MACF +#if (DEVELOPMENT || DEBUG) + if (uap->opt & RB_PANIC) { + /* on dev/debug kernels: allow anyone to call panic */ + goto skip_cred_check; + } +#endif if (error) return (error); my_cred = kauth_cred_proc_ref(p); error = mac_system_check_reboot(my_cred, uap->opt); kauth_cred_unref(&my_cred); +#if (DEVELOPMENT || DEBUG) +skip_cred_check: +#endif #endif if (!error) { OSBitOrAtomic(P_REBOOT, &p->p_flag); /* No more signals for this proc */ diff --git a/bsd/kern/kpi_mbuf.c b/bsd/kern/kpi_mbuf.c index 587394e49..6cb79b10e 100644 --- a/bsd/kern/kpi_mbuf.c +++ b/bsd/kern/kpi_mbuf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2015 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1572,15 +1572,16 @@ mbuf_last_pkt(const mbuf_t m, u_int32_t *retval) errno_t mbuf_get_timestamp(mbuf_t m, u_int64_t *ts, boolean_t *valid) { - if (m == NULL || !(m->m_flags & M_PKTHDR) || ts == NULL || - valid == NULL) + if (m == NULL || !(m->m_flags & M_PKTHDR) || ts == NULL) return (EINVAL); - if ((m->m_pkthdr.pkt_flags & PKTF_DRV_TS_VALID) == 0) { - *valid = FALSE; + if ((m->m_pkthdr.pkt_flags & PKTF_TS_VALID) == 0) { + if (valid != NULL) + *valid = FALSE; *ts = 0; } else { - *valid = TRUE; + if (valid != NULL) + *valid = TRUE; *ts = m->m_pkthdr.pkt_timestamp; } return (0); @@ -1593,10 +1594,10 @@ mbuf_set_timestamp(mbuf_t m, u_int64_t ts, boolean_t valid) return (EINVAL); if (valid == FALSE) { - m->m_pkthdr.pkt_flags &= ~PKTF_DRV_TS_VALID; + m->m_pkthdr.pkt_flags &= ~PKTF_TS_VALID; m->m_pkthdr.pkt_timestamp = 0; } else { - m->m_pkthdr.pkt_flags |= PKTF_DRV_TS_VALID; + m->m_pkthdr.pkt_flags |= PKTF_TS_VALID; m->m_pkthdr.pkt_timestamp = ts; } return (0); @@ -1846,7 +1847,7 @@ m_do_tx_compl_callback(struct mbuf *m, struct ifnet *ifp) #if (DEBUG || DEVELOPMENT) if (mbuf_tx_compl_debug != 0 && ifp != NULL && (ifp->if_xflags & IFXF_TIMESTAMP_ENABLED) != 0 && - (m->m_pkthdr.pkt_flags & PKTF_DRV_TS_VALID) == 0) { + (m->m_pkthdr.pkt_flags & PKTF_TS_VALID) == 0) { struct timespec now; nanouptime(&now); @@ -1866,7 +1867,9 @@ m_do_tx_compl_callback(struct mbuf *m, struct ifnet *ifp) if (callback != NULL) { callback(m->m_pkthdr.pkt_compl_context, - ifp, m->m_pkthdr.pkt_timestamp, + ifp, + (m->m_pkthdr.pkt_flags & PKTF_TS_VALID) ? + m->m_pkthdr.pkt_timestamp: 0, m->m_pkthdr.drv_tx_compl_arg, m->m_pkthdr.drv_tx_compl_data, m->m_pkthdr.drv_tx_status); diff --git a/bsd/kern/kpi_socket.c b/bsd/kern/kpi_socket.c index 2f1b1d96a..a7b17264d 100644 --- a/bsd/kern/kpi_socket.c +++ b/bsd/kern/kpi_socket.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -44,17 +44,30 @@ #include #include #include +#include #include #include +#include static errno_t sock_send_internal(socket_t, const struct msghdr *, mbuf_t, int, size_t *); -static void sock_setupcalls_common(socket_t, sock_upcall, void *, - sock_upcall, void *); + +#undef sock_accept +#undef sock_socket +errno_t sock_accept(socket_t so, struct sockaddr *from, int fromlen, + int flags, sock_upcall callback, void *cookie, socket_t *new_so); +errno_t sock_socket(int domain, int type, int protocol, sock_upcall callback, + void *context, socket_t *new_so); + +static errno_t sock_accept_common(socket_t sock, struct sockaddr *from, + int fromlen, int flags, sock_upcall callback, void *cookie, + socket_t *new_sock, bool is_internal); +static errno_t sock_socket_common(int domain, int type, int protocol, + sock_upcall callback, void *context, socket_t *new_so, bool is_internal); errno_t -sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags, - sock_upcall callback, void *cookie, socket_t *new_sock) +sock_accept_common(socket_t sock, struct sockaddr *from, int fromlen, int flags, + sock_upcall callback, void *cookie, socket_t *new_sock, bool is_internal) { struct sockaddr *sa; struct socket *new_so; @@ -82,7 +95,7 @@ check_again: } if (sock->so_proto->pr_getlock != NULL) { - mutex_held = (*sock->so_proto->pr_getlock)(sock, 0); + mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK); dosocklock = 1; } else { mutex_held = sock->so_proto->pr_domain->dom_mtx; @@ -121,6 +134,15 @@ check_again: so_release_accept_list(sock); + /* + * Count the accepted socket as an in-kernel socket + */ + new_so->so_flags1 |= SOF1_IN_KERNEL_SOCKET; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_total); + if (is_internal) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_os_total); + } + /* * Pass the pre-accepted socket to any interested socket filter(s). * Upon failure, the socket would have been closed by the callee. @@ -142,7 +164,7 @@ check_again: } if (dosocklock) { - lck_mtx_assert(new_so->so_proto->pr_getlock(new_so, 0), + LCK_MTX_ASSERT(new_so->so_proto->pr_getlock(new_so, 0), LCK_MTX_ASSERT_NOTOWNED); socket_lock(new_so, 1); } @@ -153,7 +175,11 @@ check_again: /* see comments in sock_setupcall() */ if (callback != NULL) { - sock_setupcalls_common(new_so, callback, cookie, NULL, NULL); +#if CONFIG_EMBEDDED + sock_setupcalls_locked(new_so, callback, cookie, callback, cookie, 0); +#else + sock_setupcalls_locked(new_so, callback, cookie, NULL, NULL, 0); +#endif /* !CONFIG_EMBEDDED */ } if (sa != NULL && from != NULL) { @@ -178,6 +204,22 @@ check_again: return (error); } +errno_t +sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags, + sock_upcall callback, void *cookie, socket_t *new_sock) +{ + return (sock_accept_common(sock, from, fromlen, flags, + callback, cookie, new_sock, false)); +} + +errno_t +sock_accept_internal(socket_t sock, struct sockaddr *from, int fromlen, int flags, + sock_upcall callback, void *cookie, socket_t *new_sock) +{ + return (sock_accept_common(sock, from, fromlen, flags, + callback, cookie, new_sock, true)); +} + errno_t sock_bind(socket_t sock, const struct sockaddr *to) { @@ -247,7 +289,7 @@ sock_connect(socket_t sock, const struct sockaddr *to, int flags) } if (sock->so_proto->pr_getlock != NULL) - mutex_held = (*sock->so_proto->pr_getlock)(sock, 0); + mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK); else mutex_held = sock->so_proto->pr_domain->dom_mtx; @@ -313,7 +355,7 @@ sock_connectwait(socket_t sock, const struct timeval *tv) } if (sock->so_proto->pr_getlock != NULL) - mutex_held = (*sock->so_proto->pr_getlock)(sock, 0); + mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK); else mutex_held = sock->so_proto->pr_domain->dom_mtx; @@ -892,10 +934,9 @@ sock_shutdown(socket_t sock, int how) return (soshutdown(sock, how)); } - errno_t -sock_socket(int domain, int type, int protocol, sock_upcall callback, - void *context, socket_t *new_so) +sock_socket_common(int domain, int type, int protocol, sock_upcall callback, + void *context, socket_t *new_so, bool is_internal) { int error = 0; @@ -905,10 +946,18 @@ sock_socket(int domain, int type, int protocol, sock_upcall callback, /* socreate will create an initial so_count */ error = socreate(domain, new_so, type, protocol); if (error == 0) { + /* + * This is an in-kernel socket + */ + (*new_so)->so_flags1 |= SOF1_IN_KERNEL_SOCKET; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_total); + if (is_internal) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_os_total); + } + /* see comments in sock_setupcall() */ if (callback != NULL) { - sock_setupcalls_common(*new_so, callback, context, - NULL, NULL); + sock_setupcall(*new_so, callback, context); } /* * last_pid and last_upid should be zero for sockets @@ -920,6 +969,22 @@ sock_socket(int domain, int type, int protocol, sock_upcall callback, return (error); } +errno_t +sock_socket_internal(int domain, int type, int protocol, sock_upcall callback, + void *context, socket_t *new_so) +{ + return (sock_socket_common(domain, type, protocol, callback, + context, new_so, true)); +} + +errno_t +sock_socket(int domain, int type, int protocol, sock_upcall callback, + void *context, socket_t *new_so) +{ + return (sock_socket_common(domain, type, protocol, callback, + context, new_so, false)); +} + void sock_close(socket_t sock) { @@ -1135,26 +1200,30 @@ socket_defunct(struct proc *p, socket_t so, int level) return (retval); } -static void -sock_setupcalls_common(socket_t sock, sock_upcall rcallback, void *rcontext, - sock_upcall wcallback, void *wcontext) +void +sock_setupcalls_locked(socket_t sock, sock_upcall rcallback, void *rcontext, + sock_upcall wcallback, void *wcontext, int locked) { if (rcallback != NULL) { sock->so_rcv.sb_flags |= SB_UPCALL; + if (locked) + sock->so_rcv.sb_flags |= SB_UPCALL_LOCK; sock->so_rcv.sb_upcall = rcallback; sock->so_rcv.sb_upcallarg = rcontext; } else { - sock->so_rcv.sb_flags &= ~SB_UPCALL; + sock->so_rcv.sb_flags &= ~(SB_UPCALL | SB_UPCALL_LOCK); sock->so_rcv.sb_upcall = NULL; sock->so_rcv.sb_upcallarg = NULL; } if (wcallback != NULL) { sock->so_snd.sb_flags |= SB_UPCALL; + if (locked) + sock->so_snd.sb_flags |= SB_UPCALL_LOCK; sock->so_snd.sb_upcall = wcallback; sock->so_snd.sb_upcallarg = wcontext; } else { - sock->so_snd.sb_flags &= ~SB_UPCALL; + sock->so_snd.sb_flags &= ~(SB_UPCALL | SB_UPCALL_LOCK); sock->so_snd.sb_upcall = NULL; sock->so_snd.sb_upcallarg = NULL; } @@ -1176,7 +1245,11 @@ sock_setupcall(socket_t sock, sock_upcall callback, void *context) * the read and write callbacks and their respective parameters. */ socket_lock(sock, 1); - sock_setupcalls_common(sock, callback, context, NULL, NULL); +#if CONFIG_EMBEDDED + sock_setupcalls_locked(sock, callback, context, callback, context, 0); +#else + sock_setupcalls_locked(sock, callback, context, NULL, NULL, 0); +#endif /* !CONFIG_EMBEDDED */ socket_unlock(sock, 1); return (0); @@ -1193,23 +1266,21 @@ sock_setupcalls(socket_t sock, sock_upcall rcallback, void *rcontext, * Note that we don't wait for any in progress upcall to complete. */ socket_lock(sock, 1); - sock_setupcalls_common(sock, rcallback, rcontext, wcallback, wcontext); + sock_setupcalls_locked(sock, rcallback, rcontext, wcallback, wcontext, 0); socket_unlock(sock, 1); return (0); } -errno_t -sock_catchevents(socket_t sock, sock_evupcall ecallback, void *econtext, +void +sock_catchevents_locked(socket_t sock, sock_evupcall ecallback, void *econtext, u_int32_t emask) { - if (sock == NULL) - return (EINVAL); + socket_lock_assert_owned(sock); /* * Note that we don't wait for any in progress upcall to complete. */ - socket_lock(sock, 1); if (ecallback != NULL) { sock->so_event = ecallback; sock->so_eventarg = econtext; @@ -1219,6 +1290,17 @@ sock_catchevents(socket_t sock, sock_evupcall ecallback, void *econtext, sock->so_eventarg = NULL; sock->so_eventmask = 0; } +} + +errno_t +sock_catchevents(socket_t sock, sock_evupcall ecallback, void *econtext, + u_int32_t emask) +{ + if (sock == NULL) + return (EINVAL); + + socket_lock(sock, 1); + sock_catchevents_locked(sock, ecallback, econtext, emask); socket_unlock(sock, 1); return (0); diff --git a/bsd/kern/kpi_socketfilter.c b/bsd/kern/kpi_socketfilter.c index e8e4115f0..138c6b299 100644 --- a/bsd/kern/kpi_socketfilter.c +++ b/bsd/kern/kpi_socketfilter.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,7 @@ #include #include +#include #include #define SFEF_ATTACHED 0x1 /* SFE is on socket list */ @@ -91,6 +93,13 @@ static thread_t sock_filter_cleanup_thread = NULL; static void sflt_cleanup_thread(void *, wait_result_t); static void sflt_detach_locked(struct socket_filter_entry *entry); +#undef sflt_register +static errno_t sflt_register_common(const struct sflt_filter *filter, int domain, + int type, int protocol, bool is_internal); +errno_t sflt_register(const struct sflt_filter *filter, int domain, + int type, int protocol); + + #pragma mark -- Internal State Management -- __private_extern__ int @@ -1249,9 +1258,9 @@ struct solist { struct socket *so; }; -errno_t -sflt_register(const struct sflt_filter *filter, int domain, int type, - int protocol) +static errno_t +sflt_register_common(const struct sflt_filter *filter, int domain, int type, + int protocol, bool is_internal) { struct socket_filter *sock_filt = NULL; struct socket_filter *match = NULL; @@ -1317,6 +1326,12 @@ sflt_register(const struct sflt_filter *filter, int domain, int type, sock_filt->sf_proto = pr; } sflt_retain_locked(sock_filt); + + OSIncrementAtomic64(&net_api_stats.nas_sfltr_register_count); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sfltr_register_total); + if (is_internal) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sfltr_register_os_total); + } } lck_rw_unlock_exclusive(sock_filter_lock); @@ -1414,6 +1429,20 @@ sflt_register(const struct sflt_filter *filter, int domain, int type, return (error); } +errno_t +sflt_register_internal(const struct sflt_filter *filter, int domain, int type, + int protocol) +{ + return (sflt_register_common(filter, domain, type, protocol, true)); +} + +errno_t +sflt_register(const struct sflt_filter *filter, int domain, int type, + int protocol) +{ + return (sflt_register_common(filter, domain, type, protocol, false)); +} + errno_t sflt_unregister(sflt_handle handle) { @@ -1427,6 +1456,8 @@ sflt_unregister(sflt_handle handle) } if (filter) { + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_sfltr_register_count) > 0); + /* Remove it from the global list */ TAILQ_REMOVE(&sock_filter_head, filter, sf_global_next); diff --git a/bsd/kern/mach_loader.c b/bsd/kern/mach_loader.c index ca2fb4338..43f1a78b0 100644 --- a/bsd/kern/mach_loader.c +++ b/bsd/kern/mach_loader.c @@ -248,7 +248,7 @@ struct macho_data; static load_return_t get_macho_vnode( - char *path, + const char *path, integer_t archbits, struct mach_header *mach_header, off_t *file_offset, @@ -285,6 +285,7 @@ note_all_image_info_section(const struct segment_command_64 *scp, } *sectionp; unsigned int i; + if (strncmp(scp->segname, "__DATA", sizeof(scp->segname)) != 0) return; for (i = 0; i < scp->nsects; ++i) { @@ -302,6 +303,15 @@ note_all_image_info_section(const struct segment_command_64 *scp, } } +#if __arm64__ +/* + * Allow bypassing some security rules (hard pagezero, no write+execute) + * in exchange for better binary compatibility for legacy apps built + * before 16KB-alignment was enforced. + */ +int fourk_binary_compatibility_unsafe = TRUE; +int fourk_binary_compatibility_allow_wx = FALSE; +#endif /* __arm64__ */ load_return_t load_machfile( @@ -324,8 +334,10 @@ load_machfile( int in_exec = (imgp->ip_flags & IMGPF_EXEC); task_t task = current_task(); proc_t p = current_proc(); - mach_vm_offset_t aslr_offset = 0; - mach_vm_offset_t dyld_aslr_offset = 0; + int64_t aslr_page_offset = 0; + int64_t dyld_aslr_page_offset = 0; + int64_t aslr_section_size = 0; + int64_t aslr_section_offset = 0; kern_return_t kret; if (macho_size > file_size) { @@ -348,7 +360,14 @@ load_machfile( vm_compute_max_offset(result->is64bit), TRUE); -#if (__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS) +#if defined(__arm64__) + if (result->is64bit) { + /* enforce 16KB alignment of VM map entries */ + vm_map_set_page_shift(map, SIXTEENK_PAGE_SHIFT); + } else { + vm_map_set_page_shift(map, page_shift_user32); + } +#elif (__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS) /* enforce 16KB alignment for watch targets with new ABI */ vm_map_set_page_shift(map, SIXTEENK_PAGE_SHIFT); #endif /* __arm64__ */ @@ -367,24 +386,25 @@ load_machfile( * normally permits it. */ if ((header->flags & MH_NO_HEAP_EXECUTION) && !(imgp->ip_flags & IMGPF_ALLOW_DATA_EXEC)) vm_map_disallow_data_exec(map); - + /* * Compute a random offset for ASLR, and an independent random offset for dyld. */ if (!(imgp->ip_flags & IMGPF_DISABLE_ASLR)) { - uint64_t max_slide_pages; + vm_map_get_max_aslr_slide_section(map, &aslr_section_offset, &aslr_section_size); + aslr_section_offset = (random() % aslr_section_offset) * aslr_section_size; - max_slide_pages = vm_map_get_max_aslr_slide_pages(map); + aslr_page_offset = random(); + aslr_page_offset %= vm_map_get_max_aslr_slide_pages(map); + aslr_page_offset <<= vm_map_page_shift(map); - aslr_offset = random(); - aslr_offset %= max_slide_pages; - aslr_offset <<= vm_map_page_shift(map); + dyld_aslr_page_offset = random(); + dyld_aslr_page_offset %= vm_map_get_max_loader_aslr_slide_pages(map); + dyld_aslr_page_offset <<= vm_map_page_shift(map); - dyld_aslr_offset = random(); - dyld_aslr_offset %= max_slide_pages; - dyld_aslr_offset <<= vm_map_page_shift(map); + aslr_page_offset += aslr_section_offset; } - + if (!result) result = &myresult; @@ -396,7 +416,7 @@ load_machfile( result->is64bit = ((imgp->ip_flags & IMGPF_IS_64BIT) == IMGPF_IS_64BIT); lret = parse_machfile(vp, map, thread, header, file_offset, macho_size, - 0, (int64_t)aslr_offset, (int64_t)dyld_aslr_offset, result, + 0, aslr_page_offset, dyld_aslr_page_offset, result, NULL, imgp); if (lret != LOAD_SUCCESS) { @@ -411,12 +431,44 @@ load_machfile( if (!result->is64bit) { enforce_hard_pagezero = FALSE; } -#endif + + /* + * For processes with IMGPF_HIGH_BITS_ASLR, add a few random high bits + * to the start address for "anywhere" memory allocations. + */ +#define VM_MAP_HIGH_START_BITS_COUNT 8 +#define VM_MAP_HIGH_START_BITS_SHIFT 27 + if (result->is64bit && + (imgp->ip_flags & IMGPF_HIGH_BITS_ASLR)) { + int random_bits; + vm_map_offset_t high_start; + + random_bits = random(); + random_bits &= (1 << VM_MAP_HIGH_START_BITS_COUNT)-1; + high_start = (((vm_map_offset_t)random_bits) + << VM_MAP_HIGH_START_BITS_SHIFT); + vm_map_set_high_start(map, high_start); + } +#endif /* __x86_64__ */ + /* * Check to see if the page zero is enforced by the map->min_offset. */ if (enforce_hard_pagezero && (vm_map_has_hard_pagezero(map, 0x1000) == FALSE)) { +#if __arm64__ + if (!result->is64bit && /* not 64-bit */ + !(header->flags & MH_PIE) && /* not PIE */ + (vm_map_page_shift(map) != FOURK_PAGE_SHIFT || + PAGE_SHIFT != FOURK_PAGE_SHIFT) && /* page size != 4KB */ + result->has_pagezero && /* has a "soft" page zero */ + fourk_binary_compatibility_unsafe) { + /* + * For backwards compatibility of "4K" apps on + * a 16K system, do not enforce a hard page zero... + */ + } else +#endif /* __arm64__ */ { vm_map_deallocate(map); /* will lose pmap reference too */ return (LOAD_BADMACHO); @@ -459,8 +511,7 @@ load_machfile( workqueue_mark_exiting(p); task_complete_halt(task); workqueue_exit(p); - kqueue_dealloc(p->p_wqkqueue); - p->p_wqkqueue = NULL; + /* * Roll up accounting info to new task. The roll up is done after * task_complete_halt to make sure the thread accounting info is @@ -514,9 +565,8 @@ parse_machfile( integer_t dlarchbits = 0; void * control; load_return_t ret = LOAD_SUCCESS; - caddr_t addr; - void * kl_addr; - vm_size_t size,kl_size; + void * addr; + vm_size_t alloc_size, cmds_size; size_t offset; size_t oldoffset; /* for overflow check */ int pass; @@ -532,6 +582,14 @@ parse_machfile( boolean_t dyld_no_load_addr = FALSE; boolean_t is_dyld = FALSE; vm_map_offset_t effective_page_mask = MAX(PAGE_MASK, vm_map_page_mask(map)); +#if __arm64__ + uint32_t pagezero_end = 0; + uint32_t executable_end = 0; + uint32_t writable_start = 0; + vm_map_size_t effective_page_size; + + effective_page_size = MAX(PAGE_SIZE, vm_map_page_size(map)); +#endif /* __arm64__ */ if (header->magic == MH_MAGIC_64 || header->magic == MH_CIGAM_64) { @@ -563,6 +621,20 @@ parse_machfile( if (depth != 1) { return (LOAD_FAILURE); } +#if CONFIG_EMBEDDED + if (header->flags & MH_DYLDLINK) { + /* Check properties of dynamic executables */ + if (!(header->flags & MH_PIE) && pie_required(header->cputype, header->cpusubtype & ~CPU_SUBTYPE_MASK)) { + return (LOAD_FAILURE); + } + result->needs_dynlinker = TRUE; + } else { + /* Check properties of static executables (disallowed except for development) */ +#if !(DEVELOPMENT || DEBUG) + return (LOAD_FAILURE); +#endif + } +#endif /* CONFIG_EMBEDDED */ break; case MH_DYLINKER: @@ -581,43 +653,32 @@ parse_machfile( */ control = ubc_getobject(vp, UBC_FLAGS_NONE); - /* - * Map portion that must be accessible directly into - * kernel's map. - */ - if ((off_t)(mach_header_sz + header->sizeofcmds) > macho_size) - return(LOAD_BADMACHO); - - /* - * Round size of Mach-O commands up to page boundry. - */ - size = round_page(mach_header_sz + header->sizeofcmds); - if (size <= 0) - return(LOAD_BADMACHO); + /* ensure header + sizeofcmds falls within the file */ + if (os_add_overflow(mach_header_sz, header->sizeofcmds, &cmds_size) || + (off_t)cmds_size > macho_size || + round_page_overflow(cmds_size, &alloc_size)) { + return LOAD_BADMACHO; + } /* * Map the load commands into kernel memory. */ - addr = 0; - kl_size = size; - kl_addr = kalloc(size); - addr = (caddr_t)kl_addr; - if (addr == NULL) - return(LOAD_NOSPACE); + addr = kalloc(alloc_size); + if (addr == NULL) { + return LOAD_NOSPACE; + } - error = vn_rdwr(UIO_READ, vp, addr, size, file_offset, + error = vn_rdwr(UIO_READ, vp, addr, alloc_size, file_offset, UIO_SYSSPACE, 0, kauth_cred_get(), &resid, p); if (error) { - if (kl_addr) - kfree(kl_addr, kl_size); - return(LOAD_IOERROR); + kfree(addr, alloc_size); + return LOAD_IOERROR; } if (resid) { /* We must be able to read in as much as the mach_header indicated */ - if (kl_addr) - kfree(kl_addr, kl_size); - return(LOAD_BADMACHO); + kfree(addr, alloc_size); + return LOAD_BADMACHO; } /* @@ -637,6 +698,11 @@ parse_machfile( */ boolean_t slide_realign = FALSE; +#if __arm64__ + if (!abi64) { + slide_realign = TRUE; + } +#endif for (pass = 0; pass <= 3; pass++) { @@ -645,6 +711,60 @@ parse_machfile( * address, pass 0 can be skipped */ continue; } else if (pass == 1) { +#if __arm64__ + boolean_t is_pie; + int64_t adjust; + + is_pie = ((header->flags & MH_PIE) != 0); + if (pagezero_end != 0 && + pagezero_end < effective_page_size) { + /* need at least 1 page for PAGEZERO */ + adjust = effective_page_size; + MACHO_PRINTF(("pagezero boundary at " + "0x%llx; adjust slide from " + "0x%llx to 0x%llx%s\n", + (uint64_t) pagezero_end, + slide, + slide + adjust, + (is_pie + ? "" + : " BUT NO PIE ****** :-("))); + if (is_pie) { + slide += adjust; + pagezero_end += adjust; + executable_end += adjust; + writable_start += adjust; + } + } + if (pagezero_end != 0) { + result->has_pagezero = TRUE; + } + if (executable_end == writable_start && + (executable_end & effective_page_mask) != 0 && + (executable_end & FOURK_PAGE_MASK) == 0) { + + /* + * The TEXT/DATA boundary is 4K-aligned but + * not page-aligned. Adjust the slide to make + * it page-aligned and avoid having a page + * with both write and execute permissions. + */ + adjust = + (effective_page_size - + (executable_end & effective_page_mask)); + MACHO_PRINTF(("page-unaligned X-W boundary at " + "0x%llx; adjust slide from " + "0x%llx to 0x%llx%s\n", + (uint64_t) executable_end, + slide, + slide + adjust, + (is_pie + ? "" + : " BUT NO PIE ****** :-("))); + if (is_pie) + slide += adjust; + } +#endif /* __arm64__ */ if (dyld_no_load_addr && binresult) { /* @@ -684,12 +804,18 @@ parse_machfile( ncmds = header->ncmds; while (ncmds--) { + + /* ensure enough space for a minimal load command */ + if (offset + sizeof(struct load_command) > cmds_size) { + ret = LOAD_BADMACHO; + break; + } + /* * Get a pointer to the command. */ lcp = (struct load_command *)(addr + offset); oldoffset = offset; - offset += lcp->cmdsize; /* * Perform prevalidation of the struct load_command @@ -699,9 +825,9 @@ parse_machfile( * straddle or exist past the reserved section at the * start of the image. */ - if (oldoffset > offset || - lcp->cmdsize < sizeof(struct load_command) || - offset > header->sizeofcmds + mach_header_sz) { + if (os_add_overflow(offset, lcp->cmdsize, &offset) || + lcp->cmdsize < sizeof(struct load_command) || + offset > cmds_size) { ret = LOAD_BADMACHO; break; } @@ -723,6 +849,31 @@ parse_machfile( } } +#if __arm64__ + assert(!abi64); + + if (scp->initprot == 0 && scp->maxprot == 0 && scp->vmaddr == 0) { + /* PAGEZERO */ + if (os_add3_overflow(scp->vmaddr, scp->vmsize, slide, &pagezero_end)) { + ret = LOAD_BADMACHO; + break; + } + } + if (scp->initprot & VM_PROT_EXECUTE) { + /* TEXT */ + if (os_add3_overflow(scp->vmaddr, scp->vmsize, slide, &executable_end)) { + ret = LOAD_BADMACHO; + break; + } + } + if (scp->initprot & VM_PROT_WRITE) { + /* DATA */ + if (os_add_overflow(scp->vmaddr, slide, &writable_start)) { + ret = LOAD_BADMACHO; + break; + } + } +#endif /* __arm64__ */ break; } @@ -850,7 +1001,7 @@ parse_machfile( case LC_UUID: if (pass == 1 && depth == 1) { ret = load_uuid((struct uuid_command *) lcp, - (char *)addr + mach_header_sz + header->sizeofcmds, + (char *)addr + cmds_size, result); } break; @@ -893,7 +1044,7 @@ parse_machfile( if (cs_debug > 10) printf("validating initial pages of %s\n", vp->v_name); - while (off < size && ret == LOAD_SUCCESS) { + while (off < alloc_size && ret == LOAD_SUCCESS) { tainted = CS_VALIDATE_TAINTED; valid = cs_validate_range(vp, @@ -975,6 +1126,7 @@ parse_machfile( if (cs_enforcement(NULL)) { ret = LOAD_FAILURE; } else { +#if !CONFIG_EMBEDDED /* * No embedded signatures: look for detached by taskgated, * this is only done on OSX, on embedded platforms we expect everything @@ -995,6 +1147,7 @@ parse_machfile( /* get flags to be applied to the process */ result->csflags |= cs_flag_data; } +#endif } } @@ -1016,6 +1169,11 @@ parse_machfile( if (result->thread_count == 0) { ret = LOAD_FAILURE; } +#if CONFIG_EMBEDDED + if (result->needs_dynlinker && !(result->csflags & CS_DYLD_PLATFORM)) { + ret = LOAD_FAILURE; + } +#endif } } @@ -1023,11 +1181,9 @@ parse_machfile( ret = LOAD_BADMACHO_UPX; } - if (kl_addr) { - kfree(kl_addr, kl_size); - } + kfree(addr, alloc_size); - return(ret); + return ret; } #if CONFIG_CODE_DECRYPTION @@ -1155,10 +1311,10 @@ map_segment( vm_prot_t initprot, vm_prot_t maxprot) { - int extra_vm_flags, cur_extra_vm_flags; vm_map_offset_t cur_offset, cur_start, cur_end; kern_return_t ret; vm_map_offset_t effective_page_mask; + vm_map_kernel_flags_t vmk_flags, cur_vmk_flags; if (vm_end < vm_start || file_end < file_start) { @@ -1172,24 +1328,71 @@ map_segment( effective_page_mask = MAX(PAGE_MASK, vm_map_page_mask(map)); - extra_vm_flags = 0; + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; if (vm_map_page_aligned(vm_start, effective_page_mask) && vm_map_page_aligned(vm_end, effective_page_mask) && vm_map_page_aligned(file_start, effective_page_mask) && vm_map_page_aligned(file_end, effective_page_mask)) { /* all page-aligned and map-aligned: proceed */ } else { +#if __arm64__ + /* use an intermediate "4K" pager */ + vmk_flags.vmkf_fourk = TRUE; +#else /* __arm64__ */ panic("map_segment: unexpected mis-alignment " "vm[0x%llx:0x%llx] file[0x%llx:0x%llx]\n", (uint64_t) vm_start, (uint64_t) vm_end, (uint64_t) file_start, (uint64_t) file_end); +#endif /* __arm64__ */ } cur_offset = 0; cur_start = vm_start; cur_end = vm_start; +#if __arm64__ + if (!vm_map_page_aligned(vm_start, effective_page_mask)) { + /* one 4K pager for the 1st page */ + cur_end = vm_map_round_page(cur_start, effective_page_mask); + if (cur_end > vm_end) { + cur_end = vm_start + (file_end - file_start); + } + if (control != MEMORY_OBJECT_CONTROL_NULL) { + ret = vm_map_enter_mem_object_control( + map, + &cur_start, + cur_end - cur_start, + (mach_vm_offset_t)0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + control, + file_start + cur_offset, + TRUE, /* copy */ + initprot, maxprot, + VM_INHERIT_DEFAULT); + } else { + ret = vm_map_enter_mem_object( + map, + &cur_start, + cur_end - cur_start, + (mach_vm_offset_t)0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + IPC_PORT_NULL, + 0, /* offset */ + TRUE, /* copy */ + initprot, maxprot, + VM_INHERIT_DEFAULT); + } + if (ret != KERN_SUCCESS) { + return (LOAD_NOSPACE); + } + cur_offset += cur_end - cur_start; + } +#endif /* __arm64__ */ if (cur_end >= vm_start + (file_end - file_start)) { /* all mapped: done */ goto done; @@ -1203,10 +1406,10 @@ map_segment( if ((vm_start & effective_page_mask) != (file_start & effective_page_mask)) { /* one 4K pager for the middle */ - cur_extra_vm_flags = extra_vm_flags; + cur_vmk_flags = vmk_flags; } else { /* regular mapping for the middle */ - cur_extra_vm_flags = 0; + cur_vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; } cur_end = vm_map_trunc_page(vm_start + (file_end - file_start), @@ -1217,7 +1420,9 @@ map_segment( &cur_start, cur_end - cur_start, (mach_vm_offset_t)0, - VM_FLAGS_FIXED | cur_extra_vm_flags, + VM_FLAGS_FIXED, + cur_vmk_flags, + VM_KERN_MEMORY_NONE, control, file_start + cur_offset, TRUE, /* copy */ @@ -1229,7 +1434,9 @@ map_segment( &cur_start, cur_end - cur_start, (mach_vm_offset_t)0, - VM_FLAGS_FIXED | cur_extra_vm_flags, + VM_FLAGS_FIXED, + cur_vmk_flags, + VM_KERN_MEMORY_NONE, IPC_PORT_NULL, 0, /* offset */ TRUE, /* copy */ @@ -1246,6 +1453,46 @@ map_segment( goto done; } cur_start = cur_end; +#if __arm64__ + if (!vm_map_page_aligned(vm_start + (file_end - file_start), + effective_page_mask)) { + /* one 4K pager for the last page */ + cur_end = vm_start + (file_end - file_start); + if (control != MEMORY_OBJECT_CONTROL_NULL) { + ret = vm_map_enter_mem_object_control( + map, + &cur_start, + cur_end - cur_start, + (mach_vm_offset_t)0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + control, + file_start + cur_offset, + TRUE, /* copy */ + initprot, maxprot, + VM_INHERIT_DEFAULT); + } else { + ret = vm_map_enter_mem_object( + map, + &cur_start, + cur_end - cur_start, + (mach_vm_offset_t)0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + IPC_PORT_NULL, + 0, /* offset */ + TRUE, /* copy */ + initprot, maxprot, + VM_INHERIT_DEFAULT); + } + if (ret != KERN_SUCCESS) { + return (LOAD_NOSPACE); + } + cur_offset += cur_end - cur_start; + } +#endif /* __arm64__ */ done: assert(cur_end >= vm_start + (file_end - file_start)); return LOAD_SUCCESS; @@ -1279,6 +1526,10 @@ load_segment( boolean_t verbose; vm_map_size_t effective_page_size; vm_map_offset_t effective_page_mask; +#if __arm64__ + vm_map_kernel_flags_t vmk_flags; + boolean_t fourk_align; +#endif /* __arm64__ */ effective_page_size = MAX(PAGE_SIZE, vm_map_page_size(map)); effective_page_mask = MAX(PAGE_MASK, vm_map_page_mask(map)); @@ -1287,9 +1538,24 @@ load_segment( if (LC_SEGMENT_64 == lcp->cmd) { segment_command_size = sizeof(struct segment_command_64); single_section_size = sizeof(struct section_64); +#if __arm64__ + /* 64-bit binary: should already be 16K-aligned */ + fourk_align = FALSE; +#endif /* __arm64__ */ } else { segment_command_size = sizeof(struct segment_command); single_section_size = sizeof(struct section); +#if __arm64__ + /* 32-bit binary: might need 4K-alignment */ + if (effective_page_size != FOURK_PAGE_SIZE) { + /* not using 4K page size: need fourk_pager */ + fourk_align = TRUE; + verbose = TRUE; + } else { + /* using 4K page size: no need for re-alignment */ + fourk_align = FALSE; + } +#endif /* __arm64__ */ } if (lcp->cmdsize < segment_command_size) return (LOAD_BADMACHO); @@ -1336,6 +1602,17 @@ load_segment( */ file_offset = pager_offset + scp->fileoff; /* limited to 32 bits */ file_size = scp->filesize; +#if __arm64__ + if (fourk_align) { + if ((file_offset & FOURK_PAGE_MASK) != 0) { + /* + * we can't mmap() it if it's not at least 4KB-aligned + * in the file + */ + return LOAD_BADMACHO; + } + } else +#endif /* __arm64__ */ if ((file_offset & PAGE_MASK_64) != 0 || /* we can't mmap() it if it's not page-aligned in the file */ (file_offset & vm_map_page_mask(map)) != 0) { @@ -1393,6 +1670,13 @@ load_segment( "page_zero up to 0x%llx\n", (uint64_t) vm_end)); } +#if __arm64__ + if (fourk_align) { + /* raise min_offset as much as page-alignment allows */ + vm_end_aligned = vm_map_trunc_page(vm_end, + effective_page_mask); + } else +#endif /* __arm64__ */ { vm_end = vm_map_round_page(vm_end, PAGE_MASK_64); @@ -1400,14 +1684,67 @@ load_segment( } ret = vm_map_raise_min_offset(map, vm_end_aligned); +#if __arm64__ + if (ret == 0 && + vm_end > vm_end_aligned) { + /* use fourk_pager to map the rest of pagezero */ + assert(fourk_align); + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_fourk = TRUE; + ret = vm_map_enter_mem_object( + map, + &vm_end_aligned, + vm_end - vm_end_aligned, + (mach_vm_offset_t) 0, /* mask */ + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + IPC_PORT_NULL, + 0, + FALSE, /* copy */ + (scp->initprot & VM_PROT_ALL), + (scp->maxprot & VM_PROT_ALL), + VM_INHERIT_DEFAULT); + } +#endif /* __arm64__ */ if (ret != KERN_SUCCESS) { return (LOAD_FAILURE); } return (LOAD_SUCCESS); } else { +#if CONFIG_EMBEDDED + /* not PAGEZERO: should not be mapped at address 0 */ + if (filetype != MH_DYLINKER && scp->vmaddr == 0) { + return LOAD_BADMACHO; + } +#endif /* CONFIG_EMBEDDED */ } +#if __arm64__ + if (fourk_align) { + /* 4K-align */ + file_start = vm_map_trunc_page(file_offset, + FOURK_PAGE_MASK); + file_end = vm_map_round_page(file_offset + file_size, + FOURK_PAGE_MASK); + vm_start = vm_map_trunc_page(vm_offset, + FOURK_PAGE_MASK); + vm_end = vm_map_round_page(vm_offset + vm_size, + FOURK_PAGE_MASK); + if (!strncmp(scp->segname, "__LINKEDIT", 11) && + page_aligned(file_start) && + vm_map_page_aligned(file_start, vm_map_page_mask(map)) && + page_aligned(vm_start) && + vm_map_page_aligned(vm_start, vm_map_page_mask(map))) { + /* XXX last segment: ignore mis-aligned tail */ + file_end = vm_map_round_page(file_end, + effective_page_mask); + vm_end = vm_map_round_page(vm_end, + effective_page_mask); + } + } else +#endif /* __arm64__ */ { file_start = vm_map_trunc_page(file_offset, effective_page_mask); @@ -1463,7 +1800,7 @@ load_segment( if (delta_size > 0) { mach_vm_offset_t tmp; - ret = mach_vm_allocate(kernel_map, &tmp, delta_size, VM_FLAGS_ANYWHERE| VM_MAKE_TAG(VM_KERN_MEMORY_BSD)); + ret = mach_vm_allocate_kernel(kernel_map, &tmp, delta_size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_BSD); if (ret != KERN_SUCCESS) { return(LOAD_RESOURCE); } @@ -1879,6 +2216,15 @@ extern char dyld_alt_path[]; extern int use_alt_dyld; #endif +static uint64_t get_va_fsid(struct vnode_attr *vap) +{ + if (VATTR_IS_SUPPORTED(vap, va_fsid64)) { + return *(uint64_t *)&vap->va_fsid64; + } else { + return vap->va_fsid; + } +} + static load_return_t load_dylinker( struct dylinker_command *lcp, @@ -1891,8 +2237,7 @@ load_dylinker( struct image_params *imgp ) { - char *name; - char *p; + const char *name; struct vnode *vp = NULLVP; /* set by get_macho_vnode() */ struct mach_header *header; off_t file_offset = 0; /* set by get_macho_vnode() */ @@ -1906,19 +2251,17 @@ load_dylinker( struct macho_data __macho_data; } *dyld_data; - if (lcp->cmdsize < sizeof(*lcp)) - return (LOAD_BADMACHO); + if (lcp->cmdsize < sizeof(*lcp) || lcp->name.offset >= lcp->cmdsize) + return LOAD_BADMACHO; - name = (char *)lcp + lcp->name.offset; + name = (const char *)lcp + lcp->name.offset; - /* - * Check for a proper null terminated string. - */ - p = name; - do { - if (p >= (char *)lcp + lcp->cmdsize) - return(LOAD_BADMACHO); - } while (*p++); + /* Check for a proper null terminated string. */ + size_t maxsz = lcp->cmdsize - lcp->name.offset; + size_t namelen = strnlen(name, maxsz); + if (namelen >= maxsz) { + return LOAD_BADMACHO; + } #if (DEVELOPMENT || DEBUG) @@ -1985,6 +2328,17 @@ load_dylinker( } } + struct vnode_attr va; + VATTR_INIT(&va); + VATTR_WANTED(&va, va_fsid64); + VATTR_WANTED(&va, va_fsid); + VATTR_WANTED(&va, va_fileid); + int error = vnode_getattr(vp, &va, imgp->ip_vfs_context); + if (error == 0) { + imgp->ip_dyld_fsid = get_va_fsid(&va); + imgp->ip_dyld_fsobjid = va.va_fileid; + } + vnode_put(vp); novp_out: FREE(dyld_data, M_TEMP); @@ -2088,7 +2442,7 @@ load_code_signature( out: if (ret == LOAD_SUCCESS) { if (blob == NULL) - panic("sucess, but no blob!"); + panic("success, but no blob!"); result->csflags |= blob->csb_flags; result->platform_binary = blob->csb_platform_binary; @@ -2259,7 +2613,7 @@ remap_now: static load_return_t get_macho_vnode( - char *path, + const char *path, integer_t archbits, struct mach_header *mach_header, off_t *file_offset, diff --git a/bsd/kern/mach_loader.h b/bsd/kern/mach_loader.h index 760ea45d9..d58ae5a4a 100644 --- a/bsd/kern/mach_loader.h +++ b/bsd/kern/mach_loader.h @@ -63,6 +63,7 @@ typedef struct _load_result { mach_vm_address_t all_image_info_addr; mach_vm_size_t all_image_info_size; + int thread_count; unsigned int /* boolean_t */ unixproc :1, diff --git a/bsd/kern/mach_process.c b/bsd/kern/mach_process.c index 184f568a4..72d262def 100644 --- a/bsd/kern/mach_process.c +++ b/bsd/kern/mach_process.c @@ -95,6 +95,10 @@ #include +#if CONFIG_MACF +#include +#endif + /* XXX ken/bsd_kern.c - prototype should be in common header */ int get_task_userstop(task_t); @@ -128,6 +132,10 @@ ptrace(struct proc *p, struct ptrace_args *uap, int32_t *retval) AUDIT_ARG(value32, uap->data); if (uap->req == PT_DENY_ATTACH) { +#if (DEVELOPMENT || DEBUG) && CONFIG_EMBEDDED + if (PE_i_can_has_debugger(NULL)) + return(0); +#endif proc_lock(p); if (ISSET(p->p_lflag, P_LTRACED)) { proc_unlock(p); @@ -227,6 +235,12 @@ retry_trace_me:; #pragma clang diagnostic pop int err; +#if CONFIG_EMBEDDED + if (tr_sigexc == 0) { + error = ENOTSUP; + goto out; + } +#endif if ( kauth_authorize_process(proc_ucred(p), KAUTH_PROCESS_CANTRACE, t, (uintptr_t)&err, 0, 0) == 0 ) { diff --git a/bsd/kern/makesyscalls.sh b/bsd/kern/makesyscalls.sh index 29dd74b74..411d5ce61 100755 --- a/bsd/kern/makesyscalls.sh +++ b/bsd/kern/makesyscalls.sh @@ -35,6 +35,7 @@ output_syshdrfile=0 output_syscalltablefile=0 output_auditevfile=0 output_tracecodes=0 +output_systrace=0 use_stdout=0 @@ -50,7 +51,7 @@ syscallprefix="SYS_" switchname="sysent" namesname="syscallnames" tracecodename="syscall.codes" - +systraceargsfile="systrace_args.c" # tmp files: syslegal="sysent.syslegal.$$" sysent="sysent.switch.$$" @@ -61,10 +62,13 @@ syscallnamestempfile="syscallnamesfile.$$" syshdrtempfile="syshdrtempfile.$$" audittempfile="audittempfile.$$" tracecodetempfile="tracecodetempfile.$$" +systraceargstempfile="systraceargstempfile.$$" +systraceargdesctempfile="systraceargdesctempfile.$$" +systracerettempfile="systracerettempfile.$$" -trap "rm $syslegal $sysent $sysinc $sysarg $sysprotoend $syscallnamestempfile $syshdrtempfile $audittempfile $tracecodetempfile" 0 +trap "rm $syslegal $sysent $sysinc $sysarg $sysprotoend $syscallnamestempfile $syshdrtempfile $audittempfile $tracecodetempfile $systraceargstempfile $systraceargdesctempfile $systracerettempfile" 0 -touch $syslegal $sysent $sysinc $sysarg $sysprotoend $syscallnamestempfile $syshdrtempfile $audittempfile $tracecodetempfile +touch $syslegal $sysent $sysinc $sysarg $sysprotoend $syscallnamestempfile $syshdrtempfile $audittempfile $tracecodetempfile $systraceargstempfile $systraceargdesctempfile $systracerettempfile case $# in 0) @@ -93,6 +97,9 @@ if [ -n "$1" ]; then audit) output_auditevfile=1 ;; + systrace) + output_systrace=1 + ;; trace) output_tracecodes=1 use_stdout=1 @@ -141,6 +148,9 @@ s/\$//g syscallnamestempfile = \"$syscallnamestempfile\" syshdrfile = \"$syshdrfile\" syshdrtempfile = \"$syshdrtempfile\" + systraceargstempfile = \"$systraceargstempfile\" + systraceargdesctempfile = \"$systraceargdesctempfile\" + systracerettempfile = \"$systracerettempfile\" audittempfile = \"$audittempfile\" tracecodetempfile = \"$tracecodetempfile\" syscallprefix = \"$syscallprefix\" @@ -233,6 +243,19 @@ s/\$//g printf "#include \n\n" > audittempfile printf "#if CONFIG_AUDIT\n\n" > audittempfile printf "au_event_t sys_au_event[] = {\n" > audittempfile + + printf "/*\n * System call argument to DTrace register array conversion.\n */\n" > systraceargstempfile + printf "#include \n" > systraceargstempfile + printf "void\nsystrace_args(int sysnum, void *params, uint64_t *uarg)\n{\n" > systraceargstempfile + printf "\tint64_t *iarg = (int64_t *) uarg;\n" > systraceargstempfile + printf "\tswitch (sysnum) {\n" > systraceargstempfile + + printf "void\nsystrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)\n{\n\tconst char *p = NULL;\n" > systraceargdesctempfile + printf "\tswitch (sysnum) {\n" > systraceargdesctempfile + + printf "void\nsystrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)\n{\n\tconst char *p = NULL;\n" > systracerettempfile + printf "\tswitch (sysnum) {\n" > systracerettempfile + next } NF == 0 || $1 ~ /^;/ { @@ -248,6 +271,9 @@ s/\$//g print > syscallnamestempfile print > sysprotoend print > audittempfile + print > systraceargstempfile + print > systraceargdesctempfile + print > systracerettempfile savesyscall = syscall_num skip_for_header = 0 next @@ -258,6 +284,9 @@ s/\$//g print > syscallnamestempfile print > sysprotoend print > audittempfile + print > systraceargstempfile + print > systraceargdesctempfile + print > systracerettempfile syscall_num = savesyscall skip_for_header = 1 next @@ -268,6 +297,9 @@ s/\$//g print > syscallnamestempfile print > sysprotoend print > audittempfile + print > systraceargstempfile + print > systraceargdesctempfile + print > systracerettempfile skip_for_header = 0 next } @@ -467,6 +499,40 @@ s/\$//g size32 = 0 if ((funcname != "nosys" && funcname != "enosys") || (syscall_num == 0 && funcname == "nosys")) { + printf("\t/* %s */\n\tcase %d: {\n", funcname, syscall_num) > systraceargstempfile + printf("\t/* %s */\n\tcase %d:\n", funcname, syscall_num) > systraceargdesctempfile + printf("\t/* %s */\n\tcase %d:\n", funcname, syscall_num) > systracerettempfile + if (argc > 0) { + printf("\t\tswitch(ndx) {\n") > systraceargdesctempfile + printf("\t\tstruct %s *p = params;\n", argalias) > systraceargstempfile + for (i = 1; i <= argc; i++) { + arg = argtype[i] + sub("__restrict$", "", arg) + if (index(arg, "*") > 0) + printf("\t\tcase %d:\n\t\t\tp = \"userland %s\";\n\t\t\tbreak;\n", i - 1, arg) > systraceargdesctempfile + else + printf("\t\tcase %d:\n\t\t\tp = \"%s\";\n\t\t\tbreak;\n", i - 1, arg) > systraceargdesctempfile + if (index(arg, "*") > 0 || arg == "caddr_t") + printf("\t\tuarg[%d] = (intptr_t) p->%s; /* %s */\n", \ + i - 1, \ + argname[i], arg) > systraceargstempfile + else if (substr(arg, 1, 1) == "u" || arg == "size_t") + printf("\t\tuarg[%d] = p->%s; /* %s */\n", \ + i - 1, \ + argname[i], arg) > systraceargstempfile + else + printf("\t\tiarg[%d] = p->%s; /* %s */\n", \ + i - 1, \ + argname[i], arg) > systraceargstempfile + } + printf("\t\tdefault:\n\t\t\tbreak;\n\t\t};\n") > systraceargdesctempfile + + } + printf("\t\tbreak;\n\t}\n", argc) > systraceargstempfile + printf("\t\tif (ndx == 0 || ndx == 1)\n") > systracerettempfile + printf("\t\t\tp = \"%s\";\n", returntype) > systracerettempfile + printf("\t\tbreak;\n") > systracerettempfile + printf("\t\tbreak;\n") > systraceargdesctempfile if (argc != 0) { if (add_sysproto_entry == 1) { printf("struct %s {\n", argalias) > sysarg @@ -707,6 +773,10 @@ s/\$//g printf("#endif /* !%s */\n", syscall_h) > syshdrtempfile printf("};\n\n") > audittempfile printf("#endif /* AUDIT */\n") > audittempfile + + printf "\tdefault:\n\t\tbreak;\n\t};\n}\n" > systraceargstempfile + printf "\tdefault:\n\t\tbreak;\n\t};\n\tif (p != NULL)\n\t\tstrlcpy(desc, p, descsz);\n}\n" > systraceargdesctempfile + printf "\tdefault:\n\t\tbreak;\n\t};\n\tif (p != NULL)\n\t\tstrlcpy(desc, p, descsz);\n}\n" > systracerettempfile } ' # define value in syscall table file to permit redifintion because of the way @@ -733,6 +803,12 @@ if [ $output_auditevfile -eq 1 ]; then cat $syslegal $audittempfile > $auditevfile fi +if [ $output_systrace -eq 1 ]; then + cat $systraceargstempfile > $systraceargsfile + cat $systraceargdesctempfile >> $systraceargsfile + cat $systracerettempfile >> $systraceargsfile +fi + if [ $output_tracecodes -eq 1 ]; then if [ $use_stdout -eq 1 ]; then cat $tracecodetempfile @@ -740,3 +816,4 @@ if [ $output_tracecodes -eq 1 ]; then cat $tracecodetempfile > $tracecodename fi fi + diff --git a/bsd/kern/mcache.c b/bsd/kern/mcache.c index 823fbf95e..0794dc1db 100644 --- a/bsd/kern/mcache.c +++ b/bsd/kern/mcache.c @@ -69,7 +69,7 @@ #include #define MCACHE_SIZE(n) \ - ((size_t)(&((mcache_t *)0)->mc_cpu[n])) + __builtin_offsetof(mcache_t, mc_cpu[n]) /* Allocate extra in case we need to manually align the pointer */ #define MCACHE_ALLOC_SIZE \ @@ -154,6 +154,7 @@ static void mcache_bkt_purge(mcache_t *); static void mcache_bkt_destroy(mcache_t *, mcache_bkttype_t *, mcache_bkt_t *, int); static void mcache_bkt_ws_update(mcache_t *); +static void mcache_bkt_ws_zero(mcache_t *); static void mcache_bkt_ws_reap(mcache_t *); static void mcache_dispatch(void (*)(void *), void *); static void mcache_cache_reap(mcache_t *); @@ -307,11 +308,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align, return (NULL); } - if (!(wait & MCR_NOSLEEP)) - buf = zalloc(mcache_zone); - else - buf = zalloc_noblock(mcache_zone); - + buf = zalloc(mcache_zone); if (buf == NULL) goto fail; @@ -333,10 +330,14 @@ mcache_create_common(const char *name, size_t bufsize, size_t align, * Guaranteed alignment is valid only when we use the internal * slab allocator (currently set to use the zone allocator). */ - if (!need_zone) + if (!need_zone) { align = 1; - else if (align == 0) - align = MCACHE_ALIGN; + } else { + /* Enforce 64-bit minimum alignment for zone-based buffers */ + if (align == 0) + align = MCACHE_ALIGN; + align = P2ROUNDUP(align, MCACHE_ALIGN); + } if ((align & (align - 1)) != 0) panic("mcache_create: bad alignment %lu", align); @@ -368,9 +369,8 @@ mcache_create_common(const char *name, size_t bufsize, size_t align, */ chunksize = MAX(bufsize, sizeof (u_int64_t)); if (need_zone) { - /* Enforce 64-bit minimum alignment for zone-based buffers */ - align = MAX(align, sizeof (u_int64_t)); - chunksize += sizeof (void *) + align; + VERIFY(align != 0 && (align % MCACHE_ALIGN) == 0); + chunksize += sizeof (uint64_t) + align; chunksize = P2ROUNDUP(chunksize, align); if ((cp->mc_slab_zone = zinit(chunksize, 64 * 1024 * ncpu, PAGE_SIZE, cp->mc_name)) == NULL) @@ -898,11 +898,12 @@ mcache_destroy(mcache_t *cp) * implementation uses the zone allocator for simplicity reasons. */ static unsigned int -mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) +mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, + int wait) { +#pragma unused(wait) mcache_t *cp = arg; unsigned int need = num; - size_t offset = 0; size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t)); u_int32_t flags = cp->mc_flags; void *buf, *base, **pbuf; @@ -910,26 +911,14 @@ mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) *list = NULL; - /* - * The address of the object returned to the caller is an - * offset from the 64-bit aligned base address only if the - * cache's alignment requirement is neither 1 nor 8 bytes. - */ - if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t)) - offset = cp->mc_align; - for (;;) { - if (!(wait & MCR_NOSLEEP)) - buf = zalloc(cp->mc_slab_zone); - else - buf = zalloc_noblock(cp->mc_slab_zone); - + buf = zalloc(cp->mc_slab_zone); if (buf == NULL) break; - /* Get the 64-bit aligned base address for this object */ + /* Get the aligned base address for this object */ base = (void *)P2ROUNDUP((intptr_t)buf + sizeof (u_int64_t), - sizeof (u_int64_t)); + cp->mc_align); /* * Wind back a pointer size from the aligned base and @@ -938,6 +927,9 @@ mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) pbuf = (void **)((intptr_t)base - sizeof (void *)); *pbuf = buf; + VERIFY (((intptr_t)base + cp->mc_bufsize) <= + ((intptr_t)buf + cp->mc_chunksize)); + /* * If auditing is enabled, patternize the contents of * the buffer starting from the 64-bit aligned base to @@ -951,14 +943,8 @@ mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) mcache_set_pattern(MCACHE_FREE_PATTERN, base, rsize); } - /* - * Fix up the object's address to fulfill the cache's - * alignment requirement (if needed) and return this - * to the caller. - */ - VERIFY(((intptr_t)base + offset + cp->mc_bufsize) <= - ((intptr_t)buf + cp->mc_chunksize)); - *list = (mcache_obj_t *)((intptr_t)base + offset); + VERIFY(IS_P2ALIGNED(base, cp->mc_align)); + *list = (mcache_obj_t *)base; (*list)->obj_next = NULL; list = *plist = &(*list)->obj_next; @@ -979,40 +965,31 @@ mcache_slab_free(void *arg, mcache_obj_t *list, __unused boolean_t purged) { mcache_t *cp = arg; mcache_obj_t *nlist; - size_t offset = 0; size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t)); u_int32_t flags = cp->mc_flags; void *base; void **pbuf; - /* - * The address of the object is an offset from a 64-bit - * aligned base address only if the cache's alignment - * requirement is neither 1 nor 8 bytes. - */ - if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t)) - offset = cp->mc_align; - for (;;) { nlist = list->obj_next; list->obj_next = NULL; - /* Get the 64-bit aligned base address of this object */ - base = (void *)((intptr_t)list - offset); - VERIFY(IS_P2ALIGNED(base, sizeof (u_int64_t))); + base = list; + VERIFY(IS_P2ALIGNED(base, cp->mc_align)); /* Get the original address since we're about to free it */ pbuf = (void **)((intptr_t)base - sizeof (void *)); + VERIFY(((intptr_t)base + cp->mc_bufsize) <= + ((intptr_t)*pbuf + cp->mc_chunksize)); + if (flags & MCF_DEBUG) { VERIFY(((intptr_t)base + rsize) <= ((intptr_t)*pbuf + cp->mc_chunksize)); - mcache_audit_free_verify(NULL, base, offset, rsize); + mcache_audit_free_verify(NULL, base, 0, rsize); } /* Free it to zone */ - VERIFY(((intptr_t)base + offset + cp->mc_bufsize) <= - ((intptr_t)*pbuf + cp->mc_chunksize)); zfree(cp->mc_slab_zone, *pbuf); /* No more objects to free; return to mcache */ @@ -1028,24 +1005,14 @@ static void mcache_slab_audit(void *arg, mcache_obj_t *list, boolean_t alloc) { mcache_t *cp = arg; - size_t offset = 0; size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t)); void *base, **pbuf; - /* - * The address of the object returned to the caller is an - * offset from the 64-bit aligned base address only if the - * cache's alignment requirement is neither 1 nor 8 bytes. - */ - if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t)) - offset = cp->mc_align; - while (list != NULL) { mcache_obj_t *next = list->obj_next; - /* Get the 64-bit aligned base address of this object */ - base = (void *)((intptr_t)list - offset); - VERIFY(IS_P2ALIGNED(base, sizeof (u_int64_t))); + base = list; + VERIFY(IS_P2ALIGNED(base, cp->mc_align)); /* Get the original address */ pbuf = (void **)((intptr_t)base - sizeof (void *)); @@ -1056,7 +1023,7 @@ mcache_slab_audit(void *arg, mcache_obj_t *list, boolean_t alloc) if (!alloc) mcache_set_pattern(MCACHE_FREE_PATTERN, base, rsize); else - mcache_audit_free_verify_set(NULL, base, offset, rsize); + mcache_audit_free_verify_set(NULL, base, 0, rsize); list = list->obj_next = next; } @@ -1181,13 +1148,7 @@ mcache_bkt_purge(mcache_t *cp) mcache_bkt_destroy(cp, btp, pbp, pobjs); } - /* - * Updating the working set back to back essentially sets - * the working set size to zero, so everything is reapable. - */ - mcache_bkt_ws_update(cp); - mcache_bkt_ws_update(cp); - + mcache_bkt_ws_zero(cp); mcache_bkt_ws_reap(cp); } @@ -1246,6 +1207,22 @@ mcache_bkt_ws_update(mcache_t *cp) MCACHE_UNLOCK(&cp->mc_bkt_lock); } +/* + * Mark everything as eligible for reaping (working set is zero). + */ +static void +mcache_bkt_ws_zero(mcache_t *cp) +{ + MCACHE_LOCK(&cp->mc_bkt_lock); + + cp->mc_full.bl_reaplimit = cp->mc_full.bl_total; + cp->mc_full.bl_min = cp->mc_full.bl_total; + cp->mc_empty.bl_reaplimit = cp->mc_empty.bl_total; + cp->mc_empty.bl_min = cp->mc_empty.bl_total; + + MCACHE_UNLOCK(&cp->mc_bkt_lock); +} + /* * Reap all buckets that are beyond the working set. */ @@ -1314,6 +1291,18 @@ mcache_reap(void) mcache_dispatch(mcache_reap_start, flag); } +__private_extern__ void +mcache_reap_now(mcache_t *cp, boolean_t purge) +{ + if (purge) { + mcache_bkt_purge(cp); + mcache_cache_bkt_enable(cp); + } else { + mcache_bkt_ws_zero(cp); + mcache_bkt_ws_reap(cp); + } +} + static void mcache_cache_reap(mcache_t *cp) { diff --git a/bsd/kern/policy_check.c b/bsd/kern/policy_check.c index cbaf30ca4..85a92f5fe 100644 --- a/bsd/kern/policy_check.c +++ b/bsd/kern/policy_check.c @@ -13,6 +13,7 @@ #include #include +#include #include /* OSBPrintBacktrace */ @@ -118,7 +119,7 @@ common_hook(void) return rv; } -#if (MAC_POLICY_OPS_VERSION != 47) +#if (MAC_POLICY_OPS_VERSION != 52) # error "struct mac_policy_ops doesn't match definition in mac_policy.h" #endif /* @@ -127,7 +128,7 @@ common_hook(void) * Please note that this struct initialization should be kept in sync with * security/mac_policy.h (mac_policy_ops struct definition). */ -static struct mac_policy_ops policy_ops = { +const static struct mac_policy_ops policy_ops = { CHECK_SET_HOOK(audit_check_postselect) CHECK_SET_HOOK(audit_check_preselect) @@ -275,7 +276,7 @@ static struct mac_policy_ops policy_ops = { CHECK_SET_HOOK(proc_check_set_host_exception_port) CHECK_SET_HOOK(exc_action_check_exception_send) CHECK_SET_HOOK(exc_action_label_associate) - CHECK_SET_HOOK(exc_action_label_copy) + CHECK_SET_HOOK(exc_action_label_populate) CHECK_SET_HOOK(exc_action_label_destroy) CHECK_SET_HOOK(exc_action_label_init) CHECK_SET_HOOK(exc_action_label_update) @@ -284,8 +285,8 @@ static struct mac_policy_ops policy_ops = { .mpo_reserved2 = (mpo_reserved_hook_t *)common_hook, .mpo_reserved3 = (mpo_reserved_hook_t *)common_hook, .mpo_reserved4 = (mpo_reserved_hook_t *)common_hook, - .mpo_reserved5 = (mpo_reserved_hook_t *)common_hook, - .mpo_reserved6 = (mpo_reserved_hook_t *)common_hook, + CHECK_SET_HOOK(skywalk_flow_check_connect) + CHECK_SET_HOOK(skywalk_flow_check_listen) CHECK_SET_HOOK(posixsem_check_create) CHECK_SET_HOOK(posixsem_check_open) @@ -363,7 +364,7 @@ static struct mac_policy_ops policy_ops = { CHECK_SET_HOOK(system_check_settime) CHECK_SET_HOOK(system_check_swapoff) CHECK_SET_HOOK(system_check_swapon) - .mpo_reserved7 = (mpo_reserved_hook_t *)common_hook, + CHECK_SET_HOOK(socket_check_ioctl) CHECK_SET_HOOK(sysvmsg_label_associate) CHECK_SET_HOOK(sysvmsg_label_destroy) @@ -396,7 +397,7 @@ static struct mac_policy_ops policy_ops = { CHECK_SET_HOOK(sysvshm_label_init) CHECK_SET_HOOK(sysvshm_label_recycle) - .mpo_reserved8 = (mpo_reserved_hook_t *)common_hook, + CHECK_SET_HOOK(proc_notify_exit) CHECK_SET_HOOK(mount_check_snapshot_revert) CHECK_SET_HOOK(vnode_check_getattr) CHECK_SET_HOOK(mount_check_snapshot_create) @@ -493,7 +494,7 @@ static struct mac_policy_ops policy_ops = { CHECK_SET_HOOK(system_check_kas_info) - CHECK_SET_HOOK(proc_check_cpumon) + CHECK_SET_HOOK(vnode_check_lookup_preflight) CHECK_SET_HOOK(vnode_notify_open) @@ -519,7 +520,7 @@ static struct mac_policy_ops policy_ops = { /* * Policy definition */ -static struct mac_policy_conf policy_conf = { +static SECURITY_READ_ONLY_LATE(struct mac_policy_conf) policy_conf = { .mpc_name = "CHECK", .mpc_fullname = "Check Assumptions Policy", .mpc_field_off = NULL, /* no label slot */ @@ -530,7 +531,7 @@ static struct mac_policy_conf policy_conf = { .mpc_runtime_flags = 0, }; -static mac_policy_handle_t policy_handle; +static SECURITY_READ_ONLY_LATE(mac_policy_handle_t) policy_handle; /* * Init routine; for a loadable policy, this would be called during the KEXT diff --git a/bsd/kern/posix_sem.c b/bsd/kern/posix_sem.c index ca7ee4c6a..9dc882363 100644 --- a/bsd/kern/posix_sem.c +++ b/bsd/kern/posix_sem.c @@ -168,20 +168,21 @@ struct psemstats psemstats; /* cache effectiveness statistics */ static int psem_access(struct pseminfo *pinfo, int mode, kauth_cred_t cred); static int psem_cache_search(struct pseminfo **, - struct psemname *, struct psemcache **); + struct psemname *, struct psemcache **); static int psem_delete(struct pseminfo * pinfo); static int psem_read (struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int psem_write (struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int psem_ioctl (struct fileproc *fp, u_long com, - caddr_t data, vfs_context_t ctx); + caddr_t data, vfs_context_t ctx); static int psem_select (struct fileproc *fp, int which, void *wql, vfs_context_t ctx); static int psem_closefile (struct fileglob *fp, vfs_context_t ctx); static int psem_unlink_internal(struct pseminfo *pinfo, struct psemcache *pcache); -static int psem_kqfilter (struct fileproc *fp, struct knote *kn, vfs_context_t ctx); +static int psem_kqfilter (struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx); static const struct fileops psemops = { .fo_type = DTYPE_PSXSEM, @@ -1093,36 +1094,36 @@ psem_delete(struct pseminfo * pinfo) } static int -psem_read(__unused struct fileproc *fp, __unused struct uio *uio, - __unused int flags, __unused vfs_context_t ctx) +psem_read(__unused struct fileproc *fp, __unused struct uio *uio, + __unused int flags, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -psem_write(__unused struct fileproc *fp, __unused struct uio *uio, - __unused int flags, __unused vfs_context_t ctx) +psem_write(__unused struct fileproc *fp, __unused struct uio *uio, + __unused int flags, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -psem_ioctl(__unused struct fileproc *fp, __unused u_long com, - __unused caddr_t data, __unused vfs_context_t ctx) +psem_ioctl(__unused struct fileproc *fp, __unused u_long com, + __unused caddr_t data, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -psem_select(__unused struct fileproc *fp, __unused int which, - __unused void *wql, __unused vfs_context_t ctx) +psem_select(__unused struct fileproc *fp, __unused int which, + __unused void *wql, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -psem_kqfilter(__unused struct fileproc *fp, struct knote *kn, - __unused vfs_context_t ctx) +psem_kqfilter(__unused struct fileproc *fp, struct knote *kn, + __unused struct kevent_internal_s *kev, __unused vfs_context_t ctx) { kn->kn_flags = EV_ERROR; kn->kn_data = ENOTSUP; diff --git a/bsd/kern/posix_shm.c b/bsd/kern/posix_shm.c index 2ddd346bc..169eba5c5 100644 --- a/bsd/kern/posix_shm.c +++ b/bsd/kern/posix_shm.c @@ -176,16 +176,17 @@ long pshmnument; /* number of cache entries allocated */ struct pshmstats pshmstats; /* cache effectiveness statistics */ static int pshm_read (struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int pshm_write (struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int pshm_ioctl (struct fileproc *fp, u_long com, - caddr_t data, vfs_context_t ctx); + caddr_t data, vfs_context_t ctx); static int pshm_select (struct fileproc *fp, int which, void *wql, vfs_context_t ctx); static int pshm_close(struct pshminfo *pinfo, int dropref); static int pshm_closefile (struct fileglob *fg, vfs_context_t ctx); -static int pshm_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx); +static int pshm_kqfilter(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx); int pshm_access(struct pshminfo *pinfo, int mode, kauth_cred_t cred, proc_t p); int pshm_cache_purge_all(proc_t p); @@ -869,6 +870,7 @@ pshm_mmap(__unused proc_t p, struct mmap_args *uap, user_addr_t *retval, struct vm_object_offset_t map_pos; vm_map_t user_map; int alloc_flags; + vm_map_kernel_flags_t vmk_flags; boolean_t docow; kern_return_t kret; struct pshminfo * pinfo; @@ -945,13 +947,15 @@ pshm_mmap(__unused proc_t p, struct mmap_args *uap, user_addr_t *retval, struct docow = FALSE; mapped_size = 0; - - /* reserver the entire space first... */ + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + /* reserve the entire space first... */ kret = vm_map_enter_mem_object(user_map, &user_addr, user_size, 0, alloc_flags, + vmk_flags, + VM_KERN_MEMORY_NONE, IPC_PORT_NULL, 0, FALSE, @@ -978,12 +982,15 @@ pshm_mmap(__unused proc_t p, struct mmap_args *uap, user_addr_t *retval, struct if (map_size > user_size) { map_size = user_size; } + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; kret = vm_map_enter_mem_object( user_map, &user_addr, map_size, 0, VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, + vmk_flags, + VM_KERN_MEMORY_NONE, pshmobj->pshmo_memobject, file_pos - map_pos, docow, @@ -1255,36 +1262,36 @@ pshm_closefile(struct fileglob *fg, __unused vfs_context_t ctx) } static int -pshm_read(__unused struct fileproc *fp, __unused struct uio *uio, - __unused int flags, __unused vfs_context_t ctx) +pshm_read(__unused struct fileproc *fp, __unused struct uio *uio, + __unused int flags, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -pshm_write(__unused struct fileproc *fp, __unused struct uio *uio, - __unused int flags, __unused vfs_context_t ctx) +pshm_write(__unused struct fileproc *fp, __unused struct uio *uio, + __unused int flags, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -pshm_ioctl(__unused struct fileproc *fp, __unused u_long com, - __unused caddr_t data, __unused vfs_context_t ctx) +pshm_ioctl(__unused struct fileproc *fp, __unused u_long com, + __unused caddr_t data, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -pshm_select(__unused struct fileproc *fp, __unused int which, __unused void *wql, - __unused vfs_context_t ctx) +pshm_select(__unused struct fileproc *fp, __unused int which, __unused void *wql, + __unused vfs_context_t ctx) { return(ENOTSUP); } static int pshm_kqfilter(__unused struct fileproc *fp, struct knote *kn, - __unused vfs_context_t ctx) + __unused struct kevent_internal_s *kev, __unused vfs_context_t ctx) { kn->kn_flags = EV_ERROR; kn->kn_data = ENOTSUP; diff --git a/bsd/kern/proc_info.c b/bsd/kern/proc_info.c index e0707e35a..9efb0b1ca 100644 --- a/bsd/kern/proc_info.c +++ b/bsd/kern/proc_info.c @@ -84,7 +84,7 @@ #include -/* Needed by proc_pidnoteexit() */ +/* Needed by proc_pidnoteexit(), proc_pidlistuptrs() */ #include #include @@ -93,6 +93,10 @@ #include #endif +#if CONFIG_MACF +#include +#endif + struct pshmnode; struct psemnode; struct pipe; @@ -101,6 +105,7 @@ struct atalk; uint64_t get_dispatchqueue_offset_from_proc(void *); uint64_t get_dispatchqueue_serialno_offset_from_proc(void *); +uint64_t get_return_to_kernel_offset_from_proc(void *p); int proc_info_internal(int callnum, int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t buffersize, int32_t * retval); /* @@ -166,7 +171,8 @@ void __attribute__ ((noinline)) proc_pidcoalitioninfo(proc_t p, struct proc_pidc int __attribute__ ((noinline)) proc_pidnoteexit(proc_t p, uint64_t arg, uint32_t *data); int __attribute__ ((noinline)) proc_pidexitreasoninfo(proc_t p, struct proc_exitreasoninfo *peri, struct proc_exitreasonbasicinfo *pberi); int __attribute__ ((noinline)) proc_pidoriginatorpid_uuid(uuid_t uuid, uint32_t buffersize, pid_t *pid); - +int __attribute__ ((noinline)) proc_pidlistuptrs(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retval); +int __attribute__ ((noinline)) proc_piddynkqueueinfo(pid_t pid, int flavor, kqueue_id_t id, user_addr_t buffer, uint32_t buffersize, int32_t *retval); /* protos for proc_pidfdinfo calls */ int __attribute__ ((noinline)) pid_vnodeinfo(vnode_t vp, uint32_t vid, struct fileproc * fp,proc_t proc, int fd, user_addr_t buffer, uint32_t buffersize, int32_t * retval); @@ -187,9 +193,6 @@ int proc_security_policy(proc_t targetp, int callnum, int flavor, boolean_t chec static void munge_vinfo_stat(struct stat64 *sbp, struct vinfo_stat *vsbp); static int proc_piduuidinfo(pid_t pid, uuid_t uuid_buf, uint32_t buffersize); int proc_pidpathinfo_internal(proc_t p, __unused uint64_t arg, char *buf, uint32_t buffersize, __unused int32_t *retval); -int proc_listfd_kqueue(proc_t p, int32_t *fdlist, int len); -int proc_kqueue_udata_info(proc_t p, int32_t fd, uint64_t *buffer, int bufsize); -int proc_list_uptrs(proc_t p, uint64_t *udata_buffer, int size); extern int cansignal(struct proc *, kauth_cred_t, struct proc *, int, int); extern int proc_get_rusage(proc_t proc, int flavor, user_addr_t buffer, int is_zombie); @@ -217,6 +220,16 @@ uint64_t get_dispatchqueue_serialno_offset_from_proc(void *p) } } +uint64_t get_return_to_kernel_offset_from_proc(void *p) +{ + if (p != NULL) { + proc_t pself = (proc_t)p; + return (pself->p_return_to_kernel_offset); + } else { + return (uint64_t)0; + } +} + /***************************** proc_info ********************/ int @@ -257,8 +270,10 @@ proc_info_internal(int callnum, int pid, int flavor, uint64_t arg, user_addr_t b buffersize, retval); case PROC_INFO_CALL_CANUSEFGHW: return proc_can_use_foreground_hw(pid, buffer, buffersize, retval); + case PROC_INFO_CALL_PIDDYNKQUEUEINFO: + return proc_piddynkqueueinfo(pid, flavor, (kqueue_id_t)arg, buffer, buffersize, retval); default: - return(EINVAL); + return EINVAL; } return(EINVAL); @@ -268,10 +283,12 @@ proc_info_internal(int callnum, int pid, int flavor, uint64_t arg, user_addr_t b int proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t buffersize, int32_t * retval) { - int numprocs, wantpids; + uint32_t numprocs = 0; + uint32_t wantpids; char * kbuf; int * ptr; - int n, skip; + uint32_t n; + int skip; struct proc * p; struct tty * tp; int error = 0; @@ -283,7 +300,7 @@ proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t bu /* if the buffer is null, return num of procs */ if (buffer == (user_addr_t)0) { - *retval = ((nprocs+20) * sizeof(int)); + *retval = ((nprocs + 20) * sizeof(int)); return(0); } @@ -291,13 +308,17 @@ proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t bu return(ENOMEM); } wantpids = buffersize/sizeof(int); - numprocs = nprocs+20; - if (numprocs > wantpids) + if ((nprocs + 20) > 0) { + numprocs = (uint32_t)(nprocs + 20); + } + if (numprocs > wantpids) { numprocs = wantpids; + } kbuf = (char *)kalloc((vm_size_t)(numprocs * sizeof(int))); - if (kbuf == NULL) + if (kbuf == NULL) { return(ENOMEM); + } bzero(kbuf, sizeof(int)); proc_list_lock(); @@ -358,6 +379,11 @@ proc_loop: skip = 1; } break; + case PROC_KDBG_ONLY: + if (p->p_kdebug == 0) { + skip = 1; + } + break; default: skip = 1; break; @@ -393,7 +419,8 @@ proc_loop: int proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retval) { - int numfds, needfds; + uint32_t numfds = 0; + uint32_t needfds; char * kbuf; struct proc_fdinfo * pfd; struct fileproc * fp; @@ -401,7 +428,9 @@ proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retv int count = 0; int error = 0; - numfds = p->p_fd->fd_nfiles; + if (p->p_fd->fd_nfiles > 0) { + numfds = (uint32_t)p->p_fd->fd_nfiles; + } if (buffer == (user_addr_t) 0) { numfds += 20; @@ -412,8 +441,9 @@ proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retv /* buffersize is big enough atleast for one struct */ needfds = buffersize/sizeof(struct proc_fdinfo); - if (numfds > needfds) + if (numfds > needfds) { numfds = needfds; + } kbuf = (char *)kalloc((vm_size_t)(numfds * sizeof(struct proc_fdinfo))); if (kbuf == NULL) @@ -424,7 +454,7 @@ proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retv pfd = (struct proc_fdinfo *)kbuf; - for (n = 0; ((n < numfds) && (n < p->p_fd->fd_nfiles)); n++) { + for (n = 0; ((n < (int)numfds) && (n < p->p_fd->fd_nfiles)); n++) { if (((fp = p->p_fd->fd_ofiles[n]) != 0) && ((p->p_fd->fd_ofileflags[n] & UF_RESERVED) == 0)) { file_type_t fdtype = FILEGLOB_DTYPE(fp->f_fglob); @@ -621,8 +651,10 @@ proc_pidbsdinfo(proc_t p, struct proc_bsdinfo * pbsd, int zombie) pbsd->pbi_flags |= PROC_FLAG_CTTY; } +#if !CONFIG_EMBEDDED if ((p->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP) pbsd->pbi_flags |= PROC_FLAG_DELAYIDLESLEEP; +#endif /* !CONFIG_EMBEDDED */ switch(PROC_CONTROL_STATE(p)) { case P_PCTHROTTLE: @@ -700,8 +732,10 @@ proc_pidshortbsdinfo(proc_t p, struct proc_bsdshortinfo * pbsd_shortp, int zombi pbsd_shortp->pbsi_flags |= PROC_FLAG_PSUGID; if ((p->p_flag & P_EXEC) == P_EXEC) pbsd_shortp->pbsi_flags |= PROC_FLAG_EXEC; +#if !CONFIG_EMBEDDED if ((p->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP) pbsd_shortp->pbsi_flags |= PROC_FLAG_DELAYIDLESLEEP; +#endif /* !CONFIG_EMBEDDED */ switch(PROC_CONTROL_STATE(p)) { case P_PCTHROTTLE: @@ -886,20 +920,22 @@ proc_pidthreadpathinfo(proc_t p, uint64_t arg, struct proc_threadwithpathinfo * int proc_pidlistthreads(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retval) { - int count = 0; + uint32_t count = 0; int ret = 0; int error = 0; void * kbuf; - int numthreads; + uint32_t numthreads = 0; - - count = buffersize/(sizeof(uint64_t)); - numthreads = get_numthreads(p->task); + int num = get_numthreads(p->task) + 10; + if (num > 0) { + numthreads = (uint32_t)num; + } - numthreads += 10; + count = buffersize/(sizeof(uint64_t)); - if (numthreads > count) + if (numthreads > count) { numthreads = count; + } kbuf = (void *)kalloc(numthreads * sizeof(uint64_t)); if (kbuf == NULL) @@ -1647,7 +1683,7 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t bu switch (flavor) { case PROC_PIDLISTFDS: size = PROC_PIDLISTFD_SIZE; - if (buffer == (user_addr_t)0) + if (buffer == USER_ADDR_NULL) size = 0; break; case PROC_PIDTBSDINFO: @@ -1732,6 +1768,18 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t bu case PROC_PIDREGIONPATHINFO3: size = PROC_PIDREGIONPATHINFO3_SIZE; break; + case PROC_PIDLISTUPTRS: + size = PROC_PIDLISTUPTRS_SIZE; + if (buffer == USER_ADDR_NULL) { + size = 0; + } + break; + case PROC_PIDLISTDYNKQUEUES: + size = PROC_PIDLISTDYNKQUEUES_SIZE; + if (buffer == USER_ADDR_NULL) { + size = 0; + } + break; default: return(EINVAL); } @@ -2007,8 +2055,17 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t bu } break; + case PROC_PIDLISTUPTRS: + error = proc_pidlistuptrs(p, buffer, buffersize, retval); + break; + + case PROC_PIDLISTDYNKQUEUES: + error = kevent_copyout_proc_dynkqids(p, buffer, buffersize, retval); + break; + default: error = ENOTSUP; + break; } out: @@ -2356,7 +2413,7 @@ proc_pidfdinfo(int pid, int flavor, int fd, user_addr_t buffer, uint32_t buffer struct kqueue * kq; if (fd == -1) { - if ((kq = p->p_wqkqueue) == NULL) { + if ((kq = p->p_fd->fd_wqkqueue) == NULL) { /* wqkqueue is initialized on-demand */ error = 0; break; @@ -2374,7 +2431,7 @@ proc_pidfdinfo(int pid, int flavor, int fd, user_addr_t buffer, uint32_t buffer struct kqueue * kq; if (fd == -1) { - if ((kq = p->p_wqkqueue) == NULL) { + if ((kq = p->p_fd->fd_wqkqueue) == NULL) { /* wqkqueue is initialized on-demand */ error = 0; break; @@ -2401,119 +2458,53 @@ out: return(error); } -int -proc_listfd_kqueue(proc_t p, int32_t *fdlist, int len) -{ - int numfds; - struct fileproc * fp; - int n; - int count = 0; - - numfds = p->p_fd->fd_nfiles; - if (len < numfds) { - return -1; - } - - proc_fdlock(p); - for (n = 0; ((n < numfds) && (n < p->p_fd->fd_nfiles)); n++) { - if (((fp = p->p_fd->fd_ofiles[n]) != 0) - && ((p->p_fd->fd_ofileflags[n] & UF_RESERVED) == 0) - && (FILEGLOB_DTYPE(fp->f_fglob) == PROX_FDTYPE_KQUEUE)) { - fdlist[count++] = n; - } - } - proc_fdunlock(p); - return count; -} +#define MAX_UPTRS 16392 int -proc_kqueue_udata_info(proc_t p, int32_t fd, uint64_t *buffer, int bufsize) +proc_pidlistuptrs(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retval) { - struct kqueue *kq; - struct fileproc * fp = NULL; - int retval; + uint32_t count = 0; + int error = 0; + void *kbuf = NULL; + int32_t nuptrs = 0; - if (fd == -1) { - /* wqkqueue is initialized on-demand */ - if ((kq = p->p_wqkqueue) == NULL) { - return 0; + if (buffer != USER_ADDR_NULL) { + count = buffersize / sizeof(uint64_t); + if (count > MAX_UPTRS) { + count = MAX_UPTRS; + buffersize = count * sizeof(uint64_t); } - } else { - int error = fp_getfkq(p, fd, &fp, &kq); - if (error != 0) { - return 0; + if (count > 0) { + kbuf = kalloc(buffersize); + assert(kbuf != NULL); } - } - - retval = pid_kqueue_udatainfo(p, kq, buffer, bufsize); - if (fp) { - fp_drop(p, fd, fp , 0); - } - - return retval; -} - -int -proc_list_uptrs(proc_t p, uint64_t *udata_buffer, int size) -{ - int32_t *fdlist = NULL; - int nfds; - int i; - int count = 0; - int ret; - int knote_max = 4096; - uint64_t *buffer; - int bufsize = knote_max * sizeof(uint64_t); - - fdlist = (int32_t *)kalloc((OPEN_MAX + 1) * sizeof(int32_t)); - if (!fdlist) { - return -1; - } - - nfds = proc_listfd_kqueue(p, &fdlist[1], OPEN_MAX); - if (nfds < 0 || nfds > OPEN_MAX) { - kfree(fdlist, (OPEN_MAX + 1) * sizeof(int32_t)); - return 0; - } - - /* Add FD -1, the implicit workq kqueue */ - fdlist[0] = -1; - nfds++; - - if (size == 0) { - bufsize = 0; - buffer = NULL; } else { - bufsize = knote_max * sizeof(uint64_t); - buffer = (uint64_t *)kalloc(bufsize); + buffersize = 0; } - for (i = 0; i < nfds; i++) { -again: - ret = proc_kqueue_udata_info(p, fdlist[i], buffer, bufsize); - if (bufsize != 0 && ret > knote_max) { - kfree(buffer, bufsize); - knote_max = ret + 32; - bufsize = knote_max * sizeof(uint64_t); - buffer = kalloc(bufsize); - goto again; - } - - if (ret == 0) - continue; + nuptrs = kevent_proc_copy_uptrs(p, kbuf, buffersize); - /* Copy the udata ptrs */ - if (size >= (int)((count + ret) * sizeof(uint64_t))) { - memcpy(&udata_buffer[count], buffer, ret * sizeof(uint64_t)); + if (kbuf) { + size_t copysize; + if (os_mul_overflow(nuptrs, sizeof(uint64_t), ©size)) { + error = ERANGE; + goto out; + } + if (copysize > buffersize) { + copysize = buffersize; } - count = count + ret; + error = copyout(kbuf, buffer, copysize); } - kfree(fdlist, (OPEN_MAX + 1) * sizeof(int32_t)); - if (buffer) { - kfree(buffer, bufsize); +out: + *retval = nuptrs; + + if (kbuf) { + kfree(kbuf, buffersize); + kbuf = NULL; } - return count; + + return error; } /* @@ -3174,3 +3165,41 @@ proc_pidnoteexit(proc_t p, uint64_t flags, uint32_t *data) return (0); } +int +proc_piddynkqueueinfo(int pid, int flavor, kqueue_id_t kq_id, + user_addr_t ubuf, uint32_t bufsize, int32_t *retval) +{ + proc_t p; + int err; + + if (ubuf == USER_ADDR_NULL) { + return EFAULT; + } + + p = proc_find(pid); + if (p == PROC_NULL) { + return ESRCH; + } + + err = proc_security_policy(p, PROC_INFO_CALL_PIDDYNKQUEUEINFO, 0, CHECK_SAME_USER); + if (err) { + goto out; + } + + switch (flavor) { + case PROC_PIDDYNKQUEUE_INFO: + err = kevent_copyout_dynkqinfo(p, kq_id, ubuf, bufsize, retval); + break; + case PROC_PIDDYNKQUEUE_EXTINFO: + err = kevent_copyout_dynkqextinfo(p, kq_id, ubuf, bufsize, retval); + break; + default: + err = ENOTSUP; + break; + } + +out: + proc_rele(p); + + return err; +} diff --git a/bsd/kern/process_policy.c b/bsd/kern/process_policy.c index bcf69d7c8..0a0b43e66 100644 --- a/bsd/kern/process_policy.c +++ b/bsd/kern/process_policy.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -43,8 +44,6 @@ #include #include -#include - #include #include #include @@ -72,6 +71,15 @@ #include #include +#if CONFIG_EMBEDDED +#include +#endif /* CONFIG_EMBEDDED */ + +#if CONFIG_MACF +#include +#include +#endif /* CONFIG_MACF */ + static int handle_lowresource(int scope, int action, int policy, int policy_subtype, user_addr_t attrp, proc_t proc, uint64_t target_threadid); static int handle_cpuuse(int action, user_addr_t attrp, proc_t proc, uint64_t target_threadid); static int handle_apptype(int scope, int action, int policy, int policy_subtype, user_addr_t attrp, proc_t proc, uint64_t target_threadid); @@ -80,6 +88,9 @@ static int handle_boost(int scope, int action, int policy, int policy_subtype, u extern kern_return_t task_suspend(task_t); extern kern_return_t task_resume(task_t); +#if CONFIG_EMBEDDED +static int handle_applifecycle(int scope, int action, int policy, int policy_subtype, user_addr_t attrp, proc_t proc, uint64_t target_threadid); +#endif /* CONFIG_EMBEDDED */ /***************************** process_policy ********************/ @@ -104,8 +115,13 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus pid_t target_pid = uap->target_pid; uint64_t target_threadid = uap->target_threadid; proc_t target_proc = PROC_NULL; +#if CONFIG_MACF || !CONFIG_EMBEDDED proc_t curp = current_proc(); +#endif kauth_cred_t my_cred; +#if CONFIG_EMBEDDED + kauth_cred_t target_cred; +#endif if ((scope != PROC_POLICY_SCOPE_PROCESS) && (scope != PROC_POLICY_SCOPE_THREAD)) { return(EINVAL); @@ -121,6 +137,13 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus my_cred = kauth_cred_get(); +#if CONFIG_EMBEDDED + target_cred = kauth_cred_proc_ref(target_proc); + + if (!kauth_cred_issuser(my_cred) && kauth_cred_getruid(my_cred) && + kauth_cred_getuid(my_cred) != kauth_cred_getuid(target_cred) && + kauth_cred_getruid(my_cred) != kauth_cred_getuid(target_cred)) +#else /* * Resoure starvation control can be used by unpriv resource owner but priv at the time of ownership claim. This is * checked in low resource handle routine. So bypass the checks here. @@ -128,6 +151,7 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus if ((policy != PROC_POLICY_RESOURCE_STARVATION) && (policy != PROC_POLICY_APPTYPE) && (!kauth_cred_issuser(my_cred) && curp != p)) +#endif { error = EPERM; goto out; @@ -137,6 +161,10 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus switch (policy) { case PROC_POLICY_BOOST: case PROC_POLICY_RESOURCE_USAGE: +#if CONFIG_EMBEDDED + case PROC_POLICY_APPTYPE: + case PROC_POLICY_APP_LIFECYCLE: +#endif /* These policies do their own appropriate mac checks */ break; default: @@ -175,6 +203,11 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus error = handle_cpuuse(action, attrp, target_proc, target_threadid); break; +#if CONFIG_EMBEDDED + case PROC_POLICY_APP_LIFECYCLE: + error = handle_applifecycle(scope, action, policy, policy_subtype, attrp, target_proc, target_threadid); + break; +#endif /* CONFIG_EMBEDDED */ case PROC_POLICY_APPTYPE: error = handle_apptype(scope, action, policy, policy_subtype, attrp, target_proc, target_threadid); break; @@ -188,6 +221,9 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus out: proc_rele(target_proc); +#if CONFIG_EMBEDDED + kauth_cred_unref(&target_cred); +#endif return(error); } @@ -217,40 +253,42 @@ static int handle_cpuuse(int action, user_addr_t attrp, proc_t proc, __unused uint64_t target_threadid) { proc_policy_cpuusage_attr_t cpuattr; -#if CONFIG_MACF +#if CONFIG_MACF || !CONFIG_EMBEDDED proc_t curp = current_proc(); #endif - int entitled = FALSE; + Boolean privileged = FALSE; Boolean canEnable = FALSE; uint64_t interval = -1ULL; int error = 0; uint8_t percentage; +#if !CONFIG_EMBEDDED + /* On macOS, tasks can only set and clear their own CPU limits. */ + if ((action == PROC_POLICY_ACTION_APPLY || action == PROC_POLICY_ACTION_RESTORE) + && curp != proc) { + return (EPERM); + } + /* No privilege required on macOS. */ + privileged = TRUE; +#endif + #if CONFIG_MACF - /* - * iOS only allows processes to override their own CPU usage monitor - * parameters if they have com.apple.private.kernel.override-cpumon. - * - * Until rdar://24799462 improves our scheme, we are also using the - * same entitlement to indicate which processes can resume monitoring - * when they otherwise wouldn't be able to. - */ - entitled = (mac_proc_check_cpumon(curp) == 0) ? TRUE : FALSE; - canEnable = (entitled && action == PROC_POLICY_ACTION_ENABLE); + /* Is caller privileged to set less-restrictive scheduling parameters? */ + if (!privileged) { + privileged = (priv_check_cred(kauth_cred_get(), PRIV_PROC_CPUMON_OVERRIDE, 0) == 0); + } + canEnable = (privileged && action == PROC_POLICY_ACTION_ENABLE); if (!canEnable && curp != proc) { - /* can the current process change scheduling parameters? */ + /* + * Can the current process change scheduling parameters for + * the target process? + */ error = mac_proc_check_sched(curp, proc); if (error) return error; } #endif - // on macOS tasks can only set and clear their own CPU limits - if ((action == PROC_POLICY_ACTION_APPLY || action == PROC_POLICY_ACTION_RESTORE) - && proc != current_proc()) { - return (EPERM); - } - switch (action) { case PROC_POLICY_ACTION_GET: error = proc_get_task_ruse_cpu(proc->task, &cpuattr.ppattr_cpu_attr, @@ -286,12 +324,12 @@ handle_cpuuse(int action, user_addr_t attrp, proc_t proc, __unused uint64_t targ cpuattr.ppattr_cpu_percentage, interval, cpuattr.ppattr_cpu_attr_deadline, - entitled); + privileged); break; /* restore process to prior state */ case PROC_POLICY_ACTION_RESTORE: - error = proc_clear_task_ruse_cpu(proc->task, entitled); + error = proc_clear_task_ruse_cpu(proc->task, privileged); break; /* re-enable suspended monitor */ @@ -310,6 +348,78 @@ handle_cpuuse(int action, user_addr_t attrp, proc_t proc, __unused uint64_t targ return(error); } +#if CONFIG_EMBEDDED +static int +handle_applifecycle(__unused int scope, + int action, + __unused int policy, + int policy_subtype, + user_addr_t attrp, + proc_t proc, + uint64_t target_threadid) +{ + int error = 0; + int state = 0; + + switch(policy_subtype) { + case PROC_POLICY_APPLIFE_NONE: + error = 0; + break; + + case PROC_POLICY_APPLIFE_STATE: + /* appstate is no longer supported */ + error = ENOTSUP; + break; + + case PROC_POLICY_APPLIFE_DEVSTATUS: +#if CONFIG_MACF + /* ToDo - this should be a generic check, since we could potentially hang other behaviours here. */ + error = mac_proc_check_suspend_resume(current_proc(), MAC_PROC_CHECK_HIBERNATE); + if (error) { + error = EPERM; + goto out; + } +#endif +#if CONFIG_MEMORYSTATUS + if (action == PROC_POLICY_ACTION_APPLY) { + /* Used as a freeze hint */ + memorystatus_on_inactivity(proc); + + /* in future use devicestatus for pid_socketshutdown() */ + error = 0; + } else +#endif + { + error = EINVAL; + } + break; + + case PROC_POLICY_APPLIFE_PIDBIND: +#if CONFIG_MACF + error = mac_proc_check_suspend_resume(current_proc(), MAC_PROC_CHECK_PIDBIND); + if (error) { + error = EPERM; + goto out; + } +#endif + error = copyin((user_addr_t)attrp, (int *)&state, sizeof(int)); + if (error != 0) + goto out; + if (action == PROC_POLICY_ACTION_APPLY) { + /* bind the thread in target_thread in current process to target_proc */ + error = proc_lf_pidbind(current_task(), target_threadid, proc->task, state); + } else + error = EINVAL; + break; + default: + error = EINVAL; + break; + } + +out: + return(error); +} +#endif /* CONFIG_EMBEDDED */ static int handle_apptype( int scope, @@ -499,7 +609,7 @@ proc_get_originatorbgstate(uint32_t *is_backgrounded) { uint32_t bgstate; proc_t p = current_proc(); - uint32_t flagsp; + uint32_t flagsp = 0; kern_return_t kr; pid_t pid; int ret; diff --git a/bsd/kern/pthread_shims.c b/bsd/kern/pthread_shims.c index 76e76c957..c8e42fc8c 100644 --- a/bsd/kern/pthread_shims.c +++ b/bsd/kern/pthread_shims.c @@ -28,6 +28,7 @@ #define PTHREAD_INTERNAL 1 +#include #include #include #include @@ -41,7 +42,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -53,7 +56,11 @@ #define PTHREAD_SHIMS_VERSION 1 /* on arm, the callbacks function has two #ifdef arm ponters */ +#if defined(__arm__) +#define PTHREAD_CALLBACK_MEMBER map_is_1gb +#else #define PTHREAD_CALLBACK_MEMBER ml_get_max_cpus +#endif /* compile time asserts to check the length of structures in pthread_shims.h */ static_assert((sizeof(struct pthread_functions_s) - offsetof(struct pthread_functions_s, psynch_rw_yieldwrlock) - sizeof(void*)) == (sizeof(void*) * 100)); @@ -63,10 +70,6 @@ static_assert((sizeof(struct pthread_callbacks_s) - offsetof(struct pthread_call extern kern_return_t mach_port_deallocate(ipc_space_t, mach_port_name_t); extern kern_return_t semaphore_signal_internal_trap(mach_port_name_t); -/* Used for stackshot introspection */ -extern void kdp_pthread_find_owner(thread_t thread, struct stackshot_thread_waitinfo *waitinfo); -extern void* kdp_pthread_get_thread_kwq(thread_t thread); - #define PTHREAD_STRUCT_ACCESSOR(get, set, rettype, structtype, member) \ static rettype \ get(structtype x) { \ @@ -84,7 +87,10 @@ PTHREAD_STRUCT_ACCESSOR(proc_get_stack_addr_hint, proc_set_stack_addr_hint, user PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_offset, proc_set_dispatchqueue_offset, uint64_t, struct proc*, p_dispatchqueue_offset); PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_serialno_offset, proc_set_dispatchqueue_serialno_offset, uint64_t, struct proc*, p_dispatchqueue_serialno_offset); PTHREAD_STRUCT_ACCESSOR(proc_get_pthread_tsd_offset, proc_set_pthread_tsd_offset, uint32_t, struct proc *, p_pth_tsd_offset); +PTHREAD_STRUCT_ACCESSOR(proc_get_mach_thread_self_tsd_offset, proc_set_mach_thread_self_tsd_offset, uint64_t, struct proc *, p_mach_thread_self_offset); PTHREAD_STRUCT_ACCESSOR(proc_get_pthhash, proc_set_pthhash, void*, struct proc*, p_pthhash); +PTHREAD_STRUCT_ACCESSOR(proc_get_return_to_kernel_offset, proc_set_return_to_kernel_offset, uint64_t, struct proc*, p_return_to_kernel_offset); +PTHREAD_STRUCT_ACCESSOR(proc_get_user_stack, proc_set_user_stack, user_addr_t, struct proc*, user_stack); PTHREAD_STRUCT_ACCESSOR(uthread_get_threadlist, uthread_set_threadlist, void*, struct uthread*, uu_threadlist); PTHREAD_STRUCT_ACCESSOR(uthread_get_sigmask, uthread_set_sigmask, sigset_t, struct uthread*, uu_sigmask); @@ -184,6 +190,14 @@ qos_main_thread_active(void) return TRUE; } +#if defined(__arm__) +/* On iOS, the stack placement depends on the address space size */ +static uint32_t +map_is_1gb(vm_map_t map) +{ + return ((!vm_map_is_64bit(map)) && (get_map_max(map) == ml_get_max_offset(FALSE, MACHINE_MAX_OFFSET_MIN))); +} +#endif static int proc_usynch_get_requested_thread_qos(struct uthread *uth) { @@ -501,10 +515,17 @@ thread_qos_from_pthread_priority(unsigned long priority, unsigned long *flags) unsigned long pthread_priority_canonicalize(unsigned long priority, boolean_t propagation) { - if (pthread_functions->pthread_priority_canonicalize2) { - return pthread_functions->pthread_priority_canonicalize2(priority, propagation); + return pthread_functions->pthread_priority_canonicalize2(priority, propagation); +} + +boolean_t +workq_thread_has_been_unbound(thread_t th, int qos_class) +{ + if (pthread_functions->workq_thread_has_been_unbound) { + return pthread_functions->workq_thread_has_been_unbound(th, qos_class); } else { - return pthread_functions->pthread_priority_canonicalize(priority); + panic("pthread kext does not support workq_thread_has_been_unbound"); + return false; } } @@ -524,6 +545,28 @@ kdp_pthread_get_thread_kwq(thread_t thread) return NULL; } +static void +thread_will_park_or_terminate(thread_t thread) +{ + if (thread_owned_workloops_count(thread)) { + (void)kevent_exit_on_workloop_ownership_leak(thread); + } +} + +#if defined(__arm64__) +static unsigned __int128 +atomic_fetch_add_128_relaxed(_Atomic unsigned __int128 *ptr, unsigned __int128 value) +{ + return atomic_fetch_add_explicit(ptr, value, memory_order_relaxed); +} + +static unsigned __int128 +atomic_load_128_relaxed(_Atomic unsigned __int128 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_relaxed); +} +#endif + /* * The callbacks structure (defined in pthread_shims.h) contains a collection * of kernel functions that were not deemed sensible to expose as a KPI to all @@ -559,7 +602,9 @@ static const struct pthread_callbacks_s pthread_callbacks = { .vm_map_page_info = vm_map_page_info, .vm_map_switch = vm_map_switch, .thread_set_wq_state32 = thread_set_wq_state32, +#if !defined(__arm__) .thread_set_wq_state64 = thread_set_wq_state64, +#endif .uthread_get_threadlist = uthread_get_threadlist, .uthread_set_threadlist = uthread_set_threadlist, @@ -593,6 +638,8 @@ static const struct pthread_callbacks_s pthread_callbacks = { .zfree = zfree, .zinit = zinit, + .workloop_fulfill_threadreq = workloop_fulfill_threadreq, + .__pthread_testcancel = __pthread_testcancel, .mach_port_deallocate = mach_port_deallocate, @@ -604,6 +651,13 @@ static const struct pthread_callbacks_s pthread_callbacks = { .convert_thread_to_port = convert_thread_to_port, .ml_get_max_cpus = (void*)ml_get_max_cpus, +#if defined(__arm__) + .map_is_1gb = map_is_1gb, +#endif +#if defined(__arm64__) + .atomic_fetch_add_128_relaxed = atomic_fetch_add_128_relaxed, + .atomic_load_128_relaxed = atomic_load_128_relaxed, +#endif .proc_get_dispatchqueue_serialno_offset = proc_get_dispatchqueue_serialno_offset, .proc_set_dispatchqueue_serialno_offset = proc_set_dispatchqueue_serialno_offset, @@ -612,6 +666,8 @@ static const struct pthread_callbacks_s pthread_callbacks = { .proc_set_stack_addr_hint = proc_set_stack_addr_hint, .proc_get_pthread_tsd_offset = proc_get_pthread_tsd_offset, .proc_set_pthread_tsd_offset = proc_set_pthread_tsd_offset, + .proc_get_mach_thread_self_tsd_offset = proc_get_mach_thread_self_tsd_offset, + .proc_set_mach_thread_self_tsd_offset = proc_set_mach_thread_self_tsd_offset, .thread_set_tsd_base = thread_set_tsd_base, @@ -632,6 +688,15 @@ static const struct pthread_callbacks_s pthread_callbacks = { .proc_usynch_thread_qos_squash_override_for_resource = proc_usynch_thread_qos_squash_override_for_resource, .task_get_default_manager_qos = task_get_default_manager_qos, .thread_create_workq_waiting = thread_create_workq_waiting, + + .proc_get_return_to_kernel_offset = proc_get_return_to_kernel_offset, + .proc_set_return_to_kernel_offset = proc_set_return_to_kernel_offset, + .thread_will_park_or_terminate = thread_will_park_or_terminate, + + .qos_max_parallelism = qos_max_parallelism, + + .proc_get_user_stack = proc_get_user_stack, + .proc_set_user_stack = proc_set_user_stack, }; pthread_callbacks_t pthread_kern = &pthread_callbacks; diff --git a/bsd/kern/stackshot.c b/bsd/kern/stackshot.c index 5910e059f..cb3918bae 100644 --- a/bsd/kern/stackshot.c +++ b/bsd/kern/stackshot.c @@ -178,7 +178,8 @@ kern_stack_snapshot_with_reason(__unused char *reason) config.sc_pid = -1; config.sc_flags = (STACKSHOT_SAVE_LOADINFO | STACKSHOT_GET_GLOBAL_MEM_STATS | STACKSHOT_SAVE_IN_KERNEL_BUFFER | - STACKSHOT_KCDATA_FORMAT | STACKSHOT_ENABLE_UUID_FAULTING); + STACKSHOT_KCDATA_FORMAT | STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_THREAD_WAITINFO | + STACKSHOT_NO_IO_STATS); config.sc_delta_timestamp = 0; config.sc_out_buffer_addr = 0; config.sc_out_size_addr = 0; diff --git a/bsd/kern/subr_eventhandler.c b/bsd/kern/subr_eventhandler.c new file mode 100644 index 000000000..65c5975e3 --- /dev/null +++ b/bsd/kern/subr_eventhandler.c @@ -0,0 +1,359 @@ +/* + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/*- + * Copyright (c) 1999 Michael Smith + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int evh_debug = 0; + +MALLOC_DEFINE(M_EVENTHANDLER, "eventhandler", "Event handler records"); + +SYSCTL_NODE(_kern, OID_AUTO, eventhandler, CTLFLAG_RW | CTLFLAG_LOCKED, + 0, "Eventhandler"); +SYSCTL_INT(_kern_eventhandler, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED, + &evh_debug, 0, "Eventhandler debug mode"); + +struct eventhandler_entry_arg eventhandler_entry_dummy_arg = {{0}}; + +/* List of 'slow' lists */ +static struct eventhandler_lists_ctxt evthdlr_lists_ctxt_glb; +static lck_grp_attr_t *eventhandler_mutex_grp_attr; +static lck_grp_t *eventhandler_mutex_grp; +static lck_attr_t *eventhandler_mutex_attr; + +static lck_grp_attr_t *el_lock_grp_attr; +lck_grp_t *el_lock_grp; +lck_attr_t *el_lock_attr; + +struct eventhandler_entry_generic +{ + struct eventhandler_entry ee; + void (* func)(void); +}; + +static struct eventhandler_list *_eventhandler_find_list( + struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, const char *name); + +void +eventhandler_lists_ctxt_init(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt) +{ + VERIFY(evthdlr_lists_ctxt != NULL); + + TAILQ_INIT(&evthdlr_lists_ctxt->eventhandler_lists); + evthdlr_lists_ctxt->eventhandler_lists_initted = 1; + lck_mtx_init(&evthdlr_lists_ctxt->eventhandler_mutex, + eventhandler_mutex_grp, eventhandler_mutex_attr); +} + +/* + * Initialize the eventhandler mutex and list. + */ +void +eventhandler_init(void) +{ + eventhandler_mutex_grp_attr = lck_grp_attr_alloc_init(); + eventhandler_mutex_grp = lck_grp_alloc_init("eventhandler", + eventhandler_mutex_grp_attr); + eventhandler_mutex_attr = lck_attr_alloc_init(); + + el_lock_grp_attr = lck_grp_attr_alloc_init(); + el_lock_grp = lck_grp_alloc_init("eventhandler list", + el_lock_grp_attr); + el_lock_attr = lck_attr_alloc_init(); + + eventhandler_lists_ctxt_init(&evthdlr_lists_ctxt_glb); +} + +/* + * Insertion is O(n) due to the priority scan, but optimises to O(1) + * if all priorities are identical. + */ +static eventhandler_tag +eventhandler_register_internal( + struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, + struct eventhandler_list *list, + const char *name, eventhandler_tag epn) +{ + struct eventhandler_list *new_list; + struct eventhandler_entry *ep; + + if (evthdlr_lists_ctxt == NULL) + evthdlr_lists_ctxt = &evthdlr_lists_ctxt_glb; + + VERIFY(evthdlr_lists_ctxt->eventhandler_lists_initted); /* eventhandler registered too early */ + VERIFY(epn != NULL); /* cannot register NULL event */ + + /* lock the eventhandler lists */ + lck_mtx_lock(&evthdlr_lists_ctxt->eventhandler_mutex); + + /* Do we need to find/create the (slow) list? */ + if (list == NULL) { + /* look for a matching, existing list */ + list = _eventhandler_find_list(evthdlr_lists_ctxt, name); + + /* Do we need to create the list? */ + if (list == NULL) { + lck_mtx_unlock(&evthdlr_lists_ctxt->eventhandler_mutex); + + MALLOC(new_list, struct eventhandler_list *, + sizeof(struct eventhandler_list) + strlen(name) + 1, + M_EVENTHANDLER, M_WAITOK); + + /* If someone else created it already, then use that one. */ + lck_mtx_lock(&evthdlr_lists_ctxt->eventhandler_mutex); + list = _eventhandler_find_list(evthdlr_lists_ctxt, name); + if (list != NULL) { + FREE(new_list, M_EVENTHANDLER); + } else { + evhlog((LOG_DEBUG, "%s: creating list \"%s\"", __func__, name)); + list = new_list; + list->el_flags = 0; + list->el_runcount = 0; + bzero(&list->el_lock, sizeof(list->el_lock)); + list->el_name = (char *)list + sizeof(struct eventhandler_list); + strlcpy(list->el_name, name, strlen(name) + 1); + TAILQ_INSERT_HEAD(&evthdlr_lists_ctxt->eventhandler_lists, list, el_link); + } + } + } + if (!(list->el_flags & EHL_INITTED)) { + TAILQ_INIT(&list->el_entries); + EHL_LOCK_INIT(list); + list->el_flags |= EHL_INITTED; + } + lck_mtx_unlock(&evthdlr_lists_ctxt->eventhandler_mutex); + + KASSERT(epn->ee_priority != EHE_DEAD_PRIORITY, + ("%s: handler for %s registered with dead priority", __func__, name)); + + /* sort it into the list */ + evhlog((LOG_DEBUG, "%s: adding item %p (function %p to \"%s\"", __func__, VM_KERNEL_ADDRPERM(epn), + VM_KERNEL_UNSLIDE(((struct eventhandler_entry_generic *)epn)->func), name)); + EHL_LOCK(list); + TAILQ_FOREACH(ep, &list->el_entries, ee_link) { + if (ep->ee_priority != EHE_DEAD_PRIORITY && + epn->ee_priority < ep->ee_priority) { + TAILQ_INSERT_BEFORE(ep, epn, ee_link); + break; + } + } + if (ep == NULL) + TAILQ_INSERT_TAIL(&list->el_entries, epn, ee_link); + EHL_UNLOCK(list); + return(epn); +} + +eventhandler_tag +eventhandler_register(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, + struct eventhandler_list *list, const char *name, + void *func, struct eventhandler_entry_arg arg, int priority) +{ + struct eventhandler_entry_generic *eg; + + /* allocate an entry for this handler, populate it */ + MALLOC(eg, struct eventhandler_entry_generic *, + sizeof(struct eventhandler_entry_generic), + M_EVENTHANDLER, M_WAITOK | M_ZERO); + + eg->func = func; + eg->ee.ee_arg = arg; + eg->ee.ee_priority = priority; + + return (eventhandler_register_internal(evthdlr_lists_ctxt, list, name, &eg->ee)); +} + +void +eventhandler_deregister(struct eventhandler_list *list, eventhandler_tag tag) +{ + struct eventhandler_entry *ep = tag; + + EHL_LOCK_ASSERT(list, LCK_MTX_ASSERT_OWNED); + if (ep != NULL) { + /* remove just this entry */ + if (list->el_runcount == 0) { + evhlog((LOG_DEBUG, "%s: removing item %p from \"%s\"", __func__, VM_KERNEL_ADDRPERM(ep), + list->el_name)); + /* + * We may have purged the list because of certain events. + * Make sure that is not the case when a specific entry + * is being removed. + */ + if (!TAILQ_EMPTY(&list->el_entries)) + TAILQ_REMOVE(&list->el_entries, ep, ee_link); + FREE(ep, M_EVENTHANDLER); + } else { + evhlog((LOG_DEBUG, "%s: marking item %p from \"%s\" as dead", __func__, + VM_KERNEL_ADDRPERM(ep), list->el_name)); + ep->ee_priority = EHE_DEAD_PRIORITY; + } + } else { + /* remove entire list */ + if (list->el_runcount == 0) { + evhlog((LOG_DEBUG, "%s: removing all items from \"%s\"", __func__, + list->el_name)); + while (!TAILQ_EMPTY(&list->el_entries)) { + ep = TAILQ_FIRST(&list->el_entries); + TAILQ_REMOVE(&list->el_entries, ep, ee_link); + FREE(ep, M_EVENTHANDLER); + } + } else { + evhlog((LOG_DEBUG, "%s: marking all items from \"%s\" as dead", + __func__, list->el_name)); + TAILQ_FOREACH(ep, &list->el_entries, ee_link) + ep->ee_priority = EHE_DEAD_PRIORITY; + } + } + while (list->el_runcount > 0) + msleep((caddr_t)list, &list->el_lock, 0, "evhrm", 0); + EHL_UNLOCK(list); +} + +/* + * Internal version for use when eventhandler list is already locked. + */ +static struct eventhandler_list * +_eventhandler_find_list(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, + const char *name) +{ + struct eventhandler_list *list; + + VERIFY(evthdlr_lists_ctxt != NULL); + + LCK_MTX_ASSERT(&evthdlr_lists_ctxt->eventhandler_mutex, LCK_MTX_ASSERT_OWNED); + TAILQ_FOREACH(list, &evthdlr_lists_ctxt->eventhandler_lists, el_link) { + if (!strcmp(name, list->el_name)) + break; + } + return (list); +} + +/* + * Lookup a "slow" list by name. Returns with the list locked. + */ +struct eventhandler_list * +eventhandler_find_list(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, + const char *name) +{ + struct eventhandler_list *list; + + if (evthdlr_lists_ctxt == NULL) + evthdlr_lists_ctxt = &evthdlr_lists_ctxt_glb; + + if (!evthdlr_lists_ctxt->eventhandler_lists_initted) + return(NULL); + + /* scan looking for the requested list */ + lck_mtx_lock(&evthdlr_lists_ctxt->eventhandler_mutex); + list = _eventhandler_find_list(evthdlr_lists_ctxt, name); + if (list != NULL) + EHL_LOCK(list); + lck_mtx_unlock(&evthdlr_lists_ctxt->eventhandler_mutex); + + return(list); +} + +/* + * Prune "dead" entries from an eventhandler list. + */ +void +eventhandler_prune_list(struct eventhandler_list *list) +{ + struct eventhandler_entry *ep, *en; + int pruned = 0; + + evhlog((LOG_DEBUG, "%s: pruning list \"%s\"", __func__, list->el_name)); + EHL_LOCK_ASSERT(list, LCK_MTX_ASSERT_OWNED); + TAILQ_FOREACH_SAFE(ep, &list->el_entries, ee_link, en) { + if (ep->ee_priority == EHE_DEAD_PRIORITY) { + TAILQ_REMOVE(&list->el_entries, ep, ee_link); + FREE(ep, M_EVENTHANDLER); + pruned++; + } + } + if (pruned > 0) + wakeup(list); +} + +/* + * This should be called when last reference to an object + * is being released. + * The individual event type lists must be purged when the object + * becomes defunct. + */ +void +eventhandler_lists_ctxt_destroy(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt) +{ + struct eventhandler_list *list = NULL; + struct eventhandler_list *list_next = NULL; + + lck_mtx_lock(&evthdlr_lists_ctxt->eventhandler_mutex); + TAILQ_FOREACH_SAFE(list, &evthdlr_lists_ctxt->eventhandler_lists, + el_link, list_next) { + VERIFY(TAILQ_EMPTY(&list->el_entries)); + EHL_LOCK_DESTROY(list); + FREE(list, M_EVENTHANDLER); + } + lck_mtx_unlock(&evthdlr_lists_ctxt->eventhandler_mutex); + lck_mtx_destroy(&evthdlr_lists_ctxt->eventhandler_mutex, + eventhandler_mutex_grp); + return; +} diff --git a/bsd/kern/subr_log.c b/bsd/kern/subr_log.c index d0a78427b..5a748b5b5 100644 --- a/bsd/kern/subr_log.c +++ b/bsd/kern/subr_log.c @@ -690,7 +690,7 @@ oslogwakeup(void) static void oslog_streamwakeup_locked(void) { - lck_spin_assert(&oslog_stream_lock, LCK_ASSERT_OWNED); + LCK_SPIN_ASSERT(&oslog_stream_lock, LCK_ASSERT_OWNED); if (!oslog_stream_open) { return; } @@ -777,7 +777,7 @@ oslogioctl(__unused dev_t dev, u_long com, caddr_t data, __unused int flag, __un /* return number of characters immediately available */ case LOGBUFFERMAP: - kernel_firehose_buffer = kernel_firehose_addr; + kernel_firehose_buffer = (firehose_buffer_t)kernel_firehose_addr; ret = mach_make_memory_entry_64(kernel_map, &buffer_size, @@ -786,11 +786,12 @@ oslogioctl(__unused dev_t dev, u_long com, caddr_t data, __unused int flag, __un &mem_entry_ptr, MACH_PORT_NULL); if (ret == KERN_SUCCESS) { - ret = mach_vm_map(get_task_map(current_task()), + ret = mach_vm_map_kernel(get_task_map(current_task()), &user_addr, buffer_size, 0, /* mask */ VM_FLAGS_ANYWHERE, + VM_KERN_MEMORY_NONE, mem_entry_ptr, 0, /* offset */ FALSE, /* copy */ @@ -868,9 +869,9 @@ oslog_init(void) panic("Failed to allocate memory for firehose logging buffer"); } kernel_firehose_addr += PAGE_SIZE; - bzero(kernel_firehose_addr, size); + bzero((void *)kernel_firehose_addr, size); /* register buffer with firehose */ - kernel_firehose_addr = __firehose_buffer_create((size_t *) &size); + kernel_firehose_addr = (vm_offset_t)__firehose_buffer_create((size_t *) &size); kprintf("oslog_init completed\n"); } @@ -907,7 +908,7 @@ oslog_stream_find_free_buf_entry_locked(void) struct msgbuf *mbp; oslog_stream_buf_entry_t buf_entry = NULL; - lck_spin_assert(&oslog_stream_lock, LCK_ASSERT_OWNED); + LCK_SPIN_ASSERT(&oslog_stream_lock, LCK_ASSERT_OWNED); mbp = oslog_streambufp; @@ -946,7 +947,7 @@ oslog_stream_find_free_buf_entry_locked(void) void oslog_streamwrite_metadata_locked(oslog_stream_buf_entry_t m_entry) { - lck_spin_assert(&oslog_stream_lock, LCK_ASSERT_OWNED); + LCK_SPIN_ASSERT(&oslog_stream_lock, LCK_ASSERT_OWNED); STAILQ_INSERT_TAIL(&oslog_stream_buf_head, m_entry, buf_entries); return; @@ -956,7 +957,7 @@ static void oslog_streamwrite_append_bytes(const char *buffer, int buflen) { struct msgbuf *mbp; - lck_spin_assert(&oslog_stream_lock, LCK_ASSERT_OWNED); + LCK_SPIN_ASSERT(&oslog_stream_lock, LCK_ASSERT_OWNED); mbp = oslog_streambufp; // Check if we have enough space in the stream buffer to write the data @@ -995,7 +996,7 @@ oslog_streamwrite_locked(firehose_tracepoint_id_u ftid, uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data); int ft_length = ft_size + publen; - lck_spin_assert(&oslog_stream_lock, LCK_ASSERT_OWNED); + LCK_SPIN_ASSERT(&oslog_stream_lock, LCK_ASSERT_OWNED); mbp = oslog_streambufp; if (ft_length > mbp->msg_size) { diff --git a/bsd/kern/subr_prf.c b/bsd/kern/subr_prf.c index ebca1b666..f254816fa 100644 --- a/bsd/kern/subr_prf.c +++ b/bsd/kern/subr_prf.c @@ -98,7 +98,6 @@ #include #include /* for cpu_number() */ -#include #include #include @@ -117,10 +116,10 @@ struct snprintf_arg { /* * In case console is off, - * panicstr contains argument to last + * debugger_panic_str contains argument to last * call to panic. */ -extern const char *panicstr; +extern const char *debugger_panic_str; extern void cnputc(char); /* standard console putc */ void (*v_putc)(char) = cnputc; /* routine to putc on virtual console */ @@ -417,7 +416,7 @@ putchar(int c, void *arg) struct putchar_args *pca = arg; char **sp = (char**) pca->tty; - if (panicstr) + if (debugger_panic_str) constty = 0; if ((pca->flags & TOCONS) && pca->tty == NULL && constty) { pca->tty = constty; @@ -449,6 +448,7 @@ vprintf_log_locked(const char *fmt, va_list ap) return 0; } +#if !CONFIG_EMBEDDED /* * Scaled down version of vsprintf(3). @@ -471,6 +471,7 @@ vsprintf(char *buf, const char *cfmt, va_list ap) } return 0; } +#endif /* !CONFIG_EMBEDDED */ /* * Scaled down version of snprintf(3). diff --git a/bsd/kern/subr_prof.c b/bsd/kern/subr_prof.c index 80b6edc27..20a0f5be3 100644 --- a/bsd/kern/subr_prof.c +++ b/bsd/kern/subr_prof.c @@ -70,7 +70,6 @@ #include #include #include -#include #include #include diff --git a/bsd/kern/sys_coalition.c b/bsd/kern/sys_coalition.c index e35a8a878..85100962e 100644 --- a/bsd/kern/sys_coalition.c +++ b/bsd/kern/sys_coalition.c @@ -1,4 +1,5 @@ #include +#include #include #include @@ -31,6 +32,7 @@ coalition_create_syscall(user_addr_t cidp, uint32_t flags) uint64_t cid; coalition_t coal; int type = COALITION_CREATE_FLAGS_GET_TYPE(flags); + int role = COALITION_CREATE_FLAGS_GET_ROLE(flags); boolean_t privileged = !!(flags & COALITION_CREATE_FLAGS_PRIVILEGED); if ((flags & (~COALITION_CREATE_FLAGS_MASK)) != 0) @@ -38,7 +40,7 @@ coalition_create_syscall(user_addr_t cidp, uint32_t flags) if (type < 0 || type > COALITION_TYPE_MAX) return EINVAL; - kr = coalition_create_internal(type, privileged, &coal); + kr = coalition_create_internal(type, role, privileged, &coal); if (kr != KERN_SUCCESS) { /* for now, the only kr is KERN_RESOURCE_SHORTAGE */ error = ENOMEM; @@ -118,7 +120,7 @@ coalition_request_terminate_syscall(user_addr_t cidp, uint32_t flags) * Request the kernel to deallocate the coalition identified by ID, which * must be both terminated and empty. This balances the reference taken * in coalition_create. - * The memory containig the coalition object may not be freed just yet, if + * The memory containing the coalition object may not be freed just yet, if * other kernel operations still hold references to it. * * Returns: @@ -232,6 +234,26 @@ coalition_info_resource_usage(coalition_t coal, user_addr_t buffer, user_size_t return copyout(&cru, buffer, MIN(bufsize, sizeof(cru))); } +#define coalition_info_set_name_internal(...) 0 + +static int +coalition_info_efficiency(coalition_t coal, user_addr_t buffer, user_size_t bufsize) +{ + int error = 0; + if (coalition_type(coal) != COALITION_TYPE_JETSAM) + return EINVAL; + uint64_t flags = 0; + error = copyin(buffer, &flags, MIN(bufsize, sizeof(flags))); + if (error) + return error; + if ((flags & COALITION_EFFICIENCY_VALID_FLAGS) == 0) + return EINVAL; + if (flags & COALITION_FLAGS_EFFICIENT) { + coalition_set_efficient(coal); + } + return error; +} + int coalition_info(proc_t p, struct coalition_info_args *uap, __unused int32_t *retval) { user_addr_t cidp = uap->cid; @@ -271,6 +293,12 @@ int coalition_info(proc_t p, struct coalition_info_args *uap, __unused int32_t * case COALITION_INFO_RESOURCE_USAGE: error = coalition_info_resource_usage(coal, buffer, bufsize); break; + case COALITION_INFO_SET_NAME: + error = coalition_info_set_name_internal(coal, buffer, bufsize); + break; + case COALITION_INFO_SET_EFFICIENCY: + error = coalition_info_efficiency(coal, buffer, bufsize); + break; default: error = EINVAL; } @@ -280,7 +308,7 @@ bad: return error; } -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG static int sysctl_coalition_get_ids SYSCTL_HANDLER_ARGS { #pragma unused(oidp, arg1, arg2) diff --git a/bsd/kern/sys_generic.c b/bsd/kern/sys_generic.c index 2d1431763..9cfbe7e91 100644 --- a/bsd/kern/sys_generic.c +++ b/bsd/kern/sys_generic.c @@ -97,6 +97,7 @@ #include #include +#include #include #include #include @@ -140,6 +141,10 @@ #include #include +#if CONFIG_MACF +#include +#endif + /* XXX should be in a header file somewhere */ void evsofree(struct socket *); void evpipefree(struct pipe *); @@ -159,7 +164,6 @@ __private_extern__ int dofilewrite(vfs_context_t ctx, struct fileproc *fp, __private_extern__ int preparefileread(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_vnode); __private_extern__ void donefileread(struct proc *p, struct fileproc *fp_ret, int fd); - /* Conflict wait queue for when selects collide (opaque type) */ struct waitq select_conflict_queue; @@ -770,7 +774,7 @@ ioctl(struct proc *p, struct ioctl_args *uap, __unused int32_t *retval) boolean_t is64bit = FALSE; int tmp = 0; #define STK_PARAMS 128 - char stkbuf[STK_PARAMS]; + char stkbuf[STK_PARAMS] = {}; int fd = uap->fd; u_long com = uap->com; struct vfs_context context = *vfs_context_current(); @@ -3111,6 +3115,14 @@ gethostuuid(struct proc *p, struct gethostuuid_args *uap, __unused int32_t *retv __darwin_uuid_t uuid_kern; /* for IOKit call */ if (!uap->spi) { +#if CONFIG_EMBEDDED +#if CONFIG_MACF + if ((error = mac_system_check_info(kauth_cred_get(), "hw.uuid")) != 0) { + /* EPERM invokes userspace upcall if present */ + return (error); + } +#endif +#endif } /* Convert the 32/64 bit timespec into a mach_timespec_t */ @@ -3289,7 +3301,7 @@ telemetry(__unused struct proc *p, struct telemetry_args *args, __unused int32_t return (error); } -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG #if CONFIG_WAITQ_DEBUG static uint64_t g_wqset_num = 0; struct g_wqset { @@ -3669,4 +3681,6 @@ SYSCTL_PROC(_kern, OID_AUTO, wqset_clear_preposts, CTLTYPE_QUAD | CTLFLAG_RW | C 0, 0, sysctl_wqset_clear_preposts, "Q", "clear preposts on given waitq set"); #endif /* CONFIG_WAITQ_DEBUG */ -#endif /* defined(DEVELOPMENT) || defined(DEBUG) */ +#endif /* DEVELOPMENT || DEBUG */ + + diff --git a/bsd/kern/sys_pipe.c b/bsd/kern/sys_pipe.c index f6adf702a..9e8b346e9 100644 --- a/bsd/kern/sys_pipe.c +++ b/bsd/kern/sys_pipe.c @@ -146,6 +146,11 @@ #include #include #include +#include + +#if CONFIG_MACF +#include +#endif #define f_flag f_fglob->fg_flag #define f_msgcount f_fglob->fg_msgcount @@ -158,14 +163,14 @@ * interfaces to the outside world exported through file operations */ static int pipe_read(struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int pipe_write(struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int pipe_close(struct fileglob *fg, vfs_context_t ctx); static int pipe_select(struct fileproc *fp, int which, void * wql, vfs_context_t ctx); static int pipe_kqfilter(struct fileproc *fp, struct knote *kn, - vfs_context_t ctx); + struct kevent_internal_s *kev, vfs_context_t ctx); static int pipe_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, vfs_context_t ctx); static int pipe_drain(struct fileproc *fp,vfs_context_t ctx); @@ -191,7 +196,7 @@ static int filt_pipewrite(struct knote *kn, long hint); static int filt_pipewritetouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_pipewriteprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops pipe_rfiltops = { +SECURITY_READ_ONLY_EARLY(struct filterops) pipe_rfiltops = { .f_isfd = 1, .f_detach = filt_pipedetach, .f_event = filt_piperead, @@ -199,7 +204,7 @@ struct filterops pipe_rfiltops = { .f_process = filt_pipereadprocess, }; -struct filterops pipe_wfiltops = { +SECURITY_READ_ONLY_EARLY(struct filterops) pipe_wfiltops = { .f_isfd = 1, .f_detach = filt_pipedetach, .f_event = filt_pipewrite, @@ -299,6 +304,7 @@ pipeinit(void) } +#ifndef CONFIG_EMBEDDED /* Bitmap for things to touch in pipe_touch() */ #define PIPE_ATIME 0x00000001 /* time of last access */ #define PIPE_MTIME 0x00000002 /* time of last modification */ @@ -307,25 +313,26 @@ pipeinit(void) static void pipe_touch(struct pipe *tpipe, int touch) { - struct timeval now; + struct timespec now; - microtime(&now); + nanotime(&now); if (touch & PIPE_ATIME) { tpipe->st_atimespec.tv_sec = now.tv_sec; - tpipe->st_atimespec.tv_nsec = now.tv_usec * 1000; + tpipe->st_atimespec.tv_nsec = now.tv_nsec; } if (touch & PIPE_MTIME) { tpipe->st_mtimespec.tv_sec = now.tv_sec; - tpipe->st_mtimespec.tv_nsec = now.tv_usec * 1000; + tpipe->st_mtimespec.tv_nsec = now.tv_nsec; } if (touch & PIPE_CTIME) { tpipe->st_ctimespec.tv_sec = now.tv_sec; - tpipe->st_ctimespec.tv_nsec = now.tv_usec * 1000; + tpipe->st_ctimespec.tv_nsec = now.tv_nsec; } } +#endif static const unsigned int pipesize_blocks[] = {512,1024,2048,4096, 4096 * 2, PIPE_SIZE , PIPE_SIZE * 4 }; @@ -658,8 +665,10 @@ pipe_create(struct pipe **cpipep) */ bzero(cpipe, sizeof *cpipe); +#ifndef CONFIG_EMBEDDED /* Initial times are all the time of creation of the pipe */ pipe_touch(cpipe, PIPE_ATIME | PIPE_MTIME | PIPE_CTIME); +#endif return (0); } @@ -860,8 +869,10 @@ unlocked_error: if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) > 0) pipeselwakeup(rpipe, rpipe->pipe_peer); +#ifndef CONFIG_EMBEDDED /* update last read time */ pipe_touch(rpipe, PIPE_ATIME); +#endif PIPE_UNLOCK(rpipe); @@ -1117,9 +1128,11 @@ pipe_write(struct fileproc *fp, struct uio *uio, __unused int flags, pipeselwakeup(wpipe, wpipe); } +#ifndef CONFIG_EMBEDDED /* Update modification, status change (# of bytes in pipe) times */ pipe_touch(rpipe, PIPE_MTIME | PIPE_CTIME); pipe_touch(wpipe, PIPE_MTIME | PIPE_CTIME); +#endif PIPE_UNLOCK(rpipe); return (error); @@ -1536,7 +1549,8 @@ filt_pipewriteprocess(struct knote *kn, struct filt_process_s *data, struct keve /*ARGSUSED*/ static int -pipe_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused vfs_context_t ctx) +pipe_kqfilter(__unused struct fileproc *fp, struct knote *kn, + __unused struct kevent_internal_s *kev, __unused vfs_context_t ctx) { struct pipe *cpipe = (struct pipe *)kn->kn_fp->f_data; int res; @@ -1623,7 +1637,7 @@ fill_pipeinfo(struct pipe * cpipe, struct pipe_info * pinfo) #if CONFIG_MACF int error; #endif - struct timeval now; + struct timespec now; struct vinfo_stat * ub; int pipe_size = 0; int pipe_count; @@ -1676,15 +1690,15 @@ fill_pipeinfo(struct pipe * cpipe, struct pipe_info * pinfo) ub->vst_uid = kauth_getuid(); ub->vst_gid = kauth_getgid(); - microtime(&now); + nanotime(&now); ub->vst_atime = now.tv_sec; - ub->vst_atimensec = now.tv_usec * 1000; + ub->vst_atimensec = now.tv_nsec; ub->vst_mtime = now.tv_sec; - ub->vst_mtimensec = now.tv_usec * 1000; + ub->vst_mtimensec = now.tv_nsec; ub->vst_ctime = now.tv_sec; - ub->vst_ctimensec = now.tv_usec * 1000; + ub->vst_ctimensec = now.tv_nsec; /* * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen, st_uid, st_gid. diff --git a/bsd/kern/sys_socket.c b/bsd/kern/sys_socket.c index 7b9e78b6f..cc4d778bd 100644 --- a/bsd/kern/sys_socket.c +++ b/bsd/kern/sys_socket.c @@ -189,6 +189,12 @@ soioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p) int error = 0; int int_arg; +#if CONFIG_MACF_SOCKET_SUBSET + error = mac_socket_check_ioctl(kauth_cred_get(), so, cmd); + if (error) + return (error); +#endif + socket_lock(so, 1); /* call the socket filter's ioctl handler anything but ours */ @@ -374,7 +380,7 @@ soo_stat(struct socket *so, void *ub, int isstat64) /* warning avoidance ; protected by isstat64 */ struct stat64 *sb64 = (struct stat64 *)0; -#if CONFIG_MACF_SOCKET +#if CONFIG_MACF_SOCKET_SUBSET ret = mac_socket_check_stat(kauth_cred_get(), so); if (ret) return (ret); diff --git a/bsd/kern/sys_ulock.c b/bsd/kern/sys_ulock.c index 4da93446c..c8bf0da86 100644 --- a/bsd/kern/sys_ulock.c +++ b/bsd/kern/sys_ulock.c @@ -131,7 +131,6 @@ typedef struct ull { static const bool ull_debug = false; extern void ulock_initialize(void); -extern void kdp_ulock_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); #define ULL_MUST_EXIST 0x0001 static ull_t *ull_get(ulk_t *, uint32_t); @@ -141,7 +140,6 @@ static thread_t ull_promote_owner_locked(ull_t* ull, thread_t thread); #if DEVELOPMENT || DEBUG static int ull_simulate_copyin_fault = 0; -static int ull_panic_on_corruption = 0; static void ull_dump(ull_t *ull) @@ -210,13 +208,6 @@ ulock_initialize(void) 0, "ulocks"); zone_change(ull_zone, Z_NOENCRYPT, TRUE); - -#if DEVELOPMENT || DEBUG - if (!PE_parse_boot_argn("ulock_panic_on_corruption", - &ull_panic_on_corruption, sizeof(ull_panic_on_corruption))) { - ull_panic_on_corruption = 0; - } -#endif } #if DEVELOPMENT || DEBUG @@ -282,7 +273,7 @@ ull_free(ull_t *ull) { assert(ull->ull_owner == THREAD_NULL); - lck_mtx_assert(&ull->ull_lock, LCK_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&ull->ull_lock, LCK_ASSERT_NOTOWNED); lck_mtx_destroy(&ull->ull_lock, ull_lck_grp); @@ -501,17 +492,6 @@ ulock_wait(struct proc *p, struct ulock_wait_args *args, int32_t *retval) /* HACK: don't bail on MACH_PORT_DEAD, to avoid blowing up the no-tsd pthread lock */ if (owner_name != MACH_PORT_DEAD && owner_thread == THREAD_NULL) { -#if DEBUG || DEVELOPMENT - if (ull_panic_on_corruption) { - if (flags & ULF_NO_ERRNO) { - // ULF_NO_ERRNO is used by libplatform ulocks, but not libdispatch ones. - // Don't panic on libdispatch ulock corruptions; the userspace likely - // mismanaged a dispatch queue. - panic("ulock_wait: ulock is corrupted; value=0x%x, ull=%p", - (uint32_t)(args->value), ull); - } - } -#endif /* * Translation failed - even though the lock value is up to date, * whatever was stored in the lock wasn't actually a thread port. @@ -733,7 +713,7 @@ ulock_wake(struct proc *p, struct ulock_wake_args *args, __unused int32_t *retva } else { /* * TODO: WAITQ_SELECT_MAX_PRI forces a linear scan of the (hashed) global waitq. - * Move to a ulock-private, priority sorted waitq to avoid that. + * Move to a ulock-private, priority sorted waitq (i.e. SYNC_POLICY_FIXED_PRIORITY) to avoid that. * * TODO: 'owner is not current_thread (or null)' likely means we can avoid this wakeup * diff --git a/bsd/kern/sys_work_interval.c b/bsd/kern/sys_work_interval.c index 53d4a2930..561a1a0bb 100644 --- a/bsd/kern/sys_work_interval.c +++ b/bsd/kern/sys_work_interval.c @@ -33,44 +33,70 @@ #include #include #include -#include +#include #include int -work_interval_ctl(__unused proc_t p, struct work_interval_ctl_args *uap, __unused int32_t *retval) +work_interval_ctl(__unused proc_t p, struct work_interval_ctl_args *uap, + __unused int32_t *retval) { - uint32_t operation = uap->operation; - int error = 0; - kern_return_t kret = KERN_SUCCESS; - uint64_t work_interval_id; - struct work_interval_notification notification; + uint32_t operation = uap->operation; + int error = 0; + kern_return_t kret = KERN_SUCCESS; + struct work_interval_notification notification; + + /* Two different structs, because headers are complicated */ + struct work_interval_create_params create_params; + struct kern_work_interval_create_args create_args; switch (operation) { case WORK_INTERVAL_OPERATION_CREATE: - if (uap->arg == USER_ADDR_NULL || uap->work_interval_id != 0) { + return ENOTSUP; + case WORK_INTERVAL_OPERATION_CREATE2: + if (uap->arg == USER_ADDR_NULL || uap->work_interval_id != 0) + return EINVAL; + if (uap->len < sizeof(create_params)) return EINVAL; - } - if (uap->len < sizeof(work_interval_id)) { - return ERANGE; - } /* * Privilege check performed up-front, and then the work * ID is allocated for use by the thread */ - error = priv_check_cred(kauth_cred_get(), PRIV_WORK_INTERVAL, 0); - if (error) { - return (error); - } + if ((error = priv_check_cred(kauth_cred_get(), PRIV_WORK_INTERVAL, 0))) + return error; - kret = thread_policy_create_work_interval(current_thread(), - &work_interval_id); - if (kret == KERN_SUCCESS) { - error = copyout(&work_interval_id, uap->arg, sizeof(work_interval_id)); - } else { - error = EINVAL; - } + if ((error = copyin(uap->arg, &create_params, sizeof(create_params)))) + return error; + + create_args = (struct kern_work_interval_create_args) { + .wica_id = create_params.wicp_id, + .wica_port = create_params.wicp_port, + .wica_create_flags = create_params.wicp_create_flags, + }; + + kret = kern_work_interval_create(current_thread(), &create_args); + + /* thread already has a work interval */ + if (kret == KERN_FAILURE) + return EALREADY; + + /* port copyout failed */ + if (kret == KERN_RESOURCE_SHORTAGE) + return ENOMEM; + + /* some other failure */ + if (kret != KERN_SUCCESS) + return EINVAL; + + create_params = (struct work_interval_create_params) { + .wicp_id = create_args.wica_id, + .wicp_port = create_args.wica_port, + .wicp_create_flags = create_args.wica_create_flags, + }; + + if ((error = copyout(&create_params, uap->arg, sizeof(create_params)))) + return error; break; case WORK_INTERVAL_OPERATION_DESTROY: @@ -83,48 +109,61 @@ work_interval_ctl(__unused proc_t p, struct work_interval_ctl_args *uap, __unuse * operation would have allocated a work interval ID for the current * thread, which the scheduler will validate. */ - kret = thread_policy_destroy_work_interval(current_thread(), - uap->work_interval_id); - if (kret != KERN_SUCCESS) { - error = EINVAL; - } + kret = kern_work_interval_destroy(current_thread(), uap->work_interval_id); + if (kret != KERN_SUCCESS) + return EINVAL; break; case WORK_INTERVAL_OPERATION_NOTIFY: - if (uap->arg == USER_ADDR_NULL || uap->work_interval_id == 0) { + if (uap->arg == USER_ADDR_NULL || uap->work_interval_id == 0) return EINVAL; - } - if (uap->len < sizeof(notification)) { + + if (uap->len < sizeof(notification)) return EINVAL; - } /* * No privilege check, we assume a previous WORK_INTERVAL_OPERATION_CREATE * operation would have allocated a work interval ID for the current * thread, which the scheduler will validate. */ - error = copyin(uap->arg, ¬ification, sizeof(notification)); - if (error) { - break; - } + if ((error = copyin(uap->arg, ¬ification, sizeof(notification)))) + return error; - kret = sched_work_interval_notify(current_thread(), - uap->work_interval_id, - notification.start, - notification.finish, - notification.deadline, - notification.next_start, - notification.flags); - if (kret != KERN_SUCCESS) { - error = EINVAL; - break; + struct kern_work_interval_args kwi_args = { + .work_interval_id = uap->work_interval_id, + .start = notification.start, + .finish = notification.finish, + .deadline = notification.deadline, + .next_start = notification.next_start, + .notify_flags = notification.notify_flags, + .create_flags = notification.create_flags, + }; + + kret = kern_work_interval_notify(current_thread(), &kwi_args); + if (kret != KERN_SUCCESS) + return EINVAL; + + break; + case WORK_INTERVAL_OPERATION_JOIN: + if (uap->arg != USER_ADDR_NULL) { + return EINVAL; } + /* + * No privilege check, because the work interval port + * is a capability. + */ + kret = kern_work_interval_join(current_thread(), + (mach_port_name_t)uap->work_interval_id); + if (kret != KERN_SUCCESS) + return EINVAL; + break; + default: - error = ENOTSUP; - break; + return ENOTSUP; } return (error); } + diff --git a/bsd/kern/syscalls.master b/bsd/kern/syscalls.master index 8b3b9b169..f18cc9121 100644 --- a/bsd/kern/syscalls.master +++ b/bsd/kern/syscalls.master @@ -262,16 +262,16 @@ 174 AUE_NULL ALL { int nosys(void); } { old getdents } 175 AUE_NULL ALL { int nosys(void); } { old gc_control } 176 AUE_NULL ALL { int nosys(void); } { old add_profil } -177 AUE_KDEBUGTRACE ALL { int kdebug_typefilter(void** addr, size_t* size) NO_SYSCALL_STUB; } -178 AUE_KDEBUGTRACE ALL { uint64_t kdebug_trace_string(uint32_t debugid, uint64_t str_id, const char *str) NO_SYSCALL_STUB; } -179 AUE_KDEBUGTRACE ALL { int kdebug_trace64(uint32_t code, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4) NO_SYSCALL_STUB; } -180 AUE_KDEBUGTRACE ALL { int kdebug_trace(uint32_t code, u_long arg1, u_long arg2, u_long arg3, u_long arg4) NO_SYSCALL_STUB; } +177 AUE_NULL ALL { int kdebug_typefilter(void** addr, size_t* size) NO_SYSCALL_STUB; } +178 AUE_NULL ALL { uint64_t kdebug_trace_string(uint32_t debugid, uint64_t str_id, const char *str) NO_SYSCALL_STUB; } +179 AUE_NULL ALL { int kdebug_trace64(uint32_t code, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4) NO_SYSCALL_STUB; } +180 AUE_NULL ALL { int kdebug_trace(uint32_t code, u_long arg1, u_long arg2, u_long arg3, u_long arg4) NO_SYSCALL_STUB; } 181 AUE_SETGID ALL { int setgid(gid_t gid); } 182 AUE_SETEGID ALL { int setegid(gid_t egid); } 183 AUE_SETEUID ALL { int seteuid(uid_t euid); } 184 AUE_SIGRETURN ALL { int sigreturn(struct ucontext *uctx, int infostyle) NO_SYSCALL_STUB; } 185 AUE_NULL ALL { int enosys(void); } { old chud } -186 AUE_NULL ALL { int nosys(void); } +186 AUE_NULL ALL { int thread_selfcounts(int type, user_addr_t buf, user_size_t nbytes); } 187 AUE_FDATASYNC ALL { int fdatasync(int fd); } 188 AUE_STAT ALL { int stat(user_addr_t path, user_addr_t ub); } 189 AUE_FSTAT ALL { int fstat(int fd, user_addr_t ub); } @@ -565,7 +565,7 @@ 372 AUE_NULL ALL { uint64_t thread_selfid (void) NO_SYSCALL_STUB; } 373 AUE_LEDGER ALL { int ledger(int cmd, caddr_t arg1, caddr_t arg2, caddr_t arg3); } 374 AUE_NULL ALL { int kevent_qos(int fd, const struct kevent_qos_s *changelist, int nchanges, struct kevent_qos_s *eventlist, int nevents, void *data_out, size_t *data_available, unsigned int flags); } -375 AUE_NULL ALL { int nosys(void); } +375 AUE_NULL ALL { int kevent_id(uint64_t id, const struct kevent_qos_s *changelist, int nchanges, struct kevent_qos_s *eventlist, int nevents, void *data_out, size_t *data_available, unsigned int flags); } 376 AUE_NULL ALL { int nosys(void); } 377 AUE_NULL ALL { int nosys(void); } 378 AUE_NULL ALL { int nosys(void); } @@ -666,8 +666,16 @@ 432 AUE_NULL ALL { int audit_session_port(au_asid_t asid, user_addr_t portnamep); } 433 AUE_NULL ALL { int pid_suspend(int pid); } 434 AUE_NULL ALL { int pid_resume(int pid); } +#if CONFIG_EMBEDDED +435 AUE_NULL ALL { int pid_hibernate(int pid); } +#else 435 AUE_NULL ALL { int nosys(void); } +#endif +#if SOCKETS +436 AUE_NULL ALL { int pid_shutdown_sockets(int pid, int level); } +#else 436 AUE_NULL ALL { int nosys(void); } +#endif 437 AUE_NULL ALL { int nosys(void); } { old shared_region_slide_np } 438 AUE_NULL ALL { int shared_region_map_and_slide_np(int fd, uint32_t count, const struct shared_file_mapping_np *mappings, uint32_t slide, uint64_t* slide_start, uint32_t slide_size) NO_SYSCALL_STUB; } 439 AUE_NULL ALL { int kas_info(int selector, void *value, size_t *size); } @@ -676,7 +684,7 @@ #else 440 AUE_NULL ALL { int nosys(void); } #endif -441 AUE_OPEN_RWTC ALL { int guarded_open_np(const char *path, const guardid_t *guard, u_int guardflags, int flags, int mode) NO_SYSCALL_STUB; } +441 AUE_OPEN_RWTC ALL { int guarded_open_np(user_addr_t path, const guardid_t *guard, u_int guardflags, int flags, int mode) NO_SYSCALL_STUB; } 442 AUE_CLOSE ALL { int guarded_close_np(int fd, const guardid_t *guard); } 443 AUE_KQUEUE ALL { int guarded_kqueue_np(const guardid_t *guard, u_int guardflags); } 444 AUE_NULL ALL { int change_fdguard_np(int fd, const guardid_t *guard, u_int guardflags, const guardid_t *nguard, u_int nguardflags, int *fdflagsp); } @@ -752,7 +760,7 @@ #else 483 AUE_NULL ALL { int enosys(void); } #endif /* CSR */ -484 AUE_NULL ALL { int guarded_open_dprotected_np(const char *path, const guardid_t *guard, u_int guardflags, int flags, int dpclass, int dpflags, int mode) NO_SYSCALL_STUB; } +484 AUE_NULL ALL { int guarded_open_dprotected_np(user_addr_t path, const guardid_t *guard, u_int guardflags, int flags, int dpclass, int dpflags, int mode) NO_SYSCALL_STUB; } 485 AUE_NULL ALL { user_ssize_t guarded_write_np(int fd, const guardid_t *guard, user_addr_t cbuf, user_size_t nbyte); } 486 AUE_PWRITE ALL { user_ssize_t guarded_pwrite_np(int fd, const guardid_t *guard, user_addr_t buf, user_size_t nbyte, off_t offset); } 487 AUE_WRITEV ALL { user_ssize_t guarded_writev_np(int fd, const guardid_t *guard, struct iovec *iovp, int iovcnt); } @@ -815,3 +823,16 @@ 519 AUE_NULL ALL { int enosys(void); } 520 AUE_KILL ALL { int terminate_with_payload(int pid, uint32_t reason_namespace, uint64_t reason_code, void *payload, uint32_t payload_size, const char *reason_string, uint64_t reason_flags) NO_SYSCALL_STUB; } 521 AUE_EXIT ALL { void abort_with_payload(uint32_t reason_namespace, uint64_t reason_code, void *payload, uint32_t payload_size, const char *reason_string, uint64_t reason_flags) NO_SYSCALL_STUB; } +#if NECP +522 AUE_NECP ALL { int necp_session_open(int flags); } } +523 AUE_NECP ALL { int necp_session_action(int necp_fd, uint32_t action, uint8_t *in_buffer, size_t in_buffer_length, uint8_t *out_buffer, size_t out_buffer_length); } +#else /* NECP */ +522 AUE_NULL ALL { int enosys(void); } +523 AUE_NULL ALL { int enosys(void); } +#endif /* NECP */ +524 AUE_SETATTRLISTAT ALL { int setattrlistat(int fd, const char *path, struct attrlist *alist, void *attributeBuffer, size_t bufferSize, uint32_t options); } +525 AUE_NET ALL { int net_qos_guideline(struct net_qos_param *param, uint32_t param_len); } +526 AUE_FMOUNT ALL { int fmount(const char *type, int fd, int flags, void *data); } +527 AUE_NULL ALL { int ntp_adjtime(struct timex *tp); } +528 AUE_NULL ALL { int ntp_gettime(struct ntptimeval *ntvp); } +529 AUE_NULL ALL { int os_fault_with_payload(uint32_t reason_namespace, uint64_t reason_code, void *payload, uint32_t payload_size, const char *reason_string, uint64_t reason_flags); } diff --git a/bsd/kern/sysv_msg.c b/bsd/kern/sysv_msg.c index 56c670424..fad2cfbf7 100644 --- a/bsd/kern/sysv_msg.c +++ b/bsd/kern/sysv_msg.c @@ -67,6 +67,10 @@ #include #include +#if CONFIG_MACF +#include +#endif + #if SYSV_MSG static int msginit(void *); @@ -268,7 +272,7 @@ msginit(__unused void *dummy) } MALLOC(msqids, struct msqid_kernel *, - sizeof(struct user_msqid_ds) * msginfo.msgmni, + sizeof(struct msqid_kernel) * msginfo.msgmni, M_SHM, M_WAITOK); if (msqids == NULL) { printf("msginit: can't allocate msqids"); diff --git a/bsd/kern/sysv_shm.c b/bsd/kern/sysv_shm.c index 4a47b7ac2..a962e9ab1 100644 --- a/bsd/kern/sysv_shm.c +++ b/bsd/kern/sysv_shm.c @@ -483,11 +483,12 @@ shmat(struct proc *p, struct shmat_args *uap, user_addr_t *retval) mapped_size = 0; /* first reserve enough space... */ - rv = mach_vm_map(current_map(), + rv = mach_vm_map_kernel(current_map(), &attach_va, map_size, 0, vm_flags, + VM_KERN_MEMORY_NONE, IPC_PORT_NULL, 0, FALSE, @@ -511,6 +512,8 @@ shmat(struct proc *p, struct shmat_args *uap, user_addr_t *retval) shm_handle->shm_handle_size, /* segment size */ (mach_vm_offset_t)0, /* alignment mask */ VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, shm_handle->shm_object, (mach_vm_offset_t)0, FALSE, @@ -622,7 +625,7 @@ shmctl(__unused struct proc *p, struct shmctl_args *uap, int32_t *retval) error = copyout(&shmid_ds, uap->buf, sizeof(shmid_ds)); } else { - struct user32_shmid_ds shmid_ds32; + struct user32_shmid_ds shmid_ds32 = {}; shmid_ds_64to32(&shmseg->u, &shmid_ds32); /* Clear kernel reserved pointer before copying to user space */ @@ -1166,7 +1169,7 @@ IPCS_shm_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, struct user32_IPCS_command u32; struct user_IPCS_command u64; } ipcs; - struct user32_shmid_ds shmid_ds32; /* post conversion, 32 bit version */ + struct user32_shmid_ds shmid_ds32 = {}; /* post conversion, 32 bit version */ struct user_shmid_ds shmid_ds; /* 64 bit version */ void *shmid_dsp; size_t ipcs_sz = sizeof(struct user_IPCS_command); diff --git a/bsd/kern/trace_codes b/bsd/kern/trace_codes index 1b7187a01..981ab7f57 100644 --- a/bsd/kern/trace_codes +++ b/bsd/kern/trace_codes @@ -107,7 +107,7 @@ 0x10c00bc MSC_kern_invalid_#47 0x10c00c0 MSC_macx_swapon 0x10c00c4 MSC_macx_swapoff -0x10c00c8 MSC_kern_invalid_#50 +0x10c00c8 MSC_thread_get_special_reply_port 0x10c00cc MSC_macx_triggers 0x10c00d0 MSC_macx_backing_store_suspend 0x10c00d4 MSC_macx_backing_store_recovery @@ -228,11 +228,19 @@ 0x1300420 MACH_vm_pageout_cache_evict 0x1300424 MACH_vm_pageout_thread_block 0x1300428 MACH_vm_pageout_jetsam +0x130042c MACH_vm_info1 +0x1300430 MACH_vm_info2 +0x1300434 MACH_vm_info3 +0x1300438 MACH_vm_info4 +0x130043c MACH_vm_info5 +0x1300440 MACH_vm_info6 +0x1300444 MACH_vm_info7 0x1300480 MACH_vm_upl_page_wait 0x1300484 MACH_vm_iopl_page_wait 0x1300488 MACH_vm_page_wait_block 0x130048C MACH_vm_page_sleep 0x1300490 MACH_vm_page_expedite +0x1300494 MACH_vm_page_expedite_no_memory 0x13004c0 MACH_vm_pressure_event 0x1300500 MACH_vm_data_write 0x1320000 vm_disconnect_all_page_mappings @@ -253,6 +261,8 @@ 0x1400024 MACH_IDLE 0x1400028 MACH_STACK_DEPTH 0x140002c MACH_MOVED +0x1400030 MACH_PSET_LOAD_AVERAGE +0x1400034 MACH_AMP_DEBUG 0x1400038 MACH_FAILSAFE 0x140003C MACH_BLOCK 0x1400040 MACH_WAIT @@ -284,6 +294,10 @@ 0x14000B4 MACH_SCHED_LOAD 0x14000B8 MACH_REC_CORES_FAILSAFE 0x14000BC MACH_SCHED_QUANTUM_EXPIRED +0x14000C0 MACH_EXEC_PROMOTE +0x14000C4 MACH_EXEC_DEMOTE +0x14000C8 MACH_AMP_SIGNAL_SPILL +0x14000CC MACH_AMP_STEAL 0x1500000 MACH_MSGID_INVALID 0x1600000 MTX_SLEEP 0x1600004 MTX_SLEEP_DEADLINE @@ -329,7 +343,11 @@ 0x1700034 PMAP_flush_kernel_TLBS 0x1700038 PMAP_flush_delayed_TLBS 0x170003c PMAP_flush_TLBS_TO +0x1700040 PMAP_flush_EPT +0x1700044 PMAP_fast_fault 0x1800000 MACH_CLOCK_EPOCH_CHANGE +0x1800004 MACH_CLOCK_BRIDGE_RCV_TS +0x1800008 MACH_CLOCK_BRIDGE_REMOTE_TIME 0x1900000 MP_TLB_FLUSH 0x1900004 MP_CPUS_CALL 0x1900008 MP_CPUS_CALL_LOCAL @@ -355,6 +373,17 @@ 0x1a30008 ENERGY_PERF_GPU_TIME 0x1a40000 SYSDIAGNOSE_notify_user 0x1a50000 ZALLOC_ZCRAM +0x1a60000 THREAD_GROUP_NEW +0x1a60004 THREAD_GROUP_FREE +0x1a60008 THREAD_GROUP_SET +0x1a6000c THREAD_GROUP_NAME +0x1a60010 THREAD_GROUP_NAME_FREE +0x1a60014 THREAD_GROUP_FLAGS +0x1a70000 COALITION_NEW +0x1a70004 COALITION_FREE +0x1a70008 COALITION_ADOPT +0x1a7000c COALITION_REMOVE +0x1a70010 COALITION_THREAD_GROUP_SET 0x2010000 L_IP_In_Beg 0x2010004 L_IP_Out_Beg 0x2010008 L_IP_In_End @@ -856,12 +885,34 @@ 0x402002C MEMSTAT_dirty_clear 0x4020030 MEMSTAT_grp_set_properties 0x4020034 MEMSTAT_do_kill +0x4030004 KEVENT_kq_processing_begin +0x4030008 KEVENT_kq_processing_end +0x403000c KEVENT_kqwq_processing_begin +0x4030010 KEVENT_kqwq_processing_end +0x4030014 KEVENT_kqwq_bind +0x4030018 KEVENT_kqwq_unbind +0x403001c KEVENT_kqwq_thread_request +0x4030020 KEVENT_kqwl_processing_begin +0x4030024 KEVENT_kqwl_processing_end +0x4030028 KEVENT_kqwl_thread_request +0x403002c KEVENT_kqwl_thread_adjust +0x4030030 KEVENT_kq_register +0x4030034 KEVENT_kqwq_register +0x4030038 KEVENT_kqwl_register +0x403003c KEVENT_knote_activate +0x4030040 KEVENT_kq_process +0x4030044 KEVENT_kqwq_process +0x4030048 KEVENT_kqwl_process +0x403004c KEVENT_kqwl_bind +0x4030050 KEVENT_kqwl_unbind +0x4030054 KEVENT_knote_enable 0x40e0104 BSC_msync_extended_info 0x40e0264 BSC_pread_extended_info 0x40e0268 BSC_pwrite_extended_info 0x40e0314 BSC_mmap_extended_info 0x40f0314 BSC_mmap_extended_info2 0x5000004 INTC_Handler +0x5000008 INTC_Spurious 0x5010004 WL_CheckForWork 0x5010008 WL_RunEventSources 0x5020004 IES_client @@ -938,6 +989,8 @@ 0x50700ec PM_DriverResponseDelay 0x50700f0 PM_PCIDevChangeStart 0x50700f4 PM_PCIDevChangeDone +0x50700f8 PM_SleepWakeMessage +0x50700fc PM_DriverPSChangeDelay 0x5080004 IOSERVICE_BUSY 0x5080008 IOSERVICE_NONBUSY 0x508000c IOSERVICE_MODULESTALL @@ -1135,6 +1188,11 @@ 0x5330024 HIBERNATE_aes_decrypt_cbc 0x5330028 HIBERNATE_flush_compressor 0x533002c HIBERNATE_fastwake_warmup +0x5330030 HIBERNATE_teardown +0x5330034 HIBERNATE_rebuild +0x5330038 HIBERNATE_stats +0x533003c HIBERNATE_idle_kernel +0x5350000 BOOTER_timestamps 0x7000004 TRACE_DATA_NEWTHREAD 0x7000008 TRACE_DATA_EXEC 0x700000c TRACE_DATA_THREAD_TERMINATE @@ -1149,6 +1207,7 @@ 0x7020008 TRACE_LOST_EVENTS 0x702000c TRACE_WRITING_EVENTS 0x7020010 TRACE_INFO_STRING +0x7020014 TRACE_RETROGRADE_EVENTS 0x8000000 USER_TEST 0x8000004 USER_run 0x8000008 USER_join @@ -1206,130 +1265,10 @@ 0xa008000 P_CS_SYNC_DISK 0xa008004 P_CS_WaitForBuffer 0xa008008 P_CS_NoBuffer -0xb000000 AFP_asp_tcp_usr_send -0xb000004 AFP_asp_tcp_usr_send_after_Request -0xb000008 AFP_asp_tcp_usr_send_after_FindDSIReq -0xb00000c AFP_asp_tcp_usr_send_after_Reply -0xb000010 AFP_asp_tcp_slowtimo -0xb000014 AFP_asp_tcp_usr_control -0xb000018 AFP_asp_tcp_fasttimo -0xb000020 AFP_Send -0xb000024 AFP_Send_before_sosend -0xb000028 AFP_Send_after_sosend -0xb00002c AFP_Send_before_write -0xb000030 AFP_Send_after_write -0xb000040 AFP_Reply -0xb000044 AFP_Reply_rcvdAlready -0xb000048 AFP_Reply_before_RcvLock -0xb00004c AFP_Reply_fail_RcvLock -0xb000050 AFP_Reply_before_ReadDSIHdr -0xb000054 AFP_Reply_after_ReadDSIHdr -0xb000058 AFP_Reply_fail_ReadDSIHdr -0xb00005c AFP_Reply_after_FindDSIReqInfo -0xb000060 AFP_Reply_SetAFPCmd -0xb000064 AFP_Reply_before_ReadDSIPacket -0xb000068 AFP_Reply_setRcvdReplyLen -0xb000070 AFP_SendReply -0xb000080 AFP_CreateDSIHeader -0xb000084 AFP_CreateDSIHeader_after_GetReqID -0xb000090 AFP_Request -0xb0000a0 AFP_ReceiveLock -0xb0000b0 AFP_ReceiveWakeUp -0xb0000c0 AFP_ReceiveUnLock -0xb0000e0 AFP_SendLock -0xb0000e4 AFP_SendUnLock -0xb0000f0 AFP_SendQueueLock -0xb000100 AFP_SendQueueUnLock -0xb000110 AFP_ReadDSIHeader -0xb000120 AFP_Receive -0xb000124 AFP_Receive_before_sorcv -0xb000128 AFP_Receive_after_sorcv -0xb000130 AFP_ReadDSIPacket -0xb000140 AFP_DoCopyOut -0xb000150 AFP_DoCopyIn -0xb000160 AFP_CheckRcvTickle -0xb000164 AFP_CheckRcvTickleTO -0xb000170 AFP_CheckSendTickle -0xb000180 AFP_CheckIncomingPkts -0xb000190 AFP_ProcessOptions -0xb000200 AFP_FindDSIReqInfo -0xb000204 AFP_FindDSIReqInfo_foundReqInfo -0xb000208 AFP_FindDSIReqInfo_flags -0xb00020c AFP_FindDSIReqLeave -0xb000210 AFP_UsrDisconnect -0xc000000 AFPVFS_UserReply -0xc000004 AFPVFS_UserReplyGetMbuf -0xc000008 AFPVFS_UserReplysosend -0xc000010 AFPVFS_UserCommand -0xc000018 AFPVFS_UserCommandsosend -0xc000020 AFPVFS_ReadFork -0xc000024 AFPVFS_ReadForkFillQPB -0xc000028 AFPVFS_ReadForkNbrRequests -0xc00002c AFPVFS_ReadForkSendQPB -0xc000030 AFPVFS_ReadForkSendErr -0xc000040 AFPVFS_ReadForkGetReply -0xc000044 AFPVFS_ReadForkGetReplyResult -0xc000050 AFPVFS_WriteFork -0xc000054 AFPVFS_WriteForkFillQPB -0xc000058 AFPVFS_WriteForkNbrRequests -0xc00005c AFPVFS_WriteForkSendQPB -0xc000060 AFPVFS_WriteForkSendErr -0xc000064 AFPVFS_WriteForkGetReply -0xc000068 AFPVFS_WriteForkGetReplyResult -0xc000070 AFPVFS_GetAttr -0xc000080 AFPVFS_SetAttr -0xc000090 AFPVFS_GetAttrList -0xc0000a0 AFPVFS_SetAttrList -0xc0000b0 AFPVFS_FSCTL -0xc0000c0 AFPVFS_LookUp -0xc0000d0 AFPVFS_CacheLookUp -0xc0000e0 AFPVFS_Write -0xc0000e4 AFPVFS_WriteNoCluster -0xc0000e8 AFPVFS_WriteDone -0xc0000f0 AFPVFS_DoWrite -0xc000100 AFPVFS_Lock -0xc000110 AFPVFS_Statfs -0xc000120 AFPVFS_Sync -0xc000130 AFPVFS_VGet -0xc000140 AFPVFS_FlushFiles -0xc000150 AFPVFS_Create -0xc000160 AFPVFS_Mknod -0xc000170 AFPVFS_Open -0xc000180 AFPVFS_Close -0xc000190 AFPVFS_Access -0xc000194 AFPVFS_AccessUID -0xc000198 AFPVFS_AccessGID -0xc00019c AFPVFS_AccessWID -0xc0001a0 AFPVFS_Writeperm -0xc0001b0 AFPVFS_Chmod -0xc0001c0 AFPVFS_Chflags -0xc0001d0 AFPVFS_Exchange -0xc0001e0 AFPVFS_Chid -0xc0001f0 AFPVFS_Fsync -0xc000200 AFPVFS_Remove -0xc000210 AFPVFS_Rename -0xc000220 AFPVFS_Copyfile -0xc000230 AFPVFS_Mkdir -0xc000240 AFPVFS_Symlink -0xc000250 AFPVFS_Readdir -0xc000260 AFPVFS_Readdirattr -0xc000264 AFPVFS_Readdirattr1 -0xc000268 AFPVFS_Readdirattr2 -0xc00026c AFPVFS_Readdirattr3 -0xc000270 AFPVFS_Readlink -0xc000280 AFPVFS_Abortop -0xc000290 AFPVFS_Inactive -0xc0002a0 AFPVFS_Reclaim -0xc0002b0 AFPVFS_Unlock -0xc0002c0 AFPVFS_Islocked -0xc0002d0 AFPVFS_Pathconf -0xc0002e0 AFPVFS_Update -0xc0002f0 AFPVFS_Makenode -0xc000300 AFPVFS_Allocate -0xc000310 AFPVFS_Search -0xc000320 AFPVFS_Reconnect -0xc0003e0 AFPVFS_Rmdir -0xc0003f0 AFPVFS_Vinit +0xc010000 MT_InstrsCycles +0xc010004 MT_InsCyc_CPU_CSwitch +0xcfe0000 MT_TmpThread +0xcff0000 MT_TmpCPU 0x11000000 DNC_PURGE1 0x11000004 DNC_PURGE2 0x11000008 DNC_FOUND @@ -1451,43 +1390,6 @@ 0x20000004 RTC_sync_TBR 0x21010000 SCROLL_BEGIN 0x21020000 BOOT_BEGIN -0x21030200 LOGIN_BEGIN -0x21030204 LOGINWINDOW_LAUNCHED -0x21030208 LOGINWINDOW_LAUNCHES_SA -0x2103020c LOGINWINDOW_GUI_APPEARS -0x21030210 LOGINWINDOW_LOGIN_CLICKED -0x21030214 LOGINWINDOW_ASKS_AUTH -0x21030218 LOGINWINDOW_AUTH_SUCCEEDED -0x2103021c LOGINWINDOW_LAUNCHES_DOCK -0x21030220 LOGINWINDOW_LAUNCHES_SUIS -0x21030224 LOGINWINDOW_LAUNCHES_FINDER -0x21030228 LOGINWINDOW_DOCK_LAUNCHED -0x2103022c LOGINWINDOW_SUIS_LAUNCHED -0x21030230 LOGINWINDOW_FINDER_LAUNCHED -0x21030234 LOGINWINDOW_LOGOUT_CLICKED -0x21030238 LOGINWINDOW_QUIT_FGAPPS -0x2103023c LOGINWINDOW_FGAPPS_QUIT -0x21030240 LOGINWINDOW_QUIT_SUIS -0x21030244 LOGINWINDOW_SUIS_DIES -0x21030248 LOGINWINDOW_QUIT_FINDER -0x2103024c LOGINWINDOW_FINDER_DIES -0x21030250 LOGINWINDOW_QUIT_DOCK -0x21030254 LOGINWINDOW_DOCK_DIES -0x21030258 LOGINWINDOW_EXIT -0x2103025c LOGINWINDOW_FUS_SELUSERNAME -0x21030260 LOGINWINDOW_FUS_SELLOGINWIND -0x21030270 LOGIN_APPLICATION_EXECUTING -0x21030274 LOGIN_APPLICATION_USABLE -0x21030300 LOGIN_END -0x21030500 LOGINWINDOW_APP_TERMINATION_REQUEST -0x21030504 LOGINWINDOW_LOGOUT_START -0x21030508 LOGINWINDOW_DESKTOP_UP -0x2103050c LOGINWINDOW_DESKTOP_UP_NOTIFICATION -0x21030510 LOGINWINDOW_DISPLAYWAKE -0x21030514 LOGINWINDOW_SYSTEMWAKE -0x21030518 LOGINWINDOW_UNLOCKUI_TRIGGERED -0x2103051c LOGINWINDOW_UNLOCKUI_SHOWN -0x21030520 LOGINWINDOW_NO_LOCKUI_SHOWN 0x21040000 APP_AudioOverload 0x21050000 TOOL_PRIVATE_1 0x21050004 TOOL_PRIVATE_2 @@ -1561,6 +1463,10 @@ 0x25010038 PERF_THD_Disp_Data_32 0x2501003c PERF_THD_Sched_Data1_32 0x25010040 PERF_THD_Sched_Data2_32 +0x25010044 PERF_THD_Inscyc_Data +0x25010048 PERF_THD_Inscyc_Data_32 +0x2501004c PERF_THD_Sched_Data_2 +0x25010050 PERF_THD_Sched_Data2_32_2 0x25020000 PERF_STK_KSample 0x25020004 PERF_STK_USched 0x25020008 PERF_STK_USample @@ -1571,7 +1477,7 @@ 0x2502001c PERF_STK_Error 0x25020020 PERF_STK_Backtrace 0x25020024 PERF_STK_Log -0x25030000 PERF_TMR_AllSched +0x25030000 PERF_TMR_Fire 0x25030004 PERF_TMR_Schedule 0x25030008 PERF_TMR_Handler 0x25040000 PERF_ATS_Thread @@ -1606,6 +1512,7 @@ 0x25080010 PERF_TK_Snap_Data2_32 0x250a0000 PERF_MI_Sample 0x250a0004 PERF_MI_Data +0x250a0008 PERF_MI_SysMem_Data 0x26100008 imp_assertion_hold 0x2610000c imp_assertion_hold_ext 0x26100020 imp_assertion_externalize @@ -1631,6 +1538,10 @@ 0x261a0004 imp_usynch_remove_override 0x261b0000 imp_donor_update_live_donor 0x261b0004 imp_donor_init_donor_state +0x261d0000 imp_sync_ipc_qos_applied +0x261d0004 imp_sync_ipc_qos_removed +0x261d0008 imp_sync_ipc_qos_overflow +0x261d000c imp_sync_ipc_qos_underflow 0x26210010 imp_task_int_bg 0x26210014 imp_task_ext_bg 0x26210020 imp_thread_int_bg @@ -1660,6 +1571,11 @@ 0x26310018 imp_task_watchers_bg 0x26320028 imp_thread_pidbind_bg 0x26330028 imp_thread_workq_bg +0x26350028 imp_thread_qos +0x26360028 imp_thread_qos_override +0x26380028 imp_thread_qos_and_relprio +0x263c0028 imp_thread_qos_promote +0x263d0028 imp_thread_qos_ipc_override 0x27000000 PERF_PCEVENT 0x27001000 PERF_CPU_IDLE 0x27001100 PERF_CPU_IDLE_TIMER @@ -1686,6 +1602,7 @@ 0x2700E030 PERF_SRAMEMA_DOM3 0x28100004 BANK_SETTLE_CPU_TIME 0x28100008 BANK_SECURE_ORIGINATOR_CHANGED +0x2810000c BANK_SETTLE_ENERGY 0x2a100004 ATM_MIN_CALLED 0x2a100008 ATM_LINK_LIST_TRIM 0x2a200004 ATM_VALUE_REPLACED @@ -1706,7 +1623,6 @@ 0xff000334 MSG_host_get_io_master 0xff000338 MSG_host_get_clock_service 0xff00033c MSG_kmod_get_info -0xff000340 MSG_host_zone_info 0xff000344 MSG_host_virtual_physical_table_info 0xff000348 MSG_host_ipc_hash_info 0xff00034c MSG_enable_bluebox diff --git a/bsd/kern/tty.c b/bsd/kern/tty.c index 580852a0c..61d59ee62 100644 --- a/bsd/kern/tty.c +++ b/bsd/kern/tty.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997-2007 Apple Inc. All rights reserved. + * Copyright (c) 1997-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -113,18 +113,8 @@ #include #include #include /* averunnable */ - -/* - * Debugging assertions for tty locks - */ -#define TTY_DEBUG 1 -#if TTY_DEBUG -#define TTY_LOCK_OWNED(tp) do {lck_mtx_assert(&tp->t_lock, LCK_MTX_ASSERT_OWNED); } while (0) -#define TTY_LOCK_NOTOWNED(tp) do {lck_mtx_assert(&tp->t_lock, LCK_MTX_ASSERT_NOTOWNED); } while (0) -#else -#define TTY_LOCK_OWNED(tp) -#define TTY_LOCK_NOTOWNED(tp) -#endif +#include +#include static lck_grp_t *tty_lck_grp; static lck_grp_attr_t *tty_lck_grp_attr; @@ -142,7 +132,7 @@ static void ttyunblock(struct tty *tp); static int ttywflush(struct tty *tp); static int proc_compare(proc_t p1, proc_t p2); -static void ttyhold(struct tty *tp); +void ttyhold(struct tty *tp); static void ttydeallocate(struct tty *tp); static int isctty(proc_t p, struct tty *tp); @@ -330,7 +320,6 @@ tty_unlock(struct tty *tp) lck_mtx_unlock(&tp->t_lock); } - /* * ttyopen (LDISC) * @@ -372,6 +361,7 @@ ttyclose(struct tty *tp) { struct pgrp * oldpg; struct session * oldsessp; + struct knote *kn; TTY_LOCK_OWNED(tp); /* debug assert */ @@ -408,8 +398,15 @@ ttyclose(struct tty *tp) pg_rele(oldpg); tty_lock(tp); tp->t_state = 0; + SLIST_FOREACH(kn, &tp->t_wsel.si_note, kn_selnext) { + KNOTE_DETACH(&tp->t_wsel.si_note, kn); + } selthreadclear(&tp->t_wsel); + SLIST_FOREACH(kn, &tp->t_rsel.si_note, kn_selnext) { + KNOTE_DETACH(&tp->t_rsel.si_note, kn); + } selthreadclear(&tp->t_rsel); + return (0); } @@ -1517,18 +1514,27 @@ int ttyselect(struct tty *tp, int rw, void *wql, proc_t p) { int retval = 0; + /* + * Attaching knotes to TTYs needs to call selrecord in order to hook + * up the waitq to the selinfo, regardless of data being ready. See + * filt_ttyattach. + */ + bool needs_selrecord = rw & FMARK; + rw &= ~FMARK; - if (tp == NULL) - return (ENXIO); + if (tp == NULL) { + return ENXIO; + } - TTY_LOCK_OWNED(tp); /* debug assert */ + TTY_LOCK_OWNED(tp); + + if (tp->t_state & TS_ZOMBIE) { + retval = 1; + goto out; + } switch (rw) { case FREAD: - if (ISSET(tp->t_state, TS_ZOMBIE)) { - return(1); - } - retval = ttnread(tp); if (retval > 0) { break; @@ -1537,12 +1543,8 @@ ttyselect(struct tty *tp, int rw, void *wql, proc_t p) selrecord(p, &tp->t_rsel, wql); break; case FWRITE: - if (ISSET(tp->t_state, TS_ZOMBIE)) { - return(1); - } - if ((tp->t_outq.c_cc <= tp->t_lowat) && - ISSET(tp->t_state, TS_CONNECTED)) { + (tp->t_state & TS_CONNECTED)) { retval = tp->t_hiwat - tp->t_outq.c_cc; break; } @@ -1550,6 +1552,19 @@ ttyselect(struct tty *tp, int rw, void *wql, proc_t p) selrecord(p, &tp->t_wsel, wql); break; } + +out: + if (retval > 0 && needs_selrecord) { + switch (rw) { + case FREAD: + selrecord(p, &tp->t_rsel, wql); + break; + case FWRITE: + selrecord(p, &tp->t_wsel, wql); + break; + } + } + return retval; } @@ -3066,7 +3081,7 @@ ttymalloc(void) /* * Increment the reference count on a tty. */ -static void +void ttyhold(struct tty *tp) { TTY_LOCK_OWNED(tp); @@ -3146,3 +3161,387 @@ isctty_sp(proc_t p, struct tty *tp, struct session *sessp) return(sessp == tp->t_session && p->p_flag & P_CONTROLT); } + + +static int filt_ttyattach(struct knote *kn, struct kevent_internal_s *kev); +static void filt_ttydetach(struct knote *kn); +static int filt_ttyevent(struct knote *kn, long hint); +static int filt_ttytouch(struct knote *kn, struct kevent_internal_s *kev); +static int filt_ttyprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); + +SECURITY_READ_ONLY_EARLY(struct filterops) tty_filtops = { + .f_isfd = 1, + .f_attach = filt_ttyattach, + .f_detach = filt_ttydetach, + .f_event = filt_ttyevent, + .f_touch = filt_ttytouch, + .f_process = filt_ttyprocess +}; + +/* + * Called with struct tty locked. Returns non-zero if there is data to be read + * or written. + */ +static int +filt_tty_common(struct knote *kn, struct tty *tp) +{ + int retval = 0; + + TTY_LOCK_OWNED(tp); /* debug assert */ + + if (tp->t_state & TS_ZOMBIE) { + kn->kn_flags |= EV_EOF; + return 1; + } + + switch (knote_get_seltype(kn)) { + case FREAD: + retval = ttnread(tp); + break; + case FWRITE: + if ((tp->t_outq.c_cc <= tp->t_lowat) && + (tp->t_state & TS_CONNECTED)) { + retval = tp->t_hiwat - tp->t_outq.c_cc; + } + break; + } + + kn->kn_data = retval; + + /* + * TODO(mwidmann, jandrus): For native knote low watermark support, + * check the kn_sfflags for NOTE_LOWAT and check against kn_sdata. + * + * res = ((kn->kn_sfflags & NOTE_LOWAT) != 0) ? + * (kn->kn_data >= kn->kn_sdata) : kn->kn_data; + */ + + return retval; +} + +/* + * Find the struct tty from a waitq, which is a member of one of the two struct + * selinfos inside the struct tty. Use the seltype to determine which selinfo. + */ +static struct tty * +tty_from_waitq(struct waitq *wq, int seltype) +{ + struct selinfo *si; + struct tty *tp = NULL; + + /* + * The waitq is part of the selinfo structure managed by the driver. For + * certain drivers, we want to hook the knote into the selinfo + * structure's si_note field so selwakeup can call KNOTE. + * + * While 'wq' is not really a queue element, this macro only uses the + * pointer to calculate the offset into a structure given an element + * name. + */ + si = qe_element(wq, struct selinfo, si_waitq); + + /* + * For TTY drivers, the selinfo structure is somewhere in the struct + * tty. There are two different selinfo structures, and the one used + * corresponds to the type of filter requested. + * + * While 'si' is not really a queue element, this macro only uses the + * pointer to calculate the offset into a structure given an element + * name. + */ + switch (seltype) { + case FREAD: + tp = qe_element(si, struct tty, t_rsel); + break; + case FWRITE: + tp = qe_element(si, struct tty, t_wsel); + break; + } + + return tp; +} + +static struct tty * +tty_from_knote(struct knote *kn) +{ + return (struct tty *)kn->kn_hook; +} + +/* + * Try to lock the TTY structure associated with a knote. + * + * On success, this function returns a locked TTY structure. Otherwise, NULL is + * returned. + */ +__attribute__((warn_unused_result)) +static struct tty * +tty_lock_from_knote(struct knote *kn) +{ + struct tty *tp = tty_from_knote(kn); + if (tp) { + tty_lock(tp); + } + + return tp; +} + +/* + * Set the knote's struct tty to the kn_hook field. + * + * The idea is to fake a call to select with our own waitq set. If the driver + * calls selrecord, we'll get a link to their waitq and access to the tty + * structure. + * + * Returns -1 on failure, with the error set in the knote, or selres on success. + */ +static int +tty_set_knote_hook(struct knote *kn) +{ + uthread_t uth; + vfs_context_t ctx; + vnode_t vp; + kern_return_t kr; + struct waitq *wq = NULL; + struct waitq_set *old_wqs; + struct waitq_set tmp_wqs; + uint64_t rsvd, rsvd_arg; + uint64_t *rlptr = NULL; + int selres = -1; + struct tty *tp; + + uth = get_bsdthread_info(current_thread()); + + ctx = vfs_context_current(); + vp = (vnode_t)kn->kn_fp->f_fglob->fg_data; + + /* + * Reserve a link element to avoid potential allocation under + * a spinlock. + */ + rsvd = rsvd_arg = waitq_link_reserve(NULL); + rlptr = (void *)&rsvd_arg; + + /* + * Trick selrecord into hooking a known waitq set into the device's selinfo + * waitq. Once the link is in place, we can get back into the selinfo from + * the waitq and subsequently the tty (see tty_from_waitq). + * + * We can't use a real waitq set (such as the kqueue's) because wakeups + * might happen before we can unlink it. + */ + kr = waitq_set_init(&tmp_wqs, SYNC_POLICY_FIFO | SYNC_POLICY_PREPOST, NULL, + NULL); + assert(kr == KERN_SUCCESS); + + old_wqs = uth->uu_wqset; + uth->uu_wqset = &tmp_wqs; + /* + * FMARK forces selects to always call selrecord, even if data is + * available. See ttselect, ptsselect, ptcselect. + * + * selres also contains the data currently available in the tty. + */ + selres = VNOP_SELECT(vp, knote_get_seltype(kn) | FMARK, 0, rlptr, ctx); + uth->uu_wqset = old_wqs; + + /* + * Make sure to cleanup the reserved link - this guards against + * drivers that may not actually call selrecord(). + */ + waitq_link_release(rsvd); + if (rsvd == rsvd_arg) { + /* + * The driver didn't call selrecord -- there's no tty hooked up so we + * can't attach. + */ + knote_set_error(kn, ENOTTY); + selres = -1; + goto out; + } + + /* rlptr may not point to a properly aligned pointer */ + memcpy(&wq, rlptr, sizeof(void *)); + + tp = tty_from_waitq(wq, knote_get_seltype(kn)); + assert(tp != NULL); + + /* + * Take a reference and stash the tty in the knote. + */ + tty_lock(tp); + ttyhold(tp); + kn->kn_hook = tp; + tty_unlock(tp); + +out: + /* + * Cleaning up the wqset will unlink its waitq and clean up any preposts + * that occurred as a result of data coming in while the tty was attached. + */ + waitq_set_deinit(&tmp_wqs); + + return selres; +} + +static int +filt_ttyattach(struct knote *kn, __unused struct kevent_internal_s *kev) +{ + int selres = 0; + struct tty *tp; + + /* + * This function should be called from filt_specattach (spec_vnops.c), + * so most of the knote data structure should already be initialized. + */ + + /* don't support offsets in ttys or drivers that don't use struct tty */ + if (kn->kn_vnode_use_ofst || !kn->kn_vnode_kqok) { + knote_set_error(kn, ENOTSUP); + return 0; + } + + /* + * Connect the struct tty to the knote through the selinfo structure + * referenced by the waitq within the selinfo. + */ + selres = tty_set_knote_hook(kn); + if (selres < 0) { + return 0; + } + + /* + * Attach the knote to selinfo's klist. + */ + tp = tty_lock_from_knote(kn); + if (!tp) { + knote_set_error(kn, ENOENT); + return 0; + } + + switch (knote_get_seltype(kn)) { + case FREAD: + KNOTE_ATTACH(&tp->t_rsel.si_note, kn); + break; + case FWRITE: + KNOTE_ATTACH(&tp->t_wsel.si_note, kn); + break; + } + + tty_unlock(tp); + + return selres; +} + +static void +filt_ttydetach(struct knote *kn) +{ + struct tty *tp; + + tp = tty_lock_from_knote(kn); + if (!tp) { + knote_set_error(kn, ENOENT); + return; + } + + struct selinfo *si = NULL; + switch (knote_get_seltype(kn)) { + case FREAD: + si = &tp->t_rsel; + break; + case FWRITE: + si = &tp->t_wsel; + break; + /* knote_get_seltype will panic on default */ + } + + KNOTE_DETACH(&si->si_note, kn); + kn->kn_hook = NULL; + + tty_unlock(tp); + ttyfree(tp); +} + +static int +filt_ttyevent(struct knote *kn, long hint) +{ + int ret; + struct tty *tp; + bool revoked = hint & NOTE_REVOKE; + hint &= ~NOTE_REVOKE; + + tp = tty_from_knote(kn); + if (!tp) { + knote_set_error(kn, ENOENT); + return 0; + } + + if (!hint) { + tty_lock(tp); + } + + if (revoked) { + kn->kn_flags |= EV_EOF | EV_ONESHOT; + ret = 1; + } else { + ret = filt_tty_common(kn, tp); + } + + if (!hint) { + tty_unlock(tp); + } + + return ret; +} + +static int +filt_ttytouch(struct knote *kn, struct kevent_internal_s *kev) +{ + struct tty *tp; + int res = 0; + + tp = tty_lock_from_knote(kn); + if (!tp) { + knote_set_error(kn, ENOENT); + return 0; + } + + kn->kn_sdata = kev->data; + kn->kn_sfflags = kev->fflags; + if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) + kn->kn_udata = kev->udata; + + if (kn->kn_vnode_kqok) { + res = filt_tty_common(kn, tp); + } + + tty_unlock(tp); + + return res; +} + +static int +filt_ttyprocess(struct knote *kn, __unused struct filt_process_s *data, struct kevent_internal_s *kev) +{ + struct tty *tp; + int res; + + tp = tty_lock_from_knote(kn); + if (!tp) { + knote_set_error(kn, ENOENT); + return 0; + } + + res = filt_tty_common(kn, tp); + + if (res) { + *kev = kn->kn_kevent; + if (kn->kn_flags & EV_CLEAR) { + kn->kn_fflags = 0; + kn->kn_data = 0; + } + } + + tty_unlock(tp); + + return res; +} diff --git a/bsd/kern/tty_ptmx.c b/bsd/kern/tty_ptmx.c index 43db005d0..cf89b93eb 100644 --- a/bsd/kern/tty_ptmx.c +++ b/bsd/kern/tty_ptmx.c @@ -2,7 +2,7 @@ * Copyright (c) 1997-2013 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* @@ -82,6 +82,7 @@ #include #include #include /* DEVFS_LOCK()/DEVFS_UNLOCK() */ +#include #if CONFIG_MACF #include @@ -169,39 +170,6 @@ SYSCTL_PROC(_kern_tty, OID_AUTO, ptmx_max, static int ptmx_clone(dev_t dev, int minor); -/* - * Set of locks to keep the interaction between kevents and revoke - * from causing havoc. - */ - -#define LOG2_PTSD_KE_NLCK 2 -#define PTSD_KE_NLCK (1l << LOG2_PTSD_KE_NLCK) -#define PTSD_KE_LOCK_INDEX(x) ((x) & (PTSD_KE_NLCK - 1)) - -static lck_mtx_t ptsd_kevent_lock[PTSD_KE_NLCK]; - -static void -ptsd_kevent_lock_init(void) -{ - int i; - lck_grp_t *lgrp = lck_grp_alloc_init("ptsd kevent", LCK_GRP_ATTR_NULL); - - for (i = 0; i < PTSD_KE_NLCK; i++) - lck_mtx_init(&ptsd_kevent_lock[i], lgrp, LCK_ATTR_NULL); -} - -static void -ptsd_kevent_mtx_lock(int minor) -{ - lck_mtx_lock(&ptsd_kevent_lock[PTSD_KE_LOCK_INDEX(minor)]); -} - -static void -ptsd_kevent_mtx_unlock(int minor) -{ - lck_mtx_unlock(&ptsd_kevent_lock[PTSD_KE_LOCK_INDEX(minor)]); -} - static struct tty_dev_t _ptmx_driver; int @@ -213,12 +181,12 @@ ptmx_init( __unused int config_count) */ /* Get a major number for /dev/ptmx */ - if((ptmx_major = cdevsw_add(-15, &ptmx_cdev)) == -1) { + if ((ptmx_major = cdevsw_add(-15, &ptmx_cdev)) == -1) { printf("ptmx_init: failed to obtain /dev/ptmx major number\n"); return (ENOENT); } - if (cdevsw_setkqueueok(ptmx_major, &ptmx_cdev, 0) == -1) { + if (cdevsw_setkqueueok(ptmx_major, &ptmx_cdev, CDEVSW_IS_PTC) == -1) { panic("Failed to set flags on ptmx cdevsw entry."); } @@ -228,16 +196,11 @@ ptmx_init( __unused int config_count) printf("ptmx_init: failed to obtain /dev/ptmx major number\n"); return (ENOENT); } - - if (cdevsw_setkqueueok(ptsd_major, &ptsd_cdev, 0) == -1) { + + if (cdevsw_setkqueueok(ptsd_major, &ptsd_cdev, CDEVSW_IS_PTS) == -1) { panic("Failed to set flags on ptmx cdevsw entry."); } - /* - * Locks to guard against races between revoke and kevents - */ - ptsd_kevent_lock_init(); - /* Create the /dev/ptmx device {,0} */ (void)devfs_make_node_clone(makedev(ptmx_major, 0), DEVFS_CHAR, UID_ROOT, GID_TTY, 0666, @@ -341,8 +304,8 @@ ptmx_get_ioctl(int minor, int open_flag) _state.pis_total += PTMX_GROW_VECTOR; if (old_pis_ioctl_list) FREE(old_pis_ioctl_list, M_TTYS); - } - + } + /* is minor in range now? */ if (minor < 0 || minor >= _state.pis_total) { ttyfree(new_ptmx_ioctl->pt_tty); @@ -350,14 +313,14 @@ ptmx_get_ioctl(int minor, int open_flag) FREE(new_ptmx_ioctl, M_TTYS); return (NULL); } - + if (_state.pis_ioctl_list[minor] != NULL) { ttyfree(new_ptmx_ioctl->pt_tty); DEVFS_UNLOCK(); FREE(new_ptmx_ioctl, M_TTYS); /* Special error value so we know to redrive the open, we've been raced */ - return (struct ptmx_ioctl*)-1; + return (struct ptmx_ioctl*)-1; } @@ -381,11 +344,11 @@ ptmx_get_ioctl(int minor, int open_flag) printf("devfs_make_node() call failed for ptmx_get_ioctl()!!!!\n"); } } - + if (minor < 0 || minor >= _state.pis_total) { return (NULL); } - + return (_state.pis_ioctl_list[minor]); } @@ -398,7 +361,7 @@ ptmx_free_ioctl(int minor, int open_flag) struct ptmx_ioctl *old_ptmx_ioctl = NULL; DEVFS_LOCK(); - + if (minor < 0 || minor >= _state.pis_total) { DEVFS_UNLOCK(); return (-1); @@ -498,22 +461,20 @@ ptmx_clone(__unused dev_t dev, int action) /* * kqueue support. */ -int ptsd_kqfilter(dev_t, struct knote *); +int ptsd_kqfilter(dev_t dev, struct knote *kn); static void ptsd_kqops_detach(struct knote *); static int ptsd_kqops_event(struct knote *, long); static int ptsd_kqops_touch(struct knote *kn, struct kevent_internal_s *kev); static int ptsd_kqops_process(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops ptsd_kqops = { +SECURITY_READ_ONLY_EARLY(struct filterops) ptsd_kqops = { .f_isfd = 1, + /* attach is handled by ptsd_kqfilter -- the dev node must be passed in */ .f_detach = ptsd_kqops_detach, .f_event = ptsd_kqops_event, .f_touch = ptsd_kqops_touch, .f_process = ptsd_kqops_process, -}; - -#define PTSD_KNOTE_VALID NULL -#define PTSD_KNOTE_REVOKED ((void *)-911l) +}; /* * In the normal case, by the time the driver_close() routine is called @@ -527,243 +488,210 @@ struct filterops ptsd_kqops = { static void ptsd_kqops_detach(struct knote *kn) { - struct ptmx_ioctl *pti; struct tty *tp; - dev_t dev, lockdev = (dev_t)kn->kn_hookid; - - ptsd_kevent_mtx_lock(minor(lockdev)); - - if ((dev = (dev_t)kn->kn_hookid) != 0) { - pti = ptmx_get_ioctl(minor(dev), 0); - if (pti != NULL && (tp = pti->pt_tty) != NULL) { - tty_lock(tp); - if (kn->kn_filter == EVFILT_READ) - KNOTE_DETACH(&tp->t_rsel.si_note, kn); - else - KNOTE_DETACH(&tp->t_wsel.si_note, kn); - tty_unlock(tp); - kn->kn_hookid = 0; + + tp = kn->kn_hook; + assert(tp != NULL); + + tty_lock(tp); + + /* + * Only detach knotes from open ttys -- ttyclose detaches all knotes + * under the lock and unsets TS_ISOPEN. + */ + if (tp->t_state & TS_ISOPEN) { + switch (kn->kn_filter) { + case EVFILT_READ: + KNOTE_DETACH(&tp->t_rsel.si_note, kn); + break; + + case EVFILT_WRITE: + KNOTE_DETACH(&tp->t_wsel.si_note, kn); + break; + + default: + panic("invalid knote %p detach, filter: %d", kn, kn->kn_filter); + break; } } - ptsd_kevent_mtx_unlock(minor(lockdev)); + kn->kn_hook = NULL; + tty_unlock(tp); + + ttyfree(tp); } static int -ptsd_kqops_common(struct knote *kn, dev_t dev, long hint) +ptsd_kqops_common(struct knote *kn, struct tty *tp) { - struct ptmx_ioctl *pti; - struct tty *tp; int retval = 0; - do { - if (kn->kn_hook != PTSD_KNOTE_VALID ) { - /* We were revoked */ - kn->kn_data = 0; - kn->kn_flags |= EV_EOF; - retval = 1; - break; - } + TTY_LOCK_OWNED(tp); - pti = ptmx_get_ioctl(minor(dev), 0); - if (pti == NULL || (tp = pti->pt_tty) == NULL) { - kn->kn_data = ENXIO; - kn->kn_flags |= EV_ERROR; + switch (kn->kn_filter) { + case EVFILT_READ: + kn->kn_data = ttnread(tp); + if (kn->kn_data > 0) { retval = 1; - break; } + break; - if (hint == 0) - tty_lock(tp); - - if (kn->kn_filter == EVFILT_READ) { - kn->kn_data = ttnread(tp); - if (kn->kn_data > 0) - retval = 1; - if (ISSET(tp->t_state, TS_ZOMBIE)) { - kn->kn_flags |= EV_EOF; - retval = 1; - } - } else { /* EVFILT_WRITE */ - if ((tp->t_outq.c_cc <= tp->t_lowat) && - ISSET(tp->t_state, TS_CONNECTED)) { - kn->kn_data = tp->t_outq.c_cn - tp->t_outq.c_cc; - retval = 1; - } - if (ISSET(tp->t_state, TS_ZOMBIE)) { - kn->kn_flags |= EV_EOF; - retval = 1; - } + case EVFILT_WRITE: + if ((tp->t_outq.c_cc <= tp->t_lowat) && + (tp->t_state & TS_CONNECTED)) { + kn->kn_data = tp->t_outq.c_cn - tp->t_outq.c_cc; + retval = 1; } + break; - if (hint == 0) - tty_unlock(tp); + default: + panic("ptsd kevent: unexpected filter: %d, kn = %p, tty = %p", + kn->kn_filter, kn, tp); + break; + } - } while (0); + if (tp->t_state & TS_ZOMBIE) { + kn->kn_flags |= EV_EOF; + retval = 1; + } - return (retval); -} + return retval; +} static int ptsd_kqops_event(struct knote *kn, long hint) { - dev_t dev = (dev_t)kn->kn_hookid; - int res; + struct tty *tp = kn->kn_hook; + int ret; + bool revoked = hint & NOTE_REVOKE; + hint &= ~NOTE_REVOKE; - ptsd_kevent_mtx_lock(minor(dev)); - res = ptsd_kqops_common(kn, dev, hint); - ptsd_kevent_mtx_unlock(minor(dev)); - return res; + if (!hint) { + tty_lock(tp); + } + + if (revoked) { + kn->kn_flags |= EV_EOF | EV_ONESHOT; + ret = 1; + } else { + ret = ptsd_kqops_common(kn, tp); + } + + if (!hint) { + tty_unlock(tp); + } + + return ret; } - static int ptsd_kqops_touch(struct knote *kn, struct kevent_internal_s *kev) { - dev_t dev = (dev_t)kn->kn_hookid; - int res; + struct tty *tp; + int ret; + + tp = kn->kn_hook; - ptsd_kevent_mtx_lock(minor(dev)); + tty_lock(tp); /* accept new kevent state */ kn->kn_sfflags = kev->fflags; kn->kn_sdata = kev->data; - if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) + if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) { kn->kn_udata = kev->udata; + } /* recapture fired state of knote */ - res = ptsd_kqops_common(kn, dev, 0); + ret = ptsd_kqops_common(kn, tp); - ptsd_kevent_mtx_unlock(minor(dev)); + tty_unlock(tp); - return res; + return ret; } static int -ptsd_kqops_process(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev) +ptsd_kqops_process(struct knote *kn, __unused struct filt_process_s *data, + struct kevent_internal_s *kev) { -#pragma unused(data) - dev_t dev = (dev_t)kn->kn_hookid; - int res; + struct tty *tp = kn->kn_hook; + int ret; - ptsd_kevent_mtx_lock(minor(dev)); - res = ptsd_kqops_common(kn, dev, 0); - if (res) { + tty_lock(tp); + ret = ptsd_kqops_common(kn, tp); + if (ret) { *kev = kn->kn_kevent; if (kn->kn_flags & EV_CLEAR) { kn->kn_fflags = 0; kn->kn_data = 0; } } - ptsd_kevent_mtx_unlock(minor(dev)); - return res; + tty_unlock(tp); + + return ret; } int ptsd_kqfilter(dev_t dev, struct knote *kn) { - struct tty *tp = NULL; + struct tty *tp = NULL; struct ptmx_ioctl *pti = NULL; - int retval = 0; + int ret; /* make sure we're talking about the right device type */ if (cdevsw[major(dev)].d_open != ptsopen) { - kn->kn_flags = EV_ERROR; - kn->kn_data = EINVAL; + knote_set_error(kn, ENODEV); return 0; } if ((pti = ptmx_get_ioctl(minor(dev), 0)) == NULL) { - kn->kn_flags = EV_ERROR; - kn->kn_data = ENXIO; - return 0; + knote_set_error(kn, ENXIO); + return 0; } tp = pti->pt_tty; tty_lock(tp); - kn->kn_hookid = dev; - kn->kn_hook = PTSD_KNOTE_VALID; - kn->kn_filtid = EVFILTID_PTSD; - - switch (kn->kn_filter) { - case EVFILT_READ: - KNOTE_ATTACH(&tp->t_rsel.si_note, kn); - break; - case EVFILT_WRITE: - KNOTE_ATTACH(&tp->t_wsel.si_note, kn); - break; - default: - kn->kn_flags = EV_ERROR; - kn->kn_data = EINVAL; - break; - } - - tty_unlock(tp); + assert(tp->t_state & TS_ISOPEN); - ptsd_kevent_mtx_lock(minor(dev)); + kn->kn_filtid = EVFILTID_PTSD; + /* the tty will be freed when detaching the knote */ + ttyhold(tp); + kn->kn_hook = tp; + + switch (kn->kn_filter) { + case EVFILT_READ: + KNOTE_ATTACH(&tp->t_rsel.si_note, kn); + break; + case EVFILT_WRITE: + KNOTE_ATTACH(&tp->t_wsel.si_note, kn); + break; + default: + panic("ptsd kevent: unexpected filter: %d, kn = %p, tty = %p", + kn->kn_filter, kn, tp); + break; + } /* capture current event state */ - retval = ptsd_kqops_common(kn, dev, 0); + ret = ptsd_kqops_common(kn, tp); - ptsd_kevent_mtx_unlock(minor(dev)); + tty_unlock(tp); - return (retval); + return ret; } /* * Support for revoke(2). - * - * Mark all the kn_hook fields so that future invocations of the - * f_event op will just say "EOF" *without* looking at the - * ptmx_ioctl structure (which may disappear or be recycled at - * the end of ptsd_close). Issue wakeups to post that EOF to - * anyone listening. And finally remove the knotes from the - * tty's klists to keep ttyclose() happy, and set the hookid to - * zero to make the final detach passively successful. */ static void -ptsd_revoke_knotes(int minor, struct tty *tp) +ptsd_revoke_knotes(__unused int minor, struct tty *tp) { - struct klist *list; - struct knote *kn, *tkn; - - /* (Hold and drop the right locks in the right order.) */ - - ptsd_kevent_mtx_lock(minor); tty_lock(tp); - list = &tp->t_rsel.si_note; - SLIST_FOREACH(kn, list, kn_selnext) - kn->kn_hook = PTSD_KNOTE_REVOKED; - - list = &tp->t_wsel.si_note; - SLIST_FOREACH(kn, list, kn_selnext) - kn->kn_hook = PTSD_KNOTE_REVOKED; - - tty_unlock(tp); - ptsd_kevent_mtx_unlock(minor); - - tty_lock(tp); ttwakeup(tp); - ttwwakeup(tp); - tty_unlock(tp); + KNOTE(&tp->t_rsel.si_note, NOTE_REVOKE | 1 /* the lock is already held */); - ptsd_kevent_mtx_lock(minor); - tty_lock(tp); - - list = &tp->t_rsel.si_note; - SLIST_FOREACH_SAFE(kn, list, kn_selnext, tkn) { - (void) KNOTE_DETACH(list, kn); - kn->kn_hookid = 0; - } - - list = &tp->t_wsel.si_note; - SLIST_FOREACH_SAFE(kn, list, kn_selnext, tkn) { - (void) KNOTE_DETACH(list, kn); - kn->kn_hookid = 0; - } + ttwwakeup(tp); + KNOTE(&tp->t_wsel.si_note, NOTE_REVOKE | 1); tty_unlock(tp); - ptsd_kevent_mtx_unlock(minor); } diff --git a/bsd/kern/tty_pty.c b/bsd/kern/tty_pty.c index 7c4f14e6a..c5f899a46 100644 --- a/bsd/kern/tty_pty.c +++ b/bsd/kern/tty_pty.c @@ -92,12 +92,9 @@ #warning You have only one pty defined, redefining to 32. #endif -#define PTY_MAJOR 5 -#define TTY_MAJOR 4 - /* - * pts == /dev/tty[pqrsPQRS][0123456789abcdefghijklmnopqrstuv] - * ptc == /dev/pty[pqrsPQRS][0123456789abcdefghijklmnopqrstuv] + * pts == /dev/tty[pqrsPQRS][0-9a-v] + * ptc == /dev/pty[pqrsPQRS][0-9a-v] */ static struct ptmx_ioctl pt_ioctl[NPTY]; @@ -154,23 +151,32 @@ pty_init(int n_ptys) for (j = 0; j < 10; j++) { for (i = 0; i < HEX_BASE; i++) { int m = j * HEX_BASE + i; - if (m >= n_ptys) + if (m >= n_ptys) { goto done; - pt_ioctl[m].pt_devhandle = devfs_make_node(makedev(TTY_MAJOR, m), - DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, - "tty%c%x", j + START_CHAR, i); - (void)devfs_make_node(makedev(PTY_MAJOR, m), - DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, - "pty%c%x", j + START_CHAR, i); + } + pt_ioctl[m].pt_devhandle = devfs_make_node(makedev(PTS_MAJOR, m), + DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, + "tty%c%x", j + START_CHAR, i); + (void)devfs_make_node(makedev(PTC_MAJOR, m), + DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, + "pty%c%x", j + START_CHAR, i); } } + done: - _pty_driver.master = PTY_MAJOR; - _pty_driver.slave = TTY_MAJOR; + _pty_driver.master = PTC_MAJOR; + _pty_driver.slave = PTS_MAJOR; _pty_driver.open_reset = 1; _pty_driver.open = &pty_get_ioctl; _pty_driver.name = &pty_get_name; tty_dev_register(&_pty_driver); - return (0); + + if (cdevsw_setkqueueok(PTC_MAJOR, &cdevsw[PTC_MAJOR], CDEVSW_IS_PTC) == -1) { + panic("Can't mark ptc as kqueue ok"); + } + if (cdevsw_setkqueueok(PTS_MAJOR, &cdevsw[PTS_MAJOR], CDEVSW_IS_PTS) == -1) { + panic("Can't mark pts as kqueue ok"); + } + return 0; } #endif // DEVFS diff --git a/bsd/kern/ubc_subr.c b/bsd/kern/ubc_subr.c index 8e6f56929..e05d13c3e 100644 --- a/bsd/kern/ubc_subr.c +++ b/bsd/kern/ubc_subr.c @@ -65,6 +65,7 @@ #include #include #include +#include #include #include /* last */ @@ -158,6 +159,12 @@ struct cs_hash { cs_md_final cs_final; }; +uint8_t cs_hash_type( + struct cs_hash const * const cs_hash) +{ + return cs_hash->cs_type; +} + static const struct cs_hash cs_hash_sha1 = { .cs_type = CS_HASHTYPE_SHA1, .cs_size = CS_SHA1_LEN, @@ -495,6 +502,9 @@ cs_validate_csblob( uint32_t n, count; const CS_CodeDirectory *best_cd = NULL; unsigned int best_rank = 0; +#if PLATFORM_WatchOS + const CS_CodeDirectory *sha1_cd = NULL; +#endif if (length < sizeof(CS_SuperBlob)) return EBADEXEC; @@ -543,6 +553,15 @@ cs_validate_csblob( printf("multiple hash=%d CodeDirectories in signature; rejecting\n", best_cd->hashType); return EBADEXEC; } +#if PLATFORM_WatchOS + if (candidate->hashType == CS_HASHTYPE_SHA1) { + if (sha1_cd != NULL) { + printf("multiple sha1 CodeDirectories in signature; rejecting\n"); + return EBADEXEC; + } + sha1_cd = candidate; + } +#endif } else if (type == CSSLOT_ENTITLEMENTS) { if (ntohl(subBlob->magic) != CSMAGIC_EMBEDDED_ENTITLEMENTS) { return EBADEXEC; @@ -555,6 +574,37 @@ cs_validate_csblob( } } +#if PLATFORM_WatchOS + /* To keep watchOS fast enough, we have to resort to sha1 for + * some code. + * + * At the time of writing this comment, known sha1 attacks are + * collision attacks (not preimage or second preimage + * attacks), which do not apply to platform binaries since + * they have a fixed hash in the trust cache. Given this + * property, we only prefer sha1 code directories for adhoc + * signatures, which always have to be in a trust cache to be + * valid (can-load-cdhash does not exist for watchOS). Those + * are, incidentally, also the platform binaries, for which we + * care about the performance hit that sha256 would bring us. + * + * Platform binaries may still contain a (not chosen) sha256 + * code directory, which keeps software updates that switch to + * sha256-only small. + */ + + if (*rcd != NULL && sha1_cd != NULL && (ntohl(sha1_cd->flags) & CS_ADHOC)) { + if (sha1_cd->flags != (*rcd)->flags) { + printf("mismatched flags between hash %d (flags: %#x) and sha1 (flags: %#x) cd.\n", + (int)(*rcd)->hashType, (*rcd)->flags, sha1_cd->flags); + *rcd = NULL; + return EBADEXEC; + } + + *rcd = sha1_cd; + } +#endif + } else if (ntohl(blob->magic) == CSMAGIC_CODEDIRECTORY) { if ((error = cs_validate_codedirectory((const CS_CodeDirectory *)(const void *)addr, length)) != 0) @@ -1080,7 +1130,7 @@ errno_t ubc_setsize_ex(struct vnode *vp, off_t nsize, ubc_setsize_opts_t opts) * zero the tail of this page if it's currently * present in the cache */ - kret = ubc_create_upl(vp, lastpg, PAGE_SIZE, &upl, &pl, UPL_SET_LITE); + kret = ubc_create_upl_kernel(vp, lastpg, PAGE_SIZE, &upl, &pl, UPL_SET_LITE, VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("ubc_setsize: ubc_create_upl (error = %d)\n", kret); @@ -2285,13 +2335,26 @@ ubc_range_op( * ubc_upl_abort(), or ubc_upl_abort_range(). */ kern_return_t -ubc_create_upl( +ubc_create_upl_external( struct vnode *vp, off_t f_offset, int bufsize, upl_t *uplp, upl_page_info_t **plp, int uplflags) +{ + return (ubc_create_upl_kernel(vp, f_offset, bufsize, uplp, plp, uplflags, vm_tag_bt())); +} + +kern_return_t +ubc_create_upl_kernel( + struct vnode *vp, + off_t f_offset, + int bufsize, + upl_t *uplp, + upl_page_info_t **plp, + int uplflags, + vm_tag_t tag) { memory_object_control_t control; kern_return_t kr; @@ -2351,7 +2414,7 @@ ubc_create_upl( if (control == MEMORY_OBJECT_CONTROL_NULL) return KERN_INVALID_ARGUMENT; - kr = memory_object_upl_request(control, f_offset, bufsize, uplp, NULL, NULL, uplflags); + kr = memory_object_upl_request(control, f_offset, bufsize, uplp, NULL, NULL, uplflags, tag); if (kr == KERN_SUCCESS && plp != NULL) *plp = UPL_GET_INTERNAL_PAGE_LIST(*uplp); return kr; @@ -3080,6 +3143,7 @@ ubc_cs_blob_add( blob->csb_mem_offset = 0; blob->csb_mem_kaddr = *addr; blob->csb_flags = 0; + blob->csb_signer_type = CS_SIGNER_TYPE_UNKNOWN; blob->csb_platform_binary = 0; blob->csb_platform_path = 0; blob->csb_teamid = NULL; @@ -3127,20 +3191,20 @@ ubc_cs_blob_add( kr); } } else { - memcpy(new_blob_addr, blob->csb_mem_kaddr, size); + memcpy((void *)new_blob_addr, (void *)blob->csb_mem_kaddr, size); if (cd == NULL) { new_cd = NULL; } else { - new_cd = ((uintptr_t)cd + new_cd = (void *)(((uintptr_t)cd - (uintptr_t)blob->csb_mem_kaddr - + (uintptr_t)new_blob_addr); + + (uintptr_t)new_blob_addr)); } if (entitlements == NULL) { new_entitlements = NULL; } else { - new_entitlements = ((uintptr_t)entitlements + new_entitlements = (void *)(((uintptr_t)entitlements - (uintptr_t)blob->csb_mem_kaddr - + (uintptr_t)new_blob_addr); + + (uintptr_t)new_blob_addr)); } // printf("CODE SIGNING: %s:%d kaddr 0x%llx cd %p ents %p -> blob 0x%llx cd %p ents %p\n", __FUNCTION__, __LINE__, (uint64_t)blob->csb_mem_kaddr, cd, entitlements, (uint64_t)new_blob_addr, new_cd, new_entitlements); ubc_cs_blob_deallocate(blob->csb_mem_kaddr, @@ -3187,8 +3251,10 @@ ubc_cs_blob_add( */ #if CONFIG_MACF unsigned int cs_flags = blob->csb_flags; - error = mac_vnode_check_signature(vp, blob, imgp, &cs_flags, flags); + unsigned int signer_type = blob->csb_signer_type; + error = mac_vnode_check_signature(vp, blob, imgp, &cs_flags, &signer_type, flags); blob->csb_flags = cs_flags; + blob->csb_signer_type = signer_type; if (error) { if (cs_debug) @@ -3201,8 +3267,8 @@ ubc_cs_blob_add( error = EPERM; goto out; } -#endif - +#endif + if (blob->csb_flags & CS_PLATFORM_BINARY) { if (cs_debug > 1) printf("check_signature[pid: %d]: platform binary\n", current_proc()->p_pid); @@ -3252,8 +3318,11 @@ ubc_cs_blob_add( oblob = oblob->csb_next) { off_t oblob_start_offset, oblob_end_offset; - /* check for conflicting teamid */ - if (blob->csb_platform_binary) { //platform binary needs to be the same for app slices + if (blob->csb_signer_type != oblob->csb_signer_type) { // signer type needs to be the same for slices + vnode_unlock(vp); + error = EALREADY; + goto out; + } else if (blob->csb_platform_binary) { //platform binary needs to be the same for app slices if (!oblob->csb_platform_binary) { vnode_unlock(vp); error = EALREADY; @@ -3575,20 +3644,22 @@ ubc_cs_blob_revalidate( assert(size == blob->csb_mem_size); unsigned int cs_flags = (ntohl(cd->flags) & CS_ALLOWED_MACHO) | CS_VALID; - + unsigned int signer_type = CS_SIGNER_TYPE_UNKNOWN; /* callout to mac_vnode_check_signature */ #if CONFIG_MACF - error = mac_vnode_check_signature(vp, blob, imgp, &cs_flags, flags); + error = mac_vnode_check_signature(vp, blob, imgp, &cs_flags, &signer_type, flags); if (cs_debug && error) { printf("revalidate: check_signature[pid: %d], error = %d\n", current_proc()->p_pid, error); } #else (void)flags; + (void)signer_type; #endif /* update generation number if success */ vnode_lock_spin(vp); blob->csb_flags = cs_flags; + blob->csb_signer_type = signer_type; if (UBCINFOEXISTS(vp)) { if (error == 0) vp->v_ubcinfo->cs_add_gen = cs_blob_generation_count; @@ -3957,7 +4028,7 @@ ubc_cs_is_range_codesigned( } #if CHECK_CS_VALIDATION_BITMAP -#define stob(s) ((atop_64((s)) + 07) >> 3) +#define stob(s) (((atop_64(round_page_64(s))) + 07) >> 3) extern boolean_t root_fs_upgrade_try; /* diff --git a/bsd/kern/uipc_domain.c b/bsd/kern/uipc_domain.c index 7fde6ee3e..e03a08e6d 100644 --- a/bsd/kern/uipc_domain.c +++ b/bsd/kern/uipc_domain.c @@ -76,10 +76,14 @@ #include #include +#include #include #include +/* Eventhandler context for protocol events */ +struct eventhandler_lists_ctxt protoctl_evhdlr_ctxt; + static void pr_init_old(struct protosw *, struct domain *); static void init_proto(struct protosw *, struct domain *); static void attach_proto(struct protosw *, struct domain *); @@ -102,7 +106,7 @@ static lck_grp_attr_t *domain_proto_mtx_grp_attr; decl_lck_mtx_data(static, domain_proto_mtx); decl_lck_mtx_data(static, domain_timeout_mtx); -static u_int64_t _net_uptime; +u_int64_t _net_uptime; #if (DEVELOPMENT || DEBUG) @@ -637,7 +641,7 @@ done: static void domain_sched_timeout(void) { - lck_mtx_assert(&domain_timeout_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_timeout_mtx, LCK_MTX_ASSERT_OWNED); if (!domain_timeout_run && domain_draining) { domain_timeout_run = TRUE; @@ -705,6 +709,7 @@ domaininit(void) struct domain *dp; domain_guard_t guard; + eventhandler_lists_ctxt_init(&protoctl_evhdlr_ctxt); /* * allocate lock group attribute and group for domain mutexes */ @@ -940,31 +945,32 @@ pfctlinput2(int cmd, struct sockaddr *sa, void *ctlparam) TAILQ_FOREACH(dp, &domains, dom_entry) { TAILQ_FOREACH(pp, &dp->dom_protosw, pr_entry) { if (pp->pr_ctlinput != NULL) - (*pp->pr_ctlinput)(cmd, sa, ctlparam); + (*pp->pr_ctlinput)(cmd, sa, ctlparam, NULL); } } domain_guard_release(guard); } void -net_update_uptime(void) +net_update_uptime_with_time(const struct timeval *tvp) { - struct timeval tv; - - microuptime(&tv); - _net_uptime = tv.tv_sec; + _net_uptime = tvp->tv_sec; /* * Round up the timer to the nearest integer value because otherwise * we might setup networking timers that are off by almost 1 second. */ - if (tv.tv_usec > 500000) + if (tvp->tv_usec > 500000) _net_uptime++; } void -net_update_uptime_secs(uint64_t secs) +net_update_uptime(void) { - _net_uptime = secs; + struct timeval tv; + + microuptime(&tv); + + net_update_uptime_with_time(&tv); } /* @@ -997,13 +1003,13 @@ net_uptime(void) void domain_proto_mtx_lock_assert_held(void) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); } void domain_proto_mtx_lock_assert_notheld(void) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); } domain_guard_t @@ -1013,11 +1019,11 @@ domain_guard_deploy(void) marks = net_thread_marks_push(NET_THREAD_HELD_DOMAIN); if (marks != net_thread_marks_none) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(&domain_proto_mtx); } else - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); return ((domain_guard_t)(const void*)marks); } @@ -1028,12 +1034,12 @@ domain_guard_release(domain_guard_t guard) net_thread_marks_t marks = (net_thread_marks_t)(const void*)guard; if (marks != net_thread_marks_none) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); lck_mtx_unlock(&domain_proto_mtx); net_thread_marks_pop(marks); } else - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); } domain_unguard_t @@ -1043,11 +1049,11 @@ domain_unguard_deploy(void) marks = net_thread_unmarks_push(NET_THREAD_HELD_DOMAIN); if (marks != net_thread_marks_none) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); lck_mtx_unlock(&domain_proto_mtx); } else - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); return ((domain_unguard_t)(const void*)marks); } @@ -1058,14 +1064,15 @@ domain_unguard_release(domain_unguard_t unguard) net_thread_marks_t marks = (net_thread_marks_t)(const void*)unguard; if (marks != net_thread_marks_none) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(&domain_proto_mtx); net_thread_unmarks_pop(marks); } else - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); } + #if (DEVELOPMENT || DEBUG) static int @@ -1085,4 +1092,3 @@ sysctl_do_drain_domains SYSCTL_HANDLER_ARGS } #endif /* DEVELOPMENT || DEBUG */ - \ No newline at end of file diff --git a/bsd/kern/uipc_mbuf.c b/bsd/kern/uipc_mbuf.c index cee3242d1..04df5a515 100644 --- a/bsd/kern/uipc_mbuf.c +++ b/bsd/kern/uipc_mbuf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2016 Apple Inc. All rights reserved. + * Copyright (c) 1998-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -317,9 +317,6 @@ static lck_grp_attr_t *mbuf_mlock_grp_attr; /* Back-end (common) layer */ static boolean_t mbuf_worker_needs_wakeup; /* wait channel for mbuf worker */ static int mbuf_worker_ready; /* worker thread is runnable */ -static int mbuf_expand_mcl; /* number of cluster creation requets */ -static int mbuf_expand_big; /* number of big cluster creation requests */ -static int mbuf_expand_16k; /* number of 16KB cluster creation requests */ static int ncpu; /* number of CPUs */ static ppnum_t *mcl_paddr; /* Array of cluster physical addresses */ static ppnum_t mcl_pages; /* Size of array (# physical pages) */ @@ -496,7 +493,7 @@ static struct mleak_table mleak_table; static mleak_stat_t *mleak_stat; #define MLEAK_STAT_SIZE(n) \ - ((size_t)(&((mleak_stat_t *)0)->ml_trace[n])) + __builtin_offsetof(mleak_stat_t, ml_trace[n]) struct mallocation { mcache_obj_t *element; /* the alloc'ed element, NULL if unused */ @@ -588,6 +585,7 @@ typedef struct { int mtbl_maxlimit; /* maximum allowed */ u_int32_t mtbl_wantpurge; /* purge during next reclaim */ uint32_t mtbl_avgtotal; /* average total on iOS */ + u_int32_t mtbl_expand; /* worker should expand the class */ } mbuf_table_t; #define m_class(c) mbuf_table[c].mtbl_class @@ -613,6 +611,7 @@ typedef struct { #define m_ctotal(c) mbuf_table[c].mtbl_stats->mbcl_ctotal #define m_peak(c) mbuf_table[c].mtbl_stats->mbcl_peak_reported #define m_release_cnt(c) mbuf_table[c].mtbl_stats->mbcl_release_cnt +#define m_region_expand(c) mbuf_table[c].mtbl_expand static mbuf_table_t mbuf_table[] = { /* @@ -621,13 +620,13 @@ static mbuf_table_t mbuf_table[] = { * usage patterns on iOS. */ { MC_MBUF, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_MBUF)), - NULL, NULL, 0, 0, 0, 0, 3000 }, + NULL, NULL, 0, 0, 0, 0, 3000, 0 }, { MC_CL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_CL)), - NULL, NULL, 0, 0, 0, 0, 2000 }, + NULL, NULL, 0, 0, 0, 0, 2000, 0 }, { MC_BIGCL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_BIGCL)), - NULL, NULL, 0, 0, 0, 0, 1000 }, + NULL, NULL, 0, 0, 0, 0, 1000, 0 }, { MC_16KCL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_16KCL)), - NULL, NULL, 0, 0, 0, 0, 1000 }, + NULL, NULL, 0, 0, 0, 0, 200, 0 }, /* * The following are special caches; they serve as intermediate * caches backed by the above rudimentary caches. Each object @@ -636,9 +635,9 @@ static mbuf_table_t mbuf_table[] = { * deal with the slab structures; instead, the constructed * cached elements are simply stored in the freelists. */ - { MC_MBUF_CL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 2000 }, - { MC_MBUF_BIGCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 1000 }, - { MC_MBUF_16KCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 1000 }, + { MC_MBUF_CL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 2000, 0 }, + { MC_MBUF_BIGCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 1000, 0 }, + { MC_MBUF_16KCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 200, 0 }, }; #define NELEM(a) (sizeof (a) / sizeof ((a)[0])) @@ -665,8 +664,13 @@ static char *mbuf_dump_buf; * mb_drain_maxint controls the amount of time to wait (in seconds) before * consecutive calls to m_drain(). */ +#if CONFIG_EMBEDDED +static unsigned int mb_watchdog = 1; +static unsigned int mb_drain_maxint = 60; +#else static unsigned int mb_watchdog = 0; static unsigned int mb_drain_maxint = 0; +#endif /* CONFIG_EMBEDDED */ uintptr_t mb_obscure_extfree __attribute__((visibility("hidden"))); uintptr_t mb_obscure_extref __attribute__((visibility("hidden"))); @@ -932,7 +936,7 @@ struct mb_stat *mb_stat; struct omb_stat *omb_stat; /* For backwards compatibility */ #define MB_STAT_SIZE(n) \ - ((size_t)(&((mb_stat_t *)0)->mbs_class[n])) + __builtin_offsetof(mb_stat_t, mbs_class[n]) #define OMB_STAT_SIZE(n) \ ((size_t)(&((struct omb_stat *)0)->mbs_class[n])) @@ -999,7 +1003,7 @@ mbuf_mtypes_sync(boolean_t locked) mtypes_cpu_t mtc; if (locked) - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); bzero(&mtc, sizeof (mtc)); for (m = 0; m < ncpu; m++) { @@ -1037,7 +1041,7 @@ mbuf_stat_sync(void) mcache_t *cp; int k, m, bktsize; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); for (k = 0; k < NELEM(mbuf_table); k++) { cp = m_cache(k); @@ -1455,6 +1459,7 @@ mbinit(void) _CASSERT(MBUF_TSO_IPV6 == CSUM_TSO_IPV6); _CASSERT(MBUF_CSUM_REQ_SUM16 == CSUM_PARTIAL); _CASSERT(MBUF_CSUM_TCP_SUM16 == MBUF_CSUM_REQ_SUM16); + _CASSERT(MBUF_CSUM_REQ_ZERO_INVERT == CSUM_ZERO_INVERT); _CASSERT(MBUF_CSUM_REQ_IP == CSUM_IP); _CASSERT(MBUF_CSUM_REQ_TCP == CSUM_TCP); _CASSERT(MBUF_CSUM_REQ_UDP == CSUM_UDP); @@ -1571,6 +1576,21 @@ mbinit(void) mleak_activate(); + /* + * Allocate structure for per-CPU statistics that's aligned + * on the CPU cache boundary; this code assumes that we never + * uninitialize this framework, since the original address + * before alignment is not saved. + */ + ncpu = ml_get_max_cpus(); + MALLOC(buf, void *, MBUF_MTYPES_SIZE(ncpu) + CPU_CACHE_LINE_SIZE, + M_TEMP, M_WAITOK); + VERIFY(buf != NULL); + + mbuf_mtypes = (mbuf_mtypes_t *)P2ROUNDUP((intptr_t)buf, + CPU_CACHE_LINE_SIZE); + bzero(mbuf_mtypes, MBUF_MTYPES_SIZE(ncpu)); + /* Calculate the number of pages assigned to the cluster pool */ mcl_pages = (nmbclusters << MCLSHIFT) / PAGE_SIZE; MALLOC(mcl_paddr, ppnum_t *, mcl_pages * sizeof (ppnum_t), @@ -1661,21 +1681,6 @@ mbinit(void) (void *)(uintptr_t)m, flags, MCR_SLEEP); } - /* - * Allocate structure for per-CPU statistics that's aligned - * on the CPU cache boundary; this code assumes that we never - * uninitialize this framework, since the original address - * before alignment is not saved. - */ - ncpu = ml_get_max_cpus(); - MALLOC(buf, void *, MBUF_MTYPES_SIZE(ncpu) + CPU_CACHE_LINE_SIZE, - M_TEMP, M_WAITOK); - VERIFY(buf != NULL); - - mbuf_mtypes = (mbuf_mtypes_t *)P2ROUNDUP((intptr_t)buf, - CPU_CACHE_LINE_SIZE); - bzero(mbuf_mtypes, MBUF_MTYPES_SIZE(ncpu)); - /* * Set the max limit on sb_max to be 1/16 th of the size of * memory allocated for mbuf clusters. @@ -1742,7 +1747,7 @@ slab_alloc(mbuf_class_t class, int wait) mcl_slab_t *sp; mcache_obj_t *buf; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* This should always be NULL for us */ VERIFY(m_cobjlist(class) == NULL); @@ -1876,7 +1881,7 @@ slab_free(mbuf_class_t class, mcache_obj_t *buf) boolean_t reinit_supercl = false; mbuf_class_t super_class; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); VERIFY(class != MC_16KCL || njcl > 0); VERIFY(buf->obj_next == NULL); @@ -2147,7 +2152,7 @@ mbuf_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) mbuf_sleep(class, need, wait)) break; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); } } @@ -2280,7 +2285,7 @@ cslab_alloc(mbuf_class_t class, mcache_obj_t ***plist, unsigned int num) VERIFY(need > 0); VERIFY(class != MC_MBUF_16KCL || njcl > 0); - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* Get what we can from the freelist */ while ((*list = m_cobjlist(class)) != NULL) { @@ -2344,7 +2349,7 @@ cslab_free(mbuf_class_t class, mcache_obj_t *list, int purged) ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class)); VERIFY(class != MC_MBUF_16KCL || njcl > 0); - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); if (class == MC_MBUF_CL) { cl_class = MC_CL; @@ -2805,7 +2810,7 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) else class = MC_16KCL; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* * Multiple threads may attempt to populate the cluster map one @@ -2819,7 +2824,7 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) mb_clalloc_waiters++; (void) msleep(mb_clalloc_waitchan, mbuf_mlock, (PZERO-1), "m_clalloc", NULL); - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); } /* We are busy now; tell everyone else to go away */ @@ -2830,7 +2835,7 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) * to grow the pool asynchronously using the mbuf worker thread. */ i = m_howmany(num, bufsize); - if (i == 0 || (wait & M_DONTWAIT)) + if (i <= 0 || (wait & M_DONTWAIT)) goto out; lck_mtx_unlock(mbuf_mlock); @@ -3000,7 +3005,7 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) return (count); out: - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* We're done; let others enter */ mb_clalloc_busy = FALSE; @@ -3024,8 +3029,8 @@ out: * at this time. */ i += m_total(MC_BIGCL); - if (i > mbuf_expand_big) { - mbuf_expand_big = i; + if (i > m_region_expand(MC_BIGCL)) { + m_region_expand(MC_BIGCL) = i; } } if (m_infree(MC_BIGCL) >= num) @@ -3037,8 +3042,8 @@ out: * at this time. */ i += m_total(MC_16KCL); - if (i > mbuf_expand_16k) { - mbuf_expand_16k = i; + if (i > m_region_expand(MC_16KCL)) { + m_region_expand(MC_16KCL) = i; } } if (m_infree(MC_16KCL) >= num) @@ -3060,7 +3065,7 @@ freelist_populate(mbuf_class_t class, unsigned int num, int wait) VERIFY(class == MC_MBUF || class == MC_CL || class == MC_BIGCL || class == MC_16KCL); - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); VERIFY(PAGE_SIZE == m_maxsize(MC_BIGCL) || PAGE_SIZE == m_maxsize(MC_16KCL)); @@ -3083,12 +3088,6 @@ freelist_populate(mbuf_class_t class, unsigned int num, int wait) i = m_clalloc(numpages, wait, m_maxsize(super_class)); - /* Respect the minimum limit of super class */ - if (m_total(super_class) == m_maxlimit(super_class) && - m_infree(super_class) <= m_minlimit(super_class)) - if (wait & MCR_COMP) - return (0); - /* how many objects will we cut the page into? */ int numobj = PAGE_SIZE / m_maxsize(class); @@ -3138,6 +3137,7 @@ freelist_populate(mbuf_class_t class, unsigned int num, int wait) mbstat.m_bigclusters = m_total(MC_BIGCL); m_total(class) += numobj; + VERIFY(m_total(class) <= m_maxlimit(class)); m_infree(class) += numobj; if (!mb_peak_newreport && mbuf_report_usage(class)) @@ -3204,7 +3204,7 @@ freelist_populate(mbuf_class_t class, unsigned int num, int wait) static void freelist_init(mbuf_class_t class) { - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); VERIFY(class == MC_CL || class == MC_BIGCL); VERIFY(m_total(class) == 0); @@ -3269,7 +3269,7 @@ mbuf_steal(mbuf_class_t class, unsigned int num) mcache_obj_t **list = ⊤ unsigned int tot = 0; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); switch (class) { case MC_MBUF: @@ -3305,7 +3305,7 @@ m_reclaim(mbuf_class_t class, unsigned int num, boolean_t comp) { int m, bmap = 0; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); VERIFY(m_total(MC_CL) <= m_maxlimit(MC_CL)); VERIFY(m_total(MC_BIGCL) <= m_maxlimit(MC_BIGCL)); @@ -3962,10 +3962,11 @@ m_classifier_init(struct mbuf *m, uint32_t pktf_mask) (void) m_set_service_class(m, MBUF_SC_BE); if (!(m->m_pkthdr.pkt_flags & PKTF_IFAINFO)) m->m_pkthdr.pkt_ifainfo = 0; -#if MEASURE_BW - m->m_pkthdr.pkt_bwseq = 0; -#endif /* MEASURE_BW */ - m->m_pkthdr.pkt_timestamp = 0; + /* + * Preserve timestamp if requested + */ + if (!(m->m_pkthdr.pkt_flags & PKTF_TS_VALID)) + m->m_pkthdr.pkt_timestamp = 0; } void @@ -3980,9 +3981,6 @@ m_copy_classifier(struct mbuf *to, struct mbuf *from) to->m_pkthdr.pkt_flags = from->m_pkthdr.pkt_flags; (void) m_set_service_class(to, from->m_pkthdr.pkt_svc); to->m_pkthdr.pkt_ifainfo = from->m_pkthdr.pkt_ifainfo; -#if MEASURE_BW - to->m_pkthdr.pkt_bwseq = from->m_pkthdr.pkt_bwseq; -#endif /* MEASURE_BW */ } /* @@ -5039,23 +5037,33 @@ nospace: void m_copydata(struct mbuf *m, int off, int len, void *vp) { + int off0 = off, len0 = len; + struct mbuf *m0 = m; unsigned count; char *cp = vp; - if (off < 0 || len < 0) - panic("m_copydata: invalid offset %d or len %d", off, len); + if (__improbable(off < 0 || len < 0)) { + panic("%s: invalid offset %d or len %d", __func__, off, len); + /* NOTREACHED */ + } while (off > 0) { - if (m == NULL) - panic("m_copydata: invalid mbuf chain"); + if (__improbable(m == NULL)) { + panic("%s: invalid mbuf chain %p [off %d, len %d]", + __func__, m0, off0, len0); + /* NOTREACHED */ + } if (off < m->m_len) break; off -= m->m_len; m = m->m_next; } while (len > 0) { - if (m == NULL) - panic("m_copydata: invalid mbuf chain"); + if (__improbable(m == NULL)) { + panic("%s: invalid mbuf chain %p [off %d, len %d]", + __func__, m0, off0, len0); + /* NOTREACHED */ + } count = MIN(m->m_len - off, len); bcopy(MTOD(m, caddr_t) + off, cp, count); len -= count; @@ -5445,7 +5453,7 @@ m_howmany(int num, size_t bufsize) VERIFY(bufsize == m_maxsize(MC_BIGCL) || bufsize == m_maxsize(MC_16KCL)); - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* Numbers in 2K cluster units */ m_mbclusters = m_total(MC_MBUF) >> NMBPCLSHIFT; @@ -5535,9 +5543,8 @@ m_howmany(int num, size_t bufsize) i = MAX(i, j); /* Check to ensure we don't go over limit */ - if (i + m_16kclusters >= m_maxlimit(MC_16KCL)) - i = m_maxlimit(MC_16KCL) - m_16kclusters; - VERIFY((m_total(MC_16KCL) + i) <= m_maxlimit(MC_16KCL)); + if ((i + m_total(MC_16KCL)) >= m_maxlimit(MC_16KCL)) + i = m_maxlimit(MC_16KCL) - m_total(MC_16KCL); } return (i); } @@ -6196,6 +6203,8 @@ m_defrag_offset(struct mbuf *m0, u_int32_t off, int how) if (length > MCLBYTES) length = MCLBYTES; length -= ((m_new == m_final) ? off : 0); + if (length < 0) + goto nospace; if (m_new == NULL) { if (length > MLEN) @@ -6357,7 +6366,7 @@ mbuf_sleep(mbuf_class_t class, unsigned int num, int wait) { boolean_t mcache_retry = FALSE; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* Check if there's anything at the cache layer */ if (mbuf_cached_above(class, wait)) { @@ -6399,6 +6408,14 @@ mbuf_sleep(mbuf_class_t class, unsigned int num, int wait) mbuf_watchdog(); mb_waiters++; + m_region_expand(class) += num; + /* wake up the worker thread */ + if (class > MC_MBUF && mbuf_worker_ready && + mbuf_worker_needs_wakeup) { + wakeup((caddr_t)&mbuf_worker_needs_wakeup); + mbuf_worker_needs_wakeup = FALSE; + } + (void) msleep(mb_waitchan, mbuf_mlock, (PZERO-1), m_cname(class), NULL); /* We are now up; stop getting notified until next round */ @@ -6425,41 +6442,41 @@ mbuf_worker_thread(void) while (1) { lck_mtx_lock(mbuf_mlock); mbuf_expand = 0; - if (mbuf_expand_mcl) { + if (m_region_expand(MC_CL) > 0) { int n; /* Adjust to current number of cluster in use */ - n = mbuf_expand_mcl - + n = m_region_expand(MC_CL) - (m_total(MC_CL) - m_infree(MC_CL)); if ((n + m_total(MC_CL)) > m_maxlimit(MC_CL)) n = m_maxlimit(MC_CL) - m_total(MC_CL); - mbuf_expand_mcl = 0; + m_region_expand(MC_CL) = 0; if (n > 0 && freelist_populate(MC_CL, n, M_WAIT) > 0) mbuf_expand++; } - if (mbuf_expand_big) { + if (m_region_expand(MC_BIGCL) > 0) { int n; /* Adjust to current number of 4 KB cluster in use */ - n = mbuf_expand_big - + n = m_region_expand(MC_BIGCL) - (m_total(MC_BIGCL) - m_infree(MC_BIGCL)); if ((n + m_total(MC_BIGCL)) > m_maxlimit(MC_BIGCL)) n = m_maxlimit(MC_BIGCL) - m_total(MC_BIGCL); - mbuf_expand_big = 0; + m_region_expand(MC_BIGCL) = 0; if (n > 0 && freelist_populate(MC_BIGCL, n, M_WAIT) > 0) mbuf_expand++; } - if (mbuf_expand_16k) { + if (m_region_expand(MC_16KCL) > 0) { int n; /* Adjust to current number of 16 KB cluster in use */ - n = mbuf_expand_16k - + n = m_region_expand(MC_16KCL) - (m_total(MC_16KCL) - m_infree(MC_16KCL)); if ((n + m_total(MC_16KCL)) > m_maxlimit(MC_16KCL)) n = m_maxlimit(MC_16KCL) - m_total(MC_16KCL); - mbuf_expand_16k = 0; + m_region_expand(MC_16KCL) = 0; if (n > 0) (void) freelist_populate(MC_16KCL, n, M_WAIT); @@ -6501,7 +6518,7 @@ slab_get(void *buf) mcl_slabg_t *slg; unsigned int ix, k; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); VERIFY(MBUF_IN_MAP(buf)); ix = ((unsigned char *)buf - mbutl) >> MBSHIFT; @@ -6686,7 +6703,7 @@ mcl_audit_init(void *buf, mcache_audit_t **mca_list, VERIFY(ix < maxclaudit); /* Make sure we haven't been here before */ - for (i = 0; i < NMBPG; i++) + for (i = 0; i < num; i++) VERIFY(mclaudit[ix].cl_audit[i] == NULL); mca = mca_tail = *mca_list; @@ -7616,8 +7633,8 @@ m_set_ext(struct mbuf *m, struct ext_ref *rfa, m_ext_free_func_t ext_free, m->m_ext.ext_free = (m_ext_free_func_t) (((uintptr_t)ext_free) ^ rfa->ext_token); if (ext_arg != NULL) { - m->m_ext.ext_arg = (((uintptr_t)ext_arg) ^ - rfa->ext_token); + m->m_ext.ext_arg = + (caddr_t)(((uintptr_t)ext_arg) ^ rfa->ext_token); } else { m->m_ext.ext_arg = NULL; } @@ -7633,10 +7650,12 @@ m_set_ext(struct mbuf *m, struct ext_ref *rfa, m_ext_free_func_t ext_free, * to obscure the ext_free and ext_arg pointers. */ if (ext_free != NULL) { - m->m_ext.ext_free = ((uintptr_t)ext_free ^ + m->m_ext.ext_free = + (m_ext_free_func_t)((uintptr_t)ext_free ^ mb_obscure_extfree); if (ext_arg != NULL) { - m->m_ext.ext_arg = ((uintptr_t)ext_arg ^ + m->m_ext.ext_arg = + (caddr_t)((uintptr_t)ext_arg ^ mb_obscure_extfree); } else { m->m_ext.ext_arg = NULL; @@ -7667,7 +7686,7 @@ m_get_ext_free(struct mbuf *m) rfa = m_get_rfa(m); if (rfa == NULL) - return ((uintptr_t)m->m_ext.ext_free ^ mb_obscure_extfree); + return ((m_ext_free_func_t)((uintptr_t)m->m_ext.ext_free ^ mb_obscure_extfree)); else return ((m_ext_free_func_t)(((uintptr_t)m->m_ext.ext_free) ^ rfa->ext_token)); @@ -7682,7 +7701,7 @@ m_get_ext_arg(struct mbuf *m) rfa = m_get_rfa(m); if (rfa == NULL) { - return ((uintptr_t)m->m_ext.ext_arg ^ mb_obscure_extfree); + return ((caddr_t)((uintptr_t)m->m_ext.ext_arg ^ mb_obscure_extfree)); } else { return ((caddr_t)(((uintptr_t)m->m_ext.ext_arg) ^ rfa->ext_token)); @@ -7919,8 +7938,14 @@ m_drain(void) 0); nsp->sl_flags = 0; } - if (mclaudit != NULL) - mcl_audit_free(sp->sl_base, 1); + if (mclaudit != NULL) { + if (sp->sl_len == PAGE_SIZE) { + mcl_audit_free(sp->sl_base, + NMBPG); + } else { + mcl_audit_free(sp->sl_base, 1); + } + } break; default: /* diff --git a/bsd/kern/uipc_mbuf2.c b/bsd/kern/uipc_mbuf2.c index 74ac53ba7..fc22ee904 100644 --- a/bsd/kern/uipc_mbuf2.c +++ b/bsd/kern/uipc_mbuf2.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -664,9 +664,8 @@ m_tag_init(struct mbuf *m, int all) * (e.g. m_dup_pkthdr), don't zero them out. */ if (all) { - bzero(m_pftag(m), sizeof (struct pf_mtag)); - bzero(&m->m_pkthdr.proto_mtag, sizeof (m->m_pkthdr.proto_mtag)); - bzero(&m->m_pkthdr.necp_mtag, sizeof (m->m_pkthdr.necp_mtag)); + bzero(&m->m_pkthdr.builtin_mtag._net_mtag, + sizeof (m->m_pkthdr.builtin_mtag._net_mtag)); } } @@ -825,27 +824,75 @@ m_service_class_from_val(u_int32_t v) } uint16_t -m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t ulpoff, uint32_t sum) +m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t dataoff, + uint32_t datalen, uint32_t sum) { - int len = (ulpoff - start); + uint32_t total_sub = 0; /* total to subtract */ + uint32_t mlen = m_pktlen(m); /* frame length */ + uint32_t bytes = (dataoff + datalen); /* bytes covered by sum */ + int len; + ASSERT(bytes <= mlen); + + /* + * Take care of excluding (len > 0) or including (len < 0) + * extraneous octets at the beginning of the packet, taking + * into account the start offset. + */ + len = (dataoff - start); + if (len > 0) + total_sub = m_sum16(m, start, len); + else if (len < 0) + sum += m_sum16(m, dataoff, -len); + + /* + * Take care of excluding any postpended extraneous octets. + */ + len = (mlen - bytes); if (len > 0) { - uint32_t adj = m_sum16(m, start, len); - if (adj >= sum) - sum = ~(adj - sum) & 0xffff; + struct mbuf *m0 = m; + uint32_t extra = m_sum16(m, bytes, len); + uint32_t off = bytes, off0 = off; + + while (off > 0) { + if (__improbable(m == NULL)) { + panic("%s: invalid mbuf chain %p [off %u, " + "len %u]", __func__, m0, off0, len); + /* NOTREACHED */ + } + if (off < m->m_len) + break; + off -= m->m_len; + m = m->m_next; + } + + /* if we started on odd-alignment, swap the value */ + if ((uintptr_t)(mtod(m, uint8_t *) + off) & 1) + total_sub += ((extra << 8) & 0xffff) | (extra >> 8); + else + total_sub += extra; + + total_sub = (total_sub >> 16) + (total_sub & 0xffff); + } + + /* + * 1's complement subtract any extraneous octets. + */ + if (total_sub != 0) { + if (total_sub >= sum) + sum = ~(total_sub - sum) & 0xffff; else - sum -= adj; - } else if (len < 0) { - sum += m_sum16(m, ulpoff, -len); + sum -= total_sub; } - ADDCARRY(sum); + /* fold 32-bit to 16-bit */ + sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */ + sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */ + sum = (sum >> 16) + (sum & 0xffff); /* final carry */ - return (sum); + return (sum & 0xffff); } -extern int cpu_in_cksum(struct mbuf *m, int len, int off, uint32_t initial_sum); - uint16_t m_sum16(struct mbuf *m, uint32_t off, uint32_t len) { @@ -859,9 +906,10 @@ m_sum16(struct mbuf *m, uint32_t off, uint32_t len) * a M_PKTHDR one. */ if ((mlen = m_length2(m, NULL)) < (off + len)) { - panic("%s: mbuf len (%d) < off+len (%d+%d)\n", __func__, - mlen, off, len); + panic("%s: mbuf %p len (%d) < off+len (%d+%d)\n", __func__, + m, mlen, off, len); + /* NOTREACHED */ } - return (~cpu_in_cksum(m, len, off, 0) & 0xffff); + return (os_cpu_in_cksum_mbuf(m, len, off, 0)); } diff --git a/bsd/kern/uipc_socket.c b/bsd/kern/uipc_socket.c index aa41e355a..1c5af8cf3 100644 --- a/bsd/kern/uipc_socket.c +++ b/bsd/kern/uipc_socket.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2016 Apple Inc. All rights reserved. + * Copyright (c) 1998-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -98,6 +98,7 @@ #include #include #include +#include #include #include #include @@ -119,9 +120,9 @@ #include #include #include +#include #if CONFIG_MACF -#include #include #endif /* MAC */ @@ -159,19 +160,19 @@ static lck_mtx_t *so_cache_mtx; #include -static int filt_sorattach(struct knote *kn); +static int filt_sorattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_sordetach(struct knote *kn); static int filt_soread(struct knote *kn, long hint); static int filt_sortouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_sorprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static int filt_sowattach(struct knote *kn); +static int filt_sowattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_sowdetach(struct knote *kn); static int filt_sowrite(struct knote *kn, long hint); static int filt_sowtouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_sowprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static int filt_sockattach(struct knote *kn); +static int filt_sockattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_sockdetach(struct knote *kn); static int filt_sockev(struct knote *kn, long hint); static int filt_socktouch(struct knote *kn, struct kevent_internal_s *kev); @@ -180,7 +181,7 @@ static int filt_sockprocess(struct knote *kn, struct filt_process_s *data, struc static int sooptcopyin_timeval(struct sockopt *, struct timeval *); static int sooptcopyout_timeval(struct sockopt *, const struct timeval *); -struct filterops soread_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) soread_filtops = { .f_isfd = 1, .f_attach = filt_sorattach, .f_detach = filt_sordetach, @@ -189,7 +190,7 @@ struct filterops soread_filtops = { .f_process = filt_sorprocess, }; -struct filterops sowrite_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) sowrite_filtops = { .f_isfd = 1, .f_attach = filt_sowattach, .f_detach = filt_sowdetach, @@ -198,7 +199,7 @@ struct filterops sowrite_filtops = { .f_process = filt_sowprocess, }; -struct filterops sock_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) sock_filtops = { .f_isfd = 1, .f_attach = filt_sockattach, .f_detach = filt_sockdetach, @@ -207,7 +208,7 @@ struct filterops sock_filtops = { .f_process = filt_sockprocess, }; -struct filterops soexcept_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) soexcept_filtops = { .f_isfd = 1, .f_attach = filt_sorattach, .f_detach = filt_sordetach, @@ -368,6 +369,7 @@ SYSCTL_STRUCT(_kern_ipc, OID_AUTO, extbkidlestat, CTLFLAG_RD | CTLFLAG_LOCKED, int so_set_extended_bk_idle(struct socket *, int); + /* * SOTCDB_NO_DSCP is set by default, to prevent the networking stack from * setting the DSCP code on the packet based on the service class; see @@ -630,6 +632,12 @@ soalloc(int waitok, int dom, int type) if (so != NULL) { so->so_gencnt = OSIncrementAtomic64((SInt64 *)&so_gencnt); so->so_zone = socket_zone; + + /* + * Increment the socket allocation statistics + */ + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_alloc_total); + #if CONFIG_MACF_SOCKET /* Convert waitok to M_WAITOK/M_NOWAIT for MAC Framework. */ if (mac_socket_label_init(so, !waitok) != 0) { @@ -677,19 +685,48 @@ socreate_internal(int dom, struct socket **aso, int type, int proto, if (so == NULL) return (ENOBUFS); + switch (dom) { + case PF_LOCAL: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_local_total); + break; + case PF_INET: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_inet_total); + if (type == SOCK_STREAM) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_stream_total); + } else { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_total); + } + break; + case PF_ROUTE: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_route_total); + break; + case PF_NDRV: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_ndrv_total); + break; + case PF_KEY: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_key_total); + break; + case PF_INET6: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_inet6_total); + if (type == SOCK_STREAM) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet6_stream_total); + } else { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet6_dgram_total); + } + break; + case PF_SYSTEM: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_system_total); + break; + case PF_MULTIPATH: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_multipath_total); + break; + default: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_other_total); + break; + } + if (flags & SOCF_ASYNC) so->so_state |= SS_NBIO; -#if MULTIPATH - if (flags & SOCF_MP_SUBFLOW) { - /* - * A multipath subflow socket is used internally in the kernel, - * therefore it does not have a file desciptor associated by - * default. - */ - so->so_state |= SS_NOFDREF; - so->so_flags |= SOF_MP_SUBFLOW; - } -#endif /* MULTIPATH */ TAILQ_INIT(&so->so_incomp); TAILQ_INIT(&so->so_comp); @@ -864,7 +901,6 @@ sobindlock(struct socket *so, struct sockaddr *nam, int dolock) if (dolock) socket_lock(so, 1); - VERIFY(so->so_usecount > 1); so_update_last_owner_locked(so, p); so_update_policy(so); @@ -1067,8 +1103,8 @@ so_acquire_accept_list(struct socket *head, struct socket *so) if (head->so_proto->pr_getlock == NULL) { return; } - mutex_held = (*head->so_proto->pr_getlock)(head, 0); - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + mutex_held = (*head->so_proto->pr_getlock)(head, PR_F_WILLUNLOCK); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (!(head->so_flags1 & SOF1_ACCEPT_LIST_HELD)) { head->so_flags1 |= SOF1_ACCEPT_LIST_HELD; @@ -1097,8 +1133,8 @@ so_release_accept_list(struct socket *head) lck_mtx_t *mutex_held; mutex_held = (*head->so_proto->pr_getlock)(head, 0); - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); - + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); + head->so_flags1 &= ~SOF1_ACCEPT_LIST_HELD; wakeup((caddr_t)&head->so_incomp); } @@ -1159,8 +1195,8 @@ sofreelastref(struct socket *so, int dealloc) } else { if (head->so_proto->pr_getlock != NULL) { so_release_accept_list(head); - socket_unlock(head, 1); - } + socket_unlock(head, 1); + } printf("sofree: not queued\n"); } } @@ -1188,10 +1224,10 @@ soclose_wait_locked(struct socket *so) lck_mtx_t *mutex_held; if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); /* * Double check here and return if there's no outstanding upcall; @@ -1202,9 +1238,10 @@ soclose_wait_locked(struct socket *so) so->so_rcv.sb_flags &= ~SB_UPCALL; so->so_snd.sb_flags &= ~SB_UPCALL; so->so_flags |= SOF_CLOSEWAIT; + (void) msleep((caddr_t)&so->so_upcallusecount, mutex_held, (PZERO - 1), "soclose_wait_locked", NULL); - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); so->so_flags &= ~SOF_CLOSEWAIT; } @@ -1365,7 +1402,7 @@ again: (so->so_state & SS_NBIO)) goto drop; if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; while (so->so_state & SS_ISCONNECTED) { @@ -1408,9 +1445,6 @@ discard: } so->so_state |= SS_NOFDREF; - if (so->so_flags & SOF_MP_SUBFLOW) - so->so_flags &= ~SOF_MP_SUBFLOW; - if ((so->so_flags & SOF_KNOTE) != 0) KNOTE(&so->so_klist, SO_FILT_HINT_LOCKED); @@ -1461,7 +1495,7 @@ soabort(struct socket *so) mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); #endif if ((so->so_flags & SOF_ABORTED) == 0) { @@ -1547,7 +1581,7 @@ soacceptfilter(struct socket *so, struct socket *head) * the following is done while holding the lock since * the socket has been exposed to the filter(s) earlier. */ - so->so_state &= ~SS_COMP; + so->so_state &= ~SS_NOFDREF; socket_unlock(so, 1); soclose(so); /* Propagate socket filter's error code to the caller */ @@ -1802,12 +1836,6 @@ sodisconnectx(struct socket *so, sae_associd_t aid, sae_connid_t cid) return (error); } -int -sopeelofflocked(struct socket *so, sae_associd_t aid, struct socket **psop) -{ - return ((*so->so_proto->pr_usrreqs->pru_peeloff)(so, aid, psop)); -} - #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? 0 : SBL_WAIT) /* @@ -1893,18 +1921,9 @@ defunct: if ((so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) { if (((so->so_state & SS_ISCONFIRMING) == 0) && (resid != 0 || clen == 0) && - !(so->so_flags1 & SOF1_PRECONNECT_DATA)) { -#if MPTCP - /* - * MPTCP Fast Join sends data before the - * socket is truly connected. - */ - if ((so->so_flags & (SOF_MP_SUBFLOW | - SOF_MPTCP_FASTJOIN)) != - (SOF_MP_SUBFLOW | SOF_MPTCP_FASTJOIN)) -#endif /* MPTCP */ + !(so->so_flags1 & SOF1_PRECONNECT_DATA)) return (ENOTCONN); - } + } else if (addr == 0 && !(flags&MSG_HOLD)) { return ((so->so_proto->pr_flags & PR_CONNREQUIRED) ? ENOTCONN : EDESTADDRREQ); @@ -2072,8 +2091,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, if (so->so_type != SOCK_STREAM && (flags & MSG_OOB) != 0) { error = EOPNOTSUPP; - socket_unlock(so, 1); - goto out; + goto out_locked; } /* @@ -2092,8 +2110,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, if (resid < 0 || resid > INT_MAX || (so->so_type == SOCK_STREAM && !(so->so_flags & SOF_ENABLE_MSGS) && (flags & MSG_EOR))) { error = EINVAL; - socket_unlock(so, 1); - goto out; + goto out_locked; } dontroute = (flags & MSG_DONTROUTE) && @@ -2111,7 +2128,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, error = sosendcheck(so, addr, resid, clen, atomic, flags, &sblocked, control); if (error) - goto release; + goto out_locked; mp = ⊤ if (so->so_flags & SOF_ENABLE_MSGS) @@ -2296,7 +2313,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, if (freelist == NULL) { error = ENOBUFS; socket_lock(so, 0); - goto release; + goto out_locked; } /* * For datagram protocols, @@ -2352,7 +2369,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, socket_lock(so, 0); if (error) - goto release; + goto out_locked; } if (flags & (MSG_HOLD|MSG_SEND)) { @@ -2372,7 +2389,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, so->so_tail = mb1; if (flags & MSG_HOLD) { top = NULL; - goto release; + goto out_locked; } top = so->so_temp; } @@ -2407,7 +2424,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, control = NULL; top = NULL; } - goto release; + goto out_locked; } #if CONTENT_FILTER /* @@ -2423,7 +2440,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, control = NULL; top = NULL; } - goto release; + goto out_locked; } #endif /* CONTENT_FILTER */ } @@ -2450,16 +2467,15 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, top = NULL; mp = ⊤ if (error) - goto release; + goto out_locked; } while (resid && space > 0); } while (resid); -release: +out_locked: if (sblocked) sbunlock(&so->so_snd, FALSE); /* will unlock socket */ else socket_unlock(so, 1); -out: if (top != NULL) m_freem(top); if (control != NULL) @@ -2469,12 +2485,7 @@ out: if (control_copy != NULL) m_freem(control_copy); - /* - * One write has been done. This was enough. Get back to "normal" - * behavior. - */ - if (so->so_flags1 & SOF1_PRECONNECT_DATA) - so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + soclearfastopen(so); if (en_tracing) { /* resid passed here is the bytes left in uio */ @@ -4485,7 +4496,7 @@ sorflush(struct socket *so) else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); #endif /* notyet */ sflt_notify(so, sock_evt_flush_read, NULL); @@ -4664,19 +4675,25 @@ sooptcopyin_timeval(struct sockopt *sopt, struct timeval *tv_p) return (0); } -static int -soopt_cred_check(struct socket *so, int priv) +int +soopt_cred_check(struct socket *so, int priv, boolean_t allow_root) { kauth_cred_t cred = NULL; proc_t ep = PROC_NULL; - int error; + uid_t uid; + int error = 0; if (so->so_flags & SOF_DELEGATED) { ep = proc_find(so->e_pid); if (ep) cred = kauth_cred_proc_ref(ep); } - error = priv_check_cred(cred ? cred : so->so_cred, priv, 0); + + uid = kauth_cred_getuid(cred ? cred : so->so_cred); + + /* uid is 0 for root */ + if (uid != 0 || !allow_root) + error = priv_check_cred(cred ? cred : so->so_cred, priv, 0); if (cred) kauth_cred_unref(&cred); if (ep != PROC_NULL) @@ -4966,7 +4983,7 @@ sosetoptlock(struct socket *so, struct sockopt *sopt, int dolock) goto out; if (optval != 0) { error = soopt_cred_check(so, - PRIV_NET_RESTRICTED_AWDL); + PRIV_NET_RESTRICTED_AWDL, false); if (error == 0) inp_set_awdl_unrestricted( sotoinpcb(so)); @@ -4985,7 +5002,7 @@ sosetoptlock(struct socket *so, struct sockopt *sopt, int dolock) if (optval != 0 && inp_get_intcoproc_allowed(sotoinpcb(so)) == FALSE) { error = soopt_cred_check(so, - PRIV_NET_RESTRICTED_INTCOPROC); + PRIV_NET_RESTRICTED_INTCOPROC, false); if (error == 0) inp_set_intcoproc_allowed( sotoinpcb(so)); @@ -5232,27 +5249,50 @@ sosetoptlock(struct socket *so, struct sockopt *sopt, int dolock) case SO_NECP_ATTRIBUTES: error = necp_set_socket_attributes(so, sopt); break; -#endif /* NECP */ -#if MPTCP - case SO_MPTCP_FASTJOIN: - if (!((so->so_flags & SOF_MP_SUBFLOW) || - ((SOCK_CHECK_DOM(so, PF_MULTIPATH)) && - (SOCK_CHECK_PROTO(so, IPPROTO_TCP))))) { - error = ENOPROTOOPT; + case SO_NECP_CLIENTUUID: + if (SOCK_DOM(so) == PF_MULTIPATH) { + /* Handled by MPTCP itself */ break; } - error = sooptcopyin(sopt, &optval, sizeof (optval), - sizeof (optval)); - if (error != 0) + if (SOCK_DOM(so) != PF_INET && SOCK_DOM(so) != PF_INET6) { + error = EINVAL; goto out; - if (optval == 0) - so->so_flags &= ~SOF_MPTCP_FASTJOIN; - else - so->so_flags |= SOF_MPTCP_FASTJOIN; + } + + struct inpcb *inp = sotoinpcb(so); + if (!uuid_is_null(inp->necp_client_uuid)) { + // Clear out the old client UUID if present + necp_inpcb_remove_cb(inp); + } + + error = sooptcopyin(sopt, &inp->necp_client_uuid, + sizeof(uuid_t), sizeof(uuid_t)); + if (error != 0) { + goto out; + } + + if (uuid_is_null(inp->necp_client_uuid)) { + error = EINVAL; + goto out; + } + + error = necp_client_register_socket_flow(so->last_pid, + inp->necp_client_uuid, inp); + if (error != 0) { + uuid_clear(inp->necp_client_uuid); + goto out; + } + + if (inp->inp_lport != 0) { + // There is bound local port, so this is not + // a fresh socket. Assign to the client. + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + } + break; -#endif /* MPTCP */ +#endif /* NECP */ case SO_EXTENDED_BK_IDLE: error = sooptcopyin(sopt, &optval, sizeof (optval), @@ -5349,8 +5389,8 @@ sooptcopyout_timeval(struct sockopt *sopt, const struct timeval *tv_p) { int error; size_t len; - struct user64_timeval tv64; - struct user32_timeval tv32; + struct user64_timeval tv64 = {}; + struct user32_timeval tv32 = {}; const void * val; size_t valsize; @@ -5694,6 +5734,23 @@ integer: case SO_NECP_ATTRIBUTES: error = necp_get_socket_attributes(so, sopt); break; + + case SO_NECP_CLIENTUUID: + { + uuid_t *ncu; + + if (SOCK_DOM(so) == PF_MULTIPATH) { + ncu = &mpsotomppcb(so)->necp_client_uuid; + } else if (SOCK_DOM(so) == PF_INET || SOCK_DOM(so) == PF_INET6) { + ncu = &sotoinpcb(so)->necp_client_uuid; + } else { + error = EINVAL; + goto out; + } + + error = sooptcopyout(sopt, ncu, sizeof(uuid_t)); + break; + } #endif /* NECP */ #if CONTENT_FILTER @@ -5708,19 +5765,6 @@ integer: } #endif /* CONTENT_FILTER */ -#if MPTCP - case SO_MPTCP_FASTJOIN: - if (!((so->so_flags & SOF_MP_SUBFLOW) || - ((SOCK_CHECK_DOM(so, PF_MULTIPATH)) && - (SOCK_CHECK_PROTO(so, IPPROTO_TCP))))) { - error = ENOPROTOOPT; - break; - } - optval = (so->so_flags & SOF_MPTCP_FASTJOIN); - /* Fixed along with rdar://19391339 */ - goto integer; -#endif /* MPTCP */ - case SO_EXTENDED_BK_IDLE: optval = (so->so_flags1 & SOF1_EXTEND_BK_IDLE_WANTED); goto integer; @@ -5940,7 +5984,8 @@ sopoll(struct socket *so, int events, kauth_cred_t cred, void * wql) } int -soo_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) +soo_kqfilter(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx) { #pragma unused(fp) #if !CONFIG_MACF_SOCKET @@ -5987,7 +6032,7 @@ soo_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) * call the appropriate sub-filter attach * with the socket still locked */ - result = knote_fops(kn)->f_attach(kn); + result = knote_fops(kn)->f_attach(kn, kev); socket_unlock(so, 1); @@ -6067,7 +6112,7 @@ filt_soread_common(struct knote *kn, struct socket *so) } static int -filt_sorattach(struct knote *kn) +filt_sorattach(struct knote *kn, __unused struct kevent_internal_s *kev) { struct socket *so = (struct socket *)kn->kn_fp->f_fglob->fg_data; @@ -6234,7 +6279,7 @@ filt_sowrite_common(struct knote *kn, struct socket *so) } static int -filt_sowattach(struct knote *kn) +filt_sowattach(struct knote *kn, __unused struct kevent_internal_s *kev) { struct socket *so = (struct socket *)kn->kn_fp->f_fglob->fg_data; @@ -6426,7 +6471,7 @@ filt_sockev_common(struct knote *kn, struct socket *so, long ev_hint) } static int -filt_sockattach(struct knote *kn) +filt_sockattach(struct knote *kn, __unused struct kevent_internal_s *kev) { struct socket *so = (struct socket *)kn->kn_fp->f_fglob->fg_data; @@ -6610,19 +6655,18 @@ solockhistory_nr(struct socket *so) return (lock_history_str); } -int +void socket_lock(struct socket *so, int refcount) { - int error = 0; void *lr_saved; lr_saved = __builtin_return_address(0); if (so->so_proto->pr_lock) { - error = (*so->so_proto->pr_lock)(so, refcount, lr_saved); + (*so->so_proto->pr_lock)(so, refcount, lr_saved); } else { #ifdef MORE_LOCKING_DEBUG - lck_mtx_assert(so->so_proto->pr_domain->dom_mtx, + LCK_MTX_ASSERT(so->so_proto->pr_domain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED); #endif lck_mtx_lock(so->so_proto->pr_domain->dom_mtx); @@ -6631,14 +6675,37 @@ socket_lock(struct socket *so, int refcount) so->lock_lr[so->next_lock_lr] = lr_saved; so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX; } +} - return (error); +void +socket_lock_assert_owned(struct socket *so) +{ + lck_mtx_t *mutex_held; + + if (so->so_proto->pr_getlock != NULL) + mutex_held = (*so->so_proto->pr_getlock)(so, 0); + else + mutex_held = so->so_proto->pr_domain->dom_mtx; + + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); } int +socket_try_lock(struct socket *so) +{ + lck_mtx_t *mtx; + + if (so->so_proto->pr_getlock != NULL) + mtx = (*so->so_proto->pr_getlock)(so, 0); + else + mtx = so->so_proto->pr_domain->dom_mtx; + + return (lck_mtx_try_lock(mtx)); +} + +void socket_unlock(struct socket *so, int refcount) { - int error = 0; void *lr_saved; lck_mtx_t *mutex_held; @@ -6650,11 +6717,11 @@ socket_unlock(struct socket *so, int refcount) } if (so && so->so_proto->pr_unlock) { - error = (*so->so_proto->pr_unlock)(so, refcount, lr_saved); + (*so->so_proto->pr_unlock)(so, refcount, lr_saved); } else { mutex_held = so->so_proto->pr_domain->dom_mtx; #ifdef MORE_LOCKING_DEBUG - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); #endif so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; @@ -6674,8 +6741,6 @@ socket_unlock(struct socket *so, int refcount) } lck_mtx_unlock(mutex_held); } - - return (error); } /* Called with socket locked, will unlock socket */ @@ -6688,7 +6753,7 @@ sofree(struct socket *so) mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); sofreelastref(so, 0); } @@ -6919,7 +6984,7 @@ sodefunct(struct proc *p, struct socket *so, int level) * Explicitly handle connectionless-protocol disconnection * and release any remaining data in the socket buffers. */ - if (!(so->so_flags & SS_ISDISCONNECTED)) + if (!(so->so_state & SS_ISDISCONNECTED)) (void) soisdisconnected(so); if (so->so_error == 0) @@ -6993,6 +7058,12 @@ so_set_extended_bk_idle(struct socket *so, int optval) struct filedesc *fdp; int count = 0; + /* + * Unlock socket to avoid lock ordering issue with + * the proc fd table lock + */ + socket_unlock(so, 0); + proc_fdlock(p); fdp = p->p_fd; @@ -7012,6 +7083,10 @@ so_set_extended_bk_idle(struct socket *so, int optval) if (count >= soextbkidlestat.so_xbkidle_maxperproc) break; } + proc_fdunlock(p); + + socket_lock(so, 0); + if (count >= soextbkidlestat.so_xbkidle_maxperproc) { OSIncrementAtomic(&soextbkidlestat.so_xbkidle_toomany); error = EBUSY; @@ -7029,8 +7104,6 @@ so_set_extended_bk_idle(struct socket *so, int optval) SOCK_DOM(so), SOCK_TYPE(so), (so->so_flags1 & SOF1_EXTEND_BK_IDLE_WANTED) ? "is" : "not"); - - proc_fdunlock(p); } return (error); @@ -7145,6 +7218,7 @@ so_set_recv_anyif(struct socket *so, int optval) sotoinpcb(so)->inp_flags &= ~INP_RECV_ANYIF; } + return (ret); } @@ -7212,6 +7286,9 @@ so_set_restrictions(struct socket *so, uint32_t vals) } } + if (SOCK_DOM(so) == PF_MULTIPATH) + mptcp_set_restrictions(so); + return (0); } diff --git a/bsd/kern/uipc_socket2.c b/bsd/kern/uipc_socket2.c index 45c151848..08ec21e97 100644 --- a/bsd/kern/uipc_socket2.c +++ b/bsd/kern/uipc_socket2.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2017 Apple Inc. All rights reserved. + * Copyright (c) 1998-2015 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -89,6 +89,7 @@ #include #include #include +#include #include #include @@ -142,10 +143,6 @@ int32_t total_sbmb_cnt_floor __attribute__((aligned(8))) = 0; int32_t total_sbmb_cnt_peak __attribute__((aligned(8))) = 0; int64_t sbmb_limreached __attribute__((aligned(8))) = 0; -/* Control whether to throttle sockets eligible to be throttled */ -__private_extern__ u_int32_t net_io_policy_throttled = 0; -static int sysctl_io_policy_throttled SYSCTL_HANDLER_ARGS; - u_int32_t net_io_policy_log = 0; /* log socket policy changes */ #if CONFIG_PROC_UUID_POLICY u_int32_t net_io_policy_uuid = 1; /* enable UUID socket policy */ @@ -202,7 +199,7 @@ soisconnected(struct socket *so) if (so->so_head != NULL && (so->so_state & SS_INCOMP)) { struct socket *head = so->so_head; int locked = 0; - + /* * Enforce lock order when the protocol has per socket locks */ @@ -233,7 +230,7 @@ soisconnected(struct socket *so) if (locked != 0) { socket_unlock(head, 1); socket_lock(so, 0); - } + } } else if (locked != 0) { so_release_accept_list(head); socket_unlock(head, 1); @@ -327,7 +324,7 @@ sonewconn_internal(struct socket *head, int connstatus) mutex_held = (*head->so_proto->pr_getlock)(head, 0); else mutex_held = head->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (!soqlencomp) { /* @@ -551,11 +548,11 @@ sbwait(struct sockbuf *sb) } if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); ts.tv_sec = sb->sb_timeo.tv_sec; ts.tv_nsec = sb->sb_timeo.tv_usec * 1000; @@ -630,15 +627,18 @@ sowakeup(struct socket *so, struct sockbuf *sb) if (sb->sb_flags & SB_UPCALL) { void (*sb_upcall)(struct socket *, void *, int); caddr_t sb_upcallarg; + int lock = !(sb->sb_flags & SB_UPCALL_LOCK); sb_upcall = sb->sb_upcall; sb_upcallarg = sb->sb_upcallarg; /* Let close know that we're about to do an upcall */ so->so_upcallusecount++; - socket_unlock(so, 0); + if (lock) + socket_unlock(so, 0); (*sb_upcall)(so, sb_upcallarg, M_DONTWAIT); - socket_lock(so, 0); + if (lock) + socket_lock(so, 0); so->so_upcallusecount--; /* Tell close that it's safe to proceed */ @@ -897,7 +897,7 @@ sbcheck(struct sockbuf *sb) else mutex_held = sb->sb_so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (sbchecking == 0) return; @@ -1468,9 +1468,6 @@ sbappendmptcpstream_rcv(struct sockbuf *sb, struct mbuf *m) SBLASTMBUFCHK(sb, __func__); - if (mptcp_adj_rmap(so, m) != 0) - return (0); - /* No filter support (SB_RECV) on mptcp subflow sockets */ sbcompress(sb, m, sb->sb_mbtail); @@ -1862,14 +1859,18 @@ sbdrop(struct sockbuf *sb, int len) next = (m = sb->sb_mb) ? m->m_nextpkt : 0; #if MPTCP - if ((m != NULL) && (len > 0) && - (!(sb->sb_flags & SB_RECV)) && + if (m != NULL && len > 0 && !(sb->sb_flags & SB_RECV) && ((sb->sb_so->so_flags & SOF_MP_SUBFLOW) || - ((SOCK_CHECK_DOM(sb->sb_so, PF_MULTIPATH)) && - (SOCK_CHECK_PROTO(sb->sb_so, IPPROTO_TCP)))) && - (!(sb->sb_so->so_flags1 & SOF1_POST_FALLBACK_SYNC))) { + (SOCK_CHECK_DOM(sb->sb_so, PF_MULTIPATH) && + SOCK_CHECK_PROTO(sb->sb_so, IPPROTO_TCP))) && + !(sb->sb_so->so_flags1 & SOF1_POST_FALLBACK_SYNC)) { mptcp_preproc_sbdrop(sb->sb_so, m, (unsigned int)len); } + if (m != NULL && len > 0 && !(sb->sb_flags & SB_RECV) && + (sb->sb_so->so_flags & SOF_MP_SUBFLOW) && + (sb->sb_so->so_flags1 & SOF1_POST_FALLBACK_SYNC)) { + mptcp_fallback_sbdrop(sb->sb_so, m, len); + } #endif /* MPTCP */ KERNEL_DEBUG((DBG_FNC_SBDROP | DBG_FUNC_START), sb, len, 0, 0, 0); @@ -2137,13 +2138,6 @@ pru_listen_notsupp(struct socket *so, struct proc *p) return (EOPNOTSUPP); } -int -pru_peeloff_notsupp(struct socket *so, sae_associd_t aid, struct socket **psop) -{ -#pragma unused(so, aid, psop) - return (EOPNOTSUPP); -} - int pru_peeraddr_notsupp(struct socket *so, struct sockaddr **nam) { @@ -2290,7 +2284,6 @@ pru_sanitize(struct pr_usrreqs *pru) DEFAULT(pru->pru_disconnect, pru_disconnect_notsupp); DEFAULT(pru->pru_disconnectx, pru_disconnectx_notsupp); DEFAULT(pru->pru_listen, pru_listen_notsupp); - DEFAULT(pru->pru_peeloff, pru_peeloff_notsupp); DEFAULT(pru->pru_peeraddr, pru_peeraddr_notsupp); DEFAULT(pru->pru_rcvd, pru_rcvd_notsupp); DEFAULT(pru->pru_rcvoob, pru_rcvoob_notsupp); @@ -2363,7 +2356,7 @@ int msgq_sbspace(struct socket *so, struct mbuf *control) { int space = 0, error; - u_int32_t msgpri; + u_int32_t msgpri = 0; VERIFY(so->so_type == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP); if (control != NULL) { @@ -2566,11 +2559,11 @@ sblock(struct sockbuf *sb, uint32_t flags) * us the lock. This will be fixed in future. */ if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); sb->sb_wantlock++; VERIFY(sb->sb_wantlock != 0); @@ -2659,11 +2652,11 @@ sbunlock(struct sockbuf *sb, boolean_t keeplocked) lck_mtx_t *mutex_held; if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); VERIFY(so->so_usecount > 0); so->so_usecount--; @@ -2711,18 +2704,10 @@ soevupcall(struct socket *so, u_int32_t hint) { if (so->so_event != NULL) { caddr_t so_eventarg = so->so_eventarg; - int locked = hint & SO_FILT_HINT_LOCKED; hint &= so->so_eventmask; - if (hint != 0) { - if (locked) - socket_unlock(so, 0); - + if (hint != 0) so->so_event(so, so_eventarg, hint); - - if (locked) - socket_lock(so, 0); - } } } @@ -2836,6 +2821,7 @@ sotoxsocket(struct socket *so, struct xsocket *xso) } +#if !CONFIG_EMBEDDED void sotoxsocket64(struct socket *so, struct xsocket64 *xso) @@ -2865,6 +2851,7 @@ sotoxsocket64(struct socket *so, struct xsocket64 *xso) xso->so_uid = kauth_cred_getuid(so->so_cred); } +#endif /* !CONFIG_EMBEDDED */ /* * This does the same for sockbufs. Note that the xsockbuf structure, @@ -2894,12 +2881,7 @@ sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb) inline int soisthrottled(struct socket *so) { - /* - * On non-embedded, we rely on implicit throttling by the - * application, as we're missing the system wide "decision maker" - */ - return ( - (so->so_flags1 & SOF1_TRAFFIC_MGT_SO_BACKGROUND)); + return (so->so_flags1 & SOF1_TRAFFIC_MGT_SO_BACKGROUND); } inline int @@ -2930,6 +2912,16 @@ soissrcbesteffort(struct socket *so) so->so_traffic_class == SO_TC_OAM); } +void +soclearfastopen(struct socket *so) +{ + if (so->so_flags1 & SOF1_PRECONNECT_DATA) + so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + + if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) + so->so_flags1 &= ~SOF1_DATA_IDEMPOTENT; +} + void sonullevent(struct socket *so, void *arg, uint32_t hint) { @@ -2963,27 +2955,6 @@ sysctl_sb_max SYSCTL_HANDLER_ARGS return (error); } -static int -sysctl_io_policy_throttled SYSCTL_HANDLER_ARGS -{ -#pragma unused(arg1, arg2) - int i, err; - - i = net_io_policy_throttled; - - err = sysctl_handle_int(oidp, &i, 0, req); - if (err != 0 || req->newptr == USER_ADDR_NULL) - return (err); - - if (i != net_io_policy_throttled) - SOTHROTTLELOG("throttle: network IO policy throttling is " - "now %s\n", i ? "ON" : "OFF"); - - net_io_policy_throttled = i; - - return (err); -} - SYSCTL_PROC(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &sb_max, 0, &sysctl_sb_max, "IU", "Maximum socket buffer size"); @@ -3004,8 +2975,39 @@ SYSCTL_INT(_kern_ipc, KIPC_SOQLIMITCOMPAT, soqlimitcompat, CTLFLAG_RW | CTLFLAG_LOCKED, &soqlimitcompat, 1, "Enable socket queue limit compatibility"); -SYSCTL_INT(_kern_ipc, OID_AUTO, soqlencomp, CTLFLAG_RW | CTLFLAG_LOCKED, - &soqlencomp, 0, "Listen backlog represents only complete queue"); +/* + * Hack alert -- rdar://33572856 + * A loopback test we cannot change was failing because it sets + * SO_SENDTIMEO to 5 seconds and that's also the value + * of the minimum persist timer. Because of the persist timer, + * the connection was not idle for 5 seconds and SO_SNDTIMEO + * was not triggering at 5 seconds causing the test failure. + * As a workaround we check the sysctl soqlencomp the test is already + * setting to set disable auto tuning of the receive buffer. + */ + +extern u_int32_t tcp_do_autorcvbuf; + +static int +sysctl_soqlencomp SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + u_int32_t new_value; + int changed = 0; + int error = sysctl_io_number(req, soqlencomp, sizeof (u_int32_t), + &new_value, &changed); + if (!error && changed) { + soqlencomp = new_value; + if (new_value != 0) { + tcp_do_autorcvbuf = 0; + tcptv_persmin_val = 6 * TCP_RETRANSHZ; + } + } + return (error); +} +SYSCTL_PROC(_kern_ipc, OID_AUTO, soqlencomp, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &soqlencomp, 0, &sysctl_soqlencomp, "IU", ""); SYSCTL_INT(_kern_ipc, OID_AUTO, sbmb_cnt, CTLFLAG_RD | CTLFLAG_LOCKED, &total_sbmb_cnt, 0, ""); @@ -3019,10 +3021,6 @@ SYSCTL_QUAD(_kern_ipc, OID_AUTO, sbmb_limreached, CTLFLAG_RD | CTLFLAG_LOCKED, SYSCTL_NODE(_kern_ipc, OID_AUTO, io_policy, CTLFLAG_RW, 0, "network IO policy"); -SYSCTL_PROC(_kern_ipc_io_policy, OID_AUTO, throttled, - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &net_io_policy_throttled, 0, - sysctl_io_policy_throttled, "I", ""); - SYSCTL_INT(_kern_ipc_io_policy, OID_AUTO, log, CTLFLAG_RW | CTLFLAG_LOCKED, &net_io_policy_log, 0, ""); diff --git a/bsd/kern/uipc_syscalls.c b/bsd/kern/uipc_syscalls.c index 0aaba750a..53043d70b 100644 --- a/bsd/kern/uipc_syscalls.c +++ b/bsd/kern/uipc_syscalls.c @@ -156,7 +156,6 @@ static int connectx_nocancel(struct proc *, struct connectx_args *, int *); static int connectitx(struct socket *, struct sockaddr *, struct sockaddr *, struct proc *, uint32_t, sae_associd_t, sae_connid_t *, uio_t, unsigned int, user_ssize_t *); -static int peeloff_nocancel(struct proc *, struct peeloff_args *, int *); static int disconnectx_nocancel(struct proc *, struct disconnectx_args *, int *); static int socket_common(struct proc *, int, int, int, pid_t, int32_t *, int); @@ -439,7 +438,7 @@ accept_nocancel(struct proc *p, struct accept_nocancel_args *uap, socket_lock(head, 1); if (head->so_proto->pr_getlock != NULL) { - mutex_held = (*head->so_proto->pr_getlock)(head, 0); + mutex_held = (*head->so_proto->pr_getlock)(head, PR_F_WILLUNLOCK); dosocklock = 1; } else { mutex_held = head->so_proto->pr_domain->dom_mtx; @@ -931,7 +930,7 @@ connectit(struct socket *so, struct sockaddr *sa) lck_mtx_t *mutex_held; if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; error = msleep((caddr_t)&so->so_timeo, mutex_held, @@ -979,7 +978,7 @@ connectitx(struct socket *so, struct sockaddr *src, so->so_flags1 |= SOF1_DATA_IDEMPOTENT; if (flags & CONNECT_DATA_AUTHENTICATED) - so->so_flags |= SOF1_DATA_AUTHENTICATED; + so->so_flags1 |= SOF1_DATA_AUTHENTICATED; } /* @@ -1029,7 +1028,7 @@ connectitx(struct socket *so, struct sockaddr *src, lck_mtx_t *mutex_held; if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; error = msleep((caddr_t)&so->so_timeo, mutex_held, @@ -1052,80 +1051,13 @@ out: int peeloff(struct proc *p, struct peeloff_args *uap, int *retval) { +#pragma unused(p, uap, retval) /* * Due to similiarity with a POSIX interface, define as * an unofficial cancellation point. */ __pthread_testcancel(1); - return (peeloff_nocancel(p, uap, retval)); -} - -static int -peeloff_nocancel(struct proc *p, struct peeloff_args *uap, int *retval) -{ - struct fileproc *fp; - struct socket *mp_so, *so = NULL; - int newfd, fd = uap->s; - short fflag; /* type must match fp->f_flag */ - int error; - - *retval = -1; - - error = fp_getfsock(p, fd, &fp, &mp_so); - if (error != 0) { - if (error == EOPNOTSUPP) - error = ENOTSOCK; - goto out_nofile; - } - if (mp_so == NULL) { - error = EBADF; - goto out; - } - - socket_lock(mp_so, 1); - error = sopeelofflocked(mp_so, uap->aid, &so); - if (error != 0) { - socket_unlock(mp_so, 1); - goto out; - } - VERIFY(so != NULL); - socket_unlock(mp_so, 0); /* keep ref on mp_so for us */ - - fflag = fp->f_flag; - error = falloc(p, &fp, &newfd, vfs_context_current()); - if (error != 0) { - /* drop this socket (probably ran out of file descriptors) */ - soclose(so); - sodereference(mp_so); /* our mp_so ref */ - goto out; - } - - fp->f_flag = fflag; - fp->f_ops = &socketops; - fp->f_data = (caddr_t)so; - - /* - * If the socket has been marked as inactive by sosetdefunct(), - * disallow further operations on it. - */ - if (so->so_flags & SOF_DEFUNCT) { - sodefunct(current_proc(), so, - SHUTDOWN_SOCKET_LEVEL_DISCONNECT_INTERNAL); - } - - proc_fdlock(p); - procfdtbl_releasefd(p, newfd, NULL); - fp_drop(p, newfd, fp, 1); - proc_fdunlock(p); - - sodereference(mp_so); /* our mp_so ref */ - *retval = newfd; - -out: - file_drop(fd); - -out_nofile: - return (error); + return (0); } int diff --git a/bsd/kern/uipc_usrreq.c b/bsd/kern/uipc_usrreq.c index 4f31897bd..b8b429c08 100644 --- a/bsd/kern/uipc_usrreq.c +++ b/bsd/kern/uipc_usrreq.c @@ -305,7 +305,7 @@ uipc_detach(struct socket *so) if (unp == 0) return (EINVAL); - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); unp_detach(unp); return (0); } @@ -987,8 +987,18 @@ unp_bind( return (EAFNOSUPPORT); } + /* + * Check if the socket is already bound to an address + */ if (unp->unp_vnode != NULL) return (EINVAL); + /* + * Check if the socket may have been shut down + */ + if ((so->so_state & (SS_CANTRCVMORE | SS_CANTSENDMORE)) == + (SS_CANTRCVMORE | SS_CANTSENDMORE)) + return (EINVAL); + namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path); if (namelen <= 0) return (EINVAL); @@ -1311,7 +1321,7 @@ decref_out: } out: - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); vnode_put(vp); return (error); } @@ -1332,8 +1342,8 @@ unp_connect2(struct socket *so, struct socket *so2) unp2 = sotounpcb(so2); - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); /* Verify both sockets are still opened */ if (unp == 0 || unp2 == 0) @@ -1397,8 +1407,8 @@ unp_connect2(struct socket *so, struct socket *so2) default: panic("unknown socket type %d in unp_connect2", so->so_type); } - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); return (0); } @@ -1460,8 +1470,8 @@ try_again: } so_locked = 1; - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); /* Check for the UNP_DONTDISCONNECT flag, if it * is set, release both sockets and go to sleep @@ -1500,7 +1510,7 @@ try_again: case SOCK_STREAM: unp2->unp_conn = NULL; - VERIFY(so2->so_usecount > 0); + VERIFY(so->so_usecount > 0); so->so_usecount--; /* Set the socket state correctly but do a wakeup later when @@ -1535,7 +1545,7 @@ out: socket_lock(so,0); soisdisconnected(so); } - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); return; } @@ -1707,6 +1717,7 @@ SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist, (caddr_t)(long)SOCK_STREAM, 0, unp_pcblist, "S,xunpcb", "List of active local stream sockets"); +#if !CONFIG_EMBEDDED static int unp_pcblist64 SYSCTL_HANDLER_ARGS @@ -1855,6 +1866,7 @@ SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist64, (caddr_t)(long)SOCK_STREAM, 0, unp_pcblist64, "S,xunpcb64", "List of active local stream sockets 64 bit"); +#endif /* !CONFIG_EMBEDDED */ static void unp_shutdown(struct unpcb *unp) @@ -2461,7 +2473,7 @@ unp_unlock(struct socket *so, int refcount, void * lr) } else { mutex_held = &((struct unpcb *)so->so_pcb)->unp_mtx; } - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; @@ -2485,7 +2497,7 @@ unp_unlock(struct socket *so, int refcount, void * lr) } lck_mtx_t * -unp_getlock(struct socket *so, __unused int locktype) +unp_getlock(struct socket *so, __unused int flags) { struct unpcb *unp = (struct unpcb *)so->so_pcb; diff --git a/bsd/libkern/libkern.h b/bsd/libkern/libkern.h index bfef07c6a..e2a47fdef 100644 --- a/bsd/libkern/libkern.h +++ b/bsd/libkern/libkern.h @@ -78,6 +78,9 @@ #include #include +#if defined(__arm__) || defined(__arm64__) +#include /* for _ARM_ARCH_* */ +#endif #ifdef __APPLE_API_OBSOLETE /* BCD conversions. */ @@ -140,6 +143,9 @@ ulmin(u_int32_t a, u_int32_t b) /* Prototypes for non-quad routines. */ extern int ffs(int); +extern int ffsll(unsigned long long); +extern int fls(int); +extern int flsll(unsigned long long); extern u_int32_t random(void); extern int scanc(u_int, u_char *, const u_char *, int); extern int skpc(int, int, char *); @@ -154,7 +160,7 @@ extern void url_decode(char *str); int snprintf(char *, size_t, const char *, ...) __printflike(3,4); /* sprintf() is being deprecated. Please use snprintf() instead. */ -int sprintf(char *bufp, const char *, ...) __deprecated; +int sprintf(char *bufp, const char *, ...) __deprecated __printflike(2,3); int sscanf(const char *, char const *, ...) __scanflike(2,3); int printf(const char *, ...) __printflike(1,2); @@ -184,19 +190,23 @@ extern int copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_si int vsscanf(const char *, char const *, va_list); -extern int vprintf(const char *, va_list); -extern int vsnprintf(char *, size_t, const char *, va_list); +extern int vprintf(const char *, va_list) __printflike(1,0); +extern int vsnprintf(char *, size_t, const char *, va_list) __printflike(3,0); #if XNU_KERNEL_PRIVATE -extern int vprintf_log_locked(const char *, va_list); +extern int vprintf_log_locked(const char *, va_list) __printflike(1,0); extern void osobject_retain(void * object); extern void osobject_release(void * object); #endif /* vsprintf() is being deprecated. Please use vsnprintf() instead. */ -extern int vsprintf(char *bufp, const char *, va_list) __deprecated; +extern int vsprintf(char *bufp, const char *, va_list) __deprecated __printflike(2,0); #ifdef KERNEL_PRIVATE +#ifdef __arm__ +void flush_inner_dcaches(void); +void clean_inner_dcaches(void); +#endif extern void invalidate_icache(vm_offset_t, unsigned, int); extern void flush_dcache(vm_offset_t, unsigned, int); #else @@ -210,8 +220,13 @@ extern void flush_dcache64(addr64_t, unsigned, int); static inline int clz(unsigned int num) { +#if (__arm__ || __arm64__) + // On ARM, clz(0) is defined to return number of bits in the input type + return __builtin_clz(num); +#else // On Intel, clz(0) is undefined return num ? __builtin_clz(num) : sizeof(num) * CHAR_BIT; +#endif } __END_DECLS diff --git a/bsd/libkern/url_encode.c b/bsd/libkern/url_encode.c index 28534ada1..0e3cf1562 100644 --- a/bsd/libkern/url_encode.c +++ b/bsd/libkern/url_encode.c @@ -74,10 +74,7 @@ url_decode(char *str) * string down a few characters */ *esc++ = c; str = memmove(esc, str, strlen(str)+1); - } else { - str++; } - } else { str++; } diff --git a/bsd/machine/Makefile b/bsd/machine/Makefile index 5b190e442..bffe6a814 100644 --- a/bsd/machine/Makefile +++ b/bsd/machine/Makefile @@ -20,7 +20,7 @@ KERNELFILES = \ disklabel.h \ byte_order.h endian.h \ limits.h param.h profile.h \ - signal.h spl.h types.h \ + signal.h types.h \ vmparam.h _types.h _limits.h _param.h \ _mcontext.h diff --git a/bsd/machine/_limits.h b/bsd/machine/_limits.h index c1d8abd07..736a5886b 100644 --- a/bsd/machine/_limits.h +++ b/bsd/machine/_limits.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/_limits.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/_limits.h" #else #error architecture not supported #endif diff --git a/bsd/machine/_mcontext.h b/bsd/machine/_mcontext.h index ee9b1d943..e22043742 100644 --- a/bsd/machine/_mcontext.h +++ b/bsd/machine/_mcontext.h @@ -27,6 +27,8 @@ */ #if defined (__i386__) || defined (__x86_64__) #include "i386/_mcontext.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/_mcontext.h" #else #error architecture not supported #endif diff --git a/bsd/machine/_param.h b/bsd/machine/_param.h index beb2cb939..96b0c2fef 100644 --- a/bsd/machine/_param.h +++ b/bsd/machine/_param.h @@ -27,6 +27,8 @@ */ #if defined (__i386__) || defined (__x86_64__) #include "i386/_param.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/_param.h" #else #error architecture not supported #endif diff --git a/bsd/machine/_types.h b/bsd/machine/_types.h index 92c65bf6c..be86a2368 100644 --- a/bsd/machine/_types.h +++ b/bsd/machine/_types.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/_types.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/_types.h" #else #error architecture not supported #endif diff --git a/bsd/machine/disklabel.h b/bsd/machine/disklabel.h index 490bbda8a..a29df81d3 100644 --- a/bsd/machine/disklabel.h +++ b/bsd/machine/disklabel.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/disklabel.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/disklabel.h" #else #error architecture not supported #endif diff --git a/bsd/machine/endian.h b/bsd/machine/endian.h index 871af6483..9cefbf79a 100644 --- a/bsd/machine/endian.h +++ b/bsd/machine/endian.h @@ -33,6 +33,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/endian.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/endian.h" #else #error architecture not supported #endif diff --git a/bsd/machine/exec.h b/bsd/machine/exec.h index a5712128a..d4bc6a86a 100644 --- a/bsd/machine/exec.h +++ b/bsd/machine/exec.h @@ -46,6 +46,8 @@ boolean_t pie_required(cpu_type_t, cpu_subtype_t); #if defined (__i386__) || defined(__x86_64__) #include "i386/exec.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/exec.h" #else #error architecture not supported #endif diff --git a/bsd/machine/fasttrap_isa.h b/bsd/machine/fasttrap_isa.h index cfe9e297a..7f31b4eec 100644 --- a/bsd/machine/fasttrap_isa.h +++ b/bsd/machine/fasttrap_isa.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/fasttrap_isa.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/fasttrap_isa.h" #else #error architecture not supported #endif diff --git a/bsd/machine/limits.h b/bsd/machine/limits.h index e96709f89..39b348a7b 100644 --- a/bsd/machine/limits.h +++ b/bsd/machine/limits.h @@ -4,6 +4,8 @@ This file is public domain. */ #if defined (__i386__) || defined(__x86_64__) #include +#elif defined (__arm__) || defined (__arm64__) +#include #else #error architecture not supported #endif diff --git a/bsd/machine/param.h b/bsd/machine/param.h index 2724da7e1..74b280059 100644 --- a/bsd/machine/param.h +++ b/bsd/machine/param.h @@ -33,6 +33,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/param.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/param.h" #else #error architecture not supported #endif diff --git a/bsd/machine/profile.h b/bsd/machine/profile.h index cc8a5eac0..14f2977ee 100644 --- a/bsd/machine/profile.h +++ b/bsd/machine/profile.h @@ -35,6 +35,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/profile.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/profile.h" #else #error architecture not supported #endif diff --git a/bsd/machine/psl.h b/bsd/machine/psl.h index 01c6e0a25..6c260a01b 100644 --- a/bsd/machine/psl.h +++ b/bsd/machine/psl.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/psl.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/psl.h" #else #error architecture not supported #endif diff --git a/bsd/machine/ptrace.h b/bsd/machine/ptrace.h index 3320c2226..cb5ecd990 100644 --- a/bsd/machine/ptrace.h +++ b/bsd/machine/ptrace.h @@ -33,6 +33,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/ptrace.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/ptrace.h" #else #error architecture not supported #endif diff --git a/bsd/machine/reboot.h b/bsd/machine/reboot.h index 864f1970c..0a00f2ec2 100644 --- a/bsd/machine/reboot.h +++ b/bsd/machine/reboot.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/reboot.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/reboot.h" #else #error architecture not supported #endif diff --git a/bsd/machine/reg.h b/bsd/machine/reg.h index 30e5dc524..8f4128740 100644 --- a/bsd/machine/reg.h +++ b/bsd/machine/reg.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/reg.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/reg.h" #else #error architecture not supported #endif diff --git a/bsd/machine/signal.h b/bsd/machine/signal.h index 4b7f69c19..46b23f231 100644 --- a/bsd/machine/signal.h +++ b/bsd/machine/signal.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/signal.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/signal.h" #else #error architecture not supported #endif diff --git a/bsd/machine/smp.h b/bsd/machine/smp.h new file mode 100644 index 000000000..f97a38fe8 --- /dev/null +++ b/bsd/machine/smp.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _BSD_MACHINE_SMP_H_ +#define _BSD_MACHINE_SMP_H_ + +#if defined (__i386__) || defined(__x86_64__) +#include "i386/smp.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/smp.h" +#else +#error architecture not supported +#endif + +#endif /* _BSD_MACHINE_SMP_H_ */ diff --git a/bsd/machine/types.h b/bsd/machine/types.h index 5d6d4db44..c14795279 100644 --- a/bsd/machine/types.h +++ b/bsd/machine/types.h @@ -33,6 +33,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/types.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/types.h" #else #error architecture not supported #endif diff --git a/bsd/machine/vmparam.h b/bsd/machine/vmparam.h index 54b212382..3817b5a67 100644 --- a/bsd/machine/vmparam.h +++ b/bsd/machine/vmparam.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/vmparam.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/vmparam.h" #else #error architecture not supported #endif diff --git a/bsd/man/man2/Makefile b/bsd/man/man2/Makefile index 53d932f6d..331ca334e 100644 --- a/bsd/man/man2/Makefile +++ b/bsd/man/man2/Makefile @@ -62,13 +62,19 @@ DATAFILES = \ fpathconf.2 \ fremovexattr.2 \ fsetxattr.2 \ + fsgetpath.2 \ fstat.2 \ fstat64.2 \ fstatat.2 \ fstatfs.2 \ fstatfs64.2 \ fsync.2 \ + fs_snapshot_create.2 \ + fs_snapshot_list.2 \ + fs_snapshot_delete.2 \ + fs_snapshot_rename.2 \ ftruncate.2 \ + futimens.2 \ futimes.2 \ getattrlist.2 \ getattrlistat.2 \ @@ -143,7 +149,6 @@ DATAFILES = \ poll.2 \ posix_madvise.2 \ pread.2 \ - profil.2 \ pselect.2 \ pthread_setugid_np.2 \ ptrace.2 \ @@ -179,6 +184,7 @@ DATAFILES = \ sendmsg.2 \ sendto.2 \ setattrlist.2 \ + setattrlistat.2 \ setaudit.2 \ setaudit_addr.2 \ setauid.2 \ @@ -227,6 +233,7 @@ DATAFILES = \ unlink.2 \ unlinkat.2 \ unmount.2 \ + utimensat.2 \ utimes.2 \ vfork.2 \ wait.2 \ diff --git a/bsd/man/man2/clonefile.2 b/bsd/man/man2/clonefile.2 index c9fd137e6..114321e0d 100644 --- a/bsd/man/man2/clonefile.2 +++ b/bsd/man/man2/clonefile.2 @@ -61,7 +61,9 @@ or .Xr mkdirat 2 or .Xr symlinkat 2 -if the current user does not have privileges to change ownership. +if the current user does not have privileges to change ownership. If the optional +flag CLONE_NOOWNERCOPY is passed, the ownership information is the same as if the +the current user does not have privileges to change ownership . .It @@ -133,6 +135,17 @@ names a symbolic link. . .El .Pp +.Bl -tag -width CLONE_NOOWNERCOPY +. +.It CLONE_NOOWNERCOPY +Don't copy ownership information from the source when run called with superuser privileges. +The symbolic link is itself cloned if +.Fa src +names a symbolic link. +. +.El +.Pp +The The .Fn clonefile , .Fn clonefileat diff --git a/bsd/man/man2/connectx.2 b/bsd/man/man2/connectx.2 index 7fbca576e..6ee2ba8b2 100644 --- a/bsd/man/man2/connectx.2 +++ b/bsd/man/man2/connectx.2 @@ -154,15 +154,6 @@ specifies the length of that buffer. .\" .Fn connectx .\" calls. .\" .Pp -.\" If the initial connection is established without any protocol-level -.\" multipath association, the error -.\" .Er EPROTO -.\" will be returned, and the connection can be extracted to a new socket with -.\" the same properties of -.\" .Fa socket , -.\" by calling -.\" .Xr peeloff 2 . -.\" .Pp .\" An association representing one or more connections, or a single connection .\" may be dissolved by calling .\" .Xr disconnectx 2 . @@ -363,11 +354,6 @@ Because .Fa socket is listening, no connection is allowed. .\" ========== -.\".It Bq Er EPROTO -.\"The connection was successfully established without any protocol-level -.\"association. The connection can be extracted to a new socket using -.\".Xr peeloff 2 . -.\" ========== .\".It Bq Er EPROTOTYPE .\".Fa address .\"has a different type than the socket @@ -385,7 +371,6 @@ Connection establishment timed out without establishing a connection. .Xr disconnectx 2 , .Xr getsockopt 2 , .\".Xr kqueue 2 , -.\".Xr peeloff 2 , .\".Xr shutdown 2 , .Xr select 2 , .Xr socket 2 , diff --git a/bsd/man/man2/exchangedata.2 b/bsd/man/man2/exchangedata.2 index 83dc23c1b..10a22ae34 100644 --- a/bsd/man/man2/exchangedata.2 +++ b/bsd/man/man2/exchangedata.2 @@ -182,10 +182,12 @@ An I/O error occurred while reading from or writing to the file system. . .Sh SEE ALSO . -.Xr getattrlist 2 +.Xr getattrlist 2 , +.Xr rename 2 . .Sh HISTORY A .Fn exchangedata function call appeared in Darwin 1.3.1 (Mac OS X version 10.0). . +It was deprecated in macOS 10.13. diff --git a/bsd/man/man2/fcntl.2 b/bsd/man/man2/fcntl.2 index dc3a20805..b55972bd9 100644 --- a/bsd/man/man2/fcntl.2 +++ b/bsd/man/man2/fcntl.2 @@ -188,11 +188,11 @@ A value of zero in turns data caching on. .It Dv F_LOG2PHYS Get disk device information. -Currently this only includes the +Currently this only returns the disk device address that corresponds -to the current file offset. Note that if the -file offset is not backed by physical blocks -we can return -1 as the offset. This is subject +to the current file offset. Note that the system +may return -1 as the disk device address if the file is not +backed by physical blocks. This is subject to change. .It Dv F_LOG2PHYS_EXT Variant of F_LOG2PHYS that uses the passed in diff --git a/bsd/man/man2/fs_snapshot_create.2 b/bsd/man/man2/fs_snapshot_create.2 new file mode 100644 index 000000000..57e3b3d34 --- /dev/null +++ b/bsd/man/man2/fs_snapshot_create.2 @@ -0,0 +1,201 @@ +.\" Copyright (c) 2017 Apple Computer, Inc. All rights reserved. +.\" +.\" The contents of this file constitute Original Code as defined in and +.\" are subject to the Apple Public Source License Version 1.1 (the +.\" "License"). You may not use this file except in compliance with the +.\" License. Please obtain a copy of the License at +.\" http://www.apple.com/publicsource and read it before using this file. +.\" +.\" This Original Code and all software distributed under the License are +.\" distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER +.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, +.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, +.\" FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the +.\" License for the specific language governing rights and limitations +.\" under the License. +.\" +.\" @(#)fs_snapshot_create.2 +. +.Dd July 4th, 2017 +.Dt FS_SNAPSHOT_CREATE 2 +.Os Darwin +.Sh NAME +.Nm fs_snasphot_create +.Nd create read only snapshot of a mounted filesystem +.Sh SYNOPSIS +.Fd #include +.Fd #include +.Pp +.Ft int +.Fn fs_snapshot_create "int dirfd" "const char * name" "uint32_t flags" +. +.Ft int +.Fn fs_snapshot_delete "int dirfd" "const char * name" "uint32_t flags" +. +.Ft int +.Fn fs_snapshot_list "int dirfd" "struct attrlist * name" "void * attrbuf" "size_t bufsize" "uint32_t flags" +. +.Ft int +.Fn fs_snapshot_rename "int dirfd" "const char * old" "const char * new" "uint32_t flags" +. +.Ft int +.Fn fs_snapshot_mount "int dirfd" "const char * dir" "const char * snapshot" "uint32_t flags" +. +.Ft int +.Fn fs_snapshot_revert "int dirfd" "const char * name" "uint32_t flags" +. +.Sh DESCRIPTION +The +.Fn fs_snapshot_create +function, for supported Filesystems, causes a snapshot of the Filesystem to be created. A snapshot is a read only copy +of the filesystem frozen at a point in time. The Filesystem is identified by the the +.Fa dirfd +parameter which should be a file descriptor associated with the root directory of the filesystem for the snapshot is to be created. +.Fa name +can be any valid name for a component name (except . and ..). +. +The +.Fn fs_snapshot_delete +function causes the named snapshot +.Fa name +to be deleted and the +.Fn fs_snapshot_rename +function causes the named snapshot +.Fa old +to be renamed to the name +.Fa new . +Available snapshots along with their attributes can be listed by calling +.Fn fs_snapshot_list +which is to be used in exactly the same way as +.Xr getattrlistbulk 2 . +. +The +.Fa flags +parameter specifies the options that can be passed. No options are currently defined. +.Pp +Snapshots may be useful for backing up the Filesystem and to restore the Filesystem to a previous state. +Snapshots are expected to consume no additional storage on creation but might consume additional storage as the active +Filesystem is modified. Similarly deletion of files on the active filesystem may not result in the storage being available +if the snapshot contains the file. Additionally, the underlying Filesystem may impose a limit on the number +of snapshots that can be taken. For supporting Filesystems, a snapshot may be used as a source for a mount. This can be done +by the +.Fn fs_snapshot_mount +function. The snapshot will be mounted read only. When a snapshot is mounted, it cannot be deleted but it can be renamed. +To revert the filesystem to a previous snapshot, the +.Fn fs_snapshot_revert +can be used. It should be noted that reverting a filesystem to a snapshot is a destructive operation and causes all +changes made to the filesystem (including snapshots created after the snapshot being reverted to) to be lost. +. +.Pp +All snapshot functions require superuser privileges and also require an additional entitlement. +. +.Sh RETURN VALUES +Upon successful completion, +.Fn fs_snapshot_create +, +.Fn fs_snapshot_delete +, +.Fn fs_snapshot_rename +and +.Fn fs_snapshot_list +returns 0. Otherwise, a value of -1 is returned and errno is set to indicate the error. +.Pp +.Sh COMPATIBILITY +Not all volumes support snapshots. A volume can be tested for snapshot support +by using +.Xr getattrlist 2 +to get the volume capabilities attribute ATTR_VOL_CAPABILITIES, and then testing the VOL_CAP_INT_SNAPSHOT flag. +.Pp +.Sh ERRORS +The +.Fn fs_snapshot_create +, +.Fn fs_snapshot_delete +, +.Fn fs_snapshot_rename +and +.Fn fs_snapshot_list +function will fail if: +.Bl -tag -width Er +. +.It Bq Er EACCES +Read permissions are denied for the caller on the filesystem +. +.It Bq Er ENOTSUP +The underlying filesystem does not support this call. +. +.It Bq Er EINVAL +The value of the +.Fa flags +parameter is invalid. +. +.It Bq Er ENOSPC +There is no free space remaining on the file system containing the file. +. +.It Bq Er ENOSPC +The limit for the maximum number of snapshots for a filesystem has been reached. +. +.It Bq Er EIO +An I/O error occurred while reading from or writing to the file system. +. +.It Bq Er EPERM +The calling process does not have appropriate privileges. +. +.It Bq Er EROFS +The requested operation requires modifications in a read-only file system. +. +.It Bq Er ENAMETOOLONG +The length of a component of a pathname is longer than {NAME_MAX}. +. +.It Bq Er EBADF +dirfd is not a valid file descriptor. +. +.It Bq Er ENOTDIR +dirfd is a file descriptor associated with a non-directory file. +.El +.Pp +In addition, the +.Fn fs_snapshot_create +or +.Fn fs_snapshot_rename +functions may fail with the following errors +.Bl -tag -width Er +.It Bq Er EEXIST +The The named snapshot to be created already exists or the new name already +exists for the snapshot being renamed. +. +.El +.Pp +.Fn fs_snapshot_create +or +.Fn fs_snapshot_rename +functions may fail with the following errors +.Bl -tag -width Er +.It Bq Er ENOENT +The named snapshot does not exist. +.El +. +.Pp +.Fn fs_snapshot_delete +function may fail with +.Bl -tag -width Er +.It Bq Er EBUSY +The named snapshot is currently mounted. +.El +. +.Sh SEE ALSO +. +.Xr getattrlist 2 , +.Xr getattrlistbulk 2 +. +.Sh HISTORY +The +.Fn fs_snapshot_create +, +.Fn fs_snapshot_delete +, +.Fn fs_snapshot_delete +and +.Fn fs_snapshot_list +function calls appeared in macOS version 10.13 +. diff --git a/bsd/man/man2/fs_snapshot_delete.2 b/bsd/man/man2/fs_snapshot_delete.2 new file mode 100644 index 000000000..20620315a --- /dev/null +++ b/bsd/man/man2/fs_snapshot_delete.2 @@ -0,0 +1 @@ +.so man2/fs_snapshot_create.2 diff --git a/bsd/man/man2/fs_snapshot_list.2 b/bsd/man/man2/fs_snapshot_list.2 new file mode 100644 index 000000000..20620315a --- /dev/null +++ b/bsd/man/man2/fs_snapshot_list.2 @@ -0,0 +1 @@ +.so man2/fs_snapshot_create.2 diff --git a/bsd/man/man2/fs_snapshot_rename.2 b/bsd/man/man2/fs_snapshot_rename.2 new file mode 100644 index 000000000..20620315a --- /dev/null +++ b/bsd/man/man2/fs_snapshot_rename.2 @@ -0,0 +1 @@ +.so man2/fs_snapshot_create.2 diff --git a/bsd/man/man2/fsgetpath.2 b/bsd/man/man2/fsgetpath.2 new file mode 100644 index 000000000..317c45cb7 --- /dev/null +++ b/bsd/man/man2/fsgetpath.2 @@ -0,0 +1,126 @@ +.\" Copyright (c) 2017 Apple Computer, Inc. All rights reserved. +.\" +.\" The contents of this file constitute Original Code as defined in and +.\" are subject to the Apple Public Source License Version 1.1 (the +.\" "License"). You may not use this file except in compliance with the +.\" License. Please obtain a copy of the License at +.\" http://www.apple.com/publicsource and read it before using this file. +.\" +.\" This Original Code and all software distributed under the License are +.\" distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER +.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, +.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, +.\" FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the +.\" License for the specific language governing rights and limitations +.\" under the License. +.\" +.\" @(#)fsgetpath.2 +. +.Dd July 27, 2017 +.Dt FSGETPATH 2 +.Os Darwin +.Sh NAME +.Nm fsgetpath +.Nd get the path associated with filesystem node identifier (inode number/link id/object id) +.Sh SYNOPSIS +.Fd #include +.Fd #include +.Pp +.Ft ssize_t +.Fn fsgetpath "char * restrict_buf" "size_t buflen" "fsid_t * fsid" "uint64_t obj_id" +. +.Sh DESCRIPTION +The +.Fn fsgetpath +function returns the path in a caller provided buffer +.Fa restrict_buf +of length indicated by +.Fa buflen +associated with a filesystem object identified by +.Fa fsid +and +.Fa obj_id. +.Fa fsid +is a pointer to a structure which identifies a filesystem to which the object belongs. +It is obtained by the value returned for ATTR_CMN_FSID in a previous call to +.Xr getattrlist 2 +or the +.Fa f_fsid +field of the +.Vt statfs +structure returned by +.Xr statfs 2 . +.Fa obj_id +can be any one of of a object identifier i.e. ATTR_CMN_FILEID returned by +.Xr getattrlist 2 +or +.Fa st_ino +field of the +.Vt stat +structure returned by +.Xr stat 2 +or a link id returned in ATTR_CMNEXT_LINKID by a previous call to +.Xr getattrlist 2 . +Using a linkid will result in a more accurate path in case the filesystem object is a +hard link. If a inode number is passed and the object is a hard link, any one of the +multiple paths to that filesystem object may be returned. +.Sh RETURN VALUES +Upon successful completion, +.Fn fsgetpath +returns the path length. Otherwise, a value of -1 is returned and errno is set to indicate the error. +.Pp +.Sh COMPATIBILITY +Not all volumes support +.Fn fsgetpath . +A volume can be tested for +.Fn fsgetpath +support by using +.Xr getattrlist 2 +to get the volume capabilities attribute ATTR_VOL_CAPABILITIES, and then testing the VOL_CAP_FMT_PATH_FROM_ID flag. +.Pp +.Sh ERRORS +The +.Fn fsgetpath +function will fail if: +.Bl -tag -width Er +. +.It Bq Er EACCES +Read permissions are denied on any component of the pathname. +. +.It Bq Er ENOTSUP +The underlying filesystem does not support this call. +. +.It Bq Er EINVAL +.Fa buflen +is larger than PAGE_SIZE +. +.It Bq Er EIO +An I/O error occurred while reading from the file system. +. +.It Bq Er EPERM +The calling process does not have appropriate privileges. +. +.It Bq Er ENOENT +The Filesystem object does not exist. +. +.It Bq Er EFAULT +restrict_buf points to memory not valid in the callers address space. +. +.It Bq Er ENOSPC +restrict_buf is not large enough to hold the path. +. +.El +. +.Pp +. +.Sh SEE ALSO +. +.Xr getattrlist 2 +.Xr statfs 2 +.Xr stat 2 +. +.Sh HISTORY +The +.Fn fsgetpath +function call appeared in macOS version 10.13 +. diff --git a/bsd/man/man2/futimens.2 b/bsd/man/man2/futimens.2 new file mode 100644 index 000000000..a365c7b53 --- /dev/null +++ b/bsd/man/man2/futimens.2 @@ -0,0 +1 @@ +.so man2/utimensat.2 diff --git a/bsd/man/man2/getattrlist.2 b/bsd/man/man2/getattrlist.2 index d0c23207c..a92c08dcd 100644 --- a/bsd/man/man2/getattrlist.2 +++ b/bsd/man/man2/getattrlist.2 @@ -435,14 +435,19 @@ An structure that uniquely identifies the file system object within a mounted volume for the duration of it's mount; this identifier is not guaranteed to be persistent for the volume and may change every time the volume is mounted. -If the VOL_CAP_FMT_64BIT_OBJECT_IDS capability is set, this is instead a 64-bit -object identifier. .Pp On HFS+ volumes, the ATTR_CMN_OBJID of a file system object is distinct from the ATTR_CMN_OBJID of any hard link to that file system object. Although the ATTR_CMN_OBJID of a file system object may appear similar (in whole or in part) to it's ATTR_CMN_FILEID (see description of ATTR_CMN_FILEID below), \fBno relation between the two attributes should ever be implied.\fP +.Pp +ATTR_CMN_OBJID is deprecated sarting with macOS 10.13, iOS 11.0, watchOS 4.0 and +tvOS 11.0 and ATTR_CMNEXT_LINKID should be used in its place. +ATTR_CMN_OBJID can only be used on older operating systems only if the file +system doesn't 64 bit IDs. See the +.Fn getLinkIDInfo +function in the EXAMPLES section. . .It ATTR_CMN_OBJPERMANENTID An @@ -450,8 +455,6 @@ An structure that uniquely and persistently identifies the file system object within its volume; persistence implies that this attribute is unaffected by mount/unmount operations on the volume. -If the VOL_CAP_FMT_64BIT_OBJECT_IDS capability is set, this is instead a 64-bit -object identifier. .Pp Some file systems can not return this attribute when the volume is mounted read-only and will fail the request with error @@ -467,8 +470,6 @@ structure that uniquely identifies the parent directory of the file system object within a mounted volume, for the duration of the volume mount; this identifier is not guaranteed to be persistent for the volume and may change every time the volume is mounted. -If the VOL_CAP_FMT_64BIT_OBJECT_IDS capability is set, this is instead a 64-bit -object identifier. .Pp . If a file system object is hard linked from multiple directories, the parent @@ -677,7 +678,7 @@ Analoguous to .It ATTR_CMN_FILEID A .Vt u_int64_t -that uniquely identifies the file system object within it's mounted volume. +that uniquely identifies the file system object within its mounted volume. Equivalent to .Fa st_ino field of the @@ -1168,6 +1169,18 @@ An containing the number of bytes that are \fBnot\fP trapped inside a clone or snapshot, and which would be freed immediately if the file were deleted. . +.It ATTR_CMNEXT_LINKID +A +.Vt u_int64_t +that uniquely identifies the file system object within a mounted volume for the +duration of its mount. +.Pp +On HFS+ and APFS volumes, the ATTR_CMNEXT_LINKID of a file system +object is distinct from the ATTR_CMNEXT_LINKID of any hard link to that file +system object. Although the ATTR_CMNEXT_LINKID of a file system object may appear +similar (in whole or in part) to its ATTR_CMN_FILEID (see description of +ATTR_CMN_FILEID above), \fBno relation between the two attributes should ever be implied.\fP +. .El . .Sh VOLUME CAPABILITIES @@ -1528,6 +1541,12 @@ See .Xr clonefileat 2 for more details. . +.It VOL_CAP_INT_SNAPSHOT +If this bit is set, the file system supports snapshots. +See +.Xr fs_snapshot_create 2 +for more details. +. .It VOL_CAP_INT_NAMEDSTREAMS If this bit is set, the volume format implementation supports native named streams. @@ -2050,6 +2069,56 @@ main(int argc, char **argv) return 0; } .Ed +.Pp + The getLinkIDInfo() function determines if ATTR_CMNEXT_LINKID and ATTR_CMN_OBJID + are valid to use on the file system specified by path. +. +.Bd -literal +int getLinkIDInfo(const char *path, bool *cmnExtLinkIDValid, bool *cmnObjIDValid) +{ + int result; + struct statfs statfsBuf; + struct attrlist attrList; + struct volAttrsBuf { + u_int32_t length; + vol_capabilities_attr_t capabilities; + vol_attributes_attr_t attributes; + } __attribute__((aligned(4), packed)); + struct volAttrsBuf volAttrs; +.Pp + memset(&attrList, 0, sizeof(attrList)); + attrList.bitmapcount = ATTR_BIT_MAP_COUNT; + attrList.volattr = ATTR_VOL_INFO | ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES; + // get the file system's mount point path for the input path + result = statfs(path, &statfsBuf); + if ( result == 0 ) { + // get the supported capabilities and attributes + result = getattrlist(statfsBuf.f_mntonname, &attrList, &volAttrs, sizeof(volAttrs), FSOPT_ATTR_CMN_EXTENDED); + if ( result == 0 ) { + if ( volAttrs.attributes.validattr.forkattr & ATTR_CMNEXT_LINKID ) { + // ATTR_CMNEXT_LINKID is available; do not use ATTR_CMN_OBJID + *cmnExtLinkIDValid = true; + *cmnObjIDValid = false; + } + else { + // ATTR_CMNEXT_LINKID is not available + cmnExtLinkIDValid = false; + // ATTR_CMN_OBJID can only be used if the file system does not use 64-bit object IDs + if ( (volAttrs.capabilities.capabilities[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_64BIT_OBJECT_IDS) && (volAttrs.capabilities.valid[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_64BIT_OBJECT_IDS) ) { + *cmnObjIDValid = false; + } + else { + *cmnObjIDValid = true; + } + } + } + } + if ( result != 0 ) { + *cmnExtLinkIDValid = *cmnObjIDValid = false; + } + return result; +} +.Ed .Pp . .Sh SEE ALSO diff --git a/bsd/man/man2/kqueue.2 b/bsd/man/man2/kqueue.2 index 6f1ce36ef..d1a535542 100644 --- a/bsd/man/man2/kqueue.2 +++ b/bsd/man/man2/kqueue.2 @@ -595,41 +595,65 @@ area provided to .Fn kevent_qos if there is enough space remaining there. .It EVFILT_TIMER -Establishes an interval timer with the data -timer identified by -.Va ident . -When adding a timer, +Establishes an interval timer identified by +.Va ident +where .Va data -specifies the timeout period and +specifies the timeout period (in milliseconds). +.Pp .Va fflags -can be set to one of the following: -.Bl -tag -width NOTE_ABSOLUTE +can include one of the following flags to specify a different unit: +.Bl -tag -width NOTE_NSECONDS .It NOTE_SECONDS -data is in seconds +.Va data +is in seconds .It NOTE_USECONDS -data is in microseconds +.Va data +is in microseconds .It NOTE_NSECONDS -data is in nanoseconds -.It NOTE_ABSOLUTE -data is an absolute timeout +.Va data +is in nanoseconds +.It NOTE_MACHTIME +.Va data +is in Mach absolute time units +.El +.Pp +.Va fflags +can also include +.Dv NOTE_ABSOLUTE, +which establishes an +.Dv EV_ONESHOT +timer with an absolute deadline instead of an interval. +The absolute deadline is expressed in terms of +.Xr gettimeofday 2 . +With +.Dv NOTE_MACHTIME, +the deadline is expressed in terms of +.Fn mach_absolute_time . +.Pp +The timer can be coalesced with other timers to save power. The following flags can be set in +.Va fflags +to modify this behavior: +.Bl -tag -width NOTE_BACKGROUND .It NOTE_CRITICAL -system makes a best effort to fire this timer as scheduled. +override default power-saving techniques to more strictly respect the leeway value .It NOTE_BACKGROUND -system has extra leeway to coalesce this timer. +apply more power-saving techniques to coalesce this timer with other timers .It NOTE_LEEWAY -ext[1] holds user-supplied slop in deadline for timer coalescing. +.Va ext[1] +holds user-supplied slop in deadline for timer coalescing. .El .Pp -If fflags is not set, the default is milliseconds. The timer will be periodic unless EV_ONESHOT is specified. +The timer will be periodic unless +.Dv EV_ONESHOT +is specified. On return, .Va data -contains the number of times the timeout has expired since the last call to -.Fn kevent , -.Fn kevent64 -or -.Fn kevent_qos . - -This filter automatically sets the EV_CLEAR flag internally. +contains the number of times the timeout has expired since the last arming or last delivery of the timer event. +.Pp +This filter automatically sets the +.Dv EV_CLEAR +flag. .El .Pp ---- diff --git a/bsd/man/man2/mount.2 b/bsd/man/man2/mount.2 index e3245d9f3..586707fa6 100644 --- a/bsd/man/man2/mount.2 +++ b/bsd/man/man2/mount.2 @@ -39,6 +39,7 @@ .Os BSD 4 .Sh NAME .Nm mount , +.Nm fmount, .Nm unmount .Nd mount or dismount a filesystem .Sh SYNOPSIS @@ -47,6 +48,8 @@ .Ft int .Fn mount "const char *type" "const char *dir" "int flags" "void *data" .Ft int +.Fn fmount "const char *type" "int fd" "int flags" "void *data" +.Ft int .Fn unmount "const char *dir" "int flags" .Sh DESCRIPTION The @@ -122,6 +125,15 @@ The format for these argument structures is described in the manual page for each filesystem. .Pp The +.Fn fmount +function call is equivalent to the +.Fn mount +function call, except in the use of the second argument. +It takes an open file descriptor representing mount point +instead of the string literal containing full path to the mount +point in the filesystem hierarchy. +.Pp +The .Fn unmount function call disassociates the filesystem from the specified mount point @@ -139,7 +151,9 @@ even if the filesystem is later remounted. .Sh RETURN VALUES The .Fn mount -returns the value 0 if the mount was successful, otherwise -1 is returned +and +.Fn fmount +return the value 0 if the mount was successful, otherwise -1 is returned and the variable .Va errno is set to indicate the error. @@ -151,6 +165,8 @@ and the variable is set to indicate the error. .Sh ERRORS .Fn mount +and +.Fn fmount will fail when one of the following occurs: .Bl -tag -width [ENAMETOOLONG] .It Bq Er EPERM @@ -217,7 +233,8 @@ points outside the process's allocated address space. .El .Sh SEE ALSO .Xr mount 8 , -.Xr unmount 8 +.Xr unmount 8 , +.Xr open 2 .Sh BUGS Some of the error codes need translation to more obvious messages. .Sh HISTORY @@ -226,3 +243,5 @@ and .Fn unmount function calls appeared in .At v6 . +.Fn fmount +function call first appeared in macOS version 10.13. diff --git a/bsd/man/man2/peeloff.2 b/bsd/man/man2/peeloff.2 deleted file mode 100644 index 3ba0acb18..000000000 --- a/bsd/man/man2/peeloff.2 +++ /dev/null @@ -1,99 +0,0 @@ -.\" -.\" Copyright (c) 2012 Apple Inc. All rights reserved. -.\" -.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@ -.\" -.\" This file contains Original Code and/or Modifications of Original Code -.\" as defined in and that are subject to the Apple Public Source License -.\" Version 2.0 (the 'License'). You may not use this file except in -.\" compliance with the License. The rights granted to you under the License -.\" may not be used to create, or enable the creation or redistribution of, -.\" unlawful or unlicensed copies of an Apple operating system, or to -.\" circumvent, violate, or enable the circumvention or violation of, any -.\" terms of an Apple operating system software license agreement. -.\" -.\" Please obtain a copy of the License at -.\" http://www.opensource.apple.com/apsl/ and read it before using this file. -.\" -.\" The Original Code and all software distributed under the License are -.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER -.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, -.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, -.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. -.\" Please see the License for the specific language governing rights and -.\" limitations under the License. -.\" -.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@ -.\" -.Dd November 14, 2012 -.Dt PEELOFF 2 -.Os Darwin -.Sh NAME -.Nm peeloff -.Nd extracts an association from a socket -.Sh SYNOPSIS -.Fd #include -.Ft int -.Fo peeloff -.Fa "int socket" -.Fa "sae_associd_t associd" -.Fc -.Sh DESCRIPTION -The parameter -.Fa socket -is a socket. The communication domain of the socket determines the -availability and behavior of -.Fn peeloff . -In general, -.Fn peeloff -attempts to extract the association specified by -.Fa associd -into its own separate socket. -.Pp -The parameter -.Fa associd -specifies the association identifier. It may be set to -.Dv SAE_ASSOCID_ANY -when there is only one association present; or one of the identifiers -returned from -.Xr getassocids 3 . -.Sh RETURN VALUES -The -.Fn peeloff -function returns -1 on error and the global variable -.Va errno -is set to indicate the error. If it succeeds, it returns a non-negative -integer that is a descriptor for the extracted association. -.Sh ERRORS -The -.Fn peeloff -system call succeeds unless: -.Bl -tag -width Er -.\" =========== -.It Bq Er EBADF -.Fa Socket -is not a valid descriptor. -.\" =========== -.It Bq Er EINVAL -The -.Fa associd -argument is invalid; cannot be extracted; or the underlying protocol -is no longer attached to -.Fa socket . -.\" =========== -.It Bq Er ENOTSOCK -.Fa Socket -is a file, not a socket. -.El -.Sh SEE ALSO -.Xr connectx 2 , -.Xr disconnectx 2 , -.Xr socket 2 , -.Xr getassocids 3 , -.Xr getconnids 3 , -.Xr getconninfo 3 , -.Xr compat 5 -.Sh HISTORY -The -.Fn peeloff -function call appeared in Darwin 13.0.0 diff --git a/bsd/man/man2/posix_spawn.2 b/bsd/man/man2/posix_spawn.2 index aa339f665..83dea9536 100644 --- a/bsd/man/man2/posix_spawn.2 +++ b/bsd/man/man2/posix_spawn.2 @@ -324,6 +324,9 @@ A component of the path prefix is not a directory. .It Bq Er ETXTBSY The new process file is a pure procedure (shared text) file that is currently open for writing or reading by some process. +.\" ========== +.It Bq Er EBADARCH +The new process file has no architectures appropriate for the current system. .El .Pp Additionally, they may fail for any of the reasons listed in diff --git a/bsd/man/man2/profil.2 b/bsd/man/man2/profil.2 deleted file mode 100644 index 40ee9a463..000000000 --- a/bsd/man/man2/profil.2 +++ /dev/null @@ -1,144 +0,0 @@ -.\" $NetBSD: profil.2,v 1.3 1995/11/22 23:07:23 cgd Exp $ -.\" -.\" Copyright (c) 1993 -.\" The Regents of the University of California. All rights reserved. -.\" -.\" This code is derived from software contributed to Berkeley by -.\" Donn Seeley of BSDI. -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted provided that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" 3. All advertising materials mentioning features or use of this software -.\" must display the following acknowledgement: -.\" This product includes software developed by the University of -.\" California, Berkeley and its contributors. -.\" 4. Neither the name of the University nor the names of its contributors -.\" may be used to endorse or promote products derived from this software -.\" without specific prior written permission. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND -.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE -.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -.\" SUCH DAMAGE. -.\" -.\" @(#)profil.2 8.1 (Berkeley) 6/4/93 -.\" -.Dd September 26, 2008 -.Dt PROFIL 2 -.Os -.Sh NAME -.Nm profil -.Nd control process profiling -.Sh LIBRARY -.Lb libc -.Sh SYNOPSIS -.In unistd.h -.Ft int -.Fn profil "char *samples" "size_t size" "u_long offset" "u_int scale" -.Sh DESCRIPTION -.Pp -.Fd -- This function is now deprecated. It will always return EINVAL. -- -.Pp -The intended replacements are the user-level developer tools, like CHUD and dtrace. -.Pp ------- -.Pp -The -.Fn profil -function enables or disables -program counter profiling of the current process. -If profiling is enabled, -then at every clock tick, -the kernel updates an appropriate count in the -.Fa samples -buffer. -.Pp -The buffer -.Fa samples -contains -.Fa size -bytes and is divided into -a series of 16-bit bins. -Each bin counts the number of times the program counter -was in a particular address range in the process -when a clock tick occurred while profiling was enabled. -For a given program counter address, -the number of the corresponding bin is given -by the relation: -.Bd -literal -offset indent -[(pc - offset) / 2] * scale / 65536 -.Ed -.Pp -The -.Fa offset -parameter is the lowest address at which -the kernel takes program counter samples. -The -.Fa scale -parameter ranges from 1 to 65536 and -can be used to change the span of the bins. -A scale of 65536 maps each bin to 2 bytes of address range; -a scale of 32768 gives 4 bytes, 16384 gives 8 bytes and so on. -Intermediate values provide approximate intermediate ranges. -A -.Fa scale -value of 0 disables profiling. -.Sh RETURN VALUES -If the -.Fa scale -value is nonzero and the buffer -.Fa samples -contains an illegal address, -.Fn profil -returns \-1, -profiling is terminated and -.Va errno -is set appropriately. -Otherwise -.Fn profil -returns 0. -.Sh FILES -.Bl -tag -width /usr/lib/gcrt0.o -compact -.It Pa /usr/lib/gcrt0.o -profiling C run-time startup file -.It Pa gmon.out -conventional name for profiling output file -.El -.Sh ERRORS -The following error may be reported: -.Bl -tag -width Er -.It Bq Er EFAULT -The buffer -.Fa samples -contains an invalid address. -.El -.Sh SEE ALSO -.Xr gprof 1 -.Sh HISTORY -The -.Fn profil -function appeared in -.At v7 . -.Sh BUGS -This routine should be named -.Fn profile . -.Pp -The -.Fa samples -argument should really be a vector of type -.Fa "unsigned short" . -.Pp -The format of the gmon.out file is undocumented. diff --git a/bsd/man/man2/readlink.2 b/bsd/man/man2/readlink.2 index 375eca319..2940e9ced 100644 --- a/bsd/man/man2/readlink.2 +++ b/bsd/man/man2/readlink.2 @@ -85,6 +85,7 @@ in the .Fa fd parameter, the current working directory is used and the behavior is identical to a call to +.Fn readlink . .Sh RETURN VALUES The call returns the count of characters placed in the buffer if it succeeds, or a -1 if an error occurs, placing the error diff --git a/bsd/man/man2/setattrlist.2 b/bsd/man/man2/setattrlist.2 index 68898eb3c..4dcc7340d 100644 --- a/bsd/man/man2/setattrlist.2 +++ b/bsd/man/man2/setattrlist.2 @@ -21,15 +21,18 @@ .Os Darwin .Sh NAME .Nm setattrlist , -.Nm fsetattrlist +.Nm fsetattrlist , +.Nm setattrlistat .Nd set file system attributes .Sh SYNOPSIS .Fd #include .Fd #include .Ft int -.Fn setattrlist "const char* path" "struct attrlist * attrList" "void * attrBuf" "size_t attrBufSize" "unsigned long options" +.Fn setattrlist "const char * path" "struct attrlist * attrList" "void * attrBuf" "size_t attrBufSize" "unsigned long options" .Ft int .Fn fsetattrlist "int fd" "struct attrlist * attrList" "void * attrBuf" "size_t attrBufSize" "unsigned long options" +.Ft int +.Fn setattrlistat "int dir_fd" "const char * path" "struct attrlist * attrList" "void * attrBuf" "size_t attrBufSize" "uint32_t options" . .Sh DESCRIPTION The @@ -59,6 +62,27 @@ The .Fa options parameter lets you control specific aspects of the function's behaviour. .Pp +The +.Fn setattrlistat +system call is equivalent to +.Fn setattrlist +except in the case where +.Fa path +specifies a relative path. +In this case the attributes are set for the file system object named by +path relative to the directory associated with the file descriptor +.Fa fd +instead of the current working directory. +If +.Fn setattrlistat +is passed the special value +.Dv AT_FDCWD +in the +.Fa fd +parameter, the current working directory is used and the behavior is +identical to a call to +.Fn setattrlist . +.Pp . The functions are only supported by certain volume format implementations. @@ -122,6 +146,8 @@ ATTR_CMN_FLAGS ATTR_CMN_EXTENDED_SECURITY .It ATTR_CMN_GRPUUID +.It +ATTR_CMN_ADDEDTIME .Pp .It ATTR_VOL_NAME @@ -151,6 +177,8 @@ ATTR_CMN_CRTIME ATTR_CMN_MODTIME .It ATTR_CMN_ACCTIME +.It +ATTR_CMN_ADDEDTIME .Pp ATTR_CMN_CHGTIME .Fa cannot be set programmatically. Any attempt to set change time is ignored. @@ -336,6 +364,31 @@ is too small to hold all the attributes that you are trying to set. An I/O error occurred while reading from or writing to the file system. .El .Pp +.Pp +In addition to the errors returned by the +.Fn setattrlist , +the +.Fn setattrlistat +function may fail if: +.Bl -tag -width Er +.It Bq Er EBADF +The +.Fa path +argument does not specify an absolute path and the +.Fa fd +argument is neither +.Dv AT_FDCWD +nor a valid file descriptor open for searching. +.It Bq Er ENOTDIR +The +.Fa path +argument is not an absolute path and +.Fa fd +is neither +.Dv AT_FDCWD +nor a file descriptor associated with a directory. +.El +.Pp . .Sh CAVEATS . @@ -443,5 +496,6 @@ static int FInfoDemo( .Sh HISTORY A .Fn setattrlist -function call appeared in Darwin 1.3.1 (Mac OS X version 10.0). +function call appeared in Darwin 1.3.1 (Mac OS X version 10.0). The setatrlistat function call first +appeared in macOS version 10.13. . diff --git a/bsd/man/man2/setattrlistat.2 b/bsd/man/man2/setattrlistat.2 new file mode 100644 index 000000000..f823a2cac --- /dev/null +++ b/bsd/man/man2/setattrlistat.2 @@ -0,0 +1 @@ +.so man2/setattrlist.2 diff --git a/bsd/man/man2/utimensat.2 b/bsd/man/man2/utimensat.2 new file mode 100644 index 000000000..0b5e8f8e8 --- /dev/null +++ b/bsd/man/man2/utimensat.2 @@ -0,0 +1,256 @@ +.\" $NetBSD: utimes.2,v 1.13 1999/03/22 19:45:11 garbled Exp $ +.\" +.\" Copyright (c) 1990, 1993 +.\" The Regents of the University of California. All rights reserved. +.\" Copyright (c) 2012, Jilles Tjoelker +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" 3. Neither the name of the University nor the names of its contributors +.\" may be used to endorse or promote products derived from this software +.\" without specific prior written permission. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" @(#)utimes.2 8.1 (Berkeley) 6/4/93 +.\" $FreeBSD$ +.\" +.Dd January 17, 2016 +.Dt UTIMENSAT 2 +.Os +.Sh NAME +.Nm futimens , +.Nm utimensat +.Nd set file access and modification times +.Sh SYNOPSIS +.In sys/stat.h +.Ft int +.Fn futimens "int fd" "const struct timespec times[2]" +.Ft int +.Fo utimensat +.Fa "int fd" +.Fa "const char *path" +.Fa "const struct timespec times[2]" +.Fa "int flag" +.Fc +.Sh DESCRIPTION +The access and modification times of the file named by +.Fa path +or referenced by +.Fa fd +are changed as specified by the argument +.Fa times . +The inode-change-time of the file is set to the current time. +.Pp +If +.Fa path +specifies a relative path, +it is relative to the current working directory if +.Fa fd +is +.Dv AT_FDCWD +and otherwise relative to the directory associated with the file descriptor +.Fa fd . +.Pp +The +.Va tv_nsec +field of a +.Vt timespec +structure +can be set to the special value +.Dv UTIME_NOW +to set the current time, or to +.Dv UTIME_OMIT +to leave the time unchanged. +In either case, the +.Va tv_sec +field is ignored. +.Pp +If +.Fa times +is +.No non- Ns Dv NULL , +it is assumed to point to an array of two timespec structures. +The access time is set to the value of the first element, and the +modification time is set to the value of the second element. +If +.Fa times +is +.Dv NULL , +this is equivalent to passing +a pointer to an array of two timespec structures +with both +.Va tv_nsec +fields set to +.Dv UTIME_NOW . +.Pp +If both +.Va tv_nsec +fields are +.Dv UTIME_OMIT , +the timestamps remain unchanged and +no permissions are needed for the file itself, +although search permissions may be required for the path prefix. +The call may or may not succeed if the named file does not exist. +.Pp +If both +.Va tv_nsec +fields are +.Dv UTIME_NOW , +the caller must be the owner of the file, have permission to +write the file, or be the super-user. +.Pp +For all other values of the timestamps, +the caller must be the owner of the file or be the super-user. +.Pp +The values for the +.Fa flag +argument of the +.Fn utimensat +system call +are constructed by a bitwise-inclusive OR of flags from the following list, +defined in +.In fcntl.h : +.Bl -tag -width indent +.It Dv AT_SYMLINK_NOFOLLOW +If +.Fa path +names a symbolic link, the symbolic link's times are changed. +By default, +.Fn utimensat +changes the times of the file referenced by the symbolic link. +.El +.Sh RETURN VALUES +.Rv -std +.Sh ERRORS +These system calls will fail if: +.Bl -tag -width Er +.It Bq Er EACCES +The +.Fa times +argument is +.Dv NULL , +or both +.Va tv_nsec +values are +.Dv UTIME_NOW , +and the effective user ID of the process does not +match the owner of the file, and is not the super-user, and write +access is denied. +.It Bq Er EINVAL +The +.Va tv_nsec +component of at least one of the values specified by the +.Fa times +argument has a value less than 0 or greater than 999999999 and is not equal to +.Dv UTIME_NOW +or +.Dv UTIME_OMIT . +.It Bq Er EIO +An I/O error occurred while reading or writing the affected inode. +.It Bq Er EPERM +The +.Fa times +argument is not +.Dv NULL +nor are both +.Va tv_nsec +values +.Dv UTIME_NOW , +nor are both +.Va tv_nsec +values +.Dv UTIME_OMIT +and the calling process's effective user ID +does not match the owner of the file and is not the super-user. +.It Bq Er EPERM +The named file has its immutable or append-only flag set, see the +.Xr chflags 2 +manual page for more information. +.It Bq Er EROFS +The file system containing the file is mounted read-only. +.El +.Pp +The +.Fn futimens +system call +will fail if: +.Bl -tag -width Er +.It Bq Er EBADF +The +.Fa fd +argument +does not refer to a valid descriptor. +.El +.Pp +The +.Fn utimensat +system call +will fail if: +.Bl -tag -width Er +.It Bq Er EACCES +Search permission is denied for a component of the path prefix. +.It Bq Er EBADF +The +.Fa path +argument does not specify an absolute path and the +.Fa fd +argument is neither +.Dv AT_FDCWD +nor a valid file descriptor. +.It Bq Er EFAULT +The +.Fa path +argument +points outside the process's allocated address space. +.It Bq Er ELOOP +Too many symbolic links were encountered in translating the pathname. +.It Bq Er ENAMETOOLONG +A component of a pathname exceeded +.Dv NAME_MAX +characters, or an entire path name exceeded +.Dv PATH_MAX +characters. +.It Bq Er ENOENT +The named file does not exist. +.It Bq Er ENOTDIR +A component of the path prefix is not a directory. +.It Bq Er ENOTDIR +The +.Fa path +argument is not an absolute path and +.Fa fd +is neither +.Dv AT_FDCWD +nor a file descriptor associated with a directory. +.El +.Sh SEE ALSO +.Xr chflags 2 , +.Xr stat 2 , +.Xr symlink 2 , +.Xr utimes 2 , +.Xr utime 3 , +.Xr symlink 7 +.Sh STANDARDS +The +.Fn futimens +and +.Fn utimensat +system calls are expected to conform to +.St -p1003.1-2008 . diff --git a/bsd/man/man9/monotonic.9 b/bsd/man/man9/monotonic.9 new file mode 100644 index 000000000..38dbe2aa6 --- /dev/null +++ b/bsd/man/man9/monotonic.9 @@ -0,0 +1,81 @@ +.\" Copyright (c) 2017, Apple Inc. All rights reserved. +.\" +.Dd March 1, 2017 +.Dt MONOTONIC 9 +.Os Darwin +.\" +.Sh NAME +.Nm monotonic +.Nd performance counter access system +.\" +.Sh DESCRIPTION +.Nm +allows kernel and user space clients to configure and read hardware performance +counters. The hardware counters can be virtualized to count per-thread and +per-process. +.Nm +is split into three major layers: +.Bl -dash +.It +The machine-dependent implementations manipulate hardware registers to configure +and access the counters. This layer provides a machine-independent interface +that can be used by the next layer. +.It +A set of hooks and kernel routines manage the counters and provide higher-level +abstractions, like 64-bit counters and counting only events that occurred on a +thread or in a process. +.It +A user space interface that is presented as device nodes under +.Pa /dev/monotonic . +See +.Xr monotonic 4 . +Mach thread and task ports are used for the per-thread and per-process counts, +with special inspection routines. Some counter values are also reflected into +.Fn getrusage , +for use after a process has exited. See +.Xr getrusage 2 . +.El +.Pp +.\".Sh DIAGNOSTICS +.\" +.Sh SEE ALSO +.Xr count 1 , +.Xr mperf 1 , +.Xr easyperf 1 , +.Xr getrusage 2 , +.Xr monotonic 4 , +.Xr perf 1 +.\" +.Sh HISTORY +.Nm +replaces the kernel performance counter system, kpc. For the time being, +.Nm +backs portions of the existing kpc +.Fn sysctl +interface. Prior to kpc, the AppleProfileFamily kernel extensions provided +performance counter interfaces. The kernel extensions themselves expanded upon +functionality provided for PowerPC by CHUD. +.\".Sh CAVEATS +.\" +.Sh SECURITY CONSIDERATIONS +.Pp +.Pp +.Bl -dash +.It +Hardware performance counters are an ideal tool for side-channel attacks. By +observing how the counters are affected by an otherwise opaque process, an +attacker can obtain sensitive data or key material. +.Pp +For this reason, the hardware performance counters cannot be queried directly +from user space. Instead, all processes, including those owned by root, can +only query the thread and process counters if they have the corresponding Mach +thread or task port. +.It +When used in sampling mode, hardware performance counters can induce interrupt +storms that translate to denial-of-service attacks on a system. Even a careless +user can stumble over this issue, since reasonable periods for some events are +far too aggressive for others. +.Pp +If a hardware performance counter takes too many interrupts in a short amount of +time, it will be disabled. +.El diff --git a/bsd/miscfs/devfs/devfs.h b/bsd/miscfs/devfs/devfs.h index 6615b97b5..c2cc577c5 100644 --- a/bsd/miscfs/devfs/devfs.h +++ b/bsd/miscfs/devfs/devfs.h @@ -57,6 +57,7 @@ #define _MISCFS_DEVFS_DEVFS_H_ #include +#include #define DEVFS_CHAR 0 #define DEVFS_BLOCK 1 diff --git a/bsd/miscfs/devfs/devfs_fdesc_support.c b/bsd/miscfs/devfs/devfs_fdesc_support.c index bf4e3bb06..5d9355efc 100644 --- a/bsd/miscfs/devfs/devfs_fdesc_support.c +++ b/bsd/miscfs/devfs/devfs_fdesc_support.c @@ -568,9 +568,15 @@ devfs_devfd_readdir(struct vnop_readdir_args *ap) if (ap->a_flags & (VNODE_READDIR_EXTENDED | VNODE_READDIR_REQSEEKOFF)) return (EINVAL); + /* + * There needs to be space for at least one entry. + */ + if (uio_resid(uio) < UIO_MX) + return (EINVAL); + i = uio->uio_offset / UIO_MX; error = 0; - while (uio_resid(uio) > 0) { + while (uio_resid(uio) >= UIO_MX) { if (i >= p->p_fd->fd_nfiles) break; diff --git a/bsd/miscfs/devfs/devfs_tree.c b/bsd/miscfs/devfs/devfs_tree.c index 21912549a..adbc2e78f 100644 --- a/bsd/miscfs/devfs/devfs_tree.c +++ b/bsd/miscfs/devfs/devfs_tree.c @@ -328,7 +328,7 @@ dev_finddir(const char * path, while (*scan != '/' && *scan) scan++; - strlcpy(component, start, scan - start); + strlcpy(component, start, (scan - start) + 1); if (*scan == '/') scan++; diff --git a/bsd/miscfs/devfs/devfs_vnops.c b/bsd/miscfs/devfs/devfs_vnops.c index fbd3246f7..41029dd7c 100644 --- a/bsd/miscfs/devfs/devfs_vnops.c +++ b/bsd/miscfs/devfs/devfs_vnops.c @@ -115,6 +115,7 @@ static int devfs_update(struct vnode *vp, struct timeval *access, void devfs_rele_node(devnode_t *); static void devfs_consider_time_update(devnode_t *dnp, uint32_t just_changed_flags); static boolean_t devfs_update_needed(long now_s, long last_s); +static boolean_t devfs_is_name_protected(struct vnode *dvp, const char *name); void dn_times_locked(devnode_t * dnp, struct timeval *t1, struct timeval *t2, struct timeval *t3, uint32_t just_changed_flags); void dn_times_now(devnode_t *dnp, uint32_t just_changed_flags); void dn_mark_for_delayed_times_update(devnode_t *dnp, uint32_t just_changed_flags); @@ -184,6 +185,33 @@ dn_times_now(devnode_t * dnp, uint32_t just_changed_flags) DEVFS_ATTR_UNLOCK(); } +/* + * Critical devfs devices cannot be renamed or removed. + * However, links to them may be moved/unlinked. So we block + * remove/rename on a per-name basis, rather than per-node. + */ +static boolean_t +devfs_is_name_protected(struct vnode *dvp, const char *name) +{ + /* + * Only names in root are protected. E.g. /dev/null is protected, + * but /dev/foo/null isn't. + */ + if (!vnode_isvroot(dvp)) + return FALSE; + + if ((strcmp("console", name) == 0) || + (strcmp("tty", name) == 0) || + (strcmp("null", name) == 0) || + (strcmp("zero", name) == 0) || + (strcmp("klog", name) == 0)) { + + return TRUE; + } + + return FALSE; +} + /* * Convert a component of a pathname into a pointer to a locked node. @@ -795,6 +823,7 @@ devfs_vnop_remove(struct vnop_remove_args *ap) * are the end of the path. Get pointers to all our * devfs structures. */ + DEVFS_LOCK(); tp = VTODN(vp); @@ -808,6 +837,14 @@ devfs_vnop_remove(struct vnop_remove_args *ap) goto abort; } + /* + * Don't allow removing critical devfs devices + */ + if (devfs_is_name_protected(dvp, cnp->cn_nameptr)) { + error = EINVAL; + goto abort; +} + /* * Make sure that we don't try do something stupid */ @@ -1006,6 +1043,15 @@ devfs_rename(struct vnop_rename_args *ap) doingdirectory++; } + /* + * Don't allow renaming critical devfs devices + */ + if (devfs_is_name_protected(fdvp, fcnp->cn_nameptr) || + devfs_is_name_protected(tdvp, tcnp->cn_nameptr)) { + error = EINVAL; + goto out; + } + /* * If ".." must be changed (ie the directory gets a new * parent) then the source directory must not be in the @@ -1570,7 +1616,7 @@ static struct vnodeopv_entry_desc devfs_vnodeop_entries[] = { #if CONFIG_MACF { &vnop_setlabel_desc, (VOPFUNC)devfs_setlabel }, /* setlabel */ #endif - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc devfs_vnodeop_opv_desc = { &devfs_vnodeop_p, devfs_vnodeop_entries }; @@ -1616,7 +1662,7 @@ static struct vnodeopv_entry_desc devfs_spec_vnodeop_entries[] = { #if CONFIG_MACF { &vnop_setlabel_desc, (VOPFUNC)devfs_setlabel }, /* setlabel */ #endif - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc devfs_spec_vnodeop_opv_desc = { &devfs_spec_vnodeop_p, devfs_spec_vnodeop_entries }; @@ -1640,7 +1686,7 @@ static struct vnodeopv_entry_desc devfs_devfd_vnodeop_entries[] = { #if CONFIG_MACF { &vnop_setlabel_desc, (VOPFUNC)devfs_setlabel }, /* setlabel */ #endif - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc devfs_devfd_vnodeop_opv_desc = { &devfs_devfd_vnodeop_p, devfs_devfd_vnodeop_entries}; diff --git a/bsd/miscfs/devfs/devfsdefs.h b/bsd/miscfs/devfs/devfsdefs.h index 79e99f512..6fdac8849 100644 --- a/bsd/miscfs/devfs/devfsdefs.h +++ b/bsd/miscfs/devfs/devfsdefs.h @@ -72,8 +72,6 @@ #include -#include - __BEGIN_DECLS #ifdef __APPLE_API_PRIVATE #define DEVMAXNAMESIZE 32 /* XXX */ diff --git a/bsd/miscfs/fifofs/fifo_vnops.c b/bsd/miscfs/fifofs/fifo_vnops.c index 0b7bc1d3d..6146bcbfa 100644 --- a/bsd/miscfs/fifofs/fifo_vnops.c +++ b/bsd/miscfs/fifofs/fifo_vnops.c @@ -117,7 +117,7 @@ struct vnodeopv_entry_desc fifo_vnodeop_entries[] = { { &vnop_blktooff_desc, (VOPFUNC)err_blktooff }, /* blktooff */ { &vnop_offtoblk_desc, (VOPFUNC)err_offtoblk }, /* offtoblk */ { &vnop_blockmap_desc, (VOPFUNC)err_blockmap }, /* blockmap */ - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc fifo_vnodeop_opv_desc = { &fifo_vnodeop_p, fifo_vnodeop_entries }; diff --git a/bsd/miscfs/nullfs/nullfs.h b/bsd/miscfs/nullfs/nullfs.h index 5d55e2c8b..e29b9e696 100644 --- a/bsd/miscfs/nullfs/nullfs.h +++ b/bsd/miscfs/nullfs/nullfs.h @@ -163,4 +163,4 @@ __END_DECLS #endif /* KERNEL */ -#endif \ No newline at end of file +#endif diff --git a/bsd/miscfs/routefs/routefs_ops.c b/bsd/miscfs/routefs/routefs_ops.c index 194325406..db6db101d 100644 --- a/bsd/miscfs/routefs/routefs_ops.c +++ b/bsd/miscfs/routefs/routefs_ops.c @@ -553,7 +553,7 @@ static struct vnodeopv_entry_desc routefs_vnodeop_entries[] = { #if CONFIG_MACF { &vnop_setlabel_desc, (VOPFUNC)routefserr_setlabel }, /* setlabel */ #endif - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc routefs_vnodeop_opv_desc = { &routefs_vnodeop_p, routefs_vnodeop_entries }; diff --git a/bsd/miscfs/specfs/spec_vnops.c b/bsd/miscfs/specfs/spec_vnops.c index f698e5d68..625e9736f 100644 --- a/bsd/miscfs/specfs/spec_vnops.c +++ b/bsd/miscfs/specfs/spec_vnops.c @@ -84,6 +84,7 @@ #include #include #include +#include #include #include @@ -91,18 +92,18 @@ #include #include #include +#include #include #include +#include /* XXX following three prototypes should be in a header file somewhere */ extern dev_t chrtoblk(dev_t dev); extern boolean_t iskmemdev(dev_t dev); extern int bpfkqfilter(dev_t dev, struct knote *kn); -extern int ptsd_kqfilter(dev_t dev, struct knote *kn); - -extern int ignore_is_ssd; +extern int ptsd_kqfilter(dev_t, struct knote *); struct vnode *speclisth[SPECHSZ]; @@ -155,7 +156,7 @@ struct vnodeopv_entry_desc spec_vnodeop_entries[] = { { &vnop_blktooff_desc, (VOPFUNC)spec_blktooff }, /* blktooff */ { &vnop_offtoblk_desc, (VOPFUNC)spec_offtoblk }, /* offtoblk */ { &vnop_blockmap_desc, (VOPFUNC)spec_blockmap }, /* blockmap */ - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc spec_vnodeop_opv_desc = { &spec_vnodeop_p, spec_vnodeop_entries }; @@ -239,6 +240,7 @@ static void throttle_info_end_io_internal(struct _throttle_io_info_t *info, int static int throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap); static int throttle_get_thread_throttle_level(uthread_t ut); static int throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier); +void throttle_info_mount_reset_period(mount_t mp, int isssd); /* * Trivial lookup routine that always fails. @@ -725,10 +727,10 @@ spec_select(struct vnop_select_args *ap) } } -static int filt_specattach(struct knote *kn); +static int filt_specattach(struct knote *kn, struct kevent_internal_s *kev); int -spec_kqfilter(vnode_t vp, struct knote *kn) +spec_kqfilter(vnode_t vp, struct knote *kn, struct kevent_internal_s *kev) { dev_t dev; @@ -741,7 +743,7 @@ spec_kqfilter(vnode_t vp, struct knote *kn) * Try a bpf device, as defined in bsd/net/bpf.c * If it doesn't error out the attach, then it * claimed it. Otherwise, fall through and try - * a regular spec attach. + * other attaches. */ int32_t tmp_flags = kn->kn_flags; int64_t tmp_data = kn->kn_data; @@ -755,8 +757,31 @@ spec_kqfilter(vnode_t vp, struct knote *kn) kn->kn_data = tmp_data; #endif + if (major(dev) > nchrdev) { + knote_set_error(kn, ENXIO); + return 0; + } + + kn->kn_vnode_kqok = !!(cdevsw_flags[major(dev)] & CDEVSW_SELECT_KQUEUE); + kn->kn_vnode_use_ofst = !!(cdevsw_flags[major(dev)] & CDEVSW_USE_OFFSET); + + if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) { + kn->kn_filtid = EVFILTID_PTSD; + return ptsd_kqfilter(dev, kn); + } else if (cdevsw[major(dev)].d_type == D_TTY && + !(cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) && + kn->kn_vnode_kqok) { + /* + * TTYs from drivers that use struct ttys use their own filter + * routines. The PTC driver doesn't use the tty for character + * counts, so it must go through the select fallback. + */ + kn->kn_filtid = EVFILTID_TTY; + return knote_fops(kn)->f_attach(kn, kev); + } + /* Try to attach to other char special devices */ - return filt_specattach(kn); + return filt_specattach(kn, kev); } /* @@ -1503,6 +1528,27 @@ throttle_info_mount_rel(mount_t mp) mp->mnt_throttle_info = NULL; } +/* + * Reset throttling periods for the given mount point + * + * private interface used by disk conditioner to reset + * throttling periods when 'is_ssd' status changes + */ +void +throttle_info_mount_reset_period(mount_t mp, int isssd) +{ + struct _throttle_io_info_t *info; + + if (mp == NULL) + info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1]; + else if (mp->mnt_throttle_info == NULL) + info = &_throttle_io_info[mp->mnt_devbsdunit]; + else + info = mp->mnt_throttle_info; + + throttle_init_throttle_period(info, isssd); +} + void throttle_info_get_last_io_time(mount_t mp, struct timeval *tv) { @@ -1535,7 +1581,6 @@ update_last_io_time(mount_t mp) mp->mnt_last_write_completed_timestamp = info->throttle_last_write_timestamp; } - int throttle_get_io_policy(uthread_t *ut) { @@ -2031,7 +2076,7 @@ void *throttle_info_update_by_mount(mount_t mp) ut = get_bsdthread_info(current_thread()); if (mp != NULL) { - if ((mp->mnt_kern_flag & MNTK_SSD) && !ignore_is_ssd) + if (disk_conditioner_mount_is_ssd(mp)) isssd = TRUE; info = &_throttle_io_info[mp->mnt_devbsdunit]; } else @@ -2158,6 +2203,11 @@ int throttle_lowpri_window(void) return ut->uu_lowpri_window; } + +#if CONFIG_IOSCHED +int upl_get_cached_tier(void *); +#endif + int spec_strategy(struct vnop_strategy_args *ap) { @@ -2176,14 +2226,35 @@ spec_strategy(struct vnop_strategy_args *ap) boolean_t upgrade = FALSE; int code = 0; +#if !CONFIG_EMBEDDED proc_t curproc = current_proc(); +#endif /* !CONFIG_EMBEDDED */ bp = ap->a_bp; bdev = buf_device(bp); mp = buf_vnode(bp)->v_mount; bap = &bp->b_attr; +#if CONFIG_IOSCHED + if (bp->b_flags & B_CLUSTER) { + + io_tier = upl_get_cached_tier(bp->b_upl); + + if (io_tier == -1) + io_tier = throttle_get_io_policy(&ut); +#if DEVELOPMENT || DEBUG + else { + int my_io_tier = throttle_get_io_policy(&ut); + + if (io_tier != my_io_tier) + KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, IO_TIER_UPL_MISMATCH)) | DBG_FUNC_NONE, buf_kernel_addrperm_addr(bp), my_io_tier, io_tier, 0, 0); + } +#endif + } else + io_tier = throttle_get_io_policy(&ut); +#else io_tier = throttle_get_io_policy(&ut); +#endif passive = throttle_get_passive_io_policy(&ut); /* @@ -2233,8 +2304,10 @@ spec_strategy(struct vnop_strategy_args *ap) bap->ba_flags |= BA_PASSIVE; } +#if !CONFIG_EMBEDDED if ((curproc != NULL) && ((curproc->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP)) bap->ba_flags |= BA_DELAYIDLESLEEP; +#endif /* !CONFIG_EMBEDDED */ bflags = bp->b_flags; @@ -2275,7 +2348,7 @@ spec_strategy(struct vnop_strategy_args *ap) thread_update_io_stats(current_thread(), buf_count(bp), code); if (mp != NULL) { - if ((mp->mnt_kern_flag & MNTK_SSD) && !ignore_is_ssd) + if (disk_conditioner_mount_is_ssd(mp)) isssd = TRUE; /* * Partially initialized mounts don't have a final devbsdunit and should not be tracked. @@ -2327,6 +2400,11 @@ spec_strategy(struct vnop_strategy_args *ap) typedef int strategy_fcn_ret_t(struct buf *bp); strategy_ret = (*(strategy_fcn_ret_t*)bdevsw[major(bdev)].d_strategy)(bp); + + // disk conditioner needs to track when this I/O actually starts + // which means track it after `strategy` which may include delays + // from inflight I/Os + microuptime(&bp->b_timestamp_tv); if (IO_SATISFIED_BY_CACHE == strategy_ret) { /* @@ -2562,37 +2640,136 @@ spec_offtoblk(struct vnop_offtoblk_args *ap) } static void filt_specdetach(struct knote *kn); -static int filt_spec(struct knote *kn, long hint); +static int filt_specevent(struct knote *kn, long hint); static int filt_spectouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_specprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); static unsigned filt_specpeek(struct knote *kn); -struct filterops spec_filtops = { - .f_isfd = 1, - .f_attach = filt_specattach, - .f_detach = filt_specdetach, - .f_event = filt_spec, - .f_touch = filt_spectouch, - .f_process = filt_specprocess, - .f_peek = filt_specpeek +SECURITY_READ_ONLY_EARLY(struct filterops) spec_filtops = { + .f_isfd = 1, + .f_attach = filt_specattach, + .f_detach = filt_specdetach, + .f_event = filt_specevent, + .f_touch = filt_spectouch, + .f_process = filt_specprocess, + .f_peek = filt_specpeek }; + +/* + * Given a waitq that is assumed to be embedded within a selinfo structure, + * return the containing selinfo structure. While 'wq' is not really a queue + * element, this macro simply does the offset_of calculation to get back to a + * containing struct given the struct type and member name. + */ +#define selinfo_from_waitq(wq) \ + qe_element((wq), struct selinfo, si_waitq) + static int -filter_to_seltype(int16_t filter) +spec_knote_select_and_link(struct knote *kn) { - switch (filter) { - case EVFILT_READ: - return FREAD; - case EVFILT_WRITE: - return FWRITE; - default: - panic("filt_to_seltype(): invalid filter %d\n", filter); + uthread_t uth; + vfs_context_t ctx; + vnode_t vp; + struct waitq_set *old_wqs; + uint64_t rsvd, rsvd_arg; + uint64_t *rlptr = NULL; + struct selinfo *si = NULL; + int selres = 0; + + uth = get_bsdthread_info(current_thread()); + + ctx = vfs_context_current(); + vp = (vnode_t)kn->kn_fp->f_fglob->fg_data; + + int error = vnode_getwithvid(vp, kn->kn_hookid); + if (error != 0) { + knote_set_error(kn, ENOENT); return 0; } + + /* + * This function may be called many times to link or re-link the + * underlying vnode to the kqueue. If we've already linked the two, + * we will have a valid kn_hook_data which ties us to the underlying + * device's waitq via a the waitq's prepost table object. However, + * devices can abort any select action by calling selthreadclear(). + * This is OK because the table object will be invalidated by the + * driver (through a call to selthreadclear), so any attempt to access + * the associated waitq will fail because the table object is invalid. + * + * Even if we've already registered, we need to pass a pointer + * to a reserved link structure. Otherwise, selrecord() will + * infer that we're in the second pass of select() and won't + * actually do anything! + */ + rsvd = rsvd_arg = waitq_link_reserve(NULL); + rlptr = (void *)&rsvd_arg; + + /* + * Trick selrecord() into hooking kqueue's wait queue set + * set into device's selinfo wait queue + */ + old_wqs = uth->uu_wqset; + uth->uu_wqset = &(knote_get_kq(kn)->kq_wqs); + selres = VNOP_SELECT(vp, knote_get_seltype(kn), 0, rlptr, ctx); + uth->uu_wqset = old_wqs; + + /* + * make sure to cleanup the reserved link - this guards against + * drivers that may not actually call selrecord(). + */ + waitq_link_release(rsvd); + if (rsvd != rsvd_arg) { + /* the driver / handler called selrecord() */ + struct waitq *wq; + memcpy(&wq, rlptr, sizeof(void *)); + + /* + * The waitq is part of the selinfo structure managed by the + * driver. For certain drivers, we want to hook the knote into + * the selinfo structure's si_note field so selwakeup can call + * KNOTE. + */ + si = selinfo_from_waitq(wq); + + /* + * The waitq_get_prepost_id() function will (potentially) + * allocate a prepost table object for the waitq and return + * the table object's ID to us. It will also set the + * waitq_prepost_id field within the waitq structure. + * + * We can just overwrite kn_hook_data because it's simply a + * table ID used to grab a reference when needed. + * + * We have a reference on the vnode, so we know that the + * device won't go away while we get this ID. + */ + kn->kn_hook_data = waitq_get_prepost_id(wq); + } else { + assert(selres != 0); + } + + vnode_put(vp); + + return selres; } -static int -filt_specattach(struct knote *kn) +static void filt_spec_common(struct knote *kn, int selres) +{ + if (kn->kn_vnode_use_ofst) { + if (kn->kn_fp->f_fglob->fg_offset >= (uint32_t)selres) { + kn->kn_data = 0; + } else { + kn->kn_data = ((uint32_t)selres) - kn->kn_fp->f_fglob->fg_offset; + } + } else { + kn->kn_data = selres; + } +} + +static int +filt_specattach(struct knote *kn, __unused struct kevent_internal_s *kev) { vnode_t vp; dev_t dev; @@ -2603,12 +2780,6 @@ filt_specattach(struct knote *kn) dev = vnode_specrdev(vp); - if (major(dev) > nchrdev) { - kn->kn_flags |= EV_ERROR; - kn->kn_data = ENXIO; - return 0; - } - /* * For a few special kinds of devices, we can attach knotes with * no restrictions because their "select" vectors return the amount @@ -2616,25 +2787,32 @@ filt_specattach(struct knote *kn) * data of 1, indicating that the caller doesn't care about actual * data counts, just an indication that the device has data. */ - - if ((cdevsw_flags[major(dev)] & CDEVSW_SELECT_KQUEUE) == 0 && + if (!kn->kn_vnode_kqok && ((kn->kn_sfflags & NOTE_LOWAT) == 0 || kn->kn_sdata != 1)) { - kn->kn_flags |= EV_ERROR; - kn->kn_data = EINVAL; + knote_set_error(kn, EINVAL); return 0; } - kn->kn_hook_data = 0; + /* + * This forces the select fallback to call through VNOP_SELECT and hook + * up selinfo on every filter routine. + * + * Pseudo-terminal controllers are opted out of native kevent support -- + * remove this when they get their own EVFILTID. + */ + if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) { + kn->kn_vnode_kqok = 0; + } kn->kn_filtid = EVFILTID_SPEC; + kn->kn_hook_data = 0; kn->kn_hookid = vnode_vid(vp); knote_markstayactive(kn); - - return 0; + return spec_knote_select_and_link(kn); } -static void +static void filt_specdetach(struct knote *kn) { knote_clearstayactive(kn); @@ -2657,15 +2835,16 @@ filt_specdetach(struct knote *kn) } } -static int -filt_spec(__unused struct knote *kn, __unused long hint) +static int +filt_specevent(struct knote *kn, __unused long hint) { - panic("filt_spec()"); + /* + * Nothing should call knote or knote_vanish on this knote. + */ + panic("filt_specevent(%p)", kn); return 0; } - - static int filt_spectouch(struct knote *kn, struct kevent_internal_s *kev) { @@ -2674,7 +2853,10 @@ filt_spectouch(struct knote *kn, struct kevent_internal_s *kev) if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) kn->kn_udata = kev->udata; - /* stayqueued knotes don't need hints from touch */ + if (kev->flags & EV_ENABLE) { + return spec_knote_select_and_link(kn); + } + return 0; } @@ -2684,96 +2866,26 @@ filt_specprocess(struct knote *kn, struct filt_process_s *data, struct kevent_in #pragma unused(data) vnode_t vp; uthread_t uth; - struct waitq_set *old_wqs; vfs_context_t ctx; int res; int selres; int error; - int use_offset; - dev_t dev; - uint64_t flags; - uint64_t rsvd, rsvd_arg; - uint64_t *rlptr = NULL; uth = get_bsdthread_info(current_thread()); ctx = vfs_context_current(); vp = (vnode_t)kn->kn_fp->f_fglob->fg_data; - /* JMM - locking against touches? */ + /* FIXME JMM - locking against touches? */ error = vnode_getwithvid(vp, kn->kn_hookid); if (error != 0) { kn->kn_flags |= (EV_EOF | EV_ONESHOT); - *kev = kn->kn_kevent; + *kev = kn->kn_kevent; return 1; } - - dev = vnode_specrdev(vp); - flags = cdevsw_flags[major(dev)]; - use_offset = ((flags & CDEVSW_USE_OFFSET) != 0); - /* - * This function may be called many times to link or re-link the - * underlying vnode to the kqueue. If we've already linked the two, - * we will have a valid kn_hook_data which ties us to the underlying - * device's waitq via a the waitq's prepost table object. However, - * devices can abort any select action by calling selthreadclear(). - * This is OK because the table object will be invalidated by the - * driver (through a call to selthreadclear), so any attempt to access - * the associated waitq will fail because the table object is invalid. - * - * Even if we've already registered, we need to pass a pointer - * to a reserved link structure. Otherwise, selrecord() will - * infer that we're in the second pass of select() and won't - * actually do anything! - */ - rsvd = rsvd_arg = waitq_link_reserve(NULL); - rlptr = (void *)&rsvd_arg; - - /* - * Trick selrecord() into hooking kqueue's wait queue set - * set into device's selinfo wait queue - */ - old_wqs = uth->uu_wqset; - uth->uu_wqset = &(knote_get_kq(kn)->kq_wqs); - selres = VNOP_SELECT(vp, filter_to_seltype(kn->kn_filter), - 0, rlptr, ctx); - uth->uu_wqset = old_wqs; - - /* - * make sure to cleanup the reserved link - this guards against - * drivers that may not actually call selrecord(). - */ - waitq_link_release(rsvd); - if (rsvd != rsvd_arg) { - /* the driver / handler called selrecord() */ - struct waitq *wq; - memcpy(&wq, rlptr, sizeof(void *)); - - /* - * The waitq_get_prepost_id() function will (potentially) - * allocate a prepost table object for the waitq and return - * the table object's ID to us. It will also set the - * waitq_prepost_id field within the waitq structure. - * - * We can just overwrite kn_hook_data because it's simply a - * table ID used to grab a reference when needed. - * - * We have a reference on the vnode, so we know that the - * device won't go away while we get this ID. - */ - kn->kn_hook_data = waitq_get_prepost_id(wq); - } - - if (use_offset) { - if (kn->kn_fp->f_fglob->fg_offset >= (uint32_t)selres) { - kn->kn_data = 0; - } else { - kn->kn_data = ((uint32_t)selres) - kn->kn_fp->f_fglob->fg_offset; - } - } else { - kn->kn_data = selres; - } + selres = spec_knote_select_and_link(kn); + filt_spec_common(kn, selres); vnode_put(vp); @@ -2794,64 +2906,11 @@ filt_specprocess(struct knote *kn, struct filt_process_s *data, struct kevent_in static unsigned filt_specpeek(struct knote *kn) { - vnode_t vp; - uthread_t uth; - struct waitq_set *old_wqs; - vfs_context_t ctx; - int error, selres; - uint64_t rsvd, rsvd_arg; - uint64_t *rlptr = NULL; - - uth = get_bsdthread_info(current_thread()); - ctx = vfs_context_current(); - vp = (vnode_t)kn->kn_fp->f_fglob->fg_data; - - error = vnode_getwithvid(vp, kn->kn_hookid); - if (error != 0) { - return 1; /* Just like VNOP_SELECT() on recycled vnode */ - } - - /* - * Even if we've already registered, we need to pass a pointer - * to a reserved link structure. Otherwise, selrecord() will - * infer that we're in the second pass of select() and won't - * actually do anything! - */ - rsvd = rsvd_arg = waitq_link_reserve(NULL); - rlptr = (void *)&rsvd_arg; + int selres = 0; - old_wqs = uth->uu_wqset; - uth->uu_wqset = &(knote_get_kq(kn)->kq_wqs); - selres = VNOP_SELECT(vp, filter_to_seltype(kn->kn_filter), - 0, (void *)rlptr, ctx); - uth->uu_wqset = old_wqs; + selres = spec_knote_select_and_link(kn); + filt_spec_common(kn, selres); - /* - * make sure to cleanup the reserved link - this guards against - * drivers that may not actually call selrecord() - */ - waitq_link_release(rsvd); - if (rsvd != rsvd_arg) { - /* the driver / handler called selrecord() */ - struct waitq *wq; - memcpy(&wq, rlptr, sizeof(void *)); - - /* - * The waitq_get_prepost_id() function will (potentially) - * allocate a prepost table object for the waitq and return - * the table object's ID to us. It will also set the - * waitq_prepost_id field within the waitq structure. - * - * We can just overwrite kn_hook_data because it's simply a - * table ID used to grab a reference when needed. - * - * We have a reference on the vnode, so we know that the - * device won't go away while we get this ID. - */ - kn->kn_hook_data = waitq_get_prepost_id(wq); - } - - vnode_put(vp); - return selres; + return kn->kn_data; } diff --git a/bsd/miscfs/specfs/specdev.h b/bsd/miscfs/specfs/specdev.h index 2b14d796b..3d6d0258f 100644 --- a/bsd/miscfs/specfs/specdev.h +++ b/bsd/miscfs/specfs/specdev.h @@ -135,7 +135,7 @@ int spec_blktooff (struct vnop_blktooff_args *); int spec_offtoblk (struct vnop_offtoblk_args *); int spec_fsync_internal (vnode_t, int, vfs_context_t); int spec_blockmap (struct vnop_blockmap_args *); -int spec_kqfilter (vnode_t vp, struct knote *kn); +int spec_kqfilter (vnode_t vp, struct knote *kn, struct kevent_internal_s *kev); #endif /* BSD_KERNEL_PRIVATE */ int spec_ebadf(void *); diff --git a/bsd/net/Makefile b/bsd/net/Makefile index 234c2a15b..98db2444f 100644 --- a/bsd/net/Makefile +++ b/bsd/net/Makefile @@ -50,10 +50,12 @@ PRIVATE_DATAFILES = \ if_utun.h \ if_var.h \ if_vlan_var.h \ + if_fake_var.h \ iptap.h \ lacp.h \ ndrv_var.h \ necp.h \ + net_api_stats.h \ netsrc.h \ network_agent.h \ ntstat.h \ diff --git a/bsd/net/altq/altq.h b/bsd/net/altq/altq.h index 590c6810b..a3b18841f 100644 --- a/bsd/net/altq/altq.h +++ b/bsd/net/altq/altq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -65,11 +65,8 @@ #define ALTQT_CBQ PKTSCHEDT_CBQ /* cbq */ #define ALTQT_HFSC PKTSCHEDT_HFSC /* hfsc */ #define ALTQT_PRIQ PKTSCHEDT_PRIQ /* priority queue */ -#define ALTQT_FAIRQ PKTSCHEDT_FAIRQ /* fairq */ -#define ALTQT_QFQ PKTSCHEDT_QFQ /* quick fair queueing */ +#define ALTQT_FAIRQ PKTSCHEDT_FAIRQ /* fairq */ +#define ALTQT_QFQ PKTSCHEDT_QFQ /* quick fair queueing */ #define ALTQT_MAX PKTSCHEDT_MAX /* should be max disc type + 1 */ #endif /* PRIVATE */ -#ifdef BSD_KERNEL_PRIVATE -#include -#endif /* BSD_KERNEL_PRIVATE */ #endif /* _ALTQ_ALTQ_H_ */ diff --git a/bsd/net/altq/altq_cbq.c b/bsd/net/altq/altq_cbq.c deleted file mode 100644 index 8ced30eeb..000000000 --- a/bsd/net/altq/altq_cbq.c +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_cbq.c,v 1.23 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_cbq.c,v 1.9 2000/12/14 08:12:45 thorpej Exp $ */ - -/* - * Copyright (c) Sun Microsystems, Inc. 1993-1998 All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the SMCC Technology - * Development Group at Sun Microsystems, Inc. - * - * 4. The name of the Sun Microsystems, Inc nor may not be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * SUN MICROSYSTEMS DOES NOT CLAIM MERCHANTABILITY OF THIS SOFTWARE OR THE - * SUITABILITY OF THIS SOFTWARE FOR ANY PARTICULAR PURPOSE. The software is - * provided "as is" without express or implied warranty of any kind. - * - * These notices must be retained in any copies of any part of this software. - */ - -#if PF_ALTQ && PKTSCHED_CBQ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -/* - * Forward Declarations. - */ -static int altq_cbq_request(struct ifaltq *, enum altrq, void *); -static int altq_cbq_enqueue(struct ifaltq *, struct mbuf *); -static struct mbuf *altq_cbq_dequeue(struct ifaltq *, enum altdq_op); - -int -altq_cbq_pfattach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - - IFCQ_LOCK(&ifp->if_snd); - error = altq_attach(IFCQ_ALTQ(&ifp->if_snd), ALTQT_CBQ, a->altq_disc, - altq_cbq_enqueue, altq_cbq_dequeue, NULL, altq_cbq_request); - IFCQ_UNLOCK(&ifp->if_snd); - - return (error); -} - -int -altq_cbq_add(struct pf_altq *a) -{ - cbq_state_t *cbqp; - struct ifnet *ifp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - if (!ALTQ_IS_READY(IFCQ_ALTQ(&ifp->if_snd))) - return (ENODEV); - - cbqp = cbq_alloc(ifp, M_WAITOK, TRUE); - if (cbqp == NULL) - return (ENOMEM); - - /* keep the state in pf_altq */ - a->altq_disc = cbqp; - - return (0); -} - -int -altq_cbq_remove(struct pf_altq *a) -{ - cbq_state_t *cbqp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((cbqp = a->altq_disc) == NULL) - return (EINVAL); - a->altq_disc = NULL; - - return (cbq_destroy(cbqp)); -} - -int -altq_cbq_add_queue(struct pf_altq *a) -{ - struct cbq_opts *opts = &a->pq_u.cbq_opts; - cbq_state_t *cbqp; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((cbqp = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(cbqp->ifnp.ifq_); - err = cbq_add_queue(cbqp, a->qlimit, a->priority, - opts->minburst, opts->maxburst, opts->pktsize, opts->maxpktsize, - opts->ns_per_byte, opts->maxidle, opts->minidle, opts->offtime, - opts->flags, a->parent_qid, a->qid, NULL); - IFCQ_UNLOCK(cbqp->ifnp.ifq_); - - return (err); -} - -int -altq_cbq_remove_queue(struct pf_altq *a) -{ - cbq_state_t *cbqp; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((cbqp = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(cbqp->ifnp.ifq_); - err = cbq_remove_queue(cbqp, a->qid); - IFCQ_UNLOCK(cbqp->ifnp.ifq_); - - return (err); -} - -int -altq_cbq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - struct ifclassq *ifq = NULL; - cbq_state_t *cbqp; - class_stats_t stats; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((unsigned)*nbytes < sizeof (stats)) - return (EINVAL); - - if ((cbqp = altq_lookup(a->ifname, ALTQT_CBQ)) == NULL) - return (EBADF); - - ifq = cbqp->ifnp.ifq_; - IFCQ_LOCK_ASSERT_HELD(ifq); /* lock held by altq_lookup */ - error = cbq_get_class_stats(cbqp, a->qid, &stats); - IFCQ_UNLOCK(ifq); - if (error != 0) - return (error); - - if ((error = copyout((caddr_t)&stats, (user_addr_t)(uintptr_t)ubuf, - sizeof (stats))) != 0) - return (error); - - *nbytes = sizeof (stats); - - return (0); -} - -static int -altq_cbq_request(struct ifaltq *altq, enum altrq req, void *arg) -{ - cbq_state_t *cbqp = (cbq_state_t *)altq->altq_disc; - - switch (req) { - case ALTRQ_PURGE: - cbq_purge(cbqp); - break; - - case ALTRQ_PURGE_SC: - /* not supported for ALTQ instance */ - break; - - case ALTRQ_EVENT: - cbq_event(cbqp, (cqev_t)arg); - break; - - case ALTRQ_THROTTLE: - default: - break; - } - return (0); -} - -/* - * altq_cbq_enqueue is an enqueue function to be registered to - * (*altq_enqueue) in struct ifaltq. - */ -static int -altq_cbq_enqueue(struct ifaltq *altq, struct mbuf *m) -{ - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - printf("%s: packet for %s does not have pkthdr\n", __func__, - if_name(altq->altq_ifcq->ifcq_ifp)); - m_freem(m); - return (ENOBUFS); - } - - return (cbq_enqueue(altq->altq_disc, NULL, m, m_pftag(m))); -} - -/* - * altq_cbq_dequeue is a dequeue function to be registered to - * (*altq_dequeue) in struct ifaltq. - * - * note: ALTDQ_POLL returns the next packet without removing the packet - * from the queue. ALTDQ_REMOVE is a normal dequeue operation. - * ALTDQ_REMOVE must return the same packet if called immediately - * after ALTDQ_POLL. - */ -static struct mbuf * -altq_cbq_dequeue(struct ifaltq *altq, enum altdq_op op) -{ - return (cbq_dequeue(altq->altq_disc, (cqdq_op_t)op)); -} -#endif /* PF_ALTQ && PKTSCHED_CBQ */ diff --git a/bsd/net/altq/altq_cbq.h b/bsd/net/altq/altq_cbq.h index fba7310c9..3a47e8df0 100644 --- a/bsd/net/altq/altq_cbq.h +++ b/bsd/net/altq/altq_cbq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -66,21 +66,4 @@ #include #include -#ifdef BSD_KERNEL_PRIVATE -#ifdef __cplusplus -extern "C" { -#endif - -extern int altq_cbq_pfattach(struct pf_altq *); -extern int altq_cbq_add(struct pf_altq *); -extern int altq_cbq_remove(struct pf_altq *); -extern int altq_cbq_add_queue(struct pf_altq *); -extern int altq_cbq_remove_queue(struct pf_altq *); -extern int altq_cbq_getqstats(struct pf_altq *, void *, int *); - -#ifdef __cplusplus -} -#endif - -#endif /* BSD_KERNEL_PRIVATE */ #endif /* !_NET_ALTQ_ALTQ_CBQ_H_ */ diff --git a/bsd/net/altq/altq_fairq.c b/bsd/net/altq/altq_fairq.c deleted file mode 100644 index 69dcf2209..000000000 --- a/bsd/net/altq/altq_fairq.c +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* - * Copyright (c) 2008 The DragonFly Project. All rights reserved. - * - * This code is derived from software contributed to The DragonFly Project - * by Matthew Dillon - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of The DragonFly Project nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific, prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $DragonFly: src/sys/net/altq/altq_fairq.c,v 1.2 2008/05/14 11:59:23 sephe Exp $ - */ -/* - * Matt: I gutted altq_priq.c and used it as a skeleton on which to build - * fairq. The fairq algorithm is completely different then priq, of course, - * but because I used priq's skeleton I believe I should include priq's - * copyright. - * - * Copyright (C) 2000-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#if PF_ALTQ && PKTSCHED_FAIRQ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -/* - * function prototypes - */ -static int altq_fairq_enqueue(struct ifaltq *, struct mbuf *); -static struct mbuf *altq_fairq_dequeue(struct ifaltq *, enum altdq_op); -static int altq_fairq_request(struct ifaltq *, enum altrq, void *); - -int -altq_fairq_pfattach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - - IFCQ_LOCK(&ifp->if_snd); - error = altq_attach(IFCQ_ALTQ(&ifp->if_snd), ALTQT_FAIRQ, a->altq_disc, - altq_fairq_enqueue, altq_fairq_dequeue, NULL, altq_fairq_request); - IFCQ_UNLOCK(&ifp->if_snd); - - return (error); -} - -int -altq_fairq_add(struct pf_altq *a) -{ - struct fairq_if *fif; - struct ifnet *ifp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - if (!ALTQ_IS_READY(IFCQ_ALTQ(&ifp->if_snd))) - return (ENODEV); - - fif = fairq_alloc(ifp, M_WAITOK, TRUE); - if (fif == NULL) - return (ENOMEM); - - /* keep the state in pf_altq */ - a->altq_disc = fif; - - return (0); -} - -int -altq_fairq_remove(struct pf_altq *a) -{ - struct fairq_if *fif; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((fif = a->altq_disc) == NULL) - return (EINVAL); - a->altq_disc = NULL; - - return (fairq_destroy(fif)); -} - -int -altq_fairq_add_queue(struct pf_altq *a) -{ - struct fairq_if *fif; - struct fairq_opts *opts = &a->pq_u.fairq_opts; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((fif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(fif->fif_ifq); - err = fairq_add_queue(fif, a->priority, a->qlimit, a->bandwidth, - opts->nbuckets, opts->flags, opts->hogs_m1, opts->lssc_m1, - opts->lssc_d, opts->lssc_m2, a->qid, NULL); - IFCQ_UNLOCK(fif->fif_ifq); - - return (err); -} - -int -altq_fairq_remove_queue(struct pf_altq *a) -{ - struct fairq_if *fif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((fif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(fif->fif_ifq); - err = fairq_remove_queue(fif, a->qid); - IFCQ_UNLOCK(fif->fif_ifq); - - return (err); -} - -int -altq_fairq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - struct ifclassq *ifq = NULL; - struct fairq_if *fif; - struct fairq_classstats stats; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((unsigned)*nbytes < sizeof (stats)) - return (EINVAL); - - if ((fif = altq_lookup(a->ifname, ALTQT_FAIRQ)) == NULL) - return (EBADF); - - ifq = fif->fif_ifq; - IFCQ_LOCK_ASSERT_HELD(ifq); /* lock held by altq_lookup */ - error = fairq_get_class_stats(fif, a->qid, &stats); - IFCQ_UNLOCK(ifq); - if (error != 0) - return (error); - - if ((error = copyout((caddr_t)&stats, (user_addr_t)(uintptr_t)ubuf, - sizeof (stats))) != 0) - return (error); - - *nbytes = sizeof (stats); - - return (0); -} - -static int -altq_fairq_request(struct ifaltq *altq, enum altrq req, void *arg) -{ - struct fairq_if *fif = (struct fairq_if *)altq->altq_disc; - - switch (req) { - case ALTRQ_PURGE: - fairq_purge(fif); - break; - - case ALTRQ_PURGE_SC: - /* not supported for ALTQ instance */ - break; - - case ALTRQ_EVENT: - fairq_event(fif, (cqev_t)arg); - break; - - case ALTRQ_THROTTLE: - default: - break; - } - return (0); -} - -/* - * altq_fairq_enqueue is an enqueue function to be registered to - * (*altq_enqueue) in struct ifaltq. - */ -static int -altq_fairq_enqueue(struct ifaltq *altq, struct mbuf *m) -{ - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - printf("%s: packet for %s does not have pkthdr\n", __func__, - if_name(altq->altq_ifcq->ifcq_ifp)); - m_freem(m); - return (ENOBUFS); - } - - return (fairq_enqueue(altq->altq_disc, NULL, m, m_pftag(m))); -} - -/* - * altq_fairq_dequeue is a dequeue function to be registered to - * (*altq_dequeue) in struct ifaltq. - * - * note: ALTDQ_POLL returns the next packet without removing the packet - * from the queue. ALTDQ_REMOVE is a normal dequeue operation. - * ALTDQ_REMOVE must return the same packet if called immediately - * after ALTDQ_POLL. - */ -static struct mbuf * -altq_fairq_dequeue(struct ifaltq *altq, enum altdq_op op) -{ - return (fairq_dequeue(altq->altq_disc, (cqdq_op_t)op)); -} -#endif /* PF_ALTQ && PKTSCHED_FAIRQ */ diff --git a/bsd/net/altq/altq_fairq.h b/bsd/net/altq/altq_fairq.h index d9d536ca8..f9c20940a 100644 --- a/bsd/net/altq/altq_fairq.h +++ b/bsd/net/altq/altq_fairq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -69,20 +69,4 @@ #include #include -#ifdef BSD_KERNEL_PRIVATE -#ifdef __cplusplus -extern "C" { -#endif - -extern int altq_fairq_pfattach(struct pf_altq *); -extern int altq_fairq_add(struct pf_altq *); -extern int altq_fairq_remove(struct pf_altq *); -extern int altq_fairq_add_queue(struct pf_altq *); -extern int altq_fairq_remove_queue(struct pf_altq *); -extern int altq_fairq_getqstats(struct pf_altq *, void *, int *); - -#ifdef __cplusplus -} -#endif -#endif /* BSD_KERNEL_PRIVATE */ #endif /* _NET_ALTQ_ALTQ_FAIRQ_H_ */ diff --git a/bsd/net/altq/altq_hfsc.c b/bsd/net/altq/altq_hfsc.c deleted file mode 100644 index adaf0d35c..000000000 --- a/bsd/net/altq/altq_hfsc.c +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_hfsc.c,v 1.25 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_hfsc.c,v 1.17 2002/11/29 07:48:33 kjc Exp $ */ - -/* - * Copyright (c) 1997-1999 Carnegie Mellon University. All Rights Reserved. - * - * Permission to use, copy, modify, and distribute this software and - * its documentation is hereby granted (including for commercial or - * for-profit use), provided that both the copyright notice and this - * permission notice appear in all copies of the software, derivative - * works, or modified versions, and any portions thereof. - * - * THIS SOFTWARE IS EXPERIMENTAL AND IS KNOWN TO HAVE BUGS, SOME OF - * WHICH MAY HAVE SERIOUS CONSEQUENCES. CARNEGIE MELLON PROVIDES THIS - * SOFTWARE IN ITS ``AS IS'' CONDITION, AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT - * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE - * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH - * DAMAGE. - * - * Carnegie Mellon encourages (but does not require) users of this - * software to return any improvements or extensions that they make, - * and to grant Carnegie Mellon the rights to redistribute these - * changes without encumbrance. - */ - -#include - -#if PF_ALTQ && PKTSCHED_HFSC - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -/* - * function prototypes - */ -static int altq_hfsc_request(struct ifaltq *, enum altrq, void *); -static int altq_hfsc_enqueue(struct ifaltq *, struct mbuf *); -static struct mbuf *altq_hfsc_dequeue(struct ifaltq *, enum altdq_op); - -int -altq_hfsc_pfattach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - - IFCQ_LOCK(&ifp->if_snd); - error = altq_attach(IFCQ_ALTQ(&ifp->if_snd), ALTQT_HFSC, a->altq_disc, - altq_hfsc_enqueue, altq_hfsc_dequeue, NULL, altq_hfsc_request); - IFCQ_UNLOCK(&ifp->if_snd); - - return (error); -} - -int -altq_hfsc_add(struct pf_altq *a) -{ - struct hfsc_if *hif; - struct ifnet *ifp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - if (!ALTQ_IS_READY(IFCQ_ALTQ(&ifp->if_snd))) - return (ENODEV); - - hif = hfsc_alloc(ifp, M_WAITOK, TRUE); - if (hif == NULL) - return (ENOMEM); - - /* keep the state in pf_altq */ - a->altq_disc = hif; - - return (0); -} - -int -altq_hfsc_remove(struct pf_altq *a) -{ - struct hfsc_if *hif; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((hif = a->altq_disc) == NULL) - return (EINVAL); - a->altq_disc = NULL; - - return (hfsc_destroy(hif)); -} - -int -altq_hfsc_add_queue(struct pf_altq *a) -{ - struct hfsc_if *hif; - struct hfsc_opts *opts = &a->pq_u.hfsc_opts; - struct service_curve rtsc, lssc, ulsc; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((hif = a->altq_disc) == NULL) - return (EINVAL); - - bzero(&rtsc, sizeof (rtsc)); - bzero(&lssc, sizeof (lssc)); - bzero(&ulsc, sizeof (ulsc)); - - rtsc.m1 = opts->rtsc_m1; - rtsc.d = opts->rtsc_d; - rtsc.m2 = opts->rtsc_m2; - rtsc.fl = opts->rtsc_fl; - lssc.m1 = opts->lssc_m1; - lssc.d = opts->lssc_d; - lssc.m2 = opts->lssc_m2; - lssc.fl = opts->lssc_fl; - ulsc.m1 = opts->ulsc_m1; - ulsc.d = opts->ulsc_d; - ulsc.m2 = opts->ulsc_m2; - ulsc.fl = opts->ulsc_fl; - - IFCQ_LOCK(hif->hif_ifq); - err = hfsc_add_queue(hif, &rtsc, &lssc, &ulsc, a->qlimit, - opts->flags, a->parent_qid, a->qid, NULL); - IFCQ_UNLOCK(hif->hif_ifq); - - return (err); -} - -int -altq_hfsc_remove_queue(struct pf_altq *a) -{ - struct hfsc_if *hif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((hif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(hif->hif_ifq); - err = hfsc_remove_queue(hif, a->qid); - IFCQ_UNLOCK(hif->hif_ifq); - - return (err); -} - -int -altq_hfsc_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - struct ifclassq *ifq = NULL; - struct hfsc_if *hif; - struct hfsc_classstats stats; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((unsigned)*nbytes < sizeof (stats)) - return (EINVAL); - - if ((hif = altq_lookup(a->ifname, ALTQT_HFSC)) == NULL) - return (EBADF); - - ifq = hif->hif_ifq; - IFCQ_LOCK_ASSERT_HELD(ifq); /* lock held by altq_lookup */ - error = hfsc_get_class_stats(hif, a->qid, &stats); - IFCQ_UNLOCK(ifq); - if (error != 0) - return (error); - - if ((error = copyout((caddr_t)&stats, (user_addr_t)(uintptr_t)ubuf, - sizeof (stats))) != 0) - return (error); - - *nbytes = sizeof (stats); - - return (0); -} - -static int -altq_hfsc_request(struct ifaltq *altq, enum altrq req, void *arg) -{ - struct hfsc_if *hif = (struct hfsc_if *)altq->altq_disc; - - switch (req) { - case ALTRQ_PURGE: - hfsc_purge(hif); - break; - - case ALTRQ_PURGE_SC: - /* not supported for ALTQ instance */ - break; - - case ALTRQ_EVENT: - hfsc_event(hif, (cqev_t)arg); - break; - - case ALTRQ_THROTTLE: - default: - break; - } - return (0); -} - -/* - * altq_hfsc_enqueue is an enqueue function to be registered to - * (*altq_enqueue) in struct ifaltq. - */ -static int -altq_hfsc_enqueue(struct ifaltq *altq, struct mbuf *m) -{ - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - printf("%s: packet for %s does not have pkthdr\n", __func__, - if_name(altq->altq_ifcq->ifcq_ifp)); - m_freem(m); - return (ENOBUFS); - } - - return (hfsc_enqueue(altq->altq_disc, NULL, m, m_pftag(m))); -} - -/* - * altq_hfsc_dequeue is a dequeue function to be registered to - * (*altq_dequeue) in struct ifaltq. - * - * note: ALTDQ_POLL returns the next packet without removing the packet - * from the queue. ALTDQ_REMOVE is a normal dequeue operation. - * ALTDQ_REMOVE must return the same packet if called immediately - * after ALTDQ_POLL. - */ -static struct mbuf * -altq_hfsc_dequeue(struct ifaltq *altq, enum altdq_op op) -{ - return (hfsc_dequeue(altq->altq_disc, (cqdq_op_t)op)); -} -#endif /* PF_ALTQ && PKTSCHED_HFSC */ diff --git a/bsd/net/altq/altq_hfsc.h b/bsd/net/altq/altq_hfsc.h index 6b46293e7..0addc4fea 100644 --- a/bsd/net/altq/altq_hfsc.h +++ b/bsd/net/altq/altq_hfsc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -65,20 +65,4 @@ #include #include -#ifdef BSD_KERNEL_PRIVATE -#ifdef __cplusplus -extern "C" { -#endif - -extern int altq_hfsc_pfattach(struct pf_altq *); -extern int altq_hfsc_add(struct pf_altq *); -extern int altq_hfsc_remove(struct pf_altq *); -extern int altq_hfsc_add_queue(struct pf_altq *); -extern int altq_hfsc_remove_queue(struct pf_altq *); -extern int altq_hfsc_getqstats(struct pf_altq *, void *, int *); - -#ifdef __cplusplus -} -#endif -#endif /* BSD_KERNEL_PRIVATE */ #endif /* _NET_ALTQ_ALTQ_HFSC_H_ */ diff --git a/bsd/net/altq/altq_priq.c b/bsd/net/altq/altq_priq.c deleted file mode 100644 index a86a48383..000000000 --- a/bsd/net/altq/altq_priq.c +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Copyright (c) 2007-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_priq.c,v 1.21 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_priq.c,v 1.1 2000/10/18 09:15:23 kjc Exp $ */ - -/* - * Copyright (C) 2000-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * priority queue - */ - -#if PF_ALTQ && PKTSCHED_PRIQ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -/* - * function prototypes - */ -static int altq_priq_enqueue(struct ifaltq *, struct mbuf *); -static struct mbuf *altq_priq_dequeue(struct ifaltq *, enum altdq_op); -static int altq_priq_request(struct ifaltq *, enum altrq, void *); - -int -altq_priq_pfattach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - - IFCQ_LOCK(&ifp->if_snd); - error = altq_attach(IFCQ_ALTQ(&ifp->if_snd), ALTQT_PRIQ, a->altq_disc, - altq_priq_enqueue, altq_priq_dequeue, NULL, altq_priq_request); - IFCQ_UNLOCK(&ifp->if_snd); - - return (error); -} - -int -altq_priq_add(struct pf_altq *a) -{ - struct priq_if *pif; - struct ifnet *ifp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - if (!ALTQ_IS_READY(IFCQ_ALTQ(&ifp->if_snd))) - return (ENODEV); - - pif = priq_alloc(ifp, M_WAITOK, TRUE); - if (pif == NULL) - return (ENOMEM); - - /* keep the state in pf_altq */ - a->altq_disc = pif; - - return (0); -} - -int -altq_priq_remove(struct pf_altq *a) -{ - struct priq_if *pif; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((pif = a->altq_disc) == NULL) - return (EINVAL); - a->altq_disc = NULL; - - return (priq_destroy(pif)); -} - -int -altq_priq_add_queue(struct pf_altq *a) -{ - struct priq_if *pif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((pif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(pif->pif_ifq); - err = priq_add_queue(pif, a->priority, a->qlimit, - a->pq_u.priq_opts.flags, a->qid, NULL); - IFCQ_UNLOCK(pif->pif_ifq); - - return (err); -} - -int -altq_priq_remove_queue(struct pf_altq *a) -{ - struct priq_if *pif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((pif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(pif->pif_ifq); - err = priq_remove_queue(pif, a->qid); - IFCQ_UNLOCK(pif->pif_ifq); - - return (err); -} - -int -altq_priq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - struct ifclassq *ifq = NULL; - struct priq_if *pif; - struct priq_classstats stats; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((unsigned)*nbytes < sizeof (stats)) - return (EINVAL); - - if ((pif = altq_lookup(a->ifname, ALTQT_PRIQ)) == NULL) - return (EBADF); - - ifq = pif->pif_ifq; - IFCQ_LOCK_ASSERT_HELD(ifq); /* lock held by altq_lookup */ - error = priq_get_class_stats(pif, a->qid, &stats); - IFCQ_UNLOCK(ifq); - if (error != 0) - return (error); - - if ((error = copyout((caddr_t)&stats, (user_addr_t)(uintptr_t)ubuf, - sizeof (stats))) != 0) - return (error); - - *nbytes = sizeof (stats); - - return (0); -} - -static int -altq_priq_request(struct ifaltq *altq, enum altrq req, void *arg) -{ - struct priq_if *pif = (struct priq_if *)altq->altq_disc; - - switch (req) { - case ALTRQ_PURGE: - priq_purge(pif); - break; - - case ALTRQ_PURGE_SC: - case ALTRQ_THROTTLE: - /* not supported for ALTQ instance */ - break; - - case ALTRQ_EVENT: - priq_event(pif, (cqev_t)arg); - break; - } - return (0); -} - -/* - * altq_priq_enqueue is an enqueue function to be registered to - * (*altq_enqueue) in struct ifaltq. - */ -static int -altq_priq_enqueue(struct ifaltq *altq, struct mbuf *m) -{ - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - printf("%s: packet for %s does not have pkthdr\n", __func__, - if_name(altq->altq_ifcq->ifcq_ifp)); - m_freem(m); - return (ENOBUFS); - } - - return (priq_enqueue(altq->altq_disc, NULL, m, m_pftag(m))); -} - -/* - * altq_priq_dequeue is a dequeue function to be registered to - * (*altq_dequeue) in struct ifaltq. - * - * note: ALTDQ_POLL returns the next packet without removing the packet - * from the queue. ALTDQ_REMOVE is a normal dequeue operation. - * ALTDQ_REMOVE must return the same packet if called immediately - * after ALTDQ_POLL. - */ -static struct mbuf * -altq_priq_dequeue(struct ifaltq *altq, enum altdq_op op) -{ - return (priq_dequeue(altq->altq_disc, (cqdq_op_t)op)); -} -#endif /* PF_ALTQ && PKTSCHED_PRIQ */ diff --git a/bsd/net/altq/altq_priq.h b/bsd/net/altq/altq_priq.h index f6b6372e9..f1f92e939 100644 --- a/bsd/net/altq/altq_priq.h +++ b/bsd/net/altq/altq_priq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -61,20 +61,4 @@ #include #include -#ifdef BSD_KERNEL_PRIVATE -#ifdef __cplusplus -extern "C" { -#endif - -extern int altq_priq_pfattach(struct pf_altq *); -extern int altq_priq_add(struct pf_altq *); -extern int altq_priq_remove(struct pf_altq *); -extern int altq_priq_add_queue(struct pf_altq *); -extern int altq_priq_remove_queue(struct pf_altq *); -extern int altq_priq_getqstats(struct pf_altq *, void *, int *); - -#ifdef __cplusplus -} -#endif -#endif /* BSD_KERNEL_PRIVATE */ #endif /* _NET_ALTQ_ALTQ_PRIQ_H_ */ diff --git a/bsd/net/altq/altq_qfq.c b/bsd/net/altq/altq_qfq.c deleted file mode 100644 index 0f7c52983..000000000 --- a/bsd/net/altq/altq_qfq.c +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* - * quick fair queueing - */ - -#if PF_ALTQ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -/* - * function prototypes - */ -static int altq_qfq_enqueue(struct ifaltq *, struct mbuf *); -static struct mbuf *altq_qfq_dequeue(struct ifaltq *, enum altdq_op); -static int altq_qfq_request(struct ifaltq *, enum altrq, void *); - -int -altq_qfq_pfattach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - - IFCQ_LOCK(&ifp->if_snd); - error = altq_attach(IFCQ_ALTQ(&ifp->if_snd), ALTQT_QFQ, a->altq_disc, - altq_qfq_enqueue, altq_qfq_dequeue, NULL, altq_qfq_request); - IFCQ_UNLOCK(&ifp->if_snd); - - return (error); -} - -int -altq_qfq_add(struct pf_altq *a) -{ - struct qfq_if *qif; - struct ifnet *ifp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - if (!ALTQ_IS_READY(IFCQ_ALTQ(&ifp->if_snd))) - return (ENODEV); - - qif = qfq_alloc(ifp, M_WAITOK, TRUE); - if (qif == NULL) - return (ENOMEM); - - /* keep the state in pf_altq */ - a->altq_disc = qif; - - return (0); -} - -int -altq_qfq_remove(struct pf_altq *a) -{ - struct qfq_if *qif; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((qif = a->altq_disc) == NULL) - return (EINVAL); - a->altq_disc = NULL; - - return (qfq_destroy(qif)); -} - -int -altq_qfq_add_queue(struct pf_altq *a) -{ - struct qfq_if *qif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((qif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(qif->qif_ifq); - err = qfq_add_queue(qif, a->qlimit, a->weight, a->pq_u.qfq_opts.lmax, - a->pq_u.qfq_opts.flags, a->qid, NULL); - IFCQ_UNLOCK(qif->qif_ifq); - - return (err); -} - -int -altq_qfq_remove_queue(struct pf_altq *a) -{ - struct qfq_if *qif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((qif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(qif->qif_ifq); - err = qfq_remove_queue(qif, a->qid); - IFCQ_UNLOCK(qif->qif_ifq); - - return (err); -} - -int -altq_qfq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - struct ifclassq *ifq = NULL; - struct qfq_if *qif; - struct qfq_classstats stats; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((unsigned)*nbytes < sizeof (stats)) - return (EINVAL); - - if ((qif = altq_lookup(a->ifname, ALTQT_QFQ)) == NULL) - return (EBADF); - - ifq = qif->qif_ifq; - IFCQ_LOCK_ASSERT_HELD(ifq); /* lock held by altq_lookup */ - error = qfq_get_class_stats(qif, a->qid, &stats); - IFCQ_UNLOCK(ifq); - if (error != 0) - return (error); - - if ((error = copyout((caddr_t)&stats, (user_addr_t)(uintptr_t)ubuf, - sizeof (stats))) != 0) - return (error); - - *nbytes = sizeof (stats); - - return (0); -} - -static int -altq_qfq_request(struct ifaltq *altq, enum altrq req, void *arg) -{ - struct qfq_if *qif = (struct qfq_if *)altq->altq_disc; - - switch (req) { - case ALTRQ_PURGE: - qfq_purge(qif); - break; - - case ALTRQ_PURGE_SC: - /* not supported for ALTQ instance */ - break; - - case ALTRQ_EVENT: - qfq_event(qif, (cqev_t)arg); - break; - - case ALTRQ_THROTTLE: - default: - break; - } - return (0); -} - -/* - * altq_qfq_enqueue is an enqueue function to be registered to - * (*altq_enqueue) in struct ifaltq. - */ -static int -altq_qfq_enqueue(struct ifaltq *altq, struct mbuf *m) -{ - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - printf("%s: packet for %s does not have pkthdr\n", __func__, - if_name(altq->altq_ifcq->ifcq_ifp)); - m_freem(m); - return (ENOBUFS); - } - - return (qfq_enqueue(altq->altq_disc, NULL, m, m_pftag(m))); -} - -/* - * altq_qfq_dequeue is a dequeue function to be registered to - * (*altq_dequeue) in struct ifaltq. - * - * note: ALTDQ_POLL returns the next packet without removing the packet - * from the queue. ALTDQ_REMOVE is a normal dequeue operation. - * ALTDQ_REMOVE must return the same packet if called immediately - * after ALTDQ_POLL. - */ -static struct mbuf * -altq_qfq_dequeue(struct ifaltq *altq, enum altdq_op op) -{ - return (qfq_dequeue(altq->altq_disc, (cqdq_op_t)op)); -} -#endif /* PF_ALTQ */ diff --git a/bsd/net/altq/altq_qfq.h b/bsd/net/altq/altq_qfq.h index 790742229..942fca369 100644 --- a/bsd/net/altq/altq_qfq.h +++ b/bsd/net/altq/altq_qfq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -33,20 +33,4 @@ #include #include -#ifdef BSD_KERNEL_PRIVATE -#ifdef __cplusplus -extern "C" { -#endif - -extern int altq_qfq_pfattach(struct pf_altq *); -extern int altq_qfq_add(struct pf_altq *); -extern int altq_qfq_remove(struct pf_altq *); -extern int altq_qfq_add_queue(struct pf_altq *); -extern int altq_qfq_remove_queue(struct pf_altq *); -extern int altq_qfq_getqstats(struct pf_altq *, void *, int *); - -#ifdef __cplusplus -} -#endif -#endif /* BSD_KERNEL_PRIVATE */ #endif /* _NET_ALTQ_ALTQ_QFQ_H_ */ diff --git a/bsd/net/altq/altq_subr.c b/bsd/net/altq/altq_subr.c deleted file mode 100644 index 5b00e6f5b..000000000 --- a/bsd/net/altq/altq_subr.c +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Copyright (c) 2007-2011 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_subr.c,v 1.24 2007/12/11 00:30:14 mikeb Exp $ */ -/* $KAME: altq_subr.c,v 1.11 2002/01/11 08:11:49 kjc Exp $ */ - -/* - * Copyright (C) 1997-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -SYSCTL_NODE(_net, OID_AUTO, altq, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "ALTQ"); - -static u_int32_t altq_debug; -SYSCTL_UINT(_net_altq, OID_AUTO, debug, CTLFLAG_RW, &altq_debug, 0, - "Enable ALTQ debugging"); - -/* - * look up the queue state by the interface name and the queueing type; - * upon success, returns with the interface send queue lock held, and - * the caller is responsible for releasing it. - */ -void * -altq_lookup(char *name, u_int32_t type) -{ - struct ifnet *ifp; - void *state = NULL; - - if ((ifp = ifunit(name)) != NULL) { - IFCQ_LOCK(&ifp->if_snd); - if (type != ALTQT_NONE && - IFCQ_ALTQ(&ifp->if_snd)->altq_type == type) - state = IFCQ_ALTQ(&ifp->if_snd)->altq_disc; - if (state == NULL) - IFCQ_UNLOCK(&ifp->if_snd); - } - - if (state != NULL) - IFCQ_LOCK_ASSERT_HELD(&ifp->if_snd); - - return (state); -} - -int -altq_attach(struct ifaltq *altq, u_int32_t type, void *discipline, - altq_enq_func enqueue, altq_deq_func dequeue, - altq_deq_sc_func dequeue_sc, altq_req_func request) -{ - IFCQ_LOCK_ASSERT_HELD(altq->altq_ifcq); - - if (!ALTQ_IS_READY(altq)) - return (ENXIO); - - VERIFY(enqueue != NULL); - VERIFY(!(dequeue != NULL && dequeue_sc != NULL)); - VERIFY(request != NULL); - - altq->altq_type = type; - altq->altq_disc = discipline; - altq->altq_enqueue = enqueue; - altq->altq_dequeue = dequeue; - altq->altq_dequeue_sc = dequeue_sc; - altq->altq_request = request; - altq->altq_flags &= (ALTQF_CANTCHANGE|ALTQF_ENABLED); - - return (0); -} - -int -altq_detach(struct ifaltq *altq) -{ - IFCQ_LOCK_ASSERT_HELD(altq->altq_ifcq); - - if (!ALTQ_IS_READY(altq)) - return (ENXIO); - if (ALTQ_IS_ENABLED(altq)) - return (EBUSY); - if (!ALTQ_IS_ATTACHED(altq)) - return (0); - - altq->altq_type = ALTQT_NONE; - altq->altq_disc = NULL; - altq->altq_enqueue = NULL; - altq->altq_dequeue = NULL; - altq->altq_dequeue_sc = NULL; - altq->altq_request = NULL; - altq->altq_flags &= ALTQF_CANTCHANGE; - - return (0); -} - -int -altq_enable(struct ifaltq *altq) -{ - struct ifclassq *ifq = altq->altq_ifcq; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (!ALTQ_IS_READY(altq)) - return (ENXIO); - if (ALTQ_IS_ENABLED(altq)) - return (0); - - altq->altq_flags |= ALTQF_ENABLED; - - return (0); -} - -int -altq_disable(struct ifaltq *altq) -{ - struct ifclassq *ifq = altq->altq_ifcq; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (!ALTQ_IS_ENABLED(altq)) - return (0); - - if_qflush(ifq->ifcq_ifp, 1); - - altq->altq_flags &= ~ALTQF_ENABLED; - - return (0); -} - -/* - * add a discipline or a queue - */ -int -altq_add(struct pf_altq *a) -{ - int error = 0; - - VERIFY(machclk_freq != 0); - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if (a->qname[0] != 0) - return (altq_add_queue(a)); - - switch (a->scheduler) { -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_add(a); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_add(a); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_add(a); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_add(a); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_add(a); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * remove a discipline or a queue - */ -int -altq_remove(struct pf_altq *a) -{ - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if (a->qname[0] != 0) - return (altq_remove_queue(a)); - - switch (a->scheduler) { -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_remove(a); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_remove(a); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_remove(a); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_remove(a); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_remove(a); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * add a queue to the discipline - */ -int -altq_add_queue(struct pf_altq *a) -{ - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - switch (a->scheduler) { -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_add_queue(a); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_add_queue(a); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_add_queue(a); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_add_queue(a); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_add_queue(a); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * remove a queue from the discipline - */ -int -altq_remove_queue(struct pf_altq *a) -{ - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - switch (a->scheduler) { -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_remove_queue(a); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_remove_queue(a); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_remove_queue(a); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_remove_queue(a); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_remove_queue(a); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * get queue statistics - */ -int -altq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - switch (a->scheduler) { -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_getqstats(a, ubuf, nbytes); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_getqstats(a, ubuf, nbytes); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_getqstats(a, ubuf, nbytes); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_getqstats(a, ubuf, nbytes); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_getqstats(a, ubuf, nbytes); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * attach a discipline to the interface. if one already exists, it is - * overridden. - */ -int -altq_pfattach(struct pf_altq *a) -{ - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - switch (a->scheduler) { - case ALTQT_NONE: - break; -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_pfattach(a); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_pfattach(a); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_pfattach(a); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_pfattach(a); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_pfattach(a); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * detach a discipline from the interface. - * it is possible that the discipline was already overridden by another - * discipline. - */ -int -altq_pfdetach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - - /* if this discipline is no longer referenced, just return */ - IFCQ_LOCK(&ifp->if_snd); - if (a->altq_disc == NULL || - a->altq_disc != IFCQ_ALTQ(&ifp->if_snd)->altq_disc) { - IFCQ_UNLOCK(&ifp->if_snd); - return (0); - } - - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(&ifp->if_snd))) - error = altq_disable(IFCQ_ALTQ(&ifp->if_snd)); - if (error == 0) - error = altq_detach(IFCQ_ALTQ(&ifp->if_snd)); - IFCQ_UNLOCK(&ifp->if_snd); - return (error); -} - - diff --git a/bsd/net/altq/altq_var.h b/bsd/net/altq/altq_var.h deleted file mode 100644 index e866a4dc4..000000000 --- a/bsd/net/altq/altq_var.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2011 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $NetBSD: altq_var.h,v 1.10 2006/10/15 13:17:13 peter Exp $ */ -/* $KAME: altq_var.h,v 1.18 2005/04/13 03:44:25 suz Exp $ */ - -/* - * Copyright (C) 1998-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#ifndef _NET_ALTQ_ALTQ_VAR_H_ -#define _NET_ALTQ_ALTQ_VAR_H_ - -#ifdef BSD_KERNEL_PRIVATE -#if PF_ALTQ -#include -#include -#include -#include -#include -#include -#if PKTSCHED_HFSC -#include -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ -#include -#endif /* PKTSCHED_FAIRQ */ -#if PKTSCHED_CBQ -#include -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ -#include -#endif /* PKTSCHED_PRIQ */ -#include - -struct pf_altq; - -extern void *altq_lookup(char *, u_int32_t); -extern int altq_pfattach(struct pf_altq *); -extern int altq_pfdetach(struct pf_altq *); -extern int altq_add(struct pf_altq *); -extern int altq_remove(struct pf_altq *); -extern int altq_add_queue(struct pf_altq *); -extern int altq_remove_queue(struct pf_altq *); -extern int altq_getqstats(struct pf_altq *, void *, int *); - -#endif /* PF_ALTQ */ -#endif /* BSD_KERNEL_PRIVATE */ -#endif /* _NET_ALTQ_ALTQ_VAR_H_ */ diff --git a/bsd/net/altq/if_altq.h b/bsd/net/altq/if_altq.h deleted file mode 100644 index 6d634cf5d..000000000 --- a/bsd/net/altq/if_altq.h +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2008-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ -/* $OpenBSD: if_altq.h,v 1.11 2007/11/18 12:51:48 mpf Exp $ */ -/* $KAME: if_altq.h,v 1.6 2001/01/29 19:59:09 itojun Exp $ */ - -/* - * Copyright (C) 1997-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#ifndef _NET_ALTQ_IF_ALTQ_H_ -#define _NET_ALTQ_IF_ALTQ_H_ - -#ifdef BSD_KERNEL_PRIVATE -#if PF_ALTQ -#include - -/* altq request types */ -typedef enum altrq { - ALTRQ_PURGE = CLASSQRQ_PURGE, /* purge all packets */ - ALTRQ_PURGE_SC = CLASSQRQ_PURGE_SC, /* purge SC flow */ - ALTRQ_EVENT = CLASSQRQ_EVENT, /* interface events */ - ALTRQ_THROTTLE = CLASSQRQ_THROTTLE, /* throttle packets */ -} altrq_t; - -struct ifaltq; -enum altdq_op; - -typedef int (*altq_enq_func)(struct ifaltq *, struct mbuf *); -typedef struct mbuf *(*altq_deq_func)(struct ifaltq *, enum altdq_op); -typedef struct mbuf *(*altq_deq_sc_func)(struct ifaltq *, - mbuf_svc_class_t, enum altdq_op); -typedef int (*altq_req_func)(struct ifaltq *, enum altrq, void *); - -/* - * Structure defining a queue for a network interface. - */ -struct ifaltq { - struct ifclassq *altq_ifcq; /* back pointer to interface queue */ - - /* alternate queueing related fields */ - u_int32_t altq_type; /* discipline type */ - u_int32_t altq_flags; /* flags (e.g. ready, in-use) */ - void *altq_disc; /* for discipline-specific use */ - - altq_enq_func altq_enqueue; - altq_deq_func altq_dequeue; - altq_deq_sc_func altq_dequeue_sc; - altq_req_func altq_request; -}; - -/* altq_flags */ -#define ALTQF_READY 0x01 /* driver supports alternate queueing */ -#define ALTQF_ENABLED 0x02 /* altq is in use */ -#define ALTQF_DRIVER1 0x40 /* driver specific */ - -/* altq_flags set internally only: */ -#define ALTQF_CANTCHANGE (ALTQF_READY) - -/* altq_dequeue op arg */ -typedef enum altdq_op { - ALTDQ_REMOVE = CLASSQDQ_REMOVE, /* dequeue mbuf from the queue */ - ALTDQ_POLL = CLASSQDQ_POLL, /* don't dequeue mbuf from the queue */ -} altdq_op_t; - -#define ALTQ_IS_READY(_altq) ((_altq)->altq_flags & ALTQF_READY) -#define ALTQ_IS_ENABLED(_altq) ((_altq)->altq_flags & ALTQF_ENABLED) -#define ALTQ_IS_ATTACHED(_altq) ((_altq)->altq_disc != NULL) - -#define ALTQ_ENQUEUE(_altq, _m, _err) do { \ - (_err) = (*(_altq)->altq_enqueue)(_altq, _m); \ -} while (0) - -#define ALTQ_DEQUEUE(_altq, _m) do { \ - (_m) = (*(_altq)->altq_dequeue)(_altq, ALTDQ_REMOVE); \ -} while (0) - -#define ALTQ_DEQUEUE_SC(_altq, _sc, _m) do { \ - (_m) = (*(_altq)->altq_dequeue_sc)(_altq, _sc, ALTDQ_REMOVE); \ -} while (0) - -#define ALTQ_POLL(_altq, _m) do { \ - (_m) = (*(_altq)->altq_dequeue)(_altq, ALTDQ_POLL); \ -} while (0) - -#define ALTQ_POLL_SC(_altq, _sc, _m) do { \ - (_m) = (*(_altq)->altq_dequeue_sc)(_altq, _sc, ALTDQ_POLL); \ -} while (0) - -#define ALTQ_PURGE(_altq) do { \ - (void) (*(_altq)->altq_request)(_altq, ALTRQ_PURGE, NULL); \ -} while (0) - -#define ALTQ_PURGE_SC(_altq, _sc, _flow, _packets, _bytes) do { \ - cqrq_purge_sc_t _req = { _sc, _flow, 0, 0 }; \ - (void) (*(_altq)->altq_request)(_altq, ALTRQ_PURGE_SC, &_req); \ - (_packets) = _req.packets; \ - (_bytes) = _req.bytes; \ -} while (0) - -#define ALTQ_UPDATE(_altq, _ev) do { \ - (void) (*(_altq)->altq_request)(_altq, ALTRQ_EVENT, \ - (void *)(_ev)); \ -} while (0) - -#define ALTQ_SET_READY(_altq) do { \ - IFCQ_LOCK_ASSERT_HELD((_altq)->altq_ifcq); \ - (_altq)->altq_flags |= ALTQF_READY; \ -} while (0) - -#define ALTQ_CLEAR_READY(_altq) do { \ - IFCQ_LOCK_ASSERT_HELD((_altq)->altq_ifcq); \ - (_altq)->altq_flags &= ~ALTQF_READY; \ -} while (0) - -extern int altq_attach(struct ifaltq *, u_int32_t, void *, - altq_enq_func, altq_deq_func, altq_deq_sc_func, altq_req_func); -extern int altq_detach(struct ifaltq *); -extern int altq_enable(struct ifaltq *); -extern int altq_disable(struct ifaltq *); -#endif /* PF_ALTQ */ -#endif /* BSD_KERNEL_PRIVATE */ -#endif /* _NET_ALTQ_IF_ALTQ_H_ */ diff --git a/bsd/net/bpf.c b/bsd/net/bpf.c index 66e0c3b54..3bf795149 100644 --- a/bsd/net/bpf.c +++ b/bsd/net/bpf.c @@ -126,6 +126,7 @@ #include #include +#include #if CONFIG_MACF_NET #include @@ -139,6 +140,8 @@ extern int tvtohz(struct timeval *); #define PRINET 26 /* interruptible */ +typedef void (*pktcopyfunc_t)(const void *, void *, size_t); + /* * The default read buffer size is patchable. */ @@ -156,7 +159,11 @@ SYSCTL_UINT(_debug, OID_AUTO, bpf_maxdevices, CTLFLAG_RW | CTLFLAG_LOCKED, * For OS X is off by default so process need to use the ioctl BPF_WANT_PKTAP * explicitly to be able to use DLT_PKTAP. */ +#if CONFIG_EMBEDDED +static unsigned int bpf_wantpktap = 1; +#else static unsigned int bpf_wantpktap = 0; +#endif SYSCTL_UINT(_debug, OID_AUTO, bpf_wantpktap, CTLFLAG_RW | CTLFLAG_LOCKED, &bpf_wantpktap, 0, ""); @@ -190,21 +197,18 @@ static lck_grp_t *bpf_mlock_grp; static lck_grp_attr_t *bpf_mlock_grp_attr; static lck_attr_t *bpf_mlock_attr; -static mbuf_tag_id_t bpf_mtag_id; #endif /* __APPLE__ */ static int bpf_allocbufs(struct bpf_d *); static errno_t bpf_attachd(struct bpf_d *d, struct bpf_if *bp); static int bpf_detachd(struct bpf_d *d, int); static void bpf_freed(struct bpf_d *); -static void bpf_mcopy(const void *, void *, size_t); static int bpf_movein(struct uio *, int, struct mbuf **, struct sockaddr *, int *); -static int bpf_setif(struct bpf_d *, ifnet_t ifp, u_int32_t dlt); +static int bpf_setif(struct bpf_d *, ifnet_t ifp); static void bpf_timed_out(void *, void *); static void bpf_wakeup(struct bpf_d *); -static void catchpacket(struct bpf_d *, u_char *, struct mbuf *, u_int, - u_int, int, void (*)(const void *, void *, size_t)); +static void catchpacket(struct bpf_d *, struct bpf_packet *, u_int, int); static void reset_d(struct bpf_d *); static int bpf_setf(struct bpf_d *, u_int, user_addr_t, u_long); static int bpf_getdltlist(struct bpf_d *, caddr_t, struct proc *); @@ -503,26 +507,18 @@ bpf_attachd(struct bpf_d *d, struct bpf_if *bp) struct bpf_if *tmp, *primary = NULL; for (tmp = bpf_iflist; tmp; tmp = tmp->bif_next) { - if (tmp->bif_ifp != bp->bif_ifp) - continue; - primary = tmp; - /* - * Make DLT_PKTAP only if process knows how - * to deal with it, otherwise find another one - */ - if (tmp->bif_dlt == DLT_PKTAP && - !(d->bd_flags & BPF_WANT_PKTAP)) - continue; - break; + if (tmp->bif_ifp == bp->bif_ifp) { + primary = tmp; + break; + } } bp->bif_ifp->if_bpf = primary; } - /* Only call dlil_set_bpf_tap for primary dlt */ if (bp->bif_ifp->if_bpf == bp) - dlil_set_bpf_tap(bp->bif_ifp, BPF_TAP_INPUT_OUTPUT, bpf_tap_callback); - - if (bp->bif_tap) + dlil_set_bpf_tap(bp->bif_ifp, BPF_TAP_INPUT_OUTPUT, bpf_tap_callback); + + if (bp->bif_tap != NULL) error = bp->bif_tap(bp->bif_ifp, bp->bif_dlt, BPF_TAP_INPUT_OUTPUT); } @@ -531,12 +527,11 @@ bpf_attachd(struct bpf_d *d, struct bpf_if *bp) */ d->bd_flags &= ~(BPF_DETACHING | BPF_DETACHED); - if (bp->bif_ifp->if_bpf != NULL && - bp->bif_ifp->if_bpf->bif_dlt == DLT_PKTAP) + if (bp->bif_dlt == DLT_PKTAP) { d->bd_flags |= BPF_FINALIZE_PKTAP; - else + } else { d->bd_flags &= ~BPF_FINALIZE_PKTAP; - + } return error; } @@ -699,7 +694,7 @@ bpf_acquire_d(struct bpf_d *d) { void *lr_saved = __builtin_return_address(0); - lck_mtx_assert(bpf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED); d->bd_refcnt += 1; @@ -712,7 +707,7 @@ bpf_release_d(struct bpf_d *d) { void *lr_saved = __builtin_return_address(0); - lck_mtx_assert(bpf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED); if (d->bd_refcnt <= 0) panic("%s: %p refcnt <= 0", __func__, d); @@ -1056,7 +1051,7 @@ bpfread(dev_t dev, struct uio *uio, int ioflag) } if (error == EINTR || error == ERESTART) { - if (d->bd_hbuf) { + if (d->bd_hbuf != NULL) { /* * Because we msleep, the hold buffer might * be filled when we wake up. Avoid rotating @@ -1064,7 +1059,7 @@ bpfread(dev_t dev, struct uio *uio, int ioflag) */ break; } - if (d->bd_slen) { + if (d->bd_slen != 0) { /* * Sometimes we may be interrupted often and * the sleep above will not timeout. @@ -1077,6 +1072,11 @@ bpfread(dev_t dev, struct uio *uio, int ioflag) } bpf_release_d(d); lck_mtx_unlock(bpf_mlock); + if (error == ERESTART) { + printf("%s: %llx ERESTART to EINTR\n", + __func__, (uint64_t)VM_KERNEL_ADDRPERM(d)); + error = EINTR; + } return (error); } if (error == EWOULDBLOCK) { @@ -1147,6 +1147,7 @@ bpfread(dev_t dev, struct uio *uio, int ioflag) } ehp->bh_flowid = 0; } + if (flags & BPF_FINALIZE_PKTAP) { struct pktap_header *pktaphdr; @@ -1617,6 +1618,11 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, u_int dlt; bcopy(addr, &dlt, sizeof (dlt)); + + if (dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) { + printf("BIOCSDLT downgrade DLT_PKTAP to DLT_RAW\n"); + dlt = DLT_RAW; + } error = bpf_setdlt(d, dlt); } break; @@ -1647,7 +1653,7 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, if (ifp == NULL) error = ENXIO; else - error = bpf_setif(d, ifp, 0); + error = bpf_setif(d, ifp); break; } @@ -1949,7 +1955,7 @@ bpf_setf(struct bpf_d *d, u_int bf_len, user_addr_t bf_insns, * Return an errno or 0. */ static int -bpf_setif(struct bpf_d *d, ifnet_t theywant, u_int32_t dlt) +bpf_setif(struct bpf_d *d, ifnet_t theywant) { struct bpf_if *bp; int error; @@ -1966,14 +1972,12 @@ bpf_setif(struct bpf_d *d, ifnet_t theywant, u_int32_t dlt) for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) { struct ifnet *ifp = bp->bif_ifp; - if (ifp == 0 || ifp != theywant || (dlt != 0 && dlt != bp->bif_dlt)) + if (ifp == 0 || ifp != theywant) continue; /* - * If the process knows how to deal with DLT_PKTAP, use it - * by default + * Do not use DLT_PKTAP, unless requested explicitly */ - if (dlt == 0 && bp->bif_dlt == DLT_PKTAP && - !(d->bd_flags & BPF_WANT_PKTAP)) + if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) continue; /* * We found the requested interface. @@ -2030,7 +2034,7 @@ bpf_getdltlist(struct bpf_d *d, caddr_t addr, struct proc *p) if (bp->bif_ifp != ifp) continue; /* - * Return DLT_PKTAP only to processes that know how to handle it + * Do not use DLT_PKTAP, unless requested explicitly */ if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) continue; @@ -2073,8 +2077,15 @@ bpf_setdlt(struct bpf_d *d, uint32_t dlt) ifp = d->bd_bif->bif_ifp; for (bp = bpf_iflist; bp; bp = bp->bif_next) { - if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) + if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) { + /* + * Do not use DLT_PKTAP, unless requested explicitly + */ + if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) { + continue; + } break; + } } if (bp != NULL) { opromisc = d->bd_promisc; @@ -2202,7 +2213,7 @@ static int filt_bpfread(struct knote *, long); static int filt_bpftouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_bpfprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops bpfread_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) bpfread_filtops = { .f_isfd = 1, .f_detach = filt_bpfdetach, .f_event = filt_bpfread, @@ -2373,12 +2384,11 @@ filt_bpfprocess(struct knote *kn, struct filt_process_s *data, struct kevent_int /* * Copy data from an mbuf chain into a buffer. This code is derived - * from m_copydata in sys/uipc_mbuf.c. + * from m_copydata in kern/uipc_mbuf.c. */ static void -bpf_mcopy(const void *src_arg, void *dst_arg, size_t len) +bpf_mcopy(struct mbuf * m, void *dst_arg, size_t len) { - struct mbuf *m = (struct mbuf *)(uintptr_t)(src_arg); u_int count; u_char *dst; @@ -2398,19 +2408,18 @@ static inline void bpf_tap_imp( ifnet_t ifp, u_int32_t dlt, - mbuf_t m, - void* hdr, - size_t hlen, + struct bpf_packet *bpf_pkt, int outbound) { + struct bpf_d *d; + u_int slen; struct bpf_if *bp; - struct mbuf *savedm = m; /* * It's possible that we get here after the bpf descriptor has been * detached from the interface; in such a case we simply return. * Lock ordering is important since we can be called asynchronously - * (from the IOKit) to process an inbound packet; when that happens + * (from IOKit) to process an inbound packet; when that happens * we would have been holding its "gateLock" and will be acquiring * "bpf_mlock" upon entering this routine. Due to that, we release * "bpf_mlock" prior to calling ifnet_set_promiscuous (which will @@ -2423,62 +2432,70 @@ bpf_tap_imp( lck_mtx_unlock(bpf_mlock); return; } - bp = ifp->if_bpf; - for (bp = ifp->if_bpf; bp && bp->bif_ifp == ifp && - (dlt != 0 && bp->bif_dlt != dlt); bp = bp->bif_next) - ; - if (bp && bp->bif_ifp == ifp && bp->bif_dlist != NULL) { - struct bpf_d *d; - struct m_hdr hack_hdr; - u_int pktlen = 0; - u_int slen = 0; - struct mbuf *m0; - - if (hdr) { - /* - * This is gross. We mock up an mbuf that points to the - * header buffer. This means we don't have to copy the - * header. A number of interfaces prepended headers just - * for bpf by allocating an mbuf on the stack. We want to - * give developers an easy way to prepend a header for bpf. - * Since a developer allocating an mbuf on the stack is bad, - * we do even worse here, allocating only a header to point - * to a buffer the developer supplied. This makes assumptions - * that bpf_filter and catchpacket will not look at anything - * in the mbuf other than the header. This was true at the - * time this code was written. - */ - hack_hdr.mh_next = m; - hack_hdr.mh_nextpkt = NULL; - hack_hdr.mh_len = hlen; - hack_hdr.mh_data = hdr; - hack_hdr.mh_type = m->m_type; - hack_hdr.mh_flags = 0; - - __IGNORE_WCASTALIGN(m = (mbuf_t)&hack_hdr); + for (bp = ifp->if_bpf; bp != NULL; bp = bp->bif_next) { + if (bp->bif_ifp != ifp) { + /* wrong interface */ + bp = NULL; + break; } - - for (m0 = m; m0 != 0; m0 = m0->m_next) - pktlen += m0->m_len; - - for (d = bp->bif_dlist; d; d = d->bd_next) { - if (outbound && !d->bd_seesent) - continue; - ++d->bd_rcount; - slen = bpf_filter(d->bd_filter, (u_char *)m, pktlen, 0); - if (slen != 0) { + if (dlt == 0 || bp->bif_dlt == dlt) { + /* tapping default DLT or DLT matches */ + break; + } + } + if (bp == NULL) { + goto done; + } + for (d = bp->bif_dlist; d; d = d->bd_next) { + if (outbound && !d->bd_seesent) + continue; + ++d->bd_rcount; + slen = bpf_filter(d->bd_filter, (u_char *)bpf_pkt, + bpf_pkt->bpfp_total_length, 0); + if (slen != 0) { #if CONFIG_MACF_NET - if (mac_bpfdesc_check_receive(d, bp->bif_ifp) != 0) - continue; + if (mac_bpfdesc_check_receive(d, bp->bif_ifp) != 0) + continue; #endif - catchpacket(d, (u_char *)m, savedm, pktlen, - slen, outbound, bpf_mcopy); - } + catchpacket(d, bpf_pkt, slen, outbound); } } + + done: lck_mtx_unlock(bpf_mlock); } +static inline void +bpf_tap_mbuf( + ifnet_t ifp, + u_int32_t dlt, + mbuf_t m, + void* hdr, + size_t hlen, + int outbound) +{ + struct bpf_packet bpf_pkt; + struct mbuf *m0; + + if (ifp->if_bpf == NULL) { + /* quickly check without taking lock */ + return; + } + bpf_pkt.bpfp_type = BPF_PACKET_TYPE_MBUF; + bpf_pkt.bpfp_mbuf = m; + bpf_pkt.bpfp_total_length = 0; + for (m0 = m; m0 != NULL; m0 = m0->m_next) + bpf_pkt.bpfp_total_length += m0->m_len; + bpf_pkt.bpfp_header = hdr; + if (hdr != NULL) { + bpf_pkt.bpfp_total_length += hlen; + bpf_pkt.bpfp_header_length = hlen; + } else { + bpf_pkt.bpfp_header_length = 0; + } + bpf_tap_imp(ifp, dlt, &bpf_pkt, outbound); +} + void bpf_tap_out( ifnet_t ifp, @@ -2487,7 +2504,7 @@ bpf_tap_out( void* hdr, size_t hlen) { - bpf_tap_imp(ifp, dlt, m, hdr, hlen, 1); + bpf_tap_mbuf(ifp, dlt, m, hdr, hlen, 1); } void @@ -2498,29 +2515,50 @@ bpf_tap_in( void* hdr, size_t hlen) { - bpf_tap_imp(ifp, dlt, m, hdr, hlen, 0); + bpf_tap_mbuf(ifp, dlt, m, hdr, hlen, 0); } /* Callback registered with Ethernet driver. */ static int bpf_tap_callback(struct ifnet *ifp, struct mbuf *m) { - bpf_tap_imp(ifp, 0, m, NULL, 0, mbuf_pkthdr_rcvif(m) == NULL); + bpf_tap_mbuf(ifp, 0, m, NULL, 0, mbuf_pkthdr_rcvif(m) == NULL); return 0; } + +static void +copy_bpf_packet(struct bpf_packet * pkt, void * dst, size_t len) +{ + /* copy the optional header */ + if (pkt->bpfp_header_length != 0) { + size_t count = min(len, pkt->bpfp_header_length); + bcopy(pkt->bpfp_header, dst, count); + len -= count; + dst += count; + } + if (len == 0) { + /* nothing past the header */ + return; + } + /* copy the packet */ + switch (pkt->bpfp_type) { + case BPF_PACKET_TYPE_MBUF: + bpf_mcopy(pkt->bpfp_mbuf, dst, len); + break; + default: + break; + } +} + /* * Move the packet data from interface memory (pkt) into the * store buffer. Return 1 if it's time to wakeup a listener (buffer full), - * otherwise 0. "copy" is the routine called to do the actual data - * transfer. bcopy is passed in to copy contiguous chunks, while - * bpf_mcopy is passed in to copy mbuf chains. In the latter case, - * pkt is really an mbuf. + * otherwise 0. */ static void -catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, - u_int snaplen, int outbound, - void (*cpfn)(const void *, void *, size_t)) +catchpacket(struct bpf_d *d, struct bpf_packet * pkt, + u_int snaplen, int outbound) { struct bpf_hdr *hp; struct bpf_hdr_ext *ehp; @@ -2529,8 +2567,6 @@ catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, int do_wakeup = 0; u_char *payload; struct timeval tv; - struct m_tag *mt = NULL; - struct bpf_mtag *bt = NULL; hdrlen = (d->bd_flags & BPF_EXTENDED_HDR) ? d->bd_bif->bif_exthdrlen : d->bd_bif->bif_hdrlen; @@ -2540,7 +2576,7 @@ catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, * much. Otherwise, transfer the whole packet (unless * we hit the buffer size limit). */ - totlen = hdrlen + min(snaplen, pktlen); + totlen = hdrlen + min(snaplen, pkt->bpfp_total_length); if (totlen > d->bd_bufsize) totlen = d->bd_bufsize; @@ -2596,26 +2632,27 @@ catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, */ microtime(&tv); if (d->bd_flags & BPF_EXTENDED_HDR) { + struct mbuf *m; + + m = (pkt->bpfp_type == BPF_PACKET_TYPE_MBUF) + ? pkt->bpfp_mbuf : NULL; ehp = (struct bpf_hdr_ext *)(void *)(d->bd_sbuf + curlen); memset(ehp, 0, sizeof(*ehp)); ehp->bh_tstamp.tv_sec = tv.tv_sec; ehp->bh_tstamp.tv_usec = tv.tv_usec; - ehp->bh_datalen = pktlen; + + ehp->bh_datalen = pkt->bpfp_total_length; ehp->bh_hdrlen = hdrlen; - ehp->bh_caplen = totlen - hdrlen; - mt = m_tag_locate(m, bpf_mtag_id, 0, NULL); - if (mt && mt->m_tag_len >= sizeof(*bt)) { - bt = (struct bpf_mtag *)(mt + 1); - ehp->bh_pid = bt->bt_pid; - strlcpy(ehp->bh_comm, bt->bt_comm, - sizeof(ehp->bh_comm)); - ehp->bh_svc = so_svc2tc(bt->bt_svc); - if (bt->bt_direction == BPF_MTAG_DIR_OUT) + caplen = ehp->bh_caplen = totlen - hdrlen; + if (m == NULL) { + if (outbound) { ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_OUT; - else + } else { ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_IN; - m_tag_delete(m, mt); + } } else if (outbound) { + ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_OUT; + /* only do lookups on non-raw INPCB */ if ((m->m_pkthdr.pkt_flags & (PKTF_FLOW_ID| PKTF_FLOW_LOCALSRC|PKTF_FLOW_RAWSOCK)) == @@ -2625,7 +2662,6 @@ catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, ehp->bh_proto = m->m_pkthdr.pkt_proto; } ehp->bh_svc = so_svc2tc(m->m_pkthdr.pkt_svc); - ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_OUT; if (m->m_pkthdr.pkt_flags & PKTF_TCP_REXMT) ehp->bh_pktflags |= BPF_PKTFLAGS_TCP_REXMT; if (m->m_pkthdr.pkt_flags & PKTF_START_SEQ) @@ -2641,21 +2677,19 @@ catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, } else ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_IN; payload = (u_char *)ehp + hdrlen; - caplen = ehp->bh_caplen; } else { hp = (struct bpf_hdr *)(void *)(d->bd_sbuf + curlen); hp->bh_tstamp.tv_sec = tv.tv_sec; hp->bh_tstamp.tv_usec = tv.tv_usec; - hp->bh_datalen = pktlen; + hp->bh_datalen = pkt->bpfp_total_length; hp->bh_hdrlen = hdrlen; - hp->bh_caplen = totlen - hdrlen; + caplen = hp->bh_caplen = totlen - hdrlen; payload = (u_char *)hp + hdrlen; - caplen = hp->bh_caplen; } /* * Copy the packet data into the store buffer and update its length. */ - (*cpfn)(pkt, payload, caplen); + copy_bpf_packet(pkt, payload, caplen); d->bd_slen = curlen + totlen; d->bd_scnt += 1; @@ -2744,10 +2778,13 @@ bpf_attach( bpf_send_func send, bpf_tap_func tap) { + struct bpf_if *bp; struct bpf_if *bp_new; - struct bpf_if *bp_temp; + struct bpf_if *bp_before_first = NULL; struct bpf_if *bp_first = NULL; - + struct bpf_if *bp_last = NULL; + boolean_t found; + bp_new = (struct bpf_if *) _MALLOC(sizeof(*bp_new), M_DEVBUF, M_WAIT | M_ZERO); if (bp_new == 0) @@ -2756,20 +2793,34 @@ bpf_attach( lck_mtx_lock(bpf_mlock); /* - * Check if this interface/dlt is already attached, record first - * attachment for this interface. + * Check if this interface/dlt is already attached. Remember the + * first and last attachment for this interface, as well as the + * element before the first attachment. */ - for (bp_temp = bpf_iflist; bp_temp && (bp_temp->bif_ifp != ifp || - bp_temp->bif_dlt != dlt); bp_temp = bp_temp->bif_next) { - if (bp_temp->bif_ifp == ifp && bp_first == NULL) - bp_first = bp_temp; + found = FALSE; + for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { + if (bp->bif_ifp != ifp) { + if (bp_first != NULL) { + /* no more elements for this interface */ + break; + } + bp_before_first = bp; + } else { + if (bp->bif_dlt == dlt) { + found = TRUE; + break; + } + if (bp_first == NULL) { + bp_first = bp; + } + bp_last = bp; + } } - - if (bp_temp != NULL) { + if (found) { + lck_mtx_unlock(bpf_mlock); printf("bpfattach - %s with dlt %d is already attached\n", if_name(ifp), dlt); FREE(bp_new, M_DEVBUF); - lck_mtx_unlock(bpf_mlock); return EEXIST; } @@ -2784,9 +2835,21 @@ bpf_attach( bpf_iflist = bp_new; } else { - /* Add this after the first entry for this interface */ - bp_new->bif_next = bp_first->bif_next; - bp_first->bif_next = bp_new; + if (ifnet_type(ifp) == IFT_ETHER && dlt == DLT_EN10MB) { + /* Make this the first entry for this interface */ + if (bp_before_first != NULL) { + /* point the previous to us */ + bp_before_first->bif_next = bp_new; + } else { + /* we're the new head */ + bpf_iflist = bp_new; + } + bp_new->bif_next = bp_first; + } else { + /* Add this after the last entry for this interface */ + bp_new->bif_next = bp_last->bif_next; + bp_last->bif_next = bp_new; + } } /* @@ -2825,8 +2888,7 @@ bpfdetach(struct ifnet *ifp) struct bpf_d *d; if (bpf_debug != 0) - printf("%s: %s\n", - __func__, if_name(ifp)); + printf("%s: %s\n", __func__, if_name(ifp)); lck_mtx_lock(bpf_mlock); @@ -2900,8 +2962,6 @@ bpf_init(__unused void *unused) for (i = 0 ; i < NBPFILTER; i++) bpf_make_dev_t(maj); - - VERIFY(mbuf_tag_id_find(BPF_CONTROL_NAME, &bpf_mtag_id) == 0); } #else cdevsw_add(&bpf_cdevsw); diff --git a/bsd/net/bpf.h b/bsd/net/bpf.h index edd79c7f7..ff4eb1bff 100644 --- a/bsd/net/bpf.h +++ b/bsd/net/bpf.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1203,8 +1203,12 @@ struct bpf_mtag { */ #define DLT_NFC_LLCP 245 +/* + * USB packets, beginning with a Darwin (macOS, etc.) USB header. + */ +#define DLT_USB_DARWIN 266 -#define DLT_MATCHING_MAX 245 /* highest value in the "matching" range */ +#define DLT_MATCHING_MAX 266 /* highest value in the "matching" range */ /* * The instruction encodings. @@ -1299,6 +1303,21 @@ struct bpf_dltlist { struct ifnet; struct mbuf; +#define BPF_PACKET_TYPE_MBUF 0 + +struct bpf_packet { + int bpfp_type; + void * bpfp_header; /* optional */ + size_t bpfp_header_length; + union { + struct mbuf *bpfpu_mbuf; + void * bpfpu_ptr; + } bpfp_u; +#define bpfp_mbuf bpfp_u.bpfpu_mbuf +#define bpfp_ptr bpfp_u.bpfpu_ptr + size_t bpfp_total_length; /* length including optional header */ +}; + extern int bpf_validate(const struct bpf_insn *, int); extern void bpfdetach(struct ifnet *); extern void bpfilterattach(int); @@ -1341,7 +1360,7 @@ typedef u_int32_t bpf_tap_mode; @param packet The packet to be sent. */ typedef errno_t (*bpf_send_func)(ifnet_t interface, u_int32_t data_link_type, - mbuf_t packet); + mbuf_t packet); /*! @typedef bpf_tap_func @@ -1359,7 +1378,7 @@ typedef errno_t (*bpf_send_func)(ifnet_t interface, u_int32_t data_link_type, @param direction The direction of the tap. */ typedef errno_t (*bpf_tap_func)(ifnet_t interface, u_int32_t data_link_type, - bpf_tap_mode direction); + bpf_tap_mode direction); /*! @function bpfattach @@ -1412,7 +1431,7 @@ extern void bpf_tap_in(ifnet_t interface, u_int32_t dlt, mbuf_t packet, /*! @function bpf_tap_out - @discussion Call this function when your interface trasmits a + @discussion Call this function when your interface transmits a packet. This function will check if any bpf devices need a a copy of the packet. @param interface The interface the packet was or will be transmitted on. diff --git a/bsd/net/bpf_filter.c b/bsd/net/bpf_filter.c index 362472a95..80e31cd06 100644 --- a/bsd/net/bpf_filter.c +++ b/bsd/net/bpf_filter.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2011 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -100,106 +100,189 @@ #endif #include #ifdef KERNEL -#define MINDEX(m, k) \ -{ \ - unsigned int len = m->m_len; \ - \ - while (k >= len) { \ - k -= len; \ - m = m->m_next; \ - if (m == 0) \ - return 0; \ - len = m->m_len; \ - } \ -} extern unsigned int bpf_maxbufsize; -static u_int16_t m_xhalf(struct mbuf *m, bpf_u_int32 k, int *err); -static u_int32_t m_xword(struct mbuf *m, bpf_u_int32 k, int *err); +static inline u_int32_t +get_word_from_buffers(u_char * cp, u_char * np, int num_from_cp) +{ + u_int32_t val; -static u_int32_t -m_xword(struct mbuf *m, bpf_u_int32 k, int *err) + switch (num_from_cp) { + case 1: + val = ((u_int32_t)cp[0] << 24) | + ((u_int32_t)np[0] << 16) | + ((u_int32_t)np[1] << 8) | + (u_int32_t)np[2]; + break; + + case 2: + val = ((u_int32_t)cp[0] << 24) | + ((u_int32_t)cp[1] << 16) | + ((u_int32_t)np[0] << 8) | + (u_int32_t)np[1]; + break; + default: + val = ((u_int32_t)cp[0] << 24) | + ((u_int32_t)cp[1] << 16) | + ((u_int32_t)cp[2] << 8) | + (u_int32_t)np[0]; + break; + } + return (val); +} + +static u_char * +m_hdr_offset(struct mbuf **m_p, void * hdr, size_t hdrlen, bpf_u_int32 * k_p, + size_t * len_p) { + u_char *cp; + bpf_u_int32 k = *k_p; size_t len; - u_char *cp, *np; - struct mbuf *m0; - - len = m->m_len; - while (k >= len) { - k -= len; - m = m->m_next; - if (m == 0) - goto bad; + + if (k >= hdrlen) { + struct mbuf *m = *m_p; + + /* there's no header or the offset we want is past the header */ + k -= hdrlen; len = m->m_len; + while (k >= len) { + k -= len; + m = m->m_next; + if (m == NULL) + return (NULL); + len = m->m_len; + } + cp = mtod(m, u_char *) + k; + + /* return next mbuf, in case it's needed */ + *m_p = m->m_next; + + /* update the offset */ + *k_p = k; + } else { + len = hdrlen; + cp = (u_char *)hdr + k; } - cp = mtod(m, u_char *) + k; + *len_p = len; + return (cp); +} + +static u_int32_t +m_xword(struct mbuf *m, void * hdr, size_t hdrlen, bpf_u_int32 k, int *err) +{ + size_t len; + u_char *cp, *np; + + cp = m_hdr_offset(&m, hdr, hdrlen, &k, &len); + if (cp == NULL) + goto bad; if (len - k >= 4) { *err = 0; return EXTRACT_LONG(cp); } - m0 = m->m_next; - if (m0 == 0 || m0->m_len + len - k < 4) + if (m == 0 || m->m_len + len - k < 4) goto bad; *err = 0; - np = mtod(m0, u_char *); - switch (len - k) { - - case 1: - return - ((u_int32_t)cp[0] << 24) | - ((u_int32_t)np[0] << 16) | - ((u_int32_t)np[1] << 8) | - (u_int32_t)np[2]; - - case 2: - return - ((u_int32_t)cp[0] << 24) | - ((u_int32_t)cp[1] << 16) | - ((u_int32_t)np[0] << 8) | - (u_int32_t)np[1]; + np = mtod(m, u_char *); + return get_word_from_buffers(cp, np, len - k); - default: - return - ((u_int32_t)cp[0] << 24) | - ((u_int32_t)cp[1] << 16) | - ((u_int32_t)cp[2] << 8) | - (u_int32_t)np[0]; - } bad: *err = 1; return 0; } static u_int16_t -m_xhalf(struct mbuf *m, bpf_u_int32 k, int *err) +m_xhalf(struct mbuf *m, void * hdr, size_t hdrlen, bpf_u_int32 k, int *err) { size_t len; u_char *cp; - struct mbuf *m0; - - len = m->m_len; - while (k >= len) { - k -= len; - m = m->m_next; - if (m == 0) - goto bad; - len = m->m_len; - } - cp = mtod(m, u_char *) + k; + + cp = m_hdr_offset(&m, hdr, hdrlen, &k, &len); + if (cp == NULL) + goto bad; if (len - k >= 2) { *err = 0; return EXTRACT_SHORT(cp); } - m0 = m->m_next; - if (m0 == 0) + if (m == 0) goto bad; *err = 0; - return (cp[0] << 8) | mtod(m0, u_char *)[0]; + return (cp[0] << 8) | mtod(m, u_char *)[0]; bad: *err = 1; return 0; } + +static u_int8_t +m_xbyte(struct mbuf *m, void * hdr, size_t hdrlen, bpf_u_int32 k, int *err) +{ + size_t len; + u_char *cp; + + cp = m_hdr_offset(&m, hdr, hdrlen, &k, &len); + if (cp == NULL) + goto bad; + *err = 0; + return (*cp); + bad: + *err = 1; + return 0; + +} + + +static u_int32_t +bp_xword(struct bpf_packet *bp, bpf_u_int32 k, int *err) +{ + void * hdr = bp->bpfp_header; + size_t hdrlen = bp->bpfp_header_length; + + switch (bp->bpfp_type) { + case BPF_PACKET_TYPE_MBUF: + return m_xword(bp->bpfp_mbuf, hdr, hdrlen, k, err); + default: + break; + } + *err = 1; + return 0; + +} + +static u_int16_t +bp_xhalf(struct bpf_packet *bp, bpf_u_int32 k, int *err) +{ + void * hdr = bp->bpfp_header; + size_t hdrlen = bp->bpfp_header_length; + + switch (bp->bpfp_type) { + case BPF_PACKET_TYPE_MBUF: + return m_xhalf(bp->bpfp_mbuf, hdr, hdrlen, k, err); + default: + break; + } + *err = 1; + return 0; + +} + +static u_int8_t +bp_xbyte(struct bpf_packet *bp, bpf_u_int32 k, int *err) +{ + void * hdr = bp->bpfp_header; + size_t hdrlen = bp->bpfp_header_length; + + switch (bp->bpfp_type) { + case BPF_PACKET_TYPE_MBUF: + return m_xbyte(bp->bpfp_mbuf, hdr, hdrlen, k, err); + default: + break; + } + *err = 1; + return 0; + +} + #endif /* @@ -213,6 +296,10 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) u_int32_t A = 0, X = 0; bpf_u_int32 k; int32_t mem[BPF_MEMWORDS]; +#ifdef KERNEL + int merr; + struct bpf_packet * bp = (struct bpf_packet *)(void *)p; +#endif /* KERNEL */ bzero(mem, sizeof(mem)); @@ -230,9 +317,9 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) default: #ifdef KERNEL return 0; -#else +#else /* KERNEL */ abort(); -#endif +#endif /* KERNEL */ case BPF_RET|BPF_K: return (u_int)pc->k; @@ -243,23 +330,21 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) k = pc->k; if (k > buflen || sizeof(int32_t) > buflen - k) { #ifdef KERNEL - int merr; - if (buflen != 0) return 0; - A = m_xword((struct mbuf *)(void *)p, k, &merr); + A = bp_xword(bp, k, &merr); if (merr != 0) return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } #if BPF_ALIGN if (((intptr_t)(p + k) & 3) != 0) A = EXTRACT_LONG(&p[k]); else -#endif +#endif /* BPF_ALIGN */ A = ntohl(*(int32_t *)(void *)(p + k)); continue; @@ -267,15 +352,15 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) k = pc->k; if (k > buflen || sizeof(int16_t) > buflen - k) { #ifdef KERNEL - int merr; - if (buflen != 0) return 0; - A = m_xhalf((struct mbuf *)(void *)p, k, &merr); + A = bp_xhalf(bp, k, &merr); + if (merr != 0) + return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } A = EXTRACT_SHORT(&p[k]); continue; @@ -284,17 +369,15 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) k = pc->k; if (k >= buflen) { #ifdef KERNEL - struct mbuf *m; - if (buflen != 0) return 0; - m = (struct mbuf *)(void *)p; - MINDEX(m, k); - A = mtod(m, u_char *)[k]; + A = bp_xbyte(bp, k, &merr); + if (merr != 0) + return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } A = p[k]; continue; @@ -312,23 +395,21 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) if (pc->k > buflen || X > buflen - pc->k || sizeof(int32_t) > buflen - k) { #ifdef KERNEL - int merr; - if (buflen != 0) return 0; - A = m_xword((struct mbuf *)(void *)p, k, &merr); + A = bp_xword(bp, k, &merr); if (merr != 0) return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } #if BPF_ALIGN if (((intptr_t)(p + k) & 3) != 0) A = EXTRACT_LONG(&p[k]); else -#endif +#endif /* BPF_ALIGN */ A = ntohl(*(int32_t *)(void *)(p + k)); continue; @@ -337,17 +418,15 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) if (X > buflen || pc->k > buflen - X || sizeof(int16_t) > buflen - k) { #ifdef KERNEL - int merr; - if (buflen != 0) return 0; - A = m_xhalf((struct mbuf *)(void *)p, k, &merr); + A = bp_xhalf(bp, k, &merr); if (merr != 0) return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } A = EXTRACT_SHORT(&p[k]); continue; @@ -356,17 +435,15 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) k = X + pc->k; if (pc->k >= buflen || X >= buflen - pc->k) { #ifdef KERNEL - struct mbuf *m; - if (buflen != 0) return 0; - m = (struct mbuf *)(void *)p; - MINDEX(m, k); - A = mtod(m, u_char *)[k]; + A = bp_xbyte(bp, k, &merr); + if (merr != 0) + return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } A = p[k]; continue; @@ -375,13 +452,12 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) k = pc->k; if (k >= buflen) { #ifdef KERNEL - struct mbuf *m; - if (buflen != 0) return 0; - m = (struct mbuf *)(void *)p; - MINDEX(m, k); - X = (mtod(m, u_char *)[k] & 0xf) << 2; + X = bp_xbyte(bp, k, &merr); + if (merr != 0) + return 0; + X = (X & 0xf) << 2; continue; #else return 0; diff --git a/bsd/net/bridgestp.c b/bsd/net/bridgestp.c index 10d86fa4a..972a1ae14 100644 --- a/bsd/net/bridgestp.c +++ b/bsd/net/bridgestp.c @@ -121,7 +121,7 @@ static void bstp_task_drain(struct bstp_task *); #define BSTP_LOCK_DESTROY(_bs) lck_mtx_free((_bs)->bs_mtx, bstp_lock_grp) #define BSTP_LOCK(_bs) lck_mtx_lock((_bs)->bs_mtx) #define BSTP_UNLOCK(_bs) lck_mtx_unlock((_bs)->bs_mtx) -#define BSTP_LOCK_ASSERT(_bs) lck_mtx_assert((_bs)->bs_mtx, LCK_MTX_ASSERT_OWNED) +#define BSTP_LOCK_ASSERT(_bs) LCK_MTX_ASSERT((_bs)->bs_mtx, LCK_MTX_ASSERT_OWNED) #ifdef BRIDGESTP_DEBUG diff --git a/bsd/net/classq/classq.c b/bsd/net/classq/classq.c index 67c1f44ea..35d86188f 100644 --- a/bsd/net/classq/classq.c +++ b/bsd/net/classq/classq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -73,6 +73,7 @@ #include + u_int32_t classq_verbose = 0; /* more noise if greater than 1 */ SYSCTL_NODE(_net, OID_AUTO, classq, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "classq"); @@ -81,44 +82,100 @@ SYSCTL_UINT(_net_classq, OID_AUTO, verbose, CTLFLAG_RW|CTLFLAG_LOCKED, &classq_verbose, 0, "Class queue verbosity level"); void -_qinit(class_queue_t *q, int type, int lim) +_qinit(class_queue_t *q, int type, int lim, classq_pkt_type_t ptype) { - MBUFQ_INIT(&q->mbufq); + switch (ptype) { + case QP_MBUF: + MBUFQ_INIT(&qmbufq(q)); + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } + qlimit(q) = lim; qlen(q) = 0; qsize(q) = 0; qtype(q) = type; + qptype(q) = ptype; qstate(q) = QS_RUNNING; } /* add a packet at the tail of the queue */ void -_addq(class_queue_t *q, struct mbuf *m) +_addq(class_queue_t *q, void *pkt) { - MBUFQ_ENQUEUE(&q->mbufq, m); + uint32_t size = 0; + + switch (qptype(q)) { + case QP_MBUF: { + struct mbuf *m = pkt; + MBUFQ_ENQUEUE(&qmbufq(q), m); + size = m_length(m); + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } + qlen(q)++; VERIFY(qlen(q) != 0); - qsize(q) += m_length(m); + qsize(q) += size; } /* add one or more packets at the tail of the queue */ void -_addq_multi(class_queue_t *q, struct mbuf *m_head, struct mbuf *m_tail, +_addq_multi(class_queue_t *q, void *pkt_head, void *pkt_tail, u_int32_t cnt, u_int32_t size) { - MBUFQ_ENQUEUE_MULTI(&q->mbufq, m_head, m_tail); + switch (qptype(q)) { + case QP_MBUF: { + struct mbuf *m_head = pkt_head; + struct mbuf *m_tail = pkt_tail; + MBUFQ_ENQUEUE_MULTI(&qmbufq(q), m_head, m_tail); + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } + qlen(q) += cnt; qsize(q) += size; } /* get a packet at the head of the queue */ -struct mbuf * +void * _getq(class_queue_t *q) { - struct mbuf *m; + void *pkt = NULL; + uint32_t pkt_len; + + switch (qptype(q)) { + case QP_MBUF: { + struct mbuf *m; + MBUFQ_DEQUEUE(&qmbufq(q), m); + if (m != NULL) { + pkt_len = m_length(m); + pkt = m; + } + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } - MBUFQ_DEQUEUE(&q->mbufq, m); - if (m == NULL) { + if (pkt == NULL) { VERIFY(qlen(q) == 0); if (qsize(q) > 0) qsize(q) = 0; @@ -128,73 +185,97 @@ _getq(class_queue_t *q) qlen(q)--; /* qsize is an approximation, so adjust if necessary */ - if (((int)qsize(q) - m_length(m)) > 0) - qsize(q) -= m_length(m); + if (((int)qsize(q) - pkt_len) > 0) + qsize(q) -= pkt_len; else if (qsize(q) != 0) qsize(q) = 0; - return (m); + return (pkt); } -static struct mbuf * +static void * _getq_flow_or_scidx(class_queue_t *q, u_int32_t val, boolean_t isflowid) { - struct mbuf *m, *m_tmp; - - MBUFQ_FOREACH_SAFE(m, &q->mbufq, m_tmp) { - if ((isflowid && (val == 0 || ((m->m_flags & M_PKTHDR) && - m->m_pkthdr.pkt_flowid == val))) || - (!isflowid && - MBUF_SCIDX(mbuf_get_service_class(m)) < val)) { - /* remove it from the class queue */ - MBUFQ_REMOVE(&q->mbufq, m); - MBUFQ_NEXT(m) = NULL; - break; + void *pkt = NULL; + uint32_t pkt_len; + + switch (qptype(q)) { + case QP_MBUF: { + struct mbuf *m, *m_tmp; + + MBUFQ_FOREACH_SAFE(m, &qmbufq(q), m_tmp) { + if ((isflowid && (val == 0 || + ((m->m_flags & M_PKTHDR) && + m->m_pkthdr.pkt_flowid == val))) || + (!isflowid && + MBUF_SCIDX(mbuf_get_service_class(m)) < val)) { + /* remove it from the class queue */ + MBUFQ_REMOVE(&qmbufq(q), m); + MBUFQ_NEXT(m) = NULL; + break; + } + } + if (m != NULL) { + pkt = m; + pkt_len = m_length(m); } + break; } - if (m != NULL) { - u_int32_t l = m_length(m); + default: + VERIFY(0); + /* NOTREACHED */ + } + + if (pkt != NULL) { VERIFY(qlen(q) > 0); qlen(q)--; /* qsize is an approximation, so adjust if necessary */ - if (((int)qsize(q) - l) > 0) - qsize(q) -= l; + if (((int)qsize(q) - pkt_len) > 0) + qsize(q) -= pkt_len; else if (qsize(q) != 0) qsize(q) = 0; } - return (m); - + return (pkt); } /* get a packet of a specific flow beginning from the head of the queue */ -struct mbuf * +void * _getq_flow(class_queue_t *q, u_int32_t flow) { return (_getq_flow_or_scidx(q, flow, TRUE)); } /* Get a packet whose MBUF_SCIDX() < scidx from head of queue */ -struct mbuf * +void * _getq_scidx_lt(class_queue_t *q, u_int32_t scidx) { return (_getq_flow_or_scidx(q, scidx, FALSE)); } -/* get all packets starting from the head of the queue */ -struct mbuf * -_getq_all(class_queue_t *q, struct mbuf **last, u_int32_t *qlenp, +/* get all packets (chained) starting from the head of the queue */ +void * +_getq_all(class_queue_t *q, void **last, u_int32_t *qlenp, u_int64_t *qsizep) { - struct mbuf *m; + void *pkt = NULL; + + switch (qptype(q)) { + case QP_MBUF: + pkt = MBUFQ_FIRST(&qmbufq(q)); + if (last != NULL) + *last = MBUFQ_LAST(&qmbufq(q)); + MBUFQ_INIT(&qmbufq(q)); + break; - m = MBUFQ_FIRST(&q->mbufq); - if (last != NULL) - *last = MBUFQ_LAST(&q->mbufq); - MBUFQ_INIT(&q->mbufq); + + default: + VERIFY(0); + /* NOTREACHED */ + } if (qlenp != NULL) *qlenp = qlen(q); @@ -204,14 +285,13 @@ _getq_all(class_queue_t *q, struct mbuf **last, u_int32_t *qlenp, qlen(q) = 0; qsize(q) = 0; - return (m); + return (pkt); } -/* drop a packet at the tail of the queue */ -struct mbuf * -_getq_tail(class_queue_t *q) +static inline struct mbuf * +_getq_tail_mbuf(class_queue_t *q) { - struct mq_head *head = &q->mbufq; + struct mq_head *head = &qmbufq(q); struct mbuf *m = MBUFQ_LAST(head); if (m != NULL) { @@ -247,15 +327,36 @@ _getq_tail(class_queue_t *q) return (m); } -/* randomly select a packet in the queue */ -struct mbuf * -_getq_random(class_queue_t *q) +/* drop a packet at the tail of the queue */ +void * +_getq_tail(class_queue_t *q) +{ + void *t = NULL; + + switch (qptype(q)) { + case QP_MBUF: + t = _getq_tail_mbuf(q); + break; + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (t); +} + +static inline struct mbuf * +_getq_random_mbuf(class_queue_t *q) { - struct mq_head *head = &q->mbufq; + struct mq_head *head = &qmbufq(q); struct mbuf *m = NULL; unsigned int n; u_int32_t rnd; + /* XXX: Add support for Kernel packet when needed */ + VERIFY((qptype(q) == QP_MBUF)); + n = qlen(q); if (n == 0) { VERIFY(MBUFQ_EMPTY(head)); @@ -265,7 +366,7 @@ _getq_random(class_queue_t *q) } m = MBUFQ_FIRST(head); - read_random(&rnd, sizeof (rnd)); + read_frandom(&rnd, sizeof (rnd)); n = (rnd % n) + 1; if (n == 1) { @@ -301,11 +402,29 @@ _getq_random(class_queue_t *q) return (m); } -/* remove a packet from the queue */ -void -_removeq(class_queue_t *q, struct mbuf *m) +/* randomly select a packet in the queue */ +void * +_getq_random(class_queue_t *q) +{ + void *r = NULL; + + switch (qptype(q)) { + case QP_MBUF: + r = _getq_random_mbuf(q); + break; + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (r); +} + +static inline void +_removeq_mbuf(class_queue_t *q, struct mbuf *m) { - struct mq_head *head = &q->mbufq; + struct mq_head *head = &qmbufq(q); struct mbuf *m0, **mtail; m0 = MBUFQ_FIRST(head); @@ -339,14 +458,30 @@ _removeq(class_queue_t *q, struct mbuf *m) MBUFQ_NEXT(m) = NULL; } +/* remove a packet from the queue */ +void +_removeq(class_queue_t *q, void *pkt) +{ + switch (qptype(q)) { + case QP_MBUF: + _removeq_mbuf(q, pkt); + break; + + default: + VERIFY(0); + /* NOTREACHED */ + } +} + void _flushq(class_queue_t *q) { (void) _flushq_flow(q, 0, NULL, NULL); } -void -_flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len) +static inline void +_flushq_flow_mbuf(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, + u_int32_t *len) { MBUFQ_HEAD(mq_freeq) freeq; struct mbuf *m, *m_tmp; @@ -354,11 +489,11 @@ _flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len) MBUFQ_INIT(&freeq); - MBUFQ_FOREACH_SAFE(m, &q->mbufq, m_tmp) { + MBUFQ_FOREACH_SAFE(m, &qmbufq(q), m_tmp) { if (flow == 0 || ((m->m_flags & M_PKTHDR) && m->m_pkthdr.pkt_flowid == flow)) { /* remove it from the class queue */ - MBUFQ_REMOVE(&q->mbufq, m); + MBUFQ_REMOVE(&qmbufq(q), m); MBUFQ_NEXT(m) = NULL; /* and add it to the free queue */ @@ -389,3 +524,17 @@ _flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len) if (len != NULL) *len = l; } + +void +_flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len) +{ + switch (qptype(q)) { + case QP_MBUF: + _flushq_flow_mbuf(q, flow, cnt, len); + break; + + default: + VERIFY(0); + /* NOTREACHED */ + } +} diff --git a/bsd/net/classq/classq.h b/bsd/net/classq/classq.h index 750ce5452..f36f9d727 100644 --- a/bsd/net/classq/classq.h +++ b/bsd/net/classq/classq.h @@ -72,15 +72,20 @@ extern "C" { #endif +/* + * Packet types + */ +typedef enum classq_pkt_type { + QP_INVALID = 0, + QP_MBUF, /* mbuf packet */ +} classq_pkt_type_t; + /* * Packet Queue types */ typedef enum classq_type { Q_DROPHEAD, Q_DROPTAIL, - Q_RED, - Q_RIO, - Q_BLUE, Q_SFB } classq_type_t; @@ -114,21 +119,26 @@ struct pktcntr { * Packet Queue structures and macros to manipulate them. */ typedef struct _class_queue_ { - MBUFQ_HEAD(mq_head) mbufq; /* Packet queue */ + union { + MBUFQ_HEAD(mq_head) __mbufq; /* mbuf packet queue */ + } __pktq_u; u_int32_t qlen; /* Queue length (in number of packets) */ u_int32_t qlim; /* Queue limit (in number of packets*) */ u_int64_t qsize; /* Approx. queue size (in number of bytes) */ classq_type_t qtype; /* Queue type */ classq_state_t qstate; /* Queue state */ + classq_pkt_type_t qptype; /* Packet type */ } class_queue_t; +#define qmbufq(q) (q)->__pktq_u.__mbufq /* Get mbuf packet queue */ +#define qptype(q) (q)->qptype /* Get queue packet type */ #define qtype(q) (q)->qtype /* Get queue type */ #define qstate(q) (q)->qstate /* Get queue state */ #define qlimit(q) (q)->qlim /* Max packets to be queued */ #define qlen(q) (q)->qlen /* Current queue length. */ #define qsize(q) (q)->qsize /* Approx. bytes in queue */ -/* #define qtail(q) MBUFQ_LAST(&(q)->mbufq) */ -#define qhead(q) MBUFQ_FIRST(&(q)->mbufq) + +#define qhead(q) MBUFQ_FIRST(&qmbufq(q)) #define qempty(q) (qlen(q) == 0) /* Is the queue empty?? */ #define q_is_red(q) (qtype(q) == Q_RED) /* Is the queue a RED queue */ @@ -157,18 +167,16 @@ extern u_int32_t classq_verbose; SYSCTL_DECL(_net_classq); -extern void _qinit(class_queue_t *, int, int); -extern void _addq(class_queue_t *, struct mbuf *); -extern void _addq_multi(class_queue_t *, struct mbuf *, struct mbuf *, - u_int32_t, u_int32_t); -extern struct mbuf *_getq(class_queue_t *); -extern struct mbuf *_getq_all(class_queue_t *, struct mbuf **, - u_int32_t *, u_int64_t *); -extern struct mbuf *_getq_tail(class_queue_t *); -extern struct mbuf *_getq_random(class_queue_t *); -extern struct mbuf *_getq_flow(class_queue_t *, u_int32_t); -extern struct mbuf *_getq_scidx_lt(class_queue_t *, u_int32_t); -extern void _removeq(class_queue_t *, struct mbuf *); +extern void _qinit(class_queue_t *, int, int, classq_pkt_type_t); +extern void _addq(class_queue_t *, void *); +extern void _addq_multi(class_queue_t *, void *, void *, u_int32_t, u_int32_t); +extern void *_getq(class_queue_t *); +extern void *_getq_all(class_queue_t *, void **, u_int32_t *, u_int64_t *); +extern void *_getq_tail(class_queue_t *); +extern void *_getq_random(class_queue_t *); +extern void *_getq_flow(class_queue_t *, u_int32_t); +extern void *_getq_scidx_lt(class_queue_t *, u_int32_t); +extern void _removeq(class_queue_t *, void *); extern void _flushq(class_queue_t *); extern void _flushq_flow(class_queue_t *, u_int32_t, u_int32_t *, u_int32_t *); diff --git a/bsd/net/classq/classq_blue.c b/bsd/net/classq/classq_blue.c deleted file mode 100644 index fdf21b069..000000000 --- a/bsd/net/classq/classq_blue.c +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Copyright (c) 2007-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $NetBSD: altq_blue.c,v 1.21 2006/11/16 01:32:37 christos Exp $ */ -/* $KAME: altq_blue.c,v 1.15 2005/04/13 03:44:24 suz Exp $ */ - -/* - * Copyright (C) 1997-2002 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ -/* - * Copyright (c) 1990-1994 Regents of the University of California. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Computer Systems - * Engineering Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include - -#if CLASSQ_BLUE - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include -#include -#include -#if INET6 -#include -#endif - -#include -#include -#include - -/* - * Blue is proposed and implemented by Wu-chang Feng . - * more information on Blue is available from - * http://www.eecs.umich.edu/~wuchang/blue/ - */ - -#define BLUE_LIMIT 200 /* default max queue lenght */ - -#define BLUE_ZONE_MAX 32 /* maximum elements in zone */ -#define BLUE_ZONE_NAME "classq_blue" /* zone name */ - -static unsigned int blue_size; /* size of zone element */ -static struct zone *blue_zone; /* zone for blue */ - -/* internal function prototypes */ -static struct mbuf *blue_getq_flow(struct blue *, class_queue_t *, - u_int32_t, boolean_t); -static int blue_drop_early(struct blue *); - -void -blue_init(void) -{ - _CASSERT(BLUEF_ECN4 == CLASSQF_ECN4); - _CASSERT(BLUEF_ECN6 == CLASSQF_ECN6); - - blue_size = sizeof (struct blue); - blue_zone = zinit(blue_size, BLUE_ZONE_MAX * blue_size, - 0, BLUE_ZONE_NAME); - if (blue_zone == NULL) { - panic("%s: failed allocating %s", __func__, BLUE_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(blue_zone, Z_EXPAND, TRUE); - zone_change(blue_zone, Z_CALLERACCT, TRUE); -} - -/* - * blue support routines - */ -struct blue * -blue_alloc(struct ifnet *ifp, u_int32_t max_pmark, u_int32_t hold_time, - u_int32_t flags) -{ - struct blue *bp; - - VERIFY(ifp != NULL); - - bp = zalloc(blue_zone); - if (bp == NULL) - return (NULL); - - bzero(bp, blue_size); - bp->blue_idle = 1; - bp->blue_ifp = ifp; - bp->blue_flags = (flags & BLUEF_USERFLAGS); -#if !PF_ECN - if (bp->blue_flags & BLUEF_ECN) { - bp->blue_flags &= ~BLUEF_ECN; - log(LOG_ERR, "%s: BLUE ECN not available; ignoring " - "BLUEF_ECN flag!\n", if_name(ifp)); - } -#endif /* !PF_ECN */ - - - if (max_pmark == 0) - bp->blue_max_pmark = 1000; - else - bp->blue_max_pmark = max_pmark; - - if (hold_time == 0) - bp->blue_hold_time = 50000; - else - bp->blue_hold_time = hold_time; - - microuptime(&bp->blue_last); - - return (bp); -} - -void -blue_destroy(struct blue *bp) -{ - zfree(blue_zone, bp); -} - -void -blue_getstats(struct blue *bp, struct blue_stats *sp) -{ - sp->q_pmark = bp->blue_pmark; - sp->drop_forced = bp->blue_stats.drop_forced; - sp->drop_unforced = bp->blue_stats.drop_unforced; - sp->marked_packets = bp->blue_stats.marked_packets; -} - -#define DTYPE_NODROP 0 /* no drop */ -#define DTYPE_FORCED 1 /* a "forced" drop */ -#define DTYPE_EARLY 2 /* an "unforced" (early) drop */ - -int -blue_addq(struct blue *bp, class_queue_t *q, struct mbuf *m, - struct pf_mtag *tag) -{ -#if !PF_ECN -#pragma unused(tag) -#endif /* !PF_ECN */ - int droptype; - - /* - * if we were idle, this is an enqueue onto an empty queue - * and we should decrement marking probability - */ - if (bp->blue_idle) { - struct timeval now; - u_int32_t t; - - bp->blue_idle = 0; - microuptime(&now); - t = (now.tv_sec - bp->blue_last.tv_sec); - if (t > 1) { - bp->blue_pmark = 1; - microuptime(&bp->blue_last); - } else { - t = t * 1000000 + (now.tv_usec - bp->blue_last.tv_usec); - if (t > bp->blue_hold_time) { - bp->blue_pmark--; - if (bp->blue_pmark < 0) - bp->blue_pmark = 0; - microuptime(&bp->blue_last); - } - } - } - - /* see if we drop early */ - droptype = DTYPE_NODROP; - if (blue_drop_early(bp) && qlen(q) > 1) { - /* mark or drop by blue */ -#if PF_ECN - if ((bp->blue_flags & BLUEF_ECN) && - (tag->pftag_proto == IPPROTO_TCP) && /* only for TCP */ - mark_ecn(m, tag, bp->blue_flags)) { - /* successfully marked. do not drop. */ - bp->blue_stats.marked_packets++; - } else -#endif /* PF_ECN */ - { - /* unforced drop by blue */ - droptype = DTYPE_EARLY; - } - } - - /* if the queue length hits the hard limit, it's a forced drop */ - if (droptype == DTYPE_NODROP && qlen(q) >= qlimit(q)) - droptype = DTYPE_FORCED; - - /* if successful or forced drop, enqueue this packet. */ - if (droptype != DTYPE_EARLY) - _addq(q, m); - - if (droptype != DTYPE_NODROP) { - if (droptype == DTYPE_EARLY) { - /* drop the incoming packet */ - bp->blue_stats.drop_unforced++; - } else { - struct timeval now; - u_int32_t t; - /* forced drop, select a victim packet in the queue. */ - m = _getq_random(q); - microuptime(&now); - t = (now.tv_sec - bp->blue_last.tv_sec); - t = t * 1000000 + (now.tv_usec - bp->blue_last.tv_usec); - if (t > bp->blue_hold_time) { - bp->blue_pmark += bp->blue_max_pmark >> 3; - if (bp->blue_pmark > bp->blue_max_pmark) - bp->blue_pmark = bp->blue_max_pmark; - microuptime(&bp->blue_last); - } - bp->blue_stats.drop_forced++; - } - IFCQ_CONVERT_LOCK(&bp->blue_ifp->if_snd); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - /* successfully queued */ - return (CLASSQEQ_SUCCESS); -} - -static struct mbuf * -blue_getq_flow(struct blue *bp, class_queue_t *q, u_int32_t flow, - boolean_t purge) -{ -#pragma unused(purge) - struct mbuf *m; - - /* flow of 0 means head of queue */ - if ((m = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) { - if (bp->blue_idle == 0) { - bp->blue_idle = 1; - microuptime(&bp->blue_last); - } - return (NULL); - } - - bp->blue_idle = 0; - return (m); -} - -struct mbuf * -blue_getq(struct blue *bp, class_queue_t *q) -{ - return (blue_getq_flow(bp, q, 0, FALSE)); -} - -void -blue_purgeq(struct blue *bp, class_queue_t *q, u_int32_t flow, - u_int32_t *packets, u_int32_t *bytes) -{ - u_int32_t cnt = 0, len = 0; - struct mbuf *m; - - IFCQ_CONVERT_LOCK(&bp->blue_ifp->if_snd); - - while ((m = blue_getq_flow(bp, q, flow, TRUE)) != NULL) { - cnt++; - len += m_pktlen(m); - m_freem(m); - } - - if (packets != NULL) - *packets = cnt; - if (bytes != NULL) - *bytes = len; -} - -/* - * early-drop probability is kept in blue_pmark - */ -static int -blue_drop_early(struct blue *bp) -{ - if ((RandomULong() % (unsigned)bp->blue_max_pmark) < - (unsigned)bp->blue_pmark) { - /* drop or mark */ - return (1); - } - /* no drop/mark */ - return (0); -} - -void -blue_updateq(struct blue *bp, cqev_t ev) -{ -#pragma unused(bp, ev) - /* nothing for now */ -} - -int -blue_suspendq(struct blue *bp, class_queue_t *q, boolean_t on) -{ -#pragma unused(bp, q, on) - return (ENOTSUP); -} -#endif /* CLASSQ_BLUE */ diff --git a/bsd/net/classq/classq_blue.h b/bsd/net/classq/classq_blue.h index e6c546e23..777122b94 100644 --- a/bsd/net/classq/classq_blue.h +++ b/bsd/net/classq/classq_blue.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -75,51 +75,6 @@ struct blue_stats { u_int64_t marked_packets; }; -#ifdef BSD_KERNEL_PRIVATE -/* blue flags */ -#define BLUEF_ECN4 0x01 /* use packet marking for IPv4 packets */ -#define BLUEF_ECN6 0x02 /* use packet marking for IPv6 packets */ -#define BLUEF_ECN (BLUEF_ECN4 | BLUEF_ECN6) - -#define BLUEF_USERFLAGS \ - (BLUEF_ECN4 | BLUEF_ECN6) - -typedef struct blue { - u_int32_t blue_flags; /* blue flags */ - - /* blue parameters */ - int32_t blue_pmark; /* 0-1000 (mark probability*10000) */ - int32_t blue_max_pmark; /* sets precision of marking probability */ - u_int32_t blue_hold_time; /* hold time in usec */ - struct ifnet *blue_ifp; /* back pointer to ifnet */ - - /* variables for internal use */ - u_int32_t blue_idle; /* queue was empty */ - struct timeval blue_last; /* timestamp when the queue becomes idle */ - - /* statistics */ - struct { - struct pktcntr xmit_cnt; - struct pktcntr drop_cnt; - u_int64_t drop_forced; - u_int64_t drop_unforced; - u_int64_t marked_packets; - } blue_stats; -} blue_t; - -extern void blue_init(void); -extern struct blue *blue_alloc(struct ifnet *, u_int32_t, u_int32_t, u_int32_t); -extern void blue_destroy(struct blue *); -extern int blue_addq(struct blue *, class_queue_t *, struct mbuf *, - struct pf_mtag *); -extern struct mbuf *blue_getq(struct blue *, class_queue_t *); -extern void blue_purgeq(struct blue *, class_queue_t *, u_int32_t, - u_int32_t *, u_int32_t *); -extern void blue_getstats(struct blue *, struct blue_stats *); -extern void blue_updateq(struct blue *, cqev_t); -extern int blue_suspendq(struct blue *, class_queue_t *, boolean_t); -#endif /* BSD_KERNEL_PRIVATE */ - #ifdef __cplusplus } #endif diff --git a/bsd/net/classq/classq_fq_codel.c b/bsd/net/classq/classq_fq_codel.c index f78da89d6..3710ad112 100644 --- a/bsd/net/classq/classq_fq_codel.c +++ b/bsd/net/classq/classq_fq_codel.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Apple Inc. All rights reserved. + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -41,15 +41,19 @@ #include #include +#include +#include +#include #include #include static struct zone *flowq_zone = NULL; -static size_t flowq_size; #define FQ_ZONE_MAX (32 * 1024) /* across all interfaces */ -#define FQ_SEQ_LT(a,b) ((int)((a)-(b)) < 0) -#define FQ_SEQ_GT(a,b) ((int)((a)-(b)) > 0) + +#define DTYPE_NODROP 0 /* no drop */ +#define DTYPE_FORCED 1 /* a "forced" drop */ +#define DTYPE_EARLY 2 /* an "unforced" (early) drop */ void fq_codel_init(void) @@ -57,9 +61,8 @@ fq_codel_init(void) if (flowq_zone != NULL) return; - flowq_size = sizeof (fq_t); - flowq_zone = zinit(flowq_size, FQ_ZONE_MAX * flowq_size, - 0, "flowq_zone"); + flowq_zone = zinit(sizeof (struct flowq), + FQ_ZONE_MAX * sizeof (struct flowq), 0, "flowq_zone"); if (flowq_zone == NULL) { panic("%s: failed to allocate flowq_zone", __func__); /* NOTREACHED */ @@ -69,27 +72,29 @@ fq_codel_init(void) } fq_t * -fq_alloc(int how) +fq_alloc(classq_pkt_type_t ptype) { fq_t *fq = NULL; - fq = (how == M_WAITOK) ? zalloc(flowq_zone) : - zalloc_noblock(flowq_zone); + fq = zalloc(flowq_zone); if (fq == NULL) { log(LOG_ERR, "%s: unable to allocate from flowq_zone\n"); return (NULL); } - bzero(fq, flowq_size); - MBUFQ_INIT(&fq->fq_mbufq); + bzero(fq, sizeof (*fq)); + fq->fq_ptype = ptype; + if (ptype == QP_MBUF) { + MBUFQ_INIT(&fq->fq_mbufq); + } return (fq); } void fq_destroy(fq_t *fq) { - VERIFY(MBUFQ_EMPTY(&fq->fq_mbufq)); + VERIFY(fq_empty(fq)); VERIFY(!(fq->fq_flags & (FQF_NEW_FLOW | FQF_OLD_FLOW))); - bzero(fq, flowq_size); + VERIFY(fq->fq_bytes == 0); zfree(flowq_zone, fq); } @@ -99,7 +104,7 @@ fq_detect_dequeue_stall(fq_if_t *fqs, fq_t *flowq, fq_if_classq_t *fq_cl, { u_int64_t maxgetqtime; if (FQ_IS_DELAYHIGH(flowq) || flowq->fq_getqtime == 0 || - MBUFQ_EMPTY(&flowq->fq_mbufq) || + fq_empty(flowq) || flowq->fq_bytes < FQ_MIN_FC_THRESHOLD_BYTES) return; maxgetqtime = flowq->fq_getqtime + fqs->fqs_update_interval; @@ -116,61 +121,78 @@ fq_detect_dequeue_stall(fq_if_t *fqs, fq_t *flowq, fq_if_classq_t *fq_cl, void fq_head_drop(fq_if_t *fqs, fq_t *fq) { - struct mbuf *m = NULL; + pktsched_pkt_t pkt; + uint32_t *pkt_flags; + uint64_t *pkt_timestamp; struct ifclassq *ifq = fqs->fqs_ifq; - m = fq_getq_flow(fqs, fq); - if (m == NULL) + _PKTSCHED_PKT_INIT(&pkt); + if (fq_getq_flow_internal(fqs, fq, &pkt) == NULL) return; - IFCQ_DROP_ADD(ifq, 1, m_length(m)); + pktsched_get_pkt_vars(&pkt, &pkt_flags, &pkt_timestamp, NULL, NULL, + NULL, NULL); + + *pkt_timestamp = 0; + if (pkt.pktsched_ptype == QP_MBUF) + *pkt_flags &= ~PKTF_PRIV_GUARDED; + + IFCQ_DROP_ADD(ifq, 1, pktsched_get_pkt_len(&pkt)); IFCQ_CONVERT_LOCK(ifq); - m_freem(m); + pktsched_free_pkt(&pkt); } int -fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) +fq_addq(fq_if_t *fqs, pktsched_pkt_t *pkt, fq_if_classq_t *fq_cl) { - struct pkthdr *pkt = &m->m_pkthdr; int droptype = DTYPE_NODROP, fc_adv = 0, ret = CLASSQEQ_SUCCESS; u_int64_t now; fq_t *fq = NULL; + uint64_t *pkt_timestamp; + uint32_t *pkt_flags; + uint32_t pkt_flowid, pkt_tx_start_seq; + uint8_t pkt_proto, pkt_flowsrc; + + pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, &pkt_flowid, + &pkt_flowsrc, &pkt_proto, &pkt_tx_start_seq); + + if (pkt->pktsched_ptype == QP_MBUF) { + /* See comments in */ + VERIFY(!(*pkt_flags & PKTF_PRIV_GUARDED)); + *pkt_flags |= PKTF_PRIV_GUARDED; + } - VERIFY(!(pkt->pkt_flags & PKTF_PRIV_GUARDED)); - pkt->pkt_flags |= PKTF_PRIV_GUARDED; - - if (pkt->pkt_timestamp > 0) { - now = pkt->pkt_timestamp; + if (*pkt_timestamp > 0) { + now = *pkt_timestamp; } else { - now = mach_absolute_time(); - pkt->pkt_timestamp = now; + struct timespec now_ts; + nanouptime(&now_ts); + now = (now_ts.tv_sec * NSEC_PER_SEC) + now_ts.tv_nsec; + *pkt_timestamp = now; } /* find the flowq for this packet */ - fq = fq_if_hash_pkt(fqs, pkt->pkt_flowid, m_get_service_class(m), - now, TRUE); + fq = fq_if_hash_pkt(fqs, pkt_flowid, pktsched_get_pkt_svc(pkt), + now, TRUE, pkt->pktsched_ptype); if (fq == NULL) { /* drop the packet if we could not allocate a flow queue */ fq_cl->fcl_stat.fcl_drop_memfailure++; IFCQ_CONVERT_LOCK(fqs->fqs_ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); + return (CLASSQEQ_DROP); } - - VERIFY(fq_cl->fcl_service_class == - (u_int32_t)mbuf_get_service_class(m)); + VERIFY(fq->fq_ptype == pkt->pktsched_ptype); fq_detect_dequeue_stall(fqs, fq, fq_cl, &now); if (FQ_IS_DELAYHIGH(fq)) { if ((fq->fq_flags & FQF_FLOWCTL_CAPABLE) && - (pkt->pkt_flags & PKTF_FLOW_ADV)) { + (*pkt_flags & PKTF_FLOW_ADV)) { fc_adv = 1; /* * If the flow is suspended or it is not * TCP, drop the packet */ - if (pkt->pkt_proto != IPPROTO_TCP) { + if (pkt_proto != IPPROTO_TCP) { droptype = DTYPE_EARLY; fq_cl->fcl_stat.fcl_drop_early++; } @@ -179,7 +201,7 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) * Need to drop a packet, instead of dropping this * one, try to drop from the head of the queue */ - if (!MBUFQ_EMPTY(&fq->fq_mbufq)) { + if (!fq_empty(fq)) { fq_head_drop(fqs, fq); droptype = DTYPE_NODROP; } else { @@ -190,28 +212,17 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) } - /* - * check if this packet is a retransmission of another pkt already - * in the queue - */ - if ((pkt->pkt_flags & (PKTF_TCP_REXMT|PKTF_START_SEQ)) == - (PKTF_TCP_REXMT|PKTF_START_SEQ) && fq->fq_dequeue_seq != 0) { - if (FQ_SEQ_GT(pkt->tx_start_seq, fq->fq_dequeue_seq)) { - fq_cl->fcl_stat.fcl_dup_rexmts++; - droptype = DTYPE_FORCED; - } - } - /* Set the return code correctly */ if (fc_adv == 1 && droptype != DTYPE_FORCED) { - if (fq_if_add_fcentry(fqs, pkt, fq_cl)) { + if (fq_if_add_fcentry(fqs, pkt, pkt_flowid, pkt_flowsrc, + fq_cl)) { fq->fq_flags |= FQF_FLOWCTL_ON; /* deliver flow control advisory error */ if (droptype == DTYPE_NODROP) { ret = CLASSQEQ_SUCCESS_FC; } else { /* dropped due to flow control */ - ret = CLASSQEQ_DROPPED_FC; + ret = CLASSQEQ_DROP_FC; } } else { /* @@ -219,7 +230,7 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) * better to drop */ droptype = DTYPE_FORCED; - ret = CLASSQEQ_DROPPED_FC; + ret = CLASSQEQ_DROP_FC; fq_cl->fcl_stat.fcl_flow_control_fail++; } } @@ -231,13 +242,38 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) * tail drop. */ if (droptype == DTYPE_NODROP && fq_if_at_drop_limit(fqs)) { - fq_if_drop_packet(fqs); + if (fqs->fqs_large_flow == fq) { + /* + * Drop from the head of the current fq. Since a + * new packet will be added to the tail, it is ok + * to leave fq in place. + */ + fq_head_drop(fqs, fq); + } else { + if (fqs->fqs_large_flow == NULL) { + droptype = DTYPE_FORCED; + fq_cl->fcl_stat.fcl_drop_overflow++; + + /* + * if this fq was freshly created and there + * is nothing to enqueue, free it + */ + if (fq_empty(fq) && !(fq->fq_flags & + (FQF_NEW_FLOW | FQF_OLD_FLOW))) { + fq_if_destroy_flow(fqs, fq_cl, fq); + fq = NULL; + } + } else { + fq_if_drop_packet(fqs); + } + } } if (droptype == DTYPE_NODROP) { - MBUFQ_ENQUEUE(&fq->fq_mbufq, m); - fq->fq_bytes += m_length(m); - fq_cl->fcl_stat.fcl_byte_cnt += m_length(m); + uint32_t pkt_len = pktsched_get_pkt_len(pkt); + fq_enqueue(fq, pkt->pktsched_pkt); + fq->fq_bytes += pkt_len; + fq_cl->fcl_stat.fcl_byte_cnt += pkt_len; fq_cl->fcl_stat.fcl_pkt_cnt++; /* @@ -247,8 +283,7 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) fq_if_is_flow_heavy(fqs, fq); } else { IFCQ_CONVERT_LOCK(fqs->fqs_ifq); - m_freem(m); - return ((ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROPPED); + return ((ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROP); } /* @@ -267,79 +302,97 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) return (ret); } -struct mbuf * -fq_getq_flow(fq_if_t *fqs, fq_t *fq) +void * +fq_getq_flow_internal(fq_if_t *fqs, fq_t *fq, pktsched_pkt_t *pkt) { - struct mbuf *m = NULL; - struct ifclassq *ifq = fqs->fqs_ifq; + void *p; + uint32_t plen; fq_if_classq_t *fq_cl; - u_int64_t now; - int64_t qdelay; - struct pkthdr *pkt; - u_int32_t mlen; + struct ifclassq *ifq = fqs->fqs_ifq; - MBUFQ_DEQUEUE(&fq->fq_mbufq, m); - if (m == NULL) + fq_dequeue(fq, p); + if (p == NULL) return (NULL); - mlen = m_length(m); + pktsched_pkt_encap(pkt, fq->fq_ptype, p); + plen = pktsched_get_pkt_len(pkt); - VERIFY(fq->fq_bytes >= mlen); - fq->fq_bytes -= mlen; + VERIFY(fq->fq_bytes >= plen); + fq->fq_bytes -= plen; fq_cl = &fqs->fqs_classq[fq->fq_sc_index]; - fq_cl->fcl_stat.fcl_byte_cnt -= mlen; + fq_cl->fcl_stat.fcl_byte_cnt -= plen; fq_cl->fcl_stat.fcl_pkt_cnt--; IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, mlen); + IFCQ_DEC_BYTES(ifq, plen); - pkt = &m->m_pkthdr; - now = mach_absolute_time(); + /* Reset getqtime so that we don't count idle times */ + if (fq_empty(fq)) + fq->fq_getqtime = 0; + + return (p); +} + +void * +fq_getq_flow(fq_if_t *fqs, fq_t *fq, pktsched_pkt_t *pkt) +{ + void *p; + fq_if_classq_t *fq_cl; + u_int64_t now; + int64_t qdelay = 0; + struct timespec now_ts; + uint32_t *pkt_flags, pkt_tx_start_seq; + uint64_t *pkt_timestamp; + + p = fq_getq_flow_internal(fqs, fq, pkt); + if (p == NULL) + return (NULL); + + pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, NULL, NULL, + NULL, &pkt_tx_start_seq); + + nanouptime(&now_ts); + now = (now_ts.tv_sec * NSEC_PER_SEC) + now_ts.tv_nsec; /* this will compute qdelay in nanoseconds */ - qdelay = now - pkt->pkt_timestamp; + if (now > *pkt_timestamp) + qdelay = now - *pkt_timestamp; + fq_cl = &fqs->fqs_classq[fq->fq_sc_index]; if (fq->fq_min_qdelay == 0 || (qdelay > 0 && (u_int64_t)qdelay < fq->fq_min_qdelay)) fq->fq_min_qdelay = qdelay; - if (now >= fq->fq_updatetime || MBUFQ_EMPTY(&fq->fq_mbufq)) { - if (fq->fq_min_qdelay >= fqs->fqs_target_qdelay) { + if (now >= fq->fq_updatetime) { + if (fq->fq_min_qdelay > fqs->fqs_target_qdelay) { if (!FQ_IS_DELAYHIGH(fq)) FQ_SET_DELAY_HIGH(fq); - } - - if (!FQ_IS_DELAYHIGH(fq) || MBUFQ_EMPTY(&fq->fq_mbufq)) { + } else { FQ_CLEAR_DELAY_HIGH(fq); - if (fq->fq_flags & FQF_FLOWCTL_ON) { - fq_if_flow_feedback(fqs, fq, fq_cl); - } } + /* Reset measured queue delay and update time */ fq->fq_updatetime = now + fqs->fqs_update_interval; fq->fq_min_qdelay = 0; } + if (!FQ_IS_DELAYHIGH(fq) || fq_empty(fq)) { + FQ_CLEAR_DELAY_HIGH(fq); + if (fq->fq_flags & FQF_FLOWCTL_ON) { + fq_if_flow_feedback(fqs, fq, fq_cl); + } + } - if ((pkt->pkt_flags & PKTF_START_SEQ) && (fq->fq_dequeue_seq == 0 || - (FQ_SEQ_LT(fq->fq_dequeue_seq, pkt->tx_start_seq)))) - fq->fq_dequeue_seq = pkt->tx_start_seq; - - pkt->pkt_timestamp = 0; - pkt->pkt_flags &= ~PKTF_PRIV_GUARDED; - - if (MBUFQ_EMPTY(&fq->fq_mbufq)) { - /* - * Remove from large_flow field, if this happened to be - * the one that is tagged. - */ - if (fqs->fqs_large_flow == fq) - fqs->fqs_large_flow = NULL; - + if (fq_empty(fq)) { /* Reset getqtime so that we don't count idle times */ fq->fq_getqtime = 0; } else { fq->fq_getqtime = now; } + fq_if_is_flow_heavy(fqs, fq); + + *pkt_timestamp = 0; + if (pkt->pktsched_ptype == QP_MBUF) + *pkt_flags &= ~PKTF_PRIV_GUARDED; - return (m); + return (p); } diff --git a/bsd/net/classq/classq_fq_codel.h b/bsd/net/classq/classq_fq_codel.h index 35f8341b2..6683256c1 100644 --- a/bsd/net/classq/classq_fq_codel.h +++ b/bsd/net/classq/classq_fq_codel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Apple Inc. All rights reserved. + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,7 +40,7 @@ extern "C" { #endif #define FQ_MIN_FC_THRESHOLD_BYTES 7500 -#define FQ_IS_DELAYHIGH(_fq_) ((_fq_)->fq_flags & FQF_DELAY_HIGH) +#define FQ_IS_DELAYHIGH(_fq_) ((_fq_)->fq_flags & FQF_DELAY_HIGH) #define FQ_SET_DELAY_HIGH(_fq_) do { \ (_fq_)->fq_flags |= FQF_DELAY_HIGH; \ } while (0) @@ -49,7 +49,9 @@ extern "C" { } while (0) typedef struct flowq { - MBUFQ_HEAD(pktq_head) fq_mbufq; /* Packet queue */ + union { + MBUFQ_HEAD(mbufq_head) __mbufq; /* mbuf packet queue */ + } __fq_pktq_u; #define FQF_FLOWCTL_CAPABLE 0x01 /* Use flow control instead of drop */ #define FQF_DELAY_HIGH 0x02 /* Min delay is greater than target */ #define FQF_NEW_FLOW 0x04 /* Currently on new flows queue */ @@ -65,19 +67,34 @@ typedef struct flowq { SLIST_ENTRY(flowq) fq_hashlink; /* for flow queue hash table */ STAILQ_ENTRY(flowq) fq_actlink; /* for new/old flow queues */ u_int32_t fq_flowhash; /* Flow hash */ - u_int32_t fq_dequeue_seq; /* Last dequeue seq */ + classq_pkt_type_t fq_ptype; /* Packet type */ } fq_t; +#define fq_mbufq __fq_pktq_u.__mbufq + +#define fq_empty(_q) MBUFQ_EMPTY(&(_q)->fq_mbufq) + +#define fq_enqueue(_q, _p) MBUFQ_ENQUEUE(&(_q)->fq_mbufq, (mbuf_t)_p) + +#define fq_dequeue(_q, _p) do { \ + mbuf_t _m; \ + MBUFQ_DEQUEUE(&(_q)->fq_mbufq, _m); \ + (_p) = _m; \ +} while (0) + struct fq_codel_sched_data; struct fq_if_classq; /* Function definitions */ extern void fq_codel_init(void); -extern fq_t *fq_alloc(int); +extern fq_t *fq_alloc(classq_pkt_type_t); extern void fq_destroy(fq_t *); -extern int fq_addq(struct fq_codel_sched_data *, struct mbuf *, +extern int fq_addq(struct fq_codel_sched_data *, pktsched_pkt_t *, struct fq_if_classq *); -extern struct mbuf *fq_getq_flow(struct fq_codel_sched_data *, fq_t *); +extern void *fq_getq_flow(struct fq_codel_sched_data *, fq_t *, + pktsched_pkt_t *); +extern void *fq_getq_flow_internal(struct fq_codel_sched_data *, + fq_t *, pktsched_pkt_t *); extern void fq_head_drop(struct fq_codel_sched_data *, fq_t *); #ifdef __cplusplus diff --git a/bsd/net/classq/classq_red.c b/bsd/net/classq/classq_red.c deleted file mode 100644 index 63cbd8ede..000000000 --- a/bsd/net/classq/classq_red.c +++ /dev/null @@ -1,630 +0,0 @@ -/* - * Copyright (c) 2007-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_red.c,v 1.14 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_red.c,v 1.10 2002/04/03 05:38:51 kjc Exp $ */ - -/* - * Copyright (C) 1997-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ -/* - * Copyright (c) 1990-1994 Regents of the University of California. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Computer Systems - * Engineering Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include - -#if CLASSQ_RED - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#if INET6 -#include -#endif - -#include -#include - -/* - * ALTQ/RED (Random Early Detection) implementation using 32-bit - * fixed-point calculation. - * - * written by kjc using the ns code as a reference. - * you can learn more about red and ns from Sally's home page at - * http://www-nrg.ee.lbl.gov/floyd/ - * - * most of the red parameter values are fixed in this implementation - * to prevent fixed-point overflow/underflow. - * if you change the parameters, watch out for overflow/underflow! - * - * the parameters used are recommended values by Sally. - * the corresponding ns config looks: - * q_weight=0.00195 - * minthresh=5 maxthresh=15 queue-size=60 - * linterm=30 - * dropmech=drop-tail - * bytes=false (can't be handled by 32-bit fixed-point) - * doubleq=false dqthresh=false - * wait=true - */ -/* - * alternative red parameters for a slow link. - * - * assume the queue length becomes from zero to L and keeps L, it takes - * N packets for q_avg to reach 63% of L. - * when q_weight is 0.002, N is about 500 packets. - * for a slow link like dial-up, 500 packets takes more than 1 minute! - * when q_weight is 0.008, N is about 127 packets. - * when q_weight is 0.016, N is about 63 packets. - * bursts of 50 packets are allowed for 0.002, bursts of 25 packets - * are allowed for 0.016. - * see Sally's paper for more details. - */ -/* normal red parameters */ -#define W_WEIGHT 512 /* inverse of weight of EWMA (511/512) */ - /* q_weight = 0.00195 */ - -/* red parameters for a slow link */ -#define W_WEIGHT_1 128 /* inverse of weight of EWMA (127/128) */ - /* q_weight = 0.0078125 */ - -/* red parameters for a very slow link (e.g., dialup) */ -#define W_WEIGHT_2 64 /* inverse of weight of EWMA (63/64) */ - /* q_weight = 0.015625 */ - -/* fixed-point uses 12-bit decimal places */ -#define FP_SHIFT 12 /* fixed-point shift */ - -/* red parameters for drop probability */ -#define INV_P_MAX 10 /* inverse of max drop probability */ -#define TH_MIN 5 /* min threshold */ -#define TH_MAX 15 /* max threshold */ - -#define RED_LIMIT 60 /* default max queue lenght */ - -#define RED_ZONE_MAX 32 /* maximum elements in zone */ -#define RED_ZONE_NAME "classq_red" /* zone name */ - -static unsigned int red_size; /* size of zone element */ -static struct zone *red_zone; /* zone for red */ - -/* - * our default policy for forced-drop is drop-tail. - * (in altq-1.1.2 or earlier, the default was random-drop. - * but it makes more sense to punish the cause of the surge.) - * to switch to the random-drop policy, define "RED_RANDOM_DROP". - */ - -/* default red parameter values */ -static int default_th_min = TH_MIN; -static int default_th_max = TH_MAX; -static int default_inv_pmax = INV_P_MAX; - -static struct mbuf *red_getq_flow(struct red *, class_queue_t *, - u_int32_t, boolean_t); - -void -red_init(void) -{ - _CASSERT(REDF_ECN4 == CLASSQF_ECN4); - _CASSERT(REDF_ECN6 == CLASSQF_ECN6); - - red_size = sizeof (red_t); - red_zone = zinit(red_size, RED_ZONE_MAX * red_size, - 0, RED_ZONE_NAME); - if (red_zone == NULL) { - panic("%s: failed allocating %s", __func__, RED_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(red_zone, Z_EXPAND, TRUE); - zone_change(red_zone, Z_CALLERACCT, TRUE); -} - -/* - * red support routines - */ -red_t * -red_alloc(struct ifnet *ifp, int weight, int inv_pmax, int th_min, - int th_max, int flags, int pkttime) -{ - red_t *rp; - int w, i; - int npkts_per_sec; - - VERIFY(ifp != NULL); - - rp = zalloc(red_zone); - if (rp == NULL) - return (NULL); - - bzero(rp, red_size); - rp->red_avg = 0; - rp->red_idle = 1; - - if (weight == 0) - rp->red_weight = W_WEIGHT; - else - rp->red_weight = weight; - if (inv_pmax == 0) - rp->red_inv_pmax = default_inv_pmax; - else - rp->red_inv_pmax = inv_pmax; - if (th_min == 0) - rp->red_thmin = default_th_min; - else - rp->red_thmin = th_min; - if (th_max == 0) - rp->red_thmax = default_th_max; - else - rp->red_thmax = th_max; - - rp->red_ifp = ifp; - rp->red_flags = (flags & REDF_USERFLAGS); -#if !PF_ECN - if (rp->red_flags & REDF_ECN) { - rp->red_flags &= ~REDF_ECN; - log(LOG_ERR, "%s: RED ECN not available; ignoring " - "REDF_ECN flag!\n", if_name(ifp)); - } -#endif /* !PF_ECN */ - - if (pkttime == 0) - /* default packet time: 1000 bytes / 10Mbps * 8 * 1000000 */ - rp->red_pkttime = 800; - else - rp->red_pkttime = pkttime; - - if (weight == 0) { - /* when the link is very slow, adjust red parameters */ - npkts_per_sec = 1000000 / rp->red_pkttime; - if (npkts_per_sec < 50) { - /* up to about 400Kbps */ - rp->red_weight = W_WEIGHT_2; - } else if (npkts_per_sec < 300) { - /* up to about 2.4Mbps */ - rp->red_weight = W_WEIGHT_1; - } - } - - /* calculate wshift. weight must be power of 2 */ - w = rp->red_weight; - for (i = 0; w > 1; i++) - w = w >> 1; - rp->red_wshift = i; - w = 1 << rp->red_wshift; - if (w != rp->red_weight) { - printf("invalid weight value %d for red! use %d\n", - rp->red_weight, w); - rp->red_weight = w; - } - - /* - * thmin_s and thmax_s are scaled versions of th_min and th_max - * to be compared with avg. - */ - rp->red_thmin_s = rp->red_thmin << (rp->red_wshift + FP_SHIFT); - rp->red_thmax_s = rp->red_thmax << (rp->red_wshift + FP_SHIFT); - - /* - * precompute probability denominator - * probd = (2 * (TH_MAX-TH_MIN) / pmax) in fixed-point - */ - rp->red_probd = (2 * (rp->red_thmax - rp->red_thmin) * - rp->red_inv_pmax) << FP_SHIFT; - - /* allocate weight table */ - rp->red_wtab = wtab_alloc(rp->red_weight); - if (rp->red_wtab == NULL) { - red_destroy(rp); - return (NULL); - } - - microuptime(&rp->red_last); - return (rp); -} - -void -red_destroy(red_t *rp) -{ - if (rp->red_wtab != NULL) { - wtab_destroy(rp->red_wtab); - rp->red_wtab = NULL; - } - zfree(red_zone, rp); -} - -void -red_getstats(red_t *rp, struct red_stats *sp) -{ - sp->q_avg = rp->red_avg >> rp->red_wshift; - sp->drop_forced = rp->red_stats.drop_forced; - sp->drop_unforced = rp->red_stats.drop_unforced; - sp->marked_packets = rp->red_stats.marked_packets; -} - -int -red_addq(red_t *rp, class_queue_t *q, struct mbuf *m, struct pf_mtag *tag) -{ -#if !PF_ECN -#pragma unused(tag) -#endif /* !PF_ECN */ - int avg, droptype; - int n; - - avg = rp->red_avg; - - /* - * if we were idle, we pretend that n packets arrived during - * the idle period. - */ - if (rp->red_idle) { - struct timeval now; - int t; - - rp->red_idle = 0; - microuptime(&now); - t = (now.tv_sec - rp->red_last.tv_sec); - if (t > 60) { - /* - * being idle for more than 1 minute, set avg to zero. - * this prevents t from overflow. - */ - avg = 0; - } else { - t = t * 1000000 + (now.tv_usec - rp->red_last.tv_usec); - n = t / rp->red_pkttime - 1; - - /* the following line does (avg = (1 - Wq)^n * avg) */ - if (n > 0) - avg = (avg >> FP_SHIFT) * - pow_w(rp->red_wtab, n); - } - } - - /* run estimator. (note: avg is scaled by WEIGHT in fixed-point) */ - avg += (qlen(q) << FP_SHIFT) - (avg >> rp->red_wshift); - rp->red_avg = avg; /* save the new value */ - - /* - * red_count keeps a tally of arriving traffic that has not - * been dropped. - */ - rp->red_count++; - - /* see if we drop early */ - droptype = DTYPE_NODROP; - if (avg >= rp->red_thmin_s && qlen(q) > 1) { - if (avg >= rp->red_thmax_s) { - /* avg >= th_max: forced drop */ - droptype = DTYPE_FORCED; - } else if (rp->red_old == 0) { - /* first exceeds th_min */ - rp->red_count = 1; - rp->red_old = 1; - } else if (drop_early((avg - rp->red_thmin_s) >> rp->red_wshift, - rp->red_probd, rp->red_count)) { - /* mark or drop by red */ -#if PF_ECN - if ((rp->red_flags & REDF_ECN) && - (tag->pftag_proto == IPPROTO_TCP) && /* only TCP */ - mark_ecn(m, tag, rp->red_flags)) { - /* successfully marked. do not drop. */ - rp->red_count = 0; - rp->red_stats.marked_packets++; - } else -#endif /* PF_ECN */ - { - /* unforced drop by red */ - droptype = DTYPE_EARLY; - } - } - } else { - /* avg < th_min */ - rp->red_old = 0; - } - - /* - * if the queue length hits the hard limit, it's a forced drop. - */ - if (droptype == DTYPE_NODROP && qlen(q) >= qlimit(q)) - droptype = DTYPE_FORCED; - -#ifdef RED_RANDOM_DROP - /* if successful or forced drop, enqueue this packet. */ - if (droptype != DTYPE_EARLY) - _addq(q, m); -#else - /* if successful, enqueue this packet. */ - if (droptype == DTYPE_NODROP) - _addq(q, m); -#endif - if (droptype != DTYPE_NODROP) { - if (droptype == DTYPE_EARLY) { - /* drop the incoming packet */ - rp->red_stats.drop_unforced++; - } else { - /* forced drop, select a victim packet in the queue. */ -#ifdef RED_RANDOM_DROP - m = _getq_random(q); -#endif - rp->red_stats.drop_forced++; - } - rp->red_count = 0; - IFCQ_CONVERT_LOCK(&rp->red_ifp->if_snd); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - /* successfully queued */ - return (CLASSQEQ_SUCCESS); -} - -/* - * early-drop probability is calculated as follows: - * prob = p_max * (avg - th_min) / (th_max - th_min) - * prob_a = prob / (2 - count*prob) - * = (avg-th_min) / (2*(th_max-th_min)*inv_p_max - count*(avg-th_min)) - * here prob_a increases as successive undrop count increases. - * (prob_a starts from prob/2, becomes prob when (count == (1 / prob)), - * becomes 1 when (count >= (2 / prob))). - */ -int -drop_early(int fp_len, int fp_probd, int count) -{ - int d; /* denominator of drop-probability */ - - d = fp_probd - count * fp_len; - if (d <= 0) - /* count exceeds the hard limit: drop or mark */ - return (1); - - /* - * now the range of d is [1..600] in fixed-point. (when - * th_max-th_min=10 and p_max=1/30) - * drop probability = (avg - TH_MIN) / d - */ - - if ((RandomULong() % d) < (unsigned)fp_len) { - /* drop or mark */ - return (1); - } - /* no drop/mark */ - return (0); -} - -static struct mbuf * -red_getq_flow(struct red *rp, class_queue_t *q, u_int32_t flow, boolean_t purge) -{ -#pragma unused(purge) - struct mbuf *m; - - /* flow of 0 means head of queue */ - if ((m = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) { - if (rp->red_idle == 0) { - rp->red_idle = 1; - microuptime(&rp->red_last); - } - return (NULL); - } - - rp->red_idle = 0; - return (m); -} - -struct mbuf * -red_getq(red_t *rp, class_queue_t *q) -{ - return (red_getq_flow(rp, q, 0, FALSE)); -} - -void -red_purgeq(struct red *rp, class_queue_t *q, u_int32_t flow, u_int32_t *packets, - u_int32_t *bytes) -{ - u_int32_t cnt = 0, len = 0; - struct mbuf *m; - - IFCQ_CONVERT_LOCK(&rp->red_ifp->if_snd); - - while ((m = red_getq_flow(rp, q, flow, TRUE)) != NULL) { - cnt++; - len += m_pktlen(m); - m_freem(m); - } - - if (packets != NULL) - *packets = cnt; - if (bytes != NULL) - *bytes = len; -} - -void -red_updateq(red_t *rp, cqev_t ev) -{ -#pragma unused(rp, ev) - /* nothing for now */ -} - -int -red_suspendq(red_t *rp, class_queue_t *q, boolean_t on) -{ -#pragma unused(rp, q, on) - return (ENOTSUP); -} - -/* - * helper routine to calibrate avg during idle. - * pow_w(wtab, n) returns (1 - Wq)^n in fixed-point - * here Wq = 1/weight and the code assumes Wq is close to zero. - * - * w_tab[n] holds ((1 - Wq)^(2^n)) in fixed-point. - */ -static struct wtab *wtab_list = NULL; /* pointer to wtab list */ - -struct wtab * -wtab_alloc(int weight) -{ - struct wtab *w; - int i; - - for (w = wtab_list; w != NULL; w = w->w_next) - if (w->w_weight == weight) { - w->w_refcount++; - return (w); - } - - w = _MALLOC(sizeof (struct wtab), M_DEVBUF, M_WAITOK|M_ZERO); - if (w == NULL) - return (NULL); - - w->w_weight = weight; - w->w_refcount = 1; - w->w_next = wtab_list; - wtab_list = w; - - /* initialize the weight table */ - w->w_tab[0] = ((weight - 1) << FP_SHIFT) / weight; - for (i = 1; i < 32; i++) { - w->w_tab[i] = (w->w_tab[i-1] * w->w_tab[i-1]) >> FP_SHIFT; - if (w->w_tab[i] == 0 && w->w_param_max == 0) - w->w_param_max = 1 << i; - } - - return (w); -} - -void -wtab_destroy(struct wtab *w) -{ - struct wtab *prev; - - if (--w->w_refcount > 0) - return; - - if (wtab_list == w) - wtab_list = w->w_next; - else for (prev = wtab_list; prev->w_next != NULL; prev = prev->w_next) - if (prev->w_next == w) { - prev->w_next = w->w_next; - break; - } - - _FREE(w, M_DEVBUF); -} - -int32_t -pow_w(struct wtab *w, int n) -{ - int i, bit; - int32_t val; - - if (n >= w->w_param_max) - return (0); - - val = 1 << FP_SHIFT; - if (n <= 0) - return (val); - - bit = 1; - i = 0; - while (n) { - if (n & bit) { - val = (val * w->w_tab[i]) >> FP_SHIFT; - n &= ~bit; - } - i++; - bit <<= 1; - } - return (val); -} - -#endif /* CLASSQ_RED */ diff --git a/bsd/net/classq/classq_red.h b/bsd/net/classq/classq_red.h index 58956b504..ceae286dc 100644 --- a/bsd/net/classq/classq_red.h +++ b/bsd/net/classq/classq_red.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -85,82 +85,6 @@ struct red_stats { u_int32_t marked_packets; }; -#ifdef BSD_KERNEL_PRIVATE -/* weight table structure for idle time calibration */ -struct wtab { - struct wtab *w_next; - int w_weight; - int w_param_max; - int w_refcount; - int32_t w_tab[32]; -}; - -/* red flags */ -#define REDF_ECN4 0x01 /* use packet marking for IPv4 packets */ -#define REDF_ECN6 0x02 /* use packet marking for IPv6 packets */ -#define REDF_ECN (REDF_ECN4 | REDF_ECN6) -#define REDF_FLOWVALVE 0x04 /* use flowvalve (aka penalty-box) */ - -#define REDF_USERFLAGS \ - (REDF_ECN4 | REDF_ECN6 | REDF_FLOWVALVE) - -typedef struct red { - int red_pkttime; /* average packet time in micro sec */ - /* used for idle calibration */ - int red_flags; /* red flags */ - struct ifnet *red_ifp; /* back pointer to ifnet */ - - /* red parameters */ - int red_weight; /* weight for EWMA */ - int red_inv_pmax; /* inverse of max drop probability */ - int red_thmin; /* red min threshold */ - int red_thmax; /* red max threshold */ - - /* variables for internal use */ - int red_wshift; /* log(red_weight) */ - int red_thmin_s; /* th_min scaled by avgshift */ - int red_thmax_s; /* th_max scaled by avgshift */ - int red_probd; /* drop probability denominator */ - - int red_avg; /* queue len avg scaled by avgshift */ - int red_count; /* packet count since last dropped/ */ - /* marked packet */ - int red_idle; /* queue was empty */ - int red_old; /* avg is above th_min */ - struct wtab *red_wtab; /* weight table */ - struct timeval red_last; /* time when the queue becomes idle */ - - struct { - struct pktcntr xmit_cnt; - struct pktcntr drop_cnt; - u_int32_t drop_forced; - u_int32_t drop_unforced; - u_int32_t marked_packets; - } red_stats; -} red_t; - -/* red drop types */ -#define DTYPE_NODROP 0 /* no drop */ -#define DTYPE_FORCED 1 /* a "forced" drop */ -#define DTYPE_EARLY 2 /* an "unforced" (early) drop */ - -extern void red_init(void); -extern red_t *red_alloc(struct ifnet *, int, int, int, int, int, int); -extern void red_destroy(red_t *); -extern void red_getstats(red_t *, struct red_stats *); -extern int red_addq(red_t *, class_queue_t *, struct mbuf *, struct pf_mtag *); -extern struct mbuf *red_getq(red_t *, class_queue_t *); -extern void red_purgeq(struct red *, class_queue_t *, u_int32_t, - u_int32_t *, u_int32_t *); -extern void red_updateq(red_t *, cqev_t); -extern int red_suspendq(red_t *, class_queue_t *, boolean_t); - -extern int drop_early(int, int, int); -extern struct wtab *wtab_alloc(int); -extern void wtab_destroy(struct wtab *); -extern int32_t pow_w(struct wtab *, int); -#endif /* BSD_KERNEL_PRIVATE */ - #ifdef __cplusplus } #endif diff --git a/bsd/net/classq/classq_rio.c b/bsd/net/classq/classq_rio.c deleted file mode 100644 index 91f7da00a..000000000 --- a/bsd/net/classq/classq_rio.c +++ /dev/null @@ -1,554 +0,0 @@ -/* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_rio.c,v 1.11 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_rio.c,v 1.8 2000/12/14 08:12:46 thorpej Exp $ */ - -/* - * Copyright (C) 1998-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -/* - * Copyright (c) 1990-1994 Regents of the University of California. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Computer Systems - * Engineering Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include - -#if CLASSQ_RIO - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#if INET6 -#include -#endif - -#include -#include -#include - -/* - * RIO: RED with IN/OUT bit - * described in - * "Explicit Allocation of Best Effort Packet Delivery Service" - * David D. Clark and Wenjia Fang, MIT Lab for Computer Science - * http://diffserv.lcs.mit.edu/Papers/exp-alloc-ddc-wf.{ps,pdf} - * - * this implementation is extended to support more than 2 drop precedence - * values as described in RFC2597 (Assured Forwarding PHB Group). - * - */ -/* - * AF DS (differentiated service) codepoints. - * (classes can be mapped to CBQ or H-FSC classes.) - * - * 0 1 2 3 4 5 6 7 - * +---+---+---+---+---+---+---+---+ - * | CLASS |DropPre| 0 | CU | - * +---+---+---+---+---+---+---+---+ - * - * class 1: 001 - * class 2: 010 - * class 3: 011 - * class 4: 100 - * - * low drop prec: 01 - * medium drop prec: 10 - * high drop prec: 11 - */ - -/* normal red parameters */ -#define W_WEIGHT 512 /* inverse of weight of EWMA (511/512) */ - /* q_weight = 0.00195 */ - -/* red parameters for a slow link */ -#define W_WEIGHT_1 128 /* inverse of weight of EWMA (127/128) */ - /* q_weight = 0.0078125 */ - -/* red parameters for a very slow link (e.g., dialup) */ -#define W_WEIGHT_2 64 /* inverse of weight of EWMA (63/64) */ - /* q_weight = 0.015625 */ - -/* fixed-point uses 12-bit decimal places */ -#define FP_SHIFT 12 /* fixed-point shift */ - -/* red parameters for drop probability */ -#define INV_P_MAX 10 /* inverse of max drop probability */ -#define TH_MIN 5 /* min threshold */ -#define TH_MAX 15 /* max threshold */ - -#define RIO_LIMIT 60 /* default max queue lenght */ - -/* default rio parameter values */ -static struct redparams default_rio_params[RIO_NDROPPREC] = { - /* th_min, th_max, inv_pmax */ - { TH_MAX * 2 + TH_MIN, TH_MAX * 3, INV_P_MAX }, /* low drop precedence */ - { TH_MAX + TH_MIN, TH_MAX * 2, INV_P_MAX }, /* medium drop precedence */ - { TH_MIN, TH_MAX, INV_P_MAX } /* high drop precedence */ -}; - -#define RIO_ZONE_MAX 32 /* maximum elements in zone */ -#define RIO_ZONE_NAME "classq_rio" /* zone name */ - -static unsigned int rio_size; /* size of zone element */ -static struct zone *rio_zone; /* zone for rio */ - -/* internal function prototypes */ -static struct mbuf *rio_getq_flow(struct rio *, class_queue_t *, - u_int32_t, boolean_t); -static int dscp2index(u_int8_t); - -void -rio_init(void) -{ - _CASSERT(RIOF_ECN4 == CLASSQF_ECN4); - _CASSERT(RIOF_ECN6 == CLASSQF_ECN6); - - rio_size = sizeof (rio_t); - rio_zone = zinit(rio_size, RIO_ZONE_MAX * rio_size, - 0, RIO_ZONE_NAME); - if (rio_zone == NULL) { - panic("%s: failed allocating %s", __func__, RIO_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(rio_zone, Z_EXPAND, TRUE); - zone_change(rio_zone, Z_CALLERACCT, TRUE); -} - -rio_t * -rio_alloc(struct ifnet *ifp, int weight, struct redparams *params, - int flags, int pkttime) -{ - rio_t *rp; - int w, i; - int npkts_per_sec; - - VERIFY(ifp != NULL); - - rp = zalloc(rio_zone); - if (rp == NULL) - return (NULL); - - bzero(rp, rio_size); - rp->rio_ifp = ifp; - rp->rio_flags = (flags & RIOF_USERFLAGS); -#if !PF_ECN - if (rp->rio_flags & RIOF_ECN) { - rp->rio_flags &= ~RIOF_ECN; - log(LOG_ERR, "%s: RIO ECN not available; ignoring " - "RIOF_ECN flag!\n", if_name(ifp)); - } - if (rp->rio_flags & RIOF_CLEARDSCP) { - rp->rio_flags &= ~RIOF_CLEARDSCP; - log(LOG_ERR, "%s: RIO ECN not available; ignoring " - "RIOF_CLEARDSCP flag!\n", if_name(ifp)); - } -#endif /* !PF_ECN */ - - if (pkttime == 0) - /* default packet time: 1000 bytes / 10Mbps * 8 * 1000000 */ - rp->rio_pkttime = 800; - else - rp->rio_pkttime = pkttime; - - if (weight != 0) - rp->rio_weight = weight; - else { - /* use default */ - rp->rio_weight = W_WEIGHT; - - /* when the link is very slow, adjust red parameters */ - npkts_per_sec = 1000000 / rp->rio_pkttime; - if (npkts_per_sec < 50) { - /* up to about 400Kbps */ - rp->rio_weight = W_WEIGHT_2; - } else if (npkts_per_sec < 300) { - /* up to about 2.4Mbps */ - rp->rio_weight = W_WEIGHT_1; - } - } - - /* calculate wshift. weight must be power of 2 */ - w = rp->rio_weight; - for (i = 0; w > 1; i++) - w = w >> 1; - rp->rio_wshift = i; - w = 1 << rp->rio_wshift; - if (w != rp->rio_weight) { - printf("invalid weight value %d for red! use %d\n", - rp->rio_weight, w); - rp->rio_weight = w; - } - - /* allocate weight table */ - rp->rio_wtab = wtab_alloc(rp->rio_weight); - if (rp->rio_wtab == NULL) { - rio_destroy(rp); - return (NULL); - } - - for (i = 0; i < RIO_NDROPPREC; i++) { - struct dropprec_state *prec = &rp->rio_precstate[i]; - - prec->avg = 0; - prec->idle = 1; - - if (params == NULL || params[i].inv_pmax == 0) - prec->inv_pmax = default_rio_params[i].inv_pmax; - else - prec->inv_pmax = params[i].inv_pmax; - if (params == NULL || params[i].th_min == 0) - prec->th_min = default_rio_params[i].th_min; - else - prec->th_min = params[i].th_min; - if (params == NULL || params[i].th_max == 0) - prec->th_max = default_rio_params[i].th_max; - else - prec->th_max = params[i].th_max; - - /* - * th_min_s and th_max_s are scaled versions of th_min - * and th_max to be compared with avg. - */ - prec->th_min_s = prec->th_min << (rp->rio_wshift + FP_SHIFT); - prec->th_max_s = prec->th_max << (rp->rio_wshift + FP_SHIFT); - - /* - * precompute probability denominator - * probd = (2 * (TH_MAX-TH_MIN) / pmax) in fixed-point - */ - prec->probd = (2 * (prec->th_max - prec->th_min) * - prec->inv_pmax) << FP_SHIFT; - - microuptime(&prec->last); - } - - return (rp); -} - -void -rio_destroy(rio_t *rp) -{ - if (rp->rio_wtab != NULL) { - wtab_destroy(rp->rio_wtab); - rp->rio_wtab = NULL; - } - zfree(rio_zone, rp); -} - -void -rio_getstats(rio_t *rp, struct red_stats *sp) -{ - int i; - - for (i = 0; i < RIO_NDROPPREC; i++) { - bcopy(&rp->q_stats[i], sp, sizeof (struct red_stats)); - sp->q_avg = rp->rio_precstate[i].avg >> rp->rio_wshift; - sp++; - } -} - -#if (RIO_NDROPPREC == 3) -/* - * internally, a drop precedence value is converted to an index - * starting from 0. - */ -static int -dscp2index(u_int8_t dscp) -{ -#define AF_DROPPRECMASK 0x18 - - int dpindex = dscp & AF_DROPPRECMASK; - - if (dpindex == 0) - return (0); - return ((dpindex >> 3) - 1); -} -#endif - -/* Store RIO precindex in the module private scratch space */ -#define pkt_precidx pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32 - -#define RIOM_SET_PRECINDEX(pkt, idx) do { \ - (pkt)->pkt_precidx = (idx); \ -} while (0) - -#define RIOM_GET_PRECINDEX(pkt) \ - ({ u_int32_t idx; idx = (pkt)->pkt_precidx; \ - RIOM_SET_PRECINDEX(pkt, 0); idx; }) - -int -rio_addq(rio_t *rp, class_queue_t *q, struct mbuf *m, struct pf_mtag *tag) -{ -#if !PF_ECN -#pragma unused(tag) -#endif /* !PF_ECN */ -#define DSCP_MASK 0xfc - int avg, droptype; - u_int8_t dsfield, odsfield; - int dpindex, i, n, t; - struct timeval now; - struct dropprec_state *prec; - -#if PF_ECN - dsfield = odsfield = read_dsfield(m, tag); -#else - dsfield = odsfield = 0; -#endif /* !PF_ECN */ - dpindex = dscp2index(dsfield); - - /* - * update avg of the precedence states whose drop precedence - * is larger than or equal to the drop precedence of the packet - */ - now.tv_sec = 0; - for (i = dpindex; i < RIO_NDROPPREC; i++) { - prec = &rp->rio_precstate[i]; - avg = prec->avg; - if (prec->idle) { - prec->idle = 0; - if (now.tv_sec == 0) - microuptime(&now); - t = (now.tv_sec - prec->last.tv_sec); - if (t > 60) - avg = 0; - else { - t = t * 1000000 + - (now.tv_usec - prec->last.tv_usec); - n = t / rp->rio_pkttime; - /* calculate (avg = (1 - Wq)^n * avg) */ - if (n > 0) { - avg = (avg >> FP_SHIFT) * - pow_w(rp->rio_wtab, n); - } - } - } - - /* run estimator. (avg is scaled by WEIGHT in fixed-point) */ - avg += (prec->qlen << FP_SHIFT) - (avg >> rp->rio_wshift); - prec->avg = avg; /* save the new value */ - /* - * count keeps a tally of arriving traffic that has not - * been dropped. - */ - prec->count++; - } - - prec = &rp->rio_precstate[dpindex]; - avg = prec->avg; - - /* see if we drop early */ - droptype = DTYPE_NODROP; - if (avg >= prec->th_min_s && prec->qlen > 1) { - if (avg >= prec->th_max_s) { - /* avg >= th_max: forced drop */ - droptype = DTYPE_FORCED; - } else if (prec->old == 0) { - /* first exceeds th_min */ - prec->count = 1; - prec->old = 1; - } else if (drop_early((avg - prec->th_min_s) >> rp->rio_wshift, - prec->probd, prec->count)) { - /* unforced drop by red */ - droptype = DTYPE_EARLY; - } - } else { - /* avg < th_min */ - prec->old = 0; - } - - /* - * if the queue length hits the hard limit, it's a forced drop. - */ - if (droptype == DTYPE_NODROP && qlen(q) >= qlimit(q)) - droptype = DTYPE_FORCED; - - if (droptype != DTYPE_NODROP) { - /* always drop incoming packet (as opposed to randomdrop) */ - for (i = dpindex; i < RIO_NDROPPREC; i++) - rp->rio_precstate[i].count = 0; - - if (droptype == DTYPE_EARLY) - rp->q_stats[dpindex].drop_unforced++; - else - rp->q_stats[dpindex].drop_forced++; - - IFCQ_CONVERT_LOCK(&rp->rio_ifp->if_snd); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - - for (i = dpindex; i < RIO_NDROPPREC; i++) - rp->rio_precstate[i].qlen++; - - /* save drop precedence index in mbuf hdr */ - RIOM_SET_PRECINDEX(&m->m_pkthdr, dpindex); - - if (rp->rio_flags & RIOF_CLEARDSCP) - dsfield &= ~DSCP_MASK; - -#if PF_ECN - if (dsfield != odsfield) - write_dsfield(m, tag, dsfield); -#endif /* PF_ECN */ - - _addq(q, m); - - return (CLASSQEQ_SUCCESS); -} - -static struct mbuf * -rio_getq_flow(struct rio *rp, class_queue_t *q, u_int32_t flow, boolean_t purge) -{ -#pragma unused(purge) - struct mbuf *m; - int dpindex, i; - - /* flow of 0 means head of queue */ - if ((m = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) - return (NULL); - - VERIFY(m->m_flags & M_PKTHDR); - - dpindex = RIOM_GET_PRECINDEX(&m->m_pkthdr); - for (i = dpindex; i < RIO_NDROPPREC; i++) { - if (--rp->rio_precstate[i].qlen == 0) { - if (rp->rio_precstate[i].idle == 0) { - rp->rio_precstate[i].idle = 1; - microuptime(&rp->rio_precstate[i].last); - } - } - } - return (m); -} - -struct mbuf * -rio_getq(rio_t *rp, class_queue_t *q) -{ - return (rio_getq_flow(rp, q, 0, FALSE)); -} - -void -rio_purgeq(struct rio *rp, class_queue_t *q, u_int32_t flow, u_int32_t *packets, - u_int32_t *bytes) -{ - u_int32_t cnt = 0, len = 0; - struct mbuf *m; - - IFCQ_CONVERT_LOCK(&rp->rio_ifp->if_snd); - - while ((m = rio_getq_flow(rp, q, flow, TRUE)) != NULL) { - cnt++; - len += m_pktlen(m); - m_freem(m); - } - - if (packets != NULL) - *packets = cnt; - if (bytes != NULL) - *bytes = len; -} - -void -rio_updateq(rio_t *rp, cqev_t ev) -{ -#pragma unused(rp, ev) - /* nothing for now */ -} - -int -rio_suspendq(rio_t *rp, class_queue_t *q, boolean_t on) -{ -#pragma unused(rp, q, on) - return (ENOTSUP); -} -#endif /* CLASSQ_RIO */ diff --git a/bsd/net/classq/classq_rio.h b/bsd/net/classq/classq_rio.h index fb3c24199..8f0a12a07 100644 --- a/bsd/net/classq/classq_rio.h +++ b/bsd/net/classq/classq_rio.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -73,65 +73,6 @@ extern "C" { */ #define RIO_NDROPPREC 3 /* number of drop precedence values */ -#ifdef BSD_KERNEL_PRIVATE -/* rio flags */ -#define RIOF_ECN4 0x01 /* use packet marking for IPv4 packets */ -#define RIOF_ECN6 0x02 /* use packet marking for IPv6 packets */ -#define RIOF_ECN (RIOF_ECN4 | RIOF_ECN6) -#define RIOF_CLEARDSCP 0x200 /* clear diffserv codepoint */ - -#define RIOF_USERFLAGS \ - (RIOF_ECN4 | RIOF_ECN6 | RIOF_CLEARDSCP) - -typedef struct rio { - /* per drop precedence structure */ - struct dropprec_state { - /* red parameters */ - int inv_pmax; /* inverse of max drop probability */ - int th_min; /* red min threshold */ - int th_max; /* red max threshold */ - - /* variables for internal use */ - int th_min_s; /* th_min scaled by avgshift */ - int th_max_s; /* th_max scaled by avgshift */ - int probd; /* drop probability denominator */ - - int qlen; /* queue length */ - int avg; /* (scaled) queue length average */ - int count; /* packet count since the last */ - /* dropped/marked packet */ - int idle; /* queue was empty */ - int old; /* avg is above th_min */ - struct timeval last; /* timestamp when queue becomes idle */ - } rio_precstate[RIO_NDROPPREC]; - - int rio_wshift; /* log(red_weight) */ - int rio_weight; /* weight for EWMA */ - struct wtab *rio_wtab; /* weight table */ - - int rio_pkttime; /* average packet time in micro sec */ - /* used for idle calibration */ - int rio_flags; /* rio flags */ - struct ifnet *rio_ifp; /* back pointer to ifnet */ - - u_int8_t rio_codepoint; /* codepoint value to tag packets */ - u_int8_t rio_codepointmask; /* codepoint mask bits */ - - struct red_stats q_stats[RIO_NDROPPREC]; /* statistics */ -} rio_t; - -extern void rio_init(void); -extern rio_t *rio_alloc(struct ifnet *, int, struct redparams *, int, int); -extern void rio_destroy(rio_t *); -extern void rio_getstats(rio_t *, struct red_stats *); -extern int rio_addq(rio_t *, class_queue_t *, struct mbuf *, struct pf_mtag *); -extern struct mbuf *rio_getq(rio_t *, class_queue_t *); -extern void rio_purgeq(struct rio *, class_queue_t *, u_int32_t, - u_int32_t *, u_int32_t *); -extern void rio_updateq(rio_t *, cqev_t); -extern int rio_suspendq(rio_t *, class_queue_t *, boolean_t); -#endif /* BSD_KERNEL_PRIVATE */ - #ifdef __cplusplus } #endif diff --git a/bsd/net/classq/classq_sfb.c b/bsd/net/classq/classq_sfb.c index c679ca43f..438abf2c3 100644 --- a/bsd/net/classq/classq_sfb.c +++ b/bsd/net/classq/classq_sfb.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -220,12 +220,6 @@ /* Place the flow control entries in current bin on level 0 */ #define SFB_FC_LEVEL 0 -/* Store SFB hash and flags in the module private scratch space */ -#define pkt_sfb_hash8 pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val8 -#define pkt_sfb_hash16 pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val16 -#define pkt_sfb_hash32 pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32 -#define pkt_sfb_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32 - static unsigned int sfb_size; /* size of zone element */ static struct zone *sfb_zone; /* zone for sfb */ @@ -237,28 +231,29 @@ static struct zone *sfb_fcl_zone; /* zone for sfb_fc_lists */ /* internal function prototypes */ static u_int32_t sfb_random(struct sfb *); -static struct mbuf *sfb_getq_flow(struct sfb *, class_queue_t *, u_int32_t, - boolean_t); +static void *sfb_getq_flow(struct sfb *, class_queue_t *, u_int32_t, boolean_t, + pktsched_pkt_t *); static void sfb_resetq(struct sfb *, cqev_t); static void sfb_calc_holdtime(struct sfb *, u_int64_t); static void sfb_calc_pboxtime(struct sfb *, u_int64_t); static void sfb_calc_hinterval(struct sfb *, u_int64_t *); static void sfb_calc_update_interval(struct sfb *, u_int64_t); static void sfb_swap_bins(struct sfb *, u_int32_t); -static inline int sfb_pcheck(struct sfb *, struct pkthdr *); -static int sfb_penalize(struct sfb *, struct pkthdr *, struct timespec *); +static inline int sfb_pcheck(struct sfb *, uint32_t); +static int sfb_penalize(struct sfb *, uint32_t, uint32_t *, struct timespec *); static void sfb_adjust_bin(struct sfb *, struct sfbbinstats *, struct timespec *, struct timespec *, boolean_t); static void sfb_decrement_bin(struct sfb *, struct sfbbinstats *, struct timespec *, struct timespec *); static void sfb_increment_bin(struct sfb *, struct sfbbinstats *, struct timespec *, struct timespec *); -static inline void sfb_dq_update_bins(struct sfb *, struct pkthdr *, +static inline void sfb_dq_update_bins(struct sfb *, uint32_t, uint32_t, struct timespec *, u_int32_t qsize); -static inline void sfb_eq_update_bins(struct sfb *, struct pkthdr *); -static int sfb_drop_early(struct sfb *, struct pkthdr *, u_int16_t *, +static inline void sfb_eq_update_bins(struct sfb *, uint32_t, uint32_t); +static int sfb_drop_early(struct sfb *, uint32_t, u_int16_t *, struct timespec *); -static boolean_t sfb_bin_addfcentry(struct sfb *, struct pkthdr *); +static boolean_t sfb_bin_addfcentry(struct sfb *, pktsched_pkt_t *, + uint32_t, uint8_t, uint32_t); static void sfb_fclist_append(struct sfb *, struct sfb_fcl *); static void sfb_fclists_clean(struct sfb *sp); static int sfb_bin_mark_or_drop(struct sfb *sp, struct sfbbinstats *bin); @@ -409,7 +404,7 @@ sfb_calc_pboxtime(struct sfb *sp, u_int64_t outbw) static void sfb_calc_hinterval(struct sfb *sp, u_int64_t *t) { - u_int64_t hinterval; + u_int64_t hinterval = 0; struct timespec now; if (t != NULL) { @@ -487,7 +482,7 @@ sfb_alloc(struct ifnet *ifp, u_int32_t qid, u_int32_t qlim, u_int32_t flags) } #endif /* !PF_ECN */ - sfb_resetq(sp, -1); + sfb_resetq(sp, CLASSQ_EV_INIT); return (sp); } @@ -496,7 +491,6 @@ static void sfb_fclist_append(struct sfb *sp, struct sfb_fcl *fcl) { IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - VERIFY(STAILQ_EMPTY(&fcl->fclist) || fcl->cnt > 0); sp->sfb_stats.flow_feedback += fcl->cnt; fcl->cnt = 0; @@ -670,11 +664,12 @@ sfb_swap_bins(struct sfb *sp, u_int32_t len) } static inline int -sfb_pcheck(struct sfb *sp, struct pkthdr *pkt) +sfb_pcheck(struct sfb *sp, uint32_t pkt_sfb_hash) { #if SFB_LEVELS != 2 int i, n; #endif /* SFB_LEVELS != 2 */ + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; int s; s = sp->sfb_current; @@ -689,17 +684,17 @@ sfb_pcheck(struct sfb *sp, struct pkthdr *pkt) * Level 0: bin index at [0] for set 0; [2] for set 1 * Level 1: bin index at [1] for set 0; [3] for set 1 */ - if (SFB_BINST(sp, 0, SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]), + if (SFB_BINST(sp, 0, SFB_BINMASK(pkt_sfb_hash8[(s << 1)]), s)->pmark < SFB_PMARK_TH || - SFB_BINST(sp, 1, SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]), + SFB_BINST(sp, 1, SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]), s)->pmark < SFB_PMARK_TH) return (0); #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); + n = SFB_BINMASK(pkt_sfb_hash8[i]); else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); if (SFB_BINST(sp, i, n, s)->pmark < SFB_PMARK_TH) return (0); @@ -709,12 +704,14 @@ sfb_pcheck(struct sfb *sp, struct pkthdr *pkt) } static int -sfb_penalize(struct sfb *sp, struct pkthdr *pkt, struct timespec *now) +sfb_penalize(struct sfb *sp, uint32_t pkt_sfb_hash, uint32_t *pkt_sfb_flags, + struct timespec *now) { struct timespec delta = { 0, 0 }; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; /* If minimum pmark of current bins is < SFB_PMARK_TH, we're done */ - if (!sfb_ratelimit || !sfb_pcheck(sp, pkt)) + if (!sfb_ratelimit || !sfb_pcheck(sp, pkt_sfb_hash)) return (0); net_timersub(now, &sp->sfb_pboxfreeze, &delta); @@ -733,22 +730,22 @@ sfb_penalize(struct sfb *sp, struct pkthdr *pkt, struct timespec *now) */ #if SFB_LEVELS == 2 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(w << 1)]); + n = SFB_BINMASK(pkt_sfb_hash8[(w << 1)]); bin = SFB_BINST(sp, 0, n, w); if (bin->pkts >= sp->sfb_allocation) sfb_increment_bin(sp, bin, SFB_BINFT(sp, 0, n, w), now); /* Level 0: bin index at [1] for set 0; [3] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(w << 1) + 1]); + n = SFB_BINMASK(pkt_sfb_hash8[(w << 1) + 1]); bin = SFB_BINST(sp, 1, n, w); if (bin->pkts >= sp->sfb_allocation) sfb_increment_bin(sp, bin, SFB_BINFT(sp, 1, n, w), now); #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { if (w == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); + n = SFB_BINMASK(pkt_sfb_hash8[i]); else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); bin = SFB_BINST(sp, i, n, w); if (bin->pkts >= sp->sfb_allocation) { @@ -761,7 +758,7 @@ sfb_penalize(struct sfb *sp, struct pkthdr *pkt, struct timespec *now) } /* non-conformant or else misclassified flow; queue it anyway */ - pkt->pkt_sfb_flags |= SFB_PKT_PBOX; + *pkt_sfb_flags |= SFB_PKT_PBOX; *(&sp->sfb_pboxfreeze) = *now; return (0); @@ -807,7 +804,7 @@ sfb_increment_bin(struct sfb *sp, struct sfbbinstats *bin, struct timespec *ft, } static inline void -sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, +sfb_dq_update_bins(struct sfb *sp, uint32_t pkt_sfb_hash, uint32_t pkt_len, struct timespec *now, u_int32_t qsize) { #if SFB_LEVELS != 2 || SFB_FC_LEVEL != 0 @@ -816,6 +813,7 @@ sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, struct sfbbinstats *bin; int s, n; struct sfb_fcl *fcl = NULL; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); @@ -825,12 +823,12 @@ sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, */ #if SFB_LEVELS == 2 && SFB_FC_LEVEL == 0 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1)]); bin = SFB_BINST(sp, 0, n, s); - VERIFY(bin->pkts > 0 && bin->bytes >= (u_int32_t)pkt->len); + VERIFY(bin->pkts > 0 && bin->bytes >= pkt_len); bin->pkts--; - bin->bytes -= pkt->len; + bin->bytes -= pkt_len; if (bin->pkts == 0) sfb_decrement_bin(sp, bin, SFB_BINFT(sp, 0, n, s), now); @@ -850,26 +848,26 @@ sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, fcl = NULL; /* Level 1: bin index at [1] for set 0; [3] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]); bin = SFB_BINST(sp, 1, n, s); - VERIFY(bin->pkts > 0 && bin->bytes >= (u_int64_t)pkt->len); + VERIFY(bin->pkts > 0 && bin->bytes >= (u_int64_t)pkt_len); bin->pkts--; - bin->bytes -= pkt->len; + bin->bytes -= pkt_len; if (bin->pkts == 0) sfb_decrement_bin(sp, bin, SFB_BINFT(sp, 1, n, s), now); #else /* SFB_LEVELS != 2 || SFB_FC_LEVEL != 0 */ for (i = 0; i < SFB_LEVELS; i++) { if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); + n = SFB_BINMASK(pkt_sfb_hash8[i]); else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); bin = SFB_BINST(sp, i, n, s); - VERIFY(bin->pkts > 0 && bin->bytes >= pkt->len); + VERIFY(bin->pkts > 0 && bin->bytes >= pkt_len); bin->pkts--; - bin->bytes -= pkt->len; + bin->bytes -= pkt_len; if (bin->pkts == 0) sfb_decrement_bin(sp, bin, SFB_BINFT(sp, i, n, s), now); @@ -890,13 +888,14 @@ sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, } static inline void -sfb_eq_update_bins(struct sfb *sp, struct pkthdr *pkt) +sfb_eq_update_bins(struct sfb *sp, uint32_t pkt_sfb_hash, uint32_t pkt_len) { #if SFB_LEVELS != 2 int i, n; #endif /* SFB_LEVELS != 2 */ int s; struct sfbbinstats *bin; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); @@ -906,44 +905,42 @@ sfb_eq_update_bins(struct sfb *sp, struct pkthdr *pkt) #if SFB_LEVELS == 2 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ bin = SFB_BINST(sp, 0, - SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]), s); + SFB_BINMASK(pkt_sfb_hash8[(s << 1)]), s); bin->pkts++; - bin->bytes += pkt->len; + bin->bytes += pkt_len; /* Level 1: bin index at [1] for set 0; [3] for set 1 */ bin = SFB_BINST(sp, 1, - SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]), s); + SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]), s); bin->pkts++; - bin->bytes += pkt->len; + bin->bytes += pkt_len; #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); + n = SFB_BINMASK(pkt_sfb_hash8[i]); else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); bin = SFB_BINST(sp, i, n, s); bin->pkts++; - bin->bytes += pkt->len; + bin->bytes += pkt_len; } #endif /* SFB_LEVELS != 2 */ } static boolean_t -sfb_bin_addfcentry(struct sfb *sp, struct pkthdr *pkt) +sfb_bin_addfcentry(struct sfb *sp, pktsched_pkt_t *pkt, uint32_t pkt_sfb_hash, + uint8_t flowsrc, uint32_t flowid) { struct flowadv_fcentry *fce; - u_int32_t flowsrc, flowid; struct sfb_fcl *fcl; int s; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); - flowsrc = pkt->pkt_flowsrc; - flowid = pkt->pkt_flowid; - if (flowid == 0) { sp->sfb_stats.null_flowid++; return (FALSE); @@ -953,9 +950,9 @@ sfb_bin_addfcentry(struct sfb *sp, struct pkthdr *pkt) * Use value at index 0 for set 0 and * value at index 2 for set 1 */ - fcl = SFB_FC_LIST(sp, SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)])); + fcl = SFB_FC_LIST(sp, SFB_BINMASK(pkt_sfb_hash8[(s << 1)])); STAILQ_FOREACH(fce, &fcl->fclist, fce_link) { - if (fce->fce_flowsrc == flowsrc && + if ((uint8_t)fce->fce_flowsrc_type == flowsrc && fce->fce_flowid == flowid) { /* Already on flow control list; just return */ return (TRUE); @@ -963,10 +960,8 @@ sfb_bin_addfcentry(struct sfb *sp, struct pkthdr *pkt) } IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - fce = flowadv_alloc_entry(M_WAITOK); + fce = pktsched_alloc_fcentry(pkt, sp->sfb_ifp, M_WAITOK); if (fce != NULL) { - fce->fce_flowsrc = flowsrc; - fce->fce_flowid = flowid; STAILQ_INSERT_TAIL(&fcl->fclist, fce, fce_link); fcl->cnt++; sp->sfb_stats.flow_controlled++; @@ -1003,7 +998,7 @@ sfb_bin_mark_or_drop(struct sfb *sp, struct sfbbinstats *bin) * early-drop probability is kept in pmark of each bin of the flow */ static int -sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, +sfb_drop_early(struct sfb *sp, uint32_t pkt_sfb_hash, u_int16_t *pmin, struct timespec *now) { #if SFB_LEVELS != 2 @@ -1011,6 +1006,7 @@ sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, #endif /* SFB_LEVELS != 2 */ struct sfbbinstats *bin; int s, n, ret = 0; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); @@ -1022,7 +1018,7 @@ sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, */ #if SFB_LEVELS == 2 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1)]); bin = SFB_BINST(sp, 0, n, s); if (*pmin > (u_int16_t)bin->pmark) *pmin = (u_int16_t)bin->pmark; @@ -1035,7 +1031,7 @@ sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, ret = sfb_bin_mark_or_drop(sp, bin); /* Level 1: bin index at [1] for set 0; [3] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]); bin = SFB_BINST(sp, 1, n, s); if (*pmin > (u_int16_t)bin->pmark) *pmin = (u_int16_t)bin->pmark; @@ -1045,9 +1041,9 @@ sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); + n = SFB_BINMASK(pkt_sfb_hash8[i]); else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); bin = SFB_BINST(sp, i, n, s); if (*pmin > (u_int16_t)bin->pmark) @@ -1095,31 +1091,45 @@ sfb_detect_dequeue_stall(struct sfb *sp, class_queue_t *q, #define DTYPE_EARLY 2 /* an "unforced" (early) drop */ int -sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) +sfb_addq(struct sfb *sp, class_queue_t *q, pktsched_pkt_t *pkt, + struct pf_mtag *t) { #if !PF_ECN #pragma unused(t) #endif /* !PF_ECN */ - struct pkthdr *pkt = &m->m_pkthdr; struct timespec now; int droptype, s; - u_int16_t pmin; + uint16_t pmin; int fc_adv = 0; int ret = CLASSQEQ_SUCCESS; - u_int32_t maxqsize = 0; + uint32_t maxqsize = 0; + uint64_t *pkt_timestamp; + uint32_t *pkt_sfb_hash; + uint16_t *pkt_sfb_hash16; + uint32_t *pkt_sfb_flags; + uint32_t pkt_flowid; + uint32_t *pkt_flags; + uint8_t pkt_proto, pkt_flowsrc; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); - /* See comments in */ - VERIFY(!(pkt->pkt_flags & PKTF_PRIV_GUARDED)); - pkt->pkt_flags |= PKTF_PRIV_GUARDED; + pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, &pkt_flowid, + &pkt_flowsrc, &pkt_proto, NULL); + pkt_sfb_hash = pktsched_get_pkt_sfb_vars(pkt, &pkt_sfb_flags); + pkt_sfb_hash16 = (uint16_t *)pkt_sfb_hash; - if (pkt->pkt_timestamp > 0) { - net_nsectimer(&pkt->pkt_timestamp, &now); + if (pkt->pktsched_ptype == QP_MBUF) { + /* See comments in */ + VERIFY(!(*pkt_flags & PKTF_PRIV_GUARDED)); + *pkt_flags |= PKTF_PRIV_GUARDED; + } + + if (*pkt_timestamp > 0) { + net_nsectimer(pkt_timestamp, &now); } else { nanouptime(&now); - net_timernsec(&now, &pkt->pkt_timestamp); + net_timernsec(&now, pkt_timestamp); } /* time to swap the bins? */ @@ -1142,12 +1152,12 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) if (qsize(q) == 0 && !net_timerisset(&sp->sfb_getqtime)) *(&sp->sfb_getqtime) = *(&now); - pkt->pkt_sfb_flags = 0; - pkt->pkt_sfb_hash16[s] = - (SFB_HASH(&pkt->pkt_flowid, sizeof (pkt->pkt_flowid), + *pkt_sfb_flags = 0; + pkt_sfb_hash16[s] = + (SFB_HASH(&pkt_flowid, sizeof (pkt_flowid), (*sp->sfb_bins)[s].fudge) & SFB_HASHMASK); - pkt->pkt_sfb_hash16[s ^ 1] = - (SFB_HASH(&pkt->pkt_flowid, sizeof (pkt->pkt_flowid), + pkt_sfb_hash16[s ^ 1] = + (SFB_HASH(&pkt_flowid, sizeof (pkt_flowid), (*sp->sfb_bins)[s ^ 1].fudge) & SFB_HASHMASK); /* check if the queue has been stalled */ @@ -1155,21 +1165,23 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) /* see if we drop early */ droptype = DTYPE_NODROP; - if (sfb_drop_early(sp, pkt, &pmin, &now)) { + if (sfb_drop_early(sp, *pkt_sfb_hash, &pmin, &now)) { /* flow control, mark or drop by sfb */ if ((sp->sfb_flags & SFBF_FLOWCTL) && - (pkt->pkt_flags & PKTF_FLOW_ADV)) { + (*pkt_flags & PKTF_FLOW_ADV)) { fc_adv = 1; /* drop all during suspension or for non-TCP */ if ((sp->sfb_flags & SFBF_SUSPENDED) || - pkt->pkt_proto != IPPROTO_TCP) { + pkt_proto != IPPROTO_TCP) { droptype = DTYPE_EARLY; sp->sfb_stats.drop_early++; } } #if PF_ECN + /* XXX: only supported for mbuf */ else if ((sp->sfb_flags & SFBF_ECN) && - (pkt->pkt_proto == IPPROTO_TCP) && /* only for TCP */ + (pkt->pktsched_ptype == QP_MBUF) && + (pkt_proto == IPPROTO_TCP) && /* only for TCP */ ((sfb_random(sp) & SFB_MAX_PMARK) <= pmin) && mark_ecn(m, t, sp->sfb_flags) && !(sp->sfb_flags & SFBF_SUSPENDED)) { @@ -1185,7 +1197,8 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) } /* non-responsive flow penalty? */ - if (droptype == DTYPE_NODROP && sfb_penalize(sp, pkt, &now)) { + if (droptype == DTYPE_NODROP && sfb_penalize(sp, *pkt_sfb_hash, + pkt_sfb_flags, &now)) { droptype = DTYPE_FORCED; sp->sfb_stats.drop_pbox++; } @@ -1200,9 +1213,9 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) * drop */ if (droptype == DTYPE_NODROP && qlen(q) >= maxqsize) { - if (pkt->pkt_proto == IPPROTO_TCP && + if (pkt_proto == IPPROTO_TCP && qlen(q) < (maxqsize + (maxqsize >> 1)) && - ((pkt->pkt_flags & PKTF_TCP_REXMT) || + ((*pkt_flags & PKTF_TCP_REXMT) || (sp->sfb_flags & SFBF_LAST_PKT_DROPPED))) { /* * At some level, dropping packets will make the @@ -1221,30 +1234,32 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) } if (fc_adv == 1 && droptype != DTYPE_FORCED && - sfb_bin_addfcentry(sp, pkt)) { + sfb_bin_addfcentry(sp, pkt, *pkt_sfb_hash, pkt_flowsrc, + pkt_flowid)) { /* deliver flow control advisory error */ if (droptype == DTYPE_NODROP) { ret = CLASSQEQ_SUCCESS_FC; VERIFY(!(sp->sfb_flags & SFBF_SUSPENDED)); } else if (sp->sfb_flags & SFBF_SUSPENDED) { - /* dropped due to suspension */ - ret = CLASSQEQ_DROPPED_SP; + /* drop due to suspension */ + ret = CLASSQEQ_DROP_SP; } else { - /* dropped due to flow-control */ - ret = CLASSQEQ_DROPPED_FC; + /* drop due to flow-control */ + ret = CLASSQEQ_DROP_FC; } } /* if successful enqueue this packet, else drop it */ if (droptype == DTYPE_NODROP) { - _addq(q, m); + VERIFY(pkt->pktsched_ptype == qptype(q)); + _addq(q, pkt->pktsched_pkt); } else { IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - m_freem(m); - return ((ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROPPED); + return ((ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROP); } - if (!(pkt->pkt_sfb_flags & SFB_PKT_PBOX)) - sfb_eq_update_bins(sp, pkt); + if (!(*pkt_sfb_flags & SFB_PKT_PBOX)) + sfb_eq_update_bins(sp, *pkt_sfb_hash, + pktsched_get_pkt_len(pkt)); else sp->sfb_stats.pbox_packets++; @@ -1252,12 +1267,17 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) return (ret); } -static struct mbuf * -sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) +static void * +sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge, + pktsched_pkt_t *pkt) { struct timespec now; - struct mbuf *m; - struct pkthdr *pkt; + classq_pkt_type_t ptype; + uint64_t *pkt_timestamp; + uint32_t *pkt_flags; + uint32_t *pkt_sfb_flags; + uint32_t *pkt_sfb_hash; + void *p; if (!purge && (sp->sfb_flags & SFBF_SUSPENDED)) return (NULL); @@ -1265,16 +1285,21 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) nanouptime(&now); /* flow of 0 means head of queue */ - if ((m = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) { + if ((p = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) { if (!purge) net_timerclear(&sp->sfb_getqtime); return (NULL); } - VERIFY(m->m_flags & M_PKTHDR); + ptype = qptype(q); + pktsched_pkt_encap(pkt, ptype, p); + pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, NULL, + NULL, NULL, NULL); + pkt_sfb_hash = pktsched_get_pkt_sfb_vars(pkt, &pkt_sfb_flags); - pkt = &m->m_pkthdr; - VERIFY(pkt->pkt_flags & PKTF_PRIV_GUARDED); + /* See comments in */ + if (ptype == QP_MBUF) + VERIFY(*pkt_flags & PKTF_PRIV_GUARDED); if (!purge) { /* calculate EWMA of dequeues */ @@ -1306,8 +1331,8 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) if (!purge && SFB_QUEUE_DELAYBASED(sp)) { u_int64_t dequeue_ns, queue_delay = 0; net_timernsec(&now, &dequeue_ns); - if (dequeue_ns > pkt->pkt_timestamp) - queue_delay = dequeue_ns - pkt->pkt_timestamp; + if (dequeue_ns > *pkt_timestamp) + queue_delay = dequeue_ns - *pkt_timestamp; if (sp->sfb_min_qdelay == 0 || (queue_delay > 0 && queue_delay < sp->sfb_min_qdelay)) @@ -1326,7 +1351,7 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) sp->sfb_min_qdelay = 0; } } - pkt->pkt_timestamp = 0; + *pkt_timestamp = 0; /* * Clearpkts are the ones which were in the queue when the hash @@ -1337,18 +1362,20 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) * this reason. A rule of thumb is to set it to K*D, where D is * the time taken to drain queue. */ - if (pkt->pkt_sfb_flags & SFB_PKT_PBOX) { - pkt->pkt_sfb_flags &= ~SFB_PKT_PBOX; + if (*pkt_sfb_flags & SFB_PKT_PBOX) { + *pkt_sfb_flags &= ~SFB_PKT_PBOX; if (sp->sfb_clearpkts > 0) sp->sfb_clearpkts--; } else if (sp->sfb_clearpkts > 0) { sp->sfb_clearpkts--; } else { - sfb_dq_update_bins(sp, pkt, &now, qsize(q)); + sfb_dq_update_bins(sp, *pkt_sfb_hash, pktsched_get_pkt_len(pkt), + &now, qsize(q)); } /* See comments in */ - pkt->pkt_flags &= ~PKTF_PRIV_GUARDED; + if (ptype == QP_MBUF) + *pkt_flags &= ~PKTF_PRIV_GUARDED; /* * If the queue becomes empty before the update interval, reset @@ -1361,14 +1388,13 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) net_timerclear(&sp->sfb_update_time); net_timerclear(&sp->sfb_getqtime); } - - return (m); + return (p); } -struct mbuf * -sfb_getq(struct sfb *sp, class_queue_t *q) +void +sfb_getq(struct sfb *sp, class_queue_t *q, pktsched_pkt_t *pkt) { - return (sfb_getq_flow(sp, q, 0, FALSE)); + sfb_getq_flow(sp, q, 0, FALSE, pkt); } void @@ -1376,14 +1402,13 @@ sfb_purgeq(struct sfb *sp, class_queue_t *q, u_int32_t flow, u_int32_t *packets, u_int32_t *bytes) { u_int32_t cnt = 0, len = 0; - struct mbuf *m; + pktsched_pkt_t pkt; IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - - while ((m = sfb_getq_flow(sp, q, flow, TRUE)) != NULL) { + while (sfb_getq_flow(sp, q, flow, TRUE, &pkt) != NULL) { cnt++; - len += m_pktlen(m); - m_freem(m); + len += pktsched_get_pkt_len(&pkt); + pktsched_free_pkt(&pkt); } if (packets != NULL) diff --git a/bsd/net/classq/classq_sfb.h b/bsd/net/classq/classq_sfb.h index 2a28a7192..480ee5d73 100644 --- a/bsd/net/classq/classq_sfb.h +++ b/bsd/net/classq/classq_sfb.h @@ -104,7 +104,7 @@ struct sfb_fcl { #define SFBF_FLOWCTL 0x04 /* enable flow control advisories */ #define SFBF_DELAYBASED 0x08 /* queueing is delay based */ #define SFBF_DELAYHIGH 0x10 /* Estimated delay is greater than target */ -#define SFBF_LAST_PKT_DROPPED 0x20 /* Last packet dropped */ +#define SFBF_LAST_PKT_DROPPED 0x20 /* Last packet dropped */ #define SFBF_SUSPENDED 0x1000 /* queue is suspended */ #define SFBF_USERFLAGS \ @@ -126,7 +126,7 @@ typedef struct sfb { /* target queue delay and interval for queue sizing */ u_int64_t sfb_target_qdelay; struct timespec sfb_update_interval; - u_int64_t sfb_fc_threshold; /* for flow control feedback */ + u_int64_t sfb_fc_threshold; /* for flow control feedback */ /* variables for computing estimated delay of the queue */ u_int64_t sfb_min_qdelay; @@ -154,9 +154,9 @@ typedef struct sfb { extern void sfb_init(void); extern struct sfb *sfb_alloc(struct ifnet *, u_int32_t, u_int32_t, u_int32_t); extern void sfb_destroy(struct sfb *); -extern int sfb_addq(struct sfb *, class_queue_t *, struct mbuf *, +extern int sfb_addq(struct sfb *, class_queue_t *, pktsched_pkt_t *, struct pf_mtag *); -extern struct mbuf *sfb_getq(struct sfb *, class_queue_t *); +extern void sfb_getq(struct sfb *, class_queue_t *, pktsched_pkt_t *); extern void sfb_purgeq(struct sfb *, class_queue_t *, u_int32_t, u_int32_t *, u_int32_t *); extern void sfb_getstats(struct sfb *, struct sfb_stats *); diff --git a/bsd/net/classq/classq_subr.c b/bsd/net/classq/classq_subr.c index 55f42daf6..669e45ad6 100644 --- a/bsd/net/classq/classq_subr.c +++ b/bsd/net/classq/classq_subr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2015 Apple Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,32 +40,18 @@ #include #include #include -#if CLASSQ_RED -#include -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO -#include -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE -#include -#endif /* CLASSQ_BLUE */ #include #include #include #include -#if PF_ALTQ -#include -#endif /* PF_ALTQ */ static errno_t ifclassq_dequeue_common(struct ifclassq *, mbuf_svc_class_t, - u_int32_t, u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, - u_int32_t *, boolean_t); -static struct mbuf *ifclassq_poll_common(struct ifclassq *, - mbuf_svc_class_t, boolean_t); -static struct mbuf *ifclassq_tbr_dequeue_common(struct ifclassq *, int, - mbuf_svc_class_t, boolean_t); + u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *, + boolean_t, classq_pkt_type_t *); +static void *ifclassq_tbr_dequeue_common(struct ifclassq *, mbuf_svc_class_t, + boolean_t, classq_pkt_type_t *); static u_int64_t ifclassq_target_qdelay = 0; SYSCTL_QUAD(_net_classq, OID_AUTO, target_qdelay, CTLFLAG_RW|CTLFLAG_LOCKED, @@ -85,21 +71,12 @@ classq_init(void) _CASSERT(MBUF_SC_BE == 0); _CASSERT(IFCQ_SC_MAX == MBUF_SC_MAX_CLASSES); -#if CLASSQ_RED - red_init(); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - rio_init(); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - blue_init(); -#endif /* CLASSQ_BLUE */ sfb_init(); fq_codel_scheduler_init(); if (!PE_parse_boot_argn("fq_codel", &ifclassq_sched_fq_codel, sizeof (ifclassq_sched_fq_codel))) - ifclassq_sched_fq_codel = 0; + ifclassq_sched_fq_codel = 1; } int @@ -147,26 +124,7 @@ ifclassq_setup(struct ifnet *ifp, u_int32_t sflags, boolean_t reuse) if (err == 0) ifq->ifcq_flags = (IFCQF_READY | IFCQF_ENABLED); } - -#if PF_ALTQ - ifq->ifcq_drain = 0; - IFCQ_ALTQ(ifq)->altq_ifcq = ifq; - VERIFY(IFCQ_ALTQ(ifq)->altq_type == ALTQT_NONE); - VERIFY(IFCQ_ALTQ(ifq)->altq_flags == 0); - VERIFY(IFCQ_ALTQ(ifq)->altq_disc == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_enqueue == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue_sc == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_request == NULL); - - if ((ifp->if_eflags & IFEF_TXSTART) && - ifp->if_output_sched_model != IFNET_SCHED_MODEL_DRIVER_MANAGED) - ALTQ_SET_READY(IFCQ_ALTQ(ifq)); - else - ALTQ_CLEAR_READY(IFCQ_ALTQ(ifq)); -#endif /* PF_ALTQ */ IFCQ_UNLOCK(ifq); - return (err); } @@ -176,24 +134,6 @@ ifclassq_teardown(struct ifnet *ifp) struct ifclassq *ifq = &ifp->if_snd; IFCQ_LOCK(ifq); -#if PF_ALTQ - if (ALTQ_IS_READY(IFCQ_ALTQ(ifq))) { - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) - altq_disable(IFCQ_ALTQ(ifq)); - if (ALTQ_IS_ATTACHED(IFCQ_ALTQ(ifq))) - altq_detach(IFCQ_ALTQ(ifq)); - IFCQ_ALTQ(ifq)->altq_flags = 0; - } - ifq->ifcq_drain = 0; - IFCQ_ALTQ(ifq)->altq_ifcq = NULL; - VERIFY(IFCQ_ALTQ(ifq)->altq_type == ALTQT_NONE); - VERIFY(IFCQ_ALTQ(ifq)->altq_flags == 0); - VERIFY(IFCQ_ALTQ(ifq)->altq_disc == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_enqueue == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue_sc == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_request == NULL); -#endif /* PF_ALTQ */ if (IFCQ_IS_READY(ifq)) { if (IFCQ_TBR_IS_ENABLED(ifq)) { @@ -228,6 +168,7 @@ int ifclassq_pktsched_setup(struct ifclassq *ifq) { struct ifnet *ifp = ifq->ifcq_ifp; + classq_pkt_type_t ptype = QP_MBUF; int err = 0; IFCQ_LOCK_ASSERT_HELD(ifq); @@ -235,21 +176,27 @@ ifclassq_pktsched_setup(struct ifclassq *ifq) switch (ifp->if_output_sched_model) { case IFNET_SCHED_MODEL_DRIVER_MANAGED: - err = pktsched_setup(ifq, PKTSCHEDT_TCQ, ifq->ifcq_sflags); + if (ifclassq_sched_fq_codel != 0) { + err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL, + ifq->ifcq_sflags, ptype); + } else { + err = pktsched_setup(ifq, PKTSCHEDT_TCQ, + ifq->ifcq_sflags, ptype); + } break; case IFNET_SCHED_MODEL_NORMAL: if (ifclassq_sched_fq_codel != 0) { err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL, - ifq->ifcq_sflags); + ifq->ifcq_sflags, ptype); } else { err = pktsched_setup(ifq, PKTSCHEDT_QFQ, - ifq->ifcq_sflags); + ifq->ifcq_sflags, ptype); } break; case IFNET_SCHED_MODEL_FQ_CODEL: err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL, - ifq->ifcq_sflags); + ifq->ifcq_sflags, ptype); break; default: VERIFY(0); @@ -295,74 +242,114 @@ ifclassq_get_len(struct ifclassq *ifq, mbuf_svc_class_t sc, u_int32_t *packets, return (err); } +inline void +ifclassq_set_packet_metadata(struct ifclassq *ifq, struct ifnet *ifp, + void *p, classq_pkt_type_t ptype) +{ + if (!IFNET_IS_CELLULAR(ifp)) + return; + + switch (ptype) { + case QP_MBUF: { + struct mbuf *m = p; + m->m_pkthdr.pkt_flags |= PKTF_VALID_UNSENT_DATA; + m->m_pkthdr.bufstatus_if = IFCQ_BYTES(ifq); + m->m_pkthdr.bufstatus_sndbuf = ifp->if_sndbyte_unsent; + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } +} + errno_t -ifclassq_enqueue(struct ifclassq *ifq, struct mbuf *m) +ifclassq_enqueue(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype, + boolean_t *pdrop) { errno_t err; - IFCQ_LOCK_SPIN(ifq); + switch (ptype) { + case QP_MBUF: + IFCQ_LOCK_SPIN(ifq); + break; -#if PF_ALTQ - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) { - ALTQ_ENQUEUE(IFCQ_ALTQ(ifq), m, err); - } else { - u_int32_t qlen = IFCQ_LEN(ifq); - IFCQ_ENQUEUE(ifq, m, err); - if (IFCQ_LEN(ifq) > qlen) - ifq->ifcq_drain += (IFCQ_LEN(ifq) - qlen); + default: + IFCQ_LOCK(ifq); + break; } -#else /* !PF_ALTQ */ - IFCQ_ENQUEUE(ifq, m, err); -#endif /* PF_ALTQ */ + IFCQ_ENQUEUE(ifq, p, ptype, err, pdrop); IFCQ_UNLOCK(ifq); - return (err); } errno_t ifclassq_dequeue(struct ifclassq *ifq, u_int32_t pkt_limit, - u_int32_t byte_limit, struct mbuf **head, - struct mbuf **tail, u_int32_t *cnt, u_int32_t *len) + u_int32_t byte_limit, void **head, void **tail, + u_int32_t *cnt, u_int32_t *len, classq_pkt_type_t *ptype) { return (ifclassq_dequeue_common(ifq, MBUF_SC_UNSPEC, pkt_limit, - byte_limit, head, tail, cnt, len, FALSE)); + byte_limit, head, tail, cnt, len, FALSE, ptype)); } errno_t ifclassq_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc, - u_int32_t pkt_limit, struct mbuf **head, struct mbuf **tail, - u_int32_t *cnt, u_int32_t *len) + u_int32_t pkt_limit, u_int32_t byte_limit, void **head, void **tail, + u_int32_t *cnt, u_int32_t *len, classq_pkt_type_t *ptype) { - return (ifclassq_dequeue_common(ifq, sc, pkt_limit, - CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, head, tail, cnt, len, TRUE)); + return (ifclassq_dequeue_common(ifq, sc, pkt_limit, byte_limit, + head, tail, cnt, len, TRUE, ptype)); } static errno_t ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc, - u_int32_t pkt_limit, u_int32_t byte_limit, struct mbuf **head, - struct mbuf **tail, u_int32_t *cnt, u_int32_t *len, boolean_t drvmgt) + u_int32_t pkt_limit, u_int32_t byte_limit, void **head, + void **tail, u_int32_t *cnt, u_int32_t *len, boolean_t drvmgt, + classq_pkt_type_t *ptype) { struct ifnet *ifp = ifq->ifcq_ifp; - u_int32_t i = 0, l = 0; - struct mbuf **first, *last; -#if PF_ALTQ - struct ifaltq *altq = IFCQ_ALTQ(ifq); - boolean_t draining; -#endif /* PF_ALTQ */ + u_int32_t i = 0, l = 0, lock_spin = 1 ; + void **first, *last; VERIFY(!drvmgt || MBUF_VALID_SC(sc)); + *ptype = 0; + + + if (IFCQ_TBR_IS_ENABLED(ifq)) + goto dequeue_loop; + /* * If the scheduler support dequeueing multiple packets at the * same time, call that one instead. */ + if (drvmgt && ifq->ifcq_dequeue_sc_multi != NULL) { + int err; - if (ifq->ifcq_dequeue_multi != NULL) { + if (lock_spin) + IFCQ_LOCK_SPIN(ifq); + else + IFCQ_LOCK(ifq); + err = ifq->ifcq_dequeue_sc_multi(ifq, sc, pkt_limit, + byte_limit, head, tail, cnt, len, ptype); + IFCQ_UNLOCK(ifq); + + if (err == 0 && (*head) == NULL) + err = EAGAIN; + return (err); + } else if (ifq->ifcq_dequeue_multi != NULL) { int err; - IFCQ_LOCK_SPIN(ifq); - err = ifq->ifcq_dequeue_multi(ifq, CLASSQDQ_REMOVE, - pkt_limit, byte_limit, head, tail, cnt, len); + + if (lock_spin) + IFCQ_LOCK_SPIN(ifq); + else + IFCQ_LOCK(ifq); + + err = ifq->ifcq_dequeue_multi(ifq, pkt_limit, byte_limit, + head, tail, cnt, len, ptype); IFCQ_UNLOCK(ifq); if (err == 0 && (*head) == NULL) @@ -370,75 +357,50 @@ ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc, return (err); } +dequeue_loop: *head = NULL; first = &(*head); last = NULL; - IFCQ_LOCK_SPIN(ifq); + if (lock_spin) + IFCQ_LOCK_SPIN(ifq); + else + IFCQ_LOCK(ifq); while (i < pkt_limit && l < byte_limit) { -#if PF_ALTQ - u_int32_t qlen; - - qlen = IFCQ_LEN(ifq); - draining = IFCQ_IS_DRAINING(ifq); - - if (drvmgt) { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head); - else if (draining) - IFCQ_DEQUEUE_SC(ifq, sc, *head); - else if (ALTQ_IS_ENABLED(altq)) - ALTQ_DEQUEUE_SC(altq, sc, *head); - else - *head = NULL; - } else { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_DEQUEUE(ifq, *head); - else if (draining) - IFCQ_DEQUEUE(ifq, *head); - else if (ALTQ_IS_ENABLED(altq)) - ALTQ_DEQUEUE(altq, *head); - else - *head = NULL; - } - - if (draining && *head != NULL) { - VERIFY(ifq->ifcq_drain >= (qlen - IFCQ_LEN(ifq))); - ifq->ifcq_drain -= (qlen - IFCQ_LEN(ifq)); - } -#else /* ! PF_ALTQ */ + classq_pkt_type_t tmp_ptype; if (drvmgt) { if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head); + IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head, &tmp_ptype); else - IFCQ_DEQUEUE_SC(ifq, sc, *head); + IFCQ_DEQUEUE_SC(ifq, sc, *head, &tmp_ptype); } else { if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_DEQUEUE(ifq, *head); + IFCQ_TBR_DEQUEUE(ifq, *head, &tmp_ptype); else - IFCQ_DEQUEUE(ifq, *head); + IFCQ_DEQUEUE(ifq, *head, &tmp_ptype); } -#endif /* !PF_ALTQ */ if (*head == NULL) break; - (*head)->m_nextpkt = NULL; - last = *head; + switch (tmp_ptype) { + case QP_MBUF: + (*((mbuf_t *)head))->m_nextpkt = NULL; + last = *head; + l += (*((mbuf_t *)head))->m_pkthdr.len; + ifclassq_set_packet_metadata(ifq, ifp, (*head), + QP_MBUF); + head = (void **)&(*((mbuf_t *)head))->m_nextpkt; + break; - l += (*head)->m_pkthdr.len; -#if MEASURE_BW - (*head)->m_pkthdr.pkt_bwseq = - atomic_add_64_ov(&(ifp->if_bw.cur_seq), m_pktlen(*head)); -#endif /* MEASURE_BW */ - if (IFNET_IS_CELLULAR(ifp)) { - (*head)->m_pkthdr.pkt_flags |= PKTF_VALID_UNSENT_DATA; - (*head)->m_pkthdr.bufstatus_if = IFCQ_BYTES(ifq); - (*head)->m_pkthdr.bufstatus_sndbuf = ifp->if_sndbyte_unsent; + default: + VERIFY(0); + /* NOTREACHED */ } - head = &(*head)->m_nextpkt; + + *ptype = tmp_ptype; i++; } @@ -454,76 +416,11 @@ ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc, return ((*first != NULL) ? 0 : EAGAIN); } -struct mbuf * -ifclassq_poll(struct ifclassq *ifq) -{ - return (ifclassq_poll_common(ifq, MBUF_SC_UNSPEC, FALSE)); -} - -struct mbuf * -ifclassq_poll_sc(struct ifclassq *ifq, mbuf_svc_class_t sc) -{ - return (ifclassq_poll_common(ifq, sc, TRUE)); -} - -static struct mbuf * -ifclassq_poll_common(struct ifclassq *ifq, mbuf_svc_class_t sc, - boolean_t drvmgt) -{ -#if PF_ALTQ - struct ifaltq *altq = IFCQ_ALTQ(ifq); -#endif /* PF_ALTQ */ - struct mbuf *m; - - VERIFY(!drvmgt || MBUF_VALID_SC(sc)); - -#if PF_ALTQ - if (drvmgt) { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_POLL_SC(ifq, sc, m); - else if (IFCQ_IS_DRAINING(ifq)) - IFCQ_POLL_SC(ifq, sc, m); - else if (ALTQ_IS_ENABLED(altq)) - ALTQ_POLL_SC(altq, sc, m); - else - m = NULL; - } else { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_POLL(ifq, m); - else if (IFCQ_IS_DRAINING(ifq)) - IFCQ_POLL(ifq, m); - else if (ALTQ_IS_ENABLED(altq)) - ALTQ_POLL(altq, m); - else - m = NULL; - } -#else /* ! PF_ALTQ */ - if (drvmgt) { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_POLL_SC(ifq, sc, m); - else - IFCQ_POLL_SC(ifq, sc, m); - } else { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_POLL(ifq, m); - else - IFCQ_POLL(ifq, m); - } -#endif /* !PF_ALTQ */ - - return (m); -} - void ifclassq_update(struct ifclassq *ifq, cqev_t ev) { IFCQ_LOCK_ASSERT_HELD(ifq); VERIFY(IFCQ_IS_READY(ifq)); - -#if PF_ALTQ - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) - ALTQ_UPDATE(IFCQ_ALTQ(ifq), ev); -#endif /* PF_ALTQ */ IFCQ_UPDATE(ifq, ev); } @@ -531,13 +428,12 @@ int ifclassq_attach(struct ifclassq *ifq, u_int32_t type, void *discipline, ifclassq_enq_func enqueue, ifclassq_deq_func dequeue, ifclassq_deq_sc_func dequeue_sc, ifclassq_deq_multi_func dequeue_multi, - ifclassq_req_func request) + ifclassq_deq_sc_multi_func dequeue_sc_multi, ifclassq_req_func request) { IFCQ_LOCK_ASSERT_HELD(ifq); VERIFY(ifq->ifcq_disc == NULL); VERIFY(enqueue != NULL); - VERIFY(!(dequeue != NULL && dequeue_sc != NULL)); VERIFY(request != NULL); ifq->ifcq_type = type; @@ -546,6 +442,7 @@ ifclassq_attach(struct ifclassq *ifq, u_int32_t type, void *discipline, ifq->ifcq_dequeue = dequeue; ifq->ifcq_dequeue_sc = dequeue_sc; ifq->ifcq_dequeue_multi = dequeue_multi; + ifq->ifcq_dequeue_sc_multi = dequeue_sc_multi; ifq->ifcq_request = request; return (0); @@ -652,24 +549,25 @@ ifclassq_ev2str(cqev_t ev) #define TBR_SCALE(x) ((int64_t)(x) << TBR_SHIFT) #define TBR_UNSCALE(x) ((x) >> TBR_SHIFT) -struct mbuf * -ifclassq_tbr_dequeue(struct ifclassq *ifq, int op) +void * +ifclassq_tbr_dequeue(struct ifclassq *ifq, classq_pkt_type_t *ptype) { - return (ifclassq_tbr_dequeue_common(ifq, op, MBUF_SC_UNSPEC, FALSE)); + return (ifclassq_tbr_dequeue_common(ifq, MBUF_SC_UNSPEC, FALSE, ptype)); } -struct mbuf * -ifclassq_tbr_dequeue_sc(struct ifclassq *ifq, int op, mbuf_svc_class_t sc) +void * +ifclassq_tbr_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc, + classq_pkt_type_t *ptype) { - return (ifclassq_tbr_dequeue_common(ifq, op, sc, TRUE)); + return (ifclassq_tbr_dequeue_common(ifq, sc, TRUE, ptype)); } -static struct mbuf * -ifclassq_tbr_dequeue_common(struct ifclassq *ifq, int op, - mbuf_svc_class_t sc, boolean_t drvmgt) +static void * +ifclassq_tbr_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc, + boolean_t drvmgt, classq_pkt_type_t *ptype) { struct tb_regulator *tbr; - struct mbuf *m; + void *p; int64_t interval; u_int64_t now; @@ -679,64 +577,46 @@ ifclassq_tbr_dequeue_common(struct ifclassq *ifq, int op, VERIFY(IFCQ_TBR_IS_ENABLED(ifq)); tbr = &ifq->ifcq_tbr; - if (op == CLASSQDQ_REMOVE && tbr->tbr_lastop == CLASSQDQ_POLL) { - /* if this is a remove after poll, bypass tbr check */ - } else { - /* update token only when it is negative */ - if (tbr->tbr_token <= 0) { - now = read_machclk(); - interval = now - tbr->tbr_last; - if (interval >= tbr->tbr_filluptime) { + /* update token only when it is negative */ + if (tbr->tbr_token <= 0) { + now = read_machclk(); + interval = now - tbr->tbr_last; + if (interval >= tbr->tbr_filluptime) { + tbr->tbr_token = tbr->tbr_depth; + } else { + tbr->tbr_token += interval * tbr->tbr_rate; + if (tbr->tbr_token > tbr->tbr_depth) tbr->tbr_token = tbr->tbr_depth; - } else { - tbr->tbr_token += interval * tbr->tbr_rate; - if (tbr->tbr_token > tbr->tbr_depth) - tbr->tbr_token = tbr->tbr_depth; - } - tbr->tbr_last = now; } - /* if token is still negative, don't allow dequeue */ - if (tbr->tbr_token <= 0) - return (NULL); + tbr->tbr_last = now; } + /* if token is still negative, don't allow dequeue */ + if (tbr->tbr_token <= 0) + return (NULL); /* * ifclassq takes precedence over ALTQ queue; * ifcq_drain count is adjusted by the caller. */ -#if PF_ALTQ - if (IFCQ_IS_DRAINING(ifq)) { -#endif /* PF_ALTQ */ - if (op == CLASSQDQ_POLL) { - if (drvmgt) - IFCQ_POLL_SC(ifq, sc, m); - else - IFCQ_POLL(ifq, m); - } else { - if (drvmgt) - IFCQ_DEQUEUE_SC(ifq, sc, m); - else - IFCQ_DEQUEUE(ifq, m); - } -#if PF_ALTQ - } else { - struct ifaltq *altq = IFCQ_ALTQ(ifq); - if (ALTQ_IS_ENABLED(altq)) { - if (drvmgt) - m = (*altq->altq_dequeue_sc)(altq, sc, op); - else - m = (*altq->altq_dequeue)(altq, op); - } else { - m = NULL; + if (drvmgt) + IFCQ_DEQUEUE_SC(ifq, sc, p, ptype); + else + IFCQ_DEQUEUE(ifq, p, ptype); + + if (p != NULL) { + switch (*ptype) { + case QP_MBUF: + tbr->tbr_token -= TBR_SCALE(m_pktlen((mbuf_t)p)); + break; + + + default: + VERIFY(0); + /* NOTREACHED */ } } -#endif /* PF_ALTQ */ - - if (m != NULL && op == CLASSQDQ_REMOVE) - tbr->tbr_token -= TBR_SCALE(m_pktlen(m)); - tbr->tbr_lastop = op; - return (m); + return (p); } /* @@ -840,7 +720,6 @@ ifclassq_tbr_set(struct ifclassq *ifq, struct tb_profile *profile, } tbr->tbr_token = tbr->tbr_depth; tbr->tbr_last = read_machclk(); - tbr->tbr_lastop = CLASSQDQ_REMOVE; if (tbr->tbr_rate > 0 && (ifp->if_flags & IFF_UP)) { struct timespec ts = @@ -874,18 +753,18 @@ ifclassq_tbr_set(struct ifclassq *ifq, struct tb_profile *profile, void ifclassq_calc_target_qdelay(struct ifnet *ifp, u_int64_t *if_target_qdelay) { - u_int64_t target_qdelay = 0; - target_qdelay = IFCQ_TARGET_QDELAY(&ifp->if_snd); + u_int64_t qdelay = 0; + qdelay = IFCQ_TARGET_QDELAY(&ifp->if_snd); if (ifclassq_target_qdelay != 0) - target_qdelay = ifclassq_target_qdelay; + qdelay = ifclassq_target_qdelay; /* * If we do not know the effective bandwidth, use the default * target queue delay. */ - if (target_qdelay == 0) - target_qdelay = IFQ_TARGET_DELAY; + if (qdelay == 0) + qdelay = IFQ_TARGET_DELAY; /* * If a delay has been added to ifnet start callback for @@ -894,9 +773,9 @@ ifclassq_calc_target_qdelay(struct ifnet *ifp, u_int64_t *if_target_qdelay) */ if ((ifp->if_eflags & IFEF_ENQUEUE_MULTI) && ifp->if_start_delay_timeout > 0) - target_qdelay += ifp->if_start_delay_timeout; + qdelay += ifp->if_start_delay_timeout; - *(if_target_qdelay) = target_qdelay; + *(if_target_qdelay) = qdelay; } void diff --git a/bsd/net/classq/if_classq.h b/bsd/net/classq/if_classq.h index de8ddc60c..148426903 100644 --- a/bsd/net/classq/if_classq.h +++ b/bsd/net/classq/if_classq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -34,11 +34,9 @@ #ifdef BSD_KERNEL_PRIVATE #include -/* classq dequeue op arg */ -typedef enum cqdq_op { - CLASSQDQ_REMOVE = 1, /* dequeue mbuf from the queue */ - CLASSQDQ_POLL = 2, /* don't dequeue mbuf from the queue */ -} cqdq_op_t; + +/* maximum number of packets stored across all queues */ +#define IFCQ_DEFAULT_PKT_DROP_LIMIT 2048 /* classq request types */ typedef enum cqrq { @@ -70,10 +68,6 @@ typedef struct cqrq_stat_sc { u_int32_t bytes; /* (out) bytes enqueued */ } cqrq_stat_sc_t; -#if PF_ALTQ -#include -#endif /* PF_ALTQ */ - /* * A token-bucket regulator limits the rate that a network driver can * dequeue packets from the output queue. Modern cards are able to buffer @@ -92,7 +86,6 @@ struct tb_regulator { int64_t tbr_filluptime; /* (scaled) time to fill up bucket */ u_int64_t tbr_last; /* last time token was updated */ - int tbr_lastop; /* last dequeue operation type */ /* needed for poll-and-dequeue */ }; @@ -107,13 +100,16 @@ struct ifclassq; enum cqdq_op; enum cqrq; -typedef int (*ifclassq_enq_func)(struct ifclassq *, struct mbuf *); -typedef struct mbuf *(*ifclassq_deq_func)(struct ifclassq *, enum cqdq_op); -typedef struct mbuf *(*ifclassq_deq_sc_func)(struct ifclassq *, - mbuf_svc_class_t, enum cqdq_op); -typedef int (*ifclassq_deq_multi_func)(struct ifclassq *, enum cqdq_op, - u_int32_t, u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, - u_int32_t *); +typedef int (*ifclassq_enq_func)(struct ifclassq *, void *, classq_pkt_type_t, + boolean_t *); +typedef void *(*ifclassq_deq_func)(struct ifclassq *, classq_pkt_type_t *); +typedef void *(*ifclassq_deq_sc_func)(struct ifclassq *, + mbuf_svc_class_t, classq_pkt_type_t *); +typedef int (*ifclassq_deq_multi_func)(struct ifclassq *, u_int32_t, + u_int32_t, void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *); +typedef int (*ifclassq_deq_sc_multi_func)(struct ifclassq *, + mbuf_svc_class_t, u_int32_t, u_int32_t, void **, void **, + u_int32_t *, u_int32_t *, classq_pkt_type_t *); typedef int (*ifclassq_req_func)(struct ifclassq *, enum cqrq, void *); /* @@ -133,6 +129,7 @@ struct ifclassq { u_int32_t ifcq_sflags; /* scheduler flags */ u_int32_t ifcq_target_qdelay; /* target queue delay */ u_int32_t ifcq_bytes; /* bytes count */ + u_int32_t ifcq_pkt_drop_limit; void *ifcq_disc; /* for scheduler-specific use */ /* * ifcq_disc_slots[] represents the leaf classes configured for the @@ -162,22 +159,13 @@ struct ifclassq { ifclassq_deq_func ifcq_dequeue; ifclassq_deq_sc_func ifcq_dequeue_sc; ifclassq_deq_multi_func ifcq_dequeue_multi; + ifclassq_deq_sc_multi_func ifcq_dequeue_sc_multi; ifclassq_req_func ifcq_request; /* token bucket regulator */ struct tb_regulator ifcq_tbr; /* TBR */ - -#if PF_ALTQ - u_int32_t ifcq_drain; - struct ifaltq ifcq_altq; -#endif /* PF_ALTQ */ }; -#if PF_ALTQ -#define IFCQ_ALTQ(_ifcq) (&(_ifcq)->ifcq_altq) -#define IFCQ_IS_DRAINING(_ifcq) ((_ifcq)->ifcq_drain > 0) -#endif /* PF_ALTQ */ - /* ifcq_flags */ #define IFCQF_READY 0x01 /* ifclassq supports discipline */ #define IFCQF_ENABLED 0x02 /* ifclassq is in use */ @@ -188,17 +176,20 @@ struct ifclassq { #define IFCQ_TBR_IS_ENABLED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_TBR) /* classq enqueue return value */ -#define CLASSQEQ_DROPPED (-1) /* packet dropped (freed) */ -#define CLASSQEQ_SUCCESS 0 /* success, packet enqueued */ -#define CLASSQEQ_SUCCESS_FC 1 /* packet enqueued; */ - /* give flow control feedback */ -#define CLASSQEQ_DROPPED_FC 2 /* packet dropped; */ - /* give flow control feedback */ -#define CLASSQEQ_DROPPED_SP 3 /* packet dropped due to suspension; */ - /* give flow control feedback */ +/* packet has to be dropped */ +#define CLASSQEQ_DROP (-1) +/* packet successfully enqueued */ +#define CLASSQEQ_SUCCESS 0 +/* packet enqueued; give flow control feedback */ +#define CLASSQEQ_SUCCESS_FC 1 +/* packet needs to be dropped due to flowcontrol; give flow control feedback */ +#define CLASSQEQ_DROP_FC 2 +/* packet needs to be dropped due to suspension; give flow control feedback */ +#define CLASSQEQ_DROP_SP 3 /* interface event argument for CLASSQRQ_EVENT */ typedef enum cqev { + CLASSQ_EV_INIT = 0, CLASSQ_EV_LINK_BANDWIDTH = 1, /* link bandwidth has changed */ CLASSQ_EV_LINK_LATENCY = 2, /* link latency has changed */ CLASSQ_EV_LINK_MTU = 3, /* link MTU has changed */ @@ -207,11 +198,7 @@ typedef enum cqev { } cqev_t; #endif /* BSD_KERNEL_PRIVATE */ -#include -#include #include -#include -#include #include #include @@ -225,11 +212,7 @@ struct if_ifclassq_stats { struct pktcntr ifqs_dropcnt; u_int32_t ifqs_scheduler; union { - struct priq_classstats ifqs_priq_stats; - struct fairq_classstats ifqs_fairq_stats; struct tcq_classstats ifqs_tcq_stats; - struct cbq_classstats ifqs_cbq_stats; - struct hfsc_classstats ifqs_hfsc_stats; struct qfq_classstats ifqs_qfq_stats; struct fq_codel_classstats ifqs_fq_codel_stats; }; @@ -244,10 +227,10 @@ struct if_ifclassq_stats { * For ifclassq lock */ #define IFCQ_LOCK_ASSERT_HELD(_ifcq) \ - lck_mtx_assert(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_OWNED) #define IFCQ_LOCK_ASSERT_NOTHELD(_ifcq) \ - lck_mtx_assert(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_NOTOWNED) #define IFCQ_LOCK(_ifcq) \ lck_mtx_lock(&(_ifcq)->ifcq_lock) @@ -266,40 +249,24 @@ struct if_ifclassq_stats { /* * For ifclassq operations */ -#define IFCQ_ENQUEUE(_ifq, _m, _err) do { \ - (_err) = (*(_ifq)->ifcq_enqueue)(_ifq, _m); \ +#define IFCQ_ENQUEUE(_ifq, _p, _t, _err, _drop) do { \ + (_err) = (*(_ifq)->ifcq_enqueue)(_ifq, _p, _t, _drop); \ } while (0) -#define IFCQ_DEQUEUE(_ifq, _m) do { \ - (_m) = (*(_ifq)->ifcq_dequeue)(_ifq, CLASSQDQ_REMOVE); \ +#define IFCQ_DEQUEUE(_ifq, _p, _t) do { \ + (_p) = (*(_ifq)->ifcq_dequeue)(_ifq, _t); \ } while (0) -#define IFCQ_DEQUEUE_SC(_ifq, _sc, _m) do { \ - (_m) = (*(_ifq)->ifcq_dequeue_sc)(_ifq, _sc, CLASSQDQ_REMOVE); \ +#define IFCQ_DEQUEUE_SC(_ifq, _sc, _p, _t) do { \ + (_p) = (*(_ifq)->ifcq_dequeue_sc)(_ifq, _sc, _t); \ } while (0) -#define IFCQ_TBR_DEQUEUE(_ifcq, _m) do { \ - (_m) = ifclassq_tbr_dequeue(_ifcq, CLASSQDQ_REMOVE); \ +#define IFCQ_TBR_DEQUEUE(_ifcq, _p, _t) do { \ + (_p) = ifclassq_tbr_dequeue(_ifcq, _t); \ } while (0) -#define IFCQ_TBR_DEQUEUE_SC(_ifcq, _sc, _m) do { \ - (_m) = ifclassq_tbr_dequeue_sc(_ifcq, CLASSQDQ_REMOVE, _sc); \ -} while (0) - -#define IFCQ_POLL(_ifq, _m) do { \ - (_m) = (*(_ifq)->ifcq_dequeue)(_ifq, CLASSQDQ_POLL); \ -} while (0) - -#define IFCQ_POLL_SC(_ifq, _sc, _m) do { \ - (_m) = (*(_ifq)->ifcq_dequeue_sc)(_ifq, _sc, CLASSQDQ_POLL); \ -} while (0) - -#define IFCQ_TBR_POLL(_ifcq, _m) do { \ - (_m) = ifclassq_tbr_dequeue(_ifcq, CLASSQDQ_POLL); \ -} while (0) - -#define IFCQ_TBR_POLL_SC(_ifcq, _sc, _m) do { \ - (_m) = ifclassq_tbr_dequeue_sc(_ifcq, CLASSQDQ_POLL, _sc); \ +#define IFCQ_TBR_DEQUEUE_SC(_ifcq, _sc, _p, _t) do { \ + (_p) = ifclassq_tbr_dequeue_sc(_ifcq, _sc, _t); \ } while (0) #define IFCQ_PURGE(_ifq) do { \ @@ -349,9 +316,9 @@ struct if_ifclassq_stats { #define IFCQ_SET_MAXLEN(_ifcq, _len) ((_ifcq)->ifcq_maxlen = (_len)) #define IFCQ_TARGET_QDELAY(_ifcq) ((_ifcq)->ifcq_target_qdelay) #define IFCQ_BYTES(_ifcq) ((_ifcq)->ifcq_bytes) -#define IFCQ_INC_BYTES(_ifcq, _len) \ +#define IFCQ_INC_BYTES(_ifcq, _len) \ ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes + (_len)) -#define IFCQ_DEC_BYTES(_ifcq, _len) \ +#define IFCQ_DEC_BYTES(_ifcq, _len) \ ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes - (_len)) #define IFCQ_XMIT_ADD(_ifcq, _pkt, _len) do { \ @@ -362,6 +329,8 @@ struct if_ifclassq_stats { PKTCNTR_ADD(&(_ifcq)->ifcq_dropcnt, _pkt, _len); \ } while (0) +#define IFCQ_PKT_DROP_LIMIT(_ifcq) ((_ifcq)->ifcq_pkt_drop_limit) + extern int ifclassq_setup(struct ifnet *, u_int32_t, boolean_t); extern void ifclassq_teardown(struct ifnet *); extern int ifclassq_pktsched_setup(struct ifclassq *); @@ -369,28 +338,33 @@ extern void ifclassq_set_maxlen(struct ifclassq *, u_int32_t); extern u_int32_t ifclassq_get_maxlen(struct ifclassq *); extern int ifclassq_get_len(struct ifclassq *, mbuf_svc_class_t, u_int32_t *, u_int32_t *); -extern errno_t ifclassq_enqueue(struct ifclassq *, struct mbuf *); +extern errno_t ifclassq_enqueue(struct ifclassq *, void *, classq_pkt_type_t, + boolean_t *); extern errno_t ifclassq_dequeue(struct ifclassq *, u_int32_t, u_int32_t, - struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *); + void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *); extern errno_t ifclassq_dequeue_sc(struct ifclassq *, mbuf_svc_class_t, - u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *); -extern struct mbuf *ifclassq_poll(struct ifclassq *); -extern struct mbuf *ifclassq_poll_sc(struct ifclassq *, mbuf_svc_class_t); + u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *, + classq_pkt_type_t *); +extern void *ifclassq_poll(struct ifclassq *, classq_pkt_type_t *); +extern void *ifclassq_poll_sc(struct ifclassq *, mbuf_svc_class_t, + classq_pkt_type_t *); extern void ifclassq_update(struct ifclassq *, cqev_t); extern int ifclassq_attach(struct ifclassq *, u_int32_t, void *, ifclassq_enq_func, ifclassq_deq_func, ifclassq_deq_sc_func, - ifclassq_deq_multi_func, ifclassq_req_func); + ifclassq_deq_multi_func, ifclassq_deq_sc_multi_func, ifclassq_req_func); extern int ifclassq_detach(struct ifclassq *); extern int ifclassq_getqstats(struct ifclassq *, u_int32_t, void *, u_int32_t *); extern const char *ifclassq_ev2str(cqev_t); extern int ifclassq_tbr_set(struct ifclassq *, struct tb_profile *, boolean_t); -extern struct mbuf *ifclassq_tbr_dequeue(struct ifclassq *, int); -extern struct mbuf *ifclassq_tbr_dequeue_sc(struct ifclassq *, int, - mbuf_svc_class_t); +extern void *ifclassq_tbr_dequeue(struct ifclassq *, classq_pkt_type_t *); +extern void *ifclassq_tbr_dequeue_sc(struct ifclassq *, mbuf_svc_class_t, + classq_pkt_type_t *); extern void ifclassq_calc_target_qdelay(struct ifnet *ifp, u_int64_t *if_target_qdelay); extern void ifclassq_calc_update_interval(u_int64_t *update_interval); +extern void ifclassq_set_packet_metadata(struct ifclassq *ifq, + struct ifnet *ifp, void *p, classq_pkt_type_t ptype); #endif /* BSD_KERNEL_PRIVATE */ #endif /* PRIVATE */ diff --git a/bsd/net/content_filter.c b/bsd/net/content_filter.c index 68cdae1e8..4a685a80a 100644 --- a/bsd/net/content_filter.c +++ b/bsd/net/content_filter.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -32,7 +32,7 @@ * also kept in kernel buffer until the user space agents makes a pass or drop * decision. This unidirectional flow of content avoids unnecessary data copies * back to the kernel. - * * + * * A user space filter agent opens a kernel control socket with the name * CONTENT_FILTER_CONTROL_NAME to attach to the socket content filter subsystem. * When connected, a "struct content_filter" is created and set as the @@ -54,7 +54,7 @@ * NECP FILTER CONTROL UNIT * * A user space filter agent uses the Network Extension Control Policy (NECP) - * database specify which TCP/IP sockets needs to be filtered. The NECP + * database to specify which TCP/IP sockets need to be filtered. The NECP * criteria may be based on a variety of properties like user ID or proc UUID. * * The NECP "filter control unit" is used by the socket content filter subsystem @@ -77,7 +77,7 @@ * 4) The NECP filter control unit is then used to find the corresponding * kernel control socket instance. * - * Note: NECP currently supports a ingle filter control unit per TCP/IP socket + * Note: NECP currently supports a single filter control unit per TCP/IP socket * but this restriction may be soon lifted. * * @@ -117,16 +117,19 @@ * * After a CFM_OP_SOCKET_ATTACHED is delivered, CFM_OP_DATA_OUT and * CFM_OP_DATA_OUT events are not delivered until a CFM_OP_DATA_UPDATE - * action message is send by the user space filter agent. + * action message is sent by the user space filter agent. * * Note: absolute 64 bits offsets should be large enough for the foreseeable - * future. A 64-bits counter will wrap after 468 years are 10 Gbit/sec: + * future. A 64-bits counter will wrap after 468 years at 10 Gbit/sec: * 2E64 / ((10E9 / 8) * 60 * 60 * 24 * 365.25) = 467.63 * - * They are two kinds of content filter actions: + * They are two kinds of primary content filter actions: * - CFM_OP_DATA_UPDATE: to update pass or peek offsets for each direction. * - CFM_OP_DROP: to shutdown socket and disallow further data flow * + * There is also an action to mark a given client flow as already filtered + * at a higher level, CFM_OP_BLESS_CLIENT. + * * * ACTION MESSAGES * @@ -196,7 +199,7 @@ * CONTENT FILTER QUEUES * * Data that is being filtered is steered away from the TCP/IP socket buffer - * and instead will sit in one of three content filter queue until the data + * and instead will sit in one of three content filter queues until the data * can be re-injected into the TCP/IP socket buffer. * * A content filter queue is represented by "struct cfil_queue" that contains @@ -209,7 +212,7 @@ * b) The "cfe_pending_q" of "struct cfil_entry" * c) The "cfi_inject_q" of "struct cfil_info" * - * Note: The seqyence (a),(b) may be repeated several times if there are more + * Note: The sequence (a),(b) may be repeated several times if there is more * than one content filter attached to the TCP/IP socket. * * The "cfe_ctl_q" queue holds data than cannot be delivered to the @@ -417,6 +420,16 @@ struct cfil_entry { #define CFEF_SENT_SOCK_CLOSED 0x0040 /* closed event was sent */ #define CFEF_CFIL_DETACHED 0x0080 /* filter was detached */ + +#define CFI_ADD_TIME_LOG(cfil, t1, t0, op) \ + struct timeval _tdiff; \ + if ((cfil)->cfi_op_list_ctr < CFI_MAX_TIME_LOG_ENTRY) { \ + timersub(t1, t0, &_tdiff); \ + (cfil)->cfi_op_time[(cfil)->cfi_op_list_ctr] = (uint32_t)(_tdiff.tv_sec * 1000 + _tdiff.tv_usec / 1000);\ + (cfil)->cfi_op_list[(cfil)->cfi_op_list_ctr] = (unsigned char)op; \ + (cfil)->cfi_op_list_ctr ++; \ + } + /* * struct cfil_info * @@ -427,6 +440,10 @@ struct cfil_info { struct socket *cfi_so; uint64_t cfi_flags; uint64_t cfi_sock_id; + struct timeval64 cfi_first_event; + uint32_t cfi_op_list_ctr; + uint32_t cfi_op_time[CFI_MAX_TIME_LOG_ENTRY]; /* time interval in microseconds since first event */ + unsigned char cfi_op_list[CFI_MAX_TIME_LOG_ENTRY]; struct cfi_buf { /* @@ -451,7 +468,7 @@ struct cfil_info { } cfi_snd, cfi_rcv; struct cfil_entry cfi_entries[MAX_CONTENT_FILTER]; -}; +} __attribute__((aligned(8))); #define CFIF_DROP 0x0001 /* drop action applied */ #define CFIF_CLOSE_WAIT 0x0002 /* waiting for filter to close */ @@ -527,6 +544,7 @@ SYSCTL_STRUCT(_net_cfil, OID_AUTO, stats, CTLFLAG_RD|CTLFLAG_LOCKED, static int cfil_action_data_pass(struct socket *, uint32_t, int, uint64_t, uint64_t); static int cfil_action_drop(struct socket *, uint32_t); +static int cfil_action_bless_client(uint32_t, struct cfil_msg_hdr *); static int cfil_dispatch_closed_event(struct socket *, int); static int cfil_data_common(struct socket *, int, struct sockaddr *, struct mbuf *, struct mbuf *, uint32_t); @@ -541,6 +559,7 @@ static void cfil_info_free(struct socket *, struct cfil_info *); static struct cfil_info * cfil_info_alloc(struct socket *); static int cfil_info_attach_unit(struct socket *, uint32_t); static struct socket * cfil_socket_from_sock_id(cfil_sock_id_t); +static struct socket *cfil_socket_from_client_uuid(uuid_t, bool *); static int cfil_service_pending_queue(struct socket *, uint32_t, int); static int cfil_data_service_ctl_q(struct socket *, uint32_t, int); static void cfil_info_verify(struct cfil_info *); @@ -647,23 +666,13 @@ cfil_rw_lock_exclusive_to_shared(lck_rw_t *lck) static void cfil_rw_lock_assert_held(lck_rw_t *lck, int exclusive) { - lck_rw_assert(lck, +#if !MACH_ASSERT +#pragma unused(lck, exclusive) +#endif + LCK_RW_ASSERT(lck, exclusive ? LCK_RW_ASSERT_EXCLUSIVE : LCK_RW_ASSERT_HELD); } -static void -socket_lock_assert_owned(struct socket *so) -{ - lck_mtx_t *mutex_held; - - if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); - else - mutex_held = so->so_proto->pr_domain->dom_mtx; - - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); -} - /* * Return the number of bytes in the mbuf chain using the same * method as m_length() or sballoc() @@ -1131,11 +1140,11 @@ cfil_acquire_sockbuf(struct socket *so, int outgoing) while ((sb->sb_flags & SB_LOCK) || (sb->sb_cfil_thread != NULL && sb->sb_cfil_thread != tp)) { if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); sb->sb_wantlock++; VERIFY(sb->sb_wantlock != 0); @@ -1244,6 +1253,28 @@ cfil_socket_from_sock_id(cfil_sock_id_t cfil_sock_id) return (so); } +static struct socket * +cfil_socket_from_client_uuid(uuid_t necp_client_uuid, bool *cfil_attached) +{ + struct socket *so = NULL; + struct inpcb *inp = NULL; + struct inpcbinfo *pcbinfo = &tcbinfo; + + lck_rw_lock_shared(pcbinfo->ipi_lock); + LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) { + if (inp->inp_state != INPCB_STATE_DEAD && + inp->inp_socket != NULL && + uuid_compare(inp->necp_client_uuid, necp_client_uuid) == 0) { + *cfil_attached = (inp->inp_socket->so_cfil != NULL); + so = inp->inp_socket; + break; + } + } + lck_rw_done(pcbinfo->ipi_lock); + + return (so); +} + static errno_t cfil_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, mbuf_t m, int flags) @@ -1295,6 +1326,17 @@ cfil_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, mbuf_t m, case CFM_OP_DROP: OSIncrementAtomic(&cfil_stats.cfs_ctl_action_drop); break; + case CFM_OP_BLESS_CLIENT: + if (msghdr->cfm_len != sizeof(struct cfil_msg_bless_client)) { + OSIncrementAtomic(&cfil_stats.cfs_ctl_action_bad_len); + error = EINVAL; + CFIL_LOG(LOG_ERR, "bad len: %u for op %u", + msghdr->cfm_len, + msghdr->cfm_op); + goto done; + } + error = cfil_action_bless_client(kcunit, msghdr); + goto done; default: OSIncrementAtomic(&cfil_stats.cfs_ctl_action_bad_op); CFIL_LOG(LOG_ERR, "bad op %u", msghdr->cfm_op); @@ -1360,6 +1402,7 @@ cfil_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, mbuf_t m, } microuptime(&entry->cfe_last_action); + CFI_ADD_TIME_LOG(so->so_cfil, &entry->cfe_last_action, &so->so_cfil->cfi_first_event, msghdr->cfm_op); action_msg = (struct cfil_msg_action *)msghdr; @@ -1440,9 +1483,88 @@ cfil_ctl_getopt(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, error = EINVAL; goto done; } - if (data != NULL) + if (data != NULL) { *(uint32_t *)data = cfc->cf_necp_control_unit; + } break; + case CFIL_OPT_GET_SOCKET_INFO: + if (*len != sizeof(struct cfil_opt_sock_info)) { + CFIL_LOG(LOG_ERR, "len does not match %lu", *len); + error = EINVAL; + goto done; + } + if (data == NULL) { + CFIL_LOG(LOG_ERR, "data not passed"); + error = EINVAL; + goto done; + } + + struct cfil_opt_sock_info *sock_info = + (struct cfil_opt_sock_info *) data; + struct socket *sock = + cfil_socket_from_sock_id(sock_info->cfs_sock_id); + if (sock == NULL) { + CFIL_LOG(LOG_NOTICE, "bad sock_id %llx", + sock_info->cfs_sock_id); + error = ENOENT; + goto done; + } + + // Unlock here so that we never hold both cfil_lck_rw and the + // socket_lock at the same time. Otherwise, this can deadlock + // because soclose() takes the socket_lock and then exclusive + // cfil_lck_rw and we require the opposite order. + + // WARNING: Be sure to never use anything protected + // by cfil_lck_rw beyond this point. + // WARNING: Be sure to avoid fallthrough and + // goto return_already_unlocked from this branch. + cfil_rw_unlock_shared(&cfil_lck_rw); + + socket_lock(sock, 1); + + if (sock->so_cfil == NULL) { + CFIL_LOG(LOG_NOTICE, "so %llx not attached, cannot fetch info", + (uint64_t)VM_KERNEL_ADDRPERM(sock)); + error = EINVAL; + socket_unlock(sock, 1); + goto return_already_unlocked; + } + + // Fill out family, type, and protocol + sock_info->cfs_sock_family = sock->so_proto->pr_domain->dom_family; + sock_info->cfs_sock_type = sock->so_proto->pr_type; + sock_info->cfs_sock_protocol = sock->so_proto->pr_protocol; + + // Source and destination addresses + struct inpcb *inp = sotoinpcb(sock); + if (inp->inp_vflag & INP_IPV6) { + fill_ip6_sockaddr_4_6(&sock_info->cfs_local, + &inp->in6p_laddr, inp->inp_lport); + fill_ip6_sockaddr_4_6(&sock_info->cfs_remote, + &inp->in6p_faddr, inp->inp_fport); + } else if (inp->inp_vflag & INP_IPV4) { + fill_ip_sockaddr_4_6(&sock_info->cfs_local, + inp->inp_laddr, inp->inp_lport); + fill_ip_sockaddr_4_6(&sock_info->cfs_remote, + inp->inp_faddr, inp->inp_fport); + } + + // Set the pid info + sock_info->cfs_pid = sock->last_pid; + memcpy(sock_info->cfs_uuid, sock->last_uuid, sizeof(uuid_t)); + + if (sock->so_flags & SOF_DELEGATED) { + sock_info->cfs_e_pid = sock->e_pid; + memcpy(sock_info->cfs_e_uuid, sock->e_uuid, sizeof(uuid_t)); + } else { + sock_info->cfs_e_pid = sock->last_pid; + memcpy(sock_info->cfs_e_uuid, sock->last_uuid, sizeof(uuid_t)); + } + + socket_unlock(sock, 1); + + goto return_already_unlocked; default: error = ENOPROTOOPT; break; @@ -1450,6 +1572,10 @@ cfil_ctl_getopt(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, done: cfil_rw_unlock_shared(&cfil_lck_rw); + return (error); + +return_already_unlocked: + return (error); } @@ -1547,7 +1673,7 @@ cfil_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, int flags) cfc->cf_flags &= ~CFF_FLOW_CONTROLLED; cfil_rw_lock_exclusive_to_shared(&cfil_lck_rw); - lck_rw_assert(&cfil_lck_rw, LCK_RW_ASSERT_SHARED); + LCK_RW_ASSERT(&cfil_lck_rw, LCK_RW_ASSERT_SHARED); } /* * Flow control will be raised again as soon as an entry cannot enqueue @@ -1915,7 +2041,8 @@ cfil_sock_attach(struct socket *so) so->so_proto->pr_domain->dom_family != PF_INET6) || so->so_proto->pr_type != SOCK_STREAM || so->so_proto->pr_protocol != IPPROTO_TCP || - (so->so_flags & SOF_MP_SUBFLOW) != 0) + (so->so_flags & SOF_MP_SUBFLOW) != 0 || + (so->so_flags1 & SOF1_CONTENT_FILTER_SKIP) != 0) goto done; filter_control_unit = necp_socket_get_content_filter_control_unit(so); @@ -1990,7 +2117,7 @@ cfil_dispatch_attach_event(struct socket *so, uint32_t filter_control_unit) struct cfil_entry *entry = NULL; struct cfil_msg_sock_attached msg_attached; uint32_t kcunit; - struct content_filter *cfc; + struct content_filter *cfc = NULL; socket_lock_assert_owned(so); @@ -2063,6 +2190,9 @@ cfil_dispatch_attach_event(struct socket *so, uint32_t filter_control_unit) goto done; } microuptime(&entry->cfe_last_event); + so->so_cfil->cfi_first_event.tv_sec = entry->cfe_last_event.tv_sec; + so->so_cfil->cfi_first_event.tv_usec = entry->cfe_last_event.tv_usec; + entry->cfe_flags |= CFEF_SENT_SOCK_ATTACHED; OSIncrementAtomic(&cfil_stats.cfs_attach_event_ok); done: @@ -2158,6 +2288,7 @@ cfil_dispatch_disconnect_event(struct socket *so, uint32_t kcunit, int outgoing) goto done; } microuptime(&entry->cfe_last_event); + CFI_ADD_TIME_LOG(so->so_cfil, &entry->cfe_last_event, &so->so_cfil->cfi_first_event, msg_disconnected.cfm_op); /* Remember we have sent the disconnection message */ if (outgoing) { @@ -2193,7 +2324,7 @@ int cfil_dispatch_closed_event(struct socket *so, int kcunit) { struct cfil_entry *entry; - struct cfil_msg_hdr msg_closed; + struct cfil_msg_sock_closed msg_closed; errno_t error = 0; struct content_filter *cfc; @@ -2222,23 +2353,42 @@ cfil_dispatch_closed_event(struct socket *so, int kcunit) if ((entry->cfe_flags & CFEF_SENT_SOCK_ATTACHED) == 0) goto done; - bzero(&msg_closed, sizeof(struct cfil_msg_hdr)); - msg_closed.cfm_len = sizeof(struct cfil_msg_hdr); - msg_closed.cfm_version = CFM_VERSION_CURRENT; - msg_closed.cfm_type = CFM_TYPE_EVENT; - msg_closed.cfm_op = CFM_OP_SOCKET_CLOSED; - msg_closed.cfm_sock_id = entry->cfe_cfil_info->cfi_sock_id; + microuptime(&entry->cfe_last_event); + CFI_ADD_TIME_LOG(so->so_cfil, &entry->cfe_last_event, &so->so_cfil->cfi_first_event, CFM_OP_SOCKET_CLOSED); + + bzero(&msg_closed, sizeof(struct cfil_msg_sock_closed)); + msg_closed.cfc_msghdr.cfm_len = sizeof(struct cfil_msg_sock_closed); + msg_closed.cfc_msghdr.cfm_version = CFM_VERSION_CURRENT; + msg_closed.cfc_msghdr.cfm_type = CFM_TYPE_EVENT; + msg_closed.cfc_msghdr.cfm_op = CFM_OP_SOCKET_CLOSED; + msg_closed.cfc_msghdr.cfm_sock_id = entry->cfe_cfil_info->cfi_sock_id; + msg_closed.cfc_first_event.tv_sec = so->so_cfil->cfi_first_event.tv_sec; + msg_closed.cfc_first_event.tv_usec = so->so_cfil->cfi_first_event.tv_usec; + memcpy(msg_closed.cfc_op_time, so->so_cfil->cfi_op_time, sizeof(uint32_t)*CFI_MAX_TIME_LOG_ENTRY); + memcpy(msg_closed.cfc_op_list, so->so_cfil->cfi_op_list, sizeof(unsigned char)*CFI_MAX_TIME_LOG_ENTRY); + msg_closed.cfc_op_list_ctr = so->so_cfil->cfi_op_list_ctr; + + CFIL_LOG(LOG_INFO, "sock id %llu, op ctr %d, start time %llu.%llu", msg_closed.cfc_msghdr.cfm_sock_id, so->so_cfil->cfi_op_list_ctr, so->so_cfil->cfi_first_event.tv_sec, so->so_cfil->cfi_first_event.tv_usec); + /* for debugging + if (msg_closed.cfc_op_list_ctr > CFI_MAX_TIME_LOG_ENTRY) { + msg_closed.cfc_op_list_ctr = CFI_MAX_TIME_LOG_ENTRY; // just in case + } + for (unsigned int i = 0; i < msg_closed.cfc_op_list_ctr ; i++) { + CFIL_LOG(LOG_ERR, "MD: socket %llu event %2u, time + %u msec", msg_closed.cfc_msghdr.cfm_sock_id, (unsigned short)msg_closed.cfc_op_list[i], msg_closed.cfc_op_time[i]); + } + */ + error = ctl_enqueuedata(entry->cfe_filter->cf_kcref, entry->cfe_filter->cf_kcunit, &msg_closed, - sizeof(struct cfil_msg_hdr), + sizeof(struct cfil_msg_sock_closed), CTL_DATA_EOR); if (error != 0) { CFIL_LOG(LOG_ERR, "ctl_enqueuedata() failed: %d", error); goto done; } - microuptime(&entry->cfe_last_event); + entry->cfe_flags |= CFEF_SENT_SOCK_CLOSED; OSIncrementAtomic(&cfil_stats.cfs_closed_event_ok); done: @@ -2305,6 +2455,7 @@ cfil_dispatch_data_event(struct socket *so, uint32_t kcunit, int outgoing, struct cfil_entry *entry; struct cfe_buf *entrybuf; struct content_filter *cfc; + struct timeval tv; cfil_rw_lock_shared(&cfil_lck_rw); @@ -2396,6 +2547,9 @@ cfil_dispatch_data_event(struct socket *so, uint32_t kcunit, int outgoing, } } + microuptime(&tv); + CFI_ADD_TIME_LOG(so->so_cfil, &tv, &so->so_cfil->cfi_first_event, data_req->cfd_msghdr.cfm_op); + /* Pass the message to the content filter */ error = ctl_enqueuembuf(entry->cfe_filter->cf_kcref, entry->cfe_filter->cf_kcunit, @@ -3230,6 +3384,35 @@ done: return (error); } +int +cfil_action_bless_client(uint32_t kcunit, struct cfil_msg_hdr *msghdr) +{ + errno_t error = 0; + + cfil_rw_lock_exclusive(&cfil_lck_rw); + + bool cfil_attached = false; + struct cfil_msg_bless_client *blessmsg = (struct cfil_msg_bless_client *)msghdr; + struct socket *so = cfil_socket_from_client_uuid(blessmsg->cfb_client_uuid, &cfil_attached); + if (so == NULL) { + error = ENOENT; + } else { + // The client gets a pass automatically + socket_lock(so, 1); + if (cfil_attached) { + (void)cfil_action_data_pass(so, kcunit, 1, CFM_MAX_OFFSET, CFM_MAX_OFFSET); + (void)cfil_action_data_pass(so, kcunit, 0, CFM_MAX_OFFSET, CFM_MAX_OFFSET); + } else { + so->so_flags1 |= SOF1_CONTENT_FILTER_SKIP; + } + socket_unlock(so, 1); + } + + cfil_rw_unlock_exclusive(&cfil_lck_rw); + + return (error); +} + static int cfil_update_entry_offsets(struct socket *so, int outgoing, unsigned int datalen) { @@ -3608,10 +3791,10 @@ cfil_sock_close_wait(struct socket *so) CFIL_LOG(LOG_INFO, "so %llx", (uint64_t)VM_KERNEL_ADDRPERM(so)); if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); while (cfil_filters_attached(so)) { /* diff --git a/bsd/net/content_filter.h b/bsd/net/content_filter.h index 7291b2fb4..e4d1ce5d4 100644 --- a/bsd/net/content_filter.h +++ b/bsd/net/content_filter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -49,6 +49,14 @@ __BEGIN_DECLS */ #define CONTENT_FILTER_CONTROL_NAME "com.apple.content-filter" +/* + * Opaque socket identifier + */ +typedef uint64_t cfil_sock_id_t; + +#define CFIL_SOCK_ID_NONE UINT64_MAX + + /* * CFIL_OPT_NECP_CONTROL_UNIT * To set or get the NECP filter control unit for the kernel control socket @@ -56,6 +64,30 @@ __BEGIN_DECLS */ #define CFIL_OPT_NECP_CONTROL_UNIT 1 /* uint32_t */ +/* + * CFIL_OPT_GET_SOCKET_INFO + * To get information about a given socket that is being filtered. + */ +#define CFIL_OPT_GET_SOCKET_INFO 2 /* uint32_t */ + +/* + * struct cfil_opt_sock_info + * + * Contains information about a socket that is being filtered. + */ +struct cfil_opt_sock_info { + cfil_sock_id_t cfs_sock_id; + int cfs_sock_family; /* e.g. PF_INET */ + int cfs_sock_type; /* e.g. SOCK_STREAM */ + int cfs_sock_protocol; /* e.g. IPPROTO_TCP */ + union sockaddr_in_4_6 cfs_local; + union sockaddr_in_4_6 cfs_remote; + pid_t cfs_pid; + pid_t cfs_e_pid; + uuid_t cfs_uuid; + uuid_t cfs_e_uuid; +}; + /* * How many filter may be active simultaneously */ @@ -87,13 +119,7 @@ __BEGIN_DECLS */ #define CFM_OP_DATA_UPDATE 16 /* update pass or peek offsets */ #define CFM_OP_DROP 17 /* shutdown socket, no more data */ - -/* - * Opaque socket identifier - */ -typedef uint64_t cfil_sock_id_t; - -#define CFIL_SOCK_ID_NONE UINT64_MAX +#define CFM_OP_BLESS_CLIENT 18 /* mark a client flow as already filtered, passes a uuid */ /* * struct cfil_msg_hdr @@ -158,6 +184,27 @@ struct cfil_msg_data_event { /* Actual content data immediatly follows */ }; +#define CFI_MAX_TIME_LOG_ENTRY 6 +/* + * struct cfil_msg_sock_closed + * + * Information about a socket being closed to the content filter + * + * Action: No reply is expected as this does not block the closing of the + * TCP/IP. + * + * Valid Types: CFM_TYPE_EVENT + * + * Valid Op: CFM_OP_SOCKET_CLOSED + */ +struct cfil_msg_sock_closed { + struct cfil_msg_hdr cfc_msghdr; + struct timeval64 cfc_first_event; + uint32_t cfc_op_list_ctr; + uint32_t cfc_op_time[CFI_MAX_TIME_LOG_ENTRY]; /* time interval in microseconds since first event */ + unsigned char cfc_op_list[CFI_MAX_TIME_LOG_ENTRY]; +} __attribute__((aligned(8))); + /* * struct cfil_msg_action * @@ -183,6 +230,20 @@ struct cfil_msg_action { uint64_t cfa_out_peek_offset; }; +/* + * struct cfil_msg_bless_client + * + * Marks a client UUID as already filtered at a higher level. + * + * Valid Type: CFM_TYPE_ACTION + * + * Valid Ops: CFM_OP_BLESS_CLIENT + */ +struct cfil_msg_bless_client { + struct cfil_msg_hdr cfb_msghdr; + uuid_t cfb_client_uuid; +}; + #define CFM_MAX_OFFSET UINT64_MAX /* @@ -361,8 +422,8 @@ extern void cfil_sock_buf_update(struct sockbuf *sb); extern cfil_sock_id_t cfil_sock_id_from_socket(struct socket *so); -__END_DECLS - #endif /* BSD_KERNEL_PRIVATE */ +__END_DECLS + #endif /* __CONTENT_FILTER_H__ */ diff --git a/bsd/net/dlil.c b/bsd/net/dlil.c index b38d4e597..50e85b274 100644 --- a/bsd/net/dlil.c +++ b/bsd/net/dlil.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999-2016 Apple Inc. All rights reserved. + * Copyright (c) 1999-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -73,6 +73,8 @@ #include #include #include +#include +#include #if INET #include @@ -113,15 +115,13 @@ #if PF #include #endif /* PF */ -#if PF_ALTQ -#include -#endif /* PF_ALTQ */ #include #if NECP #include #endif /* NECP */ + #define DBG_LAYER_BEG DLILDBG_CODE(DBG_DLIL_STATIC, 0) #define DBG_LAYER_END DLILDBG_CODE(DBG_DLIL_STATIC, 2) #define DBG_FNC_DLIL_INPUT DLILDBG_CODE(DBG_DLIL_STATIC, (1 << 8)) @@ -186,7 +186,6 @@ struct if_proto { SLIST_HEAD(proto_hash_entry, if_proto); -#define DLIL_SDLMAXLEN 64 #define DLIL_SDLDATALEN \ (DLIL_SDLMAXLEN - offsetof(struct sockaddr_dl, sdl_data[0])) @@ -353,8 +352,6 @@ static errno_t ifproto_media_send_arp(struct ifnet *, u_short, const struct sockaddr_dl *, const struct sockaddr *, const struct sockaddr_dl *, const struct sockaddr *); -static errno_t ifp_if_output(struct ifnet *, struct mbuf *); -static void ifp_if_start(struct ifnet *); static errno_t ifp_if_input(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail, const struct ifnet_stat_increment_param *s, boolean_t poll, struct thread *tp); @@ -367,8 +364,14 @@ static errno_t ifp_if_add_proto(struct ifnet *, protocol_family_t, const struct ifnet_demux_desc *, u_int32_t); static errno_t ifp_if_del_proto(struct ifnet *, protocol_family_t); static errno_t ifp_if_check_multi(struct ifnet *, const struct sockaddr *); +#if CONFIG_EMBEDDED +static errno_t ifp_if_framer(struct ifnet *, struct mbuf **, + const struct sockaddr *, const char *, const char *, + u_int32_t *, u_int32_t *); +#else static errno_t ifp_if_framer(struct ifnet *, struct mbuf **, const struct sockaddr *, const char *, const char *); +#endif /* CONFIG_EMBEDDED */ static errno_t ifp_if_framer_extended(struct ifnet *, struct mbuf **, const struct sockaddr *, const char *, const char *, u_int32_t *, u_int32_t *); @@ -391,9 +394,9 @@ static void dlil_input_packet_list_common(struct ifnet *, struct mbuf *, static errno_t ifnet_input_common(struct ifnet *, struct mbuf *, struct mbuf *, const struct ifnet_stat_increment_param *, boolean_t, boolean_t); -#if DEBUG +#if DEBUG || DEVELOPMENT static void dlil_verify_sum16(void); -#endif /* DEBUG */ +#endif /* DEBUG || DEVELOPMENT */ static void dlil_output_cksum_dbg(struct ifnet *, struct mbuf *, uint32_t, protocol_family_t); static void dlil_input_cksum_dbg(struct ifnet *, struct mbuf *, char *, @@ -408,6 +411,8 @@ static struct ifnet *ifnet_detaching_dequeue(void); static void ifnet_start_thread_fn(void *, wait_result_t); static void ifnet_poll_thread_fn(void *, wait_result_t); static void ifnet_poll(struct ifnet *); +static errno_t ifnet_enqueue_common(struct ifnet *, void *, + classq_pkt_type_t, boolean_t, boolean_t *); static void ifp_src_route_copyout(struct ifnet *, struct route *); static void ifp_src_route_copyin(struct ifnet *, struct route *); @@ -432,6 +437,10 @@ static int sysctl_get_ports_used SYSCTL_HANDLER_ARGS; struct chain_len_stats tx_chain_len_stats; static int sysctl_tx_chain_len_stats SYSCTL_HANDLER_ARGS; +#if TEST_INPUT_THREAD_TERMINATION +static int sysctl_input_thread_termination_spin SYSCTL_HANDLER_ARGS; +#endif /* TEST_INPUT_THREAD_TERMINATION */ + /* The following are protected by dlil_ifnet_lock */ static TAILQ_HEAD(, ifnet) ifnet_detaching_head; static u_int32_t ifnet_detaching_cnt; @@ -489,8 +498,12 @@ extern uint32_t tcp_count_opportunistic(unsigned int ifindex, __private_extern__ void link_rtrequest(int, struct rtentry *, struct sockaddr *); #if CONFIG_MACF +#ifdef CONFIG_EMBEDDED +int dlil_lladdr_ckreq = 1; +#else int dlil_lladdr_ckreq = 0; #endif +#endif #if DEBUG int dlil_verbose = 1; @@ -588,13 +601,14 @@ SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll, 0, sysctl_rxpoll, "I", "enable opportunistic input polling"); -u_int32_t if_bw_smoothing_val = 3; -SYSCTL_UINT(_net_link_generic_system, OID_AUTO, if_bw_smoothing_val, - CTLFLAG_RW | CTLFLAG_LOCKED, &if_bw_smoothing_val, 0, ""); - -u_int32_t if_bw_measure_size = 10; -SYSCTL_INT(_net_link_generic_system, OID_AUTO, if_bw_measure_size, - CTLFLAG_RW | CTLFLAG_LOCKED, &if_bw_measure_size, 0, ""); +#if TEST_INPUT_THREAD_TERMINATION +static u_int32_t if_input_thread_termination_spin = 0; +SYSCTL_PROC(_net_link_generic_system, OID_AUTO, input_thread_termination_spin, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &if_input_thread_termination_spin, 0, + sysctl_input_thread_termination_spin, + "I", "input thread termination spin limit"); +#endif /* TEST_INPUT_THREAD_TERMINATION */ static u_int32_t cur_dlil_input_threads = 0; SYSCTL_UINT(_net_link_generic_system, OID_AUTO, dlil_input_threads, @@ -723,12 +737,25 @@ SYSCTL_UINT(_net_link_generic_system, OID_AUTO, tx_chain_len_count, SYSCTL_NODE(_net_link_generic_system, OID_AUTO, get_ports_used, CTLFLAG_RD | CTLFLAG_LOCKED, sysctl_get_ports_used, ""); +static uint32_t threshold_notify = 1; /* enable/disable */ +SYSCTL_UINT(_net_link_generic_system, OID_AUTO, threshold_notify, + CTLFLAG_RW | CTLFLAG_LOCKED, &threshold_notify, 0, ""); + +static uint32_t threshold_interval = 2; /* in seconds */ +SYSCTL_UINT(_net_link_generic_system, OID_AUTO, threshold_interval, + CTLFLAG_RW | CTLFLAG_LOCKED, &threshold_interval, 0, ""); + #if (DEVELOPMENT || DEBUG) static int sysctl_get_kao_frames SYSCTL_HANDLER_ARGS; SYSCTL_NODE(_net_link_generic_system, OID_AUTO, get_kao_frames, CTLFLAG_RD | CTLFLAG_LOCKED, sysctl_get_kao_frames, ""); #endif /* DEVELOPMENT || DEBUG */ +struct net_api_stats net_api_stats; +SYSCTL_STRUCT(_net, OID_AUTO, api_stats, CTLFLAG_RD|CTLFLAG_LOCKED, + &net_api_stats, net_api_stats, ""); + + unsigned int net_rxpoll = 1; unsigned int net_affinity = 1; static kern_return_t dlil_affinity_set(struct thread *, u_int32_t); @@ -738,6 +765,24 @@ extern u_int32_t inject_buckets; static lck_grp_attr_t *dlil_grp_attributes = NULL; static lck_attr_t *dlil_lck_attributes = NULL; +/* DLIL data threshold thread call */ +static void dlil_dt_tcall_fn(thread_call_param_t, thread_call_param_t); + +static void dlil_mit_tcall_fn(thread_call_param_t, thread_call_param_t); + +uint32_t dlil_rcv_mit_pkts_min = 5; +uint32_t dlil_rcv_mit_pkts_max = 64; +uint32_t dlil_rcv_mit_interval = (500 * 1000); + +#if (DEVELOPMENT || DEBUG) +SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_pkts_min, + CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_pkts_min, 0, ""); +SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_pkts_max, + CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_pkts_max, 0, ""); +SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_interval, + CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_interval, 0, ""); +#endif /* DEVELOPMENT || DEBUG */ + #define DLIL_INPUT_CHECK(m, ifp) { \ struct ifnet *_rcvif = mbuf_pkthdr_rcvif(m); \ @@ -878,6 +923,9 @@ if_proto_free(struct if_proto *proto) __private_extern__ void ifnet_lock_assert(struct ifnet *ifp, ifnet_lock_assert_t what) { +#if !MACH_ASSERT +#pragma unused(ifp) +#endif unsigned int type = 0; int ass = 1; @@ -904,7 +952,7 @@ ifnet_lock_assert(struct ifnet *ifp, ifnet_lock_assert_t what) /* NOTREACHED */ } if (ass) - lck_rw_assert(&ifp->if_lock, type); + LCK_RW_ASSERT(&ifp->if_lock, type); } __private_extern__ void @@ -986,7 +1034,7 @@ ifnet_head_done(void) __private_extern__ void ifnet_head_assert_exclusive(void) { - lck_rw_assert(&ifnet_head_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&ifnet_head_lock, LCK_RW_ASSERT_EXCLUSIVE); } /* @@ -1208,11 +1256,11 @@ dlil_create_input_thread(ifnet_t ifp, struct dlil_threading_info *inp) limit = (u_int32_t)-1; } - _qinit(&inp->rcvq_pkts, Q_DROPTAIL, limit); + _qinit(&inp->rcvq_pkts, Q_DROPTAIL, limit, QP_MBUF); if (inp == dlil_main_input_thread) { struct dlil_main_threading_info *inpm = (struct dlil_main_threading_info *)inp; - _qinit(&inpm->lo_rcvq_pkts, Q_DROPTAIL, limit); + _qinit(&inpm->lo_rcvq_pkts, Q_DROPTAIL, limit, QP_MBUF); } error = kernel_thread_start(func, inp, &inp->input_thr); @@ -1231,7 +1279,7 @@ dlil_create_input_thread(ifnet_t ifp, struct dlil_threading_info *inp) * Randomize to reduce the probability * of affinity tag namespace collision. */ - read_random(&tag, sizeof (tag)); + read_frandom(&tag, sizeof (tag)); if (dlil_affinity_set(tp, tag) == KERN_SUCCESS) { thread_reference(tp); inp->tag = tag; @@ -1251,23 +1299,37 @@ dlil_create_input_thread(ifnet_t ifp, struct dlil_threading_info *inp) return (error); } -static void -dlil_terminate_input_thread(struct dlil_threading_info *inp) +#if TEST_INPUT_THREAD_TERMINATION +static int +sysctl_input_thread_termination_spin SYSCTL_HANDLER_ARGS { - struct ifnet *ifp; +#pragma unused(arg1, arg2) + uint32_t i; + int err; - VERIFY(current_thread() == inp->input_thr); - VERIFY(inp != dlil_main_input_thread); + i = if_input_thread_termination_spin; - OSAddAtomic(-1, &cur_dlil_input_threads); + err = sysctl_handle_int(oidp, &i, 0, req); + if (err != 0 || req->newptr == USER_ADDR_NULL) + return (err); + + if (net_rxpoll == 0) + return (ENXIO); + if_input_thread_termination_spin = i; + return (err); +} +#endif /* TEST_INPUT_THREAD_TERMINATION */ + +static void +dlil_clean_threading_info(struct dlil_threading_info *inp) +{ lck_mtx_destroy(&inp->input_lck, inp->lck_grp); lck_grp_free(inp->lck_grp); inp->input_waiting = 0; inp->wtot = 0; bzero(inp->input_name, sizeof (inp->input_name)); - ifp = inp->ifp; inp->ifp = NULL; VERIFY(qhead(&inp->rcvq_pkts) == NULL && qempty(&inp->rcvq_pkts)); qlimit(&inp->rcvq_pkts) = 0; @@ -1293,15 +1355,44 @@ dlil_terminate_input_thread(struct dlil_threading_info *inp) #if IFNET_INPUT_SANITY_CHK inp->input_mbuf_cnt = 0; #endif /* IFNET_INPUT_SANITY_CHK */ +} - if (dlil_verbose) { - printf("%s: input thread terminated\n", - if_name(ifp)); +static void +dlil_terminate_input_thread(struct dlil_threading_info *inp) +{ + struct ifnet *ifp = inp->ifp; + + VERIFY(current_thread() == inp->input_thr); + VERIFY(inp != dlil_main_input_thread); + + OSAddAtomic(-1, &cur_dlil_input_threads); + +#if TEST_INPUT_THREAD_TERMINATION + { /* do something useless that won't get optimized away */ + uint32_t v = 1; + for (uint32_t i = 0; + i < if_input_thread_termination_spin; + i++) { + v = (i + 1) * v; + } + printf("the value is %d\n", v); } +#endif /* TEST_INPUT_THREAD_TERMINATION */ + + lck_mtx_lock_spin(&inp->input_lck); + VERIFY((inp->input_waiting & DLIL_INPUT_TERMINATE) != 0); + inp->input_waiting |= DLIL_INPUT_TERMINATE_COMPLETE; + wakeup_one((caddr_t)&inp->input_waiting); + lck_mtx_unlock(&inp->input_lck); /* for the extra refcnt from kernel_thread_start() */ thread_deallocate(current_thread()); + if (dlil_verbose) { + printf("%s: input thread terminated\n", + if_name(ifp)); + } + /* this is the end */ thread_terminate(current_thread()); /* NOTREACHED */ @@ -1370,6 +1461,7 @@ dlil_init(void) _CASSERT(IF_HWASSIST_CSUM_UDPIPV6 == IFNET_CSUM_UDPIPV6); _CASSERT(IF_HWASSIST_CSUM_FRAGMENT_IPV6 == IFNET_IPV6_FRAGMENT); _CASSERT(IF_HWASSIST_CSUM_PARTIAL == IFNET_CSUM_PARTIAL); + _CASSERT(IF_HWASSIST_CSUM_ZERO_INVERT == IFNET_CSUM_ZERO_INVERT); _CASSERT(IF_HWASSIST_VLAN_TAGGING == IFNET_VLAN_TAGGING); _CASSERT(IF_HWASSIST_VLAN_MTU == IFNET_VLAN_MTU); _CASSERT(IF_HWASSIST_TSO_V4 == IFNET_TSO_IPV4); @@ -1387,6 +1479,7 @@ dlil_init(void) _CASSERT(CSUM_UDPIPV6 == IF_HWASSIST_CSUM_UDPIPV6); _CASSERT(CSUM_FRAGMENT_IPV6 == IF_HWASSIST_CSUM_FRAGMENT_IPV6); _CASSERT(CSUM_PARTIAL == IF_HWASSIST_CSUM_PARTIAL); + _CASSERT(CSUM_ZERO_INVERT == IF_HWASSIST_CSUM_ZERO_INVERT); _CASSERT(CSUM_VLAN_TAG_VALID == IF_HWASSIST_VLAN_TAGGING); /* @@ -1524,6 +1617,7 @@ dlil_init(void) zone_change(dlif_udpstat_zone, Z_CALLERACCT, FALSE); ifnet_llreach_init(); + eventhandler_lists_ctxt_init(&ifnet_evhdlr_ctxt); TAILQ_INIT(&dlil_ifnet_head); TAILQ_INIT(&ifnet_head); @@ -1590,10 +1684,13 @@ dlil_init(void) /* Initialize the service class to dscp map */ net_qos_map_init(); -#if DEBUG +#if DEBUG || DEVELOPMENT /* Run self-tests */ dlil_verify_sum16(); -#endif /* DEBUG */ +#endif /* DEBUG || DEVELOPMENT */ + + /* Initialize link layer table */ + lltable_glbl_init(); /* * Create and start up the main DLIL input thread and the interface @@ -1607,12 +1704,13 @@ dlil_init(void) /* NOTREACHED */ } thread_deallocate(thread); + } static void if_flt_monitor_busy(struct ifnet *ifp) { - lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); ++ifp->if_flt_busy; VERIFY(ifp->if_flt_busy != 0); @@ -1627,7 +1725,7 @@ if_flt_monitor_unbusy(struct ifnet *ifp) static void if_flt_monitor_enter(struct ifnet *ifp) { - lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); while (ifp->if_flt_busy) { ++ifp->if_flt_waiters; @@ -1640,7 +1738,7 @@ if_flt_monitor_enter(struct ifnet *ifp) static void if_flt_monitor_leave(struct ifnet *ifp) { - lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); VERIFY(ifp->if_flt_busy != 0); --ifp->if_flt_busy; @@ -1692,7 +1790,7 @@ dlil_attach_filter(struct ifnet *ifp, const struct iff_filter *if_filter, lck_mtx_lock(&ifp->if_flt_lock); if_flt_monitor_enter(ifp); - lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); TAILQ_INSERT_TAIL(&ifp->if_flt_head, filter, filt_next); if_flt_monitor_leave(ifp); @@ -1708,6 +1806,11 @@ dlil_attach_filter(struct ifnet *ifp, const struct iff_filter *if_filter, OSAddAtomic(1, &dlil_filter_disable_tso_count); routegenid_update(); } + OSIncrementAtomic64(&net_api_stats.nas_iflt_attach_count); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_iflt_attach_total); + if ((filter->filt_flags & DLIL_IFF_INTERNAL)) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_iflt_attach_os_total); + } if (dlil_verbose) { printf("%s: %s filter attached\n", if_name(ifp), if_filter->iff_name); @@ -1753,7 +1856,7 @@ dlil_detach_filter_internal(interface_filter_t filter, int detached) lck_mtx_lock(&ifp->if_flt_lock); if_flt_monitor_enter(ifp); - lck_mtx_assert(&ifp->if_flt_lock, + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); /* Remove the filter from the list */ @@ -1795,6 +1898,8 @@ destroy: routegenid_update(); } + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_iflt_attach_count) > 0); + /* Free the filter */ zfree(dlif_filt_zone, filter); filter = NULL; @@ -1958,7 +2063,7 @@ dlil_input_thread_func(void *v, wait_result_t w) inp->wtot = 0; - dlil_input_stats_sync(ifp, inp); + dlil_input_stats_sync(ifp, inp); lck_mtx_unlock(&inp->input_lck); @@ -2033,7 +2138,12 @@ dlil_rxpoll_input_thread_func(void *v, wait_result_t w) if (inp->input_waiting & DLIL_INPUT_TERMINATE) { /* Free up pending packets */ + lck_mtx_convert_spin(&inp->input_lck); _flushq(&inp->rcvq_pkts); + if (inp->input_mit_tcall != NULL) { + if (thread_call_isactive(inp->input_mit_tcall)) + thread_call_cancel(inp->input_mit_tcall); + } lck_mtx_unlock(&inp->input_lck); dlil_terminate_input_thread(inp); @@ -2257,7 +2367,7 @@ dlil_rxpoll_set_params(struct ifnet *ifp, struct ifnet_poll_params *p, if (!locked) lck_mtx_lock(&inp->input_lck); - lck_mtx_assert(&inp->input_lck, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&inp->input_lck, LCK_MTX_ASSERT_OWNED); /* * Normally, we'd reset the parameters to the auto-tuned values @@ -2380,7 +2490,7 @@ static errno_t ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail, const struct ifnet_stat_increment_param *s, boolean_t ext, boolean_t poll) { - ifnet_input_handler_func handler_func; + dlil_input_func input_func; struct ifnet_stat_increment_param _s; u_int32_t m_cnt = 0, m_size = 0; struct mbuf *last; @@ -2407,8 +2517,8 @@ ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail, return (EINVAL); } - handler_func = ifp->if_input_handler; - VERIFY(handler_func != NULL); + input_func = ifp->if_input_dlil; + VERIFY(input_func != NULL); if (m_tail == NULL) { last = m_head; @@ -2476,7 +2586,7 @@ ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail, _s.packets_in = m_cnt; _s.bytes_in = m_size; - err = (*handler_func)(ifp, m_head, m_tail, s, poll, current_thread()); + err = (*input_func)(ifp, m_head, m_tail, s, poll, current_thread()); if (ifp != lo_ifp) { /* Release the IO refcnt */ @@ -2486,31 +2596,6 @@ ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail, return (err); } -errno_t -ifnet_set_input_handler(struct ifnet *ifp, ifnet_input_handler_func fn) -{ - return (atomic_test_set_ptr(&ifp->if_input_handler, - dlil_input_handler, fn) ? 0 : EBUSY); -} - -void -ifnet_reset_input_handler(struct ifnet *ifp) -{ - atomic_set_ptr(&ifp->if_input_handler, dlil_input_handler); -} - -errno_t -ifnet_set_output_handler(struct ifnet *ifp, ifnet_output_handler_func fn) -{ - return (atomic_test_set_ptr(&ifp->if_output_handler, - dlil_output_handler, fn) ? 0 : EBUSY); -} - -void -ifnet_reset_output_handler(struct ifnet *ifp) -{ - atomic_set_ptr(&ifp->if_output_handler, dlil_output_handler); -} errno_t dlil_output_handler(struct ifnet *ifp, struct mbuf *m) @@ -2555,7 +2640,7 @@ dlil_input_handler(struct ifnet *ifp, struct mbuf *m_head, /* * Take a reference on the current thread; during detach, - * we will need to refer to it in order ot tear down its + * we will need to refer to it in order to tear down its * affinity. */ thread_reference(tp); @@ -2612,16 +2697,31 @@ dlil_input_handler(struct ifnet *ifp, struct mbuf *m_head, if (inp == dlil_main_input_thread) dlil_input_stats_sync(ifp, inp); - inp->input_waiting |= DLIL_INPUT_WAITING; - if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) { - inp->wtot++; - wakeup_one((caddr_t)&inp->input_waiting); + if (qlen(&inp->rcvq_pkts) >= dlil_rcv_mit_pkts_min && + qlen(&inp->rcvq_pkts) < dlil_rcv_mit_pkts_max && + (ifp->if_family == IFNET_FAMILY_ETHERNET || + ifp->if_type == IFT_CELLULAR) + ) { + if (!thread_call_isactive(inp->input_mit_tcall)) { + uint64_t deadline; + clock_interval_to_deadline(dlil_rcv_mit_interval, + 1, &deadline); + (void) thread_call_enter_delayed( + inp->input_mit_tcall, deadline); + } + } else { + inp->input_waiting |= DLIL_INPUT_WAITING; + if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) { + inp->wtot++; + wakeup_one((caddr_t)&inp->input_waiting); + } } lck_mtx_unlock(&inp->input_lck); return (0); } + static void ifnet_start_common(struct ifnet *ifp, int resetfc) { @@ -2645,7 +2745,8 @@ ifnet_start_common(struct ifnet *ifp, int resetfc) (resetfc || !(ifp->if_eflags & IFEF_ENQUEUE_MULTI) || IFCQ_LEN(&ifp->if_snd) >= ifp->if_start_delay_qlen || ifp->if_start_delayed == 0)) { - wakeup_one((caddr_t)&ifp->if_start_thread); + (void) thread_wakeup_thread((caddr_t)&ifp->if_start_thread, + ifp->if_start_thread); } lck_mtx_unlock(&ifp->if_start_lock); } @@ -2669,7 +2770,8 @@ ifnet_start_thread_fn(void *v, wait_result_t w) /* Construct the name for this thread, and then apply it. */ bzero(thread_name, sizeof(thread_name)); - snprintf(thread_name, sizeof(thread_name), "ifnet_start_%s", ifp->if_xname); + (void) snprintf(thread_name, sizeof (thread_name), + "ifnet_start_%s", ifp->if_xname); thread_set_thread_name(ifp->if_start_thread, thread_name); /* @@ -2699,17 +2801,16 @@ ifnet_start_thread_fn(void *v, wait_result_t w) } } - snprintf(ifname, sizeof (ifname), "%s_starter", - if_name(ifp)); + (void) snprintf(ifname, sizeof (ifname), "%s_starter", if_name(ifp)); lck_mtx_lock_spin(&ifp->if_start_lock); for (;;) { - if (ifp->if_start_thread != NULL) + if (ifp->if_start_thread != NULL) { (void) msleep(&ifp->if_start_thread, &ifp->if_start_lock, (PZERO - 1) | PSPIN, ifname, ts); - + } /* interface is detached? */ if (ifp->if_start_thread == THREAD_NULL) { ifnet_set_start_cycle(ifp, NULL); @@ -3131,34 +3232,86 @@ ifnet_get_rcvq_maxlen(struct ifnet *ifp, u_int32_t *maxqlen) return (0); } -errno_t -ifnet_enqueue(struct ifnet *ifp, struct mbuf *m) +void +ifnet_enqueue_multi_setup(struct ifnet *ifp, uint16_t delay_qlen, + uint16_t delay_timeout) +{ + if (delay_qlen > 0 && delay_timeout > 0) { + ifp->if_eflags |= IFEF_ENQUEUE_MULTI; + ifp->if_start_delay_qlen = min(100, delay_qlen); + ifp->if_start_delay_timeout = min(20000, delay_timeout); + /* convert timeout to nanoseconds */ + ifp->if_start_delay_timeout *= 1000; + kprintf("%s: forced IFEF_ENQUEUE_MULTI qlen %u timeout %u\n", + ifp->if_xname, (uint32_t)delay_qlen, + (uint32_t)delay_timeout); + } else { + ifp->if_eflags &= ~IFEF_ENQUEUE_MULTI; + } +} + +static inline errno_t +ifnet_enqueue_common(struct ifnet *ifp, void *p, classq_pkt_type_t ptype, + boolean_t flush, boolean_t *pdrop) { - int error; + volatile uint64_t *fg_ts = NULL; + volatile uint64_t *rt_ts = NULL; + struct mbuf *m = p; struct timespec now; - u_int64_t now_nsec; + u_int64_t now_nsec = 0; + int error = 0; - if (ifp == NULL || m == NULL || !(m->m_flags & M_PKTHDR) || - m->m_nextpkt != NULL) { - if (m != NULL) - m_freem_list(m); - return (EINVAL); - } else if (!(ifp->if_eflags & IFEF_TXSTART) || - !(ifp->if_refflags & IFRF_ATTACHED)) { - /* flag tested without lock for performance */ - m_freem(m); - return (ENXIO); - } else if (!(ifp->if_flags & IFF_UP)) { - m_freem(m); - return (ENETDOWN); - } + ASSERT(ifp->if_eflags & IFEF_TXSTART); + + /* + * If packet already carries a timestamp, either from dlil_output() + * or from flowswitch, use it here. Otherwise, record timestamp. + * PKTF_TS_VALID is always cleared prior to entering classq, i.e. + * the timestamp value is used internally there. + */ + switch (ptype) { + case QP_MBUF: + ASSERT(m->m_flags & M_PKTHDR); + ASSERT(m->m_nextpkt == NULL); + + if (!(m->m_pkthdr.pkt_flags & PKTF_TS_VALID) || + m->m_pkthdr.pkt_timestamp == 0) { + nanouptime(&now); + net_timernsec(&now, &now_nsec); + m->m_pkthdr.pkt_timestamp = now_nsec; + } + m->m_pkthdr.pkt_flags &= ~PKTF_TS_VALID; + /* + * If the packet service class is not background, + * update the timestamp to indicate recent activity + * on a foreground socket. + */ + if ((m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) && + m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) { + if (!(m->m_pkthdr.pkt_flags & PKTF_SO_BACKGROUND)) { + ifp->if_fg_sendts = _net_uptime; + if (fg_ts != NULL) + *fg_ts = _net_uptime; + } + if (m->m_pkthdr.pkt_flags & PKTF_SO_REALTIME) { + ifp->if_rt_sendts = _net_uptime; + if (rt_ts != NULL) + *rt_ts = _net_uptime; + } + } + break; - nanouptime(&now); - net_timernsec(&now, &now_nsec); - m->m_pkthdr.pkt_timestamp = now_nsec; - m->m_pkthdr.pkt_flags &= ~PKTF_DRV_TS_VALID; + + default: + VERIFY(0); + /* NOTREACHED */ + } if (ifp->if_eflags & IFEF_ENQUEUE_MULTI) { + if (now_nsec == 0) { + nanouptime(&now); + net_timernsec(&now, &now_nsec); + } /* * If the driver chose to delay start callback for * coalescing multiple packets, Then use the following @@ -3213,8 +3366,17 @@ ifnet_enqueue(struct ifnet *ifp, struct mbuf *m) ifp->if_eflags &= ~(IFEF_DELAY_START); } - /* enqueue the packet */ - error = ifclassq_enqueue(&ifp->if_snd, m); + switch (ptype) { + case QP_MBUF: + /* enqueue the packet (caller consumes object) */ + error = ifclassq_enqueue(&ifp->if_snd, m, QP_MBUF, pdrop); + m = NULL; + break; + + + default: + break; + } /* * Tell the driver to start dequeueing; do this even when the queue @@ -3222,16 +3384,51 @@ ifnet_enqueue(struct ifnet *ifp, struct mbuf *m) * be dequeueing from other unsuspended queues. */ if (!(ifp->if_eflags & IFEF_ENQUEUE_MULTI) && - (error == 0 || error == EQFULL || error == EQSUSPENDED)) + ((error == 0 && flush) || error == EQFULL || error == EQSUSPENDED)) ifnet_start(ifp); return (error); } +errno_t +ifnet_enqueue(struct ifnet *ifp, struct mbuf *m) +{ + boolean_t pdrop; + return (ifnet_enqueue_mbuf(ifp, m, TRUE, &pdrop)); +} + +errno_t +ifnet_enqueue_mbuf(struct ifnet *ifp, struct mbuf *m, boolean_t flush, + boolean_t *pdrop) +{ + if (ifp == NULL || m == NULL || !(m->m_flags & M_PKTHDR) || + m->m_nextpkt != NULL) { + if (m != NULL) { + m_freem_list(m); + *pdrop = TRUE; + } + return (EINVAL); + } else if (!(ifp->if_eflags & IFEF_TXSTART) || + !IF_FULLY_ATTACHED(ifp)) { + /* flag tested without lock for performance */ + m_freem(m); + *pdrop = TRUE; + return (ENXIO); + } else if (!(ifp->if_flags & IFF_UP)) { + m_freem(m); + *pdrop = TRUE; + return (ENETDOWN); + } + + return (ifnet_enqueue_common(ifp, m, QP_MBUF, flush, pdrop)); +} + + errno_t ifnet_dequeue(struct ifnet *ifp, struct mbuf **mp) { errno_t rc; + classq_pkt_type_t ptype; if (ifp == NULL || mp == NULL) return (EINVAL); else if (!(ifp->if_eflags & IFEF_TXSTART) || @@ -3239,8 +3436,10 @@ ifnet_dequeue(struct ifnet *ifp, struct mbuf **mp) return (ENXIO); if (!ifnet_is_attached(ifp, 1)) return (ENXIO); + rc = ifclassq_dequeue(&ifp->if_snd, 1, CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, - mp, NULL, NULL, NULL); + (void **)mp, NULL, NULL, NULL, &ptype); + VERIFY((*mp == NULL) || (ptype == QP_MBUF)); ifnet_decr_iorefcnt(ifp); return (rc); @@ -3251,6 +3450,7 @@ ifnet_dequeue_service_class(struct ifnet *ifp, mbuf_svc_class_t sc, struct mbuf **mp) { errno_t rc; + classq_pkt_type_t ptype; if (ifp == NULL || mp == NULL || !MBUF_VALID_SC(sc)) return (EINVAL); else if (!(ifp->if_eflags & IFEF_TXSTART) || @@ -3259,7 +3459,10 @@ ifnet_dequeue_service_class(struct ifnet *ifp, mbuf_svc_class_t sc, if (!ifnet_is_attached(ifp, 1)) return (ENXIO); - rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, 1, mp, NULL, NULL, NULL); + rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, 1, + CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)mp, NULL, NULL, + NULL, &ptype); + VERIFY((*mp == NULL) || (ptype == QP_MBUF)); ifnet_decr_iorefcnt(ifp); return (rc); } @@ -3269,6 +3472,7 @@ ifnet_dequeue_multi(struct ifnet *ifp, u_int32_t pkt_limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt, u_int32_t *len) { errno_t rc; + classq_pkt_type_t ptype; if (ifp == NULL || head == NULL || pkt_limit < 1) return (EINVAL); else if (!(ifp->if_eflags & IFEF_TXSTART) || @@ -3278,7 +3482,9 @@ ifnet_dequeue_multi(struct ifnet *ifp, u_int32_t pkt_limit, return (ENXIO); rc = ifclassq_dequeue(&ifp->if_snd, pkt_limit, - CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, head, tail, cnt, len); + CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)head, (void **)tail, cnt, + len, &ptype); + VERIFY((*head == NULL) || (ptype == QP_MBUF)); ifnet_decr_iorefcnt(ifp); return (rc); } @@ -3288,6 +3494,7 @@ ifnet_dequeue_multi_bytes(struct ifnet *ifp, u_int32_t byte_limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt, u_int32_t *len) { errno_t rc; + classq_pkt_type_t ptype; if (ifp == NULL || head == NULL || byte_limit < 1) return (EINVAL); else if (!(ifp->if_eflags & IFEF_TXSTART) || @@ -3297,7 +3504,8 @@ ifnet_dequeue_multi_bytes(struct ifnet *ifp, u_int32_t byte_limit, return (ENXIO); rc = ifclassq_dequeue(&ifp->if_snd, CLASSQ_DEQUEUE_MAX_PKT_LIMIT, - byte_limit, head, tail, cnt, len); + byte_limit, (void **)head, (void **)tail, cnt, len, &ptype); + VERIFY((*head == NULL) || (ptype == QP_MBUF)); ifnet_decr_iorefcnt(ifp); return (rc); } @@ -3308,6 +3516,7 @@ ifnet_dequeue_service_class_multi(struct ifnet *ifp, mbuf_svc_class_t sc, u_int32_t *len) { errno_t rc; + classq_pkt_type_t ptype; if (ifp == NULL || head == NULL || pkt_limit < 1 || !MBUF_VALID_SC(sc)) return (EINVAL); @@ -3316,12 +3525,16 @@ ifnet_dequeue_service_class_multi(struct ifnet *ifp, mbuf_svc_class_t sc, return (ENXIO); if (!ifnet_is_attached(ifp, 1)) return (ENXIO); - rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, pkt_limit, head, - tail, cnt, len); + + rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, pkt_limit, + CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)head, + (void **)tail, cnt, len, &ptype); + VERIFY((*head == NULL) || (ptype == QP_MBUF)); ifnet_decr_iorefcnt(ifp); return (rc); } +#if !CONFIG_EMBEDDED errno_t ifnet_framer_stub(struct ifnet *ifp, struct mbuf **m, const struct sockaddr *dest, const char *dest_linkaddr, @@ -3334,6 +3547,7 @@ ifnet_framer_stub(struct ifnet *ifp, struct mbuf **m, return (ifp->if_framer_legacy(ifp, m, dest, dest_linkaddr, frame_type)); } +#endif /* !CONFIG_EMBEDDED */ static int dlil_interface_filters_input(struct ifnet *ifp, struct mbuf **m_p, @@ -3522,17 +3736,12 @@ dlil_input_stats_sync(struct ifnet *ifp, struct dlil_threading_info *inp) atomic_add_64(&ifp->if_data.ifi_iqdrops, s->dropped); s->dropped = 0; } - /* - * If we went over the threshold, notify NetworkStatistics. - */ - if (ifp->if_data_threshold && - (ifp->if_ibytes + ifp->if_obytes) - ifp->if_dt_bytes > - ifp->if_data_threshold) { - ifp->if_dt_bytes = ifp->if_ibytes + ifp->if_obytes; + if (ifp->if_data_threshold != 0) { lck_mtx_convert_spin(&inp->input_lck); - nstat_ifnet_threshold_reached(ifp->if_index); + ifnet_notify_data_threshold(ifp); } + /* * No need for atomic operations as they are modified here * only from within the DLIL input thread context. @@ -3612,7 +3821,10 @@ dlil_input_packet_list_common(struct ifnet *ifp_param, struct mbuf *m, goto next; } iorefcnt = 1; - pktf_mask = 0; + /* + * Preserve the time stamp if it was set. + */ + pktf_mask = PKTF_TS_VALID; } else { /* * If this arrived on lo0, preserve interface addr @@ -3954,81 +4166,6 @@ dlil_get_socket_type(struct mbuf **mp, int family, int raw) } #endif -/* - * This is mostly called from the context of the DLIL input thread; - * because of that there is no need for atomic operations. - */ -static __inline void -ifp_inc_traffic_class_in(struct ifnet *ifp, struct mbuf *m) -{ - if (!(m->m_flags & M_PKTHDR)) - return; - - switch (m_get_traffic_class(m)) { - case MBUF_TC_BE: - ifp->if_tc.ifi_ibepackets++; - ifp->if_tc.ifi_ibebytes += m->m_pkthdr.len; - break; - case MBUF_TC_BK: - ifp->if_tc.ifi_ibkpackets++; - ifp->if_tc.ifi_ibkbytes += m->m_pkthdr.len; - break; - case MBUF_TC_VI: - ifp->if_tc.ifi_ivipackets++; - ifp->if_tc.ifi_ivibytes += m->m_pkthdr.len; - break; - case MBUF_TC_VO: - ifp->if_tc.ifi_ivopackets++; - ifp->if_tc.ifi_ivobytes += m->m_pkthdr.len; - break; - default: - break; - } - - if (mbuf_is_traffic_class_privileged(m)) { - ifp->if_tc.ifi_ipvpackets++; - ifp->if_tc.ifi_ipvbytes += m->m_pkthdr.len; - } -} - -/* - * This is called from DLIL output, hence multiple threads could end - * up modifying the statistics. We trade off acccuracy for performance - * by not using atomic operations here. - */ -static __inline void -ifp_inc_traffic_class_out(struct ifnet *ifp, struct mbuf *m) -{ - if (!(m->m_flags & M_PKTHDR)) - return; - - switch (m_get_traffic_class(m)) { - case MBUF_TC_BE: - ifp->if_tc.ifi_obepackets++; - ifp->if_tc.ifi_obebytes += m->m_pkthdr.len; - break; - case MBUF_TC_BK: - ifp->if_tc.ifi_obkpackets++; - ifp->if_tc.ifi_obkbytes += m->m_pkthdr.len; - break; - case MBUF_TC_VI: - ifp->if_tc.ifi_ovipackets++; - ifp->if_tc.ifi_ovibytes += m->m_pkthdr.len; - break; - case MBUF_TC_VO: - ifp->if_tc.ifi_ovopackets++; - ifp->if_tc.ifi_ovobytes += m->m_pkthdr.len; - break; - default: - break; - } - - if (mbuf_is_traffic_class_privileged(m)) { - ifp->if_tc.ifi_opvpackets++; - ifp->if_tc.ifi_opvbytes += m->m_pkthdr.len; - } -} - static void dlil_count_chain_len(mbuf_t m, struct chain_len_stats *cls) { @@ -4082,7 +4219,6 @@ errno_t dlil_output(ifnet_t ifp, protocol_family_t proto_family, mbuf_t packetlist, void *route, const struct sockaddr *dest, int raw, struct flowadv *adv) { - ifnet_output_handler_func handler_func; char *frame_type = NULL; char *dst_linkaddr = NULL; int retval = 0; @@ -4096,6 +4232,8 @@ dlil_output(ifnet_t ifp, protocol_family_t proto_family, mbuf_t packetlist, u_int32_t pre = 0, post = 0; u_int32_t fpkts = 0, fbytes = 0; int32_t flen = 0; + struct timespec now; + u_int64_t now_nsec; KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_START, 0, 0, 0, 0, 0); @@ -4109,8 +4247,7 @@ dlil_output(ifnet_t ifp, protocol_family_t proto_family, mbuf_t packetlist, } iorefcnt = 1; - handler_func = ifp->if_output_handler; - VERIFY(handler_func != NULL); + VERIFY(ifp->if_output_dlil != NULL); /* update the driver's multicast filter, if needed */ if (ifp->if_updatemcasts > 0 && if_mcasts_update(ifp) == 0) @@ -4276,20 +4413,6 @@ preout_again: goto cleanup; } - /* - * If the packet service class is not background, - * update the timestamp to indicate recent activity - * on a foreground socket. - */ - if ((m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) && - m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) { - if (!(m->m_pkthdr.pkt_flags & PKTF_SO_BACKGROUND)) - ifp->if_fg_sendts = net_uptime(); - - if (m->m_pkthdr.pkt_flags & PKTF_SO_REALTIME) - ifp->if_rt_sendts = net_uptime(); - } - ifp_inc_traffic_class_out(ifp, m); pktap_output(ifp, proto_family, m, pre, post); @@ -4300,6 +4423,29 @@ preout_again: dlil_count_chain_len(m, &tx_chain_len_stats); } + /* + * Record timestamp; ifnet_enqueue() will use this info + * rather than redoing the work. An optimization could + * involve doing this just once at the top, if there are + * no interface filters attached, but that's probably + * not a big deal. + */ + nanouptime(&now); + net_timernsec(&now, &now_nsec); + (void) mbuf_set_timestamp(m, now_nsec, TRUE); + + /* + * Discard partial sum information if this packet originated + * from another interface; the packet would already have the + * final checksum and we shouldn't recompute it. + */ + if ((m->m_pkthdr.pkt_flags & PKTF_FORWARDED) && + (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + m->m_pkthdr.csum_flags &= ~CSUM_TX_FLAGS; + m->m_pkthdr.csum_data = 0; + } + /* * Finally, call the driver. */ @@ -4319,7 +4465,7 @@ preout_again: } KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START, 0, 0, 0, 0, 0); - retval = (*handler_func)(ifp, m); + retval = (*ifp->if_output_dlil)(ifp, m); if (retval == EQFULL || retval == EQSUSPENDED) { if (adv != NULL && adv->code == FADV_SUCCESS) { adv->code = (retval == EQFULL ? @@ -4354,7 +4500,7 @@ next: KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START, 0, 0, 0, 0, 0); if (ifp->if_eflags & IFEF_SENDLIST) { - retval = (*handler_func)(ifp, send_head); + retval = (*ifp->if_output_dlil)(ifp, send_head); if (retval == EQFULL || retval == EQSUSPENDED) { if (adv != NULL) { adv->code = (retval == EQFULL ? @@ -4379,7 +4525,7 @@ next: send_m = send_head; send_head = send_m->m_nextpkt; send_m->m_nextpkt = NULL; - retval = (*handler_func)(ifp, send_m); + retval = (*ifp->if_output_dlil)(ifp, send_m); if (retval == EQFULL || retval == EQSUSPENDED) { if (adv != NULL) { adv->code = (retval == EQFULL ? @@ -4853,7 +4999,7 @@ ifnet_lookup(struct ifnet *ifp) { struct ifnet *_ifp; - lck_rw_assert(&ifnet_head_lock, LCK_RW_ASSERT_HELD); + LCK_RW_ASSERT(&ifnet_head_lock, LCK_RW_ASSERT_HELD); TAILQ_FOREACH(_ifp, &ifnet_head, if_link) { if (_ifp == ifp) break; @@ -4872,8 +5018,7 @@ ifnet_is_attached(struct ifnet *ifp, int refio) int ret; lck_mtx_lock_spin(&ifp->if_ref_lock); - if ((ret = ((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) == - IFRF_ATTACHED))) { + if ((ret = IF_FULLY_ATTACHED(ifp))) { if (refio > 0) ifp->if_refio++; } @@ -4891,8 +5036,7 @@ void ifnet_incr_iorefcnt(struct ifnet *ifp) { lck_mtx_lock_spin(&ifp->if_ref_lock); - VERIFY((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) == - IFRF_ATTACHED); + VERIFY(IF_FULLY_ATTACHED(ifp)); VERIFY(ifp->if_refio > 0); ifp->if_refio++; lck_mtx_unlock(&ifp->if_ref_lock); @@ -4903,17 +5047,16 @@ ifnet_decr_iorefcnt(struct ifnet *ifp) { lck_mtx_lock_spin(&ifp->if_ref_lock); VERIFY(ifp->if_refio > 0); - VERIFY((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) != 0); + VERIFY(ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)); ifp->if_refio--; /* * if there are no more outstanding io references, wakeup the * ifnet_detach thread if detaching flag is set. */ - if (ifp->if_refio == 0 && - (ifp->if_refflags & IFRF_DETACHING) != 0) { + if (ifp->if_refio == 0 && (ifp->if_refflags & IFRF_DETACHING)) wakeup(&(ifp->if_refio)); - } + lck_mtx_unlock(&ifp->if_ref_lock); } @@ -4967,26 +5110,39 @@ errno_t dlil_if_free(struct ifnet *ifp) { struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp; + bool need_release = FALSE; if (dl_if == NULL) return (EINVAL); lck_mtx_lock_spin(&dl_if->dl_if_lock); - if (dl_if->dl_if_refcnt == 0) { + switch (dl_if->dl_if_refcnt) { + case 0: panic("%s: negative refcnt for ifp=%p", __func__, ifp); /* NOTREACHED */ + break; + case 1: + if ((ifp->if_refflags & IFRF_EMBRYONIC) != 0) { + need_release = TRUE; + } + break; + default: + break; } --dl_if->dl_if_refcnt; if (dl_if->dl_if_trace != NULL) (*dl_if->dl_if_trace)(dl_if, FALSE); lck_mtx_unlock(&dl_if->dl_if_lock); - + if (need_release) { + dlil_if_release(ifp); + } return (0); } static errno_t dlil_attach_protocol_internal(struct if_proto *proto, - const struct ifnet_demux_desc *demux_list, u_int32_t demux_count) + const struct ifnet_demux_desc *demux_list, u_int32_t demux_count, + uint32_t * proto_count) { struct kev_dl_proto_data ev_pr_data; struct ifnet *ifp = proto->ifp; @@ -5041,6 +5197,9 @@ dlil_attach_protocol_internal(struct if_proto *proto, dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_PROTO_ATTACHED, (struct net_event_data *)&ev_pr_data, sizeof (struct kev_dl_proto_data)); + if (proto_count != NULL) { + *proto_count = ev_pr_data.proto_remaining_count; + } return (retval); } @@ -5050,6 +5209,7 @@ ifnet_attach_protocol(ifnet_t ifp, protocol_family_t protocol, { int retval = 0; struct if_proto *ifproto = NULL; + uint32_t proto_count = 0; ifnet_head_lock_shared(); if (ifp == NULL || protocol == 0 || proto_details == NULL) { @@ -5082,21 +5242,25 @@ ifnet_attach_protocol(ifnet_t ifp, protocol_family_t protocol, ifproto->kpi.v1.send_arp = proto_details->send_arp; retval = dlil_attach_protocol_internal(ifproto, - proto_details->demux_list, proto_details->demux_count); - - if (dlil_verbose) { - printf("%s: attached v1 protocol %d\n", if_name(ifp), - protocol); - } + proto_details->demux_list, proto_details->demux_count, + &proto_count); end: if (retval != 0 && retval != EEXIST && ifp != NULL) { DLIL_PRINTF("%s: failed to attach v1 protocol %d (err=%d)\n", if_name(ifp), protocol, retval); + } else { + if (dlil_verbose) { + printf("%s: attached v1 protocol %d (count = %d)\n", + if_name(ifp), + protocol, proto_count); + } } ifnet_head_done(); - if (retval != 0 && ifproto != NULL) + if (retval == 0) { + } else if (ifproto != NULL) { zfree(dlif_proto_zone, ifproto); + } return (retval); } @@ -5106,6 +5270,7 @@ ifnet_attach_protocol_v2(ifnet_t ifp, protocol_family_t protocol, { int retval = 0; struct if_proto *ifproto = NULL; + uint32_t proto_count = 0; ifnet_head_lock_shared(); if (ifp == NULL || protocol == 0 || proto_details == NULL) { @@ -5138,21 +5303,25 @@ ifnet_attach_protocol_v2(ifnet_t ifp, protocol_family_t protocol, ifproto->kpi.v2.send_arp = proto_details->send_arp; retval = dlil_attach_protocol_internal(ifproto, - proto_details->demux_list, proto_details->demux_count); - - if (dlil_verbose) { - printf("%s: attached v2 protocol %d\n", if_name(ifp), - protocol); - } + proto_details->demux_list, proto_details->demux_count, + &proto_count); end: if (retval != 0 && retval != EEXIST && ifp != NULL) { DLIL_PRINTF("%s: failed to attach v2 protocol %d (err=%d)\n", if_name(ifp), protocol, retval); + } else { + if (dlil_verbose) { + printf("%s: attached v2 protocol %d (count = %d)\n", + if_name(ifp), + protocol, proto_count); + } } ifnet_head_done(); - if (retval != 0 && ifproto != NULL) + if (retval == 0) { + } else if (ifproto != NULL) { zfree(dlif_proto_zone, ifproto); + } return (retval); } @@ -5316,8 +5485,8 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) } lck_mtx_lock_spin(&ifp->if_ref_lock); - if (ifp->if_refflags & IFRF_ATTACHED) { - panic_plain("%s: flags mismatch (attached set) ifp=%p", + if (!(ifp->if_refflags & IFRF_EMBRYONIC)) { + panic_plain("%s: flags mismatch (embryonic not set) ifp=%p", __func__, ifp); /* NOTREACHED */ } @@ -5396,7 +5565,6 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) VERIFY(ifindex2ifnet[ifp->if_index] == NULL); /* allocate (if needed) and initialize a link address */ - VERIFY(!(dl_if->dl_if_flags & DLIF_REUSE) || ifp->if_lladdr != NULL); ifa = dlil_alloc_lladdr(ifp, ll_addr); if (ifa == NULL) { ifnet_lock_done(ifp); @@ -5453,6 +5621,10 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) if (if_delaybased_queue) sflags |= PKTSCHEDF_QALG_DELAYBASED; + if (ifp->if_output_sched_model == + IFNET_SCHED_MODEL_DRIVER_MANAGED) + sflags |= PKTSCHEDF_QALG_DRIVER_MANAGED; + /* Initialize transmit queue(s) */ err = ifclassq_setup(ifp, sflags, (dl_if->dl_if_flags & DLIF_REUSE)); if (err != 0) { @@ -5500,6 +5672,12 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) } } + if (ifp->if_inp != NULL && ifp->if_inp->input_mit_tcall == NULL) { + ifp->if_inp->input_mit_tcall = + thread_call_allocate_with_priority(dlil_mit_tcall_fn, + ifp, THREAD_CALL_PRIORITY_KERNEL); + } + /* * If the driver supports the new transmit model, calculate flow hash * and create a workloop starter thread to invoke the if_start callback @@ -5508,19 +5686,19 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) if (ifp->if_eflags & IFEF_TXSTART) { ifp->if_flowhash = ifnet_calc_flowhash(ifp); VERIFY(ifp->if_flowhash != 0); - - VERIFY(ifp->if_start != NULL); VERIFY(ifp->if_start_thread == THREAD_NULL); ifnet_set_start_cycle(ifp, NULL); ifp->if_start_active = 0; ifp->if_start_req = 0; ifp->if_start_flags = 0; - if ((err = kernel_thread_start(ifnet_start_thread_fn, ifp, - &ifp->if_start_thread)) != KERN_SUCCESS) { - panic_plain("%s: ifp=%p couldn't get a start thread; " + VERIFY(ifp->if_start != NULL); + if ((err = kernel_thread_start(ifnet_start_thread_fn, + ifp, &ifp->if_start_thread)) != KERN_SUCCESS) { + panic_plain("%s: " + "ifp=%p couldn't get a start thread; " "err=%d", __func__, ifp, err); - /* NOTREACHED */ + /* NOTREACHED */ } ml_thread_policy(ifp->if_start_thread, MACHINE_GROUP, (MACHINE_NETWORK_GROUP|MACHINE_NETWORK_WORKLOOP)); @@ -5577,7 +5755,10 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) /* Clear logging parameters */ bzero(&ifp->if_log, sizeof (ifp->if_log)); + + /* Clear foreground/realtime activity timestamps */ ifp->if_fg_sendts = 0; + ifp->if_rt_sendts = 0; VERIFY(ifp->if_delegated.ifp == NULL); VERIFY(ifp->if_delegated.type == 0); @@ -5636,6 +5817,7 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) ifnet_lock_done(ifp); ifnet_head_done(); + lck_mtx_lock(&ifp->if_cached_route_lock); /* Enable forwarding cached route */ ifp->if_fwd_cacheok = 1; @@ -5679,6 +5861,7 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) #endif /* INET6 */ VERIFY(ifp->if_data_threshold == 0); + VERIFY(ifp->if_dt_tcall != NULL); /* * Finally, mark this ifnet as attached. @@ -5686,7 +5869,7 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) lck_mtx_lock(rnh_lock); ifnet_lock_exclusive(ifp); lck_mtx_lock_spin(&ifp->if_ref_lock); - ifp->if_refflags = IFRF_ATTACHED; + ifp->if_refflags = IFRF_ATTACHED; /* clears embryonic */ lck_mtx_unlock(&ifp->if_ref_lock); if (net_rtref) { /* boot-args override; enable idle notification */ @@ -5887,6 +6070,7 @@ ifnet_detach(ifnet_t ifp) lck_mtx_unlock(rnh_lock); return (ENXIO); } + VERIFY(!(ifp->if_refflags & IFRF_EMBRYONIC)); /* Indicate this interface is being detached */ ifp->if_refflags &= ~IFRF_ATTACHED; ifp->if_refflags |= IFRF_DETACHING; @@ -5933,6 +6117,7 @@ ifnet_detach(ifnet_t ifp) ifnet_head_done(); lck_mtx_unlock(rnh_lock); + /* Release reference held on the delegated interface */ if (delegated_ifp != NULL) ifnet_release(delegated_ifp); @@ -5982,7 +6167,11 @@ ifnet_detach(ifnet_t ifp) ifp->if_fwd_cacheok = 0; lck_mtx_unlock(&ifp->if_cached_route_lock); + /* Disable data threshold and wait for any pending event posting */ ifp->if_data_threshold = 0; + VERIFY(ifp->if_dt_tcall != NULL); + (void) thread_call_cancel_wait(ifp->if_dt_tcall); + /* * Drain any deferred IGMPv3/MLDv2 query responses, but keep the * references to the info structures and leave them attached to @@ -6114,7 +6303,7 @@ ifnet_detach_final(struct ifnet *ifp) lck_mtx_lock(&ifp->if_flt_lock); if_flt_monitor_enter(ifp); - lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); fhead = ifp->if_flt_head; TAILQ_INIT(&ifp->if_flt_head); @@ -6242,22 +6431,36 @@ ifnet_detach_final(struct ifnet *ifp) /* disassociate ifp DLIL input thread */ ifp->if_inp = NULL; + /* tell the input thread to terminate */ lck_mtx_lock_spin(&inp->input_lck); inp->input_waiting |= DLIL_INPUT_TERMINATE; if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) { wakeup_one((caddr_t)&inp->input_waiting); } lck_mtx_unlock(&inp->input_lck); + + /* wait for the input thread to terminate */ + lck_mtx_lock_spin(&inp->input_lck); + while ((inp->input_waiting & DLIL_INPUT_TERMINATE_COMPLETE) + == 0) { + (void) msleep(&inp->input_waiting, &inp->input_lck, + (PZERO - 1) | PSPIN, inp->input_name, NULL); + } + lck_mtx_unlock(&inp->input_lck); + + /* clean-up input thread state */ + dlil_clean_threading_info(inp); + } /* The driver might unload, so point these to ourselves */ if_free = ifp->if_free; - ifp->if_output_handler = ifp_if_output; + ifp->if_output_dlil = ifp_if_output; ifp->if_output = ifp_if_output; ifp->if_pre_enqueue = ifp_if_output; ifp->if_start = ifp_if_start; ifp->if_output_ctl = ifp_if_ctl; - ifp->if_input_handler = ifp_if_input; + ifp->if_input_dlil = ifp_if_input; ifp->if_input_poll = ifp_if_input_poll; ifp->if_input_ctl = ifp_if_ctl; ifp->if_ioctl = ifp_if_ioctl; @@ -6288,6 +6491,7 @@ ifnet_detach_final(struct ifnet *ifp) ifp->if_eflags &= ~IFEF_QOSMARKING_ENABLED; if_set_qosmarking_mode(ifp, IFRTYPE_QOSMARKING_MODE_NONE); + ifnet_lock_done(ifp); #if PF @@ -6319,6 +6523,8 @@ ifnet_detach_final(struct ifnet *ifp) lck_mtx_unlock(&ifp->if_cached_route_lock); VERIFY(ifp->if_data_threshold == 0); + VERIFY(ifp->if_dt_tcall != NULL); + VERIFY(!thread_call_isactive(ifp->if_dt_tcall)); ifnet_llreach_ifdetach(ifp); @@ -6345,7 +6551,7 @@ ifnet_detach_final(struct ifnet *ifp) ifnet_release(ifp); } -static errno_t +errno_t ifp_if_output(struct ifnet *ifp, struct mbuf *m) { #pragma unused(ifp) @@ -6353,7 +6559,7 @@ ifp_if_output(struct ifnet *ifp, struct mbuf *m) return (0); } -static void +void ifp_if_start(struct ifnet *ifp) { ifnet_purge(ifp); @@ -6421,12 +6627,23 @@ ifp_if_check_multi(struct ifnet *ifp, const struct sockaddr *sa) return (EOPNOTSUPP); } +#if CONFIG_EMBEDDED +static errno_t +ifp_if_framer(struct ifnet *ifp, struct mbuf **m, + const struct sockaddr *sa, const char *ll, const char *t, + u_int32_t *pre, u_int32_t *post) +#else static errno_t ifp_if_framer(struct ifnet *ifp, struct mbuf **m, const struct sockaddr *sa, const char *ll, const char *t) +#endif /* !CONFIG_EMBEDDED */ { #pragma unused(ifp, m, sa, ll, t) +#if CONFIG_EMBEDDED + return (ifp_if_framer_extended(ifp, m, sa, ll, t, pre, post)); +#else return (ifp_if_framer_extended(ifp, m, sa, ll, t, NULL, NULL)); +#endif /* !CONFIG_EMBEDDED */ } static errno_t @@ -6535,7 +6752,7 @@ int dlil_if_acquire(u_int32_t family, const void *uniqueid, MALLOC(dlifp1->dl_if_uniqueid, void *, uniqueid_len, M_NKE, M_WAITOK); if (dlifp1->dl_if_uniqueid == NULL) { - zfree(dlif_zone, dlifp1); + zfree(dlif_zone, buf); ret = ENOMEM; goto end; } @@ -6557,6 +6774,7 @@ int dlil_if_acquire(u_int32_t family, const void *uniqueid, ifp1->if_desc.ifd_len = 0; ifp1->if_desc.ifd_desc = dlifp1->dl_if_descstorage; + #if CONFIG_MACF_NET mac_ifnet_label_init(ifp1); #endif @@ -6601,6 +6819,14 @@ int dlil_if_acquire(u_int32_t family, const void *uniqueid, lck_mtx_init(&ifp1->if_poll_lock, ifnet_rcv_lock_group, ifnet_lock_attr); + /* thread call allocation is done with sleeping zalloc */ + ifp1->if_dt_tcall = thread_call_allocate_with_options(dlil_dt_tcall_fn, + ifp1, THREAD_CALL_PRIORITY_KERNEL, THREAD_CALL_OPTIONS_ONCE); + if (ifp1->if_dt_tcall == NULL) { + panic_plain("%s: couldn't create if_dt_tcall", __func__); + /* NOTREACHED */ + } + TAILQ_INSERT_TAIL(&dlil_ifnet_head, dlifp1, dl_if_link); *ifp = ifp1; @@ -6619,6 +6845,11 @@ dlil_if_release(ifnet_t ifp) { struct dlil_ifnet *dlifp = (struct dlil_ifnet *)ifp; + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ifnet_alloc_count) > 0); + if (!(ifp->if_xflags & IFXF_ALLOC_KPI)) { + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ifnet_alloc_os_count) > 0); + } + ifnet_lock_exclusive(ifp); lck_mtx_lock(&dlifp->dl_if_lock); dlifp->dl_if_flags &= ~DLIF_INUSE; @@ -6656,7 +6887,7 @@ dlil_if_unlock(void) __private_extern__ void dlil_if_lock_assert(void) { - lck_mtx_assert(&dlil_ifnet_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dlil_ifnet_lock, LCK_MTX_ASSERT_OWNED); } __private_extern__ void @@ -6750,17 +6981,16 @@ ifnet_cached_rtlookup_inet(struct ifnet *ifp, struct in_addr src_ip) } dst->sin_addr = src_ip; - if (src_rt.ro_rt == NULL) { - src_rt.ro_rt = rtalloc1_scoped((struct sockaddr *)dst, - 0, 0, ifp->if_index); + VERIFY(src_rt.ro_rt == NULL); + src_rt.ro_rt = rtalloc1_scoped((struct sockaddr *)dst, + 0, 0, ifp->if_index); - if (src_rt.ro_rt != NULL) { - /* retain a ref, copyin consumes one */ - struct rtentry *rte = src_rt.ro_rt; - RT_ADDREF(rte); - ifp_src_route_copyin(ifp, &src_rt); - src_rt.ro_rt = rte; - } + if (src_rt.ro_rt != NULL) { + /* retain a ref, copyin consumes one */ + struct rtentry *rte = src_rt.ro_rt; + RT_ADDREF(rte); + ifp_src_route_copyin(ifp, &src_rt); + src_rt.ro_rt = rte; } } @@ -6814,12 +7044,21 @@ if_lqm_update(struct ifnet *ifp, int lqm, int locked) VERIFY(lqm >= IFNET_LQM_MIN && lqm <= IFNET_LQM_MAX); /* Normalize to edge */ - if (lqm >= 0 && lqm <= IFNET_LQM_THRESH_BAD) - lqm = IFNET_LQM_THRESH_BAD; - else if (lqm > IFNET_LQM_THRESH_BAD && lqm <= IFNET_LQM_THRESH_POOR) + if (lqm >= 0 && lqm <= IFNET_LQM_THRESH_ABORT) { + lqm = IFNET_LQM_THRESH_ABORT; + atomic_bitset_32(&tcbinfo.ipi_flags, + INPCBINFO_HANDLE_LQM_ABORT); + inpcb_timer_sched(&tcbinfo, INPCB_TIMER_FAST); + } else if (lqm > IFNET_LQM_THRESH_ABORT && + lqm <= IFNET_LQM_THRESH_MINIMALLY_VIABLE) { + lqm = IFNET_LQM_THRESH_MINIMALLY_VIABLE; + } else if (lqm > IFNET_LQM_THRESH_MINIMALLY_VIABLE && + lqm <= IFNET_LQM_THRESH_POOR) { lqm = IFNET_LQM_THRESH_POOR; - else if (lqm > IFNET_LQM_THRESH_POOR && lqm <= IFNET_LQM_THRESH_GOOD) + } else if (lqm > IFNET_LQM_THRESH_POOR && + lqm <= IFNET_LQM_THRESH_GOOD) { lqm = IFNET_LQM_THRESH_GOOD; + } /* * Take the lock if needed @@ -7000,6 +7239,10 @@ if_probe_connectivity(struct ifnet *ifp, u_int32_t conn_probe) ifp->if_eflags |= IFEF_PROBE_CONNECTIVITY; ifnet_lock_done(ifp); +#if NECP + necp_update_all_clients(); +#endif /* NECP */ + tcp_probe_connectivity(ifp, conn_probe); return (0); } @@ -7265,19 +7508,7 @@ dlil_ifaddr_bytes(const struct sockaddr_dl *sdl, size_t *sizep, [0] = 2 }; - switch (sdl->sdl_type) { - case IFT_ETHER: - VERIFY(size == ETHER_ADDR_LEN); - bytes = unspec; - break; - case IFT_IEEE1394: - VERIFY(size == FIREWIRE_EUI64_LEN); - bytes = unspec; - break; - default: - VERIFY(FALSE); - break; - }; + bytes = unspec; } } #else @@ -7402,11 +7633,6 @@ ifnet_set_throttle(struct ifnet *ifp, u_int32_t level) switch (level) { case IFNET_THROTTLE_OFF: case IFNET_THROTTLE_OPPORTUNISTIC: -#if PF_ALTQ - /* Throttling works only for IFCQ, not ALTQ instances */ - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) - return (ENXIO); -#endif /* PF_ALTQ */ break; default: return (EINVAL); @@ -7569,7 +7795,7 @@ ifnet_flowid(struct ifnet *ifp, uint32_t *flowid) if (ifp == NULL || flowid == NULL) { return (EINVAL); } else if (!(ifp->if_eflags & IFEF_TXSTART) || - !(ifp->if_refflags & IFRF_ATTACHED)) { + !IF_FULLY_ATTACHED(ifp)) { return (ENXIO); } @@ -7586,7 +7812,7 @@ ifnet_disable_output(struct ifnet *ifp) if (ifp == NULL) { return (EINVAL); } else if (!(ifp->if_eflags & IFEF_TXSTART) || - !(ifp->if_refflags & IFRF_ATTACHED)) { + !IF_FULLY_ATTACHED(ifp)) { return (ENXIO); } @@ -7604,7 +7830,7 @@ ifnet_enable_output(struct ifnet *ifp) if (ifp == NULL) { return (EINVAL); } else if (!(ifp->if_eflags & IFEF_TXSTART) || - !(ifp->if_refflags & IFRF_ATTACHED)) { + !IF_FULLY_ATTACHED(ifp)) { return (ENXIO); } @@ -7838,7 +8064,7 @@ ifnet_get_netsignature(struct ifnet *ifp, uint8_t family, uint8_t *len, { int error = 0; - if (ifp == NULL || len == NULL || flags == NULL || data == NULL) + if (ifp == NULL || len == NULL || data == NULL) return (EINVAL); switch (family) { @@ -7883,12 +8109,104 @@ ifnet_get_netsignature(struct ifnet *ifp, uint8_t family, uint8_t *len, break; } - if (error == 0) + if (error == 0 && flags != NULL) *flags = 0; return (error); } +#if INET6 +int +ifnet_set_nat64prefix(struct ifnet *ifp, struct ipv6_prefix *prefixes) +{ + int i, error = 0, one_set = 0; + + if_inet6data_lock_exclusive(ifp); + + if (IN6_IFEXTRA(ifp) == NULL) { + error = ENOMEM; + goto out; + } + + for (i = 0; i < NAT64_MAX_NUM_PREFIXES; i++) { + uint32_t prefix_len = + prefixes[i].prefix_len; + struct in6_addr *prefix = + &prefixes[i].ipv6_prefix; + + if (prefix_len == 0) { + /* Allow clearing the signature */ + IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len = 0; + bzero(&IN6_IFEXTRA(ifp)->nat64_prefixes[i].ipv6_prefix, + sizeof(struct in6_addr)); + + continue; + } else if (prefix_len != NAT64_PREFIX_LEN_32 && + prefix_len != NAT64_PREFIX_LEN_40 && + prefix_len != NAT64_PREFIX_LEN_48 && + prefix_len != NAT64_PREFIX_LEN_56 && + prefix_len != NAT64_PREFIX_LEN_64 && + prefix_len != NAT64_PREFIX_LEN_96) { + error = EINVAL; + goto out; + } + + if (IN6_IS_SCOPE_EMBED(prefix)) { + error = EINVAL; + goto out; + } + + IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len = prefix_len; + bcopy(prefix, &IN6_IFEXTRA(ifp)->nat64_prefixes[i].ipv6_prefix, + sizeof(struct in6_addr)); + one_set = 1; + } + +out: + if_inet6data_lock_done(ifp); + + if (error == 0 && one_set != 0) + necp_update_all_clients(); + + return (error); +} + +int +ifnet_get_nat64prefix(struct ifnet *ifp, struct ipv6_prefix *prefixes) +{ + int i, found_one = 0, error = 0; + + if (ifp == NULL) + return (EINVAL); + + if_inet6data_lock_shared(ifp); + + if (IN6_IFEXTRA(ifp) == NULL) { + error = ENOMEM; + goto out; + } + + for (i = 0; i < NAT64_MAX_NUM_PREFIXES; i++) { + if (IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len != 0) + found_one = 1; + } + + if (found_one == 0) { + error = ENOENT; + goto out; + } + + if (prefixes) + bcopy(IN6_IFEXTRA(ifp)->nat64_prefixes, prefixes, + sizeof(IN6_IFEXTRA(ifp)->nat64_prefixes)); + +out: + if_inet6data_lock_done(ifp); + + return (error); +} +#endif + static void dlil_output_cksum_dbg(struct ifnet *ifp, struct mbuf *m, uint32_t hoff, protocol_family_t pf) @@ -7932,7 +8250,7 @@ static void dlil_input_cksum_dbg(struct ifnet *ifp, struct mbuf *m, char *frame_header, protocol_family_t pf) { - uint16_t sum; + uint16_t sum = 0; uint32_t hlen; if (frame_header == NULL || @@ -8042,7 +8360,8 @@ dlil_input_cksum_dbg(struct ifnet *ifp, struct mbuf *m, char *frame_header, if (aoff == rxoff || aoff > (uint32_t)m->m_pkthdr.len) return; - sum = m_adj_sum16(m, rxoff, aoff, sum); + sum = m_adj_sum16(m, rxoff, aoff, + m_pktlen(m) - aoff, sum); m->m_pkthdr.csum_rx_val = sum; m->m_pkthdr.csum_rx_start = (aoff + hlen); @@ -8137,7 +8456,7 @@ sysctl_tx_chain_len_stats SYSCTL_HANDLER_ARGS } -#if DEBUG +#if DEBUG || DEVELOPMENT /* Blob for sum16 verification */ static uint8_t sumdata[] = { 0x1f, 0x8b, 0x08, 0x08, 0x4c, 0xe5, 0x9a, 0x4f, 0x00, 0x03, @@ -8177,19 +8496,26 @@ static uint8_t sumdata[] = { /* Precomputed 16-bit 1's complement sums for various spans of the above data */ static struct { - int len; - uint16_t sum; + boolean_t init; + uint16_t len; + uint16_t sumr; /* reference */ + uint16_t sumrp; /* reference, precomputed */ } sumtbl[] = { - { 11, 0xcb6d }, - { 20, 0x20dd }, - { 27, 0xbabd }, - { 32, 0xf3e8 }, - { 37, 0x197d }, - { 43, 0x9eae }, - { 64, 0x4678 }, - { 127, 0x9399 }, - { 256, 0xd147 }, - { 325, 0x0358 } + { FALSE, 0, 0, 0x0000 }, + { FALSE, 1, 0, 0x001f }, + { FALSE, 2, 0, 0x8b1f }, + { FALSE, 3, 0, 0x8b27 }, + { FALSE, 7, 0, 0x790e }, + { FALSE, 11, 0, 0xcb6d }, + { FALSE, 20, 0, 0x20dd }, + { FALSE, 27, 0, 0xbabd }, + { FALSE, 32, 0, 0xf3e8 }, + { FALSE, 37, 0, 0x197d }, + { FALSE, 43, 0, 0x9eae }, + { FALSE, 64, 0, 0x4678 }, + { FALSE, 127, 0, 0x9399 }, + { FALSE, 256, 0, 0xd147 }, + { FALSE, 325, 0, 0x0358 }, }; #define SUMTBL_MAX ((int)sizeof (sumtbl) / (int)sizeof (sumtbl[0])) @@ -8203,6 +8529,8 @@ dlil_verify_sum16(void) /* Make sure test data plus extra room for alignment fits in cluster */ _CASSERT((sizeof (sumdata) + (sizeof (uint64_t) * 2)) <= MCLBYTES); + kprintf("DLIL: running SUM16 self-tests ... "); + m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR); MH_ALIGN(m, sizeof (uint32_t)); /* 32-bit starting alignment */ buf = mtod(m, uint8_t *); /* base address */ @@ -8213,7 +8541,7 @@ dlil_verify_sum16(void) /* Verify for all possible alignments */ for (i = 0; i < (int)sizeof (uint64_t); i++) { - uint16_t sum; + uint16_t sum, sumr; uint8_t *c; /* Copy over test data to mbuf */ @@ -8226,11 +8554,25 @@ dlil_verify_sum16(void) m->m_len = len; sum = m_sum16(m, 0, len); + if (!sumtbl[n].init) { + sumr = in_cksum_mbuf_ref(m, len, 0, 0); + sumtbl[n].sumr = sumr; + sumtbl[n].init = TRUE; + } else { + sumr = sumtbl[n].sumr; + } + /* Something is horribly broken; stop now */ - if (sum != sumtbl[n].sum) { - panic("%s: broken m_sum16 for len=%d align=%d " - "sum=0x%04x [expected=0x%04x]\n", __func__, - len, i, sum, sumtbl[n].sum); + if (sumr != sumtbl[n].sumrp) { + panic_plain("\n%s: broken in_cksum_mbuf_ref() " + "for len=%d align=%d sum=0x%04x " + "[expected=0x%04x]\n", __func__, + len, i, sum, sumr); + /* NOTREACHED */ + } else if (sum != sumr) { + panic_plain("\n%s: broken m_sum16() for len=%d " + "align=%d sum=0x%04x [expected=0x%04x]\n", + __func__, len, i, sum, sumr); /* NOTREACHED */ } @@ -8240,10 +8582,10 @@ dlil_verify_sum16(void) sum = m_sum16(m, i, len); /* Something is horribly broken; stop now */ - if (sum != sumtbl[n].sum) { - panic("%s: broken m_sum16 for len=%d offset=%d " - "sum=0x%04x [expected=0x%04x]\n", __func__, - len, i, sum, sumtbl[n].sum); + if (sum != sumr) { + panic_plain("\n%s: broken m_sum16() for len=%d " + "offset=%d sum=0x%04x [expected=0x%04x]\n", + __func__, len, i, sum, sumr); /* NOTREACHED */ } #if INET @@ -8251,10 +8593,10 @@ dlil_verify_sum16(void) sum = b_sum16(c, len); /* Something is horribly broken; stop now */ - if (sum != sumtbl[n].sum) { - panic("%s: broken b_sum16 for len=%d align=%d " - "sum=0x%04x [expected=0x%04x]\n", __func__, - len, i, sum, sumtbl[n].sum); + if (sum != sumr) { + panic_plain("\n%s: broken b_sum16() for len=%d " + "align=%d sum=0x%04x [expected=0x%04x]\n", + __func__, len, i, sum, sumr); /* NOTREACHED */ } #endif /* INET */ @@ -8262,9 +8604,9 @@ dlil_verify_sum16(void) } m_freem(m); - printf("DLIL: SUM16 self-tests PASSED\n"); + kprintf("PASSED\n"); } -#endif /* DEBUG */ +#endif /* DEBUG || DEVELOPMENT */ #define CASE_STRINGIFY(x) case x: return #x @@ -8353,7 +8695,7 @@ sysctl_get_ports_used SYSCTL_HANDLER_ARGS ifp = ifindex2ifnet[idx]; ifnet_head_done(); - bitfield = _MALLOC(bitstr_size(65536), M_TEMP, M_WAITOK); + bitfield = _MALLOC(bitstr_size(65536), M_TEMP, M_WAITOK | M_ZERO); if (bitfield == NULL) { error = ENOMEM; goto done; @@ -8371,6 +8713,48 @@ done: return (error); } +static void +dlil_dt_tcall_fn(thread_call_param_t arg0, thread_call_param_t arg1) +{ +#pragma unused(arg1) + struct ifnet *ifp = arg0; + + if (ifnet_is_attached(ifp, 1)) { + nstat_ifnet_threshold_reached(ifp->if_index); + ifnet_decr_iorefcnt(ifp); + } +} + +void +ifnet_notify_data_threshold(struct ifnet *ifp) +{ + uint64_t bytes = (ifp->if_ibytes + ifp->if_obytes); + uint64_t oldbytes = ifp->if_dt_bytes; + + ASSERT(ifp->if_dt_tcall != NULL); + + /* + * If we went over the threshold, notify NetworkStatistics. + * We rate-limit it based on the threshold interval value. + */ + if (threshold_notify && (bytes - oldbytes) > ifp->if_data_threshold && + OSCompareAndSwap64(oldbytes, bytes, &ifp->if_dt_bytes) && + !thread_call_isactive(ifp->if_dt_tcall)) { + uint64_t tival = (threshold_interval * NSEC_PER_SEC); + uint64_t now = mach_absolute_time(), deadline = now; + uint64_t ival; + + if (tival != 0) { + nanoseconds_to_absolutetime(tival, &ival); + clock_deadline_for_periodic_event(ival, now, &deadline); + (void) thread_call_enter_delayed(ifp->if_dt_tcall, + deadline); + } else { + (void) thread_call_enter(ifp->if_dt_tcall); + } + } +} + #if (DEVELOPMENT || DEBUG) /* * The sysctl variable name contains the input parameters of @@ -8468,3 +8852,34 @@ done: return (error); } #endif /* DEVELOPMENT || DEBUG */ + +void +ifnet_update_stats_per_flow(struct ifnet_stats_per_flow *ifs, + struct ifnet *ifp) +{ + tcp_update_stats_per_flow(ifs, ifp); +} + +static void +dlil_mit_tcall_fn(thread_call_param_t arg0, thread_call_param_t arg1) +{ +#pragma unused(arg1) + struct ifnet *ifp = (struct ifnet *)arg0; + struct dlil_threading_info *inp = ifp->if_inp; + + ifnet_lock_shared(ifp); + if (!IF_FULLY_ATTACHED(ifp) || inp == NULL) { + ifnet_lock_done(ifp); + return; + } + + lck_mtx_lock_spin(&inp->input_lck); + inp->input_waiting |= DLIL_INPUT_WAITING; + if (!(inp->input_waiting & DLIL_INPUT_RUNNING) || + !qempty(&inp->rcvq_pkts)) { + inp->wtot++; + wakeup_one((caddr_t)&inp->input_waiting); + } + lck_mtx_unlock(&inp->input_lck); + ifnet_lock_done(ifp); +} diff --git a/bsd/net/dlil.h b/bsd/net/dlil.h index a412d25fc..5a6c669e4 100644 --- a/bsd/net/dlil.h +++ b/bsd/net/dlil.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999-2016 Apple Inc. All rights reserved. + * Copyright (c) 1999-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -202,6 +202,7 @@ struct dlil_threading_info { */ u_int64_t input_mbuf_cnt; /* total # of packets processed */ #endif + thread_call_t input_mit_tcall; /* coalescing input processing */ }; /* @@ -221,11 +222,13 @@ struct dlil_main_threading_info { #define DLIL_PROTO_REGISTER 0x20000000 #define DLIL_PROTO_WAITING 0x10000000 #define DLIL_INPUT_TERMINATE 0x08000000 +#define DLIL_INPUT_TERMINATE_COMPLETE 0x04000000 /* * Flags for dlil_attach_filter() */ #define DLIL_IFF_TSO 0x01 /* Interface filter supports TSO */ +#define DLIL_IFF_INTERNAL 0x02 /* Apple internal -- do not count towards stats */ extern int dlil_verbose; extern uint32_t hwcksum_dbg; @@ -236,6 +239,8 @@ extern struct dlil_threading_info *dlil_main_input_thread; extern void dlil_init(void); extern errno_t ifp_if_ioctl(struct ifnet *, unsigned long, void *); +extern errno_t ifp_if_output(struct ifnet *, struct mbuf *); +extern void ifp_if_start(struct ifnet *); extern errno_t dlil_set_bpf_tap(ifnet_t, bpf_tap_mode, bpf_packet_func); @@ -325,6 +330,7 @@ extern int dlil_post_complete_msg(struct ifnet *, struct kev_msg *); extern int dlil_alloc_local_stats(struct ifnet *); + /* * dlil_if_acquire is obsolete. Use ifnet_allocate. */ @@ -364,6 +370,83 @@ extern errno_t dlil_input_handler(struct ifnet *, struct mbuf *, struct mbuf *, const struct ifnet_stat_increment_param *, boolean_t, struct thread *); + +/* + * This is mostly called from the context of the DLIL input thread; + * because of that there is no need for atomic operations. + */ +__attribute__((always_inline)) +static inline void +ifp_inc_traffic_class_in(struct ifnet *ifp, struct mbuf *m) +{ + if (!(m->m_flags & M_PKTHDR)) + return; + + switch (m_get_traffic_class(m)) { + case MBUF_TC_BE: + ifp->if_tc.ifi_ibepackets++; + ifp->if_tc.ifi_ibebytes += m->m_pkthdr.len; + break; + case MBUF_TC_BK: + ifp->if_tc.ifi_ibkpackets++; + ifp->if_tc.ifi_ibkbytes += m->m_pkthdr.len; + break; + case MBUF_TC_VI: + ifp->if_tc.ifi_ivipackets++; + ifp->if_tc.ifi_ivibytes += m->m_pkthdr.len; + break; + case MBUF_TC_VO: + ifp->if_tc.ifi_ivopackets++; + ifp->if_tc.ifi_ivobytes += m->m_pkthdr.len; + break; + default: + break; + } + + if (mbuf_is_traffic_class_privileged(m)) { + ifp->if_tc.ifi_ipvpackets++; + ifp->if_tc.ifi_ipvbytes += m->m_pkthdr.len; + } +} + +/* + * This is called from DLIL output, hence multiple threads could end + * up modifying the statistics. We trade off acccuracy for performance + * by not using atomic operations here. + */ +__attribute__((always_inline)) +static inline void +ifp_inc_traffic_class_out(struct ifnet *ifp, struct mbuf *m) +{ + if (!(m->m_flags & M_PKTHDR)) + return; + + switch (m_get_traffic_class(m)) { + case MBUF_TC_BE: + ifp->if_tc.ifi_obepackets++; + ifp->if_tc.ifi_obebytes += m->m_pkthdr.len; + break; + case MBUF_TC_BK: + ifp->if_tc.ifi_obkpackets++; + ifp->if_tc.ifi_obkbytes += m->m_pkthdr.len; + break; + case MBUF_TC_VI: + ifp->if_tc.ifi_ovipackets++; + ifp->if_tc.ifi_ovibytes += m->m_pkthdr.len; + break; + case MBUF_TC_VO: + ifp->if_tc.ifi_ovopackets++; + ifp->if_tc.ifi_ovobytes += m->m_pkthdr.len; + break; + default: + break; + } + + if (mbuf_is_traffic_class_privileged(m)) { + ifp->if_tc.ifi_opvpackets++; + ifp->if_tc.ifi_opvbytes += m->m_pkthdr.len; + } +} #endif /* BSD_KERNEL_PRIVATE */ #endif /* KERNEL_PRIVATE */ #endif /* KERNEL */ diff --git a/bsd/net/ether_if_module.c b/bsd/net/ether_if_module.c index 8fd6074f1..3335bb6c4 100644 --- a/bsd/net/ether_if_module.c +++ b/bsd/net/ether_if_module.c @@ -102,11 +102,12 @@ #if IF_BRIDGE #include #endif /* IF_BRIDGE */ +#if IF_FAKE +#include +#endif /* IF_FAKE */ #include -#define memcpy(x,y,z) bcopy(y, x, z) - SYSCTL_DECL(_net_link); SYSCTL_NODE(_net_link, IFT_ETHER, ether, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Ethernet"); @@ -118,7 +119,11 @@ struct en_desc { }; /* descriptors are allocated in blocks of ETHER_DESC_BLK_SIZE */ +#if CONFIG_EMBEDDED +#define ETHER_DESC_BLK_SIZE (2) /* IP, ARP */ +#else #define ETHER_DESC_BLK_SIZE (10) +#endif /* * Header for the demux list, hangs off of IFP at if_family_cookie @@ -636,6 +641,10 @@ ether_family_init(void) #if IF_BRIDGE bridgeattach(0); #endif /* IF_BRIDGE */ +#if IF_FAKE + if_fake_init(); +#endif /* IF_FAKE */ + done: return (error); diff --git a/bsd/net/ethernet.h b/bsd/net/ethernet.h index 5f29b6e71..3f61bc94f 100644 --- a/bsd/net/ethernet.h +++ b/bsd/net/ethernet.h @@ -130,6 +130,17 @@ struct ether_addr *ether_aton(const char *); #ifdef BSD_KERNEL_PRIVATE extern u_char etherbroadcastaddr[ETHER_ADDR_LEN]; +#if defined (__arm__) + +#include + +static __inline__ int +_ether_cmp(const void * a, const void * b) +{ + return (memcmp(a, b, ETHER_ADDR_LEN)); +} + +#else /* __arm__ */ static __inline__ int _ether_cmp(const void * a, const void * b) @@ -145,6 +156,7 @@ _ether_cmp(const void * a, const void * b) return (0); } +#endif /* __arm__ */ #endif /* BSD_KERNEL_PRIVATE */ #define ETHER_IS_MULTICAST(addr) (*(addr) & 0x01) /* is address mcast/bcast? */ diff --git a/bsd/net/flowadv.c b/bsd/net/flowadv.c index 14ad67758..214c5c715 100644 --- a/bsd/net/flowadv.c +++ b/bsd/net/flowadv.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -196,7 +196,7 @@ flowadv_thread_cont(int err) { #pragma unused(err) for (;;) { - lck_mtx_assert(&fadv_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&fadv_lock, LCK_MTX_ASSERT_OWNED); while (STAILQ_EMPTY(&fadv_list)) { VERIFY(!fadv_active); (void) msleep0(&fadv_list, &fadv_lock, (PSOCK | PSPIN), @@ -215,7 +215,7 @@ flowadv_thread_cont(int err) STAILQ_NEXT(fce, fce_link) = NULL; lck_mtx_unlock(&fadv_lock); - switch (fce->fce_flowsrc) { + switch (fce->fce_flowsrc_type) { case FLOWSRC_INPCB: inp_flowadv(fce->fce_flowid); break; @@ -224,6 +224,7 @@ flowadv_thread_cont(int err) ifnet_flowadv(fce->fce_flowid); break; + case FLOWSRC_PF: default: break; diff --git a/bsd/net/flowadv.h b/bsd/net/flowadv.h index f56eb4b8b..76ae8a498 100644 --- a/bsd/net/flowadv.h +++ b/bsd/net/flowadv.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -33,6 +33,7 @@ #include #include + #define FADV_SUCCESS 0 /* success */ #define FADV_FLOW_CONTROLLED 1 /* regular flow control */ #define FADV_SUSPENDED 2 /* flow control due to suspension */ @@ -44,7 +45,7 @@ struct flowadv { #ifdef BSD_KERNEL_PRIVATE struct flowadv_fcentry { STAILQ_ENTRY(flowadv_fcentry) fce_link; - u_int32_t fce_flowsrc; /* FLOWSRC values */ + u_int32_t fce_flowsrc_type; /* FLOWSRC values */ u_int32_t fce_flowid; }; diff --git a/bsd/net/flowhash.c b/bsd/net/flowhash.c index a45796023..85761bd68 100644 --- a/bsd/net/flowhash.c +++ b/bsd/net/flowhash.c @@ -64,17 +64,16 @@ static inline u_int64_t mh3_fmix64(u_int64_t); /* * The following hash algorithms are selected based on performance: * - * Intel 32-bit: MurmurHash3_x86_32 - * Intel 64-bit: MurmurHash3_x64_128 - * ARM, et al: JHash + * 64-bit: MurmurHash3_x64_128 + * 32-bit: JHash */ -#if defined(__x86_64__) +#if defined(__LP64__) net_flowhash_fn_t *net_flowhash = net_flowhash_mh3_x64_128; -#else /* !__i386__ && !__x86_64__ */ +#else /* !__LP64__ */ net_flowhash_fn_t *net_flowhash = net_flowhash_jhash; -#endif /* !__i386__ && !__x86_64__ */ +#endif /* !__LP64__ */ -#if defined(__i386__) || defined(__x86_64__) +#if defined(__i386__) || defined(__x86_64__) || defined(__arm64__) static inline u_int32_t getblock32(const u_int32_t *p, int i) { @@ -86,7 +85,7 @@ getblock64(const u_int64_t *p, int i) { return (p[i]); } -#else /* !__i386__ && !__x86_64 */ +#else /* !__i386__ && !__x86_64__ && !__arm64__*/ static inline u_int32_t getblock32(const u_int32_t *p, int i) { @@ -146,7 +145,7 @@ getblock64(const u_int64_t *p, int i) } return (value); } -#endif /* !__i386__ && !__x86_64 */ +#endif /* !__i386__ && !__x86_64 && !__arm64__ */ static inline u_int32_t mh3_fmix32(u_int32_t h) @@ -255,20 +254,44 @@ net_flowhash_mh3_x64_128(const void *key, u_int32_t len, const u_int32_t seed) k2 = getblock64(blocks, i * 2 + 1); k1 *= MH3_X64_128_C1; +#if defined(__x86_64__) + __asm__ ( "rol $31, %[k1]\n\t" :[k1] "+r" (k1) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[k1], %[k1], #(64-31)\n\t" :[k1] "+r" (k1) : :); +#else /* !__x86_64__ && !__arm64__ */ k1 = ROTL64(k1, 31); +#endif /* !__x86_64__ && !__arm64__ */ k1 *= MH3_X64_128_C2; h1 ^= k1; - h1 = ROTL64(h1, 27); +#if defined(__x86_64__) + __asm__ ( "rol $27, %[h1]\n\t" :[h1] "+r" (h1) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[h1], %[h1], #(64-27)\n\t" :[h1] "+r" (h1) : :); +#else /* !__x86_64__ && !__arm64__ */ + h1 = ROTL64(h1, 27); +#endif /* !__x86_64__ && !__arm64__ */ h1 += h2; h1 = h1 * 5 + 0x52dce729; k2 *= MH3_X64_128_C2; - k2 = ROTL64(k2, 33); +#if defined(__x86_64__) + __asm__ ( "rol $33, %[k2]\n\t" :[k2] "+r" (k2) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[k2], %[k2], #(64-33)\n\t" :[k2] "+r" (k2) : :); +#else /* !__x86_64__ && !__arm64__ */ + k2 = ROTL64(k2, 33); +#endif /* !__x86_64__ && !__arm64__ */ k2 *= MH3_X64_128_C1; h2 ^= k2; - h2 = ROTL64(h2, 31); +#if defined(__x86_64__) + __asm__ ( "rol $31, %[h2]\n\t" :[h2] "+r" (h2) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[h2], %[h2], #(64-31)\n\t" :[h2] "+r" (h2) : :); +#else /* !__x86_64__ && !__arm64__ */ + h2 = ROTL64(h2, 31); +#endif /* !__x86_64__ && !__arm64__ */ h2 += h1; h2 = h2 * 5+ 0x38495ab5; } @@ -300,7 +323,13 @@ net_flowhash_mh3_x64_128(const void *key, u_int32_t len, const u_int32_t seed) case 9: k2 ^= ((u_int64_t)tail[8]) << 0; k2 *= MH3_X64_128_C2; - k2 = ROTL64(k2, 33); +#if defined(__x86_64__) + __asm__ ( "rol $33, %[k2]\n\t" :[k2] "+r" (k2) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[k2], %[k2], #(64-33)\n\t" :[k2] "+r" (k2) : :); +#else /* !__x86_64__ && !__arm64__ */ + k2 = ROTL64(k2, 33); +#endif /* !__x86_64__ && !__arm64__ */ k2 *= MH3_X64_128_C1; h2 ^= k2; /* FALLTHRU */ @@ -328,7 +357,13 @@ net_flowhash_mh3_x64_128(const void *key, u_int32_t len, const u_int32_t seed) case 1: k1 ^= ((u_int64_t)tail[0]) << 0; k1 *= MH3_X64_128_C1; - k1 = ROTL64(k1, 31); +#if defined(__x86_64__) + __asm__ ( "rol $31, %[k1]\n\t" :[k1] "+r" (k1) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[k1], %[k1], #(64-31)\n\t" :[k1] "+r" (k1) : :); +#else /* !__x86_64__ && !__arm64__ */ + k1 = ROTL64(k1, 31); +#endif /* !__x86_64__ && !__arm64__ */ k1 *= MH3_X64_128_C2; h1 ^= k1; }; diff --git a/bsd/net/if.c b/bsd/net/if.c index 1bb5a6e1f..244132134 100644 --- a/bsd/net/if.c +++ b/bsd/net/if.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -99,10 +99,11 @@ #include #include #include - #include #include #include +#include + #include #include @@ -129,10 +130,6 @@ #include #endif -#if PF_ALTQ -#include -#endif /* !PF_ALTQ */ - /* * System initialization */ @@ -246,6 +243,9 @@ static uint32_t if_verbose = 0; SYSCTL_INT(_net_link_generic_system, OID_AUTO, if_verbose, CTLFLAG_RW | CTLFLAG_LOCKED, &if_verbose, 0, ""); +/* Eventhandler context for interface events */ +struct eventhandler_lists_ctxt ifnet_evhdlr_ctxt; + void ifa_init(void) { @@ -384,6 +384,7 @@ if_detach_ifa_common(struct ifnet *ifp, struct ifaddr *ifa, int link) if (ifa->ifa_detached != NULL) (*ifa->ifa_detached)(ifa); + } #define INITIAL_IF_INDEXLIM 8 @@ -1238,7 +1239,7 @@ link_rtrequest(int cmd, struct rtentry *rt, struct sockaddr *sa) struct ifnet *ifp; void (*ifa_rtrequest)(int, struct rtentry *, struct sockaddr *); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (cmd != RTM_ADD || ((ifa = rt->rt_ifa) == 0) || @@ -1369,12 +1370,6 @@ if_qflush(struct ifnet *ifp, int ifq_locked) if (IFCQ_IS_ENABLED(ifq)) IFCQ_PURGE(ifq); -#if PF_ALTQ - if (IFCQ_IS_DRAINING(ifq)) - ifq->ifcq_drain = 0; - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) - ALTQ_PURGE(IFCQ_ALTQ(ifq)); -#endif /* PF_ALTQ */ VERIFY(IFCQ_IS_EMPTY(ifq)); @@ -1398,14 +1393,6 @@ if_qflush_sc(struct ifnet *ifp, mbuf_svc_class_t sc, u_int32_t flow, if (IFCQ_IS_ENABLED(ifq)) IFCQ_PURGE_SC(ifq, sc, flow, cnt, len); -#if PF_ALTQ - if (IFCQ_IS_DRAINING(ifq)) { - VERIFY((signed)(ifq->ifcq_drain - cnt) >= 0); - ifq->ifcq_drain -= cnt; - } - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) - ALTQ_PURGE_SC(IFCQ_ALTQ(ifq), sc, flow, a_cnt, a_len); -#endif /* PF_ALTQ */ if (!ifq_locked) IFCQ_UNLOCK(ifq); @@ -1695,16 +1682,10 @@ ifioctl_linkparams(struct ifnet *ifp, u_long cmd, caddr_t data, struct proc *p) u_int64_t tbr_bw = 0, tbr_pct = 0; IFCQ_LOCK(ifq); -#if PF_ALTQ - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) { - sched_type = IFCQ_ALTQ(ifq)->altq_type; - flags |= IFLPRF_ALTQ; - } else -#endif /* PF_ALTQ */ - { - if (IFCQ_IS_ENABLED(ifq)) - sched_type = ifq->ifcq_type; - } + + if (IFCQ_IS_ENABLED(ifq)) + sched_type = ifq->ifcq_type; + bcopy(&sched_type, &iflpr->iflpr_output_sched, sizeof (iflpr->iflpr_output_sched)); @@ -1843,6 +1824,116 @@ ifioctl_getnetagents(struct ifnet *ifp, u_int32_t *count, user_addr_t uuid_p) #define IF_MAXAGENTS 64 #define IF_AGENT_INCREMENT 8 +static int +if_add_netagent_locked(struct ifnet *ifp, uuid_t new_agent_uuid) +{ + uuid_t *first_empty_slot = NULL; + u_int32_t index = 0; + bool already_added = FALSE; + + if (ifp->if_agentids != NULL) { + for (index = 0; index < ifp->if_agentcount; index++) { + uuid_t *netagent_uuid = &(ifp->if_agentids[index]); + if (uuid_compare(*netagent_uuid, new_agent_uuid) == 0) { + /* Already present, ignore */ + already_added = TRUE; + break; + } + if (first_empty_slot == NULL && + uuid_is_null(*netagent_uuid)) { + first_empty_slot = netagent_uuid; + } + } + } + if (already_added) { + /* Already added agent, don't return an error */ + return (0); + } + if (first_empty_slot == NULL) { + if (ifp->if_agentcount >= IF_MAXAGENTS) { + /* No room for another netagent UUID, bail */ + return (ENOMEM); + } else { + /* Calculate new array size */ + u_int32_t new_agent_count = + MIN(ifp->if_agentcount + IF_AGENT_INCREMENT, + IF_MAXAGENTS); + + /* Reallocate array */ + uuid_t *new_agent_array = _REALLOC(ifp->if_agentids, + sizeof(uuid_t) * new_agent_count, M_NETAGENT, + M_WAITOK | M_ZERO); + if (new_agent_array == NULL) { + return (ENOMEM); + } + + /* Save new array */ + ifp->if_agentids = new_agent_array; + + /* Set first empty slot */ + first_empty_slot = + &(ifp->if_agentids[ifp->if_agentcount]); + + /* Save new array length */ + ifp->if_agentcount = new_agent_count; + } + } + uuid_copy(*first_empty_slot, new_agent_uuid); + netagent_post_updated_interfaces(new_agent_uuid); + return (0); +} + +int +if_add_netagent(struct ifnet *ifp, uuid_t new_agent_uuid) +{ + VERIFY(ifp != NULL); + + ifnet_lock_exclusive(ifp); + + int error = if_add_netagent_locked(ifp, new_agent_uuid); + + ifnet_lock_done(ifp); + + return (error); +} + +static int +if_delete_netagent_locked(struct ifnet *ifp, uuid_t remove_agent_uuid) +{ + u_int32_t index = 0; + bool removed_agent_id = FALSE; + + if (ifp->if_agentids != NULL) { + for (index = 0; index < ifp->if_agentcount; index++) { + uuid_t *netagent_uuid = &(ifp->if_agentids[index]); + if (uuid_compare(*netagent_uuid, + remove_agent_uuid) == 0) { + uuid_clear(*netagent_uuid); + removed_agent_id = TRUE; + break; + } + } + } + if (removed_agent_id) + netagent_post_updated_interfaces(remove_agent_uuid); + + return (0); +} + +int +if_delete_netagent(struct ifnet *ifp, uuid_t remove_agent_uuid) +{ + VERIFY(ifp != NULL); + + ifnet_lock_exclusive(ifp); + + int error = if_delete_netagent_locked(ifp, remove_agent_uuid); + + ifnet_lock_done(ifp); + + return (error); +} + static __attribute__((noinline)) int ifioctl_netagent(struct ifnet *ifp, u_long cmd, caddr_t data, struct proc *p) { @@ -1852,7 +1943,6 @@ ifioctl_netagent(struct ifnet *ifp, u_long cmd, caddr_t data, struct proc *p) struct if_agentidsreq64 s64; } u; int error = 0; - u_int32_t index = 0; VERIFY(ifp != NULL); @@ -1870,82 +1960,19 @@ ifioctl_netagent(struct ifnet *ifp, u_long cmd, caddr_t data, struct proc *p) switch (cmd) { case SIOCAIFAGENTID: { /* struct if_agentidreq */ - uuid_t *first_empty_slot = NULL; // TODO: Use priv_check_cred() instead of root check if ((error = proc_suser(p)) != 0) { break; } - bool already_added = FALSE; - if (ifp->if_agentids != NULL) { - for (index = 0; index < ifp->if_agentcount; index++) { - uuid_t *netagent_uuid = &(ifp->if_agentids[index]); - if (uuid_compare(*netagent_uuid, ifar->ifar_uuid) == 0) { - /* Already present, ignore */ - already_added = TRUE; - break; - } - if (first_empty_slot == NULL && - uuid_is_null(*netagent_uuid)) { - first_empty_slot = netagent_uuid; - } - } - } - if (already_added) { - /* Already added agent, don't return an error */ - break; - } - if (first_empty_slot == NULL) { - if (ifp->if_agentcount >= IF_MAXAGENTS) { - /* No room for another netagent UUID, bail */ - error = ENOMEM; - break; - } else { - /* Calculate new array size */ - u_int32_t new_agent_count = - MIN(ifp->if_agentcount + IF_AGENT_INCREMENT, IF_MAXAGENTS); - - /* Reallocate array */ - uuid_t *new_agent_array = _REALLOC(ifp->if_agentids, - sizeof(uuid_t) * new_agent_count, M_NETAGENT, - M_WAITOK | M_ZERO); - if (new_agent_array == NULL) { - error = ENOMEM; - break; - } - - /* Save new array */ - ifp->if_agentids = new_agent_array; - - /* Set first empty slot */ - first_empty_slot = &(ifp->if_agentids[ifp->if_agentcount]); - - /* Save new array length */ - ifp->if_agentcount = new_agent_count; - } - } - uuid_copy(*first_empty_slot, ifar->ifar_uuid); - netagent_post_updated_interfaces(ifar->ifar_uuid); + error = if_add_netagent_locked(ifp, ifar->ifar_uuid); break; } case SIOCDIFAGENTID: { /* struct if_agentidreq */ - bool removed_agent_id = FALSE; // TODO: Use priv_check_cred() instead of root check if ((error = proc_suser(p)) != 0) { break; } - if (ifp->if_agentids != NULL) { - for (index = 0; index < ifp->if_agentcount; index++) { - uuid_t *netagent_uuid = &(ifp->if_agentids[index]); - if (uuid_compare(*netagent_uuid, ifar->ifar_uuid) == 0) { - uuid_clear(*netagent_uuid); - removed_agent_id = TRUE; - break; - } - } - } - if (removed_agent_id) { - netagent_post_updated_interfaces(ifar->ifar_uuid); - } + error = if_delete_netagent_locked(ifp, ifar->ifar_uuid); break; } case SIOCGIFAGENTIDS32: { /* struct if_agentidsreq32 */ @@ -2231,6 +2258,34 @@ ifioctl_netsignature(struct ifnet *ifp, u_long cmd, caddr_t data) return (error); } +#if INET6 +static __attribute__((noinline)) int +ifioctl_nat64prefix(struct ifnet *ifp, u_long cmd, caddr_t data) +{ + struct if_nat64req *ifnat64 = (struct if_nat64req *)(void *)data; + int error = 0; + + VERIFY(ifp != NULL); + + switch (cmd) { + case SIOCSIFNAT64PREFIX: /* struct if_nat64req */ + error = ifnet_set_nat64prefix(ifp, ifnat64->ifnat64_prefixes); + break; + + case SIOCGIFNAT64PREFIX: /* struct if_nat64req */ + error = ifnet_get_nat64prefix(ifp, ifnat64->ifnat64_prefixes); + break; + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (error); +} +#endif + + /* * Interface ioctls. * @@ -2351,19 +2406,13 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p) case SIOCSIFDISABLEOUTPUT: /* struct ifreq */ #endif /* (DEBUG || DEVELOPMENT) */ case SIOCGECNMODE: /* struct ifreq */ - case SIOCSECNMODE: { /* struct ifreq */ - struct ifreq ifr; - bcopy(data, &ifr, sizeof (ifr)); - ifr.ifr_name[IFNAMSIZ - 1] = '\0'; - bcopy(&ifr.ifr_name, ifname, IFNAMSIZ); - error = ifioctl_ifreq(so, cmd, &ifr, p); - bcopy(&ifr, data, sizeof (ifr)); - goto done; - } + case SIOCSECNMODE: case SIOCSQOSMARKINGMODE: /* struct ifreq */ case SIOCSQOSMARKINGENABLED: /* struct ifreq */ case SIOCGQOSMARKINGMODE: /* struct ifreq */ case SIOCGQOSMARKINGENABLED: /* struct ifreq */ + case SIOCSIFLOWINTERNET: /* struct ifreq */ + case SIOCGIFLOWINTERNET: /* struct ifreq */ { /* struct ifreq */ struct ifreq ifr; bcopy(data, &ifr, sizeof (ifr)); @@ -2548,6 +2597,12 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p) error = ifioctl_netsignature(ifp, cmd, data); break; +#if INET6 + case SIOCSIFNAT64PREFIX: /* struct if_nsreq */ + case SIOCGIFNAT64PREFIX: /* struct if_nsreq */ + error = ifioctl_nat64prefix(ifp, cmd, data); + break; +#endif default: if (so->so_proto == NULL) { error = EOPNOTSUPP; @@ -2967,10 +3022,10 @@ ifioctl_ifreq(struct socket *so, u_long cmd, struct ifreq *ifr, struct proc *p) case SIOCGIFLINKQUALITYMETRIC: ifnet_lock_shared(ifp); if ((ifp->if_interface_state.valid_bitmask & - IF_INTERFACE_STATE_LQM_STATE_VALID)) + IF_INTERFACE_STATE_LQM_STATE_VALID)) { ifr->ifr_link_quality_metric = ifp->if_interface_state.lqm_state; - else if ((ifp->if_refflags & IFRF_ATTACHED)) { + } else if (IF_FULLY_ATTACHED(ifp)) { ifr->ifr_link_quality_metric = IFNET_LQM_THRESH_UNKNOWN; } else { @@ -3252,6 +3307,33 @@ ifioctl_ifreq(struct socket *so, u_long cmd, struct ifreq *ifr, struct proc *p) error = EINVAL; #endif /* (DEBUG || DEVELOPMENT) */ break; + case SIOCSIFLOWINTERNET: + if ((error = priv_check_cred(kauth_cred_get(), + PRIV_NET_INTERFACE_CONTROL, 0)) != 0) + return (error); + + ifnet_lock_exclusive(ifp); + if (ifr->ifr_low_internet & IFRTYPE_LOW_INTERNET_ENABLE_UL) + ifp->if_xflags |= IFXF_LOW_INTERNET_UL; + else + ifp->if_xflags &= ~(IFXF_LOW_INTERNET_UL); + if (ifr->ifr_low_internet & IFRTYPE_LOW_INTERNET_ENABLE_DL) + ifp->if_xflags |= IFXF_LOW_INTERNET_DL; + else + ifp->if_xflags &= ~(IFXF_LOW_INTERNET_DL); + ifnet_lock_done(ifp); + break; + case SIOCGIFLOWINTERNET: + ifnet_lock_shared(ifp); + ifr->ifr_low_internet = 0; + if (ifp->if_xflags & IFXF_LOW_INTERNET_UL) + ifr->ifr_low_internet |= + IFRTYPE_LOW_INTERNET_ENABLE_UL; + if (ifp->if_xflags & IFXF_LOW_INTERNET_DL) + ifr->ifr_low_internet |= + IFRTYPE_LOW_INTERNET_ENABLE_DL; + ifnet_lock_done(ifp); + break; default: VERIFY(0); /* NOTREACHED */ @@ -4827,3 +4909,62 @@ ifioctl_cassert(void) ; } } + +/* + * XXX: This API is only used by BSD stack and for now will always return 0. + * For Skywalk native drivers, preamble space need not be allocated in mbuf + * as the preamble will be reserved in the translated skywalk packet + * which is transmitted to the driver. + * For Skywalk compat drivers currently headroom is always set to zero. + */ +uint32_t +ifnet_mbuf_packetpreamblelen(struct ifnet *ifp) +{ +#pragma unused(ifp) + return (0); +} + +/* The following is used to enqueue work items for interface events */ +struct intf_event { + struct ifnet *ifp; + union sockaddr_in_4_6 addr; + uint32_t intf_event_code; +}; + +static void +intf_event_callback(void *arg) +{ + struct intf_event *p_intf_ev = (struct intf_event *)arg; + + /* Call this before we walk the tree */ + EVENTHANDLER_INVOKE(&ifnet_evhdlr_ctxt, ifnet_event, p_intf_ev->ifp, + (struct sockaddr *)&(p_intf_ev->addr), p_intf_ev->intf_event_code); +} + +struct intf_event_nwk_wq_entry { + struct nwk_wq_entry nwk_wqe; + struct intf_event intf_ev_arg; +}; + +void +intf_event_enqueue_nwk_wq_entry(struct ifnet *ifp, struct sockaddr *addrp, + uint32_t intf_event_code) +{ +#pragma unused(addrp) + struct intf_event_nwk_wq_entry *p_intf_ev = NULL; + + MALLOC(p_intf_ev, struct intf_event_nwk_wq_entry *, + sizeof(struct intf_event_nwk_wq_entry), + M_NWKWQ, M_WAITOK | M_ZERO); + + p_intf_ev->intf_ev_arg.ifp = ifp; + /* + * XXX Not using addr in the arg. This will be used + * once we need IP address add/delete events + */ + p_intf_ev->intf_ev_arg.intf_event_code = intf_event_code; + p_intf_ev->nwk_wqe.func = intf_event_callback; + p_intf_ev->nwk_wqe.is_arg_managed = TRUE; + p_intf_ev->nwk_wqe.arg = &p_intf_ev->intf_ev_arg; + nwk_wq_enqueue((struct nwk_wq_entry*)p_intf_ev); +} diff --git a/bsd/net/if.h b/bsd/net/if.h index 9d93391f4..2bb7c6655 100644 --- a/bsd/net/if.h +++ b/bsd/net/if.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -137,7 +137,6 @@ struct if_clonereq32 { #define IFEF_VLAN 0x00000200 /* interface has one or more vlans */ #define IFEF_BOND 0x00000400 /* interface is part of bond */ #define IFEF_ARPLL 0x00000800 /* ARP for IPv4LL addresses */ -/* #define IFEF_NOWINDOWSCALE 0x00001000 */ /* Don't scale TCP window on iface */ /* * XXX IFEF_NOAUTOIPV6LL is deprecated and should be done away with. * Configd pretty much manages the interface configuration. @@ -170,7 +169,12 @@ struct if_clonereq32 { * Extra flags */ #define IFXF_WAKE_ON_MAGIC_PACKET 0x00000001 /* wake on magic packet */ -#define IFXF_TIMESTAMP_ENABLED 0x00000002 /* time stamping enabled */ +#define IFXF_TIMESTAMP_ENABLED 0x00000002 /* time stamping enabled */ +#define IFXF_NX_NOAUTO 0x00000004 /* no auto config nexus */ +#define IFXF_MULTISTACK_BPF_TAP 0x00000008 /* multistack bpf tap */ +#define IFXF_LOW_INTERNET_UL 0x00000010 /* Uplink Low Internet is confirmed */ +#define IFXF_LOW_INTERNET_DL 0x00000020 /* Downlink Low Internet is confirmed */ +#define IFXF_ALLOC_KPI 0x00000040 /* Allocated via the ifnet_alloc KPI */ /* * Current requirements for an AWDL interface. Setting/clearing IFEF_AWDL @@ -217,7 +221,7 @@ struct if_clonereq32 { * contains the enabled optional features & capabilites that can be used * individually per packet and are specified in the mbuf pkthdr.csum_flags * field. IFCAP_* and IFNET_* do not match one to one and IFNET_* may be - * more detailed or differenciated than IFCAP_*. + * more detailed or differentiated than IFCAP_*. * IFNET_* hwassist flags have corresponding CSUM_* in sys/mbuf.h */ #define IFCAP_RXCSUM 0x00001 /* can offload checksum on RX */ @@ -233,13 +237,16 @@ struct if_clonereq32 { #define IFCAP_SKYWALK 0x00400 /* Skywalk mode supported/enabled */ #define IFCAP_HW_TIMESTAMP 0x00800 /* Time stamping in hardware */ #define IFCAP_SW_TIMESTAMP 0x01000 /* Time stamping in software */ +#define IFCAP_CSUM_PARTIAL 0x02000 /* can offload partial checksum */ +#define IFCAP_CSUM_ZERO_INVERT 0x04000 /* can invert 0 to -0 (0xffff) */ #define IFCAP_HWCSUM (IFCAP_RXCSUM | IFCAP_TXCSUM) #define IFCAP_TSO (IFCAP_TSO4 | IFCAP_TSO6) #define IFCAP_VALID (IFCAP_HWCSUM | IFCAP_TSO | IFCAP_LRO | IFCAP_VLAN_MTU | \ IFCAP_VLAN_HWTAGGING | IFCAP_JUMBO_MTU | IFCAP_AV | IFCAP_TXSTATUS | \ - IFCAP_SKYWALK | IFCAP_SW_TIMESTAMP | IFCAP_HW_TIMESTAMP) + IFCAP_SKYWALK | IFCAP_SW_TIMESTAMP | IFCAP_HW_TIMESTAMP | \ + IFCAP_CSUM_PARTIAL | IFCAP_CSUM_ZERO_INVERT) #define IFQ_MAXLEN 128 #define IFNET_SLOWHZ 1 /* granularity is 1 second */ @@ -460,6 +467,7 @@ struct ifreq { #define IFRTYPE_SUBFAMILY_RESERVED 5 #define IFRTYPE_SUBFAMILY_INTCOPROC 6 } ifru_type; +#endif /* PRIVATE */ u_int32_t ifru_functional_type; #define IFRTYPE_FUNCTIONAL_UNKNOWN 0 #define IFRTYPE_FUNCTIONAL_LOOPBACK 1 @@ -467,8 +475,9 @@ struct ifreq { #define IFRTYPE_FUNCTIONAL_WIFI_INFRA 3 #define IFRTYPE_FUNCTIONAL_WIFI_AWDL 4 #define IFRTYPE_FUNCTIONAL_CELLULAR 5 -#define IFRTYPE_FUNCTIONAL_INTCOPROC 6 +#define IFRTYPE_FUNCTIONAL_INTCOPROC 6 #define IFRTYPE_FUNCTIONAL_LAST 6 +#ifdef PRIVATE u_int32_t ifru_expensive; u_int32_t ifru_2kcl; struct { @@ -486,6 +495,11 @@ struct ifreq { #define IFRTYPE_QOSMARKING_FASTLANE 1 u_int32_t ifru_qosmarking_enabled; u_int32_t ifru_disable_output; + u_int32_t ifru_low_internet; +#define IFRTYPE_LOW_INTERNET_DISABLE_UL_DL 0x0000 +#define IFRTYPE_LOW_INTERNET_ENABLE_UL 0x0001 +#define IFRTYPE_LOW_INTERNET_ENABLE_DL 0x0002 + #endif /* PRIVATE */ } ifr_ifru; #define ifr_addr ifr_ifru.ifru_addr /* address */ @@ -534,6 +548,7 @@ struct ifreq { #define ifr_qosmarking_enabled ifr_ifru.ifru_qosmarking_enabled #define ifr_fastlane_enabled ifr_qosmarking_enabled #define ifr_disable_output ifr_ifru.ifru_disable_output +#define ifr_low_internet ifr_ifru.ifru_low_internet #endif /* PRIVATE */ }; @@ -696,10 +711,13 @@ struct kev_dl_proto_data { enum { IFNET_LQM_THRESH_OFF = (-2), IFNET_LQM_THRESH_UNKNOWN = (-1), - IFNET_LQM_THRESH_BAD = 10, + IFNET_LQM_THRESH_ABORT = 10, + IFNET_LQM_THRESH_MINIMALLY_VIABLE = 20, IFNET_LQM_THRESH_POOR = 50, IFNET_LQM_THRESH_GOOD = 100 }; +#define IFNET_LQM_THRESH_BAD IFNET_LQM_THRESH_ABORT + #ifdef XNU_KERNEL_PRIVATE #define IFNET_LQM_MIN IFNET_LQM_THRESH_OFF #define IFNET_LQM_MAX IFNET_LQM_THRESH_GOOD @@ -864,6 +882,17 @@ struct if_agentidsreq { uuid_t *ifar_uuids; /* array of agent UUIDs */ }; +/* + * Structure for SIOCGIFNEXUS + */ +struct if_nexusreq { + char ifnr_name[IFNAMSIZ]; /* interface name */ + uint64_t ifnr_flags; /* unused, must be zero */ + uuid_t ifnr_netif; /* netif nexus instance UUID */ + uuid_t ifnr_multistack; /* multistack nexus UUID */ + uint64_t ifnr_reserved[5]; +}; + #ifdef BSD_KERNEL_PRIVATE struct if_agentidsreq32 { char ifar_name[IFNAMSIZ]; @@ -914,6 +943,30 @@ struct if_nsreq { u_int8_t ifnsr_data[IFNET_SIGNATURELEN]; }; + +#define NAT64_PREFIX_LEN_32 4 +#define NAT64_PREFIX_LEN_40 5 +#define NAT64_PREFIX_LEN_48 6 +#define NAT64_PREFIX_LEN_56 7 +#define NAT64_PREFIX_LEN_64 8 +#define NAT64_PREFIX_LEN_96 12 +#define NAT64_PREFIX_LEN_MAX NAT64_PREFIX_LEN_96 + +#define NAT64_MAX_NUM_PREFIXES 4 + +struct ipv6_prefix { + struct in6_addr ipv6_prefix; + uint32_t prefix_len; +}; + +/* + * Structure for SIOC[S/G]IFNAT64PREFIX + */ +struct if_nat64req { + char ifnat64_name[IFNAMSIZ]; + struct ipv6_prefix ifnat64_prefixes[NAT64_MAX_NUM_PREFIXES]; +}; + /* * Structure for SIOC[S/G]IFORDER * diff --git a/bsd/net/if_bond.c b/bsd/net/if_bond.c index 34f6e03d4..202f6c092 100644 --- a/bsd/net/if_bond.c +++ b/bsd/net/if_bond.c @@ -134,14 +134,14 @@ bond_lock_init(void) static __inline__ void bond_assert_lock_held(void) { - lck_mtx_assert(bond_lck_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(bond_lck_mtx, LCK_MTX_ASSERT_OWNED); return; } static __inline__ void bond_assert_lock_not_held(void) { - lck_mtx_assert(bond_lck_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(bond_lck_mtx, LCK_MTX_ASSERT_NOTOWNED); return; } diff --git a/bsd/net/if_bridge.c b/bsd/net/if_bridge.c index 9a2b34f3d..76de0c2bb 100644 --- a/bsd/net/if_bridge.c +++ b/bsd/net/if_bridge.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2015 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -139,6 +139,7 @@ #include #include #include +#include #include /* for struct arpcom */ #include @@ -175,10 +176,11 @@ #include #include + #if BRIDGE_DEBUG #define BR_DBGF_LIFECYCLE 0x0001 #define BR_DBGF_INPUT 0x0002 -#define BR_DBGF_OUTPPUT 0x0004 +#define BR_DBGF_OUTPUT 0x0004 #define BR_DBGF_RT_TABLE 0x0008 #define BR_DBGF_DELAYED_CALL 0x0010 #define BR_DBGF_IOCTL 0x0020 @@ -190,9 +192,9 @@ #define _BRIDGE_LOCK(_sc) lck_mtx_lock(&(_sc)->sc_mtx) #define _BRIDGE_UNLOCK(_sc) lck_mtx_unlock(&(_sc)->sc_mtx) #define BRIDGE_LOCK_ASSERT_HELD(_sc) \ - lck_mtx_assert(&(_sc)->sc_mtx, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_sc)->sc_mtx, LCK_MTX_ASSERT_OWNED) #define BRIDGE_LOCK_ASSERT_NOTHELD(_sc) \ - lck_mtx_assert(&(_sc)->sc_mtx, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_sc)->sc_mtx, LCK_MTX_ASSERT_NOTOWNED) #if BRIDGE_DEBUG @@ -353,14 +355,42 @@ struct bridge_delayed_call { #define BDCF_OUTSTANDING 0x01 /* Delayed call has been scheduled */ #define BDCF_CANCELLING 0x02 /* May be waiting for call completion */ + /* * Software state for each bridge. */ - LIST_HEAD(_bridge_rtnode_list, bridge_rtnode); +typedef struct { + struct _bridge_rtnode_list *bb_rthash; /* our forwarding table */ + struct _bridge_rtnode_list bb_rtlist; /* list version of above */ + uint32_t bb_rthash_key; /* key for hash */ + uint32_t bb_rthash_size; /* size of the hash table */ + struct bridge_delayed_call bb_aging_timer; + struct bridge_delayed_call bb_resize_call; + TAILQ_HEAD(, bridge_iflist) bb_spanlist; /* span ports list */ + struct bstp_state bb_stp; /* STP state */ + bpf_packet_func bb_bpf_input; + bpf_packet_func bb_bpf_output; +} bridge_bsd, *bridge_bsd_t; + +#define sc_rthash sc_u.scu_bsd.bb_rthash +#define sc_rtlist sc_u.scu_bsd.bb_rtlist +#define sc_rthash_key sc_u.scu_bsd.bb_rthash_key +#define sc_rthash_size sc_u.scu_bsd.bb_rthash_size +#define sc_aging_timer sc_u.scu_bsd.bb_aging_timer +#define sc_resize_call sc_u.scu_bsd.bb_resize_call +#define sc_spanlist sc_u.scu_bsd.bb_spanlist +#define sc_stp sc_u.scu_bsd.bb_stp +#define sc_bpf_input sc_u.scu_bsd.bb_bpf_input +#define sc_bpf_output sc_u.scu_bsd.bb_bpf_output + struct bridge_softc { struct ifnet *sc_ifp; /* make this an interface */ + u_int32_t sc_flags; + union { + bridge_bsd scu_bsd; + } sc_u; LIST_ENTRY(bridge_softc) sc_list; decl_lck_mtx_data(, sc_mtx); void *sc_cv; @@ -370,23 +400,11 @@ struct bridge_softc { uint32_t sc_iflist_ref; /* refcount for sc_iflist */ uint32_t sc_iflist_xcnt; /* refcount for sc_iflist */ TAILQ_HEAD(, bridge_iflist) sc_iflist; /* member interface list */ - struct _bridge_rtnode_list *sc_rthash; /* our forwarding table */ - struct _bridge_rtnode_list sc_rtlist; /* list version of above */ - uint32_t sc_rthash_key; /* key for hash */ - uint32_t sc_rthash_size; /* size of the hash table */ - TAILQ_HEAD(, bridge_iflist) sc_spanlist; /* span ports list */ - struct bstp_state sc_stp; /* STP state */ uint32_t sc_brtexceeded; /* # of cache drops */ uint32_t sc_filter_flags; /* ipf and flags */ struct ifnet *sc_ifaddr; /* member mac copied from */ u_char sc_defaddr[6]; /* Default MAC address */ - char sc_if_xname[IFNAMSIZ]; - bpf_packet_func sc_bpf_input; - bpf_packet_func sc_bpf_output; - u_int32_t sc_flags; - struct bridge_delayed_call sc_aging_timer; - struct bridge_delayed_call sc_resize_call; #if BRIDGE_DEBUG /* @@ -402,6 +420,19 @@ struct bridge_softc { #define SCF_DETACHING 0x01 #define SCF_RESIZING 0x02 #define SCF_MEDIA_ACTIVE 0x04 +#define SCF_BSD_MODE 0x08 + +static inline void +bridge_set_bsd_mode(struct bridge_softc * sc) +{ + sc->sc_flags |= SCF_BSD_MODE; +} + +static inline boolean_t +bridge_in_bsd_mode(const struct bridge_softc * sc) +{ + return ((sc->sc_flags & SCF_BSD_MODE) != 0); +} struct bridge_hostfilter_stats bridge_hostfilter_stats; @@ -555,6 +586,7 @@ static void bridge_cancel_delayed_call(struct bridge_delayed_call *); static void bridge_cleanup_delayed_call(struct bridge_delayed_call *); static int bridge_host_filter(struct bridge_iflist *, struct mbuf *); + #define m_copypacket(m, how) m_copym(m, 0, M_COPYALL, how) /* The default bridge vlan is 1 (IEEE 802.1Q-2003 Table 9-2) */ @@ -602,6 +634,14 @@ SYSCTL_INT(_net_link_bridge, OID_AUTO, delayed_callback_delay, "Delay before calling delayed function"); #endif +static int bridge_bsd_mode = 1; +#if (DEVELOPMENT || DEBUG) +SYSCTL_INT(_net_link_bridge, OID_AUTO, bsd_mode, + CTLFLAG_RW|CTLFLAG_LOCKED, + &bridge_bsd_mode, 0, + "Bridge using bsd mode"); +#endif /* (DEVELOPMENT || DEBUG) */ + SYSCTL_STRUCT(_net_link_bridge, OID_AUTO, hostfilterstats, CTLFLAG_RD | CTLFLAG_LOCKED, &bridge_hostfilter_stats, bridge_hostfilter_stats, ""); @@ -863,7 +903,7 @@ static void printf_ether_header(struct ether_header *); static void printf_mbuf_data(mbuf_t, size_t, size_t); static void printf_mbuf_pkthdr(mbuf_t, const char *, const char *); static void printf_mbuf(mbuf_t, const char *, const char *); -static void link_print(struct sockaddr_dl *); +static void link_print(struct bridge_softc * sc); static void bridge_lock(struct bridge_softc *); static void bridge_unlock(struct bridge_softc *); @@ -1047,18 +1087,29 @@ printf_ether_header(struct ether_header *eh) } static void -link_print(struct sockaddr_dl *dl_p) +link_print(struct bridge_softc * sc) { int i; + uint32_t sdl_buffer[offsetof(struct sockaddr_dl, sdl_data) + + IFNAMSIZ + ETHER_ADDR_LEN]; + struct sockaddr_dl *sdl = (struct sockaddr_dl *)sdl_buffer; + + memset(sdl, 0, sizeof (sdl_buffer)); + sdl->sdl_family = AF_LINK; + sdl->sdl_nlen = strlen(sc->sc_if_xname); + sdl->sdl_alen = ETHER_ADDR_LEN; + sdl->sdl_len = offsetof(struct sockaddr_dl, sdl_data); + memcpy(sdl->sdl_data, sc->sc_if_xname, sdl->sdl_nlen); + memcpy(LLADDR(sdl), sc->sc_defaddr, ETHER_ADDR_LEN); #if 1 printf("sdl len %d index %d family %d type 0x%x nlen %d alen %d" - " slen %d addr ", dl_p->sdl_len, dl_p->sdl_index, - dl_p->sdl_family, dl_p->sdl_type, dl_p->sdl_nlen, - dl_p->sdl_alen, dl_p->sdl_slen); + " slen %d addr ", sdl->sdl_len, sdl->sdl_index, + sdl->sdl_family, sdl->sdl_type, sdl->sdl_nlen, + sdl->sdl_alen, sdl->sdl_slen); #endif - for (i = 0; i < dl_p->sdl_alen; i++) - printf("%s%x", i ? ":" : "", (CONST_LLADDR(dl_p))[i]); + for (i = 0; i < sdl->sdl_alen; i++) + printf("%s%x", i ? ":" : "", (CONST_LLADDR(sdl))[i]); printf("\n"); } @@ -1150,6 +1201,38 @@ SYSCTL_PROC(_net_link_bridge, OID_AUTO, ipfw, CTLTYPE_INT|CTLFLAG_RW, &pfil_ipfw, 0, &sysctl_pfil_ipfw, "I", "Layer2 filter with IPFW"); #endif /* PFIL_HOOKS */ +static errno_t +bridge_ifnet_set_attrs(struct ifnet * ifp) +{ + errno_t error; + + error = ifnet_set_mtu(ifp, ETHERMTU); + if (error != 0) { + printf("%s: ifnet_set_mtu failed %d\n", __func__, error); + goto done; + } + error = ifnet_set_addrlen(ifp, ETHER_ADDR_LEN); + if (error != 0) { + printf("%s: ifnet_set_addrlen failed %d\n", __func__, error); + goto done; + } + error = ifnet_set_hdrlen(ifp, ETHER_HDR_LEN); + if (error != 0) { + printf("%s: ifnet_set_hdrlen failed %d\n", __func__, error); + goto done; + } + error = ifnet_set_flags(ifp, + IFF_BROADCAST | IFF_SIMPLEX | IFF_NOTRAILERS | IFF_MULTICAST, + 0xffff); + + if (error != 0) { + printf("%s: ifnet_set_flags failed %d\n", __func__, error); + goto done; + } + done: + return (error); +} + /* * bridge_clone_create: * @@ -1163,9 +1246,6 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) struct bridge_softc *sc, *sc2; struct ifnet_init_eparams init_params; errno_t error = 0; - uint32_t sdl_buffer[offsetof(struct sockaddr_dl, sdl_data) + - IFNAMSIZ + ETHER_ADDR_LEN]; - struct sockaddr_dl *sdl = (struct sockaddr_dl *)sdl_buffer; uint8_t eth_hostid[ETHER_ADDR_LEN]; int fb, retry, has_hostid; @@ -1184,15 +1264,11 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) sc->sc_filter_flags &= ~IFBF_FILT_USEIPF; #endif - /* Initialize our routing table. */ - error = bridge_rtable_init(sc); - if (error != 0) { - printf("%s: bridge_rtable_init failed %d\n", __func__, error); - goto done; + if (bridge_bsd_mode != 0) { + bridge_set_bsd_mode(sc); } TAILQ_INIT(&sc->sc_iflist); - TAILQ_INIT(&sc->sc_spanlist); /* use the interface name as the unique id for ifp recycle */ snprintf(sc->sc_if_xname, sizeof (sc->sc_if_xname), "%s%d", @@ -1200,11 +1276,22 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) bzero(&init_params, sizeof (init_params)); init_params.ver = IFNET_INIT_CURRENT_VERSION; init_params.len = sizeof (init_params); - if (if_bridge_txstart) { - init_params.start = bridge_start; - } else { - init_params.flags = IFNET_INIT_LEGACY; - init_params.output = bridge_output; + if (bridge_in_bsd_mode(sc)) { + /* Initialize our routing table. */ + error = bridge_rtable_init(sc); + if (error != 0) { + printf("%s: bridge_rtable_init failed %d\n", + __func__, error); + goto done; + } + TAILQ_INIT(&sc->sc_spanlist); + if (if_bridge_txstart) { + init_params.start = bridge_start; + } else { + init_params.flags = IFNET_INIT_LEGACY; + init_params.output = bridge_output; + } + init_params.set_bpf_tap = bridge_set_bpf_tap; } init_params.uniqueid = sc->sc_if_xname; init_params.uniqueid_len = strlen(sc->sc_if_xname); @@ -1220,38 +1307,24 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) init_params.framer_extended = ether_frameout_extended; init_params.softc = sc; init_params.ioctl = bridge_ioctl; - init_params.set_bpf_tap = bridge_set_bpf_tap; init_params.detach = bridge_detach; init_params.broadcast_addr = etherbroadcastaddr; init_params.broadcast_len = ETHER_ADDR_LEN; - error = ifnet_allocate_extended(&init_params, &ifp); - if (error != 0) { - printf("%s: ifnet_allocate failed %d\n", __func__, error); - goto done; - } - sc->sc_ifp = ifp; - error = ifnet_set_mtu(ifp, ETHERMTU); - if (error != 0) { - printf("%s: ifnet_set_mtu failed %d\n", __func__, error); - goto done; - } - error = ifnet_set_addrlen(ifp, ETHER_ADDR_LEN); - if (error != 0) { - printf("%s: ifnet_set_addrlen failed %d\n", __func__, error); - goto done; - } - error = ifnet_set_hdrlen(ifp, ETHER_HDR_LEN); - if (error != 0) { - printf("%s: ifnet_set_hdrlen failed %d\n", __func__, error); - goto done; - } - error = ifnet_set_flags(ifp, - IFF_BROADCAST | IFF_SIMPLEX | IFF_NOTRAILERS | IFF_MULTICAST, - 0xffff); - if (error != 0) { - printf("%s: ifnet_set_flags failed %d\n", __func__, error); - goto done; + if (bridge_in_bsd_mode(sc)) { + error = ifnet_allocate_extended(&init_params, &ifp); + if (error != 0) { + printf("%s: ifnet_allocate failed %d\n", + __func__, error); + goto done; + } + sc->sc_ifp = ifp; + error = bridge_ifnet_set_attrs(ifp); + if (error != 0) { + printf("%s: bridge_ifnet_set_attrs failed %d\n", + __func__, error); + goto done; + } } /* @@ -1260,7 +1333,7 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) * Since we are using random ethernet addresses for the bridge, it is * possible that we might have address collisions, so make sure that * this hardware address isn't already in use on another bridge. - * The first try uses the "hostid" and falls back to read_random(); + * The first try uses the "hostid" and falls back to read_frandom(); * for "hostid", we use the MAC address of the first-encountered * Ethernet-type interface that is currently configured. */ @@ -1268,7 +1341,7 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) has_hostid = (uuid_get_ethernet(ð_hostid[0]) == 0); for (retry = 1; retry != 0; ) { if (fb || has_hostid == 0) { - read_random(&sc->sc_defaddr, ETHER_ADDR_LEN); + read_frandom(&sc->sc_defaddr, ETHER_ADDR_LEN); sc->sc_defaddr[0] &= ~1; /* clear multicast bit */ sc->sc_defaddr[0] |= 2; /* set the LAA bit */ } else { @@ -1301,25 +1374,18 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) lck_mtx_unlock(&bridge_list_mtx); } - memset(sdl, 0, sizeof (sdl_buffer)); - sdl->sdl_family = AF_LINK; - sdl->sdl_nlen = strlen(sc->sc_if_xname); - sdl->sdl_alen = ETHER_ADDR_LEN; - sdl->sdl_len = offsetof(struct sockaddr_dl, sdl_data); - memcpy(sdl->sdl_data, sc->sc_if_xname, sdl->sdl_nlen); - memcpy(LLADDR(sdl), sc->sc_defaddr, ETHER_ADDR_LEN); - sc->sc_flags &= ~SCF_MEDIA_ACTIVE; #if BRIDGE_DEBUG if (if_bridge_debug & BR_DBGF_LIFECYCLE) - link_print(sdl); + link_print(sc); #endif - - error = ifnet_attach(ifp, NULL); - if (error != 0) { - printf("%s: ifnet_attach failed %d\n", __func__, error); - goto done; + if (bridge_in_bsd_mode(sc)) { + error = ifnet_attach(ifp, NULL); + if (error != 0) { + printf("%s: ifnet_attach failed %d\n", __func__, error); + goto done; + } } error = ifnet_set_lladdr_and_type(ifp, sc->sc_defaddr, ETHER_ADDR_LEN, @@ -1330,19 +1396,20 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) goto done; } - ifnet_set_offload(ifp, - IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | - IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6 | IFNET_MULTIPAGES); - - error = bridge_set_tso(sc); - if (error != 0) { - printf("%s: bridge_set_tso failed %d\n", __func__, error); - goto done; - } - + if (bridge_in_bsd_mode(sc)) { + ifnet_set_offload(ifp, + IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | + IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6 | IFNET_MULTIPAGES); + error = bridge_set_tso(sc); + if (error != 0) { + printf("%s: bridge_set_tso failed %d\n", + __func__, error); + goto done; + } #if BRIDGESTP - bstp_attach(&sc->sc_stp, &bridge_ops); + bstp_attach(&sc->sc_stp, &bridge_ops); #endif /* BRIDGESTP */ + } lck_mtx_lock(&bridge_list_mtx); LIST_INSERT_HEAD(&bridge_list, sc, sc_list); @@ -1382,10 +1449,12 @@ bridge_clone_destroy(struct ifnet *ifp) bridge_ifstop(ifp, 1); - bridge_cancel_delayed_call(&sc->sc_resize_call); + if (bridge_in_bsd_mode(sc)) { + bridge_cancel_delayed_call(&sc->sc_resize_call); - bridge_cleanup_delayed_call(&sc->sc_resize_call); - bridge_cleanup_delayed_call(&sc->sc_aging_timer); + bridge_cleanup_delayed_call(&sc->sc_resize_call); + bridge_cleanup_delayed_call(&sc->sc_aging_timer); + } error = ifnet_set_flags(ifp, 0, IFF_UP); if (error != 0) { @@ -1395,24 +1464,18 @@ bridge_clone_destroy(struct ifnet *ifp) while ((bif = TAILQ_FIRST(&sc->sc_iflist)) != NULL) bridge_delete_member(sc, bif, 0); - while ((bif = TAILQ_FIRST(&sc->sc_spanlist)) != NULL) { - bridge_delete_span(sc, bif); + if (bridge_in_bsd_mode(sc)) { + while ((bif = TAILQ_FIRST(&sc->sc_spanlist)) != NULL) { + bridge_delete_span(sc, bif); + } + BRIDGE_UNLOCK(sc); } - BRIDGE_UNLOCK(sc); - error = ifnet_detach(ifp); if (error != 0) { - panic("bridge_clone_destroy: ifnet_detach(%p) failed %d\n", - ifp, error); - if ((sc = (struct bridge_softc *)ifnet_softc(ifp)) != NULL) { - BRIDGE_LOCK(sc); - sc->sc_flags &= ~SCF_DETACHING; - BRIDGE_UNLOCK(sc); - } - return (0); + panic("%s: ifnet_detach(%p) failed %d\n", + __func__, ifp, error); } - return (0); } @@ -1915,7 +1978,7 @@ bridge_iff_output(void *cookie, ifnet_t ifp, protocol_family_t protocol, goto out; #if BRIDGE_DEBUG - if (if_bridge_debug & BR_DBGF_OUTPPUT) { + if (if_bridge_debug & BR_DBGF_OUTPUT) { printf("%s: %s from %s m 0x%llx data 0x%llx\n", __func__, sc->sc_ifp->if_xname, ifp->if_xname, (uint64_t)VM_KERNEL_ADDRPERM(m), @@ -2083,10 +2146,13 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, int lladdr_changed = 0, error, filt_attached; uint8_t eaddr[ETHER_ADDR_LEN]; u_int32_t event_code = 0; + boolean_t bsd_mode; BRIDGE_LOCK_ASSERT_HELD(sc); VERIFY(ifs != NULL); + bsd_mode = bridge_in_bsd_mode(sc); + /* * First, remove the member from the list first so it cannot be found anymore * when we release the bridge lock below @@ -2134,8 +2200,9 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, BRIDGE_LOCK(sc); } #if BRIDGESTP - if (bif->bif_ifflags & IFBIF_STP) + if (bsd_mode && (bif->bif_ifflags & IFBIF_STP) != 0) { bstp_disable(&bif->bif_stp); + } #endif /* BRIDGESTP */ /* @@ -2167,7 +2234,10 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, printf("%s: bridge_set_tso failed %d\n", __func__, error); } - bridge_rtdelete(sc, ifs, IFBF_FLUSHALL); + if (bsd_mode) { + bridge_rtdelete(sc, ifs, IFBF_FLUSHALL); + } + KASSERT(bif->bif_addrcnt == 0, ("%s: %d bridge routes referenced", __func__, bif->bif_addrcnt)); @@ -2178,7 +2248,9 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, */ event_code = bridge_updatelinkstatus(sc); - BRIDGE_UNLOCK(sc); + if (bsd_mode) { + BRIDGE_UNLOCK(sc); + } if (lladdr_changed && (error = ifnet_set_lladdr(bifp, eaddr, ETHER_ADDR_LEN)) != 0) @@ -2188,7 +2260,9 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, bridge_link_event(bifp, event_code); #if BRIDGESTP - bstp_destroy(&bif->bif_stp); /* prepare to free */ + if (bsd_mode) { + bstp_destroy(&bif->bif_stp); /* prepare to free */ + } #endif /* BRIDGESTP */ if (filt_attached) @@ -2231,6 +2305,7 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) uint8_t eaddr[ETHER_ADDR_LEN]; struct iff_filter iff; u_int32_t event_code = 0; + boolean_t bsd_mode = bridge_in_bsd_mode(sc); ifs = ifunit(req->ifbr_ifsname); if (ifs == NULL) @@ -2238,10 +2313,12 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) if (ifs->if_ioctl == NULL) /* must be supported */ return (EINVAL); - /* If it's in the span list, it can't be a member. */ - TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) - if (ifs == bif->bif_ifp) - return (EBUSY); + if (bsd_mode) { + /* If it's in the span list, it can't be a member. */ + TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) + if (ifs == bif->bif_ifp) + return (EBUSY); + } if (ifs->if_bridge == sc) return (EEXIST); @@ -2261,7 +2338,7 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) return (EINVAL); } - bif = _MALLOC(sizeof (*bif), M_DEVBUF, M_NOWAIT | M_ZERO); + bif = _MALLOC(sizeof (*bif), M_DEVBUF, M_WAITOK | M_ZERO); if (bif == NULL) return (ENOMEM); @@ -2298,7 +2375,9 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) ifs->if_bridge = sc; #if BRIDGESTP - bstp_create(&sc->sc_stp, &bif->bif_stp, bif->bif_ifp); + if (bsd_mode) { + bstp_create(&sc->sc_stp, &bif->bif_stp, bif->bif_ifp); + } #endif /* BRIDGESTP */ /* @@ -2348,7 +2427,9 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) /* * Respect lock ordering with DLIL lock for the following operations */ - BRIDGE_UNLOCK(sc); + if (bsd_mode) { + BRIDGE_UNLOCK(sc); + } /* * install an interface filter @@ -2356,13 +2437,16 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) memset(&iff, 0, sizeof (struct iff_filter)); iff.iff_cookie = bif; iff.iff_name = "com.apple.kernel.bsd.net.if_bridge"; - iff.iff_input = bridge_iff_input; + if (bsd_mode) { + iff.iff_input = bridge_iff_input; #if BRIDGE_MEMBER_OUT_FILTER - iff.iff_output = bridge_iff_output; + iff.iff_output = bridge_iff_output; #endif /* BRIDGE_MEMBER_OUT_FILTER */ + } iff.iff_event = bridge_iff_event; iff.iff_detached = bridge_iff_detached; - error = dlil_attach_filter(ifs, &iff, &bif->bif_iff_ref, DLIL_IFF_TSO); + error = dlil_attach_filter(ifs, &iff, &bif->bif_iff_ref, + DLIL_IFF_TSO | DLIL_IFF_INTERNAL); if (error != 0) { printf("%s: iflt_attach failed %d\n", __func__, error); BRIDGE_LOCK(sc); @@ -2426,38 +2510,41 @@ bridge_ioctl_gifflags(struct bridge_softc *sc, void *arg) { struct ifbreq *req = arg; struct bridge_iflist *bif; - struct bstp_port *bp; bif = bridge_lookup_member(sc, req->ifbr_ifsname); if (bif == NULL) return (ENOENT); - bp = &bif->bif_stp; + if (bridge_in_bsd_mode(sc)) { + struct bstp_port *bp; + + bp = &bif->bif_stp; + req->ifbr_state = bp->bp_state; + req->ifbr_priority = bp->bp_priority; + req->ifbr_path_cost = bp->bp_path_cost; + req->ifbr_proto = bp->bp_protover; + req->ifbr_role = bp->bp_role; + req->ifbr_stpflags = bp->bp_flags; + /* Copy STP state options as flags */ + if (bp->bp_operedge) + req->ifbr_ifsflags |= IFBIF_BSTP_EDGE; + if (bp->bp_flags & BSTP_PORT_AUTOEDGE) + req->ifbr_ifsflags |= IFBIF_BSTP_AUTOEDGE; + if (bp->bp_ptp_link) + req->ifbr_ifsflags |= IFBIF_BSTP_PTP; + if (bp->bp_flags & BSTP_PORT_AUTOPTP) + req->ifbr_ifsflags |= IFBIF_BSTP_AUTOPTP; + if (bp->bp_flags & BSTP_PORT_ADMEDGE) + req->ifbr_ifsflags |= IFBIF_BSTP_ADMEDGE; + if (bp->bp_flags & BSTP_PORT_ADMCOST) + req->ifbr_ifsflags |= IFBIF_BSTP_ADMCOST; + } req->ifbr_ifsflags = bif->bif_ifflags; - req->ifbr_state = bp->bp_state; - req->ifbr_priority = bp->bp_priority; - req->ifbr_path_cost = bp->bp_path_cost; req->ifbr_portno = bif->bif_ifp->if_index & 0xfff; - req->ifbr_proto = bp->bp_protover; - req->ifbr_role = bp->bp_role; - req->ifbr_stpflags = bp->bp_flags; req->ifbr_addrcnt = bif->bif_addrcnt; req->ifbr_addrmax = bif->bif_addrmax; req->ifbr_addrexceeded = bif->bif_addrexceeded; - /* Copy STP state options as flags */ - if (bp->bp_operedge) - req->ifbr_ifsflags |= IFBIF_BSTP_EDGE; - if (bp->bp_flags & BSTP_PORT_AUTOEDGE) - req->ifbr_ifsflags |= IFBIF_BSTP_AUTOEDGE; - if (bp->bp_ptp_link) - req->ifbr_ifsflags |= IFBIF_BSTP_PTP; - if (bp->bp_flags & BSTP_PORT_AUTOPTP) - req->ifbr_ifsflags |= IFBIF_BSTP_AUTOPTP; - if (bp->bp_flags & BSTP_PORT_ADMEDGE) - req->ifbr_ifsflags |= IFBIF_BSTP_ADMEDGE; - if (bp->bp_flags & BSTP_PORT_ADMCOST) - req->ifbr_ifsflags |= IFBIF_BSTP_ADMCOST; return (0); } @@ -2471,6 +2558,10 @@ bridge_ioctl_sifflags(struct bridge_softc *sc, void *arg) int error; #endif /* BRIDGESTP */ + if (!bridge_in_bsd_mode(sc)) { + return (EINVAL); + } + bif = bridge_lookup_member(sc, req->ifbr_ifsname); if (bif == NULL) return (ENOENT); @@ -2516,8 +2607,9 @@ bridge_ioctl_scache(struct bridge_softc *sc, void *arg) struct ifbrparam *param = arg; sc->sc_brtmax = param->ifbrp_csize; - bridge_rttrim(sc); - + if (bridge_in_bsd_mode(sc)) { + bridge_rttrim(sc); + } return (0); } @@ -2540,8 +2632,10 @@ bridge_ioctl_gcache(struct bridge_softc *sc, void *arg) count = 0; \ TAILQ_FOREACH(bif, &sc->sc_iflist, bif_next) \ count++; \ - TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) \ - count++; \ + if (bridge_in_bsd_mode(sc)) { \ + TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) \ + count++; \ + } \ \ buflen = sizeof (breq) * count; \ if (bifc->ifbic_len == 0) { \ @@ -2571,18 +2665,22 @@ bridge_ioctl_gcache(struct bridge_softc *sc, void *arg) buf += sizeof (breq); \ len -= sizeof (breq); \ } \ - TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) { \ - if (len < sizeof (breq)) \ - break; \ + if (bridge_in_bsd_mode(sc)) { \ + TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) { \ + if (len < sizeof (breq)) \ + break; \ \ - snprintf(breq.ifbr_ifsname, sizeof (breq.ifbr_ifsname), \ - "%s", bif->bif_ifp->if_xname); \ - breq.ifbr_ifsflags = bif->bif_ifflags; \ - breq.ifbr_portno = bif->bif_ifp->if_index & 0xfff; \ - memcpy(buf, &breq, sizeof (breq)); \ - count++; \ - buf += sizeof (breq); \ - len -= sizeof (breq); \ + snprintf(breq.ifbr_ifsname, \ + sizeof (breq.ifbr_ifsname), \ + "%s", bif->bif_ifp->if_xname); \ + breq.ifbr_ifsflags = bif->bif_ifflags; \ + breq.ifbr_portno \ + = bif->bif_ifp->if_index & 0xfff; \ + memcpy(buf, &breq, sizeof (breq)); \ + count++; \ + buf += sizeof (breq); \ + len -= sizeof (breq); \ + } \ } \ \ BRIDGE_UNLOCK(sc); \ @@ -2616,14 +2714,19 @@ bridge_ioctl_gifs32(struct bridge_softc *sc, void *arg) #define BRIDGE_IOCTL_RTS do { \ struct bridge_rtnode *brt; \ - char *buf, *outbuf; \ + char *buf; \ + char *outbuf = NULL; \ unsigned int count, buflen, len; \ unsigned long now; \ \ if (bac->ifbac_len == 0) \ return (0); \ \ + bzero(&bareq, sizeof (bareq)); \ count = 0; \ + if (!bridge_in_bsd_mode(sc)) { \ + goto out; \ + } \ LIST_FOREACH(brt, &sc->sc_rtlist, brt_list) \ count++; \ buflen = sizeof (bareq) * count; \ @@ -2635,12 +2738,11 @@ bridge_ioctl_gifs32(struct bridge_softc *sc, void *arg) count = 0; \ buf = outbuf; \ len = min(bac->ifbac_len, buflen); \ - bzero(&bareq, sizeof (bareq)); \ LIST_FOREACH(brt, &sc->sc_rtlist, brt_list) { \ if (len < sizeof (bareq)) \ goto out; \ snprintf(bareq.ifba_ifsname, sizeof (bareq.ifba_ifsname), \ - "%s", brt->brt_ifp->if_xname); \ + "%s", brt->brt_ifp->if_xname); \ memcpy(bareq.ifba_dst, brt->brt_addr, sizeof (brt->brt_addr)); \ bareq.ifba_vlan = brt->brt_vlan; \ if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) { \ @@ -2658,11 +2760,13 @@ bridge_ioctl_gifs32(struct bridge_softc *sc, void *arg) len -= sizeof (bareq); \ } \ out: \ - BRIDGE_UNLOCK(sc); \ bac->ifbac_len = sizeof (bareq) * count; \ - error = copyout(outbuf, bac->ifbac_req, bac->ifbac_len); \ - BRIDGE_LOCK(sc); \ - _FREE(outbuf, M_TEMP); \ + if (outbuf != NULL) { \ + BRIDGE_UNLOCK(sc); \ + error = copyout(outbuf, bac->ifbac_req, bac->ifbac_len); \ + _FREE(outbuf, M_TEMP); \ + BRIDGE_LOCK(sc); \ + } \ return (error); \ } while (0) @@ -2674,7 +2778,6 @@ bridge_ioctl_rts64(struct bridge_softc *sc, void *arg) int error = 0; BRIDGE_IOCTL_RTS; - return (error); } @@ -2686,7 +2789,6 @@ bridge_ioctl_rts32(struct bridge_softc *sc, void *arg) int error = 0; BRIDGE_IOCTL_RTS; - return (error); } @@ -2697,6 +2799,10 @@ bridge_ioctl_saddr32(struct bridge_softc *sc, void *arg) struct bridge_iflist *bif; int error; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } + bif = bridge_lookup_member(sc, req->ifba_ifsname); if (bif == NULL) return (ENOENT); @@ -2714,6 +2820,10 @@ bridge_ioctl_saddr64(struct bridge_softc *sc, void *arg) struct bridge_iflist *bif; int error; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } + bif = bridge_lookup_member(sc, req->ifba_ifsname); if (bif == NULL) return (ENOENT); @@ -2747,6 +2857,9 @@ bridge_ioctl_daddr32(struct bridge_softc *sc, void *arg) { struct ifbareq32 *req = arg; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } return (bridge_rtdaddr(sc, req->ifba_dst, req->ifba_vlan)); } @@ -2755,6 +2868,9 @@ bridge_ioctl_daddr64(struct bridge_softc *sc, void *arg) { struct ifbareq64 *req = arg; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } return (bridge_rtdaddr(sc, req->ifba_dst, req->ifba_vlan)); } @@ -2763,6 +2879,9 @@ bridge_ioctl_flush(struct bridge_softc *sc, void *arg) { struct ifbreq *req = arg; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } bridge_rtflush(sc, req->ifbr_ifsflags); return (0); } @@ -2773,6 +2892,9 @@ bridge_ioctl_gpri(struct bridge_softc *sc, void *arg) struct ifbrparam *param = arg; struct bstp_state *bs = &sc->sc_stp; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } param->ifbrp_prio = bs->bs_bridge_priority; return (0); } @@ -2783,6 +2905,9 @@ bridge_ioctl_spri(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_priority(&sc->sc_stp, param->ifbrp_prio)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -2796,6 +2921,9 @@ bridge_ioctl_ght(struct bridge_softc *sc, void *arg) struct ifbrparam *param = arg; struct bstp_state *bs = &sc->sc_stp; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } param->ifbrp_hellotime = bs->bs_bridge_htime >> 8; return (0); } @@ -2806,6 +2934,9 @@ bridge_ioctl_sht(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_htime(&sc->sc_stp, param->ifbrp_hellotime)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -2816,9 +2947,14 @@ bridge_ioctl_sht(struct bridge_softc *sc, void *arg) static int bridge_ioctl_gfd(struct bridge_softc *sc, void *arg) { - struct ifbrparam *param = arg; - struct bstp_state *bs = &sc->sc_stp; + struct ifbrparam *param; + struct bstp_state *bs; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } + param = arg; + bs = &sc->sc_stp; param->ifbrp_fwddelay = bs->bs_bridge_fdelay >> 8; return (0); } @@ -2829,6 +2965,9 @@ bridge_ioctl_sfd(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_fdelay(&sc->sc_stp, param->ifbrp_fwddelay)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -2839,9 +2978,14 @@ bridge_ioctl_sfd(struct bridge_softc *sc, void *arg) static int bridge_ioctl_gma(struct bridge_softc *sc, void *arg) { - struct ifbrparam *param = arg; - struct bstp_state *bs = &sc->sc_stp; + struct ifbrparam *param; + struct bstp_state *bs; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } + param = arg; + bs = &sc->sc_stp; param->ifbrp_maxage = bs->bs_bridge_max_age >> 8; return (0); } @@ -2852,6 +2996,9 @@ bridge_ioctl_sma(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_maxage(&sc->sc_stp, param->ifbrp_maxage)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -2866,6 +3013,9 @@ bridge_ioctl_sifprio(struct bridge_softc *sc, void *arg) struct ifbreq *req = arg; struct bridge_iflist *bif; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } bif = bridge_lookup_member(sc, req->ifbr_ifsname); if (bif == NULL) return (ENOENT); @@ -2884,6 +3034,9 @@ bridge_ioctl_sifcost(struct bridge_softc *sc, void *arg) struct ifbreq *req = arg; struct bridge_iflist *bif; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } bif = bridge_lookup_member(sc, req->ifbr_ifsname); if (bif == NULL) return (ENOENT); @@ -2944,6 +3097,9 @@ bridge_ioctl_addspan(struct bridge_softc *sc, void *arg) struct bridge_iflist *bif = NULL; struct ifnet *ifs; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } ifs = ifunit(req->ifbr_ifsname); if (ifs == NULL) return (ENOENT); @@ -2966,7 +3122,7 @@ bridge_ioctl_addspan(struct bridge_softc *sc, void *arg) return (EINVAL); } - bif = _MALLOC(sizeof (*bif), M_DEVBUF, M_NOWAIT | M_ZERO); + bif = _MALLOC(sizeof (*bif), M_DEVBUF, M_WAITOK | M_ZERO); if (bif == NULL) return (ENOMEM); @@ -2987,6 +3143,9 @@ bridge_ioctl_delspan(struct bridge_softc *sc, void *arg) struct bridge_iflist *bif; struct ifnet *ifs; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } ifs = ifunit(req->ifbr_ifsname); if (ifs == NULL) return (ENOENT); @@ -3033,8 +3192,9 @@ bridge_ioctl_gbparam32(struct bridge_softc *sc, void *arg) { struct ifbropreq32 *req = arg; - BRIDGE_IOCTL_GBPARAM; - + if (bridge_in_bsd_mode(sc)) { + BRIDGE_IOCTL_GBPARAM; + } return (0); } @@ -3043,8 +3203,9 @@ bridge_ioctl_gbparam64(struct bridge_softc *sc, void *arg) { struct ifbropreq64 *req = arg; - BRIDGE_IOCTL_GBPARAM; - + if (bridge_in_bsd_mode(sc)) { + BRIDGE_IOCTL_GBPARAM; + } return (0); } @@ -3119,8 +3280,9 @@ bridge_ioctl_gifsstp32(struct bridge_softc *sc, void *arg) struct ifbpstpconf32 *bifstp = arg; int error = 0; - BRIDGE_IOCTL_GIFSSTP; - + if (bridge_in_bsd_mode(sc)) { + BRIDGE_IOCTL_GIFSSTP; + } return (error); } @@ -3130,8 +3292,9 @@ bridge_ioctl_gifsstp64(struct bridge_softc *sc, void *arg) struct ifbpstpconf64 *bifstp = arg; int error = 0; - BRIDGE_IOCTL_GIFSSTP; - + if (bridge_in_bsd_mode(sc)) { + BRIDGE_IOCTL_GIFSSTP; + } return (error); } @@ -3141,6 +3304,9 @@ bridge_ioctl_sproto(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_protocol(&sc->sc_stp, param->ifbrp_proto)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -3154,6 +3320,9 @@ bridge_ioctl_stxhc(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_holdcount(&sc->sc_stp, param->ifbrp_txhc)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -3192,6 +3361,8 @@ bridge_ioctl_shostfilter(struct bridge_softc *sc, void *arg) if (bif == NULL) return (ENOENT); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_vmnet_total); + if (req->ifbrhf_flags & IFBRHF_ENABLED) { bif->bif_flags |= BIFF_HOST_FILTER; @@ -3247,17 +3418,18 @@ bridge_ifdetach(struct bridge_iflist *bif, struct ifnet *ifp) BRIDGE_UNLOCK(sc); return; } - /* Check if the interface is a span port */ lck_mtx_lock(&bridge_list_mtx); LIST_FOREACH(sc, &bridge_list, sc_list) { - BRIDGE_LOCK(sc); - TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) - if (ifp == bif->bif_ifp) { - bridge_delete_span(sc, bif); - break; - } - BRIDGE_UNLOCK(sc); + if (bridge_in_bsd_mode(sc)) { + BRIDGE_LOCK(sc); + TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) + if (ifp == bif->bif_ifp) { + bridge_delete_span(sc, bif); + break; + } + BRIDGE_UNLOCK(sc); + } } lck_mtx_unlock(&bridge_list_mtx); } @@ -3546,17 +3718,17 @@ bridge_init(struct ifnet *ifp) error = ifnet_set_flags(ifp, IFF_RUNNING, IFF_RUNNING); - /* - * Calling bridge_aging_timer() is OK as there are no entries to - * age so we're just going to arm the timer - */ - bridge_aging_timer(sc); - + if (bridge_in_bsd_mode(sc)) { + /* + * Calling bridge_aging_timer() is OK as there are no entries to + * age so we're just going to arm the timer + */ + bridge_aging_timer(sc); #if BRIDGESTP - if (error == 0) - bstp_init(&sc->sc_stp); /* Initialize Spanning Tree */ + if (error == 0) + bstp_init(&sc->sc_stp); /* Initialize Spanning Tree */ #endif /* BRIDGESTP */ - + } return (error); } @@ -3576,14 +3748,15 @@ bridge_ifstop(struct ifnet *ifp, int disable) if ((ifnet_flags(ifp) & IFF_RUNNING) == 0) return; - bridge_cancel_delayed_call(&sc->sc_aging_timer); + if (bridge_in_bsd_mode(sc)) { + bridge_cancel_delayed_call(&sc->sc_aging_timer); #if BRIDGESTP - bstp_stop(&sc->sc_stp); + bstp_stop(&sc->sc_stp); #endif /* BRIDGESTP */ - bridge_rtflush(sc, IFBF_FLUSHDYN); - + bridge_rtflush(sc, IFBF_FLUSHDYN); + } (void) ifnet_set_flags(ifp, 0, IFF_RUNNING); } @@ -3720,7 +3893,7 @@ bridge_member_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa, uint16_t vlan; #if BRIDGE_DEBUG - if (if_bridge_debug & BR_DBGF_OUTPPUT) + if (if_bridge_debug & BR_DBGF_OUTPUT) printf("%s: ifp %s\n", __func__, ifp->if_xname); #endif /* BRIDGE_DEBUG */ @@ -3851,6 +4024,8 @@ bridge_output(struct ifnet *ifp, struct mbuf *m) dst_if = NULL; BRIDGE_LOCK(sc); + ASSERT(bridge_in_bsd_mode(sc)); + if (!(m->m_flags & (M_BCAST|M_MCAST))) dst_if = bridge_rtlookup(sc, eh->ether_dhost, 0); @@ -3977,9 +4152,10 @@ bridge_forward(struct bridge_softc *sc, struct bridge_iflist *sbif, int error; BRIDGE_LOCK_ASSERT_HELD(sc); + ASSERT(bridge_in_bsd_mode(sc)); #if BRIDGE_DEBUG - if (if_bridge_debug & BR_DBGF_OUTPPUT) + if (if_bridge_debug & BR_DBGF_OUTPUT) printf("%s: %s m 0x%llx\n", __func__, sc->sc_ifp->if_xname, (uint64_t)VM_KERNEL_ADDRPERM(m)); #endif /* BRIDGE_DEBUG */ @@ -4162,6 +4338,7 @@ bridge_input(struct ifnet *ifp, struct mbuf *m, void *frame_header) uint16_t vlan; int error; + ASSERT(bridge_in_bsd_mode(sc)); #if BRIDGE_DEBUG if (if_bridge_debug & BR_DBGF_INPUT) printf("%s: %s from %s m 0x%llx data 0x%llx\n", __func__, @@ -4643,6 +4820,7 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t *dst, uint16_t vlan, int error; BRIDGE_LOCK_ASSERT_HELD(sc); + ASSERT(bridge_in_bsd_mode(sc)); /* Check the source address is valid and not multicast. */ if (ETHER_IS_MULTICAST(dst) || @@ -4891,6 +5069,8 @@ bridge_rtable_init(struct bridge_softc *sc) { u_int32_t i; + ASSERT(bridge_in_bsd_mode(sc)); + sc->sc_rthash = _MALLOC(sizeof (*sc->sc_rthash) * BRIDGE_RTHASH_SIZE, M_DEVBUF, M_WAITOK | M_ZERO); if (sc->sc_rthash == NULL) { @@ -5102,6 +5282,7 @@ bridge_rtnode_lookup(struct bridge_softc *sc, const uint8_t *addr, int dir; BRIDGE_LOCK_ASSERT_HELD(sc); + ASSERT(bridge_in_bsd_mode(sc)); hash = bridge_rthash(sc, addr); LIST_FOREACH(brt, &sc->sc_rthash[hash], brt_hash) { @@ -5793,7 +5974,7 @@ bridge_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, bpf_packet_func bpf_callback) if (sc == NULL || (sc->sc_flags & SCF_DETACHING)) { return (ENODEV); } - + ASSERT(bridge_in_bsd_mode(sc)); switch (mode) { case BPF_TAP_DISABLE: sc->sc_bpf_input = sc->sc_bpf_output = NULL; @@ -5832,8 +6013,10 @@ bridge_detach(ifnet_t ifp) bstp_detach(&sc->sc_stp); #endif /* BRIDGESTP */ - /* Tear down the routing table. */ - bridge_rtable_fini(sc); + if (bridge_in_bsd_mode(sc)) { + /* Tear down the routing table. */ + bridge_rtable_fini(sc); + } lck_mtx_lock(&bridge_list_mtx); LIST_REMOVE(sc, sc_list); @@ -5856,6 +6039,7 @@ bridge_bpf_input(ifnet_t ifp, struct mbuf *m) { struct bridge_softc *sc = (struct bridge_softc *)ifnet_softc(ifp); + ASSERT(bridge_in_bsd_mode(sc)); if (sc->sc_bpf_input) { if (mbuf_pkthdr_rcvif(m) != ifp) { printf("%s: rcvif: 0x%llx != ifp 0x%llx\n", __func__, @@ -5877,6 +6061,7 @@ bridge_bpf_output(ifnet_t ifp, struct mbuf *m) { struct bridge_softc *sc = (struct bridge_softc *)ifnet_softc(ifp); + ASSERT(bridge_in_bsd_mode(sc)); if (sc->sc_bpf_output) { (*sc->sc_bpf_output)(ifp, m); } @@ -6194,3 +6379,5 @@ done: } return (error); } + + diff --git a/bsd/net/if_dl.h b/bsd/net/if_dl.h index 3d086e402..55d504dc8 100644 --- a/bsd/net/if_dl.h +++ b/bsd/net/if_dl.h @@ -65,6 +65,12 @@ #define _NET_IF_DL_H_ #include +#include + +#ifdef BSD_KERNEL_PRIVATE +#define DLIL_SDLMAXLEN 64 +#endif /* BSD_KERNEL_PRIVATE */ + /* * A Link-Level Sockaddr may specify the interface in one of two * ways: either by means of a system-provided index number (computed diff --git a/bsd/net/if_fake.c b/bsd/net/if_fake.c new file mode 100644 index 000000000..3af936b87 --- /dev/null +++ b/bsd/net/if_fake.c @@ -0,0 +1,1029 @@ +/* + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + * if_fake.c + * - fake network interface used for testing + * - "feth" (e.g. "feth0", "feth1") is a virtual ethernet interface that allows + * two instances to have their output/input paths "crossed-over" so that + * output on one is input on the other + */ + +/* + * Modification History: + * + * September 9, 2015 Dieter Siegmund (dieter@apple.com) + * - created + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include + +#ifdef INET +#include +#include +#endif + +#include +#include + +#define FAKE_ETHER_NAME "feth" + +SYSCTL_DECL(_net_link); +SYSCTL_NODE(_net_link, OID_AUTO, fake, CTLFLAG_RW|CTLFLAG_LOCKED, 0, + "Fake interface"); + +static int if_fake_txstart = 1; +SYSCTL_INT(_net_link_fake, OID_AUTO, txstart, CTLFLAG_RW | CTLFLAG_LOCKED, + &if_fake_txstart, 0, "Fake interface TXSTART mode"); + +static int if_fake_hwcsum = 0; +SYSCTL_INT(_net_link_fake, OID_AUTO, hwcsum, CTLFLAG_RW | CTLFLAG_LOCKED, + &if_fake_hwcsum, 0, "Fake interface simulate hardware checksum"); + +static int if_fake_nxattach = 0; +SYSCTL_INT(_net_link_fake, OID_AUTO, nxattach, CTLFLAG_RW | CTLFLAG_LOCKED, + &if_fake_nxattach, 0, "Fake interface auto-attach nexus"); + +static int if_fake_bsd_mode = 1; +SYSCTL_INT(_net_link_fake, OID_AUTO, bsd_mode, CTLFLAG_RW | CTLFLAG_LOCKED, + &if_fake_bsd_mode, 0, "Fake interface attach as BSD interface"); + +static int if_fake_debug = 0; +SYSCTL_INT(_net_link_fake, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED, + &if_fake_debug, 0, "Fake interface debug logs"); + +/** + ** virtual ethernet structures, types + **/ + +typedef uint16_t iff_flags_t; +#define IFF_FLAGS_HWCSUM 0x0001 +#define IFF_FLAGS_BSD_MODE 0x0002 +#define IFF_FLAGS_DETACHING 0x0004 + + +struct if_fake { + char iff_name[IFNAMSIZ]; /* our unique id */ + ifnet_t iff_ifp; + iff_flags_t iff_flags; + uint32_t iff_retain_count; + ifnet_t iff_peer; /* the other end */ + int iff_media_current; + int iff_media_active; + uint32_t iff_media_count; + int iff_media_list[IF_FAKE_MEDIA_LIST_MAX]; + struct mbuf * iff_pending_tx_packet; + boolean_t iff_start_busy; +}; + +typedef struct if_fake * if_fake_ref; + +static if_fake_ref +ifnet_get_if_fake(ifnet_t ifp); + +#define FETH_DPRINTF(fmt, ...) \ + { if (if_fake_debug != 0) printf("%s " fmt, __func__, ## __VA_ARGS__); } + +static inline boolean_t +feth_in_bsd_mode(if_fake_ref fakeif) +{ + return ((fakeif->iff_flags & IFF_FLAGS_BSD_MODE) != 0); +} + +static inline void +feth_set_detaching(if_fake_ref fakeif) +{ + fakeif->iff_flags |= IFF_FLAGS_DETACHING; +} + +static inline boolean_t +feth_is_detaching(if_fake_ref fakeif) +{ + return ((fakeif->iff_flags & IFF_FLAGS_DETACHING) != 0); +} + + +#define M_FAKE M_DEVBUF + +static int feth_clone_create(struct if_clone *, u_int32_t, void *); +static int feth_clone_destroy(ifnet_t); +static int feth_output(ifnet_t ifp, struct mbuf *m); +static void feth_start(ifnet_t ifp); +static int feth_ioctl(ifnet_t ifp, u_long cmd, void * addr); +static int feth_config(ifnet_t ifp, ifnet_t peer); +static void feth_if_free(ifnet_t ifp); +static void feth_ifnet_set_attrs(if_fake_ref fakeif, ifnet_t ifp); +static void feth_free(if_fake_ref fakeif); + +static struct if_clone +feth_cloner = IF_CLONE_INITIALIZER(FAKE_ETHER_NAME, + feth_clone_create, + feth_clone_destroy, + 0, + IF_MAXUNIT); +static void interface_link_event(ifnet_t ifp, u_int32_t event_code); + +/* some media words to pretend to be ethernet */ +static int default_media_words[] = { + IFM_MAKEWORD(IFM_ETHER, 0, 0, 0), + IFM_MAKEWORD(IFM_ETHER, IFM_10G_T, IFM_FDX, 0), + IFM_MAKEWORD(IFM_ETHER, IFM_2500_T, IFM_FDX, 0), + IFM_MAKEWORD(IFM_ETHER, IFM_5000_T, IFM_FDX, 0), +}; +#define default_media_words_count (sizeof(default_media_words) \ + / sizeof (default_media_words[0])) + +/** + ** veth locks + **/ +static inline lck_grp_t * +my_lck_grp_alloc_init(const char * grp_name) +{ + lck_grp_t * grp; + lck_grp_attr_t * grp_attrs; + + grp_attrs = lck_grp_attr_alloc_init(); + grp = lck_grp_alloc_init(grp_name, grp_attrs); + lck_grp_attr_free(grp_attrs); + return (grp); +} + +static inline lck_mtx_t * +my_lck_mtx_alloc_init(lck_grp_t * lck_grp) +{ + lck_attr_t * lck_attrs; + lck_mtx_t * lck_mtx; + + lck_attrs = lck_attr_alloc_init(); + lck_mtx = lck_mtx_alloc_init(lck_grp, lck_attrs); + lck_attr_free(lck_attrs); + return (lck_mtx); +} + +static lck_mtx_t * feth_lck_mtx; + +static inline void +feth_lock_init(void) +{ + lck_grp_t * feth_lck_grp; + + feth_lck_grp = my_lck_grp_alloc_init("fake"); + feth_lck_mtx = my_lck_mtx_alloc_init(feth_lck_grp); +} + +#if 0 +static inline void +feth_assert_lock_not_held(void) +{ + LCK_MTX_ASSERT(feth_lck_mtx, LCK_MTX_ASSERT_NOTOWNED); + return; +} +#endif + +static inline void +feth_lock(void) +{ + lck_mtx_lock(feth_lck_mtx); + return; +} + +static inline void +feth_unlock(void) +{ + lck_mtx_unlock(feth_lck_mtx); + return; +} + +static inline int +feth_max_mtu(void) +{ + if (njcl > 0) { + return (M16KCLBYTES - ETHER_HDR_LEN); + } + return (MBIGCLBYTES - ETHER_HDR_LEN); +} + +static void +feth_free(if_fake_ref fakeif) +{ + assert(fakeif->iff_retain_count == 0); + if (feth_in_bsd_mode(fakeif)) { + if (fakeif->iff_pending_tx_packet) { + m_freem(fakeif->iff_pending_tx_packet); + } + } + + FETH_DPRINTF("%s\n", fakeif->iff_name); + FREE(fakeif, M_FAKE); +} + +static void +feth_release(if_fake_ref fakeif) +{ + u_int32_t old_retain_count; + + old_retain_count = OSDecrementAtomic(&fakeif->iff_retain_count); + switch (old_retain_count) { + case 0: + assert(old_retain_count != 0); + break; + case 1: + feth_free(fakeif); + break; + default: + break; + } + return; +} + + +/** + ** feth interface routines + **/ +static void +feth_ifnet_set_attrs(if_fake_ref fakeif, ifnet_t ifp) +{ + (void)ifnet_set_capabilities_enabled(ifp, 0, -1); + ifnet_set_addrlen(ifp, ETHER_ADDR_LEN); + ifnet_set_baudrate(ifp, 0); + ifnet_set_mtu(ifp, ETHERMTU); + ifnet_set_flags(ifp, + IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX, + 0xffff); + ifnet_set_hdrlen(ifp, sizeof(struct ether_header)); + if ((fakeif->iff_flags & IFF_FLAGS_HWCSUM) != 0) { + ifnet_set_offload(ifp, + IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | + IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6); + } else { + ifnet_set_offload(ifp, 0); + } +} + +static void +interface_link_event(ifnet_t ifp, u_int32_t event_code) +{ + struct { + struct kern_event_msg header; + u_int32_t unit; + char if_name[IFNAMSIZ]; + } event; + + bzero(&event, sizeof(event)); + event.header.total_size = sizeof(event); + event.header.vendor_code = KEV_VENDOR_APPLE; + event.header.kev_class = KEV_NETWORK_CLASS; + event.header.kev_subclass = KEV_DL_SUBCLASS; + event.header.event_code = event_code; + event.header.event_data[0] = ifnet_family(ifp); + event.unit = (u_int32_t) ifnet_unit(ifp); + strlcpy(event.if_name, ifnet_name(ifp), IFNAMSIZ); + ifnet_event(ifp, &event.header); + return; +} + +static if_fake_ref +ifnet_get_if_fake(ifnet_t ifp) +{ + return ((if_fake_ref)ifnet_softc(ifp)); +} + +static int +feth_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) +{ + int error; + if_fake_ref fakeif; + struct ifnet_init_eparams feth_init; + ifnet_t ifp; + uint8_t mac_address[ETHER_ADDR_LEN]; + + fakeif = _MALLOC(sizeof(struct if_fake), M_FAKE, M_WAITOK | M_ZERO); + if (fakeif == NULL) { + return ENOBUFS; + } + fakeif->iff_retain_count = 1; +#define FAKE_ETHER_NAME_LEN (sizeof(FAKE_ETHER_NAME) - 1) + _CASSERT(FAKE_ETHER_NAME_LEN == 4); + bcopy(FAKE_ETHER_NAME, mac_address, FAKE_ETHER_NAME_LEN); + mac_address[ETHER_ADDR_LEN - 2] = (unit & 0xff00) >> 8; + mac_address[ETHER_ADDR_LEN - 1] = unit & 0xff; + if (if_fake_bsd_mode != 0) { + fakeif->iff_flags |= IFF_FLAGS_BSD_MODE; + } + if (if_fake_hwcsum != 0) { + fakeif->iff_flags |= IFF_FLAGS_HWCSUM; + } + + /* use the interface name as the unique id for ifp recycle */ + if ((unsigned int) + snprintf(fakeif->iff_name, sizeof(fakeif->iff_name), "%s%d", + ifc->ifc_name, unit) >= sizeof(fakeif->iff_name)) { + feth_release(fakeif); + return (EINVAL); + } + bzero(&feth_init, sizeof(feth_init)); + feth_init.ver = IFNET_INIT_CURRENT_VERSION; + feth_init.len = sizeof (feth_init); + if (feth_in_bsd_mode(fakeif)) { + if (if_fake_txstart != 0) { + feth_init.start = feth_start; + } else { + feth_init.flags |= IFNET_INIT_LEGACY; + feth_init.output = feth_output; + } + } + if (if_fake_nxattach == 0) { + feth_init.flags |= IFNET_INIT_NX_NOAUTO; + } + feth_init.uniqueid = fakeif->iff_name; + feth_init.uniqueid_len = strlen(fakeif->iff_name); + feth_init.name = ifc->ifc_name; + feth_init.unit = unit; + feth_init.family = IFNET_FAMILY_ETHERNET; + feth_init.type = IFT_ETHER; + feth_init.demux = ether_demux; + feth_init.add_proto = ether_add_proto; + feth_init.del_proto = ether_del_proto; + feth_init.check_multi = ether_check_multi; + feth_init.framer_extended = ether_frameout_extended; + feth_init.softc = fakeif; + feth_init.ioctl = feth_ioctl; + feth_init.set_bpf_tap = NULL; + feth_init.detach = feth_if_free; + feth_init.broadcast_addr = etherbroadcastaddr; + feth_init.broadcast_len = ETHER_ADDR_LEN; + if (feth_in_bsd_mode(fakeif)) { + error = ifnet_allocate_extended(&feth_init, &ifp); + if (error) { + feth_release(fakeif); + return (error); + } + feth_ifnet_set_attrs(fakeif, ifp); + } + fakeif->iff_media_count = default_media_words_count; + bcopy(default_media_words, fakeif->iff_media_list, + sizeof(default_media_words)); + if (feth_in_bsd_mode(fakeif)) { + error = ifnet_attach(ifp, NULL); + if (error) { + ifnet_release(ifp); + feth_release(fakeif); + return (error); + } + fakeif->iff_ifp = ifp; + } + + ifnet_set_lladdr(ifp, mac_address, sizeof(mac_address)); + + /* attach as ethernet */ + bpfattach(ifp, DLT_EN10MB, sizeof(struct ether_header)); + return (0); +} + +static int +feth_clone_destroy(ifnet_t ifp) +{ + if_fake_ref fakeif; + + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif == NULL || feth_is_detaching(fakeif)) { + feth_unlock(); + return (0); + } + feth_set_detaching(fakeif); + feth_unlock(); + + feth_config(ifp, NULL); + ifnet_detach(ifp); + return 0; +} + +static void +feth_enqueue_input(ifnet_t ifp, struct mbuf * m) +{ + struct ifnet_stat_increment_param stats = {}; + + stats.packets_in = 1; + stats.bytes_in = (uint32_t)mbuf_pkthdr_len(m) + ETHER_HDR_LEN; + ifnet_input(ifp, m, &stats); +} + +static struct mbuf * +copy_mbuf(struct mbuf *m) +{ + struct mbuf * copy_m; + uint32_t pkt_len; + uint32_t offset; + + if ((m->m_flags & M_PKTHDR) == 0) { + return (NULL); + } + pkt_len = m->m_pkthdr.len; + MGETHDR(copy_m, M_DONTWAIT, MT_DATA); + if (copy_m == NULL) { + goto failed; + } + if (pkt_len > MHLEN) { + if (pkt_len <= MCLBYTES) { + MCLGET(copy_m, M_DONTWAIT); + } else if (pkt_len <= MBIGCLBYTES) { + copy_m = m_mbigget(copy_m, M_DONTWAIT); + } else if (pkt_len <= M16KCLBYTES && njcl > 0) { + copy_m = m_m16kget(copy_m, M_DONTWAIT); + } else { + printf("if_fake: copy_mbuf(): packet too large %d\n", + pkt_len); + goto failed; + } + if (copy_m == NULL || (copy_m->m_flags & M_EXT) == 0) { + goto failed; + } + } + mbuf_setlen(copy_m, pkt_len); + copy_m->m_pkthdr.len = pkt_len; + offset = 0; + while (m != NULL && offset < pkt_len) { + uint32_t frag_len; + + frag_len = m->m_len; + if (frag_len > (pkt_len - offset)) { + printf("if_fake_: Large mbuf fragment %d > %d\n", + frag_len, (pkt_len - offset)); + goto failed; + } + m_copydata(m, 0, frag_len, mtod(copy_m, void *) + offset); + offset += frag_len; + m = m->m_next; + } + return (copy_m); + + failed: + if (copy_m != NULL) { + m_freem(copy_m); + } + return (NULL); +} + +static void +feth_output_common(ifnet_t ifp, struct mbuf * m, ifnet_t peer, + iff_flags_t flags) +{ + void * frame_header; + + frame_header = mbuf_data(m); + if ((flags & IFF_FLAGS_HWCSUM) != 0) { + m->m_pkthdr.csum_data = 0xffff; + m->m_pkthdr.csum_flags = + CSUM_DATA_VALID | CSUM_PSEUDO_HDR | + CSUM_IP_CHECKED | CSUM_IP_VALID; + } + + (void)ifnet_stat_increment_out(ifp, 1, m->m_pkthdr.len, 0); + bpf_tap_out(ifp, DLT_EN10MB, m, NULL, 0); + + (void)mbuf_pkthdr_setrcvif(m, peer); + mbuf_pkthdr_setheader(m, frame_header); + mbuf_pkthdr_adjustlen(m, - ETHER_HDR_LEN); + (void)mbuf_setdata(m, (char *)mbuf_data(m) + ETHER_HDR_LEN, + mbuf_len(m) - ETHER_HDR_LEN); + bpf_tap_in(peer, DLT_EN10MB, m, frame_header, + sizeof(struct ether_header)); + feth_enqueue_input(peer, m); +} + +static void +feth_start(ifnet_t ifp) +{ + struct mbuf * copy_m = NULL; + if_fake_ref fakeif; + iff_flags_t flags = 0; + ifnet_t peer = NULL; + struct mbuf * m; + struct mbuf * save_m; + + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif->iff_start_busy) { + feth_unlock(); + printf("if_fake: start is busy\n"); + return; + } + if (fakeif != NULL) { + peer = fakeif->iff_peer; + flags = fakeif->iff_flags; + } + + /* check for pending TX */ + m = fakeif->iff_pending_tx_packet; + if (m != NULL) { + if (peer != NULL) { + copy_m = copy_mbuf(m); + if (copy_m == NULL) { + feth_unlock(); + return; + } + } + fakeif->iff_pending_tx_packet = NULL; + m_freem(m); + m = NULL; + } + fakeif->iff_start_busy = TRUE; + feth_unlock(); + save_m = NULL; + for (;;) { + if (copy_m != NULL) { + assert(peer != NULL); + feth_output_common(ifp, copy_m, peer, flags); + copy_m = NULL; + } + if (ifnet_dequeue(ifp, &m) != 0) { + break; + } + if (peer == NULL) { + m_freem(m); + } else { + copy_m = copy_mbuf(m); + if (copy_m == NULL) { + save_m = m; + break; + } + m_freem(m); + } + } + peer = NULL; + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif != NULL) { + fakeif->iff_start_busy = FALSE; + if (save_m != NULL && fakeif->iff_peer != NULL) { + /* save it for next time */ + fakeif->iff_pending_tx_packet = save_m; + save_m = NULL; + } + } + feth_unlock(); + if (save_m != NULL) { + /* didn't save packet, so free it */ + m_freem(save_m); + } +} + +static int +feth_output(ifnet_t ifp, struct mbuf * m) +{ + struct mbuf * copy_m; + if_fake_ref fakeif; + iff_flags_t flags; + ifnet_t peer = NULL; + + if (m == NULL) { + return (0); + } + copy_m = copy_mbuf(m); + m_freem(m); + m = NULL; + if (copy_m == NULL) { + /* count this as an output error */ + ifnet_stat_increment_out(ifp, 0, 0, 1); + return (0); + } + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif != NULL) { + peer = fakeif->iff_peer; + flags = fakeif->iff_flags; + } + feth_unlock(); + if (peer == NULL) { + m_freem(copy_m); + ifnet_stat_increment_out(ifp, 0, 0, 1); + return (0); + } + feth_output_common(ifp, copy_m, peer, flags); + return (0); +} + +static int +feth_config(ifnet_t ifp, ifnet_t peer) +{ + int connected = FALSE; + int disconnected = FALSE; + int error = 0; + if_fake_ref fakeif = NULL; + + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif == NULL) { + error = EINVAL; + goto done; + } + if (peer != NULL) { + /* connect to peer */ + if_fake_ref peer_fakeif; + + peer_fakeif = ifnet_get_if_fake(peer); + if (peer_fakeif == NULL) { + error = EINVAL; + goto done; + } + if (feth_is_detaching(fakeif) || + feth_is_detaching(peer_fakeif) || + peer_fakeif->iff_peer != NULL || + fakeif->iff_peer != NULL) { + error = EBUSY; + goto done; + } + fakeif->iff_peer = peer; + peer_fakeif->iff_peer = ifp; + connected = TRUE; + } + else if (fakeif->iff_peer != NULL) { + /* disconnect from peer */ + if_fake_ref peer_fakeif; + + peer = fakeif->iff_peer; + peer_fakeif = ifnet_get_if_fake(peer); + if (peer_fakeif == NULL) { + /* should not happen */ + error = EINVAL; + goto done; + } + fakeif->iff_peer = NULL; + peer_fakeif->iff_peer = NULL; + disconnected = TRUE; + } + + done: + feth_unlock(); + + /* generate link status event if we connect or disconnect */ + if (connected) { + ifnet_set_flags(ifp, IFF_RUNNING, IFF_RUNNING); + ifnet_set_flags(peer, IFF_RUNNING, IFF_RUNNING); + interface_link_event(ifp, KEV_DL_LINK_ON); + interface_link_event(peer, KEV_DL_LINK_ON); + } + else if (disconnected) { + ifnet_set_flags(ifp, 0, IFF_RUNNING); + ifnet_set_flags(peer, 0, IFF_RUNNING); + interface_link_event(ifp, KEV_DL_LINK_OFF); + interface_link_event(peer, KEV_DL_LINK_OFF); + } + return (error); +} + +static int +feth_set_media(ifnet_t ifp, struct if_fake_request * iffr) +{ + if_fake_ref fakeif; + int error; + + if (iffr->iffr_media.iffm_count > IF_FAKE_MEDIA_LIST_MAX) { + /* list is too long */ + return (EINVAL); + } + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif == NULL) { + error = EINVAL; + goto done; + } + fakeif->iff_media_count = iffr->iffr_media.iffm_count; + bcopy(iffr->iffr_media.iffm_list, fakeif->iff_media_list, + iffr->iffr_media.iffm_count * sizeof(fakeif->iff_media_list[0])); +#if 0 + /* XXX: "auto-negotiate" active with peer? */ + /* generate link status event? */ + fakeif->iff_media_current = iffr->iffr_media.iffm_current; +#endif + error = 0; + done: + feth_unlock(); + return (error); +} + +static int +if_fake_request_copyin(user_addr_t user_addr, + struct if_fake_request *iffr, u_int32_t len) +{ + int error; + + if (user_addr == USER_ADDR_NULL || len < sizeof(*iffr)) { + error = EINVAL; + goto done; + } + error = copyin(user_addr, iffr, sizeof(*iffr)); + if (error != 0) { + goto done; + } + if (iffr->iffr_reserved[0] != 0 || iffr->iffr_reserved[1] != 0 || + iffr->iffr_reserved[2] != 0 || iffr->iffr_reserved[3] != 0) { + error = EINVAL; + goto done; + } + done: + return (error); +} + +static int +feth_set_drvspec(ifnet_t ifp, uint32_t cmd, u_int32_t len, + user_addr_t user_addr) +{ + int error; + struct if_fake_request iffr; + ifnet_t peer; + + switch (cmd) { + case IF_FAKE_S_CMD_SET_PEER: + error = if_fake_request_copyin(user_addr, &iffr, len); + if (error != 0) { + break; + } + if (iffr.iffr_peer_name[0] == '\0') { + error = feth_config(ifp, NULL); + break; + } + + /* ensure nul termination */ + iffr.iffr_peer_name[IFNAMSIZ - 1] = '\0'; + peer = ifunit(iffr.iffr_peer_name); + if (peer == NULL) { + error = ENXIO; + break; + } + if (ifnet_type(peer) != IFT_ETHER) { + error = EINVAL; + break; + } + if (strcmp(ifnet_name(peer), FAKE_ETHER_NAME) != 0) { + error = EINVAL; + break; + } + error = feth_config(ifp, peer); + break; + case IF_FAKE_S_CMD_SET_MEDIA: + error = if_fake_request_copyin(user_addr, &iffr, len); + if (error != 0) { + break; + } + error = feth_set_media(ifp, &iffr); + break; + default: + error = EOPNOTSUPP; + break; + } + return (error); +} + +static int +feth_get_drvspec(ifnet_t ifp, u_int32_t cmd, u_int32_t len, + user_addr_t user_addr) +{ + int error = EOPNOTSUPP; + if_fake_ref fakeif; + struct if_fake_request iffr; + ifnet_t peer; + + switch (cmd) { + case IF_FAKE_G_CMD_GET_PEER: + if (len < sizeof(iffr)) { + error = EINVAL; + break; + } + feth_lock(); + fakeif = (if_fake_ref)ifnet_softc(ifp); + if (fakeif == NULL) { + feth_unlock(); + error = EOPNOTSUPP; + break; + } + peer = fakeif->iff_peer; + feth_unlock(); + bzero(&iffr, sizeof(iffr)); + if (peer != NULL) { + strlcpy(iffr.iffr_peer_name, + if_name(peer), + sizeof(iffr.iffr_peer_name)); + } + error = copyout(&iffr, user_addr, sizeof(iffr)); + break; + default: + break; + } + return (error); +} + +union ifdrvu { + struct ifdrv32 *ifdrvu_32; + struct ifdrv64 *ifdrvu_64; + void *ifdrvu_p; +}; + +static int +feth_ioctl(ifnet_t ifp, u_long cmd, void * data) +{ + unsigned int count; + struct ifdevmtu * devmtu_p; + union ifdrvu drv; + uint32_t drv_cmd; + uint32_t drv_len; + boolean_t drv_set_command = FALSE; + int error = 0; + struct ifmediareq * ifmr; + struct ifreq * ifr; + if_fake_ref fakeif; + int status; + user_addr_t user_addr; + + ifr = (struct ifreq *)data; + switch (cmd) { + case SIOCSIFADDR: + ifnet_set_flags(ifp, IFF_UP, IFF_UP); + break; + + case SIOCGIFMEDIA32: + case SIOCGIFMEDIA64: + feth_lock(); + fakeif = (if_fake_ref)ifnet_softc(ifp); + if (fakeif == NULL) { + feth_unlock(); + return (EOPNOTSUPP); + } + status = (fakeif->iff_peer != NULL) + ? (IFM_AVALID | IFM_ACTIVE) : IFM_AVALID; + ifmr = (struct ifmediareq *)data; + user_addr = (cmd == SIOCGIFMEDIA64) ? + ((struct ifmediareq64 *)ifmr)->ifmu_ulist : + CAST_USER_ADDR_T(((struct ifmediareq32 *)ifmr)->ifmu_ulist); + count = ifmr->ifm_count; + ifmr->ifm_active = IFM_ETHER; + ifmr->ifm_current = IFM_ETHER; + ifmr->ifm_mask = 0; + ifmr->ifm_status = status; + if (user_addr == USER_ADDR_NULL) { + ifmr->ifm_count = fakeif->iff_media_count; + } + else if (count > 0) { + if (count > fakeif->iff_media_count) { + count = fakeif->iff_media_count; + } + ifmr->ifm_count = count; + error = copyout(&fakeif->iff_media_list, user_addr, + count * sizeof(int)); + } + feth_unlock(); + break; + + case SIOCGIFDEVMTU: + devmtu_p = &ifr->ifr_devmtu; + devmtu_p->ifdm_current = ifnet_mtu(ifp); + devmtu_p->ifdm_max = feth_max_mtu(); + devmtu_p->ifdm_min = IF_MINMTU; + break; + + case SIOCSIFMTU: + if (ifr->ifr_mtu > feth_max_mtu() || ifr->ifr_mtu < IF_MINMTU) { + error = EINVAL; + } else { + error = ifnet_set_mtu(ifp, ifr->ifr_mtu); + } + break; + + case SIOCSDRVSPEC32: + case SIOCSDRVSPEC64: + error = proc_suser(current_proc()); + if (error != 0) { + break; + } + drv_set_command = TRUE; + /* FALL THROUGH */ + case SIOCGDRVSPEC32: + case SIOCGDRVSPEC64: + drv.ifdrvu_p = data; + if (cmd == SIOCGDRVSPEC32 || cmd == SIOCSDRVSPEC32) { + drv_cmd = drv.ifdrvu_32->ifd_cmd; + drv_len = drv.ifdrvu_32->ifd_len; + user_addr = CAST_USER_ADDR_T(drv.ifdrvu_32->ifd_data); + + } else { + drv_cmd = drv.ifdrvu_64->ifd_cmd; + drv_len = drv.ifdrvu_64->ifd_len; + user_addr = drv.ifdrvu_64->ifd_data; + } + if (drv_set_command) { + error = feth_set_drvspec(ifp, drv_cmd, drv_len, + user_addr); + } else { + error = feth_get_drvspec(ifp, drv_cmd, drv_len, + user_addr); + } + break; + + case SIOCSIFLLADDR: + error = ifnet_set_lladdr(ifp, ifr->ifr_addr.sa_data, + ifr->ifr_addr.sa_len); + break; + + case SIOCSIFFLAGS: + error = 0; + break; + + case SIOCADDMULTI: + case SIOCDELMULTI: + error = 0; + break; + default: + error = EOPNOTSUPP; + break; + } + return error; +} + +static void +feth_if_free(ifnet_t ifp) +{ + if_fake_ref fakeif; + + if (ifp == NULL) { + return; + } + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif == NULL) { + feth_unlock(); + return; + } + ifp->if_softc = NULL; + feth_unlock(); + feth_release(fakeif); + ifnet_release(ifp); + return; +} + +__private_extern__ void +if_fake_init(void) +{ + int error; + + feth_lock_init(); + error = if_clone_attach(&feth_cloner); + if (error != 0) { + return; + } + return; +} diff --git a/bsd/net/if_fake_var.h b/bsd/net/if_fake_var.h new file mode 100644 index 000000000..b6b147070 --- /dev/null +++ b/bsd/net/if_fake_var.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2015-2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _NET_IF_FAKE_VAR_H_ +#define _NET_IF_FAKE_VAR_H_ 1 + +#include + +#ifdef KERNEL_PRIVATE +__private_extern__ void +if_fake_init(void); +#endif /* KERNEL_PRIVATE */ + +/* + * SIOCSDRVSPEC + */ +enum { + IF_FAKE_S_CMD_NONE = 0, + IF_FAKE_S_CMD_SET_PEER = 1, + IF_FAKE_S_CMD_SET_MEDIA = 2, +}; + +/* + * SIOCGDRVSPEC + */ +enum { + IF_FAKE_G_CMD_NONE = 0, + IF_FAKE_G_CMD_GET_PEER = 1, +}; + +#define IF_FAKE_MEDIA_LIST_MAX 27 + +struct if_fake_media { + int32_t iffm_current; + uint32_t iffm_count; + uint32_t iffm_reserved[3]; + int32_t iffm_list[IF_FAKE_MEDIA_LIST_MAX]; +}; + +struct if_fake_request { + uint64_t iffr_reserved[4]; + union { + char iffru_buf[128]; /* stable size */ + struct if_fake_media iffru_media; + char iffru_peer_name[IFNAMSIZ]; /* if name, e.g. "en0" */ + } iffr_u; +#define iffr_peer_name iffr_u.iffru_peer_name +#define iffr_media iffr_u.iffru_media +}; + +#endif /* _NET_IF_FAKE_VAR_H_ */ diff --git a/bsd/net/if_gif.c b/bsd/net/if_gif.c index f144822ba..9e2e6c6ea 100644 --- a/bsd/net/if_gif.c +++ b/bsd/net/if_gif.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -319,7 +319,7 @@ static int gif_clone_create(struct if_clone *ifc, uint32_t unit, __unused void *params) { struct gif_softc *sc = NULL; - struct ifnet_init_params gif_init_params; + struct ifnet_init_eparams gif_init_params; errno_t error = 0; lck_mtx_lock(gif_mtx); @@ -345,6 +345,9 @@ gif_clone_create(struct if_clone *ifc, uint32_t unit, __unused void *params) lck_mtx_init(&sc->gif_lock, gif_mtx_grp, gif_mtx_attr); bzero(&gif_init_params, sizeof (gif_init_params)); + gif_init_params.ver = IFNET_INIT_CURRENT_VERSION; + gif_init_params.len = sizeof (gif_init_params); + gif_init_params.flags = IFNET_INIT_LEGACY; gif_init_params.uniqueid = sc->gif_ifname; gif_init_params.uniqueid_len = strlen(sc->gif_ifname); gif_init_params.name = GIFNAME; @@ -360,7 +363,7 @@ gif_clone_create(struct if_clone *ifc, uint32_t unit, __unused void *params) gif_init_params.set_bpf_tap = gif_set_bpf_tap; gif_init_params.detach = gif_detach; - error = ifnet_allocate(&gif_init_params, &sc->gif_if); + error = ifnet_allocate_extended(&gif_init_params, &sc->gif_if); if (error != 0) { printf("gif_clone_create, ifnet_allocate failed - %d\n", error); _FREE(sc, M_DEVBUF); diff --git a/bsd/net/if_gif.h b/bsd/net/if_gif.h index 7fe954f2f..619653bee 100644 --- a/bsd/net/if_gif.h +++ b/bsd/net/if_gif.h @@ -99,7 +99,7 @@ struct gif_softc { #define GIF_LOCK(_sc) lck_mtx_lock(&(_sc)->gif_lock) #define GIF_UNLOCK(_sc) lck_mtx_unlock(&(_sc)->gif_lock) -#define GIF_LOCK_ASSERT(_sc) lck_mtx_assert(&(_sc)->gif_lock, \ +#define GIF_LOCK_ASSERT(_sc) LCK_MTX_ASSERT(&(_sc)->gif_lock, \ LCK_MTX_ASSERT_OWNED) #define gif_ro gifsc_gifscr.gifscr_ro diff --git a/bsd/net/if_ipsec.c b/bsd/net/if_ipsec.c index 9e98a05b9..8ffac41fe 100644 --- a/bsd/net/if_ipsec.c +++ b/bsd/net/if_ipsec.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -45,14 +45,20 @@ #include #include #include +#include +#include #include #include #include #include #include +#include + +#define IPSEC_NEXUS 0 extern int net_qos_policy_restricted; extern int net_qos_policy_restrict_avapps; +extern unsigned int if_enable_netagent; /* Kernel Control functions */ static errno_t ipsec_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, @@ -67,7 +73,9 @@ static errno_t ipsec_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unit int opt, void *data, size_t len); /* Network Interface functions */ +#if !IPSEC_NEXUS static void ipsec_start(ifnet_t interface); +#endif // !IPSEC_NEXUS static errno_t ipsec_output(ifnet_t interface, mbuf_t data); static errno_t ipsec_demux(ifnet_t interface, mbuf_t data, char *frame_header, protocol_family_t *protocol); @@ -86,116 +94,2068 @@ static errno_t ipsec_proto_pre_output(ifnet_t interface, protocol_family_t proto mbuf_t *packet, const struct sockaddr *dest, void *route, char *frame_type, char *link_layer_dest); -static kern_ctl_ref ipsec_kctlref; -static u_int32_t ipsec_family; +static kern_ctl_ref ipsec_kctlref; +static u_int32_t ipsec_family; +static lck_attr_t *ipsec_lck_attr; +static lck_grp_attr_t *ipsec_lck_grp_attr; +static lck_grp_t *ipsec_lck_grp; +static lck_mtx_t ipsec_lock; + +#if IPSEC_NEXUS + +SYSCTL_DECL(_net_ipsec); +SYSCTL_NODE(_net, OID_AUTO, ipsec, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IPsec"); +static int if_ipsec_verify_interface_creation = 0; +SYSCTL_INT(_net_ipsec, OID_AUTO, verify_interface_creation, CTLFLAG_RW | CTLFLAG_LOCKED, &if_ipsec_verify_interface_creation, 0, ""); + +#define IPSEC_IF_VERIFY(_e) if (unlikely(if_ipsec_verify_interface_creation)) { VERIFY(_e); } + +#define IPSEC_IF_DEFAULT_SLOT_SIZE 4096 +#define IPSEC_IF_DEFAULT_RING_SIZE 64 +#define IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE 64 +#define IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE 128 + +#define IPSEC_IF_MIN_RING_SIZE 16 +#define IPSEC_IF_MAX_RING_SIZE 1024 + +static int sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS; +static int sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS; +static int sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS; + +static int if_ipsec_ring_size = IPSEC_IF_DEFAULT_RING_SIZE; +static int if_ipsec_tx_fsw_ring_size = IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE; +static int if_ipsec_rx_fsw_ring_size = IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE; + +SYSCTL_PROC(_net_ipsec, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_ipsec_ring_size, IPSEC_IF_DEFAULT_RING_SIZE, &sysctl_if_ipsec_ring_size, "I", ""); +SYSCTL_PROC(_net_ipsec, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_ipsec_tx_fsw_ring_size, IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_ipsec_tx_fsw_ring_size, "I", ""); +SYSCTL_PROC(_net_ipsec, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_ipsec_rx_fsw_ring_size, IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_ipsec_rx_fsw_ring_size, "I", ""); + +static errno_t +ipsec_register_nexus(void); + +typedef struct ipsec_nx { + uuid_t if_provider; + uuid_t if_instance; + uuid_t ms_provider; + uuid_t ms_instance; + uuid_t ms_device; + uuid_t ms_host; + uuid_t ms_agent; +} *ipsec_nx_t; + +static nexus_controller_t ipsec_ncd; +static int ipsec_ncd_refcount; +static uuid_t ipsec_kpipe_uuid; + +#endif // IPSEC_NEXUS + +/* Control block allocated for each kernel control connection */ +struct ipsec_pcb { + TAILQ_ENTRY(ipsec_pcb) ipsec_chain; + kern_ctl_ref ipsec_ctlref; + ifnet_t ipsec_ifp; + u_int32_t ipsec_unit; + u_int32_t ipsec_unique_id; + u_int32_t ipsec_flags; + u_int32_t ipsec_input_frag_size; + bool ipsec_frag_size_set; + int ipsec_ext_ifdata_stats; + mbuf_svc_class_t ipsec_output_service_class; + char ipsec_if_xname[IFXNAMSIZ]; + char ipsec_unique_name[IFXNAMSIZ]; + // PCB lock protects state fields, like ipsec_kpipe_enabled + decl_lck_rw_data(, ipsec_pcb_lock); + bool ipsec_output_disabled; + +#if IPSEC_NEXUS + lck_mtx_t ipsec_input_chain_lock; + struct mbuf * ipsec_input_chain; + struct mbuf * ipsec_input_chain_last; + // Input chain lock protects the list of input mbufs + // The input chain lock must be taken AFTER the PCB lock if both are held + struct ipsec_nx ipsec_nx; + int ipsec_kpipe_enabled; + uuid_t ipsec_kpipe_uuid; + void * ipsec_kpipe_rxring; + void * ipsec_kpipe_txring; + + kern_nexus_t ipsec_netif_nexus; + void * ipsec_netif_rxring; + void * ipsec_netif_txring; + uint64_t ipsec_netif_txring_size; +#endif // IPSEC_NEXUS +}; + +TAILQ_HEAD(ipsec_list, ipsec_pcb) ipsec_head; + +#define IPSEC_PCB_ZONE_MAX 32 +#define IPSEC_PCB_ZONE_NAME "net.if_ipsec" + +static unsigned int ipsec_pcb_size; /* size of zone element */ +static struct zone *ipsec_pcb_zone; /* zone for ipsec_pcb */ + +#define IPSECQ_MAXLEN 256 + +#if IPSEC_NEXUS +static int +sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_ipsec_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < IPSEC_IF_MIN_RING_SIZE || + value > IPSEC_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_ipsec_ring_size = value; + + return (0); +} + +static int +sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_ipsec_tx_fsw_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < IPSEC_IF_MIN_RING_SIZE || + value > IPSEC_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_ipsec_tx_fsw_ring_size = value; + + return (0); +} + +static int +sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_ipsec_rx_fsw_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < IPSEC_IF_MIN_RING_SIZE || + value > IPSEC_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_ipsec_rx_fsw_ring_size = value; + + return (0); +} +#endif // IPSEC_NEXUS + +errno_t +ipsec_register_control(void) +{ + struct kern_ctl_reg kern_ctl; + errno_t result = 0; + + /* Find a unique value for our interface family */ + result = mbuf_tag_id_find(IPSEC_CONTROL_NAME, &ipsec_family); + if (result != 0) { + printf("ipsec_register_control - mbuf_tag_id_find_internal failed: %d\n", result); + return result; + } + + ipsec_pcb_size = sizeof(struct ipsec_pcb); + ipsec_pcb_zone = zinit(ipsec_pcb_size, + IPSEC_PCB_ZONE_MAX * ipsec_pcb_size, + 0, IPSEC_PCB_ZONE_NAME); + if (ipsec_pcb_zone == NULL) { + printf("ipsec_register_control - zinit(ipsec_pcb) failed"); + return ENOMEM; + } + +#if IPSEC_NEXUS + ipsec_register_nexus(); +#endif // IPSEC_NEXUS + + TAILQ_INIT(&ipsec_head); + + bzero(&kern_ctl, sizeof(kern_ctl)); + strlcpy(kern_ctl.ctl_name, IPSEC_CONTROL_NAME, sizeof(kern_ctl.ctl_name)); + kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0; + kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED; /* Require root */ + kern_ctl.ctl_sendsize = 64 * 1024; + kern_ctl.ctl_recvsize = 64 * 1024; + kern_ctl.ctl_connect = ipsec_ctl_connect; + kern_ctl.ctl_disconnect = ipsec_ctl_disconnect; + kern_ctl.ctl_send = ipsec_ctl_send; + kern_ctl.ctl_setopt = ipsec_ctl_setopt; + kern_ctl.ctl_getopt = ipsec_ctl_getopt; + + result = ctl_register(&kern_ctl, &ipsec_kctlref); + if (result != 0) { + printf("ipsec_register_control - ctl_register failed: %d\n", result); + return result; + } + + /* Register the protocol plumbers */ + if ((result = proto_register_plumber(PF_INET, ipsec_family, + ipsec_attach_proto, NULL)) != 0) { + printf("ipsec_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n", + ipsec_family, result); + ctl_deregister(ipsec_kctlref); + return result; + } + + /* Register the protocol plumbers */ + if ((result = proto_register_plumber(PF_INET6, ipsec_family, + ipsec_attach_proto, NULL)) != 0) { + proto_unregister_plumber(PF_INET, ipsec_family); + ctl_deregister(ipsec_kctlref); + printf("ipsec_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n", + ipsec_family, result); + return result; + } + + ipsec_lck_attr = lck_attr_alloc_init(); + ipsec_lck_grp_attr = lck_grp_attr_alloc_init(); + ipsec_lck_grp = lck_grp_alloc_init("ipsec", ipsec_lck_grp_attr); + lck_mtx_init(&ipsec_lock, ipsec_lck_grp, ipsec_lck_attr); + + return 0; +} + +/* Helpers */ +int +ipsec_interface_isvalid (ifnet_t interface) +{ + struct ipsec_pcb *pcb = NULL; + + if (interface == NULL) + return 0; + + pcb = ifnet_softc(interface); + + if (pcb == NULL) + return 0; + + /* When ctl disconnects, ipsec_unit is set to 0 */ + if (pcb->ipsec_unit == 0) + return 0; + + return 1; +} + +static errno_t +ipsec_ifnet_set_attrs(ifnet_t ifp) +{ + /* Set flags and additional information. */ + ifnet_set_mtu(ifp, 1500); + ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); + + /* The interface must generate its own IPv6 LinkLocal address, + * if possible following the recommendation of RFC2472 to the 64bit interface ID + */ + ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); + +#if !IPSEC_NEXUS + /* Reset the stats in case as the interface may have been recycled */ + struct ifnet_stats_param stats; + bzero(&stats, sizeof(struct ifnet_stats_param)); + ifnet_set_stat(ifp, &stats); +#endif // !IPSEC_NEXUS + + return (0); +} + +#if IPSEC_NEXUS + +static uuid_t ipsec_nx_dom_prov; + +static errno_t +ipsec_nxdp_init(__unused kern_nexus_domain_provider_t domprov) +{ + return 0; +} + +static void +ipsec_nxdp_fini(__unused kern_nexus_domain_provider_t domprov) +{ + // Ignore +} + +static errno_t +ipsec_register_nexus(void) +{ + const struct kern_nexus_domain_provider_init dp_init = { + .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxdpi_flags = 0, + .nxdpi_init = ipsec_nxdp_init, + .nxdpi_fini = ipsec_nxdp_fini + }; + errno_t err = 0; + + /* ipsec_nxdp_init() is called before this function returns */ + err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF, + (const uint8_t *) "com.apple.ipsec", + &dp_init, sizeof(dp_init), + &ipsec_nx_dom_prov); + if (err != 0) { + printf("%s: failed to register domain provider\n", __func__); + return (err); + } + return (0); +} + +static errno_t +ipsec_netif_prepare(kern_nexus_t nexus, ifnet_t ifp) +{ + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + pcb->ipsec_netif_nexus = nexus; + return (ipsec_ifnet_set_attrs(ifp)); +} + +static errno_t +ipsec_nexus_pre_connect(kern_nexus_provider_t nxprov, + proc_t p, kern_nexus_t nexus, + nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx) +{ +#pragma unused(nxprov, p) +#pragma unused(nexus, nexus_port, channel, ch_ctx) + return (0); +} + +static errno_t +ipsec_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, channel) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + boolean_t ok = ifnet_is_attached(pcb->ipsec_ifp, 1); + return (ok ? 0 : ENXIO); +} + +static void +ipsec_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, nexus, channel) +} + +static void +ipsec_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, nexus, channel) +} + +static void +ipsec_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, channel) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->ipsec_netif_nexus == nexus) { + pcb->ipsec_netif_nexus = NULL; + } + ifnet_decr_iorefcnt(pcb->ipsec_ifp); +} + +static errno_t +ipsec_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, + void **ring_ctx) +{ +#pragma unused(nxprov) +#pragma unused(channel) +#pragma unused(ring_ctx) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + if (!is_tx_ring) { + VERIFY(pcb->ipsec_kpipe_rxring == NULL); + pcb->ipsec_kpipe_rxring = ring; + } else { + VERIFY(pcb->ipsec_kpipe_txring == NULL); + pcb->ipsec_kpipe_txring = ring; + } + return 0; +} + +static void +ipsec_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring) +{ +#pragma unused(nxprov) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->ipsec_kpipe_rxring == ring) { + pcb->ipsec_kpipe_rxring = NULL; + } else if (pcb->ipsec_kpipe_txring == ring) { + pcb->ipsec_kpipe_txring = NULL; + } +} + +static errno_t +ipsec_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t tx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + int channel_enabled = pcb->ipsec_kpipe_enabled; + if (!channel_enabled) { + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to write, bail + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + // Signal the netif ring to read + kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring; + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + return 0; +} + +static mbuf_t +ipsec_encrypt_mbuf(ifnet_t interface, + mbuf_t data) +{ + struct ipsec_output_state ipsec_state; + int error = 0; + uint32_t af; + + // Make sure this packet isn't looping through the interface + if (necp_get_last_interface_index_from_packet(data) == interface->if_index) { + error = -1; + goto ipsec_output_err; + } + + // Mark the interface so NECP can evaluate tunnel policy + necp_mark_packet_from_interface(data, interface); + + struct ip *ip = mtod(data, struct ip *); + u_int ip_version = ip->ip_v; + + switch (ip_version) { + case 4: { + af = AF_INET; + + memset(&ipsec_state, 0, sizeof(ipsec_state)); + ipsec_state.m = data; + ipsec_state.dst = (struct sockaddr *)&ip->ip_dst; + memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); + + error = ipsec4_interface_output(&ipsec_state, interface); + if (error == 0 && ipsec_state.tunneled == 6) { + // Tunneled in IPv6 - packet is gone + // TODO: Don't lose mbuf + data = NULL; + goto done; + } + + data = ipsec_state.m; + if (error || data == NULL) { + if (error) { + printf("ipsec_encrypt_mbuf: ipsec4_output error %d\n", error); + } + goto ipsec_output_err; + } + goto done; + } + case 6: { + af = AF_INET6; + + data = ipsec6_splithdr(data); + if (data == NULL) { + printf("ipsec_encrypt_mbuf: ipsec6_splithdr returned NULL\n"); + goto ipsec_output_err; + } + + struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *); + + memset(&ipsec_state, 0, sizeof(ipsec_state)); + ipsec_state.m = data; + ipsec_state.dst = (struct sockaddr *)&ip6->ip6_dst; + memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); + + error = ipsec6_interface_output(&ipsec_state, interface, &ip6->ip6_nxt, ipsec_state.m); + if (error == 0 && ipsec_state.tunneled == 4) { + // Tunneled in IPv4 - packet is gone + // TODO: Don't lose mbuf + data = NULL; + goto done; + } + data = ipsec_state.m; + if (error || data == NULL) { + if (error) { + printf("ipsec_encrypt_mbuf: ipsec6_output error %d\n", error); + } + goto ipsec_output_err; + } + goto done; + } + default: { + printf("ipsec_encrypt_mbuf: Received unknown packet version %d\n", ip_version); + error = -1; + goto ipsec_output_err; + } + } + +done: + return data; + +ipsec_output_err: + if (data) { + mbuf_freem(data); + } + return NULL; +} + +static errno_t +ipsec_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t rx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + struct kern_channel_ring_stat_increment rx_ring_stats; + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + int channel_enabled = pcb->ipsec_kpipe_enabled; + if (!channel_enabled) { + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + // Reclaim user-released slots + (void) kern_channel_reclaim(rx_ring); + + uint32_t avail = kern_channel_available_slot_count(rx_ring); + if (avail == 0) { + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + kern_channel_ring_t tx_ring = pcb->ipsec_netif_txring; + if (tx_ring == NULL) { + // Net-If TX ring not set up yet, nothing to read + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->ipsec_netif_nexus)->nif_stats; + + // Unlock ipsec before entering ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + (void)kr_enter(tx_ring, TRUE); + + // Lock again after entering and validate + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + if (tx_ring != pcb->ipsec_netif_txring) { + // Ring no longer valid + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + kr_exit(tx_ring); + return 0; + } + + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to read, don't bother signalling + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + kr_exit(tx_ring); + return 0; + } + + struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; + VERIFY(rx_pp != NULL); + bzero(&rx_ring_stats, sizeof(rx_ring_stats)); + kern_channel_slot_t rx_pslot = NULL; + kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); + + while (rx_slot != NULL && tx_slot != NULL) { + size_t length = 0; + mbuf_t data = NULL; + errno_t error = 0; + + // Allocate rx packet + kern_packet_t rx_ph = 0; + error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + printf("ipsec_kpipe_sync_rx %s: failed to allocate packet\n", + pcb->ipsec_ifp->if_xname); + break; + } + + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + if (tx_ph == 0) { + continue; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != NULL); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0); + + length = MIN(kern_packet_get_data_length(tx_ph), + IPSEC_IF_DEFAULT_SLOT_SIZE); + + // Increment TX stats + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + + if (length > 0) { + error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); + if (error == 0) { + error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); + if (error == 0) { + // Encrypt and send packet + data = ipsec_encrypt_mbuf(pcb->ipsec_ifp, data); + } else { + printf("ipsec_kpipe_sync_rx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } + } else { + printf("ipsec_kpipe_sync_rx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + } else { + printf("ipsec_kpipe_sync_rx %s - 0 length packet\n", pcb->ipsec_ifp->if_xname); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + + if (data == NULL) { + printf("ipsec_kpipe_sync_rx %s: no encrypted packet to send\n", pcb->ipsec_ifp->if_xname); + kern_pbufpool_free(rx_pp, rx_ph); + break; + } + + length = mbuf_pkthdr_len(data); + if (length > rx_pp->pp_buflet_size) { + // Flush data + mbuf_freem(data); + kern_pbufpool_free(rx_pp, rx_ph); + printf("ipsec_kpipe_sync_rx %s: encrypted packet length %zu > %u\n", + pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size); + continue; + } + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from mbuf to buflet + mbuf_copydata(data, 0, length, (void *)rx_baddr); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_TXPKTS); + STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + if (!pcb->ipsec_ext_ifdata_stats) { + ifnet_stat_increment_out(pcb->ipsec_ifp, 1, length, 0); + } + + mbuf_freem(data); + + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + + if (rx_pslot) { + kern_channel_advance_slot(rx_ring, rx_pslot); + kern_channel_increment_ring_net_stats(rx_ring, pcb->ipsec_ifp, &rx_ring_stats); + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + if (pcb->ipsec_output_disabled) { + errno_t error = ifnet_enable_output(pcb->ipsec_ifp); + if (error != 0) { + printf("ipsec_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error); + } else { + pcb->ipsec_output_disabled = false; + } + } + + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + if (tx_pslot != NULL) { + kern_channel_notify(tx_ring, 0); + } + kr_exit(tx_ring); + + return 0; +} + +static errno_t +ipsec_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, + void **ring_ctx) +{ +#pragma unused(nxprov) +#pragma unused(channel) +#pragma unused(ring_ctx) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + if (!is_tx_ring) { + VERIFY(pcb->ipsec_netif_rxring == NULL); + pcb->ipsec_netif_rxring = ring; + } else { + VERIFY(pcb->ipsec_netif_txring == NULL); + pcb->ipsec_netif_txring = ring; + } + return 0; +} + +static void +ipsec_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring) +{ +#pragma unused(nxprov) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->ipsec_netif_rxring == ring) { + pcb->ipsec_netif_rxring = NULL; + } else if (pcb->ipsec_netif_txring == ring) { + pcb->ipsec_netif_txring = NULL; + } +} + +static bool +ipsec_netif_check_policy(mbuf_t data) +{ + necp_kernel_policy_result necp_result = 0; + necp_kernel_policy_result_parameter necp_result_parameter = {}; + uint32_t necp_matched_policy_id = 0; + + // This packet has been marked with IP level policy, do not mark again. + if (data && data->m_pkthdr.necp_mtag.necp_policy_id >= NECP_KERNEL_POLICY_ID_FIRST_VALID_IP) { + return (true); + } + + size_t length = mbuf_pkthdr_len(data); + if (length < sizeof(struct ip)) { + return (false); + } + + struct ip *ip = mtod(data, struct ip *); + u_int ip_version = ip->ip_v; + switch (ip_version) { + case 4: { + necp_matched_policy_id = necp_ip_output_find_policy_match(data, 0, NULL, + &necp_result, &necp_result_parameter); + break; + } + case 6: { + necp_matched_policy_id = necp_ip6_output_find_policy_match(data, 0, NULL, + &necp_result, &necp_result_parameter); + break; + } + default: { + return (false); + } + } + + if (necp_result == NECP_KERNEL_POLICY_RESULT_DROP || + necp_result == NECP_KERNEL_POLICY_RESULT_SOCKET_DIVERT) { + /* Drop and flow divert packets should be blocked at the IP layer */ + return (false); + } + + necp_mark_packet_from_ip(data, necp_matched_policy_id); + return (true); +} + +static errno_t +ipsec_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t tx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + + STATS_INC(nifs, NETIF_STATS_TXSYNC); + + if (tx_slot == NULL) { + // Nothing to write, don't bother signalling + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + if (pcb->ipsec_kpipe_enabled) { + kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring; + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + // Signal the kernel pipe ring to read + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + return 0; + } + + // If we're here, we're injecting into the BSD stack + while (tx_slot != NULL) { + size_t length = 0; + mbuf_t data = NULL; + + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + if (tx_ph == 0) { + continue; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != 0); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0); + + length = MIN(kern_packet_get_data_length(tx_ph), + IPSEC_IF_DEFAULT_SLOT_SIZE); + + if (length > 0) { + errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); + if (error == 0) { + error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); + if (error == 0) { + // Mark packet from policy + uint32_t policy_id = kern_packet_get_policy_id(tx_ph); + necp_mark_packet_from_ip(data, policy_id); + + // Check policy with NECP + if (!ipsec_netif_check_policy(data)) { + printf("ipsec_netif_sync_tx %s - failed policy check\n", pcb->ipsec_ifp->if_xname); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } else { + // Send through encryption + error = ipsec_output(pcb->ipsec_ifp, data); + if (error != 0) { + printf("ipsec_netif_sync_tx %s - ipsec_output error %d\n", pcb->ipsec_ifp->if_xname, error); + } + } + } else { + printf("ipsec_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } + } else { + printf("ipsec_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + } else { + printf("ipsec_netif_sync_tx %s - 0 length packet\n", pcb->ipsec_ifp->if_xname); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + + if (data == NULL) { + printf("ipsec_netif_sync_tx %s: no encrypted packet to send\n", pcb->ipsec_ifp->if_xname); + break; + } + + STATS_INC(nifs, NETIF_STATS_TXPKTS); + STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF); + + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + return 0; +} + +static errno_t +ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring, __unused uint32_t flags) +{ +#pragma unused(nxprov) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + boolean_t more = false; + errno_t rc = 0; + do { + rc = kern_channel_tx_refill(ring, UINT32_MAX, UINT32_MAX, true, &more); + if (rc != 0 && rc != EAGAIN && rc != EBUSY) { + printf("%s, tx refill failed %d\n", __func__, rc); + } + } while ((rc == 0) && more); + + if (pcb->ipsec_kpipe_enabled && !pcb->ipsec_output_disabled) { + uint32_t tx_available = kern_channel_available_slot_count(ring); + if (pcb->ipsec_netif_txring_size > 0 && + tx_available >= pcb->ipsec_netif_txring_size - 1) { + // No room left in tx ring, disable output for now + errno_t error = ifnet_disable_output(pcb->ipsec_ifp); + if (error != 0) { + printf("ipsec_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error); + } else { + pcb->ipsec_output_disabled = true; + } + } + } + + if (pcb->ipsec_kpipe_enabled && + (((rc != 0) && (rc != EAGAIN)) || pcb->ipsec_output_disabled)) { + kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring; + + // Unlock while calling notify + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + // Signal the kernel pipe ring to read + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + } else { + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + } + + return (0); +} + +static errno_t +ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t rx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + struct kern_channel_ring_stat_increment rx_ring_stats; + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + // Reclaim user-released slots + (void) kern_channel_reclaim(rx_ring); + + STATS_INC(nifs, NETIF_STATS_RXSYNC); + + uint32_t avail = kern_channel_available_slot_count(rx_ring); + if (avail == 0) { + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; + VERIFY(rx_pp != NULL); + bzero(&rx_ring_stats, sizeof(rx_ring_stats)); + kern_channel_slot_t rx_pslot = NULL; + kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); + + while (rx_slot != NULL) { + // Check for a waiting packet + lck_mtx_lock(&pcb->ipsec_input_chain_lock); + mbuf_t data = pcb->ipsec_input_chain; + if (data == NULL) { + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + break; + } + + // Allocate rx packet + kern_packet_t rx_ph = 0; + errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: failed to allocate packet\n", + pcb->ipsec_ifp->if_xname); + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + break; + } + + // Advance waiting packets + pcb->ipsec_input_chain = data->m_nextpkt; + data->m_nextpkt = NULL; + if (pcb->ipsec_input_chain == NULL) { + pcb->ipsec_input_chain_last = NULL; + } + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + + size_t length = mbuf_pkthdr_len(data); + + if (length < sizeof(struct ip)) { + // Flush data + mbuf_freem(data); + kern_pbufpool_free(rx_pp, rx_ph); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: legacy decrypted packet length cannot hold IP %zu < %zu\n", + pcb->ipsec_ifp->if_xname, length, sizeof(struct ip)); + continue; + } + + uint32_t af = 0; + struct ip *ip = mtod(data, struct ip *); + u_int ip_version = ip->ip_v; + switch (ip_version) { + case 4: { + af = AF_INET; + break; + } + case 6: { + af = AF_INET6; + break; + } + default: { + printf("ipsec_netif_sync_rx %s: legacy unknown ip version %u\n", + pcb->ipsec_ifp->if_xname, ip_version); + break; + } + } + + if (length > rx_pp->pp_buflet_size || + (pcb->ipsec_frag_size_set && length > pcb->ipsec_input_frag_size)) { + + // We need to fragment to send up into the netif + + u_int32_t fragment_mtu = rx_pp->pp_buflet_size; + if (pcb->ipsec_frag_size_set && + pcb->ipsec_input_frag_size < rx_pp->pp_buflet_size) { + fragment_mtu = pcb->ipsec_input_frag_size; + } + + mbuf_t fragment_chain = NULL; + switch (af) { + case AF_INET: { + // ip_fragment expects the length in host order + ip->ip_len = ntohs(ip->ip_len); + + // ip_fragment will modify the original data, don't free + int fragment_error = ip_fragment(data, pcb->ipsec_ifp, fragment_mtu, TRUE); + if (fragment_error == 0 && data != NULL) { + fragment_chain = data; + } else { + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: failed to fragment IPv4 packet of length %zu (%d)\n", + pcb->ipsec_ifp->if_xname, length, fragment_error); + } + break; + } + case AF_INET6: { + if (length < sizeof(struct ip6_hdr)) { + mbuf_freem(data); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: failed to fragment IPv6 packet of length %zu < %zu\n", + pcb->ipsec_ifp->if_xname, length, sizeof(struct ip6_hdr)); + } else { + + // ip6_do_fragmentation will free the original data on success only + struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *); + struct ip6_exthdrs exthdrs; + memset(&exthdrs, 0, sizeof(exthdrs)); + + int fragment_error = ip6_do_fragmentation(&data, 0, pcb->ipsec_ifp, sizeof(struct ip6_hdr), + ip6, &exthdrs, fragment_mtu, ip6->ip6_nxt); + if (fragment_error == 0 && data != NULL) { + fragment_chain = data; + } else { + mbuf_freem(data); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: failed to fragment IPv6 packet of length %zu (%d)\n", + pcb->ipsec_ifp->if_xname, length, fragment_error); + } + } + break; + } + default: { + // Cannot fragment unknown families + mbuf_freem(data); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: uknown legacy decrypted packet length %zu > %u\n", + pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size); + break; + } + } + + if (fragment_chain != NULL) { + // Add fragments to chain before continuing + lck_mtx_lock(&pcb->ipsec_input_chain_lock); + if (pcb->ipsec_input_chain != NULL) { + pcb->ipsec_input_chain_last->m_nextpkt = fragment_chain; + } else { + pcb->ipsec_input_chain = fragment_chain; + } + while (fragment_chain->m_nextpkt) { + VERIFY(fragment_chain != fragment_chain->m_nextpkt); + fragment_chain = fragment_chain->m_nextpkt; + } + pcb->ipsec_input_chain_last = fragment_chain; + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + } + + // Make sure to free unused rx packet + kern_pbufpool_free(rx_pp, rx_ph); + + continue; + } + + mbuf_pkthdr_setrcvif(data, pcb->ipsec_ifp); + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from mbuf to buflet + mbuf_copydata(data, 0, length, (void *)rx_baddr); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_set_link_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_set_network_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_RXPKTS); + STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF); + bpf_tap_packet_in(pcb->ipsec_ifp, DLT_RAW, rx_ph, NULL, 0); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + if (!pcb->ipsec_ext_ifdata_stats) { + ifnet_stat_increment_in(pcb->ipsec_ifp, 1, length, 0); + } + + mbuf_freem(data); + + // Advance ring + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_ring_t tx_ring = pcb->ipsec_kpipe_txring; + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = NULL; + if (tx_ring == NULL) { + // Net-If TX ring not set up yet, nothing to read + goto done; + } + + + // Unlock ipsec before entering ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + (void)kr_enter(tx_ring, TRUE); + + // Lock again after entering and validate + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + if (tx_ring != pcb->ipsec_kpipe_txring) { + goto done; + } + + tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to read, don't bother signalling + goto done; + } + + while (rx_slot != NULL && tx_slot != NULL) { + size_t length = 0; + mbuf_t data = NULL; + errno_t error = 0; + uint32_t af; + + // Allocate rx packet + kern_packet_t rx_ph = 0; + error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: failed to allocate packet\n", + pcb->ipsec_ifp->if_xname); + break; + } + + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + if (tx_ph == 0) { + continue; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != 0); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + length = MIN(kern_packet_get_data_length(tx_ph), + IPSEC_IF_DEFAULT_SLOT_SIZE); + + // Increment TX stats + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + + if (length >= sizeof(struct ip)) { + error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); + if (error == 0) { + error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); + if (error == 0) { + struct ip *ip = mtod(data, struct ip *); + u_int ip_version = ip->ip_v; + switch (ip_version) { + case 4: { + af = AF_INET; + ip->ip_len = ntohs(ip->ip_len) - sizeof(struct ip); + ip->ip_off = ntohs(ip->ip_off); + + if (length < ip->ip_len) { + printf("ipsec_netif_sync_rx %s: IPv4 packet length too short (%zu < %u)\n", + pcb->ipsec_ifp->if_xname, length, ip->ip_len); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } else { + data = esp4_input_extended(data, sizeof(struct ip), pcb->ipsec_ifp); + } + break; + } + case 6: { + if (length < sizeof(struct ip6_hdr)) { + printf("ipsec_netif_sync_rx %s: IPv6 packet length too short for header %zu\n", + pcb->ipsec_ifp->if_xname, length); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } else { + af = AF_INET6; + struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *); + const size_t ip6_len = sizeof(*ip6) + ntohs(ip6->ip6_plen); + if (length < ip6_len) { + printf("ipsec_netif_sync_rx %s: IPv6 packet length too short (%zu < %zu)\n", + pcb->ipsec_ifp->if_xname, length, ip6_len); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } else { + int offset = sizeof(struct ip6_hdr); + esp6_input_extended(&data, &offset, ip6->ip6_nxt, pcb->ipsec_ifp); + } + } + break; + } + default: { + printf("ipsec_netif_sync_rx %s: unknown ip version %u\n", + pcb->ipsec_ifp->if_xname, ip_version); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + break; + } + } + } else { + printf("ipsec_netif_sync_rx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } + } else { + printf("ipsec_netif_sync_rx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + } else { + printf("ipsec_netif_sync_rx %s - bad packet length %zu\n", pcb->ipsec_ifp->if_xname, length); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + + if (data == NULL) { + // Failed to get decrypted data data + kern_pbufpool_free(rx_pp, rx_ph); + continue; + } + + length = mbuf_pkthdr_len(data); + if (length > rx_pp->pp_buflet_size) { + // Flush data + mbuf_freem(data); + kern_pbufpool_free(rx_pp, rx_ph); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: decrypted packet length %zu > %u\n", + pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size); + continue; + } + + mbuf_pkthdr_setrcvif(data, pcb->ipsec_ifp); + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from mbuf to buflet + mbuf_copydata(data, 0, length, (void *)rx_baddr); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_set_link_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_set_network_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_RXPKTS); + STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT); + bpf_tap_packet_in(pcb->ipsec_ifp, DLT_RAW, rx_ph, NULL, 0); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + if (!pcb->ipsec_ext_ifdata_stats) { + ifnet_stat_increment_in(pcb->ipsec_ifp, 1, length, 0); + } + + mbuf_freem(data); + + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + +done: + if (rx_pslot) { + kern_channel_advance_slot(rx_ring, rx_pslot); + kern_channel_increment_ring_net_stats(rx_ring, pcb->ipsec_ifp, &rx_ring_stats); + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + if (tx_ring != NULL) { + if (tx_pslot != NULL) { + kern_channel_notify(tx_ring, 0); + } + kr_exit(tx_ring); + } + + return 0; +} + +static errno_t +ipsec_nexus_ifattach(struct ipsec_pcb *pcb, + struct ifnet_init_eparams *init_params, + struct ifnet **ifp) +{ + errno_t err; + nexus_controller_t controller = kern_nexus_shared_controller(); + struct kern_nexus_net_init net_init; + + nexus_name_t provider_name; + snprintf((char *)provider_name, sizeof(provider_name), + "com.apple.netif.ipsec%d", pcb->ipsec_unit); + + struct kern_nexus_provider_init prov_init = { + .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxpi_flags = NXPIF_VIRTUAL_DEVICE, + .nxpi_pre_connect = ipsec_nexus_pre_connect, + .nxpi_connected = ipsec_nexus_connected, + .nxpi_pre_disconnect = ipsec_netif_pre_disconnect, + .nxpi_disconnected = ipsec_nexus_disconnected, + .nxpi_ring_init = ipsec_netif_ring_init, + .nxpi_ring_fini = ipsec_netif_ring_fini, + .nxpi_slot_init = NULL, + .nxpi_slot_fini = NULL, + .nxpi_sync_tx = ipsec_netif_sync_tx, + .nxpi_sync_rx = ipsec_netif_sync_rx, + .nxpi_tx_doorbell = ipsec_netif_tx_doorbell, + }; + + nexus_attr_t nxa = NULL; + err = kern_nexus_attr_create(&nxa); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __func__, err); + goto failed; + } + + uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE; + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(err == 0); + + // Reset ring size for netif nexus to limit memory usage + uint64_t ring_size = if_ipsec_ring_size; + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); + VERIFY(err == 0); + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); + VERIFY(err == 0); + + pcb->ipsec_netif_txring_size = ring_size; + + err = kern_nexus_controller_register_provider(controller, + ipsec_nx_dom_prov, + provider_name, + &prov_init, + sizeof(prov_init), + nxa, + &pcb->ipsec_nx.if_provider); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s register provider failed, error %d\n", + __func__, err); + goto failed; + } + + bzero(&net_init, sizeof(net_init)); + net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION; + net_init.nxneti_flags = 0; + net_init.nxneti_eparams = init_params; + net_init.nxneti_lladdr = NULL; + net_init.nxneti_prepare = ipsec_netif_prepare; + err = kern_nexus_controller_alloc_net_provider_instance(controller, + pcb->ipsec_nx.if_provider, + pcb, + &pcb->ipsec_nx.if_instance, + &net_init, + ifp); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s alloc_net_provider_instance failed, %d\n", + __func__, err); + kern_nexus_controller_deregister_provider(controller, + pcb->ipsec_nx.if_provider); + uuid_clear(pcb->ipsec_nx.if_provider); + goto failed; + } + +failed: + if (nxa) { + kern_nexus_attr_destroy(nxa); + } + return (err); +} + +static void +ipsec_detach_provider_and_instance(uuid_t provider, uuid_t instance) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err; + + if (!uuid_is_null(instance)) { + err = kern_nexus_controller_free_provider_instance(controller, + instance); + if (err != 0) { + printf("%s free_provider_instance failed %d\n", + __func__, err); + } + uuid_clear(instance); + } + if (!uuid_is_null(provider)) { + err = kern_nexus_controller_deregister_provider(controller, + provider); + if (err != 0) { + printf("%s deregister_provider %d\n", __func__, err); + } + uuid_clear(provider); + } + return; +} + +static void +ipsec_nexus_detach(ipsec_nx_t nx) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err; + + if (!uuid_is_null(nx->ms_host)) { + err = kern_nexus_ifdetach(controller, + nx->ms_instance, + nx->ms_host); + if (err != 0) { + printf("%s: kern_nexus_ifdetach ms host failed %d\n", + __func__, err); + } + } + + if (!uuid_is_null(nx->ms_device)) { + err = kern_nexus_ifdetach(controller, + nx->ms_instance, + nx->ms_device); + if (err != 0) { + printf("%s: kern_nexus_ifdetach ms device failed %d\n", + __func__, err); + } + } + + ipsec_detach_provider_and_instance(nx->if_provider, + nx->if_instance); + ipsec_detach_provider_and_instance(nx->ms_provider, + nx->ms_instance); + + memset(nx, 0, sizeof(*nx)); +} + +static errno_t +ipsec_create_fs_provider_and_instance(uint32_t subtype, const char *type_name, + const char *ifname, + uuid_t *provider, uuid_t *instance) +{ + nexus_attr_t attr = NULL; + nexus_controller_t controller = kern_nexus_shared_controller(); + uuid_t dom_prov; + errno_t err; + struct kern_nexus_init init; + nexus_name_t provider_name; + + err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH, + &dom_prov); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s can't get %s provider, error %d\n", + __func__, type_name, err); + goto failed; + } + + err = kern_nexus_attr_create(&attr); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __func__, err); + goto failed; + } + + err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype); + VERIFY(err == 0); + + uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(err == 0); + + // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif. + uint64_t tx_ring_size = if_ipsec_tx_fsw_ring_size; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size); + VERIFY(err == 0); + uint64_t rx_ring_size = if_ipsec_rx_fsw_ring_size; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size); + VERIFY(err == 0); + + snprintf((char *)provider_name, sizeof(provider_name), + "com.apple.%s.%s", type_name, ifname); + err = kern_nexus_controller_register_provider(controller, + dom_prov, + provider_name, + NULL, + 0, + attr, + provider); + kern_nexus_attr_destroy(attr); + attr = NULL; + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s register %s provider failed, error %d\n", + __func__, type_name, err); + goto failed; + } + bzero(&init, sizeof (init)); + init.nxi_version = KERN_NEXUS_CURRENT_VERSION; + err = kern_nexus_controller_alloc_provider_instance(controller, + *provider, + NULL, + instance, &init); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s alloc_provider_instance %s failed, %d\n", + __func__, type_name, err); + kern_nexus_controller_deregister_provider(controller, + *provider); + uuid_clear(*provider); + } +failed: + return (err); +} + +static errno_t +ipsec_multistack_attach(struct ipsec_pcb *pcb) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err = 0; + ipsec_nx_t nx = &pcb->ipsec_nx; + + // Allocate multistack flowswitch + err = ipsec_create_fs_provider_and_instance(NEXUS_EXTENSION_FSW_TYPE_MULTISTACK, + "multistack", + pcb->ipsec_ifp->if_xname, + &nx->ms_provider, + &nx->ms_instance); + if (err != 0) { + printf("%s: failed to create bridge provider and instance\n", + __func__); + goto failed; + } + + // Attach multistack to device port + err = kern_nexus_ifattach(controller, nx->ms_instance, + NULL, nx->if_instance, + FALSE, &nx->ms_device); + if (err != 0) { + printf("%s kern_nexus_ifattach ms device %d\n", __func__, err); + goto failed; + } + + // Attach multistack to host port + err = kern_nexus_ifattach(controller, nx->ms_instance, + NULL, nx->if_instance, + TRUE, &nx->ms_host); + if (err != 0) { + printf("%s kern_nexus_ifattach ms host %d\n", __func__, err); + goto failed; + } + + // Extract the agent UUID and save for later + struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false); + if (multistack_nx != NULL) { + struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx); + if (flowswitch != NULL) { + FSW_RLOCK(flowswitch); + struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private; + if (ms_context != NULL) { + uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid); + } else { + printf("ipsec_multistack_attach - fsw_ms_context is NULL\n"); + } + FSW_UNLOCK(flowswitch); + } else { + printf("ipsec_multistack_attach - flowswitch is NULL\n"); + } + nx_release(multistack_nx); + } else { + printf("ipsec_multistack_attach - unable to find multistack nexus\n"); + } + + return (0); + +failed: + ipsec_nexus_detach(nx); + + errno_t detach_error = 0; + if ((detach_error = ifnet_detach(pcb->ipsec_ifp)) != 0) { + panic("ipsec_multistack_attach - ifnet_detach failed: %d\n", detach_error); + /* NOT REACHED */ + } + + return (err); +} + +#pragma mark Kernel Pipe Nexus + +static errno_t +ipsec_register_kernel_pipe_nexus(void) +{ + nexus_attr_t nxa = NULL; + errno_t result; + + lck_mtx_lock(&ipsec_lock); + if (ipsec_ncd_refcount++) { + lck_mtx_unlock(&ipsec_lock); + return 0; + } + + result = kern_nexus_controller_create(&ipsec_ncd); + if (result) { + printf("%s: kern_nexus_controller_create failed: %d\n", + __FUNCTION__, result); + goto done; + } + + uuid_t dom_prov; + result = kern_nexus_get_builtin_domain_provider( + NEXUS_TYPE_KERNEL_PIPE, &dom_prov); + if (result) { + printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n", + __FUNCTION__, result); + goto done; + } + + struct kern_nexus_provider_init prov_init = { + .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxpi_flags = NXPIF_VIRTUAL_DEVICE, + .nxpi_pre_connect = ipsec_nexus_pre_connect, + .nxpi_connected = ipsec_nexus_connected, + .nxpi_pre_disconnect = ipsec_nexus_pre_disconnect, + .nxpi_disconnected = ipsec_nexus_disconnected, + .nxpi_ring_init = ipsec_kpipe_ring_init, + .nxpi_ring_fini = ipsec_kpipe_ring_fini, + .nxpi_slot_init = NULL, + .nxpi_slot_fini = NULL, + .nxpi_sync_tx = ipsec_kpipe_sync_tx, + .nxpi_sync_rx = ipsec_kpipe_sync_rx, + .nxpi_tx_doorbell = NULL, + }; + + result = kern_nexus_attr_create(&nxa); + if (result) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __FUNCTION__, result); + goto done; + } + + uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE; + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(result == 0); + + // Reset ring size for kernel pipe nexus to limit memory usage + uint64_t ring_size = if_ipsec_ring_size; + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); + VERIFY(result == 0); + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); + VERIFY(result == 0); + + result = kern_nexus_controller_register_provider(ipsec_ncd, + dom_prov, + (const uint8_t *)"com.apple.nexus.ipsec.kpipe", + &prov_init, + sizeof(prov_init), + nxa, + &ipsec_kpipe_uuid); + if (result) { + printf("%s: kern_nexus_controller_register_provider failed: %d\n", + __FUNCTION__, result); + goto done; + } + +done: + if (nxa) { + kern_nexus_attr_destroy(nxa); + } + + if (result) { + if (ipsec_ncd) { + kern_nexus_controller_destroy(ipsec_ncd); + ipsec_ncd = NULL; + } + ipsec_ncd_refcount = 0; + } + + lck_mtx_unlock(&ipsec_lock); + + return result; +} + +static void +ipsec_unregister_kernel_pipe_nexus(void) +{ + lck_mtx_lock(&ipsec_lock); + + VERIFY(ipsec_ncd_refcount > 0); + + if (--ipsec_ncd_refcount == 0) { + kern_nexus_controller_destroy(ipsec_ncd); + ipsec_ncd = NULL; + } + + lck_mtx_unlock(&ipsec_lock); +} + +// For use by socket option, not internally +static errno_t +ipsec_disable_channel(struct ipsec_pcb *pcb) +{ + errno_t result; + int enabled; + uuid_t uuid; + + lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock); + + enabled = pcb->ipsec_kpipe_enabled; + uuid_copy(uuid, pcb->ipsec_kpipe_uuid); + + VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid) == !enabled); -#define IPSECQ_MAXLEN 256 + pcb->ipsec_kpipe_enabled = 0; + uuid_clear(pcb->ipsec_kpipe_uuid); -errno_t -ipsec_register_control(void) + lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); + + if (enabled) { + result = kern_nexus_controller_free_provider_instance(ipsec_ncd, uuid); + } else { + result = ENXIO; + } + + if (!result) { + ipsec_unregister_kernel_pipe_nexus(); + } + + return result; +} + +static errno_t +ipsec_enable_channel(struct ipsec_pcb *pcb, struct proc *proc) { - struct kern_ctl_reg kern_ctl; - errno_t result = 0; - - /* Find a unique value for our interface family */ - result = mbuf_tag_id_find(IPSEC_CONTROL_NAME, &ipsec_family); - if (result != 0) { - printf("ipsec_register_control - mbuf_tag_id_find_internal failed: %d\n", result); + struct kern_nexus_init init; + errno_t result; + + result = ipsec_register_kernel_pipe_nexus(); + if (result) { return result; } - - bzero(&kern_ctl, sizeof(kern_ctl)); - strlcpy(kern_ctl.ctl_name, IPSEC_CONTROL_NAME, sizeof(kern_ctl.ctl_name)); - kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0; - kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED; /* Require root */ - kern_ctl.ctl_sendsize = 64 * 1024; - kern_ctl.ctl_recvsize = 64 * 1024; - kern_ctl.ctl_connect = ipsec_ctl_connect; - kern_ctl.ctl_disconnect = ipsec_ctl_disconnect; - kern_ctl.ctl_send = ipsec_ctl_send; - kern_ctl.ctl_setopt = ipsec_ctl_setopt; - kern_ctl.ctl_getopt = ipsec_ctl_getopt; - - result = ctl_register(&kern_ctl, &ipsec_kctlref); - if (result != 0) { - printf("ipsec_register_control - ctl_register failed: %d\n", result); - return result; + + VERIFY(ipsec_ncd); + + lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock); + + if (pcb->ipsec_kpipe_enabled) { + result = EEXIST; // return success instead? + goto done; } - - /* Register the protocol plumbers */ - if ((result = proto_register_plumber(PF_INET, ipsec_family, - ipsec_attach_proto, NULL)) != 0) { - printf("ipsec_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n", - ipsec_family, result); - ctl_deregister(ipsec_kctlref); - return result; + + VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid)); + bzero(&init, sizeof (init)); + init.nxi_version = KERN_NEXUS_CURRENT_VERSION; + result = kern_nexus_controller_alloc_provider_instance(ipsec_ncd, + ipsec_kpipe_uuid, pcb, &pcb->ipsec_kpipe_uuid, &init); + if (result) { + goto done; } + + nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT; + result = kern_nexus_controller_bind_provider_instance(ipsec_ncd, + pcb->ipsec_kpipe_uuid, &port, + proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID); + if (result) { + kern_nexus_controller_free_provider_instance(ipsec_ncd, + pcb->ipsec_kpipe_uuid); + uuid_clear(pcb->ipsec_kpipe_uuid); + goto done; + } + + pcb->ipsec_kpipe_enabled = 1; + +done: + lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); - /* Register the protocol plumbers */ - if ((result = proto_register_plumber(PF_INET6, ipsec_family, - ipsec_attach_proto, NULL)) != 0) { - proto_unregister_plumber(PF_INET, ipsec_family); - ctl_deregister(ipsec_kctlref); - printf("ipsec_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n", - ipsec_family, result); - return result; + if (result) { + ipsec_unregister_kernel_pipe_nexus(); } - return 0; + return result; } -/* Helpers */ -int -ipsec_interface_isvalid (ifnet_t interface) -{ - struct ipsec_pcb *pcb = NULL; - - if (interface == NULL) - return 0; - - pcb = ifnet_softc(interface); - - if (pcb == NULL) - return 0; - - /* When ctl disconnects, ipsec_unit is set to 0 */ - if (pcb->ipsec_unit == 0) - return 0; - - return 1; -} +#endif // IPSEC_NEXUS + /* Kernel control functions */ +static inline void +ipsec_free_pcb(struct ipsec_pcb *pcb) +{ +#if IPSEC_NEXUS + mbuf_freem_list(pcb->ipsec_input_chain); + lck_mtx_destroy(&pcb->ipsec_input_chain_lock, ipsec_lck_grp); +#endif // IPSEC_NEXUS + lck_rw_destroy(&pcb->ipsec_pcb_lock, ipsec_lck_grp); + lck_mtx_lock(&ipsec_lock); + TAILQ_REMOVE(&ipsec_head, pcb, ipsec_chain); + lck_mtx_unlock(&ipsec_lock); + zfree(ipsec_pcb_zone, pcb); +} + static errno_t -ipsec_ctl_connect(kern_ctl_ref kctlref, - struct sockaddr_ctl *sac, - void **unitinfo) +ipsec_ctl_connect(kern_ctl_ref kctlref, + struct sockaddr_ctl *sac, + void **unitinfo) { - struct ifnet_init_eparams ipsec_init; - struct ipsec_pcb *pcb; - errno_t result; - struct ifnet_stats_param stats; + struct ifnet_init_eparams ipsec_init = {}; + errno_t result = 0; - /* kernel control allocates, interface frees */ - MALLOC(pcb, struct ipsec_pcb *, sizeof(*pcb), M_DEVBUF, M_WAITOK | M_ZERO); + struct ipsec_pcb *pcb = zalloc(ipsec_pcb_zone); + memset(pcb, 0, sizeof(*pcb)); /* Setup the protocol control block */ *unitinfo = pcb; pcb->ipsec_ctlref = kctlref; pcb->ipsec_unit = sac->sc_unit; pcb->ipsec_output_service_class = MBUF_SC_OAM; - - printf("ipsec_ctl_connect: creating interface ipsec%d\n", pcb->ipsec_unit - 1); - + + lck_mtx_lock(&ipsec_lock); + + /* Find some open interface id */ + u_int32_t chosen_unique_id = 1; + struct ipsec_pcb *next_pcb = TAILQ_LAST(&ipsec_head, ipsec_list); + if (next_pcb != NULL) { + /* List was not empty, add one to the last item */ + chosen_unique_id = next_pcb->ipsec_unique_id + 1; + next_pcb = NULL; + + /* + * If this wrapped the id number, start looking at + * the front of the list for an unused id. + */ + if (chosen_unique_id == 0) { + /* Find the next unused ID */ + chosen_unique_id = 1; + TAILQ_FOREACH(next_pcb, &ipsec_head, ipsec_chain) { + if (next_pcb->ipsec_unique_id > chosen_unique_id) { + /* We found a gap */ + break; + } + + chosen_unique_id = next_pcb->ipsec_unique_id + 1; + } + } + } + + pcb->ipsec_unique_id = chosen_unique_id; + + if (next_pcb != NULL) { + TAILQ_INSERT_BEFORE(next_pcb, pcb, ipsec_chain); + } else { + TAILQ_INSERT_TAIL(&ipsec_head, pcb, ipsec_chain); + } + lck_mtx_unlock(&ipsec_lock); + + snprintf(pcb->ipsec_if_xname, sizeof(pcb->ipsec_if_xname), "ipsec%d", pcb->ipsec_unit - 1); + snprintf(pcb->ipsec_unique_name, sizeof(pcb->ipsec_unique_name), "ipsecid%d", pcb->ipsec_unique_id - 1); + printf("ipsec_ctl_connect: creating interface %s (id %s)\n", pcb->ipsec_if_xname, pcb->ipsec_unique_name); + + lck_rw_init(&pcb->ipsec_pcb_lock, ipsec_lck_grp, ipsec_lck_attr); +#if IPSEC_NEXUS + lck_mtx_init(&pcb->ipsec_input_chain_lock, ipsec_lck_grp, ipsec_lck_attr); +#endif // IPSEC_NEXUS + /* Create the interface */ bzero(&ipsec_init, sizeof(ipsec_init)); ipsec_init.ver = IFNET_INIT_CURRENT_VERSION; ipsec_init.len = sizeof (ipsec_init); - ipsec_init.name = "ipsec"; + +#if IPSEC_NEXUS + ipsec_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO); +#else // IPSEC_NEXUS + ipsec_init.flags = IFNET_INIT_NX_NOAUTO; ipsec_init.start = ipsec_start; +#endif // IPSEC_NEXUS + ipsec_init.name = "ipsec"; ipsec_init.unit = pcb->ipsec_unit - 1; + ipsec_init.uniqueid = pcb->ipsec_unique_name; + ipsec_init.uniqueid_len = strlen(pcb->ipsec_unique_name); ipsec_init.family = ipsec_family; + ipsec_init.subfamily = IFNET_SUBFAMILY_IPSEC; ipsec_init.type = IFT_OTHER; ipsec_init.demux = ipsec_demux; ipsec_init.add_proto = ipsec_add_proto; @@ -203,44 +2163,51 @@ ipsec_ctl_connect(kern_ctl_ref kctlref, ipsec_init.softc = pcb; ipsec_init.ioctl = ipsec_ioctl; ipsec_init.detach = ipsec_detached; - + +#if IPSEC_NEXUS + result = ipsec_nexus_ifattach(pcb, &ipsec_init, &pcb->ipsec_ifp); + if (result != 0) { + printf("ipsec_ctl_connect - ipsec_nexus_ifattach failed: %d\n", result); + ipsec_free_pcb(pcb); + *unitinfo = NULL; + return result; + } + + result = ipsec_multistack_attach(pcb); + if (result != 0) { + printf("ipsec_ctl_connect - ipsec_multistack_attach failed: %d\n", result); + *unitinfo = NULL; + return result; + } + +#else // IPSEC_NEXUS result = ifnet_allocate_extended(&ipsec_init, &pcb->ipsec_ifp); if (result != 0) { printf("ipsec_ctl_connect - ifnet_allocate failed: %d\n", result); + ipsec_free_pcb(pcb); *unitinfo = NULL; - FREE(pcb, M_DEVBUF); return result; } - - /* Set flags and additional information. */ - ifnet_set_mtu(pcb->ipsec_ifp, 1500); - ifnet_set_flags(pcb->ipsec_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); - - /* The interface must generate its own IPv6 LinkLocal address, - * if possible following the recommendation of RFC2472 to the 64bit interface ID - */ - ifnet_set_eflags(pcb->ipsec_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); - - /* Reset the stats in case as the interface may have been recycled */ - bzero(&stats, sizeof(struct ifnet_stats_param)); - ifnet_set_stat(pcb->ipsec_ifp, &stats); - + ipsec_ifnet_set_attrs(pcb->ipsec_ifp); + /* Attach the interface */ result = ifnet_attach(pcb->ipsec_ifp, NULL); if (result != 0) { - printf("ipsec_ctl_connect - ifnet_allocate failed: %d\n", result); + printf("ipsec_ctl_connect - ifnet_attach failed: %d\n", result); ifnet_release(pcb->ipsec_ifp); + ipsec_free_pcb(pcb); *unitinfo = NULL; - FREE(pcb, M_DEVBUF); - } else { - /* Attach to bpf */ - bpfattach(pcb->ipsec_ifp, DLT_NULL, 4); - - /* The interfaces resoures allocated, mark it as running */ - ifnet_set_flags(pcb->ipsec_ifp, IFF_RUNNING, IFF_RUNNING); + return (result); } - - return result; +#endif // IPSEC_NEXUS + + /* Attach to bpf */ + bpfattach(pcb->ipsec_ifp, DLT_RAW, 0); + + /* The interfaces resoures allocated, mark it as running */ + ifnet_set_flags(pcb->ipsec_ifp, IFF_RUNNING, IFF_RUNNING); + + return (0); } static errno_t @@ -397,18 +2364,46 @@ ipsec_ctl_disconnect(__unused kern_ctl_ref kctlref, __unused u_int32_t unit, void *unitinfo) { - struct ipsec_pcb *pcb = unitinfo; - ifnet_t ifp = NULL; - errno_t result = 0; + struct ipsec_pcb *pcb = unitinfo; + ifnet_t ifp = NULL; + errno_t result = 0; - if (pcb == NULL) + if (pcb == NULL) { return EINVAL; + } + +#if IPSEC_NEXUS + // Tell the nexus to stop all rings + if (pcb->ipsec_netif_nexus != NULL) { + kern_nexus_stop(pcb->ipsec_netif_nexus); + } +#endif // IPSEC_NEXUS + + lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock); + +#if IPSEC_NEXUS + uuid_t kpipe_uuid; + uuid_copy(kpipe_uuid, pcb->ipsec_kpipe_uuid); + uuid_clear(pcb->ipsec_kpipe_uuid); + pcb->ipsec_kpipe_enabled = FALSE; +#endif // IPSEC_NEXUS ifp = pcb->ipsec_ifp; VERIFY(ifp != NULL); pcb->ipsec_ctlref = NULL; - pcb->ipsec_unit = 0; - + + /* + * Quiesce the interface and flush any pending outbound packets. + */ + if_down(ifp); + + /* Increment refcnt, but detach interface */ + ifnet_incr_iorefcnt(ifp); + if ((result = ifnet_detach(ifp)) != 0) { + panic("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result); + /* NOT REACHED */ + } + /* * We want to do everything in our power to ensure that the interface * really goes away when the socket is closed. We must remove IP/IPv6 @@ -419,10 +2414,20 @@ ipsec_ctl_disconnect(__unused kern_ctl_ref kctlref, ipsec_cleanup_family(ifp, AF_INET); ipsec_cleanup_family(ifp, AF_INET6); - - if ((result = ifnet_detach(ifp)) != 0) { - printf("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result); + + lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); + +#if IPSEC_NEXUS + if (!uuid_is_null(kpipe_uuid)) { + if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) { + ipsec_unregister_kernel_pipe_nexus(); + } } + ipsec_nexus_detach(&pcb->ipsec_nx); +#endif // IPSEC_NEXUS + + /* Decrement refcnt to finish detaching and freeing */ + ifnet_decr_iorefcnt(ifp); return 0; } @@ -540,6 +2545,59 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, pcb->ipsec_output_service_class); break; } + +#if IPSEC_NEXUS + case IPSEC_OPT_ENABLE_CHANNEL: { + if (len != sizeof(int)) { + result = EMSGSIZE; + break; + } + if (*(int *)data) { + result = ipsec_enable_channel(pcb, current_proc()); + } else { + result = ipsec_disable_channel(pcb); + } + break; + } + + case IPSEC_OPT_ENABLE_FLOWSWITCH: { + if (len != sizeof(int)) { + result = EMSGSIZE; + break; + } + if (!if_enable_netagent) { + result = ENOTSUP; + break; + } + if (*(int *)data) { + if (!uuid_is_null(pcb->ipsec_nx.ms_agent)) { + if_add_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.ms_agent); + } + } else { + if (!uuid_is_null(pcb->ipsec_nx.ms_agent)) { + if_delete_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.ms_agent); + } + } + break; + } + + case IPSEC_OPT_INPUT_FRAG_SIZE: { + if (len != sizeof(u_int32_t)) { + result = EMSGSIZE; + break; + } + u_int32_t input_frag_size = *(u_int32_t *)data; + if (input_frag_size <= sizeof(struct ip6_hdr)) { + pcb->ipsec_frag_size_set = FALSE; + pcb->ipsec_input_frag_size = 0; + } else { + printf("SET FRAG SIZE TO %u\n", input_frag_size); + pcb->ipsec_frag_size_set = TRUE; + pcb->ipsec_input_frag_size = input_frag_size; + } + break; + } +#endif // IPSEC_NEXUS default: result = ENOPROTOOPT; @@ -550,46 +2608,81 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, } static errno_t -ipsec_ctl_getopt(__unused kern_ctl_ref kctlref, - __unused u_int32_t unit, - void *unitinfo, - int opt, - void *data, - size_t *len) +ipsec_ctl_getopt(__unused kern_ctl_ref kctlref, + __unused u_int32_t unit, + void *unitinfo, + int opt, + void *data, + size_t *len) { - struct ipsec_pcb *pcb = unitinfo; - errno_t result = 0; + struct ipsec_pcb *pcb = unitinfo; + errno_t result = 0; switch (opt) { - case IPSEC_OPT_FLAGS: - if (*len != sizeof(u_int32_t)) + case IPSEC_OPT_FLAGS: { + if (*len != sizeof(u_int32_t)) { result = EMSGSIZE; - else + } else { *(u_int32_t *)data = pcb->ipsec_flags; + } break; + } - case IPSEC_OPT_EXT_IFDATA_STATS: - if (*len != sizeof(int)) + case IPSEC_OPT_EXT_IFDATA_STATS: { + if (*len != sizeof(int)) { result = EMSGSIZE; - else + } else { *(int *)data = (pcb->ipsec_ext_ifdata_stats) ? 1 : 0; + } break; + } - case IPSEC_OPT_IFNAME: - *len = snprintf(data, *len, "%s%d", ifnet_name(pcb->ipsec_ifp), ifnet_unit(pcb->ipsec_ifp)) + 1; + case IPSEC_OPT_IFNAME: { + if (*len < MIN(strlen(pcb->ipsec_if_xname) + 1, sizeof(pcb->ipsec_if_xname))) { + result = EMSGSIZE; + } else { + *len = snprintf(data, *len, "%s", pcb->ipsec_if_xname) + 1; + } break; + } case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS: { if (*len != sizeof(int)) { result = EMSGSIZE; - break; + } else { + *(int *)data = so_svc2tc(pcb->ipsec_output_service_class); } - *(int *)data = so_svc2tc(pcb->ipsec_output_service_class); break; } - default: + +#if IPSEC_NEXUS + case IPSEC_OPT_GET_CHANNEL_UUID: { + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + if (uuid_is_null(pcb->ipsec_kpipe_uuid)) { + result = ENXIO; + } else if (*len != sizeof(uuid_t)) { + result = EMSGSIZE; + } else { + uuid_copy(data, pcb->ipsec_kpipe_uuid); + } + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + break; + } + + case IPSEC_OPT_INPUT_FRAG_SIZE: { + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *(u_int32_t *)data = pcb->ipsec_input_frag_size; + } + break; + } +#endif // IPSEC_NEXUS + + default: { result = ENOPROTOOPT; break; + } } return result; @@ -597,10 +2690,10 @@ ipsec_ctl_getopt(__unused kern_ctl_ref kctlref, /* Network Interface functions */ static errno_t -ipsec_output(ifnet_t interface, - mbuf_t data) +ipsec_output(ifnet_t interface, + mbuf_t data) { - struct ipsec_pcb *pcb = ifnet_softc(interface); + struct ipsec_pcb *pcb = ifnet_softc(interface); struct ipsec_output_state ipsec_state; struct route ro; struct route_in6 ro6; @@ -611,13 +2704,12 @@ ipsec_output(ifnet_t interface, struct ip6_out_args ip6oa; int error = 0; u_int ip_version = 0; - uint32_t af; int flags = 0; struct flowadv *adv = NULL; // Make sure this packet isn't looping through the interface if (necp_get_last_interface_index_from_packet(data) == interface->if_index) { - error = -1; + error = EINVAL; goto ipsec_output_err; } @@ -628,16 +2720,12 @@ ipsec_output(ifnet_t interface, ip_version = ip->ip_v; switch (ip_version) { - case 4: - /* Tap */ - af = AF_INET; - bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af)); - + case 4: { /* Apply encryption */ - bzero(&ipsec_state, sizeof(ipsec_state)); + memset(&ipsec_state, 0, sizeof(ipsec_state)); ipsec_state.m = data; ipsec_state.dst = (struct sockaddr *)&ip->ip_dst; - bzero(&ipsec_state.ro, sizeof(ipsec_state.ro)); + memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); error = ipsec4_interface_output(&ipsec_state, interface); /* Tunneled in IPv6 - packet is gone */ @@ -647,7 +2735,9 @@ ipsec_output(ifnet_t interface, data = ipsec_state.m; if (error || data == NULL) { - printf("ipsec_output: ipsec4_output error %d.\n", error); + if (error) { + printf("ipsec_output: ipsec4_output error %d.\n", error); + } goto ipsec_output_err; } @@ -668,12 +2758,12 @@ ipsec_output(ifnet_t interface, ifnet_stat_increment_out(interface, 1, length, 0); /* Send to ip_output */ - bzero(&ro, sizeof(ro)); + memset(&ro, 0, sizeof(ro)); - flags = IP_OUTARGS | /* Passing out args to specify interface */ - IP_NOIPSEC; /* To ensure the packet doesn't go through ipsec twice */ + flags = (IP_OUTARGS | /* Passing out args to specify interface */ + IP_NOIPSEC); /* To ensure the packet doesn't go through ipsec twice */ - bzero(&ipoa, sizeof(ipoa)); + memset(&ipoa, 0, sizeof(ipoa)); ipoa.ipoa_flowadv.code = 0; ipoa.ipoa_flags = IPOAF_SELECT_SRCIF | IPOAF_BOUND_SRCADDR; if (ipsec_state.outgoing_if) { @@ -684,7 +2774,7 @@ ipsec_output(ifnet_t interface, adv = &ipoa.ipoa_flowadv; - (void) ip_output(data, NULL, &ro, flags, NULL, &ipoa); + (void)ip_output(data, NULL, &ro, flags, NULL, &ipoa); data = NULL; if (adv->code == FADV_FLOW_CONTROLLED || adv->code == FADV_SUSPENDED) { @@ -693,10 +2783,8 @@ ipsec_output(ifnet_t interface, } goto done; - case 6: - af = AF_INET6; - bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af)); - + } + case 6: { data = ipsec6_splithdr(data); if (data == NULL) { printf("ipsec_output: ipsec6_splithdr returned NULL\n"); @@ -705,17 +2793,20 @@ ipsec_output(ifnet_t interface, ip6 = mtod(data, struct ip6_hdr *); - bzero(&ipsec_state, sizeof(ipsec_state)); + memset(&ipsec_state, 0, sizeof(ipsec_state)); ipsec_state.m = data; ipsec_state.dst = (struct sockaddr *)&ip6->ip6_dst; - bzero(&ipsec_state.ro, sizeof(ipsec_state.ro)); + memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); error = ipsec6_interface_output(&ipsec_state, interface, &ip6->ip6_nxt, ipsec_state.m); - if (error == 0 && ipsec_state.tunneled == 4) /* tunneled in IPv4 - packet is gone */ + if (error == 0 && ipsec_state.tunneled == 4) { /* tunneled in IPv4 - packet is gone */ goto done; + } data = ipsec_state.m; if (error || data == NULL) { - printf("ipsec_output: ipsec6_output error %d.\n", error); + if (error) { + printf("ipsec_output: ipsec6_output error %d\n", error); + } goto ipsec_output_err; } @@ -731,11 +2822,11 @@ ipsec_output(ifnet_t interface, ifnet_stat_increment_out(interface, 1, length, 0); /* Send to ip6_output */ - bzero(&ro6, sizeof(ro6)); + memset(&ro6, 0, sizeof(ro6)); flags = IPV6_OUTARGS; - bzero(&ip6oa, sizeof(ip6oa)); + memset(&ip6oa, 0, sizeof(ip6oa)); ip6oa.ip6oa_flowadv.code = 0; ip6oa.ip6oa_flags = IP6OAF_SELECT_SRCIF | IP6OAF_BOUND_SRCADDR; if (ipsec_state.outgoing_if) { @@ -755,10 +2846,12 @@ ipsec_output(ifnet_t interface, } goto done; - default: + } + default: { printf("ipsec_output: Received unknown packet version %d.\n", ip_version); - error = -1; + error = EINVAL; goto ipsec_output_err; + } } done: @@ -770,11 +2863,14 @@ ipsec_output_err: goto done; } +#if !IPSEC_NEXUS static void ipsec_start(ifnet_t interface) { mbuf_t data; + struct ipsec_pcb *pcb = ifnet_softc(interface); + VERIFY(pcb != NULL); for (;;) { if (ifnet_dequeue(interface, &data) != 0) break; @@ -782,6 +2878,7 @@ ipsec_start(ifnet_t interface) break; } } +#endif // !IPSEC_NEXUS /* Network Interface functions */ static errno_t @@ -843,14 +2940,21 @@ ipsec_del_proto(__unused ifnet_t interface, } static errno_t -ipsec_ioctl(ifnet_t interface, - u_long command, - void *data) +ipsec_ioctl(ifnet_t interface, + u_long command, + void *data) { errno_t result = 0; switch(command) { case SIOCSIFMTU: +#if IPSEC_NEXUS + // Make sure we can fit packets in the channel buffers + if (((uint64_t)((struct ifreq*)data)->ifr_mtu) > IPSEC_IF_DEFAULT_SLOT_SIZE) { + ifnet_set_mtu(interface, IPSEC_IF_DEFAULT_SLOT_SIZE); + break; + } +#endif // IPSEC_NEXUS ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu); break; @@ -866,12 +2970,11 @@ ipsec_ioctl(ifnet_t interface, } static void -ipsec_detached( - ifnet_t interface) +ipsec_detached(ifnet_t interface) { - struct ipsec_pcb *pcb = ifnet_softc(interface); - - ifnet_release(pcb->ipsec_ifp); + struct ipsec_pcb *pcb = ifnet_softc(interface); + (void)ifnet_release(interface); + ipsec_free_pcb(pcb); } /* Protocol Handlers */ @@ -882,16 +2985,7 @@ ipsec_proto_input(ifnet_t interface, mbuf_t m, __unused char *frame_header) { - struct ip *ip; - uint32_t af = 0; - ip = mtod(m, struct ip *); - if (ip->ip_v == 4) - af = AF_INET; - else if (ip->ip_v == 6) - af = AF_INET6; - mbuf_pkthdr_setrcvif(m, interface); - bpf_tap_in(interface, DLT_NULL, m, &af, sizeof(af)); pktap_input(interface, protocol, m, NULL); if (proto_input(protocol, m) != 0) { @@ -938,6 +3032,38 @@ ipsec_attach_proto(ifnet_t interface, return result; } +#if IPSEC_NEXUS +errno_t +ipsec_inject_inbound_packet(ifnet_t interface, + mbuf_t packet) +{ + struct ipsec_pcb *pcb = ifnet_softc(interface); + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + lck_mtx_lock(&pcb->ipsec_input_chain_lock); + if (pcb->ipsec_input_chain != NULL) { + pcb->ipsec_input_chain_last->m_nextpkt = packet; + } else { + pcb->ipsec_input_chain = packet; + } + while (packet->m_nextpkt) { + VERIFY(packet != packet->m_nextpkt); + packet = packet->m_nextpkt; + } + pcb->ipsec_input_chain_last = packet; + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + + kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring; + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + + return (0); +} +#else // IPSEC_NEXUS errno_t ipsec_inject_inbound_packet(ifnet_t interface, mbuf_t packet) @@ -947,9 +3073,10 @@ ipsec_inject_inbound_packet(ifnet_t interface, if ((error = ipsec_demux(interface, packet, NULL, &protocol)) != 0) { return error; } - + return ipsec_proto_input(interface, protocol, packet, NULL); } +#endif // IPSEC_NEXUS void ipsec_set_pkthdr_for_interface(ifnet_t interface, mbuf_t packet, int family) diff --git a/bsd/net/if_ipsec.h b/bsd/net/if_ipsec.h index 31195e7e4..159ef2036 100644 --- a/bsd/net/if_ipsec.h +++ b/bsd/net/if_ipsec.h @@ -35,15 +35,6 @@ #include #include -/* Control block allocated for each kernel control connection */ -struct ipsec_pcb { - kern_ctl_ref ipsec_ctlref; - ifnet_t ipsec_ifp; - u_int32_t ipsec_unit; - u_int32_t ipsec_flags; - int ipsec_ext_ifdata_stats; - mbuf_svc_class_t ipsec_output_service_class; -}; errno_t ipsec_register_control(void); @@ -76,6 +67,11 @@ void ipsec_set_ip6oa_for_interface(ifnet_t interface, struct ip6_out_args *ip6oa #define IPSEC_OPT_INC_IFDATA_STATS_OUT 5 /* set to increment stat counters (type struct ipsec_stats_param) */ #define IPSEC_OPT_SET_DELEGATE_INTERFACE 6 /* set the delegate interface (char[]) */ #define IPSEC_OPT_OUTPUT_TRAFFIC_CLASS 7 /* set the traffic class for packets leaving the interface, see sys/socket.h */ +#define IPSEC_OPT_ENABLE_CHANNEL 8 /* enable a kernel pipe nexus that allows the owner to open a channel to act as a driver */ +#define IPSEC_OPT_GET_CHANNEL_UUID 9 /* get the uuid of the kernel pipe nexus instance */ +#define IPSEC_OPT_ENABLE_FLOWSWITCH 10 /* enable a flowswitch nexus that clients can use */ +#define IPSEC_OPT_INPUT_FRAG_SIZE 11 /* set the maximum size of input packets before fragmenting as a uint32_t */ + /* * ipsec stats parameter structure */ diff --git a/bsd/net/if_llatbl.c b/bsd/net/if_llatbl.c new file mode 100644 index 000000000..7b7eab342 --- /dev/null +++ b/bsd/net/if_llatbl.c @@ -0,0 +1,860 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (c) 2004 Luigi Rizzo, Alessandro Cerri. All rights reserved. + * Copyright (c) 2004-2008 Qing Li. All rights reserved. + * Copyright (c) 2008 Kip Macy. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +MALLOC_DEFINE(M_LLTABLE, "lltable", "link level address tables"); + +static SLIST_HEAD(, lltable) lltables = SLIST_HEAD_INITIALIZER(lltables); + +static lck_grp_attr_t *lltable_rwlock_grp_attr; +static lck_grp_t *lltable_rwlock_grp; +static lck_attr_t *lltable_rwlock_attr; + +static lck_grp_attr_t *lle_lock_grp_attr = NULL; +lck_grp_t *lle_lock_grp = NULL; +lck_attr_t *lle_lock_attr = NULL; + +decl_lck_rw_data(, lltable_rwlock_data); +lck_rw_t *lltable_rwlock = &lltable_rwlock_data; + +#if 0 +static void lltable_unlink(struct lltable *llt); +#endif +static void llentries_unlink(struct lltable *llt, struct llentries *head); + +static void htable_unlink_entry(struct llentry *lle); +static void htable_link_entry(struct lltable *llt, struct llentry *lle); +static int htable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, + void *farg); + +void +lltable_glbl_init() +{ + lltable_rwlock_grp_attr = lck_grp_attr_alloc_init(); + lltable_rwlock_grp = lck_grp_alloc_init("lltable_rwlock", + lltable_rwlock_grp_attr); + lltable_rwlock_attr = lck_attr_alloc_init(); + lck_rw_init(lltable_rwlock, lltable_rwlock_grp, + lltable_rwlock_attr); + + lle_lock_grp_attr = lck_grp_attr_alloc_init(); + lle_lock_grp = lck_grp_alloc_init("lle locks", lle_lock_grp_attr); + lle_lock_attr = lck_attr_alloc_init(); +} + +/* + * Dump lle state for a specific address family. + */ +static int +lltable_dump_af(struct lltable *llt, struct sysctl_req *wr) +{ + int error; + + LLTABLE_LOCK_ASSERT(); + + if (llt->llt_ifp->if_flags & IFF_LOOPBACK) + return (0); + error = 0; + + IF_AFDATA_RLOCK(llt->llt_ifp, llt->llt_af); + error = lltable_foreach_lle(llt, + (llt_foreach_cb_t *)llt->llt_dump_entry, wr); + IF_AFDATA_RUNLOCK(llt->llt_ifp, llt->llt_af); + + return (error); +} + +/* + * Dump arp state for a specific address family. + */ +int +lltable_sysctl_dumparp(int af, struct sysctl_req *wr) +{ + struct lltable *llt = NULL; + int error = 0; + + LLTABLE_RLOCK(); + SLIST_FOREACH(llt, &lltables, llt_link) { + if (llt->llt_af == af) { + error = lltable_dump_af(llt, wr); + if (error != 0) + goto done; + } + } +done: + LLTABLE_RUNLOCK(); + return (error); +} + +/* + * Common function helpers for chained hash table. + */ + +/* + * Runs specified callback for each entry in @llt. + * Caller does the locking. + * + */ +static int +htable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, void *farg) +{ + struct llentry *lle, *next; + int i, error; + + error = 0; + + for (i = 0; i < llt->llt_hsize; i++) { + LIST_FOREACH_SAFE(lle, &llt->lle_head[i], lle_next, next) { + error = f(llt, lle, farg); + if (error != 0) + break; + } + } + + return (error); +} + +static void +htable_link_entry(struct lltable *llt, struct llentry *lle) +{ + struct llentries *lleh; + uint32_t hashidx; + + if ((lle->la_flags & LLE_LINKED) != 0) + return; + + IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp, llt->llt_af); + + hashidx = llt->llt_hash(lle, llt->llt_hsize); + lleh = &llt->lle_head[hashidx]; + + lle->lle_tbl = llt; + lle->lle_head = lleh; + lle->la_flags |= LLE_LINKED; + LIST_INSERT_HEAD(lleh, lle, lle_next); +} + +static void +htable_unlink_entry(struct llentry *lle) +{ + if ((lle->la_flags & LLE_LINKED) != 0) { + IF_AFDATA_WLOCK_ASSERT(lle->lle_tbl->llt_ifp, lle->lle_tbl->llt_af); + LIST_REMOVE(lle, lle_next); + lle->la_flags &= ~(LLE_VALID | LLE_LINKED); +#if 0 + lle->lle_tbl = NULL; + lle->lle_head = NULL; +#endif + } +} + +struct prefix_match_data { + const struct sockaddr *addr; + const struct sockaddr *mask; + struct llentries dchain; + u_int flags; +}; + +static int +htable_prefix_free_cb(struct lltable *llt, struct llentry *lle, void *farg) +{ + struct prefix_match_data *pmd; + + pmd = (struct prefix_match_data *)farg; + + if (llt->llt_match_prefix(pmd->addr, pmd->mask, pmd->flags, lle)) { + LLE_WLOCK(lle); + LIST_INSERT_HEAD(&pmd->dchain, lle, lle_chain); + } + + return (0); +} + +static void +htable_prefix_free(struct lltable *llt, const struct sockaddr *addr, + const struct sockaddr *mask, u_int flags) +{ + struct llentry *lle, *next; + struct prefix_match_data pmd; + + bzero(&pmd, sizeof(pmd)); + pmd.addr = addr; + pmd.mask = mask; + pmd.flags = flags; + LIST_INIT(&pmd.dchain); + + IF_AFDATA_WLOCK(llt->llt_ifp, llt->llt_af); + /* Push matching lles to chain */ + lltable_foreach_lle(llt, htable_prefix_free_cb, &pmd); + + llentries_unlink(llt, &pmd.dchain); + IF_AFDATA_WUNLOCK(llt->llt_ifp, llt->llt_af); + + LIST_FOREACH_SAFE(lle, &pmd.dchain, lle_chain, next) + lltable_free_entry(llt, lle); +} + +static void +htable_free_tbl(struct lltable *llt) +{ + + FREE(llt->lle_head, M_LLTABLE); + FREE(llt, M_LLTABLE); +} + +static void +llentries_unlink(struct lltable *llt, struct llentries *head) +{ + struct llentry *lle, *next; + + LIST_FOREACH_SAFE(lle, head, lle_chain, next) + llt->llt_unlink_entry(lle); +} + +/* + * Helper function used to drop all mbufs in hold queue. + * + * Returns the number of held packets, if any, that were dropped. + */ +size_t +lltable_drop_entry_queue(struct llentry *lle) +{ + size_t pkts_dropped; + struct mbuf *next; + + LLE_WLOCK_ASSERT(lle); + + pkts_dropped = 0; + while ((lle->la_numheld > 0) && (lle->la_hold != NULL)) { + next = lle->la_hold->m_nextpkt; + m_freem(lle->la_hold); + lle->la_hold = next; + lle->la_numheld--; + pkts_dropped++; + } + + KASSERT(lle->la_numheld == 0, + ("%s: la_numheld %d > 0, pkts_droped %zd", __func__, + lle->la_numheld, pkts_dropped)); + + return (pkts_dropped); +} + +void +lltable_set_entry_addr(struct ifnet *ifp, struct llentry *lle, + const char *ll_addr) +{ + bcopy(ll_addr, &lle->ll_addr, ifp->if_addrlen); + lle->la_flags |= LLE_VALID; + lle->r_flags |= RLLE_VALID; +} + +#if 0 +/* + * XXX The following is related to a change to cache destination layer 2 + * header cached in the entry instead of just the destination mac address + * Right now leaving this code out and just storing the destination's mac + * information. + */ +/* + * Tries to update @lle link-level address. + * Since update requires AFDATA WLOCK, function + * drops @lle lock, acquires AFDATA lock and then acquires + * @lle lock to maintain lock order. + * + * Returns 1 on success. + */ +int +lltable_try_set_entry_addr(struct ifnet *ifp, struct llentry *lle, + const char *linkhdr, size_t linkhdrsize, int lladdr_off) +{ + /* Perform real LLE update */ + /* use afdata WLOCK to update fields */ + LLE_WLOCK_ASSERT(lle); + LLE_ADDREF(lle); + LLE_WUNLOCK(lle); + IF_AFDATA_WLOCK(ifp, lle->lle_tbl->llt_af); + LLE_WLOCK(lle); + + /* + * Since we droppped LLE lock, other thread might have deleted + * this lle. Check and return + */ + if ((lle->la_flags & LLE_DELETED) != 0) { + IF_AFDATA_WUNLOCK(ifp, lle->lle_tbl->llt_af); + LLE_FREE_LOCKED(lle); + return (0); + } + + /* Update data */ + lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize, lladdr_off); + + IF_AFDATA_WUNLOCK(ifp, lle->lle_tbl->llt_af); + + LLE_REMREF(lle); + + return (1); +} + +/* + * Helper function used to pre-compute full/partial link-layer + * header data suitable for feeding into if_output(). + */ +int +lltable_calc_llheader(struct ifnet *ifp, int family, char *lladdr, + char *buf, size_t *bufsize, int *lladdr_off) +{ + struct if_encap_req ereq; + int error; + + bzero(buf, *bufsize); + bzero(&ereq, sizeof(ereq)); + ereq.buf = buf; + ereq.bufsize = *bufsize; + ereq.rtype = IFENCAP_LL; + ereq.family = family; + ereq.lladdr = lladdr; + ereq.lladdr_len = ifp->if_addrlen; + error = ifp->if_requestencap(ifp, &ereq); + if (error == 0) { + *bufsize = ereq.bufsize; + *lladdr_off = ereq.lladdr_off; + } + + return (error); +} + +/* + * Update link-layer header for given @lle after + * interface lladdr was changed. + */ +static int +llentry_update_ifaddr(struct lltable *llt, struct llentry *lle, void *farg) +{ + struct ifnet *ifp; + u_char linkhdr[LLE_MAX_LINKHDR]; + size_t linkhdrsize; + u_char *lladdr; + int lladdr_off; + + ifp = (struct ifnet *)farg; + + lladdr = (void *)lle->ll_addr; + + LLE_WLOCK(lle); + if ((lle->la_flags & LLE_VALID) == 0) { + LLE_WUNLOCK(lle); + return (0); + } + + if ((lle->la_flags & LLE_IFADDR) != 0) + lladdr = (void *)IF_LLADDR(ifp); + + linkhdrsize = sizeof(linkhdr); + lltable_calc_llheader(ifp, llt->llt_af, (void *)lladdr, (void *)linkhdr, &linkhdrsize, + &lladdr_off); + memcpy(lle->r_linkdata, linkhdr, linkhdrsize); + LLE_WUNLOCK(lle); + + return (0); +} + +/* + * Update all calculated headers for given @llt + */ +void +lltable_update_ifaddr(struct lltable *llt) +{ + + if (llt->llt_ifp->if_flags & IFF_LOOPBACK) + return; + + IF_AFDATA_WLOCK(llt->llt_ifp, llt->llt_af); + lltable_foreach_lle(llt, llentry_update_ifaddr, llt->llt_ifp); + IF_AFDATA_WUNLOCK(llt->llt_ifp, llt->llt_af); +} +#endif + +/* + * + * Performs generic cleanup routines and frees lle. + * + * Called for non-linked entries, with callouts and + * other AF-specific cleanups performed. + * + * @lle must be passed WLOCK'ed + * + * Returns the number of held packets, if any, that were dropped. + */ +size_t +llentry_free(struct llentry *lle) +{ + size_t pkts_dropped; + + LLE_WLOCK_ASSERT(lle); + + KASSERT((lle->la_flags & LLE_LINKED) == 0, ("freeing linked lle")); + + pkts_dropped = lltable_drop_entry_queue(lle); + + LLE_FREE_LOCKED(lle); + + return (pkts_dropped); +} + +/* + * (al)locate an llentry for address dst (equivalent to rtalloc for new-arp). + * + * If found the llentry * is returned referenced and unlocked. + */ +struct llentry * +llentry_alloc(struct ifnet *ifp, struct lltable *lt, + struct sockaddr_storage *dst) +{ + struct llentry *la, *la_tmp; + + IF_AFDATA_RLOCK(ifp, lt->llt_af); + la = lla_lookup(lt, LLE_EXCLUSIVE, (struct sockaddr *)dst); + IF_AFDATA_RUNLOCK(ifp, lt->llt_af); + + if (la != NULL) { + LLE_ADDREF(la); + LLE_WUNLOCK(la); + return (la); + } + + if ((ifp->if_flags & IFF_NOARP) == 0) { + la = lltable_alloc_entry(lt, 0, (struct sockaddr *)dst); + if (la == NULL) + return (NULL); + IF_AFDATA_WLOCK(ifp, lt->llt_af); + LLE_WLOCK(la); + /* Prefer any existing LLE over newly-created one */ + la_tmp = lla_lookup(lt, LLE_EXCLUSIVE, (struct sockaddr *)dst); + if (la_tmp == NULL) + lltable_link_entry(lt, la); + IF_AFDATA_WUNLOCK(ifp, lt->llt_af); + if (la_tmp != NULL) { + lltable_free_entry(lt, la); + la = la_tmp; + } + LLE_ADDREF(la); + LLE_WUNLOCK(la); + } + + return (la); +} + +/* + * Free all entries from given table and free itself. + */ + +static int +lltable_free_cb(struct lltable *llt, struct llentry *lle, void *farg) +{ +#pragma unused(llt) + struct llentries *dchain; + + dchain = (struct llentries *)farg; + + LLE_WLOCK(lle); + LIST_INSERT_HEAD(dchain, lle, lle_chain); + + return (0); +} + +/* + * Free all entries from given table and free itself. + */ +void +lltable_free(struct lltable *llt) +{ + struct llentry *lle, *next; + struct llentries dchain; + + KASSERT(llt != NULL, ("%s: llt is NULL", __func__)); + + //lltable_unlink(llt); + + LIST_INIT(&dchain); + IF_AFDATA_WLOCK(llt->llt_ifp, llt->llt_af); + /* Push all lles to @dchain */ + lltable_foreach_lle(llt, lltable_free_cb, &dchain); + llentries_unlink(llt, &dchain); + IF_AFDATA_WUNLOCK(llt->llt_ifp, llt->llt_af); + + LIST_FOREACH_SAFE(lle, &dchain, lle_chain, next) { +#if 0 + if (thread_call_cancel(lle->lle_timer) == TRUE) + LLE_REMREF(lle); +#endif + llentry_free(lle); + } + + /* XXX We recycle network interfaces so we only purge */ + /* llt->llt_free_tbl(llt); */ +} + +#if 0 +void +lltable_drain(int af) +{ + struct lltable *llt; + struct llentry *lle; + register int i; + + LLTABLE_RLOCK(); + SLIST_FOREACH(llt, &lltables, llt_link) { + if (llt->llt_af != af) + continue; + + for (i=0; i < llt->llt_hsize; i++) { + LIST_FOREACH(lle, &llt->lle_head[i], lle_next) { + LLE_WLOCK(lle); + if (lle->la_hold) { + m_freem(lle->la_hold); + lle->la_hold = NULL; + } + LLE_WUNLOCK(lle); + } + } + } + LLTABLE_RUNLOCK(); +} +#endif + +/* + * Deletes an address from given lltable. + * Used for userland interaction to remove + * individual entries. Skips entries added by OS. + */ +int +lltable_delete_addr(struct lltable *llt, u_int flags, + const struct sockaddr *l3addr) +{ + struct llentry *lle; + struct ifnet *ifp; + + ifp = llt->llt_ifp; + IF_AFDATA_WLOCK(ifp, llt->llt_af); + lle = lla_lookup(llt, LLE_EXCLUSIVE, l3addr); + + if (lle == NULL) { + IF_AFDATA_WUNLOCK(ifp, llt->llt_af); + return (ENOENT); + } + if ((lle->la_flags & LLE_IFADDR) != 0 && (flags & LLE_IFADDR) == 0) { + IF_AFDATA_WUNLOCK(ifp, llt->llt_af); + LLE_WUNLOCK(lle); + return (EPERM); + } + + lltable_unlink_entry(llt, lle); + IF_AFDATA_WUNLOCK(ifp, llt->llt_af); + + llt->llt_delete_entry(llt, lle); + + return (0); +} + +void +lltable_prefix_free(int af, struct sockaddr *addr, struct sockaddr *mask, + u_int flags) +{ + struct lltable *llt; + + LLTABLE_RLOCK(); + SLIST_FOREACH(llt, &lltables, llt_link) { + if (llt->llt_af != af) + continue; + + llt->llt_prefix_free(llt, addr, mask, flags); + } + LLTABLE_RUNLOCK(); +} + +struct lltable * +lltable_allocate_htbl(uint32_t hsize) +{ + struct lltable *llt; + int i; + + MALLOC(llt, struct lltable *, sizeof(struct lltable), M_LLTABLE, M_WAITOK | M_ZERO); + llt->llt_hsize = hsize; + MALLOC(llt->lle_head, struct llentries *, sizeof(struct llentries) * hsize, + M_LLTABLE, M_WAITOK | M_ZERO); + + for (i = 0; i < llt->llt_hsize; i++) + LIST_INIT(&llt->lle_head[i]); + + /* Set some default callbacks */ + llt->llt_link_entry = htable_link_entry; + llt->llt_unlink_entry = htable_unlink_entry; + llt->llt_prefix_free = htable_prefix_free; + llt->llt_foreach_entry = htable_foreach_lle; + llt->llt_free_tbl = htable_free_tbl; + + return (llt); +} + +/* + * Links lltable to global llt list. + */ +void +lltable_link(struct lltable *llt) +{ + LLTABLE_WLOCK(); + SLIST_INSERT_HEAD(&lltables, llt, llt_link); + LLTABLE_WUNLOCK(); +} + +#if 0 +static void +lltable_unlink(struct lltable *llt) +{ + LLTABLE_WLOCK(); + SLIST_REMOVE(&lltables, llt, lltable, llt_link); + LLTABLE_WUNLOCK(); +} +#endif + +/* + * External methods used by lltable consumers + */ + +int +lltable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, void *farg) +{ + return (llt->llt_foreach_entry(llt, f, farg)); +} + +struct llentry * +lltable_alloc_entry(struct lltable *llt, u_int flags, + const struct sockaddr *l3addr) +{ + return (llt->llt_alloc_entry(llt, flags, l3addr)); +} + +void +lltable_free_entry(struct lltable *llt, struct llentry *lle) +{ + llt->llt_free_entry(llt, lle); +} + +void +lltable_link_entry(struct lltable *llt, struct llentry *lle) +{ + llt->llt_link_entry(llt, lle); +} + +void +lltable_unlink_entry(struct lltable *llt, struct llentry *lle) +{ + llt->llt_unlink_entry(lle); +} + +void +lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa) +{ + struct lltable *llt; + + llt = lle->lle_tbl; + llt->llt_fill_sa_entry(lle, sa); +} + +struct ifnet * +lltable_get_ifp(const struct lltable *llt) +{ + return (llt->llt_ifp); +} + +int +lltable_get_af(const struct lltable *llt) +{ + return (llt->llt_af); +} + +#define ifnet_byindex(index) ifindex2ifnet[(index)] + +/* + * Called in route_output when rtm_flags contains RTF_LLDATA. + */ +int +lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info) +{ + struct sockaddr_dl *dl = + (struct sockaddr_dl *)(void *)info->rti_info[RTAX_GATEWAY]; + struct sockaddr *dst = (struct sockaddr *)info->rti_info[RTAX_DST]; + struct ifnet *ifp; + struct lltable *llt; + struct llentry *lle, *lle_tmp; + u_int laflags = 0; + int error; + + KASSERT(dl != NULL && dl->sdl_family == AF_LINK, + ("%s: invalid dl\n", __func__)); + + ifp = ifnet_byindex(dl->sdl_index); + if (ifp == NULL) { + log(LOG_INFO, "%s: invalid ifp (sdl_index %d)\n", + __func__, dl->sdl_index); + return EINVAL; + } + + /* XXX linked list may be too expensive */ + LLTABLE_RLOCK(); + SLIST_FOREACH(llt, &lltables, llt_link) { + if (llt->llt_af == dst->sa_family && + llt->llt_ifp == ifp) + break; + } + LLTABLE_RUNLOCK(); + KASSERT(llt != NULL, ("Yep, ugly hacks are bad\n")); + + error = 0; + + switch (rtm->rtm_type) { + case RTM_ADD: + /* Add static LLE */ + laflags = 0; + if (rtm->rtm_rmx.rmx_expire == 0) + laflags = LLE_STATIC; + lle = lltable_alloc_entry(llt, laflags, dst); + if (lle == NULL) + return (ENOMEM); +#if 0 + linkhdrsize = sizeof(linkhdr); + if (lltable_calc_llheader(ifp, dst->sa_family, LLADDR(dl), + (void *)linkhdr, &linkhdrsize, &lladdr_off) != 0) + return (EINVAL); +#endif + lltable_set_entry_addr(ifp, lle, LLADDR(dl)); + + if (rtm->rtm_flags & RTF_ANNOUNCE) + lle->la_flags |= LLE_PUB; + lle->la_expire = rtm->rtm_rmx.rmx_expire; + + laflags = lle->la_flags; + + /* Try to link new entry */ + lle_tmp = NULL; + IF_AFDATA_WLOCK(ifp, llt->llt_af); + LLE_WLOCK(lle); + lle_tmp = lla_lookup(llt, LLE_EXCLUSIVE, dst); + if (lle_tmp != NULL) { + /* Check if we are trying to replace immutable entry */ + if ((lle_tmp->la_flags & LLE_IFADDR) != 0) { + IF_AFDATA_WUNLOCK(ifp, llt->llt_af); + LLE_WUNLOCK(lle_tmp); + lltable_free_entry(llt, lle); + return (EPERM); + } + /* Unlink existing entry from table */ + lltable_unlink_entry(llt, lle_tmp); + } + lltable_link_entry(llt, lle); + IF_AFDATA_WUNLOCK(ifp, llt->llt_af); + + if (lle_tmp != NULL) { + EVENTHANDLER_INVOKE(NULL, lle_event, lle_tmp, LLENTRY_EXPIRED); + lltable_free_entry(llt, lle_tmp); + } + + /* + * By invoking LLE handler here we might get + * two events on static LLE entry insertion + * in routing socket. However, since we might have + * other subscribers we need to generate this event. + */ + EVENTHANDLER_INVOKE(NULL, lle_event, lle, LLENTRY_RESOLVED); + LLE_WUNLOCK(lle); +#ifdef INET + /* gratuitous ARP */ + if ((laflags & LLE_PUB) && dst->sa_family == AF_INET) + dlil_send_arp(ifp, ARPOP_REQUEST, NULL, dst, NULL, dst, 0); +#endif + + break; + + case RTM_DELETE: + return (lltable_delete_addr(llt, 0, dst)); + + default: + error = EINVAL; + } + + return (error); +} + diff --git a/bsd/net/if_llatbl.h b/bsd/net/if_llatbl.h new file mode 100644 index 000000000..9b4697fb6 --- /dev/null +++ b/bsd/net/if_llatbl.h @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (c) 2004 Luigi Rizzo, Alessandro Cerri. All rights reserved. + * Copyright (c) 2004-2008 Qing Li. All rights reserved. + * Copyright (c) 2008 Kip Macy. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include + +#ifndef _NET_IF_LLATBL_H_ +#define _NET_IF_LLATBL_H_ + +#include +#include +#include +#include +#include + +struct ifnet; +struct sysctl_req; +struct rt_msghdr; +struct rt_addrinfo; + +struct llentry; +LIST_HEAD(llentries, llentry); + +extern lck_rw_t *lltable_rwlock; +#define LLTABLE_RLOCK() lck_rw_lock_shared(lltable_rwlock) +#define LLTABLE_RUNLOCK() lck_rw_done(lltable_rwlock) +#define LLTABLE_WLOCK() lck_rw_lock_exclusive(lltable_rwlock) +#define LLTABLE_WUNLOCK() lck_rw_done(lltable_rwlock) +#define LLTABLE_LOCK_ASSERT() LCK_RW_ASSERT(lltable_rwlock, LCK_RW_ASSERT_EXCLUSIVE) + +#define LLE_MAX_LINKHDR 24 /* Full IB header */ +/* + * Code referencing llentry must at least hold + * a shared lock + */ +struct llentry { + LIST_ENTRY(llentry) lle_next; + union { + struct in_addr addr4; + struct in6_addr addr6; + } r_l3addr; +#if 0 + char r_linkdata[LLE_MAX_LINKHDR]; /* L2 data */ + uint8_t r_hdrlen; /* length for LL header */ + uint8_t spare0[3]; +#endif + uint16_t r_flags; /* LLE runtime flags */ + uint16_t r_skip_req; /* feedback from fast path */ + + struct lltable *lle_tbl; + struct llentries *lle_head; + void (*lle_free)(struct llentry *); + struct mbuf *la_hold; + int la_numheld; /* # of packets currently held */ + u_int64_t la_expire; + uint16_t la_flags; + uint16_t la_asked; + uint16_t la_preempt; + int16_t ln_state; /* IPv6 has ND6_LLINFO_NOSTATE == -2 */ + uint16_t ln_router; + time_t ln_ntick; + time_t lle_remtime; /* Real time remaining */ + time_t lle_hittime; /* Time when r_skip_req was unset */ + int lle_refcnt; + union { + uint64_t mac_aligned; + uint16_t mac16[3]; + } ll_addr; + LIST_ENTRY(llentry) lle_chain; /* chain of deleted items */ + thread_call_t lle_timer; + u_int64_t ln_lastused; /* last used timestamp */ + struct if_llreach *ln_llreach; /* link-layer reachability record */ + decl_lck_rw_data(, lle_lock); + decl_lck_mtx_data(, req_mtx); +}; + +extern lck_grp_t *lle_lock_grp; +extern lck_attr_t *lle_lock_attr; + +#define LLE_WLOCK(lle) lck_rw_lock_exclusive(&(lle)->lle_lock) +#define LLE_RLOCK(lle) lck_rw_lock_shared(&(lle)->lle_lock) +#define LLE_WUNLOCK(lle) lck_rw_done(&(lle)->lle_lock) +#define LLE_RUNLOCK(lle) lck_rw_done(&(lle)->lle_lock) +#define LLE_DOWNGRADE(lle) lck_rw_lock_exclusive_to_shared(&(lle)->lle_lock) +#define LLE_TRY_UPGRADE(lle) lck_rw_lock_shared_to_exclusive(&(lle)->lle_lock) +#define LLE_LOCK_INIT(lle) lck_rw_init(&(lle)->lle_lock, lle_lock_grp, lle_lock_attr) +#define LLE_LOCK_DESTROY(lle) lck_rw_destroy(&(lle)->lle_lock, lle_lock_grp) +#define LLE_WLOCK_ASSERT(lle) LCK_RW_ASSERT(&(lle)->lle_lock, LCK_RW_ASSERT_EXCLUSIVE) + +#define LLE_REQ_INIT(lle) lck_mtx_init(&(lle)->req_mtx, lle_lock_grp, lle_lock_attr) +#define LLE_REQ_DESTROY(lle) lck_mtx_destroy(&(lle)->req_mtx, lle_lock_grp) +#define LLE_REQ_LOCK(lle) lck_mtx_lock(&(lle)->req_mtx) +#define LLE_REQ_UNLOCK(lle) lck_mtx_unlock(&(lle)->req_mtx) + +#define LLE_IS_VALID(lle) (((lle) != NULL) && ((lle) != (void *)-1)) + +#define LLE_ADDREF(lle) do { \ + LLE_WLOCK_ASSERT(lle); \ + VERIFY((lle)->lle_refcnt >= 0); \ + (lle)->lle_refcnt++; \ +} while (0) + +#define LLE_REMREF(lle) do { \ + LLE_WLOCK_ASSERT(lle); \ + VERIFY((lle)->lle_refcnt > 0); \ + (lle)->lle_refcnt--; \ +} while (0) + +#define LLE_FREE_LOCKED(lle) do { \ + if ((lle)->lle_refcnt == 1) \ + (lle)->lle_free(lle); \ + else { \ + LLE_REMREF(lle); \ + LLE_WUNLOCK(lle); \ + } \ + /* guard against invalid refs */ \ + (lle) = NULL; \ +} while (0) + +#define LLE_FREE(lle) do { \ + LLE_WLOCK(lle); \ + LLE_FREE_LOCKED(lle); \ +} while (0) + +typedef struct llentry *(llt_lookup_t)(struct lltable *, u_int flags, + const struct sockaddr *l3addr); +typedef struct llentry *(llt_alloc_t)(struct lltable *, u_int flags, + const struct sockaddr *l3addr); +typedef void (llt_delete_t)(struct lltable *, struct llentry *); +typedef void (llt_prefix_free_t)(struct lltable *, + const struct sockaddr *addr, const struct sockaddr *mask, u_int flags); +typedef int (llt_dump_entry_t)(struct lltable *, struct llentry *, + struct sysctl_req *); +typedef uint32_t (llt_hash_t)(const struct llentry *, uint32_t); +typedef int (llt_match_prefix_t)(const struct sockaddr *, + const struct sockaddr *, u_int, struct llentry *); +typedef void (llt_free_entry_t)(struct lltable *, struct llentry *); +typedef void (llt_fill_sa_entry_t)(const struct llentry *, struct sockaddr *); +typedef void (llt_free_tbl_t)(struct lltable *); +typedef void (llt_link_entry_t)(struct lltable *, struct llentry *); +typedef void (llt_unlink_entry_t)(struct llentry *); + +typedef int (llt_foreach_cb_t)(struct lltable *, struct llentry *, void *); +typedef int (llt_foreach_entry_t)(struct lltable *, llt_foreach_cb_t *, void *); + +struct lltable { + SLIST_ENTRY(lltable) llt_link; + int llt_af; + int llt_hsize; + struct llentries *lle_head; + struct ifnet *llt_ifp; + + llt_lookup_t *llt_lookup; + llt_alloc_t *llt_alloc_entry; + llt_delete_t *llt_delete_entry; + llt_prefix_free_t *llt_prefix_free; + llt_dump_entry_t *llt_dump_entry; + llt_hash_t *llt_hash; + llt_match_prefix_t *llt_match_prefix; + llt_free_entry_t *llt_free_entry; + llt_foreach_entry_t *llt_foreach_entry; + llt_link_entry_t *llt_link_entry; + llt_unlink_entry_t *llt_unlink_entry; + llt_fill_sa_entry_t *llt_fill_sa_entry; + llt_free_tbl_t *llt_free_tbl; +}; + +#ifdef MALLOC_DECLARE +MALLOC_DECLARE(M_LLTABLE); +#endif + +/* + * LLentry flags + */ +#define LLE_DELETED 0x0001 /* entry must be deleted */ +#define LLE_STATIC 0x0002 /* entry is static */ +#define LLE_IFADDR 0x0004 /* entry is interface addr */ +#define LLE_VALID 0x0008 /* ll_addr is valid */ +#define LLE_REDIRECT 0x0010 /* installed by redirect; has host rtentry */ +#define LLE_PUB 0x0020 /* publish entry ??? */ +#define LLE_LINKED 0x0040 /* linked to lookup structure */ +/* LLE request flags */ +#define LLE_EXCLUSIVE 0x2000 /* return lle xlocked */ +#define LLE_UNLOCKED 0x4000 /* return lle unlocked */ +#define LLE_ADDRONLY 0x4000 /* return lladdr instead of full header */ +#define LLE_CREATE 0x8000 /* hint to avoid lle lookup */ + +/* LLE flags used by fastpath code */ +#define RLLE_VALID 0x0001 /* entry is valid */ +#define RLLE_IFADDR LLE_IFADDR /* entry is ifaddr */ + +#define LLATBL_HASH(key, mask) \ + (((((((key >> 8) ^ key) >> 8) ^ key) >> 8) ^ key) & mask) + +void lltable_glbl_init(void); +struct lltable *lltable_allocate_htbl(uint32_t hsize); +void lltable_free(struct lltable *); +void lltable_link(struct lltable *llt); +void lltable_prefix_free(int, struct sockaddr *, + struct sockaddr *, u_int); +#if 0 +void lltable_drain(int); +#endif +int lltable_sysctl_dumparp(int, struct sysctl_req *); + +size_t llentry_free(struct llentry *); +struct llentry *llentry_alloc(struct ifnet *, struct lltable *, + struct sockaddr_storage *); + +/* helper functions */ +size_t lltable_drop_entry_queue(struct llentry *); +void lltable_set_entry_addr(struct ifnet *ifp, struct llentry *lle, + const char *ll_addr); +int lltable_try_set_entry_addr(struct ifnet *ifp, struct llentry *lle, + const char *linkhdr, size_t linkhdrsize, int lladdr_off); + +int lltable_calc_llheader(struct ifnet *ifp, int family, char *lladdr, + char *buf, size_t *bufsize, int *lladdr_off); +void lltable_update_ifaddr(struct lltable *llt); +struct llentry *lltable_alloc_entry(struct lltable *llt, u_int flags, + const struct sockaddr *l4addr); +void lltable_free_entry(struct lltable *llt, struct llentry *lle); +int lltable_delete_addr(struct lltable *llt, u_int flags, + const struct sockaddr *l3addr); +void lltable_link_entry(struct lltable *llt, struct llentry *lle); +void lltable_unlink_entry(struct lltable *llt, struct llentry *lle); +void lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa); +struct ifnet *lltable_get_ifp(const struct lltable *llt); +int lltable_get_af(const struct lltable *llt); + +int lltable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, + void *farg); +/* + * Generic link layer address lookup function. + */ +static __inline struct llentry * +lla_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr) +{ + return (llt->llt_lookup(llt, flags, l3addr)); +} + +int lla_rt_output(struct rt_msghdr *, struct rt_addrinfo *); + +enum { + LLENTRY_RESOLVED, + LLENTRY_TIMEDOUT, + LLENTRY_DELETED, + LLENTRY_EXPIRED, +}; + +typedef void (*lle_event_fn)(struct eventhandler_entry_arg, struct llentry *, int); +EVENTHANDLER_DECLARE(lle_event, lle_event_fn); +#endif /* _NET_IF_LLATBL_H_ */ diff --git a/bsd/net/if_llreach.c b/bsd/net/if_llreach.c index 2012d8c05..764e72bd8 100644 --- a/bsd/net/if_llreach.c +++ b/bsd/net/if_llreach.c @@ -320,7 +320,7 @@ found: if (!lck_rw_lock_shared_to_exclusive(&ifp->if_llreach_lock)) lck_rw_lock_exclusive(&ifp->if_llreach_lock); - lck_rw_assert(&ifp->if_llreach_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&ifp->if_llreach_lock, LCK_RW_ASSERT_EXCLUSIVE); /* in case things have changed while becoming writer */ lr = RB_FIND(ll_reach_tree, &ifp->if_ll_srcs, &find); diff --git a/bsd/net/if_llreach.h b/bsd/net/if_llreach.h index c27ff9ded..63c86aef4 100644 --- a/bsd/net/if_llreach.h +++ b/bsd/net/if_llreach.h @@ -97,10 +97,10 @@ RB_PROTOTYPE_SC_PREV(__private_extern__, ll_reach_tree, if_llreach, ls_link, ifllr_cmp); #define IFLR_LOCK_ASSERT_HELD(_iflr) \ - lck_mtx_assert(&(_iflr)->lr_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_iflr)->lr_lock, LCK_MTX_ASSERT_OWNED) #define IFLR_LOCK_ASSERT_NOTHELD(_iflr) \ - lck_mtx_assert(&(_iflr)->lr_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_iflr)->lr_lock, LCK_MTX_ASSERT_NOTOWNED) #define IFLR_LOCK(_iflr) \ lck_mtx_lock(&(_iflr)->lr_lock) diff --git a/bsd/net/if_loop.c b/bsd/net/if_loop.c index 3baa27434..00a8345e3 100644 --- a/bsd/net/if_loop.c +++ b/bsd/net/if_loop.c @@ -177,15 +177,6 @@ SYSCTL_DECL(_net_link); SYSCTL_NODE(_net_link, OID_AUTO, loopback, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "loopback interface"); -#define LO_BW_SLEEP 10 -static u_int32_t lo_bw_sleep_usec = LO_BW_SLEEP; -SYSCTL_UINT(_net_link_loopback, OID_AUTO, bw_sleep_usec, - CTLFLAG_RW | CTLFLAG_LOCKED, &lo_bw_sleep_usec, LO_BW_SLEEP, ""); - -static u_int32_t lo_bw_measure = 0; -SYSCTL_UINT(_net_link_loopback, OID_AUTO, bw_measure, - CTLFLAG_RW | CTLFLAG_LOCKED, &lo_bw_measure, 0, ""); - static u_int32_t lo_dequeue_max = LOSNDQ_MAXLEN; SYSCTL_PROC(_net_link_loopback, OID_AUTO, max_dequeue, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &lo_dequeue_max, LOSNDQ_MAXLEN, @@ -405,8 +396,6 @@ lo_start(struct ifnet *ifp) for (;;) { struct mbuf *m = NULL, *m_tail = NULL; u_int32_t cnt, len = 0; - int sleep_chan = 0; - struct timespec ts; if (lo_sched_model == IFNET_SCHED_MODEL_NORMAL) { if (ifnet_dequeue_multi(ifp, lo_dequeue_max, &m, @@ -420,21 +409,6 @@ lo_start(struct ifnet *ifp) } LO_BPF_TAP_OUT_MULTI(m); - - if (lo_bw_measure) { - if (cnt >= if_bw_measure_size) - ifnet_transmit_burst_start(ifp, m); - if (lo_bw_sleep_usec > 0) { - bzero(&ts, sizeof(ts)); - ts.tv_nsec = (lo_bw_sleep_usec << 10) * cnt; - - /* Add msleep with timeout */ - (void) msleep(&sleep_chan, NULL, - PSOCK, "lo_start", &ts); - } - if (cnt >= if_bw_measure_size) - ifnet_transmit_burst_end(ifp, m_tail); - } lo_tx_compl(ifp, m); /* stats are required for extended variant */ @@ -683,6 +657,7 @@ loopattach(void) lo_init.flags = IFNET_INIT_LEGACY; lo_init.output = lo_output; } + lo_init.flags |= IFNET_INIT_NX_NOAUTO; lo_init.name = "lo"; lo_init.unit = 0; lo_init.family = IFNET_FAMILY_LOOPBACK; diff --git a/bsd/net/if_pflog.c b/bsd/net/if_pflog.c index 3bc86c820..1c9113d70 100644 --- a/bsd/net/if_pflog.c +++ b/bsd/net/if_pflog.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -136,7 +136,7 @@ static int pflog_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) { struct pflog_softc *pflogif; - struct ifnet_init_params pf_init; + struct ifnet_init_eparams pf_init; int error = 0; if (unit >= PFLOGIFS_MAX) { @@ -153,6 +153,9 @@ pflog_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) } bzero(&pf_init, sizeof (pf_init)); + pf_init.ver = IFNET_INIT_CURRENT_VERSION; + pf_init.len = sizeof (pf_init); + pf_init.flags = IFNET_INIT_LEGACY; pf_init.name = ifc->ifc_name; pf_init.unit = unit; pf_init.type = IFT_PFLOG; @@ -168,7 +171,7 @@ pflog_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) bzero(pflogif, sizeof (*pflogif)); pflogif->sc_unit = unit; - error = ifnet_allocate(&pf_init, &pflogif->sc_if); + error = ifnet_allocate_extended(&pf_init, &pflogif->sc_if); if (error != 0) { printf("%s: ifnet_allocate failed - %d\n", __func__, error); _FREE(pflogif, M_DEVBUF); @@ -281,17 +284,18 @@ pflogfree(struct ifnet *ifp) } int -pflog_packet(struct pfi_kif *kif, struct mbuf *m, sa_family_t af, u_int8_t dir, +pflog_packet(struct pfi_kif *kif, pbuf_t *pbuf, sa_family_t af, u_int8_t dir, u_int8_t reason, struct pf_rule *rm, struct pf_rule *am, struct pf_ruleset *ruleset, struct pf_pdesc *pd) { #if NBPFILTER > 0 struct ifnet *ifn; struct pfloghdr hdr; + struct mbuf *m; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); - if (kif == NULL || m == NULL || rm == NULL || pd == NULL) + if (kif == NULL || !pbuf_is_valid(pbuf) || rm == NULL || pd == NULL) return (-1); if (rm->logif >= PFLOGIFS_MAX || @@ -299,6 +303,9 @@ pflog_packet(struct pfi_kif *kif, struct mbuf *m, sa_family_t af, u_int8_t dir, return (0); } + if ((m = pbuf_to_mbuf(pbuf, FALSE)) == NULL) + return (0); + bzero(&hdr, sizeof (hdr)); hdr.length = PFLOG_REAL_HDRLEN; hdr.af = af; diff --git a/bsd/net/if_stf.c b/bsd/net/if_stf.c index 62f662115..ac4950dd1 100644 --- a/bsd/net/if_stf.c +++ b/bsd/net/if_stf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -312,7 +312,7 @@ stfattach(void) struct stf_softc *sc; int error; const struct encaptab *p; - struct ifnet_init_params stf_init; + struct ifnet_init_eparams stf_init; stfinit(); @@ -338,6 +338,9 @@ stfattach(void) lck_mtx_init(&sc->sc_ro_mtx, stf_mtx_grp, LCK_ATTR_NULL); bzero(&stf_init, sizeof(stf_init)); + stf_init.ver = IFNET_INIT_CURRENT_VERSION; + stf_init.len = sizeof (stf_init); + stf_init.flags = IFNET_INIT_LEGACY; stf_init.name = "stf"; stf_init.unit = 0; stf_init.type = IFT_STF; @@ -350,7 +353,7 @@ stfattach(void) stf_init.ioctl = stf_ioctl; stf_init.set_bpf_tap = stf_set_bpf_tap; - error = ifnet_allocate(&stf_init, &sc->sc_if); + error = ifnet_allocate_extended(&stf_init, &sc->sc_if); if (error != 0) { printf("stfattach, ifnet_allocate failed - %d\n", error); encap_detach(sc->encap_cookie); diff --git a/bsd/net/if_utun.c b/bsd/net/if_utun.c index 35868bbd2..5e01509df 100644 --- a/bsd/net/if_utun.c +++ b/bsd/net/if_utun.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -48,11 +48,69 @@ This kernel control will register an interface for every client that connects. #include #include #include +#include #include #include #include - - +#include +#include + +#define UTUN_NEXUS 0 + +extern unsigned int if_enable_netagent; + +#if UTUN_NEXUS +static nexus_controller_t utun_ncd; +static int utun_ncd_refcount; +static uuid_t utun_kpipe_uuid; +static uuid_t utun_nx_dom_prov; + +typedef struct utun_nx { + uuid_t if_provider; + uuid_t if_instance; + uuid_t ms_provider; + uuid_t ms_instance; + uuid_t ms_device; + uuid_t ms_host; + uuid_t ms_agent; +} *utun_nx_t; + +#endif // UTUN_NEXUS + +/* Control block allocated for each kernel control connection */ +struct utun_pcb { + TAILQ_ENTRY(utun_pcb) utun_chain; + kern_ctl_ref utun_ctlref; + ifnet_t utun_ifp; + u_int32_t utun_unit; + u_int32_t utun_unique_id; + u_int32_t utun_flags; + int utun_ext_ifdata_stats; + u_int32_t utun_max_pending_packets; + char utun_if_xname[IFXNAMSIZ]; + char utun_unique_name[IFXNAMSIZ]; + // PCB lock protects state fields and rings + decl_lck_rw_data(, utun_pcb_lock); + struct mbuf * utun_input_chain; + struct mbuf * utun_input_chain_last; + // Input chain lock protects the list of input mbufs + // The input chain lock must be taken AFTER the PCB lock if both are held + lck_mtx_t utun_input_chain_lock; + bool utun_output_disabled; + +#if UTUN_NEXUS + struct utun_nx utun_nx; + int utun_kpipe_enabled; + uuid_t utun_kpipe_uuid; + void * utun_kpipe_rxring; + void * utun_kpipe_txring; + + kern_nexus_t utun_netif_nexus; + void * utun_netif_rxring; + void * utun_netif_txring; + uint64_t utun_netif_txring_size; +#endif // UTUN_NEXUS +}; /* Kernel Control functions */ static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, @@ -69,13 +127,15 @@ static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags); /* Network Interface functions */ +#if !UTUN_NEXUS static void utun_start(ifnet_t interface); +static errno_t utun_framer(ifnet_t interface, mbuf_t *packet, + const struct sockaddr *dest, const char *desk_linkaddr, + const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len); +#endif // !UTUN_NEXUS static errno_t utun_output(ifnet_t interface, mbuf_t data); static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header, protocol_family_t *protocol); -static errno_t utun_framer(ifnet_t interface, mbuf_t *packet, - const struct sockaddr *dest, const char *desk_linkaddr, - const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len); static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol, const struct ifnet_demux_desc *demux_array, u_int32_t demux_count); @@ -92,19 +152,1156 @@ static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protoc char *frame_type, char *link_layer_dest); static errno_t utun_pkt_input (struct utun_pcb *pcb, mbuf_t m); +#if UTUN_NEXUS + +#define UTUN_IF_DEFAULT_SLOT_SIZE 4096 +#define UTUN_IF_DEFAULT_RING_SIZE 64 +#define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64 +#define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128 +#define UTUN_IF_HEADROOM_SIZE 32 + +#define UTUN_IF_MIN_RING_SIZE 16 +#define UTUN_IF_MAX_RING_SIZE 1024 + +static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS; +static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS; +static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS; + +static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE; +static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE; +static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE; + +SYSCTL_DECL(_net_utun); +SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun"); + +SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", ""); +SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", ""); +SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", ""); + +static errno_t +utun_register_nexus(void); + +static errno_t +utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp); +static errno_t +utun_nexus_pre_connect(kern_nexus_provider_t nxprov, + proc_t p, kern_nexus_t nexus, + nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx); +static errno_t +utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel); +static void +utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel); +static void +utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel); +static void +utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel); +static errno_t +utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, + void **ring_ctx); +static void +utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring); +static errno_t +utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring, uint32_t flags); +static errno_t +utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring, uint32_t flags); +#endif // UTUN_NEXUS #define UTUN_DEFAULT_MTU 1500 #define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0)) static kern_ctl_ref utun_kctlref; static u_int32_t utun_family; +static lck_attr_t *utun_lck_attr; +static lck_grp_attr_t *utun_lck_grp_attr; +static lck_grp_t *utun_lck_grp; +static lck_mtx_t utun_lock; + +TAILQ_HEAD(utun_list, utun_pcb) utun_head; + +#define UTUN_PCB_ZONE_MAX 32 +#define UTUN_PCB_ZONE_NAME "net.if_utun" + +static unsigned int utun_pcb_size; /* size of zone element */ +static struct zone *utun_pcb_zone; /* zone for utun_pcb */ + +#if UTUN_NEXUS + +static int +sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_utun_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < UTUN_IF_MIN_RING_SIZE || + value > UTUN_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_utun_ring_size = value; + + return (0); +} + +static int +sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_utun_tx_fsw_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < UTUN_IF_MIN_RING_SIZE || + value > UTUN_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_utun_tx_fsw_ring_size = value; + + return (0); +} + +static int +sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_utun_rx_fsw_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < UTUN_IF_MIN_RING_SIZE || + value > UTUN_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_utun_rx_fsw_ring_size = value; + + return (0); +} + +static errno_t +utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, + void **ring_ctx) +{ +#pragma unused(nxprov) +#pragma unused(channel) +#pragma unused(ring_ctx) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + if (!is_tx_ring) { + VERIFY(pcb->utun_netif_rxring == NULL); + pcb->utun_netif_rxring = ring; + } else { + VERIFY(pcb->utun_netif_txring == NULL); + pcb->utun_netif_txring = ring; + } + return 0; +} + +static void +utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring) +{ +#pragma unused(nxprov) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->utun_netif_rxring == ring) { + pcb->utun_netif_rxring = NULL; + } else if (pcb->utun_netif_txring == ring) { + pcb->utun_netif_txring = NULL; + } +} + +static errno_t +utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t tx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; + + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + + STATS_INC(nifs, NETIF_STATS_TXSYNC); + + if (tx_slot == NULL) { + // Nothing to write, don't bother signalling + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + if (pcb->utun_kpipe_enabled) { + kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring; + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + // Signal the kernel pipe ring to read + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + return 0; + } + + // If we're here, we're injecting into the utun kernel control socket + while (tx_slot != NULL) { + size_t length = 0; + mbuf_t data = NULL; + + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + if (tx_ph == 0) { + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + continue; + } + (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + + /* tx_baddr is the absolute buffer address */ + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != 0); + + bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0); + + uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf); + uint32_t tx_length = kern_buflet_get_data_length(tx_buf); + + // The offset must be large enough for the headers + VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb)); + + // Find family + uint32_t af = 0; + uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset); + u_int ip_version = (vhl >> 4); + switch (ip_version) { + case 4: { + af = AF_INET; + break; + } + case 6: { + af = AF_INET6; + break; + } + default: { + printf("utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n", + pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length, + UTUN_HEADER_SIZE(pcb)); + break; + } + } + + tx_offset -= UTUN_HEADER_SIZE(pcb); + tx_length += UTUN_HEADER_SIZE(pcb); + tx_baddr += tx_offset; + + length = MIN(tx_length, UTUN_IF_DEFAULT_SLOT_SIZE); + + // Copy in family + memcpy(tx_baddr, &af, sizeof(af)); + if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) { + kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af))); + } + + if (length > 0) { + errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); + if (error == 0) { + error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); + if (error == 0) { + error = utun_output(pcb->utun_ifp, data); + if (error != 0) { + printf("utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error); + } + } else { + printf("utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } + } else { + printf("utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + } else { + printf("utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + + kern_pbufpool_free(tx_ring->ckr_pp, tx_ph); + + if (data == NULL) { + continue; + } + + STATS_INC(nifs, NETIF_STATS_TXPKTS); + STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF); + + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + return 0; +} + +static errno_t +utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring, __unused uint32_t flags) +{ +#pragma unused(nxprov) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + boolean_t more = false; + errno_t rc = 0; + do { + /* Refill and sync the ring */ + rc = kern_channel_tx_refill(ring, UINT32_MAX, UINT32_MAX, true, &more); + if (rc != 0 && rc != EAGAIN && rc != EBUSY) { + printf("%s, tx refill failed %d\n", __func__, rc); + } + } while ((rc == 0) && more); + + if (pcb->utun_kpipe_enabled && !pcb->utun_output_disabled) { + uint32_t tx_available = kern_channel_available_slot_count(ring); + if (pcb->utun_netif_txring_size > 0 && + tx_available >= pcb->utun_netif_txring_size - 1) { + // No room left in tx ring, disable output for now + errno_t error = ifnet_disable_output(pcb->utun_ifp); + if (error != 0) { + printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error); + } else { + pcb->utun_output_disabled = true; + } + } + } + if (pcb->utun_kpipe_enabled && + (((rc != 0) && (rc != EAGAIN)) || pcb->utun_output_disabled)) { + kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring; + + // Unlock while calling notify + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + // Signal the kernel pipe ring to read + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + } else { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + } + + return (0); +} + +static errno_t +utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t rx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + struct kern_channel_ring_stat_increment rx_ring_stats; + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; + + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + // Reclaim user-released slots + (void) kern_channel_reclaim(rx_ring); + + STATS_INC(nifs, NETIF_STATS_RXSYNC); + + uint32_t avail = kern_channel_available_slot_count(rx_ring); + if (avail == 0) { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; + VERIFY(rx_pp != NULL); + bzero(&rx_ring_stats, sizeof(rx_ring_stats)); + kern_channel_slot_t rx_pslot = NULL; + kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); + + while (rx_slot != NULL) { + // Check for a waiting packet + lck_mtx_lock(&pcb->utun_input_chain_lock); + mbuf_t data = pcb->utun_input_chain; + if (data == NULL) { + lck_mtx_unlock(&pcb->utun_input_chain_lock); + break; + } + + // Allocate rx packet + kern_packet_t rx_ph = 0; + errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("utun_netif_sync_rx %s: failed to allocate packet\n", + pcb->utun_ifp->if_xname); + lck_mtx_unlock(&pcb->utun_input_chain_lock); + break; + } + + // Advance waiting packets + pcb->utun_input_chain = data->m_nextpkt; + data->m_nextpkt = NULL; + if (pcb->utun_input_chain == NULL) { + pcb->utun_input_chain_last = NULL; + } + lck_mtx_unlock(&pcb->utun_input_chain_lock); + + size_t header_offset = UTUN_HEADER_SIZE(pcb); + size_t length = mbuf_pkthdr_len(data); + + if (length < header_offset) { + // mbuf is too small + mbuf_freem(data); + kern_pbufpool_free(rx_pp, rx_ph); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n", + pcb->utun_ifp->if_xname, length, header_offset); + continue; + } + + length -= header_offset; + if (length > rx_pp->pp_buflet_size) { + // Flush data + mbuf_freem(data); + kern_pbufpool_free(rx_pp, rx_ph); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("utun_netif_sync_rx %s: legacy packet length %zu > %u\n", + pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size); + continue; + } + + mbuf_pkthdr_setrcvif(data, pcb->utun_ifp); + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from mbuf to buflet + mbuf_copydata(data, header_offset, length, (void *)rx_baddr); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_set_link_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_set_network_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_RXPKTS); + STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF); + bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + mbuf_freem(data); + + // Advance ring + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring; + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = NULL; + if (tx_ring == NULL) { + // Net-If TX ring not set up yet, nothing to read + goto done; + } + + // Unlock utun before entering ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + (void)kr_enter(tx_ring, TRUE); + + // Lock again after entering and validate + lck_rw_lock_shared(&pcb->utun_pcb_lock); + if (tx_ring != pcb->utun_kpipe_txring) { + goto done; + } + + tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to read, don't bother signalling + goto done; + } + + while (rx_slot != NULL && tx_slot != NULL) { + // Allocate rx packet + kern_packet_t rx_ph = 0; + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */ + if (tx_ph == 0) { + continue; + } + + errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("utun_netif_sync_rx %s: failed to allocate packet\n", + pcb->utun_ifp->if_xname); + break; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != 0); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + // Check packet length + size_t header_offset = UTUN_HEADER_SIZE(pcb); + uint32_t tx_length = kern_packet_get_data_length(tx_ph); + if (tx_length < header_offset) { + // Packet is too small + kern_pbufpool_free(rx_pp, rx_ph); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("utun_netif_sync_rx %s: packet length too short for header %u < %zu\n", + pcb->utun_ifp->if_xname, tx_length, header_offset); + continue; + } + + size_t length = MIN(tx_length - header_offset, + UTUN_IF_DEFAULT_SLOT_SIZE); + + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from tx to rx + memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_set_link_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_set_network_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_RXPKTS); + STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT); + bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + +done: + if (rx_pslot) { + kern_channel_advance_slot(rx_ring, rx_pslot); + kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats); + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + if (tx_ring != NULL) { + if (tx_pslot != NULL) { + kern_channel_notify(tx_ring, 0); + } + kr_exit(tx_ring); + } + + return 0; +} + +static errno_t +utun_nexus_ifattach(struct utun_pcb *pcb, + struct ifnet_init_eparams *init_params, + struct ifnet **ifp) +{ + errno_t err; + nexus_controller_t controller = kern_nexus_shared_controller(); + struct kern_nexus_net_init net_init; + + nexus_name_t provider_name; + snprintf((char *)provider_name, sizeof(provider_name), + "com.apple.netif.utun%d", pcb->utun_unit); + + struct kern_nexus_provider_init prov_init = { + .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxpi_flags = NXPIF_VIRTUAL_DEVICE, + .nxpi_pre_connect = utun_nexus_pre_connect, + .nxpi_connected = utun_nexus_connected, + .nxpi_pre_disconnect = utun_netif_pre_disconnect, + .nxpi_disconnected = utun_nexus_disconnected, + .nxpi_ring_init = utun_netif_ring_init, + .nxpi_ring_fini = utun_netif_ring_fini, + .nxpi_slot_init = NULL, + .nxpi_slot_fini = NULL, + .nxpi_sync_tx = utun_netif_sync_tx, + .nxpi_sync_rx = utun_netif_sync_rx, + .nxpi_tx_doorbell = utun_netif_tx_doorbell, + }; + + nexus_attr_t nxa = NULL; + err = kern_nexus_attr_create(&nxa); + if (err != 0) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __func__, err); + goto failed; + } + + uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE; + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(err == 0); + + // Reset ring size for netif nexus to limit memory usage + uint64_t ring_size = if_utun_ring_size; + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); + VERIFY(err == 0); + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); + VERIFY(err == 0); + + pcb->utun_netif_txring_size = ring_size; + + err = kern_nexus_controller_register_provider(controller, + utun_nx_dom_prov, + provider_name, + &prov_init, + sizeof(prov_init), + nxa, + &pcb->utun_nx.if_provider); + if (err != 0) { + printf("%s register provider failed, error %d\n", + __func__, err); + goto failed; + } + + bzero(&net_init, sizeof(net_init)); + net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION; + net_init.nxneti_flags = 0; + net_init.nxneti_eparams = init_params; + net_init.nxneti_lladdr = NULL; + net_init.nxneti_prepare = utun_netif_prepare; + err = kern_nexus_controller_alloc_net_provider_instance(controller, + pcb->utun_nx.if_provider, + pcb, + &pcb->utun_nx.if_instance, + &net_init, + ifp); + if (err != 0) { + printf("%s alloc_net_provider_instance failed, %d\n", + __func__, err); + kern_nexus_controller_deregister_provider(controller, + pcb->utun_nx.if_provider); + uuid_clear(pcb->utun_nx.if_provider); + goto failed; + } + +failed: + if (nxa) { + kern_nexus_attr_destroy(nxa); + } + return (err); +} + +static void +utun_detach_provider_and_instance(uuid_t provider, uuid_t instance) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err; + + if (!uuid_is_null(instance)) { + err = kern_nexus_controller_free_provider_instance(controller, + instance); + if (err != 0) { + printf("%s free_provider_instance failed %d\n", + __func__, err); + } + uuid_clear(instance); + } + if (!uuid_is_null(provider)) { + err = kern_nexus_controller_deregister_provider(controller, + provider); + if (err != 0) { + printf("%s deregister_provider %d\n", __func__, err); + } + uuid_clear(provider); + } + return; +} + +static void +utun_nexus_detach(utun_nx_t nx) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err; + + if (!uuid_is_null(nx->ms_host)) { + err = kern_nexus_ifdetach(controller, + nx->ms_instance, + nx->ms_host); + if (err != 0) { + printf("%s: kern_nexus_ifdetach ms host failed %d\n", + __func__, err); + } + } + + if (!uuid_is_null(nx->ms_device)) { + err = kern_nexus_ifdetach(controller, + nx->ms_instance, + nx->ms_device); + if (err != 0) { + printf("%s: kern_nexus_ifdetach ms device failed %d\n", + __func__, err); + } + } + + utun_detach_provider_and_instance(nx->if_provider, + nx->if_instance); + utun_detach_provider_and_instance(nx->ms_provider, + nx->ms_instance); + + memset(nx, 0, sizeof(*nx)); +} + +static errno_t +utun_create_fs_provider_and_instance(uint32_t subtype, const char *type_name, + const char *ifname, + uuid_t *provider, uuid_t *instance) +{ + nexus_attr_t attr = NULL; + nexus_controller_t controller = kern_nexus_shared_controller(); + uuid_t dom_prov; + errno_t err; + struct kern_nexus_init init; + nexus_name_t provider_name; + + err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH, + &dom_prov); + if (err != 0) { + printf("%s can't get %s provider, error %d\n", + __func__, type_name, err); + goto failed; + } + + err = kern_nexus_attr_create(&attr); + if (err != 0) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __func__, err); + goto failed; + } + + err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype); + VERIFY(err == 0); + + uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(err == 0); + + // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif. + uint64_t tx_ring_size = if_utun_tx_fsw_ring_size; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size); + VERIFY(err == 0); + uint64_t rx_ring_size = if_utun_rx_fsw_ring_size; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size); + VERIFY(err == 0); + + snprintf((char *)provider_name, sizeof(provider_name), + "com.apple.%s.%s", type_name, ifname); + err = kern_nexus_controller_register_provider(controller, + dom_prov, + provider_name, + NULL, + 0, + attr, + provider); + kern_nexus_attr_destroy(attr); + attr = NULL; + if (err != 0) { + printf("%s register %s provider failed, error %d\n", + __func__, type_name, err); + goto failed; + } + bzero(&init, sizeof (init)); + init.nxi_version = KERN_NEXUS_CURRENT_VERSION; + err = kern_nexus_controller_alloc_provider_instance(controller, + *provider, + NULL, + instance, &init); + if (err != 0) { + printf("%s alloc_provider_instance %s failed, %d\n", + __func__, type_name, err); + kern_nexus_controller_deregister_provider(controller, + *provider); + uuid_clear(*provider); + } +failed: + return (err); +} + +static errno_t +utun_multistack_attach(struct utun_pcb *pcb) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err = 0; + utun_nx_t nx = &pcb->utun_nx; + + // Allocate multistack flowswitch + err = utun_create_fs_provider_and_instance(NEXUS_EXTENSION_FSW_TYPE_MULTISTACK, + "multistack", + pcb->utun_ifp->if_xname, + &nx->ms_provider, + &nx->ms_instance); + if (err != 0) { + printf("%s: failed to create bridge provider and instance\n", + __func__); + goto failed; + } + + // Attach multistack to device port + err = kern_nexus_ifattach(controller, nx->ms_instance, + NULL, nx->if_instance, + FALSE, &nx->ms_device); + if (err != 0) { + printf("%s kern_nexus_ifattach ms device %d\n", __func__, err); + goto failed; + } + + // Attach multistack to host port + err = kern_nexus_ifattach(controller, nx->ms_instance, + NULL, nx->if_instance, + TRUE, &nx->ms_host); + if (err != 0) { + printf("%s kern_nexus_ifattach ms host %d\n", __func__, err); + goto failed; + } + + // Extract the agent UUID and save for later + struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false); + if (multistack_nx != NULL) { + struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx); + if (flowswitch != NULL) { + FSW_RLOCK(flowswitch); + struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private; + if (ms_context != NULL) { + uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid); + } else { + printf("utun_multistack_attach - fsw_ms_context is NULL\n"); + } + FSW_UNLOCK(flowswitch); + } else { + printf("utun_multistack_attach - flowswitch is NULL\n"); + } + nx_release(multistack_nx); + } else { + printf("utun_multistack_attach - unable to find multistack nexus\n"); + } + + return (0); + +failed: + utun_nexus_detach(nx); + + errno_t detach_error = 0; + if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) { + panic("utun_multistack_attach - ifnet_detach failed: %d\n", detach_error); + /* NOT REACHED */ + } + + return (err); +} + +static errno_t +utun_register_kernel_pipe_nexus(void) +{ + nexus_attr_t nxa = NULL; + errno_t result; + + lck_mtx_lock(&utun_lock); + if (utun_ncd_refcount++) { + lck_mtx_unlock(&utun_lock); + return 0; + } + + result = kern_nexus_controller_create(&utun_ncd); + if (result) { + printf("%s: kern_nexus_controller_create failed: %d\n", + __FUNCTION__, result); + goto done; + } + + uuid_t dom_prov; + result = kern_nexus_get_builtin_domain_provider( + NEXUS_TYPE_KERNEL_PIPE, &dom_prov); + if (result) { + printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n", + __FUNCTION__, result); + goto done; + } + + struct kern_nexus_provider_init prov_init = { + .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxpi_flags = NXPIF_VIRTUAL_DEVICE, + .nxpi_pre_connect = utun_nexus_pre_connect, + .nxpi_connected = utun_nexus_connected, + .nxpi_pre_disconnect = utun_nexus_pre_disconnect, + .nxpi_disconnected = utun_nexus_disconnected, + .nxpi_ring_init = utun_kpipe_ring_init, + .nxpi_ring_fini = utun_kpipe_ring_fini, + .nxpi_slot_init = NULL, + .nxpi_slot_fini = NULL, + .nxpi_sync_tx = utun_kpipe_sync_tx, + .nxpi_sync_rx = utun_kpipe_sync_rx, + .nxpi_tx_doorbell = NULL, + }; + + result = kern_nexus_attr_create(&nxa); + if (result) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __FUNCTION__, result); + goto done; + } + + uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE; + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(result == 0); + + // Reset ring size for kernel pipe nexus to limit memory usage + uint64_t ring_size = if_utun_ring_size; + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); + VERIFY(result == 0); + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); + VERIFY(result == 0); + + result = kern_nexus_controller_register_provider(utun_ncd, + dom_prov, + (const uint8_t *)"com.apple.nexus.utun.kpipe", + &prov_init, + sizeof(prov_init), + nxa, + &utun_kpipe_uuid); + if (result) { + printf("%s: kern_nexus_controller_register_provider failed: %d\n", + __FUNCTION__, result); + goto done; + } + +done: + if (nxa) { + kern_nexus_attr_destroy(nxa); + } + + if (result) { + if (utun_ncd) { + kern_nexus_controller_destroy(utun_ncd); + utun_ncd = NULL; + } + utun_ncd_refcount = 0; + } + + lck_mtx_unlock(&utun_lock); + + return result; +} + +static void +utun_unregister_kernel_pipe_nexus(void) +{ + lck_mtx_lock(&utun_lock); + + VERIFY(utun_ncd_refcount > 0); + + if (--utun_ncd_refcount == 0) { + kern_nexus_controller_destroy(utun_ncd); + utun_ncd = NULL; + } + + lck_mtx_unlock(&utun_lock); +} + +// For use by socket option, not internally +static errno_t +utun_disable_channel(struct utun_pcb *pcb) +{ + errno_t result; + int enabled; + uuid_t uuid; + + lck_rw_lock_exclusive(&pcb->utun_pcb_lock); + + enabled = pcb->utun_kpipe_enabled; + uuid_copy(uuid, pcb->utun_kpipe_uuid); + + VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled); + + pcb->utun_kpipe_enabled = 0; + uuid_clear(pcb->utun_kpipe_uuid); + + lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); + + if (enabled) { + result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid); + } else { + result = ENXIO; + } + + if (!result) { + utun_unregister_kernel_pipe_nexus(); + } + + return result; +} + +static errno_t +utun_enable_channel(struct utun_pcb *pcb, struct proc *proc) +{ + struct kern_nexus_init init; + errno_t result; + + result = utun_register_kernel_pipe_nexus(); + if (result) { + return result; + } + + VERIFY(utun_ncd); + + lck_rw_lock_exclusive(&pcb->utun_pcb_lock); + + if (pcb->utun_kpipe_enabled) { + result = EEXIST; // return success instead? + goto done; + } + + /* + * Make sure we can fit packets in the channel buffers and + * Allow an extra 4 bytes for the protocol number header in the channel + */ + if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > UTUN_IF_DEFAULT_SLOT_SIZE) { + result = EOPNOTSUPP; + goto done; + } + + VERIFY(uuid_is_null(pcb->utun_kpipe_uuid)); + bzero(&init, sizeof (init)); + init.nxi_version = KERN_NEXUS_CURRENT_VERSION; + result = kern_nexus_controller_alloc_provider_instance(utun_ncd, + utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init); + if (result) { + goto done; + } + + nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT; + result = kern_nexus_controller_bind_provider_instance(utun_ncd, + pcb->utun_kpipe_uuid, &port, + proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID); + if (result) { + kern_nexus_controller_free_provider_instance(utun_ncd, + pcb->utun_kpipe_uuid); + uuid_clear(pcb->utun_kpipe_uuid); + goto done; + } + + pcb->utun_kpipe_enabled = 1; + +done: + lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); + + if (result) { + utun_unregister_kernel_pipe_nexus(); + } + + return result; +} + +#endif // UTUN_NEXUS errno_t utun_register_control(void) { - struct kern_ctl_reg kern_ctl; - errno_t result = 0; + struct kern_ctl_reg kern_ctl; + errno_t result = 0; /* Find a unique value for our interface family */ result = mbuf_tag_id_find(UTUN_CONTROL_NAME, &utun_family); @@ -112,6 +1309,21 @@ utun_register_control(void) printf("utun_register_control - mbuf_tag_id_find_internal failed: %d\n", result); return result; } + + utun_pcb_size = sizeof(struct utun_pcb); + utun_pcb_zone = zinit(utun_pcb_size, + UTUN_PCB_ZONE_MAX * utun_pcb_size, + 0, UTUN_PCB_ZONE_NAME); + if (utun_pcb_zone == NULL) { + printf("utun_register_control - zinit(utun_pcb) failed"); + return ENOMEM; + } + +#if UTUN_NEXUS + utun_register_nexus(); +#endif // UTUN_NEXUS + + TAILQ_INIT(&utun_head); bzero(&kern_ctl, sizeof(kern_ctl)); strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name)); @@ -151,51 +1363,137 @@ utun_register_control(void) return result; } - + utun_lck_attr = lck_attr_alloc_init(); + utun_lck_grp_attr = lck_grp_attr_alloc_init(); + utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr); + +#if UTUN_NEXUS + lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr); +#endif // UTUN_NEXUS + return 0; } /* Kernel control functions */ +static inline void +utun_free_pcb(struct utun_pcb *pcb) +{ +#ifdef UTUN_NEXUS + mbuf_freem_list(pcb->utun_input_chain); + lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp); +#endif // UTUN_NEXUS + lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp); + lck_mtx_lock(&utun_lock); + TAILQ_REMOVE(&utun_head, pcb, utun_chain); + lck_mtx_unlock(&utun_lock); + zfree(utun_pcb_zone, pcb); +} + static errno_t -utun_ctl_connect( - kern_ctl_ref kctlref, - struct sockaddr_ctl *sac, - void **unitinfo) +utun_ctl_connect(kern_ctl_ref kctlref, + struct sockaddr_ctl *sac, + void **unitinfo) { - struct ifnet_init_eparams utun_init; - struct utun_pcb *pcb; - errno_t result; - struct ifnet_stats_param stats; + struct ifnet_init_eparams utun_init = {}; + errno_t result = 0; - /* kernel control allocates, interface frees */ - MALLOC(pcb, struct utun_pcb *, sizeof(*pcb), M_DEVBUF, M_WAITOK | M_ZERO); + struct utun_pcb *pcb = zalloc(utun_pcb_zone); + memset(pcb, 0, sizeof(*pcb)); *unitinfo = pcb; pcb->utun_ctlref = kctlref; pcb->utun_unit = sac->sc_unit; pcb->utun_max_pending_packets = 1; - - printf("utun_ctl_connect: creating interface utun%d\n", pcb->utun_unit - 1); + + lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr); + lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr); + + lck_mtx_lock(&utun_lock); + + /* Find some open interface id */ + u_int32_t chosen_unique_id = 1; + struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list); + if (next_pcb != NULL) { + /* List was not empty, add one to the last item */ + chosen_unique_id = next_pcb->utun_unique_id + 1; + next_pcb = NULL; + + /* + * If this wrapped the id number, start looking at + * the front of the list for an unused id. + */ + if (chosen_unique_id == 0) { + /* Find the next unused ID */ + chosen_unique_id = 1; + TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) { + if (next_pcb->utun_unique_id > chosen_unique_id) { + /* We found a gap */ + break; + } + + chosen_unique_id = next_pcb->utun_unique_id + 1; + } + } + } + + pcb->utun_unique_id = chosen_unique_id; + + if (next_pcb != NULL) { + TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain); + } else { + TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain); + } + lck_mtx_unlock(&utun_lock); + + snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1); + snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1); + printf("utun_ctl_connect: creating interface %s (id %s)\n", pcb->utun_if_xname, pcb->utun_unique_name); /* Create the interface */ bzero(&utun_init, sizeof(utun_init)); utun_init.ver = IFNET_INIT_CURRENT_VERSION; utun_init.len = sizeof (utun_init); - utun_init.name = "utun"; + +#if UTUN_NEXUS + utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO); + utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE; +#else // UTUN_NEXUS + utun_init.flags = IFNET_INIT_NX_NOAUTO; utun_init.start = utun_start; + utun_init.framer_extended = utun_framer; +#endif // UTUN_NEXUS + utun_init.name = "utun"; utun_init.unit = pcb->utun_unit - 1; + utun_init.uniqueid = pcb->utun_unique_name; + utun_init.uniqueid_len = strlen(pcb->utun_unique_name); utun_init.family = utun_family; utun_init.subfamily = IFNET_SUBFAMILY_UTUN; utun_init.type = IFT_OTHER; utun_init.demux = utun_demux; - utun_init.framer_extended = utun_framer; utun_init.add_proto = utun_add_proto; utun_init.del_proto = utun_del_proto; utun_init.softc = pcb; utun_init.ioctl = utun_ioctl; utun_init.detach = utun_detached; +#if UTUN_NEXUS + result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp); + if (result != 0) { + printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result); + utun_free_pcb(pcb); + *unitinfo = NULL; + return result; + } + + result = utun_multistack_attach(pcb); + if (result != 0) { + printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result); + *unitinfo = NULL; + return result; + } + +#else // UTUN_NEXUS /* * Upon success, this holds an ifnet reference which we will * release via ifnet_release() at final detach time. @@ -203,8 +1501,8 @@ utun_ctl_connect( result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp); if (result != 0) { printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result); + utun_free_pcb(pcb); *unitinfo = NULL; - FREE(pcb, M_DEVBUF); return result; } @@ -218,31 +1516,34 @@ utun_ctl_connect( ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); /* Reset the stats in case as the interface may have been recycled */ + struct ifnet_stats_param stats; bzero(&stats, sizeof(struct ifnet_stats_param)); ifnet_set_stat(pcb->utun_ifp, &stats); /* Attach the interface */ result = ifnet_attach(pcb->utun_ifp, NULL); if (result != 0) { - printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result); + printf("utun_ctl_connect - ifnet_attach failed: %d\n", result); /* Release reference now since attach failed */ ifnet_release(pcb->utun_ifp); + utun_free_pcb(pcb); *unitinfo = NULL; - FREE(pcb, M_DEVBUF); - } else { - /* Attach to bpf */ - bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb)); - /* The interfaces resoures allocated, mark it as running */ - ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING); + return (result); } +#endif // UTUN_NEXUS + + /* Attach to bpf */ + bpfattach(pcb->utun_ifp, DLT_RAW, 0); + /* The interfaces resoures allocated, mark it as running */ + ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING); + return result; } static errno_t -utun_detach_ip( - ifnet_t interface, - protocol_family_t protocol, - socket_t pf_socket) +utun_detach_ip(ifnet_t interface, + protocol_family_t protocol, + socket_t pf_socket) { errno_t result = EPROTONOSUPPORT; @@ -255,8 +1556,7 @@ utun_detach_ip( ifnet_name(interface), ifnet_unit(interface)); result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr); - } - else if (protocol == PF_INET6) { + } else if (protocol == PF_INET6) { struct in6_ifreq ifr6; bzero(&ifr6, sizeof(ifr6)); @@ -270,17 +1570,16 @@ utun_detach_ip( } static void -utun_remove_address( - ifnet_t interface, - protocol_family_t protocol, - ifaddr_t address, - socket_t pf_socket) +utun_remove_address(ifnet_t interface, + protocol_family_t protocol, + ifaddr_t address, + socket_t pf_socket) { errno_t result = 0; /* Attempt a detach */ if (protocol == PF_INET) { - struct ifreq ifr; + struct ifreq ifr; bzero(&ifr, sizeof(ifr)); snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d", @@ -288,16 +1587,14 @@ utun_remove_address( result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr)); if (result != 0) { printf("utun_remove_address - ifaddr_address failed: %d", result); - } - else { + } else { result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr); if (result != 0) { printf("utun_remove_address - SIOCDIFADDR failed: %d", result); } } - } - else if (protocol == PF_INET6) { - struct in6_ifreq ifr6; + } else if (protocol == PF_INET6) { + struct in6_ifreq ifr6; bzero(&ifr6, sizeof(ifr6)); snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d", @@ -307,8 +1604,7 @@ utun_remove_address( if (result != 0) { printf("utun_remove_address - ifaddr_address failed (v6): %d", result); - } - else { + } else { result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6); if (result != 0) { printf("utun_remove_address - SIOCDIFADDR_IN6 failed: %d", @@ -319,14 +1615,13 @@ utun_remove_address( } static void -utun_cleanup_family( - ifnet_t interface, - protocol_family_t protocol) -{ - errno_t result = 0; - socket_t pf_socket = NULL; - ifaddr_t *addresses = NULL; - int i; +utun_cleanup_family(ifnet_t interface, + protocol_family_t protocol) +{ + errno_t result = 0; + socket_t pf_socket = NULL; + ifaddr_t *addresses = NULL; + int i; if (protocol != PF_INET && protocol != PF_INET6) { printf("utun_cleanup_family - invalid protocol family %d\n", protocol); @@ -349,8 +1644,7 @@ utun_cleanup_family( if (result == 0 || result == ENXIO) { /* We are done! We either detached or weren't attached. */ goto cleanup; - } - else if (result != EBUSY) { + } else if (result != EBUSY) { /* Uh, not really sure what happened here... */ printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result); goto cleanup; @@ -383,31 +1677,58 @@ utun_cleanup_family( } cleanup: - if (pf_socket != NULL) + if (pf_socket != NULL) { sock_close(pf_socket); + } - if (addresses != NULL) + if (addresses != NULL) { ifnet_free_address_list(addresses); + } } static errno_t -utun_ctl_disconnect( - __unused kern_ctl_ref kctlref, - __unused u_int32_t unit, - void *unitinfo) +utun_ctl_disconnect(__unused kern_ctl_ref kctlref, + __unused u_int32_t unit, + void *unitinfo) { struct utun_pcb *pcb = unitinfo; - ifnet_t ifp = NULL; - errno_t result = 0; + ifnet_t ifp = NULL; + errno_t result = 0; + + if (pcb == NULL) { + return EINVAL; + } + +#if UTUN_NEXUS + // Tell the nexus to stop all rings + if (pcb->utun_netif_nexus != NULL) { + kern_nexus_stop(pcb->utun_netif_nexus); + } +#endif // UTUN_NEXUS - if (pcb == NULL) - return EINVAL; + lck_rw_lock_exclusive(&pcb->utun_pcb_lock); +#if UTUN_NEXUS + uuid_t kpipe_uuid; + uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid); + uuid_clear(pcb->utun_kpipe_uuid); + pcb->utun_kpipe_enabled = FALSE; +#endif // UTUN_NEXUS ifp = pcb->utun_ifp; VERIFY(ifp != NULL); pcb->utun_ctlref = NULL; - pcb->utun_unit = 0; + + /* + * Quiesce the interface and flush any pending outbound packets. + */ + if_down(ifp); + + /* Increment refcnt, but detach interface */ + ifnet_incr_iorefcnt(ifp); + if ((result = ifnet_detach(ifp)) != 0) { + panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result); + } /* * We want to do everything in our power to ensure that the interface @@ -418,25 +1739,29 @@ utun_ctl_disconnect( utun_cleanup_family(ifp, AF_INET); utun_cleanup_family(ifp, AF_INET6); - /* - * Detach now; utun_detach() will be called asynchronously once - * the I/O reference count drops to 0. There we will invoke - * ifnet_release(). - */ - if ((result = ifnet_detach(ifp)) != 0) { - printf("utun_ctl_disconnect - ifnet_detach failed: %d\n", result); + lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); + +#if UTUN_NEXUS + if (!uuid_is_null(kpipe_uuid)) { + if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) { + utun_unregister_kernel_pipe_nexus(); + } } + utun_nexus_detach(&pcb->utun_nx); +#endif // UTUN_NEXUS + + /* Decrement refcnt to finish detaching and freeing */ + ifnet_decr_iorefcnt(ifp); return 0; } static errno_t -utun_ctl_send( - __unused kern_ctl_ref kctlref, - __unused u_int32_t unit, - void *unitinfo, - mbuf_t m, - __unused int flags) +utun_ctl_send(__unused kern_ctl_ref kctlref, + __unused u_int32_t unit, + void *unitinfo, + mbuf_t m, + __unused int flags) { /* * The userland ABI requires the first four bytes have the protocol family @@ -452,16 +1777,15 @@ utun_ctl_send( } static errno_t -utun_ctl_setopt( - __unused kern_ctl_ref kctlref, - __unused u_int32_t unit, - void *unitinfo, - int opt, - void *data, - size_t len) +utun_ctl_setopt(__unused kern_ctl_ref kctlref, + __unused u_int32_t unit, + void *unitinfo, + int opt, + void *data, + size_t len) { - struct utun_pcb *pcb = unitinfo; - errno_t result = 0; + struct utun_pcb *pcb = unitinfo; + errno_t result = 0; /* check for privileges for privileged options */ switch (opt) { case UTUN_OPT_FLAGS: @@ -478,14 +1802,7 @@ utun_ctl_setopt( if (len != sizeof(u_int32_t)) { result = EMSGSIZE; } else { - u_int32_t old_flags = pcb->utun_flags; pcb->utun_flags = *(u_int32_t *)data; - - if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) { - // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf - bpfdetach(pcb->utun_ifp); - bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb)); - } } break; @@ -551,6 +1868,41 @@ utun_ctl_setopt( pcb->utun_max_pending_packets = max_pending_packets; break; } +#if UTUN_NEXUS + case UTUN_OPT_ENABLE_CHANNEL: { + if (len != sizeof(int)) { + result = EMSGSIZE; + break; + } + if (*(int *)data) { + result = utun_enable_channel(pcb, current_proc()); + } else { + result = utun_disable_channel(pcb); + } + break; + } + case UTUN_OPT_ENABLE_FLOWSWITCH: { + if (len != sizeof(int)) { + result = EMSGSIZE; + break; + } + if (!if_enable_netagent) { + result = ENOTSUP; + break; + } + if (uuid_is_null(pcb->utun_nx.ms_agent)) { + result = ENOENT; + break; + } + + if (*(int *)data) { + if_add_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent); + } else { + if_delete_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent); + } + break; + } +#endif // UTUN_NEXUS default: { result = ENOPROTOOPT; break; @@ -561,42 +1913,63 @@ utun_ctl_setopt( } static errno_t -utun_ctl_getopt( - __unused kern_ctl_ref kctlref, - __unused u_int32_t unit, - void *unitinfo, - int opt, - void *data, - size_t *len) +utun_ctl_getopt(__unused kern_ctl_ref kctlref, + __unused u_int32_t unit, + void *unitinfo, + int opt, + void *data, + size_t *len) { - struct utun_pcb *pcb = unitinfo; - errno_t result = 0; + struct utun_pcb *pcb = unitinfo; + errno_t result = 0; switch (opt) { case UTUN_OPT_FLAGS: - if (*len != sizeof(u_int32_t)) + if (*len != sizeof(u_int32_t)) { result = EMSGSIZE; - else + } else { *(u_int32_t *)data = pcb->utun_flags; + } break; case UTUN_OPT_EXT_IFDATA_STATS: - if (*len != sizeof(int)) + if (*len != sizeof(int)) { result = EMSGSIZE; - else + } else { *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0; + } break; case UTUN_OPT_IFNAME: - *len = snprintf(data, *len, "%s%d", ifnet_name(pcb->utun_ifp), ifnet_unit(pcb->utun_ifp)) + 1; + if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) { + result = EMSGSIZE; + } else { + *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1; + } break; case UTUN_OPT_MAX_PENDING_PACKETS: { - *len = sizeof(u_int32_t); - *((u_int32_t *)data) = pcb->utun_max_pending_packets; + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *((u_int32_t *)data) = pcb->utun_max_pending_packets; + } break; } +#if UTUN_NEXUS + case UTUN_OPT_GET_CHANNEL_UUID: + lck_rw_lock_shared(&pcb->utun_pcb_lock); + if (uuid_is_null(pcb->utun_kpipe_uuid)) { + result = ENXIO; + } else if (*len != sizeof(uuid_t)) { + result = EMSGSIZE; + } else { + uuid_copy(data, pcb->utun_kpipe_uuid); + } + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + break; +#endif // UTUN_NEXUS default: result = ENOPROTOOPT; @@ -638,6 +2011,7 @@ utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags) } /* Network Interface functions */ +#if !UTUN_NEXUS static void utun_start(ifnet_t interface) { @@ -646,6 +2020,20 @@ utun_start(ifnet_t interface) VERIFY(pcb != NULL); +#if UTUN_NEXUS + lck_rw_lock_shared(&pcb->utun_pcb_lock); + if (pcb->utun_kpipe_enabled) { + /* It's possible to have channels enabled, but not yet have the channel opened, + * in which case the rxring will not be set + */ + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + if (pcb->utun_kpipe_rxring != NULL) { + kern_channel_notify(pcb->utun_kpipe_rxring, 0); + } + return; + } + lck_rw_unlock_shared(&pcb->utun_pcb_lock); +#endif // UTUN_NEXUS for (;;) { bool can_accept_packets = true; @@ -678,26 +2066,25 @@ utun_start(ifnet_t interface) break; } ifnet_lock_done(pcb->utun_ifp); - if (ifnet_dequeue(interface, &data) != 0) + if (ifnet_dequeue(interface, &data) != 0) { break; - if (utun_output(interface, data) != 0) + } + if (utun_output(interface, data) != 0) { break; + } } } +#endif // !UTUN_NEXUS static errno_t utun_output(ifnet_t interface, mbuf_t data) { struct utun_pcb *pcb = ifnet_softc(interface); - errno_t result; + errno_t result; VERIFY(interface == pcb->utun_ifp); - - if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) { - bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0); - } - + if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) { /* flush data */ mbuf_freem(data); @@ -720,47 +2107,64 @@ utun_output(ifnet_t interface, if (result != 0) { mbuf_freem(data); printf("utun_output - ctl_enqueuembuf failed: %d\n", result); - +#if !UTUN_NEXUS ifnet_stat_increment_out(interface, 0, 0, 1); - } - else { - if (!pcb->utun_ext_ifdata_stats) + } else { + if (!pcb->utun_ext_ifdata_stats) { ifnet_stat_increment_out(interface, 1, length, 0); + } +#endif // !UTUN_NEXUS } - } - else + } else { mbuf_freem(data); + } return 0; } static errno_t -utun_demux( - __unused ifnet_t interface, - mbuf_t data, - __unused char *frame_header, - protocol_family_t *protocol) +utun_demux(__unused ifnet_t interface, + mbuf_t data, + __unused char *frame_header, + protocol_family_t *protocol) { + struct ip *ip; + u_int ip_version; + while (data != NULL && mbuf_len(data) < 1) { data = mbuf_next(data); } - + if (data == NULL) return ENOENT; - - *protocol = *(u_int32_t *)mbuf_data(data); + + ip = mtod(data, struct ip *); + ip_version = ip->ip_v; + + switch(ip_version) { + case 4: + *protocol = PF_INET; + return 0; + case 6: + *protocol = PF_INET6; + return 0; + default: + *protocol = 0; + break; + } + return 0; } +#if !UTUN_NEXUS static errno_t -utun_framer( - __unused ifnet_t interface, - mbuf_t *packet, - __unused const struct sockaddr *dest, +utun_framer(ifnet_t interface, + mbuf_t *packet, + __unused const struct sockaddr *dest, __unused const char *desk_linkaddr, const char *frame_type, - u_int32_t *prepend_len, + u_int32_t *prepend_len, u_int32_t *postpend_len) { struct utun_pcb *pcb = ifnet_softc(interface); @@ -775,10 +2179,12 @@ utun_framer( // just return, because the buffer was freed in mbuf_prepend return EJUSTRETURN; } - if (prepend_len != NULL) + if (prepend_len != NULL) { *prepend_len = header_length; - if (postpend_len != NULL) + } + if (postpend_len != NULL) { *postpend_len = 0; + } // place protocol number at the beginning of the mbuf *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type; @@ -786,13 +2192,13 @@ utun_framer( return 0; } +#endif // !UTUN_NEXUS static errno_t -utun_add_proto( - __unused ifnet_t interface, - protocol_family_t protocol, - __unused const struct ifnet_demux_desc *demux_array, - __unused u_int32_t demux_count) +utun_add_proto(__unused ifnet_t interface, + protocol_family_t protocol, + __unused const struct ifnet_demux_desc *demux_array, + __unused u_int32_t demux_count) { switch(protocol) { case PF_INET: @@ -807,23 +2213,31 @@ utun_add_proto( } static errno_t -utun_del_proto( - __unused ifnet_t interface, - __unused protocol_family_t protocol) +utun_del_proto(__unused ifnet_t interface, + __unused protocol_family_t protocol) { return 0; } static errno_t -utun_ioctl( - ifnet_t interface, - u_long command, - void *data) +utun_ioctl(ifnet_t interface, + u_long command, + void *data) { errno_t result = 0; switch(command) { case SIOCSIFMTU: +#if UTUN_NEXUS + { + // Make sure we can fit packets in the channel buffers + // Allow for the headroom in the slot + if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > UTUN_IF_DEFAULT_SLOT_SIZE) { + ifnet_set_mtu(interface, UTUN_IF_DEFAULT_SLOT_SIZE - UTUN_IF_HEADROOM_SIZE); + break; + } + } +#endif // UTUN_NEXUS ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu); break; @@ -839,64 +2253,57 @@ utun_ioctl( } static void -utun_detached( - ifnet_t interface) +utun_detached(ifnet_t interface) { struct utun_pcb *pcb = ifnet_softc(interface); - - FREE(pcb, M_DEVBUF); - /* Release reference acquired via ifnet_allocate_extended() */ - (void) ifnet_release(interface); + (void)ifnet_release(interface); + utun_free_pcb(pcb); } /* Protocol Handlers */ static errno_t -utun_proto_input( - ifnet_t interface, - protocol_family_t protocol, - mbuf_t m, - __unused char *frame_header) +utun_proto_input(__unused ifnet_t interface, + protocol_family_t protocol, + mbuf_t m, + __unused char *frame_header) { - - // remove protocol family first - struct utun_pcb *pcb = ifnet_softc(interface); - mbuf_adj(m, UTUN_HEADER_SIZE(pcb)); - if (proto_input(protocol, m) != 0) { m_freem(m); +#if !UTUN_NEXUS + ifnet_stat_increment_in(interface, 0, 0, 1); + } else { + ifnet_stat_increment_in(interface, 1, m->m_pkthdr.len, 0); +#endif // UTUN_NEXUS } return 0; } static errno_t -utun_proto_pre_output( - __unused ifnet_t interface, - protocol_family_t protocol, - __unused mbuf_t *packet, - __unused const struct sockaddr *dest, - __unused void *route, - char *frame_type, - __unused char *link_layer_dest) +utun_proto_pre_output(__unused ifnet_t interface, + protocol_family_t protocol, + __unused mbuf_t *packet, + __unused const struct sockaddr *dest, + __unused void *route, + char *frame_type, + __unused char *link_layer_dest) { *(protocol_family_t *)(void *)frame_type = protocol; return 0; } static errno_t -utun_attach_proto( - ifnet_t interface, - protocol_family_t protocol) +utun_attach_proto(ifnet_t interface, + protocol_family_t protocol) { struct ifnet_attach_proto_param proto; - errno_t result; bzero(&proto, sizeof(proto)); proto.input = utun_proto_input; proto.pre_output = utun_proto_pre_output; - result = ifnet_attach_protocol(interface, protocol, &proto); + errno_t result = ifnet_attach_protocol(interface, protocol, &proto); if (result != 0 && result != EEXIST) { printf("utun_attach_inet - ifnet_attach_protocol %d failed: %d\n", protocol, result); @@ -905,6 +2312,35 @@ utun_attach_proto( return result; } +#if UTUN_NEXUS +static errno_t +utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet) +{ + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + lck_mtx_lock(&pcb->utun_input_chain_lock); + if (pcb->utun_input_chain != NULL) { + pcb->utun_input_chain_last->m_nextpkt = packet; + } else { + pcb->utun_input_chain = packet; + } + while (packet->m_nextpkt) { + VERIFY(packet != packet->m_nextpkt); + packet = packet->m_nextpkt; + } + pcb->utun_input_chain_last = packet; + lck_mtx_unlock(&pcb->utun_input_chain_lock); + + kern_channel_ring_t rx_ring = pcb->utun_netif_rxring; + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + + return (0); +} +#else static errno_t utun_pkt_input (struct utun_pcb *pcb, mbuf_t m) { @@ -915,7 +2351,7 @@ utun_pkt_input (struct utun_pcb *pcb, mbuf_t m) if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE(pcb)) { protocol = *(u_int32_t *)mbuf_data(m); - + bpf_tap_in(pcb->utun_ifp, DLT_NULL, m, 0, 0); } if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) { @@ -926,7 +2362,7 @@ utun_pkt_input (struct utun_pcb *pcb, mbuf_t m) if (!pcb->utun_ext_ifdata_stats) { struct ifnet_stat_increment_param incs; - + bzero(&incs, sizeof(incs)); incs.packets_in = 1; incs.bytes_in = mbuf_pkthdr_len(m); @@ -936,18 +2372,400 @@ utun_pkt_input (struct utun_pcb *pcb, mbuf_t m) } if (result != 0) { ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1); - + printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result); mbuf_freem(m); } return 0; } +#endif // UTUN_NEXUS + + +#if UTUN_NEXUS + +static errno_t +utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov) +{ + return 0; +} + +static void +utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov) +{ + // Ignore +} + +static errno_t +utun_register_nexus(void) +{ + const struct kern_nexus_domain_provider_init dp_init = { + .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxdpi_flags = 0, + .nxdpi_init = utun_nxdp_init, + .nxdpi_fini = utun_nxdp_fini + }; + errno_t err = 0; + + /* utun_nxdp_init() is called before this function returns */ + err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF, + (const uint8_t *) "com.apple.utun", + &dp_init, sizeof(dp_init), + &utun_nx_dom_prov); + if (err != 0) { + printf("%s: failed to register domain provider\n", __func__); + return (err); + } + return (0); +} + +static errno_t +utun_ifnet_set_attrs(ifnet_t ifp) +{ + /* Set flags and additional information. */ + ifnet_set_mtu(ifp, 1500); + ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); + + /* The interface must generate its own IPv6 LinkLocal address, + * if possible following the recommendation of RFC2472 to the 64bit interface ID + */ + ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); + + return (0); +} + +static errno_t +utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp) +{ + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + pcb->utun_netif_nexus = nexus; + return (utun_ifnet_set_attrs(ifp)); +} + +static errno_t +utun_nexus_pre_connect(kern_nexus_provider_t nxprov, + proc_t p, kern_nexus_t nexus, + nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx) +{ +#pragma unused(nxprov, p) +#pragma unused(nexus, nexus_port, channel, ch_ctx) + return (0); +} + +static errno_t +utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, channel) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1); + return (ok ? 0 : ENXIO); +} + +static void +utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, nexus, channel) +} + +static void +utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, nexus, channel) +} + +static void +utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, channel) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->utun_netif_nexus == nexus) { + pcb->utun_netif_nexus = NULL; + } + ifnet_decr_iorefcnt(pcb->utun_ifp); +} + +static errno_t +utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel, kern_channel_ring_t ring, + boolean_t is_tx_ring, void **ring_ctx) +{ +#pragma unused(nxprov) +#pragma unused(channel) +#pragma unused(ring_ctx) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + if (!is_tx_ring) { + VERIFY(pcb->utun_kpipe_rxring == NULL); + pcb->utun_kpipe_rxring = ring; + } else { + VERIFY(pcb->utun_kpipe_txring == NULL); + pcb->utun_kpipe_txring = ring; + } + return 0; +} + +static void +utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring) +{ +#pragma unused(nxprov) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->utun_kpipe_rxring == ring) { + pcb->utun_kpipe_rxring = NULL; + } else if (pcb->utun_kpipe_txring == ring) { + pcb->utun_kpipe_txring = NULL; + } +} + +static errno_t +utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t tx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + + lck_rw_lock_shared(&pcb->utun_pcb_lock); + int channel_enabled = pcb->utun_kpipe_enabled; + if (!channel_enabled) { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to write, bail + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + // Signal the netif ring to read + kern_channel_ring_t rx_ring = pcb->utun_netif_rxring; + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + + return 0; +} + +static errno_t +utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t rx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + struct kern_channel_ring_stat_increment rx_ring_stats; + + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + int channel_enabled = pcb->utun_kpipe_enabled; + if (!channel_enabled) { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + /* reclaim user-released slots */ + (void) kern_channel_reclaim(rx_ring); + + uint32_t avail = kern_channel_available_slot_count(rx_ring); + if (avail == 0) { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + kern_channel_ring_t tx_ring = pcb->utun_netif_txring; + if (tx_ring == NULL || + pcb->utun_netif_nexus == NULL) { + // Net-If TX ring not set up yet, nothing to read + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats; + + // Unlock utun before entering ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + (void)kr_enter(tx_ring, TRUE); + + // Lock again after entering and validate + lck_rw_lock_shared(&pcb->utun_pcb_lock); + if (tx_ring != pcb->utun_netif_txring) { + // Ring no longer valid + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + kr_exit(tx_ring); + return 0; + } + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to read, don't bother signalling + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + kr_exit(tx_ring); + return 0; + } + + struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; + VERIFY(rx_pp != NULL); + bzero(&rx_ring_stats, sizeof(rx_ring_stats)); + kern_channel_slot_t rx_pslot = NULL; + kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); + + while (rx_slot != NULL && tx_slot != NULL) { + size_t length; + kern_buflet_t rx_buf; + void *rx_baddr; + + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */ + if (tx_ph == 0) { + continue; + } + + // Allocate rx packet + kern_packet_t rx_ph = 0; + errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + printf("utun_kpipe_sync_rx %s: failed to allocate packet\n", + pcb->utun_ifp->if_xname); + break; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != NULL); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0); + + length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb), + UTUN_IF_DEFAULT_SLOT_SIZE); + + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + + if (length < UTUN_HEADER_SIZE(pcb) || + length > UTUN_IF_DEFAULT_SLOT_SIZE || + length > rx_pp->pp_buflet_size || + (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) { + /* flush data */ + kern_pbufpool_free(rx_pp, rx_ph); + printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n", + pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb)); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + continue; + } + + /* fillout packet */ + rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Find family + uint32_t af = 0; + uint8_t vhl = *(uint8_t *)(tx_baddr); + u_int ip_version = (vhl >> 4); + switch (ip_version) { + case 4: { + af = AF_INET; + break; + } + case 6: { + af = AF_INET6; + break; + } + default: { + printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n", + pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb)); + break; + } + } + + // Copy header + af = htonl(af); + memcpy((void *)rx_baddr, &af, sizeof(af)); + if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) { + kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af))); + } + + // Copy data from tx to rx + memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb)); + kern_packet_clear_flow_uuid(rx_ph); // zero flow id + + /* finalize and attach the packet */ + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_TXPKTS); + STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + + if (rx_pslot) { + kern_channel_advance_slot(rx_ring, rx_pslot); + kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats); + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + if (pcb->utun_output_disabled) { + errno_t error = ifnet_enable_output(pcb->utun_ifp); + if (error != 0) { + printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error); + } else { + pcb->utun_output_disabled = false; + } + } + + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + if (tx_pslot != NULL) { + kern_channel_notify(tx_ring, 0); + } + kr_exit(tx_ring); + + return 0; +} +#endif // UTUN_NEXUS /* - * These are place holders until coreTLS kext stops caling them + * These are place holders until coreTLS kext stops calling them */ errno_t utun_ctl_register_dtls (void *reg); int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family); diff --git a/bsd/net/if_utun.h b/bsd/net/if_utun.h index b75476582..008924154 100644 --- a/bsd/net/if_utun.h +++ b/bsd/net/if_utun.h @@ -34,20 +34,6 @@ #include -/* Control block allocated for each kernel control connection */ -struct utun_pcb { - kern_ctl_ref utun_ctlref; - ifnet_t utun_ifp; - u_int32_t utun_unit; - u_int32_t utun_flags; - int utun_ext_ifdata_stats; - u_int32_t utun_max_pending_packets; - int utun_channel_enabled; - uuid_t utun_channel_uuid; - void * utun_channel_rxring; - u_int32_t utun_channel_max_pktlen; -}; - void* utun_alloc(size_t size); void utun_free(void *ptr); errno_t utun_register_control(void); @@ -73,6 +59,7 @@ errno_t utun_register_control(void); from the control socket at a time */ #define UTUN_OPT_ENABLE_CHANNEL 17 #define UTUN_OPT_GET_CHANNEL_UUID 18 +#define UTUN_OPT_ENABLE_FLOWSWITCH 19 /* * Flags for by UTUN_OPT_FLAGS */ diff --git a/bsd/net/if_var.h b/bsd/net/if_var.h index bd3bdeba7..0541857f0 100644 --- a/bsd/net/if_var.h +++ b/bsd/net/if_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -75,6 +75,9 @@ #ifdef PRIVATE #include #endif +#ifdef BSD_KERN_PRIVATE +#include +#endif #ifdef KERNEL #include @@ -366,10 +369,66 @@ struct if_tcp_ecn_stat { u_int64_t ecn_total_conn; u_int64_t ecn_fallback_droprst; u_int64_t ecn_fallback_droprxmt; + u_int64_t ecn_fallback_synrst; struct if_tcp_ecn_perf_stat ecn_on; struct if_tcp_ecn_perf_stat ecn_off; }; +struct if_lim_perf_stat { + u_int64_t lim_dl_max_bandwidth; /* bits per second */ + u_int64_t lim_ul_max_bandwidth; /* bits per second */ + u_int64_t lim_total_txpkts; /* Total transmit packets, count */ + u_int64_t lim_total_rxpkts; /* Total receive packets, count */ + u_int64_t lim_total_retxpkts; /* Total retransmit packets */ + u_int64_t lim_packet_loss_percent; /* Packet loss rate */ + u_int64_t lim_total_oopkts; /* Total out-of-order packets */ + u_int64_t lim_packet_ooo_percent; /* Out-of-order packet rate */ + u_int64_t lim_rtt_variance; /* RTT variance, milliseconds */ + u_int64_t lim_rtt_average; /* RTT average, milliseconds */ + u_int64_t lim_rtt_min; /* RTT minimum, milliseconds */ + u_int64_t lim_conn_timeouts; /* connection timeouts */ + u_int64_t lim_conn_attempts; /* connection attempts */ + u_int64_t lim_conn_timeout_percent; /* Rate of connection timeouts */ + u_int64_t lim_bk_txpkts; /* Transmit packets with BK service class, that use delay based algorithms */ + u_int64_t lim_dl_detected:1, /* Low internet */ + lim_ul_detected:1; +}; + +#define IF_VAR_H_HAS_IFNET_STATS_PER_FLOW 1 +struct ifnet_stats_per_flow { + u_int64_t bk_txpackets; + u_int64_t txpackets; + u_int64_t rxpackets; + u_int32_t txretransmitbytes; + u_int32_t rxoutoforderbytes; + u_int32_t rxmitpkts; + u_int32_t rcvoopack; + u_int32_t pawsdrop; + u_int32_t sack_recovery_episodes; + u_int32_t reordered_pkts; + u_int32_t dsack_sent; + u_int32_t dsack_recvd; + u_int32_t srtt; + u_int32_t rttupdated; + u_int32_t rttvar; + u_int32_t rttmin; + u_int32_t bw_sndbw_max; + u_int32_t bw_rcvbw_max; + u_int32_t ecn_recv_ece; + u_int32_t ecn_recv_ce; + u_int16_t ecn_flags; + u_int16_t ipv4:1, + local:1, + connreset:1, + conntimeout:1, + rxmit_drop:1, + ecn_fallback_synloss:1, + ecn_fallback_droprst:1, + ecn_fallback_droprxmt:1, + ecn_fallback_ce:1, + ecn_fallback_reorder:1; +}; + /* * Interface link status report -- includes statistics related to * the link layer technology sent by the driver. The driver will monitor @@ -572,7 +631,7 @@ struct chain_len_stats { uint64_t cls_three; uint64_t cls_four; uint64_t cls_five_or_more; -}; +} __attribute__((__aligned__(sizeof (uint64_t)))); #endif /* PRIVATE */ @@ -610,7 +669,6 @@ struct if_data_internal { u_int32_t ifi_mtu; /* maximum transmission unit */ u_int32_t ifi_metric; /* routing metric (external only) */ u_int32_t ifi_baudrate; /* linespeed */ - u_int32_t ifi_preamblelen;/* length of the packet preamble */ /* volatile statistics */ u_int64_t ifi_ipackets; /* packets received on interface */ u_int64_t ifi_ierrors; /* input errors on interface */ @@ -634,25 +692,6 @@ struct if_data_internal { u_int32_t ifi_tso_v4_mtu; /* TCP Segment Offload IPv4 maximum segment size */ u_int32_t ifi_tso_v6_mtu; /* TCP Segment Offload IPv6 maximum segment size */ }; - -#if MEASURE_BW -/* - * Fields per interface to measure perceived bandwidth. - */ -struct if_measured_bw { - u_int64_t bw; /* measured bandwidth in bytes per ms */ - u_int64_t bytes; /* XXX not needed */ - u_int64_t ts; /* XXX not needed */ - u_int64_t cur_seq __attribute((aligned(8))); /* current sequence for marking a packet */ - u_int64_t start_ts; /* time at which a measurement started */ - u_int64_t start_seq; /* sequence at which a measurement should start */ - u_int64_t last_seq; /* last recorded seq */ - u_int64_t last_ts; /* last recorded ts */ - u_int32_t flags __attribute__((aligned(4))); /* flags */ -#define IF_MEASURED_BW_INPROGRESS 0x1 -#define IF_MEASURED_BW_CALCULATION 0x2 -}; -#endif /* MEASURE_BW */ #endif /* BSD_KERNEL_PRIVATE */ #ifdef PRIVATE @@ -662,7 +701,6 @@ struct if_measured_bw { #define if_physical if_data.ifi_physical #define if_addrlen if_data.ifi_addrlen #define if_hdrlen if_data.ifi_hdrlen -#define if_preamblelen if_data.ifi_preamblelen #define if_metric if_data.ifi_metric #define if_baudrate if_data.ifi_baudrate #define if_hwassist if_data.ifi_hwassist @@ -720,7 +758,7 @@ TAILQ_HEAD(ddesc_head_name, dlil_demux_desc); #ifdef PRIVATE /* - * All of the following IF_HWASSIST_* flags are defined in kpi_inteface.h as + * All of the following IF_HWASSIST_* flags are defined in kpi_interface.h as * IFNET_* flags. These are redefined here as constants to avoid failures to * build user level programs that can not include kpi_interface.h. It is * important to keep this in sync with the definitions in kpi_interface.h. @@ -737,6 +775,7 @@ TAILQ_HEAD(ddesc_head_name, dlil_demux_desc); #define IF_HWASSIST_CSUM_UDPIPV6 0x0040 /* will csum UDPv6, IFNET_CSUM_UDP */ #define IF_HWASSIST_CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation, IFNET_IPV6_FRAGMENT */ #define IF_HWASSIST_CSUM_PARTIAL 0x1000 /* simple Sum16 computation, IFNET_CSUM_PARTIAL */ +#define IF_HWASSIST_CSUM_ZERO_INVERT 0x2000 /* capable of inverting csum of 0 to -0 (0xffff) */ #define IF_HWASSIST_CSUM_MASK 0xffff #define IF_HWASSIST_CSUM_FLAGS(hwassist) ((hwassist) & IF_HWASSIST_CSUM_MASK) @@ -767,6 +806,12 @@ TAILQ_HEAD(ddesc_head_name, dlil_demux_desc); RB_HEAD(ll_reach_tree, if_llreach); /* define struct ll_reach_tree */ + +typedef errno_t (*dlil_input_func)(ifnet_t ifp, mbuf_t m_head, + mbuf_t m_tail, const struct ifnet_stat_increment_param *s, + boolean_t poll, struct thread *tp); +typedef errno_t (*dlil_output_func)(ifnet_t interface, mbuf_t data); + #define if_name(ifp) ifp->if_xname /* * Structure defining a network interface. @@ -817,12 +862,12 @@ struct ifnet { ifnet_family_t if_family; /* value assigned by Apple */ ifnet_subfamily_t if_subfamily; /* value assigned by Apple */ uintptr_t if_family_cookie; - ifnet_output_handler_func if_output_handler; + volatile dlil_input_func if_input_dlil; + volatile dlil_output_func if_output_dlil; + volatile ifnet_start_func if_start; ifnet_output_func if_output; ifnet_pre_enqueue_func if_pre_enqueue; - ifnet_start_func if_start; ifnet_ctl_func if_output_ctl; - ifnet_input_handler_func if_input_handler; ifnet_input_poll_func if_input_poll; ifnet_ctl_func if_input_ctl; ifnet_ioctl_func if_ioctl; @@ -882,6 +927,9 @@ struct ifnet { struct dlil_threading_info *if_inp; + /* allocated once along with dlil_ifnet and is never freed */ + thread_call_t if_dt_tcall; + struct { u_int32_t length; union { @@ -923,9 +971,6 @@ struct ifnet { struct mld_ifinfo *if_mli; /* for MLDv2 */ #endif /* INET6 */ -#if MEASURE_BW - struct if_measured_bw if_bw; -#endif /* MEASURE_BW */ struct tcpstat_local *if_tcp_stat; /* TCP specific stats */ struct udpstat_local *if_udp_stat; /* UDP specific stats */ @@ -972,8 +1017,26 @@ struct ifnet { struct if_interface_state if_interface_state; struct if_tcp_ecn_stat *if_ipv4_stat; struct if_tcp_ecn_stat *if_ipv6_stat; + + struct if_lim_perf_stat if_lim_stat; }; +/* Interface event handling declarations */ +extern struct eventhandler_lists_ctxt ifnet_evhdlr_ctxt; + +typedef enum { + INTF_EVENT_CODE_CREATED, + INTF_EVENT_CODE_REMOVED, + INTF_EVENT_CODE_STATUS_UPDATE, + INTF_EVENT_CODE_IPADDR_ATTACHED, + INTF_EVENT_CODE_IPADDR_DETACHED, + INTF_EVENT_CODE_LLADDR_UPDATE, + INTF_EVENT_CODE_MTU_CHANGED, +} intf_event_code_t; + +typedef void (*ifnet_event_fn)(struct eventhandler_entry_arg, struct ifnet *, struct sockaddr *, intf_event_code_t); +EVENTHANDLER_DECLARE(ifnet_event, ifnet_event_fn); + #define IF_TCP_STATINC(_ifp, _s) do { \ if ((_ifp)->if_tcp_stat != NULL) \ atomic_add_64(&(_ifp)->if_tcp_stat->_s, 1); \ @@ -987,9 +1050,14 @@ struct ifnet { /* * Valid values for if_refflags */ -#define IFRF_ATTACHED 0x1 /* ifnet attach is completely done */ -#define IFRF_DETACHING 0x2 /* detach has been requested */ - +#define IFRF_EMBRYONIC 0x1 /* ifnet is allocated; awaiting attach */ +#define IFRF_ATTACHED 0x2 /* ifnet attach is completely done */ +#define IFRF_DETACHING 0x4 /* detach has been requested */ +#define IFRF_ATTACH_MASK \ + (IFRF_EMBRYONIC|IFRF_ATTACHED|IFRF_DETACHING) + +#define IF_FULLY_ATTACHED(_ifp) \ + (((_ifp)->if_refflags & IFRF_ATTACH_MASK) == IFRF_ATTACHED) /* * Valid values for if_start_flags */ @@ -1112,7 +1180,16 @@ struct ifaddr { (struct ifaddr *, int); void (*ifa_attached)(struct ifaddr *); /* callback fn for attaching */ void (*ifa_detached)(struct ifaddr *); /* callback fn for detaching */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Align to 64-bit since we cast to this to struct in6_ifaddr, which is + * 64-bit aligned + */ +} __attribute__ ((aligned(8))); +#else }; +#endif /* @@ -1133,10 +1210,10 @@ struct ifaddr { #define IFD_NOTREADY 0x40 /* embryonic; not yet ready */ #define IFA_LOCK_ASSERT_HELD(_ifa) \ - lck_mtx_assert(&(_ifa)->ifa_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_ifa)->ifa_lock, LCK_MTX_ASSERT_OWNED) #define IFA_LOCK_ASSERT_NOTHELD(_ifa) \ - lck_mtx_assert(&(_ifa)->ifa_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_ifa)->ifa_lock, LCK_MTX_ASSERT_NOTOWNED) #define IFA_LOCK(_ifa) \ lck_mtx_lock(&(_ifa)->ifa_lock) @@ -1194,10 +1271,10 @@ struct ifmultiaddr { #define IFMAF_ANONYMOUS 0x1 /* has anonymous request ref(s) held */ #define IFMA_LOCK_ASSERT_HELD(_ifma) \ - lck_mtx_assert(&(_ifma)->ifma_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_ifma)->ifma_lock, LCK_MTX_ASSERT_OWNED) #define IFMA_LOCK_ASSERT_NOTHELD(_ifma) \ - lck_mtx_assert(&(_ifma)->ifma_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_ifma)->ifma_lock, LCK_MTX_ASSERT_NOTOWNED) #define IFMA_LOCK(_ifma) \ lck_mtx_lock(&(_ifma)->ifma_lock) @@ -1323,8 +1400,6 @@ extern lck_grp_t *ifa_mtx_grp; extern lck_grp_t *ifnet_lock_group; extern lck_attr_t *ifnet_lock_attr; extern ifnet_t lo_ifp; -extern uint32_t if_bw_measure_size; -extern u_int32_t if_bw_smoothing_val; extern int if_addmulti(struct ifnet *, const struct sockaddr *, struct ifmultiaddr **); @@ -1452,6 +1527,171 @@ extern int ifnet_set_log(struct ifnet *, int32_t, uint32_t, int32_t, int32_t); extern int ifnet_get_log(struct ifnet *, int32_t *, uint32_t *, int32_t *, int32_t *); extern int ifnet_notify_address(struct ifnet *, int); +extern void ifnet_notify_data_threshold(struct ifnet *); + +#define IF_AFDATA_RLOCK if_afdata_rlock +#define IF_AFDATA_RUNLOCK if_afdata_unlock +#define IF_AFDATA_WLOCK if_afdata_wlock +#define IF_AFDATA_WUNLOCK if_afdata_unlock +#define IF_AFDATA_WLOCK_ASSERT if_afdata_wlock_assert +#define IF_AFDATA_LOCK_ASSERT if_afdata_lock_assert +#define IF_AFDATA_UNLOCK_ASSERT if_afdata_unlock_assert + +static inline void +if_afdata_rlock (struct ifnet *ifp, int af) +{ + switch (af) { +#if INET + case AF_INET: + lck_rw_lock_shared(&ifp->if_inetdata_lock); + break; +#endif +#if INET6 + case AF_INET6: + lck_rw_lock_shared(&ifp->if_inet6data_lock); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_runlock (struct ifnet *ifp, int af) +{ + switch (af) { +#if INET + case AF_INET: + lck_rw_done(&ifp->if_inetdata_lock); + break; +#endif +#if INET6 + case AF_INET6: + lck_rw_done(&ifp->if_inet6data_lock); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_wlock (struct ifnet *ifp, int af) +{ + switch (af) { +#if INET + case AF_INET: + lck_rw_lock_exclusive(&ifp->if_inetdata_lock); + break; +#endif +#if INET6 + case AF_INET6: + lck_rw_lock_exclusive(&ifp->if_inet6data_lock); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_unlock (struct ifnet *ifp, int af) +{ + switch (af) { +#if INET + case AF_INET: + lck_rw_done(&ifp->if_inetdata_lock); + break; +#endif +#if INET6 + case AF_INET6: + lck_rw_done(&ifp->if_inet6data_lock); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_wlock_assert (struct ifnet *ifp, int af) +{ +#if !MACH_ASSERT +#pragma unused(ifp) +#endif + switch (af) { +#if INET + case AF_INET: + LCK_RW_ASSERT(&ifp->if_inetdata_lock, LCK_RW_ASSERT_EXCLUSIVE); + break; +#endif +#if INET6 + case AF_INET6: + LCK_RW_ASSERT(&ifp->if_inet6data_lock, LCK_RW_ASSERT_EXCLUSIVE); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_unlock_assert (struct ifnet *ifp, int af) +{ +#if !MACH_ASSERT +#pragma unused(ifp) +#endif + switch (af) { +#if INET + case AF_INET: + LCK_RW_ASSERT(&ifp->if_inetdata_lock, LCK_RW_ASSERT_NOTHELD); + break; +#endif +#if INET6 + case AF_INET6: + LCK_RW_ASSERT(&ifp->if_inet6data_lock, LCK_RW_ASSERT_NOTHELD); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_lock_assert (struct ifnet *ifp, int af) +{ +#if !MACH_ASSERT +#pragma unused(ifp) +#endif + switch (af) { +#if INET + case AF_INET: + LCK_RW_ASSERT(&ifp->if_inetdata_lock, LCK_RW_ASSERT_HELD); + break; +#endif +#if INET6 + case AF_INET6: + LCK_RW_ASSERT(&ifp->if_inet6data_lock, LCK_RW_ASSERT_HELD); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} #if INET6 struct in6_addr; @@ -1511,17 +1751,39 @@ __private_extern__ int ifnet_set_netsignature(struct ifnet *, uint8_t, __private_extern__ int ifnet_get_netsignature(struct ifnet *, uint8_t, uint8_t *, uint16_t *, uint8_t *); +#if INET6 +struct ipv6_prefix; +__private_extern__ int ifnet_set_nat64prefix(struct ifnet *, + struct ipv6_prefix *); +__private_extern__ int ifnet_get_nat64prefix(struct ifnet *, + struct ipv6_prefix *); +#endif + /* Required exclusive ifnet_head lock */ __private_extern__ void ifnet_remove_from_ordered_list(struct ifnet *); __private_extern__ void ifnet_increment_generation(struct ifnet *); __private_extern__ u_int32_t ifnet_get_generation(struct ifnet *); -extern int if_set_qosmarking_mode(struct ifnet *, u_int32_t); +/* Adding and deleting netagents will take ifnet lock */ +__private_extern__ int if_add_netagent(struct ifnet *, uuid_t); +__private_extern__ int if_delete_netagent(struct ifnet *, uuid_t); +extern int if_set_qosmarking_mode(struct ifnet *, u_int32_t); +__private_extern__ uint32_t ifnet_mbuf_packetpreamblelen(struct ifnet *); +__private_extern__ void intf_event_enqueue_nwk_wq_entry(struct ifnet *ifp, + struct sockaddr *addrp, uint32_t intf_event_code); +__private_extern__ void ifnet_update_stats_per_flow(struct ifnet_stats_per_flow *, + struct ifnet *); +#if !CONFIG_EMBEDDED __private_extern__ errno_t ifnet_framer_stub(struct ifnet *, struct mbuf **, const struct sockaddr *, const char *, const char *, u_int32_t *, u_int32_t *); +#endif /* !CONFIG_EMBEDDED */ +__private_extern__ void ifnet_enqueue_multi_setup(struct ifnet *, uint16_t, + uint16_t); +__private_extern__ errno_t ifnet_enqueue_mbuf(struct ifnet *, struct mbuf *, + boolean_t, boolean_t *); #endif /* BSD_KERNEL_PRIVATE */ #ifdef XNU_KERNEL_PRIVATE /* for uuid.c */ diff --git a/bsd/net/if_vlan.c b/bsd/net/if_vlan.c index 2dabd32b2..737ce40ad 100644 --- a/bsd/net/if_vlan.c +++ b/bsd/net/if_vlan.c @@ -109,9 +109,6 @@ #define VLANNAME "vlan" -typedef int (bpf_callback_func)(struct ifnet *, struct mbuf *); -typedef int (if_set_bpf_tap_func)(struct ifnet *ifp, int mode, bpf_callback_func * func); - /** ** vlan locks **/ @@ -153,14 +150,14 @@ vlan_lock_init(void) static __inline__ void vlan_assert_lock_held(void) { - lck_mtx_assert(vlan_lck_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(vlan_lck_mtx, LCK_MTX_ASSERT_OWNED); return; } static __inline__ void vlan_assert_lock_not_held(void) { - lck_mtx_assert(vlan_lck_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(vlan_lck_mtx, LCK_MTX_ASSERT_NOTOWNED); return; } @@ -224,8 +221,6 @@ struct ifvlan { #define IFVF_DETACHING 0x2 /* interface is detaching */ #define IFVF_READY 0x4 /* interface is ready */ u_int32_t ifv_flags; - bpf_packet_func ifv_bpf_input; - bpf_packet_func ifv_bpf_output; int32_t ifv_retain_count; u_int32_t ifv_signature; /* IFV_SIGNATURE */ }; @@ -382,8 +377,6 @@ static int vlan_input(ifnet_t ifp, protocol_family_t protocol, mbuf_t m, char *frame_header); static int vlan_output(struct ifnet *ifp, struct mbuf *m); static int vlan_ioctl(ifnet_t ifp, u_long cmd, void * addr); -static int vlan_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, - bpf_packet_func func); static int vlan_attach_protocol(struct ifnet *ifp); static int vlan_detach_protocol(struct ifnet *ifp); static int vlan_setmulti(struct ifnet *ifp); @@ -566,39 +559,6 @@ siocsifaltmtu(struct ifnet * ifp, int mtu) return (ifnet_ioctl(ifp, 0, SIOCSIFALTMTU, &ifr)); } -static __inline__ void -vlan_bpf_output(struct ifnet * ifp, struct mbuf * m, - bpf_packet_func func) -{ - if (func != NULL) { - (*func)(ifp, m); - } - return; -} - -static __inline__ void -vlan_bpf_input(struct ifnet * ifp, struct mbuf * m, - bpf_packet_func func, char * frame_header, - int frame_header_len, int encap_len) -{ - if (func != NULL) { - if (encap_len > 0) { - /* present the right header to bpf */ - bcopy(frame_header, frame_header + encap_len, frame_header_len); - } - m->m_data -= frame_header_len; - m->m_len += frame_header_len; - (*func)(ifp, m); - m->m_data += frame_header_len; - m->m_len -= frame_header_len; - if (encap_len > 0) { - /* restore the header */ - bcopy(frame_header + encap_len, frame_header, frame_header_len); - } - } - return; -} - /** ** vlan_parent synchronization routines **/ @@ -1010,7 +970,7 @@ vlan_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) vlan_init.framer_extended = ether_frameout_extended; vlan_init.softc = ifv; vlan_init.ioctl = vlan_ioctl; - vlan_init.set_bpf_tap = vlan_set_bpf_tap; + vlan_init.set_bpf_tap = NULL; vlan_init.detach = vlan_if_free; vlan_init.broadcast_addr = etherbroadcastaddr; vlan_init.broadcast_len = ETHER_ADDR_LEN; @@ -1075,45 +1035,9 @@ vlan_clone_destroy(struct ifnet *ifp) return 0; } -static int -vlan_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, bpf_packet_func func) -{ - ifvlan_ref ifv; - - vlan_lock(); - ifv = ifnet_get_ifvlan_retained(ifp); - if (ifv == NULL) { - vlan_unlock(); - return (ENODEV); - } - switch (mode) { - case BPF_TAP_DISABLE: - ifv->ifv_bpf_input = ifv->ifv_bpf_output = NULL; - break; - - case BPF_TAP_INPUT: - ifv->ifv_bpf_input = func; - break; - - case BPF_TAP_OUTPUT: - ifv->ifv_bpf_output = func; - break; - - case BPF_TAP_INPUT_OUTPUT: - ifv->ifv_bpf_input = ifv->ifv_bpf_output = func; - break; - default: - break; - } - vlan_unlock(); - ifvlan_release(ifv); - return 0; -} - static int vlan_output(struct ifnet * ifp, struct mbuf * m) { - bpf_packet_func bpf_func; struct ether_vlan_header * evl; int encaplen; ifvlan_ref ifv; @@ -1143,7 +1067,6 @@ vlan_output(struct ifnet * ifp, struct mbuf * m) p = vlp->vlp_ifp; (void)ifnet_stat_increment_out(ifp, 1, m->m_pkthdr.len, 0); soft_vlan = (ifnet_offload(p) & IF_HWASSIST_VLAN_TAGGING) == 0; - bpf_func = ifv->ifv_bpf_output; tag = ifv->ifv_tag; encaplen = ifv->ifv_encaplen; vlan_unlock(); @@ -1151,7 +1074,7 @@ vlan_output(struct ifnet * ifp, struct mbuf * m) ifvlan_release(ifv); vlan_parent_release(vlp); - vlan_bpf_output(ifp, m, bpf_func); + bpf_tap_out(ifp, DLT_EN10MB, m, NULL, 0); /* do not run parent's if_output() if the parent is not up */ if ((ifnet_flags(p) & (IFF_UP | IFF_RUNNING)) != (IFF_UP | IFF_RUNNING)) { @@ -1230,7 +1153,6 @@ static int vlan_input(ifnet_t p, __unused protocol_family_t protocol, mbuf_t m, char *frame_header) { - bpf_packet_func bpf_func = NULL; struct ether_vlan_header * evl; struct ifnet * ifp = NULL; int soft_vlan = 0; @@ -1294,7 +1216,6 @@ vlan_input(ifnet_t p, __unused protocol_family_t protocol, m_freem(m); return 0; } - bpf_func = ifv->ifv_bpf_input; vlan_unlock(); } if (soft_vlan) { @@ -1313,8 +1234,7 @@ vlan_input(ifnet_t p, __unused protocol_family_t protocol, m->m_pkthdr.pkt_hdr = frame_header; (void)ifnet_stat_increment_in(ifp, 1, m->m_pkthdr.len + ETHER_HDR_LEN, 0); - vlan_bpf_input(ifp, m, bpf_func, frame_header, ETHER_HDR_LEN, - soft_vlan ? ETHER_VLAN_ENCAP_LEN : 0); + bpf_tap_in(ifp, DLT_EN10MB, m, frame_header, ETHER_HDR_LEN); /* We found a vlan interface, inject on that interface. */ dlil_input_packet_list(ifp, m); } else { @@ -1899,6 +1819,8 @@ vlan_ioctl(ifnet_t ifp, u_long cmd, void * data) break; } p = NULL; + /* ensure nul termination */ + vlr.vlr_parent[IFNAMSIZ - 1] = '\0'; if (vlr.vlr_parent[0] != '\0') { if (vlr.vlr_tag & ~EVL_VLID_MASK) { /* diff --git a/bsd/net/iptap.c b/bsd/net/iptap.c index ead29d8f2..a4c2cabdb 100644 --- a/bsd/net/iptap.c +++ b/bsd/net/iptap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999-2013 Apple Inc. All rights reserved. + * Copyright (c) 1999-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -187,7 +187,7 @@ iptap_clone_create(struct if_clone *ifc, u_int32_t unit, void *params) int error = 0; struct iptap_softc *iptap = NULL; - struct ifnet_init_params if_init; + struct ifnet_init_eparams if_init; iptap = _MALLOC(sizeof(struct iptap_softc), M_DEVBUF, M_WAITOK | M_ZERO); if (iptap == NULL) { @@ -201,7 +201,10 @@ iptap_clone_create(struct if_clone *ifc, u_int32_t unit, void *params) * We do not use a set_bpf_tap() function as we rather rely on the more * accurate callback passed to bpf_attach() */ - bzero(&if_init, sizeof(struct ifnet_init_params)); + bzero(&if_init, sizeof(if_init)); + if_init.ver = IFNET_INIT_CURRENT_VERSION; + if_init.len = sizeof (if_init); + if_init.flags = IFNET_INIT_LEGACY; if_init.name = ifc->ifc_name; if_init.unit = unit; if_init.type = IFT_OTHER; @@ -214,7 +217,7 @@ iptap_clone_create(struct if_clone *ifc, u_int32_t unit, void *params) if_init.ioctl = iptap_ioctl; if_init.detach = iptap_detach; - error = ifnet_allocate(&if_init, &iptap->iptap_ifp); + error = ifnet_allocate_extended(&if_init, &iptap->iptap_ifp); if (error != 0) { printf("%s: ifnet_allocate failed, error %d\n", __func__, error); goto done; @@ -581,6 +584,23 @@ iptap_bpf_tap(struct mbuf *m, u_int32_t proto, int outgoing) struct iptap_softc *iptap; void (*bpf_tap_func)(ifnet_t , u_int32_t , mbuf_t , void * , size_t ) = outgoing ? bpf_tap_out : bpf_tap_in; + uint16_t src_scope_id = 0; + uint16_t dst_scope_id = 0; + + if (proto == AF_INET6) { + struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); + /* + * Clear the embedded scope ID + */ + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { + src_scope_id = ip6->ip6_src.s6_addr16[1]; + ip6->ip6_src.s6_addr16[1] = 0; + } + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) { + dst_scope_id = ip6->ip6_dst.s6_addr16[1]; + ip6->ip6_dst.s6_addr16[1] = 0; + } + } iptap_lock_shared(); @@ -625,4 +645,18 @@ iptap_bpf_tap(struct mbuf *m, u_int32_t proto, int outgoing) } iptap_lock_done(); + + if (proto == AF_INET6) { + struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); + + /* + * Restore the embedded scope ID + */ + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { + ip6->ip6_src.s6_addr16[1] = src_scope_id; + } + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) { + ip6->ip6_dst.s6_addr16[1] = dst_scope_id; + } + } } diff --git a/bsd/net/kpi_interface.c b/bsd/net/kpi_interface.c index d28af82ac..05e7cfc57 100644 --- a/bsd/net/kpi_interface.c +++ b/bsd/net/kpi_interface.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2016 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -70,6 +71,7 @@ #include #endif #include +#include #include "net/net_str_id.h" @@ -78,6 +80,15 @@ #include #endif + +#undef ifnet_allocate +errno_t ifnet_allocate(const struct ifnet_init_params *init, + ifnet_t *ifp); + +static errno_t ifnet_allocate_common(const struct ifnet_init_params *init, + ifnet_t *ifp, bool is_internal); + + #define TOUCHLASTCHANGE(__if_lastchange) { \ (__if_lastchange)->tv_sec = net_uptime(); \ (__if_lastchange)->tv_usec = 0; \ @@ -117,7 +128,8 @@ ifnet_kpi_free(ifnet_t ifp) } errno_t -ifnet_allocate(const struct ifnet_init_params *init, ifnet_t *interface) +ifnet_allocate_common(const struct ifnet_init_params *init, + ifnet_t *ifp, bool is_internal) { struct ifnet_init_eparams einit; @@ -125,7 +137,10 @@ ifnet_allocate(const struct ifnet_init_params *init, ifnet_t *interface) einit.ver = IFNET_INIT_CURRENT_VERSION; einit.len = sizeof (einit); - einit.flags = IFNET_INIT_LEGACY; + einit.flags = IFNET_INIT_LEGACY | IFNET_INIT_NX_NOAUTO; + if (!is_internal) { + einit.flags |= IFNET_INIT_ALLOC_KPI; + } einit.uniqueid = init->uniqueid; einit.uniqueid_len = init->uniqueid_len; einit.name = init->name; @@ -146,7 +161,19 @@ ifnet_allocate(const struct ifnet_init_params *init, ifnet_t *interface) einit.broadcast_addr = init->broadcast_addr; einit.broadcast_len = init->broadcast_len; - return (ifnet_allocate_extended(&einit, interface)); + return (ifnet_allocate_extended(&einit, ifp)); +} + +errno_t +ifnet_allocate_internal(const struct ifnet_init_params *init, ifnet_t *ifp) +{ + return (ifnet_allocate_common(init, ifp, true)); +} + +errno_t +ifnet_allocate(const struct ifnet_init_params *init, ifnet_t *ifp) +{ + return (ifnet_allocate_common(init, ifp, false)); } errno_t @@ -169,8 +196,10 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, (einit.type & 0xFFFFFF00) != 0 || einit.type == 0) return (EINVAL); + if (einit.flags & IFNET_INIT_LEGACY) { - if (einit.output == NULL || einit.flags != IFNET_INIT_LEGACY) + if (einit.output == NULL || + (einit.flags & IFNET_INIT_INPUT_POLL)) return (EINVAL); einit.pre_enqueue = NULL; @@ -198,8 +227,8 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, if (einit.uniqueid == NULL) { /* Initialize external name (name + unit) */ - snprintf(if_xname, IFXNAMSIZ, - "%s%d", einit.name, einit.unit); + (void) snprintf(if_xname, sizeof (if_xname), "%s%d", + einit.name, einit.unit); einit.uniqueid = if_xname; einit.uniqueid_len = strlen(if_xname); } @@ -268,12 +297,17 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, * Internally, DLIL will only use the extended callback * variant which is represented by if_framer. */ +#if CONFIG_EMBEDDED + if (ifp->if_framer == NULL && ifp->if_framer_legacy != NULL) + ifp->if_framer = ifp->if_framer_legacy; +#else /* !CONFIG_EMBEDDED */ if (ifp->if_framer == NULL && ifp->if_framer_legacy != NULL) { if (ifp->if_framer_legacy == ether_frameout) ifp->if_framer = ether_frameout_extended; else ifp->if_framer = ifnet_framer_stub; } +#endif /* !CONFIG_EMBEDDED */ if (ifp->if_output_bw.eff_bw > ifp->if_output_bw.max_bw) ifp->if_output_bw.max_bw = ifp->if_output_bw.eff_bw; @@ -313,6 +347,7 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, if (ifp->if_ioctl == NULL) ifp->if_ioctl = ifp_if_ioctl; + ifp->if_eflags = 0; if (ifp->if_start != NULL) { ifp->if_eflags |= IFEF_TXSTART; if (ifp->if_pre_enqueue == NULL) @@ -327,8 +362,8 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, else ifp->if_eflags &= ~IFEF_RXPOLL; - ifp->if_output_handler = dlil_output_handler; - ifp->if_input_handler = dlil_input_handler; + ifp->if_output_dlil = dlil_output_handler; + ifp->if_input_dlil = dlil_input_handler; VERIFY(!(einit.flags & IFNET_INIT_LEGACY) || (ifp->if_pre_enqueue == NULL && ifp->if_start == NULL && @@ -359,6 +394,8 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, bzero(&ifp->if_broadcast, sizeof (ifp->if_broadcast)); } + ifp->if_xflags = 0; + /* * output target queue delay is specified in millisecond * convert it to nanoseconds @@ -367,15 +404,29 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, einit.output_target_qdelay * 1000 * 1000; IFCQ_MAXLEN(&ifp->if_snd) = einit.sndq_maxlen; - if (einit.start_delay_qlen > 0 && - einit.start_delay_timeout > 0) { - ifp->if_eflags |= IFEF_ENQUEUE_MULTI; - ifp->if_start_delay_qlen = - min(100, einit.start_delay_qlen); - ifp->if_start_delay_timeout = - min(20000, einit.start_delay_timeout); - /* convert timeout to nanoseconds */ - ifp->if_start_delay_timeout *= 1000; + ifnet_enqueue_multi_setup(ifp, einit.start_delay_qlen, + einit.start_delay_timeout); + + IFCQ_PKT_DROP_LIMIT(&ifp->if_snd) = IFCQ_DEFAULT_PKT_DROP_LIMIT; + + /* + * Set embryonic flag; this will be cleared + * later when it is fully attached. + */ + ifp->if_refflags = IFRF_EMBRYONIC; + + /* + * Count the newly allocated ifnet + */ + OSIncrementAtomic64(&net_api_stats.nas_ifnet_alloc_count); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_ifnet_alloc_total); + if (einit.flags & IFNET_INIT_ALLOC_KPI) { + ifp->if_xflags |= IFXF_ALLOC_KPI; + } else { + OSIncrementAtomic64( + &net_api_stats.nas_ifnet_alloc_os_count); + INC_ATOMIC_INT64_LIM( + net_api_stats.nas_ifnet_alloc_os_total); } if (error == 0) { @@ -387,17 +438,6 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, *interface = NULL; } } - - /* - * Note: We should do something here to indicate that we haven't been - * attached yet. By doing so, we can catch the case in ifnet_release - * where the reference count reaches zero and call the recycle - * function. If the interface is attached, the interface will be - * recycled when the interface's if_free function is called. If the - * interface is never attached, the if_free function will never be - * called and the interface will never be recycled. - */ - return (error); } @@ -617,7 +657,7 @@ ifnet_set_idle_flags_locked(ifnet_t ifp, u_int32_t new_flags, u_int32_t mask) if (ifp == NULL) return (EINVAL); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); ifnet_lock_assert(ifp, IFNET_LCK_ASSERT_EXCLUSIVE); /* @@ -848,9 +888,10 @@ ifnet_capabilities_enabled(ifnet_t ifp) static const ifnet_offload_t offload_mask = (IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | IFNET_CSUM_FRAGMENT | IFNET_IP_FRAGMENT | IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6 | - IFNET_IPV6_FRAGMENT | IFNET_CSUM_PARTIAL | IFNET_VLAN_TAGGING | - IFNET_VLAN_MTU | IFNET_MULTIPAGES | IFNET_TSO_IPV4 | IFNET_TSO_IPV6 | - IFNET_TX_STATUS | IFNET_HW_TIMESTAMP | IFNET_SW_TIMESTAMP); + IFNET_IPV6_FRAGMENT | IFNET_CSUM_PARTIAL | IFNET_CSUM_ZERO_INVERT | + IFNET_VLAN_TAGGING | IFNET_VLAN_MTU | IFNET_MULTIPAGES | + IFNET_TSO_IPV4 | IFNET_TSO_IPV6 | IFNET_TX_STATUS | IFNET_HW_TIMESTAMP | + IFNET_SW_TIMESTAMP); static const ifnet_offload_t any_offload_csum = IFNET_CHECKSUMF; @@ -864,6 +905,7 @@ ifnet_set_offload(ifnet_t interface, ifnet_offload_t offload) ifnet_lock_exclusive(interface); interface->if_hwassist = (offload & offload_mask); + /* * Hardware capable of partial checksum offload is * flexible enough to handle any transports utilizing @@ -897,6 +939,10 @@ ifnet_set_offload(ifnet_t interface, ifnet_offload_t offload) ifcaps |= IFCAP_HW_TIMESTAMP; if ((offload & IFNET_SW_TIMESTAMP)) ifcaps |= IFCAP_SW_TIMESTAMP; + if ((offload & IFNET_CSUM_PARTIAL)) + ifcaps |= IFCAP_CSUM_PARTIAL; + if ((offload & IFNET_CSUM_ZERO_INVERT)) + ifcaps |= IFCAP_CSUM_ZERO_INVERT; if (ifcaps != 0) { (void) ifnet_set_capabilities_supported(interface, ifcaps, IFCAP_VALID); @@ -1525,6 +1571,9 @@ ifnet_stat_increment(struct ifnet *ifp, /* Touch the last change time. */ TOUCHLASTCHANGE(&ifp->if_lastchange); + if (ifp->if_data_threshold != 0) + ifnet_notify_data_threshold(ifp); + return (0); } @@ -1544,6 +1593,9 @@ ifnet_stat_increment_in(struct ifnet *ifp, u_int32_t packets_in, TOUCHLASTCHANGE(&ifp->if_lastchange); + if (ifp->if_data_threshold != 0) + ifnet_notify_data_threshold(ifp); + return (0); } @@ -1563,6 +1615,9 @@ ifnet_stat_increment_out(struct ifnet *ifp, u_int32_t packets_out, TOUCHLASTCHANGE(&ifp->if_lastchange); + if (ifp->if_data_threshold != 0) + ifnet_notify_data_threshold(ifp); + return (0); } @@ -1589,6 +1644,9 @@ ifnet_set_stat(struct ifnet *ifp, const struct ifnet_stats_param *s) /* Touch the last change time. */ TOUCHLASTCHANGE(&ifp->if_lastchange); + if (ifp->if_data_threshold != 0) + ifnet_notify_data_threshold(ifp); + return (0); } @@ -1612,6 +1670,9 @@ ifnet_stat(struct ifnet *ifp, struct ifnet_stats_param *s) atomic_get_64(s->dropped, &ifp->if_data.ifi_iqdrops); atomic_get_64(s->no_protocol, &ifp->if_data.ifi_noproto); + if (ifp->if_data_threshold != 0) + ifnet_notify_data_threshold(ifp); + return (0); } @@ -1964,6 +2025,8 @@ ifnet_set_lladdr_internal(ifnet_t interface, const void *lladdr, /* Generate a kernel event */ if (error == 0) { + intf_event_enqueue_nwk_wq_entry(interface, NULL, + INTF_EVENT_CODE_LLADDR_UPDATE); dlil_post_msg(interface, KEV_DL_SUBCLASS, KEV_DL_LINK_ADDRESS_CHANGED, NULL, 0); } @@ -2199,97 +2262,6 @@ ifnet_list_free(ifnet_t *interfaces) FREE(interfaces, M_TEMP); } -void -ifnet_transmit_burst_start(ifnet_t ifp, mbuf_t pkt) -{ -#if MEASURE_BW - uint32_t orig_flags; - - if (ifp == NULL || !(pkt->m_flags & M_PKTHDR)) - return; - - orig_flags = OSBitOrAtomic(IF_MEASURED_BW_INPROGRESS, - &ifp->if_bw.flags); - if (orig_flags & IF_MEASURED_BW_INPROGRESS) { - /* There is already a measurement in progress; skip this one */ - return; - } - - ifp->if_bw.start_seq = pkt->m_pkthdr.pkt_bwseq; - ifp->if_bw.start_ts = mach_absolute_time(); -#else /* !MEASURE_BW */ -#pragma unused(ifp, pkt) -#endif /* !MEASURE_BW */ -} - -void -ifnet_transmit_burst_end(ifnet_t ifp, mbuf_t pkt) -{ -#if MEASURE_BW - uint64_t oseq, ots, bytes, ts, t; - uint32_t flags; - - if (ifp == NULL || !(pkt->m_flags & M_PKTHDR)) - return; - - flags = OSBitOrAtomic(IF_MEASURED_BW_CALCULATION, &ifp->if_bw.flags); - - /* If a calculation is already in progress, just return */ - if (flags & IF_MEASURED_BW_CALCULATION) - return; - - /* Check if a measurement was started at all */ - if (!(flags & IF_MEASURED_BW_INPROGRESS)) { - /* - * It is an error to call burst_end before burst_start. - * Reset the calculation flag and return. - */ - goto done; - } - - oseq = pkt->m_pkthdr.pkt_bwseq; - ots = mach_absolute_time(); - - if (ifp->if_bw.start_seq > 0 && oseq > ifp->if_bw.start_seq) { - ts = ots - ifp->if_bw.start_ts; - if (ts > 0) { - absolutetime_to_nanoseconds(ts, &t); - bytes = oseq - ifp->if_bw.start_seq; - ifp->if_bw.bytes = bytes; - ifp->if_bw.ts = ts; - - if (t > 0) { - uint64_t bw = 0; - - /* Compute bandwidth as bytes/ms */ - bw = (bytes * NSEC_PER_MSEC) / t; - if (bw > 0) { - if (ifp->if_bw.bw > 0) { - u_int32_t shft; - - shft = if_bw_smoothing_val; - /* Compute EWMA of bw */ - ifp->if_bw.bw = (bw + - ((ifp->if_bw.bw << shft) - - ifp->if_bw.bw)) >> shft; - } else { - ifp->if_bw.bw = bw; - } - } - } - ifp->if_bw.last_seq = oseq; - ifp->if_bw.last_ts = ots; - } - } - -done: - flags = ~(IF_MEASURED_BW_INPROGRESS | IF_MEASURED_BW_CALCULATION); - OSBitAndAtomic(flags, &ifp->if_bw.flags); -#else /* !MEASURE_BW */ -#pragma unused(ifp, pkt) -#endif /* !MEASURE_BW */ -} - /*************************************************************************/ /* ifaddr_t accessors */ /*************************************************************************/ @@ -2627,17 +2599,6 @@ ifnet_get_local_ports_extended(ifnet_t ifp, protocol_family_t protocol, u_int32_t ifindex; u_int32_t inp_flags = 0; - inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_WILDCARDOK) ? - INPCB_GET_PORTS_USED_WILDCARDOK : 0); - inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_NOWAKEUPOK) ? - INPCB_GET_PORTS_USED_NOWAKEUPOK : 0); - inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_RECVANYIFONLY) ? - INPCB_GET_PORTS_USED_RECVANYIFONLY : 0); - inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_EXTBGIDLEONLY) ? - INPCB_GET_PORTS_USED_EXTBGIDLEONLY : 0); - inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_ACTIVEONLY) ? - INPCB_GET_PORTS_USED_ACTIVEONLY : 0); - if (bitfield == NULL) return (EINVAL); @@ -2653,14 +2614,27 @@ ifnet_get_local_ports_extended(ifnet_t ifp, protocol_family_t protocol, /* bit string is long enough to hold 16-bit port values */ bzero(bitfield, bitstr_size(65536)); - ifindex = (ifp != NULL) ? ifp->if_index : 0; - - if (!(flags & IFNET_GET_LOCAL_PORTS_TCPONLY)) - udp_get_ports_used(ifindex, protocol, inp_flags, bitfield); - - if (!(flags & IFNET_GET_LOCAL_PORTS_UDPONLY)) - tcp_get_ports_used(ifindex, protocol, inp_flags, bitfield); - + inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_WILDCARDOK) ? + INPCB_GET_PORTS_USED_WILDCARDOK : 0); + inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_NOWAKEUPOK) ? + INPCB_GET_PORTS_USED_NOWAKEUPOK : 0); + inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_RECVANYIFONLY) ? + INPCB_GET_PORTS_USED_RECVANYIFONLY : 0); + inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_EXTBGIDLEONLY) ? + INPCB_GET_PORTS_USED_EXTBGIDLEONLY : 0); + inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_ACTIVEONLY) ? + INPCB_GET_PORTS_USED_ACTIVEONLY : 0); + + + ifindex = (ifp != NULL) ? ifp->if_index : 0; + + if (!(flags & IFNET_GET_LOCAL_PORTS_TCPONLY)) + udp_get_ports_used(ifindex, protocol, inp_flags, + bitfield); + + if (!(flags & IFNET_GET_LOCAL_PORTS_UDPONLY)) + tcp_get_ports_used(ifindex, protocol, inp_flags, + bitfield); return (0); } @@ -2669,7 +2643,7 @@ ifnet_get_local_ports(ifnet_t ifp, u_int8_t *bitfield) { u_int32_t flags = IFNET_GET_LOCAL_PORTS_WILDCARDOK; return (ifnet_get_local_ports_extended(ifp, PF_UNSPEC, flags, - bitfield)); + bitfield)); } errno_t @@ -2945,6 +2919,9 @@ ifnet_link_status_report(ifnet_t ifp, const void *buffer, atomic_bitset_32(&tcbinfo.ipi_flags, INPCBINFO_UPDATE_MSS); inpcb_timer_sched(&tcbinfo, INPCB_TIMER_FAST); +#if NECP + necp_update_all_clients(); +#endif } /* Finally copy the new information */ @@ -3021,39 +2998,6 @@ done: return (err); } -/*************************************************************************/ -/* Packet preamble */ -/*************************************************************************/ - -#define MAX_IF_PACKET_PREAMBLE_LEN 32 - -errno_t -ifnet_set_packetpreamblelen(ifnet_t interface, u_int32_t len) -{ - errno_t err = 0; - - if (interface == NULL || len > MAX_IF_PACKET_PREAMBLE_LEN) { - err = EINVAL; - goto done; - } - interface->if_data.ifi_preamblelen = len; -done: - return (err); -} - -u_int32_t -ifnet_packetpreamblelen(ifnet_t interface) -{ - return ((interface == NULL) ? 0 : interface->if_data.ifi_preamblelen); -} - -u_int32_t -ifnet_maxpacketpreamblelen(void) -{ - return (MAX_IF_PACKET_PREAMBLE_LEN); -} - - /*************************************************************************/ /* Fastlane QoS Ca */ /*************************************************************************/ @@ -3092,8 +3036,7 @@ ifnet_get_unsent_bytes(ifnet_t interface, int64_t *unsent_bytes) bytes = *unsent_bytes = 0; - if ((interface->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) != - IFRF_ATTACHED) + if (!IF_FULLY_ATTACHED(interface)) return (ENXIO); bytes = interface->if_sndbyte_unsent; @@ -3113,15 +3056,15 @@ ifnet_get_buffer_status(const ifnet_t ifp, ifnet_buffer_status_t *buf_status) bzero(buf_status, sizeof (*buf_status)); - if ((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) != - IFRF_ATTACHED) + if (!IF_FULLY_ATTACHED(ifp)) return (ENXIO); - buf_status->buf_sndbuf = ifp->if_sndbyte_unsent; - if (ifp->if_eflags & IFEF_TXSTART) buf_status->buf_interface = IFCQ_BYTES(&ifp->if_snd); + buf_status->buf_sndbuf = ((buf_status->buf_interface != 0) || + (ifp->if_sndbyte_unsent != 0)) ? 1 : 0; + return (0); } @@ -3133,8 +3076,7 @@ ifnet_normalise_unsent_data(void) ifnet_head_lock_shared(); TAILQ_FOREACH(ifp, &ifnet_head, if_link) { ifnet_lock_exclusive(ifp); - if ((ifp->if_refflags & (IFRF_ATTACHED|IFRF_DETACHING)) != - IFRF_ATTACHED) { + if (!IF_FULLY_ATTACHED(ifp)) { ifnet_lock_done(ifp); continue; } diff --git a/bsd/net/kpi_interface.h b/bsd/net/kpi_interface.h index c94f294f0..15b7fd09f 100644 --- a/bsd/net/kpi_interface.h +++ b/bsd/net/kpi_interface.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2016 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -49,10 +49,18 @@ struct if_interface_state; #include #ifdef XNU_KERNEL_PRIVATE +#if CONFIG_EMBEDDED +#define KPI_INTERFACE_EMBEDDED 1 +#else #define KPI_INTERFACE_EMBEDDED 0 +#endif +#else +#if TARGET_OS_EMBEDDED +#define KPI_INTERFACE_EMBEDDED 1 #else #define KPI_INTERFACE_EMBEDDED 0 #endif +#endif struct timeval; struct sockaddr; @@ -124,6 +132,7 @@ enum { IFNET_SUBFAMILY_RESERVED = 5, IFNET_SUBFAMILY_INTCOPROC = 6, IFNET_SUBFAMILY_UTUN = 7, + IFNET_SUBFAMILY_IPSEC = 8, }; /* @@ -219,6 +228,7 @@ enum { #ifdef KERNEL_PRIVATE IFNET_CSUM_PARTIAL = 0x00001000, IFNET_CSUM_SUM16 = IFNET_CSUM_PARTIAL, + IFNET_CSUM_ZERO_INVERT = 0x00002000, #endif /* KERNEL_PRIVATE */ IFNET_VLAN_TAGGING = 0x00010000, IFNET_VLAN_MTU = 0x00020000, @@ -238,14 +248,14 @@ typedef u_int32_t ifnet_offload_t; #ifdef KERNEL_PRIVATE #define IFNET_OFFLOADF_BITS \ "\020\1CSUM_IP\2CSUM_TCP\3CSUM_UDP\4CSUM_IP_FRAGS\5IP_FRAGMENT" \ - "\6CSUM_TCPIPV6\7CSUM_UDPIPV6\10IPV6_FRAGMENT\15CSUM_PARTIAL" \ - "\20VLAN_TAGGING\21VLAN_MTU\25MULTIPAGES\26TSO_IPV4\27TSO_IPV6" \ - "\30TXSTATUS\31HW_TIMESTAMP\32SW_TIMESTAMP" + "\6CSUM_TCPIPV6\7CSUM_UDPIPV6\10IPV6_FRAGMENT\15CSUM_PARTIAL" \ + "\16CSUM_ZERO_INVERT\20VLAN_TAGGING\21VLAN_MTU\25MULTIPAGES" \ + "\26TSO_IPV4\27TSO_IPV6\30TXSTATUS\31HW_TIMESTAMP\32SW_TIMESTAMP" #define IFNET_CHECKSUMF \ (IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | \ IFNET_CSUM_FRAGMENT | IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6 | \ - IFNET_CSUM_PARTIAL) + IFNET_CSUM_PARTIAL | IFNET_CSUM_ZERO_INVERT) #define IFNET_TSOF \ (IFNET_TSO_IPV4 | IFNET_TSO_IPV6) @@ -698,6 +708,8 @@ struct ifnet_init_params { /* Valid values for flags */ #define IFNET_INIT_LEGACY 0x1 /* legacy network interface model */ #define IFNET_INIT_INPUT_POLL 0x2 /* opportunistic input polling model */ +#define IFNET_INIT_NX_NOAUTO 0x4 /* do not auto config nexus */ +#define IFNET_INIT_ALLOC_KPI 0x8 /* allocated via the ifnet_alloc() KPI */ /* @typedef ifnet_pre_enqueue_func @@ -747,20 +759,6 @@ typedef void (*ifnet_input_poll_func)(ifnet_t interface, u_int32_t flags, u_int32_t max_count, mbuf_t *first_packet, mbuf_t *last_packet, u_int32_t *cnt, u_int32_t *len); -#ifdef BSD_KERNEL_PRIVATE -struct thread; -typedef errno_t (*ifnet_input_handler_func)(ifnet_t ifp, mbuf_t m_head, - mbuf_t m_tail, const struct ifnet_stat_increment_param *s, - boolean_t poll, struct thread *tp); -typedef errno_t (*ifnet_output_handler_func)(ifnet_t interface, mbuf_t data); - -extern errno_t ifnet_set_input_handler(struct ifnet *ifp, - ifnet_input_handler_func fn); -extern errno_t ifnet_set_output_handler(struct ifnet *ifp, - ifnet_output_handler_func fn); -extern void ifnet_reset_input_handler(struct ifnet *ifp); -extern void ifnet_reset_output_handler(struct ifnet *ifp); -#endif /* BSD_KERNEL_PRIVATE */ /* @enum Interface control commands @abstract Constants defining control commands. @@ -1052,6 +1050,13 @@ typedef errno_t (*ifnet_ctl_func)(ifnet_t interface, ifnet_ctl_cmd_t cmd, @field broadcast_addr The link-layer broadcast address for this interface. @field broadcast_len The length of the link-layer broadcast address. + @field tx_headroom The amount of headroom space to be reserved in the + packet being transmitted on the interface, specified in bytes. + Must be a multiple of 8 bytes. + @field tx_trailer The amount of trailer space to be reserved in the + packet being transmitted on the interface, specified in bytes. + @field rx_mit_ival mitigation interval for the rx mitigation logic, + specified in microseconds. */ struct ifnet_init_eparams { u_int32_t ver; /* required */ @@ -1104,11 +1109,14 @@ struct ifnet_init_eparams { u_int32_t broadcast_len; /* required for non point-to-point interfaces */ ifnet_framer_extended_func framer_extended; /* optional */ ifnet_subfamily_t subfamily; /* optional */ + u_int16_t tx_headroom; /* optional */ + u_int16_t tx_trailer; /* optional */ + u_int32_t rx_mit_ival; /* optional */ #if !defined(__LP64__) - u_int64_t _____reserved[3]; /* for future use */ + u_int64_t ____reserved[2]; /* for future use */ #else - u_int32_t ____reserved; /* pad */ - u_int64_t _____reserved[2]; /* for future use */ + u_int32_t ____reserved; /* for future use */ + u_int64_t _____reserved[1]; /* for future use */ #endif /* __LP64__ */ }; #endif /* KERNEL_PRIVATE */ @@ -1236,8 +1244,16 @@ __BEGIN_DECLS if an interface with the same uniqueid and family has already been allocated and is in use. */ +#ifdef KERNEL_PRIVATE +extern errno_t ifnet_allocate_internal(const struct ifnet_init_params *init, + ifnet_t *interface); + +#define ifnet_allocate(init, interface) \ + ifnet_allocate_internal((init), (interface)) +#else extern errno_t ifnet_allocate(const struct ifnet_init_params *init, ifnet_t *interface); +#endif /* KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE /* @@ -1583,39 +1599,6 @@ extern errno_t ifnet_poll_params(ifnet_t interface, */ extern void ifnet_start(ifnet_t interface); -/* - @function ifnet_transmit_burst_start - @discussion Inform the kernel about the beginning of transmission - of a burst. This function should be called when a burst of - packets are scheduled to get transmitted over the link. The - callback will be used by the system to start measuring - bandwidth available on that link. The driver may choose to - adopt this scheme for uplink bandwidth measurement, in case - the information can't be obtained from the hardware. Else - it may alternatively inform the network stack about the - information using ifnet_set_bandwidths. - @param interface The interface. - @param mbuf_t The first packet in a burst of packets that has been - scheduled to transmit. -*/ -extern void ifnet_transmit_burst_start(ifnet_t interface, mbuf_t pkt); - -/* - @function ifnet_transmit_burst_end - @discussion Inform the kernel about the end of transmission of a burst. - This function should be called when the transmission of a burst - of packets is done. This information will be used by the - system to estimate bandwidth available on that link. The - driver may choose to adopt this scheme for uplink bandwidth - measurement, in case the information can't be obtained from - the hardware. Else it may alternatively inform the network - stack about the information using ifnet_set_bandwidths. - @param interface The interface. - @param mbuf_t The last packet in the burst that has been successfully - transmitted. -*/ -extern void ifnet_transmit_burst_end(ifnet_t interface, mbuf_t pkt); - /* @function ifnet_flowid @discussion Returns the interface flow ID value, which can be used @@ -3237,7 +3220,7 @@ extern errno_t ifnet_get_local_ports(ifnet_t ifp, u_int8_t *bitfield); IFNET_GET_LOCAL_PORTS_EXTBGIDLEONLY: When bit is set, the port is in the list only if the socket has the option SO_EXTENDED_BK_IDLE set - IFNET_GET_LOCAL_PORTS_ACTIVETCPONLY: When bit is set, the + IFNET_GET_LOCAL_PORTS_ACTIVEONLY: When bit is set, the port is in the list only if the socket is not in a final TCP state or the connection is not idle in a final TCP state @param bitfield A pointer to 8192 bytes. @@ -3518,46 +3501,6 @@ extern errno_t ifnet_get_keepalive_offload_frames(ifnet_t ifp, extern errno_t ifnet_link_status_report(ifnet_t ifp, const void *buffer, size_t buffer_len); -/*************************************************************************/ -/* Packet preamble */ -/*************************************************************************/ -/*! - @function ifnet_set_packetpreamblelen - @discussion - Allows a driver to specify a leading space to be - reserved in front of the link layer header. - The preamble is logically adjoining the link layer which - itself is logically contiguous to the network protocol header - (e.g. IP). - There is no guarantee that packets being sent to the - driver has leading space reserved for the preamble. - There is also no guarantee the packet will be laid out in a - contiguous block of memory. - The network protocol header is 32 bit aligned and this dictates - the alignment of the link layer header which in turn affects - the alignment the packet preamble. - This function is intended to be called by the driver. A kext - must not call this function on an interface the kext does not - own. - @param interface The interface. - @param len The length of the packet preamble. - @result 0 on success otherwise the errno error. - */ -extern errno_t ifnet_set_packetpreamblelen(ifnet_t interface, u_int32_t len); - -/*! - @function ifnet_packetpreamblelen - @param interface The interface. - @result The current packet preamble length. - */ -extern u_int32_t ifnet_packetpreamblelen(ifnet_t interface); - -/*! - @function ifnet_maxpacketpreamblelen - @result The maximum packet preamble length supported by the system - */ -extern u_int32_t ifnet_maxpacketpreamblelen(void); - /*************************************************************************/ /* QoS Fastlane */ /*************************************************************************/ diff --git a/bsd/net/kpi_interfacefilter.c b/bsd/net/kpi_interfacefilter.c index 82ccd2d3e..47d03f9d4 100644 --- a/bsd/net/kpi_interfacefilter.c +++ b/bsd/net/kpi_interfacefilter.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003,2013 Apple Inc. All rights reserved. + * Copyright (c) 2003,2013,2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -34,6 +34,26 @@ #include #include +#undef iflt_attach +errno_t +iflt_attach( + ifnet_t interface, + const struct iff_filter *filter, + interface_filter_t *filter_ref); + + +errno_t +iflt_attach_internal( + ifnet_t interface, + const struct iff_filter *filter, + interface_filter_t *filter_ref) +{ + if (interface == NULL) return ENOENT; + + return dlil_attach_filter(interface, filter, filter_ref, + DLIL_IFF_INTERNAL); +} + errno_t iflt_attach( ifnet_t interface, diff --git a/bsd/net/kpi_interfacefilter.h b/bsd/net/kpi_interfacefilter.h index e1ea99aef..e5ac569e1 100644 --- a/bsd/net/kpi_interfacefilter.h +++ b/bsd/net/kpi_interfacefilter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2003,2008,2017 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -204,8 +204,16 @@ struct iff_filter { @param filter_ref A reference to the filter used to detach. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t iflt_attach_internal(ifnet_t interface, const struct iff_filter *filter, + interface_filter_t *filter_ref); + +#define iflt_attach(interface, filter, filter_ref) \ + iflt_attach_internal((interface), (filter), (filter_ref)) +#else extern errno_t iflt_attach(ifnet_t interface, const struct iff_filter *filter, interface_filter_t *filter_ref); +#endif /* KERNEL_PRIVATE */ /*! @function iflt_detach diff --git a/bsd/net/kpi_protocol.c b/bsd/net/kpi_protocol.c index f35b2b10b..c6314269d 100644 --- a/bsd/net/kpi_protocol.c +++ b/bsd/net/kpi_protocol.c @@ -193,7 +193,7 @@ proto_input_run(void) mbuf_t packet_list; int i, locked = 0; - lck_mtx_assert(&inp->input_lck, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&inp->input_lck, LCK_MTX_ASSERT_NOTOWNED); if (inp->input_waiting & DLIL_PROTO_REGISTER) { lck_mtx_lock_spin(&inp->input_lck); diff --git a/bsd/net/ndrv.c b/bsd/net/ndrv.c index 87b6a7749..41603c5b2 100644 --- a/bsd/net/ndrv.c +++ b/bsd/net/ndrv.c @@ -78,8 +78,6 @@ #endif #include -#include - static unsigned int ndrv_multi_max_count = NDRV_DMUX_MAX_DESCR; SYSCTL_UINT(_net, OID_AUTO, ndrv_multi_max_count, CTLFLAG_RW | CTLFLAG_LOCKED, &ndrv_multi_max_count, 0, "Number of allowed multicast addresses per NRDV socket"); @@ -194,7 +192,7 @@ ndrv_input( return EJUSTRETURN; bcopy(frame_header, m->m_data, ifnet_hdrlen(ifp)); - lck_mtx_assert(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(ndrvdomain->dom_mtx); if (sbappendaddr(&(so->so_rcv), (struct sockaddr *)&ndrvsrc, m, (struct mbuf *)0, &error) != 0) { @@ -299,7 +297,7 @@ ndrv_event(struct ifnet *ifp, __unused protocol_family_t protocol, event->kev_class == KEV_NETWORK_CLASS && event->kev_subclass == KEV_DL_SUBCLASS && event->event_code == KEV_DL_IF_DETACHING) { - lck_mtx_assert(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(ndrvdomain->dom_mtx); ndrv_handle_ifp_detach(ifnet_family(ifp), ifnet_unit(ifp)); lck_mtx_unlock(ndrvdomain->dom_mtx); @@ -407,7 +405,7 @@ ndrv_disconnect(struct socket *so) static int ndrv_shutdown(struct socket *so) { - lck_mtx_assert(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_OWNED); socantsendmore(so); return 0; } @@ -866,7 +864,7 @@ ndrv_handle_ifp_detach(u_int32_t family, short unit) so = np->nd_socket; /* Make sure sending returns an error */ - lck_mtx_assert(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_OWNED); socantsendmore(so); socantrcvmore(so); } diff --git a/bsd/net/necp.c b/bsd/net/necp.c index cf7e64dba..22f5afbd5 100644 --- a/bsd/net/necp.c +++ b/bsd/net/necp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -203,6 +205,10 @@ u_int32_t necp_session_count = 0; #define NECP_MAX_POLICY_RESULT_SIZE 512 #define NECP_MAX_ROUTE_RULES_ARRAY_SIZE 1024 #define NECP_MAX_CONDITIONS_ARRAY_SIZE 4096 +#define NECP_MAX_POLICY_LIST_COUNT 1024 + +// Cap the policy size at the max result + conditions size, with room for extra TLVs +#define NECP_MAX_POLICY_SIZE (1024 + NECP_MAX_POLICY_RESULT_SIZE + NECP_MAX_CONDITIONS_ARRAY_SIZE) struct necp_service_registration { LIST_ENTRY(necp_service_registration) session_chain; @@ -211,10 +217,13 @@ struct necp_service_registration { }; struct necp_session { + u_int8_t necp_fd_type; u_int32_t control_unit; u_int32_t session_priority; // Descriptive priority rating u_int32_t session_order; + decl_lck_mtx_data(, lock); + bool proc_locked; // Messages must come from proc_uuid uuid_t proc_uuid; int proc_pid; @@ -223,8 +232,15 @@ struct necp_session { LIST_HEAD(_policies, necp_session_policy) policies; LIST_HEAD(_services, necp_service_registration) services; + + TAILQ_ENTRY(necp_session) chain; }; +#define NECP_SESSION_LOCK(_s) lck_mtx_lock(&_s->lock) +#define NECP_SESSION_UNLOCK(_s) lck_mtx_unlock(&_s->lock) + +static TAILQ_HEAD(_necp_session_list, necp_session) necp_session_list; + struct necp_socket_info { pid_t pid; uid_t uid; @@ -298,16 +314,18 @@ static LIST_HEAD(_necpkernelipoutputpolicies, necp_kernel_ip_output_policy) necp #define NECP_IP_OUTPUT_MAP_ID_TO_BUCKET(id) (id ? (id%(NECP_KERNEL_IP_OUTPUT_POLICIES_MAP_NUM_ID_BUCKETS - 1) + 1) : 0) static struct necp_kernel_ip_output_policy **necp_kernel_ip_output_policies_map[NECP_KERNEL_IP_OUTPUT_POLICIES_MAP_NUM_ID_BUCKETS]; -static struct necp_session *necp_create_session(u_int32_t control_unit); +static struct necp_session *necp_create_session(void); static void necp_delete_session(struct necp_session *session); -static void necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); +static necp_policy_id necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_t packet, + u_int8_t *tlv_buffer, size_t tlv_buffer_length, int offset, int *error); static void necp_handle_policy_get(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_policy_delete(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_policy_apply_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_policy_list_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_policy_delete_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); -static void necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); +static int necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, + user_addr_t out_buffer, size_t out_buffer_length, int offset); static void necp_handle_set_session_priority(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_lock_session_to_proc(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_register_service(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); @@ -381,6 +399,7 @@ static char *necp_create_trimmed_domain(char *string, size_t length); static inline int necp_count_dots(char *string, size_t length); static char *necp_copy_string(char *string, size_t length); +static bool necp_update_qos_marking(struct ifnet *ifp, u_int32_t route_rule_id); #define ROUTE_RULE_IS_AGGREGATE(ruleid) (ruleid > UINT16_MAX) @@ -436,30 +455,654 @@ necp_allocate_new_session_order(u_int32_t priority, u_int32_t control_unit) priority = NECP_SESSION_PRIORITY_DEFAULT; } - // Use the control unit to decide the offset into the priority list - new_order = (control_unit) + ((priority - 1) * 1000); - - return (new_order); -} - -static inline u_int32_t -necp_get_first_order_for_priority(u_int32_t priority) -{ - return (((priority - 1) * 1000) + 1); -} + // Use the control unit to decide the offset into the priority list + new_order = (control_unit) + ((priority - 1) * 1000); + + return (new_order); +} + +static inline u_int32_t +necp_get_first_order_for_priority(u_int32_t priority) +{ + return (((priority - 1) * 1000) + 1); +} + +// Sysctl handler +static int +sysctl_handle_necp_level SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); + if (necp_drop_all_level == 0) { + necp_drop_all_order = 0; + } else { + necp_drop_all_order = necp_get_first_order_for_priority(necp_drop_all_level); + } + return (error); +} + +// Session fd + +static int noop_read(struct fileproc *, struct uio *, int, vfs_context_t); +static int noop_write(struct fileproc *, struct uio *, int, vfs_context_t); +static int noop_ioctl(struct fileproc *, unsigned long, caddr_t, + vfs_context_t); +static int noop_select(struct fileproc *, int, void *, vfs_context_t); +static int necp_session_op_close(struct fileglob *, vfs_context_t); +static int noop_kqfilter(struct fileproc *, struct knote *, + struct kevent_internal_s *, vfs_context_t); + +static const struct fileops necp_session_fd_ops = { + .fo_type = DTYPE_NETPOLICY, + .fo_read = noop_read, + .fo_write = noop_write, + .fo_ioctl = noop_ioctl, + .fo_select = noop_select, + .fo_close = necp_session_op_close, + .fo_kqfilter = noop_kqfilter, + .fo_drain = NULL, +}; + +static int +noop_read(struct fileproc *fp, struct uio *uio, int flags, vfs_context_t ctx) +{ +#pragma unused(fp, uio, flags, ctx) + return (ENXIO); +} + +static int +noop_write(struct fileproc *fp, struct uio *uio, int flags, + vfs_context_t ctx) +{ +#pragma unused(fp, uio, flags, ctx) + return (ENXIO); +} + +static int +noop_ioctl(struct fileproc *fp, unsigned long com, caddr_t data, + vfs_context_t ctx) +{ +#pragma unused(fp, com, data, ctx) + return (ENOTTY); +} + +static int +noop_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx) +{ +#pragma unused(fp, which, wql, ctx) + return (ENXIO); +} + +static int +noop_kqfilter(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx) +{ +#pragma unused(fp, kn, kev, ctx) + return (ENXIO); +} + +int +necp_session_open(struct proc *p, struct necp_session_open_args *uap, int *retval) +{ +#pragma unused(uap) + int error = 0; + struct necp_session *session = NULL; + struct fileproc *fp = NULL; + int fd = -1; + + uid_t uid = kauth_cred_getuid(proc_ucred(p)); + if (uid != 0 && priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NECP_POLICIES, 0) != 0) { + NECPLOG0(LOG_ERR, "Process does not hold necessary entitlement to open NECP session"); + error = EACCES; + goto done; + } + + error = falloc(p, &fp, &fd, vfs_context_current()); + if (error != 0) { + goto done; + } + + session = necp_create_session(); + if (session == NULL) { + error = ENOMEM; + goto done; + } + + fp->f_fglob->fg_flag = 0; + fp->f_fglob->fg_ops = &necp_session_fd_ops; + fp->f_fglob->fg_data = session; + + proc_fdlock(p); + FDFLAGS_SET(p, fd, (UF_EXCLOSE | UF_FORKCLOSE)); + procfdtbl_releasefd(p, fd, NULL); + fp_drop(p, fd, fp, 1); + proc_fdunlock(p); + + *retval = fd; +done: + if (error != 0) { + if (fp != NULL) { + fp_free(p, fd, fp); + fp = NULL; + } + } + + return (error); +} + +static int +necp_session_op_close(struct fileglob *fg, vfs_context_t ctx) +{ +#pragma unused(ctx) + struct necp_session *session = (struct necp_session *)fg->fg_data; + fg->fg_data = NULL; + + if (session != NULL) { + necp_policy_mark_all_for_deletion(session); + necp_policy_apply_all(session); + necp_delete_session(session); + return (0); + } else { + return (ENOENT); + } +} + +static int +necp_session_find_from_fd(int fd, struct necp_session **session) +{ + proc_t p = current_proc(); + struct fileproc *fp = NULL; + int error = 0; + + proc_fdlock_spin(p); + if ((error = fp_lookup(p, fd, &fp, 1)) != 0) { + goto done; + } + if (fp->f_fglob->fg_ops->fo_type != DTYPE_NETPOLICY) { + fp_drop(p, fd, fp, 1); + error = ENODEV; + goto done; + } + *session = (struct necp_session *)fp->f_fglob->fg_data; + +done: + proc_fdunlock(p); + return (error); +} + +static int +necp_session_add_policy(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + u_int8_t *tlv_buffer = NULL; + + if (uap->in_buffer_length == 0 || uap->in_buffer_length > NECP_MAX_POLICY_SIZE || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_add_policy invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + if (uap->out_buffer_length < sizeof(necp_policy_id) || uap->out_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_add_policy invalid output buffer (%zu)", uap->out_buffer_length); + error = EINVAL; + goto done; + } + + if ((tlv_buffer = _MALLOC(uap->in_buffer_length, M_NECP, M_WAITOK | M_ZERO)) == NULL) { + error = ENOMEM; + goto done; + } + + error = copyin(uap->in_buffer, tlv_buffer, uap->in_buffer_length); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_add_policy tlv copyin error (%d)", error); + goto done; + } + + necp_policy_id new_policy_id = necp_handle_policy_add(session, 0, NULL, tlv_buffer, uap->in_buffer_length, 0, &error); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_add_policy failed to add policy (%d)", error); + goto done; + } + + error = copyout(&new_policy_id, uap->out_buffer, sizeof(new_policy_id)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_add_policy policy_id copyout error (%d)", error); + goto done; + } + +done: + if (tlv_buffer != NULL) { + FREE(tlv_buffer, M_NECP); + tlv_buffer = NULL; + } + *retval = error; + + return (error); +} + +static int +necp_session_get_policy(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + u_int8_t *response = NULL; + + if (uap->in_buffer_length < sizeof(necp_policy_id) || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_get_policy invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + necp_policy_id policy_id = 0; + error = copyin(uap->in_buffer, &policy_id, sizeof(policy_id)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_get_policy policy_id copyin error (%d)", error); + goto done; + } + + struct necp_session_policy *policy = necp_policy_find(session, policy_id); + if (policy == NULL || policy->pending_deletion) { + NECPLOG(LOG_ERR, "Failed to find policy with id %d", policy_id); + error = ENOENT; + goto done; + } + + u_int32_t order_tlv_size = sizeof(u_int8_t) + sizeof(u_int32_t) + sizeof(necp_policy_order); + u_int32_t result_tlv_size = (policy->result_size ? (sizeof(u_int8_t) + sizeof(u_int32_t) + policy->result_size) : 0); + u_int32_t response_size = order_tlv_size + result_tlv_size + policy->conditions_size; + + if (uap->out_buffer_length < response_size || uap->out_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_get_policy buffer not large enough (%u < %u)", uap->out_buffer_length, response_size); + error = EINVAL; + goto done; + } + + if (response_size > NECP_MAX_POLICY_SIZE) { + NECPLOG(LOG_ERR, "necp_session_get_policy size too large to copy (%u)", response_size); + error = EINVAL; + goto done; + } + + MALLOC(response, u_int8_t *, response_size, M_NECP, M_WAITOK | M_ZERO); + if (response == NULL) { + error = ENOMEM; + goto done; + } + + u_int8_t *cursor = response; + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ORDER, sizeof(necp_policy_order), &policy->order, response, response_size); + if (result_tlv_size) { + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_RESULT, policy->result_size, &policy->result, response, response_size); + } + if (policy->conditions_size) { + memcpy(((u_int8_t *)(void *)(cursor)), policy->conditions, policy->conditions_size); + } + + error = copyout(response, uap->out_buffer, response_size); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_get_policy TLV copyout error (%d)", error); + goto done; + } + +done: + if (response != NULL) { + FREE(response, M_NECP); + response = NULL; + } + *retval = error; + + return (error); +} + +static int +necp_session_delete_policy(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + + if (uap->in_buffer_length < sizeof(necp_policy_id) || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_delete_policy invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + necp_policy_id delete_policy_id = 0; + error = copyin(uap->in_buffer, &delete_policy_id, sizeof(delete_policy_id)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_delete_policy policy_id copyin error (%d)", error); + goto done; + } + + struct necp_session_policy *policy = necp_policy_find(session, delete_policy_id); + if (policy == NULL || policy->pending_deletion) { + NECPLOG(LOG_ERR, "necp_session_delete_policy failed to find policy with id %u", delete_policy_id); + error = ENOENT; + goto done; + } + + necp_policy_mark_for_deletion(session, policy); +done: + *retval = error; + return (error); +} + +static int +necp_session_apply_all(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ +#pragma unused(uap) + necp_policy_apply_all(session); + *retval = 0; + return (0); +} + +static int +necp_session_list_all(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + u_int32_t tlv_size = (sizeof(u_int8_t) + sizeof(u_int32_t) + sizeof(necp_policy_id)); + u_int32_t response_size = 0; + u_int8_t *response = NULL; + int num_policies = 0; + int cur_policy_index = 0; + int error = 0; + struct necp_session_policy *policy; + + LIST_FOREACH(policy, &session->policies, chain) { + if (!policy->pending_deletion) { + num_policies++; + } + } + + if (num_policies > NECP_MAX_POLICY_LIST_COUNT) { + NECPLOG(LOG_ERR, "necp_session_list_all size too large to copy (%u policies)", num_policies); + error = EINVAL; + goto done; + } + + response_size = num_policies * tlv_size; + if (uap->out_buffer_length < response_size || uap->out_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_list_all buffer not large enough (%u < %u)", uap->out_buffer_length, response_size); + error = EINVAL; + goto done; + } + + // Create a response with one Policy ID TLV for each policy + MALLOC(response, u_int8_t *, response_size, M_NECP, M_WAITOK | M_ZERO); + if (response == NULL) { + error = ENOMEM; + goto done; + } + + u_int8_t *cursor = response; + LIST_FOREACH(policy, &session->policies, chain) { + if (!policy->pending_deletion && cur_policy_index < num_policies) { + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(u_int32_t), &policy->id, response, response_size); + cur_policy_index++; + } + } + + error = copyout(response, uap->out_buffer, response_size); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_list_all TLV copyout error (%d)", error); + goto done; + } + +done: + if (response != NULL) { + FREE(response, M_NECP); + response = NULL; + } + *retval = error; + + return (error); +} + + +static int +necp_session_delete_all(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ +#pragma unused(uap) + necp_policy_mark_all_for_deletion(session); + *retval = 0; + return (0); +} + +static int +necp_session_set_session_priority(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + struct necp_session_policy *policy = NULL; + struct necp_session_policy *temp_policy = NULL; + + if (uap->in_buffer_length < sizeof(necp_session_priority) || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_set_session_priority invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + necp_session_priority requested_session_priority = 0; + error = copyin(uap->in_buffer, &requested_session_priority, sizeof(requested_session_priority)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_set_session_priority priority copyin error (%d)", error); + goto done; + } + + // Enforce special session priorities with entitlements + if (requested_session_priority == NECP_SESSION_PRIORITY_CONTROL || + requested_session_priority == NECP_SESSION_PRIORITY_PRIVILEGED_TUNNEL) { + errno_t cred_result = priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NECP_POLICIES, 0); + if (cred_result != 0) { + NECPLOG(LOG_ERR, "Session does not hold necessary entitlement to claim priority level %d", requested_session_priority); + error = EPERM; + goto done; + } + } + + if (session->session_priority != requested_session_priority) { + session->session_priority = requested_session_priority; + session->session_order = necp_allocate_new_session_order(session->session_priority, session->control_unit); + session->dirty = TRUE; + + // Mark all policies as needing updates + LIST_FOREACH_SAFE(policy, &session->policies, chain, temp_policy) { + policy->pending_update = TRUE; + } + } + +done: + *retval = error; + return (error); +} + +static int +necp_session_lock_to_process(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ +#pragma unused(uap) + session->proc_locked = TRUE; + *retval = 0; + return (0); +} + +static int +necp_session_register_service(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + struct necp_service_registration *new_service = NULL; + + if (uap->in_buffer_length < sizeof(uuid_t) || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_register_service invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + uuid_t service_uuid; + error = copyin(uap->in_buffer, service_uuid, sizeof(service_uuid)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_register_service uuid copyin error (%d)", error); + goto done; + } + + MALLOC(new_service, struct necp_service_registration *, sizeof(*new_service), M_NECP, M_WAITOK | M_ZERO); + if (new_service == NULL) { + NECPLOG0(LOG_ERR, "Failed to allocate service registration"); + error = ENOMEM; + goto done; + } + + lck_rw_lock_exclusive(&necp_kernel_policy_lock); + new_service->service_id = necp_create_uuid_service_id_mapping(service_uuid); + LIST_INSERT_HEAD(&session->services, new_service, session_chain); + LIST_INSERT_HEAD(&necp_registered_service_list, new_service, kernel_chain); + lck_rw_done(&necp_kernel_policy_lock); + +done: + *retval = error; + return (error); +} + +static int +necp_session_unregister_service(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + struct necp_service_registration *service = NULL; + struct necp_service_registration *temp_service = NULL; + struct necp_uuid_id_mapping *mapping = NULL; + + if (uap->in_buffer_length < sizeof(uuid_t) || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_unregister_service invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + uuid_t service_uuid; + error = copyin(uap->in_buffer, service_uuid, sizeof(service_uuid)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_unregister_service uuid copyin error (%d)", error); + goto done; + } + + // Remove all matching services for this session + lck_rw_lock_exclusive(&necp_kernel_policy_lock); + mapping = necp_uuid_lookup_service_id_locked(service_uuid); + if (mapping != NULL) { + LIST_FOREACH_SAFE(service, &session->services, session_chain, temp_service) { + if (service->service_id == mapping->id) { + LIST_REMOVE(service, session_chain); + LIST_REMOVE(service, kernel_chain); + FREE(service, M_NECP); + } + } + necp_remove_uuid_service_id_mapping(service_uuid); + } + lck_rw_done(&necp_kernel_policy_lock); + +done: + *retval = error; + return (error); +} + +static int +necp_session_dump_all(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + + if (uap->out_buffer_length == 0 || uap->out_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_dump_all invalid output buffer (%zu)", uap->out_buffer_length); + error = EINVAL; + goto done; + } + + error = necp_handle_policy_dump_all(session, 0, NULL, uap->out_buffer, uap->out_buffer_length, 0); +done: + *retval = error; + return (error); +} + +int +necp_session_action(struct proc *p, struct necp_session_action_args *uap, int *retval) +{ +#pragma unused(p) + int error = 0; + int return_value = 0; + struct necp_session *session = NULL; + error = necp_session_find_from_fd(uap->necp_fd, &session); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_action find fd error (%d)", error); + return (error); + } + + NECP_SESSION_LOCK(session); + + if (session->proc_locked) { + // Verify that the calling process is allowed to do actions + uuid_t proc_uuid; + proc_getexecutableuuid(current_proc(), proc_uuid, sizeof(proc_uuid)); + if (uuid_compare(proc_uuid, session->proc_uuid) != 0) { + error = EPERM; + goto done; + } + } else { + // If not locked, update the proc_uuid and proc_pid of the session + proc_getexecutableuuid(current_proc(), session->proc_uuid, sizeof(session->proc_uuid)); + session->proc_pid = proc_pid(current_proc()); + } + + u_int32_t action = uap->action; + switch (action) { + case NECP_SESSION_ACTION_POLICY_ADD: { + return_value = necp_session_add_policy(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_GET: { + return_value = necp_session_get_policy(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_DELETE: { + return_value = necp_session_delete_policy(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_APPLY_ALL: { + return_value = necp_session_apply_all(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_LIST_ALL: { + return_value = necp_session_list_all(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_DELETE_ALL: { + return_value = necp_session_delete_all(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_SET_SESSION_PRIORITY: { + return_value = necp_session_set_session_priority(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_LOCK_SESSION_TO_PROC: { + return_value = necp_session_lock_to_process(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_REGISTER_SERVICE: { + return_value = necp_session_register_service(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_UNREGISTER_SERVICE: { + return_value = necp_session_unregister_service(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_DUMP_ALL: { + return_value = necp_session_dump_all(session, uap, retval); + break; + } + default: { + NECPLOG(LOG_ERR, "necp_session_action unknown action (%u)", action); + return_value = EINVAL; + break; + } + } + +done: + NECP_SESSION_UNLOCK(session); + file_drop(uap->necp_fd); -// Sysctl handler -static int -sysctl_handle_necp_level SYSCTL_HANDLER_ARGS -{ -#pragma unused(arg1, arg2) - int error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); - if (necp_drop_all_level == 0) { - necp_drop_all_order = 0; - } else { - necp_drop_all_order = necp_get_first_order_for_priority(necp_drop_all_level); - } - return (error); + return (return_value); } // Kernel Control functions @@ -531,6 +1174,8 @@ necp_init(void) necp_client_init(); + TAILQ_INIT(&necp_session_list); + LIST_INIT(&necp_kernel_socket_policies); LIST_INIT(&necp_kernel_ip_output_policies); @@ -664,8 +1309,8 @@ necp_post_change_event(struct kev_necp_policies_changed_data *necp_event_data) static errno_t necp_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, void **unitinfo) { -#pragma unused(kctlref) - *unitinfo = necp_create_session(sac->sc_unit); +#pragma unused(kctlref, sac) + *unitinfo = necp_create_session(); if (*unitinfo == NULL) { // Could not allocate session return (ENOBUFS); @@ -763,20 +1408,6 @@ necp_packet_get_tlv_at_offset(mbuf_t packet, int tlv_offset, u_int32_t buff_len, return (0); } -static int -necp_packet_get_tlv(mbuf_t packet, int offset, u_int8_t type, u_int32_t buff_len, void *buff, u_int32_t *value_size) -{ - int error = 0; - int tlv_offset; - - tlv_offset = necp_packet_find_tlv(packet, offset, type, &error, 0); - if (tlv_offset < 0) { - return (error); - } - - return (necp_packet_get_tlv_at_offset(packet, tlv_offset, buff_len, buff, value_size)); -} - static u_int8_t * necp_buffer_write_packet_header(u_int8_t *buffer, u_int8_t packet_type, u_int8_t flags, u_int32_t message_id) { @@ -786,41 +1417,66 @@ necp_buffer_write_packet_header(u_int8_t *buffer, u_int8_t packet_type, u_int8_t return (buffer + sizeof(struct necp_packet_header)); } +static inline bool +necp_buffer_write_tlv_validate(u_int8_t *cursor, u_int8_t type, u_int32_t length, + u_int8_t *buffer, u_int32_t buffer_length) +{ + if (cursor < buffer || (uintptr_t)(cursor - buffer) > buffer_length) { + NECPLOG0(LOG_ERR, "Cannot write TLV in buffer (invalid cursor)"); + return (false); + } + u_int8_t *next_tlv = (u_int8_t *)(cursor + sizeof(type) + sizeof(length) + length); + if (next_tlv <= buffer || // make sure the next TLV start doesn't overflow + (uintptr_t)(next_tlv - buffer) > buffer_length) { // make sure the next TLV has enough room in buffer + NECPLOG(LOG_ERR, "Cannot write TLV in buffer (TLV length %u, buffer length %u)", + length, buffer_length); + return (false); + } + return (true); +} u_int8_t * -necp_buffer_write_tlv_if_different(u_int8_t *buffer, const u_int8_t *max, u_int8_t type, - u_int32_t length, const void *value, bool *updated) +necp_buffer_write_tlv_if_different(u_int8_t *cursor, u_int8_t type, + u_int32_t length, const void *value, bool *updated, + u_int8_t *buffer, u_int32_t buffer_length) { - u_int8_t *next_tlv = (u_int8_t *)(buffer + sizeof(type) + sizeof(length) + length); - if (next_tlv <= max) { - if (*updated || *(u_int8_t *)(buffer) != type) { - *(u_int8_t *)(buffer) = type; - *updated = TRUE; - } - if (*updated || *(u_int32_t *)(void *)(buffer + sizeof(type)) != length) { - *(u_int32_t *)(void *)(buffer + sizeof(type)) = length; + if (!necp_buffer_write_tlv_validate(cursor, type, length, buffer, buffer_length)) { + return (NULL); + } + u_int8_t *next_tlv = (u_int8_t *)(cursor + sizeof(type) + sizeof(length) + length); + if (*updated || *(u_int8_t *)(cursor) != type) { + *(u_int8_t *)(cursor) = type; + *updated = TRUE; + } + if (*updated || *(u_int32_t *)(void *)(cursor + sizeof(type)) != length) { + *(u_int32_t *)(void *)(cursor + sizeof(type)) = length; + *updated = TRUE; + } + if (length > 0) { + if (*updated || memcmp((u_int8_t *)(cursor + sizeof(type) + sizeof(length)), value, length) != 0) { + memcpy((u_int8_t *)(cursor + sizeof(type) + sizeof(length)), value, length); *updated = TRUE; } - if (length > 0) { - if (*updated || memcmp((u_int8_t *)(buffer + sizeof(type) + sizeof(length)), value, length) != 0) { - memcpy((u_int8_t *)(buffer + sizeof(type) + sizeof(length)), value, length); - *updated = TRUE; - } - } } return (next_tlv); } u_int8_t * -necp_buffer_write_tlv(u_int8_t *buffer, u_int8_t type, u_int32_t length, const void *value) +necp_buffer_write_tlv(u_int8_t *cursor, u_int8_t type, + u_int32_t length, const void *value, + u_int8_t *buffer, u_int32_t buffer_length) { - *(u_int8_t *)(buffer) = type; - *(u_int32_t *)(void *)(buffer + sizeof(type)) = length; + if (!necp_buffer_write_tlv_validate(cursor, type, length, buffer, buffer_length)) { + return (NULL); + } + u_int8_t *next_tlv = (u_int8_t *)(cursor + sizeof(type) + sizeof(length) + length); + *(u_int8_t *)(cursor) = type; + *(u_int32_t *)(void *)(cursor + sizeof(type)) = length; if (length > 0) { - memcpy((u_int8_t *)(buffer + sizeof(type) + sizeof(length)), value, length); + memcpy((u_int8_t *)(cursor + sizeof(type) + sizeof(length)), value, length); } - return ((u_int8_t *)(buffer + sizeof(type) + sizeof(length) + length)); + return (next_tlv); } u_int8_t @@ -905,6 +1561,90 @@ necp_buffer_find_tlv(u_int8_t *buffer, u_int32_t buffer_length, int offset, u_in } } +static int +necp_find_tlv(mbuf_t packet, u_int8_t *buffer, u_int32_t buffer_length, int offset, u_int8_t type, int *err, int next) +{ + int cursor = -1; + if (packet != NULL) { + cursor = necp_packet_find_tlv(packet, offset, type, err, next); + } else if (buffer != NULL) { + cursor = necp_buffer_find_tlv(buffer, buffer_length, offset, type, next); + } + return (cursor); +} + +static int +necp_get_tlv_at_offset(mbuf_t packet, u_int8_t *buffer, u_int32_t buffer_length, + int tlv_offset, u_int32_t out_buffer_length, void *out_buffer, u_int32_t *value_size) +{ + if (packet != NULL) { + // Handle mbuf parsing + return necp_packet_get_tlv_at_offset(packet, tlv_offset, out_buffer_length, out_buffer, value_size); + } + + if (buffer == NULL) { + NECPLOG0(LOG_ERR, "necp_get_tlv_at_offset buffer is NULL"); + return (EINVAL); + } + + // Handle buffer parsing + + // Validate that buffer has enough room for any TLV + if (tlv_offset + sizeof(u_int8_t) + sizeof(u_int32_t) > buffer_length) { + NECPLOG(LOG_ERR, "necp_get_tlv_at_offset buffer_length is too small for TLV (%u < %u)", + buffer_length, tlv_offset + sizeof(u_int8_t) + sizeof(u_int32_t)); + return (EINVAL); + } + + // Validate that buffer has enough room for this TLV + u_int32_t tlv_length = necp_buffer_get_tlv_length(buffer, tlv_offset); + if (tlv_length > buffer_length - (tlv_offset + sizeof(u_int8_t) + sizeof(u_int32_t))) { + NECPLOG(LOG_ERR, "necp_get_tlv_at_offset buffer_length is too small for TLV of length %u (%u < %u)", + tlv_length, buffer_length, tlv_offset + sizeof(u_int8_t) + sizeof(u_int32_t) + tlv_length); + return (EINVAL); + } + + if (out_buffer != NULL && out_buffer_length > 0) { + // Validate that out buffer is large enough for value + if (out_buffer_length < tlv_length) { + NECPLOG(LOG_ERR, "necp_get_tlv_at_offset out_buffer_length is too small for TLV value (%u < %u)", + out_buffer_length, tlv_length); + return (EINVAL); + } + + // Get value pointer + u_int8_t *tlv_value = necp_buffer_get_tlv_value(buffer, tlv_offset, NULL); + if (tlv_value == NULL) { + NECPLOG0(LOG_ERR, "necp_get_tlv_at_offset tlv_value is NULL"); + return (ENOENT); + } + + // Copy value + memcpy(out_buffer, tlv_value, tlv_length); + } + + // Copy out length + if (value_size != NULL) { + *value_size = tlv_length; + } + + return (0); +} + +static int +necp_get_tlv(mbuf_t packet, u_int8_t *buffer, u_int32_t buffer_length, + int offset, u_int8_t type, u_int32_t buff_len, void *buff, u_int32_t *value_size) +{ + int error = 0; + + int tlv_offset = necp_find_tlv(packet, buffer, buffer_length, offset, type, &error, 0); + if (tlv_offset < 0) { + return (error); + } + + return (necp_get_tlv_at_offset(packet, buffer, buffer_length, tlv_offset, buff_len, buff, value_size)); +} + static bool necp_send_ctl_data(struct necp_session *session, u_int8_t *buffer, size_t buffer_size) { @@ -932,7 +1672,7 @@ necp_send_success_response(struct necp_session *session, u_int8_t packet_type, u } cursor = response; cursor = necp_buffer_write_packet_header(cursor, packet_type, NECP_PACKET_FLAGS_RESPONSE, message_id); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_NIL, 0, NULL); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_NIL, 0, NULL, response, response_size); if (!(success = necp_send_ctl_data(session, (u_int8_t *)response, response_size))) { NECPLOG0(LOG_ERR, "Failed to send response"); @@ -955,7 +1695,7 @@ necp_send_error_response(struct necp_session *session, u_int8_t packet_type, u_i } cursor = response; cursor = necp_buffer_write_packet_header(cursor, packet_type, NECP_PACKET_FLAGS_RESPONSE, message_id); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ERROR, sizeof(error), &error); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ERROR, sizeof(error), &error, response, response_size); if (!(success = necp_send_ctl_data(session, (u_int8_t *)response, response_size))) { NECPLOG0(LOG_ERR, "Failed to send response"); @@ -978,7 +1718,7 @@ necp_send_policy_id_response(struct necp_session *session, u_int8_t packet_type, } cursor = response; cursor = necp_buffer_write_packet_header(cursor, packet_type, NECP_PACKET_FLAGS_RESPONSE, message_id); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id, response, response_size); if (!(success = necp_send_ctl_data(session, (u_int8_t *)response, response_size))) { NECPLOG0(LOG_ERR, "Failed to send response"); @@ -1031,7 +1771,7 @@ necp_ctl_send(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, mbuf_t packe switch (header.packet_type) { case NECP_PACKET_TYPE_POLICY_ADD: { - necp_handle_policy_add(session, header.message_id, packet, sizeof(header)); + necp_handle_policy_add(session, header.message_id, packet, NULL, 0, sizeof(header), NULL); break; } case NECP_PACKET_TYPE_POLICY_GET: { @@ -1055,7 +1795,7 @@ necp_ctl_send(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, mbuf_t packe break; } case NECP_PACKET_TYPE_POLICY_DUMP_ALL: { - necp_handle_policy_dump_all(session, header.message_id, packet, sizeof(header)); + necp_handle_policy_dump_all(session, header.message_id, packet, 0, 0, sizeof(header)); break; } case NECP_PACKET_TYPE_SET_SESSION_PRIORITY: { @@ -1108,29 +1848,55 @@ necp_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int opt, v } // Session Management + static struct necp_session * -necp_create_session(u_int32_t control_unit) +necp_create_session(void) { struct necp_session *new_session = NULL; - MALLOC(new_session, struct necp_session *, sizeof(*new_session), M_NECP, M_WAITOK); + MALLOC(new_session, struct necp_session *, sizeof(*new_session), M_NECP, M_WAITOK | M_ZERO); if (new_session == NULL) { goto done; } - if (necp_debug) { - NECPLOG(LOG_DEBUG, "Create NECP session, control unit %d", control_unit); - } - memset(new_session, 0, sizeof(*new_session)); + + new_session->necp_fd_type = necp_fd_type_session; new_session->session_priority = NECP_SESSION_PRIORITY_UNKNOWN; - new_session->session_order = necp_allocate_new_session_order(new_session->session_priority, control_unit); - new_session->control_unit = control_unit; new_session->dirty = FALSE; LIST_INIT(&new_session->policies); + lck_mtx_init(&new_session->lock, necp_kernel_policy_mtx_grp, necp_kernel_policy_mtx_attr); + // Take the lock lck_rw_lock_exclusive(&necp_kernel_policy_lock); + + // Find the next available control unit + u_int32_t control_unit = 1; + struct necp_session *next_session = NULL; + TAILQ_FOREACH(next_session, &necp_session_list, chain) { + if (next_session->control_unit > control_unit) { + // Found a gap, grab this control unit + break; + } + + // Try the next control unit, loop around + control_unit = next_session->control_unit + 1; + } + + new_session->control_unit = control_unit; + new_session->session_order = necp_allocate_new_session_order(new_session->session_priority, control_unit); + + if (next_session != NULL) { + TAILQ_INSERT_BEFORE(next_session, new_session, chain); + } else { + TAILQ_INSERT_TAIL(&necp_session_list, new_session, chain); + } + necp_session_count++; lck_rw_done(&necp_kernel_policy_lock); + if (necp_debug) { + NECPLOG(LOG_DEBUG, "Created NECP session, control unit %d", control_unit); + } + done: return (new_session); } @@ -1151,11 +1917,14 @@ necp_delete_session(struct necp_session *session) if (necp_debug) { NECPLOG0(LOG_DEBUG, "Deleted NECP session"); } - FREE(session, M_NECP); lck_rw_lock_exclusive(&necp_kernel_policy_lock); + TAILQ_REMOVE(&necp_session_list, session, chain); necp_session_count--; lck_rw_done(&necp_kernel_policy_lock); + + lck_mtx_destroy(&session->lock, necp_kernel_policy_mtx_grp); + FREE(session, M_NECP); } } @@ -1467,6 +2236,30 @@ necp_policy_route_rule_is_valid(u_int8_t *buffer, u_int32_t length) return (validated); } +static int +necp_get_posix_error_for_necp_error(int response_error) +{ + switch (response_error) { + case NECP_ERROR_UNKNOWN_PACKET_TYPE: + case NECP_ERROR_INVALID_TLV: + case NECP_ERROR_POLICY_RESULT_INVALID: + case NECP_ERROR_POLICY_CONDITIONS_INVALID: + case NECP_ERROR_ROUTE_RULES_INVALID: { + return (EINVAL); + } + case NECP_ERROR_POLICY_ID_NOT_FOUND: { + return (ENOENT); + } + case NECP_ERROR_INVALID_PROCESS: { + return (EPERM); + } + case NECP_ERROR_INTERNAL: + default: { + return (ENOMEM); + } + } +} + static void necp_handle_set_session_priority(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset) { @@ -1477,7 +2270,7 @@ necp_handle_set_session_priority(struct necp_session *session, u_int32_t message u_int32_t requested_session_priority = NECP_SESSION_PRIORITY_UNKNOWN; // Read policy id - error = necp_packet_get_tlv(packet, offset, NECP_TLV_SESSION_PRIORITY, sizeof(requested_session_priority), &requested_session_priority, NULL); + error = necp_get_tlv(packet, NULL, 0, offset, NECP_TLV_SESSION_PRIORITY, sizeof(requested_session_priority), &requested_session_priority, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get session priority: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1550,7 +2343,7 @@ necp_handle_register_service(struct necp_session *session, u_int32_t message_id, } // Read service uuid - error = necp_packet_get_tlv(packet, offset, NECP_TLV_SERVICE_UUID, sizeof(uuid_t), service_uuid, NULL); + error = necp_get_tlv(packet, NULL, 0, offset, NECP_TLV_SERVICE_UUID, sizeof(uuid_t), service_uuid, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get service UUID: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1595,7 +2388,7 @@ necp_handle_unregister_service(struct necp_session *session, u_int32_t message_i } // Read service uuid - error = necp_packet_get_tlv(packet, offset, NECP_TLV_SERVICE_UUID, sizeof(uuid_t), service_uuid, NULL); + error = necp_get_tlv(packet, NULL, 0, offset, NECP_TLV_SERVICE_UUID, sizeof(uuid_t), service_uuid, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get service UUID: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1623,8 +2416,9 @@ fail: necp_send_error_response(session, NECP_PACKET_TYPE_UNREGISTER_SERVICE, message_id, response_error); } -static void -necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset) +static necp_policy_id +necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_t packet, + u_int8_t *tlv_buffer, size_t tlv_buffer_length, int offset, int *return_error) { bool has_default_condition = FALSE; bool has_non_default_condition = FALSE; @@ -1651,7 +2445,7 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ u_int32_t policy_result_size = 0; // Read policy order - error = necp_packet_get_tlv(packet, offset, NECP_TLV_POLICY_ORDER, sizeof(order), &order, NULL); + error = necp_get_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_POLICY_ORDER, sizeof(order), &order, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get policy order: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1659,8 +2453,8 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ } // Read policy result - cursor = necp_packet_find_tlv(packet, offset, NECP_TLV_POLICY_RESULT, &error, 0); - error = necp_packet_get_tlv_at_offset(packet, cursor, 0, NULL, &policy_result_size); + cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_POLICY_RESULT, &error, 0); + error = necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, 0, NULL, &policy_result_size); if (error || policy_result_size == 0) { NECPLOG(LOG_ERR, "Failed to get policy result length: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1677,7 +2471,7 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ response_error = NECP_ERROR_INTERNAL; goto fail; } - error = necp_packet_get_tlv_at_offset(packet, cursor, policy_result_size, policy_result, NULL); + error = necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, policy_result_size, policy_result, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get policy result: %d", error); response_error = NECP_ERROR_POLICY_RESULT_INVALID; @@ -1691,11 +2485,11 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ if (necp_policy_result_requires_route_rules(policy_result, policy_result_size)) { // Read route rules conditions - for (cursor = necp_packet_find_tlv(packet, offset, NECP_TLV_ROUTE_RULE, &error, 0); + for (cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_ROUTE_RULE, &error, 0); cursor >= 0; - cursor = necp_packet_find_tlv(packet, cursor, NECP_TLV_ROUTE_RULE, &error, 1)) { + cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, cursor, NECP_TLV_ROUTE_RULE, &error, 1)) { u_int32_t route_rule_size = 0; - necp_packet_get_tlv_at_offset(packet, cursor, 0, NULL, &route_rule_size); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, 0, NULL, &route_rule_size); if (route_rule_size > 0) { route_rules_array_size += (sizeof(u_int8_t) + sizeof(u_int32_t) + route_rule_size); } @@ -1719,12 +2513,12 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ } route_rules_array_cursor = 0; - for (cursor = necp_packet_find_tlv(packet, offset, NECP_TLV_ROUTE_RULE, &error, 0); + for (cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_ROUTE_RULE, &error, 0); cursor >= 0; - cursor = necp_packet_find_tlv(packet, cursor, NECP_TLV_ROUTE_RULE, &error, 1)) { + cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, cursor, NECP_TLV_ROUTE_RULE, &error, 1)) { u_int8_t route_rule_type = NECP_TLV_ROUTE_RULE; u_int32_t route_rule_size = 0; - necp_packet_get_tlv_at_offset(packet, cursor, 0, NULL, &route_rule_size); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, 0, NULL, &route_rule_size); if (route_rule_size > 0 && route_rule_size <= (route_rules_array_size - route_rules_array_cursor)) { // Add type memcpy((route_rules_array + route_rules_array_cursor), &route_rule_type, sizeof(route_rule_type)); @@ -1735,7 +2529,7 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ route_rules_array_cursor += sizeof(route_rule_size); // Add value - necp_packet_get_tlv_at_offset(packet, cursor, route_rule_size, (route_rules_array + route_rules_array_cursor), NULL); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, route_rule_size, (route_rules_array + route_rules_array_cursor), NULL); if (!necp_policy_route_rule_is_valid((route_rules_array + route_rules_array_cursor), route_rule_size)) { NECPLOG0(LOG_ERR, "Failed to validate policy route rule"); @@ -1758,11 +2552,11 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ } // Read policy conditions - for (cursor = necp_packet_find_tlv(packet, offset, NECP_TLV_POLICY_CONDITION, &error, 0); + for (cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_POLICY_CONDITION, &error, 0); cursor >= 0; - cursor = necp_packet_find_tlv(packet, cursor, NECP_TLV_POLICY_CONDITION, &error, 1)) { + cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, cursor, NECP_TLV_POLICY_CONDITION, &error, 1)) { u_int32_t condition_size = 0; - necp_packet_get_tlv_at_offset(packet, cursor, 0, NULL, &condition_size); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, 0, NULL, &condition_size); if (condition_size > 0) { conditions_array_size += (sizeof(u_int8_t) + sizeof(u_int32_t) + condition_size); @@ -1787,12 +2581,12 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ } conditions_array_cursor = 0; - for (cursor = necp_packet_find_tlv(packet, offset, NECP_TLV_POLICY_CONDITION, &error, 0); + for (cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_POLICY_CONDITION, &error, 0); cursor >= 0; - cursor = necp_packet_find_tlv(packet, cursor, NECP_TLV_POLICY_CONDITION, &error, 1)) { + cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, cursor, NECP_TLV_POLICY_CONDITION, &error, 1)) { u_int8_t condition_type = NECP_TLV_POLICY_CONDITION; u_int32_t condition_size = 0; - necp_packet_get_tlv_at_offset(packet, cursor, 0, NULL, &condition_size); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, 0, NULL, &condition_size); if (condition_size > 0 && condition_size <= (conditions_array_size - conditions_array_cursor)) { // Add type memcpy((conditions_array + conditions_array_cursor), &condition_type, sizeof(condition_type)); @@ -1803,7 +2597,7 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ conditions_array_cursor += sizeof(condition_size); // Add value - necp_packet_get_tlv_at_offset(packet, cursor, condition_size, (conditions_array + conditions_array_cursor), NULL); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, condition_size, (conditions_array + conditions_array_cursor), NULL); if (!necp_policy_condition_is_valid((conditions_array + conditions_array_cursor), condition_size, necp_policy_result_get_type_from_buffer(policy_result, policy_result_size))) { NECPLOG0(LOG_ERR, "Failed to validate policy condition"); response_error = NECP_ERROR_POLICY_CONDITIONS_INVALID; @@ -1858,8 +2652,10 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ goto fail; } - necp_send_policy_id_response(session, NECP_PACKET_TYPE_POLICY_ADD, message_id, policy->id); - return; + if (packet != NULL) { + necp_send_policy_id_response(session, NECP_PACKET_TYPE_POLICY_ADD, message_id, policy->id); + } + return (policy->id); fail: if (policy_result != NULL) { @@ -1872,7 +2668,13 @@ fail: FREE(route_rules_array, M_NECP); } - necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_ADD, message_id, response_error); + if (packet != NULL) { + necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_ADD, message_id, response_error); + } + if (return_error != NULL) { + *return_error = necp_get_posix_error_for_necp_error(response_error); + } + return (0); } static void @@ -1891,7 +2693,7 @@ necp_handle_policy_get(struct necp_session *session, u_int32_t message_id, mbuf_ struct necp_session_policy *policy = NULL; // Read policy id - error = necp_packet_get_tlv(packet, offset, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id, NULL); + error = necp_get_tlv(packet, NULL, 0, offset, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get policy id: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1910,16 +2712,16 @@ necp_handle_policy_get(struct necp_session *session, u_int32_t message_id, mbuf_ response_size = sizeof(struct necp_packet_header) + order_tlv_size + result_tlv_size + policy->conditions_size; MALLOC(response, u_int8_t *, response_size, M_NECP, M_WAITOK); if (response == NULL) { - necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_LIST_ALL, message_id, NECP_ERROR_INTERNAL); + necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_GET, message_id, NECP_ERROR_INTERNAL); return; } cursor = response; cursor = necp_buffer_write_packet_header(cursor, NECP_PACKET_TYPE_POLICY_GET, NECP_PACKET_FLAGS_RESPONSE, message_id); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ORDER, sizeof(necp_policy_order), &policy->order); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ORDER, sizeof(necp_policy_order), &policy->order, response, response_size); if (result_tlv_size) { - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_RESULT, policy->result_size, &policy->result); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_RESULT, policy->result_size, &policy->result, response, response_size); } if (policy->conditions_size) { memcpy(((u_int8_t *)(void *)(cursor)), policy->conditions, policy->conditions_size); @@ -1946,7 +2748,7 @@ necp_handle_policy_delete(struct necp_session *session, u_int32_t message_id, mb struct necp_session_policy *policy = NULL; // Read policy id - error = necp_packet_get_tlv(packet, offset, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id, NULL); + error = necp_get_tlv(packet, NULL, 0, offset, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get policy id: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -2008,7 +2810,7 @@ necp_handle_policy_list_all(struct necp_session *session, u_int32_t message_id, LIST_FOREACH(policy, &session->policies, chain) { if (!policy->pending_deletion && cur_policy_index < num_policies) { - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(u_int32_t), &policy->id); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(u_int32_t), &policy->id, response, response_size); cur_policy_index++; } } @@ -2147,21 +2949,23 @@ necp_policy_get_new_id(void) * } * ... */ -static void -necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset) +static int +necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, + user_addr_t out_buffer, size_t out_buffer_length, int offset) { -#pragma unused(packet, offset) +#pragma unused(offset) struct necp_kernel_socket_policy *policy = NULL; int policy_i; int policy_count = 0; u_int8_t **tlv_buffer_pointers = NULL; u_int32_t *tlv_buffer_lengths = NULL; - int total_tlv_len = 0; + u_int32_t total_tlv_len = 0; u_int8_t *result_buf = NULL; u_int8_t *result_buf_cursor = result_buf; char result_string[MAX_RESULT_STRING_LEN]; char proc_name_string[MAXCOMLEN + 1]; + int error_code = 0; bool error_occured = false; u_int32_t response_error = NECP_ERROR_INTERNAL; @@ -2181,7 +2985,9 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, // LOCK lck_rw_lock_shared(&necp_kernel_policy_lock); - NECPLOG0(LOG_DEBUG, "Gathering policies"); + if (necp_debug) { + NECPLOG0(LOG_DEBUG, "Gathering policies"); + } policy_count = necp_kernel_application_policies_count; @@ -2209,7 +3015,9 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, u_int16_t proc_name_len = strlen(proc_name_string) + 1; u_int16_t result_string_len = strlen(result_string) + 1; - NECPLOG(LOG_DEBUG, "Policy: process: %s, result: %s", proc_name_string, result_string); + if (necp_debug) { + NECPLOG(LOG_DEBUG, "Policy: process: %s, result: %s", proc_name_string, result_string); + } u_int32_t total_allocated_bytes = sizeof(u_int8_t) + sizeof(u_int32_t) + sizeof(policy->id) + // NECP_TLV_POLICY_ID sizeof(u_int8_t) + sizeof(u_int32_t) + sizeof(policy->order) + // NECP_TLV_POLICY_ORDER @@ -2312,11 +3120,11 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, } u_int8_t *cursor = tlv_buffer; - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(policy->id), &policy->id); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ORDER, sizeof(necp_policy_order), &policy->order); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_SESSION_ORDER, sizeof(policy->session_order), &policy->session_order); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_RESULT_STRING, result_string_len , result_string); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_OWNER, proc_name_len , proc_name_string); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(policy->id), &policy->id, tlv_buffer, total_allocated_bytes); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ORDER, sizeof(necp_policy_order), &policy->order, tlv_buffer, total_allocated_bytes); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_SESSION_ORDER, sizeof(policy->session_order), &policy->session_order, tlv_buffer, total_allocated_bytes); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_RESULT_STRING, result_string_len, result_string, tlv_buffer, total_allocated_bytes); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_OWNER, proc_name_len, proc_name_string, tlv_buffer, total_allocated_bytes); #define N_QUICK 256 u_int8_t q_cond_buf[N_QUICK]; // Minor optimization @@ -2336,63 +3144,76 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, memset(cond_buf, 0, condition_tlv_length); u_int8_t *cond_buf_cursor = cond_buf; if (condition_mask == NECP_POLICY_CONDITION_DEFAULT) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_DEFAULT, 0, ""); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_DEFAULT, 0, "", cond_buf, condition_tlv_length); } else { if (condition_mask & NECP_KERNEL_CONDITION_ALL_INTERFACES) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ALL_INTERFACES, 0, ""); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ALL_INTERFACES, 0, "", cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_BOUND_INTERFACE) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_BOUND_INTERFACE, strlen(if_name) + 1, if_name); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_BOUND_INTERFACE, strlen(if_name) + 1, + if_name, cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_PROTOCOL) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_IP_PROTOCOL, sizeof(policy->cond_protocol), &policy->cond_protocol); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_IP_PROTOCOL, sizeof(policy->cond_protocol), &policy->cond_protocol, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_APP_ID) { struct necp_uuid_id_mapping *entry = necp_uuid_lookup_uuid_with_app_id_locked(policy->cond_app_id); if (entry != NULL) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_APPLICATION, sizeof(entry->uuid), entry->uuid); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_APPLICATION, sizeof(entry->uuid), entry->uuid, + cond_buf, condition_tlv_length); } } if (condition_mask & NECP_KERNEL_CONDITION_REAL_APP_ID) { struct necp_uuid_id_mapping *entry = necp_uuid_lookup_uuid_with_app_id_locked(policy->cond_real_app_id); if (entry != NULL) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REAL_APPLICATION, sizeof(entry->uuid), entry->uuid); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REAL_APPLICATION, sizeof(entry->uuid), entry->uuid, + cond_buf, condition_tlv_length); } } if (condition_mask & NECP_KERNEL_CONDITION_DOMAIN) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_DOMAIN, strlen(policy->cond_domain) + 1, policy->cond_domain); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_DOMAIN, strlen(policy->cond_domain) + 1, policy->cond_domain, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_ACCOUNT_ID) { if (account_id_entry != NULL) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ACCOUNT, strlen(account_id_entry->string) + 1, account_id_entry->string); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ACCOUNT, strlen(account_id_entry->string) + 1, account_id_entry->string, + cond_buf, condition_tlv_length); } } if (condition_mask & NECP_KERNEL_CONDITION_PID) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_PID, sizeof(policy->cond_pid), &policy->cond_pid); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_PID, sizeof(policy->cond_pid), &policy->cond_pid, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_UID) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_UID, sizeof(policy->cond_uid), &policy->cond_uid); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_UID, sizeof(policy->cond_uid), &policy->cond_uid, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_TRAFFIC_CLASS) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_TRAFFIC_CLASS, sizeof(policy->cond_traffic_class), &policy->cond_traffic_class); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_TRAFFIC_CLASS, sizeof(policy->cond_traffic_class), &policy->cond_traffic_class, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_ENTITLEMENT) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ENTITLEMENT, 0, ""); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ENTITLEMENT, 0, "", + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_CUSTOM_ENTITLEMENT) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ENTITLEMENT, strlen(policy->cond_custom_entitlement) + 1, policy->cond_custom_entitlement); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ENTITLEMENT, strlen(policy->cond_custom_entitlement) + 1, policy->cond_custom_entitlement, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_LOCAL_START) { if (condition_mask & NECP_KERNEL_CONDITION_LOCAL_END) { struct necp_policy_condition_addr_range range; memcpy(&range.start_address, &policy->cond_local_start, sizeof(policy->cond_local_start)); memcpy(&range.end_address, &policy->cond_local_end, sizeof(policy->cond_local_end)); - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_LOCAL_ADDR_RANGE, sizeof(range), &range); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_LOCAL_ADDR_RANGE, sizeof(range), &range, + cond_buf, condition_tlv_length); } else { struct necp_policy_condition_addr addr; addr.prefix = policy->cond_local_prefix; memcpy(&addr.address, &policy->cond_local_start, sizeof(policy->cond_local_start)); - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_LOCAL_ADDR, sizeof(addr), &addr); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_LOCAL_ADDR, sizeof(addr), &addr, + cond_buf, condition_tlv_length); } } if (condition_mask & NECP_KERNEL_CONDITION_REMOTE_START) { @@ -2400,17 +3221,19 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, struct necp_policy_condition_addr_range range; memcpy(&range.start_address, &policy->cond_remote_start, sizeof(policy->cond_remote_start)); memcpy(&range.end_address, &policy->cond_remote_end, sizeof(policy->cond_remote_end)); - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REMOTE_ADDR_RANGE, sizeof(range), &range); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REMOTE_ADDR_RANGE, sizeof(range), &range, + cond_buf, condition_tlv_length); } else { struct necp_policy_condition_addr addr; addr.prefix = policy->cond_remote_prefix; memcpy(&addr.address, &policy->cond_remote_start, sizeof(policy->cond_remote_start)); - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REMOTE_ADDR, sizeof(addr), &addr); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REMOTE_ADDR, sizeof(addr), &addr, + cond_buf, condition_tlv_length); } } } - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_CONDITION, cond_buf_cursor - cond_buf, cond_buf); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_CONDITION, cond_buf_cursor - cond_buf, cond_buf, tlv_buffer, total_allocated_bytes); if (cond_buf != q_cond_buf) { FREE(cond_buf, M_NECP); } @@ -2425,36 +3248,77 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, // UNLOCK lck_rw_done(&necp_kernel_policy_lock); - u_int32_t total_result_length = sizeof(struct necp_packet_header) + total_tlv_len; - MALLOC(result_buf, u_int8_t *, total_result_length, M_NECP, M_NOWAIT | M_ZERO); - if (result_buf == NULL) { - NECPLOG(LOG_DEBUG, "Failed to allocate result_buffer (%u bytes)", total_result_length); - REPORT_ERROR(NECP_ERROR_INTERNAL); - } + // Send packet + if (packet != NULL) { + u_int32_t total_result_length = sizeof(struct necp_packet_header) + total_tlv_len; + + // Allow malloc to wait, since the total buffer may be large and we are not holding any locks + MALLOC(result_buf, u_int8_t *, total_result_length, M_NECP, M_WAITOK | M_ZERO); + if (result_buf == NULL) { + NECPLOG(LOG_DEBUG, "Failed to allocate result_buffer (%u bytes)", total_result_length); + REPORT_ERROR(NECP_ERROR_INTERNAL); + } - result_buf_cursor = result_buf; - result_buf_cursor = necp_buffer_write_packet_header(result_buf_cursor, NECP_PACKET_TYPE_POLICY_DUMP_ALL, NECP_PACKET_FLAGS_RESPONSE, message_id); + result_buf_cursor = result_buf; + result_buf_cursor = necp_buffer_write_packet_header(result_buf_cursor, NECP_PACKET_TYPE_POLICY_DUMP_ALL, NECP_PACKET_FLAGS_RESPONSE, message_id); + + for (int i = 0; i < policy_count; i++) { + if (tlv_buffer_pointers[i] != NULL) { + result_buf_cursor = necp_buffer_write_tlv(result_buf_cursor, NECP_TLV_POLICY_DUMP, tlv_buffer_lengths[i], tlv_buffer_pointers[i], result_buf, total_result_length); + } + } - for (int i = 0; i < policy_count; i++) { - if (tlv_buffer_pointers[i] != NULL) { - result_buf_cursor = necp_buffer_write_tlv(result_buf_cursor, NECP_TLV_POLICY_DUMP, tlv_buffer_lengths[i], tlv_buffer_pointers[i]); + if (!necp_send_ctl_data(session, result_buf, result_buf_cursor - result_buf)) { + NECPLOG(LOG_ERR, "Failed to send response (%u bytes)", result_buf_cursor - result_buf); + } else { + NECPLOG(LOG_ERR, "Sent data worth %u bytes. Total result buffer length was %u bytes", result_buf_cursor - result_buf, total_result_length); } } - if (!necp_send_ctl_data(session, result_buf, result_buf_cursor - result_buf)) { - NECPLOG(LOG_ERR, "Failed to send response (%u bytes)", result_buf_cursor - result_buf); - } else { - NECPLOG(LOG_ERR, "Sent data worth %u bytes. Total result buffer length was %u bytes", result_buf_cursor - result_buf, total_result_length); + // Copy out + if (out_buffer != 0) { + if (out_buffer_length < total_tlv_len + sizeof(u_int32_t)) { + NECPLOG(LOG_DEBUG, "out_buffer_length too small (%u < %u)", out_buffer_length, total_tlv_len + sizeof(u_int32_t)); + REPORT_ERROR(NECP_ERROR_INVALID_TLV); + } + + // Allow malloc to wait, since the total buffer may be large and we are not holding any locks + MALLOC(result_buf, u_int8_t *, total_tlv_len + sizeof(u_int32_t), M_NECP, M_WAITOK | M_ZERO); + if (result_buf == NULL) { + NECPLOG(LOG_DEBUG, "Failed to allocate result_buffer (%u bytes)", total_tlv_len + sizeof(u_int32_t)); + REPORT_ERROR(NECP_ERROR_INTERNAL); + } + + // Add four bytes for total length at the start + memcpy(result_buf, &total_tlv_len, sizeof(u_int32_t)); + + // Copy the TLVs + result_buf_cursor = result_buf + sizeof(u_int32_t); + for (int i = 0; i < policy_count; i++) { + if (tlv_buffer_pointers[i] != NULL) { + result_buf_cursor = necp_buffer_write_tlv(result_buf_cursor, NECP_TLV_POLICY_DUMP, tlv_buffer_lengths[i], tlv_buffer_pointers[i], + result_buf, total_tlv_len + sizeof(u_int32_t)); + } + } + + int copy_error = copyout(result_buf, out_buffer, total_tlv_len + sizeof(u_int32_t)); + if (copy_error) { + NECPLOG(LOG_DEBUG, "Failed to copy out result_buffer (%u bytes)", total_tlv_len + sizeof(u_int32_t)); + REPORT_ERROR(NECP_ERROR_INTERNAL); + } } done: if (error_occured) { - if(!necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_DUMP_ALL, message_id, response_error)) { - NECPLOG0(LOG_ERR, "Failed to send error response"); - } else { - NECPLOG0(LOG_ERR, "Sent error response"); + if (packet != NULL) { + if(!necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_DUMP_ALL, message_id, response_error)) { + NECPLOG0(LOG_ERR, "Failed to send error response"); + } else { + NECPLOG0(LOG_ERR, "Sent error response"); + } } + error_code = necp_get_posix_error_for_necp_error(response_error); } if (result_buf != NULL) { @@ -2478,6 +3342,8 @@ done: #undef RESET_COND_BUF #undef REPORT_ERROR #undef UNLOCK_AND_REPORT_ERROR + + return (error_code); } static struct necp_session_policy * @@ -2495,7 +3361,7 @@ necp_policy_create(struct necp_session *session, necp_policy_order order, u_int8 goto done; } - memset(new_policy, 0, sizeof(*new_policy)); + memset(new_policy, 0, sizeof(*new_policy)); // M_ZERO is not supported for MALLOC_ZONE new_policy->applied = FALSE; new_policy->pending_deletion = FALSE; new_policy->pending_update = FALSE; @@ -2634,7 +3500,7 @@ necp_policy_unapply(struct necp_session_policy *policy) return (FALSE); } - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); // Release local uuid mappings if (!uuid_is_null(policy->applied_app_uuid)) { @@ -2749,7 +3615,7 @@ necp_policy_apply(struct necp_session *session, struct necp_session_policy *poli return (FALSE); } - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); // Process conditions while (offset < policy->conditions_size) { @@ -3268,18 +4134,30 @@ necp_policy_apply_all(struct necp_session *session) // --------------------- // Kernel policies are derived from session policies static necp_kernel_policy_id -necp_kernel_policy_get_new_id(void) +necp_kernel_policy_get_new_id(bool socket_level) { + static necp_kernel_policy_id necp_last_kernel_socket_policy_id = 0; + static necp_kernel_policy_id necp_last_kernel_ip_policy_id = 0; + necp_kernel_policy_id newid = NECP_KERNEL_POLICY_ID_NONE; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); - necp_last_kernel_policy_id++; - if (necp_last_kernel_policy_id < NECP_KERNEL_POLICY_ID_FIRST_VALID) { - necp_last_kernel_policy_id = NECP_KERNEL_POLICY_ID_FIRST_VALID; + if (socket_level) { + necp_last_kernel_socket_policy_id++; + if (necp_last_kernel_socket_policy_id < NECP_KERNEL_POLICY_ID_FIRST_VALID_SOCKET || + necp_last_kernel_socket_policy_id >= NECP_KERNEL_POLICY_ID_FIRST_VALID_IP) { + necp_last_kernel_socket_policy_id = NECP_KERNEL_POLICY_ID_FIRST_VALID_SOCKET; + } + newid = necp_last_kernel_socket_policy_id; + } else { + necp_last_kernel_ip_policy_id++; + if (necp_last_kernel_ip_policy_id < NECP_KERNEL_POLICY_ID_FIRST_VALID_IP) { + necp_last_kernel_ip_policy_id = NECP_KERNEL_POLICY_ID_FIRST_VALID_IP; + } + newid = necp_last_kernel_ip_policy_id; } - newid = necp_last_kernel_policy_id; if (newid == NECP_KERNEL_POLICY_ID_NONE) { NECPLOG0(LOG_DEBUG, "Allocate kernel policy id failed.\n"); return (0); @@ -3300,9 +4178,9 @@ necp_kernel_socket_policy_add(necp_policy_id parent_policy_id, necp_policy_order goto done; } - memset(new_kernel_policy, 0, sizeof(*new_kernel_policy)); + memset(new_kernel_policy, 0, sizeof(*new_kernel_policy)); // M_ZERO is not supported for MALLOC_ZONE new_kernel_policy->parent_policy_id = parent_policy_id; - new_kernel_policy->id = necp_kernel_policy_get_new_id(); + new_kernel_policy->id = necp_kernel_policy_get_new_id(true); new_kernel_policy->order = order; new_kernel_policy->session_order = session_order; new_kernel_policy->session_pid = session_pid; @@ -3416,7 +4294,7 @@ necp_kernel_socket_policy_delete(necp_kernel_policy_id policy_id) { struct necp_kernel_socket_policy *policy = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); policy = necp_kernel_socket_policy_find(policy_id); if (policy) { @@ -3881,7 +4759,7 @@ necp_kernel_socket_policies_reprocess(void) int app_layer_current_free_index = 0; struct necp_kernel_socket_policy *kernel_policy = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); // Reset mask to 0 necp_kernel_application_policies_condition_mask = 0; @@ -4003,7 +4881,7 @@ necp_get_new_string_id(void) { u_int32_t newid = 0; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); necp_last_string_id++; if (necp_last_string_id < 1) { @@ -4057,7 +4935,7 @@ necp_create_string_to_id_mapping(struct necp_string_id_mapping_list *list, char u_int32_t string_id = 0; struct necp_string_id_mapping *existing_mapping = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); existing_mapping = necp_lookup_string_to_id_locked(list, string); if (existing_mapping != NULL) { @@ -4091,7 +4969,7 @@ necp_remove_string_to_id_mapping(struct necp_string_id_mapping_list *list, char { struct necp_string_id_mapping *existing_mapping = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); existing_mapping = necp_lookup_string_to_id_locked(list, string); if (existing_mapping != NULL) { @@ -4111,7 +4989,7 @@ necp_get_new_route_rule_id(void) { u_int32_t newid = 0; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); necp_last_route_rule_id++; if (necp_last_route_rule_id < 1 || necp_last_route_rule_id > UINT16_MAX) { @@ -4132,7 +5010,7 @@ necp_get_new_aggregate_route_rule_id(void) { u_int32_t newid = 0; - lck_rw_assert(&necp_route_rule_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_route_rule_lock, LCK_RW_ASSERT_EXCLUSIVE); necp_last_aggregate_route_rule_id++; if (necp_last_aggregate_route_rule_id <= UINT16_MAX) { @@ -4232,7 +5110,7 @@ necp_create_route_rule(struct necp_route_rule_list *list, u_int8_t *route_rules_ u_int8_t if_actions[MAX_ROUTE_RULE_INTERFACES]; memset(&if_actions, 0, sizeof(if_actions)); - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); if (route_rules_array == NULL || route_rules_array_size == 0) { return (0); @@ -4346,7 +5224,7 @@ necp_remove_route_rule(struct necp_route_rule_list *list, u_int32_t route_rule_i { struct necp_route_rule *existing_rule = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); existing_rule = necp_lookup_route_rule_locked(list, route_rule_id); if (existing_rule != NULL) { @@ -4423,7 +5301,7 @@ necp_get_new_uuid_id(void) { u_int32_t newid = 0; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); necp_last_uuid_id++; if (necp_last_uuid_id < (NECP_NULL_SERVICE_ID + 1)) { @@ -4480,7 +5358,7 @@ necp_create_uuid_app_id_mapping(uuid_t uuid, bool *allocated_mapping, bool uuid_ u_int32_t local_id = 0; struct necp_uuid_id_mapping *existing_mapping = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); if (allocated_mapping) { *allocated_mapping = FALSE; @@ -4524,7 +5402,7 @@ necp_remove_uuid_app_id_mapping(uuid_t uuid, bool *removed_mapping, bool uuid_po { struct necp_uuid_id_mapping *existing_mapping = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); if (removed_mapping) { *removed_mapping = FALSE; @@ -4608,7 +5486,7 @@ necp_create_uuid_service_id_mapping(uuid_t uuid) return (NECP_NULL_SERVICE_ID); } - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); existing_mapping = necp_uuid_lookup_service_id_locked(uuid); if (existing_mapping != NULL) { @@ -4640,7 +5518,7 @@ necp_remove_uuid_service_id_mapping(uuid_t uuid) return (TRUE); } - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); existing_mapping = necp_uuid_lookup_app_id_locked(uuid); if (existing_mapping != NULL) { @@ -4658,7 +5536,7 @@ necp_remove_uuid_service_id_mapping(uuid_t uuid) static bool necp_kernel_socket_policies_update_uuid_table(void) { - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); if (necp_uuid_app_id_mappings_dirty) { if (proc_uuid_policy_kernel(PROC_UUID_POLICY_OPERATION_CLEAR, NULL, PROC_UUID_NECP_APP_POLICY) < 0) { @@ -4697,9 +5575,9 @@ necp_kernel_ip_output_policy_add(necp_policy_id parent_policy_id, necp_policy_or goto done; } - memset(new_kernel_policy, 0, sizeof(*new_kernel_policy)); + memset(new_kernel_policy, 0, sizeof(*new_kernel_policy)); // M_ZERO is not supported for MALLOC_ZONE new_kernel_policy->parent_policy_id = parent_policy_id; - new_kernel_policy->id = necp_kernel_policy_get_new_id(); + new_kernel_policy->id = necp_kernel_policy_get_new_id(false); new_kernel_policy->suborder = suborder; new_kernel_policy->order = order; new_kernel_policy->session_order = session_order; @@ -4788,7 +5666,7 @@ necp_kernel_ip_output_policy_delete(necp_kernel_policy_id policy_id) { struct necp_kernel_ip_output_policy *policy = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); policy = necp_kernel_ip_output_policy_find(policy_id); if (policy) { @@ -4971,7 +5849,7 @@ necp_kernel_ip_output_policies_reprocess(void) int bucket_current_free_index[NECP_KERNEL_IP_OUTPUT_POLICIES_MAP_NUM_ID_BUCKETS]; struct necp_kernel_ip_output_policy *kernel_policy = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); // Reset mask to 0 necp_kernel_ip_output_policies_condition_mask = 0; @@ -5261,7 +6139,10 @@ necp_application_find_policy_match_internal(proc_t proc, u_int32_t parameters_size, struct necp_aggregate_result *returned_result, u_int32_t *flags, - u_int required_interface_index) + u_int required_interface_index, + const union necp_sockaddr_union *override_local_addr, + const union necp_sockaddr_union *override_remote_addr, + struct rtentry **returned_route, bool ignore_address) { int error = 0; size_t offset = 0; @@ -5276,14 +6157,23 @@ necp_application_find_policy_match_internal(proc_t proc, u_int16_t protocol = 0; u_int32_t bound_interface_index = required_interface_index; u_int32_t traffic_class = 0; + u_int32_t client_flags = 0; union necp_sockaddr_union local_addr; union necp_sockaddr_union remote_addr; bool no_remote_addr = FALSE; u_int8_t remote_family = 0; bool no_local_addr = FALSE; - memset(&local_addr, 0, sizeof(local_addr)); - memset(&remote_addr, 0, sizeof(remote_addr)); + if (override_local_addr) { + memcpy(&local_addr, override_local_addr, sizeof(local_addr)); + } else { + memset(&local_addr, 0, sizeof(local_addr)); + } + if (override_remote_addr) { + memcpy(&remote_addr, override_remote_addr, sizeof(remote_addr)); + } else { + memset(&remote_addr, 0, sizeof(remote_addr)); + } // Initialize UID, PID, and UUIDs to the current process uid_t uid = kauth_cred_getuid(proc_ucred(proc)); @@ -5423,6 +6313,10 @@ necp_application_find_policy_match_internal(proc_t proc, break; } case NECP_CLIENT_PARAMETER_LOCAL_ADDRESS: { + if (ignore_address) { + break; + } + if (length >= sizeof(struct necp_policy_condition_addr)) { struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; if (necp_address_is_valid(&address_struct->address.sa)) { @@ -5432,6 +6326,10 @@ necp_application_find_policy_match_internal(proc_t proc, break; } case NECP_CLIENT_PARAMETER_REMOTE_ADDRESS: { + if (ignore_address) { + break; + } + if (length >= sizeof(struct necp_policy_condition_addr)) { struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; if (necp_address_is_valid(&address_struct->address.sa)) { @@ -5440,6 +6338,11 @@ necp_application_find_policy_match_internal(proc_t proc, } break; } + case NECP_CLIENT_PARAMETER_FLAGS: { + if (length >= sizeof(client_flags)) { + memcpy(&client_flags, value, sizeof(client_flags)); + } + } default: { break; } @@ -5459,6 +6362,10 @@ necp_application_find_policy_match_internal(proc_t proc, returned_result->policy_id = matched_policy->id; returned_result->routing_result = matched_policy->result; memcpy(&returned_result->routing_result_parameter, &matched_policy->result_parameter, sizeof(returned_result->routing_result_parameter)); + } else if (necp_drop_all_order > 0) { + // Mark socket as a drop if drop_all is set + returned_result->policy_id = NECP_KERNEL_POLICY_ID_NO_MATCH; + returned_result->routing_result = NECP_KERNEL_POLICY_RESULT_DROP; } else { returned_result->policy_id = 0; returned_result->routing_result = NECP_KERNEL_POLICY_RESULT_NONE; @@ -5522,70 +6429,76 @@ necp_application_find_policy_match_internal(proc_t proc, remote_family = remote_addr.sa.sa_family; } - if (no_remote_addr) { - memset(&remote_addr, 0, sizeof(remote_addr)); - if (remote_family == AF_INET6) { - // Reset address to :: - remote_addr.sa.sa_family = AF_INET6; - remote_addr.sa.sa_len = sizeof(struct sockaddr_in6); - } else { - // Reset address to 0.0.0.0 - remote_addr.sa.sa_family = AF_INET; - remote_addr.sa.sa_len = sizeof(struct sockaddr_in); - } - } - + returned_result->routed_interface_index = 0; struct rtentry *rt = NULL; - rt = rtalloc1_scoped((struct sockaddr *)&remote_addr, 0, 0, - output_bound_interface); + if (!no_local_addr && (client_flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) != 0) { + // Treat the output bound interface as the routed interface for local address + // validation later. + returned_result->routed_interface_index = output_bound_interface; + } else { + if (no_remote_addr) { + memset(&remote_addr, 0, sizeof(remote_addr)); + if (remote_family == AF_INET6) { + // Reset address to :: + remote_addr.sa.sa_family = AF_INET6; + remote_addr.sa.sa_len = sizeof(struct sockaddr_in6); + } else { + // Reset address to 0.0.0.0 + remote_addr.sa.sa_family = AF_INET; + remote_addr.sa.sa_len = sizeof(struct sockaddr_in); + } + } - if (no_remote_addr && remote_family == 0 && - (rt == NULL || rt->rt_ifp == NULL)) { - // Route lookup for default IPv4 failed, try IPv6 + rt = rtalloc1_scoped((struct sockaddr *)&remote_addr, 0, 0, + output_bound_interface); - // Cleanup old route if necessary - if (rt != NULL) { - rtfree(rt); - rt = NULL; - } + if (no_remote_addr && remote_family == 0 && + (rt == NULL || rt->rt_ifp == NULL)) { + // Route lookup for default IPv4 failed, try IPv6 - // Reset address to :: - memset(&remote_addr, 0, sizeof(remote_addr)); - remote_addr.sa.sa_family = AF_INET6; - remote_addr.sa.sa_len = sizeof(struct sockaddr_in6); + // Cleanup old route if necessary + if (rt != NULL) { + rtfree(rt); + rt = NULL; + } - // Get route - rt = rtalloc1_scoped((struct sockaddr *)&remote_addr, 0, 0, - output_bound_interface); - } + // Reset address to :: + memset(&remote_addr, 0, sizeof(remote_addr)); + remote_addr.sa.sa_family = AF_INET6; + remote_addr.sa.sa_len = sizeof(struct sockaddr_in6); - returned_result->routed_interface_index = 0; - if (rt != NULL && - rt->rt_ifp != NULL) { - returned_result->routed_interface_index = rt->rt_ifp->if_index; - /* - * For local addresses, we allow the interface scope to be - * either the loopback interface or the interface hosting the - * local address. - */ - if (bound_interface_index != IFSCOPE_NONE && - rt->rt_ifa != NULL && rt->rt_ifa->ifa_ifp && - (output_bound_interface == lo_ifp->if_index || - rt->rt_ifp->if_index == lo_ifp->if_index || - rt->rt_ifa->ifa_ifp->if_index == bound_interface_index)) { - struct sockaddr_storage dst; - unsigned int ifscope = bound_interface_index; + // Get route + rt = rtalloc1_scoped((struct sockaddr *)&remote_addr, 0, 0, + output_bound_interface); + } + if (rt != NULL && + rt->rt_ifp != NULL) { + returned_result->routed_interface_index = rt->rt_ifp->if_index; /* - * Transform dst into the internal routing table form + * For local addresses, we allow the interface scope to be + * either the loopback interface or the interface hosting the + * local address. */ - (void) sa_copy((struct sockaddr *)&remote_addr, - &dst, &ifscope); + if (bound_interface_index != IFSCOPE_NONE && + rt->rt_ifa != NULL && rt->rt_ifa->ifa_ifp && + (output_bound_interface == lo_ifp->if_index || + rt->rt_ifp->if_index == lo_ifp->if_index || + rt->rt_ifa->ifa_ifp->if_index == bound_interface_index)) { + struct sockaddr_storage dst; + unsigned int ifscope = bound_interface_index; - if ((rt->rt_ifp->if_index == lo_ifp->if_index) || - rt_ifa_is_dst((struct sockaddr *)&dst, rt->rt_ifa)) - returned_result->routed_interface_index = - bound_interface_index; + /* + * Transform dst into the internal routing table form + */ + (void) sa_copy((struct sockaddr *)&remote_addr, + &dst, &ifscope); + + if ((rt->rt_ifp->if_index == lo_ifp->if_index) || + rt_ifa_is_dst((struct sockaddr *)&dst, rt->rt_ifa)) + returned_result->routed_interface_index = + bound_interface_index; + } } } @@ -5619,35 +6532,80 @@ necp_application_find_policy_match_internal(proc_t proc, } if (flags != NULL) { - // Check for local/direct - bool is_local = FALSE; - if (rt != NULL && (rt->rt_flags & RTF_LOCAL)) { - is_local = TRUE; - } else if (returned_result->routed_interface_index != 0 && - !no_remote_addr) { - // Check if remote address is an interface address - struct ifaddr *ifa = ifa_ifwithaddr(&remote_addr.sa); - if (ifa != NULL && ifa->ifa_ifp != NULL) { - u_int if_index_for_remote_addr = ifa->ifa_ifp->if_index; - if (if_index_for_remote_addr == returned_result->routed_interface_index || - if_index_for_remote_addr == lo_ifp->if_index) { - is_local = TRUE; + if ((client_flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) == 0) { + // Check for local/direct + bool is_local = FALSE; + if (rt != NULL && (rt->rt_flags & RTF_LOCAL)) { + is_local = TRUE; + } else if (returned_result->routed_interface_index != 0 && + !no_remote_addr) { + // Check if remote address is an interface address + struct ifaddr *ifa = ifa_ifwithaddr(&remote_addr.sa); + if (ifa != NULL && ifa->ifa_ifp != NULL) { + u_int if_index_for_remote_addr = ifa->ifa_ifp->if_index; + if (if_index_for_remote_addr == returned_result->routed_interface_index || + if_index_for_remote_addr == lo_ifp->if_index) { + is_local = TRUE; + } + } + if (ifa != NULL) { + ifaddr_release(ifa); + ifa = NULL; } } - if (ifa != NULL) { - ifaddr_release(ifa); - ifa = NULL; + + if (is_local) { + *flags |= (NECP_CLIENT_RESULT_FLAG_IS_LOCAL | NECP_CLIENT_RESULT_FLAG_IS_DIRECT); + } else { + if (rt != NULL && + !(rt->rt_flags & RTF_GATEWAY) && + (rt->rt_ifa && rt->rt_ifa->ifa_ifp && !(rt->rt_ifa->ifa_ifp->if_flags & IFF_POINTOPOINT))) { + // Route is directly accessible + *flags |= NECP_CLIENT_RESULT_FLAG_IS_DIRECT; + } } - } - if (is_local) { - *flags |= (NECP_CLIENT_RESULT_FLAG_IS_LOCAL | NECP_CLIENT_RESULT_FLAG_IS_DIRECT); - } else { if (rt != NULL && - !(rt->rt_flags & RTF_GATEWAY) && - (rt->rt_ifa && rt->rt_ifa->ifa_ifp && !(rt->rt_ifa->ifa_ifp->if_flags & IFF_POINTOPOINT))) { - // Route is directly accessible - *flags |= NECP_CLIENT_RESULT_FLAG_IS_DIRECT; + rt->rt_ifp != NULL) { + // Check probe status + if (rt->rt_ifp->if_eflags & IFEF_PROBE_CONNECTIVITY) { + *flags |= NECP_CLIENT_RESULT_FLAG_PROBE_CONNECTIVITY; + } + + if (rt->rt_ifp->if_type == IFT_CELLULAR) { + struct if_cellular_status_v1 *ifsr; + + ifnet_lock_shared(rt->rt_ifp); + lck_rw_lock_exclusive(&rt->rt_ifp->if_link_status_lock); + + if (rt->rt_ifp->if_link_status != NULL) { + ifsr = &rt->rt_ifp->if_link_status->ifsr_u.ifsr_cell.if_cell_u.if_status_v1; + + if (ifsr->valid_bitmask & IF_CELL_UL_MSS_RECOMMENDED_VALID) { + if (ifsr->mss_recommended == IF_CELL_UL_MSS_RECOMMENDED_NONE) { + returned_result->mss_recommended = NECP_CLIENT_RESULT_RECOMMENDED_MSS_NONE; + } else if (ifsr->mss_recommended == IF_CELL_UL_MSS_RECOMMENDED_MEDIUM) { + returned_result->mss_recommended = NECP_CLIENT_RESULT_RECOMMENDED_MSS_MEDIUM; + } else if (ifsr->mss_recommended == IF_CELL_UL_MSS_RECOMMENDED_LOW) { + returned_result->mss_recommended = NECP_CLIENT_RESULT_RECOMMENDED_MSS_LOW; + } + } + } + lck_rw_done(&rt->rt_ifp->if_link_status_lock); + ifnet_lock_done(rt->rt_ifp); + } + + // Check link quality + if ((client_flags & NECP_CLIENT_PARAMETER_FLAG_DISCRETIONARY) && + (rt->rt_ifp->if_interface_state.valid_bitmask & IF_INTERFACE_STATE_LQM_STATE_VALID) && + rt->rt_ifp->if_interface_state.lqm_state == IFNET_LQM_THRESH_ABORT) { + *flags |= NECP_CLIENT_RESULT_FLAG_LINK_QUALITY_ABORT; + } + + // Check QoS marking (fastlane) + if (necp_update_qos_marking(rt->rt_ifp, route_rule_id)) { + *flags |= NECP_CLIENT_RESULT_FLAG_ALLOW_QOS_MARKING; + } } } @@ -5701,7 +6659,11 @@ necp_application_find_policy_match_internal(proc_t proc, } if (rt != NULL) { - rtfree(rt); + if (returned_route != NULL) { + *returned_route = rt; + } else { + rtfree(rt); + } rt = NULL; } // Unlock @@ -7109,7 +8071,7 @@ necp_buffer_compare_with_bit_prefix(u_int8_t *p1, u_int8_t *p2, u_int32_t bits) } static bool -necp_socket_update_qos_marking_inner(struct ifnet *ifp, u_int32_t route_rule_id) +necp_update_qos_marking(struct ifnet *ifp, u_int32_t route_rule_id) { bool qos_marking = FALSE; int exception_index = 0; @@ -7170,8 +8132,9 @@ necp_socket_update_qos_marking(struct inpcb *inp, struct rtentry *route, struct bool qos_marking = FALSE; struct ifnet *ifp = interface = NULL; - ASSERT(net_qos_policy_restricted != 0); - + if (net_qos_policy_restricted == 0) { + return; + } if (inp->inp_socket == NULL) { return; } @@ -7207,14 +8170,14 @@ necp_socket_update_qos_marking(struct inpcb *inp, struct rtentry *route, struct if (sub_route_rule_id == 0) { break; } - qos_marking = necp_socket_update_qos_marking_inner(ifp, sub_route_rule_id); + qos_marking = necp_update_qos_marking(ifp, sub_route_rule_id); if (qos_marking == TRUE) { break; } } } } else { - qos_marking = necp_socket_update_qos_marking_inner(ifp, route_rule_id); + qos_marking = necp_update_qos_marking(ifp, route_rule_id); } /* * Now that we have an interface we remember the gencount @@ -7452,16 +8415,18 @@ necp_socket_is_allowed_to_send_recv_internal(struct inpcb *inp, struct sockaddr if ((necp_socket_is_connected(inp) || (override_local_addr == NULL && override_remote_addr == NULL)) && inp->inp_policyresult.policy_id != NECP_KERNEL_POLICY_ID_NONE) { bool policies_have_changed = FALSE; bool route_allowed = TRUE; - lck_rw_lock_shared(&necp_kernel_policy_lock); + if (inp->inp_policyresult.policy_gencount != necp_kernel_socket_policies_gencount) { policies_have_changed = TRUE; } else { - if (inp->inp_policyresult.results.route_rule_id != 0 && - !necp_route_is_allowed(route, interface, inp->inp_policyresult.results.route_rule_id, &interface_type_denied)) { - route_allowed = FALSE; + if (inp->inp_policyresult.results.route_rule_id != 0) { + lck_rw_lock_shared(&necp_kernel_policy_lock); + if (!necp_route_is_allowed(route, interface, inp->inp_policyresult.results.route_rule_id, &interface_type_denied)) { + route_allowed = FALSE; + } + lck_rw_done(&necp_kernel_policy_lock); } } - lck_rw_done(&necp_kernel_policy_lock); if (!policies_have_changed) { if (!route_allowed || diff --git a/bsd/net/necp.h b/bsd/net/necp.h index ec33fb120..63a9fe967 100644 --- a/bsd/net/necp.h +++ b/bsd/net/necp.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -33,6 +33,7 @@ #ifdef PRIVATE #include +#include #include #include @@ -64,6 +65,21 @@ struct necp_packet_header { #define NECP_PACKET_TYPE_UNREGISTER_SERVICE 10 #define NECP_PACKET_TYPE_POLICY_DUMP_ALL 11 +/* + * Session actions + */ +#define NECP_SESSION_ACTION_POLICY_ADD 1 // In: Policy TLVs Out: necp_policy_id +#define NECP_SESSION_ACTION_POLICY_GET 2 // In: necp_policy_id Out: Policy TLVs +#define NECP_SESSION_ACTION_POLICY_DELETE 3 // In: necp_policy_id Out: None +#define NECP_SESSION_ACTION_POLICY_APPLY_ALL 4 // In: None Out: None +#define NECP_SESSION_ACTION_POLICY_LIST_ALL 5 // In: None Out: TLVs of IDs +#define NECP_SESSION_ACTION_POLICY_DELETE_ALL 6 // In: None Out: None +#define NECP_SESSION_ACTION_SET_SESSION_PRIORITY 7 // In: necp_session_priority Out: None +#define NECP_SESSION_ACTION_LOCK_SESSION_TO_PROC 8 // In: None Out: None +#define NECP_SESSION_ACTION_REGISTER_SERVICE 9 // In: uuid_t Out: None +#define NECP_SESSION_ACTION_UNREGISTER_SERVICE 10 // In: uuid_t Out: None +#define NECP_SESSION_ACTION_POLICY_DUMP_ALL 11 // In: None Out: uint32_t bytes length, then Policy TLVs + /* * Control message flags */ @@ -207,6 +223,7 @@ struct necp_policy_condition_addr_range { typedef u_int32_t necp_policy_id; typedef u_int32_t necp_policy_order; +typedef u_int32_t necp_session_priority; typedef u_int32_t necp_kernel_policy_result; typedef u_int32_t necp_kernel_policy_filter; @@ -220,6 +237,8 @@ typedef union { #define NECP_SERVICE_FLAGS_REGISTERED 0x01 #define NECP_MAX_NETAGENTS 8 + +#define NECP_TFO_COOKIE_LEN_MAX 16 struct necp_aggregate_result { necp_kernel_policy_result routing_result; necp_kernel_policy_routing_result_parameter routing_result_parameter; @@ -232,6 +251,7 @@ struct necp_aggregate_result { u_int32_t policy_id; uuid_t netagents[NECP_MAX_NETAGENTS]; u_int32_t netagent_flags[NECP_MAX_NETAGENTS]; + u_int8_t mss_recommended; }; /* @@ -239,7 +259,7 @@ struct necp_aggregate_result { * but they get entangled with #defines for v4 etc in pfvar.h and it may be better practice * to have separate definitions here. */ -typedef struct necp_stat_counts +struct necp_stat_counts { /* Counters */ u_int64_t necp_stat_rxpackets __attribute__((aligned(8))); @@ -258,13 +278,7 @@ typedef struct necp_stat_counts u_int32_t necp_stat_avg_rtt; u_int32_t necp_stat_var_rtt; - u_int64_t necp_stat_cell_rxbytes __attribute__((aligned(8))); - u_int64_t necp_stat_cell_txbytes __attribute__((aligned(8))); - u_int64_t necp_stat_wifi_rxbytes __attribute__((aligned(8))); - u_int64_t necp_stat_wifi_txbytes __attribute__((aligned(8))); - u_int64_t necp_stat_wired_rxbytes __attribute__((aligned(8))); - u_int64_t necp_stat_wired_txbytes __attribute__((aligned(8))); -} necp_stat_counts; +}; // Note, some metadata is implicit in the necp client itself: // From the process itself : pid, upid, uuid, proc name. @@ -272,17 +286,11 @@ typedef struct necp_stat_counts // // The following may well be supplied via future necp client parameters, // but they are here so they don't get forgotten. -typedef struct necp_basic_metadata +struct necp_basic_metadata { u_int32_t rcvbufsize; u_int32_t rcvbufused; - - u_int64_t eupid; // Not currently used by NetworkStatistics, could skip. - u_int32_t epid; - - uuid_t vuuid; // Effective UUID as given from voucher - uint16_t ifnet_properties; -} necp_basic_metadata; +}; struct necp_tcp_probe_status { unsigned int probe_activated : 1; @@ -291,7 +299,7 @@ struct necp_tcp_probe_status { unsigned int conn_probe_failed : 1; }; -typedef struct necp_extra_tcp_metadata +struct necp_extra_tcp_metadata { struct necp_tcp_probe_status probestatus; @@ -300,42 +308,106 @@ typedef struct necp_extra_tcp_metadata u_int32_t txunacked; u_int32_t txwindow; u_int32_t txcwindow; + u_int32_t flags; // use SOF_* + u_int32_t flags1; // use SOF1_* u_int32_t traffic_mgt_flags; u_int32_t cc_alg_index; u_int32_t state; -} necp_extra_tcp_metadata; + activity_bitmap_t activity_bitmap; +}; -typedef struct necp_stats_hdr { +struct necp_stats_hdr { u_int32_t necp_stats_type __attribute__((aligned(8))); u_int32_t necp_stats_ver; - u_int64_t necp_stats_event; -} necp_stats_hdr; + u_int64_t __necp_stats_reserved; // Pad the field for future use +}; #define NECP_CLIENT_STATISTICS_TYPE_TCP 1 // Identifies use of necp_tcp_stats #define NECP_CLIENT_STATISTICS_TYPE_UDP 2 // Identifies use of necp_udp_stats #define NECP_CLIENT_STATISTICS_TYPE_TCP_VER_1 1 // Currently supported version for TCP #define NECP_CLIENT_STATISTICS_TYPE_UDP_VER_1 1 // Currently supported version for UDP -typedef struct necp_tcp_stats { - necp_stats_hdr necp_tcp_hdr; - necp_stat_counts necp_tcp_counts; - necp_basic_metadata necp_tcp_basic; - necp_extra_tcp_metadata necp_tcp_extra; -} necp_tcp_stats; +#define NECP_CLIENT_STATISTICS_TYPE_TCP_CURRENT_VER NECP_CLIENT_STATISTICS_TYPE_TCP_VER_1 +#define NECP_CLIENT_STATISTICS_TYPE_UDP_CURRENT_VER NECP_CLIENT_STATISTICS_TYPE_UDP_VER_1 + +#define NECP_CLIENT_STATISTICS_EVENT_INIT 0x00000000 // Register the flow +#define NECP_CLIENT_STATISTICS_EVENT_TIME_WAIT 0x00000001 // The flow is effectively finished but waiting on timer + +struct necp_tcp_stats { + struct necp_stats_hdr necp_tcp_hdr; + struct necp_stat_counts necp_tcp_counts; + struct necp_basic_metadata necp_tcp_basic; + struct necp_extra_tcp_metadata necp_tcp_extra; +}; -typedef struct necp_udp_stats { - necp_stats_hdr necp_udp_hdr; - necp_stat_counts necp_udp_counts; - necp_basic_metadata necp_udp_basic; -} necp_udp_stats; +struct necp_udp_stats { + struct necp_stats_hdr necp_udp_hdr; + struct necp_stat_counts necp_udp_counts; + struct necp_basic_metadata necp_udp_basic; +}; typedef struct necp_all_stats { union { - necp_tcp_stats tcp_stats; - necp_udp_stats udp_stats; + struct necp_tcp_stats tcp_stats; + struct necp_udp_stats udp_stats; } all_stats_u; } necp_all_stats; +// Memory for statistics is requested via a necp_stats_bufreq +// +struct necp_stats_bufreq { + u_int32_t necp_stats_bufreq_id __attribute__((aligned(8))); + u_int32_t necp_stats_bufreq_type; // NECP_CLIENT_STATISTICS_TYPE_* + u_int32_t necp_stats_bufreq_ver; // NECP_CLIENT_STATISTICS_TYPE_*_VER + u_int32_t necp_stats_bufreq_size; + union { + void *necp_stats_bufreq_addr; + mach_vm_address_t necp_stats_bufreq_uaddr; + }; +}; + +#define NECP_CLIENT_STATISTICS_BUFREQ_ID 0xbf // Distinguishes from statistics actions taking a necp_all_stats struct + +// There is a limit to the number of statistics structures that may be allocated per process, subject to change +// +#define NECP_MAX_PER_PROCESS_CLIENT_STATISTICS_STRUCTS 512 + +#define NECP_TCP_ECN_HEURISTICS_SYN_RST 1 +typedef struct necp_tcp_ecn_cache { + u_int8_t necp_tcp_ecn_heuristics_success:1; + u_int8_t necp_tcp_ecn_heuristics_loss:1; + u_int8_t necp_tcp_ecn_heuristics_drop_rst:1; + u_int8_t necp_tcp_ecn_heuristics_drop_rxmt:1; + u_int8_t necp_tcp_ecn_heuristics_aggressive:1; + u_int8_t necp_tcp_ecn_heuristics_syn_rst:1; +} necp_tcp_ecn_cache; + +#define NECP_TCP_TFO_HEURISTICS_RST 1 +typedef struct necp_tcp_tfo_cache { + u_int8_t necp_tcp_tfo_cookie[NECP_TFO_COOKIE_LEN_MAX]; + u_int8_t necp_tcp_tfo_cookie_len; + u_int8_t necp_tcp_tfo_heuristics_success:1; // TFO succeeded with data in the SYN + u_int8_t necp_tcp_tfo_heuristics_loss:1; // TFO SYN-loss with data + u_int8_t necp_tcp_tfo_heuristics_middlebox:1; // TFO middlebox detected + u_int8_t necp_tcp_tfo_heuristics_success_req:1; // TFO succeeded with the TFO-option in the SYN + u_int8_t necp_tcp_tfo_heuristics_loss_req:1; // TFO SYN-loss with the TFO-option + u_int8_t necp_tcp_tfo_heuristics_rst_data:1; // Recevied RST upon SYN with data in the SYN + u_int8_t necp_tcp_tfo_heuristics_rst_req:1; // Received RST upon SYN with the TFO-option +} necp_tcp_tfo_cache; + +#define NECP_CLIENT_CACHE_TYPE_ECN 1 // Identifies use of necp_tcp_ecn_cache +#define NECP_CLIENT_CACHE_TYPE_TFO 2 // Identifies use of necp_tcp_tfo_cache + +#define NECP_CLIENT_CACHE_TYPE_ECN_VER_1 1 // Currently supported version for ECN +#define NECP_CLIENT_CACHE_TYPE_TFO_VER_1 1 // Currently supported version for TFO + +typedef struct necp_cache_buffer { + u_int8_t necp_cache_buf_type; // NECP_CLIENT_CACHE_TYPE_* + u_int8_t necp_cache_buf_ver; // NECP_CLIENT_CACHE_TYPE_*_VER + u_int32_t necp_cache_buf_size; + mach_vm_address_t necp_cache_buf_addr; +} necp_cache_buffer; + /* * NECP Client definitions */ @@ -343,18 +415,27 @@ typedef struct necp_all_stats { #define NECP_MAX_CLIENT_RESULT_SIZE 512 #define NECP_OPEN_FLAG_OBSERVER 0x01 // Observers can query clients they don't own +#define NECP_OPEN_FLAG_BACKGROUND 0x02 // Mark this fd as backgrounded +#define NECP_OPEN_FLAG_PUSH_OBSERVER 0x04 // When used with the OBSERVER flag, allows updates to be pushed. Adding clients is not allowed in this mode. + +#define NECP_FD_SUPPORTS_GUARD 1 #define NECP_CLIENT_ACTION_ADD 1 // Register a new client. Input: parameters in buffer; Output: client_id -#define NECP_CLIENT_ACTION_REMOVE 2 // Unregister a client. Input: client_id +#define NECP_CLIENT_ACTION_REMOVE 2 // Unregister a client. Input: client_id, optional struct ifnet_stats_per_flow #define NECP_CLIENT_ACTION_COPY_PARAMETERS 3 // Copy client parameters. Input: client_id; Output: parameters in buffer #define NECP_CLIENT_ACTION_COPY_RESULT 4 // Copy client result. Input: client_id; Output: result in buffer #define NECP_CLIENT_ACTION_COPY_LIST 5 // Copy all client IDs. Output: struct necp_client_list in buffer -#define NECP_CLIENT_ACTION_REQUEST_NEXUS_INSTANCE 6 // Request a nexus instance from a nexus provider +#define NECP_CLIENT_ACTION_REQUEST_NEXUS_INSTANCE 6 // Request a nexus instance from a nexus provider, optional struct necp_stats_bufreq #define NECP_CLIENT_ACTION_AGENT 7 // Interact with agent. Input: client_id, agent parameters #define NECP_CLIENT_ACTION_COPY_AGENT 8 // Copy agent content. Input: agent UUID; Output: struct netagent #define NECP_CLIENT_ACTION_COPY_INTERFACE 9 // Copy interface details. Input: ifindex cast to UUID; Output: struct necp_interface_details -#define NECP_CLIENT_ACTION_SET_STATISTICS 10 // Start/update/complete per-flow statistics. Input: client_id, statistics area +#define NECP_CLIENT_ACTION_SET_STATISTICS 10 // Deprecated +#define NECP_CLIENT_ACTION_COPY_ROUTE_STATISTICS 11 // Get route statistics. Input: client_id; Output: struct necp_stat_counts #define NECP_CLIENT_ACTION_AGENT_USE 12 // Return the use count and increment the use count. Input/Output: struct necp_agent_use_parameters +#define NECP_CLIENT_ACTION_MAP_SYSCTLS 13 // Get the read-only sysctls memory location. Output: mach_vm_address_t +#define NECP_CLIENT_ACTION_UPDATE_CACHE 14 // Update heuristics and cache +#define NECP_CLIENT_ACTION_COPY_CLIENT_UPDATE 15 // Fetch an updated client for push-mode observer. Output: Client id, struct necp_client_observer_update in buffer +#define NECP_CLIENT_ACTION_COPY_UPDATED_RESULT 16 // Copy client result only if changed. Input: client_id; Output: result in buffer #define NECP_CLIENT_PARAMETER_APPLICATION NECP_POLICY_CONDITION_APPLICATION // Requires entitlement #define NECP_CLIENT_PARAMETER_REAL_APPLICATION NECP_POLICY_CONDITION_REAL_APPLICATION // Requires entitlement @@ -367,6 +448,7 @@ typedef struct necp_all_stats { #define NECP_CLIENT_PARAMETER_IP_PROTOCOL NECP_POLICY_CONDITION_IP_PROTOCOL #define NECP_CLIENT_PARAMETER_LOCAL_ADDRESS NECP_POLICY_CONDITION_LOCAL_ADDR #define NECP_CLIENT_PARAMETER_REMOTE_ADDRESS NECP_POLICY_CONDITION_REMOTE_ADDR +#define NECP_CLIENT_PARAMETER_NEXUS_KEY 102 // "Prohibit" will never choose an interface with that property #define NECP_CLIENT_PARAMETER_PROHIBIT_INTERFACE 100 // String, interface name @@ -390,7 +472,18 @@ typedef struct necp_all_stats { #define NECP_CLIENT_PARAMETER_LOCAL_ENDPOINT 200 // struct necp_client_endpoint #define NECP_CLIENT_PARAMETER_REMOTE_ENDPOINT 201 // struct necp_client_endpoint -#define NECP_CLIENT_PARAMETER_RESERVED_START 1000 // Parameters 1000 and higher are reserved for custom userspace options +#define NECP_CLIENT_PARAMETER_BROWSE_CATEGORY 202 // struct necp_client_endpoint + +#define NECP_CLIENT_PARAMETER_FLAGS 250 // u_int32_t, see NECP_CLIENT_PAREMETER_FLAG_* values + +#define NECP_CLIENT_PARAMETER_FLAG_MULTIPATH 0x0001 // Get multipath interface results +#define NECP_CLIENT_PARAMETER_FLAG_BROWSE 0x0002 // Agent assertions on nexuses are requests to browse +#define NECP_CLIENT_PARAMETER_FLAG_PROHIBIT_EXPENSIVE 0x0004 // Prohibit expensive interfaces +#define NECP_CLIENT_PARAMETER_FLAG_LISTENER 0x0008 // Client is interested in listening for inbound connections +#define NECP_CLIENT_PARAMETER_FLAG_DISCRETIONARY 0x0010 // Client's traffic is discretionary, and eligible for early defuncting +#define NECP_CLIENT_PARAMETER_FLAG_ECN_ENABLE 0x0020 // Client is requesting to enable ECN +#define NECP_CLIENT_PARAMETER_FLAG_ECN_DISABLE 0x0040 // Client is requesting to disable ECN +#define NECP_CLIENT_PARAMETER_FLAG_TFO_ENABLE 0x0080 // Client is requesting to enable TFO #define NECP_CLIENT_RESULT_CLIENT_ID 1 // uuid_t #define NECP_CLIENT_RESULT_POLICY_RESULT 2 // u_int32_t @@ -400,17 +493,50 @@ typedef struct necp_all_stats { #define NECP_CLIENT_RESULT_NETAGENT 6 // struct necp_client_result_netagent #define NECP_CLIENT_RESULT_FLAGS 7 // u_int32_t, see NECP_CLIENT_RESULT_FLAG_* values #define NECP_CLIENT_RESULT_INTERFACE 8 // struct necp_client_result_interface +#define NECP_CLIENT_RESULT_MULTIPATH_INTERFACE 9 // struct necp_client_result_interface +#define NECP_CLIENT_RESULT_EFFECTIVE_MTU 10 // u_int32_t +#define NECP_CLIENT_RESULT_FLOW 11 // TLV array of a single flow's state +#define NECP_CLIENT_RESULT_PROTO_CTL_EVENT 12 +#define NECP_CLIENT_RESULT_TFO_COOKIE 13 // NECP_TFO_COOKIE_LEN_MAX +#define NECP_CLIENT_RESULT_TFO_FLAGS 14 // u_int8_t +#define NECP_CLIENT_RESULT_RECOMMENDED_MSS 15 // u_int8_t #define NECP_CLIENT_RESULT_NEXUS_INSTANCE 100 // uuid_t #define NECP_CLIENT_RESULT_NEXUS_PORT 101 // u_int16_t +#define NECP_CLIENT_RESULT_NEXUS_KEY 102 // uuid_t +#define NECP_CLIENT_RESULT_NEXUS_PORT_FLOW_INDEX 103 // u_int32_t #define NECP_CLIENT_RESULT_LOCAL_ENDPOINT 200 // struct necp_client_endpoint #define NECP_CLIENT_RESULT_REMOTE_ENDPOINT 201 // struct necp_client_endpoint +#define NECP_CLIENT_RESULT_DISCOVERED_ENDPOINT 202 // struct necp_client_endpoint, result of browse +#define NECP_CLIENT_RESULT_EFFECTIVE_TRAFFIC_CLASS 210 // u_int32_t +#define NECP_CLIENT_RESULT_TRAFFIC_MGMT_BG 211 // u_int32_t, 1: background, 0: not background #define NECP_CLIENT_RESULT_FLAG_IS_LOCAL 0x0001 // Routes to this device #define NECP_CLIENT_RESULT_FLAG_IS_DIRECT 0x0002 // Routes to directly accessible peer #define NECP_CLIENT_RESULT_FLAG_HAS_IPV4 0x0004 // Supports IPv4 #define NECP_CLIENT_RESULT_FLAG_HAS_IPV6 0x0008 // Supports IPv6 +#define NECP_CLIENT_RESULT_FLAG_DEFUNCT 0x0010 // Defunct +#define NECP_CLIENT_RESULT_FLAG_SATISFIED 0x0020 // Satisfied path +#define NECP_CLIENT_RESULT_FLAG_FLOW_ASSIGNED 0x0040 // Assigned, the flow is active +#define NECP_CLIENT_RESULT_FLAG_FLOW_VIABLE 0x0080 // Viable, the flow has a valid route +#define NECP_CLIENT_RESULT_FLAG_PROBE_CONNECTIVITY 0x0100 // Flow should probe connectivity +#define NECP_CLIENT_RESULT_FLAG_ECN_ENABLED 0x0200 // ECN should be used +#define NECP_CLIENT_RESULT_FLAG_FAST_OPEN_BLOCKED 0x0400 // Fast open should not be used +#define NECP_CLIENT_RESULT_FLAG_LINK_QUALITY_ABORT 0x0800 // Link quality is very bad, recommend close connections +#define NECP_CLIENT_RESULT_FLAG_ALLOW_QOS_MARKING 0x1000 // QoS marking is allowed + +#define NECP_CLIENT_RESULT_FAST_OPEN_SND_PROBE 0x01 // DEPRECATED - Fast open send probe +#define NECP_CLIENT_RESULT_FAST_OPEN_RCV_PROBE 0x02 // DEPRECATED - Fast open receive probe + +#define NECP_CLIENT_RESULT_RECOMMENDED_MSS_NONE 0x01 +#define NECP_CLIENT_RESULT_RECOMMENDED_MSS_LOW 0x02 +#define NECP_CLIENT_RESULT_RECOMMENDED_MSS_MEDIUM 0x04 + +struct necp_interface_signature { + u_int8_t signature[IFNET_SIGNATURELEN]; + u_int8_t signature_len; +}; struct necp_interface_details { char name[IFXNAMSIZ]; @@ -420,11 +546,13 @@ struct necp_interface_details { u_int32_t delegate_index; u_int32_t flags; // see NECP_INTERFACE_FLAG_* u_int32_t mtu; - u_int8_t ipv4_signature[IFNET_SIGNATURELEN]; - u_int8_t ipv6_signature[IFNET_SIGNATURELEN]; + struct necp_interface_signature ipv4_signature; + struct necp_interface_signature ipv6_signature; }; #define NECP_INTERFACE_FLAG_EXPENSIVE 0x0001 +#define NECP_INTERFACE_FLAG_TXSTART 0X0002 +#define NECP_INTERFACE_FLAG_NOACKPRI 0x0004 struct necp_client_parameter_netagent_type { char netagent_domain[32]; @@ -470,16 +598,35 @@ struct necp_agent_use_parameters { uint64_t out_use_count; }; +struct necp_client_flow_protoctl_event { + uint32_t protoctl_event_code; + uint32_t protoctl_event_val; + /* TCP seq number is in host byte order */ + uint32_t protoctl_event_tcp_seq_num; +}; + +#define NECP_CLIENT_UPDATE_TYPE_PARAMETERS 1 // Parameters, for a new client +#define NECP_CLIENT_UPDATE_TYPE_RESULT 2 // Result, for a udpated client +#define NECP_CLIENT_UPDATE_TYPE_REMOVE 3 // Empty, for a removed client + +struct necp_client_observer_update { + u_int32_t update_type; // NECP_CLIENT_UPDATE_TYPE_* + u_int8_t tlv_buffer[0]; // Parameters or result as TLVs, based on type +}; + #ifdef BSD_KERNEL_PRIVATE #include #include #include #include +#include #include #include #include #include +SYSCTL_DECL(_net_necp); + #define NECPLOG(level, format, ...) do { \ log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s: " format "\n", __FUNCTION__, __VA_ARGS__); \ } while (0) @@ -488,19 +635,53 @@ struct necp_agent_use_parameters { log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s: %s\n", __FUNCTION__, msg); \ } while (0) +enum necp_fd_type_t { + necp_fd_type_invalid = 0, + necp_fd_type_session = 1, + necp_fd_type_client = 2, +}; + +union necp_sockaddr_union { + struct sockaddr sa; + struct sockaddr_in sin; + struct sockaddr_in6 sin6; +}; + +/* + * kstats + * The ustats and kstats region are mirrored. So when we allocate with + * skmem_cache from kstats region, we also get an ustats object. To tie them + * together, kstats has an extra *necp_stats_ustats pointer pointing to the + * ustats object + */ +struct necp_all_kstats { + struct necp_all_stats necp_stats_comm; /* kernel private stats snapshot */ + struct necp_all_stats *necp_stats_ustats; /* points to user-visible stats (in shared ustats region) */ +}; + extern errno_t necp_client_init(void); extern int necp_application_find_policy_match_internal(proc_t proc, u_int8_t *parameters, u_int32_t parameters_size, - struct necp_aggregate_result *returned_result, - u_int32_t *flags, u_int required_interface_index); + struct necp_aggregate_result *returned_result, + u_int32_t *flags, u_int required_interface_index, + const union necp_sockaddr_union *override_local_addr, + const union necp_sockaddr_union *override_remote_addr, + struct rtentry **returned_route, bool ignore_address); /* * TLV utilities * * Note that these functions (other than necp_buffer_find_tlv) do not check the length of the entire buffer, * so the caller must be sure that the entire TLV is within bounds. */ -extern u_int8_t *necp_buffer_write_tlv(u_int8_t *buffer, u_int8_t type, u_int32_t length, const void *value); -extern u_int8_t *necp_buffer_write_tlv_if_different(u_int8_t *buffer, const u_int8_t *max, u_int8_t type, - u_int32_t length, const void *value, bool *updated); +struct necp_tlv_header { + u_int8_t type; + u_int32_t length; +} __attribute__((__packed__)); + +extern u_int8_t *necp_buffer_write_tlv(u_int8_t *cursor, u_int8_t type, u_int32_t length, const void *value, + u_int8_t *buffer, u_int32_t buffer_length); +extern u_int8_t *necp_buffer_write_tlv_if_different(u_int8_t *cursor, u_int8_t type, + u_int32_t length, const void *value, bool *updated, + u_int8_t *buffer, u_int32_t buffer_length); extern u_int8_t necp_buffer_get_tlv_type(u_int8_t *buffer, int tlv_offset); extern u_int32_t necp_buffer_get_tlv_length(u_int8_t *buffer, int tlv_offset); extern u_int8_t *necp_buffer_get_tlv_value(u_int8_t *buffer, int tlv_offset, u_int32_t *value_size); @@ -514,6 +695,15 @@ extern int necp_buffer_find_tlv(u_int8_t *buffer, u_int32_t buffer_length, int o #define NECPCTL_SOCKET_NON_APP_POLICY_COUNT 6 /* Count of non-per-app socket-level policies */ #define NECPCTL_IP_POLICY_COUNT 7 /* Count of all ip-level policies */ #define NECPCTL_SESSION_COUNT 8 /* Count of NECP sessions */ +#define NECPCTL_CLIENT_FD_COUNT 9 /* Count of NECP client fds */ +#define NECPCTL_CLIENT_COUNT 10 /* Count of NECP clients */ +#define NECPCTL_ARENA_COUNT 11 /* Count of NECP arenas (stats, etc) */ +#define NECPCTL_NEXUS_FLOW_COUNT 12 /* Count of NECP nexus flows */ +#define NECPCTL_SOCKET_FLOW_COUNT 13 /* Count of NECP socket flows */ +#define NECPCTL_IF_FLOW_COUNT 14 /* Count of NECP socket flows */ +#define NECPCTL_OBSERVER_FD_COUNT 15 /* Count of NECP observer fds */ +#define NECPCTL_OBSERVER_MESSAGE_LIMIT 16 /* Number of of NECP observer messages allowed to be queued */ +#define NECPCTL_SYSCTL_ARENA_COUNT 17 /* Count of sysctl arenas */ #define NECPCTL_NAMES { \ { 0, 0 }, \ @@ -526,7 +716,8 @@ extern int necp_buffer_find_tlv(u_int8_t *buffer, u_int32_t buffer_length, int o typedef u_int32_t necp_kernel_policy_id; #define NECP_KERNEL_POLICY_ID_NONE 0 #define NECP_KERNEL_POLICY_ID_NO_MATCH 1 -#define NECP_KERNEL_POLICY_ID_FIRST_VALID 2 +#define NECP_KERNEL_POLICY_ID_FIRST_VALID_SOCKET 2 +#define NECP_KERNEL_POLICY_ID_FIRST_VALID_IP UINT16_MAX typedef u_int32_t necp_app_id; @@ -562,12 +753,6 @@ typedef union { necp_kernel_policy_service service; } necp_kernel_policy_result_parameter; -union necp_sockaddr_union { - struct sockaddr sa; - struct sockaddr_in sin; - struct sockaddr_in6 sin6; -}; - enum necp_boolean_state { necp_boolean_state_unknown = 0, necp_boolean_state_false = 1, @@ -652,7 +837,6 @@ struct necp_session_policy { uuid_t applied_app_uuid; uuid_t applied_real_app_uuid; - char *applied_domain; char *applied_account; uuid_t applied_result_uuid; @@ -683,7 +867,9 @@ extern errno_t necp_init(void); extern errno_t necp_set_socket_attributes(struct socket *so, struct sockopt *sopt); extern errno_t necp_get_socket_attributes(struct socket *so, struct sockopt *sopt); +extern void necp_inpcb_remove_cb(struct inpcb *inp); extern void necp_inpcb_dispose(struct inpcb *inp); +extern void necp_mppcb_dispose(struct mppcb *mpp); extern u_int32_t necp_socket_get_content_filter_control_unit(struct socket *so); @@ -737,9 +923,82 @@ extern void necp_update_all_clients(void); // Handle general re-evaluate event extern void necp_force_update_client(uuid_t client_id, uuid_t remove_netagent_uuid); // Cause a single client to get an update event +extern void necp_client_early_close(uuid_t client_id); // Cause a single client to close stats, etc + +extern void necp_set_client_as_background(proc_t proc, struct fileproc *fp, bool background); // Set all clients for an fp as background or not + +extern void necp_defunct_client(proc_t proc, struct fileproc *fp); // Set all clients for an fp as defunct + +extern int necp_client_register_socket_flow(pid_t pid, uuid_t client_id, struct inpcb *inp); + +extern int necp_client_register_multipath_cb(pid_t pid, uuid_t client_id, struct mppcb *mpp); + +extern int necp_client_assign_from_socket(pid_t pid, uuid_t client_id, struct inpcb *inp); + extern int necp_assign_client_result(uuid_t netagent_uuid, uuid_t client_id, u_int8_t *assigned_results, size_t assigned_results_length); +struct skmem_obj_info; // forward declaration +extern int necp_stats_ctor(struct skmem_obj_info *oi, struct skmem_obj_info *oim, void *arg, uint32_t skmflag); +extern int necp_stats_dtor(void *addr, void *arg); + +/* value to denote invalid flow advisory index */ +struct netagent_session; +extern int +necp_update_flow_protoctl_event(uuid_t netagent_uuid, uuid_t client_id, + uint32_t protoctl_event_code, uint32_t protoctl_event_val, + uint32_t protoctl_event_tcp_seq_num); + +#define NECP_FLOWADV_IDX_INVALID UINT32_MAX +extern void *necp_create_nexus_assign_message(uuid_t nexus_instance, u_int32_t nexus_port, void *key, uint32_t key_length, + struct necp_client_endpoint *local_endpoint, struct necp_client_endpoint *remote_endpoint, + u_int32_t flow_adv_index, size_t *message_length); + +struct necp_client_nexus_parameters { + pid_t pid; + pid_t epid; + uuid_t euuid; + union necp_sockaddr_union local_addr; + union necp_sockaddr_union remote_addr; + u_int16_t ip_protocol; + u_int32_t traffic_class; + necp_policy_id policy_id; + unsigned is_listener:1; + unsigned allow_qos_marking:1; +}; +extern int necp_client_copy_parameters(uuid_t client_uuid, struct necp_client_nexus_parameters *parameters); + +#define NECP_CLIENT_CBACTION_NONVIABLE 1 +#define NECP_CLIENT_CBACTION_VIABLE 2 +#define NECP_CLIENT_CBACTION_INITIAL 3 + +struct necp_client_flow { + LIST_ENTRY(necp_client_flow) flow_chain; + unsigned invalid : 1; + unsigned nexus : 1; // If true, flow is a nexus; if false, flow is attached to socket + unsigned socket : 1; + unsigned viable : 1; + unsigned requested_nexus : 1; + unsigned assigned : 1; + unsigned has_protoctl_event : 1; + unsigned check_tcp_heuristics : 1; + union { + uuid_t nexus_agent; + struct { + void *socket_handle; + void (*cb)(void *handle, int action, struct necp_client_flow *flow); + }; + } u; + uint32_t interface_index; + uint16_t interface_flags; + uint32_t necp_flow_flags; + struct necp_client_flow_protoctl_event protoctl_event; + union necp_sockaddr_union local_addr; + union necp_sockaddr_union remote_addr; + + size_t assigned_results_length; + u_int8_t *assigned_results; +}; #endif /* BSD_KERNEL_PRIVATE */ #ifndef KERNEL @@ -750,6 +1009,12 @@ extern int necp_open(int flags); extern int necp_client_action(int necp_fd, uint32_t action, uuid_t client_id, size_t client_id_len, uint8_t *buffer, size_t buffer_size); +extern int necp_session_open(int flags); + +extern int necp_session_action(int necp_fd, uint32_t action, + uint8_t *in_buffer, size_t in_buffer_length, + uint8_t *out_buffer, size_t out_buffer_length); + #endif /* !KERNEL */ #endif /* PRIVATE */ diff --git a/bsd/net/necp_client.c b/bsd/net/necp_client.c index 33adf5a22..0cb79a5b7 100644 --- a/bsd/net/necp_client.c +++ b/bsd/net/necp_client.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -27,31 +27,45 @@ */ #include -#include -#include -#include -#include + +#include +#include + #include -#include + #include -#include -#include -#include -#include +#include +#include +#include +#include +#include + +#include #include #include -#include -#include +#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include + +#include #include +#include +#include +#include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + /* * NECP Client Architecture @@ -131,41 +145,71 @@ extern u_int32_t necp_debug; +// proc_best_name() is declared here in advance of it landing in a header file. +// See comment in kern_proc.c +extern char *proc_best_name(proc_t p); + static int noop_read(struct fileproc *, struct uio *, int, vfs_context_t); static int noop_write(struct fileproc *, struct uio *, int, vfs_context_t); static int noop_ioctl(struct fileproc *, unsigned long, caddr_t, - vfs_context_t); + vfs_context_t); static int necpop_select(struct fileproc *, int, void *, vfs_context_t); static int necpop_close(struct fileglob *, vfs_context_t); -static int necpop_kqfilter(struct fileproc *, struct knote *, vfs_context_t); +static int necpop_kqfilter(struct fileproc *, struct knote *, + struct kevent_internal_s *kev, vfs_context_t); // Timer functions static int necp_timeout_microseconds = 1000 * 100; // 100ms static int necp_timeout_leeway_microseconds = 1000 * 500; // 500ms + +static int necp_client_fd_count = 0; +static int necp_observer_fd_count = 0; +static int necp_client_count = 0; +static int necp_socket_flow_count = 0; +static int necp_if_flow_count = 0; +static int necp_observer_message_limit = 256; + +SYSCTL_INT(_net_necp, NECPCTL_CLIENT_FD_COUNT, client_fd_count, CTLFLAG_LOCKED | CTLFLAG_RD, &necp_client_fd_count, 0, ""); +SYSCTL_INT(_net_necp, NECPCTL_OBSERVER_FD_COUNT, observer_fd_count, CTLFLAG_LOCKED | CTLFLAG_RD, &necp_observer_fd_count, 0, ""); +SYSCTL_INT(_net_necp, NECPCTL_CLIENT_COUNT, client_count, CTLFLAG_LOCKED | CTLFLAG_RD, &necp_client_count, 0, ""); +SYSCTL_INT(_net_necp, NECPCTL_SOCKET_FLOW_COUNT, socket_flow_count, CTLFLAG_LOCKED | CTLFLAG_RD, &necp_socket_flow_count, 0, ""); +SYSCTL_INT(_net_necp, NECPCTL_IF_FLOW_COUNT, if_flow_count, CTLFLAG_LOCKED | CTLFLAG_RD, &necp_if_flow_count, 0, ""); +SYSCTL_INT(_net_necp, NECPCTL_OBSERVER_MESSAGE_LIMIT, observer_message_limit, CTLFLAG_LOCKED | CTLFLAG_RW, &necp_observer_message_limit, 256, ""); + +#define NECP_MAX_CLIENT_LIST_SIZE 1024 * 1024 // 1MB + extern int tvtohz(struct timeval *); +extern unsigned int get_maxmtu(struct rtentry *); // Parsed parameters -#define NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR 0x0001 -#define NECP_PARSED_PARAMETERS_FIELD_REMOTE_ADDR 0x0002 -#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IF 0x0004 -#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_IF 0x0008 -#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE 0x0010 -#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_IFTYPE 0x0020 -#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_AGENT 0x0040 -#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_AGENT 0x0080 -#define NECP_PARSED_PARAMETERS_FIELD_PREFERRED_AGENT 0x0100 -#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_AGENT_TYPE 0x0200 -#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_AGENT_TYPE 0x0400 -#define NECP_PARSED_PARAMETERS_FIELD_PREFERRED_AGENT_TYPE 0x0800 +#define NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR 0x00001 +#define NECP_PARSED_PARAMETERS_FIELD_REMOTE_ADDR 0x00002 +#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IF 0x00004 +#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_IF 0x00008 +#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE 0x00010 +#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_IFTYPE 0x00020 +#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_AGENT 0x00040 +#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_AGENT 0x00080 +#define NECP_PARSED_PARAMETERS_FIELD_PREFERRED_AGENT 0x00100 +#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_AGENT_TYPE 0x00200 +#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_AGENT_TYPE 0x00400 +#define NECP_PARSED_PARAMETERS_FIELD_PREFERRED_AGENT_TYPE 0x00800 +#define NECP_PARSED_PARAMETERS_FIELD_FLAGS 0x01000 +#define NECP_PARSED_PARAMETERS_FIELD_IP_PROTOCOL 0x02000 +#define NECP_PARSED_PARAMETERS_FIELD_EFFECTIVE_PID 0x04000 +#define NECP_PARSED_PARAMETERS_FIELD_EFFECTIVE_UUID 0x08000 +#define NECP_PARSED_PARAMETERS_FIELD_TRAFFIC_CLASS 0x10000 +#define NECP_PARSED_PARAMETERS_FIELD_LOCAL_PORT 0x20000 #define NECP_MAX_PARSED_PARAMETERS 16 struct necp_client_parsed_parameters { u_int32_t valid_fields; + u_int32_t flags; union necp_sockaddr_union local_addr; union necp_sockaddr_union remote_addr; u_int32_t required_interface_index; char prohibited_interfaces[IFXNAMSIZ][NECP_MAX_PARSED_PARAMETERS]; - u_int8_t required_interface_types[NECP_MAX_PARSED_PARAMETERS]; + u_int8_t required_interface_type; u_int8_t prohibited_interface_types[NECP_MAX_PARSED_PARAMETERS]; struct necp_client_parameter_netagent_type required_netagent_types[NECP_MAX_PARSED_PARAMETERS]; struct necp_client_parameter_netagent_type prohibited_netagent_types[NECP_MAX_PARSED_PARAMETERS]; @@ -173,9 +217,23 @@ struct necp_client_parsed_parameters { uuid_t required_netagents[NECP_MAX_PARSED_PARAMETERS]; uuid_t prohibited_netagents[NECP_MAX_PARSED_PARAMETERS]; uuid_t preferred_netagents[NECP_MAX_PARSED_PARAMETERS]; + u_int16_t ip_protocol; + pid_t effective_pid; + uuid_t effective_uuid; + u_int32_t traffic_class; }; -static bool necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_parameters, u_int *return_ifindex); +static bool +necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_parameters, + u_int *return_ifindex); + +static bool +necp_ifnet_matches_local_address(struct ifnet *ifp, struct sockaddr *sa); + +static bool +necp_ifnet_matches_parameters(struct ifnet *ifp, + struct necp_client_parsed_parameters *parsed_parameters, + u_int32_t *preferred_count, bool ignore_require_if); static const struct fileops necp_fd_ops = { .fo_type = DTYPE_NETPOLICY, @@ -193,48 +251,194 @@ struct necp_client_assertion { uuid_t asserted_netagent; }; +struct necp_client_flow_header { + struct necp_tlv_header outer_header; + struct necp_tlv_header flags_tlv_header; + u_int32_t flags_value; + struct necp_tlv_header interface_tlv_header; + struct necp_client_result_interface interface_value; +} __attribute__((__packed__)); + +struct necp_client_flow_protoctl_event_header { + struct necp_tlv_header protoctl_tlv_header; + struct necp_client_flow_protoctl_event protoctl_event; +} __attribute__((__packed__)); + +struct necp_client_nexus_flow_header { + struct necp_client_flow_header flow_header; + struct necp_tlv_header agent_tlv_header; + struct necp_client_result_netagent agent_value; + struct necp_tlv_header tfo_cookie_tlv_header; + u_int8_t tfo_cookie_value[NECP_TFO_COOKIE_LEN_MAX]; +} __attribute__((__packed__)); + struct necp_client { - LIST_ENTRY(necp_client) chain; + RB_ENTRY(necp_client) link; + RB_ENTRY(necp_client) global_link; + LIST_ENTRY(necp_client) collect_stats_chain; + + decl_lck_mtx_data(, lock); + decl_lck_mtx_data(, route_lock); + uint32_t reference_count; uuid_t client_id; - bool result_read; - bool assigned_result_read; + unsigned result_read : 1; + unsigned flow_result_read : 1; + unsigned allow_multiple_flows : 1; + + unsigned defunct : 1; + unsigned background : 1; + unsigned background_update : 1; + unsigned platform_binary : 1; size_t result_length; u_int8_t result[NECP_MAX_CLIENT_RESULT_SIZE]; - uuid_t nexus_agent; - size_t assigned_results_length; - u_int8_t *assigned_results; + necp_policy_id policy_id; + + u_int16_t ip_protocol; + int proc_pid; + LIST_HEAD(_necp_client_flow_list, necp_client_flow) flow_list; LIST_HEAD(_necp_client_assertion_list, necp_client_assertion) assertion_list; - user_addr_t stats_uaddr; - user_size_t stats_ulen; - nstat_userland_context stats_handler_context; - necp_stats_hdr *stats_area; + struct rtentry *current_route; + + void *interface_handle; + void (*interface_cb)(void *handle, int action, struct necp_client_flow *flow); size_t parameters_length; u_int8_t parameters[0]; }; +#define NECP_CLIENT_LOCK(_c) lck_mtx_lock(&_c->lock) +#define NECP_CLIENT_UNLOCK(_c) lck_mtx_unlock(&_c->lock) +#define NECP_CLIENT_ASSERT_LOCKED(_c) LCK_MTX_ASSERT(&_c->lock, LCK_MTX_ASSERT_OWNED) +#define NECP_CLIENT_ASSERT_UNLOCKED(_c) LCK_MTX_ASSERT(&_c->lock, LCK_MTX_ASSERT_NOTOWNED) + +#define NECP_CLIENT_ROUTE_LOCK(_c) lck_mtx_lock(&_c->route_lock) +#define NECP_CLIENT_ROUTE_UNLOCK(_c) lck_mtx_unlock(&_c->route_lock) + +static void necp_client_retain_locked(struct necp_client *client); +static void necp_client_retain(struct necp_client *client); +static bool necp_client_release_locked(struct necp_client *client); + +static void +necp_client_add_assertion(struct necp_client *client, uuid_t netagent_uuid); + +static bool +necp_client_remove_assertion(struct necp_client *client, uuid_t netagent_uuid); + +LIST_HEAD(_necp_client_list, necp_client); +static struct _necp_client_list necp_collect_stats_client_list; + +struct necp_client_defunct { + LIST_ENTRY(necp_client_defunct) chain; + + uuid_t client_id; + uuid_t nexus_agent; + int proc_pid; +}; + +LIST_HEAD(_necp_client_defunct_list, necp_client_defunct); + +static int necp_client_id_cmp(struct necp_client *client0, struct necp_client *client1); + +RB_HEAD(_necp_client_tree, necp_client); +RB_PROTOTYPE_PREV(_necp_client_tree, necp_client, link, necp_client_id_cmp); +RB_GENERATE_PREV(_necp_client_tree, necp_client, link, necp_client_id_cmp); + +RB_HEAD(_necp_client_global_tree, necp_client); +RB_PROTOTYPE_PREV(_necp_client_global_tree, necp_client, global_link, necp_client_id_cmp); +RB_GENERATE_PREV(_necp_client_global_tree, necp_client, global_link, necp_client_id_cmp); + +static struct _necp_client_global_tree necp_client_global_tree; + +struct necp_client_update { + TAILQ_ENTRY(necp_client_update) chain; + + uuid_t client_id; + + size_t update_length; + struct necp_client_observer_update update; +}; + struct necp_fd_data { + u_int8_t necp_fd_type; LIST_ENTRY(necp_fd_data) chain; - LIST_HEAD(_clients, necp_client) clients; + struct _necp_client_tree clients; + TAILQ_HEAD(_necp_client_update_list, necp_client_update) update_list; + int update_count; int flags; int proc_pid; decl_lck_mtx_data(, fd_lock); struct selinfo si; }; +#define NECP_FD_LOCK(_f) lck_mtx_lock(&_f->fd_lock) +#define NECP_FD_UNLOCK(_f) lck_mtx_unlock(&_f->fd_lock) +#define NECP_FD_ASSERT_LOCKED(_f) LCK_MTX_ASSERT(&_f->fd_lock, LCK_MTX_ASSERT_OWNED) +#define NECP_FD_ASSERT_UNLOCKED(_f) LCK_MTX_ASSERT(&_f->fd_lock, LCK_MTX_ASSERT_NOTOWNED) + static LIST_HEAD(_necp_fd_list, necp_fd_data) necp_fd_list; +static LIST_HEAD(_necp_fd_observer_list, necp_fd_data) necp_fd_observer_list; + +#define NECP_CLIENT_FD_ZONE_MAX 128 +#define NECP_CLIENT_FD_ZONE_NAME "necp.clientfd" + +static unsigned int necp_client_fd_size; /* size of zone element */ +static struct zone *necp_client_fd_zone; /* zone for necp_fd_data */ + +#define NECP_FLOW_ZONE_MAX 512 +#define NECP_FLOW_ZONE_NAME "necp.flow" + +static unsigned int necp_flow_size; /* size of zone element */ +static struct zone *necp_flow_zone; /* zone for necp_client_flow */ static lck_grp_attr_t *necp_fd_grp_attr = NULL; static lck_attr_t *necp_fd_mtx_attr = NULL; static lck_grp_t *necp_fd_mtx_grp = NULL; + decl_lck_rw_data(static, necp_fd_lock); +decl_lck_rw_data(static, necp_observer_lock); +decl_lck_rw_data(static, necp_client_tree_lock); +decl_lck_rw_data(static, necp_collect_stats_list_lock); + +#define NECP_STATS_LIST_LOCK_EXCLUSIVE() lck_rw_lock_exclusive(&necp_collect_stats_list_lock) +#define NECP_STATS_LIST_LOCK_SHARED() lck_rw_lock_shared(&necp_collect_stats_list_lock) +#define NECP_STATS_LIST_UNLOCK() lck_rw_done(&necp_collect_stats_list_lock) + +#define NECP_CLIENT_TREE_LOCK_EXCLUSIVE() lck_rw_lock_exclusive(&necp_client_tree_lock) +#define NECP_CLIENT_TREE_LOCK_SHARED() lck_rw_lock_shared(&necp_client_tree_lock) +#define NECP_CLIENT_TREE_UNLOCK() lck_rw_done(&necp_client_tree_lock) + +#define NECP_FD_LIST_LOCK_EXCLUSIVE() lck_rw_lock_exclusive(&necp_fd_lock) +#define NECP_FD_LIST_LOCK_SHARED() lck_rw_lock_shared(&necp_fd_lock) +#define NECP_FD_LIST_UNLOCK() lck_rw_done(&necp_fd_lock) + +#define NECP_OBSERVER_LIST_LOCK_EXCLUSIVE() lck_rw_lock_exclusive(&necp_observer_lock) +#define NECP_OBSERVER_LIST_LOCK_SHARED() lck_rw_lock_shared(&necp_observer_lock) +#define NECP_OBSERVER_LIST_UNLOCK() lck_rw_done(&necp_observer_lock) + +// Locking Notes + +// Take NECP_FD_LIST_LOCK when accessing or modifying the necp_fd_list +// Take NECP_CLIENT_TREE_LOCK when accessing or modifying the necp_client_global_tree +// Take NECP_STATS_LIST_LOCK when accessing or modifying the necp_collect_stats_client_list +// Take NECP_FD_LOCK when accessing or modifying an necp_fd_data entry +// Take NECP_CLIENT_LOCK when accessing or modifying a single necp_client +// Take NECP_CLIENT_ROUTE_LOCK when accessing or modifying a client's route + +// Precedence, where 1 is the first lock that must be taken +// 1. NECP_FD_LIST_LOCK +// 2. NECP_FD_LOCK (any) +// 3. NECP_CLIENT_TREE_LOCK +// 4. NECP_CLIENT_LOCK (any) +// 5. NECP_STATS_LIST_LOCK +// 6. NECP_CLIENT_ROUTE_LOCK (any) + +static thread_call_t necp_client_update_tcall; -static thread_call_t necp_client_tcall; /// NECP file descriptor functions @@ -267,7 +471,7 @@ necp_fd_notify(struct necp_fd_data *fd_data, bool locked) struct selinfo *si = &fd_data->si; if (!locked) { - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); } selwakeup(si); @@ -277,7 +481,7 @@ necp_fd_notify(struct necp_fd_data *fd_data, bool locked) KNOTE(&si->si_note, 1); // notification if (!locked) { - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); } } @@ -286,27 +490,59 @@ necp_fd_poll(struct necp_fd_data *fd_data, int events, void *wql, struct proc *p { #pragma unused(wql, p, is_kevent) u_int revents = 0; - struct necp_client *client = NULL; - bool has_unread_clients = FALSE; u_int want_rx = events & (POLLIN | POLLRDNORM); if (want_rx) { - - LIST_FOREACH(client, &fd_data->clients, chain) { - if (!client->result_read || !client->assigned_result_read) { - has_unread_clients = TRUE; - break; + if (fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER) { + // Push-mode observers are readable when they have a new update + if (!TAILQ_EMPTY(&fd_data->update_list)) { + revents |= want_rx; + } + } else { + // Standard fds are readable when some client is unread + struct necp_client *client = NULL; + bool has_unread_clients = FALSE; + RB_FOREACH(client, _necp_client_tree, &fd_data->clients) { + NECP_CLIENT_LOCK(client); + if (!client->result_read || !client->flow_result_read) { + has_unread_clients = TRUE; + } + NECP_CLIENT_UNLOCK(client); + if (has_unread_clients) { + break; + } } - } - if (has_unread_clients) { - revents |= want_rx; + if (has_unread_clients) { + revents |= want_rx; + } } } return (revents); } +static struct necp_client * +necp_client_fd_find_client_and_lock(struct necp_fd_data *client_fd, uuid_t client_id) +{ + struct necp_client find; + NECP_FD_ASSERT_LOCKED(client_fd); + uuid_copy(find.client_id, client_id); + struct necp_client *client = RB_FIND(_necp_client_tree, &client_fd->clients, &find); + + if (client != NULL) { + NECP_CLIENT_LOCK(client); + } + + return (client); +} + +static inline int +necp_client_id_cmp(struct necp_client *client0, struct necp_client *client1) +{ + return (uuid_compare(client0->client_id, client1->client_id)); +} + static int necpop_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx) { @@ -335,9 +571,9 @@ necpop_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx) } } - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); revents = necp_fd_poll(fd_data, events, wql, procp, 0); - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); return ((events & revents) ? 1 : 0); } @@ -348,9 +584,9 @@ necp_fd_knrdetach(struct knote *kn) struct necp_fd_data *fd_data = (struct necp_fd_data *)kn->kn_hook; struct selinfo *si = &fd_data->si; - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); KNOTE_DETACH(&si->si_note, kn); - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); } static int @@ -370,17 +606,17 @@ necp_fd_knrprocess(struct knote *kn, struct filt_process_s *data, struct kevent_ fd_data = (struct necp_fd_data *)kn->kn_hook; - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); revents = necp_fd_poll(fd_data, POLLIN, NULL, current_proc(), 1); res = ((revents & POLLIN) != 0); if (res) { *kev = kn->kn_kevent; } - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); return (res); } -static int +static int necp_fd_knrtouch(struct knote *kn, struct kevent_internal_s *kev) { #pragma unused(kev) @@ -389,16 +625,16 @@ necp_fd_knrtouch(struct knote *kn, struct kevent_internal_s *kev) fd_data = (struct necp_fd_data *)kn->kn_hook; - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) kn->kn_udata = kev->udata; revents = necp_fd_poll(fd_data, POLLIN, NULL, current_proc(), 1); - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); return ((revents & POLLIN) != 0); } -struct filterops necp_fd_rfiltops = { +SECURITY_READ_ONLY_EARLY(struct filterops) necp_fd_rfiltops = { .f_isfd = 1, .f_detach = necp_fd_knrdetach, .f_event = necp_fd_knread, @@ -407,7 +643,8 @@ struct filterops necp_fd_rfiltops = { }; static int -necpop_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) +necpop_kqfilter(struct fileproc *fp, struct knote *kn, + __unused struct kevent_internal_s *kev, vfs_context_t ctx) { #pragma unused(fp, ctx) struct necp_fd_data *fd_data = NULL; @@ -428,77 +665,326 @@ necpop_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) return (0); } - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); kn->kn_filtid = EVFILTID_NECP_FD; kn->kn_hook = fd_data; KNOTE_ATTACH(&fd_data->si.si_note, kn); revents = necp_fd_poll(fd_data, POLLIN, NULL, current_proc(), 1); - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); return ((revents & POLLIN) != 0); } + +static bool +necp_set_client_defunct(struct necp_client *client) +{ + bool updated = FALSE; + u_int32_t flags = 0; + u_int32_t value_size = 0; + + client->defunct = TRUE; + + u_int8_t *flags_pointer = necp_buffer_get_tlv_value(client->result, 0, &value_size); + if (flags_pointer && value_size == sizeof(flags)) { + memcpy(&flags, flags_pointer, value_size); + + flags |= NECP_CLIENT_RESULT_FLAG_DEFUNCT; + + (void)necp_buffer_write_tlv_if_different(client->result, NECP_CLIENT_RESULT_FLAGS, + sizeof(flags), &flags, &updated, client->result, sizeof(client->result)); + } + + return (updated); +} + static void -necp_destroy_client_stats(struct necp_client *client) +necp_defunct_client_for_policy(struct necp_client *client, + struct _necp_client_defunct_list *defunct_list) { - if ((client->stats_area != NULL) && - (client->stats_handler_context != NULL) && - (client->stats_uaddr != 0)) { - // Close old stats if required. - int error = copyin(client->stats_uaddr, client->stats_area, client->stats_ulen); - if (error) { - NECPLOG(LOG_ERR, "necp_destroy_client_stats copyin error on close (%d)", error); - // Not much we can for an error on an obsolete address + if (!client->defunct) { + bool needs_defunct = false; + struct necp_client_flow *search_flow = NULL; + LIST_FOREACH(search_flow, &client->flow_list, flow_chain) { + if (search_flow->nexus && + !uuid_is_null(search_flow->u.nexus_agent) && + search_flow->requested_nexus) { + + // Save defunct values for the nexus + if (defunct_list != NULL) { + // Sleeping alloc won't fail; copy only what's necessary + struct necp_client_defunct *client_defunct = _MALLOC(sizeof (struct necp_client_defunct), + M_NECP, M_WAITOK | M_ZERO); + uuid_copy(client_defunct->nexus_agent, search_flow->u.nexus_agent); + uuid_copy(client_defunct->client_id, client->client_id); + client_defunct->proc_pid = client->proc_pid; + + // Add to the list provided by caller + LIST_INSERT_HEAD(defunct_list, client_defunct, chain); + } + + needs_defunct = true; + } + } + + if (needs_defunct) { + // Only set defunct if there was some assigned flow + client->defunct = true; } - ntstat_userland_stats_close(client->stats_handler_context); - FREE(client->stats_area, M_NECP); - client->stats_area = NULL; - client->stats_handler_context = NULL; - client->stats_uaddr = 0; - client->stats_ulen = 0; } } static void -necp_destroy_client(struct necp_client *client) +necp_client_free(struct necp_client *client) +{ + NECP_CLIENT_ASSERT_LOCKED(client); + + NECP_CLIENT_UNLOCK(client); + + lck_mtx_destroy(&client->route_lock, necp_fd_mtx_grp); + lck_mtx_destroy(&client->lock, necp_fd_mtx_grp); + + FREE(client, M_NECP); +} + +static void +necp_client_retain_locked(struct necp_client *client) +{ + NECP_CLIENT_ASSERT_LOCKED(client); + + client->reference_count++; + ASSERT(client->reference_count != 0); +} + +static void +necp_client_retain(struct necp_client *client) +{ + NECP_CLIENT_LOCK(client); + necp_client_retain_locked(client); + NECP_CLIENT_UNLOCK(client); +} + +static bool +necp_client_release_locked(struct necp_client *client) +{ + NECP_CLIENT_ASSERT_LOCKED(client); + + uint32_t old_ref = client->reference_count; + + ASSERT(client->reference_count != 0); + if (--client->reference_count == 0) { + necp_client_free(client); + } + + return (old_ref == 1); +} + + +static void +necp_client_update_observer_add_internal(struct necp_fd_data *observer_fd, struct necp_client *client) +{ + NECP_FD_LOCK(observer_fd); + + if (observer_fd->update_count >= necp_observer_message_limit) { + NECP_FD_UNLOCK(observer_fd); + return; + } + + struct necp_client_update *client_update = _MALLOC(sizeof(struct necp_client_update) + client->parameters_length, + M_NECP, M_WAITOK | M_ZERO); + if (client_update != NULL) { + client_update->update_length = sizeof(struct necp_client_observer_update) + client->parameters_length; + uuid_copy(client_update->client_id, client->client_id); + client_update->update.update_type = NECP_CLIENT_UPDATE_TYPE_PARAMETERS; + memcpy(client_update->update.tlv_buffer, client->parameters, client->parameters_length); + TAILQ_INSERT_TAIL(&observer_fd->update_list, client_update, chain); + observer_fd->update_count++; + + necp_fd_notify(observer_fd, true); + } + + NECP_FD_UNLOCK(observer_fd); +} + +static void +necp_client_update_observer_update_internal(struct necp_fd_data *observer_fd, struct necp_client *client) +{ + NECP_FD_LOCK(observer_fd); + + if (observer_fd->update_count >= necp_observer_message_limit) { + NECP_FD_UNLOCK(observer_fd); + return; + } + + struct necp_client_update *client_update = _MALLOC(sizeof(struct necp_client_update) + client->result_length, + M_NECP, M_WAITOK | M_ZERO); + if (client_update != NULL) { + client_update->update_length = sizeof(struct necp_client_observer_update) + client->result_length; + uuid_copy(client_update->client_id, client->client_id); + client_update->update.update_type = NECP_CLIENT_UPDATE_TYPE_RESULT; + memcpy(client_update->update.tlv_buffer, client->result, client->result_length); + TAILQ_INSERT_TAIL(&observer_fd->update_list, client_update, chain); + observer_fd->update_count++; + + necp_fd_notify(observer_fd, true); + } + + NECP_FD_UNLOCK(observer_fd); +} + +static void +necp_client_update_observer_remove_internal(struct necp_fd_data *observer_fd, struct necp_client *client) +{ + NECP_FD_LOCK(observer_fd); + + if (observer_fd->update_count >= necp_observer_message_limit) { + NECP_FD_UNLOCK(observer_fd); + return; + } + + struct necp_client_update *client_update = _MALLOC(sizeof(struct necp_client_update), + M_NECP, M_WAITOK | M_ZERO); + if (client_update != NULL) { + client_update->update_length = sizeof(struct necp_client_observer_update); + uuid_copy(client_update->client_id, client->client_id); + client_update->update.update_type = NECP_CLIENT_UPDATE_TYPE_REMOVE; + TAILQ_INSERT_TAIL(&observer_fd->update_list, client_update, chain); + observer_fd->update_count++; + + necp_fd_notify(observer_fd, true); + } + + NECP_FD_UNLOCK(observer_fd); +} + +static void +necp_client_update_observer_add(struct necp_client *client) +{ + NECP_OBSERVER_LIST_LOCK_SHARED(); + + if (LIST_EMPTY(&necp_fd_observer_list)) { + // No observers, bail + NECP_OBSERVER_LIST_UNLOCK(); + return; + } + + struct necp_fd_data *observer_fd = NULL; + LIST_FOREACH(observer_fd, &necp_fd_observer_list, chain) { + necp_client_update_observer_add_internal(observer_fd, client); + } + + NECP_OBSERVER_LIST_UNLOCK(); +} + +static void +necp_client_update_observer_update(struct necp_client *client) +{ + NECP_OBSERVER_LIST_LOCK_SHARED(); + + if (LIST_EMPTY(&necp_fd_observer_list)) { + // No observers, bail + NECP_OBSERVER_LIST_UNLOCK(); + return; + } + + struct necp_fd_data *observer_fd = NULL; + LIST_FOREACH(observer_fd, &necp_fd_observer_list, chain) { + necp_client_update_observer_update_internal(observer_fd, client); + } + + NECP_OBSERVER_LIST_UNLOCK(); +} + +static void +necp_client_update_observer_remove(struct necp_client *client) { - // Remove from list - LIST_REMOVE(client, chain); + NECP_OBSERVER_LIST_LOCK_SHARED(); + + if (LIST_EMPTY(&necp_fd_observer_list)) { + // No observers, bail + NECP_OBSERVER_LIST_UNLOCK(); + return; + } + + struct necp_fd_data *observer_fd = NULL; + LIST_FOREACH(observer_fd, &necp_fd_observer_list, chain) { + necp_client_update_observer_remove_internal(observer_fd, client); + } - // Remove nexus assignment - if (client->assigned_results != NULL) { - if (!uuid_is_null(client->nexus_agent)) { - int netagent_error = netagent_client_message(client->nexus_agent, client->client_id, - NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS); - if (netagent_error != 0) { + NECP_OBSERVER_LIST_UNLOCK(); +} + +static void +necp_destroy_client(struct necp_client *client, pid_t pid, bool abort) +{ + NECP_CLIENT_ASSERT_UNLOCKED(client); + + necp_client_update_observer_remove(client); + + NECP_CLIENT_LOCK(client); + + // Free route + NECP_CLIENT_ROUTE_LOCK(client); + if (client->current_route != NULL) { + rtfree(client->current_route); + client->current_route = NULL; + } + NECP_CLIENT_ROUTE_UNLOCK(client); + + // Remove flow assignments + struct necp_client_flow *search_flow = NULL; + struct necp_client_flow *temp_flow = NULL; + LIST_FOREACH_SAFE(search_flow, &client->flow_list, flow_chain, temp_flow) { + if (search_flow->nexus && + !uuid_is_null(search_flow->u.nexus_agent) && + search_flow->requested_nexus) { + // Note that if we had defuncted the client earlier, this would result in a harmless ENOENT + int netagent_error = netagent_client_message(search_flow->u.nexus_agent, client->client_id, pid, + abort ? NETAGENT_MESSAGE_TYPE_ABORT_NEXUS : NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS); + if (netagent_error != 0 && netagent_error != ENOENT) { NECPLOG(LOG_ERR, "necp_client_remove close nexus error (%d)", netagent_error); } + uuid_clear(search_flow->u.nexus_agent); } - FREE(client->assigned_results, M_NETAGENT); + if (search_flow->assigned_results != NULL) { + FREE(search_flow->assigned_results, M_NETAGENT); + search_flow->assigned_results = NULL; + } + LIST_REMOVE(search_flow, flow_chain); + if (search_flow->socket) { + OSDecrementAtomic(&necp_socket_flow_count); + } else { + OSDecrementAtomic(&necp_if_flow_count); + } + zfree(necp_flow_zone, search_flow); } // Remove agent assertions struct necp_client_assertion *search_assertion = NULL; struct necp_client_assertion *temp_assertion = NULL; LIST_FOREACH_SAFE(search_assertion, &client->assertion_list, assertion_chain, temp_assertion) { - int netagent_error = netagent_client_message(search_assertion->asserted_netagent, client->client_id, NETAGENT_MESSAGE_TYPE_CLIENT_UNASSERT); + int netagent_error = netagent_client_message(search_assertion->asserted_netagent, client->client_id, pid, NETAGENT_MESSAGE_TYPE_CLIENT_UNASSERT); if (netagent_error != 0) { - NECPLOG(LOG_ERR, "necp_client_remove unassert agent error (%d)", netagent_error); + NECPLOG((netagent_error == ENOENT ? LOG_DEBUG : LOG_ERR), + "necp_client_remove unassert agent error (%d)", netagent_error); } LIST_REMOVE(search_assertion, assertion_chain); FREE(search_assertion, M_NECP); } - necp_destroy_client_stats(client); - FREE(client, M_NECP); + if (!necp_client_release_locked(client)) { + NECP_CLIENT_UNLOCK(client); + } + + OSDecrementAtomic(&necp_client_count); } static int necpop_close(struct fileglob *fg, vfs_context_t ctx) { -#pragma unused(fg, ctx) +#pragma unused(ctx) struct necp_fd_data *fd_data = NULL; int error = 0; @@ -506,26 +992,61 @@ necpop_close(struct fileglob *fg, vfs_context_t ctx) fg->fg_data = NULL; if (fd_data != NULL) { - lck_rw_lock_exclusive(&necp_fd_lock); + struct _necp_client_tree clients_to_close; + RB_INIT(&clients_to_close); + + // Remove from list quickly + if (fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER) { + NECP_OBSERVER_LIST_LOCK_EXCLUSIVE(); + LIST_REMOVE(fd_data, chain); + NECP_OBSERVER_LIST_UNLOCK(); + } else { + NECP_FD_LIST_LOCK_EXCLUSIVE(); + LIST_REMOVE(fd_data, chain); + NECP_FD_LIST_UNLOCK(); + } - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); + pid_t pid = fd_data->proc_pid; struct necp_client *client = NULL; struct necp_client *temp_client = NULL; - LIST_FOREACH_SAFE(client, &fd_data->clients, chain, temp_client) { - necp_destroy_client(client); + RB_FOREACH_SAFE(client, _necp_client_tree, &fd_data->clients, temp_client) { + NECP_CLIENT_TREE_LOCK_EXCLUSIVE(); + RB_REMOVE(_necp_client_global_tree, &necp_client_global_tree, client); + NECP_CLIENT_TREE_UNLOCK(); + RB_REMOVE(_necp_client_tree, &fd_data->clients, client); + RB_INSERT(_necp_client_tree, &clients_to_close, client); } - lck_mtx_unlock(&fd_data->fd_lock); + + struct necp_client_update *client_update = NULL; + struct necp_client_update *temp_update = NULL; + TAILQ_FOREACH_SAFE(client_update, &fd_data->update_list, chain, temp_update) { + // Flush pending updates + TAILQ_REMOVE(&fd_data->update_list, client_update, chain); + FREE(client_update, M_NECP); + } + fd_data->update_count = 0; + + + NECP_FD_UNLOCK(fd_data); selthreadclear(&fd_data->si); lck_mtx_destroy(&fd_data->fd_lock, necp_fd_mtx_grp); - LIST_REMOVE(fd_data, chain); - - lck_rw_done(&necp_fd_lock); + if (fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER) { + OSDecrementAtomic(&necp_observer_fd_count); + } else { + OSDecrementAtomic(&necp_client_fd_count); + } - FREE(fd_data, M_NECP); + zfree(necp_client_fd_zone, fd_data); fd_data = NULL; + + RB_FOREACH_SAFE(client, _necp_client_tree, &clients_to_close, temp_client) { + RB_REMOVE(_necp_client_tree, &clients_to_close, client); + necp_destroy_client(client, pid, true); + } } return (error); @@ -533,6 +1054,13 @@ necpop_close(struct fileglob *fg, vfs_context_t ctx) /// NECP client utilities +static inline bool +necp_address_is_wildcard(const union necp_sockaddr_union * const addr) +{ + return ((addr->sa.sa_family == AF_INET && addr->sin.sin_addr.s_addr == INADDR_ANY) || + (addr->sa.sa_family == AF_INET6 && IN6_IS_ADDR_UNSPECIFIED(&addr->sin6.sin6_addr))); +} + static int necp_find_fd_data(int fd, struct necp_fd_data **fd_data) { @@ -556,15 +1084,183 @@ done: return (error); } -static bool -necp_netagent_applies_to_client(__unused struct necp_client *client, struct necp_client_parsed_parameters *parameters, uuid_t netagent_uuid) -{ - bool applies = FALSE; - u_int32_t flags = netagent_get_flags(netagent_uuid); - if (!(flags & NETAGENT_FLAG_REGISTERED)) { - // Unregistered agents never apply - return (applies); - } + +static void +necp_client_add_socket_flow(struct necp_client *client, struct inpcb *inp) +{ + struct necp_client_flow *new_flow = zalloc(necp_flow_zone); + if (new_flow == NULL) { + NECPLOG0(LOG_ERR, "Failed to allocate socket flow"); + return; + } + + memset(new_flow, 0, sizeof(*new_flow)); + + new_flow->socket = TRUE; + new_flow->u.socket_handle = inp; + new_flow->u.cb = inp->necp_cb; + + OSIncrementAtomic(&necp_socket_flow_count); + + LIST_INSERT_HEAD(&client->flow_list, new_flow, flow_chain); +} + +static void +necp_client_add_interface_flow(struct necp_client *client, uint32_t interface_index) +{ + struct necp_client_flow *new_flow = zalloc(necp_flow_zone); + if (new_flow == NULL) { + NECPLOG0(LOG_ERR, "Failed to allocate interface flow"); + return; + } + + memset(new_flow, 0, sizeof(*new_flow)); + + // Neither nexus nor socket + new_flow->interface_index = interface_index; + new_flow->u.socket_handle = client->interface_handle; + new_flow->u.cb = client->interface_cb; + + OSIncrementAtomic(&necp_if_flow_count); + + LIST_INSERT_HEAD(&client->flow_list, new_flow, flow_chain); +} + +static void +necp_client_add_interface_flow_if_needed(struct necp_client *client, uint32_t interface_index) +{ + if (!client->allow_multiple_flows || + interface_index == IFSCOPE_NONE) { + // Interface not set, or client not allowed to use this mode + return; + } + + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (!flow->nexus && !flow->socket && flow->interface_index == interface_index) { + // Already have the flow + flow->invalid = FALSE; + + flow->u.socket_handle = client->interface_handle; + flow->u.cb = client->interface_cb; + return; + } + } + + necp_client_add_interface_flow(client, interface_index); +} + +static bool +necp_client_flow_is_viable(proc_t proc, struct necp_client *client, + struct necp_client_flow *flow) +{ + struct necp_aggregate_result result; + bool ignore_address = (client->allow_multiple_flows && !flow->nexus && !flow->socket); + + flow->necp_flow_flags = 0; + int error = necp_application_find_policy_match_internal(proc, client->parameters, + (u_int32_t)client->parameters_length, + &result, &flow->necp_flow_flags, + flow->interface_index, + &flow->local_addr, &flow->remote_addr, NULL, ignore_address); + + return (error == 0 && + result.routed_interface_index != IFSCOPE_NONE && + result.routing_result != NECP_KERNEL_POLICY_RESULT_DROP); +} + +static bool +necp_client_update_flows(proc_t proc, struct necp_client *client) +{ + bool client_updated = FALSE; + struct necp_client_flow *flow = NULL; + struct necp_client_flow *temp_flow = NULL; + LIST_FOREACH_SAFE(flow, &client->flow_list, flow_chain, temp_flow) { + // Check policy result for flow + int old_flags = flow->necp_flow_flags; + bool viable = necp_client_flow_is_viable(proc, client, flow); + + // TODO: Defunct nexus flows that are blocked by policy + + if (flow->viable != viable) { + flow->viable = viable; + client_updated = TRUE; + } + + if ((old_flags & (NECP_CLIENT_RESULT_FLAG_HAS_IPV4 | NECP_CLIENT_RESULT_FLAG_HAS_IPV6)) != + (flow->necp_flow_flags & (NECP_CLIENT_RESULT_FLAG_HAS_IPV4 | NECP_CLIENT_RESULT_FLAG_HAS_IPV6))) { + client_updated = TRUE; + } + + if (flow->viable && client_updated && (flow->socket || (!flow->socket && !flow->nexus)) && flow->u.cb) { + flow->u.cb(flow->u.socket_handle, NECP_CLIENT_CBACTION_VIABLE, flow); + } + + if (!flow->viable || flow->invalid) { + if (client_updated && (flow->socket || (!flow->socket && !flow->nexus)) && flow->u.cb) { + flow->u.cb(flow->u.socket_handle, NECP_CLIENT_CBACTION_NONVIABLE, flow); + } + // The callback might change the viable-flag of the + // flow depending on its policy. Thus, we need to + // check again the flags after the callback. + } + + + // Handle flows that no longer match + if (!flow->viable || flow->invalid) { + // Drop them as long as they aren't assigned data + if (!flow->requested_nexus && !flow->assigned) { + if (flow->assigned_results != NULL) { + FREE(flow->assigned_results, M_NETAGENT); + flow->assigned_results = NULL; + client_updated = TRUE; + } + LIST_REMOVE(flow, flow_chain); + if (flow->socket) { + OSDecrementAtomic(&necp_socket_flow_count); + } else { + OSDecrementAtomic(&necp_if_flow_count); + } + zfree(necp_flow_zone, flow); + } + } + } + + return (client_updated); +} + +static void +necp_client_mark_all_nonsocket_flows_as_invalid(struct necp_client *client) +{ + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (!flow->socket) { // Socket flows are not marked as invalid + flow->invalid = TRUE; + } + } +} + +static bool +necp_netagent_applies_to_client(__unused struct necp_client *client, + const struct necp_client_parsed_parameters *parameters, + uuid_t netagent_uuid, bool allow_nexus, + uint32_t interface_index, u_int16_t interface_flags) +{ +#pragma unused(interface_index, interface_flags) + bool applies = FALSE; + u_int32_t flags = netagent_get_flags(netagent_uuid); + if (!(flags & NETAGENT_FLAG_REGISTERED)) { + // Unregistered agents never apply + return (applies); + } + + if (!allow_nexus && + (flags & NETAGENT_FLAG_NEXUS_PROVIDER)) { + // Hide nexus providers unless allowed + // Direct interfaces and direct policies are allowed to use a nexus + // Delegate interfaces or re-scoped interfaces are not allowed + return (applies); + } if (flags & NETAGENT_FLAG_SPECIFIC_USE_ONLY) { // Specific use agents only apply when required @@ -619,15 +1315,39 @@ necp_netagent_applies_to_client(__unused struct necp_client *client, struct necp applies = TRUE; } - if (applies && - (flags & NETAGENT_FLAG_NEXUS_PROVIDER) && - uuid_is_null(client->nexus_agent)) { - uuid_copy(client->nexus_agent, netagent_uuid); - } return (applies); } +static void +necp_client_add_agent_flows_for_interface(struct necp_client *client, + const struct necp_client_parsed_parameters *parsed_parameters, + ifnet_t ifp) +{ + if (ifp != NULL && ifp->if_agentids != NULL) { + for (u_int32_t i = 0; i < ifp->if_agentcount; i++) { + if (uuid_is_null(ifp->if_agentids[i])) { + continue; + } + u_int16_t if_flags = nstat_ifnet_to_flags(ifp); + // Relies on the side effect that nexus agents that apply will create flows + (void)necp_netagent_applies_to_client(client, parsed_parameters, ifp->if_agentids[i], TRUE, ifp->if_index, if_flags); + } + } +} + +static inline bool +necp_client_address_is_valid(struct sockaddr *address) +{ + if (address->sa_family == AF_INET) { + return (address->sa_len == sizeof(struct sockaddr_in)); + } else if (address->sa_family == AF_INET6) { + return (address->sa_len == sizeof(struct sockaddr_in6)); + } else { + return (FALSE); + } +} + static int necp_client_parse_parameters(u_int8_t *parameters, u_int32_t parameters_size, @@ -637,7 +1357,6 @@ necp_client_parse_parameters(u_int8_t *parameters, size_t offset = 0; u_int32_t num_prohibited_interfaces = 0; - u_int32_t num_required_interface_types = 0; u_int32_t num_prohibited_interface_types = 0; u_int32_t num_required_agents = 0; u_int32_t num_prohibited_agents = 0; @@ -652,11 +1371,11 @@ necp_client_parse_parameters(u_int8_t *parameters, memset(parsed_parameters, 0, sizeof(struct necp_client_parsed_parameters)); - while ((offset + sizeof(u_int8_t) + sizeof(u_int32_t)) <= parameters_size) { + while ((offset + sizeof(struct necp_tlv_header)) <= parameters_size) { u_int8_t type = necp_buffer_get_tlv_type(parameters, offset); u_int32_t length = necp_buffer_get_tlv_length(parameters, offset); - if (length > (parameters_size - (offset + sizeof(u_int8_t) + sizeof(u_int32_t)))) { + if (length > (parameters_size - (offset + sizeof(struct necp_tlv_header)))) { // If the length is larger than what can fit in the remaining parameters size, bail NECPLOG(LOG_ERR, "Invalid TLV length (%u)", length); break; @@ -683,11 +1402,15 @@ necp_client_parse_parameters(u_int8_t *parameters, case NECP_CLIENT_PARAMETER_LOCAL_ADDRESS: { if (length >= sizeof(struct necp_policy_condition_addr)) { struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; - if ((address_struct->address.sa.sa_family == AF_INET || - address_struct->address.sa.sa_family == AF_INET6) && - address_struct->address.sa.sa_len <= length) { + if (necp_client_address_is_valid(&address_struct->address.sa)) { memcpy(&parsed_parameters->local_addr, &address_struct->address, sizeof(address_struct->address)); - parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR; + if (!necp_address_is_wildcard(&parsed_parameters->local_addr)) { + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR; + } + if ((parsed_parameters->local_addr.sa.sa_family == AF_INET && parsed_parameters->local_addr.sin.sin_port) || + (parsed_parameters->local_addr.sa.sa_family == AF_INET6 && parsed_parameters->local_addr.sin6.sin6_port)) { + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_PORT; + } } } break; @@ -695,11 +1418,15 @@ necp_client_parse_parameters(u_int8_t *parameters, case NECP_CLIENT_PARAMETER_LOCAL_ENDPOINT: { if (length >= sizeof(struct necp_client_endpoint)) { struct necp_client_endpoint *endpoint = (struct necp_client_endpoint *)(void *)value; - if ((endpoint->u.endpoint.endpoint_family == AF_INET || - endpoint->u.endpoint.endpoint_family == AF_INET6) && - endpoint->u.endpoint.endpoint_length <= length) { + if (necp_client_address_is_valid(&endpoint->u.sa)) { memcpy(&parsed_parameters->local_addr, &endpoint->u.sa, sizeof(union necp_sockaddr_union)); - parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR; + if (!necp_address_is_wildcard(&parsed_parameters->local_addr)) { + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR; + } + if ((parsed_parameters->local_addr.sa.sa_family == AF_INET && parsed_parameters->local_addr.sin.sin_port) || + (parsed_parameters->local_addr.sa.sa_family == AF_INET6 && parsed_parameters->local_addr.sin6.sin6_port)) { + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_PORT; + } } } break; @@ -707,9 +1434,7 @@ necp_client_parse_parameters(u_int8_t *parameters, case NECP_CLIENT_PARAMETER_REMOTE_ADDRESS: { if (length >= sizeof(struct necp_policy_condition_addr)) { struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; - if ((address_struct->address.sa.sa_family == AF_INET || - address_struct->address.sa.sa_family == AF_INET6) && - address_struct->address.sa.sa_len <= length) { + if (necp_client_address_is_valid(&address_struct->address.sa)) { memcpy(&parsed_parameters->remote_addr, &address_struct->address, sizeof(address_struct->address)); parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_REMOTE_ADDR; } @@ -719,9 +1444,7 @@ necp_client_parse_parameters(u_int8_t *parameters, case NECP_CLIENT_PARAMETER_REMOTE_ENDPOINT: { if (length >= sizeof(struct necp_client_endpoint)) { struct necp_client_endpoint *endpoint = (struct necp_client_endpoint *)(void *)value; - if ((endpoint->u.endpoint.endpoint_family == AF_INET || - endpoint->u.endpoint.endpoint_family == AF_INET6) && - endpoint->u.endpoint.endpoint_length <= length) { + if (necp_client_address_is_valid(&endpoint->u.sa)) { memcpy(&parsed_parameters->remote_addr, &endpoint->u.sa, sizeof(union necp_sockaddr_union)); parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_REMOTE_ADDR; } @@ -741,13 +1464,14 @@ necp_client_parse_parameters(u_int8_t *parameters, break; } case NECP_CLIENT_PARAMETER_REQUIRE_IF_TYPE: { - if (num_required_interface_types >= NECP_MAX_PARSED_PARAMETERS) { + if (parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE) { break; } if (length >= sizeof(u_int8_t)) { - memcpy(&parsed_parameters->required_interface_types[num_required_interface_types], value, sizeof(u_int8_t)); - num_required_interface_types++; - parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE; + memcpy(&parsed_parameters->required_interface_type, value, sizeof(u_int8_t)); + if (parsed_parameters->required_interface_type) { + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE; + } } break; } @@ -828,6 +1552,89 @@ necp_client_parse_parameters(u_int8_t *parameters, } break; } + case NECP_CLIENT_PARAMETER_FLAGS: { + if (length >= sizeof(u_int32_t)) { + memcpy(&parsed_parameters->flags, value, sizeof(parsed_parameters->flags)); + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_FLAGS; + } + break; + } + case NECP_CLIENT_PARAMETER_IP_PROTOCOL: { + if (length >= sizeof(parsed_parameters->ip_protocol)) { + memcpy(&parsed_parameters->ip_protocol, value, sizeof(parsed_parameters->ip_protocol)); + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_IP_PROTOCOL; + } + break; + } + case NECP_CLIENT_PARAMETER_PID: { + if (length >= sizeof(parsed_parameters->effective_pid)) { + memcpy(&parsed_parameters->effective_pid, value, sizeof(parsed_parameters->effective_pid)); + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_EFFECTIVE_PID; + } + break; + } + case NECP_CLIENT_PARAMETER_APPLICATION: { + if (length >= sizeof(parsed_parameters->effective_uuid)) { + memcpy(&parsed_parameters->effective_uuid, value, sizeof(parsed_parameters->effective_uuid)); + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_EFFECTIVE_UUID; + } + break; + } + case NECP_CLIENT_PARAMETER_TRAFFIC_CLASS: { + if (length >= sizeof(parsed_parameters->traffic_class)) { + memcpy(&parsed_parameters->traffic_class, value, sizeof(parsed_parameters->traffic_class)); + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_TRAFFIC_CLASS; + } + break; + } + default: { + break; + } + } + } + } + + offset += sizeof(struct necp_tlv_header) + length; + } + + return (error); +} + +static int +necp_client_parse_result(u_int8_t *result, + u_int32_t result_size, + union necp_sockaddr_union *local_address, + union necp_sockaddr_union *remote_address) +{ + int error = 0; + size_t offset = 0; + + while ((offset + sizeof(struct necp_tlv_header)) <= result_size) { + u_int8_t type = necp_buffer_get_tlv_type(result, offset); + u_int32_t length = necp_buffer_get_tlv_length(result, offset); + + if (length > 0 && (offset + sizeof(struct necp_tlv_header) + length) <= result_size) { + u_int8_t *value = necp_buffer_get_tlv_value(result, offset, NULL); + if (value != NULL) { + switch (type) { + case NECP_CLIENT_RESULT_LOCAL_ENDPOINT: { + if (length >= sizeof(struct necp_client_endpoint)) { + struct necp_client_endpoint *endpoint = (struct necp_client_endpoint *)(void *)value; + if (local_address != NULL && necp_client_address_is_valid(&endpoint->u.sa)) { + memcpy(local_address, &endpoint->u.sa, endpoint->u.sa.sa_len); + } + } + break; + } + case NECP_CLIENT_RESULT_REMOTE_ENDPOINT: { + if (length >= sizeof(struct necp_client_endpoint)) { + struct necp_client_endpoint *endpoint = (struct necp_client_endpoint *)(void *)value; + if (remote_address != NULL && necp_client_address_is_valid(&endpoint->u.sa)) { + memcpy(remote_address, &endpoint->u.sa, endpoint->u.sa.sa_len); + } + } + break; + } default: { break; } @@ -835,56 +1642,462 @@ necp_client_parse_parameters(u_int8_t *parameters, } } - offset += sizeof(u_int8_t) + sizeof(u_int32_t) + length; + offset += sizeof(struct necp_tlv_header) + length; } return (error); } int -necp_assign_client_result(uuid_t netagent_uuid, uuid_t client_id, - u_int8_t *assigned_results, size_t assigned_results_length) +necp_client_register_socket_flow(pid_t pid, uuid_t client_id, struct inpcb *inp) +{ + int error = 0; + bool found_client = FALSE; + + NECP_CLIENT_TREE_LOCK_SHARED(); + + struct necp_client find; + uuid_copy(find.client_id, client_id); + struct necp_client *client = RB_FIND(_necp_client_global_tree, &necp_client_global_tree, &find); + if (client != NULL) { + NECP_CLIENT_LOCK(client); + + if (!pid || client->proc_pid == pid) { + // Found the right client! + found_client = TRUE; + necp_client_add_socket_flow(client, inp); + } + + NECP_CLIENT_UNLOCK(client); + } + + NECP_CLIENT_TREE_UNLOCK(); + + if (!found_client) { + error = ENOENT; + } else { + /* + * Count the sockets that have the NECP client UUID set + */ + struct socket *so = inp->inp_socket; + if (!(so->so_flags1 & SOF1_HAS_NECP_CLIENT_UUID)) { + so->so_flags1 |= SOF1_HAS_NECP_CLIENT_UUID; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_necp_clientuuid_total); + } + } + + return (error); +} + +static void +necp_client_add_multipath_cb(struct necp_client *client, struct mppcb *mpp) +{ + struct necp_client_flow *flow = NULL; + + client->interface_handle = mpp; + client->interface_cb = mpp->necp_cb; + + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (flow->nexus || flow->socket) { + continue; + } + + flow->u.socket_handle = mpp; + flow->u.cb = mpp->necp_cb; + + if (flow->viable && flow->u.cb) { + flow->u.cb(mpp, NECP_CLIENT_CBACTION_INITIAL, flow); + } + } +} + +int +necp_client_register_multipath_cb(pid_t pid, uuid_t client_id, struct mppcb *mpp) +{ + int error = 0; + bool found_client = FALSE; + + NECP_CLIENT_TREE_LOCK_SHARED(); + + struct necp_client find; + uuid_copy(find.client_id, client_id); + struct necp_client *client = RB_FIND(_necp_client_global_tree, &necp_client_global_tree, &find); + if (client != NULL) { + NECP_CLIENT_LOCK(client); + + if (!pid || client->proc_pid == pid) { + // Found the right client! + found_client = TRUE; + necp_client_add_multipath_cb(client, mpp); + } + + NECP_CLIENT_UNLOCK(client); + } + + NECP_CLIENT_TREE_UNLOCK(); + + if (!found_client) { + error = ENOENT; + } + + return (error); +} + +#define NETAGENT_DOMAIN_NETEXT "NetworkExtension" +#define NETAGENT_TYPE_PATHCTRL "PathController" + +static int +necp_client_unregister_socket_flow(uuid_t client_id, void *handle) { int error = 0; struct necp_fd_data *client_fd = NULL; bool found_client = FALSE; bool client_updated = FALSE; - lck_rw_lock_shared(&necp_fd_lock); + NECP_FD_LIST_LOCK_SHARED(); + LIST_FOREACH(client_fd, &necp_fd_list, chain) { + NECP_FD_LOCK(client_fd); + + struct necp_client *client = necp_client_fd_find_client_and_lock(client_fd, client_id); + if (client != NULL) { + // Found the right client! + found_client = TRUE; + + // Remove flow assignment + struct necp_client_flow *search_flow = NULL; + struct necp_client_flow *temp_flow = NULL; + LIST_FOREACH_SAFE(search_flow, &client->flow_list, flow_chain, temp_flow) { + if (search_flow->socket && search_flow->u.socket_handle == handle) { + if (search_flow->assigned_results != NULL) { + FREE(search_flow->assigned_results, M_NETAGENT); + search_flow->assigned_results = NULL; + } + client_updated = TRUE; + LIST_REMOVE(search_flow, flow_chain); + OSDecrementAtomic(&necp_socket_flow_count); + zfree(necp_flow_zone, search_flow); + } + } + + NECP_CLIENT_UNLOCK(client); + } + + if (client_updated) { + client->flow_result_read = FALSE; + necp_fd_notify(client_fd, true); + } + NECP_FD_UNLOCK(client_fd); + + if (found_client) { + break; + } + } + NECP_FD_LIST_UNLOCK(); + + if (!found_client) { + error = ENOENT; + } + + return (error); +} + +static int +necp_client_unregister_multipath_cb(uuid_t client_id, void *handle) +{ + int error = 0; + bool found_client = FALSE; + + NECP_CLIENT_TREE_LOCK_SHARED(); + + struct necp_client find; + uuid_copy(find.client_id, client_id); + struct necp_client *client = RB_FIND(_necp_client_global_tree, &necp_client_global_tree, &find); + if (client != NULL) { + NECP_CLIENT_LOCK(client); + + // Found the right client! + found_client = TRUE; + + // Remove flow assignment + struct necp_client_flow *search_flow = NULL; + struct necp_client_flow *temp_flow = NULL; + LIST_FOREACH_SAFE(search_flow, &client->flow_list, flow_chain, temp_flow) { + if (!search_flow->socket && !search_flow->nexus && + search_flow->u.socket_handle == handle) { + search_flow->u.socket_handle = NULL; + search_flow->u.cb = NULL; + } + } + client->interface_handle = NULL; + client->interface_cb = NULL; + + NECP_CLIENT_UNLOCK(client); + } + + NECP_CLIENT_TREE_UNLOCK(); + + if (!found_client) { + error = ENOENT; + } + + return (error); +} + +int +necp_client_assign_from_socket(pid_t pid, uuid_t client_id, struct inpcb *inp) +{ + int error = 0; + struct necp_fd_data *client_fd = NULL; + bool found_client = FALSE; + bool client_updated = FALSE; + + NECP_FD_LIST_LOCK_SHARED(); LIST_FOREACH(client_fd, &necp_fd_list, chain) { - struct necp_client *client = NULL; - lck_mtx_lock(&client_fd->fd_lock); - LIST_FOREACH(client, &client_fd->clients, chain) { - if (uuid_compare(client->client_id, client_id) == 0) { - // Found the right client! - found_client = TRUE; + if (pid && client_fd->proc_pid != pid) { + continue; + } - if (uuid_compare(client->nexus_agent, netagent_uuid) == 0) { - // Verify that the client nexus agent matches - if (client->assigned_results != NULL) { - // Release prior result - FREE(client->assigned_results, M_NETAGENT); + proc_t proc = proc_find(client_fd->proc_pid); + if (proc == PROC_NULL) { + continue; + } + + NECP_FD_LOCK(client_fd); + + struct necp_client *client = necp_client_fd_find_client_and_lock(client_fd, client_id); + if (client != NULL) { + // Found the right client! + found_client = TRUE; + + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (flow->socket && flow->u.socket_handle == inp) { + // Release prior results and route + if (flow->assigned_results != NULL) { + FREE(flow->assigned_results, M_NETAGENT); + flow->assigned_results = NULL; + } + + ifnet_t ifp = NULL; + if ((inp->inp_flags & INP_BOUND_IF) && inp->inp_boundifp) { + ifp = inp->inp_boundifp; + } else { + ifp = inp->inp_last_outifp; + } + + if (ifp != NULL) { + flow->interface_index = ifp->if_index; + } else { + flow->interface_index = IFSCOPE_NONE; } - client->assigned_results = assigned_results; - client->assigned_results_length = assigned_results_length; - client->assigned_result_read = FALSE; + + if (inp->inp_vflag & INP_IPV4) { + flow->local_addr.sin.sin_family = AF_INET; + flow->local_addr.sin.sin_len = sizeof(struct sockaddr_in); + flow->local_addr.sin.sin_port = inp->inp_lport; + memcpy(&flow->local_addr.sin.sin_addr, &inp->inp_laddr, sizeof(struct in_addr)); + + flow->remote_addr.sin.sin_family = AF_INET; + flow->remote_addr.sin.sin_len = sizeof(struct sockaddr_in); + flow->remote_addr.sin.sin_port = inp->inp_fport; + memcpy(&flow->remote_addr.sin.sin_addr, &inp->inp_faddr, sizeof(struct in_addr)); + } else if (inp->inp_vflag & INP_IPV6) { + in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, &flow->local_addr.sin6, sizeof(flow->local_addr)); + in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, &flow->remote_addr.sin6, sizeof(flow->remote_addr)); + } + + flow->viable = necp_client_flow_is_viable(proc, client, flow); + + uuid_t empty_uuid; + uuid_clear(empty_uuid); + flow->assigned = TRUE; + flow->assigned_results = necp_create_nexus_assign_message(empty_uuid, 0, NULL, 0, + (struct necp_client_endpoint *)&flow->local_addr, + (struct necp_client_endpoint *)&flow->remote_addr, + 0, &flow->assigned_results_length); + client->flow_result_read = FALSE; client_updated = TRUE; + break; } } + + NECP_CLIENT_UNLOCK(client); } if (client_updated) { necp_fd_notify(client_fd, true); } - lck_mtx_unlock(&client_fd->fd_lock); + NECP_FD_UNLOCK(client_fd); + + proc_rele(proc); + proc = PROC_NULL; if (found_client) { break; } } + NECP_FD_LIST_UNLOCK(); - lck_rw_done(&necp_fd_lock); + if (!found_client) { + error = ENOENT; + } else if (!client_updated) { + error = EINVAL; + } + + return (error); +} + +int +necp_update_flow_protoctl_event(uuid_t netagent_uuid, uuid_t client_id, + uint32_t protoctl_event_code, uint32_t protoctl_event_val, + uint32_t protoctl_event_tcp_seq_number) +{ + int error = 0; + struct necp_fd_data *client_fd = NULL; + bool found_client = FALSE; + bool client_updated = FALSE; + + NECP_FD_LIST_LOCK_SHARED(); + LIST_FOREACH(client_fd, &necp_fd_list, chain) { + proc_t proc = proc_find(client_fd->proc_pid); + if (proc == PROC_NULL) { + continue; + } + + NECP_FD_LOCK(client_fd); + + struct necp_client *client = necp_client_fd_find_client_and_lock(client_fd, client_id); + if (client != NULL) { + /* Found the right client! */ + found_client = TRUE; + + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + // Verify that the client nexus agent matches + if (flow->nexus && + uuid_compare(flow->u.nexus_agent, + netagent_uuid) == 0) { + flow->has_protoctl_event = TRUE; + flow->protoctl_event.protoctl_event_code = protoctl_event_code; + flow->protoctl_event.protoctl_event_val = protoctl_event_val; + flow->protoctl_event.protoctl_event_tcp_seq_num = protoctl_event_tcp_seq_number; + client->flow_result_read = FALSE; + client_updated = TRUE; + break; + } + } + + NECP_CLIENT_UNLOCK(client); + } + + if (client_updated) { + necp_fd_notify(client_fd, true); + } + + NECP_FD_UNLOCK(client_fd); + proc_rele(proc); + proc = PROC_NULL; + + if (found_client) { + break; + } + } + NECP_FD_LIST_UNLOCK(); + if (!found_client) { + error = ENOENT; + } else if (!client_updated) { + error = EINVAL; + } + return (error); +} + +static bool +necp_assign_client_result_locked(struct proc *proc, struct necp_fd_data *client_fd, struct necp_client *client, uuid_t netagent_uuid, u_int8_t *assigned_results, size_t assigned_results_length) +{ + bool client_updated = FALSE; + + NECP_FD_ASSERT_LOCKED(client_fd); + NECP_CLIENT_ASSERT_LOCKED(client); + + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + // Verify that the client nexus agent matches + if (flow->nexus && + uuid_compare(flow->u.nexus_agent, netagent_uuid) == 0) { + // Release prior results and route + if (flow->assigned_results != NULL) { + FREE(flow->assigned_results, M_NETAGENT); + flow->assigned_results = NULL; + } + + if (assigned_results != NULL && assigned_results_length > 0) { + int error = necp_client_parse_result(assigned_results, (u_int32_t)assigned_results_length, + &flow->local_addr, &flow->remote_addr); + VERIFY(error == 0); + } + + flow->viable = necp_client_flow_is_viable(proc, client, flow); + + flow->assigned = TRUE; + flow->assigned_results = assigned_results; + flow->assigned_results_length = assigned_results_length; + client->flow_result_read = FALSE; + client_updated = TRUE; + break; + } + } + + if (client_updated) { + necp_fd_notify(client_fd, true); + } + + // if not updated, client must free assigned_results + return (client_updated); +} + +int +necp_assign_client_result(uuid_t netagent_uuid, uuid_t client_id, + u_int8_t *assigned_results, size_t assigned_results_length) +{ + int error = 0; + struct necp_fd_data *client_fd = NULL; + bool found_client = FALSE; + bool client_updated = FALSE; + + NECP_FD_LIST_LOCK_SHARED(); + + LIST_FOREACH(client_fd, &necp_fd_list, chain) { + proc_t proc = proc_find(client_fd->proc_pid); + if (proc == PROC_NULL) { + continue; + } + + NECP_FD_LOCK(client_fd); + struct necp_client *client = necp_client_fd_find_client_and_lock(client_fd, client_id); + if (client != NULL) { + // Found the right client! + found_client = TRUE; + + if (necp_assign_client_result_locked(proc, client_fd, client, netagent_uuid, assigned_results, assigned_results_length)) { + client_updated = TRUE; + } + + NECP_CLIENT_UNLOCK(client); + } + NECP_FD_UNLOCK(client_fd); + + proc_rele(proc); + proc = PROC_NULL; + + if (found_client) { + break; + } + } + + NECP_FD_LIST_UNLOCK(); + + // upon error, client must free assigned_results if (!found_client) { error = ENOENT; } else if (!client_updated) { @@ -898,81 +2111,177 @@ necp_assign_client_result(uuid_t netagent_uuid, uuid_t client_id, static bool necp_update_client_result(proc_t proc, - struct necp_client *client) + struct necp_fd_data *client_fd, + struct necp_client *client, + struct _necp_client_defunct_list *defunct_list) { struct necp_client_result_netagent netagent; struct necp_aggregate_result result; - struct necp_client_parsed_parameters parsed_parameters; + struct necp_client_parsed_parameters *parsed_parameters = NULL; u_int32_t flags = 0; + struct rtentry *route = NULL; - uuid_clear(client->nexus_agent); + NECP_CLIENT_ASSERT_LOCKED(client); - int error = necp_client_parse_parameters(client->parameters, (u_int32_t)client->parameters_length, &parsed_parameters); + MALLOC(parsed_parameters, struct necp_client_parsed_parameters *, sizeof(*parsed_parameters), M_NECP, (M_WAITOK | M_ZERO)); + if (parsed_parameters == NULL) { + NECPLOG0(LOG_ERR, "Failed to allocate parsed parameters"); + return (FALSE); + } + + // Nexus flows will be brought back if they are still valid + necp_client_mark_all_nonsocket_flows_as_invalid(client); + + int error = necp_client_parse_parameters(client->parameters, (u_int32_t)client->parameters_length, parsed_parameters); if (error != 0) { + FREE(parsed_parameters, M_NECP); return (FALSE); } + // Update saved IP protocol + client->ip_protocol = parsed_parameters->ip_protocol; + // Check parameters to find best interface u_int matching_if_index = 0; - if (necp_find_matching_interface_index(&parsed_parameters, &matching_if_index)) { + if (necp_find_matching_interface_index(parsed_parameters, &matching_if_index)) { if (matching_if_index != 0) { - parsed_parameters.required_interface_index = matching_if_index; + parsed_parameters->required_interface_index = matching_if_index; } // Interface found or not needed, match policy. - error = necp_application_find_policy_match_internal(proc, client->parameters, (u_int32_t)client->parameters_length, &result, &flags, matching_if_index); + error = necp_application_find_policy_match_internal(proc, client->parameters, + (u_int32_t)client->parameters_length, + &result, &flags, matching_if_index, + NULL, NULL, &route, false); if (error != 0) { + if (route != NULL) { + rtfree(route); + } + FREE(parsed_parameters, M_NECP); return (FALSE); } + + // Reset current route + NECP_CLIENT_ROUTE_LOCK(client); + if (client->current_route != NULL) { + rtfree(client->current_route); + } + client->current_route = route; + NECP_CLIENT_ROUTE_UNLOCK(client); } else { // Interface not found. Clear out the whole result, make everything fail. memset(&result, 0, sizeof(result)); } + // Save the last policy id on the client + client->policy_id = result.policy_id; + + if ((parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_MULTIPATH) || + ((parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) && + result.routing_result != NECP_KERNEL_POLICY_RESULT_SOCKET_SCOPED)) { + client->allow_multiple_flows = TRUE; + } else { + client->allow_multiple_flows = FALSE; + } + // If the original request was scoped, and the policy result matches, make sure the result is scoped if ((result.routing_result == NECP_KERNEL_POLICY_RESULT_NONE || result.routing_result == NECP_KERNEL_POLICY_RESULT_PASS) && result.routed_interface_index != IFSCOPE_NONE && - parsed_parameters.required_interface_index == result.routed_interface_index) { + parsed_parameters->required_interface_index == result.routed_interface_index) { result.routing_result = NECP_KERNEL_POLICY_RESULT_SOCKET_SCOPED; result.routing_result_parameter.scoped_interface_index = result.routed_interface_index; } + if (defunct_list != NULL && + result.routing_result == NECP_KERNEL_POLICY_RESULT_DROP) { + // If we are forced to drop the client, defunct it if it has flows + necp_defunct_client_for_policy(client, defunct_list); + } + + // Recalculate flags + if (client->defunct) { + flags |= NECP_CLIENT_RESULT_FLAG_DEFUNCT; + } + if (parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) { + // Listeners are valid as long as they aren't dropped + if (result.routing_result != NECP_KERNEL_POLICY_RESULT_DROP) { + flags |= NECP_CLIENT_RESULT_FLAG_SATISFIED; + } + } else if (result.routed_interface_index != 0) { + // Clients without flows determine viability based on having some routable interface + flags |= NECP_CLIENT_RESULT_FLAG_SATISFIED; + } + bool updated = FALSE; u_int8_t *cursor = client->result; - const u_int8_t *max = client->result + NECP_MAX_CLIENT_RESULT_SIZE; - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_CLIENT_ID, sizeof(uuid_t), client->client_id, &updated); - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_POLICY_RESULT, sizeof(result.routing_result), &result.routing_result, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_FLAGS, sizeof(flags), &flags, &updated, client->result, sizeof(client->result)); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_CLIENT_ID, sizeof(uuid_t), client->client_id, &updated, + client->result, sizeof(client->result)); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_POLICY_RESULT, sizeof(result.routing_result), &result.routing_result, &updated, + client->result, sizeof(client->result)); if (result.routing_result_parameter.tunnel_interface_index != 0) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_POLICY_RESULT_PARAMETER, - sizeof(result.routing_result_parameter), &result.routing_result_parameter, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_POLICY_RESULT_PARAMETER, + sizeof(result.routing_result_parameter), &result.routing_result_parameter, &updated, + client->result, sizeof(client->result)); } if (result.filter_control_unit != 0) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_FILTER_CONTROL_UNIT, - sizeof(result.filter_control_unit), &result.filter_control_unit, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_FILTER_CONTROL_UNIT, + sizeof(result.filter_control_unit), &result.filter_control_unit, &updated, + client->result, sizeof(client->result)); } if (result.routed_interface_index != 0) { u_int routed_interface_index = result.routed_interface_index; if (result.routing_result == NECP_KERNEL_POLICY_RESULT_IP_TUNNEL && - parsed_parameters.required_interface_index != IFSCOPE_NONE && - parsed_parameters.required_interface_index != result.routed_interface_index) { - routed_interface_index = parsed_parameters.required_interface_index; + parsed_parameters->required_interface_index != IFSCOPE_NONE && + parsed_parameters->required_interface_index != result.routed_interface_index) { + routed_interface_index = parsed_parameters->required_interface_index; } - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_INTERFACE_INDEX, - sizeof(routed_interface_index), &routed_interface_index, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_INTERFACE_INDEX, + sizeof(routed_interface_index), &routed_interface_index, &updated, + client->result, sizeof(client->result)); + } + if (client_fd && client_fd->flags & NECP_OPEN_FLAG_BACKGROUND) { + u_int32_t effective_traffic_class = SO_TC_BK_SYS; + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_EFFECTIVE_TRAFFIC_CLASS, + sizeof(effective_traffic_class), &effective_traffic_class, &updated, + client->result, sizeof(client->result)); + } + if (client->background_update) { + u_int32_t background = client->background; + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_TRAFFIC_MGMT_BG, + sizeof(background), &background, &updated, + client->result, sizeof(client->result)); + if (updated) { + client->background_update = 0; + } } - if (flags != 0) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_FLAGS, - sizeof(flags), &flags, &updated); + NECP_CLIENT_ROUTE_LOCK(client); + if (client->current_route != NULL) { + const u_int32_t route_mtu = get_maxmtu(client->current_route); + if (route_mtu != 0) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_EFFECTIVE_MTU, + sizeof(route_mtu), &route_mtu, &updated, + client->result, sizeof(client->result)); + } } + NECP_CLIENT_ROUTE_UNLOCK(client); + + if (result.mss_recommended != 0) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_RECOMMENDED_MSS, + sizeof(result.mss_recommended), &result.mss_recommended, &updated, + client->result, sizeof(client->result)); + } + for (int i = 0; i < NECP_MAX_NETAGENTS; i++) { if (uuid_is_null(result.netagents[i])) { break; } uuid_copy(netagent.netagent_uuid, result.netagents[i]); netagent.generation = netagent_get_generation(netagent.netagent_uuid); - if (necp_netagent_applies_to_client(client, &parsed_parameters, netagent.netagent_uuid)) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated); + if (necp_netagent_applies_to_client(client, parsed_parameters, netagent.netagent_uuid, TRUE, 0, 0)) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated, + client->result, sizeof(client->result)); } } @@ -983,10 +2292,10 @@ necp_update_client_result(proc_t proc, if (result.routed_interface_index != IFSCOPE_NONE && result.routed_interface_index <= (u_int32_t)if_index) { direct_interface = ifindex2ifnet[result.routed_interface_index]; - } else if (parsed_parameters.required_interface_index != IFSCOPE_NONE && - parsed_parameters.required_interface_index <= (u_int32_t)if_index) { + } else if (parsed_parameters->required_interface_index != IFSCOPE_NONE && + parsed_parameters->required_interface_index <= (u_int32_t)if_index) { // If the request was scoped, but the route didn't match, still grab the agents - direct_interface = ifindex2ifnet[parsed_parameters.required_interface_index]; + direct_interface = ifindex2ifnet[parsed_parameters->required_interface_index]; } else if (result.routed_interface_index == IFSCOPE_NONE && result.routing_result == NECP_KERNEL_POLICY_RESULT_SOCKET_SCOPED && result.routing_result_parameter.scoped_interface_index != IFSCOPE_NONE) { @@ -996,30 +2305,59 @@ necp_update_client_result(proc_t proc, delegate_interface = direct_interface->if_delegated.ifp; } if (result.routing_result == NECP_KERNEL_POLICY_RESULT_IP_TUNNEL && - parsed_parameters.required_interface_index != IFSCOPE_NONE && - parsed_parameters.required_interface_index != result.routing_result_parameter.tunnel_interface_index && - parsed_parameters.required_interface_index <= (u_int32_t)if_index) { - original_scoped_interface = ifindex2ifnet[parsed_parameters.required_interface_index]; + parsed_parameters->required_interface_index != IFSCOPE_NONE && + parsed_parameters->required_interface_index != result.routing_result_parameter.tunnel_interface_index && + parsed_parameters->required_interface_index <= (u_int32_t)if_index) { + original_scoped_interface = ifindex2ifnet[parsed_parameters->required_interface_index]; } // Add interfaces if (original_scoped_interface != NULL) { struct necp_client_result_interface interface_struct; interface_struct.index = original_scoped_interface->if_index; interface_struct.generation = ifnet_get_generation(original_scoped_interface); - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated, + client->result, sizeof(client->result)); } if (direct_interface != NULL) { struct necp_client_result_interface interface_struct; interface_struct.index = direct_interface->if_index; interface_struct.generation = ifnet_get_generation(direct_interface); - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated, + client->result, sizeof(client->result)); } if (delegate_interface != NULL) { struct necp_client_result_interface interface_struct; interface_struct.index = delegate_interface->if_index; interface_struct.generation = ifnet_get_generation(delegate_interface); - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated, + client->result, sizeof(client->result)); + } + + // Update multipath/listener interface flows + if (parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_MULTIPATH) { + // Get multipath interface options from ordered list + struct ifnet *multi_interface = NULL; + TAILQ_FOREACH(multi_interface, &ifnet_ordered_head, if_ordered_link) { + if (necp_ifnet_matches_parameters(multi_interface, parsed_parameters, NULL, true)) { + // Add multipath interface flows for kernel MPTCP + necp_client_add_interface_flow_if_needed(client, multi_interface->if_index); + + // Add nexus agents for multipath + necp_client_add_agent_flows_for_interface(client, parsed_parameters, multi_interface); + } + } + } else if ((parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) && + result.routing_result != NECP_KERNEL_POLICY_RESULT_SOCKET_SCOPED) { + // Get listener interface options from global list + struct ifnet *listen_interface = NULL; + TAILQ_FOREACH(listen_interface, &ifnet_head, if_link) { + if (necp_ifnet_matches_parameters(listen_interface, parsed_parameters, NULL, false)) { + // Add nexus agents for listeners + necp_client_add_agent_flows_for_interface(client, parsed_parameters, listen_interface); + } + } } + // Add agents if (original_scoped_interface != NULL) { ifnet_lock_shared(original_scoped_interface); @@ -1028,10 +2366,12 @@ necp_update_client_result(proc_t proc, if (uuid_is_null(original_scoped_interface->if_agentids[i])) { continue; } + u_int16_t if_flags = nstat_ifnet_to_flags(original_scoped_interface); uuid_copy(netagent.netagent_uuid, original_scoped_interface->if_agentids[i]); netagent.generation = netagent_get_generation(netagent.netagent_uuid); - if (necp_netagent_applies_to_client(client, &parsed_parameters, netagent.netagent_uuid)) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated); + if (necp_netagent_applies_to_client(client, parsed_parameters, netagent.netagent_uuid, FALSE, original_scoped_interface->if_index, if_flags)) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated, + client->result, sizeof(client->result)); } } } @@ -1044,10 +2384,12 @@ necp_update_client_result(proc_t proc, if (uuid_is_null(direct_interface->if_agentids[i])) { continue; } + u_int16_t if_flags = nstat_ifnet_to_flags(direct_interface); uuid_copy(netagent.netagent_uuid, direct_interface->if_agentids[i]); netagent.generation = netagent_get_generation(netagent.netagent_uuid); - if (necp_netagent_applies_to_client(client, &parsed_parameters, netagent.netagent_uuid)) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated); + if (necp_netagent_applies_to_client(client, parsed_parameters, netagent.netagent_uuid, TRUE, direct_interface->if_index, if_flags)) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated, + client->result, sizeof(client->result)); } } } @@ -1060,10 +2402,12 @@ necp_update_client_result(proc_t proc, if (uuid_is_null(delegate_interface->if_agentids[i])) { continue; } + u_int16_t if_flags = nstat_ifnet_to_flags(delegate_interface); uuid_copy(netagent.netagent_uuid, delegate_interface->if_agentids[i]); netagent.generation = netagent_get_generation(netagent.netagent_uuid); - if (necp_netagent_applies_to_client(client, &parsed_parameters, netagent.netagent_uuid)) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated); + if (necp_netagent_applies_to_client(client, parsed_parameters, netagent.netagent_uuid, FALSE, delegate_interface->if_index, if_flags)) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated, + client->result, sizeof(client->result)); } } } @@ -1076,62 +2420,262 @@ necp_update_client_result(proc_t proc, client->result_length = new_result_length; updated = TRUE; } + + // Update flow viability/flags + if (necp_client_update_flows(proc, client)) { + updated = TRUE; + } + if (updated) { client->result_read = FALSE; + necp_client_update_observer_update(client); } + FREE(parsed_parameters, M_NECP); return (updated); } +static inline void +necp_defunct_client_fd_locked(struct necp_fd_data *client_fd, struct _necp_client_defunct_list *defunct_list) +{ + bool updated_result = FALSE; + struct necp_client *client = NULL; + + NECP_FD_ASSERT_LOCKED(client_fd); + RB_FOREACH(client, _necp_client_tree, &client_fd->clients) { + NECP_CLIENT_LOCK(client); + if (!client->defunct) { + updated_result = necp_set_client_defunct(client); + + // Prepare close events to be sent to the nexus to effectively remove the flows + struct necp_client_flow *search_flow = NULL; + LIST_FOREACH(search_flow, &client->flow_list, flow_chain) { + if (search_flow->nexus && + !uuid_is_null(search_flow->u.nexus_agent) && + search_flow->requested_nexus) { + + struct necp_client_defunct *client_defunct; + + // Sleeping alloc won't fail; copy only what's necessary + client_defunct = _MALLOC(sizeof (struct necp_client_defunct), M_NECP, M_WAITOK | M_ZERO); + uuid_copy(client_defunct->nexus_agent, search_flow->u.nexus_agent); + uuid_copy(client_defunct->client_id, client->client_id); + client_defunct->proc_pid = client->proc_pid; + + // Add to the list provided by caller + LIST_INSERT_HEAD(defunct_list, client_defunct, chain); + } + } + } + NECP_CLIENT_UNLOCK(client); + } + if (updated_result) { + necp_fd_notify(client_fd, true); + } +} + +static inline void +necp_update_client_fd_locked(struct necp_fd_data *client_fd, + proc_t proc, + struct _necp_client_defunct_list *defunct_list) +{ + struct necp_client *client = NULL; + bool updated_result = FALSE; + NECP_FD_ASSERT_LOCKED(client_fd); + RB_FOREACH(client, _necp_client_tree, &client_fd->clients) { + NECP_CLIENT_LOCK(client); + if (necp_update_client_result(proc, client_fd, client, defunct_list)) { + updated_result = TRUE; + } + NECP_CLIENT_UNLOCK(client); + } + if (updated_result) { + necp_fd_notify(client_fd, true); + } +} + + static void necp_update_all_clients_callout(__unused thread_call_param_t dummy, __unused thread_call_param_t arg) { -#pragma unused(arg) struct necp_fd_data *client_fd = NULL; - lck_rw_lock_shared(&necp_fd_lock); + struct _necp_client_defunct_list defunct_list; + LIST_INIT(&defunct_list); + + NECP_FD_LIST_LOCK_SHARED(); LIST_FOREACH(client_fd, &necp_fd_list, chain) { - bool updated_result = FALSE; - struct necp_client *client = NULL; proc_t proc = proc_find(client_fd->proc_pid); - if (proc == NULL) { + if (proc == PROC_NULL) { continue; } - lck_mtx_lock(&client_fd->fd_lock); - LIST_FOREACH(client, &client_fd->clients, chain) { - if (necp_update_client_result(proc, client)) { - updated_result = TRUE; + // Update all clients on one fd + NECP_FD_LOCK(client_fd); + necp_update_client_fd_locked(client_fd, proc, &defunct_list); + NECP_FD_UNLOCK(client_fd); + + proc_rele(proc); + proc = PROC_NULL; + } + + NECP_FD_LIST_UNLOCK(); + + // Handle the case in which some clients became newly defunct + if (!LIST_EMPTY(&defunct_list)) { + struct necp_client_defunct *client_defunct = NULL; + struct necp_client_defunct *temp_client_defunct = NULL; + + // For each newly defunct client, send a message to the nexus to remove the flow + LIST_FOREACH_SAFE(client_defunct, &defunct_list, chain, temp_client_defunct) { + if (!uuid_is_null(client_defunct->nexus_agent)) { + int netagent_error = netagent_client_message(client_defunct->nexus_agent, + client_defunct->client_id, + client_defunct->proc_pid, + NETAGENT_MESSAGE_TYPE_ABORT_NEXUS); + if (netagent_error != 0) { + NECPLOG((netagent_error == ENOENT ? LOG_DEBUG : LOG_ERR), "necp_update_client abort nexus error (%d)", netagent_error); + } } + LIST_REMOVE(client_defunct, chain); + FREE(client_defunct, M_NECP); } - if (updated_result) { - necp_fd_notify(client_fd, true); + } + ASSERT(LIST_EMPTY(&defunct_list)); +} + +void +necp_update_all_clients(void) +{ + if (necp_client_update_tcall == NULL) { + // Don't try to update clients if the module is not initialized + return; + } + + uint64_t deadline = 0; + uint64_t leeway = 0; + clock_interval_to_deadline(necp_timeout_microseconds, NSEC_PER_USEC, &deadline); + clock_interval_to_absolutetime_interval(necp_timeout_leeway_microseconds, NSEC_PER_USEC, &leeway); + + thread_call_enter_delayed_with_leeway(necp_client_update_tcall, NULL, + deadline, leeway, THREAD_CALL_DELAY_LEEWAY); +} + +void +necp_set_client_as_background(proc_t proc, + struct fileproc *fp, + bool background) +{ + bool updated_result = FALSE; + struct necp_client *client = NULL; + + if (proc == PROC_NULL) { + NECPLOG0(LOG_ERR, "NULL proc"); + return; + } + + if (fp == NULL) { + NECPLOG0(LOG_ERR, "NULL fp"); + return; + } + + struct necp_fd_data *client_fd = (struct necp_fd_data *)fp->f_fglob->fg_data; + if (client_fd == NULL) { + NECPLOG0(LOG_ERR, "Could not find client structure for backgrounded client"); + return; + } + + if (client_fd->necp_fd_type != necp_fd_type_client) { + // Not a client fd, ignore + NECPLOG0(LOG_ERR, "Not a client fd, ignore"); + return; + } + + NECP_FD_LOCK(client_fd); + + RB_FOREACH(client, _necp_client_tree, &client_fd->clients) { + NECP_CLIENT_LOCK(client); + + bool has_assigned_flow = FALSE; + struct necp_client_flow *search_flow = NULL; + LIST_FOREACH(search_flow, &client->flow_list, flow_chain) { + if (search_flow->assigned) { + has_assigned_flow = TRUE; + break; + } } - lck_mtx_unlock(&client_fd->fd_lock); - proc_rele(proc); + if (has_assigned_flow) { + client->background = background; + client->background_update = TRUE; + updated_result = TRUE; + } + + NECP_CLIENT_UNLOCK(client); + } + if (updated_result) { + necp_update_client_fd_locked(client_fd, proc, NULL); + } + NECP_FD_UNLOCK(client_fd); +} + +void +necp_defunct_client(proc_t proc, + struct fileproc *fp) +{ + struct _necp_client_defunct_list defunct_list; + + if (proc == PROC_NULL) { + NECPLOG0(LOG_ERR, "NULL proc passed to set as background"); + return; + } + + if (fp == NULL) { + NECPLOG0(LOG_ERR, "NULL fp passed to set as background"); + return; } - lck_rw_done(&necp_fd_lock); -} + struct necp_fd_data *client_fd = (struct necp_fd_data *)fp->f_fglob->fg_data; + if (client_fd == NULL) { + NECPLOG0(LOG_ERR, "Could not find client structure for backgrounded client"); + return; + } -void -necp_update_all_clients(void) -{ - if (necp_client_tcall == NULL) { - // Don't try to update clients if the module is not initialized + if (client_fd->necp_fd_type != necp_fd_type_client) { + // Not a client fd, ignore return; } - uint64_t deadline = 0; - uint64_t leeway = 0; - clock_interval_to_deadline(necp_timeout_microseconds, NSEC_PER_USEC, &deadline); - clock_interval_to_absolutetime_interval(necp_timeout_leeway_microseconds, NSEC_PER_USEC, &leeway); + // Our local temporary list + LIST_INIT(&defunct_list); - thread_call_enter_delayed_with_leeway(necp_client_tcall, NULL, - deadline, leeway, THREAD_CALL_DELAY_LEEWAY); + // Need to hold lock so ntstats defunct the same set of clients + NECP_FD_LOCK(client_fd); + necp_defunct_client_fd_locked(client_fd, &defunct_list); + NECP_FD_UNLOCK(client_fd); + + if (!LIST_EMPTY(&defunct_list)) { + struct necp_client_defunct *client_defunct = NULL; + struct necp_client_defunct *temp_client_defunct = NULL; + + // For each defunct client, remove flow from the nexus + LIST_FOREACH_SAFE(client_defunct, &defunct_list, chain, temp_client_defunct) { + if (!uuid_is_null(client_defunct->nexus_agent)) { + int netagent_error = netagent_client_message(client_defunct->nexus_agent, + client_defunct->client_id, + client_defunct->proc_pid, + NETAGENT_MESSAGE_TYPE_ABORT_NEXUS); + if (netagent_error != 0) { + NECPLOG((netagent_error == ENOENT ? LOG_DEBUG : LOG_ERR), "necp_defunct_client abort nexus error (%d)", netagent_error); + } + } + LIST_REMOVE(client_defunct, chain); + FREE(client_defunct, M_NECP); + } + } + ASSERT(LIST_EMPTY(&defunct_list)); } static void @@ -1140,11 +2684,11 @@ necp_client_remove_agent_from_result(struct necp_client *client, uuid_t netagent size_t offset = 0; u_int8_t *result_buffer = client->result; - while ((offset + sizeof(u_int8_t) + sizeof(u_int32_t)) <= client->result_length) { + while ((offset + sizeof(struct necp_tlv_header)) <= client->result_length) { u_int8_t type = necp_buffer_get_tlv_type(result_buffer, offset); u_int32_t length = necp_buffer_get_tlv_length(result_buffer, offset); - size_t tlv_total_length = (sizeof(u_int8_t) + sizeof(u_int32_t) + length); + size_t tlv_total_length = (sizeof(struct necp_tlv_header) + length); if (type == NECP_CLIENT_RESULT_NETAGENT && length == sizeof(struct necp_client_result_netagent) && (offset + tlv_total_length) <= client->result_length) { @@ -1158,7 +2702,7 @@ necp_client_remove_agent_from_result(struct necp_client *client, uuid_t netagent result_buffer + offset + tlv_total_length, client->result_length - (offset + tlv_total_length)); client->result_length -= tlv_total_length; - memset(result_buffer + client->result_length, 0, NECP_MAX_CLIENT_RESULT_SIZE - client->result_length); + memset(result_buffer + client->result_length, 0, sizeof(client->result) - client->result_length); continue; } } @@ -1172,36 +2716,35 @@ necp_force_update_client(uuid_t client_id, uuid_t remove_netagent_uuid) { struct necp_fd_data *client_fd = NULL; - lck_rw_lock_shared(&necp_fd_lock); + NECP_FD_LIST_LOCK_SHARED(); LIST_FOREACH(client_fd, &necp_fd_list, chain) { bool updated_result = FALSE; - struct necp_client *client = NULL; - lck_mtx_lock(&client_fd->fd_lock); - LIST_FOREACH(client, &client_fd->clients, chain) { - if (uuid_compare(client->client_id, client_id) == 0) { - if (!uuid_is_null(remove_netagent_uuid)) { - necp_client_remove_agent_from_result(client, remove_netagent_uuid); - } - client->assigned_result_read = FALSE; - updated_result = TRUE; - // Found the client, break - break; + NECP_FD_LOCK(client_fd); + struct necp_client *client = necp_client_fd_find_client_and_lock(client_fd, client_id); + if (client != NULL) { + if (!uuid_is_null(remove_netagent_uuid)) { + necp_client_remove_agent_from_result(client, remove_netagent_uuid); } + client->flow_result_read = FALSE; + // Found the client, break + updated_result = TRUE; + NECP_CLIENT_UNLOCK(client); } if (updated_result) { necp_fd_notify(client_fd, true); } - lck_mtx_unlock(&client_fd->fd_lock); + NECP_FD_UNLOCK(client_fd); if (updated_result) { // Found the client, break break; } } - lck_rw_done(&necp_fd_lock); + NECP_FD_LIST_UNLOCK(); } + /// Interface matching #define NECP_PARSED_PARAMETERS_INTERESTING_IFNET_FIELDS (NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR | \ @@ -1346,10 +2889,28 @@ necp_ifnet_matches_local_address(struct ifnet *ifp, struct sockaddr *sa) return (matched_local_address); } +static bool +necp_interface_type_is_primary_eligible(u_int8_t interface_type) +{ + switch (interface_type) { + // These types can never be primary, so a client requesting these types is allowed + // to match an interface that isn't currently eligible to be primary (has default + // route, dns, etc) + case IFRTYPE_FUNCTIONAL_WIFI_AWDL: + case IFRTYPE_FUNCTIONAL_INTCOPROC: + return false; + default: + break; + } + return true; +} + +#define NECP_IFP_IS_ON_ORDERED_LIST(_ifp) ((_ifp)->if_ordered_link.tqe_next != NULL || (_ifp)->if_ordered_link.tqe_prev != NULL) + static bool necp_ifnet_matches_parameters(struct ifnet *ifp, struct necp_client_parsed_parameters *parsed_parameters, - u_int32_t *preferred_count) + u_int32_t *preferred_count, bool ignore_require_if) { if (preferred_count) { *preferred_count = 0; @@ -1361,18 +2922,19 @@ necp_ifnet_matches_parameters(struct ifnet *ifp, } } - if (parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE) { - for (int i = 0; i < NECP_MAX_PARSED_PARAMETERS; i++) { - if (parsed_parameters->required_interface_types[i] == 0) { - break; - } - - if (!necp_ifnet_matches_type(ifp, parsed_parameters->required_interface_types[i], FALSE)) { - return (FALSE); - } + if (parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_FLAGS) { + if ((parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_PROHIBIT_EXPENSIVE) && + IFNET_IS_EXPENSIVE(ifp)) { + return (FALSE); } } + if (!ignore_require_if && + (parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE) && + !necp_ifnet_matches_type(ifp, parsed_parameters->required_interface_type, FALSE)) { + return (FALSE); + } + if (parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_IFTYPE) { for (int i = 0; i < NECP_MAX_PARSED_PARAMETERS; i++) { if (parsed_parameters->prohibited_interface_types[i] == 0) { @@ -1502,9 +3064,9 @@ necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_ if (!(parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_SCOPED_IFNET_FIELDS)) { // We do have fields to match, but they are only prohibitory - // If the first interface in the list matches, we don't need to scope + // If the first interface in the list matches, or there are no ordered interfaces, we don't need to scope ifp = TAILQ_FIRST(&ifnet_ordered_head); - if (ifp && necp_ifnet_matches_parameters(ifp, parsed_parameters, NULL)) { + if (ifp == NULL || necp_ifnet_matches_parameters(ifp, parsed_parameters, NULL, false)) { // Don't set return_ifindex, so the client doesn't need to scope ifnet_head_done(); return (TRUE); @@ -1514,7 +3076,7 @@ necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_ // First check the ordered interface list TAILQ_FOREACH(ifp, &ifnet_ordered_head, if_ordered_link) { u_int32_t preferred_count = 0; - if (necp_ifnet_matches_parameters(ifp, parsed_parameters, &preferred_count)) { + if (necp_ifnet_matches_parameters(ifp, parsed_parameters, &preferred_count, false)) { if (preferred_count > best_preferred_count || *return_ifindex == 0) { @@ -1531,16 +3093,16 @@ necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_ // Then check the remaining interfaces if ((parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_SCOPED_IFNET_FIELDS) && - !(parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE) && + ((!(parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE)) || + !necp_interface_type_is_primary_eligible(parsed_parameters->required_interface_type)) && *return_ifindex == 0) { TAILQ_FOREACH(ifp, &ifnet_head, if_link) { u_int32_t preferred_count = 0; - if (ifp->if_ordered_link.tqe_next != NULL || - ifp->if_ordered_link.tqe_prev != NULL) { + if (NECP_IFP_IS_ON_ORDERED_LIST(ifp)) { // This interface was in the ordered list, skip continue; } - if (necp_ifnet_matches_parameters(ifp, parsed_parameters, &preferred_count)) { + if (necp_ifnet_matches_parameters(ifp, parsed_parameters, &preferred_count, false)) { if (preferred_count > best_preferred_count || *return_ifindex == 0) { @@ -1568,186 +3130,6 @@ necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_ return (*return_ifindex != 0); } -static void -necp_find_netstat_data(struct necp_client *client, union necp_sockaddr_union *local, union necp_sockaddr_union *remote, u_int32_t *ifindex, uuid_t euuid, u_int32_t *traffic_class) -{ - size_t offset = 0; - u_int8_t *parameters; - u_int32_t parameters_size; - - parameters = client->parameters; - parameters_size = (u_int32_t)client->parameters_length; - - while ((offset + sizeof(u_int8_t) + sizeof(u_int32_t)) <= parameters_size) { - u_int8_t type = necp_buffer_get_tlv_type(parameters, offset); - u_int32_t length = necp_buffer_get_tlv_length(parameters, offset); - - if (length > (parameters_size - (offset + sizeof(u_int8_t) + sizeof(u_int32_t)))) { - // If the length is larger than what can fit in the remaining parameters size, bail - NECPLOG(LOG_ERR, "Invalid TLV length (%u)", length); - break; - } - - if (length > 0) { - u_int8_t *value = necp_buffer_get_tlv_value(parameters, offset, NULL); - if (value != NULL) { - switch (type) { - case NECP_CLIENT_PARAMETER_REAL_APPLICATION: { - if (length >= sizeof(uuid_t)) { - uuid_copy(euuid, value); - } - break; - } - case NECP_CLIENT_PARAMETER_TRAFFIC_CLASS: { - if (length >= sizeof(u_int32_t)) { - memcpy(traffic_class, value, sizeof(u_int32_t)); - } - break; - } - case NECP_CLIENT_PARAMETER_BOUND_INTERFACE: { - if (length <= IFXNAMSIZ && length > 0) { - ifnet_t bound_interface = NULL; - char interface_name[IFXNAMSIZ]; - memcpy(interface_name, value, length); - interface_name[length - 1] = 0; // Make sure the string is NULL terminated - if (ifnet_find_by_name(interface_name, &bound_interface) == 0) { - *ifindex = bound_interface->if_index; - ifnet_release(bound_interface); - } - } - break; - } - case NECP_CLIENT_PARAMETER_LOCAL_ADDRESS: { - if (length >= sizeof(struct necp_policy_condition_addr)) { - struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; - memcpy(local, &address_struct->address, sizeof(address_struct->address)); - } - break; - } - case NECP_CLIENT_PARAMETER_REMOTE_ADDRESS: { - if (length >= sizeof(struct necp_policy_condition_addr)) { - struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; - memcpy(remote, &address_struct->address, sizeof(address_struct->address)); - } - break; - } - default: { - break; - } - } - } - } - offset += sizeof(u_int8_t) + sizeof(u_int32_t) + length; - } -} - -static void -necp_fillout_current_process_details(u_int32_t *pid, u_int64_t *upid, unsigned char *uuid, char *pname, size_t len) -{ - *pid = proc_selfpid(); - *upid = proc_uniqueid(current_proc()); - proc_selfname(pname, (int) len); - proc_getexecutableuuid(current_proc(), uuid, sizeof(uuid_t)); -} - -// Called from NetworkStatistics when it wishes to collect latest information for a TCP flow. -// It is a responsibility of NetworkStatistics to have previously zeroed any supplied memory. -static bool -necp_request_tcp_netstats(userland_stats_provider_context *ctx, - nstat_counts *countsp, - void *metadatap) -{ - if (ctx == NULL) { - return false; - } - - struct necp_client *client = (struct necp_client *)ctx; - struct necp_tcp_stats *tcpstats = (struct necp_tcp_stats *)client->stats_area; - if (tcpstats == NULL) { - return false; - } - - if (countsp) { - *countsp = *((struct nstat_counts *)&tcpstats->necp_tcp_counts); - } - - if (metadatap) { - nstat_tcp_descriptor *desc = (nstat_tcp_descriptor *)metadatap; - - // Metadata for the process - necp_fillout_current_process_details(&desc->pid, &desc->upid, desc->uuid, desc->pname, sizeof(desc->pname)); - - // Metadata that the necp client should have in TLV format. - necp_find_netstat_data(client, (union necp_sockaddr_union *)&desc->local, (union necp_sockaddr_union *)&desc->remote, &desc->ifindex, desc->euuid, &desc->traffic_class); - - // Basic metadata - desc->rcvbufsize = tcpstats->necp_tcp_basic.rcvbufsize; - desc->rcvbufused = tcpstats->necp_tcp_basic.rcvbufused; - desc->eupid = tcpstats->necp_tcp_basic.eupid; - desc->epid = tcpstats->necp_tcp_basic.epid; - memcpy(desc->vuuid, tcpstats->necp_tcp_basic.vuuid, sizeof(desc->vuuid)); - desc->ifnet_properties = tcpstats->necp_tcp_basic.ifnet_properties; - - // Additional TCP specific data - desc->sndbufsize = tcpstats->necp_tcp_extra.sndbufsize; - desc->sndbufused = tcpstats->necp_tcp_extra.sndbufused; - desc->txunacked = tcpstats->necp_tcp_extra.txunacked; - desc->txwindow = tcpstats->necp_tcp_extra.txwindow; - desc->txcwindow = tcpstats->necp_tcp_extra.txcwindow; - desc->traffic_mgt_flags = tcpstats->necp_tcp_extra.traffic_mgt_flags; - - if (tcpstats->necp_tcp_extra.cc_alg_index < TCP_CC_ALGO_COUNT) { - strlcpy(desc->cc_algo, tcp_cc_algo_list[tcpstats->necp_tcp_extra.cc_alg_index]->name, sizeof(desc->cc_algo)); - } else { - strlcpy(desc->cc_algo, "unknown", sizeof(desc->cc_algo)); - } - - desc->connstatus.write_probe_failed = tcpstats->necp_tcp_extra.probestatus.write_probe_failed; - desc->connstatus.read_probe_failed = tcpstats->necp_tcp_extra.probestatus.read_probe_failed; - desc->connstatus.conn_probe_failed = tcpstats->necp_tcp_extra.probestatus.conn_probe_failed; - } - return true; -} - -// Called from NetworkStatistics when it wishes to collect latest information for a UDP flow. -static bool -necp_request_udp_netstats(userland_stats_provider_context *ctx, - nstat_counts *countsp, - void *metadatap) -{ - if (ctx == NULL) { - return false; - } - - struct necp_client *client = (struct necp_client *)ctx; - struct necp_udp_stats *udpstats = (struct necp_udp_stats *)client->stats_area; - if (udpstats == NULL) { - return false; - } - - if (countsp) { - *countsp = *((struct nstat_counts *)&udpstats->necp_udp_counts); - } - - if (metadatap) { - nstat_udp_descriptor *desc = (nstat_udp_descriptor *)metadatap; - - // Metadata for the process - necp_fillout_current_process_details(&desc->pid, &desc->upid, desc->uuid, desc->pname, sizeof(desc->pname)); - - // Metadata that the necp client should have in TLV format. - necp_find_netstat_data(client, (union necp_sockaddr_union *)&desc->local, (union necp_sockaddr_union *)&desc->remote, &desc->ifindex, desc->euuid, &desc->traffic_class); - - // Basic metadata is all that is required for UDP - desc->rcvbufsize = udpstats->necp_udp_basic.rcvbufsize; - desc->rcvbufused = udpstats->necp_udp_basic.rcvbufused; - desc->eupid = udpstats->necp_udp_basic.eupid; - desc->epid = udpstats->necp_udp_basic.epid; - memcpy(desc->vuuid, udpstats->necp_udp_basic.vuuid, sizeof(desc->euuid)); - desc->ifnet_properties = udpstats->necp_udp_basic.ifnet_properties; - } - return true; -} static int necp_skywalk_priv_check_cred(proc_t p, kauth_cred_t cred) @@ -1769,7 +3151,7 @@ necp_open(struct proc *p, struct necp_open_args *uap, int *retval) if (uap->flags & NECP_OPEN_FLAG_OBSERVER) { if (necp_skywalk_priv_check_cred(p, kauth_cred_get()) != 0 && - priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) != 0) { + priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) != 0) { NECPLOG0(LOG_ERR, "Client does not hold necessary entitlement to observe other NECP clients"); error = EACCES; goto done; @@ -1781,14 +3163,17 @@ necp_open(struct proc *p, struct necp_open_args *uap, int *retval) goto done; } - if ((fd_data = _MALLOC(sizeof(struct necp_fd_data), M_NECP, - M_WAITOK | M_ZERO)) == NULL) { + if ((fd_data = zalloc(necp_client_fd_zone)) == NULL) { error = ENOMEM; goto done; } + memset(fd_data, 0, sizeof(*fd_data)); + + fd_data->necp_fd_type = necp_fd_type_client; fd_data->flags = uap->flags; - LIST_INIT(&fd_data->clients); + RB_INIT(&fd_data->clients); + TAILQ_INIT(&fd_data->update_list); lck_mtx_init(&fd_data->fd_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); klist_init(&fd_data->si.si_note); fd_data->proc_pid = proc_pid(p); @@ -1805,9 +3190,28 @@ necp_open(struct proc *p, struct necp_open_args *uap, int *retval) *retval = fd; - lck_rw_lock_exclusive(&necp_fd_lock); - LIST_INSERT_HEAD(&necp_fd_list, fd_data, chain); - lck_rw_done(&necp_fd_lock); + if (fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER) { + NECP_OBSERVER_LIST_LOCK_EXCLUSIVE(); + LIST_INSERT_HEAD(&necp_fd_observer_list, fd_data, chain); + OSIncrementAtomic(&necp_observer_fd_count); + NECP_OBSERVER_LIST_UNLOCK(); + + // Walk all existing clients and add them + NECP_CLIENT_TREE_LOCK_SHARED(); + struct necp_client *existing_client = NULL; + RB_FOREACH(existing_client, _necp_client_global_tree, &necp_client_global_tree) { + NECP_CLIENT_LOCK(existing_client); + necp_client_update_observer_add_internal(fd_data, existing_client); + necp_client_update_observer_update_internal(fd_data, existing_client); + NECP_CLIENT_UNLOCK(existing_client); + } + NECP_CLIENT_TREE_UNLOCK(); + } else { + NECP_FD_LIST_LOCK_EXCLUSIVE(); + LIST_INSERT_HEAD(&necp_fd_list, fd_data, chain); + OSIncrementAtomic(&necp_client_fd_count); + NECP_FD_LIST_UNLOCK(); + } proc_fdunlock(p); @@ -1818,7 +3222,7 @@ done: fp = NULL; } if (fd_data != NULL) { - FREE(fd_data, M_NECP); + zfree(necp_client_fd_zone, fd_data); fd_data = NULL; } } @@ -1827,15 +3231,19 @@ done: } static int -necp_client_add(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) +necp_client_add(struct proc *p, struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { int error = 0; struct necp_client *client = NULL; + if (fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER) { + NECPLOG0(LOG_ERR, "NECP client observers with push enabled may not add their own clients"); + return (EINVAL); + } + if (uap->client_id == 0 || uap->client_id_len != sizeof(uuid_t) || uap->buffer_size == 0 || uap->buffer_size > NECP_MAX_CLIENT_PARAMETERS_SIZE || uap->buffer == 0) { - error = EINVAL; - goto done; + return (EINVAL); } if ((client = _MALLOC(sizeof(struct necp_client) + uap->buffer_size, M_NECP, @@ -1850,10 +3258,17 @@ necp_client_add(struct necp_fd_data *fd_data, struct necp_client_action_args *ua goto done; } + lck_mtx_init(&client->lock, necp_fd_mtx_grp, necp_fd_mtx_attr); + lck_mtx_init(&client->route_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); + necp_client_retain(client); // Hold our reference until close + client->parameters_length = uap->buffer_size; + client->proc_pid = fd_data->proc_pid; // Save off proc pid in case the client will persist past fd + client->platform_binary = ((csproc_get_platform_binary(p) == 0) ? 0 : 1); uuid_generate_random(client->client_id); LIST_INIT(&client->assertion_list); + LIST_INIT(&client->flow_list); error = copyout(client->client_id, uap->client_id, sizeof(uuid_t)); if (error) { @@ -1861,12 +3276,20 @@ necp_client_add(struct necp_fd_data *fd_data, struct necp_client_action_args *ua goto done; } - lck_mtx_lock(&fd_data->fd_lock); - LIST_INSERT_HEAD(&fd_data->clients, client, chain); + necp_client_update_observer_add(client); + + NECP_FD_LOCK(fd_data); + RB_INSERT(_necp_client_tree, &fd_data->clients, client); + OSIncrementAtomic(&necp_client_count); + NECP_CLIENT_TREE_LOCK_EXCLUSIVE(); + RB_INSERT(_necp_client_global_tree, &necp_client_global_tree, client); + NECP_CLIENT_TREE_UNLOCK(); // Prime the client result - (void)necp_update_client_result(current_proc(), client); - lck_mtx_unlock(&fd_data->fd_lock); + NECP_CLIENT_LOCK(client); + (void)necp_update_client_result(current_proc(), fd_data, client, NULL); + NECP_CLIENT_UNLOCK(client); + NECP_FD_UNLOCK(fd_data); done: if (error != 0) { if (client != NULL) { @@ -1885,7 +3308,8 @@ necp_client_remove(struct necp_fd_data *fd_data, struct necp_client_action_args int error = 0; struct necp_client *client = NULL; struct necp_client *temp_client = NULL; - uuid_t client_id; + uuid_t client_id = {}; + struct ifnet_stats_per_flow flow_ifnet_stats = {}; if (uap->client_id == 0 || uap->client_id_len != sizeof(uuid_t)) { error = EINVAL; @@ -1898,19 +3322,128 @@ necp_client_remove(struct necp_fd_data *fd_data, struct necp_client_action_args goto done; } - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH_SAFE(client, &fd_data->clients, chain, temp_client) { + if (uap->buffer != 0 && uap->buffer_size == sizeof(flow_ifnet_stats)) { + error = copyin(uap->buffer, &flow_ifnet_stats, uap->buffer_size); + if (error) { + NECPLOG(LOG_ERR, "necp_client_remove flow_ifnet_stats copyin error (%d)", error); + // Not fatal + } + } else if (uap->buffer != 0) { + NECPLOG(LOG_ERR, "necp_client_remove unexpected parameters length (%zu)", uap->buffer_size); + } + + struct _necp_client_tree clients_to_close; + RB_INIT(&clients_to_close); + NECP_FD_LOCK(fd_data); + pid_t pid = fd_data->proc_pid; + RB_FOREACH_SAFE(client, _necp_client_tree, &fd_data->clients, temp_client) { if (uuid_compare(client->client_id, client_id) == 0) { - necp_destroy_client(client); + NECP_CLIENT_TREE_LOCK_EXCLUSIVE(); + RB_REMOVE(_necp_client_global_tree, &necp_client_global_tree, client); + NECP_CLIENT_TREE_UNLOCK(); + RB_REMOVE(_necp_client_tree, &fd_data->clients, client); + RB_INSERT(_necp_client_tree, &clients_to_close, client); } } - lck_mtx_unlock(&fd_data->fd_lock); + + + NECP_FD_UNLOCK(fd_data); + + RB_FOREACH_SAFE(client, _necp_client_tree, &clients_to_close, temp_client) { + RB_REMOVE(_necp_client_tree, &clients_to_close, client); + necp_destroy_client(client, pid, true); + } done: *retval = error; return (error); } +static int +necp_client_check_tcp_heuristics(struct necp_client *client, struct necp_client_flow *flow, u_int32_t *flags, u_int8_t *tfo_cookie, u_int8_t *tfo_cookie_len) +{ + struct necp_client_parsed_parameters parsed_parameters; + int error = 0; + + error = necp_client_parse_parameters(client->parameters, + (u_int32_t)client->parameters_length, + &parsed_parameters); + if (error) { + NECPLOG(LOG_ERR, "necp_client_parse_parameters error (%d)", error); + return (error); + } + + if ((flow->remote_addr.sa.sa_family != AF_INET && + flow->remote_addr.sa.sa_family != AF_INET6) || + (flow->local_addr.sa.sa_family != AF_INET && + flow->local_addr.sa.sa_family != AF_INET6)) { + return (EINVAL); + } + + NECP_CLIENT_ROUTE_LOCK(client); + + if (client->current_route == NULL) { + error = ENOENT; + goto do_unlock; + } + + bool check_ecn = false; + do { + if ((parsed_parameters.flags & NECP_CLIENT_PARAMETER_FLAG_ECN_ENABLE) == + NECP_CLIENT_PARAMETER_FLAG_ECN_ENABLE) { + check_ecn = true; + break; + } + + if ((parsed_parameters.flags & NECP_CLIENT_PARAMETER_FLAG_ECN_DISABLE) == + NECP_CLIENT_PARAMETER_FLAG_ECN_DISABLE) { + break; + } + + if (client->current_route != NULL) { + if (client->current_route->rt_ifp->if_eflags & IFEF_ECN_ENABLE) { + check_ecn = true; + break; + } + if (client->current_route->rt_ifp->if_eflags & IFEF_ECN_DISABLE) { + break; + } + } + + bool inbound = ((parsed_parameters.flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) == 0); + if ((inbound && tcp_ecn_inbound == 1) || + (!inbound && tcp_ecn_outbound == 1)) { + check_ecn = true; + } + } while (false); + + if (check_ecn) { + if (tcp_heuristic_do_ecn_with_address(client->current_route->rt_ifp, + (union sockaddr_in_4_6 *)&flow->local_addr)) { + *flags |= NECP_CLIENT_RESULT_FLAG_ECN_ENABLED; + } + } + + if ((parsed_parameters.flags & NECP_CLIENT_PARAMETER_FLAG_TFO_ENABLE) == + NECP_CLIENT_PARAMETER_FLAG_TFO_ENABLE) { + + if (!tcp_heuristic_do_tfo_with_address(client->current_route->rt_ifp, + (union sockaddr_in_4_6 *)&flow->local_addr, + (union sockaddr_in_4_6 *)&flow->remote_addr, + tfo_cookie, tfo_cookie_len)) { + *flags |= NECP_CLIENT_RESULT_FLAG_FAST_OPEN_BLOCKED; + *tfo_cookie_len = 0; + } + } else { + *flags |= NECP_CLIENT_RESULT_FLAG_FAST_OPEN_BLOCKED; + *tfo_cookie_len = 0; + } +do_unlock: + NECP_CLIENT_ROUTE_UNLOCK(client); + + return (error); +} + static int necp_client_copy_internal(struct necp_client *client, bool client_is_observed, struct necp_client_action_args *uap, int *retval) { @@ -1927,8 +3460,31 @@ necp_client_copy_internal(struct necp_client *client, bool client_is_observed, s goto done; } *retval = client->parameters_length; - } else if (uap->action == NECP_CLIENT_ACTION_COPY_RESULT) { - if (uap->buffer_size < (client->result_length + client->assigned_results_length)) { + } else if (uap->action == NECP_CLIENT_ACTION_COPY_UPDATED_RESULT && + client->result_read && client->flow_result_read) { + // Copy updates only, but nothing to read + // Just return 0 for bytes read + *retval = 0; + } else if (uap->action == NECP_CLIENT_ACTION_COPY_RESULT || + uap->action == NECP_CLIENT_ACTION_COPY_UPDATED_RESULT) { + size_t assigned_results_size = 0; + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (flow->nexus || (flow->socket && flow->assigned)) { + size_t header_length = 0; + if (flow->nexus) { + header_length = sizeof(struct necp_client_nexus_flow_header); + } else { + header_length = sizeof(struct necp_client_flow_header); + } + assigned_results_size += (header_length + flow->assigned_results_length); + + if (flow->has_protoctl_event) { + assigned_results_size += sizeof(struct necp_client_flow_protoctl_event_header); + } + } + } + if (uap->buffer_size < (client->result_length + assigned_results_size)) { error = EINVAL; goto done; } @@ -1937,20 +3493,148 @@ necp_client_copy_internal(struct necp_client *client, bool client_is_observed, s NECPLOG(LOG_ERR, "necp_client_copy result copyout error (%d)", error); goto done; } - if (client->assigned_results_length && client->assigned_results) { - error = copyout(client->assigned_results, uap->buffer + client->result_length, client->assigned_results_length); - if (error) { - NECPLOG(LOG_ERR, "necp_client_copy assigned results copyout error (%d)", error); - goto done; + + size_t assigned_results_cursor = 0; + + flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (flow->nexus || (flow->socket && flow->assigned)) { + // Write TLV headers + struct necp_client_nexus_flow_header header; + u_int32_t length = 0; + u_int32_t flags = 0; + u_int8_t tfo_cookie_len = 0; + u_int8_t type = 0; + + if (flow->nexus) { + if (flow->check_tcp_heuristics) { + u_int8_t tfo_cookie[NECP_TFO_COOKIE_LEN_MAX]; + tfo_cookie_len = NECP_TFO_COOKIE_LEN_MAX; + + if (necp_client_check_tcp_heuristics(client, flow, &flags, + tfo_cookie, &tfo_cookie_len) != 0) { + tfo_cookie_len = 0; + } else { + flow->check_tcp_heuristics = FALSE; + + if (tfo_cookie_len != 0) { + type = NECP_CLIENT_RESULT_TFO_COOKIE; + length = tfo_cookie_len; + memcpy(&header.tfo_cookie_tlv_header.type, &type, sizeof(type)); + memcpy(&header.tfo_cookie_tlv_header.length, &length, sizeof(length)); + memcpy(&header.tfo_cookie_value, tfo_cookie, tfo_cookie_len); + } + } + } + } + + size_t header_length = 0; + if (flow->nexus) { + if (tfo_cookie_len != 0) { + header_length = sizeof(struct necp_client_nexus_flow_header) - (NECP_TFO_COOKIE_LEN_MAX - tfo_cookie_len); + } else { + header_length = sizeof(struct necp_client_nexus_flow_header) - sizeof(struct necp_tlv_header) - NECP_TFO_COOKIE_LEN_MAX; + } + } else { + header_length = sizeof(struct necp_client_flow_header); + } + + type = NECP_CLIENT_RESULT_FLAGS; + length = sizeof(header.flow_header.flags_value); + memcpy(&header.flow_header.flags_tlv_header.type, &type, sizeof(type)); + memcpy(&header.flow_header.flags_tlv_header.length, &length, sizeof(length)); + if (flow->assigned) { + flags |= NECP_CLIENT_RESULT_FLAG_FLOW_ASSIGNED; + } + if (flow->viable) { + flags |= NECP_CLIENT_RESULT_FLAG_FLOW_VIABLE; + } + memcpy(&header.flow_header.flags_value, &flags, sizeof(flags)); + + type = NECP_CLIENT_RESULT_INTERFACE; + length = sizeof(header.flow_header.interface_value); + memcpy(&header.flow_header.interface_tlv_header.type, &type, sizeof(type)); + memcpy(&header.flow_header.interface_tlv_header.length, &length, sizeof(length)); + + struct necp_client_result_interface interface_struct; + interface_struct.generation = 0; + interface_struct.index = flow->interface_index; + + memcpy(&header.flow_header.interface_value, &interface_struct, sizeof(interface_struct)); + if (flow->nexus) { + type = NECP_CLIENT_RESULT_NETAGENT; + length = sizeof(header.agent_value); + memcpy(&header.agent_tlv_header.type, &type, sizeof(type)); + memcpy(&header.agent_tlv_header.length, &length, sizeof(length)); + + struct necp_client_result_netagent agent_struct; + agent_struct.generation = 0; + uuid_copy(agent_struct.netagent_uuid, flow->u.nexus_agent); + + memcpy(&header.agent_value, &agent_struct, sizeof(agent_struct)); + } + + // Don't include outer TLV header in length field + type = NECP_CLIENT_RESULT_FLOW; + length = (header_length - sizeof(struct necp_tlv_header) + flow->assigned_results_length); + if (flow->has_protoctl_event) { + length += sizeof(struct necp_client_flow_protoctl_event_header); + } + memcpy(&header.flow_header.outer_header.type, &type, sizeof(type)); + memcpy(&header.flow_header.outer_header.length, &length, sizeof(length)); + + error = copyout(&header, uap->buffer + client->result_length + assigned_results_cursor, header_length); + if (error) { + NECPLOG(LOG_ERR, "necp_client_copy assigned results tlv_header copyout error (%d)", error); + goto done; + } + assigned_results_cursor += header_length; + + if (flow->assigned_results && flow->assigned_results_length) { + // Write inner TLVs + error = copyout(flow->assigned_results, uap->buffer + client->result_length + assigned_results_cursor, + flow->assigned_results_length); + if (error) { + NECPLOG(LOG_ERR, "necp_client_copy assigned results copyout error (%d)", error); + goto done; + } + } + assigned_results_cursor += flow->assigned_results_length; + + /* Read the protocol event and reset it */ + if (flow->has_protoctl_event) { + struct necp_client_flow_protoctl_event_header protoctl_event_header; + + type = NECP_CLIENT_RESULT_PROTO_CTL_EVENT; + length = sizeof(protoctl_event_header.protoctl_event); + + memcpy(&protoctl_event_header.protoctl_tlv_header.type, &type, sizeof(type)); + memcpy(&protoctl_event_header.protoctl_tlv_header.length, &length, sizeof(length)); + memcpy(&protoctl_event_header.protoctl_event, &flow->protoctl_event, + sizeof(flow->protoctl_event)); + + error = copyout(&protoctl_event_header, uap->buffer + client->result_length + assigned_results_cursor, + sizeof(protoctl_event_header)); + + if (error) { + NECPLOG(LOG_ERR, "necp_client_copy protocol control event results" + " tlv_header copyout error (%d)", error); + goto done; + } + assigned_results_cursor += sizeof(protoctl_event_header); + flow->has_protoctl_event = FALSE; + flow->protoctl_event.protoctl_event_code = 0; + flow->protoctl_event.protoctl_event_val = 0; + flow->protoctl_event.protoctl_event_tcp_seq_num = 0; + } } - *retval = client->result_length + client->assigned_results_length; - } else { - *retval = client->result_length; } + *retval = client->result_length + assigned_results_cursor; + if (!client_is_observed) { client->result_read = TRUE; - client->assigned_result_read = TRUE; + client->flow_result_read = TRUE; } } @@ -1975,7 +3659,8 @@ necp_client_copy(struct necp_fd_data *fd_data, struct necp_client_action_args *u } if (uap->action != NECP_CLIENT_ACTION_COPY_PARAMETERS && - uap->action != NECP_CLIENT_ACTION_COPY_RESULT) { + uap->action != NECP_CLIENT_ACTION_COPY_RESULT && + uap->action != NECP_CLIENT_ACTION_COPY_UPDATED_RESULT) { error = EINVAL; goto done; } @@ -1994,16 +3679,19 @@ necp_client_copy(struct necp_fd_data *fd_data, struct necp_client_action_args *u } } - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH(find_client, &fd_data->clients, chain) { - if (uap->action == NECP_CLIENT_ACTION_COPY_RESULT && + NECP_FD_LOCK(fd_data); + RB_FOREACH(find_client, _necp_client_tree, &fd_data->clients) { + NECP_CLIENT_LOCK(find_client); + if ((uap->action == NECP_CLIENT_ACTION_COPY_RESULT || uap->action == NECP_CLIENT_ACTION_COPY_UPDATED_RESULT) && uuid_is_null(client_id)) { - if (!find_client->result_read || !find_client->assigned_result_read) { + if (!find_client->result_read || !find_client->flow_result_read) { client = find_client; - break; } } else if (uuid_compare(find_client->client_id, client_id) == 0) { client = find_client; + } + NECP_CLIENT_UNLOCK(find_client); + if (client != NULL) { break; } } @@ -2013,44 +3701,35 @@ necp_client_copy(struct necp_fd_data *fd_data, struct necp_client_action_args *u } // Unlock our own client before moving on or returning - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); if (client == NULL) { if (fd_data->flags & NECP_OPEN_FLAG_OBSERVER) { // Observers are allowed to lookup clients on other fds - // Lock list - lck_rw_lock_shared(&necp_fd_lock); - struct necp_fd_data *client_fd = NULL; - LIST_FOREACH(client_fd, &necp_fd_list, chain) { - // Lock client - lck_mtx_lock(&client_fd->fd_lock); - find_client = NULL; - LIST_FOREACH(find_client, &client_fd->clients, chain) { - if (uuid_compare(find_client->client_id, client_id) == 0) { - client = find_client; - break; - } - } + // Lock tree + NECP_CLIENT_TREE_LOCK_SHARED(); - if (client != NULL) { - // Matched, copy out data - error = necp_client_copy_internal(client, TRUE, uap, retval); - } + bool found_client = FALSE; - // Unlock client - lck_mtx_unlock(&client_fd->fd_lock); + struct necp_client find; + uuid_copy(find.client_id, client_id); + client = RB_FIND(_necp_client_global_tree, &necp_client_global_tree, &find); + if (client != NULL) { + NECP_CLIENT_LOCK(client); - if (client != NULL) { - break; - } + // Matched, copy out data + found_client = TRUE; + error = necp_client_copy_internal(client, TRUE, uap, retval); + + NECP_CLIENT_UNLOCK(client); } - // Unlock list - lck_rw_done(&necp_fd_lock); + // Unlock tree + NECP_CLIENT_TREE_UNLOCK(); // No client found, fail - if (client == NULL) { + if (!found_client) { error = ENOENT; goto done; } @@ -2065,6 +3744,137 @@ done: return (error); } +static int +necp_client_copy_client_update(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) +{ + int error = 0; + + *retval = 0; + + if (!(fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER)) { + NECPLOG0(LOG_ERR, "NECP fd is not observer, cannot copy client update"); + return (EINVAL); + } + + if (uap->client_id_len != sizeof(uuid_t) || uap->client_id == 0) { + NECPLOG0(LOG_ERR, "Client id invalid, cannot copy client update"); + return (EINVAL); + } + + if (uap->buffer_size == 0 || uap->buffer == 0) { + NECPLOG0(LOG_ERR, "Buffer invalid, cannot copy client update"); + return (EINVAL); + } + + NECP_FD_LOCK(fd_data); + struct necp_client_update *client_update = TAILQ_FIRST(&fd_data->update_list); + if (client_update != NULL) { + TAILQ_REMOVE(&fd_data->update_list, client_update, chain); + VERIFY(fd_data->update_count > 0); + fd_data->update_count--; + } + NECP_FD_UNLOCK(fd_data); + + if (client_update != NULL) { + error = copyout(client_update->client_id, uap->client_id, sizeof(uuid_t)); + if (error) { + NECPLOG(LOG_ERR, "Copy client update copyout client id error (%d)", error); + } else { + if (uap->buffer_size < client_update->update_length) { + NECPLOG(LOG_ERR, "Buffer size cannot hold update (%zu < %zu)", uap->buffer_size, client_update->update_length); + error = EINVAL; + } else { + error = copyout(&client_update->update, uap->buffer, client_update->update_length); + if (error) { + NECPLOG(LOG_ERR, "Copy client update copyout error (%d)", error); + } else { + *retval = client_update->update_length; + } + } + } + + FREE(client_update, M_NECP); + client_update = NULL; + } else { + error = ENOENT; + } + + return (error); +} + +static int +necp_client_copy_parameters_locked(struct necp_client *client, struct necp_client_nexus_parameters *parameters) +{ + VERIFY(parameters != NULL); + + struct necp_client_parsed_parameters parsed_parameters = {}; + int error = necp_client_parse_parameters(client->parameters, (u_int32_t)client->parameters_length, &parsed_parameters); + + parameters->pid = client->proc_pid; + if (parsed_parameters.valid_fields & NECP_PARSED_PARAMETERS_FIELD_EFFECTIVE_PID) { + parameters->epid = parsed_parameters.effective_pid; + } else { + parameters->epid = parameters->pid; + } + memcpy(¶meters->local_addr, &parsed_parameters.local_addr, sizeof(parameters->local_addr)); + memcpy(¶meters->remote_addr, &parsed_parameters.remote_addr, sizeof(parameters->remote_addr)); + parameters->ip_protocol = parsed_parameters.ip_protocol; + parameters->traffic_class = parsed_parameters.traffic_class; + uuid_copy(parameters->euuid, parsed_parameters.effective_uuid); + parameters->is_listener = (parsed_parameters.flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) ? 1 : 0; + parameters->policy_id = client->policy_id; + + // parse client result flag + u_int32_t client_result_flags = 0; + u_int32_t value_size = 0; + u_int8_t *flags_pointer = NULL; + flags_pointer = necp_buffer_get_tlv_value(client->result, 0, &value_size); + if (flags_pointer && value_size == sizeof(client_result_flags)) { + memcpy(&client_result_flags, flags_pointer, value_size); + } + parameters->allow_qos_marking = (client_result_flags & NECP_CLIENT_RESULT_FLAG_ALLOW_QOS_MARKING) ? 1 : 0; + + return (error); +} + +int +necp_client_copy_parameters(uuid_t client_id, struct necp_client_nexus_parameters *parameters) +{ + int error = 0; + struct necp_client *client = NULL; + + if (parameters == NULL) { + return EINVAL; + } + + // Lock tree + NECP_CLIENT_TREE_LOCK_SHARED(); + + bool found_client = FALSE; + struct necp_client find; + uuid_copy(find.client_id, client_id); + client = RB_FIND(_necp_client_global_tree, &necp_client_global_tree, &find); + if (client != NULL) { + NECP_CLIENT_LOCK(client); + + // Matched, parse parameters + found_client = TRUE; + error = necp_client_copy_parameters_locked(client, parameters); + + NECP_CLIENT_UNLOCK(client); + } + + // Unlock tree + NECP_CLIENT_TREE_UNLOCK(); + + // No client found, fail + if (!found_client) { + return ENOENT; + } + + return error; +} + static int necp_client_list(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { @@ -2073,6 +3883,7 @@ necp_client_list(struct necp_fd_data *fd_data, struct necp_client_action_args *u uuid_t *list = NULL; u_int32_t requested_client_count = 0; u_int32_t client_count = 0; + size_t copy_buffer_size = 0; if (uap->buffer_size < sizeof(requested_client_count) || uap->buffer == 0) { error = EINVAL; @@ -2090,38 +3901,45 @@ necp_client_list(struct necp_fd_data *fd_data, struct necp_client_action_args *u goto done; } - if (uap->buffer_size != (sizeof(requested_client_count) + requested_client_count * sizeof(uuid_t))) { + if (os_mul_overflow(sizeof(uuid_t), requested_client_count, ©_buffer_size)) { + error = ERANGE; + goto done; + } + + if (uap->buffer_size - sizeof(requested_client_count) != copy_buffer_size) { + error = EINVAL; + goto done; + } + + if (copy_buffer_size > NECP_MAX_CLIENT_LIST_SIZE) { error = EINVAL; goto done; } if (requested_client_count > 0) { - if ((list = _MALLOC(requested_client_count * sizeof(uuid_t), M_NECP, M_WAITOK | M_ZERO)) == NULL) { + if ((list = _MALLOC(copy_buffer_size, M_NECP, M_WAITOK | M_ZERO)) == NULL) { error = ENOMEM; goto done; } } - // Lock list - lck_rw_lock_shared(&necp_fd_lock); - struct necp_fd_data *client_fd = NULL; - LIST_FOREACH(client_fd, &necp_fd_list, chain) { - // Lock client - lck_mtx_lock(&client_fd->fd_lock); - find_client = NULL; - LIST_FOREACH(find_client, &client_fd->clients, chain) { - if (!uuid_is_null(find_client->client_id)) { - if (client_count < requested_client_count) { - uuid_copy(list[client_count], find_client->client_id); - } - client_count++; + // Lock tree + NECP_CLIENT_TREE_LOCK_SHARED(); + + find_client = NULL; + RB_FOREACH(find_client, _necp_client_global_tree, &necp_client_global_tree) { + NECP_CLIENT_LOCK(find_client); + if (!uuid_is_null(find_client->client_id)) { + if (client_count < requested_client_count) { + uuid_copy(list[client_count], find_client->client_id); } + client_count++; } - lck_mtx_unlock(&client_fd->fd_lock); + NECP_CLIENT_UNLOCK(find_client); } - // Unlock list - lck_rw_done(&necp_fd_lock); + // Unlock tree + NECP_CLIENT_TREE_UNLOCK(); error = copyout(&client_count, uap->buffer, sizeof(client_count)); if (error) { @@ -2132,7 +3950,7 @@ necp_client_list(struct necp_fd_data *fd_data, struct necp_client_action_args *u if (requested_client_count > 0 && client_count > 0 && list != NULL) { - error = copyout(list, uap->buffer + sizeof(client_count), requested_client_count * sizeof(uuid_t)); + error = copyout(list, uap->buffer + sizeof(client_count), copy_buffer_size); if (error) { NECPLOG(LOG_ERR, "necp_client_list client count copyout error (%d)", error); goto done; @@ -2143,55 +3961,11 @@ done: FREE(list, M_NECP); } *retval = error; - - return (error); -} - -static int -necp_client_request_nexus(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) -{ - int error = 0; - struct necp_client *client = NULL; - uuid_t client_id; - bool requested_nexus = FALSE; - - if (uap->client_id == 0 || uap->client_id_len != sizeof(uuid_t)) { - error = EINVAL; - goto done; - } - - error = copyin(uap->client_id, client_id, sizeof(uuid_t)); - if (error) { - NECPLOG(LOG_ERR, "necp_client_request_nexus copyin client_id error (%d)", error); - goto done; - } - - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH(client, &fd_data->clients, chain) { - if (uuid_compare(client->client_id, client_id) == 0) { - // Request from nexus agent - if (!uuid_is_null(client->nexus_agent)) { - error = netagent_client_message(client->nexus_agent, client->client_id, - NETAGENT_MESSAGE_TYPE_REQUEST_NEXUS); - if (error == 0) { - requested_nexus = TRUE; - } - } - break; - } - } - lck_mtx_unlock(&fd_data->fd_lock); - - if (!requested_nexus && - error == 0) { - error = ENOENT; - } -done: - *retval = error; return (error); } + static void necp_client_add_assertion(struct necp_client *client, uuid_t netagent_uuid) { @@ -2234,7 +4008,6 @@ static int necp_client_agent_action(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { int error = 0; - struct necp_client *matched_client = NULL; struct necp_client *client = NULL; uuid_t client_id; bool acted_on_agent = FALSE; @@ -2264,20 +4037,15 @@ necp_client_agent_action(struct necp_fd_data *fd_data, struct necp_client_action goto done; } - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH(client, &fd_data->clients, chain) { - if (uuid_compare(client->client_id, client_id) == 0) { - matched_client = client; - break; - } - } - if (matched_client) { + NECP_FD_LOCK(fd_data); + client = necp_client_fd_find_client_and_lock(fd_data, client_id); + if (client != NULL) { size_t offset = 0; - while ((offset + sizeof(u_int8_t) + sizeof(u_int32_t)) <= parameters_size) { + while ((offset + sizeof(struct necp_tlv_header)) <= parameters_size) { u_int8_t type = necp_buffer_get_tlv_type(parameters, offset); u_int32_t length = necp_buffer_get_tlv_length(parameters, offset); - if (length > (parameters_size - (offset + sizeof(u_int8_t) + sizeof(u_int32_t)))) { + if (length > (parameters_size - (offset + sizeof(struct necp_tlv_header)))) { // If the length is larger than what can fit in the remaining parameters size, bail NECPLOG(LOG_ERR, "Invalid TLV length (%u)", length); break; @@ -2310,8 +4078,15 @@ necp_client_agent_action(struct necp_fd_data *fd_data, struct necp_client_action } } - error = netagent_client_message(agent_uuid, client_id, - netagent_message_type); + struct necp_client_nexus_parameters parsed_parameters = {}; + necp_client_copy_parameters_locked(client, &parsed_parameters); + + error = netagent_client_message_with_params(agent_uuid, + client_id, + fd_data->proc_pid, + netagent_message_type, + &parsed_parameters, + NULL, NULL); if (error == 0) { acted_on_agent = TRUE; } else { @@ -2325,10 +4100,12 @@ necp_client_agent_action(struct necp_fd_data *fd_data, struct necp_client_action } } - offset += sizeof(u_int8_t) + sizeof(u_int32_t) + length; + offset += sizeof(struct necp_tlv_header) + length; } + + NECP_CLIENT_UNLOCK(client); } - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); if (!acted_on_agent && error == 0) { @@ -2340,7 +4117,7 @@ done: FREE(parameters, M_NECP); parameters = NULL; } - + return (error); } @@ -2365,12 +4142,12 @@ necp_client_copy_agent(__unused struct necp_fd_data *fd_data, struct necp_client error = netagent_copyout(agent_uuid, uap->buffer, uap->buffer_size); if (error) { - NECPLOG(LOG_ERR, "necp_client_copy_agent netagent_copyout error (%d)", error); + // netagent_copyout already logs appropriate errors goto done; } done: *retval = error; - + return (error); } @@ -2378,7 +4155,6 @@ static int necp_client_agent_use(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { int error = 0; - struct necp_client *matched_client = NULL; struct necp_client *client = NULL; uuid_t client_id; struct necp_agent_use_parameters parameters; @@ -2401,21 +4177,16 @@ necp_client_agent_use(struct necp_fd_data *fd_data, struct necp_client_action_ar goto done; } - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH(client, &fd_data->clients, chain) { - if (uuid_compare(client->client_id, client_id) == 0) { - matched_client = client; - break; - } - } - - if (matched_client) { + NECP_FD_LOCK(fd_data); + client = necp_client_fd_find_client_and_lock(fd_data, client_id); + if (client != NULL) { error = netagent_use(parameters.agent_uuid, ¶meters.out_use_count); + NECP_CLIENT_UNLOCK(client); } else { error = ENOENT; } - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); if (error == 0) { error = copyout(¶meters, uap->buffer, uap->buffer_size); @@ -2478,17 +4249,29 @@ necp_client_copy_interface(__unused struct necp_fd_data *fd_data, struct necp_cl if (IFNET_IS_EXPENSIVE(interface)) { interface_details.flags |= NECP_INTERFACE_FLAG_EXPENSIVE; } + if ((interface->if_eflags & IFEF_TXSTART) == IFEF_TXSTART) { + interface_details.flags |= NECP_INTERFACE_FLAG_TXSTART; + } + if ((interface->if_eflags & IFEF_NOACKPRI) == IFEF_NOACKPRI) { + interface_details.flags |= NECP_INTERFACE_FLAG_NOACKPRI; + } interface_details.mtu = interface->if_mtu; - u_int8_t ipv4_signature_len = sizeof(interface_details.ipv4_signature); + u_int8_t ipv4_signature_len = sizeof(interface_details.ipv4_signature.signature); u_int16_t ipv4_signature_flags; - ifnet_get_netsignature(interface, AF_INET, &ipv4_signature_len, &ipv4_signature_flags, - (u_int8_t *)&interface_details.ipv4_signature); + if (ifnet_get_netsignature(interface, AF_INET, &ipv4_signature_len, &ipv4_signature_flags, + (u_int8_t *)&interface_details.ipv4_signature) != 0) { + ipv4_signature_len = 0; + } + interface_details.ipv4_signature.signature_len = ipv4_signature_len; - u_int8_t ipv6_signature_len = sizeof(interface_details.ipv6_signature); + u_int8_t ipv6_signature_len = sizeof(interface_details.ipv6_signature.signature); u_int16_t ipv6_signature_flags; - ifnet_get_netsignature(interface, AF_INET6, &ipv6_signature_len, &ipv6_signature_flags, - (u_int8_t *)&interface_details.ipv6_signature); + if (ifnet_get_netsignature(interface, AF_INET6, &ipv6_signature_len, &ipv6_signature_flags, + (u_int8_t *)&interface_details.ipv6_signature) != 0) { + ipv6_signature_len = 0; + } + interface_details.ipv6_signature.signature_len = ipv6_signature_len; } ifnet_head_done(); @@ -2504,106 +4287,74 @@ done: return (error); } + static int -necp_client_stats_action(struct necp_client *client, user_addr_t buffer, user_size_t buffer_size) +necp_client_copy_route_statistics(__unused struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { int error = 0; - struct necp_stats_hdr *stats_hdr = NULL; - - if (client->stats_area) { - // Close old stats if required. - if ((client->stats_uaddr != buffer) || (client->stats_ulen != buffer_size)) { - necp_destroy_client_stats(client); - } - } - - if ((buffer == 0) || (buffer_size == 0)) { - goto done; - } - - if (client->stats_area) { - // An update - error = copyin(client->stats_uaddr, client->stats_area, client->stats_ulen); - if (error) { - NECPLOG(LOG_ERR, "necp_client_stats_action copyin error on update (%d)", error); - } else { - // Future use - check - stats_hdr = (necp_stats_hdr *)client->stats_area; - if (stats_hdr->necp_stats_event != 0) { - ntstat_userland_stats_event(client->stats_handler_context, (userland_stats_event_t)stats_hdr->necp_stats_event); - } - } - goto done; - } + struct necp_client *client = NULL; + uuid_t client_id; - // A create - if ((buffer_size > sizeof(necp_all_stats)) || (buffer_size < sizeof(necp_stats_hdr))) { + if (uap->client_id == 0 || uap->client_id_len != sizeof(uuid_t) || + uap->buffer_size < sizeof(struct necp_stat_counts) || uap->buffer == 0) { + NECPLOG0(LOG_ERR, "necp_client_copy_route_statistics bad input"); error = EINVAL; goto done; } - if ((stats_hdr = _MALLOC(buffer_size, M_NECP, M_WAITOK | M_ZERO)) == NULL) { - error = ENOMEM; - goto done; - } - - client->stats_handler_context = NULL; - client->stats_uaddr = buffer; - client->stats_ulen = buffer_size; - client->stats_area = stats_hdr; - error = copyin(client->stats_uaddr, client->stats_area, client->stats_ulen); + error = copyin(uap->client_id, client_id, sizeof(uuid_t)); if (error) { - NECPLOG(LOG_ERR, "necp_client_stats_action copyin error on create (%d)", error); + NECPLOG(LOG_ERR, "necp_client_copy_route_statistics copyin client_id error (%d)", error); goto done; } - switch (stats_hdr->necp_stats_type) { - case NECP_CLIENT_STATISTICS_TYPE_TCP: { - if (stats_hdr->necp_stats_ver == NECP_CLIENT_STATISTICS_TYPE_TCP_VER_1) { - client->stats_handler_context = ntstat_userland_stats_open((userland_stats_provider_context *)client, - NSTAT_PROVIDER_TCP_USERLAND, 0, necp_request_tcp_netstats); - if (client->stats_handler_context == NULL) { - error = EIO; - } - } else { - error = ENOTSUP; - } - break; - } - case NECP_CLIENT_STATISTICS_TYPE_UDP: { - if (stats_hdr->necp_stats_ver != NECP_CLIENT_STATISTICS_TYPE_UDP_VER_1) { - client->stats_handler_context = ntstat_userland_stats_open((userland_stats_provider_context *)client, - NSTAT_PROVIDER_UDP_USERLAND, 0, necp_request_udp_netstats); - if (client->stats_handler_context == NULL) { - error = EIO; - } - } else { - error = ENOTSUP; - } - break; + // Lock + NECP_FD_LOCK(fd_data); + client = necp_client_fd_find_client_and_lock(fd_data, client_id); + if (client != NULL) { + NECP_CLIENT_ROUTE_LOCK(client); + struct nstat_counts route_stats = {}; + if (client->current_route != NULL && client->current_route->rt_stats != NULL) { + struct nstat_counts *rt_stats = client->current_route->rt_stats; + atomic_get_64(route_stats.nstat_rxpackets, &rt_stats->nstat_rxpackets); + atomic_get_64(route_stats.nstat_rxbytes, &rt_stats->nstat_rxbytes); + atomic_get_64(route_stats.nstat_txpackets, &rt_stats->nstat_txpackets); + atomic_get_64(route_stats.nstat_txbytes, &rt_stats->nstat_txbytes); + route_stats.nstat_rxduplicatebytes = rt_stats->nstat_rxduplicatebytes; + route_stats.nstat_rxoutoforderbytes = rt_stats->nstat_rxoutoforderbytes; + route_stats.nstat_txretransmit = rt_stats->nstat_txretransmit; + route_stats.nstat_connectattempts = rt_stats->nstat_connectattempts; + route_stats.nstat_connectsuccesses = rt_stats->nstat_connectsuccesses; + route_stats.nstat_min_rtt = rt_stats->nstat_min_rtt; + route_stats.nstat_avg_rtt = rt_stats->nstat_avg_rtt; + route_stats.nstat_var_rtt = rt_stats->nstat_var_rtt; } - default: { - error = ENOTSUP; - break; + + // Unlock before copying out + NECP_CLIENT_ROUTE_UNLOCK(client); + NECP_CLIENT_UNLOCK(client); + NECP_FD_UNLOCK(fd_data); + + error = copyout(&route_stats, uap->buffer, sizeof(route_stats)); + if (error) { + NECPLOG(LOG_ERR, "necp_client_copy_route_statistics copyout error (%d)", error); } + } else { + // Unlock + NECP_FD_UNLOCK(fd_data); + error = ENOENT; } -done: - if ((error) && (stats_hdr != NULL)) { - FREE(stats_hdr, M_NECP); - client->stats_area = NULL; - client->stats_handler_context = NULL; - client->stats_uaddr = 0; - client->stats_ulen = 0; - } + +done: + *retval = error; return (error); } static int -necp_client_set_statistics(__unused struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) +necp_client_update_cache(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { int error = 0; - struct necp_client *find_client = NULL; struct necp_client *client = NULL; uuid_t client_id; @@ -2614,24 +4365,102 @@ necp_client_set_statistics(__unused struct necp_fd_data *fd_data, struct necp_cl error = copyin(uap->client_id, client_id, sizeof(uuid_t)); if (error) { - NECPLOG(LOG_ERR, "necp_client_set_statistics copyin client_id error (%d)", error); + NECPLOG(LOG_ERR, "necp_client_update_cache copyin client_id error (%d)", error); goto done; } - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH(find_client, &fd_data->clients, chain) { - if (uuid_compare(find_client->client_id, client_id) == 0) { - client = find_client; - break; - } + NECP_FD_LOCK(fd_data); + client = necp_client_fd_find_client_and_lock(fd_data, client_id); + if (client == NULL) { + NECP_FD_UNLOCK(fd_data); + error = ENOENT; + goto done; + } + + NECP_CLIENT_ROUTE_LOCK(client); + // This needs to be changed when TFO/ECN is supported by multiple flows + struct necp_client_flow *flow = LIST_FIRST(&client->flow_list); + if (flow == NULL || + (flow->remote_addr.sa.sa_family != AF_INET && + flow->remote_addr.sa.sa_family != AF_INET6) || + (flow->local_addr.sa.sa_family != AF_INET && + flow->local_addr.sa.sa_family != AF_INET6)) { + error = EINVAL; + NECPLOG(LOG_ERR, "necp_client_update_cache no flow error (%d)", error); + goto done_unlock; + } + + necp_cache_buffer cache_buffer; + memset(&cache_buffer, 0, sizeof(cache_buffer)); + + if (uap->buffer_size != sizeof(necp_cache_buffer) || + uap->buffer == USER_ADDR_NULL) { + error = EINVAL; + goto done_unlock; } - if (client) { - error = necp_client_stats_action(client, uap->buffer, uap->buffer_size); + error = copyin(uap->buffer, &cache_buffer, sizeof(cache_buffer)); + if (error) { + NECPLOG(LOG_ERR, "necp_client_update_cache copyin cache buffer error (%d)", error); + goto done_unlock; + } + + if (cache_buffer.necp_cache_buf_type == NECP_CLIENT_CACHE_TYPE_ECN && + cache_buffer.necp_cache_buf_ver == NECP_CLIENT_CACHE_TYPE_ECN_VER_1) { + if (cache_buffer.necp_cache_buf_size != sizeof(necp_tcp_ecn_cache) || + cache_buffer.necp_cache_buf_addr == USER_ADDR_NULL) { + error = EINVAL; + goto done_unlock; + } + + necp_tcp_ecn_cache ecn_cache_buffer; + memset(&ecn_cache_buffer, 0, sizeof(ecn_cache_buffer)); + + error = copyin(cache_buffer.necp_cache_buf_addr, &ecn_cache_buffer, sizeof(necp_tcp_ecn_cache)); + if (error) { + NECPLOG(LOG_ERR, "necp_client_update_cache copyin ecn cache buffer error (%d)", error); + goto done_unlock; + } + + if (client->current_route != NULL && client->current_route->rt_ifp != NULL) { + if (!client->platform_binary) { + ecn_cache_buffer.necp_tcp_ecn_heuristics_success = 0; + } + tcp_heuristics_ecn_update(&ecn_cache_buffer, client->current_route->rt_ifp, + (union sockaddr_in_4_6 *)&flow->local_addr); + } + } else if (cache_buffer.necp_cache_buf_type == NECP_CLIENT_CACHE_TYPE_TFO && + cache_buffer.necp_cache_buf_ver == NECP_CLIENT_CACHE_TYPE_TFO_VER_1) { + if (cache_buffer.necp_cache_buf_size != sizeof(necp_tcp_tfo_cache) || + cache_buffer.necp_cache_buf_addr == USER_ADDR_NULL) { + error = EINVAL; + goto done_unlock; + } + + necp_tcp_tfo_cache tfo_cache_buffer; + memset(&tfo_cache_buffer, 0, sizeof(tfo_cache_buffer)); + + error = copyin(cache_buffer.necp_cache_buf_addr, &tfo_cache_buffer, sizeof(necp_tcp_tfo_cache)); + if (error) { + NECPLOG(LOG_ERR, "necp_client_update_cache copyin tfo cache buffer error (%d)", error); + goto done_unlock; + } + + if (client->current_route != NULL && client->current_route->rt_ifp != NULL) { + if (!client->platform_binary) { + tfo_cache_buffer.necp_tcp_tfo_heuristics_success = 0; + } + tcp_heuristics_tfo_update(&tfo_cache_buffer, client->current_route->rt_ifp, + (union sockaddr_in_4_6 *)&flow->local_addr, + (union sockaddr_in_4_6 *)&flow->remote_addr); + } } else { - error = ENOENT; + error = EINVAL; } - lck_mtx_unlock(&fd_data->fd_lock); +done_unlock: + NECP_CLIENT_ROUTE_UNLOCK(client); + NECP_CLIENT_UNLOCK(client); + NECP_FD_UNLOCK(fd_data); done: *retval = error; return (error); @@ -2653,7 +4482,7 @@ necp_client_action(struct proc *p, struct necp_client_action_args *uap, int *ret u_int32_t action = uap->action; switch (action) { case NECP_CLIENT_ACTION_ADD: { - return_value = necp_client_add(fd_data, uap, retval); + return_value = necp_client_add(p, fd_data, uap, retval); break; } case NECP_CLIENT_ACTION_REMOVE: { @@ -2661,7 +4490,8 @@ necp_client_action(struct proc *p, struct necp_client_action_args *uap, int *ret break; } case NECP_CLIENT_ACTION_COPY_PARAMETERS: - case NECP_CLIENT_ACTION_COPY_RESULT: { + case NECP_CLIENT_ACTION_COPY_RESULT: + case NECP_CLIENT_ACTION_COPY_UPDATED_RESULT: { return_value = necp_client_copy(fd_data, uap, retval); break; } @@ -2669,10 +4499,6 @@ necp_client_action(struct proc *p, struct necp_client_action_args *uap, int *ret return_value = necp_client_list(fd_data, uap, retval); break; } - case NECP_CLIENT_ACTION_REQUEST_NEXUS_INSTANCE: { - return_value = necp_client_request_nexus(fd_data, uap, retval); - break; - } case NECP_CLIENT_ACTION_AGENT: { return_value = necp_client_agent_action(fd_data, uap, retval); break; @@ -2689,8 +4515,16 @@ necp_client_action(struct proc *p, struct necp_client_action_args *uap, int *ret return_value = necp_client_copy_interface(fd_data, uap, retval); break; } - case NECP_CLIENT_ACTION_SET_STATISTICS: { - return_value = necp_client_set_statistics(fd_data, uap, retval); + case NECP_CLIENT_ACTION_COPY_ROUTE_STATISTICS: { + return_value = necp_client_copy_route_statistics(fd_data, uap, retval); + break; + } + case NECP_CLIENT_ACTION_UPDATE_CACHE: { + return_value = necp_client_update_cache(fd_data, uap, retval); + break; + } + case NECP_CLIENT_ACTION_COPY_CLIENT_UPDATE: { + return_value = necp_client_copy_client_update(fd_data, uap, retval); break; } default: { @@ -2736,7 +4570,8 @@ necp_match_policy(struct proc *p, struct necp_match_policy_args *uap, int32_t *r goto done; } - error = necp_application_find_policy_match_internal(p, parameters, uap->parameters_size, &returned_result, NULL, 0); + error = necp_application_find_policy_match_internal(p, parameters, uap->parameters_size, + &returned_result, NULL, 0, NULL, NULL, NULL, false); if (error) { goto done; } @@ -2827,7 +4662,7 @@ necp_set_socket_attributes(struct socket *so, struct sockopt *sopt) size_t valsize = sopt->sopt_valsize; if (valsize == 0 || - valsize > ((sizeof(u_int8_t) + sizeof(u_int32_t) + NECP_MAX_SOCKET_ATTRIBUTE_STRING_LENGTH) * 2)) { + valsize > ((sizeof(struct necp_tlv_header) + NECP_MAX_SOCKET_ATTRIBUTE_STRING_LENGTH) * 2)) { goto done; } @@ -2874,10 +4709,10 @@ necp_get_socket_attributes(struct socket *so, struct sockopt *sopt) struct inpcb *inp = sotoinpcb(so); if (inp->inp_necp_attributes.inp_domain != NULL) { - valsize += sizeof(u_int8_t) + sizeof(u_int32_t) + strlen(inp->inp_necp_attributes.inp_domain); + valsize += sizeof(struct necp_tlv_header) + strlen(inp->inp_necp_attributes.inp_domain); } if (inp->inp_necp_attributes.inp_account != NULL) { - valsize += sizeof(u_int8_t) + sizeof(u_int32_t) + strlen(inp->inp_necp_attributes.inp_account); + valsize += sizeof(struct necp_tlv_header) + strlen(inp->inp_necp_attributes.inp_account); } if (valsize == 0) { goto done; @@ -2890,11 +4725,13 @@ necp_get_socket_attributes(struct socket *so, struct sockopt *sopt) cursor = buffer; if (inp->inp_necp_attributes.inp_domain != NULL) { - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ATTRIBUTE_DOMAIN, strlen(inp->inp_necp_attributes.inp_domain), inp->inp_necp_attributes.inp_domain); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ATTRIBUTE_DOMAIN, strlen(inp->inp_necp_attributes.inp_domain), inp->inp_necp_attributes.inp_domain, + buffer, valsize); } if (inp->inp_necp_attributes.inp_account != NULL) { - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ATTRIBUTE_ACCOUNT, strlen(inp->inp_necp_attributes.inp_account), inp->inp_necp_attributes.inp_account); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ATTRIBUTE_ACCOUNT, strlen(inp->inp_necp_attributes.inp_account), inp->inp_necp_attributes.inp_account, + buffer, valsize); } error = sooptcopyout(sopt, buffer, valsize); @@ -2905,10 +4742,79 @@ done: if (buffer != NULL) { FREE(buffer, M_NECP); } - + return (error); } +void * +necp_create_nexus_assign_message(uuid_t nexus_instance, u_int32_t nexus_port, void *key, uint32_t key_length, + struct necp_client_endpoint *local_endpoint, struct necp_client_endpoint *remote_endpoint, + u_int32_t flow_adv_index, size_t *message_length) +{ + u_int8_t *buffer = NULL; + u_int8_t *cursor = NULL; + size_t valsize = 0; + bool has_nexus_assignment = FALSE; + + + if (!uuid_is_null(nexus_instance)) { + has_nexus_assignment = TRUE; + valsize += sizeof(struct necp_tlv_header) + sizeof(uuid_t); + valsize += sizeof(struct necp_tlv_header) + sizeof(u_int32_t); + } + if (flow_adv_index != NECP_FLOWADV_IDX_INVALID) { + valsize += sizeof(struct necp_tlv_header) + sizeof(u_int32_t); + } + if (key != NULL && key_length > 0) { + valsize += sizeof(struct necp_tlv_header) + key_length; + } + if (local_endpoint != NULL) { + valsize += sizeof(struct necp_tlv_header) + sizeof(struct necp_client_endpoint); + } + if (remote_endpoint != NULL) { + valsize += sizeof(struct necp_tlv_header) + sizeof(struct necp_client_endpoint); + } + if (valsize == 0) { + return (NULL); + } + + MALLOC(buffer, u_int8_t *, valsize, M_NETAGENT, M_WAITOK | M_ZERO); // Use M_NETAGENT area, since it is expected upon free + if (buffer == NULL) { + return (NULL); + } + + cursor = buffer; + if (has_nexus_assignment) { + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_RESULT_NEXUS_INSTANCE, sizeof(uuid_t), nexus_instance, buffer, valsize); + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_RESULT_NEXUS_PORT, sizeof(u_int32_t), &nexus_port, buffer, valsize); + } + if (flow_adv_index != NECP_FLOWADV_IDX_INVALID) { + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_RESULT_NEXUS_PORT_FLOW_INDEX, sizeof(u_int32_t), &flow_adv_index, buffer, valsize); + } + if (key != NULL && key_length > 0) { + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_PARAMETER_NEXUS_KEY, key_length, key, buffer, valsize); + } + if (local_endpoint != NULL) { + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_RESULT_LOCAL_ENDPOINT, sizeof(struct necp_client_endpoint), local_endpoint, buffer, valsize); + } + if (remote_endpoint != NULL) { + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_RESULT_REMOTE_ENDPOINT, sizeof(struct necp_client_endpoint), remote_endpoint, buffer, valsize); + } + + *message_length = valsize; + + return (buffer); +} + +void +necp_inpcb_remove_cb(struct inpcb *inp) +{ + if (!uuid_is_null(inp->necp_client_uuid)) { + necp_client_unregister_socket_flow(inp->necp_client_uuid, inp); + uuid_clear(inp->necp_client_uuid); + } +} + void necp_inpcb_dispose(struct inpcb *inp) { @@ -2922,6 +4828,15 @@ necp_inpcb_dispose(struct inpcb *inp) } } +void +necp_mppcb_dispose(struct mppcb *mpp) +{ + if (!uuid_is_null(mpp->necp_client_uuid)) { + necp_client_unregister_multipath_cb(mpp->necp_client_uuid, mpp); + uuid_clear(mpp->necp_client_uuid); + } +} + /// Module init errno_t @@ -2950,16 +4865,40 @@ necp_client_init(void) goto done; } - necp_client_tcall = thread_call_allocate(necp_update_all_clients_callout, NULL); - if (necp_client_tcall == NULL) { - NECPLOG0(LOG_ERR, "thread_call_allocate failed"); + necp_client_fd_size = sizeof(struct necp_fd_data); + necp_client_fd_zone = zinit(necp_client_fd_size, + NECP_CLIENT_FD_ZONE_MAX * necp_client_fd_size, + 0, NECP_CLIENT_FD_ZONE_NAME); + if (necp_client_fd_zone == NULL) { + NECPLOG0(LOG_ERR, "zinit(necp_client_fd) failed"); + result = ENOMEM; + goto done; + } + + necp_flow_size = sizeof(struct necp_client_flow); + necp_flow_zone = zinit(necp_flow_size, + NECP_FLOW_ZONE_MAX * necp_flow_size, + 0, NECP_FLOW_ZONE_NAME); + if (necp_flow_zone == NULL) { + NECPLOG0(LOG_ERR, "zinit(necp_flow) failed"); result = ENOMEM; goto done; } + necp_client_update_tcall = thread_call_allocate_with_options(necp_update_all_clients_callout, NULL, + THREAD_CALL_PRIORITY_KERNEL, THREAD_CALL_OPTIONS_ONCE); + VERIFY(necp_client_update_tcall != NULL); + lck_rw_init(&necp_fd_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); + lck_rw_init(&necp_observer_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); + lck_rw_init(&necp_client_tree_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); + lck_rw_init(&necp_collect_stats_list_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); LIST_INIT(&necp_fd_list); + LIST_INIT(&necp_fd_observer_list); + LIST_INIT(&necp_collect_stats_client_list); + + RB_INIT(&necp_client_global_tree); done: if (result != 0) { diff --git a/bsd/net/net_api_stats.h b/bsd/net/net_api_stats.h new file mode 100644 index 000000000..af986b05e --- /dev/null +++ b/bsd/net/net_api_stats.h @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef __NET_API_STATS__ +#define __NET_API_STATS__ + +#ifdef PRIVATE +#include + +/* + * net_api_stats counts the usage of the networking APIs + * + * Note: we are using signed 64 bit values to detect and prevent wrap around + */ +struct net_api_stats { + /* + * Interface Filters + */ + int64_t nas_iflt_attach_count; // Currently attached + int64_t nas_iflt_attach_total; // Total number of attachments + int64_t nas_iflt_attach_os_total; + + /* + * IP Filters + */ + int64_t nas_ipf_add_count; // Currently attached + int64_t nas_ipf_add_total; // Total number of attachments + int64_t nas_ipf_add_os_total; + + /* + * Socket Filters + */ + int64_t nas_sfltr_register_count; // Currently attached + int64_t nas_sfltr_register_total; // Total number of attachments + int64_t nas_sfltr_register_os_total; + + /* + * Sockets + */ + int64_t nas_socket_alloc_total; + int64_t nas_socket_in_kernel_total; + int64_t nas_socket_in_kernel_os_total; + int64_t nas_socket_necp_clientuuid_total; + + /* + * Sockets per protocol domains + */ + int64_t nas_socket_domain_local_total; + int64_t nas_socket_domain_route_total; + int64_t nas_socket_domain_inet_total; + int64_t nas_socket_domain_inet6_total; + int64_t nas_socket_domain_system_total; + int64_t nas_socket_domain_multipath_total; + int64_t nas_socket_domain_key_total; + int64_t nas_socket_domain_ndrv_total; + int64_t nas_socket_domain_other_total; + + /* + * Sockets per domain and type + */ + int64_t nas_socket_inet_stream_total; + int64_t nas_socket_inet_dgram_total; + int64_t nas_socket_inet_dgram_connected; + int64_t nas_socket_inet_dgram_dns; // port 53 + int64_t nas_socket_inet_dgram_no_data; // typically for interface ioctl + + int64_t nas_socket_inet6_stream_total; + int64_t nas_socket_inet6_dgram_total; + int64_t nas_socket_inet6_dgram_connected; + int64_t nas_socket_inet6_dgram_dns; // port 53 + int64_t nas_socket_inet6_dgram_no_data; // typically for interface ioctl + + /* + * Multicast join + */ + int64_t nas_socket_mcast_join_total; + int64_t nas_socket_mcast_join_os_total; + + /* + * IPv6 Extension Header Socket API + */ + int64_t nas_sock_inet6_stream_exthdr_in; + int64_t nas_sock_inet6_stream_exthdr_out; + int64_t nas_sock_inet6_dgram_exthdr_in; + int64_t nas_sock_inet6_dgram_exthdr_out; + + /* + * Nexus flows + */ + int64_t nas_nx_flow_inet_stream_total; + int64_t nas_nx_flow_inet_dgram_total; + + int64_t nas_nx_flow_inet6_stream_total; + int64_t nas_nx_flow_inet6_dgram_total; + + /* + * Interfaces + */ + int64_t nas_ifnet_alloc_count; + int64_t nas_ifnet_alloc_total; + int64_t nas_ifnet_alloc_os_count; + int64_t nas_ifnet_alloc_os_total; + + /* + * PF + */ + int64_t nas_pf_addrule_total; + int64_t nas_pf_addrule_os; + + /* + * vmnet API + */ + int64_t nas_vmnet_total; +}; + +#ifdef XNU_KERNEL_PRIVATE +extern struct net_api_stats net_api_stats; + +/* + * Increment up to the max value of int64_t + */ +#define INC_ATOMIC_INT64_LIM(counter) { \ + int64_t val; \ + do { \ + val = counter; \ + if (val >= INT64_MAX) { \ + break; \ + } \ + } while (!OSCompareAndSwap64(val, val + 1, &(counter))); \ +} +#endif /* XNU_KERNEL_PRIVATE */ + +#endif /* PRIVATE */ + +#endif /* __NET_API_STATS__ */ diff --git a/bsd/net/net_kev.h b/bsd/net/net_kev.h index ba1de1cbe..366b801a3 100644 --- a/bsd/net/net_kev.h +++ b/bsd/net/net_kev.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Apple Inc. All rights reserved. + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -101,6 +101,7 @@ #define KEV_INET6_NEW_LL_ADDR 4 /* Autoconf LL address appeared */ #define KEV_INET6_NEW_RTADV_ADDR 5 /* Autoconf address has appeared */ #define KEV_INET6_DEFROUTER 6 /* Default router detected */ +#define KEV_INET6_REQUEST_NAT64_PREFIX 7 /* Asking for the NAT64-prefix */ #ifdef PRIVATE #define KEV_ND6_SUBCLASS 7 /* IPv6 NDP subclass */ @@ -108,6 +109,13 @@ #define KEV_ND6_RA 1 #define KEV_ND6_NDFAILURE 2 /* IPv6 neighbor cache entry expiry */ #define KEV_ND6_NDALIVE 3 /* IPv6 neighbor reachable */ +#define KEV_ND6_DAD_FAILURE 4 /* IPv6 address failed DAD */ +#define KEV_ND6_DAD_SUCCESS 5 /* IPv6 address completed DAD */ +#define KEV_ND6_ADDR_DETACHED 6 /* IPv6 address is deemed detached */ +#define KEV_ND6_ADDR_DEPRECATED 7 /* IPv6 address's preferred lifetime expired */ +#define KEV_ND6_ADDR_EXPIRED 8 /* IPv6 address has expired */ +#define KEV_ND6_RTR_EXPIRED 9 /* IPv6 default router has expired */ +#define KEV_ND6_PFX_EXPIRED 10 /* IPv6 prefix has expired */ #define KEV_NECP_SUBCLASS 8 /* NECP subclasss */ /* KEV_NECP_SUBCLASS event codes */ @@ -127,8 +135,11 @@ #define KEV_NETEVENT_SUBCLASS 11 /* Generic Net events subclass */ /* KEV_NETEVENT_SUBCLASS event codes */ #define KEV_NETEVENT_APNFALLBACK 1 -#endif /* PRIVATE */ -#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ +#define KEV_MPTCP_SUBCLASS 12 /* Global MPTCP events subclass */ +/* KEV_MPTCP_SUBCLASS event codes */ +#define KEV_MPTCP_CELLUSE 1 +#endif /* PRIVATE */ +#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ #endif /* _NET_NETKEV_H_ */ diff --git a/bsd/net/net_stubs.c b/bsd/net/net_stubs.c index 31ac0627a..137682285 100644 --- a/bsd/net/net_stubs.c +++ b/bsd/net/net_stubs.c @@ -86,6 +86,7 @@ STUB(ifmaddr_release); STUB(ifnet_add_multicast); STUB(ifnet_addrlen); STUB(ifnet_allocate); +STUB(ifnet_allocate_internal); STUB(ifnet_attach); STUB(ifnet_attach_protocol); STUB(ifnet_baudrate); @@ -337,8 +338,6 @@ STUB(ifnet_set_poll_params); STUB(ifnet_set_rcvq_maxlen); STUB(ifnet_set_sndq_maxlen); STUB(ifnet_start); -STUB(ifnet_transmit_burst_end); -STUB(ifnet_transmit_burst_start); STUB(ifnet_tx_compl_status); STUB(ifnet_tx_compl); STUB(ifnet_flowid); @@ -346,9 +345,6 @@ STUB(ifnet_enable_output); STUB(ifnet_disable_output); STUB(ifnet_get_keepalive_offload_frames); STUB(ifnet_link_status_report); -STUB(ifnet_set_packetpreamblelen); -STUB(ifnet_packetpreamblelen); -STUB(ifnet_maxpacketpreamblelen); STUB(ifnet_set_fastlane_capable); STUB(ifnet_get_fastlane_capable); STUB(ifnet_get_unsent_bytes); @@ -454,6 +450,12 @@ STUB(arp_ifinit); STUB(arp_lookup_ip); STUB(ip_gre_register_input); STUB(sock_iskernel); +STUB(iflt_attach_internal); +STUB(ipf_addv4_internal); +STUB(ipf_addv6_internal); +STUB(sflt_register_internal); +STUB(sock_accept_internal); +STUB(sock_socket_internal); #undef STUB /* diff --git a/bsd/net/netsrc.c b/bsd/net/netsrc.c index 052858985..17f00fead 100644 --- a/bsd/net/netsrc.c +++ b/bsd/net/netsrc.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -26,6 +26,19 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +// Include netinet/in.h first. net/netsrc.h depends on netinet/in.h but +// netinet/in.h doesn't work with -Wpadded, -Wpacked. +#include + +#pragma clang diagnostic push +#pragma clang diagnostic error "-Wpadded" +#pragma clang diagnostic error "-Wpacked" +// This header defines structures shared with user space, so we need to ensure there is +// no compiler inserted padding in case the user space process isn't using the same +// architecture as the kernel (example: i386 process with x86_64 kernel). +#include +#pragma clang diagnostic pop + #include #include #include @@ -48,29 +61,7 @@ #include #include -#include - -static errno_t netsrc_ctlsend(kern_ctl_ref, uint32_t, void *, mbuf_t, int); -static errno_t netsrc_ctlconnect(kern_ctl_ref, struct sockaddr_ctl *, void **); -static errno_t netsrc_ipv4(kern_ctl_ref, uint32_t, struct netsrc_req *); -static errno_t netsrc_ipv6(kern_ctl_ref, uint32_t, struct netsrc_req *); - -static kern_ctl_ref netsrc_ctlref = NULL; - -__private_extern__ void -netsrc_init(void) -{ - errno_t error; - struct kern_ctl_reg netsrc_ctl = { - .ctl_connect = netsrc_ctlconnect, - .ctl_send = netsrc_ctlsend, - }; - - strlcpy(netsrc_ctl.ctl_name, NETSRC_CTLNAME, sizeof(NETSRC_CTLNAME)); - - if ((error = ctl_register(&netsrc_ctl, &netsrc_ctlref))) - printf("%s: ctl_register failed %d\n", __func__, error); -} +#include static errno_t netsrc_ctlconnect(kern_ctl_ref kctl, struct sockaddr_ctl *sac, void **uinfo) @@ -84,6 +75,198 @@ netsrc_ctlconnect(kern_ctl_ref kctl, struct sockaddr_ctl *sac, void **uinfo) return (0); } +static errno_t +netsrc_reply(kern_ctl_ref kctl, uint32_t unit, uint16_t version, + struct netsrc_rep *reply) +{ + switch (version) { + case NETSRC_CURVERS: + return ctl_enqueuedata(kctl, unit, reply, + sizeof(*reply), CTL_DATA_EOR); + case NETSRC_VERSION1: { + if ((reply->nrp_flags & NETSRC_FLAG_ROUTEABLE) == 0) { + return EHOSTUNREACH; + } +#define NETSRC_FLAG_V1_MASK (NETSRC_IP6_FLAG_TENTATIVE | \ + NETSRC_IP6_FLAG_TEMPORARY | \ + NETSRC_IP6_FLAG_DEPRECATED | \ + NETSRC_IP6_FLAG_OPTIMISTIC | \ + NETSRC_IP6_FLAG_SECURED) + struct netsrc_repv1 v1 = { + .nrp_src = reply->nrp_src, + .nrp_flags = (reply->nrp_flags & NETSRC_FLAG_V1_MASK), + .nrp_label = reply->nrp_label, + .nrp_precedence = reply->nrp_precedence, + .nrp_dstlabel = reply->nrp_dstlabel, + .nrp_dstprecedence = reply->nrp_dstprecedence + }; + return ctl_enqueuedata(kctl, unit, &v1, sizeof(v1), CTL_DATA_EOR); + } + } + return EINVAL; +} + +static void +netsrc_common(struct rtentry *rt, struct netsrc_rep *reply) +{ + if (!rt) { + return; + } + + // Gather statistics information + struct nstat_counts *rt_stats = rt->rt_stats; + if (rt_stats) { + reply->nrp_min_rtt = rt_stats->nstat_min_rtt; + reply->nrp_connection_attempts = rt_stats->nstat_connectattempts; + reply->nrp_connection_successes = rt_stats->nstat_connectsuccesses; + } + + // If this route didn't have any stats, check its parent + if (reply->nrp_min_rtt == 0) { + // Is this lock necessary? + RT_LOCK(rt); + if (rt->rt_parent) { + rt_stats = rt->rt_parent->rt_stats; + if (rt_stats) { + reply->nrp_min_rtt = rt_stats->nstat_min_rtt; + reply->nrp_connection_attempts = rt_stats->nstat_connectattempts; + reply->nrp_connection_successes = rt_stats->nstat_connectsuccesses; + } + } + RT_UNLOCK(rt); + } + reply->nrp_ifindex = rt->rt_ifp ? rt->rt_ifp->if_index : 0; + + if (rt->rt_ifp->if_eflags & IFEF_AWDL) { + reply->nrp_flags |= NETSRC_FLAG_AWDL; + } + if (rt->rt_flags & RTF_LOCAL) { + reply->nrp_flags |= NETSRC_FLAG_DIRECT; + } else if (!(rt->rt_flags & RTF_GATEWAY) && + (rt->rt_ifa && rt->rt_ifa->ifa_ifp && + !(rt->rt_ifa->ifa_ifp->if_flags & IFF_POINTOPOINT))) { + reply->nrp_flags |= NETSRC_FLAG_DIRECT; + } +} + +static struct in6_addrpolicy * +lookup_policy(struct sockaddr* sa) +{ + // alignment fun - if sa_family is AF_INET or AF_INET6, this is one of those + // addresses and it should be aligned, so this should be safe. + union sockaddr_in_4_6 *addr = (union sockaddr_in_4_6 *)(void*)sa; + if (addr->sa.sa_family == AF_INET6) { + return in6_addrsel_lookup_policy(&addr->sin6); + } else if (sa->sa_family == AF_INET) { + struct sockaddr_in6 mapped = { + .sin6_family = AF_INET6, + .sin6_len = sizeof(mapped), + .sin6_addr = IN6ADDR_V4MAPPED_INIT, + }; + mapped.sin6_addr.s6_addr32[3] = addr->sin.sin_addr.s_addr; + return in6_addrsel_lookup_policy(&mapped); + } + return NULL; +} + +static void +netsrc_policy_common(struct netsrc_req *request, struct netsrc_rep *reply) +{ + // Destination policy + struct in6_addrpolicy *policy = lookup_policy(&request->nrq_dst.sa); + if (policy != NULL && policy->label != -1) { + reply->nrp_dstlabel = policy->label; + reply->nrp_dstprecedence = policy->preced; + } + + // Source policy + policy = lookup_policy(&reply->nrp_src.sa); + if (policy != NULL && policy->label != -1) { + reply->nrp_label = policy->label; + reply->nrp_precedence = policy->preced; + } +} + +static errno_t +netsrc_ipv6(kern_ctl_ref kctl, uint32_t unit, struct netsrc_req *request) +{ + struct route_in6 ro = { + .ro_dst = request->nrq_sin6, + }; + + int error = 0; + struct in6_addr storage, *in6 = in6_selectsrc(&request->nrq_sin6, NULL, + NULL, &ro, NULL, &storage, + request->nrq_ifscope, &error); + struct netsrc_rep reply = { + .nrp_sin6.sin6_family = AF_INET6, + .nrp_sin6.sin6_len = sizeof(reply.nrp_sin6), + .nrp_sin6.sin6_addr = in6 ? *in6 : (struct in6_addr){}, + }; + netsrc_common(ro.ro_rt, &reply); + if (ro.ro_srcia == NULL && in6 != NULL) { + ro.ro_srcia = (struct ifaddr *)ifa_foraddr6_scoped(in6, reply.nrp_ifindex); + } + if (ro.ro_srcia) { + struct in6_ifaddr *ia = (struct in6_ifaddr *)ro.ro_srcia; +#define IA_TO_NRP_FLAG(flag) \ + if (ia->ia6_flags & IN6_IFF_##flag) { \ + reply.nrp_flags |= NETSRC_FLAG_IP6_##flag; \ + } + IA_TO_NRP_FLAG(TENTATIVE); + IA_TO_NRP_FLAG(TEMPORARY); + IA_TO_NRP_FLAG(DEPRECATED); + IA_TO_NRP_FLAG(OPTIMISTIC); + IA_TO_NRP_FLAG(SECURED); + IA_TO_NRP_FLAG(DYNAMIC); + IA_TO_NRP_FLAG(AUTOCONF); +#undef IA_TO_NRP_FLAG + reply.nrp_flags |= NETSRC_FLAG_ROUTEABLE; + } + ROUTE_RELEASE(&ro); + netsrc_policy_common(request, &reply); + return netsrc_reply(kctl, unit, request->nrq_ver, &reply); +} + +static errno_t +netsrc_ipv4(kern_ctl_ref kctl, uint32_t unit, struct netsrc_req *request) +{ + // Unfortunately, IPv4 doesn't have a function like in6_selectsrc + // Look up the route + lck_mtx_lock(rnh_lock); + struct rtentry *rt = rt_lookup(TRUE, &request->nrq_dst.sa, + NULL, rt_tables[AF_INET], + request->nrq_ifscope); + lck_mtx_unlock(rnh_lock); + + // Look up the ifa + struct netsrc_rep reply = {}; + if (rt) { + struct in_ifaddr *ia = NULL; + lck_rw_lock_shared(in_ifaddr_rwlock); + TAILQ_FOREACH(ia, &in_ifaddrhead, ia_link) { + IFA_LOCK_SPIN(&ia->ia_ifa); + if (ia->ia_ifp == rt->rt_ifp) { + IFA_ADDREF_LOCKED(&ia->ia_ifa); + break; + } + IFA_UNLOCK(&ia->ia_ifa); + } + lck_rw_done(in_ifaddr_rwlock); + + if (ia) { + reply.nrp_sin = *IA_SIN(ia); + IFA_REMREF_LOCKED(&ia->ia_ifa); + IFA_UNLOCK(&ia->ia_ifa); + reply.nrp_flags |= NETSRC_FLAG_ROUTEABLE; + } + netsrc_common(rt, &reply); + rtfree(rt); + } + netsrc_policy_common(request, &reply); + return netsrc_reply(kctl, unit, request->nrq_ver, &reply); +} + static errno_t netsrc_ctlsend(kern_ctl_ref kctl, uint32_t unit, void *uinfo, mbuf_t m, int flags) @@ -102,17 +285,26 @@ netsrc_ctlsend(kern_ctl_ref kctl, uint32_t unit, void *uinfo, mbuf_t m, mbuf_copydata(m, 0, sizeof(storage), &storage); nrq = &storage; } - /* We only have one version right now. */ - if (nrq->nrq_ver != NETSRC_VERSION1) { + if (nrq->nrq_ver > NETSRC_CURVERS) { error = EINVAL; goto out; } switch (nrq->nrq_sin.sin_family) { case AF_INET: - error = netsrc_ipv4(kctl, unit, nrq); + if (nrq->nrq_sin.sin_len < sizeof (nrq->nrq_sin) || + nrq->nrq_sin.sin_addr.s_addr == INADDR_ANY) { + error = EINVAL; + } else { + error = netsrc_ipv4(kctl, unit, nrq); + } break; case AF_INET6: - error = netsrc_ipv6(kctl, unit, nrq); + if (nrq->nrq_sin6.sin6_len < sizeof(nrq->nrq_sin6) || + IN6_IS_ADDR_UNSPECIFIED(&nrq->nrq_sin6.sin6_addr)) { + error = EINVAL; + } else { + error = netsrc_ipv6(kctl, unit, nrq); + } break; default: printf("%s: invalid family\n", __func__); @@ -125,132 +317,19 @@ out: } -static errno_t -netsrc_ipv4(kern_ctl_ref kctl, uint32_t unit, struct netsrc_req *nrq) +__private_extern__ void +netsrc_init(void) { - errno_t error = EHOSTUNREACH; - struct sockaddr_in *dstsin; - struct rtentry *rt; - struct in_ifaddr *ia; - struct netsrc_rep nrp; - struct sockaddr_in6 v4entry = { - .sin6_family = AF_INET6, - .sin6_len = sizeof(struct sockaddr_in6), - .sin6_addr = IN6ADDR_V4MAPPED_INIT, + struct kern_ctl_reg netsrc_ctl = { + .ctl_connect = netsrc_ctlconnect, + .ctl_send = netsrc_ctlsend, }; - struct in6_addrpolicy *policy; - dstsin = &nrq->nrq_sin; + strlcpy(netsrc_ctl.ctl_name, NETSRC_CTLNAME, sizeof(netsrc_ctl.ctl_name)); - if (dstsin->sin_len < sizeof (*dstsin) || - dstsin->sin_addr.s_addr == INADDR_ANY) - return (EINVAL); - - lck_mtx_lock(rnh_lock); - rt = rt_lookup(TRUE, (struct sockaddr *)dstsin, NULL, - rt_tables[AF_INET], nrq->nrq_ifscope); - lck_mtx_unlock(rnh_lock); - if (!rt) - return (EHOSTUNREACH); - lck_rw_lock_shared(in_ifaddr_rwlock); - TAILQ_FOREACH(ia, &in_ifaddrhead, ia_link) { - IFA_LOCK_SPIN(&ia->ia_ifa); - if (ia->ia_ifp == rt->rt_ifp) { - memset(&nrp, 0, sizeof(nrp)); - memcpy(&nrp.nrp_sin, IA_SIN(ia), sizeof(nrp.nrp_sin)); - IFA_UNLOCK(&ia->ia_ifa); - v4entry.sin6_addr.s6_addr32[3] = - nrp.nrp_sin.sin_addr.s_addr; - policy = in6_addrsel_lookup_policy(&v4entry); - if (policy->label != -1) { - nrp.nrp_label = policy->label; - nrp.nrp_precedence = policy->preced; - /* XXX might not be true */ - nrp.nrp_dstlabel = policy->label; - nrp.nrp_dstprecedence = policy->preced; - } - error = ctl_enqueuedata(kctl, unit, &nrp, - sizeof(nrp), CTL_DATA_EOR); - break; - } - IFA_UNLOCK(&ia->ia_ifa); - } - lck_rw_done(in_ifaddr_rwlock); - if (rt) - rtfree(rt); - - return (error); -} - -static errno_t -netsrc_ipv6(kern_ctl_ref kctl, uint32_t unit, struct netsrc_req *nrq) -{ - struct sockaddr_in6 *dstsin6; - struct in6_addr *in6, storage; - struct in6_ifaddr *ia; - struct route_in6 ro; - int error = EHOSTUNREACH; - struct netsrc_rep nrp; - - dstsin6 = &nrq->nrq_sin6; - - if (dstsin6->sin6_len < sizeof (*dstsin6) || - IN6_IS_ADDR_UNSPECIFIED(&dstsin6->sin6_addr)) - return (EINVAL); - - memset(&ro, 0, sizeof(ro)); - lck_mtx_lock(rnh_lock); - ro.ro_rt = rt_lookup(TRUE, (struct sockaddr *)dstsin6, NULL, - rt_tables[AF_INET6], nrq->nrq_ifscope); - lck_mtx_unlock(rnh_lock); - if (!ro.ro_rt) - return (EHOSTUNREACH); - in6 = in6_selectsrc(dstsin6, NULL, NULL, &ro, NULL, &storage, - nrq->nrq_ifscope, &error); - ROUTE_RELEASE(&ro); - if (!in6 || error) - return (error); - memset(&nrp, 0, sizeof(nrp)); - nrp.nrp_sin6.sin6_family = AF_INET6; - nrp.nrp_sin6.sin6_len = sizeof(nrp.nrp_sin6); - memcpy(&nrp.nrp_sin6.sin6_addr, in6, sizeof(nrp.nrp_sin6.sin6_addr)); - lck_rw_lock_shared(&in6_ifaddr_rwlock); - for (ia = in6_ifaddrs; ia; ia = ia->ia_next) { - if (memcmp(&ia->ia_addr.sin6_addr, in6, sizeof(*in6)) == 0) { - struct sockaddr_in6 sin6; - struct in6_addrpolicy *policy; - - if (ia->ia6_flags & IN6_IFF_TEMPORARY) - nrp.nrp_flags |= NETSRC_IP6_FLAG_TEMPORARY; - if (ia->ia6_flags & IN6_IFF_TENTATIVE) - nrp.nrp_flags |= NETSRC_IP6_FLAG_TENTATIVE; - if (ia->ia6_flags & IN6_IFF_DEPRECATED) - nrp.nrp_flags |= NETSRC_IP6_FLAG_DEPRECATED; - if (ia->ia6_flags & IN6_IFF_OPTIMISTIC) - nrp.nrp_flags |= NETSRC_IP6_FLAG_OPTIMISTIC; - if (ia->ia6_flags & IN6_IFF_SECURED) - nrp.nrp_flags |= NETSRC_IP6_FLAG_SECURED; - sin6.sin6_family = AF_INET6; - sin6.sin6_len = sizeof(sin6); - memcpy(&sin6.sin6_addr, in6, sizeof(*in6)); - policy = in6_addrsel_lookup_policy(&sin6); - if (policy->label != -1) { - nrp.nrp_label = policy->label; - nrp.nrp_precedence = policy->preced; - } - memcpy(&sin6.sin6_addr, &dstsin6->sin6_addr, - sizeof(dstsin6->sin6_addr)); - policy = in6_addrsel_lookup_policy(&sin6); - if (policy->label != -1) { - nrp.nrp_dstlabel = policy->label; - nrp.nrp_dstprecedence = policy->preced; - } - break; - } + static kern_ctl_ref netsrc_ctlref = NULL; + errno_t error = ctl_register(&netsrc_ctl, &netsrc_ctlref); + if (error != 0) { + printf("%s: ctl_register failed %d\n", __func__, error); } - lck_rw_done(&in6_ifaddr_rwlock); - error = ctl_enqueuedata(kctl, unit, &nrp, sizeof(nrp), - CTL_DATA_EOR); - - return (error); } diff --git a/bsd/net/netsrc.h b/bsd/net/netsrc.h index 13fcd456b..02d43fa19 100644 --- a/bsd/net/netsrc.h +++ b/bsd/net/netsrc.h @@ -28,28 +28,31 @@ #ifndef __NET_NETSRC_H__ +#include + #define NETSRC_CTLNAME "com.apple.netsrc" #define NETSRC_VERSION1 1 -#define NETSRC_CURVERS NETSRC_VERSION1 +#define NETSRC_VERSION2 2 +#define NETSRC_CURVERS NETSRC_VERSION2 struct netsrc_req { unsigned int nrq_ver; unsigned int nrq_ifscope; union { - struct sockaddr_in _usin; - struct sockaddr_in6 _usin6; - } _usa; + union sockaddr_in_4_6 nrq_dst; + union sockaddr_in_4_6 _usa; + }; }; -#define nrq_sin _usa._usin -#define nrq_sin6 _usa._usin6 +#define nrq_sin _usa.sin +#define nrq_sin6 _usa.sin6 -struct netsrc_rep { +struct netsrc_repv1 { union { - struct sockaddr_in _usin; - struct sockaddr_in6 _usin6; - } _usa; + union sockaddr_in_4_6 nrp_src; + union sockaddr_in_4_6 _usa; + }; #define NETSRC_IP6_FLAG_TENTATIVE 0x0001 #define NETSRC_IP6_FLAG_TEMPORARY 0x0002 #define NETSRC_IP6_FLAG_DEPRECATED 0x0004 @@ -60,10 +63,41 @@ struct netsrc_rep { uint16_t nrp_precedence; uint16_t nrp_dstlabel; uint16_t nrp_dstprecedence; + uint16_t nrp_unused; // Padding }; -#define nrp_sin _usa._usin -#define nrp_sin6 _usa._usin6 +struct netsrc_repv2 { + union { + union sockaddr_in_4_6 nrp_src; + union sockaddr_in_4_6 _usa; + }; + uint32_t nrp_min_rtt; + uint32_t nrp_connection_attempts; + uint32_t nrp_connection_successes; + // Continues from above, fixes naming +#define NETSRC_FLAG_IP6_TENTATIVE NETSRC_IP6_FLAG_TENTATIVE +#define NETSRC_FLAG_IP6_TEMPORARY NETSRC_IP6_FLAG_TEMPORARY +#define NETSRC_FLAG_IP6_DEPRECATED NETSRC_IP6_FLAG_DEPRECATED +#define NETSRC_FLAG_IP6_OPTIMISTIC NETSRC_IP6_FLAG_OPTIMISTIC +#define NETSRC_FLAG_IP6_SECURED NETSRC_IP6_FLAG_SECURED +#define NETSRC_FLAG_ROUTEABLE 0x00000020 +#define NETSRC_FLAG_DIRECT 0x00000040 +#define NETSRC_FLAG_AWDL 0x00000080 +#define NETSRC_FLAG_IP6_DYNAMIC 0x00000100 +#define NETSRC_FLAG_IP6_AUTOCONF 0x00000200 + uint32_t nrp_flags; + uint16_t nrp_label; + uint16_t nrp_precedence; + uint16_t nrp_dstlabel; + uint16_t nrp_dstprecedence; + uint16_t nrp_ifindex; + uint16_t nrp_unused; // Padding +}; + +#define netsrc_rep netsrc_repv2 + +#define nrp_sin nrp_src.sin +#define nrp_sin6 nrp_src.sin6 #ifdef KERNEL_PRIVATE __private_extern__ void netsrc_init(void); diff --git a/bsd/net/network_agent.c b/bsd/net/network_agent.c index a51eacfb8..14fe6dc99 100644 --- a/bsd/net/network_agent.c +++ b/bsd/net/network_agent.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2016, 2017 Apple Inc. All rights reserved. + * Copyright (c) 2014-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -80,6 +80,8 @@ LIST_HEAD(netagent_client_list_s, netagent_client); struct netagent_wrapper { LIST_ENTRY(netagent_wrapper) master_chain; u_int32_t control_unit; + netagent_event_f event_handler; + void *event_context; u_int32_t generation; u_int64_t use_count; struct netagent_client_list_s pending_triggers_list; @@ -87,8 +89,10 @@ struct netagent_wrapper { }; struct netagent_session { - u_int32_t control_unit; + u_int32_t control_unit; // A control unit of 0 indicates an agent owned by the kernel struct netagent_wrapper *wrapper; + netagent_event_f event_handler; + void *event_context; }; typedef enum { @@ -608,6 +612,18 @@ done: return (new_session); } +netagent_session_t netagent_create(netagent_event_f event_handler, void *context) +{ + struct netagent_session *session = netagent_create_session(0); + if (session == NULL) { + return NULL; + } + + session->event_handler = event_handler; + session->event_context = context; + return session; +} + static void netagent_free_wrapper(struct netagent_wrapper *wrapper) { @@ -668,6 +684,11 @@ netagent_delete_session(struct netagent_session *session) } } +void netagent_destroy(netagent_session_t session) +{ + return netagent_delete_session((struct netagent_session *)session); +} + static int netagent_packet_get_netagent_data_size(mbuf_t packet, int offset, int *err) { @@ -693,6 +714,8 @@ netagent_handle_register_inner(struct netagent_session *session, struct netagent lck_rw_lock_exclusive(&netagent_lock); new_wrapper->control_unit = session->control_unit; + new_wrapper->event_handler = session->event_handler; + new_wrapper->event_context = session->event_context; new_wrapper->generation = g_next_generation++; session->wrapper = new_wrapper; @@ -710,6 +733,56 @@ netagent_handle_register_inner(struct netagent_session *session, struct netagent return 0; } +errno_t +netagent_register(netagent_session_t _session, struct netagent *agent) +{ + int data_size = 0; + struct netagent_wrapper *new_wrapper = NULL; + + struct netagent_session *session = (struct netagent_session *)_session; + if (session == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot register agent on NULL session"); + return EINVAL; + } + + if (agent == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot register NULL agent"); + return EINVAL; + } + + if (session->wrapper != NULL) { + NETAGENTLOG0(LOG_ERR, "Session already has a registered agent"); + return EINVAL; + } + + data_size = agent->netagent_data_size; + if (data_size < 0 || data_size > NETAGENT_MAX_DATA_SIZE) { + NETAGENTLOG(LOG_ERR, "Register message size could not be read, data_size %d", + data_size); + return EINVAL; + } + + MALLOC(new_wrapper, struct netagent_wrapper *, sizeof(*new_wrapper) + data_size, M_NETAGENT, M_WAITOK); + if (new_wrapper == NULL) { + NETAGENTLOG0(LOG_ERR, "Failed to allocate agent"); + return ENOMEM; + } + + memset(new_wrapper, 0, sizeof(*new_wrapper) + data_size); + memcpy(&new_wrapper->netagent, agent, sizeof(struct netagent) + data_size); + + int error = netagent_handle_register_inner(session, new_wrapper); + if (error != 0) { + FREE(new_wrapper, M_NETAGENT); + return error; + } + + NETAGENTLOG0(LOG_DEBUG, "Registered new agent"); + netagent_post_event(new_wrapper->netagent.netagent_uuid, KEV_NETAGENT_REGISTERED, TRUE); + + return 0; +} + static errno_t netagent_handle_register_setopt(struct netagent_session *session, u_int8_t *payload, u_int32_t payload_length) @@ -846,6 +919,19 @@ fail: netagent_send_error_response(session, NETAGENT_MESSAGE_TYPE_REGISTER, message_id, response_error); } +errno_t +netagent_unregister(netagent_session_t _session) +{ + struct netagent_session *session = (struct netagent_session *)_session; + if (session == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot unregister NULL session"); + return EINVAL; + } + + netagent_unregister_session_wrapper(session); + return 0; +} + static errno_t netagent_handle_unregister_setopt(struct netagent_session *session, u_int8_t *payload, u_int32_t payload_length) @@ -978,6 +1064,8 @@ netagent_handle_update_inner(struct netagent_session *session, struct netagent_w netagent_free_wrapper(session->wrapper); session->wrapper = new_wrapper; new_wrapper->control_unit = session->control_unit; + new_wrapper->event_handler = session->event_handler; + new_wrapper->event_context = session->event_context; LIST_INSERT_HEAD(&master_netagent_list, new_wrapper, master_chain); LIST_INIT(&new_wrapper->pending_triggers_list); @@ -989,6 +1077,59 @@ netagent_handle_update_inner(struct netagent_session *session, struct netagent_w return response_error; } +errno_t +netagent_update(netagent_session_t _session, struct netagent *agent) +{ + u_int8_t agent_changed; + int data_size = 0; + struct netagent_wrapper *new_wrapper = NULL; + + struct netagent_session *session = (struct netagent_session *)_session; + if (session == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot update agent on NULL session"); + return EINVAL; + } + + if (agent == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot register NULL agent"); + return EINVAL; + } + + if (session->wrapper == NULL) { + NETAGENTLOG0(LOG_ERR, "Session has no agent to update"); + return EINVAL; + } + + data_size = agent->netagent_data_size; + if (data_size > NETAGENT_MAX_DATA_SIZE) { + NETAGENTLOG(LOG_ERR, "Update message size (%u > %u) too large", data_size, NETAGENT_MAX_DATA_SIZE); + return EINVAL; + } + + MALLOC(new_wrapper, struct netagent_wrapper *, sizeof(*new_wrapper) + data_size, M_NETAGENT, M_WAITOK); + if (new_wrapper == NULL) { + NETAGENTLOG0(LOG_ERR, "Failed to allocate agent"); + return ENOMEM; + } + + memset(new_wrapper, 0, sizeof(*new_wrapper) + data_size); + memcpy(&new_wrapper->netagent, agent, sizeof(struct netagent) + data_size); + + int error = netagent_handle_update_inner(session, new_wrapper, data_size, &agent_changed, kNetagentErrorDomainPOSIX); + if (error == 0) { + netagent_post_event(session->wrapper->netagent.netagent_uuid, KEV_NETAGENT_UPDATED, agent_changed); + if (agent_changed == FALSE) { + // The session wrapper does not need the "new_wrapper" as nothing changed + FREE(new_wrapper, M_NETAGENT); + } + } else { + FREE(new_wrapper, M_NETAGENT); + return error; + } + + return 0; +} + static errno_t netagent_handle_update_setopt(struct netagent_session *session, u_int8_t *payload, u_int32_t payload_length) { @@ -1182,6 +1323,57 @@ fail: netagent_send_error_response(session, NETAGENT_MESSAGE_TYPE_GET, message_id, response_error); } +errno_t +netagent_assign_nexus(netagent_session_t _session, uuid_t necp_client_uuid, + void *assign_message, size_t assigned_results_length) +{ + struct netagent_session *session = (struct netagent_session *)_session; + if (session == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot assign nexus from NULL session"); + return EINVAL; + } + + if (session->wrapper == NULL) { + NETAGENTLOG0(LOG_ERR, "Session has no agent"); + return ENOENT; + } + + // Note that if the error is 0, NECP has taken over our malloc'ed buffer + int error = necp_assign_client_result(session->wrapper->netagent.netagent_uuid, necp_client_uuid, assign_message, assigned_results_length); + if (error) { + // necp_assign_client_result returns POSIX errors; don't error for ENOENT + NETAGENTLOG((error == ENOENT ? LOG_DEBUG : LOG_ERR), "Client assignment failed: %d", error); + return error; + } + + NETAGENTLOG0(LOG_DEBUG, "Agent assigned nexus properties to client"); + return 0; +} + +errno_t +netagent_update_flow_protoctl_event(netagent_session_t _session, + uuid_t client_id, uint32_t protoctl_event_code, + uint32_t protoctl_event_val, uint32_t protoctl_event_tcp_seq_number) +{ + struct netagent_session *session = (struct netagent_session *)_session; + int error = 0; + + if (session == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot assign nexus from NULL session"); + return (EINVAL); + } + + if (session->wrapper == NULL) { + NETAGENTLOG0(LOG_ERR, "Session has no agent"); + return (ENOENT); + } + + error = necp_update_flow_protoctl_event(session->wrapper->netagent.netagent_uuid, + client_id, protoctl_event_code, protoctl_event_val, protoctl_event_tcp_seq_number); + + return (error); +} + static errno_t netagent_handle_assign_nexus_setopt(struct netagent_session *session, u_int8_t *payload, u_int32_t payload_length) @@ -1676,7 +1868,13 @@ done: } int -netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t message_type) +netagent_client_message_with_params(uuid_t agent_uuid, + uuid_t necp_client_uuid, + pid_t pid, + u_int8_t message_type, + struct necp_client_nexus_parameters *parameters, + void **assigned_results, + size_t *assigned_results_length) { int error = 0; @@ -1684,7 +1882,8 @@ netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t mes message_type != NETAGENT_MESSAGE_TYPE_CLIENT_ASSERT && message_type != NETAGENT_MESSAGE_TYPE_CLIENT_UNASSERT && message_type != NETAGENT_MESSAGE_TYPE_REQUEST_NEXUS && - message_type != NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS) { + message_type != NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS && + message_type != NETAGENT_MESSAGE_TYPE_ABORT_NEXUS) { NETAGENTLOG(LOG_ERR, "Client netagent message type (%d) is invalid", message_type); return(EINVAL); } @@ -1693,7 +1892,7 @@ netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t mes bool should_unlock = TRUE; struct netagent_wrapper *wrapper = netagent_find_agent_with_uuid(agent_uuid); if (wrapper == NULL) { - NETAGENTLOG0(LOG_ERR, "Requested netagent for nexus instance could not be found"); + NETAGENTLOG0(LOG_DEBUG, "Requested netagent for nexus instance could not be found"); error = ENOENT; goto done; } @@ -1704,19 +1903,25 @@ netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t mes // Don't log, since this is a common case used to trigger events that cellular data is blocked, etc. error = ENOTSUP; - struct proc *p = current_proc(); - pid_t current_pid = 0; - uuid_t current_proc_uuid; - uuid_clear(current_proc_uuid); - if (p != NULL) { - current_pid = proc_pid(p); - proc_getexecutableuuid(p, current_proc_uuid, sizeof(current_proc_uuid)); + + pid_t report_pid = 0; + uuid_t report_proc_uuid = {}; + if (parameters != NULL) { + report_pid = parameters->epid; + uuid_copy(report_proc_uuid, parameters->euuid); + } else { + struct proc *p = current_proc(); + if (p != NULL) { + report_pid = proc_pid(p); + proc_getexecutableuuid(p, report_proc_uuid, sizeof(report_proc_uuid)); + } } - netagent_send_cellular_failed_event(wrapper, current_pid, current_proc_uuid); + netagent_send_cellular_failed_event(wrapper, report_pid, report_proc_uuid); goto done; } } else if (message_type == NETAGENT_MESSAGE_TYPE_REQUEST_NEXUS || - message_type == NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS) { + message_type == NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS || + message_type == NETAGENT_MESSAGE_TYPE_ABORT_NEXUS) { if ((wrapper->netagent.netagent_flags & NETAGENT_FLAG_NEXUS_PROVIDER) == 0) { NETAGENTLOG0(LOG_ERR, "Requested netagent for nexus instance is not a nexus provider"); // Agent is not a nexus provider @@ -1732,29 +1937,54 @@ netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t mes } } - error = netagent_send_client_message(wrapper, necp_client_uuid, message_type); - if (error == 0 && message_type == NETAGENT_MESSAGE_TYPE_CLIENT_TRIGGER) { - if (lck_rw_lock_shared_to_exclusive(&netagent_lock)) { - // Grab the lock exclusively to add a pending client to the list - struct netagent_client *new_pending_client = NULL; - MALLOC(new_pending_client, struct netagent_client *, sizeof(*new_pending_client), M_NETAGENT, M_WAITOK); - if (new_pending_client == NULL) { - NETAGENTLOG0(LOG_ERR, "Failed to allocate client for trigger"); - } else { - uuid_copy(new_pending_client->client_id, necp_client_uuid); - struct proc *p = current_proc(); - if (p != NULL) { - new_pending_client->client_pid = proc_pid(p); - proc_getexecutableuuid(p, new_pending_client->client_proc_uuid, sizeof(new_pending_client->client_proc_uuid)); + if (wrapper->control_unit == 0) { + should_unlock = FALSE; + lck_rw_done(&netagent_lock); + if (wrapper->event_handler == NULL) { + // No event handler registered for kernel agent + error = EINVAL; + } else { + error = wrapper->event_handler(message_type, necp_client_uuid, pid, wrapper->event_context, parameters, assigned_results, assigned_results_length); + if (error != 0) { + VERIFY(assigned_results == NULL || *assigned_results == NULL); + VERIFY(assigned_results_length == NULL || *assigned_results_length == 0); + } + } + } else { + // ABORT_NEXUS is kernel-private, so translate it for userspace nexus + if (message_type == NETAGENT_MESSAGE_TYPE_ABORT_NEXUS) { + message_type = NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS; + } + + error = netagent_send_client_message(wrapper, necp_client_uuid, message_type); + if (error == 0 && message_type == NETAGENT_MESSAGE_TYPE_CLIENT_TRIGGER) { + if (lck_rw_lock_shared_to_exclusive(&netagent_lock)) { + // Grab the lock exclusively to add a pending client to the list + struct netagent_client *new_pending_client = NULL; + MALLOC(new_pending_client, struct netagent_client *, sizeof(*new_pending_client), M_NETAGENT, M_WAITOK); + if (new_pending_client == NULL) { + NETAGENTLOG0(LOG_ERR, "Failed to allocate client for trigger"); + } else { + uuid_copy(new_pending_client->client_id, necp_client_uuid); + if (parameters != NULL) { + new_pending_client->client_pid = parameters->epid; + uuid_copy(new_pending_client->client_proc_uuid, parameters->euuid); + } else { + struct proc *p = current_proc(); + if (p != NULL) { + new_pending_client->client_pid = proc_pid(p); + proc_getexecutableuuid(p, new_pending_client->client_proc_uuid, sizeof(new_pending_client->client_proc_uuid)); + } + } + LIST_INSERT_HEAD(&wrapper->pending_triggers_list, new_pending_client, client_chain); } - LIST_INSERT_HEAD(&wrapper->pending_triggers_list, new_pending_client, client_chain); + } else { + // If lck_rw_lock_shared_to_exclusive fails, it unlocks automatically + should_unlock = FALSE; } - } else { - // If lck_rw_lock_shared_to_exclusive fails, it unlocks automatically - should_unlock = FALSE; } } - NETAGENTLOG((error ? LOG_ERR : LOG_INFO), "Send message %d for client (error %d)", message_type, error); + NETAGENTLOG(((error && error != ENOENT) ? LOG_ERR : LOG_INFO), "Send message %d for client (error %d)", message_type, error); done: if (should_unlock) { lck_rw_done(&netagent_lock); @@ -1762,6 +1992,12 @@ done: return (error); } +int +netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, pid_t pid, u_int8_t message_type) +{ + return (netagent_client_message_with_params(agent_uuid, necp_client_uuid, pid, message_type, NULL, NULL, NULL)); +} + int netagent_use(uuid_t agent_uuid, uint64_t *out_use_count) { @@ -1795,7 +2031,7 @@ netagent_copyout(uuid_t agent_uuid, user_addr_t user_addr, u_int32_t user_size) lck_rw_lock_shared(&netagent_lock); struct netagent_wrapper *wrapper = netagent_find_agent_with_uuid(agent_uuid); if (wrapper == NULL) { - NETAGENTLOG0(LOG_ERR, "Requested netagent for nexus instance could not be found"); + NETAGENTLOG0(LOG_DEBUG, "Requested netagent for nexus instance could not be found"); error = ENOENT; goto done; } diff --git a/bsd/net/network_agent.h b/bsd/net/network_agent.h index f459e2f6d..3e2c86417 100644 --- a/bsd/net/network_agent.h +++ b/bsd/net/network_agent.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2016 Apple Inc. All rights reserved. + * Copyright (c) 2014-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -89,6 +89,7 @@ struct netagent_assign_nexus_message { #define NETAGENT_OPTION_TYPE_UPDATE NETAGENT_MESSAGE_TYPE_UPDATE // Pass netagent to update, no return value #define NETAGENT_OPTION_TYPE_ASSIGN_NEXUS NETAGENT_MESSAGE_TYPE_ASSIGN_NEXUS // Pass struct netagent_assign_nexus_message #define NETAGENT_OPTION_TYPE_USE_COUNT 16 // Pass use count to set, get current use count +#define NETAGENT_MESSAGE_TYPE_ABORT_NEXUS 17 // Kernel private #define NETAGENT_MESSAGE_FLAGS_RESPONSE 0x01 // Used for acks, errors, and query responses @@ -115,6 +116,7 @@ struct netagent_assign_nexus_message { #define NETAGENT_FLAG_SPECIFIC_USE_ONLY 0x0020 // Agent should only be used and activated when specifically required #define NETAGENT_FLAG_NETWORK_PROVIDER 0x0040 // Agent provides network access #define NETAGENT_FLAG_NEXUS_PROVIDER 0x0080 // Agent provides a skywalk nexus +#define NETAGENT_FLAG_SUPPORTS_BROWSE 0x0100 // Assertions will cause agent to fill in browse endpoints #define NETAGENT_NEXUS_MAX_REQUEST_TYPES 16 #define NETAGENT_NEXUS_MAX_RESOLUTION_TYPE_PAIRS 16 @@ -125,11 +127,19 @@ struct netagent_assign_nexus_message { #define NETAGENT_NEXUS_FRAME_TYPE_TRANSPORT 3 #define NETAGENT_NEXUS_FRAME_TYPE_APPLICATION 4 +#define NETAGENT_NEXUS_ENDPOINT_TYPE_ADDRESS 1 +#define NETAGENT_NEXUS_ENDPOINT_TYPE_HOST 2 +#define NETAGENT_NEXUS_ENDPOINT_TYPE_BONJOUR 3 + +#define NETAGENT_NEXUS_FLAG_SUPPORTS_USER_PACKET_POOL 0x1 +#define NETAGENT_NEXUS_FLAG_ASSERT_UNSUPPORTED 0x2 // No calls to assert the agent are required + struct netagent_nexus { u_int32_t frame_type; u_int32_t endpoint_assignment_type; u_int32_t endpoint_request_types[NETAGENT_NEXUS_MAX_REQUEST_TYPES]; u_int32_t endpoint_resolution_type_pairs[NETAGENT_NEXUS_MAX_RESOLUTION_TYPE_PAIRS * 2]; + u_int32_t nexus_flags; }; #define NETAGENT_TRIGGER_FLAG_USER 0x0001 // Userspace triggered agent @@ -196,6 +206,8 @@ struct netagentlist_req64 { user64_addr_t data __attribute__((aligned(8))); }; +struct necp_client_nexus_parameters; + // Kernel accessors extern void netagent_post_updated_interfaces(uuid_t uuid); // To be called from interface ioctls @@ -207,11 +219,60 @@ extern bool netagent_get_agent_domain_and_type(uuid_t uuid, char *domain, char * extern int netagent_kernel_trigger(uuid_t uuid); -extern int netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t message_type); +extern int netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, pid_t pid, u_int8_t message_type); + +extern int netagent_client_message_with_params(uuid_t agent_uuid, + uuid_t necp_client_uuid, + pid_t pid, + u_int8_t message_type, + struct necp_client_nexus_parameters *parameters, + void **assigned_results, + size_t *assigned_results_length); extern int netagent_copyout(uuid_t uuid, user_addr_t user_addr, u_int32_t user_size); + +// Kernel agent management + +typedef void * netagent_session_t; + +struct netagent_nexus_agent { + struct netagent agent; + struct netagent_nexus nexus_data; +}; + +#define NETAGENT_EVENT_TRIGGER NETAGENT_MESSAGE_TYPE_CLIENT_TRIGGER +#define NETAGENT_EVENT_ASSERT NETAGENT_MESSAGE_TYPE_CLIENT_ASSERT +#define NETAGENT_EVENT_UNASSERT NETAGENT_MESSAGE_TYPE_CLIENT_UNASSERT +#define NETAGENT_EVENT_NEXUS_FLOW_INSERT NETAGENT_MESSAGE_TYPE_REQUEST_NEXUS +#define NETAGENT_EVENT_NEXUS_FLOW_REMOVE NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS +#define NETAGENT_EVENT_NEXUS_FLOW_ABORT NETAGENT_MESSAGE_TYPE_ABORT_NEXUS + +typedef errno_t (*netagent_event_f)(u_int8_t event, uuid_t necp_client_uuid, pid_t pid, void *context, struct necp_client_nexus_parameters *parameters, void **assigned_results, size_t *assigned_results_length); + +extern netagent_session_t netagent_create(netagent_event_f event_handler, void *handle); + +extern void netagent_destroy(netagent_session_t session); + +extern errno_t netagent_register(netagent_session_t session, struct netagent *agent); + +extern errno_t netagent_update(netagent_session_t session, struct netagent *agent); + +extern errno_t netagent_unregister(netagent_session_t session); + +extern errno_t netagent_assign_nexus(netagent_session_t _session, + uuid_t necp_client_uuid, + void *assign_message, + size_t assigned_results_length); // Length of assigned_results_length + +extern errno_t netagent_update_flow_protoctl_event(netagent_session_t _session, + uuid_t client_id, + uint32_t protoctl_event_code, + uint32_t protoctl_event_val, + uint32_t protoctl_event_tcp_seq_number); + extern int netagent_use(uuid_t agent_uuid, uint64_t *out_use_count); + #endif /* BSD_KERNEL_PRIVATE */ #ifndef KERNEL diff --git a/bsd/net/ntstat.c b/bsd/net/ntstat.c index 1135bc55e..1d51a92e7 100644 --- a/bsd/net/ntstat.c +++ b/bsd/net/ntstat.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2016 Apple Inc. All rights reserved. + * Copyright (c) 2010-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -50,12 +50,25 @@ #include #include #include + +// These includes appear in ntstat.h but we include them here first so they won't trigger +// any clang diagnostic errors. +#include +#include +#include + +#pragma clang diagnostic push +#pragma clang diagnostic error "-Wpadded" +#pragma clang diagnostic error "-Wpacked" +// This header defines structures shared with user space, so we need to ensure there is +// no compiler inserted padding in case the user space process isn't using the same +// architecture as the kernel (example: i386 process with x86_64 kernel). #include +#pragma clang diagnostic pop #include #include #include -#include #include #include #include @@ -64,6 +77,8 @@ #include #include +extern unsigned int if_enable_netagent; + __private_extern__ int nstat_collect = 1; #if (DEBUG || DEVELOPMENT) @@ -71,7 +86,11 @@ SYSCTL_INT(_net, OID_AUTO, statistics, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_collect, 0, "Collect detailed statistics"); #endif /* (DEBUG || DEVELOPMENT) */ +#if CONFIG_EMBEDDED +static int nstat_privcheck = 1; +#else static int nstat_privcheck = 0; +#endif SYSCTL_INT(_net, OID_AUTO, statistics_privcheck, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_privcheck, 0, "Entitlement check"); @@ -94,6 +113,33 @@ static struct nstat_stats nstat_stats; SYSCTL_STRUCT(_net_stats, OID_AUTO, stats, CTLFLAG_RD | CTLFLAG_LOCKED, &nstat_stats, nstat_stats, ""); +static u_int32_t nstat_lim_interval = 30 * 60; /* Report interval, seconds */ +static u_int32_t nstat_lim_min_tx_pkts = 100; +static u_int32_t nstat_lim_min_rx_pkts = 100; +#if (DEBUG || DEVELOPMENT) +SYSCTL_INT(_net_stats, OID_AUTO, lim_report_interval, + CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_interval, 0, + "Low internet stat report interval"); + +SYSCTL_INT(_net_stats, OID_AUTO, lim_min_tx_pkts, + CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_min_tx_pkts, 0, + "Low Internet, min transmit packets threshold"); + +SYSCTL_INT(_net_stats, OID_AUTO, lim_min_rx_pkts, + CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_min_rx_pkts, 0, + "Low Internet, min receive packets threshold"); +#endif /* DEBUG || DEVELOPMENT */ + +static struct net_api_stats net_api_stats_before; +static u_int64_t net_api_stats_last_report_time; +#define NET_API_STATS_REPORT_INTERVAL (12 * 60 * 60) /* 12 hours, in seconds */ +static u_int32_t net_api_stats_report_interval = NET_API_STATS_REPORT_INTERVAL; + +#if (DEBUG || DEVELOPMENT) +SYSCTL_UINT(_net_stats, OID_AUTO, api_report_interval, + CTLFLAG_RW | CTLFLAG_LOCKED, &net_api_stats_report_interval, 0, ""); +#endif /* DEBUG || DEVELOPMENT */ + enum { NSTAT_FLAG_CLEANUP = (1 << 0), @@ -102,7 +148,14 @@ enum NSTAT_FLAG_SYSINFO_SUBSCRIBED = (1 << 3), }; +#if CONFIG_EMBEDDED +#define QUERY_CONTINUATION_SRC_COUNT 50 +#else #define QUERY_CONTINUATION_SRC_COUNT 100 +#endif + +typedef TAILQ_HEAD(, nstat_src) tailq_head_nstat_src; +typedef TAILQ_ENTRY(nstat_src) tailq_entry_nstat_src; typedef struct nstat_provider_filter { @@ -117,11 +170,11 @@ typedef struct nstat_control_state { struct nstat_control_state *ncs_next; u_int32_t ncs_watching; - decl_lck_mtx_data(, mtx); + decl_lck_mtx_data(, ncs_mtx); kern_ctl_ref ncs_kctl; u_int32_t ncs_unit; nstat_src_ref_t ncs_next_srcref; - struct nstat_src *ncs_srcs; + tailq_head_nstat_src ncs_src_queue; mbuf_t ncs_accumulated; u_int32_t ncs_flags; nstat_provider_filter ncs_provider_filters[NSTAT_PROVIDER_COUNT]; @@ -138,7 +191,7 @@ typedef struct nstat_provider errno_t (*nstat_lookup)(const void *data, u_int32_t length, nstat_provider_cookie_t *out_cookie); int (*nstat_gone)(nstat_provider_cookie_t cookie); errno_t (*nstat_counts)(nstat_provider_cookie_t cookie, struct nstat_counts *out_counts, int *out_gone); - errno_t (*nstat_watcher_add)(nstat_control_state *state); + errno_t (*nstat_watcher_add)(nstat_control_state *state, nstat_msg_add_all_srcs *req); void (*nstat_watcher_remove)(nstat_control_state *state); errno_t (*nstat_copy_descriptor)(nstat_provider_cookie_t cookie, void *data, u_int32_t len); void (*nstat_release)(nstat_provider_cookie_t cookie, boolean_t locked); @@ -151,9 +204,13 @@ typedef STAILQ_ENTRY(nstat_src) stailq_entry_nstat_src; typedef TAILQ_HEAD(, nstat_tu_shadow) tailq_head_tu_shadow; typedef TAILQ_ENTRY(nstat_tu_shadow) tailq_entry_tu_shadow; +typedef TAILQ_HEAD(, nstat_procdetails) tailq_head_procdetails; +typedef TAILQ_ENTRY(nstat_procdetails) tailq_entry_procdetails; + typedef struct nstat_src { - struct nstat_src *next; + tailq_entry_nstat_src ns_control_link; // All sources for the nstat_control_state, for iterating over. + nstat_control_state *ns_control; // The nstat_control_state that this is a source for nstat_src_ref_t srcref; nstat_provider *provider; nstat_provider_cookie_t cookie; @@ -172,11 +229,12 @@ static bool nstat_control_reporting_allowed(nstat_control_state *state, nstat_s static boolean_t nstat_control_begin_query(nstat_control_state *state, const nstat_msg_hdr *hdrp); static u_int16_t nstat_control_end_query(nstat_control_state *state, nstat_src *last_src, boolean_t partial); static void nstat_ifnet_report_ecn_stats(void); +static void nstat_ifnet_report_lim_stats(void); +static void nstat_net_api_report_stats(void); +static errno_t nstat_set_provider_filter( nstat_control_state *state, nstat_msg_add_all_srcs *req); static u_int32_t nstat_udp_watchers = 0; -static u_int32_t nstat_userland_udp_watchers = 0; static u_int32_t nstat_tcp_watchers = 0; -static u_int32_t nstat_userland_tcp_watchers = 0; static void nstat_control_register(void); @@ -185,7 +243,7 @@ static void nstat_control_register(void); * * socket_lock (inpcb) * nstat_mtx - * state->mtx + * state->ncs_mtx */ static volatile OSMallocTag nstat_malloc_tag = NULL; static nstat_control_state *nstat_controls = NULL; @@ -234,28 +292,7 @@ nstat_ip_to_sockaddr( sin->sin_addr = *ip; } -static void -nstat_ip6_to_sockaddr( - const struct in6_addr *ip6, - u_int16_t port, - struct sockaddr_in6 *sin6, - u_int32_t maxlen) -{ - if (maxlen < sizeof(struct sockaddr_in6)) - return; - - sin6->sin6_family = AF_INET6; - sin6->sin6_len = sizeof(*sin6); - sin6->sin6_port = port; - sin6->sin6_addr = *ip6; - if (IN6_IS_SCOPE_EMBED(&sin6->sin6_addr)) - { - sin6->sin6_scope_id = ntohs(sin6->sin6_addr.s6_addr16[1]); - sin6->sin6_addr.s6_addr16[1] = 0; - } -} - -static u_int16_t +u_int16_t nstat_ifnet_to_flags( struct ifnet *ifp) { @@ -362,9 +399,7 @@ nstat_lookup_entry( static void nstat_init_route_provider(void); static void nstat_init_tcp_provider(void); -static void nstat_init_userland_tcp_provider(void); static void nstat_init_udp_provider(void); -static void nstat_init_userland_udp_provider(void); static void nstat_init_ifnet_provider(void); __private_extern__ void @@ -383,9 +418,7 @@ nstat_init(void) // we need to initialize other things, we do it here as this code path will only be hit once; nstat_init_route_provider(); nstat_init_tcp_provider(); - nstat_init_userland_tcp_provider(); nstat_init_udp_provider(); - nstat_init_userland_udp_provider(); nstat_init_ifnet_provider(); nstat_control_register(); } @@ -555,7 +588,7 @@ nstat_route_walktree_add( struct rtentry *rt = (struct rtentry *)rn; nstat_control_state *state = (nstat_control_state*)context; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* RTF_UP can't change while rnh_lock is held */ if ((rt->rt_flags & RTF_UP) != 0) @@ -584,23 +617,34 @@ nstat_route_walktree_add( static errno_t nstat_route_add_watcher( - nstat_control_state *state) + nstat_control_state *state, + nstat_msg_add_all_srcs *req) { int i; errno_t result = 0; - OSIncrementAtomic(&nstat_route_watchers); lck_mtx_lock(rnh_lock); - for (i = 1; i < AF_MAX; i++) + + result = nstat_set_provider_filter(state, req); + if (result == 0) { - struct radix_node_head *rnh; - rnh = rt_tables[i]; - if (!rnh) continue; + OSIncrementAtomic(&nstat_route_watchers); - result = rnh->rnh_walktree(rnh, nstat_route_walktree_add, state); - if (result != 0) + for (i = 1; i < AF_MAX; i++) { - break; + struct radix_node_head *rnh; + rnh = rt_tables[i]; + if (!rnh) continue; + + result = rnh->rnh_walktree(rnh, nstat_route_walktree_add, state); + if (result != 0) + { + // This is probably resource exhaustion. + // There currently isn't a good way to recover from this. + // Least bad seems to be to give up on the add-all but leave + // the watcher in place. + break; + } } } lck_mtx_unlock(rnh_lock); @@ -726,7 +770,7 @@ nstat_init_route_provider(void) #pragma mark -- Route Collection -- -static struct nstat_counts* +__private_extern__ struct nstat_counts* nstat_route_attach( struct rtentry *rte) { @@ -850,68 +894,99 @@ nstat_route_rx( } } +/* atomically average current value at _val_addr with _new_val and store */ +#define NSTAT_EWMA_ATOMIC(_val_addr, _new_val, _decay) do { \ + volatile uint32_t _old_val; \ + volatile uint32_t _avg; \ + do { \ + _old_val = *_val_addr; \ + if (_old_val == 0) \ + { \ + _avg = _new_val; \ + } \ + else \ + { \ + _avg = _old_val - (_old_val >> _decay) + (_new_val >> _decay); \ + } \ + if (_old_val == _avg) break; \ + } while (!OSCompareAndSwap(_old_val, _avg, _val_addr)); \ +} while (0); + +/* atomically compute minimum of current value at _val_addr with _new_val and store */ +#define NSTAT_MIN_ATOMIC(_val_addr, _new_val) do { \ + volatile uint32_t _old_val; \ + do { \ + _old_val = *_val_addr; \ + if (_old_val != 0 && _old_val < _new_val) \ + { \ + break; \ + } \ + } while (!OSCompareAndSwap(_old_val, _new_val, _val_addr)); \ +} while (0); + __private_extern__ void nstat_route_rtt( struct rtentry *rte, u_int32_t rtt, u_int32_t rtt_var) { - const int32_t factor = 8; + const uint32_t decay = 3; while (rte) { struct nstat_counts* stats = nstat_route_attach(rte); if (stats) { - int32_t oldrtt; - int32_t newrtt; - - // average - do - { - oldrtt = stats->nstat_avg_rtt; - if (oldrtt == 0) - { - newrtt = rtt; - } - else - { - newrtt = oldrtt - (oldrtt - (int32_t)rtt) / factor; - } - if (oldrtt == newrtt) break; - } while (!OSCompareAndSwap(oldrtt, newrtt, &stats->nstat_avg_rtt)); + NSTAT_EWMA_ATOMIC(&stats->nstat_avg_rtt, rtt, decay); + NSTAT_MIN_ATOMIC(&stats->nstat_min_rtt, rtt); + NSTAT_EWMA_ATOMIC(&stats->nstat_var_rtt, rtt_var, decay); + } + rte = rte->rt_parent; + } +} - // minimum - do - { - oldrtt = stats->nstat_min_rtt; - if (oldrtt != 0 && oldrtt < (int32_t)rtt) - { - break; - } - } while (!OSCompareAndSwap(oldrtt, rtt, &stats->nstat_min_rtt)); +__private_extern__ void +nstat_route_update( + struct rtentry *rte, + uint32_t connect_attempts, + uint32_t connect_successes, + uint32_t rx_packets, + uint32_t rx_bytes, + uint32_t rx_duplicatebytes, + uint32_t rx_outoforderbytes, + uint32_t tx_packets, + uint32_t tx_bytes, + uint32_t tx_retransmit, + uint32_t rtt, + uint32_t rtt_var) +{ + const uint32_t decay = 3; - // variance - do - { - oldrtt = stats->nstat_var_rtt; - if (oldrtt == 0) - { - newrtt = rtt_var; - } - else - { - newrtt = oldrtt - (oldrtt - (int32_t)rtt_var) / factor; - } - if (oldrtt == newrtt) break; - } while (!OSCompareAndSwap(oldrtt, newrtt, &stats->nstat_var_rtt)); + while (rte) + { + struct nstat_counts* stats = nstat_route_attach(rte); + if (stats) + { + OSAddAtomic(connect_attempts, &stats->nstat_connectattempts); + OSAddAtomic(connect_successes, &stats->nstat_connectsuccesses); + OSAddAtomic64((SInt64)tx_packets, (SInt64*)&stats->nstat_txpackets); + OSAddAtomic64((SInt64)tx_bytes, (SInt64*)&stats->nstat_txbytes); + OSAddAtomic(tx_retransmit, &stats->nstat_txretransmit); + OSAddAtomic64((SInt64)rx_packets, (SInt64*)&stats->nstat_rxpackets); + OSAddAtomic64((SInt64)rx_bytes, (SInt64*)&stats->nstat_rxbytes); + OSAddAtomic(rx_outoforderbytes, &stats->nstat_rxoutoforderbytes); + OSAddAtomic(rx_duplicatebytes, &stats->nstat_rxduplicatebytes); + + if (rtt != 0) { + NSTAT_EWMA_ATOMIC(&stats->nstat_avg_rtt, rtt, decay); + NSTAT_MIN_ATOMIC(&stats->nstat_min_rtt, rtt); + NSTAT_EWMA_ATOMIC(&stats->nstat_var_rtt, rtt_var, decay); + } } - rte = rte->rt_parent; } } - #pragma mark -- TCP Kernel Provider -- /* @@ -955,7 +1030,7 @@ nstat_tucookie_alloc_internal( if (cookie == NULL) return NULL; if (!locked) - lck_mtx_assert(&nstat_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&nstat_mtx, LCK_MTX_ASSERT_NOTOWNED); if (ref && in_pcb_checkstate(inp, WNT_ACQUIRE, locked) == WNT_STOPUSING) { OSFree(cookie, sizeof(*cookie), nstat_malloc_tag); @@ -1188,31 +1263,46 @@ nstat_tcp_release( static errno_t nstat_tcp_add_watcher( - nstat_control_state *state) -{ - OSIncrementAtomic(&nstat_tcp_watchers); + nstat_control_state *state, + nstat_msg_add_all_srcs *req) +{ + // There is a tricky issue around getting all TCP sockets added once + // and only once. nstat_tcp_new_pcb() is called prior to the new item + // being placed on any lists where it might be found. + // By locking the tcbinfo.ipi_lock prior to marking the state as a watcher, + // it should be impossible for a new socket to be added twice. + // On the other hand, there is still a timing issue where a new socket + // results in a call to nstat_tcp_new_pcb() before this watcher + // is instantiated and yet the socket doesn't make it into ipi_listhead + // prior to the scan. - lck_rw_lock_shared(tcbinfo.ipi_lock); + errno_t result; - // Add all current tcp inpcbs. Ignore those in timewait - struct inpcb *inp; - struct nstat_tucookie *cookie; - LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list) - { - cookie = nstat_tucookie_alloc_ref(inp); - if (cookie == NULL) - continue; - if (nstat_control_source_add(0, state, &nstat_tcp_provider, - cookie) != 0) + lck_rw_lock_shared(tcbinfo.ipi_lock); + result = nstat_set_provider_filter(state, req); + if (result == 0) { + OSIncrementAtomic(&nstat_tcp_watchers); + + // Add all current tcp inpcbs. Ignore those in timewait + struct inpcb *inp; + struct nstat_tucookie *cookie; + LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list) { - nstat_tucookie_release(cookie); - break; + cookie = nstat_tucookie_alloc_ref(inp); + if (cookie == NULL) + continue; + if (nstat_control_source_add(0, state, &nstat_tcp_provider, + cookie) != 0) + { + nstat_tucookie_release(cookie); + break; + } } } lck_rw_done(tcbinfo.ipi_lock); - return 0; + return result; } static void @@ -1228,6 +1318,8 @@ nstat_tcp_new_pcb( { struct nstat_tucookie *cookie; + inp->inp_start_timestamp = mach_continuous_time(); + if (nstat_tcp_watchers == 0) return; @@ -1260,20 +1352,20 @@ __private_extern__ void nstat_pcb_detach(struct inpcb *inp) { nstat_control_state *state; - nstat_src *src, *prevsrc; - nstat_src *dead_list = NULL; + nstat_src *src; + tailq_head_nstat_src dead_list; struct nstat_tucookie *tucookie; errno_t result; if (inp == NULL || (nstat_tcp_watchers == 0 && nstat_udp_watchers == 0)) return; + TAILQ_INIT(&dead_list); lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (prevsrc = NULL, src = state->ncs_srcs; src; - prevsrc = src, src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { nstat_provider_id_t provider_id = src->provider->nstat_provider_id; if (provider_id == NSTAT_PROVIDER_TCP_KERNEL || provider_id == NSTAT_PROVIDER_UDP_KERNEL) @@ -1288,22 +1380,16 @@ nstat_pcb_detach(struct inpcb *inp) { result = nstat_control_send_goodbye(state, src); - if (prevsrc) - prevsrc->next = src->next; - else - state->ncs_srcs = src->next; - - src->next = dead_list; - dead_list = src; + TAILQ_REMOVE(&state->ncs_src_queue, src, ns_control_link); + TAILQ_INSERT_TAIL(&dead_list, src, ns_control_link); } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); - while (dead_list) { - src = dead_list; - dead_list = src->next; - + while ((src = TAILQ_FIRST(&dead_list))) + { + TAILQ_REMOVE(&dead_list, src, ns_control_link); nstat_control_cleanup_source(NULL, src, TRUE); } } @@ -1321,19 +1407,19 @@ nstat_pcb_cache(struct inpcb *inp) VERIFY(SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP); lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { tucookie = (struct nstat_tucookie *)src->cookie; if (tucookie->inp == inp) { if (inp->inp_vflag & INP_IPV6) { - nstat_ip6_to_sockaddr(&inp->in6p_laddr, + in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, &tucookie->local.v6, sizeof(tucookie->local)); - nstat_ip6_to_sockaddr(&inp->in6p_faddr, + in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, &tucookie->remote.v6, sizeof(tucookie->remote)); @@ -1358,7 +1444,7 @@ nstat_pcb_cache(struct inpcb *inp) break; } } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); } @@ -1376,8 +1462,8 @@ nstat_pcb_invalidate_cache(struct inpcb *inp) VERIFY(SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP); lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { tucookie = (struct nstat_tucookie *)src->cookie; if (tucookie->inp == inp) @@ -1386,7 +1472,7 @@ nstat_pcb_invalidate_cache(struct inpcb *inp) break; } } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); } @@ -1414,9 +1500,9 @@ nstat_tcp_copy_descriptor( if (inp->inp_vflag & INP_IPV6) { - nstat_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, + in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, &desc->local.v6, sizeof(desc->local)); - nstat_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, + in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, &desc->remote.v6, sizeof(desc->remote)); } else if (inp->inp_vflag & INP_IPV4) @@ -1484,6 +1570,9 @@ nstat_tcp_copy_descriptor( tcp_get_connectivity_status(tp, &desc->connstatus); desc->ifnet_properties = nstat_inpcb_to_flags(inp); + inp_get_activity_bitmap(inp, &desc->activity_bitmap); + desc->start_timestamp = inp->inp_start_timestamp; + desc->timestamp = mach_continuous_time(); return 0; } @@ -1667,31 +1756,48 @@ nstat_udp_release( static errno_t nstat_udp_add_watcher( - nstat_control_state *state) -{ - struct inpcb *inp; - struct nstat_tucookie *cookie; + nstat_control_state *state, + nstat_msg_add_all_srcs *req) +{ + // There is a tricky issue around getting all UDP sockets added once + // and only once. nstat_udp_new_pcb() is called prior to the new item + // being placed on any lists where it might be found. + // By locking the udpinfo.ipi_lock prior to marking the state as a watcher, + // it should be impossible for a new socket to be added twice. + // On the other hand, there is still a timing issue where a new socket + // results in a call to nstat_udp_new_pcb() before this watcher + // is instantiated and yet the socket doesn't make it into ipi_listhead + // prior to the scan. - OSIncrementAtomic(&nstat_udp_watchers); + errno_t result; lck_rw_lock_shared(udbinfo.ipi_lock); - // Add all current UDP inpcbs. - LIST_FOREACH(inp, udbinfo.ipi_listhead, inp_list) - { - cookie = nstat_tucookie_alloc_ref(inp); - if (cookie == NULL) - continue; - if (nstat_control_source_add(0, state, &nstat_udp_provider, - cookie) != 0) + result = nstat_set_provider_filter(state, req); + + if (result == 0) { + struct inpcb *inp; + struct nstat_tucookie *cookie; + + OSIncrementAtomic(&nstat_udp_watchers); + + // Add all current UDP inpcbs. + LIST_FOREACH(inp, udbinfo.ipi_listhead, inp_list) { - nstat_tucookie_release(cookie); - break; + cookie = nstat_tucookie_alloc_ref(inp); + if (cookie == NULL) + continue; + if (nstat_control_source_add(0, state, &nstat_udp_provider, + cookie) != 0) + { + nstat_tucookie_release(cookie); + break; + } } } lck_rw_done(udbinfo.ipi_lock); - return 0; + return result; } static void @@ -1707,6 +1813,8 @@ nstat_udp_new_pcb( { struct nstat_tucookie *cookie; + inp->inp_start_timestamp = mach_continuous_time(); + if (nstat_udp_watchers == 0) return; @@ -1759,9 +1867,9 @@ nstat_udp_copy_descriptor( if (tucookie->cached == false) { if (inp->inp_vflag & INP_IPV6) { - nstat_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, + in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, &desc->local.v6, sizeof(desc->local.v6)); - nstat_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, + in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, &desc->remote.v6, sizeof(desc->remote.v6)); } else if (inp->inp_vflag & INP_IPV4) @@ -1830,6 +1938,9 @@ nstat_udp_copy_descriptor( desc->rcvbufsize = so->so_rcv.sb_hiwat; desc->rcvbufused = so->so_rcv.sb_cc; desc->traffic_class = so->so_traffic_class; + inp_get_activity_bitmap(inp, &desc->activity_bitmap); + desc->start_timestamp = inp->inp_start_timestamp; + desc->timestamp = mach_continuous_time(); } return 0; @@ -1850,435 +1961,18 @@ nstat_init_udp_provider(void) nstat_udp_provider.nstat_descriptor_length = sizeof(nstat_udp_descriptor); nstat_udp_provider.nstat_lookup = nstat_udp_lookup; nstat_udp_provider.nstat_gone = nstat_udp_gone; - nstat_udp_provider.nstat_counts = nstat_udp_counts; - nstat_udp_provider.nstat_watcher_add = nstat_udp_add_watcher; - nstat_udp_provider.nstat_watcher_remove = nstat_udp_remove_watcher; - nstat_udp_provider.nstat_copy_descriptor = nstat_udp_copy_descriptor; - nstat_udp_provider.nstat_release = nstat_udp_release; - nstat_udp_provider.nstat_reporting_allowed = nstat_udp_reporting_allowed; - nstat_udp_provider.next = nstat_providers; - nstat_providers = &nstat_udp_provider; -} - -#pragma mark -- TCP/UDP Userland - -// Almost all of this infrastucture is common to both TCP and UDP - -static nstat_provider nstat_userland_tcp_provider; -static nstat_provider nstat_userland_udp_provider; - - -struct nstat_tu_shadow { - tailq_entry_tu_shadow shad_link; - userland_stats_request_vals_fn *shad_getvals_fn; - userland_stats_provider_context *shad_provider_context; - u_int64_t shad_properties; - int shad_provider; - uint32_t shad_magic; -}; - -// Magic number checking should remain in place until the userland provider has been fully proven -#define TU_SHADOW_MAGIC 0xfeedf00d -#define TU_SHADOW_UNMAGIC 0xdeaddeed - -static tailq_head_tu_shadow nstat_userprot_shad_head = TAILQ_HEAD_INITIALIZER(nstat_userprot_shad_head); - -static errno_t -nstat_userland_tu_lookup( - __unused const void *data, - __unused u_int32_t length, - __unused nstat_provider_cookie_t *out_cookie) -{ - // Looking up a specific connection is not supported - return ENOTSUP; -} - -static int -nstat_userland_tu_gone( - __unused nstat_provider_cookie_t cookie) -{ - // Returns non-zero if the source has gone. - // We don't keep a source hanging around, so the answer is always 0 - return 0; -} - -static errno_t -nstat_userland_tu_counts( - nstat_provider_cookie_t cookie, - struct nstat_counts *out_counts, - int *out_gone) - { - struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, out_counts, NULL); - - if (out_gone) *out_gone = 0; - - return (result)? 0 : EIO; -} - - -static errno_t -nstat_userland_tu_copy_descriptor( - nstat_provider_cookie_t cookie, - void *data, - __unused u_int32_t len) -{ - struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, data); - - return (result)? 0 : EIO; -} - -static void -nstat_userland_tu_release( - __unused nstat_provider_cookie_t cookie, - __unused int locked) -{ - // Called when a nstat_src is detached. - // We don't reference count or ask for delayed release so nothing to do here. -} - -static bool -check_reporting_for_user(nstat_provider_filter *filter, pid_t pid, pid_t epid, uuid_t *uuid, uuid_t *euuid) -{ - bool retval = true; - - if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) - { - retval = false; - - if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_PID) != 0) && - (filter->npf_pid == pid)) - { - retval = true; - } - else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EPID) != 0) && - (filter->npf_pid == epid)) - { - retval = true; - } - else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_UUID) != 0) && - (memcmp(filter->npf_uuid, uuid, sizeof(*uuid)) == 0)) - { - retval = true; - } - else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EUUID) != 0) && - (memcmp(filter->npf_uuid, euuid, sizeof(*euuid)) == 0)) - { - retval = true; - } - } - return retval; -} - -static bool -nstat_userland_tcp_reporting_allowed(nstat_provider_cookie_t cookie, nstat_provider_filter *filter) -{ - bool retval = true; - - if ((filter->npf_flags & (NSTAT_FILTER_IFNET_FLAGS|NSTAT_FILTER_SPECIFIC_USER)) != 0) - { - nstat_tcp_descriptor tcp_desc; // Stack allocation - OK or pushing the limits too far? - struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; - - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - if ((*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, &tcp_desc)) - { - if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) - { - if ((filter->npf_flags & tcp_desc.ifnet_properties) == 0) - { - return false; - } - } - if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) - { - retval = check_reporting_for_user(filter, (pid_t)tcp_desc.pid, (pid_t)tcp_desc.epid, - &tcp_desc.uuid, &tcp_desc.euuid); - } - } - else - { - retval = false; // No further information, so might as well give up now. - } - } - return retval; -} - -static bool -nstat_userland_udp_reporting_allowed(nstat_provider_cookie_t cookie, nstat_provider_filter *filter) -{ - bool retval = true; - - if ((filter->npf_flags & (NSTAT_FILTER_IFNET_FLAGS|NSTAT_FILTER_SPECIFIC_USER)) != 0) - { - nstat_udp_descriptor udp_desc; // Stack allocation - OK or pushing the limits too far? - struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; - - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - if ((*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, &udp_desc)) - { - if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) - { - if ((filter->npf_flags & udp_desc.ifnet_properties) == 0) - { - return false; - } - } - if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) - { - retval = check_reporting_for_user(filter, (pid_t)udp_desc.pid, (pid_t)udp_desc.epid, - &udp_desc.uuid, &udp_desc.euuid); - } - } - else - { - retval = false; // No further information, so might as well give up now. - } - } - return retval; -} - - - -static errno_t -nstat_userland_tcp_add_watcher( - nstat_control_state *state) -{ - struct nstat_tu_shadow *shad; - - OSIncrementAtomic(&nstat_userland_tcp_watchers); - - lck_mtx_lock(&nstat_mtx); - - TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) { - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - if (shad->shad_provider == NSTAT_PROVIDER_TCP_USERLAND) - { - int result = nstat_control_source_add(0, state, &nstat_userland_tcp_provider, shad); - if (result != 0) - { - printf("%s - nstat_control_source_add returned %d\n", __func__, result); - } - } - } - lck_mtx_unlock(&nstat_mtx); - - return 0; -} - -static errno_t -nstat_userland_udp_add_watcher( - nstat_control_state *state) -{ - struct nstat_tu_shadow *shad; - - OSIncrementAtomic(&nstat_userland_udp_watchers); - - lck_mtx_lock(&nstat_mtx); - - TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) { - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - if (shad->shad_provider == NSTAT_PROVIDER_UDP_USERLAND) - { - int result = nstat_control_source_add(0, state, &nstat_userland_udp_provider, shad); - if (result != 0) - { - printf("%s - nstat_control_source_add returned %d\n", __func__, result); - } - } - } - lck_mtx_unlock(&nstat_mtx); - - return 0; -} - - -static void -nstat_userland_tcp_remove_watcher( - __unused nstat_control_state *state) -{ - OSDecrementAtomic(&nstat_userland_tcp_watchers); -} - -static void -nstat_userland_udp_remove_watcher( - __unused nstat_control_state *state) -{ - OSDecrementAtomic(&nstat_userland_udp_watchers); -} - -static void -nstat_init_userland_tcp_provider(void) -{ - bzero(&nstat_userland_tcp_provider, sizeof(nstat_tcp_provider)); - nstat_userland_tcp_provider.nstat_descriptor_length = sizeof(nstat_tcp_descriptor); - nstat_userland_tcp_provider.nstat_provider_id = NSTAT_PROVIDER_TCP_USERLAND; - nstat_userland_tcp_provider.nstat_lookup = nstat_userland_tu_lookup; - nstat_userland_tcp_provider.nstat_gone = nstat_userland_tu_gone; - nstat_userland_tcp_provider.nstat_counts = nstat_userland_tu_counts; - nstat_userland_tcp_provider.nstat_release = nstat_userland_tu_release; - nstat_userland_tcp_provider.nstat_watcher_add = nstat_userland_tcp_add_watcher; - nstat_userland_tcp_provider.nstat_watcher_remove = nstat_userland_tcp_remove_watcher; - nstat_userland_tcp_provider.nstat_copy_descriptor = nstat_userland_tu_copy_descriptor; - nstat_userland_tcp_provider.nstat_reporting_allowed = nstat_userland_tcp_reporting_allowed; - nstat_userland_tcp_provider.next = nstat_providers; - nstat_providers = &nstat_userland_tcp_provider; -} - - -static void -nstat_init_userland_udp_provider(void) -{ - bzero(&nstat_userland_udp_provider, sizeof(nstat_udp_provider)); - nstat_userland_udp_provider.nstat_descriptor_length = sizeof(nstat_udp_descriptor); - nstat_userland_udp_provider.nstat_provider_id = NSTAT_PROVIDER_UDP_USERLAND; - nstat_userland_udp_provider.nstat_lookup = nstat_userland_tu_lookup; - nstat_userland_udp_provider.nstat_gone = nstat_userland_tu_gone; - nstat_userland_udp_provider.nstat_counts = nstat_userland_tu_counts; - nstat_userland_udp_provider.nstat_release = nstat_userland_tu_release; - nstat_userland_udp_provider.nstat_watcher_add = nstat_userland_udp_add_watcher; - nstat_userland_udp_provider.nstat_watcher_remove = nstat_userland_udp_remove_watcher; - nstat_userland_udp_provider.nstat_copy_descriptor = nstat_userland_tu_copy_descriptor; - nstat_userland_udp_provider.nstat_reporting_allowed = nstat_userland_udp_reporting_allowed; - nstat_userland_udp_provider.next = nstat_providers; - nstat_providers = &nstat_userland_udp_provider; -} - - - -// Things get started with a call to netstats to say that there’s a new connection: -__private_extern__ nstat_userland_context -ntstat_userland_stats_open(userland_stats_provider_context *ctx, - int provider_id, - u_int64_t properties, - userland_stats_request_vals_fn req_fn) -{ - struct nstat_tu_shadow *shad; - - if ((provider_id != NSTAT_PROVIDER_TCP_USERLAND) && (provider_id != NSTAT_PROVIDER_UDP_USERLAND)) - { - printf("%s - incorrect provider is supplied, %d\n", __func__, provider_id); - return NULL; - } - - shad = OSMalloc(sizeof(*shad), nstat_malloc_tag); - if (shad == NULL) - return NULL; - - shad->shad_getvals_fn = req_fn; - shad->shad_provider_context = ctx; - shad->shad_provider = provider_id; - shad->shad_properties = properties; - shad->shad_magic = TU_SHADOW_MAGIC; - - lck_mtx_lock(&nstat_mtx); - nstat_control_state *state; - - // Even if there are no watchers, we save the shadow structure - TAILQ_INSERT_HEAD(&nstat_userprot_shad_head, shad, shad_link); - - for (state = nstat_controls; state; state = state->ncs_next) - { - if ((state->ncs_watching & (1 << provider_id)) != 0) - { - // this client is watching tcp/udp userland - // Link to it. - int result = nstat_control_source_add(0, state, &nstat_userland_tcp_provider, shad); - if (result != 0) - { - printf("%s - nstat_control_source_add returned %d\n", __func__, result); - } - } - } - lck_mtx_unlock(&nstat_mtx); - - return (nstat_userland_context)shad; -} - - -__private_extern__ void -ntstat_userland_stats_close(nstat_userland_context nstat_ctx) -{ - struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)nstat_ctx; - nstat_src *dead_list = NULL; - - if (shad == NULL) - return; - - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - lck_mtx_lock(&nstat_mtx); - if (nstat_userland_udp_watchers != 0 || nstat_userland_tcp_watchers != 0) - { - nstat_control_state *state; - nstat_src *src, *prevsrc; - errno_t result; - - for (state = nstat_controls; state; state = state->ncs_next) - { - lck_mtx_lock(&state->mtx); - for (prevsrc = NULL, src = state->ncs_srcs; src; - prevsrc = src, src = src->next) - { - if (shad == (struct nstat_tu_shadow *)src->cookie) - break; - } - - if (src) - { - result = nstat_control_send_goodbye(state, src); - - if (prevsrc) - prevsrc->next = src->next; - else - state->ncs_srcs = src->next; - - src->next = dead_list; - dead_list = src; - } - lck_mtx_unlock(&state->mtx); - } - } - TAILQ_REMOVE(&nstat_userprot_shad_head, shad, shad_link); - - lck_mtx_unlock(&nstat_mtx); - - while (dead_list) - { - nstat_src *src; - src = dead_list; - dead_list = src->next; - - nstat_control_cleanup_source(NULL, src, TRUE); - } - - shad->shad_magic = TU_SHADOW_UNMAGIC; - - OSFree(shad, sizeof(*shad), nstat_malloc_tag); -} - - -__private_extern__ void -ntstat_userland_stats_event( - __unused nstat_userland_context context, - __unused userland_stats_event_t event) -{ - // This is a dummy for when we hook up event reporting to NetworkStatistics. - // See NetworkStatistics should provide opt-in notifications + nstat_udp_provider.nstat_counts = nstat_udp_counts; + nstat_udp_provider.nstat_watcher_add = nstat_udp_add_watcher; + nstat_udp_provider.nstat_watcher_remove = nstat_udp_remove_watcher; + nstat_udp_provider.nstat_copy_descriptor = nstat_udp_copy_descriptor; + nstat_udp_provider.nstat_release = nstat_udp_release; + nstat_udp_provider.nstat_reporting_allowed = nstat_udp_reporting_allowed; + nstat_udp_provider.next = nstat_providers; + nstat_providers = &nstat_udp_provider; } - #pragma mark -- ifnet Provider -- static nstat_provider nstat_ifnet_provider; @@ -2353,14 +2047,14 @@ nstat_ifnet_lookup( lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { if (src->provider != &nstat_ifnet_provider) continue; nstat_control_send_description(state, src, 0, 0); } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); } @@ -2435,8 +2129,8 @@ nstat_ifnet_release( lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { /* Skip the provider we are about to detach. */ if (src->provider != &nstat_ifnet_provider || @@ -2446,7 +2140,7 @@ nstat_ifnet_release( if (ifcookie->threshold < minthreshold) minthreshold = ifcookie->threshold; } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); /* @@ -2778,8 +2472,7 @@ nstat_ifnet_report_ecn_stats(void) if (ifp->if_ipv4_stat == NULL || ifp->if_ipv6_stat == NULL) continue; - if ((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) != - IFRF_ATTACHED) + if (!IF_FULLY_ATTACHED(ifp)) continue; /* Limit reporting to Wifi, Ethernet and cellular. */ @@ -2831,6 +2524,133 @@ v6: } +/* Some thresholds to determine Low Iternet mode */ +#define NSTAT_LIM_DL_MAX_BANDWIDTH_THRESHOLD 1000000 /* 1 Mbps */ +#define NSTAT_LIM_UL_MAX_BANDWIDTH_THRESHOLD 500000 /* 500 Kbps */ +#define NSTAT_LIM_UL_MIN_RTT_THRESHOLD 1000 /* 1 second */ +#define NSTAT_LIM_CONN_TIMEOUT_PERCENT_THRESHOLD (10 << 10) /* 10 percent connection timeouts */ +#define NSTAT_LIM_PACKET_LOSS_PERCENT_THRESHOLD (2 << 10) /* 2 percent packet loss rate */ + +static boolean_t +nstat_lim_activity_check(struct if_lim_perf_stat *st) +{ + /* check that the current activity is enough to report stats */ + if (st->lim_total_txpkts < nstat_lim_min_tx_pkts || + st->lim_total_rxpkts < nstat_lim_min_rx_pkts || + st->lim_conn_attempts == 0) + return (FALSE); + + /* + * Compute percentages if there was enough activity. Use + * shift-left by 10 to preserve precision. + */ + st->lim_packet_loss_percent = ((st->lim_total_retxpkts << 10) / + st->lim_total_txpkts) * 100; + + st->lim_packet_ooo_percent = ((st->lim_total_oopkts << 10) / + st->lim_total_rxpkts) * 100; + + st->lim_conn_timeout_percent = ((st->lim_conn_timeouts << 10) / + st->lim_conn_attempts) * 100; + + /* + * Is Low Internet detected? First order metrics are bandwidth + * and RTT. If these metrics are below the minimum thresholds + * defined then the network attachment can be classified as + * having Low Internet capacity. + * + * High connection timeout rate also indicates Low Internet + * capacity. + */ + if (st->lim_dl_max_bandwidth > 0 && + st->lim_dl_max_bandwidth <= NSTAT_LIM_DL_MAX_BANDWIDTH_THRESHOLD) + st->lim_dl_detected = 1; + + if ((st->lim_ul_max_bandwidth > 0 && + st->lim_ul_max_bandwidth <= NSTAT_LIM_UL_MAX_BANDWIDTH_THRESHOLD) || + st->lim_rtt_min >= NSTAT_LIM_UL_MIN_RTT_THRESHOLD) + st->lim_ul_detected = 1; + + if (st->lim_conn_attempts > 20 && + st->lim_conn_timeout_percent >= + NSTAT_LIM_CONN_TIMEOUT_PERCENT_THRESHOLD) + st->lim_ul_detected = 1; + /* + * Second order metrics: If there was high packet loss even after + * using delay based algorithms then we classify it as Low Internet + * again + */ + if (st->lim_bk_txpkts >= nstat_lim_min_tx_pkts && + st->lim_packet_loss_percent >= + NSTAT_LIM_PACKET_LOSS_PERCENT_THRESHOLD) + st->lim_ul_detected = 1; + return (TRUE); +} + +static u_int64_t nstat_lim_last_report_time = 0; +static void +nstat_ifnet_report_lim_stats(void) +{ + u_int64_t uptime; + struct nstat_sysinfo_data data; + struct nstat_sysinfo_lim_stats *st; + struct ifnet *ifp; + int err; + + uptime = net_uptime(); + + if ((u_int32_t)(uptime - nstat_lim_last_report_time) < + nstat_lim_interval) + return; + + nstat_lim_last_report_time = uptime; + data.flags = NSTAT_SYSINFO_LIM_STATS; + st = &data.u.lim_stats; + data.unsent_data_cnt = 0; + + ifnet_head_lock_shared(); + TAILQ_FOREACH(ifp, &ifnet_head, if_link) { + if (!IF_FULLY_ATTACHED(ifp)) + continue; + + /* Limit reporting to Wifi, Ethernet and cellular */ + if (!(IFNET_IS_ETHERNET(ifp) || IFNET_IS_CELLULAR(ifp))) + continue; + + if (!nstat_lim_activity_check(&ifp->if_lim_stat)) + continue; + + bzero(st, sizeof(*st)); + st->ifnet_siglen = sizeof (st->ifnet_signature); + err = ifnet_get_netsignature(ifp, AF_INET, + (u_int8_t *)&st->ifnet_siglen, NULL, + st->ifnet_signature); + if (err != 0) { + err = ifnet_get_netsignature(ifp, AF_INET6, + (u_int8_t *)&st->ifnet_siglen, NULL, + st->ifnet_signature); + if (err != 0) + continue; + } + ifnet_lock_shared(ifp); + if (IFNET_IS_CELLULAR(ifp)) { + st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_CELLULAR; + } else if (IFNET_IS_WIFI(ifp)) { + st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_WIFI; + } else { + st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_ETHERNET; + } + bcopy(&ifp->if_lim_stat, &st->lim_stat, + sizeof(st->lim_stat)); + + /* Zero the stats in ifp */ + bzero(&ifp->if_lim_stat, sizeof(ifp->if_lim_stat)); + ifnet_lock_done(ifp); + nstat_sysinfo_send_data(&data); + } + ifnet_head_done(); +} + static errno_t nstat_ifnet_copy_descriptor( nstat_provider_cookie_t cookie, @@ -2890,8 +2710,8 @@ nstat_ifnet_threshold_reached(unsigned int ifindex) lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { if (src->provider != &nstat_ifnet_provider) continue; @@ -2901,7 +2721,7 @@ nstat_ifnet_threshold_reached(unsigned int ifindex) continue; nstat_control_send_counts(state, src, 0, 0, NULL); } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); } @@ -2913,6 +2733,18 @@ nstat_set_keyval_scalar(nstat_sysinfo_keyval *kv, int key, u_int32_t val) kv->nstat_sysinfo_key = key; kv->nstat_sysinfo_flags = NSTAT_SYSINFO_FLAG_SCALAR; kv->u.nstat_sysinfo_scalar = val; + kv->nstat_sysinfo_valsize = sizeof(kv->u.nstat_sysinfo_scalar); +} + +static void +nstat_set_keyval_string(nstat_sysinfo_keyval *kv, int key, u_int8_t *buf, + u_int32_t len) +{ + kv->nstat_sysinfo_key = key; + kv->nstat_sysinfo_flags = NSTAT_SYSINFO_FLAG_STRING; + kv->nstat_sysinfo_valsize = min(len, + NSTAT_SYSINFO_KEYVAL_STRING_MAXSIZE); + bcopy(buf, kv->u.nstat_sysinfo_string, kv->nstat_sysinfo_valsize); } static void @@ -2938,8 +2770,7 @@ nstat_sysinfo_send_data_internal( sizeof(u_int32_t); break; case NSTAT_SYSINFO_TCP_STATS: - nkeyvals = sizeof(struct nstat_sysinfo_tcp_stats) / - sizeof(u_int32_t); + nkeyvals = NSTAT_SYSINFO_TCP_STATS_COUNT; break; case NSTAT_SYSINFO_IFNET_ECN_STATS: nkeyvals = (sizeof(struct if_tcp_ecn_stat) / @@ -2951,6 +2782,12 @@ nstat_sysinfo_send_data_internal( /* One key for unsent data. */ nkeyvals++; break; + case NSTAT_SYSINFO_LIM_STATS: + nkeyvals = NSTAT_LIM_STAT_KEYVAL_COUNT; + break; + case NSTAT_SYSINFO_NET_API_STATS: + nkeyvals = NSTAT_NET_API_STAT_KEYVAL_COUNT; + break; default: return; } @@ -3125,6 +2962,90 @@ nstat_sysinfo_send_data_internal( nstat_set_keyval_scalar(&kv[i++], NSTAT_SYSINFO_TFO_SEND_BLACKHOLE, data->u.tcp_stats.tfo_sndblackhole); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_ATTEMPT, + data->u.tcp_stats.mptcp_handover_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_INTERACTIVE_ATTEMPT, + data->u.tcp_stats.mptcp_interactive_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_AGGREGATE_ATTEMPT, + data->u.tcp_stats.mptcp_aggregate_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_HANDOVER_ATTEMPT, + data->u.tcp_stats.mptcp_fp_handover_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_ATTEMPT, + data->u.tcp_stats.mptcp_fp_interactive_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_ATTEMPT, + data->u.tcp_stats.mptcp_fp_aggregate_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HEURISTIC_FALLBACK, + data->u.tcp_stats.mptcp_heuristic_fallback); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_HEURISTIC_FALLBACK, + data->u.tcp_stats.mptcp_fp_heuristic_fallback); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_WIFI, + data->u.tcp_stats.mptcp_handover_success_wifi); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_CELL, + data->u.tcp_stats.mptcp_handover_success_cell); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_INTERACTIVE_SUCCESS, + data->u.tcp_stats.mptcp_interactive_success); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_AGGREGATE_SUCCESS, + data->u.tcp_stats.mptcp_aggregate_success); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_WIFI, + data->u.tcp_stats.mptcp_fp_handover_success_wifi); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_CELL, + data->u.tcp_stats.mptcp_fp_handover_success_cell); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_SUCCESS, + data->u.tcp_stats.mptcp_fp_interactive_success); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_SUCCESS, + data->u.tcp_stats.mptcp_fp_aggregate_success); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_FROM_WIFI, + data->u.tcp_stats.mptcp_handover_cell_from_wifi); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_WIFI_FROM_CELL, + data->u.tcp_stats.mptcp_handover_wifi_from_cell); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_FROM_WIFI, + data->u.tcp_stats.mptcp_interactive_cell_from_wifi); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_BYTES, + data->u.tcp_stats.mptcp_handover_cell_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_BYTES, + data->u.tcp_stats.mptcp_interactive_cell_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_AGGREGATE_CELL_BYTES, + data->u.tcp_stats.mptcp_aggregate_cell_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_ALL_BYTES, + data->u.tcp_stats.mptcp_handover_all_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_INTERACTIVE_ALL_BYTES, + data->u.tcp_stats.mptcp_interactive_all_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_AGGREGATE_ALL_BYTES, + data->u.tcp_stats.mptcp_aggregate_all_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_BACK_TO_WIFI, + data->u.tcp_stats.mptcp_back_to_wifi); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_WIFI_PROXY, + data->u.tcp_stats.mptcp_wifi_proxy); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_CELL_PROXY, + data->u.tcp_stats.mptcp_cell_proxy); VERIFY(i == nkeyvals); break; } @@ -3271,6 +3192,191 @@ nstat_sysinfo_send_data_internal( nstat_set_keyval_scalar(&kv[i++], NSTAT_SYSINFO_ECN_IFNET_FALLBACK_DROPRXMT, data->u.ifnet_ecn_stats.ecn_stat.ecn_fallback_droprxmt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_ECN_IFNET_FALLBACK_SYNRST, + data->u.ifnet_ecn_stats.ecn_stat.ecn_fallback_synrst); + break; + } + case NSTAT_SYSINFO_LIM_STATS: + { + nstat_set_keyval_string(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_SIGNATURE, + data->u.lim_stats.ifnet_signature, + data->u.lim_stats.ifnet_siglen); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_DL_MAX_BANDWIDTH, + data->u.lim_stats.lim_stat.lim_dl_max_bandwidth); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_UL_MAX_BANDWIDTH, + data->u.lim_stats.lim_stat.lim_ul_max_bandwidth); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_PACKET_LOSS_PERCENT, + data->u.lim_stats.lim_stat.lim_packet_loss_percent); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_PACKET_OOO_PERCENT, + data->u.lim_stats.lim_stat.lim_packet_ooo_percent); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_RTT_VARIANCE, + data->u.lim_stats.lim_stat.lim_rtt_variance); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_RTT_MIN, + data->u.lim_stats.lim_stat.lim_rtt_min); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_RTT_AVG, + data->u.lim_stats.lim_stat.lim_rtt_average); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_CONN_TIMEOUT_PERCENT, + data->u.lim_stats.lim_stat.lim_conn_timeout_percent); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_DL_DETECTED, + data->u.lim_stats.lim_stat.lim_dl_detected); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_UL_DETECTED, + data->u.lim_stats.lim_stat.lim_ul_detected); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_TYPE, + data->u.lim_stats.ifnet_type); + break; + } + case NSTAT_SYSINFO_NET_API_STATS: + { + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IF_FLTR_ATTACH, + data->u.net_api_stats.net_api_stats.nas_iflt_attach_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IF_FLTR_ATTACH_OS, + data->u.net_api_stats.net_api_stats.nas_iflt_attach_os_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IP_FLTR_ADD, + data->u.net_api_stats.net_api_stats.nas_ipf_add_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IP_FLTR_ADD_OS, + data->u.net_api_stats.net_api_stats.nas_ipf_add_os_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH, + data->u.net_api_stats.net_api_stats.nas_sfltr_register_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH_OS, + data->u.net_api_stats.net_api_stats.nas_sfltr_register_os_total); + + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_ALLOC_TOTAL, + data->u.net_api_stats.net_api_stats.nas_socket_alloc_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL, + data->u.net_api_stats.net_api_stats.nas_socket_in_kernel_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL_OS, + data->u.net_api_stats.net_api_stats.nas_socket_in_kernel_os_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_NECP_CLIENTUUID, + data->u.net_api_stats.net_api_stats.nas_socket_necp_clientuuid_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_LOCAL, + data->u.net_api_stats.net_api_stats.nas_socket_domain_local_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_ROUTE, + data->u.net_api_stats.net_api_stats.nas_socket_domain_route_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_INET, + data->u.net_api_stats.net_api_stats.nas_socket_domain_inet_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_INET6, + data->u.net_api_stats.net_api_stats.nas_socket_domain_inet6_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_SYSTEM, + data->u.net_api_stats.net_api_stats.nas_socket_domain_system_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_MULTIPATH, + data->u.net_api_stats.net_api_stats.nas_socket_domain_multipath_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_KEY, + data->u.net_api_stats.net_api_stats.nas_socket_domain_key_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_NDRV, + data->u.net_api_stats.net_api_stats.nas_socket_domain_ndrv_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_OTHER, + data->u.net_api_stats.net_api_stats.nas_socket_domain_other_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_STREAM, + data->u.net_api_stats.net_api_stats.nas_socket_inet_stream_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_DGRAM, + data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_DGRAM_CONNECTED, + data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_connected); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_DGRAM_DNS, + data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_dns); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_DGRAM_NO_DATA, + data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_no_data); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET6_STREAM, + data->u.net_api_stats.net_api_stats.nas_socket_inet6_stream_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET6_DGRAM, + data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_CONNECTED, + data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_connected); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_DNS, + data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_dns); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_NO_DATA, + data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_no_data); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN, + data->u.net_api_stats.net_api_stats.nas_socket_mcast_join_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN_OS, + data->u.net_api_stats.net_api_stats.nas_socket_mcast_join_os_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_NEXUS_FLOW_INET_STREAM, + data->u.net_api_stats.net_api_stats.nas_nx_flow_inet_stream_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_NEXUS_FLOW_INET_DATAGRAM, + data->u.net_api_stats.net_api_stats.nas_nx_flow_inet_dgram_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_STREAM, + data->u.net_api_stats.net_api_stats.nas_nx_flow_inet6_stream_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_DATAGRAM, + data->u.net_api_stats.net_api_stats.nas_nx_flow_inet6_dgram_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IFNET_ALLOC, + data->u.net_api_stats.net_api_stats.nas_ifnet_alloc_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IFNET_ALLOC_OS, + data->u.net_api_stats.net_api_stats.nas_ifnet_alloc_os_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_PF_ADDRULE, + data->u.net_api_stats.net_api_stats.nas_pf_addrule_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_PF_ADDRULE_OS, + data->u.net_api_stats.net_api_stats.nas_pf_addrule_os); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_VMNET_START, + data->u.net_api_stats.net_api_stats.nas_vmnet_total); + + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_REPORT_INTERVAL, + data->u.net_api_stats.report_interval); + break; } } @@ -3303,14 +3409,12 @@ nstat_sysinfo_send_data( nstat_control_state *control; lck_mtx_lock(&nstat_mtx); - for (control = nstat_controls; control; control = control->ncs_next) - { - lck_mtx_lock(&control->mtx); - if ((control->ncs_flags & NSTAT_FLAG_SYSINFO_SUBSCRIBED) != 0) - { + for (control = nstat_controls; control; control = control->ncs_next) { + lck_mtx_lock(&control->ncs_mtx); + if ((control->ncs_flags & NSTAT_FLAG_SYSINFO_SUBSCRIBED) != 0) { nstat_sysinfo_send_data_internal(control, data); } - lck_mtx_unlock(&control->mtx); + lck_mtx_unlock(&control->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); } @@ -3321,8 +3425,124 @@ nstat_sysinfo_generate_report(void) mbuf_report_peak_usage(); tcp_report_stats(); nstat_ifnet_report_ecn_stats(); + nstat_ifnet_report_lim_stats(); + nstat_net_api_report_stats(); +} + +#pragma mark -- net_api -- + +static struct net_api_stats net_api_stats_before; +static u_int64_t net_api_stats_last_report_time; + +static void +nstat_net_api_report_stats(void) +{ + struct nstat_sysinfo_data data; + struct nstat_sysinfo_net_api_stats *st = &data.u.net_api_stats; + u_int64_t uptime; + + uptime = net_uptime(); + + if ((u_int32_t)(uptime - net_api_stats_last_report_time) < + net_api_stats_report_interval) + return; + + st->report_interval = uptime - net_api_stats_last_report_time; + net_api_stats_last_report_time = uptime; + + data.flags = NSTAT_SYSINFO_NET_API_STATS; + data.unsent_data_cnt = 0; + + /* + * Some of the fields in the report are the current value and + * other fields are the delta from the last report: + * - Report difference for the per flow counters as they increase + * with time + * - Report current value for other counters as they tend not to change + * much with time + */ +#define STATCOPY(f) \ + (st->net_api_stats.f = net_api_stats.f) +#define STATDIFF(f) \ + (st->net_api_stats.f = net_api_stats.f - net_api_stats_before.f) + + STATCOPY(nas_iflt_attach_count); + STATCOPY(nas_iflt_attach_total); + STATCOPY(nas_iflt_attach_os_total); + + STATCOPY(nas_ipf_add_count); + STATCOPY(nas_ipf_add_total); + STATCOPY(nas_ipf_add_os_total); + + STATCOPY(nas_sfltr_register_count); + STATCOPY(nas_sfltr_register_total); + STATCOPY(nas_sfltr_register_os_total); + + STATDIFF(nas_socket_alloc_total); + STATDIFF(nas_socket_in_kernel_total); + STATDIFF(nas_socket_in_kernel_os_total); + STATDIFF(nas_socket_necp_clientuuid_total); + + STATDIFF(nas_socket_domain_local_total); + STATDIFF(nas_socket_domain_route_total); + STATDIFF(nas_socket_domain_inet_total); + STATDIFF(nas_socket_domain_inet6_total); + STATDIFF(nas_socket_domain_system_total); + STATDIFF(nas_socket_domain_multipath_total); + STATDIFF(nas_socket_domain_key_total); + STATDIFF(nas_socket_domain_ndrv_total); + STATDIFF(nas_socket_domain_other_total); + + STATDIFF(nas_socket_inet_stream_total); + STATDIFF(nas_socket_inet_dgram_total); + STATDIFF(nas_socket_inet_dgram_connected); + STATDIFF(nas_socket_inet_dgram_dns); + STATDIFF(nas_socket_inet_dgram_no_data); + + STATDIFF(nas_socket_inet6_stream_total); + STATDIFF(nas_socket_inet6_dgram_total); + STATDIFF(nas_socket_inet6_dgram_connected); + STATDIFF(nas_socket_inet6_dgram_dns); + STATDIFF(nas_socket_inet6_dgram_no_data); + + STATDIFF(nas_socket_mcast_join_total); + STATDIFF(nas_socket_mcast_join_os_total); + + STATDIFF(nas_sock_inet6_stream_exthdr_in); + STATDIFF(nas_sock_inet6_stream_exthdr_out); + STATDIFF(nas_sock_inet6_dgram_exthdr_in); + STATDIFF(nas_sock_inet6_dgram_exthdr_out); + + STATDIFF(nas_nx_flow_inet_stream_total); + STATDIFF(nas_nx_flow_inet_dgram_total); + + STATDIFF(nas_nx_flow_inet6_stream_total); + STATDIFF(nas_nx_flow_inet6_dgram_total); + + STATCOPY(nas_ifnet_alloc_count); + STATCOPY(nas_ifnet_alloc_total); + STATCOPY(nas_ifnet_alloc_os_count); + STATCOPY(nas_ifnet_alloc_os_total); + + STATCOPY(nas_pf_addrule_total); + STATCOPY(nas_pf_addrule_os); + + STATCOPY(nas_vmnet_total); + +#undef STATCOPY +#undef STATDIFF + + nstat_sysinfo_send_data(&data); + + /* + * Save a copy of the current fields so we can diff them the next time + */ + memcpy(&net_api_stats_before, &net_api_stats, + sizeof(struct net_api_stats)); + _CASSERT(sizeof (net_api_stats_before) == sizeof (net_api_stats)); } + #pragma mark -- Kernel Control Socket -- static kern_ctl_ref nstat_ctlref = NULL; @@ -3491,44 +3711,38 @@ nstat_idle_check( __unused thread_call_param_t p0, __unused thread_call_param_t p1) { + nstat_control_state *control; + nstat_src *src, *tmpsrc; + tailq_head_nstat_src dead_list; + TAILQ_INIT(&dead_list); + lck_mtx_lock(&nstat_mtx); nstat_idle_time = 0; - nstat_control_state *control; - nstat_src *dead = NULL; - nstat_src *dead_list = NULL; for (control = nstat_controls; control; control = control->ncs_next) { - lck_mtx_lock(&control->mtx); - nstat_src **srcpp = &control->ncs_srcs; - + lck_mtx_lock(&control->ncs_mtx); if (!(control->ncs_flags & NSTAT_FLAG_REQCOUNTS)) { - while(*srcpp != NULL) + TAILQ_FOREACH_SAFE(src, &control->ncs_src_queue, ns_control_link, tmpsrc) { - if ((*srcpp)->provider->nstat_gone((*srcpp)->cookie)) + if (src->provider->nstat_gone(src->cookie)) { errno_t result; // Pull it off the list - dead = *srcpp; - *srcpp = (*srcpp)->next; + TAILQ_REMOVE(&control->ncs_src_queue, src, ns_control_link); - result = nstat_control_send_goodbye(control, dead); + result = nstat_control_send_goodbye(control, src); // Put this on the list to release later - dead->next = dead_list; - dead_list = dead; - } - else - { - srcpp = &(*srcpp)->next; + TAILQ_INSERT_TAIL(&dead_list, src, ns_control_link); } } } control->ncs_flags &= ~NSTAT_FLAG_REQCOUNTS; - lck_mtx_unlock(&control->mtx); + lck_mtx_unlock(&control->ncs_mtx); } if (nstat_controls) @@ -3543,14 +3757,13 @@ nstat_idle_check( nstat_sysinfo_generate_report(); // Release the sources now that we aren't holding lots of locks - while (dead_list) + while ((src = TAILQ_FIRST(&dead_list))) { - dead = dead_list; - dead_list = dead->next; - - nstat_control_cleanup_source(NULL, dead, FALSE); + TAILQ_REMOVE(&dead_list, src, ns_control_link); + nstat_control_cleanup_source(NULL, src, FALSE); } + return NULL; } @@ -3629,7 +3842,7 @@ nstat_control_connect( if (state == NULL) return ENOMEM; bzero(state, sizeof(*state)); - lck_mtx_init(&state->mtx, nstat_lck_grp, NULL); + lck_mtx_init(&state->ncs_mtx, nstat_lck_grp, NULL); state->ncs_kctl = kctl; state->ncs_unit = sac->sc_unit; state->ncs_flags = NSTAT_FLAG_REQCOUNTS; @@ -3658,6 +3871,10 @@ nstat_control_disconnect( { u_int32_t watching; nstat_control_state *state = (nstat_control_state*)uinfo; + tailq_head_nstat_src cleanup_list; + nstat_src *src; + + TAILQ_INIT(&cleanup_list); // pull it out of the global list of states lck_mtx_lock(&nstat_mtx); @@ -3672,7 +3889,7 @@ nstat_control_disconnect( } lck_mtx_unlock(&nstat_mtx); - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); // Stop watching for sources nstat_provider *provider; watching = state->ncs_watching; @@ -3696,22 +3913,16 @@ nstat_control_disconnect( } // Copy out the list of sources - nstat_src *srcs = state->ncs_srcs; - state->ncs_srcs = NULL; - lck_mtx_unlock(&state->mtx); + TAILQ_CONCAT(&cleanup_list, &state->ncs_src_queue, ns_control_link); + lck_mtx_unlock(&state->ncs_mtx); - while (srcs) + while ((src = TAILQ_FIRST(&cleanup_list))) { - nstat_src *src; - - // pull it out of the list - src = srcs; - srcs = src->next; - - // clean it up + TAILQ_REMOVE(&cleanup_list, src, ns_control_link); nstat_control_cleanup_source(NULL, src, FALSE); } - lck_mtx_destroy(&state->mtx, nstat_lck_grp); + + lck_mtx_destroy(&state->ncs_mtx, nstat_lck_grp); OSFree(state, sizeof(*state), nstat_malloc_tag); return 0; @@ -4071,8 +4282,8 @@ nstat_control_handle_add_request( return EINVAL; } - nstat_provider *provider; - nstat_provider_cookie_t cookie; + nstat_provider *provider = NULL; + nstat_provider_cookie_t cookie = NULL; nstat_msg_add_src_req *req = mbuf_data(m); if (mbuf_pkthdr_len(m) > mbuf_len(m)) { @@ -4102,6 +4313,26 @@ nstat_control_handle_add_request( return result; } +static errno_t +nstat_set_provider_filter( + nstat_control_state *state, + nstat_msg_add_all_srcs *req) +{ + nstat_provider_id_t provider_id = req->provider; + + u_int32_t prev_ncs_watching = atomic_or_32_ov(&state->ncs_watching, (1 << provider_id)); + + if ((prev_ncs_watching & (1 << provider_id)) != 0) + return EALREADY; + + state->ncs_watching |= (1 << provider_id); + state->ncs_provider_filters[provider_id].npf_flags = req->filter; + state->ncs_provider_filters[provider_id].npf_events = req->events; + state->ncs_provider_filters[provider_id].npf_pid = req->target_pid; + uuid_copy(state->ncs_provider_filters[provider_id].npf_uuid, req->target_uuid); + return 0; +} + static errno_t nstat_control_handle_add_all( nstat_control_state *state, @@ -4115,7 +4346,6 @@ nstat_control_handle_add_all( return EINVAL; } - nstat_msg_add_all_srcs *req = mbuf_data(m); if (req->provider > NSTAT_PROVIDER_LAST) return ENOENT; @@ -4131,33 +4361,21 @@ nstat_control_handle_add_all( return result; } - // Make sure we don't add the provider twice - lck_mtx_lock(&state->mtx); - if ((state->ncs_watching & (1 << provider->nstat_provider_id)) != 0) - result = EALREADY; - state->ncs_watching |= (1 << provider->nstat_provider_id); - lck_mtx_unlock(&state->mtx); - if (result != 0) return result; + lck_mtx_lock(&state->ncs_mtx); + if (req->filter & NSTAT_FILTER_SUPPRESS_SRC_ADDED) + { + // Suppression of source messages implicitly requires the use of update messages + state->ncs_flags |= NSTAT_FLAG_SUPPORTS_UPDATES; + } + lck_mtx_unlock(&state->ncs_mtx); - state->ncs_provider_filters[req->provider].npf_flags = req->filter; - state->ncs_provider_filters[req->provider].npf_events = req->events; - state->ncs_provider_filters[req->provider].npf_pid = req->target_pid; - memcpy(state->ncs_provider_filters[req->provider].npf_uuid, req->target_uuid, - sizeof(state->ncs_provider_filters[req->provider].npf_uuid)); + // rdar://problem/30301300 Different providers require different synchronization + // to ensure that a new entry does not get double counted due to being added prior + // to all current provider entries being added. Hence pass the provider the details + // in the original request for this to be applied atomically - result = provider->nstat_watcher_add(state); - if (result != 0) - { - state->ncs_provider_filters[req->provider].npf_flags = 0; - state->ncs_provider_filters[req->provider].npf_events = 0; - state->ncs_provider_filters[req->provider].npf_pid = 0; - bzero(state->ncs_provider_filters[req->provider].npf_uuid, - sizeof(state->ncs_provider_filters[req->provider].npf_uuid)); + result = provider->nstat_watcher_add(state, req); - lck_mtx_lock(&state->mtx); - state->ncs_watching &= ~(1 << provider->nstat_provider_id); - lck_mtx_unlock(&state->mtx); - } if (result == 0) nstat_enqueue_success(req->hdr.context, state, 0); @@ -4183,6 +4401,11 @@ nstat_control_source_add( (provider_filter_flagss & NSTAT_FILTER_PROVIDER_NOZEROBYTES) ? NSTAT_FILTER_NOZEROBYTES : 0; + if (provider_filter_flagss & NSTAT_FILTER_TCP_NO_EARLY_CLOSE) + { + src_filter |= NSTAT_FILTER_TCP_NO_EARLY_CLOSE; + } + if (tell_user) { unsigned int one = 1; @@ -4211,7 +4434,7 @@ nstat_control_source_add( } // Fill in the source, including picking an unused source ref - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); src->srcref = nstat_control_next_src_ref(state); if (srcrefp) @@ -4219,7 +4442,7 @@ nstat_control_source_add( if (state->ncs_flags & NSTAT_FLAG_CLEANUP || src->srcref == NSTAT_SRC_REF_INVALID) { - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); OSFree(src, sizeof(*src), nstat_malloc_tag); if (msg) mbuf_freem(msg); return EINVAL; @@ -4237,17 +4460,17 @@ nstat_control_source_add( if (result != 0) { nstat_stats.nstat_srcaddedfailures += 1; - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); OSFree(src, sizeof(*src), nstat_malloc_tag); mbuf_freem(msg); return result; } } // Put the source in the list - src->next = state->ncs_srcs; - state->ncs_srcs = src; + TAILQ_INSERT_HEAD(&state->ncs_src_queue, src, ns_control_link); + src->ns_control = state; - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); return 0; } @@ -4257,29 +4480,30 @@ nstat_control_handle_remove_request( nstat_control_state *state, mbuf_t m) { - nstat_src_ref_t srcref = NSTAT_SRC_REF_INVALID; + nstat_src_ref_t srcref = NSTAT_SRC_REF_INVALID; + nstat_src *src; if (mbuf_copydata(m, offsetof(nstat_msg_rem_src_req, srcref), sizeof(srcref), &srcref) != 0) { return EINVAL; } - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); // Remove this source as we look for it - nstat_src **nextp; - nstat_src *src = NULL; - for (nextp = &state->ncs_srcs; *nextp; nextp = &(*nextp)->next) + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { - if ((*nextp)->srcref == srcref) + if (src->srcref == srcref) { - src = *nextp; - *nextp = src->next; break; } } + if (src) + { + TAILQ_REMOVE(&state->ncs_src_queue, src, ns_control_link); + } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); if (src) nstat_control_cleanup_source(state, src, FALSE); @@ -4299,7 +4523,7 @@ nstat_control_handle_query_request( // using this socket, one for read and one for write. Two threads probably // won't work with this code anyhow since we don't have proper locking in // place yet. - nstat_src *dead_srcs = NULL; + tailq_head_nstat_src dead_list; errno_t result = ENOENT; nstat_msg_query_src_req req; @@ -4309,14 +4533,15 @@ nstat_control_handle_query_request( } const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL); + TAILQ_INIT(&dead_list); - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); if (all_srcs) { state->ncs_flags |= NSTAT_FLAG_REQCOUNTS; } - nstat_src **srcpp = &state->ncs_srcs; + nstat_src *src, *tmpsrc; u_int64_t src_count = 0; boolean_t partial = FALSE; @@ -4326,14 +4551,11 @@ nstat_control_handle_query_request( */ partial = nstat_control_begin_query(state, &req.hdr); - while (*srcpp != NULL - && (!partial || src_count < QUERY_CONTINUATION_SRC_COUNT)) + + TAILQ_FOREACH_SAFE(src, &state->ncs_src_queue, ns_control_link, tmpsrc) { - nstat_src *src = NULL; - int gone; + int gone = 0; - src = *srcpp; - gone = 0; // XXX ignore IFACE types? if (all_srcs || src->srcref == req.srcref) { @@ -4380,7 +4602,7 @@ nstat_control_handle_query_request( // send one last descriptor message so client may see last state // If we can't send the notification now, it // will be sent in the idle cleanup. - result = nstat_control_send_description(state, *srcpp, 0, 0); + result = nstat_control_send_description(state, src, 0, 0); if (result != 0) { nstat_stats.nstat_control_send_description_failures++; @@ -4391,28 +4613,30 @@ nstat_control_handle_query_request( } // pull src out of the list - *srcpp = src->next; - - src->next = dead_srcs; - dead_srcs = src; + TAILQ_REMOVE(&state->ncs_src_queue, src, ns_control_link); + TAILQ_INSERT_TAIL(&dead_list, src, ns_control_link); } - else + + if (all_srcs) { - srcpp = &(*srcpp)->next; + if (src_count >= QUERY_CONTINUATION_SRC_COUNT) + { + break; + } } - - if (!all_srcs && req.srcref == src->srcref) + else if (req.srcref == src->srcref) { break; } } + nstat_flush_accumulated_msgs(state); u_int16_t flags = 0; if (req.srcref == NSTAT_SRC_REF_ALL) - flags = nstat_control_end_query(state, *srcpp, partial); + flags = nstat_control_end_query(state, src, partial); - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); /* * If an error occurred enqueueing data, then allow the error to @@ -4425,14 +4649,9 @@ nstat_control_handle_query_request( result = 0; } - while (dead_srcs) + while ((src = TAILQ_FIRST(&dead_list))) { - nstat_src *src; - - src = dead_srcs; - dead_srcs = src->next; - - // release src and send notification + TAILQ_REMOVE(&dead_list, src, ns_control_link); nstat_control_cleanup_source(state, src, FALSE); } @@ -4453,7 +4672,7 @@ nstat_control_handle_get_src_description( return EINVAL; } - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); u_int64_t src_count = 0; boolean_t partial = FALSE; const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL); @@ -4464,9 +4683,7 @@ nstat_control_handle_get_src_description( */ partial = nstat_control_begin_query(state, &req.hdr); - for (src = state->ncs_srcs; - src && (!partial || src_count < QUERY_CONTINUATION_SRC_COUNT); - src = src->next) + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { if (all_srcs || src->srcref == req.srcref) { @@ -4498,6 +4715,10 @@ nstat_control_handle_get_src_description( */ src->seq = state->ncs_seq; src_count++; + if (src_count >= QUERY_CONTINUATION_SRC_COUNT) + { + break; + } } } @@ -4513,7 +4734,7 @@ nstat_control_handle_get_src_description( if (req.srcref == NSTAT_SRC_REF_ALL) flags = nstat_control_end_query(state, src, partial); - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); /* * If an error occurred enqueueing data, then allow the error to * propagate to nstat_control_send. This way, the error is sent to @@ -4542,14 +4763,16 @@ nstat_control_handle_set_filter( req.srcref == NSTAT_SRC_REF_INVALID) return EINVAL; - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) + { if (req.srcref == src->srcref) { src->filter = req.filter; break; } - lck_mtx_unlock(&state->mtx); + } + lck_mtx_unlock(&state->ncs_mtx); if (src == NULL) return ENOENT; @@ -4645,16 +4868,16 @@ nstat_control_handle_get_update( return EINVAL; } - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); state->ncs_flags |= NSTAT_FLAG_SUPPORTS_UPDATES; errno_t result = ENOENT; - nstat_src *src; - nstat_src *dead_srcs = NULL; - nstat_src **srcpp = &state->ncs_srcs; + nstat_src *src, *tmpsrc; + tailq_head_nstat_src dead_list; u_int64_t src_count = 0; boolean_t partial = FALSE; + TAILQ_INIT(&dead_list); /* * Error handling policy and sequence number generation is folded into @@ -4662,14 +4885,11 @@ nstat_control_handle_get_update( */ partial = nstat_control_begin_query(state, &req.hdr); - while (*srcpp != NULL - && (FALSE == partial - || src_count < QUERY_CONTINUATION_SRC_COUNT)) + TAILQ_FOREACH_SAFE(src, &state->ncs_src_queue, ns_control_link, tmpsrc) { int gone; gone = 0; - src = *srcpp; if (nstat_control_reporting_allowed(state, src)) { /* skip this source if it has the current state @@ -4706,20 +4926,18 @@ nstat_control_handle_get_update( if (gone) { // pull src out of the list - *srcpp = src->next; - - src->next = dead_srcs; - dead_srcs = src; - } - else - { - srcpp = &(*srcpp)->next; + TAILQ_REMOVE(&state->ncs_src_queue, src, ns_control_link); + TAILQ_INSERT_TAIL(&dead_list, src, ns_control_link); } if (req.srcref != NSTAT_SRC_REF_ALL && req.srcref == src->srcref) { break; } + if (src_count >= QUERY_CONTINUATION_SRC_COUNT) + { + break; + } } nstat_flush_accumulated_msgs(state); @@ -4727,9 +4945,9 @@ nstat_control_handle_get_update( u_int16_t flags = 0; if (req.srcref == NSTAT_SRC_REF_ALL) - flags = nstat_control_end_query(state, *srcpp, partial); + flags = nstat_control_end_query(state, src, partial); - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); /* * If an error occurred enqueueing data, then allow the error to * propagate to nstat_control_send. This way, the error is sent to @@ -4741,11 +4959,9 @@ nstat_control_handle_get_update( result = 0; } - while (dead_srcs) + while ((src = TAILQ_FIRST(&dead_list))) { - src = dead_srcs; - dead_srcs = src->next; - + TAILQ_REMOVE(&dead_list, src, ns_control_link); // release src and send notification nstat_control_cleanup_source(state, src, FALSE); } @@ -4764,9 +4980,9 @@ nstat_control_handle_subscribe_sysinfo( return result; } - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); state->ncs_flags |= NSTAT_FLAG_SYSINFO_SUBSCRIBED; - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); return 0; } @@ -4890,3 +5106,5 @@ nstat_control_send( return result; } + + diff --git a/bsd/net/ntstat.h b/bsd/net/ntstat.h index e686d877d..af474f67a 100644 --- a/bsd/net/ntstat.h +++ b/bsd/net/ntstat.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2016 Apple Inc. All rights reserved. + * Copyright (c) 2010-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,13 +30,14 @@ #include #include #include +#include +#include #include #ifdef PRIVATE -#pragma pack(push, 4) #pragma mark -- Common Data Structures -- -#define __NSTAT_REVISION__ 8 +#define __NSTAT_REVISION__ 9 typedef u_int32_t nstat_provider_id_t; typedef u_int64_t nstat_src_ref_t; @@ -60,10 +61,17 @@ enum typedef struct nstat_counts { /* Counters */ - u_int64_t nstat_rxpackets __attribute__((aligned(8))); - u_int64_t nstat_rxbytes __attribute__((aligned(8))); - u_int64_t nstat_txpackets __attribute__((aligned(8))); - u_int64_t nstat_txbytes __attribute__((aligned(8))); + u_int64_t nstat_rxpackets __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_rxbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_txpackets __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_txbytes __attribute__((aligned(sizeof(u_int64_t)))); + + u_int64_t nstat_cell_rxbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_cell_txbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_wifi_rxbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_wifi_txbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_wired_rxbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_wired_txbytes __attribute__((aligned(sizeof(u_int64_t)))); u_int32_t nstat_rxduplicatebytes; u_int32_t nstat_rxoutoforderbytes; @@ -75,15 +83,9 @@ typedef struct nstat_counts u_int32_t nstat_min_rtt; u_int32_t nstat_avg_rtt; u_int32_t nstat_var_rtt; - - u_int64_t nstat_cell_rxbytes __attribute__((aligned(8))); - u_int64_t nstat_cell_txbytes __attribute__((aligned(8))); - u_int64_t nstat_wifi_rxbytes __attribute__((aligned(8))); - u_int64_t nstat_wifi_txbytes __attribute__((aligned(8))); - u_int64_t nstat_wired_rxbytes __attribute__((aligned(8))); - u_int64_t nstat_wired_txbytes __attribute__((aligned(8))); } nstat_counts; +#define NSTAT_SYSINFO_KEYVAL_STRING_MAXSIZE 24 typedef struct nstat_sysinfo_keyval { u_int32_t nstat_sysinfo_key; @@ -91,11 +93,15 @@ typedef struct nstat_sysinfo_keyval union { int64_t nstat_sysinfo_scalar; double nstat_sysinfo_distribution; + u_int8_t nstat_sysinfo_string[NSTAT_SYSINFO_KEYVAL_STRING_MAXSIZE]; } u; -} __attribute__((packed)) nstat_sysinfo_keyval; + u_int32_t nstat_sysinfo_valsize; + u_int8_t reserved[4]; +} nstat_sysinfo_keyval; #define NSTAT_SYSINFO_FLAG_SCALAR 0x0001 #define NSTAT_SYSINFO_FLAG_DISTRIBUTION 0x0002 +#define NSTAT_SYSINFO_FLAG_STRING 0x0004 #define NSTAT_MAX_MSG_SIZE 4096 @@ -105,7 +111,7 @@ typedef struct nstat_sysinfo_counts u_int32_t nstat_sysinfo_len; u_int32_t pad; u_int8_t nstat_sysinfo_keyvals[]; -} __attribute__((packed)) nstat_sysinfo_counts; +} nstat_sysinfo_counts; enum { @@ -208,10 +214,116 @@ enum ,NSTAT_SYSINFO_IFNET_UNSENT_DATA = 97 ,NSTAT_SYSINFO_ECN_IFNET_FALLBACK_DROPRST = 98 ,NSTAT_SYSINFO_ECN_IFNET_FALLBACK_DROPRXMT = 99 + ,NSTAT_SYSINFO_LIM_IFNET_SIGNATURE = 100 + ,NSTAT_SYSINFO_LIM_IFNET_DL_MAX_BANDWIDTH = 101 + ,NSTAT_SYSINFO_LIM_IFNET_UL_MAX_BANDWIDTH = 102 + ,NSTAT_SYSINFO_LIM_IFNET_PACKET_LOSS_PERCENT = 103 + ,NSTAT_SYSINFO_LIM_IFNET_PACKET_OOO_PERCENT = 104 + ,NSTAT_SYSINFO_LIM_IFNET_RTT_VARIANCE = 105 + ,NSTAT_SYSINFO_LIM_IFNET_RTT_MIN = 106 + ,NSTAT_SYSINFO_LIM_IFNET_RTT_AVG = 107 + ,NSTAT_SYSINFO_LIM_IFNET_CONN_TIMEOUT_PERCENT = 108 + ,NSTAT_SYSINFO_LIM_IFNET_DL_DETECTED = 109 + ,NSTAT_SYSINFO_LIM_IFNET_UL_DETECTED = 110 + ,NSTAT_SYSINFO_LIM_IFNET_TYPE = 111 + + ,NSTAT_SYSINFO_API_IF_FLTR_ATTACH = 112 + ,NSTAT_SYSINFO_API_IF_FLTR_ATTACH_OS = 113 + ,NSTAT_SYSINFO_API_IP_FLTR_ADD = 114 + ,NSTAT_SYSINFO_API_IP_FLTR_ADD_OS = 115 + ,NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH = 116 + ,NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH_OS = 117 + + ,NSTAT_SYSINFO_API_SOCK_ALLOC_TOTAL = 118 + ,NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL = 119 + ,NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL_OS = 120 + ,NSTAT_SYSINFO_API_SOCK_NECP_CLIENTUUID = 121 + + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_LOCAL = 122 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_ROUTE = 123 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_INET = 124 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_INET6 = 125 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_SYSTEM = 126 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_MULTIPATH = 127 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_KEY = 128 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_NDRV = 129 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_OTHER = 130 + + ,NSTAT_SYSINFO_API_SOCK_INET_STREAM= 131 + ,NSTAT_SYSINFO_API_SOCK_INET_DGRAM = 132 + ,NSTAT_SYSINFO_API_SOCK_INET_DGRAM_CONNECTED = 133 + ,NSTAT_SYSINFO_API_SOCK_INET_DGRAM_DNS = 134 + ,NSTAT_SYSINFO_API_SOCK_INET_DGRAM_NO_DATA = 135 + + ,NSTAT_SYSINFO_API_SOCK_INET6_STREAM= 136 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM = 137 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_CONNECTED = 138 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_DNS = 139 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_NO_DATA = 140 + + ,NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN = 141 + ,NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN_OS = 142 + + ,NSTAT_SYSINFO_API_SOCK_INET6_STREAM_EXTHDR_IN = 143 + ,NSTAT_SYSINFO_API_SOCK_INET6_STREAM_EXTHDR_OUT = 144 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_EXTHDR_IN = 145 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_EXTHDR_OUT = 146 + + ,NSTAT_SYSINFO_API_NEXUS_FLOW_INET_STREAM = 147 + ,NSTAT_SYSINFO_API_NEXUS_FLOW_INET_DATAGRAM = 148 + + ,NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_STREAM = 149 + ,NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_DATAGRAM = 150 + + ,NSTAT_SYSINFO_API_IFNET_ALLOC = 151 + ,NSTAT_SYSINFO_API_IFNET_ALLOC_OS = 152 + + ,NSTAT_SYSINFO_API_PF_ADDRULE = 153 + ,NSTAT_SYSINFO_API_PF_ADDRULE_OS = 154 + + ,NSTAT_SYSINFO_API_VMNET_START = 155 + + ,NSTAT_SYSINFO_API_IF_NETAGENT_ENABLED = 156 + + ,NSTAT_SYSINFO_API_REPORT_INTERVAL = 157 + + ,NSTAT_SYSINFO_MPTCP_HANDOVER_ATTEMPT = 158 + ,NSTAT_SYSINFO_MPTCP_INTERACTIVE_ATTEMPT = 159 + ,NSTAT_SYSINFO_MPTCP_AGGREGATE_ATTEMPT = 160 + ,NSTAT_SYSINFO_MPTCP_FP_HANDOVER_ATTEMPT = 161 /* _FP_ stands for first-party */ + ,NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_ATTEMPT = 162 + ,NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_ATTEMPT = 163 + ,NSTAT_SYSINFO_MPTCP_HEURISTIC_FALLBACK = 164 + ,NSTAT_SYSINFO_MPTCP_FP_HEURISTIC_FALLBACK = 165 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_WIFI = 166 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_CELL = 167 + ,NSTAT_SYSINFO_MPTCP_INTERACTIVE_SUCCESS = 168 + ,NSTAT_SYSINFO_MPTCP_AGGREGATE_SUCCESS = 169 + ,NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_WIFI = 170 + ,NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_CELL = 171 + ,NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_SUCCESS = 172 + ,NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_SUCCESS = 173 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_FROM_WIFI = 174 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_WIFI_FROM_CELL = 175 + ,NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_FROM_WIFI = 176 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_BYTES = 177 + ,NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_BYTES = 178 + ,NSTAT_SYSINFO_MPTCP_AGGREGATE_CELL_BYTES = 179 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_ALL_BYTES = 180 + ,NSTAT_SYSINFO_MPTCP_INTERACTIVE_ALL_BYTES = 181 + ,NSTAT_SYSINFO_MPTCP_AGGREGATE_ALL_BYTES = 182 + ,NSTAT_SYSINFO_MPTCP_BACK_TO_WIFI = 183 + ,NSTAT_SYSINFO_MPTCP_WIFI_PROXY = 184 + ,NSTAT_SYSINFO_MPTCP_CELL_PROXY = 185 + ,NSTAT_SYSINFO_ECN_IFNET_FALLBACK_SYNRST = 186 + // NSTAT_SYSINFO_ENUM_VERSION must be updated any time a value is added -#define NSTAT_SYSINFO_ENUM_VERSION 20160715 +#define NSTAT_SYSINFO_ENUM_VERSION 20170623 }; +#define NSTAT_SYSINFO_API_FIRST NSTAT_SYSINFO_API_IF_FLTR_ATTACH +#define NSTAT_SYSINFO_API_LAST NSTAT_SYSINFO_API_REPORT_INTERVAL + #pragma mark -- Network Statistics Providers -- @@ -226,6 +338,9 @@ enum #define NSTAT_IFNET_IS_EXPENSIVE 0x40 #define NSTAT_IFNET_IS_VPN 0x80 #define NSTAT_IFNET_VIA_CELLFALLBACK 0x100 +// Temporary properties of use for bringing up userland providers +#define NSTAT_IFNET_ROUTE_VALUE_UNOBTAINABLE 0x1000 +#define NSTAT_IFNET_FLOWSWITCH_VALUE_UNOBTAINABLE 0x2000 enum @@ -273,20 +388,14 @@ typedef struct nstat_tcp_add_param typedef struct nstat_tcp_descriptor { - union - { - struct sockaddr_in v4; - struct sockaddr_in6 v6; - } local; + u_int64_t upid __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t eupid __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t start_timestamp __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t timestamp __attribute__((aligned(sizeof(u_int64_t)))); - union - { - struct sockaddr_in v4; - struct sockaddr_in6 v6; - } remote; + activity_bitmap_t activity_bitmap; u_int32_t ifindex; - u_int32_t state; u_int32_t sndbufsize; @@ -298,25 +407,49 @@ typedef struct nstat_tcp_descriptor u_int32_t txcwindow; u_int32_t traffic_class; u_int32_t traffic_mgt_flags; - char cc_algo[16]; - u_int64_t upid; u_int32_t pid; - char pname[64]; - u_int64_t eupid; u_int32_t epid; + union + { + struct sockaddr_in v4; + struct sockaddr_in6 v6; + } local; + + union + { + struct sockaddr_in v4; + struct sockaddr_in6 v6; + } remote; + + char cc_algo[16]; + char pname[64]; + uuid_t uuid; uuid_t euuid; uuid_t vuuid; - struct tcp_conn_status connstatus; + union { + struct tcp_conn_status connstatus; + // On armv7k, tcp_conn_status is 1 byte instead of 4 + uint8_t __pad_connstatus[4]; + }; uint16_t ifnet_properties __attribute__((aligned(4))); + + u_int8_t reserved[6]; } nstat_tcp_descriptor; typedef struct nstat_tcp_add_param nstat_udp_add_param; typedef struct nstat_udp_descriptor { + u_int64_t upid __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t eupid __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t start_timestamp __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t timestamp __attribute__((aligned(sizeof(u_int64_t)))); + + activity_bitmap_t activity_bitmap; + union { struct sockaddr_in v4; @@ -335,23 +468,23 @@ typedef struct nstat_udp_descriptor u_int32_t rcvbufused; u_int32_t traffic_class; - u_int64_t upid; u_int32_t pid; char pname[64]; - u_int64_t eupid; u_int32_t epid; uuid_t uuid; uuid_t euuid; uuid_t vuuid; uint16_t ifnet_properties; + + u_int8_t reserved[6]; } nstat_udp_descriptor; typedef struct nstat_route_descriptor { - u_int64_t id; - u_int64_t parent_id; - u_int64_t gateway_id; + u_int64_t id __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t parent_id __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t gateway_id __attribute__((aligned(sizeof(u_int64_t)))); union { @@ -377,12 +510,15 @@ typedef struct nstat_route_descriptor u_int32_t ifindex; u_int32_t flags; + u_int8_t reserved[4]; } nstat_route_descriptor; typedef struct nstat_ifnet_add_param { + u_int64_t threshold __attribute__((aligned(sizeof(u_int64_t)))); u_int32_t ifindex; - u_int64_t threshold; + + u_int8_t reserved[4]; } nstat_ifnet_add_param; typedef struct nstat_ifnet_desc_cellular_status @@ -436,6 +572,7 @@ typedef struct nstat_ifnet_desc_cellular_status #define NSTAT_IFNET_DESC_MSS_RECOMMENDED_MEDIUM 0x1 #define NSTAT_IFNET_DESC_MSS_RECOMMENDED_LOW 0x2 u_int16_t mss_recommended; /* recommended MSS */ + u_int8_t reserved[2]; } nstat_ifnet_desc_cellular_status; typedef struct nstat_ifnet_desc_wifi_status { @@ -508,6 +645,7 @@ enum NSTAT_IFNET_DESC_LINK_STATUS_TYPE_NONE = 0 ,NSTAT_IFNET_DESC_LINK_STATUS_TYPE_CELLULAR = 1 ,NSTAT_IFNET_DESC_LINK_STATUS_TYPE_WIFI = 2 + ,NSTAT_IFNET_DESC_LINK_STATUS_TYPE_ETHERNET = 3 }; typedef struct nstat_ifnet_desc_link_status @@ -524,12 +662,13 @@ typedef struct nstat_ifnet_desc_link_status #endif typedef struct nstat_ifnet_descriptor { - char name[IFNAMSIZ+1]; + u_int64_t threshold __attribute__((aligned(sizeof(u_int64_t)))); u_int32_t ifindex; - u_int64_t threshold; - unsigned int type; - char description[IF_DESCSIZE]; nstat_ifnet_desc_link_status link_status; + unsigned int type; + char description[IF_DESCSIZE]; + char name[IFNAMSIZ+1]; + u_int8_t reserved[3]; } nstat_ifnet_descriptor; typedef struct nstat_sysinfo_descriptor @@ -545,7 +684,9 @@ typedef struct nstat_sysinfo_add_param #define NSTAT_SYSINFO_MBUF_STATS 0x0001 #define NSTAT_SYSINFO_TCP_STATS 0x0002 -#define NSTAT_SYSINFO_IFNET_ECN_STATS 0x0003 +#define NSTAT_SYSINFO_IFNET_ECN_STATS 0x0003 +#define NSTAT_SYSINFO_LIM_STATS 0x0004 /* Low Internet mode stats */ +#define NSTAT_SYSINFO_NET_API_STATS 0x0005 /* API and KPI stats */ #pragma mark -- Network Statistics User Client -- @@ -604,6 +745,7 @@ enum ,NSTAT_FILTER_TCP_NO_LISTENER = 0x00001000 ,NSTAT_FILTER_TCP_ONLY_LISTENER = 0x00002000 ,NSTAT_FILTER_TCP_INTERFACE_ATTACH = 0x00004000 + ,NSTAT_FILTER_TCP_NO_EARLY_CLOSE = 0x00008000 ,NSTAT_FILTER_TCP_FLAGS = 0x0000F000 ,NSTAT_FILTER_UDP_INTERFACE_ATTACH = 0x00010000 @@ -629,7 +771,7 @@ enum typedef struct nstat_msg_hdr { - u_int64_t context; + u_int64_t context __attribute__((aligned(sizeof(u_int64_t)))); u_int32_t type; u_int16_t length; u_int16_t flags; @@ -639,21 +781,45 @@ typedef struct nstat_msg_error { nstat_msg_hdr hdr; u_int32_t error; // errno error + u_int8_t reserved[4]; } nstat_msg_error; +#define NSTAT_ADD_SRC_FIELDS \ + nstat_msg_hdr hdr; \ + nstat_provider_id_t provider; \ + u_int8_t reserved[4] \ + typedef struct nstat_msg_add_src { - nstat_msg_hdr hdr; - nstat_provider_id_t provider; - u_int8_t param[]; + NSTAT_ADD_SRC_FIELDS; + u_int8_t param[]; } nstat_msg_add_src_req; +typedef struct nstat_msg_add_src_header +{ + NSTAT_ADD_SRC_FIELDS; +} nstat_msg_add_src_header; + +typedef struct nstat_msg_add_src_convenient +{ + nstat_msg_add_src_header hdr; + union { + nstat_route_add_param route; + nstat_tcp_add_param tcp; + nstat_udp_add_param udp; + nstat_ifnet_add_param ifnet; + nstat_sysinfo_add_param sysinfo; + }; +} nstat_msg_add_src_convenient; + +#undef NSTAT_ADD_SRC_FIELDS + typedef struct nstat_msg_add_all_srcs { nstat_msg_hdr hdr; + u_int64_t filter __attribute__((aligned(sizeof(u_int64_t)))); + nstat_event_flags_t events __attribute__((aligned(sizeof(u_int64_t)))); nstat_provider_id_t provider; - u_int64_t filter; - nstat_event_flags_t events; pid_t target_pid; uuid_t target_uuid; } nstat_msg_add_all_srcs; @@ -661,76 +827,122 @@ typedef struct nstat_msg_add_all_srcs typedef struct nstat_msg_src_added { nstat_msg_hdr hdr; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); nstat_provider_id_t provider; - nstat_src_ref_t srcref; + u_int8_t reserved[4]; } nstat_msg_src_added; typedef struct nstat_msg_rem_src { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); } nstat_msg_rem_src_req; typedef struct nstat_msg_get_src_description { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); } nstat_msg_get_src_description; typedef struct nstat_msg_set_filter { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; - u_int32_t filter; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); + u_int32_t filter; + u_int8_t reserved[4]; } nstat_msg_set_filter; +#define NSTAT_SRC_DESCRIPTION_FIELDS \ + nstat_msg_hdr hdr; \ + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); \ + nstat_event_flags_t event_flags __attribute__((aligned(sizeof(u_int64_t)))); \ + nstat_provider_id_t provider; \ + u_int8_t reserved[4] + typedef struct nstat_msg_src_description { - nstat_msg_hdr hdr; - nstat_src_ref_t srcref; - nstat_event_flags_t event_flags; - nstat_provider_id_t provider; - u_int8_t data[]; + NSTAT_SRC_DESCRIPTION_FIELDS; + u_int8_t data[]; } nstat_msg_src_description; +typedef struct nstat_msg_src_description_header +{ + NSTAT_SRC_DESCRIPTION_FIELDS; +} nstat_msg_src_description_header; + +typedef struct nstat_msg_src_description_convenient +{ + nstat_msg_src_description_header hdr; + union { + nstat_tcp_descriptor tcp; + nstat_udp_descriptor udp; + nstat_route_descriptor route; + nstat_ifnet_descriptor ifnet; + nstat_sysinfo_descriptor sysinfo; + }; +} nstat_msg_src_description_convenient; + +#undef NSTAT_SRC_DESCRIPTION_FIELDS + typedef struct nstat_msg_query_src { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); } nstat_msg_query_src_req; typedef struct nstat_msg_src_counts { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; - nstat_event_flags_t event_flags; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); + nstat_event_flags_t event_flags __attribute__((aligned(sizeof(u_int64_t)))); nstat_counts counts; } nstat_msg_src_counts; +#define NSTAT_SRC_UPDATE_FIELDS \ + nstat_msg_hdr hdr; \ + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); \ + nstat_event_flags_t event_flags __attribute__((aligned(sizeof(u_int64_t)))); \ + nstat_counts counts; \ + nstat_provider_id_t provider; \ + u_int8_t reserved[4] + typedef struct nstat_msg_src_update { - nstat_msg_hdr hdr; - nstat_src_ref_t srcref; - nstat_event_flags_t event_flags; - nstat_counts counts; - nstat_provider_id_t provider; - u_int8_t data[]; + NSTAT_SRC_UPDATE_FIELDS; + u_int8_t data[]; } nstat_msg_src_update; +typedef struct nstat_msg_src_update_hdr +{ + NSTAT_SRC_UPDATE_FIELDS; +} nstat_msg_src_update_hdr; + +typedef struct nstat_msg_src_update_convenient +{ + nstat_msg_src_update_hdr hdr; + union { + nstat_tcp_descriptor tcp; + nstat_udp_descriptor udp; + nstat_route_descriptor route; + nstat_ifnet_descriptor ifnet; + nstat_sysinfo_descriptor sysinfo; + }; +} nstat_msg_src_update_convenient; + +#undef NSTAT_SRC_UPDATE_FIELDS + typedef struct nstat_msg_src_removed { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); } nstat_msg_src_removed; typedef struct nstat_msg_sysinfo_counts { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); nstat_sysinfo_counts counts; -} __attribute__((packed)) nstat_msg_sysinfo_counts; - -#pragma pack(pop) +} nstat_msg_sysinfo_counts; #pragma mark -- Statitiscs about Network Statistics -- @@ -775,6 +987,7 @@ typedef struct nstat_sysinfo_mbuf_stats typedef struct nstat_sysinfo_tcp_stats { + /* When adding/removing here, also adjust NSTAT_SYSINFO_TCP_STATS_COUNT */ u_int32_t ipv4_avgrtt; /* Average RTT for IPv4 */ u_int32_t ipv6_avgrtt; /* Average RTT for IPv6 */ u_int32_t send_plr; /* Average uplink packet loss rate */ @@ -817,7 +1030,37 @@ typedef struct nstat_sysinfo_tcp_stats u_int32_t tfo_no_cookie_rcv; /* We asked for a cookie but didn't get one */ u_int32_t tfo_heuristics_disable; /* TFO got disabled due to heuristics */ u_int32_t tfo_sndblackhole; /* TFO got blackholed in the sending direction */ + u_int32_t mptcp_handover_attempt; /* Total number of MPTCP-attempts using handover mode */ + u_int32_t mptcp_interactive_attempt; /* Total number of MPTCP-attempts using interactive mode */ + u_int32_t mptcp_aggregate_attempt; /* Total number of MPTCP-attempts using aggregate mode */ + u_int32_t mptcp_fp_handover_attempt; /* Same as previous three but only for first-party apps */ + u_int32_t mptcp_fp_interactive_attempt; + u_int32_t mptcp_fp_aggregate_attempt; + u_int32_t mptcp_heuristic_fallback; /* Total number of MPTCP-connections that fell back due to heuristics */ + u_int32_t mptcp_fp_heuristic_fallback; /* Same as previous but for first-party apps */ + u_int32_t mptcp_handover_success_wifi; /* Total number of successfull handover-mode connections that *started* on WiFi */ + u_int32_t mptcp_handover_success_cell; /* Total number of successfull handover-mode connections that *started* on Cell */ + u_int32_t mptcp_interactive_success; /* Total number of interactive-mode connections that negotiated MPTCP */ + u_int32_t mptcp_aggregate_success; /* Same as previous but for aggregate */ + u_int32_t mptcp_fp_handover_success_wifi; /* Same as previous four, but for first-party apps */ + u_int32_t mptcp_fp_handover_success_cell; + u_int32_t mptcp_fp_interactive_success; + u_int32_t mptcp_fp_aggregate_success; + u_int32_t mptcp_handover_cell_from_wifi; /* Total number of connections that use cell in handover-mode (coming from WiFi) */ + u_int32_t mptcp_handover_wifi_from_cell; /* Total number of connections that use WiFi in handover-mode (coming from cell) */ + u_int32_t mptcp_interactive_cell_from_wifi; /* Total number of connections that use cell in interactive mode (coming from WiFi) */ + u_int32_t mptcp_back_to_wifi; /* Total number of connections that succeed to move traffic away from cell (when starting on cell) */ + u_int64_t mptcp_handover_cell_bytes; /* Total number of bytes sent on cell in handover-mode (on new subflows, ignoring initial one) */ + u_int64_t mptcp_interactive_cell_bytes; /* Same as previous but for interactive */ + u_int64_t mptcp_aggregate_cell_bytes; + u_int64_t mptcp_handover_all_bytes; /* Total number of bytes sent in handover */ + u_int64_t mptcp_interactive_all_bytes; + u_int64_t mptcp_aggregate_all_bytes; + u_int32_t mptcp_wifi_proxy; /* Total number of new subflows that fell back to regular TCP on cell */ + u_int32_t mptcp_cell_proxy; /* Total number of new subflows that fell back to regular TCP on WiFi */ + /* When adding/removing here, also adjust NSTAT_SYSINFO_TCP_STATS_COUNT */ } nstat_sysinfo_tcp_stats; +#define NSTAT_SYSINFO_TCP_STATS_COUNT 70 enum { NSTAT_IFNET_ECN_PROTO_IPV4 = 1 @@ -836,15 +1079,33 @@ typedef struct nstat_sysinfo_ifnet_ecn_stats { struct if_tcp_ecn_stat ecn_stat; } nstat_sysinfo_ifnet_ecn_stats; +/* Total number of Low Internet stats that will be reported */ +#define NSTAT_LIM_STAT_KEYVAL_COUNT 12 +typedef struct nstat_sysinfo_lim_stats { + u_int8_t ifnet_signature[NSTAT_SYSINFO_KEYVAL_STRING_MAXSIZE]; + u_int32_t ifnet_siglen; + u_int32_t ifnet_type; + struct if_lim_perf_stat lim_stat; +} nstat_sysinfo_lim_stats; + +#define NSTAT_NET_API_STAT_KEYVAL_COUNT (NSTAT_SYSINFO_API_LAST - NSTAT_SYSINFO_API_FIRST + 1) +typedef struct nstat_sysinfo_net_api_stats { + u_int32_t report_interval; + u_int32_t _padding; + struct net_api_stats net_api_stats; +} nstat_sysinfo_net_api_stats; + typedef struct nstat_sysinfo_data { - u_int32_t flags; + uint32_t flags; + uint32_t unsent_data_cnt; /* Before sleeping */ union { nstat_sysinfo_mbuf_stats mb_stats; nstat_sysinfo_tcp_stats tcp_stats; nstat_sysinfo_ifnet_ecn_stats ifnet_ecn_stats; + nstat_sysinfo_lim_stats lim_stats; + nstat_sysinfo_net_api_stats net_api_stats; } u; - uint32_t unsent_data_cnt; /* Before sleeping */ } nstat_sysinfo_data; #pragma mark -- Generic Network Statistics Provider -- @@ -876,6 +1137,11 @@ void nstat_route_connect_success(struct rtentry *rte); void nstat_route_tx(struct rtentry *rte, u_int32_t packets, u_int32_t bytes, u_int32_t flags); void nstat_route_rx(struct rtentry *rte, u_int32_t packets, u_int32_t bytes, u_int32_t flags); void nstat_route_rtt(struct rtentry *rte, u_int32_t rtt, u_int32_t rtt_var); +void nstat_route_update(struct rtentry *rte, uint32_t connect_attempts, uint32_t connect_successes, + uint32_t rx_packets, uint32_t rx_bytes, uint32_t rx_duplicatebytes, uint32_t rx_outoforderbytes, + uint32_t tx_packets, uint32_t tx_bytes, uint32_t tx_retransmit, + uint32_t rtt, uint32_t rtt_var); +struct nstat_counts* nstat_route_attach(struct rtentry *rte); void nstat_route_detach(struct rtentry *rte); // watcher support @@ -892,37 +1158,9 @@ void nstat_ifnet_threshold_reached(unsigned int ifindex); void nstat_sysinfo_send_data(struct nstat_sysinfo_data *); -// Userland stats reporting - -// Each side, NetworkStatistics and the kernel provider for userland, -// pass opaque references. -typedef void *userland_stats_provider_context; -typedef void *nstat_userland_context; - -// When things have been set up, Netstats can request a refresh of its data. -typedef bool (userland_stats_request_vals_fn)(userland_stats_provider_context *ctx, - nstat_counts *countsp, - void *metadatap); - -// Things get started with a call to netstats to say that there’s a new connection: -nstat_userland_context ntstat_userland_stats_open(userland_stats_provider_context *ctx, - int provider_id, - u_int64_t properties, - userland_stats_request_vals_fn req_fn); - -void ntstat_userland_stats_close(nstat_userland_context nstat_ctx); - - -// There may be other occasions where the stats have changed and NECP should push the new values. -// This is provisional, ahead of full implementation. - -typedef enum { - USERLAND_STATS_WILL_UPDATE, - USERLAND_STATS_DID_UPDATE -} userland_stats_event_t; - -void ntstat_userland_stats_event(nstat_userland_context nstat_ctx, userland_stats_event_t event); +// Utilities for userland stats reporting +u_int16_t nstat_ifnet_to_flags(struct ifnet *ifp); // locked_add_64 uses atomic operations on 32bit so the 64bit // value can be properly read. The values are only ever incremented diff --git a/bsd/net/nwk_wq.c b/bsd/net/nwk_wq.c new file mode 100644 index 000000000..400fd1312 --- /dev/null +++ b/bsd/net/nwk_wq.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +MALLOC_DEFINE(M_NWKWQ, "nwkwq", "Network work-queue"); + +static TAILQ_HEAD(, nwk_wq_entry) nwk_wq_head; +decl_lck_mtx_data(static, nwk_wq_lock); + +/* Lock group and attributes */ +static lck_grp_attr_t *nwk_wq_lock_grp_attributes = NULL; +static lck_grp_t *nwk_wq_lock_group = NULL; + +/* Lock and lock attributes */ +static lck_attr_t *nwk_wq_lock_attributes = NULL; +decl_lck_mtx_data(static, nwk_wq_lock); + +/* Wait channel for Network work queue */ +static void *nwk_wq_waitch = NULL; +static void nwk_wq_thread_func(void *, wait_result_t); + +static int nwk_wq_thread_cont(int err); +static void nwk_wq_thread_func(void *v, wait_result_t w); + +void +nwk_wq_init (void) +{ + thread_t nwk_wq_thread = THREAD_NULL; + + TAILQ_INIT(&nwk_wq_head); + nwk_wq_lock_grp_attributes = lck_grp_attr_alloc_init(); + nwk_wq_lock_group = lck_grp_alloc_init("Network work queue lock", + nwk_wq_lock_grp_attributes); + + nwk_wq_lock_attributes = lck_attr_alloc_init(); + lck_mtx_init(&nwk_wq_lock, nwk_wq_lock_group, nwk_wq_lock_attributes); + if (kernel_thread_start(nwk_wq_thread_func, + NULL, &nwk_wq_thread) != KERN_SUCCESS) { + panic_plain("%s: couldn't create network work queue thread", __func__); + /* NOTREACHED */ + } + thread_deallocate(nwk_wq_thread); +} + +static int +nwk_wq_thread_cont(int err) +{ + TAILQ_HEAD(, nwk_wq_entry) temp_nwk_wq_head; + struct nwk_wq_entry *nwk_item; + struct nwk_wq_entry *nwk_item_next; + +#pragma unused(err) + for (;;) { + nwk_item = NULL; + nwk_item_next = NULL; + TAILQ_INIT(&temp_nwk_wq_head); + + LCK_MTX_ASSERT(&nwk_wq_lock, LCK_MTX_ASSERT_OWNED); + while (TAILQ_FIRST(&nwk_wq_head) == NULL) { + (void) msleep0(&nwk_wq_waitch, &nwk_wq_lock, + (PZERO - 1), "nwk_wq_thread_cont", 0, + nwk_wq_thread_cont); + /* NOTREACHED */ + } + + TAILQ_SWAP(&temp_nwk_wq_head, &nwk_wq_head, nwk_wq_entry, nwk_wq_link); + VERIFY(TAILQ_EMPTY(&nwk_wq_head)); + lck_mtx_unlock(&nwk_wq_lock); + + VERIFY(TAILQ_FIRST(&temp_nwk_wq_head) != NULL); + TAILQ_FOREACH_SAFE(nwk_item, &temp_nwk_wq_head, nwk_wq_link, nwk_item_next) { + nwk_item->func(nwk_item->arg); + if (nwk_item->is_arg_managed == FALSE) + FREE(nwk_item->arg, M_NWKWQ); + FREE(nwk_item, M_NWKWQ); + } + lck_mtx_lock(&nwk_wq_lock); + } +} + +static void +nwk_wq_thread_func(void *v, wait_result_t w) +{ +#pragma unused(v, w) + lck_mtx_lock(&nwk_wq_lock); + (void) msleep0(&nwk_wq_waitch, &nwk_wq_lock, + (PZERO - 1), "nwk_wq_thread_func", 0, nwk_wq_thread_cont); + /* + * msleep0() shouldn't have returned as PCATCH was not set; + * therefore assert in this case. + */ + lck_mtx_unlock(&nwk_wq_lock); + VERIFY(0); +} + +void +nwk_wq_enqueue(struct nwk_wq_entry *nwk_item) +{ + lck_mtx_lock(&nwk_wq_lock); + TAILQ_INSERT_TAIL(&nwk_wq_head, nwk_item, nwk_wq_link); + lck_mtx_unlock(&nwk_wq_lock); + wakeup((caddr_t)&nwk_wq_waitch); +} + diff --git a/bsd/net/nwk_wq.h b/bsd/net/nwk_wq.h new file mode 100644 index 000000000..80d8e4851 --- /dev/null +++ b/bsd/net/nwk_wq.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef NWK_WQ_H +#define NWK_WQ_H +#include +#include + +#ifdef BSD_KERNEL_PRIVATE +struct nwk_wq_entry { + void (* func) (void *); + void *arg; + boolean_t is_arg_managed; + TAILQ_ENTRY(nwk_wq_entry) nwk_wq_link; +}; + +void nwk_wq_init (void); +void nwk_wq_enqueue(struct nwk_wq_entry *nwk_item); +#endif /* BSD_KERNEL_PRIVATE */ +#endif /* NWK_WQ_H */ + diff --git a/bsd/net/packet_mangler.c b/bsd/net/packet_mangler.c index 5a1776d74..d6c8b9dbd 100644 --- a/bsd/net/packet_mangler.c +++ b/bsd/net/packet_mangler.c @@ -822,7 +822,7 @@ static errno_t pktmnglr_ipfilter_output(void *cookie, mbuf_t *data, ipf_pktopts_ if (ip.ip_v != 4) { PKT_MNGLR_LOG(LOG_INFO, "%s:%d Not handling IP version %d\n", - __FILE__, __LINE__, ip.ip_v); + __func__, __LINE__, ip.ip_v); goto output_done; } diff --git a/bsd/net/pf.c b/bsd/net/pf.c index 70ea3cf33..6ce23690e 100644 --- a/bsd/net/pf.c +++ b/bsd/net/pf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -68,7 +68,6 @@ #include #include #include -#include #include #include #include @@ -83,6 +82,7 @@ #include #include +#include #include #include @@ -162,14 +162,6 @@ struct pf_state_tree_ext_gwy pf_statetbl_ext_gwy; struct pf_palist pf_pabuf; struct pf_status pf_status; -#if PF_ALTQ -struct pf_altqqueue pf_altqs[2]; -struct pf_altqqueue *pf_altqs_active; -struct pf_altqqueue *pf_altqs_inactive; -u_int32_t ticket_altqs_active; -u_int32_t ticket_altqs_inactive; -int altqs_inactive_open; -#endif /* PF_ALTQ */ u_int32_t ticket_pabuf; static MD5_CTX pf_tcp_secret_ctx; @@ -186,9 +178,6 @@ static struct pf_anchor_stackframe { struct pool pf_src_tree_pl, pf_rule_pl, pf_pooladdr_pl; struct pool pf_state_pl, pf_state_key_pl; -#if PF_ALTQ -struct pool pf_altq_pl; -#endif /* PF_ALTQ */ typedef void (*hook_fn_t)(void *); @@ -218,11 +207,11 @@ static void pf_init_threshold(struct pf_threshold *, u_int32_t, static void pf_add_threshold(struct pf_threshold *); static int pf_check_threshold(struct pf_threshold *); -static void pf_change_ap(int, struct mbuf *, struct pf_addr *, +static void pf_change_ap(int, pbuf_t *, struct pf_addr *, u_int16_t *, u_int16_t *, u_int16_t *, struct pf_addr *, u_int16_t, u_int8_t, sa_family_t, sa_family_t, int); -static int pf_modulate_sack(struct mbuf *, int, struct pf_pdesc *, +static int pf_modulate_sack(pbuf_t *, int, struct pf_pdesc *, struct tcphdr *, struct pf_state_peer *); #if INET6 static void pf_change_a6(struct pf_addr *, u_int16_t *, @@ -240,46 +229,47 @@ static void pf_send_tcp(const struct pf_rule *, sa_family_t, u_int16_t, u_int16_t, u_int32_t, u_int32_t, u_int8_t, u_int16_t, u_int16_t, u_int8_t, int, u_int16_t, struct ether_header *, struct ifnet *); -static void pf_send_icmp(struct mbuf *, u_int8_t, u_int8_t, +static void pf_send_icmp(pbuf_t *, u_int8_t, u_int8_t, sa_family_t, struct pf_rule *); -static struct pf_rule *pf_match_translation(struct pf_pdesc *, struct mbuf *, +static struct pf_rule *pf_match_translation(struct pf_pdesc *, pbuf_t *, int, int, struct pfi_kif *, struct pf_addr *, union pf_state_xport *, struct pf_addr *, union pf_state_xport *, int); static struct pf_rule *pf_get_translation_aux(struct pf_pdesc *, - struct mbuf *, int, int, struct pfi_kif *, + pbuf_t *, int, int, struct pfi_kif *, struct pf_src_node **, struct pf_addr *, union pf_state_xport *, struct pf_addr *, - union pf_state_xport *, union pf_state_xport *); + union pf_state_xport *, union pf_state_xport * + ); static void pf_attach_state(struct pf_state_key *, struct pf_state *, int); static void pf_detach_state(struct pf_state *, int); static u_int32_t pf_tcp_iss(struct pf_pdesc *); static int pf_test_rule(struct pf_rule **, struct pf_state **, - int, struct pfi_kif *, struct mbuf *, int, + int, struct pfi_kif *, pbuf_t *, int, void *, struct pf_pdesc *, struct pf_rule **, struct pf_ruleset **, struct ifqueue *); #if DUMMYNET static int pf_test_dummynet(struct pf_rule **, int, - struct pfi_kif *, struct mbuf **, + struct pfi_kif *, pbuf_t **, struct pf_pdesc *, struct ip_fw_args *); #endif /* DUMMYNET */ static int pf_test_fragment(struct pf_rule **, int, - struct pfi_kif *, struct mbuf *, void *, + struct pfi_kif *, pbuf_t *, void *, struct pf_pdesc *, struct pf_rule **, struct pf_ruleset **); static int pf_test_state_tcp(struct pf_state **, int, - struct pfi_kif *, struct mbuf *, int, + struct pfi_kif *, pbuf_t *, int, void *, struct pf_pdesc *, u_short *); static int pf_test_state_udp(struct pf_state **, int, - struct pfi_kif *, struct mbuf *, int, + struct pfi_kif *, pbuf_t *, int, void *, struct pf_pdesc *, u_short *); static int pf_test_state_icmp(struct pf_state **, int, - struct pfi_kif *, struct mbuf *, int, + struct pfi_kif *, pbuf_t *, int, void *, struct pf_pdesc *, u_short *); static int pf_test_state_other(struct pf_state **, int, struct pfi_kif *, struct pf_pdesc *); -static int pf_match_tag(struct mbuf *, struct pf_rule *, +static int pf_match_tag(struct pf_rule *, struct pf_mtag *, int *); static void pf_hash(struct pf_addr *, struct pf_addr *, struct pf_poolhashkey *, sa_family_t); @@ -290,24 +280,25 @@ static int pf_get_sport(struct pf_pdesc *, struct pfi_kif *, struct pf_rule *, struct pf_addr *, union pf_state_xport *, struct pf_addr *, union pf_state_xport *, struct pf_addr *, - union pf_state_xport *, struct pf_src_node **); -static void pf_route(struct mbuf **, struct pf_rule *, int, + union pf_state_xport *, struct pf_src_node ** + ); +static void pf_route(pbuf_t **, struct pf_rule *, int, struct ifnet *, struct pf_state *, struct pf_pdesc *); #if INET6 -static void pf_route6(struct mbuf **, struct pf_rule *, int, +static void pf_route6(pbuf_t **, struct pf_rule *, int, struct ifnet *, struct pf_state *, struct pf_pdesc *); #endif /* INET6 */ -static u_int8_t pf_get_wscale(struct mbuf *, int, u_int16_t, +static u_int8_t pf_get_wscale(pbuf_t *, int, u_int16_t, sa_family_t); -static u_int16_t pf_get_mss(struct mbuf *, int, u_int16_t, +static u_int16_t pf_get_mss(pbuf_t *, int, u_int16_t, sa_family_t); static u_int16_t pf_calc_mss(struct pf_addr *, sa_family_t, u_int16_t); static void pf_set_rt_ifp(struct pf_state *, struct pf_addr *, sa_family_t af); -static int pf_check_proto_cksum(struct mbuf *, int, int, +static int pf_check_proto_cksum(pbuf_t *, int, int, u_int8_t, sa_family_t); static int pf_addr_wrap_neq(struct pf_addr_wrap *, struct pf_addr_wrap *); @@ -345,35 +336,37 @@ struct pf_pool_limit pf_pool_limits[PF_LIMIT_MAX] = { { &pfr_kentry_pl, PFR_KENTRY_HIWAT }, }; -struct mbuf * -pf_lazy_makewritable(struct pf_pdesc *pd, struct mbuf *m, int len) +void * +pf_lazy_makewritable(struct pf_pdesc *pd, pbuf_t *pbuf, int len) { + void *p; + if (pd->lmw < 0) - return (0); + return (NULL); - VERIFY(m == pd->mp); + VERIFY(pbuf == pd->mp); + p = pbuf->pb_data; if (len > pd->lmw) { - if (m_makewritable(&m, 0, len, M_DONTWAIT)) + if ((p = pbuf_ensure_writable(pbuf, len)) == NULL) len = -1; pd->lmw = len; - if (len >= 0 && m != pd->mp) { - pd->mp = m; - pd->pf_mtag = pf_find_mtag(m); + if (len >= 0) { + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); switch (pd->af) { case AF_INET: { - struct ip *h = mtod(m, struct ip *); - pd->src = (struct pf_addr *)&h->ip_src; - pd->dst = (struct pf_addr *)&h->ip_dst; + struct ip *h = p; + pd->src = (struct pf_addr *)(uintptr_t)&h->ip_src; + pd->dst = (struct pf_addr *)(uintptr_t)&h->ip_dst; pd->ip_sum = &h->ip_sum; break; } #if INET6 case AF_INET6: { - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); - pd->src = (struct pf_addr *)&h->ip6_src; - pd->dst = (struct pf_addr *)&h->ip6_dst; + struct ip6_hdr *h = p; + pd->src = (struct pf_addr *)(uintptr_t)&h->ip6_src; + pd->dst = (struct pf_addr *)(uintptr_t)&h->ip6_dst; break; } #endif /* INET6 */ @@ -381,7 +374,7 @@ pf_lazy_makewritable(struct pf_pdesc *pd, struct mbuf *m, int len) } } - return (len < 0 ? 0 : m); + return (len < 0 ? NULL : p); } static const int * @@ -1222,13 +1215,13 @@ pf_src_connlimit(struct pf_state **state) #if INET case AF_INET: p.pfra_net = 32; - p.pfra_ip4addr = (*state)->src_node->addr.v4; + p.pfra_ip4addr = (*state)->src_node->addr.v4addr; break; #endif /* INET */ #if INET6 case AF_INET6: p.pfra_net = 128; - p.pfra_ip6addr = (*state)->src_node->addr.v6; + p.pfra_ip6addr = (*state)->src_node->addr.v6addr; break; #endif /* INET6 */ } @@ -1532,7 +1525,7 @@ pf_state_expires(const struct pf_state *state) u_int32_t end; u_int32_t states; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); /* handle all PFTM_* > PFTM_MAX here */ if (state->timeout == PFTM_PURGE) @@ -1567,7 +1560,7 @@ pf_purge_expired_src_nodes(void) { struct pf_src_node *cur, *next; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); for (cur = RB_MIN(pf_src_tree, &tree_src_tracking); cur; cur = next) { next = RB_NEXT(pf_src_tree, &tree_src_tracking, cur); @@ -1592,7 +1585,7 @@ pf_src_tree_remove_state(struct pf_state *s) { u_int32_t t; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (s->src_node != NULL) { if (s->src.tcp_est) { @@ -1622,7 +1615,7 @@ pf_src_tree_remove_state(struct pf_state *s) void pf_unlink_state(struct pf_state *cur) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (cur->src.state == PF_TCPS_PROXY_DST) { pf_send_tcp(cur->rule.ptr, cur->state_key->af_lan, @@ -1649,7 +1642,7 @@ pf_unlink_state(struct pf_state *cur) void pf_free_state(struct pf_state *cur) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); #if NPFSYNC if (pfsyncif != NULL && (pfsyncif->sc_bulk_send_next == cur || @@ -1689,7 +1682,7 @@ pf_purge_expired_states(u_int32_t maxcheck) static struct pf_state *cur = NULL; struct pf_state *next; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); while (maxcheck--) { /* wrap to start of list when we hit the end */ @@ -1716,7 +1709,7 @@ pf_purge_expired_states(u_int32_t maxcheck) int pf_tbladdr_setup(struct pf_ruleset *rs, struct pf_addr_wrap *aw) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (aw->type != PF_ADDR_TABLE) return (0); @@ -1728,7 +1721,7 @@ pf_tbladdr_setup(struct pf_ruleset *rs, struct pf_addr_wrap *aw) void pf_tbladdr_remove(struct pf_addr_wrap *aw) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (aw->type != PF_ADDR_TABLE || aw->p.tbl == NULL) return; @@ -1741,7 +1734,7 @@ pf_tbladdr_copyout(struct pf_addr_wrap *aw) { struct pfr_ktable *kt = aw->p.tbl; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (aw->type != PF_ADDR_TABLE || kt == NULL) return; @@ -2096,7 +2089,7 @@ pf_cksum_fixup(u_int16_t cksum, u_int16_t old, u_int16_t new, u_int8_t udp) * only the checksum is recalculated & updated. */ static void -pf_change_ap(int dir, struct mbuf *m, struct pf_addr *a, u_int16_t *p, +pf_change_ap(int dir, pbuf_t *pbuf, struct pf_addr *a, u_int16_t *p, u_int16_t *ic, u_int16_t *pc, struct pf_addr *an, u_int16_t pn, u_int8_t u, sa_family_t af, sa_family_t afn, int ua) { @@ -2126,9 +2119,8 @@ pf_change_ap(int dir, struct mbuf *m, struct pf_addr *a, u_int16_t *p, * will have UDP/TCP CSUM flag set (gets set in protocol * output). */ - if (dir == PF_OUT && m != NULL && - (m->m_flags & M_PKTHDR) && - (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))) { + if (dir == PF_OUT && pbuf != NULL && + (*pbuf->pb_csum_flags & (CSUM_TCP | CSUM_UDP))) { /* Pseudo-header checksum does not include ports */ *pc = ~pf_cksum_fixup(pf_cksum_fixup(~*pc, ao.addr16[0], an->addr16[0], u), @@ -2175,9 +2167,8 @@ pf_change_ap(int dir, struct mbuf *m, struct pf_addr *a, u_int16_t *p, * will have UDP/TCP CSUM flag set (gets set in protocol * output). */ - if (dir == PF_OUT && m != NULL && - (m->m_flags & M_PKTHDR) && - (m->m_pkthdr.csum_flags & (CSUM_TCPIPV6 | + if (dir == PF_OUT && pbuf != NULL && + (*pbuf->pb_csum_flags & (CSUM_TCPIPV6 | CSUM_UDPIPV6))) { /* Pseudo-header checksum does not include ports */ *pc = @@ -2280,7 +2271,7 @@ pf_change_addr(struct pf_addr *a, u_int16_t *c, struct pf_addr *an, u_int8_t u, case AF_INET: switch (afn) { case AF_INET: - pf_change_a(a, c, an->v4.s_addr, u); + pf_change_a(a, c, an->v4addr.s_addr, u); break; case AF_INET6: *c = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup( @@ -2413,7 +2404,7 @@ pf_change_icmp(struct pf_addr *ia, u_int16_t *ip, struct pf_addr *oa, * (credits to Krzysztof Pfaff for report and patch) */ static int -pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd, +pf_modulate_sack(pbuf_t *pbuf, int off, struct pf_pdesc *pd, struct tcphdr *th, struct pf_state_peer *dst) { int hlen = (th->th_off << 2) - sizeof (*th), thoptlen = hlen; @@ -2423,7 +2414,7 @@ pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd, #define TCPOLEN_SACKLEN (TCPOLEN_SACK + 2) if (hlen < TCPOLEN_SACKLEN || - !pf_pull_hdr(m, off + sizeof (*th), opts, hlen, NULL, NULL, pd->af)) + !pf_pull_hdr(pbuf, off + sizeof (*th), opts, hlen, NULL, NULL, pd->af)) return (0); while (hlen >= TCPOLEN_SACKLEN) { @@ -2461,14 +2452,25 @@ pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd, } if (copyback) { - m = pf_lazy_makewritable(pd, m, copyback); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, copyback) == NULL) return (-1); - m_copyback(m, off + sizeof (*th), thoptlen, opts); + pbuf_copy_back(pbuf, off + sizeof (*th), thoptlen, opts); } return (copyback); } +/* + * XXX + * + * The following functions (pf_send_tcp and pf_send_icmp) are somewhat + * special in that they originate "spurious" packets rather than + * filter/NAT existing packets. As such, they're not a great fit for + * the 'pbuf' shim, which assumes the underlying packet buffers are + * allocated elsewhere. + * + * Since these functions are rarely used, we'll carry on allocating mbufs + * and passing them to the IP stack for eventual routing. + */ static void pf_send_tcp(const struct pf_rule *r, sa_family_t af, const struct pf_addr *saddr, const struct pf_addr *daddr, @@ -2515,10 +2517,8 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af, if (m == NULL) return; - if ((pf_mtag = pf_get_mtag(m)) == NULL) { - m_free(m); + if ((pf_mtag = pf_get_mtag(m)) == NULL) return; - } if (tag) pf_mtag->pftag_flags |= PF_TAG_GENERATED; @@ -2527,11 +2527,6 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af, if (r != NULL && PF_RTABLEID_IS_VALID(r->rtableid)) pf_mtag->pftag_rtableid = r->rtableid; -#if PF_ALTQ - if (altq_allowed && r != NULL && r->qid) - pf_mtag->pftag_qid = r->qid; -#endif /* PF_ALTQ */ - #if PF_ECN /* add hints for ecn */ pf_mtag->pftag_hdr = mtod(m, struct ip *); @@ -2567,8 +2562,8 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af, /* IP header fields included in the TCP checksum */ h->ip_p = IPPROTO_TCP; h->ip_len = htons(tlen); - h->ip_src.s_addr = saddr->v4.s_addr; - h->ip_dst.s_addr = daddr->v4.s_addr; + h->ip_src.s_addr = saddr->v4addr.s_addr; + h->ip_dst.s_addr = daddr->v4addr.s_addr; th = (struct tcphdr *)(void *)((caddr_t)h + sizeof (struct ip)); break; @@ -2580,8 +2575,8 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af, /* IP header fields included in the TCP checksum */ h6->ip6_nxt = IPPROTO_TCP; h6->ip6_plen = htons(tlen); - memcpy(&h6->ip6_src, &saddr->v6, sizeof (struct in6_addr)); - memcpy(&h6->ip6_dst, &daddr->v6, sizeof (struct in6_addr)); + memcpy(&h6->ip6_src, &saddr->v6addr, sizeof (struct in6_addr)); + memcpy(&h6->ip6_dst, &daddr->v6addr, sizeof (struct in6_addr)); th = (struct tcphdr *)(void *) ((caddr_t)h6 + sizeof (struct ip6_hdr)); @@ -2655,13 +2650,13 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af, } static void -pf_send_icmp(struct mbuf *m, u_int8_t type, u_int8_t code, sa_family_t af, +pf_send_icmp(pbuf_t *pbuf, u_int8_t type, u_int8_t code, sa_family_t af, struct pf_rule *r) { struct mbuf *m0; struct pf_mtag *pf_mtag; - m0 = m_copy(m, 0, M_COPYALL); + m0 = pbuf_clone_to_mbuf(pbuf); if (m0 == NULL) return; @@ -2673,11 +2668,6 @@ pf_send_icmp(struct mbuf *m, u_int8_t type, u_int8_t code, sa_family_t af, if (PF_RTABLEID_IS_VALID(r->rtableid)) pf_mtag->pftag_rtableid = r->rtableid; -#if PF_ALTQ - if (altq_allowed && r->qid) - pf_mtag->pftag_qid = r->qid; -#endif /* PF_ALTQ */ - #if PF_ECN /* add hints for ecn */ pf_mtag->pftag_hdr = mtod(m0, struct ip *); @@ -2886,10 +2876,9 @@ pf_match_gid(u_int8_t op, gid_t a1, gid_t a2, gid_t g) } static int -pf_match_tag(struct mbuf *m, struct pf_rule *r, struct pf_mtag *pf_mtag, +pf_match_tag(struct pf_rule *r, struct pf_mtag *pf_mtag, int *tag) { -#pragma unused(m) if (*tag == -1) *tag = pf_mtag->pftag_tag; @@ -2898,14 +2887,14 @@ pf_match_tag(struct mbuf *m, struct pf_rule *r, struct pf_mtag *pf_mtag, } int -pf_tag_packet(struct mbuf *m, struct pf_mtag *pf_mtag, int tag, +pf_tag_packet(pbuf_t *pbuf, struct pf_mtag *pf_mtag, int tag, unsigned int rtableid, struct pf_pdesc *pd) { if (tag <= 0 && !PF_RTABLEID_IS_VALID(rtableid) && (pd == NULL || !(pd->pktflags & PKTF_FLOW_ID))) return (0); - if (pf_mtag == NULL && (pf_mtag = pf_get_mtag(m)) == NULL) + if (pf_mtag == NULL && (pf_mtag = pf_get_mtag_pbuf(pbuf)) == NULL) return (1); if (tag > 0) @@ -2913,10 +2902,10 @@ pf_tag_packet(struct mbuf *m, struct pf_mtag *pf_mtag, int tag, if (PF_RTABLEID_IS_VALID(rtableid)) pf_mtag->pftag_rtableid = rtableid; if (pd != NULL && (pd->pktflags & PKTF_FLOW_ID)) { - m->m_pkthdr.pkt_flowsrc = pd->flowsrc; - m->m_pkthdr.pkt_flowid = pd->flowhash; - m->m_pkthdr.pkt_flags |= pd->pktflags; - m->m_pkthdr.pkt_proto = pd->proto; + *pbuf->pb_flowsrc = pd->flowsrc; + *pbuf->pb_flowid = pd->flowhash; + *pbuf->pb_flags |= pd->pktflags; + *pbuf->pb_proto = pd->proto; } return (0); @@ -3310,7 +3299,8 @@ static int pf_get_sport(struct pf_pdesc *pd, struct pfi_kif *kif, struct pf_rule *r, struct pf_addr *saddr, union pf_state_xport *sxport, struct pf_addr *daddr, union pf_state_xport *dxport, struct pf_addr *naddr, - union pf_state_xport *nxport, struct pf_src_node **sn) + union pf_state_xport *nxport, struct pf_src_node **sn + ) { #pragma unused(kif) struct pf_state_key_cmp key; @@ -3423,11 +3413,14 @@ pf_get_sport(struct pf_pdesc *pd, struct pfi_kif *kif, struct pf_rule *r, return (0); } else if (low == 0 && high == 0) { key.gwy.xport = *nxport; - if (pf_find_state_all(&key, PF_IN, NULL) == NULL) + if (pf_find_state_all(&key, PF_IN, NULL) == NULL + ) { return (0); + } } else if (low == high) { key.gwy.xport.port = htons(low); - if (pf_find_state_all(&key, PF_IN, NULL) == NULL) { + if (pf_find_state_all(&key, PF_IN, NULL) == NULL + ) { nxport->port = htons(low); return (0); } @@ -3443,16 +3436,16 @@ pf_get_sport(struct pf_pdesc *pd, struct pfi_kif *kif, struct pf_rule *r, /* low <= cut <= high */ for (tmp = cut; tmp <= high; ++(tmp)) { key.gwy.xport.port = htons(tmp); - if (pf_find_state_all(&key, PF_IN, NULL) == - NULL) { + if (pf_find_state_all(&key, PF_IN, NULL) == NULL + ) { nxport->port = htons(tmp); return (0); } } for (tmp = cut - 1; tmp >= low; --(tmp)) { key.gwy.xport.port = htons(tmp); - if (pf_find_state_all(&key, PF_IN, NULL) == - NULL) { + if (pf_find_state_all(&key, PF_IN, NULL) == NULL + ) { nxport->port = htons(tmp); return (0); } @@ -3477,7 +3470,7 @@ pf_get_sport(struct pf_pdesc *pd, struct pfi_kif *kif, struct pf_rule *r, } static struct pf_rule * -pf_match_translation(struct pf_pdesc *pd, struct mbuf *m, int off, +pf_match_translation(struct pf_pdesc *pd, pbuf_t *pbuf, int off, int direction, struct pfi_kif *kif, struct pf_addr *saddr, union pf_state_xport *sxport, struct pf_addr *daddr, union pf_state_xport *dxport, int rs_num) @@ -3544,10 +3537,10 @@ pf_match_translation(struct pf_pdesc *pd, struct mbuf *m, int off, else if (dst && !pf_match_xport(r->proto, r->proto_variant, &dst->xport, dxport)) r = r->skip[PF_SKIP_DST_PORT].ptr; - else if (r->match_tag && !pf_match_tag(m, r, pd->pf_mtag, &tag)) + else if (r->match_tag && !pf_match_tag(r, pd->pf_mtag, &tag)) r = TAILQ_NEXT(r, entries); else if (r->os_fingerprint != PF_OSFP_ANY && (pd->proto != - IPPROTO_TCP || !pf_osfp_match(pf_osfp_fingerprint(pd, m, + IPPROTO_TCP || !pf_osfp_match(pf_osfp_fingerprint(pd, pbuf, off, pd->hdr.tcp), r->os_fingerprint))) r = TAILQ_NEXT(r, entries); else { @@ -3565,7 +3558,7 @@ pf_match_translation(struct pf_pdesc *pd, struct mbuf *m, int off, pf_step_out_of_anchor(&asd, &ruleset, rs_num, &r, NULL, NULL); } - if (pf_tag_packet(m, pd->pf_mtag, tag, rtableid, NULL)) + if (pf_tag_packet(pbuf, pd->pf_mtag, tag, rtableid, NULL)) return (NULL); if (rm != NULL && (rm->action == PF_NONAT || rm->action == PF_NORDR || rm->action == PF_NOBINAT || @@ -3577,7 +3570,7 @@ pf_match_translation(struct pf_pdesc *pd, struct mbuf *m, int off, /* * Get address translation information for NAT/BINAT/RDR * pd : pf packet descriptor - * m : mbuf holding the packet + * pbuf : pbuf holding the packet * off : offset to protocol header * direction : direction of packet * kif : pf interface info obtained from the packet's recv interface @@ -3592,28 +3585,29 @@ pf_match_translation(struct pf_pdesc *pd, struct mbuf *m, int off, * pd->ndaddr */ static struct pf_rule * -pf_get_translation_aux(struct pf_pdesc *pd, struct mbuf *m, int off, +pf_get_translation_aux(struct pf_pdesc *pd, pbuf_t *pbuf, int off, int direction, struct pfi_kif *kif, struct pf_src_node **sn, struct pf_addr *saddr, union pf_state_xport *sxport, struct pf_addr *daddr, - union pf_state_xport *dxport, union pf_state_xport *nsxport) + union pf_state_xport *dxport, union pf_state_xport *nsxport + ) { struct pf_rule *r = NULL; pd->naf = pd->af; if (direction == PF_OUT) { - r = pf_match_translation(pd, m, off, direction, kif, saddr, + r = pf_match_translation(pd, pbuf, off, direction, kif, saddr, sxport, daddr, dxport, PF_RULESET_BINAT); if (r == NULL) - r = pf_match_translation(pd, m, off, direction, kif, + r = pf_match_translation(pd, pbuf, off, direction, kif, saddr, sxport, daddr, dxport, PF_RULESET_RDR); if (r == NULL) - r = pf_match_translation(pd, m, off, direction, kif, + r = pf_match_translation(pd, pbuf, off, direction, kif, saddr, sxport, daddr, dxport, PF_RULESET_NAT); } else { - r = pf_match_translation(pd, m, off, direction, kif, saddr, + r = pf_match_translation(pd, pbuf, off, direction, kif, saddr, sxport, daddr, dxport, PF_RULESET_RDR); if (r == NULL) - r = pf_match_translation(pd, m, off, direction, kif, + r = pf_match_translation(pd, pbuf, off, direction, kif, saddr, sxport, daddr, dxport, PF_RULESET_BINAT); } @@ -3643,7 +3637,9 @@ pf_get_translation_aux(struct pf_pdesc *pd, struct mbuf *m, int off, return (NULL); if (pf_get_sport(pd, kif, r, saddr, sxport, daddr, - dxport, nsaddr, nsxport, sn)) { + dxport, nsaddr, nsxport, sn + )) + { DPFPRINTF(PF_DEBUG_MISC, ("pf: NAT proxy port allocation " "(%u-%u) failed\n", @@ -3656,7 +3652,7 @@ pf_get_translation_aux(struct pf_pdesc *pd, struct mbuf *m, int off, * from the last 32 bits of synthesized IPv6 address */ if (r->action == PF_NAT64) { - ndaddr->v4.s_addr = daddr->addr32[3]; + ndaddr->v4addr.s_addr = daddr->addr32[3]; pd->naf = AF_INET; } break; @@ -3877,7 +3873,7 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd) switch (pd->af) { #if INET case AF_INET: - inp = in_pcblookup_hash_exists(pi, saddr->v4, sport, daddr->v4, dport, + inp = in_pcblookup_hash_exists(pi, saddr->v4addr, sport, daddr->v4addr, dport, 0, &pd->lookup.uid, &pd->lookup.gid, NULL); #if INET6 if (inp == 0) { @@ -3885,19 +3881,19 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd) memset(&s6, 0, sizeof (s6)); s6.s6_addr16[5] = htons(0xffff); - memcpy(&s6.s6_addr32[3], &saddr->v4, - sizeof (saddr->v4)); + memcpy(&s6.s6_addr32[3], &saddr->v4addr, + sizeof (saddr->v4addr)); memset(&d6, 0, sizeof (d6)); d6.s6_addr16[5] = htons(0xffff); - memcpy(&d6.s6_addr32[3], &daddr->v4, - sizeof (daddr->v4)); + memcpy(&d6.s6_addr32[3], &daddr->v4addr, + sizeof (daddr->v4addr)); inp = in6_pcblookup_hash_exists(pi, &s6, sport, &d6, dport, 0, &pd->lookup.uid, &pd->lookup.gid, NULL); if (inp == 0) { - inp = in_pcblookup_hash_exists(pi, saddr->v4, sport, - daddr->v4, dport, INPLOOKUP_WILDCARD, &pd->lookup.uid, &pd->lookup.gid, NULL); + inp = in_pcblookup_hash_exists(pi, saddr->v4addr, sport, + daddr->v4addr, dport, INPLOOKUP_WILDCARD, &pd->lookup.uid, &pd->lookup.gid, NULL); if (inp == 0) { inp = in6_pcblookup_hash_exists(pi, &s6, sport, &d6, dport, INPLOOKUP_WILDCARD, @@ -3909,8 +3905,8 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd) } #else if (inp == 0) { - inp = in_pcblookup_hash_exists(pi, saddr->v4, sport, - daddr->v4, dport, INPLOOKUP_WILDCARD, + inp = in_pcblookup_hash_exists(pi, saddr->v4addr, sport, + daddr->v4addr, dport, INPLOOKUP_WILDCARD, &pd->lookup.uid, &pd->lookup.gid, NULL); if (inp == 0) return (-1); @@ -3920,11 +3916,11 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd) #endif /* INET */ #if INET6 case AF_INET6: - inp = in6_pcblookup_hash_exists(pi, &saddr->v6, sport, &daddr->v6, + inp = in6_pcblookup_hash_exists(pi, &saddr->v6addr, sport, &daddr->v6addr, dport, 0, &pd->lookup.uid, &pd->lookup.gid, NULL); if (inp == 0) { - inp = in6_pcblookup_hash_exists(pi, &saddr->v6, sport, - &daddr->v6, dport, INPLOOKUP_WILDCARD, + inp = in6_pcblookup_hash_exists(pi, &saddr->v6addr, sport, + &daddr->v6addr, dport, INPLOOKUP_WILDCARD, &pd->lookup.uid, &pd->lookup.gid, NULL); if (inp == 0) return (-1); @@ -3940,7 +3936,7 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd) } static u_int8_t -pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af) +pf_get_wscale(pbuf_t *pbuf, int off, u_int16_t th_off, sa_family_t af) { int hlen; u_int8_t hdr[60]; @@ -3950,7 +3946,7 @@ pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af) hlen = th_off << 2; /* hlen <= sizeof (hdr) */ if (hlen <= (int)sizeof (struct tcphdr)) return (0); - if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af)) + if (!pf_pull_hdr(pbuf, off, hdr, hlen, NULL, NULL, af)) return (0); opt = hdr + sizeof (struct tcphdr); hlen -= sizeof (struct tcphdr); @@ -3980,7 +3976,7 @@ pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af) } static u_int16_t -pf_get_mss(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af) +pf_get_mss(pbuf_t *pbuf, int off, u_int16_t th_off, sa_family_t af) { int hlen; u_int8_t hdr[60]; @@ -3990,7 +3986,7 @@ pf_get_mss(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af) hlen = th_off << 2; /* hlen <= sizeof (hdr) */ if (hlen <= (int)sizeof (struct tcphdr)) return (0); - if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af)) + if (!pf_pull_hdr(pbuf, off, hdr, hlen, NULL, NULL, af)) return (0); opt = hdr + sizeof (struct tcphdr); hlen -= sizeof (struct tcphdr); @@ -4042,7 +4038,7 @@ pf_calc_mss(struct pf_addr *addr, sa_family_t af, u_int16_t offer) dst = (struct sockaddr_in *)(void *)&ro.ro_dst; dst->sin_family = AF_INET; dst->sin_len = sizeof (*dst); - dst->sin_addr = addr->v4; + dst->sin_addr = addr->v4addr; rtalloc(&ro); rt = ro.ro_rt; break; @@ -4054,7 +4050,7 @@ pf_calc_mss(struct pf_addr *addr, sa_family_t af, u_int16_t offer) dst6 = (struct sockaddr_in6 *)(void *)&ro6.ro_dst; dst6->sin6_family = AF_INET6; dst6->sin6_len = sizeof (*dst6); - dst6->sin6_addr = addr->v6; + dst6->sin6_addr = addr->v6addr; rtalloc((struct route *)&ro); rt = ro6.ro_rt; break; @@ -4166,7 +4162,7 @@ pf_tcp_iss(struct pf_pdesc *pd) u_int32_t digest[4]; if (pf_tcp_secret_init == 0) { - read_random(pf_tcp_secret, sizeof (pf_tcp_secret)); + read_frandom(pf_tcp_secret, sizeof (pf_tcp_secret)); MD5Init(&pf_tcp_secret_ctx); MD5Update(&pf_tcp_secret_ctx, pf_tcp_secret, sizeof (pf_tcp_secret)); @@ -4177,11 +4173,11 @@ pf_tcp_iss(struct pf_pdesc *pd) MD5Update(&ctx, (char *)&pd->hdr.tcp->th_sport, sizeof (u_short)); MD5Update(&ctx, (char *)&pd->hdr.tcp->th_dport, sizeof (u_short)); if (pd->af == AF_INET6) { - MD5Update(&ctx, (char *)&pd->src->v6, sizeof (struct in6_addr)); - MD5Update(&ctx, (char *)&pd->dst->v6, sizeof (struct in6_addr)); + MD5Update(&ctx, (char *)&pd->src->v6addr, sizeof (struct in6_addr)); + MD5Update(&ctx, (char *)&pd->dst->v6addr, sizeof (struct in6_addr)); } else { - MD5Update(&ctx, (char *)&pd->src->v4, sizeof (struct in_addr)); - MD5Update(&ctx, (char *)&pd->dst->v4, sizeof (struct in_addr)); + MD5Update(&ctx, (char *)&pd->src->v4addr, sizeof (struct in_addr)); + MD5Update(&ctx, (char *)&pd->dst->v4addr, sizeof (struct in_addr)); } MD5Final((u_char *)digest, &ctx); pf_tcp_iss_off += 4096; @@ -4190,60 +4186,54 @@ pf_tcp_iss(struct pf_pdesc *pd) /* * This routine is called to perform address family translation on the - * inner IP header (that may come as payload) of an ICMP(v4/6) error + * inner IP header (that may come as payload) of an ICMP(v4addr/6) error * response. */ static int -pf_change_icmp_af(struct mbuf *m, int off, +pf_change_icmp_af(pbuf_t *pbuf, int off, struct pf_pdesc *pd, struct pf_pdesc *pd2, struct pf_addr *src, struct pf_addr *dst, sa_family_t af, sa_family_t naf) { - struct mbuf *n = NULL; struct ip *ip4 = NULL; struct ip6_hdr *ip6 = NULL; - int hlen, olen, mlen; + void *hdr; + int hlen, olen; if (af == naf || (af != AF_INET && af != AF_INET6) || (naf != AF_INET && naf != AF_INET6)) return (-1); - /* split the mbuf chain on the inner ip/ip6 header boundary */ - if ((n = m_split(m, off, M_DONTWAIT)) == NULL) - return (-1); - /* old header */ olen = pd2->off - off; /* new header */ hlen = naf == AF_INET ? sizeof(*ip4) : sizeof(*ip6); - /* trim old header */ - m_adj(n, olen); - - /* prepend a new one */ - if (M_PREPEND(n, hlen, M_DONTWAIT, 0) == NULL) + /* Modify the pbuf to accommodate the new header */ + hdr = pbuf_resize_segment(pbuf, off, olen, hlen); + if (hdr == NULL) return (-1); /* translate inner ip/ip6 header */ switch (naf) { case AF_INET: - ip4 = mtod(n, struct ip *); + ip4 = hdr; bzero(ip4, sizeof(*ip4)); ip4->ip_v = IPVERSION; ip4->ip_hl = sizeof(*ip4) >> 2; ip4->ip_len = htons(sizeof(*ip4) + pd2->tot_len - olen); - ip4->ip_id = htons(ip_randomid()); + ip4->ip_id = rfc6864 ? 0 : htons(ip_randomid()); ip4->ip_off = htons(IP_DF); ip4->ip_ttl = pd2->ttl; if (pd2->proto == IPPROTO_ICMPV6) ip4->ip_p = IPPROTO_ICMP; else ip4->ip_p = pd2->proto; - ip4->ip_src = src->v4; - ip4->ip_dst = dst->v4; - ip4->ip_sum = in_cksum(n, ip4->ip_hl << 2); + ip4->ip_src = src->v4addr; + ip4->ip_dst = dst->v4addr; + ip4->ip_sum = pbuf_inet_cksum(pbuf, 0, 0, ip4->ip_hl << 2); break; case AF_INET6: - ip6 = mtod(n, struct ip6_hdr *); + ip6 = hdr; bzero(ip6, sizeof(*ip6)); ip6->ip6_vfc = IPV6_VERSION; ip6->ip6_plen = htons(pd2->tot_len - olen); @@ -4255,8 +4245,8 @@ pf_change_icmp_af(struct mbuf *m, int off, ip6->ip6_hlim = IPV6_DEFHLIM; else ip6->ip6_hlim = pd2->ttl; - ip6->ip6_src = src->v6; - ip6->ip6_dst = dst->v6; + ip6->ip6_src = src->v6addr; + ip6->ip6_dst = dst->v6addr; break; } @@ -4264,11 +4254,6 @@ pf_change_icmp_af(struct mbuf *m, int off, pd2->off += hlen - olen; pd->tot_len += hlen - olen; - /* merge modified inner packet with the original header */ - mlen = n->m_pkthdr.len; - m_cat(m, n); - m->m_pkthdr.len += mlen; - return (0); } @@ -4471,10 +4456,12 @@ pf_translate_icmp_af(int af, void *arg) return (0); } +/* Note: frees pbuf if PF_NAT64 is returned */ static int -pf_nat64_ipv6(struct mbuf *m, int off, struct pf_pdesc *pd) +pf_nat64_ipv6(pbuf_t *pbuf, int off, struct pf_pdesc *pd) { struct ip *ip4; + struct mbuf *m; /* * ip_input asserts for rcvif to be not NULL @@ -4484,17 +4471,13 @@ pf_nat64_ipv6(struct mbuf *m, int off, struct pf_pdesc *pd) * 2. If IPv6 stack in kernel internally generates a * message destined for a synthesized IPv6 end-point. */ - if (m->m_pkthdr.rcvif == NULL) + if (pbuf->pb_ifp == NULL) return (PF_DROP); - /* trim the old header */ - m_adj(m, off); - - /* prepend the new one */ - if (M_PREPEND(m, sizeof(*ip4), M_DONTWAIT, 0) == NULL) + ip4 = (struct ip *)pbuf_resize_segment(pbuf, 0, off, sizeof(*ip4)); + if (ip4 == NULL) return (PF_DROP); - ip4 = mtod(m, struct ip *); ip4->ip_v = 4; ip4->ip_hl = 5; ip4->ip_tos = pd->tos & htonl(0x0ff00000); @@ -4504,81 +4487,88 @@ pf_nat64_ipv6(struct mbuf *m, int off, struct pf_pdesc *pd) ip4->ip_ttl = pd->ttl; ip4->ip_p = pd->proto; ip4->ip_sum = 0; - ip4->ip_src = pd->naddr.v4; - ip4->ip_dst = pd->ndaddr.v4; - ip4->ip_sum = in_cksum(m, ip4->ip_hl << 2); + ip4->ip_src = pd->naddr.v4addr; + ip4->ip_dst = pd->ndaddr.v4addr; + ip4->ip_sum = pbuf_inet_cksum(pbuf, 0, 0, ip4->ip_hl << 2); /* recalculate icmp checksums */ if (pd->proto == IPPROTO_ICMP) { - struct mbuf *mp; struct icmp *icmp; - int moff, hlen = sizeof(*ip4); + int hlen = sizeof(*ip4); - if ((mp = m_pulldown(m, hlen, ICMP_MINLEN, &moff)) == NULL) - return (PF_DROP); + icmp = (struct icmp *)pbuf_contig_segment(pbuf, hlen, + ICMP_MINLEN); + if (icmp == NULL) + return (PF_NAT64); - icmp = (struct icmp *)(void *)(mtod(mp, char *) + moff); icmp->icmp_cksum = 0; - icmp->icmp_cksum = inet_cksum(m, 0, hlen, + icmp->icmp_cksum = pbuf_inet_cksum(pbuf, 0, hlen, ntohs(ip4->ip_len) - hlen); } - ip_input(m); + if ((m = pbuf_to_mbuf(pbuf, TRUE)) != NULL) + ip_input(m); + return (PF_NAT64); } static int -pf_nat64_ipv4(struct mbuf *m, int off, struct pf_pdesc *pd) +pf_nat64_ipv4(pbuf_t *pbuf, int off, struct pf_pdesc *pd) { struct ip6_hdr *ip6; + struct mbuf *m; - if (m->m_pkthdr.rcvif == NULL) + if (pbuf->pb_ifp == NULL) return (PF_DROP); - m_adj(m, off); - if (M_PREPEND(m, sizeof(*ip6), M_DONTWAIT, 0) == NULL) + ip6 = (struct ip6_hdr *)pbuf_resize_segment(pbuf, 0, off, sizeof(*ip6)); + if (ip6 == NULL) return (PF_DROP); - ip6 = mtod(m, struct ip6_hdr *); ip6->ip6_vfc = htonl((6 << 28) | (pd->tos << 20)); ip6->ip6_plen = htons(pd->tot_len - off); ip6->ip6_nxt = pd->proto; ip6->ip6_hlim = pd->ttl; - ip6->ip6_src = pd->naddr.v6; - ip6->ip6_dst = pd->ndaddr.v6; + ip6->ip6_src = pd->naddr.v6addr; + ip6->ip6_dst = pd->ndaddr.v6addr; /* recalculate icmp6 checksums */ if (pd->proto == IPPROTO_ICMPV6) { - struct mbuf *mp; struct icmp6_hdr *icmp6; - int moff, hlen = sizeof(*ip6); + int hlen = sizeof(*ip6); - if ((mp = m_pulldown(m, hlen, sizeof(*icmp6), &moff)) == NULL) + icmp6 = (struct icmp6_hdr *)pbuf_contig_segment(pbuf, hlen, + sizeof(*icmp6)); + if (icmp6 == NULL) return (PF_DROP); - icmp6 = (struct icmp6_hdr *)(void *)(mtod(mp, char *) + moff); icmp6->icmp6_cksum = 0; - icmp6->icmp6_cksum = inet6_cksum(m, IPPROTO_ICMPV6, hlen, - ntohs(ip6->ip6_plen)); + icmp6->icmp6_cksum = pbuf_inet6_cksum(pbuf, + IPPROTO_ICMPV6, hlen, + ntohs(ip6->ip6_plen)); } else if (pd->proto == IPPROTO_UDP) { - struct mbuf *mp; struct udphdr *uh; - int moff, hlen = sizeof(*ip6); - if ((mp = m_pulldown(m, hlen, sizeof(*uh), &moff)) == NULL) + int hlen = sizeof(*ip6); + + uh = (struct udphdr *)pbuf_contig_segment(pbuf, hlen, + sizeof(*uh)); + if (uh == NULL) return (PF_DROP); - uh = (struct udphdr *)(void *)(mtod(mp, char *) + moff); + if (uh->uh_sum == 0) - uh->uh_sum = inet6_cksum(m, IPPROTO_UDP, hlen, - ntohs(ip6->ip6_plen)); + uh->uh_sum = pbuf_inet6_cksum(pbuf, IPPROTO_UDP, + hlen, ntohs(ip6->ip6_plen)); } - ip6_input(m); + if ((m = pbuf_to_mbuf(pbuf, TRUE)) != NULL) + ip6_input(m); + return (PF_NAT64); } static int pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, - struct pfi_kif *kif, struct mbuf *m, int off, void *h, + struct pfi_kif *kif, pbuf_t *pbuf, int off, void *h, struct pf_pdesc *pd, struct pf_rule **am, struct pf_ruleset **rsm, struct ifqueue *ifq) { @@ -4605,7 +4595,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, union pf_state_xport bxport, bdxport, nxport, sxport, dxport; struct pf_state_key psk; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (direction == PF_IN && pf_check_congestion(ifq)) { REASON_SET(&reason, PFRES_CONGEST); @@ -4686,9 +4676,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, nxport = dxport; /* check packet for BINAT/NAT/RDR */ - if ((nr = pf_get_translation_aux(pd, m, off, direction, kif, &nsn, - saddr, &sxport, daddr, &dxport, &nxport)) != - NULL) { + if ((nr = pf_get_translation_aux(pd, pbuf, off, direction, kif, &nsn, + saddr, &sxport, daddr, &dxport, &nxport + )) != NULL) { int ua; u_int16_t dport; @@ -4765,8 +4755,8 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, * Will cross the bridge when it comes. */ if (PF_ANEQ(saddr, &pd->naddr, pd->af)) { - pf_change_a(&saddr->v4.s_addr, pd->ip_sum, - pd->naddr.v4.s_addr, 0); + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, + pd->naddr.v4addr.s_addr, 0); pd->hdr.icmp->icmp_cksum = pf_cksum_fixup( pd->hdr.icmp->icmp_cksum, sxport.port, nxport.port, 0); @@ -4774,8 +4764,8 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, } if (PF_ANEQ(daddr, &pd->ndaddr, pd->af)) { - pf_change_a(&daddr->v4.s_addr, pd->ip_sum, - pd->ndaddr.v4.s_addr, 0); + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, + pd->ndaddr.v4addr.s_addr, 0); } ++rewrite; break; @@ -4817,14 +4807,14 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, #if INET case AF_INET: if (PF_ANEQ(saddr, &pd->naddr, pd->af)) { - pf_change_a(&saddr->v4.s_addr, + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, - pd->naddr.v4.s_addr, 0); + pd->naddr.v4addr.s_addr, 0); } if (PF_ANEQ(daddr, &pd->ndaddr, pd->af)) { - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - pd->ndaddr.v4.s_addr, 0); + pd->ndaddr.v4addr.s_addr, 0); } break; #endif /* INET */ @@ -4847,13 +4837,13 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, #if INET case AF_INET: if (PF_ANEQ(saddr, &pd->naddr, pd->af)) { - pf_change_a(&saddr->v4.s_addr, - pd->ip_sum, pd->naddr.v4.s_addr, 0); + pf_change_a(&saddr->v4addr.s_addr, + pd->ip_sum, pd->naddr.v4addr.s_addr, 0); } if (PF_ANEQ(daddr, &pd->ndaddr, pd->af)) { - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - pd->ndaddr.v4.s_addr, 0); + pd->ndaddr.v4addr.s_addr, 0); } break; #endif /* INET */ @@ -4900,6 +4890,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, r = NULL; pd->nat_rule = nr; pd->af = pd->naf; + } else { } if (nr && nr->tag > 0) @@ -4971,11 +4962,11 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, r = TAILQ_NEXT(r, entries); else if (r->prob && r->prob <= (RandomULong() % (UINT_MAX - 1) + 1)) r = TAILQ_NEXT(r, entries); - else if (r->match_tag && !pf_match_tag(m, r, pd->pf_mtag, &tag)) + else if (r->match_tag && !pf_match_tag(r, pd->pf_mtag, &tag)) r = TAILQ_NEXT(r, entries); else if (r->os_fingerprint != PF_OSFP_ANY && (pd->proto != IPPROTO_TCP || !pf_osfp_match( - pf_osfp_fingerprint(pd, m, off, th), + pf_osfp_fingerprint(pd, pbuf, off, th), r->os_fingerprint))) r = TAILQ_NEXT(r, entries); else { @@ -5010,15 +5001,14 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, if (rewrite < off + hdrlen) rewrite = off + hdrlen; - m = pf_lazy_makewritable(pd, m, rewrite); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, rewrite) == NULL) { REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } - m_copyback(m, off, hdrlen, pd->hdr.any); + pbuf_copy_back(pbuf, off, hdrlen, pd->hdr.any); } - PFLOG_PACKET(kif, h, m, pd->af, direction, reason, + PFLOG_PACKET(kif, h, pbuf, pd->af, direction, reason, r->log ? r : nr, a, ruleset, pd); } @@ -5060,9 +5050,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, switch (af) { #if INET case AF_INET: - pf_change_a(&saddr->v4.s_addr, + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, - pd->baddr.v4.s_addr, 0); + pd->baddr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -5078,9 +5068,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, switch (af) { #if INET case AF_INET: - pf_change_a(&saddr->v4.s_addr, + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, - pd->baddr.v4.s_addr, 0); + pd->baddr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -5094,9 +5084,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, default: switch (af) { case AF_INET: - pf_change_a(&saddr->v4.s_addr, + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, - pd->baddr.v4.s_addr, 0); + pd->baddr.v4addr.s_addr, 0); break; case AF_INET6: PF_ACPY(saddr, &pd->baddr, af); @@ -5136,9 +5126,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, switch (af) { #if INET case AF_INET: - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - pd->bdaddr.v4.s_addr, 0); + pd->bdaddr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -5153,9 +5143,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, switch (af) { #if INET case AF_INET: - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - pd->bdaddr.v4.s_addr, 0); + pd->bdaddr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -5169,9 +5159,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, default: switch (af) { case AF_INET: - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - pd->bdaddr.v4.s_addr, 0); + pd->bdaddr.v4addr.s_addr, 0); break; #if INET6 case AF_INET6: @@ -5195,19 +5185,19 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, switch (pd->af) { case AF_INET: - h4 = mtod(m, struct ip *); + h4 = pbuf->pb_data; len = ntohs(h4->ip_len) - off; break; #if INET6 case AF_INET6: - h6 = mtod(m, struct ip6_hdr *); + h6 = pbuf->pb_data; len = ntohs(h6->ip6_plen) - (off - sizeof (*h6)); break; #endif /* INET6 */ } - if (pf_check_proto_cksum(m, off, len, IPPROTO_TCP, + if (pf_check_proto_cksum(pbuf, off, len, IPPROTO_TCP, pd->af)) REASON_SET(&reason, PFRES_PROTCKSUM); else { @@ -5223,17 +5213,18 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, } else if (pd->proto != IPPROTO_ICMP && pd->af == AF_INET && pd->proto != IPPROTO_ESP && pd->proto != IPPROTO_AH && r->return_icmp) - pf_send_icmp(m, r->return_icmp >> 8, + pf_send_icmp(pbuf, r->return_icmp >> 8, r->return_icmp & 255, pd->af, r); else if (pd->proto != IPPROTO_ICMPV6 && af == AF_INET6 && pd->proto != IPPROTO_ESP && pd->proto != IPPROTO_AH && r->return_icmp6) - pf_send_icmp(m, r->return_icmp6 >> 8, + pf_send_icmp(pbuf, r->return_icmp6 >> 8, r->return_icmp6 & 255, pd->af, r); } - if (r->action == PF_DROP) + if (r->action == PF_DROP) { return (PF_DROP); + } /* prepare state key, for flowhash and/or the state (if created) */ bzero(&psk, sizeof (psk)); @@ -5268,7 +5259,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, * NAT64 requires protocol translation between ICMPv4 * and ICMPv6. TCP and UDP do not require protocol * translation. To avoid adding complexity just to - * handle ICMP(v4/v6), we always lookup for + * handle ICMP(v4addr/v6addr), we always lookup for * proto = IPPROTO_ICMP on both LAN and WAN side */ psk.proto = IPPROTO_ICMP; @@ -5310,7 +5301,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, * NAT64 requires protocol translation between ICMPv4 * and ICMPv6. TCP and UDP do not require protocol * translation. To avoid adding complexity just to - * handle ICMP(v4/v6), we always lookup for + * handle ICMP(v4addr/v6addr), we always lookup for * proto = IPPROTO_ICMP on both LAN and WAN side */ psk.proto = IPPROTO_ICMP; @@ -5382,7 +5373,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, pd->pktflags &= ~PKTF_FLOW_ADV; } - if (pf_tag_packet(m, pd->pf_mtag, tag, rtableid, pd)) { + if (pf_tag_packet(pbuf, pd->pf_mtag, tag, rtableid, pd)) { REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } @@ -5396,14 +5387,15 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, struct pf_ike_hdr ike; if (pd->proto == IPPROTO_UDP) { - size_t plen = m->m_pkthdr.len - off - sizeof (*uh); + size_t plen = pbuf->pb_packet_len - off - sizeof(*uh); if (ntohs(uh->uh_sport) == PF_IKE_PORT && ntohs(uh->uh_dport) == PF_IKE_PORT && plen >= PF_IKE_PACKET_MINSIZE) { if (plen > PF_IKE_PACKET_MINSIZE) plen = PF_IKE_PACKET_MINSIZE; - m_copydata(m, off + sizeof (*uh), plen, &ike); + pbuf_copy_data(pbuf, off + sizeof (*uh), plen, + &ike); } } @@ -5507,7 +5499,7 @@ cleanup: s->src.seqdiff = 0; if (th->th_flags & TH_SYN) { s->src.seqhi++; - s->src.wscale = pf_get_wscale(m, off, + s->src.wscale = pf_get_wscale(pbuf, off, th->th_off, af); } s->src.max_win = MAX(ntohs(th->th_win), 1); @@ -5569,7 +5561,7 @@ cleanup: } if (pd->proto == IPPROTO_TCP) { if ((pd->flags & PFDESC_TCP_NORM) && - pf_normalize_tcp_init(m, off, pd, th, &s->src, + pf_normalize_tcp_init(pbuf, off, pd, th, &s->src, &s->dst)) { REASON_SET(&reason, PFRES_MEMORY); pf_src_tree_remove_state(s); @@ -5578,7 +5570,7 @@ cleanup: return (PF_DROP); } if ((pd->flags & PFDESC_TCP_NORM) && s->src.scrub && - pf_normalize_tcp_stateful(m, off, pd, &reason, + pf_normalize_tcp_stateful(pbuf, off, pd, &reason, th, s, &s->src, &s->dst, &rewrite)) { /* This really shouldn't happen!!! */ DPFPRINTF(PF_DEBUG_URGENT, @@ -5595,12 +5587,17 @@ cleanup: /* allocate state key and import values from psk */ if ((sk = pf_alloc_state_key(s, &psk)) == NULL) { REASON_SET(&reason, PFRES_MEMORY); + /* + * XXXSCW: This will leak the freshly-allocated + * state structure 's'. Although it should + * eventually be aged-out and removed. + */ goto cleanup; } pf_set_rt_ifp(s, saddr, af); /* needs s->state_key set */ - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? if (sk->app_state == 0) { switch (pd->proto) { @@ -5668,8 +5665,9 @@ cleanup: STATE_DEC_COUNTERS(s); pool_put(&pf_state_pl, s); return (PF_DROP); - } else + } else { *sm = s; + } if (tag > 0) { pf_tag_ref(tag); s->tag = tag; @@ -5696,7 +5694,7 @@ cleanup: } s->src.seqhi = htonl(random()); /* Find mss option */ - mss = pf_get_mss(m, off, th->th_off, af); + mss = pf_get_mss(pbuf, off, th->th_off, af); mss = pf_calc_mss(saddr, af, mss); mss = pf_calc_mss(daddr, af, mss); s->src.mss = mss; @@ -5729,7 +5727,7 @@ cleanup: REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } } } @@ -5739,23 +5737,60 @@ cleanup: if (rewrite < off + hdrlen) rewrite = off + hdrlen; - m = pf_lazy_makewritable(pd, pd->mp, rewrite); - if (!m) { + if (pf_lazy_makewritable(pd, pd->mp, rewrite) == NULL) { REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } - m_copyback(m, off, hdrlen, pd->hdr.any); + pbuf_copy_back(pbuf, off, hdrlen, pd->hdr.any); if (af == AF_INET6 && pd->naf == AF_INET) - return pf_nat64_ipv6(m, off, pd); + return pf_nat64_ipv6(pbuf, off, pd); else if (af == AF_INET && pd->naf == AF_INET6) - return pf_nat64_ipv4(m, off, pd); + return pf_nat64_ipv4(pbuf, off, pd); } return (PF_PASS); } +boolean_t is_nlc_enabled_glb = FALSE; + +static inline boolean_t +pf_is_dummynet_enabled(void) +{ +#if DUMMYNET + if (__probable(!PF_IS_ENABLED)) + return (FALSE); + + if (__probable(!DUMMYNET_LOADED)) + return (FALSE); + + if (__probable(TAILQ_EMPTY(pf_main_ruleset. + rules[PF_RULESET_DUMMYNET].active.ptr))) + return (FALSE); + + return (TRUE); +#else + return (FALSE); +#endif /* DUMMYNET */ +} + +boolean_t +pf_is_nlc_enabled(void) +{ +#if DUMMYNET + if (__probable(!pf_is_dummynet_enabled())) + return (FALSE); + + if (__probable(!is_nlc_enabled_glb)) + return (FALSE); + + return (TRUE); +#else + return (FALSE); +#endif /* DUMMYNET */ +} + #if DUMMYNET /* * When pf_test_dummynet() returns PF_PASS, the rule matching parameter "rm" @@ -5766,9 +5801,9 @@ cleanup: */ static int pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, - struct mbuf **m0, struct pf_pdesc *pd, struct ip_fw_args *fwa) + pbuf_t **pbuf0, struct pf_pdesc *pd, struct ip_fw_args *fwa) { - struct mbuf *m = *m0; + pbuf_t *pbuf = *pbuf0; struct pf_rule *am = NULL; struct pf_ruleset *rsm = NULL; struct pf_addr *saddr = pd->src, *daddr = pd->dst; @@ -5787,12 +5822,9 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, struct pf_rule *prev_matching_rule = fwa ? fwa->fwa_pf_rule : NULL; int found_prev_rule = (prev_matching_rule) ? 0 : 1; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); - if (!DUMMYNET_LOADED) - return (PF_PASS); - - if (TAILQ_EMPTY(pf_main_ruleset.rules[PF_RULESET_DUMMYNET].active.ptr)) + if (!pf_is_dummynet_enabled()) return (PF_PASS); bzero(&dnflow, sizeof(dnflow)); @@ -5896,7 +5928,7 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, r = TAILQ_NEXT(r, entries); else if (r->prob && r->prob <= (RandomULong() % (UINT_MAX - 1) + 1)) r = TAILQ_NEXT(r, entries); - else if (r->match_tag && !pf_match_tag(m, r, pd->pf_mtag, &tag)) + else if (r->match_tag && !pf_match_tag(r, pd->pf_mtag, &tag)) r = TAILQ_NEXT(r, entries); else { /* @@ -5937,7 +5969,7 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, REASON_SET(&reason, PFRES_DUMMYNET); if (r->log) { - PFLOG_PACKET(kif, h, m, af, direction, reason, r, + PFLOG_PACKET(kif, h, pbuf, af, direction, reason, r, a, ruleset, pd); } @@ -5949,13 +5981,14 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, return (PF_PASS); } - if (pf_tag_packet(m, pd->pf_mtag, tag, rtableid, pd)) { + if (pf_tag_packet(pbuf, pd->pf_mtag, tag, rtableid, pd)) { REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } if (r->dnpipe && ip_dn_io_ptr != NULL) { + struct mbuf *m; int dirndx = (direction == PF_OUT); r->packets[dirndx]++; @@ -5968,13 +6001,13 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, switch (af) { case AF_INET: dnflow.fwa_id.addr_type = 4; - dnflow.fwa_id.src_ip = ntohl(saddr->v4.s_addr); - dnflow.fwa_id.dst_ip = ntohl(daddr->v4.s_addr); + dnflow.fwa_id.src_ip = ntohl(saddr->v4addr.s_addr); + dnflow.fwa_id.dst_ip = ntohl(daddr->v4addr.s_addr); break; case AF_INET6: dnflow.fwa_id.addr_type = 6; - dnflow.fwa_id.src_ip6 = saddr->v6; - dnflow.fwa_id.dst_ip6 = saddr->v6; + dnflow.fwa_id.src_ip6 = saddr->v6addr; + dnflow.fwa_id.dst_ip6 = saddr->v6addr; break; } @@ -5998,7 +6031,7 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, } if (af == AF_INET) { - struct ip *iphdr = mtod(m, struct ip *); + struct ip *iphdr = pbuf->pb_data; NTOHS(iphdr->ip_len); NTOHS(iphdr->ip_off); } @@ -6006,18 +6039,20 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, * Don't need to unlock pf_lock as NET_THREAD_HELD_PF * allows for recursive behavior */ - ip_dn_io_ptr(m, - dnflow.fwa_cookie, - af == AF_INET ? - direction == PF_IN ? DN_TO_IP_IN : DN_TO_IP_OUT : - direction == PF_IN ? DN_TO_IP6_IN : DN_TO_IP6_OUT, - &dnflow, DN_CLIENT_PF); + m = pbuf_to_mbuf(pbuf, TRUE); + if (m != NULL) { + ip_dn_io_ptr(m, + dnflow.fwa_cookie, (af == AF_INET) ? + ((direction==PF_IN) ? DN_TO_IP_IN : DN_TO_IP_OUT) : + ((direction==PF_IN) ? DN_TO_IP6_IN : DN_TO_IP6_OUT), + &dnflow, DN_CLIENT_PF); + } /* * The packet is siphoned out by dummynet so return a NULL - * mbuf so the caller can still return success. + * pbuf so the caller can still return success. */ - *m0 = NULL; + *pbuf0 = NULL; return (PF_PASS); } @@ -6028,7 +6063,7 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, static int pf_test_fragment(struct pf_rule **rm, int direction, struct pfi_kif *kif, - struct mbuf *m, void *h, struct pf_pdesc *pd, struct pf_rule **am, + pbuf_t *pbuf, void *h, struct pf_pdesc *pd, struct pf_rule **am, struct pf_ruleset **rsm) { #pragma unused(h) @@ -6081,7 +6116,7 @@ pf_test_fragment(struct pf_rule **rm, int direction, struct pfi_kif *kif, r = TAILQ_NEXT(r, entries); else if (r->prob && r->prob <= (RandomULong() % (UINT_MAX - 1) + 1)) r = TAILQ_NEXT(r, entries); - else if (r->match_tag && !pf_match_tag(m, r, pd->pf_mtag, &tag)) + else if (r->match_tag && !pf_match_tag(r, pd->pf_mtag, &tag)) r = TAILQ_NEXT(r, entries); else { if (r->anchor == NULL) { @@ -6107,13 +6142,13 @@ pf_test_fragment(struct pf_rule **rm, int direction, struct pfi_kif *kif, REASON_SET(&reason, PFRES_MATCH); if (r->log) - PFLOG_PACKET(kif, h, m, af, direction, reason, r, a, ruleset, + PFLOG_PACKET(kif, h, pbuf, af, direction, reason, r, a, ruleset, pd); if (r->action != PF_PASS) return (PF_DROP); - if (pf_tag_packet(m, pd->pf_mtag, tag, -1, NULL)) { + if (pf_tag_packet(pbuf, pd->pf_mtag, tag, -1, NULL)) { REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } @@ -6138,7 +6173,7 @@ pf_pptp_handler(struct pf_state *s, int direction, int off, u_int8_t *pac_state; u_int8_t *pns_state; enum { PF_PPTP_PASS, PF_PPTP_INSERT_GRE, PF_PPTP_REMOVE_GRE } op; - struct mbuf *m; + pbuf_t *pbuf; struct pf_state_key *sk; struct pf_state_key *gsk; struct pf_app_state *gas; @@ -6150,12 +6185,12 @@ pf_pptp_handler(struct pf_state *s, int direction, int off, if (gs) gs->expire = pf_time_second(); - m = pd->mp; - plen = min(sizeof (cm), m->m_pkthdr.len - off); + pbuf = pd->mp; + plen = min(sizeof (cm), pbuf->pb_packet_len - off); if (plen < PF_PPTP_CTRL_MSG_MINSIZE) return; tlen = plen - PF_PPTP_CTRL_MSG_MINSIZE; - m_copydata(m, off, plen, &cm); + pbuf_copy_data(pbuf, off, plen, &cm); if (ntohl(cm.hdr.magic) != PF_PPTP_MAGIC_NUMBER) return; @@ -6399,14 +6434,13 @@ pf_pptp_handler(struct pf_state *s, int direction, int off, gsk->lan.xport.call_id, gsk->gwy.xport.call_id, 0); } - m = pf_lazy_makewritable(pd, m, off + plen); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, off + plen) == NULL) { pptps->grev1_state = NULL; STATE_DEC_COUNTERS(gs); pool_put(&pf_state_pl, gs); return; } - m_copyback(m, off, plen, &cm); + pbuf_copy_back(pbuf, off, plen, &cm); } switch (op) { @@ -6499,7 +6533,7 @@ pf_ike_compare(struct pf_app_state *a, struct pf_app_state *b) } static int -pf_do_nat64(struct pf_state_key *sk, struct pf_pdesc *pd, struct mbuf *m, +pf_do_nat64(struct pf_state_key *sk, struct pf_pdesc *pd, pbuf_t *pbuf, int off) { if (pd->af == AF_INET) { @@ -6510,7 +6544,7 @@ pf_do_nat64(struct pf_state_key *sk, struct pf_pdesc *pd, struct mbuf *m, pd->naddr = sk->gwy.addr; pd->ndaddr = sk->ext_gwy.addr; } - return (pf_nat64_ipv4(m, off, pd)); + return (pf_nat64_ipv4(pbuf, off, pd)); } else if (pd->af == AF_INET6) { if (pd->af != sk->af_lan) { @@ -6520,14 +6554,14 @@ pf_do_nat64(struct pf_state_key *sk, struct pf_pdesc *pd, struct mbuf *m, pd->naddr = sk->gwy.addr; pd->ndaddr = sk->ext_gwy.addr; } - return (pf_nat64_ipv6(m, off, pd)); + return (pf_nat64_ipv6(pbuf, off, pd)); } return (PF_DROP); } static int pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, - struct mbuf *m, int off, void *h, struct pf_pdesc *pd, + pbuf_t *pbuf, int off, void *h, struct pf_pdesc *pd, u_short *reason) { #pragma unused(h) @@ -6710,7 +6744,7 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, if ((pd->flags & PFDESC_TCP_NORM || dst->scrub) && src->scrub == NULL) { - if (pf_normalize_tcp_init(m, off, pd, th, src, dst)) { + if (pf_normalize_tcp_init(pbuf, off, pd, th, src, dst)) { REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } @@ -6734,8 +6768,8 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, if (th->th_flags & TH_SYN) { end++; if (dst->wscale & PF_WSCALE_FLAG) { - src->wscale = pf_get_wscale(m, off, th->th_off, - pd->af); + src->wscale = pf_get_wscale(pbuf, off, + th->th_off, pd->af); if (src->wscale & PF_WSCALE_FLAG) { /* * Remove scale factor from initial @@ -6836,13 +6870,13 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, * options anyway. */ if (dst->seqdiff && (th->th_off << 2) > (int)sizeof (struct tcphdr)) { - copyback = pf_modulate_sack(m, off, pd, th, dst); + copyback = pf_modulate_sack(pbuf, off, pd, th, dst); if (copyback == -1) { REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } @@ -6861,11 +6895,11 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, /* Require an exact/+1 sequence match on resets when possible */ if (dst->scrub || src->scrub) { - if (pf_normalize_tcp_stateful(m, off, pd, reason, th, + if (pf_normalize_tcp_stateful(pbuf, off, pd, reason, th, *state, src, dst, ©back)) return (PF_DROP); - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } /* update max window */ @@ -6961,10 +6995,10 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, } if (dst->scrub || src->scrub) { - if (pf_normalize_tcp_stateful(m, off, pd, reason, th, + if (pf_normalize_tcp_stateful(pbuf, off, pd, reason, th, *state, src, dst, ©back)) return (PF_DROP); - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } /* update max window */ @@ -7040,7 +7074,7 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } /* translate source/destination address, if necessary */ @@ -7092,24 +7126,23 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, } if (copyback) { - m = pf_lazy_makewritable(pd, m, copyback); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, copyback) == NULL) { REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } /* Copyback sequence modulation or stateful scrub changes */ - m_copyback(m, off, sizeof (*th), th); + pbuf_copy_back(pbuf, off, sizeof (*th), th); if (sk->af_lan != sk->af_gwy) - return (pf_do_nat64(sk, pd, m, off)); + return (pf_do_nat64(sk, pd, pbuf, off)); } return (PF_PASS); } static int pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif, - struct mbuf *m, int off, void *h, struct pf_pdesc *pd, u_short *reason) + pbuf_t *pbuf, int off, void *h, struct pf_pdesc *pd, u_short *reason) { #pragma unused(h) struct pf_state_peer *src, *dst; @@ -7150,7 +7183,7 @@ pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif, if (ntohs(uh->uh_sport) == PF_IKE_PORT && ntohs(uh->uh_dport) == PF_IKE_PORT) { struct pf_ike_hdr ike; - size_t plen = m->m_pkthdr.len - off - sizeof (*uh); + size_t plen = pbuf->pb_packet_len - off - sizeof (*uh); if (plen < PF_IKE_PACKET_MINSIZE) { DPFPRINTF(PF_DEBUG_MISC, ("pf: IKE message too small.\n")); @@ -7159,7 +7192,7 @@ pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif, if (plen > sizeof (ike)) plen = sizeof (ike); - m_copydata(m, off + sizeof (*uh), plen, &ike); + pbuf_copy_data(pbuf, off + sizeof (*uh), plen, &ike); if (ike.initiator_cookie) { key.app_state = &as; @@ -7254,13 +7287,12 @@ pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif, REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } /* translate source/destination address, if necessary */ if (STATE_TRANSLATE(sk)) { - m = pf_lazy_makewritable(pd, m, off + sizeof (*uh)); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, off + sizeof (*uh)) == NULL) { REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } @@ -7309,22 +7341,22 @@ pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif, } } - m_copyback(m, off, sizeof (*uh), uh); + pbuf_copy_back(pbuf, off, sizeof (*uh), uh); if (sk->af_lan != sk->af_gwy) - return (pf_do_nat64(sk, pd, m, off)); + return (pf_do_nat64(sk, pd, pbuf, off)); } return (PF_PASS); } static int pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, - struct mbuf *m, int off, void *h, struct pf_pdesc *pd, u_short *reason) + pbuf_t *pbuf, int off, void *h, struct pf_pdesc *pd, u_short *reason) { #pragma unused(h) struct pf_addr *saddr = pd->src, *daddr = pd->dst; - struct in_addr srcv4_inaddr = saddr->v4; - u_int16_t icmpid = 0, *icmpsum; - u_int8_t icmptype; + struct in_addr srcv4_inaddr = saddr->v4addr; + u_int16_t icmpid = 0, *icmpsum = NULL; + u_int8_t icmptype = 0; int state_icmp = 0; struct pf_state_key_cmp key; struct pf_state_key *sk; @@ -7374,7 +7406,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, * NAT64 requires protocol translation between ICMPv4 * and ICMPv6. TCP and UDP do not require protocol * translation. To avoid adding complexity just to - * handle ICMP(v4/v6), we always lookup for + * handle ICMP(v4addr/v6addr), we always lookup for * proto = IPPROTO_ICMP on both LAN and WAN side */ key.proto = IPPROTO_ICMP; @@ -7404,20 +7436,19 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&saddr->v4.s_addr, + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, - sk->gwy.addr.v4.s_addr, 0); + sk->gwy.addr.v4addr.s_addr, 0); pd->hdr.icmp->icmp_cksum = pf_cksum_fixup( pd->hdr.icmp->icmp_cksum, icmpid, sk->gwy.xport.port, 0); pd->hdr.icmp->icmp_id = sk->gwy.xport.port; - m = pf_lazy_makewritable(pd, m, - off + ICMP_MINLEN); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off + ICMP_MINLEN) == NULL) return (PF_DROP); - m_copyback(m, off, ICMP_MINLEN, + pbuf_copy_back(pbuf, off, ICMP_MINLEN, pd->hdr.icmp); break; #endif /* INET */ @@ -7426,11 +7457,11 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, pf_change_a6(saddr, &pd->hdr.icmp6->icmp6_cksum, &sk->gwy.addr, 0); - m = pf_lazy_makewritable(pd, m, - off + sizeof (struct icmp6_hdr)); - if (!m) + if (pf_lazy_makewritable(pd, NULL, + off + sizeof (struct icmp6_hdr)) == + NULL) return (PF_DROP); - m_copyback(m, off, + pbuf_copy_back(pbuf, off, sizeof (struct icmp6_hdr), pd->hdr.icmp6); break; @@ -7449,9 +7480,9 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, } else { - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - sk->lan.addr.v4.s_addr, 0); + sk->lan.addr.v4addr.s_addr, 0); pd->hdr.icmp->icmp_cksum = pf_cksum_fixup( @@ -7462,15 +7493,14 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, sk->lan.xport.port; } - m = pf_lazy_makewritable(pd, m, - off + ICMP_MINLEN); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off + ICMP_MINLEN) == NULL) return (PF_DROP); - m_copyback(m, off, ICMP_MINLEN, + pbuf_copy_back(pbuf, off, ICMP_MINLEN, pd->hdr.icmp); if (sk->af_lan != sk->af_gwy) - return (pf_do_nat64(sk, pd, m, - off)); + return (pf_do_nat64(sk, pd, + pbuf, off)); break; #endif /* INET */ #if INET6 @@ -7486,16 +7516,16 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, &pd->hdr.icmp6->icmp6_cksum, &sk->lan.addr, 0); } - m = pf_lazy_makewritable(pd, m, - off + sizeof (struct icmp6_hdr)); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off + sizeof (struct icmp6_hdr)) == + NULL) return (PF_DROP); - m_copyback(m, off, + pbuf_copy_back(pbuf, off, sizeof (struct icmp6_hdr), pd->hdr.icmp6); if (sk->af_lan != sk->af_gwy) - return (pf_do_nat64(sk, pd, m, - off)); + return (pf_do_nat64(sk, pd, + pbuf, off)); break; #endif /* INET6 */ } @@ -7529,7 +7559,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, /* offset of h2 in mbuf chain */ ipoff2 = off + ICMP_MINLEN; - if (!pf_pull_hdr(m, ipoff2, &h2, sizeof (h2), + if (!pf_pull_hdr(pbuf, ipoff2, &h2, sizeof (h2), NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short " @@ -7560,7 +7590,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, case AF_INET6: ipoff2 = off + sizeof (struct icmp6_hdr); - if (!pf_pull_hdr(m, ipoff2, &h2_6, sizeof (h2_6), + if (!pf_pull_hdr(pbuf, ipoff2, &h2_6, sizeof (h2_6), NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short " @@ -7568,8 +7598,8 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, return (PF_DROP); } pd2.proto = h2_6.ip6_nxt; - pd2.src = (struct pf_addr *)&h2_6.ip6_src; - pd2.dst = (struct pf_addr *)&h2_6.ip6_dst; + pd2.src = (struct pf_addr *)(uintptr_t)&h2_6.ip6_src; + pd2.dst = (struct pf_addr *)(uintptr_t)&h2_6.ip6_dst; pd2.ip_sum = NULL; off2 = ipoff2 + sizeof (h2_6); do { @@ -7588,7 +7618,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, /* get next header and header length */ struct ip6_ext opt6; - if (!pf_pull_hdr(m, off2, &opt6, + if (!pf_pull_hdr(pbuf, off2, &opt6, sizeof (opt6), NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, @@ -7627,7 +7657,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, * expected. Don't access any TCP header fields after * th_seq, an ackskew test is not possible. */ - if (!pf_pull_hdr(m, off2, &th, 8, NULL, reason, + if (!pf_pull_hdr(pbuf, off2, &th, 8, NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short " @@ -7711,12 +7741,12 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, if (pf_translate_icmp_af(pd->naf, pd->hdr.icmp)) return (PF_DROP); - m = - pf_lazy_makewritable(pd, m, off2 + 8); - if (!m) + + if (pf_lazy_makewritable(pd, pbuf, + off2 + 8) == NULL) return (PF_DROP); - m_copyback(m, pd->off, + pbuf_copy_back(pbuf, pd->off, sizeof(struct icmp6_hdr), pd->hdr.icmp6); @@ -7724,7 +7754,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, * translate inner ip header within the * ICMP message */ - if (pf_change_icmp_af(m, ipoff2, pd, + if (pf_change_icmp_af(pbuf, ipoff2, pd, &pd2, &saddr2->addr, &daddr2->addr, pd->af, pd->naf)) return (PF_DROP); @@ -7750,7 +7780,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, daddr2->xport.port, 0, pd2.af, pd2.naf, 0); - m_copyback(m, pd2.off, 8, &th); + pbuf_copy_back(pbuf, pd2.off, 8, &th); /* translate outer ip header */ PF_ACPY(&pd->naddr, &daddr2->addr, @@ -7761,10 +7791,10 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, memcpy(&pd->naddr.addr32[3], &srcv4_inaddr, sizeof(pd->naddr.addr32[3])); - return (pf_nat64_ipv4(m, off, + return (pf_nat64_ipv4(pbuf, off, pd)); } else { - return (pf_nat64_ipv6(m, off, + return (pf_nat64_ipv6(pbuf, off, pd)); } } @@ -7785,29 +7815,29 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, } if (copyback) { - m = pf_lazy_makewritable(pd, m, off2 + 8); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, off2 + 8) == + NULL) return (PF_DROP); switch (pd2.af) { #if INET case AF_INET: - m_copyback(m, off, ICMP_MINLEN, + pbuf_copy_back(pbuf, off, ICMP_MINLEN, pd->hdr.icmp); - m_copyback(m, ipoff2, sizeof (h2), + pbuf_copy_back(pbuf, ipoff2, sizeof(h2), &h2); break; #endif /* INET */ #if INET6 case AF_INET6: - m_copyback(m, off, + pbuf_copy_back(pbuf, off, sizeof (struct icmp6_hdr), pd->hdr.icmp6); - m_copyback(m, ipoff2, sizeof (h2_6), - &h2_6); + pbuf_copy_back(pbuf, ipoff2, + sizeof (h2_6), &h2_6); break; #endif /* INET6 */ } - m_copyback(m, off2, 8, &th); + pbuf_copy_back(pbuf, off2, 8, &th); } return (PF_PASS); @@ -7815,7 +7845,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, case IPPROTO_UDP: { struct udphdr uh; int dx, action; - if (!pf_pull_hdr(m, off2, &uh, sizeof (uh), + if (!pf_pull_hdr(pbuf, off2, &uh, sizeof (uh), NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short " @@ -7842,8 +7872,8 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, if (ntohs(uh.uh_sport) == PF_IKE_PORT && ntohs(uh.uh_dport) == PF_IKE_PORT) { struct pf_ike_hdr ike; - size_t plen = - m->m_pkthdr.len - off2 - sizeof (uh); + size_t plen = pbuf->pb_packet_len - off2 - + sizeof (uh); if (direction == PF_IN && plen < 8 /* PF_IKE_PACKET_MINSIZE */) { DPFPRINTF(PF_DEBUG_MISC, ("pf: " @@ -7854,7 +7884,8 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, if (plen > sizeof (ike)) plen = sizeof (ike); - m_copydata(m, off + sizeof (uh), plen, &ike); + pbuf_copy_data(pbuf, off + sizeof (uh), plen, + &ike); key.app_state = &as; as.compare_lan_ext = pf_ike_compare; @@ -7914,12 +7945,11 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, if (pf_translate_icmp_af(pd->naf, pd->hdr.icmp)) return (PF_DROP); - m = - pf_lazy_makewritable(pd, m, off2 + 8); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off2 + 8) == NULL) return (PF_DROP); - m_copyback(m, pd->off, + pbuf_copy_back(pbuf, pd->off, sizeof(struct icmp6_hdr), pd->hdr.icmp6); @@ -7927,7 +7957,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, * translate inner ip header within the * ICMP message */ - if (pf_change_icmp_af(m, ipoff2, pd, + if (pf_change_icmp_af(pbuf, ipoff2, pd, &pd2, &saddr2->addr, &daddr2->addr, pd->af, pd->naf)) return (PF_DROP); @@ -7953,7 +7983,8 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, daddr2->xport.port, 0, pd2.af, pd2.naf, 0); - m_copyback(m, pd2.off, sizeof(uh), &uh); + pbuf_copy_back(pbuf, pd2.off, + sizeof(uh), &uh); /* translate outer ip header */ PF_ACPY(&pd->naddr, &daddr2->addr, @@ -7964,10 +7995,10 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, memcpy(&pd->naddr.addr32[3], &srcv4_inaddr, sizeof(pd->naddr.addr32[3])); - return (pf_nat64_ipv4(m, off, + return (pf_nat64_ipv4(pbuf, off, pd)); } else { - return (pf_nat64_ipv6(m, off, + return (pf_nat64_ipv6(pbuf, off, pd)); } } @@ -7984,29 +8015,29 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, pd2.ip_sum, icmpsum, pd->ip_sum, 1, pd2.af); } - m = pf_lazy_makewritable(pd, m, - off2 + sizeof (uh)); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off2 + sizeof (uh)) == NULL) return (PF_DROP); switch (pd2.af) { #if INET case AF_INET: - m_copyback(m, off, ICMP_MINLEN, + pbuf_copy_back(pbuf, off, ICMP_MINLEN, pd->hdr.icmp); - m_copyback(m, ipoff2, sizeof (h2), &h2); + pbuf_copy_back(pbuf, ipoff2, + sizeof (h2), &h2); break; #endif /* INET */ #if INET6 case AF_INET6: - m_copyback(m, off, + pbuf_copy_back(pbuf, off, sizeof (struct icmp6_hdr), pd->hdr.icmp6); - m_copyback(m, ipoff2, sizeof (h2_6), - &h2_6); + pbuf_copy_back(pbuf, ipoff2, + sizeof (h2_6), &h2_6); break; #endif /* INET6 */ } - m_copyback(m, off2, sizeof (uh), &uh); + pbuf_copy_back(pbuf, off2, sizeof (uh), &uh); } return (PF_PASS); @@ -8015,7 +8046,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, case IPPROTO_ICMP: { struct icmp iih; - if (!pf_pull_hdr(m, off2, &iih, ICMP_MINLEN, + if (!pf_pull_hdr(pbuf, off2, &iih, ICMP_MINLEN, NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short i" @@ -8055,13 +8086,13 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, pd2.ip_sum, icmpsum, pd->ip_sum, 0, AF_INET); } - m = pf_lazy_makewritable(pd, m, - off2 + ICMP_MINLEN); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off2 + ICMP_MINLEN) == NULL) return (PF_DROP); - m_copyback(m, off, ICMP_MINLEN, pd->hdr.icmp); - m_copyback(m, ipoff2, sizeof (h2), &h2); - m_copyback(m, off2, ICMP_MINLEN, &iih); + pbuf_copy_back(pbuf, off, ICMP_MINLEN, + pd->hdr.icmp); + pbuf_copy_back(pbuf, ipoff2, sizeof (h2), &h2); + pbuf_copy_back(pbuf, off2, ICMP_MINLEN, &iih); } return (PF_PASS); @@ -8071,7 +8102,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, case IPPROTO_ICMPV6: { struct icmp6_hdr iih; - if (!pf_pull_hdr(m, off2, &iih, + if (!pf_pull_hdr(pbuf, off2, &iih, sizeof (struct icmp6_hdr), NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short " @@ -8111,15 +8142,15 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, pd2.ip_sum, icmpsum, pd->ip_sum, 0, AF_INET6); } - m = pf_lazy_makewritable(pd, m, off2 + - sizeof (struct icmp6_hdr)); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, off2 + + sizeof (struct icmp6_hdr)) == NULL) return (PF_DROP); - m_copyback(m, off, sizeof (struct icmp6_hdr), - pd->hdr.icmp6); - m_copyback(m, ipoff2, sizeof (h2_6), &h2_6); - m_copyback(m, off2, sizeof (struct icmp6_hdr), - &iih); + pbuf_copy_back(pbuf, off, + sizeof (struct icmp6_hdr), pd->hdr.icmp6); + pbuf_copy_back(pbuf, ipoff2, sizeof (h2_6), + &h2_6); + pbuf_copy_back(pbuf, off2, + sizeof (struct icmp6_hdr), &iih); } return (PF_PASS); @@ -8159,22 +8190,32 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd2.af) { #if INET case AF_INET: - m = pf_lazy_makewritable(pd, m, - ipoff2 + sizeof (h2)); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + ipoff2 + sizeof (h2)) == NULL) return (PF_DROP); + /* + * + * Xnu was missing the following... + */ + pbuf_copy_back(pbuf, off, ICMP_MINLEN, + pd->hdr.icmp); + pbuf_copy_back(pbuf, ipoff2, + sizeof(h2), &h2); + break; + /* + * + */ #endif /* INET */ #if INET6 case AF_INET6: - m = pf_lazy_makewritable(pd, m, - ipoff2 + sizeof (h2_6)); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + ipoff2 + sizeof (h2_6)) == NULL) return (PF_DROP); - m_copyback(m, off, - sizeof (struct icmp6_hdr), - pd->hdr.icmp6); - m_copyback(m, ipoff2, sizeof (h2_6), - &h2_6); + pbuf_copy_back(pbuf, off, + sizeof (struct icmp6_hdr), + pd->hdr.icmp6); + pbuf_copy_back(pbuf, ipoff2, + sizeof (h2_6), &h2_6); break; #endif /* INET6 */ } @@ -8194,7 +8235,6 @@ pf_test_state_grev1(struct pf_state **state, int direction, struct pf_state_peer *dst; struct pf_state_key_cmp key; struct pf_grev1_hdr *grev1 = pd->hdr.grev1; - struct mbuf *m; key.app_state = 0; key.proto = IPPROTO_GRE; @@ -8247,9 +8287,9 @@ pf_test_state_grev1(struct pf_state **state, int direction, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->src->v4.s_addr, + pf_change_a(&pd->src->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->gwy.addr.v4.s_addr, 0); + (*state)->state_key->gwy.addr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -8265,9 +8305,9 @@ pf_test_state_grev1(struct pf_state **state, int direction, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->dst->v4.s_addr, + pf_change_a(&pd->dst->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->lan.addr.v4.s_addr, 0); + (*state)->state_key->lan.addr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -8279,10 +8319,10 @@ pf_test_state_grev1(struct pf_state **state, int direction, } } - m = pf_lazy_makewritable(pd, pd->mp, off + sizeof (*grev1)); - if (!m) + if (pf_lazy_makewritable(pd, pd->mp, off + sizeof (*grev1)) == + NULL) return (PF_DROP); - m_copyback(m, off, sizeof (*grev1), grev1); + pbuf_copy_back(pd->mp, off, sizeof (*grev1), grev1); } return (PF_PASS); @@ -8418,9 +8458,9 @@ pf_test_state_esp(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->src->v4.s_addr, + pf_change_a(&pd->src->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->gwy.addr.v4.s_addr, 0); + (*state)->state_key->gwy.addr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -8434,9 +8474,9 @@ pf_test_state_esp(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->dst->v4.s_addr, + pf_change_a(&pd->dst->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->lan.addr.v4.s_addr, 0); + (*state)->state_key->lan.addr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -8504,9 +8544,9 @@ pf_test_state_other(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->src->v4.s_addr, + pf_change_a(&pd->src->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->gwy.addr.v4.s_addr, + (*state)->state_key->gwy.addr.v4addr.s_addr, 0); break; #endif /* INET */ @@ -8521,9 +8561,9 @@ pf_test_state_other(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->dst->v4.s_addr, + pf_change_a(&pd->dst->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->lan.addr.v4.s_addr, + (*state)->state_key->lan.addr.v4addr.s_addr, 0); break; #endif /* INET */ @@ -8545,13 +8585,13 @@ pf_test_state_other(struct pf_state **state, int direction, struct pfi_kif *kif, * h must be at "ipoff" on the mbuf chain. */ void * -pf_pull_hdr(struct mbuf *m, int off, void *p, int len, +pf_pull_hdr(pbuf_t *pbuf, int off, void *p, int len, u_short *actionp, u_short *reasonp, sa_family_t af) { switch (af) { #if INET case AF_INET: { - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; u_int16_t fragoff = (ntohs(h->ip_off) & IP_OFFMASK) << 3; if (fragoff) { @@ -8563,7 +8603,7 @@ pf_pull_hdr(struct mbuf *m, int off, void *p, int len, } return (NULL); } - if (m->m_pkthdr.len < off + len || + if (pbuf->pb_packet_len < (unsigned)(off + len) || ntohs(h->ip_len) < off + len) { ACTION_SET(actionp, PF_DROP); REASON_SET(reasonp, PFRES_SHORT); @@ -8574,9 +8614,9 @@ pf_pull_hdr(struct mbuf *m, int off, void *p, int len, #endif /* INET */ #if INET6 case AF_INET6: { - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; - if (m->m_pkthdr.len < off + len || + if (pbuf->pb_packet_len < (unsigned)(off + len) || (ntohs(h->ip6_plen) + sizeof (struct ip6_hdr)) < (unsigned)(off + len)) { ACTION_SET(actionp, PF_DROP); @@ -8587,7 +8627,7 @@ pf_pull_hdr(struct mbuf *m, int off, void *p, int len, } #endif /* INET6 */ } - m_copydata(m, off, len, p); + pbuf_copy_data(pbuf, off, len, p); return (p); } @@ -8610,14 +8650,14 @@ pf_routable(struct pf_addr *addr, sa_family_t af, struct pfi_kif *kif) dst = satosin(&ro.ro_dst); dst->sin_family = AF_INET; dst->sin_len = sizeof (*dst); - dst->sin_addr = addr->v4; + dst->sin_addr = addr->v4addr; break; #if INET6 case AF_INET6: dst6 = (struct sockaddr_in6 *)&ro.ro_dst; dst6->sin6_family = AF_INET6; dst6->sin6_len = sizeof (*dst6); - dst6->sin6_addr = addr->v6; + dst6->sin6_addr = addr->v6addr; break; #endif /* INET6 */ default: @@ -8656,14 +8696,14 @@ pf_rtlabel_match(struct pf_addr *addr, sa_family_t af, struct pf_addr_wrap *aw) dst = satosin(&ro.ro_dst); dst->sin_family = AF_INET; dst->sin_len = sizeof (*dst); - dst->sin_addr = addr->v4; + dst->sin_addr = addr->v4addr; break; #if INET6 case AF_INET6: dst6 = (struct sockaddr_in6 *)&ro.ro_dst; dst6->sin6_family = AF_INET6; dst6->sin6_len = sizeof (*dst6); - dst6->sin6_addr = addr->v6; + dst6->sin6_addr = addr->v6addr; break; #endif /* INET6 */ default: @@ -8680,7 +8720,7 @@ pf_rtlabel_match(struct pf_addr *addr, sa_family_t af, struct pf_addr_wrap *aw) #if INET static void -pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, +pf_route(pbuf_t **pbufp, struct pf_rule *r, int dir, struct ifnet *oifp, struct pf_state *s, struct pf_pdesc *pd) { #pragma unused(pd) @@ -8697,28 +8737,41 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, bzero(&iproute, sizeof (iproute)); - if (m == NULL || *m == NULL || r == NULL || + if (pbufp == NULL || !pbuf_is_valid(*pbufp) || r == NULL || (dir != PF_IN && dir != PF_OUT) || oifp == NULL) panic("pf_route: invalid parameters"); if (pd->pf_mtag->pftag_routed++ > 3) { - m0 = *m; - *m = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; + m0 = NULL; goto bad; } - if (r->rt == PF_DUPTO) { - if ((m0 = m_copym(*m, 0, M_COPYALL, M_NOWAIT)) == NULL) - return; - } else { - if ((r->rt == PF_REPLYTO) == (r->direction == dir)) - return; - m0 = *m; + /* + * Since this is something of an edge case and may involve the + * host stack (for routing, at least for now), we convert the + * incoming pbuf into an mbuf. + */ + if (r->rt == PF_DUPTO) + m0 = pbuf_clone_to_mbuf(*pbufp); + else + if ((r->rt == PF_REPLYTO) == (r->direction == dir)) + return; + else { + /* We're going to consume this packet */ + m0 = pbuf_to_mbuf(*pbufp, TRUE); + *pbufp = NULL; } + if (m0 == NULL) + goto bad; + + /* We now have the packet in an mbuf (m0) */ + if (m0->m_len < (int)sizeof (struct ip)) { DPFPRINTF(PF_DEBUG_URGENT, - ("pf_route: m0->m_len < sizeof (struct ip)\n")); + ("pf_route: packet length < sizeof (struct ip)\n")); goto bad; } @@ -8753,13 +8806,13 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, pf_map_addr(AF_INET, r, (struct pf_addr *)&ip->ip_src, &naddr, NULL, &sn); if (!PF_AZERO(&naddr, AF_INET)) - dst->sin_addr.s_addr = naddr.v4.s_addr; + dst->sin_addr.s_addr = naddr.v4addr.s_addr; ifp = r->rpool.cur->kif ? r->rpool.cur->kif->pfik_ifp : NULL; } else { if (!PF_AZERO(&s->rt_addr, AF_INET)) dst->sin_addr.s_addr = - s->rt_addr.v4.s_addr; + s->rt_addr.v4addr.s_addr; ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL; } } @@ -8767,13 +8820,13 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, goto bad; if (oifp != ifp) { - if (pf_test(PF_OUT, ifp, &m0, NULL, NULL) != PF_PASS) + if (pf_test_mbuf(PF_OUT, ifp, &m0, NULL, NULL) != PF_PASS) goto bad; else if (m0 == NULL) goto done; if (m0->m_len < (int)sizeof (struct ip)) { DPFPRINTF(PF_DEBUG_URGENT, - ("pf_route: m0->m_len < sizeof (struct ip)\n")); + ("pf_route: packet length < sizeof (struct ip)\n")); goto bad; } ip = mtod(m0, struct ip *); @@ -8840,21 +8893,19 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, ipstat.ips_fragmented++; done: - if (r->rt != PF_DUPTO) - *m = NULL; - ROUTE_RELEASE(&iproute); return; bad: - m_freem(m0); + if (m0) + m_freem(m0); goto done; } #endif /* INET */ #if INET6 static void -pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, +pf_route6(pbuf_t **pbufp, struct pf_rule *r, int dir, struct ifnet *oifp, struct pf_state *s, struct pf_pdesc *pd) { #pragma unused(pd) @@ -8868,25 +8919,36 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, struct pf_src_node *sn = NULL; int error = 0; - if (m == NULL || *m == NULL || r == NULL || + if (pbufp == NULL || !pbuf_is_valid(*pbufp) || r == NULL || (dir != PF_IN && dir != PF_OUT) || oifp == NULL) panic("pf_route6: invalid parameters"); if (pd->pf_mtag->pftag_routed++ > 3) { - m0 = *m; - *m = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; + m0 = NULL; goto bad; } + /* + * Since this is something of an edge case and may involve the + * host stack (for routing, at least for now), we convert the + * incoming pbuf into an mbuf. + */ if (r->rt == PF_DUPTO) { - if ((m0 = m_copym(*m, 0, M_COPYALL, M_NOWAIT)) == NULL) - return; - } else { - if ((r->rt == PF_REPLYTO) == (r->direction == dir)) - return; - m0 = *m; + m0 = pbuf_clone_to_mbuf(*pbufp); + } else + if ((r->rt == PF_REPLYTO) == (r->direction == dir)) + return; + else { + /* We're about to consume this packet */ + m0 = pbuf_to_mbuf(*pbufp, TRUE); + *pbufp = NULL; } + if (m0 == NULL) + goto bad; + if (m0->m_len < (int)sizeof (struct ip6_hdr)) { DPFPRINTF(PF_DEBUG_URGENT, ("pf_route6: m0->m_len < sizeof (struct ip6_hdr)\n")); @@ -8901,7 +8963,7 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, dst->sin6_len = sizeof (*dst); dst->sin6_addr = ip6->ip6_dst; - /* Cheat. XXX why only in the v6 case??? */ + /* Cheat. XXX why only in the v6addr case??? */ if (r->rt == PF_FASTROUTE) { struct pf_mtag *pf_mtag; @@ -8918,7 +8980,7 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, goto bad; } if (s == NULL) { - pf_map_addr(AF_INET6, r, (struct pf_addr *)&ip6->ip6_src, + pf_map_addr(AF_INET6, r, (struct pf_addr *)(uintptr_t)&ip6->ip6_src, &naddr, NULL, &sn); if (!PF_AZERO(&naddr, AF_INET6)) PF_ACPY((struct pf_addr *)&dst->sin6_addr, @@ -8934,7 +8996,7 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, goto bad; if (oifp != ifp) { - if (pf_test6(PF_OUT, ifp, &m0, NULL, NULL) != PF_PASS) + if (pf_test6_mbuf(PF_OUT, ifp, &m0, NULL, NULL) != PF_PASS) goto bad; else if (m0 == NULL) goto done; @@ -8963,12 +9025,11 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, } done: - if (r->rt != PF_DUPTO) - *m = NULL; return; bad: - m_freem(m0); + if (m0) + m_freem(m0); goto done; } #endif /* INET6 */ @@ -8981,7 +9042,7 @@ bad: * returns 0 when the checksum is valid, otherwise returns 1. */ static int -pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, +pf_check_proto_cksum(pbuf_t *pbuf, int off, int len, u_int8_t p, sa_family_t af) { u_int16_t sum; @@ -8995,10 +9056,10 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, * is partially-computed (only 16-bit summation), do it in * software below. */ - if ((m->m_pkthdr.csum_flags & + if ((*pbuf->pb_csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == (CSUM_DATA_VALID | CSUM_PSEUDO_HDR) && - (m->m_pkthdr.csum_data ^ 0xffff) == 0) { + (*pbuf->pb_csum_data ^ 0xffff) == 0) { return (0); } break; @@ -9012,12 +9073,13 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, } if (off < (int)sizeof (struct ip) || len < (int)sizeof (struct udphdr)) return (1); - if (m->m_pkthdr.len < off + len) + if (pbuf->pb_packet_len < (unsigned)(off + len)) return (1); switch (af) { #if INET case AF_INET: if (p == IPPROTO_ICMP) { +#if 0 if (m->m_len < off) return (1); m->m_data += off; @@ -9025,18 +9087,23 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sum = in_cksum(m, len); m->m_data -= off; m->m_len += off; +#else + if (pbuf->pb_contig_len < (unsigned)off) + return (1); + sum = pbuf_inet_cksum(pbuf, 0, off, len); +#endif } else { - if (m->m_len < (int)sizeof (struct ip)) + if (pbuf->pb_contig_len < (int)sizeof (struct ip)) return (1); - sum = inet_cksum(m, p, off, len); + sum = pbuf_inet_cksum(pbuf, p, off, len); } break; #endif /* INET */ #if INET6 case AF_INET6: - if (m->m_len < (int)sizeof (struct ip6_hdr)) + if (pbuf->pb_contig_len < (int)sizeof (struct ip6_hdr)) return (1); - sum = inet6_cksum(m, p, off, len); + sum = pbuf_inet6_cksum(pbuf, p, off, len); break; #endif /* INET6 */ default: @@ -9067,15 +9134,37 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, #if INET #define PF_APPLE_UPDATE_PDESC_IPv4() \ do { \ - if (m && pd.mp && m != pd.mp) { \ - m = pd.mp; \ - h = mtod(m, struct ip *); \ - pd.pf_mtag = pf_get_mtag(m); \ + if (pbuf && pd.mp && pbuf != pd.mp) { \ + pbuf = pd.mp; \ + h = pbuf->pb_data; \ + pd.pf_mtag = pf_get_mtag_pbuf(pbuf); \ } \ } while (0) int -pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, +pf_test_mbuf(int dir, struct ifnet *ifp, struct mbuf **m0, + struct ether_header *eh, struct ip_fw_args *fwa) +{ + pbuf_t pbuf_store, *pbuf; + int rv; + + pbuf_init_mbuf(&pbuf_store, *m0, (*m0)->m_pkthdr.rcvif); + pbuf = &pbuf_store; + + rv = pf_test(dir, ifp, &pbuf, eh, fwa); + + if (pbuf_is_valid(pbuf)) { + *m0 = pbuf->pb_mbuf; + pbuf->pb_mbuf = NULL; + pbuf_destroy(pbuf); + } else + *m0 = NULL; + + return (rv); +} + +int +pf_test(int dir, struct ifnet *ifp, pbuf_t **pbufp, struct ether_header *eh, struct ip_fw_args *fwa) { #if !DUMMYNET @@ -9083,7 +9172,7 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, #endif struct pfi_kif *kif; u_short action = PF_PASS, reason = 0, log = 0; - struct mbuf *m = *m0; + pbuf_t *pbuf = *pbufp; struct ip *h = 0; struct pf_rule *a = NULL, *r = &pf_default_rule, *tr, *nr; struct pf_state *s = NULL; @@ -9092,16 +9181,16 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, struct pf_pdesc pd; int off, dirndx, pqid = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (!pf_status.running) return (PF_PASS); memset(&pd, 0, sizeof (pd)); - if ((pd.pf_mtag = pf_get_mtag(m)) == NULL) { + if ((pd.pf_mtag = pf_get_mtag_pbuf(pbuf)) == NULL) { DPFPRINTF(PF_DEBUG_URGENT, - ("pf_test: pf_get_mtag returned NULL\n")); + ("pf_test: pf_get_mtag_pbuf returned NULL\n")); return (PF_DROP); } @@ -9118,13 +9207,11 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, if (kif->pfik_flags & PFI_IFLAG_SKIP) return (PF_PASS); - VERIFY(m->m_flags & M_PKTHDR); - /* initialize enough of pd for the done label */ - h = mtod(m, struct ip *); - pd.mp = m; + h = pbuf->pb_data; + pd.mp = pbuf; pd.lmw = 0; - pd.pf_mtag = pf_get_mtag(m); + pd.pf_mtag = pf_get_mtag_pbuf(pbuf); pd.src = (struct pf_addr *)&h->ip_src; pd.dst = (struct pf_addr *)&h->ip_dst; PF_ACPY(&pd.baddr, pd.src, AF_INET); @@ -9138,7 +9225,7 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, pd.tot_len = ntohs(h->ip_len); pd.eh = eh; - if (m->m_pkthdr.len < (int)sizeof (*h)) { + if (pbuf->pb_packet_len < (int)sizeof (*h)) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); log = 1; @@ -9151,8 +9238,7 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, #endif /* DUMMYNET */ /* We do IP header normalization and packet reassembly here */ - action = pf_normalize_ip(m0, dir, kif, &reason, &pd); - pd.mp = m = *m0; + action = pf_normalize_ip(pbuf, dir, kif, &reason, &pd); if (action != PF_PASS || pd.lmw < 0) { action = PF_DROP; goto done; @@ -9161,8 +9247,8 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, #if DUMMYNET nonormalize: #endif /* DUMMYNET */ - m = *m0; /* pf_normalize messes with m0 */ - h = mtod(m, struct ip *); + /* pf_normalize can mess with pb_data */ + h = pbuf->pb_data; off = h->ip_hl << 2; if (off < (int)sizeof (*h)) { @@ -9179,20 +9265,20 @@ nonormalize: pd.ip_sum = &h->ip_sum; pd.proto = h->ip_p; pd.proto_variant = 0; - pd.mp = m; + pd.mp = pbuf; pd.lmw = 0; - pd.pf_mtag = pf_get_mtag(m); + pd.pf_mtag = pf_get_mtag_pbuf(pbuf); pd.af = AF_INET; pd.tos = h->ip_tos; pd.ttl = h->ip_ttl; - pd.sc = MBUF_SCIDX(mbuf_get_service_class(m)); + pd.sc = MBUF_SCIDX(pbuf_get_service_class(pbuf)); pd.tot_len = ntohs(h->ip_len); pd.eh = eh; - if (m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) { - pd.flowsrc = m->m_pkthdr.pkt_flowsrc; - pd.flowhash = m->m_pkthdr.pkt_flowid; - pd.pktflags = (m->m_pkthdr.pkt_flags & PKTF_FLOW_MASK); + if (*pbuf->pb_flags & PKTF_FLOW_ID) { + pd.flowsrc = *pbuf->pb_flowsrc; + pd.flowhash = *pbuf->pb_flowid; + pd.pktflags = *pbuf->pb_flags & PKTF_FLOW_MASK; } /* handle fragments that didn't get reassembled by normalization */ @@ -9200,13 +9286,13 @@ nonormalize: pd.flags |= PFDESC_IP_FRAG; #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_test_fragment(&r, dir, kif, m, h, + action = pf_test_fragment(&r, dir, kif, pbuf, h, &pd, &a, &ruleset); goto done; } @@ -9216,7 +9302,7 @@ nonormalize: case IPPROTO_TCP: { struct tcphdr th; pd.hdr.tcp = &th; - if (!pf_pull_hdr(m, off, &th, sizeof (th), + if (!pf_pull_hdr(pbuf, off, &th, sizeof (th), &action, &reason, AF_INET)) { log = action != PF_PASS; goto done; @@ -9226,19 +9312,19 @@ nonormalize: pqid = 1; #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_normalize_tcp(dir, kif, m, 0, off, h, &pd); + action = pf_normalize_tcp(dir, kif, pbuf, 0, off, h, &pd); if (pd.lmw < 0) goto done; PF_APPLE_UPDATE_PDESC_IPv4(); if (action == PF_DROP) goto done; - action = pf_test_state_tcp(&s, dir, kif, m, off, h, &pd, + action = pf_test_state_tcp(&s, dir, kif, pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; @@ -9254,7 +9340,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9262,13 +9348,13 @@ nonormalize: struct udphdr uh; pd.hdr.udp = &uh; - if (!pf_pull_hdr(m, off, &uh, sizeof (uh), + if (!pf_pull_hdr(pbuf, off, &uh, sizeof (uh), &action, &reason, AF_INET)) { log = action != PF_PASS; goto done; } if (uh.uh_dport == 0 || - ntohs(uh.uh_ulen) > m->m_pkthdr.len - off || + ntohs(uh.uh_ulen) > pbuf->pb_packet_len - off || ntohs(uh.uh_ulen) < sizeof (struct udphdr)) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); @@ -9276,13 +9362,13 @@ nonormalize: } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_test_state_udp(&s, dir, kif, m, off, h, &pd, + action = pf_test_state_udp(&s, dir, kif, pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; @@ -9298,7 +9384,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9306,20 +9392,20 @@ nonormalize: struct icmp ih; pd.hdr.icmp = &ih; - if (!pf_pull_hdr(m, off, &ih, ICMP_MINLEN, + if (!pf_pull_hdr(pbuf, off, &ih, ICMP_MINLEN, &action, &reason, AF_INET)) { log = action != PF_PASS; goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_test_state_icmp(&s, dir, kif, m, off, h, &pd, + action = pf_test_state_icmp(&s, dir, kif, pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; @@ -9335,7 +9421,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9343,16 +9429,16 @@ nonormalize: struct pf_esp_hdr esp; pd.hdr.esp = &esp; - if (!pf_pull_hdr(m, off, &esp, sizeof (esp), &action, &reason, + if (!pf_pull_hdr(pbuf, off, &esp, sizeof (esp), &action, &reason, AF_INET)) { log = action != PF_PASS; goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ @@ -9369,30 +9455,30 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } case IPPROTO_GRE: { struct pf_grev1_hdr grev1; pd.hdr.grev1 = &grev1; - if (!pf_pull_hdr(m, off, &grev1, sizeof (grev1), &action, + if (!pf_pull_hdr(pbuf, off, &grev1, sizeof (grev1), &action, &reason, AF_INET)) { log = (action != PF_PASS); goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ if ((ntohs(grev1.flags) & PF_GRE_FLAG_VERSION_MASK) == 1 && ntohs(grev1.protocol_type) == PF_GRE_PPP_ETHERTYPE) { if (ntohs(grev1.payload_length) > - m->m_pkthdr.len - off) { + pbuf->pb_packet_len - off) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); goto done; @@ -9410,8 +9496,8 @@ nonormalize: log = s->log; break; } else if (s == NULL) { - action = pf_test_rule(&r, &s, dir, kif, m, off, - h, &pd, &a, &ruleset, NULL); + action = pf_test_rule(&r, &s, dir, kif, pbuf, + off, h, &pd, &a, &ruleset, NULL); if (action == PF_PASS) break; } @@ -9423,9 +9509,9 @@ nonormalize: default: #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ @@ -9441,18 +9527,18 @@ nonormalize: a = s->anchor.ptr; log = s->log; } else if (s == NULL) - action = pf_test_rule(&r, &s, dir, kif, m, off, h, + action = pf_test_rule(&r, &s, dir, kif, pbuf, off, h, &pd, &a, &ruleset, NULL); break; } done: if (action == PF_NAT64) { - *m0 = NULL; + *pbufp = NULL; return (action); } - *m0 = pd.mp; + *pbufp = pd.mp; PF_APPLE_UPDATE_PDESC_IPv4(); if (action != PF_DROP) { @@ -9468,18 +9554,10 @@ done: if ((s && s->tag) || PF_RTABLEID_IS_VALID(r->rtableid) || (pd.pktflags & PKTF_FLOW_ID)) - (void) pf_tag_packet(m, pd.pf_mtag, s ? s->tag : 0, + (void) pf_tag_packet(pbuf, pd.pf_mtag, s ? s->tag : 0, r->rtableid, &pd); if (action == PF_PASS) { -#if PF_ALTQ - if (altq_allowed && r->qid) { - if (pqid || (pd.tos & IPTOS_LOWDELAY)) - pd.pf_mtag->pftag_qid = r->pqid; - else - pd.pf_mtag->pftag_qid = r->qid; - } -#endif /* PF_ALTQ */ #if PF_ECN /* add hints for ecn */ pd.pf_mtag->pftag_hdr = h; @@ -9488,7 +9566,7 @@ done: pd.pf_mtag->pftag_flags |= PF_TAG_HDR_INET; #endif /* PF_ECN */ /* record protocol */ - m->m_pkthdr.pkt_proto = pd.proto; + *pbuf->pb_proto = pd.proto; /* * connections redirected to loopback should not match sockets @@ -9500,7 +9578,7 @@ done: s->nat_rule.ptr != NULL && (s->nat_rule.ptr->action == PF_RDR || s->nat_rule.ptr->action == PF_BINAT) && - (ntohl(pd.dst->v4.s_addr) >> IN_CLASSA_NSHIFT) + (ntohl(pd.dst->v4addr.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET) pd.pf_mtag->pftag_flags |= PF_TAG_TRANSLATE_LOCALHOST; } @@ -9514,7 +9592,7 @@ done: lr = s->nat_rule.ptr; else lr = r; - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, lr, a, ruleset, + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, lr, a, ruleset, &pd); } @@ -9584,30 +9662,30 @@ done: tr->dst.neg); } - VERIFY(m == NULL || pd.mp == NULL || pd.mp == m); + VERIFY(pbuf == NULL || pd.mp == NULL || pd.mp == pbuf); - if (*m0) { + if (*pbufp) { if (pd.lmw < 0) { REASON_SET(&reason, PFRES_MEMORY); action = PF_DROP; } if (action == PF_DROP) { - m_freem(*m0); - *m0 = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; return (PF_DROP); } - *m0 = m; + *pbufp = pbuf; } if (action == PF_SYNPROXY_DROP) { - m_freem(*m0); - *m0 = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; action = PF_PASS; } else if (r->rt) - /* pf_route can free the mbuf causing *m0 to become NULL */ - pf_route(m0, r, dir, kif->pfik_ifp, s, &pd); + /* pf_route can free the pbuf causing *pbufp to become NULL */ + pf_route(pbufp, r, dir, kif->pfik_ifp, s, &pd); return (action); } @@ -9616,16 +9694,36 @@ done: #if INET6 #define PF_APPLE_UPDATE_PDESC_IPv6() \ do { \ - if (m && pd.mp && m != pd.mp) { \ - if (n == m) \ - n = pd.mp; \ - m = pd.mp; \ - h = mtod(m, struct ip6_hdr *); \ + if (pbuf && pd.mp && pbuf != pd.mp) { \ + pbuf = pd.mp; \ } \ + h = pbuf->pb_data; \ } while (0) int -pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, +pf_test6_mbuf(int dir, struct ifnet *ifp, struct mbuf **m0, + struct ether_header *eh, struct ip_fw_args *fwa) +{ + pbuf_t pbuf_store, *pbuf; + int rv; + + pbuf_init_mbuf(&pbuf_store, *m0, (*m0)->m_pkthdr.rcvif); + pbuf = &pbuf_store; + + rv = pf_test6(dir, ifp, &pbuf, eh, fwa); + + if (pbuf_is_valid(pbuf)) { + *m0 = pbuf->pb_mbuf; + pbuf->pb_mbuf = NULL; + pbuf_destroy(pbuf); + } else + *m0 = NULL; + + return (rv); +} + +int +pf_test6(int dir, struct ifnet *ifp, pbuf_t **pbufp, struct ether_header *eh, struct ip_fw_args *fwa) { #if !DUMMYNET @@ -9633,7 +9731,7 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, #endif struct pfi_kif *kif; u_short action = PF_PASS, reason = 0, log = 0; - struct mbuf *m = *m0, *n = NULL; + pbuf_t *pbuf = *pbufp; struct ip6_hdr *h; struct pf_rule *a = NULL, *r = &pf_default_rule, *tr, *nr; struct pf_state *s = NULL; @@ -9643,16 +9741,16 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, int off, terminal = 0, dirndx, rh_cnt = 0; u_int8_t nxt; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (!pf_status.running) return (PF_PASS); memset(&pd, 0, sizeof (pd)); - if ((pd.pf_mtag = pf_get_mtag(m)) == NULL) { + if ((pd.pf_mtag = pf_get_mtag_pbuf(pbuf)) == NULL) { DPFPRINTF(PF_DEBUG_URGENT, - ("pf_test6: pf_get_mtag returned NULL\n")); + ("pf_test6: pf_get_mtag_pbuf returned NULL\n")); return (PF_DROP); } @@ -9669,17 +9767,15 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, if (kif->pfik_flags & PFI_IFLAG_SKIP) return (PF_PASS); - VERIFY(m->m_flags & M_PKTHDR); - - h = mtod(m, struct ip6_hdr *); + h = pbuf->pb_data; nxt = h->ip6_nxt; - off = ((caddr_t)h - m->m_data) + sizeof(struct ip6_hdr); - pd.mp = m; + off = ((caddr_t)h - (caddr_t)pbuf->pb_data) + sizeof(struct ip6_hdr); + pd.mp = pbuf; pd.lmw = 0; - pd.pf_mtag = pf_get_mtag(m); - pd.src = (struct pf_addr *)&h->ip6_src; - pd.dst = (struct pf_addr *)&h->ip6_dst; + pd.pf_mtag = pf_get_mtag_pbuf(pbuf); + pd.src = (struct pf_addr *)(uintptr_t)&h->ip6_src; + pd.dst = (struct pf_addr *)(uintptr_t)&h->ip6_dst; PF_ACPY(&pd.baddr, pd.src, AF_INET6); PF_ACPY(&pd.bdaddr, pd.dst, AF_INET6); pd.ip_sum = NULL; @@ -9688,17 +9784,17 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, pd.proto_variant = 0; pd.tos = 0; pd.ttl = h->ip6_hlim; - pd.sc = MBUF_SCIDX(mbuf_get_service_class(m)); + pd.sc = MBUF_SCIDX(pbuf_get_service_class(pbuf)); pd.tot_len = ntohs(h->ip6_plen) + sizeof(struct ip6_hdr); pd.eh = eh; - if (m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) { - pd.flowsrc = m->m_pkthdr.pkt_flowsrc; - pd.flowhash = m->m_pkthdr.pkt_flowid; - pd.pktflags = (m->m_pkthdr.pkt_flags & PKTF_FLOW_MASK); + if (*pbuf->pb_flags & PKTF_FLOW_ID) { + pd.flowsrc = *pbuf->pb_flowsrc; + pd.flowhash = *pbuf->pb_flowid; + pd.pktflags = (*pbuf->pb_flags & PKTF_FLOW_MASK); } - if (m->m_pkthdr.len < (int)sizeof (*h)) { + if (pbuf->pb_packet_len < (int)sizeof (*h)) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); log = 1; @@ -9711,8 +9807,7 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, #endif /* DUMMYNET */ /* We do IP header normalization and packet reassembly here */ - action = pf_normalize_ip6(m0, dir, kif, &reason, &pd); - pd.mp = m = *m0; + action = pf_normalize_ip6(pbuf, dir, kif, &reason, &pd); if (action != PF_PASS || pd.lmw < 0) { action = PF_DROP; goto done; @@ -9721,7 +9816,7 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, #if DUMMYNET nonormalize: #endif /* DUMMYNET */ - h = mtod(m, struct ip6_hdr *); + h = pbuf->pb_data; #if 1 /* @@ -9735,8 +9830,8 @@ nonormalize: } #endif - pd.src = (struct pf_addr *)&h->ip6_src; - pd.dst = (struct pf_addr *)&h->ip6_dst; + pd.src = (struct pf_addr *)(uintptr_t)&h->ip6_src; + pd.dst = (struct pf_addr *)(uintptr_t)&h->ip6_dst; PF_ACPY(&pd.baddr, pd.src, AF_INET6); PF_ACPY(&pd.bdaddr, pd.dst, AF_INET6); pd.ip_sum = NULL; @@ -9746,12 +9841,12 @@ nonormalize: pd.tot_len = ntohs(h->ip6_plen) + sizeof (struct ip6_hdr); pd.eh = eh; - off = ((caddr_t)h - m->m_data) + sizeof (struct ip6_hdr); + off = ((caddr_t)h - (caddr_t)pbuf->pb_data) + sizeof (struct ip6_hdr); pd.proto = h->ip6_nxt; pd.proto_variant = 0; - pd.mp = m; + pd.mp = pbuf; pd.lmw = 0; - pd.pf_mtag = pf_get_mtag(m); + pd.pf_mtag = pf_get_mtag_pbuf(pbuf); do { switch (nxt) { @@ -9759,7 +9854,7 @@ nonormalize: struct ip6_frag ip6f; pd.flags |= PFDESC_IP_FRAG; - if (!pf_pull_hdr(m, off, &ip6f, sizeof ip6f, NULL, + if (!pf_pull_hdr(pbuf, off, &ip6f, sizeof ip6f, NULL, &reason, pd.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: IPv6 short fragment header\n")); @@ -9771,14 +9866,15 @@ nonormalize: pd.proto = nxt = ip6f.ip6f_nxt; #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, + fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_test_fragment(&r, dir, kif, m, h, &pd, &a, - &ruleset); + action = pf_test_fragment(&r, dir, kif, pbuf, h, &pd, + &a, &ruleset); if (action == PF_DROP) { REASON_SET(&reason, PFRES_FRAG); log = 1; @@ -9795,7 +9891,7 @@ nonormalize: /* get next header and header length */ struct ip6_ext opt6; - if (!pf_pull_hdr(m, off, &opt6, sizeof(opt6), + if (!pf_pull_hdr(pbuf, off, &opt6, sizeof(opt6), NULL, &reason, pd.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: IPv6 short opt\n")); @@ -9817,9 +9913,6 @@ nonormalize: } } while (!terminal); - /* if there's no routing header, use unmodified mbuf for checksumming */ - if (!n) - n = m; switch (pd.proto) { @@ -9827,7 +9920,7 @@ nonormalize: struct tcphdr th; pd.hdr.tcp = &th; - if (!pf_pull_hdr(m, off, &th, sizeof (th), + if (!pf_pull_hdr(pbuf, off, &th, sizeof (th), &action, &reason, AF_INET6)) { log = action != PF_PASS; goto done; @@ -9835,19 +9928,19 @@ nonormalize: pd.p_len = pd.tot_len - off - (th.th_off << 2); #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_normalize_tcp(dir, kif, m, 0, off, h, &pd); + action = pf_normalize_tcp(dir, kif, pbuf, 0, off, h, &pd); if (pd.lmw < 0) goto done; PF_APPLE_UPDATE_PDESC_IPv6(); if (action == PF_DROP) goto done; - action = pf_test_state_tcp(&s, dir, kif, m, off, h, &pd, + action = pf_test_state_tcp(&s, dir, kif, pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; @@ -9863,7 +9956,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9871,13 +9964,13 @@ nonormalize: struct udphdr uh; pd.hdr.udp = &uh; - if (!pf_pull_hdr(m, off, &uh, sizeof (uh), + if (!pf_pull_hdr(pbuf, off, &uh, sizeof (uh), &action, &reason, AF_INET6)) { log = action != PF_PASS; goto done; } if (uh.uh_dport == 0 || - ntohs(uh.uh_ulen) > m->m_pkthdr.len - off || + ntohs(uh.uh_ulen) > pbuf->pb_packet_len - off || ntohs(uh.uh_ulen) < sizeof (struct udphdr)) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); @@ -9885,13 +9978,13 @@ nonormalize: } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_test_state_udp(&s, dir, kif, m, off, h, &pd, + action = pf_test_state_udp(&s, dir, kif, pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; @@ -9907,7 +10000,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9915,21 +10008,21 @@ nonormalize: struct icmp6_hdr ih; pd.hdr.icmp6 = &ih; - if (!pf_pull_hdr(m, off, &ih, sizeof (ih), + if (!pf_pull_hdr(pbuf, off, &ih, sizeof (ih), &action, &reason, AF_INET6)) { log = action != PF_PASS; goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ action = pf_test_state_icmp(&s, dir, kif, - m, off, h, &pd, &reason); + pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; if (pd.lmw < 0) @@ -9944,7 +10037,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9952,16 +10045,16 @@ nonormalize: struct pf_esp_hdr esp; pd.hdr.esp = &esp; - if (!pf_pull_hdr(m, off, &esp, sizeof (esp), &action, &reason, - AF_INET6)) { + if (!pf_pull_hdr(pbuf, off, &esp, sizeof (esp), &action, + &reason, AF_INET6)) { log = action != PF_PASS; goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ @@ -9978,7 +10071,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9986,23 +10079,23 @@ nonormalize: struct pf_grev1_hdr grev1; pd.hdr.grev1 = &grev1; - if (!pf_pull_hdr(m, off, &grev1, sizeof (grev1), &action, + if (!pf_pull_hdr(pbuf, off, &grev1, sizeof (grev1), &action, &reason, AF_INET6)) { log = (action != PF_PASS); goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ if ((ntohs(grev1.flags) & PF_GRE_FLAG_VERSION_MASK) == 1 && ntohs(grev1.protocol_type) == PF_GRE_PPP_ETHERTYPE) { if (ntohs(grev1.payload_length) > - m->m_pkthdr.len - off) { + pbuf->pb_packet_len - off) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); goto done; @@ -10020,8 +10113,8 @@ nonormalize: log = s->log; break; } else if (s == NULL) { - action = pf_test_rule(&r, &s, dir, kif, m, off, - h, &pd, &a, &ruleset, NULL); + action = pf_test_rule(&r, &s, dir, kif, pbuf, + off, h, &pd, &a, &ruleset, NULL); if (action == PF_PASS) break; } @@ -10033,9 +10126,9 @@ nonormalize: default: #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ @@ -10051,25 +10144,20 @@ nonormalize: a = s->anchor.ptr; log = s->log; } else if (s == NULL) - action = pf_test_rule(&r, &s, dir, kif, m, off, h, + action = pf_test_rule(&r, &s, dir, kif, pbuf, off, h, &pd, &a, &ruleset, NULL); break; } done: if (action == PF_NAT64) { - *m0 = NULL; + *pbufp = NULL; return (action); } - *m0 = pd.mp; + *pbufp = pd.mp; PF_APPLE_UPDATE_PDESC_IPv6(); - if (n != m) { - m_freem(n); - n = NULL; - } - /* handle dangerous IPv6 extension headers. */ if (action != PF_DROP) { if (action == PF_PASS && rh_cnt && @@ -10078,23 +10166,15 @@ done: REASON_SET(&reason, PFRES_IPOPTIONS); log = 1; DPFPRINTF(PF_DEBUG_MISC, - ("pf: dropping packet with dangerous v6 headers\n")); + ("pf: dropping packet with dangerous v6addr headers\n")); } if ((s && s->tag) || PF_RTABLEID_IS_VALID(r->rtableid) || (pd.pktflags & PKTF_FLOW_ID)) - (void) pf_tag_packet(m, pd.pf_mtag, s ? s->tag : 0, + (void) pf_tag_packet(pbuf, pd.pf_mtag, s ? s->tag : 0, r->rtableid, &pd); if (action == PF_PASS) { -#if PF_ALTQ - if (altq_allowed && r->qid) { - if (pd.tos & IPTOS_LOWDELAY) - pd.pf_mtag->pftag_qid = r->pqid; - else - pd.pf_mtag->pftag_qid = r->qid; - } -#endif /* PF_ALTQ */ #if PF_ECN /* add hints for ecn */ pd.pf_mtag->pftag_hdr = h; @@ -10103,13 +10183,13 @@ done: pd.pf_mtag->pftag_flags |= PF_TAG_HDR_INET6; #endif /* PF_ECN */ /* record protocol */ - m->m_pkthdr.pkt_proto = pd.proto; + *pbuf->pb_proto = pd.proto; if (dir == PF_IN && (pd.proto == IPPROTO_TCP || pd.proto == IPPROTO_UDP) && s != NULL && s->nat_rule.ptr != NULL && (s->nat_rule.ptr->action == PF_RDR || s->nat_rule.ptr->action == PF_BINAT) && - IN6_IS_ADDR_LOOPBACK(&pd.dst->v6)) + IN6_IS_ADDR_LOOPBACK(&pd.dst->v6addr)) pd.pf_mtag->pftag_flags |= PF_TAG_TRANSLATE_LOCALHOST; } } @@ -10123,7 +10203,7 @@ done: lr = s->nat_rule.ptr; else lr = r; - PFLOG_PACKET(kif, h, m, AF_INET6, dir, reason, lr, a, ruleset, + PFLOG_PACKET(kif, h, pbuf, AF_INET6, dir, reason, lr, a, ruleset, &pd); } @@ -10201,35 +10281,30 @@ done: /* pf_route6 can free the mbuf causing *m0 to become NULL */ pf_route6(m0, r, dir, kif->pfik_ifp, s, &pd); #else - VERIFY(m == NULL || pd.mp == NULL || pd.mp == m); + VERIFY(pbuf == NULL || pd.mp == NULL || pd.mp == pbuf); - if (*m0) { + if (*pbufp) { if (pd.lmw < 0) { REASON_SET(&reason, PFRES_MEMORY); action = PF_DROP; } if (action == PF_DROP) { - m_freem(*m0); - *m0 = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; return (PF_DROP); } - *m0 = m; + *pbufp = pbuf; } if (action == PF_SYNPROXY_DROP) { - m_freem(*m0); - *m0 = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; action = PF_PASS; } else if (r->rt) { - if (action == PF_PASS) { - m = *m0; - h = mtod(m, struct ip6_hdr *); - } - /* pf_route6 can free the mbuf causing *m0 to become NULL */ - pf_route6(m0, r, dir, kif->pfik_ifp, s, &pd); + pf_route6(pbufp, r, dir, kif->pfik_ifp, s, &pd); } #endif /* 0 */ @@ -10284,7 +10359,7 @@ pool_get(struct pool *pp, int flags) { void *buf; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (pp->pool_count > pp->pool_limit) { DPFPRINTF(PF_DEBUG_NOISY, @@ -10306,18 +10381,23 @@ pool_get(struct pool *pp, int flags) void pool_put(struct pool *pp, void *v) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); zfree(pp->pool_zone, v); VERIFY(pp->pool_count != 0); pp->pool_count--; } +struct pf_mtag * +pf_find_mtag_pbuf(pbuf_t *pbuf) +{ + + return (pbuf->pb_pftag); +} + struct pf_mtag * pf_find_mtag(struct mbuf *m) { - if (!(m->m_flags & M_PKTHDR)) - return (NULL); return (m_pftag(m)); } @@ -10328,6 +10408,12 @@ pf_get_mtag(struct mbuf *m) return (pf_find_mtag(m)); } +struct pf_mtag * +pf_get_mtag_pbuf(pbuf_t *pbuf) +{ + return (pf_find_mtag_pbuf(pbuf)); +} + uint64_t pf_time_second(void) { diff --git a/bsd/net/pf_if.c b/bsd/net/pf_if.c index 05f265677..f67d06a5d 100644 --- a/bsd/net/pf_if.c +++ b/bsd/net/pf_if.c @@ -231,7 +231,7 @@ pfi_attach_ifnet(struct ifnet *ifp) { struct pfi_kif *kif; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); pfi_update++; if ((kif = pfi_kif_get(if_name(ifp))) == NULL) @@ -253,7 +253,7 @@ pfi_detach_ifnet(struct ifnet *ifp) { struct pfi_kif *kif; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if ((kif = (struct pfi_kif *)ifp->if_pf_kif) == NULL) return; @@ -310,7 +310,7 @@ pfi_dynaddr_setup(struct pf_addr_wrap *aw, sa_family_t af) struct pf_ruleset *ruleset = NULL; int rv = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (aw->type != PF_ADDR_DYNIFTL) return (0); @@ -378,7 +378,7 @@ pfi_kif_update(struct pfi_kif *kif) { struct pfi_dynaddr *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); /* update all dynaddr */ TAILQ_FOREACH(p, &kif->pfik_dynaddrs, entry) @@ -576,7 +576,7 @@ pfi_kifaddr_update(void *v) { struct pfi_kif *kif = (struct pfi_kif *)v; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); pfi_update++; pfi_kif_update(kif); @@ -595,7 +595,7 @@ pfi_update_status(const char *name, struct pf_status *pfs) struct pfi_kif_cmp key; int i, j, k; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); strlcpy(key.pfik_name, name, sizeof (key.pfik_name)); p = RB_FIND(pfi_ifhead, &pfi_ifs, (struct pfi_kif *)(void *)&key); @@ -627,7 +627,7 @@ pfi_get_ifaces(const char *name, user_addr_t buf, int *size) struct pfi_kif *p, *nextp; int n = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); for (p = RB_MIN(pfi_ifhead, &pfi_ifs); p; p = nextp) { nextp = RB_NEXT(pfi_ifhead, &pfi_ifs, p); @@ -689,7 +689,7 @@ pfi_set_flags(const char *name, int flags) { struct pfi_kif *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); RB_FOREACH(p, pfi_ifhead, &pfi_ifs) { if (pfi_skip_if(name, p)) @@ -704,7 +704,7 @@ pfi_clear_flags(const char *name, int flags) { struct pfi_kif *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); RB_FOREACH(p, pfi_ifhead, &pfi_ifs) { if (pfi_skip_if(name, p)) diff --git a/bsd/net/pf_ioctl.c b/bsd/net/pf_ioctl.c index 517ee6856..05bb21b56 100644 --- a/bsd/net/pf_ioctl.c +++ b/bsd/net/pf_ioctl.c @@ -86,6 +86,7 @@ #include #include #include +#include #include #include @@ -123,15 +124,6 @@ struct ip_fw_args; #include #endif /* INET6 */ -#if PF_ALTQ -#include -#include -#include -#include -#include -#include -#endif /* PF_ALTQ */ - #include #if 0 @@ -167,15 +159,6 @@ static struct pf_pool *pf_get_pool(char *, u_int32_t, u_int8_t, u_int32_t, u_int8_t, u_int8_t, u_int8_t); static void pf_mv_pool(struct pf_palist *, struct pf_palist *); static void pf_empty_pool(struct pf_palist *); -#if PF_ALTQ -static int pf_begin_altq(u_int32_t *); -static int pf_rollback_altq(u_int32_t); -static int pf_commit_altq(u_int32_t); -static int pf_enable_altq(struct pf_altq *); -static int pf_disable_altq(struct pf_altq *); -static void pf_altq_copyin(struct pf_altq *, struct pf_altq *); -static void pf_altq_copyout(struct pf_altq *, struct pf_altq *); -#endif /* PF_ALTQ */ static int pf_begin_rules(u_int32_t *, int, const char *); static int pf_rollback_rules(u_int32_t, int, char *); static int pf_setup_pfsync_matching(struct pf_ruleset *); @@ -235,10 +218,6 @@ static void pf_detach_hooks(void); */ int pf_is_enabled = 0; -#if PF_ALTQ -u_int32_t altq_allowed = 0; -#endif /* PF_ALTQ */ - u_int32_t pf_hash_seed; int16_t pf_nat64_configured = 0; @@ -254,19 +233,10 @@ SLIST_HEAD(list_head, pfioc_kernel_token); static struct list_head token_list_head; struct pf_rule pf_default_rule; -#if PF_ALTQ -static int pf_altq_running; -#endif /* PF_ALTQ */ #define TAGID_MAX 50000 -#if !PF_ALTQ static TAILQ_HEAD(pf_tags, pf_tagname) pf_tags = TAILQ_HEAD_INITIALIZER(pf_tags); -#else /* PF_ALTQ */ -static TAILQ_HEAD(pf_tags, pf_tagname) - pf_tags = TAILQ_HEAD_INITIALIZER(pf_tags), - pf_qids = TAILQ_HEAD_INITIALIZER(pf_qids); -#endif /* PF_ALTQ */ #if (PF_QNAME_SIZE != PF_TAG_NAME_SIZE) #error PF_QNAME_SIZE must be equal to PF_TAG_NAME_SIZE @@ -377,7 +347,7 @@ generate_token(struct proc *p) new_token = _MALLOC(sizeof (struct pfioc_kernel_token), M_TEMP, M_WAITOK|M_ZERO); - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (new_token == NULL) { /* malloc failed! bail! */ @@ -404,7 +374,7 @@ remove_token(struct pfioc_remove_token *tok) { struct pfioc_kernel_token *entry, *tmp; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); SLIST_FOREACH_SAFE(entry, &token_list_head, next, tmp) { if (tok->token_value == entry->token.token_value) { @@ -425,7 +395,7 @@ invalidate_all_tokens(void) { struct pfioc_kernel_token *entry, *tmp; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); SLIST_FOREACH_SAFE(entry, &token_list_head, next, tmp) { SLIST_REMOVE(&token_list_head, entry, pfioc_kernel_token, next); @@ -462,10 +432,6 @@ pfinit(void) "pfstatekeypl", NULL); pool_init(&pf_app_state_pl, sizeof (struct pf_app_state), 0, 0, 0, "pfappstatepl", NULL); -#if PF_ALTQ - pool_init(&pf_altq_pl, sizeof (struct pf_altq), 0, 0, 0, "pfaltqpl", - NULL); -#endif /* PF_ALTQ */ pool_init(&pf_pooladdr_pl, sizeof (struct pf_pooladdr), 0, 0, 0, "pfpooladdrpl", NULL); pfr_initialize(); @@ -484,21 +450,6 @@ pfinit(void) pf_init_ruleset(&pf_main_ruleset); TAILQ_INIT(&pf_pabuf); TAILQ_INIT(&state_list); -#if PF_ALTQ - TAILQ_INIT(&pf_altqs[0]); - TAILQ_INIT(&pf_altqs[1]); - pf_altqs_active = &pf_altqs[0]; - pf_altqs_inactive = &pf_altqs[1]; - - PE_parse_boot_argn("altq", &altq_allowed, sizeof (altq_allowed)); - - _CASSERT(ALTRQ_PURGE == CLASSQRQ_PURGE); - _CASSERT(ALTRQ_PURGE_SC == CLASSQRQ_PURGE_SC); - _CASSERT(ALTRQ_EVENT == CLASSQRQ_EVENT); - - _CASSERT(ALTDQ_REMOVE == CLASSQDQ_REMOVE); - _CASSERT(ALTDQ_POLL == CLASSQDQ_POLL); -#endif /* PF_ALTQ */ _CASSERT((SC_BE & SCIDX_MASK) == SCIDX_BE); _CASSERT((SC_BK_SYS & SCIDX_MASK) == SCIDX_BK_SYS); @@ -571,6 +522,9 @@ pfinit(void) UID_ROOT, GID_WHEEL, 0600, "pfm", 0); pf_attach_hooks(); +#if DUMMYNET + dummynet_init(); +#endif } #if 0 @@ -594,10 +548,6 @@ pfdetach(void) for (i = 0; i < PF_RULESET_MAX; i++) if (pf_begin_rules(&ticket, i, &r) == 0) pf_commit_rules(ticket, i, &r); -#if PF_ALTQ - if (pf_begin_altq(&ticket) == 0) - pf_commit_altq(ticket); -#endif /* PF_ALTQ */ /* clear states */ RB_FOREACH(state, pf_state_tree_id, &tree_id) { @@ -639,9 +589,6 @@ pfdetach(void) /* destroy the pools */ pool_destroy(&pf_pooladdr_pl); -#if PF_ALTQ - pool_destroy(&pf_altq_pl); -#endif /* PF_ALTQ */ pool_destroy(&pf_state_pl); pool_destroy(&pf_rule_pl); pool_destroy(&pf_src_tree_pl); @@ -783,13 +730,6 @@ pf_rm_rule(struct pf_rulequeue *rulequeue, struct pf_rule *rule) return; pf_tag_unref(rule->tag); pf_tag_unref(rule->match_tag); -#if PF_ALTQ - if (altq_allowed) { - if (rule->pqid != rule->qid) - pf_qid_unref(rule->pqid); - pf_qid_unref(rule->qid); - } -#endif /* PF_ALTQ */ pf_rtlabel_remove(&rule->src.addr); pf_rtlabel_remove(&rule->dst.addr); pfi_dynaddr_remove(&rule->src.addr); @@ -930,236 +870,6 @@ pf_rtlabel_copyout(struct pf_addr_wrap *a) #pragma unused(a) } -#if PF_ALTQ -u_int32_t -pf_qname2qid(char *qname) -{ - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - return ((u_int32_t)tagname2tag(&pf_qids, qname)); -} - -void -pf_qid2qname(u_int32_t qid, char *p) -{ - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - tag2tagname(&pf_qids, (u_int16_t)qid, p); -} - -void -pf_qid_unref(u_int32_t qid) -{ - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - tag_unref(&pf_qids, (u_int16_t)qid); -} - -static int -pf_begin_altq(u_int32_t *ticket) -{ - struct pf_altq *altq; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - /* Purge the old altq list */ - while ((altq = TAILQ_FIRST(pf_altqs_inactive)) != NULL) { - TAILQ_REMOVE(pf_altqs_inactive, altq, entries); - if (altq->qname[0] == '\0') { - /* detach and destroy the discipline */ - error = altq_remove(altq); - } else - pf_qid_unref(altq->qid); - pool_put(&pf_altq_pl, altq); - } - if (error) - return (error); - *ticket = ++ticket_altqs_inactive; - altqs_inactive_open = 1; - return (0); -} - -static int -pf_rollback_altq(u_int32_t ticket) -{ - struct pf_altq *altq; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if (!altqs_inactive_open || ticket != ticket_altqs_inactive) - return (0); - /* Purge the old altq list */ - while ((altq = TAILQ_FIRST(pf_altqs_inactive)) != NULL) { - TAILQ_REMOVE(pf_altqs_inactive, altq, entries); - if (altq->qname[0] == '\0') { - /* detach and destroy the discipline */ - error = altq_remove(altq); - } else - pf_qid_unref(altq->qid); - pool_put(&pf_altq_pl, altq); - } - altqs_inactive_open = 0; - return (error); -} - -static int -pf_commit_altq(u_int32_t ticket) -{ - struct pf_altqqueue *old_altqs; - struct pf_altq *altq; - int err, error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if (!altqs_inactive_open || ticket != ticket_altqs_inactive) - return (EBUSY); - - /* swap altqs, keep the old. */ - old_altqs = pf_altqs_active; - pf_altqs_active = pf_altqs_inactive; - pf_altqs_inactive = old_altqs; - ticket_altqs_active = ticket_altqs_inactive; - - /* Attach new disciplines */ - TAILQ_FOREACH(altq, pf_altqs_active, entries) { - if (altq->qname[0] == '\0') { - /* attach the discipline */ - error = altq_pfattach(altq); - if (error == 0 && pf_altq_running) - error = pf_enable_altq(altq); - if (error != 0) { - return (error); - } - } - } - - /* Purge the old altq list */ - while ((altq = TAILQ_FIRST(pf_altqs_inactive)) != NULL) { - TAILQ_REMOVE(pf_altqs_inactive, altq, entries); - if (altq->qname[0] == '\0') { - /* detach and destroy the discipline */ - if (pf_altq_running) - error = pf_disable_altq(altq); - err = altq_pfdetach(altq); - if (err != 0 && error == 0) - error = err; - err = altq_remove(altq); - if (err != 0 && error == 0) - error = err; - } else - pf_qid_unref(altq->qid); - pool_put(&pf_altq_pl, altq); - } - - altqs_inactive_open = 0; - return (error); -} - -static int -pf_enable_altq(struct pf_altq *altq) -{ - struct ifnet *ifp; - struct ifclassq *ifq; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(altq->ifname)) == NULL) - return (EINVAL); - - ifq = &ifp->if_snd; - IFCQ_LOCK(ifq); - if (IFCQ_ALTQ(ifq)->altq_type != ALTQT_NONE) - error = altq_enable(IFCQ_ALTQ(ifq)); - - /* set or clear tokenbucket regulator */ - if (error == 0 && ifp != NULL && ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) { - struct tb_profile tb = { 0, 0, 0 }; - - if (altq->aflags & PF_ALTQF_TBR) { - if (altq->bwtype != PF_ALTQ_BW_ABSOLUTE && - altq->bwtype != PF_ALTQ_BW_PERCENT) { - error = EINVAL; - } else { - if (altq->bwtype == PF_ALTQ_BW_ABSOLUTE) - tb.rate = altq->ifbandwidth; - else - tb.percent = altq->ifbandwidth; - tb.depth = altq->tbrsize; - error = ifclassq_tbr_set(ifq, &tb, TRUE); - } - } else if (IFCQ_TBR_IS_ENABLED(ifq)) { - error = ifclassq_tbr_set(ifq, &tb, TRUE); - } - } - IFCQ_UNLOCK(ifq); - - return (error); -} - -static int -pf_disable_altq(struct pf_altq *altq) -{ - struct ifnet *ifp; - struct ifclassq *ifq; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(altq->ifname)) == NULL) - return (EINVAL); - - /* - * when the discipline is no longer referenced, it was overridden - * by a new one. if so, just return. - */ - ifq = &ifp->if_snd; - IFCQ_LOCK(ifq); - if (altq->altq_disc != IFCQ_ALTQ(ifq)->altq_disc) { - IFCQ_UNLOCK(ifq); - return (0); - } - - error = altq_disable(IFCQ_ALTQ(ifq)); - - if (error == 0 && IFCQ_TBR_IS_ENABLED(ifq)) { - /* clear tokenbucket regulator */ - struct tb_profile tb = { 0, 0, 0 }; - error = ifclassq_tbr_set(ifq, &tb, TRUE); - } - IFCQ_UNLOCK(ifq); - - return (error); -} - -static void -pf_altq_copyin(struct pf_altq *src, struct pf_altq *dst) -{ - bcopy(src, dst, sizeof (struct pf_altq)); - - dst->ifname[sizeof (dst->ifname) - 1] = '\0'; - dst->qname[sizeof (dst->qname) - 1] = '\0'; - dst->parent[sizeof (dst->parent) - 1] = '\0'; - dst->altq_disc = NULL; - dst->entries.tqe_next = NULL; - dst->entries.tqe_prev = NULL; -} - -static void -pf_altq_copyout(struct pf_altq *src, struct pf_altq *dst) -{ - struct pf_altq pa; - - bcopy(src, &pa, sizeof (struct pf_altq)); - pa.altq_disc = NULL; - pa.entries.tqe_next = NULL; - pa.entries.tqe_prev = NULL; - bcopy(&pa, dst, sizeof (struct pf_altq)); -} -#endif /* PF_ALTQ */ - static int pf_begin_rules(u_int32_t *ticket, int rs_num, const char *anchor) { @@ -1301,7 +1011,7 @@ pf_commit_rules(u_int32_t ticket, int rs_num, char *anchor) int error; u_int32_t old_rcount; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (rs_num < 0 || rs_num >= PF_RULESET_MAX) return (EINVAL); @@ -1572,7 +1282,7 @@ pf_setup_pfsync_matching(struct pf_ruleset *rs) static void pf_start(void) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); VERIFY(pf_is_enabled == 0); @@ -1590,7 +1300,7 @@ pf_start(void) static void pf_stop(void) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); VERIFY(pf_is_enabled); @@ -1739,22 +1449,6 @@ pfioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p) return (EACCES); } -#if PF_ALTQ - switch (cmd) { - case DIOCSTARTALTQ: - case DIOCSTOPALTQ: - case DIOCADDALTQ: - case DIOCGETALTQS: - case DIOCGETALTQ: - case DIOCCHANGEALTQ: - case DIOCGETQSTATS: - /* fail if ALTQ is disabled */ - if (!altq_allowed) - return (ENODEV); - break; - } -#endif /* PF_ALTQ */ - if (flags & FWRITE) lck_rw_lock_exclusive(pf_perim_lock); else @@ -2006,171 +1700,6 @@ pfioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p) break; } -#if PF_ALTQ - case DIOCSTARTALTQ: { - struct pf_altq *altq; - - VERIFY(altq_allowed); - /* enable all altq interfaces on active list */ - TAILQ_FOREACH(altq, pf_altqs_active, entries) { - if (altq->qname[0] == '\0') { - error = pf_enable_altq(altq); - if (error != 0) - break; - } - } - if (error == 0) - pf_altq_running = 1; - DPFPRINTF(PF_DEBUG_MISC, ("altq: started\n")); - break; - } - - case DIOCSTOPALTQ: { - struct pf_altq *altq; - - VERIFY(altq_allowed); - /* disable all altq interfaces on active list */ - TAILQ_FOREACH(altq, pf_altqs_active, entries) { - if (altq->qname[0] == '\0') { - error = pf_disable_altq(altq); - if (error != 0) - break; - } - } - if (error == 0) - pf_altq_running = 0; - DPFPRINTF(PF_DEBUG_MISC, ("altq: stopped\n")); - break; - } - - case DIOCADDALTQ: { /* struct pfioc_altq */ - struct pfioc_altq *pa = (struct pfioc_altq *)(void *)addr; - struct pf_altq *altq, *a; - u_int32_t ticket; - - VERIFY(altq_allowed); - bcopy(&pa->ticket, &ticket, sizeof (ticket)); - if (ticket != ticket_altqs_inactive) { - error = EBUSY; - break; - } - altq = pool_get(&pf_altq_pl, PR_WAITOK); - if (altq == NULL) { - error = ENOMEM; - break; - } - pf_altq_copyin(&pa->altq, altq); - - /* - * if this is for a queue, find the discipline and - * copy the necessary fields - */ - if (altq->qname[0] != '\0') { - if ((altq->qid = pf_qname2qid(altq->qname)) == 0) { - error = EBUSY; - pool_put(&pf_altq_pl, altq); - break; - } - altq->altq_disc = NULL; - TAILQ_FOREACH(a, pf_altqs_inactive, entries) { - if (strncmp(a->ifname, altq->ifname, - IFNAMSIZ) == 0 && a->qname[0] == '\0') { - altq->altq_disc = a->altq_disc; - break; - } - } - } - - error = altq_add(altq); - if (error) { - pool_put(&pf_altq_pl, altq); - break; - } - - TAILQ_INSERT_TAIL(pf_altqs_inactive, altq, entries); - pf_altq_copyout(altq, &pa->altq); - break; - } - - case DIOCGETALTQS: { - struct pfioc_altq *pa = (struct pfioc_altq *)(void *)addr; - struct pf_altq *altq; - u_int32_t nr; - - VERIFY(altq_allowed); - nr = 0; - TAILQ_FOREACH(altq, pf_altqs_active, entries) - nr++; - bcopy(&nr, &pa->nr, sizeof (nr)); - bcopy(&ticket_altqs_active, &pa->ticket, sizeof (pa->ticket)); - break; - } - - case DIOCGETALTQ: { - struct pfioc_altq *pa = (struct pfioc_altq *)(void *)addr; - struct pf_altq *altq; - u_int32_t nr, pa_nr, ticket; - - VERIFY(altq_allowed); - bcopy(&pa->ticket, &ticket, sizeof (ticket)); - if (ticket != ticket_altqs_active) { - error = EBUSY; - break; - } - bcopy(&pa->nr, &pa_nr, sizeof (pa_nr)); - nr = 0; - altq = TAILQ_FIRST(pf_altqs_active); - while ((altq != NULL) && (nr < pa_nr)) { - altq = TAILQ_NEXT(altq, entries); - nr++; - } - if (altq == NULL) { - error = EBUSY; - break; - } - pf_altq_copyout(altq, &pa->altq); - break; - } - - case DIOCCHANGEALTQ: - VERIFY(altq_allowed); - /* CHANGEALTQ not supported yet! */ - error = ENODEV; - break; - - case DIOCGETQSTATS: { - struct pfioc_qstats *pq = (struct pfioc_qstats *)(void *)addr; - struct pf_altq *altq; - u_int32_t nr, pq_nr, ticket; - int nbytes; - - VERIFY(altq_allowed); - bcopy(&pq->ticket, &ticket, sizeof (ticket)); - if (ticket != ticket_altqs_active) { - error = EBUSY; - break; - } - bcopy(&pq->nr, &pq_nr, sizeof (pq_nr)); - nr = 0; - altq = TAILQ_FIRST(pf_altqs_active); - while ((altq != NULL) && (nr < pq_nr)) { - altq = TAILQ_NEXT(altq, entries); - nr++; - } - if (altq == NULL) { - error = EBUSY; - break; - } - bcopy(&pq->nbytes, &nbytes, sizeof (nbytes)); - error = altq_getqstats(altq, pq->buf, &nbytes); - if (error == 0) { - pq->scheduler = altq->scheduler; - bcopy(&nbytes, &pq->nbytes, sizeof (nbytes)); - } - break; - } -#endif /* PF_ALTQ */ - case DIOCBEGINADDRS: /* struct pfioc_pooladdr */ case DIOCADDADDR: /* struct pfioc_pooladdr */ case DIOCGETADDRS: /* struct pfioc_pooladdr */ @@ -3037,19 +2566,6 @@ pf_rule_setup(struct pfioc_rule *pr, struct pf_rule *rule, } pfi_kif_ref(rule->kif, PFI_KIF_REF_RULE); } -#if PF_ALTQ - /* set queue IDs */ - if (altq_allowed && rule->qname[0] != '\0') { - if ((rule->qid = pf_qname2qid(rule->qname)) == 0) - error = EBUSY; - else if (rule->pqname[0] != '\0') { - if ((rule->pqid = - pf_qname2qid(rule->pqname)) == 0) - error = EBUSY; - } else - rule->pqid = rule->qid; - } -#endif /* PF_ALTQ */ if (rule->tagname[0]) if ((rule->tag = pf_tagname2tag(rule->tagname)) == 0) error = EBUSY; @@ -3191,6 +2707,38 @@ pfioctl_ioc_rule(u_long cmd, int minordev, struct pfioc_rule *pr, struct proc *p if (rule->action == PF_NAT64) atomic_add_16(&pf_nat64_configured, 1); + + if (pr->anchor_call[0] == '\0') { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_pf_addrule_total); + if (rule->rule_flag & PFRULE_PFM) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_pf_addrule_os); + } + } + +#if DUMMYNET + if (rule->action == PF_DUMMYNET) { + struct dummynet_event dn_event; + uint32_t direction = DN_INOUT;; + bzero(&dn_event, sizeof(dn_event)); + + dn_event.dn_event_code = DUMMYNET_RULE_CONFIG; + + if (rule->direction == PF_IN) + direction = DN_IN; + else if (rule->direction == PF_OUT) + direction = DN_OUT; + + dn_event.dn_event_rule_config.dir = direction; + dn_event.dn_event_rule_config.af = rule->af; + dn_event.dn_event_rule_config.proto = rule->proto; + dn_event.dn_event_rule_config.src_port = rule->src.xport.range.port[0]; + dn_event.dn_event_rule_config.dst_port = rule->dst.xport.range.port[0]; + strlcpy(dn_event.dn_event_rule_config.ifname, rule->ifname, + sizeof(dn_event.dn_event_rule_config.ifname)); + + dummynet_event_enqueue_nwk_wq_entry(&dn_event); + } +#endif break; } @@ -3355,20 +2903,6 @@ pfioctl_ioc_rule(u_long cmd, int minordev, struct pfioc_rule *pr, struct proc *p } else newrule->kif = NULL; -#if PF_ALTQ - /* set queue IDs */ - if (altq_allowed && newrule->qname[0] != '\0') { - if ((newrule->qid = - pf_qname2qid(newrule->qname)) == 0) - error = EBUSY; - else if (newrule->pqname[0] != '\0') { - if ((newrule->pqid = - pf_qname2qid(newrule->pqname)) == 0) - error = EBUSY; - } else - newrule->pqid = newrule->qid; - } -#endif /* PF_ALTQ */ if (newrule->tagname[0]) if ((newrule->tag = pf_tagname2tag(newrule->tagname)) == 0) @@ -3593,6 +3127,13 @@ pfioctl_ioc_rule(u_long cmd, int minordev, struct pfioc_rule *pr, struct proc *p pffwrules++; if (rule->action == PF_NAT64) atomic_add_16(&pf_nat64_configured, 1); + + if (pr->anchor_call[0] == '\0') { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_pf_addrule_total); + if (rule->rule_flag & PFRULE_PFM) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_pf_addrule_os); + } + } break; } @@ -4401,22 +3942,6 @@ pfioctl_ioc_trans(u_long cmd, struct pfioc_trans_32 *io32, ioe->anchor[sizeof (ioe->anchor) - 1] = '\0'; switch (ioe->rs_num) { case PF_RULESET_ALTQ: -#if PF_ALTQ - if (altq_allowed) { - if (ioe->anchor[0]) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - error = EINVAL; - goto fail; - } - error = pf_begin_altq(&ioe->ticket); - if (error != 0) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - goto fail; - } - } -#endif /* PF_ALTQ */ break; case PF_RULESET_TABLE: bzero(table, sizeof (*table)); @@ -4471,22 +3996,6 @@ pfioctl_ioc_trans(u_long cmd, struct pfioc_trans_32 *io32, ioe->anchor[sizeof (ioe->anchor) - 1] = '\0'; switch (ioe->rs_num) { case PF_RULESET_ALTQ: -#if PF_ALTQ - if (altq_allowed) { - if (ioe->anchor[0]) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - error = EINVAL; - goto fail; - } - error = pf_rollback_altq(ioe->ticket); - if (error != 0) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - goto fail; /* really bad */ - } - } -#endif /* PF_ALTQ */ break; case PF_RULESET_TABLE: bzero(table, sizeof (*table)); @@ -4538,24 +4047,6 @@ pfioctl_ioc_trans(u_long cmd, struct pfioc_trans_32 *io32, ioe->anchor[sizeof (ioe->anchor) - 1] = '\0'; switch (ioe->rs_num) { case PF_RULESET_ALTQ: -#if PF_ALTQ - if (altq_allowed) { - if (ioe->anchor[0]) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - error = EINVAL; - goto fail; - } - if (!altqs_inactive_open || - ioe->ticket != - ticket_altqs_inactive) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - error = EBUSY; - goto fail; - } - } -#endif /* PF_ALTQ */ break; case PF_RULESET_TABLE: rs = pf_find_ruleset(ioe->anchor); @@ -4600,14 +4091,6 @@ pfioctl_ioc_trans(u_long cmd, struct pfioc_trans_32 *io32, ioe->anchor[sizeof (ioe->anchor) - 1] = '\0'; switch (ioe->rs_num) { case PF_RULESET_ALTQ: -#if PF_ALTQ - if (altq_allowed && - (error = pf_commit_altq(ioe->ticket))) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - goto fail; /* really bad */ - } -#endif /* PF_ALTQ */ break; case PF_RULESET_TABLE: bzero(table, sizeof (*table)); @@ -4967,7 +4450,7 @@ pf_inet_hook(struct ifnet *ifp, struct mbuf **mp, int input, HTONS(ip->ip_len); HTONS(ip->ip_off); #endif - if (pf_test(input ? PF_IN : PF_OUT, ifp, mp, NULL, fwa) != PF_PASS) { + if (pf_test_mbuf(input ? PF_IN : PF_OUT, ifp, mp, NULL, fwa) != PF_PASS) { if (*mp != NULL) { m_freem(*mp); *mp = NULL; @@ -5018,7 +4501,7 @@ pf_inet6_hook(struct ifnet *ifp, struct mbuf **mp, int input, } } - if (pf_test6(input ? PF_IN : PF_OUT, ifp, mp, NULL, fwa) != PF_PASS) { + if (pf_test6_mbuf(input ? PF_IN : PF_OUT, ifp, mp, NULL, fwa) != PF_PASS) { if (*mp != NULL) { m_freem(*mp); *mp = NULL; diff --git a/bsd/net/pf_norm.c b/bsd/net/pf_norm.c index 547ff705f..3cb22e9ce 100644 --- a/bsd/net/pf_norm.c +++ b/bsd/net/pf_norm.c @@ -134,6 +134,8 @@ struct pf_fragment { LIST_HEAD(pf_fragq, pf_frent) fru_queue; /* buffering */ LIST_HEAD(pf_cacheq, pf_frcache) fru_cache; /* non-buf */ } fr_u; + uint32_t fr_csum_flags; /* checksum flags */ + uint32_t fr_csum; /* partial checksum value */ }; static TAILQ_HEAD(pf_fragqueue, pf_fragment) pf_fragqueue; @@ -160,7 +162,7 @@ static __inline struct pf_fragment * static __inline struct pf_fragment * pf_find_fragment_by_ipv6_header(struct ip6_hdr *, struct ip6_frag *, struct pf_frag_tree *); -static struct mbuf *pf_reassemble(struct mbuf **, struct pf_fragment **, +static struct mbuf *pf_reassemble(struct mbuf *, struct pf_fragment **, struct pf_frent *, int); static struct mbuf *pf_fragcache(struct mbuf **, struct ip *, struct pf_fragment **, int, int, int *); @@ -169,7 +171,7 @@ static struct mbuf *pf_reassemble6(struct mbuf **, struct pf_fragment **, static struct mbuf *pf_frag6cache(struct mbuf **, struct ip6_hdr*, struct ip6_frag *, struct pf_fragment **, int, int, int, int *); static int pf_normalize_tcpopt(struct pf_rule *, int, struct pfi_kif *, - struct pf_pdesc *, struct mbuf *, struct tcphdr *, int, int *); + struct pf_pdesc *, pbuf_t *, struct tcphdr *, int, int *); #define DPFPRINTF(x) do { \ if (pf_status.debug >= PF_DEBUG_MISC) { \ @@ -246,13 +248,13 @@ pf_frag_compare(struct pf_fragment *a, struct pf_fragment *b) case AF_INET: if ((diff = a->fr_id - b->fr_id)) return (diff); - else if (sa->v4.s_addr < sb->v4.s_addr) + else if (sa->v4addr.s_addr < sb->v4addr.s_addr) return (-1); - else if (sa->v4.s_addr > sb->v4.s_addr) + else if (sa->v4addr.s_addr > sb->v4addr.s_addr) return (1); - else if (da->v4.s_addr < db->v4.s_addr) + else if (da->v4addr.s_addr < db->v4addr.s_addr) return (-1); - else if (da->v4.s_addr > db->v4.s_addr) + else if (da->v4addr.s_addr > db->v4addr.s_addr) return (1); break; #endif @@ -432,8 +434,8 @@ pf_ip6hdr2key(struct pf_fragment *key, struct ip6_hdr *ip6, key->fr_p = fh->ip6f_nxt; key->fr_id6 = fh->ip6f_ident; key->fr_af = AF_INET6; - key->fr_srcx.v6 = ip6->ip6_src; - key->fr_dstx.v6 = ip6->ip6_dst; + key->fr_srcx.v6addr = ip6->ip6_src; + key->fr_dstx.v6addr = ip6->ip6_dst; } static void @@ -442,8 +444,8 @@ pf_ip2key(struct pf_fragment *key, struct ip *ip) key->fr_p = ip->ip_p; key->fr_id = ip->ip_id; key->fr_af = AF_INET; - key->fr_srcx.v4.s_addr = ip->ip_src.s_addr; - key->fr_dstx.v4.s_addr = ip->ip_dst.s_addr; + key->fr_srcx.v4addr.s_addr = ip->ip_src.s_addr; + key->fr_dstx.v4addr.s_addr = ip->ip_dst.s_addr; } static struct pf_fragment * @@ -502,20 +504,79 @@ pf_remove_fragment(struct pf_fragment *frag) #define FR_IP_OFF(fr) ((ntohs((fr)->fr_ip->ip_off) & IP_OFFMASK) << 3) static struct mbuf * -pf_reassemble(struct mbuf **m0, struct pf_fragment **frag, +pf_reassemble(struct mbuf *m0, struct pf_fragment **frag, struct pf_frent *frent, int mff) { - struct mbuf *m = *m0, *m2; + struct mbuf *m = m0, *m2; struct pf_frent *frea, *next; struct pf_frent *frep = NULL; struct ip *ip = frent->fr_ip; - int hlen = ip->ip_hl << 2; + uint32_t hlen = ip->ip_hl << 2; u_int16_t off = (ntohs(ip->ip_off) & IP_OFFMASK) << 3; u_int16_t ip_len = ntohs(ip->ip_len) - ip->ip_hl * 4; u_int16_t fr_max = ip_len + off; + uint32_t csum, csum_flags; VERIFY(*frag == NULL || BUFFER_FRAGMENTS(*frag)); + /* + * Leverage partial checksum offload for IP fragments. Narrow down + * the scope to cover only UDP without IP options, as that is the + * most common case. + * + * Perform 1's complement adjustment of octets that got included/ + * excluded in the hardware-calculated checksum value. Ignore cases + * where the value includes the entire IPv4 header span, as the sum + * for those octets would already be 0 by the time we get here; IP + * has already performed its header checksum validation. Also take + * care of any trailing bytes and subtract out their partial sum. + */ + if (ip->ip_p == IPPROTO_UDP && hlen == sizeof (struct ip) && + (m->m_pkthdr.csum_flags & + (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) == + (CSUM_DATA_VALID | CSUM_PARTIAL)) { + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - ntohs(ip->ip_len)); + uint32_t swbytes = (uint32_t)trailer; + + csum = m->m_pkthdr.csum_rx_val; + + ASSERT(trailer >= 0); + if ((start != 0 && start != hlen) || trailer != 0) { +#if BYTE_ORDER != BIG_ENDIAN + if (start < hlen) { + HTONS(ip->ip_len); + HTONS(ip->ip_off); + } +#endif /* BYTE_ORDER != BIG_ENDIAN */ + /* callee folds in sum */ + csum = m_adj_sum16(m, start, hlen, + (ip->ip_len - hlen), csum); + if (hlen > start) + swbytes += (hlen - start); + else + swbytes += (start - hlen); +#if BYTE_ORDER != BIG_ENDIAN + if (start < hlen) { + NTOHS(ip->ip_off); + NTOHS(ip->ip_len); + } +#endif /* BYTE_ORDER != BIG_ENDIAN */ + } + csum_flags = m->m_pkthdr.csum_flags; + + if (swbytes != 0) + udp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); + } else { + csum = 0; + csum_flags = 0; + } + + /* Invalidate checksum */ + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + /* Strip off ip header */ m->m_data += hlen; m->m_len -= hlen; @@ -533,11 +594,15 @@ pf_reassemble(struct mbuf **m0, struct pf_fragment **frag, (*frag)->fr_flags = 0; (*frag)->fr_max = 0; (*frag)->fr_af = AF_INET; - (*frag)->fr_srcx.v4 = frent->fr_ip->ip_src; - (*frag)->fr_dstx.v4 = frent->fr_ip->ip_dst; + (*frag)->fr_srcx.v4addr = frent->fr_ip->ip_src; + (*frag)->fr_dstx.v4addr = frent->fr_ip->ip_dst; (*frag)->fr_p = frent->fr_ip->ip_p; (*frag)->fr_id = frent->fr_ip->ip_id; (*frag)->fr_timeout = pf_time_second(); + if (csum_flags != 0) { + (*frag)->fr_csum_flags = csum_flags; + (*frag)->fr_csum = csum; + } LIST_INIT(&(*frag)->fr_queue); RB_INSERT(pf_frag_tree, &pf_frag_tree, *frag); @@ -548,6 +613,16 @@ pf_reassemble(struct mbuf **m0, struct pf_fragment **frag, goto insert; } + /* + * If this fragment contains similar checksum offload info + * as that of the existing ones, accumulate checksum. Otherwise, + * invalidate checksum offload info for the entire datagram. + */ + if (csum_flags != 0 && csum_flags == (*frag)->fr_csum_flags) + (*frag)->fr_csum += csum; + else if ((*frag)->fr_csum_flags != 0) + (*frag)->fr_csum_flags = 0; + /* * Find a fragment after the current one: * - off contains the real shifted offset. @@ -665,8 +740,26 @@ insert: m_cat(m, m2); } - ip->ip_src = (*frag)->fr_srcx.v4; - ip->ip_dst = (*frag)->fr_dstx.v4; + ip->ip_src = (*frag)->fr_srcx.v4addr; + ip->ip_dst = (*frag)->fr_dstx.v4addr; + + if ((*frag)->fr_csum_flags != 0) { + csum = (*frag)->fr_csum; + + ADDCARRY(csum); + + m->m_pkthdr.csum_rx_val = csum; + m->m_pkthdr.csum_rx_start = sizeof (struct ip); + m->m_pkthdr.csum_flags = (*frag)->fr_csum_flags; + } else if ((m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) || + (m->m_pkthdr.pkt_flags & PKTF_LOOP)) { + /* loopback checksums are always OK */ + m->m_pkthdr.csum_data = 0xffff; + m->m_pkthdr.csum_flags &= ~CSUM_PARTIAL; + m->m_pkthdr.csum_flags = + CSUM_DATA_VALID | CSUM_PSEUDO_HDR | + CSUM_IP_CHECKED | CSUM_IP_VALID; + } /* Remove from fragment queue */ pf_remove_fragment(*frag); @@ -733,8 +826,8 @@ pf_fragcache(struct mbuf **m0, struct ip *h, struct pf_fragment **frag, int mff, (*frag)->fr_flags = PFFRAG_NOBUFFER; (*frag)->fr_max = 0; (*frag)->fr_af = AF_INET; - (*frag)->fr_srcx.v4 = h->ip_src; - (*frag)->fr_dstx.v4 = h->ip_dst; + (*frag)->fr_srcx.v4addr = h->ip_src; + (*frag)->fr_dstx.v4addr = h->ip_dst; (*frag)->fr_p = h->ip_p; (*frag)->fr_id = h->ip_id; (*frag)->fr_timeout = pf_time_second(); @@ -1008,19 +1101,84 @@ pf_reassemble6(struct mbuf **m0, struct pf_fragment **frag, struct mbuf *m, *m2; struct pf_frent *frea, *frep, *next; struct ip6_hdr *ip6; + struct ip6_frag *ip6f; int plen, off, fr_max; + uint32_t uoff, csum, csum_flags; VERIFY(*frag == NULL || BUFFER_FRAGMENTS(*frag)); m = *m0; frep = NULL; ip6 = frent->fr_ip6; + ip6f = &frent->fr_ip6f_opt; off = FR_IP6_OFF(frent); + uoff = frent->fr_ip6f_hlen; plen = FR_IP6_PLEN(frent); fr_max = off + plen - (frent->fr_ip6f_hlen - sizeof *ip6); DPFPRINTF(("0x%llx IPv6 frag plen %u off %u fr_ip6f_hlen %u " "fr_max %u m_len %u\n", (uint64_t)VM_KERNEL_ADDRPERM(m), plen, off, frent->fr_ip6f_hlen, fr_max, m->m_len)); + + /* + * Leverage partial checksum offload for simple UDP/IP fragments, + * as that is the most common case. + * + * Perform 1's complement adjustment of octets that got included/ + * excluded in the hardware-calculated checksum value. Also take + * care of any trailing bytes and subtract out their partial sum. + */ + if (ip6f->ip6f_nxt == IPPROTO_UDP && + uoff == (sizeof (*ip6) + sizeof (*ip6f)) && + (m->m_pkthdr.csum_flags & + (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) == + (CSUM_DATA_VALID | CSUM_PARTIAL)) { + uint32_t start = m->m_pkthdr.csum_rx_start; + uint32_t ip_len = (sizeof (*ip6) + ntohs(ip6->ip6_plen)); + int32_t trailer = (m_pktlen(m) - ip_len); + uint32_t swbytes = (uint32_t)trailer; + + csum = m->m_pkthdr.csum_rx_val; + + ASSERT(trailer >= 0); + if (start != uoff || trailer != 0) { + uint16_t s = 0, d = 0; + + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { + s = ip6->ip6_src.s6_addr16[1]; + ip6->ip6_src.s6_addr16[1] = 0 ; + } + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) { + d = ip6->ip6_dst.s6_addr16[1]; + ip6->ip6_dst.s6_addr16[1] = 0; + } + + /* callee folds in sum */ + csum = m_adj_sum16(m, start, uoff, + (ip_len - uoff), csum); + if (uoff > start) + swbytes += (uoff - start); + else + swbytes += (start - uoff); + + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) + ip6->ip6_src.s6_addr16[1] = s; + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) + ip6->ip6_dst.s6_addr16[1] = d; + + } + csum_flags = m->m_pkthdr.csum_flags; + + if (swbytes != 0) + udp_in6_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); + } else { + csum = 0; + csum_flags = 0; + } + + /* Invalidate checksum */ + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; /* strip off headers up to the fragment payload */ m->m_data += frent->fr_ip6f_hlen; @@ -1039,11 +1197,15 @@ pf_reassemble6(struct mbuf **m0, struct pf_fragment **frag, (*frag)->fr_flags = 0; (*frag)->fr_max = 0; (*frag)->fr_af = AF_INET6; - (*frag)->fr_srcx.v6 = frent->fr_ip6->ip6_src; - (*frag)->fr_dstx.v6 = frent->fr_ip6->ip6_dst; + (*frag)->fr_srcx.v6addr = frent->fr_ip6->ip6_src; + (*frag)->fr_dstx.v6addr = frent->fr_ip6->ip6_dst; (*frag)->fr_p = frent->fr_ip6f_opt.ip6f_nxt; (*frag)->fr_id6 = frent->fr_ip6f_opt.ip6f_ident; (*frag)->fr_timeout = pf_time_second(); + if (csum_flags != 0) { + (*frag)->fr_csum_flags = csum_flags; + (*frag)->fr_csum = csum; + } LIST_INIT(&(*frag)->fr_queue); RB_INSERT(pf_frag_tree, &pf_frag_tree, *frag); @@ -1053,6 +1215,16 @@ pf_reassemble6(struct mbuf **m0, struct pf_fragment **frag, frep = NULL; goto insert; } + + /* + * If this fragment contains similar checksum offload info + * as that of the existing ones, accumulate checksum. Otherwise, + * invalidate checksum offload info for the entire datagram. + */ + if (csum_flags != 0 && csum_flags == (*frag)->fr_csum_flags) + (*frag)->fr_csum += csum; + else if ((*frag)->fr_csum_flags != 0) + (*frag)->fr_csum_flags = 0; /* * Find a fragment after the current one: @@ -1159,8 +1331,24 @@ pf_reassemble6(struct mbuf **m0, struct pf_fragment **frag, ip6 = frent->fr_ip6; ip6->ip6_nxt = (*frag)->fr_p; ip6->ip6_plen = htons(off); - ip6->ip6_src = (*frag)->fr_srcx.v6; - ip6->ip6_dst = (*frag)->fr_dstx.v6; + ip6->ip6_src = (*frag)->fr_srcx.v6addr; + ip6->ip6_dst = (*frag)->fr_dstx.v6addr; + + if ((*frag)->fr_csum_flags != 0) { + csum = (*frag)->fr_csum; + + ADDCARRY(csum); + + m->m_pkthdr.csum_rx_val = csum; + m->m_pkthdr.csum_rx_start = sizeof (struct ip6_hdr); + m->m_pkthdr.csum_flags = (*frag)->fr_csum_flags; + } else if ((m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) || + (m->m_pkthdr.pkt_flags & PKTF_LOOP)) { + /* loopback checksums are always OK */ + m->m_pkthdr.csum_data = 0xffff; + m->m_pkthdr.csum_flags &= ~CSUM_PARTIAL; + m->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR; + } /* Remove from fragment queue */ pf_remove_fragment(*frag); @@ -1263,8 +1451,8 @@ pf_frag6cache(struct mbuf **m0, struct ip6_hdr *h, struct ip6_frag *fh, (*frag)->fr_flags = PFFRAG_NOBUFFER; (*frag)->fr_max = 0; (*frag)->fr_af = AF_INET6; - (*frag)->fr_srcx.v6 = h->ip6_src; - (*frag)->fr_dstx.v6 = h->ip6_dst; + (*frag)->fr_srcx.v6addr = h->ip6_src; + (*frag)->fr_dstx.v6addr = h->ip6_dst; (*frag)->fr_p = fh->ip6f_nxt; (*frag)->fr_id6 = fh->ip6f_ident; (*frag)->fr_timeout = pf_time_second(); @@ -1530,14 +1718,14 @@ pf_frag6cache(struct mbuf **m0, struct ip6_hdr *h, struct ip6_frag *fh, } int -pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, +pf_normalize_ip(pbuf_t *pbuf, int dir, struct pfi_kif *kif, u_short *reason, struct pf_pdesc *pd) { - struct mbuf *m = *m0; + struct mbuf *m; struct pf_rule *r; struct pf_frent *frent; struct pf_fragment *frag = NULL; - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; int mff = (ntohs(h->ip_off) & IP_MF); int hlen = h->ip_hl << 2; u_int16_t fragoff = (ntohs(h->ip_off) & IP_OFFMASK) << 3; @@ -1546,6 +1734,7 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, int ip_off; int asd = 0; struct pf_ruleset *ruleset = NULL; + struct ifnet *ifp = pbuf->pb_ifp; r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_SCRUB].active.ptr); while (r != NULL) { @@ -1639,10 +1828,21 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, fr_max > frag->fr_max) goto bad; + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + REASON_SET(reason, PFRES_MEMORY); + return (PF_DROP); + } + + VERIFY(!pbuf_is_valid(pbuf)); + + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip *); + /* Get an entry for the fragment queue */ frent = pool_get(&pf_frent_pl, PR_NOWAIT); if (frent == NULL) { REASON_SET(reason, PFRES_MEMORY); + m_freem(m); return (PF_DROP); } pf_nfrents++; @@ -1652,29 +1852,34 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, /* Might return a completely reassembled mbuf, or NULL */ DPFPRINTF(("reass IPv4 frag %d @ %d-%d\n", ntohs(h->ip_id), fragoff, fr_max)); - *m0 = m = pf_reassemble(m0, &frag, frent, mff); + m = pf_reassemble(m, &frag, frent, mff); if (m == NULL) return (PF_DROP); VERIFY(m->m_flags & M_PKTHDR); + pbuf_init_mbuf(pbuf, m, ifp); /* use mtag from concatenated mbuf chain */ - pd->pf_mtag = pf_find_mtag(m); + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); +#if 0 +// SCW: This check is superfluous #if DIAGNOSTIC if (pd->pf_mtag == NULL) { printf("%s: pf_find_mtag returned NULL(1)\n", __func__); if ((pd->pf_mtag = pf_get_mtag(m)) == NULL) { m_freem(m); - m = *m0 = NULL; + m = NULL; goto no_mem; } } #endif - if (frag != NULL && (frag->fr_flags & PFFRAG_DROP)) - goto drop; +#endif h = mtod(m, struct ip *); + + if (frag != NULL && (frag->fr_flags & PFFRAG_DROP)) + goto drop; } else { /* non-buffering fragment cache (drops or masks overlaps) */ int nomem = 0; @@ -1698,33 +1903,49 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, goto bad; } - *m0 = m = pf_fragcache(m0, h, &frag, mff, + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + REASON_SET(reason, PFRES_MEMORY); + goto bad; + } + + VERIFY(!pbuf_is_valid(pbuf)); + + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip *); + + m = pf_fragcache(&m, h, &frag, mff, (r->rule_flag & PFRULE_FRAGDROP) ? 1 : 0, &nomem); if (m == NULL) { + // Note: pf_fragcache() has already m_freem'd the mbuf if (nomem) goto no_mem; goto drop; } VERIFY(m->m_flags & M_PKTHDR); + pbuf_init_mbuf(pbuf, m, ifp); /* use mtag from copied and trimmed mbuf chain */ - pd->pf_mtag = pf_find_mtag(m); + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); +#if 0 +// SCW: This check is superfluous #if DIAGNOSTIC if (pd->pf_mtag == NULL) { printf("%s: pf_find_mtag returned NULL(2)\n", __func__); if ((pd->pf_mtag = pf_get_mtag(m)) == NULL) { m_freem(m); - m = *m0 = NULL; + m = NULL; goto no_mem; } } +#endif #endif if (dir == PF_IN) pd->pf_mtag->pftag_flags |= PF_TAG_FRAGCACHE; if (frag != NULL && (frag->fr_flags & PFFRAG_DROP)) goto drop; + goto fragment_pass; } @@ -1747,7 +1968,11 @@ no_fragment: if (r->rule_flag & PFRULE_RANDOMID) { u_int16_t oip_id = h->ip_id; - h->ip_id = ip_randomid(); + if (rfc6864 && IP_OFF_IS_ATOMIC(ntohs(h->ip_off))) { + h->ip_id = 0; + } else { + h->ip_id = ip_randomid(); + } h->ip_sum = pf_cksum_fixup(h->ip_sum, oip_id, h->ip_id, 0); } if ((r->rule_flag & (PFRULE_FRAGCROP|PFRULE_FRAGDROP)) == 0) @@ -1769,15 +1994,15 @@ fragment_pass: no_mem: REASON_SET(reason, PFRES_MEMORY); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, *reason, r, + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); drop: REASON_SET(reason, PFRES_NORM); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, *reason, r, + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); @@ -1789,20 +2014,20 @@ bad: pf_free_fragment(frag); REASON_SET(reason, PFRES_FRAG); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, *reason, r, NULL, NULL, pd); + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); } #if INET6 int -pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, +pf_normalize_ip6(pbuf_t *pbuf, int dir, struct pfi_kif *kif, u_short *reason, struct pf_pdesc *pd) { - struct mbuf *m = *m0; + struct mbuf *m; struct pf_rule *r; - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; int off; struct ip6_ext ext; /* adi XXX */ @@ -1823,6 +2048,7 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, u_int16_t fr_max; int asd = 0; struct pf_ruleset *ruleset = NULL; + struct ifnet *ifp = pbuf->pb_ifp; r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_SCRUB].active.ptr); while (r != NULL) { @@ -1838,11 +2064,11 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, r = r->skip[PF_SKIP_PROTO].ptr; #endif else if (PF_MISMATCHAW(&r->src.addr, - (struct pf_addr *)&h->ip6_src, AF_INET6, + (struct pf_addr *)(uintptr_t)&h->ip6_src, AF_INET6, r->src.neg, kif)) r = r->skip[PF_SKIP_SRC_ADDR].ptr; else if (PF_MISMATCHAW(&r->dst.addr, - (struct pf_addr *)&h->ip6_dst, AF_INET6, + (struct pf_addr *)(uintptr_t)&h->ip6_dst, AF_INET6, r->dst.neg, NULL)) r = r->skip[PF_SKIP_DST_ADDR].ptr; else { @@ -1865,7 +2091,8 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, } /* Check for illegal packets */ - if ((int)(sizeof (struct ip6_hdr) + IPV6_MAXPACKET) < m->m_pkthdr.len) + if ((uint32_t)(sizeof (struct ip6_hdr) + IPV6_MAXPACKET) < + pbuf->pb_packet_len) goto drop; off = sizeof (struct ip6_hdr); @@ -1879,7 +2106,7 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, case IPPROTO_AH: case IPPROTO_ROUTING: case IPPROTO_DSTOPTS: - if (!pf_pull_hdr(m, off, &ext, sizeof (ext), NULL, + if (!pf_pull_hdr(pbuf, off, &ext, sizeof (ext), NULL, NULL, AF_INET6)) goto shortpkt; /* @@ -1964,7 +2191,7 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, plen = ntohs(h->ip6_plen); if (plen == 0) goto drop; - if ((int)(sizeof (struct ip6_hdr) + plen) > m->m_pkthdr.len) + if ((uint32_t)(sizeof (struct ip6_hdr) + plen) > pbuf->pb_packet_len) goto shortpkt; /* Enforce a minimum ttl, may cause endless packet loops */ @@ -1978,7 +2205,7 @@ fragment: goto drop; plen = ntohs(h->ip6_plen); - if (!pf_pull_hdr(m, off, &frag, sizeof (frag), NULL, NULL, AF_INET6)) + if (!pf_pull_hdr(pbuf, off, &frag, sizeof (frag), NULL, NULL, AF_INET6)) goto shortpkt; fragoff = ntohs(frag.ip6f_offlg & IP6F_OFF_MASK); pd->proto = frag.ip6f_nxt; @@ -1988,9 +2215,10 @@ fragment: goto badfrag; fr_max = fragoff + plen - (off - sizeof(struct ip6_hdr)); - DPFPRINTF(("0x%llx IPv6 frag plen %u mff %d off %u fragoff %u " - "fr_max %u\n", (uint64_t)VM_KERNEL_ADDRPERM(m), plen, mff, off, - fragoff, fr_max)); +// XXX SCW: mbuf-specific +// DPFPRINTF(("0x%llx IPv6 frag plen %u mff %d off %u fragoff %u " +// "fr_max %u\n", (uint64_t)VM_KERNEL_ADDRPERM(m), plen, mff, off, +// fragoff, fr_max)); if ((r->rule_flag & (PFRULE_FRAGCROP|PFRULE_FRAGDROP)) == 0) { /* Fully buffer all of the fragments */ @@ -2003,13 +2231,22 @@ fragment: if (pff != NULL && (pff->fr_flags & PFFRAG_SEENLAST) && fr_max > pff->fr_max) goto badfrag; + + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + REASON_SET(reason, PFRES_MEMORY); + return (PF_DROP); + } + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip6_hdr *); + /* Get an entry for the fragment queue */ frent = pool_get(&pf_frent_pl, PR_NOWAIT); if (frent == NULL) { REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } + pf_nfrents++; frent->fr_ip6 = h; frent->fr_m = m; @@ -2019,15 +2256,16 @@ fragment: /* Might return a completely reassembled mbuf, or NULL */ DPFPRINTF(("reass IPv6 frag %d @ %d-%d\n", ntohl(frag.ip6f_ident), fragoff, fr_max)); - *m0 = m = pf_reassemble6(m0, &pff, frent, mff); + m = pf_reassemble6(&m, &pff, frent, mff); if (m == NULL) return (PF_DROP); + + pbuf_init_mbuf(pbuf, m, ifp); + h = pbuf->pb_data; if (pff != NULL && (pff->fr_flags & PFFRAG_DROP)) goto drop; - - h = mtod(m, struct ip6_hdr *); } else if (dir == PF_IN || !(pd->pf_mtag->pftag_flags & PF_TAG_FRAGCACHE)) { /* non-buffering fragment cache (overlaps: see RFC 5722) */ @@ -2044,14 +2282,26 @@ fragment: goto badfrag; } - *m0 = m = pf_frag6cache(m0, h, &frag, &pff, off, mff, + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + goto no_mem; + } + + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip6_hdr *); + + m = pf_frag6cache(&m, h, &frag, &pff, off, mff, (r->rule_flag & PFRULE_FRAGDROP) ? 1 : 0, &nomem); if (m == NULL) { + // Note: pf_frag6cache() has already m_freem'd the mbuf if (nomem) goto no_mem; goto drop; } + pbuf_init_mbuf(pbuf, m, ifp); + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); + h = pbuf->pb_data; + if (dir == PF_IN) pd->pf_mtag->pftag_flags |= PF_TAG_FRAGCACHE; @@ -2084,14 +2334,14 @@ fragment: dropout: if (pff != NULL) pf_free_fragment(pff); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET6, dir, *reason, r, NULL, NULL, pd); + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET6, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); } #endif /* INET6 */ int -pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, +pf_normalize_tcp(int dir, struct pfi_kif *kif, pbuf_t *pbuf, int ipoff, int off, void *h, struct pf_pdesc *pd) { #pragma unused(ipoff, h) @@ -2134,7 +2384,7 @@ pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, &r->dst.xport, &dxport)) r = r->skip[PF_SKIP_DST_PORT].ptr; else if (r->os_fingerprint != PF_OSFP_ANY && - !pf_osfp_match(pf_osfp_fingerprint(pd, m, off, th), + !pf_osfp_match(pf_osfp_fingerprint(pd, pbuf, off, th), r->os_fingerprint)) r = TAILQ_NEXT(r, entries); else { @@ -2208,25 +2458,24 @@ pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, /* copy back packet headers if we sanitized */ /* Process options */ if (r->max_mss) { - int rv = pf_normalize_tcpopt(r, dir, kif, pd, m, th, off, + int rv = pf_normalize_tcpopt(r, dir, kif, pd, pbuf, th, off, &rewrite); if (rv == PF_DROP) return rv; - m = pd->mp; + pbuf = pd->mp; } if (rewrite) { - struct mbuf *mw = pf_lazy_makewritable(pd, m, - off + sizeof (*th)); - if (!mw) { + if (pf_lazy_makewritable(pd, pbuf, + off + sizeof (*th)) == NULL) { REASON_SET(&reason, PFRES_MEMORY); if (r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, r, 0, 0, pd); return PF_DROP; } - m_copyback(mw, off, sizeof (*th), th); + pbuf_copy_back(pbuf, off, sizeof (*th), th); } return (PF_PASS); @@ -2234,12 +2483,12 @@ pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, tcp_drop: REASON_SET(&reason, PFRES_NORM); if (rm != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, r, NULL, NULL, pd); + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, r, NULL, NULL, pd); return (PF_DROP); } int -pf_normalize_tcp_init(struct mbuf *m, int off, struct pf_pdesc *pd, +pf_normalize_tcp_init(pbuf_t *pbuf, int off, struct pf_pdesc *pd, struct tcphdr *th, struct pf_state_peer *src, struct pf_state_peer *dst) { #pragma unused(dst) @@ -2257,14 +2506,14 @@ pf_normalize_tcp_init(struct mbuf *m, int off, struct pf_pdesc *pd, switch (pd->af) { #if INET case AF_INET: { - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; src->scrub->pfss_ttl = h->ip_ttl; break; } #endif /* INET */ #if INET6 case AF_INET6: { - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; src->scrub->pfss_ttl = h->ip6_hlim; break; } @@ -2281,7 +2530,7 @@ pf_normalize_tcp_init(struct mbuf *m, int off, struct pf_pdesc *pd, if (th->th_off > (sizeof (struct tcphdr) >> 2) && src->scrub && - pf_pull_hdr(m, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { + pf_pull_hdr(pbuf, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { /* Diddle with TCP options */ int hlen; opt = hdr + sizeof (struct tcphdr); @@ -2334,12 +2583,12 @@ pf_normalize_tcp_cleanup(struct pf_state *state) } int -pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, +pf_normalize_tcp_stateful(pbuf_t *pbuf, int off, struct pf_pdesc *pd, u_short *reason, struct tcphdr *th, struct pf_state *state, struct pf_state_peer *src, struct pf_state_peer *dst, int *writeback) { struct timeval uptime; - u_int32_t tsval, tsecr; + u_int32_t tsval = 0, tsecr = 0; u_int tsval_from_last; u_int8_t hdr[60]; u_int8_t *opt; @@ -2357,7 +2606,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, #if INET case AF_INET: { if (src->scrub) { - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; if (h->ip_ttl > src->scrub->pfss_ttl) src->scrub->pfss_ttl = h->ip_ttl; h->ip_ttl = src->scrub->pfss_ttl; @@ -2368,7 +2617,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, #if INET6 case AF_INET6: { if (src->scrub) { - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; if (h->ip6_hlim > src->scrub->pfss_ttl) src->scrub->pfss_ttl = h->ip6_hlim; h->ip6_hlim = src->scrub->pfss_ttl; @@ -2381,7 +2630,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, if (th->th_off > (sizeof (struct tcphdr) >> 2) && ((src->scrub && (src->scrub->pfss_flags & PFSS_TIMESTAMP)) || (dst->scrub && (dst->scrub->pfss_flags & PFSS_TIMESTAMP))) && - pf_pull_hdr(m, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { + pf_pull_hdr(pbuf, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { /* Diddle with TCP options */ int hlen; opt = hdr + sizeof (struct tcphdr); @@ -2451,13 +2700,13 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, /* Copyback the options, caller copys back header */ int optoff = off + sizeof (*th); int optlen = (th->th_off << 2) - sizeof (*th); - m = pf_lazy_makewritable(pd, m, optoff + optlen); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, optoff + optlen) == + NULL) { REASON_SET(reason, PFRES_MEMORY); return PF_DROP; } *writeback = optoff + optlen; - m_copyback(m, optoff, optlen, hdr + sizeof (*th)); + pbuf_copy_back(pbuf, optoff, optlen, hdr + sizeof(*th)); } } @@ -2735,7 +2984,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, static int pf_normalize_tcpopt(struct pf_rule *r, int dir, struct pfi_kif *kif, - struct pf_pdesc *pd, struct mbuf *m, struct tcphdr *th, int off, + struct pf_pdesc *pd, pbuf_t *pbuf, struct tcphdr *th, int off, int *rewrptr) { #pragma unused(dir, kif) @@ -2750,7 +2999,7 @@ pf_normalize_tcpopt(struct pf_rule *r, int dir, struct pfi_kif *kif, thoff = th->th_off << 2; cnt = thoff - sizeof (struct tcphdr); - if (cnt > 0 && !pf_pull_hdr(m, off + sizeof (*th), opts, cnt, + if (cnt > 0 && !pf_pull_hdr(pbuf, off + sizeof (*th), opts, cnt, NULL, NULL, af)) return PF_DROP; @@ -2776,8 +3025,8 @@ pf_normalize_tcpopt(struct pf_rule *r, int dir, struct pfi_kif *kif, * Only do the TCP checksum fixup if delayed * checksum calculation will not be performed. */ - if (m->m_pkthdr.rcvif || - !(m->m_pkthdr.csum_flags & CSUM_TCP)) + if (pbuf->pb_ifp || + !(*pbuf->pb_csum_flags & CSUM_TCP)) th->th_sum = pf_cksum_fixup(th->th_sum, *mss, htons(r->max_mss), 0); *mss = htons(r->max_mss); @@ -2790,21 +3039,21 @@ pf_normalize_tcpopt(struct pf_rule *r, int dir, struct pfi_kif *kif, } if (rewrite) { - struct mbuf *mw; u_short reason; - mw = pf_lazy_makewritable(pd, pd->mp, - off + sizeof (*th) + thoff); - if (!mw) { + VERIFY(pbuf == pd->mp); + + if (pf_lazy_makewritable(pd, pd->mp, + off + sizeof (*th) + thoff) == NULL) { REASON_SET(&reason, PFRES_MEMORY); if (r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, r, 0, 0, pd); return PF_DROP; } *rewrptr = 1; - m_copyback(mw, off + sizeof (*th), thoff - sizeof (*th), opts); + pbuf_copy_back(pd->mp, off + sizeof (*th), thoff - sizeof (*th), opts); } return PF_PASS; diff --git a/bsd/net/pf_osfp.c b/bsd/net/pf_osfp.c index e04a94e0f..2dc1e241c 100644 --- a/bsd/net/pf_osfp.c +++ b/bsd/net/pf_osfp.c @@ -86,7 +86,7 @@ static void pf_osfp_insert(struct pf_osfp_list *, struct pf_os_fingerprint *); * Returns the list of possible OSes. */ struct pf_osfp_enlist * -pf_osfp_fingerprint(struct pf_pdesc *pd, struct mbuf *m, int off, +pf_osfp_fingerprint(struct pf_pdesc *pd, pbuf_t *pbuf, int off, const struct tcphdr *tcp) { struct ip *ip; @@ -99,13 +99,13 @@ pf_osfp_fingerprint(struct pf_pdesc *pd, struct mbuf *m, int off, return (NULL); if (pd->af == PF_INET) { - ip = mtod(m, struct ip *); + ip = pbuf->pb_data; ip6 = (struct ip6_hdr *)NULL; } else { ip = (struct ip *)NULL; - ip6 = mtod(m, struct ip6_hdr *); + ip6 = pbuf->pb_data; } - if (!pf_pull_hdr(m, off, hdr, tcp->th_off << 2, NULL, NULL, + if (!pf_pull_hdr(pbuf, off, hdr, tcp->th_off << 2, NULL, NULL, pd->af)) return (NULL); diff --git a/bsd/net/pf_pbuf.c b/bsd/net/pf_pbuf.c new file mode 100644 index 000000000..a5b69b226 --- /dev/null +++ b/bsd/net/pf_pbuf.c @@ -0,0 +1,410 @@ +/* + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +void +pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp) +{ + + VERIFY((m->m_flags & M_PKTHDR) != 0); + + pbuf->pb_type = PBUF_TYPE_MBUF; + pbuf->pb_mbuf = m; + pbuf->pb_ifp = ifp; + pbuf->pb_next = NULL; + pbuf_sync(pbuf); +} + +void +pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp) +{ + + pbuf->pb_type = PBUF_TYPE_MEMORY; + pbuf->pb_memory = *mp; + pbuf->pb_ifp = ifp; + pbuf->pb_next = NULL; + pbuf_sync(pbuf); +} + +void +pbuf_destroy(pbuf_t *pbuf) +{ + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + if (pbuf->pb_mbuf) { + m_freem(pbuf->pb_mbuf); + pbuf->pb_mbuf = NULL; + } + } else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) { + VERIFY(pbuf->pb_memory.pm_buffer != NULL); + (void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory, + PBUF_ACTION_DESTROY); + } else { + VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE); + } + + memset(pbuf, 0, sizeof(*pbuf)); +} + +void +pbuf_sync(pbuf_t *pbuf) +{ + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + struct mbuf *m = pbuf->pb_mbuf; + + VERIFY(m != NULL); + VERIFY(m->m_flags & M_PKTHDR); + + pbuf->pb_data = mtod(m, void *); + pbuf->pb_packet_len = m->m_pkthdr.len; + pbuf->pb_contig_len = m->m_len; + pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags; + pbuf->pb_csum_data = &m->m_pkthdr.csum_data; + pbuf->pb_proto = &m->m_pkthdr.pkt_proto; + pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc; + pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid; + pbuf->pb_flags = &m->m_pkthdr.pkt_flags; + pbuf->pb_pftag = m_pftag(m); + } else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) { + struct pbuf_memory *nm = &pbuf->pb_memory; + + VERIFY(nm->pm_buffer != NULL); + VERIFY(nm->pm_buffer_len != 0); + VERIFY(nm->pm_len != 0); + VERIFY(nm->pm_len <= nm->pm_buffer_len); + VERIFY(nm->pm_offset < nm->pm_len); + + pbuf->pb_data = &nm->pm_buffer[nm->pm_offset]; + pbuf->pb_packet_len = nm->pm_len; + pbuf->pb_contig_len = nm->pm_len; + pbuf->pb_csum_flags = &nm->pm_csum_flags; + pbuf->pb_csum_data = &nm->pm_csum_data; + pbuf->pb_proto = &nm->pm_proto; + pbuf->pb_flowsrc = &nm->pm_flowsrc; + pbuf->pb_flowid = &nm->pm_flowid; + pbuf->pb_flags = &nm->pm_flags; + pbuf->pb_pftag = &nm->pm_pftag; + } else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); +} + +struct mbuf * +pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr) +{ + struct mbuf *m = NULL; + + pbuf_sync(pbuf); + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + m = pbuf->pb_mbuf; + if (release_ptr) { + pbuf->pb_mbuf = NULL; + pbuf_destroy(pbuf); + } + } else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) { + if (pbuf->pb_packet_len > (u_int)MHLEN) { + if (pbuf->pb_packet_len > (u_int)MCLBYTES) { + printf("%s: packet too big for cluster (%u)\n", + __func__, pbuf->pb_packet_len); + return (NULL); + } + m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR); + } else { + m = m_gethdr(M_DONTWAIT, MT_DATA); + } + if (m == NULL) + return (NULL); + + m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data); + m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags; + m->m_pkthdr.csum_data = *pbuf->pb_csum_data; + m->m_pkthdr.pkt_proto = *pbuf->pb_proto; + m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc; + m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid; + m->m_pkthdr.pkt_flags = *pbuf->pb_flags; + + if (pbuf->pb_pftag != NULL) { + struct pf_mtag *pftag = m_pftag(m); + + if (pftag != NULL) + *pftag = *pbuf->pb_pftag; + } + + if (release_ptr) + pbuf_destroy(pbuf); + } + + return (m); +} + +struct mbuf * +pbuf_clone_to_mbuf(pbuf_t *pbuf) +{ + struct mbuf *m = NULL; + + pbuf_sync(pbuf); + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL); + else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) + m = pbuf_to_mbuf(pbuf, FALSE); + else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); + + return (m); +} + +void * +pbuf_ensure_writable(pbuf_t *pbuf, size_t len) +{ + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + struct mbuf *m = pbuf->pb_mbuf; + + if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT)) + return (NULL); + + if (pbuf->pb_mbuf == NULL) { + pbuf_destroy(pbuf); + return (NULL); + } + + if (m != pbuf->pb_mbuf) + pbuf_sync(pbuf); + + } else + if (pbuf->pb_type != PBUF_TYPE_MEMORY) + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); + + return (pbuf->pb_data); +} + +void * +pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen) +{ + void *rv = NULL; + + VERIFY(off >= 0); + VERIFY((u_int)off <= pbuf->pb_packet_len); + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + struct mbuf *m, *n; + + VERIFY(pbuf->pb_mbuf != NULL); + + m = pbuf->pb_mbuf; + + if (off > 0) { + /* Split the mbuf chain at the specified boundary */ + if ((n = m_split(m, off, M_DONTWAIT)) == NULL) + return (NULL); + } else { + n = m; + } + + /* Trim old length */ + m_adj(n, olen); + + /* Prepend new length */ + if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL) + return (NULL); + + rv = mtod(n, void *); + + if (off > 0) { + /* Merge the two chains */ + int mlen; + + mlen = n->m_pkthdr.len; + m_cat(m, n); + m->m_pkthdr.len += mlen; + } else { + /* The new mbuf becomes the packet header */ + pbuf->pb_mbuf = n; + } + + pbuf_sync(pbuf); + } else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) { + struct pbuf_memory *nm = &pbuf->pb_memory; + u_int true_offset, move_len; + int delta_len; + uint8_t *psrc, *pdst; + + delta_len = nlen - olen; + VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <= + nm->pm_buffer_len); + + true_offset = (u_int)off + nm->pm_offset; + rv = &nm->pm_buffer[true_offset]; + psrc = &nm->pm_buffer[true_offset + (u_int)olen]; + pdst = &nm->pm_buffer[true_offset + (u_int)nlen]; + move_len = pbuf->pb_packet_len - ((u_int)off + olen); + memmove(pdst, psrc, move_len); + + nm->pm_len += delta_len; + + VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len); + + pbuf_sync(pbuf); + } else + panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type); + + return (rv); +} + +void * +pbuf_contig_segment(pbuf_t *pbuf, int off, int len) +{ + void *rv = NULL; + + VERIFY(off >= 0); + VERIFY(len >= 0); + VERIFY((u_int)(off + len) < pbuf->pb_packet_len); + + /* + * Note: If this fails, then the pbuf is destroyed. This is a + * side-effect of m_pulldown(). + * + * PF expects this behaviour so it's not a real problem. + */ + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + struct mbuf *n; + int moff; + + n = m_pulldown(pbuf->pb_mbuf, off, len, &moff); + if (n == NULL) { + /* mbuf is freed by m_pulldown() in this case */ + pbuf->pb_mbuf = NULL; + pbuf_destroy(pbuf); + return (NULL); + } + + pbuf_sync(pbuf); + + rv = (void *)(mtod(n, uint8_t *) + moff); + } else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) { + /* + * This always succeeds since memory pbufs are fully contig. + */ + rv = (void *)(uintptr_t)(((uint8_t *)pbuf->pb_data)[off]); + } else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); + + return (rv); +} + +void +pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *src) +{ + + VERIFY(off >= 0); + VERIFY(len >= 0); + VERIFY((u_int)(off + len) <= pbuf->pb_packet_len); + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + m_copyback(pbuf->pb_mbuf, off, len, src); + else + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + if (len) + memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len); + } else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); +} + +void +pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *dst) +{ + + VERIFY(off >= 0); + VERIFY(len >= 0); + VERIFY((u_int)(off + len) <= pbuf->pb_packet_len); + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + m_copydata(pbuf->pb_mbuf, off, len, dst); + else + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + if (len) + memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len); + } else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); +} + +uint16_t +pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len) +{ + uint16_t sum = 0; + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len); + else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) + sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len); + else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); + + return (sum); +} + +uint16_t +pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len) +{ + uint16_t sum = 0; + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len); + else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) + sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len); + else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); + + return (sum); +} + +mbuf_svc_class_t +pbuf_get_service_class(const pbuf_t *pbuf) +{ + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + return m_get_service_class(pbuf->pb_mbuf); + + VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY); + + return (MBUF_SC_BE); +} diff --git a/bsd/net/pf_pbuf.h b/bsd/net/pf_pbuf.h new file mode 100644 index 000000000..55c7f0aa8 --- /dev/null +++ b/bsd/net/pf_pbuf.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#ifndef __PBUF_H__ +#define __PBUF_H__ + +#include + +enum pbuf_type { + PBUF_TYPE_ZOMBIE = 0, + PBUF_TYPE_MBUF, + PBUF_TYPE_MEMORY +}; + +enum pbuf_action { + PBUF_ACTION_DESTROY +}; + +#define PBUF_ACTION_RV_SUCCESS 0 +#define PBUF_ACTION_RV_FAILURE (-1) + +struct pbuf_memory { + uint8_t *pm_buffer; // Pointer to start of buffer + u_int pm_buffer_len; // Total length of buffer + u_int pm_offset; // Offset to start of payload + u_int pm_len; // Length of payload + uint32_t pm_csum_flags; + uint32_t pm_csum_data; + uint8_t pm_proto; + uint8_t pm_flowsrc; + uint32_t pm_flowid; + uint32_t pm_flags; + struct pf_mtag pm_pftag; + int (*pm_action)(struct pbuf_memory *, enum pbuf_action); + void *pm_action_cookie; +}; + +typedef struct pbuf { + enum pbuf_type pb_type; + union { + struct mbuf *pbu_mbuf; + struct pbuf_memory pbu_memory; + } pb_u; +#define pb_mbuf pb_u.pbu_mbuf +#define pb_memory pb_u.pbu_memory + + void *pb_data; + uint32_t pb_packet_len; + uint32_t pb_contig_len; + uint32_t *pb_csum_flags; + uint32_t *pb_csum_data; + uint8_t *pb_proto; + uint8_t *pb_flowsrc; + uint32_t *pb_flowid; + uint32_t *pb_flags; + struct pf_mtag *pb_pftag; + struct ifnet *pb_ifp; + struct pbuf *pb_next; +} pbuf_t; + +#define pbuf_is_valid(pb) (!((pb) == NULL || (pb)->pb_type == PBUF_TYPE_ZOMBIE)) + +void pbuf_init_mbuf(pbuf_t *, struct mbuf *, struct ifnet *); +void pbuf_init_memory(pbuf_t *, const struct pbuf_memory *, + struct ifnet *); +void pbuf_destroy(pbuf_t *); +void pbuf_sync(pbuf_t *); + +struct mbuf *pbuf_to_mbuf(pbuf_t *, boolean_t); +struct mbuf *pbuf_clone_to_mbuf(pbuf_t *); + +void * pbuf_ensure_contig(pbuf_t *, size_t); +void * pbuf_ensure_writable(pbuf_t *, size_t); + +void * pbuf_resize_segment(pbuf_t *, int off, int olen, int nlen); +void * pbuf_contig_segment(pbuf_t *, int off, int len); + +void pbuf_copy_data(pbuf_t *, int, int, void *); +void pbuf_copy_back(pbuf_t *, int, int, void *); + +uint16_t pbuf_inet_cksum(const pbuf_t *, uint32_t, uint32_t, uint32_t); +uint16_t pbuf_inet6_cksum(const pbuf_t *, uint32_t, uint32_t, uint32_t); + +mbuf_svc_class_t pbuf_get_service_class(const pbuf_t *); + +#endif /* __PBUF_H__ */ diff --git a/bsd/net/pf_ruleset.c b/bsd/net/pf_ruleset.c index f03f3297f..ff4c3f904 100644 --- a/bsd/net/pf_ruleset.c +++ b/bsd/net/pf_ruleset.c @@ -74,6 +74,7 @@ #endif /* KERNEL */ #include +#include #include #include #include @@ -245,7 +246,7 @@ pf_find_ruleset_with_owner(const char *path, const char *owner, int is_anchor, struct pf_ruleset * pf_find_or_create_ruleset(const char *path) { - char *p, *q, *r; + char *p, *q = NULL, *r; struct pf_ruleset *ruleset; struct pf_anchor *anchor = 0, *dup, *parent = NULL; @@ -329,6 +330,11 @@ pf_find_or_create_ruleset(const char *path) q = r + 1; else *q = 0; +#if DUMMYNET + if(strncmp("com.apple.nlc", anchor->name, + sizeof("com.apple.nlc")) == 0) + is_nlc_enabled_glb = TRUE; +#endif } rs_free(p); return (anchor ? &anchor->ruleset : 0); @@ -352,6 +358,16 @@ pf_remove_if_empty_ruleset(struct pf_ruleset *ruleset) ruleset->rules[i].inactive.open) return; RB_REMOVE(pf_anchor_global, &pf_anchors, ruleset->anchor); +#if DUMMYNET + if(strncmp("com.apple.nlc", ruleset->anchor->name, + sizeof("com.apple.nlc")) == 0) { + struct dummynet_event dn_event; + bzero(&dn_event, sizeof(dn_event)); + dn_event.dn_event_code = DUMMYNET_NLC_DISABLED; + dummynet_event_enqueue_nwk_wq_entry(&dn_event); + is_nlc_enabled_glb = FALSE; + } +#endif if ((parent = ruleset->anchor->parent) != NULL) RB_REMOVE(pf_anchor_node, &parent->children, ruleset->anchor); diff --git a/bsd/net/pf_table.c b/bsd/net/pf_table.c index 17ff0ab1a..5ccaf0426 100644 --- a/bsd/net/pf_table.c +++ b/bsd/net/pf_table.c @@ -793,7 +793,7 @@ pfr_lookup_addr(struct pfr_ktable *kt, struct pfr_addr *ad, int exact) struct radix_node_head *head; struct pfr_kentry *ke; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); bzero(&sa, sizeof (sa)); if (ad->pfra_af == AF_INET) { @@ -938,7 +938,7 @@ pfr_clstats_kentries(struct pfr_kentryworkq *workq, u_int64_t tzero, { struct pfr_kentry *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); SLIST_FOREACH(p, workq, pfrke_workq) { if (negchange) @@ -996,7 +996,7 @@ pfr_route_kentry(struct pfr_ktable *kt, struct pfr_kentry *ke) struct radix_node *rn; struct radix_node_head *head; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); bzero(ke->pfrke_node, sizeof (ke->pfrke_node)); if (ke->pfrke_af == AF_INET) @@ -1022,7 +1022,7 @@ pfr_unroute_kentry(struct pfr_ktable *kt, struct pfr_kentry *ke) struct radix_node *rn; struct radix_node_head *head; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (ke->pfrke_af == AF_INET) head = kt->pfrkt_ip4; @@ -1066,7 +1066,7 @@ pfr_walktree(struct radix_node *rn, void *arg) struct pfr_walktree *w = arg; int flags = w->pfrw_flags; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); switch (w->pfrw_op) { case PFRW_MARK: @@ -1150,7 +1150,7 @@ pfr_clr_tables(struct pfr_table *filter, int *ndel, int flags) struct pfr_ktable *p; int xdel = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY | PFR_FLAG_ALLRSETS); @@ -1187,7 +1187,7 @@ pfr_add_tables(user_addr_t tbl, int size, int *nadd, int flags) int i, rv, xadd = 0; u_int64_t tzero = pf_calendar_time_second(); - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY); SLIST_INIT(&addq); @@ -1265,7 +1265,7 @@ pfr_del_tables(user_addr_t tbl, int size, int *ndel, int flags) struct pfr_ktable *p, *q, key; int i, xdel = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY); SLIST_INIT(&workq); @@ -1340,7 +1340,7 @@ pfr_get_tstats(struct pfr_table *filter, user_addr_t tbl, int *size, int n, nn; u_int64_t tzero = pf_calendar_time_second(); - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); /* XXX PFR_FLAG_CLSTATS disabled */ ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_ALLRSETS); @@ -1384,7 +1384,7 @@ pfr_clr_tstats(user_addr_t tbl, int size, int *nzero, int flags) int i, xzero = 0; u_int64_t tzero = pf_calendar_time_second(); - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY | PFR_FLAG_ADDRSTOO); @@ -1417,7 +1417,7 @@ pfr_set_tflags(user_addr_t tbl, int size, int setflag, int clrflag, struct pfr_ktable *p, *q, key; int i, xchange = 0, xdel = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY); if ((setflag & ~PFR_TFLAG_USRMASK) || @@ -1470,7 +1470,7 @@ pfr_ina_begin(struct pfr_table *trs, u_int32_t *ticket, int *ndel, int flags) struct pf_ruleset *rs; int xdel = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_DUMMY); rs = pf_find_or_create_ruleset(trs->pfrt_anchor); @@ -1509,7 +1509,7 @@ pfr_ina_define(struct pfr_table *tbl, user_addr_t addr, int size, struct pf_ruleset *rs; int i, rv, xadd = 0, xaddr = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_DUMMY | PFR_FLAG_ADDRSTOO); if (size && !(flags & PFR_FLAG_ADDRSTOO)) @@ -1608,7 +1608,7 @@ pfr_ina_rollback(struct pfr_table *trs, u_int32_t ticket, int *ndel, int flags) struct pf_ruleset *rs; int xdel = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_DUMMY); rs = pf_find_ruleset(trs->pfrt_anchor); @@ -1643,7 +1643,7 @@ pfr_ina_commit(struct pfr_table *trs, u_int32_t ticket, int *nadd, int xadd = 0, xchange = 0; u_int64_t tzero = pf_calendar_time_second(); - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY); rs = pf_find_ruleset(trs->pfrt_anchor); @@ -1684,7 +1684,7 @@ pfr_commit_ktable(struct pfr_ktable *kt, u_int64_t tzero) struct pfr_ktable *shadow = kt->pfrkt_shadow; int nflags; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (shadow->pfrkt_cnt == NO_ADDRESSES) { if (!(kt->pfrkt_flags & PFR_TFLAG_ACTIVE)) @@ -1825,7 +1825,7 @@ pfr_insert_ktables(struct pfr_ktableworkq *workq) { struct pfr_ktable *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); SLIST_FOREACH(p, workq, pfrkt_workq) pfr_insert_ktable(p); @@ -1834,7 +1834,7 @@ pfr_insert_ktables(struct pfr_ktableworkq *workq) static void pfr_insert_ktable(struct pfr_ktable *kt) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); RB_INSERT(pfr_ktablehead, &pfr_ktables, kt); pfr_ktable_cnt++; @@ -1849,7 +1849,7 @@ pfr_setflags_ktables(struct pfr_ktableworkq *workq) { struct pfr_ktable *p, *q; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); for (p = SLIST_FIRST(workq); p; p = q) { q = SLIST_NEXT(p, pfrkt_workq); @@ -1862,7 +1862,7 @@ pfr_setflags_ktable(struct pfr_ktable *kt, int newf) { struct pfr_kentryworkq addrq; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (!(newf & PFR_TFLAG_REFERENCED) && !(newf & PFR_TFLAG_PERSIST)) @@ -1896,7 +1896,7 @@ pfr_clstats_ktables(struct pfr_ktableworkq *workq, u_int64_t tzero, int recurse) { struct pfr_ktable *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); SLIST_FOREACH(p, workq, pfrkt_workq) pfr_clstats_ktable(p, tzero, recurse); @@ -1907,7 +1907,7 @@ pfr_clstats_ktable(struct pfr_ktable *kt, u_int64_t tzero, int recurse) { struct pfr_kentryworkq addrq; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (recurse) { pfr_enqueue_addrs(kt, &addrq, NULL, 0); @@ -1925,7 +1925,7 @@ pfr_create_ktable(struct pfr_table *tbl, u_int64_t tzero, int attachruleset) struct pfr_ktable *kt; struct pf_ruleset *rs; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); kt = pool_get(&pfr_ktable_pl, PR_WAITOK); if (kt == NULL) @@ -1960,7 +1960,7 @@ pfr_destroy_ktables(struct pfr_ktableworkq *workq, int flushaddr) { struct pfr_ktable *p, *q; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); for (p = SLIST_FIRST(workq); p; p = q) { q = SLIST_NEXT(p, pfrkt_workq); @@ -1973,7 +1973,7 @@ pfr_destroy_ktable(struct pfr_ktable *kt, int flushaddr) { struct pfr_kentryworkq addrq; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (flushaddr) { pfr_enqueue_addrs(kt, &addrq, NULL, 0); @@ -2006,7 +2006,7 @@ pfr_ktable_compare(struct pfr_ktable *p, struct pfr_ktable *q) static struct pfr_ktable * pfr_lookup_table(struct pfr_table *tbl) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); /* struct pfr_ktable start like a struct pfr_table */ return (RB_FIND(pfr_ktablehead, &pfr_ktables, @@ -2019,7 +2019,7 @@ pfr_match_addr(struct pfr_ktable *kt, struct pf_addr *a, sa_family_t af) struct pfr_kentry *ke = NULL; int match; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (!(kt->pfrkt_flags & PFR_TFLAG_ACTIVE) && kt->pfrkt_root != NULL) kt = kt->pfrkt_root; @@ -2058,7 +2058,7 @@ pfr_update_stats(struct pfr_ktable *kt, struct pf_addr *a, sa_family_t af, { struct pfr_kentry *ke = NULL; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (!(kt->pfrkt_flags & PFR_TFLAG_ACTIVE) && kt->pfrkt_root != NULL) kt = kt->pfrkt_root; @@ -2105,7 +2105,7 @@ pfr_attach_table(struct pf_ruleset *rs, char *name) struct pfr_table tbl; struct pf_anchor *ac = rs->anchor; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); bzero(&tbl, sizeof (tbl)); strlcpy(tbl.pfrt_name, name, sizeof (tbl.pfrt_name)); @@ -2139,7 +2139,7 @@ pfr_attach_table(struct pf_ruleset *rs, char *name) void pfr_detach_table(struct pfr_ktable *kt) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (kt->pfrkt_refcnt[PFR_REFCNT_RULE] <= 0) printf("pfr_detach_table: refcount = %d.\n", @@ -2157,7 +2157,7 @@ pfr_pool_get(struct pfr_ktable *kt, int *pidx, struct pf_addr *counter, union sockaddr_union mask; int idx = -1, use_counter = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (af == AF_INET) addr = (struct pf_addr *)&pfr_sin.sin_addr; @@ -2247,7 +2247,7 @@ pfr_kentry_byidx(struct pfr_ktable *kt, int idx, int af) { struct pfr_walktree w; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); bzero(&w, sizeof (w)); w.pfrw_op = PFRW_POOL_GET; @@ -2276,7 +2276,7 @@ pfr_dynaddr_update(struct pfr_ktable *kt, struct pfi_dynaddr *dyn) { struct pfr_walktree w; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); bzero(&w, sizeof (w)); w.pfrw_op = PFRW_DYNADDR_UPDATE; diff --git a/bsd/net/pfkeyv2.h b/bsd/net/pfkeyv2.h index 97d6280fa..74ec44859 100644 --- a/bsd/net/pfkeyv2.h +++ b/bsd/net/pfkeyv2.h @@ -432,6 +432,7 @@ struct sadb_sastat { #define SADB_X_EALG_AESCBC 12 #define SADB_X_EALG_AES 12 #define SADB_X_EALG_AES_GCM 13 +#define SADB_X_EALG_CHACHA20POLY1305 14 /* private allocations should use 249-255 (RFC2407) */ #if 1 /*nonstandard */ @@ -470,6 +471,8 @@ struct sadb_sastat { #define SADB_X_EXT_PZERO 0x0200 /* zero padding for ESP */ #define SADB_X_EXT_PMASK 0x0300 /* mask for padding flag */ +#define SADB_X_EXT_IIV 0x0400 /* Implicit IV */ + #ifdef PRIVATE #define SADB_X_EXT_NATT_DETECTED_PEER 0x1000 #define SADB_X_EXT_ESP_KEEPALIVE 0x2000 diff --git a/bsd/net/pfvar.h b/bsd/net/pfvar.h index 11e771bf5..d7ea4c6d1 100644 --- a/bsd/net/pfvar.h +++ b/bsd/net/pfvar.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007-2015 Apple Inc. All rights reserved. + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -91,6 +91,8 @@ extern "C" { #include #include +#include + #if BYTE_ORDER == BIG_ENDIAN #define htobe64(x) (x) @@ -231,17 +233,17 @@ enum { PF_ADDR_ADDRMASK, PF_ADDR_NOROUTE, PF_ADDR_DYNIFTL, struct pf_addr { union { - struct in_addr v4; - struct in6_addr v6; - u_int8_t addr8[16]; - u_int16_t addr16[8]; - u_int32_t addr32[4]; + struct in_addr _v4addr; + struct in6_addr _v6addr; + u_int8_t _addr8[16]; + u_int16_t _addr16[8]; + u_int32_t _addr32[4]; } pfa; /* 128-bit address */ -#define v4 pfa.v4 -#define v6 pfa.v6 -#define addr8 pfa.addr8 -#define addr16 pfa.addr16 -#define addr32 pfa.addr32 +#define v4addr pfa._v4addr +#define v6addr pfa._v6addr +#define addr8 pfa._addr8 +#define addr16 pfa._addr16 +#define addr32 pfa._addr32 }; #define PF_TABLE_NAME_SIZE 32 @@ -1431,7 +1433,7 @@ struct pf_pdesc { struct pf_addr *dst; struct ether_header *eh; - struct mbuf *mp; + pbuf_t *mp; int lmw; /* lazy writable offset */ struct pf_mtag *pf_mtag; u_int16_t *ip_sum; @@ -2165,15 +2167,6 @@ TAILQ_HEAD(pf_poolqueue, pf_pool); extern struct pf_poolqueue pf_pools[2]; extern struct pf_palist pf_pabuf; extern u_int32_t ticket_pabuf; -#if PF_ALTQ -TAILQ_HEAD(pf_altqqueue, pf_altq); -extern struct pf_altqqueue pf_altqs[2]; -extern u_int32_t ticket_altqs_active; -extern u_int32_t ticket_altqs_inactive; -extern int altqs_inactive_open; -extern struct pf_altqqueue *pf_altqs_active; -extern struct pf_altqqueue *pf_altqs_inactive; -#endif /* PF_ALTQ */ extern struct pf_poolqueue *pf_pools_active; extern struct pf_poolqueue *pf_pools_inactive; @@ -2187,9 +2180,6 @@ __private_extern__ u_int32_t pf_calc_state_key_flowhash(struct pf_state_key *); extern struct pool pf_src_tree_pl, pf_rule_pl; extern struct pool pf_state_pl, pf_state_key_pl, pf_pooladdr_pl; extern struct pool pf_state_scrub_pl; -#if PF_ALTQ -extern struct pool pf_altq_pl; -#endif /* PF_ALTQ */ extern struct pool pf_app_state_pl; extern struct thread *pf_purge_thread; @@ -2219,25 +2209,33 @@ __private_extern__ void pf_addrcpy(struct pf_addr *, struct pf_addr *, __private_extern__ void pf_rm_rule(struct pf_rulequeue *, struct pf_rule *); struct ip_fw_args; + +extern boolean_t is_nlc_enabled_glb; +extern boolean_t pf_is_nlc_enabled(void); + #if INET -__private_extern__ int pf_test(int, struct ifnet *, struct mbuf **, +__private_extern__ int pf_test(int, struct ifnet *, pbuf_t **, + struct ether_header *, struct ip_fw_args *); +__private_extern__ int pf_test_mbuf(int, struct ifnet *, struct mbuf **, struct ether_header *, struct ip_fw_args *); #endif /* INET */ #if INET6 -__private_extern__ int pf_test6(int, struct ifnet *, struct mbuf **, +__private_extern__ int pf_test6(int, struct ifnet *, pbuf_t **, + struct ether_header *, struct ip_fw_args *); +__private_extern__ int pf_test6_mbuf(int, struct ifnet *, struct mbuf **, struct ether_header *, struct ip_fw_args *); __private_extern__ void pf_poolmask(struct pf_addr *, struct pf_addr *, struct pf_addr *, struct pf_addr *, u_int8_t); __private_extern__ void pf_addr_inc(struct pf_addr *, sa_family_t); #endif /* INET6 */ -__private_extern__ struct mbuf *pf_lazy_makewritable(struct pf_pdesc *, - struct mbuf *, int); -__private_extern__ void *pf_pull_hdr(struct mbuf *, int, void *, int, +__private_extern__ void *pf_lazy_makewritable(struct pf_pdesc *, + pbuf_t *, int); +__private_extern__ void *pf_pull_hdr(pbuf_t *, int, void *, int, u_short *, u_short *, sa_family_t); __private_extern__ void pf_change_a(void *, u_int16_t *, u_int32_t, u_int8_t); -__private_extern__ int pflog_packet(struct pfi_kif *, struct mbuf *, +__private_extern__ int pflog_packet(struct pfi_kif *, pbuf_t *, sa_family_t, u_int8_t, u_int8_t, struct pf_rule *, struct pf_rule *, struct pf_ruleset *, struct pf_pdesc *); __private_extern__ int pf_match_addr(u_int8_t, struct pf_addr *, @@ -2253,17 +2251,17 @@ __private_extern__ int pf_match_gid(u_int8_t, gid_t, gid_t, gid_t); __private_extern__ void pf_normalize_init(void); __private_extern__ int pf_normalize_isempty(void); -__private_extern__ int pf_normalize_ip(struct mbuf **, int, struct pfi_kif *, +__private_extern__ int pf_normalize_ip(pbuf_t *, int, struct pfi_kif *, u_short *, struct pf_pdesc *); -__private_extern__ int pf_normalize_ip6(struct mbuf **, int, struct pfi_kif *, +__private_extern__ int pf_normalize_ip6(pbuf_t *, int, struct pfi_kif *, u_short *, struct pf_pdesc *); -__private_extern__ int pf_normalize_tcp(int, struct pfi_kif *, struct mbuf *, +__private_extern__ int pf_normalize_tcp(int, struct pfi_kif *, pbuf_t *, int, int, void *, struct pf_pdesc *); __private_extern__ void pf_normalize_tcp_cleanup(struct pf_state *); -__private_extern__ int pf_normalize_tcp_init(struct mbuf *, int, +__private_extern__ int pf_normalize_tcp_init(pbuf_t *, int, struct pf_pdesc *, struct tcphdr *, struct pf_state_peer *, struct pf_state_peer *); -__private_extern__ int pf_normalize_tcp_stateful(struct mbuf *, int, +__private_extern__ int pf_normalize_tcp_stateful(pbuf_t *, int, struct pf_pdesc *, u_short *, struct tcphdr *, struct pf_state *, struct pf_state_peer *, struct pf_state_peer *, int *); __private_extern__ u_int64_t pf_state_expires(const struct pf_state *); @@ -2347,7 +2345,7 @@ __private_extern__ u_int16_t pf_tagname2tag(char *); __private_extern__ void pf_tag2tagname(u_int16_t, char *); __private_extern__ void pf_tag_ref(u_int16_t); __private_extern__ void pf_tag_unref(u_int16_t); -__private_extern__ int pf_tag_packet(struct mbuf *, struct pf_mtag *, +__private_extern__ int pf_tag_packet(pbuf_t *, struct pf_mtag *, int, unsigned int, struct pf_pdesc *); __private_extern__ void pf_step_into_anchor(int *, struct pf_ruleset **, int, struct pf_rule **, struct pf_rule **, int *); @@ -2384,10 +2382,6 @@ extern int16_t pf_nat64_configured; #define PF_IS_ENABLED (pf_is_enabled != 0) extern u_int32_t pf_hash_seed; -#if PF_ALTQ -extern u_int32_t altq_allowed; -#endif /* PF_ALTQ */ - /* these ruleset functions can be linked into userland programs (pfctl) */ __private_extern__ int pf_get_ruleset_number(u_int8_t); __private_extern__ void pf_init_ruleset(struct pf_ruleset *); @@ -2406,7 +2400,7 @@ __private_extern__ void pf_rs_initialize(void); __private_extern__ int pf_osfp_add(struct pf_osfp_ioctl *); __private_extern__ struct pf_osfp_enlist *pf_osfp_fingerprint(struct pf_pdesc *, - struct mbuf *, int, const struct tcphdr *); + pbuf_t *, int, const struct tcphdr *); __private_extern__ struct pf_osfp_enlist *pf_osfp_fingerprint_hdr( const struct ip *, const struct ip6_hdr *, const struct tcphdr *); __private_extern__ void pf_osfp_flush(void); @@ -2415,7 +2409,9 @@ __private_extern__ void pf_osfp_initialize(void); __private_extern__ int pf_osfp_match(struct pf_osfp_enlist *, pf_osfp_t); __private_extern__ struct pf_os_fingerprint *pf_osfp_validate(void); __private_extern__ struct pf_mtag *pf_find_mtag(struct mbuf *); +__private_extern__ struct pf_mtag *pf_find_mtag_pbuf(pbuf_t *); __private_extern__ struct pf_mtag *pf_get_mtag(struct mbuf *); +__private_extern__ struct pf_mtag *pf_get_mtag_pbuf(pbuf_t *); #else /* !KERNEL */ extern struct pf_anchor_global pf_anchors; extern struct pf_anchor pf_main_anchor; diff --git a/bsd/net/pktap.c b/bsd/net/pktap.c index 2b5c5e0ce..6608e9df3 100644 --- a/bsd/net/pktap.c +++ b/bsd/net/pktap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -76,7 +76,7 @@ struct pktap_softc { }; #ifndef PKTAP_DEBUG -#define PKTAP_DEBUG 1 +#define PKTAP_DEBUG 0 #endif /* PKTAP_DEBUG */ #define PKTAP_FILTER_OK 0 /* Packet passes filter checks */ @@ -88,8 +88,8 @@ SYSCTL_DECL(_net_link); SYSCTL_NODE(_net_link, IFT_PKTAP, pktap, CTLFLAG_RW |CTLFLAG_LOCKED, 0, "pktap virtual interface"); -static int pktap_total_tap_count = 0; -SYSCTL_INT(_net_link_pktap, OID_AUTO, total_tap_count, +uint32_t pktap_total_tap_count = 0; +SYSCTL_UINT(_net_link_pktap, OID_AUTO, total_tap_count, CTLFLAG_RD | CTLFLAG_LOCKED, &pktap_total_tap_count, 0, ""); static u_int64_t pktap_count_unknown_if_type = 0; @@ -208,7 +208,7 @@ pktap_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) { int error = 0; struct pktap_softc *pktap = NULL; - struct ifnet_init_params if_init; + struct ifnet_init_eparams if_init; PKTAP_LOG(PKTP_LOG_FUNC, "unit %u\n", unit); @@ -228,9 +228,15 @@ pktap_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) pktap->pktp_filters[0].filter_param = PKTAP_FILTER_PARAM_IF_TYPE; pktap->pktp_filters[0].filter_param_if_type = IFT_ETHER; +#if CONFIG_EMBEDDED + pktap->pktp_filters[1].filter_op = PKTAP_FILTER_OP_PASS; + pktap->pktp_filters[1].filter_param = PKTAP_FILTER_PARAM_IF_TYPE; + pktap->pktp_filters[1].filter_param_if_type = IFT_CELLULAR; +#else /* CONFIG_EMBEDDED */ pktap->pktp_filters[1].filter_op = PKTAP_FILTER_OP_PASS; pktap->pktp_filters[1].filter_param = PKTAP_FILTER_PARAM_IF_TYPE; pktap->pktp_filters[1].filter_param_if_type = IFT_IEEE1394; +#endif /* CONFIG_EMBEDDED */ #if (DEVELOPMENT || DEBUG) pktap->pktp_filters[2].filter_op = PKTAP_FILTER_OP_PASS; @@ -242,7 +248,10 @@ pktap_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) * We do not use a set_bpf_tap() function as we rather rely on the more * accurate callback passed to bpf_attach() */ - bzero(&if_init, sizeof(struct ifnet_init_params)); + bzero(&if_init, sizeof(if_init)); + if_init.ver = IFNET_INIT_CURRENT_VERSION; + if_init.len = sizeof (if_init); + if_init.flags = IFNET_INIT_LEGACY; if_init.name = ifc->ifc_name; if_init.unit = unit; if_init.type = IFT_PKTAP; @@ -255,7 +264,7 @@ pktap_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) if_init.ioctl = pktap_ioctl; if_init.detach = pktap_detach; - error = ifnet_allocate(&if_init, &pktap->pktp_ifp); + error = ifnet_allocate_extended(&if_init, &pktap->pktp_ifp); if (error != 0) { printf("%s: ifnet_allocate failed, error %d\n", __func__, error); @@ -835,7 +844,7 @@ pktap_fill_proc_info(struct pktap_header *hdr, protocol_family_t proto, errno_t error; size_t hlen; struct in_addr faddr, laddr; - u_short fport, lport; + u_short fport = 0, lport = 0; struct inpcbinfo *pcbinfo = NULL; int wildcard = 0; @@ -899,7 +908,7 @@ pktap_fill_proc_info(struct pktap_header *hdr, protocol_family_t proto, errno_t error; struct in6_addr *faddr; struct in6_addr *laddr; - u_short fport, lport; + u_short fport = 0, lport = 0; struct inpcbinfo *pcbinfo = NULL; int wildcard = 0; @@ -1082,15 +1091,22 @@ pktap_bpf_tap(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m, goto done; if (proto == AF_INET6 && (size_t) m_pktlen(m) - 4 < sizeof(struct ip6_hdr)) goto done; + /* - * Skip the protocol in the mbuf as it's in network order + * Handle two cases: + * - The old utun encapsulation with the protocol family in network order + * - A raw IPv4 or IPv6 packet */ - pre = 4; - data_adjust = 4; - hdr->pth_dlt = DLT_NULL; - hdr_buffer.proto = proto; - hdr_size = sizeof(hdr_buffer); - break; + uint8_t data = *(uint8_t *)mbuf_data(m); + if ((data >> 4) == 4 || (data >> 4) == 6) { + pre = 4; + } else { + /* + * Skip the protocol in the mbuf as it's in network order + */ + pre = 4; + data_adjust = 4; + } } hdr->pth_dlt = DLT_NULL; hdr_buffer.proto = proto; @@ -1109,8 +1125,8 @@ pktap_bpf_tap(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m, ifp->if_type, ifp->if_xname); pktap_count_unknown_if_type += 1; } else { - snprintf(hdr->pth_ifname, sizeof(hdr->pth_ifname), "%s", - ifp->if_xname); + strlcpy(hdr->pth_ifname, ifp->if_xname, + sizeof(hdr->pth_ifname)); hdr->pth_flags |= outgoing ? PTH_FLAG_DIR_OUT : PTH_FLAG_DIR_IN; hdr->pth_protocol_family = proto; hdr->pth_frame_pre_length = pre + pre_adjust; @@ -1146,13 +1162,15 @@ __private_extern__ void pktap_input(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m, char *frame_header) { - char *hdr = (char *)mbuf_data(m); - char *start = (char *)mbuf_datastart(m); + char *hdr; + char *start; /* Fast path */ if (pktap_total_tap_count == 0) return; + hdr = (char *)mbuf_data(m); + start = (char *)mbuf_datastart(m); /* Make sure the frame header is fully contained in the mbuf */ if (frame_header != NULL && frame_header >= start && frame_header <= hdr) { size_t o_len = m->m_len; @@ -1186,3 +1204,4 @@ pktap_output(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m, pktap_bpf_tap(ifp, proto, m, pre, post, 1); } + diff --git a/bsd/net/pktap.h b/bsd/net/pktap.h index ef5ec8c7c..74b0b5bd1 100644 --- a/bsd/net/pktap.h +++ b/bsd/net/pktap.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2014 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -137,9 +137,11 @@ struct pktap_header { #endif /* BSD_KERNEL_PRIVATE */ #define PTH_FLAG_TSTAMP 0x2000 /* Has time stamp */ #define PTH_FLAG_NEW_FLOW 0x4000 /* Packet from a new flow */ - +#define PTH_FLAG_MSFSW 0x8000 /* Multi stack flow switch */ #ifdef BSD_KERNEL_PRIVATE +extern uint32_t pktap_total_tap_count; + extern void pktap_init(void); extern void pktap_input(struct ifnet *, protocol_family_t, struct mbuf *, char *); extern void pktap_output(struct ifnet *, protocol_family_t, struct mbuf *, diff --git a/bsd/net/pktsched/pktsched.c b/bsd/net/pktsched/pktsched.c index 451aa1709..e757dc893 100644 --- a/bsd/net/pktsched/pktsched.c +++ b/bsd/net/pktsched/pktsched.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -46,21 +46,10 @@ #include #include #include -#if PKTSCHED_PRIQ -#include -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_FAIRQ -#include -#endif /* PKTSCHED_FAIRQ */ -#if PKTSCHED_CBQ -#include -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_HFSC -#include -#endif /* PKTSCHED_HFSC */ #include + u_int32_t machclk_freq = 0; u_int64_t machclk_per_sec = 0; u_int32_t pktsched_verbose; /* more noise if greater than 1 */ @@ -83,18 +72,6 @@ pktsched_init(void) tcq_init(); qfq_init(); -#if PKTSCHED_PRIQ - priq_init(); -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_FAIRQ - fairq_init(); -#endif /* PKTSCHED_FAIRQ */ -#if PKTSCHED_CBQ - cbq_init(); -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_HFSC - hfsc_init(); -#endif /* PKTSCHED_HFSC */ } static void @@ -129,10 +106,10 @@ pktsched_nsecs_to_abstime(u_int64_t nsecs) } int -pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags) +pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags, + classq_pkt_type_t ptype) { int error = 0; - u_int32_t qflags = sflags; u_int32_t rflags; IFCQ_LOCK_ASSERT_HELD(ifq); @@ -143,17 +120,6 @@ pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags) if (ifq->ifcq_type == scheduler) return (0); - qflags &= (PKTSCHEDF_QALG_RED | PKTSCHEDF_QALG_RIO | - PKTSCHEDF_QALG_BLUE | PKTSCHEDF_QALG_SFB); - - /* These are mutually exclusive */ - if (qflags != 0 && - qflags != PKTSCHEDF_QALG_RED && qflags != PKTSCHEDF_QALG_RIO && - qflags != PKTSCHEDF_QALG_BLUE && qflags != PKTSCHEDF_QALG_SFB) { - panic("%s: RED|RIO|BLUE|SFB mutually exclusive\n", __func__); - /* NOTREACHED */ - } - /* * Remember the flags that need to be restored upon success, as * they may be cleared when we tear down existing scheduler. @@ -173,21 +139,15 @@ pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags) } switch (scheduler) { -#if PKTSCHED_PRIQ - case PKTSCHEDT_PRIQ: - error = priq_setup_ifclassq(ifq, sflags); - break; -#endif /* PKTSCHED_PRIQ */ - case PKTSCHEDT_TCQ: - error = tcq_setup_ifclassq(ifq, sflags); + error = tcq_setup_ifclassq(ifq, sflags, ptype); break; case PKTSCHEDT_QFQ: - error = qfq_setup_ifclassq(ifq, sflags); + error = qfq_setup_ifclassq(ifq, sflags, ptype); break; case PKTSCHEDT_FQ_CODEL: - error = fq_if_setup_ifclassq(ifq, sflags); + error = fq_if_setup_ifclassq(ifq, sflags, ptype); break; default: error = ENXIO; @@ -216,12 +176,6 @@ pktsched_teardown(struct ifclassq *ifq) case PKTSCHEDT_NONE: break; -#if PKTSCHED_PRIQ - case PKTSCHEDT_PRIQ: - error = priq_teardown_ifclassq(ifq); - break; -#endif /* PKTSCHED_PRIQ */ - case PKTSCHEDT_TCQ: error = tcq_teardown_ifclassq(ifq); break; @@ -249,12 +203,6 @@ pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid, IFCQ_LOCK_ASSERT_HELD(ifq); switch (ifq->ifcq_type) { -#if PKTSCHED_PRIQ - case PKTSCHEDT_PRIQ: - error = priq_getqstats_ifclassq(ifq, qid, ifqs); - break; -#endif /* PKTSCHED_PRIQ */ - case PKTSCHEDT_TCQ: error = tcq_getqstats_ifclassq(ifq, qid, ifqs); break; @@ -273,3 +221,161 @@ pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid, return (error); } + +void +pktsched_pkt_encap(pktsched_pkt_t *pkt, classq_pkt_type_t ptype, void *pp) +{ + pkt->pktsched_ptype = ptype; + pkt->pktsched_pkt = pp; + + switch (ptype) { + case QP_MBUF: + pkt->pktsched_plen = + (uint32_t)m_pktlen((struct mbuf *)pkt->pktsched_pkt); + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } +} + +void +pktsched_free_pkt(pktsched_pkt_t *pkt) +{ + switch (pkt->pktsched_ptype) { + case QP_MBUF: + m_freem(pkt->pktsched_pkt); + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } + + pkt->pktsched_pkt = NULL; + pkt->pktsched_plen = 0; + pkt->pktsched_ptype = 0; +} + +uint32_t +pktsched_get_pkt_len(pktsched_pkt_t *pkt) +{ + return (pkt->pktsched_plen); +} + +mbuf_svc_class_t +pktsched_get_pkt_svc(pktsched_pkt_t *pkt) +{ + mbuf_svc_class_t svc = MBUF_SC_UNSPEC; + + switch (pkt->pktsched_ptype) { + case QP_MBUF: + svc = m_get_service_class((mbuf_t)pkt->pktsched_pkt); + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (svc); +} + +void +pktsched_get_pkt_vars(pktsched_pkt_t *pkt, uint32_t **flags, + uint64_t **timestamp, uint32_t *flowid, uint8_t *flowsrc, uint8_t *proto, + uint32_t *tcp_start_seq) +{ + switch (pkt->pktsched_ptype) { + case QP_MBUF: { + struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt; + struct pkthdr *pkth = &m->m_pkthdr; + + if (flags != NULL) + *flags = &pkth->pkt_flags; + if (timestamp != NULL) + *timestamp = &pkth->pkt_timestamp; + if (flowid != NULL) + *flowid = pkth->pkt_flowid; + if (flowsrc != NULL) + *flowsrc = pkth->pkt_flowsrc; + if (proto != NULL) + *proto = pkth->pkt_proto; + /* + * caller should use this value only if PKTF_START_SEQ + * is set in the mbuf packet flags + */ + if (tcp_start_seq != NULL) + *tcp_start_seq = pkth->tx_start_seq; + + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } +} + +struct flowadv_fcentry * +pktsched_alloc_fcentry(pktsched_pkt_t *pkt, struct ifnet *ifp, int how) +{ +#pragma unused(ifp) + struct flowadv_fcentry *fce = NULL; + + switch (pkt->pktsched_ptype) { + case QP_MBUF: { + struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt; + + fce = flowadv_alloc_entry(how); + if (fce == NULL) + break; + + _CASSERT(sizeof (m->m_pkthdr.pkt_flowid) == + sizeof (fce->fce_flowid)); + + fce->fce_flowsrc_type = m->m_pkthdr.pkt_flowsrc; + fce->fce_flowid = m->m_pkthdr.pkt_flowid; + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (fce); +} + +uint32_t * +pktsched_get_pkt_sfb_vars(pktsched_pkt_t *pkt, uint32_t **sfb_flags) +{ + uint32_t *hashp = NULL; + + switch (pkt->pktsched_ptype) { + case QP_MBUF: { + struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt; + struct pkthdr *pkth = &m->m_pkthdr; + + _CASSERT(sizeof (pkth->pkt_mpriv_hash) == sizeof (uint32_t)); + _CASSERT(sizeof (pkth->pkt_mpriv_flags) == sizeof (uint32_t)); + + *sfb_flags = &pkth->pkt_mpriv_flags; + hashp = &pkth->pkt_mpriv_hash; + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (hashp); +} diff --git a/bsd/net/pktsched/pktsched.h b/bsd/net/pktsched/pktsched.h index fb25dfcef..6c2a8eaae 100644 --- a/bsd/net/pktsched/pktsched.h +++ b/bsd/net/pktsched/pktsched.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -51,14 +51,26 @@ extern "C" { #include /* flags for pktsched_setup */ -#define PKTSCHEDF_QALG_RED 0x1 /* use RED */ -#define PKTSCHEDF_QALG_RIO 0x2 /* use RIO */ -#define PKTSCHEDF_QALG_BLUE 0x4 /* use BLUE */ -#define PKTSCHEDF_QALG_SFB 0x8 /* use SFB */ -#define PKTSCHEDF_QALG_ECN 0x10 /* enable ECN */ -#define PKTSCHEDF_QALG_FLOWCTL 0x20 /* enable flow control advisories */ -#define PKTSCHEDF_QALG_DELAYBASED 0x40 /* Delay based queueing */ -#define PKTSCHEDF_QALG_FQ_CODEL 0x80 /* Flow queueing with Codel */ +#define PKTSCHEDF_QALG_SFB 0x01 /* use SFB */ +#define PKTSCHEDF_QALG_ECN 0x02 /* enable ECN */ +#define PKTSCHEDF_QALG_FLOWCTL 0x04 /* enable flow control advisories */ +#define PKTSCHEDF_QALG_DELAYBASED 0x08 /* Delay based queueing */ +#define PKTSCHEDF_QALG_DRIVER_MANAGED 0x10 /* driver managed */ + +typedef struct _pktsched_pkt_ { + classq_pkt_type_t __ptype; + uint32_t __plen; + void *__pkt; +#define pktsched_ptype __ptype +#define pktsched_plen __plen +#define pktsched_pkt __pkt +} pktsched_pkt_t; + +#define _PKTSCHED_PKT_INIT(_p) do { \ + (_p)->pktsched_ptype = QP_INVALID; \ + (_p)->pktsched_plen = 0; \ + (_p)->pktsched_pkt = NULL; \ +} while (0) /* macro for timeout/untimeout */ /* use old-style timeout/untimeout */ @@ -144,12 +156,22 @@ SYSCTL_DECL(_net_pktsched); struct if_ifclassq_stats; extern void pktsched_init(void); -extern int pktsched_setup(struct ifclassq *, u_int32_t, u_int32_t); +extern int pktsched_setup(struct ifclassq *, u_int32_t, u_int32_t, + classq_pkt_type_t); extern int pktsched_teardown(struct ifclassq *); extern int pktsched_getqstats(struct ifclassq *, u_int32_t, struct if_ifclassq_stats *); extern u_int64_t pktsched_abs_to_nsecs(u_int64_t); extern u_int64_t pktsched_nsecs_to_abstime(u_int64_t); +extern void pktsched_free_pkt(pktsched_pkt_t *); +extern uint32_t pktsched_get_pkt_len(pktsched_pkt_t *); +extern void pktsched_get_pkt_vars(pktsched_pkt_t *, uint32_t **, uint64_t **, + uint32_t *, uint8_t *, uint8_t *, uint32_t *); +extern uint32_t *pktsched_get_pkt_sfb_vars(pktsched_pkt_t *, uint32_t **); +extern void pktsched_pkt_encap(pktsched_pkt_t *, classq_pkt_type_t, void *); +extern mbuf_svc_class_t pktsched_get_pkt_svc(pktsched_pkt_t *); +extern struct flowadv_fcentry *pktsched_alloc_fcentry(pktsched_pkt_t *, + struct ifnet *, int); #endif /* BSD_KERNEL_PRIVATE */ #ifdef __cplusplus diff --git a/bsd/net/pktsched/pktsched_cbq.c b/bsd/net/pktsched/pktsched_cbq.c deleted file mode 100644 index 41b1f8ede..000000000 --- a/bsd/net/pktsched/pktsched_cbq.c +++ /dev/null @@ -1,705 +0,0 @@ -/* - * Copyright (c) 2007-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_cbq.c,v 1.23 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_cbq.c,v 1.9 2000/12/14 08:12:45 thorpej Exp $ */ - -/* - * Copyright (c) Sun Microsystems, Inc. 1993-1998 All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the SMCC Technology - * Development Group at Sun Microsystems, Inc. - * - * 4. The name of the Sun Microsystems, Inc nor may not be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * SUN MICROSYSTEMS DOES NOT CLAIM MERCHANTABILITY OF THIS SOFTWARE OR THE - * SUITABILITY OF THIS SOFTWARE FOR ANY PARTICULAR PURPOSE. The software is - * provided "as is" without express or implied warranty of any kind. - * - * These notices must be retained in any copies of any part of this software. - */ - -#if PKTSCHED_CBQ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#include -#include - -/* - * Forward Declarations. - */ -#if 0 -static int cbq_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *cbq_dequeue_ifclassq(struct ifclassq *, cqdq_op_t); -static int cbq_request_ifclassq(struct ifclassq *, cqrq_t, void *); -#endif -static int cbq_class_destroy(cbq_state_t *, struct rm_class *); -static int cbq_destroy_locked(cbq_state_t *); -static struct rm_class *cbq_clh_to_clp(cbq_state_t *, u_int32_t); -static const char *cbq_style(cbq_state_t *); -static int cbq_clear_interface(cbq_state_t *); -static void cbqrestart(struct ifclassq *); - -#define CBQ_ZONE_MAX 32 /* maximum elements in zone */ -#define CBQ_ZONE_NAME "pktsched_cbq" /* zone name */ - -static unsigned int cbq_size; /* size of zone element */ -static struct zone *cbq_zone; /* zone for cbq */ - -void -cbq_init(void) -{ - _CASSERT(CBQCLF_RED == RMCF_RED); - _CASSERT(CBQCLF_ECN == RMCF_ECN); - _CASSERT(CBQCLF_RIO == RMCF_RIO); - _CASSERT(CBQCLF_FLOWVALVE == RMCF_FLOWVALVE); - _CASSERT(CBQCLF_CLEARDSCP == RMCF_CLEARDSCP); - _CASSERT(CBQCLF_WRR == RMCF_WRR); - _CASSERT(CBQCLF_EFFICIENT == RMCF_EFFICIENT); - _CASSERT(CBQCLF_BLUE == RMCF_BLUE); - _CASSERT(CBQCLF_SFB == RMCF_SFB); - _CASSERT(CBQCLF_FLOWCTL == RMCF_FLOWCTL); - _CASSERT(CBQCLF_LAZY == RMCF_LAZY); - - cbq_size = sizeof (cbq_state_t); - cbq_zone = zinit(cbq_size, CBQ_ZONE_MAX * cbq_size, 0, CBQ_ZONE_NAME); - if (cbq_zone == NULL) { - panic("%s: failed allocating %s", __func__, CBQ_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(cbq_zone, Z_EXPAND, TRUE); - zone_change(cbq_zone, Z_CALLERACCT, TRUE); - - rmclass_init(); -} - -cbq_state_t * -cbq_alloc(struct ifnet *ifp, int how, boolean_t altq) -{ - cbq_state_t *cbqp; - - /* allocate and initialize cbq_state_t */ - cbqp = (how == M_WAITOK) ? zalloc(cbq_zone) : zalloc_noblock(cbq_zone); - if (cbqp == NULL) - return (NULL); - - bzero(cbqp, cbq_size); - CALLOUT_INIT(&cbqp->cbq_callout); - cbqp->cbq_qlen = 0; - cbqp->ifnp.ifq_ = &ifp->if_snd; /* keep the ifclassq */ - if (altq) - cbqp->cbq_flags |= CBQSF_ALTQ; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler allocated\n", - if_name(ifp), cbq_style(cbqp)); - } - - return (cbqp); -} - -int -cbq_destroy(cbq_state_t *cbqp) -{ - struct ifclassq *ifq = cbqp->ifnp.ifq_; - int err; - - IFCQ_LOCK(ifq); - err = cbq_destroy_locked(cbqp); - IFCQ_UNLOCK(ifq); - - return (err); -} - -static int -cbq_destroy_locked(cbq_state_t *cbqp) -{ - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - (void) cbq_clear_interface(cbqp); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler destroyed\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - } - - if (cbqp->ifnp.default_) - cbq_class_destroy(cbqp, cbqp->ifnp.default_); - if (cbqp->ifnp.root_) - cbq_class_destroy(cbqp, cbqp->ifnp.root_); - - /* deallocate cbq_state_t */ - zfree(cbq_zone, cbqp); - - return (0); -} - -int -cbq_add_queue(cbq_state_t *cbqp, u_int32_t qlimit, u_int32_t priority, - u_int32_t minburst, u_int32_t maxburst, u_int32_t pktsize, - u_int32_t maxpktsize, u_int32_t ns_per_byte, u_int32_t maxidle, int minidle, - u_int32_t offtime, u_int32_t flags, u_int32_t parent_qid, u_int32_t qid, - struct rm_class **clp) -{ -#pragma unused(minburst, maxburst, maxpktsize) - struct rm_class *borrow, *parent; - struct rm_class *cl; - int i, error; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - /* Sanitize flags unless internally configured */ - if (cbqp->cbq_flags & CBQSF_ALTQ) - flags &= CBQCLF_USERFLAGS; - - /* - * find a free slot in the class table. if the slot matching - * the lower bits of qid is free, use this slot. otherwise, - * use the first free slot. - */ - i = qid % CBQ_MAX_CLASSES; - if (cbqp->cbq_class_tbl[i] != NULL) { - for (i = 0; i < CBQ_MAX_CLASSES; i++) - if (cbqp->cbq_class_tbl[i] == NULL) - break; - if (i == CBQ_MAX_CLASSES) - return (EINVAL); - } - - /* check parameters */ - if (priority >= CBQ_MAXPRI) - return (EINVAL); - - if (ns_per_byte == 0) { - log(LOG_ERR, "%s: %s invalid inverse data rate\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - - /* Get pointers to parent and borrow classes. */ - parent = cbq_clh_to_clp(cbqp, parent_qid); - if (flags & CBQCLF_BORROW) - borrow = parent; - else - borrow = NULL; - - /* - * A class must borrow from its parent or it can not - * borrow at all. Hence, borrow can be null. - */ - if (parent == NULL && (flags & CBQCLF_ROOTCLASS) == 0) { - log(LOG_ERR, "%s: %s no parent class!\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - - if ((borrow != parent) && (borrow != NULL)) { - log(LOG_ERR, "%s: %s borrow class != parent\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - - /* - * check parameters - */ - switch (flags & CBQCLF_CLASSMASK) { - case CBQCLF_ROOTCLASS: - if (parent != NULL) { - log(LOG_ERR, "%s: %s parent exists\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - if (cbqp->ifnp.root_) { - log(LOG_ERR, "%s: %s root class exists\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - break; - case CBQCLF_DEFCLASS: - if (cbqp->ifnp.default_) { - log(LOG_ERR, "%s: %s default class exists\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - break; - case 0: - break; - default: - /* more than two flags bits set */ - log(LOG_ERR, "%s: %s invalid class flags 0x%x\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp), - (flags & CBQCLF_CLASSMASK)); - return (EINVAL); - } - - /* - * create a class. if this is a root class, initialize the - * interface. - */ - if ((flags & CBQCLF_CLASSMASK) == CBQCLF_ROOTCLASS) { - error = rmc_init(cbqp->ifnp.ifq_, &cbqp->ifnp, ns_per_byte, - cbqrestart, qid, qlimit, RM_MAXQUEUED, maxidle, minidle, - offtime, flags); - if (error != 0) - return (error); - cl = cbqp->ifnp.root_; - } else { - cl = rmc_newclass(priority, &cbqp->ifnp, ns_per_byte, - rmc_delay_action, qid, qlimit, parent, borrow, maxidle, - minidle, offtime, pktsize, flags); - } - if (cl == NULL) - return (ENOMEM); - - /* return handle to user space. */ - cl->stats_.handle = qid; - cl->stats_.depth = cl->depth_; - - /* save the allocated class */ - cbqp->cbq_class_tbl[i] = cl; - - if ((flags & CBQCLF_CLASSMASK) == CBQCLF_DEFCLASS) - cbqp->ifnp.default_ = cl; - - if (clp != NULL) - *clp = cl; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s created qid=%d pri=%d qlimit=%d " - "flags=%b\n", if_name(CBQS_IFP(cbqp)), cbq_style(cbqp), - qid, priority, qlimit, flags, CBQCLF_BITS); - } - - return (0); -} - -int -cbq_remove_queue(cbq_state_t *cbqp, u_int32_t qid) -{ - struct rm_class *cl; - int i; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - if ((cl = cbq_clh_to_clp(cbqp, qid)) == NULL) - return (EINVAL); - - /* if we are a parent class, then return an error. */ - if (RMC_IS_A_PARENT_CLASS(cl)) - return (EINVAL); - - /* delete the class */ - rmc_delete_class(&cbqp->ifnp, cl); - - /* - * free the class handle - */ - for (i = 0; i < CBQ_MAX_CLASSES; i++) { - if (cbqp->cbq_class_tbl[i] == cl) { - cbqp->cbq_class_tbl[i] = NULL; - if (cl == cbqp->ifnp.root_) - cbqp->ifnp.root_ = NULL; - if (cl == cbqp->ifnp.default_) - cbqp->ifnp.default_ = NULL; - break; - } - } - return (0); -} - -/* - * int - * cbq_class_destroy(cbq_mod_state_t *, struct rm_class *) - This - * function destroys a given traffic class. Before destroying - * the class, all traffic for that class is released. - */ -static int -cbq_class_destroy(cbq_state_t *cbqp, struct rm_class *cl) -{ - int i; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s destroyed qid=%d pri=%d\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp), - cl->stats_.handle, cl->pri_); - } - - /* delete the class */ - rmc_delete_class(&cbqp->ifnp, cl); - - /* - * free the class handle - */ - for (i = 0; i < CBQ_MAX_CLASSES; i++) - if (cbqp->cbq_class_tbl[i] == cl) - cbqp->cbq_class_tbl[i] = NULL; - - if (cl == cbqp->ifnp.root_) - cbqp->ifnp.root_ = NULL; - if (cl == cbqp->ifnp.default_) - cbqp->ifnp.default_ = NULL; - - return (0); -} - -/* convert class handle to class pointer */ -static struct rm_class * -cbq_clh_to_clp(cbq_state_t *cbqp, u_int32_t chandle) -{ - int i; - struct rm_class *cl; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - /* - * first, try optimistically the slot matching the lower bits of - * the handle. if it fails, do the linear table search. - */ - i = chandle % CBQ_MAX_CLASSES; - if ((cl = cbqp->cbq_class_tbl[i]) != NULL && - cl->stats_.handle == chandle) - return (cl); - for (i = 0; i < CBQ_MAX_CLASSES; i++) - if ((cl = cbqp->cbq_class_tbl[i]) != NULL && - cl->stats_.handle == chandle) - return (cl); - return (NULL); -} - -static const char * -cbq_style(cbq_state_t *cbqp) -{ - return ((cbqp->cbq_flags & CBQSF_ALTQ) ? "ALTQ_CBQ" : "CBQ"); -} - -static int -cbq_clear_interface(cbq_state_t *cbqp) -{ - int again, i; - struct rm_class *cl; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - /* clear out the classes now */ - do { - again = 0; - for (i = 0; i < CBQ_MAX_CLASSES; i++) { - if ((cl = cbqp->cbq_class_tbl[i]) != NULL) { - if (RMC_IS_A_PARENT_CLASS(cl)) - again++; - else { - cbq_class_destroy(cbqp, cl); - cbqp->cbq_class_tbl[i] = NULL; - if (cl == cbqp->ifnp.root_) - cbqp->ifnp.root_ = NULL; - if (cl == cbqp->ifnp.default_) - cbqp->ifnp.default_ = NULL; - } - } - } - } while (again); - - return (0); -} - -/* copy the stats info in rm_class to class_states_t */ -int -cbq_get_class_stats(cbq_state_t *cbqp, u_int32_t qid, class_stats_t *statsp) -{ - struct rm_class *cl; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - if ((cl = cbq_clh_to_clp(cbqp, qid)) == NULL) - return (EINVAL); - - statsp->xmit_cnt = cl->stats_.xmit_cnt; - statsp->drop_cnt = cl->stats_.drop_cnt; - statsp->over = cl->stats_.over; - statsp->borrows = cl->stats_.borrows; - statsp->overactions = cl->stats_.overactions; - statsp->delays = cl->stats_.delays; - - statsp->depth = cl->depth_; - statsp->priority = cl->pri_; - statsp->maxidle = cl->maxidle_; - statsp->minidle = cl->minidle_; - statsp->offtime = cl->offtime_; - statsp->qmax = qlimit(&cl->q_); - statsp->ns_per_byte = cl->ns_per_byte_; - statsp->wrr_allot = cl->w_allotment_; - statsp->qcnt = qlen(&cl->q_); - statsp->avgidle = cl->avgidle_; - - statsp->qtype = qtype(&cl->q_); - statsp->qstate = qstate(&cl->q_); -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - red_getstats(cl->red_, &statsp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - rio_getstats(cl->rio_, &statsp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - blue_getstats(cl->blue_, &statsp->blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_) && cl->sfb_ != NULL) - sfb_getstats(cl->sfb_, &statsp->sfb); - - return (0); -} - -int -cbq_enqueue(cbq_state_t *cbqp, struct rm_class *cl, struct mbuf *m, - struct pf_mtag *t) -{ - struct ifclassq *ifq = cbqp->ifnp.ifq_; - int len, ret; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - log(LOG_ERR, "%s: packet for %s does not have pkthdr\n", - if_name(ifq->ifcq_ifp)); - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - - if (cl == NULL) { -#if PF_ALTQ - cl = cbq_clh_to_clp(cbqp, t->pftag_qid); -#else /* !PF_ALTQ */ - cl = cbq_clh_to_clp(cbqp, 0); -#endif /* !PF_ALTQ */ - if (cl == NULL) { - cl = cbqp->ifnp.default_; - if (cl == NULL) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - } - } - - len = m_pktlen(m); - - ret = rmc_queue_packet(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - /* packet has been freed in rmc_queue_packet */ - PKTCNTR_ADD(&cl->stats_.drop_cnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT REACHED */ - } - } - - /* successfully queued. */ - ++cbqp->cbq_qlen; - IFCQ_INC_LEN(ifq); - IFCQ_INC_BYTES(ifq, len); - - return (ret); -} - -struct mbuf * -cbq_dequeue(cbq_state_t *cbqp, cqdq_op_t op) -{ - struct ifclassq *ifq = cbqp->ifnp.ifq_; - struct mbuf *m; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - m = rmc_dequeue_next(&cbqp->ifnp, op); - - if (m && op == CLASSQDQ_REMOVE) { - --cbqp->cbq_qlen; /* decrement # of packets in cbq */ - IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, m_pktlen(m)); - IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m)); - - /* Update the class. */ - rmc_update_class_util(&cbqp->ifnp); - } - return (m); -} - -/* - * void - * cbqrestart(queue_t *) - Restart sending of data. - * called from rmc_restart via timeout after waking up - * a suspended class. - * Returns: NONE - */ - -static void -cbqrestart(struct ifclassq *ifq) -{ - u_int32_t qlen; - - IFCQ_LOCK(ifq); - qlen = IFCQ_LEN(ifq); - IFCQ_UNLOCK(ifq); - - if (qlen > 0) - ifnet_start(ifq->ifcq_ifp); -} - -void -cbq_purge(cbq_state_t *cbqp) -{ - struct rm_class *cl; - int i; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - for (i = 0; i < CBQ_MAX_CLASSES; i++) { - if ((cl = cbqp->cbq_class_tbl[i]) != NULL) { - if (!qempty(&cl->q_) && pktsched_verbose) { - log(LOG_DEBUG, "%s: %s purge qid=%d pri=%d " - "qlen=%d\n", if_name(CBQS_IFP(cbqp)), - cbq_style(cbqp), cl->stats_.handle, - cl->pri_, qlen(&cl->q_)); - } - rmc_dropall(cl); - } - } -} - -void -cbq_event(cbq_state_t *cbqp, cqev_t ev) -{ - struct rm_class *cl; - int i; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - for (i = 0; i < CBQ_MAX_CLASSES; i++) { - if ((cl = cbqp->cbq_class_tbl[i]) != NULL) { - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s update qid=%d pri=%d " - "event=%s\n", if_name(CBQS_IFP(cbqp)), - cbq_style(cbqp), cl->stats_.handle, - cl->pri_, ifclassq_ev2str(ev)); - } - rmc_updateq(cl, ev); - } - } -} - -int -cqb_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) -{ -#pragma unused(ifq, flags) - return (ENXIO); /* not yet */ -} - -int -cbq_teardown_ifclassq(struct ifclassq *ifq) -{ - cbq_state_t *cbqp = ifq->ifcq_disc; - int i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(cbqp != NULL && ifq->ifcq_type == PKTSCHEDT_CBQ); - - (void) cbq_destroy_locked(cbqp); - - ifq->ifcq_disc = NULL; - for (i = 0; i < IFCQ_SC_MAX; i++) { - ifq->ifcq_disc_slots[i].qid = 0; - ifq->ifcq_disc_slots[i].cl = NULL; - } - - return (ifclassq_detach(ifq)); -} - -int -cbq_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t slot, - struct if_ifclassq_stats *ifqs) -{ - cbq_state_t *cbqp = ifq->ifcq_disc; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(ifq->ifcq_type == PKTSCHEDT_CBQ); - - if (slot >= IFCQ_SC_MAX) - return (EINVAL); - - return (cbq_get_class_stats(cbqp, ifq->ifcq_disc_slots[slot].qid, - &ifqs->ifqs_cbq_stats)); -} -#endif /* PKTSCHED_CBQ */ diff --git a/bsd/net/pktsched/pktsched_cbq.h b/bsd/net/pktsched/pktsched_cbq.h index 15fe1b0b3..0553397d7 100644 --- a/bsd/net/pktsched/pktsched_cbq.h +++ b/bsd/net/pktsched/pktsched_cbq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -152,51 +152,6 @@ typedef struct cbq_classstats { classq_state_t qstate; } class_stats_t; -#ifdef BSD_KERNEL_PRIVATE -/* - * Define macros only good for kernel drivers and modules. - */ -#define CBQ_WATCHDOG (hz / 20) -#define CBQ_TIMEOUT 10 -#define CBQ_LS_TIMEOUT (20 * hz / 1000) - -#define CBQ_MAX_CLASSES 256 - -/* cbqstate flags */ -#define CBQSF_ALTQ 0x1 /* configured via PF/ALTQ */ - -/* - * Define State structures. - */ -typedef struct cbqstate { - int cbq_qlen; /* # of packets in cbq */ - u_int32_t cbq_flags; /* flags */ - struct rm_class *cbq_class_tbl[CBQ_MAX_CLASSES]; - - struct rm_ifdat ifnp; - struct callout cbq_callout; /* for timeouts */ -} cbq_state_t; - -#define CBQS_IFP(_cs) ((_cs)->ifnp.ifq_->ifcq_ifp) - -extern void cbq_init(void); -extern cbq_state_t *cbq_alloc(struct ifnet *, int, boolean_t); -extern int cbq_destroy(cbq_state_t *); -extern void cbq_purge(cbq_state_t *); -extern void cbq_event(cbq_state_t *, cqev_t); -extern int cbq_add_queue(cbq_state_t *, u_int32_t, u_int32_t, u_int32_t, - u_int32_t, u_int32_t, u_int32_t, u_int32_t, u_int32_t, int, u_int32_t, - u_int32_t, u_int32_t, u_int32_t, struct rm_class **); -extern int cbq_remove_queue(cbq_state_t *, u_int32_t); -extern int cbq_get_class_stats(cbq_state_t *, u_int32_t, class_stats_t *); -extern int cbq_enqueue(cbq_state_t *, struct rm_class *, struct mbuf *, - struct pf_mtag *); -extern struct mbuf *cbq_dequeue(cbq_state_t *, cqdq_op_t); -extern int cqb_setup_ifclassq(struct ifclassq *, u_int32_t); -extern int cbq_teardown_ifclassq(struct ifclassq *); -extern int cbq_getqstats_ifclassq(struct ifclassq *, u_int32_t, - struct if_ifclassq_stats *); -#endif /* BSD_KERNEL_PRIVATE */ #ifdef __cplusplus } #endif diff --git a/bsd/net/pktsched/pktsched_fairq.c b/bsd/net/pktsched/pktsched_fairq.c deleted file mode 100644 index 7e61e04c1..000000000 --- a/bsd/net/pktsched/pktsched_fairq.c +++ /dev/null @@ -1,1300 +0,0 @@ -/* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* - * Copyright (c) 2008 The DragonFly Project. All rights reserved. - * - * This code is derived from software contributed to The DragonFly Project - * by Matthew Dillon - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of The DragonFly Project nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific, prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $DragonFly: src/sys/net/altq/altq_fairq.c,v 1.2 2008/05/14 11:59:23 sephe Exp $ - */ -/* - * Matt: I gutted altq_priq.c and used it as a skeleton on which to build - * fairq. The fairq algorithm is completely different then priq, of course, - * but because I used priq's skeleton I believe I should include priq's - * copyright. - * - * Copyright (C) 2000-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * FAIRQ - take traffic classified by keep state (hashed into - * pf->pftag_flowhash) and bucketize it. Fairly extract - * the first packet from each bucket in a round-robin fashion. - * - * TODO - better overall qlimit support (right now it is per-bucket). - * - NOTE: red etc is per bucket, not overall. - * - better service curve support. - * - * EXAMPLE: - * - * altq on em0 fairq bandwidth 650Kb queue { std, bulk } - * queue std priority 3 bandwidth 200Kb \ - * fairq (buckets 64, default, hogs 1Kb) qlimit 50 - * queue bulk priority 2 bandwidth 100Kb \ - * fairq (buckets 64, hogs 1Kb) qlimit 50 - * - * NOTE: When the aggregate bandwidth is less than the link bandwidth - * any remaining bandwidth is dynamically assigned using the - * existing bandwidth specs as weightings. - * - * pass out on em0 from any to any keep state queue std - * pass out on em0 inet proto tcp ..... port ... keep state queue bulk - */ - -#if PKTSCHED_FAIRQ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#include -#include - -/* - * function prototypes - */ -#if 0 -static int fairq_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *fairq_dequeue_ifclassq(struct ifclassq *, cqdq_op_t); -static int fairq_request_ifclassq(struct ifclassq *, cqrq_t, void *); -#endif -static int fairq_clear_interface(struct fairq_if *); -static inline int fairq_addq(struct fairq_class *, struct mbuf *, - struct pf_mtag *); -static inline struct mbuf *fairq_getq(struct fairq_class *, u_int64_t); -static inline struct mbuf *fairq_pollq(struct fairq_class *, u_int64_t, int *); -static fairq_bucket_t *fairq_selectq(struct fairq_class *, int); -static void fairq_purgeq(struct fairq_if *, struct fairq_class *, u_int32_t, - u_int32_t *, u_int32_t *); -static void fairq_updateq(struct fairq_if *, struct fairq_class *, cqev_t); -static struct fairq_class *fairq_class_create(struct fairq_if *, int, u_int32_t, - u_int64_t, u_int32_t, int, u_int64_t, u_int64_t, u_int64_t, u_int64_t, - u_int32_t); -static int fairq_class_destroy(struct fairq_if *, struct fairq_class *); -static int fairq_destroy_locked(struct fairq_if *); -static inline struct fairq_class *fairq_clh_to_clp(struct fairq_if *, - u_int32_t); -static const char *fairq_style(struct fairq_if *); - -#define FAIRQ_ZONE_MAX 32 /* maximum elements in zone */ -#define FAIRQ_ZONE_NAME "pktsched_fairq" /* zone name */ - -static unsigned int fairq_size; /* size of zone element */ -static struct zone *fairq_zone; /* zone for fairq */ - -#define FAIRQ_CL_ZONE_MAX 32 /* maximum elements in zone */ -#define FAIRQ_CL_ZONE_NAME "pktsched_fairq_cl" /* zone name */ - -static unsigned int fairq_cl_size; /* size of zone element */ -static struct zone *fairq_cl_zone; /* zone for fairq */ - -void -fairq_init(void) -{ - fairq_size = sizeof (struct fairq_if); - fairq_zone = zinit(fairq_size, FAIRQ_ZONE_MAX * fairq_size, - 0, FAIRQ_ZONE_NAME); - if (fairq_zone == NULL) { - panic("%s: failed allocating %s", __func__, FAIRQ_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(fairq_zone, Z_EXPAND, TRUE); - zone_change(fairq_zone, Z_CALLERACCT, TRUE); - - fairq_cl_size = sizeof (struct fairq_class); - fairq_cl_zone = zinit(fairq_cl_size, FAIRQ_CL_ZONE_MAX * fairq_cl_size, - 0, FAIRQ_CL_ZONE_NAME); - if (fairq_cl_zone == NULL) { - panic("%s: failed allocating %s", __func__, FAIRQ_CL_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(fairq_cl_zone, Z_EXPAND, TRUE); - zone_change(fairq_cl_zone, Z_CALLERACCT, TRUE); -} - -struct fairq_if * -fairq_alloc(struct ifnet *ifp, int how, boolean_t altq) -{ - struct fairq_if *fif; - - fif = (how == M_WAITOK) ? - zalloc(fairq_zone) : zalloc_noblock(fairq_zone); - if (fif == NULL) - return (NULL); - - bzero(fif, fairq_size); - fif->fif_maxpri = -1; - fif->fif_ifq = &ifp->if_snd; - if (altq) - fif->fif_flags |= FAIRQIFF_ALTQ; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler allocated\n", - if_name(ifp), fairq_style(fif)); - } - - return (fif); -} - -int -fairq_destroy(struct fairq_if *fif) -{ - struct ifclassq *ifq = fif->fif_ifq; - int err; - - IFCQ_LOCK(ifq); - err = fairq_destroy_locked(fif); - IFCQ_UNLOCK(ifq); - - return (err); -} - -static int -fairq_destroy_locked(struct fairq_if *fif) -{ - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - (void) fairq_clear_interface(fif); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler destroyed\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif)); - } - - zfree(fairq_zone, fif); - - return (0); -} - -/* - * bring the interface back to the initial state by discarding - * all the filters and classes. - */ -static int -fairq_clear_interface(struct fairq_if *fif) -{ - struct fairq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - /* clear out the classes */ - for (pri = 0; pri <= fif->fif_maxpri; pri++) - if ((cl = fif->fif_classes[pri]) != NULL) - fairq_class_destroy(fif, cl); - - return (0); -} - -/* discard all the queued packets on the interface */ -void -fairq_purge(struct fairq_if *fif) -{ - struct fairq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - for (pri = 0; pri <= fif->fif_maxpri; pri++) { - if ((cl = fif->fif_classes[pri]) != NULL && cl->cl_head) - fairq_purgeq(fif, cl, 0, NULL, NULL); - } -#if !PF_ALTQ - /* - * This assertion is safe to be made only when PF_ALTQ is not - * configured; otherwise, IFCQ_LEN represents the sum of the - * packets managed by ifcq_disc and altq_disc instances, which - * is possible when transitioning between the two. - */ - VERIFY(IFCQ_LEN(fif->fif_ifq) == 0); -#endif /* !PF_ALTQ */ -} - -void -fairq_event(struct fairq_if *fif, cqev_t ev) -{ - struct fairq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - for (pri = 0; pri <= fif->fif_maxpri; pri++) - if ((cl = fif->fif_classes[pri]) != NULL) - fairq_updateq(fif, cl, ev); -} - -int -fairq_add_queue(struct fairq_if *fif, int priority, u_int32_t qlimit, - u_int64_t bandwidth, u_int32_t nbuckets, int flags, u_int64_t hogs_m1, - u_int64_t lssc_m1, u_int64_t lssc_d, u_int64_t lssc_m2, u_int32_t qid, - struct fairq_class **clp) -{ - struct fairq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - /* check parameters */ - if (priority >= FAIRQ_MAXPRI) - return (EINVAL); - if (bandwidth == 0 || (bandwidth / 8) == 0) - return (EINVAL); - if (fif->fif_classes[priority] != NULL) - return (EBUSY); - if (fairq_clh_to_clp(fif, qid) != NULL) - return (EBUSY); - - cl = fairq_class_create(fif, priority, qlimit, bandwidth, - nbuckets, flags, hogs_m1, lssc_m1, lssc_d, lssc_m2, qid); - if (cl == NULL) - return (ENOMEM); - - if (clp != NULL) - *clp = cl; - - return (0); -} - -static struct fairq_class * -fairq_class_create(struct fairq_if *fif, int pri, u_int32_t qlimit, - u_int64_t bandwidth, u_int32_t nbuckets, int flags, u_int64_t hogs_m1, - u_int64_t lssc_m1, u_int64_t lssc_d, u_int64_t lssc_m2, u_int32_t qid) -{ -#pragma unused(lssc_d, lssc_m2) - struct ifnet *ifp; - struct ifclassq *ifq; - struct fairq_class *cl; - u_int32_t i; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - /* Sanitize flags unless internally configured */ - if (fif->fif_flags & FAIRQIFF_ALTQ) - flags &= FARF_USERFLAGS; - -#if !CLASSQ_RED - if (flags & FARF_RED) { - log(LOG_ERR, "%s: %s RED not available!\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif)); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & FARF_RIO) { - log(LOG_ERR, "%s: %s RIO not available!\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif)); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & FARF_BLUE) { - log(LOG_ERR, "%s: %s BLUE not available!\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif)); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) && - (flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) != FARF_RED && - (flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) != FARF_RIO && - (flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) != FARF_BLUE && - (flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) != FARF_SFB) { - log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif)); - return (NULL); - } - - if (bandwidth == 0 || (bandwidth / 8) == 0) { - log(LOG_ERR, "%s: %s invalid data rate %llu\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif), bandwidth); - return (NULL); - } - - if (nbuckets == 0) - nbuckets = 256; - if (nbuckets > FAIRQ_MAX_BUCKETS) - nbuckets = FAIRQ_MAX_BUCKETS; - /* enforce power-of-2 size */ - while ((nbuckets ^ (nbuckets - 1)) != ((nbuckets << 1) - 1)) - ++nbuckets; - - ifq = fif->fif_ifq; - ifp = FAIRQIF_IFP(fif); - - if ((cl = fif->fif_classes[pri]) != NULL) { - /* modify the class instead of creating a new one */ - if (cl->cl_head) - fairq_purgeq(fif, cl, 0, NULL, NULL); -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - cl->cl_qtype = Q_DROPTAIL; - cl->cl_qstate = QS_RUNNING; - } else { - cl = zalloc(fairq_cl_zone); - if (cl == NULL) - goto err_ret; - bzero(cl, fairq_cl_size); - cl->cl_nbuckets = nbuckets; - cl->cl_nbucket_mask = nbuckets - 1; - - cl->cl_buckets = _MALLOC(sizeof (struct fairq_bucket) * - cl->cl_nbuckets, M_DEVBUF, M_WAITOK|M_ZERO); - if (cl->cl_buckets == NULL) - goto err_buckets; - cl->cl_head = NULL; - } - - fif->fif_classes[pri] = cl; - if (flags & FARF_DEFAULTCLASS) - fif->fif_default = cl; - if (qlimit == 0 || qlimit > IFCQ_MAXLEN(ifq)) { - qlimit = IFCQ_MAXLEN(ifq); - if (qlimit == 0) - qlimit = DEFAULT_QLIMIT; /* use default */ - } - cl->cl_qlimit = qlimit; - for (i = 0; i < cl->cl_nbuckets; ++i) { - _qinit(&cl->cl_buckets[i].queue, Q_DROPTAIL, qlimit); - } - cl->cl_bandwidth = bandwidth / 8; /* cvt to bytes per second */ - cl->cl_qtype = Q_DROPTAIL; - cl->cl_qstate = QS_RUNNING; - cl->cl_flags = flags; - cl->cl_pri = pri; - if (pri > fif->fif_maxpri) - fif->fif_maxpri = pri; - cl->cl_fif = fif; - cl->cl_handle = qid; - cl->cl_hogs_m1 = hogs_m1 / 8; - cl->cl_lssc_m1 = lssc_m1 / 8; /* NOT YET USED */ - cl->cl_bw_current = 0; - - if (flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) { -#if CLASSQ_RED || CLASSQ_RIO - u_int64_t ifbandwidth = ifnet_output_linkrate(ifp); - int pkttime; -#endif /* CLASSQ_RED || CLASSQ_RIO */ - - cl->cl_qflags = 0; - if (flags & FARF_ECN) { - if (flags & FARF_BLUE) - cl->cl_qflags |= BLUEF_ECN; - else if (flags & FARF_SFB) - cl->cl_qflags |= SFBF_ECN; - else if (flags & FARF_RED) - cl->cl_qflags |= REDF_ECN; - else if (flags & FARF_RIO) - cl->cl_qflags |= RIOF_ECN; - } - if (flags & FARF_FLOWCTL) { - if (flags & FARF_SFB) - cl->cl_qflags |= SFBF_FLOWCTL; - } - if (flags & FARF_CLEARDSCP) { - if (flags & FARF_RIO) - cl->cl_qflags |= RIOF_CLEARDSCP; - } -#if CLASSQ_RED || CLASSQ_RIO - /* - * XXX: RED & RIO should be watching link speed and MTU - * events and recompute pkttime accordingly. - */ - if (ifbandwidth < 8) - pkttime = 1000 * 1000 * 1000; /* 1 sec */ - else - pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 / - (ifbandwidth / 8); - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RIO - if (flags & FARF_RIO) { - cl->cl_rio = - rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime); - if (cl->cl_rio != NULL) - cl->cl_qtype = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (flags & FARF_RED) { - cl->cl_red = red_alloc(ifp, 0, 0, - cl->cl_qlimit * 10/100, - cl->cl_qlimit * 30/100, - cl->cl_qflags, pkttime); - if (cl->cl_red != NULL) - cl->cl_qtype = Q_RED; - } -#endif /* CLASSQ_RED */ -#endif /* CLASSQ_RED || CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & FARF_BLUE) { - cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags); - if (cl->cl_blue != NULL) - cl->cl_qtype = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & FARF_SFB) { - if (!(cl->cl_flags & FARF_LAZY)) - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - cl->cl_qlimit, cl->cl_qflags); - if (cl->cl_sfb != NULL || (cl->cl_flags & FARF_LAZY)) - cl->cl_qtype = Q_SFB; - } - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s created qid=%d pri=%d qlimit=%d " - "flags=%b\n", if_name(ifp), fairq_style(fif), - cl->cl_handle, cl->cl_pri, cl->cl_qlimit, flags, FARF_BITS); - } - - return (cl); - -err_buckets: - if (cl->cl_buckets != NULL) - _FREE(cl->cl_buckets, M_DEVBUF); -err_ret: - if (cl != NULL) { - if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - cl->cl_qtype = Q_DROPTAIL; - cl->cl_qstate = QS_RUNNING; - } - zfree(fairq_cl_zone, cl); - } - return (NULL); -} - -int -fairq_remove_queue(struct fairq_if *fif, u_int32_t qid) -{ - struct fairq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - if ((cl = fairq_clh_to_clp(fif, qid)) == NULL) - return (EINVAL); - - return (fairq_class_destroy(fif, cl)); -} - -static int -fairq_class_destroy(struct fairq_if *fif, struct fairq_class *cl) -{ - struct ifclassq *ifq = fif->fif_ifq; - int pri; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (cl->cl_head) - fairq_purgeq(fif, cl, 0, NULL, NULL); - - fif->fif_classes[cl->cl_pri] = NULL; - if (fif->fif_poll_cache == cl) - fif->fif_poll_cache = NULL; - if (fif->fif_maxpri == cl->cl_pri) { - for (pri = cl->cl_pri; pri >= 0; pri--) - if (fif->fif_classes[pri] != NULL) { - fif->fif_maxpri = pri; - break; - } - if (pri < 0) - fif->fif_maxpri = -1; - } - - if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - cl->cl_qtype = Q_DROPTAIL; - cl->cl_qstate = QS_RUNNING; - } - - if (fif->fif_default == cl) - fif->fif_default = NULL; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s destroyed qid=%d pri=%d\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif), - cl->cl_handle, cl->cl_pri); - } - - _FREE(cl->cl_buckets, M_DEVBUF); - cl->cl_head = NULL; /* sanity */ - cl->cl_polled = NULL; /* sanity */ - cl->cl_buckets = NULL; /* sanity */ - - zfree(fairq_cl_zone, cl); - - return (0); -} - -int -fairq_enqueue(struct fairq_if *fif, struct fairq_class *cl, struct mbuf *m, - struct pf_mtag *t) -{ - struct ifclassq *ifq = fif->fif_ifq; - int len, ret; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(cl == NULL || cl->cl_fif == fif); - - if (cl == NULL) { -#if PF_ALTQ - cl = fairq_clh_to_clp(fif, t->pftag_qid); -#else /* !PF_ALTQ */ - cl = fairq_clh_to_clp(fif, 0); -#endif /* !PF_ALTQ */ - if (cl == NULL) { - cl = fif->fif_default; - if (cl == NULL) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - } - } - - cl->cl_flags |= FARF_HAS_PACKETS; - len = m_pktlen(m); - - ret = fairq_addq(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - - /* packet has been freed in fairq_addq */ - PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT REACHED */ - } - } - IFCQ_INC_LEN(ifq); - IFCQ_INC_BYTES(ifq, len); - - /* successfully queued. */ - return (ret); -} - -/* - * note: CLASSQDQ_POLL returns the next packet without removing the packet - * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation. - * CLASSQDQ_REMOVE must return the same packet if called immediately - * after CLASSQDQ_POLL. - */ -struct mbuf * -fairq_dequeue(struct fairq_if *fif, cqdq_op_t op) -{ - struct ifclassq *ifq = fif->fif_ifq; - struct fairq_class *cl; - struct fairq_class *best_cl; - struct mbuf *best_m; - struct mbuf *m; - u_int64_t cur_time = read_machclk(); - u_int32_t best_scale; - u_int32_t scale; - int pri; - int hit_limit; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (IFCQ_IS_EMPTY(ifq)) { - /* no packet in the queue */ - return (NULL); - } - - if (fif->fif_poll_cache && op == CLASSQDQ_REMOVE) { - best_cl = fif->fif_poll_cache; - m = fairq_getq(best_cl, cur_time); - fif->fif_poll_cache = NULL; - if (m != NULL) { - IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, m_pktlen(m)); - IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m)); - PKTCNTR_ADD(&best_cl->cl_xmitcnt, 1, m_pktlen(m)); - } - } else { - best_cl = NULL; - best_m = NULL; - best_scale = 0xFFFFFFFFU; - - for (pri = fif->fif_maxpri; pri >= 0; pri--) { - if ((cl = fif->fif_classes[pri]) == NULL) - continue; - if ((cl->cl_flags & FARF_HAS_PACKETS) == 0) - continue; - m = fairq_pollq(cl, cur_time, &hit_limit); - if (m == NULL) { - cl->cl_flags &= ~FARF_HAS_PACKETS; - continue; - } - - /* - * We can halt the search immediately if the queue - * did not hit its bandwidth limit. - */ - if (hit_limit == 0) { - best_cl = cl; - best_m = m; - break; - } - - /* - * Otherwise calculate the scale factor and select - * the queue with the lowest scale factor. This - * apportions any unused bandwidth weighted by - * the relative bandwidth specification. - */ - scale = cl->cl_bw_current * 100 / cl->cl_bandwidth; - if (scale < best_scale) { - best_cl = cl; - best_m = m; - best_scale = scale; - } - } - - if (op == CLASSQDQ_POLL) { - fif->fif_poll_cache = best_cl; - m = best_m; - } else if (best_cl != NULL) { - m = fairq_getq(best_cl, cur_time); - if (m != NULL) { - IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, m_pktlen(m)); - IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m)); - PKTCNTR_ADD(&best_cl->cl_xmitcnt, 1, - m_pktlen(m)); - } - } else { - m = NULL; - } - } - return (m); -} - -static inline int -fairq_addq(struct fairq_class *cl, struct mbuf *m, struct pf_mtag *t) -{ - struct ifclassq *ifq = cl->cl_fif->fif_ifq; - fairq_bucket_t *b; - u_int32_t hash = m->m_pkthdr.pkt_flowid; - u_int32_t hindex; - u_int64_t bw; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - /* - * If the packet doesn't have any keep state put it on the end of - * our queue. XXX this can result in out of order delivery. - */ - if (hash == 0) { - if (cl->cl_head) - b = cl->cl_head->prev; - else - b = &cl->cl_buckets[0]; - } else { - hindex = (hash & cl->cl_nbucket_mask); - b = &cl->cl_buckets[hindex]; - } - - /* - * Add the bucket to the end of the circular list of active buckets. - * - * As a special case we add the bucket to the beginning of the list - * instead of the end if it was not previously on the list and if - * its traffic is less then the hog level. - */ - if (b->in_use == 0) { - b->in_use = 1; - if (cl->cl_head == NULL) { - cl->cl_head = b; - b->next = b; - b->prev = b; - } else { - b->next = cl->cl_head; - b->prev = cl->cl_head->prev; - b->prev->next = b; - b->next->prev = b; - - if (b->bw_delta && cl->cl_hogs_m1) { - bw = b->bw_bytes * machclk_freq / b->bw_delta; - if (bw < cl->cl_hogs_m1) - cl->cl_head = b; - } - } - } - -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - return (rio_addq(cl->cl_rio, &b->queue, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - return (red_addq(cl->cl_red, &b->queue, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - return (blue_addq(cl->cl_blue, &b->queue, m, t)); - else -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB) { - if (cl->cl_sfb == NULL) { - struct ifnet *ifp = FAIRQIF_IFP(cl->cl_fif); - - VERIFY(cl->cl_flags & FARF_LAZY); - IFCQ_CONVERT_LOCK(ifq); - - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - cl->cl_qlimit, cl->cl_qflags); - if (cl->cl_sfb == NULL) { - /* fall back to droptail */ - cl->cl_qtype = Q_DROPTAIL; - cl->cl_flags &= ~FARF_SFB; - cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL); - - log(LOG_ERR, "%s: %s SFB lazy allocation " - "failed for qid=%d pri=%d, falling back " - "to DROPTAIL\n", if_name(ifp), - fairq_style(cl->cl_fif), cl->cl_handle, - cl->cl_pri); - } - } - if (cl->cl_sfb != NULL) - return (sfb_addq(cl->cl_sfb, &b->queue, m, t)); - } else if (qlen(&b->queue) >= qlimit(&b->queue)) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - -#if PF_ECN - if (cl->cl_flags & FARF_CLEARDSCP) - write_dsfield(m, t, 0); -#endif /* PF_ECN */ - - _addq(&b->queue, m); - - return (0); -} - -static inline struct mbuf * -fairq_getq(struct fairq_class *cl, u_int64_t cur_time) -{ - fairq_bucket_t *b; - struct mbuf *m; - - IFCQ_LOCK_ASSERT_HELD(cl->cl_fif->fif_ifq); - - b = fairq_selectq(cl, 0); - if (b == NULL) - m = NULL; -#if CLASSQ_RIO - else if (cl->cl_qtype == Q_RIO) - m = rio_getq(cl->cl_rio, &b->queue); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - else if (cl->cl_qtype == Q_RED) - m = red_getq(cl->cl_red, &b->queue); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - else if (cl->cl_qtype == Q_BLUE) - m = blue_getq(cl->cl_blue, &b->queue); -#endif /* CLASSQ_BLUE */ - else if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - m = sfb_getq(cl->cl_sfb, &b->queue); - else - m = _getq(&b->queue); - - /* - * Calculate the BW change - */ - if (m != NULL) { - u_int64_t delta; - - /* - * Per-class bandwidth calculation - */ - delta = (cur_time - cl->cl_last_time); - if (delta > machclk_freq * 8) - delta = machclk_freq * 8; - cl->cl_bw_delta += delta; - cl->cl_bw_bytes += m->m_pkthdr.len; - cl->cl_last_time = cur_time; - if (cl->cl_bw_delta > machclk_freq) { - cl->cl_bw_delta -= cl->cl_bw_delta >> 2; - cl->cl_bw_bytes -= cl->cl_bw_bytes >> 2; - } - - /* - * Per-bucket bandwidth calculation - */ - delta = (cur_time - b->last_time); - if (delta > machclk_freq * 8) - delta = machclk_freq * 8; - b->bw_delta += delta; - b->bw_bytes += m->m_pkthdr.len; - b->last_time = cur_time; - if (b->bw_delta > machclk_freq) { - b->bw_delta -= b->bw_delta >> 2; - b->bw_bytes -= b->bw_bytes >> 2; - } - } - return (m); -} - -/* - * Figure out what the next packet would be if there were no limits. If - * this class hits its bandwidth limit *hit_limit is set to no-zero, otherwise - * it is set to 0. A non-NULL mbuf is returned either way. - */ -static inline struct mbuf * -fairq_pollq(struct fairq_class *cl, u_int64_t cur_time, int *hit_limit) -{ - fairq_bucket_t *b; - struct mbuf *m; - u_int64_t delta; - u_int64_t bw; - - IFCQ_LOCK_ASSERT_HELD(cl->cl_fif->fif_ifq); - - *hit_limit = 0; - b = fairq_selectq(cl, 1); - if (b == NULL) - return (NULL); - m = qhead(&b->queue); - - /* - * Did this packet exceed the class bandwidth? Calculate the - * bandwidth component of the packet. - * - * - Calculate bytes per second - */ - delta = cur_time - cl->cl_last_time; - if (delta > machclk_freq * 8) - delta = machclk_freq * 8; - cl->cl_bw_delta += delta; - cl->cl_last_time = cur_time; - if (cl->cl_bw_delta) { - bw = cl->cl_bw_bytes * machclk_freq / cl->cl_bw_delta; - - if (bw > cl->cl_bandwidth) - *hit_limit = 1; - cl->cl_bw_current = bw; -#if 0 - printf("BW %6lld relative to %6u %d queue 0x%llx\n", - bw, cl->cl_bandwidth, *hit_limit, - (uint64_t)VM_KERNEL_ADDRPERM(b)); -#endif - } - return (m); -} - -/* - * Locate the next queue we want to pull a packet out of. This code - * is also responsible for removing empty buckets from the circular list. - */ -static fairq_bucket_t * -fairq_selectq(struct fairq_class *cl, int ispoll) -{ - fairq_bucket_t *b; - u_int64_t bw; - - IFCQ_LOCK_ASSERT_HELD(cl->cl_fif->fif_ifq); - - if (ispoll == 0 && cl->cl_polled) { - b = cl->cl_polled; - cl->cl_polled = NULL; - return (b); - } - - while ((b = cl->cl_head) != NULL) { - /* - * Remove empty queues from consideration - */ - if (qempty(&b->queue)) { - b->in_use = 0; - cl->cl_head = b->next; - if (cl->cl_head == b) { - cl->cl_head = NULL; - } else { - b->next->prev = b->prev; - b->prev->next = b->next; - } - continue; - } - - /* - * Advance the round robin. Queues with bandwidths less - * then the hog bandwidth are allowed to burst. - */ - if (cl->cl_hogs_m1 == 0) { - cl->cl_head = b->next; - } else if (b->bw_delta) { - bw = b->bw_bytes * machclk_freq / b->bw_delta; - if (bw >= cl->cl_hogs_m1) { - cl->cl_head = b->next; - } - /* - * XXX TODO - - */ - } - - /* - * Return bucket b. - */ - break; - } - if (ispoll) - cl->cl_polled = b; - return (b); -} - -static void -fairq_purgeq(struct fairq_if *fif, struct fairq_class *cl, u_int32_t flow, - u_int32_t *packets, u_int32_t *bytes) -{ - struct ifclassq *ifq = fif->fif_ifq; - u_int32_t _cnt = 0, _len = 0; - fairq_bucket_t *b; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - /* become regular mutex before freeing mbufs */ - IFCQ_CONVERT_LOCK(ifq); - - while ((b = fairq_selectq(cl, 0)) != NULL) { - u_int32_t cnt, len, qlen; - - if ((qlen = qlen(&b->queue)) == 0) - continue; - -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - rio_purgeq(cl->cl_rio, &b->queue, flow, &cnt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - red_purgeq(cl->cl_red, &b->queue, flow, &cnt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - blue_purgeq(cl->cl_blue, &b->queue, flow, &cnt, &len); - else -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - sfb_purgeq(cl->cl_sfb, &b->queue, flow, &cnt, &len); - else - _flushq_flow(&b->queue, flow, &cnt, &len); - - if (cnt == 0) - continue; - - VERIFY(qlen(&b->queue) == (qlen - cnt)); - - PKTCNTR_ADD(&cl->cl_dropcnt, cnt, len); - IFCQ_DROP_ADD(ifq, cnt, len); - - VERIFY(((signed)IFCQ_LEN(ifq) - cnt) >= 0); - IFCQ_LEN(ifq) -= cnt; - - _cnt += cnt; - _len += len; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s purge qid=%d pri=%d " - "qlen=[%d,%d] cnt=%d len=%d flow=0x%x\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif), - cl->cl_handle, cl->cl_pri, qlen, qlen(&b->queue), - cnt, len, flow); - } - } - - if (packets != NULL) - *packets = _cnt; - if (bytes != NULL) - *bytes = _len; -} - -static void -fairq_updateq(struct fairq_if *fif, struct fairq_class *cl, cqev_t ev) -{ - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s update qid=%d pri=%d event=%s\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif), - cl->cl_handle, cl->cl_pri, ifclassq_ev2str(ev)); - } - -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - return (rio_updateq(cl->cl_rio, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - return (red_updateq(cl->cl_red, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - return (blue_updateq(cl->cl_blue, ev)); -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - return (sfb_updateq(cl->cl_sfb, ev)); -} - -int -fairq_get_class_stats(struct fairq_if *fif, u_int32_t qid, - struct fairq_classstats *sp) -{ - struct fairq_class *cl; - fairq_bucket_t *b; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - if ((cl = fairq_clh_to_clp(fif, qid)) == NULL) - return (EINVAL); - - sp->class_handle = cl->cl_handle; - sp->priority = cl->cl_pri; - sp->qlimit = cl->cl_qlimit; - sp->xmit_cnt = cl->cl_xmitcnt; - sp->drop_cnt = cl->cl_dropcnt; - sp->qtype = cl->cl_qtype; - sp->qstate = cl->cl_qstate; - sp->qlength = 0; - - if (cl->cl_head) { - b = cl->cl_head; - do { - sp->qlength += qlen(&b->queue); - b = b->next; - } while (b != cl->cl_head); - } - -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - red_getstats(cl->cl_red, &sp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - rio_getstats(cl->cl_rio, &sp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - blue_getstats(cl->cl_blue, &sp->blue); -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - sfb_getstats(cl->cl_sfb, &sp->sfb); - - return (0); -} - -/* convert a class handle to the corresponding class pointer */ -static inline struct fairq_class * -fairq_clh_to_clp(struct fairq_if *fif, u_int32_t chandle) -{ - struct fairq_class *cl; - int idx; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - for (idx = fif->fif_maxpri; idx >= 0; idx--) - if ((cl = fif->fif_classes[idx]) != NULL && - cl->cl_handle == chandle) - return (cl); - - return (NULL); -} - -static const char * -fairq_style(struct fairq_if *fif) -{ - return ((fif->fif_flags & FAIRQIFF_ALTQ) ? "ALTQ_FAIRQ" : "FAIRQ"); -} - -int -fairq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) -{ -#pragma unused(ifq, flags) - return (ENXIO); /* not yet */ -} - -int -fairq_teardown_ifclassq(struct ifclassq *ifq) -{ - struct fairq_if *fif = ifq->ifcq_disc; - int i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(fif != NULL && ifq->ifcq_type == PKTSCHEDT_FAIRQ); - - (void) fairq_destroy_locked(fif); - - ifq->ifcq_disc = NULL; - for (i = 0; i < IFCQ_SC_MAX; i++) { - ifq->ifcq_disc_slots[i].qid = 0; - ifq->ifcq_disc_slots[i].cl = NULL; - } - - return (ifclassq_detach(ifq)); -} - -int -fairq_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t slot, - struct if_ifclassq_stats *ifqs) -{ - struct fairq_if *fif = ifq->ifcq_disc; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(ifq->ifcq_type == PKTSCHEDT_FAIRQ); - - if (slot >= IFCQ_SC_MAX) - return (EINVAL); - - return (fairq_get_class_stats(fif, ifq->ifcq_disc_slots[slot].qid, - &ifqs->ifqs_fairq_stats)); -} -#endif /* PKTSCHED_FAIRQ */ diff --git a/bsd/net/pktsched/pktsched_fairq.h b/bsd/net/pktsched/pktsched_fairq.h index 910172950..bbe88c451 100644 --- a/bsd/net/pktsched/pktsched_fairq.h +++ b/bsd/net/pktsched/pktsched_fairq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -133,97 +133,6 @@ struct fairq_classstats { classq_state_t qstate; }; -#ifdef BSD_KERNEL_PRIVATE - -typedef struct fairq_bucket { - struct fairq_bucket *next; /* circular list */ - struct fairq_bucket *prev; /* circular list */ - class_queue_t queue; /* the actual queue */ - u_int64_t bw_bytes; /* statistics used to calculate bw */ - u_int64_t bw_delta; /* statistics used to calculate bw */ - u_int64_t last_time; - int in_use; -} fairq_bucket_t; - -struct fairq_class { - u_int32_t cl_handle; /* class handle */ - u_int32_t cl_nbuckets; /* (power of 2) */ - u_int32_t cl_nbucket_mask; /* bucket mask */ - u_int32_t cl_qflags; /* class queue flags */ - fairq_bucket_t *cl_buckets; - fairq_bucket_t *cl_head; /* head of circular bucket list */ - fairq_bucket_t *cl_polled; - union { - void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ - struct sfb *sfb; /* SFB state */ - } cl_qalg; - u_int64_t cl_hogs_m1; - u_int64_t cl_lssc_m1; - u_int64_t cl_bandwidth; - u_int64_t cl_bw_current; - u_int64_t cl_bw_bytes; - u_int64_t cl_bw_delta; - u_int64_t cl_last_time; - classq_type_t cl_qtype; /* rollup */ - classq_state_t cl_qstate; /* state */ - int cl_qlimit; - int cl_pri; /* priority */ - int cl_flags; /* class flags */ - struct fairq_if *cl_fif; /* back pointer to fif */ - - /* round robin index */ - - /* statistics */ - struct pktcntr cl_xmitcnt; /* transmitted packet counter */ - struct pktcntr cl_dropcnt; /* dropped packet counter */ -}; - -#define cl_red cl_qalg.red -#define cl_rio cl_qalg.rio -#define cl_blue cl_qalg.blue -#define cl_sfb cl_qalg.sfb - -/* fairq_if flags */ -#define FAIRQIFF_ALTQ 0x1 /* configured via PF/ALTQ */ - -/* - * fairq interface state - */ -struct fairq_if { - struct ifclassq *fif_ifq; /* backpointer to ifclassq */ - int fif_maxpri; /* max priority in use */ - u_int32_t fif_flags; /* flags */ - struct fairq_class *fif_poll_cache; /* cached poll */ - struct fairq_class *fif_default; /* default class */ - struct fairq_class *fif_classes[FAIRQ_MAXPRI]; /* classes */ -}; - -#define FAIRQIF_IFP(_fif) ((_fif)->fif_ifq->ifcq_ifp) - -struct if_ifclassq_stats; - -extern void fairq_init(void); -extern struct fairq_if *fairq_alloc(struct ifnet *, int, boolean_t); -extern int fairq_destroy(struct fairq_if *); -extern void fairq_purge(struct fairq_if *); -extern void fairq_event(struct fairq_if *, cqev_t); -extern int fairq_add_queue(struct fairq_if *, int, u_int32_t, u_int64_t, - u_int32_t, int, u_int64_t, u_int64_t, u_int64_t, u_int64_t, u_int32_t, - struct fairq_class **); -extern int fairq_remove_queue(struct fairq_if *, u_int32_t); -extern int fairq_get_class_stats(struct fairq_if *, u_int32_t, - struct fairq_classstats *); -extern int fairq_enqueue(struct fairq_if *, struct fairq_class *, - struct mbuf *, struct pf_mtag *); -extern struct mbuf *fairq_dequeue(struct fairq_if *, cqdq_op_t); -extern int fairq_setup_ifclassq(struct ifclassq *, u_int32_t); -extern int fairq_teardown_ifclassq(struct ifclassq *ifq); -extern int fairq_getqstats_ifclassq(struct ifclassq *, u_int32_t, - struct if_ifclassq_stats *); -#endif /* BSD_KERNEL_PRIVATE */ #ifdef __cplusplus } #endif diff --git a/bsd/net/pktsched/pktsched_fq_codel.c b/bsd/net/pktsched/pktsched_fq_codel.c index f7eef8378..c30dc2eb1 100644 --- a/bsd/net/pktsched/pktsched_fq_codel.c +++ b/bsd/net/pktsched/pktsched_fq_codel.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Apple Inc. All rights reserved. + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -35,21 +35,26 @@ #include #include - static size_t fq_if_size; static struct zone *fq_if_zone; -static fq_if_t *fq_if_alloc(struct ifnet *ifp, int how); +static fq_if_t *fq_if_alloc(struct ifnet *, classq_pkt_type_t); static void fq_if_destroy(fq_if_t *fqs); static void fq_if_classq_init(fq_if_t *fqs, u_int32_t priority, u_int32_t quantum, u_int32_t drr_max, u_int32_t svc_class); -static int fq_if_enqueue_classq(struct ifclassq *ifq, struct mbuf *m); -static struct mbuf *fq_if_dequeue_classq(struct ifclassq *ifq, cqdq_op_t); -static int fq_if_dequeue_classq_multi(struct ifclassq *, cqdq_op_t, - u_int32_t, u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, - u_int32_t *); +static int fq_if_enqueue_classq(struct ifclassq *ifq, void *p, + classq_pkt_type_t ptype, boolean_t *pdrop); +static void *fq_if_dequeue_classq(struct ifclassq *, classq_pkt_type_t *); +static int fq_if_dequeue_classq_multi(struct ifclassq *, u_int32_t, + u_int32_t, void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *); +static void *fq_if_dequeue_sc_classq(struct ifclassq *, mbuf_svc_class_t, + classq_pkt_type_t *); +static int fq_if_dequeue_sc_classq_multi(struct ifclassq *, + mbuf_svc_class_t, u_int32_t, u_int32_t, void **, + void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *); static void fq_if_dequeue(fq_if_t *, fq_if_classq_t *, u_int32_t, - u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *); + u_int32_t, void **, void **, u_int32_t *, u_int32_t *, + boolean_t drvmgmt, classq_pkt_type_t *); static int fq_if_request_classq(struct ifclassq *ifq, cqrq_t op, void *arg); void fq_if_stat_sc(fq_if_t *fqs, cqrq_stat_sc_t *stat); static void fq_if_purge(fq_if_t *); @@ -59,8 +64,6 @@ static void fq_if_empty_new_flow(fq_t *fq, fq_if_classq_t *fq_cl, bool add_to_old); static void fq_if_empty_old_flow(fq_if_t *fqs, fq_if_classq_t *fq_cl, fq_t *fq, bool remove_hash); -static void fq_if_destroy_flow(fq_if_t *fqs, fq_if_classq_t *fq_cl, - fq_t *fq); #define FQ_IF_ZONE_MAX 32 /* Maximum elements in zone */ #define FQ_IF_ZONE_NAME "pktsched_fq_if" /* zone for fq_if class */ @@ -72,6 +75,68 @@ static void fq_if_destroy_flow(fq_if_t *fqs, fq_if_classq_t *fq_cl, (STAILQ_EMPTY(&(_fcl_)->fcl_new_flows) && \ STAILQ_EMPTY(&(_fcl_)->fcl_old_flows)) +typedef void (* fq_if_append_pkt_t)(void *, void *); +typedef boolean_t (* fq_getq_flow_t)(fq_if_t *, fq_if_classq_t *, fq_t *, + u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *, + boolean_t *, u_int32_t); + +static void +fq_if_append_mbuf(void *pkt, void *next_pkt) +{ + ((mbuf_t)pkt)->m_nextpkt = (mbuf_t)next_pkt; +} + + + +static boolean_t +fq_getq_flow_mbuf(fq_if_t *fqs, fq_if_classq_t *fq_cl, fq_t *fq, + u_int32_t byte_limit, u_int32_t pkt_limit, void **top, void **last, + u_int32_t *byte_cnt, u_int32_t *pkt_cnt, boolean_t *qempty, + u_int32_t pflags) +{ + struct mbuf *m; + u_int32_t plen; + pktsched_pkt_t pkt; + boolean_t limit_reached = FALSE; + struct ifclassq *ifq = fqs->fqs_ifq; + struct ifnet *ifp = ifq->ifcq_ifp; + + while (fq->fq_deficit > 0 && limit_reached == FALSE && + !MBUFQ_EMPTY(&fq->fq_mbufq)) { + + _PKTSCHED_PKT_INIT(&pkt); + m = fq_getq_flow(fqs, fq, &pkt); + ASSERT(pkt.pktsched_ptype == QP_MBUF); + + plen = pktsched_get_pkt_len(&pkt); + fq->fq_deficit -= plen; + m->m_pkthdr.pkt_flags |= pflags; + + if (*top == NULL) { + *top = m; + } else { + ASSERT(*last != NULL); + ASSERT((*(struct mbuf **)last)->m_nextpkt == NULL); + (*(struct mbuf **)last)->m_nextpkt = m; + } + *last = m; + (*(mbuf_t *)last)->m_nextpkt = NULL; + fq_cl->fcl_stat.fcl_dequeue++; + fq_cl->fcl_stat.fcl_dequeue_bytes += plen; + *pkt_cnt += 1; + *byte_cnt += plen; + + ifclassq_set_packet_metadata(ifq, ifp, m, QP_MBUF); + + /* Check if the limit is reached */ + if (*pkt_cnt >= pkt_limit || *byte_cnt >= byte_limit) + limit_reached = TRUE; + } + + *qempty = MBUFQ_EMPTY(&fq->fq_mbufq); + return (limit_reached); +} + void fq_codel_scheduler_init(void) { @@ -91,23 +156,25 @@ fq_codel_scheduler_init(void) } fq_if_t * -fq_if_alloc(struct ifnet *ifp, int how) +fq_if_alloc(struct ifnet *ifp, classq_pkt_type_t ptype) { fq_if_t *fqs; - fqs = (how == M_WAITOK) ? zalloc(fq_if_zone) : - zalloc_noblock(fq_if_zone); + fqs = zalloc(fq_if_zone); if (fqs == NULL) return (NULL); bzero(fqs, fq_if_size); fqs->fqs_ifq = &ifp->if_snd; + fqs->fqs_ptype = ptype; /* Calculate target queue delay */ ifclassq_calc_target_qdelay(ifp, &fqs->fqs_target_qdelay); /* Calculate update interval */ ifclassq_calc_update_interval(&fqs->fqs_update_interval); - fqs->fqs_pkt_droplimit = FQ_IF_MAX_PKT_LIMIT; + + /* Configure packet drop limit across all queues */ + fqs->fqs_pkt_droplimit = IFCQ_PKT_DROP_LIMIT(&ifp->if_snd); STAILQ_INIT(&fqs->fqs_fclist); return (fqs); } @@ -115,17 +182,44 @@ fq_if_alloc(struct ifnet *ifp, int how) void fq_if_destroy(fq_if_t *fqs) { - IFCQ_LOCK_ASSERT_HELD(fqs->fqs_ifq); fq_if_purge(fqs); fqs->fqs_ifq = NULL; zfree(fq_if_zone, fqs); } static inline u_int32_t -fq_if_service_to_priority(mbuf_svc_class_t svc) +fq_if_service_to_priority(fq_if_t *fqs, mbuf_svc_class_t svc) { u_int32_t pri; + if (fqs->fqs_flags & FQS_DRIVER_MANAGED) { + switch (svc) { + case MBUF_SC_BK_SYS: + case MBUF_SC_BK: + pri = FQ_IF_BK_INDEX; + break; + case MBUF_SC_BE: + case MBUF_SC_RD: + case MBUF_SC_OAM: + pri = FQ_IF_BE_INDEX; + break; + case MBUF_SC_AV: + case MBUF_SC_RV: + case MBUF_SC_VI: + pri = FQ_IF_VI_INDEX; + break; + case MBUF_SC_VO: + case MBUF_SC_CTL: + pri = FQ_IF_VO_INDEX; + break; + default: + pri = FQ_IF_BE_INDEX; /* Use best effort by default */ + break; + } + return (pri); + } + + /* scheduler is not managed by the driver */ switch (svc) { case MBUF_SC_BK_SYS: pri = FQ_IF_BK_SYS_INDEX; @@ -183,24 +277,28 @@ fq_if_classq_init(fq_if_t *fqs, u_int32_t pri, u_int32_t quantum, } int -fq_if_enqueue_classq(struct ifclassq *ifq, struct mbuf *m) +fq_if_enqueue_classq(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype, + boolean_t *pdrop) { u_int32_t pri; fq_if_t *fqs; fq_if_classq_t *fq_cl; int ret, len; mbuf_svc_class_t svc; + pktsched_pkt_t pkt; IFCQ_LOCK_ASSERT_HELD(ifq); - if (!(m->m_flags & M_PKTHDR)) { + if ((ptype == QP_MBUF) && !(((mbuf_t)p)->m_flags & M_PKTHDR)) { IFCQ_CONVERT_LOCK(ifq); - m_freem(m); + m_freem((mbuf_t)p); + *pdrop = TRUE; return (ENOBUFS); } + pktsched_pkt_encap(&pkt, ptype, p); fqs = (fq_if_t *)ifq->ifcq_disc; - svc = mbuf_get_service_class(m); - pri = fq_if_service_to_priority(svc); + svc = pktsched_get_pkt_svc(&pkt); + pri = fq_if_service_to_priority(fqs, svc); VERIFY(pri >= 0 && pri < FQ_IF_MAX_CLASSES); fq_cl = &fqs->fqs_classq[pri]; @@ -208,13 +306,15 @@ fq_if_enqueue_classq(struct ifclassq *ifq, struct mbuf *m) /* BK_SYS is currently throttled */ fq_cl->fcl_stat.fcl_throttle_drops++; IFCQ_CONVERT_LOCK(ifq); - m_freem(m); + pktsched_free_pkt(&pkt); + *pdrop = TRUE; return (EQSUSPENDED); } - len = m_length(m); - ret = fq_addq(fqs, m, fq_cl); - if (!FQ_IF_CLASSQ_IDLE(fq_cl)) { + len = pktsched_get_pkt_len(&pkt); + ret = fq_addq(fqs, &pkt, fq_cl); + if (!(fqs->fqs_flags & FQS_DRIVER_MANAGED) && + !FQ_IF_CLASSQ_IDLE(fq_cl)) { if (((fqs->fqs_bitmaps[FQ_IF_ER] | fqs->fqs_bitmaps[FQ_IF_EB]) & (1 << pri)) == 0) { /* @@ -229,56 +329,91 @@ fq_if_enqueue_classq(struct ifclassq *ifq, struct mbuf *m) if (ret == CLASSQEQ_SUCCESS_FC) { /* packet enqueued, return advisory feedback */ ret = EQFULL; + *pdrop = FALSE; } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); + *pdrop = TRUE; + VERIFY(ret == CLASSQEQ_DROP || + ret == CLASSQEQ_DROP_FC || + ret == CLASSQEQ_DROP_SP); + pktsched_free_pkt(&pkt); switch (ret) { - case CLASSQEQ_DROPPED: + case CLASSQEQ_DROP: return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: + case CLASSQEQ_DROP_FC: return (EQFULL); - case CLASSQEQ_DROPPED_SP: + case CLASSQEQ_DROP_SP: return (EQSUSPENDED); } } + } else { + *pdrop = FALSE; } IFCQ_INC_LEN(ifq); IFCQ_INC_BYTES(ifq, len); return (ret); } -struct mbuf * -fq_if_dequeue_classq(struct ifclassq *ifq, cqdq_op_t op) +static void * +fq_if_dequeue_classq(struct ifclassq *ifq, classq_pkt_type_t *ptype) { - struct mbuf *top; + void *top; - (void) fq_if_dequeue_classq_multi(ifq, op, 1, - CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, &top, NULL, NULL, NULL); + (void) fq_if_dequeue_classq_multi(ifq, 1, + CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, &top, NULL, NULL, NULL, ptype); + return (top); +} +static void * +fq_if_dequeue_sc_classq(struct ifclassq *ifq, mbuf_svc_class_t svc, + classq_pkt_type_t *ptype) +{ + void *top; + fq_if_t *fqs = (fq_if_t *)ifq->ifcq_disc; + fq_if_classq_t *fq_cl; + u_int32_t pri; + + pri = fq_if_service_to_priority(fqs, svc); + fq_cl = &fqs->fqs_classq[pri]; + + fq_if_dequeue(fqs, fq_cl, 1, CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, + &top, NULL, NULL, NULL, TRUE, ptype); return (top); } int -fq_if_dequeue_classq_multi(struct ifclassq *ifq, cqdq_op_t op, - u_int32_t maxpktcnt, u_int32_t maxbytecnt, struct mbuf **first_packet, - struct mbuf **last_packet, u_int32_t *retpktcnt, u_int32_t *retbytecnt) +fq_if_dequeue_classq_multi(struct ifclassq *ifq, u_int32_t maxpktcnt, + u_int32_t maxbytecnt, void **first_packet, + void **last_packet, u_int32_t *retpktcnt, u_int32_t *retbytecnt, + classq_pkt_type_t *ptype) { -#pragma unused(op) - struct mbuf *top = NULL, *tail = NULL, *first, *last; + void *top = NULL, *tail = NULL, *first, *last; u_int32_t pktcnt = 0, bytecnt = 0, total_pktcnt, total_bytecnt; fq_if_t *fqs; fq_if_classq_t *fq_cl; int pri; + fq_if_append_pkt_t append_pkt; IFCQ_LOCK_ASSERT_HELD(ifq); fqs = (fq_if_t *)ifq->ifcq_disc; + switch (fqs->fqs_ptype) { + case QP_MBUF: + append_pkt = fq_if_append_mbuf; + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } + first = last = NULL; total_pktcnt = total_bytecnt = 0; + *ptype = fqs->fqs_ptype; for (;;) { + classq_pkt_type_t tmp_ptype; if (fqs->fqs_bitmaps[FQ_IF_ER] == 0 && fqs->fqs_bitmaps[FQ_IF_EB] == 0) { fqs->fqs_bitmaps[FQ_IF_EB] = fqs->fqs_bitmaps[FQ_IF_IB]; @@ -313,21 +448,22 @@ fq_if_dequeue_classq_multi(struct ifclassq *ifq, cqdq_op_t op, } fq_if_dequeue(fqs, fq_cl, (maxpktcnt - total_pktcnt), (maxbytecnt - total_bytecnt), &top, &tail, &pktcnt, - &bytecnt); + &bytecnt, FALSE, &tmp_ptype); if (top != NULL) { - VERIFY(pktcnt > 0 && bytecnt > 0); + ASSERT(tmp_ptype == *ptype); + ASSERT(pktcnt > 0 && bytecnt > 0); if (first == NULL) { first = top; last = tail; total_pktcnt = pktcnt; total_bytecnt = bytecnt; } else { - last->m_nextpkt = top; + append_pkt(last, top); last = tail; total_pktcnt += pktcnt; total_bytecnt += bytecnt; } - last->m_nextpkt = NULL; + append_pkt(last, NULL); fq_cl->fcl_budget -= bytecnt; pktcnt = 0; bytecnt = 0; @@ -378,21 +514,94 @@ state_change: return (0); } +int +fq_if_dequeue_sc_classq_multi(struct ifclassq *ifq, mbuf_svc_class_t svc, + u_int32_t maxpktcnt, u_int32_t maxbytecnt, void **first_packet, + void **last_packet, u_int32_t *retpktcnt, u_int32_t *retbytecnt, + classq_pkt_type_t *ptype) +{ +#pragma unused(maxpktcnt, maxbytecnt, first_packet, last_packet, retpktcnt, retbytecnt) + fq_if_t *fqs = (fq_if_t *)ifq->ifcq_disc; + u_int32_t pri; + u_int32_t total_pktcnt = 0, total_bytecnt = 0; + fq_if_classq_t *fq_cl; + void *first = NULL, *last = NULL; + fq_if_append_pkt_t append_pkt; + + switch (fqs->fqs_ptype) { + case QP_MBUF: + append_pkt = fq_if_append_mbuf; + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } + + pri = fq_if_service_to_priority(fqs, svc); + fq_cl = &fqs->fqs_classq[pri]; + + /* + * Now we have the queue for a particular service class. We need + * to dequeue as many packets as needed, first from the new flows + * and then from the old flows. + */ + while (total_pktcnt < maxpktcnt && total_bytecnt < maxbytecnt && + fq_cl->fcl_stat.fcl_pkt_cnt > 0) { + void *top, *tail; + u_int32_t pktcnt = 0, bytecnt = 0; + fq_if_dequeue(fqs, fq_cl, (maxpktcnt - total_pktcnt), + (maxbytecnt - total_bytecnt), &top, &tail, &pktcnt, + &bytecnt, TRUE, ptype); + if (first == NULL) { + first = top; + total_pktcnt = pktcnt; + total_bytecnt = bytecnt; + } else { + append_pkt(last, top); + total_pktcnt += pktcnt; + total_bytecnt += bytecnt; + } + last = tail; + } + if (first != NULL) { + if (first_packet != NULL) + *first_packet = first; + if (last_packet != NULL) + *last_packet = last; + if (retpktcnt != NULL) + *retpktcnt = total_pktcnt; + if (retbytecnt != NULL) + *retbytecnt = total_bytecnt; + } else { + if (first_packet != NULL) + *first_packet = NULL; + if (last_packet != NULL) + *last_packet = NULL; + if (retpktcnt != NULL) + *retpktcnt = 0; + if (retbytecnt != NULL) + *retbytecnt = 0; + } + return (0); +} + static void fq_if_purge_flow(fq_if_t *fqs, fq_t *fq, u_int32_t *pktsp, u_int32_t *bytesp) { fq_if_classq_t *fq_cl; u_int32_t pkts, bytes; - struct mbuf *m; + pktsched_pkt_t pkt; fq_cl = &fqs->fqs_classq[fq->fq_sc_index]; pkts = bytes = 0; - while ((m = fq_getq_flow(fqs, fq)) != NULL) { + _PKTSCHED_PKT_INIT(&pkt); + while (fq_getq_flow(fqs, fq, &pkt) != NULL) { pkts++; - bytes += m_length(m); - m_freem(m); - m = NULL; + bytes += pktsched_get_pkt_len(&pkt); + pktsched_free_pkt(&pkt); } IFCQ_DROP_ADD(fqs->fqs_ifq, pkts, bytes); @@ -471,7 +680,8 @@ fq_if_purge_sc(fq_if_t *fqs, cqrq_purge_sc_t *req) req->packets = req->bytes = 0; VERIFY(req->flow != 0); - fq = fq_if_hash_pkt(fqs, req->flow, req->sc, 0, FALSE); + /* packet type is needed only if we want to create a flow queue */ + fq = fq_if_hash_pkt(fqs, req->flow, req->sc, 0, FALSE, QP_INVALID); if (fq != NULL) fq_if_purge_flow(fqs, fq, &req->packets, &req->bytes); @@ -514,7 +724,9 @@ fq_if_throttle(fq_if_t *fqs, cqrq_throttle_t *tr) { struct ifclassq *ifq = fqs->fqs_ifq; int index; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); if (!tr->set) { @@ -526,7 +738,7 @@ fq_if_throttle(fq_if_t *fqs, cqrq_throttle_t *tr) return (EALREADY); /* Throttling is allowed on BK_SYS class only */ - index = fq_if_service_to_priority(MBUF_SC_BK_SYS); + index = fq_if_service_to_priority(fqs, MBUF_SC_BK_SYS); switch (tr->level) { case IFNET_THROTTLE_OFF: fq_if_classq_resume(fqs, &fqs->fqs_classq[index]); @@ -549,7 +761,7 @@ fq_if_stat_sc(fq_if_t *fqs, cqrq_stat_sc_t *stat) if (stat == NULL) return; - pri = fq_if_service_to_priority(stat->sc); + pri = fq_if_service_to_priority(fqs, stat->sc); fq_cl = &fqs->fqs_classq[pri]; stat->packets = fq_cl->fcl_stat.fcl_pkt_cnt; stat->bytes = fq_cl->fcl_stat.fcl_byte_cnt; @@ -588,7 +800,8 @@ fq_if_request_classq(struct ifclassq *ifq, cqrq_t rq, void *arg) } int -fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) +fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags, + classq_pkt_type_t ptype) { #pragma unused(flags) struct ifnet *ifp = ifq->ifcq_ifp; @@ -599,24 +812,47 @@ fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) VERIFY(ifq->ifcq_disc == NULL); VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); - fqs = fq_if_alloc(ifp, M_WAITOK); + fqs = fq_if_alloc(ifp, ptype); if (fqs == NULL) return (ENOMEM); - fq_if_classq_init(fqs, FQ_IF_BK_SYS_INDEX, 1500, 2, MBUF_SC_BK_SYS); - fq_if_classq_init(fqs, FQ_IF_BK_INDEX, 1500, 2, MBUF_SC_BK); - fq_if_classq_init(fqs, FQ_IF_BE_INDEX, 1500, 4, MBUF_SC_BE); - fq_if_classq_init(fqs, FQ_IF_RD_INDEX, 1500, 4, MBUF_SC_RD); - fq_if_classq_init(fqs, FQ_IF_OAM_INDEX, 1500, 4, MBUF_SC_OAM); - fq_if_classq_init(fqs, FQ_IF_AV_INDEX, 3000, 6, MBUF_SC_AV); - fq_if_classq_init(fqs, FQ_IF_RV_INDEX, 3000, 6, MBUF_SC_RV); - fq_if_classq_init(fqs, FQ_IF_VI_INDEX, 3000, 6, MBUF_SC_VI); - fq_if_classq_init(fqs, FQ_IF_VO_INDEX, 600, 8, MBUF_SC_VO); - fq_if_classq_init(fqs, FQ_IF_CTL_INDEX, 600, 8, MBUF_SC_CTL); + if (flags & PKTSCHEDF_QALG_DRIVER_MANAGED) { + fqs->fqs_flags |= FQS_DRIVER_MANAGED; + fq_if_classq_init(fqs, FQ_IF_BK_INDEX, 1500, + 2, MBUF_SC_BK); + fq_if_classq_init(fqs, FQ_IF_BE_INDEX, 1500, + 4, MBUF_SC_BE); + fq_if_classq_init(fqs, FQ_IF_VI_INDEX, 3000, + 6, MBUF_SC_VI); + fq_if_classq_init(fqs, FQ_IF_VO_INDEX, 600, + 8, MBUF_SC_VO); + } else { + fq_if_classq_init(fqs, FQ_IF_BK_SYS_INDEX, 1500, + 2, MBUF_SC_BK_SYS); + fq_if_classq_init(fqs, FQ_IF_BK_INDEX, 1500, + 2, MBUF_SC_BK); + fq_if_classq_init(fqs, FQ_IF_BE_INDEX, 1500, + 4, MBUF_SC_BE); + fq_if_classq_init(fqs, FQ_IF_RD_INDEX, 1500, + 4, MBUF_SC_RD); + fq_if_classq_init(fqs, FQ_IF_OAM_INDEX, 1500, + 4, MBUF_SC_OAM); + fq_if_classq_init(fqs, FQ_IF_AV_INDEX, 3000, + 6, MBUF_SC_AV); + fq_if_classq_init(fqs, FQ_IF_RV_INDEX, 3000, + 6, MBUF_SC_RV); + fq_if_classq_init(fqs, FQ_IF_VI_INDEX, 3000, + 6, MBUF_SC_VI); + fq_if_classq_init(fqs, FQ_IF_VO_INDEX, 600, + 8, MBUF_SC_VO); + fq_if_classq_init(fqs, FQ_IF_CTL_INDEX, 600, + 8, MBUF_SC_CTL); + } err = ifclassq_attach(ifq, PKTSCHEDT_FQ_CODEL, fqs, - fq_if_enqueue_classq, fq_if_dequeue_classq, NULL, - fq_if_dequeue_classq_multi, fq_if_request_classq); + fq_if_enqueue_classq, fq_if_dequeue_classq, + fq_if_dequeue_sc_classq, fq_if_dequeue_classq_multi, + fq_if_dequeue_sc_classq_multi, fq_if_request_classq); if (err != 0) { printf("%s: error from ifclassq_attach, " @@ -628,7 +864,7 @@ fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) fq_t * fq_if_hash_pkt(fq_if_t *fqs, u_int32_t flowid, mbuf_svc_class_t svc_class, - u_int64_t now, boolean_t create) + u_int64_t now, boolean_t create, classq_pkt_type_t ptype) { fq_t *fq = NULL; flowq_list_t *fq_list; @@ -636,7 +872,7 @@ fq_if_hash_pkt(fq_if_t *fqs, u_int32_t flowid, mbuf_svc_class_t svc_class, u_int8_t fqs_hash_id; u_int8_t scidx; - scidx = fq_if_service_to_priority(svc_class); + scidx = fq_if_service_to_priority(fqs, svc_class); fqs_hash_id = FQ_IF_FLOW_HASH_ID(flowid); @@ -648,15 +884,16 @@ fq_if_hash_pkt(fq_if_t *fqs, u_int32_t flowid, mbuf_svc_class_t svc_class, break; } if (fq == NULL && create == TRUE) { + ASSERT(ptype == QP_MBUF); + /* If the flow is not already on the list, allocate it */ IFCQ_CONVERT_LOCK(fqs->fqs_ifq); - fq = fq_alloc(M_WAITOK); + fq = fq_alloc(ptype); if (fq != NULL) { fq->fq_flowhash = flowid; fq->fq_sc_index = scidx; fq->fq_updatetime = now + fqs->fqs_update_interval; fq_cl = &fqs->fqs_classq[scidx]; - fq->fq_flags = FQF_FLOWCTL_CAPABLE; SLIST_INSERT_HEAD(fq_list, fq, fq_hashlink); fq_cl->fcl_stat.fcl_flows_cnt++; @@ -667,13 +904,13 @@ fq_if_hash_pkt(fq_if_t *fqs, u_int32_t flowid, mbuf_svc_class_t svc_class, * If getq time is not set because this is the first packet or after * idle time, set it now so that we can detect a stall. */ - if (fq->fq_getqtime == 0) + if (fq != NULL && fq->fq_getqtime == 0) fq->fq_getqtime = now; return (fq); } -static void +void fq_if_destroy_flow(fq_if_t *fqs, fq_if_classq_t *fq_cl, fq_t *fq) { u_int8_t hash_id; @@ -734,20 +971,30 @@ inline void fq_if_drop_packet(fq_if_t *fqs) { fq_t *fq = fqs->fqs_large_flow; - struct mbuf *m; fq_if_classq_t *fq_cl; + pktsched_pkt_t pkt; + uint32_t *pkt_flags; + uint64_t *pkt_timestamp; if (fq == NULL) return; - /* mbufq can not be empty on the largest flow */ - VERIFY(!MBUFQ_EMPTY(&fq->fq_mbufq)); + /* queue can not be empty on the largest flow */ + VERIFY(!fq_empty(fq)); fq_cl = &fqs->fqs_classq[fq->fq_sc_index]; + _PKTSCHED_PKT_INIT(&pkt); + (void)fq_getq_flow_internal(fqs, fq, &pkt); - m = fq_getq_flow(fqs, fq); + pktsched_get_pkt_vars(&pkt, &pkt_flags, &pkt_timestamp, NULL, NULL, + NULL, NULL); IFCQ_CONVERT_LOCK(fqs->fqs_ifq); - if (MBUFQ_EMPTY(&fq->fq_mbufq)) { + *pkt_timestamp = 0; + if (pkt.pktsched_ptype == QP_MBUF) + *pkt_flags &= ~PKTF_PRIV_GUARDED; + + if (fq_empty(fq)) { + fqs->fqs_large_flow = NULL; if (fq->fq_flags & FQF_OLD_FLOW) { fq_if_empty_old_flow(fqs, fq_cl, fq, true); } else { @@ -755,18 +1002,28 @@ fq_if_drop_packet(fq_if_t *fqs) fq_if_empty_new_flow(fq, fq_cl, true); } } - IFCQ_DROP_ADD(fqs->fqs_ifq, 1, m_length(m)); + IFCQ_DROP_ADD(fqs->fqs_ifq, 1, pktsched_get_pkt_len(&pkt)); - m_freem(m); + pktsched_free_pkt(&pkt); fq_cl->fcl_stat.fcl_drop_overflow++; } inline void fq_if_is_flow_heavy(fq_if_t *fqs, fq_t *fq) { - fq_t *prev_fq = fqs->fqs_large_flow; - if (prev_fq == NULL && !MBUFQ_EMPTY(&fq->fq_mbufq)) { - fqs->fqs_large_flow = fq; + fq_t *prev_fq; + + if (fqs->fqs_large_flow != NULL && + fqs->fqs_large_flow->fq_bytes < FQ_IF_LARGE_FLOW_BYTE_LIMIT) + fqs->fqs_large_flow = NULL; + + if (fq == NULL || fq->fq_bytes < FQ_IF_LARGE_FLOW_BYTE_LIMIT) + return; + + prev_fq = fqs->fqs_large_flow; + if (prev_fq == NULL) { + if (!fq_empty(fq)) + fqs->fqs_large_flow = fq; return; } else if (fq->fq_bytes > prev_fq->fq_bytes) { fqs->fqs_large_flow = fq; @@ -774,27 +1031,21 @@ fq_if_is_flow_heavy(fq_if_t *fqs, fq_t *fq) } boolean_t -fq_if_add_fcentry(fq_if_t *fqs, struct pkthdr *pkt, fq_if_classq_t *fq_cl) +fq_if_add_fcentry(fq_if_t *fqs, pktsched_pkt_t *pkt, uint32_t flowid, + uint8_t flowsrc, fq_if_classq_t *fq_cl) { struct flowadv_fcentry *fce; - u_int32_t flowsrc, flowid; - - flowsrc = pkt->pkt_flowsrc; - flowid = pkt->pkt_flowid; STAILQ_FOREACH(fce, &fqs->fqs_fclist, fce_link) { - if (fce->fce_flowsrc == flowsrc && + if ((uint8_t)fce->fce_flowsrc_type == flowsrc && fce->fce_flowid == flowid) { /* Already on flowcontrol list */ return (TRUE); } } - IFCQ_CONVERT_LOCK(fqs->fqs_ifq); - fce = flowadv_alloc_entry(M_WAITOK); + fce = pktsched_alloc_fcentry(pkt, fqs->fqs_ifq->ifcq_ifp, M_WAITOK); if (fce != NULL) { - fce->fce_flowsrc = flowsrc; - fce->fce_flowid = flowid; /* XXX Add number of bytes in the queue */ STAILQ_INSERT_TAIL(&fqs->fqs_fclist, fce, fce_link); fq_cl->fcl_stat.fcl_flow_control++; @@ -824,92 +1075,65 @@ fq_if_flow_feedback(fq_if_t *fqs, fq_t *fq, fq_if_classq_t *fq_cl) void fq_if_dequeue(fq_if_t *fqs, fq_if_classq_t *fq_cl, u_int32_t pktlimit, - u_int32_t bytelimit, struct mbuf **top, struct mbuf **tail, - u_int32_t *retpktcnt, u_int32_t *retbytecnt) + u_int32_t bytelimit, void **top, void **tail, + u_int32_t *retpktcnt, u_int32_t *retbytecnt, boolean_t drvmgmt, + classq_pkt_type_t *ptype) { fq_t *fq = NULL, *tfq = NULL; - struct mbuf *m = NULL, *last = NULL; flowq_stailq_t temp_stailq; - u_int32_t pktcnt, bytecnt, mlen; - boolean_t limit_reached = FALSE; + u_int32_t pktcnt, bytecnt; + boolean_t qempty, limit_reached = FALSE; + void *last = NULL; + fq_getq_flow_t fq_getq_flow_fn; + + switch (fqs->fqs_ptype) { + case QP_MBUF: + fq_getq_flow_fn = fq_getq_flow_mbuf; + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } /* * maximum byte limit should not be greater than the budget for * this class */ - if ((int32_t)bytelimit > fq_cl->fcl_budget) + if ((int32_t)bytelimit > fq_cl->fcl_budget && !drvmgmt) bytelimit = fq_cl->fcl_budget; VERIFY(pktlimit > 0 && bytelimit > 0 && top != NULL); *top = NULL; + *ptype = fqs->fqs_ptype; pktcnt = bytecnt = 0; STAILQ_INIT(&temp_stailq); STAILQ_FOREACH_SAFE(fq, &fq_cl->fcl_new_flows, fq_actlink, tfq) { - VERIFY((fq->fq_flags & (FQF_NEW_FLOW|FQF_OLD_FLOW)) == + ASSERT((fq->fq_flags & (FQF_NEW_FLOW|FQF_OLD_FLOW)) == FQF_NEW_FLOW); - while (fq->fq_deficit > 0 && limit_reached == FALSE && - !MBUFQ_EMPTY(&fq->fq_mbufq)) { - - m = fq_getq_flow(fqs, fq); - m->m_pkthdr.pkt_flags |= PKTF_NEW_FLOW; - mlen = m_length(m); - fq->fq_deficit -= mlen; - - if (*top == NULL) { - *top = m; - } else { - last->m_nextpkt = m; - } - last = m; - last->m_nextpkt = NULL; - fq_cl->fcl_stat.fcl_dequeue++; - fq_cl->fcl_stat.fcl_dequeue_bytes += mlen; - pktcnt++; - bytecnt += mlen; + limit_reached = fq_getq_flow_fn(fqs, fq_cl, fq, bytelimit, + pktlimit, top, &last, &bytecnt, &pktcnt, &qempty, + PKTF_NEW_FLOW); - /* Check if the limit is reached */ - if (pktcnt >= pktlimit || bytecnt >= bytelimit) - limit_reached = TRUE; - } - - if (fq->fq_deficit <= 0 || MBUFQ_EMPTY(&fq->fq_mbufq)) { + if (fq->fq_deficit <= 0 || qempty) fq_if_empty_new_flow(fq, fq_cl, true); - fq->fq_deficit += fq_cl->fcl_quantum; - } - if (limit_reached == TRUE) + fq->fq_deficit += fq_cl->fcl_quantum; + if (limit_reached) goto done; } STAILQ_FOREACH_SAFE(fq, &fq_cl->fcl_old_flows, fq_actlink, tfq) { VERIFY((fq->fq_flags & (FQF_NEW_FLOW|FQF_OLD_FLOW)) == FQF_OLD_FLOW); - while (fq->fq_deficit > 0 && !MBUFQ_EMPTY(&fq->fq_mbufq) && - limit_reached == FALSE) { - m = fq_getq_flow(fqs, fq); - mlen = m_length(m); - fq->fq_deficit -= mlen; - if (*top == NULL) { - *top = m; - } else { - last->m_nextpkt = m; - } - last = m; - last->m_nextpkt = NULL; - fq_cl->fcl_stat.fcl_dequeue++; - fq_cl->fcl_stat.fcl_dequeue_bytes += mlen; - pktcnt++; - bytecnt += mlen; + limit_reached = fq_getq_flow_fn(fqs, fq_cl, fq, bytelimit, + pktlimit, top, &last, &bytecnt, &pktcnt, &qempty, 0); - /* Check if the limit is reached */ - if (pktcnt >= pktlimit || bytecnt >= bytelimit) - limit_reached = TRUE; - } - - if (MBUFQ_EMPTY(&fq->fq_mbufq)) { + if (qempty) { fq_if_empty_old_flow(fqs, fq_cl, fq, true); } else if (fq->fq_deficit <= 0) { STAILQ_REMOVE(&fq_cl->fcl_old_flows, fq, @@ -922,8 +1146,7 @@ fq_if_dequeue(fq_if_t *fqs, fq_if_classq_t *fq_cl, u_int32_t pktlimit, STAILQ_INSERT_TAIL(&temp_stailq, fq, fq_actlink); fq->fq_deficit += fq_cl->fcl_quantum; } - - if (limit_reached == TRUE) + if (limit_reached) break; } @@ -955,10 +1178,29 @@ fq_if_teardown_ifclassq(struct ifclassq *ifq) fq_if_destroy(fqs); ifq->ifcq_disc = NULL; - return (ifclassq_detach(ifq)); } +static void +fq_export_flowstats(fq_if_t *fqs, fq_t *fq, + struct fq_codel_flowstats *flowstat) +{ + bzero(flowstat, sizeof (*flowstat)); + flowstat->fqst_min_qdelay = fq->fq_min_qdelay; + flowstat->fqst_bytes = fq->fq_bytes; + flowstat->fqst_flowhash = fq->fq_flowhash; + if (fq->fq_flags & FQF_NEW_FLOW) + flowstat->fqst_flags |= FQ_FLOWSTATS_NEW_FLOW; + if (fq->fq_flags & FQF_OLD_FLOW) + flowstat->fqst_flags |= FQ_FLOWSTATS_OLD_FLOW; + if (fq->fq_flags & FQF_DELAY_HIGH) + flowstat->fqst_flags |= FQ_FLOWSTATS_DELAY_HIGH; + if (fq->fq_flags & FQF_FLOWCTL_ON) + flowstat->fqst_flags |= FQ_FLOWSTATS_FLOWCTL_ON; + if (fqs->fqs_large_flow == fq) + flowstat->fqst_flags |= FQ_FLOWSTATS_LARGE_FLOW; +} + int fq_if_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t qid, struct if_ifclassq_stats *ifqs) @@ -966,6 +1208,8 @@ fq_if_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t qid, struct fq_codel_classstats *fcls; fq_if_classq_t *fq_cl; fq_if_t *fqs; + fq_t *fq = NULL; + u_int32_t i, flowstat_cnt; if (qid >= FQ_IF_MAX_CLASSES) return (EINVAL); @@ -1002,5 +1246,28 @@ fq_if_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t qid, fcls->fcls_throttle_drops = fq_cl->fcl_stat.fcl_throttle_drops; fcls->fcls_dup_rexmts = fq_cl->fcl_stat.fcl_dup_rexmts; + /* Gather per flow stats */ + flowstat_cnt = min((fcls->fcls_newflows_cnt + + fcls->fcls_oldflows_cnt), FQ_IF_MAX_FLOWSTATS); + i = 0; + STAILQ_FOREACH(fq, &fq_cl->fcl_new_flows, fq_actlink) { + if (i >= fcls->fcls_newflows_cnt || i >= flowstat_cnt) + break; + + /* leave space for a few old flows */ + if ((flowstat_cnt - i) < fcls->fcls_oldflows_cnt && + i >= (FQ_IF_MAX_FLOWSTATS >> 1)) + break; + fq_export_flowstats(fqs, fq, &fcls->fcls_flowstats[i]); + i++; + } + STAILQ_FOREACH(fq, &fq_cl->fcl_old_flows, fq_actlink) { + if (i >= flowstat_cnt) + break; + fq_export_flowstats(fqs, fq, &fcls->fcls_flowstats[i]); + i++; + } + VERIFY(i <= flowstat_cnt); + fcls->fcls_flowstats_cnt = i; return (0); } diff --git a/bsd/net/pktsched/pktsched_fq_codel.h b/bsd/net/pktsched/pktsched_fq_codel.h index 8d760a409..be7629a71 100644 --- a/bsd/net/pktsched/pktsched_fq_codel.h +++ b/bsd/net/pktsched/pktsched_fq_codel.h @@ -73,15 +73,14 @@ struct fcl_stat { #define FQ_IF_HASH_TAG_MASK 0xFF #define FQ_IF_HASH_TABLE_SIZE (1 << FQ_IF_HASH_TAG_SIZE) -/* maximum number f packets stored across all queues */ -#define FQ_IF_MAX_PKT_LIMIT 2048 - /* Set the quantum to be one MTU */ #define FQ_IF_DEFAULT_QUANTUM 1500 /* Max number of service classes currently supported */ #define FQ_IF_MAX_CLASSES 10 +#define FQ_IF_LARGE_FLOW_BYTE_LIMIT 15000 + struct flowq; typedef u_int32_t pktsched_bitmap_t; struct if_ifclassq_stats; @@ -130,13 +129,30 @@ typedef struct fq_codel_sched_data { pktsched_bitmap_t fqs_bitmaps[FQ_IF_MAX_STATE]; u_int32_t fqs_pkt_droplimit; /* drop limit */ u_int8_t fqs_throttle; /* throttle on or off */ + u_int8_t fqs_flags; /* flags */ +#define FQS_DRIVER_MANAGED 0x1 fq_if_classq_t fqs_classq[FQ_IF_MAX_CLASSES]; /* class queues */ struct flowadv_fclist fqs_fclist; /* flow control state */ struct flowq *fqs_large_flow; /* flow has highest number of bytes */ + classq_pkt_type_t fqs_ptype; } fq_if_t; #endif /* BSD_KERNEL_PRIVATE */ +struct fq_codel_flowstats { + u_int32_t fqst_min_qdelay; +#define FQ_FLOWSTATS_OLD_FLOW 0x1 +#define FQ_FLOWSTATS_NEW_FLOW 0x2 +#define FQ_FLOWSTATS_LARGE_FLOW 0x4 +#define FQ_FLOWSTATS_DELAY_HIGH 0x8 +#define FQ_FLOWSTATS_FLOWCTL_ON 0x10 + u_int32_t fqst_flags; + u_int32_t fqst_bytes; + u_int32_t fqst_flowhash; +}; + +#define FQ_IF_MAX_FLOWSTATS 20 + struct fq_codel_classstats { u_int32_t fcls_pri; u_int32_t fcls_service_class; @@ -163,23 +179,28 @@ struct fq_codel_classstats { u_int32_t fcls_throttle_off; u_int32_t fcls_throttle_drops; u_int32_t fcls_dup_rexmts; + u_int32_t fcls_flowstats_cnt; + struct fq_codel_flowstats fcls_flowstats[FQ_IF_MAX_FLOWSTATS]; }; #ifdef BSD_KERNEL_PRIVATE extern void fq_codel_scheduler_init(void); extern struct flowq *fq_if_hash_pkt(fq_if_t *, u_int32_t, mbuf_svc_class_t, - u_int64_t, boolean_t); + u_int64_t, boolean_t, classq_pkt_type_t); extern boolean_t fq_if_at_drop_limit(fq_if_t *); extern void fq_if_drop_packet(fq_if_t *); extern void fq_if_is_flow_heavy(fq_if_t *, struct flowq *); -extern boolean_t fq_if_add_fcentry(fq_if_t *, struct pkthdr *, - fq_if_classq_t *); +extern boolean_t fq_if_add_fcentry(fq_if_t *, pktsched_pkt_t *, uint32_t, + uint8_t, fq_if_classq_t *); extern void fq_if_flow_feedback(fq_if_t *, struct flowq *, fq_if_classq_t *); -extern int fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags); +extern int fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags, + classq_pkt_type_t ptype); extern int fq_if_teardown_ifclassq(struct ifclassq *ifq); extern int fq_if_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t qid, struct if_ifclassq_stats *ifqs); +extern void fq_if_destroy_flow(fq_if_t *, fq_if_classq_t *, + struct flowq *); #endif /* BSD_KERNEL_PRIVATE */ diff --git a/bsd/net/pktsched/pktsched_hfsc.c b/bsd/net/pktsched/pktsched_hfsc.c deleted file mode 100644 index 365d16f01..000000000 --- a/bsd/net/pktsched/pktsched_hfsc.c +++ /dev/null @@ -1,2065 +0,0 @@ -/* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_hfsc.c,v 1.25 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_hfsc.c,v 1.17 2002/11/29 07:48:33 kjc Exp $ */ - -/* - * Copyright (c) 1997-1999 Carnegie Mellon University. All Rights Reserved. - * - * Permission to use, copy, modify, and distribute this software and - * its documentation is hereby granted (including for commercial or - * for-profit use), provided that both the copyright notice and this - * permission notice appear in all copies of the software, derivative - * works, or modified versions, and any portions thereof. - * - * THIS SOFTWARE IS EXPERIMENTAL AND IS KNOWN TO HAVE BUGS, SOME OF - * WHICH MAY HAVE SERIOUS CONSEQUENCES. CARNEGIE MELLON PROVIDES THIS - * SOFTWARE IN ITS ``AS IS'' CONDITION, AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT - * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE - * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH - * DAMAGE. - * - * Carnegie Mellon encourages (but does not require) users of this - * software to return any improvements or extensions that they make, - * and to grant Carnegie Mellon the rights to redistribute these - * changes without encumbrance. - */ -/* - * H-FSC is described in Proceedings of SIGCOMM'97, - * "A Hierarchical Fair Service Curve Algorithm for Link-Sharing, - * Real-Time and Priority Service" - * by Ion Stoica, Hui Zhang, and T. S. Eugene Ng. - * - * Oleg Cherevko added the upperlimit for link-sharing. - * when a class has an upperlimit, the fit-time is computed from the - * upperlimit service curve. the link-sharing scheduler does not schedule - * a class whose fit-time exceeds the current time. - */ - -#if PKTSCHED_HFSC - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#include -#include - -/* - * function prototypes - */ -#if 0 -static int hfsc_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *hfsc_dequeue_ifclassq(struct ifclassq *, cqdq_op_t); -static int hfsc_request_ifclassq(struct ifclassq *, cqrq_t, void *); -#endif -static int hfsc_addq(struct hfsc_class *, struct mbuf *, struct pf_mtag *); -static struct mbuf *hfsc_getq(struct hfsc_class *); -static struct mbuf *hfsc_pollq(struct hfsc_class *); -static void hfsc_purgeq(struct hfsc_if *, struct hfsc_class *, u_int32_t, - u_int32_t *, u_int32_t *); -static void hfsc_print_sc(struct hfsc_if *, u_int32_t, u_int64_t, - struct service_curve *, struct internal_sc *, const char *); -static void hfsc_updateq_linkrate(struct hfsc_if *, struct hfsc_class *); -static void hfsc_updateq(struct hfsc_if *, struct hfsc_class *, cqev_t); - -static int hfsc_clear_interface(struct hfsc_if *); -static struct hfsc_class *hfsc_class_create(struct hfsc_if *, - struct service_curve *, struct service_curve *, struct service_curve *, - struct hfsc_class *, u_int32_t, int, u_int32_t); -static int hfsc_class_destroy(struct hfsc_if *, struct hfsc_class *); -static int hfsc_destroy_locked(struct hfsc_if *); -static struct hfsc_class *hfsc_nextclass(struct hfsc_class *); -static struct hfsc_class *hfsc_clh_to_clp(struct hfsc_if *, u_int32_t); -static const char *hfsc_style(struct hfsc_if *); - -static void set_active(struct hfsc_class *, u_int32_t); -static void set_passive(struct hfsc_class *); - -static void init_ed(struct hfsc_class *, u_int32_t); -static void update_ed(struct hfsc_class *, u_int32_t); -static void update_d(struct hfsc_class *, u_int32_t); -static void init_vf(struct hfsc_class *, u_int32_t); -static void update_vf(struct hfsc_class *, u_int32_t, u_int64_t); -static void update_cfmin(struct hfsc_class *); -static void ellist_insert(struct hfsc_class *); -static void ellist_remove(struct hfsc_class *); -static void ellist_update(struct hfsc_class *); -static struct hfsc_class *ellist_get_mindl(ellist_t *, u_int64_t); -static void actlist_insert(struct hfsc_class *); -static void actlist_remove(struct hfsc_class *); -static void actlist_update(struct hfsc_class *); -static struct hfsc_class *actlist_firstfit(struct hfsc_class *, u_int64_t); - -static inline u_int64_t seg_x2y(u_int64_t, u_int64_t); -static inline u_int64_t seg_y2x(u_int64_t, u_int64_t); -static inline u_int64_t m2sm(u_int64_t); -static inline u_int64_t m2ism(u_int64_t); -static inline u_int64_t d2dx(u_int64_t); -static u_int64_t sm2m(u_int64_t); -static u_int64_t dx2d(u_int64_t); - -static boolean_t sc2isc(struct hfsc_class *, struct service_curve *, - struct internal_sc *, u_int64_t); -static void rtsc_init(struct runtime_sc *, struct internal_sc *, - u_int64_t, u_int64_t); -static u_int64_t rtsc_y2x(struct runtime_sc *, u_int64_t); -static u_int64_t rtsc_x2y(struct runtime_sc *, u_int64_t); -static void rtsc_min(struct runtime_sc *, struct internal_sc *, - u_int64_t, u_int64_t); - -#define HFSC_ZONE_MAX 32 /* maximum elements in zone */ -#define HFSC_ZONE_NAME "pktsched_hfsc" /* zone name */ - -static unsigned int hfsc_size; /* size of zone element */ -static struct zone *hfsc_zone; /* zone for hfsc_if */ - -#define HFSC_CL_ZONE_MAX 32 /* maximum elements in zone */ -#define HFSC_CL_ZONE_NAME "pktsched_hfsc_cl" /* zone name */ - -static unsigned int hfsc_cl_size; /* size of zone element */ -static struct zone *hfsc_cl_zone; /* zone for hfsc_class */ - -/* - * macros - */ -#define HFSC_IS_A_PARENT_CLASS(cl) ((cl)->cl_children != NULL) - -#define HT_INFINITY 0xffffffffffffffffLL /* infinite time value */ - -void -hfsc_init(void) -{ - hfsc_size = sizeof (struct hfsc_if); - hfsc_zone = zinit(hfsc_size, HFSC_ZONE_MAX * hfsc_size, - 0, HFSC_ZONE_NAME); - if (hfsc_zone == NULL) { - panic("%s: failed allocating %s", __func__, HFSC_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(hfsc_zone, Z_EXPAND, TRUE); - zone_change(hfsc_zone, Z_CALLERACCT, TRUE); - - hfsc_cl_size = sizeof (struct hfsc_class); - hfsc_cl_zone = zinit(hfsc_cl_size, HFSC_CL_ZONE_MAX * hfsc_cl_size, - 0, HFSC_CL_ZONE_NAME); - if (hfsc_cl_zone == NULL) { - panic("%s: failed allocating %s", __func__, HFSC_CL_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(hfsc_cl_zone, Z_EXPAND, TRUE); - zone_change(hfsc_cl_zone, Z_CALLERACCT, TRUE); -} - -struct hfsc_if * -hfsc_alloc(struct ifnet *ifp, int how, boolean_t altq) -{ - struct hfsc_if *hif; - - hif = (how == M_WAITOK) ? zalloc(hfsc_zone) : zalloc_noblock(hfsc_zone); - if (hif == NULL) - return (NULL); - - bzero(hif, hfsc_size); - TAILQ_INIT(&hif->hif_eligible); - hif->hif_ifq = &ifp->if_snd; - if (altq) { - hif->hif_maxclasses = HFSC_MAX_CLASSES; - hif->hif_flags |= HFSCIFF_ALTQ; - } else { - hif->hif_maxclasses = IFCQ_SC_MAX + 1; /* incl. root class */ - } - - if ((hif->hif_class_tbl = _MALLOC(sizeof (struct hfsc_class *) * - hif->hif_maxclasses, M_DEVBUF, M_WAITOK|M_ZERO)) == NULL) { - log(LOG_ERR, "%s: %s unable to allocate class table array\n", - if_name(ifp), hfsc_style(hif)); - goto error; - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler allocated\n", - if_name(ifp), hfsc_style(hif)); - } - - return (hif); - -error: - if (hif->hif_class_tbl != NULL) { - _FREE(hif->hif_class_tbl, M_DEVBUF); - hif->hif_class_tbl = NULL; - } - zfree(hfsc_zone, hif); - - return (NULL); -} - -int -hfsc_destroy(struct hfsc_if *hif) -{ - struct ifclassq *ifq = hif->hif_ifq; - int err; - - IFCQ_LOCK(ifq); - err = hfsc_destroy_locked(hif); - IFCQ_UNLOCK(ifq); - - return (err); -} - -static int -hfsc_destroy_locked(struct hfsc_if *hif) -{ - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - (void) hfsc_clear_interface(hif); - (void) hfsc_class_destroy(hif, hif->hif_rootclass); - - VERIFY(hif->hif_class_tbl != NULL); - _FREE(hif->hif_class_tbl, M_DEVBUF); - hif->hif_class_tbl = NULL; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler destroyed\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif)); - } - - zfree(hfsc_zone, hif); - - return (0); -} - -/* - * bring the interface back to the initial state by discarding - * all the filters and classes except the root class. - */ -static int -hfsc_clear_interface(struct hfsc_if *hif) -{ - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - /* clear out the classes */ - while (hif->hif_rootclass != NULL && - (cl = hif->hif_rootclass->cl_children) != NULL) { - /* - * remove the first leaf class found in the hierarchy - * then start over - */ - for (; cl != NULL; cl = hfsc_nextclass(cl)) { - if (!HFSC_IS_A_PARENT_CLASS(cl)) { - (void) hfsc_class_destroy(hif, cl); - break; - } - } - } - - return (0); -} - -/* discard all the queued packets on the interface */ -void -hfsc_purge(struct hfsc_if *hif) -{ - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - for (cl = hif->hif_rootclass; cl != NULL; cl = hfsc_nextclass(cl)) { - if (!qempty(&cl->cl_q)) - hfsc_purgeq(hif, cl, 0, NULL, NULL); - } -#if !PF_ALTQ - /* - * This assertion is safe to be made only when PF_ALTQ is not - * configured; otherwise, IFCQ_LEN represents the sum of the - * packets managed by ifcq_disc and altq_disc instances, which - * is possible when transitioning between the two. - */ - VERIFY(IFCQ_LEN(hif->hif_ifq) == 0); -#endif /* !PF_ALTQ */ -} - -void -hfsc_event(struct hfsc_if *hif, cqev_t ev) -{ - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - for (cl = hif->hif_rootclass; cl != NULL; cl = hfsc_nextclass(cl)) - hfsc_updateq(hif, cl, ev); -} - -int -hfsc_add_queue(struct hfsc_if *hif, struct service_curve *rtsc, - struct service_curve *lssc, struct service_curve *ulsc, - u_int32_t qlimit, int flags, u_int32_t parent_qid, u_int32_t qid, - struct hfsc_class **clp) -{ - struct hfsc_class *cl = NULL, *parent; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - if (parent_qid == HFSC_NULLCLASS_HANDLE && hif->hif_rootclass == NULL) - parent = NULL; - else if ((parent = hfsc_clh_to_clp(hif, parent_qid)) == NULL) - return (EINVAL); - - if (hfsc_clh_to_clp(hif, qid) != NULL) - return (EBUSY); - - cl = hfsc_class_create(hif, rtsc, lssc, ulsc, parent, - qlimit, flags, qid); - if (cl == NULL) - return (ENOMEM); - - if (clp != NULL) - *clp = cl; - - return (0); -} - -static struct hfsc_class * -hfsc_class_create(struct hfsc_if *hif, struct service_curve *rsc, - struct service_curve *fsc, struct service_curve *usc, - struct hfsc_class *parent, u_int32_t qlimit, int flags, u_int32_t qid) -{ - struct ifnet *ifp; - struct ifclassq *ifq; - struct hfsc_class *cl, *p; - u_int64_t eff_rate; - u_int32_t i; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - /* Sanitize flags unless internally configured */ - if (hif->hif_flags & HFSCIFF_ALTQ) - flags &= HFCF_USERFLAGS; - - if (hif->hif_classes >= hif->hif_maxclasses) { - log(LOG_ERR, "%s: %s out of classes! (max %d)\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif), - hif->hif_maxclasses); - return (NULL); - } - -#if !CLASSQ_RED - if (flags & HFCF_RED) { - log(LOG_ERR, "%s: %s RED not available!\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif)); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & HFCF_RIO) { - log(LOG_ERR, "%s: %s RIO not available!\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif)); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & HFCF_BLUE) { - log(LOG_ERR, "%s: %s BLUE not available!\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif)); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) && - (flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) != HFCF_RED && - (flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) != HFCF_RIO && - (flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) != HFCF_BLUE && - (flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) != HFCF_SFB) { - log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif)); - return (NULL); - } - - cl = zalloc(hfsc_cl_zone); - if (cl == NULL) - return (NULL); - - bzero(cl, hfsc_cl_size); - TAILQ_INIT(&cl->cl_actc); - ifq = hif->hif_ifq; - ifp = HFSCIF_IFP(hif); - - if (qlimit == 0 || qlimit > IFCQ_MAXLEN(ifq)) { - qlimit = IFCQ_MAXLEN(ifq); - if (qlimit == 0) - qlimit = DEFAULT_QLIMIT; /* use default */ - } - _qinit(&cl->cl_q, Q_DROPTAIL, qlimit); - - cl->cl_flags = flags; - if (flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) { -#if CLASSQ_RED || CLASSQ_RIO - int pkttime; -#endif /* CLASSQ_RED || CLASSQ_RIO */ - u_int64_t m2; - - m2 = 0; - if (rsc != NULL && rsc->m2 > m2) - m2 = rsc->m2; - if (fsc != NULL && fsc->m2 > m2) - m2 = fsc->m2; - if (usc != NULL && usc->m2 > m2) - m2 = usc->m2; - - cl->cl_qflags = 0; - if (flags & HFCF_ECN) { - if (flags & HFCF_BLUE) - cl->cl_qflags |= BLUEF_ECN; - else if (flags & HFCF_SFB) - cl->cl_qflags |= SFBF_ECN; - else if (flags & HFCF_RED) - cl->cl_qflags |= REDF_ECN; - else if (flags & HFCF_RIO) - cl->cl_qflags |= RIOF_ECN; - } - if (flags & HFCF_FLOWCTL) { - if (flags & HFCF_SFB) - cl->cl_qflags |= SFBF_FLOWCTL; - } - if (flags & HFCF_CLEARDSCP) { - if (flags & HFCF_RIO) - cl->cl_qflags |= RIOF_CLEARDSCP; - } -#if CLASSQ_RED || CLASSQ_RIO - /* - * XXX: RED & RIO should be watching link speed and MTU - * events and recompute pkttime accordingly. - */ - if (m2 < 8) - pkttime = 1000 * 1000 * 1000; /* 1 sec */ - else - pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 / - (m2 / 8); - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RED - if (flags & HFCF_RED) { - cl->cl_red = red_alloc(ifp, 0, 0, - qlimit(&cl->cl_q) * 10/100, - qlimit(&cl->cl_q) * 30/100, - cl->cl_qflags, pkttime); - if (cl->cl_red != NULL) - qtype(&cl->cl_q) = Q_RED; - } -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (flags & HFCF_RIO) { - cl->cl_rio = - rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime); - if (cl->cl_rio != NULL) - qtype(&cl->cl_q) = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#endif /* CLASSQ_RED || CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & HFCF_BLUE) { - cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags); - if (cl->cl_blue != NULL) - qtype(&cl->cl_q) = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & HFCF_SFB) { - if (!(cl->cl_flags & HFCF_LAZY)) - cl->cl_sfb = sfb_alloc(ifp, qid, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb != NULL || (cl->cl_flags & HFCF_LAZY)) - qtype(&cl->cl_q) = Q_SFB; - } - } - - cl->cl_id = hif->hif_classid++; - cl->cl_handle = qid; - cl->cl_hif = hif; - cl->cl_parent = parent; - - eff_rate = ifnet_output_linkrate(HFSCIF_IFP(hif)); - hif->hif_eff_rate = eff_rate; - - if (rsc != NULL && (rsc->m1 != 0 || rsc->m2 != 0) && - (!(rsc->fl & HFSCF_M1_PCT) || (rsc->m1 > 0 && rsc->m1 <= 100)) && - (!(rsc->fl & HFSCF_M2_PCT) || (rsc->m2 > 0 && rsc->m2 <= 100))) { - rsc->fl &= HFSCF_USERFLAGS; - cl->cl_flags |= HFCF_RSC; - cl->cl_rsc0 = *rsc; - (void) sc2isc(cl, &cl->cl_rsc0, &cl->cl_rsc, eff_rate); - rtsc_init(&cl->cl_deadline, &cl->cl_rsc, 0, 0); - rtsc_init(&cl->cl_eligible, &cl->cl_rsc, 0, 0); - } - if (fsc != NULL && (fsc->m1 != 0 || fsc->m2 != 0) && - (!(fsc->fl & HFSCF_M1_PCT) || (fsc->m1 > 0 && fsc->m1 <= 100)) && - (!(fsc->fl & HFSCF_M2_PCT) || (fsc->m2 > 0 && fsc->m2 <= 100))) { - fsc->fl &= HFSCF_USERFLAGS; - cl->cl_flags |= HFCF_FSC; - cl->cl_fsc0 = *fsc; - (void) sc2isc(cl, &cl->cl_fsc0, &cl->cl_fsc, eff_rate); - rtsc_init(&cl->cl_virtual, &cl->cl_fsc, 0, 0); - } - if (usc != NULL && (usc->m1 != 0 || usc->m2 != 0) && - (!(usc->fl & HFSCF_M1_PCT) || (usc->m1 > 0 && usc->m1 <= 100)) && - (!(usc->fl & HFSCF_M2_PCT) || (usc->m2 > 0 && usc->m2 <= 100))) { - usc->fl &= HFSCF_USERFLAGS; - cl->cl_flags |= HFCF_USC; - cl->cl_usc0 = *usc; - (void) sc2isc(cl, &cl->cl_usc0, &cl->cl_usc, eff_rate); - rtsc_init(&cl->cl_ulimit, &cl->cl_usc, 0, 0); - } - - /* - * find a free slot in the class table. if the slot matching - * the lower bits of qid is free, use this slot. otherwise, - * use the first free slot. - */ - i = qid % hif->hif_maxclasses; - if (hif->hif_class_tbl[i] == NULL) { - hif->hif_class_tbl[i] = cl; - } else { - for (i = 0; i < hif->hif_maxclasses; i++) - if (hif->hif_class_tbl[i] == NULL) { - hif->hif_class_tbl[i] = cl; - break; - } - if (i == hif->hif_maxclasses) { - goto err_ret; - } - } - hif->hif_classes++; - - if (flags & HFCF_DEFAULTCLASS) - hif->hif_defaultclass = cl; - - if (parent == NULL) { - /* this is root class */ - hif->hif_rootclass = cl; - } else { - /* add this class to the children list of the parent */ - if ((p = parent->cl_children) == NULL) - parent->cl_children = cl; - else { - while (p->cl_siblings != NULL) - p = p->cl_siblings; - p->cl_siblings = cl; - } - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s created qid=%d pqid=%d qlimit=%d " - "flags=%b\n", if_name(ifp), hfsc_style(hif), cl->cl_handle, - (cl->cl_parent != NULL) ? cl->cl_parent->cl_handle : 0, - qlimit(&cl->cl_q), cl->cl_flags, HFCF_BITS); - if (cl->cl_flags & HFCF_RSC) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - &cl->cl_rsc0, &cl->cl_rsc, "rsc"); - } - if (cl->cl_flags & HFCF_FSC) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - &cl->cl_fsc0, &cl->cl_fsc, "fsc"); - } - if (cl->cl_flags & HFCF_USC) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - &cl->cl_usc0, &cl->cl_usc, "usc"); - } - } - - return (cl); - -err_ret: - if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - qtype(&cl->cl_q) = Q_DROPTAIL; - qstate(&cl->cl_q) = QS_RUNNING; - } - zfree(hfsc_cl_zone, cl); - return (NULL); -} - -int -hfsc_remove_queue(struct hfsc_if *hif, u_int32_t qid) -{ - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - if ((cl = hfsc_clh_to_clp(hif, qid)) == NULL) - return (EINVAL); - - return (hfsc_class_destroy(hif, cl)); -} - -static int -hfsc_class_destroy(struct hfsc_if *hif, struct hfsc_class *cl) -{ - u_int32_t i; - - if (cl == NULL) - return (0); - - if (HFSC_IS_A_PARENT_CLASS(cl)) - return (EBUSY); - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - if (!qempty(&cl->cl_q)) - hfsc_purgeq(hif, cl, 0, NULL, NULL); - - if (cl->cl_parent == NULL) { - /* this is root class */ - } else { - struct hfsc_class *p = cl->cl_parent->cl_children; - - if (p == cl) - cl->cl_parent->cl_children = cl->cl_siblings; - else do { - if (p->cl_siblings == cl) { - p->cl_siblings = cl->cl_siblings; - break; - } - } while ((p = p->cl_siblings) != NULL); - VERIFY(p != NULL); - } - - for (i = 0; i < hif->hif_maxclasses; i++) - if (hif->hif_class_tbl[i] == cl) { - hif->hif_class_tbl[i] = NULL; - break; - } - - hif->hif_classes--; - - if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - qtype(&cl->cl_q) = Q_DROPTAIL; - qstate(&cl->cl_q) = QS_RUNNING; - } - - if (cl == hif->hif_rootclass) - hif->hif_rootclass = NULL; - if (cl == hif->hif_defaultclass) - hif->hif_defaultclass = NULL; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s destroyed qid=%d slot=%d\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif), - cl->cl_handle, cl->cl_id); - } - - zfree(hfsc_cl_zone, cl); - - return (0); -} - -/* - * hfsc_nextclass returns the next class in the tree. - * usage: - * for (cl = hif->hif_rootclass; cl != NULL; cl = hfsc_nextclass(cl)) - * do_something; - */ -static struct hfsc_class * -hfsc_nextclass(struct hfsc_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_hif->hif_ifq); - - if (cl->cl_children != NULL) - cl = cl->cl_children; - else if (cl->cl_siblings != NULL) - cl = cl->cl_siblings; - else { - while ((cl = cl->cl_parent) != NULL) - if (cl->cl_siblings) { - cl = cl->cl_siblings; - break; - } - } - - return (cl); -} - -int -hfsc_enqueue(struct hfsc_if *hif, struct hfsc_class *cl, struct mbuf *m, - struct pf_mtag *t) -{ - struct ifclassq *ifq = hif->hif_ifq; - u_int32_t len; - int ret; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(cl == NULL || cl->cl_hif == hif); - - if (cl == NULL) { -#if PF_ALTQ - cl = hfsc_clh_to_clp(hif, t->pftag_qid); -#else /* !PF_ALTQ */ - cl = hfsc_clh_to_clp(hif, 0); -#endif /* !PF_ALTQ */ - if (cl == NULL || HFSC_IS_A_PARENT_CLASS(cl)) { - cl = hif->hif_defaultclass; - if (cl == NULL) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - } - } - - len = m_pktlen(m); - - ret = hfsc_addq(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - /* packet has been freed in hfsc_addq */ - PKTCNTR_ADD(&cl->cl_stats.drop_cnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT_REACHED */ - } - } - IFCQ_INC_LEN(ifq); - IFCQ_INC_BYTES(ifq, len); - cl->cl_hif->hif_packets++; - - /* successfully queued. */ - if (qlen(&cl->cl_q) == 1) - set_active(cl, len); - - return (ret); -} - -/* - * note: CLASSQDQ_POLL returns the next packet without removing the packet - * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation. - * CLASSQDQ_REMOVE must return the same packet if called immediately - * after CLASSQDQ_POLL. - */ -struct mbuf * -hfsc_dequeue(struct hfsc_if *hif, cqdq_op_t op) -{ - struct ifclassq *ifq = hif->hif_ifq; - struct hfsc_class *cl; - struct mbuf *m; - u_int32_t len, next_len; - int realtime = 0; - u_int64_t cur_time; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (hif->hif_packets == 0) - /* no packet in the tree */ - return (NULL); - - cur_time = read_machclk(); - - if (op == CLASSQDQ_REMOVE && hif->hif_pollcache != NULL) { - - cl = hif->hif_pollcache; - hif->hif_pollcache = NULL; - /* check if the class was scheduled by real-time criteria */ - if (cl->cl_flags & HFCF_RSC) - realtime = (cl->cl_e <= cur_time); - } else { - /* - * if there are eligible classes, use real-time criteria. - * find the class with the minimum deadline among - * the eligible classes. - */ - if ((cl = ellist_get_mindl(&hif->hif_eligible, cur_time)) - != NULL) { - realtime = 1; - } else { - int fits = 0; - /* - * use link-sharing criteria - * get the class with the minimum vt in the hierarchy - */ - cl = hif->hif_rootclass; - while (HFSC_IS_A_PARENT_CLASS(cl)) { - - cl = actlist_firstfit(cl, cur_time); - if (cl == NULL) { - if (fits > 0) - log(LOG_ERR, "%s: %s " - "%d fit but none found\n", - if_name(HFSCIF_IFP(hif)), - hfsc_style(hif), fits); - return (NULL); - } - /* - * update parent's cl_cvtmin. - * don't update if the new vt is smaller. - */ - if (cl->cl_parent->cl_cvtmin < cl->cl_vt) - cl->cl_parent->cl_cvtmin = cl->cl_vt; - fits++; - } - } - - if (op == CLASSQDQ_POLL) { - hif->hif_pollcache = cl; - m = hfsc_pollq(cl); - return (m); - } - } - - m = hfsc_getq(cl); - VERIFY(m != NULL); - len = m_pktlen(m); - cl->cl_hif->hif_packets--; - IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, len); - IFCQ_XMIT_ADD(ifq, 1, len); - PKTCNTR_ADD(&cl->cl_stats.xmit_cnt, 1, len); - - update_vf(cl, len, cur_time); - if (realtime) - cl->cl_cumul += len; - - if (!qempty(&cl->cl_q)) { - if (cl->cl_flags & HFCF_RSC) { - /* update ed */ - next_len = m_pktlen(qhead(&cl->cl_q)); - - if (realtime) - update_ed(cl, next_len); - else - update_d(cl, next_len); - } - } else { - /* the class becomes passive */ - set_passive(cl); - } - - return (m); - -} - -static int -hfsc_addq(struct hfsc_class *cl, struct mbuf *m, struct pf_mtag *t) -{ - struct ifclassq *ifq = cl->cl_hif->hif_ifq; - - IFCQ_LOCK_ASSERT_HELD(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_addq(cl->cl_rio, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_addq(cl->cl_red, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_addq(cl->cl_blue, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q)) { - if (cl->cl_sfb == NULL) { - struct ifnet *ifp = HFSCIF_IFP(cl->cl_hif); - - VERIFY(cl->cl_flags & HFCF_LAZY); - IFCQ_CONVERT_LOCK(ifq); - - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb == NULL) { - /* fall back to droptail */ - qtype(&cl->cl_q) = Q_DROPTAIL; - cl->cl_flags &= ~HFCF_SFB; - cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL); - - log(LOG_ERR, "%s: %s SFB lazy allocation " - "failed for qid=%d slot=%d, falling back " - "to DROPTAIL\n", if_name(ifp), - hfsc_style(cl->cl_hif), cl->cl_handle, - cl->cl_id); - } - } - if (cl->cl_sfb != NULL) - return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t)); - } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - -#if PF_ECN - if (cl->cl_flags & HFCF_CLEARDSCP) - write_dsfield(m, t, 0); -#endif /* PF_ECN */ - - _addq(&cl->cl_q, m); - - return (0); -} - -static struct mbuf * -hfsc_getq(struct hfsc_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_hif->hif_ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_getq(cl->cl_rio, &cl->cl_q)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_getq(cl->cl_red, &cl->cl_q)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_getq(cl->cl_blue, &cl->cl_q)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_getq(cl->cl_sfb, &cl->cl_q)); - - return (_getq(&cl->cl_q)); -} - -static struct mbuf * -hfsc_pollq(struct hfsc_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_hif->hif_ifq); - - return (qhead(&cl->cl_q)); -} - -static void -hfsc_purgeq(struct hfsc_if *hif, struct hfsc_class *cl, u_int32_t flow, - u_int32_t *packets, u_int32_t *bytes) -{ - struct ifclassq *ifq = hif->hif_ifq; - u_int32_t cnt = 0, len = 0, qlen; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if ((qlen = qlen(&cl->cl_q)) == 0) { - VERIFY(hif->hif_packets == 0); - goto done; - } - - /* become regular mutex before freeing mbufs */ - IFCQ_CONVERT_LOCK(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len); - else - _flushq_flow(&cl->cl_q, flow, &cnt, &len); - - if (cnt > 0) { - VERIFY(qlen(&cl->cl_q) == (qlen - cnt)); - - PKTCNTR_ADD(&cl->cl_stats.drop_cnt, cnt, len); - IFCQ_DROP_ADD(ifq, cnt, len); - - VERIFY(hif->hif_packets >= cnt); - hif->hif_packets -= cnt; - - VERIFY(((signed)IFCQ_LEN(ifq) - cnt) >= 0); - IFCQ_LEN(ifq) -= cnt; - - if (qempty(&cl->cl_q)) { - update_vf(cl, 0, 0); /* remove cl from the actlist */ - set_passive(cl); - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s purge qid=%d slot=%d " - "qlen=[%d,%d] cnt=%d len=%d flow=0x%x\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif), - cl->cl_handle, cl->cl_id, qlen, qlen(&cl->cl_q), - cnt, len, flow); - } - } -done: - if (packets != NULL) - *packets = cnt; - if (bytes != NULL) - *bytes = len; -} - -static void -hfsc_print_sc(struct hfsc_if *hif, u_int32_t qid, u_int64_t eff_rate, - struct service_curve *sc, struct internal_sc *isc, const char *which) -{ - struct ifnet *ifp = HFSCIF_IFP(hif); - - log(LOG_DEBUG, "%s: %s qid=%d {%s_m1=%llu%s [%llu], " - "%s_d=%u msec, %s_m2=%llu%s [%llu]} linkrate=%llu bps\n", - if_name(ifp), hfsc_style(hif), qid, - which, sc->m1, (sc->fl & HFSCF_M1_PCT) ? "%" : " bps", isc->sm1, - which, sc->d, - which, sc->m2, (sc->fl & HFSCF_M2_PCT) ? "%" : " bps", isc->sm2, - eff_rate); -} - -static void -hfsc_updateq_linkrate(struct hfsc_if *hif, struct hfsc_class *cl) -{ - u_int64_t eff_rate = ifnet_output_linkrate(HFSCIF_IFP(hif)); - struct service_curve *sc; - struct internal_sc *isc; - - /* Update parameters only if rate has changed */ - if (eff_rate == hif->hif_eff_rate) - return; - - sc = &cl->cl_rsc0; - isc = &cl->cl_rsc; - if ((cl->cl_flags & HFCF_RSC) && sc2isc(cl, sc, isc, eff_rate)) { - rtsc_init(&cl->cl_deadline, isc, 0, 0); - rtsc_init(&cl->cl_eligible, isc, 0, 0); - if (pktsched_verbose) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - sc, isc, "rsc"); - } - } - sc = &cl->cl_fsc0; - isc = &cl->cl_fsc; - if ((cl->cl_flags & HFCF_FSC) && sc2isc(cl, sc, isc, eff_rate)) { - rtsc_init(&cl->cl_virtual, isc, 0, 0); - if (pktsched_verbose) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - sc, isc, "fsc"); - } - } - sc = &cl->cl_usc0; - isc = &cl->cl_usc; - if ((cl->cl_flags & HFCF_USC) && sc2isc(cl, sc, isc, eff_rate)) { - rtsc_init(&cl->cl_ulimit, isc, 0, 0); - if (pktsched_verbose) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - sc, isc, "usc"); - } - } -} - -static void -hfsc_updateq(struct hfsc_if *hif, struct hfsc_class *cl, cqev_t ev) -{ - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s update qid=%d slot=%d event=%s\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif), - cl->cl_handle, cl->cl_id, ifclassq_ev2str(ev)); - } - - if (ev == CLASSQ_EV_LINK_BANDWIDTH) - hfsc_updateq_linkrate(hif, cl); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_updateq(cl->cl_rio, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_updateq(cl->cl_red, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_updateq(cl->cl_blue, ev)); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_updateq(cl->cl_sfb, ev)); -} - -static void -set_active(struct hfsc_class *cl, u_int32_t len) -{ - if (cl->cl_flags & HFCF_RSC) - init_ed(cl, len); - if (cl->cl_flags & HFCF_FSC) - init_vf(cl, len); - - cl->cl_stats.period++; -} - -static void -set_passive(struct hfsc_class *cl) -{ - if (cl->cl_flags & HFCF_RSC) - ellist_remove(cl); - - /* - * actlist is now handled in update_vf() so that update_vf(cl, 0, 0) - * needs to be called explicitly to remove a class from actlist - */ -} - -static void -init_ed(struct hfsc_class *cl, u_int32_t next_len) -{ - u_int64_t cur_time; - - cur_time = read_machclk(); - - /* update the deadline curve */ - rtsc_min(&cl->cl_deadline, &cl->cl_rsc, cur_time, cl->cl_cumul); - - /* - * update the eligible curve. - * for concave, it is equal to the deadline curve. - * for convex, it is a linear curve with slope m2. - */ - cl->cl_eligible = cl->cl_deadline; - if (cl->cl_rsc.sm1 <= cl->cl_rsc.sm2) { - cl->cl_eligible.dx = 0; - cl->cl_eligible.dy = 0; - } - - /* compute e and d */ - cl->cl_e = rtsc_y2x(&cl->cl_eligible, cl->cl_cumul); - cl->cl_d = rtsc_y2x(&cl->cl_deadline, cl->cl_cumul + next_len); - - ellist_insert(cl); -} - -static void -update_ed(struct hfsc_class *cl, u_int32_t next_len) -{ - cl->cl_e = rtsc_y2x(&cl->cl_eligible, cl->cl_cumul); - cl->cl_d = rtsc_y2x(&cl->cl_deadline, cl->cl_cumul + next_len); - - ellist_update(cl); -} - -static void -update_d(struct hfsc_class *cl, u_int32_t next_len) -{ - cl->cl_d = rtsc_y2x(&cl->cl_deadline, cl->cl_cumul + next_len); -} - -static void -init_vf(struct hfsc_class *cl, u_int32_t len) -{ -#pragma unused(len) - struct hfsc_class *max_cl, *p; - u_int64_t vt, f, cur_time; - int go_active; - - cur_time = 0; - go_active = 1; - for (; cl->cl_parent != NULL; cl = cl->cl_parent) { - - if (go_active && cl->cl_nactive++ == 0) - go_active = 1; - else - go_active = 0; - - if (go_active) { - max_cl = actlist_last(&cl->cl_parent->cl_actc); - if (max_cl != NULL) { - /* - * set vt to the average of the min and max - * classes. if the parent's period didn't - * change, don't decrease vt of the class. - */ - vt = max_cl->cl_vt; - if (cl->cl_parent->cl_cvtmin != 0) - vt = (cl->cl_parent->cl_cvtmin + vt)/2; - - if (cl->cl_parent->cl_vtperiod != - cl->cl_parentperiod || vt > cl->cl_vt) - cl->cl_vt = vt; - } else { - /* - * first child for a new parent backlog period. - * add parent's cvtmax to vtoff of children - * to make a new vt (vtoff + vt) larger than - * the vt in the last period for all children. - */ - vt = cl->cl_parent->cl_cvtmax; - for (p = cl->cl_parent->cl_children; p != NULL; - p = p->cl_siblings) - p->cl_vtoff += vt; - cl->cl_vt = 0; - cl->cl_parent->cl_cvtmax = 0; - cl->cl_parent->cl_cvtmin = 0; - } - cl->cl_initvt = cl->cl_vt; - - /* update the virtual curve */ - vt = cl->cl_vt + cl->cl_vtoff; - rtsc_min(&cl->cl_virtual, &cl->cl_fsc, - vt, cl->cl_total); - if (cl->cl_virtual.x == vt) { - cl->cl_virtual.x -= cl->cl_vtoff; - cl->cl_vtoff = 0; - } - cl->cl_vtadj = 0; - - cl->cl_vtperiod++; /* increment vt period */ - cl->cl_parentperiod = cl->cl_parent->cl_vtperiod; - if (cl->cl_parent->cl_nactive == 0) - cl->cl_parentperiod++; - cl->cl_f = 0; - - actlist_insert(cl); - - if (cl->cl_flags & HFCF_USC) { - /* class has upper limit curve */ - if (cur_time == 0) - cur_time = read_machclk(); - - /* update the ulimit curve */ - rtsc_min(&cl->cl_ulimit, &cl->cl_usc, cur_time, - cl->cl_total); - /* compute myf */ - cl->cl_myf = rtsc_y2x(&cl->cl_ulimit, - cl->cl_total); - cl->cl_myfadj = 0; - } - } - - if (cl->cl_myf > cl->cl_cfmin) - f = cl->cl_myf; - else - f = cl->cl_cfmin; - if (f != cl->cl_f) { - cl->cl_f = f; - update_cfmin(cl->cl_parent); - } - } -} - -static void -update_vf(struct hfsc_class *cl, u_int32_t len, u_int64_t cur_time) -{ -#pragma unused(cur_time) -#if 0 - u_int64_t myf_bound, delta; -#endif - u_int64_t f; - int go_passive; - - go_passive = (qempty(&cl->cl_q) && (cl->cl_flags & HFCF_FSC)); - - for (; cl->cl_parent != NULL; cl = cl->cl_parent) { - - cl->cl_total += len; - - if (!(cl->cl_flags & HFCF_FSC) || cl->cl_nactive == 0) - continue; - - if (go_passive && --cl->cl_nactive == 0) - go_passive = 1; - else - go_passive = 0; - - if (go_passive) { - /* no more active child, going passive */ - - /* update cvtmax of the parent class */ - if (cl->cl_vt > cl->cl_parent->cl_cvtmax) - cl->cl_parent->cl_cvtmax = cl->cl_vt; - - /* remove this class from the vt list */ - actlist_remove(cl); - - update_cfmin(cl->cl_parent); - - continue; - } - - /* - * update vt and f - */ - cl->cl_vt = rtsc_y2x(&cl->cl_virtual, cl->cl_total) - - cl->cl_vtoff + cl->cl_vtadj; - - /* - * if vt of the class is smaller than cvtmin, - * the class was skipped in the past due to non-fit. - * if so, we need to adjust vtadj. - */ - if (cl->cl_vt < cl->cl_parent->cl_cvtmin) { - cl->cl_vtadj += cl->cl_parent->cl_cvtmin - cl->cl_vt; - cl->cl_vt = cl->cl_parent->cl_cvtmin; - } - - /* update the vt list */ - actlist_update(cl); - - if (cl->cl_flags & HFCF_USC) { - cl->cl_myf = cl->cl_myfadj + - rtsc_y2x(&cl->cl_ulimit, cl->cl_total); -#if 0 - /* - * if myf lags behind by more than one clock tick - * from the current time, adjust myfadj to prevent - * a rate-limited class from going greedy. - * in a steady state under rate-limiting, myf - * fluctuates within one clock tick. - */ - myf_bound = cur_time - machclk_per_tick; - if (cl->cl_myf < myf_bound) { - delta = cur_time - cl->cl_myf; - cl->cl_myfadj += delta; - cl->cl_myf += delta; - } -#endif - } - - /* cl_f is max(cl_myf, cl_cfmin) */ - if (cl->cl_myf > cl->cl_cfmin) - f = cl->cl_myf; - else - f = cl->cl_cfmin; - if (f != cl->cl_f) { - cl->cl_f = f; - update_cfmin(cl->cl_parent); - } - } -} - -static void -update_cfmin(struct hfsc_class *cl) -{ - struct hfsc_class *p; - u_int64_t cfmin; - - if (TAILQ_EMPTY(&cl->cl_actc)) { - cl->cl_cfmin = 0; - return; - } - cfmin = HT_INFINITY; - TAILQ_FOREACH(p, &cl->cl_actc, cl_actlist) { - if (p->cl_f == 0) { - cl->cl_cfmin = 0; - return; - } - if (p->cl_f < cfmin) - cfmin = p->cl_f; - } - cl->cl_cfmin = cfmin; -} - -/* - * TAILQ based ellist and actlist implementation - * (ion wanted to make a calendar queue based implementation) - */ -/* - * eligible list holds backlogged classes being sorted by their eligible times. - * there is one eligible list per interface. - */ - -static void -ellist_insert(struct hfsc_class *cl) -{ - struct hfsc_if *hif = cl->cl_hif; - struct hfsc_class *p; - - /* check the last entry first */ - if ((p = TAILQ_LAST(&hif->hif_eligible, _eligible)) == NULL || - p->cl_e <= cl->cl_e) { - TAILQ_INSERT_TAIL(&hif->hif_eligible, cl, cl_ellist); - return; - } - - TAILQ_FOREACH(p, &hif->hif_eligible, cl_ellist) { - if (cl->cl_e < p->cl_e) { - TAILQ_INSERT_BEFORE(p, cl, cl_ellist); - return; - } - } - VERIFY(0); /* should not reach here */ -} - -static void -ellist_remove(struct hfsc_class *cl) -{ - struct hfsc_if *hif = cl->cl_hif; - - TAILQ_REMOVE(&hif->hif_eligible, cl, cl_ellist); -} - -static void -ellist_update(struct hfsc_class *cl) -{ - struct hfsc_if *hif = cl->cl_hif; - struct hfsc_class *p, *last; - - /* - * the eligible time of a class increases monotonically. - * if the next entry has a larger eligible time, nothing to do. - */ - p = TAILQ_NEXT(cl, cl_ellist); - if (p == NULL || cl->cl_e <= p->cl_e) - return; - - /* check the last entry */ - last = TAILQ_LAST(&hif->hif_eligible, _eligible); - VERIFY(last != NULL); - if (last->cl_e <= cl->cl_e) { - TAILQ_REMOVE(&hif->hif_eligible, cl, cl_ellist); - TAILQ_INSERT_TAIL(&hif->hif_eligible, cl, cl_ellist); - return; - } - - /* - * the new position must be between the next entry - * and the last entry - */ - while ((p = TAILQ_NEXT(p, cl_ellist)) != NULL) { - if (cl->cl_e < p->cl_e) { - TAILQ_REMOVE(&hif->hif_eligible, cl, cl_ellist); - TAILQ_INSERT_BEFORE(p, cl, cl_ellist); - return; - } - } - VERIFY(0); /* should not reach here */ -} - -/* find the class with the minimum deadline among the eligible classes */ -static struct hfsc_class * -ellist_get_mindl(ellist_t *head, u_int64_t cur_time) -{ - struct hfsc_class *p, *cl = NULL; - - TAILQ_FOREACH(p, head, cl_ellist) { - if (p->cl_e > cur_time) - break; - if (cl == NULL || p->cl_d < cl->cl_d) - cl = p; - } - return (cl); -} - -/* - * active children list holds backlogged child classes being sorted - * by their virtual time. - * each intermediate class has one active children list. - */ - -static void -actlist_insert(struct hfsc_class *cl) -{ - struct hfsc_class *p; - - /* check the last entry first */ - if ((p = TAILQ_LAST(&cl->cl_parent->cl_actc, _active)) == NULL || - p->cl_vt <= cl->cl_vt) { - TAILQ_INSERT_TAIL(&cl->cl_parent->cl_actc, cl, cl_actlist); - return; - } - - TAILQ_FOREACH(p, &cl->cl_parent->cl_actc, cl_actlist) { - if (cl->cl_vt < p->cl_vt) { - TAILQ_INSERT_BEFORE(p, cl, cl_actlist); - return; - } - } - VERIFY(0); /* should not reach here */ -} - -static void -actlist_remove(struct hfsc_class *cl) -{ - TAILQ_REMOVE(&cl->cl_parent->cl_actc, cl, cl_actlist); -} - -static void -actlist_update(struct hfsc_class *cl) -{ - struct hfsc_class *p, *last; - - /* - * the virtual time of a class increases monotonically during its - * backlogged period. - * if the next entry has a larger virtual time, nothing to do. - */ - p = TAILQ_NEXT(cl, cl_actlist); - if (p == NULL || cl->cl_vt < p->cl_vt) - return; - - /* check the last entry */ - last = TAILQ_LAST(&cl->cl_parent->cl_actc, _active); - VERIFY(last != NULL); - if (last->cl_vt <= cl->cl_vt) { - TAILQ_REMOVE(&cl->cl_parent->cl_actc, cl, cl_actlist); - TAILQ_INSERT_TAIL(&cl->cl_parent->cl_actc, cl, cl_actlist); - return; - } - - /* - * the new position must be between the next entry - * and the last entry - */ - while ((p = TAILQ_NEXT(p, cl_actlist)) != NULL) { - if (cl->cl_vt < p->cl_vt) { - TAILQ_REMOVE(&cl->cl_parent->cl_actc, cl, cl_actlist); - TAILQ_INSERT_BEFORE(p, cl, cl_actlist); - return; - } - } - VERIFY(0); /* should not reach here */ -} - -static struct hfsc_class * -actlist_firstfit(struct hfsc_class *cl, u_int64_t cur_time) -{ - struct hfsc_class *p; - - TAILQ_FOREACH(p, &cl->cl_actc, cl_actlist) { - if (p->cl_f <= cur_time) - return (p); - } - return (NULL); -} - -/* - * service curve support functions - * - * external service curve parameters - * m: bits/sec - * d: msec - * internal service curve parameters - * sm: (bytes/tsc_interval) << SM_SHIFT - * ism: (tsc_count/byte) << ISM_SHIFT - * dx: tsc_count - * - * SM_SHIFT and ISM_SHIFT are scaled in order to keep effective digits. - * we should be able to handle 100K-1Gbps linkspeed with 200Hz-1GHz CPU - * speed. SM_SHIFT and ISM_SHIFT are selected to have at least 3 effective - * digits in decimal using the following table. - * - * bits/sec 100Kbps 1Mbps 10Mbps 100Mbps 1Gbps - * ----------+------------------------------------------------------- - * bytes/nsec 12.5e-6 125e-6 1250e-6 12500e-6 125000e-6 - * sm(500MHz) 25.0e-6 250e-6 2500e-6 25000e-6 250000e-6 - * sm(200MHz) 62.5e-6 625e-6 6250e-6 62500e-6 625000e-6 - * - * nsec/byte 80000 8000 800 80 8 - * ism(500MHz) 40000 4000 400 40 4 - * ism(200MHz) 16000 1600 160 16 1.6 - */ -#define SM_SHIFT 24 -#define ISM_SHIFT 10 - -#define SM_MASK ((1LL << SM_SHIFT) - 1) -#define ISM_MASK ((1LL << ISM_SHIFT) - 1) - -static inline u_int64_t -seg_x2y(u_int64_t x, u_int64_t sm) -{ - u_int64_t y; - - /* - * compute - * y = x * sm >> SM_SHIFT - * but divide it for the upper and lower bits to avoid overflow - */ - y = (x >> SM_SHIFT) * sm + (((x & SM_MASK) * sm) >> SM_SHIFT); - return (y); -} - -static inline u_int64_t -seg_y2x(u_int64_t y, u_int64_t ism) -{ - u_int64_t x; - - if (y == 0) - x = 0; - else if (ism == HT_INFINITY) - x = HT_INFINITY; - else { - x = (y >> ISM_SHIFT) * ism - + (((y & ISM_MASK) * ism) >> ISM_SHIFT); - } - return (x); -} - -static inline u_int64_t -m2sm(u_int64_t m) -{ - u_int64_t sm; - - sm = (m << SM_SHIFT) / 8 / machclk_freq; - return (sm); -} - -static inline u_int64_t -m2ism(u_int64_t m) -{ - u_int64_t ism; - - if (m == 0) - ism = HT_INFINITY; - else - ism = ((u_int64_t)machclk_freq << ISM_SHIFT) * 8 / m; - return (ism); -} - -static inline u_int64_t -d2dx(u_int64_t d) -{ - u_int64_t dx; - - dx = (d * machclk_freq) / 1000; - return (dx); -} - -static u_int64_t -sm2m(u_int64_t sm) -{ - u_int64_t m; - - m = (sm * 8 * machclk_freq) >> SM_SHIFT; - return (m); -} - -static u_int64_t -dx2d(u_int64_t dx) -{ - u_int64_t d; - - d = dx * 1000 / machclk_freq; - return (d); -} - -static boolean_t -sc2isc(struct hfsc_class *cl, struct service_curve *sc, struct internal_sc *isc, - u_int64_t eff_rate) -{ - struct hfsc_if *hif = cl->cl_hif; - struct internal_sc oisc = *isc; - u_int64_t m1, m2; - - if (eff_rate == 0 && (sc->fl & (HFSCF_M1_PCT | HFSCF_M2_PCT))) { - /* - * If service curve is configured with percentage and the - * effective uplink rate is not known, assume this is a - * transient case, and that the rate will be updated in - * the near future via CLASSQ_EV_LINK_SPEED. Pick a - * reasonable number for now, e.g. 10 Mbps. - */ - eff_rate = (10 * 1000 * 1000); - - log(LOG_WARNING, "%s: %s qid=%d slot=%d eff_rate unknown; " - "using temporary rate %llu bps\n", if_name(HFSCIF_IFP(hif)), - hfsc_style(hif), cl->cl_handle, cl->cl_id, eff_rate); - } - - m1 = sc->m1; - if (sc->fl & HFSCF_M1_PCT) { - VERIFY(m1 > 0 && m1 <= 100); - m1 = (eff_rate * m1) / 100; - } - - m2 = sc->m2; - if (sc->fl & HFSCF_M2_PCT) { - VERIFY(m2 > 0 && m2 <= 100); - m2 = (eff_rate * m2) / 100; - } - - isc->sm1 = m2sm(m1); - isc->ism1 = m2ism(m1); - isc->dx = d2dx(sc->d); - isc->dy = seg_x2y(isc->dx, isc->sm1); - isc->sm2 = m2sm(m2); - isc->ism2 = m2ism(m2); - - /* return non-zero if there's any change */ - return (bcmp(&oisc, isc, sizeof (*isc))); -} - -/* - * initialize the runtime service curve with the given internal - * service curve starting at (x, y). - */ -static void -rtsc_init(struct runtime_sc *rtsc, struct internal_sc *isc, u_int64_t x, - u_int64_t y) -{ - rtsc->x = x; - rtsc->y = y; - rtsc->sm1 = isc->sm1; - rtsc->ism1 = isc->ism1; - rtsc->dx = isc->dx; - rtsc->dy = isc->dy; - rtsc->sm2 = isc->sm2; - rtsc->ism2 = isc->ism2; -} - -/* - * calculate the y-projection of the runtime service curve by the - * given x-projection value - */ -static u_int64_t -rtsc_y2x(struct runtime_sc *rtsc, u_int64_t y) -{ - u_int64_t x; - - if (y < rtsc->y) - x = rtsc->x; - else if (y <= rtsc->y + rtsc->dy) { - /* x belongs to the 1st segment */ - if (rtsc->dy == 0) - x = rtsc->x + rtsc->dx; - else - x = rtsc->x + seg_y2x(y - rtsc->y, rtsc->ism1); - } else { - /* x belongs to the 2nd segment */ - x = rtsc->x + rtsc->dx - + seg_y2x(y - rtsc->y - rtsc->dy, rtsc->ism2); - } - return (x); -} - -static u_int64_t -rtsc_x2y(struct runtime_sc *rtsc, u_int64_t x) -{ - u_int64_t y; - - if (x <= rtsc->x) - y = rtsc->y; - else if (x <= rtsc->x + rtsc->dx) - /* y belongs to the 1st segment */ - y = rtsc->y + seg_x2y(x - rtsc->x, rtsc->sm1); - else - /* y belongs to the 2nd segment */ - y = rtsc->y + rtsc->dy - + seg_x2y(x - rtsc->x - rtsc->dx, rtsc->sm2); - return (y); -} - -/* - * update the runtime service curve by taking the minimum of the current - * runtime service curve and the service curve starting at (x, y). - */ -static void -rtsc_min(struct runtime_sc *rtsc, struct internal_sc *isc, u_int64_t x, - u_int64_t y) -{ - u_int64_t y1, y2, dx, dy; - - if (isc->sm1 <= isc->sm2) { - /* service curve is convex */ - y1 = rtsc_x2y(rtsc, x); - if (y1 < y) - /* the current rtsc is smaller */ - return; - rtsc->x = x; - rtsc->y = y; - return; - } - - /* - * service curve is concave - * compute the two y values of the current rtsc - * y1: at x - * y2: at (x + dx) - */ - y1 = rtsc_x2y(rtsc, x); - if (y1 <= y) { - /* rtsc is below isc, no change to rtsc */ - return; - } - - y2 = rtsc_x2y(rtsc, x + isc->dx); - if (y2 >= y + isc->dy) { - /* rtsc is above isc, replace rtsc by isc */ - rtsc->x = x; - rtsc->y = y; - rtsc->dx = isc->dx; - rtsc->dy = isc->dy; - return; - } - - /* - * the two curves intersect - * compute the offsets (dx, dy) using the reverse - * function of seg_x2y() - * seg_x2y(dx, sm1) == seg_x2y(dx, sm2) + (y1 - y) - */ - dx = ((y1 - y) << SM_SHIFT) / (isc->sm1 - isc->sm2); - /* - * check if (x, y1) belongs to the 1st segment of rtsc. - * if so, add the offset. - */ - if (rtsc->x + rtsc->dx > x) - dx += rtsc->x + rtsc->dx - x; - dy = seg_x2y(dx, isc->sm1); - - rtsc->x = x; - rtsc->y = y; - rtsc->dx = dx; - rtsc->dy = dy; -} - -int -hfsc_get_class_stats(struct hfsc_if *hif, u_int32_t qid, - struct hfsc_classstats *sp) -{ - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - if ((cl = hfsc_clh_to_clp(hif, qid)) == NULL) - return (EINVAL); - - sp->class_id = cl->cl_id; - sp->class_handle = cl->cl_handle; - - if (cl->cl_flags & HFCF_RSC) { - sp->rsc.m1 = sm2m(cl->cl_rsc.sm1); - sp->rsc.d = dx2d(cl->cl_rsc.dx); - sp->rsc.m2 = sm2m(cl->cl_rsc.sm2); - } else { - sp->rsc.m1 = 0; - sp->rsc.d = 0; - sp->rsc.m2 = 0; - } - if (cl->cl_flags & HFCF_FSC) { - sp->fsc.m1 = sm2m(cl->cl_fsc.sm1); - sp->fsc.d = dx2d(cl->cl_fsc.dx); - sp->fsc.m2 = sm2m(cl->cl_fsc.sm2); - } else { - sp->fsc.m1 = 0; - sp->fsc.d = 0; - sp->fsc.m2 = 0; - } - if (cl->cl_flags & HFCF_USC) { - sp->usc.m1 = sm2m(cl->cl_usc.sm1); - sp->usc.d = dx2d(cl->cl_usc.dx); - sp->usc.m2 = sm2m(cl->cl_usc.sm2); - } else { - sp->usc.m1 = 0; - sp->usc.d = 0; - sp->usc.m2 = 0; - } - - sp->total = cl->cl_total; - sp->cumul = cl->cl_cumul; - - sp->d = cl->cl_d; - sp->e = cl->cl_e; - sp->vt = cl->cl_vt; - sp->f = cl->cl_f; - - sp->initvt = cl->cl_initvt; - sp->vtperiod = cl->cl_vtperiod; - sp->parentperiod = cl->cl_parentperiod; - sp->nactive = cl->cl_nactive; - sp->vtoff = cl->cl_vtoff; - sp->cvtmax = cl->cl_cvtmax; - sp->myf = cl->cl_myf; - sp->cfmin = cl->cl_cfmin; - sp->cvtmin = cl->cl_cvtmin; - sp->myfadj = cl->cl_myfadj; - sp->vtadj = cl->cl_vtadj; - - sp->cur_time = read_machclk(); - sp->machclk_freq = machclk_freq; - - sp->qlength = qlen(&cl->cl_q); - sp->qlimit = qlimit(&cl->cl_q); - sp->xmit_cnt = cl->cl_stats.xmit_cnt; - sp->drop_cnt = cl->cl_stats.drop_cnt; - sp->period = cl->cl_stats.period; - - sp->qtype = qtype(&cl->cl_q); - sp->qstate = qstate(&cl->cl_q); -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_getstats(cl->cl_red, &sp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_getstats(cl->cl_rio, &sp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_getstats(cl->cl_blue, &sp->blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_getstats(cl->cl_sfb, &sp->sfb); - - return (0); -} - -/* convert a class handle to the corresponding class pointer */ -static struct hfsc_class * -hfsc_clh_to_clp(struct hfsc_if *hif, u_int32_t chandle) -{ - u_int32_t i; - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - /* - * first, try optimistically the slot matching the lower bits of - * the handle. if it fails, do the linear table search. - */ - i = chandle % hif->hif_maxclasses; - if ((cl = hif->hif_class_tbl[i]) != NULL && cl->cl_handle == chandle) - return (cl); - for (i = 0; i < hif->hif_maxclasses; i++) - if ((cl = hif->hif_class_tbl[i]) != NULL && - cl->cl_handle == chandle) - return (cl); - return (NULL); -} - -static const char * -hfsc_style(struct hfsc_if *hif) -{ - return ((hif->hif_flags & HFSCIFF_ALTQ) ? "ALTQ_HFSC" : "HFSC"); -} - -int -hfsc_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) -{ -#pragma unused(ifq, flags) - return (ENXIO); /* not yet */ -} - -int -hfsc_teardown_ifclassq(struct ifclassq *ifq) -{ - struct hfsc_if *hif = ifq->ifcq_disc; - int i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(hif != NULL && ifq->ifcq_type == PKTSCHEDT_HFSC); - - (void) hfsc_destroy_locked(hif); - - ifq->ifcq_disc = NULL; - for (i = 0; i < IFCQ_SC_MAX; i++) { - ifq->ifcq_disc_slots[i].qid = 0; - ifq->ifcq_disc_slots[i].cl = NULL; - } - - return (ifclassq_detach(ifq)); -} - -int -hfsc_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t slot, - struct if_ifclassq_stats *ifqs) -{ - struct hfsc_if *hif = ifq->ifcq_disc; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(ifq->ifcq_type == PKTSCHEDT_HFSC); - - if (slot >= IFCQ_SC_MAX) - return (EINVAL); - - return (hfsc_get_class_stats(hif, ifq->ifcq_disc_slots[slot].qid, - &ifqs->ifqs_hfsc_stats)); -} -#endif /* PKTSCHED_HFSC */ diff --git a/bsd/net/pktsched/pktsched_hfsc.h b/bsd/net/pktsched/pktsched_hfsc.h index d22b95380..7f14cdcb9 100644 --- a/bsd/net/pktsched/pktsched_hfsc.h +++ b/bsd/net/pktsched/pktsched_hfsc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -173,181 +173,6 @@ struct hfsc_classstats { classq_state_t qstate; }; -#ifdef BSD_KERNEL_PRIVATE -#include -/* - * kernel internal service curve representation - * coordinates are given by 64 bit unsigned integers. - * x-axis: unit is clock count. for the intel x86 architecture, - * the raw Pentium TSC (Timestamp Counter) value is used. - * virtual time is also calculated in this time scale. - * y-axis: unit is byte. - * - * the service curve parameters are converted to the internal - * representation. - * the slope values are scaled to avoid overflow. - * the inverse slope values as well as the y-projection of the 1st - * segment are kept in order to to avoid 64-bit divide operations - * that are expensive on 32-bit architectures. - * - * note: Intel Pentium TSC never wraps around in several thousands of years. - * x-axis doesn't wrap around for 1089 years with 1GHz clock. - * y-axis doesn't wrap around for 4358 years with 1Gbps bandwidth. - */ - -/* kernel internal representation of a service curve */ -struct internal_sc { - u_int64_t sm1; /* scaled slope of the 1st segment */ - u_int64_t ism1; /* scaled inverse-slope of the 1st segment */ - u_int64_t dx; /* the x-projection of the 1st segment */ - u_int64_t dy; /* the y-projection of the 1st segment */ - u_int64_t sm2; /* scaled slope of the 2nd segment */ - u_int64_t ism2; /* scaled inverse-slope of the 2nd segment */ -}; - -/* runtime service curve */ -struct runtime_sc { - u_int64_t x; /* current starting position on x-axis */ - u_int64_t y; /* current starting position on x-axis */ - u_int64_t sm1; /* scaled slope of the 1st segment */ - u_int64_t ism1; /* scaled inverse-slope of the 1st segment */ - u_int64_t dx; /* the x-projection of the 1st segment */ - u_int64_t dy; /* the y-projection of the 1st segment */ - u_int64_t sm2; /* scaled slope of the 2nd segment */ - u_int64_t ism2; /* scaled inverse-slope of the 2nd segment */ -}; - -/* for TAILQ based ellist and actlist implementation */ -struct hfsc_class; -typedef TAILQ_HEAD(_eligible, hfsc_class) ellist_t; -typedef TAILQ_ENTRY(hfsc_class) elentry_t; -typedef TAILQ_HEAD(_active, hfsc_class) actlist_t; -typedef TAILQ_ENTRY(hfsc_class) actentry_t; -#define ellist_first(s) TAILQ_FIRST(s) -#define actlist_first(s) TAILQ_FIRST(s) -#define actlist_last(s) TAILQ_LAST(s, _active) - -struct hfsc_class { - u_int32_t cl_id; /* class id (just for debug) */ - u_int32_t cl_handle; /* class handle */ - struct hfsc_if *cl_hif; /* back pointer to struct hfsc_if */ - u_int32_t cl_flags; /* misc flags */ - - struct hfsc_class *cl_parent; /* parent class */ - struct hfsc_class *cl_siblings; /* sibling classes */ - struct hfsc_class *cl_children; /* child classes */ - - class_queue_t cl_q; /* class queue structure */ - u_int32_t cl_qflags; /* class queue flags */ - union { - void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ - struct sfb *sfb; /* SFB state */ - } cl_qalg; - - u_int64_t cl_total; /* total work in bytes */ - u_int64_t cl_cumul; /* cumulative work in bytes */ - /* done by real-time criteria */ - u_int64_t cl_d; /* deadline */ - u_int64_t cl_e; /* eligible time */ - u_int64_t cl_vt; /* virtual time */ - u_int64_t cl_f; /* time when this class will fit for */ - /* link-sharing, max(myf, cfmin) */ - u_int64_t cl_myf; /* my fit-time (as calculated from */ - /* this class's own upperlimit */ - /* curve) */ - u_int64_t cl_myfadj; /* my fit-time adjustment */ - /* (to cancel history dependence) */ - u_int64_t cl_cfmin; /* earliest children's fit-time (used */ - /* with cl_myf to obtain cl_f) */ - u_int64_t cl_cvtmin; /* minimal virtual time among the */ - /* children fit for link-sharing */ - /* (monotonic within a period) */ - u_int64_t cl_vtadj; /* intra-period cumulative vt */ - /* adjustment */ - u_int64_t cl_vtoff; /* inter-period cumulative vt offset */ - u_int64_t cl_cvtmax; /* max child's vt in the last period */ - - u_int64_t cl_initvt; /* init virtual time (for debugging) */ - - struct service_curve cl_rsc0; /* external real-time service curve */ - struct service_curve cl_fsc0; /* external fair service curve */ - struct service_curve cl_usc0; /* external uppperlimit service curve */ - struct internal_sc cl_rsc; /* internal real-time service curve */ - struct internal_sc cl_fsc; /* internal fair service curve */ - struct internal_sc cl_usc; /* internal upperlimit service curve */ - struct runtime_sc cl_deadline; /* deadline curve */ - struct runtime_sc cl_eligible; /* eligible curve */ - struct runtime_sc cl_virtual; /* virtual curve */ - struct runtime_sc cl_ulimit; /* upperlimit curve */ - - u_int32_t cl_vtperiod; /* vt period sequence no */ - u_int32_t cl_parentperiod; /* parent's vt period seqno */ - u_int32_t cl_nactive; /* number of active children */ - actlist_t cl_actc; /* active children list */ - - actentry_t cl_actlist; /* active children list entry */ - elentry_t cl_ellist; /* eligible list entry */ - - struct { - struct pktcntr xmit_cnt; - struct pktcntr drop_cnt; - u_int32_t period; - } cl_stats; -}; - -#define cl_red cl_qalg.red -#define cl_rio cl_qalg.rio -#define cl_blue cl_qalg.blue -#define cl_sfb cl_qalg.sfb - -/* hfsc_if flags */ -#define HFSCIFF_ALTQ 0x1 /* configured via PF/ALTQ */ - -/* - * hfsc interface state - */ -struct hfsc_if { - struct ifclassq *hif_ifq; /* backpointer to ifclassq */ - struct hfsc_class *hif_rootclass; /* root class */ - struct hfsc_class *hif_defaultclass; /* default class */ - struct hfsc_class **hif_class_tbl; - struct hfsc_class *hif_pollcache; /* cache for poll operation */ - - u_int32_t hif_flags; /* flags */ - u_int32_t hif_maxclasses; /* max # of classes in table */ - u_int32_t hif_classes; /* # of classes in the tree */ - u_int32_t hif_packets; /* # of packets in the tree */ - u_int32_t hif_classid; /* class id sequence number */ - u_int64_t hif_eff_rate; /* last known effective rate */ - - ellist_t hif_eligible; /* eligible list */ -}; - -#define HFSCIF_IFP(_hif) ((_hif)->hif_ifq->ifcq_ifp) - -extern void hfsc_init(void); -extern struct hfsc_if *hfsc_alloc(struct ifnet *, int, boolean_t); -extern int hfsc_destroy(struct hfsc_if *); -extern void hfsc_purge(struct hfsc_if *); -extern void hfsc_event(struct hfsc_if *, cqev_t); -extern int hfsc_add_queue(struct hfsc_if *, struct service_curve *, - struct service_curve *, struct service_curve *, u_int32_t, int, - u_int32_t, u_int32_t, struct hfsc_class **); -extern int hfsc_remove_queue(struct hfsc_if *, u_int32_t); -extern int hfsc_get_class_stats(struct hfsc_if *, u_int32_t, - struct hfsc_classstats *); -extern int hfsc_enqueue(struct hfsc_if *, struct hfsc_class *, - struct mbuf *, struct pf_mtag *); -extern struct mbuf *hfsc_dequeue(struct hfsc_if *, cqdq_op_t); -extern int hfsc_setup_ifclassq(struct ifclassq *, u_int32_t); -extern int hfsc_teardown_ifclassq(struct ifclassq *); -extern int hfsc_getqstats_ifclassq(struct ifclassq *, u_int32_t, - struct if_ifclassq_stats *); -#endif /* BSD_KERNEL_PRIVATE */ - #ifdef __cplusplus } #endif diff --git a/bsd/net/pktsched/pktsched_priq.c b/bsd/net/pktsched/pktsched_priq.c deleted file mode 100644 index ffbf5cf28..000000000 --- a/bsd/net/pktsched/pktsched_priq.c +++ /dev/null @@ -1,1309 +0,0 @@ -/* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_priq.c,v 1.21 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_priq.c,v 1.1 2000/10/18 09:15:23 kjc Exp $ */ - -/* - * Copyright (C) 2000-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * priority queue - */ - -#if PKTSCHED_PRIQ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#include -#include - -/* - * function prototypes - */ -static int priq_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *priq_dequeue_ifclassq(struct ifclassq *, cqdq_op_t); -static int priq_request_ifclassq(struct ifclassq *, cqrq_t, void *); -static int priq_clear_interface(struct priq_if *); -static struct priq_class *priq_class_create(struct priq_if *, int, u_int32_t, - int, u_int32_t); -static int priq_class_destroy(struct priq_if *, struct priq_class *); -static int priq_destroy_locked(struct priq_if *); -static inline int priq_addq(struct priq_class *, struct mbuf *, - struct pf_mtag *); -static inline struct mbuf *priq_getq(struct priq_class *); -static inline struct mbuf *priq_pollq(struct priq_class *); -static void priq_purgeq(struct priq_if *, struct priq_class *, u_int32_t, - u_int32_t *, u_int32_t *); -static void priq_purge_sc(struct priq_if *, cqrq_purge_sc_t *); -static void priq_updateq(struct priq_if *, struct priq_class *, cqev_t); -static int priq_throttle(struct priq_if *, cqrq_throttle_t *); -static int priq_resumeq(struct priq_if *, struct priq_class *); -static int priq_suspendq(struct priq_if *, struct priq_class *); -static int priq_stat_sc(struct priq_if *, cqrq_stat_sc_t *); -static inline struct priq_class *priq_clh_to_clp(struct priq_if *, u_int32_t); -static const char *priq_style(struct priq_if *); - -#define PRIQ_ZONE_MAX 32 /* maximum elements in zone */ -#define PRIQ_ZONE_NAME "pktsched_priq" /* zone name */ - -static unsigned int priq_size; /* size of zone element */ -static struct zone *priq_zone; /* zone for priq */ - -#define PRIQ_CL_ZONE_MAX 32 /* maximum elements in zone */ -#define PRIQ_CL_ZONE_NAME "pktsched_priq_cl" /* zone name */ - -static unsigned int priq_cl_size; /* size of zone element */ -static struct zone *priq_cl_zone; /* zone for priq_class */ - -void -priq_init(void) -{ - priq_size = sizeof (struct priq_if); - priq_zone = zinit(priq_size, PRIQ_ZONE_MAX * priq_size, - 0, PRIQ_ZONE_NAME); - if (priq_zone == NULL) { - panic("%s: failed allocating %s", __func__, PRIQ_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(priq_zone, Z_EXPAND, TRUE); - zone_change(priq_zone, Z_CALLERACCT, TRUE); - - priq_cl_size = sizeof (struct priq_class); - priq_cl_zone = zinit(priq_cl_size, PRIQ_CL_ZONE_MAX * priq_cl_size, - 0, PRIQ_CL_ZONE_NAME); - if (priq_cl_zone == NULL) { - panic("%s: failed allocating %s", __func__, PRIQ_CL_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(priq_cl_zone, Z_EXPAND, TRUE); - zone_change(priq_cl_zone, Z_CALLERACCT, TRUE); -} - -struct priq_if * -priq_alloc(struct ifnet *ifp, int how, boolean_t altq) -{ - struct priq_if *pif; - - pif = (how == M_WAITOK) ? zalloc(priq_zone) : zalloc_noblock(priq_zone); - if (pif == NULL) - return (NULL); - - bzero(pif, priq_size); - pif->pif_maxpri = -1; - pif->pif_ifq = &ifp->if_snd; - if (altq) - pif->pif_flags |= PRIQIFF_ALTQ; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler allocated\n", - if_name(ifp), priq_style(pif)); - } - - return (pif); -} - -int -priq_destroy(struct priq_if *pif) -{ - struct ifclassq *ifq = pif->pif_ifq; - int err; - - IFCQ_LOCK(ifq); - err = priq_destroy_locked(pif); - IFCQ_UNLOCK(ifq); - - return (err); -} - -static int -priq_destroy_locked(struct priq_if *pif) -{ - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - (void) priq_clear_interface(pif); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler destroyed\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif)); - } - - zfree(priq_zone, pif); - - return (0); -} - -/* - * bring the interface back to the initial state by discarding - * all the filters and classes. - */ -static int -priq_clear_interface(struct priq_if *pif) -{ - struct priq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - /* clear out the classes */ - for (pri = 0; pri <= pif->pif_maxpri; pri++) - if ((cl = pif->pif_classes[pri]) != NULL) - priq_class_destroy(pif, cl); - - return (0); -} - -/* discard all the queued packets on the interface */ -void -priq_purge(struct priq_if *pif) -{ - struct priq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - for (pri = 0; pri <= pif->pif_maxpri; pri++) { - if ((cl = pif->pif_classes[pri]) != NULL && !qempty(&cl->cl_q)) - priq_purgeq(pif, cl, 0, NULL, NULL); - } -#if !PF_ALTQ - /* - * This assertion is safe to be made only when PF_ALTQ is not - * configured; otherwise, IFCQ_LEN represents the sum of the - * packets managed by ifcq_disc and altq_disc instances, which - * is possible when transitioning between the two. - */ - VERIFY(IFCQ_LEN(pif->pif_ifq) == 0); -#endif /* !PF_ALTQ */ -} - -static void -priq_purge_sc(struct priq_if *pif, cqrq_purge_sc_t *pr) -{ - struct ifclassq *ifq = pif->pif_ifq; - u_int32_t i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - VERIFY(pr->sc == MBUF_SC_UNSPEC || MBUF_VALID_SC(pr->sc)); - VERIFY(pr->flow != 0); - - if (pr->sc != MBUF_SC_UNSPEC) { - i = MBUF_SCIDX(pr->sc); - VERIFY(i < IFCQ_SC_MAX); - - priq_purgeq(pif, ifq->ifcq_disc_slots[i].cl, - pr->flow, &pr->packets, &pr->bytes); - } else { - u_int32_t cnt, len; - - pr->packets = 0; - pr->bytes = 0; - - for (i = 0; i < IFCQ_SC_MAX; i++) { - priq_purgeq(pif, ifq->ifcq_disc_slots[i].cl, - pr->flow, &cnt, &len); - pr->packets += cnt; - pr->bytes += len; - } - } -} - -void -priq_event(struct priq_if *pif, cqev_t ev) -{ - struct priq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - for (pri = 0; pri <= pif->pif_maxpri; pri++) - if ((cl = pif->pif_classes[pri]) != NULL) - priq_updateq(pif, cl, ev); -} - -int -priq_add_queue(struct priq_if *pif, int priority, u_int32_t qlimit, - int flags, u_int32_t qid, struct priq_class **clp) -{ - struct priq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - /* check parameters */ - if (priority >= PRIQ_MAXPRI) - return (EINVAL); - if (pif->pif_classes[priority] != NULL) - return (EBUSY); - if (priq_clh_to_clp(pif, qid) != NULL) - return (EBUSY); - - cl = priq_class_create(pif, priority, qlimit, flags, qid); - if (cl == NULL) - return (ENOMEM); - - if (clp != NULL) - *clp = cl; - - return (0); -} - -static struct priq_class * -priq_class_create(struct priq_if *pif, int pri, u_int32_t qlimit, - int flags, u_int32_t qid) -{ - struct ifnet *ifp; - struct ifclassq *ifq; - struct priq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - /* Sanitize flags unless internally configured */ - if (pif->pif_flags & PRIQIFF_ALTQ) - flags &= PRCF_USERFLAGS; - -#if !CLASSQ_RED - if (flags & PRCF_RED) { - log(LOG_ERR, "%s: %s RED not available!\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif)); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & PRCF_RIO) { - log(LOG_ERR, "%s: %s RIO not available!\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif)); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & PRCF_BLUE) { - log(LOG_ERR, "%s: %s BLUE not available!\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif)); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) && - (flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) != PRCF_RED && - (flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) != PRCF_RIO && - (flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) != PRCF_BLUE && - (flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) != PRCF_SFB) { - log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif)); - return (NULL); - } - - ifq = pif->pif_ifq; - ifp = PRIQIF_IFP(pif); - - if ((cl = pif->pif_classes[pri]) != NULL) { - /* modify the class instead of creating a new one */ - if (!qempty(&cl->cl_q)) - priq_purgeq(pif, cl, 0, NULL, NULL); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - qtype(&cl->cl_q) = Q_DROPTAIL; - qstate(&cl->cl_q) = QS_RUNNING; - } else { - cl = zalloc(priq_cl_zone); - if (cl == NULL) - return (NULL); - - bzero(cl, priq_cl_size); - } - - pif->pif_classes[pri] = cl; - if (flags & PRCF_DEFAULTCLASS) - pif->pif_default = cl; - if (qlimit == 0 || qlimit > IFCQ_MAXLEN(ifq)) { - qlimit = IFCQ_MAXLEN(ifq); - if (qlimit == 0) - qlimit = DEFAULT_QLIMIT; /* use default */ - } - _qinit(&cl->cl_q, Q_DROPTAIL, qlimit); - cl->cl_flags = flags; - cl->cl_pri = pri; - if (pri > pif->pif_maxpri) - pif->pif_maxpri = pri; - cl->cl_pif = pif; - cl->cl_handle = qid; - - if (flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) { -#if CLASSQ_RED || CLASSQ_RIO - u_int64_t ifbandwidth = ifnet_output_linkrate(ifp); - int pkttime; -#endif /* CLASSQ_RED || CLASSQ_RIO */ - - cl->cl_qflags = 0; - if (flags & PRCF_ECN) { - if (flags & PRCF_BLUE) - cl->cl_qflags |= BLUEF_ECN; - else if (flags & PRCF_SFB) - cl->cl_qflags |= SFBF_ECN; - else if (flags & PRCF_RED) - cl->cl_qflags |= REDF_ECN; - else if (flags & PRCF_RIO) - cl->cl_qflags |= RIOF_ECN; - } - if (flags & PRCF_FLOWCTL) { - if (flags & PRCF_SFB) - cl->cl_qflags |= SFBF_FLOWCTL; - } - if (flags & PRCF_CLEARDSCP) { - if (flags & PRCF_RIO) - cl->cl_qflags |= RIOF_CLEARDSCP; - } -#if CLASSQ_RED || CLASSQ_RIO - /* - * XXX: RED & RIO should be watching link speed and MTU - * events and recompute pkttime accordingly. - */ - if (ifbandwidth < 8) - pkttime = 1000 * 1000 * 1000; /* 1 sec */ - else - pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 / - (ifbandwidth / 8); - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RED - if (flags & PRCF_RED) { - cl->cl_red = red_alloc(ifp, 0, 0, - qlimit(&cl->cl_q) * 10/100, - qlimit(&cl->cl_q) * 30/100, - cl->cl_qflags, pkttime); - if (cl->cl_red != NULL) - qtype(&cl->cl_q) = Q_RED; - } -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (flags & PRCF_RIO) { - cl->cl_rio = - rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime); - if (cl->cl_rio != NULL) - qtype(&cl->cl_q) = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#endif /* CLASSQ_RED || CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & PRCF_BLUE) { - cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags); - if (cl->cl_blue != NULL) - qtype(&cl->cl_q) = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & PRCF_SFB) { - if (!(cl->cl_flags & PRCF_LAZY)) - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb != NULL || (cl->cl_flags & PRCF_LAZY)) - qtype(&cl->cl_q) = Q_SFB; - } - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s created qid=%d pri=%d qlimit=%d " - "flags=%b\n", if_name(ifp), priq_style(pif), - cl->cl_handle, cl->cl_pri, qlimit, flags, PRCF_BITS); - } - - return (cl); -} - -int -priq_remove_queue(struct priq_if *pif, u_int32_t qid) -{ - struct priq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - if ((cl = priq_clh_to_clp(pif, qid)) == NULL) - return (EINVAL); - - return (priq_class_destroy(pif, cl)); -} - -static int -priq_class_destroy(struct priq_if *pif, struct priq_class *cl) -{ - struct ifclassq *ifq = pif->pif_ifq; - int pri; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (!qempty(&cl->cl_q)) - priq_purgeq(pif, cl, 0, NULL, NULL); - - VERIFY(cl->cl_pri < PRIQ_MAXPRI); - VERIFY(!pktsched_bit_tst(cl->cl_pri, &pif->pif_bitmap)); - - pif->pif_classes[cl->cl_pri] = NULL; - if (pif->pif_maxpri == cl->cl_pri) { - for (pri = cl->cl_pri; pri >= 0; pri--) - if (pif->pif_classes[pri] != NULL) { - pif->pif_maxpri = pri; - break; - } - if (pri < 0) - pif->pif_maxpri = -1; - } - - if (pif->pif_default == cl) - pif->pif_default = NULL; - - if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - qtype(&cl->cl_q) = Q_DROPTAIL; - qstate(&cl->cl_q) = QS_RUNNING; - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s destroyed qid=%d pri=%d\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif), - cl->cl_handle, cl->cl_pri); - } - - zfree(priq_cl_zone, cl); - - return (0); -} - -int -priq_enqueue(struct priq_if *pif, struct priq_class *cl, struct mbuf *m, - struct pf_mtag *t) -{ - struct ifclassq *ifq = pif->pif_ifq; - u_int32_t pri; - int len, ret; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(cl == NULL || cl->cl_pif == pif); - - if (cl == NULL) { -#if PF_ALTQ - cl = priq_clh_to_clp(pif, t->pftag_qid); -#else /* !PF_ALTQ */ - cl = priq_clh_to_clp(pif, 0); -#endif /* !PF_ALTQ */ - if (cl == NULL) { - cl = pif->pif_default; - if (cl == NULL) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - } - } - pri = cl->cl_pri; - VERIFY(pri < PRIQ_MAXPRI); - - len = m_pktlen(m); - - ret = priq_addq(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - /* packet has been freed in priq_addq */ - PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT REACHED */ - } - } - IFCQ_INC_LEN(ifq); - IFCQ_INC_BYTES(ifq, len); - - /* class is now active; indicate it as such */ - if (!pktsched_bit_tst(pri, &pif->pif_bitmap)) - pktsched_bit_set(pri, &pif->pif_bitmap); - - /* successfully queued. */ - return (ret); -} - -/* - * note: CLASSQDQ_POLL returns the next packet without removing the packet - * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation. - * CLASSQDQ_REMOVE must return the same packet if called immediately - * after CLASSQDQ_POLL. - */ -struct mbuf * -priq_dequeue(struct priq_if *pif, cqdq_op_t op) -{ - struct ifclassq *ifq = pif->pif_ifq; - struct priq_class *cl; - struct mbuf *m; - u_int32_t pri, len; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (pif->pif_bitmap == 0) { - /* no active class; nothing to dequeue */ - return (NULL); - } - VERIFY(!IFCQ_IS_EMPTY(ifq)); - - pri = pktsched_fls(pif->pif_bitmap) - 1; /* zero based */ - VERIFY(pri < PRIQ_MAXPRI); - cl = pif->pif_classes[pri]; - VERIFY(cl != NULL && !qempty(&cl->cl_q)); - - if (op == CLASSQDQ_POLL) - return (priq_pollq(cl)); - - m = priq_getq(cl); - VERIFY(m != NULL); /* qalg must be work conserving */ - len = m_pktlen(m); - - IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, len); - if (qempty(&cl->cl_q)) { - cl->cl_period++; - /* class is now inactive; indicate it as such */ - pktsched_bit_clr(pri, &pif->pif_bitmap); - } - PKTCNTR_ADD(&cl->cl_xmitcnt, 1, len); - IFCQ_XMIT_ADD(ifq, 1, len); - - return (m); -} - -static inline int -priq_addq(struct priq_class *cl, struct mbuf *m, struct pf_mtag *t) -{ - struct priq_if *pif = cl->cl_pif; - struct ifclassq *ifq = pif->pif_ifq; - - IFCQ_LOCK_ASSERT_HELD(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_addq(cl->cl_rio, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_addq(cl->cl_red, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_addq(cl->cl_blue, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q)) { - if (cl->cl_sfb == NULL) { - struct ifnet *ifp = PRIQIF_IFP(pif); - - VERIFY(cl->cl_flags & PRCF_LAZY); - cl->cl_flags &= ~PRCF_LAZY; - IFCQ_CONVERT_LOCK(ifq); - - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb == NULL) { - /* fall back to droptail */ - qtype(&cl->cl_q) = Q_DROPTAIL; - cl->cl_flags &= ~PRCF_SFB; - cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL); - - log(LOG_ERR, "%s: %s SFB lazy allocation " - "failed for qid=%d pri=%d, falling back " - "to DROPTAIL\n", if_name(ifp), - priq_style(pif), cl->cl_handle, - cl->cl_pri); - } else if (pif->pif_throttle != IFNET_THROTTLE_OFF) { - /* if there's pending throttling, set it */ - cqrq_throttle_t tr = { 1, pif->pif_throttle }; - int err = priq_throttle(pif, &tr); - - if (err == EALREADY) - err = 0; - if (err != 0) { - tr.level = IFNET_THROTTLE_OFF; - (void) priq_throttle(pif, &tr); - } - } - } - if (cl->cl_sfb != NULL) - return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t)); - } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - -#if PF_ECN - if (cl->cl_flags & PRCF_CLEARDSCP) - write_dsfield(m, t, 0); -#endif /* PF_ECN */ - - _addq(&cl->cl_q, m); - - return (0); -} - -static inline struct mbuf * -priq_getq(struct priq_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_pif->pif_ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_getq(cl->cl_rio, &cl->cl_q)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_getq(cl->cl_red, &cl->cl_q)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_getq(cl->cl_blue, &cl->cl_q)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_getq(cl->cl_sfb, &cl->cl_q)); - - return (_getq(&cl->cl_q)); -} - -static inline struct mbuf * -priq_pollq(struct priq_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_pif->pif_ifq); - - return (qhead(&cl->cl_q)); -} - -static void -priq_purgeq(struct priq_if *pif, struct priq_class *cl, u_int32_t flow, - u_int32_t *packets, u_int32_t *bytes) -{ - struct ifclassq *ifq = pif->pif_ifq; - u_int32_t cnt = 0, len = 0, qlen; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if ((qlen = qlen(&cl->cl_q)) == 0) { - VERIFY(!pktsched_bit_tst(cl->cl_pri, &pif->pif_bitmap)); - goto done; - } - - /* become regular mutex before freeing mbufs */ - IFCQ_CONVERT_LOCK(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len); - else - _flushq_flow(&cl->cl_q, flow, &cnt, &len); - - if (cnt > 0) { - VERIFY(qlen(&cl->cl_q) == (qlen - cnt)); - - PKTCNTR_ADD(&cl->cl_dropcnt, cnt, len); - IFCQ_DROP_ADD(ifq, cnt, len); - - VERIFY(((signed)IFCQ_LEN(ifq) - cnt) >= 0); - IFCQ_LEN(ifq) -= cnt; - - if (qempty(&cl->cl_q)) - pktsched_bit_clr(cl->cl_pri, &pif->pif_bitmap); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s purge qid=%d pri=%d " - "qlen=[%d,%d] cnt=%d len=%d flow=0x%x\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif), - cl->cl_handle, cl->cl_pri, qlen, qlen(&cl->cl_q), - cnt, len, flow); - } - } -done: - if (packets != NULL) - *packets = cnt; - if (bytes != NULL) - *bytes = len; -} - -static void -priq_updateq(struct priq_if *pif, struct priq_class *cl, cqev_t ev) -{ - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s update qid=%d pri=%d event=%s\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif), - cl->cl_handle, cl->cl_pri, ifclassq_ev2str(ev)); - } - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_updateq(cl->cl_rio, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_updateq(cl->cl_red, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_updateq(cl->cl_blue, ev)); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_updateq(cl->cl_sfb, ev)); -} - -int -priq_get_class_stats(struct priq_if *pif, u_int32_t qid, - struct priq_classstats *sp) -{ - struct priq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - if ((cl = priq_clh_to_clp(pif, qid)) == NULL) - return (EINVAL); - - sp->class_handle = cl->cl_handle; - sp->priority = cl->cl_pri; - sp->qlength = qlen(&cl->cl_q); - sp->qlimit = qlimit(&cl->cl_q); - sp->period = cl->cl_period; - sp->xmitcnt = cl->cl_xmitcnt; - sp->dropcnt = cl->cl_dropcnt; - - sp->qtype = qtype(&cl->cl_q); - sp->qstate = qstate(&cl->cl_q); -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_getstats(cl->cl_red, &sp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_getstats(cl->cl_rio, &sp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_getstats(cl->cl_blue, &sp->blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_getstats(cl->cl_sfb, &sp->sfb); - - return (0); -} - -static int -priq_stat_sc(struct priq_if *pif, cqrq_stat_sc_t *sr) -{ - struct ifclassq *ifq = pif->pif_ifq; - struct priq_class *cl; - u_int32_t i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - VERIFY(sr->sc == MBUF_SC_UNSPEC || MBUF_VALID_SC(sr->sc)); - - i = MBUF_SCIDX(sr->sc); - VERIFY(i < IFCQ_SC_MAX); - - cl = ifq->ifcq_disc_slots[i].cl; - sr->packets = qlen(&cl->cl_q); - sr->bytes = qsize(&cl->cl_q); - - return (0); -} - -/* convert a class handle to the corresponding class pointer */ -static inline struct priq_class * -priq_clh_to_clp(struct priq_if *pif, u_int32_t chandle) -{ - struct priq_class *cl; - int idx; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - for (idx = pif->pif_maxpri; idx >= 0; idx--) - if ((cl = pif->pif_classes[idx]) != NULL && - cl->cl_handle == chandle) - return (cl); - - return (NULL); -} - -static const char * -priq_style(struct priq_if *pif) -{ - return ((pif->pif_flags & PRIQIFF_ALTQ) ? "ALTQ_PRIQ" : "PRIQ"); -} - -/* - * priq_enqueue_ifclassq is an enqueue function to be registered to - * (*ifcq_enqueue) in struct ifclassq. - */ -static int -priq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) -{ - u_int32_t i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - log(LOG_ERR, "%s: packet does not have pkthdr\n", - if_name(ifq->ifcq_ifp)); - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - - i = MBUF_SCIDX(mbuf_get_service_class(m)); - VERIFY((u_int32_t)i < IFCQ_SC_MAX); - - return (priq_enqueue(ifq->ifcq_disc, - ifq->ifcq_disc_slots[i].cl, m, m_pftag(m))); -} - -/* - * priq_dequeue_ifclassq is a dequeue function to be registered to - * (*ifcq_dequeue) in struct ifclass. - * - * note: CLASSQDQ_POLL returns the next packet without removing the packet - * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation. - * CLASSQDQ_REMOVE must return the same packet if called immediately - * after CLASSQDQ_POLL. - */ -static struct mbuf * -priq_dequeue_ifclassq(struct ifclassq *ifq, cqdq_op_t op) -{ - return (priq_dequeue(ifq->ifcq_disc, op)); -} - -static int -priq_request_ifclassq(struct ifclassq *ifq, cqrq_t req, void *arg) -{ - struct priq_if *pif = (struct priq_if *)ifq->ifcq_disc; - int err = 0; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - switch (req) { - case CLASSQRQ_PURGE: - priq_purge(pif); - break; - - case CLASSQRQ_PURGE_SC: - priq_purge_sc(pif, (cqrq_purge_sc_t *)arg); - break; - - case CLASSQRQ_EVENT: - priq_event(pif, (cqev_t)arg); - break; - - case CLASSQRQ_THROTTLE: - err = priq_throttle(pif, (cqrq_throttle_t *)arg); - break; - - case CLASSQRQ_STAT_SC: - err = priq_stat_sc(pif, (cqrq_stat_sc_t *)arg); - break; - } - return (err); -} - -int -priq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) -{ - struct ifnet *ifp = ifq->ifcq_ifp; - struct priq_class *cl0, *cl1, *cl2, *cl3, *cl4; - struct priq_class *cl5, *cl6, *cl7, *cl8, *cl9; - struct priq_if *pif; - u_int32_t maxlen = 0, qflags = 0; - int err = 0; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(ifq->ifcq_disc == NULL); - VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); - - if (flags & PKTSCHEDF_QALG_RED) - qflags |= PRCF_RED; - if (flags & PKTSCHEDF_QALG_RIO) - qflags |= PRCF_RIO; - if (flags & PKTSCHEDF_QALG_BLUE) - qflags |= PRCF_BLUE; - if (flags & PKTSCHEDF_QALG_SFB) - qflags |= PRCF_SFB; - if (flags & PKTSCHEDF_QALG_ECN) - qflags |= PRCF_ECN; - if (flags & PKTSCHEDF_QALG_FLOWCTL) - qflags |= PRCF_FLOWCTL; - - pif = priq_alloc(ifp, M_WAITOK, FALSE); - if (pif == NULL) - return (ENOMEM); - - if ((maxlen = IFCQ_MAXLEN(ifq)) == 0) - maxlen = if_sndq_maxlen; - - if ((err = priq_add_queue(pif, 0, maxlen, - qflags | PRCF_LAZY, SCIDX_BK_SYS, &cl0)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 1, maxlen, - qflags | PRCF_LAZY, SCIDX_BK, &cl1)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 2, maxlen, - qflags | PRCF_DEFAULTCLASS, SCIDX_BE, &cl2)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 3, maxlen, - qflags | PRCF_LAZY, SCIDX_RD, &cl3)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 4, maxlen, - qflags | PRCF_LAZY, SCIDX_OAM, &cl4)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 5, maxlen, - qflags | PRCF_LAZY, SCIDX_AV, &cl5)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 6, maxlen, - qflags | PRCF_LAZY, SCIDX_RV, &cl6)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 7, maxlen, - qflags | PRCF_LAZY, SCIDX_VI, &cl7)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 8, maxlen, - qflags | PRCF_LAZY, SCIDX_VO, &cl8)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 9, maxlen, - qflags, SCIDX_CTL, &cl9)) != 0) - goto cleanup; - - err = ifclassq_attach(ifq, PKTSCHEDT_PRIQ, pif, - priq_enqueue_ifclassq, priq_dequeue_ifclassq, NULL, - NULL, priq_request_ifclassq); - - /* cache these for faster lookup */ - if (err == 0) { - ifq->ifcq_disc_slots[SCIDX_BK_SYS].qid = SCIDX_BK_SYS; - ifq->ifcq_disc_slots[SCIDX_BK_SYS].cl = cl0; - - ifq->ifcq_disc_slots[SCIDX_BK].qid = SCIDX_BK; - ifq->ifcq_disc_slots[SCIDX_BK].cl = cl1; - - ifq->ifcq_disc_slots[SCIDX_BE].qid = SCIDX_BE; - ifq->ifcq_disc_slots[SCIDX_BE].cl = cl2; - - ifq->ifcq_disc_slots[SCIDX_RD].qid = SCIDX_RD; - ifq->ifcq_disc_slots[SCIDX_RD].cl = cl3; - - ifq->ifcq_disc_slots[SCIDX_OAM].qid = SCIDX_OAM; - ifq->ifcq_disc_slots[SCIDX_OAM].cl = cl4; - - ifq->ifcq_disc_slots[SCIDX_AV].qid = SCIDX_AV; - ifq->ifcq_disc_slots[SCIDX_AV].cl = cl5; - - ifq->ifcq_disc_slots[SCIDX_RV].qid = SCIDX_RV; - ifq->ifcq_disc_slots[SCIDX_RV].cl = cl6; - - ifq->ifcq_disc_slots[SCIDX_VI].qid = SCIDX_VI; - ifq->ifcq_disc_slots[SCIDX_VI].cl = cl7; - - ifq->ifcq_disc_slots[SCIDX_VO].qid = SCIDX_VO; - ifq->ifcq_disc_slots[SCIDX_VO].cl = cl8; - - ifq->ifcq_disc_slots[SCIDX_CTL].qid = SCIDX_CTL; - ifq->ifcq_disc_slots[SCIDX_CTL].cl = cl9; - } - -cleanup: - if (err != 0) - (void) priq_destroy_locked(pif); - - return (err); -} - -int -priq_teardown_ifclassq(struct ifclassq *ifq) -{ - struct priq_if *pif = ifq->ifcq_disc; - int i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(pif != NULL && ifq->ifcq_type == PKTSCHEDT_PRIQ); - - (void) priq_destroy_locked(pif); - - ifq->ifcq_disc = NULL; - for (i = 0; i < IFCQ_SC_MAX; i++) { - ifq->ifcq_disc_slots[i].qid = 0; - ifq->ifcq_disc_slots[i].cl = NULL; - } - - return (ifclassq_detach(ifq)); -} - -int -priq_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t slot, - struct if_ifclassq_stats *ifqs) -{ - struct priq_if *pif = ifq->ifcq_disc; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(ifq->ifcq_type == PKTSCHEDT_PRIQ); - - if (slot >= IFCQ_SC_MAX) - return (EINVAL); - - return (priq_get_class_stats(pif, ifq->ifcq_disc_slots[slot].qid, - &ifqs->ifqs_priq_stats)); -} - -static int -priq_throttle(struct priq_if *pif, cqrq_throttle_t *tr) -{ - struct ifclassq *ifq = pif->pif_ifq; - struct priq_class *cl; - int err = 0; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(!(pif->pif_flags & PRIQIFF_ALTQ)); - - if (!tr->set) { - tr->level = pif->pif_throttle; - return (0); - } - - if (tr->level == pif->pif_throttle) - return (EALREADY); - - /* Current throttling levels only involve BK_SYS class */ - cl = ifq->ifcq_disc_slots[SCIDX_BK_SYS].cl; - - switch (tr->level) { - case IFNET_THROTTLE_OFF: - err = priq_resumeq(pif, cl); - break; - - case IFNET_THROTTLE_OPPORTUNISTIC: - err = priq_suspendq(pif, cl); - break; - - default: - VERIFY(0); - /* NOTREACHED */ - } - - if (err == 0 || err == ENXIO) { - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s throttling level %sset %d->%d\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif), - (err == 0) ? "" : "lazy ", pif->pif_throttle, - tr->level); - } - pif->pif_throttle = tr->level; - if (err != 0) - err = 0; - else - priq_purgeq(pif, cl, 0, NULL, NULL); - } else { - log(LOG_ERR, "%s: %s unable to set throttling level " - "%d->%d [error=%d]\n", if_name(PRIQIF_IFP(pif)), - priq_style(pif), pif->pif_throttle, tr->level, err); - } - - return (err); -} - -static int -priq_resumeq(struct priq_if *pif, struct priq_class *cl) -{ - struct ifclassq *ifq = pif->pif_ifq; - int err = 0; - - IFCQ_LOCK_ASSERT_HELD(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, FALSE); - - if (err == 0) - qstate(&cl->cl_q) = QS_RUNNING; - - return (err); -} - -static int -priq_suspendq(struct priq_if *pif, struct priq_class *cl) -{ - struct ifclassq *ifq = pif->pif_ifq; - int err = 0; - - IFCQ_LOCK_ASSERT_HELD(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q)) { - if (cl->cl_sfb != NULL) { - err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, TRUE); - } else { - VERIFY(cl->cl_flags & PRCF_LAZY); - err = ENXIO; /* delayed throttling */ - } - } - - if (err == 0 || err == ENXIO) - qstate(&cl->cl_q) = QS_SUSPENDED; - - return (err); -} -#endif /* PKTSCHED_PRIQ */ diff --git a/bsd/net/pktsched/pktsched_priq.h b/bsd/net/pktsched/pktsched_priq.h index 4dc9b74bc..858cf9ef3 100644 --- a/bsd/net/pktsched/pktsched_priq.h +++ b/bsd/net/pktsched/pktsched_priq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -118,73 +118,6 @@ struct priq_classstats { classq_state_t qstate; }; -#ifdef BSD_KERNEL_PRIVATE -struct priq_class { - u_int32_t cl_handle; /* class handle */ - class_queue_t cl_q; /* class queue structure */ - u_int32_t cl_qflags; /* class queue flags */ - union { - void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ - struct sfb *sfb; /* SFB state */ - } cl_qalg; - int32_t cl_pri; /* priority */ - u_int32_t cl_flags; /* class flags */ - struct priq_if *cl_pif; /* back pointer to pif */ - - /* statistics */ - u_int32_t cl_period; /* backlog period */ - struct pktcntr cl_xmitcnt; /* transmitted packet counter */ - struct pktcntr cl_dropcnt; /* dropped packet counter */ -}; - -#define cl_red cl_qalg.red -#define cl_rio cl_qalg.rio -#define cl_blue cl_qalg.blue -#define cl_sfb cl_qalg.sfb - -/* priq_if flags */ -#define PRIQIFF_ALTQ 0x1 /* configured via PF/ALTQ */ - -/* - * priq interface state - */ -struct priq_if { - struct ifclassq *pif_ifq; /* backpointer to ifclassq */ - int pif_maxpri; /* max priority in use */ - u_int32_t pif_flags; /* flags */ - u_int32_t pif_throttle; /* throttling level */ - pktsched_bitmap_t pif_bitmap; /* active class bitmap */ - struct priq_class *pif_default; /* default class */ - struct priq_class *pif_classes[PRIQ_MAXPRI]; /* classes */ -}; - -#define PRIQIF_IFP(_pif) ((_pif)->pif_ifq->ifcq_ifp) - -struct if_ifclassq_stats; - -extern void priq_init(void); -extern struct priq_if *priq_alloc(struct ifnet *, int, boolean_t); -extern int priq_destroy(struct priq_if *); -extern void priq_purge(struct priq_if *); -extern void priq_event(struct priq_if *, cqev_t); -extern int priq_add_queue(struct priq_if *, int, u_int32_t, int, u_int32_t, - struct priq_class **); -extern int priq_remove_queue(struct priq_if *, u_int32_t); -extern int priq_get_class_stats(struct priq_if *, u_int32_t, - struct priq_classstats *); -extern int priq_enqueue(struct priq_if *, struct priq_class *, struct mbuf *, - struct pf_mtag *); -extern struct mbuf *priq_dequeue(struct priq_if *, cqdq_op_t); -extern int priq_setup_ifclassq(struct ifclassq *, u_int32_t); -extern int priq_teardown_ifclassq(struct ifclassq *ifq); -extern int priq_getqstats_ifclassq(struct ifclassq *, u_int32_t, - struct if_ifclassq_stats *); -extern int priq_set_throttle(struct ifclassq *, u_int32_t); -extern int priq_get_throttle(struct ifclassq *, u_int32_t *); -#endif /* BSD_KERNEL_PRIVATE */ #ifdef __cplusplus } #endif diff --git a/bsd/net/pktsched/pktsched_qfq.c b/bsd/net/pktsched/pktsched_qfq.c index b1a88d435..862ee8711 100644 --- a/bsd/net/pktsched/pktsched_qfq.c +++ b/bsd/net/pktsched/pktsched_qfq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -78,20 +78,22 @@ #include #include + /* * function prototypes */ -static int qfq_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *qfq_dequeue_ifclassq(struct ifclassq *, cqdq_op_t); +static int qfq_enqueue_ifclassq(struct ifclassq *, void *, classq_pkt_type_t, + boolean_t *); +static void *qfq_dequeue_ifclassq(struct ifclassq *, classq_pkt_type_t *); static int qfq_request_ifclassq(struct ifclassq *, cqrq_t, void *); static int qfq_clear_interface(struct qfq_if *); static struct qfq_class *qfq_class_create(struct qfq_if *, u_int32_t, - u_int32_t, u_int32_t, u_int32_t, u_int32_t); + u_int32_t, u_int32_t, u_int32_t, u_int32_t, classq_pkt_type_t); static int qfq_class_destroy(struct qfq_if *, struct qfq_class *); static int qfq_destroy_locked(struct qfq_if *); -static inline int qfq_addq(struct qfq_class *, struct mbuf *, struct pf_mtag *); -static inline struct mbuf *qfq_getq(struct qfq_class *); -static inline struct mbuf *qfq_pollq(struct qfq_class *); +static inline int qfq_addq(struct qfq_class *, pktsched_pkt_t *, + struct pf_mtag *); +static inline void qfq_getq(struct qfq_class *, pktsched_pkt_t *); static void qfq_purgeq(struct qfq_if *, struct qfq_class *, u_int32_t, u_int32_t *, u_int32_t *); static void qfq_purge_sc(struct qfq_if *, cqrq_purge_sc_t *); @@ -179,7 +181,7 @@ qfq_init(void) } struct qfq_if * -qfq_alloc(struct ifnet *ifp, int how, boolean_t altq) +qfq_alloc(struct ifnet *ifp, int how) { struct qfq_if *qif; @@ -189,22 +191,17 @@ qfq_alloc(struct ifnet *ifp, int how, boolean_t altq) bzero(qif, qfq_size); qif->qif_ifq = &ifp->if_snd; - if (altq) { - qif->qif_maxclasses = QFQ_MAX_CLASSES; - qif->qif_maxslots = QFQ_MAX_SLOTS; - qif->qif_flags |= QFQIFF_ALTQ; - } else { - qif->qif_maxclasses = IFCQ_SC_MAX; - /* - * TODO: adi@apple.com - * - * Ideally I would like to have the following - * but QFQ needs further modifications. - * - * qif->qif_maxslots = IFCQ_SC_MAX; - */ - qif->qif_maxslots = QFQ_MAX_SLOTS; - } + + qif->qif_maxclasses = IFCQ_SC_MAX; + /* + * TODO: adi@apple.com + * + * Ideally I would like to have the following + * but QFQ needs further modifications. + * + * qif->qif_maxslots = IFCQ_SC_MAX; + */ + qif->qif_maxslots = QFQ_MAX_SLOTS; if ((qif->qif_class_tbl = _MALLOC(sizeof (struct qfq_class *) * qif->qif_maxclasses, M_DEVBUF, M_WAITOK|M_ZERO)) == NULL) { @@ -325,15 +322,7 @@ qfq_purge(struct qfq_if *qif) if ((cl = qif->qif_class_tbl[i]) != NULL) qfq_purgeq(qif, cl, 0, NULL, NULL); } -#if !PF_ALTQ - /* - * This assertion is safe to be made only when PF_ALTQ is not - * configured; otherwise, IFCQ_LEN represents the sum of the - * packets managed by ifcq_disc and altq_disc instances, which - * is possible when transitioning between the two. - */ VERIFY(IFCQ_LEN(qif->qif_ifq) == 0); -#endif /* !PF_ALTQ */ } static void @@ -383,7 +372,8 @@ qfq_event(struct qfq_if *qif, cqev_t ev) int qfq_add_queue(struct qfq_if *qif, u_int32_t qlimit, u_int32_t weight, - u_int32_t maxsz, u_int32_t flags, u_int32_t qid, struct qfq_class **clp) + u_int32_t maxsz, u_int32_t flags, u_int32_t qid, struct qfq_class **clp, + classq_pkt_type_t ptype) { struct qfq_class *cl; u_int32_t w; @@ -404,7 +394,7 @@ qfq_add_queue(struct qfq_if *qif, u_int32_t qlimit, u_int32_t weight, if (maxsz == 0 || maxsz > (1 << QFQ_MTU_SHIFT)) return (EINVAL); - cl = qfq_class_create(qif, weight, qlimit, flags, maxsz, qid); + cl = qfq_class_create(qif, weight, qlimit, flags, maxsz, qid, ptype); if (cl == NULL) return (ENOMEM); @@ -416,7 +406,7 @@ qfq_add_queue(struct qfq_if *qif, u_int32_t qlimit, u_int32_t weight, static struct qfq_class * qfq_class_create(struct qfq_if *qif, u_int32_t weight, u_int32_t qlimit, - u_int32_t flags, u_int32_t maxsz, u_int32_t qid) + u_int32_t flags, u_int32_t maxsz, u_int32_t qid, classq_pkt_type_t ptype) { struct ifnet *ifp; struct ifclassq *ifq; @@ -427,10 +417,6 @@ qfq_class_create(struct qfq_if *qif, u_int32_t weight, u_int32_t qlimit, IFCQ_LOCK_ASSERT_HELD(qif->qif_ifq); - /* Sanitize flags unless internally configured */ - if (qif->qif_flags & QFQIFF_ALTQ) - flags &= QFCF_USERFLAGS; - if (qif->qif_classes >= qif->qif_maxclasses) { log(LOG_ERR, "%s: %s out of classes! (max %d)\n", if_name(QFQIF_IFP(qif)), qfq_style(qif), @@ -438,41 +424,6 @@ qfq_class_create(struct qfq_if *qif, u_int32_t weight, u_int32_t qlimit, return (NULL); } -#if !CLASSQ_RED - if (flags & QFCF_RED) { - log(LOG_ERR, "%s: %s RED not available!\n", - if_name(QFQIF_IFP(qif)), qfq_style(qif)); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & QFCF_RIO) { - log(LOG_ERR, "%s: %s RIO not available!\n", - if_name(QFQIF_IFP(qif)), qfq_style(qif)); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & QFCF_BLUE) { - log(LOG_ERR, "%s: %s BLUE not available!\n", - if_name(QFQIF_IFP(qif)), qfq_style(qif)); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) && - (flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) != QFCF_RED && - (flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) != QFCF_RIO && - (flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) != QFCF_BLUE && - (flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) != QFCF_SFB) { - log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n", - if_name(QFQIF_IFP(qif)), qfq_style(qif)); - return (NULL); - } - ifq = qif->qif_ifq; ifp = QFQIF_IFP(qif); @@ -487,7 +438,7 @@ qfq_class_create(struct qfq_if *qif, u_int32_t weight, u_int32_t qlimit, if (qlimit == 0) qlimit = DEFAULT_QLIMIT; /* use default */ } - _qinit(&cl->cl_q, Q_DROPTAIL, qlimit); + _qinit(&cl->cl_q, Q_DROPTAIL, qlimit, ptype); cl->cl_qif = qif; cl->cl_flags = flags; cl->cl_handle = qid; @@ -561,80 +512,22 @@ qfq_class_create(struct qfq_if *qif, u_int32_t weight, u_int32_t qlimit, if (flags & QFCF_DEFAULTCLASS) qif->qif_default = cl; - if (flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) { -#if CLASSQ_RED || CLASSQ_RIO - u_int64_t ifbandwidth = ifnet_output_linkrate(ifp); - int pkttime; -#endif /* CLASSQ_RED || CLASSQ_RIO */ - + if (flags & QFCF_SFB) { cl->cl_qflags = 0; if (flags & QFCF_ECN) { - if (flags & QFCF_BLUE) - cl->cl_qflags |= BLUEF_ECN; - else if (flags & QFCF_SFB) - cl->cl_qflags |= SFBF_ECN; - else if (flags & QFCF_RED) - cl->cl_qflags |= REDF_ECN; - else if (flags & QFCF_RIO) - cl->cl_qflags |= RIOF_ECN; + cl->cl_qflags |= SFBF_ECN; } if (flags & QFCF_FLOWCTL) { - if (flags & QFCF_SFB) - cl->cl_qflags |= SFBF_FLOWCTL; + cl->cl_qflags |= SFBF_FLOWCTL; } if (flags & QFCF_DELAYBASED) { - if (flags & QFCF_SFB) - cl->cl_qflags |= SFBF_DELAYBASED; - } - if (flags & QFCF_CLEARDSCP) { - if (flags & QFCF_RIO) - cl->cl_qflags |= RIOF_CLEARDSCP; - } -#if CLASSQ_RED || CLASSQ_RIO - /* - * XXX: RED & RIO should be watching link speed and MTU - * events and recompute pkttime accordingly. - */ - if (ifbandwidth < 8) - pkttime = 1000 * 1000 * 1000; /* 1 sec */ - else - pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 / - (ifbandwidth / 8); - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RED - if (flags & QFCF_RED) { - cl->cl_red = red_alloc(ifp, 0, 0, - qlimit(&cl->cl_q) * 10/100, - qlimit(&cl->cl_q) * 30/100, - cl->cl_qflags, pkttime); - if (cl->cl_red != NULL) - qtype(&cl->cl_q) = Q_RED; - } -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (flags & QFCF_RIO) { - cl->cl_rio = - rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime); - if (cl->cl_rio != NULL) - qtype(&cl->cl_q) = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#endif /* CLASSQ_RED || CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & QFCF_BLUE) { - cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags); - if (cl->cl_blue != NULL) - qtype(&cl->cl_q) = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & QFCF_SFB) { - if (!(cl->cl_flags & QFCF_LAZY)) - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb != NULL || (cl->cl_flags & QFCF_LAZY)) - qtype(&cl->cl_q) = Q_SFB; + cl->cl_qflags |= SFBF_DELAYBASED; } + if (!(cl->cl_flags & QFCF_LAZY)) + cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, + qlimit(&cl->cl_q), cl->cl_qflags); + if (cl->cl_sfb != NULL || (cl->cl_flags & QFCF_LAZY)) + qtype(&cl->cl_q) = Q_SFB; } if (pktsched_verbose) { @@ -665,6 +558,9 @@ qfq_class_destroy(struct qfq_if *qif, struct qfq_class *cl) { struct ifclassq *ifq = qif->qif_ifq; int i; +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); @@ -684,18 +580,6 @@ qfq_class_destroy(struct qfq_if *qif, struct qfq_class *cl) qif->qif_classes--; if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_destroy(cl->cl_sfb); cl->cl_qalg.ptr = NULL; @@ -911,7 +795,7 @@ qfq_update_class(struct qfq_if *qif, struct qfq_group *grp, u_int32_t len; u_int64_t roundedS; - len = m_pktlen(qhead(&cl->cl_q)); + len = m_pktlen((struct mbuf *)qhead(&cl->cl_q)); cl->cl_F = cl->cl_S + (u_int64_t)len * cl->cl_inv_w; roundedS = qfq_round_down(cl->cl_S, grp->qfg_slot_shift); if (roundedS == grp->qfg_S) @@ -929,19 +813,20 @@ qfq_update_class(struct qfq_if *qif, struct qfq_group *grp, * CLASSQDQ_REMOVE must return the same packet if called immediately * after CLASSQDQ_POLL. */ -struct mbuf * -qfq_dequeue(struct qfq_if *qif, cqdq_op_t op) +void +qfq_dequeue(struct qfq_if *qif, pktsched_pkt_t *pkt) { pktsched_bitmap_t er_bits = qif->qif_bitmaps[ER]; struct ifclassq *ifq = qif->qif_ifq; struct qfq_group *grp; struct qfq_class *cl; - struct mbuf *m; u_int64_t old_V; u_int32_t len; IFCQ_LOCK_ASSERT_HELD(ifq); + pkt->pktsched_pkt = NULL; + for (;;) { if (er_bits == 0) { #if QFQ_DEBUG @@ -949,7 +834,7 @@ qfq_dequeue(struct qfq_if *qif, cqdq_op_t op) qfq_dump_sched(qif, "start dequeue"); #endif /* QFQ_DEBUG */ /* no eligible and ready packet */ - return (NULL); + return; } grp = qfq_ffs(qif, er_bits); /* if group is non-empty, use it */ @@ -965,12 +850,9 @@ qfq_dequeue(struct qfq_if *qif, cqdq_op_t op) cl = grp->qfg_slots[grp->qfg_front]; VERIFY(cl != NULL && !qempty(&cl->cl_q)); - if (op == CLASSQDQ_POLL) - return (qfq_pollq(cl)); - - m = qfq_getq(cl); - VERIFY(m != NULL); /* qalg must be work conserving */ - len = m_pktlen(m); + qfq_getq(cl, pkt); + VERIFY(pkt->pktsched_pkt != NULL); /* qalg must be work conserving */ + len = pktsched_get_pkt_len(pkt); #if QFQ_DEBUG qif->qif_queued--; @@ -987,9 +869,10 @@ qfq_dequeue(struct qfq_if *qif, cqdq_op_t op) qif->qif_V += (u_int64_t)len * QFQ_IWSUM; if (pktsched_verbose > 2) { - log(LOG_DEBUG, "%s: %s qid=%d dequeue m=0x%llx F=0x%llx " + log(LOG_DEBUG, "%s: %s qid=%d dequeue pkt=0x%llx F=0x%llx " "V=0x%llx", if_name(QFQIF_IFP(qif)), qfq_style(qif), - cl->cl_handle, (uint64_t)VM_KERNEL_ADDRPERM(m), cl->cl_F, + cl->cl_handle, + (uint64_t)VM_KERNEL_ADDRPERM(pkt->pktsched_pkt), cl->cl_F, qif->qif_V); } @@ -1026,8 +909,6 @@ skip_unblock: if (!qif->qif_bitmaps[ER] && qif->qif_queued && pktsched_verbose > 1) qfq_dump_sched(qif, "end dequeue"); #endif /* QFQ_DEBUG */ - - return (m); } /* @@ -1070,7 +951,7 @@ qfq_update_start(struct qfq_if *qif, struct qfq_class *cl) } int -qfq_enqueue(struct qfq_if *qif, struct qfq_class *cl, struct mbuf *m, +qfq_enqueue(struct qfq_if *qif, struct qfq_class *cl, pktsched_pkt_t *pkt, struct pf_mtag *t) { struct ifclassq *ifq = qif->qif_ifq; @@ -1082,45 +963,27 @@ qfq_enqueue(struct qfq_if *qif, struct qfq_class *cl, struct mbuf *m, VERIFY(cl == NULL || cl->cl_qif == qif); if (cl == NULL) { -#if PF_ALTQ - cl = qfq_clh_to_clp(qif, t->pftag_qid); -#else /* !PF_ALTQ */ cl = qfq_clh_to_clp(qif, 0); -#endif /* !PF_ALTQ */ if (cl == NULL) { cl = qif->qif_default; if (cl == NULL) { IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); + return (CLASSQEQ_DROP); } } } - len = m_pktlen(m); - - ret = qfq_addq(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - /* packet has been freed in qfq_addq */ - PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT REACHED */ - } + VERIFY(pkt->pktsched_ptype == qptype(&cl->cl_q)); + len = pktsched_get_pkt_len(pkt); + + ret = qfq_addq(cl, pkt, t); + if ((ret != 0) && (ret != CLASSQEQ_SUCCESS_FC)) { + VERIFY(ret == CLASSQEQ_DROP || + ret == CLASSQEQ_DROP_FC || + ret == CLASSQEQ_DROP_SP); + PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); + IFCQ_DROP_ADD(ifq, 1, len); + return (ret); } IFCQ_INC_LEN(ifq); IFCQ_INC_BYTES(ifq, len); @@ -1174,7 +1037,8 @@ qfq_enqueue(struct qfq_if *qif, struct qfq_class *cl, struct mbuf *m, log(LOG_DEBUG, "%s: %s qid=%d enqueue m=0x%llx state=%s 0x%x " "S=0x%llx F=0x%llx V=0x%llx\n", if_name(QFQIF_IFP(qif)), qfq_style(qif), cl->cl_handle, - (uint64_t)VM_KERNEL_ADDRPERM(m), qfq_state2str(s), + (uint64_t)VM_KERNEL_ADDRPERM(pkt->pktsched_pkt), + qfq_state2str(s), qif->qif_bitmaps[s], cl->cl_S, cl->cl_F, qif->qif_V); } @@ -1311,36 +1175,21 @@ qfq_state2str(int s) } static inline int -qfq_addq(struct qfq_class *cl, struct mbuf *m, struct pf_mtag *t) +qfq_addq(struct qfq_class *cl, pktsched_pkt_t *pkt, struct pf_mtag *t) { struct qfq_if *qif = cl->cl_qif; struct ifclassq *ifq = qif->qif_ifq; IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_addq(cl->cl_rio, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_addq(cl->cl_red, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_addq(cl->cl_blue, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q)) { if (cl->cl_sfb == NULL) { struct ifnet *ifp = QFQIF_IFP(qif); VERIFY(cl->cl_flags & QFCF_LAZY); cl->cl_flags &= ~QFCF_LAZY; - IFCQ_CONVERT_LOCK(ifq); + IFCQ_CONVERT_LOCK(ifq); cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, qlimit(&cl->cl_q), cl->cl_qflags); if (cl->cl_sfb == NULL) { @@ -1368,55 +1217,34 @@ qfq_addq(struct qfq_class *cl, struct mbuf *m, struct pf_mtag *t) } } if (cl->cl_sfb != NULL) - return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t)); + return (sfb_addq(cl->cl_sfb, &cl->cl_q, pkt, t)); } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) { IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); + return (CLASSQEQ_DROP); } #if PF_ECN - if (cl->cl_flags & QFCF_CLEARDSCP) + if (cl->cl_flags & QFCF_CLEARDSCP) { + /* not supported for non-mbuf type packets */ + VERIFY(pkt->pktsched_ptype == QP_MBUF); write_dsfield(m, t, 0); + } #endif /* PF_ECN */ - _addq(&cl->cl_q, m); - + VERIFY(pkt->pktsched_ptype == qptype(&cl->cl_q)); + _addq(&cl->cl_q, pkt->pktsched_pkt); return (0); } -static inline struct mbuf * -qfq_getq(struct qfq_class *cl) +static inline void +qfq_getq(struct qfq_class *cl, pktsched_pkt_t *pkt) { IFCQ_LOCK_ASSERT_HELD(cl->cl_qif->qif_ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_getq(cl->cl_rio, &cl->cl_q)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_getq(cl->cl_red, &cl->cl_q)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_getq(cl->cl_blue, &cl->cl_q)); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_getq(cl->cl_sfb, &cl->cl_q)); - - return (_getq(&cl->cl_q)); -} - -static inline struct mbuf * -qfq_pollq(struct qfq_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_qif->qif_ifq); + return (sfb_getq(cl->cl_sfb, &cl->cl_q, pkt)); - return (qhead(&cl->cl_q)); + return (pktsched_pkt_encap(pkt, qptype(&cl->cl_q), _getq(&cl->cl_q))); } static void @@ -1431,24 +1259,7 @@ qfq_purgeq(struct qfq_if *qif, struct qfq_class *cl, u_int32_t flow, if ((qlen = qlen(&cl->cl_q)) == 0) goto done; - /* become regular mutex before freeing mbufs */ IFCQ_CONVERT_LOCK(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len); else @@ -1498,18 +1309,6 @@ qfq_updateq(struct qfq_if *qif, struct qfq_class *cl, cqev_t ev) ifclassq_ev2str(ev)); } -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_updateq(cl->cl_rio, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_updateq(cl->cl_red, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_updateq(cl->cl_blue, ev)); -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) return (sfb_updateq(cl->cl_sfb, ev)); } @@ -1537,18 +1336,7 @@ qfq_get_class_stats(struct qfq_if *qif, u_int32_t qid, sp->qtype = qtype(&cl->cl_q); sp->qstate = qstate(&cl->cl_q); -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_getstats(cl->cl_red, &sp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_getstats(cl->cl_rio, &sp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_getstats(cl->cl_blue, &sp->blue); -#endif /* CLASSQ_BLUE */ + if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_getstats(cl->cl_sfb, &sp->sfb); @@ -1603,7 +1391,8 @@ qfq_clh_to_clp(struct qfq_if *qif, u_int32_t chandle) static const char * qfq_style(struct qfq_if *qif) { - return ((qif->qif_flags & QFQIFF_ALTQ) ? "ALTQ_QFQ" : "QFQ"); +#pragma unused(qif) + return ("QFQ"); } /* @@ -1726,26 +1515,73 @@ qfq_dump_sched(struct qfq_if *qif, const char *msg) * (*ifcq_enqueue) in struct ifclassq. */ static int -qfq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) +qfq_enqueue_ifclassq(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype, + boolean_t *pdrop) { - u_int32_t i; + u_int32_t i = 0; + int ret; + pktsched_pkt_t pkt; + struct pf_mtag *t = NULL; IFCQ_LOCK_ASSERT_HELD(ifq); - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - log(LOG_ERR, "%s: packet does not have pkthdr\n", - if_name(ifq->ifcq_ifp)); - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); + switch (ptype) { + case QP_MBUF: { + struct mbuf *m = p; + if (!(m->m_flags & M_PKTHDR)) { + /* should not happen */ + log(LOG_ERR, "%s: packet does not have pkthdr\n", + if_name(ifq->ifcq_ifp)); + IFCQ_CONVERT_LOCK(ifq); + m_freem(m); + *pdrop = TRUE; + return (ENOBUFS); + } + i = MBUF_SCIDX(mbuf_get_service_class(m)); + t = m_pftag(m); + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ } - i = MBUF_SCIDX(mbuf_get_service_class(m)); VERIFY((u_int32_t)i < IFCQ_SC_MAX); - return (qfq_enqueue(ifq->ifcq_disc, - ifq->ifcq_disc_slots[i].cl, m, m_pftag(m))); + pktsched_pkt_encap(&pkt, ptype, p); + + ret = qfq_enqueue(ifq->ifcq_disc, + ifq->ifcq_disc_slots[i].cl, &pkt, t); + + if ((ret != 0) && (ret != CLASSQEQ_SUCCESS_FC)) { + pktsched_free_pkt(&pkt); + *pdrop = TRUE; + } else { + *pdrop = FALSE; + } + + switch (ret) { + case CLASSQEQ_DROP: + ret = ENOBUFS; + break; + case CLASSQEQ_DROP_FC: + ret = EQFULL; + break; + case CLASSQEQ_DROP_SP: + ret = EQSUSPENDED; + break; + case CLASSQEQ_SUCCESS_FC: + ret = EQFULL; + break; + case CLASSQEQ_SUCCESS: + ret = 0; + break; + default: + VERIFY(0); + } + return (ret); } /* @@ -1757,10 +1593,14 @@ qfq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) * CLASSQDQ_REMOVE must return the same packet if called immediately * after CLASSQDQ_POLL. */ -static struct mbuf * -qfq_dequeue_ifclassq(struct ifclassq *ifq, cqdq_op_t op) +static void * +qfq_dequeue_ifclassq(struct ifclassq *ifq, classq_pkt_type_t *ptype) { - return (qfq_dequeue(ifq->ifcq_disc, op)); + pktsched_pkt_t pkt; + bzero(&pkt, sizeof (pkt)); + qfq_dequeue(ifq->ifcq_disc, &pkt); + *ptype = pkt.pktsched_ptype; + return (pkt.pktsched_pkt); } static int @@ -1795,7 +1635,8 @@ qfq_request_ifclassq(struct ifclassq *ifq, cqrq_t req, void *arg) } int -qfq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) +qfq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags, + classq_pkt_type_t ptype) { struct ifnet *ifp = ifq->ifcq_ifp; struct qfq_class *cl0, *cl1, *cl2, *cl3, *cl4; @@ -1808,12 +1649,6 @@ qfq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) VERIFY(ifq->ifcq_disc == NULL); VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); - if (flags & PKTSCHEDF_QALG_RED) - qflags |= QFCF_RED; - if (flags & PKTSCHEDF_QALG_RIO) - qflags |= QFCF_RIO; - if (flags & PKTSCHEDF_QALG_BLUE) - qflags |= QFCF_BLUE; if (flags & PKTSCHEDF_QALG_SFB) qflags |= QFCF_SFB; if (flags & PKTSCHEDF_QALG_ECN) @@ -1823,7 +1658,7 @@ qfq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) if (flags & PKTSCHEDF_QALG_DELAYBASED) qflags |= QFCF_DELAYBASED; - qif = qfq_alloc(ifp, M_WAITOK, FALSE); + qif = qfq_alloc(ifp, M_WAITOK); if (qif == NULL) return (ENOMEM); @@ -1831,48 +1666,48 @@ qfq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) maxlen = if_sndq_maxlen; if ((err = qfq_add_queue(qif, maxlen, 300, 1200, - qflags | QFCF_LAZY, SCIDX_BK_SYS, &cl0)) != 0) + qflags | QFCF_LAZY, SCIDX_BK_SYS, &cl0, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 600, 1400, - qflags | QFCF_LAZY, SCIDX_BK, &cl1)) != 0) + qflags | QFCF_LAZY, SCIDX_BK, &cl1, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 2400, 600, - qflags | QFCF_DEFAULTCLASS, SCIDX_BE, &cl2)) != 0) + qflags | QFCF_DEFAULTCLASS, SCIDX_BE, &cl2, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 2700, 600, - qflags | QFCF_LAZY, SCIDX_RD, &cl3)) != 0) + qflags | QFCF_LAZY, SCIDX_RD, &cl3, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 3000, 400, - qflags | QFCF_LAZY, SCIDX_OAM, &cl4)) != 0) + qflags | QFCF_LAZY, SCIDX_OAM, &cl4, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 8000, 1000, - qflags | QFCF_LAZY, SCIDX_AV, &cl5)) != 0) + qflags | QFCF_LAZY, SCIDX_AV, &cl5, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 15000, 1200, - qflags | QFCF_LAZY, SCIDX_RV, &cl6)) != 0) + qflags | QFCF_LAZY, SCIDX_RV, &cl6, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 20000, 1400, - qflags | QFCF_LAZY, SCIDX_VI, &cl7)) != 0) + qflags | QFCF_LAZY, SCIDX_VI, &cl7, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 23000, 200, - qflags | QFCF_LAZY, SCIDX_VO, &cl8)) != 0) + qflags | QFCF_LAZY, SCIDX_VO, &cl8, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 25000, 200, - qflags, SCIDX_CTL, &cl9)) != 0) + qflags, SCIDX_CTL, &cl9, ptype)) != 0) goto cleanup; err = ifclassq_attach(ifq, PKTSCHEDT_QFQ, qif, qfq_enqueue_ifclassq, qfq_dequeue_ifclassq, NULL, - NULL, qfq_request_ifclassq); + NULL, NULL, qfq_request_ifclassq); /* cache these for faster lookup */ if (err == 0) { @@ -1958,7 +1793,6 @@ qfq_throttle(struct qfq_if *qif, cqrq_throttle_t *tr) int err = 0; IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(!(qif->qif_flags & QFQIFF_ALTQ)); if (!tr->set) { tr->level = qif->qif_throttle; @@ -2011,24 +1845,11 @@ qfq_resumeq(struct qfq_if *qif, struct qfq_class *cl) { struct ifclassq *ifq = qif->qif_ifq; int err = 0; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, FALSE); @@ -2043,24 +1864,11 @@ qfq_suspendq(struct qfq_if *qif, struct qfq_class *cl) { struct ifclassq *ifq = qif->qif_ifq; int err = 0; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q)) { if (cl->cl_sfb != NULL) { err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, TRUE); diff --git a/bsd/net/pktsched/pktsched_qfq.h b/bsd/net/pktsched/pktsched_qfq.h index ca3a2c4c3..15ce5a323 100644 --- a/bsd/net/pktsched/pktsched_qfq.h +++ b/bsd/net/pktsched/pktsched_qfq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -204,9 +204,6 @@ struct qfq_class { u_int32_t cl_qflags; /* class queue flags */ union { void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ struct sfb *sfb; /* SFB state */ } cl_qalg; struct qfq_if *cl_qif; /* back pointer to qif */ @@ -229,9 +226,6 @@ struct qfq_class { struct pktcntr cl_dropcnt; /* dropped packet counter */ }; -#define cl_red cl_qalg.red -#define cl_rio cl_qalg.rio -#define cl_blue cl_qalg.blue #define cl_sfb cl_qalg.sfb /* @@ -249,15 +243,11 @@ struct qfq_group { struct qfq_class **qfg_slots; }; -/* qfq_if flags */ -#define QFQIFF_ALTQ 0x1 /* configured via PF/ALTQ */ - /* * qfq interface state */ struct qfq_if { struct ifclassq *qif_ifq; /* backpointer to ifclassq */ - u_int32_t qif_flags; /* flags */ u_int32_t qif_throttle; /* throttling level */ u_int8_t qif_classes; /* # of classes in table */ u_int8_t qif_maxclasses; /* max # of classes in table */ @@ -281,19 +271,19 @@ struct qfq_if { struct if_ifclassq_stats; extern void qfq_init(void); -extern struct qfq_if *qfq_alloc(struct ifnet *, int, boolean_t); +extern struct qfq_if *qfq_alloc(struct ifnet *, int); extern int qfq_destroy(struct qfq_if *); extern void qfq_purge(struct qfq_if *); extern void qfq_event(struct qfq_if *, cqev_t); extern int qfq_add_queue(struct qfq_if *, u_int32_t, u_int32_t, u_int32_t, - u_int32_t, u_int32_t, struct qfq_class **); + u_int32_t, u_int32_t, struct qfq_class **, classq_pkt_type_t); extern int qfq_remove_queue(struct qfq_if *, u_int32_t); extern int qfq_get_class_stats(struct qfq_if *, u_int32_t, struct qfq_classstats *); -extern int qfq_enqueue(struct qfq_if *, struct qfq_class *, struct mbuf *, +extern int qfq_enqueue(struct qfq_if *, struct qfq_class *, pktsched_pkt_t *, struct pf_mtag *); -extern struct mbuf *qfq_dequeue(struct qfq_if *, cqdq_op_t); -extern int qfq_setup_ifclassq(struct ifclassq *, u_int32_t); +extern void qfq_dequeue(struct qfq_if *, pktsched_pkt_t *); +extern int qfq_setup_ifclassq(struct ifclassq *, u_int32_t, classq_pkt_type_t); extern int qfq_teardown_ifclassq(struct ifclassq *ifq); extern int qfq_getqstats_ifclassq(struct ifclassq *, u_int32_t, struct if_ifclassq_stats *); diff --git a/bsd/net/pktsched/pktsched_rmclass.c b/bsd/net/pktsched/pktsched_rmclass.c deleted file mode 100644 index 89b98e14b..000000000 --- a/bsd/net/pktsched/pktsched_rmclass.c +++ /dev/null @@ -1,1852 +0,0 @@ -/* - * Copyright (c) 2007-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_rmclass.c,v 1.13 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_rmclass.c,v 1.10 2001/02/09 07:20:40 kjc Exp $ */ - -/* - * Copyright (c) 1991-1997 Regents of the University of California. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Network Research - * Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * LBL code modified by speer@eng.sun.com, May 1977. - * For questions and/or comments, please send mail to cbq@ee.lbl.gov - */ - -#include - -#ident "@(#)rm_class.c 1.48 97/12/05 SMI" - -#if PKTSCHED_CBQ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * Local Macros - */ - -#define reset_cutoff(ifd) { ifd->cutoff_ = RM_MAXDEPTH; } - -/* - * Local routines. - */ - -static int rmc_satisfied(struct rm_class *, struct timeval *); -static void rmc_wrr_set_weights(struct rm_ifdat *); -static void rmc_depth_compute(struct rm_class *); -static void rmc_depth_recompute(rm_class_t *); - -static struct mbuf *_rmc_wrr_dequeue_next(struct rm_ifdat *, cqdq_op_t); -static struct mbuf *_rmc_prr_dequeue_next(struct rm_ifdat *, cqdq_op_t); - -static int _rmc_addq(rm_class_t *, struct mbuf *, struct pf_mtag *); -static void _rmc_dropq(rm_class_t *); -static struct mbuf *_rmc_getq(rm_class_t *); -static struct mbuf *_rmc_pollq(rm_class_t *); - -static int rmc_under_limit(struct rm_class *, struct timeval *); -static void rmc_tl_satisfied(struct rm_ifdat *, struct timeval *); -static void rmc_drop_action(struct rm_class *); -static void rmc_restart(struct rm_class *); -static void rmc_root_overlimit(rm_class_t *, rm_class_t *); - -#define RMC_ZONE_MAX 32 /* maximum elements in zone */ -#define RMC_ZONE_NAME "pktsched_cbq_cl" /* zone name (CBQ for now) */ - -static unsigned int rmc_size; /* size of zone element */ -static struct zone *rmc_zone; /* zone for rm_class */ - -void -rmclass_init(void) -{ - if (rmc_zone != NULL) - return; - - rmc_size = sizeof (struct rm_class); - rmc_zone = zinit(rmc_size, RMC_ZONE_MAX * rmc_size, 0, RMC_ZONE_NAME); - if (rmc_zone == NULL) { - panic("%s: failed allocating %s", __func__, RMC_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(rmc_zone, Z_EXPAND, TRUE); - zone_change(rmc_zone, Z_CALLERACCT, TRUE); -} - -#define BORROW_OFFTIME -/* - * BORROW_OFFTIME (experimental): - * borrow the offtime of the class borrowing from. - * the reason is that when its own offtime is set, the class is unable - * to borrow much, especially when cutoff is taking effect. - * but when the borrowed class is overloaded (advidle is close to minidle), - * use the borrowing class's offtime to avoid overload. - */ -#define ADJUST_CUTOFF -/* - * ADJUST_CUTOFF (experimental): - * if no underlimit class is found due to cutoff, increase cutoff and - * retry the scheduling loop. - * also, don't invoke delay_actions while cutoff is taking effect, - * since a sleeping class won't have a chance to be scheduled in the - * next loop. - * - * now heuristics for setting the top-level variable (cutoff_) becomes: - * 1. if a packet arrives for a not-overlimit class, set cutoff - * to the depth of the class. - * 2. if cutoff is i, and a packet arrives for an overlimit class - * with an underlimit ancestor at a lower level than i (say j), - * then set cutoff to j. - * 3. at scheduling a packet, if there is no underlimit class - * due to the current cutoff level, increase cutoff by 1 and - * then try to schedule again. - */ - -/* - * rm_class_t * - * rmc_newclass(...) - Create a new resource management class at priority - * 'pri' on the interface given by 'ifd'. - * - * nsecPerByte is the data rate of the interface in nanoseconds/byte. - * E.g., 800 for a 10Mb/s ethernet. If the class gets less - * than 100% of the bandwidth, this number should be the - * 'effective' rate for the class. Let f be the - * bandwidth fraction allocated to this class, and let - * nsPerByte be the data rate of the output link in - * nanoseconds/byte. Then nsecPerByte is set to - * nsPerByte / f. E.g., 1600 (= 800 / .5) - * for a class that gets 50% of an ethernet's bandwidth. - * - * action the routine to call when the class is over limit. - * - * maxq max allowable queue size for class (in packets). - * - * parent parent class pointer. - * - * borrow class to borrow from (should be either 'parent' or null). - * - * maxidle max value allowed for class 'idle' time estimate (this - * parameter determines how large an initial burst of packets - * can be before overlimit action is invoked. - * - * offtime how long 'delay' action will delay when class goes over - * limit (this parameter determines the steady-state burst - * size when a class is running over its limit). - * - * Maxidle and offtime have to be computed from the following: If the - * average packet size is s, the bandwidth fraction allocated to this - * class is f, we want to allow b packet bursts, and the gain of the - * averaging filter is g (= 1 - 2^(-RM_FILTER_GAIN)), then: - * - * ptime = s * nsPerByte * (1 - f) / f - * maxidle = ptime * (1 - g^b) / g^b - * minidle = -ptime * (1 / (f - 1)) - * offtime = ptime * (1 + 1/(1 - g) * (1 - g^(b - 1)) / g^(b - 1) - * - * Operationally, it's convenient to specify maxidle & offtime in units - * independent of the link bandwidth so the maxidle & offtime passed to - * this routine are the above values multiplied by 8*f/(1000*nsPerByte). - * (The constant factor is a scale factor needed to make the parameters - * integers. This scaling also means that the 'unscaled' values of - * maxidle*nsecPerByte/8 and offtime*nsecPerByte/8 will be in microseconds, - * not nanoseconds.) Also note that the 'idle' filter computation keeps - * an estimate scaled upward by 2^RM_FILTER_GAIN so the passed value of - * maxidle also must be scaled upward by this value. Thus, the passed - * values for maxidle and offtime can be computed as follows: - * - * maxidle = maxidle * 2^RM_FILTER_GAIN * 8 / (1000 * nsecPerByte) - * offtime = offtime * 8 / (1000 * nsecPerByte) - * - * When USE_HRTIME is employed, then maxidle and offtime become: - * maxidle = maxilde * (8.0 / nsecPerByte); - * offtime = offtime * (8.0 / nsecPerByte); - */ -struct rm_class * -rmc_newclass(int pri, struct rm_ifdat *ifd, u_int32_t nsecPerByte, - void (*action)(rm_class_t *, rm_class_t *), u_int32_t qid, u_int32_t maxq, - struct rm_class *parent, struct rm_class *borrow, u_int32_t maxidle, - int minidle, u_int32_t offtime, int pktsize, int flags) -{ - struct ifnet *ifp; - struct ifclassq *ifq; - struct rm_class *cl; - struct rm_class *peer; - - if (nsecPerByte == 0) { - log(LOG_ERR, "%s: invalid inverse data rate\n", __func__); - return (NULL); - } - - if (pri >= RM_MAXPRIO) { - log(LOG_ERR, "%s: priority %d out of range! (max %d)\n", - __func__, pri, RM_MAXPRIO - 1); - return (NULL); - } - -#if !CLASSQ_RED - if (flags & RMCF_RED) { - log(LOG_ERR, "%s: RED not configured for CBQ!\n", __func__); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & RMCF_RIO) { - log(LOG_ERR, "%s: RIO not configured for CBQ!\n", __func__); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & RMCF_BLUE) { - log(LOG_ERR, "%s: BLUE not configured for CBQ!\n", __func__); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) && - (flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) != RMCF_RED && - (flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) != RMCF_RIO && - (flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) != RMCF_BLUE && - (flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) != RMCF_SFB) { - log(LOG_ERR, "%s: RED|RIO|BLUE|SFB mutually exclusive\n", - __func__); - return (NULL); - } - - cl = zalloc(rmc_zone); - if (cl == NULL) - return (NULL); - - bzero(cl, rmc_size); - CALLOUT_INIT(&cl->callout_); - - /* - * Class initialization. - */ - cl->children_ = NULL; - cl->parent_ = parent; - cl->borrow_ = borrow; - cl->leaf_ = 1; - cl->ifdat_ = ifd; - cl->pri_ = pri; - cl->allotment_ = RM_NS_PER_SEC / nsecPerByte; /* Bytes per sec */ - cl->depth_ = 0; - cl->qthresh_ = 0; - cl->ns_per_byte_ = nsecPerByte; - - ifq = ifd->ifq_; - ifp = ifq->ifcq_ifp; - - if (maxq == 0 || maxq > IFCQ_MAXLEN(ifq)) { - maxq = IFCQ_MAXLEN(ifq); - if (maxq == 0) - maxq = DEFAULT_QLIMIT; /* use default */ - } - _qinit(&cl->q_, Q_DROPHEAD, maxq); - - cl->flags_ = flags; - - cl->minidle_ = (minidle * (int)nsecPerByte) / 8; - if (cl->minidle_ > 0) - cl->minidle_ = 0; - - cl->maxidle_ = (maxidle * nsecPerByte) / 8; - if (cl->maxidle_ == 0) - cl->maxidle_ = 1; - - cl->avgidle_ = cl->maxidle_; - cl->offtime_ = ((offtime * nsecPerByte) / 8) >> RM_FILTER_GAIN; - if (cl->offtime_ == 0) - cl->offtime_ = 1; - - cl->overlimit = action; - - if (flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) { - int pkttime; - - cl->qflags_ = 0; - if (flags & RMCF_ECN) { - if (flags & RMCF_BLUE) - cl->qflags_ |= BLUEF_ECN; - else if (flags & RMCF_SFB) - cl->qflags_ |= SFBF_ECN; - else if (flags & RMCF_RED) - cl->qflags_ |= REDF_ECN; - else if (flags & RMCF_RIO) - cl->qflags_ |= RIOF_ECN; - } - if (flags & RMCF_FLOWCTL) { - if (flags & RMCF_SFB) - cl->qflags_ |= SFBF_FLOWCTL; - } - if (flags & RMCF_FLOWVALVE) { - if (flags & RMCF_RED) - cl->qflags_ |= REDF_FLOWVALVE; - } - if (flags & RMCF_CLEARDSCP) { - if (flags & RMCF_RIO) - cl->qflags_ |= RIOF_CLEARDSCP; - } - pkttime = nsecPerByte * pktsize / 1000; - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RED - if (flags & RMCF_RED) { - cl->red_ = red_alloc(ifp, 0, 0, - qlimit(&cl->q_) * 10/100, - qlimit(&cl->q_) * 30/100, - cl->qflags_, pkttime); - if (cl->red_ != NULL) - qtype(&cl->q_) = Q_RED; - } -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (flags & RMCF_RIO) { - cl->rio_ = - rio_alloc(ifp, 0, NULL, cl->qflags_, pkttime); - if (cl->rio_ != NULL) - qtype(&cl->q_) = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & RMCF_BLUE) { - cl->blue_ = blue_alloc(ifp, 0, 0, cl->qflags_); - if (cl->blue_ != NULL) - qtype(&cl->q_) = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & RMCF_SFB) { - if (!(cl->flags_ & RMCF_LAZY)) - cl->sfb_ = sfb_alloc(ifp, qid, - qlimit(&cl->q_), cl->qflags_); - if (cl->sfb_ != NULL || (cl->flags_ & RMCF_LAZY)) - qtype(&cl->q_) = Q_SFB; - } - } - - /* - * put the class into the class tree - */ - if ((peer = ifd->active_[pri]) != NULL) { - /* find the last class at this pri */ - cl->peer_ = peer; - while (peer->peer_ != ifd->active_[pri]) - peer = peer->peer_; - peer->peer_ = cl; - } else { - ifd->active_[pri] = cl; - cl->peer_ = cl; - } - - if (cl->parent_) { - cl->next_ = parent->children_; - parent->children_ = cl; - parent->leaf_ = 0; - } - - /* - * Compute the depth of this class and its ancestors in the class - * hierarchy. - */ - rmc_depth_compute(cl); - - /* - * If CBQ's WRR is enabled, then initialize the class WRR state. - */ - if (ifd->wrr_) { - ifd->num_[pri]++; - ifd->alloc_[pri] += cl->allotment_; - rmc_wrr_set_weights(ifd); - } - return (cl); -} - -int -rmc_modclass(struct rm_class *cl, u_int32_t nsecPerByte, int maxq, - u_int32_t maxidle, int minidle, u_int32_t offtime, int pktsize) -{ -#pragma unused(pktsize) - struct rm_ifdat *ifd; - u_int32_t old_allotment; - - ifd = cl->ifdat_; - old_allotment = cl->allotment_; - - cl->allotment_ = RM_NS_PER_SEC / nsecPerByte; /* Bytes per sec */ - cl->qthresh_ = 0; - cl->ns_per_byte_ = nsecPerByte; - - qlimit(&cl->q_) = maxq; - - cl->minidle_ = (minidle * nsecPerByte) / 8; - if (cl->minidle_ > 0) - cl->minidle_ = 0; - - cl->maxidle_ = (maxidle * nsecPerByte) / 8; - if (cl->maxidle_ == 0) - cl->maxidle_ = 1; - - cl->avgidle_ = cl->maxidle_; - cl->offtime_ = ((offtime * nsecPerByte) / 8) >> RM_FILTER_GAIN; - if (cl->offtime_ == 0) - cl->offtime_ = 1; - - /* - * If CBQ's WRR is enabled, then initialize the class WRR state. - */ - if (ifd->wrr_) { - ifd->alloc_[cl->pri_] += cl->allotment_ - old_allotment; - rmc_wrr_set_weights(ifd); - } - return (0); -} - -/* - * static void - * rmc_wrr_set_weights(struct rm_ifdat *ifdat) - This function computes - * the appropriate run robin weights for the CBQ weighted round robin - * algorithm. - * - * Returns: NONE - */ - -static void -rmc_wrr_set_weights(struct rm_ifdat *ifd) -{ - int i; - struct rm_class *cl, *clh; - - for (i = 0; i < RM_MAXPRIO; i++) { - /* - * This is inverted from that of the simulator to - * maintain precision. - */ - if (ifd->num_[i] == 0) { - ifd->M_[i] = 0; - } else { - ifd->M_[i] = - ifd->alloc_[i] / (ifd->num_[i] * ifd->maxpkt_); - } - /* - * Compute the weighted allotment for each class. - * This takes the expensive div instruction out - * of the main loop for the wrr scheduling path. - * These only get recomputed when a class comes or - * goes. - */ - if (ifd->active_[i] != NULL) { - clh = cl = ifd->active_[i]; - do { - /* safe-guard for slow link or alloc_ == 0 */ - if (ifd->M_[i] == 0) { - cl->w_allotment_ = 0; - } else { - cl->w_allotment_ = - cl->allotment_ / ifd->M_[i]; - } - cl = cl->peer_; - } while ((cl != NULL) && (cl != clh)); - } - } -} - -int -rmc_get_weight(struct rm_ifdat *ifd, int pri) -{ - if ((pri >= 0) && (pri < RM_MAXPRIO)) - return (ifd->M_[pri]); - else - return (0); -} - -/* - * static void - * rmc_depth_compute(struct rm_class *cl) - This function computes the - * appropriate depth of class 'cl' and its ancestors. - * - * Returns: NONE - */ - -static void -rmc_depth_compute(struct rm_class *cl) -{ - rm_class_t *t = cl, *p; - - /* - * Recompute the depth for the branch of the tree. - */ - while (t != NULL) { - p = t->parent_; - if (p && (t->depth_ >= p->depth_)) { - p->depth_ = t->depth_ + 1; - t = p; - } else - t = NULL; - } -} - -/* - * static void - * rmc_depth_recompute(struct rm_class *cl) - This function re-computes - * the depth of the tree after a class has been deleted. - * - * Returns: NONE - */ - -static void -rmc_depth_recompute(rm_class_t *cl) -{ - rm_class_t *p, *t; - - p = cl; - while (p != NULL) { - if ((t = p->children_) == NULL) { - p->depth_ = 0; - } else { - int cdepth = 0; - - while (t != NULL) { - if (t->depth_ > cdepth) - cdepth = t->depth_; - t = t->next_; - } - - if (p->depth_ == cdepth + 1) - /* no change to this parent */ - return; - - p->depth_ = cdepth + 1; - } - - p = p->parent_; - } -} - -/* - * void - * rmc_delete_class(struct rm_ifdat *ifdat, struct rm_class *cl) - This - * function deletes a class from the link-sharing structure and frees - * all resources associated with the class. - * - * Returns: NONE - */ - -void -rmc_delete_class(struct rm_ifdat *ifd, struct rm_class *cl) -{ - struct rm_class *p, *head, *previous; - - VERIFY(cl->children_ == NULL); - - if (cl->sleeping_) - CALLOUT_STOP(&cl->callout_); - - /* - * Free packets in the packet queue. - * XXX - this may not be a desired behavior. Packets should be - * re-queued. - */ - rmc_dropall(cl); - - /* - * If the class has a parent, then remove the class from the - * class from the parent's children chain. - */ - if (cl->parent_ != NULL) { - head = cl->parent_->children_; - p = previous = head; - if (head->next_ == NULL) { - VERIFY(head == cl); - cl->parent_->children_ = NULL; - cl->parent_->leaf_ = 1; - } else while (p != NULL) { - if (p == cl) { - if (cl == head) - cl->parent_->children_ = cl->next_; - else - previous->next_ = cl->next_; - cl->next_ = NULL; - p = NULL; - } else { - previous = p; - p = p->next_; - } - } - } - - /* - * Delete class from class priority peer list. - */ - if ((p = ifd->active_[cl->pri_]) != NULL) { - /* - * If there is more than one member of this priority - * level, then look for class(cl) in the priority level. - */ - if (p != p->peer_) { - while (p->peer_ != cl) - p = p->peer_; - p->peer_ = cl->peer_; - - if (ifd->active_[cl->pri_] == cl) - ifd->active_[cl->pri_] = cl->peer_; - } else { - VERIFY(p == cl); - ifd->active_[cl->pri_] = NULL; - } - } - - /* - * Recompute the WRR weights. - */ - if (ifd->wrr_) { - ifd->alloc_[cl->pri_] -= cl->allotment_; - ifd->num_[cl->pri_]--; - rmc_wrr_set_weights(ifd); - } - - /* - * Re-compute the depth of the tree. - */ - rmc_depth_recompute(cl->parent_); - - /* - * Free the class structure. - */ - if (cl->qalg_.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - rio_destroy(cl->rio_); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - red_destroy(cl->red_); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - blue_destroy(cl->blue_); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_) && cl->sfb_ != NULL) - sfb_destroy(cl->sfb_); - cl->qalg_.ptr = NULL; - qtype(&cl->q_) = Q_DROPTAIL; - qstate(&cl->q_) = QS_RUNNING; - } - zfree(rmc_zone, cl); -} - - -/* - * int - * rmc_init(...) - Initialize the resource management data structures - * associated with the output portion of interface 'ifp'. 'ifd' is - * where the structures will be built (for backwards compatibility, the - * structures aren't kept in the ifnet struct). 'nsecPerByte' - * gives the link speed (inverse of bandwidth) in nanoseconds/byte. - * 'restart' is the driver-specific routine that the generic 'delay - * until under limit' action will call to restart output. `maxq' - * is the queue size of the 'link' & 'default' classes. 'maxqueued' - * is the maximum number of packets that the resource management - * code will allow to be queued 'downstream' (this is typically 1). - * - * Returns: 0 on success - */ - -int -rmc_init(struct ifclassq *ifq, struct rm_ifdat *ifd, u_int32_t nsecPerByte, - void (*restart)(struct ifclassq *), u_int32_t qid, int maxq, int maxqueued, - u_int32_t maxidle, int minidle, u_int32_t offtime, int flags) -{ - struct ifnet *ifp = ifq->ifcq_ifp; - int i, mtu; - - /* - * Initialize the CBQ tracing/debug facility. - */ - CBQTRACEINIT(); - - if (nsecPerByte == 0) { - log(LOG_ERR, "%s: %s: invalid inverse data rate)\n", - __func__, if_name(ifp)); - return (EINVAL); - } - - mtu = ifp->if_mtu; - if (mtu < 1) { - log(LOG_ERR, "%s: %s: invalid MTU (interface not " - "initialized?)\n", __func__, if_name(ifp)); - return (EINVAL); - } - bzero((char *)ifd, sizeof (*ifd)); - - ifd->ifq_ = ifq; - ifd->restart = restart; - ifd->maxqueued_ = maxqueued; - ifd->ns_per_byte_ = nsecPerByte; - ifd->maxpkt_ = mtu; - ifd->wrr_ = (flags & RMCF_WRR) ? 1 : 0; - ifd->efficient_ = (flags & RMCF_EFFICIENT) ? 1 : 0; -#if 1 - ifd->maxiftime_ = mtu * nsecPerByte / 1000 * 16; - if (mtu * nsecPerByte > 10 * 1000000) - ifd->maxiftime_ /= 4; -#endif - - reset_cutoff(ifd); - CBQTRACE(rmc_init, 'INIT', ifd->cutoff_); - - /* - * Initialize the CBQ's WRR state. - */ - for (i = 0; i < RM_MAXPRIO; i++) { - ifd->alloc_[i] = 0; - ifd->M_[i] = 0; - ifd->num_[i] = 0; - ifd->na_[i] = 0; - ifd->active_[i] = NULL; - } - - /* - * Initialize current packet state. - */ - ifd->qi_ = 0; - ifd->qo_ = 0; - for (i = 0; i < RM_MAXQUEUED; i++) { - ifd->class_[i] = NULL; - ifd->curlen_[i] = 0; - ifd->borrowed_[i] = NULL; - } - - /* - * Create the root class of the link-sharing structure. - */ - if ((ifd->root_ = rmc_newclass(0, ifd, nsecPerByte, - rmc_root_overlimit, qid, maxq, 0, 0, maxidle, minidle, offtime, - 0, 0)) == NULL) { - log(LOG_ERR, "rmc_init: root class not allocated\n"); - return (ENOMEM); - } - ifd->root_->depth_ = 0; - - return (0); -} - -/* - * void - * rmc_queue_packet(struct rm_class *cl, struct mbuf *m) - Add packet given by - * mbuf 'm' to queue for resource class 'cl'. This routine is called - * by a driver's if_output routine. This routine must be called with - * output packet completion interrupts locked out (to avoid racing with - * rmc_dequeue_next). - * - * Returns: 0 on successful queueing - * CLASSQEQ_DROPPED when packet drop occurs - */ -int -rmc_queue_packet(struct rm_class *cl, struct mbuf *m, struct pf_mtag *t) -{ - struct timeval now; - struct rm_ifdat *ifd = cl->ifdat_; - int cpri = cl->pri_; - int is_empty = qempty(&cl->q_); - int ret = 0; - - RM_GETTIME(now); - if (ifd->cutoff_ > 0) { - if (TV_LT(&cl->undertime_, &now)) { - if (ifd->cutoff_ > cl->depth_) - ifd->cutoff_ = cl->depth_; - CBQTRACE(rmc_queue_packet, 'ffoc', cl->depth_); - } else { - /* - * the class is overlimit. if the class has - * underlimit ancestors, set cutoff to the lowest - * depth among them. - */ - struct rm_class *borrow = cl->borrow_; - - while (borrow != NULL && - borrow->depth_ < ifd->cutoff_) { - if (TV_LT(&borrow->undertime_, &now)) { - ifd->cutoff_ = borrow->depth_; - CBQTRACE(rmc_queue_packet, 'ffob', - ifd->cutoff_); - break; - } - borrow = borrow->borrow_; - } - } - } - - ret = _rmc_addq(cl, m, t); - if (ret != 0 && - (ret == CLASSQEQ_DROPPED || ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP)) { - /* failed */ - return (ret); - } - VERIFY(ret == 0 || ret == CLASSQEQ_SUCCESS_FC); - if (is_empty) { - CBQTRACE(rmc_queue_packet, 'type', cl->stats_.handle); - ifd->na_[cpri]++; - } - - if (qlen(&cl->q_) > qlimit(&cl->q_)) { - /* note: qlimit can be set to 0 or 1 */ - rmc_drop_action(cl); - return (CLASSQEQ_DROPPED); - } - return (ret); -} - -/* - * void - * rmc_tl_satisfied(struct rm_ifdat *ifd, struct timeval *now) - Check all - * classes to see if there are satified. - */ - -static void -rmc_tl_satisfied(struct rm_ifdat *ifd, struct timeval *now) -{ - int i; - rm_class_t *p, *bp; - - for (i = RM_MAXPRIO - 1; i >= 0; i--) { - if ((bp = ifd->active_[i]) != NULL) { - p = bp; - do { - if (!rmc_satisfied(p, now)) { - ifd->cutoff_ = p->depth_; - return; - } - p = p->peer_; - } while (p != bp); - } - } - - reset_cutoff(ifd); -} - -/* - * rmc_satisfied - Return 1 of the class is satisfied. O, otherwise. - */ - -static int -rmc_satisfied(struct rm_class *cl, struct timeval *now) -{ - rm_class_t *p; - - if (cl == NULL) - return (1); - if (TV_LT(now, &cl->undertime_)) - return (1); - if (cl->depth_ == 0) { - if (!cl->sleeping_ && (qlen(&cl->q_) > cl->qthresh_)) - return (0); - else - return (1); - } - if (cl->children_ != NULL) { - p = cl->children_; - while (p != NULL) { - if (!rmc_satisfied(p, now)) - return (0); - p = p->next_; - } - } - - return (1); -} - -/* - * Return 1 if class 'cl' is under limit or can borrow from a parent, - * 0 if overlimit. As a side-effect, this routine will invoke the - * class overlimit action if the class if overlimit. - */ - -static int -rmc_under_limit(struct rm_class *cl, struct timeval *now) -{ - rm_class_t *p = cl; - rm_class_t *top; - struct rm_ifdat *ifd = cl->ifdat_; - - ifd->borrowed_[ifd->qi_] = NULL; - /* - * If cl is the root class, then always return that it is - * underlimit. Otherwise, check to see if the class is underlimit. - */ - if (cl->parent_ == NULL) - return (1); - - if (cl->sleeping_) { - if (TV_LT(now, &cl->undertime_)) - return (0); - - CALLOUT_STOP(&cl->callout_); - cl->sleeping_ = 0; - cl->undertime_.tv_sec = 0; - return (1); - } - - top = NULL; - while (cl->undertime_.tv_sec && TV_LT(now, &cl->undertime_)) { - if (((cl = cl->borrow_) == NULL) || - (cl->depth_ > ifd->cutoff_)) { -#ifdef ADJUST_CUTOFF - if (cl != NULL) - /* - * cutoff is taking effect, just - * return false without calling - * the delay action. - */ - return (0); -#endif -#ifdef BORROW_OFFTIME - /* - * check if the class can borrow offtime too. - * borrow offtime from the top of the borrow - * chain if the top class is not overloaded. - */ - if (cl != NULL) { - /* - * cutoff is taking effect, use this - * class as top. - */ - top = cl; - CBQTRACE(rmc_under_limit, 'ffou', ifd->cutoff_); - } - if (top != NULL && top->avgidle_ == top->minidle_) - top = NULL; - p->overtime_ = *now; - (p->overlimit)(p, top); -#else - p->overtime_ = *now; - (p->overlimit)(p, NULL); -#endif - return (0); - } - top = cl; - } - - if (cl != p) - ifd->borrowed_[ifd->qi_] = cl; - return (1); -} - -/* - * _rmc_wrr_dequeue_next() - This is scheduler for WRR as opposed to - * Packet-by-packet round robin. - * - * The heart of the weighted round-robin scheduler, which decides which - * class next gets to send a packet. Highest priority first, then - * weighted round-robin within priorites. - * - * Each able-to-send class gets to send until its byte allocation is - * exhausted. Thus, the active pointer is only changed after a class has - * exhausted its allocation. - * - * If the scheduler finds no class that is underlimit or able to borrow, - * then the first class found that had a nonzero queue and is allowed to - * borrow gets to send. - */ - -static struct mbuf * -_rmc_wrr_dequeue_next(struct rm_ifdat *ifd, cqdq_op_t op) -{ - struct rm_class *cl = NULL, *first = NULL; - u_int32_t deficit; - int cpri; - struct mbuf *m; - struct timeval now; - - RM_GETTIME(now); - - /* - * if the driver polls the top of the queue and then removes - * the polled packet, we must return the same packet. - */ - if (op == CLASSQDQ_REMOVE && ifd->pollcache_) { - cl = ifd->pollcache_; - cpri = cl->pri_; - if (ifd->efficient_) { - /* check if this class is overlimit */ - if (cl->undertime_.tv_sec != 0 && - rmc_under_limit(cl, &now) == 0) - first = cl; - } - ifd->pollcache_ = NULL; - goto _wrr_out; - } else { - /* mode == CLASSQDQ_POLL || pollcache == NULL */ - ifd->pollcache_ = NULL; - ifd->borrowed_[ifd->qi_] = NULL; - } -#ifdef ADJUST_CUTOFF -_again: -#endif - for (cpri = RM_MAXPRIO - 1; cpri >= 0; cpri--) { - if (ifd->na_[cpri] == 0) - continue; - deficit = 0; - /* - * Loop through twice for a priority level, if some class - * was unable to send a packet the first round because - * of the weighted round-robin mechanism. - * During the second loop at this level, deficit==2. - * (This second loop is not needed if for every class, - * "M[cl->pri_])" times "cl->allotment" is greater than - * the byte size for the largest packet in the class.) - */ -_wrr_loop: - cl = ifd->active_[cpri]; - VERIFY(cl != NULL); - do { - if ((deficit < 2) && (cl->bytes_alloc_ <= 0)) - cl->bytes_alloc_ += cl->w_allotment_; - if (!qempty(&cl->q_)) { - if ((cl->undertime_.tv_sec == 0) || - rmc_under_limit(cl, &now)) { - if (cl->bytes_alloc_ > 0 || deficit > 1) - goto _wrr_out; - - /* underlimit but no alloc */ - deficit = 1; -#if 1 - ifd->borrowed_[ifd->qi_] = NULL; -#endif - } else if (first == NULL && cl->borrow_ != NULL) - first = cl; /* borrowing candidate */ - } - - cl->bytes_alloc_ = 0; - cl = cl->peer_; - } while (cl != ifd->active_[cpri]); - - if (deficit == 1) { - /* first loop found an underlimit class with deficit */ - /* Loop on same priority level, with new deficit. */ - deficit = 2; - goto _wrr_loop; - } - } - -#ifdef ADJUST_CUTOFF - /* - * no underlimit class found. if cutoff is taking effect, - * increase cutoff and try again. - */ - if (first != NULL && ifd->cutoff_ < ifd->root_->depth_) { - ifd->cutoff_++; - CBQTRACE(_rmc_wrr_dequeue_next, 'ojda', ifd->cutoff_); - goto _again; - } -#endif /* ADJUST_CUTOFF */ - /* - * If LINK_EFFICIENCY is turned on, then the first overlimit - * class we encounter will send a packet if all the classes - * of the link-sharing structure are overlimit. - */ - reset_cutoff(ifd); - CBQTRACE(_rmc_wrr_dequeue_next, 'otsr', ifd->cutoff_); - - if (!ifd->efficient_ || first == NULL) - return (NULL); - - cl = first; - cpri = cl->pri_; -#if 0 /* too time-consuming for nothing */ - if (cl->sleeping_) - CALLOUT_STOP(&cl->callout_); - cl->sleeping_ = 0; - cl->undertime_.tv_sec = 0; -#endif - ifd->borrowed_[ifd->qi_] = cl->borrow_; - ifd->cutoff_ = cl->borrow_->depth_; - - /* - * Deque the packet and do the book keeping... - */ -_wrr_out: - if (op == CLASSQDQ_REMOVE) { - m = _rmc_getq(cl); - if (m == NULL) - return (NULL); - - if (qempty(&cl->q_)) - ifd->na_[cpri]--; - - /* - * Update class statistics and link data. - */ - if (cl->bytes_alloc_ > 0) - cl->bytes_alloc_ -= m_pktlen(m); - - if ((cl->bytes_alloc_ <= 0) || first == cl) - ifd->active_[cl->pri_] = cl->peer_; - else - ifd->active_[cl->pri_] = cl; - - ifd->class_[ifd->qi_] = cl; - ifd->curlen_[ifd->qi_] = m_pktlen(m); - ifd->now_[ifd->qi_] = now; - ifd->qi_ = (ifd->qi_ + 1) % ifd->maxqueued_; - ifd->queued_++; - } else { - /* mode == ALTDQ_PPOLL */ - m = _rmc_pollq(cl); - ifd->pollcache_ = cl; - } - return (m); -} - -/* - * Dequeue & return next packet from the highest priority class that - * has a packet to send & has enough allocation to send it. This - * routine is called by a driver whenever it needs a new packet to - * output. - */ -static struct mbuf * -_rmc_prr_dequeue_next(struct rm_ifdat *ifd, cqdq_op_t op) -{ - struct mbuf *m; - int cpri; - struct rm_class *cl, *first = NULL; - struct timeval now; - - RM_GETTIME(now); - - /* - * if the driver polls the top of the queue and then removes - * the polled packet, we must return the same packet. - */ - if (op == CLASSQDQ_REMOVE && ifd->pollcache_) { - cl = ifd->pollcache_; - cpri = cl->pri_; - ifd->pollcache_ = NULL; - goto _prr_out; - } else { - /* mode == CLASSQDQ_POLL || pollcache == NULL */ - ifd->pollcache_ = NULL; - ifd->borrowed_[ifd->qi_] = NULL; - } -#ifdef ADJUST_CUTOFF -_again: -#endif - for (cpri = RM_MAXPRIO - 1; cpri >= 0; cpri--) { - if (ifd->na_[cpri] == 0) - continue; - cl = ifd->active_[cpri]; - VERIFY(cl != NULL); - do { - if (!qempty(&cl->q_)) { - if ((cl->undertime_.tv_sec == 0) || - rmc_under_limit(cl, &now)) - goto _prr_out; - if (first == NULL && cl->borrow_ != NULL) - first = cl; - } - cl = cl->peer_; - } while (cl != ifd->active_[cpri]); - } - -#ifdef ADJUST_CUTOFF - /* - * no underlimit class found. if cutoff is taking effect, increase - * cutoff and try again. - */ - if (first != NULL && ifd->cutoff_ < ifd->root_->depth_) { - ifd->cutoff_++; - goto _again; - } -#endif /* ADJUST_CUTOFF */ - /* - * If LINK_EFFICIENCY is turned on, then the first overlimit - * class we encounter will send a packet if all the classes - * of the link-sharing structure are overlimit. - */ - reset_cutoff(ifd); - if (!ifd->efficient_ || first == NULL) - return (NULL); - - cl = first; - cpri = cl->pri_; -#if 0 /* too time-consuming for nothing */ - if (cl->sleeping_) - CALLOUT_STOP(&cl->callout_); - cl->sleeping_ = 0; - cl->undertime_.tv_sec = 0; -#endif - ifd->borrowed_[ifd->qi_] = cl->borrow_; - ifd->cutoff_ = cl->borrow_->depth_; - - /* - * Deque the packet and do the book keeping... - */ -_prr_out: - if (op == CLASSQDQ_REMOVE) { - m = _rmc_getq(cl); - if (m == NULL) - return (NULL); - - if (qempty(&cl->q_)) - ifd->na_[cpri]--; - - ifd->active_[cpri] = cl->peer_; - - ifd->class_[ifd->qi_] = cl; - ifd->curlen_[ifd->qi_] = m_pktlen(m); - ifd->now_[ifd->qi_] = now; - ifd->qi_ = (ifd->qi_ + 1) % ifd->maxqueued_; - ifd->queued_++; - } else { - /* mode == CLASSQDQ_POLL */ - m = _rmc_pollq(cl); - ifd->pollcache_ = cl; - } - return (m); -} - -/* - * struct mbuf * - * rmc_dequeue_next(struct rm_ifdat *ifd, struct timeval *now) - this function - * is invoked by the packet driver to get the next packet to be - * dequeued and output on the link. If WRR is enabled, then the - * WRR dequeue next routine will determine the next packet to sent. - * Otherwise, packet-by-packet round robin is invoked. - * - * Returns: NULL, if a packet is not available or if all - * classes are overlimit. - * - * Otherwise, Pointer to the next packet. - */ - -struct mbuf * -rmc_dequeue_next(struct rm_ifdat *ifd, cqdq_op_t mode) -{ - if (ifd->queued_ >= ifd->maxqueued_) - return (NULL); - else if (ifd->wrr_) - return (_rmc_wrr_dequeue_next(ifd, mode)); - else - return (_rmc_prr_dequeue_next(ifd, mode)); -} - -/* - * Update the utilization estimate for the packet that just completed. - * The packet's class & the parent(s) of that class all get their - * estimators updated. This routine is called by the driver's output- - * packet-completion interrupt service routine. - */ - -/* - * a macro to approximate "divide by 1000" that gives 0.000999, - * if a value has enough effective digits. - * (on pentium, mul takes 9 cycles but div takes 46!) - */ -#define NSEC_TO_USEC(t) (((t) >> 10) + ((t) >> 16) + ((t) >> 17)) -void -rmc_update_class_util(struct rm_ifdat *ifd) -{ - int idle, avgidle, pktlen; - int pkt_time, tidle; - rm_class_t *cl, *borrowed; - rm_class_t *borrows; - struct timeval *nowp; - - /* - * Get the most recent completed class. - */ - if ((cl = ifd->class_[ifd->qo_]) == NULL) - return; - - pktlen = ifd->curlen_[ifd->qo_]; - borrowed = ifd->borrowed_[ifd->qo_]; - borrows = borrowed; - - PKTCNTR_ADD(&cl->stats_.xmit_cnt, 1, pktlen); - - /* - * Run estimator on class and its ancestors. - */ - /* - * rm_update_class_util is designed to be called when the - * transfer is completed from a xmit complete interrupt, - * but most drivers don't implement an upcall for that. - * so, just use estimated completion time. - * as a result, ifd->qi_ and ifd->qo_ are always synced. - */ - nowp = &ifd->now_[ifd->qo_]; - /* get pkt_time (for link) in usec */ -#if 1 /* use approximation */ - pkt_time = ifd->curlen_[ifd->qo_] * ifd->ns_per_byte_; - pkt_time = NSEC_TO_USEC(pkt_time); -#else - pkt_time = ifd->curlen_[ifd->qo_] * ifd->ns_per_byte_ / 1000; -#endif -#if 1 /* ALTQ4PPP */ - if (TV_LT(nowp, &ifd->ifnow_)) { - int iftime; - - /* - * make sure the estimated completion time does not go - * too far. it can happen when the link layer supports - * data compression or the interface speed is set to - * a much lower value. - */ - TV_DELTA(&ifd->ifnow_, nowp, iftime); - if (iftime+pkt_time < ifd->maxiftime_) { - TV_ADD_DELTA(&ifd->ifnow_, pkt_time, &ifd->ifnow_); - } else { - TV_ADD_DELTA(nowp, ifd->maxiftime_, &ifd->ifnow_); - } - } else { - TV_ADD_DELTA(nowp, pkt_time, &ifd->ifnow_); - } -#else - if (TV_LT(nowp, &ifd->ifnow_)) { - TV_ADD_DELTA(&ifd->ifnow_, pkt_time, &ifd->ifnow_); - } else { - TV_ADD_DELTA(nowp, pkt_time, &ifd->ifnow_); - } -#endif - - while (cl != NULL) { - TV_DELTA(&ifd->ifnow_, &cl->last_, idle); - if (idle >= 2000000) - /* - * this class is idle enough, reset avgidle. - * (TV_DELTA returns 2000000 us when delta is large.) - */ - cl->avgidle_ = cl->maxidle_; - - /* get pkt_time (for class) in usec */ -#if 1 /* use approximation */ - pkt_time = pktlen * cl->ns_per_byte_; - pkt_time = NSEC_TO_USEC(pkt_time); -#else - pkt_time = pktlen * cl->ns_per_byte_ / 1000; -#endif - idle -= pkt_time; - - avgidle = cl->avgidle_; - avgidle += idle - (avgidle >> RM_FILTER_GAIN); - cl->avgidle_ = avgidle; - - /* Are we overlimit ? */ - if (avgidle <= 0) { - CBQTRACE(rmc_update_class_util, 'milo', - cl->stats_.handle); - /* - * need some lower bound for avgidle, otherwise - * a borrowing class gets unbounded penalty. - */ - if (avgidle < cl->minidle_) - avgidle = cl->avgidle_ = cl->minidle_; - - /* set next idle to make avgidle 0 */ - tidle = pkt_time + - (((1 - RM_POWER) * avgidle) >> RM_FILTER_GAIN); - TV_ADD_DELTA(nowp, tidle, &cl->undertime_); - ++cl->stats_.over; - } else { - cl->avgidle_ = - (avgidle > cl->maxidle_) ? cl->maxidle_ : avgidle; - cl->undertime_.tv_sec = 0; - if (cl->sleeping_) { - CALLOUT_STOP(&cl->callout_); - cl->sleeping_ = 0; - } - } - - if (borrows != NULL) { - if (borrows != cl) - ++cl->stats_.borrows; - else - borrows = NULL; - } - cl->last_ = ifd->ifnow_; - cl->last_pkttime_ = pkt_time; - -#if 1 - if (cl->parent_ == NULL) { - /* take stats of root class */ - PKTCNTR_ADD(&cl->stats_.xmit_cnt, 1, pktlen); - } -#endif - - cl = cl->parent_; - } - - /* - * Check to see if cutoff needs to set to a new level. - */ - cl = ifd->class_[ifd->qo_]; - if (borrowed && (ifd->cutoff_ >= borrowed->depth_)) { - if ((qlen(&cl->q_) <= 0) || - TV_LT(nowp, &borrowed->undertime_)) { - rmc_tl_satisfied(ifd, nowp); - CBQTRACE(rmc_update_class_util, 'broe', ifd->cutoff_); - } else { - ifd->cutoff_ = borrowed->depth_; - CBQTRACE(rmc_update_class_util, 'ffob', - borrowed->depth_); - } - } - - /* - * Release class slot - */ - ifd->borrowed_[ifd->qo_] = NULL; - ifd->class_[ifd->qo_] = NULL; - ifd->qo_ = (ifd->qo_ + 1) % ifd->maxqueued_; - ifd->queued_--; -} - -/* - * void - * rmc_drop_action(struct rm_class *cl) - Generic (not protocol-specific) - * over-limit action routines. These get invoked by rmc_under_limit() - * if a class with packets to send if over its bandwidth limit & can't - * borrow from a parent class. - * - * Returns: NONE - */ - -static void -rmc_drop_action(struct rm_class *cl) -{ - struct rm_ifdat *ifd = cl->ifdat_; - - VERIFY(qlen(&cl->q_) > 0); - IFCQ_CONVERT_LOCK(ifd->ifq_); - _rmc_dropq(cl); - if (qempty(&cl->q_)) - ifd->na_[cl->pri_]--; -} - -void -rmc_drop(struct rm_class *cl, u_int32_t flow, u_int32_t *packets, - u_int32_t *bytes) -{ - struct rm_ifdat *ifd = cl->ifdat_; - struct ifclassq *ifq = ifd->ifq_; - u_int32_t pkt = 0, len = 0, qlen; - - if ((qlen = qlen(&cl->q_)) != 0) { - IFCQ_CONVERT_LOCK(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - rio_purgeq(cl->rio_, &cl->q_, flow, &pkt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - red_purgeq(cl->red_, &cl->q_, flow, &pkt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - blue_purgeq(cl->blue_, &cl->q_, flow, &pkt, &len); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_) && cl->sfb_ != NULL) - sfb_purgeq(cl->sfb_, &cl->q_, flow, &pkt, &len); - else - _flushq_flow(&cl->q_, flow, &pkt, &len); - - if (pkt > 0) { - VERIFY(qlen(&cl->q_) == (qlen - pkt)); - - PKTCNTR_ADD(&cl->stats_.drop_cnt, pkt, len); - IFCQ_DROP_ADD(ifq, pkt, len); - - VERIFY(((signed)IFCQ_LEN(ifq) - pkt) >= 0); - IFCQ_LEN(ifq) -= pkt; - - if (qempty(&cl->q_)) - ifd->na_[cl->pri_]--; - } - } - if (packets != NULL) - *packets = pkt; - if (bytes != NULL) - *bytes = len; -} - -void -rmc_dropall(struct rm_class *cl) -{ - rmc_drop(cl, 0, NULL, NULL); -} - -/* - * void - * rmc_delay_action(struct rm_class *cl) - This function is the generic CBQ - * delay action routine. It is invoked via rmc_under_limit when the - * packet is discoverd to be overlimit. - * - * If the delay action is result of borrow class being overlimit, then - * delay for the offtime of the borrowing class that is overlimit. - * - * Returns: NONE - */ - -void -rmc_delay_action(struct rm_class *cl, struct rm_class *borrow) -{ - int ndelay, t, extradelay; - - cl->stats_.overactions++; - TV_DELTA(&cl->undertime_, &cl->overtime_, ndelay); -#ifndef BORROW_OFFTIME - ndelay += cl->offtime_; -#endif - - if (!cl->sleeping_) { - CBQTRACE(rmc_delay_action, 'yled', cl->stats_.handle); -#ifdef BORROW_OFFTIME - if (borrow != NULL) - extradelay = borrow->offtime_; - else -#endif - extradelay = cl->offtime_; - - /* - * XXX recalculate suspend time: - * current undertime is (tidle + pkt_time) calculated - * from the last transmission. - * tidle: time required to bring avgidle back to 0 - * pkt_time: target waiting time for this class - * we need to replace pkt_time by offtime - */ - extradelay -= cl->last_pkttime_; - if (extradelay > 0) { - TV_ADD_DELTA(&cl->undertime_, extradelay, - &cl->undertime_); - ndelay += extradelay; - } - - cl->sleeping_ = 1; - cl->stats_.delays++; - - /* - * Since packets are phased randomly with respect to the - * clock, 1 tick (the next clock tick) can be an arbitrarily - * short time so we have to wait for at least two ticks. - * NOTE: If there's no other traffic, we need the timer as - * a 'backstop' to restart this class. - */ - if (ndelay > tick * 2) { - /* - * FreeBSD rounds up the tick; - * other BSDs round down the tick. - */ - t = hzto(&cl->undertime_) + 1; - } else { - t = 2; - } - CALLOUT_RESET(&cl->callout_, t, - (timeout_t *)rmc_restart, (caddr_t)cl); - } -} - -/* - * void - * rmc_restart() - is just a helper routine for rmc_delay_action -- it is - * called by the system timer code & is responsible checking if the - * class is still sleeping (it might have been restarted as a side - * effect of the queue scan on a packet arrival) and, if so, restarting - * output for the class. Inspecting the class state & restarting output - * require locking the class structure. In general the driver is - * responsible for locking but this is the only routine that is not - * called directly or indirectly from the interface driver so it has - * know about system locking conventions. - * - * Returns: NONE - */ - -static void -rmc_restart(struct rm_class *cl) -{ - struct rm_ifdat *ifd = cl->ifdat_; - - if (cl->sleeping_) { - cl->sleeping_ = 0; - cl->undertime_.tv_sec = 0; - - if (ifd->queued_ < ifd->maxqueued_ && ifd->restart != NULL) { - CBQTRACE(rmc_restart, 'trts', cl->stats_.handle); - (ifd->restart)(ifd->ifq_); - } - } -} - -/* - * void - * rmc_root_overlimit(struct rm_class *cl) - This the generic overlimit - * handling routine for the root class of the link sharing structure. - * - * Returns: NONE - */ -static void -rmc_root_overlimit(struct rm_class *cl, - struct rm_class *borrow) -{ -#pragma unused(cl, borrow) - panic("rmc_root_overlimit"); -} - -/* - * Packet Queue handling routines. Eventually, this is to localize the - * effects on the code whether queues are red queues or droptail - * queues. - */ - -static int -_rmc_addq(rm_class_t *cl, struct mbuf *m, struct pf_mtag *t) -{ -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - return (rio_addq(cl->rio_, &cl->q_, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - return (red_addq(cl->red_, &cl->q_, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - return (blue_addq(cl->blue_, &cl->q_, m, t)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_)) { - if (cl->sfb_ == NULL) { - struct ifclassq *ifq = cl->ifdat_->ifq_; - struct ifnet *ifp = ifq->ifcq_ifp; - - VERIFY(cl->flags_ & RMCF_LAZY); - IFCQ_CONVERT_LOCK(ifq); - - cl->sfb_ = sfb_alloc(ifp, cl->stats_.handle, - qlimit(&cl->q_), cl->qflags_); - if (cl->sfb_ == NULL) { - /* fall back to droptail */ - qtype(&cl->q_) = Q_DROPTAIL; - cl->flags_ &= ~RMCF_SFB; - cl->qflags_ &= ~(SFBF_ECN | SFBF_FLOWCTL); - - log(LOG_ERR, "%s: CBQ SFB lazy allocation " - "failed for qid=%d pri=%d, falling back " - "to DROPTAIL\n", if_name(ifp), - cl->stats_.handle, cl->pri_); - } - } - if (cl->sfb_ != NULL) - return (sfb_addq(cl->sfb_, &cl->q_, m, t)); - } -#if PF_ECN - else if (cl->flags_ & RMCF_CLEARDSCP) - write_dsfield(m, t, 0); -#endif /* PF_ECN */ - - /* test for qlen > qlimit is done by caller */ - _addq(&cl->q_, m); - return (0); -} - -/* note: _rmc_dropq is not called for red */ -static void -_rmc_dropq(rm_class_t *cl) -{ - struct mbuf *m; - - if ((m = _rmc_getq(cl)) != NULL) - m_freem(m); -} - -static struct mbuf * -_rmc_getq(rm_class_t *cl) -{ -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - return (rio_getq(cl->rio_, &cl->q_)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - return (red_getq(cl->red_, &cl->q_)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - return (blue_getq(cl->blue_, &cl->q_)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_) && cl->sfb_ != NULL) - return (sfb_getq(cl->sfb_, &cl->q_)); - - return (_getq(&cl->q_)); -} - -static struct mbuf * -_rmc_pollq(rm_class_t *cl) -{ - return (qhead(&cl->q_)); -} - -void -rmc_updateq(rm_class_t *cl, cqev_t ev) -{ -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - return (rio_updateq(cl->rio_, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - return (red_updateq(cl->red_, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - return (blue_updateq(cl->blue_, ev)); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_) && cl->sfb_ != NULL) - return (sfb_updateq(cl->sfb_, ev)); -} - -#ifdef CBQ_TRACE - -struct cbqtrace cbqtrace_buffer[NCBQTRACE+1]; -struct cbqtrace *cbqtrace_ptr = NULL; -int cbqtrace_count; - -/* - * DDB hook to trace cbq events: - * the last 1024 events are held in a circular buffer. - * use "call cbqtrace_dump(N)" to display 20 events from Nth event. - */ -void cbqtrace_dump(int); -static char *rmc_funcname(void *); - -static struct rmc_funcs { - void *func; - char *name; -} rmc_funcs[] = -{ - rmc_init, "rmc_init", - rmc_queue_packet, "rmc_queue_packet", - rmc_under_limit, "rmc_under_limit", - rmc_update_class_util, "rmc_update_class_util", - rmc_delay_action, "rmc_delay_action", - rmc_restart, "rmc_restart", - _rmc_wrr_dequeue_next, "_rmc_wrr_dequeue_next", - NULL, NULL -}; - -static char * -rmc_funcname(void *func) -{ - struct rmc_funcs *fp; - - for (fp = rmc_funcs; fp->func != NULL; fp++) - if (fp->func == func) - return (fp->name); - return ("unknown"); -} - -void -cbqtrace_dump(int counter) -{ - int i, *p; - char *cp; - - counter = counter % NCBQTRACE; - p = (int *)&cbqtrace_buffer[counter]; - - for (i = 0; i < 20; i++) { - log(LOG_DEBUG, "[0x%x] ", *p++); - log(LOG_DEBUG, "%s: ", rmc_funcname((void *)*p++)); - cp = (char *)p++; - log(LOG_DEBUG, "%c%c%c%c: ", cp[0], cp[1], cp[2], cp[3]); - log(LOG_DEBUG, "%d\n", *p++); - - if (p >= (int *)&cbqtrace_buffer[NCBQTRACE]) - p = (int *)cbqtrace_buffer; - } -} -#endif /* CBQ_TRACE */ -#endif /* PKTSCHED_CBQ */ diff --git a/bsd/net/pktsched/pktsched_rmclass.h b/bsd/net/pktsched/pktsched_rmclass.h index d5f6b13b2..b467fa835 100644 --- a/bsd/net/pktsched/pktsched_rmclass.h +++ b/bsd/net/pktsched/pktsched_rmclass.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -90,225 +90,6 @@ extern "C" { #define RMCF_BLUE 0x10000 /* use BLUE */ #define RMCF_SFB 0x20000 /* use SFB */ #define RMCF_FLOWCTL 0x40000 /* enable flow control advisories */ -#ifdef BSD_KERNEL_PRIVATE -#define RMCF_LAZY 0x10000000 /* on-demand resource allocation */ - -typedef struct rm_ifdat rm_ifdat_t; -typedef struct rm_class rm_class_t; - -struct red; -struct rio; -struct blue; -struct sfb; - -/* - * Macros for dealing with time values. We assume all times are - * 'timevals'. `microuptime' is used to get the best available clock - * resolution. If `microuptime' *doesn't* return a value that's about - * ten times smaller than the average packet time on the fastest - * link that will use these routines, a slightly different clock - * scheme than this one should be used. - * (Bias due to truncation error in this scheme will overestimate utilization - * and discriminate against high bandwidth classes. To remove this bias an - * integrator needs to be added. The simplest integrator uses a history of - * 10 * avg.packet.time / min.tick.time packet completion entries. This is - * straight forward to add but we don't want to pay the extra memory - * traffic to maintain it if it's not necessary (occasionally a vendor - * accidentally builds a workstation with a decent clock - e.g., Sun & HP).) - */ - -#define RM_GETTIME(now) microuptime(&now) - -#define TV_LT(a, b) (((a)->tv_sec < (b)->tv_sec) || \ - (((a)->tv_usec < (b)->tv_usec) && ((a)->tv_sec <= (b)->tv_sec))) - -#define TV_DELTA(a, b, delta) { \ - int xxs; \ - \ - delta = (a)->tv_usec - (b)->tv_usec; \ - if ((xxs = (a)->tv_sec - (b)->tv_sec)) { \ - switch (xxs) { \ - default: \ - /* \ - * if (xxs < 0) \ - * printf("rm_class: bogus time values\n"); \ - */ \ - delta = 0; \ - /* fall through */ \ - case 2: \ - delta += 1000000; \ - /* fall through */ \ - case 1: \ - delta += 1000000; \ - break; \ - } \ - } \ -} - -#define TV_ADD_DELTA(a, delta, res) { \ - int xxus = (a)->tv_usec + (delta); \ - \ - (res)->tv_sec = (a)->tv_sec; \ - while (xxus >= 1000000) { \ - ++((res)->tv_sec); \ - xxus -= 1000000; \ - } \ - (res)->tv_usec = xxus; \ -} - -#define RM_TIMEOUT 2 /* 1 Clock tick. */ - -#if 1 -#define RM_MAXQUEUED 1 /* this isn't used in ALTQ/CBQ */ -#else -#define RM_MAXQUEUED 16 /* Max number of packets downstream of CBQ */ -#endif -#define RM_MAXQUEUE 64 /* Max queue length */ -#define RM_FILTER_GAIN 5 /* log2 of gain, e.g., 5 => 31/32 */ -#define RM_POWER (1 << RM_FILTER_GAIN) -#define RM_MAXDEPTH 32 -#define RM_NS_PER_SEC (1000000000) - -typedef struct _rm_class_stats_ { - u_int32_t handle; - u_int32_t depth; - - struct pktcntr xmit_cnt; /* packets sent in this class */ - struct pktcntr drop_cnt; /* dropped packets */ - u_int32_t over; /* # times went over limit */ - u_int32_t borrows; /* # times tried to borrow */ - u_int32_t overactions; /* # times invoked overlimit action */ - u_int32_t delays; /* # times invoked delay actions */ -} rm_class_stats_t; - -/* - * CBQ Class state structure - */ -struct rm_class { - class_queue_t q_; /* Queue of packets */ - rm_ifdat_t *ifdat_; - int pri_; /* Class priority. */ - int depth_; /* Class depth */ - u_int32_t ns_per_byte_; /* NanoSeconds per byte. */ - u_int32_t maxrate_; /* Bytes per second for this class. */ - u_int32_t allotment_; /* Fraction of link bandwidth. */ - u_int32_t w_allotment_; /* Weighted allotment for WRR */ - int bytes_alloc_; /* Allocation for round of WRR */ - - int avgidle_; - int maxidle_; - int minidle_; - int offtime_; - int sleeping_; /* != 0 if delaying */ - u_int32_t qthresh_; /* Threshold for formal link sharing */ - int leaf_; /* Note whether leaf class or not */ - - rm_class_t *children_; /* Children of this class */ - rm_class_t *next_; /* Next pointer, used if child */ - - rm_class_t *peer_; /* Peer class */ - rm_class_t *borrow_; /* Borrow class */ - rm_class_t *parent_; /* Parent class */ - - void (*overlimit)(struct rm_class *, struct rm_class *); - void (*drop)(struct rm_class *); /* Class drop action. */ - - union { - void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ - struct sfb *sfb; /* SFB state */ - } qalg_; - int flags_; - u_int32_t qflags_; - - int last_pkttime_; /* saved pkt_time */ - struct timeval undertime_; /* time can next send */ - struct timeval last_; /* time last packet sent */ - struct timeval overtime_; - struct callout callout_; /* for timeout() calls */ - - rm_class_stats_t stats_; /* Class Statistics */ -}; - -#define red_ qalg_.red -#define rio_ qalg_.rio -#define blue_ qalg_.blue -#define sfb_ qalg_.sfb - -/* - * CBQ Interface state - */ -struct rm_ifdat { - int queued_; /* # pkts queued downstream */ - int efficient_; /* Link Efficency bit */ - int wrr_; /* Enable Weighted Round-Robin */ - u_long ns_per_byte_; /* Link byte speed. */ - int maxqueued_; /* Max packets to queue */ - int maxpkt_; /* Max packet size. */ - int qi_; /* In/out pointers for downstream */ - int qo_; /* packets */ - - /* - * Active class state and WRR state. - */ - rm_class_t *active_[RM_MAXPRIO]; /* Active cl's in each pri */ - int na_[RM_MAXPRIO]; /* # of active cl's in a pri */ - int num_[RM_MAXPRIO]; /* # of cl's per pri */ - int alloc_[RM_MAXPRIO]; /* Byte Allocation */ - u_long M_[RM_MAXPRIO]; /* WRR weights. */ - - /* - * Network Interface/Solaris Queue state pointer. - */ - struct ifclassq *ifq_; - rm_class_t *default_; /* Default Pkt class, BE */ - rm_class_t *root_; /* Root Link class. */ - rm_class_t *ctl_; /* Control Traffic class. */ - void (*restart)(struct ifclassq *); /* Restart routine. */ - - /* - * Current packet downstream packet state and dynamic state. - */ - rm_class_t *borrowed_[RM_MAXQUEUED]; /* Class borrowed last */ - rm_class_t *class_[RM_MAXQUEUED]; /* class sending */ - int curlen_[RM_MAXQUEUED]; /* Current pktlen */ - struct timeval now_[RM_MAXQUEUED]; /* Current packet time */ - int is_overlimit_[RM_MAXQUEUED]; /* Current packet time */ - - int cutoff_; /* Cut-off depth for borrowing */ - - struct timeval ifnow_; /* expected xmit completion time */ -#if 1 /* ALTQ4PPP */ - int maxiftime_; /* max delay inside interface */ -#endif - rm_class_t *pollcache_; /* cached rm_class by poll operation */ -}; - -#define RMC_IS_A_PARENT_CLASS(cl) ((cl)->children_ != NULL) - -extern void rmclass_init(void); -extern rm_class_t *rmc_newclass(int, struct rm_ifdat *, u_int32_t, - void (*)(struct rm_class *, struct rm_class *), u_int32_t, - u_int32_t, struct rm_class *, struct rm_class *, - u_int32_t, int, u_int32_t, int, int); -extern void rmc_delete_class(struct rm_ifdat *, struct rm_class *); -extern int rmc_modclass(struct rm_class *, u_int32_t, int, u_int32_t, - int, u_int32_t, int); -extern int rmc_init(struct ifclassq *, struct rm_ifdat *, u_int32_t, - void (*)(struct ifclassq *), u_int32_t, int, int, u_int32_t, - int, u_int32_t, int); -extern int rmc_queue_packet(struct rm_class *, struct mbuf *, struct pf_mtag *); -extern struct mbuf *rmc_dequeue_next(struct rm_ifdat *, cqdq_op_t); -extern void rmc_update_class_util(struct rm_ifdat *); -extern void rmc_delay_action(struct rm_class *, struct rm_class *); -extern void rmc_drop(struct rm_class *, u_int32_t, u_int32_t *, u_int32_t *); -extern void rmc_dropall(struct rm_class *); -extern int rmc_get_weight(struct rm_ifdat *, int); -extern void rmc_updateq(struct rm_class *, cqev_t); - -#endif /* BSD_KERNEL_PRIVATE */ #ifdef __cplusplus } diff --git a/bsd/net/pktsched/pktsched_rmclass_debug.h b/bsd/net/pktsched/pktsched_rmclass_debug.h deleted file mode 100644 index dd3f364f5..000000000 --- a/bsd/net/pktsched/pktsched_rmclass_debug.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2011 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $NetBSD: altq_rmclass_debug.h,v 1.7 2006/10/12 19:59:08 peter Exp $ */ -/* $KAME: altq_rmclass_debug.h,v 1.3 2002/11/29 04:36:24 kjc Exp $ */ - -/* - * Copyright (c) Sun Microsystems, Inc. 1998 All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the SMCC Technology - * Development Group at Sun Microsystems, Inc. - * - * 4. The name of the Sun Microsystems, Inc nor may not be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * SUN MICROSYSTEMS DOES NOT CLAIM MERCHANTABILITY OF THIS SOFTWARE OR THE - * SUITABILITY OF THIS SOFTWARE FOR ANY PARTICULAR PURPOSE. The software is - * provided "as is" without express or implied warranty of any kind. - * - * These notices must be retained in any copies of any part of this software. - */ - -#ifndef _NET_PKTSCHED_PKTSCHED_RMCLASS_DEBUG_H_ -#define _NET_PKTSCHED_PKTSCHED_RMCLASS_DEBUG_H_ - -/* #pragma ident "@(#)rm_class_debug.h 1.7 98/05/04 SMI" */ - -/* - * Cbq debugging macros - */ - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef BSD_KERNEL_PRIVATE - -#ifdef CBQ_TRACE -#ifndef NCBQTRACE -#define NCBQTRACE (16 * 1024) -#endif - -/* - * To view the trace output, using adb, type: - * adb -k /dev/ksyms /dev/mem , then type - * cbqtrace_count/D to get the count, then type - * cbqtrace_buffer,0tcount/Dp4C" "Xn - * This will dump the trace buffer from 0 to count. - */ -/* - * in ALTQ, "call cbqtrace_dump(N)" from DDB to display 20 events - * from Nth event in the circular buffer. - */ - -struct cbqtrace { - int count; - int function; /* address of function */ - int trace_action; /* descriptive 4 characters */ - int object; /* object operated on */ -}; - -extern struct cbqtrace cbqtrace_buffer[]; -extern struct cbqtrace *cbqtrace_ptr; -extern int cbqtrace_count; - -#define CBQTRACEINIT() { \ - if (cbqtrace_ptr == NULL) \ - cbqtrace_ptr = cbqtrace_buffer; \ - else { \ - cbqtrace_ptr = cbqtrace_buffer; \ - bzero((void *)cbqtrace_ptr, sizeof (cbqtrace_buffer)); \ - cbqtrace_count = 0; \ - } \ -} - -#define CBQTRACE(func, act, obj) { \ - int *_p = &cbqtrace_ptr->count; \ - *_p++ = ++cbqtrace_count; \ - *_p++ = (int)(func); \ - *_p++ = (int)(act); \ - *_p++ = (int)(obj); \ - if ((struct cbqtrace *)(void *)_p >= &cbqtrace_buffer[NCBQTRACE]) \ - cbqtrace_ptr = cbqtrace_buffer; \ - else \ - cbqtrace_ptr = (struct cbqtrace *)(void *)_p; \ - } -#else - -/* If no tracing, define no-ops */ -#define CBQTRACEINIT() -#define CBQTRACE(a, b, c) - -#endif /* !CBQ_TRACE */ - -#endif /* BSD_KERNEL_PRIVATE */ - -#ifdef __cplusplus -} -#endif - -#endif /* _NET_PKTSCHED_PKTSCHED_RMCLASS_DEBUG_H_ */ diff --git a/bsd/net/pktsched/pktsched_tcq.c b/bsd/net/pktsched/pktsched_tcq.c index d3e64b5e1..d83fbb253 100644 --- a/bsd/net/pktsched/pktsched_tcq.c +++ b/bsd/net/pktsched/pktsched_tcq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -47,22 +47,23 @@ #include #include + /* * function prototypes */ -static int tcq_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *tcq_dequeue_tc_ifclassq(struct ifclassq *, - mbuf_svc_class_t, cqdq_op_t); +static int tcq_enqueue_ifclassq(struct ifclassq *, void *, classq_pkt_type_t, + boolean_t *); +static void *tcq_dequeue_tc_ifclassq(struct ifclassq *, mbuf_svc_class_t, + classq_pkt_type_t *); static int tcq_request_ifclassq(struct ifclassq *, cqrq_t, void *); static int tcq_clear_interface(struct tcq_if *); static struct tcq_class *tcq_class_create(struct tcq_if *, int, u_int32_t, - int, u_int32_t); + int, u_int32_t, classq_pkt_type_t); static int tcq_class_destroy(struct tcq_if *, struct tcq_class *); static int tcq_destroy_locked(struct tcq_if *); -static inline int tcq_addq(struct tcq_class *, struct mbuf *, +static inline int tcq_addq(struct tcq_class *, pktsched_pkt_t *, struct pf_mtag *); -static inline struct mbuf *tcq_getq(struct tcq_class *); -static inline struct mbuf *tcq_pollq(struct tcq_class *); +static inline void tcq_getq(struct tcq_class *, pktsched_pkt_t *); static void tcq_purgeq(struct tcq_if *, struct tcq_class *, u_int32_t, u_int32_t *, u_int32_t *); static void tcq_purge_sc(struct tcq_if *, cqrq_purge_sc_t *); @@ -71,8 +72,8 @@ static int tcq_throttle(struct tcq_if *, cqrq_throttle_t *); static int tcq_resumeq(struct tcq_if *, struct tcq_class *); static int tcq_suspendq(struct tcq_if *, struct tcq_class *); static int tcq_stat_sc(struct tcq_if *, cqrq_stat_sc_t *); -static struct mbuf *tcq_dequeue_cl(struct tcq_if *, struct tcq_class *, - mbuf_svc_class_t, cqdq_op_t); +static void tcq_dequeue_cl(struct tcq_if *, struct tcq_class *, + mbuf_svc_class_t, pktsched_pkt_t *); static inline struct tcq_class *tcq_clh_to_clp(struct tcq_if *, u_int32_t); static const char *tcq_style(struct tcq_if *); @@ -113,7 +114,7 @@ tcq_init(void) } struct tcq_if * -tcq_alloc(struct ifnet *ifp, int how, boolean_t altq) +tcq_alloc(struct ifnet *ifp, int how) { struct tcq_if *tif; @@ -124,8 +125,6 @@ tcq_alloc(struct ifnet *ifp, int how, boolean_t altq) bzero(tif, tcq_size); tif->tif_maxpri = -1; tif->tif_ifq = &ifp->if_snd; - if (altq) - tif->tif_flags |= TCQIFF_ALTQ; if (pktsched_verbose) { log(LOG_DEBUG, "%s: %s scheduler allocated\n", @@ -198,15 +197,7 @@ tcq_purge(struct tcq_if *tif) if ((cl = tif->tif_classes[pri]) != NULL && !qempty(&cl->cl_q)) tcq_purgeq(tif, cl, 0, NULL, NULL); } -#if !PF_ALTQ - /* - * This assertion is safe to be made only when PF_ALTQ is not - * configured; otherwise, IFCQ_LEN represents the sum of the - * packets managed by ifcq_disc and altq_disc instances, which - * is possible when transitioning between the two. - */ VERIFY(IFCQ_LEN(tif->tif_ifq) == 0); -#endif /* !PF_ALTQ */ } static void @@ -256,7 +247,7 @@ tcq_event(struct tcq_if *tif, cqev_t ev) int tcq_add_queue(struct tcq_if *tif, int priority, u_int32_t qlimit, - int flags, u_int32_t qid, struct tcq_class **clp) + int flags, u_int32_t qid, struct tcq_class **clp, classq_pkt_type_t ptype) { struct tcq_class *cl; @@ -270,7 +261,7 @@ tcq_add_queue(struct tcq_if *tif, int priority, u_int32_t qlimit, if (tcq_clh_to_clp(tif, qid) != NULL) return (EBUSY); - cl = tcq_class_create(tif, priority, qlimit, flags, qid); + cl = tcq_class_create(tif, priority, qlimit, flags, qid, ptype); if (cl == NULL) return (ENOMEM); @@ -282,7 +273,7 @@ tcq_add_queue(struct tcq_if *tif, int priority, u_int32_t qlimit, static struct tcq_class * tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit, - int flags, u_int32_t qid) + int flags, u_int32_t qid, classq_pkt_type_t ptype) { struct ifnet *ifp; struct ifclassq *ifq; @@ -290,45 +281,6 @@ tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit, IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq); - /* Sanitize flags unless internally configured */ - if (tif->tif_flags & TCQIFF_ALTQ) - flags &= TQCF_USERFLAGS; - -#if !CLASSQ_RED - if (flags & TQCF_RED) { - log(LOG_ERR, "%s: %s RED not available!\n", - if_name(TCQIF_IFP(tif)), tcq_style(tif)); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & TQCF_RIO) { - log(LOG_ERR, "%s: %s RIO not available!\n", - if_name(TCQIF_IFP(tif)), tcq_style(tif)); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & TQCF_BLUE) { - log(LOG_ERR, "%s: %s BLUE not available!\n", - if_name(TCQIF_IFP(tif)), tcq_style(tif)); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) && - (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_RED && - (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_RIO && - (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_BLUE && - (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_SFB) { - log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n", - if_name(TCQIF_IFP(tif)), tcq_style(tif)); - return (NULL); - } - ifq = tif->tif_ifq; ifp = TCQIF_IFP(tif); @@ -336,23 +288,13 @@ tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit, /* modify the class instead of creating a new one */ if (!qempty(&cl->cl_q)) tcq_purgeq(tif, cl, 0, NULL, NULL); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ + if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_destroy(cl->cl_sfb); cl->cl_qalg.ptr = NULL; qtype(&cl->cl_q) = Q_DROPTAIL; qstate(&cl->cl_q) = QS_RUNNING; + VERIFY(qptype(&cl->cl_q) == ptype); } else { cl = zalloc(tcq_cl_zone); if (cl == NULL) @@ -369,7 +311,7 @@ tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit, if (qlimit == 0) qlimit = DEFAULT_QLIMIT; /* use default */ } - _qinit(&cl->cl_q, Q_DROPTAIL, qlimit); + _qinit(&cl->cl_q, Q_DROPTAIL, qlimit, ptype); cl->cl_flags = flags; cl->cl_pri = pri; if (pri > tif->tif_maxpri) @@ -377,80 +319,22 @@ tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit, cl->cl_tif = tif; cl->cl_handle = qid; - if (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) { -#if CLASSQ_RED || CLASSQ_RIO - u_int64_t ifbandwidth = ifnet_output_linkrate(ifp); - int pkttime; -#endif /* CLASSQ_RED || CLASSQ_RIO */ - + if (flags & TQCF_SFB) { cl->cl_qflags = 0; if (flags & TQCF_ECN) { - if (flags & TQCF_BLUE) - cl->cl_qflags |= BLUEF_ECN; - else if (flags & TQCF_SFB) - cl->cl_qflags |= SFBF_ECN; - else if (flags & TQCF_RED) - cl->cl_qflags |= REDF_ECN; - else if (flags & TQCF_RIO) - cl->cl_qflags |= RIOF_ECN; + cl->cl_qflags |= SFBF_ECN; } if (flags & TQCF_FLOWCTL) { - if (flags & TQCF_SFB) - cl->cl_qflags |= SFBF_FLOWCTL; + cl->cl_qflags |= SFBF_FLOWCTL; } if (flags & TQCF_DELAYBASED) { - if (flags & TQCF_SFB) - cl->cl_qflags |= SFBF_DELAYBASED; - } - if (flags & TQCF_CLEARDSCP) { - if (flags & TQCF_RIO) - cl->cl_qflags |= RIOF_CLEARDSCP; - } -#if CLASSQ_RED || CLASSQ_RIO - /* - * XXX: RED & RIO should be watching link speed and MTU - * events and recompute pkttime accordingly. - */ - if (ifbandwidth < 8) - pkttime = 1000 * 1000 * 1000; /* 1 sec */ - else - pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 / - (ifbandwidth / 8); - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RED - if (flags & TQCF_RED) { - cl->cl_red = red_alloc(ifp, 0, 0, - qlimit(&cl->cl_q) * 10/100, - qlimit(&cl->cl_q) * 30/100, - cl->cl_qflags, pkttime); - if (cl->cl_red != NULL) - qtype(&cl->cl_q) = Q_RED; - } -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (flags & TQCF_RIO) { - cl->cl_rio = - rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime); - if (cl->cl_rio != NULL) - qtype(&cl->cl_q) = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#endif /* CLASSQ_RED || CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & TQCF_BLUE) { - cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags); - if (cl->cl_blue != NULL) - qtype(&cl->cl_q) = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & TQCF_SFB) { - if (!(cl->cl_flags & TQCF_LAZY)) - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb != NULL || (cl->cl_flags & TQCF_LAZY)) - qtype(&cl->cl_q) = Q_SFB; + cl->cl_qflags |= SFBF_DELAYBASED; } + if (!(cl->cl_flags & TQCF_LAZY)) + cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, + qlimit(&cl->cl_q), cl->cl_qflags); + if (cl->cl_sfb != NULL || (cl->cl_flags & TQCF_LAZY)) + qtype(&cl->cl_q) = Q_SFB; } if (pktsched_verbose) { @@ -480,7 +364,9 @@ tcq_class_destroy(struct tcq_if *tif, struct tcq_class *cl) { struct ifclassq *ifq = tif->tif_ifq; int pri; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); if (!qempty(&cl->cl_q)) @@ -501,18 +387,6 @@ tcq_class_destroy(struct tcq_if *tif, struct tcq_class *cl) tif->tif_default = NULL; if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_destroy(cl->cl_sfb); cl->cl_qalg.ptr = NULL; @@ -531,7 +405,7 @@ tcq_class_destroy(struct tcq_if *tif, struct tcq_class *cl) } int -tcq_enqueue(struct tcq_if *tif, struct tcq_class *cl, struct mbuf *m, +tcq_enqueue(struct tcq_if *tif, struct tcq_class *cl, pktsched_pkt_t *pkt, struct pf_mtag *t) { struct ifclassq *ifq = tif->tif_ifq; @@ -541,45 +415,27 @@ tcq_enqueue(struct tcq_if *tif, struct tcq_class *cl, struct mbuf *m, VERIFY(cl == NULL || cl->cl_tif == tif); if (cl == NULL) { -#if PF_ALTQ - cl = tcq_clh_to_clp(tif, t->pftag_qid); -#else /* !PF_ALTQ */ cl = tcq_clh_to_clp(tif, 0); -#endif /* !PF_ALTQ */ if (cl == NULL) { cl = tif->tif_default; if (cl == NULL) { IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); + return (CLASSQEQ_DROP); } } } - len = m_pktlen(m); - - ret = tcq_addq(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - /* packet has been freed in tcq_addq */ - PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT REACHED */ - } + VERIFY(pkt->pktsched_ptype == qptype(&cl->cl_q)); + len = pktsched_get_pkt_len(pkt); + + ret = tcq_addq(cl, pkt, t); + if ((ret != 0) && (ret != CLASSQEQ_SUCCESS_FC)) { + VERIFY(ret == CLASSQEQ_DROP || + ret == CLASSQEQ_DROP_FC || + ret == CLASSQEQ_DROP_SP); + PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); + IFCQ_DROP_ADD(ifq, 1, len); + return (ret); } IFCQ_INC_LEN(ifq); IFCQ_INC_BYTES(ifq, len); @@ -594,70 +450,56 @@ tcq_enqueue(struct tcq_if *tif, struct tcq_class *cl, struct mbuf *m, * CLASSQDQ_REMOVE must return the same packet if called immediately * after CLASSQDQ_POLL. */ -struct mbuf * -tcq_dequeue_tc(struct tcq_if *tif, mbuf_svc_class_t sc, cqdq_op_t op) +void +tcq_dequeue_tc(struct tcq_if *tif, mbuf_svc_class_t sc, pktsched_pkt_t *pkt) { - return (tcq_dequeue_cl(tif, NULL, sc, op)); + tcq_dequeue_cl(tif, NULL, sc, pkt); } -static struct mbuf * -tcq_dequeue_cl(struct tcq_if *tif, struct tcq_class *cl, - mbuf_svc_class_t sc, cqdq_op_t op) +static void +tcq_dequeue_cl(struct tcq_if *tif, struct tcq_class *cl, mbuf_svc_class_t sc, + pktsched_pkt_t *pkt) { struct ifclassq *ifq = tif->tif_ifq; - struct mbuf *m; + uint32_t len; IFCQ_LOCK_ASSERT_HELD(ifq); if (cl == NULL) { cl = tcq_clh_to_clp(tif, MBUF_SCIDX(sc)); - if (cl == NULL) - return (NULL); + if (cl == NULL) { + pkt->pktsched_pkt = NULL; + return; + } } - if (qempty(&cl->cl_q)) - return (NULL); + if (qempty(&cl->cl_q)) { + pkt->pktsched_pkt = NULL; + return; + } VERIFY(!IFCQ_IS_EMPTY(ifq)); - if (op == CLASSQDQ_POLL) - return (tcq_pollq(cl)); - - m = tcq_getq(cl); - if (m != NULL) { + tcq_getq(cl, pkt); + if (pkt->pktsched_pkt != NULL) { + len = pktsched_get_pkt_len(pkt); IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, m_pktlen(m)); + IFCQ_DEC_BYTES(ifq, len); if (qempty(&cl->cl_q)) cl->cl_period++; - PKTCNTR_ADD(&cl->cl_xmitcnt, 1, m_pktlen(m)); - IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m)); + PKTCNTR_ADD(&cl->cl_xmitcnt, 1, len); + IFCQ_XMIT_ADD(ifq, 1, len); } - return (m); } static inline int -tcq_addq(struct tcq_class *cl, struct mbuf *m, struct pf_mtag *t) +tcq_addq(struct tcq_class *cl, pktsched_pkt_t *pkt, struct pf_mtag *t) { struct tcq_if *tif = cl->cl_tif; struct ifclassq *ifq = tif->tif_ifq; IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_addq(cl->cl_rio, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_addq(cl->cl_red, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_addq(cl->cl_blue, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q)) { if (cl->cl_sfb == NULL) { struct ifnet *ifp = TCQIF_IFP(tif); @@ -693,55 +535,35 @@ tcq_addq(struct tcq_class *cl, struct mbuf *m, struct pf_mtag *t) } } if (cl->cl_sfb != NULL) - return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t)); + return (sfb_addq(cl->cl_sfb, &cl->cl_q, pkt, t)); } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) { IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); + return (CLASSQEQ_DROP); } #if PF_ECN if (cl->cl_flags & TQCF_CLEARDSCP) + /* not supported for skywalk packets */ + VERIFY(pkt->pktsched_ptype == QP_MBUF); write_dsfield(m, t, 0); #endif /* PF_ECN */ - _addq(&cl->cl_q, m); + VERIFY(pkt->pktsched_ptype == qptype(&cl->cl_q)); + _addq(&cl->cl_q, pkt->pktsched_pkt); return (0); } -static inline struct mbuf * -tcq_getq(struct tcq_class *cl) +static inline void +tcq_getq(struct tcq_class *cl, pktsched_pkt_t *pkt) { IFCQ_LOCK_ASSERT_HELD(cl->cl_tif->tif_ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_getq(cl->cl_rio, &cl->cl_q)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_getq(cl->cl_red, &cl->cl_q)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_getq(cl->cl_blue, &cl->cl_q)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_getq(cl->cl_sfb, &cl->cl_q)); - - return (_getq(&cl->cl_q)); -} - -static inline struct mbuf * -tcq_pollq(struct tcq_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_tif->tif_ifq); + if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) { + return (sfb_getq(cl->cl_sfb, &cl->cl_q, pkt)); + } - return (qhead(&cl->cl_q)); + return (pktsched_pkt_encap(pkt, qptype(&cl->cl_q), _getq(&cl->cl_q))); } static void @@ -756,24 +578,7 @@ tcq_purgeq(struct tcq_if *tif, struct tcq_class *cl, u_int32_t flow, if ((qlen = qlen(&cl->cl_q)) == 0) goto done; - /* become regular mutex before freeing mbufs */ IFCQ_CONVERT_LOCK(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len); else @@ -814,18 +619,6 @@ tcq_updateq(struct tcq_if *tif, struct tcq_class *cl, cqev_t ev) cl->cl_handle, cl->cl_pri, ifclassq_ev2str(ev)); } -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_updateq(cl->cl_rio, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_updateq(cl->cl_red, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_updateq(cl->cl_blue, ev)); -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) return (sfb_updateq(cl->cl_sfb, ev)); } @@ -851,18 +644,7 @@ tcq_get_class_stats(struct tcq_if *tif, u_int32_t qid, sp->qtype = qtype(&cl->cl_q); sp->qstate = qstate(&cl->cl_q); -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_getstats(cl->cl_red, &sp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_getstats(cl->cl_rio, &sp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_getstats(cl->cl_blue, &sp->blue); -#endif /* CLASSQ_BLUE */ + if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_getstats(cl->cl_sfb, &sp->sfb); @@ -910,7 +692,8 @@ tcq_clh_to_clp(struct tcq_if *tif, u_int32_t chandle) static const char * tcq_style(struct tcq_if *tif) { - return ((tif->tif_flags & TCQIFF_ALTQ) ? "ALTQ_TCQ" : "TCQ"); +#pragma unused(tif) + return ("TCQ"); } /* @@ -918,26 +701,64 @@ tcq_style(struct tcq_if *tif) * (*ifcq_enqueue) in struct ifclassq. */ static int -tcq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) +tcq_enqueue_ifclassq(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype, + boolean_t *pdrop) { - u_int32_t i; + u_int32_t i = 0; + int ret; + pktsched_pkt_t pkt; + struct pf_mtag *t = NULL; IFCQ_LOCK_ASSERT_HELD(ifq); - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - log(LOG_ERR, "%s: packet does not have pkthdr\n", - if_name(ifq->ifcq_ifp)); - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); + if (ptype == QP_MBUF) { + struct mbuf *m = p; + if (!(m->m_flags & M_PKTHDR)) { + /* should not happen */ + log(LOG_ERR, "%s: packet does not have pkthdr\n", + if_name(ifq->ifcq_ifp)); + IFCQ_CONVERT_LOCK(ifq); + m_freem(m); + *pdrop = TRUE; + return (ENOBUFS); + } + t = m_pftag(m); + i = MBUF_SCIDX(mbuf_get_service_class(m)); } - - i = MBUF_SCIDX(mbuf_get_service_class(m)); VERIFY((u_int32_t)i < IFCQ_SC_MAX); - return (tcq_enqueue(ifq->ifcq_disc, - ifq->ifcq_disc_slots[i].cl, m, m_pftag(m))); + pktsched_pkt_encap(&pkt, ptype, p); + + ret = tcq_enqueue(ifq->ifcq_disc, + ifq->ifcq_disc_slots[i].cl, &pkt, t); + + if ((ret != 0) && (ret != CLASSQEQ_SUCCESS_FC)) { + pktsched_free_pkt(&pkt); + *pdrop = TRUE; + } else { + *pdrop = FALSE; + } + + switch (ret) { + case CLASSQEQ_DROP: + ret = ENOBUFS; + break; + case CLASSQEQ_DROP_FC: + ret = EQFULL; + break; + case CLASSQEQ_DROP_SP: + ret = EQSUSPENDED; + break; + case CLASSQEQ_SUCCESS_FC: + ret = EQFULL; + break; + case CLASSQEQ_SUCCESS: + ret = 0; + break; + default: + VERIFY(0); + } + return (ret); } /* @@ -949,16 +770,19 @@ tcq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) * CLASSQDQ_REMOVE must return the same packet if called immediately * after CLASSQDQ_POLL. */ -static struct mbuf * +static void * tcq_dequeue_tc_ifclassq(struct ifclassq *ifq, mbuf_svc_class_t sc, - cqdq_op_t op) + classq_pkt_type_t *ptype) { + pktsched_pkt_t pkt; u_int32_t i = MBUF_SCIDX(sc); VERIFY((u_int32_t)i < IFCQ_SC_MAX); - return (tcq_dequeue_cl(ifq->ifcq_disc, - ifq->ifcq_disc_slots[i].cl, sc, op)); + bzero(&pkt, sizeof (pkt)); + (tcq_dequeue_cl(ifq->ifcq_disc, ifq->ifcq_disc_slots[i].cl, sc, &pkt)); + *ptype = pkt.pktsched_ptype; + return (pkt.pktsched_pkt); } static int @@ -994,7 +818,8 @@ tcq_request_ifclassq(struct ifclassq *ifq, cqrq_t req, void *arg) } int -tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) +tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags, + classq_pkt_type_t ptype) { struct ifnet *ifp = ifq->ifcq_ifp; struct tcq_class *cl0, *cl1, *cl2, *cl3; @@ -1006,12 +831,6 @@ tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) VERIFY(ifq->ifcq_disc == NULL); VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); - if (flags & PKTSCHEDF_QALG_RED) - qflags |= TQCF_RED; - if (flags & PKTSCHEDF_QALG_RIO) - qflags |= TQCF_RIO; - if (flags & PKTSCHEDF_QALG_BLUE) - qflags |= TQCF_BLUE; if (flags & PKTSCHEDF_QALG_SFB) qflags |= TQCF_SFB; if (flags & PKTSCHEDF_QALG_ECN) @@ -1021,7 +840,7 @@ tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) if (flags & PKTSCHEDF_QALG_DELAYBASED) qflags |= TQCF_DELAYBASED; - tif = tcq_alloc(ifp, M_WAITOK, FALSE); + tif = tcq_alloc(ifp, M_WAITOK); if (tif == NULL) return (ENOMEM); @@ -1029,24 +848,24 @@ tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) maxlen = if_sndq_maxlen; if ((err = tcq_add_queue(tif, 0, maxlen, - qflags | PRCF_LAZY, SCIDX_BK, &cl0)) != 0) + qflags | TQCF_LAZY, SCIDX_BK, &cl0, ptype)) != 0) goto cleanup; if ((err = tcq_add_queue(tif, 1, maxlen, - qflags | TQCF_DEFAULTCLASS, SCIDX_BE, &cl1)) != 0) + qflags | TQCF_DEFAULTCLASS, SCIDX_BE, &cl1, ptype)) != 0) goto cleanup; if ((err = tcq_add_queue(tif, 2, maxlen, - qflags | PRCF_LAZY, SCIDX_VI, &cl2)) != 0) + qflags | TQCF_LAZY, SCIDX_VI, &cl2, ptype)) != 0) goto cleanup; if ((err = tcq_add_queue(tif, 3, maxlen, - qflags, SCIDX_VO, &cl3)) != 0) + qflags, SCIDX_VO, &cl3, ptype)) != 0) goto cleanup; err = ifclassq_attach(ifq, PKTSCHEDT_TCQ, tif, tcq_enqueue_ifclassq, NULL, tcq_dequeue_tc_ifclassq, - NULL, tcq_request_ifclassq); + NULL, NULL, tcq_request_ifclassq); /* cache these for faster lookup */ if (err == 0) { @@ -1136,7 +955,6 @@ tcq_throttle(struct tcq_if *tif, cqrq_throttle_t *tr) int err = 0; IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(!(tif->tif_flags & TCQIFF_ALTQ)); if (!tr->set) { tr->level = tif->tif_throttle; @@ -1189,24 +1007,11 @@ tcq_resumeq(struct tcq_if *tif, struct tcq_class *cl) { struct ifclassq *ifq = tif->tif_ifq; int err = 0; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, FALSE); @@ -1221,24 +1026,11 @@ tcq_suspendq(struct tcq_if *tif, struct tcq_class *cl) { struct ifclassq *ifq = tif->tif_ifq; int err = 0; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q)) { if (cl->cl_sfb != NULL) { err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, TRUE); diff --git a/bsd/net/pktsched/pktsched_tcq.h b/bsd/net/pktsched/pktsched_tcq.h index 57bb9fea0..91c2f71e8 100644 --- a/bsd/net/pktsched/pktsched_tcq.h +++ b/bsd/net/pktsched/pktsched_tcq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -52,7 +52,7 @@ extern "C" { #define TQCF_SFB 0x0200 /* use SFB */ #define TQCF_FLOWCTL 0x0400 /* enable flow control advisories */ #define TQCF_DEFAULTCLASS 0x1000 /* default class */ -#define TQCF_DELAYBASED 0x2000 /* queue sizing is delay based */ +#define TQCF_DELAYBASED 0x2000 /* queue sizing is delay based */ #ifdef BSD_KERNEL_PRIVATE #define TQCF_LAZY 0x10000000 /* on-demand resource allocation */ #endif /* BSD_KERNEL_PRIVATE */ @@ -98,9 +98,6 @@ struct tcq_class { u_int32_t cl_qflags; /* class queue flags */ union { void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ struct sfb *sfb; /* SFB state */ } cl_qalg; int32_t cl_pri; /* priority */ @@ -113,21 +110,14 @@ struct tcq_class { struct pktcntr cl_dropcnt; /* dropped packet counter */ }; -#define cl_red cl_qalg.red -#define cl_rio cl_qalg.rio -#define cl_blue cl_qalg.blue #define cl_sfb cl_qalg.sfb -/* tcq_if flags */ -#define TCQIFF_ALTQ 0x1 /* configured via PF/ALTQ */ - /* * tcq interface state */ struct tcq_if { struct ifclassq *tif_ifq; /* backpointer to ifclassq */ int tif_maxpri; /* max priority in use */ - u_int32_t tif_flags; /* flags */ u_int32_t tif_throttle; /* throttling level */ struct tcq_class *tif_default; /* default class */ struct tcq_class *tif_classes[TCQ_MAXPRI]; /* classes */ @@ -138,20 +128,19 @@ struct tcq_if { struct if_ifclassq_stats; extern void tcq_init(void); -extern struct tcq_if *tcq_alloc(struct ifnet *, int, boolean_t); +extern struct tcq_if *tcq_alloc(struct ifnet *, int); extern int tcq_destroy(struct tcq_if *); extern void tcq_purge(struct tcq_if *); extern void tcq_event(struct tcq_if *, cqev_t); extern int tcq_add_queue(struct tcq_if *, int, u_int32_t, int, u_int32_t, - struct tcq_class **); + struct tcq_class **, classq_pkt_type_t); extern int tcq_remove_queue(struct tcq_if *, u_int32_t); extern int tcq_get_class_stats(struct tcq_if *, u_int32_t, struct tcq_classstats *); -extern int tcq_enqueue(struct tcq_if *, struct tcq_class *, struct mbuf *, +extern int tcq_enqueue(struct tcq_if *, struct tcq_class *, pktsched_pkt_t *, struct pf_mtag *); -extern struct mbuf *tcq_dequeue_tc(struct tcq_if *, mbuf_svc_class_t, - cqdq_op_t); -extern int tcq_setup_ifclassq(struct ifclassq *, u_int32_t); +extern void tcq_dequeue_tc(struct tcq_if *, mbuf_svc_class_t, pktsched_pkt_t *); +extern int tcq_setup_ifclassq(struct ifclassq *, u_int32_t, classq_pkt_type_t); extern int tcq_teardown_ifclassq(struct ifclassq *ifq); extern int tcq_getqstats_ifclassq(struct ifclassq *, u_int32_t qid, struct if_ifclassq_stats *); diff --git a/bsd/net/radix.h b/bsd/net/radix.h index d48399aae..78f251b22 100644 --- a/bsd/net/radix.h +++ b/bsd/net/radix.h @@ -102,7 +102,15 @@ struct radix_node { struct radix_node *rn_ybro; #endif +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Aligned to 64-bit since this is cast to rtentry, which is 64-bit aligned. + */ +} __attribute__ ((aligned(8))); +#else }; +#endif #define rn_dupedkey rn_u.rn_leaf.rn_Dupedkey #define rn_key rn_u.rn_leaf.rn_Key diff --git a/bsd/net/raw_cb.h b/bsd/net/raw_cb.h index c944ec005..39a63716c 100644 --- a/bsd/net/raw_cb.h +++ b/bsd/net/raw_cb.h @@ -90,7 +90,7 @@ extern LIST_HEAD(rawcb_list_head, rawcb) rawcb_list; __BEGIN_DECLS extern int raw_attach(struct socket *, int); -extern void raw_ctlinput(int, struct sockaddr *, void *); +extern void raw_ctlinput(int, struct sockaddr *, void *, struct ifnet *); extern void raw_detach(struct rawcb *); extern void raw_disconnect(struct rawcb *); extern void raw_init(struct protosw *, struct domain *); diff --git a/bsd/net/raw_usrreq.c b/bsd/net/raw_usrreq.c index 657f47d06..462842129 100644 --- a/bsd/net/raw_usrreq.c +++ b/bsd/net/raw_usrreq.c @@ -175,7 +175,8 @@ raw_input(struct mbuf *m0, struct sockproto *proto, struct sockaddr *src, /*ARGSUSED*/ void -raw_ctlinput(int cmd, __unused struct sockaddr *arg, __unused void *dummy) +raw_ctlinput(int cmd, __unused struct sockaddr *arg, __unused void *dummy, + __unused struct ifnet *ifp) { if (cmd < 0 || cmd >= PRC_NCMDS) @@ -193,7 +194,7 @@ raw_uabort(struct socket *so) mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (rp == 0) return EINVAL; @@ -242,7 +243,7 @@ raw_udetach(struct socket *so) mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (rp == 0) return EINVAL; @@ -296,7 +297,7 @@ raw_usend(struct socket *so, int flags, struct mbuf *m, mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (rp == 0) { error = EINVAL; @@ -348,7 +349,7 @@ raw_ushutdown(struct socket *so) mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (rp == 0) return EINVAL; diff --git a/bsd/net/route.c b/bsd/net/route.c index 96633f833..80d336a40 100644 --- a/bsd/net/route.c +++ b/bsd/net/route.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -83,11 +83,16 @@ #include #include #include +#include +#if NECP +#include +#endif /* NECP */ #include #include #include #include +#include #if INET6 #include @@ -208,7 +213,7 @@ extern void kdp_set_gateway_mac(void *gatewaymac); -__private_extern__ struct rtstat rtstat = { 0, 0, 0, 0, 0 }; +__private_extern__ struct rtstat rtstat = { 0, 0, 0, 0, 0, 0 }; struct radix_node_head *rt_tables[AF_MAX+1]; decl_lck_mtx_data(, rnh_lock_data); /* global routing tables mutex */ @@ -224,7 +229,7 @@ static lck_grp_attr_t *rte_mtx_grp_attr; int rttrash = 0; /* routes not in table but not freed */ -unsigned int rte_debug; +unsigned int rte_debug = 0; /* Possible flags for rte_debug */ #define RTD_DEBUG 0x1 /* enable or disable rtentry debug facility */ @@ -319,6 +324,7 @@ static void rtfree_common(struct rtentry *, boolean_t); static void rte_if_ref(struct ifnet *, int); static void rt_set_idleref(struct rtentry *); static void rt_clear_idleref(struct rtentry *); +static void route_event_callback(void *); static void rt_str4(struct rtentry *, char *, uint32_t, char *, uint32_t); #if INET6 static void rt_str6(struct rtentry *, char *, uint32_t, char *, uint32_t); @@ -413,6 +419,8 @@ route_init(void) #if INET6 _CASSERT(offsetof(struct route, ro_rt) == offsetof(struct route_in6, ro_rt)); + _CASSERT(offsetof(struct route, ro_lle) == + offsetof(struct route_in6, ro_lle)); _CASSERT(offsetof(struct route, ro_srcia) == offsetof(struct route_in6, ro_srcia)); _CASSERT(offsetof(struct route, ro_flags) == @@ -699,7 +707,7 @@ sa_trim(struct sockaddr *sa, int skip) */ struct sockaddr * rtm_scrub(int type, int idx, struct sockaddr *hint, struct sockaddr *sa, - void *buf, uint32_t buflen, kauth_cred_t *credp, uint32_t rtm_hint_flags) + void *buf, uint32_t buflen, kauth_cred_t *credp) { struct sockaddr_storage *ss = (struct sockaddr_storage *)buf; struct sockaddr *ret = sa; @@ -782,8 +790,7 @@ rtm_scrub(int type, int idx, struct sockaddr *hint, struct sockaddr *sa, /* fallthrough */ } case RTAX_IFP: { - if (sa->sa_family == AF_LINK && credp && - (rtm_hint_flags & RTMF_HIDE_LLADDR)) { + if (sa->sa_family == AF_LINK && credp) { struct sockaddr_dl *sdl = SDL(buf); const void *bytes; size_t size; @@ -893,7 +900,7 @@ rtalloc_ign_common_locked(struct route *ro, uint32_t ignore, void rtalloc_ign(struct route *ro, uint32_t ignore) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); rtalloc_ign_common_locked(ro, ignore, IFSCOPE_NONE); lck_mtx_unlock(rnh_lock); @@ -902,7 +909,7 @@ rtalloc_ign(struct route *ro, uint32_t ignore) void rtalloc_scoped_ign(struct route *ro, uint32_t ignore, unsigned int ifscope) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); rtalloc_ign_common_locked(ro, ignore, ifscope); lck_mtx_unlock(rnh_lock); @@ -1039,7 +1046,7 @@ struct rtentry * rtalloc1(struct sockaddr *dst, int report, uint32_t ignflags) { struct rtentry *entry; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); entry = rtalloc1_locked(dst, report, ignflags); lck_mtx_unlock(rnh_lock); @@ -1051,7 +1058,7 @@ rtalloc1_scoped(struct sockaddr *dst, int report, uint32_t ignflags, unsigned int ifscope) { struct rtentry *entry; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); entry = rtalloc1_scoped_locked(dst, report, ignflags, ifscope); lck_mtx_unlock(rnh_lock); @@ -1073,7 +1080,7 @@ rtfree_common(struct rtentry *rt, boolean_t locked) { struct radix_node_head *rnh; - lck_mtx_assert(rnh_lock, locked ? + LCK_MTX_ASSERT(rnh_lock, locked ? LCK_MTX_ASSERT_OWNED : LCK_MTX_ASSERT_NOTOWNED); /* @@ -1119,7 +1126,7 @@ rtfree_common(struct rtentry *rt, boolean_t locked) */ RT_CONVERT_LOCK(rt); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* Negative refcnt must never happen */ if (rt->rt_refcnt != 0) { @@ -1154,6 +1161,7 @@ rtfree_common(struct rtentry *rt, boolean_t locked) struct rtentry *rt_parent; struct ifaddr *rt_ifa; + rt->rt_flags |= RTF_DEAD; if (rt->rt_nodes->rn_flags & (RNF_ACTIVE | RNF_ROOT)) { panic("rt %p freed while in radix tree\n", rt); /* NOTREACHED */ @@ -1189,11 +1197,15 @@ rtfree_common(struct rtentry *rt, boolean_t locked) rt->rt_llinfo = NULL; } + /* Destroy eventhandler lists context */ + eventhandler_lists_ctxt_destroy(&rt->rt_evhdlr_ctxt); + /* * Route is no longer in the tree and refcnt is 0; * we have exclusive access, so destroy it. */ RT_UNLOCK(rt); + rte_lock_destroy(rt); if (rt_parent != NULL) rtfree_locked(rt_parent); @@ -1216,7 +1228,6 @@ rtfree_common(struct rtentry *rt, boolean_t locked) /* * and the rtentry itself of course */ - rte_lock_destroy(rt); rte_free(rt); } else { /* @@ -1289,6 +1300,7 @@ rtref(struct rtentry *p) { RT_LOCK_ASSERT_HELD(p); + VERIFY((p->rt_flags & RTF_DEAD) == 0); if (++p->rt_refcnt == 0) { panic("%s(%p) bad refcnt\n", __func__, p); /* NOTREACHED */ @@ -1321,7 +1333,7 @@ rtref_audit(struct rtentry_dbg *rte) void rtsetifa(struct rtentry *rt, struct ifaddr *ifa) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); @@ -1363,7 +1375,7 @@ rtredirect(struct ifnet *ifp, struct sockaddr *dst, struct sockaddr *gateway, struct sockaddr_storage ss; int af = src->sa_family; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); /* @@ -1554,7 +1566,7 @@ ifa_ifwithroute_common_locked(int flags, const struct sockaddr *dst, struct rtentry *rt = NULL; struct sockaddr_storage dst_ss, gw_ss; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* * Just in case the sockaddr passed in by the caller @@ -1734,7 +1746,7 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, #define senderr(x) { error = x; goto bad; } - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* * Find the correct routing tree to use for this Address Family */ @@ -1785,6 +1797,8 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, switch (req) { case RTM_DELETE: { struct rtentry *gwrt = NULL; + boolean_t was_router = FALSE; + uint32_t old_rt_refcnt = 0; /* * Remove the item from the tree and return it. * Complain if it is not there and do no more processing. @@ -1798,6 +1812,7 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, rt = (struct rtentry *)rn; RT_LOCK(rt); + old_rt_refcnt = rt->rt_refcnt; rt->rt_flags &= ~RTF_UP; /* * Release any idle reference count held on the interface @@ -1824,10 +1839,21 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, * Clear RTF_ROUTER if it's set. */ if (rt->rt_flags & RTF_ROUTER) { + was_router = TRUE; VERIFY(rt->rt_flags & RTF_HOST); rt->rt_flags &= ~RTF_ROUTER; } + /* + * Enqueue work item to invoke callback for this route entry + * + * If the old count is 0, it implies that last reference is being + * removed and there's no one listening for this route event. + */ + if (old_rt_refcnt != 0) + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_ENTRY_DELETED, NULL, TRUE); + /* * Now search what's left of the subtree for any cloned * routes which might have been formed from this node. @@ -1840,6 +1866,15 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, RT_LOCK(rt); } + if (was_router) { + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_DELETED); + RT_UNLOCK(rt); + (void) rnh->rnh_walktree(rnh, + route_event_walktree, (void *)&rt_ev); + RT_LOCK(rt); + } + /* * Remove any external references we may have. */ @@ -1878,6 +1913,16 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, IFSCOPE_NONE); } +#if NECP + /* + * If this is a change in a default route, update + * necp client watchers to re-evaluate + */ + if (SA_DEFAULT(rt_key(rt))) { + necp_update_all_clients(); + } +#endif /* NECP */ + RT_UNLOCK(rt); /* @@ -2008,10 +2053,15 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, if (ifa == NULL) senderr(ENETUNREACH); makeroute: + /* + * We land up here for both RTM_RESOLVE and RTM_ADD + * when we decide to create a route. + */ if ((rt = rte_alloc()) == NULL) senderr(ENOBUFS); Bzero(rt, sizeof(*rt)); rte_lock_init(rt); + eventhandler_lists_ctxt_init(&rt->rt_evhdlr_ctxt); getmicrotime(&caltime); rt->base_calendartime = caltime.tv_sec; rt->base_uptime = net_uptime(); @@ -2172,6 +2222,16 @@ makeroute: rt->rt_ifp->if_index); } +#if NECP + /* + * If this is a change in a default route, update + * necp client watchers to re-evaluate + */ + if (SA_DEFAULT(rt_key(rt))) { + necp_update_all_clients(); + } +#endif /* NECP */ + /* * actually return a resultant rtentry and * give the caller a single reference. @@ -2225,7 +2285,7 @@ rtrequest(int req, struct sockaddr *dst, struct sockaddr *gateway, struct sockaddr *netmask, int flags, struct rtentry **ret_nrt) { int error; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); error = rtrequest_locked(req, dst, gateway, netmask, flags, ret_nrt); lck_mtx_unlock(rnh_lock); @@ -2238,7 +2298,7 @@ rtrequest_scoped(int req, struct sockaddr *dst, struct sockaddr *gateway, unsigned int ifscope) { int error; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); error = rtrequest_scoped_locked(req, dst, gateway, netmask, flags, ret_nrt, ifscope); @@ -2259,7 +2319,7 @@ rt_fixdelete(struct radix_node *rn, void *vp) struct rtentry *rt = (struct rtentry *)rn; struct rtentry *rt0 = vp; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); if (rt->rt_parent == rt0 && @@ -2300,7 +2360,7 @@ rt_fixchange(struct radix_node *rn, void *vp) u_char *xk1, *xm1, *xk2, *xmp; int i, len; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); @@ -2394,7 +2454,7 @@ rt_setgate(struct rtentry *rt, struct sockaddr *dst, struct sockaddr *gate) } rnh = rt_tables[dst->sa_family]; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); /* @@ -2546,6 +2606,16 @@ rt_setgate(struct rtentry *rt, struct sockaddr *dst, struct sockaddr *gate) rt->rt_ifp->if_index); } +#if NECP + /* + * If this is a change in a default route, update + * necp client watchers to re-evaluate + */ + if (SA_DEFAULT(dst)) { + necp_update_all_clients(); + } +#endif /* NECP */ + /* * Tell the kernel debugger about the new default gateway * if the gateway route uses the primary interface, or @@ -2647,7 +2717,7 @@ rt_set_gwroute(struct rtentry *rt, struct sockaddr *dst, struct rtentry *gwrt) { boolean_t gwrt_isrouter; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (gwrt != NULL) @@ -2833,7 +2903,7 @@ rt_lookup_common(boolean_t lookup_only, boolean_t coarse, struct sockaddr *dst, #endif VERIFY(!coarse || ifscope == IFSCOPE_NONE); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); #if INET6 /* * While we have rnh_lock held, see if we need to schedule the timer. @@ -3109,7 +3179,7 @@ rtinit(struct ifaddr *ifa, int cmd, int flags) { int error; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); error = rtinit_locked(ifa, cmd, flags); @@ -3137,7 +3207,7 @@ rtinit_locked(struct ifaddr *ifa, int cmd, int flags) * changing (e.g. in_ifinit), so it is safe to access its * ifa_{dst}addr (here and down below) without locking. */ - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (flags & RTF_HOST) { dst = ifa->ifa_dstaddr; @@ -3681,6 +3751,10 @@ route_copyout(struct route *dst, const struct route *src, size_t length) if (dst->ro_rt != NULL) RT_ADDREF(dst->ro_rt); + /* Hold one reference for the local copy of struct lle */ + if (dst->ro_lle != NULL) + LLE_ADDREF(dst->ro_lle); + /* Hold one reference for the local copy of struct ifaddr */ if (dst->ro_srcia != NULL) IFA_ADDREF(dst->ro_srcia); @@ -3689,8 +3763,18 @@ route_copyout(struct route *dst, const struct route *src, size_t length) void route_copyin(struct route *src, struct route *dst, size_t length) { - /* No cached route at the destination? */ + /* + * No cached route at the destination? + * If none, then remove old references if present + * and copy entire src route. + */ if (dst->ro_rt == NULL) { + /* + * Ditch the cached link layer reference (dst) + * since we're about to take everything there is in src + */ + if (dst->ro_lle != NULL) + LLE_REMREF(dst->ro_lle); /* * Ditch the address in the cached copy (dst) since * we're about to take everything there is in src. @@ -3698,42 +3782,78 @@ route_copyin(struct route *src, struct route *dst, size_t length) if (dst->ro_srcia != NULL) IFA_REMREF(dst->ro_srcia); /* - * Copy everything (rt, srcia, flags, dst) from src; the + * Copy everything (rt, ro_lle, srcia, flags, dst) from src; the * references to rt and/or srcia were held at the time * of storage and are kept intact. */ bcopy(src, dst, length); - } else if (src->ro_rt != NULL) { - /* - * If the same, update srcia and flags, and ditch the route - * in the local copy. Else ditch the one that is currently - * cached, and cache the new route. - */ - if (dst->ro_rt == src->ro_rt) { - dst->ro_flags = src->ro_flags; - if (dst->ro_srcia != src->ro_srcia) { - if (dst->ro_srcia != NULL) - IFA_REMREF(dst->ro_srcia); - dst->ro_srcia = src->ro_srcia; - } else if (src->ro_srcia != NULL) { - IFA_REMREF(src->ro_srcia); - } - rtfree(src->ro_rt); - } else { - rtfree(dst->ro_rt); + goto done; + } + + /* + * We know dst->ro_rt is not NULL here. + * If the src->ro_rt is the same, update ro_lle, srcia and flags + * and ditch the route in the local copy. + */ + if (dst->ro_rt == src->ro_rt) { + dst->ro_flags = src->ro_flags; + + if (dst->ro_lle != src->ro_lle) { + if (dst->ro_lle != NULL) + LLE_REMREF(dst->ro_lle); + dst->ro_lle = src->ro_lle; + } else if (src->ro_lle != NULL) { + LLE_REMREF(src->ro_lle); + } + + if (dst->ro_srcia != src->ro_srcia) { if (dst->ro_srcia != NULL) IFA_REMREF(dst->ro_srcia); - bcopy(src, dst, length); + dst->ro_srcia = src->ro_srcia; + } else if (src->ro_srcia != NULL) { + IFA_REMREF(src->ro_srcia); } - } else if (src->ro_srcia != NULL) { + rtfree(src->ro_rt); + goto done; + } + + /* + * If they are dst's ro_rt is not equal to src's, + * and src'd rt is not NULL, then remove old references + * if present and copy entire src route. + */ + if (src->ro_rt != NULL) { + rtfree(dst->ro_rt); + + if (dst->ro_lle != NULL) + LLE_REMREF(dst->ro_lle); + if (dst->ro_srcia != NULL) + IFA_REMREF(dst->ro_srcia); + bcopy(src, dst, length); + goto done; + } + + /* + * Here, dst's cached route is not NULL but source's is. + * Just get rid of all the other cached reference in src. + */ + if (src->ro_srcia != NULL) { /* * Ditch src address in the local copy (src) since we're * not caching the route entry anyway (ro_rt is NULL). */ IFA_REMREF(src->ro_srcia); } - + if (src->ro_lle != NULL) { + /* + * Ditch cache lle in the local copy (src) since we're + * not caching the route anyway (ro_rt is NULL). + */ + LLE_REMREF(src->ro_lle); + } +done: /* This function consumes the references on src */ + src->ro_lle = NULL; src->ro_rt = NULL; src->ro_srcia = NULL; } @@ -4075,3 +4195,173 @@ rt_str(struct rtentry *rt, char *ds, uint32_t dslen, char *gs, uint32_t gslen) break; } } + +void route_event_init(struct route_event *p_route_ev, struct rtentry *rt, + struct rtentry *gwrt, int route_ev_code) +{ + VERIFY(p_route_ev != NULL); + bzero(p_route_ev, sizeof(*p_route_ev)); + + p_route_ev->rt = rt; + p_route_ev->gwrt = gwrt; + p_route_ev->route_event_code = route_ev_code; +} + +static void +route_event_callback(void *arg) +{ + struct route_event *p_rt_ev = (struct route_event *)arg; + struct rtentry *rt = p_rt_ev->rt; + eventhandler_tag evtag = p_rt_ev->evtag; + int route_ev_code = p_rt_ev->route_event_code; + + if (route_ev_code == ROUTE_EVHDLR_DEREGISTER) { + VERIFY(evtag != NULL); + EVENTHANDLER_DEREGISTER(&rt->rt_evhdlr_ctxt, route_event, + evtag); + rtfree(rt); + return; + } + + EVENTHANDLER_INVOKE(&rt->rt_evhdlr_ctxt, route_event, rt_key(rt), + route_ev_code, (struct sockaddr *)&p_rt_ev->rt_addr, + rt->rt_flags); + + /* The code enqueuing the route event held a reference */ + rtfree(rt); + /* XXX No reference is taken on gwrt */ +} + +int +route_event_walktree(struct radix_node *rn, void *arg) +{ + struct route_event *p_route_ev = (struct route_event *)arg; + struct rtentry *rt = (struct rtentry *)rn; + struct rtentry *gwrt = p_route_ev->rt; + + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); + + RT_LOCK(rt); + + /* Return if the entry is pending cleanup */ + if (rt->rt_flags & RTPRF_OURS) { + RT_UNLOCK(rt); + return (0); + } + + /* Return if it is not an indirect route */ + if (!(rt->rt_flags & RTF_GATEWAY)) { + RT_UNLOCK(rt); + return (0); + } + + if (rt->rt_gwroute != gwrt) { + RT_UNLOCK(rt); + return (0); + } + + route_event_enqueue_nwk_wq_entry(rt, gwrt, p_route_ev->route_event_code, + NULL, TRUE); + RT_UNLOCK(rt); + + return (0); +} + +struct route_event_nwk_wq_entry +{ + struct nwk_wq_entry nwk_wqe; + struct route_event rt_ev_arg; +}; + +void +route_event_enqueue_nwk_wq_entry(struct rtentry *rt, struct rtentry *gwrt, + uint32_t route_event_code, eventhandler_tag evtag, boolean_t rt_locked) +{ + struct route_event_nwk_wq_entry *p_rt_ev = NULL; + struct sockaddr *p_gw_saddr = NULL; + + MALLOC(p_rt_ev, struct route_event_nwk_wq_entry *, + sizeof(struct route_event_nwk_wq_entry), + M_NWKWQ, M_WAITOK | M_ZERO); + + /* + * If the intent is to de-register, don't take + * reference, route event registration already takes + * a reference on route. + */ + if (route_event_code != ROUTE_EVHDLR_DEREGISTER) { + /* The reference is released by route_event_callback */ + if (rt_locked) + RT_ADDREF_LOCKED(rt); + else + RT_ADDREF(rt); + } + + p_rt_ev->rt_ev_arg.rt = rt; + p_rt_ev->rt_ev_arg.gwrt = gwrt; + p_rt_ev->rt_ev_arg.evtag = evtag; + + if (gwrt != NULL) + p_gw_saddr = gwrt->rt_gateway; + else + p_gw_saddr = rt->rt_gateway; + + VERIFY(p_gw_saddr->sa_len <= sizeof(p_rt_ev->rt_ev_arg.rt_addr)); + bcopy(p_gw_saddr, &(p_rt_ev->rt_ev_arg.rt_addr), p_gw_saddr->sa_len); + + p_rt_ev->rt_ev_arg.route_event_code = route_event_code; + p_rt_ev->nwk_wqe.func = route_event_callback; + p_rt_ev->nwk_wqe.is_arg_managed = TRUE; + p_rt_ev->nwk_wqe.arg = &p_rt_ev->rt_ev_arg; + nwk_wq_enqueue((struct nwk_wq_entry*)p_rt_ev); +} + +const char * +route_event2str(int route_event) +{ + const char *route_event_str = "ROUTE_EVENT_UNKNOWN"; + switch (route_event) { + case ROUTE_STATUS_UPDATE: + route_event_str = "ROUTE_STATUS_UPDATE"; + break; + case ROUTE_ENTRY_REFRESH: + route_event_str = "ROUTE_ENTRY_REFRESH"; + break; + case ROUTE_ENTRY_DELETED: + route_event_str = "ROUTE_ENTRY_DELETED"; + break; + case ROUTE_LLENTRY_RESOLVED: + route_event_str = "ROUTE_LLENTRY_RESOLVED"; + break; + case ROUTE_LLENTRY_UNREACH: + route_event_str = "ROUTE_LLENTRY_UNREACH"; + break; + case ROUTE_LLENTRY_CHANGED: + route_event_str = "ROUTE_LLENTRY_CHANGED"; + break; + case ROUTE_LLENTRY_STALE: + route_event_str = "ROUTE_LLENTRY_STALE"; + break; + case ROUTE_LLENTRY_TIMEDOUT: + route_event_str = "ROUTE_LLENTRY_TIMEDOUT"; + break; + case ROUTE_LLENTRY_DELETED: + route_event_str = "ROUTE_LLENTRY_DELETED"; + break; + case ROUTE_LLENTRY_EXPIRED: + route_event_str = "ROUTE_LLENTRY_EXPIRED"; + break; + case ROUTE_LLENTRY_PROBED: + route_event_str = "ROUTE_LLENTRY_PROBED"; + break; + case ROUTE_EVHDLR_DEREGISTER: + route_event_str = "ROUTE_EVHDLR_DEREGISTER"; + break; + default: + /* Init'd to ROUTE_EVENT_UNKNOWN */ + break; + } + return route_event_str; +} + + diff --git a/bsd/net/route.h b/bsd/net/route.h index e02d052d9..1e01812ec 100644 --- a/bsd/net/route.h +++ b/bsd/net/route.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -83,7 +83,8 @@ struct rt_metrics { u_int32_t rmx_rtt; /* estimated round trip time */ u_int32_t rmx_rttvar; /* estimated rtt variance */ u_int32_t rmx_pksent; /* packets sent using this route */ - u_int32_t rmx_filler[4]; /* will be used for T/TCP later */ + u_int32_t rmx_state; /* route state */ + u_int32_t rmx_filler[3]; /* will be used for T/TCP later */ }; /* @@ -102,6 +103,9 @@ struct route_old { #ifdef BSD_KERNEL_PRIVATE #include #include +#include +#include +#include /* * Kernel resident routing tables. @@ -123,17 +127,30 @@ struct rt_reach_info; */ struct route { /* - * N.B: struct route must begin with ro_{rt,srcia,flags} + * N.B: struct route must begin with ro_{rt, lle, srcia, flags} * because the code does some casts of a 'struct route_in6 *' * to a 'struct route *'. */ struct rtentry *ro_rt; + struct llentry *ro_lle; + struct ifaddr *ro_srcia; uint32_t ro_flags; /* route flags (see below) */ struct sockaddr ro_dst; }; -#define ROF_SRCIF_SELECTED 0x1 /* source interface was selected */ +#define ROF_SRCIF_SELECTED 0x0001 /* source interface was selected */ +#if 0 +/* XXX These will be used in the changes coming in later */ +#define ROF_NORTREF 0x0002 /* doesn't hold reference on ro_rt */ +#define ROF_L2_ME 0x0004 /* dst L2 addr is our address */ +#define ROF_MAY_LOOP 0x0008 /* dst may require loop copy */ +#define ROF_HAS_HEADER 0x0010 /* mbuf already have its header prepended */ +#define ROF_REJECT 0x0020 /* Destination is reject */ +#define ROF_BLACKHOLE 0x0040 /* Destination is blackhole */ +#define ROF_HAS_GW 0x0080 /* Destination has GW */ +#endif +#define ROF_LLE_CACHE 0x0100 /* Cache link layer */ #define ROUTE_UNUSABLE(_ro) \ ((_ro)->ro_rt == NULL || \ @@ -154,6 +171,11 @@ struct route { (_ro)->ro_srcia = NULL; \ (_ro)->ro_flags &= ~ROF_SRCIF_SELECTED; \ } \ + if ((_ro)->ro_lle != NULL) { \ + LLE_REMREF((_ro)->ro_lle); \ + (_ro)->ro_lle = NULL; \ + (_ro)->ro_flags &= ~ROF_LLE_CACHE; \ + } \ } while (0) #define ROUTE_RELEASE_LOCKED(_ro) _ROUTE_RELEASE_COMMON(_ro, TRUE) @@ -210,8 +232,31 @@ struct rtentry { u_int32_t rtt_min; /* minimum RTT computed from history */ u_int32_t rtt_expire_ts; /* RTT history expire timestamp */ u_int8_t rtt_index; /* Index into RTT history */ + /* Event handler context for the rtentrt */ + struct eventhandler_lists_ctxt rt_evhdlr_ctxt; +}; + +enum { + ROUTE_STATUS_UPDATE = 1, + ROUTE_ENTRY_REFRESH, + ROUTE_ENTRY_DELETED, + ROUTE_LLENTRY_RESOLVED, + ROUTE_LLENTRY_UNREACH, + ROUTE_LLENTRY_CHANGED, + ROUTE_LLENTRY_STALE, + ROUTE_LLENTRY_TIMEDOUT, + ROUTE_LLENTRY_DELETED, + ROUTE_LLENTRY_EXPIRED, + ROUTE_LLENTRY_PROBED, + ROUTE_EVHDLR_DEREGISTER, }; +extern const char * route_event2str(int route_event); + +typedef void (*route_event_fn) (struct eventhandler_entry_arg, + struct sockaddr *, int, struct sockaddr *, int); +EVENTHANDLER_DECLARE(route_event, route_event_fn); + /* * Synchronize route entry's generation ID with the tree's. */ @@ -239,7 +284,9 @@ struct rtentry { #define RTF_DELCLONE 0x80 /* delete cloned route */ #define RTF_CLONING 0x100 /* generate new routes on use */ #define RTF_XRESOLVE 0x200 /* external daemon resolves name */ -#define RTF_LLINFO 0x400 /* generated by link layer (e.g. ARP) */ +#define RTF_LLINFO 0x400 /* DEPRECATED - exists ONLY for backward + compatibility */ +#define RTF_LLDATA 0x400 /* used by apps to add/del L2 entries */ #define RTF_STATIC 0x800 /* manually added */ #define RTF_BLACKHOLE 0x1000 /* just discard pkts (during updates) */ #define RTF_NOIFREF 0x2000 /* not eligible for RTF_IFREF */ @@ -259,8 +306,10 @@ struct rtentry { #define RTF_IFREF 0x4000000 /* route holds a ref to interface */ #define RTF_PROXY 0x8000000 /* proxying, no interface scope */ #define RTF_ROUTER 0x10000000 /* host is a router */ - /* 0x20000000 and up unassigned */ +#define RTF_DEAD 0x20000000 /* Route entry is being freed */ + /* 0x40000000 and up unassigned */ +#define RTPRF_OURS RTF_PROTO3 /* set on routes we manage */ #define RTF_BITS \ "\020\1UP\2GATEWAY\3HOST\4REJECT\5DYNAMIC\6MODIFIED\7DONE" \ "\10DELCLONE\11CLONING\12XRESOLVE\13LLINFO\14STATIC\15BLACKHOLE" \ @@ -268,6 +317,8 @@ struct rtentry { "\25PINNED\26LOCAL\27BROADCAST\30MULTICAST\31IFSCOPE\32CONDEMNED" \ "\33IFREF\34PROXY\35ROUTER" +#define IS_DIRECT_HOSTROUTE(rt) \ + (((rt)->rt_flags & (RTF_HOST | RTF_GATEWAY)) == RTF_HOST) /* * Routing statistics. */ @@ -277,6 +328,7 @@ struct rtstat { short rts_newgateway; /* routes modified by redirects */ short rts_unreach; /* lookups which failed */ short rts_wildcard; /* lookups satisfied by a wildcard */ + short rts_badrtgwroute; /* route to gateway is not direct */ }; /* @@ -384,10 +436,6 @@ struct rt_msghdr_ext { #define RTM_GET_EXT 0x15 #endif /* PRIVATE */ -#ifdef BSD_KERNEL_PRIVATE -/* RTM flags */ -#define RTMF_HIDE_LLADDR 0x00000001 -#endif /* BSD_KERNEL_PRIVATE */ /* * Bitmask values for rtm_inits and rmx_locks. */ @@ -454,10 +502,10 @@ typedef struct ctrace { extern void ctrace_record(ctrace_t *); #define RT_LOCK_ASSERT_HELD(_rt) \ - lck_mtx_assert(&(_rt)->rt_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_rt)->rt_lock, LCK_MTX_ASSERT_OWNED) #define RT_LOCK_ASSERT_NOTHELD(_rt) \ - lck_mtx_assert(&(_rt)->rt_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_rt)->rt_lock, LCK_MTX_ASSERT_NOTOWNED) #define RT_LOCK(_rt) do { \ rt_lock(_rt, FALSE); \ @@ -581,7 +629,7 @@ extern unsigned int sin6_get_ifscope(struct sockaddr *); extern void rt_lock(struct rtentry *, boolean_t); extern void rt_unlock(struct rtentry *); extern struct sockaddr *rtm_scrub(int, int, struct sockaddr *, - struct sockaddr *, void *, uint32_t, kauth_cred_t *, uint32_t); + struct sockaddr *, void *, uint32_t, kauth_cred_t *); extern boolean_t rt_validate(struct rtentry *); extern void rt_set_proxy(struct rtentry *, boolean_t); extern void rt_set_gwroute(struct rtentry *, struct sockaddr *, @@ -597,5 +645,38 @@ extern void route_copyout(struct route *, const struct route *, size_t); extern boolean_t rt_ifa_is_dst(struct sockaddr *, struct ifaddr *); extern struct sockaddr *sa_copy(struct sockaddr *, struct sockaddr_storage *, unsigned int *); + +/* + * The following is used to enqueue work items for route events + * and also used to pass route event while walking the tree + */ +struct route_event { + struct rtentry *rt; + /* + * There's no reference taken on gwrt. + * We only use it to check whether we should + * point to rt_gateway or the embedded rt_addr + * structure. + */ + struct rtentry *gwrt; + union { + union sockaddr_in_4_6 _rtev_ipaddr; + struct sockaddr_dl _rtev_lladdr; + char _rtev_addr_bytes[DLIL_SDLMAXLEN]; + } rt_addr; + uint32_t route_event_code; + eventhandler_tag evtag; +}; + +#define rtev_ipaddr rt_addr._rtev_ipaddr +#define rtev_lladdr rt_addr._rtev_lladdr +#define rtev_addr_bytes rt_addr._rtev_addr_bytes + +extern void route_event_init(struct route_event *p_route_ev, struct rtentry *rt, + struct rtentry *gwrt, int route_ev_code); +extern int route_event_walktree(struct radix_node *rn, void *arg); +extern void route_event_enqueue_nwk_wq_entry(struct rtentry *, struct rtentry *, + uint32_t, eventhandler_tag, boolean_t); + #endif /* BSD_KERNEL_PRIVATE */ #endif /* _NET_ROUTE_H_ */ diff --git a/bsd/net/rtsock.c b/bsd/net/rtsock.c index f66639877..457eaf4eb 100644 --- a/bsd/net/rtsock.c +++ b/bsd/net/rtsock.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -75,6 +75,7 @@ #include #include #include +#include #include #include @@ -131,7 +132,7 @@ static void rt_setif(struct rtentry *, struct sockaddr *, struct sockaddr *, static int rt_xaddrs(caddr_t, caddr_t, struct rt_addrinfo *); static struct mbuf *rt_msg1(int, struct rt_addrinfo *); static int rt_msg2(int, struct rt_addrinfo *, caddr_t, struct walkarg *, - kauth_cred_t *, uint32_t); + kauth_cred_t *); static int sysctl_dumpentry(struct radix_node *rn, void *vw); static int sysctl_dumpentry_ext(struct radix_node *rn, void *vw); static int sysctl_iflist(int af, struct walkarg *w); @@ -145,6 +146,11 @@ SYSCTL_NODE(_net, PF_ROUTE, routetable, CTLFLAG_RD | CTLFLAG_LOCKED, SYSCTL_NODE(_net, OID_AUTO, route, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "routing"); +/* Align x to 1024 (only power of 2) assuming x is positive */ +#define ALIGN_BYTES(x) do { \ + x = P2ALIGN(x, 1024); \ +} while(0) + #define ROUNDUP32(a) \ ((a) > 0 ? (1 + (((a) - 1) | (sizeof (uint32_t) - 1))) : \ sizeof (uint32_t)) @@ -307,7 +313,7 @@ route_output(struct mbuf *m, struct socket *so) int sendonlytoself = 0; unsigned int ifscope = IFSCOPE_NONE; struct rawcb *rp = NULL; - uint32_t rtm_hint_flags = 0; + boolean_t is_router = FALSE; #define senderr(e) { error = (e); goto flush; } if (m == NULL || ((m->m_len < sizeof (intptr_t)) && (m = m_pullup(m, sizeof (intptr_t))) == NULL)) @@ -535,9 +541,6 @@ route_output(struct mbuf *m, struct socket *so) senderr(ESRCH); RT_LOCK(rt); - if (rt->rt_ifp == lo_ifp) - rtm_hint_flags |= RTMF_HIDE_LLADDR; - /* * Holding rnh_lock here prevents the possibility of * ifa from changing (e.g. in_ifinit), so it is safe @@ -577,9 +580,7 @@ report: } if (ifa2 != NULL) IFA_LOCK(ifa2); - - len = rt_msg2(rtm->rtm_type, &info, NULL, NULL, &cred, rtm_hint_flags); - + len = rt_msg2(rtm->rtm_type, &info, NULL, NULL, &cred); if (ifa2 != NULL) IFA_UNLOCK(ifa2); if (len > rtm->rtm_msglen) { @@ -596,10 +597,8 @@ report: } if (ifa2 != NULL) IFA_LOCK(ifa2); - (void) rt_msg2(rtm->rtm_type, &info, (caddr_t)rtm, - NULL, &cred, rtm_hint_flags); - + NULL, &cred); if (ifa2 != NULL) IFA_UNLOCK(ifa2); rtm->rtm_flags = rt->rt_flags; @@ -607,10 +606,14 @@ report: rtm->rtm_addrs = info.rti_addrs; if (ifa2 != NULL) IFA_REMREF(ifa2); + + kauth_cred_unref(&cred); break; } case RTM_CHANGE: + is_router = (rt->rt_flags & RTF_ROUTER) ? TRUE : FALSE; + if (info.rti_info[RTAX_GATEWAY] != NULL && (error = rt_setgate(rt, rt_key(rt), info.rti_info[RTAX_GATEWAY]))) { @@ -623,7 +626,7 @@ report: * the required gateway, then just use the old one. * This can happen if the user tries to change the * flags on the default route without changing the - * default gateway. Changing flags still doesn't work. + * default gateway. Changing flags still doesn't work. */ if ((rt->rt_flags & RTF_GATEWAY) && info.rti_info[RTAX_GATEWAY] == NULL) @@ -646,6 +649,24 @@ report: } if (info.rti_info[RTAX_GENMASK]) rt->rt_genmask = info.rti_info[RTAX_GENMASK]; + + /* + * Enqueue work item to invoke callback for this route entry + * This may not be needed always, but for now issue it anytime + * RTM_CHANGE gets called. + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, ROUTE_ENTRY_REFRESH, NULL, TRUE); + /* + * If the route is for a router, walk the tree to send refresh + * event to protocol cloned entries + */ + if (is_router) { + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_ENTRY_REFRESH); + RT_UNLOCK(rt); + (void) rnh->rnh_walktree(rnh, route_event_walktree, (void *)&rt_ev); + RT_LOCK(rt); + } /* FALLTHRU */ case RTM_LOCK: rt->rt_rmx.rmx_locks &= ~(rtm->rtm_inits); @@ -822,7 +843,7 @@ rt_setif(struct rtentry *rt, struct sockaddr *Ifpaddr, struct sockaddr *Ifaaddr, struct ifnet *ifp = NULL; void (*ifa_rtrequest)(int, struct rtentry *, struct sockaddr *); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); @@ -1082,7 +1103,7 @@ rt_msg1(int type, struct rt_addrinfo *rtinfo) /* Scrub away any trace of embedded interface scope */ sa = rtm_scrub(type, i, hint, sa, &ssbuf, - sizeof (ssbuf), NULL, 0); + sizeof (ssbuf), NULL); break; default: @@ -1107,7 +1128,7 @@ rt_msg1(int type, struct rt_addrinfo *rtinfo) static int rt_msg2(int type, struct rt_addrinfo *rtinfo, caddr_t cp, struct walkarg *w, - kauth_cred_t* credp, uint32_t rtm_hint_flags) + kauth_cred_t* credp) { int i; int len, dlen, rlen, second_time = 0; @@ -1173,12 +1194,12 @@ again: /* Scrub away any trace of embedded interface scope */ sa = rtm_scrub(type, i, hint, sa, &ssbuf, - sizeof (ssbuf), NULL, rtm_hint_flags); + sizeof (ssbuf), NULL); break; case RTAX_GATEWAY: case RTAX_IFP: sa = rtm_scrub(type, i, NULL, sa, &ssbuf, - sizeof (ssbuf), credp, rtm_hint_flags); + sizeof (ssbuf), credp); break; default: @@ -1298,7 +1319,7 @@ rt_newaddrmsg(int cmd, struct ifaddr *ifa, int error, struct rtentry *rt) struct ifnet *ifp = ifa->ifa_ifp; struct sockproto route_proto = { PF_ROUTE, 0 }; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (route_cb.any_count == 0) @@ -1493,7 +1514,6 @@ sysctl_dumpentry(struct radix_node *rn, void *vw) int error = 0, size; struct rt_addrinfo info; kauth_cred_t cred; - uint32_t rtm_hint_flags = 0; cred = kauth_cred_proc_ref(current_proc()); @@ -1506,11 +1526,8 @@ sysctl_dumpentry(struct radix_node *rn, void *vw) info.rti_info[RTAX_NETMASK] = rt_mask(rt); info.rti_info[RTAX_GENMASK] = rt->rt_genmask; - if (rt->rt_ifp == lo_ifp) - rtm_hint_flags |= RTMF_HIDE_LLADDR; - if (w->w_op != NET_RT_DUMP2) { - size = rt_msg2(RTM_GET, &info, NULL, w, &cred, rtm_hint_flags); + size = rt_msg2(RTM_GET, &info, NULL, w, &cred); if (w->w_req != NULL && w->w_tmem != NULL) { struct rt_msghdr *rtm = (struct rt_msghdr *)(void *)w->w_tmem; @@ -1526,7 +1543,7 @@ sysctl_dumpentry(struct radix_node *rn, void *vw) error = SYSCTL_OUT(w->w_req, (caddr_t)rtm, size); } } else { - size = rt_msg2(RTM_GET2, &info, NULL, w, &cred, rtm_hint_flags); + size = rt_msg2(RTM_GET2, &info, NULL, w, &cred); if (w->w_req != NULL && w->w_tmem != NULL) { struct rt_msghdr2 *rtm = (struct rt_msghdr2 *)(void *)w->w_tmem; @@ -1563,7 +1580,6 @@ sysctl_dumpentry_ext(struct radix_node *rn, void *vw) int error = 0, size; struct rt_addrinfo info; kauth_cred_t cred; - uint32_t rtm_hint_flags = 0; cred = kauth_cred_proc_ref(current_proc()); @@ -1576,10 +1592,7 @@ sysctl_dumpentry_ext(struct radix_node *rn, void *vw) info.rti_info[RTAX_NETMASK] = rt_mask(rt); info.rti_info[RTAX_GENMASK] = rt->rt_genmask; - if (rt->rt_ifp == lo_ifp) - rtm_hint_flags |= RTMF_HIDE_LLADDR; - - size = rt_msg2(RTM_GET_EXT, &info, NULL, w, &cred, rtm_hint_flags); + size = rt_msg2(RTM_GET_EXT, &info, NULL, w, &cred); if (w->w_req != NULL && w->w_tmem != NULL) { struct rt_msghdr_ext *ertm = (struct rt_msghdr_ext *)(void *)w->w_tmem; @@ -1628,7 +1641,7 @@ sysctl_iflist(int af, struct walkarg *w) struct ifnet *ifp; struct ifaddr *ifa; struct rt_addrinfo info; - int len, error = 0; + int len = 0, error = 0; int pass = 0; int total_len = 0, current_len = 0; char *total_buffer = NULL, *cp = NULL; @@ -1655,7 +1668,7 @@ sysctl_iflist(int af, struct walkarg *w) */ ifa = ifp->if_lladdr; info.rti_info[RTAX_IFP] = ifa->ifa_addr; - len = rt_msg2(RTM_IFINFO, &info, NULL, NULL, &cred, RTMF_HIDE_LLADDR); + len = rt_msg2(RTM_IFINFO, &info, NULL, NULL, &cred); if (pass == 0) { total_len += len; } else { @@ -1668,7 +1681,7 @@ sysctl_iflist(int af, struct walkarg *w) } info.rti_info[RTAX_IFP] = ifa->ifa_addr; len = rt_msg2(RTM_IFINFO, &info, - (caddr_t)cp, NULL, &cred, RTMF_HIDE_LLADDR); + (caddr_t)cp, NULL, &cred); info.rti_info[RTAX_IFP] = NULL; ifm = (struct if_msghdr *)(void *)cp; @@ -1677,6 +1690,14 @@ sysctl_iflist(int af, struct walkarg *w) if_data_internal_to_if_data(ifp, &ifp->if_data, &ifm->ifm_data); ifm->ifm_addrs = info.rti_addrs; + /* + * + * Round bytes only for non-platform + */ + if (!csproc_get_platform_binary(w->w_req->p)) { + ALIGN_BYTES(ifm->ifm_data.ifi_ibytes); + ALIGN_BYTES(ifm->ifm_data.ifi_obytes); + } cp += len; VERIFY(IS_P2ALIGNED(cp, sizeof (u_int32_t))); @@ -1692,7 +1713,7 @@ sysctl_iflist(int af, struct walkarg *w) info.rti_info[RTAX_NETMASK] = ifa->ifa_netmask; info.rti_info[RTAX_BRD] = ifa->ifa_dstaddr; len = rt_msg2(RTM_NEWADDR, &info, NULL, NULL, - &cred, RTMF_HIDE_LLADDR); + &cred); if (pass == 0) { total_len += len; } else { @@ -1704,7 +1725,7 @@ sysctl_iflist(int af, struct walkarg *w) break; } len = rt_msg2(RTM_NEWADDR, &info, - (caddr_t)cp, NULL, &cred, RTMF_HIDE_LLADDR); + (caddr_t)cp, NULL, &cred); ifam = (struct ifa_msghdr *)(void *)cp; ifam->ifam_index = @@ -1770,7 +1791,7 @@ sysctl_iflist2(int af, struct walkarg *w) struct ifnet *ifp; struct ifaddr *ifa; struct rt_addrinfo info; - int len, error = 0; + int len = 0, error = 0; int pass = 0; int total_len = 0, current_len = 0; char *total_buffer = NULL, *cp = NULL; @@ -1799,7 +1820,7 @@ sysctl_iflist2(int af, struct walkarg *w) */ ifa = ifp->if_lladdr; info.rti_info[RTAX_IFP] = ifa->ifa_addr; - len = rt_msg2(RTM_IFINFO2, &info, NULL, NULL, &cred, RTMF_HIDE_LLADDR); + len = rt_msg2(RTM_IFINFO2, &info, NULL, NULL, &cred); if (pass == 0) { total_len += len; } else { @@ -1812,7 +1833,7 @@ sysctl_iflist2(int af, struct walkarg *w) } info.rti_info[RTAX_IFP] = ifa->ifa_addr; len = rt_msg2(RTM_IFINFO2, &info, - (caddr_t)cp, NULL, &cred, RTMF_HIDE_LLADDR); + (caddr_t)cp, NULL, &cred); info.rti_info[RTAX_IFP] = NULL; ifm = (struct if_msghdr2 *)(void *)cp; @@ -1826,6 +1847,14 @@ sysctl_iflist2(int af, struct walkarg *w) ifm->ifm_timer = ifp->if_timer; if_data_internal_to_if_data64(ifp, &ifp->if_data, &ifm->ifm_data); + /* + * + * Round bytes only for non-platform + */ + if (!csproc_get_platform_binary(w->w_req->p)) { + ALIGN_BYTES(ifm->ifm_data.ifi_ibytes); + ALIGN_BYTES(ifm->ifm_data.ifi_obytes); + } cp += len; VERIFY(IS_P2ALIGNED(cp, sizeof (u_int32_t))); @@ -1841,7 +1870,7 @@ sysctl_iflist2(int af, struct walkarg *w) info.rti_info[RTAX_NETMASK] = ifa->ifa_netmask; info.rti_info[RTAX_BRD] = ifa->ifa_dstaddr; len = rt_msg2(RTM_NEWADDR, &info, NULL, NULL, - &cred, RTMF_HIDE_LLADDR); + &cred); if (pass == 0) { total_len += len; } else { @@ -1853,7 +1882,7 @@ sysctl_iflist2(int af, struct walkarg *w) break; } len = rt_msg2(RTM_NEWADDR, &info, - (caddr_t)cp, NULL, &cred, RTMF_HIDE_LLADDR); + (caddr_t)cp, NULL, &cred); ifam = (struct ifa_msghdr *)(void *)cp; ifam->ifam_index = @@ -1897,7 +1926,7 @@ sysctl_iflist2(int af, struct walkarg *w) info.rti_info[RTAX_GATEWAY] = ifma->ifma_ll->ifma_addr; len = rt_msg2(RTM_NEWMADDR2, &info, NULL, NULL, - &cred, RTMF_HIDE_LLADDR); + &cred); if (pass == 0) { total_len += len; } else { @@ -1909,7 +1938,7 @@ sysctl_iflist2(int af, struct walkarg *w) break; } len = rt_msg2(RTM_NEWMADDR2, &info, - (caddr_t)cp, NULL, &cred, RTMF_HIDE_LLADDR); + (caddr_t)cp, NULL, &cred); ifmam = (struct ifma_msghdr2 *)(void *)cp; diff --git a/bsd/net/skmem_sysctl.c b/bsd/net/skmem_sysctl.c new file mode 100644 index 000000000..3fe18dc63 --- /dev/null +++ b/bsd/net/skmem_sysctl.c @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include + diff --git a/bsd/netinet/Makefile b/bsd/netinet/Makefile index ea575bddc..6eddebfe3 100644 --- a/bsd/netinet/Makefile +++ b/bsd/netinet/Makefile @@ -35,7 +35,8 @@ PRIVATE_DATAFILES = \ tcp_debug.h \ tcp_var.h \ tcp_cache.h \ - udp.h + udp.h \ + in_stat.h PRIVATE_KERNELFILES = ${KERNELFILES} \ ip_ecn.h ip_encap.h diff --git a/bsd/netinet/cpu_in_cksum.c b/bsd/netinet/cpu_in_cksum_gen.c similarity index 71% rename from bsd/netinet/cpu_in_cksum.c rename to bsd/netinet/cpu_in_cksum_gen.c index a579371a4..3d88e15e4 100644 --- a/bsd/netinet/cpu_in_cksum.c +++ b/bsd/netinet/cpu_in_cksum_gen.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -55,18 +55,124 @@ * SUCH DAMAGE. */ +#ifdef KERNEL #include -#include #include #include #include #include -#include #include +#include +#include +#define CKSUM_ERR(fmt, args...) kprintf(fmt, ## args) +#else /* !KERNEL */ +#ifndef LIBSYSCALL_INTERFACE +#error "LIBSYSCALL_INTERFACE not defined" +#endif /* !LIBSYSCALL_INTERFACE */ +#include +#include +#include +#include +#include +#include +#endif /* !KERNEL */ + +/* compile time assert */ +#ifndef _CASSERT +#define _CASSERT(x) _Static_assert(x, "compile-time assertion failed") +#endif /* !_CASSERT */ + +#ifndef VERIFY +#define VERIFY(EX) ((void)0) +#endif /* !VERIFY */ + +#ifndef CKSUM_ERR +#define CKSUM_ERR(fmt, args...) ((void)0) +#endif /* !CKSUM_ERR */ -int cpu_in_cksum(struct mbuf *, int, int, uint32_t); +#define PREDICT_TRUE(x) __builtin_expect(!!((long)(x)), 1L) +#define PREDICT_FALSE(x) __builtin_expect(!!((long)(x)), 0L) -#define PREDICT_FALSE(_exp) __builtin_expect((_exp), 0) +/* fake mbuf struct used only for calling os_cpu_in_cksum_mbuf() */ +struct _mbuf { + struct _mbuf *_m_next; + void *_m_pad; + uint8_t *_m_data; + int32_t _m_len; +}; + +extern uint32_t os_cpu_in_cksum(const void *, uint32_t, uint32_t); +extern uint32_t os_cpu_in_cksum_mbuf(struct _mbuf *, int, int, uint32_t); + +uint32_t +os_cpu_in_cksum(const void *data, uint32_t len, uint32_t initial_sum) +{ + /* + * If data is 4-bytes aligned, length is multiple of 4-bytes, + * and the amount to checksum is small, this would be quicker; + * this is suitable for IPv4 header. + */ + if (IS_P2ALIGNED(data, sizeof (uint32_t)) && + len <= 64 && (len & 3) == 0) { + uint8_t *p = __DECONST(uint8_t *, data); + uint64_t sum = initial_sum; + + if (PREDICT_TRUE(len == 20)) { /* simple IPv4 header */ + sum += *(uint32_t *)(void *)p; + sum += *(uint32_t *)(void *)(p + 4); + sum += *(uint32_t *)(void *)(p + 8); + sum += *(uint32_t *)(void *)(p + 12); + sum += *(uint32_t *)(void *)(p + 16); + } else { + while (len) { + sum += *(uint32_t *)(void *)p; + p += 4; + len -= 4; + } + } + + /* fold 64-bit to 16-bit (deferred carries) */ + sum = (sum >> 32) + (sum & 0xffffffff); /* 33-bit */ + sum = (sum >> 16) + (sum & 0xffff); /* 17-bit + carry */ + sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */ + sum = (sum >> 16) + (sum & 0xffff); /* final carry */ + + return (sum & 0xffff); + } + + /* + * Otherwise, let os_cpu_in_cksum_mbuf() handle it; it only looks + * at 3 fields: {next,data,len}, and since it doesn't care about + * the authenticity of the mbuf, we use a fake one here. Make + * sure the offsets are as expected. + */ +#if defined(__LP64__) + _CASSERT(offsetof(struct _mbuf, _m_next) == 0); + _CASSERT(offsetof(struct _mbuf, _m_data) == 16); + _CASSERT(offsetof(struct _mbuf, _m_len) == 24); +#else /* !__LP64__ */ + _CASSERT(offsetof(struct _mbuf, _m_next) == 0); + _CASSERT(offsetof(struct _mbuf, _m_data) == 8); + _CASSERT(offsetof(struct _mbuf, _m_len) == 12); +#endif /* !__LP64__ */ +#ifdef KERNEL + _CASSERT(offsetof(struct _mbuf, _m_next) == + offsetof(struct mbuf, m_next)); + _CASSERT(offsetof(struct _mbuf, _m_data) == + offsetof(struct mbuf, m_data)); + _CASSERT(offsetof(struct _mbuf, _m_len) == + offsetof(struct mbuf, m_len)); +#endif /* KERNEL */ + struct _mbuf m = { + ._m_next = NULL, + ._m_data = __DECONST(uint8_t *, data), + ._m_len = len, + }; + + return (os_cpu_in_cksum_mbuf(&m, len, 0, initial_sum)); +} + +#if defined(__i386__) || defined(__x86_64__) /* * Checksum routine for Internet Protocol family headers (Portable Version). @@ -88,10 +194,10 @@ int cpu_in_cksum(struct mbuf *, int, int, uint32_t); * reduction is done to avoid carry in long packets. */ -#if ULONG_MAX == 0xffffffffUL +#if !defined(__LP64__) /* 32-bit version */ -int -cpu_in_cksum(struct mbuf *m, int len, int off, uint32_t initial_sum) +uint32_t +os_cpu_in_cksum_mbuf(struct _mbuf *m, int len, int off, uint32_t initial_sum) { int mlen; uint32_t sum, partial; @@ -108,28 +214,28 @@ cpu_in_cksum(struct mbuf *m, int len, int off, uint32_t initial_sum) for (;;) { if (PREDICT_FALSE(m == NULL)) { - printf("%s: out of data\n", __func__); - return (-1); + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); } - mlen = m->m_len; + mlen = m->_m_len; if (mlen > off) { mlen -= off; - data = mtod(m, uint8_t *) + off; + data = m->_m_data + off; goto post_initial_offset; } off -= mlen; if (len == 0) break; - m = m->m_next; + m = m->_m_next; } - for (; len > 0; m = m->m_next) { + for (; len > 0; m = m->_m_next) { if (PREDICT_FALSE(m == NULL)) { - printf("%s: out of data\n", __func__); - return (-1); + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); } - mlen = m->m_len; - data = mtod(m, uint8_t *); + mlen = m->_m_len; + data = m->_m_data; post_initial_offset: if (mlen == 0) continue; @@ -231,13 +337,13 @@ post_initial_offset: } final_acc = ((sum >> 16) & 0xffff) + (sum & 0xffff); final_acc = (final_acc >> 16) + (final_acc & 0xffff); - return (~final_acc & 0xffff); + return (final_acc & 0xffff); } -#else +#else /* __LP64__ */ /* 64-bit version */ -int -cpu_in_cksum(struct mbuf *m, int len, int off, uint32_t initial_sum) +uint32_t +os_cpu_in_cksum_mbuf(struct _mbuf *m, int len, int off, uint32_t initial_sum) { int mlen; uint64_t sum, partial; @@ -254,28 +360,28 @@ cpu_in_cksum(struct mbuf *m, int len, int off, uint32_t initial_sum) for (;;) { if (PREDICT_FALSE(m == NULL)) { - printf("%s: out of data\n", __func__); - return (-1); + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); } - mlen = m->m_len; + mlen = m->_m_len; if (mlen > off) { mlen -= off; - data = mtod(m, uint8_t *) + off; + data = m->_m_data + off; goto post_initial_offset; } off -= mlen; if (len == 0) break; - m = m->m_next; + m = m->_m_next; } - for (; len > 0; m = m->m_next) { + for (; len > 0; m = m->_m_next) { if (PREDICT_FALSE(m == NULL)) { - printf("%s: out of data\n", __func__); - return (-1); + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); } - mlen = m->m_len; - data = mtod(m, uint8_t *); + mlen = m->_m_len; + data = m->_m_data; post_initial_offset: if (mlen == 0) continue; @@ -391,6 +497,8 @@ trailing_bytes: ((sum >> 16) & 0xffff) + (sum & 0xffff); final_acc = (final_acc >> 16) + (final_acc & 0xffff); final_acc = (final_acc >> 16) + (final_acc & 0xffff); - return (~final_acc & 0xffff); + return (final_acc & 0xffff); } -#endif /* ULONG_MAX != 0xffffffffUL */ +#endif /* __LP64 */ + +#endif /* __i386__ || __x86_64__ */ diff --git a/bsd/netinet/flow_divert.c b/bsd/netinet/flow_divert.c index 1e46e42c4..c5e974aab 100644 --- a/bsd/netinet/flow_divert.c +++ b/bsd/netinet/flow_divert.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -601,6 +601,7 @@ flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boo INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len); } + inp_set_activity_bitmap(inp); } static errno_t @@ -1037,7 +1038,11 @@ flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr if (signing_id != NULL) { uint16_t result = NULL_TRIE_IDX; lck_rw_lock_shared(&fd_cb->group->lck); - result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id); + if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) { + result = 1; + } else { + result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id); + } lck_rw_done(&fd_cb->group->lck); if (result != NULL_TRIE_IDX) { error = 0; @@ -1360,9 +1365,11 @@ flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t } } - last = m_last(packet); - mbuf_setnext(last, data); - mbuf_pkthdr_adjustlen(packet, data_len); + if (data_len > 0 && data != NULL) { + last = m_last(packet); + mbuf_setnext(last, data); + mbuf_pkthdr_adjustlen(packet, data_len); + } error = flow_divert_send_packet(fd_cb, packet, force); if (error) { @@ -1447,11 +1454,15 @@ flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force) } } data_len = mbuf_pkthdr_len(m); - FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len); - error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data); - if (error) { - FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); - break; + if (data_len > 0) { + FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len); + error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data); + if (error) { + FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); + break; + } + } else { + data = NULL; } error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force); if (error) { @@ -1551,7 +1562,7 @@ flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct soc } } } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { - if (to_send) { + if (to_send || mbuf_pkthdr_len(data) == 0) { error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE); if (error) { FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send); @@ -2041,6 +2052,7 @@ flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, in int error = 0; uint32_t key_size = 0; int log_level; + uint32_t flags = 0; error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size); if (error) { @@ -2072,6 +2084,11 @@ flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, in group->token_key_size = key_size; + error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL); + if (!error) { + group->flags = flags; + } + lck_rw_done(&group->lck); } @@ -3133,7 +3150,7 @@ flow_divert_preconnect(struct socket *so) fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; } - so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + soclearfastopen(so); return error; } diff --git a/bsd/netinet/flow_divert.h b/bsd/netinet/flow_divert.h index 47abceaa3..c430a3935 100644 --- a/bsd/netinet/flow_divert.h +++ b/bsd/netinet/flow_divert.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -84,6 +84,7 @@ struct flow_divert_group { MBUFQ_HEAD(send_queue_head) send_queue; uint8_t *token_key; size_t token_key_size; + uint32_t flags; struct flow_divert_trie signing_id_trie; }; diff --git a/bsd/netinet/flow_divert_proto.h b/bsd/netinet/flow_divert_proto.h index 934746d01..675444b77 100644 --- a/bsd/netinet/flow_divert_proto.h +++ b/bsd/netinet/flow_divert_proto.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -81,6 +81,8 @@ #define FLOW_DIVERT_TOKEN_FLAG_TFO 0x0000002 #define FLOW_DIVERT_TOKEN_FLAG_MPTCP 0x0000004 +#define FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP 0x0000001 + struct flow_divert_packet_header { uint8_t packet_type; uint32_t conn_id; diff --git a/bsd/netinet/icmp6.h b/bsd/netinet/icmp6.h index 52228f43c..2176a0299 100644 --- a/bsd/netinet/icmp6.h +++ b/bsd/netinet/icmp6.h @@ -329,7 +329,7 @@ struct nd_opt_hdr { /* Neighbor discovery option header */ #define ND_OPT_REDIRECTED_HEADER 4 #define ND_OPT_MTU 5 #define ND_OPT_NONCE 14 /* RFC 3971 */ -#define ND_OPT_RDNSS 25 /* RFC 5006 */ +#define ND_OPT_RDNSS 25 /* RFC 6106 */ #define ND_OPT_DNSSL 31 /* RFC 6106 */ #define ND_OPT_ROUTE_INFO 200 /* draft-ietf-ipngwg-router-preference, not officially assigned yet */ diff --git a/bsd/netinet/igmp.c b/bsd/netinet/igmp.c index f2cd0966e..b96b869fa 100644 --- a/bsd/netinet/igmp.c +++ b/bsd/netinet/igmp.c @@ -190,9 +190,9 @@ static int current_state_timers_running; /* IGMPv1/v2 host #define IGMP_LOCK() \ lck_mtx_lock(&igmp_mtx) #define IGMP_LOCK_ASSERT_HELD() \ - lck_mtx_assert(&igmp_mtx, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&igmp_mtx, LCK_MTX_ASSERT_OWNED) #define IGMP_LOCK_ASSERT_NOTHELD() \ - lck_mtx_assert(&igmp_mtx, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&igmp_mtx, LCK_MTX_ASSERT_NOTOWNED) #define IGMP_UNLOCK() \ lck_mtx_unlock(&igmp_mtx) @@ -555,7 +555,7 @@ igmp_ra_alloc(void) MGET(m, M_WAITOK, MT_DATA); p = mtod(m, struct ipoption *); p->ipopt_dst.s_addr = INADDR_ANY; - p->ipopt_list[0] = IPOPT_RA; /* Router Alert Option */ + p->ipopt_list[0] = (char)IPOPT_RA; /* Router Alert Option */ p->ipopt_list[1] = 0x04; /* 4 bytes long */ p->ipopt_list[2] = IPOPT_EOL; /* End of IP option list */ p->ipopt_list[3] = 0x00; /* pad byte */ diff --git a/bsd/netinet/igmp_var.h b/bsd/netinet/igmp_var.h index 0d299ef24..22181c9e3 100644 --- a/bsd/netinet/igmp_var.h +++ b/bsd/netinet/igmp_var.h @@ -273,10 +273,10 @@ struct igmp_ifinfo { }; #define IGI_LOCK_ASSERT_HELD(_igi) \ - lck_mtx_assert(&(_igi)->igi_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_igi)->igi_lock, LCK_MTX_ASSERT_OWNED) #define IGI_LOCK_ASSERT_NOTHELD(_igi) \ - lck_mtx_assert(&(_igi)->igi_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_igi)->igi_lock, LCK_MTX_ASSERT_NOTOWNED) #define IGI_LOCK(_igi) \ lck_mtx_lock(&(_igi)->igi_lock) diff --git a/bsd/netinet/in.c b/bsd/netinet/in.c index 0645f9670..cbb7f8cb6 100644 --- a/bsd/netinet/in.c +++ b/bsd/netinet/in.c @@ -83,6 +83,8 @@ #include #include #include +#include +#include #if PF #include #endif /* PF */ @@ -95,6 +97,7 @@ #include #include #include +#include static int inctl_associd(struct socket *, u_long, caddr_t); static int inctl_connid(struct socket *, u_long, caddr_t); @@ -135,9 +138,24 @@ static void in_ifaddr_trace(struct ifaddr *, int); static int in_getassocids(struct socket *, uint32_t *, user_addr_t); static int in_getconnids(struct socket *, sae_associd_t, uint32_t *, user_addr_t); -static int in_getconninfo(struct socket *, sae_connid_t, uint32_t *, - uint32_t *, int32_t *, user_addr_t, socklen_t *, user_addr_t, socklen_t *, - uint32_t *, user_addr_t, uint32_t *); + +/* IPv4 Layer 2 neighbor cache management routines */ +static void in_lltable_destroy_lle_unlocked(struct llentry *lle); +static void in_lltable_destroy_lle(struct llentry *lle); +static struct llentry *in_lltable_new(struct in_addr addr4, u_int flags); +static int in_lltable_match_prefix(const struct sockaddr *saddr, + const struct sockaddr *smask, u_int flags, struct llentry *lle); +static void in_lltable_free_entry(struct lltable *llt, struct llentry *lle); +static int in_lltable_rtcheck(struct ifnet *ifp, u_int flags, const struct sockaddr *l3addr); +static inline uint32_t in_lltable_hash_dst(const struct in_addr dst, uint32_t hsize); +static uint32_t in_lltable_hash(const struct llentry *lle, uint32_t hsize); +static void in_lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa); +static inline struct llentry * in_lltable_find_dst(struct lltable *llt, struct in_addr dst); +static void in_lltable_delete_entry(struct lltable *llt, struct llentry *lle); +static struct llentry * in_lltable_alloc(struct lltable *llt, u_int flags, const struct sockaddr *l3addr); +static struct llentry * in_lltable_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr); +static int in_lltable_dump_entry(struct lltable *llt, struct llentry *lle, struct sysctl_req *wr); +static struct lltable * in_lltattach(struct ifnet *ifp); static int subnetsarelocal = 0; SYSCTL_INT(_net_inet_ip, OID_AUTO, subnets_are_local, @@ -353,6 +371,7 @@ in_domifattach(struct ifnet *ifp) pbuf = (void **)((intptr_t)base - sizeof (void *)); *pbuf = ext; ifp->if_inetdata = base; + IN_IFEXTRA(ifp)->ii_llt = in_lltattach(ifp); VERIFY(IS_P2ALIGNED(ifp->if_inetdata, sizeof (uint64_t))); } done: @@ -1529,7 +1548,7 @@ in_ifscrub(struct ifnet *ifp, struct in_ifaddr *ia, int locked) static void in_iahash_remove(struct in_ifaddr *ia) { - lck_rw_assert(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); IFA_LOCK_ASSERT_HELD(&ia->ia_ifa); if (!IA_IS_HASHED(ia)) { @@ -1551,7 +1570,7 @@ in_iahash_remove(struct in_ifaddr *ia) static void in_iahash_insert(struct in_ifaddr *ia) { - lck_rw_assert(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); IFA_LOCK_ASSERT_HELD(&ia->ia_ifa); if (ia->ia_addr.sin_family != AF_INET) { @@ -1581,7 +1600,7 @@ in_iahash_insert_ptp(struct in_ifaddr *ia) struct in_ifaddr *tmp_ifa; struct ifnet *tmp_ifp; - lck_rw_assert(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); IFA_LOCK_ASSERT_HELD(&ia->ia_ifa); if (ia->ia_addr.sin_family != AF_INET) { @@ -2135,13 +2154,12 @@ in_getconnids(struct socket *so, sae_associd_t aid, uint32_t *cnt, /* * Handle SIOCGCONNINFO ioctl for PF_INET domain. */ -static int +int in_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len, user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type, user_addr_t aux_data, uint32_t *aux_len) { -#pragma unused(aux_data) struct inpcb *inp = sotoinpcb(so); struct sockaddr_in sin; struct ifnet *ifp = NULL; @@ -2209,8 +2227,6 @@ in_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, } } - *aux_type = 0; - *aux_len = 0; if (SOCK_PROTO(so) == IPPROTO_TCP) { struct conninfo_tcp tcp_ci; @@ -2228,8 +2244,362 @@ in_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, *aux_len = copy_len; } } + } else { + *aux_type = 0; + *aux_len = 0; } out: return (error); } + +struct in_llentry { + struct llentry base; +}; + +#define IN_LLTBL_DEFAULT_HSIZE 32 +#define IN_LLTBL_HASH(k, h) \ + ((((((((k) >> 8) ^ (k)) >> 8) ^ (k)) >> 8) ^ (k)) & ((h) - 1)) + +/* + * Do actual deallocation of @lle. + */ +static void +in_lltable_destroy_lle_unlocked(struct llentry *lle) +{ + LLE_LOCK_DESTROY(lle); + LLE_REQ_DESTROY(lle); + FREE(lle, M_LLTABLE); +} + +/* + * Called by LLE_FREE_LOCKED when number of references + * drops to zero. + */ +static void +in_lltable_destroy_lle(struct llentry *lle) +{ + LLE_WUNLOCK(lle); + in_lltable_destroy_lle_unlocked(lle); +} + +static struct llentry * +in_lltable_new(struct in_addr addr4, u_int flags) +{ +#pragma unused(flags) + struct in_llentry *lle; + + MALLOC(lle, struct in_llentry *, sizeof(struct in_llentry), M_LLTABLE, M_NOWAIT | M_ZERO); + if (lle == NULL) /* NB: caller generates msg */ + return NULL; + + /* + * For IPv4 this will trigger "arpresolve" to generate + * an ARP request. + */ + lle->base.la_expire = net_uptime(); /* mark expired */ + lle->base.r_l3addr.addr4 = addr4; + lle->base.lle_refcnt = 1; + lle->base.lle_free = in_lltable_destroy_lle; + + LLE_LOCK_INIT(&lle->base); + LLE_REQ_INIT(&lle->base); + //callout_init(&lle->base.lle_timer, 1); + + return (&lle->base); +} + +#define IN_ARE_MASKED_ADDR_EQUAL(d, a, m) ( \ + ((((d).s_addr ^ (a).s_addr) & (m).s_addr)) == 0 ) + +static int +in_lltable_match_prefix(const struct sockaddr *saddr, + const struct sockaddr *smask, u_int flags, struct llentry *lle) +{ + struct in_addr addr, mask, lle_addr; + + addr = ((const struct sockaddr_in *)(const void *)saddr)->sin_addr; + mask = ((const struct sockaddr_in *)(const void *)smask)->sin_addr; + lle_addr.s_addr = ntohl(lle->r_l3addr.addr4.s_addr); + + if (IN_ARE_MASKED_ADDR_EQUAL(lle_addr, addr, mask) == 0) + return (0); + + if (lle->la_flags & LLE_IFADDR) { + /* + * Delete LLE_IFADDR records IFF address & flag matches. + * Note that addr is the interface address within prefix + * being matched. + * Note also we should handle 'ifdown' cases without removing + * ifaddr macs. + */ + if (addr.s_addr == lle_addr.s_addr && (flags & LLE_STATIC) != 0) + return (1); + return (0); + } + + /* flags & LLE_STATIC means deleting both dynamic and static entries */ + if ((flags & LLE_STATIC) || !(lle->la_flags & LLE_STATIC)) + return (1); + + return (0); +} + +static void +in_lltable_free_entry(struct lltable *llt, struct llentry *lle) +{ + struct ifnet *ifp; + size_t pkts_dropped; + + LLE_WLOCK_ASSERT(lle); + KASSERT(llt != NULL, ("lltable is NULL")); + + /* Unlink entry from table if not already */ + if ((lle->la_flags & LLE_LINKED) != 0) { + ifp = llt->llt_ifp; + IF_AFDATA_WLOCK_ASSERT(ifp, llt->llt_af); + lltable_unlink_entry(llt, lle); + } + +#if 0 + /* cancel timer */ + if (callout_stop(&lle->lle_timer) > 0) + LLE_REMREF(lle); +#endif + /* Drop hold queue */ + pkts_dropped = llentry_free(lle); + arpstat.dropped += pkts_dropped; +} + + +static int +in_lltable_rtcheck(struct ifnet *ifp, u_int flags, const struct sockaddr *l3addr) +{ +#pragma unused(flags) + struct rtentry *rt; + + KASSERT(l3addr->sa_family == AF_INET, + ("sin_family %d", l3addr->sa_family)); + + /* XXX rtalloc1 should take a const param */ + rt = rtalloc1(__DECONST(struct sockaddr *, l3addr), 0, 0); + if (rt == NULL || (rt->rt_flags & RTF_GATEWAY) || rt->rt_ifp != ifp) { + log(LOG_INFO, "IPv4 address: \"%s\" is not on the network\n", + inet_ntoa(((const struct sockaddr_in *)(const void *)l3addr)->sin_addr)); + if (rt != NULL) + rtfree_locked(rt); + return (EINVAL); + } + rtfree_locked(rt); + return 0; +} + +static inline uint32_t +in_lltable_hash_dst(const struct in_addr dst, uint32_t hsize) +{ + return (IN_LLTBL_HASH(dst.s_addr, hsize)); +} + +static uint32_t +in_lltable_hash(const struct llentry *lle, uint32_t hsize) +{ + return (in_lltable_hash_dst(lle->r_l3addr.addr4, hsize)); +} + + +static void +in_lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa) +{ + struct sockaddr_in *sin; + + sin = (struct sockaddr_in *)(void *)sa; + bzero(sin, sizeof(*sin)); + sin->sin_family = AF_INET; + sin->sin_len = sizeof(*sin); + sin->sin_addr = lle->r_l3addr.addr4; +} + +static inline struct llentry * +in_lltable_find_dst(struct lltable *llt, struct in_addr dst) +{ + struct llentry *lle; + struct llentries *lleh; + u_int hashidx; + + hashidx = in_lltable_hash_dst(dst, llt->llt_hsize); + lleh = &llt->lle_head[hashidx]; + LIST_FOREACH(lle, lleh, lle_next) { + if (lle->la_flags & LLE_DELETED) + continue; + if (lle->r_l3addr.addr4.s_addr == dst.s_addr) + break; + } + + return (lle); +} + +static void +in_lltable_delete_entry(struct lltable *llt, struct llentry *lle) +{ +#pragma unused(llt) + lle->la_flags |= LLE_DELETED; + //EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED); +#ifdef DIAGNOSTIC + log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle); +#endif + llentry_free(lle); +} + +static struct llentry * +in_lltable_alloc(struct lltable *llt, u_int flags, const struct sockaddr *l3addr) +{ + const struct sockaddr_in *sin = (const struct sockaddr_in *) (const void *)l3addr; + struct ifnet *ifp = llt->llt_ifp; + struct llentry *lle; + + KASSERT(l3addr->sa_family == AF_INET, + ("sin_family %d", l3addr->sa_family)); + + /* + * A route that covers the given address must have + * been installed 1st because we are doing a resolution, + * verify this. + */ + if (!(flags & LLE_IFADDR) && + in_lltable_rtcheck(ifp, flags, l3addr) != 0) + return (NULL); + + lle = in_lltable_new(sin->sin_addr, flags); + if (lle == NULL) { + log(LOG_INFO, "lla_lookup: new lle malloc failed\n"); + return (NULL); + } + lle->la_flags = flags & ~LLE_CREATE; + if (flags & LLE_STATIC) + lle->r_flags |= RLLE_VALID; + if ((flags & LLE_IFADDR) == LLE_IFADDR) { + lltable_set_entry_addr(ifp, lle, LLADDR(SDL(ifp->if_lladdr->ifa_addr))); + lle->la_flags |= LLE_STATIC; + lle->r_flags |= (RLLE_VALID | RLLE_IFADDR); + } + return (lle); +} + +/* + * Return NULL if not found or marked for deletion. + * If found return lle read locked. + */ +static struct llentry * +in_lltable_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr) +{ + const struct sockaddr_in *sin = (const struct sockaddr_in *)(const void *)l3addr; + struct llentry *lle; + + IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp, llt->llt_af); + + KASSERT(l3addr->sa_family == AF_INET, + ("sin_family %d", l3addr->sa_family)); + lle = in_lltable_find_dst(llt, sin->sin_addr); + + if (lle == NULL) + return (NULL); + + KASSERT((flags & (LLE_UNLOCKED|LLE_EXCLUSIVE)) != + (LLE_UNLOCKED|LLE_EXCLUSIVE),("wrong lle request flags: 0x%X", + flags)); + + if (flags & LLE_UNLOCKED) + return (lle); + + if (flags & LLE_EXCLUSIVE) + LLE_WLOCK(lle); + else + LLE_RLOCK(lle); + + return (lle); +} + +static int +in_lltable_dump_entry(struct lltable *llt, struct llentry *lle, + struct sysctl_req *wr) +{ + struct ifnet *ifp = llt->llt_ifp; + /* XXX stack use */ + struct { + struct rt_msghdr rtm; + struct sockaddr_in sin; + struct sockaddr_dl sdl; + } arpc; + struct sockaddr_dl *sdl; + int error; + + bzero(&arpc, sizeof(arpc)); + /* skip deleted entries */ + if ((lle->la_flags & LLE_DELETED) == LLE_DELETED) + return (0); + /* Skip if jailed and not a valid IP of the prison. */ + lltable_fill_sa_entry(lle,(struct sockaddr *)&arpc.sin); + /* + * produce a msg made of: + * struct rt_msghdr; + * struct sockaddr_in; (IPv4) + * struct sockaddr_dl; + */ + arpc.rtm.rtm_msglen = sizeof(arpc); + arpc.rtm.rtm_version = RTM_VERSION; + arpc.rtm.rtm_type = RTM_GET; + arpc.rtm.rtm_flags = RTF_UP; + arpc.rtm.rtm_addrs = RTA_DST | RTA_GATEWAY; + + /* publish */ + if (lle->la_flags & LLE_PUB) + arpc.rtm.rtm_flags |= RTF_ANNOUNCE; + + sdl = &arpc.sdl; + sdl->sdl_family = AF_LINK; + sdl->sdl_len = sizeof(*sdl); + sdl->sdl_index = ifp->if_index; + sdl->sdl_type = ifp->if_type; + if ((lle->la_flags & LLE_VALID) == LLE_VALID) { + sdl->sdl_alen = ifp->if_addrlen; + bcopy(&lle->ll_addr, LLADDR(sdl), ifp->if_addrlen); + } else { + sdl->sdl_alen = 0; + bzero(LLADDR(sdl), ifp->if_addrlen); + } + + arpc.rtm.rtm_rmx.rmx_expire = + lle->la_flags & LLE_STATIC ? 0 : lle->la_expire; + arpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA); + if (lle->la_flags & LLE_STATIC) + arpc.rtm.rtm_flags |= RTF_STATIC; + if (lle->la_flags & LLE_IFADDR) + arpc.rtm.rtm_flags |= RTF_PINNED; + arpc.rtm.rtm_flags |= RTF_PINNED; + arpc.rtm.rtm_index = ifp->if_index; + error = SYSCTL_OUT(wr, &arpc, sizeof(arpc)); + + return (error); +} + +static struct lltable * +in_lltattach(struct ifnet *ifp) +{ + struct lltable *llt; + + llt = lltable_allocate_htbl(IN_LLTBL_DEFAULT_HSIZE); + llt->llt_af = AF_INET; + llt->llt_ifp = ifp; + + llt->llt_lookup = in_lltable_lookup; + llt->llt_alloc_entry = in_lltable_alloc; + llt->llt_delete_entry = in_lltable_delete_entry; + llt->llt_dump_entry = in_lltable_dump_entry; + llt->llt_hash = in_lltable_hash; + llt->llt_fill_sa_entry = in_lltable_fill_sa_entry; + llt->llt_free_entry = in_lltable_free_entry; + llt->llt_match_prefix = in_lltable_match_prefix; + lltable_link(llt); + + return (llt); +} diff --git a/bsd/netinet/in.h b/bsd/netinet/in.h index 74918ba34..07732679a 100644 --- a/bsd/netinet/in.h +++ b/bsd/netinet/in.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -162,9 +162,9 @@ #define IPPROTO_SWIPE 53 /* IP with encryption */ #define IPPROTO_NHRP 54 /* Next Hop Resolution */ /* 55-57: Unassigned */ -#define IPPROTO_ICMPV6 58 /* ICMP6 */ -#define IPPROTO_NONE 59 /* IP6 no next header */ -#define IPPROTO_DSTOPTS 60 /* IP6 destination option */ +#define IPPROTO_ICMPV6 58 /* ICMP6 */ +#define IPPROTO_NONE 59 /* IP6 no next header */ +#define IPPROTO_DSTOPTS 60 /* IP6 destination option */ #define IPPROTO_AHIP 61 /* any host internal protocol */ #define IPPROTO_CFTP 62 /* CFTP */ #define IPPROTO_HELLO 63 /* "hello" routing protocol */ @@ -364,10 +364,19 @@ struct in_addr { (((u_int32_t)(i) & 0xfff00000) == 0xac100000) || \ (((u_int32_t)(i) & 0xffff0000) == 0xc0a80000)) +#ifdef PRIVATE +#define IN_SHARED_ADDRESS_SPACE(i) ((((u_int32_t)(i)) & (u_int32_t)0xffc00000) \ + == (u_int32_t)0x64400000) + +#define IN_DS_LITE(i) ((((u_int32_t)(i)) & (u_int32_t)0xfffffff8) == (u_int32_t)0xc0000000) + +#define IN_6TO4_RELAY_ANYCAST(i) ((((u_int32_t)(i)) & (u_int32_t)IN_CLASSC_NET) == (u_int32_t)0xc0586300) +#endif + #define IN_LOCAL_GROUP(i) (((u_int32_t)(i) & 0xffffff00) == 0xe0000000) - + #define IN_ANY_LOCAL(i) (IN_LINKLOCAL(i) || IN_LOCAL_GROUP(i)) -#endif +#endif /* __APPLE__ */ #define IN_LOOPBACKNET 127 /* official! */ #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ @@ -851,17 +860,31 @@ extern boolean_t in_broadcast(struct in_addr, struct ifnet *); extern boolean_t in_canforward(struct in_addr); extern u_int32_t in_netof(struct in_addr); +extern uint32_t os_cpu_in_cksum_mbuf(struct mbuf *m, int len, int off, + uint32_t initial_sum); + extern uint16_t inet_cksum(struct mbuf *, uint32_t, uint32_t, uint32_t); +extern uint16_t inet_cksum_buffer(const void *, uint32_t, uint32_t, uint32_t); extern uint16_t in_addword(uint16_t, uint16_t); extern uint16_t in_pseudo(uint32_t, uint32_t, uint32_t); extern uint16_t in_pseudo64(uint64_t, uint64_t, uint64_t); extern uint16_t in_cksum_hdr_opt(const struct ip *); extern uint16_t ip_cksum_hdr_dir(struct mbuf *, uint32_t, int); +extern uint16_t ip_cksum_hdr_dir_buffer(const void *, uint32_t, uint32_t, int); extern uint32_t in_finalize_cksum(struct mbuf *, uint32_t, uint32_t); extern uint16_t b_sum16(const void *buf, int len); +#if DEBUG || DEVELOPMENT +extern uint32_t in_cksum_mbuf_ref(struct mbuf *, int, int, uint32_t); +#endif /* DEBUG || DEVELOPMENT */ + +extern int in_getconninfo(struct socket *, sae_connid_t, uint32_t *, + uint32_t *, int32_t *, user_addr_t, socklen_t *, user_addr_t, socklen_t *, + uint32_t *, user_addr_t, uint32_t *); #define in_cksum(_m, _l) \ inet_cksum(_m, 0, 0, _l) +#define in_cksum_buffer(_b, _l) \ + inet_cksum_buffer(_b, 0, 0, _l) #define ip_cksum_hdr_in(_m, _l) \ ip_cksum_hdr_dir(_m, _l, 0) #define ip_cksum_hdr_out(_m, _l) \ diff --git a/bsd/netinet/in_arp.c b/bsd/netinet/in_arp.c index e7eafd51d..674da52bf 100644 --- a/bsd/netinet/in_arp.c +++ b/bsd/netinet/in_arp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2016 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -79,6 +79,7 @@ #include #include #include +#include #include #include @@ -121,20 +122,22 @@ struct llinfo_arp { * The following are protected by rnh_lock */ LIST_ENTRY(llinfo_arp) la_le; - struct rtentry *la_rt; + struct rtentry *la_rt; /* * The following are protected by rt_lock */ - class_queue_t la_holdq; /* packets awaiting resolution */ - struct if_llreach *la_llreach; /* link-layer reachability record */ - u_int64_t la_lastused; /* last used timestamp */ - u_int32_t la_asked; /* # of requests sent */ - u_int32_t la_maxtries; /* retry limit */ - u_int64_t la_probeexp; /* probe deadline timestamp */ + class_queue_t la_holdq; /* packets awaiting resolution */ + struct if_llreach *la_llreach; /* link-layer reachability record */ + u_int64_t la_lastused; /* last used timestamp */ + u_int32_t la_asked; /* # of requests sent */ + u_int32_t la_maxtries; /* retry limit */ + u_int64_t la_probeexp; /* probe deadline timestamp */ + u_int32_t la_prbreq_cnt; /* probe request count */ u_int32_t la_flags; -#define LLINFO_RTRFAIL_EVTSENT 0x1 /* sent an ARP event */ -#define LLINFO_PROBING 0x2 /* waiting for an ARP reply */ +#define LLINFO_RTRFAIL_EVTSENT 0x1 /* sent an ARP event */ +#define LLINFO_PROBING 0x2 /* waiting for an ARP reply */ }; + static LIST_HEAD(, llinfo_arp) llinfo_arp; static thread_call_t arp_timeout_tcall; @@ -163,7 +166,7 @@ static void arp_llinfo_refresh(struct rtentry *); static __inline void arp_llreach_use(struct llinfo_arp *); static __inline int arp_llreach_reachable(struct llinfo_arp *); static void arp_llreach_alloc(struct rtentry *, struct ifnet *, void *, - unsigned int, boolean_t); + unsigned int, boolean_t, uint32_t *); extern int tvtohz(struct timeval *); @@ -290,7 +293,7 @@ arp_llinfo_alloc(int how) * a head drop, details in arp_llinfo_addq(). */ _qinit(&la->la_holdq, Q_DROPHEAD, (arp_maxhold == 0) ? - (uint32_t)-1 : arp_maxhold); + (uint32_t)-1 : arp_maxhold, QP_MBUF); } return (la); @@ -349,11 +352,13 @@ arp_llinfo_flushq(struct llinfo_arp *la) { uint32_t held = qlen(&la->la_holdq); - atomic_add_32(&arpstat.purged, held); - atomic_add_32(&arpstat.held, -held); - _flushq(&la->la_holdq); + if (held != 0) { + atomic_add_32(&arpstat.purged, held); + atomic_add_32(&arpstat.held, -held); + _flushq(&la->la_holdq); + } + la->la_prbreq_cnt = 0; VERIFY(qempty(&la->la_holdq)); - return (held); } @@ -523,7 +528,7 @@ arp_llreach_reachable(struct llinfo_arp *la) */ static void arp_llreach_alloc(struct rtentry *rt, struct ifnet *ifp, void *addr, - unsigned int alen, boolean_t solicited) + unsigned int alen, boolean_t solicited, uint32_t *p_rt_event_code) { VERIFY(rt->rt_expire == 0 || rt->rt_rmx.rmx_expire != 0); VERIFY(rt->rt_expire != 0 || rt->rt_rmx.rmx_expire == 0); @@ -554,7 +559,15 @@ arp_llreach_alloc(struct rtentry *rt, struct ifnet *ifp, void *addr, lr = NULL; why = " for different target HW address; " "using new llreach record"; + *p_rt_event_code = ROUTE_LLENTRY_CHANGED; } else { + /* + * If we were doing unicast probing, we need to + * deliver an event for neighbor cache resolution + */ + if (lr->lr_probes != 0) + *p_rt_event_code = ROUTE_LLENTRY_RESOLVED; + lr->lr_probes = 0; /* reset probe count */ IFLR_UNLOCK(lr); if (solicited) { @@ -572,6 +585,7 @@ arp_llreach_alloc(struct rtentry *rt, struct ifnet *ifp, void *addr, if (why == NULL) why = "creating new llreach record"; } + *p_rt_event_code = ROUTE_LLENTRY_RESOLVED; } if (arp_verbose > 1 && lr != NULL && why != NULL) { @@ -605,7 +619,7 @@ arptfree(struct llinfo_arp *la, void *arg) struct rtentry *rt = la->la_rt; uint64_t timenow; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* rnh_lock acquired by caller protects rt from going away */ RT_LOCK(rt); @@ -623,9 +637,22 @@ arptfree(struct llinfo_arp *la, void *arg) if (sdl != NULL) sdl->sdl_alen = 0; (void) arp_llinfo_flushq(la); + /* + * Enqueue work item to invoke callback for this route entry + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_UNREACH, NULL, TRUE); } + /* + * The following is mostly being used to arm the timer + * again and for logging. + * qlen is used to re-arm the timer. Therefore, pure probe + * requests can be considered as 0 length packets + * contributing only to length but not to the size. + */ ap->qlen += qlen(&la->la_holdq); + ap->qlen += la->la_prbreq_cnt; ap->qsize += qsize(&la->la_holdq); if (rt->rt_expire == 0 || (rt->rt_flags & RTF_STATIC)) { @@ -742,7 +769,7 @@ arp_timeout(thread_call_param_t arg0, thread_call_param_t arg1) static void arp_sched_timeout(struct timeval *atv) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (!arp_timeout_run) { struct timeval tv; @@ -811,7 +838,7 @@ arp_probe(thread_call_param_t arg0, thread_call_param_t arg1) static void arp_sched_probe(struct timeval *atv) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (!arp_probe_run) { struct timeval tv; @@ -856,7 +883,7 @@ arp_rtrequest(int req, struct rtentry *rt, struct sockaddr *sa) char buf[MAX_IPv4_STR_LEN]; VERIFY(arpinit_done); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (rt->rt_flags & RTF_GATEWAY) @@ -1160,6 +1187,19 @@ arp_lookup_route(const struct in_addr *addr, int create, int proxy, return (0); } +boolean_t +arp_is_entry_probing (route_t p_route) +{ + struct llinfo_arp *llinfo = p_route->rt_llinfo; + + if (llinfo != NULL && + llinfo->la_llreach != NULL && + llinfo->la_llreach->lr_probes != 0) + return (TRUE); + + return (FALSE); +} + /* * This is the ARP pre-output routine; care must be taken to ensure that * the "hint" route never gets freed via rtfree(), since the caller may @@ -1182,6 +1222,7 @@ arp_lookup_ip(ifnet_t ifp, const struct sockaddr_in *net_dest, struct sockaddr *sa; uint32_t rtflags; struct sockaddr_dl sdl; + boolean_t send_probe_notif = FALSE; if (ifp == NULL || net_dest == NULL) return (EINVAL); @@ -1316,6 +1357,13 @@ arp_lookup_ip(ifnet_t ifp, const struct sockaddr_in *net_dest, if (lr->lr_probes == 0) { llinfo->la_probeexp = (timenow + arpt_probe); llinfo->la_flags |= LLINFO_PROBING; + /* + * Provide notification that ARP unicast + * probing has started. + * We only do it for the first unicast probe + * attempt. + */ + send_probe_notif = TRUE; } /* @@ -1371,7 +1419,8 @@ arp_lookup_ip(ifnet_t ifp, const struct sockaddr_in *net_dest, */ if (packet != NULL) arp_llinfo_addq(llinfo, packet); - + else + llinfo->la_prbreq_cnt++; /* * Regardless of permanent vs. expirable entry, we need to * avoid having packets sit in la_holdq forever; thus mark the @@ -1380,10 +1429,11 @@ arp_lookup_ip(ifnet_t ifp, const struct sockaddr_in *net_dest, * moment we get an ARP reply. */ probing = TRUE; - if (qlen(&llinfo->la_holdq) == 1) { + if ((qlen(&llinfo->la_holdq) + llinfo->la_prbreq_cnt) == 1) { llinfo->la_probeexp = (timenow + arpt_probe); llinfo->la_flags |= LLINFO_PROBING; } + if (route->rt_expire) { route->rt_flags &= ~RTF_REJECT; if (llinfo->la_asked == 0 || route->rt_expire != timenow) { @@ -1464,6 +1514,12 @@ arp_lookup_ip(ifnet_t ifp, const struct sockaddr_in *net_dest, VERIFY(_m == packet); } result = EHOSTUNREACH; + + /* + * Enqueue work item to invoke callback for this route entry + */ + route_event_enqueue_nwk_wq_entry(route, NULL, + ROUTE_LLENTRY_UNREACH, NULL, TRUE); goto release; } } @@ -1477,6 +1533,30 @@ release: atomic_add_32(&arpstat.dropped, 1); if (route != NULL) { + if (send_probe_notif) { + route_event_enqueue_nwk_wq_entry(route, NULL, + ROUTE_LLENTRY_PROBED, NULL, TRUE); + + if (route->rt_flags & RTF_ROUTER) { + struct radix_node_head *rnh = NULL; + struct route_event rt_ev; + route_event_init(&rt_ev, route, NULL, ROUTE_LLENTRY_PROBED); + /* + * We already have a reference on rt. The function + * frees it before returning. + */ + RT_UNLOCK(route); + lck_mtx_lock(rnh_lock); + rnh = rt_tables[AF_INET]; + + if (rnh != NULL) + (void) rnh->rnh_walktree(rnh, + route_event_walktree, (void *)&rt_ev); + lck_mtx_unlock(rnh_lock); + RT_LOCK(route); + } + } + if (route == hint) { RT_REMREF_LOCKED(route); RT_UNLOCK(route); @@ -1512,6 +1592,7 @@ arp_ip_handle_input(ifnet_t ifp, u_short arpop, errno_t error; int created_announcement = 0; int bridged = 0, is_bridge = 0; + uint32_t rt_evcode = 0; /* * Here and other places within this routine where we don't hold @@ -1927,7 +2008,7 @@ match: /* cache the gateway (sender HW) address */ arp_llreach_alloc(route, ifp, LLADDR(gateway), gateway->sdl_alen, - (arpop == ARPOP_REPLY)); + (arpop == ARPOP_REPLY), &rt_evcode); llinfo = route->rt_llinfo; /* send a notification that the route is back up */ @@ -1956,6 +2037,34 @@ match: /* Update the llinfo, send out all queued packets at once */ llinfo->la_asked = 0; llinfo->la_flags &= ~LLINFO_PROBING; + llinfo->la_prbreq_cnt = 0; + + if (rt_evcode) { + /* + * Enqueue work item to invoke callback for this route entry + */ + route_event_enqueue_nwk_wq_entry(route, NULL, rt_evcode, NULL, TRUE); + + if (route->rt_flags & RTF_ROUTER) { + struct radix_node_head *rnh = NULL; + struct route_event rt_ev; + route_event_init(&rt_ev, route, NULL, rt_evcode); + /* + * We already have a reference on rt. The function + * frees it before returning. + */ + RT_UNLOCK(route); + lck_mtx_lock(rnh_lock); + rnh = rt_tables[AF_INET]; + + if (rnh != NULL) + (void) rnh->rnh_walktree(rnh, route_event_walktree, + (void *)&rt_ev); + lck_mtx_unlock(rnh_lock); + RT_LOCK(route); + } + } + if (!qempty(&llinfo->la_holdq)) { uint32_t held; struct mbuf *m0 = @@ -1973,7 +2082,6 @@ match: route = NULL; } - respond: if (route != NULL) { /* Mark use timestamp if we're going to send a reply */ diff --git a/bsd/netinet/in_arp.h b/bsd/netinet/in_arp.h index fdbc9e2d1..51310486a 100644 --- a/bsd/netinet/in_arp.h +++ b/bsd/netinet/in_arp.h @@ -61,6 +61,7 @@ struct sockaddr_in; * the packet. */ #ifdef BSD_KERNEL_PRIVATE +extern boolean_t arp_is_entry_probing (route_t p_route); extern errno_t arp_lookup_ip(ifnet_t interface, const struct sockaddr_in *ip_dest, struct sockaddr_dl *ll_dest, size_t ll_dest_len, route_t hint, mbuf_t packet); diff --git a/bsd/netinet/in_cksum.c b/bsd/netinet/in_cksum.c index bc302ae30..29594c123 100644 --- a/bsd/netinet/in_cksum.c +++ b/bsd/netinet/in_cksum.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2012 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -94,258 +94,7 @@ union q_util { uint64_t q; }; -#define PREDICT_FALSE(_exp) __builtin_expect((_exp), 0) - -static uint16_t in_cksumdata(const void *buf, int len); - -/* - * Portable version of 16-bit 1's complement sum function that works - * on a contiguous buffer. This is used mainly for instances where - * the caller is certain about the buffer requirements, e.g. for IP - * header checksum calculation, though it is capable of being used - * on any arbitrary data span. The platform-specific cpu_in_cksum() - * routine might be better-optmized, so use that instead for large - * data span. - * - * The logic is borrowed from - */ - -#if ULONG_MAX == 0xffffffffUL -/* 32-bit version */ -static uint16_t -in_cksumdata(const void *buf, int mlen) -{ - uint32_t sum, partial; - unsigned int final_acc; - const uint8_t *data = (const uint8_t *)buf; - boolean_t needs_swap, started_on_odd; - - VERIFY(mlen >= 0); - - needs_swap = FALSE; - started_on_odd = FALSE; - - sum = 0; - partial = 0; - - if ((uintptr_t)data & 1) { - /* Align on word boundary */ - started_on_odd = !started_on_odd; -#if BYTE_ORDER == LITTLE_ENDIAN - partial = *data << 8; -#else - partial = *data; -#endif - ++data; - --mlen; - } - needs_swap = started_on_odd; - while (mlen >= 32) { - __builtin_prefetch(data + 32); - partial += *(const uint16_t *)(const void *)data; - partial += *(const uint16_t *)(const void *)(data + 2); - partial += *(const uint16_t *)(const void *)(data + 4); - partial += *(const uint16_t *)(const void *)(data + 6); - partial += *(const uint16_t *)(const void *)(data + 8); - partial += *(const uint16_t *)(const void *)(data + 10); - partial += *(const uint16_t *)(const void *)(data + 12); - partial += *(const uint16_t *)(const void *)(data + 14); - partial += *(const uint16_t *)(const void *)(data + 16); - partial += *(const uint16_t *)(const void *)(data + 18); - partial += *(const uint16_t *)(const void *)(data + 20); - partial += *(const uint16_t *)(const void *)(data + 22); - partial += *(const uint16_t *)(const void *)(data + 24); - partial += *(const uint16_t *)(const void *)(data + 26); - partial += *(const uint16_t *)(const void *)(data + 28); - partial += *(const uint16_t *)(const void *)(data + 30); - data += 32; - mlen -= 32; - if (PREDICT_FALSE(partial & 0xc0000000)) { - if (needs_swap) - partial = (partial << 8) + - (partial >> 24); - sum += (partial >> 16); - sum += (partial & 0xffff); - partial = 0; - } - } - if (mlen & 16) { - partial += *(const uint16_t *)(const void *)data; - partial += *(const uint16_t *)(const void *)(data + 2); - partial += *(const uint16_t *)(const void *)(data + 4); - partial += *(const uint16_t *)(const void *)(data + 6); - partial += *(const uint16_t *)(const void *)(data + 8); - partial += *(const uint16_t *)(const void *)(data + 10); - partial += *(const uint16_t *)(const void *)(data + 12); - partial += *(const uint16_t *)(const void *)(data + 14); - data += 16; - mlen -= 16; - } - /* - * mlen is not updated below as the remaining tests - * are using bit masks, which are not affected. - */ - if (mlen & 8) { - partial += *(const uint16_t *)(const void *)data; - partial += *(const uint16_t *)(const void *)(data + 2); - partial += *(const uint16_t *)(const void *)(data + 4); - partial += *(const uint16_t *)(const void *)(data + 6); - data += 8; - } - if (mlen & 4) { - partial += *(const uint16_t *)(const void *)data; - partial += *(const uint16_t *)(const void *)(data + 2); - data += 4; - } - if (mlen & 2) { - partial += *(const uint16_t *)(const void *)data; - data += 2; - } - if (mlen & 1) { -#if BYTE_ORDER == LITTLE_ENDIAN - partial += *data; -#else - partial += *data << 8; -#endif - started_on_odd = !started_on_odd; - } - - if (needs_swap) - partial = (partial << 8) + (partial >> 24); - sum += (partial >> 16) + (partial & 0xffff); - sum = (sum >> 16) + (sum & 0xffff); - - final_acc = ((sum >> 16) & 0xffff) + (sum & 0xffff); - final_acc = (final_acc >> 16) + (final_acc & 0xffff); - - return (final_acc); -} - -#else -/* 64-bit version */ -static uint16_t -in_cksumdata(const void *buf, int mlen) -{ - uint64_t sum, partial; - unsigned int final_acc; - const uint8_t *data = (const uint8_t *)buf; - boolean_t needs_swap, started_on_odd; - - VERIFY(mlen >= 0); - - needs_swap = FALSE; - started_on_odd = FALSE; - - sum = 0; - partial = 0; - - if ((uintptr_t)data & 1) { - /* Align on word boundary */ - started_on_odd = !started_on_odd; -#if BYTE_ORDER == LITTLE_ENDIAN - partial = *data << 8; -#else - partial = *data; -#endif - ++data; - --mlen; - } - needs_swap = started_on_odd; - if ((uintptr_t)data & 2) { - if (mlen < 2) - goto trailing_bytes; - partial += *(const uint16_t *)(const void *)data; - data += 2; - mlen -= 2; - } - while (mlen >= 64) { - __builtin_prefetch(data + 32); - __builtin_prefetch(data + 64); - partial += *(const uint32_t *)(const void *)data; - partial += *(const uint32_t *)(const void *)(data + 4); - partial += *(const uint32_t *)(const void *)(data + 8); - partial += *(const uint32_t *)(const void *)(data + 12); - partial += *(const uint32_t *)(const void *)(data + 16); - partial += *(const uint32_t *)(const void *)(data + 20); - partial += *(const uint32_t *)(const void *)(data + 24); - partial += *(const uint32_t *)(const void *)(data + 28); - partial += *(const uint32_t *)(const void *)(data + 32); - partial += *(const uint32_t *)(const void *)(data + 36); - partial += *(const uint32_t *)(const void *)(data + 40); - partial += *(const uint32_t *)(const void *)(data + 44); - partial += *(const uint32_t *)(const void *)(data + 48); - partial += *(const uint32_t *)(const void *)(data + 52); - partial += *(const uint32_t *)(const void *)(data + 56); - partial += *(const uint32_t *)(const void *)(data + 60); - data += 64; - mlen -= 64; - if (PREDICT_FALSE(partial & (3ULL << 62))) { - if (needs_swap) - partial = (partial << 8) + - (partial >> 56); - sum += (partial >> 32); - sum += (partial & 0xffffffff); - partial = 0; - } - } - /* - * mlen is not updated below as the remaining tests - * are using bit masks, which are not affected. - */ - if (mlen & 32) { - partial += *(const uint32_t *)(const void *)data; - partial += *(const uint32_t *)(const void *)(data + 4); - partial += *(const uint32_t *)(const void *)(data + 8); - partial += *(const uint32_t *)(const void *)(data + 12); - partial += *(const uint32_t *)(const void *)(data + 16); - partial += *(const uint32_t *)(const void *)(data + 20); - partial += *(const uint32_t *)(const void *)(data + 24); - partial += *(const uint32_t *)(const void *)(data + 28); - data += 32; - } - if (mlen & 16) { - partial += *(const uint32_t *)(const void *)data; - partial += *(const uint32_t *)(const void *)(data + 4); - partial += *(const uint32_t *)(const void *)(data + 8); - partial += *(const uint32_t *)(const void *)(data + 12); - data += 16; - } - if (mlen & 8) { - partial += *(const uint32_t *)(const void *)data; - partial += *(const uint32_t *)(const void *)(data + 4); - data += 8; - } - if (mlen & 4) { - partial += *(const uint32_t *)(const void *)data; - data += 4; - } - if (mlen & 2) { - partial += *(const uint16_t *)(const void *)data; - data += 2; - } -trailing_bytes: - if (mlen & 1) { -#if BYTE_ORDER == LITTLE_ENDIAN - partial += *data; -#else - partial += *data << 8; -#endif - started_on_odd = !started_on_odd; - } - - if (needs_swap) - partial = (partial << 8) + (partial >> 56); - sum += (partial >> 32) + (partial & 0xffffffff); - sum = (sum >> 32) + (sum & 0xffffffff); - - final_acc = (sum >> 48) + ((sum >> 32) & 0xffff) + - ((sum >> 16) & 0xffff) + (sum & 0xffff); - final_acc = (final_acc >> 16) + (final_acc & 0xffff); - final_acc = (final_acc >> 16) + (final_acc & 0xffff); - - return (final_acc); -} -#endif /* ULONG_MAX != 0xffffffffUL */ +extern uint32_t os_cpu_in_cksum(const void *, uint32_t, uint32_t); /* * Perform 16-bit 1's complement sum on a contiguous span. @@ -353,7 +102,7 @@ trailing_bytes: uint16_t b_sum16(const void *buf, int len) { - return (in_cksumdata(buf, len)); + return (os_cpu_in_cksum(buf, len, 0)); } uint16_t inet_cksum_simple(struct mbuf *, int); @@ -433,6 +182,27 @@ ip_cksum_hdr_dir(struct mbuf *m, uint32_t hlen, int out) return (inet_cksum(m, 0, 0, hlen)); } +uint16_t +ip_cksum_hdr_dir_buffer(const void *buffer, uint32_t hlen, uint32_t len, + int out) +{ + const struct ip *ip = buffer; + + if (out) { + ipstat.ips_snd_swcsum++; + ipstat.ips_snd_swcsum_bytes += hlen; + } else { + ipstat.ips_rcv_swcsum++; + ipstat.ips_rcv_swcsum_bytes += hlen; + } + + if (hlen == sizeof (*ip) && + len >= sizeof (*ip) && IP_HDR_ALIGNED_P(ip)) + return (in_cksum_hdr(ip)); + + return (inet_cksum_buffer(buffer, 0, 0, hlen)); +} + /* * m MUST contain at least an IP header, if nxt is specified; * nxt is the upper layer protocol number; @@ -488,3 +258,369 @@ inet_cksum(struct mbuf *m, uint32_t nxt, uint32_t off, uint32_t len) return (~sum & 0xffff); } + +/* + * buffer MUST contain at least an IP header, if nxt is specified; + * nxt is the upper layer protocol number; + * off is an offset where TCP/UDP/ICMP header starts; + * len is a total length of a transport segment (e.g. TCP header + TCP payload) + */ +uint16_t +inet_cksum_buffer(const void *buffer, uint32_t nxt, uint32_t off, + uint32_t len) +{ + uint32_t sum; + + if (off >= len) + panic("%s: off (%d) >= len (%d)", __func__, off, len); + + sum = b_sum16(&((const uint8_t *)buffer)[off], len); + + /* include pseudo header checksum? */ + if (nxt != 0) { + const struct ip *ip; + unsigned char buf[sizeof ((*ip))] __attribute__((aligned(8))); + + /* + * In case the IP header is not contiguous, or not 32-bit + * aligned, copy it to a local buffer. Note here that we + * expect the data pointer to point to the IP header. + */ + if (!IP_HDR_ALIGNED_P(buffer)) { + memcpy(buf, buffer, sizeof (*ip)); + ip = (const struct ip *)(const void *)buf; + } else { + ip = (const struct ip *)buffer; + } + + /* add pseudo header checksum */ + sum += in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, + htonl(len + nxt)); + + /* fold in carry bits */ + ADDCARRY(sum); + } + + return (~sum & 0xffff); +} + +#if DEBUG || DEVELOPMENT +#include +#include + +#define CKSUM_ERR kprintf + +/* + * The following routines implement the portable, reference implementation + * of os_cpu_in_cksum_mbuf(). This is currently used only for validating + * the correctness of the platform-specific implementation, at boot time + * in dlil_verify_sum16(). It returns the 32-bit accumulator without doing + * a 1's complement on it. + */ +#if !defined(__LP64__) +/* 32-bit version */ +uint32_t +in_cksum_mbuf_ref(struct mbuf *m, int len, int off, uint32_t initial_sum) +{ + int mlen; + uint32_t sum, partial; + unsigned int final_acc; + uint8_t *data; + boolean_t needs_swap, started_on_odd; + + VERIFY(len >= 0); + VERIFY(off >= 0); + + needs_swap = FALSE; + started_on_odd = FALSE; + sum = (initial_sum >> 16) + (initial_sum & 0xffff); + + for (;;) { + if (__improbable(m == NULL)) { + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); + } + mlen = m->m_len; + if (mlen > off) { + mlen -= off; + data = mtod(m, uint8_t *) + off; + goto post_initial_offset; + } + off -= mlen; + if (len == 0) + break; + m = m->m_next; + } + + for (; len > 0; m = m->m_next) { + if (__improbable(m == NULL)) { + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); + } + mlen = m->m_len; + data = mtod(m, uint8_t *); +post_initial_offset: + if (mlen == 0) + continue; + if (mlen > len) + mlen = len; + len -= mlen; + + partial = 0; + if ((uintptr_t)data & 1) { + /* Align on word boundary */ + started_on_odd = !started_on_odd; +#if BYTE_ORDER == LITTLE_ENDIAN + partial = *data << 8; +#else /* BYTE_ORDER != LITTLE_ENDIAN */ + partial = *data; +#endif /* BYTE_ORDER != LITTLE_ENDIAN */ + ++data; + --mlen; + } + needs_swap = started_on_odd; + while (mlen >= 32) { + __builtin_prefetch(data + 32); + partial += *(uint16_t *)(void *)data; + partial += *(uint16_t *)(void *)(data + 2); + partial += *(uint16_t *)(void *)(data + 4); + partial += *(uint16_t *)(void *)(data + 6); + partial += *(uint16_t *)(void *)(data + 8); + partial += *(uint16_t *)(void *)(data + 10); + partial += *(uint16_t *)(void *)(data + 12); + partial += *(uint16_t *)(void *)(data + 14); + partial += *(uint16_t *)(void *)(data + 16); + partial += *(uint16_t *)(void *)(data + 18); + partial += *(uint16_t *)(void *)(data + 20); + partial += *(uint16_t *)(void *)(data + 22); + partial += *(uint16_t *)(void *)(data + 24); + partial += *(uint16_t *)(void *)(data + 26); + partial += *(uint16_t *)(void *)(data + 28); + partial += *(uint16_t *)(void *)(data + 30); + data += 32; + mlen -= 32; + if (__improbable(partial & 0xc0000000)) { + if (needs_swap) + partial = (partial << 8) + + (partial >> 24); + sum += (partial >> 16); + sum += (partial & 0xffff); + partial = 0; + } + } + if (mlen & 16) { + partial += *(uint16_t *)(void *)data; + partial += *(uint16_t *)(void *)(data + 2); + partial += *(uint16_t *)(void *)(data + 4); + partial += *(uint16_t *)(void *)(data + 6); + partial += *(uint16_t *)(void *)(data + 8); + partial += *(uint16_t *)(void *)(data + 10); + partial += *(uint16_t *)(void *)(data + 12); + partial += *(uint16_t *)(void *)(data + 14); + data += 16; + mlen -= 16; + } + /* + * mlen is not updated below as the remaining tests + * are using bit masks, which are not affected. + */ + if (mlen & 8) { + partial += *(uint16_t *)(void *)data; + partial += *(uint16_t *)(void *)(data + 2); + partial += *(uint16_t *)(void *)(data + 4); + partial += *(uint16_t *)(void *)(data + 6); + data += 8; + } + if (mlen & 4) { + partial += *(uint16_t *)(void *)data; + partial += *(uint16_t *)(void *)(data + 2); + data += 4; + } + if (mlen & 2) { + partial += *(uint16_t *)(void *)data; + data += 2; + } + if (mlen & 1) { +#if BYTE_ORDER == LITTLE_ENDIAN + partial += *data; +#else /* BYTE_ORDER != LITTLE_ENDIAN */ + partial += *data << 8; +#endif /* BYTE_ORDER != LITTLE_ENDIAN */ + started_on_odd = !started_on_odd; + } + + if (needs_swap) + partial = (partial << 8) + (partial >> 24); + sum += (partial >> 16) + (partial & 0xffff); + /* + * Reduce sum to allow potential byte swap + * in the next iteration without carry. + */ + sum = (sum >> 16) + (sum & 0xffff); + } + final_acc = ((sum >> 16) & 0xffff) + (sum & 0xffff); + final_acc = (final_acc >> 16) + (final_acc & 0xffff); + return (final_acc & 0xffff); +} + +#else /* __LP64__ */ +/* 64-bit version */ +uint32_t +in_cksum_mbuf_ref(struct mbuf *m, int len, int off, uint32_t initial_sum) +{ + int mlen; + uint64_t sum, partial; + unsigned int final_acc; + uint8_t *data; + boolean_t needs_swap, started_on_odd; + + VERIFY(len >= 0); + VERIFY(off >= 0); + + needs_swap = FALSE; + started_on_odd = FALSE; + sum = initial_sum; + + for (;;) { + if (__improbable(m == NULL)) { + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); + } + mlen = m->m_len; + if (mlen > off) { + mlen -= off; + data = mtod(m, uint8_t *) + off; + goto post_initial_offset; + } + off -= mlen; + if (len == 0) + break; + m = m->m_next; + } + + for (; len > 0; m = m->m_next) { + if (__improbable(m == NULL)) { + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); + } + mlen = m->m_len; + data = mtod(m, uint8_t *); +post_initial_offset: + if (mlen == 0) + continue; + if (mlen > len) + mlen = len; + len -= mlen; + + partial = 0; + if ((uintptr_t)data & 1) { + /* Align on word boundary */ + started_on_odd = !started_on_odd; +#if BYTE_ORDER == LITTLE_ENDIAN + partial = *data << 8; +#else /* BYTE_ORDER != LITTLE_ENDIAN */ + partial = *data; +#endif /* BYTE_ORDER != LITTLE_ENDIAN */ + ++data; + --mlen; + } + needs_swap = started_on_odd; + if ((uintptr_t)data & 2) { + if (mlen < 2) + goto trailing_bytes; + partial += *(uint16_t *)(void *)data; + data += 2; + mlen -= 2; + } + while (mlen >= 64) { + __builtin_prefetch(data + 32); + __builtin_prefetch(data + 64); + partial += *(uint32_t *)(void *)data; + partial += *(uint32_t *)(void *)(data + 4); + partial += *(uint32_t *)(void *)(data + 8); + partial += *(uint32_t *)(void *)(data + 12); + partial += *(uint32_t *)(void *)(data + 16); + partial += *(uint32_t *)(void *)(data + 20); + partial += *(uint32_t *)(void *)(data + 24); + partial += *(uint32_t *)(void *)(data + 28); + partial += *(uint32_t *)(void *)(data + 32); + partial += *(uint32_t *)(void *)(data + 36); + partial += *(uint32_t *)(void *)(data + 40); + partial += *(uint32_t *)(void *)(data + 44); + partial += *(uint32_t *)(void *)(data + 48); + partial += *(uint32_t *)(void *)(data + 52); + partial += *(uint32_t *)(void *)(data + 56); + partial += *(uint32_t *)(void *)(data + 60); + data += 64; + mlen -= 64; + if (__improbable(partial & (3ULL << 62))) { + if (needs_swap) + partial = (partial << 8) + + (partial >> 56); + sum += (partial >> 32); + sum += (partial & 0xffffffff); + partial = 0; + } + } + /* + * mlen is not updated below as the remaining tests + * are using bit masks, which are not affected. + */ + if (mlen & 32) { + partial += *(uint32_t *)(void *)data; + partial += *(uint32_t *)(void *)(data + 4); + partial += *(uint32_t *)(void *)(data + 8); + partial += *(uint32_t *)(void *)(data + 12); + partial += *(uint32_t *)(void *)(data + 16); + partial += *(uint32_t *)(void *)(data + 20); + partial += *(uint32_t *)(void *)(data + 24); + partial += *(uint32_t *)(void *)(data + 28); + data += 32; + } + if (mlen & 16) { + partial += *(uint32_t *)(void *)data; + partial += *(uint32_t *)(void *)(data + 4); + partial += *(uint32_t *)(void *)(data + 8); + partial += *(uint32_t *)(void *)(data + 12); + data += 16; + } + if (mlen & 8) { + partial += *(uint32_t *)(void *)data; + partial += *(uint32_t *)(void *)(data + 4); + data += 8; + } + if (mlen & 4) { + partial += *(uint32_t *)(void *)data; + data += 4; + } + if (mlen & 2) { + partial += *(uint16_t *)(void *)data; + data += 2; + } +trailing_bytes: + if (mlen & 1) { +#if BYTE_ORDER == LITTLE_ENDIAN + partial += *data; +#else /* BYTE_ORDER != LITTLE_ENDIAN */ + partial += *data << 8; +#endif /* BYTE_ORDER != LITTLE_ENDIAN */ + started_on_odd = !started_on_odd; + } + + if (needs_swap) + partial = (partial << 8) + (partial >> 56); + sum += (partial >> 32) + (partial & 0xffffffff); + /* + * Reduce sum to allow potential byte swap + * in the next iteration without carry. + */ + sum = (sum >> 32) + (sum & 0xffffffff); + } + final_acc = (sum >> 48) + ((sum >> 32) & 0xffff) + + ((sum >> 16) & 0xffff) + (sum & 0xffff); + final_acc = (final_acc >> 16) + (final_acc & 0xffff); + final_acc = (final_acc >> 16) + (final_acc & 0xffff); + return (final_acc & 0xffff); +} +#endif /* __LP64 */ +#endif /* DEBUG || DEVELOPMENT */ diff --git a/bsd/netinet/in_mcast.c b/bsd/netinet/in_mcast.c index 1d1b56563..3d8ca9699 100644 --- a/bsd/netinet/in_mcast.c +++ b/bsd/netinet/in_mcast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2016 Apple Inc. All rights reserved. + * Copyright (c) 2010-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -80,6 +80,7 @@ #include #include +#include #include #include @@ -89,17 +90,6 @@ #include #include -#ifndef __SOCKUNION_DECLARED -union sockunion { - struct sockaddr_storage ss; - struct sockaddr sa; - struct sockaddr_dl sdl; - struct sockaddr_in sin; -}; -typedef union sockunion sockunion_t; -#define __SOCKUNION_DECLARED -#endif /* __SOCKUNION_DECLARED */ - /* * Functions with non-static linkage defined in this file should be * declared in in_var.h: @@ -128,10 +118,10 @@ static void imf_rollback(struct in_mfilter *); static void imf_reap(struct in_mfilter *); static int imo_grow(struct ip_moptions *, size_t); static size_t imo_match_group(const struct ip_moptions *, - const struct ifnet *, const struct sockaddr *); + const struct ifnet *, const struct sockaddr_in *); static struct in_msource * imo_match_source(const struct ip_moptions *, const size_t, - const struct sockaddr *); + const struct sockaddr_in *); static void ims_merge(struct ip_msource *ims, const struct in_msource *lims, const int rollback); static int in_getmulti(struct ifnet *, const struct in_addr *, @@ -335,16 +325,14 @@ imo_grow(struct ip_moptions *imo, size_t newmax) */ static size_t imo_match_group(const struct ip_moptions *imo, const struct ifnet *ifp, - const struct sockaddr *group) + const struct sockaddr_in *group) { - const struct sockaddr_in *gsin; struct in_multi *pinm; int idx; int nmships; IMO_LOCK_ASSERT_HELD(__DECONST(struct ip_moptions *, imo)); - gsin = (struct sockaddr_in *)(uintptr_t)(size_t)group; /* The imo_membership array may be lazy allocated. */ if (imo->imo_membership == NULL || imo->imo_num_memberships == 0) @@ -357,7 +345,7 @@ imo_match_group(const struct ip_moptions *imo, const struct ifnet *ifp, continue; INM_LOCK(pinm); if ((ifp == NULL || (pinm->inm_ifp == ifp)) && - in_hosteq(pinm->inm_addr, gsin->sin_addr)) { + in_hosteq(pinm->inm_addr, group->sin_addr)) { INM_UNLOCK(pinm); break; } @@ -378,16 +366,15 @@ imo_match_group(const struct ip_moptions *imo, const struct ifnet *ifp, */ static struct in_msource * imo_match_source(const struct ip_moptions *imo, const size_t gidx, - const struct sockaddr *src) + const struct sockaddr_in *src) { struct ip_msource find; struct in_mfilter *imf; struct ip_msource *ims; - const sockunion_t *psa; IMO_LOCK_ASSERT_HELD(__DECONST(struct ip_moptions *, imo)); - VERIFY(src->sa_family == AF_INET); + VERIFY(src->sin_family == AF_INET); VERIFY(gidx != (size_t)-1 && gidx < imo->imo_num_memberships); /* The imo_mfilters array may be lazy allocated. */ @@ -396,8 +383,7 @@ imo_match_source(const struct ip_moptions *imo, const size_t gidx, imf = &imo->imo_mfilters[gidx]; /* Source trees are keyed in host byte order. */ - psa = (sockunion_t *)(uintptr_t)(size_t)src; - find.ims_haddr = ntohl(psa->sin.sin_addr.s_addr); + find.ims_haddr = ntohl(src->sin_addr.s_addr); ims = RB_FIND(ip_msource_tree, &imf->imf_sources, &find); return ((struct in_msource *)ims); @@ -411,7 +397,7 @@ imo_match_source(const struct ip_moptions *imo, const size_t gidx, */ int imo_multi_filter(const struct ip_moptions *imo, const struct ifnet *ifp, - const struct sockaddr *group, const struct sockaddr *src) + const struct sockaddr_in *group, const struct sockaddr_in *src) { size_t gidx; struct in_msource *ims; @@ -1077,7 +1063,7 @@ ims_merge(struct ip_msource *ims, const struct in_msource *lims, static int inm_merge(struct in_multi *inm, /*const*/ struct in_mfilter *imf) { - struct ip_msource *ims, *nims; + struct ip_msource *ims, *nims = NULL; struct in_msource *lims; int schanged, error; int nsrc0, nsrc1; @@ -1455,7 +1441,7 @@ static int inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) { struct group_source_req gsr; - sockunion_t *gsa, *ssa; + struct sockaddr_in *gsa, *ssa; struct ifnet *ifp; struct in_mfilter *imf; struct ip_moptions *imo; @@ -1473,8 +1459,8 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) doblock = 0; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - ssa = (sockunion_t *)&gsr.gsr_source; + gsa = (struct sockaddr_in *)&gsr.gsr_group; + ssa = (struct sockaddr_in *)&gsr.gsr_source; switch (sopt->sopt_name) { case IP_BLOCK_SOURCE: @@ -1487,13 +1473,13 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - gsa->sin.sin_family = AF_INET; - gsa->sin.sin_len = sizeof(struct sockaddr_in); - gsa->sin.sin_addr = mreqs.imr_multiaddr; + gsa->sin_family = AF_INET; + gsa->sin_len = sizeof(struct sockaddr_in); + gsa->sin_addr = mreqs.imr_multiaddr; - ssa->sin.sin_family = AF_INET; - ssa->sin.sin_len = sizeof(struct sockaddr_in); - ssa->sin.sin_addr = mreqs.imr_sourceaddr; + ssa->sin_family = AF_INET; + ssa->sin_len = sizeof(struct sockaddr_in); + ssa->sin_addr = mreqs.imr_sourceaddr; if (!in_nullhost(mreqs.imr_interface)) ifp = ip_multicast_if(&mreqs.imr_interface, &ifindex); @@ -1515,12 +1501,12 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin.sin_family != AF_INET || - gsa->sin.sin_len != sizeof(struct sockaddr_in)) + if (gsa->sin_family != AF_INET || + gsa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); - if (ssa->sin.sin_family != AF_INET || - ssa->sin.sin_len != sizeof(struct sockaddr_in)) + if (ssa->sin_family != AF_INET || + ssa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); ifnet_head_lock_shared(); @@ -1546,7 +1532,7 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) return (EOPNOTSUPP); } - if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr))) + if (!IN_MULTICAST(ntohl(gsa->sin_addr.s_addr))) return (EINVAL); /* @@ -1557,7 +1543,7 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IMO_LOCK(imo); - idx = imo_match_group(imo, ifp, &gsa->sa); + idx = imo_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->imo_mfilters == NULL) { error = EADDRNOTAVAIL; goto out_imo_locked; @@ -1583,9 +1569,9 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) * Asked to unblock, but nothing to unblock. * If adding a new block entry, allocate it. */ - ims = imo_match_source(imo, idx, &ssa->sa); + ims = imo_match_source(imo, idx, ssa); if ((ims != NULL && doblock) || (ims == NULL && !doblock)) { - IGMP_INET_PRINTF(ssa->sin.sin_addr, + IGMP_INET_PRINTF(ssa->sin_addr, ("%s: source %s %spresent\n", __func__, _igmp_inet_buf, doblock ? "" : "not ")); error = EADDRNOTAVAIL; @@ -1597,12 +1583,12 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) */ if (doblock) { IGMP_PRINTF(("%s: %s source\n", __func__, "block")); - ims = imf_graft(imf, fmode, &ssa->sin); + ims = imf_graft(imf, fmode, ssa); if (ims == NULL) error = ENOMEM; } else { IGMP_PRINTF(("%s: %s source\n", __func__, "allow")); - error = imf_prune(imf, &ssa->sin); + error = imf_prune(imf, ssa); } if (error) { @@ -1713,7 +1699,7 @@ inp_get_source_filters(struct inpcb *inp, struct sockopt *sopt) { struct __msfilterreq64 msfr, msfr64; struct __msfilterreq32 msfr32; - sockunion_t *gsa; + struct sockaddr_in *gsa; struct ifnet *ifp; struct ip_moptions *imo; struct in_mfilter *imf; @@ -1770,8 +1756,9 @@ inp_get_source_filters(struct inpcb *inp, struct sockopt *sopt) /* * Lookup group on the socket. */ - gsa = (sockunion_t *)&msfr.msfr_group; - idx = imo_match_group(imo, ifp, &gsa->sa); + gsa = (struct sockaddr_in *)&msfr.msfr_group; + + idx = imo_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->imo_mfilters == NULL) { IMO_UNLOCK(imo); return (EADDRNOTAVAIL); @@ -2063,7 +2050,7 @@ int inp_join_group(struct inpcb *inp, struct sockopt *sopt) { struct group_source_req gsr; - sockunion_t *gsa, *ssa; + struct sockaddr_in *gsa, *ssa; struct ifnet *ifp; struct in_mfilter *imf; struct ip_moptions *imo; @@ -2080,10 +2067,10 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) is_new = 0; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - gsa->ss.ss_family = AF_UNSPEC; - ssa = (sockunion_t *)&gsr.gsr_source; - ssa->ss.ss_family = AF_UNSPEC; + gsa = (struct sockaddr_in *)&gsr.gsr_group; + gsa->sin_family = AF_UNSPEC; + ssa = (struct sockaddr_in *)&gsr.gsr_source; + ssa->sin_family = AF_UNSPEC; switch (sopt->sopt_name) { case IP_ADD_MEMBERSHIP: @@ -2112,21 +2099,20 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) return (error); } - gsa->sin.sin_family = AF_INET; - gsa->sin.sin_len = sizeof(struct sockaddr_in); - gsa->sin.sin_addr = mreqs.imr_multiaddr; + gsa->sin_family = AF_INET; + gsa->sin_len = sizeof(struct sockaddr_in); + gsa->sin_addr = mreqs.imr_multiaddr; if (sopt->sopt_name == IP_ADD_SOURCE_MEMBERSHIP) { - ssa->sin.sin_family = AF_INET; - ssa->sin.sin_len = sizeof(struct sockaddr_in); - ssa->sin.sin_addr = mreqs.imr_sourceaddr; + ssa->sin_family = AF_INET; + ssa->sin_len = sizeof(struct sockaddr_in); + ssa->sin_addr = mreqs.imr_sourceaddr; } - if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr))) + if (!IN_MULTICAST(ntohl(gsa->sin_addr.s_addr))) return (EINVAL); - ifp = inp_lookup_mcast_ifp(inp, &gsa->sin, - mreqs.imr_interface); + ifp = inp_lookup_mcast_ifp(inp, gsa, mreqs.imr_interface); IGMP_INET_PRINTF(mreqs.imr_interface, ("%s: imr_interface = %s, ifp = 0x%llx\n", __func__, _igmp_inet_buf, (uint64_t)VM_KERNEL_ADDRPERM(ifp))); @@ -2147,23 +2133,23 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin.sin_family != AF_INET || - gsa->sin.sin_len != sizeof(struct sockaddr_in)) + if (gsa->sin_family != AF_INET || + gsa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); /* * Overwrite the port field if present, as the sockaddr * being copied in may be matched with a binary comparison. */ - gsa->sin.sin_port = 0; + gsa->sin_port = 0; if (sopt->sopt_name == MCAST_JOIN_SOURCE_GROUP) { - if (ssa->sin.sin_family != AF_INET || - ssa->sin.sin_len != sizeof(struct sockaddr_in)) + if (ssa->sin_family != AF_INET || + ssa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); - ssa->sin.sin_port = 0; + ssa->sin_port = 0; } - if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr))) + if (!IN_MULTICAST(ntohl(gsa->sin_addr.s_addr))) return (EINVAL); ifnet_head_lock_shared(); @@ -2186,18 +2172,26 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) if (ifp == NULL || (ifp->if_flags & IFF_MULTICAST) == 0) return (EADDRNOTAVAIL); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_mcast_join_total); + /* + * TBD: revisit the criteria for non-OS initiated joins + */ + if (inp->inp_lport == htons(5353)) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_mcast_join_os_total); + } + imo = inp_findmoptions(inp); if (imo == NULL) return (ENOMEM); IMO_LOCK(imo); - idx = imo_match_group(imo, ifp, &gsa->sa); + idx = imo_match_group(imo, ifp, gsa); if (idx == (size_t)-1) { is_new = 1; } else { inm = imo->imo_membership[idx]; imf = &imo->imo_mfilters[idx]; - if (ssa->ss.ss_family != AF_UNSPEC) { + if (ssa->sin_family != AF_UNSPEC) { /* * MCAST_JOIN_SOURCE_GROUP on an exclusive membership * is an error. On an existing inclusive membership, @@ -2223,7 +2217,7 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) * full-state SSM API with the delta-based API, * which is discouraged in the relevant RFCs. */ - lims = imo_match_source(imo, idx, &ssa->sa); + lims = imo_match_source(imo, idx, ssa); if (lims != NULL /*&& lims->imsl_st[1] == MCAST_INCLUDE*/) { error = EADDRNOTAVAIL; @@ -2281,7 +2275,7 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) * been allocated yet if this is a new membership, however, * the in_mfilter slot will be allocated and must be initialized. */ - if (ssa->ss.ss_family != AF_UNSPEC) { + if (ssa->sin_family != AF_UNSPEC) { /* Membership starts in IN mode */ if (is_new) { IGMP_PRINTF(("%s: new join w/source\n", __func__)); @@ -2289,7 +2283,7 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) } else { IGMP_PRINTF(("%s: %s source\n", __func__, "allow")); } - lims = imf_graft(imf, MCAST_INCLUDE, &ssa->sin); + lims = imf_graft(imf, MCAST_INCLUDE, ssa); if (lims == NULL) { IGMP_PRINTF(("%s: merge imf state failed\n", __func__)); @@ -2319,7 +2313,7 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) socket_unlock(inp->inp_socket, 0); VERIFY(inm == NULL); - error = in_joingroup(ifp, &gsa->sin.sin_addr, imf, &inm); + error = in_joingroup(ifp, &gsa->sin_addr, imf, &inm); socket_lock(inp->inp_socket, 0); IMO_REMREF(imo); @@ -2388,7 +2382,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) { struct group_source_req gsr; struct ip_mreq_source mreqs; - sockunion_t *gsa, *ssa; + struct sockaddr_in *gsa, *ssa; struct ifnet *ifp; struct in_mfilter *imf; struct ip_moptions *imo; @@ -2405,10 +2399,8 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) is_final = 1; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - gsa->ss.ss_family = AF_UNSPEC; - ssa = (sockunion_t *)&gsr.gsr_source; - ssa->ss.ss_family = AF_UNSPEC; + gsa = (struct sockaddr_in *)&gsr.gsr_group; + ssa = (struct sockaddr_in *)&gsr.gsr_source; switch (sopt->sopt_name) { case IP_DROP_MEMBERSHIP: @@ -2432,14 +2424,14 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - gsa->sin.sin_family = AF_INET; - gsa->sin.sin_len = sizeof(struct sockaddr_in); - gsa->sin.sin_addr = mreqs.imr_multiaddr; + gsa->sin_family = AF_INET; + gsa->sin_len = sizeof(struct sockaddr_in); + gsa->sin_addr = mreqs.imr_multiaddr; if (sopt->sopt_name == IP_DROP_SOURCE_MEMBERSHIP) { - ssa->sin.sin_family = AF_INET; - ssa->sin.sin_len = sizeof(struct sockaddr_in); - ssa->sin.sin_addr = mreqs.imr_sourceaddr; + ssa->sin_family = AF_INET; + ssa->sin_len = sizeof(struct sockaddr_in); + ssa->sin_addr = mreqs.imr_sourceaddr; } /* * Attempt to look up hinted ifp from interface address. @@ -2471,13 +2463,13 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin.sin_family != AF_INET || - gsa->sin.sin_len != sizeof(struct sockaddr_in)) + if (gsa->sin_family != AF_INET || + gsa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); if (sopt->sopt_name == MCAST_LEAVE_SOURCE_GROUP) { - if (ssa->sin.sin_family != AF_INET || - ssa->sin.sin_len != sizeof(struct sockaddr_in)) + if (ssa->sin_family != AF_INET || + ssa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); } @@ -2498,7 +2490,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) return (EOPNOTSUPP); } - if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr))) + if (!IN_MULTICAST(ntohl(gsa->sin_addr.s_addr))) return (EINVAL); /* @@ -2509,7 +2501,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IMO_LOCK(imo); - idx = imo_match_group(imo, ifp, &gsa->sa); + idx = imo_match_group(imo, ifp, gsa); if (idx == (size_t)-1) { error = EADDRNOTAVAIL; goto out_locked; @@ -2517,7 +2509,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) inm = imo->imo_membership[idx]; imf = &imo->imo_mfilters[idx]; - if (ssa->ss.ss_family != AF_UNSPEC) { + if (ssa->sin_family != AF_UNSPEC) { IGMP_PRINTF(("%s: opt=%d is_final=0\n", __func__, sopt->sopt_name)); is_final = 0; @@ -2538,16 +2530,16 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) error = EADDRNOTAVAIL; goto out_locked; } - ims = imo_match_source(imo, idx, &ssa->sa); + ims = imo_match_source(imo, idx, ssa); if (ims == NULL) { - IGMP_INET_PRINTF(ssa->sin.sin_addr, + IGMP_INET_PRINTF(ssa->sin_addr, ("%s: source %s %spresent\n", __func__, _igmp_inet_buf, "not ")); error = EADDRNOTAVAIL; goto out_locked; } IGMP_PRINTF(("%s: %s source\n", __func__, "block")); - error = imf_prune(imf, &ssa->sin); + error = imf_prune(imf, ssa); if (error) { IGMP_PRINTF(("%s: merge imf state failed\n", __func__)); @@ -2647,6 +2639,7 @@ inp_set_multicast_if(struct inpcb *inp, struct sockopt *sopt) int error = 0 ; unsigned int ifindex = 0; + bzero(&addr, sizeof(addr)); if (sopt->sopt_valsize == sizeof(struct ip_mreqn)) { /* * An interface index was specified using the @@ -2728,7 +2721,7 @@ inp_set_source_filters(struct inpcb *inp, struct sockopt *sopt) { struct __msfilterreq64 msfr, msfr64; struct __msfilterreq32 msfr32; - sockunion_t *gsa; + struct sockaddr_in *gsa; struct ifnet *ifp; struct in_mfilter *imf; struct ip_moptions *imo; @@ -2773,11 +2766,11 @@ inp_set_source_filters(struct inpcb *inp, struct sockopt *sopt) msfr.msfr_group.ss_len != sizeof(struct sockaddr_in)) return (EINVAL); - gsa = (sockunion_t *)&msfr.msfr_group; - if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr))) + gsa = (struct sockaddr_in *)&msfr.msfr_group; + if (!IN_MULTICAST(ntohl(gsa->sin_addr.s_addr))) return (EINVAL); - gsa->sin.sin_port = 0; /* ignore port */ + gsa->sin_port = 0; /* ignore port */ ifnet_head_lock_shared(); if (msfr.msfr_ifindex == 0 || (u_int)if_index < msfr.msfr_ifindex) { @@ -2798,7 +2791,7 @@ inp_set_source_filters(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IMO_LOCK(imo); - idx = imo_match_group(imo, ifp, &gsa->sa); + idx = imo_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->imo_mfilters == NULL) { error = EADDRNOTAVAIL; goto out_imo_locked; @@ -3571,7 +3564,10 @@ in_multihead_lock_shared(void) void in_multihead_lock_assert(int what) { - lck_rw_assert(&in_multihead_lock, what); +#if !MACH_ASSERT +#pragma unused(what) +#endif + LCK_RW_ASSERT(&in_multihead_lock, what); } void diff --git a/bsd/netinet/in_pcb.c b/bsd/netinet/in_pcb.c index c20961c22..3371c71b3 100644 --- a/bsd/netinet/in_pcb.c +++ b/bsd/netinet/in_pcb.c @@ -135,12 +135,6 @@ static boolean_t intcoproc_unrestricted = FALSE; extern char *proc_best_name(proc_t); -/* - * If the total number of gc reqs is above a threshold, schedule - * garbage collect timer sooner - */ -static boolean_t inpcb_toomany_gcreq = FALSE; - #define INPCB_GCREQ_THRESHOLD 50000 static thread_call_t inpcb_thread_call, inpcb_fast_thread_call; @@ -221,7 +215,16 @@ SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, static uint32_t apn_fallbk_debug = 0; #define apn_fallbk_log(x) do { if (apn_fallbk_debug >= 1) log x; } while (0) +#if CONFIG_EMBEDDED +static boolean_t apn_fallbk_enabled = TRUE; + +SYSCTL_DECL(_net_inet); +SYSCTL_NODE(_net_inet, OID_AUTO, apn_fallback, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "APN Fallback"); +SYSCTL_UINT(_net_inet_apn_fallback, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED, + &apn_fallbk_debug, 0, "APN fallback debug enable"); +#else static boolean_t apn_fallbk_enabled = FALSE; +#endif extern int udp_use_randomport; extern int tcp_use_randomport; @@ -315,16 +318,10 @@ in_pcbinit(void) static void inpcb_timeout(void *arg0, void *arg1) { -#pragma unused(arg0) +#pragma unused(arg0, arg1) struct inpcbinfo *ipi; boolean_t t, gc; struct intimercount gccnt, tmcnt; - boolean_t toomany_gc = FALSE; - - if (arg1 != NULL) { - VERIFY(arg1 == &inpcb_toomany_gcreq); - toomany_gc = *(boolean_t *)arg1; - } /* * Update coarse-grained networking timestamp (in sec.); the idea @@ -368,7 +365,7 @@ inpcb_timeout(void *arg0, void *arg1) ipi->ipi_timer(ipi); tmcnt.intimer_lazy += ipi->ipi_timer_req.intimer_lazy; - tmcnt.intimer_lazy += + tmcnt.intimer_fast += ipi->ipi_timer_req.intimer_fast; tmcnt.intimer_nodelay += ipi->ipi_timer_req.intimer_nodelay; @@ -386,12 +383,8 @@ inpcb_timeout(void *arg0, void *arg1) inpcb_ticking = INPCB_HAVE_TIMER_REQ(tmcnt); /* re-arm the timer if there's work to do */ - if (toomany_gc) { - inpcb_toomany_gcreq = FALSE; - } else { - inpcb_timeout_run--; - VERIFY(inpcb_timeout_run >= 0 && inpcb_timeout_run < 2); - } + inpcb_timeout_run--; + VERIFY(inpcb_timeout_run >= 0 && inpcb_timeout_run < 2); if (gccnt.intimer_nodelay > 0 || tmcnt.intimer_nodelay > 0) inpcb_sched_timeout(); @@ -422,7 +415,7 @@ _inpcb_sched_timeout(unsigned int offset) uint64_t deadline, leeway; clock_interval_to_deadline(1, NSEC_PER_SEC, &deadline); - lck_mtx_assert(&inpcb_timeout_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&inpcb_timeout_lock, LCK_MTX_ASSERT_OWNED); if (inpcb_timeout_run == 0 && (inpcb_garbage_collecting || inpcb_ticking)) { lck_mtx_convert_spin(&inpcb_timeout_lock); @@ -457,25 +450,14 @@ void inpcb_gc_sched(struct inpcbinfo *ipi, u_int32_t type) { u_int32_t gccnt; - uint64_t deadline; lck_mtx_lock_spin(&inpcb_timeout_lock); inpcb_garbage_collecting = TRUE; gccnt = ipi->ipi_gc_req.intimer_nodelay + ipi->ipi_gc_req.intimer_fast; - if (gccnt > INPCB_GCREQ_THRESHOLD && !inpcb_toomany_gcreq) { - inpcb_toomany_gcreq = TRUE; - - /* - * There are toomany pcbs waiting to be garbage collected, - * schedule a much faster timeout in addition to - * the caller's request - */ - lck_mtx_convert_spin(&inpcb_timeout_lock); - clock_interval_to_deadline(100, NSEC_PER_MSEC, &deadline); - thread_call_enter1_delayed(inpcb_thread_call, - &inpcb_toomany_gcreq, deadline); + if (gccnt > INPCB_GCREQ_THRESHOLD) { + type = INPCB_TIMER_FAST; } switch (type) { @@ -681,7 +663,7 @@ in_pcblookup_local_and_cleanup(struct inpcbinfo *pcbinfo, struct in_addr laddr, if (inp != NULL && inp->inp_wantcnt == WNT_STOPUSING) { struct socket *so = inp->inp_socket; - lck_mtx_lock(&inp->inpcb_mtx); + socket_lock(so, 0); if (so->so_usecount == 0) { if (inp->inp_state != INPCB_STATE_DEAD) @@ -689,7 +671,7 @@ in_pcblookup_local_and_cleanup(struct inpcbinfo *pcbinfo, struct in_addr laddr, in_pcbdispose(inp); /* will unlock & destroy */ inp = NULL; } else { - lck_mtx_unlock(&inp->inpcb_mtx); + socket_unlock(so, 0); } } @@ -824,6 +806,7 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) struct inpcb *t; uid_t u; +#if !CONFIG_EMBEDDED if (ntohs(lport) < IPPORT_RESERVED) { cred = kauth_cred_proc_ref(p); error = priv_check_cred(cred, @@ -835,6 +818,7 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) return (EACCES); } } +#endif /* !CONFIG_EMBEDDED */ if (!IN_MULTICAST(ntohl(SIN(nam)->sin_addr.s_addr)) && (u = kauth_cred_getuid(so->so_cred)) != 0 && (t = in_pcblookup_local_and_cleanup( @@ -891,6 +875,7 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) if (lport == 0) { u_short first, last; int count; + bool found; randomport = (so->so_flags & SOF_BINDRANDOMPORT) || (so->so_type == SOCK_STREAM ? tcp_use_randomport : @@ -935,16 +920,22 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) * is not being tested on each round of the loop. */ if (first > last) { + struct in_addr lookup_addr; + /* * counting down */ if (randomport) { - read_random(&rand_port, sizeof (rand_port)); + read_frandom(&rand_port, sizeof (rand_port)); *lastport = first - (rand_port % (first - last)); } count = first - last; + lookup_addr = (laddr.s_addr != INADDR_ANY) ? laddr : + inp->inp_laddr; + + found = false; do { if (count-- < 0) { /* completely used? */ lck_rw_done(pcbinfo->ipi_lock); @@ -955,20 +946,27 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) if (*lastport > first || *lastport < last) *lastport = first; lport = htons(*lastport); - } while (in_pcblookup_local_and_cleanup(pcbinfo, - ((laddr.s_addr != INADDR_ANY) ? laddr : - inp->inp_laddr), lport, wild)); + + found = in_pcblookup_local_and_cleanup(pcbinfo, + lookup_addr, lport, wild) == NULL; + } while (!found); } else { + struct in_addr lookup_addr; + /* * counting up */ if (randomport) { - read_random(&rand_port, sizeof (rand_port)); + read_frandom(&rand_port, sizeof (rand_port)); *lastport = first + (rand_port % (first - last)); } count = last - first; + lookup_addr = (laddr.s_addr != INADDR_ANY) ? laddr : + inp->inp_laddr; + + found = false; do { if (count-- < 0) { /* completely used? */ lck_rw_done(pcbinfo->ipi_lock); @@ -979,9 +977,10 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) if (*lastport < first || *lastport > last) *lastport = first; lport = htons(*lastport); - } while (in_pcblookup_local_and_cleanup(pcbinfo, - ((laddr.s_addr != INADDR_ANY) ? laddr : - inp->inp_laddr), lport, wild)); + + found = in_pcblookup_local_and_cleanup(pcbinfo, + lookup_addr, lport, wild) == NULL; + } while (!found); } } socket_lock(so, 0); @@ -1586,6 +1585,12 @@ in_pcbdetach(struct inpcb *inp) } #endif /* IPSEC */ + if (inp->inp_stat != NULL && SOCK_PROTO(so) == IPPROTO_UDP) { + if (inp->inp_stat->rxpackets == 0 && inp->inp_stat->txpackets == 0) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_no_data); + } + } + /* * Let NetworkStatistics know this PCB is going away * before we detach it. @@ -1664,7 +1669,7 @@ in_pcbdispose(struct inpcb *inp) } } - lck_rw_assert(ipi->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(ipi->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); inp->inp_gencnt = ++ipi->ipi_gencnt; /* access ipi in in_pcbremlists */ @@ -1687,6 +1692,11 @@ in_pcbdispose(struct inpcb *inp) /* NOTREACHED */ } lck_mtx_unlock(&inp->inpcb_mtx); + +#if NECP + necp_inpcb_remove_cb(inp); +#endif /* NECP */ + lck_mtx_destroy(&inp->inpcb_mtx, ipi->ipi_lock_grp); } /* makes sure we're not called twice from so_close */ @@ -1747,9 +1757,9 @@ in_getsockaddr(struct socket *so, struct sockaddr **nam) } int -in_getsockaddr_s(struct socket *so, struct sockaddr_storage *ss) +in_getsockaddr_s(struct socket *so, struct sockaddr_in *ss) { - struct sockaddr_in *sin = SIN(ss); + struct sockaddr_in *sin = ss; struct inpcb *inp; VERIFY(ss != NULL); @@ -1758,12 +1768,8 @@ in_getsockaddr_s(struct socket *so, struct sockaddr_storage *ss) sin->sin_family = AF_INET; sin->sin_len = sizeof (*sin); - if ((inp = sotoinpcb(so)) == NULL -#if NECP - || (necp_socket_should_use_flow_divert(inp)) -#endif /* NECP */ - ) - return (inp == NULL ? EINVAL : EPROTOTYPE); + if ((inp = sotoinpcb(so)) == NULL) + return (EINVAL); sin->sin_port = inp->inp_lport; sin->sin_addr = inp->inp_laddr; @@ -1797,31 +1803,6 @@ in_getpeeraddr(struct socket *so, struct sockaddr **nam) return (0); } -int -in_getpeeraddr_s(struct socket *so, struct sockaddr_storage *ss) -{ - struct sockaddr_in *sin = SIN(ss); - struct inpcb *inp; - - VERIFY(ss != NULL); - bzero(ss, sizeof (*ss)); - - sin->sin_family = AF_INET; - sin->sin_len = sizeof (*sin); - - if ((inp = sotoinpcb(so)) == NULL -#if NECP - || (necp_socket_should_use_flow_divert(inp)) -#endif /* NECP */ - ) { - return (inp == NULL ? EINVAL : EPROTOTYPE); - } - - sin->sin_port = inp->inp_fport; - sin->sin_addr = inp->inp_faddr; - return (0); -} - void in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno, void (*notify)(struct inpcb *, int)) @@ -2346,6 +2327,8 @@ in_pcbinshash(struct inpcb *inp, int locked) } VERIFY(!(inp->inp_flags2 & INP2_INHASHLIST)); + + inp->inp_phd = phd; LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); LIST_INSERT_HEAD(pcbhash, inp, inp_hash); @@ -2594,6 +2577,7 @@ inpcb_to_compat(struct inpcb *inp, struct inpcb_compat *inp_compat) inp_compat->inp_depend6.inp6_hops = inp->inp_depend6.inp6_hops; } +#if !CONFIG_EMBEDDED void inpcb_to_xinpcb64(struct inpcb *inp, struct xinpcb64 *xinp) { @@ -2613,6 +2597,7 @@ inpcb_to_xinpcb64(struct inpcb *inp, struct xinpcb64 *xinp) xinp->inp_depend6.inp6_ifindex = 0; xinp->inp_depend6.inp6_hops = inp->inp_depend6.inp6_hops; } +#endif /* !CONFIG_EMBEDDED */ /* * The following routines implement this scheme: @@ -2645,7 +2630,7 @@ inp_route_copyout(struct inpcb *inp, struct route *dst) { struct route *src = &inp->inp_route; - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(inp->inp_socket); /* * If the route in the PCB is stale or not for IPv4, blow it away; @@ -2662,7 +2647,7 @@ inp_route_copyin(struct inpcb *inp, struct route *src) { struct route *dst = &inp->inp_route; - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(inp->inp_socket); /* Minor sanity check */ if (src->ro_rt != NULL && rt_key(src->ro_rt)->sa_family != AF_INET) @@ -3487,3 +3472,16 @@ inp_decr_sndbytes_allunsent(struct socket *so, u_int32_t th_ack) len = inp_get_sndbytes_allunsent(so, th_ack); inp_decr_sndbytes_unsent(so, len); } + + +inline void +inp_set_activity_bitmap(struct inpcb *inp) +{ + in_stat_set_activity_bitmap(&inp->inp_nw_activity, net_uptime()); +} + +inline void +inp_get_activity_bitmap(struct inpcb *inp, activity_bitmap_t *ab) +{ + bcopy(&inp->inp_nw_activity, ab, sizeof (*ab)); +} diff --git a/bsd/netinet/in_pcb.h b/bsd/netinet/in_pcb.h index 11ad54fe2..588a4d054 100644 --- a/bsd/netinet/in_pcb.h +++ b/bsd/netinet/in_pcb.h @@ -78,16 +78,17 @@ #include #include #include +#include #endif /* BSD_KERNEL_PRIVATE */ +#if IPSEC #include /* for IPSEC */ +#endif /* IPSEC */ + #if NECP #include #endif -#if IPSEC -#include /* for IPSEC */ -#endif #ifdef BSD_KERNEL_PRIVATE /* @@ -219,6 +220,8 @@ struct inpcb { char *inp_account; } inp_necp_attributes; struct necp_inpcb_result inp_policyresult; + uuid_t necp_client_uuid; + void (*necp_cb)(void *, int, struct necp_client_flow *); #endif u_char *inp_keepalive_data; /* for keepalive offload */ u_int8_t inp_keepalive_datalen; /* keepalive data length */ @@ -233,6 +236,8 @@ struct inpcb { u_int8_t inp_cstat_store[sizeof (struct inp_stat) + sizeof (u_int64_t)]; u_int8_t inp_wstat_store[sizeof (struct inp_stat) + sizeof (u_int64_t)]; u_int8_t inp_Wstat_store[sizeof (struct inp_stat) + sizeof (u_int64_t)]; + activity_bitmap_t inp_nw_activity; + u_int64_t inp_start_timestamp; }; #define INP_ADD_STAT(_inp, _cnt_cellular, _cnt_wifi, _cnt_wired, _a, _n)\ @@ -367,6 +372,7 @@ struct xinpcb { u_quad_t xi_alignment_hack; }; +#if !CONFIG_EMBEDDED struct inpcb64_list_entry { u_int64_t le_next; u_int64_t le_prev; @@ -408,6 +414,7 @@ struct xinpcb64 { struct xsocket64 xi_socket; u_quad_t xi_alignment_hack; }; +#endif /* !CONFIG_EMBEDDED */ #ifdef PRIVATE struct xinpcb_list_entry { @@ -596,6 +603,7 @@ struct inpcbinfo { lck_grp_attr_t *ipi_lock_grp_attr; #define INPCBINFO_UPDATE_MSS 0x1 +#define INPCBINFO_HANDLE_LQM_ABORT 0x2 u_int32_t ipi_flags; }; @@ -699,6 +707,7 @@ struct inpcbinfo { #define INP2_AWDL_UNRESTRICTED 0x00000020 /* AWDL restricted mode allowed */ #define INP2_KEEPALIVE_OFFLOAD 0x00000040 /* Enable UDP or TCP keepalive offload */ #define INP2_INTCOPROC_ALLOWED 0x00000080 /* Allow communication via internal co-processor interfaces */ +#define INP2_CONNECT_IN_PROGRESS 0x00000100 /* A connect call is in progress, so binds are intermediate steps */ /* * Flags passed to in_pcblookup*() functions. @@ -768,13 +777,14 @@ extern void in_pcbnotifyall(struct inpcbinfo *, struct in_addr, int, void (*)(struct inpcb *, int)); extern void in_pcbrehash(struct inpcb *); extern int in_getpeeraddr(struct socket *, struct sockaddr **); -extern int in_getpeeraddr_s(struct socket *, struct sockaddr_storage *); extern int in_getsockaddr(struct socket *, struct sockaddr **); -extern int in_getsockaddr_s(struct socket *, struct sockaddr_storage *); +extern int in_getsockaddr_s(struct socket *, struct sockaddr_in *); extern int in_pcb_checkstate(struct inpcb *, int, int); extern void in_pcbremlists(struct inpcb *); extern void inpcb_to_compat(struct inpcb *, struct inpcb_compat *); +#if !CONFIG_EMBEDDED extern void inpcb_to_xinpcb64(struct inpcb *, struct xinpcb64 *); +#endif extern int get_pcblist_n(short, struct sysctl_req *, struct inpcbinfo *); #define INPCB_GET_PORTS_USED_WILDCARDOK 0x01 @@ -825,6 +835,8 @@ extern void inp_incr_sndbytes_unsent(struct socket *, int32_t); extern void inp_decr_sndbytes_unsent(struct socket *, int32_t); extern int32_t inp_get_sndbytes_allunsent(struct socket *, u_int32_t); extern void inp_decr_sndbytes_allunsent(struct socket *, u_int32_t); +extern void inp_set_activity_bitmap(struct inpcb *inp); +extern void inp_get_activity_bitmap(struct inpcb *inp, activity_bitmap_t *b); #endif /* BSD_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE /* exported for PPP */ diff --git a/bsd/netinet/in_pcblist.c b/bsd/netinet/in_pcblist.c index 5e667775b..c67d76635 100644 --- a/bsd/netinet/in_pcblist.c +++ b/bsd/netinet/in_pcblist.c @@ -71,6 +71,7 @@ #include #include +#include #include #include @@ -269,22 +270,22 @@ get_pcblist_n(short proto, struct sysctl_req *req, struct inpcbinfo *pcbinfo) if (proto == IPPROTO_TCP) item_size += ROUNDUP64(sizeof (struct xtcpcb_n)); - /* - * The process of preparing the PCB list is too time-consuming and - * resource-intensive to repeat twice on every request. - */ - lck_rw_lock_exclusive(pcbinfo->ipi_lock); if (req->oldptr == USER_ADDR_NULL) { n = pcbinfo->ipi_count; - req->oldidx = 2 * (sizeof (xig)) + (n + n/8) * item_size; - goto done; + req->oldidx = 2 * (sizeof (xig)) + (n + n/8 + 1) * item_size; + return 0; } if (req->newptr != USER_ADDR_NULL) { - error = EPERM; - goto done; + return EPERM; } + + /* + * The process of preparing the PCB list is too time-consuming and + * resource-intensive to repeat twice on every request. + */ + lck_rw_lock_exclusive(pcbinfo->ipi_lock); /* * OK, now we're committed to doing something. */ @@ -303,7 +304,7 @@ get_pcblist_n(short proto, struct sysctl_req *req, struct inpcbinfo *pcbinfo) /* * We are done if there is no pcb */ - if (n == 0) { + if (xig.xig_count == 0) { goto done; } @@ -375,8 +376,12 @@ get_pcblist_n(short proto, struct sysctl_req *req, struct inpcbinfo *pcbinfo) inp->inp_ppcb, xt); } error = SYSCTL_OUT(req, buf, item_size); + if (error) { + break; + } } } + if (!error) { /* * Give the user an updated idea of our state. @@ -394,6 +399,7 @@ get_pcblist_n(short proto, struct sysctl_req *req, struct inpcbinfo *pcbinfo) } done: lck_rw_done(pcbinfo->ipi_lock); + if (inp_list != NULL) FREE(inp_list, M_TEMP); if (buf != NULL) @@ -517,7 +523,7 @@ inpcb_get_ports_used(uint32_t ifindex, int protocol, uint32_t flags, port = ntohs(inp->inp_lport); if (port == 0) continue; - bit_set(bitfield, port); + bitstr_set(bitfield, port); } lck_rw_done(pcbinfo->ipi_lock); } diff --git a/bsd/netinet/in_rmx.c b/bsd/netinet/in_rmx.c index fa4b580dc..1c0fc3696 100644 --- a/bsd/netinet/in_rmx.c +++ b/bsd/netinet/in_rmx.c @@ -103,8 +103,6 @@ static int in_rtqkill(struct radix_node *, void *); static int in_ifadownkill(struct radix_node *, void *); -#define RTPRF_OURS RTF_PROTO3 /* set on routes we manage */ - /* * Do what we need to do when inserting a route. */ @@ -119,7 +117,7 @@ in_addroute(void *v_arg, void *n_arg, struct radix_node_head *head, uint32_t flags = rt->rt_flags; boolean_t verbose = (rt_verbose > 1); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (verbose) @@ -252,7 +250,7 @@ in_deleteroute(void *v_arg, void *netmask_arg, struct radix_node_head *head) { struct radix_node *rn; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); rn = rn_delete(v_arg, netmask_arg, head); if (rt_verbose > 1 && rn != NULL) { @@ -362,11 +360,11 @@ in_clsroute(struct radix_node *rn, struct radix_node_head *head) struct rtentry *rt = (struct rtentry *)rn; boolean_t verbose = (rt_verbose > 1); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (!(rt->rt_flags & RTF_UP)) - return; /* prophylactic measures */ + return; /* prophylactic measures */ if ((rt->rt_flags & (RTF_LLINFO | RTF_HOST)) != RTF_HOST) return; @@ -464,7 +462,7 @@ in_rtqkill(struct radix_node *rn, void *rock) int err; timenow = net_uptime(); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); if (rt->rt_flags & RTPRF_OURS) { @@ -483,6 +481,7 @@ in_rtqkill(struct radix_node *rn, void *rock) rt, rt->rt_refcnt); /* NOTREACHED */ } + if (verbose) { log(LOG_DEBUG, "%s: deleting route to " "%s->%s->%s, flags=%b, draining=%d\n", @@ -617,7 +616,7 @@ in_rtqtimo(void *targ) static void in_sched_rtqtimo(struct timeval *atv) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (!in_rtqtimo_run) { struct timeval tv; @@ -712,7 +711,7 @@ in_ifadownkill(struct radix_node *rn, void *xap) boolean_t verbose = (rt_verbose != 0); int err; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); if (rt->rt_ifa == ap->ifa && @@ -765,7 +764,7 @@ in_ifadown(struct ifaddr *ifa, int delete) struct in_ifadown_arg arg; struct radix_node_head *rnh; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* * Holding rnh_lock here prevents the possibility of diff --git a/bsd/netinet/in_stat.c b/bsd/netinet/in_stat.c new file mode 100644 index 000000000..c7f36defb --- /dev/null +++ b/bsd/netinet/in_stat.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include + +#define IN_STAT_ACTIVITY_GRANULARITY 8 /* 8 sec granularity */ +#define IN_STAT_ACTIVITY_TIME_SEC_SHIFT 3 /* 8 sec per bit */ +#define IN_STAT_ACTIVITY_BITMAP_TOTAL_SIZE ((uint64_t) 128) +#define IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE ((uint64_t) 64) +#define IN_STAT_ACTIVITY_TOTAL_TIME ((uint64_t) (8 * 128)) +#define IN_STAT_SET_MOST_SIGNIFICANT_BIT ((u_int64_t )0x8000000000000000) + +void +in_stat_set_activity_bitmap(activity_bitmap_t *activity, uint64_t now) +{ + uint64_t elapsed_time, slot; + uint64_t *bitmap; + if (activity->start == 0) + activity->start = now; + elapsed_time = now - activity->start; + + slot = elapsed_time >> IN_STAT_ACTIVITY_TIME_SEC_SHIFT; + if (slot < IN_STAT_ACTIVITY_BITMAP_TOTAL_SIZE) { + if (slot < IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE) { + bitmap = &activity->bitmap[0]; + } else { + bitmap = &activity->bitmap[1]; + slot -= IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE; + } + *bitmap |= (((u_int64_t) 1) << slot); + } else { + if (slot >= (IN_STAT_ACTIVITY_BITMAP_TOTAL_SIZE * 2)) { + activity->start = now - IN_STAT_ACTIVITY_TOTAL_TIME; + activity->bitmap[0] = activity->bitmap[1] = 0; + } else { + uint64_t shift = + slot - (IN_STAT_ACTIVITY_BITMAP_TOTAL_SIZE - 1); + /* + * Move the start time and bitmap forward to + * cover the lost time + */ + activity->start += + (shift << IN_STAT_ACTIVITY_TIME_SEC_SHIFT); + if (shift > IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE) { + activity->bitmap[0] = activity->bitmap[1]; + activity->bitmap[1] = 0; + shift -= IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE; + if (shift == IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE) + activity->bitmap[0] = 0; + else + activity->bitmap[0] >>= shift; + } else { + uint64_t mask_lower, tmp; + uint64_t b1_low, b0_high; + + /* + * generate a mask with all of lower + * 'shift' bits set + */ + tmp = (((uint64_t)1) << (shift - 1)); + mask_lower = ((tmp - 1) ^ tmp); + activity->bitmap[0] >>= shift; + b1_low = (activity->bitmap[1] & mask_lower); + + b0_high = (b1_low << + (IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE - + shift)); + activity->bitmap[0] |= b0_high; + activity->bitmap[1] >>= shift; + } + } + activity->bitmap[1] |= IN_STAT_SET_MOST_SIGNIFICANT_BIT; + } +} diff --git a/bsd/netinet/in_stat.h b/bsd/netinet/in_stat.h new file mode 100644 index 000000000..0c31148c7 --- /dev/null +++ b/bsd/netinet/in_stat.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef __NETINET_IN_STAT_H__ +#define __NETINET_IN_STAT_H__ + +#ifdef PRIVATE + +#include + +typedef struct activity_bitmap { + uint64_t start; /* Start timestamp using uptime */ + uint64_t bitmap[2]; /* 128 bit map, each bit == 8 sec */ +} activity_bitmap_t; + +#endif /* PRIVATE */ + +#ifdef BSD_KERNEL_PRIVATE + +extern void in_stat_set_activity_bitmap(activity_bitmap_t *activity, uint64_t now); + +#endif /* BSD_KERNEL_PRIVATE */ + +#endif /* __NETINET_IN_STAT_H__ */ diff --git a/bsd/netinet/in_tclass.c b/bsd/netinet/in_tclass.c index 3fe179ce9..5d627e551 100644 --- a/bsd/netinet/in_tclass.c +++ b/bsd/netinet/in_tclass.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2016 Apple Inc. All rights reserved. + * Copyright (c) 2009-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -1093,10 +1094,8 @@ set_tcp_stream_priority(struct socket *so) * happen when there is no indication of foreground * activity. */ - if (soissrcbackground(so) && - ((outifp->if_fg_sendts > 0 && - (int)(uptime - outifp->if_fg_sendts) <= - TCP_BG_SWITCH_TIME) || net_io_policy_throttled)) + if (soissrcbackground(so) && outifp->if_fg_sendts > 0 && + (int)(uptime - outifp->if_fg_sendts) <= TCP_BG_SWITCH_TIME) fg_active = true; /* @@ -1818,3 +1817,83 @@ sysctl_reset_dscp_to_wifi_ac_map SYSCTL_HANDLER_ARGS return (0); } + +/* + * Returns whether a large upload or download transfer should be marked as + * BK service type for network activity. This is a system level + * hint/suggestion to classify application traffic based on statistics + * collected from the current network attachment + * + * Returns 1 for BK and 0 for default + */ + +int +net_qos_guideline(struct proc *p, struct net_qos_guideline_args *arg, + int *retval) +{ +#pragma unused(p) +#define RETURN_USE_BK 1 +#define RETURN_USE_DEFAULT 0 + struct net_qos_param qos_arg; + struct ifnet *ipv4_primary, *ipv6_primary; + int err = 0; + + if (arg->param == USER_ADDR_NULL || retval == NULL || + arg->param_len != sizeof (qos_arg)) { + return (EINVAL); + } + err = copyin(arg->param, (caddr_t) &qos_arg, sizeof (qos_arg)); + if (err != 0) + return (err); + + *retval = RETURN_USE_DEFAULT; + ipv4_primary = ifindex2ifnet[get_primary_ifscope(AF_INET)]; + ipv6_primary = ifindex2ifnet[get_primary_ifscope(AF_INET6)]; + + /* + * If either of the interfaces is in Low Internet mode, enable + * background delay based algorithms on this transfer + */ + if (qos_arg.nq_uplink) { + if ((ipv4_primary != NULL && + (ipv4_primary->if_xflags & IFXF_LOW_INTERNET_UL)) || + (ipv6_primary != NULL && + (ipv6_primary->if_xflags & IFXF_LOW_INTERNET_UL))) { + *retval = RETURN_USE_BK; + return (0); + } + } else { + if ((ipv4_primary != NULL && + (ipv4_primary->if_xflags & IFXF_LOW_INTERNET_DL)) || + (ipv6_primary != NULL && + (ipv6_primary->if_xflags & IFXF_LOW_INTERNET_DL))) { + *retval = RETURN_USE_BK; + return (0); + } + } + + /* + * Some times IPv4 and IPv6 primary interfaces can be different. + * In this case, if either of them is non-cellular, we should mark + * the transfer as BK as it can potentially get used based on + * the host name resolution + */ + if (ipv4_primary != NULL && IFNET_IS_EXPENSIVE(ipv4_primary) && + ipv6_primary != NULL && IFNET_IS_EXPENSIVE(ipv6_primary)) { + if (qos_arg.nq_use_expensive) { + return (0); + } else { + *retval = RETURN_USE_BK; + return (0); + } + } + if (qos_arg.nq_transfer_size >= 5 * 1024 * 1024) { + *retval = RETURN_USE_BK; + return (0); + } + + +#undef RETURN_USE_BK +#undef RETURN_USE_DEFAULT + return (0); +} diff --git a/bsd/netinet/in_tclass.h b/bsd/netinet/in_tclass.h index 430de9f27..d42713162 100644 --- a/bsd/netinet/in_tclass.h +++ b/bsd/netinet/in_tclass.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -64,6 +64,29 @@ struct so_tcdbg { #define NET_QOS_MARKING_POLICY_ENABLE QOS_MODE_MARKING_POLICY_ENABLE /* obsolete, to be removed */ #define NET_QOS_MARKING_POLICY_DISABLE QOS_MODE_MARKING_POLICY_DISABLE /* obsolete, to be removed */ +struct net_qos_param { + u_int64_t nq_transfer_size; /* transfer size in bytes */ + u_int32_t nq_use_expensive:1, /* allowed = 1 otherwise 0 */ + nq_uplink:1; /* uplink = 1 otherwise 0 */ + u_int32_t nq_unused; /* for future expansion */ +}; + +#ifndef KERNEL + +/* + * Returns whether a large upload or download transfer should be marked as + * BK service type for network activity. This is a system level + * hint/suggestion to classify application traffic based on statistics + * collected from the current network attachment + * + * @param param transfer parameters + * @param param_len parameter length + * @return returns 1 for BK and 0 for default + */ +extern int net_qos_guideline(struct net_qos_param *param, size_t param_len); + +#endif /* !KERNEL */ + #ifdef BSD_KERNEL_PRIVATE extern int net_qos_policy_restricted; diff --git a/bsd/netinet/in_var.h b/bsd/netinet/in_var.h index 7368ed28d..5b0506120 100644 --- a/bsd/netinet/in_var.h +++ b/bsd/netinet/in_var.h @@ -147,6 +147,7 @@ struct kev_in_portinuse { #ifdef BSD_KERNEL_PRIVATE #include #include +#include #include #include /* @@ -335,10 +336,10 @@ struct in_multi { }; #define INM_LOCK_ASSERT_HELD(_inm) \ - lck_mtx_assert(&(_inm)->inm_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_inm)->inm_lock, LCK_MTX_ASSERT_OWNED) #define INM_LOCK_ASSERT_NOTHELD(_inm) \ - lck_mtx_assert(&(_inm)->inm_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_inm)->inm_lock, LCK_MTX_ASSERT_NOTOWNED) #define INM_LOCK(_inm) \ lck_mtx_lock(&(_inm)->inm_lock) @@ -462,14 +463,17 @@ struct inpcb; struct in_ifextra { uint32_t netsig_len; u_int8_t netsig[IFNET_SIGNATURELEN]; + struct lltable *ii_llt; /* ARP state */ }; #define IN_IFEXTRA(_ifp) ((struct in_ifextra *)(_ifp->if_inetdata)) +#define LLTABLE(ifp) ((IN_IFEXTRA(ifp) == NULL) ? NULL : IN_IFEXTRA(ifp)->ii_llt) extern u_int32_t ipv4_ll_arp_aware; extern void in_ifaddr_init(void); -extern int imo_multi_filter(const struct ip_moptions *, const struct ifnet *, - const struct sockaddr *, const struct sockaddr *); +extern int imo_multi_filter(const struct ip_moptions *, + const struct ifnet *, const struct sockaddr_in *, + const struct sockaddr_in *); extern int imo_clone(struct inpcb *, struct inpcb *); extern void inm_commit(struct in_multi *); extern void inm_clear_recorded(struct in_multi *); diff --git a/bsd/netinet/ip_compat.h b/bsd/netinet/ip_compat.h index 70d0fadf4..5fb846683 100644 --- a/bsd/netinet/ip_compat.h +++ b/bsd/netinet/ip_compat.h @@ -334,9 +334,9 @@ extern vm_map_t kmem_map; # define SPL_X(x) (void) splx(x) # else # if !SOLARIS && !defined(linux) -# define SPL_IMP(x) x = splimp() -# define SPL_NET(x) x = splnet() -# define SPL_X(x) (void) splx(x) +# define SPL_IMP(x) ; +# define SPL_NET(x) ; +# define SPL_X(x) ; # endif # endif /* NetBSD && NetBSD <= 1991011 && NetBSD >= 199407 */ # define PANIC(x,y) if (x) panic y diff --git a/bsd/netinet/ip_divert.c b/bsd/netinet/ip_divert.c index 083b1484b..4981eb25e 100644 --- a/bsd/netinet/ip_divert.c +++ b/bsd/netinet/ip_divert.c @@ -757,7 +757,7 @@ div_unlock(struct socket *so, int refcount, void *lr) lck_rw_done(divcbinfo.ipi_lock); return (0); } - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; lck_mtx_unlock(mutex_held); @@ -765,17 +765,17 @@ div_unlock(struct socket *so, int refcount, void *lr) } __private_extern__ lck_mtx_t * -div_getlock(struct socket *so, __unused int locktype) +div_getlock(struct socket *so, __unused int flags) { struct inpcb *inpcb = (struct inpcb *)so->so_pcb; - + if (so->so_pcb) { if (so->so_usecount < 0) - panic("div_getlock: so=%p usecount=%x lrh= %s\n", + panic("div_getlock: so=%p usecount=%x lrh= %s\n", so, so->so_usecount, solockhistory_nr(so)); return(&inpcb->inpcb_mtx); } else { - panic("div_getlock: so=%p NULL NO PCB lrh= %s\n", + panic("div_getlock: so=%p NULL NO PCB lrh= %s\n", so, solockhistory_nr(so)); return (so->so_proto->pr_domain->dom_mtx); } diff --git a/bsd/netinet/ip_dummynet.c b/bsd/netinet/ip_dummynet.c index b40ca4612..122698fa4 100644 --- a/bsd/netinet/ip_dummynet.c +++ b/bsd/netinet/ip_dummynet.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -85,6 +85,7 @@ #include #include /* XXX */ #include +#include #include #include #include @@ -95,6 +96,8 @@ #if DUMMYNET #include #endif /* DUMMYNET */ +#include +#include #include #include #include @@ -167,7 +170,6 @@ static void dummynet_send(struct mbuf *m); static struct dn_pipe_head pipehash[HASHSIZE]; /* all pipes */ static struct dn_flow_set_head flowsethash[HASHSIZE]; /* all flowsets */ - #ifdef SYSCTL_NODE SYSCTL_NODE(_net_inet_ip, OID_AUTO, dummynet, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "Dummynet"); @@ -207,13 +209,6 @@ SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED, #define DPRINTF(X) #endif -/* contrary to the comment above random(), it does not actually - * return a value [0, 2^31 - 1], which breaks plr amongst other - * things. Masking it should work even if the behavior of - * the function is fixed. - */ -#define MY_RANDOM (random() & 0x7FFFFFFF) - /* dummynet lock */ static lck_grp_t *dn_mutex_grp; static lck_grp_attr_t *dn_mutex_grp_attr; @@ -229,8 +224,6 @@ static void dummynet_flush(void); void dummynet_drain(void); static ip_dn_io_t dummynet_io; -int if_tx_rdy(struct ifnet *ifp); - static void cp_flow_set_to_64_user(struct dn_flow_set *set, struct dn_flow_set_64 *fs_bp); static void cp_queue_to_64_user( struct dn_flow_queue *q, struct dn_flow_queue_64 *qp); static char *cp_pipe_to_64_user(struct dn_pipe *p, struct dn_pipe_64 *pipe_bp); @@ -243,6 +236,16 @@ static char *cp_pipe_to_32_user(struct dn_pipe *p, struct dn_pipe_32 *pipe_bp); static char* dn_copy_set_32(struct dn_flow_set *set, char *bp); static int cp_pipe_from_user_32( struct sockopt *sopt, struct dn_pipe *p ); +struct eventhandler_lists_ctxt dummynet_evhdlr_ctxt; + +uint32_t my_random(void) +{ + uint32_t val; + read_frandom(&val, sizeof(val)); + val &= 0x7FFFFFFF; + + return (val); +} /* * Heap management functions. @@ -730,7 +733,7 @@ transmit_event(struct dn_pipe *pipe, struct mbuf **head, struct mbuf **tail) struct dn_pkt_tag *pkt = NULL; u_int64_t schedule_time; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); ASSERT(serialize >= 0); if (serialize == 0) { while ((m = pipe->head) != NULL) { @@ -817,7 +820,7 @@ ready_event(struct dn_flow_queue *q, struct mbuf **head, struct mbuf **tail) struct dn_pipe *p = q->fs->pipe ; int p_was_empty ; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); if (p == NULL) { printf("dummynet: ready_event pipe is gone\n"); @@ -884,7 +887,7 @@ ready_event_wfq(struct dn_pipe *p, struct mbuf **head, struct mbuf **tail) struct dn_heap *neh = &(p->not_eligible_heap) ; int64_t p_numbytes = p->numbytes; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); if (p->if_name[0] == 0) /* tx clock is simulated */ p_numbytes += ( curr_time - p->sched_time ) * p->bandwidth; @@ -1145,58 +1148,6 @@ dummynet_send(struct mbuf *m) } } - - -/* - * called by an interface when tx_rdy occurs. - */ -int -if_tx_rdy(struct ifnet *ifp) -{ - struct dn_pipe *p; - struct mbuf *head = NULL, *tail = NULL; - int i; - - lck_mtx_lock(dn_mutex); - - for (i = 0; i < HASHSIZE; i++) - SLIST_FOREACH(p, &pipehash[i], next) - if (p->ifp == ifp) - break ; - if (p == NULL) { - char buf[32]; - snprintf(buf, sizeof(buf), "%s", if_name(ifp)); - for (i = 0; i < HASHSIZE; i++) - SLIST_FOREACH(p, &pipehash[i], next) - if (!strcmp(p->if_name, buf) ) { - p->ifp = ifp ; - DPRINTF(("dummynet: ++ tx rdy from %s (now found)\n", buf)); - break ; - } - } - if (p != NULL) { - DPRINTF(("dummynet: ++ tx rdy from %s - qlen %d\n", if_name(ifp), - IFCQ_LEN(&ifp->if_snd))); - p->numbytes = 0 ; /* mark ready for I/O */ - ready_event_wfq(p, &head, &tail); - } - - if (head != NULL) { - serialize++; - } - - lck_mtx_unlock(dn_mutex); - - /* Send out the de-queued list of ready-to-send packets */ - if (head != NULL) { - dummynet_send(head); - lck_mtx_lock(dn_mutex); - serialize--; - lck_mtx_unlock(dn_mutex); - } - return 0; -} - /* * Unconditionally expire empty queues in case of shortage. * Returns the number of queues freed. @@ -1460,7 +1411,7 @@ red_drops(struct dn_flow_set *fs, struct dn_flow_queue *q, int len) if (fs->flags_fs & DN_QSIZE_IS_BYTES) p_b = (p_b * len) / fs->max_pkt_size; if (++q->count == 0) - q->random = MY_RANDOM & 0xffff; + q->random = (my_random() & 0xffff); else { /* * q->count counts packets arrived since last drop, so a greater @@ -1470,7 +1421,7 @@ red_drops(struct dn_flow_set *fs, struct dn_flow_queue *q, int len) q->count = 0; DPRINTF(("dummynet: - red drop")); /* after a drop we calculate a new random value */ - q->random = MY_RANDOM & 0xffff; + q->random = (my_random() & 0xffff); return 1; /* drop */ } } @@ -1602,7 +1553,7 @@ dummynet_io(struct mbuf *m, int pipe_nr, int dir, struct ip_fw_args *fwa, int cl */ q->tot_bytes += len ; q->tot_pkts++ ; - if ( fs->plr && (MY_RANDOM < fs->plr) ) + if ( fs->plr && (my_random() < fs->plr)) goto dropit ; /* random pkt drop */ if ( fs->flags_fs & DN_QSIZE_IS_BYTES) { if (q->len_bytes > fs->qsize) @@ -1815,7 +1766,7 @@ purge_flow_set(struct dn_flow_set *fs, int all) struct dn_flow_queue *q, *qn ; int i ; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); for (i = 0 ; i <= fs->rq_size ; i++ ) { for (q = fs->rq[i] ; q ; q = qn ) { @@ -2065,7 +2016,6 @@ set_fs_parms(struct dn_flow_set *x, struct dn_flow_set *src) /* * setup pipe or queue parameters. */ - static int config_pipe(struct dn_pipe *p) { @@ -2087,7 +2037,7 @@ config_pipe(struct dn_pipe *p) return EINVAL ; if (p->pipe_nr != 0) { /* this is a pipe */ struct dn_pipe *x, *b; - + struct dummynet_event dn_event; lck_mtx_lock(dn_mutex); /* locate pipe */ @@ -2133,6 +2083,14 @@ config_pipe(struct dn_pipe *p) x, next); } lck_mtx_unlock(dn_mutex); + + bzero(&dn_event, sizeof(dn_event)); + dn_event.dn_event_code = DUMMYNET_PIPE_CONFIG; + dn_event.dn_event_pipe_config.bandwidth = p->bandwidth; + dn_event.dn_event_pipe_config.delay = p->delay; + dn_event.dn_event_pipe_config.plr = pfs->plr; + + dummynet_event_enqueue_nwk_wq_entry(&dn_event); } else { /* config queue */ struct dn_flow_set *x, *b ; @@ -2232,7 +2190,7 @@ dummynet_drain(void) struct mbuf *m, *mnext; int i; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); heap_free(&ready_heap); heap_free(&wfq_ready_heap); @@ -2348,7 +2306,7 @@ char* dn_copy_set_32(struct dn_flow_set *set, char *bp) struct dn_flow_queue *q; struct dn_flow_queue_32 *qp = (struct dn_flow_queue_32 *)bp; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); for (i = 0 ; i <= set->rq_size ; i++) for (q = set->rq[i] ; q ; q = q->next, qp++ ) { @@ -2380,7 +2338,7 @@ char* dn_copy_set_64(struct dn_flow_set *set, char *bp) struct dn_flow_queue *q; struct dn_flow_queue_64 *qp = (struct dn_flow_queue_64 *)bp; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); for (i = 0 ; i <= set->rq_size ; i++) for (q = set->rq[i] ; q ; q = q->next, qp++ ) { @@ -2417,7 +2375,7 @@ dn_calc_size(int is64user) size_t setsize; int i; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); if ( is64user ){ pipesize = sizeof(struct dn_pipe_64); queuesize = sizeof(struct dn_flow_queue_64); @@ -2445,70 +2403,74 @@ dn_calc_size(int is64user) static int dummynet_get(struct sockopt *sopt) { - char *buf, *bp=NULL; /* bp is the "copy-pointer" */ - size_t size ; - struct dn_flow_set *set ; - struct dn_pipe *p ; - int error=0, i ; - int is64user = 0; - - /* XXX lock held too long */ - lck_mtx_lock(dn_mutex); - /* - * XXX: Ugly, but we need to allocate memory with M_WAITOK flag and we - * cannot use this flag while holding a mutex. - */ + char *buf = NULL, *bp = NULL; /* bp is the "copy-pointer" */ + size_t size = 0; + struct dn_flow_set *set; + struct dn_pipe *p; + int error = 0, i; + int is64user = 0; + + /* XXX lock held too long */ + lck_mtx_lock(dn_mutex); + /* + * XXX: Ugly, but we need to allocate memory with M_WAITOK flag + * and we cannot use this flag while holding a mutex. + */ if (proc_is64bit(sopt->sopt_p)) is64user = 1; - for (i = 0; i < 10; i++) { + for (i = 0; i < 10; i++) { size = dn_calc_size(is64user); lck_mtx_unlock(dn_mutex); buf = _MALLOC(size, M_TEMP, M_WAITOK); if (buf == NULL) - return ENOBUFS; + return(ENOBUFS); lck_mtx_lock(dn_mutex); if (size == dn_calc_size(is64user)) break; FREE(buf, M_TEMP); buf = NULL; - } - if (buf == NULL) { + } + if (buf == NULL) { lck_mtx_unlock(dn_mutex); - return ENOBUFS ; - } - + return(ENOBUFS); + } - bp = buf; - for (i = 0; i < HASHSIZE; i++) - SLIST_FOREACH(p, &pipehash[i], next) { - /* - * copy pipe descriptor into *bp, convert delay back to ms, - * then copy the flow_set descriptor(s) one at a time. - * After each flow_set, copy the queue descriptor it owns. - */ - if ( is64user ){ - bp = cp_pipe_to_64_user(p, (struct dn_pipe_64 *)bp); + bp = buf; + for (i = 0; i < HASHSIZE; i++) { + SLIST_FOREACH(p, &pipehash[i], next) { + /* + * copy pipe descriptor into *bp, convert delay + * back to ms, then copy the flow_set descriptor(s) + * one at a time. After each flow_set, copy the + * queue descriptor it owns. + */ + if ( is64user ) { + bp = cp_pipe_to_64_user(p, + (struct dn_pipe_64 *)bp); + } else { + bp = cp_pipe_to_32_user(p, + (struct dn_pipe_32 *)bp); + } } - else{ - bp = cp_pipe_to_32_user(p, (struct dn_pipe_32 *)bp); + } + for (i = 0; i < HASHSIZE; i++) { + SLIST_FOREACH(set, &flowsethash[i], next) { + struct dn_flow_set_64 *fs_bp = + (struct dn_flow_set_64 *)bp ; + cp_flow_set_to_64_user(set, fs_bp); + /* XXX same hack as above */ + fs_bp->next = CAST_DOWN(user64_addr_t, + DN_IS_QUEUE); + fs_bp->pipe = USER_ADDR_NULL; + fs_bp->rq = USER_ADDR_NULL ; + bp += sizeof(struct dn_flow_set_64); + bp = dn_copy_set_64( set, bp ); } - } - for (i = 0; i < HASHSIZE; i++) - SLIST_FOREACH(set, &flowsethash[i], next) { - struct dn_flow_set_64 *fs_bp = (struct dn_flow_set_64 *)bp ; - cp_flow_set_to_64_user(set, fs_bp); - /* XXX same hack as above */ - fs_bp->next = CAST_DOWN(user64_addr_t, DN_IS_QUEUE); - fs_bp->pipe = USER_ADDR_NULL; - fs_bp->rq = USER_ADDR_NULL ; - bp += sizeof(struct dn_flow_set_64); - bp = dn_copy_set_64( set, bp ); - } - lck_mtx_unlock(dn_mutex); - - error = sooptcopyout(sopt, buf, size); - FREE(buf, M_TEMP); - return error ; + } + lck_mtx_unlock(dn_mutex); + error = sooptcopyout(sopt, buf, size); + FREE(buf, M_TEMP); + return(error); } /* @@ -2564,6 +2526,12 @@ ip_dn_ctl(struct sockopt *sopt) return error ; } +void +dummynet_init(void) +{ + eventhandler_lists_ctxt_init(&dummynet_evhdlr_ctxt); +} + void ip_dn_init(void) { @@ -2594,8 +2562,41 @@ ip_dn_init(void) default_rule.cmd[0].len = 1; default_rule.cmd[0].opcode = #ifdef IPFIREWALL_DEFAULT_TO_ACCEPT - (1) ? O_ACCEPT : + (1) ? O_ACCEPT : #endif - O_DENY; + O_DENY; #endif } + +struct dn_event_nwk_wq_entry +{ + struct nwk_wq_entry nwk_wqe; + struct dummynet_event dn_ev_arg; +}; + +static void +dummynet_event_callback(void *arg) +{ + struct dummynet_event *p_dn_ev = (struct dummynet_event *)arg; + + EVENTHANDLER_INVOKE(&dummynet_evhdlr_ctxt, dummynet_event, p_dn_ev); + return; +} + +void +dummynet_event_enqueue_nwk_wq_entry(struct dummynet_event *p_dn_event) +{ + struct dn_event_nwk_wq_entry *p_dn_ev = NULL; + + MALLOC(p_dn_ev, struct dn_event_nwk_wq_entry *, + sizeof(struct dn_event_nwk_wq_entry), + M_NWKWQ, M_WAITOK | M_ZERO); + + p_dn_ev->nwk_wqe.func = dummynet_event_callback; + p_dn_ev->nwk_wqe.is_arg_managed = TRUE; + p_dn_ev->nwk_wqe.arg = &p_dn_ev->dn_ev_arg; + + bcopy(p_dn_event, &(p_dn_ev->dn_ev_arg), + sizeof(struct dummynet_event)); + nwk_wq_enqueue((struct nwk_wq_entry*)p_dn_ev); +} diff --git a/bsd/netinet/ip_dummynet.h b/bsd/netinet/ip_dummynet.h index dde647d0b..7472e958c 100644 --- a/bsd/netinet/ip_dummynet.h +++ b/bsd/netinet/ip_dummynet.h @@ -60,7 +60,6 @@ #include #ifdef PRIVATE - #include /* Apply ipv6 mask on ipv6 addr */ @@ -156,8 +155,11 @@ struct dn_heap { * processing requirements. */ #ifdef KERNEL +#include +#include #include /* for ip_out_args */ #include /* for ip6_out_args */ +#include #include /* for ip6_out_args */ struct dn_pkt_tag { @@ -418,7 +420,7 @@ struct dn_pipe { /* a pipe */ SLIST_HEAD(dn_pipe_head, dn_pipe); #ifdef BSD_KERNEL_PRIVATE - +extern uint32_t my_random(void); void ip_dn_init(void); /* called from raw_ip.c:load_ipfw() */ typedef int ip_dn_ctl_t(struct sockopt *); /* raw_ip.c */ @@ -678,8 +680,6 @@ struct dn_pipe_64 { /* a pipe */ struct dn_flow_set_64 fs ; /* used with fixed-rate flows */ }; - - /* * Return the IPFW rule associated with the dummynet tag; if any. * Make sure that the dummynet tag is not reused by lower layers. @@ -695,6 +695,64 @@ ip_dn_claim_rule(struct mbuf *m) } else return (NULL); } + +#include +/* Dummynet event handling declarations */ +extern struct eventhandler_lists_ctxt dummynet_evhdlr_ctxt; +extern void dummynet_init(void); + +struct dn_pipe_mini_config { + uint32_t bandwidth; + uint32_t delay; + uint32_t plr; +}; + +struct dn_rule_mini_config { + uint32_t dir; + uint32_t af; + uint32_t proto; + /* + * XXX PF rules actually define ranges of ports and + * along with range goes an opcode ((not) equal to, less than + * greater than, etc. + * For now the following works assuming there's no port range + * and the rule is for specific port. + * Also the operation is assumed as equal to. + */ + uint32_t src_port; + uint32_t dst_port; + char ifname[IFXNAMSIZ]; +}; + +struct dummynet_event { + uint32_t dn_event_code; + union { + struct dn_pipe_mini_config _dnev_pipe_config; + struct dn_rule_mini_config _dnev_rule_config; + } dn_event; +}; + +#define dn_event_pipe_config dn_event._dnev_pipe_config +#define dn_event_rule_config dn_event._dnev_rule_config + +extern void dummynet_event_enqueue_nwk_wq_entry(struct dummynet_event *); + +enum { + DUMMYNET_RULE_CONFIG, + DUMMYNET_RULE_DELETE, + DUMMYNET_PIPE_CONFIG, + DUMMYNET_PIPE_DELETE, + DUMMYNET_NLC_DISABLED, +}; + +enum { DN_INOUT, DN_IN, DN_OUT }; +/* + * The signature for the callback is: + * eventhandler_entry_arg __unused + * dummynet_event pointer to dummynet event object + */ +typedef void (*dummynet_event_fn) (struct eventhandler_entry_arg, struct dummynet_event *); +EVENTHANDLER_DECLARE(dummynet_event, dummynet_event_fn); #endif /* BSD_KERNEL_PRIVATE */ #endif /* PRIVATE */ #endif /* _IP_DUMMYNET_H */ diff --git a/bsd/netinet/ip_fw2.c b/bsd/netinet/ip_fw2.c index 63543fd6c..6694975b7 100644 --- a/bsd/netinet/ip_fw2.c +++ b/bsd/netinet/ip_fw2.c @@ -65,7 +65,6 @@ #endif /* INET */ #if IPFW2 -#include #include #include diff --git a/bsd/netinet/ip_icmp.c b/bsd/netinet/ip_icmp.c index 1e2877525..3d7ecafe3 100644 --- a/bsd/netinet/ip_icmp.c +++ b/bsd/netinet/ip_icmp.c @@ -154,7 +154,11 @@ const static int icmp_datalen = 8; /* Default values in case CONFIG_ICMP_BANDLIM is not defined in the MASTER file */ #ifndef CONFIG_ICMP_BANDLIM +#if !CONFIG_EMBEDDED #define CONFIG_ICMP_BANDLIM 250 +#else /* CONFIG_EMBEDDED */ +#define CONFIG_ICMP_BANDLIM 50 +#endif /* CONFIG_EMBEDDED */ #endif /* CONFIG_ICMP_BANDLIM */ /* @@ -379,7 +383,7 @@ icmp_input(struct mbuf *m, int hlen) int icmplen; int i; struct in_ifaddr *ia; - void (*ctlfunc)(int, struct sockaddr *, void *); + void (*ctlfunc)(int, struct sockaddr *, void *, struct ifnet *); int code; /* Expect 32-bit aligned data pointer on strict-align platforms */ @@ -536,7 +540,7 @@ icmp_input(struct mbuf *m, int hlen) ctlfunc = ip_protox[icp->icmp_ip.ip_p]->pr_ctlinput; if (ctlfunc) (*ctlfunc)(code, (struct sockaddr *)&icmpsrc, - (void *)&icp->icmp_ip); + (void *)&icp->icmp_ip, m->m_pkthdr.rcvif); break; badcode: diff --git a/bsd/netinet/ip_id.c b/bsd/netinet/ip_id.c index 4feff3e9c..94c791ea0 100644 --- a/bsd/netinet/ip_id.c +++ b/bsd/netinet/ip_id.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2013 Apple Inc. All rights reserved. + * Copyright (c) 2002-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -195,14 +195,14 @@ ip_randomid(void) if (random_id_statistics && new_id != 0) random_id_collisions++; read_random(&new_id, sizeof (new_id)); - } while (bit_test(id_bits, new_id) || new_id == 0); + } while (bitstr_test(id_bits, new_id) || new_id == 0); /* * These require serialization to maintain correctness. */ lck_mtx_lock_spin(&ipid_lock); - bit_clear(id_bits, id_array[array_ptr]); - bit_set(id_bits, new_id); + bitstr_clear(id_bits, id_array[array_ptr]); + bitstr_set(id_bits, new_id); id_array[array_ptr] = new_id; if (++array_ptr == ARRAY_SIZE) array_ptr = 0; diff --git a/bsd/netinet/ip_input.c b/bsd/netinet/ip_input.c index 3b32f0dde..4e4c27040 100644 --- a/bsd/netinet/ip_input.c +++ b/bsd/netinet/ip_input.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -240,6 +240,11 @@ SYSCTL_UINT(_net_inet_ip, OID_AUTO, adj_clear_hwcksum, CTLFLAG_RW | CTLFLAG_LOCKED, &ip_adj_clear_hwcksum, 0, "Invalidate hwcksum info when adjusting length"); +static uint32_t ip_adj_partial_sum = 1; +SYSCTL_UINT(_net_inet_ip, OID_AUTO, adj_partial_sum, + CTLFLAG_RW | CTLFLAG_LOCKED, &ip_adj_partial_sum, 0, + "Perform partial sum adjustment of trailing bytes at IP layer"); + /* * XXX - Setting ip_checkinterface mostly implements the receive side of * the Strong ES model described in RFC 1122, but since the routing table @@ -844,6 +849,76 @@ ip_input_setdst_chain(struct mbuf *m, uint32_t ifindex, struct in_ifaddr *ia) } } +static void +ip_input_adjust(struct mbuf *m, struct ip *ip, struct ifnet *inifp) +{ + boolean_t adjust = TRUE; + + ASSERT(m_pktlen(m) > ip->ip_len); + + /* + * Invalidate hardware checksum info if ip_adj_clear_hwcksum + * is set; useful to handle buggy drivers. Note that this + * should not be enabled by default, as we may get here due + * to link-layer padding. + */ + if (ip_adj_clear_hwcksum && + (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) && + !(inifp->if_flags & IFF_LOOPBACK) && + !(m->m_pkthdr.pkt_flags & PKTF_LOOP)) { + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + m->m_pkthdr.csum_data = 0; + ipstat.ips_adj_hwcsum_clr++; + } + + /* + * If partial checksum information is available, subtract + * out the partial sum of postpended extraneous bytes, and + * update the checksum metadata accordingly. By doing it + * here, the upper layer transport only needs to adjust any + * prepended extraneous bytes (else it will do both.) + */ + if (ip_adj_partial_sum && + (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + m->m_pkthdr.csum_rx_val = m_adj_sum16(m, + m->m_pkthdr.csum_rx_start, m->m_pkthdr.csum_rx_start, + (ip->ip_len - m->m_pkthdr.csum_rx_start), + m->m_pkthdr.csum_rx_val); + } else if ((m->m_pkthdr.csum_flags & + (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + /* + * If packet has partial checksum info and we decided not + * to subtract the partial sum of postpended extraneous + * bytes here (not the default case), leave that work to + * be handled by the other layers. For now, only TCP, UDP + * layers are capable of dealing with this. For all other + * protocols (including fragments), trim and ditch the + * partial sum as those layers might not implement partial + * checksumming (or adjustment) at all. + */ + if ((ip->ip_off & (IP_MF | IP_OFFMASK)) == 0 && + (ip->ip_p == IPPROTO_TCP || ip->ip_p == IPPROTO_UDP)) { + adjust = FALSE; + } else { + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + m->m_pkthdr.csum_data = 0; + ipstat.ips_adj_hwcsum_clr++; + } + } + + if (adjust) { + ipstat.ips_adj++; + if (m->m_len == m->m_pkthdr.len) { + m->m_len = ip->ip_len; + m->m_pkthdr.len = ip->ip_len; + } else { + m_adj(m, ip->ip_len - m->m_pkthdr.len); + } + } +} + /* * First pass does all essential packet validation and places on a per flow * queue for doing operations that have same outcome for all packets of a flow. @@ -1123,27 +1198,7 @@ ipfw_tags_done: } if (m->m_pkthdr.len > ip->ip_len) { - /* - * Invalidate hardware checksum info if ip_adj_clear_hwcksum - * is set; useful to handle buggy drivers. Note that this - * should not be enabled by default, as we may get here due - * to link-layer padding. - */ - if (ip_adj_clear_hwcksum && - (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) && - !(inifp->if_flags & IFF_LOOPBACK) && - !(m->m_pkthdr.pkt_flags & PKTF_LOOP)) { - m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; - m->m_pkthdr.csum_data = 0; - ipstat.ips_adj_hwcsum_clr++; - } - - ipstat.ips_adj++; - if (m->m_len == m->m_pkthdr.len) { - m->m_len = ip->ip_len; - m->m_pkthdr.len = ip->ip_len; - } else - m_adj(m, ip->ip_len - m->m_pkthdr.len); + ip_input_adjust(m, ip, inifp); } /* for consistency */ @@ -1161,6 +1216,8 @@ check_with_pf: if (PF_IS_ENABLED) { int error; ip_input_cpout_args(args, &args1, &init); + ip = mtod(m, struct ip *); + src_ip = ip->ip_src; #if DUMMYNET error = pf_af_hook(inifp, NULL, &m, AF_INET, TRUE, &args1); @@ -2014,27 +2071,7 @@ tooshort: goto bad; } if (m->m_pkthdr.len > ip->ip_len) { - /* - * Invalidate hardware checksum info if ip_adj_clear_hwcksum - * is set; useful to handle buggy drivers. Note that this - * should not be enabled by default, as we may get here due - * to link-layer padding. - */ - if (ip_adj_clear_hwcksum && - (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) && - !(inifp->if_flags & IFF_LOOPBACK) && - !(m->m_pkthdr.pkt_flags & PKTF_LOOP)) { - m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; - m->m_pkthdr.csum_data = 0; - ipstat.ips_adj_hwcsum_clr++; - } - - ipstat.ips_adj++; - if (m->m_len == m->m_pkthdr.len) { - m->m_len = ip->ip_len; - m->m_pkthdr.len = ip->ip_len; - } else - m_adj(m, ip->ip_len - m->m_pkthdr.len); + ip_input_adjust(m, ip, inifp); } /* for consistency */ @@ -2446,7 +2483,7 @@ bad: static void ipq_updateparams(void) { - lck_mtx_assert(&ipqlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ipqlock, LCK_MTX_ASSERT_OWNED); /* * -1 for unlimited allocation. */ @@ -2519,7 +2556,7 @@ done: * When IPDIVERT enabled, keep additional state with each packet that * tells us if we need to divert or tee the packet we're building. * - * The IP header is *NOT* adjusted out of iplen. + * The IP header is *NOT* adjusted out of iplen (but in host byte order). */ static struct mbuf * #if IPDIVERT @@ -2617,35 +2654,49 @@ found: * * Perform 1's complement adjustment of octets that got included/ * excluded in the hardware-calculated checksum value. Ignore cases - * where the value includes or excludes the IP header span, as the - * sum for those octets would already be 0xffff and thus no-op. + * where the value includes the entire IPv4 header span, as the sum + * for those octets would already be 0 by the time we get here; IP + * has already performed its header checksum validation. Also take + * care of any trailing bytes and subtract out their partial sum. */ if (ip->ip_p == IPPROTO_UDP && hlen == sizeof (struct ip) && (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) == (CSUM_DATA_VALID | CSUM_PARTIAL)) { - uint32_t start; + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - ip->ip_len); + uint32_t swbytes = (uint32_t)trailer; - start = m->m_pkthdr.csum_rx_start; csum = m->m_pkthdr.csum_rx_val; - if (start != 0 && start != hlen) { + ASSERT(trailer >= 0); + if ((start != 0 && start != hlen) || trailer != 0) { #if BYTE_ORDER != BIG_ENDIAN if (start < hlen) { HTONS(ip->ip_len); HTONS(ip->ip_off); } -#endif +#endif /* BYTE_ORDER != BIG_ENDIAN */ /* callee folds in sum */ - csum = m_adj_sum16(m, start, hlen, csum); + csum = m_adj_sum16(m, start, hlen, + (ip->ip_len - hlen), csum); + if (hlen > start) + swbytes += (hlen - start); + else + swbytes += (start - hlen); #if BYTE_ORDER != BIG_ENDIAN if (start < hlen) { NTOHS(ip->ip_off); NTOHS(ip->ip_len); } -#endif +#endif /* BYTE_ORDER != BIG_ENDIAN */ } csum_flags = m->m_pkthdr.csum_flags; + + if (swbytes != 0) + udp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } else { csum = 0; csum_flags = 0; @@ -3019,7 +3070,7 @@ dropfrag: static void frag_freef(struct ipqhead *fhp, struct ipq *fp) { - lck_mtx_assert(&ipqlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ipqlock, LCK_MTX_ASSERT_OWNED); fp->ipq_nfrags = 0; if (fp->ipq_frags != NULL) { @@ -3085,7 +3136,7 @@ frag_timeout(void *arg) static void frag_sched_timeout(void) { - lck_mtx_assert(&ipqlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ipqlock, LCK_MTX_ASSERT_OWNED); if (!frag_timeout_run && nipq > 0) { frag_timeout_run = 1; @@ -3632,16 +3683,11 @@ ip_srcroute(void) } /* - * Strip out IP options, at higher - * level protocol in the kernel. - * Second argument is buffer to which options - * will be moved, and return value is their length. - * XXX should be deleted; last arg currently ignored. + * Strip out IP options, at higher level protocol in the kernel. */ void -ip_stripoptions(struct mbuf *m, struct mbuf *mopt) +ip_stripoptions(struct mbuf *m) { -#pragma unused(mopt) int i; struct ip *ip = mtod(m, struct ip *); caddr_t opts; @@ -3650,6 +3696,7 @@ ip_stripoptions(struct mbuf *m, struct mbuf *mopt) /* Expect 32-bit aligned data pointer on strict-align platforms */ MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m); + /* use bcopy() since it supports overlapping range */ olen = (IP_VHL_HL(ip->ip_vhl) << 2) - sizeof (struct ip); opts = (caddr_t)(ip + 1); i = m->m_len - (sizeof (struct ip) + olen); @@ -3658,6 +3705,27 @@ ip_stripoptions(struct mbuf *m, struct mbuf *mopt) if (m->m_flags & M_PKTHDR) m->m_pkthdr.len -= olen; ip->ip_vhl = IP_MAKE_VHL(IPVERSION, sizeof (struct ip) >> 2); + + /* + * We expect ip_{off,len} to be in host order by now, and + * that the original IP header length has been subtracted + * out from ip_len. Temporarily adjust ip_len for checksum + * recalculation, and restore it afterwards. + */ + ip->ip_len += sizeof (struct ip); + + /* recompute checksum now that IP header is smaller */ +#if BYTE_ORDER != BIG_ENDIAN + HTONS(ip->ip_len); + HTONS(ip->ip_off); +#endif /* BYTE_ORDER != BIG_ENDIAN */ + ip->ip_sum = in_cksum_hdr(ip); +#if BYTE_ORDER != BIG_ENDIAN + NTOHS(ip->ip_off); + NTOHS(ip->ip_len); +#endif /* BYTE_ORDER != BIG_ENDIAN */ + + ip->ip_len -= sizeof (struct ip); } u_char inetctlerrmap[PRC_NCMDS] = { diff --git a/bsd/netinet/ip_output.c b/bsd/netinet/ip_output.c index 9069bb8ba..16aff3c65 100644 --- a/bsd/netinet/ip_output.c +++ b/bsd/netinet/ip_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -207,6 +207,10 @@ SYSCTL_PROC(_net_inet_ip, OID_AUTO, output_perf_data, 0, 0, sysctl_ip_output_getperf, "S,net_perf", "IP output performance data (struct net_perf, net/net_perf.h)"); +__private_extern__ int rfc6864 = 1; +SYSCTL_INT(_net_inet_ip, OID_AUTO, rfc6864, CTLFLAG_RW | CTLFLAG_LOCKED, + &rfc6864, 0, "updated ip id field behavior"); + #define IMO_TRACE_HIST_SIZE 32 /* size of trace history */ /* For gdb */ @@ -586,7 +590,12 @@ loopit: if (!(flags & (IP_FORWARDING|IP_RAWOUTPUT))) { ip->ip_vhl = IP_MAKE_VHL(IPVERSION, hlen >> 2); ip->ip_off &= IP_DF; - ip->ip_id = ip_randomid(); + if (rfc6864 && IP_OFF_IS_ATOMIC(ip->ip_off)) { + // Per RFC6864, value of ip_id is undefined for atomic ip packets + ip->ip_id = 0; + } else { + ip->ip_id = ip_randomid(); + } OSAddAtomic(1, &ipstat.ips_localout); } else { hlen = IP_VHL_HL(ip->ip_vhl) << 2; @@ -901,7 +910,7 @@ loopit: if (IN_MULTICAST(ntohl(pkt_dst.s_addr))) { struct ifnet *srcifp = NULL; struct in_multi *inm; - u_int32_t vif; + u_int32_t vif = 0; u_int8_t ttl = IP_DEFAULT_MULTICAST_TTL; u_int8_t loop = IP_DEFAULT_MULTICAST_LOOP; @@ -1209,6 +1218,7 @@ sendit: /* Check if the interface is allowed */ if (!necp_packet_is_allowed_over_interface(m, ifp)) { error = EHOSTUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; } goto skip_ipsec; @@ -1216,6 +1226,7 @@ sendit: case NECP_KERNEL_POLICY_RESULT_SOCKET_DIVERT: /* Flow divert packets should be blocked at the IP layer */ error = EHOSTUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; case NECP_KERNEL_POLICY_RESULT_IP_TUNNEL: { /* Verify that the packet is being routed to the tunnel */ @@ -1224,6 +1235,7 @@ sendit: /* Check if the interface is allowed */ if (!necp_packet_is_allowed_over_interface(m, ifp)) { error = EHOSTUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; } goto skip_ipsec; @@ -1232,6 +1244,7 @@ sendit: /* Check if the interface is allowed */ if (!necp_packet_is_allowed_over_interface(m, policy_ifp)) { error = EHOSTUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; } @@ -1241,6 +1254,7 @@ sendit: goto skip_ipsec; } else { error = ENETUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; } } @@ -1252,6 +1266,7 @@ sendit: /* Catch-all to check if the interface is allowed */ if (!necp_packet_is_allowed_over_interface(m, ifp)) { error = EHOSTUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; } #endif /* NECP */ @@ -2256,7 +2271,8 @@ in_finalize_cksum(struct mbuf *m, uint32_t hoff, uint32_t csum_flags) ip_out_cksum_stats(ip->ip_p, len); /* RFC1122 4.1.3.4 */ - if (csum == 0 && (m->m_pkthdr.csum_flags & CSUM_UDP)) + if (csum == 0 && + (m->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_ZERO_INVERT))) csum = 0xffff; /* Insert the checksum in the ULP csum field */ @@ -2268,8 +2284,8 @@ in_finalize_cksum(struct mbuf *m, uint32_t hoff, uint32_t csum_flags) } else { bcopy(&csum, (mtod(m, char *) + offset), sizeof (csum)); } - m->m_pkthdr.csum_flags &= - ~(CSUM_DELAY_DATA | CSUM_DATA_VALID | CSUM_PARTIAL); + m->m_pkthdr.csum_flags &= ~(CSUM_DELAY_DATA | CSUM_DATA_VALID | + CSUM_PARTIAL | CSUM_ZERO_INVERT); } if (sw_csum & CSUM_DELAY_IP) { @@ -2449,8 +2465,10 @@ ip_ctloutput(struct socket *so, struct sockopt *sopt) m->m_len = sopt->sopt_valsize; error = sooptcopyin(sopt, mtod(m, char *), m->m_len, m->m_len); - if (error) + if (error) { + m_freem(m); break; + } return (ip_pcbopts(sopt->sopt_name, &inp->inp_options, m)); @@ -3089,7 +3107,7 @@ ip_mloopback(struct ifnet *srcifp, struct ifnet *origifp, struct mbuf *m, * interface itself is lo0, this will be overridden by if_loop. */ if (hwcksum_rx) { - copym->m_pkthdr.csum_flags &= ~CSUM_PARTIAL; + copym->m_pkthdr.csum_flags &= ~(CSUM_PARTIAL|CSUM_ZERO_INVERT); copym->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; copym->m_pkthdr.csum_data = 0xffff; @@ -3443,10 +3461,13 @@ ip_output_checksum(struct ifnet *ifp, struct mbuf *m, int hlen, int ip_len, /* * Partial checksum offload, if non-IP fragment, and TCP only * (no UDP support, as the hardware may not be able to convert - * +0 to -0 (0xffff) per RFC1122 4.1.3.4.) + * +0 to -0 (0xffff) per RFC1122 4.1.3.4. unless the interface + * supports "invert zero" capability.) */ if (hwcksum_tx && !tso && - (m->m_pkthdr.csum_flags & CSUM_TCP) && + ((m->m_pkthdr.csum_flags & CSUM_TCP) || + ((hwcap & CSUM_ZERO_INVERT) && + (m->m_pkthdr.csum_flags & CSUM_ZERO_INVERT))) && ip_len <= ifp->if_mtu) { uint16_t start = sizeof (struct ip); uint16_t ulpoff = m->m_pkthdr.csum_data & 0xffff; diff --git a/bsd/netinet/ip_var.h b/bsd/netinet/ip_var.h index cc54d85d8..87985cc1c 100644 --- a/bsd/netinet/ip_var.h +++ b/bsd/netinet/ip_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -150,10 +150,10 @@ struct ip_moptions { }; #define IMO_LOCK_ASSERT_HELD(_imo) \ - lck_mtx_assert(&(_imo)->imo_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_imo)->imo_lock, LCK_MTX_ASSERT_OWNED) #define IMO_LOCK_ASSERT_NOTHELD(_imo) \ - lck_mtx_assert(&(_imo)->imo_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_imo)->imo_lock, LCK_MTX_ASSERT_NOTOWNED) #define IMO_LOCK(_imo) \ lck_mtx_lock(&(_imo)->imo_lock) @@ -228,7 +228,7 @@ struct ipstat { u_int32_t ips_rxc_chainsz_gt4; /* rx chain size greater than 4 */ u_int32_t ips_rxc_notlist; /* count of pkts through ip_input */ u_int32_t ips_raw_sappend_fail; /* sock append failed */ - + u_int32_t ips_necp_policy_drop; /* NECP policy related drop */ }; struct ip_linklocal_stat { @@ -254,6 +254,7 @@ struct ip_moptions; #define IP_OUTARGS 0x100 /* has ancillary output info */ #define IP_HDR_ALIGNED_P(_ip) ((((uintptr_t)(_ip)) & ((uintptr_t)3)) == 0) +#define IP_OFF_IS_ATOMIC(_ip_off) ((_ip_off & (IP_DF | IP_MF | IP_OFFMASK)) == IP_DF) /* * On platforms which require strict alignment (currently for anything but @@ -307,6 +308,7 @@ extern int ip_use_randomid; extern u_short ip_id; /* ip packet ctr, for ids */ extern int ip_defttl; /* default IP ttl */ extern int ipforwarding; /* ip forwarding */ +extern int rfc6864; extern struct protosw *ip_protox[]; extern struct pr_usrreqs rip_usrreqs; @@ -334,7 +336,7 @@ extern struct in_ifaddr *ip_rtaddr(struct in_addr); extern int ip_savecontrol(struct inpcb *, struct mbuf **, struct ip *, struct mbuf *); extern struct mbuf *ip_srcroute(void); -extern void ip_stripoptions(struct mbuf *, struct mbuf *); +extern void ip_stripoptions(struct mbuf *); extern void ip_initid(void); extern u_int16_t ip_randomid(void); extern void ip_proto_dispatch_in_wrapper(struct mbuf *, int, u_int8_t); @@ -346,7 +348,7 @@ extern int ip_getsrcifaddr_info(struct mbuf *, uint32_t *, uint32_t *); extern int ip_getdstifaddr_info(struct mbuf *, uint32_t *, uint32_t *); extern int rip_ctloutput(struct socket *, struct sockopt *); -extern void rip_ctlinput(int, struct sockaddr *, void *); +extern void rip_ctlinput(int, struct sockaddr *, void *, struct ifnet *); extern void rip_init(struct protosw *, struct domain *); extern void rip_input(struct mbuf *, int); extern int rip_output(struct mbuf *, struct socket *, u_int32_t, struct mbuf *); diff --git a/bsd/netinet/kpi_ipfilter.c b/bsd/netinet/kpi_ipfilter.c index a63d4a583..2186c63ba 100644 --- a/bsd/netinet/kpi_ipfilter.c +++ b/bsd/netinet/kpi_ipfilter.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2004-2016 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -36,10 +36,11 @@ #include -#define _IP_VHL +#define _IP_VHL #include #include #include +#include #include #include @@ -51,12 +52,13 @@ #include #include +#include /* * kipf_lock and kipf_ref protect the linkage of the list of IP filters * An IP filter can be removed only when kipf_ref is zero - * If an IP filter cannot be removed because kipf_ref is not null, then - * the IP filter is marjed and kipf_delayed_remove is set so that when + * If an IP filter cannot be removed because kipf_ref is not null, then + * the IP filter is marjed and kipf_delayed_remove is set so that when * kipf_ref eventually goes down to zero, the IP filter is removed */ decl_lck_mtx_data(static, kipf_lock_data); @@ -69,11 +71,21 @@ __private_extern__ struct ipfilter_list ipv4_filters = TAILQ_HEAD_INITIALIZER(ip __private_extern__ struct ipfilter_list ipv6_filters = TAILQ_HEAD_INITIALIZER(ipv6_filters); __private_extern__ struct ipfilter_list tbr_filters = TAILQ_HEAD_INITIALIZER(tbr_filters); +#undef ipf_addv4 +#undef ipf_addv6 +extern errno_t ipf_addv4(const struct ipf_filter *filter, + ipfilter_t *filter_ref); +extern errno_t ipf_addv6(const struct ipf_filter *filter, + ipfilter_t *filter_ref); + +static errno_t ipf_add(const struct ipf_filter *filter, + ipfilter_t *filter_ref, struct ipfilter_list *head, bool is_internal); + __private_extern__ void ipf_ref(void) { lck_mtx_lock(kipf_lock); - kipf_ref++; + kipf_ref++; lck_mtx_unlock(kipf_lock); } @@ -82,17 +94,19 @@ ipf_unref(void) { lck_mtx_lock(kipf_lock); - if (kipf_ref == 0) - panic("ipf_unref: kipf_ref == 0\n"); - - kipf_ref--; - if (kipf_ref == 0 && kipf_delayed_remove != 0) { - struct ipfilter *filter; + if (kipf_ref == 0) + panic("ipf_unref: kipf_ref == 0\n"); + + kipf_ref--; + if (kipf_ref == 0 && kipf_delayed_remove != 0) { + struct ipfilter *filter; while ((filter = TAILQ_FIRST(&tbr_filters))) { + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ipf_add_count) > 0); + ipf_detach_func ipf_detach = filter->ipf_filter.ipf_detach; void* cookie = filter->ipf_filter.cookie; - + TAILQ_REMOVE(filter->ipf_head, filter, ipf_link); TAILQ_REMOVE(&tbr_filters, filter, ipf_tbr); kipf_delayed_remove--; @@ -104,57 +118,80 @@ ipf_unref(void) /* In case some filter got to run while we released the lock */ if (kipf_ref != 0) break; - } + } } - } + } lck_mtx_unlock(kipf_lock); } static errno_t ipf_add( - const struct ipf_filter* filter, + const struct ipf_filter *filter, ipfilter_t *filter_ref, - struct ipfilter_list *head) + struct ipfilter_list *head, + bool is_internal) { struct ipfilter *new_filter; if (filter->name == NULL || (filter->ipf_input == NULL && filter->ipf_output == NULL)) - return EINVAL; - - MALLOC(new_filter, struct ipfilter*, sizeof(*new_filter), M_IFADDR, M_WAITOK); + return (EINVAL); + + MALLOC(new_filter, struct ipfilter *, sizeof(*new_filter), M_IFADDR, M_WAITOK); if (new_filter == NULL) - return ENOMEM; - + return (ENOMEM); + lck_mtx_lock(kipf_lock); new_filter->ipf_filter = *filter; new_filter->ipf_head = head; - + TAILQ_INSERT_HEAD(head, new_filter, ipf_link); - + + OSIncrementAtomic64(&net_api_stats.nas_ipf_add_count); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_ipf_add_total); + if (is_internal) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_ipf_add_os_total); + } + lck_mtx_unlock(kipf_lock); - + *filter_ref = (ipfilter_t)new_filter; /* This will force TCP to re-evaluate its use of TSO */ OSAddAtomic(1, &kipf_count); routegenid_update(); - return 0; + return (0); +} + +errno_t +ipf_addv4_internal( + const struct ipf_filter *filter, + ipfilter_t *filter_ref) +{ + return (ipf_add(filter, filter_ref, &ipv4_filters, true)); } errno_t ipf_addv4( - const struct ipf_filter* filter, + const struct ipf_filter *filter, + ipfilter_t *filter_ref) +{ + return (ipf_add(filter, filter_ref, &ipv4_filters, false)); +} + +errno_t +ipf_addv6_internal( + const struct ipf_filter *filter, ipfilter_t *filter_ref) { - return ipf_add(filter, filter_ref, &ipv4_filters); + return (ipf_add(filter, filter_ref, &ipv6_filters, true)); } errno_t ipf_addv6( - const struct ipf_filter* filter, + const struct ipf_filter *filter, ipfilter_t *filter_ref) { - return ipf_add(filter, filter_ref, &ipv6_filters); + return (ipf_add(filter, filter_ref, &ipv6_filters, false)); } static errno_t @@ -185,20 +222,20 @@ errno_t ipf_remove( ipfilter_t filter_ref) { - struct ipfilter *match = (struct ipfilter*)filter_ref; + struct ipfilter *match = (struct ipfilter *)filter_ref; struct ipfilter_list *head; - + if (match == 0 || (match->ipf_head != &ipv4_filters && match->ipf_head != &ipv6_filters)) - return EINVAL; - + return (EINVAL); + head = match->ipf_head; - + lck_mtx_lock(kipf_lock); TAILQ_FOREACH(match, head, ipf_link) { - if (match == (struct ipfilter*)filter_ref) { + if (match == (struct ipfilter *)filter_ref) { ipf_detach_func ipf_detach = match->ipf_filter.ipf_detach; void* cookie = match->ipf_filter.cookie; - + /* * Cannot detach when they are filters running */ @@ -209,8 +246,11 @@ ipf_remove( match->ipf_filter.ipf_output = ipf_output_detached; lck_mtx_unlock(kipf_lock); } else { + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ipf_add_count) > 0); + TAILQ_REMOVE(head, match, ipf_link); lck_mtx_unlock(kipf_lock); + if (ipf_detach) ipf_detach(cookie); FREE(match, M_IFADDR); @@ -220,12 +260,12 @@ ipf_remove( routegenid_update(); } - return 0; + return (0); } } lck_mtx_unlock(kipf_lock); - - return ENOENT; + + return (ENOENT); } int log_for_en1 = 0; @@ -235,7 +275,7 @@ ipf_inject_input( mbuf_t data, ipfilter_t filter_ref) { - struct mbuf *m = (struct mbuf*)data; + struct mbuf *m = (struct mbuf *)data; struct m_tag *mtag = 0; struct ip *ip = mtod(m, struct ip *); u_int8_t vers; @@ -244,7 +284,7 @@ ipf_inject_input( protocol_family_t proto; vers = IP_VHL_V(ip->ip_vhl); - + switch (vers) { case 4: proto = PF_INET; @@ -256,7 +296,7 @@ ipf_inject_input( error = ENOTSUP; goto done; } - + if (filter_ref == 0 && m->m_pkthdr.rcvif == 0) { m->m_pkthdr.rcvif = lo_ifp; m->m_pkthdr.csum_data = 0; @@ -269,27 +309,27 @@ ipf_inject_input( } if (filter_ref != 0) { mtag = m_tag_create(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_IPFILT, - sizeof (ipfilter_t), M_NOWAIT, m); + sizeof (ipfilter_t), M_NOWAIT, m); if (mtag == NULL) { error = ENOMEM; goto done; - } - *(ipfilter_t*)(mtag+1) = filter_ref; + } + *(ipfilter_t *)(mtag+1) = filter_ref; m_tag_prepend(m, mtag); } - + error = proto_inject(proto, data); done: - return error; + return (error); } static errno_t ipf_injectv4_out(mbuf_t data, ipfilter_t filter_ref, ipf_pktopts_t options) { struct route ro; - struct ip *ip; - struct mbuf *m = (struct mbuf*)data; + struct ip *ip; + struct mbuf *m = (struct mbuf *)data; errno_t error = 0; struct m_tag *mtag = NULL; struct ip_moptions *imo = NULL; @@ -365,8 +405,8 @@ static errno_t ipf_injectv6_out(mbuf_t data, ipfilter_t filter_ref, ipf_pktopts_t options) { struct route_in6 ro; - struct ip6_hdr *ip6; - struct mbuf *m = (struct mbuf*)data; + struct ip6_hdr *ip6; + struct mbuf *m = (struct mbuf *)data; errno_t error = 0; struct m_tag *mtag = NULL; struct ip6_moptions *im6o = NULL; @@ -379,7 +419,7 @@ ipf_injectv6_out(mbuf_t data, ipfilter_t filter_ref, ipf_pktopts_t options) if (m == NULL) return (ENOMEM); } - ip6 = (struct ip6_hdr*)m_mtod(m); + ip6 = (struct ip6_hdr *)m_mtod(m); if (filter_ref != 0) { mtag = m_tag_create(KERNEL_MODULE_TAG_ID, @@ -419,7 +459,7 @@ ipf_injectv6_out(mbuf_t data, ipfilter_t filter_ref, ipf_pktopts_t options) /* * Send mbuf and ifscope information. Check for correctness - * of ifscope information is done while searching for a route in + * of ifscope information is done while searching for a route in * ip6_output. */ error = ip6_output(m, NULL, &ro, IPV6_OUTARGS, im6o, NULL, &ip6oa); @@ -440,20 +480,19 @@ ipf_inject_output( ipfilter_t filter_ref, ipf_pktopts_t options) { - struct mbuf *m = (struct mbuf*)data; + struct mbuf *m = (struct mbuf *)data; u_int8_t vers; errno_t error = 0; /* Make one byte of the header contiguous in the mbuf */ if (m->m_len < 1) { m = m_pullup(m, 1); - if (m == NULL) + if (m == NULL) goto done; } - - vers = (*(u_int8_t*)m_mtod(m)) >> 4; - switch (vers) - { + + vers = (*(u_int8_t *)m_mtod(m)) >> 4; + switch (vers) { case 4: error = ipf_injectv4_out(data, filter_ref, options); break; @@ -468,8 +507,8 @@ ipf_inject_output( break; } -done: - return error; +done: + return (error); } __private_extern__ ipfilter_t @@ -477,14 +516,14 @@ ipf_get_inject_filter(struct mbuf *m) { ipfilter_t filter_ref = 0; struct m_tag *mtag; - + mtag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_IPFILT, NULL); if (mtag) { filter_ref = *(ipfilter_t *)(mtag+1); - + m_tag_delete(m, mtag); } - return filter_ref; + return (filter_ref); } __private_extern__ int @@ -494,28 +533,28 @@ ipf_init(void) lck_grp_attr_t *grp_attributes = 0; lck_attr_t *lck_attributes = 0; lck_grp_t *lck_grp = 0; - + grp_attributes = lck_grp_attr_alloc_init(); if (grp_attributes == 0) { printf("ipf_init: lck_grp_attr_alloc_init failed\n"); error = ENOMEM; goto done; } - + lck_grp = lck_grp_alloc_init("IP Filter", grp_attributes); if (lck_grp == 0) { printf("ipf_init: lck_grp_alloc_init failed\n"); error = ENOMEM; goto done; } - + lck_attributes = lck_attr_alloc_init(); if (lck_attributes == 0) { printf("ipf_init: lck_attr_alloc_init failed\n"); error = ENOMEM; goto done; } - + lck_mtx_init(kipf_lock, lck_grp, lck_attributes); done: @@ -531,6 +570,6 @@ ipf_init(void) lck_attr_free(lck_attributes); lck_attributes = 0; } - - return error; + + return (error); } diff --git a/bsd/netinet/kpi_ipfilter.h b/bsd/netinet/kpi_ipfilter.h index 210f99f97..392d0650b 100644 --- a/bsd/netinet/kpi_ipfilter.h +++ b/bsd/netinet/kpi_ipfilter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2014 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -156,8 +156,16 @@ typedef struct opaque_ipfilter *ipfilter_t; @param filter_ref A reference to the filter used to detach it. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t ipf_addv4_internal(const struct ipf_filter *filter, + ipfilter_t *filter_ref); + +#define ipf_addv4(filter, filter_ref) \ + ipf_addv4_internal((filter), (filter_ref)) +#else extern errno_t ipf_addv4(const struct ipf_filter *filter, ipfilter_t *filter_ref); +#endif /* KERNEL_PRIVATE */ /*! @function ipf_addv6 @@ -166,8 +174,16 @@ extern errno_t ipf_addv4(const struct ipf_filter *filter, @param filter_ref A reference to the filter used to detach it. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t ipf_addv6_internal(const struct ipf_filter *filter, + ipfilter_t *filter_ref); + +#define ipf_addv6(filter, filter_ref) \ + ipf_addv6_internal((filter), (filter_ref)) +#else extern errno_t ipf_addv6(const struct ipf_filter *filter, ipfilter_t *filter_ref); +#endif /* KERNEL_PRIVATE */ /*! @function ipf_remove diff --git a/bsd/netinet/mp_pcb.c b/bsd/netinet/mp_pcb.c index 31ea83fd0..288f29c71 100644 --- a/bsd/netinet/mp_pcb.c +++ b/bsd/netinet/mp_pcb.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -43,6 +43,7 @@ #include #include +#include static lck_grp_t *mp_lock_grp; static lck_attr_t *mp_lock_attr; @@ -131,7 +132,7 @@ mp_timeout(void *arg) static void mp_sched_timeout(void) { - lck_mtx_assert(&mp_timeout_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mp_timeout_lock, LCK_MTX_ASSERT_OWNED); if (!mp_timeout_run && (mp_garbage_collecting || mp_ticking)) { lck_mtx_convert_spin(&mp_timeout_lock); @@ -199,27 +200,9 @@ int mp_pcballoc(struct socket *so, struct mppcbinfo *mppi) { struct mppcb *mpp = NULL; + int error; - VERIFY(sotomppcb(so) == NULL); - - lck_mtx_lock(&mppi->mppi_lock); - if (mppi->mppi_count >= mptcp_socket_limit) { - lck_mtx_unlock(&mppi->mppi_lock); - mptcplog((LOG_ERR, "MPTCP Socket: Reached MPTCP socket limit."), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); - /* - * This limit may be reached either because of - * a leak or a transient condition where - * MPTCP connections are not released fast - * enough. - * We return EAFNOSUPPORT here to have user - * space library fallback to TCP. - * XXX We need to revist this when we get rid - * of the current low limit imposed on MPTCP. - */ - return (EAFNOSUPPORT); - } - lck_mtx_unlock(&mppi->mppi_lock); + VERIFY(mpsotomppcb(so) == NULL); mpp = zalloc(mppi->mppi_zone); if (mpp == NULL) { @@ -233,10 +216,11 @@ mp_pcballoc(struct socket *so, struct mppcbinfo *mppi) mpp->mpp_socket = so; so->so_pcb = mpp; - if (NULL == mppi->mppi_pcbe_create(so, mpp)) { + error = mptcp_sescreate(mpp); + if (error) { lck_mtx_destroy(&mpp->mpp_lock, mppi->mppi_lock_grp); zfree(mppi->mppi_zone, mpp); - return (ENOBUFS); + return (error); } lck_mtx_lock(&mppi->mppi_lock); @@ -249,15 +233,13 @@ mp_pcballoc(struct socket *so, struct mppcbinfo *mppi) } void -mp_pcbdetach(struct mppcb *mpp) +mp_pcbdetach(struct socket *mp_so) { - struct socket *so = mpp->mpp_socket; - - VERIFY(so->so_pcb == mpp); + struct mppcb *mpp = mpsotomppcb(mp_so); mpp->mpp_state = MPPCB_STATE_DEAD; - if (!(so->so_flags & SOF_PCBCLEARING)) - so->so_flags |= SOF_PCBCLEARING; + if (!(mp_so->so_flags & SOF_PCBCLEARING)) + mp_so->so_flags |= SOF_PCBCLEARING; mp_gc_sched(); } @@ -269,23 +251,105 @@ mp_pcbdispose(struct mppcb *mpp) VERIFY(mppi != NULL); - lck_mtx_assert(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&mpp->mpp_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); + mpp_lock_assert_held(mpp); VERIFY(mpp->mpp_state == MPPCB_STATE_DEAD); - VERIFY(mpp->mpp_flags & MPP_ATTACHED); + mpp->mpp_flags &= ~MPP_ATTACHED; TAILQ_REMOVE(&mppi->mppi_pcbs, mpp, mpp_entry); VERIFY(mppi->mppi_count != 0); mppi->mppi_count--; + mpp_unlock(mpp); + +#if NECP + necp_mppcb_dispose(mpp); +#endif /* NECP */ + + lck_mtx_destroy(&mpp->mpp_lock, mppi->mppi_lock_grp); + VERIFY(mpp->mpp_socket != NULL); VERIFY(mpp->mpp_socket->so_usecount == 0); mpp->mpp_socket->so_pcb = NULL; mpp->mpp_socket = NULL; - lck_mtx_unlock(&mpp->mpp_lock); - lck_mtx_destroy(&mpp->mpp_lock, mppi->mppi_lock_grp); zfree(mppi->mppi_zone, mpp); } + +static int +mp_getaddr_v4(struct socket *mp_so, struct sockaddr **nam, boolean_t peer) +{ + struct mptses *mpte = mpsotompte(mp_so); + struct sockaddr_in *sin; + + /* + * Do the malloc first in case it blocks. + */ + MALLOC(sin, struct sockaddr_in *, sizeof (*sin), M_SONAME, M_WAITOK); + if (sin == NULL) + return (ENOBUFS); + bzero(sin, sizeof (*sin)); + sin->sin_family = AF_INET; + sin->sin_len = sizeof (*sin); + + if (!peer) { + sin->sin_port = mpte->__mpte_src_v4.sin_port; + sin->sin_addr = mpte->__mpte_src_v4.sin_addr; + } else { + sin->sin_port = mpte->__mpte_dst_v4.sin_port; + sin->sin_addr = mpte->__mpte_dst_v4.sin_addr; + } + + *nam = (struct sockaddr *)sin; + return (0); +} + +static int +mp_getaddr_v6(struct socket *mp_so, struct sockaddr **nam, boolean_t peer) +{ + struct mptses *mpte = mpsotompte(mp_so); + struct in6_addr addr; + in_port_t port; + + if (!peer) { + port = mpte->__mpte_src_v6.sin6_port; + addr = mpte->__mpte_src_v6.sin6_addr; + } else { + port = mpte->__mpte_dst_v6.sin6_port; + addr = mpte->__mpte_dst_v6.sin6_addr; + } + + *nam = in6_sockaddr(port, &addr); + if (*nam == NULL) + return (ENOBUFS); + + return (0); +} + +int +mp_getsockaddr(struct socket *mp_so, struct sockaddr **nam) +{ + struct mptses *mpte = mpsotompte(mp_so); + + if (mpte->mpte_src.sa_family == AF_INET || mpte->mpte_src.sa_family == 0) + return mp_getaddr_v4(mp_so, nam, false); + else if (mpte->mpte_src.sa_family == AF_INET) + return mp_getaddr_v6(mp_so, nam, false); + else + return (EINVAL); +} + +int +mp_getpeeraddr(struct socket *mp_so, struct sockaddr **nam) +{ + struct mptses *mpte = mpsotompte(mp_so); + + if (mpte->mpte_src.sa_family == AF_INET || mpte->mpte_src.sa_family == 0) + return mp_getaddr_v4(mp_so, nam, true); + else if (mpte->mpte_src.sa_family == AF_INET) + return mp_getaddr_v6(mp_so, nam, true); + else + return (EINVAL); +} diff --git a/bsd/netinet/mp_pcb.h b/bsd/netinet/mp_pcb.h index eba202b85..f8fb188c0 100644 --- a/bsd/netinet/mp_pcb.h +++ b/bsd/netinet/mp_pcb.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,6 +30,9 @@ #define _NETINET_MP_PCB_H_ #ifdef BSD_KERNEL_PRIVATE +#include +#include +#include #include #include #include @@ -40,6 +43,10 @@ typedef enum mppcb_state { MPPCB_STATE_DEAD = 2, } mppcb_state_t; + +/* net/necp.h already includes mp_pcb.h - so we have to forward-declare */ +struct necp_client_flow; + /* * Multipath Protocol Control Block */ @@ -47,17 +54,42 @@ struct mppcb { TAILQ_ENTRY(mppcb) mpp_entry; /* glue to all PCBs */ decl_lck_mtx_data(, mpp_lock); /* per PCB lock */ struct mppcbinfo *mpp_pcbinfo; /* PCB info */ - void *mpp_pcbe; /* ptr to per-protocol ext */ + struct mptses *mpp_pcbe; /* ptr to MPTCP-session */ struct socket *mpp_socket; /* back pointer to socket */ uint32_t mpp_flags; /* PCB flags */ mppcb_state_t mpp_state; /* PCB state */ + +#if NECP + uuid_t necp_client_uuid; + void (*necp_cb)(void *, int, struct necp_client_flow *); +#endif }; -#define sotomppcb(so) ((struct mppcb *)((so)->so_pcb)) +static inline struct mppcb * +mpsotomppcb(struct socket *mp_so) +{ + VERIFY(SOCK_DOM(mp_so) == PF_MULTIPATH); + return ((struct mppcb *)mp_so->so_pcb); +} /* valid values for mpp_flags */ -#define MPP_ATTACHED 0x1 -#define MPP_DEFUNCT 0x2 +#define MPP_ATTACHED 0x001 +#define MPP_INSIDE_OUTPUT 0x002 /* MPTCP-stack is inside mptcp_subflow_output */ +#define MPP_INSIDE_INPUT 0x004 /* MPTCP-stack is inside mptcp_subflow_input */ +#define MPP_RUPCALL 0x008 /* MPTCP-stack is handling a read upcall */ +#define MPP_WUPCALL 0x010 /* MPTCP-stack is handling a read upcall */ +#define MPP_SHOULD_WORKLOOP 0x020 /* MPTCP-stack should call the workloop function */ +#define MPP_SHOULD_RWAKEUP 0x040 /* MPTCP-stack should call sorwakeup */ +#define MPP_SHOULD_WWAKEUP 0x080 /* MPTCP-stack should call sowwakeup */ +#define MPP_CREATE_SUBFLOWS 0x100 /* This connection needs to create subflows */ +#define MPP_SET_CELLICON 0x200 /* Set the cellicon (deferred) */ +#define MPP_UNSET_CELLICON 0x400 /* Unset the cellicon (deferred) */ + +static inline boolean_t +mptcp_should_defer_upcall(struct mppcb *mpp) +{ + return !!(mpp->mpp_flags & (MPP_INSIDE_OUTPUT | MPP_INSIDE_INPUT | MPP_RUPCALL | MPP_WUPCALL)); +} /* * Multipath PCB Information @@ -74,8 +106,6 @@ struct mppcbinfo { decl_lck_mtx_data(, mppi_lock); /* global PCB lock */ uint32_t (*mppi_gc)(struct mppcbinfo *); /* garbage collector func */ uint32_t (*mppi_timer)(struct mppcbinfo *); /* timer func */ - /* Extended pcb create func */ - void *(*mppi_pcbe_create) (struct socket *mp_so, struct mppcb *mpp); }; __BEGIN_DECLS @@ -83,10 +113,13 @@ extern void mp_pcbinit(void); extern void mp_pcbinfo_attach(struct mppcbinfo *); extern int mp_pcbinfo_detach(struct mppcbinfo *); extern int mp_pcballoc(struct socket *, struct mppcbinfo *); -extern void mp_pcbdetach(struct mppcb *); +extern void mp_pcbdetach(struct socket *); extern void mp_pcbdispose(struct mppcb *); extern void mp_gc_sched(void); extern void mptcp_timer_sched(void); +extern void mptcp_handle_deferred_upcalls(struct mppcb *mpp, uint32_t flag); +extern int mp_getsockaddr(struct socket *mp_so, struct sockaddr **nam); +extern int mp_getpeeraddr(struct socket *mp_so, struct sockaddr **nam); __END_DECLS #endif /* BSD_KERNEL_PRIVATE */ diff --git a/bsd/netinet/mp_proto.c b/bsd/netinet/mp_proto.c index 65eafb0be..1093fcc9b 100644 --- a/bsd/netinet/mp_proto.c +++ b/bsd/netinet/mp_proto.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -36,19 +36,13 @@ #include #include -#if MPTCP #include -#endif /* MPTCP */ extern struct domain mpdomain_s; -static struct domain *mpdomain = NULL; static void mp_dinit(struct domain *); -lck_mtx_t *mp_domain_mutex; -static struct protosw mpsw[] = { -#if MPTCP -{ +static struct protosw mpsw = { .pr_type = SOCK_STREAM, .pr_protocol = IPPROTO_TCP, .pr_flags = PR_CONNREQUIRED|PR_MULTICONN|PR_EVCONNINFO| @@ -60,12 +54,8 @@ static struct protosw mpsw[] = { .pr_lock = mptcp_lock, .pr_unlock = mptcp_unlock, .pr_getlock = mptcp_getlock, -}, -#endif /* MPTCP */ }; -static int mp_proto_count = (sizeof (mpsw) / sizeof (struct protosw)); - struct domain mpdomain_s = { .dom_family = PF_MULTIPATH, .dom_flags = DOM_REENTRANT, @@ -77,16 +67,7 @@ struct domain mpdomain_s = { void mp_dinit(struct domain *dp) { - struct protosw *pr; - int i; - VERIFY(!(dp->dom_flags & DOM_INITIALIZED)); - VERIFY(mpdomain == NULL); - - mpdomain = dp; - - for (i = 0, pr = &mpsw[0]; i < mp_proto_count; i++, pr++) - net_add_proto(pr, dp, 1); - mp_domain_mutex = dp->dom_mtx; + net_add_proto(&mpsw, dp, 1); } diff --git a/bsd/netinet/mptcp.c b/bsd/netinet/mptcp.c index 264c9d7c1..5d901a9da 100644 --- a/bsd/netinet/mptcp.c +++ b/bsd/netinet/mptcp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -26,6 +26,59 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +/* + * A note on the MPTCP/NECP-interactions: + * + * MPTCP uses NECP-callbacks to get notified of interface/policy events. + * MPTCP registers to these events at the MPTCP-layer for interface-events + * through a call to necp_client_register_multipath_cb. + * To get per-flow events (aka per TCP-subflow), we register to it with + * necp_client_register_socket_flow. Both registrations happen by using the + * necp-client-uuid that comes from the app. + * + * The locking is rather tricky. In general, we expect the lock-ordering to + * happen from necp-fd -> necp->client -> mpp_lock. + * + * There are however some subtleties. + * + * 1. When registering the multipath_cb, we are holding the mpp_lock. This is + * safe, because it is the very first time this MPTCP-connection goes into NECP. + * As we go into NECP we take the NECP-locks and thus are guaranteed that no + * NECP-locks will deadlock us. Because these NECP-events will also first take + * the NECP-locks. Either they win the race and thus won't find our + * MPTCP-connection. Or, MPTCP wins the race and thus it will safely install + * the callbacks while holding the NECP lock. + * + * 2. When registering the subflow-callbacks we must unlock the mpp_lock. This, + * because we have already registered callbacks and we might race against an + * NECP-event that will match on our socket. So, we have to unlock to be safe. + * + * 3. When removing the multipath_cb, we do it in mp_pcbdispose(). The + * so_usecount has reached 0. We must be careful to not remove the mpp_socket + * pointers before we unregistered the callback. Because, again we might be + * racing against an NECP-event. Unregistering must happen with an unlocked + * mpp_lock, because of the lock-ordering constraint. It could be that + * before we had a chance to unregister an NECP-event triggers. That's why + * we need to check for the so_usecount in mptcp_session_necp_cb. If we get + * there while the socket is being garbage-collected, the use-count will go + * down to 0 and we exit. Removal of the multipath_cb again happens by taking + * the NECP-locks so any running NECP-events will finish first and exit cleanly. + * + * 4. When removing the subflow-callback, we do it in in_pcbdispose(). Again, + * the socket-lock must be unlocked for lock-ordering constraints. This gets a + * bit tricky here, as in tcp_garbage_collect we hold the mp_so and so lock. + * So, we drop the mp_so-lock as soon as the subflow is unlinked with + * mptcp_subflow_del. Then, in in_pcbdispose we drop the subflow-lock. + * If an NECP-event was waiting on the lock in mptcp_subflow_necp_cb, when it + * gets it, it will realize that the subflow became non-MPTCP and retry (see + * tcp_lock). Then it waits again on the subflow-lock. When we drop this lock + * in in_pcbdispose, and enter necp_inpcb_dispose, this one will have to wait + * for the NECP-lock (held by the other thread that is taking care of the NECP- + * event). So, the event now finally gets the subflow-lock and then hits an + * so_usecount that is 0 and exits. Eventually, we can remove the subflow from + * the NECP callback. + */ + #include #include #include @@ -39,11 +92,8 @@ #include #include -#include #include -#include - #include #include #include @@ -93,47 +143,6 @@ int mptcp_subflow_keeptime = 60*14; SYSCTL_INT(_net_inet_mptcp, OID_AUTO, keepalive, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_subflow_keeptime, 0, "Keepalive in seconds"); -/* - * MP_PRIO option. - */ -int mptcp_mpprio_enable = 1; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, mpprio, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_mpprio_enable, 0, "Enable MP_PRIO option"); - -/* - * REMOVE_ADDR option. - */ -int mptcp_remaddr_enable = 1; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, remaddr, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_remaddr_enable, 0, "Enable REMOVE_ADDR option"); - -/* - * FastJoin Option - */ -int mptcp_fastjoin = 1; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, fastjoin, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_fastjoin, 0, "Enable FastJoin Option"); - -int mptcp_zerortt_fastjoin = 0; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, zerortt_fastjoin, CTLFLAG_RW | - CTLFLAG_LOCKED, &mptcp_zerortt_fastjoin, 0, - "Enable Zero RTT Fast Join"); - -/* - * R/W Notification on resume - */ -int mptcp_rwnotify = 0; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rwnotify, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_rwnotify, 0, "Enable RW notify on resume"); - -/* - * Using RTT history for sending new data - */ -int mptcp_use_rtthist = 1; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rtthist, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_use_rtthist, 0, "Disable RTT History"); - -#define MPTCP_RTTHIST_MINTHRESH 500 int mptcp_rtthist_rtthresh = 600; SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rtthist_thresh, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_rtthist_rtthresh, 0, "Rtt threshold"); @@ -145,32 +154,17 @@ int mptcp_use_rto = 1; SYSCTL_INT(_net_inet_mptcp, OID_AUTO, userto, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_use_rto, 0, "Disable RTO for subflow selection"); -#define MPTCP_RTO_MINTHRESH 1000 int mptcp_rtothresh = 1500; SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rto_thresh, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_rtothresh, 0, "RTO threshold"); -/* - * Use server's chosen path for sending new data - */ -int mptcp_peerswitch = 1; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, use_peer, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_peerswitch, 0, "Use peer"); - -#define MPTCP_PEERSWITCH_CNTMIN 3 -uint32_t mptcp_peerswitch_cnt = 3; -SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, peerswitchno, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_peerswitch_cnt, 0, "Set threshold based on peer's data arrival"); - /* * Probe the preferred path, when it is not in use */ -#define MPTCP_PROBETO_MIN 500 uint32_t mptcp_probeto = 1000; SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probeto, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_probeto, 0, "Disable probing by setting to 0"); -#define MPTCP_PROBE_MX 15 uint32_t mptcp_probecnt = 5; SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probecnt, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_probecnt, 0, "Number of probe writes"); @@ -178,8 +172,174 @@ SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probecnt, CTLFLAG_RW | CTLFLAG_LOCKED, /* * Static declarations */ -static int mptcp_validate_csum(struct tcpcb *, struct mbuf *, int); -static uint16_t mptcp_input_csum(struct tcpcb *, struct mbuf *, int); +static uint16_t mptcp_input_csum(struct tcpcb *, struct mbuf *, uint64_t, + uint32_t, uint16_t, uint16_t); + +static int +mptcp_reass_present(struct socket *mp_so) +{ + struct mptcb *mp_tp = mpsotomppcb(mp_so)->mpp_pcbe->mpte_mptcb; + struct tseg_qent *q; + int dowakeup = 0; + + /* + * Present data to user, advancing rcv_nxt through + * completed sequence space. + */ + if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) + return (0); + q = LIST_FIRST(&mp_tp->mpt_segq); + if (!q || q->tqe_m->m_pkthdr.mp_dsn != mp_tp->mpt_rcvnxt) + return (0); + + /* + * If there is already another thread doing reassembly for this + * connection, it is better to let it finish the job -- + * (radar 16316196) + */ + if (mp_tp->mpt_flags & MPTCPF_REASS_INPROG) + return (0); + + mp_tp->mpt_flags |= MPTCPF_REASS_INPROG; + + do { + mp_tp->mpt_rcvnxt += q->tqe_len; + LIST_REMOVE(q, tqe_q); + if (mp_so->so_state & SS_CANTRCVMORE) { + m_freem(q->tqe_m); + } else { + if (sbappendstream(&mp_so->so_rcv, q->tqe_m)) + dowakeup = 1; + } + zfree(tcp_reass_zone, q); + mp_tp->mpt_reassqlen--; + q = LIST_FIRST(&mp_tp->mpt_segq); + } while (q && q->tqe_m->m_pkthdr.mp_dsn == mp_tp->mpt_rcvnxt); + mp_tp->mpt_flags &= ~MPTCPF_REASS_INPROG; + + if (dowakeup) + sorwakeup(mp_so); /* done with socket lock held */ + return (0); + +} + +static int +mptcp_reass(struct socket *mp_so, struct pkthdr *phdr, int *tlenp, struct mbuf *m) +{ + struct mptcb *mp_tp = mpsotomppcb(mp_so)->mpp_pcbe->mpte_mptcb; + u_int64_t mb_dsn = phdr->mp_dsn; + struct tseg_qent *q; + struct tseg_qent *p = NULL; + struct tseg_qent *nq; + struct tseg_qent *te = NULL; + u_int16_t qlimit; + + /* + * Limit the number of segments in the reassembly queue to prevent + * holding on to too many segments (and thus running out of mbufs). + * Make sure to let the missing segment through which caused this + * queue. Always keep one global queue entry spare to be able to + * process the missing segment. + */ + qlimit = min(max(100, mp_so->so_rcv.sb_hiwat >> 10), + (tcp_autorcvbuf_max >> 10)); + if (mb_dsn != mp_tp->mpt_rcvnxt && + (mp_tp->mpt_reassqlen + 1) >= qlimit) { + tcpstat.tcps_mptcp_rcvmemdrop++; + m_freem(m); + *tlenp = 0; + return (0); + } + + /* Allocate a new queue entry. If we can't, just drop the pkt. XXX */ + te = (struct tseg_qent *) zalloc(tcp_reass_zone); + if (te == NULL) { + tcpstat.tcps_mptcp_rcvmemdrop++; + m_freem(m); + return (0); + } + + mp_tp->mpt_reassqlen++; + + /* + * Find a segment which begins after this one does. + */ + LIST_FOREACH(q, &mp_tp->mpt_segq, tqe_q) { + if (MPTCP_SEQ_GT(q->tqe_m->m_pkthdr.mp_dsn, mb_dsn)) + break; + p = q; + } + + /* + * If there is a preceding segment, it may provide some of + * our data already. If so, drop the data from the incoming + * segment. If it provides all of our data, drop us. + */ + if (p != NULL) { + int64_t i; + /* conversion to int (in i) handles seq wraparound */ + i = p->tqe_m->m_pkthdr.mp_dsn + p->tqe_len - mb_dsn; + if (i > 0) { + if (i >= *tlenp) { + tcpstat.tcps_mptcp_rcvduppack++; + m_freem(m); + zfree(tcp_reass_zone, te); + te = NULL; + mp_tp->mpt_reassqlen--; + /* + * Try to present any queued data + * at the left window edge to the user. + * This is needed after the 3-WHS + * completes. + */ + goto out; + } + m_adj(m, i); + *tlenp -= i; + phdr->mp_dsn += i; + } + } + + tcpstat.tcps_mp_oodata++; + + /* + * While we overlap succeeding segments trim them or, + * if they are completely covered, dequeue them. + */ + while (q) { + int64_t i = (mb_dsn + *tlenp) - q->tqe_m->m_pkthdr.mp_dsn; + if (i <= 0) + break; + + if (i < q->tqe_len) { + q->tqe_m->m_pkthdr.mp_dsn += i; + q->tqe_len -= i; + m_adj(q->tqe_m, i); + break; + } + + nq = LIST_NEXT(q, tqe_q); + LIST_REMOVE(q, tqe_q); + m_freem(q->tqe_m); + zfree(tcp_reass_zone, q); + mp_tp->mpt_reassqlen--; + q = nq; + } + + /* Insert the new segment queue entry into place. */ + te->tqe_m = m; + te->tqe_th = NULL; + te->tqe_len = *tlenp; + + if (p == NULL) { + LIST_INSERT_HEAD(&mp_tp->mpt_segq, te, tqe_q); + } else { + LIST_INSERT_AFTER(p, te, tqe_q); + } + +out: + return (mptcp_reass_present(mp_so)); +} /* * MPTCP input, called when data has been read from a subflow socket. @@ -189,20 +349,21 @@ mptcp_input(struct mptses *mpte, struct mbuf *m) { struct socket *mp_so; struct mptcb *mp_tp = NULL; - u_int64_t mb_dsn; - u_int32_t mb_datalen; - int count = 0; + int count = 0, wakeup = 0; struct mbuf *save = NULL, *prev = NULL; struct mbuf *freelist = NULL, *tail = NULL; - boolean_t in_fallback = FALSE; VERIFY(m->m_flags & M_PKTHDR); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + + mp_so = mptetoso(mpte); + mp_tp = mpte->mpte_mptcb; DTRACE_MPTCP(input); + mp_tp->mpt_rcvwnd = mptcp_sbspace(mp_tp); + /* * Each mbuf contains MPTCP Data Sequence Map * Process the data for reassembly, delivery to MPTCP socket @@ -211,18 +372,13 @@ mptcp_input(struct mptses *mpte, struct mbuf *m) */ count = mp_so->so_rcv.sb_cc; - VERIFY(m != NULL); - mp_tp = mpte->mpte_mptcb; - VERIFY(mp_tp != NULL); - - /* Ok to check for this flag without lock as its set in this thread */ - in_fallback = (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP); - /* * In the degraded fallback case, data is accepted without DSS map */ - if (in_fallback) { + if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) { fallback: + mptcp_sbrcv_grow(mp_tp); + /* * assume degraded flow as this may be the first packet * without DSS, and the subflow state is not updated yet. @@ -235,18 +391,19 @@ fallback: struct sockbuf *, &mp_so->so_snd, struct mptses *, mpte); count = mp_so->so_rcv.sb_cc - count; - mptcplog((LOG_DEBUG, "MPTCP Receiver: Fallback read %d bytes\n", + mptcplog((LOG_DEBUG, "%s: Fallback read %d bytes\n", __func__, count), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); return; } - MPT_LOCK(mp_tp); do { + u_int64_t mb_dsn; + int32_t mb_datalen; + int64_t todrop; + /* If fallback occurs, mbufs will not have PKTF_MPTCP set */ - if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP)) { - MPT_UNLOCK(mp_tp); + if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP)) goto fallback; - } save = m->m_next; /* @@ -271,26 +428,40 @@ fallback: mb_dsn = m->m_pkthdr.mp_dsn; mb_datalen = m->m_pkthdr.mp_rlen; - if (MPTCP_SEQ_GT(mb_dsn, mp_tp->mpt_rcvatmark)) { - tcpstat.tcps_mp_oodata++; - MPT_UNLOCK(mp_tp); - m_freem(m); - return; - /* - * Reassembly queue support here in future. Per spec, - * senders must implement retransmission timer to - * retransmit unacked data. Dropping out of order - * gives a slight hit on performance but allows us to - * deploy MPTCP and protects us against in-window DoS - * attacks that attempt to use up memory by sending - * out of order data. When doing load sharing across - * subflows, out of order support is a must. - */ + todrop = (mb_dsn + mb_datalen) - (mp_tp->mpt_rcvnxt + mp_tp->mpt_rcvwnd); + if (todrop > 0) { + tcpstat.tcps_mptcp_rcvpackafterwin++; + + if (todrop >= mb_datalen) { + if (freelist == NULL) + freelist = m; + else + tail->m_next = m; + + if (prev != NULL) + tail = prev; + else + tail = m; + + m = save; + prev = save = NULL; + continue; + } else { + m_adj(m, -todrop); + mb_datalen -= todrop; + } } - if (MPTCP_SEQ_LT(mb_dsn, mp_tp->mpt_rcvatmark)) { + if (MPTCP_SEQ_GT(mb_dsn, mp_tp->mpt_rcvnxt) || + !LIST_EMPTY(&mp_tp->mpt_segq)) { + mptcp_reass(mp_so, &m->m_pkthdr, &mb_datalen, m); + + goto next; + } + + if (MPTCP_SEQ_LT(mb_dsn, mp_tp->mpt_rcvnxt)) { if (MPTCP_SEQ_LEQ((mb_dsn + mb_datalen), - mp_tp->mpt_rcvatmark)) { + mp_tp->mpt_rcvnxt)) { if (freelist == NULL) freelist = m; else @@ -305,44 +476,77 @@ fallback: prev = save = NULL; continue; } else { - m_adj(m, (mp_tp->mpt_rcvatmark - mb_dsn)); + m_adj(m, (mp_tp->mpt_rcvnxt - mb_dsn)); } - mptcplog((LOG_INFO, "MPTCP Receiver: Left Edge %llu\n", - mp_tp->mpt_rcvatmark), + mptcplog((LOG_INFO, "%s: Left Edge %llu\n", __func__, + mp_tp->mpt_rcvnxt), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); } - MPT_UNLOCK(mp_tp); - if (sbappendstream(&mp_so->so_rcv, m)) { - sorwakeup(mp_so); - } + mptcp_sbrcv_grow(mp_tp); + + if (sbappendstream(&mp_so->so_rcv, m)) + wakeup = 1; + DTRACE_MPTCP6(receive, struct mbuf *, m, struct socket *, mp_so, struct sockbuf *, &mp_so->so_rcv, struct sockbuf *, &mp_so->so_snd, struct mptses *, mpte, struct mptcb *, mp_tp); - MPT_LOCK(mp_tp); count = mp_so->so_rcv.sb_cc - count; tcpstat.tcps_mp_rcvtotal++; tcpstat.tcps_mp_rcvbytes += count; - mptcplog((LOG_DEBUG, "MPTCP Receiver: Read %d bytes\n", count), + mptcplog((LOG_DEBUG, "%s: Read %d bytes\n", __func__, count), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); - /* - * The data received at the MPTCP layer will never exceed the - * receive window because anything to the right of the - * receive window will be trimmed at the subflow level. - */ - mp_tp->mpt_rcvwnd = mptcp_sbspace(mp_tp); - mp_tp->mpt_rcvatmark += count; + mp_tp->mpt_rcvnxt += count; + +next: m = save; prev = save = NULL; count = mp_so->so_rcv.sb_cc; } while (m); - MPT_UNLOCK(mp_tp); if (freelist) m_freem(freelist); + + if (wakeup) + sorwakeup(mp_so); +} + +static boolean_t +mptcp_can_send_more(struct mptcb *mp_tp) +{ + struct socket *mp_so = mptetoso(mp_tp->mpt_mpte); + + /* + * Always send if there is data in the reinject-queue. + */ + if (mp_tp->mpt_mpte->mpte_reinjectq) + return (TRUE); + + /* + * Don't send, if: + * + * 1. snd_nxt >= snd_max : Means, basically everything has been sent. + * Except when using TFO, we might be doing a 0-byte write. + * 2. snd_una + snd_wnd <= snd_nxt: No space in the receiver's window + * 3. snd_nxt + 1 == snd_max and we are closing: A DATA_FIN is scheduled. + */ + + if (!(mp_so->so_flags1 & SOF1_PRECONNECT_DATA) && MPTCP_SEQ_GEQ(mp_tp->mpt_sndnxt, mp_tp->mpt_sndmax)) + return (FALSE); + + if (MPTCP_SEQ_LEQ(mp_tp->mpt_snduna + mp_tp->mpt_sndwnd, mp_tp->mpt_sndnxt)) + return (FALSE); + + if (mp_tp->mpt_sndnxt + 1 == mp_tp->mpt_sndmax && mp_tp->mpt_state > MPTCPS_CLOSE_WAIT) + return (FALSE); + + if (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_2) + return (FALSE); + + return (TRUE); } /* @@ -351,295 +555,357 @@ fallback: int mptcp_output(struct mptses *mpte) { + struct mptcb *mp_tp; struct mptsub *mpts; struct mptsub *mpts_tried = NULL; struct socket *mp_so; struct mptsub *preferred_mpts = NULL; + uint64_t old_snd_nxt; int error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; - if (mp_so->so_state & SS_CANTSENDMORE) { - mptcplog((LOG_DEBUG, "MPTCP Sender: cantsendmore\n"), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - return (EPIPE); - } + mpte_lock_assert_held(mpte); + mp_so = mptetoso(mpte); + mp_tp = mpte->mpte_mptcb; -try_again: - /* get the "best" subflow to be used for transmission */ - mpts = mptcp_get_subflow(mpte, NULL, &preferred_mpts); - if (mpts == NULL) { - mptcplog((LOG_ERR, "MPTCP Sender: mp_so 0x%llx no subflow\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - goto out; - } + VERIFY(!(mpte->mpte_mppcb->mpp_flags & MPP_WUPCALL)); + mpte->mpte_mppcb->mpp_flags |= MPP_WUPCALL; + + mptcplog((LOG_DEBUG, "%s: snxt %u sndmax %u suna %u swnd %u reinjectq %u state %u\n", + __func__, (uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_sndmax, + (uint32_t)mp_tp->mpt_snduna, mp_tp->mpt_sndwnd, + mpte->mpte_reinjectq ? 1 : 0, + mp_tp->mpt_state), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + + old_snd_nxt = mp_tp->mpt_sndnxt; + while (mptcp_can_send_more(mp_tp)) { + /* get the "best" subflow to be used for transmission */ + mpts = mptcp_get_subflow(mpte, NULL, &preferred_mpts); + if (mpts == NULL) { + mptcplog((LOG_INFO, "%s: no subflow\n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + break; + } - mptcplog((LOG_DEBUG, "MPTCP Sender: mp_so 0x%llx using cid %d \n", - (uint64_t)VM_KERNEL_ADDRPERM(mp_so), mpts->mpts_connid), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - - /* In case there's just one flow, we reattempt later */ - MPTS_LOCK(mpts); - if ((mpts_tried != NULL) && ((mpts == mpts_tried) || - (mpts->mpts_flags & MPTSF_FAILINGOVER))) { - MPTS_UNLOCK(mpts); - MPTS_LOCK(mpts_tried); - mpts_tried->mpts_flags &= ~MPTSF_FAILINGOVER; - mpts_tried->mpts_flags |= MPTSF_ACTIVE; - MPTS_UNLOCK(mpts_tried); - mptcp_start_timer(mpte, MPTT_REXMT); - mptcplog((LOG_DEBUG, "MPTCP Sender: mp_so 0x%llx retry later\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), + mptcplog((LOG_DEBUG, "%s: using id %u\n", __func__, mpts->mpts_connid), MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - goto out; - } - DTRACE_MPTCP3(output, struct mptses *, mpte, struct mptsub *, mpts, - struct socket *, mp_so); - error = mptcp_subflow_output(mpte, mpts); - if (error && error != EWOULDBLOCK) { - /* can be a temporary loss of source address or other error */ - mpts->mpts_flags |= MPTSF_FAILINGOVER; - mpts->mpts_flags &= ~MPTSF_ACTIVE; - mpts_tried = mpts; - MPTS_UNLOCK(mpts); - mptcplog((LOG_INFO, "MPTCP Sender: %s Error = %d \n", - __func__, error), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - goto try_again; - } - /* The model is to have only one active flow at a time */ - mpts->mpts_flags |= MPTSF_ACTIVE; - mpts->mpts_probesoon = mpts->mpts_probecnt = 0; - MPTS_UNLOCK(mpts); - - /* Allows us to update the smoothed rtt */ - if ((mptcp_probeto) && (mptcp_probeto >= MPTCP_PROBETO_MIN) && - (mpts != preferred_mpts) && (preferred_mpts != NULL)) { - MPTS_LOCK(preferred_mpts); - if (preferred_mpts->mpts_probesoon) { - if ((tcp_now - preferred_mpts->mpts_probesoon) > - mptcp_probeto) { - (void) mptcp_subflow_output(mpte, preferred_mpts); - if (preferred_mpts->mpts_probecnt >= - MIN(mptcp_probecnt, MPTCP_PROBE_MX)) { - preferred_mpts->mpts_probesoon = 0; - preferred_mpts->mpts_probecnt = 0; + /* In case there's just one flow, we reattempt later */ + if (mpts_tried != NULL && + (mpts == mpts_tried || (mpts->mpts_flags & MPTSF_FAILINGOVER))) { + mpts_tried->mpts_flags &= ~MPTSF_FAILINGOVER; + mpts_tried->mpts_flags |= MPTSF_ACTIVE; + mptcp_start_timer(mpte, MPTT_REXMT); + mptcplog((LOG_DEBUG, "%s: retry later\n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + break; + } + + /* + * Automatic sizing of send socket buffer. Increase the send + * socket buffer size if all of the following criteria are met + * 1. the receiver has enough buffer space for this data + * 2. send buffer is filled to 7/8th with data (so we actually + * have data to make use of it); + */ + if (tcp_do_autosendbuf == 1 && + (mp_so->so_snd.sb_flags & (SB_AUTOSIZE | SB_TRIM)) == SB_AUTOSIZE && + tcp_cansbgrow(&mp_so->so_snd)) { + if ((mp_tp->mpt_sndwnd / 4 * 5) >= mp_so->so_snd.sb_hiwat && + mp_so->so_snd.sb_cc >= (mp_so->so_snd.sb_hiwat / 8 * 7)) { + if (sbreserve(&mp_so->so_snd, + min(mp_so->so_snd.sb_hiwat + tcp_autosndbuf_inc, + tcp_autosndbuf_max)) == 1) { + mp_so->so_snd.sb_idealsize = mp_so->so_snd.sb_hiwat; + + mptcplog((LOG_DEBUG, "%s: increased snd hiwat to %u lowat %u\n", + __func__, mp_so->so_snd.sb_hiwat, + mp_so->so_snd.sb_lowat), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); } } - } else { - preferred_mpts->mpts_probesoon = tcp_now; - preferred_mpts->mpts_probecnt = 0; } - MPTS_UNLOCK(preferred_mpts); - } - if (mpte->mpte_active_sub == NULL) { - mpte->mpte_active_sub = mpts; - } else if (mpte->mpte_active_sub != mpts) { - mptcplog((LOG_DEBUG, "MPTCP Sender: switch [cid %d, srtt %d]" - "to [cid %d, srtt %d]\n", - mpte->mpte_active_sub->mpts_connid, - mpte->mpte_active_sub->mpts_srtt >> 5, - mpts->mpts_connid, - mpts->mpts_srtt >> 5), - MPTCP_SENDER_DBG | MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - - MPTS_LOCK(mpte->mpte_active_sub); - mpte->mpte_active_sub->mpts_flags &= ~MPTSF_ACTIVE; - mpts->mpts_peerswitch = 0; - MPTS_UNLOCK(mpte->mpte_active_sub); - mpte->mpte_active_sub = mpts; - tcpstat.tcps_mp_switches++; + DTRACE_MPTCP3(output, struct mptses *, mpte, struct mptsub *, mpts, + struct socket *, mp_so); + error = mptcp_subflow_output(mpte, mpts, 0); + if (error) { + /* can be a temporary loss of source address or other error */ + mpts->mpts_flags |= MPTSF_FAILINGOVER; + mpts->mpts_flags &= ~MPTSF_ACTIVE; + mpts_tried = mpts; + mptcplog((LOG_ERR, "%s: Error = %d mpts_flags %#x\n", __func__, + error, mpts->mpts_flags), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + break; + } + /* The model is to have only one active flow at a time */ + mpts->mpts_flags |= MPTSF_ACTIVE; + mpts->mpts_probesoon = mpts->mpts_probecnt = 0; + + /* Allows us to update the smoothed rtt */ + if (mptcp_probeto && mpts != preferred_mpts && preferred_mpts != NULL) { + if (preferred_mpts->mpts_probesoon) { + if ((tcp_now - preferred_mpts->mpts_probesoon) > mptcp_probeto) { + mptcp_subflow_output(mpte, preferred_mpts, MPTCP_SUBOUT_PROBING); + if (preferred_mpts->mpts_probecnt >= mptcp_probecnt) { + preferred_mpts->mpts_probesoon = 0; + preferred_mpts->mpts_probecnt = 0; + } + } + } else { + preferred_mpts->mpts_probesoon = tcp_now; + preferred_mpts->mpts_probecnt = 0; + } + } + + if (mpte->mpte_active_sub == NULL) { + mpte->mpte_active_sub = mpts; + } else if (mpte->mpte_active_sub != mpts) { + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); + struct tcpcb *acttp = sototcpcb(mpte->mpte_active_sub->mpts_socket); + + mptcplog((LOG_DEBUG, "%s: switch [%u, srtt %d] to [%u, srtt %d]\n", __func__, + mpte->mpte_active_sub->mpts_connid, acttp->t_srtt >> TCP_RTT_SHIFT, + mpts->mpts_connid, tp->t_srtt >> TCP_RTT_SHIFT), + (MPTCP_SENDER_DBG | MPTCP_SOCKET_DBG), MPTCP_LOGLVL_LOG); + + mpte->mpte_active_sub->mpts_flags &= ~MPTSF_ACTIVE; + mpte->mpte_active_sub = mpts; + + mptcpstats_inc_switch(mpte, mpts); + } } -out: + + mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_WUPCALL); + /* subflow errors should not be percolated back up */ return (0); } + +static struct mptsub * +mptcp_choose_subflow(struct mptsub *mpts, struct mptsub *curbest, int *currtt) +{ + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); + + /* + * Lower RTT? Take it, if it's our first one, or + * it doesn't has any loss, or the current one has + * loss as well. + */ + if (tp->t_srtt && *currtt > tp->t_srtt && + (curbest == NULL || tp->t_rxtshift == 0 || + sototcpcb(curbest->mpts_socket)->t_rxtshift)) { + *currtt = tp->t_srtt; + return (mpts); + } + + /* + * If we find a subflow without loss, take it always! + */ + if (curbest && + sototcpcb(curbest->mpts_socket)->t_rxtshift && + tp->t_rxtshift == 0) { + *currtt = tp->t_srtt; + return (mpts); + } + + return (curbest != NULL ? curbest : mpts); +} + +static struct mptsub * +mptcp_return_subflow(struct mptsub *mpts) +{ + if (mpts && mptcp_subflow_cwnd_space(mpts->mpts_socket) <= 0) + return (NULL); + + return (mpts); +} + /* * Return the most eligible subflow to be used for sending data. - * This function also serves to check if any alternate subflow is available - * or not. best and second_best flows are chosen by their priority. third_best - * could be best or second_best but is under loss at the time of evaluation. */ struct mptsub * mptcp_get_subflow(struct mptses *mpte, struct mptsub *ignore, struct mptsub **preferred) { + struct tcpcb *besttp, *secondtp; + struct inpcb *bestinp, *secondinp; struct mptsub *mpts; struct mptsub *best = NULL; struct mptsub *second_best = NULL; - struct mptsub *third_best = NULL; - struct mptsub *symptoms_best = NULL; - struct socket *so = NULL; + int exp_rtt = INT_MAX, cheap_rtt = INT_MAX; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + /* + * First Step: + * Choose the best subflow for cellular and non-cellular interfaces. + */ TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); + struct socket *so = mpts->mpts_socket; + struct tcpcb *tp = sototcpcb(so); + struct inpcb *inp = sotoinpcb(so); + + mptcplog((LOG_DEBUG, "%s mpts %u ignore %d, mpts_flags %#x, suspended %u sostate %#x tpstate %u cellular %d rtt %u rxtshift %u cheap %u exp %u cwnd %d\n", + __func__, mpts->mpts_connid, ignore ? ignore->mpts_connid : -1, mpts->mpts_flags, + INP_WAIT_FOR_IF_FEEDBACK(inp), so->so_state, tp->t_state, + inp->inp_last_outifp ? IFNET_IS_CELLULAR(inp->inp_last_outifp) : -1, + tp->t_srtt, tp->t_rxtshift, cheap_rtt, exp_rtt, + mptcp_subflow_cwnd_space(so)), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - if ((ignore) && (mpts == ignore)) { - MPTS_UNLOCK(mpts); + /* + * First, the hard conditions to reject subflows + * (e.g., not connected,...) + */ + if (mpts == ignore || inp->inp_last_outifp == NULL) + continue; + + if (INP_WAIT_FOR_IF_FEEDBACK(inp)) continue; - } /* There can only be one subflow in degraded state */ if (mpts->mpts_flags & MPTSF_MP_DEGRADED) { - MPTS_UNLOCK(mpts); best = mpts; break; } /* - * Subflows with TFO or Fastjoin allow data to be written before - * the subflow is mp capable. + * If this subflow is waiting to finally send, do it! */ - if (!(mpts->mpts_flags & MPTSF_MP_CAPABLE) && - !(mpts->mpts_flags & MPTSF_FASTJ_REQD) && - !(mpts->mpts_flags & MPTSF_TFO_REQD)) { - MPTS_UNLOCK(mpts); - continue; - } + if (so->so_flags1 & SOF1_PRECONNECT_DATA) + return (mptcp_return_subflow(mpts)); - if (mpts->mpts_flags & MPTSF_SUSPENDED) { - MPTS_UNLOCK(mpts); + /* + * Only send if the subflow is MP_CAPABLE. The exceptions to + * this rule (degraded or TFO) have been taken care of above. + */ + if (!(mpts->mpts_flags & MPTSF_MP_CAPABLE)) continue; - } - if ((mpts->mpts_flags & MPTSF_DISCONNECTED) || - (mpts->mpts_flags & MPTSF_DISCONNECTING)) { - MPTS_UNLOCK(mpts); + if ((so->so_state & SS_ISDISCONNECTED) || + !(so->so_state & SS_ISCONNECTED) || + !TCPS_HAVEESTABLISHED(tp->t_state) || + tp->t_state > TCPS_CLOSE_WAIT) continue; - } - - if (mpts->mpts_flags & MPTSF_FAILINGOVER) { - so = mpts->mpts_socket; - if ((so) && (!(so->so_flags & SOF_PCBCLEARING))) { - socket_lock(so, 1); - if ((so->so_snd.sb_cc == 0) && - (mptcp_no_rto_spike(so))) { - mpts->mpts_flags &= ~MPTSF_FAILINGOVER; - so->so_flags &= ~SOF_MP_TRYFAILOVER; - socket_unlock(so, 1); - } else { - third_best = mpts; - mptcplog((LOG_DEBUG, "MPTCP Sender: " - "%s cid %d in failover\n", - __func__, third_best->mpts_connid), - MPTCP_SENDER_DBG, - MPTCP_LOGLVL_VERBOSE); - socket_unlock(so, 1); - MPTS_UNLOCK(mpts); - continue; - } - } else { - MPTS_UNLOCK(mpts); - continue; - } - } - /* When there are no preferred flows, use first one in list */ - if ((!second_best) && !(mpts->mpts_flags & MPTSF_PREFERRED)) - second_best = mpts; - - if (mpts->mpts_flags & MPTSF_PREFERRED) { - best = mpts; - } - - MPTS_UNLOCK(mpts); + /* + * Second, the soft conditions to find the subflow with best + * conditions for each set (aka cellular vs non-cellular) + */ + if (IFNET_IS_CELLULAR(inp->inp_last_outifp)) + second_best = mptcp_choose_subflow(mpts, second_best, + &exp_rtt); + else + best = mptcp_choose_subflow(mpts, best, &cheap_rtt); } /* * If there is no preferred or backup subflow, and there is no active * subflow use the last usable subflow. */ - if (best == NULL) { - return (second_best ? second_best : third_best); - } + if (best == NULL) + return (mptcp_return_subflow(second_best)); - if (second_best == NULL) { - return (best ? best : third_best); - } + if (second_best == NULL) + return (mptcp_return_subflow(best)); + + besttp = sototcpcb(best->mpts_socket); + bestinp = sotoinpcb(best->mpts_socket); + secondtp = sototcpcb(second_best->mpts_socket); + secondinp = sotoinpcb(second_best->mpts_socket); if (preferred != NULL) - *preferred = best; - - /* Use a hint from symptomsd if it exists */ - symptoms_best = mptcp_use_symptoms_hints(best, second_best); - if (symptoms_best != NULL) - return (symptoms_best); - - /* Compare RTTs, select second_best if best's rtt exceeds rttthresh */ - if ((mptcp_use_rtthist) && - (best->mpts_srtt) && (second_best->mpts_srtt) && - (best->mpts_srtt > second_best->mpts_srtt) && - (best->mpts_srtt >= MAX((MPTCP_RTTHIST_MINTHRESH << 5), - (mptcp_rtthist_rtthresh << 5)))) { - tcpstat.tcps_mp_sel_rtt++; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s best cid %d" - " at rtt %d, second cid %d at rtt %d\n", __func__, - best->mpts_connid, best->mpts_srtt >> 5, - second_best->mpts_connid, - second_best->mpts_srtt >> 5), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return (second_best); - } + *preferred = mptcp_return_subflow(best); - /* Compare RTOs, select second_best if best's rto exceeds rtothresh */ - if ((mptcp_use_rto) && - (best->mpts_rxtcur) && (second_best->mpts_rxtcur) && - (best->mpts_rxtcur > second_best->mpts_rxtcur) && - (best->mpts_rxtcur >= - MAX(MPTCP_RTO_MINTHRESH, mptcp_rtothresh))) { - tcpstat.tcps_mp_sel_rto++; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s best cid %d" - " at rto %d, second cid %d at rto %d\n", __func__, - best->mpts_connid, best->mpts_rxtcur, - second_best->mpts_connid, second_best->mpts_rxtcur), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - - return (second_best); - } + /* + * Second Step: Among best and second_best. Choose the one that is + * most appropriate for this particular service-type. + */ + if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER) { + /* + * Only handover if Symptoms tells us to do so. + */ + if (IFNET_IS_WIFI(bestinp->inp_last_outifp) && + mptcp_is_wifi_unusable() && + besttp->t_rxtshift >= mptcp_fail_thresh) + return (mptcp_return_subflow(second_best)); + + return (mptcp_return_subflow(best)); + } else if (mpte->mpte_svctype == MPTCP_SVCTYPE_INTERACTIVE) { + int rtt_thresh = mptcp_rtthist_rtthresh << TCP_RTT_SHIFT; + int rto_thresh = mptcp_rtothresh; + + /* Adjust with symptoms information */ + if (IFNET_IS_WIFI(bestinp->inp_last_outifp) && + mptcp_is_wifi_unusable()) { + rtt_thresh /= 2; + rto_thresh /= 2; + } - /* If second_best received data, use second_best */ - if (mptcp_peerswitch && - (second_best->mpts_peerswitch > - MAX(MPTCP_PEERSWITCH_CNTMIN, mptcp_peerswitch_cnt))) { - tcpstat.tcps_mp_sel_peer++; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s: best cid %d" - " but using cid %d after receiving %d segments\n", - __func__, best->mpts_connid, second_best->mpts_connid, - second_best->mpts_peerswitch), MPTCP_SENDER_DBG, - MPTCP_LOGLVL_LOG); - return (second_best); - } - return (best); -} + if (besttp->t_srtt && secondtp->t_srtt && + besttp->t_srtt >= rtt_thresh && + secondtp->t_srtt < rtt_thresh) { + tcpstat.tcps_mp_sel_rtt++; + mptcplog((LOG_DEBUG, "%s: best cid %d at rtt %d, second cid %d at rtt %d\n", __func__, + best->mpts_connid, besttp->t_srtt >> TCP_RTT_SHIFT, + second_best->mpts_connid, + secondtp->t_srtt >> TCP_RTT_SHIFT), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + return (mptcp_return_subflow(second_best)); + } -struct mptsub * -mptcp_get_pending_subflow(struct mptses *mpte, struct mptsub *ignore) -{ - struct mptsub *mpts = NULL; + if (besttp->t_rxtshift >= mptcp_fail_thresh && + secondtp->t_rxtshift == 0) { + return (mptcp_return_subflow(second_best)); + } - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + /* Compare RTOs, select second_best if best's rto exceeds rtothresh */ + if (besttp->t_rxtcur && secondtp->t_rxtcur && + besttp->t_rxtcur >= rto_thresh && + secondtp->t_rxtcur < rto_thresh) { + tcpstat.tcps_mp_sel_rto++; + mptcplog((LOG_DEBUG, "%s: best cid %d at rto %d, second cid %d at rto %d\n", __func__, + best->mpts_connid, besttp->t_rxtcur, + second_best->mpts_connid, secondtp->t_rxtcur), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + + return (mptcp_return_subflow(second_best)); + } - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); + /* + * None of the above conditions for sending on the secondary + * were true. So, let's schedule on the best one, if he still + * has some space in the congestion-window. + */ + return (mptcp_return_subflow(best)); + } else if (mpte->mpte_svctype == MPTCP_SVCTYPE_AGGREGATE) { + struct mptsub *tmp; - if ((ignore) && (mpts == ignore)) { - MPTS_UNLOCK(mpts); - continue; + /* + * We only care about RTT when aggregating + */ + if (besttp->t_srtt > secondtp->t_srtt) { + tmp = best; + best = second_best; + besttp = secondtp; + bestinp = secondinp; + + second_best = tmp; + secondtp = sototcpcb(second_best->mpts_socket); + secondinp = sotoinpcb(second_best->mpts_socket); } - if (mpts->mpts_flags & MPTSF_CONNECT_PENDING) { - MPTS_UNLOCK(mpts); - break; - } + /* Is there still space in the congestion window? */ + if (mptcp_subflow_cwnd_space(bestinp->inp_socket) <= 0) + return (mptcp_return_subflow(second_best)); - MPTS_UNLOCK(mpts); + return (mptcp_return_subflow(best)); + } else { + panic("Unknown service-type configured for MPTCP"); } - return (mpts); + + return (NULL); } static const char * @@ -702,7 +968,7 @@ mptcp_state_to_str(mptcp_state_t state) void mptcp_close_fsm(struct mptcb *mp_tp, uint32_t event) { - MPT_LOCK_ASSERT_HELD(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); mptcp_state_t old_state = mp_tp->mpt_state; DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp, @@ -747,7 +1013,7 @@ mptcp_close_fsm(struct mptcb *mp_tp, uint32_t event) case MPTCPS_LAST_ACK: if (event == MPCE_RECV_DATA_ACK) - mp_tp->mpt_state = MPTCPS_TERMINATE; + mptcp_close(mp_tp->mpt_mpte, mp_tp); break; case MPTCPS_FIN_WAIT_2: @@ -758,54 +1024,22 @@ mptcp_close_fsm(struct mptcb *mp_tp, uint32_t event) break; case MPTCPS_TIME_WAIT: - break; - case MPTCPS_TERMINATE: break; + default: VERIFY(0); /* NOTREACHED */ } DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp, uint32_t, event); - mptcplog((LOG_INFO, "MPTCP State: %s to %s on event %s\n", + mptcplog((LOG_INFO, "%s: %s to %s on event %s\n", __func__, mptcp_state_to_str(old_state), mptcp_state_to_str(mp_tp->mpt_state), mptcp_event_to_str(event)), MPTCP_STATE_DBG, MPTCP_LOGLVL_LOG); } -/* - * Update the mptcb send state variables, but the actual sbdrop occurs - * in MPTCP layer - */ -void -mptcp_data_ack_rcvd(struct mptcb *mp_tp, struct tcpcb *tp, u_int64_t full_dack) -{ - u_int64_t acked = 0; - - acked = full_dack - mp_tp->mpt_snduna; - - if (acked) { - mp_tp->mpt_snduna += acked; - /* In degraded mode, we may get some Data ACKs */ - if ((tp->t_mpflags & TMPF_TCP_FALLBACK) && - !(mp_tp->mpt_flags & MPTCPF_POST_FALLBACK_SYNC) && - MPTCP_SEQ_GT(mp_tp->mpt_sndnxt, mp_tp->mpt_snduna)) { - /* bring back sndnxt to retransmit MPTCP data */ - mp_tp->mpt_sndnxt = mp_tp->mpt_dsn_at_csum_fail; - mp_tp->mpt_flags |= MPTCPF_POST_FALLBACK_SYNC; - tp->t_inpcb->inp_socket->so_flags1 |= - SOF1_POST_FALLBACK_SYNC; - } - } - if ((full_dack == mp_tp->mpt_sndmax) && - (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_1)) { - mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_ACK); - tp->t_mpflags &= ~TMPF_SEND_DFIN; - } -} - /* If you change this function, match up mptcp_update_rcv_state_f */ void mptcp_update_dss_rcv_state(struct mptcp_dsn_opt *dss_info, struct tcpcb *tp, @@ -819,9 +1053,7 @@ mptcp_update_dss_rcv_state(struct mptcp_dsn_opt *dss_info, struct tcpcb *tp, NTOHS(dss_info->mdss_data_len); /* XXX for autosndbuf grow sb here */ - MPT_LOCK(mp_tp); MPTCP_EXTEND_DSN(mp_tp->mpt_rcvnxt, dss_info->mdss_dsn, full_dsn); - MPT_UNLOCK(mp_tp); mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, dss_info->mdss_subflow_seqn, dss_info->mdss_data_len, csum); @@ -834,32 +1066,28 @@ mptcp_update_rcv_state_meat(struct mptcb *mp_tp, struct tcpcb *tp, uint16_t csum) { if (mdss_data_len == 0) { - mptcplog((LOG_INFO, "MPTCP Receiver: Infinite Mapping.\n"), + mptcplog((LOG_INFO, "%s: Infinite Mapping.\n", __func__), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG); if ((mp_tp->mpt_flags & MPTCPF_CHECKSUM) && (csum != 0)) { - mptcplog((LOG_ERR, "MPTCP Receiver: Bad checksum %x \n", + mptcplog((LOG_ERR, "%s: Bad checksum %x \n", __func__, csum), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); } mptcp_notify_mpfail(tp->t_inpcb->inp_socket); return; } - MPT_LOCK(mp_tp); mptcplog((LOG_DEBUG, - "MPTCP Receiver: seqn = %x len = %x full = %llx " - "rcvnxt = %llu \n", + "%s: seqn = %x len = %x full = %llx rcvnxt = %llu \n", __func__, seqn, mdss_data_len, full_dsn, mp_tp->mpt_rcvnxt), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); /* Process a Data FIN packet , handled in mptcp_do_fin_opt */ if ((seqn == 0) && (mdss_data_len == 1)) { - mptcplog((LOG_INFO, "MPTCP Receiver: Data FIN in %s state \n", + mptcplog((LOG_INFO, "%s: Data FIN in %s state \n", __func__, mptcp_state_to_str(mp_tp->mpt_state)), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG); - MPT_UNLOCK(mp_tp); return; } - MPT_UNLOCK(mp_tp); mptcp_notify_mpready(tp->t_inpcb->inp_socket); tp->t_rcv_map.mpt_dsn = full_dsn; tp->t_rcv_map.mpt_sseq = seqn; @@ -869,78 +1097,20 @@ mptcp_update_rcv_state_meat(struct mptcb *mp_tp, struct tcpcb *tp, } -void -mptcp_update_rcv_state_f(struct mptcp_dss_ack_opt *dss_info, struct tcpcb *tp, - uint16_t csum) -{ - u_int64_t full_dsn = 0; - struct mptcb *mp_tp = tptomptp(tp); - - /* - * May happen, because the caller of this function does an soevent. - * Review after rdar://problem/24083886 - */ - if (!mp_tp) - return; - - NTOHL(dss_info->mdss_dsn); - NTOHL(dss_info->mdss_subflow_seqn); - NTOHS(dss_info->mdss_data_len); - MPT_LOCK(mp_tp); - MPTCP_EXTEND_DSN(mp_tp->mpt_rcvnxt, dss_info->mdss_dsn, full_dsn); - MPT_UNLOCK(mp_tp); - mptcp_update_rcv_state_meat(mp_tp, tp, - full_dsn, - dss_info->mdss_subflow_seqn, - dss_info->mdss_data_len, - csum); -} - -void -mptcp_update_rcv_state_g(struct mptcp_dss64_ack32_opt *dss_info, - struct tcpcb *tp, uint16_t csum) -{ - u_int64_t dsn = mptcp_ntoh64(dss_info->mdss_dsn); - struct mptcb *mp_tp = tptomptp(tp); - - /* - * May happen, because the caller of this function does an soevent. - * Review after rdar://problem/24083886 - */ - if (!mp_tp) - return; - - NTOHL(dss_info->mdss_subflow_seqn); - NTOHS(dss_info->mdss_data_len); - mptcp_update_rcv_state_meat(mp_tp, tp, - dsn, - dss_info->mdss_subflow_seqn, - dss_info->mdss_data_len, - csum); -} - static int mptcp_validate_dss_map(struct socket *so, struct tcpcb *tp, struct mbuf *m, int hdrlen) { - u_int32_t sseq, datalen; + u_int32_t datalen; if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP)) return 0; - sseq = m->m_pkthdr.mp_rseq + tp->irs; datalen = m->m_pkthdr.mp_rlen; -#if 0 - /* enable this to test TCP fallback post connection establishment */ - if (SEQ_GT(sseq, (tp->irs+1))) - datalen = m->m_pkthdr.len - hdrlen - 1; -#endif - /* unacceptable DSS option, fallback to TCP */ if (m->m_pkthdr.len > ((int) datalen + hdrlen)) { - mptcplog((LOG_ERR, "MPTCP Receiver: " - "%s: mbuf len %d, MPTCP expected %d", + mptcplog((LOG_ERR, "%s: mbuf len %d, MPTCP expected %d", __func__, m->m_pkthdr.len, datalen), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG); } else { @@ -955,9 +1125,6 @@ mptcp_validate_dss_map(struct socket *so, struct tcpcb *tp, struct mbuf *m, int mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, int drop_hdrlen) { - if (mptcp_validate_csum(tp, m, drop_hdrlen) != 0) - return -1; - mptcp_insert_rmap(tp, m); if (mptcp_validate_dss_map(tp->t_inpcb->inp_socket, tp, m, drop_hdrlen) != 0) @@ -973,31 +1140,29 @@ mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, int drop_hdrlen) * DSS option. */ -static int -mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, int drop_hdrlen) +int +mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, + uint32_t sseq, uint16_t dlen, uint16_t csum) { - uint16_t mptcp_csum = 0; - mptcp_csum = mptcp_input_csum(tp, m, drop_hdrlen); + uint16_t mptcp_csum; + + mptcp_csum = mptcp_input_csum(tp, m, dsn, sseq, dlen, csum); if (mptcp_csum) { tp->t_mpflags |= TMPF_SND_MPFAIL; - tp->t_mpflags &= ~TMPF_EMBED_DSN; mptcp_notify_mpfail(tp->t_inpcb->inp_socket); m_freem(m); tcpstat.tcps_mp_badcsum++; - return -1; + return (-1); } - return 0; + return (0); } static uint16_t -mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, int off) +mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, uint32_t sseq, + uint16_t dlen, uint16_t csum) { struct mptcb *mp_tp = tptomptp(tp); uint32_t sum = 0; - uint64_t dsn; - uint32_t sseq; - uint16_t len; - uint16_t csum; if (mp_tp == NULL) return (0); @@ -1005,9 +1170,6 @@ mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, int off) if (!(mp_tp->mpt_flags & MPTCPF_CHECKSUM)) return (0); - if (!(tp->t_mpflags & TMPF_EMBED_DSN)) - return (0); - if (tp->t_mpflags & TMPF_TCP_FALLBACK) return (0); @@ -1015,62 +1177,42 @@ mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, int off) * The remote side may send a packet with fewer bytes than the * claimed DSS checksum length. */ - if ((int)m_length2(m, NULL) < (off + tp->t_rcv_map.mpt_len)) + if ((int)m_length2(m, NULL) < dlen) return (0xffff); - if (tp->t_rcv_map.mpt_len != 0) - sum = m_sum16(m, off, tp->t_rcv_map.mpt_len); + if (dlen != 0) + sum = m_sum16(m, 0, dlen); - dsn = mptcp_hton64(tp->t_rcv_map.mpt_dsn); - sseq = htonl(tp->t_rcv_map.mpt_sseq); - len = htons(tp->t_rcv_map.mpt_len); - csum = tp->t_rcv_map.mpt_csum; - sum += in_pseudo64(dsn, sseq, (len + csum)); + sum += in_pseudo64(htonll(dsn), htonl(sseq), htons(dlen) + csum); ADDCARRY(sum); DTRACE_MPTCP3(checksum__result, struct tcpcb *, tp, struct mbuf *, m, uint32_t, sum); - mptcplog((LOG_DEBUG, "MPTCP Receiver: sum = %x \n", sum), + + mptcplog((LOG_DEBUG, "%s: sum = %x \n", __func__, sum), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); return (~sum & 0xffff); } -void -mptcp_output_csum(struct tcpcb *tp, struct mbuf *m, int32_t len, - unsigned hdrlen, u_int64_t dss_val, u_int32_t *sseqp) +uint32_t +mptcp_output_csum(struct mbuf *m, uint64_t dss_val, uint32_t sseq, uint16_t dlen) { - struct mptcb *mp_tp = tptomptp(tp); u_int32_t sum = 0; - uint32_t sseq; - uint16_t dss_len; - uint16_t csum = 0; - uint16_t *csump = NULL; - - if (mp_tp == NULL) - return; - - if (!(mp_tp->mpt_flags & MPTCPF_CHECKSUM)) - return; - if (sseqp == NULL) - return; - - if (len) - sum = m_sum16(m, hdrlen, len); + if (dlen) + sum = m_sum16(m, 0, dlen); dss_val = mptcp_hton64(dss_val); - sseq = *sseqp; - dss_len = *(uint16_t *)(void *)((u_char*)sseqp + sizeof (u_int32_t)); - sum += in_pseudo64(dss_val, sseq, (dss_len + csum)); + sseq = htonl(sseq); + dlen = htons(dlen); + sum += in_pseudo64(dss_val, sseq, dlen); ADDCARRY(sum); sum = ~sum & 0xffff; - csump = (uint16_t *)(void *)((u_char*)sseqp + sizeof (u_int32_t) + - sizeof (uint16_t)); - DTRACE_MPTCP3(checksum__result, struct tcpcb *, tp, struct mbuf *, m, - uint32_t, sum); - *csump = sum; - mptcplog((LOG_DEBUG, "MPTCP Sender: sum = %x \n", sum), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + DTRACE_MPTCP2(checksum__result, struct mbuf *, m, uint32_t, sum); + mptcplog((LOG_DEBUG, "%s: sum = %x \n", __func__, sum), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + + return sum; } /* @@ -1084,11 +1226,11 @@ mptcp_no_rto_spike(struct socket *so) struct tcpcb *tp = intotcpcb(sotoinpcb(so)); int32_t spike = 0; - if (tp->t_rxtcur > MAX(mptcp_rtothresh, MPTCP_RTO_MINTHRESH)) { + if (tp->t_rxtcur > mptcp_rtothresh) { spike = tp->t_rxtcur - mptcp_rtothresh; - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: spike = %d rto = %d" - "best = %d cur = %d\n", __func__, spike, + mptcplog((LOG_DEBUG, "%s: spike = %d rto = %d best = %d cur = %d\n", + __func__, spike, tp->t_rxtcur, tp->t_rttbest >> TCP_RTT_SHIFT, tp->t_rttcur), (MPTCP_SOCKET_DBG|MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); @@ -1101,3 +1243,229 @@ mptcp_no_rto_spike(struct socket *so) return (TRUE); } } + +void +mptcp_handle_deferred_upcalls(struct mppcb *mpp, uint32_t flag) +{ + VERIFY(mpp->mpp_flags & flag); + mpp->mpp_flags &= ~flag; + + if (mptcp_should_defer_upcall(mpp)) + return; + + if (mpp->mpp_flags & MPP_SHOULD_WORKLOOP) { + mpp->mpp_flags &= ~MPP_SHOULD_WORKLOOP; + + mptcp_subflow_workloop(mpp->mpp_pcbe); + } + + if (mpp->mpp_flags & MPP_SHOULD_RWAKEUP) { + mpp->mpp_flags &= ~MPP_SHOULD_RWAKEUP; + + sorwakeup(mpp->mpp_socket); + } + + if (mpp->mpp_flags & MPP_SHOULD_WWAKEUP) { + mpp->mpp_flags &= ~MPP_SHOULD_WWAKEUP; + + sowwakeup(mpp->mpp_socket); + } + + if (mpp->mpp_flags & MPP_SET_CELLICON) { + mpp->mpp_flags &= ~MPP_SET_CELLICON; + + mptcp_set_cellicon(mpp->mpp_pcbe); + } + + if (mpp->mpp_flags & MPP_UNSET_CELLICON) { + mpp->mpp_flags &= ~MPP_UNSET_CELLICON; + + mptcp_unset_cellicon(); + } +} + +static void +mptcp_ask_for_nat64(struct ifnet *ifp) +{ + in6_post_msg(ifp, KEV_INET6_REQUEST_NAT64_PREFIX, NULL, NULL); + + mptcplog((LOG_DEBUG, "%s: asked for NAT64-prefix on %s\n", + __func__, ifp->if_name), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); +} + +static void +mptcp_reset_itfinfo(struct mpt_itf_info *info) +{ + info->ifindex = 0; + info->has_v4_conn = 0; + info->has_v6_conn = 0; +} + +void +mptcp_session_necp_cb(void *handle, int action, struct necp_client_flow *flow) +{ + struct mppcb *mp = (struct mppcb *)handle; + struct mptses *mpte = mptompte(mp); + struct socket *mp_so; + struct mptcb *mp_tp; + int locked = 0; + uint32_t i, ifindex; + + ifindex = flow->interface_index; + VERIFY(ifindex != IFSCOPE_NONE); + + /* ToDo - remove after rdar://problem/32007628 */ + if (!IF_INDEX_IN_RANGE(ifindex)) + printf("%s 1 ifindex %u not in range of flow %p action %d\n", + __func__, ifindex, flow, action); + + /* About to be garbage-collected (see note about MPTCP/NECP interactions) */ + if (mp->mpp_socket->so_usecount == 0) + return; + + if (action != NECP_CLIENT_CBACTION_INITIAL) { + mpte_lock(mpte); + locked = 1; + + /* Check again, because it might have changed while waiting */ + if (mp->mpp_socket->so_usecount == 0) + goto out; + } + + mp_tp = mpte->mpte_mptcb; + mp_so = mptetoso(mpte); + + mptcplog((LOG_DEBUG, "%s, action: %u ifindex %u usecount %u mpt_flags %#x state %u\n", + __func__, action, ifindex, mp->mpp_socket->so_usecount, mp_tp->mpt_flags, mp_tp->mpt_state), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + /* No need on fallen back sockets */ + if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) + goto out; + + if (action == NECP_CLIENT_CBACTION_NONVIABLE) { + for (i = 0; i < mpte->mpte_itfinfo_size; i++) { + if (mpte->mpte_itfinfo[i].ifindex == ifindex) + mptcp_reset_itfinfo(&mpte->mpte_itfinfo[i]); + } + + mptcp_sched_create_subflows(mpte); + } else if (action == NECP_CLIENT_CBACTION_VIABLE || + action == NECP_CLIENT_CBACTION_INITIAL) { + int found_empty = 0, empty_index = -1; + struct ifnet *ifp; + + /* ToDo - remove after rdar://problem/32007628 */ + if (!IF_INDEX_IN_RANGE(ifindex)) + printf("%s 2 ifindex %u not in range of flow %p action %d\n", + __func__, ifindex, flow, action); + + ifnet_head_lock_shared(); + ifp = ifindex2ifnet[ifindex]; + ifnet_head_done(); + + /* ToDo - remove after rdar://problem/32007628 */ + if (!IF_INDEX_IN_RANGE(ifindex)) + printf("%s 3 ifindex %u not in range of flow %p action %d\n", + __func__, ifindex, flow, action); + + if (ifp == NULL) + goto out; + + if (IFNET_IS_EXPENSIVE(ifp) && + (mp_so->so_restrictions & SO_RESTRICT_DENY_EXPENSIVE)) + goto out; + + if (IFNET_IS_CELLULAR(ifp) && + (mp_so->so_restrictions & SO_RESTRICT_DENY_CELLULAR)) + goto out; + + for (i = 0; i < mpte->mpte_itfinfo_size; i++) { + if (mpte->mpte_itfinfo[i].ifindex == 0) { + found_empty = 1; + empty_index = i; + } + + if (mpte->mpte_itfinfo[i].ifindex == ifindex) { + /* Ok, it's already there */ + goto out; + } + } + + if ((mpte->mpte_dst.sa_family == AF_INET || mpte->mpte_dst.sa_family == 0) && + !(flow->necp_flow_flags & NECP_CLIENT_RESULT_FLAG_HAS_IPV4) && + ifnet_get_nat64prefix(ifp, NULL) == ENOENT) { + mptcp_ask_for_nat64(ifp); + goto out; + } + + if (found_empty == 0) { + int new_size = mpte->mpte_itfinfo_size * 2; + struct mpt_itf_info *info = _MALLOC(sizeof(*info) * new_size, M_TEMP, M_ZERO); + + if (info == NULL) { + mptcplog((LOG_ERR, "%s malloc failed for %u\n", __func__, new_size), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + goto out; + } + + memcpy(info, mpte->mpte_itfinfo, mpte->mpte_itfinfo_size * sizeof(*info)); + + if (mpte->mpte_itfinfo_size > MPTE_ITFINFO_SIZE) + _FREE(mpte->mpte_itfinfo, M_TEMP); + + /* We allocated a new one, thus the first must be empty */ + empty_index = mpte->mpte_itfinfo_size; + + mpte->mpte_itfinfo = info; + mpte->mpte_itfinfo_size = new_size; + + mptcplog((LOG_DEBUG, "%s Needed to realloc to %u\n", __func__, new_size), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + } + + VERIFY(empty_index >= 0 && empty_index < (int)mpte->mpte_itfinfo_size); + mpte->mpte_itfinfo[empty_index].ifindex = ifindex; + mpte->mpte_itfinfo[empty_index].has_v4_conn = !!(flow->necp_flow_flags & NECP_CLIENT_RESULT_FLAG_HAS_IPV4); + mpte->mpte_itfinfo[empty_index].has_v6_conn = !!(flow->necp_flow_flags & NECP_CLIENT_RESULT_FLAG_HAS_IPV6); + + mptcp_sched_create_subflows(mpte); + } + +out: + if (locked) + mpte_unlock(mpte); +} + +void +mptcp_set_restrictions(struct socket *mp_so) +{ + struct mptses *mpte = mpsotompte(mp_so); + uint32_t i; + + mpte_lock_assert_held(mpte); + + ifnet_head_lock_shared(); + + for (i = 0; i < mpte->mpte_itfinfo_size; i++) { + struct mpt_itf_info *info = &mpte->mpte_itfinfo[i]; + uint32_t ifindex = info->ifindex; + struct ifnet *ifp; + + if (ifindex == IFSCOPE_NONE) + continue; + + ifp = ifindex2ifnet[ifindex]; + + if (IFNET_IS_EXPENSIVE(ifp) && + (mp_so->so_restrictions & SO_RESTRICT_DENY_EXPENSIVE)) + info->ifindex = IFSCOPE_NONE; + + if (IFNET_IS_CELLULAR(ifp) && + (mp_so->so_restrictions & SO_RESTRICT_DENY_CELLULAR)) + info->ifindex = IFSCOPE_NONE; + } + + ifnet_head_done(); +} + diff --git a/bsd/netinet/mptcp.h b/bsd/netinet/mptcp.h index 3ea265ebc..d0b77e6b6 100644 --- a/bsd/netinet/mptcp.h +++ b/bsd/netinet/mptcp.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2014 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -33,6 +33,8 @@ #include +#include + #if BYTE_ORDER == BIG_ENDIAN #define mptcp_hton64(x) (x) #define mptcp_ntoh64(x) (x) @@ -135,7 +137,7 @@ struct mptcp_mpjoin_opt_rsp2 { mmjo_reserved1:4; #endif u_int8_t mmjo_reserved2; - u_int8_t mmjo_mac[20]; /* This is 160 bits HMAC SHA-1 per RFC */ + u_int8_t mmjo_mac[SHA1_RESULTLEN]; /* This is 160 bits HMAC SHA-1 per RFC */ } __attribute__((__packed__)); diff --git a/bsd/netinet/mptcp_opt.c b/bsd/netinet/mptcp_opt.c index d2aec1750..8aa8a9a4e 100644 --- a/bsd/netinet/mptcp_opt.c +++ b/bsd/netinet/mptcp_opt.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -53,29 +53,22 @@ #include -/* - * SYSCTL for enforcing 64 bit dsn - */ -int32_t force_64bit_dsn = 0; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, force_64bit_dsn, - CTLFLAG_RW|CTLFLAG_LOCKED, &force_64bit_dsn, 0, - "Force MPTCP 64bit dsn"); - - static int mptcp_validate_join_hmac(struct tcpcb *, u_char*, int); static int mptcp_snd_mpprio(struct tcpcb *tp, u_char *cp, int optlen); +static void mptcp_send_remaddr_opt(struct tcpcb *, struct mptcp_remaddr_opt *); /* * MPTCP Options Output Processing */ static unsigned -mptcp_setup_first_subflow_syn_opts(struct socket *so, int flags, u_char *opt, - unsigned optlen) +mptcp_setup_first_subflow_syn_opts(struct socket *so, u_char *opt, unsigned optlen) { + struct mptcp_mpcapable_opt_common mptcp_opt; struct tcpcb *tp = sototcpcb(so); - struct mptcb *mp_tp = NULL; - mp_tp = tptomptp(tp); + struct mptcb *mp_tp = tptomptp(tp); + + mpte_lock_assert_held(mp_tp->mpt_mpte); /* * Avoid retransmitting the MP_CAPABLE option. @@ -93,72 +86,32 @@ mptcp_setup_first_subflow_syn_opts(struct socket *so, int flags, u_char *opt, return (optlen); } - if ((flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) { - struct mptcp_mpcapable_opt_rsp mptcp_opt; - mptcp_key_t mp_localkey = 0; + bzero(&mptcp_opt, sizeof (struct mptcp_mpcapable_opt_common)); - mp_localkey = mptcp_get_localkey(mp_tp); - if (mp_localkey == 0) { - /* an embryonic connection was closed from above */ - return (optlen); - } - bzero(&mptcp_opt, - sizeof (struct mptcp_mpcapable_opt_rsp)); - mptcp_opt.mmc_common.mmco_kind = TCPOPT_MULTIPATH; - mptcp_opt.mmc_common.mmco_len = - sizeof (struct mptcp_mpcapable_opt_rsp); - mptcp_opt.mmc_common.mmco_subtype = MPO_CAPABLE; - MPT_LOCK_SPIN(mp_tp); - mptcp_opt.mmc_common.mmco_version = mp_tp->mpt_version; - mptcp_opt.mmc_common.mmco_flags |= MPCAP_PROPOSAL_SBIT; - if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) - mptcp_opt.mmc_common.mmco_flags |= - MPCAP_CHECKSUM_CBIT; - MPT_UNLOCK(mp_tp); - mptcp_opt.mmc_localkey = mp_localkey; - memcpy(opt + optlen, &mptcp_opt, - mptcp_opt.mmc_common.mmco_len); - optlen += mptcp_opt.mmc_common.mmco_len; - } else { - /* Only the SYN flag is set */ - struct mptcp_mpcapable_opt_common mptcp_opt; - mptcp_key_t mp_localkey = 0; - mp_localkey = mptcp_get_localkey(mp_tp); - so->so_flags |= SOF_MPTCP_CLIENT; - if (mp_localkey == 0) { - /* an embryonic connection was closed */ - return (optlen); - } - bzero(&mptcp_opt, - sizeof (struct mptcp_mpcapable_opt_common)); - mptcp_opt.mmco_kind = TCPOPT_MULTIPATH; - mptcp_opt.mmco_len = - sizeof (struct mptcp_mpcapable_opt_common) + - sizeof (mptcp_key_t); - mptcp_opt.mmco_subtype = MPO_CAPABLE; - MPT_LOCK_SPIN(mp_tp); - mptcp_opt.mmco_version = mp_tp->mpt_version; - mptcp_opt.mmco_flags |= MPCAP_PROPOSAL_SBIT; - if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) - mptcp_opt.mmco_flags |= MPCAP_CHECKSUM_CBIT; - MPT_UNLOCK(mp_tp); - (void) memcpy(opt + optlen, &mptcp_opt, - sizeof (struct mptcp_mpcapable_opt_common)); - optlen += sizeof (struct mptcp_mpcapable_opt_common); - (void) memcpy(opt + optlen, &mp_localkey, - sizeof (mptcp_key_t)); - optlen += sizeof (mptcp_key_t); - } + mptcp_opt.mmco_kind = TCPOPT_MULTIPATH; + mptcp_opt.mmco_len = + sizeof (struct mptcp_mpcapable_opt_common) + + sizeof (mptcp_key_t); + mptcp_opt.mmco_subtype = MPO_CAPABLE; + mptcp_opt.mmco_version = mp_tp->mpt_version; + mptcp_opt.mmco_flags |= MPCAP_PROPOSAL_SBIT; + if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) + mptcp_opt.mmco_flags |= MPCAP_CHECKSUM_CBIT; + memcpy(opt + optlen, &mptcp_opt, sizeof (struct mptcp_mpcapable_opt_common)); + optlen += sizeof (struct mptcp_mpcapable_opt_common); + memcpy(opt + optlen, &mp_tp->mpt_localkey, sizeof (mptcp_key_t)); + optlen += sizeof (mptcp_key_t); return (optlen); } static unsigned -mptcp_setup_join_subflow_syn_opts(struct socket *so, int flags, u_char *opt, - unsigned optlen) +mptcp_setup_join_subflow_syn_opts(struct socket *so, u_char *opt, unsigned optlen) { + struct mptcp_mpjoin_opt_req mpjoin_req; struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = NULL; + struct mptsub *mpts; if (!inp) return (optlen); @@ -167,61 +120,37 @@ mptcp_setup_join_subflow_syn_opts(struct socket *so, int flags, u_char *opt, if (!tp) return (optlen); - if (!tp->t_mptcb) - return (optlen); + mpts = tp->t_mpsub; - if ((flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) { - struct mptcp_mpjoin_opt_rsp mpjoin_rsp; - struct mptcb *mp_tp = tptomptp(tp); + VERIFY(tptomptp(tp)); + mpte_lock_assert_held(tptomptp(tp)->mpt_mpte); - if (mp_tp == NULL) - return (optlen); + bzero(&mpjoin_req, sizeof (mpjoin_req)); + mpjoin_req.mmjo_kind = TCPOPT_MULTIPATH; + mpjoin_req.mmjo_len = sizeof (mpjoin_req); + mpjoin_req.mmjo_subtype_bkp = MPO_JOIN << 4; - MPT_LOCK(mp_tp); - if (mptcp_get_localkey(mp_tp) == 0) { - MPT_UNLOCK(mp_tp); - return (optlen); - } - MPT_UNLOCK(mp_tp); - bzero(&mpjoin_rsp, sizeof (mpjoin_rsp)); - mpjoin_rsp.mmjo_kind = TCPOPT_MULTIPATH; - mpjoin_rsp.mmjo_len = sizeof (mpjoin_rsp); - mpjoin_rsp.mmjo_subtype_bkp = MPO_JOIN << 4; - if (tp->t_mpflags & TMPF_BACKUP_PATH) - mpjoin_rsp.mmjo_subtype_bkp |= MPTCP_BACKUP; - mpjoin_rsp.mmjo_addr_id = tp->t_local_aid; - mptcp_get_rands(tp->t_local_aid, tptomptp(tp), - &mpjoin_rsp.mmjo_rand, NULL); - mpjoin_rsp.mmjo_mac = mptcp_get_trunced_hmac(tp->t_local_aid, - mp_tp); - memcpy(opt + optlen, &mpjoin_rsp, mpjoin_rsp.mmjo_len); - optlen += mpjoin_rsp.mmjo_len; + if (tp->t_mpflags & TMPF_BACKUP_PATH) { + mpjoin_req.mmjo_subtype_bkp |= MPTCP_BACKUP; + } else if (inp->inp_boundifp && IFNET_IS_CELLULAR(inp->inp_boundifp) && + mpts->mpts_mpte->mpte_svctype != MPTCP_SVCTYPE_AGGREGATE) { + mpjoin_req.mmjo_subtype_bkp |= MPTCP_BACKUP; + tp->t_mpflags |= TMPF_BACKUP_PATH; } else { - struct mptcp_mpjoin_opt_req mpjoin_req; - - bzero(&mpjoin_req, sizeof (mpjoin_req)); - mpjoin_req.mmjo_kind = TCPOPT_MULTIPATH; - mpjoin_req.mmjo_len = sizeof (mpjoin_req); - mpjoin_req.mmjo_subtype_bkp = MPO_JOIN << 4; - if (tp->t_mpflags & TMPF_BACKUP_PATH) - mpjoin_req.mmjo_subtype_bkp |= MPTCP_BACKUP; - mpjoin_req.mmjo_addr_id = tp->t_local_aid; - mpjoin_req.mmjo_peer_token = mptcp_get_remotetoken(tp->t_mptcb); - if (mpjoin_req.mmjo_peer_token == 0) { - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: peer token 0", - __func__), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); - } - mptcp_get_rands(tp->t_local_aid, tptomptp(tp), - &mpjoin_req.mmjo_rand, NULL); - memcpy(opt + optlen, &mpjoin_req, mpjoin_req.mmjo_len); - optlen += mpjoin_req.mmjo_len; - /* send an event up, if Fast Join is requested */ - if (mptcp_zerortt_fastjoin && - (so->so_flags & SOF_MPTCP_FASTJOIN)) { - soevent(so, (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MPFASTJ)); - } + mpts->mpts_flags |= MPTSF_PREFERRED; + } + + mpjoin_req.mmjo_addr_id = tp->t_local_aid; + mpjoin_req.mmjo_peer_token = tptomptp(tp)->mpt_remotetoken; + if (mpjoin_req.mmjo_peer_token == 0) { + mptcplog((LOG_DEBUG, "%s: peer token 0", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); } + mptcp_get_rands(tp->t_local_aid, tptomptp(tp), + &mpjoin_req.mmjo_rand, NULL); + memcpy(opt + optlen, &mpjoin_req, mpjoin_req.mmjo_len); + optlen += mpjoin_req.mmjo_len; + return (optlen); } @@ -241,30 +170,22 @@ mptcp_setup_join_ack_opts(struct tcpcb *tp, u_char *opt, unsigned optlen) join_rsp2.mmjo_len = sizeof (struct mptcp_mpjoin_opt_rsp2); join_rsp2.mmjo_subtype = MPO_JOIN; mptcp_get_hmac(tp->t_local_aid, tptomptp(tp), - (u_char*)&join_rsp2.mmjo_mac, - sizeof (join_rsp2.mmjo_mac)); + (u_char*)&join_rsp2.mmjo_mac); memcpy(opt + optlen, &join_rsp2, join_rsp2.mmjo_len); new_optlen = optlen + join_rsp2.mmjo_len; - tp->t_mpflags |= TMPF_FASTJOINBY2_SEND; return (new_optlen); } unsigned -mptcp_setup_syn_opts(struct socket *so, int flags, u_char *opt, unsigned optlen) +mptcp_setup_syn_opts(struct socket *so, u_char *opt, unsigned optlen) { unsigned new_optlen; - if (!(so->so_flags & SOF_MP_SEC_SUBFLOW)) { - new_optlen = mptcp_setup_first_subflow_syn_opts(so, flags, opt, - optlen); - } else { - /* - * To simulate SYN_ACK with no join opt, comment this line on - * OS X server side. This serves as a testing hook. - */ - new_optlen = mptcp_setup_join_subflow_syn_opts(so, flags, opt, - optlen); - } + if (!(so->so_flags & SOF_MP_SEC_SUBFLOW)) + new_optlen = mptcp_setup_first_subflow_syn_opts(so, opt, optlen); + else + new_optlen = mptcp_setup_join_subflow_syn_opts(so, opt, optlen); + return (new_optlen); } @@ -284,15 +205,15 @@ mptcp_send_mpfail(struct tcpcb *tp, u_char *opt, unsigned int optlen) return (optlen); } + mpte_lock_assert_held(mp_tp->mpt_mpte); + /* if option space low give up */ if ((MAX_TCPOPTLEN - optlen) < sizeof (struct mptcp_mpfail_opt)) { tp->t_mpflags &= ~TMPF_SND_MPFAIL; return (optlen); } - MPT_LOCK(mp_tp); dsn = mp_tp->mpt_rcvnxt; - MPT_UNLOCK(mp_tp); bzero(&fail_opt, sizeof (fail_opt)); fail_opt.mfail_kind = TCPOPT_MULTIPATH; @@ -302,7 +223,7 @@ mptcp_send_mpfail(struct tcpcb *tp, u_char *opt, unsigned int optlen) memcpy(opt + optlen, &fail_opt, len); optlen += len; tp->t_mpflags &= ~TMPF_SND_MPFAIL; - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: %d \n", __func__, + mptcplog((LOG_DEBUG, "%s: %d \n", __func__, tp->t_local_aid), (MPTCP_SOCKET_DBG | MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); return (optlen); @@ -315,7 +236,6 @@ mptcp_send_infinite_mapping(struct tcpcb *tp, u_char *opt, unsigned int optlen) struct mptcb *mp_tp = NULL; size_t len = sizeof (struct mptcp_dsn_opt); struct socket *so = tp->t_inpcb->inp_socket; - int error = 0; int csum_len = 0; if (!so) @@ -325,15 +245,15 @@ mptcp_send_infinite_mapping(struct tcpcb *tp, u_char *opt, unsigned int optlen) if (mp_tp == NULL) return (optlen); - MPT_LOCK(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) csum_len = 2; /* try later */ - if ((MAX_TCPOPTLEN - optlen) < (len + csum_len)) { - MPT_UNLOCK(mp_tp); + if ((MAX_TCPOPTLEN - optlen) < (len + csum_len)) return (optlen); - } + bzero(&infin_opt, sizeof (infin_opt)); infin_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; infin_opt.mdss_copt.mdss_len = len + csum_len; @@ -352,20 +272,18 @@ mptcp_send_infinite_mapping(struct tcpcb *tp, u_char *opt, unsigned int optlen) ((mp_tp->mpt_local_idsn + 1) == mp_tp->mpt_snduna)) { infin_opt.mdss_subflow_seqn = 1; - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: idsn %llu" - "snduna %llu \n", __func__, mp_tp->mpt_local_idsn, + mptcplog((LOG_DEBUG, "%s: idsn %llu snduna %llu \n", + __func__, mp_tp->mpt_local_idsn, mp_tp->mpt_snduna), (MPTCP_SOCKET_DBG | MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); } else { - infin_opt.mdss_subflow_seqn = tp->snd_una - tp->iss; + infin_opt.mdss_subflow_seqn = tp->snd_una - tp->t_mpsub->mpts_iss; } infin_opt.mdss_dsn = (u_int32_t) MPTCP_DATASEQ_LOW32(mp_tp->mpt_snduna); } - MPT_UNLOCK(mp_tp); - if (error != 0) - return (optlen); + if ((infin_opt.mdss_dsn == 0) || (infin_opt.mdss_subflow_seqn == 0)) { return (optlen); } @@ -382,15 +300,13 @@ mptcp_send_infinite_mapping(struct tcpcb *tp, u_char *opt, unsigned int optlen) optlen += csum_len; } - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: dsn = %x, seq = %x len = %x\n", - __func__, + mptcplog((LOG_DEBUG, "%s: dsn = %x, seq = %x len = %x\n", __func__, ntohl(infin_opt.mdss_dsn), ntohl(infin_opt.mdss_subflow_seqn), ntohs(infin_opt.mdss_data_len)), (MPTCP_SOCKET_DBG | MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); - /* so->so_flags &= ~SOF_MPTCP_CLIENT; */ tp->t_mpflags |= TMPF_INFIN_SENT; tcpstat.tcps_estab_fallback++; return (optlen); @@ -400,24 +316,21 @@ mptcp_send_infinite_mapping(struct tcpcb *tp, u_char *opt, unsigned int optlen) static int mptcp_ok_to_fin(struct tcpcb *tp, u_int64_t dsn, u_int32_t datalen) { - struct mptcb *mp_tp = NULL; - mp_tp = tptomptp(tp); + struct mptcb *mp_tp = tptomptp(tp); + + mpte_lock_assert_held(mp_tp->mpt_mpte); - MPT_LOCK(mp_tp); dsn = (mp_tp->mpt_sndmax & MPTCP_DATASEQ_LOW32_MASK) | dsn; - if ((dsn + datalen) == mp_tp->mpt_sndmax) { - MPT_UNLOCK(mp_tp); + if ((dsn + datalen) == mp_tp->mpt_sndmax) return (1); - } - MPT_UNLOCK(mp_tp); + return (0); } unsigned int mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, - unsigned int optlen, int flags, int datalen, - unsigned int **dss_lenp, u_int8_t **finp, u_int64_t *dss_valp, - u_int32_t **sseqp, boolean_t *p_mptcp_acknow) + unsigned int optlen, int flags, int len, + boolean_t *p_mptcp_acknow) { struct inpcb *inp = (struct inpcb *)tp->t_inpcb; struct socket *so = inp->inp_socket; @@ -425,17 +338,15 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, boolean_t do_csum = FALSE; boolean_t send_64bit_dsn = FALSE; boolean_t send_64bit_ack = FALSE; - u_int32_t old_mpt_flags = tp->t_mpflags & - (TMPF_SND_MPPRIO | TMPF_SND_REM_ADDR | TMPF_SND_MPFAIL); + u_int32_t old_mpt_flags = tp->t_mpflags & TMPF_MPTCP_SIGNALS; - if ((mptcp_enable == 0) || - (mp_tp == NULL) || - (mp_tp->mpt_flags & MPTCPF_PEEL_OFF) || - (tp->t_state == TCPS_CLOSED)) { + if (mptcp_enable == 0 || mp_tp == NULL || tp->t_state == TCPS_CLOSED) { /* do nothing */ goto ret_optlen; } + mpte_lock_assert_held(mp_tp->mpt_mpte); + if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) { do_csum = TRUE; } @@ -447,11 +358,9 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, if ((MAX_TCPOPTLEN - optlen) < sizeof (struct mptcp_mpcapable_opt_common)) { - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: no space left %d flags %x " - "tp->t_mpflags %x " - "len %d\n", __func__, optlen, flags, tp->t_mpflags, - datalen), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + mptcplog((LOG_ERR, "%s: no space left %d flags %x tp->t_mpflags %x len %d\n", + __func__, optlen, flags, tp->t_mpflags, len), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); goto ret_optlen; } @@ -463,17 +372,7 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, goto ret_optlen; } - if (((tp->t_mpflags & TMPF_FASTJOINBY2_SEND) || - (tp->t_mpflags & TMPF_FASTJOIN_SEND )) && - (datalen > 0)) { - tp->t_mpflags &= ~TMPF_FASTJOINBY2_SEND; - tp->t_mpflags &= ~TMPF_FASTJOIN_SEND; - goto fastjoin_send; - } - - if (((tp->t_mpflags & TMPF_PREESTABLISHED) && - (!(tp->t_mpflags & TMPF_SENT_KEYS)) && - (!(tp->t_mpflags & TMPF_JOINED_FLOW)))) { + if (tp->t_mpflags & TMPF_SND_KEYS) { struct mptcp_mpcapable_opt_rsp1 mptcp_opt; if ((MAX_TCPOPTLEN - optlen) < sizeof (struct mptcp_mpcapable_opt_rsp1)) @@ -486,17 +385,13 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, mptcp_opt.mmc_common.mmco_version = mp_tp->mpt_version; /* HMAC-SHA1 is the proposal */ mptcp_opt.mmc_common.mmco_flags |= MPCAP_PROPOSAL_SBIT; - MPT_LOCK(mp_tp); if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) mptcp_opt.mmc_common.mmco_flags |= MPCAP_CHECKSUM_CBIT; - mptcp_opt.mmc_localkey = mptcp_get_localkey(mp_tp); - mptcp_opt.mmc_remotekey = mptcp_get_remotekey(mp_tp); - MPT_UNLOCK(mp_tp); + mptcp_opt.mmc_localkey = mp_tp->mpt_localkey; + mptcp_opt.mmc_remotekey = mp_tp->mpt_remotekey; memcpy(opt + optlen, &mptcp_opt, mptcp_opt.mmc_common.mmco_len); optlen += mptcp_opt.mmc_common.mmco_len; - tp->t_mpflags |= TMPF_SENT_KEYS | TMPF_MPTCP_TRUE; - so->so_flags |= SOF_MPTCP_TRUE; - tp->t_mpflags &= ~TMPF_PREESTABLISHED; + tp->t_mpflags &= ~TMPF_SND_KEYS; if (!tp->t_mpuna) { tp->t_mpuna = tp->snd_una; @@ -506,17 +401,7 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, goto ret_optlen; } - if ((tp->t_mpflags & TMPF_JOINED_FLOW) && - (tp->t_mpflags & TMPF_PREESTABLISHED) && - (!(tp->t_mpflags & TMPF_RECVD_JOIN)) && - (tp->t_mpflags & TMPF_SENT_JOIN) && - (!(tp->t_mpflags & TMPF_MPTCP_TRUE))) { - MPT_LOCK(mp_tp); - if (mptcp_get_localkey(mp_tp) == 0) { - MPT_UNLOCK(mp_tp); - goto ret_optlen; - } - MPT_UNLOCK(mp_tp); + if (tp->t_mpflags & TMPF_SND_JACK) { /* Do the ACK part */ optlen = mptcp_setup_join_ack_opts(tp, opt, optlen); if (!tp->t_mpuna) { @@ -525,12 +410,13 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, /* Start a timer to retransmit the ACK */ tp->t_timer[TCPT_JACK_RXMT] = OFFSET_FROM_START(tp, tcp_jack_rxmt); + + tp->t_mpflags &= ~TMPF_SND_JACK; goto ret_optlen; } if (!(tp->t_mpflags & TMPF_MPTCP_TRUE)) goto ret_optlen; -fastjoin_send: /* * From here on, all options are sent only if MPTCP_TRUE * or when data is sent early on as in Fast Join @@ -552,45 +438,39 @@ fastjoin_send: optlen = mptcp_snd_mpprio(tp, opt, optlen); } - MPT_LOCK(mp_tp); - if ((mp_tp->mpt_flags & MPTCPF_SND_64BITDSN) || force_64bit_dsn) { + if (mp_tp->mpt_flags & MPTCPF_SND_64BITDSN) { send_64bit_dsn = TRUE; } if (mp_tp->mpt_flags & MPTCPF_SND_64BITACK) send_64bit_ack = TRUE; - MPT_UNLOCK(mp_tp); - -#define CHECK_OPTLEN { \ - if ((MAX_TCPOPTLEN - optlen) < len) { \ - mptcplog((LOG_ERR, "MPTCP Socket: " \ - "%s: len %d optlen %d \n", __func__, len, optlen), \ - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); \ - goto ret_optlen; \ - } \ +#define CHECK_OPTLEN { \ + if ((MAX_TCPOPTLEN - optlen) < dssoptlen) { \ + mptcplog((LOG_ERR, "%s: dssoptlen %d optlen %d \n", __func__, \ + dssoptlen, optlen), \ + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); \ + goto ret_optlen; \ + } \ } #define DO_FIN(dsn_opt) { \ int sndfin = 0; \ - sndfin = mptcp_ok_to_fin(tp, dsn_opt.mdss_dsn, datalen); \ + sndfin = mptcp_ok_to_fin(tp, dsn_opt.mdss_dsn, len); \ if (sndfin) { \ dsn_opt.mdss_copt.mdss_flags |= MDSS_F; \ - *finp = opt + optlen + offsetof(struct mptcp_dss_copt, \ - mdss_flags); \ dsn_opt.mdss_data_len += 1; \ } \ } #define CHECK_DATALEN { \ /* MPTCP socket does not support IP options */ \ - if ((datalen + optlen + len) > tp->t_maxopd) { \ - mptcplog((LOG_ERR, "MPTCP Socket: " \ - "%s: nosp %d len %d opt %d %d %d\n", \ - __func__, datalen, len, optlen, \ + if ((len + optlen + dssoptlen) > tp->t_maxopd) { \ + mptcplog((LOG_ERR, "%s: nosp %d len %d opt %d %d %d\n", \ + __func__, len, dssoptlen, optlen, \ tp->t_maxseg, tp->t_maxopd), \ MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); \ /* remove option length from payload len */ \ - datalen = tp->t_maxopd - optlen - len; \ + len = tp->t_maxopd - optlen - dssoptlen; \ } \ } @@ -606,10 +486,11 @@ fastjoin_send: * XXX If this delay causes issue, remove the 2-byte padding. */ struct mptcp_dss64_ack32_opt dsn_ack_opt; - unsigned int len = sizeof (dsn_ack_opt); + unsigned int dssoptlen = sizeof (dsn_ack_opt); + uint16_t dss_csum; if (do_csum) { - len += 2; + dssoptlen += 2; } CHECK_OPTLEN; @@ -617,18 +498,17 @@ fastjoin_send: bzero(&dsn_ack_opt, sizeof (dsn_ack_opt)); dsn_ack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; dsn_ack_opt.mdss_copt.mdss_subtype = MPO_DSS; - dsn_ack_opt.mdss_copt.mdss_len = len; + dsn_ack_opt.mdss_copt.mdss_len = dssoptlen; dsn_ack_opt.mdss_copt.mdss_flags |= MDSS_M | MDSS_m | MDSS_A; CHECK_DATALEN; - mptcp_output_getm_dsnmap64(so, off, (u_int32_t)datalen, - &dsn_ack_opt.mdss_dsn, - &dsn_ack_opt.mdss_subflow_seqn, - &dsn_ack_opt.mdss_data_len); - - *dss_valp = dsn_ack_opt.mdss_dsn; + mptcp_output_getm_dsnmap64(so, off, + &dsn_ack_opt.mdss_dsn, + &dsn_ack_opt.mdss_subflow_seqn, + &dsn_ack_opt.mdss_data_len, + &dss_csum); if ((dsn_ack_opt.mdss_data_len == 0) || (dsn_ack_opt.mdss_dsn == 0)) { @@ -639,30 +519,21 @@ fastjoin_send: DO_FIN(dsn_ack_opt); } - MPT_LOCK(mp_tp); dsn_ack_opt.mdss_ack = htonl(MPTCP_DATAACK_LOW32(mp_tp->mpt_rcvnxt)); - MPT_UNLOCK(mp_tp); dsn_ack_opt.mdss_dsn = mptcp_hton64(dsn_ack_opt.mdss_dsn); dsn_ack_opt.mdss_subflow_seqn = htonl( dsn_ack_opt.mdss_subflow_seqn); dsn_ack_opt.mdss_data_len = htons( dsn_ack_opt.mdss_data_len); - *dss_lenp = (unsigned int *)(void *)(opt + optlen + - offsetof(struct mptcp_dss64_ack32_opt, mdss_data_len)); memcpy(opt + optlen, &dsn_ack_opt, sizeof (dsn_ack_opt)); + if (do_csum) + *((uint16_t *)(void *)(opt + optlen + sizeof (dsn_ack_opt))) = dss_csum; - if (do_csum) { - *sseqp = (u_int32_t *)(void *)(opt + optlen + - offsetof(struct mptcp_dss64_ack32_opt, - mdss_subflow_seqn)); - } - optlen += len; - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: long DSS = %llx ACK = %llx \n", - __func__, + optlen += dssoptlen; + mptcplog((LOG_DEBUG,"%s: long DSS = %llx ACK = %llx \n", __func__, mptcp_ntoh64(dsn_ack_opt.mdss_dsn), mptcp_ntoh64(dsn_ack_opt.mdss_ack)), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); @@ -675,10 +546,11 @@ fastjoin_send: (!send_64bit_dsn) && !(tp->t_mpflags & TMPF_MPTCP_ACKNOW)) { struct mptcp_dsn_opt dsn_opt; - unsigned int len = sizeof (struct mptcp_dsn_opt); + unsigned int dssoptlen = sizeof (struct mptcp_dsn_opt); + uint16_t dss_csum; if (do_csum) { - len += 2; + dssoptlen += 2; } CHECK_OPTLEN; @@ -686,15 +558,15 @@ fastjoin_send: bzero(&dsn_opt, sizeof (dsn_opt)); dsn_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; dsn_opt.mdss_copt.mdss_subtype = MPO_DSS; - dsn_opt.mdss_copt.mdss_len = len; + dsn_opt.mdss_copt.mdss_len = dssoptlen; dsn_opt.mdss_copt.mdss_flags |= MDSS_M; CHECK_DATALEN; - mptcp_output_getm_dsnmap32(so, off, (u_int32_t)datalen, - &dsn_opt.mdss_dsn, - &dsn_opt.mdss_subflow_seqn, &dsn_opt.mdss_data_len, - dss_valp); + mptcp_output_getm_dsnmap32(so, off, &dsn_opt.mdss_dsn, + &dsn_opt.mdss_subflow_seqn, + &dsn_opt.mdss_data_len, + &dss_csum); if ((dsn_opt.mdss_data_len == 0) || (dsn_opt.mdss_dsn == 0)) { @@ -708,14 +580,11 @@ fastjoin_send: dsn_opt.mdss_dsn = htonl(dsn_opt.mdss_dsn); dsn_opt.mdss_subflow_seqn = htonl(dsn_opt.mdss_subflow_seqn); dsn_opt.mdss_data_len = htons(dsn_opt.mdss_data_len); - *dss_lenp = (unsigned int *)(void *)(opt + optlen + - offsetof(struct mptcp_dsn_opt, mdss_data_len)); memcpy(opt + optlen, &dsn_opt, sizeof (dsn_opt)); - if (do_csum) { - *sseqp = (u_int32_t *)(void *)(opt + optlen + - offsetof(struct mptcp_dsn_opt, mdss_subflow_seqn)); - } - optlen += len; + if (do_csum) + *((uint16_t *)(void *)(opt + optlen + sizeof (dsn_opt))) = dss_csum; + + optlen += dssoptlen; tp->t_mpflags &= ~TMPF_MPTCP_ACKNOW; goto ret_optlen; } @@ -727,23 +596,21 @@ fastjoin_send: !(tp->t_mpflags & TMPF_SEND_DFIN)) { struct mptcp_data_ack_opt dack_opt; - unsigned int len = 0; + unsigned int dssoptlen = 0; do_ack32_only: - len = sizeof (dack_opt); + dssoptlen = sizeof (dack_opt); CHECK_OPTLEN; - bzero(&dack_opt, len); + bzero(&dack_opt, dssoptlen); dack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; - dack_opt.mdss_copt.mdss_len = len; + dack_opt.mdss_copt.mdss_len = dssoptlen; dack_opt.mdss_copt.mdss_subtype = MPO_DSS; dack_opt.mdss_copt.mdss_flags |= MDSS_A; - MPT_LOCK_SPIN(mp_tp); dack_opt.mdss_ack = htonl(MPTCP_DATAACK_LOW32(mp_tp->mpt_rcvnxt)); - MPT_UNLOCK(mp_tp); - memcpy(opt + optlen, &dack_opt, len); - optlen += len; + memcpy(opt + optlen, &dack_opt, dssoptlen); + optlen += dssoptlen; VERIFY(optlen <= MAX_TCPOPTLEN); tp->t_mpflags &= ~TMPF_MPTCP_ACKNOW; goto ret_optlen; @@ -755,27 +622,25 @@ do_ack32_only: !(tp->t_mpflags & TMPF_SEND_DSN) && !(tp->t_mpflags & TMPF_SEND_DFIN)) { struct mptcp_data_ack64_opt dack_opt; - unsigned int len = 0; + unsigned int dssoptlen = 0; do_ack64_only: - len = sizeof (dack_opt); + dssoptlen = sizeof (dack_opt); CHECK_OPTLEN; - bzero(&dack_opt, len); + bzero(&dack_opt, dssoptlen); dack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; - dack_opt.mdss_copt.mdss_len = len; + dack_opt.mdss_copt.mdss_len = dssoptlen; dack_opt.mdss_copt.mdss_subtype = MPO_DSS; dack_opt.mdss_copt.mdss_flags |= (MDSS_A | MDSS_a); - MPT_LOCK_SPIN(mp_tp); dack_opt.mdss_ack = mptcp_hton64(mp_tp->mpt_rcvnxt); /* * The other end should retransmit 64-bit DSN until it * receives a 64-bit ACK. */ mp_tp->mpt_flags &= ~MPTCPF_SND_64BITACK; - MPT_UNLOCK(mp_tp); - memcpy(opt + optlen, &dack_opt, len); - optlen += len; + memcpy(opt + optlen, &dack_opt, dssoptlen); + optlen += dssoptlen; VERIFY(optlen <= MAX_TCPOPTLEN); tp->t_mpflags &= ~TMPF_MPTCP_ACKNOW; goto ret_optlen; @@ -787,30 +652,28 @@ do_ack64_only: (!send_64bit_ack) && (tp->t_mpflags & TMPF_MPTCP_ACKNOW)) { struct mptcp_dss_ack_opt dss_ack_opt; - unsigned int len = sizeof (dss_ack_opt); + unsigned int dssoptlen = sizeof (dss_ack_opt); + uint16_t dss_csum; if (do_csum) - len += 2; + dssoptlen += 2; CHECK_OPTLEN; bzero(&dss_ack_opt, sizeof (dss_ack_opt)); dss_ack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; - dss_ack_opt.mdss_copt.mdss_len = len; + dss_ack_opt.mdss_copt.mdss_len = dssoptlen; dss_ack_opt.mdss_copt.mdss_subtype = MPO_DSS; dss_ack_opt.mdss_copt.mdss_flags |= MDSS_A | MDSS_M; - MPT_LOCK_SPIN(mp_tp); dss_ack_opt.mdss_ack = htonl(MPTCP_DATAACK_LOW32(mp_tp->mpt_rcvnxt)); - MPT_UNLOCK(mp_tp); CHECK_DATALEN; - mptcp_output_getm_dsnmap32(so, off, (u_int32_t)datalen, - &dss_ack_opt.mdss_dsn, - &dss_ack_opt.mdss_subflow_seqn, - &dss_ack_opt.mdss_data_len, - dss_valp); + mptcp_output_getm_dsnmap32(so, off, &dss_ack_opt.mdss_dsn, + &dss_ack_opt.mdss_subflow_seqn, + &dss_ack_opt.mdss_data_len, + &dss_csum); if ((dss_ack_opt.mdss_data_len == 0) || (dss_ack_opt.mdss_dsn == 0)) { @@ -825,16 +688,11 @@ do_ack64_only: dss_ack_opt.mdss_subflow_seqn = htonl(dss_ack_opt.mdss_subflow_seqn); dss_ack_opt.mdss_data_len = htons(dss_ack_opt.mdss_data_len); - *dss_lenp = (unsigned int *)(void *)(opt + optlen + - offsetof(struct mptcp_dss_ack_opt, mdss_data_len)); memcpy(opt + optlen, &dss_ack_opt, sizeof (dss_ack_opt)); - if (do_csum) { - *sseqp = (u_int32_t *)(void *)(opt + optlen + - offsetof(struct mptcp_dss_ack_opt, - mdss_subflow_seqn)); - } + if (do_csum) + *((uint16_t *)(void *)(opt + optlen + sizeof (dss_ack_opt))) = dss_csum; - optlen += len; + optlen += dssoptlen; if (optlen > MAX_TCPOPTLEN) panic("optlen too large"); @@ -848,28 +706,28 @@ do_ack64_only: (send_64bit_ack) && (tp->t_mpflags & TMPF_MPTCP_ACKNOW)) { struct mptcp_dss32_ack64_opt dss_ack_opt; - unsigned int len = sizeof (dss_ack_opt); + unsigned int dssoptlen = sizeof (dss_ack_opt); + uint16_t dss_csum; if (do_csum) - len += 2; + dssoptlen += 2; CHECK_OPTLEN; bzero(&dss_ack_opt, sizeof (dss_ack_opt)); dss_ack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; - dss_ack_opt.mdss_copt.mdss_len = len; + dss_ack_opt.mdss_copt.mdss_len = dssoptlen; dss_ack_opt.mdss_copt.mdss_subtype = MPO_DSS; dss_ack_opt.mdss_copt.mdss_flags |= MDSS_M | MDSS_A | MDSS_a; - MPT_LOCK_SPIN(mp_tp); dss_ack_opt.mdss_ack = mptcp_hton64(mp_tp->mpt_rcvnxt); - MPT_UNLOCK(mp_tp); CHECK_DATALEN; - mptcp_output_getm_dsnmap32(so, off, (u_int32_t)datalen, - &dss_ack_opt.mdss_dsn, &dss_ack_opt.mdss_subflow_seqn, - &dss_ack_opt.mdss_data_len, dss_valp); + mptcp_output_getm_dsnmap32(so, off, &dss_ack_opt.mdss_dsn, + &dss_ack_opt.mdss_subflow_seqn, + &dss_ack_opt.mdss_data_len, + &dss_csum); if ((dss_ack_opt.mdss_data_len == 0) || (dss_ack_opt.mdss_dsn == 0)) { @@ -884,16 +742,11 @@ do_ack64_only: dss_ack_opt.mdss_subflow_seqn = htonl(dss_ack_opt.mdss_subflow_seqn); dss_ack_opt.mdss_data_len = htons(dss_ack_opt.mdss_data_len); - *dss_lenp = (unsigned int *)(void *)(opt + optlen + - offsetof(struct mptcp_dss32_ack64_opt, mdss_data_len)); memcpy(opt + optlen, &dss_ack_opt, sizeof (dss_ack_opt)); - if (do_csum) { - *sseqp = (u_int32_t *)(void *)(opt + optlen + - offsetof(struct mptcp_dss32_ack64_opt, - mdss_subflow_seqn)); - } + if (do_csum) + *((uint16_t *)(void *)(opt + optlen + sizeof (dss_ack_opt))) = dss_csum; - optlen += len; + optlen += dssoptlen; if (optlen > MAX_TCPOPTLEN) panic("optlen too large"); @@ -903,58 +756,48 @@ do_ack64_only: if (tp->t_mpflags & TMPF_SEND_DFIN) { struct mptcp_dss_ack_opt dss_ack_opt; - unsigned int len = sizeof (struct mptcp_dss_ack_opt); + unsigned int dssoptlen = sizeof (struct mptcp_dss_ack_opt); if (do_csum) - len += 2; + dssoptlen += 2; CHECK_OPTLEN; bzero(&dss_ack_opt, sizeof (dss_ack_opt)); - MPT_LOCK(mp_tp); /* * Data FIN occupies one sequence space. * Don't send it if it has been Acked. */ if (((mp_tp->mpt_sndnxt + 1) != mp_tp->mpt_sndmax) || - (mp_tp->mpt_snduna == mp_tp->mpt_sndmax)) { - MPT_UNLOCK(mp_tp); + (mp_tp->mpt_snduna == mp_tp->mpt_sndmax)) goto ret_optlen; - } dss_ack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; - dss_ack_opt.mdss_copt.mdss_len = len; + dss_ack_opt.mdss_copt.mdss_len = dssoptlen; dss_ack_opt.mdss_copt.mdss_subtype = MPO_DSS; dss_ack_opt.mdss_copt.mdss_flags |= MDSS_A | MDSS_M | MDSS_F; dss_ack_opt.mdss_ack = htonl(MPTCP_DATAACK_LOW32(mp_tp->mpt_rcvnxt)); dss_ack_opt.mdss_dsn = htonl(MPTCP_DATASEQ_LOW32(mp_tp->mpt_sndnxt)); - MPT_UNLOCK(mp_tp); dss_ack_opt.mdss_subflow_seqn = 0; dss_ack_opt.mdss_data_len = 1; dss_ack_opt.mdss_data_len = htons(dss_ack_opt.mdss_data_len); memcpy(opt + optlen, &dss_ack_opt, sizeof (dss_ack_opt)); - if (do_csum) { - *dss_valp = mp_tp->mpt_sndnxt; - *sseqp = (u_int32_t *)(void *)(opt + optlen + - offsetof(struct mptcp_dss_ack_opt, - mdss_subflow_seqn)); - } - optlen += len; + optlen += dssoptlen; } ret_optlen: if (TRUE == *p_mptcp_acknow ) { VERIFY(old_mpt_flags != 0); - u_int32_t new_mpt_flags = tp->t_mpflags & - (TMPF_SND_MPPRIO | TMPF_SND_REM_ADDR | TMPF_SND_MPFAIL); + u_int32_t new_mpt_flags = tp->t_mpflags & TMPF_MPTCP_SIGNALS; /* * If none of the above mpflags were acted on by * this routine, reset these flags and set p_mptcp_acknow * to false. + * * XXX The reset value of p_mptcp_acknow can be used * to communicate tcp_output to NOT send a pure ack without any * MPTCP options as it will be treated as a dup ack. @@ -965,16 +808,14 @@ ret_optlen: * we haven't modified the logic in tcp_output to avoid * that. */ - if ((old_mpt_flags == new_mpt_flags) || (new_mpt_flags == 0)) { - tp->t_mpflags &= ~(TMPF_SND_MPPRIO - | TMPF_SND_REM_ADDR | TMPF_SND_MPFAIL); + if (old_mpt_flags == new_mpt_flags) { + tp->t_mpflags &= ~TMPF_MPTCP_SIGNALS; *p_mptcp_acknow = FALSE; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s: no action \n", - __func__), MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: no action \n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); } else { - mptcplog((LOG_DEBUG, "MPTCP Sender: acknow set, " - "old flags %x new flags %x \n", - old_mpt_flags, new_mpt_flags), + mptcplog((LOG_DEBUG, "%s: acknow set, old flags %x new flags %x \n", + __func__, old_mpt_flags, new_mpt_flags), MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); } } @@ -993,8 +834,8 @@ mptcp_sanitize_option(struct tcpcb *tp, int mptcp_subtype) int ret = 1; if (mp_tp == NULL) { - mptcplog((LOG_ERR, "MPTCP Socket: %s: NULL mpsocket \n", - __func__), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + mptcplog((LOG_ERR, "%s: NULL mpsocket \n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); return (0); } @@ -1013,8 +854,7 @@ mptcp_sanitize_option(struct tcpcb *tp, int mptcp_subtype) break; default: ret = 0; - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: type = %d \n", __func__, + mptcplog((LOG_ERR, "%s: type = %d \n", __func__, mptcp_subtype), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); break; @@ -1048,6 +888,8 @@ mptcp_do_mpcapable_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, struct mptcp_mpcapable_opt_rsp *rsp = NULL; struct mptcb *mp_tp = tptomptp(tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + /* Only valid on SYN/ACK */ if ((th->th_flags & (TH_SYN | TH_ACK)) != (TH_SYN | TH_ACK)) return; @@ -1065,8 +907,7 @@ mptcp_do_mpcapable_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, /* A SYN/ACK contains peer's key and flags */ if (optlen != sizeof (struct mptcp_mpcapable_opt_rsp)) { /* complain */ - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: SYN_ACK optlen = %d, sizeof mp opt = %lu \n", + mptcplog((LOG_ERR, "%s: SYN_ACK optlen = %d, sizeof mp opt = %lu \n", __func__, optlen, sizeof (struct mptcp_mpcapable_opt_rsp)), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); @@ -1083,7 +924,6 @@ mptcp_do_mpcapable_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, mp_tp->mpt_flags |= MPTCPF_CHECKSUM; rsp = (struct mptcp_mpcapable_opt_rsp *)cp; - MPT_LOCK(mp_tp); mp_tp->mpt_remotekey = rsp->mmc_localkey; /* For now just downgrade to the peer's version */ mp_tp->mpt_peer_version = rsp->mmc_common.mmco_version; @@ -1093,12 +933,11 @@ mptcp_do_mpcapable_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, } if (mptcp_init_remote_parms(mp_tp) != 0) { tcpstat.tcps_invalid_mpcap++; - MPT_UNLOCK(mp_tp); return; } - MPT_UNLOCK(mp_tp); tcp_heuristic_mptcp_success(tp); - tp->t_mpflags |= TMPF_PREESTABLISHED; + tp->t_mpflags |= (TMPF_SND_KEYS | TMPF_MPTCP_TRUE); + tp->t_inpcb->inp_socket->so_flags |= SOF_MPTCP_TRUE; } @@ -1122,9 +961,8 @@ mptcp_do_mpjoin_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen) return; if (optlen != sizeof (struct mptcp_mpjoin_opt_rsp)) { - mptcplog((LOG_ERR, "MPTCP Socket: " - "SYN_ACK: unexpected optlen = %d mp " - "option = %lu\n", optlen, + mptcplog((LOG_ERR, "%s: SYN_ACK: unexpected optlen = %d mp " + "option = %lu\n", __func__, optlen, sizeof (struct mptcp_mpjoin_opt_rsp)), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); tp->t_mpflags &= ~TMPF_PREESTABLISHED; @@ -1138,77 +976,146 @@ mptcp_do_mpjoin_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen) error = mptcp_validate_join_hmac(tp, (u_char*)&join_rsp->mmjo_mac, SHA1_TRUNCATED); if (error) { - mptcplog((LOG_ERR, "MPTCP Socket: %s: " - "SYN_ACK error = %d \n", __func__, error), + mptcplog((LOG_ERR, "%s: SYN_ACK error = %d \n", __func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); tp->t_mpflags &= ~TMPF_PREESTABLISHED; /* send RST and close */ MPTCP_JOPT_ERROR_PATH(tp); return; } - tp->t_mpflags |= TMPF_SENT_JOIN; + tp->t_mpflags |= (TMPF_SENT_JOIN | TMPF_SND_JACK); } static int mptcp_validate_join_hmac(struct tcpcb *tp, u_char* hmac, int mac_len) { u_char digest[SHA1_RESULTLEN] = {0}; - struct mptcb *mp_tp = NULL; - mptcp_key_t rem_key, loc_key; + struct mptcb *mp_tp = tptomptp(tp); u_int32_t rem_rand, loc_rand; - mp_tp = tp->t_mptcb; + mpte_lock_assert_held(mp_tp->mpt_mpte); rem_rand = loc_rand = 0; - MPT_LOCK(mp_tp); - rem_key = mp_tp->mpt_remotekey; - - /* - * Can happen if the MPTCP-connection is about to be closed and we - * receive an MP_JOIN in-between the events are being handled by the - * worker thread. - */ - if (mp_tp->mpt_localkey == NULL) { - MPT_UNLOCK(mp_tp); - return (-1); - } - - loc_key = *mp_tp->mpt_localkey; - MPT_UNLOCK(mp_tp); - mptcp_get_rands(tp->t_local_aid, mp_tp, &loc_rand, &rem_rand); if ((rem_rand == 0) || (loc_rand == 0)) return (-1); - mptcp_hmac_sha1(rem_key, loc_key, rem_rand, loc_rand, - digest, sizeof (digest)); + mptcp_hmac_sha1(mp_tp->mpt_remotekey, mp_tp->mpt_localkey, rem_rand, loc_rand, + digest); if (bcmp(digest, hmac, mac_len) == 0) return (0); /* matches */ else { printf("%s: remote key %llx local key %llx remote rand %x " - "local rand %x \n", __func__, rem_key, loc_key, + "local rand %x \n", __func__, mp_tp->mpt_remotekey, mp_tp->mpt_localkey, rem_rand, loc_rand); return (-1); } } +/* + * Update the mptcb send state variables, but the actual sbdrop occurs + * in MPTCP layer + */ +void +mptcp_data_ack_rcvd(struct mptcb *mp_tp, struct tcpcb *tp, u_int64_t full_dack) +{ + u_int64_t acked = 0; + + acked = full_dack - mp_tp->mpt_snduna; + + if (acked) { + struct socket *mp_so = mptetoso(mp_tp->mpt_mpte); + + if (acked > mp_so->so_snd.sb_cc) { + if (acked > mp_so->so_snd.sb_cc + 1 || + mp_tp->mpt_state < MPTCPS_FIN_WAIT_1) + mptcplog((LOG_ERR, "%s: acked %u, sb_cc %u full %u suna %u state %u\n", + __func__, (uint32_t)acked, mp_so->so_snd.sb_cc, + (uint32_t)full_dack, (uint32_t)mp_tp->mpt_snduna, + mp_tp->mpt_state), + MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); + + sbdrop(&mp_so->so_snd, (int)mp_so->so_snd.sb_cc); + } else { + sbdrop(&mp_so->so_snd, acked); + } + + mp_tp->mpt_snduna += acked; + /* In degraded mode, we may get some Data ACKs */ + if ((tp->t_mpflags & TMPF_TCP_FALLBACK) && + !(mp_tp->mpt_flags & MPTCPF_POST_FALLBACK_SYNC) && + MPTCP_SEQ_GT(mp_tp->mpt_sndnxt, mp_tp->mpt_snduna)) { + /* bring back sndnxt to retransmit MPTCP data */ + mp_tp->mpt_sndnxt = mp_tp->mpt_dsn_at_csum_fail; + mp_tp->mpt_flags |= MPTCPF_POST_FALLBACK_SYNC; + tp->t_inpcb->inp_socket->so_flags1 |= + SOF1_POST_FALLBACK_SYNC; + } + + mptcp_clean_reinjectq(mp_tp->mpt_mpte); + + sowwakeup(mp_so); + } + if (full_dack == mp_tp->mpt_sndmax && + mp_tp->mpt_state >= MPTCPS_FIN_WAIT_1) { + mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_ACK); + tp->t_mpflags &= ~TMPF_SEND_DFIN; + } +} + +void +mptcp_update_window_fallback(struct tcpcb *tp) +{ + struct mptcb *mp_tp = tptomptp(tp); + + mpte_lock_assert_held(mp_tp->mpt_mpte); + + if (!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP)) + return; + + mptcplog((LOG_DEBUG, "%s: update window to %u\n", __func__, tp->snd_wnd), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + mp_tp->mpt_sndwnd = tp->snd_wnd; + mp_tp->mpt_sndwl1 = mp_tp->mpt_rcvnxt; + mp_tp->mpt_sndwl2 = mp_tp->mpt_snduna; + + sowwakeup(tp->t_inpcb->inp_socket); +} + +static void +mptcp_update_window(struct mptcb *mp_tp, u_int64_t ack, u_int64_t seq, + u_int32_t tiwin) +{ + /* Don't look at the window if there is no ACK flag */ + if ((SEQ_LT(mp_tp->mpt_sndwl1, seq) || + (mp_tp->mpt_sndwl1 == seq && (SEQ_LT(mp_tp->mpt_sndwl2, ack) || + (mp_tp->mpt_sndwl2 == ack && tiwin > mp_tp->mpt_sndwnd))))) { + mp_tp->mpt_sndwnd = tiwin; + mp_tp->mpt_sndwl1 = seq; + mp_tp->mpt_sndwl2 = ack; + + mptcplog((LOG_DEBUG, "%s: Updating window to %u\n", __func__, + mp_tp->mpt_sndwnd), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); + } +} + static void -mptcp_do_dss_opt_ack_meat(u_int64_t full_dack, struct tcpcb *tp) +mptcp_do_dss_opt_ack_meat(u_int64_t full_dack, u_int64_t full_dsn, + struct tcpcb *tp, u_int32_t tiwin) { struct mptcb *mp_tp = tptomptp(tp); int close_notify = 0; tp->t_mpflags |= TMPF_RCVD_DACK; - MPT_LOCK(mp_tp); if (MPTCP_SEQ_LEQ(full_dack, mp_tp->mpt_sndmax) && MPTCP_SEQ_GEQ(full_dack, mp_tp->mpt_snduna)) { mptcp_data_ack_rcvd(mp_tp, tp, full_dack); if (mp_tp->mpt_state > MPTCPS_FIN_WAIT_2) close_notify = 1; - MPT_UNLOCK(mp_tp); if (mp_tp->mpt_flags & MPTCPF_RCVD_64BITACK) { mp_tp->mpt_flags &= ~MPTCPF_RCVD_64BITACK; mp_tp->mpt_flags &= ~MPTCPF_SND_64BITDSN; @@ -1217,42 +1124,35 @@ mptcp_do_dss_opt_ack_meat(u_int64_t full_dack, struct tcpcb *tp) if (close_notify) mptcp_notify_close(tp->t_inpcb->inp_socket); } else { - MPT_UNLOCK(mp_tp); - mptcplog((LOG_ERR,"MPTCP Socket: " - "%s: unexpected dack %llx snduna %llx " - "sndmax %llx\n", __func__, full_dack, - mp_tp->mpt_snduna, mp_tp->mpt_sndmax), + mptcplog((LOG_ERR,"%s: unexpected dack %u snduna %u sndmax %u\n", __func__, + (u_int32_t)full_dack, (u_int32_t)mp_tp->mpt_snduna, + (u_int32_t)mp_tp->mpt_sndmax), (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), MPTCP_LOGLVL_LOG); } + + mptcp_update_window(mp_tp, full_dack, full_dsn, tiwin); } static void -mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) +mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp, struct tcphdr *th) { struct mptcp_dss_copt *dss_rsp = (struct mptcp_dss_copt *)cp; u_int64_t full_dack = 0; + u_int32_t tiwin = th->th_win << tp->snd_scale; struct mptcb *mp_tp = tptomptp(tp); int csum_len = 0; -#define MPTCP_DSS_OPT_SZ_CHK(len, expected_len) { \ - if (len != expected_len) { \ - mptcplog((LOG_ERR, "MPTCP Socket: " \ - "%s: bad len = %d dss: %x \n", __func__, \ - len, dss_rsp->mdss_flags), \ - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), \ - MPTCP_LOGLVL_LOG); \ - return; \ - } \ +#define MPTCP_DSS_OPT_SZ_CHK(len, expected_len) { \ + if (len != expected_len) { \ + mptcplog((LOG_ERR, "%s: bad len = %d dss: %x \n", __func__, \ + len, dss_rsp->mdss_flags), \ + (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), \ + MPTCP_LOGLVL_LOG); \ + return; \ + } \ } - /* - * mp_tp might become NULL after the call to mptcp_do_fin_opt(). - * Review after rdar://problem/24083886 - */ - if (!mp_tp) - return; - if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) csum_len = 2; @@ -1285,10 +1185,8 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) u_int32_t dack = dack_opt->mdss_ack; NTOHL(dack); - MPT_LOCK_SPIN(mp_tp); MPTCP_EXTEND_DSN(mp_tp->mpt_snduna, dack, full_dack); - MPT_UNLOCK(mp_tp); - mptcp_do_dss_opt_ack_meat(full_dack, tp); + mptcp_do_dss_opt_ack_meat(full_dack, mp_tp->mpt_sndwl1, tp, tiwin); break; } case (MDSS_M | MDSS_A): @@ -1296,23 +1194,31 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) /* 32-bit Data ACK + 32-bit DSS */ struct mptcp_dss_ack_opt *dss_ack_rsp; dss_ack_rsp = (struct mptcp_dss_ack_opt *)cp; + u_int64_t full_dsn; + uint16_t csum = 0; MPTCP_DSS_OPT_SZ_CHK(dss_ack_rsp->mdss_copt.mdss_len, sizeof (struct mptcp_dss_ack_opt) + csum_len); u_int32_t dack = dss_ack_rsp->mdss_ack; NTOHL(dack); - MPT_LOCK_SPIN(mp_tp); MPTCP_EXTEND_DSN(mp_tp->mpt_snduna, dack, full_dack); - MPT_UNLOCK(mp_tp); - mptcp_do_dss_opt_ack_meat(full_dack, tp); - if (csum_len == 0) - mptcp_update_rcv_state_f(dss_ack_rsp, tp, 0); - else - mptcp_update_rcv_state_f(dss_ack_rsp, tp, - *(uint16_t *)(void *)(cp + - (dss_ack_rsp->mdss_copt.mdss_len - - csum_len))); + + NTOHL(dss_ack_rsp->mdss_dsn); + NTOHL(dss_ack_rsp->mdss_subflow_seqn); + NTOHS(dss_ack_rsp->mdss_data_len); + MPTCP_EXTEND_DSN(mp_tp->mpt_rcvnxt, dss_ack_rsp->mdss_dsn, full_dsn); + + mptcp_do_dss_opt_ack_meat(full_dack, full_dsn, tp, tiwin); + + if (csum_len != 0) + csum = *(uint16_t *)(void *)(cp + (dss_ack_rsp->mdss_copt.mdss_len - csum_len)); + + mptcp_update_rcv_state_meat(mp_tp, tp, + full_dsn, + dss_ack_rsp->mdss_subflow_seqn, + dss_ack_rsp->mdss_data_len, + csum); break; } case (MDSS_M | MDSS_m): @@ -1321,33 +1227,24 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) struct mptcp_dsn64_opt *dsn64; dsn64 = (struct mptcp_dsn64_opt *)cp; u_int64_t full_dsn; + uint16_t csum = 0; MPTCP_DSS_OPT_SZ_CHK(dsn64->mdss_copt.mdss_len, sizeof (struct mptcp_dsn64_opt) + csum_len); - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: 64-bit M present.\n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - - MPT_LOCK_SPIN(mp_tp); mp_tp->mpt_flags |= MPTCPF_SND_64BITACK; - MPT_UNLOCK(mp_tp); full_dsn = mptcp_ntoh64(dsn64->mdss_dsn); NTOHL(dsn64->mdss_subflow_seqn); NTOHS(dsn64->mdss_data_len); - if (csum_len == 0) - mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, - dsn64->mdss_subflow_seqn, - dsn64->mdss_data_len, - 0); - else - mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, - dsn64->mdss_subflow_seqn, - dsn64->mdss_data_len, - *(uint16_t *)(void *)(cp + - dsn64->mdss_copt.mdss_len - csum_len)); + + if (csum_len != 0) + csum = *(uint16_t *)(void *)(cp + dsn64->mdss_copt.mdss_len - csum_len); + + mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, + dsn64->mdss_subflow_seqn, + dsn64->mdss_data_len, + csum); break; } case (MDSS_A | MDSS_a): @@ -1359,17 +1256,10 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) MPTCP_DSS_OPT_SZ_CHK(dack64->mdss_copt.mdss_len, sizeof (struct mptcp_data_ack64_opt)); - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: 64-bit A present. \n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - - MPT_LOCK_SPIN(mp_tp); mp_tp->mpt_flags |= MPTCPF_RCVD_64BITACK; - MPT_UNLOCK(mp_tp); full_dack = mptcp_ntoh64(dack64->mdss_ack); - mptcp_do_dss_opt_ack_meat(full_dack, tp); + mptcp_do_dss_opt_ack_meat(full_dack, mp_tp->mpt_sndwl1, tp, tiwin); break; } case (MDSS_M | MDSS_m | MDSS_A): @@ -1377,29 +1267,31 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) /* 64-bit DSS + 32-bit Data ACK */ struct mptcp_dss64_ack32_opt *dss_ack_rsp; dss_ack_rsp = (struct mptcp_dss64_ack32_opt *)cp; + u_int64_t full_dsn; + uint16_t csum = 0; MPTCP_DSS_OPT_SZ_CHK(dss_ack_rsp->mdss_copt.mdss_len, sizeof (struct mptcp_dss64_ack32_opt) + csum_len); - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: 64-bit M and 32-bit A present.\n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - u_int32_t dack = dss_ack_rsp->mdss_ack; NTOHL(dack); - MPT_LOCK_SPIN(mp_tp); mp_tp->mpt_flags |= MPTCPF_SND_64BITACK; MPTCP_EXTEND_DSN(mp_tp->mpt_snduna, dack, full_dack); - MPT_UNLOCK(mp_tp); - mptcp_do_dss_opt_ack_meat(full_dack, tp); - if (csum_len == 0) - mptcp_update_rcv_state_g(dss_ack_rsp, tp, 0); - else - mptcp_update_rcv_state_g(dss_ack_rsp, tp, - *(uint16_t *)(void *)(cp + - dss_ack_rsp->mdss_copt.mdss_len - - csum_len)); + + full_dsn = mptcp_ntoh64(dss_ack_rsp->mdss_dsn); + NTOHL(dss_ack_rsp->mdss_subflow_seqn); + NTOHS(dss_ack_rsp->mdss_data_len); + + mptcp_do_dss_opt_ack_meat(full_dack, full_dsn, tp, tiwin); + + if (csum_len != 0) + csum = *(uint16_t *)(void *)(cp + dss_ack_rsp->mdss_copt.mdss_len - csum_len); + + mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, + dss_ack_rsp->mdss_subflow_seqn, + dss_ack_rsp->mdss_data_len, + csum); + break; } case (MDSS_M | MDSS_A | MDSS_a): @@ -1413,21 +1305,15 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) dss32_ack64_opt->mdss_copt.mdss_len, sizeof (struct mptcp_dss32_ack64_opt) + csum_len); - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: 32-bit M and 64-bit A present.\n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - full_dack = mptcp_ntoh64(dss32_ack64_opt->mdss_ack); - mptcp_do_dss_opt_ack_meat(full_dack, tp); NTOHL(dss32_ack64_opt->mdss_dsn); - MPT_LOCK_SPIN(mp_tp); mp_tp->mpt_flags |= MPTCPF_RCVD_64BITACK; MPTCP_EXTEND_DSN(mp_tp->mpt_rcvnxt, dss32_ack64_opt->mdss_dsn, full_dsn); - MPT_UNLOCK(mp_tp); NTOHL(dss32_ack64_opt->mdss_subflow_seqn); NTOHS(dss32_ack64_opt->mdss_data_len); + + mptcp_do_dss_opt_ack_meat(full_dack, full_dsn, tp, tiwin); if (csum_len == 0) mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, dss32_ack64_opt->mdss_subflow_seqn, @@ -1451,18 +1337,11 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) MPTCP_DSS_OPT_SZ_CHK(dss64_ack64->mdss_copt.mdss_len, sizeof (struct mptcp_dss64_ack64_opt) + csum_len); - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: 64-bit M and 64-bit A present.\n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - - MPT_LOCK_SPIN(mp_tp); mp_tp->mpt_flags |= MPTCPF_RCVD_64BITACK; mp_tp->mpt_flags |= MPTCPF_SND_64BITACK; - MPT_UNLOCK(mp_tp); full_dsn = mptcp_ntoh64(dss64_ack64->mdss_dsn); full_dack = mptcp_ntoh64(dss64_ack64->mdss_dsn); - mptcp_do_dss_opt_ack_meat(full_dack, tp); + mptcp_do_dss_opt_ack_meat(full_dack, full_dsn, tp, tiwin); NTOHL(dss64_ack64->mdss_subflow_seqn); NTOHS(dss64_ack64->mdss_data_len); if (csum_len == 0) @@ -1479,30 +1358,22 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) break; } default: - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: File bug, DSS flags = %x\n", __func__, - dss_rsp->mdss_flags), + mptcplog((LOG_DEBUG,"%s: File bug, DSS flags = %x\n", + __func__, dss_rsp->mdss_flags), (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), MPTCP_LOGLVL_LOG); break; } } - static void mptcp_do_fin_opt(struct tcpcb *tp) { - struct mptcb *mp_tp = (struct mptcb *)tp->t_mptcb; - - mptcplog((LOG_DEBUG,"MPTCP Socket: %s \n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); + struct mptcb *mp_tp = tptomptp(tp); if (!(tp->t_mpflags & TMPF_RECV_DFIN)) { if (mp_tp != NULL) { - MPT_LOCK(mp_tp); mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN); - MPT_UNLOCK(mp_tp); if (tp->t_inpcb->inp_socket != NULL) { soevent(tp->t_inpcb->inp_socket, @@ -1525,8 +1396,8 @@ mptcp_do_fin_opt(struct tcpcb *tp) static void mptcp_do_dss_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen) { -#pragma unused(th, optlen) - struct mptcb *mp_tp = (struct mptcb *)tp->t_mptcb; +#pragma unused(optlen) + struct mptcb *mp_tp = tptomptp(tp); if (!mp_tp) return; @@ -1541,7 +1412,7 @@ mptcp_do_dss_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen) mptcp_do_fin_opt(tp); } - mptcp_do_dss_opt_meat(cp, tp); + mptcp_do_dss_opt_meat(cp, tp, th); } } } @@ -1555,20 +1426,16 @@ mptcp_do_fastclose_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th) if (th->th_flags != TH_ACK) return; - mptcplog((LOG_DEBUG,"MPTCP Socket: %s: \n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - if (fc_opt->mfast_len != sizeof (struct mptcp_fastclose_opt)) { tcpstat.tcps_invalid_opt++; return; } - mp_tp = (struct mptcb *)tp->t_mptcb; + mp_tp = tptomptp(tp); if (!mp_tp) return; - if (fc_opt->mfast_key != mptcp_get_localkey(mp_tp)) { + if (fc_opt->mfast_key != mp_tp->mpt_localkey) { tcpstat.tcps_invalid_opt++; return; } @@ -1613,18 +1480,14 @@ mptcp_do_mpfail_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th) if ((th->th_flags != TH_ACK) && (th->th_flags != TH_RST)) return; - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: \n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), MPTCP_LOGLVL_LOG); - if (fail_opt->mfail_len != sizeof (struct mptcp_mpfail_opt)) return; - mp_tp = (struct mptcb *)tp->t_mptcb; - MPT_LOCK(mp_tp); + mp_tp = tptomptp(tp); + mp_tp->mpt_flags |= MPTCPF_RECVD_MPFAIL; mp_tp->mpt_dsn_at_csum_fail = mptcp_hton64(fail_opt->mfail_dsn); - MPT_UNLOCK(mp_tp); - error = mptcp_get_map_for_dsn(tp->t_inpcb->inp_socket, + error = mptcp_get_map_for_dsn(tp->t_inpcb->inp_socket, mp_tp->mpt_dsn_at_csum_fail, &mdss_subflow_seqn); if (error == 0) { mp_tp->mpt_ssn_at_csum_fail = mdss_subflow_seqn; @@ -1638,6 +1501,12 @@ tcp_do_mptcp_options(struct tcpcb *tp, u_char *cp, struct tcphdr *th, struct tcpopt *to, int optlen) { int mptcp_subtype; + struct mptcb *mp_tp = tptomptp(tp); + + if (mp_tp == NULL) + return; + + mpte_lock_assert_held(mp_tp->mpt_mpte); /* All MPTCP options have atleast 4 bytes */ if (optlen < 4) @@ -1675,49 +1544,11 @@ tcp_do_mptcp_options(struct tcpcb *tp, u_char *cp, struct tcphdr *th, return; } -/* - * MPTCP ADD_ADDR and REMOVE_ADDR options - */ - -/* - * ADD_ADDR is only placeholder code - not sent on wire - * The ADD_ADDR option is not sent on wire because of security issues - * around connection hijacking. - */ -void -mptcp_send_addaddr_opt(struct tcpcb *tp, struct mptcp_addaddr_opt *opt) -{ - - opt->ma_kind = TCPOPT_MULTIPATH; - opt->ma_len = sizeof (struct mptcp_addaddr_opt); - opt->ma_subtype = MPO_ADD_ADDR; - opt->ma_addr_id = tp->t_local_aid; -#ifdef MPTCP_NOTYET - struct inpcb *inp = tp->t_inpcb; - if (inp->inp_vflag == AF_INET) { - opt->ma_ipver = MA_IPVer_V4; - bcopy((char *)&sin->sin_addr.s_addr, (char *)opt + opt->ma_len, - sizeof (in_addr_t)); - opt->ma_len += sizeof (in_addr_t); - } else if (inp->inp_vflag == AF_INET6) { - opt->ma_ipver = MA_IPVer_V6; - bcopy((char *)&sin6->sin6_addr, (char *)opt + opt->ma_len, - sizeof (struct in6_addr)); - opt->ma_len += sizeof (struct in6_addr); - } -#if 0 - if (tp->t_mp_port) { - /* add ports XXX */ - } -#endif -#endif -} - /* REMOVE_ADDR option is sent when a source address goes away */ -void +static void mptcp_send_remaddr_opt(struct tcpcb *tp, struct mptcp_remaddr_opt *opt) { - mptcplog((LOG_DEBUG,"MPTCP Socket: %s: local id %d remove id %d \n", + mptcplog((LOG_DEBUG,"%s: local id %d remove id %d \n", __func__, tp->t_local_aid, tp->t_rem_aid), (MPTCP_SOCKET_DBG|MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); @@ -1729,33 +1560,6 @@ mptcp_send_remaddr_opt(struct tcpcb *tp, struct mptcp_remaddr_opt *opt) tp->t_mpflags &= ~TMPF_SND_REM_ADDR; } -/* - * MPTCP MP_PRIO option - */ - -#if 0 -/* - * Current implementation drops incoming MP_PRIO option and this code is - * just a placeholder. The option is dropped because only the mobile client can - * decide which of the subflows is preferred (usually wifi is preferred - * over Cellular). - */ -void -mptcp_do_mpprio_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, - int optlen) -{ - int bkp = 0; - struct mptcp_mpprio_opt *mpprio = (struct mptcp_mpprio_opt *)cp; - - if ((tp == NULL) || !(tp->t_mpflags & TMPF_MPTCP_TRUE)) - return; - - if ((mpprio->mpprio_len != sizeof (struct mptcp_mpprio_addr_opt)) && - (mpprio->mpprio_len != sizeof (struct mptcp_mpprio_opt))) - return; -} -#endif - /* We send MP_PRIO option based on the values set by the SIOCSCONNORDER ioctl */ static int mptcp_snd_mpprio(struct tcpcb *tp, u_char *cp, int optlen) @@ -1767,11 +1571,6 @@ mptcp_snd_mpprio(struct tcpcb *tp, u_char *cp, int optlen) return (optlen); } - if (mptcp_mpprio_enable != 1) { - tp->t_mpflags &= ~TMPF_SND_MPPRIO; - return (optlen); - } - if ((MAX_TCPOPTLEN - optlen) < (int)sizeof (mpprio)) return (optlen); @@ -1786,8 +1585,9 @@ mptcp_snd_mpprio(struct tcpcb *tp, u_char *cp, int optlen) memcpy(cp + optlen, &mpprio, sizeof (mpprio)); optlen += sizeof (mpprio); tp->t_mpflags &= ~TMPF_SND_MPPRIO; - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: aid = %d \n", __func__, + mptcplog((LOG_DEBUG, "%s: aid = %d \n", __func__, tp->t_local_aid), (MPTCP_SOCKET_DBG|MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); return (optlen); } + diff --git a/bsd/netinet/mptcp_opt.h b/bsd/netinet/mptcp_opt.h index a9450dee9..785e1a998 100644 --- a/bsd/netinet/mptcp_opt.h +++ b/bsd/netinet/mptcp_opt.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -41,25 +41,19 @@ #define MPTCP_CAPABLE_RETRIES (2) __BEGIN_DECLS +extern void mptcp_data_ack_rcvd(struct mptcb *mp_tp, struct tcpcb *tp, u_int64_t full_dack); +extern void mptcp_update_window_fallback(struct tcpcb *tp); extern void tcp_do_mptcp_options(struct tcpcb *, u_char *, struct tcphdr *, struct tcpopt *, int); -extern unsigned mptcp_setup_syn_opts(struct socket *, int, u_char*, unsigned); +extern unsigned mptcp_setup_syn_opts(struct socket *, u_char*, unsigned); extern unsigned mptcp_setup_join_ack_opts(struct tcpcb *, u_char*, unsigned); -extern void mptcp_update_dss_send_state(struct mptcb *, u_int64_t); -extern void mptcp_send_addaddr_opt(struct tcpcb *, struct mptcp_addaddr_opt *); -extern void mptcp_send_remaddr_opt(struct tcpcb *, struct mptcp_remaddr_opt *); -extern unsigned int mptcp_setup_opts(struct tcpcb *, int, u_char *, - unsigned int, int, int, unsigned int **, u_int8_t **, u_int64_t *, - u_int32_t **, boolean_t *); +extern unsigned int mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, + unsigned int optlen, int flags, int len, + boolean_t *p_mptcp_acknow); extern void mptcp_update_dss_rcv_state(struct mptcp_dsn_opt *, struct tcpcb *, uint16_t); -extern void mptcp_update_rcv_state_f(struct mptcp_dss_ack_opt *, - struct tcpcb *, uint16_t); -extern void mptcp_update_rcv_state_g(struct mptcp_dss64_ack32_opt *, - struct tcpcb *, uint16_t); extern void mptcp_update_rcv_state_meat(struct mptcb *, struct tcpcb *, u_int64_t, u_int32_t, u_int16_t, uint16_t); -extern void mptcp_data_ack_rcvd(struct mptcb *, struct tcpcb *, u_int64_t); __END_DECLS #endif /* BSD_KERNEL_PRIVATE */ diff --git a/bsd/netinet/mptcp_seq.h b/bsd/netinet/mptcp_seq.h index a444baa08..c79232ee6 100644 --- a/bsd/netinet/mptcp_seq.h +++ b/bsd/netinet/mptcp_seq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * diff --git a/bsd/netinet/mptcp_subr.c b/bsd/netinet/mptcp_subr.c index 90ff36152..19e128687 100644 --- a/bsd/netinet/mptcp_subr.c +++ b/bsd/netinet/mptcp_subr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -26,26 +26,29 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#include -#include -#include +#include +#include +#include + +#include + +#include +#include +#include #include #include #include +#include +#include +#include #include #include #include -#include -#include -#include #include +#include +#include -#include -#include - -#include -#include - +#include #include #include #include @@ -57,6 +60,7 @@ #include #include #include +#include #include #include #include @@ -66,8 +70,6 @@ #endif /* INET6 */ #include -extern char *proc_best_name(proc_t); - /* * Notes on MPTCP implementation. * @@ -86,67 +88,45 @@ extern char *proc_best_name(proc_t); * PCB (mppcb) as well as the MPTCP Session (mptses). * * The MPTCP Session is an MPTCP-specific extension to the Multipath PCB; - * in particular, the list of subflows as well as the MPTCP thread. * * A functioning MPTCP Session consists of one or more subflow sockets. Each * subflow socket is essentially a regular PF_INET/PF_INET6 TCP socket, and is * represented by the mptsub structure. Because each subflow requires access * to the MPTCP Session, the MPTCP socket's so_usecount is bumped up for each - * subflow. This gets decremented prior to the subflow's destruction. The - * subflow lock (mpts_lock) is used to protect accesses to the subflow. - * - * To handle events (read, write, control) from the subflows, an MPTCP thread - * is created; currently, there is one thread per MPTCP Session. In order to - * prevent the MPTCP socket from being destroyed while being accessed by the - * MPTCP thread, we bump up the MPTCP socket's so_usecount for the thread, - * which will be decremented prior to the thread's termination. The thread - * lock (mpte_thread_lock) is used to synchronize its signalling. + * subflow. This gets decremented prior to the subflow's destruction. * - * Lock ordering is defined as follows: + * To handle events (read, write, control) from the subflows, we do direct + * upcalls into the specific function. * - * mtcbinfo (mppi_lock) - * mp_so (mpp_lock) - * mpts (mpts_lock) - * so (inpcb_mtx) - * mptcb (mpt_lock) - * - * It is not a requirement that all of the above locks need to be acquired - * in succession, but the correct lock ordering must be followed when there - * are more than one locks that need to be held. The MPTCP thread lock is - * is not constrained by this arrangement, because none of the other locks - * is ever acquired while holding mpte_thread_lock; therefore it may be called - * at any moment to signal the thread. + * The whole MPTCP connection is protected by a single lock, the MPTCP socket's + * lock. Incoming data on a subflow also ends up taking this single lock. To + * achieve the latter, tcp_lock/unlock has been changed to rather use the lock + * of the MPTCP-socket. * * An MPTCP socket will be destroyed when its so_usecount drops to zero; this * work is done by the MPTCP garbage collector which is invoked on demand by * the PF_MULTIPATH garbage collector. This process will take place once all - * of the subflows have been destroyed, and the MPTCP thread be instructed to - * self-terminate. + * of the subflows have been destroyed. */ -static void mptcp_sesdestroy(struct mptses *); -static void mptcp_thread_signal_locked(struct mptses *); -static void mptcp_thread_terminate_signal(struct mptses *); -static void mptcp_thread_dowork(struct mptses *); -static void mptcp_thread_func(void *, wait_result_t); -static void mptcp_thread_destroy(struct mptses *); -static void mptcp_key_pool_init(void); static void mptcp_attach_to_subf(struct socket *, struct mptcb *, uint8_t); static void mptcp_detach_mptcb_from_subf(struct mptcb *, struct socket *); static uint32_t mptcp_gc(struct mppcbinfo *); -static int mptcp_subflow_soclose(struct mptsub *, struct socket *); -static int mptcp_subflow_soconnectx(struct mptses *, struct mptsub *); static int mptcp_subflow_soreceive(struct socket *, struct sockaddr **, struct uio *, struct mbuf **, struct mbuf **, int *); +static int mptcp_subflow_sosend(struct socket *, struct sockaddr *, + struct uio *, struct mbuf *, struct mbuf *, int); static void mptcp_subflow_rupcall(struct socket *, void *, int); static void mptcp_subflow_input(struct mptses *, struct mptsub *); static void mptcp_subflow_wupcall(struct socket *, void *, int); -static void mptcp_subflow_eupcall(struct socket *, void *, uint32_t); -static void mptcp_update_last_owner(struct mptsub *, struct socket *); -static void mptcp_output_needed(struct mptses *mpte, struct mptsub *to_mpts); -static void mptcp_get_rtt_measurement(struct mptsub *, struct mptses *); -static void mptcp_drop_tfo_data(struct mptses *, struct mptsub *, int *); +static void mptcp_subflow_eupcall1(struct socket *, void *, uint32_t); +static void mptcp_update_last_owner(struct socket *so, struct socket *mp_so); +static void mptcp_drop_tfo_data(struct mptses *, struct mptsub *); + +static void mptcp_subflow_abort(struct mptsub *, int); + +static void mptcp_send_dfin(struct socket *so); /* * Possible return values for subflow event handlers. Note that success @@ -163,28 +143,22 @@ typedef enum { } ev_ret_t; static ev_ret_t mptcp_subflow_events(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_connreset_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_cantrcvmore_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_cantsendmore_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_timeout_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_nosrcaddr_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_failover_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_ifdenied_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_suspend_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_resume_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_connected_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_disconnected_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_mpstatus_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_mustrst_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_fastjoin_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_deleteok_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_mpcantrcvmore_ev(struct mptses *, struct mptsub *, uint64_t *); +static ev_ret_t mptcp_subflow_propagate_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_nosrcaddr_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_failover_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_ifdenied_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_connected_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_disconnected_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_mpstatus_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_mustrst_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_mpcantrcvmore_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_adaptive_rtimo_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_adaptive_wtimo_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); static const char *mptcp_evret2str(ev_ret_t); -static mptcp_key_t *mptcp_reserve_key(void); -static int mptcp_do_sha1(mptcp_key_t *, char *, int); -static void mptcp_init_local_parms(struct mptcb *); +static void mptcp_do_sha1(mptcp_key_t *, char *); +static void mptcp_init_local_parms(struct mptses *); static unsigned int mptsub_zone_size; /* size of mptsub */ static struct zone *mptsub_zone; /* zone for mptsub */ @@ -197,8 +171,6 @@ static struct zone *mpt_subauth_zone; /* zone of subf auth entry */ struct mppcbinfo mtcbinfo; -static struct mptcp_keys_pool_head mptcp_keys_pool; - #define MPTCP_SUBFLOW_WRITELEN (8 * 1024) /* bytes to write each time */ #define MPTCP_SUBFLOW_READLEN (8 * 1024) /* bytes to read each time */ @@ -206,40 +178,17 @@ SYSCTL_DECL(_net_inet); SYSCTL_NODE(_net_inet, OID_AUTO, mptcp, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "MPTCP"); -uint32_t mptcp_dbg_area = 0; /* more noise if greater than 1 */ +uint32_t mptcp_dbg_area = 31; /* more noise if greater than 1 */ SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, dbg_area, CTLFLAG_RW|CTLFLAG_LOCKED, &mptcp_dbg_area, 0, "MPTCP debug area"); -uint32_t mptcp_dbg_level = 0; +uint32_t mptcp_dbg_level = 1; SYSCTL_INT(_net_inet_mptcp, OID_AUTO, dbg_level, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_dbg_level, 0, "MPTCP debug level"); - SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, pcbcount, CTLFLAG_RD|CTLFLAG_LOCKED, &mtcbinfo.mppi_count, 0, "Number of active PCBs"); -/* - * Since there is one kernel thread per mptcp socket, imposing an artificial - * limit on number of allowed mptcp sockets. - */ -uint32_t mptcp_socket_limit = MPPCB_LIMIT; -SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, sk_lim, CTLFLAG_RW|CTLFLAG_LOCKED, - &mptcp_socket_limit, 0, "MPTCP socket limit"); - -/* - * SYSCTL to turn on delayed cellular subflow start. - */ -uint32_t mptcp_delayed_subf_start = 0; -SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, delayed, CTLFLAG_RW|CTLFLAG_LOCKED, - &mptcp_delayed_subf_start, 0, "MPTCP Delayed Subflow start"); - -/* - * sysctl to use network status hints from symptomsd - */ -uint32_t mptcp_use_symptomsd = 1; -SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, usesymptoms, CTLFLAG_RW|CTLFLAG_LOCKED, - &mptcp_use_symptomsd, 0, "MPTCP Use SymptomsD"); - static struct protosw mptcp_subflow_protosw; static struct pr_usrreqs mptcp_subflow_usrreqs; #if INET6 @@ -247,26 +196,26 @@ static struct ip6protosw mptcp_subflow_protosw6; static struct pr_usrreqs mptcp_subflow_usrreqs6; #endif /* INET6 */ +static uint8_t mptcp_create_subflows_scheduled; + typedef struct mptcp_subflow_event_entry { uint64_t sofilt_hint_mask; ev_ret_t (*sofilt_hint_ev_hdlr)( struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint); + uint64_t *p_mpsofilt_hint, + uint64_t event); } mptsub_ev_entry_t; +static uint8_t mptcp_cellicon_is_set; +static uint32_t mptcp_last_cellicon_set; +#define MPTCP_CELLICON_TOGGLE_RATE (5 * TCP_RETRANSHZ) /* Only toggle every 5 seconds */ + /* * XXX The order of the event handlers below is really - * really important. - * SO_FILT_HINT_DELETEOK event has to be handled first, - * else we may end up missing on this event. - * Please read radar://24043716 for more details. + * really important. Think twice before changing it. */ static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = { - { - .sofilt_hint_mask = SO_FILT_HINT_DELETEOK, - .sofilt_hint_ev_hdlr = mptcp_deleteok_ev, - }, { .sofilt_hint_mask = SO_FILT_HINT_MPCANTRCVMORE, .sofilt_hint_ev_hdlr = mptcp_subflow_mpcantrcvmore_ev, @@ -277,7 +226,7 @@ static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = { }, { .sofilt_hint_mask = SO_FILT_HINT_CONNRESET, - .sofilt_hint_ev_hdlr = mptcp_subflow_connreset_ev, + .sofilt_hint_ev_hdlr = mptcp_subflow_propagate_ev, }, { .sofilt_hint_mask = SO_FILT_HINT_MUSTRST, @@ -285,14 +234,11 @@ static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = { }, { .sofilt_hint_mask = SO_FILT_HINT_CANTRCVMORE, - .sofilt_hint_ev_hdlr = mptcp_subflow_cantrcvmore_ev, - }, - { .sofilt_hint_mask = SO_FILT_HINT_CANTSENDMORE, - .sofilt_hint_ev_hdlr = mptcp_subflow_cantsendmore_ev, + .sofilt_hint_ev_hdlr = mptcp_subflow_propagate_ev, }, { .sofilt_hint_mask = SO_FILT_HINT_TIMEOUT, - .sofilt_hint_ev_hdlr = mptcp_subflow_timeout_ev, + .sofilt_hint_ev_hdlr = mptcp_subflow_propagate_ev, }, { .sofilt_hint_mask = SO_FILT_HINT_NOSRCADDR, @@ -302,14 +248,6 @@ static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = { .sofilt_hint_mask = SO_FILT_HINT_IFDENIED, .sofilt_hint_ev_hdlr = mptcp_subflow_ifdenied_ev, }, - { - .sofilt_hint_mask = SO_FILT_HINT_SUSPEND, - .sofilt_hint_ev_hdlr = mptcp_subflow_suspend_ev, - }, - { - .sofilt_hint_mask = SO_FILT_HINT_RESUME, - .sofilt_hint_ev_hdlr = mptcp_subflow_resume_ev, - }, { .sofilt_hint_mask = SO_FILT_HINT_CONNECTED, .sofilt_hint_ev_hdlr = mptcp_subflow_connected_ev, @@ -323,9 +261,13 @@ static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = { .sofilt_hint_ev_hdlr = mptcp_subflow_disconnected_ev, }, { - .sofilt_hint_mask = SO_FILT_HINT_MPFASTJ, - .sofilt_hint_ev_hdlr = mptcp_fastjoin_ev, - } + .sofilt_hint_mask = SO_FILT_HINT_ADAPTIVE_RTIMO, + .sofilt_hint_ev_hdlr = mptcp_subflow_adaptive_rtimo_ev, + }, + { + .sofilt_hint_mask = SO_FILT_HINT_ADAPTIVE_WTIMO, + .sofilt_hint_ev_hdlr = mptcp_subflow_adaptive_wtimo_ev, + }, }; /* @@ -361,6 +303,7 @@ mptcp_init(struct protosw *pp, struct domain *dp) mptcp_subflow_protosw.pr_entry.tqe_prev = NULL; mptcp_subflow_protosw.pr_usrreqs = &mptcp_subflow_usrreqs; mptcp_subflow_usrreqs.pru_soreceive = mptcp_subflow_soreceive; + mptcp_subflow_usrreqs.pru_sosend = mptcp_subflow_sosend; mptcp_subflow_usrreqs.pru_rcvoob = pru_rcvoob_notsupp; /* * Socket filters shouldn't attach/detach to/from this protosw @@ -383,6 +326,7 @@ mptcp_init(struct protosw *pp, struct domain *dp) mptcp_subflow_protosw6.pr_entry.tqe_prev = NULL; mptcp_subflow_protosw6.pr_usrreqs = &mptcp_subflow_usrreqs6; mptcp_subflow_usrreqs6.pru_soreceive = mptcp_subflow_soreceive; + mptcp_subflow_usrreqs6.pru_sosend = mptcp_subflow_sosend; mptcp_subflow_usrreqs6.pru_rcvoob = pru_rcvoob_notsupp; /* * Socket filters shouldn't attach/detach to/from this protosw @@ -415,7 +359,6 @@ mptcp_init(struct protosw *pp, struct domain *dp) mtcbinfo.mppi_gc = mptcp_gc; mtcbinfo.mppi_timer = mptcp_timer; - mtcbinfo.mppi_pcbe_create = mptcp_sescreate; /* attach to MP domain for garbage collection to take place */ mp_pcbinfo_attach(&mtcbinfo); @@ -448,20 +391,82 @@ mptcp_init(struct protosw *pp, struct domain *dp) zone_change(mpt_subauth_zone, Z_CALLERACCT, FALSE); zone_change(mpt_subauth_zone, Z_EXPAND, TRUE); - /* Set up a list of unique keys */ - mptcp_key_pool_init(); + mptcp_last_cellicon_set = tcp_now; +} + +int +mptcp_get_statsindex(struct mptcp_itf_stats *stats, const struct mptsub *mpts) +{ + const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp; + + int i, index = -1; + + if (ifp == NULL) { + mptcplog((LOG_ERR, "%s: no ifp on subflow\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return (-1); + } + + for (i = 0; i < MPTCP_ITFSTATS_SIZE; i++) { + if (stats[i].ifindex == IFSCOPE_NONE) { + if (index < 0) + index = i; + continue; + } + + if (stats[i].ifindex == ifp->if_index) { + index = i; + return (index); + } + } + + if (index != -1) { + stats[index].ifindex = ifp->if_index; + if (stats[index].is_expensive == 0) + stats[index].is_expensive = IFNET_IS_CELLULAR(ifp); + } + + return (index); +} + +void +mptcpstats_inc_switch(struct mptses *mpte, const struct mptsub *mpts) +{ + int index; + + tcpstat.tcps_mp_switches++; + mpte->mpte_subflow_switches++; + + index = mptcp_get_statsindex(mpte->mpte_itfstats, mpts); + + if (index != -1) + mpte->mpte_itfstats[index].switches++; +} + +/* + * Flushes all recorded socket options from an MP socket. + */ +static void +mptcp_flush_sopts(struct mptses *mpte) +{ + struct mptopt *mpo, *tmpo; + + TAILQ_FOREACH_SAFE(mpo, &mpte->mpte_sopts, mpo_entry, tmpo) { + mptcp_sopt_remove(mpte, mpo); + mptcp_sopt_free(mpo); + } + VERIFY(TAILQ_EMPTY(&mpte->mpte_sopts)); } /* * Create an MPTCP session, called as a result of opening a MPTCP socket. */ -void * -mptcp_sescreate(struct socket *mp_so, struct mppcb *mpp) +int +mptcp_sescreate(struct mppcb *mpp) { struct mppcbinfo *mppi; struct mptses *mpte; struct mptcb *mp_tp; - int error = 0; VERIFY(mpp != NULL); mppi = mpp->mpp_pcbinfo; @@ -482,178 +487,646 @@ mptcp_sescreate(struct socket *mp_so, struct mppcb *mpp) mpte->mpte_associd = SAE_ASSOCID_ANY; mpte->mpte_connid_last = SAE_CONNID_ANY; - lck_mtx_init(&mpte->mpte_thread_lock, mppi->mppi_lock_grp, - mppi->mppi_lock_attr); - - /* - * XXX: adi@apple.com - * - * This can be rather expensive if we have lots of MPTCP sockets, - * but we need a kernel thread for this model to work. Perhaps we - * could amortize the costs by having one worker thread per a group - * of MPTCP sockets. - */ - if (kernel_thread_start(mptcp_thread_func, mpte, - &mpte->mpte_thread) != KERN_SUCCESS) { - error = ENOBUFS; - goto out; - } - mp_so->so_usecount++; /* for thread */ + mpte->mpte_itfinfo = &mpte->_mpte_itfinfo[0]; + mpte->mpte_itfinfo_size = MPTE_ITFINFO_SIZE; /* MPTCP Protocol Control Block */ bzero(mp_tp, sizeof (*mp_tp)); - lck_mtx_init(&mp_tp->mpt_lock, mppi->mppi_lock_grp, - mppi->mppi_lock_attr); mp_tp->mpt_mpte = mpte; mp_tp->mpt_state = MPTCPS_CLOSED; -out: - if (error != 0) - lck_mtx_destroy(&mpte->mpte_thread_lock, mppi->mppi_lock_grp); - DTRACE_MPTCP5(session__create, struct socket *, mp_so, - struct sockbuf *, &mp_so->so_rcv, - struct sockbuf *, &mp_so->so_snd, - struct mppcb *, mpp, int, error); - return ((error != 0) ? NULL : mpte); + DTRACE_MPTCP1(session__create, struct mppcb *, mpp); + + return (0); +} + +static void +mptcpstats_get_bytes(struct mptses *mpte, boolean_t initial_cell, + uint64_t *cellbytes, uint64_t *allbytes) +{ + int64_t mycellbytes = 0; + uint64_t myallbytes = 0; + int i; + + for (i = 0; i < MPTCP_ITFSTATS_SIZE; i++) { + if (mpte->mpte_itfstats[i].is_expensive) { + mycellbytes += mpte->mpte_itfstats[i].mpis_txbytes; + mycellbytes += mpte->mpte_itfstats[i].mpis_rxbytes; + } + + myallbytes += mpte->mpte_itfstats[i].mpis_txbytes; + myallbytes += mpte->mpte_itfstats[i].mpis_rxbytes; + } + + if (initial_cell) { + mycellbytes -= mpte->mpte_init_txbytes; + mycellbytes -= mpte->mpte_init_txbytes; + } + + if (mycellbytes < 0) { + mptcplog((LOG_ERR, "%s cellbytes is %d\n", __func__, mycellbytes), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + *cellbytes = 0; + *allbytes = 0; + } else { + *cellbytes = mycellbytes; + *allbytes = myallbytes; + } +} + +static void +mptcpstats_session_wrapup(struct mptses *mpte) +{ + boolean_t cell = mpte->mpte_initial_cell; + + switch (mpte->mpte_svctype) { + case MPTCP_SVCTYPE_HANDOVER: + if (mpte->mpte_flags & MPTE_FIRSTPARTY) { + tcpstat.tcps_mptcp_fp_handover_attempt++; + + if (cell && mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_fp_handover_success_cell++; + + if (mpte->mpte_used_wifi) + tcpstat.tcps_mptcp_handover_wifi_from_cell++; + } else if (mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_fp_handover_success_wifi++; + + if (mpte->mpte_used_cell) + tcpstat.tcps_mptcp_handover_cell_from_wifi++; + } + } else { + tcpstat.tcps_mptcp_handover_attempt++; + + if (cell && mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_handover_success_cell++; + + if (mpte->mpte_used_wifi) + tcpstat.tcps_mptcp_handover_wifi_from_cell++; + } else if (mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_handover_success_wifi++; + + if (mpte->mpte_used_cell) + tcpstat.tcps_mptcp_handover_cell_from_wifi++; + } + } + + if (mpte->mpte_handshake_success) { + uint64_t cellbytes; + uint64_t allbytes; + + mptcpstats_get_bytes(mpte, cell, &cellbytes, &allbytes); + + tcpstat.tcps_mptcp_handover_cell_bytes += cellbytes; + tcpstat.tcps_mptcp_handover_all_bytes += allbytes; + } + break; + case MPTCP_SVCTYPE_INTERACTIVE: + if (mpte->mpte_flags & MPTE_FIRSTPARTY) { + tcpstat.tcps_mptcp_fp_interactive_attempt++; + + if (mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_fp_interactive_success++; + + if (!cell && mpte->mpte_used_cell) + tcpstat.tcps_mptcp_interactive_cell_from_wifi++; + } + } else { + tcpstat.tcps_mptcp_interactive_attempt++; + + if (mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_interactive_success++; + + if (!cell && mpte->mpte_used_cell) + tcpstat.tcps_mptcp_interactive_cell_from_wifi++; + } + } + + if (mpte->mpte_handshake_success) { + uint64_t cellbytes; + uint64_t allbytes; + + mptcpstats_get_bytes(mpte, cell, &cellbytes, &allbytes); + + tcpstat.tcps_mptcp_interactive_cell_bytes += cellbytes; + tcpstat.tcps_mptcp_interactive_all_bytes += allbytes; + } + break; + case MPTCP_SVCTYPE_AGGREGATE: + if (mpte->mpte_flags & MPTE_FIRSTPARTY) { + tcpstat.tcps_mptcp_fp_aggregate_attempt++; + + if (mpte->mpte_handshake_success) + tcpstat.tcps_mptcp_fp_aggregate_success++; + } else { + tcpstat.tcps_mptcp_aggregate_attempt++; + + if (mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_aggregate_success++; + } + } + + if (mpte->mpte_handshake_success) { + uint64_t cellbytes; + uint64_t allbytes; + + mptcpstats_get_bytes(mpte, cell, &cellbytes, &allbytes); + + tcpstat.tcps_mptcp_aggregate_cell_bytes += cellbytes; + tcpstat.tcps_mptcp_aggregate_all_bytes += allbytes; + } + break; + } + + if (cell && mpte->mpte_handshake_success && mpte->mpte_used_wifi) + tcpstat.tcps_mptcp_back_to_wifi++; } /* * Destroy an MPTCP session. */ static void -mptcp_sesdestroy(struct mptses *mpte) +mptcp_session_destroy(struct mptses *mpte) { struct mptcb *mp_tp; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ mp_tp = mpte->mpte_mptcb; VERIFY(mp_tp != NULL); + mptcpstats_session_wrapup(mpte); + + mptcp_unset_cellicon(); + /* * MPTCP Multipath PCB Extension section */ mptcp_flush_sopts(mpte); VERIFY(TAILQ_EMPTY(&mpte->mpte_subflows) && mpte->mpte_numflows == 0); - lck_mtx_destroy(&mpte->mpte_thread_lock, - mpte->mpte_mppcb->mpp_pcbinfo->mppi_lock_grp); + if (mpte->mpte_itfinfo_size > MPTE_ITFINFO_SIZE) + _FREE(mpte->mpte_itfinfo, M_TEMP); + + mpte->mpte_itfinfo = NULL; + + m_freem_list(mpte->mpte_reinjectq); /* * MPTCP Protocol Control Block section */ - lck_mtx_destroy(&mp_tp->mpt_lock, - mpte->mpte_mppcb->mpp_pcbinfo->mppi_lock_grp); - DTRACE_MPTCP2(session__destroy, struct mptses *, mpte, struct mptcb *, mp_tp); } -/* - * Allocate an MPTCP socket option structure. - */ -struct mptopt * -mptcp_sopt_alloc(int how) +static boolean_t +mptcp_ok_to_create_subflows(struct mptcb *mp_tp) { - struct mptopt *mpo; + return (mp_tp->mpt_state >= MPTCPS_ESTABLISHED && + mp_tp->mpt_state < MPTCPS_TIME_WAIT && + !(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP)); +} - mpo = (how == M_WAITOK) ? zalloc(mptopt_zone) : - zalloc_noblock(mptopt_zone); - if (mpo != NULL) { - bzero(mpo, mptopt_zone_size); +static int +mptcp_synthesize_nat64(struct in6_addr *addr, uint32_t len, struct in_addr *addrv4) +{ + static const struct in6_addr well_known_prefix = { + .__u6_addr.__u6_addr8 = {0x00, 0x64, 0xff, 0x9b, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + }; + char buf[MAX_IPv6_STR_LEN]; + char *ptrv4 = (char *)addrv4; + char *ptr = (char *)addr; + + if (IN_ZERONET(addrv4->s_addr) || // 0.0.0.0/8 Source hosts on local network + IN_LOOPBACK(addrv4->s_addr) || // 127.0.0.0/8 Loopback + IN_LINKLOCAL(addrv4->s_addr) || // 169.254.0.0/16 Link Local + IN_DS_LITE(addrv4->s_addr) || // 192.0.0.0/29 DS-Lite + IN_6TO4_RELAY_ANYCAST(addrv4->s_addr) || // 192.88.99.0/24 6to4 Relay Anycast + IN_MULTICAST(addrv4->s_addr) || // 224.0.0.0/4 Multicast + INADDR_BROADCAST == addrv4->s_addr) { // 255.255.255.255/32 Limited Broadcast + return (-1); } - return (mpo); -} + /* Check for the well-known prefix */ + if (len == NAT64_PREFIX_LEN_96 && + IN6_ARE_ADDR_EQUAL(addr, &well_known_prefix)) { + if (IN_PRIVATE(addrv4->s_addr) || // 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 Private-Use + IN_SHARED_ADDRESS_SPACE(addrv4->s_addr)) // 100.64.0.0/10 Shared Address Space + return (-1); + } -/* - * Free an MPTCP socket option structure. - */ -void -mptcp_sopt_free(struct mptopt *mpo) -{ - VERIFY(!(mpo->mpo_flags & MPOF_ATTACHED)); + switch (len) { + case NAT64_PREFIX_LEN_96: + memcpy(ptr + 12, ptrv4, 4); + break; + case NAT64_PREFIX_LEN_64: + memcpy(ptr + 9, ptrv4, 4); + break; + case NAT64_PREFIX_LEN_56: + memcpy(ptr + 7, ptrv4, 1); + memcpy(ptr + 9, ptrv4 + 1, 3); + break; + case NAT64_PREFIX_LEN_48: + memcpy(ptr + 6, ptrv4, 2); + memcpy(ptr + 9, ptrv4 + 2, 2); + break; + case NAT64_PREFIX_LEN_40: + memcpy(ptr + 5, ptrv4, 3); + memcpy(ptr + 9, ptrv4 + 3, 1); + break; + case NAT64_PREFIX_LEN_32: + memcpy(ptr + 4, ptrv4, 4); + break; + default: + panic("NAT64-prefix len is wrong: %u\n", len); + } - zfree(mptopt_zone, mpo); -} + mptcplog((LOG_DEBUG, "%s: nat64prefix-len %u synthesized %s\n", __func__, + len, inet_ntop(AF_INET6, (void *)addr, buf, sizeof(buf))), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); -/* - * Add a socket option to the MPTCP socket option list. - */ -void -mptcp_sopt_insert(struct mptses *mpte, struct mptopt *mpo) -{ - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - VERIFY(!(mpo->mpo_flags & MPOF_ATTACHED)); - mpo->mpo_flags |= MPOF_ATTACHED; - TAILQ_INSERT_TAIL(&mpte->mpte_sopts, mpo, mpo_entry); + return (0); } -/* - * Remove a socket option from the MPTCP socket option list. - */ void -mptcp_sopt_remove(struct mptses *mpte, struct mptopt *mpo) +mptcp_check_subflows_and_add(struct mptses *mpte) { - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - VERIFY(mpo->mpo_flags & MPOF_ATTACHED); - mpo->mpo_flags &= ~MPOF_ATTACHED; - TAILQ_REMOVE(&mpte->mpte_sopts, mpo, mpo_entry); -} + struct mptcb *mp_tp = mpte->mpte_mptcb; + uint32_t i; -/* - * Search for an existing socket option. - */ -struct mptopt * -mptcp_sopt_find(struct mptses *mpte, struct sockopt *sopt) -{ - struct mptopt *mpo; + if (!mptcp_ok_to_create_subflows(mp_tp)) + return; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + for (i = 0; i < mpte->mpte_itfinfo_size; i++) { + struct mpt_itf_info *info; + struct mptsub *mpts; + uint32_t ifindex; + int found = 0; - TAILQ_FOREACH(mpo, &mpte->mpte_sopts, mpo_entry) { - if (mpo->mpo_level == sopt->sopt_level && - mpo->mpo_name == sopt->sopt_name) - break; - } - VERIFY(mpo == NULL || sopt->sopt_valsize == sizeof (int)); + info = &mpte->mpte_itfinfo[i]; - return (mpo); -} + if (info->no_mptcp_support) + continue; -/* - * Flushes all recorded socket options from an MP socket. - */ -void -mptcp_flush_sopts(struct mptses *mpte) -{ - struct mptopt *mpo, *tmpo; + ifindex = info->ifindex; + if (ifindex == IFSCOPE_NONE) + continue; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp; - TAILQ_FOREACH_SAFE(mpo, &mpte->mpte_sopts, mpo_entry, tmpo) { - mptcp_sopt_remove(mpte, mpo); - mptcp_sopt_free(mpo); - } - VERIFY(TAILQ_EMPTY(&mpte->mpte_sopts)); -} + if (ifp == NULL) + continue; -/* - * Allocate a MPTCP subflow structure. - */ -struct mptsub * -mptcp_subflow_alloc(int how) -{ - struct mptsub *mpts; + if (ifp->if_index == ifindex && + !(mpts->mpts_socket->so_state & SS_ISDISCONNECTED)) { + /* + * We found a subflow on this interface. + * No need to create a new one. + */ + found = 1; + break; + } + + /* + * In Handover mode, only create cell subflow if + * 1. Wi-Fi Assist is active + * 2. Symptoms marked WiFi as weak + * 3. We are experiencing RTOs or we are not sending data. + * + * This covers the scenario, where: + * 1. We send and get retransmission timeouts (thus, + * we confirmed that WiFi is indeed bad). + * 2. We are not sending and the server tries to send. + * Establshing a cell-subflow gives the server a + * chance to send us some data over cell if WiFi + * is dead. We establish the subflow with the + * backup-bit set, so the server is not allowed to + * send on this subflow as long as WiFi is providing + * good performance. + */ + if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER && + !IFNET_IS_CELLULAR(ifp) && + !(mpts->mpts_flags & (MPTSF_DISCONNECTING | MPTSF_DISCONNECTED | MPTSF_CLOSE_REQD)) && + (!mptcp_is_wifi_unusable() || + (sototcpcb(mpts->mpts_socket)->t_rxtshift < mptcp_fail_thresh && + mptetoso(mpte)->so_snd.sb_cc))) { + mptcplog((LOG_DEBUG, "%s handover, wifi state %u rxt %u ifindex %u this %u\n", + __func__, mptcp_is_wifi_unusable(), sototcpcb(mpts->mpts_socket)->t_rxtshift, ifindex, + ifp->if_index), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + found = 1; + break; + } + } + + if (!found && !(mpte->mpte_flags & MPTE_FIRSTPARTY) && + !(mpte->mpte_flags & MPTE_ACCESS_GRANTED) && + mptcp_developer_mode == 0) { + mptcp_ask_symptoms(mpte); + return; + } + + if (!found) { + struct sockaddr *dst = &mpte->mpte_dst; + struct sockaddr_in6 nat64pre; + + if (mpte->mpte_dst.sa_family == AF_INET && + !info->has_v4_conn && info->has_v6_conn) { + struct ipv6_prefix nat64prefixes[NAT64_MAX_NUM_PREFIXES]; + struct ifnet *ifp; + int error, j; + + bzero(&nat64pre, sizeof(struct sockaddr_in6)); + + ifnet_head_lock_shared(); + ifp = ifindex2ifnet[ifindex]; + ifnet_head_done(); + + error = ifnet_get_nat64prefix(ifp, nat64prefixes); + if (error) { + mptcplog((LOG_ERR, "%s: no NAT64-prefix on itf %s, error %d\n", + __func__, ifp->if_name, error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + continue; + } + + for (j = 0; j < NAT64_MAX_NUM_PREFIXES; j++) { + if (nat64prefixes[j].prefix_len != 0) + break; + } + + VERIFY(j < NAT64_MAX_NUM_PREFIXES); + + error = mptcp_synthesize_nat64(&nat64prefixes[j].ipv6_prefix, + nat64prefixes[j].prefix_len, + &mpte->__mpte_dst_v4.sin_addr); + if (error != 0) { + mptcplog((LOG_INFO, "%s: cannot synthesize this addr\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + continue; + } + + memcpy(&nat64pre.sin6_addr, + &nat64prefixes[j].ipv6_prefix, + sizeof(nat64pre.sin6_addr)); + nat64pre.sin6_len = sizeof(struct sockaddr_in6); + nat64pre.sin6_family = AF_INET6; + nat64pre.sin6_port = mpte->__mpte_dst_v6.sin6_port; + nat64pre.sin6_flowinfo = 0; + nat64pre.sin6_scope_id = 0; + + dst = (struct sockaddr *)&nat64pre; + } + + mptcp_subflow_add(mpte, NULL, dst, ifindex, NULL); + } + } +} + +/* + * Based on the MPTCP Service-type and the state of the subflows, we + * will destroy subflows here. + */ +static void +mptcp_check_subflows_and_remove(struct mptses *mpte) +{ + struct mptsub *mpts, *tmpts; + int found_working_subflow = 0, removed_some = 0; + int wifi_unusable = mptcp_is_wifi_unusable(); + + if (mpte->mpte_svctype != MPTCP_SVCTYPE_HANDOVER) + return; + + /* + * Look for a subflow that is on a non-cellular interface + * and actually works (aka, no retransmission timeout). + */ + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp; + struct socket *so; + struct tcpcb *tp; + + if (ifp == NULL || IFNET_IS_CELLULAR(ifp)) + continue; + + so = mpts->mpts_socket; + tp = sototcpcb(so); + + if (!(mpts->mpts_flags & MPTSF_CONNECTED) || + tp->t_state != TCPS_ESTABLISHED) + continue; + + /* Either this subflow is in good condition while we try to send */ + if (tp->t_rxtshift == 0 && mptetoso(mpte)->so_snd.sb_cc) + found_working_subflow = 1; - mpts = (how == M_WAITOK) ? zalloc(mptsub_zone) : - zalloc_noblock(mptsub_zone); - if (mpts != NULL) { - bzero(mpts, mptsub_zone_size); - lck_mtx_init(&mpts->mpts_lock, mtcbinfo.mppi_lock_grp, - mtcbinfo.mppi_lock_attr); + /* Or WiFi is fine */ + if (!wifi_unusable) + found_working_subflow = 1; } + /* + * Couldn't find a working subflow, let's not remove those on a cellular + * interface. + */ + if (!found_working_subflow) + return; + + TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { + const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp; + + /* Only remove cellular subflows */ + if (ifp == NULL || !IFNET_IS_CELLULAR(ifp)) + continue; + + soevent(mpts->mpts_socket, SO_FILT_HINT_LOCKED | SO_FILT_HINT_MUSTRST); + removed_some = 1; + } + + if (removed_some) + mptcp_unset_cellicon(); +} + +static void +mptcp_remove_subflows(struct mptses *mpte) +{ + struct mptsub *mpts, *tmpts; + + TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { + if (mpts->mpts_flags & MPTSF_CLOSE_REQD) { + mpts->mpts_flags &= ~MPTSF_CLOSE_REQD; + + soevent(mpts->mpts_socket, + SO_FILT_HINT_LOCKED | SO_FILT_HINT_NOSRCADDR); + } + } +} + +static void +mptcp_create_subflows(__unused void *arg) +{ + struct mppcb *mpp; + + /* + * Start with clearing, because we might be processing connections + * while a new event comes in. + */ + if (OSTestAndClear(0x01, &mptcp_create_subflows_scheduled)) + mptcplog((LOG_ERR, "%s: bit was already cleared!\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + /* Iterate over all MPTCP connections */ + + lck_mtx_lock(&mtcbinfo.mppi_lock); + + TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) { + struct mptses *mpte; + struct socket *mp_so; + + if (!(mpp->mpp_flags & MPP_CREATE_SUBFLOWS)) + continue; + + mpp_lock(mpp); + + mpp->mpp_flags &= ~MPP_CREATE_SUBFLOWS; + + mpte = mpp->mpp_pcbe; + mp_so = mpp->mpp_socket; + + VERIFY(mp_so->so_usecount > 0); + + mptcp_check_subflows_and_add(mpte); + mptcp_remove_subflows(mpte); + + mp_so->so_usecount--; /* See mptcp_sched_create_subflows */ + mpp_unlock(mpp); + } + + lck_mtx_unlock(&mtcbinfo.mppi_lock); +} + +/* + * We need this because we are coming from an NECP-event. This event gets posted + * while holding NECP-locks. The creation of the subflow however leads us back + * into NECP (e.g., to add the necp_cb and also from tcp_connect). + * So, we would deadlock there as we already hold the NECP-lock. + * + * So, let's schedule this separately. It also gives NECP the chance to make + * progress, without having to wait for MPTCP to finish its subflow creation. + */ +void +mptcp_sched_create_subflows(struct mptses *mpte) +{ + struct mppcb *mpp = mpte->mpte_mppcb; + struct mptcb *mp_tp = mpte->mpte_mptcb; + struct socket *mp_so = mpp->mpp_socket; + + if (!mptcp_ok_to_create_subflows(mp_tp)) { + mptcplog((LOG_DEBUG, "%s: not a good time for subflows, state %u flags %#x", + __func__, mp_tp->mpt_state, mp_tp->mpt_flags), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + return; + } + + if (!(mpp->mpp_flags & MPP_CREATE_SUBFLOWS)) { + mp_so->so_usecount++; /* To prevent it from being free'd in-between */ + mpp->mpp_flags |= MPP_CREATE_SUBFLOWS; + } + + if (OSTestAndSet(0x01, &mptcp_create_subflows_scheduled)) + return; + + /* Do the call in 100ms to allow NECP to schedule it on all sockets */ + timeout(mptcp_create_subflows, NULL, hz/10); +} + +/* + * Allocate an MPTCP socket option structure. + */ +struct mptopt * +mptcp_sopt_alloc(int how) +{ + struct mptopt *mpo; + + mpo = (how == M_WAITOK) ? zalloc(mptopt_zone) : + zalloc_noblock(mptopt_zone); + if (mpo != NULL) { + bzero(mpo, mptopt_zone_size); + } + + return (mpo); +} + +/* + * Free an MPTCP socket option structure. + */ +void +mptcp_sopt_free(struct mptopt *mpo) +{ + VERIFY(!(mpo->mpo_flags & MPOF_ATTACHED)); + + zfree(mptopt_zone, mpo); +} + +/* + * Add a socket option to the MPTCP socket option list. + */ +void +mptcp_sopt_insert(struct mptses *mpte, struct mptopt *mpo) +{ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + VERIFY(!(mpo->mpo_flags & MPOF_ATTACHED)); + mpo->mpo_flags |= MPOF_ATTACHED; + TAILQ_INSERT_TAIL(&mpte->mpte_sopts, mpo, mpo_entry); +} + +/* + * Remove a socket option from the MPTCP socket option list. + */ +void +mptcp_sopt_remove(struct mptses *mpte, struct mptopt *mpo) +{ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + VERIFY(mpo->mpo_flags & MPOF_ATTACHED); + mpo->mpo_flags &= ~MPOF_ATTACHED; + TAILQ_REMOVE(&mpte->mpte_sopts, mpo, mpo_entry); +} + +/* + * Search for an existing socket option. + */ +struct mptopt * +mptcp_sopt_find(struct mptses *mpte, struct sockopt *sopt) +{ + struct mptopt *mpo; + + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + + TAILQ_FOREACH(mpo, &mpte->mpte_sopts, mpo_entry) { + if (mpo->mpo_level == sopt->sopt_level && + mpo->mpo_name == sopt->sopt_name) + break; + } + VERIFY(mpo == NULL || sopt->sopt_valsize == sizeof (int)); + + return (mpo); +} + +/* + * Allocate a MPTCP subflow structure. + */ +static struct mptsub * +mptcp_subflow_alloc(void) +{ + struct mptsub *mpts = zalloc(mptsub_zone); + + if (mpts == NULL) + return (NULL); + + bzero(mpts, mptsub_zone_size); return (mpts); } @@ -661,11 +1134,9 @@ mptcp_subflow_alloc(int how) * Deallocate a subflow structure, called when all of the references held * on it have been released. This implies that the subflow has been deleted. */ -void +static void mptcp_subflow_free(struct mptsub *mpts) { - MPTS_LOCK_ASSERT_HELD(mpts); - VERIFY(mpts->mpts_refcnt == 0); VERIFY(!(mpts->mpts_flags & MPTSF_ATTACHED)); VERIFY(mpts->mpts_mpte == NULL); @@ -675,30 +1146,128 @@ mptcp_subflow_free(struct mptsub *mpts) FREE(mpts->mpts_src, M_SONAME); mpts->mpts_src = NULL; } - if (mpts->mpts_dst != NULL) { - FREE(mpts->mpts_dst, M_SONAME); - mpts->mpts_dst = NULL; - } - MPTS_UNLOCK(mpts); - lck_mtx_destroy(&mpts->mpts_lock, mtcbinfo.mppi_lock_grp); zfree(mptsub_zone, mpts); } +static void +mptcp_subflow_addref(struct mptsub *mpts) +{ + if (++mpts->mpts_refcnt == 0) + panic("%s: mpts %p wraparound refcnt\n", __func__, mpts); + /* NOTREACHED */ +} + +static void +mptcp_subflow_remref(struct mptsub *mpts) +{ + if (mpts->mpts_refcnt == 0) { + panic("%s: mpts %p negative refcnt\n", __func__, mpts); + /* NOTREACHED */ + } + if (--mpts->mpts_refcnt > 0) + return; + + /* callee will unlock and destroy lock */ + mptcp_subflow_free(mpts); +} + +static void +mptcp_subflow_attach(struct mptses *mpte, struct mptsub *mpts, struct socket *so) +{ + struct socket *mp_so = mpte->mpte_mppcb->mpp_socket; + struct tcpcb *tp = sototcpcb(so); + + /* + * From this moment on, the subflow is linked to the MPTCP-connection. + * Locking,... happens now at the MPTCP-layer + */ + tp->t_mptcb = mpte->mpte_mptcb; + so->so_flags |= SOF_MP_SUBFLOW; + mp_so->so_usecount++; + + /* + * Insert the subflow into the list, and associate the MPTCP PCB + * as well as the the subflow socket. From this point on, removing + * the subflow needs to be done via mptcp_subflow_del(). + */ + TAILQ_INSERT_TAIL(&mpte->mpte_subflows, mpts, mpts_entry); + mpte->mpte_numflows++; + + atomic_bitset_32(&mpts->mpts_flags, MPTSF_ATTACHED); + mpts->mpts_mpte = mpte; + mpts->mpts_socket = so; + tp->t_mpsub = mpts; + mptcp_subflow_addref(mpts); /* for being in MPTCP subflow list */ + mptcp_subflow_addref(mpts); /* for subflow socket */ +} + +static void +mptcp_subflow_necp_cb(void *handle, __unused int action, + __unused struct necp_client_flow *flow) +{ + struct inpcb *inp = (struct inpcb *)handle; + struct socket *so = inp->inp_socket; + struct mptsub *mpts; + struct mptses *mpte; + + if (action != NECP_CLIENT_CBACTION_NONVIABLE) + return; + + /* + * The socket is being garbage-collected. There is nothing to be done + * here. + */ + if (so->so_usecount == 0) + return; + + socket_lock(so, 1); + + /* Check again after we acquired the lock. */ + if (so->so_usecount == 0) + goto out; + + mpte = tptomptp(sototcpcb(so))->mpt_mpte; + mpts = sototcpcb(so)->t_mpsub; + + mptcplog((LOG_DEBUG, "%s: Subflow became non-viable", __func__), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); + + mpts->mpts_flags |= MPTSF_CLOSE_REQD; + + mptcp_sched_create_subflows(mpte); + + if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER) + flow->viable = 1; + +out: + socket_unlock(so, 1); +} + /* * Create an MPTCP subflow socket. */ static int mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, - struct proc *p, struct socket **so) + struct socket **so) { + lck_mtx_t *subflow_mtx; struct mptopt smpo, *mpo, *tmpo; + struct proc *p; struct socket *mp_so; int error; *so = NULL; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); + + p = proc_find(mp_so->last_pid); + if (p == PROC_NULL) { + mptcplog((LOG_ERR, "%s: Couldn't find proc for pid %u\n", __func__, mp_so->last_pid), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + return (ESRCH); + } /* * Create the subflow socket (multipath subflow, non-blocking.) @@ -708,19 +1277,49 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, * It also indicates to the underlying TCP to handle MPTCP options. * A multipath subflow socket implies SS_NOFDREF state. */ - if ((error = socreate_internal(dom, so, SOCK_STREAM, - IPPROTO_TCP, p, SOCF_ASYNC | SOCF_MP_SUBFLOW, PROC_NULL)) != 0) { - mptcplog((LOG_ERR, "MPTCP Socket: subflow socreate mp_so 0x%llx" - " unable to create subflow socket error %d\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), error), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + /* + * Unlock, because tcp_usr_attach ends up in in_pcballoc, which takes + * the ipi-lock. We cannot hold the socket-lock at that point. + */ + mpte_unlock(mpte); + error = socreate_internal(dom, so, SOCK_STREAM, IPPROTO_TCP, p, + SOCF_ASYNC, PROC_NULL); + mpte_lock(mpte); + if (error) { + mptcplog((LOG_ERR, "%s: subflow socreate mp_so 0x%llx unable to create subflow socket error %d\n", + (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + proc_rele(p); + + mptcp_subflow_free(mpts); return (error); } - socket_lock(*so, 0); - VERIFY((*so)->so_flags & SOF_MP_SUBFLOW); - VERIFY(((*so)->so_state & (SS_NBIO|SS_NOFDREF)) == - (SS_NBIO|SS_NOFDREF)); + /* + * We need to protect the setting of SOF_MP_SUBFLOW with a lock, because + * this marks the moment of lock-switch from the TCP-lock to the MPTCP-lock. + * Which is why we also need to get the lock with pr_getlock, as after + * setting the flag, socket_unlock will work on the MPTCP-level lock. + */ + subflow_mtx = ((*so)->so_proto->pr_getlock)(*so, 0); + lck_mtx_lock(subflow_mtx); + + /* + * Must be the first thing we do, to make sure all pointers for this + * subflow are set. + */ + mptcp_subflow_attach(mpte, mpts, *so); + + /* + * A multipath subflow socket is used internally in the kernel, + * therefore it does not have a file desciptor associated by + * default. + */ + (*so)->so_state |= SS_NOFDREF; + + lck_mtx_unlock(subflow_mtx); /* prevent the socket buffers from being compressed */ (*so)->so_rcv.sb_flags |= SB_NOCOMPRESS; @@ -729,10 +1328,42 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, /* Inherit preconnect and TFO data flags */ if (mp_so->so_flags1 & SOF1_PRECONNECT_DATA) (*so)->so_flags1 |= SOF1_PRECONNECT_DATA; - if (mp_so->so_flags1 & SOF1_DATA_IDEMPOTENT) (*so)->so_flags1 |= SOF1_DATA_IDEMPOTENT; + /* Inherit uuid and create the related flow. */ + if (!uuid_is_null(mpsotomppcb(mp_so)->necp_client_uuid)) { + struct mptcb *mp_tp = mpte->mpte_mptcb; + + sotoinpcb(*so)->necp_cb = mptcp_subflow_necp_cb; + + /* + * A note on the unlock: With MPTCP, we do multiple times a + * necp_client_register_socket_flow. This is problematic, + * because now the lock-ordering guarantee (first necp-locks, + * then socket-locks) is no more respected. So, we need to + * unlock here. + */ + mpte_unlock(mpte); + error = necp_client_register_socket_flow(mp_so->last_pid, + mpsotomppcb(mp_so)->necp_client_uuid, sotoinpcb(*so)); + mpte_lock(mpte); + + if (error) + goto out_err; + + /* Possible state-change during the unlock above */ + if (mp_tp->mpt_state >= MPTCPS_TIME_WAIT || + (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP)) + goto out_err; + + uuid_copy(sotoinpcb(*so)->necp_client_uuid, mpsotomppcb(mp_so)->necp_client_uuid); + } else { + mptcplog((LOG_NOTICE, "%s: uuid is not set!\n"), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + } + + /* inherit the other socket options */ bzero(&smpo, sizeof (smpo)); smpo.mpo_flags |= MPOF_SUBFLOW_OK; smpo.mpo_level = SOL_SOCKET; @@ -740,42 +1371,36 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, /* disable SIGPIPE */ smpo.mpo_name = SO_NOSIGPIPE; - if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0) - goto out; + if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != 0) + goto out_err; /* find out if the subflow's source address goes away */ smpo.mpo_name = SO_NOADDRERR; - if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0) - goto out; + if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != 0) + goto out_err; /* enable keepalive */ smpo.mpo_name = SO_KEEPALIVE; - if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0) - goto out; - - /* - * Limit the receive socket buffer size to 64k. - * - * We need to take into consideration the window scale option - * which could be negotiated in one subflow but disabled in - * another subflow. - * XXX This can be improved in the future. - */ - smpo.mpo_name = SO_RCVBUF; - smpo.mpo_intval = MPTCP_RWIN_MAX; - if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0) - goto out; - - /* N.B.: set by sosetopt */ - VERIFY(!((*so)->so_rcv.sb_flags & SB_AUTOSIZE)); - /* Prevent automatic socket buffer sizing. */ - (*so)->so_snd.sb_flags &= ~SB_AUTOSIZE; + if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != 0) + goto out_err; smpo.mpo_level = IPPROTO_TCP; smpo.mpo_intval = mptcp_subflow_keeptime; smpo.mpo_name = TCP_KEEPALIVE; - if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0) - goto out; + if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != 0) + goto out_err; + + if (mpte->mpte_mptcb->mpt_state >= MPTCPS_ESTABLISHED) { + /* + * On secondary subflows we might need to set the cell-fallback + * flag (see conditions in mptcp_subflow_sosetopt). + */ + smpo.mpo_level = SOL_SOCKET; + smpo.mpo_name = SO_MARK_CELLFALLBACK; + smpo.mpo_intval = 1; + if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != 0) + goto out_err; + } /* replay setsockopt(2) on the subflow sockets for eligible options */ TAILQ_FOREACH_SAFE(mpo, &mpte->mpte_sopts, mpo_entry, tmpo) { @@ -796,14 +1421,12 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, continue; interim = (mpo->mpo_flags & MPOF_INTERIM); - if (mptcp_subflow_sosetopt(mpte, *so, mpo) != 0 && interim) { - char buf[32]; - mptcplog((LOG_ERR, "MPTCP Socket: subflow socreate" - " mp_so 0x%llx" - " sopt %s val %d interim record removed\n", + if (mptcp_subflow_sosetopt(mpte, mpts, mpo) != 0 && interim) { + mptcplog((LOG_ERR, "%s: subflow socreate mp_so 0x%llx" + " sopt %s val %d interim record removed\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name, - buf, sizeof (buf)), mpo->mpo_intval), + mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), + mpo->mpo_intval), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); mptcp_sopt_remove(mpte, mpo); mptcp_sopt_free(mpo); @@ -816,7 +1439,6 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, * so use a customized socket receive function. We will undo * this when the socket is peeled off or closed. */ - mpts->mpts_oprotosw = (*so)->so_proto; switch (dom) { case PF_INET: (*so)->so_proto = &mptcp_subflow_protosw; @@ -831,11 +1453,20 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, /* NOTREACHED */ } -out: - socket_unlock(*so, 0); + proc_rele(p); + + DTRACE_MPTCP3(subflow__create, struct mptses *, mpte, + int, dom, int, error); + + return (0); - DTRACE_MPTCP4(subflow__create, struct mptses *, mpte, - struct mptsub *, mpts, int, dom, int, error); +out_err: + mptcp_subflow_abort(mpts, error); + + proc_rele(p); + + mptcplog((LOG_ERR, "%s: subflow socreate failed with error %d\n", + __func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); return (error); } @@ -846,95 +1477,117 @@ out: * Note that this may be called on an embryonic subflow, and the only * thing that is guaranteed valid is the protocol-user request. */ -static int -mptcp_subflow_soclose(struct mptsub *mpts, struct socket *so) +static void +mptcp_subflow_soclose(struct mptsub *mpts) { - MPTS_LOCK_ASSERT_HELD(mpts); + struct socket *so = mpts->mpts_socket; + + if (mpts->mpts_flags & MPTSF_CLOSED) + return; - socket_lock(so, 0); + VERIFY(so != NULL); VERIFY(so->so_flags & SOF_MP_SUBFLOW); VERIFY((so->so_state & (SS_NBIO|SS_NOFDREF)) == (SS_NBIO|SS_NOFDREF)); - /* restore protocol-user requests */ - VERIFY(mpts->mpts_oprotosw != NULL); - so->so_proto = mpts->mpts_oprotosw; - socket_unlock(so, 0); - - mpts->mpts_socket = NULL; /* may already be NULL */ - DTRACE_MPTCP5(subflow__close, struct mptsub *, mpts, struct socket *, so, struct sockbuf *, &so->so_rcv, struct sockbuf *, &so->so_snd, struct mptses *, mpts->mpts_mpte); - return (soclose(so)); + mpts->mpts_flags |= MPTSF_CLOSED; + + if (so->so_retaincnt == 0) { + soclose_locked(so); + + return; + } else { + VERIFY(so->so_usecount > 0); + so->so_usecount--; + } + + return; } /* * Connect an MPTCP subflow socket. * - * This may be called inline as part of adding a subflow, or asynchronously - * by the thread (upon progressing to MPTCPF_JOIN_READY). Note that in the - * pending connect case, the subflow socket may have been bound to an interface - * and/or a source IP address which may no longer be around by the time this - * routine is called; in that case the connect attempt will most likely fail. + * Note that in the pending connect case, the subflow socket may have been + * bound to an interface and/or a source IP address which may no longer be + * around by the time this routine is called; in that case the connect attempt + * will most likely fail. */ static int mptcp_subflow_soconnectx(struct mptses *mpte, struct mptsub *mpts) { - struct socket *so; + char dbuf[MAX_IPv6_STR_LEN]; + struct socket *mp_so, *so; + struct mptcb *mp_tp; + struct sockaddr *dst; + struct proc *p; int af, error; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - - VERIFY((mpts->mpts_flags & (MPTSF_CONNECTING|MPTSF_CONNECTED)) == - MPTSF_CONNECTING); - VERIFY(mpts->mpts_socket != NULL); - so = mpts->mpts_socket; - af = mpts->mpts_family; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ - if (af == AF_INET || af == AF_INET6) { - struct sockaddr *dst; - char dbuf[MAX_IPv6_STR_LEN]; + mp_so = mptetoso(mpte); + mp_tp = mpte->mpte_mptcb; - dst = mpts->mpts_dst; + p = proc_find(mp_so->last_pid); + if (p == PROC_NULL) { + mptcplog((LOG_ERR, "%s: Couldn't find proc for pid %u\n", __func__, mp_so->last_pid), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); - mptcplog((LOG_DEBUG, "MPTCP Socket: connectx mp_so 0x%llx " - "dst %s[%d] cid %d [pended %s]\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mpte->mpte_mppcb->mpp_socket), - inet_ntop(af, ((af == AF_INET) ? - (void *)&SIN(dst)->sin_addr.s_addr : - (void *)&SIN6(dst)->sin6_addr), - dbuf, sizeof (dbuf)), ((af == AF_INET) ? - ntohs(SIN(dst)->sin_port) : - ntohs(SIN6(dst)->sin6_port)), - mpts->mpts_connid, - ((mpts->mpts_flags & MPTSF_CONNECT_PENDING) ? - "YES" : "NO")), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + return (ESRCH); } + so = mpts->mpts_socket; + af = mpts->mpts_dst.sa_family; + + VERIFY((mpts->mpts_flags & (MPTSF_CONNECTING|MPTSF_CONNECTED)) == MPTSF_CONNECTING); + VERIFY(mpts->mpts_socket != NULL); + VERIFY(af == AF_INET || af == AF_INET6); + + dst = &mpts->mpts_dst; + mptcplog((LOG_DEBUG, "%s: connectx mp_so 0x%llx dst %s[%d] cid %d [pended %s]\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + inet_ntop(af, ((af == AF_INET) ? (void *)&SIN(dst)->sin_addr.s_addr : + (void *)&SIN6(dst)->sin6_addr), + dbuf, sizeof (dbuf)), + ((af == AF_INET) ? ntohs(SIN(dst)->sin_port) : ntohs(SIN6(dst)->sin6_port)), + mpts->mpts_connid, + ((mpts->mpts_flags & MPTSF_CONNECT_PENDING) ? "YES" : "NO")), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + mpts->mpts_flags &= ~MPTSF_CONNECT_PENDING; - socket_lock(so, 0); mptcp_attach_to_subf(so, mpte->mpte_mptcb, mpte->mpte_addrid_last); /* connect the subflow socket */ - error = soconnectxlocked(so, mpts->mpts_src, mpts->mpts_dst, - mpts->mpts_mpcr.mpcr_proc, mpts->mpts_mpcr.mpcr_ifscope, - mpte->mpte_associd, NULL, CONNREQF_MPTCP, - &mpts->mpts_mpcr, sizeof (mpts->mpts_mpcr), NULL, NULL); - socket_unlock(so, 0); + error = soconnectxlocked(so, mpts->mpts_src, &mpts->mpts_dst, + p, mpts->mpts_ifscope, + mpte->mpte_associd, NULL, 0, NULL, 0, NULL, NULL); + + mpts->mpts_iss = sototcpcb(so)->iss; + + /* See tcp_connect_complete */ + if (mp_tp->mpt_state < MPTCPS_ESTABLISHED && + (mp_so->so_flags1 & SOF1_PRECONNECT_DATA)) { + mp_tp->mpt_sndwnd = sototcpcb(so)->snd_wnd; + } /* Allocate a unique address id per subflow */ mpte->mpte_addrid_last++; if (mpte->mpte_addrid_last == 0) mpte->mpte_addrid_last++; + proc_rele(p); + DTRACE_MPTCP3(subflow__connect, struct mptses *, mpte, struct mptsub *, mpts, int, error); + if (error) + mptcplog((LOG_ERR, "%s: connectx failed with error %d ifscope %u\n", + __func__, error, mpts->mpts_ifscope), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); return (error); } @@ -947,12 +1600,13 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp) { #pragma unused(uio) + struct socket *mp_so = mptetoso(tptomptp(sototcpcb(so))->mpt_mpte); int flags, error = 0; struct proc *p = current_proc(); struct mbuf *m, **mp = mp0; - struct mbuf *nextrecord; + boolean_t proc_held = FALSE; - socket_lock(so, 1); + mpte_lock_assert_held(tptomptp(sototcpcb(so))->mpt_mpte); VERIFY(so->so_proto->pr_flags & PR_CONNREQUIRED); #ifdef MORE_LOCKING_DEBUG @@ -966,10 +1620,9 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, * to the MPTCP layer, so we require that the caller passes in the * expected parameters. */ - if (mp == NULL || controlp != NULL) { - socket_unlock(so, 1); + if (mp == NULL || controlp != NULL) return (EINVAL); - } + *mp = NULL; if (psa != NULL) *psa = NULL; @@ -978,10 +1631,9 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, else flags = 0; - if (flags & (MSG_PEEK|MSG_OOB|MSG_NEEDSA|MSG_WAITALL|MSG_WAITSTREAM)) { - socket_unlock(so, 1); + if (flags & (MSG_PEEK|MSG_OOB|MSG_NEEDSA|MSG_WAITALL|MSG_WAITSTREAM)) return (EOPNOTSUPP); - } + flags |= (MSG_DONTWAIT|MSG_NBIO); /* @@ -993,10 +1645,6 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, struct sockbuf *sb = &so->so_rcv; error = ENOTCONN; - SODEFUNCTLOG("%s[%d, %s]: defunct so 0x%llx [%d,%d] (%d)\n", - __func__, proc_pid(p), proc_best_name(p), - (uint64_t)VM_KERNEL_ADDRPERM(so), - SOCK_DOM(so), SOCK_TYPE(so), error); /* * This socket should have been disconnected and flushed * prior to being returned from sodefunct(); there should @@ -1004,7 +1652,6 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, */ if (so->so_state & SS_DEFUNCT) sb_empty_assert(sb, __func__); - socket_unlock(so, 1); return (error); } @@ -1025,20 +1672,16 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, * socket is closed for real, SOF_MP_SUBFLOW would be cleared. */ if ((so->so_state & (SS_NOFDREF | SS_CANTRCVMORE)) == - (SS_NOFDREF | SS_CANTRCVMORE) && !(so->so_flags & SOF_MP_SUBFLOW)) { - socket_unlock(so, 1); + (SS_NOFDREF | SS_CANTRCVMORE) && !(so->so_flags & SOF_MP_SUBFLOW)) return (0); - } /* * For consistency with soreceive() semantics, we need to obey * SB_LOCK in case some other code path has locked the buffer. */ error = sblock(&so->so_rcv, 0); - if (error != 0) { - socket_unlock(so, 1); + if (error != 0) return (error); - } m = so->so_rcv.sb_mb; if (m == NULL) { @@ -1057,158 +1700,223 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, goto release; } - if (so->so_state & SS_CANTRCVMORE) { - goto release; + if (so->so_state & SS_CANTRCVMORE) { + goto release; + } + + if (!(so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING))) { + error = ENOTCONN; + goto release; + } + + /* + * MSG_DONTWAIT is implicitly defined and this routine will + * never block, so return EWOULDBLOCK when there is nothing. + */ + error = EWOULDBLOCK; + goto release; + } + + mptcp_update_last_owner(so, mp_so); + + if (mp_so->last_pid != proc_pid(p)) { + p = proc_find(mp_so->last_pid); + if (p == PROC_NULL) { + p = current_proc(); + } else { + proc_held = TRUE; + } + } + + OSIncrementAtomicLong(&p->p_stats->p_ru.ru_msgrcv); + SBLASTRECORDCHK(&so->so_rcv, "mptcp_subflow_soreceive 1"); + SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 1"); + + while (m != NULL) { + int dlen = 0; + struct mbuf *start = m; + uint64_t dsn; + uint32_t sseq; + uint16_t orig_dlen; + uint16_t csum; + + VERIFY(m->m_nextpkt == NULL); + + if ((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)) { + orig_dlen = dlen = m->m_pkthdr.mp_rlen; + dsn = m->m_pkthdr.mp_dsn; + sseq = m->m_pkthdr.mp_rseq; + csum = m->m_pkthdr.mp_csum; + } else { + /* We did fallback */ + mptcp_adj_rmap(so, m, 0); + + sbfree(&so->so_rcv, m); + + if (mp != NULL) { + *mp = m; + mp = &m->m_next; + so->so_rcv.sb_mb = m = m->m_next; + *mp = NULL; + + } + + if (m != NULL) { + so->so_rcv.sb_lastrecord = m; + } else { + SB_EMPTY_FIXUP(&so->so_rcv); + } + + continue; } - if (!(so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING))) { - error = ENOTCONN; + /* + * Check if the full mapping is now present + */ + if ((int)so->so_rcv.sb_cc < dlen) { + mptcplog((LOG_INFO, "%s not enough data (%u) need %u\n", + __func__, so->so_rcv.sb_cc, dlen), + MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG); + + if (*mp0 == NULL) + error = EWOULDBLOCK; goto release; } - /* - * MSG_DONTWAIT is implicitly defined and this routine will - * never block, so return EWOULDBLOCK when there is nothing. - */ - error = EWOULDBLOCK; - goto release; - } + /* Now, get the full mapping */ + while (dlen > 0) { + mptcp_adj_rmap(so, m, orig_dlen - dlen); - OSIncrementAtomicLong(&p->p_stats->p_ru.ru_msgrcv); - SBLASTRECORDCHK(&so->so_rcv, "mptcp_subflow_soreceive 1"); - SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 1"); + dlen -= m->m_len; + sbfree(&so->so_rcv, m); - while (m != NULL) { - nextrecord = m->m_nextpkt; - sbfree(&so->so_rcv, m); - - if (mp != NULL) { - *mp = m; - mp = &m->m_next; - so->so_rcv.sb_mb = m = m->m_next; - *mp = NULL; + if (mp != NULL) { + *mp = m; + mp = &m->m_next; + so->so_rcv.sb_mb = m = m->m_next; + *mp = NULL; + } + + VERIFY(dlen <= 0 || m); } + VERIFY(dlen == 0); + if (m != NULL) { - m->m_nextpkt = nextrecord; - if (nextrecord == NULL) - so->so_rcv.sb_lastrecord = m; + so->so_rcv.sb_lastrecord = m; } else { - m = so->so_rcv.sb_mb = nextrecord; SB_EMPTY_FIXUP(&so->so_rcv); } + + if (mptcp_validate_csum(sototcpcb(so), start, dsn, sseq, orig_dlen, csum)) { + error = EIO; + *mp0 = NULL; + goto release; + } + SBLASTRECORDCHK(&so->so_rcv, "mptcp_subflow_soreceive 2"); SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 2"); } DTRACE_MPTCP3(subflow__receive, struct socket *, so, struct sockbuf *, &so->so_rcv, struct sockbuf *, &so->so_snd); - /* notify protocol that we drained all the data */ - if ((so->so_proto->pr_flags & PR_WANTRCVD) && so->so_pcb != NULL) - (*so->so_proto->pr_usrreqs->pru_rcvd)(so, flags); if (flagsp != NULL) *flagsp |= flags; release: - sbunlock(&so->so_rcv, FALSE); /* will unlock socket */ + sbunlock(&so->so_rcv, TRUE); + + if (proc_held) + proc_rele(p); + return (error); } - /* - * Prepare an MPTCP subflow socket for peeloff(2); basically undo - * the work done earlier when the subflow socket was created. + * MPTCP subflow socket send routine, derived from sosend(). */ -void -mptcp_subflow_sopeeloff(struct mptses *mpte, struct mptsub *mpts, - struct socket *so) +static int +mptcp_subflow_sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, + struct mbuf *top, struct mbuf *control, int flags) { - struct mptopt smpo; - struct socket *mp_so; - int p, c; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; - MPTS_LOCK_ASSERT_HELD(mpts); + struct socket *mp_so = mptetoso(tptomptp(sototcpcb(so))->mpt_mpte); + struct proc *p = current_proc(); + boolean_t en_tracing = FALSE, proc_held = FALSE; + int en_tracing_val; + int sblocked = 1; /* Pretend as if it is already locked, so we won't relock it */ + int error; - socket_lock(so, 0); - VERIFY(so->so_flags & SOF_MP_SUBFLOW); - VERIFY((so->so_state & (SS_NBIO|SS_NOFDREF)) == (SS_NBIO|SS_NOFDREF)); + VERIFY(control == NULL); + VERIFY(addr == NULL); + VERIFY(uio == NULL); + VERIFY(flags == 0); + VERIFY((so->so_flags & SOF_CONTENT_FILTER) == 0); - /* inherit MPTCP socket states */ - if (!(mp_so->so_state & SS_NBIO)) - so->so_state &= ~SS_NBIO; + VERIFY(top->m_pkthdr.len > 0 && top->m_pkthdr.len <= UINT16_MAX); + VERIFY(top->m_pkthdr.pkt_flags & PKTF_MPTCP); /* - * At this point, the socket is not yet closed, as there is at least - * one outstanding usecount previously held by mpts_socket from - * socreate(). Atomically clear SOF_MP_SUBFLOW and SS_NOFDREF here. + * trace if tracing & network (vs. unix) sockets & and + * non-loopback */ - so->so_flags &= ~SOF_MP_SUBFLOW; - so->so_state &= ~SS_NOFDREF; - so->so_flags &= ~SOF_MPTCP_TRUE; + if (ENTR_SHOULDTRACE && + (SOCK_CHECK_DOM(so, AF_INET) || SOCK_CHECK_DOM(so, AF_INET6))) { + struct inpcb *inp = sotoinpcb(so); + if (inp->inp_last_outifp != NULL && + !(inp->inp_last_outifp->if_flags & IFF_LOOPBACK)) { + en_tracing = TRUE; + en_tracing_val = top->m_pkthdr.len; + KERNEL_ENERGYTRACE(kEnTrActKernSockWrite, DBG_FUNC_START, + VM_KERNEL_ADDRPERM(so), + ((so->so_state & SS_NBIO) ? kEnTrFlagNonBlocking : 0), + (int64_t)en_tracing_val); + } + } - /* allow socket buffers to be compressed */ - so->so_rcv.sb_flags &= ~SB_NOCOMPRESS; - so->so_snd.sb_flags &= ~SB_NOCOMPRESS; + mptcp_update_last_owner(so, mp_so); - /* - * Allow socket buffer auto sizing. - * - * This will increase the current 64k buffer size to whatever is best. - */ - if (!(so->so_rcv.sb_flags & SB_USRSIZE)) - so->so_rcv.sb_flags |= SB_AUTOSIZE; - if (!(so->so_snd.sb_flags & SB_USRSIZE)) - so->so_snd.sb_flags |= SB_AUTOSIZE; + if (mp_so->last_pid != proc_pid(p)) { + p = proc_find(mp_so->last_pid); + if (p == PROC_NULL) { + p = current_proc(); + } else { + proc_held = TRUE; + } + } - /* restore protocol-user requests */ - VERIFY(mpts->mpts_oprotosw != NULL); - so->so_proto = mpts->mpts_oprotosw; +#if NECP + inp_update_necp_policy(sotoinpcb(so), NULL, NULL, 0); +#endif /* NECP */ - bzero(&smpo, sizeof (smpo)); - smpo.mpo_flags |= MPOF_SUBFLOW_OK; - smpo.mpo_level = SOL_SOCKET; + OSIncrementAtomicLong(&p->p_stats->p_ru.ru_msgsnd); - /* inherit SOF_NOSIGPIPE from parent MP socket */ - p = (mp_so->so_flags & SOF_NOSIGPIPE); - c = (so->so_flags & SOF_NOSIGPIPE); - smpo.mpo_intval = ((p - c) > 0) ? 1 : 0; - smpo.mpo_name = SO_NOSIGPIPE; - if ((p - c) != 0) - (void) mptcp_subflow_sosetopt(mpte, so, &smpo); + error = sosendcheck(so, NULL, top->m_pkthdr.len, 0, 1, 0, &sblocked, NULL); + if (error) + goto out; - /* inherit SOF_NOADDRAVAIL from parent MP socket */ - p = (mp_so->so_flags & SOF_NOADDRAVAIL); - c = (so->so_flags & SOF_NOADDRAVAIL); - smpo.mpo_intval = ((p - c) > 0) ? 1 : 0; - smpo.mpo_name = SO_NOADDRERR; - if ((p - c) != 0) - (void) mptcp_subflow_sosetopt(mpte, so, &smpo); + error = (*so->so_proto->pr_usrreqs->pru_send)(so, 0, top, NULL, NULL, p); + top = NULL; - /* inherit SO_KEEPALIVE from parent MP socket */ - p = (mp_so->so_options & SO_KEEPALIVE); - c = (so->so_options & SO_KEEPALIVE); - smpo.mpo_intval = ((p - c) > 0) ? 1 : 0; - smpo.mpo_name = SO_KEEPALIVE; - if ((p - c) != 0) - (void) mptcp_subflow_sosetopt(mpte, so, &smpo); +out: + if (top != NULL) + m_freem(top); - /* unset TCP level default keepalive option */ - p = (intotcpcb(sotoinpcb(mp_so)))->t_keepidle; - c = (intotcpcb(sotoinpcb(so)))->t_keepidle; - smpo.mpo_level = IPPROTO_TCP; - smpo.mpo_intval = 0; - smpo.mpo_name = TCP_KEEPALIVE; - if ((p - c) != 0) - (void) mptcp_subflow_sosetopt(mpte, so, &smpo); - socket_unlock(so, 0); + if (proc_held) + proc_rele(p); + + soclearfastopen(so); + + if (en_tracing) { + KERNEL_ENERGYTRACE(kEnTrActKernSockWrite, DBG_FUNC_END, + VM_KERNEL_ADDRPERM(so), + ((error == EWOULDBLOCK) ? kEnTrFlagNoWork : 0), + (int64_t)en_tracing_val); + } + + return (error); - DTRACE_MPTCP5(subflow__peeloff, struct mptses *, mpte, - struct mptsub *, mpts, struct socket *, so, - struct sockbuf *, &so->so_rcv, struct sockbuf *, &so->so_snd); } /* @@ -1216,59 +1924,70 @@ mptcp_subflow_sopeeloff(struct mptses *mpte, struct mptsub *mpts, * connected), or add a subflow to an existing MPTCP connection. */ int -mptcp_subflow_add(struct mptses *mpte, struct mptsub *mpts, - struct proc *p, uint32_t ifscope) +mptcp_subflow_add(struct mptses *mpte, struct sockaddr *src, + struct sockaddr *dst, uint32_t ifscope, sae_connid_t *pcid) { struct socket *mp_so, *so = NULL; - struct mptsub_connreq mpcr; struct mptcb *mp_tp; + struct mptsub *mpts = NULL; int af, error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; - MPT_LOCK(mp_tp); if (mp_tp->mpt_state >= MPTCPS_CLOSE_WAIT) { /* If the remote end sends Data FIN, refuse subflow adds */ + mptcplog((LOG_ERR, "%s state %u\n", __func__, mp_tp->mpt_state), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); error = ENOTCONN; - MPT_UNLOCK(mp_tp); - return (error); + goto out_err; } - MPT_UNLOCK(mp_tp); - - MPTS_LOCK(mpts); - VERIFY(!(mpts->mpts_flags & (MPTSF_CONNECTING|MPTSF_CONNECTED))); - VERIFY(mpts->mpts_mpte == NULL); - VERIFY(mpts->mpts_socket == NULL); - VERIFY(mpts->mpts_dst != NULL); - VERIFY(mpts->mpts_connid == SAE_CONNID_ANY); - af = mpts->mpts_family = mpts->mpts_dst->sa_family; + mpts = mptcp_subflow_alloc(); + if (mpts == NULL) { + mptcplog((LOG_ERR, "%s malloc subflow failed\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = ENOMEM; + goto out_err; + } - /* - * If the source address is not specified, allocate a storage for - * it, so that later on we can fill it in with the actual source - * IP address chosen by the underlying layer for the subflow after - * it is connected. - */ - if (mpts->mpts_src == NULL) { - int len = mpts->mpts_dst->sa_len; + if (src != NULL) { + int len = src->sa_len; MALLOC(mpts->mpts_src, struct sockaddr *, len, M_SONAME, M_WAITOK | M_ZERO); if (mpts->mpts_src == NULL) { - error = ENOBUFS; - goto out; + mptcplog((LOG_ERR, "%s malloc mpts_src failed", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = ENOMEM; + goto out_err; } - bzero(mpts->mpts_src, len); - mpts->mpts_src->sa_len = len; - mpts->mpts_src->sa_family = mpts->mpts_dst->sa_family; + bcopy(src, mpts->mpts_src, len); } + memcpy(&mpts->mpts_dst, dst, dst->sa_len); + + af = mpts->mpts_dst.sa_family; + + mpts->mpts_ifscope = ifscope; + /* create the subflow socket */ - if ((error = mptcp_subflow_socreate(mpte, mpts, af, p, &so)) != 0) - goto out; + if ((error = mptcp_subflow_socreate(mpte, mpts, af, &so)) != 0) + /* + * Returning (error) and not cleaning up, because up to here + * all we did is creating mpts. + * + * And the contract is that the call to mptcp_subflow_socreate, + * moves ownership of mpts to mptcp_subflow_socreate. + */ + return (error); + + /* + * We may be called from within the kernel. Still need to account this + * one to the real app. + */ + mptcp_update_last_owner(mpts->mpts_socket, mp_so); /* * Increment the counter, while avoiding 0 (SAE_CONNID_ANY) and @@ -1280,8 +1999,6 @@ mptcp_subflow_add(struct mptses *mpte, struct mptsub *mpts, mpte->mpte_connid_last++; mpts->mpts_connid = mpte->mpte_connid_last; - VERIFY(mpts->mpts_connid != SAE_CONNID_ANY && - mpts->mpts_connid != SAE_CONNID_ALL); mpts->mpts_rel_seq = 1; @@ -1290,169 +2007,45 @@ mptcp_subflow_add(struct mptses *mpte, struct mptsub *mpts, if (mpte->mpte_addrid_last == 0) mpte->mpte_addrid_last++; - /* bind subflow socket to the specified interface */ - if (ifscope != IFSCOPE_NONE) { - socket_lock(so, 0); - error = inp_bindif(sotoinpcb(so), ifscope, &mpts->mpts_outif); - if (error != 0) { - socket_unlock(so, 0); - (void) mptcp_subflow_soclose(mpts, so); - goto out; - } - VERIFY(mpts->mpts_outif != NULL); - mpts->mpts_flags |= MPTSF_BOUND_IF; - - if (IFNET_IS_EXPENSIVE(mpts->mpts_outif)) { - sototcpcb(so)->t_mpflags |= TMPF_BACKUP_PATH; - } else { - mpts->mpts_flags |= MPTSF_PREFERRED; - } - - mptcplog((LOG_DEBUG, "MPTCP Socket: subflow_add mp_so 0x%llx " - "bindif %s[%d] cid %d expensive %d\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mpts->mpts_outif->if_xname, - ifscope, mpts->mpts_connid, - IFNET_IS_EXPENSIVE(mpts->mpts_outif)), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - socket_unlock(so, 0); - } - - /* if source address and/or port is specified, bind to it */ - if (mpts->mpts_src != NULL) { - struct sockaddr *sa = mpts->mpts_src; - uint32_t mpts_flags = 0; - in_port_t lport; - - switch (af) { - case AF_INET: - if (SIN(sa)->sin_addr.s_addr != INADDR_ANY) - mpts_flags |= MPTSF_BOUND_IP; - if ((lport = SIN(sa)->sin_port) != 0) - mpts_flags |= MPTSF_BOUND_PORT; - break; -#if INET6 - case AF_INET6: - VERIFY(af == AF_INET6); - if (!IN6_IS_ADDR_UNSPECIFIED(&SIN6(sa)->sin6_addr)) - mpts_flags |= MPTSF_BOUND_IP; - if ((lport = SIN6(sa)->sin6_port) != 0) - mpts_flags |= MPTSF_BOUND_PORT; - break; -#endif /* INET6 */ - } - - error = sobindlock(so, sa, 1); /* will lock/unlock socket */ - if (error != 0) { - (void) mptcp_subflow_soclose(mpts, so); - goto out; - } - mpts->mpts_flags |= mpts_flags; - - if (af == AF_INET || af == AF_INET6) { - char sbuf[MAX_IPv6_STR_LEN]; - - mptcplog((LOG_DEBUG, "MPTCP Socket: subflow_add " - "mp_so 0x%llx bindip %s[%d] cid %d\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - inet_ntop(af, ((af == AF_INET) ? - (void *)&SIN(sa)->sin_addr.s_addr : - (void *)&SIN6(sa)->sin6_addr), sbuf, sizeof (sbuf)), - ntohs(lport), mpts->mpts_connid), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - } - } - - /* - * Insert the subflow into the list, and associate the MPTCP PCB - * as well as the the subflow socket. From this point on, removing - * the subflow needs to be done via mptcp_subflow_del(). - */ - TAILQ_INSERT_TAIL(&mpte->mpte_subflows, mpts, mpts_entry); - mpte->mpte_numflows++; - - atomic_bitset_32(&mpts->mpts_flags, MPTSF_ATTACHED); - mpts->mpts_mpte = mpte; - mpts->mpts_socket = so; - MPTS_ADDREF_LOCKED(mpts); /* for being in MPTCP subflow list */ - MPTS_ADDREF_LOCKED(mpts); /* for subflow socket */ - mp_so->so_usecount++; /* for subflow socket */ - /* register for subflow socket read/write events */ - (void) sock_setupcalls(so, mptcp_subflow_rupcall, mpts, - mptcp_subflow_wupcall, mpts); + sock_setupcalls_locked(so, mptcp_subflow_rupcall, mpts, mptcp_subflow_wupcall, mpts, 1); - /* - * Register for subflow socket control events; ignore - * SO_FILT_HINT_CONNINFO_UPDATED from below since we - * will generate it here. - */ - (void) sock_catchevents(so, mptcp_subflow_eupcall, mpts, + /* Register for subflow socket control events */ + sock_catchevents_locked(so, mptcp_subflow_eupcall1, mpts, SO_FILT_HINT_CONNRESET | SO_FILT_HINT_CANTRCVMORE | - SO_FILT_HINT_CANTSENDMORE | SO_FILT_HINT_TIMEOUT | - SO_FILT_HINT_NOSRCADDR | SO_FILT_HINT_IFDENIED | - SO_FILT_HINT_SUSPEND | SO_FILT_HINT_RESUME | - SO_FILT_HINT_CONNECTED | SO_FILT_HINT_DISCONNECTED | - SO_FILT_HINT_MPFAILOVER | SO_FILT_HINT_MPSTATUS | - SO_FILT_HINT_MUSTRST | SO_FILT_HINT_MPFASTJ | - SO_FILT_HINT_DELETEOK | SO_FILT_HINT_MPCANTRCVMORE); + SO_FILT_HINT_TIMEOUT | SO_FILT_HINT_NOSRCADDR | + SO_FILT_HINT_IFDENIED | SO_FILT_HINT_CONNECTED | + SO_FILT_HINT_DISCONNECTED | SO_FILT_HINT_MPFAILOVER | + SO_FILT_HINT_MPSTATUS | SO_FILT_HINT_MUSTRST | + SO_FILT_HINT_MPCANTRCVMORE | SO_FILT_HINT_ADAPTIVE_RTIMO | + SO_FILT_HINT_ADAPTIVE_WTIMO); /* sanity check */ VERIFY(!(mpts->mpts_flags & (MPTSF_CONNECTING|MPTSF_CONNECTED|MPTSF_CONNECT_PENDING))); - bzero(&mpcr, sizeof (mpcr)); - mpcr.mpcr_proc = p; - mpcr.mpcr_ifscope = ifscope; /* * Indicate to the TCP subflow whether or not it should establish * the initial MPTCP connection, or join an existing one. Fill * in the connection request structure with additional info needed * by the underlying TCP (to be used in the TCP options, etc.) */ - MPT_LOCK(mp_tp); if (mp_tp->mpt_state < MPTCPS_ESTABLISHED && mpte->mpte_numflows == 1) { + mpts->mpts_flags |= MPTSF_INITIAL_SUB; + if (mp_tp->mpt_state == MPTCPS_CLOSED) { - mptcp_init_local_parms(mp_tp); + mptcp_init_local_parms(mpte); } - MPT_UNLOCK(mp_tp); soisconnecting(mp_so); - mpcr.mpcr_type = MPTSUB_CONNREQ_MP_ENABLE; + + /* If fastopen is requested, set state in mpts */ + if (so->so_flags1 & SOF1_PRECONNECT_DATA) + mpts->mpts_flags |= MPTSF_TFO_REQD; } else { if (!(mp_tp->mpt_flags & MPTCPF_JOIN_READY)) mpts->mpts_flags |= MPTSF_CONNECT_PENDING; - - /* avoid starting up cellular subflow unless required */ - if ((mptcp_delayed_subf_start) && - (IFNET_IS_CELLULAR(mpts->mpts_outif))) { - mpts->mpts_flags |= MPTSF_CONNECT_PENDING; - } - MPT_UNLOCK(mp_tp); - mpcr.mpcr_type = MPTSUB_CONNREQ_MP_ADD; - } - - /* If fastjoin or fastopen is requested, set state in mpts */ - if (mpte->mpte_nummpcapflows == 0) { - if (so->so_flags1 & SOF1_PRECONNECT_DATA) { - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - mpts->mpts_flags |= MPTSF_TFO_REQD; - mpts->mpts_sndnxt = mp_tp->mpt_snduna; - } - MPT_UNLOCK(mp_tp); - } - - if (so->so_flags & SOF_MPTCP_FASTJOIN) { - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state == MPTCPS_ESTABLISHED) { - mpts->mpts_flags |= MPTSF_FASTJ_REQD; - mpts->mpts_sndnxt = mp_tp->mpt_snduna; - } - MPT_UNLOCK(mp_tp); - } } - mpts->mpts_mpcr = mpcr; mpts->mpts_flags |= MPTSF_CONNECTING; if (af == AF_INET || af == AF_INET6) { @@ -1463,76 +2056,84 @@ mptcp_subflow_add(struct mptses *mpte, struct mptsub *mpts, "[pending %s]\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), inet_ntop(af, ((af == AF_INET) ? - (void *)&SIN(mpts->mpts_dst)->sin_addr.s_addr : - (void *)&SIN6(mpts->mpts_dst)->sin6_addr), + (void *)&SIN(&mpts->mpts_dst)->sin_addr.s_addr : + (void *)&SIN6(&mpts->mpts_dst)->sin6_addr), dbuf, sizeof (dbuf)), ((af == AF_INET) ? - ntohs(SIN(mpts->mpts_dst)->sin_port) : - ntohs(SIN6(mpts->mpts_dst)->sin6_port)), + ntohs(SIN(&mpts->mpts_dst)->sin_port) : + ntohs(SIN6(&mpts->mpts_dst)->sin6_port)), mpts->mpts_connid, ((mpts->mpts_flags & MPTSF_CONNECT_PENDING) ? "YES" : "NO")), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); } /* connect right away if first attempt, or if join can be done now */ if (!(mpts->mpts_flags & MPTSF_CONNECT_PENDING)) error = mptcp_subflow_soconnectx(mpte, mpts); -out: - MPTS_UNLOCK(mpts); - if (error == 0) { - soevent(mp_so, SO_FILT_HINT_LOCKED | - SO_FILT_HINT_CONNINFO_UPDATED); - } + if (error) + goto out_err_close; + + if (pcid) + *pcid = mpts->mpts_connid; + + return (0); + +out_err_close: + mptcp_subflow_abort(mpts, error); + + return (error); + +out_err: + if (mpts) + mptcp_subflow_free(mpts); + return (error); } +void +mptcpstats_update(struct mptcp_itf_stats *stats, struct mptsub *mpts) +{ + int index = mptcp_get_statsindex(stats, mpts); + + if (index != -1) { + struct inpcb *inp = sotoinpcb(mpts->mpts_socket); + + stats[index].mpis_txbytes += inp->inp_stat->txbytes; + stats[index].mpis_rxbytes += inp->inp_stat->rxbytes; + } +} + /* * Delete/remove a subflow from an MPTCP. The underlying subflow socket * will no longer be accessible after a subflow is deleted, thus this * should occur only after the subflow socket has been disconnected. - * If peeloff(2) is called, leave the socket open. */ void -mptcp_subflow_del(struct mptses *mpte, struct mptsub *mpts, boolean_t close) +mptcp_subflow_del(struct mptses *mpte, struct mptsub *mpts) { - struct socket *mp_so, *so; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; - - MPTS_LOCK(mpts); - so = mpts->mpts_socket; - VERIFY(so != NULL); + struct socket *mp_so = mptetoso(mpte); + struct socket *so = mpts->mpts_socket; + struct tcpcb *tp = sototcpcb(so); - if (close && !((mpts->mpts_flags & MPTSF_DELETEOK) && - (mpts->mpts_flags & MPTSF_USER_DISCONNECT))) { - MPTS_UNLOCK(mpts); - mptcplog((LOG_DEBUG, "MPTCP Socket: subflow_del returning" - " mp_so 0x%llx flags %x\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpts->mpts_flags), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - return; - } + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + VERIFY(mpts->mpts_mpte == mpte); + VERIFY(mpts->mpts_flags & MPTSF_ATTACHED); + VERIFY(mpte->mpte_numflows != 0); + VERIFY(mp_so->so_usecount > 0); - mptcplog((LOG_DEBUG, "MPTCP Socket: subflow_del mp_so 0x%llx " - "[u=%d,r=%d] cid %d [close %s] %d %x error %d\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mp_so->so_usecount, - mp_so->so_retaincnt, mpts->mpts_connid, - (close ? "YES" : "NO"), mpts->mpts_soerror, - mpts->mpts_flags, - mp_so->so_error), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx [u=%d,r=%d] cid %d %x error %d\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mp_so->so_usecount, mp_so->so_retaincnt, mpts->mpts_connid, + mpts->mpts_flags, mp_so->so_error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - VERIFY(mpts->mpts_mpte == mpte); - VERIFY(mpts->mpts_connid != SAE_CONNID_ANY && - mpts->mpts_connid != SAE_CONNID_ALL); + mptcpstats_update(mpte->mpte_itfstats, mpts); + mpte->mpte_init_rxbytes = sotoinpcb(so)->inp_stat->rxbytes; + mpte->mpte_init_txbytes = sotoinpcb(so)->inp_stat->txbytes; - VERIFY(mpts->mpts_flags & MPTSF_ATTACHED); atomic_bitclear_32(&mpts->mpts_flags, MPTSF_ATTACHED); TAILQ_REMOVE(&mpte->mpte_subflows, mpts, mpts_entry); - VERIFY(mpte->mpte_numflows != 0); mpte->mpte_numflows--; if (mpte->mpte_active_sub == mpts) mpte->mpte_active_sub = NULL; @@ -1541,73 +2142,94 @@ mptcp_subflow_del(struct mptses *mpte, struct mptsub *mpts, boolean_t close) * Drop references held by this subflow socket; there * will be no further upcalls made from this point. */ - (void) sock_setupcalls(so, NULL, NULL, NULL, NULL); - (void) sock_catchevents(so, NULL, NULL, 0); + sock_setupcalls_locked(so, NULL, NULL, NULL, NULL, 0); + sock_catchevents_locked(so, NULL, NULL, 0); mptcp_detach_mptcb_from_subf(mpte->mpte_mptcb, so); - if (close) - (void) mptcp_subflow_soclose(mpts, so); - - VERIFY(mp_so->so_usecount > 0); mp_so->so_usecount--; /* for subflow socket */ mpts->mpts_mpte = NULL; mpts->mpts_socket = NULL; - MPTS_UNLOCK(mpts); - MPTS_REMREF(mpts); /* for MPTCP subflow list */ - MPTS_REMREF(mpts); /* for subflow socket */ + mptcp_subflow_remref(mpts); /* for MPTCP subflow list */ + mptcp_subflow_remref(mpts); /* for subflow socket */ + + so->so_flags &= ~SOF_MP_SUBFLOW; + tp->t_mptcb = NULL; + tp->t_mpsub = NULL; +} + +void +mptcp_subflow_shutdown(struct mptses *mpte, struct mptsub *mpts) +{ + struct socket *so = mpts->mpts_socket; + struct mptcb *mp_tp = mpte->mpte_mptcb; + int send_dfin = 0; + + if (mp_tp->mpt_state > MPTCPS_CLOSE_WAIT) + send_dfin = 1; + + if (!(so->so_state & (SS_ISDISCONNECTING | SS_ISDISCONNECTED)) && + (so->so_state & SS_ISCONNECTED)) { + mptcplog((LOG_DEBUG, "MPTCP subflow shutdown %s: cid %d fin %d\n", + __func__, mpts->mpts_connid, send_dfin), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + if (send_dfin) + mptcp_send_dfin(so); + soshutdownlock(so, SHUT_WR); + } + +} + +static void +mptcp_subflow_abort(struct mptsub *mpts, int error) +{ + struct socket *so = mpts->mpts_socket; + struct tcpcb *tp = sototcpcb(so); + + if (mpts->mpts_flags & MPTSF_DISCONNECTED) + return; + + mptcplog((LOG_DEBUG, "%s aborting connection state %u\n", __func__, tp->t_state), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - soevent(mp_so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNINFO_UPDATED); + if (tp->t_state != TCPS_CLOSED) + tcp_drop(tp, error); + + mptcp_subflow_eupcall1(so, mpts, SO_FILT_HINT_DISCONNECTED); } /* * Disconnect a subflow socket. */ void -mptcp_subflow_disconnect(struct mptses *mpte, struct mptsub *mpts, - boolean_t deleteok) +mptcp_subflow_disconnect(struct mptses *mpte, struct mptsub *mpts) { struct socket *so; struct mptcb *mp_tp; int send_dfin = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpts->mpts_mpte == mpte); VERIFY(mpts->mpts_socket != NULL); - VERIFY(mpts->mpts_connid != SAE_CONNID_ANY && - mpts->mpts_connid != SAE_CONNID_ALL); if (mpts->mpts_flags & (MPTSF_DISCONNECTING|MPTSF_DISCONNECTED)) return; mpts->mpts_flags |= MPTSF_DISCONNECTING; - /* - * If this is coming from disconnectx(2) or issued as part of - * closing the MPTCP socket, the subflow shouldn't stick around. - * Otherwise let it linger around in case the upper layers need - * to retrieve its conninfo. - */ - if (deleteok) - mpts->mpts_flags |= MPTSF_DELETEOK; - so = mpts->mpts_socket; mp_tp = mpte->mpte_mptcb; - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state > MPTCPS_ESTABLISHED) + if (mp_tp->mpt_state > MPTCPS_CLOSE_WAIT) send_dfin = 1; - MPT_UNLOCK(mp_tp); - socket_lock(so, 0); if (!(so->so_state & (SS_ISDISCONNECTING | SS_ISDISCONNECTED)) && (so->so_state & SS_ISCONNECTED)) { - mptcplog((LOG_DEBUG, "MPTCP Socket %s: cid %d fin %d " - "[linger %s]\n", __func__, mpts->mpts_connid, send_dfin, - (deleteok ? "NO" : "YES")), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "MPTCP Socket %s: cid %d fin %d\n", + __func__, mpts->mpts_connid, send_dfin), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); if (send_dfin) mptcp_send_dfin(so); @@ -1615,154 +2237,134 @@ mptcp_subflow_disconnect(struct mptses *mpte, struct mptsub *mpts, (void) soshutdownlock(so, SHUT_WR); (void) sodisconnectlocked(so); } - socket_unlock(so, 0); /* * Generate a disconnect event for this subflow socket, in case * the lower layer doesn't do it; this is needed because the - * subflow socket deletion relies on it. This will also end up - * generating SO_FILT_HINT_CONNINFO_UPDATED on the MPTCP socket; - * we cannot do that here because subflow lock is currently held. + * subflow socket deletion relies on it. */ - mptcp_subflow_eupcall(so, mpts, SO_FILT_HINT_DISCONNECTED); + mptcp_subflow_eupcall1(so, mpts, SO_FILT_HINT_DISCONNECTED); } /* - * Subflow socket read upcall. - * - * Called when the associated subflow socket posted a read event. The subflow - * socket lock has been released prior to invoking the callback. Note that the - * upcall may occur synchronously as a result of MPTCP performing an action on - * it, or asynchronously as a result of an event happening at the subflow layer. - * Therefore, to maintain lock ordering, the only lock that can be acquired - * here is the thread lock, for signalling purposes. + * Called when the associated subflow socket posted a read event. */ static void mptcp_subflow_rupcall(struct socket *so, void *arg, int waitf) { #pragma unused(so, waitf) - struct mptsub *mpts = arg; + struct mptsub *mpts = arg, *tmpts; struct mptses *mpte = mpts->mpts_mpte; - /* - * mpte should never be NULL, except in a race with - * mptcp_subflow_del - */ - if (mpte == NULL) + VERIFY(mpte != NULL); + + if (mptcp_should_defer_upcall(mpte->mpte_mppcb)) { + if (!(mpte->mpte_mppcb->mpp_flags & MPP_RUPCALL)) + mpte->mpte_mppcb->mpp_flags |= MPP_SHOULD_RWAKEUP; return; + } + + mpte->mpte_mppcb->mpp_flags |= MPP_RUPCALL; + TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { + if (mpts->mpts_socket->so_usecount == 0) { + /* Will be removed soon by tcp_garbage_collect */ + continue; + } + + mptcp_subflow_addref(mpts); + mpts->mpts_socket->so_usecount++; - lck_mtx_lock(&mpte->mpte_thread_lock); - mptcp_thread_signal_locked(mpte); - lck_mtx_unlock(&mpte->mpte_thread_lock); + mptcp_subflow_input(mpte, mpts); + + mptcp_subflow_remref(mpts); /* ours */ + + VERIFY(mpts->mpts_socket->so_usecount != 0); + mpts->mpts_socket->so_usecount--; + } + + mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_RUPCALL); } /* * Subflow socket input. - * - * Called in the context of the MPTCP thread, for reading data from the - * underlying subflow socket and delivering it to MPTCP. */ static void mptcp_subflow_input(struct mptses *mpte, struct mptsub *mpts) { + struct socket *mp_so = mptetoso(mpte); struct mbuf *m = NULL; struct socket *so; - int error; - struct mptsub *mpts_alt = NULL; + int error, wakeup = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + VERIFY(!(mpte->mpte_mppcb->mpp_flags & MPP_INSIDE_INPUT)); + mpte->mpte_mppcb->mpp_flags |= MPP_INSIDE_INPUT; DTRACE_MPTCP2(subflow__input, struct mptses *, mpte, struct mptsub *, mpts); if (!(mpts->mpts_flags & MPTSF_CONNECTED)) - return; + goto out; so = mpts->mpts_socket; error = sock_receive_internal(so, NULL, &m, 0, NULL); if (error != 0 && error != EWOULDBLOCK) { - mptcplog((LOG_ERR, "MPTCP Receiver: %s cid %d error %d\n", + mptcplog((LOG_ERR, "%s: cid %d error %d\n", __func__, mpts->mpts_connid, error), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); - MPTS_UNLOCK(mpts); - mpts_alt = mptcp_get_subflow(mpte, mpts, NULL); - if (mpts_alt == NULL) { - if (mptcp_delayed_subf_start) { - mpts_alt = mptcp_get_pending_subflow(mpte, - mpts); - if (mpts_alt) { - mptcplog((LOG_DEBUG,"MPTCP Receiver:" - " %s: pending %d\n", - __func__, mpts_alt->mpts_connid), - MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); - } else { - mptcplog((LOG_ERR, "MPTCP Receiver:" - " %s: no pending flow for cid %d", - __func__, mpts->mpts_connid), - MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); - } - } else { - mptcplog((LOG_ERR, "MPTCP Receiver: %s: no alt" - " path for cid %d\n", __func__, - mpts->mpts_connid), - MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); - } - if (error == ENODATA) { - /* - * Don't ignore ENODATA so as to discover - * nasty middleboxes. - */ - struct socket *mp_so = - mpte->mpte_mppcb->mpp_socket; - mp_so->so_error = ENODATA; - sorwakeup(mp_so); - } + if (error == ENODATA) { + /* + * Don't ignore ENODATA so as to discover + * nasty middleboxes. + */ + mp_so->so_error = ENODATA; + + wakeup = 1; + goto out; } - MPTS_LOCK(mpts); } else if (error == 0) { - mptcplog((LOG_DEBUG, "MPTCP Receiver: %s: cid %d \n", - __func__, mpts->mpts_connid), + mptcplog((LOG_DEBUG, "%s: cid %d \n", __func__, mpts->mpts_connid), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); } /* In fallback, make sure to accept data on all but one subflow */ - if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) && - (!(mpts->mpts_flags & MPTSF_ACTIVE))) { + if (m && (mpts->mpts_flags & MPTSF_MP_DEGRADED) && + !(mpts->mpts_flags & MPTSF_ACTIVE)) { + mptcplog((LOG_DEBUG, "%s: degraded and got data on non-active flow\n", + __func__), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); m_freem(m); - return; + goto out; } if (m != NULL) { + if (IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp)) { + mpte->mpte_mppcb->mpp_flags |= MPP_SET_CELLICON; - /* Did we receive data on the backup subflow? */ - if (!(mpts->mpts_flags & MPTSF_ACTIVE)) - mpts->mpts_peerswitch++; - else - mpts->mpts_peerswitch = 0; + mpte->mpte_used_cell = 1; + } else { + mpte->mpte_mppcb->mpp_flags |= MPP_UNSET_CELLICON; + + mpte->mpte_used_wifi = 1; + } - /* - * Release subflow lock since this may trigger MPTCP to send, - * possibly on a different subflow. An extra reference has - * been held on the subflow by the MPTCP thread before coming - * here, so we can be sure that it won't go away, in the event - * the MP socket lock gets released. - */ - MPTS_UNLOCK(mpts); mptcp_input(mpte, m); - MPTS_LOCK(mpts); } + + /* notify protocol that we drained all the data */ + if (error == 0 && m != NULL && + (so->so_proto->pr_flags & PR_WANTRCVD) && so->so_pcb != NULL) + (*so->so_proto->pr_usrreqs->pru_rcvd)(so, 0); + +out: + if (wakeup) + mpte->mpte_mppcb->mpp_flags |= MPP_SHOULD_RWAKEUP; + + mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_INSIDE_INPUT); } /* * Subflow socket write upcall. * - * Called when the associated subflow socket posted a read event. The subflow - * socket lock has been released prior to invoking the callback. Note that the - * upcall may occur synchronously as a result of MPTCP performing an action on - * it, or asynchronously as a result of an event happening at the subflow layer. - * Therefore, to maintain lock ordering, the only lock that can be acquired - * here is the thread lock, for signalling purposes. + * Called when the associated subflow socket posted a read event. */ static void mptcp_subflow_wupcall(struct socket *so, void *arg, int waitf) @@ -1771,18 +2373,15 @@ mptcp_subflow_wupcall(struct socket *so, void *arg, int waitf) struct mptsub *mpts = arg; struct mptses *mpte = mpts->mpts_mpte; - /* - * mpte should never be NULL except in a race with - * mptcp_subflow_del which doesn't hold socket lock across critical - * section. This upcall is made after releasing the socket lock. - * Interleaving of socket operations becomes possible therefore. - */ - if (mpte == NULL) + VERIFY(mpte != NULL); + + if (mptcp_should_defer_upcall(mpte->mpte_mppcb)) { + if (!(mpte->mpte_mppcb->mpp_flags & MPP_WUPCALL)) + mpte->mpte_mppcb->mpp_flags |= MPP_SHOULD_WWAKEUP; return; + } - lck_mtx_lock(&mpte->mpte_thread_lock); - mptcp_thread_signal_locked(mpte); - lck_mtx_unlock(&mpte->mpte_thread_lock); + mptcp_output(mpte); } /* @@ -1791,61 +2390,47 @@ mptcp_subflow_wupcall(struct socket *so, void *arg, int waitf) * Called for sending data from MPTCP to the underlying subflow socket. */ int -mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts) +mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts, int flags) { - struct socket *mp_so, *so; - size_t sb_cc = 0, tot_sent = 0; - struct mbuf *sb_mb; - int error = 0, wakeup = 0; - u_int64_t mpt_dsn = 0; struct mptcb *mp_tp = mpte->mpte_mptcb; - struct mbuf *mpt_mbuf = NULL; - u_int64_t off = 0; - struct mbuf *head, *tail; - int tcp_zero_len_write = 0; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - mp_so = mpte->mpte_mppcb->mpp_socket; + struct mbuf *sb_mb, *m, *mpt_mbuf = NULL, *head, *tail; + struct socket *mp_so, *so; + struct tcpcb *tp; + uint64_t mpt_dsn = 0, off = 0; + int sb_cc = 0, error = 0, wakeup = 0; + uint32_t dss_csum; + uint16_t tot_sent = 0; + boolean_t reinjected = FALSE; + + mpte_lock_assert_held(mpte); + + mp_so = mptetoso(mpte); so = mpts->mpts_socket; + tp = sototcpcb(so); - DTRACE_MPTCP2(subflow__output, struct mptses *, mpte, - struct mptsub *, mpts); + VERIFY(!(mpte->mpte_mppcb->mpp_flags & MPP_INSIDE_OUTPUT)); + mpte->mpte_mppcb->mpp_flags |= MPP_INSIDE_OUTPUT; - /* subflow socket is suspended? */ - if (mpts->mpts_flags & MPTSF_SUSPENDED) { - mptcplog((LOG_ERR, "MPTCP Sender: %s mp_so 0x%llx cid %d is " - "flow controlled\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpts->mpts_connid), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); - goto out; - } + VERIFY(!INP_WAIT_FOR_IF_FEEDBACK(sotoinpcb(so))); + VERIFY((mpts->mpts_flags & MPTSF_MP_CAPABLE) || + (mpts->mpts_flags & MPTSF_MP_DEGRADED) || + (mpts->mpts_flags & MPTSF_TFO_REQD)); + VERIFY(mptcp_subflow_cwnd_space(mpts->mpts_socket) > 0); - /* subflow socket is not MPTCP capable? */ - if (!(mpts->mpts_flags & MPTSF_MP_CAPABLE) && - !(mpts->mpts_flags & MPTSF_MP_DEGRADED) && - !(mpts->mpts_flags & MPTSF_FASTJ_SEND) && - !(mpts->mpts_flags & MPTSF_TFO_REQD)) { - mptcplog((LOG_ERR, "MPTCP Sender: %s mp_so 0x%llx cid %d not " - "MPTCP capable\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpts->mpts_connid), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); - goto out; - } + mptcplog((LOG_DEBUG, "%s mpts_flags %#x, mpte_flags %#x cwnd_space %u\n", + __func__, mpts->mpts_flags, mpte->mpte_flags, + mptcp_subflow_cwnd_space(so)), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + DTRACE_MPTCP2(subflow__output, struct mptses *, mpte, + struct mptsub *, mpts); /* Remove Addr Option is not sent reliably as per I-D */ if (mpte->mpte_flags & MPTE_SND_REM_ADDR) { - struct tcpcb *tp = intotcpcb(sotoinpcb(so)); tp->t_rem_aid = mpte->mpte_lost_aid; - if (mptcp_remaddr_enable) - tp->t_mpflags |= TMPF_SND_REM_ADDR; + tp->t_mpflags |= TMPF_SND_REM_ADDR; mpte->mpte_flags &= ~MPTE_SND_REM_ADDR; } - if (mpts->mpts_flags & MPTSF_TFO_REQD) { - mptcp_drop_tfo_data(mpte, mpts, &wakeup); - } - /* * The mbuf chains containing the metadata (as well as pointing to * the user data sitting at the MPTCP output queue) would then be @@ -1860,154 +2445,195 @@ mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts) * pkt_flags marked with the PKTF_MPTCP flag. */ - /* First, drop acknowledged data */ - sb_mb = mp_so->so_snd.sb_mb; + if (mpte->mpte_reinjectq) + sb_mb = mpte->mpte_reinjectq; + else + sb_mb = mp_so->so_snd.sb_mb; + if (sb_mb == NULL) { + mptcplog((LOG_ERR, "%s: No data in MPTCP-sendbuffer! smax %u snxt %u suna %u\n", + __func__, (uint32_t)mp_tp->mpt_sndmax, (uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_snduna), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); goto out; } VERIFY(sb_mb->m_pkthdr.pkt_flags & PKTF_MPTCP); - mpt_mbuf = sb_mb; - while (mpt_mbuf && mpt_mbuf->m_pkthdr.mp_rlen == 0) { - if (((so->so_state & SS_ISCONNECTED) == 0) && - (mpt_mbuf->m_next == NULL) && - (so->so_flags1 & SOF1_PRECONNECT_DATA)) { - /* - * If TFO, allow connection establishment with zero - * length write. - */ - tcp_zero_len_write = 1; - goto zero_len_write; - } - mpt_mbuf = mpt_mbuf->m_next; - } - if (mpt_mbuf && (mpt_mbuf->m_pkthdr.pkt_flags & PKTF_MPTCP)) { - mpt_dsn = mpt_mbuf->m_pkthdr.mp_dsn; - } else { - goto out; + if (sb_mb->m_pkthdr.mp_rlen == 0 && + !(so->so_state & SS_ISCONNECTED) && + (so->so_flags1 & SOF1_PRECONNECT_DATA)) { + tp->t_mpflags |= TMPF_TFO_REQUEST; + goto zero_len_write; } - MPT_LOCK(mp_tp); + mpt_dsn = sb_mb->m_pkthdr.mp_dsn; + + /* First, drop acknowledged data */ if (MPTCP_SEQ_LT(mpt_dsn, mp_tp->mpt_snduna)) { - u_int64_t len = 0; - len = mp_tp->mpt_snduna - mpt_dsn; - MPT_UNLOCK(mp_tp); - sbdrop(&mp_so->so_snd, (int)len); - wakeup = 1; - MPT_LOCK(mp_tp); + mptcplog((LOG_ERR, "%s: dropping data, should have been done earlier " + "dsn %u suna %u reinject? %u\n", + __func__, (uint32_t)mpt_dsn, + (uint32_t)mp_tp->mpt_snduna, !!mpte->mpte_reinjectq), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + if (mpte->mpte_reinjectq) { + mptcp_clean_reinjectq(mpte); + } else { + uint64_t len = 0; + len = mp_tp->mpt_snduna - mpt_dsn; + sbdrop(&mp_so->so_snd, (int)len); + wakeup = 1; + } + } + + /* Check again because of above sbdrop */ + if (mp_so->so_snd.sb_mb == NULL && mpte->mpte_reinjectq == NULL) { + mptcplog((LOG_ERR, "%s send-buffer is empty\n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + goto out; } /* * In degraded mode, we don't receive data acks, so force free * mbufs less than snd_nxt */ - if (mp_so->so_snd.sb_mb == NULL) { - MPT_UNLOCK(mp_tp); - goto out; - } - - mpt_dsn = mp_so->so_snd.sb_mb->m_pkthdr.mp_dsn; if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) && (mp_tp->mpt_flags & MPTCPF_POST_FALLBACK_SYNC) && - MPTCP_SEQ_LT(mpt_dsn, mp_tp->mpt_sndnxt)) { - u_int64_t len = 0; - len = mp_tp->mpt_sndnxt - mpt_dsn; - sbdrop(&mp_so->so_snd, (int)len); - wakeup = 1; - mp_tp->mpt_snduna = mp_tp->mpt_sndnxt; + mp_so->so_snd.sb_mb) { + mpt_dsn = mp_so->so_snd.sb_mb->m_pkthdr.mp_dsn; + if (MPTCP_SEQ_LT(mpt_dsn, mp_tp->mpt_snduna)) { + uint64_t len = 0; + len = mp_tp->mpt_snduna - mpt_dsn; + sbdrop(&mp_so->so_snd, (int)len); + wakeup = 1; + + mptcplog((LOG_ERR, "%s: dropping data in degraded mode, should have been done earlier dsn %u sndnxt %u suna %u\n", + __func__, (uint32_t)mpt_dsn, (uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_snduna), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + } } if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) && !(mp_tp->mpt_flags & MPTCPF_POST_FALLBACK_SYNC)) { mp_tp->mpt_flags |= MPTCPF_POST_FALLBACK_SYNC; so->so_flags1 |= SOF1_POST_FALLBACK_SYNC; - if (mp_tp->mpt_flags & MPTCPF_RECVD_MPFAIL) - mpts->mpts_sndnxt = mp_tp->mpt_dsn_at_csum_fail; - } - - /* - * Adjust the subflow's notion of next byte to send based on - * the last unacknowledged byte - */ - if (MPTCP_SEQ_LT(mpts->mpts_sndnxt, mp_tp->mpt_snduna)) { - mpts->mpts_sndnxt = mp_tp->mpt_snduna; } /* * Adjust the top level notion of next byte used for retransmissions * and sending FINs. */ - if (MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mp_tp->mpt_snduna)) { + if (MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mp_tp->mpt_snduna)) mp_tp->mpt_sndnxt = mp_tp->mpt_snduna; - } - /* Now determine the offset from which to start transmitting data */ - sb_mb = mp_so->so_snd.sb_mb; - sb_cc = mp_so->so_snd.sb_cc; + if (mpte->mpte_reinjectq) + sb_mb = mpte->mpte_reinjectq; + else + sb_mb = mp_so->so_snd.sb_mb; if (sb_mb == NULL) { - MPT_UNLOCK(mp_tp); + mptcplog((LOG_ERR, "%s send-buffer is still empty\n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); goto out; } - if (MPTCP_SEQ_LT(mpts->mpts_sndnxt, mp_tp->mpt_sndmax)) { - off = mpts->mpts_sndnxt - mp_tp->mpt_snduna; - sb_cc -= (size_t)off; + + if (mpte->mpte_reinjectq) { + sb_cc = sb_mb->m_pkthdr.mp_rlen; + } else if (flags & MPTCP_SUBOUT_PROBING) { + sb_cc = sb_mb->m_pkthdr.mp_rlen; + off = 0; } else { - MPT_UNLOCK(mp_tp); - goto out; + sb_cc = min(mp_so->so_snd.sb_cc, mp_tp->mpt_sndwnd); + + /* + * With TFO, there might be no data at all, thus still go into this + * code-path here. + */ + if ((mp_so->so_flags1 & SOF1_PRECONNECT_DATA) || + MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mp_tp->mpt_sndmax)) { + off = mp_tp->mpt_sndnxt - mp_tp->mpt_snduna; + sb_cc -= off; + } else { + mptcplog((LOG_ERR, "%s this should not happen: sndnxt %u sndmax %u\n", + __func__, (uint32_t)mp_tp->mpt_sndnxt, + (uint32_t)mp_tp->mpt_sndmax), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + + goto out; + } } - MPT_UNLOCK(mp_tp); - mpt_mbuf = sb_mb; + sb_cc = min(sb_cc, mptcp_subflow_cwnd_space(so)); + if (sb_cc <= 0) { + mptcplog((LOG_ERR, "%s sb_cc is %d, mp_so->sb_cc %u, sndwnd %u,sndnxt %u sndmax %u cwnd %u\n", + __func__, sb_cc, mp_so->so_snd.sb_cc, mp_tp->mpt_sndwnd, + (uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_sndmax, + mptcp_subflow_cwnd_space(so)), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + } + + sb_cc = min(sb_cc, UINT16_MAX); + + /* + * Create a DSN mapping for the data we are about to send. It all + * has the same mapping. + */ + if (mpte->mpte_reinjectq) + mpt_dsn = sb_mb->m_pkthdr.mp_dsn; + else + mpt_dsn = mp_tp->mpt_snduna + off; - while (mpt_mbuf && ((mpt_mbuf->m_pkthdr.mp_rlen == 0) || - (mpt_mbuf->m_pkthdr.mp_rlen <= (u_int32_t)off))) { + mpt_mbuf = sb_mb; + while (mpt_mbuf && mpte->mpte_reinjectq == NULL && + (mpt_mbuf->m_pkthdr.mp_rlen == 0 || + mpt_mbuf->m_pkthdr.mp_rlen <= (uint32_t)off)) { off -= mpt_mbuf->m_pkthdr.mp_rlen; mpt_mbuf = mpt_mbuf->m_next; } if (mpts->mpts_flags & MPTSF_MP_DEGRADED) - mptcplog((LOG_DEBUG, "MPTCP Sender: %s cid = %d " - "snduna = %llu sndnxt = %llu probe %d\n", - __func__, mpts->mpts_connid, - mp_tp->mpt_snduna, mpts->mpts_sndnxt, + mptcplog((LOG_DEBUG, "%s: %u snduna = %u sndnxt = %u probe %d\n", + __func__, mpts->mpts_connid, (uint32_t)mp_tp->mpt_snduna, (uint32_t)mp_tp->mpt_sndnxt, mpts->mpts_probecnt), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); VERIFY((mpt_mbuf == NULL) || (mpt_mbuf->m_pkthdr.pkt_flags & PKTF_MPTCP)); head = tail = NULL; while (tot_sent < sb_cc) { - struct mbuf *m; - size_t mlen; + ssize_t mlen; - mlen = mpt_mbuf->m_pkthdr.mp_rlen; + mlen = mpt_mbuf->m_len; mlen -= off; - if (mlen == 0) - goto out; + mlen = min(mlen, sb_cc - tot_sent); - if (mlen > sb_cc) { - panic("%s: unexpected %lu %lu \n", __func__, - mlen, sb_cc); + if (mlen < 0) { + mptcplog((LOG_ERR, "%s mlen %d mp_rlen %u off %u sb_cc %u tot_sent %u\n", + __func__, (int)mlen, mpt_mbuf->m_pkthdr.mp_rlen, + (uint32_t)off, sb_cc, tot_sent), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + goto out; } + if (mlen == 0) + goto next; + m = m_copym_mode(mpt_mbuf, (int)off, mlen, M_DONTWAIT, M_COPYM_MUST_COPY_HDR); if (m == NULL) { + mptcplog((LOG_ERR, "%s m_copym_mode failed\n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); error = ENOBUFS; break; } /* Create a DSN mapping for the data (m_copym does it) */ - mpt_dsn = mpt_mbuf->m_pkthdr.mp_dsn; VERIFY(m->m_flags & M_PKTHDR); + VERIFY(m->m_next == NULL); + m->m_pkthdr.pkt_flags |= PKTF_MPTCP; m->m_pkthdr.pkt_flags &= ~PKTF_MPSO; - m->m_pkthdr.mp_dsn = mpt_dsn + off; + m->m_pkthdr.mp_dsn = mpt_dsn; m->m_pkthdr.mp_rseq = mpts->mpts_rel_seq; - m->m_pkthdr.mp_rlen = mlen; - mpts->mpts_rel_seq += mlen; m->m_pkthdr.len = mlen; if (head == NULL) { @@ -2019,352 +2645,500 @@ mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts) tot_sent += mlen; off = 0; +next: mpt_mbuf = mpt_mbuf->m_next; } - if (head != NULL) { - struct tcpcb *tp = intotcpcb(sotoinpcb(so)); + if (mpte->mpte_reinjectq) { + reinjected = TRUE; + if (sb_cc < sb_mb->m_pkthdr.mp_rlen) { + struct mbuf *n = sb_mb; + + while (n) { + n->m_pkthdr.mp_dsn += sb_cc; + n->m_pkthdr.mp_rlen -= sb_cc; + n = n->m_next; + } + m_adj(sb_mb, sb_cc); + } else { + mpte->mpte_reinjectq = sb_mb->m_nextpkt; + m_freem(sb_mb); + } + } + + mptcplog((LOG_DEBUG, "%s: Queued dsn %u ssn %u len %u on sub %u\n", + __func__, (uint32_t)mpt_dsn, mpts->mpts_rel_seq, + tot_sent, mpts->mpts_connid), MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + + if (head && (mp_tp->mpt_flags & MPTCPF_CHECKSUM)) { + dss_csum = mptcp_output_csum(head, mpt_dsn, mpts->mpts_rel_seq, + tot_sent); + } + + /* Now, let's update rel-seq and the data-level length */ + mpts->mpts_rel_seq += tot_sent; + m = head; + while (m) { + if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) + m->m_pkthdr.mp_csum = dss_csum; + m->m_pkthdr.mp_rlen = tot_sent; + m = m->m_next; + } + + if (head != NULL) { if ((mpts->mpts_flags & MPTSF_TFO_REQD) && - (tp->t_tfo_stats == 0)) { + (tp->t_tfo_stats == 0)) tp->t_mpflags |= TMPF_TFO_REQUEST; - } else if (mpts->mpts_flags & MPTSF_FASTJ_SEND) { - tp->t_mpflags |= TMPF_FASTJOIN_SEND; - } error = sock_sendmbuf(so, NULL, head, 0, NULL); - DTRACE_MPTCP7(send, struct mbuf *, head, struct socket *, so, + DTRACE_MPTCP7(send, struct mbuf *, m, struct socket *, so, struct sockbuf *, &so->so_rcv, struct sockbuf *, &so->so_snd, struct mptses *, mpte, struct mptsub *, mpts, size_t, tot_sent); - } else if (tcp_zero_len_write == 1) { -zero_len_write: - socket_lock(so, 1); - /* Opting to call pru_send as no mbuf at subflow level */ - error = (*so->so_proto->pr_usrreqs->pru_send) - (so, 0, NULL, NULL, NULL, current_proc()); - socket_unlock(so, 1); } - if ((error == 0) || (error == EWOULDBLOCK)) { - mpts->mpts_sndnxt += tot_sent; +done_sending: + if (error == 0 || + (error == EWOULDBLOCK && (tp->t_mpflags & TMPF_TFO_REQUEST))) { + uint64_t new_sndnxt = mp_tp->mpt_sndnxt + tot_sent; if (mpts->mpts_probesoon && mpts->mpts_maxseg && tot_sent) { tcpstat.tcps_mp_num_probes++; - if (tot_sent < mpts->mpts_maxseg) + if ((uint32_t)tot_sent < mpts->mpts_maxseg) mpts->mpts_probecnt += 1; else mpts->mpts_probecnt += tot_sent/mpts->mpts_maxseg; } - MPT_LOCK(mp_tp); - - if (MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mpts->mpts_sndnxt)) { - if (MPTCP_DATASEQ_HIGH32(mpts->mpts_sndnxt) > + if (!reinjected && !(flags & MPTCP_SUBOUT_PROBING)) { + if (MPTCP_DATASEQ_HIGH32(new_sndnxt) > MPTCP_DATASEQ_HIGH32(mp_tp->mpt_sndnxt)) mp_tp->mpt_flags |= MPTCPF_SND_64BITDSN; - mp_tp->mpt_sndnxt = mpts->mpts_sndnxt; + mp_tp->mpt_sndnxt = new_sndnxt; } - mptcp_cancel_timer(mp_tp, MPTT_REXMT); - MPT_UNLOCK(mp_tp); - if (so->so_flags1 & SOF1_PRECONNECT_DATA) - so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + mptcp_cancel_timer(mp_tp, MPTT_REXMT); - /* Send once in SYN_SENT state to avoid sending SYN spam */ - if (mpts->mpts_flags & MPTSF_FASTJ_SEND) { - so->so_flags &= ~SOF_MPTCP_FASTJOIN; - mpts->mpts_flags &= ~MPTSF_FASTJ_SEND; - } + /* Must be here as mptcp_can_send_more() checks for this */ + soclearfastopen(mp_so); if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) || (mpts->mpts_probesoon != 0)) - mptcplog((LOG_DEBUG, "MPTCP Sender: %s cid %d " - "wrote %d %d probe %d probedelta %d\n", - __func__, mpts->mpts_connid, (int)tot_sent, - (int) sb_cc, mpts->mpts_probecnt, + mptcplog((LOG_DEBUG, "%s %u degraded %u wrote %d %d probe %d probedelta %d\n", + __func__, mpts->mpts_connid, + !!(mpts->mpts_flags & MPTSF_MP_DEGRADED), + tot_sent, (int) sb_cc, mpts->mpts_probecnt, (tcp_now - mpts->mpts_probesoon)), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + + if (IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp)) { + mpte->mpte_mppcb->mpp_flags |= MPP_SET_CELLICON; + + mpte->mpte_used_cell = 1; + } else { + mpte->mpte_mppcb->mpp_flags |= MPP_UNSET_CELLICON; + + mpte->mpte_used_wifi = 1; + } + + /* + * Don't propagate EWOULDBLOCK - it's already taken care of + * in mptcp_usr_send for TFO. + */ + error = 0; } else { - mptcplog((LOG_ERR, "MPTCP Sender: %s cid %d error %d len %zd\n", - __func__, mpts->mpts_connid, error, tot_sent), + mptcplog((LOG_ERR, "%s: %u error %d len %d subflags %#x sostate %#x soerror %u hiwat %u lowat %u\n", + __func__, mpts->mpts_connid, error, tot_sent, so->so_flags, so->so_state, so->so_error, so->so_snd.sb_hiwat, so->so_snd.sb_lowat), MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); } out: + if (wakeup) - sowwakeup(mp_so); + mpte->mpte_mppcb->mpp_flags |= MPP_SHOULD_WWAKEUP; + mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_INSIDE_OUTPUT); return (error); + +zero_len_write: + /* Opting to call pru_send as no mbuf at subflow level */ + error = (*so->so_proto->pr_usrreqs->pru_send)(so, 0, NULL, NULL, + NULL, current_proc()); + + goto done_sending; } -/* - * Subflow socket control event upcall. - * - * Called when the associated subflow socket posted one or more control events. - * The subflow socket lock has been released prior to invoking the callback. - * Note that the upcall may occur synchronously as a result of MPTCP performing - * an action on it, or asynchronously as a result of an event happening at the - * subflow layer. Therefore, to maintain lock ordering, the only lock that can - * be acquired here is the thread lock, for signalling purposes. - */ static void -mptcp_subflow_eupcall(struct socket *so, void *arg, uint32_t events) +mptcp_add_reinjectq(struct mptses *mpte, struct mbuf *m) { -#pragma unused(so) - struct mptsub *mpts = arg; - struct mptses *mpte = mpts->mpts_mpte; + struct mbuf *n, *prev = NULL; - VERIFY(mpte != NULL); + mptcplog((LOG_DEBUG, "%s reinjecting dsn %u dlen %u rseq %u\n", + __func__, (uint32_t)m->m_pkthdr.mp_dsn, m->m_pkthdr.mp_rlen, + m->m_pkthdr.mp_rseq), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + n = mpte->mpte_reinjectq; + + /* First, look for an mbuf n, whose data-sequence-number is bigger or + * equal than m's sequence number. + */ + while (n) { + if (MPTCP_SEQ_GEQ(n->m_pkthdr.mp_dsn, m->m_pkthdr.mp_dsn)) + break; + + prev = n; + + n = n->m_nextpkt; + } + + if (n) { + /* m is already fully covered by the next mbuf in the queue */ + if (n->m_pkthdr.mp_dsn == m->m_pkthdr.mp_dsn && + n->m_pkthdr.mp_rlen >= m->m_pkthdr.mp_rlen) { + mptcplog((LOG_DEBUG, "%s fully covered with len %u\n", + __func__, n->m_pkthdr.mp_rlen), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + goto dont_queue; + } + + /* m is covering the next mbuf entirely, thus we remove this guy */ + if (m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen >= n->m_pkthdr.mp_dsn + n->m_pkthdr.mp_rlen) { + struct mbuf *tmp = n->m_nextpkt; + + mptcplog((LOG_DEBUG, "%s m is covering that guy dsn %u len %u dsn %u len %u\n", + __func__, m->m_pkthdr.mp_dsn, m->m_pkthdr.mp_rlen, + n->m_pkthdr.mp_dsn, n->m_pkthdr.mp_rlen), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + m->m_nextpkt = NULL; + if (prev == NULL) + mpte->mpte_reinjectq = tmp; + else + prev->m_nextpkt = tmp; + + m_freem(n); + n = tmp; + } + + } + + if (prev) { + /* m is already fully covered by the previous mbuf in the queue */ + if (prev->m_pkthdr.mp_dsn + prev->m_pkthdr.mp_rlen >= m->m_pkthdr.mp_dsn + m->m_pkthdr.len) { + mptcplog((LOG_DEBUG, "%s prev covers us from %u with len %u\n", + __func__, prev->m_pkthdr.mp_dsn, prev->m_pkthdr.mp_rlen), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + goto dont_queue; + } + } + + if (prev == NULL) + mpte->mpte_reinjectq = m; + else + prev->m_nextpkt = m; - lck_mtx_lock(&mpte->mpte_thread_lock); - atomic_bitset_32(&mpts->mpts_evctl, events); - mptcp_thread_signal_locked(mpte); - lck_mtx_unlock(&mpte->mpte_thread_lock); + m->m_nextpkt = n; + + return; + +dont_queue: + m_freem(m); + return; } -/* - * Subflow socket control events. - * - * Called for handling events related to the underlying subflow socket. - */ -static ev_ret_t -mptcp_subflow_events(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +static struct mbuf * +mptcp_lookup_dsn(struct mptses *mpte, uint64_t dsn) { - uint32_t events, save_events; - ev_ret_t ret = MPTS_EVRET_OK; - int i = 0; - int mpsub_ev_entry_count = sizeof(mpsub_ev_entry_tbl)/ - sizeof(mpsub_ev_entry_tbl[0]); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + struct socket *mp_so = mptetoso(mpte); + struct mbuf *m; - /* bail if there's nothing to process */ - if ((events = mpts->mpts_evctl) == 0) - return (ret); + m = mp_so->so_snd.sb_mb; - if (events & (SO_FILT_HINT_CONNRESET|SO_FILT_HINT_MUSTRST| - SO_FILT_HINT_CANTRCVMORE|SO_FILT_HINT_CANTSENDMORE| - SO_FILT_HINT_TIMEOUT|SO_FILT_HINT_NOSRCADDR| - SO_FILT_HINT_IFDENIED|SO_FILT_HINT_SUSPEND| - SO_FILT_HINT_DISCONNECTED)) { - events |= SO_FILT_HINT_MPFAILOVER; + while (m) { + /* If this segment covers what we are looking for, return it. */ + if (MPTCP_SEQ_LEQ(m->m_pkthdr.mp_dsn, dsn) && + MPTCP_SEQ_GT(m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen, dsn)) + break; + + + /* Segment is no more in the queue */ + if (MPTCP_SEQ_GT(m->m_pkthdr.mp_dsn, dsn)) + return NULL; + + m = m->m_next; } - save_events = events; + return m; +} - DTRACE_MPTCP3(subflow__events, struct mptses *, mpte, - struct mptsub *, mpts, uint32_t, events); +static struct mbuf * +mptcp_copy_mbuf_list(struct mbuf *m, int len) +{ + struct mbuf *top = NULL, *tail = NULL; + uint64_t dsn; + uint32_t dlen, rseq; - mptcplog((LOG_DEBUG, "MPTCP Events: %s cid %d events=%b\n", __func__, - mpts->mpts_connid, events, SO_FILT_HINT_BITS), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); + dsn = m->m_pkthdr.mp_dsn; + dlen = m->m_pkthdr.mp_rlen; + rseq = m->m_pkthdr.mp_rseq; - /* - * Process all the socket filter hints and reset the hint - * once it is handled - */ - for (i = 0; (i < mpsub_ev_entry_count) && events; i++) { - /* - * Always execute the DISCONNECTED event, because it will wakeup - * the app. - */ - if ((events & mpsub_ev_entry_tbl[i].sofilt_hint_mask) && - (ret >= MPTS_EVRET_OK || - mpsub_ev_entry_tbl[i].sofilt_hint_mask == SO_FILT_HINT_DISCONNECTED)) { - ev_ret_t error = - mpsub_ev_entry_tbl[i].sofilt_hint_ev_hdlr(mpte, mpts, p_mpsofilt_hint); - events &= ~mpsub_ev_entry_tbl[i].sofilt_hint_mask; - ret = ((error >= MPTS_EVRET_OK) ? MAX(error, ret) : error); + while (len > 0) { + struct mbuf *n; + + VERIFY((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)); + + n = m_copym_mode(m, 0, m->m_len, M_DONTWAIT, M_COPYM_MUST_COPY_HDR); + if (n == NULL) { + mptcplog((LOG_ERR, "%s m_copym_mode returned NULL\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + goto err; } - } - /* - * We should be getting only events specified via sock_catchevents(), - * so loudly complain if we have any unprocessed one(s). - */ - if (events != 0 || ret < MPTS_EVRET_OK) { - mptcplog((LOG_ERR, "MPTCP Events %s%s: cid %d evret %s (%d)" - " unhandled events=%b\n", - (events != 0) && (ret == MPTS_EVRET_OK) ? "MPTCP_ERROR " : "", - __func__, mpts->mpts_connid, - mptcp_evret2str(ret), ret, events, SO_FILT_HINT_BITS), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); + VERIFY(n->m_flags & M_PKTHDR); + VERIFY(n->m_next == NULL); + VERIFY(n->m_pkthdr.mp_dsn == dsn); + VERIFY(n->m_pkthdr.mp_rlen == dlen); + VERIFY(n->m_pkthdr.mp_rseq == rseq); + VERIFY(n->m_len == m->m_len); + + n->m_pkthdr.pkt_flags |= (PKTF_MPSO | PKTF_MPTCP); + + if (top == NULL) + top = n; + + if (tail != NULL) + tail->m_next = n; + + tail = n; + + len -= m->m_len; + m = m->m_next; } - /* clear the ones we've processed */ - atomic_bitclear_32(&mpts->mpts_evctl, save_events); - return (ret); + return top; + +err: + if (top) + m_freem(top); + + return NULL; } -/* - * Handle SO_FILT_HINT_CONNRESET subflow socket event. - */ -static ev_ret_t -mptcp_subflow_connreset_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +static void +mptcp_reinject_mbufs(struct socket *so) { - struct socket *mp_so, *so; - struct mptcb *mp_tp; - boolean_t linger; + struct tcpcb *tp = sototcpcb(so); + struct mptsub *mpts = tp->t_mpsub; + struct mptcb *mp_tp = tptomptp(tp); + struct mptses *mpte = mp_tp->mpt_mpte;; + struct sockbuf *sb = &so->so_snd; + struct mbuf *m; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; - mp_tp = mpte->mpte_mptcb; - so = mpts->mpts_socket; + m = sb->sb_mb; + while (m) { + struct mbuf *n = m->m_next, *orig = m; - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); + mptcplog((LOG_DEBUG, "%s working on suna %u relseq %u iss %u len %u pktflags %#x\n", + __func__, tp->snd_una, m->m_pkthdr.mp_rseq, mpts->mpts_iss, + m->m_pkthdr.mp_rlen, m->m_pkthdr.pkt_flags), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d [linger %s]\n", __func__, - mpts->mpts_connid, (linger ? "YES" : "NO")), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + VERIFY((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)); - /* - * We got a TCP RST for this subflow connection. - * - * Right now, we simply propagate ECONNREFUSED to the MPTCP socket - * client if the MPTCP connection has not been established or - * if the connection has only one subflow and is a connection being - * resumed. Otherwise we close the socket. - */ - mptcp_subflow_disconnect(mpte, mpts, !linger); + if (m->m_pkthdr.pkt_flags & PKTF_MPTCP_REINJ) + goto next; - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - mpts->mpts_soerror = mp_so->so_error = ECONNREFUSED; - } else if (mpte->mpte_nummpcapflows < 1 || - ((mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && - (mpts->mpts_flags & MPTSF_ACTIVE))) { - mpts->mpts_soerror = mp_so->so_error = ECONNRESET; - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNRESET; + /* Has it all already been acknowledged at the data-level? */ + if (MPTCP_SEQ_GEQ(mp_tp->mpt_snduna, m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen)) + goto next; + + /* Part of this has already been acknowledged - lookup in the + * MPTCP-socket for the segment. + */ + if (SEQ_GT(tp->snd_una - mpts->mpts_iss, m->m_pkthdr.mp_rseq)) { + m = mptcp_lookup_dsn(mpte, m->m_pkthdr.mp_dsn); + if (m == NULL) + goto next; + } + + /* Copy the mbuf with headers (aka, DSN-numbers) */ + m = mptcp_copy_mbuf_list(m, m->m_pkthdr.mp_rlen); + if (m == NULL) + break; + + VERIFY(m->m_nextpkt == NULL); + + /* Now, add to the reinject-queue, eliminating overlapping + * segments + */ + mptcp_add_reinjectq(mpte, m); + + orig->m_pkthdr.pkt_flags |= PKTF_MPTCP_REINJ; + +next: + /* mp_rlen can cover multiple mbufs, so advance to the end of it. */ + while (n) { + VERIFY((n->m_flags & M_PKTHDR) && (n->m_pkthdr.pkt_flags & PKTF_MPTCP)); + + if (n->m_pkthdr.mp_dsn != orig->m_pkthdr.mp_dsn) + break; + + n->m_pkthdr.pkt_flags |= PKTF_MPTCP_REINJ; + n = n->m_next; + } + + m = n; } - MPT_UNLOCK(mp_tp); +} - /* - * Keep the subflow socket around, unless the MPTCP socket has - * been detached or the subflow has been disconnected explicitly, - * in which case it should be deleted right away. - */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); +void +mptcp_clean_reinjectq(struct mptses *mpte) +{ + struct mptcb *mp_tp = mpte->mpte_mptcb; + + mpte_lock_assert_held(mpte); + + while (mpte->mpte_reinjectq) { + struct mbuf *m = mpte->mpte_reinjectq; + + if (MPTCP_SEQ_GEQ(m->m_pkthdr.mp_dsn, mp_tp->mpt_snduna) || + MPTCP_SEQ_GEQ(m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen, mp_tp->mpt_snduna)) + break; + + mpte->mpte_reinjectq = m->m_nextpkt; + m->m_nextpkt = NULL; + m_freem(m); + } } /* - * Handle SO_FILT_HINT_CANTRCVMORE subflow socket event. + * Subflow socket control event upcall. */ -static ev_ret_t -mptcp_subflow_cantrcvmore_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +static void +mptcp_subflow_eupcall1(struct socket *so, void *arg, uint32_t events) { - struct mptcb *mp_tp; - struct socket *so; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); +#pragma unused(so) + struct mptsub *mpts = arg; + struct mptses *mpte = mpts->mpts_mpte; - mp_tp = mpte->mpte_mptcb; - so = mpts->mpts_socket; + VERIFY(mpte != NULL); + mpte_lock_assert_held(mpte); - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d\n", __func__, mpts->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + if ((mpts->mpts_evctl & events) == events) + return; - /* - * A FIN on a fallen back MPTCP-connection should be treated like a - * DATA_FIN. - */ - MPT_LOCK(mp_tp); - if ((mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && - (mpts->mpts_flags & MPTSF_ACTIVE)) { - mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN); - if (mp_tp->mpt_state == MPTCPS_CLOSE_WAIT) { - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CANTRCVMORE; - } + mpts->mpts_evctl |= events; + + if (mptcp_should_defer_upcall(mpte->mpte_mppcb)) { + mpte->mpte_mppcb->mpp_flags |= MPP_SHOULD_WORKLOOP; + return; } - MPT_UNLOCK(mp_tp); - return (MPTS_EVRET_OK); /* keep the subflow socket around */ + mptcp_subflow_workloop(mpte); } /* - * Handle SO_FILT_HINT_CANTSENDMORE subflow socket event. + * Subflow socket control events. + * + * Called for handling events related to the underlying subflow socket. */ static ev_ret_t -mptcp_subflow_cantsendmore_ev(struct mptses *mpte, struct mptsub *mpts, +mptcp_subflow_events(struct mptses *mpte, struct mptsub *mpts, uint64_t *p_mpsofilt_hint) { -#pragma unused(p_mpsofilt_hint) - struct socket *so; + ev_ret_t ret = MPTS_EVRET_OK; + int i, mpsub_ev_entry_count = sizeof(mpsub_ev_entry_tbl) / + sizeof(mpsub_ev_entry_tbl[0]); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ - so = mpts->mpts_socket; + /* bail if there's nothing to process */ + if (!mpts->mpts_evctl) + return (ret); - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d\n", __func__, mpts->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + if (mpts->mpts_evctl & (SO_FILT_HINT_CONNRESET|SO_FILT_HINT_MUSTRST| + SO_FILT_HINT_CANTSENDMORE|SO_FILT_HINT_TIMEOUT| + SO_FILT_HINT_NOSRCADDR|SO_FILT_HINT_IFDENIED| + SO_FILT_HINT_DISCONNECTED)) { + mpts->mpts_evctl |= SO_FILT_HINT_MPFAILOVER; + } - return (MPTS_EVRET_OK); /* keep the subflow socket around */ + DTRACE_MPTCP3(subflow__events, struct mptses *, mpte, + struct mptsub *, mpts, uint32_t, mpts->mpts_evctl); + + mptcplog((LOG_DEBUG, "%s cid %d events=%b\n", __func__, + mpts->mpts_connid, mpts->mpts_evctl, SO_FILT_HINT_BITS), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); + + /* + * Process all the socket filter hints and reset the hint + * once it is handled + */ + for (i = 0; i < mpsub_ev_entry_count && mpts->mpts_evctl; i++) { + /* + * Always execute the DISCONNECTED event, because it will wakeup + * the app. + */ + if ((mpts->mpts_evctl & mpsub_ev_entry_tbl[i].sofilt_hint_mask) && + (ret >= MPTS_EVRET_OK || + mpsub_ev_entry_tbl[i].sofilt_hint_mask == SO_FILT_HINT_DISCONNECTED)) { + mpts->mpts_evctl &= ~mpsub_ev_entry_tbl[i].sofilt_hint_mask; + ev_ret_t error = + mpsub_ev_entry_tbl[i].sofilt_hint_ev_hdlr(mpte, mpts, p_mpsofilt_hint, mpsub_ev_entry_tbl[i].sofilt_hint_mask); + ret = ((error >= MPTS_EVRET_OK) ? MAX(error, ret) : error); + } + } + + /* + * We should be getting only events specified via sock_catchevents(), + * so loudly complain if we have any unprocessed one(s). + */ + if (mpts->mpts_evctl || ret < MPTS_EVRET_OK) + mptcplog((LOG_WARNING, "%s%s: cid %d evret %s (%d) unhandled events=%b\n", __func__, + (mpts->mpts_evctl && ret == MPTS_EVRET_OK) ? "MPTCP_ERROR " : "", + mpts->mpts_connid, + mptcp_evret2str(ret), ret, mpts->mpts_evctl, SO_FILT_HINT_BITS), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + else + mptcplog((LOG_DEBUG, "%s: Done, events %b\n", __func__, + mpts->mpts_evctl, SO_FILT_HINT_BITS), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); + + return (ret); } -/* - * Handle SO_FILT_HINT_TIMEOUT subflow socket event. - */ static ev_ret_t -mptcp_subflow_timeout_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +mptcp_subflow_propagate_ev(struct mptses *mpte, struct mptsub *mpts, + uint64_t *p_mpsofilt_hint, uint64_t event) { -#pragma unused(p_mpsofilt_hint) struct socket *mp_so, *so; struct mptcb *mp_tp; - boolean_t linger; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; so = mpts->mpts_socket; - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); - - mptcplog((LOG_NOTICE, "MPTCP Events: " - "%s: cid %d [linger %s]\n", __func__, - mpts->mpts_connid, (linger ? "YES" : "NO")), + mptcplog((LOG_DEBUG, "%s: cid %d event %d\n", __func__, + mpts->mpts_connid, event), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - if (mpts->mpts_soerror == 0) - mpts->mpts_soerror = ETIMEDOUT; - /* - * The subflow connection has timed out. - * - * Right now, we simply propagate ETIMEDOUT to the MPTCP socket - * client if the MPTCP connection has not been established. Otherwise - * drop it. + * We got an event for this subflow that might need to be propagated, + * based on the state of the MPTCP connection. */ - mptcp_subflow_disconnect(mpte, mpts, !linger); - - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - mp_so->so_error = ETIMEDOUT; + if (mp_tp->mpt_state < MPTCPS_ESTABLISHED || + ((mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && (mpts->mpts_flags & MPTSF_ACTIVE))) { + mp_so->so_error = so->so_error; + *p_mpsofilt_hint |= event; } - MPT_UNLOCK(mp_tp); - /* - * Keep the subflow socket around, unless the MPTCP socket has - * been detached or the subflow has been disconnected explicitly, - * in which case it should be deleted right away. - */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); + return (MPTS_EVRET_OK); } /* @@ -2372,24 +3146,18 @@ mptcp_subflow_timeout_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_nosrcaddr_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { -#pragma unused(p_mpsofilt_hint) - struct socket *mp_so, *so; - struct mptcb *mp_tp; - boolean_t linger; - struct tcpcb *tp = NULL; +#pragma unused(p_mpsofilt_hint, event) + struct socket *mp_so; + struct tcpcb *tp; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; - mp_tp = mpte->mpte_mptcb; - so = mpts->mpts_socket; + mp_so = mptetoso(mpte); + tp = intotcpcb(sotoinpcb(mpts->mpts_socket)); - /* Not grabbing socket lock as t_local_aid is write once only */ - tp = intotcpcb(sotoinpcb(so)); /* * This overwrites any previous mpte_lost_aid to avoid storing * too much state when the typical case has only two subflows. @@ -2397,42 +3165,18 @@ mptcp_subflow_nosrcaddr_ev(struct mptses *mpte, struct mptsub *mpts, mpte->mpte_flags |= MPTE_SND_REM_ADDR; mpte->mpte_lost_aid = tp->t_local_aid; - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); - - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s cid %d [linger %s]\n", __func__, - mpts->mpts_connid, (linger ? "YES" : "NO")), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - - if (mpts->mpts_soerror == 0) - mpts->mpts_soerror = EADDRNOTAVAIL; + mptcplog((LOG_DEBUG, "%s cid %d\n", __func__, mpts->mpts_connid), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); /* * The subflow connection has lost its source address. - * - * Right now, we simply propagate EADDRNOTAVAIL to the MPTCP socket - * client if the MPTCP connection has not been established. If it - * has been established with one subflow , we keep the MPTCP - * connection valid without any subflows till closed by application. - * This lets tcp connection manager decide whether to close this or - * not as it reacts to reachability changes too. */ - mptcp_subflow_disconnect(mpte, mpts, !linger); + mptcp_subflow_abort(mpts, EADDRNOTAVAIL); - MPT_LOCK(mp_tp); - if ((mp_tp->mpt_state < MPTCPS_ESTABLISHED) && - (mp_so->so_flags & SOF_NOADDRAVAIL)) { - mp_so->so_error = EADDRNOTAVAIL; - } - MPT_UNLOCK(mp_tp); + if (mp_so->so_flags & SOF_NOADDRAVAIL) + mptcp_subflow_propagate_ev(mpte, mpts, p_mpsofilt_hint, event); - /* - * Keep the subflow socket around, unless the MPTCP socket has - * been detached or the subflow has been disconnected explicitly, - * in which case it should be deleted right away. - */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); + return (MPTS_EVRET_DELETE); } /* @@ -2441,19 +3185,15 @@ mptcp_subflow_nosrcaddr_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_mpcantrcvmore_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { - struct socket *so, *mp_so; +#pragma unused(event) struct mptcb *mp_tp; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - mp_so = mpte->mpte_mppcb->mpp_socket; - so = mpts->mpts_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ mp_tp = mpte->mpte_mptcb; - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d\n", __func__, mpts->mpts_connid), + mptcplog((LOG_DEBUG, "%s: cid %d\n", __func__, mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); /* @@ -2462,11 +3202,9 @@ mptcp_subflow_mpcantrcvmore_ev(struct mptses *mpte, struct mptsub *mpts, * mptcp socket and the user is notified so that it may close * the socket if needed. */ - MPT_LOCK(mp_tp); if (mp_tp->mpt_state == MPTCPS_CLOSE_WAIT) - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CANTRCVMORE; + *p_mpsofilt_hint |= SO_FILT_HINT_CANTRCVMORE; - MPT_UNLOCK(mp_tp); return (MPTS_EVRET_OK); /* keep the subflow socket around */ } @@ -2475,106 +3213,68 @@ mptcp_subflow_mpcantrcvmore_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_failover_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { +#pragma unused(event, p_mpsofilt_hint) struct mptsub *mpts_alt = NULL; - struct socket *so = NULL; + struct socket *alt_so = NULL; struct socket *mp_so; int altpath_exists = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - mp_so = mpte->mpte_mppcb->mpp_socket; - mptcplog((LOG_NOTICE, "MPTCP Events: " - "%s: mp_so 0x%llx\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + mpte_lock_assert_held(mpte); + mp_so = mptetoso(mpte); + mptcplog((LOG_NOTICE, "%s: mp_so 0x%llx\n", __func__, + (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - MPTS_UNLOCK(mpts); - mpts_alt = mptcp_get_subflow(mpte, mpts, NULL); + mptcp_reinject_mbufs(mpts->mpts_socket); + mpts_alt = mptcp_get_subflow(mpte, mpts, NULL); /* * If there is no alternate eligible subflow, ignore the * failover hint. */ if (mpts_alt == NULL) { - mptcplog((LOG_WARNING, "MPTCP Events: " - "%s: no alternate path\n", __func__), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); - - if (mptcp_delayed_subf_start) { - mpts_alt = mptcp_get_pending_subflow(mpte, mpts); - if (mpts_alt != NULL) { - MPTS_LOCK(mpts_alt); - (void) mptcp_subflow_soconnectx(mpte, - mpts_alt); - MPTS_UNLOCK(mpts_alt); - } - } - MPTS_LOCK(mpts); + mptcplog((LOG_WARNING, "%s: no alternate path\n", __func__), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + goto done; } - MPTS_LOCK(mpts_alt); + altpath_exists = 1; - so = mpts_alt->mpts_socket; + alt_so = mpts_alt->mpts_socket; if (mpts_alt->mpts_flags & MPTSF_FAILINGOVER) { - socket_lock(so, 1); /* All data acknowledged and no RTT spike */ - if ((so->so_snd.sb_cc == 0) && - (mptcp_no_rto_spike(so))) { - so->so_flags &= ~SOF_MP_TRYFAILOVER; + if (alt_so->so_snd.sb_cc == 0 && mptcp_no_rto_spike(alt_so)) { mpts_alt->mpts_flags &= ~MPTSF_FAILINGOVER; } else { /* no alternate path available */ altpath_exists = 0; } - socket_unlock(so, 1); - } - if (altpath_exists) { - mptcplog((LOG_INFO, "MPTCP Events: " - "%s: cid = %d\n", - __func__, mpts_alt->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - mpts_alt->mpts_flags |= MPTSF_ACTIVE; - mpts_alt->mpts_peerswitch = 0; - struct mptcb *mp_tp = mpte->mpte_mptcb; - /* Bring the subflow's notion of snd_nxt into the send window */ - MPT_LOCK(mp_tp); - mpts_alt->mpts_sndnxt = mp_tp->mpt_snduna; - MPT_UNLOCK(mp_tp); - mpte->mpte_active_sub = mpts_alt; - socket_lock(so, 1); - sowwakeup(so); - socket_unlock(so, 1); } - MPTS_UNLOCK(mpts_alt); if (altpath_exists) { - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNINFO_UPDATED; - mptcplog((LOG_NOTICE, "MPTCP Events: " - "%s: mp_so 0x%llx switched from " - "%d to %d\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mpts->mpts_connid, mpts_alt->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - tcpstat.tcps_mp_switches++; - } + mpts_alt->mpts_flags |= MPTSF_ACTIVE; - MPTS_LOCK(mpts); - if (altpath_exists) { + mpte->mpte_active_sub = mpts_alt; mpts->mpts_flags |= MPTSF_FAILINGOVER; mpts->mpts_flags &= ~MPTSF_ACTIVE; + + mptcplog((LOG_NOTICE, "%s: switched from %d to %d\n", + __func__, mpts->mpts_connid, mpts_alt->mpts_connid), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + + mptcpstats_inc_switch(mpte, mpts); + + sowwakeup(alt_so); } else { - mptcplog((LOG_DEBUG, "MPTCP Events %s: no alt cid = %d\n", - __func__, mpts->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: no alt cid = %d\n", __func__, + mpts->mpts_connid), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); done: - so = mpts->mpts_socket; - socket_lock(so, 1); - so->so_flags &= ~SOF_MP_TRYFAILOVER; - socket_unlock(so, 1); + mpts->mpts_socket->so_flags &= ~SOF_MP_TRYFAILOVER; } - MPTS_LOCK_ASSERT_HELD(mpts); + return (MPTS_EVRET_OK); } @@ -2583,102 +3283,23 @@ done: */ static ev_ret_t mptcp_subflow_ifdenied_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { - struct socket *mp_so, *so; - struct mptcb *mp_tp; - boolean_t linger; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; - mp_tp = mpte->mpte_mptcb; - so = mpts->mpts_socket; - - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); - - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d [linger %s]\n", __func__, - mpts->mpts_connid, (linger ? "YES" : "NO")), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - - if (mpts->mpts_soerror == 0) - mpts->mpts_soerror = EHOSTUNREACH; - /* - * The subflow connection cannot use the outgoing interface. - * - * Right now, we simply propagate EHOSTUNREACH to the MPTCP socket - * client if the MPTCP connection has not been established. If it - * has been established, let the upper layer call disconnectx. - */ - mptcp_subflow_disconnect(mpte, mpts, !linger); - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_IFDENIED; - - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - mp_so->so_error = EHOSTUNREACH; - } - MPT_UNLOCK(mp_tp); + mptcplog((LOG_DEBUG, "%s: cid %d\n", __func__, + mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); /* - * Keep the subflow socket around, unless the MPTCP socket has - * been detached or the subflow has been disconnected explicitly, - * in which case it should be deleted right away. + * The subflow connection cannot use the outgoing interface, let's + * close this subflow. */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); -} - -/* - * Handle SO_FILT_HINT_SUSPEND subflow socket event. - */ -static ev_ret_t -mptcp_subflow_suspend_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) -{ -#pragma unused(p_mpsofilt_hint) - struct socket *so; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mptcp_subflow_abort(mpts, EPERM); - so = mpts->mpts_socket; - - /* the subflow connection is being flow controlled */ - mpts->mpts_flags |= MPTSF_SUSPENDED; - - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d\n", __func__, - mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - - return (MPTS_EVRET_OK); /* keep the subflow socket around */ -} - -/* - * Handle SO_FILT_HINT_RESUME subflow socket event. - */ -static ev_ret_t -mptcp_subflow_resume_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) -{ -#pragma unused(p_mpsofilt_hint) - struct socket *so; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - - so = mpts->mpts_socket; - - /* the subflow connection is no longer flow controlled */ - mpts->mpts_flags &= ~MPTSF_SUSPENDED; - - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d\n", __func__, mpts->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + mptcp_subflow_propagate_ev(mpte, mpts, p_mpsofilt_hint, event); - return (MPTS_EVRET_OK); /* keep the subflow socket around */ + return (MPTS_EVRET_DELETE); } /* @@ -2686,45 +3307,39 @@ mptcp_subflow_resume_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { - char buf0[MAX_IPv6_STR_LEN], buf1[MAX_IPv6_STR_LEN]; - struct sockaddr_storage src; +#pragma unused(event, p_mpsofilt_hint) struct socket *mp_so, *so; + struct inpcb *inp; + struct tcpcb *tp; struct mptcb *mp_tp; - struct ifnet *outifp; - int af, error = 0; + int af; boolean_t mpok = FALSE; - boolean_t cell = FALSE; - boolean_t wifi = FALSE; - boolean_t wired = FALSE; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; - mp_tp = mpte->mpte_mptcb; - MPTS_LOCK_ASSERT_HELD(mpts); + mp_so = mptetoso(mpte); + mp_tp = mpte->mpte_mptcb; so = mpts->mpts_socket; - af = mpts->mpts_family; + tp = sototcpcb(so); + af = mpts->mpts_dst.sa_family; if (mpts->mpts_flags & MPTSF_CONNECTED) return (MPTS_EVRET_OK); if ((mpts->mpts_flags & MPTSF_DISCONNECTED) || (mpts->mpts_flags & MPTSF_DISCONNECTING)) { - socket_lock(so, 0); if (!(so->so_state & (SS_ISDISCONNECTING | SS_ISDISCONNECTED)) && (so->so_state & SS_ISCONNECTED)) { - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d disconnect before tcp connect\n", + mptcplog((LOG_DEBUG, "%s: cid %d disconnect before tcp connect\n", __func__, mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); (void) soshutdownlock(so, SHUT_RD); (void) soshutdownlock(so, SHUT_WR); (void) sodisconnectlocked(so); } - socket_unlock(so, 0); return (MPTS_EVRET_OK); } @@ -2734,19 +3349,15 @@ mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts, * * a. If MPTCP connection is not yet established, then this must be * the first subflow connection. If MPTCP failed to negotiate, - * indicate to the MPTCP socket client via EPROTO, that the - * underlying TCP connection may be peeled off via peeloff(2). - * Otherwise, mark the MPTCP socket as connected. + * fallback to regular TCP by degrading this subflow. * * b. If MPTCP connection has been established, then this must be * one of the subsequent subflow connections. If MPTCP failed - * to negotiate, disconnect the connection since peeloff(2) - * is no longer possible. + * to negotiate, disconnect the connection. * * Right now, we simply unblock any waiters at the MPTCP socket layer * if the MPTCP connection has not been established. */ - socket_lock(so, 0); if (so->so_state & SS_ISDISCONNECTED) { /* @@ -2756,203 +3367,80 @@ mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts, * where the subflow could get disconnected before the * connected event is processed. */ - socket_unlock(so, 0); return (MPTS_EVRET_OK); } - mpts->mpts_soerror = 0; - mpts->mpts_flags &= ~MPTSF_CONNECTING; - mpts->mpts_flags |= MPTSF_CONNECTED; + if (mpts->mpts_flags & MPTSF_TFO_REQD) + mptcp_drop_tfo_data(mpte, mpts); - if (!(so->so_flags1 & SOF1_DATA_IDEMPOTENT)) - mpts->mpts_flags &= ~MPTSF_TFO_REQD; + mpts->mpts_flags &= ~(MPTSF_CONNECTING | MPTSF_TFO_REQD); + mpts->mpts_flags |= MPTSF_CONNECTED; - struct tcpcb *tp = sototcpcb(so); if (tp->t_mpflags & TMPF_MPTCP_TRUE) mpts->mpts_flags |= MPTSF_MP_CAPABLE; tp->t_mpflags &= ~TMPF_TFO_REQUEST; - VERIFY(mpts->mpts_dst != NULL); - - VERIFY(mpts->mpts_src != NULL); - - /* get/check source IP address */ - switch (af) { - case AF_INET: { - error = in_getsockaddr_s(so, &src); - if (error == 0) { - struct sockaddr_in *ms = SIN(mpts->mpts_src); - struct sockaddr_in *s = SIN(&src); - - VERIFY(s->sin_len == ms->sin_len); - VERIFY(ms->sin_family == AF_INET); - - if ((mpts->mpts_flags & MPTSF_BOUND_IP) && - bcmp(&ms->sin_addr, &s->sin_addr, - sizeof (ms->sin_addr)) != 0) { - mptcplog((LOG_ERR, "MPTCP Events: " - "%s: cid %d local " - "address %s (expected %s)\n", __func__, - mpts->mpts_connid, inet_ntop(AF_INET, - (void *)&s->sin_addr.s_addr, buf0, - sizeof (buf0)), inet_ntop(AF_INET, - (void *)&ms->sin_addr.s_addr, buf1, - sizeof (buf1))), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); - } - bcopy(s, ms, sizeof (*s)); - } - break; - } -#if INET6 - case AF_INET6: { - error = in6_getsockaddr_s(so, &src); - if (error == 0) { - struct sockaddr_in6 *ms = SIN6(mpts->mpts_src); - struct sockaddr_in6 *s = SIN6(&src); - - VERIFY(s->sin6_len == ms->sin6_len); - VERIFY(ms->sin6_family == AF_INET6); - - if ((mpts->mpts_flags & MPTSF_BOUND_IP) && - bcmp(&ms->sin6_addr, &s->sin6_addr, - sizeof (ms->sin6_addr)) != 0) { - mptcplog((LOG_ERR, "MPTCP Events: " - "%s: cid %d local " - "address %s (expected %s)\n", __func__, - mpts->mpts_connid, inet_ntop(AF_INET6, - (void *)&s->sin6_addr, buf0, - sizeof (buf0)), inet_ntop(AF_INET6, - (void *)&ms->sin6_addr, buf1, - sizeof (buf1))), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); - } - bcopy(s, ms, sizeof (*s)); - } - break; - } -#endif /* INET6 */ - default: - VERIFY(0); - /* NOTREACHED */ - } - - if (error != 0) { - mptcplog((LOG_ERR, "MPTCP Events " - "%s: cid %d getsockaddr failed (%d)\n", - __func__, mpts->mpts_connid, error), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); - } - /* get/verify the outbound interface */ - outifp = sotoinpcb(so)->inp_last_outifp; /* could be NULL */ - if (mpts->mpts_flags & MPTSF_BOUND_IF) { - VERIFY(mpts->mpts_outif != NULL); - if (mpts->mpts_outif != outifp) { - mptcplog((LOG_ERR, "MPTCP Events: %s: cid %d outif %s " - "(expected %s)\n", __func__, mpts->mpts_connid, - ((outifp != NULL) ? outifp->if_xname : "NULL"), - mpts->mpts_outif->if_xname), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); - - if (outifp == NULL) - outifp = mpts->mpts_outif; - } - } else { - mpts->mpts_outif = outifp; - } - - mpts->mpts_srtt = (intotcpcb(sotoinpcb(so)))->t_srtt; - mpts->mpts_rxtcur = (intotcpcb(sotoinpcb(so)))->t_rxtcur; - mpts->mpts_maxseg = (intotcpcb(sotoinpcb(so)))->t_maxseg; - - cell = IFNET_IS_CELLULAR(mpts->mpts_outif); - wifi = (!cell && IFNET_IS_WIFI(mpts->mpts_outif)); - wired = (!wifi && IFNET_IS_WIRED(mpts->mpts_outif)); - - if (cell) - mpts->mpts_linktype |= MPTSL_CELL; - else if (wifi) - mpts->mpts_linktype |= MPTSL_WIFI; - else if (wired) - mpts->mpts_linktype |= MPTSL_WIRED; - - socket_unlock(so, 0); - - mptcplog((LOG_DEBUG, "MPTCP Sender: %s: cid %d " - "establishment srtt %d \n", __func__, - mpts->mpts_connid, (mpts->mpts_srtt >> 5)), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + inp = sotoinpcb(so); + mpts->mpts_maxseg = tp->t_maxseg; - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: cid %d outif %s %s[%d] -> %s[%d] " - "is %s\n", __func__, mpts->mpts_connid, ((outifp != NULL) ? - outifp->if_xname : "NULL"), inet_ntop(af, (af == AF_INET) ? - (void *)&SIN(mpts->mpts_src)->sin_addr.s_addr : - (void *)&SIN6(mpts->mpts_src)->sin6_addr, buf0, sizeof (buf0)), - ((af == AF_INET) ? ntohs(SIN(mpts->mpts_src)->sin_port) : - ntohs(SIN6(mpts->mpts_src)->sin6_port)), - inet_ntop(af, ((af == AF_INET) ? - (void *)&SIN(mpts->mpts_dst)->sin_addr.s_addr : - (void *)&SIN6(mpts->mpts_dst)->sin6_addr), buf1, sizeof (buf1)), - ((af == AF_INET) ? ntohs(SIN(mpts->mpts_dst)->sin_port) : - ntohs(SIN6(mpts->mpts_dst)->sin6_port)), - ((mpts->mpts_flags & MPTSF_MP_CAPABLE) ? - "MPTCP capable" : "a regular TCP")), + mptcplog((LOG_DEBUG, "%s: cid %d outif %s is %s\n", __func__, mpts->mpts_connid, + ((inp->inp_last_outifp != NULL) ? inp->inp_last_outifp->if_xname : "NULL"), + ((mpts->mpts_flags & MPTSF_MP_CAPABLE) ? "MPTCP capable" : "a regular TCP")), (MPTCP_SOCKET_DBG | MPTCP_EVENTS_DBG), MPTCP_LOGLVL_LOG); mpok = (mpts->mpts_flags & MPTSF_MP_CAPABLE); - MPTS_UNLOCK(mpts); - - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNINFO_UPDATED; - MPT_LOCK(mp_tp); if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { + mp_tp->mpt_state = MPTCPS_ESTABLISHED; + mpte->mpte_associd = mpts->mpts_connid; + DTRACE_MPTCP2(state__change, + struct mptcb *, mp_tp, + uint32_t, 0 /* event */); + + if (SOCK_DOM(so) == AF_INET) { + in_getsockaddr_s(so, &mpte->__mpte_src_v4); + } else { + in6_getsockaddr_s(so, &mpte->__mpte_src_v6); + } + /* case (a) above */ if (!mpok) { - mp_tp->mpt_flags |= MPTCPF_PEEL_OFF; - (void) mptcp_drop(mpte, mp_tp, EPROTO); - MPT_UNLOCK(mp_tp); + tcpstat.tcps_mpcap_fallback++; + + tp->t_mpflags |= TMPF_INFIN_SENT; + mptcp_notify_mpfail(so); } else { - MPT_UNLOCK(mp_tp); - mptcplog((LOG_DEBUG, "MPTCP State: " - "MPTCPS_ESTABLISHED for mp_so 0x%llx \n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), - MPTCP_STATE_DBG, MPTCP_LOGLVL_LOG); - mp_tp->mpt_state = MPTCPS_ESTABLISHED; - mpte->mpte_associd = mpts->mpts_connid; - DTRACE_MPTCP2(state__change, - struct mptcb *, mp_tp, - uint32_t, 0 /* event */); - - if (mpts->mpts_outif && - IFNET_IS_EXPENSIVE(mpts->mpts_outif)) { - sototcpcb(so)->t_mpflags |= (TMPF_BACKUP_PATH | TMPF_SND_MPPRIO); + if (IFNET_IS_CELLULAR(inp->inp_last_outifp) && + mpte->mpte_svctype != MPTCP_SVCTYPE_AGGREGATE) { + tp->t_mpflags |= (TMPF_BACKUP_PATH | TMPF_SND_MPPRIO); } else { mpts->mpts_flags |= MPTSF_PREFERRED; } mpts->mpts_flags |= MPTSF_ACTIVE; - soisconnected(mp_so); - } - MPTS_LOCK(mpts); - if (mpok) { + mpts->mpts_flags |= MPTSF_MPCAP_CTRSET; mpte->mpte_nummpcapflows++; - MPT_LOCK_SPIN(mp_tp); - /* With TFO, sndnxt may be initialized earlier */ - if (mpts->mpts_sndnxt == 0) - mpts->mpts_sndnxt = mp_tp->mpt_snduna; - MPT_UNLOCK(mp_tp); + + mptcp_check_subflows_and_add(mpte); + + if (IFNET_IS_CELLULAR(inp->inp_last_outifp)) + mpte->mpte_initial_cell = 1; + + mpte->mpte_handshake_success = 1; } + + mp_tp->mpt_sndwnd = tp->snd_wnd; + mp_tp->mpt_sndwl1 = mp_tp->mpt_rcvnxt; + mp_tp->mpt_sndwl2 = mp_tp->mpt_snduna; + soisconnected(mp_so); + + mptcplog((LOG_DEBUG, "%s: MPTCPS_ESTABLISHED for mp_so 0x%llx mpok %u\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpok), + MPTCP_STATE_DBG, MPTCP_LOGLVL_LOG); } else if (mpok) { - MPT_UNLOCK(mp_tp); - if (mptcp_rwnotify && (mpte->mpte_nummpcapflows == 0)) { - /* Experimental code, disabled by default. */ - sorwakeup(mp_so); - sowwakeup(mp_so); - } /* * case (b) above * In case of additional flows, the MPTCP socket is not @@ -2960,24 +3448,49 @@ mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts, * for 3-way handshake. TCP would have guaranteed that this * is an MPTCP subflow. */ - MPTS_LOCK(mpts); + if (IFNET_IS_CELLULAR(inp->inp_last_outifp) && + !(tp->t_mpflags & TMPF_BACKUP_PATH) && + mpte->mpte_svctype != MPTCP_SVCTYPE_AGGREGATE) { + tp->t_mpflags |= (TMPF_BACKUP_PATH | TMPF_SND_MPPRIO); + mpts->mpts_flags &= ~MPTSF_PREFERRED; + } else { + mpts->mpts_flags |= MPTSF_PREFERRED; + } + mpts->mpts_flags |= MPTSF_MPCAP_CTRSET; - mpts->mpts_flags &= ~MPTSF_FASTJ_REQD; mpte->mpte_nummpcapflows++; - MPT_LOCK_SPIN(mp_tp); - /* With Fastjoin, sndnxt is updated before connected_ev */ - if (mpts->mpts_sndnxt == 0) { - mpts->mpts_sndnxt = mp_tp->mpt_snduna; - mpts->mpts_rel_seq = 1; - } - MPT_UNLOCK(mp_tp); - mptcp_output_needed(mpte, mpts); + + mpts->mpts_rel_seq = 1; + + mptcp_check_subflows_and_remove(mpte); } else { - MPT_UNLOCK(mp_tp); - MPTS_LOCK(mpts); + unsigned int i; + + /* Mark this interface as non-MPTCP */ + for (i = 0; i < mpte->mpte_itfinfo_size; i++) { + struct mpt_itf_info *info = &mpte->mpte_itfinfo[i]; + + if (inp->inp_last_outifp->if_index == info->ifindex) { + info->no_mptcp_support = 1; + break; + } + } + + tcpstat.tcps_join_fallback++; + if (IFNET_IS_CELLULAR(inp->inp_last_outifp)) + tcpstat.tcps_mptcp_cell_proxy++; + else + tcpstat.tcps_mptcp_wifi_proxy++; + + soevent(mpts->mpts_socket, SO_FILT_HINT_LOCKED | SO_FILT_HINT_MUSTRST); + + return (MPTS_EVRET_OK); } - MPTS_LOCK_ASSERT_HELD(mpts); + /* This call, just to "book" an entry in the stats-table for this ifindex */ + mptcp_get_statsindex(mpte->mpte_itfstats, mpts); + + mptcp_output(mpte); return (MPTS_EVRET_OK); /* keep the subflow socket around */ } @@ -2987,77 +3500,56 @@ mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_disconnected_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { +#pragma unused(event, p_mpsofilt_hint) struct socket *mp_so, *so; struct mptcb *mp_tp; - boolean_t linger; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; so = mpts->mpts_socket; - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); - - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d [linger %s]\n", __func__, - mpts->mpts_connid, (linger ? "YES" : "NO")), + mptcplog((LOG_DEBUG, "%s: cid %d, so_err %d, mpt_state %u fallback %u active %u flags %#x\n", + __func__, mpts->mpts_connid, so->so_error, mp_tp->mpt_state, + !!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP), + !!(mpts->mpts_flags & MPTSF_ACTIVE), sototcpcb(so)->t_mpflags), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); if (mpts->mpts_flags & MPTSF_DISCONNECTED) - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); + return (MPTS_EVRET_DELETE); - /* - * Clear flags that are used by getconninfo to return state. - * Retain like MPTSF_DELETEOK for internal purposes. - */ - mpts->mpts_flags &= ~(MPTSF_CONNECTING|MPTSF_CONNECT_PENDING| - MPTSF_CONNECTED|MPTSF_DISCONNECTING|MPTSF_PREFERRED| - MPTSF_MP_CAPABLE|MPTSF_MP_READY|MPTSF_MP_DEGRADED| - MPTSF_SUSPENDED|MPTSF_ACTIVE); mpts->mpts_flags |= MPTSF_DISCONNECTED; - /* - * The subflow connection has been disconnected. - * - * Right now, we simply unblock any waiters at the MPTCP socket layer - * if the MPTCP connection has not been established. - */ - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNINFO_UPDATED; + /* The subflow connection has been disconnected. */ if (mpts->mpts_flags & MPTSF_MPCAP_CTRSET) { mpte->mpte_nummpcapflows--; if (mpte->mpte_active_sub == mpts) { mpte->mpte_active_sub = NULL; - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: resetting active subflow \n", + mptcplog((LOG_DEBUG, "%s: resetting active subflow \n", __func__), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); } mpts->mpts_flags &= ~MPTSF_MPCAP_CTRSET; } - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - MPT_UNLOCK(mp_tp); - MPTS_UNLOCK(mpts); - soisdisconnected(mp_so); - MPTS_LOCK(mpts); - } else { - MPT_UNLOCK(mp_tp); + if (mp_tp->mpt_state < MPTCPS_ESTABLISHED || + ((mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && (mpts->mpts_flags & MPTSF_ACTIVE)) || + (sototcpcb(so)->t_mpflags & TMPF_FASTCLOSERCV)) { + mptcp_drop(mpte, mp_tp, so->so_error); } /* - * The underlying subflow socket has been disconnected; - * it is no longer useful to us. Keep the subflow socket - * around, unless the MPTCP socket has been detached or - * the subflow has been disconnected explicitly, in which - * case it should be deleted right away. + * Clear flags that are used by getconninfo to return state. + * Retain like MPTSF_DELETEOK for internal purposes. */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); + mpts->mpts_flags &= ~(MPTSF_CONNECTING|MPTSF_CONNECT_PENDING| + MPTSF_CONNECTED|MPTSF_DISCONNECTING|MPTSF_PREFERRED| + MPTSF_MP_CAPABLE|MPTSF_MP_READY|MPTSF_MP_DEGRADED|MPTSF_ACTIVE); + + return (MPTS_EVRET_DELETE); } /* @@ -3065,23 +3557,19 @@ mptcp_subflow_disconnected_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_mpstatus_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { +#pragma unused(event, p_mpsofilt_hint) struct socket *mp_so, *so; struct mptcb *mp_tp; ev_ret_t ret = MPTS_EVRET_OK; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; - - MPTS_LOCK_ASSERT_HELD(mpts); so = mpts->mpts_socket; - socket_lock(so, 0); - MPT_LOCK(mp_tp); - if (sototcpcb(so)->t_mpflags & TMPF_MPTCP_TRUE) mpts->mpts_flags |= MPTSF_MP_CAPABLE; else @@ -3108,27 +3596,18 @@ mptcp_subflow_mpstatus_ev(struct mptses *mpte, struct mptsub *mpts, if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) { VERIFY(!(mp_tp->mpt_flags & MPTCPF_JOIN_READY)); ret = MPTS_EVRET_DISCONNECT_FALLBACK; - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | - SO_FILT_HINT_CONNINFO_UPDATED; } else if (mpts->mpts_flags & MPTSF_MP_READY) { mp_tp->mpt_flags |= MPTCPF_JOIN_READY; ret = MPTS_EVRET_CONNECT_PENDING; - } else { - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | - SO_FILT_HINT_CONNINFO_UPDATED; } - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: mp_so 0x%llx mpt_flags=%b cid %d " - "mptsf=%b\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mpte->mpte_mppcb->mpp_socket), - mp_tp->mpt_flags, MPTCPF_BITS, mpts->mpts_connid, - mpts->mpts_flags, MPTSF_BITS), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx mpt_flags=%b cid %d mptsf=%b\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mp_tp->mpt_flags, MPTCPF_BITS, mpts->mpts_connid, + mpts->mpts_flags, MPTSF_BITS), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); done: - MPT_UNLOCK(mp_tp); - socket_unlock(so, 0); return (ret); } @@ -3137,28 +3616,20 @@ done: */ static ev_ret_t mptcp_subflow_mustrst_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { +#pragma unused(event) struct socket *mp_so, *so; struct mptcb *mp_tp; - boolean_t linger, is_fastclose; + boolean_t is_fastclose; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; so = mpts->mpts_socket; - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); - - if (mpts->mpts_soerror == 0) - mpts->mpts_soerror = ECONNABORTED; - /* We got an invalid option or a fast close */ - socket_lock(so, 0); struct tcptemp *t_template; struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = NULL; @@ -3189,12 +3660,7 @@ mptcp_subflow_mustrst_ev(struct mptses *mpte, struct mptsub *mpts, so, mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); } - socket_unlock(so, 0); - mptcp_subflow_disconnect(mpte, mpts, !linger); - - *p_mpsofilt_hint |= (SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNINFO_UPDATED); - - MPT_LOCK(mp_tp); + mptcp_subflow_abort(mpts, ECONNABORTED); if (!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && is_fastclose) { *p_mpsofilt_hint |= SO_FILT_HINT_CONNRESET; @@ -3212,65 +3678,64 @@ mptcp_subflow_mustrst_ev(struct mptses *mpte, struct mptsub *mpts, if (mp_tp->mpt_gc_ticks == MPT_GC_TICKS) mp_tp->mpt_gc_ticks = MPT_GC_TICKS_FAST; - MPT_UNLOCK(mp_tp); - /* - * Keep the subflow socket around unless the subflow has been - * disconnected explicitly. - */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); + return (MPTS_EVRET_DELETE); } static ev_ret_t -mptcp_fastjoin_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +mptcp_subflow_adaptive_rtimo_ev(struct mptses *mpte, struct mptsub *mpts, + uint64_t *p_mpsofilt_hint, uint64_t event) { -#pragma unused(p_mpsofilt_hint) - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - VERIFY(mpte->mpte_mppcb != NULL); +#pragma unused(event) + bool found_active = false; + + mpts->mpts_flags |= MPTSF_READ_STALL; - if (mpte->mpte_nummpcapflows == 0) { - struct mptcb *mp_tp = mpte->mpte_mptcb; - mptcplog((LOG_DEBUG,"MPTCP Events: %s: %llx %llx \n", - __func__, mp_tp->mpt_snduna, mpts->mpts_sndnxt), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); - mpte->mpte_active_sub = mpts; - mpts->mpts_flags |= (MPTSF_FASTJ_SEND | MPTSF_ACTIVE); - MPT_LOCK(mp_tp); - /* - * If mptcp_subflow_output is called before fastjoin_ev - * then mpts->mpts_sndnxt is initialized to mp_tp->mpt_snduna - * and further mpts->mpts_sndnxt is incremented by len copied. - */ - if (mpts->mpts_sndnxt == 0) { - mpts->mpts_sndnxt = mp_tp->mpt_snduna; + if (!TCPS_HAVEESTABLISHED(tp->t_state) || + TCPS_HAVERCVDFIN2(tp->t_state)) + continue; + + if (!(mpts->mpts_flags & MPTSF_READ_STALL)) { + found_active = true; + break; } - MPT_UNLOCK(mp_tp); } + if (!found_active) + *p_mpsofilt_hint |= SO_FILT_HINT_ADAPTIVE_RTIMO; + return (MPTS_EVRET_OK); } static ev_ret_t -mptcp_deleteok_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +mptcp_subflow_adaptive_wtimo_ev(struct mptses *mpte, struct mptsub *mpts, + uint64_t *p_mpsofilt_hint, uint64_t event) { -#pragma unused(p_mpsofilt_hint) - MPTE_LOCK_ASSERT_HELD(mpte); - MPTS_LOCK_ASSERT_HELD(mpts); - VERIFY(mpte->mpte_mppcb != NULL); +#pragma unused(event) + bool found_active = false; - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s cid %d\n", __func__, mpts->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + mpts->mpts_flags |= MPTSF_WRITE_STALL; - mpts->mpts_flags |= MPTSF_DELETEOK; - if (mpts->mpts_flags & MPTSF_DISCONNECTED) - return (MPTS_EVRET_DELETE); - else - return (MPTS_EVRET_OK); + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); + + if (!TCPS_HAVEESTABLISHED(tp->t_state) || + tp->t_state > TCPS_CLOSE_WAIT) + continue; + + if (!(mpts->mpts_flags & MPTSF_WRITE_STALL)) { + found_active = true; + break; + } + } + + if (!found_active) + *p_mpsofilt_hint |= SO_FILT_HINT_ADAPTIVE_WTIMO; + + return (MPTS_EVRET_OK); } static const char * @@ -3297,63 +3762,58 @@ mptcp_evret2str(ev_ret_t ret) return (c); } -/* - * Add a reference to a subflow structure; used by MPTS_ADDREF(). - */ -void -mptcp_subflow_addref(struct mptsub *mpts, int locked) -{ - if (!locked) - MPTS_LOCK(mpts); - else - MPTS_LOCK_ASSERT_HELD(mpts); - - if (++mpts->mpts_refcnt == 0) { - panic("%s: mpts %p wraparound refcnt\n", __func__, mpts); - /* NOTREACHED */ - } - if (!locked) - MPTS_UNLOCK(mpts); -} - -/* - * Remove a reference held on a subflow structure; used by MPTS_REMREF(); - */ -void -mptcp_subflow_remref(struct mptsub *mpts) -{ - MPTS_LOCK(mpts); - if (mpts->mpts_refcnt == 0) { - panic("%s: mpts %p negative refcnt\n", __func__, mpts); - /* NOTREACHED */ - } - if (--mpts->mpts_refcnt > 0) { - MPTS_UNLOCK(mpts); - return; - } - /* callee will unlock and destroy lock */ - mptcp_subflow_free(mpts); -} - /* * Issues SOPT_SET on an MPTCP subflow socket; socket must already be locked, * caller must ensure that the option can be issued on subflow sockets, via * MPOF_SUBFLOW_OK flag. */ int -mptcp_subflow_sosetopt(struct mptses *mpte, struct socket *so, - struct mptopt *mpo) +mptcp_subflow_sosetopt(struct mptses *mpte, struct mptsub *mpts, struct mptopt *mpo) { - struct socket *mp_so; + struct socket *mp_so, *so; struct sockopt sopt; - char buf[32]; int error; VERIFY(mpo->mpo_flags & MPOF_SUBFLOW_OK); - mpo->mpo_flags &= ~MPOF_INTERIM; + mpte_lock_assert_held(mpte); + + mp_so = mptetoso(mpte); + so = mpts->mpts_socket; + + if (mpte->mpte_mptcb->mpt_state >= MPTCPS_ESTABLISHED && + mpo->mpo_level == SOL_SOCKET && + mpo->mpo_name == SO_MARK_CELLFALLBACK) { + mptcplog((LOG_DEBUG, "%s Setting CELL_FALLBACK, mpte_flags %#x, svctype %u wifi unusable %u lastcell? %d boundcell? %d\n", + __func__, mpte->mpte_flags, mpte->mpte_svctype, mptcp_is_wifi_unusable(), + sotoinpcb(so)->inp_last_outifp ? IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp) : -1, + mpts->mpts_ifscope != IFSCOPE_NONE ? IFNET_IS_CELLULAR(ifindex2ifnet[mpts->mpts_ifscope]) : -1), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + /* + * When we open a new subflow, mark it as cell fallback, if + * this subflow goes over cell. + * + * (except for first-party apps) + */ + + if (mpte->mpte_flags & MPTE_FIRSTPARTY) + return (0); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + if (sotoinpcb(so)->inp_last_outifp && + !IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp)) + return (0); + + /* + * This here is an OR, because if the app is not binding to the + * interface, then it definitely is not a cell-fallback + * connection. + */ + if (mpts->mpts_ifscope == IFSCOPE_NONE || + !IFNET_IS_CELLULAR(ifindex2ifnet[mpts->mpts_ifscope])) + return (0); + } + + mpo->mpo_flags &= ~MPOF_INTERIM; bzero(&sopt, sizeof (sopt)); sopt.sopt_dir = SOPT_SET; @@ -3363,23 +3823,21 @@ mptcp_subflow_sosetopt(struct mptses *mpte, struct socket *so, sopt.sopt_valsize = sizeof (int); sopt.sopt_p = kernproc; - error = sosetoptlock(so, &sopt, 0); /* already locked */ + error = sosetoptlock(so, &sopt, 0); if (error == 0) { - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx sopt %s " + mptcplog((LOG_INFO, "%s: mp_so 0x%llx sopt %s " "val %d set successful\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name, - buf, sizeof (buf)), mpo->mpo_intval), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), + mpo->mpo_intval), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); } else { - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: mp_so 0x%llx sopt %s " + mptcplog((LOG_ERR, "%s:mp_so 0x%llx sopt %s " "val %d set error %d\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name, - buf, sizeof (buf)), mpo->mpo_intval, error), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), + mpo->mpo_intval, error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); } return (error); } @@ -3395,12 +3853,11 @@ mptcp_subflow_sogetopt(struct mptses *mpte, struct socket *so, { struct socket *mp_so; struct sockopt sopt; - char buf[32]; int error; VERIFY(mpo->mpo_flags & MPOF_SUBFLOW_OK); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); bzero(&sopt, sizeof (sopt)); sopt.sopt_dir = SOPT_GET; @@ -3416,15 +3873,14 @@ mptcp_subflow_sogetopt(struct mptses *mpte, struct socket *so, "%s: mp_so 0x%llx sopt %s " "val %d get successful\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name, - buf, sizeof (buf)), mpo->mpo_intval), + mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), + mpo->mpo_intval), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); } else { mptcplog((LOG_ERR, "MPTCP Socket: " "%s: mp_so 0x%llx sopt %s get error %d\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(mpo->mpo_level, - mpo->mpo_name, buf, sizeof (buf)), error), + mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); } return (error); @@ -3444,7 +3900,7 @@ mptcp_gc(struct mppcbinfo *mppi) struct mppcb *mpp, *tmpp; uint32_t active = 0; - lck_mtx_assert(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); TAILQ_FOREACH_SAFE(mpp, &mppi->mppi_pcbs, mpp_entry, tmpp) { struct socket *mp_so; @@ -3466,9 +3922,9 @@ mptcp_gc(struct mppcbinfo *mppi) mp_so->so_retaincnt, mpp->mpp_state), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - if (!lck_mtx_try_lock(&mpp->mpp_lock)) { + if (!mpte_try_lock(mpte)) { mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx skipped " + "%s: mp_so 0x%llx skipped lock " "(u=%d,r=%d)\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mp_so->so_usecount, mp_so->so_retaincnt), @@ -3478,12 +3934,12 @@ mptcp_gc(struct mppcbinfo *mppi) } /* check again under the lock */ - if (mp_so->so_usecount > 1) { + if (mp_so->so_usecount > 0) { boolean_t wakeup = FALSE; struct mptsub *mpts, *tmpts; mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx skipped " + "%s: mp_so 0x%llx skipped usecount " "[u=%d,r=%d] %d %d\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mp_so->so_usecount, mp_so->so_retaincnt, @@ -3491,70 +3947,37 @@ mptcp_gc(struct mppcbinfo *mppi) mp_tp->mpt_state), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - MPT_LOCK(mp_tp); if (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_1) { if (mp_tp->mpt_gc_ticks > 0) mp_tp->mpt_gc_ticks--; if (mp_tp->mpt_gc_ticks == 0) { wakeup = TRUE; - if (mp_tp->mpt_localkey != NULL) { - mptcp_free_key( - mp_tp->mpt_localkey); - mp_tp->mpt_localkey = NULL; - } } } - MPT_UNLOCK(mp_tp); if (wakeup) { TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { - MPTS_LOCK(mpts); - mpts->mpts_flags |= MPTSF_DELETEOK; - if (mpts->mpts_soerror == 0) - mpts->mpts_soerror = ETIMEDOUT; - mptcp_subflow_eupcall(mpts->mpts_socket, + mptcp_subflow_eupcall1(mpts->mpts_socket, mpts, SO_FILT_HINT_DISCONNECTED); - MPTS_UNLOCK(mpts); } } - lck_mtx_unlock(&mpp->mpp_lock); + mpte_unlock(mpte); active++; continue; } if (mpp->mpp_state != MPPCB_STATE_DEAD) { - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx skipped " - "[u=%d,r=%d,s=%d]\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mp_so->so_usecount, mp_so->so_retaincnt, - mpp->mpp_state), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - lck_mtx_unlock(&mpp->mpp_lock); - active++; - continue; + panic("MPTCP Socket: %s: mp_so 0x%llx skipped state " + "[u=%d,r=%d,s=%d]\n", __func__, + (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mp_so->so_usecount, mp_so->so_retaincnt, + mpp->mpp_state); } - /* - * The PCB has been detached, and there is exactly 1 refnct - * held by the MPTCP thread. Signal that thread to terminate, - * after which the last refcnt will be released. That will - * allow it to be destroyed below during the next round. - */ - if (mp_so->so_usecount == 1) { - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx scheduled for " - "termination [u=%d,r=%d]\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mp_so->so_usecount, mp_so->so_retaincnt), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + if (mp_tp->mpt_state == MPTCPS_TIME_WAIT) + mptcp_close(mpte, mp_tp); - /* signal MPTCP thread to terminate */ - mptcp_thread_terminate_signal(mpte); - lck_mtx_unlock(&mpp->mpp_lock); - active++; - continue; - } + mptcp_session_destroy(mpte); mptcplog((LOG_DEBUG, "MPTCP Socket: " "%s: mp_so 0x%llx destroyed [u=%d,r=%d]\n", @@ -3582,12 +4005,10 @@ mptcp_drop(struct mptses *mpte, struct mptcb *mp_tp, int errno) { struct socket *mp_so; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPT_LOCK_ASSERT_HELD(mp_tp); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mptcb == mp_tp); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); - mp_tp->mpt_state = MPTCPS_TERMINATE; DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp, uint32_t, 0 /* event */); @@ -3607,33 +4028,20 @@ mptcp_close(struct mptses *mpte, struct mptcb *mp_tp) struct socket *mp_so = NULL; struct mptsub *mpts = NULL, *tmpts = NULL; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPT_LOCK_ASSERT_HELD(mp_tp); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mptcb == mp_tp); - mp_so = mpte->mpte_mppcb->mpp_socket; - if (mp_tp->mpt_localkey != NULL) { - mptcp_free_key(mp_tp->mpt_localkey); - mp_tp->mpt_localkey = NULL; - } + mp_so = mptetoso(mpte); - MPT_UNLOCK(mp_tp); - soisdisconnected(mp_so); + mp_tp->mpt_state = MPTCPS_TERMINATE; - MPT_LOCK(mp_tp); - if (mp_tp->mpt_flags & MPTCPF_PEEL_OFF) { - return (NULL); - } - MPT_UNLOCK(mp_tp); + mptcp_freeq(mp_tp); + + soisdisconnected(mp_so); /* Clean up all subflows */ TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { - MPTS_LOCK(mpts); - mpts->mpts_flags |= MPTSF_USER_DISCONNECT; - mptcp_subflow_disconnect(mpte, mpts, TRUE); - MPTS_UNLOCK(mpts); - mptcp_subflow_del(mpte, mpts, TRUE); + mptcp_subflow_disconnect(mpte, mpts); } - MPT_LOCK(mp_tp); return (NULL); } @@ -3645,84 +4053,34 @@ mptcp_notify_close(struct socket *so) } /* - * Signal MPTCP thread to wake up. + * MPTCP workloop. */ void -mptcp_thread_signal(struct mptses *mpte) -{ - lck_mtx_lock(&mpte->mpte_thread_lock); - mptcp_thread_signal_locked(mpte); - lck_mtx_unlock(&mpte->mpte_thread_lock); -} - -/* - * Signal MPTCP thread to wake up (locked version) - */ -static void -mptcp_thread_signal_locked(struct mptses *mpte) -{ - lck_mtx_assert(&mpte->mpte_thread_lock, LCK_MTX_ASSERT_OWNED); - - mpte->mpte_thread_reqs++; - if (!mpte->mpte_thread_active && mpte->mpte_thread != THREAD_NULL) - wakeup_one((caddr_t)&mpte->mpte_thread); -} - -/* - * Signal MPTCP thread to terminate. - */ -static void -mptcp_thread_terminate_signal(struct mptses *mpte) -{ - lck_mtx_lock(&mpte->mpte_thread_lock); - if (mpte->mpte_thread != THREAD_NULL) { - mpte->mpte_thread = THREAD_NULL; - mpte->mpte_thread_reqs++; - if (!mpte->mpte_thread_active) - wakeup_one((caddr_t)&mpte->mpte_thread); - } - lck_mtx_unlock(&mpte->mpte_thread_lock); -} - -/* - * MPTCP thread workloop. - */ -static void -mptcp_thread_dowork(struct mptses *mpte) +mptcp_subflow_workloop(struct mptses *mpte) { struct socket *mp_so; struct mptsub *mpts, *tmpts; boolean_t connect_pending = FALSE, disconnect_fallback = FALSE; - uint64_t mpsofilt_hint_mask = 0; + uint64_t mpsofilt_hint_mask = SO_FILT_HINT_LOCKED; - MPTE_LOCK(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); VERIFY(mp_so != NULL); TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { ev_ret_t ret; - MPTS_LOCK(mpts); - MPTS_ADDREF_LOCKED(mpts); /* for us */ - - /* Update process ownership based on parent mptcp socket */ - mptcp_update_last_owner(mpts, mp_so); - - mptcp_subflow_input(mpte, mpts); + if (mpts->mpts_socket->so_usecount == 0) { + /* Will be removed soon by tcp_garbage_collect */ + continue; + } - mptcp_get_rtt_measurement(mpts, mpte); + mptcp_subflow_addref(mpts); + mpts->mpts_socket->so_usecount++; ret = mptcp_subflow_events(mpte, mpts, &mpsofilt_hint_mask); - if (mpts->mpts_flags & MPTSF_ACTIVE) { - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: cid %d \n", __func__, - mpts->mpts_connid), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - (void) mptcp_subflow_output(mpte, mpts); - } - /* * If MPTCP socket is closed, disconnect all subflows. * This will generate a disconnect event which will @@ -3730,15 +4088,14 @@ mptcp_thread_dowork(struct mptses *mpte) * non-zero error to be returned above. */ if (mp_so->so_flags & SOF_PCBCLEARING) - mptcp_subflow_disconnect(mpte, mpts, FALSE); - MPTS_UNLOCK(mpts); + mptcp_subflow_disconnect(mpte, mpts); switch (ret) { case MPTS_EVRET_OK: /* nothing to do */ break; case MPTS_EVRET_DELETE: - mptcp_subflow_del(mpte, mpts, TRUE); + mptcp_subflow_soclose(mpts); break; case MPTS_EVRET_CONNECT_PENDING: connect_pending = TRUE; @@ -3754,53 +4111,43 @@ mptcp_thread_dowork(struct mptses *mpte) MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); break; } - MPTS_REMREF(mpts); /* ours */ + mptcp_subflow_remref(mpts); /* ours */ + + VERIFY(mpts->mpts_socket->so_usecount != 0); + mpts->mpts_socket->so_usecount--; } - if (mpsofilt_hint_mask) { + if (mpsofilt_hint_mask != SO_FILT_HINT_LOCKED) { + struct mptcb *mp_tp = mpte->mpte_mptcb; + + VERIFY(mpsofilt_hint_mask & SO_FILT_HINT_LOCKED); + if (mpsofilt_hint_mask & SO_FILT_HINT_CANTRCVMORE) { + mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN); socantrcvmore(mp_so); mpsofilt_hint_mask &= ~SO_FILT_HINT_CANTRCVMORE; } - if (mpsofilt_hint_mask & SO_FILT_HINT_CONNRESET) { - struct mptcb *mp_tp = mpte->mpte_mptcb; - - MPT_LOCK(mp_tp); - mptcp_drop(mpte, mp_tp, ECONNRESET); - MPT_UNLOCK(mp_tp); - } - soevent(mp_so, mpsofilt_hint_mask); } - if (!connect_pending && !disconnect_fallback) { - MPTE_UNLOCK(mpte); + if (!connect_pending && !disconnect_fallback) return; - } TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { - MPTS_LOCK(mpts); if (disconnect_fallback) { struct socket *so = NULL; struct inpcb *inp = NULL; struct tcpcb *tp = NULL; - if (mpts->mpts_flags & MPTSF_MP_DEGRADED) { - MPTS_UNLOCK(mpts); + if (mpts->mpts_flags & MPTSF_MP_DEGRADED) continue; - } mpts->mpts_flags |= MPTSF_MP_DEGRADED; if (mpts->mpts_flags & (MPTSF_DISCONNECTING| - MPTSF_DISCONNECTED|MPTSF_CONNECT_PENDING)) { - MPTS_UNLOCK(mpts); + MPTSF_DISCONNECTED|MPTSF_CONNECT_PENDING)) continue; - } - - if (mpts->mpts_flags & MPTSF_TFO_REQD) - mptcp_drop_tfo_data(mpte, mpts, NULL); so = mpts->mpts_socket; @@ -3813,7 +4160,6 @@ mptcp_thread_dowork(struct mptses *mpte) * ACTIVE one. */ - socket_lock(so, 1); inp = sotoinpcb(so); tp = intotcpcb(inp); tp->t_mpflags &= @@ -3821,26 +4167,11 @@ mptcp_thread_dowork(struct mptses *mpte) tp->t_mpflags |= TMPF_TCP_FALLBACK; if (mpts->mpts_flags & MPTSF_ACTIVE) { - socket_unlock(so, 1); - MPTS_UNLOCK(mpts); continue; } tp->t_mpflags |= TMPF_RESET; - soevent(so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_MUSTRST); - socket_unlock(so, 1); - + soevent(so, SO_FILT_HINT_MUSTRST); } else if (connect_pending) { - /* - * If delayed subflow start is set and cellular, - * delay the connect till a retransmission timeout - */ - - if ((mptcp_delayed_subf_start) && - (IFNET_IS_CELLULAR(mpts->mpts_outif))) { - MPTS_UNLOCK(mpts); - continue; - } - /* * The MPTCP connection has progressed to a state * where it supports full multipath semantics; allow @@ -3848,102 +4179,22 @@ mptcp_thread_dowork(struct mptses *mpte) * that are in the PENDING state. */ if (mpts->mpts_flags & MPTSF_CONNECT_PENDING) { - (void) mptcp_subflow_soconnectx(mpte, mpts); - } - } - MPTS_UNLOCK(mpts); - } - - MPTE_UNLOCK(mpte); -} - -/* - * MPTCP thread. - */ -static void -mptcp_thread_func(void *v, wait_result_t w) -{ -#pragma unused(w) - struct mptses *mpte = v; - struct timespec *ts = NULL; + int error = mptcp_subflow_soconnectx(mpte, mpts); - VERIFY(mpte != NULL); - - lck_mtx_lock_spin(&mpte->mpte_thread_lock); - - for (;;) { - lck_mtx_assert(&mpte->mpte_thread_lock, LCK_MTX_ASSERT_OWNED); - - if (mpte->mpte_thread != THREAD_NULL) { - (void) msleep(&mpte->mpte_thread, - &mpte->mpte_thread_lock, (PZERO - 1) | PSPIN, - __func__, ts); - } - - /* MPTCP socket is closed? */ - if (mpte->mpte_thread == THREAD_NULL) { - lck_mtx_unlock(&mpte->mpte_thread_lock); - /* callee will destroy thread lock */ - mptcp_thread_destroy(mpte); - /* NOTREACHED */ - return; - } - - mpte->mpte_thread_active = 1; - for (;;) { - uint32_t reqs = mpte->mpte_thread_reqs; - - lck_mtx_unlock(&mpte->mpte_thread_lock); - mptcp_thread_dowork(mpte); - lck_mtx_lock_spin(&mpte->mpte_thread_lock); - - /* if there's no pending request, we're done */ - if (reqs == mpte->mpte_thread_reqs || - mpte->mpte_thread == THREAD_NULL) - break; + if (error) + mptcp_subflow_abort(mpts, error); + } } - mpte->mpte_thread_reqs = 0; - mpte->mpte_thread_active = 0; } } -/* - * Destroy a MTCP thread, to be called in the MPTCP thread context - * upon receiving an indication to self-terminate. This routine - * will not return, as the current thread is terminated at the end. - */ -static void -mptcp_thread_destroy(struct mptses *mpte) -{ - struct socket *mp_so; - - MPTE_LOCK(mpte); /* same as MP socket lock */ - VERIFY(mpte->mpte_thread == THREAD_NULL); - VERIFY(mpte->mpte_mppcb != NULL); - - mptcp_sesdestroy(mpte); - - mp_so = mpte->mpte_mppcb->mpp_socket; - VERIFY(mp_so != NULL); - VERIFY(mp_so->so_usecount > 0); - mp_so->so_usecount--; /* for thread */ - mpte->mpte_mppcb->mpp_flags |= MPP_DEFUNCT; - MPTE_UNLOCK(mpte); - - /* for the extra refcnt from kernel_thread_start() */ - thread_deallocate(current_thread()); - /* this is the end */ - thread_terminate(current_thread()); - /* NOTREACHED */ -} - /* * Protocol pr_lock callback. */ int mptcp_lock(struct socket *mp_so, int refcount, void *lr) { - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); void *lr_saved; if (lr == NULL) @@ -3956,7 +4207,7 @@ mptcp_lock(struct socket *mp_so, int refcount, void *lr) mp_so, lr_saved, solockhistory_nr(mp_so)); /* NOTREACHED */ } - lck_mtx_lock(&mpp->mpp_lock); + mpp_lock(mpp); if (mp_so->so_usecount < 0) { panic("%s: so=%p so_pcb=%p lr=%p ref=%x lrh= %s\n", __func__, @@ -3976,207 +4227,58 @@ mptcp_lock(struct socket *mp_so, int refcount, void *lr) * Protocol pr_unlock callback. */ int -mptcp_unlock(struct socket *mp_so, int refcount, void *lr) -{ - struct mppcb *mpp = sotomppcb(mp_so); - void *lr_saved; - - if (lr == NULL) - lr_saved = __builtin_return_address(0); - else - lr_saved = lr; - - if (mpp == NULL) { - panic("%s: so=%p NO PCB usecount=%x lr=%p lrh= %s\n", __func__, - mp_so, mp_so->so_usecount, lr_saved, - solockhistory_nr(mp_so)); - /* NOTREACHED */ - } - lck_mtx_assert(&mpp->mpp_lock, LCK_MTX_ASSERT_OWNED); - - if (refcount != 0) - mp_so->so_usecount--; - - if (mp_so->so_usecount < 0) { - panic("%s: so=%p usecount=%x lrh= %s\n", __func__, - mp_so, mp_so->so_usecount, solockhistory_nr(mp_so)); - /* NOTREACHED */ - } - mp_so->unlock_lr[mp_so->next_unlock_lr] = lr_saved; - mp_so->next_unlock_lr = (mp_so->next_unlock_lr + 1) % SO_LCKDBG_MAX; - lck_mtx_unlock(&mpp->mpp_lock); - - return (0); -} - -/* - * Protocol pr_getlock callback. - */ -lck_mtx_t * -mptcp_getlock(struct socket *mp_so, int locktype) -{ -#pragma unused(locktype) - struct mppcb *mpp = sotomppcb(mp_so); - - if (mpp == NULL) { - panic("%s: so=%p NULL so_pcb %s\n", __func__, mp_so, - solockhistory_nr(mp_so)); - /* NOTREACHED */ - } - if (mp_so->so_usecount < 0) { - panic("%s: so=%p usecount=%x lrh= %s\n", __func__, - mp_so, mp_so->so_usecount, solockhistory_nr(mp_so)); - /* NOTREACHED */ - } - return (&mpp->mpp_lock); -} - -/* - * Key generation functions - */ -static void -mptcp_generate_unique_key(struct mptcp_key_entry *key_entry) -{ - struct mptcp_key_entry *key_elm; -try_again: - read_random(&key_entry->mkey_value, sizeof (key_entry->mkey_value)); - if (key_entry->mkey_value == 0) - goto try_again; - mptcp_do_sha1(&key_entry->mkey_value, key_entry->mkey_digest, - sizeof (key_entry->mkey_digest)); - - LIST_FOREACH(key_elm, &mptcp_keys_pool, mkey_next) { - if (key_elm->mkey_value == key_entry->mkey_value) { - goto try_again; - } - if (bcmp(key_elm->mkey_digest, key_entry->mkey_digest, 4) == - 0) { - goto try_again; - } - } -} - -static mptcp_key_t * -mptcp_reserve_key(void) -{ - struct mptcp_key_entry *key_elm; - struct mptcp_key_entry *found_elm = NULL; - - lck_mtx_lock(&mptcp_keys_pool.mkph_lock); - LIST_FOREACH(key_elm, &mptcp_keys_pool, mkey_next) { - if (key_elm->mkey_flags == MKEYF_FREE) { - key_elm->mkey_flags = MKEYF_INUSE; - found_elm = key_elm; - break; - } - } - lck_mtx_unlock(&mptcp_keys_pool.mkph_lock); - - if (found_elm) { - return (&found_elm->mkey_value); - } - - key_elm = (struct mptcp_key_entry *) - zalloc(mptcp_keys_pool.mkph_key_entry_zone); - key_elm->mkey_flags = MKEYF_INUSE; - - lck_mtx_lock(&mptcp_keys_pool.mkph_lock); - mptcp_generate_unique_key(key_elm); - LIST_INSERT_HEAD(&mptcp_keys_pool, key_elm, mkey_next); - mptcp_keys_pool.mkph_count += 1; - lck_mtx_unlock(&mptcp_keys_pool.mkph_lock); - return (&key_elm->mkey_value); -} - -static caddr_t -mptcp_get_stored_digest(mptcp_key_t *key) +mptcp_unlock(struct socket *mp_so, int refcount, void *lr) { - struct mptcp_key_entry *key_holder; - caddr_t digest = NULL; - - lck_mtx_lock(&mptcp_keys_pool.mkph_lock); - key_holder = (struct mptcp_key_entry *)(void *)((caddr_t)key - - offsetof(struct mptcp_key_entry, mkey_value)); - if (key_holder->mkey_flags != MKEYF_INUSE) - panic_plain("%s", __func__); - digest = &key_holder->mkey_digest[0]; - lck_mtx_unlock(&mptcp_keys_pool.mkph_lock); - return (digest); -} + struct mppcb *mpp = mpsotomppcb(mp_so); + void *lr_saved; -void -mptcp_free_key(mptcp_key_t *key) -{ - struct mptcp_key_entry *key_holder; - struct mptcp_key_entry *key_elm; - int pt = RandomULong(); + if (lr == NULL) + lr_saved = __builtin_return_address(0); + else + lr_saved = lr; - lck_mtx_lock(&mptcp_keys_pool.mkph_lock); - key_holder = (struct mptcp_key_entry *)(void*)((caddr_t)key - - offsetof(struct mptcp_key_entry, mkey_value)); - key_holder->mkey_flags = MKEYF_FREE; + if (mpp == NULL) { + panic("%s: so=%p NO PCB usecount=%x lr=%p lrh= %s\n", __func__, + mp_so, mp_so->so_usecount, lr_saved, + solockhistory_nr(mp_so)); + /* NOTREACHED */ + } + mpp_lock_assert_held(mpp); - LIST_REMOVE(key_holder, mkey_next); - mptcp_keys_pool.mkph_count -= 1; + if (refcount != 0) + mp_so->so_usecount--; - /* Free half the time */ - if (pt & 0x01) { - zfree(mptcp_keys_pool.mkph_key_entry_zone, key_holder); - } else { - /* Insert it at random point to avoid early reuse */ - int i = 0; - if (mptcp_keys_pool.mkph_count > 1) { - pt = pt % (mptcp_keys_pool.mkph_count - 1); - LIST_FOREACH(key_elm, &mptcp_keys_pool, mkey_next) { - if (++i >= pt) { - LIST_INSERT_AFTER(key_elm, key_holder, - mkey_next); - break; - } - } - if (i < pt) - panic("missed insertion"); - } else { - LIST_INSERT_HEAD(&mptcp_keys_pool, key_holder, - mkey_next); - } - mptcp_keys_pool.mkph_count += 1; + if (mp_so->so_usecount < 0) { + panic("%s: so=%p usecount=%x lrh= %s\n", __func__, + mp_so, mp_so->so_usecount, solockhistory_nr(mp_so)); + /* NOTREACHED */ } - lck_mtx_unlock(&mptcp_keys_pool.mkph_lock); + mp_so->unlock_lr[mp_so->next_unlock_lr] = lr_saved; + mp_so->next_unlock_lr = (mp_so->next_unlock_lr + 1) % SO_LCKDBG_MAX; + mpp_unlock(mpp); + + return (0); } -static void -mptcp_key_pool_init(void) +/* + * Protocol pr_getlock callback. + */ +lck_mtx_t * +mptcp_getlock(struct socket *mp_so, int flags) { - int i; - struct mptcp_key_entry *key_entry; - - LIST_INIT(&mptcp_keys_pool); - mptcp_keys_pool.mkph_count = 0; - - mptcp_keys_pool.mkph_key_elm_sz = (vm_size_t) - (sizeof (struct mptcp_key_entry)); - mptcp_keys_pool.mkph_key_entry_zone = zinit( - mptcp_keys_pool.mkph_key_elm_sz, - MPTCP_MX_KEY_ALLOCS * mptcp_keys_pool.mkph_key_elm_sz, - MPTCP_MX_PREALLOC_ZONE_SZ, "mptkeys"); - if (mptcp_keys_pool.mkph_key_entry_zone == NULL) { - panic("%s: unable to allocate MPTCP keys zone \n", __func__); + struct mppcb *mpp = mpsotomppcb(mp_so); + + if (mpp == NULL) { + panic("%s: so=%p NULL so_pcb %s\n", __func__, mp_so, + solockhistory_nr(mp_so)); /* NOTREACHED */ } - zone_change(mptcp_keys_pool.mkph_key_entry_zone, Z_CALLERACCT, FALSE); - zone_change(mptcp_keys_pool.mkph_key_entry_zone, Z_EXPAND, TRUE); - - for (i = 0; i < MPTCP_KEY_PREALLOCS_MX; i++) { - key_entry = (struct mptcp_key_entry *) - zalloc(mptcp_keys_pool.mkph_key_entry_zone); - key_entry->mkey_flags = MKEYF_FREE; - mptcp_generate_unique_key(key_entry); - LIST_INSERT_HEAD(&mptcp_keys_pool, key_entry, mkey_next); - mptcp_keys_pool.mkph_count += 1; + if (mp_so->so_usecount < 0) { + panic("%s: so=%p usecount=%x lrh= %s\n", __func__, + mp_so, mp_so->so_usecount, solockhistory_nr(mp_so)); + /* NOTREACHED */ } - lck_mtx_init(&mptcp_keys_pool.mkph_lock, mtcbinfo.mppi_lock_grp, - mtcbinfo.mppi_lock_attr); + return (mpp_getlock(mpp, flags)); } /* @@ -4189,10 +4291,8 @@ mptcp_attach_to_subf(struct socket *so, struct mptcb *mp_tp, { struct tcpcb *tp = sototcpcb(so); struct mptcp_subf_auth_entry *sauth_entry; - MPT_LOCK_ASSERT_NOTHELD(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); - MPT_LOCK_SPIN(mp_tp); - tp->t_mptcb = mp_tp; /* * The address ID of the first flow is implicitly 0. */ @@ -4203,7 +4303,6 @@ mptcp_attach_to_subf(struct socket *so, struct mptcb *mp_tp, tp->t_mpflags |= (TMPF_PREESTABLISHED | TMPF_JOINED_FLOW); so->so_flags |= SOF_MP_SEC_SUBFLOW; } - MPT_UNLOCK(mp_tp); sauth_entry = zalloc(mpt_subauth_zone); sauth_entry->msae_laddr_id = tp->t_local_aid; sauth_entry->msae_raddr_id = 0; @@ -4212,9 +4311,7 @@ try_again: sauth_entry->msae_laddr_rand = RandomULong(); if (sauth_entry->msae_laddr_rand == 0) goto try_again; - MPT_LOCK_SPIN(mp_tp); LIST_INSERT_HEAD(&mp_tp->mpt_subauth_list, sauth_entry, msae_next); - MPT_UNLOCK(mp_tp); } static void @@ -4224,14 +4321,10 @@ mptcp_detach_mptcb_from_subf(struct mptcb *mp_tp, struct socket *so) struct tcpcb *tp = NULL; int found = 0; - socket_lock(so, 0); tp = sototcpcb(so); - if (tp == NULL) { - socket_unlock(so, 0); + if (tp == NULL) return; - } - MPT_LOCK(mp_tp); LIST_FOREACH(sauth_entry, &mp_tp->mpt_subauth_list, msae_next) { if (sauth_entry->msae_laddr_id == tp->t_local_aid) { found = 1; @@ -4241,13 +4334,9 @@ mptcp_detach_mptcb_from_subf(struct mptcb *mp_tp, struct socket *so) if (found) { LIST_REMOVE(sauth_entry, msae_next); } - MPT_UNLOCK(mp_tp); if (found) zfree(mpt_subauth_zone, sauth_entry); - - tp->t_mptcb = NULL; - socket_unlock(so, 0); } void @@ -4255,9 +4344,8 @@ mptcp_get_rands(mptcp_addr_id addr_id, struct mptcb *mp_tp, u_int32_t *lrand, u_int32_t *rrand) { struct mptcp_subf_auth_entry *sauth_entry; - MPT_LOCK_ASSERT_NOTHELD(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); - MPT_LOCK(mp_tp); LIST_FOREACH(sauth_entry, &mp_tp->mpt_subauth_list, msae_next) { if (sauth_entry->msae_laddr_id == addr_id) { if (lrand) @@ -4267,7 +4355,6 @@ mptcp_get_rands(mptcp_addr_id addr_id, struct mptcb *mp_tp, u_int32_t *lrand, break; } } - MPT_UNLOCK(mp_tp); } void @@ -4275,9 +4362,8 @@ mptcp_set_raddr_rand(mptcp_addr_id laddr_id, struct mptcb *mp_tp, mptcp_addr_id raddr_id, u_int32_t raddr_rand) { struct mptcp_subf_auth_entry *sauth_entry; - MPT_LOCK_ASSERT_NOTHELD(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); - MPT_LOCK(mp_tp); LIST_FOREACH(sauth_entry, &mp_tp->mpt_subauth_list, msae_next) { if (sauth_entry->msae_laddr_id == laddr_id) { if ((sauth_entry->msae_raddr_id != 0) && @@ -4286,7 +4372,6 @@ mptcp_set_raddr_rand(mptcp_addr_id laddr_id, struct mptcb *mp_tp, " address ids %d %d \n", __func__, raddr_id, sauth_entry->msae_raddr_id), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - MPT_UNLOCK(mp_tp); return; } sauth_entry->msae_raddr_id = raddr_id; @@ -4297,42 +4382,34 @@ mptcp_set_raddr_rand(mptcp_addr_id laddr_id, struct mptcb *mp_tp, __func__, raddr_rand, sauth_entry->msae_raddr_rand), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - MPT_UNLOCK(mp_tp); return; } sauth_entry->msae_raddr_rand = raddr_rand; - MPT_UNLOCK(mp_tp); return; } } - MPT_UNLOCK(mp_tp); } /* * SHA1 support for MPTCP */ -static int -mptcp_do_sha1(mptcp_key_t *key, char *sha_digest, int digest_len) +static void +mptcp_do_sha1(mptcp_key_t *key, char *sha_digest) { SHA1_CTX sha1ctxt; const unsigned char *sha1_base; int sha1_size; - if (digest_len != SHA1_RESULTLEN) { - return (FALSE); - } - sha1_base = (const unsigned char *) key; sha1_size = sizeof (mptcp_key_t); SHA1Init(&sha1ctxt); SHA1Update(&sha1ctxt, sha1_base, sha1_size); SHA1Final(sha_digest, &sha1ctxt); - return (TRUE); } void mptcp_hmac_sha1(mptcp_key_t key1, mptcp_key_t key2, - u_int32_t rand1, u_int32_t rand2, u_char *digest, int digest_len) + u_int32_t rand1, u_int32_t rand2, u_char *digest) { SHA1_CTX sha1ctxt; mptcp_key_t key_ipad[8] = {0}; /* key XOR'd with inner pad */ @@ -4340,7 +4417,7 @@ mptcp_hmac_sha1(mptcp_key_t key1, mptcp_key_t key2, u_int32_t data[2]; int i; - bzero(digest, digest_len); + bzero(digest, SHA1_RESULTLEN); /* Set up the Key for HMAC */ key_ipad[0] = key1; @@ -4380,41 +4457,22 @@ mptcp_hmac_sha1(mptcp_key_t key1, mptcp_key_t key2, * corresponds to MAC-A = MAC (Key=(Key-A+Key-B), Msg=(R-A+R-B)) */ void -mptcp_get_hmac(mptcp_addr_id aid, struct mptcb *mp_tp, u_char *digest, - int digest_len) +mptcp_get_hmac(mptcp_addr_id aid, struct mptcb *mp_tp, u_char *digest) { uint32_t lrand, rrand; - mptcp_key_t localkey, remotekey; - MPT_LOCK_ASSERT_NOTHELD(mp_tp); - if (digest_len != SHA1_RESULTLEN) - return; + mpte_lock_assert_held(mp_tp->mpt_mpte); lrand = rrand = 0; mptcp_get_rands(aid, mp_tp, &lrand, &rrand); - MPT_LOCK_SPIN(mp_tp); - localkey = *mp_tp->mpt_localkey; - remotekey = mp_tp->mpt_remotekey; - MPT_UNLOCK(mp_tp); - mptcp_hmac_sha1(localkey, remotekey, lrand, rrand, digest, - digest_len); -} - -u_int64_t -mptcp_get_trunced_hmac(mptcp_addr_id aid, struct mptcb *mp_tp) -{ - u_char digest[SHA1_RESULTLEN]; - u_int64_t trunced_digest; - - mptcp_get_hmac(aid, mp_tp, &digest[0], sizeof (digest)); - bcopy(digest, &trunced_digest, 8); - return (trunced_digest); + mptcp_hmac_sha1(mp_tp->mpt_localkey, mp_tp->mpt_remotekey, lrand, rrand, + digest); } /* * Authentication data generation */ -void +static void mptcp_generate_token(char *sha_digest, int sha_digest_len, caddr_t token, int token_len) { @@ -4426,7 +4484,7 @@ mptcp_generate_token(char *sha_digest, int sha_digest_len, caddr_t token, return; } -void +static void mptcp_generate_idsn(char *sha_digest, int sha_digest_len, caddr_t idsn, int idsn_len) { @@ -4466,15 +4524,17 @@ mptcp_conn_properties(struct mptcb *mp_tp) } static void -mptcp_init_local_parms(struct mptcb *mp_tp) +mptcp_init_local_parms(struct mptses *mpte) { - caddr_t local_digest = NULL; + struct mptcb *mp_tp = mpte->mpte_mptcb; + char key_digest[SHA1_RESULTLEN]; - mp_tp->mpt_localkey = mptcp_reserve_key(); - local_digest = mptcp_get_stored_digest(mp_tp->mpt_localkey); - mptcp_generate_token(local_digest, SHA1_RESULTLEN, + read_frandom(&mp_tp->mpt_localkey, sizeof(mp_tp->mpt_localkey)); + mptcp_do_sha1(&mp_tp->mpt_localkey, key_digest); + + mptcp_generate_token(key_digest, SHA1_RESULTLEN, (caddr_t)&mp_tp->mpt_localtoken, sizeof (mp_tp->mpt_localtoken)); - mptcp_generate_idsn(local_digest, SHA1_RESULTLEN, + mptcp_generate_idsn(key_digest, SHA1_RESULTLEN, (caddr_t)&mp_tp->mpt_local_idsn, sizeof (u_int64_t)); /* The subflow SYN is also first MPTCP byte */ @@ -4487,65 +4547,25 @@ mptcp_init_local_parms(struct mptcb *mp_tp) int mptcp_init_remote_parms(struct mptcb *mp_tp) { - char remote_digest[MPTCP_SHA1_RESULTLEN]; - MPT_LOCK_ASSERT_HELD(mp_tp); + char remote_digest[SHA1_RESULTLEN]; + mpte_lock_assert_held(mp_tp->mpt_mpte); /* Only Version 0 is supported for auth purposes */ if (mp_tp->mpt_version != MPTCP_STD_VERSION_0) return (-1); /* Setup local and remote tokens and Initial DSNs */ - - if (!mptcp_do_sha1(&mp_tp->mpt_remotekey, remote_digest, - SHA1_RESULTLEN)) { - mptcplog((LOG_ERR, "MPTCP Socket: %s: unexpected failure", - __func__), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - return (-1); - } + mptcp_do_sha1(&mp_tp->mpt_remotekey, remote_digest); mptcp_generate_token(remote_digest, SHA1_RESULTLEN, (caddr_t)&mp_tp->mpt_remotetoken, sizeof (mp_tp->mpt_remotetoken)); mptcp_generate_idsn(remote_digest, SHA1_RESULTLEN, (caddr_t)&mp_tp->mpt_remote_idsn, sizeof (u_int64_t)); - mp_tp->mpt_rcvatmark = mp_tp->mpt_rcvnxt = mp_tp->mpt_remote_idsn + 1; + mp_tp->mpt_rcvnxt = mp_tp->mpt_remote_idsn + 1; return (0); } -/* - * Helper Functions - */ -mptcp_token_t -mptcp_get_localtoken(void* mptcb_arg) -{ - struct mptcb *mp_tp = (struct mptcb *)mptcb_arg; - return (mp_tp->mpt_localtoken); -} - -mptcp_token_t -mptcp_get_remotetoken(void* mptcb_arg) -{ - struct mptcb *mp_tp = (struct mptcb *)mptcb_arg; - return (mp_tp->mpt_remotetoken); -} - -u_int64_t -mptcp_get_localkey(void* mptcb_arg) -{ - struct mptcb *mp_tp = (struct mptcb *)mptcb_arg; - if (mp_tp->mpt_localkey != NULL) - return (*mp_tp->mpt_localkey); - else - return (0); -} - -u_int64_t -mptcp_get_remotekey(void* mptcb_arg) -{ - struct mptcb *mp_tp = (struct mptcb *)mptcb_arg; - return (mp_tp->mpt_remotekey); -} - -void +static void mptcp_send_dfin(struct socket *so) { struct tcpcb *tp = NULL; @@ -4575,7 +4595,8 @@ mptcp_insert_dsn(struct mppcb *mpp, struct mbuf *m) return; __IGNORE_WCASTALIGN(mp_tp = &((struct mpp_mtp *)mpp)->mtcb); - MPT_LOCK(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + while (m) { VERIFY(m->m_flags & M_PKTHDR); m->m_pkthdr.pkt_flags |= (PKTF_MPTCP | PKTF_MPSO); @@ -4584,56 +4605,97 @@ mptcp_insert_dsn(struct mppcb *mpp, struct mbuf *m) mp_tp->mpt_sndmax += m_pktlen(m); m = m->m_next; } - MPT_UNLOCK(mp_tp); +} + +void +mptcp_fallback_sbdrop(struct socket *so, struct mbuf *m, int len) +{ + struct mptcb *mp_tp = tptomptp(sototcpcb(so)); + uint64_t data_ack; + uint64_t dsn; + + if (!m || len == 0) + return; + + while (m && len > 0) { + VERIFY(m->m_flags & M_PKTHDR); + VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP); + + data_ack = m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen; + dsn = m->m_pkthdr.mp_dsn; + + len -= m->m_len; + m = m->m_next; + } + + if (m && len == 0) { + /* + * If there is one more mbuf in the chain, it automatically means + * that up to m->mp_dsn has been ack'ed. + * + * This means, we actually correct data_ack back down (compared + * to what we set inside the loop - dsn + data_len). Because in + * the loop we are "optimistic" and assume that the full mapping + * will be acked. If that's not the case and we get out of the + * loop with m != NULL, it means only up to m->mp_dsn has been + * really acked. + */ + data_ack = m->m_pkthdr.mp_dsn; + } + + if (len < 0) { + /* + * If len is negative, meaning we acked in the middle of an mbuf, + * only up to this mbuf's data-sequence number has been acked + * at the MPTCP-level. + */ + data_ack = dsn; + } + + mptcplog((LOG_DEBUG, "%s inferred ack up to %u\n", __func__, (uint32_t)data_ack), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + mptcp_data_ack_rcvd(mp_tp, sototcpcb(so), data_ack); } void mptcp_preproc_sbdrop(struct socket *so, struct mbuf *m, unsigned int len) { - u_int32_t sub_len = 0; int rewinding = 0; - if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) { - /* TFO makes things complicated. */ - if (so->so_flags1 & SOF1_TFO_REWIND) { - rewinding = 1; - so->so_flags1 &= ~SOF1_TFO_REWIND; - } + /* TFO makes things complicated. */ + if (so->so_flags1 & SOF1_TFO_REWIND) { + rewinding = 1; + so->so_flags1 &= ~SOF1_TFO_REWIND; } - while (m) { + while (m && (!(so->so_flags & SOF_MP_SUBFLOW) || rewinding)) { + u_int32_t sub_len; VERIFY(m->m_flags & M_PKTHDR); + VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP); - if (m->m_pkthdr.pkt_flags & PKTF_MPTCP) { - sub_len = m->m_pkthdr.mp_rlen; + sub_len = m->m_pkthdr.mp_rlen; - if (sub_len < len) { - m->m_pkthdr.mp_dsn += sub_len; - if (!(m->m_pkthdr.pkt_flags & PKTF_MPSO)) { - m->m_pkthdr.mp_rseq += sub_len; - } - m->m_pkthdr.mp_rlen = 0; - len -= sub_len; - } else { - /* sub_len >= len */ - if (rewinding == 0) - m->m_pkthdr.mp_dsn += len; - if (!(m->m_pkthdr.pkt_flags & PKTF_MPSO)) { - if (rewinding == 0) - m->m_pkthdr.mp_rseq += len; - } - mptcplog((LOG_DEBUG, "MPTCP Sender: " - "%s: dsn 0x%llx ssn %u len %d %d\n", - __func__, - m->m_pkthdr.mp_dsn, m->m_pkthdr.mp_rseq, - m->m_pkthdr.mp_rlen, len), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - m->m_pkthdr.mp_rlen -= len; - break; + if (sub_len < len) { + m->m_pkthdr.mp_dsn += sub_len; + if (!(m->m_pkthdr.pkt_flags & PKTF_MPSO)) { + m->m_pkthdr.mp_rseq += sub_len; } + m->m_pkthdr.mp_rlen = 0; + len -= sub_len; } else { - panic("%s: MPTCP tag not set", __func__); - /* NOTREACHED */ + /* sub_len >= len */ + if (rewinding == 0) + m->m_pkthdr.mp_dsn += len; + if (!(m->m_pkthdr.pkt_flags & PKTF_MPSO)) { + if (rewinding == 0) + m->m_pkthdr.mp_rseq += len; + } + mptcplog((LOG_DEBUG, "%s: dsn %u ssn %u len %d %d\n", + __func__, (u_int32_t)m->m_pkthdr.mp_dsn, + m->m_pkthdr.mp_rseq, m->m_pkthdr.mp_rlen, len), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + m->m_pkthdr.mp_rlen -= len; + break; } m = m->m_next; } @@ -4645,37 +4707,32 @@ mptcp_preproc_sbdrop(struct socket *so, struct mbuf *m, unsigned int len) * Received an ack without receiving a DATA_ACK. * Need to fallback to regular TCP (or destroy this subflow). */ + sototcpcb(so)->t_mpflags |= TMPF_INFIN_SENT; mptcp_notify_mpfail(so); } } /* Obtain the DSN mapping stored in the mbuf */ void -mptcp_output_getm_dsnmap32(struct socket *so, int off, uint32_t datalen, - u_int32_t *dsn, u_int32_t *relseq, u_int16_t *data_len, u_int64_t *dsn64p) +mptcp_output_getm_dsnmap32(struct socket *so, int off, + uint32_t *dsn, uint32_t *relseq, uint16_t *data_len, uint16_t *dss_csum) { u_int64_t dsn64; - mptcp_output_getm_dsnmap64(so, off, datalen, &dsn64, relseq, data_len); + mptcp_output_getm_dsnmap64(so, off, &dsn64, relseq, data_len, dss_csum); *dsn = (u_int32_t)MPTCP_DATASEQ_LOW32(dsn64); - *dsn64p = dsn64; } void -mptcp_output_getm_dsnmap64(struct socket *so, int off, uint32_t datalen, - u_int64_t *dsn, u_int32_t *relseq, u_int16_t *data_len) +mptcp_output_getm_dsnmap64(struct socket *so, int off, uint64_t *dsn, + uint32_t *relseq, uint16_t *data_len, + uint16_t *dss_csum) { struct mbuf *m = so->so_snd.sb_mb; - struct mbuf *mnext = NULL; - uint32_t runlen = 0; - u_int64_t dsn64; - uint32_t contig_len = 0; + int off_orig = off; - if (m == NULL) - return; + VERIFY(off >= 0); - if (off < 0) - return; /* * In the subflow socket, the DSN sequencing can be discontiguous, * but the subflow sequence mapping is contiguous. Use the subflow @@ -4684,97 +4741,29 @@ mptcp_output_getm_dsnmap64(struct socket *so, int off, uint32_t datalen, */ while (m) { - VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP); VERIFY(m->m_flags & M_PKTHDR); + VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP); - if ((unsigned int)off >= m->m_pkthdr.mp_rlen) { - off -= m->m_pkthdr.mp_rlen; + if (off >= m->m_len) { + off -= m->m_len; m = m->m_next; } else { break; } } - if (m == NULL) { - panic("%s: bad offset", __func__); - /* NOTREACHED */ - } - - dsn64 = m->m_pkthdr.mp_dsn + off; - *dsn = dsn64; - *relseq = m->m_pkthdr.mp_rseq + off; - - /* - * Now find the last contiguous byte and its length from - * start. - */ - runlen = m->m_pkthdr.mp_rlen - off; - contig_len = runlen; - - /* If datalen does not span multiple mbufs, return */ - if (datalen <= runlen) { - *data_len = min(datalen, UINT16_MAX); - return; - } - - mnext = m->m_next; - while (datalen > runlen) { - if (mnext == NULL) { - panic("%s: bad datalen = %d, %d %d", __func__, datalen, - runlen, off); - /* NOTREACHED */ - } - VERIFY(mnext->m_flags & M_PKTHDR); - VERIFY(mnext->m_pkthdr.pkt_flags & PKTF_MPTCP); - - /* - * case A. contiguous DSN stream - * case B. discontiguous DSN stream - */ - if (mnext->m_pkthdr.mp_dsn == (dsn64 + runlen)) { - /* case A */ - runlen += mnext->m_pkthdr.mp_rlen; - contig_len += mnext->m_pkthdr.mp_rlen; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s: contig \n", - __func__), MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - } else { - /* case B */ - mptcplog((LOG_DEBUG, "MPTCP Sender: " - "%s: discontig datalen %d contig_len %d cc %d \n", - __func__, datalen, contig_len, so->so_snd.sb_cc), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - break; - } - mnext = mnext->m_next; - } - datalen = min(datalen, UINT16_MAX); - *data_len = min(datalen, contig_len); - mptcplog((LOG_DEBUG, "MPTCP Sender: " - "%s: %llu %u %d %d \n", __func__, - *dsn, *relseq, *data_len, off), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); -} + VERIFY(m); + VERIFY(off >= 0); + VERIFY(m->m_pkthdr.mp_rlen <= UINT16_MAX); -/* - * MPTCP's notion of the next insequence Data Sequence number is adjusted - * here. It must be called from mptcp_adj_rmap() which is called only after - * reassembly of out of order data. The rcvnxt variable must - * be updated only when atleast some insequence new data is received. - */ -static void -mptcp_adj_rcvnxt(struct tcpcb *tp, struct mbuf *m) -{ - struct mptcb *mp_tp = tptomptp(tp); + *dsn = m->m_pkthdr.mp_dsn; + *relseq = m->m_pkthdr.mp_rseq; + *data_len = m->m_pkthdr.mp_rlen; + *dss_csum = m->m_pkthdr.mp_csum; - if (mp_tp == NULL) - return; - MPT_LOCK(mp_tp); - if ((MPTCP_SEQ_GEQ(mp_tp->mpt_rcvnxt, m->m_pkthdr.mp_dsn)) && - (MPTCP_SEQ_LEQ(mp_tp->mpt_rcvnxt, (m->m_pkthdr.mp_dsn + - m->m_pkthdr.mp_rlen)))) { - mp_tp->mpt_rcvnxt = m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen; - } - MPT_UNLOCK(mp_tp); + mptcplog((LOG_DEBUG, "%s: dsn %u ssn %u data_len %d off %d off_orig %d\n", + __func__, (u_int32_t)(*dsn), *relseq, *data_len, off, off_orig), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); } /* @@ -4798,61 +4787,37 @@ mptcp_insert_rmap(struct tcpcb *tp, struct mbuf *m) m->m_pkthdr.mp_dsn = tp->t_rcv_map.mpt_dsn; m->m_pkthdr.mp_rseq = tp->t_rcv_map.mpt_sseq; m->m_pkthdr.mp_rlen = tp->t_rcv_map.mpt_len; + m->m_pkthdr.mp_csum = tp->t_rcv_map.mpt_csum; m->m_pkthdr.pkt_flags |= PKTF_MPTCP; tp->t_mpflags &= ~TMPF_EMBED_DSN; tp->t_mpflags |= TMPF_MPTCP_ACKNOW; } } -int -mptcp_adj_rmap(struct socket *so, struct mbuf *m) +void +mptcp_adj_rmap(struct socket *so, struct mbuf *m, int off) { - u_int64_t dsn; - u_int32_t sseq, datalen; - struct tcpcb *tp = intotcpcb(sotoinpcb(so)); - u_int32_t old_rcvnxt = 0; + struct mptsub *mpts = sototcpcb(so)->t_mpsub; if (m_pktlen(m) == 0) - return 0; - - if (m->m_pkthdr.pkt_flags & PKTF_MPTCP) { - VERIFY(m->m_flags & M_PKTHDR); - - dsn = m->m_pkthdr.mp_dsn; - sseq = m->m_pkthdr.mp_rseq + tp->irs; - datalen = m->m_pkthdr.mp_rlen; - } else { - /* data arrived without an DSS option mapping */ - - /* initial subflow can fallback right after SYN handshake */ - mptcp_notify_mpfail(so); - return 0; - } - - /* In the common case, data is in window and in sequence */ - if (m->m_pkthdr.len == (int)datalen) { - mptcp_adj_rcvnxt(tp, m); - return 0; - } + return; - old_rcvnxt = tp->rcv_nxt - m->m_pkthdr.len; - if (SEQ_GT(old_rcvnxt, sseq)) { - /* data trimmed from the left */ - int off = old_rcvnxt - sseq; + if ((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)) { m->m_pkthdr.mp_dsn += off; m->m_pkthdr.mp_rseq += off; m->m_pkthdr.mp_rlen = m->m_pkthdr.len; - } else if (old_rcvnxt == sseq) { - /* - * data was trimmed from the right - */ - m->m_pkthdr.mp_rlen = m->m_pkthdr.len; } else { - mptcp_notify_mpfail(so); - return (-1); + if (!(mpts->mpts_flags & MPTSF_CONFIRMED)) { + /* data arrived without an DSS option mapping */ + + /* initial subflow can fallback right after SYN handshake */ + mptcp_notify_mpfail(so); + } } - mptcp_adj_rcvnxt(tp, m); - return 0; + + mpts->mpts_flags |= MPTSF_CONFIRMED; + + return; } /* @@ -4872,9 +4837,8 @@ mptcp_act_on_txfail(struct socket *so) if (tp == NULL) return; - if (so->so_flags & SOF_MP_TRYFAILOVER) { + if (so->so_flags & SOF_MP_TRYFAILOVER) return; - } so->so_flags |= SOF_MP_TRYFAILOVER; soevent(so, (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MPFAILOVER)); @@ -4903,9 +4867,8 @@ mptcp_get_map_for_dsn(struct socket *so, u_int64_t dsn_fail, u_int32_t *tcp_seq) (MPTCP_SEQ_GEQ(dsn + datalen, dsn_fail))) { off = dsn_fail - dsn; *tcp_seq = m->m_pkthdr.mp_rseq + off; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s: %llu %llu \n", - __func__, dsn, dsn_fail), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: %llu %llu \n", __func__, dsn, + dsn_fail), MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); return (0); } @@ -4917,72 +4880,234 @@ mptcp_get_map_for_dsn(struct socket *so, u_int64_t dsn_fail, u_int32_t *tcp_seq) * not much else to do. */ - mptcplog((LOG_ERR, "MPTCP Sender: " - "%s: %llu not found \n", __func__, dsn_fail), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return (-1); + mptcplog((LOG_ERR, "MPTCP Sender: " + "%s: %llu not found \n", __func__, dsn_fail), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + return (-1); +} + +/* + * Support for sending contiguous MPTCP bytes in subflow + * Also for preventing sending data with ACK in 3-way handshake + */ +int32_t +mptcp_adj_sendlen(struct socket *so, int32_t off) +{ + struct tcpcb *tp = sototcpcb(so); + struct mptsub *mpts = tp->t_mpsub; + uint64_t mdss_dsn; + uint32_t mdss_subflow_seq; + int mdss_subflow_off; + uint16_t mdss_data_len; + uint16_t dss_csum; + + mptcp_output_getm_dsnmap64(so, off, &mdss_dsn, &mdss_subflow_seq, + &mdss_data_len, &dss_csum); + + /* + * We need to compute how much of the mapping still remains. + * So, we compute the offset in the send-buffer of the dss-sub-seq. + */ + mdss_subflow_off = (mdss_subflow_seq + mpts->mpts_iss) - tp->snd_una; + + /* + * When TFO is used, we are sending the mpts->mpts_iss although the relative + * seq has been set to 1 (while it should be 0). + */ + if (tp->t_mpflags & TMPF_TFO_REQUEST) + mdss_subflow_off--; + + if (off < mdss_subflow_off) + printf("%s off %d mdss_subflow_off %d mdss_subflow_seq %u iss %u suna %u\n", __func__, + off, mdss_subflow_off, mdss_subflow_seq, mpts->mpts_iss, tp->snd_una); + VERIFY(off >= mdss_subflow_off); + + mptcplog((LOG_DEBUG, "%s dlen %u off %d sub_off %d sub_seq %u iss %u suna %u\n", + __func__, mdss_data_len, off, mdss_subflow_off, mdss_subflow_seq, + mpts->mpts_iss, tp->snd_una), MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + return (mdss_data_len - (off - mdss_subflow_off)); +} + +static uint32_t +mptcp_get_maxseg(struct mptses *mpte) +{ + struct mptsub *mpts; + uint32_t maxseg = 0; + + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); + + if (!TCPS_HAVEESTABLISHED(tp->t_state) || + TCPS_HAVERCVDFIN2(tp->t_state)) + continue; + + if (tp->t_maxseg > maxseg) + maxseg = tp->t_maxseg; + } + + return (maxseg); +} + +static uint8_t +mptcp_get_rcvscale(struct mptses *mpte) +{ + struct mptsub *mpts; + uint8_t rcvscale = UINT8_MAX; + + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); + + if (!TCPS_HAVEESTABLISHED(tp->t_state) || + TCPS_HAVERCVDFIN2(tp->t_state)) + continue; + + if (tp->rcv_scale < rcvscale) + rcvscale = tp->rcv_scale; + } + + return (rcvscale); +} + +/* Similar to tcp_sbrcv_reserve */ +static void +mptcp_sbrcv_reserve(struct mptcb *mp_tp, struct sockbuf *sbrcv, + u_int32_t newsize, u_int32_t idealsize) +{ + uint8_t rcvscale = mptcp_get_rcvscale(mp_tp->mpt_mpte); + + /* newsize should not exceed max */ + newsize = min(newsize, tcp_autorcvbuf_max); + + /* The receive window scale negotiated at the + * beginning of the connection will also set a + * limit on the socket buffer size + */ + newsize = min(newsize, TCP_MAXWIN << rcvscale); + + /* Set new socket buffer size */ + if (newsize > sbrcv->sb_hiwat && + (sbreserve(sbrcv, newsize) == 1)) { + sbrcv->sb_idealsize = min(max(sbrcv->sb_idealsize, + (idealsize != 0) ? idealsize : newsize), tcp_autorcvbuf_max); + + /* Again check the limit set by the advertised + * window scale + */ + sbrcv->sb_idealsize = min(sbrcv->sb_idealsize, + TCP_MAXWIN << rcvscale); + } +} + +void +mptcp_sbrcv_grow(struct mptcb *mp_tp) +{ + struct mptses *mpte = mp_tp->mpt_mpte; + struct socket *mp_so = mpte->mpte_mppcb->mpp_socket; + struct sockbuf *sbrcv = &mp_so->so_rcv; + uint32_t hiwat_sum = 0; + uint32_t ideal_sum = 0; + struct mptsub *mpts; + + /* + * Do not grow the receive socket buffer if + * - auto resizing is disabled, globally or on this socket + * - the high water mark already reached the maximum + * - the stream is in background and receive side is being + * throttled + * - if there are segments in reassembly queue indicating loss, + * do not need to increase recv window during recovery as more + * data is not going to be sent. A duplicate ack sent during + * recovery should not change the receive window + */ + if (tcp_do_autorcvbuf == 0 || + (sbrcv->sb_flags & SB_AUTOSIZE) == 0 || + tcp_cansbgrow(sbrcv) == 0 || + sbrcv->sb_hiwat >= tcp_autorcvbuf_max || + (mp_so->so_flags1 & SOF1_EXTEND_BK_IDLE_WANTED) || + !LIST_EMPTY(&mp_tp->mpt_segq)) { + /* Can not resize the socket buffer, just return */ + return; + } + + /* + * Ideally, we want the rbuf to be (sum_i {bw_i} * rtt_max * 2) + * + * But, for this we first need accurate receiver-RTT estimations, which + * we currently don't have. + * + * Let's use a dummy algorithm for now, just taking the sum of all + * subflow's receive-buffers. It's too low, but that's all we can get + * for now. + */ + + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + hiwat_sum += mpts->mpts_socket->so_rcv.sb_hiwat; + ideal_sum += mpts->mpts_socket->so_rcv.sb_idealsize; + } + + mptcp_sbrcv_reserve(mp_tp, sbrcv, hiwat_sum, ideal_sum); } /* - * Support for sending contiguous MPTCP bytes in subflow - * Also for preventing sending data with ACK in 3-way handshake + * Determine if we can grow the recieve socket buffer to avoid sending + * a zero window update to the peer. We allow even socket buffers that + * have fixed size (set by the application) to grow if the resource + * constraints are met. They will also be trimmed after the application + * reads data. + * + * Similar to tcp_sbrcv_grow_rwin */ -int32_t -mptcp_adj_sendlen(struct socket *so, int32_t off, int32_t len) +static void +mptcp_sbrcv_grow_rwin(struct mptcb *mp_tp, struct sockbuf *sb) { - u_int64_t mdss_dsn = 0; - u_int32_t mdss_subflow_seq = 0; - u_int16_t mdss_data_len = 0; + struct socket *mp_so = mp_tp->mpt_mpte->mpte_mppcb->mpp_socket; + u_int32_t rcvbufinc = mptcp_get_maxseg(mp_tp->mpt_mpte) << 4; + u_int32_t rcvbuf = sb->sb_hiwat; - if (len == 0) - return (len); - - mptcp_output_getm_dsnmap64(so, off, (u_int32_t)len, - &mdss_dsn, &mdss_subflow_seq, &mdss_data_len); + if (tcp_recv_bg == 1 || IS_TCP_RECV_BG(mp_so)) + return; - /* - * Special case handling for Fast Join. We want to send data right - * after ACK of the 3-way handshake, but not piggyback the data - * with the 3rd ACK of the 3WHS. TMPF_FASTJOINBY2_SEND and - * mdss_data_len control this. - */ - struct tcpcb *tp = NULL; - tp = intotcpcb(sotoinpcb(so)); - if ((tp->t_mpflags & TMPF_JOINED_FLOW) && - (tp->t_mpflags & TMPF_PREESTABLISHED) && - (!(tp->t_mpflags & TMPF_RECVD_JOIN)) && - (tp->t_mpflags & TMPF_SENT_JOIN) && - (!(tp->t_mpflags & TMPF_MPTCP_TRUE)) && - (!(tp->t_mpflags & TMPF_FASTJOINBY2_SEND))) { - mdss_data_len = 0; - tp->t_mpflags |= TMPF_FASTJOINBY2_SEND; - } - - if ((tp->t_state > TCPS_SYN_SENT) && - (tp->t_mpflags & TMPF_TFO_REQUEST)) { - mdss_data_len = 0; - tp->t_mpflags &= ~TMPF_TFO_REQUEST; + if (tcp_do_autorcvbuf == 1 && + tcp_cansbgrow(sb) && + /* Diff to tcp_sbrcv_grow_rwin */ + (mp_so->so_flags1 & SOF1_EXTEND_BK_IDLE_WANTED) == 0 && + (rcvbuf - sb->sb_cc) < rcvbufinc && + rcvbuf < tcp_autorcvbuf_max && + (sb->sb_idealsize > 0 && + sb->sb_hiwat <= (sb->sb_idealsize + rcvbufinc))) { + sbreserve(sb, min((sb->sb_hiwat + rcvbufinc), tcp_autorcvbuf_max)); } - return (mdss_data_len); } +/* Similar to tcp_sbspace */ int32_t -mptcp_sbspace(struct mptcb *mpt) +mptcp_sbspace(struct mptcb *mp_tp) { - struct sockbuf *sb; + struct sockbuf *sb = &mp_tp->mpt_mpte->mpte_mppcb->mpp_socket->so_rcv; uint32_t rcvbuf; int32_t space; + int32_t pending = 0; + + mpte_lock_assert_held(mp_tp->mpt_mpte); - MPT_LOCK_ASSERT_HELD(mpt); - MPTE_LOCK_ASSERT_HELD(mpt->mpt_mpte); + mptcp_sbrcv_grow_rwin(mp_tp, sb); - sb = &mpt->mpt_mpte->mpte_mppcb->mpp_socket->so_rcv; + /* hiwat might have changed */ rcvbuf = sb->sb_hiwat; - space = ((int32_t)imin((rcvbuf - sb->sb_cc), - (sb->sb_mbmax - sb->sb_mbcnt))); + + space = ((int32_t) imin((rcvbuf - sb->sb_cc), + (sb->sb_mbmax - sb->sb_mbcnt))); if (space < 0) space = 0; - /* XXX check if it's too small? */ + +#if CONTENT_FILTER + /* Compensate for data being processed by content filters */ + pending = cfil_sock_data_space(sb); +#endif /* CONTENT_FILTER */ + if (pending > space) + space = 0; + else + space -= pending; return (space); } @@ -5052,12 +5177,11 @@ boolean_t mptcp_ok_to_keepalive(struct mptcb *mp_tp) { boolean_t ret = 1; - VERIFY(mp_tp != NULL); - MPT_LOCK(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + if (mp_tp->mpt_state >= MPTCPS_CLOSE_WAIT) { ret = 0; } - MPT_UNLOCK(mp_tp); return (ret); } @@ -5072,34 +5196,36 @@ mptcp_adj_mss(struct tcpcb *tp, boolean_t mtudisc) #define MPTCP_COMPUTE_LEN { \ mss_lower = sizeof (struct mptcp_dss_ack_opt); \ - MPT_LOCK(mp_tp); \ if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) \ mss_lower += 2; \ else \ /* adjust to 32-bit boundary + EOL */ \ mss_lower += 2; \ - MPT_UNLOCK(mp_tp); \ } if (mp_tp == NULL) return (0); + mpte_lock_assert_held(mp_tp->mpt_mpte); + /* * For the first subflow and subsequent subflows, adjust mss for * most common MPTCP option size, for case where tcp_mss is called * during option processing and MTU discovery. */ - if ((tp->t_mpflags & TMPF_PREESTABLISHED) && - (!(tp->t_mpflags & TMPF_JOINED_FLOW))) { - MPTCP_COMPUTE_LEN; - } - - if ((tp->t_mpflags & TMPF_PREESTABLISHED) && - (tp->t_mpflags & TMPF_SENT_JOIN)) { - MPTCP_COMPUTE_LEN; - } + if (!mtudisc) { + if (tp->t_mpflags & TMPF_MPTCP_TRUE && + !(tp->t_mpflags & TMPF_JOINED_FLOW)) { + MPTCP_COMPUTE_LEN; + } - if ((mtudisc) && (tp->t_mpflags & TMPF_MPTCP_TRUE)) { - MPTCP_COMPUTE_LEN; + if (tp->t_mpflags & TMPF_PREESTABLISHED && + tp->t_mpflags & TMPF_SENT_JOIN) { + MPTCP_COMPUTE_LEN; + } + } else { + if (tp->t_mpflags & TMPF_MPTCP_TRUE) { + MPTCP_COMPUTE_LEN; + } } return (mss_lower); @@ -5109,21 +5235,15 @@ mptcp_adj_mss(struct tcpcb *tp, boolean_t mtudisc) * Update the pid, upid, uuid of the subflow so, based on parent so */ void -mptcp_update_last_owner(struct mptsub *mpts, struct socket *parent_mpso) +mptcp_update_last_owner(struct socket *so, struct socket *mp_so) { - struct socket *subflow_so = mpts->mpts_socket; - - MPTS_LOCK_ASSERT_HELD(mpts); - - socket_lock(subflow_so, 0); - if ((subflow_so->last_pid != parent_mpso->last_pid) || - (subflow_so->last_upid != parent_mpso->last_upid)) { - subflow_so->last_upid = parent_mpso->last_upid; - subflow_so->last_pid = parent_mpso->last_pid; - uuid_copy(subflow_so->last_uuid, parent_mpso->last_uuid); + if (so->last_pid != mp_so->last_pid || + so->last_upid != mp_so->last_upid) { + so->last_upid = mp_so->last_upid; + so->last_pid = mp_so->last_pid; + uuid_copy(so->last_uuid, mp_so->last_uuid); } - so_update_policy(subflow_so); - socket_unlock(subflow_so, 0); + so_update_policy(so); } static void @@ -5159,11 +5279,9 @@ fill_mptcp_subflow(struct socket *so, mptcp_flow_t *flow, struct mptsub *mpts) flow->flow_tcpci_offset = offsetof(mptcp_flow_t, flow_ci); flow->flow_flags = mpts->mpts_flags; flow->flow_cid = mpts->mpts_connid; - flow->flow_sndnxt = mpts->mpts_sndnxt; flow->flow_relseq = mpts->mpts_rel_seq; - flow->flow_soerror = mpts->mpts_soerror; + flow->flow_soerror = mpts->mpts_socket->so_error; flow->flow_probecnt = mpts->mpts_probecnt; - flow->flow_peerswitch = mpts->mpts_peerswitch; } static int @@ -5171,7 +5289,7 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS { #pragma unused(oidp, arg1, arg2) int error = 0, f; - size_t n, len; + size_t len; struct mppcb *mpp; struct mptses *mpte; struct mptcb *mp_tp; @@ -5184,8 +5302,8 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS return (EPERM); lck_mtx_lock(&mtcbinfo.mppi_lock); - n = mtcbinfo.mppi_count; if (req->oldptr == USER_ADDR_NULL) { + size_t n = mtcbinfo.mppi_count; lck_mtx_unlock(&mtcbinfo.mppi_lock); req->oldidx = (n + n/8) * sizeof(conninfo_mptcp_t) + 4 * (n + n/8) * sizeof(mptcp_flow_t); @@ -5193,19 +5311,15 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS } TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) { flows = NULL; - lck_mtx_lock(&mpp->mpp_lock); + mpp_lock(mpp); VERIFY(mpp->mpp_flags & MPP_ATTACHED); - if (mpp->mpp_flags & MPP_DEFUNCT) { - lck_mtx_unlock(&mpp->mpp_lock); - continue; - } mpte = mptompte(mpp); VERIFY(mpte != NULL); + mpte_lock_assert_held(mpte); mp_tp = mpte->mpte_mptcb; VERIFY(mp_tp != NULL); bzero(&mptcpci, sizeof(mptcpci)); - MPT_LOCK(mp_tp); mptcpci.mptcpci_state = mp_tp->mpt_state; mptcpci.mptcpci_flags = mp_tp->mpt_flags; mptcpci.mptcpci_ltoken = mp_tp->mpt_localtoken; @@ -5217,10 +5331,9 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS mptcpci.mptcpci_lidsn = mp_tp->mpt_local_idsn; mptcpci.mptcpci_sndwnd = mp_tp->mpt_sndwnd; mptcpci.mptcpci_rcvnxt = mp_tp->mpt_rcvnxt; - mptcpci.mptcpci_rcvatmark = mp_tp->mpt_rcvatmark; + mptcpci.mptcpci_rcvatmark = mp_tp->mpt_rcvnxt; mptcpci.mptcpci_ridsn = mp_tp->mpt_remote_idsn; mptcpci.mptcpci_rcvwnd = mp_tp->mpt_rcvwnd; - MPT_UNLOCK(mp_tp); mptcpci.mptcpci_nflows = mpte->mpte_numflows; mptcpci.mptcpci_mpte_flags = mpte->mpte_flags; @@ -5232,7 +5345,7 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS if (mpte->mpte_numflows != 0) { flows = _MALLOC(len, M_TEMP, M_WAITOK | M_ZERO); if (flows == NULL) { - lck_mtx_unlock(&mpp->mpp_lock); + mpp_unlock(mpp); break; } mptcpci.mptcpci_len = sizeof(mptcpci) + @@ -5244,21 +5357,17 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS error = SYSCTL_OUT(req, &mptcpci, sizeof(mptcpci)); } if (error) { - lck_mtx_unlock(&mpp->mpp_lock); + mpp_unlock(mpp); FREE(flows, M_TEMP); break; } f = 0; TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); so = mpts->mpts_socket; - socket_lock(so, 0); fill_mptcp_subflow(so, &flows[f], mpts); - socket_unlock(so, 0); - MPTS_UNLOCK(mpts); f++; } - lck_mtx_unlock(&mpp->mpp_lock); + mpp_unlock(mpp); if (flows) { error = SYSCTL_OUT(req, flows, len); FREE(flows, M_TEMP); @@ -5275,42 +5384,6 @@ SYSCTL_PROC(_net_inet_mptcp, OID_AUTO, pcblist, CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, mptcp_pcblist, "S,conninfo_mptcp_t", "List of active MPTCP connections"); -/* - * Check the health of the other subflows and do an mptcp_output if - * there is no other active or functional subflow at the time of - * call of this function. - */ -static void -mptcp_output_needed(struct mptses *mpte, struct mptsub *to_mpts) -{ - struct mptsub *from_mpts = NULL; - - MPTE_LOCK_ASSERT_HELD(mpte); - - MPTS_UNLOCK(to_mpts); - - from_mpts = mpte->mpte_active_sub; - - if (from_mpts == NULL) - goto output_needed; - - MPTS_LOCK(from_mpts); - - if ((from_mpts->mpts_flags & MPTSF_DISCONNECTED) || - (from_mpts->mpts_flags & MPTSF_DISCONNECTING)) { - MPTS_UNLOCK(from_mpts); - goto output_needed; - } - - MPTS_UNLOCK(from_mpts); - MPTS_LOCK(to_mpts); - return; - -output_needed: - mptcp_output(mpte); - MPTS_LOCK(to_mpts); -} - /* * Set notsent lowat mark on the MPTCB */ @@ -5328,7 +5401,7 @@ mptcp_set_notsent_lowat(struct mptses *mpte, int optval) else error = EINVAL; - return error; + return (error); } u_int32_t @@ -5340,13 +5413,14 @@ mptcp_get_notsent_lowat(struct mptses *mpte) mp_tp = mpte->mpte_mptcb; if (mp_tp) - return mp_tp->mpt_notsent_lowat; + return (mp_tp->mpt_notsent_lowat); else - return 0; + return (0); } int -mptcp_notsent_lowat_check(struct socket *so) { +mptcp_notsent_lowat_check(struct socket *so) +{ struct mptses *mpte; struct mppcb *mpp; struct mptcb *mp_tp; @@ -5354,15 +5428,15 @@ mptcp_notsent_lowat_check(struct socket *so) { int notsent = 0; - mpp = sotomppcb(so); + mpp = mpsotomppcb(so); if (mpp == NULL || mpp->mpp_state == MPPCB_STATE_DEAD) { return (0); } mpte = mptompte(mpp); + mpte_lock_assert_held(mpte); mp_tp = mpte->mpte_mptcb; - MPT_LOCK(mp_tp); notsent = so->so_snd.sb_cc; if ((notsent == 0) || @@ -5373,10 +5447,8 @@ mptcp_notsent_lowat_check(struct socket *so) { mp_tp->mpt_notsent_lowat, notsent, notsent - (mp_tp->mpt_sndnxt - mp_tp->mpt_snduna)), MPTCP_SENDER_DBG , MPTCP_LOGLVL_VERBOSE); - MPT_UNLOCK(mp_tp); return (1); } - MPT_UNLOCK(mp_tp); /* When Nagle's algorithm is not disabled, it is better * to wakeup the client even before there is atleast one @@ -5384,10 +5456,8 @@ mptcp_notsent_lowat_check(struct socket *so) { */ TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { int retval = 0; - MPTS_LOCK(mpts); if (mpts->mpts_flags & MPTSF_ACTIVE) { struct socket *subf_so = mpts->mpts_socket; - socket_lock(subf_so, 0); struct tcpcb *tp = intotcpcb(sotoinpcb(subf_so)); notsent = so->so_snd.sb_cc - @@ -5401,31 +5471,16 @@ mptcp_notsent_lowat_check(struct socket *so) { " nodelay false \n", mp_tp->mpt_notsent_lowat, notsent), MPTCP_SENDER_DBG , MPTCP_LOGLVL_VERBOSE); - socket_unlock(subf_so, 0); - MPTS_UNLOCK(mpts); return (retval); } - MPTS_UNLOCK(mpts); } return (0); } -static void -mptcp_get_rtt_measurement(struct mptsub *mpts, struct mptses *mpte) -{ - MPTE_LOCK_ASSERT_HELD(mpte); - MPTS_LOCK_ASSERT_HELD(mpts); - - struct socket *subflow_so = mpts->mpts_socket; - socket_lock(subflow_so, 0); - mpts->mpts_srtt = (intotcpcb(sotoinpcb(subflow_so)))->t_srtt; - mpts->mpts_rxtcur = (intotcpcb(sotoinpcb(subflow_so)))->t_rxtcur; - socket_unlock(subflow_so, 0); -} - /* Using Symptoms Advisory to detect poor WiFi or poor Cell */ static kern_ctl_ref mptcp_kern_ctrl_ref = NULL; static uint32_t mptcp_kern_skt_inuse = 0; +static uint32_t mptcp_kern_skt_unit; symptoms_advisory_t mptcp_advisory; static errno_t @@ -5433,14 +5488,141 @@ mptcp_symptoms_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, void **unitinfo) { #pragma unused(kctlref, sac, unitinfo) - /* - * We don't need to do anything here. But we can atleast ensure - * only one user opens the MPTCP_KERN_CTL_NAME control socket. - */ - if (OSCompareAndSwap(0, 1, &mptcp_kern_skt_inuse)) - return (0); + + if (OSIncrementAtomic(&mptcp_kern_skt_inuse) > 0) + mptcplog((LOG_ERR, "%s MPTCP kernel-control socket already open!", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + mptcp_kern_skt_unit = sac->sc_unit; + + return (0); +} + +static void +mptcp_allow_uuid(uuid_t uuid) +{ + struct mppcb *mpp; + + /* Iterate over all MPTCP connections */ + + lck_mtx_lock(&mtcbinfo.mppi_lock); + + TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) { + struct mptses *mpte; + struct socket *mp_so; + + mpp_lock(mpp); + + mpte = mpp->mpp_pcbe; + mp_so = mpp->mpp_socket; + + if (mp_so->so_flags & SOF_DELEGATED && + uuid_compare(uuid, mp_so->e_uuid)) + goto next; + else if (!(mp_so->so_flags & SOF_DELEGATED) && + uuid_compare(uuid, mp_so->last_uuid)) + goto next; + + mpte->mpte_flags |= MPTE_ACCESS_GRANTED; + + mptcp_check_subflows_and_add(mpte); + mptcp_remove_subflows(mpte); + + mpte->mpte_flags &= ~MPTE_ACCESS_GRANTED; + +next: + mpp_unlock(mpp); + } + + lck_mtx_unlock(&mtcbinfo.mppi_lock); +} + +static void +mptcp_wifi_status_changed(void) +{ + struct mppcb *mpp; + + /* Iterate over all MPTCP connections */ + + lck_mtx_lock(&mtcbinfo.mppi_lock); + + TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) { + struct mptses *mpte; + struct socket *mp_so; + + mpp_lock(mpp); + + mpte = mpp->mpp_pcbe; + mp_so = mpp->mpp_socket; + + /* Only handover-mode is purely driven by Symptom's Wi-Fi status */ + if (mpte->mpte_svctype != MPTCP_SVCTYPE_HANDOVER) + goto next; + + mptcp_check_subflows_and_add(mpte); + mptcp_check_subflows_and_remove(mpte); + +next: + mpp_unlock(mpp); + } + + lck_mtx_unlock(&mtcbinfo.mppi_lock); +} + +void +mptcp_ask_symptoms(struct mptses *mpte) +{ + struct mptcp_symptoms_ask_uuid ask; + struct socket *mp_so; + struct proc *p; + int pid, prio, err; + + if (mptcp_kern_skt_unit == 0) { + mptcplog((LOG_ERR, "%s skt_unit is still 0\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return; + } + + mp_so = mptetoso(mpte); + + if (mp_so->so_flags & SOF_DELEGATED) + pid = mp_so->e_pid; + else + pid = mp_so->last_pid; + + p = proc_find(pid); + if (p == PROC_NULL) { + mptcplog((LOG_ERR, "%s Couldn't find proc for pid %u\n", __func__, + pid), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return; + } + + ask.cmd = MPTCP_SYMPTOMS_ASK_UUID; + + if (mp_so->so_flags & SOF_DELEGATED) + uuid_copy(ask.uuid, mp_so->e_uuid); + else + uuid_copy(ask.uuid, mp_so->last_uuid); + + prio = proc_get_effective_task_policy(proc_task(p), TASK_POLICY_ROLE); + + if (prio == TASK_BACKGROUND_APPLICATION) + ask.priority = MPTCP_SYMPTOMS_BACKGROUND; + else if (prio == TASK_FOREGROUND_APPLICATION) + ask.priority = MPTCP_SYMPTOMS_FOREGROUND; else - return (EALREADY); + ask.priority = MPTCP_SYMPTOMS_UNKNOWN; + + mptcplog((LOG_DEBUG, "%s ask symptoms about pid %u, prio %u\n", __func__, + pid, ask.priority), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + err = ctl_enqueuedata(mptcp_kern_ctrl_ref, mptcp_kern_skt_unit, + &ask, sizeof(ask), CTL_DATA_EOR); + if (err) + mptcplog((LOG_ERR, "%s ctl_enqueuedata failed %d\n", __func__, err), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + proc_rele(p); } static errno_t @@ -5448,23 +5630,24 @@ mptcp_symptoms_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo) { #pragma unused(kctlref, kcunit, unitinfo) - if (OSCompareAndSwap(1, 0, &mptcp_kern_skt_inuse)) { - /* TBD needs to be locked if the size grows more than an int */ - bzero(&mptcp_advisory, sizeof(mptcp_advisory)); - return (0); - } - else { - return (EINVAL); - } + + OSDecrementAtomic(&mptcp_kern_skt_inuse); + + return (0); } static errno_t mptcp_symptoms_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, mbuf_t m, int flags) { -#pragma unused(kctlref, kcunit, unitinfo, flags) +#pragma unused(kctlref, unitinfo, flags) symptoms_advisory_t *sa = NULL; + if (kcunit != mptcp_kern_skt_unit) + mptcplog((LOG_ERR, "%s kcunit %u is different from expected one %u\n", + __func__, kcunit, mptcp_kern_skt_unit), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + if (mbuf_pkthdr_len(m) < sizeof(*sa)) { mbuf_freem(m); return (EINVAL); @@ -5475,41 +5658,36 @@ mptcp_symptoms_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, else return (EINVAL); - if (mptcp_advisory.sa_nwk_status_int != sa->sa_nwk_status_int) { - /* - * we could use this notification to notify all mptcp pcbs - * of the change in network status. But its difficult to - * define if sending REMOVE_ADDR or MP_PRIO is appropriate - * given that these are only soft indicators of the network - * state. Leaving this as TBD for now. - */ - } + if (sa->sa_nwk_status != SYMPTOMS_ADVISORY_NOCOMMENT && + sa->sa_nwk_status != SYMPTOMS_ADVISORY_USEAPP) { + uint8_t old_wifi_status = mptcp_advisory.sa_wifi_status; - if (sa->sa_nwk_status != SYMPTOMS_ADVISORY_NOCOMMENT) { - mptcplog((LOG_DEBUG, "MPTCP Events: %s wifi %d,%d cell %d,%d\n", - __func__, sa->sa_wifi_status, mptcp_advisory.sa_wifi_status, - sa->sa_cell_status, mptcp_advisory.sa_cell_status), - MPTCP_SOCKET_DBG | MPTCP_EVENTS_DBG, - MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: wifi %d,%d\n", + __func__, sa->sa_wifi_status, mptcp_advisory.sa_wifi_status), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); if ((sa->sa_wifi_status & (SYMPTOMS_ADVISORY_WIFI_BAD | SYMPTOMS_ADVISORY_WIFI_OK)) != - (SYMPTOMS_ADVISORY_WIFI_BAD | SYMPTOMS_ADVISORY_WIFI_OK)) { + (SYMPTOMS_ADVISORY_WIFI_BAD | SYMPTOMS_ADVISORY_WIFI_OK)) mptcp_advisory.sa_wifi_status = sa->sa_wifi_status; - } - if ((sa->sa_cell_status & - (SYMPTOMS_ADVISORY_CELL_BAD | SYMPTOMS_ADVISORY_CELL_OK)) != - (SYMPTOMS_ADVISORY_CELL_BAD | SYMPTOMS_ADVISORY_CELL_OK)) { - mptcp_advisory.sa_cell_status = sa->sa_cell_status; - } - } else { - mptcplog((LOG_DEBUG, "MPTCP Events: %s NOCOMMENT " - "wifi %d cell %d\n", __func__, - mptcp_advisory.sa_wifi_status, - mptcp_advisory.sa_cell_status), - MPTCP_SOCKET_DBG | MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + if (old_wifi_status != mptcp_advisory.sa_wifi_status) + mptcp_wifi_status_changed(); + } else if (sa->sa_nwk_status == SYMPTOMS_ADVISORY_NOCOMMENT) { + mptcplog((LOG_DEBUG, "%s: NOCOMMENT wifi %d\n", __func__, + mptcp_advisory.sa_wifi_status), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); + } else if (sa->sa_nwk_status == SYMPTOMS_ADVISORY_USEAPP) { + uuid_t uuid; + + mptcplog((LOG_DEBUG, "%s Got response about useApp\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + uuid_copy(uuid, (unsigned char *)(sa + 1)); + + mptcp_allow_uuid(uuid); } + return (0); } @@ -5537,141 +5715,172 @@ mptcp_is_wifi_unusable(void) return (mptcp_advisory.sa_wifi_status & SYMPTOMS_ADVISORY_WIFI_BAD); } -int -mptcp_is_cell_unusable(void) -{ - /* a false return val indicates there is no info or cell is ok */ - return (mptcp_advisory.sa_cell_status & SYMPTOMS_ADVISORY_CELL_BAD); -} - -struct mptsub* -mptcp_use_symptoms_hints(struct mptsub* best, struct mptsub *second_best) -{ - struct mptsub *cellsub = NULL; - struct mptsub *wifisub = NULL; - struct mptsub *wiredsub = NULL; - - VERIFY ((best != NULL) && (second_best != NULL)); - - if (!mptcp_use_symptomsd) - return (NULL); - - if (!mptcp_kern_skt_inuse) - return (NULL); - - /* - * There could be devices with more than one wifi interface or - * more than one wired or cell interfaces. - * TBD: SymptomsD is unavailable on such platforms as of now. - * Try to prefer best when possible in general. - * Also, SymptomsD sends notifications about wifi only when it - * is primary. - */ - if (best->mpts_linktype & MPTSL_WIFI) - wifisub = best; - else if (best->mpts_linktype & MPTSL_CELL) - cellsub = best; - else if (best->mpts_linktype & MPTSL_WIRED) - wiredsub = best; - - /* - * On platforms with wired paths, don't use hints about wifi or cell. - * Currently, SymptomsD is not available on platforms with wired paths. - */ - if (wiredsub) - return (NULL); - - if ((wifisub == NULL) && (second_best->mpts_linktype & MPTSL_WIFI)) - wifisub = second_best; - - if ((cellsub == NULL) && (second_best->mpts_linktype & MPTSL_CELL)) - cellsub = second_best; - - if ((wiredsub == NULL) && (second_best->mpts_linktype & MPTSL_WIRED)) - wiredsub = second_best; - - if ((wifisub == best) && mptcp_is_wifi_unusable()) { - tcpstat.tcps_mp_sel_symtomsd++; - if (mptcp_is_cell_unusable()) { - mptcplog((LOG_DEBUG, "MPTCP Sender: SymptomsD hint" - " suggests both Wifi and Cell are bad. Wired %s.", - (wiredsub == NULL) ? "none" : "present"), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return (wiredsub); - } else { - mptcplog((LOG_DEBUG, "MPTCP Sender: SymptomsD hint" - " suggests Wifi bad, Cell good. Wired %s.", - (wiredsub == NULL) ? "none" : "present"), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return ((wiredsub != NULL) ? wiredsub : cellsub); - } - } - - if ((cellsub == best) && (mptcp_is_cell_unusable())) { - tcpstat.tcps_mp_sel_symtomsd++; - if (mptcp_is_wifi_unusable()) { - mptcplog((LOG_DEBUG, "MPTCP Sender: SymptomsD hint" - " suggests both Cell and Wifi are bad. Wired %s.", - (wiredsub == NULL) ? "none" : "present"), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return (wiredsub); - } else { - mptcplog((LOG_DEBUG, "MPTCP Sender: SymptomsD hint" - " suggests Cell bad, Wifi good. Wired %s.", - (wiredsub == NULL) ? "none" : "present"), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return ((wiredsub != NULL) ? wiredsub : wifisub); - } - } - - /* little is known about the state of the network or wifi is good */ - return (NULL); -} - /* If TFO data is succesfully acked, it must be dropped from the mptcp so */ static void -mptcp_drop_tfo_data(struct mptses *mpte, struct mptsub *mpts, int *wakeup) +mptcp_drop_tfo_data(struct mptses *mpte, struct mptsub *mpts) { - struct socket *mp_so = mpte->mpte_mppcb->mpp_socket; + struct socket *mp_so = mptetoso(mpte); struct socket *so = mpts->mpts_socket; struct tcpcb *tp = intotcpcb(sotoinpcb(so)); struct mptcb *mp_tp = mpte->mpte_mptcb; /* If data was sent with SYN, rewind state */ if (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) { - mpts->mpts_flags &= ~MPTSF_TFO_REQD; - tp->t_mpflags &= ~TMPF_TFO_REQUEST; - MPT_LOCK(mp_tp); - u_int64_t mp_droplen = mpts->mpts_sndnxt - mp_tp->mpt_snduna; + u_int64_t mp_droplen = mp_tp->mpt_sndnxt - mp_tp->mpt_snduna; unsigned int tcp_droplen = tp->snd_una - tp->iss - 1; + VERIFY(mp_droplen <= (UINT_MAX)); VERIFY(mp_droplen >= tcp_droplen); + mpts->mpts_flags &= ~MPTSF_TFO_REQD; + mpts->mpts_iss += tcp_droplen; + tp->t_mpflags &= ~TMPF_TFO_REQUEST; + if (mp_droplen > tcp_droplen) { /* handle partial TCP ack */ mp_so->so_flags1 |= SOF1_TFO_REWIND; mp_tp->mpt_sndnxt = mp_tp->mpt_snduna + (mp_droplen - tcp_droplen); - mpts->mpts_sndnxt = mp_tp->mpt_sndnxt; mp_droplen = tcp_droplen; } else { /* all data on SYN was acked */ mpts->mpts_rel_seq = 1; mp_tp->mpt_sndnxt = mp_tp->mpt_snduna; - mpts->mpts_sndnxt = mp_tp->mpt_snduna; } mp_tp->mpt_sndmax -= tcp_droplen; - MPT_UNLOCK(mp_tp); if (mp_droplen != 0) { VERIFY(mp_so->so_snd.sb_mb != NULL); sbdrop(&mp_so->so_snd, (int)mp_droplen); - if (wakeup) - *wakeup = 1; } - mptcplog((LOG_ERR, "MPTCP Sender: %s mp_so 0x%llx cid %d " - "TFO tcp len %d mptcp len %d\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpts->mpts_connid, - tcp_droplen, mp_droplen), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx cid %d TFO tcp len %d mptcp len %d\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mpts->mpts_connid, tcp_droplen, mp_droplen), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + } +} + +int +mptcp_freeq(struct mptcb *mp_tp) +{ + struct tseg_qent *q; + int rv = 0; + + while ((q = LIST_FIRST(&mp_tp->mpt_segq)) != NULL) { + LIST_REMOVE(q, tqe_q); + m_freem(q->tqe_m); + zfree(tcp_reass_zone, q); + rv = 1; + } + mp_tp->mpt_reassqlen = 0; + return (rv); +} + +static int +mptcp_post_event(u_int32_t event_code, int value) +{ + struct kev_mptcp_data event_data; + struct kev_msg ev_msg; + + memset(&ev_msg, 0, sizeof(ev_msg)); + + ev_msg.vendor_code = KEV_VENDOR_APPLE; + ev_msg.kev_class = KEV_NETWORK_CLASS; + ev_msg.kev_subclass = KEV_MPTCP_SUBCLASS; + ev_msg.event_code = event_code; + + event_data.value = value; + + ev_msg.dv[0].data_ptr = &event_data; + ev_msg.dv[0].data_length = sizeof(event_data); + + return kev_post_msg(&ev_msg); +} + +void +mptcp_set_cellicon(struct mptses *mpte) +{ + int error; + + /* First-party apps (Siri) don't flip the cellicon */ + if (mpte->mpte_flags & MPTE_FIRSTPARTY) + return; + + /* Remember the last time we set the cellicon (see mptcp_unset_cellicon) */ + mptcp_last_cellicon_set = tcp_now; + + /* If cellicon is already set, get out of here! */ + if (OSTestAndSet(7, &mptcp_cellicon_is_set)) + return; + + error = mptcp_post_event(KEV_MPTCP_CELLUSE, 1); + + if (error) + mptcplog((LOG_ERR, "%s: Setting cellicon failed with %d\n", + __func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + else + mptcplog((LOG_DEBUG, "%s successfully set the cellicon\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); +} + +void +mptcp_unset_cellicon(void) +{ + int error; + + /* If cellicon is already unset, get out of here! */ + if (OSTestAndClear(7, &mptcp_cellicon_is_set)) + return; + + /* + * If during the past MPTCP_CELLICON_TOGGLE_RATE seconds we didn't + * explicitly set the cellicon (see mptcp_set_cellicon()), then we unset + * it again. + */ + if (TSTMP_GT(mptcp_last_cellicon_set + MPTCP_CELLICON_TOGGLE_RATE, + tcp_now)) { + OSTestAndSet(7, &mptcp_cellicon_is_set); + return; } + + error = mptcp_post_event(KEV_MPTCP_CELLUSE, 0); + + if (error) + mptcplog((LOG_ERR, "%s: Unsetting cellicon failed with %d\n", + __func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + else + mptcplog((LOG_DEBUG, "%s successfully unset the cellicon\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); +} + +void +mptcp_reset_rexmit_state(struct tcpcb *tp) +{ + struct mptsub *mpts; + struct inpcb *inp; + struct socket *so; + + inp = tp->t_inpcb; + if (inp == NULL) + return; + + so = inp->inp_socket; + if (so == NULL) + return; + + if (!(so->so_flags & SOF_MP_SUBFLOW)) + return; + + mpts = tp->t_mpsub; + + mpts->mpts_flags &= ~MPTSF_WRITE_STALL; + so->so_flags &= ~SOF_MP_TRYFAILOVER; +} + +void +mptcp_reset_keepalive(struct tcpcb *tp) +{ + struct mptsub *mpts = tp->t_mpsub; + + mpts->mpts_flags &= ~MPTSF_READ_STALL; } + diff --git a/bsd/netinet/mptcp_timer.c b/bsd/netinet/mptcp_timer.c index 7ac605b74..0f427e184 100644 --- a/bsd/netinet/mptcp_timer.c +++ b/bsd/netinet/mptcp_timer.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -75,8 +75,7 @@ mptcp_timer_demux(struct mptses *mpte, uint32_t now_msecs) DTRACE_MPTCP2(timer, struct mptses *, mpte, struct mptcb *, mp_tp); - MPTE_LOCK_ASSERT_HELD(mpte); - MPT_LOCK(mp_tp); + mpte_lock_assert_held(mpte); switch (mp_tp->mpt_timer_vals) { case MPTT_REXMT: if (mp_tp->mpt_rxtstart == 0) @@ -95,13 +94,11 @@ mptcp_timer_demux(struct mptses *mpte, uint32_t now_msecs) DTRACE_MPTCP1(error, struct mptcb *, mp_tp); } else { mp_tp->mpt_sndnxt = mp_tp->mpt_rtseq; - MPT_UNLOCK(mp_tp); mptcplog((LOG_DEBUG, "MPTCP Socket: " "%s: REXMT %d times.\n", __func__, mp_tp->mpt_rxtshift), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); mptcp_output(mpte); - MPT_LOCK(mp_tp); } } else { resched_timer = 1; @@ -125,7 +122,6 @@ mptcp_timer_demux(struct mptses *mpte, uint32_t now_msecs) default: break; } - MPT_UNLOCK(mp_tp); return (resched_timer); } @@ -138,7 +134,7 @@ mptcp_timer(struct mppcbinfo *mppi) u_int32_t now_msecs; uint32_t resched_timer = 0; - lck_mtx_assert(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); microuptime(&now); now_msecs = TIMEVAL_TO_HZ(now); @@ -150,17 +146,12 @@ mptcp_timer(struct mppcbinfo *mppi) VERIFY(mp_so != NULL); mpte = mptompte(mpp); VERIFY(mpte != NULL); - MPTE_LOCK(mpte); + mpte_lock(mpte); VERIFY(mpp->mpp_flags & MPP_ATTACHED); - if (mpp->mpp_flags & MPP_DEFUNCT) { - MPTE_UNLOCK(mpte); - continue; - } - if (mptcp_timer_demux(mpte, now_msecs)) resched_timer = 1; - MPTE_UNLOCK(mpte); + mpte_unlock(mpte); } return (resched_timer); @@ -178,21 +169,19 @@ mptcp_start_timer(struct mptses *mpte, int timer_type) mptcplog((LOG_DEBUG, "MPTCP Socket: %s: %d\n", __func__, timer_type), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + mpte_lock_assert_held(mpte); + switch (timer_type) { case MPTT_REXMT: - MPT_LOCK(mp_tp); mp_tp->mpt_timer_vals |= MPTT_REXMT; mp_tp->mpt_rxtstart = TIMEVAL_TO_HZ(now); mp_tp->mpt_rxtshift = 0; mp_tp->mpt_rtseq = mp_tp->mpt_sndnxt; - MPT_UNLOCK(mp_tp); break; case MPTT_TW: /* XXX: Not implemented yet */ - MPT_LOCK(mp_tp); mp_tp->mpt_timer_vals |= MPTT_TW; mp_tp->mpt_timewait = TIMEVAL_TO_HZ(now); - MPT_UNLOCK(mp_tp); break; case MPTT_FASTCLOSE: /* NO-OP */ @@ -207,7 +196,7 @@ mptcp_start_timer(struct mptses *mpte, int timer_type) void mptcp_cancel_timer(struct mptcb *mp_tp, int timer_type) { - MPT_LOCK_ASSERT_HELD(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); DTRACE_MPTCP2(cancel__timer, struct mptcb *, mp_tp, int, timer_type); switch (timer_type) { diff --git a/bsd/netinet/mptcp_timer.h b/bsd/netinet/mptcp_timer.h index 94da71dc5..35ee57c48 100644 --- a/bsd/netinet/mptcp_timer.h +++ b/bsd/netinet/mptcp_timer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * diff --git a/bsd/netinet/mptcp_usrreq.c b/bsd/netinet/mptcp_usrreq.c index cc84cb40e..1a5589901 100644 --- a/bsd/netinet/mptcp_usrreq.c +++ b/bsd/netinet/mptcp_usrreq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -37,6 +37,8 @@ #include #include #include +#include +#include #include #include @@ -54,10 +56,6 @@ static int mptcp_usr_attach(struct socket *, int, struct proc *); static int mptcp_usr_detach(struct socket *); static int mptcp_attach(struct socket *, struct proc *); -static int mptcp_detach(struct socket *, struct mppcb *); -static int mptcp_connectx(struct mptses *, struct sockaddr *, - struct sockaddr *, struct proc *, uint32_t, sae_associd_t, - sae_connid_t *, uint32_t, void *, uint32_t); static int mptcp_usr_connectx(struct socket *, struct sockaddr *, struct sockaddr *, struct proc *, uint32_t, sae_associd_t, sae_connid_t *, uint32_t, void *, uint32_t, struct uio *, user_ssize_t *); @@ -69,25 +67,20 @@ static int mptcp_getconninfo(struct mptses *, sae_connid_t *, uint32_t *, uint32_t *, user_addr_t, uint32_t *); static int mptcp_usr_control(struct socket *, u_long, caddr_t, struct ifnet *, struct proc *); -static int mptcp_disconnectx(struct mptses *, sae_associd_t, sae_connid_t); +static int mptcp_disconnect(struct mptses *); static int mptcp_usr_disconnect(struct socket *); static int mptcp_usr_disconnectx(struct socket *, sae_associd_t, sae_connid_t); static struct mptses *mptcp_usrclosed(struct mptses *); -static int mptcp_usr_peeloff(struct socket *, sae_associd_t, struct socket **); -static int mptcp_peeloff(struct mptses *, sae_associd_t, struct socket **); static int mptcp_usr_rcvd(struct socket *, int); static int mptcp_usr_send(struct socket *, int, struct mbuf *, struct sockaddr *, struct mbuf *, struct proc *); static int mptcp_usr_shutdown(struct socket *); -static int mptcp_uiotombuf(struct uio *, int, int, uint32_t, struct mbuf **); static int mptcp_usr_sosend(struct socket *, struct sockaddr *, struct uio *, struct mbuf *, struct mbuf *, int); static int mptcp_usr_socheckopt(struct socket *, struct sockopt *); -static int mptcp_setopt_apply(struct mptses *, struct mptopt *); static int mptcp_setopt(struct mptses *, struct sockopt *); static int mptcp_getopt(struct mptses *, struct sockopt *); static int mptcp_default_tcp_optval(struct mptses *, struct sockopt *, int *); -static void mptcp_connorder_helper(struct mptsub *mpts); static int mptcp_usr_preconnect(struct socket *so); struct pr_usrreqs mptcp_usrreqs = { @@ -97,16 +90,29 @@ struct pr_usrreqs mptcp_usrreqs = { .pru_detach = mptcp_usr_detach, .pru_disconnect = mptcp_usr_disconnect, .pru_disconnectx = mptcp_usr_disconnectx, - .pru_peeloff = mptcp_usr_peeloff, + .pru_peeraddr = mp_getpeeraddr, .pru_rcvd = mptcp_usr_rcvd, .pru_send = mptcp_usr_send, .pru_shutdown = mptcp_usr_shutdown, + .pru_sockaddr = mp_getsockaddr, .pru_sosend = mptcp_usr_sosend, .pru_soreceive = soreceive, .pru_socheckopt = mptcp_usr_socheckopt, .pru_preconnect = mptcp_usr_preconnect, }; + +#if (DEVELOPMENT || DEBUG) +static int mptcp_disable_entitlements = 0; +SYSCTL_INT(_net_inet_mptcp, OID_AUTO, disable_entitlements, CTLFLAG_RW | CTLFLAG_LOCKED, + &mptcp_disable_entitlements, 0, "Disable Multipath TCP Entitlement Checking"); +#endif + +int mptcp_developer_mode = 0; +SYSCTL_INT(_net_inet_mptcp, OID_AUTO, allow_aggregate, CTLFLAG_RW | CTLFLAG_LOCKED, + &mptcp_developer_mode, 0, "Allow the Multipath aggregation mode"); + + /* * Attaches an MPTCP control block to a socket. */ @@ -116,7 +122,7 @@ mptcp_usr_attach(struct socket *mp_so, int proto, struct proc *p) #pragma unused(proto) int error; - VERIFY(sotomppcb(mp_so) == NULL); + VERIFY(mpsotomppcb(mp_so) == NULL); error = mptcp_attach(mp_so, p); if (error != 0) @@ -138,14 +144,27 @@ out: static int mptcp_usr_detach(struct socket *mp_so) { - struct mppcb *mpp = sotomppcb(mp_so); - int error = 0; + struct mptses *mpte = mpsotompte(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); - VERIFY(mpp != NULL); - VERIFY(mpp->mpp_socket != NULL); + if (mpp == NULL || mpp->mpp_state == MPPCB_STATE_DEAD) { + mptcplog((LOG_ERR, "%s state: %d\n", __func__, + mpp ? mpp->mpp_state : -1), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return (EINVAL); + } - error = mptcp_detach(mp_so, mpp); - return (error); + /* + * We are done with this MPTCP socket (it has been closed); + * trigger all subflows to be disconnected, if not already, + * by initiating the PCB detach sequence (SOF_PCBCLEARING + * will be set.) + */ + mp_pcbdetach(mp_so); + + mptcp_disconnect(mpte); + + return (0); } /* @@ -162,7 +181,7 @@ mptcp_attach(struct socket *mp_so, struct proc *p) int error = 0; if (mp_so->so_snd.sb_hiwat == 0 || mp_so->so_rcv.sb_hiwat == 0) { - error = soreserve(mp_so, tcp_sendspace, MPTCP_RWIN_MAX); + error = soreserve(mp_so, tcp_sendspace, tcp_recvspace); if (error != 0) goto out; } @@ -171,6 +190,11 @@ mptcp_attach(struct socket *mp_so, struct proc *p) soreserve_preconnect(mp_so, 2048); } + if ((mp_so->so_rcv.sb_flags & SB_USRSIZE) == 0) + mp_so->so_rcv.sb_flags |= SB_AUTOSIZE; + if ((mp_so->so_snd.sb_flags & SB_USRSIZE) == 0) + mp_so->so_snd.sb_flags |= SB_AUTOSIZE; + /* * MPTCP socket buffers cannot be compressed, due to the * fact that each mbuf chained via m_next is a M_PKTHDR @@ -179,15 +203,11 @@ mptcp_attach(struct socket *mp_so, struct proc *p) mp_so->so_snd.sb_flags |= SB_NOCOMPRESS; mp_so->so_rcv.sb_flags |= SB_NOCOMPRESS; - /* Disable socket buffer auto-tuning. */ - mp_so->so_rcv.sb_flags &= ~SB_AUTOSIZE; - mp_so->so_snd.sb_flags &= ~SB_AUTOSIZE; - if ((error = mp_pcballoc(mp_so, &mtcbinfo)) != 0) { goto out; } - mpp = sotomppcb(mp_so); + mpp = mpsotomppcb(mp_so); VERIFY(mpp != NULL); mpte = (struct mptses *)mpp->mpp_pcbe; VERIFY(mpte != NULL); @@ -197,43 +217,41 @@ out: return (error); } -/* - * Called when the socket layer loses its final reference to the socket; - * at this point, there is only one case in which we will keep things - * around: time wait. - */ static int -mptcp_detach(struct socket *mp_so, struct mppcb *mpp) +mptcp_entitlement_check(struct socket *mp_so) { - struct mptses *mpte; - struct mppcbinfo *mppi; + struct mptses *mpte = mpsotompte(mp_so); - VERIFY(mp_so->so_pcb == mpp); - VERIFY(mpp->mpp_socket == mp_so); - - mppi = mpp->mpp_pcbinfo; - VERIFY(mppi != NULL); - - __IGNORE_WCASTALIGN(mpte = &((struct mpp_mtp *)mpp)->mpp_ses); - VERIFY(mpte->mpte_mppcb == mpp); + if (soopt_cred_check(mp_so, PRIV_NET_RESTRICTED_MULTIPATH_EXTENDED, TRUE) == 0) { + /* + * This means the app has the extended entitlement. Thus, + * it's a first party app and can run without restrictions. + */ + mpte->mpte_flags |= MPTE_FIRSTPARTY; + goto grant; + } - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ +#if (DEVELOPMENT || DEBUG) + if (mptcp_disable_entitlements) + goto grant; +#endif - /* - * We are done with this MPTCP socket (it has been closed); - * trigger all subflows to be disconnected, if not already, - * by initiating the PCB detach sequence (SOF_PCBCLEARING - * will be set.) - */ - mp_pcbdetach(mpp); + if (soopt_cred_check(mp_so, PRIV_NET_PRIVILEGED_MULTIPATH, TRUE)) { + mptcplog((LOG_NOTICE, "%s Multipath Capability needed\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + return (-1); + } - (void) mptcp_disconnectx(mpte, SAE_ASSOCID_ALL, SAE_CONNID_ALL); + if (mpte->mpte_svctype > MPTCP_SVCTYPE_INTERACTIVE && + mptcp_developer_mode == 0) { + mptcplog((LOG_NOTICE, "%s need to set allow_aggregate sysctl\n", + __func__), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + return (-1); + } - /* - * XXX: adi@apple.com - * - * Here, we would want to handle time wait state. - */ +grant: + mptcplog((LOG_NOTICE, "%s entitlement granted for %u\n", __func__, mpte->mpte_svctype), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); return (0); } @@ -246,74 +264,20 @@ mptcp_detach(struct socket *mp_so, struct mppcb *mpp) */ static int mptcp_connectx(struct mptses *mpte, struct sockaddr *src, - struct sockaddr *dst, struct proc *p, uint32_t ifscope, - sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg, - uint32_t arglen) + struct sockaddr *dst, uint32_t ifscope, sae_connid_t *pcid) { -#pragma unused(p, aid, flags, arg, arglen) - struct mptsub *mpts; - struct socket *mp_so; + struct socket *mp_so = mptetoso(mpte); int error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; - VERIFY(dst != NULL); VERIFY(pcid != NULL); - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx\n", __func__, + mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + DTRACE_MPTCP2(connectx, struct mptses *, mpte, struct socket *, mp_so); - DTRACE_MPTCP3(connectx, struct mptses *, mpte, sae_associd_t, aid, - struct socket *, mp_so); - - mpts = mptcp_subflow_alloc(M_WAITOK); - if (mpts == NULL) { - error = ENOBUFS; - goto out; - } - MPTS_ADDREF(mpts); /* for this routine */ - - if (src != NULL) { - int len = src->sa_len; - - MALLOC(mpts->mpts_src, struct sockaddr *, len, M_SONAME, - M_WAITOK | M_ZERO); - if (mpts->mpts_src == NULL) { - error = ENOBUFS; - goto out; - } - bcopy(src, mpts->mpts_src, len); - } - - MALLOC(mpts->mpts_dst, struct sockaddr *, dst->sa_len, M_SONAME, - M_WAITOK | M_ZERO); - if (mpts->mpts_dst == NULL) { - error = ENOBUFS; - goto out; - } - bcopy(dst, mpts->mpts_dst, dst->sa_len); - - error = mptcp_subflow_add(mpte, mpts, p, ifscope); - if (error == 0 && pcid != NULL) - *pcid = mpts->mpts_connid; - -out: - if (mpts != NULL) { - if ((error != 0) && (error != EWOULDBLOCK)) { - MPTS_LOCK(mpts); - if (mpts->mpts_flags & MPTSF_ATTACHED) { - MPTS_UNLOCK(mpts); - MPTS_REMREF(mpts); - mptcp_subflow_del(mpte, mpts, TRUE); - return (error); - } - MPTS_UNLOCK(mpts); - } - MPTS_REMREF(mpts); - } + error = mptcp_subflow_add(mpte, src, dst, ifscope, pcid); return (error); } @@ -327,30 +291,90 @@ mptcp_usr_connectx(struct socket *mp_so, struct sockaddr *src, sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg, uint32_t arglen, struct uio *auio, user_ssize_t *bytes_written) { - struct mppcb *mpp = sotomppcb(mp_so); +#pragma unused(p, aid, flags, arg, arglen) + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte = NULL; struct mptcb *mp_tp = NULL; user_ssize_t datalen; - int error = 0; if (mpp == NULL || mpp->mpp_state == MPPCB_STATE_DEAD) { + mptcplog((LOG_ERR, "%s state %d\n", __func__, + mpp ? mpp->mpp_state : -1), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); error = EINVAL; goto out; } mpte = mptompte(mpp); VERIFY(mpte != NULL); + mpte_lock_assert_held(mpte); mp_tp = mpte->mpte_mptcb; VERIFY(mp_tp != NULL); if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) { + mptcplog((LOG_ERR, "%s fell back to TCP\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); error = EINVAL; goto out; } - error = mptcp_connectx(mpte, src, dst, p, ifscope, - aid, pcid, flags, arg, arglen); + if (dst->sa_family == AF_INET && + dst->sa_len != sizeof(mpte->__mpte_dst_v4)) { + mptcplog((LOG_ERR, "%s IPv4 dst len %u\n", __func__, + dst->sa_len), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = EINVAL; + goto out; + } + + if (dst->sa_family == AF_INET6 && + dst->sa_len != sizeof(mpte->__mpte_dst_v6)) { + mptcplog((LOG_ERR, "%s IPv6 dst len %u\n", __func__, + dst->sa_len), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = EINVAL; + goto out; + } + + if (!(mpte->mpte_flags & MPTE_SVCTYPE_CHECKED)) { + if (mptcp_entitlement_check(mp_so) < 0) { + error = EPERM; + goto out; + } + + mpte->mpte_flags |= MPTE_SVCTYPE_CHECKED; + } + + if ((mp_so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0) { + memcpy(&mpte->mpte_dst, dst, dst->sa_len); + } + + if (src) { + if (src->sa_family == AF_INET && + src->sa_len != sizeof(mpte->__mpte_src_v4)) { + mptcplog((LOG_ERR, "%s IPv4 src len %u\n", __func__, + src->sa_len), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = EINVAL; + goto out; + } + + if (src->sa_family == AF_INET6 && + src->sa_len != sizeof(mpte->__mpte_src_v6)) { + mptcplog((LOG_ERR, "%s IPv6 src len %u\n", __func__, + src->sa_len), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = EINVAL; + goto out; + } + + if ((mp_so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0) { + memcpy(&mpte->mpte_src, src, src->sa_len); + } + } + + error = mptcp_connectx(mpte, src, dst, ifscope, pcid); /* If there is data, copy it */ if (auio != NULL) { @@ -358,7 +382,7 @@ mptcp_usr_connectx(struct socket *mp_so, struct sockaddr *src, socket_unlock(mp_so, 0); error = mp_so->so_proto->pr_usrreqs->pru_sosend(mp_so, NULL, (uio_t) auio, NULL, NULL, 0); - /* check if this can be supported with fast Join also. XXX */ + if (error == 0 || error == EWOULDBLOCK) *bytes_written = datalen - uio_resid(auio); @@ -366,16 +390,6 @@ mptcp_usr_connectx(struct socket *mp_so, struct sockaddr *src, error = EINPROGRESS; socket_lock(mp_so, 0); - MPT_LOCK(mp_tp); - if (mp_tp->mpt_flags & MPTCPF_PEEL_OFF) { - *bytes_written = datalen - uio_resid(auio); - /* - * Override errors like EPIPE that occur as - * a result of doing TFO during TCP fallback. - */ - error = EPROTO; - } - MPT_UNLOCK(mp_tp); } out: @@ -388,7 +402,7 @@ out: static int mptcp_getassocids(struct mptses *mpte, uint32_t *cnt, user_addr_t aidp) { - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ /* MPTCP has at most 1 association */ *cnt = (mpte->mpte_associd != SAE_ASSOCID_ANY) ? 1 : 0; @@ -411,7 +425,7 @@ mptcp_getconnids(struct mptses *mpte, sae_associd_t aid, uint32_t *cnt, struct mptsub *mpts; int error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL && aid != mpte->mpte_associd) @@ -443,237 +457,132 @@ mptcp_getconninfo(struct mptses *mpte, sae_connid_t *cid, uint32_t *flags, user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type, user_addr_t aux_data, uint32_t *aux_len) { -#pragma unused(aux_data) - struct ifnet *ifp = NULL; + struct socket *so; + struct inpcb *inp; struct mptsub *mpts; int error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - - if (*cid == SAE_CONNID_ALL) - return (EINVAL); - - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - if (mpts->mpts_connid == *cid || *cid == SAE_CONNID_ANY) - break; - } - if (mpts == NULL) - return ((*cid == SAE_CONNID_ANY) ? ENXIO : EINVAL); - - MPTS_LOCK(mpts); - ifp = mpts->mpts_outif; - *cid = mpts->mpts_connid; - *ifindex = ((ifp != NULL) ? ifp->if_index : 0); - *soerror = mpts->mpts_soerror; *flags = 0; - if (mpts->mpts_flags & MPTSF_CONNECTING) - *flags |= CIF_CONNECTING; - if (mpts->mpts_flags & MPTSF_CONNECTED) - *flags |= CIF_CONNECTED; - if (mpts->mpts_flags & MPTSF_DISCONNECTING) - *flags |= CIF_DISCONNECTING; - if (mpts->mpts_flags & MPTSF_DISCONNECTED) - *flags |= CIF_DISCONNECTED; - if (mpts->mpts_flags & MPTSF_BOUND_IF) - *flags |= CIF_BOUND_IF; - if (mpts->mpts_flags & MPTSF_BOUND_IP) - *flags |= CIF_BOUND_IP; - if (mpts->mpts_flags & MPTSF_BOUND_PORT) - *flags |= CIF_BOUND_PORT; - if (mpts->mpts_flags & MPTSF_PREFERRED) - *flags |= CIF_PREFERRED; - if (mpts->mpts_flags & MPTSF_MP_CAPABLE) - *flags |= CIF_MP_CAPABLE; - if (mpts->mpts_flags & MPTSF_MP_DEGRADED) - *flags |= CIF_MP_DEGRADED; - if (mpts->mpts_flags & MPTSF_MP_READY) - *flags |= CIF_MP_READY; - if (mpts->mpts_flags & MPTSF_ACTIVE) - *flags |= CIF_MP_ACTIVE; + *aux_type = 0; + *ifindex = 0; + *soerror = 0; + + if (*cid == SAE_CONNID_ALL) { + struct socket *mp_so = mptetoso(mpte); + struct mptcb *mp_tp = mpte->mpte_mptcb; + struct conninfo_multipathtcp mptcp_ci; + + if (*aux_len != 0 && *aux_len != sizeof(mptcp_ci)) + return (EINVAL); + + if (mp_so->so_state & SS_ISCONNECTING) + *flags |= CIF_CONNECTING; + if (mp_so->so_state & SS_ISCONNECTED) + *flags |= CIF_CONNECTED; + if (mp_so->so_state & SS_ISDISCONNECTING) + *flags |= CIF_DISCONNECTING; + if (mp_so->so_state & SS_ISDISCONNECTED) + *flags |= CIF_DISCONNECTED; + if (!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP)) + *flags |= CIF_MP_CAPABLE; + if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) + *flags |= CIF_MP_DEGRADED; + + *src_len = 0; + *dst_len = 0; + + *aux_type = CIAUX_MPTCP; + *aux_len = sizeof(mptcp_ci); - VERIFY(mpts->mpts_src != NULL); - *src_len = mpts->mpts_src->sa_len; - if (src != USER_ADDR_NULL) { - error = copyout(mpts->mpts_src, src, mpts->mpts_src->sa_len); - if (error != 0) - goto out; - } + if (aux_data != USER_ADDR_NULL) { + unsigned long i = 0; + int initial_info_set = 0; - VERIFY(mpts->mpts_dst != NULL); - *dst_len = mpts->mpts_dst->sa_len; - if (dst != USER_ADDR_NULL) { - error = copyout(mpts->mpts_dst, dst, mpts->mpts_dst->sa_len); - if (error != 0) - goto out; - } + bzero(&mptcp_ci, sizeof (mptcp_ci)); + mptcp_ci.mptcpci_subflow_count = mpte->mpte_numflows; + mptcp_ci.mptcpci_switch_count = mpte->mpte_subflow_switches; - *aux_type = 0; - *aux_len = 0; - if (mpts->mpts_socket != NULL) { - struct conninfo_tcp tcp_ci; + VERIFY(sizeof(mptcp_ci.mptcpci_itfstats) == sizeof(mpte->mpte_itfstats)); + memcpy(mptcp_ci.mptcpci_itfstats, mpte->mpte_itfstats, sizeof(mptcp_ci.mptcpci_itfstats)); - *aux_type = CIAUX_TCP; - *aux_len = sizeof (tcp_ci); + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + if (i >= sizeof(mptcp_ci.mptcpci_subflow_connids) / sizeof(sae_connid_t)) + break; + mptcp_ci.mptcpci_subflow_connids[i] = mpts->mpts_connid; - if (aux_data != USER_ADDR_NULL) { - struct socket *so = mpts->mpts_socket; - - VERIFY(SOCK_PROTO(so) == IPPROTO_TCP); - bzero(&tcp_ci, sizeof (tcp_ci)); - socket_lock(so, 0); - tcp_getconninfo(so, &tcp_ci); - socket_unlock(so, 0); - error = copyout(&tcp_ci, aux_data, sizeof (tcp_ci)); - if (error != 0) - goto out; - } - } - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: cid %d flags %x \n", - __func__, mpts->mpts_connid, mpts->mpts_flags), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + if (mpts->mpts_flags & MPTSF_INITIAL_SUB) { + inp = sotoinpcb(mpts->mpts_socket); -out: - MPTS_UNLOCK(mpts); - return (error); -} + mptcp_ci.mptcpci_init_rxbytes = inp->inp_stat->rxbytes; + mptcp_ci.mptcpci_init_txbytes = inp->inp_stat->txbytes; + initial_info_set = 1; + } -/* - * Handle SIOCSCONNORDER - */ -int -mptcp_setconnorder(struct mptses *mpte, sae_connid_t cid, uint32_t rank) -{ - struct mptsub *mpts, *mpts1; - int error = 0; + mptcpstats_update(mptcp_ci.mptcpci_itfstats, mpts); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: cid %d rank %d \n", __func__, cid, rank), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + i++; + } - if (cid == SAE_CONNID_ANY || cid == SAE_CONNID_ALL) { - error = EINVAL; - goto out; - } + if (initial_info_set == 0) { + mptcp_ci.mptcpci_init_rxbytes = mpte->mpte_init_rxbytes; + mptcp_ci.mptcpci_init_txbytes = mpte->mpte_init_txbytes; + } - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - if (mpts->mpts_connid == cid) - break; - } - if (mpts == NULL) { - error = ENXIO; - goto out; - } + if (mpte->mpte_flags & MPTE_FIRSTPARTY) + mptcp_ci.mptcpci_flags |= MPTCPCI_FIRSTPARTY; - if (rank == 0 || rank > 1) { - /* - * If rank is 0, determine whether this should be the - * primary or backup subflow, depending on what we have. - * - * Otherwise, if greater than 0, make it a backup flow. - */ - TAILQ_FOREACH(mpts1, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts1); - if (mpts1->mpts_flags & MPTSF_PREFERRED) { - MPTS_UNLOCK(mpts1); - break; + error = copyout(&mptcp_ci, aux_data, sizeof(mptcp_ci)); + if (error != 0) { + mptcplog((LOG_ERR, "%s copyout failed: %d\n", + __func__, error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return (error); } - MPTS_UNLOCK(mpts1); } - MPTS_LOCK(mpts); - mpts->mpts_flags &= ~MPTSF_PREFERRED; - mpts->mpts_rank = rank; - if (mpts1 != NULL && mpts != mpts1) { - /* preferred subflow found; set rank as necessary */ - if (rank == 0) - mpts->mpts_rank = (mpts1->mpts_rank + 1); - } else if (rank == 0) { - /* no preferred one found; promote this */ - rank = 1; - } - MPTS_UNLOCK(mpts); + return (0); } - if (rank == 1) { - /* - * If rank is 1, promote this subflow to be preferred. - */ - TAILQ_FOREACH(mpts1, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts1); - if (mpts1 != mpts && - (mpts1->mpts_flags & MPTSF_PREFERRED)) { - mpts1->mpts_flags &= ~MPTSF_PREFERRED; - if (mpte->mpte_nummpcapflows > 1) - mptcp_connorder_helper(mpts1); - } else if (mpts1 == mpts) { - mpts1->mpts_rank = 1; - if (mpts1->mpts_flags & MPTSF_MP_CAPABLE) { - mpts1->mpts_flags |= MPTSF_PREFERRED; - if (mpte->mpte_nummpcapflows > 1) - mptcp_connorder_helper(mpts1); - } - } - MPTS_UNLOCK(mpts1); - } + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + if (mpts->mpts_connid == *cid || *cid == SAE_CONNID_ANY) + break; } + if (mpts == NULL) + return ((*cid == SAE_CONNID_ANY) ? ENXIO : EINVAL); -out: - return (error); -} - -static void -mptcp_connorder_helper(struct mptsub *mpts) -{ - struct socket *so = mpts->mpts_socket; - struct tcpcb *tp = NULL; - - socket_lock(so, 0); + so = mpts->mpts_socket; + inp = sotoinpcb(so); - tp = intotcpcb(sotoinpcb(so)); - tp->t_mpflags |= TMPF_SND_MPPRIO; - if (mpts->mpts_flags & MPTSF_PREFERRED) - tp->t_mpflags &= ~TMPF_BACKUP_PATH; + if (inp->inp_vflag & INP_IPV4) + error = in_getconninfo(so, SAE_CONNID_ANY, flags, ifindex, + soerror, src, src_len, dst, dst_len, + aux_type, aux_data, aux_len); else - tp->t_mpflags |= TMPF_BACKUP_PATH; - - socket_unlock(so, 0); - -} - -/* - * Handle SIOCSGONNORDER - */ -int -mptcp_getconnorder(struct mptses *mpte, sae_connid_t cid, uint32_t *rank) -{ - struct mptsub *mpts; - int error = 0; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - VERIFY(rank != NULL); - *rank = 0; + error = in6_getconninfo(so, SAE_CONNID_ANY, flags, ifindex, + soerror, src, src_len, dst, dst_len, + aux_type, aux_data, aux_len); - if (cid == SAE_CONNID_ANY || cid == SAE_CONNID_ALL) { - error = EINVAL; - goto out; + if (error != 0) { + mptcplog((LOG_ERR, "%s error from in_getconninfo %d\n", + __func__, error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return (error); } - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - if (mpts->mpts_connid == cid) - break; - } - if (mpts == NULL) { - error = ENXIO; - goto out; - } + if (mpts->mpts_flags & MPTSF_MP_CAPABLE) + *flags |= CIF_MP_CAPABLE; + if (mpts->mpts_flags & MPTSF_MP_DEGRADED) + *flags |= CIF_MP_DEGRADED; + if (mpts->mpts_flags & MPTSF_MP_READY) + *flags |= CIF_MP_READY; + if (mpts->mpts_flags & MPTSF_ACTIVE) + *flags |= CIF_MP_ACTIVE; - MPTS_LOCK(mpts); - *rank = mpts->mpts_rank; - MPTS_UNLOCK(mpts); -out: - return (error); + mptcplog((LOG_DEBUG, "%s: cid %d flags %x \n", __func__, + mpts->mpts_connid, mpts->mpts_flags), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + return (0); } /* @@ -684,7 +593,7 @@ mptcp_usr_control(struct socket *mp_so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p) { #pragma unused(ifp, p) - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; int error = 0; @@ -695,7 +604,7 @@ mptcp_usr_control(struct socket *mp_so, u_long cmd, caddr_t data, mpte = mptompte(mpp); VERIFY(mpte != NULL); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ switch (cmd) { case SIOCGASSOCIDS32: { /* struct so_aidreq32 */ @@ -764,24 +673,6 @@ mptcp_usr_control(struct socket *mp_so, u_long cmd, caddr_t data, break; } - case SIOCSCONNORDER: { /* struct so_cordreq */ - struct so_cordreq cor; - bcopy(data, &cor, sizeof (cor)); - error = mptcp_setconnorder(mpte, cor.sco_cid, cor.sco_rank); - if (error == 0) - bcopy(&cor, data, sizeof (cor)); - break; - } - - case SIOCGCONNORDER: { /* struct so_cordreq */ - struct so_cordreq cor; - bcopy(data, &cor, sizeof (cor)); - error = mptcp_getconnorder(mpte, cor.sco_cid, &cor.sco_rank); - if (error == 0) - bcopy(&cor, data, sizeof (cor)); - break; - } - default: error = EOPNOTSUPP; break; @@ -790,131 +681,66 @@ out: return (error); } -/* - * Initiate a disconnect. MPTCP-level disconnection is specified by - * CONNID_{ANY,ALL}. Otherwise, selectively disconnect a subflow - * connection while keeping the MPTCP-level connection (association). - */ static int -mptcp_disconnectx(struct mptses *mpte, sae_associd_t aid, sae_connid_t cid) +mptcp_disconnect(struct mptses *mpte) { - struct mptsub *mpts; struct socket *mp_so; struct mptcb *mp_tp; int error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx aid %d cid %d %d\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), aid, cid, mp_so->so_error), + "%s: mp_so 0x%llx %d\n", __func__, + (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mp_so->so_error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - DTRACE_MPTCP5(disconnectx, struct mptses *, mpte, sae_associd_t, aid, - sae_connid_t, cid, struct socket *, mp_so, struct mptcb *, mp_tp); - - VERIFY(aid == SAE_ASSOCID_ANY || aid == SAE_ASSOCID_ALL || - aid == mpte->mpte_associd); - - /* terminate the association? */ - if (cid == SAE_CONNID_ANY || cid == SAE_CONNID_ALL) { - /* if we're not detached, go thru socket state checks */ - if (!(mp_so->so_flags & SOF_PCBCLEARING)) { - if (!(mp_so->so_state & (SS_ISCONNECTED| - SS_ISCONNECTING))) { - error = ENOTCONN; - goto out; - } - if (mp_so->so_state & SS_ISDISCONNECTING) { - error = EALREADY; - goto out; - } - } - MPT_LOCK(mp_tp); - mptcp_cancel_all_timers(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - (void) mptcp_close(mpte, mp_tp); - MPT_UNLOCK(mp_tp); - } else if ((mp_so->so_options & SO_LINGER) && - mp_so->so_linger == 0) { - (void) mptcp_drop(mpte, mp_tp, 0); - MPT_UNLOCK(mp_tp); - } else { - MPT_UNLOCK(mp_tp); - soisdisconnecting(mp_so); - sbflush(&mp_so->so_rcv); - if (mptcp_usrclosed(mpte) != NULL) - (void) mptcp_output(mpte); - } - } else { - bool disconnect_embryonic_subflows = false; - struct socket *so = NULL; - - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - if (mpts->mpts_connid != cid) - continue; - - MPTS_LOCK(mpts); - /* - * Check if disconnected subflow is the one used - * to initiate MPTCP connection. - * If it is and the connection is not yet join ready - * disconnect all other subflows. - */ - so = mpts->mpts_socket; - if (!(mp_tp->mpt_flags & MPTCPF_JOIN_READY) && - so && !(so->so_flags & SOF_MP_SEC_SUBFLOW)) { - disconnect_embryonic_subflows = true; - } + DTRACE_MPTCP3(disconnectx, struct mptses *, mpte, + struct socket *, mp_so, struct mptcb *, mp_tp); - mpts->mpts_flags |= MPTSF_USER_DISCONNECT; - mptcp_subflow_disconnect(mpte, mpts, FALSE); - MPTS_UNLOCK(mpts); - break; - } - - if (mpts == NULL) { - error = EINVAL; + /* if we're not detached, go thru socket state checks */ + if (!(mp_so->so_flags & SOF_PCBCLEARING)) { + if (!(mp_so->so_state & (SS_ISCONNECTED| + SS_ISCONNECTING))) { + error = ENOTCONN; goto out; } - - if (disconnect_embryonic_subflows) { - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - if (mpts->mpts_connid == cid) - continue; - MPTS_LOCK(mpts); - mptcp_subflow_disconnect(mpte, mpts, TRUE); - MPTS_UNLOCK(mpts); - } + if (mp_so->so_state & SS_ISDISCONNECTING) { + error = EALREADY; + goto out; } } - if (error == 0) - mptcp_thread_signal(mpte); - - if ((mp_so->so_state & (SS_CANTRCVMORE | SS_CANTSENDMORE)) == - (SS_CANTRCVMORE | SS_CANTSENDMORE)) { - /* the socket has been shutdown, no more sockopt's */ - mptcp_flush_sopts(mpte); + mptcp_cancel_all_timers(mp_tp); + if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { + mptcp_close(mpte, mp_tp); + } else if ((mp_so->so_options & SO_LINGER) && + mp_so->so_linger == 0) { + mptcp_drop(mpte, mp_tp, 0); + } else { + soisdisconnecting(mp_so); + sbflush(&mp_so->so_rcv); + if (mptcp_usrclosed(mpte) != NULL) + mptcp_output(mpte); } + if (error == 0) + mptcp_subflow_workloop(mpte); + out: return (error); } /* - * Wrapper function to support disconnect on socket + * Wrapper function to support disconnect on socket */ static int mptcp_usr_disconnect(struct socket *mp_so) { - int error = 0; - - error = mptcp_usr_disconnectx(mp_so, SAE_ASSOCID_ALL, SAE_CONNID_ALL); - return (error); + return (mptcp_disconnect(mpsotompte(mp_so))); } /* @@ -923,27 +749,13 @@ mptcp_usr_disconnect(struct socket *mp_so) static int mptcp_usr_disconnectx(struct socket *mp_so, sae_associd_t aid, sae_connid_t cid) { - struct mppcb *mpp = sotomppcb(mp_so); - struct mptses *mpte; - int error = 0; - - if (mpp == NULL || mpp->mpp_state == MPPCB_STATE_DEAD) { - error = EINVAL; - goto out; - } - mpte = mptompte(mpp); - VERIFY(mpte != NULL); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) + return (EINVAL); - if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL && - aid != mpte->mpte_associd) { - error = EINVAL; - goto out; - } + if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL) + return (EINVAL); - error = mptcp_disconnectx(mpte, aid, cid); -out: - return (error); + return (mptcp_usr_disconnect(mp_so)); } /* @@ -956,142 +768,29 @@ mptcp_usrclosed(struct mptses *mpte) struct mptcb *mp_tp; struct mptsub *mpts; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; - MPT_LOCK(mp_tp); mptcp_close_fsm(mp_tp, MPCE_CLOSE); if (mp_tp->mpt_state == MPTCPS_CLOSED) { mpte = mptcp_close(mpte, mp_tp); - MPT_UNLOCK(mp_tp); } else if (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_2) { - MPT_UNLOCK(mp_tp); soisdisconnected(mp_so); - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); - mpts->mpts_flags |= MPTSF_USER_DISCONNECT; - MPTS_UNLOCK(mpts); - } } else { - MPT_UNLOCK(mp_tp); - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); - mpts->mpts_flags |= MPTSF_USER_DISCONNECT; - mptcp_subflow_disconnect(mpte, mpts, FALSE); - MPTS_UNLOCK(mpts); + if ((mp_so->so_state & (SS_CANTRCVMORE|SS_CANTSENDMORE)) == + (SS_CANTRCVMORE | SS_CANTSENDMORE)) + mptcp_subflow_disconnect(mpte, mpts); + else + mptcp_subflow_shutdown(mpte, mpts); } } return (mpte); } -/* - * User-protocol pru_peeloff callback. - */ -static int -mptcp_usr_peeloff(struct socket *mp_so, sae_associd_t aid, struct socket **psop) -{ - struct mppcb *mpp = sotomppcb(mp_so); - struct mptses *mpte; - int error = 0; - - VERIFY(psop != NULL); - - if (mpp == NULL || mpp->mpp_state == MPPCB_STATE_DEAD) { - error = EINVAL; - goto out; - } - mpte = mptompte(mpp); - VERIFY(mpte != NULL); - - error = mptcp_peeloff(mpte, aid, psop); -out: - return (error); -} - -/* - * Transform a previously connected TCP subflow connection which has - * failed to negotiate MPTCP to its own socket which can be externalized - * with a file descriptor. Valid only when the MPTCP socket is not - * yet associated (MPTCP-level connection has not been established.) - */ -static int -mptcp_peeloff(struct mptses *mpte, sae_associd_t aid, struct socket **psop) -{ - struct socket *so = NULL, *mp_so; - struct mptsub *mpts; - int error = 0; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; - - VERIFY(psop != NULL); - *psop = NULL; - - DTRACE_MPTCP3(peeloff, struct mptses *, mpte, sae_associd_t, aid, - struct socket *, mp_so); - - /* peeloff cannot happen after an association is established */ - if (mpte->mpte_associd != SAE_ASSOCID_ANY) { - error = EINVAL; - goto out; - } - - if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) { - error = EINVAL; - goto out; - } - - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); - if (mpts->mpts_flags & MPTSF_MP_CAPABLE) { - panic("%s: so %p is MPTCP capable but mp_so %p " - "aid is %d\n", __func__, so, mp_so, - mpte->mpte_associd); - /* NOTREACHED */ - } - MPTS_ADDREF_LOCKED(mpts); /* for us */ - so = mpts->mpts_socket; - VERIFY(so != NULL); - /* - * This subflow socket is about to be externalized; make it - * appear as if it has the same properties as the MPTCP socket, - * undo what's done earlier in mptcp_subflow_add(). - */ - mptcp_subflow_sopeeloff(mpte, mpts, so); - MPTS_UNLOCK(mpts); - - mptcp_subflow_del(mpte, mpts, FALSE); - MPTS_REMREF(mpts); /* ours */ - /* - * XXX adi@apple.com - * - * Here we need to make sure the subflow socket is not - * flow controlled; need to clear both INP_FLOW_CONTROLLED - * and INP_FLOW_SUSPENDED on the subflow socket, since - * we will no longer be monitoring its events. - */ - break; - } - - if (so == NULL) { - error = EINVAL; - goto out; - } - *psop = so; - - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - -out: - return (error); -} - /* * After a receive, possible send some update to peer. */ @@ -1099,7 +798,7 @@ static int mptcp_usr_rcvd(struct socket *mp_so, int flags) { #pragma unused(flags) - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; int error = 0; @@ -1123,7 +822,7 @@ mptcp_usr_send(struct socket *mp_so, int prus_flags, struct mbuf *m, struct sockaddr *nam, struct mbuf *control, struct proc *p) { #pragma unused(nam, p) - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; int error = 0; @@ -1150,21 +849,16 @@ mptcp_usr_send(struct socket *mp_so, int prus_flags, struct mbuf *m, VERIFY(mpte != NULL); if (!(mp_so->so_state & SS_ISCONNECTED) && - (!(mp_so->so_flags1 & SOF1_PRECONNECT_DATA))) { + !(mp_so->so_flags1 & SOF1_PRECONNECT_DATA)) { error = ENOTCONN; goto out; } mptcp_insert_dsn(mpp, m); VERIFY(mp_so->so_snd.sb_flags & SB_NOCOMPRESS); - (void) sbappendstream(&mp_so->so_snd, m); + sbappendstream(&mp_so->so_snd, m); m = NULL; - /* - * XXX: adi@apple.com - * - * PRUS_MORETOCOME could be set, but we don't check it now. - */ error = mptcp_output(mpte); if (error != 0) goto out; @@ -1192,7 +886,7 @@ out: static int mptcp_usr_shutdown(struct socket *mp_so) { - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; int error = 0; @@ -1426,9 +1120,7 @@ out: if (control != NULL) m_freem(control); - /* clear SOF1_PRECONNECT_DATA after one write */ - if (mp_so->so_flags1 & SOF1_PRECONNECT_DATA) - mp_so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + soclearfastopen(mp_so); return (error); } @@ -1457,13 +1149,7 @@ mptcp_usr_socheckopt(struct socket *mp_so, struct sockopt *sopt) * * Need to consider the following cases: * - * a. In the event peeloff(2) occurs on the subflow socket, - * we may want to issue those options which are now - * handled at the MP socket. In that case, we will need - * to record them in mptcp_setopt() so that they can - * be replayed during peeloff. - * - * b. Certain socket options don't have a clear definition + * a. Certain socket options don't have a clear definition * on the expected behavior post connect(2). At the time * those options are issued on the MP socket, there may * be existing subflow sockets that are already connected. @@ -1489,6 +1175,12 @@ mptcp_usr_socheckopt(struct socket *mp_so, struct sockopt *sopt) case SO_DEFUNCTOK: /* MP */ case SO_ISDEFUNCT: /* MP */ case SO_TRAFFIC_CLASS_DBG: /* MP */ + case SO_DELEGATED: /* MP */ + case SO_DELEGATED_UUID: /* MP */ +#if NECP + case SO_NECP_ATTRIBUTES: + case SO_NECP_CLIENTUUID: +#endif /* NECP */ /* * Tell the caller that these options are to be processed. */ @@ -1504,9 +1196,9 @@ mptcp_usr_socheckopt(struct socket *mp_so, struct sockopt *sopt) case SO_RECV_ANYIF: /* MP + subflow */ case SO_RESTRICTIONS: /* MP + subflow */ case SO_FLUSH: /* MP + subflow */ - case SO_MPTCP_FASTJOIN: /* MP + subflow */ case SO_NOWAKEFROMSLEEP: case SO_NOAPNFALLBK: + case SO_MARK_CELLFALLBACK: /* * Tell the caller that these options are to be processed; * these will also be recorded later by mptcp_setopt(). @@ -1556,8 +1248,8 @@ mptcp_setopt_apply(struct mptses *mpte, struct mptopt *mpo) goto out; } - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); /* * Don't bother going further if there's no subflow; mark the option @@ -1581,33 +1273,25 @@ mptcp_setopt_apply(struct mptses *mpte, struct mptopt *mpo) TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { struct socket *so; - MPTS_LOCK(mpts); mpts->mpts_flags &= ~(MPTSF_SOPT_OLDVAL|MPTSF_SOPT_INPROG); mpts->mpts_oldintval = 0; smpo.mpo_intval = 0; VERIFY(mpts->mpts_socket != NULL); so = mpts->mpts_socket; - socket_lock(so, 0); if (mptcp_subflow_sogetopt(mpte, so, &smpo) == 0) { mpts->mpts_flags |= MPTSF_SOPT_OLDVAL; mpts->mpts_oldintval = smpo.mpo_intval; } - socket_unlock(so, 0); - MPTS_UNLOCK(mpts); } /* apply socket option */ TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { struct socket *so; - MPTS_LOCK(mpts); mpts->mpts_flags |= MPTSF_SOPT_INPROG; VERIFY(mpts->mpts_socket != NULL); so = mpts->mpts_socket; - socket_lock(so, 0); - error = mptcp_subflow_sosetopt(mpte, so, mpo); - socket_unlock(so, 0); - MPTS_UNLOCK(mpts); + error = mptcp_subflow_sosetopt(mpte, mpts, mpo); if (error != 0) break; } @@ -1616,32 +1300,26 @@ mptcp_setopt_apply(struct mptses *mpte, struct mptopt *mpo) TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { struct socket *so; - MPTS_LOCK(mpts); if (!(mpts->mpts_flags & MPTSF_SOPT_INPROG)) { /* clear in case it's set */ mpts->mpts_flags &= ~MPTSF_SOPT_OLDVAL; mpts->mpts_oldintval = 0; - MPTS_UNLOCK(mpts); continue; } if (!(mpts->mpts_flags & MPTSF_SOPT_OLDVAL)) { mpts->mpts_flags &= ~MPTSF_SOPT_INPROG; VERIFY(mpts->mpts_oldintval == 0); - MPTS_UNLOCK(mpts); continue; } /* error during sosetopt, so roll it back */ if (error != 0) { VERIFY(mpts->mpts_socket != NULL); so = mpts->mpts_socket; - socket_lock(so, 0); smpo.mpo_intval = mpts->mpts_oldintval; - (void) mptcp_subflow_sosetopt(mpte, so, &smpo); - socket_unlock(so, 0); + mptcp_subflow_sosetopt(mpte, mpts, &smpo); } mpts->mpts_oldintval = 0; mpts->mpts_flags &= ~(MPTSF_SOPT_OLDVAL|MPTSF_SOPT_INPROG); - MPTS_UNLOCK(mpts); } out: @@ -1654,18 +1332,17 @@ out: static int mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) { - int error = 0, optval, level, optname, rec = 1; + int error = 0, optval = 0, level, optname, rec = 1; struct mptopt smpo, *mpo = NULL; struct socket *mp_so; - char buf[32]; level = sopt->sopt_level; optname = sopt->sopt_name; VERIFY(sopt->sopt_dir == SOPT_SET); VERIFY(level == SOL_SOCKET || level == IPPROTO_TCP); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); /* * Record socket options which are applicable to subflow sockets so @@ -1684,16 +1361,46 @@ mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) case SO_RECV_ANYIF: case SO_RESTRICTIONS: case SO_NOWAKEFROMSLEEP: - case SO_MPTCP_FASTJOIN: case SO_NOAPNFALLBK: + case SO_MARK_CELLFALLBACK: /* record it */ break; case SO_FLUSH: /* don't record it */ rec = 0; break; + + /* Next ones, record at MPTCP-level */ +#if NECP + case SO_NECP_CLIENTUUID: + if (!uuid_is_null(mpsotomppcb(mp_so)->necp_client_uuid)) { + error = EINVAL; + goto out; + } + + error = sooptcopyin(sopt, &mpsotomppcb(mp_so)->necp_client_uuid, + sizeof(uuid_t), sizeof(uuid_t)); + if (error != 0) { + goto out; + } + + mpsotomppcb(mp_so)->necp_cb = mptcp_session_necp_cb; + error = necp_client_register_multipath_cb(mp_so->last_pid, + mpsotomppcb(mp_so)->necp_client_uuid, + mpsotomppcb(mp_so)); + if (error) + goto out; + + if (uuid_is_null(mpsotomppcb(mp_so)->necp_client_uuid)) { + error = EINVAL; + goto out; + } + + goto out; + case SO_NECP_ATTRIBUTES: +#endif /* NECP */ default: - /* nothing to do; just return success */ + /* nothing to do; just return */ goto out; } } else { @@ -1706,6 +1413,8 @@ mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) case TCP_CONNECTIONTIMEOUT: case TCP_RXT_CONNDROPTIME: case PERSIST_TIMEOUT: + case TCP_ADAPTIVE_READ_TIMEOUT: + case TCP_ADAPTIVE_WRITE_TIMEOUT: /* eligible; record it */ break; case TCP_NOTSENT_LOWAT: @@ -1727,6 +1436,27 @@ mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) optval); } } + goto out; + case MPTCP_SERVICE_TYPE: + /* record at MPTCP level */ + error = sooptcopyin(sopt, &optval, sizeof(optval), + sizeof(optval)); + if (error) + goto out; + if (optval < 0 || optval >= MPTCP_SVCTYPE_MAX) { + error = EINVAL; + goto out; + } + + mpte->mpte_svctype = optval; + + if (mptcp_entitlement_check(mp_so) < 0) { + error = EACCES; + goto out; + } + + mpte->mpte_flags |= MPTE_SVCTYPE_CHECKED; + goto out; default: /* not eligible */ @@ -1747,12 +1477,9 @@ mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) if (mpo == NULL) { error = ENOBUFS; } else { - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx sopt %s " - "val %d %s\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(level, optname, buf, - sizeof (buf)), optval, + mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx sopt %s val %d %s\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mptcp_sopt2str(level, optname), optval, (mpo->mpo_flags & MPOF_ATTACHED) ? "updated" : "recorded"), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); @@ -1791,19 +1518,16 @@ mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) } out: if (error == 0 && mpo != NULL) { - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: mp_so 0x%llx sopt %s val %d set %s\n", + mptcplog((LOG_INFO, "%s: mp_so 0x%llx sopt %s val %d set %s\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(level, optname, buf, - sizeof (buf)), optval, (mpo->mpo_flags & MPOF_INTERIM) ? + mptcp_sopt2str(level, optname), optval, + (mpo->mpo_flags & MPOF_INTERIM) ? "pending" : "successful"), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); } else if (error != 0) { - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: mp_so 0x%llx sopt %s can't be issued " - "error %d\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mptcp_sopt2str(level, - optname, buf, sizeof (buf)), error), + mptcplog((LOG_ERR, "%s: mp_so 0x%llx sopt %s (%d, %d) val %d can't be issued error %d\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mptcp_sopt2str(level, optname), level, optname, optval, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); } return (error); @@ -1818,7 +1542,7 @@ mptcp_getopt(struct mptses *mpte, struct sockopt *sopt) int error = 0, optval; VERIFY(sopt->sopt_dir == SOPT_GET); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ /* * We only handle SOPT_GET for TCP level socket options; we should @@ -1839,7 +1563,10 @@ mptcp_getopt(struct mptses *mpte, struct sockopt *sopt) case TCP_CONNECTIONTIMEOUT: case TCP_RXT_CONNDROPTIME: case PERSIST_TIMEOUT: + case TCP_ADAPTIVE_READ_TIMEOUT: + case TCP_ADAPTIVE_WRITE_TIMEOUT: case TCP_NOTSENT_LOWAT: + case MPTCP_SERVICE_TYPE: /* eligible; get the default value just in case */ error = mptcp_default_tcp_optval(mpte, sopt, &optval); break; @@ -1851,11 +1578,14 @@ mptcp_getopt(struct mptses *mpte, struct sockopt *sopt) switch (sopt->sopt_name) { case TCP_NOTSENT_LOWAT: - if (mpte->mpte_mppcb->mpp_socket->so_flags & SOF_NOTSENT_LOWAT) + if (mptetoso(mpte)->so_flags & SOF_NOTSENT_LOWAT) optval = mptcp_get_notsent_lowat(mpte); else optval = 0; goto out; + case MPTCP_SERVICE_TYPE: + optval = mpte->mpte_svctype; + goto out; } /* @@ -1890,7 +1620,7 @@ mptcp_default_tcp_optval(struct mptses *mpte, struct sockopt *sopt, int *optval) VERIFY(sopt->sopt_level == IPPROTO_TCP); VERIFY(sopt->sopt_dir == SOPT_GET); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ /* try to do what tcp_newtcpcb() does */ switch (sopt->sopt_name) { @@ -1901,6 +1631,9 @@ mptcp_default_tcp_optval(struct mptses *mpte, struct sockopt *sopt, int *optval) case TCP_CONNECTIONTIMEOUT: case TCP_RXT_CONNDROPTIME: case TCP_NOTSENT_LOWAT: + case TCP_ADAPTIVE_READ_TIMEOUT: + case TCP_ADAPTIVE_WRITE_TIMEOUT: + case MPTCP_SERVICE_TYPE: *optval = 0; break; @@ -1927,7 +1660,7 @@ mptcp_default_tcp_optval(struct mptses *mpte, struct sockopt *sopt, int *optval) int mptcp_ctloutput(struct socket *mp_so, struct sockopt *sopt) { - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; int error = 0; @@ -1936,16 +1669,14 @@ mptcp_ctloutput(struct socket *mp_so, struct sockopt *sopt) goto out; } mpte = mptompte(mpp); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ /* we only handle socket and TCP-level socket options for MPTCP */ if (sopt->sopt_level != SOL_SOCKET && sopt->sopt_level != IPPROTO_TCP) { - char buf[32]; mptcplog((LOG_DEBUG, "MPTCP Socket: " "%s: mp_so 0x%llx sopt %s level not " "handled\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(sopt->sopt_level, - sopt->sopt_name, buf, sizeof (buf))), + mptcp_sopt2str(sopt->sopt_level, sopt->sopt_name)), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); error = EINVAL; goto out; @@ -1964,185 +1695,153 @@ out: return (error); } -/* - * Return a string representation of - */ const char * -mptcp_sopt2str(int level, int optname, char *dst, int size) +mptcp_sopt2str(int level, int optname) { - char lbuf[32], obuf[32]; - const char *l = lbuf, *o = obuf; - - (void) snprintf(lbuf, sizeof (lbuf), "0x%x", level); - (void) snprintf(obuf, sizeof (obuf), "0x%x", optname); - switch (level) { case SOL_SOCKET: - l = "SOL_SOCKET"; switch (optname) { case SO_LINGER: - o = "SO_LINGER"; - break; + return ("SO_LINGER"); case SO_LINGER_SEC: - o = "SO_LINGER_SEC"; - break; + return ("SO_LINGER_SEC"); case SO_DEBUG: - o = "SO_DEBUG"; - break; + return ("SO_DEBUG"); case SO_KEEPALIVE: - o = "SO_KEEPALIVE"; - break; + return ("SO_KEEPALIVE"); case SO_USELOOPBACK: - o = "SO_USELOOPBACK"; - break; + return ("SO_USELOOPBACK"); case SO_TYPE: - o = "SO_TYPE"; - break; + return ("SO_TYPE"); case SO_NREAD: - o = "SO_NREAD"; - break; + return ("SO_NREAD"); case SO_NWRITE: - o = "SO_NWRITE"; - break; + return ("SO_NWRITE"); case SO_ERROR: - o = "SO_ERROR"; - break; + return ("SO_ERROR"); case SO_SNDBUF: - o = "SO_SNDBUF"; - break; + return ("SO_SNDBUF"); case SO_RCVBUF: - o = "SO_RCVBUF"; - break; + return ("SO_RCVBUF"); case SO_SNDLOWAT: - o = "SO_SNDLOWAT"; - break; + return ("SO_SNDLOWAT"); case SO_RCVLOWAT: - o = "SO_RCVLOWAT"; - break; + return ("SO_RCVLOWAT"); case SO_SNDTIMEO: - o = "SO_SNDTIMEO"; - break; + return ("SO_SNDTIMEO"); case SO_RCVTIMEO: - o = "SO_RCVTIMEO"; - break; + return ("SO_RCVTIMEO"); case SO_NKE: - o = "SO_NKE"; - break; + return ("SO_NKE"); case SO_NOSIGPIPE: - o = "SO_NOSIGPIPE"; - break; + return ("SO_NOSIGPIPE"); case SO_NOADDRERR: - o = "SO_NOADDRERR"; - break; + return ("SO_NOADDRERR"); case SO_RESTRICTIONS: - o = "SO_RESTRICTIONS"; - break; + return ("SO_RESTRICTIONS"); case SO_LABEL: - o = "SO_LABEL"; - break; + return ("SO_LABEL"); case SO_PEERLABEL: - o = "SO_PEERLABEL"; - break; + return ("SO_PEERLABEL"); case SO_RANDOMPORT: - o = "SO_RANDOMPORT"; - break; + return ("SO_RANDOMPORT"); case SO_TRAFFIC_CLASS: - o = "SO_TRAFFIC_CLASS"; - break; + return ("SO_TRAFFIC_CLASS"); case SO_RECV_TRAFFIC_CLASS: - o = "SO_RECV_TRAFFIC_CLASS"; - break; + return ("SO_RECV_TRAFFIC_CLASS"); case SO_TRAFFIC_CLASS_DBG: - o = "SO_TRAFFIC_CLASS_DBG"; - break; + return ("SO_TRAFFIC_CLASS_DBG"); case SO_PRIVILEGED_TRAFFIC_CLASS: - o = "SO_PRIVILEGED_TRAFFIC_CLASS"; - break; + return ("SO_PRIVILEGED_TRAFFIC_CLASS"); case SO_DEFUNCTOK: - o = "SO_DEFUNCTOK"; - break; + return ("SO_DEFUNCTOK"); case SO_ISDEFUNCT: - o = "SO_ISDEFUNCT"; - break; + return ("SO_ISDEFUNCT"); case SO_OPPORTUNISTIC: - o = "SO_OPPORTUNISTIC"; - break; + return ("SO_OPPORTUNISTIC"); case SO_FLUSH: - o = "SO_FLUSH"; - break; + return ("SO_FLUSH"); case SO_RECV_ANYIF: - o = "SO_RECV_ANYIF"; - break; + return ("SO_RECV_ANYIF"); case SO_NOWAKEFROMSLEEP: - o = "SO_NOWAKEFROMSLEEP"; - break; - case SO_MPTCP_FASTJOIN: - o = "SO_MPTCP_FASTJOIN"; - break; + return ("SO_NOWAKEFROMSLEEP"); case SO_NOAPNFALLBK: - o = "SO_NOAPNFALLBK"; - break; + return ("SO_NOAPNFALLBK"); + case SO_MARK_CELLFALLBACK: + return ("SO_CELLFALLBACK"); + case SO_DELEGATED: + return ("SO_DELEGATED"); + case SO_DELEGATED_UUID: + return ("SO_DELEGATED_UUID"); +#if NECP + case SO_NECP_ATTRIBUTES: + return ("SO_NECP_ATTRIBUTES"); + case SO_NECP_CLIENTUUID: + return ("SO_NECP_CLIENTUUID"); +#endif /* NECP */ } + break; case IPPROTO_TCP: - l = "IPPROTO_TCP"; switch (optname) { + case TCP_NODELAY: + return ("TCP_NODELAY"); case TCP_KEEPALIVE: - o = "TCP_KEEPALIVE"; - break; + return ("TCP_KEEPALIVE"); case TCP_KEEPINTVL: - o = "TCP_KEEPINTVL"; - break; + return ("TCP_KEEPINTVL"); case TCP_KEEPCNT: - o = "TCP_KEEPCNT"; - break; + return ("TCP_KEEPCNT"); case TCP_CONNECTIONTIMEOUT: - o = "TCP_CONNECTIONTIMEOUT"; - break; + return ("TCP_CONNECTIONTIMEOUT"); case TCP_RXT_CONNDROPTIME: - o = "TCP_RXT_CONNDROPTIME"; - break; + return ("TCP_RXT_CONNDROPTIME"); case PERSIST_TIMEOUT: - o = "PERSIST_TIMEOUT"; - break; + return ("PERSIST_TIMEOUT"); + case TCP_NOTSENT_LOWAT: + return ("NOTSENT_LOWAT"); + case TCP_ADAPTIVE_READ_TIMEOUT: + return ("ADAPTIVE_READ_TIMEOUT"); + case TCP_ADAPTIVE_WRITE_TIMEOUT: + return ("ADAPTIVE_WRITE_TIMEOUT"); + case MPTCP_SERVICE_TYPE: + return ("MPTCP_SERVICE_TYPE"); } + break; } - (void) snprintf(dst, size, "<%s,%s>", l, o); - return (dst); + return ("unknown"); } static int mptcp_usr_preconnect(struct socket *mp_so) { struct mptsub *mpts = NULL; - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; struct socket *so; struct tcpcb *tp = NULL; + int error; mpte = mptompte(mpp); VERIFY(mpte != NULL); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ mpts = mptcp_get_subflow(mpte, NULL, NULL); if (mpts == NULL) { - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: mp_so 0x%llx invalid preconnect ", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + mptcplog((LOG_ERR, "%s: mp_so 0x%llx invalid preconnect ", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); return (EINVAL); } - MPTS_LOCK(mpts); mpts->mpts_flags &= ~MPTSF_TFO_REQD; so = mpts->mpts_socket; - socket_lock(so, 0); tp = intotcpcb(sotoinpcb(so)); tp->t_mpflags &= ~TMPF_TFO_REQUEST; - int error = tcp_output(sototcpcb(so)); - socket_unlock(so, 0); - MPTS_UNLOCK(mpts); - mp_so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + error = tcp_output(sototcpcb(so)); + + soclearfastopen(mp_so); + return (error); } diff --git a/bsd/netinet/mptcp_var.h b/bsd/netinet/mptcp_var.h index 46e47dc96..2d1f99061 100644 --- a/bsd/netinet/mptcp_var.h +++ b/bsd/netinet/mptcp_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,13 +40,20 @@ #include #include #include +#include + +struct mpt_itf_info { + uint32_t ifindex; + uint32_t has_v4_conn:1, + has_v6_conn:1, + no_mptcp_support:1; +}; /* * MPTCP Session * * This is an extension to the multipath PCB specific for MPTCP, protected by - * the per-PCB mpp_lock (also the socket's lock); MPTCP thread signalling uses - * its own mpte_thread_lock due to lock ordering constraints. + * the per-PCB mpp_lock (also the socket's lock); */ struct mptses { struct mppcb *mpte_mppcb; /* back ptr to multipath PCB */ @@ -57,45 +64,170 @@ struct mptses { uint16_t mpte_nummpcapflows; /* # of MP_CAP subflows */ sae_associd_t mpte_associd; /* MPTCP association ID */ sae_connid_t mpte_connid_last; /* last used connection ID */ - /* - * Threading (protected by mpte_thread_lock) - */ - decl_lck_mtx_data(, mpte_thread_lock); /* thread lock */ - struct thread *mpte_thread; /* worker thread */ - uint32_t mpte_thread_active; /* thread is running */ - uint32_t mpte_thread_reqs; /* # of requests for thread */ + + union { + /* Source address of initial subflow */ + struct sockaddr mpte_src; + struct sockaddr_in __mpte_src_v4; + struct sockaddr_in6 __mpte_src_v6; + }; + + union { + /* Destination address of initial subflow */ + struct sockaddr mpte_dst; + struct sockaddr_in __mpte_dst_v4; + struct sockaddr_in6 __mpte_dst_v6; + }; + struct mptsub *mpte_active_sub; /* ptr to last active subf */ uint8_t mpte_flags; /* per mptcp session flags */ +#define MPTE_SND_REM_ADDR 0x01 /* Send Remove_addr option */ +#define MPTE_SVCTYPE_CHECKED 0x02 /* Did entitlement-check for service-type */ +#define MPTE_FIRSTPARTY 0x04 /* First-party app used multipath_extended entitlement */ +#define MPTE_ACCESS_GRANTED 0x08 /* Access to cellular has been granted for this connection */ + uint8_t mpte_svctype; /* MPTCP Service type */ uint8_t mpte_lost_aid; /* storing lost address id */ uint8_t mpte_addrid_last; /* storing address id parm */ + +#define MPTE_ITFINFO_SIZE 4 + uint32_t mpte_itfinfo_size; + struct mpt_itf_info _mpte_itfinfo[MPTE_ITFINFO_SIZE]; + struct mpt_itf_info *mpte_itfinfo; + + struct mbuf *mpte_reinjectq; + + /* The below is used for stats */ + uint32_t mpte_subflow_switches; /* Number of subflow-switches in sending */ + uint32_t mpte_used_cell:1, + mpte_used_wifi:1, + mpte_initial_cell:1, + mpte_handshake_success:1; + + struct mptcp_itf_stats mpte_itfstats[MPTCP_ITFSTATS_SIZE]; + uint64_t mpte_init_txbytes __attribute__((aligned(8))); + uint64_t mpte_init_rxbytes __attribute__((aligned(8))); }; -/* - * Valid values for mpte_flags. - */ -#define MPTE_SND_REM_ADDR 0x01 /* Send Remove_addr option */ +static inline struct socket * +mptetoso(struct mptses *mpte) +{ + return (mpte->mpte_mppcb->mpp_socket); +} + +static inline struct mptses * +mptompte(struct mppcb *mp) +{ + return ((struct mptses *)mp->mpp_pcbe); +} -#define mptompte(mp) ((struct mptses *)(mp)->mpp_pcbe) +static inline struct mptses * +mpsotompte(struct socket *so) +{ + return (mptompte(mpsotomppcb(so))); +} -#define MPTE_LOCK_ASSERT_HELD(_mpte) \ - lck_mtx_assert(&(_mpte)->mpte_mppcb->mpp_lock, LCK_MTX_ASSERT_OWNED) +static inline void +mpp_lock_assert_held(struct mppcb *mp) +{ +#if !MACH_ASSERT +#pragma unused(mp) +#endif + LCK_MTX_ASSERT(&mp->mpp_lock, LCK_MTX_ASSERT_OWNED); +} -#define MPTE_LOCK_ASSERT_NOTHELD(_mpte) \ - lck_mtx_assert(&(_mpte)->mpte_mppcb->mpp_lock, LCK_MTX_ASSERT_NOTOWNED) +static inline void +mpp_lock_assert_notheld(struct mppcb *mp) +{ +#if !MACH_ASSERT +#pragma unused(mp) +#endif + LCK_MTX_ASSERT(&mp->mpp_lock, LCK_MTX_ASSERT_NOTOWNED); +} -#define MPTE_LOCK(_mpte) \ - lck_mtx_lock(&(_mpte)->mpte_mppcb->mpp_lock) +static inline boolean_t +mpp_try_lock(struct mppcb *mp) +{ + if (!lck_mtx_try_lock(&mp->mpp_lock)) + return false; -#define MPTE_LOCK_SPIN(_mpte) \ - lck_mtx_lock_spin(&(_mpte)->mpte_mppcb->mpp_lock) + VERIFY(!(mp->mpp_flags & MPP_INSIDE_OUTPUT)); + VERIFY(!(mp->mpp_flags & MPP_INSIDE_INPUT)); -#define MPTE_CONVERT_LOCK(_mpte) do { \ - MPTE_LOCK_ASSERT_HELD(_mpte); \ - lck_mtx_convert_spin(&(_mpte)->mpte_mppcb->mpp_lock); \ -} while (0) + return true; +} + +static inline void +mpp_lock(struct mppcb *mp) +{ + lck_mtx_lock(&mp->mpp_lock); + VERIFY(!(mp->mpp_flags & MPP_INSIDE_OUTPUT)); + VERIFY(!(mp->mpp_flags & MPP_INSIDE_INPUT)); +} + +static inline void +mpp_unlock(struct mppcb *mp) +{ + VERIFY(!(mp->mpp_flags & MPP_INSIDE_OUTPUT)); + VERIFY(!(mp->mpp_flags & MPP_INSIDE_INPUT)); + lck_mtx_unlock(&mp->mpp_lock); +} + +static inline lck_mtx_t * +mpp_getlock(struct mppcb *mp, int flags) +{ + if (flags & PR_F_WILLUNLOCK) { + VERIFY(!(mp->mpp_flags & MPP_INSIDE_OUTPUT)); + VERIFY(!(mp->mpp_flags & MPP_INSIDE_INPUT)); + } + + return (&mp->mpp_lock); +} + +static inline void +mpte_lock_assert_held(struct mptses *mpte) +{ + mpp_lock_assert_held(mpte->mpte_mppcb); +} + +static inline void +mpte_lock_assert_notheld(struct mptses *mpte) +{ + mpp_lock_assert_notheld(mpte->mpte_mppcb); +} + +static inline boolean_t +mpte_try_lock(struct mptses *mpte) +{ + return (mpp_try_lock(mpte->mpte_mppcb)); +} + +static inline void +mpte_lock(struct mptses *mpte) +{ + mpp_lock(mpte->mpte_mppcb); +} + +static inline void +mpte_unlock(struct mptses *mpte) +{ + mpp_unlock(mpte->mpte_mppcb); +} + +static inline lck_mtx_t * +mpte_getlock(struct mptses *mpte, int flags) +{ + return mpp_getlock(mpte->mpte_mppcb, flags); +} + +static inline int +mptcp_subflow_cwnd_space(struct socket *so) +{ + struct tcpcb *tp = sototcpcb(so); + int cwnd = min(tp->snd_wnd, tp->snd_cwnd) - (tp->snd_nxt - tp->snd_una); + + return (min(cwnd, sbspace(&so->so_snd))); +} -#define MPTE_UNLOCK(_mpte) \ - lck_mtx_unlock(&(_mpte)->mpte_mppcb->mpp_lock) /* * MPTCP socket options @@ -112,56 +244,35 @@ struct mptopt { #define MPOF_SUBFLOW_OK 0x2 /* can be issued on subflow socket */ #define MPOF_INTERIM 0x4 /* has not been issued on any subflow */ -/* - * Structure passed down to TCP during subflow connection establishment - * containing information pertaining to the MPTCP. - */ -struct mptsub_connreq { - uint32_t mpcr_type; /* see MPTSUB_CONNREQ_* below */ - uint32_t mpcr_ifscope; /* ifscope parameter to connectx(2) */ - struct proc *mpcr_proc; /* process issuing connectx(2) */ -}; - -/* valid values for mpcr_type */ -#define MPTSUB_CONNREQ_MP_ENABLE 1 /* enable MPTCP */ -#define MPTSUB_CONNREQ_MP_ADD 2 /* join an existing MPTCP */ - /* * MPTCP subflow * - * Protected by the the per-subflow mpts_lock. Note that mpts_flags - * and mpts_evctl are modified via atomic operations. + * Note that mpts_flags and mpts_evctl are modified via atomic operations. */ struct mptsub { - decl_lck_mtx_data(, mpts_lock); /* per-subflow lock */ TAILQ_ENTRY(mptsub) mpts_entry; /* glue to peer subflows */ uint32_t mpts_refcnt; /* reference count */ uint32_t mpts_flags; /* see flags below */ uint32_t mpts_evctl; /* subflow control events */ - uint32_t mpts_family; /* address family */ sae_connid_t mpts_connid; /* subflow connection ID */ int mpts_oldintval; /* sopt_val before sosetopt */ - uint32_t mpts_rank; /* subflow priority/rank */ - int32_t mpts_soerror; /* most recent subflow error */ struct mptses *mpts_mpte; /* back ptr to MPTCP session */ struct socket *mpts_socket; /* subflow socket */ struct sockaddr *mpts_src; /* source address */ - struct sockaddr *mpts_dst; /* destination address */ - struct ifnet *mpts_outif; /* outbound interface */ - u_int64_t mpts_sndnxt; /* next byte to send in mp so */ + + union { + /* destination address */ + struct sockaddr mpts_dst; + struct sockaddr_in __mpts_dst_v4; + struct sockaddr_in6 __mpts_dst_v6; + }; + u_int32_t mpts_rel_seq; /* running count of subflow # */ - struct protosw *mpts_oprotosw; /* original protosw */ - struct mptsub_connreq mpts_mpcr; /* connection request */ - int32_t mpts_srtt; /* tcp's rtt estimate */ - int32_t mpts_rxtcur; /* tcp's rto estimate */ + u_int32_t mpts_iss; /* Initial sequence number, taking TFO into account */ + u_int32_t mpts_ifscope; /* scoped to the interface */ uint32_t mpts_probesoon; /* send probe after probeto */ uint32_t mpts_probecnt; /* number of probes sent */ uint32_t mpts_maxseg; /* cached value of t_maxseg */ - uint32_t mpts_peerswitch;/* no of uses of backup so */ -#define MPTSL_WIRED 0x01 -#define MPTSL_WIFI 0x02 -#define MPTSL_CELL 0x04 - uint32_t mpts_linktype; /* wired, wifi, cell */ }; /* @@ -193,58 +304,36 @@ struct mptsub { * * Keep in sync with bsd/dev/dtrace/scripts/mptcp.d. */ -#define MPTSF_ATTACHED 0x1 /* attached to MPTCP PCB */ -#define MPTSF_CONNECTING 0x2 /* connection was attempted */ -#define MPTSF_CONNECT_PENDING 0x4 /* will connect when MPTCP is ready */ -#define MPTSF_CONNECTED 0x8 /* connection is established */ -#define MPTSF_DISCONNECTING 0x10 /* disconnection was attempted */ -#define MPTSF_DISCONNECTED 0x20 /* has been disconnected */ -#define MPTSF_MP_CAPABLE 0x40 /* connected as a MPTCP subflow */ -#define MPTSF_MP_READY 0x80 /* MPTCP has been confirmed */ -#define MPTSF_MP_DEGRADED 0x100 /* has lost its MPTCP capabilities */ -#define MPTSF_SUSPENDED 0x200 /* write-side is flow controlled */ -#define MPTSF_BOUND_IF 0x400 /* subflow bound to an interface */ -#define MPTSF_BOUND_IP 0x800 /* subflow bound to a src address */ -#define MPTSF_BOUND_PORT 0x1000 /* subflow bound to a src port */ -#define MPTSF_PREFERRED 0x2000 /* primary/preferred subflow */ -#define MPTSF_SOPT_OLDVAL 0x4000 /* old option value is valid */ -#define MPTSF_SOPT_INPROG 0x8000 /* sosetopt in progress */ -#define MPTSF_DELETEOK 0x10000 /* subflow can be deleted */ -#define MPTSF_FAILINGOVER 0x20000 /* subflow not used for output */ -#define MPTSF_ACTIVE 0x40000 /* subflow currently in use */ -#define MPTSF_MPCAP_CTRSET 0x80000 /* mpcap counter */ -#define MPTSF_FASTJ_SEND 0x100000 /* send data after SYN in MP_JOIN */ -#define MPTSF_FASTJ_REQD 0x200000 /* fastjoin required */ -#define MPTSF_USER_DISCONNECT 0x400000 /* User triggered disconnect */ -#define MPTSF_TFO_REQD 0x800000 /* TFO requested */ +#define MPTSF_ATTACHED 0x00000001 /* attached to MPTCP PCB */ +#define MPTSF_CONNECTING 0x00000002 /* connection was attempted */ +#define MPTSF_CONNECT_PENDING 0x00000004 /* will connect when MPTCP is ready */ +#define MPTSF_CONNECTED 0x00000008 /* connection is established */ +#define MPTSF_DISCONNECTING 0x00000010 /* disconnection was attempted */ +#define MPTSF_DISCONNECTED 0x00000020 /* has been disconnected */ +#define MPTSF_MP_CAPABLE 0x00000040 /* connected as a MPTCP subflow */ +#define MPTSF_MP_READY 0x00000080 /* MPTCP has been confirmed */ +#define MPTSF_MP_DEGRADED 0x00000100 /* has lost its MPTCP capabilities */ +#define MPTSF_PREFERRED 0x00000200 /* primary/preferred subflow */ +#define MPTSF_SOPT_OLDVAL 0x00000400 /* old option value is valid */ +#define MPTSF_SOPT_INPROG 0x00000800 /* sosetopt in progress */ +#define MPTSF_FAILINGOVER 0x00001000 /* subflow not used for output */ +#define MPTSF_ACTIVE 0x00002000 /* subflow currently in use */ +#define MPTSF_MPCAP_CTRSET 0x00004000 /* mpcap counter */ +#define MPTSF_CLOSED 0x00008000 /* soclose_locked has been called on this subflow */ +#define MPTSF_TFO_REQD 0x00010000 /* TFO requested */ +#define MPTSF_CLOSE_REQD 0x00020000 /* A close has been requested from NECP */ +#define MPTSF_INITIAL_SUB 0x00040000 /* This is the initial subflow */ +#define MPTSF_READ_STALL 0x00080000 /* A read-stall has been detected */ +#define MPTSF_WRITE_STALL 0x00100000 /* A write-stall has been detected */ +#define MPTSF_CONFIRMED 0x00200000 /* Subflow confirmed to be MPTCP-capable */ #define MPTSF_BITS \ "\020\1ATTACHED\2CONNECTING\3PENDING\4CONNECTED\5DISCONNECTING" \ - "\6DISCONNECTED\7MP_CAPABLE\10MP_READY\11MP_DEGRADED\12SUSPENDED" \ - "\13BOUND_IF\14BOUND_IP\15BOUND_PORT\16PREFERRED\17SOPT_OLDVAL" \ - "\20SOPT_INPROG\21NOLINGER\22FAILINGOVER\23ACTIVE\24MPCAP_CTRSET" \ - "\25FASTJ_SEND\26FASTJ_REQD\27USER_DISCONNECT" - -#define MPTS_LOCK_ASSERT_HELD(_mpts) \ - lck_mtx_assert(&(_mpts)->mpts_lock, LCK_MTX_ASSERT_OWNED) - -#define MPTS_LOCK_ASSERT_NOTHELD(_mpts) \ - lck_mtx_assert(&(_mpts)->mpts_lock, LCK_MTX_ASSERT_NOTOWNED) - -#define MPTS_LOCK(_mpts) \ - lck_mtx_lock(&(_mpts)->mpts_lock) - -#define MPTS_UNLOCK(_mpts) \ - lck_mtx_unlock(&(_mpts)->mpts_lock) - -#define MPTS_ADDREF(_mpts) \ - mptcp_subflow_addref(_mpts, 0) - -#define MPTS_ADDREF_LOCKED(_mpts) \ - mptcp_subflow_addref(_mpts, 1) - -#define MPTS_REMREF(_mpts) \ - mptcp_subflow_remref(_mpts) + "\6DISCONNECTED\7MP_CAPABLE\10MP_READY\11MP_DEGRADED" \ + "\12PREFERRED\13SOPT_OLDVAL" \ + "\14SOPT_INPROG\15FAILINGOVER\16ACTIVE\17MPCAP_CTRSET" \ + "\20CLOSED\21TFO_REQD\22CLOSEREQD\23INITIALSUB\24READ_STALL" \ + "\25WRITE_STALL\26CONFIRMED" /* * MPTCP states @@ -284,17 +373,15 @@ struct mptcp_subf_auth_entry { * Keep in sync with bsd/dev/dtrace/scripts/mptcp.d. */ struct mptcb { - decl_lck_mtx_data(, mpt_lock); /* per MPTCP PCB lock */ struct mptses *mpt_mpte; /* back ptr to MPTCP session */ mptcp_state_t mpt_state; /* MPTCP state */ u_int32_t mpt_flags; /* see flags below */ - u_int32_t mpt_refcnt; /* references held on mptcb */ u_int32_t mpt_version; /* MPTCP proto version */ int mpt_softerror; /* error not yet reported */ /* * Authentication and metadata invariants */ - mptcp_key_t *mpt_localkey; /* in network byte order */ + mptcp_key_t mpt_localkey; /* in network byte order */ mptcp_key_t mpt_remotekey; /* in network byte order */ mptcp_token_t mpt_localtoken; /* HMAC SHA1 of local key */ mptcp_token_t mpt_remotetoken; /* HMAC SHA1 of remote key */ @@ -316,11 +403,12 @@ struct mptcb { u_int64_t mpt_sndmax; /* DSN of max byte sent */ u_int64_t mpt_local_idsn; /* First byte's DSN */ u_int32_t mpt_sndwnd; + u_int64_t mpt_sndwl1; + u_int64_t mpt_sndwl2; /* * Receiving side */ u_int64_t mpt_rcvnxt; /* Next expected DSN */ - u_int64_t mpt_rcvatmark; /* mpsocket marker of rcvnxt */ u_int64_t mpt_remote_idsn; /* Peer's IDSN */ u_int32_t mpt_rcvwnd; LIST_HEAD(, mptcp_subf_auth_entry) mpt_subauth_list; /* address IDs */ @@ -338,51 +426,33 @@ struct mptcb { u_int32_t mpt_notsent_lowat; /* TCP_NOTSENT_LOWAT support */ u_int32_t mpt_peer_version; /* Version from peer */ + + struct tsegqe_head mpt_segq; + u_int16_t mpt_reassqlen; /* length of reassembly queue */ }; /* valid values for mpt_flags (see also notes on mpts_flags above) */ -#define MPTCPF_CHECKSUM 0x1 /* checksum DSS option */ -#define MPTCPF_FALLBACK_TO_TCP 0x2 /* Fallback to TCP */ -#define MPTCPF_JOIN_READY 0x4 /* Ready to start 2 or more subflows */ -#define MPTCPF_RECVD_MPFAIL 0x8 /* Received MP_FAIL option */ -#define MPTCPF_PEEL_OFF 0x10 /* Peel off this socket */ -#define MPTCPF_SND_64BITDSN 0x20 /* Send full 64-bit DSN */ -#define MPTCPF_SND_64BITACK 0x40 /* Send 64-bit ACK response */ -#define MPTCPF_RCVD_64BITACK 0x80 /* Received 64-bit Data ACK */ -#define MPTCPF_POST_FALLBACK_SYNC 0x100 /* Post fallback resend data */ -#define MPTCPF_FALLBACK_HEURISTIC 0x200 /* Send SYN without MP_CAPABLE due to heuristic */ -#define MPTCPF_HEURISTIC_TRAC 0x400 /* Tracked this connection in the heuristics as a failure */ +#define MPTCPF_CHECKSUM 0x001 /* checksum DSS option */ +#define MPTCPF_FALLBACK_TO_TCP 0x002 /* Fallback to TCP */ +#define MPTCPF_JOIN_READY 0x004 /* Ready to start 2 or more subflows */ +#define MPTCPF_RECVD_MPFAIL 0x008 /* Received MP_FAIL option */ +#define MPTCPF_SND_64BITDSN 0x010 /* Send full 64-bit DSN */ +#define MPTCPF_SND_64BITACK 0x020 /* Send 64-bit ACK response */ +#define MPTCPF_RCVD_64BITACK 0x040 /* Received 64-bit Data ACK */ +#define MPTCPF_POST_FALLBACK_SYNC 0x080 /* Post fallback resend data */ +#define MPTCPF_FALLBACK_HEURISTIC 0x100 /* Send SYN without MP_CAPABLE due to heuristic */ +#define MPTCPF_HEURISTIC_TRAC 0x200 /* Tracked this connection in the heuristics as a failure */ +#define MPTCPF_REASS_INPROG 0x400 /* Reassembly is in progress */ #define MPTCPF_BITS \ - "\020\1CHECKSUM\2FALLBACK_TO_TCP\3JOIN_READY\4RECVD_MPFAIL\5PEEL_OFF" \ - "\6SND_64BITDSN\7SND_64BITACK\10RCVD_64BITACK\11POST_FALLBACK_SYNC" \ - "\12FALLBACK_HEURISTIC\13HEURISTIC_TRAC" + "\020\1CHECKSUM\2FALLBACK_TO_TCP\3JOIN_READY\4RECVD_MPFAIL" \ + "\5SND_64BITDSN\6SND_64BITACK\7RCVD_64BITACK\10POST_FALLBACK_SYNC" \ + "\11FALLBACK_HEURISTIC\12HEURISTIC_TRAC\13REASS_INPROG" /* valid values for mpt_timer_vals */ #define MPTT_REXMT 0x01 /* Starting Retransmit Timer */ #define MPTT_TW 0x02 /* Starting Timewait Timer */ #define MPTT_FASTCLOSE 0x04 /* Starting Fastclose wait timer */ -//#define MPTT_PROBE_TIMER 0x08 /* Timer for probing preferred path */ - -#define MPT_LOCK_ASSERT_HELD(_mpt) \ - lck_mtx_assert(&(_mpt)->mpt_lock, LCK_MTX_ASSERT_OWNED) - -#define MPT_LOCK_ASSERT_NOTHELD(_mpt) \ - lck_mtx_assert(&(_mpt)->mpt_lock, LCK_MTX_ASSERT_NOTOWNED) - -#define MPT_LOCK(_mpt) \ - lck_mtx_lock(&(_mpt)->mpt_lock) - -#define MPT_LOCK_SPIN(_mpt) \ - lck_mtx_lock_spin(&(_mpt)->mpt_lock) - -#define MPT_CONVERT_LOCK(_mpt) do { \ - MPT_LOCK_ASSERT_HELD(_mpt); \ - lck_mtx_convert_spin(&(_mpt)->mpt_lock); \ -} while (0) - -#define MPT_UNLOCK(_mpt) \ - lck_mtx_unlock(&(_mpt)->mpt_lock) /* events for close FSM */ #define MPCE_CLOSE 0x1 @@ -390,7 +460,10 @@ struct mptcb { #define MPCE_RECV_DATA_FIN 0x4 /* mptcb manipulation */ -#define tptomptp(tp) ((struct mptcb *)((tp)->t_mptcb)) +static inline struct mptcb *tptomptp(struct tcpcb *tp) +{ + return (tp->t_mptcb); +} /* * MPTCP control block and state structures are allocated along with @@ -410,41 +483,13 @@ extern struct mppcbinfo mtcbinfo; extern struct pr_usrreqs mptcp_usrreqs; /* Encryption algorithm related definitions */ -#define MPTCP_SHA1_RESULTLEN 20 #define SHA1_TRUNCATED 8 -/* List of valid keys to use for MPTCP connections */ -#define MPTCP_KEY_DIGEST_LEN (MPTCP_SHA1_RESULTLEN) -#define MPTCP_MX_KEY_ALLOCS (256) -#define MPTCP_KEY_PREALLOCS_MX (16) -#define MPTCP_MX_PREALLOC_ZONE_SZ (8192) - -struct mptcp_key_entry { - LIST_ENTRY(mptcp_key_entry) mkey_next; - mptcp_key_t mkey_value; -#define MKEYF_FREE 0x0 -#define MKEYF_INUSE 0x1 - u_int32_t mkey_flags; - char mkey_digest[MPTCP_KEY_DIGEST_LEN]; -}; - -/* structure for managing unique key list */ -struct mptcp_keys_pool_head { - struct mptcp_key_entry *lh_first; /* list of keys */ - u_int32_t mkph_count; /* total keys in pool */ - vm_size_t mkph_key_elm_sz; /* size of key entry */ - struct zone *mkph_key_entry_zone; /* zone for key entry */ - decl_lck_mtx_data(, mkph_lock); /* lock for key list */ -}; - -/* MPTCP Receive Window */ -#define MPTCP_RWIN_MAX (1<<16) - /* MPTCP Debugging Levels */ #define MPTCP_LOGLVL_NONE 0x0 /* No debug logging */ #define MPTCP_LOGLVL_ERR 0x1 /* Errors in execution are logged */ #define MPTCP_LOGLVL_LOG 0x2 /* Important logs */ -#define MPTCP_LOGLVL_VERBOSE 0x3 /* Verbose logs */ +#define MPTCP_LOGLVL_VERBOSE 0x4 /* Verbose logs */ /* MPTCP sub-components for debug logging */ #define MPTCP_NO_DBG 0x00 /* No areas are logged */ @@ -453,8 +498,6 @@ struct mptcp_keys_pool_head { #define MPTCP_SENDER_DBG 0x04 /* Sender side logging */ #define MPTCP_RECEIVER_DBG 0x08 /* Receiver logging */ #define MPTCP_EVENTS_DBG 0x10 /* Subflow events logging */ -#define MPTCP_ALL_DBG (MPTCP_STATE_DBG | MPTCP_SOCKET_DBG | \ - MPTCP_SENDER_DBG | MPTCP_RECEIVER_DBG | MPTCP_EVENTS_DBG) /* Mask to obtain 32-bit portion of data sequence number */ #define MPTCP_DATASEQ_LOW32_MASK (0xffffffff) @@ -510,8 +553,7 @@ struct mptcp_keys_pool_head { } #define mptcplog(x, y, z) do { \ - if ((mptcp_dbg_area & y) && \ - (mptcp_dbg_level >= z)) \ + if ((mptcp_dbg_area & y) && (mptcp_dbg_level & z)) \ log x; \ } while (0) @@ -521,54 +563,47 @@ extern int mptcp_join_retries; /* Multipath TCP Join retries */ extern int mptcp_dss_csum; /* Multipath DSS Option checksum */ extern int mptcp_fail_thresh; /* Multipath failover thresh of retransmits */ extern int mptcp_subflow_keeptime; /* Multipath subflow TCP_KEEPALIVE opt */ -extern int mptcp_mpprio_enable; /* MP_PRIO option enable/disable */ -extern int mptcp_remaddr_enable;/* REMOVE_ADDR option enable/disable */ -extern int mptcp_fastjoin; /* Enable FastJoin */ -extern int mptcp_zerortt_fastjoin; /* Enable Data after SYN Fast Join */ -extern int mptcp_rwnotify; /* Enable RW notification on resume */ extern uint32_t mptcp_dbg_level; /* Multipath TCP debugging level */ extern uint32_t mptcp_dbg_area; /* Multipath TCP debugging area */ +extern int mptcp_developer_mode; /* Allow aggregation mode */ -#define MPPCB_LIMIT 32 -extern uint32_t mptcp_socket_limit; /* max number of mptcp sockets allowed */ -extern uint32_t mptcp_delayed_subf_start; /* delayed cellular subflow start */ extern int tcp_jack_rxmt; /* Join ACK retransmission value in msecs */ __BEGIN_DECLS extern void mptcp_init(struct protosw *, struct domain *); extern int mptcp_ctloutput(struct socket *, struct sockopt *); -extern void *mptcp_sescreate(struct socket *, struct mppcb *); -extern void mptcp_drain(void); +extern int mptcp_sescreate(struct mppcb *); +extern void mptcp_check_subflows_and_add(struct mptses *); +extern int mptcp_get_statsindex(struct mptcp_itf_stats *stats, + const struct mptsub *mpts); +extern void mptcpstats_inc_switch(struct mptses *, const struct mptsub *); extern struct mptses *mptcp_drop(struct mptses *, struct mptcb *, int); extern struct mptses *mptcp_close(struct mptses *, struct mptcb *); extern int mptcp_lock(struct socket *, int, void *); extern int mptcp_unlock(struct socket *, int, void *); extern lck_mtx_t *mptcp_getlock(struct socket *, int); -extern void mptcp_thread_signal(struct mptses *); -extern void mptcp_flush_sopts(struct mptses *); -extern int mptcp_setconnorder(struct mptses *, sae_connid_t, uint32_t); -extern int mptcp_getconnorder(struct mptses *, sae_connid_t, uint32_t *); +extern void mptcp_subflow_workloop(struct mptses *); + +extern void mptcp_sched_create_subflows(struct mptses *); extern struct mptopt *mptcp_sopt_alloc(int); -extern const char *mptcp_sopt2str(int, int, char *, int); +extern const char *mptcp_sopt2str(int, int); extern void mptcp_sopt_free(struct mptopt *); extern void mptcp_sopt_insert(struct mptses *, struct mptopt *); extern void mptcp_sopt_remove(struct mptses *, struct mptopt *); extern struct mptopt *mptcp_sopt_find(struct mptses *, struct sockopt *); -extern struct mptsub *mptcp_subflow_alloc(int); -extern void mptcp_subflow_free(struct mptsub *); -extern void mptcp_subflow_addref(struct mptsub *, int); -extern int mptcp_subflow_add(struct mptses *, struct mptsub *, - struct proc *, uint32_t); -extern void mptcp_subflow_del(struct mptses *, struct mptsub *, boolean_t); -extern void mptcp_subflow_remref(struct mptsub *); -extern int mptcp_subflow_output(struct mptses *, struct mptsub *); -extern void mptcp_subflow_disconnect(struct mptses *, struct mptsub *, - boolean_t); -extern void mptcp_subflow_sopeeloff(struct mptses *, struct mptsub *, - struct socket *); -extern int mptcp_subflow_sosetopt(struct mptses *, struct socket *, +extern int mptcp_subflow_add(struct mptses *, struct sockaddr *, + struct sockaddr *, uint32_t, sae_connid_t *); +extern void mptcpstats_update(struct mptcp_itf_stats *stats, struct mptsub *mpts); +extern void mptcp_subflow_del(struct mptses *, struct mptsub *); + +#define MPTCP_SUBOUT_PROBING 0x01 +extern int mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts, int flags); +extern void mptcp_clean_reinjectq(struct mptses *mpte); +extern void mptcp_subflow_shutdown(struct mptses *, struct mptsub *); +extern void mptcp_subflow_disconnect(struct mptses *, struct mptsub *); +extern int mptcp_subflow_sosetopt(struct mptses *, struct mptsub *, struct mptopt *); extern int mptcp_subflow_sogetopt(struct mptses *, struct socket *, struct mptopt *); @@ -577,40 +612,28 @@ extern void mptcp_input(struct mptses *, struct mbuf *); extern int mptcp_output(struct mptses *); extern void mptcp_close_fsm(struct mptcb *, uint32_t); -extern mptcp_token_t mptcp_get_localtoken(void *); -extern mptcp_token_t mptcp_get_remotetoken(void *); - -extern u_int64_t mptcp_get_localkey(void *); -extern u_int64_t mptcp_get_remotekey(void *); - -extern void mptcp_free_key(mptcp_key_t *key); extern void mptcp_hmac_sha1(mptcp_key_t, mptcp_key_t, u_int32_t, u_int32_t, - u_char*, int); -extern void mptcp_get_hmac(mptcp_addr_id, struct mptcb *, u_char *, int); + u_char*); +extern void mptcp_get_hmac(mptcp_addr_id, struct mptcb *, u_char *); extern void mptcp_get_rands(mptcp_addr_id, struct mptcb *, u_int32_t *, u_int32_t *); extern void mptcp_set_raddr_rand(mptcp_addr_id, struct mptcb *, mptcp_addr_id, u_int32_t); -extern u_int64_t mptcp_get_trunced_hmac(mptcp_addr_id, struct mptcb *mp_tp); -extern void mptcp_generate_token(char *, int, caddr_t, int); -extern void mptcp_generate_idsn(char *, int, caddr_t, int); extern int mptcp_init_remote_parms(struct mptcb *); extern boolean_t mptcp_ok_to_keepalive(struct mptcb *); extern void mptcp_insert_dsn(struct mppcb *, struct mbuf *); -extern void mptcp_output_getm_dsnmap32(struct socket *, int, uint32_t, - u_int32_t *, u_int32_t *, u_int16_t *, u_int64_t *); -extern void mptcp_output_getm_dsnmap64(struct socket *, int, uint32_t, - u_int64_t *, u_int32_t *, u_int16_t *); -extern void mptcp_send_dfin(struct socket *); +extern void mptcp_output_getm_dsnmap32(struct socket *so, int off, + uint32_t *dsn, uint32_t *relseq, + uint16_t *data_len, uint16_t *dss_csum); +extern void mptcp_output_getm_dsnmap64(struct socket *so, int off, + uint64_t *dsn, uint32_t *relseq, + uint16_t *data_len, uint16_t *dss_csum); extern void mptcp_act_on_txfail(struct socket *); extern struct mptsub *mptcp_get_subflow(struct mptses *, struct mptsub *, struct mptsub **); -extern struct mptsub *mptcp_get_pending_subflow(struct mptses *, - struct mptsub *); -extern struct mptsub* mptcp_use_symptoms_hints(struct mptsub*, - struct mptsub *); extern int mptcp_get_map_for_dsn(struct socket *, u_int64_t, u_int32_t *); -extern int32_t mptcp_adj_sendlen(struct socket *so, int32_t off, int32_t len); +extern int32_t mptcp_adj_sendlen(struct socket *so, int32_t off); +extern void mptcp_sbrcv_grow(struct mptcb *mp_tp); extern int32_t mptcp_sbspace(struct mptcb *); extern void mptcp_notify_mpready(struct socket *); extern void mptcp_notify_mpfail(struct socket *); @@ -619,9 +642,18 @@ extern boolean_t mptcp_no_rto_spike(struct socket*); extern int mptcp_set_notsent_lowat(struct mptses *mpte, int optval); extern u_int32_t mptcp_get_notsent_lowat(struct mptses *mpte); extern int mptcp_notsent_lowat_check(struct socket *so); +extern void mptcp_ask_symptoms(struct mptses *mpte); extern void mptcp_control_register(void); extern int mptcp_is_wifi_unusable(void); -extern int mptcp_is_cell_unusable(void); +extern void mptcp_session_necp_cb(void *, int, struct necp_client_flow *); +extern void mptcp_set_restrictions(struct socket *mp_so); +extern int mptcp_freeq(struct mptcb *); +extern void mptcp_set_cellicon(struct mptses *mpte); +extern void mptcp_unset_cellicon(void); +extern void mptcp_reset_rexmit_state(struct tcpcb *tp); +extern void mptcp_reset_keepalive(struct tcpcb *tp); +extern int mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, + uint32_t sseq, uint16_t dlen, uint16_t csum); __END_DECLS #endif /* BSD_KERNEL_PRIVATE */ @@ -634,11 +666,9 @@ typedef struct mptcp_flow { sae_connid_t flow_cid; struct sockaddr_storage flow_src; struct sockaddr_storage flow_dst; - uint64_t flow_sndnxt; /* subflow's sndnxt snapshot */ uint32_t flow_relseq; /* last subflow rel seq# */ int32_t flow_soerror; /* subflow level error */ uint32_t flow_probecnt; /* number of probes sent */ - uint32_t flow_peerswitch;/* did peer switch */ conninfo_tcp_t flow_ci; /* must be the last field */ } mptcp_flow_t; @@ -678,7 +708,8 @@ typedef struct symptoms_advisory { uint32_t sa_nwk_status_int; struct { union { -#define SYMPTOMS_ADVISORY_NOCOMMENT 0x00 +#define SYMPTOMS_ADVISORY_NOCOMMENT 0x0000 +#define SYMPTOMS_ADVISORY_USEAPP 0xFFFF /* Very ugly workaround to avoid breaking backwards compatibility - ToDo: Fix it in +1 */ uint16_t sa_nwk_status; struct { #define SYMPTOMS_ADVISORY_WIFI_BAD 0x01 @@ -694,6 +725,19 @@ typedef struct symptoms_advisory { }; } symptoms_advisory_t; +struct mptcp_symptoms_ask_uuid { + uint32_t cmd; +#define MPTCP_SYMPTOMS_ASK_UUID 1 + uuid_t uuid; + uint32_t priority; +#define MPTCP_SYMPTOMS_UNKNOWN 0 +#define MPTCP_SYMPTOMS_BACKGROUND 1 +#define MPTCP_SYMPTOMS_FOREGROUND 2 +}; + +struct kev_mptcp_data { + int value; +}; #endif /* PRIVATE */ #endif /* _NETINET_MPTCP_VAR_H_ */ diff --git a/bsd/netinet/raw_ip.c b/bsd/netinet/raw_ip.c index 1f7ccb227..cb4e31a22 100644 --- a/bsd/netinet/raw_ip.c +++ b/bsd/netinet/raw_ip.c @@ -84,6 +84,7 @@ #include #include +#include #include #define _IP_VHL @@ -436,7 +437,7 @@ rip_output( m_freem(m); return EINVAL; } - if (ip->ip_id == 0) + if (ip->ip_id == 0 && !(rfc6864 && IP_OFF_IS_ATOMIC(ntohs(ip->ip_off)))) ip->ip_id = ip_randomid(); /* XXX prevent ip_output from overwriting header fields */ flags |= IP_RAWOUTPUT; @@ -499,13 +500,6 @@ rip_output( if (inp->inp_route.ro_rt != NULL) rt_ifp = inp->inp_route.ro_rt->rt_ifp; - printf("%s inp %p last_pid %u inp_boundifp %d inp_last_outifp %d rt_ifp %d route_rule_id %u\n", - __func__, inp, - inp->inp_socket != NULL ? inp->inp_socket->last_pid : -1, - inp->inp_boundifp != NULL ? inp->inp_boundifp->if_index : -1, - inp->inp_last_outifp != NULL ? inp->inp_last_outifp->if_index : -1, - rt_ifp != NULL ? rt_ifp->if_index : -1, - route_rule_id); necp_socket_update_qos_marking(inp, inp->inp_route.ro_rt, NULL, route_rule_id); } @@ -571,8 +565,10 @@ rip_output( * route is unicast, update outif with that of the * route interface used by IP. */ - if (rt != NULL && (outif = rt->rt_ifp) != inp->inp_last_outifp) + if (rt != NULL && + (outif = rt->rt_ifp) != inp->inp_last_outifp) { inp->inp_last_outifp = outif; + } } else { ROUTE_RELEASE(&inp->inp_route); } @@ -752,11 +748,12 @@ void rip_ctlinput( int cmd, struct sockaddr *sa, - __unused void *vip) + __unused void *vip, + __unused struct ifnet *ifp) { - struct in_ifaddr *ia; - struct ifnet *ifp; - int err; + struct in_ifaddr *ia = NULL; + struct ifnet *iaifp = NULL; + int err = 0; int flags, done = 0; switch (cmd) { @@ -816,10 +813,10 @@ rip_ctlinput( lck_rw_done(in_ifaddr_rwlock); flags = RTF_UP; - ifp = ia->ia_ifa.ifa_ifp; + iaifp = ia->ia_ifa.ifa_ifp; - if ((ifp->if_flags & IFF_LOOPBACK) - || (ifp->if_flags & IFF_POINTOPOINT)) + if ((iaifp->if_flags & IFF_LOOPBACK) + || (iaifp->if_flags & IFF_POINTOPOINT)) flags |= RTF_HOST; err = rtinit(&ia->ia_ifa, RTM_ADD, flags); @@ -940,6 +937,7 @@ rip_bind(struct socket *so, struct sockaddr *nam, struct proc *p) } inp->inp_laddr = sin.sin_addr; inp->inp_last_outifp = outif; + return (0); } @@ -962,6 +960,12 @@ rip_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p) if ((addr->sin_family != AF_INET) && (addr->sin_family != AF_IMPLINK)) return EAFNOSUPPORT; + + if (!(so->so_flags1 & SOF1_CONNECT_COUNTED)) { + so->so_flags1 |= SOF1_CONNECT_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_connected); + } + inp->inp_faddr = addr->sin_addr; soisconnected(so); @@ -1169,6 +1173,7 @@ SYSCTL_PROC(_net_inet_raw, OID_AUTO/*XXX*/, pcblist, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, rip_pcblist, "S,xinpcb", "List of active raw IP sockets"); +#if !CONFIG_EMBEDDED static int rip_pcblist64 SYSCTL_HANDLER_ARGS @@ -1272,6 +1277,7 @@ SYSCTL_PROC(_net_inet_raw, OID_AUTO, pcblist64, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, rip_pcblist64, "S,xinpcb64", "List of active raw IP sockets"); +#endif /* !CONFIG_EMBEDDED */ static int diff --git a/bsd/netinet/tcp.h b/bsd/netinet/tcp.h index 35d85c160..d960c1f77 100644 --- a/bsd/netinet/tcp.h +++ b/bsd/netinet/tcp.h @@ -66,6 +66,7 @@ #include #include #include +#include /* __uint32_t */ #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) typedef __uint32_t tcp_seq; @@ -224,8 +225,8 @@ struct tcphdr { #define TCP_CONNECTION_INFO 0x106 /* State of TCP connection */ #ifdef PRIVATE -#define TCP_INFO 0x200 /* retrieve tcp_info structure */ -#define TCP_MEASURE_SND_BW 0x202 /* Measure sender's bandwidth for this connection */ +#define TCP_INFO 0x200 /* retrieve tcp_info structure */ +#define TCP_MEASURE_SND_BW 0x202 /* Measure sender's bandwidth for this connection */ #endif /* PRIVATE */ @@ -280,6 +281,21 @@ struct tcp_notify_ack_complete { }; #define TCP_NOTIFY_ACKNOWLEDGEMENT 0x212 /* Notify when data is acknowledged */ +#define MPTCP_SERVICE_TYPE 0x213 /* MPTCP Service type */ +#define TCP_FASTOPEN_FORCE_HEURISTICS 0x214 /* Make sure TFO-heuristics never get disabled */ + +#define MPTCP_SVCTYPE_HANDOVER 0 /* Default 0 */ +#define MPTCP_SVCTYPE_INTERACTIVE 1 +#define MPTCP_SVCTYPE_AGGREGATE 2 +#define MPTCP_SVCTYPE_MAX 3 +/* + * Specify minimum time in seconds before which an established + * TCP connection will not be dropped when there is no response from the + * peer + */ +#define TCP_RXT_MINIMUM_TIMEOUT 0x215 + +#define TCP_RXT_MINIMUM_TIMEOUT_LIMIT (5 * 60) /* Limit is 5 minutes */ /* * The TCP_INFO socket option is a private API and is subject to change @@ -380,7 +396,8 @@ struct tcp_info { tcpi_tfo_no_cookie_rcv:1, /* We did not receive a cookie upon our request */ tcpi_tfo_heuristics_disable:1, /* TFO-heuristics disabled it */ tcpi_tfo_send_blackhole:1, /* A sending-blackhole got detected */ - tcpi_tfo_recv_blackhole:1; /* A receiver-blackhole got detected */ + tcpi_tfo_recv_blackhole:1, /* A receiver-blackhole got detected */ + tcpi_tfo_onebyte_proxy:1; /* A proxy acknowledges all but one byte of the SYN */ u_int16_t tcpi_ecn_client_setup:1, /* Attempted ECN setup from client side */ tcpi_ecn_server_setup:1, /* Attempted ECN setup from server side */ @@ -388,8 +405,13 @@ struct tcp_info { tcpi_ecn_lost_syn:1, /* Lost SYN with ECN setup */ tcpi_ecn_lost_synack:1, /* Lost SYN-ACK with ECN setup */ tcpi_local_peer:1, /* Local to the host or the subnet */ - tcpi_if_cell:1, /* Interface is cellular */ - tcpi_if_wifi:1; /* Interface is WiFi */ + tcpi_if_cell:1, /* Interface is cellular */ + tcpi_if_wifi:1, /* Interface is WiFi */ + tcpi_if_wired:1, /* Interface is wired - ethernet , thunderbolt etc,. */ + tcpi_if_wifi_infra:1, /* Interface is wifi infrastructure */ + tcpi_if_wifi_awdl:1, /* Interface is wifi AWDL */ + tcpi_snd_background:1, /* Using delay based algorithm on sender side */ + tcpi_rcv_background:1; /* Using delay based algorithm on receive side */ u_int32_t tcpi_ecn_recv_ce; /* Packets received with CE */ u_int32_t tcpi_ecn_recv_cwr; /* Packets received with CWR */ @@ -429,16 +451,16 @@ struct info_tuple { }; #define itpl_local_sa itpl_localaddr._itpl_sa -#define itpl_local_sin itpl_localaddr._itpl_sin +#define itpl_local_sin itpl_localaddr._itpl_sin #define itpl_local_sin6 itpl_localaddr._itpl_sin6 -#define itpl_remote_sa itpl_remoteaddr._itpl_sa +#define itpl_remote_sa itpl_remoteaddr._itpl_sa #define itpl_remote_sin itpl_remoteaddr._itpl_sin #define itpl_remote_sin6 itpl_remoteaddr._itpl_sin6 /* * TCP connection info auxiliary data (CIAUX_TCP) * - * Do not add new fields to this structure, just add them to tcp_info + * Do not add new fields to this structure, just add them to tcp_info * structure towards the end. This will preserve binary compatibility. */ typedef struct conninfo_tcp { @@ -448,6 +470,31 @@ typedef struct conninfo_tcp { #pragma pack() +struct mptcp_itf_stats { + uint16_t ifindex; + uint16_t switches; + uint32_t is_expensive:1; + uint64_t mpis_txbytes __attribute__((aligned(8))); + uint64_t mpis_rxbytes __attribute__((aligned(8))); +}; + +/* Version solely used to let libnetcore survive */ +#define CONNINFO_MPTCP_VERSION 3 +typedef struct conninfo_multipathtcp { + uint32_t mptcpci_subflow_count; + uint32_t mptcpci_switch_count; + sae_connid_t mptcpci_subflow_connids[4]; + + uint64_t mptcpci_init_rxbytes; + uint64_t mptcpci_init_txbytes; + +#define MPTCP_ITFSTATS_SIZE 4 + struct mptcp_itf_stats mptcpci_itfstats[MPTCP_ITFSTATS_SIZE]; + + uint32_t mptcpci_flags; +#define MPTCPCI_FIRSTPARTY 0x01 +} conninfo_multipathtcp_t; + #endif /* PRIVATE */ struct tcp_connection_info { @@ -488,7 +535,8 @@ struct tcp_connection_info { tcpi_tfo_heuristics_disable:1, /* TFO-heuristics disabled it */ tcpi_tfo_send_blackhole:1, /* A sending-blackhole got detected */ tcpi_tfo_recv_blackhole:1, /* A receiver-blackhole got detected */ - __pad2:18; + tcpi_tfo_onebyte_proxy:1, /* A proxy acknowledges all but one byte of the SYN */ + __pad2:17; u_int64_t tcpi_txpackets __attribute__((aligned(8))); u_int64_t tcpi_txbytes __attribute__((aligned(8))); u_int64_t tcpi_txretransmitbytes __attribute__((aligned(8))); diff --git a/bsd/netinet/tcp_cache.c b/bsd/netinet/tcp_cache.c index ecd3ad590..293a0fc0c 100644 --- a/bsd/netinet/tcp_cache.c +++ b/bsd/netinet/tcp_cache.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,7 +30,9 @@ #include #include +#include #include +#include #include #include #include @@ -38,13 +40,15 @@ #include #include +typedef union { + struct in_addr addr; + struct in6_addr addr6; +} in_4_6_addr; + struct tcp_heuristic_key { union { uint8_t thk_net_signature[IFNET_SIGNATURELEN]; - union { - struct in_addr addr; - struct in6_addr addr6; - } thk_ip; + in_4_6_addr thk_ip; }; sa_family_t thk_family; }; @@ -52,27 +56,29 @@ struct tcp_heuristic_key { struct tcp_heuristic { SLIST_ENTRY(tcp_heuristic) list; - u_int32_t th_last_access; + uint32_t th_last_access; struct tcp_heuristic_key th_key; char th_val_start[0]; /* Marker for memsetting to 0 */ - u_int8_t th_tfo_cookie_loss; /* The number of times a SYN+cookie has been lost */ - u_int8_t th_mptcp_loss; /* The number of times a SYN+MP_CAPABLE has been lost */ - u_int8_t th_ecn_loss; /* The number of times a SYN+ecn has been lost */ - u_int8_t th_ecn_aggressive; /* The number of times we did an aggressive fallback */ - u_int8_t th_ecn_droprst; /* The number of times ECN connections received a RST after first data pkt */ - u_int8_t th_ecn_droprxmt; /* The number of times ECN connection is dropped after multiple retransmits */ - u_int32_t th_tfo_fallback_trials; /* Number of times we did not try out TFO due to SYN-loss */ - u_int32_t th_tfo_cookie_backoff; /* Time until when we should not try out TFO */ - u_int32_t th_mptcp_backoff; /* Time until when we should not try out MPTCP */ - u_int32_t th_ecn_backoff; /* Time until when we should not try out ECN */ - - u_int8_t th_tfo_in_backoff:1, /* Are we avoiding TFO due to the backoff timer? */ - th_tfo_aggressive_fallback:1, /* Aggressive fallback due to nasty middlebox */ - th_tfo_snd_middlebox_supp:1, /* We are sure that the network supports TFO in upstream direction */ - th_tfo_rcv_middlebox_supp:1, /* We are sure that the network supports TFO in downstream direction*/ + uint8_t th_tfo_data_loss; /* The number of times a SYN+data has been lost */ + uint8_t th_tfo_req_loss; /* The number of times a SYN+cookie-req has been lost */ + uint8_t th_tfo_data_rst; /* The number of times a SYN+data has received a RST */ + uint8_t th_tfo_req_rst; /* The number of times a SYN+cookie-req has received a RST */ + uint8_t th_mptcp_loss; /* The number of times a SYN+MP_CAPABLE has been lost */ + uint8_t th_ecn_loss; /* The number of times a SYN+ecn has been lost */ + uint8_t th_ecn_aggressive; /* The number of times we did an aggressive fallback */ + uint8_t th_ecn_droprst; /* The number of times ECN connections received a RST after first data pkt */ + uint8_t th_ecn_droprxmt; /* The number of times ECN connection is dropped after multiple retransmits */ + uint8_t th_ecn_synrst; /* number of times RST was received in response to an ECN enabled SYN */ + uint32_t th_tfo_enabled_time; /* The moment when we reenabled TFO after backing off */ + uint32_t th_tfo_backoff_until; /* Time until when we should not try out TFO */ + uint32_t th_tfo_backoff; /* Current backoff timer */ + uint32_t th_mptcp_backoff; /* Time until when we should not try out MPTCP */ + uint32_t th_ecn_backoff; /* Time until when we should not try out ECN */ + + uint8_t th_tfo_in_backoff:1, /* Are we avoiding TFO due to the backoff timer? */ th_mptcp_in_backoff:1; /* Are we avoiding MPTCP due to the backoff timer? */ char th_val_end[0]; /* Marker for memsetting to 0 */ @@ -89,10 +95,7 @@ struct tcp_cache_key { sa_family_t tck_family; struct tcp_heuristic_key tck_src; - union { - struct in_addr addr; - struct in6_addr addr6; - } tck_dst; + in_4_6_addr tck_dst; }; struct tcp_cache { @@ -113,6 +116,13 @@ struct tcp_cache_head { lck_mtx_t tch_mtx; }; +struct tcp_cache_key_src { + struct ifnet *ifp; + in_4_6_addr laddr; + in_4_6_addr faddr; + int af; +}; + static u_int32_t tcp_cache_hash_seed; size_t tcp_cache_size; @@ -139,13 +149,24 @@ static lck_attr_t *tcp_heuristic_mtx_attr; static lck_grp_t *tcp_heuristic_mtx_grp; static lck_grp_attr_t *tcp_heuristic_mtx_grp_attr; -static int tcp_ecn_timeout = 60; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_timeout, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_ecn_timeout, 0, "Initial minutes to wait before re-trying ECN"); +static uint32_t tcp_backoff_maximum = 65536; + +SYSCTL_UINT(_net_inet_tcp, OID_AUTO, backoff_maximum, CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_backoff_maximum, 0, "Maximum time for which we won't try TFO"); + +SYSCTL_SKMEM_TCP_INT(OID_AUTO, ecn_timeout, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, tcp_ecn_timeout, 60, "Initial minutes to wait before re-trying ECN"); + +SYSCTL_SKMEM_TCP_INT(OID_AUTO, disable_tcp_heuristics, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, disable_tcp_heuristics, 0, "Set to 1, to disable all TCP heuristics (TFO, ECN, MPTCP)"); -static int disable_tcp_heuristics = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, disable_tcp_heuristics, CTLFLAG_RW | CTLFLAG_LOCKED, - &disable_tcp_heuristics, 0, "Set to 1, to disable all TCP heuristics (TFO, ECN, MPTCP)"); +static uint32_t tcp_min_to_hz(uint32_t minutes) +{ + if (minutes > 65536) + return ((uint32_t)65536 * 60 * TCP_RETRANSHZ); + + return (minutes * 60 * TCP_RETRANSHZ); +} /* * This number is coupled with tcp_ecn_timeout, because we want to prevent @@ -158,19 +179,34 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, disable_tcp_heuristics, CTLFLAG_RW | CTLFLAG #define TFO_MAX_COOKIE_LOSS 2 #define ECN_MAX_SYN_LOSS 2 #define MPTCP_MAX_SYN_LOSS 2 -#define ECN_MAX_DROPRST 2 +#define ECN_MAX_DROPRST 1 #define ECN_MAX_DROPRXMT 4 - -/* Flags for setting/unsetting loss-heuristics, limited to 1 byte */ -#define TCPCACHE_F_TFO 0x01 -#define TCPCACHE_F_ECN 0x02 -#define TCPCACHE_F_MPTCP 0x04 -#define TCPCACHE_F_ECN_DROPRST 0x08 -#define TCPCACHE_F_ECN_DROPRXMT 0x10 +#define ECN_MAX_SYNRST 4 + +/* Flags for setting/unsetting loss-heuristics, limited to 4 bytes */ +#define TCPCACHE_F_TFO_REQ 0x01 +#define TCPCACHE_F_TFO_DATA 0x02 +#define TCPCACHE_F_ECN 0x04 +#define TCPCACHE_F_MPTCP 0x08 +#define TCPCACHE_F_ECN_DROPRST 0x10 +#define TCPCACHE_F_ECN_DROPRXMT 0x20 +#define TCPCACHE_F_TFO_REQ_RST 0x40 +#define TCPCACHE_F_TFO_DATA_RST 0x80 +#define TCPCACHE_F_ECN_SYNRST 0x100 /* Always retry ECN after backing off to this level for some heuristics */ #define ECN_RETRY_LIMIT 9 +#define TCP_CACHE_INC_IFNET_STAT(_ifp_, _af_, _stat_) { \ + if ((_ifp_) != NULL) { \ + if ((_af_) == AF_INET6) { \ + (_ifp_)->if_ipv6_stat->_stat_++;\ + } else { \ + (_ifp_)->if_ipv4_stat->_stat_++;\ + }\ + }\ +} + /* * Round up to next higher power-of 2. See "Bit Twiddling Hacks". * @@ -190,17 +226,17 @@ static u_int32_t tcp_cache_roundup2(u_int32_t a) return a; } -static void tcp_cache_hash_src(struct inpcb *inp, struct tcp_heuristic_key *key) +static void tcp_cache_hash_src(struct tcp_cache_key_src *tcks, struct tcp_heuristic_key *key) { - struct ifnet *ifn = inp->inp_last_outifp; + struct ifnet *ifp = tcks->ifp; uint8_t len = sizeof(key->thk_net_signature); uint16_t flags; - if (inp->inp_vflag & INP_IPV6) { + if (tcks->af == AF_INET6) { int ret; key->thk_family = AF_INET6; - ret = ifnet_get_netsignature(ifn, AF_INET6, &len, &flags, + ret = ifnet_get_netsignature(ifp, AF_INET6, &len, &flags, key->thk_net_signature); /* @@ -209,13 +245,13 @@ static void tcp_cache_hash_src(struct inpcb *inp, struct tcp_heuristic_key *key) * in this case we should take the connection's address. */ if (ret == ENOENT || ret == EINVAL) - memcpy(&key->thk_ip.addr6, &inp->in6p_laddr, sizeof(struct in6_addr)); + memcpy(&key->thk_ip.addr6, &tcks->laddr.addr6, sizeof(struct in6_addr)); } else { int ret; key->thk_family = AF_INET; - ret = ifnet_get_netsignature(ifn, AF_INET, &len, &flags, - key->thk_net_signature); + ret = ifnet_get_netsignature(ifp, AF_INET, &len, &flags, + key->thk_net_signature); /* * ifnet_get_netsignature only returns EINVAL if ifn is NULL @@ -223,25 +259,25 @@ static void tcp_cache_hash_src(struct inpcb *inp, struct tcp_heuristic_key *key) * in this case we should take the connection's address. */ if (ret == ENOENT || ret == EINVAL) - memcpy(&key->thk_ip.addr, &inp->inp_laddr, sizeof(struct in_addr)); + memcpy(&key->thk_ip.addr, &tcks->laddr.addr, sizeof(struct in_addr)); } } -static u_int16_t tcp_cache_hash(struct inpcb *inp, struct tcp_cache_key *key) +static u_int16_t tcp_cache_hash(struct tcp_cache_key_src *tcks, struct tcp_cache_key *key) { u_int32_t hash; bzero(key, sizeof(struct tcp_cache_key)); - tcp_cache_hash_src(inp, &key->tck_src); + tcp_cache_hash_src(tcks, &key->tck_src); - if (inp->inp_vflag & INP_IPV6) { + if (tcks->af == AF_INET6) { key->tck_family = AF_INET6; - memcpy(&key->tck_dst.addr6, &inp->in6p_faddr, + memcpy(&key->tck_dst.addr6, &tcks->faddr.addr6, sizeof(struct in6_addr)); } else { key->tck_family = AF_INET; - memcpy(&key->tck_dst.addr, &inp->inp_faddr, + memcpy(&key->tck_dst.addr, &tcks->faddr.addr, sizeof(struct in_addr)); } @@ -266,17 +302,16 @@ static void tcp_cache_unlock(struct tcp_cache_head *head) * That's why we provide the head as a "return"-pointer so that the caller * can give it back to use for tcp_cache_unlock(). */ -static struct tcp_cache *tcp_getcache_with_lock(struct tcpcb *tp, int create, - struct tcp_cache_head **headarg) +static struct tcp_cache *tcp_getcache_with_lock(struct tcp_cache_key_src *tcks, + int create, struct tcp_cache_head **headarg) { - struct inpcb *inp = tp->t_inpcb; struct tcp_cache *tpcache = NULL; struct tcp_cache_head *head; struct tcp_cache_key key; u_int16_t hash; int i = 0; - hash = tcp_cache_hash(inp, &key); + hash = tcp_cache_hash(tcks, &key); head = &tcp_cache[hash]; lck_mtx_lock(&head->tch_mtx); @@ -336,13 +371,33 @@ out_null: return (NULL); } -void tcp_cache_set_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t len) +static void tcp_cache_key_src_create(struct tcpcb *tp, struct tcp_cache_key_src *tcks) +{ + struct inpcb *inp = tp->t_inpcb; + memset(tcks, 0, sizeof(*tcks)); + + tcks->ifp = inp->inp_last_outifp; + + if (inp->inp_vflag & INP_IPV6) { + memcpy(&tcks->laddr.addr6, &inp->in6p_laddr, sizeof(struct in6_addr)); + memcpy(&tcks->faddr.addr6, &inp->in6p_faddr, sizeof(struct in6_addr)); + tcks->af = AF_INET6; + } else { + memcpy(&tcks->laddr.addr, &inp->inp_laddr, sizeof(struct in_addr)); + memcpy(&tcks->faddr.addr, &inp->inp_faddr, sizeof(struct in_addr)); + tcks->af = AF_INET; + } + + return; +} + +static void tcp_cache_set_cookie_common(struct tcp_cache_key_src *tcks, u_char *cookie, u_int8_t len) { struct tcp_cache_head *head; struct tcp_cache *tpcache; /* Call lookup/create function */ - tpcache = tcp_getcache_with_lock(tp, 1, &head); + tpcache = tcp_getcache_with_lock(tcks, 1, &head); if (tpcache == NULL) return; @@ -352,23 +407,24 @@ void tcp_cache_set_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t len) tcp_cache_unlock(head); } -/* - * Get the cookie related to 'tp', and copy it into 'cookie', provided that len - * is big enough (len designates the available memory. - * Upon return, 'len' is set to the cookie's length. - * - * Returns 0 if we should request a cookie. - * Returns 1 if the cookie has been found and written. - */ -int tcp_cache_get_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t *len) +void tcp_cache_set_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t len) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + tcp_cache_set_cookie_common(&tcks, cookie, len); +} + +static int tcp_cache_get_cookie_common(struct tcp_cache_key_src *tcks, u_char *cookie, u_int8_t *len) { struct tcp_cache_head *head; struct tcp_cache *tpcache; /* Call lookup/create function */ - tpcache = tcp_getcache_with_lock(tp, 1, &head); - if (tpcache == NULL) + tpcache = tcp_getcache_with_lock(tcks, 1, &head); + if (tpcache == NULL) { return (0); + } if (tpcache->tc_tfo_cookie_len == 0) { tcp_cache_unlock(head); @@ -389,14 +445,30 @@ int tcp_cache_get_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t *len) return (1); } -unsigned int tcp_cache_get_cookie_len(struct tcpcb *tp) +/* + * Get the cookie related to 'tp', and copy it into 'cookie', provided that len + * is big enough (len designates the available memory. + * Upon return, 'len' is set to the cookie's length. + * + * Returns 0 if we should request a cookie. + * Returns 1 if the cookie has been found and written. + */ +int tcp_cache_get_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t *len) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + return tcp_cache_get_cookie_common(&tcks, cookie, len); +} + +static unsigned int tcp_cache_get_cookie_len_common(struct tcp_cache_key_src *tcks) { struct tcp_cache_head *head; struct tcp_cache *tpcache; unsigned int cookie_len; /* Call lookup/create function */ - tpcache = tcp_getcache_with_lock(tp, 1, &head); + tpcache = tcp_getcache_with_lock(tcks, 1, &head); if (tpcache == NULL) return (0); @@ -407,14 +479,21 @@ unsigned int tcp_cache_get_cookie_len(struct tcpcb *tp) return cookie_len; } -static u_int16_t tcp_heuristics_hash(struct inpcb *inp, - struct tcp_heuristic_key *key) +unsigned int tcp_cache_get_cookie_len(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + return tcp_cache_get_cookie_len_common(&tcks); +} + +static u_int16_t tcp_heuristics_hash(struct tcp_cache_key_src *tcks, struct tcp_heuristic_key *key) { u_int32_t hash; bzero(key, sizeof(struct tcp_heuristic_key)); - tcp_cache_hash_src(inp, key); + tcp_cache_hash_src(tcks, key); hash = net_flowhash(key, sizeof(struct tcp_heuristic_key), tcp_cache_hash_seed); @@ -441,17 +520,16 @@ static void tcp_heuristic_unlock(struct tcp_heuristics_head *head) * ToDo - way too much code-duplication. We should create an interface to handle * bucketized hashtables with recycling of the oldest element. */ -static struct tcp_heuristic *tcp_getheuristic_with_lock(struct tcpcb *tp, +static struct tcp_heuristic *tcp_getheuristic_with_lock(struct tcp_cache_key_src *tcks, int create, struct tcp_heuristics_head **headarg) { - struct inpcb *inp = tp->t_inpcb; struct tcp_heuristic *tpheur = NULL; struct tcp_heuristics_head *head; struct tcp_heuristic_key key; u_int16_t hash; int i = 0; - hash = tcp_heuristics_hash(inp, &key); + hash = tcp_heuristics_hash(tcks, &key); head = &tcp_heuristics[hash]; lck_mtx_lock(&head->thh_mtx); @@ -500,8 +578,9 @@ static struct tcp_heuristic *tcp_getheuristic_with_lock(struct tcpcb *tp, * near future. */ tpheur->th_ecn_backoff = tcp_now; - tpheur->th_tfo_cookie_backoff = tcp_now; + tpheur->th_tfo_backoff_until = tcp_now; tpheur->th_mptcp_backoff = tcp_now; + tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout); memcpy(&tpheur->th_key, &key, sizeof(key)); } @@ -520,7 +599,7 @@ out_null: return (NULL); } -static void tcp_heuristic_reset_loss(struct tcpcb *tp, u_int8_t flags) +static void tcp_heuristic_reset_counters(struct tcp_cache_key_src *tcks, u_int8_t flags) { struct tcp_heuristics_head *head; struct tcp_heuristic *tpheur; @@ -530,15 +609,30 @@ static void tcp_heuristic_reset_loss(struct tcpcb *tp, u_int8_t flags) * server does not support TFO. This reduces the lookup-cost on * our side. */ - tpheur = tcp_getheuristic_with_lock(tp, 0, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 0, &head); if (tpheur == NULL) return; - if (flags & TCPCACHE_F_TFO) - tpheur->th_tfo_cookie_loss = 0; + if (flags & TCPCACHE_F_TFO_DATA) { + tpheur->th_tfo_data_loss = 0; + } + + if (flags & TCPCACHE_F_TFO_REQ) { + tpheur->th_tfo_req_loss = 0; + } + + if (flags & TCPCACHE_F_TFO_DATA_RST) { + tpheur->th_tfo_data_rst = 0; + } + + if (flags & TCPCACHE_F_TFO_REQ_RST) { + tpheur->th_tfo_req_rst = 0; + } - if (flags & TCPCACHE_F_ECN) + if (flags & TCPCACHE_F_ECN) { tpheur->th_ecn_loss = 0; + tpheur->th_ecn_synrst = 0; + } if (flags & TCPCACHE_F_MPTCP) tpheur->th_mptcp_loss = 0; @@ -548,69 +642,120 @@ static void tcp_heuristic_reset_loss(struct tcpcb *tp, u_int8_t flags) void tcp_heuristic_tfo_success(struct tcpcb *tp) { - tcp_heuristic_reset_loss(tp, TCPCACHE_F_TFO); + struct tcp_cache_key_src tcks; + uint8_t flag = 0; + + tcp_cache_key_src_create(tp, &tcks); + + if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) + flag = (TCPCACHE_F_TFO_DATA | TCPCACHE_F_TFO_REQ | + TCPCACHE_F_TFO_DATA_RST | TCPCACHE_F_TFO_REQ_RST ); + if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) + flag = (TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_REQ_RST); + + tcp_heuristic_reset_counters(&tcks, flag); } void tcp_heuristic_mptcp_success(struct tcpcb *tp) { - tcp_heuristic_reset_loss(tp, TCPCACHE_F_MPTCP); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_MPTCP); } void tcp_heuristic_ecn_success(struct tcpcb *tp) { - tcp_heuristic_reset_loss(tp, TCPCACHE_F_ECN); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_ECN); } -void tcp_heuristic_tfo_rcv_good(struct tcpcb *tp) +static void __tcp_heuristic_tfo_middlebox_common(struct tcp_heuristic *tpheur) { - struct tcp_heuristics_head *head; - - struct tcp_heuristic *tpheur = tcp_getheuristic_with_lock(tp, 1, &head); - if (tpheur == NULL) + if (tpheur->th_tfo_in_backoff) return; - tpheur->th_tfo_rcv_middlebox_supp = 1; + tpheur->th_tfo_in_backoff = 1; - tcp_heuristic_unlock(head); + if (tpheur->th_tfo_enabled_time) { + uint32_t old_backoff = tpheur->th_tfo_backoff; + + tpheur->th_tfo_backoff -= (tcp_now - tpheur->th_tfo_enabled_time); + if (tpheur->th_tfo_backoff > old_backoff) + tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout); + } - tp->t_tfo_flags |= TFO_F_NO_RCVPROBING; + tpheur->th_tfo_backoff_until = tcp_now + tpheur->th_tfo_backoff; + + /* Then, increase the backoff time */ + tpheur->th_tfo_backoff *= 2; + + if (tpheur->th_tfo_backoff > tcp_min_to_hz(tcp_backoff_maximum)) + tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout); } -void tcp_heuristic_tfo_snd_good(struct tcpcb *tp) +static void tcp_heuristic_tfo_middlebox_common(struct tcp_cache_key_src *tcks) { struct tcp_heuristics_head *head; + struct tcp_heuristic *tpheur; - struct tcp_heuristic *tpheur = tcp_getheuristic_with_lock(tp, 1, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 1, &head); if (tpheur == NULL) return; - tpheur->th_tfo_snd_middlebox_supp = 1; + __tcp_heuristic_tfo_middlebox_common(tpheur); tcp_heuristic_unlock(head); - - tp->t_tfo_flags |= TFO_F_NO_SNDPROBING; } -static void tcp_heuristic_inc_loss(struct tcpcb *tp, u_int8_t flags) +static void tcp_heuristic_inc_counters(struct tcp_cache_key_src *tcks, + u_int32_t flags) { struct tcp_heuristics_head *head; struct tcp_heuristic *tpheur; - tpheur = tcp_getheuristic_with_lock(tp, 1, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 1, &head); if (tpheur == NULL) return; /* Limit to prevent integer-overflow during exponential backoff */ - if ((flags & TCPCACHE_F_TFO) && tpheur->th_tfo_cookie_loss < TCP_CACHE_OVERFLOW_PROTECT) - tpheur->th_tfo_cookie_loss++; + if ((flags & TCPCACHE_F_TFO_DATA) && tpheur->th_tfo_data_loss < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_tfo_data_loss++; + + if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) + __tcp_heuristic_tfo_middlebox_common(tpheur); + } + + if ((flags & TCPCACHE_F_TFO_REQ) && tpheur->th_tfo_req_loss < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_tfo_req_loss++; + + if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) + __tcp_heuristic_tfo_middlebox_common(tpheur); + } + + if ((flags & TCPCACHE_F_TFO_DATA_RST) && tpheur->th_tfo_data_rst < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_tfo_data_rst++; + + if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) + __tcp_heuristic_tfo_middlebox_common(tpheur); + } + + if ((flags & TCPCACHE_F_TFO_REQ_RST) && tpheur->th_tfo_req_rst < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_tfo_req_rst++; + + if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) + __tcp_heuristic_tfo_middlebox_common(tpheur); + } if ((flags & TCPCACHE_F_ECN) && tpheur->th_ecn_loss < TCP_CACHE_OVERFLOW_PROTECT) { tpheur->th_ecn_loss++; if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS) { tcpstat.tcps_ecn_fallback_synloss++; - INP_INC_IFNET_STAT(tp->t_inpcb, ecn_fallback_synloss); + TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, ecn_fallback_synloss); tpheur->th_ecn_backoff = tcp_now + - ((tcp_ecn_timeout * 60 * TCP_RETRANSHZ) << + (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_loss - ECN_MAX_SYN_LOSS)); } } @@ -624,7 +769,7 @@ static void tcp_heuristic_inc_loss(struct tcpcb *tp, u_int8_t flags) * another sysctl that is just used for testing. */ tpheur->th_mptcp_backoff = tcp_now + - ((tcp_ecn_timeout * 60 * TCP_RETRANSHZ) << + (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_mptcp_loss - MPTCP_MAX_SYN_LOSS)); } } @@ -634,79 +779,139 @@ static void tcp_heuristic_inc_loss(struct tcpcb *tp, u_int8_t flags) tpheur->th_ecn_droprst++; if (tpheur->th_ecn_droprst >= ECN_MAX_DROPRST) { tcpstat.tcps_ecn_fallback_droprst++; - INP_INC_IFNET_STAT(tp->t_inpcb, ecn_fallback_droprst); + TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, + ecn_fallback_droprst); tpheur->th_ecn_backoff = tcp_now + - ((tcp_ecn_timeout * 60 * TCP_RETRANSHZ) << + (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_droprst - ECN_MAX_DROPRST)); } } if ((flags & TCPCACHE_F_ECN_DROPRXMT) && - tpheur->th_ecn_droprst < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_ecn_droprxmt < TCP_CACHE_OVERFLOW_PROTECT) { tpheur->th_ecn_droprxmt++; if (tpheur->th_ecn_droprxmt >= ECN_MAX_DROPRXMT) { tcpstat.tcps_ecn_fallback_droprxmt++; - INP_INC_IFNET_STAT(tp->t_inpcb, ecn_fallback_droprxmt); + TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, + ecn_fallback_droprxmt); tpheur->th_ecn_backoff = tcp_now + - ((tcp_ecn_timeout * 60 * TCP_RETRANSHZ) << + (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_droprxmt - ECN_MAX_DROPRXMT)); } } + if ((flags & TCPCACHE_F_ECN_SYNRST) && + tpheur->th_ecn_synrst < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_ecn_synrst++; + if (tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) { + tcpstat.tcps_ecn_fallback_synrst++; + TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, + ecn_fallback_synrst); + tpheur->th_ecn_backoff = tcp_now + + (tcp_min_to_hz(tcp_ecn_timeout) << + (tpheur->th_ecn_synrst - ECN_MAX_SYNRST)); + } + } tcp_heuristic_unlock(head); } void tcp_heuristic_tfo_loss(struct tcpcb *tp) { - tcp_heuristic_inc_loss(tp, TCPCACHE_F_TFO); + struct tcp_cache_key_src tcks; + uint32_t flag = 0; + + tcp_cache_key_src_create(tp, &tcks); + + if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) + flag = (TCPCACHE_F_TFO_DATA | TCPCACHE_F_TFO_REQ); + if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) + flag = TCPCACHE_F_TFO_REQ; + + tcp_heuristic_inc_counters(&tcks, flag); +} + +void tcp_heuristic_tfo_rst(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + uint32_t flag = 0; + + tcp_cache_key_src_create(tp, &tcks); + + if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) + flag = (TCPCACHE_F_TFO_DATA_RST | TCPCACHE_F_TFO_REQ_RST); + if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) + flag = TCPCACHE_F_TFO_REQ_RST; + + tcp_heuristic_inc_counters(&tcks, flag); } void tcp_heuristic_mptcp_loss(struct tcpcb *tp) { - tcp_heuristic_inc_loss(tp, TCPCACHE_F_MPTCP); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_MPTCP); } void tcp_heuristic_ecn_loss(struct tcpcb *tp) { - tcp_heuristic_inc_loss(tp, TCPCACHE_F_ECN); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN); } void tcp_heuristic_ecn_droprst(struct tcpcb *tp) { - tcp_heuristic_inc_loss(tp, TCPCACHE_F_ECN_DROPRST); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRST); } void tcp_heuristic_ecn_droprxmt(struct tcpcb *tp) { - tcp_heuristic_inc_loss(tp, TCPCACHE_F_ECN_DROPRXMT); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRXMT); } -void tcp_heuristic_tfo_middlebox(struct tcpcb *tp) +void tcp_heuristic_ecn_synrst(struct tcpcb *tp) { - struct tcp_heuristics_head *head; - struct tcp_heuristic *tpheur; + struct tcp_cache_key_src tcks; - tpheur = tcp_getheuristic_with_lock(tp, 1, &head); - if (tpheur == NULL) - return; + tcp_cache_key_src_create(tp, &tcks); - tpheur->th_tfo_aggressive_fallback = 1; + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_SYNRST); +} - tcp_heuristic_unlock(head); +void tcp_heuristic_tfo_middlebox(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + + tp->t_tfo_flags |= TFO_F_HEURISTIC_DONE; + + tcp_cache_key_src_create(tp, &tcks); + tcp_heuristic_tfo_middlebox_common(&tcks); } -void tcp_heuristic_ecn_aggressive(struct tcpcb *tp) +static void tcp_heuristic_ecn_aggressive_common(struct tcp_cache_key_src *tcks) { struct tcp_heuristics_head *head; struct tcp_heuristic *tpheur; - tpheur = tcp_getheuristic_with_lock(tp, 1, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 1, &head); if (tpheur == NULL) return; /* Must be done before, otherwise we will start off with expo-backoff */ tpheur->th_ecn_backoff = tcp_now + - ((tcp_ecn_timeout * 60 * TCP_RETRANSHZ) << (tpheur->th_ecn_aggressive)); + (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_aggressive)); /* * Ugly way to prevent integer overflow... limit to prevent in @@ -718,7 +923,15 @@ void tcp_heuristic_ecn_aggressive(struct tcpcb *tp) tcp_heuristic_unlock(head); } -boolean_t tcp_heuristic_do_tfo(struct tcpcb *tp) +void tcp_heuristic_ecn_aggressive(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + tcp_heuristic_ecn_aggressive_common(&tcks); +} + +static boolean_t tcp_heuristic_do_tfo_common(struct tcp_cache_key_src *tcks) { struct tcp_heuristics_head *head; struct tcp_heuristic *tpheur; @@ -727,85 +940,75 @@ boolean_t tcp_heuristic_do_tfo(struct tcpcb *tp) return (TRUE); /* Get the tcp-heuristic. */ - tpheur = tcp_getheuristic_with_lock(tp, 0, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 0, &head); if (tpheur == NULL) return (TRUE); - if (tpheur->th_tfo_aggressive_fallback) { - /* Aggressive fallback - don't do TFO anymore... :'( */ - tcp_heuristic_unlock(head); - return (FALSE); - } + if (tpheur->th_tfo_in_backoff == 0) + goto tfo_ok; - if (tpheur->th_tfo_cookie_loss >= TFO_MAX_COOKIE_LOSS && - (tpheur->th_tfo_fallback_trials < tcp_tfo_fallback_min || - TSTMP_GT(tpheur->th_tfo_cookie_backoff, tcp_now))) { - /* - * So, when we are in SYN-loss mode we try to stop using TFO - * for the next 'tcp_tfo_fallback_min' connections. That way, - * we are sure that never more than 1 out of tcp_tfo_fallback_min - * connections will suffer from our nice little middelbox. - * - * After that we first wait for 2 minutes. If we fail again, - * we wait for yet another 60 minutes. - */ - tpheur->th_tfo_fallback_trials++; - if (tpheur->th_tfo_fallback_trials >= tcp_tfo_fallback_min && - !tpheur->th_tfo_in_backoff) { - if (tpheur->th_tfo_cookie_loss == TFO_MAX_COOKIE_LOSS) - /* Backoff for 2 minutes */ - tpheur->th_tfo_cookie_backoff = tcp_now + (60 * 2 * TCP_RETRANSHZ); - else - /* Backoff for 60 minutes */ - tpheur->th_tfo_cookie_backoff = tcp_now + (60 * 60 * TCP_RETRANSHZ); - - tpheur->th_tfo_in_backoff = 1; - } + if (TSTMP_GT(tcp_now, tpheur->th_tfo_backoff_until)) { + tpheur->th_tfo_in_backoff = 0; + tpheur->th_tfo_enabled_time = tcp_now; - tcp_heuristic_unlock(head); - return (FALSE); + goto tfo_ok; } - /* - * We give it a new shot, set trials back to 0. This allows to - * start counting again from zero in case we get yet another SYN-loss - */ - tpheur->th_tfo_fallback_trials = 0; - tpheur->th_tfo_in_backoff = 0; - - if (tpheur->th_tfo_rcv_middlebox_supp) - tp->t_tfo_flags |= TFO_F_NO_RCVPROBING; - if (tpheur->th_tfo_snd_middlebox_supp) - tp->t_tfo_flags |= TFO_F_NO_SNDPROBING; - tcp_heuristic_unlock(head); + return (FALSE); +tfo_ok: + tcp_heuristic_unlock(head); return (TRUE); } +boolean_t tcp_heuristic_do_tfo(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + if (tcp_heuristic_do_tfo_common(&tcks)) + return (TRUE); + + return (FALSE); +} + boolean_t tcp_heuristic_do_mptcp(struct tcpcb *tp) { - struct tcp_heuristics_head *head; + struct tcp_cache_key_src tcks; + struct tcp_heuristics_head *head = NULL; struct tcp_heuristic *tpheur; - boolean_t ret = TRUE; if (disable_tcp_heuristics) return (TRUE); + tcp_cache_key_src_create(tp, &tcks); + /* Get the tcp-heuristic. */ - tpheur = tcp_getheuristic_with_lock(tp, 0, &head); + tpheur = tcp_getheuristic_with_lock(&tcks, 0, &head); if (tpheur == NULL) - return ret; + return (TRUE); if (TSTMP_GT(tpheur->th_mptcp_backoff, tcp_now)) - ret = FALSE; + goto fallback; tcp_heuristic_unlock(head); - return (ret); + return (TRUE); + +fallback: + if (head) + tcp_heuristic_unlock(head); + + if (tptomptp(tp)->mpt_mpte->mpte_flags & MPTE_FIRSTPARTY) + tcpstat.tcps_mptcp_fp_heuristic_fallback++; + else + tcpstat.tcps_mptcp_heuristic_fallback++; + + return (FALSE); } -boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp) +static boolean_t tcp_heuristic_do_ecn_common(struct tcp_cache_key_src *tcks) { struct tcp_heuristics_head *head; struct tcp_heuristic *tpheur; @@ -815,7 +1018,7 @@ boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp) return (TRUE); /* Get the tcp-heuristic. */ - tpheur = tcp_getheuristic_with_lock(tp, 0, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 0, &head); if (tpheur == NULL) return ret; @@ -827,6 +1030,8 @@ boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp) tpheur->th_ecn_droprst = 0; if (tpheur->th_ecn_droprxmt >= ECN_RETRY_LIMIT) tpheur->th_ecn_droprxmt = 0; + if (tpheur->th_ecn_synrst >= ECN_RETRY_LIMIT) + tpheur->th_ecn_synrst = 0; } tcp_heuristic_unlock(head); @@ -834,6 +1039,152 @@ boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp) return (ret); } +boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + return tcp_heuristic_do_ecn_common(&tcks); +} + +boolean_t tcp_heuristic_do_ecn_with_address(struct ifnet *ifp, + union sockaddr_in_4_6 *local_address) +{ + struct tcp_cache_key_src tcks; + + memset(&tcks, 0, sizeof(tcks)); + tcks.ifp = ifp; + + calculate_tcp_clock(); + + if (local_address->sa.sa_family == AF_INET6) { + memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr)); + tcks.af = AF_INET6; + } else if (local_address->sa.sa_family == AF_INET) { + memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr)); + tcks.af = AF_INET; + } + + return tcp_heuristic_do_ecn_common(&tcks); +} + +void tcp_heuristics_ecn_update(struct necp_tcp_ecn_cache *necp_buffer, + struct ifnet *ifp, union sockaddr_in_4_6 *local_address) +{ + struct tcp_cache_key_src tcks; + + memset(&tcks, 0, sizeof(tcks)); + tcks.ifp = ifp; + + calculate_tcp_clock(); + + if (local_address->sa.sa_family == AF_INET6) { + memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr)); + tcks.af = AF_INET6; + } else if (local_address->sa.sa_family == AF_INET) { + memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr)); + tcks.af = AF_INET; + } + + if (necp_buffer->necp_tcp_ecn_heuristics_success) { + tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_ECN); + } else if (necp_buffer->necp_tcp_ecn_heuristics_loss) { + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN); + } else if (necp_buffer->necp_tcp_ecn_heuristics_drop_rst) { + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRST); + } else if (necp_buffer->necp_tcp_ecn_heuristics_drop_rxmt) { + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRXMT); + } else if (necp_buffer->necp_tcp_ecn_heuristics_syn_rst) { + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_SYNRST); + } else if (necp_buffer->necp_tcp_ecn_heuristics_aggressive) { + tcp_heuristic_ecn_aggressive_common(&tcks); + } + + return; +} + +boolean_t tcp_heuristic_do_tfo_with_address(struct ifnet *ifp, + union sockaddr_in_4_6 *local_address, union sockaddr_in_4_6 *remote_address, + u_int8_t *cookie, u_int8_t *cookie_len) +{ + struct tcp_cache_key_src tcks; + + memset(&tcks, 0, sizeof(tcks)); + tcks.ifp = ifp; + + calculate_tcp_clock(); + + if (remote_address->sa.sa_family == AF_INET6) { + memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr)); + memcpy(&tcks.faddr.addr6, &remote_address->sin6.sin6_addr, sizeof(struct in6_addr)); + tcks.af = AF_INET6; + } else if (remote_address->sa.sa_family == AF_INET) { + memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr)); + memcpy(&tcks.faddr.addr, &remote_address->sin.sin_addr, sizeof(struct in_addr)); + tcks.af = AF_INET; + } + + if (tcp_heuristic_do_tfo_common(&tcks)) { + if (!tcp_cache_get_cookie_common(&tcks, cookie, cookie_len)) { + *cookie_len = 0; + } + return TRUE; + } + + return FALSE; +} + +void tcp_heuristics_tfo_update(struct necp_tcp_tfo_cache *necp_buffer, + struct ifnet *ifp, union sockaddr_in_4_6 *local_address, + union sockaddr_in_4_6 *remote_address) +{ + struct tcp_cache_key_src tcks; + + memset(&tcks, 0, sizeof(tcks)); + tcks.ifp = ifp; + + calculate_tcp_clock(); + + if (remote_address->sa.sa_family == AF_INET6) { + memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr)); + memcpy(&tcks.faddr.addr6, &remote_address->sin6.sin6_addr, sizeof(struct in6_addr)); + tcks.af = AF_INET6; + } else if (remote_address->sa.sa_family == AF_INET) { + memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr)); + memcpy(&tcks.faddr.addr, &remote_address->sin.sin_addr, sizeof(struct in_addr)); + tcks.af = AF_INET; + } + + if (necp_buffer->necp_tcp_tfo_heuristics_success) + tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_DATA | + TCPCACHE_F_TFO_REQ_RST | TCPCACHE_F_TFO_DATA_RST); + + if (necp_buffer->necp_tcp_tfo_heuristics_success_req) + tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_REQ_RST); + + if (necp_buffer->necp_tcp_tfo_heuristics_loss) + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_DATA); + + if (necp_buffer->necp_tcp_tfo_heuristics_loss_req) + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ); + + if (necp_buffer->necp_tcp_tfo_heuristics_rst_data) + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ_RST | TCPCACHE_F_TFO_DATA_RST); + + if (necp_buffer->necp_tcp_tfo_heuristics_rst_req) + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ_RST); + + if (necp_buffer->necp_tcp_tfo_heuristics_middlebox) + tcp_heuristic_tfo_middlebox_common(&tcks); + + if (necp_buffer->necp_tcp_tfo_cookie_len != 0) { + tcp_cache_set_cookie_common(&tcks, + necp_buffer->necp_tcp_tfo_cookie, necp_buffer->necp_tcp_tfo_cookie_len); + } + + return; +} + static void sysctl_cleartfocache(void) { int i; diff --git a/bsd/netinet/tcp_cache.h b/bsd/netinet/tcp_cache.h index 4516d7578..bd7044ea3 100644 --- a/bsd/netinet/tcp_cache.h +++ b/bsd/netinet/tcp_cache.h @@ -42,10 +42,9 @@ extern int tcp_cache_get_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t *len) extern unsigned int tcp_cache_get_cookie_len(struct tcpcb *tp); extern void tcp_heuristic_tfo_loss(struct tcpcb *tp); +extern void tcp_heuristic_tfo_rst(struct tcpcb *tp); extern void tcp_heuristic_mptcp_loss(struct tcpcb *tp); extern void tcp_heuristic_ecn_loss(struct tcpcb *tp); -extern void tcp_heuristic_tfo_snd_good(struct tcpcb *tp); -extern void tcp_heuristic_tfo_rcv_good(struct tcpcb *tp); extern void tcp_heuristic_tfo_middlebox(struct tcpcb *tp); extern void tcp_heuristic_ecn_aggressive(struct tcpcb *tp); extern void tcp_heuristic_tfo_success(struct tcpcb *tp); @@ -56,6 +55,18 @@ extern boolean_t tcp_heuristic_do_mptcp(struct tcpcb *tp); extern boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp); extern void tcp_heuristic_ecn_droprst(struct tcpcb *tp); extern void tcp_heuristic_ecn_droprxmt(struct tcpcb *tp); +extern void tcp_heuristic_ecn_synrst(struct tcpcb *tp); + +extern boolean_t tcp_heuristic_do_ecn_with_address(struct ifnet *ifp, + union sockaddr_in_4_6 *local_address); +extern void tcp_heuristics_ecn_update(struct necp_tcp_ecn_cache *necp_buffer, + struct ifnet *ifp, union sockaddr_in_4_6 *local_address); +extern boolean_t tcp_heuristic_do_tfo_with_address(struct ifnet *ifp, + union sockaddr_in_4_6 *local_address, union sockaddr_in_4_6 *remote_address, + u_int8_t *cookie, u_int8_t *cookie_len); +extern void tcp_heuristics_tfo_update(struct necp_tcp_tfo_cache *necp_buffer, + struct ifnet *ifp, union sockaddr_in_4_6 *local_address, + union sockaddr_in_4_6 *remote_address); extern void tcp_cache_init(void); diff --git a/bsd/netinet/tcp_cc.c b/bsd/netinet/tcp_cc.c index a15fd6a07..1f634a174 100644 --- a/bsd/netinet/tcp_cc.c +++ b/bsd/netinet/tcp_cc.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -76,9 +76,8 @@ struct tcp_cc_debug_state { } u; }; -int tcp_cc_debug = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, cc_debug, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_cc_debug, 0, "Enable debug data collection"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, cc_debug, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_cc_debug, 0, "Enable debug data collection"); extern struct tcp_cc_algo tcp_cc_newreno; SYSCTL_INT(_net_inet_tcp, OID_AUTO, newreno_sockets, @@ -95,9 +94,8 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, cubic_sockets, CTLFLAG_RD | CTLFLAG_LOCKED,&tcp_cc_cubic.num_sockets, 0, "Number of sockets using cubic"); -int tcp_use_newreno = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, use_newreno, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_use_newreno, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, use_newreno, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_use_newreno, 0, "Use TCP NewReno by default"); static int tcp_check_cwnd_nonvalidated = 1; @@ -299,7 +297,7 @@ void tcp_bad_rexmt_fix_sndbuf(struct tcpcb *tp) sb = &tp->t_inpcb->inp_socket->so_snd; if ((sb->sb_flags & (SB_TRIM|SB_AUTOSIZE)) == (SB_TRIM|SB_AUTOSIZE)) { /* - * If there was a retransmission that was not necessary + * If there was a retransmission that was not necessary * then the size of socket buffer can be restored to * what it was before */ @@ -426,11 +424,19 @@ tcp_cc_after_idle_stretchack(struct tcpcb *tp) inline uint32_t tcp_cc_is_cwnd_nonvalidated(struct tcpcb *tp) { + struct socket *so = tp->t_inpcb->inp_socket; if (tp->t_pipeack == 0 || tcp_check_cwnd_nonvalidated == 0) { tp->t_flagsext &= ~TF_CWND_NONVALIDATED; return (0); } - if (tp->t_pipeack >= (tp->snd_cwnd) >> 1) + + /* + * The congestion window is validated if the number of bytes acked + * is more than half of the current window or if there is more + * data to send in the send socket buffer + */ + if (tp->t_pipeack >= (tp->snd_cwnd >> 1) || + (so != NULL && so->so_snd.sb_cc > tp->snd_cwnd)) tp->t_flagsext &= ~TF_CWND_NONVALIDATED; else tp->t_flagsext |= TF_CWND_NONVALIDATED; diff --git a/bsd/netinet/tcp_cc.h b/bsd/netinet/tcp_cc.h index e9df6b451..7c83900d1 100644 --- a/bsd/netinet/tcp_cc.h +++ b/bsd/netinet/tcp_cc.h @@ -74,6 +74,8 @@ #define TCP_CA_NAME_MAX 16 /* Maximum characters in the name of a CC algorithm */ +extern int tcp_recv_bg; + /* * Structure to hold definition various actions defined by a congestion * control algorithm for TCP. This can be used to change the congestion diff --git a/bsd/netinet/tcp_cubic.c b/bsd/netinet/tcp_cubic.c index 29a3aed78..c9ce59c04 100644 --- a/bsd/netinet/tcp_cubic.c +++ b/bsd/netinet/tcp_cubic.c @@ -89,20 +89,14 @@ const float tcp_cubic_backoff = 0.2; /* multiplicative decrease factor */ const float tcp_cubic_coeff = 0.4; const float tcp_cubic_fast_convergence_factor = 0.875; -static int tcp_cubic_tcp_friendliness = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, cubic_tcp_friendliness, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_cubic_tcp_friendliness, 0, - "Enable TCP friendliness"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, cubic_tcp_friendliness, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, tcp_cubic_tcp_friendliness, 0, "Enable TCP friendliness"); -static int tcp_cubic_fast_convergence = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, cubic_fast_convergence, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_cubic_fast_convergence, 0, - "Enable fast convergence"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, cubic_fast_convergence, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, tcp_cubic_fast_convergence, 0, "Enable fast convergence"); -static int tcp_cubic_use_minrtt = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, cubic_use_minrtt, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_cubic_use_minrtt, 0, - "use a min of 5 sec rtt"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, cubic_use_minrtt, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, tcp_cubic_use_minrtt, 0, "use a min of 5 sec rtt"); static int tcp_cubic_init(struct tcpcb *tp) { @@ -214,7 +208,7 @@ tcp_cubic_update(struct tcpcb *tp, u_int32_t rtt) var = (elapsed_time - tp->t_ccstate->cub_epoch_period) / TCP_RETRANSHZ; var = var * var * var * (tcp_cubic_coeff * tp->t_maxseg); - tp->t_ccstate->cub_target_win = tp->t_ccstate->cub_origin_point + var; + tp->t_ccstate->cub_target_win = (u_int32_t)(tp->t_ccstate->cub_origin_point + var); return (tp->t_ccstate->cub_target_win); } @@ -355,7 +349,7 @@ tcp_cubic_ack_rcvd(struct tcpcb *tp, struct tcphdr *th) static void tcp_cubic_pre_fr(struct tcpcb *tp) { - uint32_t win, avg; + u_int32_t win, avg; int32_t dev; tp->t_ccstate->cub_epoch_start = 0; tp->t_ccstate->cub_tcp_win = 0; @@ -382,8 +376,8 @@ tcp_cubic_pre_fr(struct tcpcb *tp) */ if (win < tp->t_ccstate->cub_last_max && tcp_cubic_fast_convergence == 1) - tp->t_ccstate->cub_last_max = win * - tcp_cubic_fast_convergence_factor; + tp->t_ccstate->cub_last_max = (u_int32_t)(win * + tcp_cubic_fast_convergence_factor); else tp->t_ccstate->cub_last_max = win; @@ -429,7 +423,7 @@ tcp_cubic_pre_fr(struct tcpcb *tp) } /* Backoff congestion window by tcp_cubic_backoff factor */ - win = win - (win * tcp_cubic_backoff); + win = (u_int32_t)(win - (win * tcp_cubic_backoff)); win = (win / tp->t_maxseg); if (win < 2) win = 2; diff --git a/bsd/netinet/tcp_fsm.h b/bsd/netinet/tcp_fsm.h index b963d865f..2a46e6ca5 100644 --- a/bsd/netinet/tcp_fsm.h +++ b/bsd/netinet/tcp_fsm.h @@ -104,6 +104,10 @@ #define TCPS_HAVERCVDSYN(s) ((s) >= TCPS_SYN_RECEIVED) #define TCPS_HAVEESTABLISHED(s) ((s) >= TCPS_ESTABLISHED) #define TCPS_HAVERCVDFIN(s) ((s) >= TCPS_TIME_WAIT) +#define TCPS_HAVERCVDFIN2(s) ((s) == TCPS_CLOSE_WAIT || \ + (s) == TCPS_CLOSING || \ + (s) == TCPS_LAST_ACK || \ + (s) == TCPS_TIME_WAIT) #ifdef KERNEL_PRIVATE #ifdef TCPOUTFLAGS diff --git a/bsd/netinet/tcp_input.c b/bsd/netinet/tcp_input.c index 7767910b8..741aa00d5 100644 --- a/bsd/netinet/tcp_input.c +++ b/bsd/netinet/tcp_input.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -79,7 +79,10 @@ #include #include #include +#if !CONFIG_EMBEDDED #include +#endif +#include #include /* before tcp_seq.h, for tcp_random18() */ #include @@ -156,8 +159,6 @@ struct tcphdr tcp_savetcp; #define TCP_RECV_THROTTLE_WIN (5 * TCP_RETRANSHZ) #define TCP_STRETCHACK_ENABLE_PKTCNT 2000 -tcp_cc tcp_ccgen; - struct tcpstat tcpstat; static int log_in_vain = 0; @@ -170,24 +171,20 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, blackhole, CTLFLAG_RW | CTLFLAG_LOCKED, &blackhole, 0, "Do not send RST when dropping refused connections"); -int tcp_delack_enabled = 3; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, delayed_ack, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_delack_enabled, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, delayed_ack, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_delack_enabled, 3, "Delay ACK to try and piggyback it onto a data packet"); -int tcp_lq_overflow = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_lq_overflow, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_lq_overflow, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, tcp_lq_overflow, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_lq_overflow, 1, "Listen Queue Overflow"); -int tcp_recv_bg = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, recvbg, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_recv_bg, 0, "Receive background"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, recvbg, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_recv_bg, 0, "Receive background"); #if TCP_DROP_SYNFIN -static int drop_synfin = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, drop_synfin, - CTLFLAG_RW | CTLFLAG_LOCKED, &drop_synfin, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, drop_synfin, + CTLFLAG_RW | CTLFLAG_LOCKED, static int, drop_synfin, 1, "Drop TCP packets with SYN+FIN set"); #endif @@ -200,32 +197,27 @@ SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, overflows, "Global number of TCP Segment Reassembly Queue Overflows"); -__private_extern__ int slowlink_wsize = 8192; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowlink_wsize, - CTLFLAG_RW | CTLFLAG_LOCKED, - &slowlink_wsize, 0, "Maximum advertised window size for slowlink"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, slowlink_wsize, CTLFLAG_RW | CTLFLAG_LOCKED, + __private_extern__ int, slowlink_wsize, 8192, + "Maximum advertised window size for slowlink"); -int maxseg_unacked = 8; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, maxseg_unacked, - CTLFLAG_RW | CTLFLAG_LOCKED, &maxseg_unacked, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, maxseg_unacked, + CTLFLAG_RW | CTLFLAG_LOCKED, int, maxseg_unacked, 8, "Maximum number of outstanding segments left unacked"); -int tcp_do_rfc3465 = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3465, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_do_rfc3465, 0, ""); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rfc3465, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_do_rfc3465, 1, ""); -int tcp_do_rfc3465_lim2 = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3465_lim2, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_rfc3465_lim2, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rfc3465_lim2, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_do_rfc3465_lim2, 1, "Appropriate bytes counting w/ L=2*SMSS"); int rtt_samples_per_slot = 20; -int tcp_allowed_iaj = ALLOWED_IAJ; int tcp_acc_iaj_high_thresh = ACC_IAJ_HIGH_THRESH; u_int32_t tcp_autorcvbuf_inc_shift = 3; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, recv_allowed_iaj, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_allowed_iaj, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, recv_allowed_iaj, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_allowed_iaj, ALLOWED_IAJ, "Allowed inter-packet arrival jiter"); #if (DEVELOPMENT || DEBUG) SYSCTL_INT(_net_inet_tcp, OID_AUTO, acc_iaj_high_thresh, @@ -237,14 +229,12 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, autorcvbufincshift, "Shift for increment in receive socket buffer size"); #endif /* (DEVELOPMENT || DEBUG) */ -u_int32_t tcp_do_autorcvbuf = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, doautorcvbuf, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_autorcvbuf, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, doautorcvbuf, + CTLFLAG_RW | CTLFLAG_LOCKED, u_int32_t, tcp_do_autorcvbuf, 1, "Enable automatic socket buffer tuning"); -u_int32_t tcp_autorcvbuf_max = 512 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, autorcvbufmax, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_autorcvbuf_max, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, autorcvbufmax, + CTLFLAG_RW | CTLFLAG_LOCKED, u_int32_t, tcp_autorcvbuf_max, 512 * 1024, "Maximum receive socket buffer size"); u_int32_t tcp_autorcvbuf_max_ca = 512 * 1024; @@ -254,7 +244,11 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, autorcvbufmaxca, "Maximum receive socket buffer size"); #endif /* (DEBUG || DEVELOPMENT) */ +#if CONFIG_EMBEDDED +int sw_lro = 1; +#else int sw_lro = 0; +#endif /* !CONFIG_EMBEDDED */ SYSCTL_INT(_net_inet_tcp, OID_AUTO, lro, CTLFLAG_RW | CTLFLAG_LOCKED, &sw_lro, 0, "Used to coalesce TCP packets"); @@ -268,8 +262,6 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, lro_startcnt, CTLFLAG_RW | CTLFLAG_LOCKED, &lro_start, 0, "Segments for starting LRO computed as power of 2"); -extern int tcp_do_autosendbuf; - int limited_txmt = 1; int early_rexmt = 1; int sack_ackadv = 1; @@ -291,7 +283,13 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, sack_ackadv, SYSCTL_INT(_net_inet_tcp, OID_AUTO, dsack_enable, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_dsack_enable, 0, "use DSACK TCP option to report duplicate segments"); + #endif /* (DEVELOPMENT || DEBUG) */ +int tcp_disable_access_to_stats = 1; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, disable_access_to_stats, + CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_disable_access_to_stats, 0, + "Disable access to tcpstat"); + extern int tcp_TCPTV_MIN; extern int tcp_acc_iaj_high; @@ -330,6 +328,8 @@ static void compute_iaj_meat(struct tcpcb *tp, uint32_t cur_iaj); static inline unsigned int tcp_maxmtu6(struct rtentry *); #endif +unsigned int get_maxmtu(struct rtentry *); + static void tcp_sbrcv_grow(struct tcpcb *tp, struct sockbuf *sb, struct tcpopt *to, u_int32_t tlen, u_int32_t rcvbuf_max); void tcp_sbrcv_trim(struct tcpcb *tp, struct sockbuf *sb); @@ -377,9 +377,8 @@ extern void ipfw_stealth_stats_incr_tcp(void); int tcp_rcvunackwin = TCPTV_UNACKWIN; int tcp_maxrcvidle = TCPTV_MAXRCVIDLE; -int tcp_rcvsspktcnt = TCP_RCV_SS_PKTCOUNT; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rcvsspktcnt, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_rcvsspktcnt, 0, "packets to be seen before receiver stretches acks"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rcvsspktcnt, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_rcvsspktcnt, TCP_RCV_SS_PKTCOUNT, "packets to be seen before receiver stretches acks"); #define DELAY_ACK(tp, th) \ (CC_ALGO(tp)->delay_ack != NULL && CC_ALGO(tp)->delay_ack(tp, th)) @@ -560,8 +559,12 @@ tcp_bwmeas_check(struct tcpcb *tp) { int32_t bw_meas_bytes; uint32_t bw, bytes, elapsed_time; + + if (SEQ_LEQ(tp->snd_una, tp->t_bwmeas->bw_start)) + return; + bw_meas_bytes = tp->snd_una - tp->t_bwmeas->bw_start; - if ((tp->t_flagsext & TF_BWMEAS_INPROGRESS) != 0 && + if ((tp->t_flagsext & TF_BWMEAS_INPROGRESS) && bw_meas_bytes >= (int32_t)(tp->t_bwmeas->bw_size)) { bytes = bw_meas_bytes; elapsed_time = tcp_now - tp->t_bwmeas->bw_ts; @@ -570,10 +573,22 @@ tcp_bwmeas_check(struct tcpcb *tp) if ( bw > 0) { if (tp->t_bwmeas->bw_sndbw > 0) { tp->t_bwmeas->bw_sndbw = - (((tp->t_bwmeas->bw_sndbw << 3) - tp->t_bwmeas->bw_sndbw) + bw) >> 3; + (((tp->t_bwmeas->bw_sndbw << 3) + - tp->t_bwmeas->bw_sndbw) + + bw) >> 3; } else { tp->t_bwmeas->bw_sndbw = bw; } + + /* Store the maximum value */ + if (tp->t_bwmeas->bw_sndbw_max == 0) { + tp->t_bwmeas->bw_sndbw_max = + tp->t_bwmeas->bw_sndbw; + } else { + tp->t_bwmeas->bw_sndbw_max = + max(tp->t_bwmeas->bw_sndbw, + tp->t_bwmeas->bw_sndbw_max); + } } } tp->t_flagsext &= ~(TF_BWMEAS_INPROGRESS); @@ -692,6 +707,7 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m, INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, *tlenp); tp->t_stat.rxduplicatebytes += *tlenp; + inp_set_activity_bitmap(inp); } m_freem(m); zfree(tcp_reass_zone, te); @@ -719,6 +735,7 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m, INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, *tlenp); tp->t_stat.rxoutoforderbytes += *tlenp; + inp_set_activity_bitmap(inp); } /* @@ -1171,6 +1188,19 @@ tcp_sbrcv_grow(struct tcpcb *tp, struct sockbuf *sbrcv, sbrcv->sb_hiwat + rcvbuf_inc, (tp->rfbuf_cnt * 2), rcvbuf_max); } + /* Measure instantaneous receive bandwidth */ + if (tp->t_bwmeas != NULL && tp->rfbuf_cnt > 0 && + TSTMP_GT(tcp_now, tp->rfbuf_ts)) { + u_int32_t rcv_bw; + rcv_bw = tp->rfbuf_cnt / + (int)(tcp_now - tp->rfbuf_ts); + if (tp->t_bwmeas->bw_rcvbw_max == 0) { + tp->t_bwmeas->bw_rcvbw_max = rcv_bw; + } else { + tp->t_bwmeas->bw_rcvbw_max = max( + tp->t_bwmeas->bw_rcvbw_max, rcv_bw); + } + } goto out; } else { tp->rfbuf_cnt += pktlen; @@ -1725,30 +1755,24 @@ tcp_tfo_synack(struct tcpcb *tp, struct tcpopt *to) static void tcp_tfo_rcv_probe(struct tcpcb *tp, int tlen) { - if (tlen == 0) { - tp->t_tfo_probe_state = TFO_PROBE_PROBING; + if (tlen != 0) + return; - /* - * We send the probe out rather quickly (after one RTO). It does not - * really hurt that much, it's only one additional segment on the wire. - */ - tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, (TCP_REXMTVAL(tp))); - } else { - /* If SYN/ACK+data, don't probe. We got the data! */ - tcp_heuristic_tfo_rcv_good(tp); - } + tp->t_tfo_probe_state = TFO_PROBE_PROBING; + + /* + * We send the probe out rather quickly (after one RTO). It does not + * really hurt that much, it's only one additional segment on the wire. + */ + tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, (TCP_REXMTVAL(tp))); } static void tcp_tfo_rcv_data(struct tcpcb *tp) { /* Transition from PROBING to NONE as data has been received */ - if (tp->t_tfo_probe_state >= TFO_PROBE_PROBING) { + if (tp->t_tfo_probe_state >= TFO_PROBE_PROBING) tp->t_tfo_probe_state = TFO_PROBE_NONE; - - /* Data has been received - we are good to go! */ - tcp_heuristic_tfo_rcv_good(tp); - } } static void @@ -1789,6 +1813,9 @@ tcp_update_window(struct tcpcb *tp, int thflags, struct tcphdr * th, tp->snd_wl2 = th->th_ack; if (tp->snd_wnd > tp->max_sndwnd) tp->max_sndwnd = tp->snd_wnd; + + if (tp->t_inpcb->inp_socket->so_flags & SOF_MP_SUBFLOW) + mptcp_update_window_fallback(tp); return (true); } return (false); @@ -1920,7 +1947,7 @@ tcp_input(struct mbuf *m, int off0) * Note: IP leaves IP header in first mbuf. */ if (off0 > sizeof (struct ip)) { - ip_stripoptions(m, (struct mbuf *)0); + ip_stripoptions(m); off0 = sizeof(struct ip); } if (m->m_len < sizeof (struct tcpiphdr)) { @@ -2209,30 +2236,39 @@ findpcb: goto dropnosock; } - tcp_lock(so, 1, 0); + socket_lock(so, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(so, 1, (void *)2); + socket_unlock(so, 1); inp = NULL; // pretend we didn't find it goto dropnosock; } #if NECP -#if INET6 - if (isipv6) { - if (!necp_socket_is_allowed_to_send_recv_v6(inp, - th->th_dport, th->th_sport, &ip6->ip6_dst, - &ip6->ip6_src, ifp, NULL, NULL)) { - IF_TCP_STATINC(ifp, badformatipsec); + if (so->so_state & SS_ISCONNECTED) { + // Connected TCP sockets have a fully-bound local and remote, + // so the policy check doesn't need to override addresses + if (!necp_socket_is_allowed_to_send_recv(inp, NULL, NULL)) { + IF_TCP_STATINC(ifp, badformat); goto drop; } - } else + } else { +#if INET6 + if (isipv6) { + if (!necp_socket_is_allowed_to_send_recv_v6(inp, + th->th_dport, th->th_sport, &ip6->ip6_dst, + &ip6->ip6_src, ifp, NULL, NULL)) { + IF_TCP_STATINC(ifp, badformat); + goto drop; + } + } else #endif - { - if (!necp_socket_is_allowed_to_send_recv_v4(inp, - th->th_dport, th->th_sport, &ip->ip_dst, &ip->ip_src, - ifp, NULL, NULL)) { - IF_TCP_STATINC(ifp, badformatipsec); - goto drop; + { + if (!necp_socket_is_allowed_to_send_recv_v4(inp, + th->th_dport, th->th_sport, &ip->ip_dst, &ip->ip_src, + ifp, NULL, NULL)) { + IF_TCP_STATINC(ifp, badformat); + goto drop; + } } } #endif /* NECP */ @@ -2433,10 +2469,10 @@ findpcb: tp = intotcpcb(inp); oso = so; - tcp_unlock(so, 0, 0); /* Unlock but keep a reference on listener for now */ + socket_unlock(so, 0); /* Unlock but keep a reference on listener for now */ so = so2; - tcp_lock(so, 1, 0); + socket_lock(so, 1); /* * Mark socket as temporary until we're * committed to keeping it. The code at @@ -2503,38 +2539,38 @@ findpcb: #endif /* INET6 */ inp->inp_laddr.s_addr = INADDR_ANY; inp->inp_lport = 0; - tcp_lock(oso, 0, 0); /* release ref on parent */ - tcp_unlock(oso, 1, 0); + socket_lock(oso, 0); /* release ref on parent */ + socket_unlock(oso, 1); goto drop; } #if INET6 if (isipv6) { - /* - * Inherit socket options from the listening - * socket. - * Note that in6p_inputopts are not (even - * should not be) copied, since it stores + /* + * Inherit socket options from the listening + * socket. + * Note that in6p_inputopts are not (even + * should not be) copied, since it stores * previously received options and is used to - * detect if each new option is different than - * the previous one and hence should be passed - * to a user. - * If we copied in6p_inputopts, a user would - * not be able to receive options just after - * calling the accept system call. - */ + * detect if each new option is different than + * the previous one and hence should be passed + * to a user. + * If we copied in6p_inputopts, a user would + * not be able to receive options just after + * calling the accept system call. + */ inp->inp_flags |= oinp->inp_flags & INP_CONTROLOPTS; - if (oinp->in6p_outputopts) - inp->in6p_outputopts = - ip6_copypktopts(oinp->in6p_outputopts, - M_NOWAIT); + if (oinp->in6p_outputopts) + inp->in6p_outputopts = + ip6_copypktopts(oinp->in6p_outputopts, + M_NOWAIT); } else #endif /* INET6 */ { inp->inp_options = ip_srcroute(); inp->inp_ip_tos = oinp->inp_ip_tos; } - tcp_lock(oso, 0, 0); + socket_lock(oso, 0); #if IPSEC /* copy old policy into new socket's */ if (sotoinpcb(oso)->inp_sp) @@ -2565,15 +2601,14 @@ findpcb: tp0->t_inpcb->inp_flags2 & INP2_KEEPALIVE_OFFLOAD; /* now drop the reference on the listener */ - tcp_unlock(oso, 1, 0); + socket_unlock(oso, 1); tcp_set_max_rwinscale(tp, so, TCP_AUTORCVBUF_MAX(ifp)); KERNEL_DEBUG(DBG_FNC_TCP_NEWCONN | DBG_FUNC_END,0,0,0,0,0); } } - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); if (tp->t_state == TCPS_ESTABLISHED && tlen > 0) { /* @@ -2693,9 +2728,13 @@ findpcb: * Segment received on connection. * Reset idle time and keep-alive timer. */ - if (TCPS_HAVEESTABLISHED(tp->t_state)) + if (TCPS_HAVEESTABLISHED(tp->t_state)) { tcp_keepalive_reset(tp); + if (tp->t_mpsub) + mptcp_reset_keepalive(tp); + } + /* * Process options if not in LISTEN state, * else do it below (after getting remote address). @@ -2707,7 +2746,7 @@ findpcb: tp->t_flags |= TF_ACKNOW; (void) tcp_output(tp); tcp_check_timer_state(tp); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; @@ -2928,7 +2967,7 @@ findpcb: tcp_tfo_rcv_ack(tp, th); tcp_check_timer_state(tp); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; } @@ -2942,7 +2981,7 @@ findpcb: */ /* - * If this is a connection in steady state, start + * If this is a connection in steady state, start * coalescing packets belonging to this flow. */ if (turnoff_lro) { @@ -2992,6 +3031,7 @@ findpcb: } INP_ADD_STAT(inp, cell, wifi, wired,rxbytes, tlen); + inp_set_activity_bitmap(inp); } /* @@ -3050,7 +3090,7 @@ findpcb: tcp_tfo_rcv_data(tp); tcp_check_timer_state(tp); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; } @@ -3062,8 +3102,7 @@ findpcb: * Receive window is amount of space in rcv queue, * but not less than advertised window. */ - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); win = tcp_sbspace(tp); if (win < 0) win = 0; @@ -3079,12 +3118,11 @@ findpcb: */ if ((tp->t_mpflags & TMPF_MPTCP_TRUE) && (mp_tp = tptomptp(tp))) { - MPT_LOCK(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); if (tp->rcv_wnd > mp_tp->mpt_rcvwnd) { tp->rcv_wnd = imax(mp_tp->mpt_rcvwnd, (int)(tp->rcv_adv - tp->rcv_nxt)); tcpstat.tcps_mp_reducedwin++; } - MPT_UNLOCK(mp_tp); } #endif /* MPTCP */ @@ -3105,8 +3143,7 @@ findpcb: struct sockaddr_in6 *sin6; #endif - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); #if INET6 if (isipv6) { MALLOC(sin6, struct sockaddr_in6 *, sizeof *sin6, @@ -3131,11 +3168,9 @@ findpcb: } else #endif { - lck_mtx_assert( - &((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); MALLOC(sin, struct sockaddr_in *, sizeof *sin, M_SONAME, - M_NOWAIT); + M_NOWAIT); if (sin == NULL) goto drop; sin->sin_family = AF_INET; @@ -3255,14 +3290,21 @@ findpcb: } if (thflags & TH_RST) { if ((thflags & TH_ACK) != 0) { -#if MPTCP - if ((so->so_flags & SOF_MPTCP_FASTJOIN) && - SEQ_GT(th->th_ack, tp->iss+1)) { - so->so_flags &= ~SOF_MPTCP_FASTJOIN; - /* ignore the RST and retransmit SYN */ - goto drop; + if (tfo_enabled(tp)) + tcp_heuristic_tfo_rst(tp); + if ((tp->ecn_flags & (TE_SETUPSENT | TE_RCVD_SYN_RST)) == TE_SETUPSENT) { + /* + * On local connections, send + * non-ECN syn one time before + * dropping the connection + */ + if (tp->t_flags & TF_LOCAL) { + tp->ecn_flags |= TE_RCVD_SYN_RST; + goto drop; + } else { + tcp_heuristic_ecn_synrst(tp); + } } -#endif /* MPTCP */ soevent(so, (SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNRESET)); @@ -3323,8 +3365,24 @@ findpcb: * We should restart the sending from what the receiver * has acknowledged immediately. */ - if (SEQ_GT(tp->snd_nxt, th->th_ack)) + if (SEQ_GT(tp->snd_nxt, th->th_ack)) { + /* + * rdar://problem/33214601 + * There is a middlebox that acks all but one + * byte and still drops the data. + */ + if ((tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && + tp->snd_max == th->th_ack + 1 && + tp->snd_max > tp->snd_una + 1) { + tcp_heuristic_tfo_middlebox(tp); + + so->so_error = ENODATA; + + tp->t_tfo_stats |= TFO_S_ONE_BYTE_PROXY; + } + tp->snd_max = tp->snd_nxt = th->th_ack; + } /* * If there's data, delay ACK; if there's also a FIN @@ -3382,11 +3440,6 @@ findpcb: if ((!(tp->t_mpflags & TMPF_MPTCP_TRUE)) && (tp->t_mpflags & TMPF_SENT_JOIN)) { isconnected = FALSE; - /* Start data xmit if fastjoin */ - if (mptcp_fastjoin && (so->so_flags & SOF_MPTCP_FASTJOIN)) { - soevent(so, (SO_FILT_HINT_LOCKED | - SO_FILT_HINT_MPFASTJ)); - } } else #endif /* MPTCP */ isconnected = TRUE; @@ -3403,14 +3456,13 @@ findpcb: if (so->so_flags & SOF_MP_SUBFLOW) so->so_flags1 |= SOF1_TFO_REWIND; #endif - if (!(tp->t_tfo_flags & TFO_F_NO_RCVPROBING)) - tcp_tfo_rcv_probe(tp, tlen); + tcp_tfo_rcv_probe(tp, tlen); } } } else { /* * Received initial SYN in SYN-SENT[*] state => simul- - * taneous open. If segment contains CC option and there is + * taneous open. If segment contains CC option and there is * a cached CC, apply TAO test; if it succeeds, connection is * half-synchronized. Otherwise, do 3-way handshake: * SYN-SENT -> SYN-RECEIVED @@ -3654,6 +3706,7 @@ trimthenstep6: INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, tlen); tp->t_stat.rxduplicatebytes += tlen; + inp_set_activity_bitmap(inp); } if (tlen > 0) goto dropafterack; @@ -3730,6 +3783,7 @@ trimthenstep6: INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, todrop); tp->t_stat.rxduplicatebytes += todrop; + inp_set_activity_bitmap(inp); } drop_hdrlen += todrop; /* drop from the top afterwards */ th->th_seq += todrop; @@ -3748,19 +3802,30 @@ trimthenstep6: * Send also a RST when we received a data segment after we've * sent our FIN when the socket is defunct. * Note that an MPTCP subflow socket would have SS_NOFDREF set - * by default so check to make sure that we test for SOF_MP_SUBFLOW - * socket flag (which would be cleared when the socket is closed.) + * by default. So, if it's an MPTCP-subflow we rather check the + * MPTCP-level's socket state for SS_NOFDREF. */ - if (!(so->so_flags & SOF_MP_SUBFLOW) && tlen && - (((so->so_state & SS_NOFDREF) && - tp->t_state > TCPS_CLOSE_WAIT) || - ((so->so_flags & SOF_DEFUNCT) && - tp->t_state > TCPS_FIN_WAIT_1))) { - tp = tcp_close(tp); - tcpstat.tcps_rcvafterclose++; - rstreason = BANDLIM_UNLIMITED; - IF_TCP_STATINC(ifp, cleanup); - goto dropwithreset; + if (tlen) { + boolean_t close_it = FALSE; + + if (!(so->so_flags & SOF_MP_SUBFLOW) && (so->so_state & SS_NOFDREF) && + tp->t_state > TCPS_CLOSE_WAIT) + close_it = TRUE; + + if ((so->so_flags & SOF_MP_SUBFLOW) && (mptetoso(tptomptp(tp)->mpt_mpte)->so_state & SS_NOFDREF) && + tp->t_state > TCPS_CLOSE_WAIT) + close_it = TRUE; + + if ((so->so_flags & SOF_DEFUNCT) && tp->t_state > TCPS_FIN_WAIT_1) + close_it = TRUE; + + if (close_it) { + tp = tcp_close(tp); + tcpstat.tcps_rcvafterclose++; + rstreason = BANDLIM_UNLIMITED; + IF_TCP_STATINC(ifp, cleanup); + goto dropwithreset; + } } /* @@ -3783,7 +3848,7 @@ trimthenstep6: SEQ_GT(th->th_seq, tp->rcv_nxt)) { iss = tcp_new_isn(tp); tp = tcp_close(tp); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); goto findpcb; } /* @@ -4044,7 +4109,7 @@ trimthenstep6: tcp_sack_doack(tp, &to, th, &sack_bytes_acked); #if MPTCP - if ((tp->t_mpuna) && (SEQ_GEQ(th->th_ack, tp->t_mpuna))) { + if (tp->t_mpuna && SEQ_GEQ(th->th_ack, tp->t_mpuna)) { if (tp->t_mpflags & TMPF_PREESTABLISHED) { /* MP TCP establishment succeeded */ tp->t_mpuna = 0; @@ -4068,7 +4133,6 @@ trimthenstep6: } } else { isconnected = TRUE; - tp->t_mpflags &= ~TMPF_SENT_KEYS; } } } @@ -4482,7 +4546,7 @@ process_ACK: if ((tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) && !(th->th_flags & TH_SYN)) - tcp_heuristic_tfo_snd_good(tp); + tp->t_tfo_flags |= TFO_F_NO_SNDPROBING; /* * If TH_ECE is received, make sure that ECN is enabled @@ -4832,6 +4896,7 @@ dodata: } INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, tlen); + inp_set_activity_bitmap(inp); } tcp_sbrcv_grow(tp, &so->so_rcv, &to, tlen, TCP_AUTORCVBUF_MAX(ifp)); @@ -4974,7 +5039,7 @@ dodata: tcp_check_timer_state(tp); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; @@ -5011,7 +5076,7 @@ dropafterack: (void) tcp_output(tp); /* Don't need to check timer state as we should have done it during tcp_output */ - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; dropwithresetnosock: @@ -5069,9 +5134,9 @@ dropwithreset: /* destroy temporarily created socket */ if (dropsocket) { (void) soabort(so); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } else if ((inp != NULL) && (nosock == 0)) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; @@ -5090,10 +5155,10 @@ drop: /* destroy temporarily created socket */ if (dropsocket) { (void) soabort(so); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } else if (nosock == 0) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; @@ -5567,6 +5632,26 @@ tcp_maxmtu6(struct rtentry *rt) } #endif +unsigned int +get_maxmtu(struct rtentry *rt) +{ + unsigned int maxmtu = 0; + + RT_LOCK_ASSERT_NOTHELD(rt); + + RT_LOCK(rt); + + if (rt_key(rt)->sa_family == AF_INET6) { + maxmtu = tcp_maxmtu6(rt); + } else { + maxmtu = tcp_maxmtu(rt); + } + + RT_UNLOCK(rt); + + return (maxmtu); +} + /* * Determine a reasonable value for maxseg size. * If the route is known, check route for mtu. @@ -5944,7 +6029,7 @@ tcp_dropdropablreq(struct socket *head) return (0); so_acquire_accept_list(head, NULL); - socket_unlock(head, NULL); + socket_unlock(head, 0); /* * Check if there is any socket in the incomp queue @@ -5965,7 +6050,7 @@ tcp_dropdropablreq(struct socket *head) * issues because it is in the incomp queue and * is not visible to others. */ - if (lck_mtx_try_lock(&inp->inpcb_mtx)) { + if (socket_try_lock(so)) { so->so_usecount++; goto found_victim; } else { @@ -6000,8 +6085,7 @@ tcp_dropdropablreq(struct socket *head) sonext = TAILQ_NEXT(so, so_list); - if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) - != WNT_STOPUSING) { + if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) != WNT_STOPUSING) { /* * Avoid the issue of a socket being accepted * by one input thread and being dropped by @@ -6009,7 +6093,7 @@ tcp_dropdropablreq(struct socket *head) * on this mutex, then grab the next socket in * line. */ - if (lck_mtx_try_lock(&inp->inpcb_mtx)) { + if (socket_try_lock(so)) { so->so_usecount++; if ((so->so_usecount == 2) && (so->so_state & SS_INCOMP) && @@ -6021,7 +6105,7 @@ tcp_dropdropablreq(struct socket *head) * used in any other way */ in_pcb_checkstate(inp, WNT_RELEASE, 1); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } } else { /* @@ -6044,7 +6128,7 @@ tcp_dropdropablreq(struct socket *head) /* Makes sure socket is still in the right state to be discarded */ if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); socket_lock(head, 0); so_release_accept_list(head); return (0); @@ -6053,7 +6137,7 @@ tcp_dropdropablreq(struct socket *head) found_victim: if (so->so_usecount != 2 || !(so->so_state & SS_INCOMP)) { /* do not discard: that socket is being accepted */ - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); socket_lock(head, 0); so_release_accept_list(head); return (0); @@ -6067,9 +6151,9 @@ found_victim: so->so_flags |= SOF_OVERFLOW; so->so_head = NULL; so_release_accept_list(head); - tcp_unlock(head, 0, 0); + socket_unlock(head, 0); - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); tp = sototcpcb(so); tcp_close(tp); @@ -6083,18 +6167,18 @@ found_victim: */ VERIFY(so->so_usecount > 0); so->so_usecount--; - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } else { /* * Unlock this socket and leave the reference on. * We need to acquire the pcbinfo lock in order to * fully dispose it off */ - tcp_unlock(so, 0, 0); + socket_unlock(so, 0); lck_rw_lock_exclusive(tcbinfo.ipi_lock); - tcp_lock(so, 0, 0); + socket_lock(so, 0); /* Release the reference held for so_incomp queue */ VERIFY(so->so_usecount > 0); so->so_usecount--; @@ -6108,7 +6192,7 @@ found_victim: * was unlocked. This socket will have to be * garbage collected later */ - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } else { /* Drop the reference held for this function */ VERIFY(so->so_usecount > 0); @@ -6120,7 +6204,7 @@ found_victim: } tcpstat.tcps_drops++; - tcp_lock(head, 0, 0); + socket_lock(head, 0); return(1); } @@ -6227,12 +6311,14 @@ tcp_getstat SYSCTL_HANDLER_ARGS #pragma unused(oidp, arg1, arg2) int error; - + struct tcpstat *stat; + stat = &tcpstat; +#if !CONFIG_EMBEDDED proc_t caller = PROC_NULL; proc_t caller_parent = PROC_NULL; char command_name[MAXCOMLEN + 1] = ""; char parent_name[MAXCOMLEN + 1] = ""; - + struct tcpstat zero_stat; if ((caller = proc_self()) != PROC_NULL) { /* get process name */ strlcpy(command_name, caller->p_comm, sizeof(command_name)); @@ -6262,12 +6348,19 @@ tcp_getstat SYSCTL_HANDLER_ARGS } if (caller != PROC_NULL) proc_rele(caller); + if (tcp_disable_access_to_stats && + !kauth_cred_issuser(kauth_cred_get())) { + bzero(&zero_stat, sizeof(zero_stat)); + stat = &zero_stat; + } + +#endif /* !CONFIG_EMBEDDED */ if (req->oldptr == 0) { req->oldlen= (size_t)sizeof(struct tcpstat); } - error = SYSCTL_OUT(req, &tcpstat, MIN(sizeof (tcpstat), req->oldlen)); + error = SYSCTL_OUT(req, stat, MIN(sizeof (tcpstat), req->oldlen)); return (error); @@ -6289,39 +6382,65 @@ tcp_input_checksum(int af, struct mbuf *m, struct tcphdr *th, int off, int tlen) if (m->m_pkthdr.pkt_flags & PKTF_SW_LRO_DID_CSUM) return (0); + /* ip_stripoptions() must have been called before we get here */ + ASSERT((ip->ip_hl << 2) == sizeof (*ip)); + if ((hwcksum_rx || (ifp->if_flags & IFF_LOOPBACK) || (m->m_pkthdr.pkt_flags & PKTF_LOOP)) && (m->m_pkthdr.csum_flags & CSUM_DATA_VALID)) { if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) { th->th_sum = m->m_pkthdr.csum_rx_val; } else { - uint16_t sum = m->m_pkthdr.csum_rx_val; - uint16_t start = m->m_pkthdr.csum_rx_start; + uint32_t sum = m->m_pkthdr.csum_rx_val; + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - (off + tlen)); /* * Perform 1's complement adjustment of octets * that got included/excluded in the hardware- * calculated checksum value. Ignore cases - * where the value includes or excludes the IP - * header span, as the sum for those octets - * would already be 0xffff and thus no-op. + * where the value already includes the entire + * IP header span, as the sum for those octets + * would already be 0 by the time we get here; + * IP has already performed its header checksum + * checks. If we do need to adjust, restore + * the original fields in the IP header when + * computing the adjustment value. Also take + * care of any trailing bytes and subtract out + * their partial sum. */ + ASSERT(trailer >= 0); if ((m->m_pkthdr.csum_flags & CSUM_PARTIAL) && - start != 0 && (off - start) != off) { -#if BYTE_ORDER != BIG_ENDIAN + ((start != 0 && start != off) || trailer)) { + uint32_t swbytes = (uint32_t)trailer; + if (start < off) { + ip->ip_len += sizeof (*ip); +#if BYTE_ORDER != BIG_ENDIAN HTONS(ip->ip_len); HTONS(ip->ip_off); +#endif /* BYTE_ORDER != BIG_ENDIAN */ } -#endif /* callee folds in sum */ - sum = m_adj_sum16(m, start, off, sum); -#if BYTE_ORDER != BIG_ENDIAN + sum = m_adj_sum16(m, start, off, + tlen, sum); + if (off > start) + swbytes += (off - start); + else + swbytes += (start - off); + if (start < off) { +#if BYTE_ORDER != BIG_ENDIAN NTOHS(ip->ip_off); NTOHS(ip->ip_len); +#endif /* BYTE_ORDER != BIG_ENDIAN */ + ip->ip_len -= sizeof (*ip); } -#endif + + if (swbytes != 0) + tcp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } /* callee folds in sum */ @@ -6364,17 +6483,22 @@ tcp_input_checksum(int af, struct mbuf *m, struct tcphdr *th, int off, int tlen) if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) { th->th_sum = m->m_pkthdr.csum_rx_val; } else { - uint16_t sum = m->m_pkthdr.csum_rx_val; - uint16_t start = m->m_pkthdr.csum_rx_start; + uint32_t sum = m->m_pkthdr.csum_rx_val; + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - (off + tlen)); /* * Perform 1's complement adjustment of octets * that got included/excluded in the hardware- - * calculated checksum value. + * calculated checksum value. Also take care + * of any trailing bytes and subtract out their + * partial sum. */ + ASSERT(trailer >= 0); if ((m->m_pkthdr.csum_flags & CSUM_PARTIAL) && - start != off) { - uint16_t s, d; + (start != off || trailer != 0)) { + uint16_t s = 0, d = 0; + uint32_t swbytes = (uint32_t)trailer; if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { s = ip6->ip6_src.s6_addr16[1]; @@ -6386,12 +6510,22 @@ tcp_input_checksum(int af, struct mbuf *m, struct tcphdr *th, int off, int tlen) } /* callee folds in sum */ - sum = m_adj_sum16(m, start, off, sum); + sum = m_adj_sum16(m, start, off, + tlen, sum); + if (off > start) + swbytes += (off - start); + else + swbytes += (start - off); if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) ip6->ip6_src.s6_addr16[1] = s; if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) ip6->ip6_dst.s6_addr16[1] = d; + + if (swbytes != 0) + tcp_in6_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } th->th_sum = in6_pseudo( diff --git a/bsd/netinet/tcp_ledbat.c b/bsd/netinet/tcp_ledbat.c index 457233d8d..fd066b42a 100644 --- a/bsd/netinet/tcp_ledbat.c +++ b/bsd/netinet/tcp_ledbat.c @@ -91,9 +91,8 @@ struct tcp_cc_algo tcp_cc_ledbat = { * The LEDBAT draft says that target queue delay MUST be 100 ms for * inter-operability. */ -int target_qdelay = 100; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, bg_target_qdelay, CTLFLAG_RW | CTLFLAG_LOCKED, - &target_qdelay , 100, "Target queuing delay"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, bg_target_qdelay, CTLFLAG_RW | CTLFLAG_LOCKED, + int, target_qdelay, 100, "Target queuing delay"); /* Allowed increase and tether are used to place an upper bound on * congestion window based on the amount of data that is outstanding. @@ -112,23 +111,21 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, bg_target_qdelay, CTLFLAG_RW | CTLFLAG_LOCKE * 'Tether' is also set to 2. We do not want this to limit the growth of cwnd * during slow-start. */ -int allowed_increase = 8; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, bg_allowed_increase, CTLFLAG_RW | CTLFLAG_LOCKED, - &allowed_increase, 1, "Additive constant used to calculate max allowed congestion window"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, bg_allowed_increase, CTLFLAG_RW | CTLFLAG_LOCKED, + int, allowed_increase, 8, + "Additive constant used to calculate max allowed congestion window"); /* Left shift for cwnd to get tether value of 2 */ -int tether_shift = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, bg_tether_shift, CTLFLAG_RW | CTLFLAG_LOCKED, - &tether_shift, 1, "Tether shift for max allowed congestion window"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, bg_tether_shift, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tether_shift, 1, "Tether shift for max allowed congestion window"); /* Start with an initial window of 2. This will help to get more accurate * minimum RTT measurement in the beginning. It will help to probe * the path slowly and will not add to the existing delay if the path is * already congested. Using 2 packets will reduce the delay induced by delayed-ack. */ -uint32_t bg_ss_fltsz = 2; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, bg_ss_fltsz, CTLFLAG_RW | CTLFLAG_LOCKED, - &bg_ss_fltsz, 2, "Initial congestion window for background transport"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, bg_ss_fltsz, CTLFLAG_RW | CTLFLAG_LOCKED, + uint32_t, bg_ss_fltsz, 2, "Initial congestion window for background transport"); extern int rtt_samples_per_slot; diff --git a/bsd/netinet/tcp_lro.c b/bsd/netinet/tcp_lro.c index 59ee6f445..bc1f21b6d 100644 --- a/bsd/netinet/tcp_lro.c +++ b/bsd/netinet/tcp_lro.c @@ -460,6 +460,7 @@ tcp_lro_process_pkt(struct mbuf *lro_mb, int drop_hdrlen) * quickly get the values now and not bother calling * tcp_dooptions(), etc. */ + bzero(&to, sizeof(to)); if ((optlen == TCPOLEN_TSTAMP_APPA || (optlen > TCPOLEN_TSTAMP_APPA && optp[TCPOLEN_TSTAMP_APPA] == TCPOPT_EOL)) && diff --git a/bsd/netinet/tcp_output.c b/bsd/netinet/tcp_output.c index f37afed3a..88ced34e9 100644 --- a/bsd/netinet/tcp_output.c +++ b/bsd/netinet/tcp_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -136,28 +136,24 @@ #define DBG_LAYER_END NETDBG_CODE(DBG_NETTCP, 3) #define DBG_FNC_TCP_OUTPUT NETDBG_CODE(DBG_NETTCP, (4 << 8) | 1) -int path_mtu_discovery = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery, - CTLFLAG_RW | CTLFLAG_LOCKED, &path_mtu_discovery, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, path_mtu_discovery, + CTLFLAG_RW | CTLFLAG_LOCKED, int, path_mtu_discovery, 1, "Enable Path MTU Discovery"); -int ss_fltsz = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowstart_flightsize, - CTLFLAG_RW | CTLFLAG_LOCKED,&ss_fltsz, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, slowstart_flightsize, + CTLFLAG_RW | CTLFLAG_LOCKED, int, ss_fltsz, 1, "Slow start flight size"); -int ss_fltsz_local = 8; /* starts with eight segments max */ -SYSCTL_INT(_net_inet_tcp, OID_AUTO, local_slowstart_flightsize, - CTLFLAG_RW | CTLFLAG_LOCKED, &ss_fltsz_local, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, local_slowstart_flightsize, + CTLFLAG_RW | CTLFLAG_LOCKED, int, ss_fltsz_local, 8, "Slow start flight size for local networks"); int tcp_do_tso = 1; SYSCTL_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_do_tso, 0, "Enable TCP Segmentation Offload"); + &tcp_do_tso, 0, "Enable TCP Segmentation Offload"); -int tcp_ecn_setup_percentage = 50; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_setup_percentage, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_setup_percentage, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, ecn_setup_percentage, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_ecn_setup_percentage, 100, "Max ECN setup percentage"); static int @@ -176,6 +172,7 @@ sysctl_change_ecn_setting SYSCTL_HANDLER_ARGS if ((tcp_ecn_outbound == 0 || tcp_ecn_outbound == 1) && (i == 0 || i == 1)) { tcp_ecn_outbound = i; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_initiate_out, tcp_ecn_outbound); return(err); } if (tcp_ecn_outbound == 2 && (i == 0 || i == 1)) { @@ -211,10 +208,13 @@ sysctl_change_ecn_setting SYSCTL_HANDLER_ARGS ifnet_head_done(); } tcp_ecn_outbound = i; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_initiate_out, tcp_ecn_outbound); } /* Change the other one too as the work is done */ - if (i == 2 || tcp_ecn_inbound == 2) + if (i == 2 || tcp_ecn_inbound == 2) { tcp_ecn_inbound = i; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_negotiate_in, tcp_ecn_inbound); + } return (err); } @@ -230,65 +230,53 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, ecn_negotiate_in, sysctl_change_ecn_setting, "IU", "Initiate ECN for inbound connections"); -int tcp_packet_chaining = 50; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, packetchain, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_packet_chaining, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, packetchain, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_packet_chaining, 50, "Enable TCP output packet chaining"); -int tcp_output_unlocked = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, socket_unlocked_on_output, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_output_unlocked, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, socket_unlocked_on_output, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_output_unlocked, 1, "Unlock TCP when sending packets down to IP"); -int tcp_do_rfc3390 = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3390, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_rfc3390, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rfc3390, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_do_rfc3390, 1, "Calculate intial slowstart cwnd depending on MSS"); -int tcp_min_iaj_win = MIN_IAJ_WIN; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, min_iaj_win, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_min_iaj_win, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, min_iaj_win, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_min_iaj_win, MIN_IAJ_WIN, "Minimum recv win based on inter-packet arrival jitter"); -int tcp_acc_iaj_react_limit = ACC_IAJ_REACT_LIMIT; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, acc_iaj_react_limit, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_acc_iaj_react_limit, 1, - "Accumulated IAJ when receiver starts to react"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, acc_iaj_react_limit, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_acc_iaj_react_limit, + ACC_IAJ_REACT_LIMIT, "Accumulated IAJ when receiver starts to react"); -uint32_t tcp_do_autosendbuf = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, doautosndbuf, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_autosendbuf, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, doautosndbuf, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_do_autosendbuf, 1, "Enable send socket buffer auto-tuning"); -uint32_t tcp_autosndbuf_inc = 8 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufinc, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_autosndbuf_inc, 1, - "Increment in send socket bufffer size"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, autosndbufinc, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_autosndbuf_inc, + 8 * 1024, "Increment in send socket bufffer size"); -uint32_t tcp_autosndbuf_max = 512 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufmax, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_autosndbuf_max, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, autosndbufmax, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_autosndbuf_max, 512 * 1024, "Maximum send socket buffer size"); -uint32_t tcp_prioritize_acks = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, ack_prioritize, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_prioritize_acks, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, ack_prioritize, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_prioritize_acks, 1, "Prioritize pure acks"); -uint32_t tcp_use_rtt_recvbg = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rtt_recvbg, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_use_rtt_recvbg, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rtt_recvbg, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_use_rtt_recvbg, 1, "Use RTT for bg recv algorithm"); -uint32_t tcp_recv_throttle_minwin = 16 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, recv_throttle_minwin, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_recv_throttle_minwin, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, recv_throttle_minwin, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_recv_throttle_minwin, 16 * 1024, "Minimum recv win for throttling"); -int32_t tcp_enable_tlp = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, enable_tlp, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, enable_tlp, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_enable_tlp, 1, "Enable Tail loss probe"); + int32_t, tcp_enable_tlp, 1, "Enable Tail loss probe"); static int32_t packchain_newlist = 0; static int32_t packchain_looped = 0; @@ -322,7 +310,8 @@ static int32_t tcp_tfo_check(struct tcpcb *tp, int32_t len) if (tp->t_flags & TF_NOOPT) goto fallback; - if (so->so_flags & SOF1_DATA_AUTHENTICATED) + if ((so->so_flags1 & SOF1_DATA_AUTHENTICATED) && + !(tp->t_flagsext & TF_FASTOPEN_HEUR)) return (len); if (!tcp_heuristic_do_tfo(tp)) { @@ -331,6 +320,9 @@ static int32_t tcp_tfo_check(struct tcpcb *tp, int32_t len) goto fallback; } + if (so->so_flags1 & SOF1_DATA_AUTHENTICATED) + return (len); + optlen += TCPOLEN_MAXSEG; if (tp->t_flags & TF_REQ_SCALE) @@ -412,7 +404,7 @@ tcp_tfo_write_cookie(struct tcpcb *tp, unsigned optlen, int32_t len, int res; u_char *bp; - if (so->so_flags & SOF1_DATA_AUTHENTICATED) { + if (so->so_flags1 & SOF1_DATA_AUTHENTICATED) { /* If there is some data, let's track it */ if (len > 0) { tp->t_tfo_stats |= TFO_S_SYN_DATA_SENT; @@ -597,12 +589,7 @@ tcp_output(struct tcpcb *tp) struct mbuf *mnext = NULL; int sackoptlen = 0; #if MPTCP - unsigned int *dlenp = NULL; - u_int8_t *finp = NULL; - u_int32_t *sseqp = NULL; - u_int64_t dss_val = 0; - boolean_t mptcp_acknow = FALSE; - boolean_t early_data_sent = FALSE; + boolean_t mptcp_acknow; #endif /* MPTCP */ boolean_t cell = FALSE; boolean_t wifi = FALSE; @@ -660,6 +647,10 @@ tcp_output(struct tcpcb *tp) #endif /* MPTCP */ again: +#if MPTCP + mptcp_acknow = FALSE; +#endif + KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_START, 0,0,0,0,0); #if INET6 @@ -982,22 +973,6 @@ after_sack_rexmit: } } -#if MPTCP - if ((tp->t_mpflags & TMPF_FASTJOIN_SEND) && - (tp->t_state == TCPS_SYN_SENT) && - (!(tp->t_flags & TF_CLOSING)) && - (so->so_snd.sb_cc != 0) && - (tp->t_rxtshift == 0)) { - flags &= ~TH_SYN; - flags |= TH_ACK; - off = 0; - len = min(so->so_snd.sb_cc, tp->t_maxseg); - early_data_sent = TRUE; - } else if (early_data_sent) { - /* for now, we allow only one data segment to be sent */ - return (0); - } -#endif /* MPTCP */ /* * Lop off SYN bit if it has already been sent. However, if this * is SYN-SENT state and if segment contains data and if we don't @@ -1019,7 +994,7 @@ after_sack_rexmit: error = tcp_ip_output(so, tp, packetlist, packchain_listadd, tp_inp_options, (so_options & SO_DONTROUTE), - (sack_rxmit | (sack_bytes_rxmt != 0)), + (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6); } @@ -1170,14 +1145,21 @@ after_sack_rexmit: if ((so->so_flags & SOF_MP_SUBFLOW) && !(tp->t_mpflags & TMPF_TCP_FALLBACK)) { int newlen = len; - if ((tp->t_state >= TCPS_ESTABLISHED) && - ((tp->t_mpflags & TMPF_SND_MPPRIO) || - (tp->t_mpflags & TMPF_SND_REM_ADDR) || - (tp->t_mpflags & TMPF_SND_MPFAIL))) { + if (tp->t_state >= TCPS_ESTABLISHED && + (tp->t_mpflags & TMPF_SND_MPPRIO || + tp->t_mpflags & TMPF_SND_REM_ADDR || + tp->t_mpflags & TMPF_SND_MPFAIL || + tp->t_mpflags & TMPF_SND_KEYS || + tp->t_mpflags & TMPF_SND_JACK)) { if (len > 0) { len = 0; } - sendalot = 1; + /* + * On a new subflow, don't try to send again, because + * we are still waiting for the fourth ack. + */ + if (!(tp->t_mpflags & TMPF_PREESTABLISHED)) + sendalot = 1; mptcp_acknow = TRUE; } else { mptcp_acknow = FALSE; @@ -1189,7 +1171,7 @@ after_sack_rexmit: * the contiguous MPTCP level. Set sendalot to send remainder. */ if (len > 0) - newlen = mptcp_adj_sendlen(so, off, len); + newlen = mptcp_adj_sendlen(so, off); if (newlen < len) { len = newlen; sendalot = 1; @@ -1243,9 +1225,8 @@ after_sack_rexmit: struct mptcb *mp_tp = tptomptp(tp); if (mp_tp != NULL) { - MPT_LOCK(mp_tp); - recwin = imin(recwin, (int)mp_tp->mpt_rcvwnd); - MPT_UNLOCK(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + recwin = imin(recwin, mptcp_sbspace(mp_tp)); } } #endif @@ -1271,7 +1252,15 @@ after_sack_rexmit: if (recwin > (int32_t)(TCP_MAXWIN << tp->rcv_scale)) recwin = (int32_t)(TCP_MAXWIN << tp->rcv_scale); - if (recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt)) + + /* + * MPTCP needs to be able to announce a smaller window than previously, + * because the other subflow may have filled up the available window- + * space. So we have to be able to go backwards and announce a smaller + * window. + */ + if (!(so->so_flags & SOF_MPTCP_TRUE) && + recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt)) recwin = (int32_t)(tp->rcv_adv - tp->rcv_nxt); /* @@ -1302,12 +1291,20 @@ after_sack_rexmit: tp->t_state != TCPS_ESTABLISHED) { if (len >= tp->t_maxseg) goto send; + if (!(tp->t_flags & TF_MORETOCOME) && (idle || tp->t_flags & TF_NODELAY || (tp->t_flags & TF_MAXSEGSNT) || ALLOW_LIMITED_TRANSMIT(tp)) && (tp->t_flags & TF_NOPUSH) == 0 && - len + off >= so->so_snd.sb_cc) + (len + off >= so->so_snd.sb_cc || + /* + * MPTCP needs to respect the DSS-mappings. So, it + * may be sending data that *could* have been + * coalesced, but cannot because of + * mptcp_adj_sendlen(). + */ + so->so_flags & SOF_MP_SUBFLOW)) goto send; if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) goto send; @@ -1441,7 +1438,7 @@ just_return: error = tcp_ip_output(so, tp, packetlist, packchain_listadd, tp_inp_options, (so_options & SO_DONTROUTE), - (sack_rxmit | (sack_bytes_rxmt != 0)), isipv6); + (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6); } /* tcp was closed while we were in ip; resume close */ if (inp->inp_sndinprog_cnt == 0 && @@ -1503,8 +1500,7 @@ send: } #if MPTCP if (mptcp_enable && (so->so_flags & SOF_MP_SUBFLOW)) { - optlen = mptcp_setup_syn_opts(so, flags, opt, - optlen); + optlen = mptcp_setup_syn_opts(so, opt, optlen); } #endif /* MPTCP */ } @@ -1572,7 +1568,7 @@ send: tp->t_mpflags |= TMPF_MPTCP_ACKNOW; } optlen = mptcp_setup_opts(tp, off, &opt[0], optlen, flags, - len, &dlenp, &finp, &dss_val, &sseqp, &mptcp_acknow); + len, &mptcp_acknow); tp->t_mpflags &= ~TMPF_SEND_DSN; } #endif /* MPTCP */ @@ -1835,43 +1831,30 @@ send: sendalot = 1; } } -#if MPTCP - /* Adjust the length in the DSS option, if it is lesser than len */ - if (dlenp) { - /* - * To test this path without SACK, artificially - * decrement len with something like - * if (len > 10) - len -= 10; - */ - if (ntohs(*dlenp) > len) { - *dlenp = htons(len); - /* Unset the FIN flag, if len was adjusted */ - if (finp) { - *finp &= ~MDSS_F; - } - sendalot = 1; - } - } -#endif /* MPTCP */ if (max_linkhdr + hdrlen > MCLBYTES) panic("tcphdr too big"); /* Check if there is enough data in the send socket - * buffer to start measuring bw + * buffer to start measuring bandwidth */ if ((tp->t_flagsext & TF_MEASURESNDBW) != 0 && (tp->t_bwmeas != NULL) && - (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0 && - (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)) >= - tp->t_bwmeas->bw_minsize) { - tp->t_bwmeas->bw_size = min( - (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)), - tp->t_bwmeas->bw_maxsize); - tp->t_flagsext |= TF_BWMEAS_INPROGRESS; - tp->t_bwmeas->bw_start = tp->snd_max; - tp->t_bwmeas->bw_ts = tcp_now; + (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0) { + tp->t_bwmeas->bw_size = min(min( + (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)), + tp->snd_cwnd), tp->snd_wnd); + if (tp->t_bwmeas->bw_minsize > 0 && + tp->t_bwmeas->bw_size < tp->t_bwmeas->bw_minsize) + tp->t_bwmeas->bw_size = 0; + if (tp->t_bwmeas->bw_maxsize > 0) + tp->t_bwmeas->bw_size = min(tp->t_bwmeas->bw_size, + tp->t_bwmeas->bw_maxsize); + if (tp->t_bwmeas->bw_size > 0) { + tp->t_flagsext |= TF_BWMEAS_INPROGRESS; + tp->t_bwmeas->bw_start = tp->snd_max; + tp->t_bwmeas->bw_ts = tcp_now; + } } VERIFY(inp->inp_flowhash != 0); @@ -1909,6 +1892,7 @@ send: } inp_decr_sndbytes_unsent(so, len); } + inp_set_activity_bitmap(inp); #if MPTCP if (tp->t_mpflags & TMPF_MPTCP_TRUE) { tcpstat.tcps_mp_sndpacks++; @@ -2051,10 +2035,6 @@ send: m->m_len = hdrlen; } m->m_pkthdr.rcvif = 0; -#if MPTCP - /* Before opt is copied to the mbuf, set the csum field */ - mptcp_output_csum(tp, m, len, hdrlen, dss_val, sseqp); -#endif /* MPTCP */ #if CONFIG_MACF_NET mac_mbuf_label_associate_inpcb(inp, m); #endif @@ -2148,12 +2128,6 @@ send: } th->th_ack = htonl(tp->rcv_nxt); tp->last_ack_sent = tp->rcv_nxt; -#if MPTCP - /* Initialize the ACK field to a value as 0 ack fields are dropped */ - if (early_data_sent) { - th->th_ack = th->th_seq + 1; - } -#endif /* MPTCP */ if (optlen) { bcopy(opt, th + 1, optlen); th->th_off = (sizeof (struct tcphdr) + optlen) >> 2; @@ -2274,6 +2248,7 @@ send: } if (SEQ_GT(tp->snd_nxt, tp->snd_max)) { tp->snd_max = tp->snd_nxt; + tp->t_sndtime = tcp_now; /* * Time this transmission if not a retransmission and * not currently timing anything. @@ -2343,7 +2318,7 @@ timer: */ pto = tp->t_rxtcur; - /* Reset the next RTO to be after PTO. */ + /* Reset the next RTO to be after PTO. */ TCPT_RANGESET(new_rto, (pto + TCP_REXMTVAL(tp)), max(tp->t_rttmin, tp->t_rttcur + 2), @@ -2366,8 +2341,10 @@ timer: ++xlen; tp->t_flags |= TF_SENTFIN; } - if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max)) + if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max)) { tp->snd_max = tp->snd_nxt + len; + tp->t_sndtime = tcp_now; + } } #if TCPDEBUG @@ -2466,12 +2443,7 @@ timer: */ m->m_pkthdr.pkt_flowsrc = FLOWSRC_INPCB; m->m_pkthdr.pkt_flowid = inp->inp_flowhash; - m->m_pkthdr.pkt_flags |= PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC; -#if MPTCP - /* Disable flow advisory when using MPTCP. */ - if (!(tp->t_mpflags & TMPF_MPTCP_TRUE)) -#endif /* MPTCP */ - m->m_pkthdr.pkt_flags |= PKTF_FLOW_ADV; + m->m_pkthdr.pkt_flags |= (PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC | PKTF_FLOW_ADV); m->m_pkthdr.pkt_proto = IPPROTO_TCP; m->m_nextpkt = NULL; @@ -2553,7 +2525,7 @@ timer: if (sendalot == 0 || (tp->t_state != TCPS_ESTABLISHED) || (tp->snd_cwnd <= (tp->snd_wnd / 8)) || - (tp->t_flags & (TH_PUSH | TF_ACKNOW)) || + (tp->t_flags & TF_ACKNOW) || (tp->t_flagsext & TF_FORCE) || tp->t_lastchain >= tcp_packet_chaining) { error = 0; @@ -2568,7 +2540,7 @@ timer: error = tcp_ip_output(so, tp, packetlist, packchain_listadd, tp_inp_options, (so_options & SO_DONTROUTE), - (sack_rxmit | (sack_bytes_rxmt != 0)), isipv6); + (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6); if (error) { /* * Take into account the rest of unsent @@ -2820,7 +2792,7 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, */ if (tcp_output_unlocked && !so->so_upcallusecount && (tp->t_state == TCPS_ESTABLISHED) && (sack_in_progress == 0) && - !IN_FASTRECOVERY(tp)) { + !IN_FASTRECOVERY(tp) && !(so->so_flags & SOF_MP_SUBFLOW)) { unlocked = TRUE; socket_unlock(so, 0); @@ -2892,7 +2864,8 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, /* * Enter flow controlled state if the connection is established - * and is not in recovery. + * and is not in recovery. Flow control is allowed only if there + * is outstanding data. * * A connection will enter suspended state even if it is in * recovery. @@ -2900,7 +2873,8 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, if (((adv->code == FADV_FLOW_CONTROLLED && !IN_FASTRECOVERY(tp)) || adv->code == FADV_SUSPENDED) && !(tp->t_flags & TF_CLOSING) && - tp->t_state == TCPS_ESTABLISHED) { + tp->t_state == TCPS_ESTABLISHED && + SEQ_GT(tp->snd_max, tp->snd_una)) { int rc; rc = inp_set_fc_state(inp, adv->code); @@ -2940,6 +2914,7 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, so->so_snd.sb_flags &= ~SB_SNDBYTE_CNT; } inp->inp_last_outifp = outif; + } if (error != 0 && ifdenied && @@ -2966,6 +2941,8 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, return (error); } +int tcptv_persmin_val = TCPTV_PERSMIN; + void tcp_setpersist(struct tcpcb *tp) { @@ -2988,7 +2965,7 @@ tcp_setpersist(struct tcpcb *tp) */ TCPT_RANGESET(tp->t_timer[TCPT_PERSIST], t * tcp_backoff[tp->t_rxtshift], - TCPTV_PERSMIN, TCPTV_PERSMAX, 0); + tcptv_persmin_val, TCPTV_PERSMAX, 0); tp->t_timer[TCPT_PERSIST] = OFFSET_FROM_START(tp, tp->t_timer[TCPT_PERSIST]); if (tp->t_rxtshift < TCP_MAXRXTSHIFT) diff --git a/bsd/netinet/tcp_sack.c b/bsd/netinet/tcp_sack.c index 4a68325db..4b4c04dbe 100644 --- a/bsd/netinet/tcp_sack.c +++ b/bsd/netinet/tcp_sack.c @@ -105,17 +105,14 @@ #include -int tcp_do_sack = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, sack, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_sack, 0, - "Enable/Disable TCP SACK support"); -static int tcp_sack_maxholes = 128; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, sack_maxholes, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_sack_maxholes, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, sack, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_do_sack, 1, "Enable/Disable TCP SACK support"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, sack_maxholes, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, tcp_sack_maxholes, 128, "Maximum number of TCP SACK holes allowed per connection"); -static int tcp_sack_globalmaxholes = 65536; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, sack_globalmaxholes, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_sack_globalmaxholes, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, sack_globalmaxholes, + CTLFLAG_RW | CTLFLAG_LOCKED, static int, tcp_sack_globalmaxholes, 65536, "Global maximum number of TCP SACK holes"); static SInt32 tcp_sack_globalholes = 0; diff --git a/bsd/netinet/tcp_seq.h b/bsd/netinet/tcp_seq.h index 5eb5c3b93..772d33f36 100644 --- a/bsd/netinet/tcp_seq.h +++ b/bsd/netinet/tcp_seq.h @@ -107,11 +107,9 @@ #define tcp_sendseqinit(tp) \ (tp)->snd_una = (tp)->snd_nxt = (tp)->snd_max = (tp)->snd_up = \ - (tp)->snd_recover = (tp)->iss + (tp)->snd_recover = (tp)->iss #define TCP_PAWS_IDLE (24 * 24 * 60 * 60 * TCP_RETRANSHZ) - /* timestamp wrap-around time */ - -extern tcp_cc tcp_ccgen; /* global connection count */ +/* timestamp wrap-around time */ #endif /* KERNEL_PRIVATE */ #endif /* _NETINET_TCP_SEQ_H_ */ diff --git a/bsd/netinet/tcp_subr.c b/bsd/netinet/tcp_subr.c index b2e18eb05..1be5b6a80 100644 --- a/bsd/netinet/tcp_subr.c +++ b/bsd/netinet/tcp_subr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -99,6 +99,7 @@ #include #if INET6 #include +#include #endif #include #if INET6 @@ -110,6 +111,7 @@ #if INET6 #include #endif +#include #include #include #include @@ -155,24 +157,21 @@ #define DBG_FNC_TCP_CLOSE NETDBG_CODE(DBG_NETTCP, ((5 << 8) | 2)) +static tcp_cc tcp_ccgen; extern int tcp_lq_overflow; extern struct tcptimerlist tcp_timer_list; extern struct tcptailq tcp_tw_tailq; -int tcp_mssdflt = TCP_MSS; -SYSCTL_INT(_net_inet_tcp, TCPCTL_MSSDFLT, mssdflt, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_mssdflt, 0, "Default TCP Maximum Segment Size"); +SYSCTL_SKMEM_TCP_INT(TCPCTL_MSSDFLT, mssdflt, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_mssdflt, TCP_MSS, "Default TCP Maximum Segment Size"); #if INET6 -int tcp_v6mssdflt = TCP6_MSS; -SYSCTL_INT(_net_inet_tcp, TCPCTL_V6MSSDFLT, v6mssdflt, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_v6mssdflt, 0, +SYSCTL_SKMEM_TCP_INT(TCPCTL_V6MSSDFLT, v6mssdflt, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_v6mssdflt, TCP6_MSS, "Default TCP Maximum Segment Size for IPv6"); #endif -extern int tcp_do_autorcvbuf; - int tcp_sysctl_fastopenkey(struct sysctl_oid *, void *, int, struct sysctl_req *); SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fastopen_key, CTLTYPE_STRING | CTLFLAG_WR, @@ -182,19 +181,19 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fastopen_key, CTLTYPE_STRING | CTLFLAG_WR, int tcp_tfo_halfcnt = 0; /* Maximum of half-open TFO connection backlog */ -int tcp_tfo_backlog = 10; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, fastopen_backlog, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_tfo_backlog, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, fastopen_backlog, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_tfo_backlog, 10, "Backlog queue for half-open TFO connections"); -int tcp_fastopen = TCP_FASTOPEN_CLIENT | TCP_FASTOPEN_SERVER; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_fastopen, 0, "Enable TCP Fastopen (RFC 7413)"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_fastopen, TCP_FASTOPEN_CLIENT | TCP_FASTOPEN_SERVER, + "Enable TCP Fastopen (RFC 7413)"); + +SYSCTL_SKMEM_TCP_INT(OID_AUTO, now_init, CTLFLAG_RD | CTLFLAG_LOCKED, + uint32_t, tcp_now_init, 0, "Initial tcp now value"); -int tcp_tfo_fallback_min = 10; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, fastopen_fallback_min, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_tfo_fallback_min, 0, - "Mininum number of trials without TFO when in fallback mode"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, microuptime_init, CTLFLAG_RD | CTLFLAG_LOCKED, + uint32_t, tcp_microuptime_init, 0, "Initial tcp uptime value in micro seconds"); /* * Minimum MSS we accept and use. This prevents DoS attacks where @@ -204,9 +203,8 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, fastopen_fallback_min, * with packet generation and sending. Set to zero to disable MINMSS * checking. This setting prevents us from sending too small packets. */ -int tcp_minmss = TCP_MINMSS; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, minmss, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_minmss, 0, "Minmum TCP Maximum Segment Size"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, minmss, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_minmss, TCP_MINMSS, "Minmum TCP Maximum Segment Size"); int tcp_do_rfc1323 = 1; #if (DEVELOPMENT || DEBUG) SYSCTL_INT(_net_inet_tcp, TCPCTL_DO_RFC1323, rfc1323, @@ -220,9 +218,8 @@ SYSCTL_INT(_net_inet_tcp, TCPCTL_DO_RFC1644, rfc1644, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_rfc1644, 0, "Enable rfc1644 (TTCP) extensions"); -static int do_tcpdrain = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, do_tcpdrain, CTLFLAG_RW | CTLFLAG_LOCKED, - &do_tcpdrain, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, do_tcpdrain, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, do_tcpdrain, 0, "Enable tcp_drain routine for extra help when low on mbufs"); SYSCTL_INT(_net_inet_tcp, OID_AUTO, pcbcount, CTLFLAG_RD | CTLFLAG_LOCKED, @@ -231,9 +228,8 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, pcbcount, CTLFLAG_RD | CTLFLAG_LOCKED, SYSCTL_INT(_net_inet_tcp, OID_AUTO, tw_pcbcount, CTLFLAG_RD | CTLFLAG_LOCKED, &tcbinfo.ipi_twcount, 0, "Number of pcbs in time-wait state"); -static int icmp_may_rst = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, icmp_may_rst, CTLFLAG_RW | CTLFLAG_LOCKED, - &icmp_may_rst, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, icmp_may_rst, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, icmp_may_rst, 1, "Certain ICMP unreachable messages may abort connections in SYN_SENT"); static int tcp_strict_rfc1948 = 0; @@ -247,23 +243,18 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, isn_reseed_interval, &tcp_isn_reseed_interval, 0, "Seconds between reseeding of ISN secret"); #endif /* (DEVELOPMENT || DEBUG) */ -int tcp_TCPTV_MIN = 100; /* 100ms minimum RTT */ -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rtt_min, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_TCPTV_MIN, 0, "min rtt value allowed"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rtt_min, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_TCPTV_MIN, 100, "min rtt value allowed"); -int tcp_rexmt_slop = TCPTV_REXMTSLOP; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rexmt_slop, CTLFLAG_RW, - &tcp_rexmt_slop, 0, "Slop added to retransmit timeout"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rexmt_slop, CTLFLAG_RW, + int, tcp_rexmt_slop, TCPTV_REXMTSLOP, "Slop added to retransmit timeout"); -__private_extern__ int tcp_use_randomport = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, randomize_ports, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_use_randomport, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, randomize_ports, CTLFLAG_RW | CTLFLAG_LOCKED, + __private_extern__ int , tcp_use_randomport, 0, "Randomize TCP port numbers"); -__private_extern__ int tcp_win_scale = 3; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, win_scale_factor, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_win_scale, 0, - "Window scaling factor"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, win_scale_factor, CTLFLAG_RW | CTLFLAG_LOCKED, + __private_extern__ int, tcp_win_scale, 3, "Window scaling factor"); static void tcp_cleartaocache(void); static void tcp_notify(struct inpcb *, int); @@ -321,6 +312,10 @@ static lck_attr_t *tcp_uptime_mtx_attr = NULL; static lck_grp_t *tcp_uptime_mtx_grp = NULL; static lck_grp_attr_t *tcp_uptime_mtx_grp_attr = NULL; int tcp_notsent_lowat_check(struct socket *so); +static void tcp_flow_lim_stats(struct ifnet_stats_per_flow *ifs, + struct if_lim_perf_stat *stat); +static void tcp_flow_ecn_perf_stats(struct ifnet_stats_per_flow *ifs, + struct if_tcp_ecn_perf_stat *stat); static aes_encrypt_ctx tfo_ctx; /* Crypto-context for TFO */ @@ -452,7 +447,7 @@ tcp_tfo_init(void) { u_char key[TCP_FASTOPEN_KEYLEN]; - read_random(key, sizeof(key)); + read_frandom(key, sizeof(key)); aes_encrypt_key128(key, &tfo_ctx); } @@ -484,11 +479,17 @@ tcp_init(struct protosw *pp, struct domain *dp) tcp_msl = TCPTV_MSL; microuptime(&tcp_uptime); - read_random(&tcp_now, sizeof(tcp_now)); + read_frandom(&tcp_now, sizeof(tcp_now)); /* Starts tcp internal clock at a random value */ tcp_now = tcp_now & 0x3fffffff; + /* expose initial uptime/now via systcl for utcp to keep time sync */ + tcp_now_init = tcp_now; + tcp_microuptime_init = tcp_uptime.tv_sec * 1000 + tcp_uptime.tv_usec; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.microuptime_init, tcp_microuptime_init); + SYSCTL_SKMEM_UPDATE_FIELD(tcp.now_init, tcp_now_init); + tcp_tfo_init(); LIST_INIT(&tcb); @@ -638,8 +639,15 @@ tcp_init(struct protosw *pp, struct domain *dp) * maximum allowed receive and send socket buffer size. */ if (nmbclusters > 30720) { - tcp_autorcvbuf_max = 1024 * 1024; - tcp_autosndbuf_max = 1024 * 1024; + #if CONFIG_EMBEDDED + tcp_autorcvbuf_max = 2 * 1024 * 1024; + tcp_autosndbuf_max = 2 * 1024 * 1024; + #else + tcp_autorcvbuf_max = 1024 * 1024; + tcp_autosndbuf_max = 1024 * 1024; + #endif /* CONFIG_EMBEDDED */ + SYSCTL_SKMEM_UPDATE_FIELD(tcp.autorcvbufmax, tcp_autorcvbuf_max); + SYSCTL_SKMEM_UPDATE_FIELD(tcp.autosndbufmax, tcp_autosndbuf_max); /* * Receive buffer max for cellular interfaces supporting @@ -941,12 +949,7 @@ tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m, /* Embed flowhash and flow control flags */ m->m_pkthdr.pkt_flowsrc = FLOWSRC_INPCB; m->m_pkthdr.pkt_flowid = tp->t_inpcb->inp_flowhash; - m->m_pkthdr.pkt_flags |= PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC; -#if MPTCP - /* Disable flow advisory when using MPTCP. */ - if (!(tp->t_mpflags & TMPF_MPTCP_TRUE)) -#endif /* MPTCP */ - m->m_pkthdr.pkt_flags |= PKTF_FLOW_ADV; + m->m_pkthdr.pkt_flags |= (PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC | PKTF_FLOW_ADV); m->m_pkthdr.pkt_proto = IPPROTO_TCP; } @@ -977,8 +980,9 @@ tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m, if (tp != NULL && ro6 != NULL && ro6->ro_rt != NULL && (outif = ro6->ro_rt->rt_ifp) != - tp->t_inpcb->in6p_last_outifp) + tp->t_inpcb->in6p_last_outifp) { tp->t_inpcb->in6p_last_outifp = outif; + } if (ro6 == &sro6) ROUTE_RELEASE(ro6); @@ -1014,9 +1018,10 @@ tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m, if (tp != NULL && sro.ro_rt != NULL && (outif = sro.ro_rt->rt_ifp) != - tp->t_inpcb->inp_last_outifp) + tp->t_inpcb->inp_last_outifp) { tp->t_inpcb->inp_last_outifp = outif; + } if (ro != &sro) { /* Synchronize cached PCB route */ inp_route_copyin(tp->t_inpcb, &sro); @@ -1100,6 +1105,14 @@ tcp_newtcpcb(struct inpcb *inp) tp->t_flagsext |= TF_RCVUNACK_WAITSS; tp->t_rexmtthresh = tcprexmtthresh; + /* Enable bandwidth measurement on this connection */ + tp->t_flagsext |= TF_MEASURESNDBW; + if (tp->t_bwmeas == NULL) { + tp->t_bwmeas = tcp_bwmeas_alloc(tp); + if (tp->t_bwmeas == NULL) + tp->t_flagsext &= ~TF_MEASURESNDBW; + } + /* Clear time wait tailq entry */ tp->t_twentry.tqe_next = NULL; tp->t_twentry.tqe_prev = NULL; @@ -1177,33 +1190,77 @@ tcp_getrt_rtt(struct tcpcb *tp, struct rtentry *rt) } static inline void -tcp_update_ecn_perf_stats(struct tcpcb *tp, +tcp_create_ifnet_stats_per_flow(struct tcpcb *tp, + struct ifnet_stats_per_flow *ifs) +{ + struct inpcb *inp; + struct socket *so; + if (tp == NULL || ifs == NULL) + return; + + bzero(ifs, sizeof(*ifs)); + inp = tp->t_inpcb; + so = inp->inp_socket; + + ifs->ipv4 = (inp->inp_vflag & INP_IPV6) ? 0 : 1; + ifs->local = (tp->t_flags & TF_LOCAL) ? 1 : 0; + ifs->connreset = (so->so_error == ECONNRESET) ? 1 : 0; + ifs->conntimeout = (so->so_error == ETIMEDOUT) ? 1 : 0; + ifs->ecn_flags = tp->ecn_flags; + ifs->txretransmitbytes = tp->t_stat.txretransmitbytes; + ifs->rxoutoforderbytes = tp->t_stat.rxoutoforderbytes; + ifs->rxmitpkts = tp->t_stat.rxmitpkts; + ifs->rcvoopack = tp->t_rcvoopack; + ifs->pawsdrop = tp->t_pawsdrop; + ifs->sack_recovery_episodes = tp->t_sack_recovery_episode; + ifs->reordered_pkts = tp->t_reordered_pkts; + ifs->dsack_sent = tp->t_dsack_sent; + ifs->dsack_recvd = tp->t_dsack_recvd; + ifs->srtt = tp->t_srtt; + ifs->rttupdated = tp->t_rttupdated; + ifs->rttvar = tp->t_rttvar; + ifs->rttmin = get_base_rtt(tp); + if (tp->t_bwmeas != NULL && tp->t_bwmeas->bw_sndbw_max > 0) { + ifs->bw_sndbw_max = tp->t_bwmeas->bw_sndbw_max; + } else { + ifs->bw_sndbw_max = 0; + } + if (tp->t_bwmeas!= NULL && tp->t_bwmeas->bw_rcvbw_max > 0) { + ifs->bw_rcvbw_max = tp->t_bwmeas->bw_rcvbw_max; + } else { + ifs->bw_rcvbw_max = 0; + } + ifs->bk_txpackets = so->so_tc_stats[MBUF_TC_BK].txpackets; + ifs->txpackets = inp->inp_stat->txpackets; + ifs->rxpackets = inp->inp_stat->rxpackets; +} + +static inline void +tcp_flow_ecn_perf_stats(struct ifnet_stats_per_flow *ifs, struct if_tcp_ecn_perf_stat *stat) { u_int64_t curval, oldval; - struct inpcb *inp = tp->t_inpcb; - stat->total_txpkts += inp->inp_stat->txpackets; - stat->total_rxpkts += inp->inp_stat->rxpackets; - stat->total_rxmitpkts += tp->t_stat.rxmitpkts; - stat->total_oopkts += tp->t_rcvoopack; - stat->total_reorderpkts += (tp->t_reordered_pkts + tp->t_pawsdrop + - tp->t_dsack_sent + tp->t_dsack_recvd); + stat->total_txpkts += ifs->txpackets; + stat->total_rxpkts += ifs->rxpackets; + stat->total_rxmitpkts += ifs->rxmitpkts; + stat->total_oopkts += ifs->rcvoopack; + stat->total_reorderpkts += (ifs->reordered_pkts + + ifs->pawsdrop + ifs->dsack_sent + ifs->dsack_recvd); /* Average RTT */ - curval = (tp->t_srtt >> TCP_RTT_SHIFT); - if (curval > 0 && tp->t_rttupdated >= 16) { + curval = ifs->srtt >> TCP_RTT_SHIFT; + if (curval > 0 && ifs->rttupdated >= 16) { if (stat->rtt_avg == 0) { stat->rtt_avg = curval; } else { oldval = stat->rtt_avg; - stat->rtt_avg = - ((oldval << 4) - oldval + curval) >> 4; + stat->rtt_avg = ((oldval << 4) - oldval + curval) >> 4; } } /* RTT variance */ - curval = tp->t_rttvar >> TCP_RTTVAR_SHIFT; - if (curval > 0 && tp->t_rttupdated >= 16) { + curval = ifs->rttvar >> TCP_RTTVAR_SHIFT; + if (curval > 0 && ifs->rttupdated >= 16) { if (stat->rtt_var == 0) { stat->rtt_var = curval; } else { @@ -1213,13 +1270,77 @@ tcp_update_ecn_perf_stats(struct tcpcb *tp, } } - /* Total number of SACK recovery episodes */ - stat->sack_episodes += tp->t_sack_recovery_episode; - - if (inp->inp_socket->so_error == ECONNRESET) + /* SACK episodes */ + stat->sack_episodes += ifs->sack_recovery_episodes; + if (ifs->connreset) stat->rst_drop++; } +static inline void +tcp_flow_lim_stats(struct ifnet_stats_per_flow *ifs, + struct if_lim_perf_stat *stat) +{ + u_int64_t curval, oldval; + + stat->lim_total_txpkts += ifs->txpackets; + stat->lim_total_rxpkts += ifs->rxpackets; + stat->lim_total_retxpkts += ifs->rxmitpkts; + stat->lim_total_oopkts += ifs->rcvoopack; + + if (ifs->bw_sndbw_max > 0) { + /* convert from bytes per ms to bits per second */ + ifs->bw_sndbw_max *= 8000; + stat->lim_ul_max_bandwidth = max(stat->lim_ul_max_bandwidth, + ifs->bw_sndbw_max); + } + + if (ifs->bw_rcvbw_max > 0) { + /* convert from bytes per ms to bits per second */ + ifs->bw_rcvbw_max *= 8000; + stat->lim_dl_max_bandwidth = max(stat->lim_dl_max_bandwidth, + ifs->bw_rcvbw_max); + } + + /* Average RTT */ + curval = ifs->srtt >> TCP_RTT_SHIFT; + if (curval > 0 && ifs->rttupdated >= 16) { + if (stat->lim_rtt_average == 0) { + stat->lim_rtt_average = curval; + } else { + oldval = stat->lim_rtt_average; + stat->lim_rtt_average = + ((oldval << 4) - oldval + curval) >> 4; + } + } + + /* RTT variance */ + curval = ifs->rttvar >> TCP_RTTVAR_SHIFT; + if (curval > 0 && ifs->rttupdated >= 16) { + if (stat->lim_rtt_variance == 0) { + stat->lim_rtt_variance = curval; + } else { + oldval = stat->lim_rtt_variance; + stat->lim_rtt_variance = + ((oldval << 4) - oldval + curval) >> 4; + } + } + + if (stat->lim_rtt_min == 0) { + stat->lim_rtt_min = ifs->rttmin; + } else { + stat->lim_rtt_min = min(stat->lim_rtt_min, ifs->rttmin); + } + + /* connection timeouts */ + stat->lim_conn_attempts++; + if (ifs->conntimeout) + stat->lim_conn_timeouts++; + + /* bytes sent using background delay-based algorithms */ + stat->lim_bk_txpkts += ifs->bk_txpackets; + +} + /* * Close a TCP control block: * discard all space held by the tcp @@ -1237,6 +1358,7 @@ tcp_close(struct tcpcb *tp) struct route *ro; struct rtentry *rt; int dosavessthresh; + struct ifnet_stats_per_flow ifs; /* tcp_close was called previously, bail */ if (inp->inp_ppcb == NULL) @@ -1398,90 +1520,9 @@ no_valid_rt: /* free the reassembly queue, if any */ (void) tcp_freeq(tp); - /* Collect ECN related statistics */ - if (tp->ecn_flags & TE_SETUPSENT) { - if (tp->ecn_flags & TE_CLIENT_SETUP) { - INP_INC_IFNET_STAT(inp, ecn_client_setup); - if (TCP_ECN_ENABLED(tp)) { - INP_INC_IFNET_STAT(inp, - ecn_client_success); - } else if (tp->ecn_flags & TE_LOST_SYN) { - INP_INC_IFNET_STAT(inp, ecn_syn_lost); - } else { - INP_INC_IFNET_STAT(inp, - ecn_peer_nosupport); - } - } else { - INP_INC_IFNET_STAT(inp, ecn_server_setup); - if (TCP_ECN_ENABLED(tp)) { - INP_INC_IFNET_STAT(inp, - ecn_server_success); - } else if (tp->ecn_flags & TE_LOST_SYNACK) { - INP_INC_IFNET_STAT(inp, - ecn_synack_lost); - } else { - INP_INC_IFNET_STAT(inp, - ecn_peer_nosupport); - } - } - } else { - INP_INC_IFNET_STAT(inp, ecn_off_conn); - } - if (TCP_ECN_ENABLED(tp)) { - if (tp->ecn_flags & TE_RECV_ECN_CE) { - tcpstat.tcps_ecn_conn_recv_ce++; - INP_INC_IFNET_STAT(inp, ecn_conn_recv_ce); - } - if (tp->ecn_flags & TE_RECV_ECN_ECE) { - tcpstat.tcps_ecn_conn_recv_ece++; - INP_INC_IFNET_STAT(inp, ecn_conn_recv_ece); - } - if (tp->ecn_flags & (TE_RECV_ECN_CE | TE_RECV_ECN_ECE)) { - if (tp->t_stat.txretransmitbytes > 0 || - tp->t_stat.rxoutoforderbytes > 0) { - tcpstat.tcps_ecn_conn_pl_ce++; - INP_INC_IFNET_STAT(inp, ecn_conn_plce); - } else { - tcpstat.tcps_ecn_conn_nopl_ce++; - INP_INC_IFNET_STAT(inp, ecn_conn_noplce); - } - } else { - if (tp->t_stat.txretransmitbytes > 0 || - tp->t_stat.rxoutoforderbytes > 0) { - tcpstat.tcps_ecn_conn_plnoce++; - INP_INC_IFNET_STAT(inp, ecn_conn_plnoce); - } - } - } - - /* Aggregate performance stats */ - if (inp->inp_last_outifp != NULL && !(tp->t_flags & TF_LOCAL)) { - struct ifnet *ifp = inp->inp_last_outifp; - ifnet_lock_shared(ifp); - if ((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) == - IFRF_ATTACHED) { - if (inp->inp_vflag & INP_IPV6) { - ifp->if_ipv6_stat->timestamp = net_uptime(); - if (TCP_ECN_ENABLED(tp)) { - tcp_update_ecn_perf_stats(tp, - &ifp->if_ipv6_stat->ecn_on); - } else { - tcp_update_ecn_perf_stats(tp, - &ifp->if_ipv6_stat->ecn_off); - } - } else { - ifp->if_ipv4_stat->timestamp = net_uptime(); - if (TCP_ECN_ENABLED(tp)) { - tcp_update_ecn_perf_stats(tp, - &ifp->if_ipv4_stat->ecn_on); - } else { - tcp_update_ecn_perf_stats(tp, - &ifp->if_ipv4_stat->ecn_off); - } - } - } - ifnet_lock_done(ifp); - } + /* performance stats per interface */ + tcp_create_ifnet_stats_per_flow(tp, &ifs); + tcp_update_stats_per_flow(&ifs, inp->inp_last_outifp); tcp_free_sackholes(tp); tcp_notify_ack_free(tp); @@ -1497,16 +1538,6 @@ no_valid_rt: m_freem_list(tp->t_pktlist_head); TCP_PKTLIST_CLEAR(tp); -#if MPTCP - /* Clear MPTCP state */ - if ((so->so_flags & SOF_MPTCP_TRUE) || - (so->so_flags & SOF_MP_SUBFLOW)) { - soevent(so, (SO_FILT_HINT_LOCKED | SO_FILT_HINT_DELETEOK)); - } - tp->t_mpflags = 0; - tp->t_mptcb = NULL; -#endif /* MPTCP */ - if (so->so_flags1 & SOF1_CACHED_IN_SOCK_LAYER) inp->inp_saved_ppcb = (caddr_t) tp; @@ -1609,11 +1640,11 @@ tcp_drain(void) LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list) { if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) != WNT_STOPUSING) { - tcp_lock(inp->inp_socket, 1, 0); + socket_lock(inp->inp_socket, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { /* lost a race, try the next one */ - tcp_unlock(inp->inp_socket, 1, 0); + socket_unlock(inp->inp_socket, 1); continue; } tp = intotcpcb(inp); @@ -1623,7 +1654,7 @@ tcp_drain(void) so_drain_extended_bk_idle(inp->inp_socket); - tcp_unlock(inp->inp_socket, 1, 0); + socket_unlock(inp->inp_socket, 1); } } lck_rw_done(tcbinfo.ipi_lock); @@ -1648,6 +1679,7 @@ tcp_notify(struct inpcb *inp, int error) tp = (struct tcpcb *)inp->inp_ppcb; + VERIFY(tp != NULL); /* * Ignore some errors if we are hooked up. * If connection hasn't completed, has retransmitted several times, @@ -1658,7 +1690,10 @@ tcp_notify(struct inpcb *inp, int error) if (tp->t_state == TCPS_ESTABLISHED && (error == EHOSTUNREACH || error == ENETUNREACH || error == EHOSTDOWN)) { - return; + if (inp->inp_route.ro_rt) { + rtfree(inp->inp_route.ro_rt); + inp->inp_route.ro_rt = (struct rtentry *)NULL; + } } else if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift > 3 && tp->t_softerror) tcp_drop(tp, error); @@ -1681,9 +1716,7 @@ tcp_bwmeas_alloc(struct tcpcb *tp) bzero(elm, bwmeas_elm_size); elm->bw_minsizepkts = TCP_BWMEAS_BURST_MINSIZE; - elm->bw_maxsizepkts = TCP_BWMEAS_BURST_MAXSIZE; elm->bw_minsize = elm->bw_minsizepkts * tp->t_maxseg; - elm->bw_maxsize = elm->bw_maxsizepkts * tp->t_maxseg; return (elm); } @@ -1854,13 +1887,13 @@ tcp_pcblist SYSCTL_HANDLER_ARGS if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) continue; - tcp_lock(inp->inp_socket, 1, NULL); + socket_lock(inp->inp_socket, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); continue; } if (inp->inp_gencnt > gencnt) { - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); continue; } @@ -1878,7 +1911,7 @@ tcp_pcblist SYSCTL_HANDLER_ARGS if (inp->inp_socket) sotoxsocket(inp->inp_socket, &xt.xt_socket); - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); error = SYSCTL_OUT(req, &xt, sizeof(xt)); } @@ -1906,6 +1939,7 @@ SYSCTL_PROC(_net_inet_tcp, TCPCTL_PCBLIST, pcblist, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, tcp_pcblist, "S,xtcpcb", "List of active TCP connections"); +#if !CONFIG_EMBEDDED static void tcpcb_to_xtcpcb64(struct tcpcb *tp, struct xtcpcb64 *otp) @@ -2029,18 +2063,18 @@ tcp_pcblist64 SYSCTL_HANDLER_ARGS for (i = 0; i < n; i++) { struct xtcpcb64 xt; struct inpcb *inp; - + inp = inp_list[i]; if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) continue; - tcp_lock(inp->inp_socket, 1, NULL); + socket_lock(inp->inp_socket, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); continue; } if (inp->inp_gencnt > gencnt) { - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); continue; } @@ -2056,7 +2090,7 @@ tcp_pcblist64 SYSCTL_HANDLER_ARGS sotoxsocket64(inp->inp_socket, &xt.xt_inpcb.xi_socket); - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); error = SYSCTL_OUT(req, &xt, sizeof(xt)); } @@ -2084,6 +2118,7 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, pcblist64, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, tcp_pcblist64, "S,xtcpcb64", "List of active TCP connections"); +#endif /* !CONFIG_EMBEDDED */ static int tcp_pcblist_n SYSCTL_HANDLER_ARGS @@ -2106,8 +2141,8 @@ __private_extern__ void tcp_get_ports_used(uint32_t ifindex, int protocol, uint32_t flags, bitstr_t *bitfield) { - inpcb_get_ports_used(ifindex, protocol, flags, - bitfield, &tcbinfo); + inpcb_get_ports_used(ifindex, protocol, flags, bitfield, + &tcbinfo); } __private_extern__ uint32_t @@ -2119,7 +2154,7 @@ tcp_count_opportunistic(unsigned int ifindex, u_int32_t flags) __private_extern__ uint32_t tcp_find_anypcb_byaddr(struct ifaddr *ifa) { - return (inpcb_find_anypcb_byaddr(ifa, &tcbinfo)); + return (inpcb_find_anypcb_byaddr(ifa, &tcbinfo)); } static void @@ -2219,14 +2254,15 @@ tcp_handle_msgsize(struct ip *ip, struct inpcb *inp) } void -tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip) +tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip, __unused struct ifnet *ifp) { tcp_seq icmp_tcp_seq; struct ip *ip = vip; struct in_addr faddr; struct inpcb *inp; struct tcpcb *tp; - + struct tcphdr *th; + struct icmp *icp; void (*notify)(struct inpcb *, int) = tcp_notify; faddr = ((struct sockaddr_in *)(void *)sa)->sin_addr; @@ -2236,121 +2272,189 @@ tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip) if ((unsigned)cmd >= PRC_NCMDS) return; + /* Source quench is deprecated */ + if (cmd == PRC_QUENCH) + return; + if (cmd == PRC_MSGSIZE) notify = tcp_mtudisc; else if (icmp_may_rst && (cmd == PRC_UNREACH_ADMIN_PROHIB || - cmd == PRC_UNREACH_PORT) && ip) + cmd == PRC_UNREACH_PORT || cmd == PRC_UNREACH_PROTOCOL || + cmd == PRC_TIMXCEED_INTRANS) && ip) notify = tcp_drop_syn_sent; - else if (PRC_IS_REDIRECT(cmd)) { - ip = 0; - notify = in_rtchange; - } else if (cmd == PRC_HOSTDEAD) - ip = 0; - /* Source quench is deprecated */ - else if (cmd == PRC_QUENCH) + /* + * Hostdead is ugly because it goes linearly through all PCBs. + * XXX: We never get this from ICMP, otherwise it makes an + * excellent DoS attack on machines with many connections. + */ + else if (cmd == PRC_HOSTDEAD) + ip = NULL; + else if (inetctlerrmap[cmd] == 0 && !PRC_IS_REDIRECT(cmd)) return; - else if (inetctlerrmap[cmd] == 0) + + + if (ip == NULL) { + in_pcbnotifyall(&tcbinfo, faddr, inetctlerrmap[cmd], notify); return; - if (ip) { - struct tcphdr th; - struct icmp *icp; + } - icp = (struct icmp *)(void *) - ((caddr_t)ip - offsetof(struct icmp, icmp_ip)); - /* - * Only the first 8 bytes of TCP header will be returned. - */ - bzero(&th, sizeof(th)); - bcopy(((caddr_t)ip + (IP_VHL_HL(ip->ip_vhl) << 2)), &th, 8); - inp = in_pcblookup_hash(&tcbinfo, faddr, th.th_dport, - ip->ip_src, th.th_sport, 0, NULL); - if (inp != NULL && inp->inp_socket != NULL) { - tcp_lock(inp->inp_socket, 1, 0); - if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == - WNT_STOPUSING) { - tcp_unlock(inp->inp_socket, 1, 0); - return; - } - icmp_tcp_seq = htonl(th.th_seq); - tp = intotcpcb(inp); - if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) && - SEQ_LT(icmp_tcp_seq, tp->snd_max)) { - if (cmd == PRC_MSGSIZE) - tcp_handle_msgsize(ip, inp); + icp = (struct icmp *)(void *) + ((caddr_t)ip - offsetof(struct icmp, icmp_ip)); + th = (struct tcphdr *)(void *)((caddr_t)ip + (IP_VHL_HL(ip->ip_vhl) << 2)); + icmp_tcp_seq = ntohl(th->th_seq); - (*notify)(inp, inetctlerrmap[cmd]); - } - tcp_unlock(inp->inp_socket, 1, 0); + inp = in_pcblookup_hash(&tcbinfo, faddr, th->th_dport, + ip->ip_src, th->th_sport, 0, NULL); + + if (inp == NULL || + inp->inp_socket == NULL) { + return; + } + + socket_lock(inp->inp_socket, 1); + if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == + WNT_STOPUSING) { + socket_unlock(inp->inp_socket, 1); + return; + } + + if (PRC_IS_REDIRECT(cmd)) { + /* signal EHOSTDOWN, as it flushes the cached route */ + (*notify)(inp, EHOSTDOWN); + } else { + tp = intotcpcb(inp); + if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) && + SEQ_LT(icmp_tcp_seq, tp->snd_max)) { + if (cmd == PRC_MSGSIZE) + tcp_handle_msgsize(ip, inp); + + (*notify)(inp, inetctlerrmap[cmd]); } - } else - in_pcbnotifyall(&tcbinfo, faddr, inetctlerrmap[cmd], notify); + } + socket_unlock(inp->inp_socket, 1); } #if INET6 void -tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d) +tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d, __unused struct ifnet *ifp) { - struct tcphdr th; + tcp_seq icmp_tcp_seq; + struct in6_addr *dst; + struct tcphdr *th; void (*notify)(struct inpcb *, int) = tcp_notify; struct ip6_hdr *ip6; struct mbuf *m; + struct inpcb *inp; + struct tcpcb *tp; + struct icmp6_hdr *icmp6; struct ip6ctlparam *ip6cp = NULL; const struct sockaddr_in6 *sa6_src = NULL; - int off; - struct tcp_portonly { - u_int16_t th_sport; - u_int16_t th_dport; - } *thp; + unsigned int mtu; + unsigned int off; if (sa->sa_family != AF_INET6 || sa->sa_len != sizeof(struct sockaddr_in6)) return; - if ((unsigned)cmd >= PRC_NCMDS) + /* Source quench is deprecated */ + if (cmd == PRC_QUENCH) return; - if (cmd == PRC_MSGSIZE) - notify = tcp_mtudisc; - else if (!PRC_IS_REDIRECT(cmd) && (inet6ctlerrmap[cmd] == 0)) - return; - /* Source quench is deprecated */ - else if (cmd == PRC_QUENCH) + if ((unsigned)cmd >= PRC_NCMDS) return; /* if the parameter is from icmp6, decode it. */ if (d != NULL) { ip6cp = (struct ip6ctlparam *)d; + icmp6 = ip6cp->ip6c_icmp6; m = ip6cp->ip6c_m; ip6 = ip6cp->ip6c_ip6; off = ip6cp->ip6c_off; sa6_src = ip6cp->ip6c_src; + dst = ip6cp->ip6c_finaldst; } else { m = NULL; ip6 = NULL; off = 0; /* fool gcc */ sa6_src = &sa6_any; + dst = NULL; } - if (ip6) { + if (cmd == PRC_MSGSIZE) + notify = tcp_mtudisc; + else if (icmp_may_rst && (cmd == PRC_UNREACH_ADMIN_PROHIB || + cmd == PRC_UNREACH_PORT || cmd == PRC_TIMXCEED_INTRANS) && + ip6 != NULL) + notify = tcp_drop_syn_sent; + /* + * Hostdead is ugly because it goes linearly through all PCBs. + * XXX: We never get this from ICMP, otherwise it makes an + * excellent DoS attack on machines with many connections. + */ + else if (cmd == PRC_HOSTDEAD) + ip6 = NULL; + else if (inet6ctlerrmap[cmd] == 0 && !PRC_IS_REDIRECT(cmd)) + return; + + + if (ip6 == NULL) { + in6_pcbnotify(&tcbinfo, sa, 0, (struct sockaddr *)(size_t)sa6_src, + 0, cmd, NULL, notify); + return; + } + + if (m == NULL || + (m->m_pkthdr.len < (int32_t) (off + offsetof(struct tcphdr, th_seq)))) + return; + + th = (struct tcphdr *)(void *)mtodo(m, off); + icmp_tcp_seq = ntohl(th->th_seq); + + if (cmd == PRC_MSGSIZE) { + mtu = ntohl(icmp6->icmp6_mtu); /* - * XXX: We assume that when IPV6 is non NULL, - * M and OFF are valid. + * If no alternative MTU was proposed, or the proposed + * MTU was too small, set to the min. */ + if (mtu < IPV6_MMTU) + mtu = IPV6_MMTU - 8; + } - /* check if we can safely examine src and dst ports */ - if (m->m_pkthdr.len < off + sizeof(*thp)) - return; + inp = in6_pcblookup_hash(&tcbinfo, &ip6->ip6_dst, th->th_dport, + &ip6->ip6_src, th->th_sport, 0, NULL); - bzero(&th, sizeof(th)); - m_copydata(m, off, sizeof(*thp), (caddr_t)&th); + if (inp == NULL || + inp->inp_socket == NULL) { + return; + } - in6_pcbnotify(&tcbinfo, sa, th.th_dport, - (struct sockaddr *)ip6cp->ip6c_src, - th.th_sport, cmd, NULL, notify); + socket_lock(inp->inp_socket, 1); + if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == + WNT_STOPUSING) { + socket_unlock(inp->inp_socket, 1); + return; + } + + if (PRC_IS_REDIRECT(cmd)) { + /* signal EHOSTDOWN, as it flushes the cached route */ + (*notify)(inp, EHOSTDOWN); } else { - in6_pcbnotify(&tcbinfo, sa, 0, - (struct sockaddr *)(size_t)sa6_src, 0, cmd, NULL, notify); + tp = intotcpcb(inp); + if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) && + SEQ_LT(icmp_tcp_seq, tp->snd_max)) { + if (cmd == PRC_MSGSIZE) { + /* + * Only process the offered MTU if it + * is smaller than the current one. + */ + if (mtu < tp->t_maxseg + + (sizeof (*th) + sizeof (*ip6))) + (*notify)(inp, inetctlerrmap[cmd]); + } else + (*notify)(inp, inetctlerrmap[cmd]); + } } + socket_unlock(inp->inp_socket, 1); } #endif /* INET6 */ @@ -2426,7 +2530,7 @@ tcp_new_isn(struct tcpcb *tp) (((u_int)isn_last_reseed + (u_int)tcp_isn_reseed_interval*hz) < (u_int)timenow.tv_sec))) { #ifdef __APPLE__ - read_random(&isn_secret, sizeof(isn_secret)); + read_frandom(&isn_secret, sizeof(isn_secret)); #else read_random_unlimited(&isn_secret, sizeof(isn_secret)); #endif @@ -2597,7 +2701,7 @@ tcp_rtlookup(struct inpcb *inp, unsigned int input_ifscope) struct rtentry *rt; struct tcpcb *tp; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); ro = &inp->inp_route; if ((rt = ro->ro_rt) != NULL) @@ -2659,8 +2763,10 @@ tcp_rtlookup(struct inpcb *inp, unsigned int input_ifscope) soif2kcl(inp->inp_socket, (rt->rt_ifp->if_eflags & IFEF_2KCL)); tcp_set_ecn(tp, rt->rt_ifp); - if (inp->inp_last_outifp == NULL) + if (inp->inp_last_outifp == NULL) { inp->inp_last_outifp = rt->rt_ifp; + + } } /* Note if the peer is local */ @@ -2685,7 +2791,7 @@ tcp_rtlookup6(struct inpcb *inp, unsigned int input_ifscope) struct rtentry *rt; struct tcpcb *tp; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); ro6 = &inp->in6p_route; if ((rt = ro6->ro_rt) != NULL) @@ -2757,8 +2863,9 @@ tcp_rtlookup6(struct inpcb *inp, unsigned int input_ifscope) soif2kcl(inp->inp_socket, (rt->rt_ifp->if_eflags & IFEF_2KCL)); tcp_set_ecn(tp, rt->rt_ifp); - if (inp->inp_last_outifp == NULL) + if (inp->inp_last_outifp == NULL) { inp->inp_last_outifp = rt->rt_ifp; + } } /* Note if the peer is local */ @@ -2875,8 +2982,36 @@ tcp_lock(struct socket *so, int refcount, void *lr) else lr_saved = lr; +retry: if (so->so_pcb != NULL) { - lck_mtx_lock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); + if (so->so_flags & SOF_MP_SUBFLOW) { + struct mptcb *mp_tp = tptomptp(sototcpcb(so)); + VERIFY(mp_tp); + + mpte_lock_assert_notheld(mp_tp->mpt_mpte); + + mpte_lock(mp_tp->mpt_mpte); + + /* + * Check if we became non-MPTCP while waiting for the lock. + * If yes, we have to retry to grab the right lock. + */ + if (!(so->so_flags & SOF_MP_SUBFLOW)) { + mpte_unlock(mp_tp->mpt_mpte); + goto retry; + } + } else { + lck_mtx_lock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); + + if (so->so_flags & SOF_MP_SUBFLOW) { + /* + * While waiting for the lock, we might have + * become MPTCP-enabled (see mptcp_subflow_socreate). + */ + lck_mtx_unlock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); + goto retry; + } + } } else { panic("tcp_lock: so=%p NO PCB! lr=%p lrh= %s\n", so, lr_saved, solockhistory_nr(so)); @@ -2926,17 +3061,27 @@ tcp_unlock(struct socket *so, int refcount, void *lr) so, so->so_usecount, lr_saved, solockhistory_nr(so)); /* NOTREACHED */ } else { - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; - lck_mtx_unlock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); + + if (so->so_flags & SOF_MP_SUBFLOW) { + struct mptcb *mp_tp = tptomptp(sototcpcb(so)); + + VERIFY(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + + mpte_unlock(mp_tp->mpt_mpte); + } else { + LCK_MTX_ASSERT(&((struct inpcb *)so->so_pcb)->inpcb_mtx, + LCK_MTX_ASSERT_OWNED); + lck_mtx_unlock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); + } } return (0); } lck_mtx_t * -tcp_getlock(struct socket *so, __unused int locktype) +tcp_getlock(struct socket *so, int flags) { struct inpcb *inp = sotoinpcb(so); @@ -2944,7 +3089,14 @@ tcp_getlock(struct socket *so, __unused int locktype) if (so->so_usecount < 0) panic("tcp_getlock: so=%p usecount=%x lrh= %s\n", so, so->so_usecount, solockhistory_nr(so)); - return (&inp->inpcb_mtx); + + if (so->so_flags & SOF_MP_SUBFLOW) { + struct mptcb *mp_tp = tptomptp(sototcpcb(so)); + + return (mpte_getlock(mp_tp->mpt_mpte, flags)); + } else { + return (&inp->inpcb_mtx); + } } else { panic("tcp_getlock: so=%p NULL so_pcb %s\n", so, solockhistory_nr(so)); @@ -3118,7 +3270,7 @@ calculate_tcp_clock(void) * is to update the counter returnable via net_uptime() when * we read time. */ - net_update_uptime_secs(now.tv_sec); + net_update_uptime_with_time(&now); timevaladd(&tv, &interval); if (timevalcmp(&now, &tv, >)) { @@ -3447,7 +3599,8 @@ tcp_make_keepalive_frame(struct tcpcb *tp, struct ifnet *ifp, ip = (__typeof__(ip))(void *)data; - ip->ip_id = ip_randomid(); + ip->ip_id = rfc6864 ? 0 : ip_randomid(); + ip->ip_off = htons(IP_DF); ip->ip_len = htons(sizeof(struct ip) + sizeof(struct tcphdr)); ip->ip_ttl = inp->inp_ip_ttl; ip->ip_tos |= (inp->inp_ip_tos & ~IPTOS_ECN_MASK); @@ -3547,43 +3700,43 @@ tcp_fill_keepalive_offload_frames(ifnet_t ifp, if (inp->inp_ppcb == NULL || in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) continue; - tcp_lock(so, 1, 0); + socket_lock(so, 1); /* Release the want count */ if (inp->inp_ppcb == NULL || (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING)) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } if ((inp->inp_vflag & INP_IPV4) && (inp->inp_laddr.s_addr == INADDR_ANY || inp->inp_faddr.s_addr == INADDR_ANY)) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } if ((inp->inp_vflag & INP_IPV6) && (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) || IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } if (inp->inp_lport == 0 || inp->inp_fport == 0) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } if (inp->inp_last_outifp == NULL || inp->inp_last_outifp->if_index != ifp->if_index) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } if ((inp->inp_vflag & INP_IPV4) && frame_data_offset + sizeof(struct ip) + sizeof(struct tcphdr) > IFNET_KEEPALIVE_OFFLOAD_FRAME_DATA_SIZE) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } else if (!(inp->inp_vflag & INP_IPV4) && frame_data_offset + sizeof(struct ip6_hdr) + sizeof(struct tcphdr) > IFNET_KEEPALIVE_OFFLOAD_FRAME_DATA_SIZE) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } /* @@ -3592,7 +3745,7 @@ tcp_fill_keepalive_offload_frames(ifnet_t ifp, * for processes that will sent and receive data */ if (tp->t_state != TCPS_ESTABLISHED) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } /* @@ -3646,7 +3799,7 @@ tcp_fill_keepalive_offload_frames(ifnet_t ifp, */ m = tcp_make_keepalive_frame(tp, ifp, TRUE); if (m == NULL) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } bcopy(m->m_data, frame->data + frame_data_offset, @@ -3658,7 +3811,7 @@ tcp_fill_keepalive_offload_frames(ifnet_t ifp, */ m = tcp_make_keepalive_frame(tp, ifp, FALSE); if (m == NULL) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } bcopy(m->m_data, frame->reply_data + frame_data_offset, @@ -3666,7 +3819,7 @@ tcp_fill_keepalive_offload_frames(ifnet_t ifp, m_freem(m); frame_index++; - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } lck_rw_done(tcbinfo.ipi_lock); *used_frames_count = frame_index; @@ -3820,3 +3973,125 @@ inp_get_sndbytes_allunsent(struct socket *so, u_int32_t th_ack) } return (0); } + +#define IFP_PER_FLOW_STAT(_ipv4_, _stat_) { \ + if (_ipv4_) { \ + ifp->if_ipv4_stat->_stat_++; \ + } else { \ + ifp->if_ipv6_stat->_stat_++; \ + } \ +} + +#define FLOW_ECN_ENABLED(_flags_) \ + ((_flags_ & (TE_ECN_ON)) == (TE_ECN_ON)) + +void tcp_update_stats_per_flow(struct ifnet_stats_per_flow *ifs, + struct ifnet *ifp) +{ + if (ifp == NULL || !IF_FULLY_ATTACHED(ifp)) + return; + + ifnet_lock_shared(ifp); + if (ifs->ecn_flags & TE_SETUPSENT) { + if (ifs->ecn_flags & TE_CLIENT_SETUP) { + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_client_setup); + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_client_success); + } else if (ifs->ecn_flags & TE_LOST_SYN) { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_syn_lost); + } else { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_peer_nosupport); + } + } else { + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_server_setup); + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_server_success); + } else if (ifs->ecn_flags & TE_LOST_SYN) { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_synack_lost); + } else { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_peer_nosupport); + } + } + } else { + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_off_conn); + } + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + if (ifs->ecn_flags & TE_RECV_ECN_CE) { + tcpstat.tcps_ecn_conn_recv_ce++; + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_conn_recv_ce); + } + if (ifs->ecn_flags & TE_RECV_ECN_ECE) { + tcpstat.tcps_ecn_conn_recv_ece++; + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_conn_recv_ece); + } + if (ifs->ecn_flags & (TE_RECV_ECN_CE | TE_RECV_ECN_ECE)) { + if (ifs->txretransmitbytes > 0 || + ifs->rxoutoforderbytes > 0) { + tcpstat.tcps_ecn_conn_pl_ce++; + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_conn_plce); + } else { + tcpstat.tcps_ecn_conn_nopl_ce++; + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_conn_noplce); + } + } else { + if (ifs->txretransmitbytes > 0 || + ifs->rxoutoforderbytes > 0) { + tcpstat.tcps_ecn_conn_plnoce++; + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_conn_plnoce); + } + } + } + + /* Other stats are interesting for non-local connections only */ + if (ifs->local) { + ifnet_lock_done(ifp); + return; + } + + if (ifs->ipv4) { + ifp->if_ipv4_stat->timestamp = net_uptime(); + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + tcp_flow_ecn_perf_stats(ifs, &ifp->if_ipv4_stat->ecn_on); + } else { + tcp_flow_ecn_perf_stats(ifs, &ifp->if_ipv4_stat->ecn_off); + } + } else { + ifp->if_ipv6_stat->timestamp = net_uptime(); + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + tcp_flow_ecn_perf_stats(ifs, &ifp->if_ipv6_stat->ecn_on); + } else { + tcp_flow_ecn_perf_stats(ifs, &ifp->if_ipv6_stat->ecn_off); + } + } + + if (ifs->rxmit_drop) { + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_on.rxmit_drop); + } else { + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_off.rxmit_drop); + } + } + if (ifs->ecn_fallback_synloss) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_fallback_synloss); + if (ifs->ecn_fallback_droprst) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_fallback_droprst); + if (ifs->ecn_fallback_droprxmt) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_fallback_droprxmt); + if (ifs->ecn_fallback_ce) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_fallback_ce); + if (ifs->ecn_fallback_reorder) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_fallback_reorder); + if (ifs->ecn_recv_ce > 0) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_recv_ce); + if (ifs->ecn_recv_ece > 0) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_recv_ece); + + tcp_flow_lim_stats(ifs, &ifp->if_lim_stat); + ifnet_lock_done(ifp); +} diff --git a/bsd/netinet/tcp_timer.c b/bsd/netinet/tcp_timer.c index 5c87a2d18..f410382ed 100644 --- a/bsd/netinet/tcp_timer.c +++ b/bsd/netinet/tcp_timer.c @@ -124,10 +124,10 @@ struct tcptailq tcp_tw_tailq; static int sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS { -#pragma unused(arg1, arg2) +#pragma unused(arg2) int error, s, tt; - tt = *(int *)oidp->oid_arg1; + tt = *(int *)arg1; s = tt * 1000 / TCP_RETRANSHZ;; error = sysctl_handle_int(oidp, &s, 0, req); @@ -138,34 +138,65 @@ sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS if (tt < 1) return (EINVAL); - *(int *)oidp->oid_arg1 = tt; - return (0); + *(int *)arg1 = tt; + SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, *(int*)arg1); + return (0); } -int tcp_keepinit; +#if SYSCTL_SKMEM +int tcp_keepinit = TCPTV_KEEP_INIT; +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_keepinit, offsetof(skmem_sysctl, tcp.keepinit), + sysctl_msec_to_ticks, "I", ""); + +int tcp_keepidle = TCPTV_KEEP_IDLE; +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_keepidle, offsetof(skmem_sysctl, tcp.keepidle), + sysctl_msec_to_ticks, "I", ""); + +int tcp_keepintvl = TCPTV_KEEPINTVL; +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_keepintvl, offsetof(skmem_sysctl, tcp.keepintvl), + sysctl_msec_to_ticks, "I", ""); + +SYSCTL_SKMEM_TCP_INT(OID_AUTO, keepcnt, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_keepcnt, TCPTV_KEEPCNT, "number of times to repeat keepalive"); + +int tcp_msl = TCPTV_MSL; +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_msl, offsetof(skmem_sysctl, tcp.msl), + sysctl_msec_to_ticks, "I", "Maximum segment lifetime"); +#else /* SYSCTL_SKMEM */ +int tcp_keepinit; SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepinit, 0, sysctl_msec_to_ticks, "I", ""); -int tcp_keepidle; +int tcp_keepidle; SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepidle, 0, sysctl_msec_to_ticks, "I", ""); -int tcp_keepintvl; +int tcp_keepintvl; SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", ""); -int tcp_keepcnt; +int tcp_keepcnt; SYSCTL_INT(_net_inet_tcp, OID_AUTO, keepcnt, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepcnt, 0, "number of times to repeat keepalive"); -int tcp_msl; +int tcp_msl; SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime"); +#endif /* SYSCTL_SKMEM */ /* * Avoid DoS via TCP Robustness in Persist Condition @@ -176,26 +207,32 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, * Expressed in milliseconds to be consistent without timeout related * values, the TCP socket option is in seconds. */ +#if SYSCTL_SKMEM +u_int32_t tcp_max_persist_timeout = 0; +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_max_persist_timeout, offsetof(skmem_sysctl, tcp.max_persist_timeout), + sysctl_msec_to_ticks, "I", "Maximum persistence timeout for ZWP"); +#else /* SYSCTL_SKMEM */ u_int32_t tcp_max_persist_timeout = 0; SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_max_persist_timeout, 0, sysctl_msec_to_ticks, "I", "Maximum persistence timeout for ZWP"); +#endif /* SYSCTL_SKMEM */ -static int always_keepalive = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, always_keepalive, - CTLFLAG_RW | CTLFLAG_LOCKED, - &always_keepalive , 0, "Assume SO_KEEPALIVE on all TCP connections"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, always_keepalive, + CTLFLAG_RW | CTLFLAG_LOCKED, static int, always_keepalive, 0, + "Assume SO_KEEPALIVE on all TCP connections"); /* * This parameter determines how long the timer list will stay in fast or * quick mode even though all connections are idle. In this state, the * timer will run more frequently anticipating new data. */ -int timer_fastmode_idlemax = TCP_FASTMODE_IDLERUN_MAX; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, timer_fastmode_idlemax, - CTLFLAG_RW | CTLFLAG_LOCKED, - &timer_fastmode_idlemax, 0, "Maximum idle generations in fast mode"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, timer_fastmode_idlemax, + CTLFLAG_RW | CTLFLAG_LOCKED, int, timer_fastmode_idlemax, + TCP_FASTMODE_IDLERUN_MAX, "Maximum idle generations in fast mode"); /* * See tcp_syn_backoff[] for interval values between SYN retransmits; @@ -204,10 +241,9 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, timer_fastmode_idlemax, * SYN retransmits. Setting it to 0 disables the dropping off of those * two options. */ -static int tcp_broken_peer_syn_rxmit_thres = 10; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, broken_peer_syn_rexmit_thres, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_broken_peer_syn_rxmit_thres, 0, - "Number of retransmitted SYNs before disabling RFC 1323 " +SYSCTL_SKMEM_TCP_INT(OID_AUTO, broken_peer_syn_rexmit_thres, + CTLFLAG_RW | CTLFLAG_LOCKED, static int, tcp_broken_peer_syn_rxmit_thres, + 10, "Number of retransmitted SYNs before disabling RFC 1323 " "options on local connections"); static int tcp_timer_advanced = 0; @@ -220,14 +256,12 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_resched_timerlist, CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_resched_timerlist, 0, "Number of times timer list was rescheduled as part of processing a packet"); -int tcp_pmtud_black_hole_detect = 1 ; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, pmtud_blackhole_detection, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_pmtud_black_hole_detect, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_detection, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_detect, 1, "Path MTU Discovery Black Hole Detection"); -int tcp_pmtud_black_hole_mss = 1200 ; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, pmtud_blackhole_mss, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_pmtud_black_hole_mss, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_mss, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_mss, 1200, "Path MTU Discovery Black Hole Detection lowered MSS"); static u_int32_t tcp_mss_rec_medium = 1200; @@ -240,7 +274,7 @@ int tcp_report_stats_interval = TCP_REPORT_STATS_INTERVAL; static boolean_t tcp_gc_done = FALSE; /* max idle probes */ -int tcp_maxpersistidle; +int tcp_maxpersistidle = TCPTV_KEEP_IDLE; /* * TCP delack timer is set to 100 ms. Since the processing of timer list @@ -311,6 +345,36 @@ struct tcp_last_report_stats { u_int32_t tcps_tfo_no_cookie_rcv; u_int32_t tcps_tfo_heuristics_disable; u_int32_t tcps_tfo_sndblackhole; + + /* MPTCP-related statistics */ + u_int32_t tcps_mptcp_handover_attempt; + u_int32_t tcps_mptcp_interactive_attempt; + u_int32_t tcps_mptcp_aggregate_attempt; + u_int32_t tcps_mptcp_fp_handover_attempt; + u_int32_t tcps_mptcp_fp_interactive_attempt; + u_int32_t tcps_mptcp_fp_aggregate_attempt; + u_int32_t tcps_mptcp_heuristic_fallback; + u_int32_t tcps_mptcp_fp_heuristic_fallback; + u_int32_t tcps_mptcp_handover_success_wifi; + u_int32_t tcps_mptcp_handover_success_cell; + u_int32_t tcps_mptcp_interactive_success; + u_int32_t tcps_mptcp_aggregate_success; + u_int32_t tcps_mptcp_fp_handover_success_wifi; + u_int32_t tcps_mptcp_fp_handover_success_cell; + u_int32_t tcps_mptcp_fp_interactive_success; + u_int32_t tcps_mptcp_fp_aggregate_success; + u_int32_t tcps_mptcp_handover_cell_from_wifi; + u_int32_t tcps_mptcp_handover_wifi_from_cell; + u_int32_t tcps_mptcp_interactive_cell_from_wifi; + u_int64_t tcps_mptcp_handover_cell_bytes; + u_int64_t tcps_mptcp_interactive_cell_bytes; + u_int64_t tcps_mptcp_aggregate_cell_bytes; + u_int64_t tcps_mptcp_handover_all_bytes; + u_int64_t tcps_mptcp_interactive_all_bytes; + u_int64_t tcps_mptcp_aggregate_all_bytes; + u_int32_t tcps_mptcp_back_to_wifi; + u_int32_t tcps_mptcp_wifi_proxy; + u_int32_t tcps_mptcp_cell_proxy; }; @@ -422,7 +486,7 @@ add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay) uint32_t timer; /* pcb list should be locked when we get here */ - lck_rw_assert(pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); /* We may get here multiple times, so check */ if (!(inp->inp_flags2 & INP2_TIMEWAIT)) { @@ -458,9 +522,9 @@ add_to_time_wait(struct tcpcb *tp, uint32_t delay) nstat_pcb_detach(tp->t_inpcb); if (!lck_rw_try_lock_exclusive(pcbinfo->ipi_lock)) { - tcp_unlock(tp->t_inpcb->inp_socket, 0, 0); + socket_unlock(tp->t_inpcb->inp_socket, 0); lck_rw_lock_exclusive(pcbinfo->ipi_lock); - tcp_lock(tp->t_inpcb->inp_socket, 0, 0); + socket_lock(tp->t_inpcb->inp_socket, 0); } add_to_time_wait_locked(tp, delay); lck_rw_done(pcbinfo->ipi_lock); @@ -481,12 +545,22 @@ static boolean_t tcp_garbage_collect(struct inpcb *inp, int istimewait) { boolean_t active = FALSE; - struct socket *so; + struct socket *so, *mp_so = NULL; struct tcpcb *tp; so = inp->inp_socket; tp = intotcpcb(inp); + if (so->so_flags & SOF_MP_SUBFLOW) { + mp_so = mptetoso(tptomptp(tp)->mpt_mpte); + if (!socket_try_lock(mp_so)) { + mp_so = NULL; + active = TRUE; + goto out; + } + mp_so->so_usecount++; + } + /* * Skip if still in use or busy; it would have been more efficient * if we were to test so_usecount against 0, but this isn't possible @@ -494,20 +568,21 @@ tcp_garbage_collect(struct inpcb *inp, int istimewait) * overflow sockets that are eligible for garbage collection have * their usecounts set to 1. */ - if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) - return (TRUE); + if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) { + active = TRUE; + goto out; + } /* Check again under the lock */ if (so->so_usecount > 1) { if (inp->inp_wantcnt == WNT_STOPUSING) active = TRUE; lck_mtx_unlock(&inp->inpcb_mtx); - return (active); + goto out; } - if (istimewait && - TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) && - tp->t_state != TCPS_CLOSED) { + if (istimewait && TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) && + tp->t_state != TCPS_CLOSED) { /* Become a regular mutex */ lck_mtx_convert_spin(&inp->inpcb_mtx); tcp_close(tp); @@ -544,10 +619,11 @@ tcp_garbage_collect(struct inpcb *inp, int istimewait) if (inp->inp_wantcnt == WNT_STOPUSING) active = TRUE; lck_mtx_unlock(&inp->inpcb_mtx); - return (active); + goto out; } else if (inp->inp_wantcnt != WNT_STOPUSING) { lck_mtx_unlock(&inp->inpcb_mtx); - return (FALSE); + active = FALSE; + goto out; } /* @@ -583,12 +659,28 @@ tcp_garbage_collect(struct inpcb *inp, int istimewait) #endif /* INET6 */ in_pcbdetach(inp); } + + if (mp_so) { + mptcp_subflow_del(tptomptp(tp)->mpt_mpte, tp->t_mpsub); + + /* so is now unlinked from mp_so - let's drop the lock */ + socket_unlock(mp_so, 1); + mp_so = NULL; + } + in_pcbdispose(inp); - return (FALSE); + active = FALSE; + goto out; } lck_mtx_unlock(&inp->inpcb_mtx); - return (TRUE); + active = TRUE; + +out: + if (mp_so) + socket_unlock(mp_so, 1); + + return (active); } /* @@ -758,6 +850,9 @@ tcp_pmtud_revert_segment_size(struct tcpcb *tp) CC_ALGO(tp)->cwnd_init(tp); tp->t_pmtud_start_ts = 0; tcpstat.tcps_pmtudbh_reverted++; + + /* change MSS according to recommendation, if there was one */ + tcp_update_mss_locked(tp->t_inpcb->inp_socket, NULL); } /* @@ -833,6 +928,20 @@ tcp_timers(struct tcpcb *tp, int timer) ((tp->t_flagsext & TF_RXTFINDROP) != 0 && (tp->t_flags & TF_SENTFIN) != 0 && tp->t_rxtshift >= 4) || (tp->t_rxtshift > 4 && last_sleep_ms >= TCP_SLEEP_TOO_LONG)) { + if (tp->t_state == TCPS_ESTABLISHED && + tp->t_rxt_minimum_timeout > 0) { + /* + * Avoid dropping a connection if minimum + * timeout is set and that time did not + * pass. We will retry sending + * retransmissions at the maximum interval + */ + if (TSTMP_LT(tcp_now, (tp->t_rxtstart + + tp->t_rxt_minimum_timeout))) { + tp->t_rxtshift = TCP_MAXRXTSHIFT - 1; + goto retransmit_packet; + } + } if ((tp->t_flagsext & TF_RXTFINDROP) != 0) { tcpstat.tcps_rxtfindrop++; } else if (last_sleep_ms >= TCP_SLEEP_TOO_LONG) { @@ -863,7 +972,7 @@ tcp_timers(struct tcpcb *tp, int timer) break; } - +retransmit_packet: tcpstat.tcps_rexmttimeo++; tp->t_accsleep_ms = accsleep_ms; @@ -886,6 +995,12 @@ tcp_timers(struct tcpcb *tp, int timer) mptcp_act_on_txfail(so); } + + if (so->so_flags & SOF_MP_SUBFLOW) { + struct mptses *mpte = tptomptp(tp)->mpt_mpte; + + mptcp_check_subflows_and_add(mpte); + } #endif /* MPTCP */ if (tp->t_adaptive_wtimo > 0 && @@ -921,7 +1036,8 @@ tcp_timers(struct tcpcb *tp, int timer) if (tp->t_state == TCPS_SYN_RECEIVED) tcp_disable_tfo(tp); - if ((tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && + if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) && + (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) && ((tp->t_state != TCPS_SYN_SENT && tp->t_rxtshift > 1) || tp->t_rxtshift > 2)) { @@ -944,12 +1060,26 @@ tcp_timers(struct tcpcb *tp, int timer) tcpstat.tcps_tfo_sndblackhole++; } + if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) && + (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) && + tp->t_rxtshift > 1) { + if (TSTMP_GT(tp->t_sndtime - 10 * TCP_RETRANSHZ, tp->t_rcvtime)) { + tcp_heuristic_tfo_middlebox(tp); + + so->so_error = ENODATA; + sorwakeup(so); + sowwakeup(so); + } + } + if (tp->t_state == TCPS_SYN_SENT) { rexmt = TCP_REXMTVAL(tp) * tcp_syn_backoff[tp->t_rxtshift]; tp->t_stat.synrxtshift = tp->t_rxtshift; /* When retransmitting, disable TFO */ - if (tfo_enabled(tp) && !(so->so_flags & SOF1_DATA_AUTHENTICATED)) { + if (tfo_enabled(tp) && + (!(so->so_flags1 & SOF1_DATA_AUTHENTICATED) || + (tp->t_flagsext & TF_FASTOPEN_HEUR))) { tp->t_flagsext &= ~TF_FASTOPEN; tp->t_tfo_flags |= TFO_F_SYN_LOSS; } @@ -1153,7 +1283,7 @@ fc_output: * Regular TCP connections do not send keepalives after closing * MPTCP must not also, after sending Data FINs. */ - struct mptcb *mp_tp = tp->t_mptcb; + struct mptcb *mp_tp = tptomptp(tp); if ((tp->t_mpflags & TMPF_MPTCP_TRUE) && (tp->t_state > TCPS_ESTABLISHED)) { goto dropit; @@ -1261,7 +1391,8 @@ fc_output: tp->t_timer[TCPT_KEEP] = min(OFFSET_FROM_START( tp, tcp_backoff[ind] * TCP_REXMTVAL(tp)), tp->t_timer[TCPT_KEEP]); - } else if (tp->t_tfo_probe_state == TFO_PROBE_WAIT_DATA) { + } else if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) && + tp->t_tfo_probe_state == TFO_PROBE_WAIT_DATA) { /* Still no data! Let's assume a TFO-error and err out... */ tcp_heuristic_tfo_middlebox(tp); @@ -1328,13 +1459,14 @@ fc_output: tcpstat.tcps_timeoutdrop++; postevent(so, 0, EV_TIMEOUT); soevent(so, - (SO_FILT_HINT_LOCKED| + (SO_FILT_HINT_LOCKED| SO_FILT_HINT_TIMEOUT)); tp = tcp_drop(tp, tp->t_softerror ? - tp->t_softerror : ETIMEDOUT); + tp->t_softerror : ETIMEDOUT); break; } tcpstat.tcps_join_rxmts++; + tp->t_mpflags |= TMPF_SND_JACK; tp->t_flags |= TF_ACKNOW; /* @@ -1448,7 +1580,7 @@ tcp_remove_timer(struct tcpcb *tp) { struct tcptimerlist *listp = &tcp_timer_list; - lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(tp->t_inpcb->inp_socket); if (!(TIMER_IS_ON_LIST(tp))) { return; } @@ -1521,10 +1653,11 @@ tcp_sched_timerlist(uint32_t offset) uint64_t deadline = 0; struct tcptimerlist *listp = &tcp_timer_list; - lck_mtx_assert(listp->mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(listp->mtx, LCK_MTX_ASSERT_OWNED); offset = min(offset, TCP_TIMERLIST_MAX_OFFSET); listp->runtime = tcp_now + offset; + listp->schedtime = tcp_now; if (listp->runtime == 0) { listp->runtime++; offset++; @@ -1560,7 +1693,7 @@ tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode, bzero(needtorun, sizeof(needtorun)); *te_mode = 0; - tcp_lock(tp->t_inpcb->inp_socket, 1, 0); + socket_lock(tp->t_inpcb->inp_socket, 1); so = tp->t_inpcb->inp_socket; /* Release the want count on inp */ @@ -1676,7 +1809,7 @@ done: offset = 0; } - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); return(offset); } @@ -1696,6 +1829,27 @@ tcp_run_timerlist(void * arg1, void * arg2) lck_mtx_lock(listp->mtx); + int32_t drift = tcp_now - listp->runtime; + if (drift <= 1) { + tcpstat.tcps_timer_drift_le_1_ms++; + } else if (drift <= 10) { + tcpstat.tcps_timer_drift_le_10_ms++; + } else if (drift <= 20) { + tcpstat.tcps_timer_drift_le_20_ms++; + } else if (drift <= 50) { + tcpstat.tcps_timer_drift_le_50_ms++; + } else if (drift <= 100) { + tcpstat.tcps_timer_drift_le_100_ms++; + } else if (drift <= 200) { + tcpstat.tcps_timer_drift_le_200_ms++; + } else if (drift <= 500) { + tcpstat.tcps_timer_drift_le_500_ms++; + } else if (drift <= 1000) { + tcpstat.tcps_timer_drift_le_1000_ms++; + } else { + tcpstat.tcps_timer_drift_gt_1000_ms++; + } + listp->running = TRUE; LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) { @@ -1975,7 +2129,7 @@ tcp_set_lotimer_index(struct tcpcb *tp) void tcp_check_timer_state(struct tcpcb *tp) { - lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(tp->t_inpcb->inp_socket); if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT) return; @@ -1999,6 +2153,19 @@ tcp_cumulative_stat(u_int32_t cur, u_int32_t *prev, u_int32_t *dest) return; } +static inline void +tcp_cumulative_stat64(u_int64_t cur, u_int64_t *prev, u_int64_t *dest) +{ + /* handle wrap around */ + int64_t diff = (int64_t) (cur - *prev); + if (diff > 0) + *dest = diff; + else + *dest = 0; + *prev = cur; + return; +} + __private_extern__ void tcp_report_stats(void) { @@ -2164,6 +2331,62 @@ tcp_report_stats(void) &prev.tcps_tfo_sndblackhole, &stat.tfo_sndblackhole); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_attempt, + &prev.tcps_mptcp_handover_attempt , &stat.mptcp_handover_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_attempt, + &prev.tcps_mptcp_interactive_attempt , &stat.mptcp_interactive_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_attempt, + &prev.tcps_mptcp_aggregate_attempt , &stat.mptcp_aggregate_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_attempt, + &prev.tcps_mptcp_fp_handover_attempt , &stat.mptcp_fp_handover_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_attempt, + &prev.tcps_mptcp_fp_interactive_attempt , &stat.mptcp_fp_interactive_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_attempt, + &prev.tcps_mptcp_fp_aggregate_attempt , &stat.mptcp_fp_aggregate_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_heuristic_fallback, + &prev.tcps_mptcp_heuristic_fallback , &stat.mptcp_heuristic_fallback); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_heuristic_fallback, + &prev.tcps_mptcp_fp_heuristic_fallback , &stat.mptcp_fp_heuristic_fallback); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_wifi, + &prev.tcps_mptcp_handover_success_wifi , &stat.mptcp_handover_success_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_cell, + &prev.tcps_mptcp_handover_success_cell , &stat.mptcp_handover_success_cell); + tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_success, + &prev.tcps_mptcp_interactive_success , &stat.mptcp_interactive_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_success, + &prev.tcps_mptcp_aggregate_success , &stat.mptcp_aggregate_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_wifi, + &prev.tcps_mptcp_fp_handover_success_wifi , &stat.mptcp_fp_handover_success_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_cell, + &prev.tcps_mptcp_fp_handover_success_cell , &stat.mptcp_fp_handover_success_cell); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_success, + &prev.tcps_mptcp_fp_interactive_success , &stat.mptcp_fp_interactive_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_success, + &prev.tcps_mptcp_fp_aggregate_success , &stat.mptcp_fp_aggregate_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_cell_from_wifi, + &prev.tcps_mptcp_handover_cell_from_wifi , &stat.mptcp_handover_cell_from_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_wifi_from_cell, + &prev.tcps_mptcp_handover_wifi_from_cell , &stat.mptcp_handover_wifi_from_cell); + tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_cell_from_wifi, + &prev.tcps_mptcp_interactive_cell_from_wifi , &stat.mptcp_interactive_cell_from_wifi); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_cell_bytes, + &prev.tcps_mptcp_handover_cell_bytes , &stat.mptcp_handover_cell_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_cell_bytes, + &prev.tcps_mptcp_interactive_cell_bytes , &stat.mptcp_interactive_cell_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_cell_bytes, + &prev.tcps_mptcp_aggregate_cell_bytes , &stat.mptcp_aggregate_cell_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_all_bytes, + &prev.tcps_mptcp_handover_all_bytes , &stat.mptcp_handover_all_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_all_bytes, + &prev.tcps_mptcp_interactive_all_bytes , &stat.mptcp_interactive_all_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_all_bytes, + &prev.tcps_mptcp_aggregate_all_bytes , &stat.mptcp_aggregate_all_bytes); + tcp_cumulative_stat(tcpstat.tcps_mptcp_back_to_wifi, + &prev.tcps_mptcp_back_to_wifi , &stat.mptcp_back_to_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_wifi_proxy, + &prev.tcps_mptcp_wifi_proxy , &stat.mptcp_wifi_proxy); + tcp_cumulative_stat(tcpstat.tcps_mptcp_cell_proxy, + &prev.tcps_mptcp_cell_proxy , &stat.mptcp_cell_proxy); nstat_sysinfo_send_data(&data); @@ -2263,6 +2486,9 @@ tcp_disable_read_probe(struct tcpcb *tp) ((tp->t_flagsext & TF_DETECT_READSTALL) || tp->t_rtimo_probes > 0)) { tcp_keepalive_reset(tp); + + if (tp->t_mpsub) + mptcp_reset_keepalive(tp); } } @@ -2297,12 +2523,12 @@ tcp_probe_connectivity(struct ifnet *ifp, u_int32_t enable) continue; /* Acquire lock to look at the state of the connection */ - tcp_lock(inp->inp_socket, 1, 0); + socket_lock(inp->inp_socket, 1); /* Release the want count */ if (inp->inp_ppcb == NULL || (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING)) { - tcp_unlock(inp->inp_socket, 1, 0); + socket_unlock(inp->inp_socket, 1); continue; } tp = intotcpcb(inp); @@ -2311,7 +2537,7 @@ tcp_probe_connectivity(struct ifnet *ifp, u_int32_t enable) else tcp_disable_read_probe(tp); - tcp_unlock(inp->inp_socket, 1, 0); + socket_unlock(inp->inp_socket, 1); } lck_rw_done(pcbinfo->ipi_lock); @@ -2384,12 +2610,9 @@ tcp_update_mss_locked(struct socket *so, struct ifnet *ifp) struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = intotcpcb(inp); - if (ifp == NULL && inp->inp_last_outifp == NULL) + if (ifp == NULL && (ifp = inp->inp_last_outifp) == NULL) return; - if (ifp == NULL) - ifp = inp->inp_last_outifp; - if (!IFNET_IS_CELLULAR(ifp)) { /* * This optimization is implemented for cellular @@ -2428,23 +2651,41 @@ tcp_itimer(struct inpcbinfo *ipi) LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) { struct socket *so; + struct ifnet *ifp; if (inp->inp_ppcb == NULL || in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) continue; so = inp->inp_socket; - tcp_lock(so, 1, 0); + ifp = inp->inp_last_outifp; + socket_lock(so, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } so_check_extended_bk_idle_time(so); if (ipi->ipi_flags & INPCBINFO_UPDATE_MSS) { tcp_update_mss_locked(so, NULL); } - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); + + /* + * Defunct all system-initiated background sockets if the + * socket is using the cellular interface and the interface + * has its LQM set to abort. + */ + if ((ipi->ipi_flags & INPCBINFO_HANDLE_LQM_ABORT) && + IS_SO_TC_BACKGROUNDSYSTEM(so->so_traffic_class) && + ifp != NULL && IFNET_IS_CELLULAR(ifp) && + (ifp->if_interface_state.valid_bitmask & + IF_INTERFACE_STATE_LQM_STATE_VALID) && + ifp->if_interface_state.lqm_state == + IFNET_LQM_THRESH_ABORT) { + socket_defunct(current_proc(), so, + SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL); + } } - ipi->ipi_flags &= ~INPCBINFO_UPDATE_MSS; + ipi->ipi_flags &= ~(INPCBINFO_UPDATE_MSS | INPCBINFO_HANDLE_LQM_ABORT); lck_rw_done(ipi->ipi_lock); } diff --git a/bsd/netinet/tcp_timer.h b/bsd/netinet/tcp_timer.h index 177cd162c..04a9eb5e4 100644 --- a/bsd/netinet/tcp_timer.h +++ b/bsd/netinet/tcp_timer.h @@ -177,6 +177,8 @@ #define TCPTV_PERSMIN ( 5*TCP_RETRANSHZ) /* retransmit persistence */ #define TCPTV_PERSMAX ( 60*TCP_RETRANSHZ) /* maximum persist interval */ +extern int tcptv_persmin_val; + #define TCPTV_KEEP_INIT ( 75*TCP_RETRANSHZ) /* connect keep alive */ #define TCPTV_KEEP_IDLE (120*60*TCP_RETRANSHZ) /* time before probing */ #define TCPTV_KEEPINTVL ( 75*TCP_RETRANSHZ) /* default probe interval */ @@ -248,6 +250,7 @@ struct tcptimerlist { lck_grp_attr_t *mtx_grp_attr; /* mutex group attributes */ thread_call_t call; /* call entry */ uint32_t runtime; /* time at which this list is going to run */ + uint32_t schedtime; /* time at which this list was scheduled */ uint32_t entries; /* Number of entries on the list */ uint32_t maxentries; /* Max number of entries at any time */ diff --git a/bsd/netinet/tcp_usrreq.c b/bsd/netinet/tcp_usrreq.c index a4ae7aaf4..0f0dc353e 100644 --- a/bsd/netinet/tcp_usrreq.c +++ b/bsd/netinet/tcp_usrreq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -70,7 +70,9 @@ #if INET6 #include #endif /* INET6 */ +#if !CONFIG_EMBEDDED #include +#endif #include #include #include @@ -101,6 +103,7 @@ #include #include #include +#include #include #if TCPDEBUG #include @@ -117,7 +120,6 @@ #include #endif /* FLOW_DIVERT */ -void tcp_fill_info(struct tcpcb *, struct tcp_info *); errno_t tcp_fill_info_for_info_tuple(struct info_tuple *, struct tcp_info *); int tcp_sysctl_info(struct sysctl_oid *, void *, int , struct sysctl_req *); @@ -209,7 +211,7 @@ tcp_usr_detach(struct socket *so) if (inp == 0 || (inp->inp_state == INPCB_STATE_DEAD)) { return EINVAL; /* XXX */ } - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); tp = intotcpcb(inp); /* In case we got disconnected from the peer */ if (tp == NULL) @@ -291,6 +293,17 @@ tcp_usr_bind(struct socket *so, struct sockaddr *nam, struct proc *p) error = in_pcbbind(inp, nam, p); if (error) goto out; + +#if NECP + /* Update NECP client with bind result if not in middle of connect */ + if ((inp->inp_flags2 & INP2_CONNECT_IN_PROGRESS) && + !uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ + COMMON_END(PRU_BIND); } @@ -396,8 +409,18 @@ static int tcp_connect_complete(struct socket *so) { struct tcpcb *tp = sototcpcb(so); + struct inpcb *inp = sotoinpcb(so); int error = 0; +#if NECP + /* Update NECP client with connected five-tuple */ + if (!uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ + /* TFO delays the tcp_output until later, when the app calls write() */ if (so->so_flags1 & SOF1_PRECONNECT_DATA) { if (!necp_socket_is_allowed_to_send_recv(sotoinpcb(so), NULL, NULL)) @@ -495,12 +518,9 @@ tcp_usr_connectx_common(struct socket *so, int af, uint32_t flags, void *arg, uint32_t arglen, struct uio *auio, user_ssize_t *bytes_written) { -#pragma unused(aid) -#if !MPTCP -#pragma unused(flags, arg, arglen) -#endif /* !MPTCP */ +#pragma unused(aid, flags, arg, arglen) struct inpcb *inp = sotoinpcb(so); - int error; + int error = 0; user_ssize_t datalen = 0; if (inp == NULL) @@ -508,6 +528,9 @@ tcp_usr_connectx_common(struct socket *so, int af, VERIFY(dst != NULL); + ASSERT(!(inp->inp_flags2 & INP2_CONNECT_IN_PROGRESS)); + inp->inp_flags2 |= INP2_CONNECT_IN_PROGRESS; + #if NECP inp_update_necp_policy(inp, src, dst, ifscope); #endif /* NECP */ @@ -516,47 +539,17 @@ tcp_usr_connectx_common(struct socket *so, int af, (tcp_fastopen & TCP_FASTOPEN_CLIENT)) sototcpcb(so)->t_flagsext |= TF_FASTOPEN; - /* - * We get here for 2 cases: - * - * a. From MPTCP, to connect a subflow. There is no need to - * bind the socket to the source address and/or interface, - * since everything has been taken care of by MPTCP. We - * simply check whether or not this is for the initial - * MPTCP connection attempt, or to join an existing one. - * - * b. From the socket layer, to connect a TCP. Perform the - * bind to source address and/or interface as necessary. - */ -#if MPTCP - if (flags & CONNREQF_MPTCP) { - struct mptsub_connreq *mpcr = arg; - - /* Check to make sure this came down from MPTCP */ - if (arg == NULL || arglen != sizeof (*mpcr)) - return (EOPNOTSUPP); - - switch (mpcr->mpcr_type) { - case MPTSUB_CONNREQ_MP_ENABLE: - break; - case MPTSUB_CONNREQ_MP_ADD: - break; - default: - return (EOPNOTSUPP); - } - } else -#endif /* MPTCP */ - { - /* bind socket to the specified interface, if requested */ - if (ifscope != IFSCOPE_NONE && - (error = inp_bindif(inp, ifscope, NULL)) != 0) - return (error); + /* bind socket to the specified interface, if requested */ + if (ifscope != IFSCOPE_NONE && + (error = inp_bindif(inp, ifscope, NULL)) != 0) { + goto done; + } - /* if source address and/or port is specified, bind to it */ - if (src != NULL) { - error = sobindlock(so, src, 0); /* already locked */ - if (error != 0) - return (error); + /* if source address and/or port is specified, bind to it */ + if (src != NULL) { + error = sobindlock(so, src, 0); /* already locked */ + if (error != 0) { + goto done; } } @@ -574,8 +567,9 @@ tcp_usr_connectx_common(struct socket *so, int af, /* NOTREACHED */ } - if (error != 0) - return (error); + if (error != 0) { + goto done; + } /* if there is data, copy it */ if (auio != NULL) { @@ -606,6 +600,8 @@ tcp_usr_connectx_common(struct socket *so, int af, if (error == 0 && pcid != NULL) *pcid = 1; /* there is only one connection in regular TCP */ +done: + inp->inp_flags2 &= ~INP2_CONNECT_IN_PROGRESS; return (error); } @@ -735,8 +731,7 @@ tcp_usr_disconnect(struct socket *so) struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp; - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); COMMON_START(); /* In case we got disconnected from the peer */ if (tp == NULL) @@ -1023,8 +1018,8 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, m_freem(control); control = NULL; } else if (control->m_len) { - /* - * if not unordered, TCP should not have + /* + * if not unordered, TCP should not have * control mbufs */ m_freem(control); @@ -1089,8 +1084,8 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, tp->t_flags &= ~TF_MORETOCOME; } } else { - if (sbspace(&so->so_snd) == 0) { - /* if no space is left in sockbuf, + if (sbspace(&so->so_snd) == 0) { + /* if no space is left in sockbuf, * do not try to squeeze in OOB traffic */ m_freem(m); error = ENOBUFS; @@ -1141,7 +1136,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, error = sbwait(&so->so_snd); } - COMMON_END((flags & PRUS_OOB) ? PRU_SENDOOB : + COMMON_END((flags & PRUS_OOB) ? PRU_SENDOOB : ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND)); } @@ -1218,8 +1213,7 @@ tcp_usr_preconnect(struct socket *so) error = tcp_output(sototcpcb(so)); - /* One read has been done. This was enough. Get back to "normal" behavior. */ - so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + soclearfastopen(so); COMMON_END(PRU_PRECONNECT); } @@ -1323,19 +1317,19 @@ tcp_connect(struct tcpcb *tp, struct sockaddr *nam, struct proc *p) if (error) goto done; - tcp_unlock(inp->inp_socket, 0, 0); + socket_unlock(inp->inp_socket, 0); oinp = in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, inp->inp_laddr.s_addr != INADDR_ANY ? inp->inp_laddr : laddr, inp->inp_lport, 0, NULL); - tcp_lock(inp->inp_socket, 0, 0); + socket_lock(inp->inp_socket, 0); if (oinp) { if (oinp != inp) /* 4143933: avoid deadlock if inp == oinp */ - tcp_lock(oinp->inp_socket, 1, 0); + socket_lock(oinp->inp_socket, 1); if (in_pcb_checkstate(oinp, WNT_RELEASE, 1) == WNT_STOPUSING) { if (oinp != inp) - tcp_unlock(oinp->inp_socket, 1, 0); + socket_unlock(oinp->inp_socket, 1); goto skip_oinp; } @@ -1348,12 +1342,12 @@ tcp_connect(struct tcpcb *tp, struct sockaddr *nam, struct proc *p) printf("tcp_connect: inp=0x%llx err=EADDRINUSE\n", (uint64_t)VM_KERNEL_ADDRPERM(inp)); if (oinp != inp) - tcp_unlock(oinp->inp_socket, 1, 0); + socket_unlock(oinp->inp_socket, 1); error = EADDRINUSE; goto done; } if (oinp != inp) - tcp_unlock(oinp->inp_socket, 1, 0); + socket_unlock(oinp->inp_socket, 1); } skip_oinp: if ((inp->inp_laddr.s_addr == INADDR_ANY ? laddr.s_addr : @@ -1372,6 +1366,7 @@ skip_oinp: inp->inp_laddr = laddr; /* no reference needed */ inp->inp_last_outifp = outif; + inp->inp_flags |= INP_INADDR_ANY; } inp->inp_faddr = sin->sin_addr; @@ -1430,14 +1425,14 @@ tcp6_connect(struct tcpcb *tp, struct sockaddr *nam, struct proc *p) error = in6_pcbladdr(inp, nam, &addr6, &outif); if (error) goto done; - tcp_unlock(inp->inp_socket, 0, 0); + socket_unlock(inp->inp_socket, 0); oinp = in6_pcblookup_hash(inp->inp_pcbinfo, &sin6->sin6_addr, sin6->sin6_port, IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) ? &addr6 : &inp->in6p_laddr, inp->inp_lport, 0, NULL); - tcp_lock(inp->inp_socket, 0, 0); + socket_lock(inp->inp_socket, 0); if (oinp) { if (oinp != inp && (otp = intotcpcb(oinp)) != NULL && otp->t_state == TCPS_TIME_WAIT && @@ -1481,7 +1476,7 @@ tcp6_connect(struct tcpcb *tp, struct sockaddr *nam, struct proc *p) soisconnecting(so); tcpstat.tcps_connattempt++; tp->t_state = TCPS_SYN_SENT; - tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, + tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, TCP_CONN_KEEPINIT(tp)); tp->iss = tcp_new_isn(tp); tcp_sendseqinit(tp); @@ -1499,11 +1494,11 @@ done: /* * Export TCP internal state information via a struct tcp_info */ -__private_extern__ void +void tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti) { struct inpcb *inp = tp->t_inpcb; - + bzero(ti, sizeof(*ti)); ti->tcpi_state = tp->t_state; @@ -1541,7 +1536,7 @@ tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti) ti->tcpi_snd_ssthresh = tp->snd_ssthresh; ti->tcpi_snd_cwnd = tp->snd_cwnd; ti->tcpi_snd_sbbytes = inp->inp_socket->so_snd.sb_cc; - + ti->tcpi_rcv_space = tp->rcv_wnd; ti->tcpi_snd_wnd = tp->snd_wnd; @@ -1604,6 +1599,7 @@ tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti) ti->tcpi_tfo_heuristics_disable = !!(tp->t_tfo_stats & TFO_S_HEURISTICS_DISABLE); ti->tcpi_tfo_send_blackhole = !!(tp->t_tfo_stats & TFO_S_SEND_BLACKHOLE); ti->tcpi_tfo_recv_blackhole = !!(tp->t_tfo_stats & TFO_S_RECV_BLACKHOLE); + ti->tcpi_tfo_onebyte_proxy = !!(tp->t_tfo_stats & TFO_S_ONE_BYTE_PROXY); ti->tcpi_ecn_client_setup = !!(tp->ecn_flags & TE_SETUPSENT); ti->tcpi_ecn_server_setup = !!(tp->ecn_flags & TE_SETUPRECEIVED); @@ -1616,9 +1612,20 @@ tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti) if (tp->t_inpcb->inp_last_outifp != NULL) { if (IFNET_IS_CELLULAR(tp->t_inpcb->inp_last_outifp)) ti->tcpi_if_cell = 1; - else if (IFNET_IS_WIFI(tp->t_inpcb->inp_last_outifp)) + if (IFNET_IS_WIFI(tp->t_inpcb->inp_last_outifp)) ti->tcpi_if_wifi = 1; + if (IFNET_IS_WIRED(tp->t_inpcb->inp_last_outifp)) + ti->tcpi_if_wired = 1; + if (IFNET_IS_WIFI_INFRA(tp->t_inpcb->inp_last_outifp)) + ti->tcpi_if_wifi_infra = 1; + if (tp->t_inpcb->inp_last_outifp->if_eflags & IFEF_AWDL) + ti->tcpi_if_wifi_awdl = 1; } + if (tp->tcp_cc_index == TCP_CC_ALGO_BACKGROUND_INDEX) + ti->tcpi_snd_background = 1; + if (tcp_recv_bg == 1 || + IS_TCP_RECV_BG(tp->t_inpcb->inp_socket)) + ti->tcpi_rcv_background = 1; ti->tcpi_ecn_recv_ce = tp->t_ecn_recv_ce; ti->tcpi_ecn_recv_cwr = tp->t_ecn_recv_cwr; @@ -1639,15 +1646,15 @@ tcp_fill_info_for_info_tuple(struct info_tuple *itpl, struct tcp_info *ti) struct inpcb *inp = NULL; struct socket *so; struct tcpcb *tp; - + if (itpl->itpl_proto == IPPROTO_TCP) pcbinfo = &tcbinfo; else return EINVAL; - + if (itpl->itpl_local_sa.sa_family == AF_INET && itpl->itpl_remote_sa.sa_family == AF_INET) { - inp = in_pcblookup_hash(pcbinfo, + inp = in_pcblookup_hash(pcbinfo, itpl->itpl_remote_sin.sin_addr, itpl->itpl_remote_sin.sin_port, itpl->itpl_local_sin.sin_addr, @@ -1657,18 +1664,18 @@ tcp_fill_info_for_info_tuple(struct info_tuple *itpl, struct tcp_info *ti) itpl->itpl_remote_sa.sa_family == AF_INET6) { struct in6_addr ina6_local; struct in6_addr ina6_remote; - + ina6_local = itpl->itpl_local_sin6.sin6_addr; - if (IN6_IS_SCOPE_LINKLOCAL(&ina6_local) && + if (IN6_IS_SCOPE_LINKLOCAL(&ina6_local) && itpl->itpl_local_sin6.sin6_scope_id) ina6_local.s6_addr16[1] = htons(itpl->itpl_local_sin6.sin6_scope_id); ina6_remote = itpl->itpl_remote_sin6.sin6_addr; - if (IN6_IS_SCOPE_LINKLOCAL(&ina6_remote) && + if (IN6_IS_SCOPE_LINKLOCAL(&ina6_remote) && itpl->itpl_remote_sin6.sin6_scope_id) ina6_remote.s6_addr16[1] = htons(itpl->itpl_remote_sin6.sin6_scope_id); - - inp = in6_pcblookup_hash(pcbinfo, + + inp = in6_pcblookup_hash(pcbinfo, &ina6_remote, itpl->itpl_remote_sin6.sin6_port, &ina6_local, @@ -1749,16 +1756,18 @@ tcp_connection_fill_info(struct tcpcb *tp, struct tcp_connection_info *tci) tci->tcpi_tfo_heuristics_disable = !!(tp->t_tfo_stats & TFO_S_HEURISTICS_DISABLE); tci->tcpi_tfo_send_blackhole = !!(tp->t_tfo_stats & TFO_S_SEND_BLACKHOLE); tci->tcpi_tfo_recv_blackhole = !!(tp->t_tfo_stats & TFO_S_RECV_BLACKHOLE); + tci->tcpi_tfo_onebyte_proxy = !!(tp->t_tfo_stats & TFO_S_ONE_BYTE_PROXY); } } -__private_extern__ int +__private_extern__ int tcp_sysctl_info(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { int error; struct tcp_info ti; struct info_tuple itpl; +#if !CONFIG_EMBEDDED proc_t caller = PROC_NULL; proc_t caller_parent = PROC_NULL; char command_name[MAXCOMLEN + 1] = ""; @@ -1794,6 +1803,7 @@ tcp_sysctl_info(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused if (caller != PROC_NULL) proc_rele(caller); +#endif /* !CONFIG_EMBEDDED */ if (req->newptr == USER_ADDR_NULL) { return EINVAL; @@ -1813,7 +1823,7 @@ tcp_sysctl_info(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused if (error != 0) { return error; } - + return 0; } @@ -1823,12 +1833,12 @@ tcp_lookup_peer_pid_locked(struct socket *so, pid_t *out_pid) int error = EHOSTUNREACH; *out_pid = -1; if ((so->so_state & SS_ISCONNECTED) == 0) return ENOTCONN; - + struct inpcb *inp = (struct inpcb*)so->so_pcb; uint16_t lport = inp->inp_lport; uint16_t fport = inp->inp_fport; struct inpcb *finp = NULL; - + if (inp->inp_vflag & INP_IPV6) { struct in6_addr laddr6 = inp->in6p_laddr; struct in6_addr faddr6 = inp->in6p_faddr; @@ -1842,13 +1852,17 @@ tcp_lookup_peer_pid_locked(struct socket *so, pid_t *out_pid) finp = in_pcblookup_hash(&tcbinfo, laddr4, lport, faddr4, fport, 0, NULL); socket_lock(so, 0); } - + if (finp) { *out_pid = finp->inp_socket->last_pid; error = 0; - in_pcb_checkstate(finp, WNT_RELEASE, 0); + /* Avoid deadlock due to same inpcb for loopback socket */ + if (inp == finp) + in_pcb_checkstate(finp, WNT_RELEASE, 1); + else + in_pcb_checkstate(finp, WNT_RELEASE, 0); } - + return error; } @@ -1988,7 +2002,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) error = EINVAL; break; } - minpkts = (in.min_burst_size != 0) ? in.min_burst_size : + minpkts = (in.min_burst_size != 0) ? in.min_burst_size : tp->t_bwmeas->bw_minsizepkts; maxpkts = (in.max_burst_size != 0) ? in.max_burst_size : tp->t_bwmeas->bw_maxsizepkts; @@ -2025,7 +2039,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) } else { tp->t_keepidle = optval * TCP_RETRANSHZ; /* reset the timer to new value */ - tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, + tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, TCP_CONN_KEEPIDLE(tp)); tcp_check_timer_state(tp); } @@ -2093,7 +2107,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) if (optval < 0 || optval > INT32_MAX) { error = EINVAL; break; - } + } if (optval != 0) inp->inp_flags2 |= INP2_KEEPALIVE_OFFLOAD; else @@ -2107,7 +2121,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) break; if (optval < 0) error = EINVAL; - else + else tp->t_persist_timeout = optval * TCP_RETRANSHZ; break; case TCP_RXT_CONNDROPTIME: @@ -2132,7 +2146,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) if (optval == 0) { so->so_flags &= ~(SOF_NOTSENT_LOWAT); tp->t_notsent_lowat = 0; - } else { + } else { so->so_flags |= SOF_NOTSENT_LOWAT; tp->t_notsent_lowat = optval; } @@ -2143,13 +2157,16 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) sizeof(optval)); if (error) break; - if (optval < 0 || + if (optval < 0 || optval > TCP_ADAPTIVE_TIMEOUT_MAX) { error = EINVAL; break; } else if (optval == 0) { tp->t_adaptive_rtimo = 0; tcp_keepalive_reset(tp); + + if (tp->t_mpsub) + mptcp_reset_keepalive(tp); } else { tp->t_adaptive_rtimo = optval; } @@ -2159,7 +2176,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) sizeof (optval)); if (error) break; - if (optval < 0 || + if (optval < 0 || optval > TCP_ADAPTIVE_TIMEOUT_MAX) { error = EINVAL; break; @@ -2261,6 +2278,27 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) else tcp_disable_tfo(tp); break; + case TCP_FASTOPEN_FORCE_HEURISTICS: + error = sooptcopyin(sopt, &optval, sizeof(optval), + sizeof(optval)); + + if (error) + break; + if (optval < 0 || optval > 1) { + error = EINVAL; + break; + } + + if (tp->t_state != TCPS_CLOSED) { + error = EINVAL; + break; + } + if (optval) + tp->t_flagsext |= TF_FASTOPEN_HEUR; + else + tp->t_flagsext &= ~TF_FASTOPEN_HEUR; + + break; case TCP_ENABLE_ECN: error = sooptcopyin(sopt, &optval, sizeof optval, sizeof optval); @@ -2271,6 +2309,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) tp->ecn_flags &= ~TE_ECN_MODE_DISABLE; } else { tp->ecn_flags &= ~TE_ECN_MODE_ENABLE; + tp->ecn_flags |= TE_ECN_MODE_DISABLE; } break; case TCP_ECN_MODE: @@ -2336,7 +2375,24 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) TRAFFIC_MGT_SO_BACKGROUND); } break; - + case TCP_RXT_MINIMUM_TIMEOUT: + error = sooptcopyin(sopt, &optval, sizeof(optval), + sizeof(optval)); + if (error) + break; + if (optval < 0) { + error = EINVAL; + break; + } + if (optval == 0) { + tp->t_rxt_minimum_timeout = 0; + } else { + tp->t_rxt_minimum_timeout = min(optval, + TCP_RXT_MINIMUM_TIMEOUT_LIMIT); + /* convert to milliseconds */ + tp->t_rxt_minimum_timeout *= TCP_RETRANSHZ; + } + break; default: error = ENOPROTOOPT; break; @@ -2400,7 +2456,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) break; case TCP_RXT_FINDROP: optval = tp->t_flagsext & TF_RXTFINDROP; - break; + break; case TCP_NOTIMEWAIT: optval = (tp->t_flagsext & TF_NOTIMEWAIT) ? 1 : 0; break; @@ -2412,6 +2468,9 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) } optval = tfo_enabled(tp); break; + case TCP_FASTOPEN_FORCE_HEURISTICS: + optval = (tp->t_flagsext & TF_FASTOPEN_HEUR) ? 1 : 0; + break; case TCP_MEASURE_SND_BW: optval = tp->t_flagsext & TF_MEASURESNDBW; break; @@ -2501,6 +2560,9 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) error = sooptcopyout(sopt, &retid, sizeof (retid)); goto done; } + case TCP_RXT_MINIMUM_TIMEOUT: + optval = tp->t_rxt_minimum_timeout / TCP_RETRANSHZ; + break; default: error = ENOPROTOOPT; break; @@ -2525,10 +2587,12 @@ u_int32_t tcp_recvspace = 1448*384; * sb_max in sbreserve. Disallow setting the tcp send and recv space * to be more than sb_max because that will cause tcp_attach to fail * (see radar 5713060) - */ + */ static int sysctl_tcp_sospace(struct sysctl_oid *oidp, __unused void *arg1, - __unused int arg2, struct sysctl_req *req) { + int arg2, struct sysctl_req *req) +{ +#pragma unused(arg2) u_int32_t new_value = 0, *space_p = NULL; int changed = 0, error = 0; u_quad_t sb_effective_max = (sb_max / (MSIZE+MCLBYTES)) * MCLBYTES; @@ -2548,6 +2612,7 @@ sysctl_tcp_sospace(struct sysctl_oid *oidp, __unused void *arg1, if (changed) { if (new_value > 0 && new_value <= sb_effective_max) { *space_p = new_value; + SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, new_value); } else { error = ERANGE; } @@ -2555,10 +2620,21 @@ sysctl_tcp_sospace(struct sysctl_oid *oidp, __unused void *arg1, return error; } +#if SYSCTL_SKMEM +SYSCTL_PROC(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_sendspace, + offsetof(skmem_sysctl, tcp.sendspace), sysctl_tcp_sospace, + "IU", "Maximum outgoing TCP datagram size"); +SYSCTL_PROC(_net_inet_tcp, TCPCTL_RECVSPACE, recvspace, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_recvspace, + offsetof(skmem_sysctl, tcp.recvspace), sysctl_tcp_sospace, + "IU", "Maximum incoming TCP datagram size"); +#else /* SYSCTL_SKMEM */ SYSCTL_PROC(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_sendspace , 0, &sysctl_tcp_sospace, "IU", "Maximum outgoing TCP datagram size"); SYSCTL_PROC(_net_inet_tcp, TCPCTL_RECVSPACE, recvspace, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_recvspace , 0, &sysctl_tcp_sospace, "IU", "Maximum incoming TCP datagram size"); +#endif /* SYSCTL_SKMEM */ /* * Attach TCP protocol to socket, allocating @@ -2689,17 +2765,17 @@ tcp_usrclosed(struct tcpcb *tp) break; case TCPS_ESTABLISHED: - DTRACE_TCP4(state__change, void, NULL, + DTRACE_TCP4(state__change, void, NULL, struct inpcb *, tp->t_inpcb, - struct tcpcb *, tp, + struct tcpcb *, tp, int32_t, TCPS_FIN_WAIT_1); tp->t_state = TCPS_FIN_WAIT_1; break; case TCPS_CLOSE_WAIT: - DTRACE_TCP4(state__change, void, NULL, + DTRACE_TCP4(state__change, void, NULL, struct inpcb *, tp->t_inpcb, - struct tcpcb *, tp, + struct tcpcb *, tp, int32_t, TCPS_LAST_ACK); tp->t_state = TCPS_LAST_ACK; break; @@ -2708,7 +2784,7 @@ tcp_usrclosed(struct tcpcb *tp) soisdisconnected(tp->t_inpcb->inp_socket); /* To prevent the connection hanging in FIN_WAIT_2 forever. */ if (tp->t_state == TCPS_FIN_WAIT_2) - tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp, + tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp, TCP_CONN_MAXIDLE(tp)); } return (tp); @@ -2743,12 +2819,12 @@ tcp_out6_cksum_stats(u_int32_t len) tcpstat.tcps_snd6_swcsum_bytes += len; } -/* +/* * When messages are enabled on a TCP socket, the message priority * is sent as a control message. This function will extract it. */ int -tcp_get_msg_priority(struct mbuf *control, uint32_t *msgpri) +tcp_get_msg_priority(struct mbuf *control, uint32_t *msgpri) { struct cmsghdr *cm; if (control == NULL) @@ -2767,7 +2843,7 @@ tcp_get_msg_priority(struct mbuf *control, uint32_t *msgpri) } } - VERIFY(*msgpri >= MSG_PRI_MIN && *msgpri <= MSG_PRI_MAX); + VERIFY(*msgpri >= MSG_PRI_MIN && *msgpri <= MSG_PRI_MAX); return (0); } #endif /* INET6 */ diff --git a/bsd/netinet/tcp_var.h b/bsd/netinet/tcp_var.h index aeb7f3b56..4a5e1d3b4 100644 --- a/bsd/netinet/tcp_var.h +++ b/bsd/netinet/tcp_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -198,14 +198,16 @@ struct tcptemp { }; struct bwmeas { - tcp_seq bw_start; /* start of bw measurement */ + tcp_seq bw_start; /* start of bw measurement */ uint32_t bw_ts; /* timestamp when bw measurement started */ - uint32_t bw_size; /* burst size in bytes for this bw measurement */ - uint32_t bw_minsizepkts; /* Min burst size as segments */ - uint32_t bw_maxsizepkts; /* Max burst size as segments */ + uint32_t bw_size; /* burst size in bytes for this bw measurement */ + uint32_t bw_minsizepkts; /* Min burst size as segments */ + uint32_t bw_maxsizepkts; /* Max burst size as segments */ uint32_t bw_minsize; /* Min size in bytes */ uint32_t bw_maxsize; /* Max size in bytes */ - uint32_t bw_sndbw; /* Measured send bw */ + uint32_t bw_sndbw; /* Measured send bandwidth */ + uint32_t bw_sndbw_max; /* Max measured bandwidth */ + uint32_t bw_rcvbw_max; /* Max receive bandwidth measured */ }; /* MPTCP Data sequence map entry */ @@ -315,6 +317,7 @@ struct tcpcb { u_int32_t t_maxopd; /* mss plus options */ u_int32_t t_rcvtime; /* time at which a packet was received */ + u_int32_t t_sndtime; /* time at which we last sent new data */ u_int32_t t_starttime; /* time connection was established */ int t_rtttime; /* tcp clock when rtt calculation was started */ tcp_seq t_rtseq; /* sequence number being timed */ @@ -398,6 +401,7 @@ struct tcpcb { #define TE_ECN_ON (TE_SETUPSENT | TE_SETUPRECEIVED) /* Indicate ECN was successfully negotiated on a connection) */ #define TE_CEHEURI_SET 0x2000 /* We did our CE-probing at the beginning */ #define TE_CLIENT_SETUP 0x4000 /* setup from client side */ +#define TE_RCVD_SYN_RST 0x8000 /* Received RST to the first ECN enabled SYN */ u_int32_t t_ecn_recv_ce; /* Received CE from the network */ u_int32_t t_ecn_recv_cwr; /* Packets received with CWR */ @@ -478,10 +482,11 @@ struct tcpcb { #define TF_PROBING 0x200000 /* Trigger probe timeout */ #define TF_FASTOPEN 0x400000 /* TCP Fastopen is enabled */ #define TF_REASS_INPROG 0x800000 /* Reassembly is in progress */ +#define TF_FASTOPEN_HEUR 0x1000000 /* Make sure that heuristics get never skipped */ #if TRAFFIC_MGT /* Inter-arrival jitter related state */ - uint32_t iaj_rcv_ts; /* tcp clock when the first packet was received */ + uint32_t iaj_rcv_ts; /* tcp clock when the first packet was received */ uint16_t iaj_size; /* Size of packet for iaj measurement */ uint8_t iaj_small_pkt; /* Count of packets smaller than iaj_size */ uint8_t t_pipeack_ind; /* index for next pipeack sample */ @@ -514,8 +519,8 @@ struct tcpcb { #if MPTCP u_int32_t t_mpflags; /* flags for multipath TCP */ -#define TMPF_PREESTABLISHED 0x00000001 /* conn in pre-established state */ -#define TMPF_SENT_KEYS 0x00000002 /* indicates that keys were sent */ +#define TMPF_PREESTABLISHED 0x00000001 /* conn in pre-established state */ +#define TMPF_SND_KEYS 0x00000002 /* indicates that keys should be send */ #define TMPF_MPTCP_TRUE 0x00000004 /* negotiated MPTCP successfully */ #define TMPF_MPTCP_RCVD_KEY 0x00000008 /* state for 3-way handshake */ #define TMPF_SND_MPPRIO 0x00000010 /* send priority of subflow */ @@ -536,12 +541,14 @@ struct tcpcb { #define TMPF_MPTCP_READY 0x00080000 /* Can send DSS options on data */ #define TMPF_INFIN_SENT 0x00100000 /* Sent infinite mapping */ #define TMPF_SND_MPFAIL 0x00200000 /* Received mapping csum failure */ -#define TMPF_FASTJOIN_SEND 0x00400000 /* Fast join early data send */ -#define TMPF_FASTJOINBY2_SEND 0x00800000 /* Fast join send after 3 WHS */ -#define TMPF_TFO_REQUEST 0x02000000 /* TFO Requested */ +#define TMPF_SND_JACK 0x00400000 /* Send a Join-ACK */ +#define TMPF_TFO_REQUEST 0x00800000 /* TFO Requested */ + +#define TMPF_MPTCP_SIGNALS (TMPF_SND_MPPRIO | TMPF_SND_REM_ADDR | TMPF_SND_MPFAIL | TMPF_SND_KEYS | TMPF_SND_JACK) tcp_seq t_mpuna; /* unacknowledged sequence */ - void *t_mptcb; /* pointer to MPTCP TCB */ + struct mptcb *t_mptcb; /* pointer to MPTCP TCB */ + struct mptsub *t_mpsub; /* pointer to the MPTCP subflow */ struct mpt_dsn_map t_rcv_map; /* Receive mapping list */ u_int8_t t_local_aid; /* Addr Id for authentication */ u_int8_t t_rem_aid; /* Addr ID of another subflow */ @@ -553,8 +560,8 @@ struct tcpcb { #define TFO_F_COOKIE_REQ 0x04 /* Client requested a new cookie */ #define TFO_F_COOKIE_SENT 0x08 /* Client did send a cookie in the SYN */ #define TFO_F_SYN_LOSS 0x10 /* A SYN-loss triggered a fallback to regular TCP on the client-side */ -#define TFO_F_NO_RCVPROBING 0x20 /* This network is guaranteed to support TFO in the downstream direction */ -#define TFO_F_NO_SNDPROBING 0x40 /* This network is guaranteed to support TFO in the upstream direction */ +#define TFO_F_NO_SNDPROBING 0x20 /* This network is guaranteed to support TFO in the upstream direction */ +#define TFO_F_HEURISTIC_DONE 0x40 /* We have already marked this network as bad */ u_int8_t t_tfo_flags; #define TFO_S_SYNDATA_RCV 0x01 /* SYN+data has been received */ #define TFO_S_COOKIEREQ_RECV 0x02 /* TFO-cookie request received */ @@ -570,6 +577,7 @@ struct tcpcb { #define TFO_S_HEURISTICS_DISABLE 0x0800 /* TFO-heuristics disabled it for this connection */ #define TFO_S_SEND_BLACKHOLE 0x1000 /* TFO got blackholed in the send direction */ #define TFO_S_RECV_BLACKHOLE 0x2000 /* TFO got blackholed in the recv direction */ +#define TFO_S_ONE_BYTE_PROXY 0x4000 /* TFO failed because of a proxy acknowledging just one byte */ u_int16_t t_tfo_stats; u_int8_t t_tfo_probes; /* TFO-probes we did send */ @@ -610,6 +618,7 @@ struct tcpcb { u_int32_t t_dsack_recvd; /* Received a valid DSACK option */ SLIST_HEAD(,tcp_notify_ack_marker) t_notify_ack; /* state for notifying data acknowledgements */ u_int32_t t_recv_throttle_ts; /* TS for start of recv throttle */ + u_int32_t t_rxt_minimum_timeout; /* minimum retransmit timeout in ms */ }; #define IN_FASTRECOVERY(tp) (tp->t_flags & TF_FASTRECOVERY) @@ -705,6 +714,7 @@ extern int tcprexmtthresh; #define TCP_RESET_REXMT_STATE(_tp_) do { \ (_tp_)->t_rxtshift = 0; \ (_tp_)->t_rxtstart = 0; \ + mptcp_reset_rexmit_state((_tp_)); \ } while(0); #define TCP_AUTORCVBUF_MAX(_ifp_) (((_ifp_) != NULL && \ @@ -782,22 +792,6 @@ struct rmxp_tao { #define intotcpcb(ip) ((struct tcpcb *)(ip)->inp_ppcb) #define sototcpcb(so) (intotcpcb(sotoinpcb(so))) -/* - * The rtt measured is in milliseconds as the timestamp granularity is - * a millisecond. The smoothed round-trip time and estimated variance - * are stored as fixed point numbers scaled by the values below. - * For convenience, these scales are also used in smoothing the average - * (smoothed = (1/scale)sample + ((scale-1)/scale)smoothed). - * With these scales, srtt has 5 bits to the right of the binary point, - * and thus an "ALPHA" of 0.875. rttvar has 4 bits to the right of the - * binary point, and is smoothed with an ALPHA of 0.75. - */ -#define TCP_RTT_SCALE 32 /* multiplier for srtt; 3 bits frac. */ -#define TCP_RTT_SHIFT 5 /* shift for srtt; 5 bits frac. */ -#define TCP_RTTVAR_SCALE 16 /* multiplier for rttvar; 4 bits */ -#define TCP_RTTVAR_SHIFT 4 /* shift for rttvar; 4 bits */ -#define TCP_DELTA_SHIFT 2 /* see tcp_input.c */ - /* TFO-specific defines */ #define TFO_COOKIE_LEN_MIN 4 #define TFO_COOKIE_LEN_DEFAULT 8 @@ -1162,6 +1156,51 @@ struct tcpstat { u_int32_t tcps_mss_to_low; /* Change MSS to low using link status report */ u_int32_t tcps_ecn_fallback_droprst; /* ECN fallback caused by connection drop due to RST */ u_int32_t tcps_ecn_fallback_droprxmt; /* ECN fallback due to drop after multiple retransmits */ + u_int32_t tcps_ecn_fallback_synrst; /* ECN fallback due to rst after syn */ + + u_int32_t tcps_mptcp_rcvmemdrop; /* MPTCP packets dropped for lack of memory */ + u_int32_t tcps_mptcp_rcvduppack; /* MPTCP duplicate-only packets received */ + u_int32_t tcps_mptcp_rcvpackafterwin; /* MPTCP packets with data after window */ + + /* TCP timer statistics */ + u_int32_t tcps_timer_drift_le_1_ms; /* Timer drift less or equal to 1 ms */ + u_int32_t tcps_timer_drift_le_10_ms; /* Timer drift less or equal to 10 ms */ + u_int32_t tcps_timer_drift_le_20_ms; /* Timer drift less or equal to 20 ms */ + u_int32_t tcps_timer_drift_le_50_ms; /* Timer drift less or equal to 50 ms */ + u_int32_t tcps_timer_drift_le_100_ms; /* Timer drift less or equal to 100 ms */ + u_int32_t tcps_timer_drift_le_200_ms; /* Timer drift less or equal to 200 ms */ + u_int32_t tcps_timer_drift_le_500_ms; /* Timer drift less or equal to 500 ms */ + u_int32_t tcps_timer_drift_le_1000_ms; /* Timer drift less or equal to 1000 ms */ + u_int32_t tcps_timer_drift_gt_1000_ms; /* Timer drift greater than 1000 ms */ + + u_int32_t tcps_mptcp_handover_attempt; /* Total number of MPTCP-attempts using handover mode */ + u_int32_t tcps_mptcp_interactive_attempt; /* Total number of MPTCP-attempts using interactive mode */ + u_int32_t tcps_mptcp_aggregate_attempt; /* Total number of MPTCP-attempts using aggregate mode */ + u_int32_t tcps_mptcp_fp_handover_attempt; /* Same as previous three but only for first-party apps */ + u_int32_t tcps_mptcp_fp_interactive_attempt; + u_int32_t tcps_mptcp_fp_aggregate_attempt; + u_int32_t tcps_mptcp_heuristic_fallback; /* Total number of MPTCP-connections that fell back due to heuristics */ + u_int32_t tcps_mptcp_fp_heuristic_fallback; /* Same as previous but for first-party apps */ + u_int32_t tcps_mptcp_handover_success_wifi; /* Total number of successfull handover-mode connections that *started* on WiFi */ + u_int32_t tcps_mptcp_handover_success_cell; /* Total number of successfull handover-mode connections that *started* on Cell */ + u_int32_t tcps_mptcp_interactive_success; /* Total number of interactive-mode connections that negotiated MPTCP */ + u_int32_t tcps_mptcp_aggregate_success; /* Same as previous but for aggregate */ + u_int32_t tcps_mptcp_fp_handover_success_wifi; /* Same as previous four, but for first-party apps */ + u_int32_t tcps_mptcp_fp_handover_success_cell; + u_int32_t tcps_mptcp_fp_interactive_success; + u_int32_t tcps_mptcp_fp_aggregate_success; + u_int32_t tcps_mptcp_handover_cell_from_wifi; /* Total number of connections that use cell in handover-mode (coming from WiFi) */ + u_int32_t tcps_mptcp_handover_wifi_from_cell; /* Total number of connections that use WiFi in handover-mode (coming from cell) */ + u_int32_t tcps_mptcp_interactive_cell_from_wifi; /* Total number of connections that use cell in interactive mode (coming from WiFi) */ + u_int64_t tcps_mptcp_handover_cell_bytes; /* Total number of bytes sent on cell in handover-mode (on new subflows, ignoring initial one) */ + u_int64_t tcps_mptcp_interactive_cell_bytes; /* Same as previous but for interactive */ + u_int64_t tcps_mptcp_aggregate_cell_bytes; + u_int64_t tcps_mptcp_handover_all_bytes; /* Total number of bytes sent in handover */ + u_int64_t tcps_mptcp_interactive_all_bytes; + u_int64_t tcps_mptcp_aggregate_all_bytes; + u_int32_t tcps_mptcp_back_to_wifi; /* Total number of connections that succeed to move traffic away from cell (when starting on cell) */ + u_int32_t tcps_mptcp_wifi_proxy; /* Total number of new subflows that fell back to regular TCP on cell */ + u_int32_t tcps_mptcp_cell_proxy; /* Total number of new subflows that fell back to regular TCP on WiFi */ }; @@ -1206,6 +1245,7 @@ struct xtcpcb { u_quad_t xt_alignment_hack; }; +#if !CONFIG_EMBEDDED struct xtcpcb64 { u_int32_t xt_len; @@ -1286,6 +1326,7 @@ struct xtcpcb64 { u_quad_t xt_alignment_hack; }; +#endif /* !CONFIG_EMBEDDED */ #ifdef PRIVATE @@ -1365,6 +1406,22 @@ struct xtcpcb_n { u_int32_t snd_ssthresh_prev; /* ssthresh prior to retransmit */ }; + /* + * The rtt measured is in milliseconds as the timestamp granularity is + * a millisecond. The smoothed round-trip time and estimated variance + * are stored as fixed point numbers scaled by the values below. + * For convenience, these scales are also used in smoothing the average + * (smoothed = (1/scale)sample + ((scale-1)/scale)smoothed). + * With these scales, srtt has 5 bits to the right of the binary point, + * and thus an "ALPHA" of 0.875. rttvar has 4 bits to the right of the + * binary point, and is smoothed with an ALPHA of 0.75. + */ +#define TCP_RTT_SCALE 32 /* multiplier for srtt; 3 bits frac. */ +#define TCP_RTT_SHIFT 5 /* shift for srtt; 5 bits frac. */ +#define TCP_RTTVAR_SCALE 16 /* multiplier for rttvar; 4 bits */ +#define TCP_RTTVAR_SHIFT 4 /* shift for rttvar; 4 bits */ +#define TCP_DELTA_SHIFT 2 /* see tcp_input.c */ + #endif /* PRIVATE */ #pragma pack() @@ -1424,20 +1481,19 @@ extern int tcp_minmss; #define TCP_FASTOPEN_SERVER 0x01 #define TCP_FASTOPEN_CLIENT 0x02 -extern int tcp_tfo_halfcnt; -extern int tcp_tfo_backlog; -extern int tcp_fastopen; -extern int tcp_tfo_fallback_min; -extern int ss_fltsz; -extern int ss_fltsz_local; -extern int tcp_do_rfc3390; /* Calculate ss_fltsz according to RFC 3390 */ +extern int tcp_tfo_halfcnt; +extern int tcp_tfo_backlog; +extern int tcp_fastopen; +extern int ss_fltsz; +extern int ss_fltsz_local; +extern int tcp_do_rfc3390; /* Calculate ss_fltsz according to RFC 3390 */ extern int tcp_do_rfc1323; extern int target_qdelay; -extern u_int32_t tcp_now; /* for RFC 1323 timestamps */ +extern u_int32_t tcp_now; /* for RFC 1323 timestamps */ extern struct timeval tcp_uptime; extern lck_spin_t *tcp_uptime_lock; -extern int tcp_delack_enabled; -extern int tcp_do_sack; /* SACK enabled/disabled */ +extern int tcp_delack_enabled; +extern int tcp_do_sack; /* SACK enabled/disabled */ extern int tcp_do_rfc3465; extern int tcp_do_rfc3465_lim2; extern int maxseg_unacked; @@ -1446,6 +1502,7 @@ extern struct zone *tcp_reass_zone; extern struct zone *tcp_rxt_seg_zone; extern int tcp_ecn_outbound; extern int tcp_ecn_inbound; +extern u_int32_t tcp_do_autorcvbuf; extern u_int32_t tcp_autorcvbuf_max; extern u_int32_t tcp_autorcvbuf_max_ca; extern u_int32_t tcp_autorcvbuf_inc_shift; @@ -1465,7 +1522,7 @@ struct tcp_respond_args { void tcp_canceltimers(struct tcpcb *); struct tcpcb * tcp_close(struct tcpcb *); -void tcp_ctlinput(int, struct sockaddr *, void *); +void tcp_ctlinput(int, struct sockaddr *, void *, struct ifnet *); int tcp_ctloutput(struct socket *, struct sockopt *); struct tcpcb * tcp_drop(struct tcpcb *, int); @@ -1497,6 +1554,7 @@ void tcp_fillheaders(struct tcpcb *, void *, void *); struct tcpcb *tcp_timers(struct tcpcb *, int); void tcp_trace(int, int, struct tcpcb *, void *, struct tcphdr *, int); +void tcp_fill_info(struct tcpcb *, struct tcp_info *); void tcp_sack_doack(struct tcpcb *, struct tcpopt *, struct tcphdr *, u_int32_t *); extern boolean_t tcp_sack_process_dsack(struct tcpcb *, struct tcpopt *, @@ -1591,11 +1649,15 @@ extern bool tcp_notify_ack_active(struct socket *so); #if MPTCP extern int mptcp_input_preproc(struct tcpcb *, struct mbuf *, int); -extern void mptcp_output_csum(struct tcpcb *, struct mbuf *, int32_t, unsigned, - u_int64_t, u_int32_t *); +extern uint32_t mptcp_output_csum(struct mbuf *m, uint64_t dss_val, + uint32_t sseq, uint16_t dlen); extern int mptcp_adj_mss(struct tcpcb *, boolean_t); extern void mptcp_insert_rmap(struct tcpcb *, struct mbuf *); #endif + +__private_extern__ void tcp_update_stats_per_flow( + struct ifnet_stats_per_flow *, struct ifnet *); + #endif /* BSD_KERNEL_RPIVATE */ #endif /* _NETINET_TCP_VAR_H_ */ diff --git a/bsd/netinet/udp_usrreq.c b/bsd/netinet/udp_usrreq.c index 698aea8b6..044486ec5 100644 --- a/bsd/netinet/udp_usrreq.c +++ b/bsd/netinet/udp_usrreq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -81,6 +81,7 @@ #include #include #include +#include #include #include @@ -201,17 +202,17 @@ struct udp_ip6 { u_char uip6_init_done : 1; }; -static int udp_abort(struct socket *); -static int udp_attach(struct socket *, int, struct proc *); -static int udp_bind(struct socket *, struct sockaddr *, struct proc *); -static int udp_connect(struct socket *, struct sockaddr *, struct proc *); -static int udp_connectx(struct socket *, struct sockaddr *, +int udp_abort(struct socket *); +int udp_attach(struct socket *, int, struct proc *); +int udp_bind(struct socket *, struct sockaddr *, struct proc *); +int udp_connect(struct socket *, struct sockaddr *, struct proc *); +int udp_connectx(struct socket *, struct sockaddr *, struct sockaddr *, struct proc *, uint32_t, sae_associd_t, sae_connid_t *, uint32_t, void *, uint32_t, struct uio *, user_ssize_t *); -static int udp_detach(struct socket *); -static int udp_disconnect(struct socket *); -static int udp_disconnectx(struct socket *, sae_associd_t, sae_connid_t); -static int udp_send(struct socket *, int, struct mbuf *, struct sockaddr *, +int udp_detach(struct socket *); +int udp_disconnect(struct socket *); +int udp_disconnectx(struct socket *, sae_associd_t, sae_connid_t); +int udp_send(struct socket *, int, struct mbuf *, struct sockaddr *, struct mbuf *, struct proc *); static void udp_append(struct inpcb *, struct ip *, struct mbuf *, int, struct sockaddr_in *, struct udp_in6 *, struct udp_ip6 *, struct ifnet *); @@ -220,7 +221,7 @@ static void udp_append(struct inpcb *, struct ip *, struct mbuf *, int, struct sockaddr_in *, struct ifnet *); #endif /* !INET6 */ static int udp_input_checksum(struct mbuf *, struct udphdr *, int, int); -static int udp_output(struct inpcb *, struct mbuf *, struct sockaddr *, +int udp_output(struct inpcb *, struct mbuf *, struct sockaddr *, struct mbuf *, struct proc *); static void ip_2_ip6_hdr(struct ip6_hdr *ip6, struct ip *ip); static void udp_gc(struct inpcbinfo *); @@ -331,7 +332,7 @@ udp_input(struct mbuf *m, int iphlen) * with options still present. */ if (iphlen > sizeof (struct ip)) { - ip_stripoptions(m, (struct mbuf *)0); + ip_stripoptions(m); iphlen = sizeof (struct ip); } @@ -493,8 +494,7 @@ udp_input(struct mbuf *m, int iphlen) group.sin_addr = ip->ip_dst; blocked = imo_multi_filter(imo, ifp, - (struct sockaddr *)&group, - (struct sockaddr *)&udp_in); + &group, &udp_in); if (blocked == MCAST_PASS) foundmembership = 1; @@ -743,6 +743,7 @@ udp_input(struct mbuf *m, int iphlen) if (nstat_collect) { INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, m->m_pkthdr.len); + inp_set_activity_bitmap(inp); } so_recv_data_stat(inp->inp_socket, m, 0); if (sbappendaddr(&inp->inp_socket->so_rcv, append_sa, @@ -850,6 +851,7 @@ udp_append(struct inpcb *last, struct ip *ip, struct mbuf *n, int off, INP_ADD_STAT(last, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(last, cell, wifi, wired, rxbytes, n->m_pkthdr.len); + inp_set_activity_bitmap(last); } so_recv_data_stat(last->inp_socket, n, 0); m_adj(n, off); @@ -878,12 +880,12 @@ udp_notify(struct inpcb *inp, int errno) } void -udp_ctlinput(int cmd, struct sockaddr *sa, void *vip) +udp_ctlinput(int cmd, struct sockaddr *sa, void *vip, __unused struct ifnet * ifp) { struct ip *ip = vip; void (*notify)(struct inpcb *, int) = udp_notify; struct in_addr faddr; - struct inpcb *inp; + struct inpcb *inp = NULL; faddr = ((struct sockaddr_in *)(void *)sa)->sin_addr; if (sa->sa_family != AF_INET || faddr.s_addr == INADDR_ANY) @@ -1172,6 +1174,7 @@ SYSCTL_PROC(_net_inet_udp, UDPCTL_PCBLIST, pcblist, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, udp_pcblist, "S,xinpcb", "List of active UDP sockets"); +#if !CONFIG_EMBEDDED static int udp_pcblist64 SYSCTL_HANDLER_ARGS @@ -1290,6 +1293,7 @@ SYSCTL_PROC(_net_inet_udp, OID_AUTO, pcblist64, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, udp_pcblist64, "S,xinpcb64", "List of active UDP sockets"); +#endif /* !CONFIG_EMBEDDED */ static int udp_pcblist_n SYSCTL_HANDLER_ARGS @@ -1306,7 +1310,8 @@ __private_extern__ void udp_get_ports_used(uint32_t ifindex, int protocol, uint32_t flags, bitstr_t *bitfield) { - inpcb_get_ports_used(ifindex, protocol, flags, bitfield, &udbinfo); + inpcb_get_ports_used(ifindex, protocol, flags, bitfield, + &udbinfo); } __private_extern__ uint32_t @@ -1318,7 +1323,7 @@ udp_count_opportunistic(unsigned int ifindex, u_int32_t flags) __private_extern__ uint32_t udp_find_anypcb_byaddr(struct ifaddr *ifa) { - return (inpcb_find_anypcb_byaddr(ifa, &udbinfo)); + return (inpcb_find_anypcb_byaddr(ifa, &udbinfo)); } static int @@ -1395,7 +1400,7 @@ udp_check_pktinfo(struct mbuf *control, struct ifnet **outif, return (0); } -static int +int udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, struct mbuf *control, struct proc *p) { @@ -1425,7 +1430,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, KERNEL_DEBUG(DBG_FNC_UDP_OUTPUT | DBG_FUNC_START, 0, 0, 0, 0, 0); - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); if (control != NULL) { sotc = so_tc_from_control(control, &netsvctype); VERIFY(outif == NULL); @@ -1622,6 +1627,11 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, if (inp->inp_flowhash == 0) inp->inp_flowhash = inp_calc_flowhash(inp); + if (fport == htons(53) && !(so->so_flags1 & SOF1_DNS_COUNTED)) { + so->so_flags1 |= SOF1_DNS_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_dns); + } + /* * Calculate data length and get a mbuf * for UDP and IP headers. @@ -1651,7 +1661,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, if (udpcksum && !(inp->inp_flags & INP_UDP_NOCKSUM)) { ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr, htons((u_short)len + sizeof (struct udphdr) + IPPROTO_UDP)); - m->m_pkthdr.csum_flags = CSUM_UDP; + m->m_pkthdr.csum_flags = (CSUM_UDP|CSUM_ZERO_INVERT); m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); } else { ui->ui_sum = 0; @@ -1735,6 +1745,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, mopts->imo_multicast_ifp != NULL) { /* no reference needed */ inp->inp_last_outifp = mopts->imo_multicast_ifp; + } IMO_UNLOCK(mopts); } @@ -1777,6 +1788,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, } INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, txbytes, len); + inp_set_activity_bitmap(inp); } if (flowadv && (adv->code == FADV_FLOW_CONTROLLED || @@ -1804,6 +1816,7 @@ abort: inp->inp_laddr = origladdr; /* XXX rehash? */ /* no reference needed */ inp->inp_last_outifp = origoutifp; + } else if (inp->inp_route.ro_rt != NULL) { struct rtentry *rt = inp->inp_route.ro_rt; struct ifnet *outifp; @@ -1828,7 +1841,7 @@ abort: sizeof(struct udphdr) + sizeof(struct ip) + ifnet_hdrlen(outifp) + - ifnet_packetpreamblelen(outifp), + ifnet_mbuf_packetpreamblelen(outifp), sizeof(u_int32_t)); } } else { @@ -1904,7 +1917,7 @@ SYSCTL_PROC(_net_inet_udp, UDPCTL_MAXDGRAM, maxdgram, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &udp_sendspace, 0, &sysctl_udp_sospace, "IU", "Maximum outgoing UDP datagram size"); -static int +int udp_abort(struct socket *so) { struct inpcb *inp; @@ -1919,7 +1932,7 @@ udp_abort(struct socket *so) return (0); } -static int +int udp_attach(struct socket *so, int proto, struct proc *p) { #pragma unused(proto) @@ -1945,7 +1958,7 @@ udp_attach(struct socket *so, int proto, struct proc *p) return (0); } -static int +int udp_bind(struct socket *so, struct sockaddr *nam, struct proc *p) { struct inpcb *inp; @@ -1959,10 +1972,22 @@ udp_bind(struct socket *so, struct sockaddr *nam, struct proc *p) if (inp == NULL) return (EINVAL); error = in_pcbbind(inp, nam, p); + +#if NECP + /* Update NECP client with bind result if not in middle of connect */ + if (error == 0 && + (inp->inp_flags2 & INP2_CONNECT_IN_PROGRESS) && + !uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ + return (error); } -static int +int udp_connect(struct socket *so, struct sockaddr *nam, struct proc *p) { struct inpcb *inp; @@ -1974,6 +1999,11 @@ udp_connect(struct socket *so, struct sockaddr *nam, struct proc *p) if (inp->inp_faddr.s_addr != INADDR_ANY) return (EISCONN); + if (!(so->so_flags1 & SOF1_CONNECT_COUNTED)) { + so->so_flags1 |= SOF1_CONNECT_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_connected); + } + #if NECP #if FLOW_DIVERT if (necp_socket_should_use_flow_divert(inp)) { @@ -1994,6 +2024,15 @@ udp_connect(struct socket *so, struct sockaddr *nam, struct proc *p) error = in_pcbconnect(inp, nam, p, IFSCOPE_NONE, NULL); if (error == 0) { +#if NECP + /* Update NECP client with connected five-tuple */ + if (!uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ + soisconnected(so); if (inp->inp_flowhash == 0) inp->inp_flowhash = inp_calc_flowhash(inp); @@ -2009,7 +2048,7 @@ udp_connectx_common(struct socket *so, int af, struct sockaddr *src, struct sock { #pragma unused(aid, flags, arg, arglen) struct inpcb *inp = sotoinpcb(so); - int error; + int error = 0; user_ssize_t datalen = 0; if (inp == NULL) @@ -2017,20 +2056,25 @@ udp_connectx_common(struct socket *so, int af, struct sockaddr *src, struct sock VERIFY(dst != NULL); + ASSERT(!(inp->inp_flags2 & INP2_CONNECT_IN_PROGRESS)); + inp->inp_flags2 |= INP2_CONNECT_IN_PROGRESS; + #if NECP inp_update_necp_policy(inp, src, dst, ifscope); #endif /* NECP */ /* bind socket to the specified interface, if requested */ if (ifscope != IFSCOPE_NONE && - (error = inp_bindif(inp, ifscope, NULL)) != 0) - return (error); + (error = inp_bindif(inp, ifscope, NULL)) != 0) { + goto done; + } /* if source address and/or port is specified, bind to it */ if (src != NULL) { error = sobindlock(so, src, 0); /* already locked */ - if (error != 0) - return (error); + if (error != 0) { + goto done; + } } switch (af) { @@ -2047,8 +2091,9 @@ udp_connectx_common(struct socket *so, int af, struct sockaddr *src, struct sock /* NOTREACHED */ } - if (error != 0) - return (error); + if (error != 0) { + goto done; + } /* * If there is data, copy it. DATA_IDEMPOTENT is ignored. @@ -2081,10 +2126,12 @@ udp_connectx_common(struct socket *so, int af, struct sockaddr *src, struct sock if (error == 0 && pcid != NULL) *pcid = 1; /* there is only 1 connection for UDP */ +done: + inp->inp_flags2 &= ~INP2_CONNECT_IN_PROGRESS; return (error); } -static int +int udp_connectx(struct socket *so, struct sockaddr *src, struct sockaddr *dst, struct proc *p, uint32_t ifscope, sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg, @@ -2094,7 +2141,7 @@ udp_connectx(struct socket *so, struct sockaddr *src, p, ifscope, aid, pcid, flags, arg, arglen, uio, bytes_written)); } -static int +int udp_detach(struct socket *so) { struct inpcb *inp; @@ -2119,7 +2166,7 @@ udp_detach(struct socket *so) return (0); } -static int +int udp_disconnect(struct socket *so) { struct inpcb *inp; @@ -2142,10 +2189,11 @@ udp_disconnect(struct socket *so) inp->inp_laddr.s_addr = INADDR_ANY; so->so_state &= ~SS_ISCONNECTED; /* XXX */ inp->inp_last_outifp = NULL; + return (0); } -static int +int udp_disconnectx(struct socket *so, sae_associd_t aid, sae_connid_t cid) { #pragma unused(cid) @@ -2155,7 +2203,7 @@ udp_disconnectx(struct socket *so, sae_associd_t aid, sae_connid_t cid) return (udp_disconnect(so)); } -static int +int udp_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr, struct mbuf *control, struct proc *p) { @@ -2209,7 +2257,7 @@ udp_lock(struct socket *so, int refcount, void *debug) lr_saved = debug; if (so->so_pcb != NULL) { - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, + LCK_MTX_ASSERT(&((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); } else { @@ -2244,7 +2292,7 @@ udp_unlock(struct socket *so, int refcount, void *debug) so, lr_saved, solockhistory_nr(so)); /* NOTREACHED */ } else { - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, + LCK_MTX_ASSERT(&((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_OWNED); so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; @@ -2254,9 +2302,9 @@ udp_unlock(struct socket *so, int refcount, void *debug) } lck_mtx_t * -udp_getlock(struct socket *so, int locktype) +udp_getlock(struct socket *so, int flags) { -#pragma unused(locktype) +#pragma unused(flags) struct inpcb *inp = sotoinpcb(so); if (so->so_pcb == NULL) { @@ -2306,7 +2354,7 @@ udp_gc(struct inpcbinfo *ipi) * Skip if busy, no hurry for cleanup. Keep gc active * and try the lock again during next round. */ - if (!lck_mtx_try_lock(&inp->inpcb_mtx)) { + if (!socket_try_lock(inp->inp_socket)) { atomic_add_32(&ipi->ipi_gc_req.intimer_fast, 1); continue; } @@ -2326,7 +2374,7 @@ udp_gc(struct inpcbinfo *ipi) } in_pcbdispose(inp); } else { - lck_mtx_unlock(&inp->inpcb_mtx); + socket_unlock(so, 0); atomic_add_32(&ipi->ipi_gc_req.intimer_fast, 1); } } @@ -2388,39 +2436,64 @@ udp_input_checksum(struct mbuf *m, struct udphdr *uh, int off, int ulen) return (0); } + /* ip_stripoptions() must have been called before we get here */ + ASSERT((ip->ip_hl << 2) == sizeof (*ip)); + if ((hwcksum_rx || (ifp->if_flags & IFF_LOOPBACK) || (m->m_pkthdr.pkt_flags & PKTF_LOOP)) && (m->m_pkthdr.csum_flags & CSUM_DATA_VALID)) { if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) { uh->uh_sum = m->m_pkthdr.csum_rx_val; } else { - uint16_t sum = m->m_pkthdr.csum_rx_val; - uint16_t start = m->m_pkthdr.csum_rx_start; + uint32_t sum = m->m_pkthdr.csum_rx_val; + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - (off + ulen)); /* * Perform 1's complement adjustment of octets * that got included/excluded in the hardware- * calculated checksum value. Ignore cases - * where the value includes or excludes the + * where the value already includes the entire * IP header span, as the sum for those octets - * would already be 0xffff and thus no-op. + * would already be 0 by the time we get here; + * IP has already performed its header checksum + * checks. If we do need to adjust, restore + * the original fields in the IP header when + * computing the adjustment value. Also take + * care of any trailing bytes and subtract out + * their partial sum. */ + ASSERT(trailer >= 0); if ((m->m_pkthdr.csum_flags & CSUM_PARTIAL) && - start != 0 && (off - start) != off) { -#if BYTE_ORDER != BIG_ENDIAN + ((start != 0 && start != off) || trailer != 0)) { + uint32_t swbytes = (uint32_t)trailer; + if (start < off) { + ip->ip_len += sizeof (*ip); +#if BYTE_ORDER != BIG_ENDIAN HTONS(ip->ip_len); HTONS(ip->ip_off); - } #endif /* BYTE_ORDER != BIG_ENDIAN */ + } /* callee folds in sum */ - sum = m_adj_sum16(m, start, off, sum); -#if BYTE_ORDER != BIG_ENDIAN + sum = m_adj_sum16(m, start, off, ulen, sum); + if (off > start) + swbytes += (off - start); + else + swbytes += (start - off); + if (start < off) { +#if BYTE_ORDER != BIG_ENDIAN NTOHS(ip->ip_off); NTOHS(ip->ip_len); - } #endif /* BYTE_ORDER != BIG_ENDIAN */ + ip->ip_len -= sizeof (*ip); + } + + if (swbytes != 0) + udp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } /* callee folds in sum */ @@ -2611,7 +2684,8 @@ udp_fill_keepalive_offload_frames(ifnet_t ifp, htons(sizeof(struct udphdr) + (u_short)inp->inp_keepalive_datalen + IPPROTO_UDP)); - m->m_pkthdr.csum_flags = CSUM_UDP; + m->m_pkthdr.csum_flags = + (CSUM_UDP|CSUM_ZERO_INVERT); m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); } @@ -2682,7 +2756,8 @@ udp_fill_keepalive_offload_frames(ifnet_t ifp, htonl(sizeof(struct udphdr) + (u_short)inp->inp_keepalive_datalen + IPPROTO_UDP)); - m->m_pkthdr.csum_flags = CSUM_UDPIPV6; + m->m_pkthdr.csum_flags = + (CSUM_UDPIPV6|CSUM_ZERO_INVERT); m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); } diff --git a/bsd/netinet/udp_var.h b/bsd/netinet/udp_var.h index 2eb1e0806..f016b31de 100644 --- a/bsd/netinet/udp_var.h +++ b/bsd/netinet/udp_var.h @@ -163,7 +163,7 @@ extern struct udpstat udpstat; extern int udp_log_in_vain; __BEGIN_DECLS -extern void udp_ctlinput(int, struct sockaddr *, void *); +extern void udp_ctlinput(int, struct sockaddr *, void *, struct ifnet *); extern int udp_ctloutput(struct socket *, struct sockopt *); extern void udp_init(struct protosw *, struct domain *); extern void udp_input(struct mbuf *, int); diff --git a/bsd/netinet6/Makefile b/bsd/netinet6/Makefile index 039ad151d..9bccd060b 100644 --- a/bsd/netinet6/Makefile +++ b/bsd/netinet6/Makefile @@ -22,9 +22,9 @@ PRIVATE_DATAFILES = \ scope6_var.h PRIVATE_KERNELFILES = \ - ah6.h esp6.h esp_rijndael.h in6_gif.h in6_ifattach.h \ - ip6_ecn.h ip6protosw.h ipcomp6.h ipsec6.h \ - tcp6_var.h udp6_var.h + ah6.h esp6.h esp_rijndael.h esp_chachapoly.h \ + in6_gif.h in6_ifattach.h ip6_ecn.h ip6protosw.h \ + ipcomp6.h ipsec6.h tcp6_var.h udp6_var.h INSTALL_MI_LIST = ${DATAFILES} diff --git a/bsd/netinet6/ah_input.c b/bsd/netinet6/ah_input.c index a6054b601..fac8da28c 100644 --- a/bsd/netinet6/ah_input.c +++ b/bsd/netinet6/ah_input.c @@ -996,7 +996,7 @@ ah6_ctlinput(int cmd, struct sockaddr *sa, void *d) struct ip6_hdr *ip6; struct mbuf *m; struct ip6ctlparam *ip6cp = NULL; - int off; + int off = 0; struct sockaddr_in6 *sa6_src, *sa6_dst; if (sa->sa_family != AF_INET6 || diff --git a/bsd/netinet6/esp.h b/bsd/netinet6/esp.h index b2deaa2f3..94f6e26b7 100644 --- a/bsd/netinet6/esp.h +++ b/bsd/netinet6/esp.h @@ -129,6 +129,7 @@ extern int esp_max_ivlen(void); /* crypt routines */ extern int esp4_output(struct mbuf *, struct secasvar *); extern void esp4_input(struct mbuf *, int off); +extern struct mbuf *esp4_input_extended(struct mbuf *, int off, ifnet_t interface); extern size_t esp_hdrsiz(struct ipsecrequest *); extern int esp_schedule(const struct esp_algorithm *, struct secasvar *); diff --git a/bsd/netinet6/esp6.h b/bsd/netinet6/esp6.h index 767d0c387..384ec59dc 100644 --- a/bsd/netinet6/esp6.h +++ b/bsd/netinet6/esp6.h @@ -70,8 +70,9 @@ extern int esp6_output(struct mbuf *, u_char *, struct mbuf *, struct secasvar *); extern int esp6_input(struct mbuf **, int *, int); +extern int esp6_input_extended(struct mbuf **mp, int *offp, int proto, ifnet_t interface); -extern void esp6_ctlinput(int, struct sockaddr *, void *); +extern void esp6_ctlinput(int, struct sockaddr *, void *, struct ifnet *); #endif /* BSD_KERNEL_PRIVATE */ #endif /* _NETINET6_ESP6_H_ */ diff --git a/bsd/netinet6/esp_chachapoly.c b/bsd/netinet6/esp_chachapoly.c new file mode 100644 index 000000000..0970f6983 --- /dev/null +++ b/bsd/netinet6/esp_chachapoly.c @@ -0,0 +1,481 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ESP_CHACHAPOLY_SALT_LEN 4 +#define ESP_CHACHAPOLY_KEY_LEN 32 +#define ESP_CHACHAPOLY_NONCE_LEN 12 + +// The minimum alignment is documented in KALLOC_LOG2_MINALIGN +// which isn't accessible from here. Current minimum is 8. +_Static_assert(_Alignof(chacha20poly1305_ctx) <= 8, + "Alignment guarantee is broken"); + +#if ((( 8 * (ESP_CHACHAPOLY_KEY_LEN + ESP_CHACHAPOLY_SALT_LEN)) != ESP_CHACHAPOLY_KEYBITS_WITH_SALT) || \ + (ESP_CHACHAPOLY_KEY_LEN != CCCHACHA20_KEY_NBYTES) || \ + (ESP_CHACHAPOLY_NONCE_LEN != CCCHACHA20POLY1305_NONCE_NBYTES)) +#error "Invalid sizes" +#endif + +extern lck_mtx_t *sadb_mutex; + +typedef struct _esp_chachapoly_ctx { + chacha20poly1305_ctx ccp_ctx; + uint8_t ccp_salt[ESP_CHACHAPOLY_SALT_LEN]; + bool ccp_implicit_iv; +} esp_chachapoly_ctx_s, *esp_chachapoly_ctx_t; + + +#define ESP_ASSERT(_cond, _format, ...) \ + do { \ + if (!(_cond)) { \ + panic("%s:%d " _format, __FUNCTION__, __LINE__, ##__VA_ARGS__); \ + } \ + } while (0) + +#define ESP_CHECK_ARG(_arg) ESP_ASSERT(_arg != NULL, #_arg "is NULL") + +#define _esp_log(_level, _format, ...) \ + log(_level, "%s:%d " _format, __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define esp_log_err(_format, ...) _esp_log(LOG_ERR, _format, ##__VA_ARGS__) + +#define _esp_packet_log(_level, _format, ...) \ + ipseclog((_level, "%s:%d " _format, __FUNCTION__, __LINE__, ##__VA_ARGS__)) +#define esp_packet_log_err(_format, ...) _esp_packet_log(LOG_ERR, _format, ##__VA_ARGS__) + +int +esp_chachapoly_mature(struct secasvar *sav) +{ + const struct esp_algorithm *algo; + + ESP_CHECK_ARG(sav); + + if ((sav->flags & SADB_X_EXT_OLD) != 0) { + esp_log_err("ChaChaPoly is incompatible with SADB_X_EXT_OLD"); + return 1; + } + if ((sav->flags & SADB_X_EXT_DERIV) != 0) { + esp_log_err("ChaChaPoly is incompatible with SADB_X_EXT_DERIV"); + return 1; + } + + if (sav->alg_enc != SADB_X_EALG_CHACHA20POLY1305) { + esp_log_err("ChaChaPoly unsupported algorithm %d", + sav->alg_enc); + return 1; + } + + if (sav->key_enc == NULL) { + esp_log_err("ChaChaPoly key is missing"); + return 1; + } + + algo = esp_algorithm_lookup(sav->alg_enc); + if (algo == NULL) { + esp_log_err("ChaChaPoly lookup failed for algorithm %d", + sav->alg_enc); + return 1; + } + + if (sav->key_enc->sadb_key_bits != ESP_CHACHAPOLY_KEYBITS_WITH_SALT) { + esp_log_err("ChaChaPoly invalid key length %d bits", + sav->key_enc->sadb_key_bits); + return 1; + } + + return 0; +} + +int +esp_chachapoly_schedlen(__unused const struct esp_algorithm *algo) +{ + return sizeof(esp_chachapoly_ctx_s); +} + +int +esp_chachapoly_schedule(__unused const struct esp_algorithm *algo, + struct secasvar *sav) +{ + esp_chachapoly_ctx_t esp_ccp_ctx; + int rc = 0; + + ESP_CHECK_ARG(sav); + if (sav->ivlen != ESP_CHACHAPOLY_IV_LEN) { + esp_log_err("Invalid ivlen %u", sav->ivlen); + return EINVAL; + } + if (_KEYLEN(sav->key_enc) != ESP_CHACHAPOLY_KEY_LEN + ESP_CHACHAPOLY_SALT_LEN) { + esp_log_err("Invalid key len %u", _KEYLEN(sav->key_enc)); + return EINVAL; + } + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); + + esp_ccp_ctx = (esp_chachapoly_ctx_t)sav->sched; + + rc = chacha20poly1305_init(&esp_ccp_ctx->ccp_ctx, + (const uint8_t *)_KEYBUF(sav->key_enc)); + if (rc != 0) { + esp_log_err("chacha20poly1305_init returned %d", rc); + return rc; + } + + memcpy(esp_ccp_ctx->ccp_salt, + (const uint8_t *)_KEYBUF(sav->key_enc) + ESP_CHACHAPOLY_KEY_LEN, + sizeof(esp_ccp_ctx->ccp_salt)); + + esp_ccp_ctx->ccp_implicit_iv = ((sav->flags & SADB_X_EXT_IIV) != 0); + + return 0; +} + +int +esp_chachapoly_encrypt_finalize(struct secasvar *sav, + unsigned char *tag, + unsigned int tag_bytes) +{ + esp_chachapoly_ctx_t esp_ccp_ctx; + int rc = 0; + + ESP_CHECK_ARG(sav); + ESP_CHECK_ARG(tag); + if (tag_bytes != ESP_CHACHAPOLY_ICV_LEN) { + esp_log_err("Invalid tag_bytes %u", tag_bytes); + return EINVAL; + } + + esp_ccp_ctx = (esp_chachapoly_ctx_t)sav->sched; + rc = chacha20poly1305_finalize(&esp_ccp_ctx->ccp_ctx, tag); + if (rc != 0) { + esp_log_err("chacha20poly1305_finalize returned %d", rc); + return rc; + } + return 0; +} + +int +esp_chachapoly_decrypt_finalize(struct secasvar *sav, + unsigned char *tag, + unsigned int tag_bytes) +{ + esp_chachapoly_ctx_t esp_ccp_ctx; + int rc = 0; + + ESP_CHECK_ARG(sav); + ESP_CHECK_ARG(tag); + if (tag_bytes != ESP_CHACHAPOLY_ICV_LEN) { + esp_log_err("Invalid tag_bytes %u", tag_bytes); + return EINVAL; + } + + esp_ccp_ctx = (esp_chachapoly_ctx_t)sav->sched; + rc = chacha20poly1305_verify(&esp_ccp_ctx->ccp_ctx, tag); + if (rc != 0) { + esp_log_err("chacha20poly1305_finalize returned %d", rc); + return rc; + } + return 0; +} + +int +esp_chachapoly_encrypt(struct mbuf *m, // head of mbuf chain + size_t off, // offset to ESP header + __unused size_t plen, + struct secasvar *sav, + __unused const struct esp_algorithm *algo, + int ivlen) +{ + struct mbuf *s = m; // this mbuf + int32_t soff = 0; // offset from the head of mbuf chain (m) to head of this mbuf (s) + int32_t sn = 0; // offset from the head of this mbuf (s) to the body + uint8_t *sp; // buffer of a given encryption round + size_t len; // length of a given encryption round + const int32_t ivoff = (int32_t)off + (int32_t)sizeof(struct newesp); // IV offset + int32_t bodyoff; // body offset + int rc = 0; // return code of corecrypto operations + struct newesp esp_hdr; // ESP header for AAD + _Static_assert(sizeof(esp_hdr) == 8, "Bad size"); + uint8_t nonce[ESP_CHACHAPOLY_NONCE_LEN]; + esp_chachapoly_ctx_t esp_ccp_ctx; + + ESP_CHECK_ARG(m); + ESP_CHECK_ARG(sav); + if (ivlen != ESP_CHACHAPOLY_IV_LEN) { + m_freem(m); + esp_log_err("Invalid ivlen %u", ivlen); + return EINVAL; + } + if (sav->ivlen != ESP_CHACHAPOLY_IV_LEN) { + m_freem(m); + esp_log_err("Invalid sav->ivlen %u", sav->ivlen); + return EINVAL; + } + + esp_ccp_ctx = (esp_chachapoly_ctx_t)sav->sched; + if (esp_ccp_ctx->ccp_implicit_iv) { + bodyoff = ivoff; + } else { + bodyoff = ivoff + ivlen; + } + // check if total packet length is enough to contain ESP + IV + if (m->m_pkthdr.len < bodyoff) { + esp_log_err("Packet too short %d < %zu", m->m_pkthdr.len, bodyoff); + m_freem(m); + return EINVAL; + } + + rc = chacha20poly1305_reset(&esp_ccp_ctx->ccp_ctx); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_reset failed %d", rc); + return rc; + } + + // RFC 7634 dictates that the 12 byte nonce must be + // the 4 byte salt followed by the 8 byte IV. + // The IV MUST be non-repeating but does not need to be unpredictable, + // so we use 4 bytes of 0 followed by the 4 byte ESP sequence number. + // this allows us to use implicit IV -- draft-mglt-ipsecme-implicit-iv + memset(sav->iv, 0, 4); + memcpy(sav->iv + 4, &sav->seq, sizeof(sav->seq)); + _Static_assert(4 + sizeof(sav->seq) == ESP_CHACHAPOLY_IV_LEN, + "Bad IV length"); + memcpy(nonce, esp_ccp_ctx->ccp_salt, ESP_CHACHAPOLY_SALT_LEN); + memcpy(nonce + ESP_CHACHAPOLY_SALT_LEN, sav->iv, ESP_CHACHAPOLY_IV_LEN); + _Static_assert(ESP_CHACHAPOLY_SALT_LEN + ESP_CHACHAPOLY_IV_LEN == sizeof(nonce), + "Bad nonce length"); + + rc = chacha20poly1305_setnonce(&esp_ccp_ctx->ccp_ctx, nonce); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_setnonce failed %d", rc); + return rc; + } + + if (!esp_ccp_ctx->ccp_implicit_iv) { + m_copyback(m, ivoff, ivlen, sav->iv); + } + cc_clear(sizeof(nonce), nonce); + + // Set Additional Authentication Data (AAD) + m_copydata(m, (int)off, sizeof(esp_hdr), (void *)&esp_hdr); + + rc = chacha20poly1305_aad(&esp_ccp_ctx->ccp_ctx, + sizeof(esp_hdr), + (void *)&esp_hdr); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_aad failed %d", rc); + return rc; + } + + // skip headers/IV + while (s != NULL && soff < bodyoff) { + if (soff + s->m_len > bodyoff) { + sn = bodyoff - soff; + break; + } + + soff += s->m_len; + s = s->m_next; + } + + while (s != NULL && soff < m->m_pkthdr.len) { + len = (size_t)(s->m_len - sn); + if (len == 0) { + // skip empty mbufs + continue; + } + sp = mtod(s, uint8_t *) + sn; + + rc = chacha20poly1305_encrypt(&esp_ccp_ctx->ccp_ctx, + len, sp, sp); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_encrypt failed %d", rc); + return rc; + } + + sn = 0; + soff += s->m_len; + s = s->m_next; + } + if (s == NULL && soff != m->m_pkthdr.len) { + m_freem(m); + esp_log_err("not enough mbufs %d %d", soff, m->m_pkthdr.len); + return EFBIG; + } + return 0; +} + +int +esp_chachapoly_decrypt(struct mbuf *m, // head of mbuf chain + size_t off, // offset to ESP header + struct secasvar *sav, + __unused const struct esp_algorithm *algo, + int ivlen) +{ + struct mbuf *s = m; // this mbuf + int32_t soff = 0; // offset from the head of mbuf chain (m) to head of this mbuf (s) + int32_t sn = 0; // offset from the head of this mbuf (s) to the body + uint8_t *sp; // buffer of a given encryption round + size_t len; // length of a given encryption round + const int32_t ivoff = (int32_t)off + (int32_t)sizeof(struct newesp); // IV offset + int32_t bodyoff; // body offset + int rc = 0; // return code of corecrypto operations + struct newesp esp_hdr; // ESP header for AAD + _Static_assert(sizeof(esp_hdr) == 8, "Bad size"); + uint8_t nonce[ESP_CHACHAPOLY_NONCE_LEN]; + esp_chachapoly_ctx_t esp_ccp_ctx; + + ESP_CHECK_ARG(m); + ESP_CHECK_ARG(sav); + if (ivlen != ESP_CHACHAPOLY_IV_LEN) { + m_freem(m); + esp_log_err("Invalid ivlen %u", ivlen); + return EINVAL; + } + if (sav->ivlen != ESP_CHACHAPOLY_IV_LEN) { + m_freem(m); + esp_log_err("Invalid sav->ivlen %u", sav->ivlen); + return EINVAL; + } + + esp_ccp_ctx = (esp_chachapoly_ctx_t)sav->sched; + if (esp_ccp_ctx->ccp_implicit_iv) { + bodyoff = ivoff; + } else { + bodyoff = ivoff + ivlen; + } + // check if total packet length is enough to contain ESP + IV + if (m->m_pkthdr.len < bodyoff) { + esp_packet_log_err("Packet too short %d < %zu", m->m_pkthdr.len, bodyoff); + m_freem(m); + return EINVAL; + } + + rc = chacha20poly1305_reset(&esp_ccp_ctx->ccp_ctx); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_reset failed %d", rc); + return rc; + } + + m_copydata(m, (int)off, sizeof(esp_hdr), (void *)&esp_hdr); + + // RFC 7634 dictates that the 12 byte nonce must be + // the 4 byte salt followed by the 8 byte IV. + memcpy(nonce, esp_ccp_ctx->ccp_salt, ESP_CHACHAPOLY_SALT_LEN); + if (esp_ccp_ctx->ccp_implicit_iv) { + // IV is implicit (4 zero bytes followed by the ESP sequence number) + memset(nonce + ESP_CHACHAPOLY_SALT_LEN, 0, 4); + memcpy(nonce + ESP_CHACHAPOLY_SALT_LEN + 4, &esp_hdr.esp_seq, sizeof(esp_hdr.esp_seq)); + _Static_assert(4 + sizeof(esp_hdr.esp_seq) == ESP_CHACHAPOLY_IV_LEN, "Bad IV length"); + } else { + // copy IV from packet + m_copydata(m, ivoff, ESP_CHACHAPOLY_IV_LEN, nonce + ESP_CHACHAPOLY_SALT_LEN); + } + _Static_assert(ESP_CHACHAPOLY_SALT_LEN + ESP_CHACHAPOLY_IV_LEN == sizeof(nonce), + "Bad nonce length"); + + rc = chacha20poly1305_setnonce(&esp_ccp_ctx->ccp_ctx, nonce); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_setnonce failed %d", rc); + return rc; + } + cc_clear(sizeof(nonce), nonce); + + // Set Additional Authentication Data (AAD) + rc = chacha20poly1305_aad(&esp_ccp_ctx->ccp_ctx, + sizeof(esp_hdr), + (void *)&esp_hdr); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_aad failed %d", rc); + return rc; + } + + // skip headers/IV + while (s != NULL && soff < bodyoff) { + if (soff + s->m_len > bodyoff) { + sn = bodyoff - soff; + break; + } + + soff += s->m_len; + s = s->m_next; + } + + while (s != NULL && soff < m->m_pkthdr.len) { + len = (size_t)(s->m_len - sn); + if (len == 0) { + // skip empty mbufs + continue; + } + sp = mtod(s, uint8_t *) + sn; + + rc = chacha20poly1305_decrypt(&esp_ccp_ctx->ccp_ctx, + len, sp, sp); + if (rc != 0) { + m_freem(m); + esp_packet_log_err("chacha20poly1305_decrypt failed %d", rc); + return rc; + } + + sn = 0; + soff += s->m_len; + s = s->m_next; + } + if (s == NULL && soff != m->m_pkthdr.len) { + m_freem(m); + esp_packet_log_err("not enough mbufs %d %d", soff, m->m_pkthdr.len); + return EFBIG; + } + return 0; +} diff --git a/bsd/netinet6/esp_chachapoly.h b/bsd/netinet6/esp_chachapoly.h new file mode 100644 index 000000000..b98b77a40 --- /dev/null +++ b/bsd/netinet6/esp_chachapoly.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include + +#ifdef BSD_KERNEL_PRIVATE + +#ifndef _ESP_CHACHA_POLY_H_ +#define _ESP_CHACHA_POLY_H_ + +#define ESP_CHACHAPOLY_PAD_BOUND 1 +#define ESP_CHACHAPOLY_IV_LEN 8 +#define ESP_CHACHAPOLY_ICV_LEN 16 +#define ESP_CHACHAPOLY_KEYBITS_WITH_SALT 288 /* 32 bytes key + 4 bytes salt */ + +int esp_chachapoly_schedlen(const struct esp_algorithm *); +int esp_chachapoly_schedule(const struct esp_algorithm *, + struct secasvar *); +int esp_chachapoly_encrypt(struct mbuf *, size_t, size_t, struct secasvar *, + const struct esp_algorithm *, int); +int esp_chachapoly_decrypt(struct mbuf *, size_t, struct secasvar *, + const struct esp_algorithm *, int); +int esp_chachapoly_encrypt_finalize(struct secasvar *, unsigned char *, unsigned int); +int esp_chachapoly_decrypt_finalize(struct secasvar *, unsigned char *, unsigned int); +int esp_chachapoly_mature(struct secasvar *); + +#endif /* _ESP_CHACHA_POLY_H_ */ +#endif /* BSD_KERNEL_PRIVATE */ diff --git a/bsd/netinet6/esp_core.c b/bsd/netinet6/esp_core.c index bc9a75ed6..a26873e45 100644 --- a/bsd/netinet6/esp_core.c +++ b/bsd/netinet6/esp_core.c @@ -98,6 +98,7 @@ #include #endif #include +#include #include #include #include @@ -183,6 +184,14 @@ static const struct esp_algorithm aes_gcm = esp_gcm_encrypt_aes, esp_gcm_schedule, 0, 0, 16, esp_gcm_decrypt_finalize, esp_gcm_encrypt_finalize}; +static const struct esp_algorithm chacha_poly = + { ESP_CHACHAPOLY_PAD_BOUND, ESP_CHACHAPOLY_IV_LEN, + esp_chachapoly_mature, ESP_CHACHAPOLY_KEYBITS_WITH_SALT, + ESP_CHACHAPOLY_KEYBITS_WITH_SALT, esp_chachapoly_schedlen, + "chacha-poly", esp_common_ivlen, esp_chachapoly_decrypt, + esp_chachapoly_encrypt, esp_chachapoly_schedule, + NULL, NULL, ESP_CHACHAPOLY_ICV_LEN, + esp_chachapoly_decrypt_finalize, esp_chachapoly_encrypt_finalize}; static const struct esp_algorithm *esp_algorithms[] = { &des_cbc, @@ -190,6 +199,7 @@ static const struct esp_algorithm *esp_algorithms[] = { &null_esp, &aes_cbc, &aes_gcm, + &chacha_poly, }; const struct esp_algorithm * @@ -206,6 +216,8 @@ esp_algorithm_lookup(int idx) return &aes_cbc; case SADB_X_EALG_AES_GCM: return &aes_gcm; + case SADB_X_EALG_CHACHA20POLY1305: + return &chacha_poly; default: return NULL; } @@ -248,6 +260,17 @@ esp_schedule(const struct esp_algorithm *algo, struct secasvar *sav) lck_mtx_unlock(sadb_mutex); return 0; } + + /* prevent disallowed implicit IV */ + if (((sav->flags & SADB_X_EXT_IIV) != 0) && + (sav->alg_enc != SADB_X_EALG_AES_GCM) && + (sav->alg_enc != SADB_X_EALG_CHACHA20POLY1305)) { + ipseclog((LOG_ERR, + "esp_schedule %s: implicit IV not allowed\n", + algo->name)); + return EINVAL; + } + /* no schedule necessary */ if (!algo->schedule || !algo->schedlen) { lck_mtx_unlock(sadb_mutex); @@ -384,7 +407,7 @@ esp_des_schedule( struct secasvar *sav) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); if (des_ecb_key_sched((des_cblock *)_KEYBUF(sav->key_enc), (des_ecb_key_schedule *)sav->sched)) return EINVAL; @@ -498,6 +521,11 @@ esp_gcm_mature(struct secasvar *sav) "esp_gcm_mature: algorithm incompatible with derived\n")); return 1; } + if (sav->flags & SADB_X_EXT_IIV) { + ipseclog((LOG_ERR, + "esp_gcm_mature: implicit IV not currently implemented\n")); + return 1; + } if (!sav->key_enc) { ipseclog((LOG_ERR, "esp_gcm_mature: no key is given.\n")); @@ -550,7 +578,7 @@ esp_3des_schedule( __unused const struct esp_algorithm *algo, struct secasvar *sav) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); if (des3_ecb_key_sched((des_cblock *)_KEYBUF(sav->key_enc), (des3_ecb_key_schedule *)sav->sched)) diff --git a/bsd/netinet6/esp_input.c b/bsd/netinet6/esp_input.c index 8056438bb..912dd3983 100644 --- a/bsd/netinet6/esp_input.c +++ b/bsd/netinet6/esp_input.c @@ -176,6 +176,12 @@ esp6_input_strip_udp_encap (struct mbuf *m, int ip6hlen) void esp4_input(struct mbuf *m, int off) +{ + (void)esp4_input_extended(m, off, NULL); +} + +struct mbuf * +esp4_input_extended(struct mbuf *m, int off, ifnet_t interface) { struct ip *ip; #if INET6 @@ -193,6 +199,7 @@ esp4_input(struct mbuf *m, int off) size_t hlen; size_t esplen; sa_family_t ifamily; + struct mbuf *out_m = NULL; KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_START, 0,0,0,0,0); /* sanity check for alignment. */ @@ -213,6 +220,8 @@ esp4_input(struct mbuf *m, int off) } } + m->m_pkthdr.csum_flags &= ~CSUM_RX_FLAGS; + /* Expect 32-bit aligned data pointer on strict-align platforms */ MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m); @@ -235,9 +244,9 @@ esp4_input(struct mbuf *m, int off) /* find the sassoc. */ spi = esp->esp_spi; - if ((sav = key_allocsa(AF_INET, - (caddr_t)&ip->ip_src, (caddr_t)&ip->ip_dst, - IPPROTO_ESP, spi)) == 0) { + if ((sav = key_allocsa_extended(AF_INET, + (caddr_t)&ip->ip_src, (caddr_t)&ip->ip_dst, + IPPROTO_ESP, spi, interface)) == 0) { ipseclog((LOG_WARNING, "IPv4 ESP input: no key association found for spi %u\n", (u_int32_t)ntohl(spi))); @@ -337,7 +346,7 @@ esp4_input(struct mbuf *m, int off) } if (cc_cmp_safe(siz, sum0, sum)) { - ipseclog((LOG_WARNING, "auth fail in IPv4 ESP input: %s %s\n", + ipseclog((LOG_WARNING, "cc_cmp fail in IPv4 ESP input: %s %s\n", ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav))); IPSEC_STAT_INCREMENT(ipsecstat.in_espauthfail); goto bad; @@ -618,6 +627,13 @@ noreplaycheck: // Input via IPSec interface if (sav->sah->ipsec_if != NULL) { + // Return mbuf + if (interface != NULL && + interface == sav->sah->ipsec_if) { + out_m = m; + goto done; + } + if (ipsec_inject_inbound_packet(sav->sah->ipsec_if, m) == 0) { m = NULL; goto done; @@ -670,6 +686,7 @@ noreplaycheck: if (nxt == IPPROTO_TCP || nxt == IPPROTO_UDP) { m->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR; m->m_pkthdr.csum_data = 0xFFFF; + _CASSERT(offsetof(struct pkthdr, csum_data) == offsetof(struct pkthdr, csum_rx_val)); } if (nxt != IPPROTO_DONE) { @@ -718,12 +735,28 @@ noreplaycheck: struct ip *, ip, struct ifnet *, m->m_pkthdr.rcvif, struct ip *, ip, struct ip6_hdr *, NULL); - // Input via IPSec interface + // Input via IPsec interface legacy path if (sav->sah->ipsec_if != NULL) { + int mlen; + if ((mlen = m_length2(m, NULL)) < hlen) { + ipseclog((LOG_DEBUG, + "IPv4 ESP input: decrypted packet too short %d < %d\n", + mlen, hlen)); + IPSEC_STAT_INCREMENT(ipsecstat.in_inval); + goto bad; + } ip->ip_len = htons(ip->ip_len + hlen); ip->ip_off = htons(ip->ip_off); ip->ip_sum = 0; ip->ip_sum = ip_cksum_hdr_in(m, hlen); + + // Return mbuf + if (interface != NULL && + interface == sav->sah->ipsec_if) { + out_m = m; + goto done; + } + if (ipsec_inject_inbound_packet(sav->sah->ipsec_if, m) == 0) { m = NULL; goto done; @@ -733,8 +766,9 @@ noreplaycheck: } ip_proto_dispatch_in(m, off, nxt, 0); - } else + } else { m_freem(m); + } m = NULL; } @@ -746,8 +780,7 @@ done: key_freesav(sav, KEY_SADB_UNLOCKED); } IPSEC_STAT_INCREMENT(ipsecstat.in_success); - return; - + return out_m; bad: if (sav) { KEYDEBUG(KEYDEBUG_IPSEC_STAMP, @@ -755,16 +788,24 @@ bad: (uint64_t)VM_KERNEL_ADDRPERM(sav))); key_freesav(sav, KEY_SADB_UNLOCKED); } - if (m) + if (m) { m_freem(m); + } KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_END, 4,0,0,0,0); - return; + return out_m; } #endif /* INET */ #if INET6 + int esp6_input(struct mbuf **mp, int *offp, int proto) +{ + return esp6_input_extended(mp, offp, proto, NULL); +} + +int +esp6_input_extended(struct mbuf **mp, int *offp, int proto, ifnet_t interface) { #pragma unused(proto) struct mbuf *m = *mp; @@ -802,6 +843,8 @@ esp6_input(struct mbuf **mp, int *offp, int proto) return IPPROTO_DONE; } #endif + m->m_pkthdr.csum_flags &= ~CSUM_RX_FLAGS; + /* Expect 32-bit data aligned pointer on strict-align platforms */ MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m); @@ -825,9 +868,9 @@ esp6_input(struct mbuf **mp, int *offp, int proto) /* find the sassoc. */ spi = esp->esp_spi; - if ((sav = key_allocsa(AF_INET6, - (caddr_t)&ip6->ip6_src, (caddr_t)&ip6->ip6_dst, - IPPROTO_ESP, spi)) == 0) { + if ((sav = key_allocsa_extended(AF_INET6, + (caddr_t)&ip6->ip6_src, (caddr_t)&ip6->ip6_dst, + IPPROTO_ESP, spi, interface)) == 0) { ipseclog((LOG_WARNING, "IPv6 ESP input: no key association found for spi %u\n", (u_int32_t)ntohl(spi))); @@ -1190,6 +1233,12 @@ noreplaycheck: // Input via IPSec interface if (sav->sah->ipsec_if != NULL) { + // Return mbuf + if (interface != NULL && + interface == sav->sah->ipsec_if) { + goto done; + } + if (ipsec_inject_inbound_packet(sav->sah->ipsec_if, m) == 0) { m = NULL; nxt = IPPROTO_DONE; @@ -1307,10 +1356,17 @@ noreplaycheck: if (nxt == IPPROTO_TCP || nxt == IPPROTO_UDP) { m->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR; m->m_pkthdr.csum_data = 0xFFFF; + _CASSERT(offsetof(struct pkthdr, csum_data) == offsetof(struct pkthdr, csum_rx_val)); } // Input via IPSec interface if (sav->sah->ipsec_if != NULL) { + // Return mbuf + if (interface != NULL && + interface == sav->sah->ipsec_if) { + goto done; + } + if (ipsec_inject_inbound_packet(sav->sah->ipsec_if, m) == 0) { m = NULL; nxt = IPPROTO_DONE; @@ -1341,13 +1397,17 @@ bad: (uint64_t)VM_KERNEL_ADDRPERM(sav))); key_freesav(sav, KEY_SADB_UNLOCKED); } - if (m) + if (m) { m_freem(m); + } + if (interface != NULL) { + *mp = NULL; + } return IPPROTO_DONE; } void -esp6_ctlinput(int cmd, struct sockaddr *sa, void *d) +esp6_ctlinput(int cmd, struct sockaddr *sa, void *d, __unused struct ifnet *ifp) { const struct newesp *espp; struct newesp esp; @@ -1355,7 +1415,7 @@ esp6_ctlinput(int cmd, struct sockaddr *sa, void *d) struct secasvar *sav; struct ip6_hdr *ip6; struct mbuf *m; - int off; + int off = 0; struct sockaddr_in6 *sa6_src, *sa6_dst; if (sa->sa_family != AF_INET6 || diff --git a/bsd/netinet6/esp_output.c b/bsd/netinet6/esp_output.c index 3b8b817d4..720846a19 100644 --- a/bsd/netinet6/esp_output.c +++ b/bsd/netinet6/esp_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -828,7 +828,7 @@ esp_output( ip6 = mtod(m, struct ip6_hdr *); udp->uh_ulen = htons(plen + siz + extendsiz + esphlen); udp->uh_sum = in6_pseudo(&ip6->ip6_src, &ip6->ip6_dst, htonl(ntohs(udp->uh_ulen) + IPPROTO_UDP)); - m->m_pkthdr.csum_flags = CSUM_UDPIPV6; + m->m_pkthdr.csum_flags = (CSUM_UDPIPV6|CSUM_ZERO_INVERT); m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); break; } diff --git a/bsd/netinet6/esp_rijndael.c b/bsd/netinet6/esp_rijndael.c index 56b560263..eba6c7fd3 100644 --- a/bsd/netinet6/esp_rijndael.c +++ b/bsd/netinet6/esp_rijndael.c @@ -109,7 +109,7 @@ esp_aes_schedule( struct secasvar *sav) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); aes_ctx *ctx = (aes_ctx*)sav->sched; aes_decrypt_key((const unsigned char *) _KEYBUF(sav->key_enc), _KEYLEN(sav->key_enc), &ctx->decrypt); @@ -287,12 +287,25 @@ esp_cbc_decrypt_aes( } else { sp_unaligned = sp; if (len > MAX_REALIGN_LEN) { + m_freem(m); + if (d0 != NULL) { + m_freem(d0); + } + if (sp_aligned != NULL) { + FREE(sp_aligned, M_SECA); + sp_aligned = NULL; + } return ENOBUFS; } if (sp_aligned == NULL) { sp_aligned = (u_int8_t *)_MALLOC(MAX_REALIGN_LEN, M_SECA, M_DONTWAIT); - if (sp_aligned == NULL) - return ENOMEM; + if (sp_aligned == NULL) { + m_freem(m); + if (d0 != NULL) { + m_freem(d0); + } + return ENOMEM; + } } sp = sp_aligned; memcpy(sp, sp_unaligned, len); @@ -482,12 +495,25 @@ esp_cbc_encrypt_aes( } else { sp_unaligned = sp; if (len > MAX_REALIGN_LEN) { + m_freem(m); + if (d0) { + m_freem(d0); + } + if (sp_aligned != NULL) { + FREE(sp_aligned, M_SECA); + sp_aligned = NULL; + } return ENOBUFS; } if (sp_aligned == NULL) { sp_aligned = (u_int8_t *)_MALLOC(MAX_REALIGN_LEN, M_SECA, M_DONTWAIT); - if (sp_aligned == NULL) + if (sp_aligned == NULL) { + m_freem(m); + if (d0) { + m_freem(d0); + } return ENOMEM; + } } sp = sp_aligned; memcpy(sp, sp_unaligned, len); @@ -556,7 +582,7 @@ int esp_gcm_schedule( __unused const struct esp_algorithm *algo, struct secasvar *sav) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); aes_gcm_ctx *ctx = (aes_gcm_ctx*)P2ROUNDUP(sav->sched, ESP_GCM_ALIGN); u_int ivlen = sav->ivlen; unsigned char nonce[ESP_GCM_SALT_LEN+ivlen]; @@ -764,12 +790,25 @@ esp_gcm_encrypt_aes( } else { sp_unaligned = sp; if (len > MAX_REALIGN_LEN) { + m_freem(m); + if (d0) { + m_freem(d0); + } + if (sp_aligned != NULL) { + FREE(sp_aligned, M_SECA); + sp_aligned = NULL; + } return ENOBUFS; } if (sp_aligned == NULL) { sp_aligned = (u_int8_t *)_MALLOC(MAX_REALIGN_LEN, M_SECA, M_DONTWAIT); - if (sp_aligned == NULL) + if (sp_aligned == NULL) { + m_freem(m); + if (d0) { + m_freem(d0); + } return ENOMEM; + } } sp = sp_aligned; memcpy(sp, sp_unaligned, len); @@ -960,12 +999,25 @@ esp_gcm_decrypt_aes( } else { sp_unaligned = sp; if (len > MAX_REALIGN_LEN) { + m_freem(m); + if (d0) { + m_freem(d0); + } + if (sp_aligned != NULL) { + FREE(sp_aligned, M_SECA); + sp_aligned = NULL; + } return ENOBUFS; } if (sp_aligned == NULL) { sp_aligned = (u_int8_t *)_MALLOC(MAX_REALIGN_LEN, M_SECA, M_DONTWAIT); - if (sp_aligned == NULL) + if (sp_aligned == NULL) { + m_freem(m); + if (d0) { + m_freem(d0); + } return ENOMEM; + } } sp = sp_aligned; memcpy(sp, sp_unaligned, len); diff --git a/bsd/netinet6/frag6.c b/bsd/netinet6/frag6.c index 8e5f416ea..5bdb1adf3 100644 --- a/bsd/netinet6/frag6.c +++ b/bsd/netinet6/frag6.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -79,6 +79,7 @@ #include #include #include +#include #include #include #include @@ -204,7 +205,7 @@ frag6_restore_context(struct mbuf *m) static void frag6_icmp6_paramprob_error(struct fq6_head *diq6) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_NOTOWNED); if (!MBUFQ_EMPTY(diq6)) { struct mbuf *merr, *merr_tmp; @@ -227,7 +228,7 @@ frag6_icmp6_paramprob_error(struct fq6_head *diq6) static void frag6_icmp6_timeex_error(struct fq6_head *diq6) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_NOTOWNED); if (!MBUFQ_EMPTY(diq6)) { struct mbuf *m, *m_tmp; @@ -387,20 +388,24 @@ frag6_input(struct mbuf **mp, int *offp, int proto) * as that is the most common case. * * Perform 1's complement adjustment of octets that got included/ - * excluded in the hardware-calculated checksum value. + * excluded in the hardware-calculated checksum value. Also take + * care of any trailing bytes and subtract out their partial sum. */ if (ip6f->ip6f_nxt == IPPROTO_UDP && offset == (sizeof (*ip6) + sizeof (*ip6f)) && (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) == (CSUM_DATA_VALID | CSUM_PARTIAL)) { - uint32_t start; + uint32_t start = m->m_pkthdr.csum_rx_start; + uint32_t ip_len = (sizeof (*ip6) + ntohs(ip6->ip6_plen)); + int32_t trailer = (m_pktlen(m) - ip_len); + uint32_t swbytes = (uint32_t)trailer; - start = m->m_pkthdr.csum_rx_start; csum = m->m_pkthdr.csum_rx_val; - if (start != offset) { - uint16_t s, d; + ASSERT(trailer >= 0); + if (start != offset || trailer != 0) { + uint16_t s = 0, d = 0; if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { s = ip6->ip6_src.s6_addr16[1]; @@ -412,7 +417,12 @@ frag6_input(struct mbuf **mp, int *offp, int proto) } /* callee folds in sum */ - csum = m_adj_sum16(m, start, offset, csum); + csum = m_adj_sum16(m, start, offset, + (ip_len - offset), csum); + if (offset > start) + swbytes += (offset - start); + else + swbytes += (start - offset); if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) ip6->ip6_src.s6_addr16[1] = s; @@ -421,6 +431,11 @@ frag6_input(struct mbuf **mp, int *offp, int proto) } csum_flags = m->m_pkthdr.csum_flags; + + if (swbytes != 0) + udp_in6_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } else { csum = 0; csum_flags = 0; @@ -873,7 +888,7 @@ frag6_freef(struct ip6q *q6, struct fq6_head *dfq6, struct fq6_head *diq6) { struct ip6asfrag *af6, *down6; - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); for (af6 = q6->ip6q_down; af6 != (struct ip6asfrag *)q6; af6 = down6) { @@ -916,7 +931,7 @@ frag6_freef(struct ip6q *q6, struct fq6_head *dfq6, struct fq6_head *diq6) void frag6_enq(struct ip6asfrag *af6, struct ip6asfrag *up6) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); af6->ip6af_up = up6; af6->ip6af_down = up6->ip6af_down; @@ -930,7 +945,7 @@ frag6_enq(struct ip6asfrag *af6, struct ip6asfrag *up6) void frag6_deq(struct ip6asfrag *af6) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); af6->ip6af_up->ip6af_down = af6->ip6af_down; af6->ip6af_down->ip6af_up = af6->ip6af_up; @@ -939,7 +954,7 @@ frag6_deq(struct ip6asfrag *af6) void frag6_insque(struct ip6q *new, struct ip6q *old) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); new->ip6q_prev = old; new->ip6q_next = old->ip6q_next; @@ -950,7 +965,7 @@ frag6_insque(struct ip6q *new, struct ip6q *old) void frag6_remque(struct ip6q *p6) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); p6->ip6q_prev->ip6q_next = p6->ip6q_next; p6->ip6q_next->ip6q_prev = p6->ip6q_prev; @@ -1021,7 +1036,7 @@ frag6_timeout(void *arg) static void frag6_sched_timeout(void) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); if (!frag6_timeout_run && frag6_nfragpackets > 0) { frag6_timeout_run = 1; @@ -1125,7 +1140,7 @@ ip6af_free(struct ip6asfrag *af6) static void ip6q_updateparams(void) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); /* * -1 for unlimited allocation. */ diff --git a/bsd/netinet6/icmp6.c b/bsd/netinet6/icmp6.c index 23a9178d3..1580cae9d 100644 --- a/bsd/netinet6/icmp6.c +++ b/bsd/netinet6/icmp6.c @@ -574,15 +574,13 @@ icmp6_input(struct mbuf **mp, int *offp, int proto) icmp6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_dstunreach); switch (code) { case ICMP6_DST_UNREACH_NOROUTE: + case ICMP6_DST_UNREACH_ADDR: /* PRC_HOSTDEAD is a DOS */ code = PRC_UNREACH_NET; break; case ICMP6_DST_UNREACH_ADMIN: icmp6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_adminprohib); code = PRC_UNREACH_PROTOCOL; /* is this a good code? */ break; - case ICMP6_DST_UNREACH_ADDR: - code = PRC_HOSTDEAD; - break; case ICMP6_DST_UNREACH_BEYONDSCOPE: /* I mean "source address was incorrect." */ code = PRC_PARAMPROB; @@ -942,7 +940,7 @@ icmp6_notify_error(struct mbuf *m, int off, int icmp6len, int code) /* Detect the upper level protocol */ { - void (*ctlfunc)(int, struct sockaddr *, void *); + void (*ctlfunc)(int, struct sockaddr *, void *, struct ifnet *); u_int8_t nxt = eip6->ip6_nxt; int eoff = off + sizeof(struct icmp6_hdr) + sizeof(struct ip6_hdr); @@ -1133,11 +1131,10 @@ icmp6_notify_error(struct mbuf *m, int off, int icmp6len, int code) icmp6_mtudisc_update(&ip6cp, 1); /*XXX*/ } - ctlfunc = (void (*)(int, struct sockaddr *, void *)) - (ip6_protox[nxt]->pr_ctlinput); + ctlfunc = ip6_protox[nxt]->pr_ctlinput; if (ctlfunc) { (void) (*ctlfunc)(code, (struct sockaddr *)&icmp6dst, - &ip6cp); + &ip6cp, m->m_pkthdr.rcvif); } } return(0); @@ -1288,7 +1285,7 @@ ni6_input(struct mbuf *m, int off) !(icmp6_nodeinfo & ICMP6_NODEINFO_TMPADDROK)) { nd6log((LOG_DEBUG, "ni6_input: ignore node info to " "a temporary address in %s:%d", - __FILE__, __LINE__)); + __func__, __LINE__)); goto bad; } } @@ -2150,7 +2147,7 @@ icmp6_reflect(struct mbuf *m, size_t off) nd6log((LOG_DEBUG, "sanity fail: off=%lx, sizeof(ip6)=%lx in %s:%d\n", (u_int32_t)off, (u_int32_t)sizeof(struct ip6_hdr), - __FILE__, __LINE__)); + __func__, __LINE__)); goto bad; } @@ -2784,7 +2781,7 @@ nolladdropt:; * and truncates if not. */ if (m0->m_next || m0->m_pkthdr.len != m0->m_len) - panic("assumption failed in %s:%d\n", __FILE__, __LINE__); + panic("assumption failed in %s:%d\n", __func__, __LINE__); if (len - sizeof(*nd_opt_rh) < m0->m_pkthdr.len) { /* not enough room, truncate */ diff --git a/bsd/netinet6/in6.c b/bsd/netinet6/in6.c index 559815be5..a76b74157 100644 --- a/bsd/netinet6/in6.c +++ b/bsd/netinet6/in6.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -105,9 +105,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -118,6 +120,7 @@ #include #include #include +#include #include #include @@ -142,6 +145,7 @@ #include #include +#include #if PF #include @@ -211,9 +215,6 @@ static int in6_to_kamescope(struct sockaddr_in6 *, struct ifnet *); static int in6_getassocids(struct socket *, uint32_t *, user_addr_t); static int in6_getconnids(struct socket *, sae_associd_t, uint32_t *, user_addr_t); -static int in6_getconninfo(struct socket *, sae_connid_t, uint32_t *, - uint32_t *, int32_t *, user_addr_t, socklen_t *, user_addr_t, socklen_t *, - uint32_t *, user_addr_t, uint32_t *); static void in6_if_up_dad_start(struct ifnet *); @@ -260,6 +261,8 @@ static struct zone *in6ifa_zone; /* zone for in6_ifaddr */ #define IN6IFA_ZONE_MAX 64 /* maximum elements in zone */ #define IN6IFA_ZONE_NAME "in6_ifaddr" /* zone name */ +struct eventhandler_lists_ctxt in6_evhdlr_ctxt; + /* * Subroutine for in6_ifaddloop() and in6_ifremloop(). * This routine does actual work. @@ -2377,7 +2380,7 @@ in6_purgeaddr(struct ifaddr *ifa) struct in6_ifaddr *ia = (struct in6_ifaddr *)ifa; struct in6_multi_mship *imm; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); /* stop DAD processing */ nd6_dad_stop(ifa); @@ -2435,7 +2438,7 @@ in6_unlink_ifa(struct in6_ifaddr *ia, struct ifnet *ifp) struct ifaddr *ifa; int unlinked; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); ifa = &ia->ia_ifa; IFA_ADDREF(ifa); @@ -2532,7 +2535,7 @@ in6_purgeif(struct ifnet *ifp) if (ifp == NULL) return; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); lck_rw_lock_exclusive(&in6_ifaddr_rwlock); ia = in6_ifaddrs; @@ -3466,6 +3469,15 @@ in6if_do_dad( (IFEF_IPV6_ND6ALT|IFEF_LOCALNET_PRIVATE|IFEF_DIRECTLINK)) return (0); + if (ifp->if_subfamily == IFNET_SUBFAMILY_IPSEC || + ifp->if_subfamily == IFNET_SUBFAMILY_UTUN) { + /* + * Ignore DAD for tunneling virtual interfaces, which get + * their IPv6 address explicitly assigned. + */ + return (0); + } + switch (ifp->if_type) { #if IFT_DUMMY case IFT_DUMMY: @@ -3680,21 +3692,23 @@ in6_post_msg(struct ifnet *ifp, u_int32_t event_code, struct in6_ifaddr *ifa, ev_msg.kev_subclass = KEV_INET6_SUBCLASS; ev_msg.event_code = event_code; - IFA_LOCK(&ifa->ia_ifa); - in6_event_data.ia_addr = ifa->ia_addr; - in6_event_data.ia_net = ifa->ia_net; - in6_event_data.ia_dstaddr = ifa->ia_dstaddr; - in6_event_data.ia_prefixmask = ifa->ia_prefixmask; - in6_event_data.ia_plen = ifa->ia_plen; - in6_event_data.ia6_flags = (u_int32_t)ifa->ia6_flags; - - /* retrieve time as calendar time (last arg is 1) */ - in6ifa_getlifetime(ifa, &ia6_lt, 1); - in6_event_data.ia_lifetime.ia6t_expire = ia6_lt.ia6t_expire; - in6_event_data.ia_lifetime.ia6t_preferred = ia6_lt.ia6t_preferred; - in6_event_data.ia_lifetime.ia6t_vltime = ia6_lt.ia6t_vltime; - in6_event_data.ia_lifetime.ia6t_pltime = ia6_lt.ia6t_pltime; - IFA_UNLOCK(&ifa->ia_ifa); + if (ifa) { + IFA_LOCK(&ifa->ia_ifa); + in6_event_data.ia_addr = ifa->ia_addr; + in6_event_data.ia_net = ifa->ia_net; + in6_event_data.ia_dstaddr = ifa->ia_dstaddr; + in6_event_data.ia_prefixmask = ifa->ia_prefixmask; + in6_event_data.ia_plen = ifa->ia_plen; + in6_event_data.ia6_flags = (u_int32_t)ifa->ia6_flags; + + /* retrieve time as calendar time (last arg is 1) */ + in6ifa_getlifetime(ifa, &ia6_lt, 1); + in6_event_data.ia_lifetime.ia6t_expire = ia6_lt.ia6t_expire; + in6_event_data.ia_lifetime.ia6t_preferred = ia6_lt.ia6t_preferred; + in6_event_data.ia_lifetime.ia6t_vltime = ia6_lt.ia6t_vltime; + in6_event_data.ia_lifetime.ia6t_pltime = ia6_lt.ia6t_pltime; + IFA_UNLOCK(&ifa->ia_ifa); + } if (ifp != NULL) { (void) strlcpy(&in6_event_data.link_data.if_name[0], @@ -3923,13 +3937,12 @@ in6_getconnids(struct socket *so, sae_associd_t aid, uint32_t *cnt, /* * Handle SIOCGCONNINFO ioctl for PF_INET6 domain. */ -static int +int in6_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len, user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type, user_addr_t aux_data, uint32_t *aux_len) { -#pragma unused(aux_data) struct in6pcb *in6p = sotoin6pcb(so); struct sockaddr_in6 sin6; struct ifnet *ifp = NULL; @@ -3997,8 +4010,6 @@ in6_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, } } - *aux_type = 0; - *aux_len = 0; if (SOCK_PROTO(so) == IPPROTO_TCP) { struct conninfo_tcp tcp_ci; @@ -4016,6 +4027,9 @@ in6_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, *aux_len = copy_len; } } + } else { + *aux_type = 0; + *aux_len = 0; } out: @@ -4053,3 +4067,528 @@ in6ioctl_cassert(void) ; } } + +struct in6_llentry { + struct llentry base; +}; + +#define IN6_LLTBL_DEFAULT_HSIZE 32 +#define IN6_LLTBL_HASH(k, h) \ + ((((((((k) >> 8) ^ (k)) >> 8) ^ (k)) >> 8) ^ (k)) & ((h) - 1)) + +/* + * Do actual deallocation of @lle. + */ +static void +in6_lltable_destroy_lle_unlocked(struct llentry *lle) +{ + LLE_LOCK_DESTROY(lle); + LLE_REQ_DESTROY(lle); + FREE(lle, M_LLTABLE); +} + +/* + * Called by LLE_FREE_LOCKED when number of references + * drops to zero. + */ +static void +in6_lltable_destroy_lle(struct llentry *lle) +{ + LLE_WUNLOCK(lle); + /* XXX TBD */ + //thread_call_free(lle->lle_timer); + in6_lltable_destroy_lle_unlocked(lle); +} + + +static struct llentry * +in6_lltable_new(const struct in6_addr *addr6, u_int flags) +{ +#pragma unused(flags) + struct in6_llentry *lle; + + MALLOC(lle, struct in6_llentry *, sizeof(struct in6_llentry), M_LLTABLE, M_NOWAIT | M_ZERO); + if (lle == NULL) /* NB: caller generates msg */ + return NULL; + + lle->base.r_l3addr.addr6 = *addr6; + lle->base.lle_refcnt = 1; + lle->base.lle_free = in6_lltable_destroy_lle; + LLE_LOCK_INIT(&lle->base); + LLE_REQ_INIT(&lle->base); +#if 0 + /* XXX TBD */ + lle->base.lle_timer = thread_call_allocate(nd6_llinfo_timer, lle); + + if (lle->base.lle_timer == NULL) { + printf("lle_timer thread call could not be allocated.\n"); + LLE_LOCK_DESTROY(&lle->base); + LLE_REQ_DESTROY(&lle->base); + FREE(lle, M_LLTABLE); + return NULL; + } +#endif + return (&lle->base); +} + +static int +in6_lltable_match_prefix(const struct sockaddr *saddr, + const struct sockaddr *smask, u_int flags, struct llentry *lle) +{ + const struct in6_addr *addr, *mask, *lle_addr; + + addr = &((const struct sockaddr_in6 *)(const void *)saddr)->sin6_addr; + mask = &((const struct sockaddr_in6 *)(const void *)smask)->sin6_addr; + lle_addr = &lle->r_l3addr.addr6; + + if (IN6_ARE_MASKED_ADDR_EQUAL(lle_addr, addr, mask) == 0) + return (0); + + if (lle->la_flags & LLE_IFADDR) { + /* + * Delete LLE_IFADDR records IFF address & flag matches. + * Note that addr is the interface address within prefix + * being matched. + */ + if (IN6_ARE_ADDR_EQUAL(addr, lle_addr) && + (flags & LLE_STATIC) != 0) + return (1); + return (0); + } + + /* flags & LLE_STATIC means deleting both dynamic and static entries */ + if ((flags & LLE_STATIC) || !(lle->la_flags & LLE_STATIC)) + return (1); + + return (0); +} + +static void +in6_lltable_free_entry(struct lltable *llt, struct llentry *lle) +{ + struct ifnet *ifp; + + LLE_WLOCK_ASSERT(lle); + KASSERT(llt != NULL, ("lltable is NULL")); + + /* Unlink entry from table */ + if ((lle->la_flags & LLE_LINKED) != 0) { + ifp = llt->llt_ifp; + if_afdata_wlock_assert(ifp, llt->llt_af); + lltable_unlink_entry(llt, lle); + } + +#if 0 + /* XXX TBD */ + if (thread_call_cancel(lle->lle_timer) == TRUE) + LLE_REMREF(lle); +#endif + llentry_free(lle); +} + +static int +in6_lltable_rtcheck(struct ifnet *ifp, + u_int flags, const struct sockaddr *l3addr) +{ +#pragma unused(flags) + struct rtentry *rt; + + KASSERT(l3addr->sa_family == AF_INET6, + ("sin_family %d", l3addr->sa_family)); + /* XXX rtalloc1 should take a const param */ + rt = rtalloc1(__DECONST(struct sockaddr *, l3addr), 0, 0); + if (rt == NULL || (rt->rt_flags & RTF_GATEWAY) || rt->rt_ifp != ifp) { + struct ifaddr *ifa; + /* + * Create an ND6 cache for an IPv6 neighbor + * that is not covered by our own prefix. + */ + /* XXX ifaof_ifpforaddr should take a const param */ + ifa = ifaof_ifpforaddr(__DECONST(struct sockaddr *, l3addr), ifp); + if (ifa != NULL) { + IFA_REMREF(ifa); + if (rt != NULL) + rtfree(rt); + return 0; + } + log(LOG_INFO, "IPv6 address: \"%s\" is not on the network\n", + ip6_sprintf(&((const struct sockaddr_in6 *)(const void *)l3addr)->sin6_addr)); + if (rt != NULL) + rtfree(rt); + return EINVAL; + } + rtfree(rt); + return 0; +} + +static inline uint32_t +in6_lltable_hash_dst(const struct in6_addr *dst, uint32_t hsize) +{ + return (IN6_LLTBL_HASH(dst->s6_addr32[3], hsize)); +} + +static uint32_t +in6_lltable_hash(const struct llentry *lle, uint32_t hsize) +{ + return (in6_lltable_hash_dst(&lle->r_l3addr.addr6, hsize)); +} + +static void +in6_lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa) +{ + struct sockaddr_in6 *sin6; + + sin6 = (struct sockaddr_in6 *)(void *)sa; + bzero(sin6, sizeof(*sin6)); + sin6->sin6_family = AF_INET6; + sin6->sin6_len = sizeof(*sin6); + sin6->sin6_addr = lle->r_l3addr.addr6; +} + +static inline struct llentry * +in6_lltable_find_dst(struct lltable *llt, const struct in6_addr *dst) +{ + struct llentry *lle; + struct llentries *lleh; + u_int hashidx; + + hashidx = in6_lltable_hash_dst(dst, llt->llt_hsize); + lleh = &llt->lle_head[hashidx]; + LIST_FOREACH(lle, lleh, lle_next) { + if (lle->la_flags & LLE_DELETED) + continue; + if (IN6_ARE_ADDR_EQUAL(&lle->r_l3addr.addr6, dst)) + break; + } + + return (lle); +} + +static void +in6_lltable_delete_entry(struct lltable *llt, struct llentry *lle) +{ +#pragma unused(llt) + lle->la_flags |= LLE_DELETED; + EVENTHANDLER_INVOKE(NULL, lle_event, lle, LLENTRY_DELETED); +#ifdef DIAGNOSTIC + log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle); +#endif + llentry_free(lle); +} + +static struct llentry * +in6_lltable_alloc(struct lltable *llt, u_int flags, + const struct sockaddr *l3addr) +{ + const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)(const void *)l3addr; + struct ifnet *ifp = llt->llt_ifp; + struct llentry *lle; + + KASSERT(l3addr->sa_family == AF_INET6, + ("sin_family %d", l3addr->sa_family)); + + /* + * A route that covers the given address must have + * been installed 1st because we are doing a resolution, + * verify this. + */ + if (!(flags & LLE_IFADDR) && + in6_lltable_rtcheck(ifp, flags, l3addr) != 0) + return (NULL); + + lle = in6_lltable_new(&sin6->sin6_addr, flags); + if (lle == NULL) { + log(LOG_INFO, "lla_lookup: new lle malloc failed\n"); + return (NULL); + } + lle->la_flags = flags; + if ((flags & LLE_IFADDR) == LLE_IFADDR) { + lltable_set_entry_addr(ifp, lle, LLADDR(SDL(ifp->if_lladdr->ifa_addr))); + lle->la_flags |= LLE_STATIC; + } + + if ((lle->la_flags & LLE_STATIC) != 0) + lle->ln_state = ND6_LLINFO_REACHABLE; + + return (lle); +} + +static struct llentry * +in6_lltable_lookup(struct lltable *llt, u_int flags, + const struct sockaddr *l3addr) +{ + const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)(const void *)l3addr; + struct llentry *lle; + + IF_AFDATA_LOCK_ASSERT(llt->llt_ifp, llt->llt_af); + KASSERT(l3addr->sa_family == AF_INET6, + ("sin_family %d", l3addr->sa_family)); + + lle = in6_lltable_find_dst(llt, &sin6->sin6_addr); + + if (lle == NULL) + return (NULL); + + KASSERT((flags & (LLE_UNLOCKED|LLE_EXCLUSIVE)) != + (LLE_UNLOCKED|LLE_EXCLUSIVE),("wrong lle request flags: 0x%X", + flags)); + + if (flags & LLE_UNLOCKED) + return (lle); + + if (flags & LLE_EXCLUSIVE) + LLE_WLOCK(lle); + else + LLE_RLOCK(lle); + return (lle); +} + +static int +in6_lltable_dump_entry(struct lltable *llt, struct llentry *lle, + struct sysctl_req *wr) +{ + struct ifnet *ifp = llt->llt_ifp; + /* XXX stack use */ + struct { + struct rt_msghdr rtm; + struct sockaddr_in6 sin6; + /* + * ndp.c assumes that sdl is word aligned + */ +#ifdef __LP64__ + uint32_t pad; +#endif + struct sockaddr_dl sdl; + } ndpc; + struct sockaddr_dl *sdl; + int error; + + bzero(&ndpc, sizeof(ndpc)); + /* skip deleted entries */ + if ((lle->la_flags & LLE_DELETED) == LLE_DELETED) + return (0); + /* Skip if jailed and not a valid IP of the prison. */ + lltable_fill_sa_entry(lle, + (struct sockaddr *)&ndpc.sin6); + /* + * produce a msg made of: + * struct rt_msghdr; + * struct sockaddr_in6 (IPv6) + * struct sockaddr_dl; + */ + ndpc.rtm.rtm_msglen = sizeof(ndpc); + ndpc.rtm.rtm_version = RTM_VERSION; + ndpc.rtm.rtm_type = RTM_GET; + ndpc.rtm.rtm_flags = RTF_UP; + ndpc.rtm.rtm_addrs = RTA_DST | RTA_GATEWAY; + + /* publish */ + if (lle->la_flags & LLE_PUB) + ndpc.rtm.rtm_flags |= RTF_ANNOUNCE; + sdl = &ndpc.sdl; + sdl->sdl_family = AF_LINK; + sdl->sdl_len = sizeof(*sdl); + sdl->sdl_index = ifp->if_index; + sdl->sdl_type = ifp->if_type; + if ((lle->la_flags & LLE_VALID) == LLE_VALID) { + sdl->sdl_alen = ifp->if_addrlen; + bcopy(&lle->ll_addr, LLADDR(sdl), ifp->if_addrlen); + } else { + sdl->sdl_alen = 0; + bzero(LLADDR(sdl), ifp->if_addrlen); + } + if (lle->la_expire != 0) { + clock_sec_t secs; + clock_usec_t usecs; + + clock_get_calendar_microtime(&secs, &usecs); + ndpc.rtm.rtm_rmx.rmx_expire = lle->la_expire + + lle->lle_remtime / hz + + secs - net_uptime(); + } + ndpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA); + if (lle->la_flags & LLE_STATIC) + ndpc.rtm.rtm_flags |= RTF_STATIC; + if (lle->la_flags & LLE_IFADDR) + ndpc.rtm.rtm_flags |= RTF_PINNED; + if (lle->ln_router != 0) + ndpc.rtm.rtm_flags |= RTF_GATEWAY; + ndpc.rtm.rtm_rmx.rmx_pksent = lle->la_asked; + /* Store state in rmx_weight value */ + ndpc.rtm.rtm_rmx.rmx_state = lle->ln_state; + ndpc.rtm.rtm_index = ifp->if_index; + error = SYSCTL_OUT(wr, &ndpc, sizeof(ndpc)); + + return (error); +} + +struct lltable * +in6_lltattach(struct ifnet *ifp) +{ + struct lltable *llt; + + llt = lltable_allocate_htbl(IN6_LLTBL_DEFAULT_HSIZE); + llt->llt_af = AF_INET6; + llt->llt_ifp = ifp; + + llt->llt_lookup = in6_lltable_lookup; + llt->llt_alloc_entry = in6_lltable_alloc; + llt->llt_delete_entry = in6_lltable_delete_entry; + llt->llt_dump_entry = in6_lltable_dump_entry; + llt->llt_hash = in6_lltable_hash; + llt->llt_fill_sa_entry = in6_lltable_fill_sa_entry; + llt->llt_free_entry = in6_lltable_free_entry; + llt->llt_match_prefix = in6_lltable_match_prefix; + lltable_link(llt); + + return (llt); +} + +void +in6_ip6_to_sockaddr(const struct in6_addr *ip6, u_int16_t port, + struct sockaddr_in6 *sin6, u_int32_t maxlen) +{ + if (maxlen < sizeof(struct sockaddr_in6)) { + return; + } + + *sin6 = (struct sockaddr_in6) { + .sin6_family = AF_INET6, + .sin6_len = sizeof(*sin6), + .sin6_port = port, + .sin6_addr = *ip6, + }; + + if (IN6_IS_SCOPE_EMBED(&sin6->sin6_addr)) + { + sin6->sin6_scope_id = ntohs(sin6->sin6_addr.s6_addr16[1]); + sin6->sin6_addr.s6_addr16[1] = 0; + } +} + +/* IPv6 events */ +struct in6_event { + in6_evhdlr_code_t in6_event_code; + struct ifnet *in6_ifp; + struct in6_addr in6_address; + uint32_t val; +}; + +struct in6_event2kev in6_event2kev_array[IN6_EVENT_MAX] = { +{ + .in6_event_code = IN6_ADDR_MARKED_DUPLICATED, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_DAD_FAILURE, + .in6_event_str = "IN6_ADDR_MARKED_DUPLICATED", +}, +{ + .in6_event_code = IN6_ADDR_MARKED_DETACHED, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_ADDR_DETACHED, + .in6_event_str = "IN6_ADDR_MARKED_DETACHED", +}, +{ + .in6_event_code = IN6_ADDR_MARKED_DEPRECATED, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_ADDR_DEPRECATED, + .in6_event_str = "IN6_ADDR_MARKED_DEPRECATED", +}, +{ + .in6_event_code = IN6_NDP_RTR_EXPIRY, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_RTR_EXPIRED, + .in6_event_str = "IN6_NDP_RTR_EXPIRY", +}, +{ + .in6_event_code = IN6_NDP_PFX_EXPIRY, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_PFX_EXPIRED, + .in6_event_str = "IN6_NDP_PFX_EXPIRY", +}, +{ + .in6_event_code = IN6_NDP_ADDR_EXPIRY, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_ADDR_EXPIRED, + .in6_event_str = "IN6_NDP_ADDR_EXPIRY", +}, +}; + +void +in6_eventhdlr_callback(struct eventhandler_entry_arg arg0 __unused, + in6_evhdlr_code_t in6_ev_code, struct ifnet *ifp, + struct in6_addr *p_addr6, uint32_t val) +{ + struct kev_msg ev_msg; + struct kev_nd6_event nd6_event; + + bzero(&ev_msg, sizeof(ev_msg)); + bzero(&nd6_event, sizeof(nd6_event)); + + nd6log0((LOG_INFO, "%s Event %s received for %s\n", + __func__, in6_event2kev_array[in6_ev_code].in6_event_str, + ip6_sprintf(p_addr6))); + + ev_msg.vendor_code = KEV_VENDOR_APPLE; + ev_msg.kev_class = KEV_NETWORK_CLASS; + ev_msg.kev_subclass = + in6_event2kev_array[in6_ev_code].in6_event_kev_subclass; + ev_msg.event_code = + in6_event2kev_array[in6_ev_code].in6_event_kev_code; + + nd6_event.link_data.if_family = ifp->if_family; + nd6_event.link_data.if_unit = ifp->if_unit; + strlcpy(nd6_event.link_data.if_name, ifp->if_name, + sizeof(nd6_event.link_data.if_name)); + + VERIFY(p_addr6 != NULL); + bcopy(p_addr6, &nd6_event.in6_address, + sizeof(nd6_event.in6_address)); + nd6_event.val = val; + + ev_msg.dv[0].data_ptr = &nd6_event; + ev_msg.dv[0].data_length = sizeof(nd6_event); + + kev_post_msg(&ev_msg); +} + +static void +in6_event_callback(void *arg) +{ + struct in6_event *p_in6_ev = (struct in6_event *)arg; + + EVENTHANDLER_INVOKE(&in6_evhdlr_ctxt, in6_event, + p_in6_ev->in6_event_code, p_in6_ev->in6_ifp, + &p_in6_ev->in6_address, p_in6_ev->val); +} + +struct in6_event_nwk_wq_entry +{ + struct nwk_wq_entry nwk_wqe; + struct in6_event in6_ev_arg; +}; + +void +in6_event_enqueue_nwk_wq_entry(in6_evhdlr_code_t in6_event_code, + struct ifnet *ifp, struct in6_addr *p_addr6, + uint32_t val) +{ + struct in6_event_nwk_wq_entry *p_in6_ev = NULL; + + MALLOC(p_in6_ev, struct in6_event_nwk_wq_entry *, + sizeof(struct in6_event_nwk_wq_entry), + M_NWKWQ, M_WAITOK | M_ZERO); + + p_in6_ev->nwk_wqe.func = in6_event_callback; + p_in6_ev->nwk_wqe.is_arg_managed = TRUE; + p_in6_ev->nwk_wqe.arg = &p_in6_ev->in6_ev_arg; + + p_in6_ev->in6_ev_arg.in6_event_code = in6_event_code; + p_in6_ev->in6_ev_arg.in6_ifp = ifp; + if (p_addr6 != NULL) { + bcopy(p_addr6, &p_in6_ev->in6_ev_arg.in6_address, + sizeof(p_in6_ev->in6_ev_arg.in6_address)); + } + p_in6_ev->in6_ev_arg.val = val; + + nwk_wq_enqueue((struct nwk_wq_entry*)p_in6_ev); +} diff --git a/bsd/netinet6/in6.h b/bsd/netinet6/in6.h index b09c026b4..3fe1484e2 100644 --- a/bsd/netinet6/in6.h +++ b/bsd/netinet6/in6.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2015 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -465,6 +465,9 @@ struct route_in6_old { #endif /* PRIVATE */ #ifdef BSD_KERNEL_PRIVATE +#include +#include + /* * IP6 route structure * @@ -479,6 +482,8 @@ struct route_in6 { * to a 'struct route *'. */ struct rtentry *ro_rt; + struct llentry *ro_lle; + struct ifaddr *ro_srcia; uint32_t ro_flags; /* route flags */ struct sockaddr_in6 ro_dst; @@ -846,13 +851,19 @@ struct cmsghdr; struct mbuf; struct ifnet; struct in6_aliasreq; +struct lltable; +extern struct lltable * in6_lltattach(struct ifnet *ifp); extern uint16_t in6_pseudo(const struct in6_addr *, const struct in6_addr *, uint32_t); extern u_int16_t inet6_cksum(struct mbuf *, uint32_t, uint32_t, uint32_t); +extern u_int16_t inet6_cksum_buffer(const uint8_t *, uint32_t, uint32_t, + uint32_t); -#define in6_cksum(_m, _n, _o, _l) \ +#define in6_cksum(_m, _n, _o, _l) \ inet6_cksum(_m, _n, _o, _l) +#define in6_cksum_buffer(_b, _n, _o, _l) \ + inet6_cksum_buffer(_b, _n, _o, _l) extern int in6_addrscope(struct in6_addr *); extern struct in6_ifaddr *in6_ifawithscope(struct ifnet *, struct in6_addr *); @@ -873,6 +884,54 @@ extern uint32_t in6_finalize_cksum(struct mbuf *, uint32_t, int32_t, ((void) in6_finalize_cksum(_m, 0, 0, -1, CSUM_DELAY_IPV6_DATA)) #define in6_delayed_cksum_offset(_m, _o, _s, _p) \ ((void) in6_finalize_cksum(_m, _o, _s, _p, CSUM_DELAY_IPV6_DATA)) + +/* IPv6 protocol events */ +extern struct eventhandler_lists_ctxt in6_evhdlr_ctxt; + +/* + * XXX Avoid reordering the enum values below. + * If the order is changed, please make sure + * in6_event2kev_array is also changed to reflect the + * change in order of the enums + */ +typedef enum { + /* Address events */ + /* + * XXX To avoid duplicacy and also for correctness + * only report these for link local and stable addresses + * NOTE: Link local address can never be marked detached + * or duplicated. + */ + IN6_ADDR_MARKED_DUPLICATED, + IN6_ADDR_MARKED_DETACHED, + IN6_ADDR_MARKED_DEPRECATED, + + /* Expiry events */ + IN6_NDP_RTR_EXPIRY, + IN6_NDP_PFX_EXPIRY, + IN6_NDP_ADDR_EXPIRY, + + /* XXX DNS expiry needs to be handled by user-space */ + /* MAX */ + IN6_EVENT_MAX, +} in6_evhdlr_code_t; + +struct in6_event2kev { + in6_evhdlr_code_t in6_event_code; + uint32_t in6_event_kev_subclass; + uint32_t in6_event_kev_code; + const char *in6_event_str; +}; +extern struct in6_event2kev in6_event2kev_array[]; + +extern void in6_eventhdlr_callback(struct eventhandler_entry_arg, in6_evhdlr_code_t, + struct ifnet *, struct in6_addr *, uint32_t); +extern void in6_event_enqueue_nwk_wq_entry(in6_evhdlr_code_t, + struct ifnet *, struct in6_addr *, uint32_t); + +typedef void (*in6_event_fn) (struct eventhandler_entry_arg, in6_evhdlr_code_t, + struct ifnet *, struct in6_addr *, uint32_t); +EVENTHANDLER_DECLARE(in6_event, in6_event_fn); #endif /* BSD_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE diff --git a/bsd/netinet6/in6_cga.c b/bsd/netinet6/in6_cga.c index 5174dfdf0..981df0a18 100644 --- a/bsd/netinet6/in6_cga.c +++ b/bsd/netinet6/in6_cga.c @@ -81,8 +81,11 @@ static struct in6_cga_singleton in6_cga = { static void in6_cga_node_lock_assert(int owned) { +#if !MACH_ASSERT +#pragma unused(owned) +#endif VERIFY(in6_cga.cga_initialized); - lck_mtx_assert(&in6_cga.cga_mutex, owned); + LCK_MTX_ASSERT(&in6_cga.cga_mutex, owned); } static boolean_t diff --git a/bsd/netinet6/in6_cksum.c b/bsd/netinet6/in6_cksum.c index 2bea8cbcc..394ea35b2 100644 --- a/bsd/netinet6/in6_cksum.c +++ b/bsd/netinet6/in6_cksum.c @@ -232,3 +232,47 @@ inet6_cksum(struct mbuf *m, uint32_t nxt, uint32_t off, uint32_t len) return (~sum & 0xffff); } + +/* + * buffer MUST contain at least an IPv6 header, if nxt is specified; + * nxt is the upper layer protocol number; + * off is an offset where TCP/UDP/ICMP6 header starts; + * len is a total length of a transport segment (e.g. TCP header + TCP payload) + */ +u_int16_t +inet6_cksum_buffer(const uint8_t *buffer, uint32_t nxt, uint32_t off, + uint32_t len) +{ + uint32_t sum; + + if (off >= len) + panic("%s: off (%d) >= len (%d)", __func__, off, len); + + sum = b_sum16(&((const uint8_t *)buffer)[off], len); + + if (nxt != 0) { + const struct ip6_hdr *ip6; + unsigned char buf[sizeof (*ip6)] __attribute__((aligned(8))); + + /* + * In case the IPv6 header is not contiguous, or not 32-bit + * aligned, copy it to a local buffer. Note here that we + * expect the data pointer to point to the IPv6 header. + */ + if (!IP6_HDR_ALIGNED_P(buffer)) { + memcpy(buf, buffer, sizeof (*ip6)); + ip6 = (const struct ip6_hdr *)(const void *)buf; + } else { + ip6 = (const struct ip6_hdr *)buffer; + } + + /* add pseudo header checksum */ + sum += in6_pseudo(&ip6->ip6_src, &ip6->ip6_dst, + htonl(nxt + len)); + + /* fold in carry bits */ + ADDCARRY(sum); + } + + return (~sum & 0xffff); +} diff --git a/bsd/netinet6/in6_ifattach.c b/bsd/netinet6/in6_ifattach.c index 804a77d8e..86759c202 100644 --- a/bsd/netinet6/in6_ifattach.c +++ b/bsd/netinet6/in6_ifattach.c @@ -72,6 +72,7 @@ #include #include #include +#include #include #include @@ -710,7 +711,7 @@ in6_ifattach_prelim(struct ifnet *ifp) * (previously, this was a silent error.) */ if ((ifp->if_flags & IFF_MULTICAST) == 0) { - nd6log((LOG_INFO, "in6_ifattach: ", + nd6log0((LOG_INFO, "in6_ifattach: ", "%s is not multicast capable, IPv6 not enabled\n", if_name(ifp))); return (EINVAL); @@ -732,6 +733,7 @@ skipmcast: pbuf = (void **)((intptr_t)base - sizeof(void *)); *pbuf = ext; ifp->if_inet6data = base; + IN6_IFEXTRA(ifp)->ii_llt = in6_lltattach(ifp); VERIFY(IS_P2ALIGNED(ifp->if_inet6data, sizeof(uint64_t))); } else { /* @@ -744,6 +746,11 @@ skipmcast: sizeof(IN6_IFEXTRA(ifp)->icmp6_ifstat)); bzero(&IN6_IFEXTRA(ifp)->in6_ifstat, sizeof(IN6_IFEXTRA(ifp)->in6_ifstat)); + IN6_IFEXTRA(ifp)->netsig_len = 0; + bzero(&IN6_IFEXTRA(ifp)->netsig, + sizeof(IN6_IFEXTRA(ifp)->netsig)); + bzero(IN6_IFEXTRA(ifp)->nat64_prefixes, sizeof(IN6_IFEXTRA(ifp)->nat64_prefixes)); + /* XXX TBD Purge the layer two table */ /* * XXX When recycling, nd_ifinfo gets initialized, other * than the lock, inside nd6_ifattach @@ -995,11 +1002,14 @@ in6_ifdetach(struct ifnet *ifp) struct in6_multi_mship *imm; int unlinked; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); /* remove neighbor management table */ nd6_purge(ifp); + if (LLTABLE6(ifp)) + lltable_free(LLTABLE6(ifp)); + /* nuke any of IPv6 addresses we have */ lck_rw_lock_exclusive(&in6_ifaddr_rwlock); ia = in6_ifaddrs; diff --git a/bsd/netinet6/in6_mcast.c b/bsd/netinet6/in6_mcast.c index 1fc7aca03..778bafa8d 100644 --- a/bsd/netinet6/in6_mcast.c +++ b/bsd/netinet6/in6_mcast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2016 Apple Inc. All rights reserved. + * Copyright (c) 2010-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -80,6 +80,7 @@ #include #include +#include #include #include @@ -96,17 +97,6 @@ #include #include -#ifndef __SOCKUNION_DECLARED -union sockunion { - struct sockaddr_storage ss; - struct sockaddr sa; - struct sockaddr_dl sdl; - struct sockaddr_in6 sin6; -}; -typedef union sockunion sockunion_t; -#define __SOCKUNION_DECLARED -#endif /* __SOCKUNION_DECLARED */ - static void im6f_commit(struct in6_mfilter *); static int im6f_get_source(struct in6_mfilter *imf, const struct sockaddr_in6 *psin, @@ -119,10 +109,10 @@ static void im6f_rollback(struct in6_mfilter *); static void im6f_reap(struct in6_mfilter *); static int im6o_grow(struct ip6_moptions *, size_t); static size_t im6o_match_group(const struct ip6_moptions *, - const struct ifnet *, const struct sockaddr *); + const struct ifnet *, const struct sockaddr_in6 *); static struct in6_msource * - im6o_match_source(const struct ip6_moptions *, const size_t, - const struct sockaddr *); + im6o_match_source(const struct ip6_moptions *, + const size_t, const struct sockaddr_in6 *); static void im6s_merge(struct ip6_msource *ims, const struct in6_msource *lims, const int rollback); static int in6_mc_get(struct ifnet *, const struct in6_addr *, @@ -339,7 +329,7 @@ im6o_grow(struct ip6_moptions *imo, size_t newmax) */ static size_t im6o_match_group(const struct ip6_moptions *imo, const struct ifnet *ifp, - const struct sockaddr *group) + const struct sockaddr_in6 *group) { const struct sockaddr_in6 *gsin6; struct in6_multi *pinm; @@ -348,7 +338,7 @@ im6o_match_group(const struct ip6_moptions *imo, const struct ifnet *ifp, IM6O_LOCK_ASSERT_HELD(__DECONST(struct ip6_moptions *, imo)); - gsin6 = (struct sockaddr_in6 *)(uintptr_t)(size_t)group; + gsin6 = group; /* The im6o_membership array may be lazy allocated. */ if (imo->im6o_membership == NULL || imo->im6o_num_memberships == 0) @@ -387,16 +377,16 @@ im6o_match_group(const struct ip6_moptions *imo, const struct ifnet *ifp, */ static struct in6_msource * im6o_match_source(const struct ip6_moptions *imo, const size_t gidx, - const struct sockaddr *src) + const struct sockaddr_in6 *src) { struct ip6_msource find; struct in6_mfilter *imf; struct ip6_msource *ims; - const sockunion_t *psa; + const struct sockaddr_in6 *psa; IM6O_LOCK_ASSERT_HELD(__DECONST(struct ip6_moptions *, imo)); - VERIFY(src->sa_family == AF_INET6); + VERIFY(src->sin6_family == AF_INET6); VERIFY(gidx != (size_t)-1 && gidx < imo->im6o_num_memberships); /* The im6o_mfilters array may be lazy allocated. */ @@ -404,8 +394,8 @@ im6o_match_source(const struct ip6_moptions *imo, const size_t gidx, return (NULL); imf = &imo->im6o_mfilters[gidx]; - psa = (sockunion_t *)(uintptr_t)(size_t)src; - find.im6s_addr = psa->sin6.sin6_addr; + psa = src; + find.im6s_addr = psa->sin6_addr; in6_clearscope(&find.im6s_addr); /* XXX */ ims = RB_FIND(ip6_msource_tree, &imf->im6f_sources, &find); @@ -420,7 +410,7 @@ im6o_match_source(const struct ip6_moptions *imo, const size_t gidx, */ int im6o_mc_filter(const struct ip6_moptions *imo, const struct ifnet *ifp, - const struct sockaddr *group, const struct sockaddr *src) + const struct sockaddr_in6 *group, const struct sockaddr_in6 *src) { size_t gidx; struct in6_msource *ims; @@ -995,7 +985,7 @@ im6s_merge(struct ip6_msource *ims, const struct in6_msource *lims, static int in6m_merge(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf) { - struct ip6_msource *ims, *nims; + struct ip6_msource *ims, *nims = NULL; struct in6_msource *lims; int schanged, error; int nsrc0, nsrc1; @@ -1403,7 +1393,7 @@ static int in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) { struct group_source_req gsr; - sockunion_t *gsa, *ssa; + struct sockaddr_in6 *gsa, *ssa; struct ifnet *ifp; struct in6_mfilter *imf; struct ip6_moptions *imo; @@ -1420,8 +1410,8 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) doblock = 0; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - ssa = (sockunion_t *)&gsr.gsr_source; + gsa = (struct sockaddr_in6 *)&gsr.gsr_group; + ssa = (struct sockaddr_in6 *)&gsr.gsr_source; switch (sopt->sopt_name) { case MCAST_BLOCK_SOURCE: @@ -1432,12 +1422,12 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin6.sin6_family != AF_INET6 || - gsa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (gsa->sin6_family != AF_INET6 || + gsa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); - if (ssa->sin6.sin6_family != AF_INET6 || - ssa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (ssa->sin6_family != AF_INET6 || + ssa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); ifnet_head_lock_shared(); @@ -1463,10 +1453,10 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) return (EOPNOTSUPP); } - if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6.sin6_addr)) + if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr)) return (EINVAL); - (void) in6_setscope(&gsa->sin6.sin6_addr, ifp, NULL); + (void) in6_setscope(&gsa->sin6_addr, ifp, NULL); /* * Check if we are actually a member of this group. @@ -1476,7 +1466,7 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IM6O_LOCK(imo); - idx = im6o_match_group(imo, ifp, &gsa->sa); + idx = im6o_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->im6o_mfilters == NULL) { error = EADDRNOTAVAIL; goto out_imo_locked; @@ -1502,10 +1492,10 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) * Asked to unblock, but nothing to unblock. * If adding a new block entry, allocate it. */ - ims = im6o_match_source(imo, idx, &ssa->sa); + ims = im6o_match_source(imo, idx, ssa); if ((ims != NULL && doblock) || (ims == NULL && !doblock)) { MLD_PRINTF(("%s: source %s %spresent\n", __func__, - ip6_sprintf(&ssa->sin6.sin6_addr), + ip6_sprintf(&ssa->sin6_addr), doblock ? "" : "not ")); error = EADDRNOTAVAIL; goto out_imo_locked; @@ -1516,12 +1506,12 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) */ if (doblock) { MLD_PRINTF(("%s: %s source\n", __func__, "block")); - ims = im6f_graft(imf, fmode, &ssa->sin6); + ims = im6f_graft(imf, fmode, ssa); if (ims == NULL) error = ENOMEM; } else { MLD_PRINTF(("%s: %s source\n", __func__, "allow")); - error = im6f_prune(imf, &ssa->sin6); + error = im6f_prune(imf, ssa); } if (error) { @@ -1631,7 +1621,7 @@ in6p_get_source_filters(struct inpcb *inp, struct sockopt *sopt) { struct __msfilterreq64 msfr, msfr64; struct __msfilterreq32 msfr32; - sockunion_t *gsa; + struct sockaddr_in6 *gsa; struct ifnet *ifp; struct ip6_moptions *imo; struct in6_mfilter *imf; @@ -1669,8 +1659,8 @@ in6p_get_source_filters(struct inpcb *inp, struct sockopt *sopt) msfr.msfr_group.ss_len != sizeof(struct sockaddr_in6)) return (EINVAL); - gsa = (sockunion_t *)&msfr.msfr_group; - if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6.sin6_addr)) + gsa = (struct sockaddr_in6 *)&msfr.msfr_group; + if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr)) return (EINVAL); ifnet_head_lock_shared(); @@ -1691,13 +1681,13 @@ in6p_get_source_filters(struct inpcb *inp, struct sockopt *sopt) if (msfr.msfr_nsrcs > in6_mcast_maxsocksrc) msfr.msfr_nsrcs = in6_mcast_maxsocksrc; - (void)in6_setscope(&gsa->sin6.sin6_addr, ifp, NULL); + (void)in6_setscope(&gsa->sin6_addr, ifp, NULL); IM6O_LOCK(imo); /* * Lookup group on the socket. */ - idx = im6o_match_group(imo, ifp, &gsa->sa); + idx = im6o_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->im6o_mfilters == NULL) { IM6O_UNLOCK(imo); return (EADDRNOTAVAIL); @@ -1952,7 +1942,7 @@ static int in6p_join_group(struct inpcb *inp, struct sockopt *sopt) { struct group_source_req gsr; - sockunion_t *gsa, *ssa; + struct sockaddr_in6 *gsa, *ssa; struct ifnet *ifp; struct in6_mfilter *imf; struct ip6_moptions *imo; @@ -1970,10 +1960,8 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) is_new = 0; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - gsa->ss.ss_family = AF_UNSPEC; - ssa = (sockunion_t *)&gsr.gsr_source; - ssa->ss.ss_family = AF_UNSPEC; + gsa = (struct sockaddr_in6 *)&gsr.gsr_group; + ssa = (struct sockaddr_in6 *)&gsr.gsr_source; /* * Chew everything into struct group_source_req. @@ -1984,7 +1972,6 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) switch (sopt->sopt_name) { case IPV6_JOIN_GROUP: { struct ipv6_mreq mreq; - struct sockaddr_in6 *gsin6; error = sooptcopyin(sopt, &mreq, sizeof(struct ipv6_mreq), sizeof(struct ipv6_mreq)); @@ -2011,19 +1998,17 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) return (inp_join_group(inp, &v4sopt)); } - gsa->sin6.sin6_family = AF_INET6; - gsa->sin6.sin6_len = sizeof(struct sockaddr_in6); - gsa->sin6.sin6_addr = mreq.ipv6mr_multiaddr; - - gsin6 = &gsa->sin6; + gsa->sin6_family = AF_INET6; + gsa->sin6_len = sizeof(struct sockaddr_in6); + gsa->sin6_addr = mreq.ipv6mr_multiaddr; /* Only allow IPv6 multicast addresses */ - if (IN6_IS_ADDR_MULTICAST(&gsin6->sin6_addr) == 0) { + if (IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr) == 0) { return (EINVAL); } if (mreq.ipv6mr_interface == 0) { - ifp = in6p_lookup_mcast_ifp(inp, gsin6); + ifp = in6p_lookup_mcast_ifp(inp, gsa); } else { ifnet_head_lock_shared(); if ((u_int)if_index < mreq.ipv6mr_interface) { @@ -2053,24 +2038,24 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin6.sin6_family != AF_INET6 || - gsa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (gsa->sin6_family != AF_INET6 || + gsa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); if (sopt->sopt_name == MCAST_JOIN_SOURCE_GROUP) { - if (ssa->sin6.sin6_family != AF_INET6 || - ssa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (ssa->sin6_family != AF_INET6 || + ssa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); - if (IN6_IS_ADDR_MULTICAST(&ssa->sin6.sin6_addr)) + if (IN6_IS_ADDR_MULTICAST(&ssa->sin6_addr)) return (EINVAL); /* * TODO: Validate embedded scope ID in source * list entry against passed-in ifp, if and only * if source list filter entry is iface or node local. */ - in6_clearscope(&ssa->sin6.sin6_addr); - ssa->sin6.sin6_port = 0; - ssa->sin6.sin6_scope_id = 0; + in6_clearscope(&ssa->sin6_addr); + ssa->sin6_port = 0; + ssa->sin6_scope_id = 0; } ifnet_head_lock_shared(); @@ -2089,28 +2074,36 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) return (EOPNOTSUPP); } - if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6.sin6_addr)) + if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr)) return (EINVAL); if (ifp == NULL || (ifp->if_flags & IFF_MULTICAST) == 0) return (EADDRNOTAVAIL); - gsa->sin6.sin6_port = 0; - gsa->sin6.sin6_scope_id = 0; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_mcast_join_total); + /* + * TBD: revisit the criteria for non-OS initiated joins + */ + if (inp->inp_lport == htons(5353)) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_mcast_join_os_total); + } + + gsa->sin6_port = 0; + gsa->sin6_scope_id = 0; /* * Always set the scope zone ID on memberships created from userland. * Use the passed-in ifp to do this. */ - (void)in6_setscope(&gsa->sin6.sin6_addr, ifp, &scopeid); + (void)in6_setscope(&gsa->sin6_addr, ifp, &scopeid); /* * Some addresses are not valid without an embedded scopeid. * This check must be present because otherwise we will later hit * a VERIFY() in in6_mc_join(). */ - if ((IN6_IS_ADDR_MC_LINKLOCAL(&gsa->sin6.sin6_addr) || - IN6_IS_ADDR_MC_INTFACELOCAL(&gsa->sin6.sin6_addr)) && - (scopeid == 0 || gsa->sin6.sin6_addr.s6_addr16[1] == 0)) + if ((IN6_IS_ADDR_MC_LINKLOCAL(&gsa->sin6_addr) || + IN6_IS_ADDR_MC_INTFACELOCAL(&gsa->sin6_addr)) && + (scopeid == 0 || gsa->sin6_addr.s6_addr16[1] == 0)) return (EINVAL); imo = in6p_findmoptions(inp); @@ -2118,13 +2111,13 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IM6O_LOCK(imo); - idx = im6o_match_group(imo, ifp, &gsa->sa); + idx = im6o_match_group(imo, ifp, gsa); if (idx == (size_t)-1) { is_new = 1; } else { inm = imo->im6o_membership[idx]; imf = &imo->im6o_mfilters[idx]; - if (ssa->ss.ss_family != AF_UNSPEC) { + if (ssa->sin6_family != AF_UNSPEC) { /* * MCAST_JOIN_SOURCE_GROUP on an exclusive membership * is an error. On an existing inclusive membership, @@ -2150,7 +2143,7 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) * full-state SSM API with the delta-based API, * which is discouraged in the relevant RFCs. */ - lims = im6o_match_source(imo, idx, &ssa->sa); + lims = im6o_match_source(imo, idx, ssa); if (lims != NULL /*&& lims->im6sl_st[1] == MCAST_INCLUDE*/) { error = EADDRNOTAVAIL; @@ -2213,7 +2206,7 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) * XXX: Should check for non-NULL lims (node exists but may * not be in-mode) for interop with full-state API. */ - if (ssa->ss.ss_family != AF_UNSPEC) { + if (ssa->sin6_family != AF_UNSPEC) { /* Membership starts in IN mode */ if (is_new) { MLD_PRINTF(("%s: new join w/source\n", __func__); @@ -2221,7 +2214,7 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) } else { MLD_PRINTF(("%s: %s source\n", __func__, "allow")); } - lims = im6f_graft(imf, MCAST_INCLUDE, &ssa->sin6); + lims = im6f_graft(imf, MCAST_INCLUDE, ssa); if (lims == NULL) { MLD_PRINTF(("%s: merge imf state failed\n", __func__)); @@ -2249,7 +2242,7 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) socket_unlock(inp->inp_socket, 0); VERIFY(inm == NULL); - error = in6_mc_join(ifp, &gsa->sin6.sin6_addr, imf, &inm, 0); + error = in6_mc_join(ifp, &gsa->sin6_addr, imf, &inm, 0); VERIFY(inm != NULL || error != 0); socket_lock(inp->inp_socket, 0); @@ -2315,7 +2308,7 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) { struct ipv6_mreq mreq; struct group_source_req gsr; - sockunion_t *gsa, *ssa; + struct sockaddr_in6 *gsa, *ssa; struct ifnet *ifp; struct in6_mfilter *imf; struct ip6_moptions *imo; @@ -2332,10 +2325,8 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) is_final = 1; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - gsa->ss.ss_family = AF_UNSPEC; - ssa = (sockunion_t *)&gsr.gsr_source; - ssa->ss.ss_family = AF_UNSPEC; + gsa = (struct sockaddr_in6 *)&gsr.gsr_group; + ssa = (struct sockaddr_in6 *)&gsr.gsr_source; /* * Chew everything passed in up into a struct group_source_req @@ -2346,7 +2337,6 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) */ switch (sopt->sopt_name) { case IPV6_LEAVE_GROUP: { - struct sockaddr_in6 *gsin6; error = sooptcopyin(sopt, &mreq, sizeof(struct ipv6_mreq), sizeof(struct ipv6_mreq)); @@ -2373,15 +2363,14 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) return (inp_leave_group(inp, &v4sopt)); } - gsa->sin6.sin6_family = AF_INET6; - gsa->sin6.sin6_len = sizeof(struct sockaddr_in6); - gsa->sin6.sin6_addr = mreq.ipv6mr_multiaddr; - gsa->sin6.sin6_port = 0; - gsa->sin6.sin6_scope_id = 0; + gsa->sin6_family = AF_INET6; + gsa->sin6_len = sizeof(struct sockaddr_in6); + gsa->sin6_addr = mreq.ipv6mr_multiaddr; + gsa->sin6_port = 0; + gsa->sin6_scope_id = 0; ifindex = mreq.ipv6mr_interface; - gsin6 = &gsa->sin6; /* Only allow IPv6 multicast addresses */ - if (IN6_IS_ADDR_MULTICAST(&gsin6->sin6_addr) == 0) { + if (IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr) == 0) { return (EINVAL); } break; @@ -2401,24 +2390,24 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin6.sin6_family != AF_INET6 || - gsa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (gsa->sin6_family != AF_INET6 || + gsa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); if (sopt->sopt_name == MCAST_LEAVE_SOURCE_GROUP) { - if (ssa->sin6.sin6_family != AF_INET6 || - ssa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (ssa->sin6_family != AF_INET6 || + ssa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); - if (IN6_IS_ADDR_MULTICAST(&ssa->sin6.sin6_addr)) + if (IN6_IS_ADDR_MULTICAST(&ssa->sin6_addr)) return (EINVAL); /* * TODO: Validate embedded scope ID in source * list entry against passed-in ifp, if and only * if source list filter entry is iface or node local. */ - in6_clearscope(&ssa->sin6.sin6_addr); + in6_clearscope(&ssa->sin6_addr); } - gsa->sin6.sin6_port = 0; - gsa->sin6.sin6_scope_id = 0; + gsa->sin6_port = 0; + gsa->sin6_scope_id = 0; ifindex = gsr.gsr_interface; break; @@ -2428,7 +2417,7 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) return (EOPNOTSUPP); } - if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6.sin6_addr)) + if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr)) return (EINVAL); /* @@ -2448,9 +2437,9 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) ifnet_head_done(); if (ifp == NULL) return (EADDRNOTAVAIL); - (void) in6_setscope(&gsa->sin6.sin6_addr, ifp, NULL); + (void) in6_setscope(&gsa->sin6_addr, ifp, NULL); } else { - error = sa6_embedscope(&gsa->sin6, ip6_use_defzone); + error = sa6_embedscope(gsa, ip6_use_defzone); if (error) return (EADDRNOTAVAIL); /* @@ -2463,12 +2452,12 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) * directly until such time as this implementation is * refactored, assuming the scope IDs are the way to go. */ - ifindex = ntohs(gsa->sin6.sin6_addr.s6_addr16[1]); + ifindex = ntohs(gsa->sin6_addr.s6_addr16[1]); if (ifindex == 0) { MLD_PRINTF(("%s: warning: no ifindex, looking up " "ifp for group %s.\n", __func__, - ip6_sprintf(&gsa->sin6.sin6_addr))); - ifp = in6p_lookup_mcast_ifp(inp, &gsa->sin6); + ip6_sprintf(&gsa->sin6_addr))); + ifp = in6p_lookup_mcast_ifp(inp, gsa); } else { if (!IF_INDEX_IN_RANGE(ifindex)) return (EADDRNOTAVAIL); @@ -2492,7 +2481,7 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IM6O_LOCK(imo); - idx = im6o_match_group(imo, ifp, &gsa->sa); + idx = im6o_match_group(imo, ifp, gsa); if (idx == (size_t)-1) { error = EADDRNOTAVAIL; goto out_locked; @@ -2500,7 +2489,7 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) inm = imo->im6o_membership[idx]; imf = &imo->im6o_mfilters[idx]; - if (ssa->ss.ss_family != AF_UNSPEC) + if (ssa->sin6_family != AF_UNSPEC) is_final = 0; /* @@ -2518,16 +2507,16 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) error = EADDRNOTAVAIL; goto out_locked; } - ims = im6o_match_source(imo, idx, &ssa->sa); + ims = im6o_match_source(imo, idx, ssa); if (ims == NULL) { MLD_PRINTF(("%s: source %s %spresent\n", __func__, - ip6_sprintf(&ssa->sin6.sin6_addr), + ip6_sprintf(&ssa->sin6_addr), "not ")); error = EADDRNOTAVAIL; goto out_locked; } MLD_PRINTF(("%s: %s source\n", __func__, "block")); - error = im6f_prune(imf, &ssa->sin6); + error = im6f_prune(imf, ssa); if (error) { MLD_PRINTF(("%s: merge imf state failed\n", __func__)); @@ -2663,7 +2652,7 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt) { struct __msfilterreq64 msfr, msfr64; struct __msfilterreq32 msfr32; - sockunion_t *gsa; + struct sockaddr_in6 *gsa; struct ifnet *ifp; struct in6_mfilter *imf; struct ip6_moptions *imo; @@ -2708,11 +2697,11 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt) msfr.msfr_group.ss_len != sizeof(struct sockaddr_in6)) return (EINVAL); - gsa = (sockunion_t *)&msfr.msfr_group; - if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6.sin6_addr)) + gsa = (struct sockaddr_in6 *)&msfr.msfr_group; + if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr)) return (EINVAL); - gsa->sin6.sin6_port = 0; /* ignore port */ + gsa->sin6_port = 0; /* ignore port */ ifnet_head_lock_shared(); if (msfr.msfr_ifindex == 0 || (u_int)if_index < msfr.msfr_ifindex) { @@ -2724,7 +2713,7 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt) if (ifp == NULL) return (EADDRNOTAVAIL); - (void)in6_setscope(&gsa->sin6.sin6_addr, ifp, NULL); + (void)in6_setscope(&gsa->sin6_addr, ifp, NULL); /* * Take the INP write lock. @@ -2735,7 +2724,7 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IM6O_LOCK(imo); - idx = im6o_match_group(imo, ifp, &gsa->sa); + idx = im6o_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->im6o_mfilters == NULL) { error = EADDRNOTAVAIL; goto out_imo_locked; @@ -3430,7 +3419,10 @@ in6_multihead_lock_shared(void) void in6_multihead_lock_assert(int what) { - lck_rw_assert(&in6_multihead_lock, what); +#if !MACH_ASSERT +#pragma unused(what) +#endif + LCK_RW_ASSERT(&in6_multihead_lock, what); } void diff --git a/bsd/netinet6/in6_pcb.c b/bsd/netinet6/in6_pcb.c index 54b9555f6..29cbedd50 100644 --- a/bsd/netinet6/in6_pcb.c +++ b/bsd/netinet6/in6_pcb.c @@ -163,7 +163,7 @@ in6_pcblookup_local_and_cleanup(struct inpcbinfo *pcbinfo, if (inp != NULL && inp->inp_wantcnt == WNT_STOPUSING) { struct socket *so = inp->inp_socket; - lck_mtx_lock(&inp->inpcb_mtx); + socket_lock(so, 0); if (so->so_usecount == 0) { if (inp->inp_state != INPCB_STATE_DEAD) @@ -171,7 +171,7 @@ in6_pcblookup_local_and_cleanup(struct inpcbinfo *pcbinfo, in_pcbdispose(inp); /* will unlock & destroy */ inp = NULL; } else { - lck_mtx_unlock(&inp->inpcb_mtx); + socket_unlock(so, 0); } } @@ -191,8 +191,10 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) int wild = 0, reuseport = (so->so_options & SO_REUSEPORT); struct ifnet *outif = NULL; struct sockaddr_in6 sin6; +#if !CONFIG_EMBEDDED int error; kauth_cred_t cred; +#endif /* !CONFIG_EMBEDDED */ if (!in6_ifaddrs) /* XXX broken! */ return (EADDRNOTAVAIL); @@ -290,6 +292,7 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) uid_t u; /* GROSS */ +#if !CONFIG_EMBEDDED if (ntohs(lport) < IPV6PORT_RESERVED) { cred = kauth_cred_proc_ref(p); error = priv_check_cred(cred, @@ -301,6 +304,7 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) return (EACCES); } } +#endif /* !CONFIG_EMBEDDED */ if (!IN6_IS_ADDR_MULTICAST(&sin6.sin6_addr) && (u = kauth_cred_getuid(so->so_cred)) != 0) { t = in6_pcblookup_local_and_cleanup(pcbinfo, @@ -525,6 +529,12 @@ in6_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct proc *p) struct ifnet *outif = NULL; struct socket *so = inp->inp_socket; + if (so->so_proto->pr_protocol == IPPROTO_UDP && + sin6->sin6_port == htons(53) && !(so->so_flags1 & SOF1_DNS_COUNTED)) { + so->so_flags1 |= SOF1_DNS_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_dns); + } + /* * Call inner routine, to assign local interface address. * in6_pcbladdr() may automatically fill in sin6_scope_id. @@ -618,13 +628,19 @@ in6_pcbdetach(struct inpcb *inp) inp, so, SOCK_PROTO(so)); /* NOTREACHED */ } - + #if IPSEC if (inp->in6p_sp != NULL) { (void) ipsec6_delete_pcbpolicy(inp); } #endif /* IPSEC */ + if (inp->inp_stat != NULL && SOCK_PROTO(so) == IPPROTO_UDP) { + if (inp->inp_stat->rxpackets == 0 && inp->inp_stat->txpackets == 0) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet6_dgram_no_data); + } + } + /* * Let NetworkStatistics know this PCB is going away * before we detach it. @@ -665,7 +681,7 @@ in6_pcbdetach(struct inpcb *inp) inp->inp_state = INPCB_STATE_DEAD; /* makes sure we're not called twice from so_close */ so->so_flags |= SOF_PCBCLEARING; - + inpcb_gc_sched(inp->inp_pcbinfo, INPCB_TIMER_FAST); /* @@ -752,7 +768,7 @@ in6_getsockaddr(struct socket *so, struct sockaddr **nam) } int -in6_getsockaddr_s(struct socket *so, struct sockaddr_storage *ss) +in6_getsockaddr_s(struct socket *so, struct sockaddr_in6 *ss) { struct inpcb *inp; struct in6_addr addr; @@ -761,17 +777,13 @@ in6_getsockaddr_s(struct socket *so, struct sockaddr_storage *ss) VERIFY(ss != NULL); bzero(ss, sizeof (*ss)); - if ((inp = sotoinpcb(so)) == NULL -#if NECP - || (necp_socket_should_use_flow_divert(inp)) -#endif /* NECP */ - ) - return (inp == NULL ? EINVAL : EPROTOTYPE); + if ((inp = sotoinpcb(so)) == NULL) + return (EINVAL); port = inp->inp_lport; addr = inp->in6p_laddr; - in6_sockaddr_s(port, &addr, SIN6(ss)); + in6_sockaddr_s(port, &addr, ss); return (0); } @@ -794,30 +806,6 @@ in6_getpeeraddr(struct socket *so, struct sockaddr **nam) return (0); } -int -in6_getpeeraddr_s(struct socket *so, struct sockaddr_storage *ss) -{ - struct inpcb *inp; - struct in6_addr addr; - in_port_t port; - - VERIFY(ss != NULL); - bzero(ss, sizeof (*ss)); - - if ((inp = sotoinpcb(so)) == NULL -#if NECP - || (necp_socket_should_use_flow_divert(inp)) -#endif /* NECP */ - ) - return (inp == NULL ? EINVAL : EPROTOTYPE); - - port = inp->inp_fport; - addr = inp->in6p_faddr; - - in6_sockaddr_s(port, &addr, SIN6(ss)); - return (0); -} - int in6_mapped_sockaddr(struct socket *so, struct sockaddr **nam) { @@ -924,13 +912,9 @@ in6_pcbnotify(struct inpcbinfo *pcbinfo, struct sockaddr *dst, u_int fport_arg, * sockets disconnected. * XXX: should we avoid to notify the value to TCP sockets? */ - if (cmd == PRC_MSGSIZE && (inp->inp_flags & IN6P_MTU) != 0 && - (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) || - IN6_ARE_ADDR_EQUAL(&inp->in6p_faddr, - &sa6_dst->sin6_addr))) { + if (cmd == PRC_MSGSIZE) ip6_notify_pmtu(inp, (struct sockaddr_in6 *)(void *)dst, (u_int32_t *)cmdarg); - } /* * Detect if we should notify the error. If no source and @@ -1340,7 +1324,7 @@ in6p_route_copyout(struct inpcb *inp, struct route_in6 *dst) { struct route_in6 *src = &inp->in6p_route; - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(inp->inp_socket); /* Minor sanity check */ if (src->ro_rt != NULL && rt_key(src->ro_rt)->sa_family != AF_INET6) @@ -1354,7 +1338,7 @@ in6p_route_copyin(struct inpcb *inp, struct route_in6 *src) { struct route_in6 *dst = &inp->in6p_route; - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(inp->inp_socket); /* Minor sanity check */ if (src->ro_rt != NULL && rt_key(src->ro_rt)->sa_family != AF_INET6) diff --git a/bsd/netinet6/in6_pcb.h b/bsd/netinet6/in6_pcb.h index 093cef2f0..01973578a 100644 --- a/bsd/netinet6/in6_pcb.h +++ b/bsd/netinet6/in6_pcb.h @@ -114,9 +114,8 @@ extern void in6_rtchange(struct inpcb *, int); extern struct sockaddr *in6_sockaddr(in_port_t port, struct in6_addr *addr_p); extern void in6_sockaddr_s(in_port_t, struct in6_addr *, struct sockaddr_in6 *); extern int in6_getpeeraddr(struct socket *, struct sockaddr **); -extern int in6_getpeeraddr_s(struct socket *, struct sockaddr_storage *); extern int in6_getsockaddr(struct socket *, struct sockaddr **); -extern int in6_getsockaddr_s(struct socket *, struct sockaddr_storage *); +extern int in6_getsockaddr_s(struct socket *, struct sockaddr_in6 *); extern int in6_mapped_sockaddr(struct socket *so, struct sockaddr **nam); extern int in6_mapped_peeraddr(struct socket *so, struct sockaddr **nam); extern int in6_selecthlim(struct in6pcb *, struct ifnet *); diff --git a/bsd/netinet6/in6_proto.c b/bsd/netinet6/in6_proto.c index 3aedbea2f..34f8f754a 100644 --- a/bsd/netinet6/in6_proto.c +++ b/bsd/netinet6/in6_proto.c @@ -569,9 +569,16 @@ SYSCTL_INT(_net_inet6_ip6, IPV6CTL_DEFHLIM, SYSCTL_PROC(_net_inet6_ip6, IPV6CTL_STATS, stats, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, ip6_getstat, "S,ip6stat", ""); + +#if (DEVELOPMENT || DEBUG) +SYSCTL_INT(_net_inet6_ip6, IPV6CTL_ACCEPT_RTADV, + accept_rtadv, CTLFLAG_RW | CTLFLAG_LOCKED, + &ip6_accept_rtadv, 0, ""); +#else SYSCTL_INT(_net_inet6_ip6, IPV6CTL_ACCEPT_RTADV, accept_rtadv, CTLFLAG_RD | CTLFLAG_LOCKED, - &ip6_accept_rtadv, 0, ""); + &ip6_accept_rtadv, 0, ""); +#endif /* (DEVELOPMENT || DEBUG) */ SYSCTL_INT(_net_inet6_ip6, IPV6CTL_KEEPFAITH, keepfaith, CTLFLAG_RD | CTLFLAG_LOCKED, &ip6_keepfaith, 0, ""); SYSCTL_INT(_net_inet6_ip6, IPV6CTL_LOG_INTERVAL, diff --git a/bsd/netinet6/in6_rmx.c b/bsd/netinet6/in6_rmx.c index ff874090c..7647eaf0c 100644 --- a/bsd/netinet6/in6_rmx.c +++ b/bsd/netinet6/in6_rmx.c @@ -143,8 +143,6 @@ static struct radix_node *in6_matroute_args(void *, struct radix_node_head *, static void in6_clsroute(struct radix_node *, struct radix_node_head *); static int in6_rtqkill(struct radix_node *, void *); -#define RTPRF_OURS RTF_PROTO3 /* set on routes we manage */ - /* * Accessed by in6_addroute(), in6_deleteroute() and in6_rtqkill(), during * which the routing lock (rnh_lock) is held and thus protects the variable. @@ -165,7 +163,7 @@ in6_addroute(void *v_arg, void *n_arg, struct radix_node_head *head, uint32_t flags = rt->rt_flags; boolean_t verbose = (rt_verbose > 1); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (verbose) @@ -334,7 +332,7 @@ in6_deleteroute(void *v_arg, void *netmask_arg, struct radix_node_head *head) { struct radix_node *rn; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); rn = rn_delete(v_arg, netmask_arg, head); if (rn != NULL) { @@ -449,11 +447,11 @@ in6_clsroute(struct radix_node *rn, struct radix_node_head *head) struct rtentry *rt = (struct rtentry *)rn; boolean_t verbose = (rt_verbose > 1); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (!(rt->rt_flags & RTF_UP)) - return; /* prophylactic measures */ + return; /* prophylactic measures */ if ((rt->rt_flags & (RTF_LLINFO | RTF_HOST)) != RTF_HOST) return; @@ -517,9 +515,9 @@ in6_clsroute(struct radix_node *rn, struct radix_node_head *head) if (verbose) { log(LOG_DEBUG, "%s: route to %s->%s->%s invalidated, " - "flags=%b, expire=T+%u\n", __func__, dbuf, gbuf, - (rt->rt_ifp != NULL) ? rt->rt_ifp->if_xname : "", - rt->rt_flags, RTF_BITS, rt->rt_expire - timenow); + "flags=%b, expire=T+%u\n", __func__, dbuf, gbuf, + (rt->rt_ifp != NULL) ? rt->rt_ifp->if_xname : "", + rt->rt_flags, RTF_BITS, rt->rt_expire - timenow); } /* We have at least one entry; arm the timer if not already */ @@ -553,7 +551,7 @@ in6_rtqkill(struct radix_node *rn, void *rock) int err; timenow = net_uptime(); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); if (rt->rt_flags & RTPRF_OURS) { @@ -574,6 +572,7 @@ in6_rtqkill(struct radix_node *rn, void *rock) rt, rt->rt_refcnt); /* NOTREACHED */ } + if (verbose) { log(LOG_DEBUG, "%s: deleting route to " "%s->%s->%s, flags=%b, draining=%d\n", @@ -708,7 +707,7 @@ in6_rtqtimo(void *targ) static void in6_sched_rtqtimo(struct timeval *atv) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (!in6_rtqtimo_run) { struct timeval tv; diff --git a/bsd/netinet6/in6_src.c b/bsd/netinet6/in6_src.c index 28e831f86..a987f4250 100644 --- a/bsd/netinet6/in6_src.c +++ b/bsd/netinet6/in6_src.c @@ -143,6 +143,11 @@ SYSCTL_INT(_net_inet6_ip6, OID_AUTO, select_src_expensive_secondary_if, CTLFLAG_RW | CTLFLAG_LOCKED, &ip6_select_src_expensive_secondary_if, 0, "allow source interface selection to use expensive secondaries"); +static int ip6_select_src_strong_end = 1; +SYSCTL_INT(_net_inet6_ip6, OID_AUTO, select_src_strong_end, + CTLFLAG_RW | CTLFLAG_LOCKED, &ip6_select_src_strong_end, 0, + "limit source address selection to outgoing interface"); + #define ADDR_LABEL_NOTAPP (-1) struct in6_addrpolicy defaultaddrpolicy; @@ -181,7 +186,7 @@ void addrsel_policy_init(void); #define SASEL_LOG(fmt, ...) \ do { \ - if (SASEL_DO_DBG(inp)) \ + if (srcsel_debug) \ printf("%s:%d " fmt "\n",\ __FUNCTION__, __LINE__, ##__VA_ARGS__); \ } while (0); \ @@ -212,147 +217,40 @@ do { \ goto out; /* XXX: we can't use 'break' here */ \ } while (0) -/* - * Regardless of error, it will return an ifp with a reference held if the - * caller provides a non-NULL ifpp. The caller is responsible for checking - * if the returned ifp is valid and release its reference at all times. - */ struct in6_addr * -in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, - struct inpcb *inp, struct route_in6 *ro, - struct ifnet **ifpp, struct in6_addr *src_storage, unsigned int ifscope, - int *errorp) +in6_selectsrc_core(struct sockaddr_in6 *dstsock, uint32_t hint_mask, + struct ifnet *ifp, int srcsel_debug, struct in6_addr *src_storage, + struct ifnet **sifp, int *errorp, struct ifaddr **ifapp) { + u_int32_t odstzone; + int bestrule = IP6S_SRCRULE_0; + struct in6_addrpolicy *dst_policy = NULL, *best_policy = NULL; struct in6_addr dst; - struct ifnet *ifp = NULL; struct in6_ifaddr *ia = NULL, *ia_best = NULL; - struct in6_pktinfo *pi = NULL; + char s_src[MAX_IPv6_STR_LEN] = {0}; + char s_dst[MAX_IPv6_STR_LEN] = {0}; + const struct in6_addr *tmp = NULL; int dst_scope = -1, best_scope = -1, best_matchlen = -1; - struct in6_addrpolicy *dst_policy = NULL, *best_policy = NULL; - u_int32_t odstzone; - int prefer_tempaddr; - struct ip6_moptions *mopts; - struct ip6_out_args ip6oa = { ifscope, { 0 }, IP6OAF_SELECT_SRCIF, 0, - SO_TC_UNSPEC, _NET_SERVICE_TYPE_UNSPEC }; - boolean_t islocal = FALSE; uint64_t secs = net_uptime(); - char s_src[MAX_IPv6_STR_LEN], s_dst[MAX_IPv6_STR_LEN]; - const struct in6_addr *tmp; - int bestrule = IP6S_SRCRULE_0; + VERIFY(dstsock != NULL); + VERIFY(src_storage != NULL); + VERIFY(ifp != NULL); - dst = dstsock->sin6_addr; /* make a copy for local operation */ - *errorp = 0; - if (ifpp != NULL) - *ifpp = NULL; + if (sifp != NULL) + *sifp = NULL; - if (inp != NULL) { - mopts = inp->in6p_moptions; - if (INP_NO_CELLULAR(inp)) - ip6oa.ip6oa_flags |= IP6OAF_NO_CELLULAR; - if (INP_NO_EXPENSIVE(inp)) - ip6oa.ip6oa_flags |= IP6OAF_NO_EXPENSIVE; - if (INP_AWDL_UNRESTRICTED(inp)) - ip6oa.ip6oa_flags |= IP6OAF_AWDL_UNRESTRICTED; - if (INP_INTCOPROC_ALLOWED(inp)) - ip6oa.ip6oa_flags |= IP6OAF_INTCOPROC_ALLOWED; - } else { - mopts = NULL; - /* Allow the kernel to retransmit packets. */ - ip6oa.ip6oa_flags |= IP6OAF_INTCOPROC_ALLOWED | - IP6OAF_AWDL_UNRESTRICTED; - } - - if (ip6oa.ip6oa_boundif != IFSCOPE_NONE) - ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF; - - /* - * If the source address is explicitly specified by the caller, - * check if the requested source address is indeed a unicast address - * assigned to the node, and can be used as the packet's source - * address. If everything is okay, use the address as source. - */ - if (opts && (pi = opts->ip6po_pktinfo) && - !IN6_IS_ADDR_UNSPECIFIED(&pi->ipi6_addr)) { - struct sockaddr_in6 srcsock; - struct in6_ifaddr *ia6; + if (ifapp != NULL) + *ifapp = NULL; - /* get the outgoing interface */ - if ((*errorp = in6_selectif(dstsock, opts, mopts, ro, &ip6oa, - &ifp)) != 0) { - src_storage = NULL; - goto done; - } - - /* - * determine the appropriate zone id of the source based on - * the zone of the destination and the outgoing interface. - * If the specified address is ambiguous wrt the scope zone, - * the interface must be specified; otherwise, ifa_ifwithaddr() - * will fail matching the address. - */ - bzero(&srcsock, sizeof (srcsock)); - srcsock.sin6_family = AF_INET6; - srcsock.sin6_len = sizeof (srcsock); - srcsock.sin6_addr = pi->ipi6_addr; - if (ifp != NULL) { - *errorp = in6_setscope(&srcsock.sin6_addr, ifp, NULL); - if (*errorp != 0) { - src_storage = NULL; - goto done; - } - } - ia6 = (struct in6_ifaddr *)ifa_ifwithaddr((struct sockaddr *) - (&srcsock)); - if (ia6 == NULL) { - *errorp = EADDRNOTAVAIL; - src_storage = NULL; - goto done; - } - IFA_LOCK_SPIN(&ia6->ia_ifa); - if ((ia6->ia6_flags & (IN6_IFF_ANYCAST | IN6_IFF_NOTREADY)) || - (inp && inp_restricted_send(inp, ia6->ia_ifa.ifa_ifp))) { - IFA_UNLOCK(&ia6->ia_ifa); - IFA_REMREF(&ia6->ia_ifa); - *errorp = EHOSTUNREACH; - src_storage = NULL; - goto done; - } - - *src_storage = satosin6(&ia6->ia_addr)->sin6_addr; - IFA_UNLOCK(&ia6->ia_ifa); - IFA_REMREF(&ia6->ia_ifa); - goto done; - } - - /* - * Otherwise, if the socket has already bound the source, just use it. - */ - if (inp != NULL && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) { - src_storage = &inp->in6p_laddr; - goto done; - } - - /* - * If the address is not specified, choose the best one based on - * the outgoing interface and the destination address. - */ - - /* get the outgoing interface */ - if ((*errorp = in6_selectif(dstsock, opts, mopts, ro, &ip6oa, - &ifp)) != 0) { - src_storage = NULL; - goto done; - } + dst = dstsock->sin6_addr; /* make a copy for local operation */ - if (SASEL_DO_DBG(inp)) { + if (srcsel_debug) { (void) inet_ntop(AF_INET6, &dst, s_dst, sizeof (s_src)); tmp = &in6addr_any; (void) inet_ntop(AF_INET6, tmp, s_src, sizeof (s_src)); - - printf("%s out src %s dst %s ifscope %d ifp %s\n", - __func__, s_src, s_dst, ifscope, - ifp ? ifp->if_xname : "NULL"); + printf("%s out src %s dst %s ifp %s\n", + __func__, s_src, s_dst, ifp->if_xname); } *errorp = in6_setscope(&dst, ifp, &odstzone); @@ -360,24 +258,37 @@ in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, src_storage = NULL; goto done; } - lck_rw_lock_shared(&in6_ifaddr_rwlock); + lck_rw_lock_shared(&in6_ifaddr_rwlock); for (ia = in6_ifaddrs; ia; ia = ia->ia_next) { int new_scope = -1, new_matchlen = -1; struct in6_addrpolicy *new_policy = NULL; - u_int32_t srczone, osrczone, dstzone; + u_int32_t srczone = 0, osrczone, dstzone; struct in6_addr src; struct ifnet *ifp1 = ia->ia_ifp; int srcrule; - if (SASEL_DO_DBG(inp)) + if (srcsel_debug) (void) inet_ntop(AF_INET6, &ia->ia_addr.sin6_addr, - s_src, sizeof (s_src)); + s_src, sizeof (s_src)); IFA_LOCK(&ia->ia_ifa); + + /* + * XXX By default we are strong end system and will + * limit candidate set of source address to the ones + * configured on the outgoing interface. + */ + if (ip6_select_src_strong_end && + ifp1 != ifp) { + SASEL_LOG("NEXT ia %s ifp1 %s address is not on outgoing " + "interface \n", s_src, ifp1->if_xname); + goto next; + } + /* * We'll never take an address that breaks the scope zone - * of the destination. We also skip an address if its zone + * of the destination. We also skip an address if its zone * does not contain the outgoing interface. * XXX: we should probably use sin6_scope_id here. */ @@ -463,53 +374,17 @@ in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, */ /* Rule 5: Prefer outgoing interface */ - if (ia_best->ia_ifp == ifp && ia->ia_ifp != ifp) - NEXTSRC(IP6S_SRCRULE_5); - if (ia_best->ia_ifp != ifp && ia->ia_ifp == ifp) - REPLACE(IP6S_SRCRULE_5); - - /* Rule 5.5: Prefer addresses in a prefix advertised by the next hop. */ - if (ro != NULL && ro->ro_rt != NULL && ia_best->ia6_ndpr != NULL && - ia->ia6_ndpr != NULL) { - struct rtentry *rta, *rtb; - int op; - - NDPR_LOCK(ia_best->ia6_ndpr); - rta = ia_best->ia6_ndpr->ndpr_rt; - if (rta != NULL) - RT_ADDREF(rta); - NDPR_UNLOCK(ia_best->ia6_ndpr); - - NDPR_LOCK(ia->ia6_ndpr); - rtb = ia->ia6_ndpr->ndpr_rt; - if (rtb != NULL) - RT_ADDREF(rtb); - NDPR_UNLOCK(ia->ia6_ndpr); - - if (rta == NULL || rtb == NULL) - op = 0; - else if (rta == ro->ro_rt && rtb != ro->ro_rt) - op = 1; - else if (rta != ro->ro_rt && rtb == ro->ro_rt) - op = 2; - else - op = 0; - - if (rta != NULL) - RT_REMREF(rta); - if (rtb != NULL) - RT_REMREF(rtb); - - switch (op) { - case 1: - NEXTSRC(IP6S_SRCRULE_5_5); - break; - case 2: - REPLACE(IP6S_SRCRULE_5_5); - break; - default: - break; - } + /* + * XXX By default we are strong end with source address + * selection. That means all address selection candidate + * addresses will be the ones hosted on the outgoing interface + * making the following check redundant. + */ + if (ip6_select_src_strong_end == 0) { + if (ia_best->ia_ifp == ifp && ia->ia_ifp != ifp) + NEXTSRC(IP6S_SRCRULE_5); + if (ia_best->ia_ifp != ifp && ia->ia_ifp == ifp) + REPLACE(IP6S_SRCRULE_5); } /* @@ -533,31 +408,17 @@ in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, * We allow users to reverse the logic by configuring * a sysctl variable, so that transparency conscious users can * always prefer stable addresses. - * Don't use temporary addresses for local destinations or - * for multicast addresses unless we were passed in an option. */ - if (IN6_IS_ADDR_MULTICAST(&dst) || - in6_matchlen(&ia_best->ia_addr.sin6_addr, &dst) >= - ia_best->ia_plen) - islocal = TRUE; - if (opts == NULL || - opts->ip6po_prefer_tempaddr == IP6PO_TEMPADDR_SYSTEM) { - prefer_tempaddr = islocal ? 0 : ip6_prefer_tempaddr; - } else if (opts->ip6po_prefer_tempaddr == - IP6PO_TEMPADDR_NOTPREFER) { - prefer_tempaddr = 0; - } else - prefer_tempaddr = 1; if (!(ia_best->ia6_flags & IN6_IFF_TEMPORARY) && (ia->ia6_flags & IN6_IFF_TEMPORARY)) { - if (prefer_tempaddr) + if (hint_mask & IPV6_SRCSEL_HINT_PREFER_TMPADDR) REPLACE(IP6S_SRCRULE_7); else NEXTSRC(IP6S_SRCRULE_7); } if ((ia_best->ia6_flags & IN6_IFF_TEMPORARY) && !(ia->ia6_flags & IN6_IFF_TEMPORARY)) { - if (prefer_tempaddr) + if (hint_mask & IPV6_SRCSEL_HINT_PREFER_TMPADDR) NEXTSRC(IP6S_SRCRULE_7); else REPLACE(IP6S_SRCRULE_7); @@ -591,7 +452,7 @@ in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, ip6_select_src_expensive_secondary_if == 0) { SASEL_LOG("NEXT ia %s ifp1 %s IFEF_EXPENSIVE\n", s_src, ifp1->if_xname); - ip6stat.ip6s_sources_skip_expensive_secondary_if++; + ip6stat.ip6s_sources_skip_expensive_secondary_if++; goto next; } SASEL_LOG("NEXT ia %s ifp1 %s last resort\n", @@ -608,7 +469,7 @@ replace: ip6_select_src_expensive_secondary_if == 0) { SASEL_LOG("NEXT ia %s ifp1 %s IFEF_EXPENSIVE\n", s_src, ifp1->if_xname); - ip6stat.ip6s_sources_skip_expensive_secondary_if++; + ip6stat.ip6s_sources_skip_expensive_secondary_if++; goto next; } bestrule = srcrule; @@ -620,7 +481,7 @@ replace: in6_matchlen(&ia->ia_addr.sin6_addr, &dst)); SASEL_LOG("NEXT ia %s ifp1 %s best_scope %d new_scope %d dst_scope %d\n", s_src, ifp1->if_xname, best_scope, new_scope, dst_scope); - IFA_ADDREF_LOCKED(&ia->ia_ifa); /* for ia_best */ + IFA_ADDREF_LOCKED(&ia->ia_ifa); /* for ia_best */ IFA_UNLOCK(&ia->ia_ifa); if (ia_best != NULL) IFA_REMREF(&ia_best->ia_ifa); @@ -632,7 +493,7 @@ next: continue; out: - IFA_ADDREF_LOCKED(&ia->ia_ifa); /* for ia_best */ + IFA_ADDREF_LOCKED(&ia->ia_ifa); /* for ia_best */ IFA_UNLOCK(&ia->ia_ifa); if (ia_best != NULL) IFA_REMREF(&ia_best->ia_ifa); @@ -642,13 +503,6 @@ out: lck_rw_done(&in6_ifaddr_rwlock); - if (ia_best != NULL && inp && - inp_restricted_send(inp, ia_best->ia_ifa.ifa_ifp)) { - IFA_REMREF(&ia_best->ia_ifa); - ia_best = NULL; - *errorp = EHOSTUNREACH; - } - if ((ia = ia_best) == NULL) { if (*errorp == 0) *errorp = EADDRNOTAVAIL; @@ -656,22 +510,191 @@ out: goto done; } + if (sifp != NULL) { + *sifp = ia->ia_ifa.ifa_ifp; + ifnet_reference(*sifp); + } + IFA_LOCK_SPIN(&ia->ia_ifa); if (bestrule < IP6S_SRCRULE_COUNT) ip6stat.ip6s_sources_rule[bestrule]++; *src_storage = satosin6(&ia->ia_addr)->sin6_addr; IFA_UNLOCK(&ia->ia_ifa); - IFA_REMREF(&ia->ia_ifa); + + if (ifapp != NULL) + *ifapp = &ia->ia_ifa; + else + IFA_REMREF(&ia->ia_ifa); + done: - if (SASEL_DO_DBG(inp)) { + if (srcsel_debug) { (void) inet_ntop(AF_INET6, &dst, s_dst, sizeof (s_src)); tmp = (src_storage != NULL) ? src_storage : &in6addr_any; (void) inet_ntop(AF_INET6, tmp, s_src, sizeof (s_src)); - printf("%s out src %s dst %s ifscope %d dst_scope %d best_scope %d\n", - __func__, s_src, s_dst, ifscope, dst_scope, best_scope); + printf("%s out src %s dst %s dst_scope %d best_scope %d\n", + __func__, s_src, s_dst, dst_scope, best_scope); } + + return (src_storage); +} + +/* + * Regardless of error, it will return an ifp with a reference held if the + * caller provides a non-NULL ifpp. The caller is responsible for checking + * if the returned ifp is valid and release its reference at all times. + */ +struct in6_addr * +in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, + struct inpcb *inp, struct route_in6 *ro, + struct ifnet **ifpp, struct in6_addr *src_storage, unsigned int ifscope, + int *errorp) +{ + struct ifnet *ifp = NULL; + struct in6_pktinfo *pi = NULL; + struct ip6_moptions *mopts; + struct ip6_out_args ip6oa = { ifscope, { 0 }, IP6OAF_SELECT_SRCIF, 0, + SO_TC_UNSPEC, _NET_SERVICE_TYPE_UNSPEC }; + boolean_t inp_debug = FALSE; + uint32_t hint_mask = 0; + int prefer_tempaddr = 0; + struct ifnet *sifp = NULL; + + *errorp = 0; + if (ifpp != NULL) + *ifpp = NULL; + + if (inp != NULL) { + inp_debug = SASEL_DO_DBG(inp); + mopts = inp->in6p_moptions; + if (INP_NO_CELLULAR(inp)) + ip6oa.ip6oa_flags |= IP6OAF_NO_CELLULAR; + if (INP_NO_EXPENSIVE(inp)) + ip6oa.ip6oa_flags |= IP6OAF_NO_EXPENSIVE; + if (INP_AWDL_UNRESTRICTED(inp)) + ip6oa.ip6oa_flags |= IP6OAF_AWDL_UNRESTRICTED; + if (INP_INTCOPROC_ALLOWED(inp)) + ip6oa.ip6oa_flags |= IP6OAF_INTCOPROC_ALLOWED; + } else { + mopts = NULL; + /* Allow the kernel to retransmit packets. */ + ip6oa.ip6oa_flags |= IP6OAF_INTCOPROC_ALLOWED | + IP6OAF_AWDL_UNRESTRICTED; + } + + if (ip6oa.ip6oa_boundif != IFSCOPE_NONE) + ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF; + + /* + * If the source address is explicitly specified by the caller, + * check if the requested source address is indeed a unicast address + * assigned to the node, and can be used as the packet's source + * address. If everything is okay, use the address as source. + */ + if (opts && (pi = opts->ip6po_pktinfo) && + !IN6_IS_ADDR_UNSPECIFIED(&pi->ipi6_addr)) { + struct sockaddr_in6 srcsock; + struct in6_ifaddr *ia6; + + /* get the outgoing interface */ + if ((*errorp = in6_selectif(dstsock, opts, mopts, ro, &ip6oa, + &ifp)) != 0) { + src_storage = NULL; + goto done; + } + + /* + * determine the appropriate zone id of the source based on + * the zone of the destination and the outgoing interface. + * If the specified address is ambiguous wrt the scope zone, + * the interface must be specified; otherwise, ifa_ifwithaddr() + * will fail matching the address. + */ + bzero(&srcsock, sizeof (srcsock)); + srcsock.sin6_family = AF_INET6; + srcsock.sin6_len = sizeof (srcsock); + srcsock.sin6_addr = pi->ipi6_addr; + if (ifp != NULL) { + *errorp = in6_setscope(&srcsock.sin6_addr, ifp, NULL); + if (*errorp != 0) { + src_storage = NULL; + goto done; + } + } + ia6 = (struct in6_ifaddr *)ifa_ifwithaddr((struct sockaddr *) + (&srcsock)); + if (ia6 == NULL) { + *errorp = EADDRNOTAVAIL; + src_storage = NULL; + goto done; + } + IFA_LOCK_SPIN(&ia6->ia_ifa); + if ((ia6->ia6_flags & (IN6_IFF_ANYCAST | IN6_IFF_NOTREADY)) || + (inp && inp_restricted_send(inp, ia6->ia_ifa.ifa_ifp))) { + IFA_UNLOCK(&ia6->ia_ifa); + IFA_REMREF(&ia6->ia_ifa); + *errorp = EHOSTUNREACH; + src_storage = NULL; + goto done; + } + + *src_storage = satosin6(&ia6->ia_addr)->sin6_addr; + IFA_UNLOCK(&ia6->ia_ifa); + IFA_REMREF(&ia6->ia_ifa); + goto done; + } + + /* + * Otherwise, if the socket has already bound the source, just use it. + */ + if (inp != NULL && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) { + src_storage = &inp->in6p_laddr; + goto done; + } + + /* + * If the address is not specified, choose the best one based on + * the outgoing interface and the destination address. + */ + /* get the outgoing interface */ + if ((*errorp = in6_selectif(dstsock, opts, mopts, ro, &ip6oa, + &ifp)) != 0) { + src_storage = NULL; + goto done; + } + + VERIFY(ifp != NULL); + + if (opts == NULL || + opts->ip6po_prefer_tempaddr == IP6PO_TEMPADDR_SYSTEM) { + prefer_tempaddr = ip6_prefer_tempaddr; + } else if (opts->ip6po_prefer_tempaddr == IP6PO_TEMPADDR_NOTPREFER) { + prefer_tempaddr = 0; + } else + prefer_tempaddr = 1; + + if (prefer_tempaddr) + hint_mask |= IPV6_SRCSEL_HINT_PREFER_TMPADDR; + + if (in6_selectsrc_core(dstsock, hint_mask, ifp, inp_debug, src_storage, + &sifp, errorp, NULL) == NULL) { + src_storage = NULL; + goto done; + } + + VERIFY(sifp != NULL); + + if (inp && inp_restricted_send(inp, sifp)) { + src_storage = NULL; + *errorp = EHOSTUNREACH; + ifnet_release(sifp); + goto done; + } else { + ifnet_release(sifp); + } + +done: if (ifpp != NULL) { /* if ifp is non-NULL, refcnt held in in6_selectif() */ *ifpp = ifp; @@ -716,21 +739,6 @@ selectroute(struct sockaddr_in6 *srcsock, struct sockaddr_in6 *dstsock, unsigned int ifscope = ((ip6oa != NULL) ? ip6oa->ip6oa_boundif : IFSCOPE_NONE); -#if 0 - char ip6buf[INET6_ADDRSTRLEN]; - - if (dstsock->sin6_addr.s6_addr32[0] == 0 && - dstsock->sin6_addr.s6_addr32[1] == 0 && - !IN6_IS_ADDR_LOOPBACK(&dstsock->sin6_addr)) { - printf("in6_selectroute: strange destination %s\n", - ip6_sprintf(ip6buf, &dstsock->sin6_addr)); - } else { - printf("in6_selectroute: destination = %s%%%d\n", - ip6_sprintf(ip6buf, &dstsock->sin6_addr), - dstsock->sin6_scope_id); /* for debug */ - } -#endif - if (retifp != NULL) *retifp = NULL; @@ -1361,6 +1369,7 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct proc *p, struct socket *so = inp->inp_socket; u_int16_t lport = 0, first, last, *lastport; int count, error = 0, wild = 0; + bool found; struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; kauth_cred_t cred; if (!locked) { /* Make sure we don't run into a deadlock: 4052373 */ @@ -1423,6 +1432,7 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct proc *p, * counting down */ count = first - last; + found = false; do { if (count-- < 0) { /* completely used? */ @@ -1440,11 +1450,14 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct proc *p, if (*lastport > first || *lastport < last) *lastport = first; lport = htons(*lastport); - } while (in6_pcblookup_local(pcbinfo, &inp->in6p_laddr, lport, - wild)); + + found = in6_pcblookup_local(pcbinfo, &inp->in6p_laddr, + lport, wild) == NULL; + } while (!found); } else { /* counting up */ count = last - first; + found = false; do { if (count-- < 0) { /* completely used? */ @@ -1462,8 +1475,10 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct proc *p, if (*lastport < first || *lastport > last) *lastport = first; lport = htons(*lastport); - } while (in6_pcblookup_local(pcbinfo, &inp->in6p_laddr, lport, - wild)); + + found = in6_pcblookup_local(pcbinfo, &inp->in6p_laddr, + lport, wild) == NULL; + } while (!found); } inp->inp_lport = lport; diff --git a/bsd/netinet6/in6_var.h b/bsd/netinet6/in6_var.h index aee032c49..8a08baa85 100644 --- a/bsd/netinet6/in6_var.h +++ b/bsd/netinet6/in6_var.h @@ -227,6 +227,11 @@ struct in6_ifstat { /* NOTE: increment on final dst if */ u_quad_t ifs6_in_mcast; /* # of inbound multicast datagrams */ u_quad_t ifs6_out_mcast; /* # of outbound multicast datagrams */ + + u_quad_t ifs6_cantfoward_icmp6; /* # of ICMPv6 packets received for unreachable dest */ + u_quad_t ifs6_addr_expiry_cnt; /* # of address expiry events (excluding privacy addresses) */ + u_quad_t ifs6_pfx_expiry_cnt; /* # of prefix expiry events */ + u_quad_t ifs6_defrtr_expiry_cnt; /* # of default router expiry events */ }; /* @@ -865,9 +870,11 @@ struct in6_multi_mship { #ifdef BSD_KERNEL_PRIVATE #include +#include + /* - * * Per-interface IPv6 structures. - * */ + * Per-interface IPv6 structures. + */ struct in6_ifextra { struct scope6_id scope6_id; struct in6_ifstat in6_ifstat; @@ -875,8 +882,11 @@ struct in6_ifextra { struct nd_ifinfo nd_ifinfo; uint32_t netsig_len; u_int8_t netsig[IFNET_SIGNATURELEN]; + struct ipv6_prefix nat64_prefixes[NAT64_MAX_NUM_PREFIXES]; + struct lltable *ii_llt; /* NDP state */ }; #define IN6_IFEXTRA(_ifp) ((struct in6_ifextra *)(_ifp->if_inet6data)) +#define LLTABLE6(ifp) ((IN6_IFEXTRA(ifp) == NULL) ? NULL : IN6_IFEXTRA(ifp)->ii_llt) #endif /* BSD_KERNEL_PRIVATE */ struct mld_ifinfo; @@ -949,10 +959,10 @@ struct in6_multi { }; #define IN6M_LOCK_ASSERT_HELD(_in6m) \ - lck_mtx_assert(&(_in6m)->in6m_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_in6m)->in6m_lock, LCK_MTX_ASSERT_OWNED) #define IN6M_LOCK_ASSERT_NOTHELD(_in6m) \ - lck_mtx_assert(&(_in6m)->in6m_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_in6m)->in6m_lock, LCK_MTX_ASSERT_NOTOWNED) #define IN6M_LOCK(_in6m) \ lck_mtx_lock(&(_in6m)->in6m_lock) @@ -1070,7 +1080,7 @@ struct ip6_pktopts; /* Multicast private KPIs. */ extern int im6o_mc_filter(const struct ip6_moptions *, const struct ifnet *, - const struct sockaddr *, const struct sockaddr *); + const struct sockaddr_in6 *, const struct sockaddr_in6 *); extern int in6_mc_join(struct ifnet *, const struct in6_addr *, struct in6_mfilter *, struct in6_multi **, int); extern int in6_mc_leave(struct in6_multi *, struct in6_mfilter *); @@ -1152,6 +1162,11 @@ extern ssize_t in6_cga_parameters_prepare(void *, size_t, const struct in6_addr *, u_int8_t, const struct in6_cga_modifier *); extern int in6_cga_generate(struct in6_cga_prepare *, u_int8_t, struct in6_addr *); +extern int in6_getconninfo(struct socket *, sae_connid_t, uint32_t *, + uint32_t *, int32_t *, user_addr_t, socklen_t *, + user_addr_t, socklen_t *, uint32_t *, user_addr_t, uint32_t *); +extern void in6_ip6_to_sockaddr(const struct in6_addr *ip6, u_int16_t port, + struct sockaddr_in6 *sin6, u_int32_t maxlen); #endif /* BSD_KERNEL_PRIVATE */ #endif /* _NETINET6_IN6_VAR_H_ */ diff --git a/bsd/netinet6/ip6_fw.c b/bsd/netinet6/ip6_fw.c index dfa12ff11..99d3e8c02 100644 --- a/bsd/netinet6/ip6_fw.c +++ b/bsd/netinet6/ip6_fw.c @@ -86,8 +86,6 @@ #endif #include -#include - #include #include #include @@ -908,7 +906,6 @@ add_entry6(struct ip6_fw_head *chainptr, struct ip6_fw *frwl) struct ip6_fw *ftmp = 0; struct ip6_fw_chain *fwc = 0, *fcp, *fcpl = 0; u_short nbr = 0; - int s; fwc = _MALLOC(sizeof *fwc, M_IP6FW, M_WAITOK); ftmp = _MALLOC(sizeof *ftmp, M_IP6FW, M_WAITOK); @@ -925,16 +922,12 @@ add_entry6(struct ip6_fw_head *chainptr, struct ip6_fw *frwl) ftmp->fw_bcnt = 0L; fwc->rule = ftmp; - s = splnet(); - if (!chainptr->lh_first) { LIST_INSERT_HEAD(chainptr, fwc, chain); - splx(s); return(0); } else if (ftmp->fw_number == (u_short)-1) { if (fwc) FREE(fwc, M_IP6FW); if (ftmp) FREE(ftmp, M_IP6FW); - splx(s); dprintf(("%s bad rule number\n", err_prefix)); return (EINVAL); } @@ -967,7 +960,6 @@ add_entry6(struct ip6_fw_head *chainptr, struct ip6_fw *frwl) } bcopy(ftmp, frwl, sizeof(struct ip6_fw)); - splx(s); return (0); } @@ -975,16 +967,12 @@ static int del_entry6(struct ip6_fw_head *chainptr, u_short number) { struct ip6_fw_chain *fcp; - int s; - - s = splnet(); fcp = chainptr->lh_first; if (number != (u_short)-1) { for (; fcp; fcp = fcp->chain.le_next) { if (fcp->rule->fw_number == number) { LIST_REMOVE(fcp, chain); - splx(s); FREE(fcp->rule, M_IP6FW); FREE(fcp, M_IP6FW); return 0; @@ -992,7 +980,6 @@ del_entry6(struct ip6_fw_head *chainptr, u_short number) } } - splx(s); return (EINVAL); } @@ -1000,20 +987,17 @@ static int zero_entry6(struct ip6_fw *frwl) { struct ip6_fw_chain *fcp; - int s; /* * It's possible to insert multiple chain entries with the * same number, so we don't stop after finding the first * match if zeroing a specific entry. */ - s = splnet(); for (fcp = ip6_fw_chain.lh_first; fcp; fcp = fcp->chain.le_next) if (!frwl || frwl->fw_number == 0 || frwl->fw_number == fcp->rule->fw_number) { fcp->rule->fw_bcnt = fcp->rule->fw_pcnt = 0; fcp->rule->timestamp = 0; } - splx(s); if (fw6_verbose) { if (frwl) @@ -1282,7 +1266,6 @@ static int ip6_fw_ctl(struct sockopt *sopt) { int error = 0; - int spl; int valsize; struct ip6_fw rule; int is64user=0; @@ -1336,8 +1319,6 @@ ip6_fw_ctl(struct sockopt *sopt) size_t size = 0; size_t rulesize = 0; - spl = splnet(); - if ( is64user ) rulesize = sizeof(struct ip6_fw_64 ); else @@ -1369,7 +1350,6 @@ ip6_fw_ctl(struct sockopt *sopt) } } - splx(spl); if (buf) { sopt->sopt_valsize = valsize; @@ -1381,7 +1361,6 @@ ip6_fw_ctl(struct sockopt *sopt) } case IPV6_FW_FLUSH: - spl = splnet(); while (ip6_fw_chain.lh_first && ip6_fw_chain.lh_first->rule->fw_number != (u_short)-1) { @@ -1390,7 +1369,6 @@ ip6_fw_ctl(struct sockopt *sopt) FREE(fcp->rule, M_IP6FW); FREE(fcp, M_IP6FW); } - splx(spl); ip6fw_kev_post_msg(KEV_IP6FW_FLUSH); break; diff --git a/bsd/netinet6/ip6_input.c b/bsd/netinet6/ip6_input.c index ec32485cb..b3c2bcfcd 100644 --- a/bsd/netinet6/ip6_input.c +++ b/bsd/netinet6/ip6_input.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -211,6 +211,11 @@ SYSCTL_UINT(_net_inet6_ip6, OID_AUTO, adj_clear_hwcksum, CTLFLAG_RW | CTLFLAG_LOCKED, &ip6_adj_clear_hwcksum, 0, "Invalidate hwcksum info when adjusting length"); +static uint32_t ip6_adj_partial_sum = 1; +SYSCTL_UINT(_net_inet6_ip6, OID_AUTO, adj_partial_sum, + CTLFLAG_RW | CTLFLAG_LOCKED, &ip6_adj_partial_sum, 0, + "Perform partial sum adjustment of trailing bytes at IP layer"); + static int ip6_input_measure = 0; SYSCTL_PROC(_net_inet6_ip6, OID_AUTO, input_perf, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, @@ -305,6 +310,14 @@ ip6_init(struct ip6protosw *pp, struct domain *dp) return; ip6_initialized = 1; + eventhandler_lists_ctxt_init(&in6_evhdlr_ctxt); + (void)EVENTHANDLER_REGISTER(&in6_evhdlr_ctxt, in6_event, + in6_eventhdlr_callback, eventhandler_entry_dummy_arg, + EVENTHANDLER_PRI_ANY); + + for (i = 0; i < IN6_EVENT_MAX; i++) + VERIFY(in6_event2kev_array[i].in6_event_code == i); + pr = pffindproto_locked(PF_INET6, IPPROTO_RAW, SOCK_RAW); if (pr == NULL) { panic("%s: Unable to find [PF_INET6,IPPROTO_RAW,SOCK_RAW]\n", @@ -453,6 +466,78 @@ ip6_init_delayed(void) #endif /* NSTF */ } +static void +ip6_input_adjust(struct mbuf *m, struct ip6_hdr *ip6, uint32_t plen, + struct ifnet *inifp) +{ + boolean_t adjust = TRUE; + uint32_t tot_len = sizeof (*ip6) + plen; + + ASSERT(m_pktlen(m) > tot_len); + + /* + * Invalidate hardware checksum info if ip6_adj_clear_hwcksum + * is set; useful to handle buggy drivers. Note that this + * should not be enabled by default, as we may get here due + * to link-layer padding. + */ + if (ip6_adj_clear_hwcksum && + (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) && + !(inifp->if_flags & IFF_LOOPBACK) && + !(m->m_pkthdr.pkt_flags & PKTF_LOOP)) { + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + m->m_pkthdr.csum_data = 0; + ip6stat.ip6s_adj_hwcsum_clr++; + } + + /* + * If partial checksum information is available, subtract + * out the partial sum of postpended extraneous bytes, and + * update the checksum metadata accordingly. By doing it + * here, the upper layer transport only needs to adjust any + * prepended extraneous bytes (else it will do both.) + */ + if (ip6_adj_partial_sum && + (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + m->m_pkthdr.csum_rx_val = m_adj_sum16(m, + m->m_pkthdr.csum_rx_start, m->m_pkthdr.csum_rx_start, + (tot_len - m->m_pkthdr.csum_rx_start), + m->m_pkthdr.csum_rx_val); + } else if ((m->m_pkthdr.csum_flags & + (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + /* + * If packet has partial checksum info and we decided not + * to subtract the partial sum of postpended extraneous + * bytes here (not the default case), leave that work to + * be handled by the other layers. For now, only TCP, UDP + * layers are capable of dealing with this. For all other + * protocols (including fragments), trim and ditch the + * partial sum as those layers might not implement partial + * checksumming (or adjustment) at all. + */ + if (ip6->ip6_nxt == IPPROTO_TCP || + ip6->ip6_nxt == IPPROTO_UDP) { + adjust = FALSE; + } else { + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + m->m_pkthdr.csum_data = 0; + ip6stat.ip6s_adj_hwcsum_clr++; + } + } + + if (adjust) { + ip6stat.ip6s_adj++; + if (m->m_len == m->m_pkthdr.len) { + m->m_len = tot_len; + m->m_pkthdr.len = tot_len; + } else { + m_adj(m, tot_len - m->m_pkthdr.len); + } + } +} + void ip6_input(struct mbuf *m) { @@ -911,6 +996,14 @@ check_with_pf: if (!ip6_forwarding) { ip6stat.ip6s_cantforward++; in6_ifstat_inc(inifp, ifs6_in_discard); + /* + * Raise a kernel event if the packet received on cellular + * interface is not intended for local host. + * For now limit it to ICMPv6 packets. + */ + if (inifp->if_type == IFT_CELLULAR && + ip6->ip6_nxt == IPPROTO_ICMPV6) + in6_ifstat_inc(inifp, ifs6_cantfoward_icmp6); goto bad; } @@ -1004,29 +1097,7 @@ hbhcheck: goto bad; } if (m->m_pkthdr.len > sizeof (struct ip6_hdr) + plen) { - /* - * Invalidate hardware checksum info if ip6_adj_clear_hwcksum - * is set; useful to handle buggy drivers. Note that this - * should not be enabled by default, as we may get here due - * to link-layer padding. - */ - if (ip6_adj_clear_hwcksum && - (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) && - !(inifp->if_flags & IFF_LOOPBACK) && - !(m->m_pkthdr.pkt_flags & PKTF_LOOP)) { - m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; - m->m_pkthdr.csum_data = 0; - ip6stat.ip6s_adj_hwcsum_clr++; - } - - ip6stat.ip6s_adj++; - if (m->m_len == m->m_pkthdr.len) { - m->m_len = sizeof (struct ip6_hdr) + plen; - m->m_pkthdr.len = sizeof (struct ip6_hdr) + plen; - } else { - m_adj(m, sizeof (struct ip6_hdr) + plen - - m->m_pkthdr.len); - } + ip6_input_adjust(m, ip6, plen, inifp); } /* @@ -1741,6 +1812,9 @@ ip6_notify_pmtu(struct inpcb *in6p, struct sockaddr_in6 *dst, u_int32_t *mtu) so = in6p->inp_socket; + if ((in6p->inp_flags & IN6P_MTU) == 0) + return; + if (mtu == NULL) return; @@ -1751,6 +1825,14 @@ ip6_notify_pmtu(struct inpcb *in6p, struct sockaddr_in6 *dst, u_int32_t *mtu) } #endif + if (IN6_IS_ADDR_UNSPECIFIED(&in6p->in6p_faddr) && + (so->so_proto == NULL || so->so_proto->pr_protocol == IPPROTO_TCP)) + return; + + if (!IN6_IS_ADDR_UNSPECIFIED(&in6p->in6p_faddr) && + !IN6_ARE_ADDR_EQUAL(&in6p->in6p_faddr, &dst->sin6_addr)) + return; + bzero(&mtuctl, sizeof (mtuctl)); /* zero-clear for safety */ mtuctl.ip6m_mtu = *mtu; mtuctl.ip6m_addr = *dst; diff --git a/bsd/netinet6/ip6_output.c b/bsd/netinet6/ip6_output.c index 8096be24f..5e541f14c 100644 --- a/bsd/netinet6/ip6_output.c +++ b/bsd/netinet6/ip6_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -118,6 +118,7 @@ #include #include #include +#include #include #include @@ -181,10 +182,6 @@ static int ip6_splithdr(struct mbuf *, struct ip6_exthdrs *); static void ip6_output_checksum(struct ifnet *, uint32_t, struct mbuf *, int, uint32_t, uint32_t); extern int udp_ctloutput(struct socket *, struct sockopt *); -static int ip6_do_fragmentation(struct mbuf **morig, - uint32_t optlen, struct ifnet *ifp, uint32_t unfragpartlen, - struct ip6_hdr *ip6, struct ip6_exthdrs *exthdrsp, uint32_t mtu, - int nxt0); static int ip6_fragment_packet(struct mbuf **m, struct ip6_pktopts *opt, struct ip6_exthdrs *exthdrsp, struct ifnet *ifp, uint32_t mtu, boolean_t alwaysfrag, uint32_t unfragpartlen, @@ -280,13 +277,13 @@ ip6_output_list(struct mbuf *m0, int packetchain, struct ip6_pktopts *opt, struct mbuf *m, *mprev; struct mbuf *sendchain = NULL, *sendchain_last = NULL; struct mbuf *inputchain = NULL; - int nxt0; + int nxt0 = 0; struct route_in6 *ro_pmtu = NULL; struct rtentry *rt = NULL; - struct sockaddr_in6 *dst, src_sa, dst_sa; + struct sockaddr_in6 *dst = NULL, src_sa, dst_sa; int error = 0; struct in6_ifaddr *ia = NULL, *src_ia = NULL; - u_int32_t mtu; + u_int32_t mtu = 0; boolean_t alwaysfrag = FALSE; u_int32_t optlen = 0, plen = 0, unfragpartlen = 0; struct ip6_rthdr *rh; @@ -573,6 +570,7 @@ loopit: * layer. */ error = EHOSTUNREACH; + ip6stat.ip6s_necp_policy_drop++; goto freehdrs; case NECP_KERNEL_POLICY_RESULT_IP_TUNNEL: { /* @@ -612,6 +610,7 @@ loopit: goto skip_ipsec; } else { error = ENETUNREACH; + ip6stat.ip6s_necp_policy_drop++; goto freehdrs; } } @@ -1175,6 +1174,7 @@ skip_ipsec: /* Catch-all to check if the interface is allowed */ if (!necp_packet_is_allowed_over_interface(m, ifp)) { error = EHOSTUNREACH; + ip6stat.ip6s_necp_policy_drop++; goto bad; } #endif /* NECP */ @@ -1662,8 +1662,19 @@ ip6_fragment_packet(struct mbuf **mptr, struct ip6_pktopts *opt, size_t tlen = m->m_pkthdr.len; boolean_t dontfrag = (opt != NULL && (opt->ip6po_flags & IP6PO_DONTFRAG)); - if (m->m_pkthdr.pkt_flags & PKTF_FORWARDED) + if (m->m_pkthdr.pkt_flags & PKTF_FORWARDED) { dontfrag = TRUE; + /* + * Discard partial sum information if this packet originated + * from another interface; the packet would already have the + * final checksum and we shouldn't recompute it. + */ + if ((m->m_pkthdr.csum_flags & (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + m->m_pkthdr.csum_flags &= ~CSUM_TX_FLAGS; + m->m_pkthdr.csum_data = 0; + } + } if (dontfrag && alwaysfrag) { /* case 4 */ /* conflicting request - can't transmit */ @@ -1720,7 +1731,7 @@ ip6_fragment_packet(struct mbuf **mptr, struct ip6_pktopts *opt, * of fragments is linked into the packet chain where morig existed. Otherwise, * an errno is returned. */ -static int +int ip6_do_fragmentation(struct mbuf **mptr, uint32_t optlen, struct ifnet *ifp, uint32_t unfragpartlen, struct ip6_hdr *ip6, struct ip6_exthdrs *exthdrsp, uint32_t mtu, int nxt0) @@ -2051,7 +2062,8 @@ in6_finalize_cksum(struct mbuf *m, uint32_t hoff, int32_t optlen, ip6_out_cksum_stats(nxt, plen - olen); /* RFC1122 4.1.3.4 */ - if (csum == 0 && (m->m_pkthdr.csum_flags & CSUM_UDPIPV6)) + if (csum == 0 && + (m->m_pkthdr.csum_flags & (CSUM_UDPIPV6|CSUM_ZERO_INVERT))) csum = 0xffff; /* Insert the checksum in the ULP csum field */ @@ -2063,8 +2075,8 @@ in6_finalize_cksum(struct mbuf *m, uint32_t hoff, int32_t optlen, } else { bcopy(&csum, (mtod(m, char *) + offset), sizeof (csum)); } - m->m_pkthdr.csum_flags &= - ~(CSUM_DELAY_IPV6_DATA | CSUM_DATA_VALID | CSUM_PARTIAL); + m->m_pkthdr.csum_flags &= ~(CSUM_DELAY_IPV6_DATA | CSUM_DATA_VALID | + CSUM_PARTIAL | CSUM_ZERO_INVERT); done: return (sw_csum); @@ -2217,6 +2229,10 @@ ip6_getpmtu(struct route_in6 *ro_pmtu, struct route_in6 *ro, u_int32_t mtu = 0; boolean_t alwaysfrag = FALSE; int error = 0; + boolean_t is_local = FALSE; + + if (IN6_IS_SCOPE_LINKLOCAL(dst)) + is_local = TRUE; if (ro_pmtu != ro) { /* The first hop and the final destination may differ. */ @@ -2287,7 +2303,7 @@ ip6_getpmtu(struct route_in6 *ro_pmtu, struct route_in6 *ro, } *mtup = mtu; - if (alwaysfragp != NULL) + if ((alwaysfragp != NULL) && !is_local) *alwaysfragp = alwaysfrag; return (error); } @@ -2319,6 +2335,7 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) privileged = (proc_suser(p) == 0); if (level == IPPROTO_IPV6) { + boolean_t capture_exthdrstat_in = FALSE; switch (op) { case SOPT_SET: switch (optname) { @@ -2445,6 +2462,7 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) break; } OPTSET(IN6P_HOPOPTS); + capture_exthdrstat_in = TRUE; break; case IPV6_RECVDSTOPTS: @@ -2454,6 +2472,7 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) break; } OPTSET(IN6P_DSTOPTS); + capture_exthdrstat_in = TRUE; break; case IPV6_RECVRTHDRDSTOPTS: @@ -2463,6 +2482,7 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) break; } OPTSET(IN6P_RTHDRDSTOPTS); + capture_exthdrstat_in = TRUE; break; case IPV6_RECVRTHDR: @@ -2472,6 +2492,7 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) break; } OPTSET(IN6P_RTHDR); + capture_exthdrstat_in = TRUE; break; case IPV6_RECVPATHMTU: @@ -2566,15 +2587,18 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) if (!privileged) return (EPERM); OPTSET2292(IN6P_HOPOPTS); + capture_exthdrstat_in = TRUE; break; case IPV6_2292DSTOPTS: if (!privileged) return (EPERM); OPTSET2292(IN6P_DSTOPTS| IN6P_RTHDRDSTOPTS); /* XXX */ + capture_exthdrstat_in = TRUE; break; case IPV6_2292RTHDR: OPTSET2292(IN6P_RTHDR); + capture_exthdrstat_in = TRUE; break; } break; @@ -2721,6 +2745,13 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) error = ENOPROTOOPT; break; } + if (capture_exthdrstat_in) { + if (uproto == IPPROTO_TCP) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sock_inet6_stream_exthdr_in); + } else if (uproto == IPPROTO_UDP) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sock_inet6_dgram_exthdr_in); + } + } break; case SOPT_GET: @@ -3520,6 +3551,7 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, { int minmtupolicy, preftemp; int error; + boolean_t capture_exthdrstat_out = FALSE; if (!sticky && !cmsg) { #ifdef DIAGNOSTIC @@ -3747,7 +3779,7 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, if (opt->ip6po_hbh == NULL) return (ENOBUFS); bcopy(hbh, opt->ip6po_hbh, hbhlen); - + capture_exthdrstat_out = TRUE; break; } @@ -3811,6 +3843,7 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, if (*newdest == NULL) return (ENOBUFS); bcopy(dest, *newdest, destlen); + capture_exthdrstat_out = TRUE; break; } @@ -3851,6 +3884,7 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, if (opt->ip6po_rthdr == NULL) return (ENOBUFS); bcopy(rth, opt->ip6po_rthdr, rthlen); + capture_exthdrstat_out = TRUE; break; } @@ -3897,6 +3931,14 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, return (ENOPROTOOPT); } /* end of switch */ + if (capture_exthdrstat_out) { + if (uproto == IPPROTO_TCP) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sock_inet6_stream_exthdr_out); + } else if (uproto == IPPROTO_UDP) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sock_inet6_dgram_exthdr_out); + } + } + return (0); } @@ -4030,13 +4072,15 @@ ip6_output_checksum(struct ifnet *ifp, uint32_t mtu, struct mbuf *m, } else if (!(sw_csum & CSUM_DELAY_IPV6_DATA) && (hwcap & CSUM_PARTIAL)) { /* - * Partial checksum offload, ere), if no extension - * headers, and TCP only (no UDP support, as the - * hardware may not be able to convert +0 to - * -0 (0xffff) per RFC1122 4.1.3.4.) + * Partial checksum offload, ere), if no extension headers, + * and TCP only (no UDP support, as the hardware may not be + * able to convert +0 to -0 (0xffff) per RFC1122 4.1.3.4. + * unless the interface supports "invert zero" capability.) */ if (hwcksum_tx && !tso && - (m->m_pkthdr.csum_flags & CSUM_TCPIPV6) && + ((m->m_pkthdr.csum_flags & CSUM_TCPIPV6) || + ((hwcap & CSUM_ZERO_INVERT) && + (m->m_pkthdr.csum_flags & CSUM_ZERO_INVERT))) && tlen <= mtu) { uint16_t start = sizeof (struct ip6_hdr); uint16_t ulpoff = diff --git a/bsd/netinet6/ip6_var.h b/bsd/netinet6/ip6_var.h index e04dee46a..153ad9dc4 100644 --- a/bsd/netinet6/ip6_var.h +++ b/bsd/netinet6/ip6_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -148,10 +148,10 @@ struct ip6_moptions { }; #define IM6O_LOCK_ASSERT_HELD(_im6o) \ - lck_mtx_assert(&(_im6o)->im6o_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_im6o)->im6o_lock, LCK_MTX_ASSERT_OWNED) #define IM6O_LOCK_ASSERT_NOTHELD(_im6o) \ - lck_mtx_assert(&(_im6o)->im6o_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_im6o)->im6o_lock, LCK_MTX_ASSERT_NOTOWNED) #define IM6O_LOCK(_im6o) \ lck_mtx_lock(&(_im6o)->im6o_lock) @@ -336,11 +336,14 @@ struct ip6stat { /* DAD NS looped back */ u_quad_t ip6s_dad_loopcount; + + /* NECP policy related drop */ + u_quad_t ip6s_necp_policy_drop; }; enum ip6s_sources_rule_index { IP6S_SRCRULE_0, IP6S_SRCRULE_1, IP6S_SRCRULE_2, IP6S_SRCRULE_3, IP6S_SRCRULE_4, - IP6S_SRCRULE_5, IP6S_SRCRULE_5_5, IP6S_SRCRULE_6, IP6S_SRCRULE_7, + IP6S_SRCRULE_5, IP6S_SRCRULE_6, IP6S_SRCRULE_7, IP6S_SRCRULE_7x, IP6S_SRCRULE_8 }; @@ -471,6 +474,7 @@ extern struct pr_usrreqs icmp6_dgram_usrreqs; struct sockopt; struct inpcb; +struct ip6_hdr; struct in6_ifaddr; struct ip6protosw; struct domain; @@ -524,6 +528,8 @@ extern void ip6_clearpktopts(struct ip6_pktopts *, int); extern struct ip6_pktopts *ip6_copypktopts(struct ip6_pktopts *, int); extern int ip6_optlen(struct inpcb *); extern void ip6_drain(void); +extern int ip6_do_fragmentation(struct mbuf **, uint32_t, struct ifnet *, uint32_t, + struct ip6_hdr *, struct ip6_exthdrs *, uint32_t, int); extern int route6_input(struct mbuf **, int *, int); @@ -532,12 +538,20 @@ extern int frag6_input(struct mbuf **, int *, int); extern void frag6_drain(void); extern int rip6_input(struct mbuf **, int *, int); -extern void rip6_ctlinput(int, struct sockaddr *, void *); +extern void rip6_ctlinput(int, struct sockaddr *, void *, struct ifnet *); extern int rip6_ctloutput(struct socket *so, struct sockopt *sopt); extern int rip6_output(struct mbuf *, struct socket *, struct sockaddr_in6 *, struct mbuf *, int); extern int dest6_input(struct mbuf **, int *, int); +/* + * IPv6 source address selection hints + */ +#define IPV6_SRCSEL_HINT_PREFER_TMPADDR 0x00000001 + +extern struct in6_addr * in6_selectsrc_core(struct sockaddr_in6 *, + uint32_t, struct ifnet *, int, + struct in6_addr *, struct ifnet **, int *, struct ifaddr **); extern struct in6_addr *in6_selectsrc(struct sockaddr_in6 *, struct ip6_pktopts *, struct inpcb *, struct route_in6 *, struct ifnet **, struct in6_addr *, unsigned int, int *); diff --git a/bsd/netinet6/ip6protosw.h b/bsd/netinet6/ip6protosw.h index 12bec55b0..135b6f344 100644 --- a/bsd/netinet6/ip6protosw.h +++ b/bsd/netinet6/ip6protosw.h @@ -168,7 +168,7 @@ struct ip6protosw { (struct mbuf *m, struct socket *so, struct sockaddr_in6 *, struct mbuf *); void (*pr_ctlinput) /* control input (from below) */ - (int, struct sockaddr *, void *); + (int, struct sockaddr *, void *, struct ifnet *); int (*pr_ctloutput) /* control output (from above) */ (struct socket *, struct sockopt *); /* @@ -184,11 +184,11 @@ struct ip6protosw { /* for compat. with IPv4 protosw */ int (*pr_sysctl)(void); /* sysctl for protocol */ int (*pr_lock) /* lock function for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); int (*pr_unlock) /* unlock for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); lck_mtx_t *(*pr_getlock) /* retrieve protocol lock */ - (struct socket *so, int locktype); + (struct socket *so, int flags); /* * misc */ diff --git a/bsd/netinet6/ipsec.c b/bsd/netinet6/ipsec.c index 61302bab6..895d63e77 100644 --- a/bsd/netinet6/ipsec.c +++ b/bsd/netinet6/ipsec.c @@ -314,7 +314,7 @@ ipsec4_getpolicybysock(struct mbuf *m, struct secpolicy *currsp = NULL; /* policy on socket */ struct secpolicy *kernsp = NULL; /* policy on kernel */ - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL || so == NULL || error == NULL) panic("ipsec4_getpolicybysock: NULL pointer was passed.\n"); @@ -518,7 +518,7 @@ ipsec4_getpolicybyaddr(struct mbuf *m, if (ipsec_bypass != 0) return 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL || error == NULL) @@ -652,7 +652,7 @@ ipsec6_getpolicybysock(struct mbuf *m, struct secpolicy *currsp = NULL; /* policy on socket */ struct secpolicy *kernsp = NULL; /* policy on kernel */ - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL || so == NULL || error == NULL) @@ -823,7 +823,7 @@ ipsec6_getpolicybyaddr(struct mbuf *m, { struct secpolicy *sp = NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL || error == NULL) @@ -1955,7 +1955,7 @@ ipsec4_in_reject_so(struct mbuf *m, struct socket *so) int error; int result; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL) return 0; /* XXX should be panic ? */ @@ -1985,7 +1985,7 @@ ipsec4_in_reject_so(struct mbuf *m, struct socket *so) int ipsec4_in_reject(struct mbuf *m, struct inpcb *inp) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (inp == NULL) return ipsec4_in_reject_so(m, NULL); if (inp->inp_socket) @@ -2010,7 +2010,7 @@ ipsec6_in_reject_so(struct mbuf *m, struct socket *so) int error; int result; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL) return 0; /* XXX should be panic ? */ @@ -2040,7 +2040,7 @@ int ipsec6_in_reject(struct mbuf *m, struct in6pcb *in6p) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (in6p == NULL) return ipsec6_in_reject_so(m, NULL); if (in6p->in6p_socket) @@ -2064,7 +2064,7 @@ ipsec_hdrsiz(struct secpolicy *sp) struct ipsecrequest *isr; size_t siz, clen; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); KEYDEBUG(KEYDEBUG_IPSEC_DATA, printf("ipsec_hdrsiz: using SP\n"); kdebug_secpolicy(sp)); @@ -2138,7 +2138,7 @@ ipsec4_hdrsiz(struct mbuf *m, u_int dir, struct inpcb *inp) int error; size_t size; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL) return 0; /* XXX should be panic ? */ @@ -2179,7 +2179,7 @@ ipsec6_hdrsiz(struct mbuf *m, u_int dir, struct in6pcb *in6p) int error; size_t size; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL) return 0; /* XXX shoud be panic ? */ @@ -2312,7 +2312,11 @@ ipsec4_encapsulate(struct mbuf *m, struct secasvar *sav) ipseclog((LOG_ERR, "IPv4 ipsec: size exceeds limit: " "leave ip_len as is (invalid packet)\n")); } - ip->ip_id = ip_randomid(); + if (rfc6864 && IP_OFF_IS_ATOMIC(ntohs(ip->ip_off))) { + ip->ip_id = 0; + } else { + ip->ip_id = ip_randomid(); + } bcopy(&((struct sockaddr_in *)&sav->sah->saidx.src)->sin_addr, &ip->ip_src, sizeof(ip->ip_src)); bcopy(&((struct sockaddr_in *)&sav->sah->saidx.dst)->sin_addr, @@ -3268,7 +3272,7 @@ ipsec4_interface_output(struct ipsec_output_state *state, ifnet_t interface) int error = 0; struct secasvar *sav = NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (!state) panic("state == NULL in ipsec4_output"); @@ -3310,7 +3314,7 @@ ipsec4_output(struct ipsec_output_state *state, struct secpolicy *sp, __unused i int error = 0; struct sockaddr_in *sin; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (!state) panic("state == NULL in ipsec4_output"); @@ -3515,7 +3519,7 @@ ipsec6_output_trans( struct sockaddr_in6 *sin6; struct secasvar *sav = NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (!state) panic("state == NULL in ipsec6_output_trans"); @@ -3914,7 +3918,7 @@ ipsec6_output_tunnel( struct secasvar *sav = NULL; int error = 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (!state) panic("state == NULL in ipsec6_output_tunnel"); @@ -4047,7 +4051,7 @@ ipsec6_interface_output(struct ipsec_output_state *state, ifnet_t interface, u_c int error = 0; struct secasvar *sav = NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (!state) panic("state == NULL in ipsec6_output"); @@ -4185,7 +4189,7 @@ ipsec4_tunnel_validate( struct secpolicy *sp; struct ip *oip; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); #if DIAGNOSTIC if (m->m_len < sizeof(struct ip)) @@ -4303,7 +4307,7 @@ ipsec6_tunnel_validate( struct secpolicy *sp; struct ip6_hdr *oip6; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); #if DIAGNOSTIC if (m->m_len < sizeof(struct ip6_hdr)) @@ -4506,7 +4510,15 @@ struct ipsec_tag { struct socket *socket; u_int32_t history_count; struct ipsec_history history[]; +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Aligning to 64-bit since we case to m_tag which is 64-bit aligned. + */ +} __attribute__ ((aligned(8))); +#else }; +#endif #define IPSEC_TAG_SIZE (MLEN - sizeof(struct m_tag)) #define IPSEC_TAG_HDR_SIZE (offsetof(struct ipsec_tag, history[0])) @@ -4674,7 +4686,7 @@ ipsec_send_natt_keepalive( struct route ro; int keepalive_interval = natt_keepalive_interval; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if ((esp_udp_encap_port & 0xFFFF) == 0 || sav->remote_ike_port == 0) return FALSE; @@ -4804,7 +4816,11 @@ ipsec_fill_offload_frame(ifnet_t ifp, break; } ip->ip_len = htons(sizeof(struct udpiphdr) + 1); - ip->ip_id = ip_randomid(); + if (rfc6864 && IP_OFF_IS_ATOMIC(htons(ip->ip_off))) { + ip->ip_id = 0; + } else { + ip->ip_id = ip_randomid(); + } ip->ip_ttl = ip_defttl; ip->ip_p = IPPROTO_UDP; ip->ip_sum = 0; diff --git a/bsd/netinet6/ipsec.h b/bsd/netinet6/ipsec.h index 823bab27e..8d0e6dd51 100644 --- a/bsd/netinet6/ipsec.h +++ b/bsd/netinet6/ipsec.h @@ -290,8 +290,13 @@ struct ipsecstat { { "esp_randpad", CTLTYPE_INT }, \ } +#if defined(__ARM__) +#define IPSEC_IS_P2ALIGNED(p) IS_P2ALIGNED(p, sizeof (u_int32_t)) +#define IPSEC_GET_P2UNALIGNED_OFS(p) (sizeof(u_int32_t) - (((uintptr_t)(p)) & ((uintptr_t)(sizeof(u_int32_t)) - 1))) +#else #define IPSEC_IS_P2ALIGNED(p) 1 #define IPSEC_GET_P2UNALIGNED_OFS(p) 0 +#endif struct ipsec_output_state { int tunneled; diff --git a/bsd/netinet6/mld6.c b/bsd/netinet6/mld6.c index 6cbcaef13..ca0f1c7fd 100644 --- a/bsd/netinet6/mld6.c +++ b/bsd/netinet6/mld6.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -182,7 +182,7 @@ static const char * mld_rec_type_to_str(const int); #endif static uint32_t mld_set_version(struct mld_ifinfo *, const int); static void mld_flush_relq(struct mld_ifinfo *, struct mld_in6m_relhead *); -static void mld_dispatch_queue(struct mld_ifinfo *, struct ifqueue *, int); +static void mld_dispatch_queue_locked(struct mld_ifinfo *, struct ifqueue *, int); static int mld_v1_input_query(struct ifnet *, const struct ip6_hdr *, /*const*/ struct mld_hdr *); static int mld_v1_input_report(struct ifnet *, struct mbuf *, @@ -228,15 +228,16 @@ static int interface_timers_running6; static int state_change_timers_running6; static int current_state_timers_running6; +static unsigned int mld_mli_list_genid; /* * Subsystem lock macros. */ #define MLD_LOCK() \ lck_mtx_lock(&mld_mtx) #define MLD_LOCK_ASSERT_HELD() \ - lck_mtx_assert(&mld_mtx, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&mld_mtx, LCK_MTX_ASSERT_OWNED) #define MLD_LOCK_ASSERT_NOTHELD() \ - lck_mtx_assert(&mld_mtx, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&mld_mtx, LCK_MTX_ASSERT_NOTOWNED) #define MLD_UNLOCK() \ lck_mtx_unlock(&mld_mtx) @@ -487,12 +488,20 @@ out_locked: * Dispatch an entire queue of pending packet chains. * * Must not be called with in6m_lock held. + * XXX This routine unlocks MLD global lock and also mli locks. + * Make sure that the calling routine takes reference on the mli + * before calling this routine. + * Also if we are traversing mli_head, remember to check for + * mli list generation count and restart the loop if generation count + * has changed. */ static void -mld_dispatch_queue(struct mld_ifinfo *mli, struct ifqueue *ifq, int limit) +mld_dispatch_queue_locked(struct mld_ifinfo *mli, struct ifqueue *ifq, int limit) { struct mbuf *m; + MLD_LOCK_ASSERT_HELD(); + if (mli != NULL) MLI_LOCK_ASSERT_HELD(mli); @@ -503,11 +512,17 @@ mld_dispatch_queue(struct mld_ifinfo *mli, struct ifqueue *ifq, int limit) MLD_PRINTF(("%s: dispatch 0x%llx from 0x%llx\n", __func__, (uint64_t)VM_KERNEL_ADDRPERM(ifq), (uint64_t)VM_KERNEL_ADDRPERM(m))); + if (mli != NULL) MLI_UNLOCK(mli); + MLD_UNLOCK(); + mld_dispatch_packet(m); + + MLD_LOCK(); if (mli != NULL) MLI_LOCK(mli); + if (--limit == 0) break; } @@ -575,6 +590,7 @@ mld_domifattach(struct ifnet *ifp, int how) ifnet_lock_done(ifp); LIST_INSERT_HEAD(&mli_head, mli, mli_link); + mld_mli_list_genid++; MLD_UNLOCK(); @@ -608,6 +624,7 @@ mld_domifreattach(struct mld_ifinfo *mli) ifnet_lock_done(ifp); LIST_INSERT_HEAD(&mli_head, mli, mli_link); + mld_mli_list_genid++; MLD_UNLOCK(); @@ -664,6 +681,7 @@ mli_delete(const struct ifnet *ifp, struct mld_in6m_relhead *in6m_dthead) LIST_REMOVE(mli, mli_link); MLI_REMREF(mli); /* release mli_head reference */ + mld_mli_list_genid++; return; } MLI_UNLOCK(mli); @@ -1563,6 +1581,8 @@ mld_timeout(void *arg) struct mld_ifinfo *mli; struct in6_multi *inm; int uri_sec = 0; + unsigned int genid = mld_mli_list_genid; + SLIST_HEAD(, in6_multi) in6m_dthead; SLIST_INIT(&in6m_dthead); @@ -1600,12 +1620,30 @@ mld_timeout(void *arg) if (interface_timers_running6) { MLD_PRINTF(("%s: interface timers running\n", __func__)); interface_timers_running6 = 0; - LIST_FOREACH(mli, &mli_head, mli_link) { + mli = LIST_FIRST(&mli_head); + + while (mli != NULL) { + if (mli->mli_flags & MLIF_PROCESSED) { + mli = LIST_NEXT(mli, mli_link); + continue; + } + MLI_LOCK(mli); if (mli->mli_version != MLD_VERSION_2) { MLI_UNLOCK(mli); + mli = LIST_NEXT(mli, mli_link); continue; } + /* + * XXX The logic below ends up calling + * mld_dispatch_packet which can unlock mli + * and the global MLD lock. + * Therefore grab a reference on MLI and also + * check for generation count to see if we should + * iterate the list again. + */ + MLI_ADDREF_LOCKED(mli); + if (mli->mli_v2_timer == 0) { /* Do nothing. */ } else if (--mli->mli_v2_timer == 0) { @@ -1614,10 +1652,27 @@ mld_timeout(void *arg) } else { interface_timers_running6 = 1; } + mli->mli_flags |= MLIF_PROCESSED; MLI_UNLOCK(mli); + MLI_REMREF(mli); + + if (genid != mld_mli_list_genid) { + MLD_PRINTF(("%s: MLD information list changed " + "in the middle of iteration! Restart iteration.\n", + __func__)); + mli = LIST_FIRST(&mli_head); + genid = mld_mli_list_genid; + } else { + mli = LIST_NEXT(mli, mli_link); + } } + + LIST_FOREACH(mli, &mli_head, mli_link) + mli->mli_flags &= ~MLIF_PROCESSED; } + + if (!current_state_timers_running6 && !state_change_timers_running6) goto out_locked; @@ -1637,9 +1692,16 @@ mld_timeout(void *arg) * MLD host report and state-change timer processing. * Note: Processing a v2 group timer may remove a node. */ - LIST_FOREACH(mli, &mli_head, mli_link) { + mli = LIST_FIRST(&mli_head); + + while (mli != NULL) { struct in6_multistep step; + if (mli->mli_flags & MLIF_PROCESSED) { + mli = LIST_NEXT(mli, mli_link); + continue; + } + MLI_LOCK(mli); ifp = mli->mli_ifp; uri_sec = MLD_RANDOM_DELAY(mli->mli_uri); @@ -1670,13 +1732,22 @@ next: } in6_multihead_lock_done(); + /* + * XXX The logic below ends up calling + * mld_dispatch_packet which can unlock mli + * and the global MLD lock. + * Therefore grab a reference on MLI and also + * check for generation count to see if we should + * iterate the list again. + */ MLI_LOCK(mli); + MLI_ADDREF_LOCKED(mli); if (mli->mli_version == MLD_VERSION_1) { - mld_dispatch_queue(mli, &mli->mli_v1q, 0); + mld_dispatch_queue_locked(mli, &mli->mli_v1q, 0); } else if (mli->mli_version == MLD_VERSION_2) { MLI_UNLOCK(mli); - mld_dispatch_queue(NULL, &qrq, 0); - mld_dispatch_queue(NULL, &scq, 0); + mld_dispatch_queue_locked(NULL, &qrq, 0); + mld_dispatch_queue_locked(NULL, &scq, 0); VERIFY(qrq.ifq_len == 0); VERIFY(scq.ifq_len == 0); MLI_LOCK(mli); @@ -1694,12 +1765,27 @@ next: */ mld_flush_relq(mli, (struct mld_in6m_relhead *)&in6m_dthead); VERIFY(SLIST_EMPTY(&mli->mli_relinmhead)); + mli->mli_flags |= MLIF_PROCESSED; MLI_UNLOCK(mli); + MLI_REMREF(mli); IF_DRAIN(&qrq); IF_DRAIN(&scq); + + if (genid != mld_mli_list_genid) { + MLD_PRINTF(("%s: MLD information list changed " + "in the middle of iteration! Restart iteration.\n", + __func__)); + mli = LIST_FIRST(&mli_head); + genid = mld_mli_list_genid; + } else { + mli = LIST_NEXT(mli, mli_link); + } } + LIST_FOREACH(mli, &mli_head, mli_link) + mli->mli_flags &= ~MLIF_PROCESSED; + out_locked: /* re-arm the timer if there's work to do */ mld_timeout_run = 0; @@ -3407,7 +3493,7 @@ next: in6_multihead_lock_done(); MLI_LOCK(mli); - mld_dispatch_queue(mli, &mli->mli_gq, MLD_MAX_RESPONSE_BURST); + mld_dispatch_queue_locked(mli, &mli->mli_gq, MLD_MAX_RESPONSE_BURST); MLI_LOCK_ASSERT_HELD(mli); /* diff --git a/bsd/netinet6/mld6_var.h b/bsd/netinet6/mld6_var.h index 249506b85..c7623ea8d 100644 --- a/bsd/netinet6/mld6_var.h +++ b/bsd/netinet6/mld6_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2012 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -90,6 +90,7 @@ struct mld_ifinfo_u { #define MLIF_SILENT 0x00000001 /* Do not use MLD on this ifp */ #define MLIF_USEALLOW 0x00000002 /* Use ALLOW/BLOCK for joins/leaves */ +#define MLIF_PROCESSED 0x00000004 /* Entry has been processed and can be skipped */ /* * MLD version tag. @@ -193,10 +194,10 @@ struct mld_ifinfo { }; #define MLI_LOCK_ASSERT_HELD(_mli) \ - lck_mtx_assert(&(_mli)->mli_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_mli)->mli_lock, LCK_MTX_ASSERT_OWNED) #define MLI_LOCK_ASSERT_NOTHELD(_mli) \ - lck_mtx_assert(&(_mli)->mli_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_mli)->mli_lock, LCK_MTX_ASSERT_NOTOWNED) #define MLI_LOCK(_mli) \ lck_mtx_lock(&(_mli)->mli_lock) diff --git a/bsd/netinet6/nd6.c b/bsd/netinet6/nd6.c index 2aef3803e..0a16cdb14 100644 --- a/bsd/netinet6/nd6.c +++ b/bsd/netinet6/nd6.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -91,6 +91,7 @@ #include #include #include +#include #include #include @@ -226,7 +227,7 @@ static int nd6_sysctl_prlist SYSCTL_HANDLER_ARGS; * Insertion and removal from llinfo_nd6 must be done with rnh_lock held. */ #define LN_DEQUEUE(_ln) do { \ - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); \ + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); \ RT_LOCK_ASSERT_HELD((_ln)->ln_rt); \ (_ln)->ln_next->ln_prev = (_ln)->ln_prev; \ (_ln)->ln_prev->ln_next = (_ln)->ln_next; \ @@ -235,7 +236,7 @@ static int nd6_sysctl_prlist SYSCTL_HANDLER_ARGS; } while (0) #define LN_INSERTHEAD(_ln) do { \ - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); \ + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); \ RT_LOCK_ASSERT_HELD((_ln)->ln_rt); \ (_ln)->ln_next = llinfo_nd6.ln_next; \ llinfo_nd6.ln_next = (_ln); \ @@ -500,7 +501,7 @@ nd6_ifreset(struct ifnet *ifp) VERIFY(NULL != ndi); VERIFY(ndi->initialized); - lck_mtx_assert(&ndi->lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ndi->lock, LCK_MTX_ASSERT_OWNED); ndi->linkmtu = ifp->if_mtu; ndi->chlim = IPV6_DEFHLIM; ndi->basereachable = REACHABLE_TIME; @@ -713,6 +714,7 @@ nd6_options(union nd_opts *ndopts) (struct nd_opt_prefix_info *)nd_opt; break; case ND_OPT_RDNSS: + case ND_OPT_DNSSL: /* ignore */ break; default: @@ -762,11 +764,12 @@ nd6_service(void *arg) struct ifnet *ifp = NULL; struct in6_ifaddr *ia6, *nia6; uint64_t timenow; - bool send_nc_failure_kev = false; + boolean_t send_nc_failure_kev = FALSE; struct nd_drhead nd_defrouter_tmp; struct nd_defrouter *ndr = NULL; + struct radix_node_head *rnh = rt_tables[AF_INET6]; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* * Since we may drop rnh_lock and nd6_mutex below, we want * to run this entire operation single threaded. @@ -778,7 +781,7 @@ nd6_service(void *arg) nd6_service_waiters++; (void) msleep(nd6_service_wc, rnh_lock, (PZERO-1), __func__, NULL); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); } /* We are busy now; tell everyone else to go away */ @@ -829,7 +832,7 @@ again: dlil_post_complete_msg(NULL, &ev_msg); } - send_nc_failure_kev = false; + send_nc_failure_kev = FALSE; ifp = NULL; /* * The global list llinfo_nd6 is modified by nd6_request() and is @@ -843,7 +846,7 @@ again: * pass thru the entries and clear the flag so they can be processed * during the next timeout. */ - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); ln = llinfo_nd6.ln_next; while (ln != NULL && ln != &llinfo_nd6) { @@ -852,6 +855,7 @@ again: struct llinfo_nd6 *next; u_int32_t retrans, flags; struct nd_ifinfo *ndi = NULL; + boolean_t is_router = FALSE; /* ln_next/prev/rt is protected by rnh_lock */ next = ln->ln_next; @@ -931,6 +935,7 @@ again: flags = ndi->flags; RT_LOCK_ASSERT_HELD(rt); + is_router = (rt->rt_flags & RTF_ROUTER) ? TRUE : FALSE; switch (ln->ln_state) { case ND6_LLINFO_INCOMPLETE: @@ -954,7 +959,7 @@ again: } else { struct mbuf *m = ln->ln_hold; ln->ln_hold = NULL; - send_nc_failure_kev = (rt->rt_flags & RTF_ROUTER) ? true : false; + send_nc_failure_kev = is_router; if (m != NULL) { RT_ADDREF_LOCKED(rt); RT_UNLOCK(rt); @@ -974,12 +979,34 @@ again: RT_UNLOCK(rt); lck_mtx_unlock(rnh_lock); } + + /* + * Enqueue work item to invoke callback for + * this route entry + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_UNREACH, NULL, FALSE); nd6_free(rt); ap->killed++; lck_mtx_lock(rnh_lock); + /* + * nd6_free above would flush out the routing table of + * any cloned routes with same next-hop. + * Walk the tree anyways as there could be static routes + * left. + * + * We also already have a reference to rt that gets freed right + * after the block below executes. Don't need an extra reference + * on rt here. + */ + if (is_router) { + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_UNREACH); + (void) rnh->rnh_walktree(rnh, route_event_walktree, (void *)&rt_ev); + } rtfree_locked(rt); } - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); goto again; case ND6_LLINFO_REACHABLE: @@ -987,8 +1014,24 @@ again: ND6_CACHE_STATE_TRANSITION(ln, ND6_LLINFO_STALE); ln_setexpire(ln, timenow + nd6_gctimer); ap->aging_lazy++; + /* + * Enqueue work item to invoke callback for + * this route entry + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_STALE, NULL, TRUE); + + RT_ADDREF_LOCKED(rt); + RT_UNLOCK(rt); + if (is_router) { + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_STALE); + (void) rnh->rnh_walktree(rnh, route_event_walktree, (void *)&rt_ev); + } + rtfree_locked(rt); + } else { + RT_UNLOCK(rt); } - RT_UNLOCK(rt); break; case ND6_LLINFO_STALE: @@ -1043,16 +1086,41 @@ again: ap->aging++; lck_mtx_lock(rnh_lock); } else { - send_nc_failure_kev = (rt->rt_flags & RTF_ROUTER) ? true : false; + is_router = (rt->rt_flags & RTF_ROUTER) ? TRUE : FALSE; + send_nc_failure_kev = is_router; RT_ADDREF_LOCKED(rt); RT_UNLOCK(rt); lck_mtx_unlock(rnh_lock); nd6_free(rt); ap->killed++; + + /* + * Enqueue work item to invoke callback for + * this route entry + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_UNREACH, NULL, FALSE); + lck_mtx_lock(rnh_lock); + /* + * nd6_free above would flush out the routing table of + * any cloned routes with same next-hop. + * Walk the tree anyways as there could be static routes + * left. + * + * We also already have a reference to rt that gets freed right + * after the block below executes. Don't need an extra reference + * on rt here. + */ + if (is_router) { + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_UNREACH); + (void) rnh->rnh_walktree(rnh, + route_event_walktree, (void *)&rt_ev); + } rtfree_locked(rt); } - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); goto again; default: @@ -1061,7 +1129,7 @@ again: } ln = next; } - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* Now clear the flag from all entries */ ln = llinfo_nd6.ln_next; @@ -1084,6 +1152,10 @@ again: TAILQ_FOREACH_SAFE(dr, &nd_defrouter, dr_entry, ndr) { ap->found++; if (dr->expire != 0 && dr->expire < timenow) { + VERIFY(dr->ifp != NULL); + in6_ifstat_inc(dr->ifp, ifs6_defrtr_expiry_cnt); + in6_event_enqueue_nwk_wq_entry(IN6_NDP_RTR_EXPIRY, dr->ifp, + &dr->rtaddr, dr->rtlifetime); if (dr->ifp != NULL && dr->ifp->if_type == IFT_CELLULAR) { /* @@ -1211,6 +1283,12 @@ addrloop: in6_purgeaddr(&ia6->ia_ifa); ap->killed++; + if ((ia6->ia6_flags & IN6_IFF_TEMPORARY) == 0) { + in6_ifstat_inc(ia6->ia_ifa.ifa_ifp, ifs6_addr_expiry_cnt); + in6_event_enqueue_nwk_wq_entry(IN6_NDP_ADDR_EXPIRY, + ia6->ia_ifa.ifa_ifp, &ia6->ia_addr.sin6_addr, + 0); + } /* Release extra reference taken above */ IFA_REMREF(&ia6->ia_ifa); goto addrloop; @@ -1228,9 +1306,17 @@ addrloop: IFA_LOCK_ASSERT_HELD(&ia6->ia_ifa); if (IFA6_IS_DEPRECATED(ia6, timenow)) { int oldflags = ia6->ia6_flags; - ia6->ia6_flags |= IN6_IFF_DEPRECATED; + /* + * Only enqueue the Deprecated event when the address just + * becomes deprecated + */ + if((oldflags & IN6_IFF_DEPRECATED) == 0) { + in6_event_enqueue_nwk_wq_entry(IN6_ADDR_MARKED_DEPRECATED, + ia6->ia_ifa.ifa_ifp, &ia6->ia_addr.sin6_addr, + 0); + } /* * If a temporary address has just become deprecated, * regenerate a new one if possible. @@ -1271,7 +1357,7 @@ addrloop: ia6->ia6_flags &= ~IN6_IFF_DEPRECATED; IFA_UNLOCK(&ia6->ia_ifa); } - lck_rw_assert(&in6_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&in6_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); /* Release extra reference taken above */ IFA_REMREF(&ia6->ia_ifa); } @@ -1304,6 +1390,11 @@ addrloop: NDPR_ADDREF_LOCKED(pr); prelist_remove(pr); NDPR_UNLOCK(pr); + + in6_ifstat_inc(pr->ndpr_ifp, ifs6_pfx_expiry_cnt); + in6_event_enqueue_nwk_wq_entry(IN6_NDP_PFX_EXPIRY, + pr->ndpr_ifp, &pr->ndpr_prefix.sin6_addr, + 0); NDPR_REMREF(pr); pfxlist_onlink_check(); pr = nd_prefix.lh_first; @@ -1406,7 +1497,7 @@ nd6_sched_timeout(struct timeval *atv, struct timeval *ltv) { struct timeval tv; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (atv == NULL) { tv.tv_usec = 0; tv.tv_sec = MAX(nd6_prune, 1); @@ -1466,7 +1557,7 @@ nd6_sched_timeout(struct timeval *atv, struct timeval *ltv) */ void nd6_post_msg(u_int32_t code, struct nd_prefix_list *prefix_list, - u_int32_t list_length, u_int32_t mtu, char *dl_addr, u_int32_t dl_addr_len) + u_int32_t list_length, u_int32_t mtu) { struct kev_msg ev_msg; struct kev_nd6_ra_data nd6_ra_msg_data; @@ -1479,9 +1570,6 @@ nd6_post_msg(u_int32_t code, struct nd_prefix_list *prefix_list, ev_msg.event_code = code; bzero(&nd6_ra_msg_data, sizeof (nd6_ra_msg_data)); - nd6_ra_msg_data.lladdrlen = (dl_addr_len <= ND6_ROUTER_LL_SIZE) ? - dl_addr_len : ND6_ROUTER_LL_SIZE; - bcopy(dl_addr, &nd6_ra_msg_data.lladdr, nd6_ra_msg_data.lladdrlen); if (mtu > 0 && mtu >= IPV6_MMTU) { nd6_ra_msg_data.mtu = mtu; @@ -1750,7 +1838,7 @@ again: */ nd6_free(rt); RT_REMREF(rt); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); goto again; } else { RT_UNLOCK(rt); @@ -1781,7 +1869,7 @@ nd6_lookup(struct in6_addr *addr6, int create, struct ifnet *ifp, int rt_locked) ifscope = (ifp != NULL) ? ifp->if_index : IFSCOPE_NONE; if (rt_locked) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); rt = rtalloc1_scoped_locked(SA(&sin6), create, 0, ifscope); } else { rt = rtalloc1_scoped(SA(&sin6), create, 0, ifscope); @@ -1924,7 +2012,7 @@ nd6_is_new_addr_neighbor(struct sockaddr_in6 *addr, struct ifnet *ifp) struct nd_prefix *pr; struct ifaddr *dstaddr; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); /* * A link-local address is always a neighbor. @@ -2001,7 +2089,7 @@ nd6_is_addr_neighbor(struct sockaddr_in6 *addr, struct ifnet *ifp, { struct rtentry *rt; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(nd6_mutex); if (nd6_is_new_addr_neighbor(addr, ifp)) { lck_mtx_unlock(nd6_mutex); @@ -2036,7 +2124,7 @@ nd6_free(struct rtentry *rt) struct in6_addr in6; struct nd_defrouter *dr; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); RT_LOCK_ASSERT_NOTHELD(rt); lck_mtx_lock(nd6_mutex); @@ -2134,7 +2222,7 @@ nd6_rtrequest(int req, struct rtentry *rt, struct sockaddr *sa) VERIFY((NULL != ndi) && (TRUE == ndi->initialized)); VERIFY(nd6_init_done); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); /* @@ -2497,7 +2585,7 @@ nd6_siocgdrlst(void *data, int data_is_64) struct nd_defrouter *dr; int i = 0; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); dr = TAILQ_FIRST(&nd_defrouter); @@ -2580,7 +2668,7 @@ nd6_siocgprlst(void *data, int data_is_64) struct nd_prefix *pr; int i = 0; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); pr = nd_prefix.lh_first; @@ -3172,7 +3260,7 @@ fail: * 1 -- y -- (7) * STALE */ - if (lladdr) { /* (3-5) and (7) */ + if (lladdr != NULL) { /* (3-5) and (7) */ /* * Record source link-layer address * XXX is it dependent to ifp->if_type? @@ -3184,7 +3272,7 @@ fail: nd6_llreach_alloc(rt, ifp, LLADDR(sdl), sdl->sdl_alen, FALSE); } - if (!is_newentry) { + if (is_newentry == 0) { if ((!olladdr && lladdr != NULL) || /* (3) */ (olladdr && lladdr != NULL && llchange)) { /* (5) */ do_update = 1; @@ -3315,6 +3403,39 @@ fail: break; } + if (do_update) { + int route_ev_code = 0; + + if (llchange) + route_ev_code = ROUTE_LLENTRY_CHANGED; + else + route_ev_code = ROUTE_LLENTRY_RESOLVED; + + /* Enqueue work item to invoke callback for this route entry */ + route_event_enqueue_nwk_wq_entry(rt, NULL, route_ev_code, NULL, TRUE); + + if (ln->ln_router || (rt->rt_flags & RTF_ROUTER)) { + struct radix_node_head *rnh = NULL; + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, llchange ? ROUTE_LLENTRY_CHANGED : + ROUTE_LLENTRY_RESOLVED); + /* + * We already have a valid reference on rt. + * The function frees that before returning. + * We therefore don't need an extra reference here + */ + RT_UNLOCK(rt); + lck_mtx_lock(rnh_lock); + + rnh = rt_tables[AF_INET6]; + if (rnh != NULL) + (void) rnh->rnh_walktree(rnh, route_event_walktree, + (void *)&rt_ev); + lck_mtx_unlock(rnh_lock); + RT_LOCK(rt); + } + } + /* * When the link-layer address of a router changes, select the * best router again. In particular, when the neighbor entry is newly @@ -3748,8 +3869,7 @@ lookup: if (ln->ln_hold) m_freem_list(ln->ln_hold); ln->ln_hold = m0; - if (ln->ln_expire != 0 && ln->ln_asked < nd6_mmaxtries && - ln->ln_expire <= timenow) { + if (!ND6_LLINFO_PERMANENT(ln) && ln->ln_asked == 0) { ln->ln_asked++; ndi = ND_IFINFO(ifp); VERIFY(ndi != NULL && ndi->initialized); @@ -3767,9 +3887,6 @@ lookup: nd6_sched_timeout(NULL, NULL); lck_mtx_unlock(rnh_lock); } else { - if(ln->ln_state == ND6_LLINFO_INCOMPLETE) { - ln_setexpire(ln, timenow); - } RT_UNLOCK(rt); } /* diff --git a/bsd/netinet6/nd6.h b/bsd/netinet6/nd6.h index bd71a5a9b..13bb3e963 100644 --- a/bsd/netinet6/nd6.h +++ b/bsd/netinet6/nd6.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -70,6 +70,7 @@ #include #include #include +#include #include struct llinfo_nd6 { @@ -117,17 +118,18 @@ struct llinfo_nd6 { #ifdef BSD_KERNEL_PRIVATE #define ND6_CACHE_STATE_TRANSITION(ln, nstate) do {\ + struct rtentry *ln_rt = (ln)->ln_rt; \ if (nd6_debug >= 1) {\ nd6log((LOG_INFO,\ "[%s:%d]: NDP cache entry changed from %s -> %s",\ - __FILE__,\ + __func__,\ __LINE__,\ ndcache_state2str((ln)->ln_state),\ ndcache_state2str(nstate)));\ - if ((ln)->ln_rt)\ + if (ln_rt != NULL)\ nd6log((LOG_INFO,\ " for address: %s.\n",\ - ip6_sprintf(&SIN6(rt_key((ln)->ln_rt))->sin6_addr)));\ + ip6_sprintf(&SIN6(rt_key(ln_rt))->sin6_addr)));\ else\ nd6log((LOG_INFO, "\n"));\ }\ @@ -532,10 +534,10 @@ struct nd_defrouter { }; #define NDDR_LOCK_ASSERT_HELD(_nddr) \ - lck_mtx_assert(&(_nddr)->nddr_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_nddr)->nddr_lock, LCK_MTX_ASSERT_OWNED) #define NDDR_LOCK_ASSERT_NOTHELD(_nddr) \ - lck_mtx_assert(&(_nddr)->nddr_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_nddr)->nddr_lock, LCK_MTX_ASSERT_NOTOWNED) #define NDDR_LOCK(_nddr) \ lck_mtx_lock(&(_nddr)->nddr_lock) @@ -610,10 +612,10 @@ struct nd_prefix { #define NDPR_KEEP_EXPIRED (1800 * 2) #define NDPR_LOCK_ASSERT_HELD(_ndpr) \ - lck_mtx_assert(&(_ndpr)->ndpr_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_ndpr)->ndpr_lock, LCK_MTX_ASSERT_OWNED) #define NDPR_LOCK_ASSERT_NOTHELD(_ndpr) \ - lck_mtx_assert(&(_ndpr)->ndpr_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_ndpr)->ndpr_lock, LCK_MTX_ASSERT_NOTOWNED) #define NDPR_LOCK(_ndpr) \ lck_mtx_lock(&(_ndpr)->ndpr_lock) @@ -691,9 +693,6 @@ struct kev_nd6_ndalive { struct net_event_data link_data; }; -/* ND6 RA L2 source address length */ -#define ND6_ROUTER_LL_SIZE 64 - struct nd6_ra_prefix { struct sockaddr_in6 prefix; struct prf_ra raflags; @@ -713,8 +712,6 @@ struct nd6_ra_prefix { #define KEV_ND6_DATA_VALID_PREFIX (0x1 << 1) struct kev_nd6_ra_data { - u_int8_t lladdr[ND6_ROUTER_LL_SIZE]; - u_int32_t lladdrlen; u_int32_t mtu; u_int32_t list_index; u_int32_t list_length; @@ -723,6 +720,12 @@ struct kev_nd6_ra_data { u_int32_t pad; }; +struct kev_nd6_event { + struct net_event_data link_data; + struct in6_addr in6_address; + uint32_t val; +}; + struct nd6_lookup_ipv6_args { char ifname[IFNAMSIZ]; struct sockaddr_in6 ip6_dest; @@ -754,6 +757,7 @@ extern int nd6_debug; extern int nd6_onlink_ns_rfc4861; extern int nd6_optimistic_dad; +#define nd6log0(x) do { log x; } while (0) #define nd6log(x) do { if (nd6_debug >= 1) log x; } while (0) #define nd6log2(x) do { if (nd6_debug >= 2) log x; } while (0) @@ -839,7 +843,7 @@ extern int nd6_storelladdr(struct ifnet *, struct rtentry *, struct mbuf *, extern int nd6_need_cache(struct ifnet *); extern void nd6_drain(void *); extern void nd6_post_msg(u_int32_t, struct nd_prefix_list *, u_int32_t, - u_int32_t, char *, u_int32_t); + u_int32_t); extern int nd6_setifinfo(struct ifnet *, u_int32_t, u_int32_t); extern const char *ndcache_state2str(short); extern void ln_setexpire(struct llinfo_nd6 *, uint64_t); diff --git a/bsd/netinet6/nd6_nbr.c b/bsd/netinet6/nd6_nbr.c index 96eb73444..855b65e4c 100644 --- a/bsd/netinet6/nd6_nbr.c +++ b/bsd/netinet6/nd6_nbr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -81,6 +81,7 @@ #include #include #include +#include #include #include @@ -121,10 +122,10 @@ static struct zone *dad_zone; /* zone for dadq */ #define DAD_ZONE_NAME "nd6_dad" /* zone name */ #define DAD_LOCK_ASSERT_HELD(_dp) \ - lck_mtx_assert(&(_dp)->dad_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_dp)->dad_lock, LCK_MTX_ASSERT_OWNED) #define DAD_LOCK_ASSERT_NOTHELD(_dp) \ - lck_mtx_assert(&(_dp)->dad_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_dp)->dad_lock, LCK_MTX_ASSERT_NOTOWNED) #define DAD_LOCK(_dp) \ lck_mtx_lock(&(_dp)->dad_lock) @@ -272,11 +273,6 @@ nd6_ns_input( boolean_t is_dad_probe; int oflgclr = 0; - if ((ifp->if_eflags & IFEF_IPV6_ND6ALT) != 0) { - nd6log((LOG_INFO, "nd6_ns_input: on ND6ALT interface!\n")); - return; - } - /* Expect 32-bit aligned data pointer on strict-align platforms */ MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m); @@ -1049,25 +1045,46 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) ND6_CACHE_STATE_TRANSITION(ln, ND6_LLINFO_STALE); ln_setexpire(ln, timenow + nd6_gctimer); } + + + /* + * Enqueue work item to invoke callback for this + * route entry + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_RESOLVED, NULL, TRUE); + if ((ln->ln_router = is_router) != 0) { + struct radix_node_head *rnh = NULL; + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_RESOLVED); /* * This means a router's state has changed from * non-reachable to probably reachable, and might * affect the status of associated prefixes.. + * We already have a reference on rt. Don't need to + * take one for the unlock/lock. */ RT_UNLOCK(rt); + lck_mtx_lock(rnh_lock); + rnh = rt_tables[AF_INET6]; + + if (rnh != NULL) + (void) rnh->rnh_walktree(rnh, route_event_walktree, + (void *)&rt_ev); + lck_mtx_unlock(rnh_lock); lck_mtx_lock(nd6_mutex); pfxlist_onlink_check(); lck_mtx_unlock(nd6_mutex); RT_LOCK(rt); } } else { - int llchange; + int llchange = 0; /* * Check if the link-layer address has changed or not. */ - if (!lladdr) + if (lladdr == NULL) llchange = 0; else { if (sdl->sdl_alen) { @@ -1080,7 +1097,7 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) } /* - * This is VERY complex. Look at it with care. + * This is VERY complex. Look at it with care. * * override solicit lladdr llchange action * (L: record lladdr) @@ -1149,6 +1166,44 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) ln_setexpire(ln, timenow + nd6_gctimer); } } + + /* + * XXX + * The above is somewhat convoluted, for now just + * issue a callback for LLENTRY changed. + */ + /* Enqueue work item to invoke callback for this route entry */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_CHANGED, NULL, TRUE); + + /* + * If the entry is no longer a router, the logic post this processing + * gets rid of all the route entries having the current entry as a next + * hop. + * So only walk the tree here when there's no such change. + */ + if (ln->ln_router && is_router) { + struct radix_node_head *rnh = NULL; + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_CHANGED); + /* + * This means a router's state has changed from + * non-reachable to probably reachable, and might + * affect the status of associated prefixes.. + * + * We already have a valid rt reference here. + * We don't need to take another one for unlock/lock. + */ + RT_UNLOCK(rt); + lck_mtx_lock(rnh_lock); + rnh = rt_tables[AF_INET6]; + + if (rnh != NULL) + (void) rnh->rnh_walktree(rnh, route_event_walktree, + (void *)&rt_ev); + lck_mtx_unlock(rnh_lock); + RT_LOCK(rt); + } } if (ln->ln_router && !is_router) { @@ -1244,13 +1299,15 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) } RT_REMREF_LOCKED(rt); RT_UNLOCK(rt); + m_freem(m); + return; bad: icmp6stat.icp6s_badna++; /* fall through */ - freeit: m_freem(m); + return; } /* @@ -1614,11 +1671,11 @@ nd6_dad_start( */ IFA_LOCK(&ia->ia_ifa); if (!(ia->ia6_flags & IN6_IFF_DADPROGRESS)) { - log(LOG_DEBUG, + nd6log0((LOG_DEBUG, "nd6_dad_start: not a tentative or optimistic address " "%s(%s)\n", ip6_sprintf(&ia->ia_addr.sin6_addr), - ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???"); + ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???")); IFA_UNLOCK(&ia->ia_ifa); return; } @@ -1642,10 +1699,10 @@ nd6_dad_start( dp = zalloc(dad_zone); if (dp == NULL) { - log(LOG_ERR, "nd6_dad_start: memory allocation failed for " + nd6log0((LOG_ERR, "nd6_dad_start: memory allocation failed for " "%s(%s)\n", ip6_sprintf(&ia->ia_addr.sin6_addr), - ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???"); + ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???")); return; } bzero(dp, dad_size); @@ -1654,7 +1711,7 @@ nd6_dad_start( /* Callee adds one reference for us */ dp = nd6_dad_attach(dp, ifa); - nd6log((LOG_DEBUG, "%s: starting %sDAD %sfor %s\n", + nd6log0((LOG_DEBUG, "%s: starting %sDAD %sfor %s\n", if_name(ifa->ifa_ifp), (ia->ia6_flags & IN6_IFF_OPTIMISTIC) ? "optimistic " : "", (tick_delay == NULL) ? "immediately " : "", @@ -1793,7 +1850,7 @@ nd6_dad_timer(struct ifaddr *ifa) /* Sanity check */ if (ia == NULL) { - log(LOG_ERR, "nd6_dad_timer: called with null parameter\n"); + nd6log0((LOG_ERR, "nd6_dad_timer: called with null parameter\n")); goto done; } @@ -1805,23 +1862,23 @@ nd6_dad_timer(struct ifaddr *ifa) dp = nd6_dad_find(ifa, NULL); if (dp == NULL) { - log(LOG_ERR, "nd6_dad_timer: DAD structure not found\n"); + nd6log0((LOG_ERR, "nd6_dad_timer: DAD structure not found\n")); goto done; } IFA_LOCK(&ia->ia_ifa); if (ia->ia6_flags & IN6_IFF_DUPLICATED) { - log(LOG_ERR, "nd6_dad_timer: called with duplicated address " + nd6log0((LOG_ERR, "nd6_dad_timer: called with duplicated address " "%s(%s)\n", ip6_sprintf(&ia->ia_addr.sin6_addr), - ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???"); + ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???")); IFA_UNLOCK(&ia->ia_ifa); goto done; } if ((ia->ia6_flags & IN6_IFF_DADPROGRESS) == 0) { - log(LOG_ERR, "nd6_dad_timer: not a tentative or optimistic " + nd6log0((LOG_ERR, "nd6_dad_timer: not a tentative or optimistic " "address %s(%s)\n", ip6_sprintf(&ia->ia_addr.sin6_addr), - ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???"); + ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???")); IFA_UNLOCK(&ia->ia_ifa); goto done; } @@ -1831,7 +1888,7 @@ nd6_dad_timer(struct ifaddr *ifa) DAD_LOCK(dp); if (dp->dad_ns_tcount > dad_maxtry) { DAD_UNLOCK(dp); - nd6log((LOG_INFO, "%s: could not run DAD, driver problem?\n", + nd6log0((LOG_INFO, "%s: could not run DAD, driver problem?\n", if_name(ifa->ifa_ifp))); nd6_dad_detach(dp, ifa); @@ -1859,7 +1916,7 @@ nd6_dad_timer(struct ifaddr *ifa) if (dp->dad_na_icount > 0 || dp->dad_ns_icount) { /* We've seen NS or NA, means DAD has failed. */ DAD_UNLOCK(dp); - nd6log((LOG_INFO, + nd6log0((LOG_INFO, "%s: duplicate IPv6 address %s [timer]\n", __func__, ip6_sprintf(&ia->ia_addr.sin6_addr), if_name(ia->ia_ifp))); @@ -1883,7 +1940,7 @@ nd6_dad_timer(struct ifaddr *ifa) * additional probes until the loopback condition * becomes clear when a looped back probe is detected. */ - nd6log((LOG_INFO, + nd6log0((LOG_INFO, "%s: a looped back NS message is " "detected during DAD for %s. " "Another DAD probe is being sent on interface.\n", @@ -1917,14 +1974,14 @@ nd6_dad_timer(struct ifaddr *ifa) nd6_unsol_na_output(ifa); } - nd6log((LOG_DEBUG, + nd6log0((LOG_DEBUG, "%s: DAD complete for %s - no duplicates found%s\n", if_name(ifa->ifa_ifp), ip6_sprintf(&ia->ia_addr.sin6_addr), txunsolna ? ", tx unsolicited NA with O=1" : ".")); if (dp->dad_ns_lcount > 0) - nd6log((LOG_DEBUG, + nd6log0((LOG_DEBUG, "%s: DAD completed while " "a looped back NS message is detected " "during DAD for %s om interface %s\n", @@ -2029,6 +2086,9 @@ nd6_dad_duplicated(struct ifaddr *ifa) ia->ia6_flags &= ~IN6_IFF_DADPROGRESS; ia->ia6_flags |= IN6_IFF_DUPLICATED; + in6_event_enqueue_nwk_wq_entry(IN6_ADDR_MARKED_DUPLICATED, + ia->ia_ifa.ifa_ifp, &ia->ia_addr.sin6_addr, + 0); IFA_UNLOCK(&ia->ia_ifa); /* increment DAD collision counter */ @@ -2382,7 +2442,7 @@ nd6_alt_node_present(struct ifnet *ifp, struct sockaddr_in6 *sin6, { struct rtentry *rt; struct llinfo_nd6 *ln; - struct if_llreach *lr; + struct if_llreach *lr = NULL; const uint16_t temp_embedded_id = sin6->sin6_addr.s6_addr16[1]; if (IN6_IS_SCOPE_LINKLOCAL(&sin6->sin6_addr) && @@ -2392,7 +2452,7 @@ nd6_alt_node_present(struct ifnet *ifp, struct sockaddr_in6 *sin6, nd6_cache_lladdr(ifp, &sin6->sin6_addr, LLADDR(sdl), sdl->sdl_alen, ND_NEIGHBOR_ADVERT, 0); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); rt = rtalloc1_scoped_locked((struct sockaddr *)sin6, 1, 0, @@ -2449,7 +2509,7 @@ nd6_alt_node_absent(struct ifnet *ifp, struct sockaddr_in6 *sin6) (temp_embedded_id == 0)) sin6->sin6_addr.s6_addr16[1] = htons(ifp->if_index); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); rt = rtalloc1_scoped_locked((struct sockaddr *)sin6, 0, 0, diff --git a/bsd/netinet6/nd6_prproxy.c b/bsd/netinet6/nd6_prproxy.c index c85830994..bb7f1448b 100644 --- a/bsd/netinet6/nd6_prproxy.c +++ b/bsd/netinet6/nd6_prproxy.c @@ -285,13 +285,13 @@ nd6_prproxy_prelist_setroute(boolean_t enable, struct nd6_prproxy_prelist *up, *down, *ndprl_tmp; struct nd_prefix *pr; - lck_mtx_assert(&proxy6_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&proxy6_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); SLIST_FOREACH_SAFE(up, up_head, ndprl_le, ndprl_tmp) { struct rtentry *rt; boolean_t prproxy, set_allmulti = FALSE; - int allmulti_sw; + int allmulti_sw = FALSE; struct ifnet *ifp = NULL; SLIST_REMOVE(up_head, up, nd6_prproxy_prelist, ndprl_le); @@ -347,7 +347,7 @@ nd6_prproxy_prelist_setroute(boolean_t enable, struct nd_prefix *pr_up; struct rtentry *rt; boolean_t prproxy, set_allmulti = FALSE; - int allmulti_sw; + int allmulti_sw = FALSE; struct ifnet *ifp = NULL; SLIST_REMOVE(down_head, down, nd6_prproxy_prelist, ndprl_le); @@ -744,7 +744,7 @@ nd6_prproxy_prelist_update(struct nd_prefix *pr_cur, struct nd_prefix *pr_up) SLIST_INIT(&down_head); VERIFY(pr_cur != NULL); - lck_mtx_assert(&proxy6_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&proxy6_lock, LCK_MTX_ASSERT_OWNED); /* * Upstream prefix. If caller did not specify one, search for one @@ -857,7 +857,7 @@ nd6_prproxy_ifaddr(struct in6_ifaddr *ia) u_int32_t pr_len; boolean_t proxied = FALSE; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); IFA_LOCK(&ia->ia_ifa); bcopy(&ia->ia_addr.sin6_addr, &addr, sizeof (addr)); diff --git a/bsd/netinet6/nd6_rtr.c b/bsd/netinet6/nd6_rtr.c index be938f59e..8502334ea 100644 --- a/bsd/netinet6/nd6_rtr.c +++ b/bsd/netinet6/nd6_rtr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -382,7 +382,10 @@ nd6_ra_input( struct nd_defrouter dr0; u_int32_t advreachable; - +#if (DEVELOPMENT || DEBUG) + if (ip6_accept_rtadv == 0) + goto freeit; +#endif /* (DEVELOPMENT || DEBUG) */ /* Expect 32-bit aligned data pointer on strict-align platforms */ MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m); @@ -685,7 +688,7 @@ skip: /* Post message */ nd6_post_msg(KEV_ND6_RA, nd_prefix_list_head, nd_prefix_list_length, - mtu, lladdr, lladdrlen); + mtu); /* * Installing a link-layer address might change the state of the @@ -751,7 +754,7 @@ defrouter_addreq(struct nd_defrouter *new, boolean_t scoped) int err; struct nd_ifinfo *ndi = ND_IFINFO(new->ifp); - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); NDDR_LOCK_ASSERT_NOTHELD(new); /* * We're free to lock and unlock NDDR because our callers @@ -854,7 +857,7 @@ defrouter_lookup( { struct nd_defrouter *dr; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); for (dr = TAILQ_FIRST(&nd_defrouter); dr; dr = TAILQ_NEXT(dr, dr_entry)) { @@ -883,7 +886,7 @@ defrouter_delreq(struct nd_defrouter *dr) unsigned int ifscope; int err; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); NDDR_LOCK_ASSERT_NOTHELD(dr); /* * We're free to lock and unlock NDDR because our callers @@ -963,7 +966,7 @@ defrouter_reset(void) { struct nd_defrouter *dr, drany; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); dr = TAILQ_FIRST(&nd_defrouter); while (dr) { @@ -1092,7 +1095,7 @@ defrtrlist_del(struct nd_defrouter *dr) struct nd_ifinfo *ndi = NULL; boolean_t resetmtu; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); #if (DEVELOPMENT || DEBUG) /* @@ -1292,7 +1295,7 @@ defrouter_select(struct ifnet *ifp) unsigned int genid = 0; boolean_t is_installed_reachable = FALSE; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); if (ifp == NULL) { nd6log2((LOG_INFO, @@ -1358,7 +1361,7 @@ defrouter_select(struct ifnet *ifp) nd_defrouter_waiters++; msleep(nd_defrouter_waitchan, nd6_mutex, (PZERO-1), __func__, NULL); - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); } nd_defrouter_busy = TRUE; @@ -1643,7 +1646,7 @@ out: NDDR_REMREF(selected_dr); if (installed_dr) NDDR_REMREF(installed_dr); - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); VERIFY(nd_defrouter_busy); nd_defrouter_busy = FALSE; if (nd_defrouter_waiters > 0) { @@ -1660,7 +1663,7 @@ defrtrlist_update_common(struct nd_defrouter *new, boolean_t scoped) struct nd_ifinfo *ndi = NULL; struct timeval caltime; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); if ((dr = defrouter_lookup(&new->rtaddr, ifp)) != NULL) { /* entry exists */ @@ -1808,7 +1811,7 @@ defrtrlist_update(struct nd_defrouter *new) { struct nd_defrouter *dr; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); dr = defrtrlist_update_common(new, (nd6_defifp != NULL && new->ifp != nd6_defifp)); @@ -1820,7 +1823,7 @@ pfxrtr_lookup(struct nd_prefix *pr, struct nd_defrouter *dr) { struct nd_pfxrouter *search; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); NDPR_LOCK_ASSERT_HELD(pr); for (search = pr->ndpr_advrtrs.lh_first; search; @@ -1837,7 +1840,7 @@ pfxrtr_add(struct nd_prefix *pr, struct nd_defrouter *dr) { struct nd_pfxrouter *new; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); NDPR_LOCK_ASSERT_NOTHELD(pr); new = zalloc(ndprtr_zone); @@ -1857,7 +1860,7 @@ pfxrtr_add(struct nd_prefix *pr, struct nd_defrouter *dr) static void pfxrtr_del(struct nd_pfxrouter *pfr, struct nd_prefix *pr) { - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); NDPR_LOCK_ASSERT_HELD(pr); pr->ndpr_genid++; LIST_REMOVE(pfr, pfr_entry); @@ -2006,7 +2009,7 @@ prelist_remove(struct nd_prefix *pr) int e; struct nd_ifinfo *ndi = NULL; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); NDPR_LOCK_ASSERT_HELD(pr); if (pr->ndpr_stateflags & NDPRF_DEFUNCT) @@ -2717,7 +2720,7 @@ find_pfxlist_reachable_router(struct nd_prefix *pr) struct in6_addr rtaddr; unsigned int genid; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); NDPR_LOCK_ASSERT_HELD(pr); genid = pr->ndpr_genid; @@ -2783,13 +2786,13 @@ pfxlist_onlink_check(void) struct ifaddr **ifap = NULL; struct nd_prefix *ndpr; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); while (nd_prefix_busy) { nd_prefix_waiters++; msleep(nd_prefix_waitchan, nd6_mutex, (PZERO-1), __func__, NULL); - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); } nd_prefix_busy = TRUE; @@ -3070,6 +3073,9 @@ pfxlist_onlink_check(void) NDPR_UNLOCK(ndpr); IFA_LOCK(&ifa->ia_ifa); ifa->ia6_flags |= IN6_IFF_DETACHED; + in6_event_enqueue_nwk_wq_entry(IN6_ADDR_MARKED_DETACHED, + ifa->ia_ifa.ifa_ifp, &ifa->ia_addr.sin6_addr, + 0); IFA_UNLOCK(&ifa->ia_ifa); } NDPR_REMREF(ndpr); @@ -3101,7 +3107,7 @@ nd6_prefix_equal_lookup(struct nd_prefix *pr, boolean_t primary_only) { struct nd_prefix *opr; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); for (opr = nd_prefix.lh_first; opr; opr = opr->ndpr_next) { if (opr == pr) @@ -3137,7 +3143,7 @@ nd6_prefix_sync(struct ifnet *ifp) struct nd_prefix *pr, *opr; int err = 0; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); if (ifp == NULL) return; @@ -3240,7 +3246,7 @@ nd6_prefix_onlink_common(struct nd_prefix *pr, boolean_t force_scoped, int error = 0, prproxy = 0; struct rtentry *rt = NULL; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ NDPR_LOCK(pr); @@ -3453,7 +3459,7 @@ nd6_prefix_offlink(struct nd_prefix *pr) struct rtentry *rt = NULL, *ndpr_rt = NULL; unsigned int ifscope; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ NDPR_LOCK(pr); @@ -3963,7 +3969,7 @@ rt6_deleteroute( struct rtentry *rt = (struct rtentry *)rn; struct in6_addr *gate = (struct in6_addr *)arg; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); if (rt->rt_gateway == NULL || rt->rt_gateway->sa_family != AF_INET6) { @@ -4010,7 +4016,7 @@ nd6_setdefaultiface( int error = 0; ifnet_t def_ifp = NULL; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); ifnet_head_lock_shared(); if (ifindex < 0 || if_index < ifindex) { diff --git a/bsd/netinet6/nd6_send.c b/bsd/netinet6/nd6_send.c index 2b99a3bf4..cc7d35a5f 100644 --- a/bsd/netinet6/nd6_send.c +++ b/bsd/netinet6/nd6_send.c @@ -53,12 +53,12 @@ SYSCTL_NODE(_net_inet6, OID_AUTO, send, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IPv6 Secure Neighbor Discovery"); static int nd6_send_opmode = ND6_SEND_OPMODE_CGA_QUIET; -SYSCTL_INT(_net_inet6_send, OID_AUTO, opstate, CTLFLAG_RD | CTLFLAG_LOCKED, - &nd6_send_opstate, 0, "current SEND operating state"); +SYSCTL_INT(_net_inet6_send, OID_AUTO, opmode, CTLFLAG_RW | CTLFLAG_LOCKED, + &nd6_send_opmode, 0, "configured SEND operating mode"); int nd6_send_opstate = ND6_SEND_OPMODE_DISABLED; -SYSCTL_INT(_net_inet6_send, OID_AUTO, opmode, CTLFLAG_RW | CTLFLAG_LOCKED, - &nd6_send_opmode, 0, "configured SEND operating mode"); +SYSCTL_INT(_net_inet6_send, OID_AUTO, opstate, CTLFLAG_RD | CTLFLAG_LOCKED, + &nd6_send_opstate, 0, "current SEND operating state"); static int sysctl_cga_parameters SYSCTL_HANDLER_ARGS; diff --git a/bsd/netinet6/raw_ip6.c b/bsd/netinet6/raw_ip6.c index 4fff23a3c..d7c6f689f 100644 --- a/bsd/netinet6/raw_ip6.c +++ b/bsd/netinet6/raw_ip6.c @@ -102,6 +102,7 @@ #include #include +#include #include #include @@ -282,7 +283,8 @@ void rip6_ctlinput( int cmd, struct sockaddr *sa, - void *d) + void *d, + __unused struct ifnet *ifp) { struct ip6_hdr *ip6; struct mbuf *m; @@ -667,8 +669,9 @@ rip6_output( * the route interface index used by IP. */ if (rt != NULL && - (outif = rt->rt_ifp) != in6p->in6p_last_outifp) + (outif = rt->rt_ifp) != in6p->in6p_last_outifp) { in6p->in6p_last_outifp = outif; + } } else { ROUTE_RELEASE(&in6p->in6p_route); } @@ -886,6 +889,7 @@ rip6_bind(struct socket *so, struct sockaddr *nam, struct proc *p) } inp->in6p_laddr = sin6.sin6_addr; inp->in6p_last_outifp = outif; + return (0); } @@ -915,6 +919,12 @@ rip6_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p) return EADDRNOTAVAIL; if (addr->sin6_family != AF_INET6) return EAFNOSUPPORT; + + if (!(so->so_flags1 & SOF1_CONNECT_COUNTED)) { + so->so_flags1 |= SOF1_CONNECT_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet6_dgram_connected); + } + #if ENABLE_DEFAULT_SCOPE if (addr->sin6_scope_id == 0) { /* not change if specified */ /* avoid overwrites */ @@ -941,6 +951,7 @@ rip6_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p) if (inp->in6p_route.ro_rt != NULL) outif = inp->in6p_route.ro_rt->rt_ifp; inp->in6p_last_outifp = outif; + soisconnected(so); return 0; } diff --git a/bsd/netinet6/tcp6_var.h b/bsd/netinet6/tcp6_var.h index 8190f5269..d5b761e3a 100644 --- a/bsd/netinet6/tcp6_var.h +++ b/bsd/netinet6/tcp6_var.h @@ -105,7 +105,7 @@ SYSCTL_DECL(_net_inet6_tcp6); extern int tcp_v6mssdflt; /* XXX */ struct ip6_hdr; -void tcp6_ctlinput(int, struct sockaddr *, void *); +void tcp6_ctlinput(int, struct sockaddr *, void *, struct ifnet *); void tcp6_init(void); int tcp6_input(struct mbuf **, int *, int); struct rtentry *tcp_rtlookup6(struct inpcb *, unsigned int); diff --git a/bsd/netinet6/udp6_output.c b/bsd/netinet6/udp6_output.c index c0cd894e4..3db74ee52 100644 --- a/bsd/netinet6/udp6_output.c +++ b/bsd/netinet6/udp6_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -314,6 +314,11 @@ udp6_output(struct in6pcb *in6p, struct mbuf *m, struct sockaddr *addr6, if (af == AF_INET) hlen = sizeof (struct ip); + if (fport == htons(53) && !(so->so_flags1 & SOF1_DNS_COUNTED)) { + so->so_flags1 |= SOF1_DNS_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_dns); + } + /* * Calculate data length and get a mbuf * for UDP and IP6 headers. @@ -353,7 +358,7 @@ udp6_output(struct in6pcb *in6p, struct mbuf *m, struct sockaddr *addr6, udp6->uh_sum = in6_pseudo(laddr, faddr, htonl(plen + IPPROTO_UDP)); - m->m_pkthdr.csum_flags = CSUM_UDPIPV6; + m->m_pkthdr.csum_flags = (CSUM_UDPIPV6|CSUM_ZERO_INVERT); m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); if (!IN6_IS_ADDR_UNSPECIFIED(laddr)) @@ -477,6 +482,7 @@ udp6_output(struct in6pcb *in6p, struct mbuf *m, struct sockaddr *addr6, } INP_ADD_STAT(in6p, cell, wifi, wired, txpackets, 1); INP_ADD_STAT(in6p, cell, wifi, wired, txbytes, ulen); + inp_set_activity_bitmap(in6p); } if (flowadv && (adv->code == FADV_FLOW_CONTROLLED || @@ -500,7 +506,7 @@ udp6_output(struct in6pcb *in6p, struct mbuf *m, struct sockaddr *addr6, sizeof(struct udphdr) + hlen + ifnet_hdrlen(outif) + - ifnet_packetpreamblelen(outif), + ifnet_mbuf_packetpreamblelen(outif), sizeof(u_int32_t)); } @@ -533,7 +539,7 @@ udp6_output(struct in6pcb *in6p, struct mbuf *m, struct sockaddr *addr6, sizeof(struct udphdr) + hlen + ifnet_hdrlen(outif) + - ifnet_packetpreamblelen(outif), + ifnet_mbuf_packetpreamblelen(outif), sizeof(u_int32_t)); } } else { diff --git a/bsd/netinet6/udp6_usrreq.c b/bsd/netinet6/udp6_usrreq.c index b939c58bb..8680560fc 100644 --- a/bsd/netinet6/udp6_usrreq.c +++ b/bsd/netinet6/udp6_usrreq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -92,11 +92,10 @@ * * @(#)udp_var.h 8.1 (Berkeley) 6/10/93 */ - -#include #include #include #include +#include #include #include #include @@ -113,6 +112,7 @@ #include #include #include +#include #include #include @@ -218,6 +218,7 @@ udp6_append(struct inpcb *last, struct ip6_hdr *ip6, if (nstat_collect) { INP_ADD_STAT(last, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(last, cell, wifi, wired, rxbytes, n->m_pkthdr.len); + inp_set_activity_bitmap(last); } so_recv_data_stat(last->in6p_socket, n, 0); if (sbappendaddr(&last->in6p_socket->so_rcv, @@ -374,8 +375,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto) mcaddr.sin6_addr = ip6->ip6_dst; blocked = im6o_mc_filter(imo, ifp, - (struct sockaddr *)&mcaddr, - (struct sockaddr *)&fromsa); + &mcaddr, &fromsa); IM6O_UNLOCK(imo); if (blocked != MCAST_PASS) { udp_unlock(in6p->in6p_socket, 1, 0); @@ -582,6 +582,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto) if (nstat_collect) { INP_ADD_STAT(in6p, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(in6p, cell, wifi, wired, rxbytes, m->m_pkthdr.len); + inp_set_activity_bitmap(in6p); } so_recv_data_stat(in6p->in6p_socket, m, 0); if (sbappendaddr(&in6p->in6p_socket->so_rcv, @@ -604,13 +605,14 @@ bad: } void -udp6_ctlinput(int cmd, struct sockaddr *sa, void *d) +udp6_ctlinput(int cmd, struct sockaddr *sa, void *d, __unused struct ifnet *ifp) { struct udphdr uh; struct ip6_hdr *ip6; struct mbuf *m; int off = 0; struct ip6ctlparam *ip6cp = NULL; + struct icmp6_hdr *icmp6 = NULL; const struct sockaddr_in6 *sa6_src = NULL; void (*notify)(struct inpcb *, int) = udp_notify; struct udp_portonly { @@ -635,6 +637,7 @@ udp6_ctlinput(int cmd, struct sockaddr *sa, void *d) /* if the parameter is from icmp6, decode it. */ if (d != NULL) { ip6cp = (struct ip6ctlparam *)d; + icmp6 = ip6cp->ip6c_icmp6; m = ip6cp->ip6c_m; ip6 = ip6cp->ip6c_ip6; off = ip6cp->ip6c_off; @@ -650,7 +653,6 @@ udp6_ctlinput(int cmd, struct sockaddr *sa, void *d) * XXX: We assume that when IPV6 is non NULL, * M and OFF are valid. */ - /* check if we can safely examine src and dst ports */ if (m->m_pkthdr.len < off + sizeof (*uhp)) return; @@ -780,6 +782,12 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct proc *p) if (inp->inp_faddr.s_addr != INADDR_ANY) return (EISCONN); + + if (!(so->so_flags1 & SOF1_CONNECT_COUNTED)) { + so->so_flags1 |= SOF1_CONNECT_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_connected); + } + in6_sin6_2_sin(&sin, sin6_p); #if defined(NECP) && defined(FLOW_DIVERT) if (should_use_flow_divert) { @@ -789,6 +797,14 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct proc *p) error = in_pcbconnect(inp, (struct sockaddr *)&sin, p, IFSCOPE_NONE, NULL); if (error == 0) { +#if NECP + /* Update NECP client with connected five-tuple */ + if (!uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ inp->inp_vflag |= INP_IPV4; inp->inp_vflag &= ~INP_IPV6; soisconnected(so); @@ -800,6 +816,11 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct proc *p) if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) return (EISCONN); + if (!(so->so_flags1 & SOF1_CONNECT_COUNTED)) { + so->so_flags1 |= SOF1_CONNECT_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet6_dgram_connected); + } + #if defined(NECP) && defined(FLOW_DIVERT) do_flow_divert: if (should_use_flow_divert) { @@ -824,6 +845,14 @@ do_flow_divert: inp->inp_vflag &= ~INP_IPV4; inp->inp_vflag |= INP_IPV6; } +#if NECP + /* Update NECP client with connected five-tuple */ + if (!uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ soisconnected(so); if (inp->inp_flowhash == 0) inp->inp_flowhash = inp_calc_flowhash(inp); @@ -890,6 +919,7 @@ udp6_disconnect(struct socket *so) inp->in6p_laddr = in6addr_any; inp->in6p_last_outifp = NULL; + so->so_state &= ~SS_ISCONNECTED; /* XXX */ return (0); } @@ -1014,17 +1044,22 @@ udp6_input_checksum(struct mbuf *m, struct udphdr *uh, int off, int ulen) if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) { uh->uh_sum = m->m_pkthdr.csum_rx_val; } else { - uint16_t sum = m->m_pkthdr.csum_rx_val; - uint16_t start = m->m_pkthdr.csum_rx_start; + uint32_t sum = m->m_pkthdr.csum_rx_val; + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - (off + ulen)); /* * Perform 1's complement adjustment of octets * that got included/excluded in the hardware- - * calculated checksum value. + * calculated checksum value. Also take care + * of any trailing bytes and subtract out + * their partial sum. */ + ASSERT(trailer >= 0); if ((m->m_pkthdr.csum_flags & CSUM_PARTIAL) && - start != off) { - uint16_t s, d; + (start != off || trailer != 0)) { + uint32_t swbytes = (uint32_t)trailer; + uint16_t s = 0, d = 0; if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { s = ip6->ip6_src.s6_addr16[1]; @@ -1036,12 +1071,21 @@ udp6_input_checksum(struct mbuf *m, struct udphdr *uh, int off, int ulen) } /* callee folds in sum */ - sum = m_adj_sum16(m, start, off, sum); + sum = m_adj_sum16(m, start, off, ulen, sum); + if (off > start) + swbytes += (off - start); + else + swbytes += (start - off); if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) ip6->ip6_src.s6_addr16[1] = s; if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) ip6->ip6_dst.s6_addr16[1] = d; + + if (swbytes != 0) + udp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } uh->uh_sum = in6_pseudo(&ip6->ip6_src, &ip6->ip6_dst, diff --git a/bsd/netinet6/udp6_var.h b/bsd/netinet6/udp6_var.h index 0758482f7..d21f257d3 100644 --- a/bsd/netinet6/udp6_var.h +++ b/bsd/netinet6/udp6_var.h @@ -98,7 +98,7 @@ SYSCTL_DECL(_net_inet6_udp6); extern struct pr_usrreqs udp6_usrreqs; -extern void udp6_ctlinput(int, struct sockaddr *, void *); +extern void udp6_ctlinput(int, struct sockaddr *, void *, struct ifnet *); extern int udp6_input(struct mbuf **, int *, int); extern int udp6_output(struct inpcb *, struct mbuf *, struct sockaddr *, struct mbuf *, struct proc *); diff --git a/bsd/netkey/key.c b/bsd/netkey/key.c index a82188ec9..d8e8a1b39 100644 --- a/bsd/netkey/key.c +++ b/bsd/netkey/key.c @@ -677,7 +677,7 @@ static void key_start_timehandler(void) { /* must be called while locked */ - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); if (key_timehandler_running == 0) { key_timehandler_running = 1; (void)timeout((void *)key_timehandler, (void *)0, hz); @@ -703,7 +703,7 @@ key_allocsp( struct secpolicy *sp; struct timeval tv; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (spidx == NULL) panic("key_allocsp: NULL pointer is passed.\n"); @@ -852,11 +852,14 @@ struct secasvar *key_alloc_outbound_sav_for_interface(ifnet_t interface, int fam if (interface == NULL) return NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(sadb_mutex); LIST_FOREACH(sah, &sahtree, chain) { + if (sah->state == SADB_SASTATE_DEAD) { + continue; + } if (sah->ipsec_if == interface && (family == AF_INET6 || family == AF_INET) && sah->dir == IPSEC_DIR_OUTBOUND) { @@ -907,7 +910,7 @@ key_checkrequest( int error; struct sockaddr_in *sin; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); *sav = NULL; @@ -1101,7 +1104,7 @@ key_do_allocsa_policy( { struct secasvar *sav, *nextsav, *candidate, *natt_candidate, *no_natt_candidate, *d; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* initialize */ candidate = NULL; @@ -1230,6 +1233,17 @@ key_allocsa( caddr_t dst, u_int proto, u_int32_t spi) +{ + return key_allocsa_extended(family, src, dst, proto, spi, NULL); +} + +struct secasvar * +key_allocsa_extended(u_int family, + caddr_t src, + caddr_t dst, + u_int proto, + u_int32_t spi, + ifnet_t interface) { struct secasvar *sav, *match; u_int stateidx, state, tmpidx, matchidx; @@ -1238,7 +1252,7 @@ key_allocsa( const u_int *saorder_state_valid; int arraysize; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (src == NULL || dst == NULL) @@ -1272,6 +1286,10 @@ key_allocsa( LIST_FOREACH(sav, &spihash[SPIHASH(spi)], spihash) { if (sav->spi != spi) continue; + if (interface != NULL && + sav->sah->ipsec_if != interface) { + continue; + } if (proto != sav->sah->saidx.proto) continue; if (family != sav->sah->saidx.src.ss_family || @@ -1445,7 +1463,7 @@ key_do_get_translated_port( struct secasvar *currsav, *nextsav, *candidate; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* initilize */ candidate = NULL; @@ -1515,7 +1533,7 @@ key_freesp( if (!locked) lck_mtx_lock(sadb_mutex); else - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); sp->refcnt--; KEYDEBUG(KEYDEBUG_IPSEC_STAMP, printf("DP freesp cause refcnt--:%d SP:0x%llx\n", @@ -1546,7 +1564,7 @@ key_freesav( if (!locked) lck_mtx_lock(sadb_mutex); else - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); sav->refcnt--; KEYDEBUG(KEYDEBUG_IPSEC_STAMP, printf("DP freesav cause refcnt--:%d SA:0x%llx SPI %u\n", @@ -1573,7 +1591,7 @@ key_delsp( if (sp == NULL) panic("key_delsp: NULL pointer is passed.\n"); - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); sp->state = IPSEC_SPSTATE_DEAD; if (sp->refcnt > 0) @@ -1625,7 +1643,7 @@ key_getsp( { struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (spidx == NULL) @@ -1654,7 +1672,7 @@ key_getspbyid( { struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(sadb_mutex); sp = __key_getspbyid(id); @@ -1668,7 +1686,7 @@ __key_getspbyid(u_int32_t id) { struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); LIST_FOREACH(sp, &sptree[IPSEC_DIR_INBOUND], chain) { if (sp->state == IPSEC_SPSTATE_DEAD) @@ -1696,7 +1714,7 @@ key_newsp(void) { struct secpolicy *newsp = NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); newsp = keydb_newsecpolicy(); if (!newsp) return newsp; @@ -1720,7 +1738,7 @@ key_msg2sp( { struct secpolicy *newsp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (xpl0 == NULL) @@ -1770,6 +1788,13 @@ key_msg2sp( xisr = (struct sadb_x_ipsecrequest *)(xpl0 + 1); while (tlen > 0) { + if (tlen < sizeof(*xisr)) { + ipseclog((LOG_DEBUG, "key_msg2sp: " + "invalid ipsecrequest.\n")); + key_freesp(newsp, KEY_SADB_UNLOCKED); + *error = EINVAL; + return NULL; + } /* length check */ if (xisr->sadb_x_ipsecrequest_len < sizeof(*xisr)) { @@ -1873,8 +1898,25 @@ key_msg2sp( /* set IP addresses if there */ if (xisr->sadb_x_ipsecrequest_len > sizeof(*xisr)) { struct sockaddr *paddr; + + if (tlen < xisr->sadb_x_ipsecrequest_len) { + ipseclog((LOG_DEBUG, "key_msg2sp: invalid request " + "address length.\n")); + key_freesp(newsp, KEY_SADB_UNLOCKED); + *error = EINVAL; + return NULL; + } paddr = (struct sockaddr *)(xisr + 1); + uint8_t src_len = paddr->sa_len; + + if (xisr->sadb_x_ipsecrequest_len < src_len) { + ipseclog((LOG_DEBUG, "key_msg2sp: invalid request " + "invalid source address length.\n")); + key_freesp(newsp, KEY_SADB_UNLOCKED); + *error = EINVAL; + return NULL; + } /* validity check */ if (paddr->sa_len @@ -1885,11 +1927,20 @@ key_msg2sp( *error = EINVAL; return NULL; } + bcopy(paddr, &(*p_isr)->saidx.src, - paddr->sa_len); + MIN(paddr->sa_len, sizeof((*p_isr)->saidx.src))); - paddr = (struct sockaddr *)((caddr_t)paddr - + paddr->sa_len); + paddr = (struct sockaddr *)((caddr_t)paddr + paddr->sa_len); + uint8_t dst_len = paddr->sa_len; + + if (xisr->sadb_x_ipsecrequest_len < (src_len + dst_len)) { + ipseclog((LOG_DEBUG, "key_msg2sp: invalid request " + "invalid dest address length.\n")); + key_freesp(newsp, KEY_SADB_UNLOCKED); + *error = EINVAL; + return NULL; + } /* validity check */ if (paddr->sa_len @@ -1900,8 +1951,9 @@ key_msg2sp( *error = EINVAL; return NULL; } + bcopy(paddr, &(*p_isr)->saidx.dst, - paddr->sa_len); + MIN(paddr->sa_len, sizeof((*p_isr)->saidx.dst))); } (*p_isr)->sp = newsp; @@ -2134,7 +2186,7 @@ key_spdadd( struct mbuf *m, const struct sadb_msghdr *mhp) { - struct sadb_address *src0, *dst0, *src1, *dst1; + struct sadb_address *src0, *dst0, *src1 = NULL, *dst1 = NULL; struct sadb_x_policy *xpl0, *xpl; struct sadb_lifetime *lft = NULL; struct secpolicyindex spidx; @@ -2150,7 +2202,7 @@ key_spdadd( int init_disabled = 0; int address_family, address_len; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2536,7 +2588,7 @@ key_spddelete( struct mbuf *m, const struct sadb_msghdr *mhp) { - struct sadb_address *src0, *dst0, *src1, *dst1; + struct sadb_address *src0, *dst0, *src1 = NULL, *dst1 = NULL; struct sadb_x_policy *xpl0; struct secpolicyindex spidx; struct secpolicy *sp; @@ -2545,7 +2597,7 @@ key_spddelete( int use_src_range = 0; int use_dst_range = 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2690,7 +2742,7 @@ key_spddelete2( u_int32_t id; struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2780,7 +2832,7 @@ key_spdenable( u_int32_t id; struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2840,7 +2892,7 @@ key_spddisable( u_int32_t id; struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2913,7 +2965,7 @@ key_spdget( struct secpolicy *sp; struct mbuf *n; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2967,7 +3019,7 @@ key_spdacquire( struct secspacq *newspacq; int error; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (sp == NULL) @@ -3305,7 +3357,7 @@ key_spdexpire( int error = EINVAL; struct sadb_lifetime *lt; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (sp == NULL) @@ -3512,7 +3564,7 @@ key_delsah( u_int stateidx, state; int zombie = 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (sah == NULL) @@ -3598,7 +3650,7 @@ key_newsav( struct secasvar *newsav; const struct sadb_sa *xsa; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (m == NULL || mhp == NULL || mhp->msg == NULL || sah == NULL) @@ -3763,7 +3815,7 @@ key_newsav2(struct secashead *sah, { struct secasvar *newsav; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (sah == NULL) @@ -3861,7 +3913,7 @@ key_delsav( struct secasvar *sav) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (sav == NULL) @@ -3930,7 +3982,7 @@ key_getsah(struct secasindex *saidx) { struct secashead *sah; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); LIST_FOREACH(sah, &sahtree, chain) { if (sah->state == SADB_SASTATE_DEAD) @@ -3948,7 +4000,7 @@ key_newsah2 (struct secasindex *saidx, { struct secashead *sah; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); sah = key_getsah(saidx); if (!sah) { @@ -3972,7 +4024,7 @@ key_checkspidup( struct secasvar *sav; u_int stateidx, state; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* check address family */ if (saidx->src.ss_family != saidx->dst.ss_family) { @@ -4002,7 +4054,7 @@ key_setspi( struct secasvar *sav, u_int32_t spi) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); sav->spi = spi; if (sav->spihash.le_prev || sav->spihash.le_next) LIST_REMOVE(sav, spihash); @@ -4024,7 +4076,7 @@ key_getsavbyspi( struct secasvar *sav, *match; u_int stateidx, state, matchidx; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); match = NULL; matchidx = _ARRAYLEN(saorder_state_alive); LIST_FOREACH(sav, &spihash[SPIHASH(spi)], spihash) { @@ -4065,7 +4117,7 @@ key_setsaval( int error = 0; struct timeval tv; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (m == NULL || mhp == NULL || mhp->msg == NULL) @@ -4406,7 +4458,7 @@ key_setsaval2(struct secasvar *sav, int error = 0; struct timeval tv; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* initialization */ sav->replay = NULL; @@ -4611,7 +4663,7 @@ key_mature( mature = 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* check SPI value */ switch (sav->sah->saidx.proto) { @@ -5296,7 +5348,7 @@ key_newbuf( { caddr_t new; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); KMALLOC_NOWAIT(new, caddr_t, len); if (new == NULL) { lck_mtx_unlock(sadb_mutex); @@ -6456,7 +6508,7 @@ key_getspi( u_int32_t reqid; int error; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -6658,7 +6710,7 @@ key_getspi2(struct sockaddr *src, u_int32_t spi; struct secasindex saidx; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* XXX boundary check against sa_len */ KEY_SETSECASIDX(proto, mode, reqid, src, dst, 0, &saidx); @@ -6719,7 +6771,7 @@ key_do_getnewspi( u_int32_t keymin, keymax; int count = key_spi_trycnt; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* set spi range to allocate */ if (spirange != NULL) { @@ -6812,7 +6864,7 @@ key_update( u_int16_t flags2; int error; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -6993,7 +7045,7 @@ key_migrate(struct socket *so, struct secasvar *sav = NULL; u_int16_t proto; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -7136,7 +7188,7 @@ key_getsavbyseq( struct secasvar *sav; u_int state; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); state = SADB_SASTATE_LARVAL; @@ -7191,7 +7243,7 @@ key_add( u_int32_t reqid; int error; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -7335,7 +7387,7 @@ key_setident( const struct sadb_ident *idsrc, *iddst; int idsrclen, iddstlen; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (sah == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -7473,7 +7525,7 @@ key_delete( struct secasvar *sav = NULL; u_int16_t proto; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -7588,7 +7640,7 @@ key_delete_all( struct secasvar *sav, *nextsav; u_int stateidx, state; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); src0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_SRC]); dst0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_DST]); @@ -7680,7 +7732,7 @@ key_get( struct secasvar *sav = NULL; u_int16_t proto; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -8121,7 +8173,7 @@ key_acquire( int error = -1; u_int32_t seq; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (saidx == NULL) @@ -8330,7 +8382,7 @@ key_getacq( { struct secacq *acq; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); LIST_FOREACH(acq, &acqtree, chain) { if (key_cmpsaidx(saidx, &acq->saidx, CMP_EXACTLY)) @@ -8346,7 +8398,7 @@ key_getacqbyseq( { struct secacq *acq; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); LIST_FOREACH(acq, &acqtree, chain) { if (acq->seq == seq) @@ -8392,7 +8444,7 @@ key_getspacq( { struct secspacq *acq; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); LIST_FOREACH(acq, &spacqtree, chain) { if (key_cmpspidx_exactly(spidx, &acq->spidx)) @@ -8796,7 +8848,7 @@ key_expire( int error = -1; struct sadb_lifetime *lt; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (sav == NULL) @@ -9020,7 +9072,7 @@ key_dump( struct mbuf *n; int error = 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -9365,7 +9417,7 @@ key_parse( int target; Boolean keyAligned = FALSE; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL || so == NULL) @@ -9580,7 +9632,7 @@ key_senderror( { struct sadb_msg *msg; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (m->m_len < sizeof(struct sadb_msg)) panic("invalid mbuf passed to key_senderror"); @@ -9721,7 +9773,7 @@ key_validate_ext( { struct sockaddr *sa; enum { NONE, ADDR } checktype = NONE; - int baselen; + int baselen = 0; const int sal = offsetof(struct sockaddr, sa_len) + sizeof(sa->sa_len); if (len != PFKEY_UNUNIT64(ext->sadb_ext_len)) @@ -9891,7 +9943,7 @@ key_sa_chgstate( if (sav->state == state) return; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); if (__LIST_CHAINED(sav)) LIST_REMOVE(sav, chain); @@ -10049,7 +10101,7 @@ key_getsastat (struct socket *so, return key_senderror(so, m, EINVAL); } - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); // exit early if there are no active SAs if (ipsec_sav_count <= 0) { @@ -10256,7 +10308,7 @@ key_delsp_for_ipsec_if (ifnet_t ipsec_if) if (ipsec_if == NULL) return; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(sadb_mutex); diff --git a/bsd/netkey/key.h b/bsd/netkey/key.h index aec0ae52d..c13c36947 100644 --- a/bsd/netkey/key.h +++ b/bsd/netkey/key.h @@ -62,6 +62,9 @@ extern int key_checkrequest(struct ipsecrequest *isr, struct secasindex *, struct secasvar **sav); extern struct secasvar *key_allocsa(u_int, caddr_t, caddr_t, u_int, u_int32_t); +struct secasvar * +key_allocsa_extended(u_int family, caddr_t src, caddr_t dst, + u_int proto, u_int32_t spi, ifnet_t interface); extern u_int16_t key_natt_get_translated_port(struct secasvar *); extern void key_freesp(struct secpolicy *, int); extern void key_freesav(struct secasvar *, int); diff --git a/bsd/netkey/keydb.c b/bsd/netkey/keydb.c index eba59f728..a5500fe9c 100644 --- a/bsd/netkey/keydb.c +++ b/bsd/netkey/keydb.c @@ -91,7 +91,7 @@ keydb_newsecpolicy(void) { struct secpolicy *p; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); return (struct secpolicy *)_MALLOC(sizeof(*p), M_SECA, M_WAITOK | M_ZERO); @@ -112,7 +112,7 @@ keydb_newsecashead(void) struct secashead *p; int i; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); p = (struct secashead *)_MALLOC(sizeof(*p), M_SECA, M_NOWAIT | M_ZERO); if (!p) { @@ -147,7 +147,7 @@ keydb_newsecasvar() { struct secasvar *p; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); p = (struct secasvar *)_MALLOC(sizeof(*p), M_SECA, M_WAITOK); if (!p) @@ -162,7 +162,7 @@ keydb_refsecasvar(p) struct secasvar *p; { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); p->refcnt++; } @@ -172,7 +172,7 @@ keydb_freesecasvar(p) struct secasvar *p; { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); p->refcnt--; /* negative refcnt will cause panic intentionally */ @@ -200,7 +200,7 @@ keydb_newsecreplay(size_t wsize) { struct secreplay *p; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); p = (struct secreplay *)_MALLOC(sizeof(*p), M_SECA, M_NOWAIT | M_ZERO); if (!p) { diff --git a/bsd/nfs/gss/gss_krb5_mech.c b/bsd/nfs/gss/gss_krb5_mech.c index 59df39f4a..0d592043a 100644 --- a/bsd/nfs/gss/gss_krb5_mech.c +++ b/bsd/nfs/gss/gss_krb5_mech.c @@ -82,7 +82,7 @@ typedef struct crypt_walker_ctx { typedef struct hmac_walker_ctx { const struct ccdigest_info *di; - cchmac_ctx_t hmac_ctx; + struct cchmac_ctx *hmac_ctx; } *hmac_walker_ctx_t; typedef size_t (*ccpad_func)(const struct ccmode_cbc *, cccbc_ctx *, cccbc_iv *, @@ -521,10 +521,10 @@ do_crypt(void *walker, uint8_t *data, uint32_t len) void do_hmac_init(hmac_walker_ctx_t wctx, crypto_ctx_t cctx, void *key) { - size_t alloc_size = cc_ctx_n(struct cchmac_ctx, cchmac_di_size(cctx->di)) * sizeof(struct cchmac_ctx); + size_t alloc_size = cchmac_di_size(cctx->di); wctx->di = cctx->di; - MALLOC(wctx->hmac_ctx.hdr, struct cchmac_ctx *, alloc_size, M_TEMP, M_WAITOK|M_ZERO); + MALLOC(wctx->hmac_ctx, struct cchmac_ctx *, alloc_size, M_TEMP, M_WAITOK|M_ZERO); cchmac_init(cctx->di, wctx->hmac_ctx, cctx->keylen, key); } @@ -622,7 +622,7 @@ krb5_mic_mbuf(crypto_ctx_t ctx, gss_buffer_t header, cchmac_update(ctx->di, wctx.hmac_ctx, trailer->length, trailer->value); cchmac_final(ctx->di, wctx.hmac_ctx, digest); - FREE(wctx.hmac_ctx.hdr, M_TEMP); + FREE(wctx.hmac_ctx, M_TEMP); if (verify) { *verify = (memcmp(mic, digest, ctx->digest_size) == 0); diff --git a/bsd/nfs/nfs.h b/bsd/nfs/nfs.h index 0232e77a1..9cacf96c5 100644 --- a/bsd/nfs/nfs.h +++ b/bsd/nfs/nfs.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -200,7 +200,7 @@ extern int nfs_ticks; #define NFS_MFLAG_MUTEJUKEBOX 8 /* don't treat jukebox errors as unresponsive */ #define NFS_MFLAG_EPHEMERAL 9 /* ephemeral (mirror) mount */ #define NFS_MFLAG_NOCALLBACK 10 /* don't provide callback RPC service */ -#define NFS_MFLAG_NONAMEDATTR 11 /* don't use named attributes */ +#define NFS_MFLAG_NAMEDATTR 11 /* don't use named attributes */ #define NFS_MFLAG_NOACL 12 /* don't support ACLs */ #define NFS_MFLAG_ACLONLY 13 /* only support ACLs - not mode */ #define NFS_MFLAG_NFC 14 /* send NFC strings */ @@ -771,14 +771,19 @@ struct nfsstats { #include /* for guid_t below */ #define MAXIDNAMELEN 1024 struct nfs_testmapid { - uint32_t ntm_name2id; /* lookup name 2 id or id 2 name */ + uint32_t ntm_lookup; /* lookup name 2 id or id 2 name */ uint32_t ntm_grpflag; /* Is this a group or user maping */ uint32_t ntm_id; /* id to map or return */ uint32_t pad; guid_t ntm_guid; /* intermidiate guid used in conversion */ char ntm_name[MAXIDNAMELEN]; /* name to map or return */ }; - + +#define NTM_ID2NAME 0 +#define NTM_NAME2ID 1 +#define NTM_NAME2GUID 2 +#define NTM_GUID2NAME 3 + /* * fs.nfs sysctl(3) identifiers */ @@ -998,9 +1003,6 @@ extern uint32_t nfs_debug_ctl; /* bits for nfs_idmap_ctrl: */ #define NFS_IDMAP_CTRL_USE_IDMAP_SERVICE 0x00000001 /* use the ID mapping service */ #define NFS_IDMAP_CTRL_FALLBACK_NO_COMMON_IDS 0x00000002 /* fallback should NOT handle common IDs like "root" and "nobody" */ -#define NFS_IDMAP_CTRL_FALLBACK_NO_WELLKNOWN_IDS 0x00000004 /* fallback should NOT handle the well known "XXX@" IDs */ -#define NFS_IDMAP_CTRL_UNKNOWN_IS_99 0x00000008 /* for unknown IDs use uid/gid 99 instead of -2/nobody */ -#define NFS_IDMAP_CTRL_COMPARE_RESULTS 0x00000010 /* compare results of ID mapping service and fallback */ #define NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS 0x00000020 /* log failed ID mapping attempts */ #define NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS 0x00000040 /* log successful ID mapping attempts */ @@ -1150,7 +1152,7 @@ extern thread_call_t nfsrv_fmod_timer_call; #endif /* nfs 4 default domain for user mapping */ -extern char nfs4_domain[MAXPATHLEN]; +extern char nfs4_default_domain[MAXPATHLEN]; __BEGIN_DECLS @@ -1261,7 +1263,7 @@ uint32_t nfs4_ace_vfsflags_to_nfsflags(uint32_t); uint32_t nfs4_ace_nfsmask_to_vfsrights(uint32_t); uint32_t nfs4_ace_vfsrights_to_nfsmask(uint32_t); int nfs4_id2guid(char *, guid_t *, int); -int nfs4_guid2id(guid_t *, char *, int *, int); +int nfs4_guid2id(guid_t *, char *, size_t *, int); int nfs_parsefattr(struct nfsm_chain *, int, struct nfs_vattr *); int nfs4_parsefattr(struct nfsm_chain *, struct nfs_fsattr *, struct nfs_vattr *, fhandle_t *, struct dqblk *, struct nfs_fs_locations *); diff --git a/bsd/nfs/nfs4_subs.c b/bsd/nfs/nfs4_subs.c index 130b04f7a..d94f41582 100644 --- a/bsd/nfs/nfs4_subs.c +++ b/bsd/nfs/nfs4_subs.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2011 Apple Inc. All rights reserved. + * Copyright (c) 2006-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -709,8 +709,8 @@ nfs4_default_attrs_for_referral_trigger( struct nfs_vattr *nvap, fhandle_t *fhp) { - struct timeval now; - microtime(&now); + struct timespec now; + nanotime(&now); int len; nvap->nva_flags = NFS_FFLAG_TRIGGER | NFS_FFLAG_TRIGGER_REFERRAL; @@ -752,17 +752,17 @@ nfs4_default_attrs_for_referral_trigger( if (!NFS_BITMAP_ISSET(nvap->nva_bitmap, NFS_FATTR_TIME_ACCESS)) { NFS_BITMAP_SET(nvap->nva_bitmap, NFS_FATTR_TIME_ACCESS); nvap->nva_timesec[NFSTIME_ACCESS] = now.tv_sec; - nvap->nva_timensec[NFSTIME_ACCESS] = now.tv_usec * 1000; + nvap->nva_timensec[NFSTIME_ACCESS] = now.tv_nsec; } if (!NFS_BITMAP_ISSET(nvap->nva_bitmap, NFS_FATTR_TIME_MODIFY)) { NFS_BITMAP_SET(nvap->nva_bitmap, NFS_FATTR_TIME_MODIFY); nvap->nva_timesec[NFSTIME_MODIFY] = now.tv_sec; - nvap->nva_timensec[NFSTIME_MODIFY] = now.tv_usec * 1000; + nvap->nva_timensec[NFSTIME_MODIFY] = now.tv_nsec; } if (!NFS_BITMAP_ISSET(nvap->nva_bitmap, NFS_FATTR_TIME_METADATA)) { NFS_BITMAP_SET(nvap->nva_bitmap, NFS_FATTR_TIME_METADATA); nvap->nva_timesec[NFSTIME_CHANGE] = now.tv_sec; - nvap->nva_timensec[NFSTIME_CHANGE] = now.tv_usec * 1000; + nvap->nva_timensec[NFSTIME_CHANGE] = now.tv_nsec; } if (!NFS_BITMAP_ISSET(nvap->nva_bitmap, NFS_FATTR_FILEID)) { NFS_BITMAP_SET(nvap->nva_bitmap, NFS_FATTR_FILEID); @@ -1012,6 +1012,158 @@ nfs4_ace_vfsrights_to_nfsmask(uint32_t vfsrights) return (nfsmask); } +/* + * nfs4_wkid2sidd:: + * mapid a wellknown identity to guid. + * Return 0 on success ENOENT if id does not map and EINVAL if the id is not a well known name. + */ +static int +nfs4_wkid2sid(const char *id, ntsid_t *sp) +{ + size_t len = strnlen(id, MAXIDNAMELEN); + + if (len == MAXIDNAMELEN || id[len-1] != '@') + return (EINVAL); + + bzero(sp, sizeof(ntsid_t)); + sp->sid_kind = 1; + sp->sid_authcount = 1; + if (!strcmp(id, "OWNER@")) { + // S-1-3-0 + sp->sid_authority[5] = 3; + sp->sid_authorities[0] = 0; + } else if (!strcmp(id, "GROUP@")) { + // S-1-3-1 + sp->sid_authority[5] = 3; + sp->sid_authorities[0] = 1; + } else if (!strcmp(id, "EVERYONE@")) { + // S-1-1-0 + sp->sid_authority[5] = 1; + sp->sid_authorities[0] = 0; + } else if (!strcmp(id, "INTERACTIVE@")) { + // S-1-5-4 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 4; + } else if (!strcmp(id, "NETWORK@")) { + // S-1-5-2 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 2; + } else if (!strcmp(id, "DIALUP@")) { + // S-1-5-1 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 1; + } else if (!strcmp(id, "BATCH@")) { + // S-1-5-3 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 3; + } else if (!strcmp(id, "ANONYMOUS@")) { + // S-1-5-7 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 7; + } else if (!strcmp(id, "AUTHENTICATED@")) { + // S-1-5-11 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 11; + } else if (!strcmp(id, "SERVICE@")) { + // S-1-5-6 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 6; + } else { + // S-1-0-0 "NOBODY" + sp->sid_authority[5] = 0; + sp->sid_authorities[0] = 0; + } + return (0); +} + +static int +nfs4_fallback_name(const char *id, int have_at) +{ + if (have_at) { + /* must be user@domain */ + /* try to identify some well-known IDs */ + if (!strncmp(id, "root@", 5)) + return (0); + else if (!strncmp(id, "wheel@", 6)) + return (0); + else if (!strncmp(id, "nobody@", 7)) + return (-2); + else if (!strncmp(id, "nfsnobody@", 10)) + return (-2); + } + return (-2); +} + +static void +nfs4_mapid_log(int error, const char *idstr, int isgroup, guid_t *gp) +{ + if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) + printf("nfs4_id2guid: idmap failed for %s %s error %d\n", idstr, isgroup ? "G" : " ", error); + if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) + printf("nfs4_id2guid: idmap for %s %s got guid " + "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x\n", + idstr, isgroup ? "G" : " ", + gp->g_guid[0], gp->g_guid[1], gp->g_guid[2], gp->g_guid[3], + gp->g_guid[4], gp->g_guid[5], gp->g_guid[6], gp->g_guid[7], + gp->g_guid[8], gp->g_guid[9], gp->g_guid[10], gp->g_guid[11], + gp->g_guid[12], gp->g_guid[13], gp->g_guid[14], gp->g_guid[15]); +} + +static char * +nfs4_map_domain(char *id, char **atp) +{ + char *at = *atp; + char *dsnode, *otw_nfs4domain; + char *new_id = NULL; + size_t otw_domain_len; + size_t otw_id_2_at_len; + int error; + + if (at == NULL) + at = strchr(id, '@'); + if (at == NULL || *at != '@') + return (NULL); + + otw_nfs4domain = at + 1; + otw_domain_len = strnlen(otw_nfs4domain, MAXPATHLEN); + otw_id_2_at_len = at - id + 1; + + MALLOC_ZONE(dsnode, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); + /* first try to map nfs4 domain to dsnode for scoped lookups */ + error = kauth_cred_nfs4domain2dsnode(otw_nfs4domain, dsnode); + if (!error) { + /* Success! Make new id be id@dsnode */ + size_t dsnode_len = strnlen(dsnode, MAXPATHLEN); + size_t new_id_len = otw_id_2_at_len + dsnode_len + 1; + char tmp; + + MALLOC(new_id, char*, new_id_len, M_TEMP, M_WAITOK); + tmp = *otw_nfs4domain; + *otw_nfs4domain = '\0'; /* Chop of the old domain */ + strlcpy(new_id, id, MAXPATHLEN); + *otw_nfs4domain = tmp; /* Be nice and preserve callers original id */ + strlcat(new_id, dsnode, MAXPATHLEN); + at = strchr(new_id, '@'); + } else { + /* Bummer:-( See if default nfs4 set for unscoped lookup */ + size_t default_domain_len = strnlen(nfs4_default_domain, MAXPATHLEN); + + if ((otw_domain_len == default_domain_len) && + (strncmp(otw_nfs4domain, nfs4_default_domain, otw_domain_len) == 0)) { + /* Woohoo! We have matching domains, do unscoped lookups */ + *at = '\0'; + } + } + FREE_ZONE(dsnode, MAXPATHLEN, M_NAMEI); + + if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS) { + printf("nfs4_id2guid: after domain mapping id is %s\n", id); + } + + *atp = at; + return (new_id); +} + /* * Map an NFSv4 ID string to a VFS guid. * @@ -1020,16 +1172,12 @@ nfs4_ace_vfsrights_to_nfsmask(uint32_t vfsrights) int nfs4_id2guid(/*const*/ char *id, guid_t *guidp, int isgroup) { - int error1 = 0, error = 0, compare; - guid_t guid1, guid2, *gp; + int error = 0; ntsid_t sid; - long num, unknown; + long num; char *p, *at, *new_id = NULL; *guidp = kauth_null_guid; - compare = ((nfs_idmap_ctrl & NFS_IDMAP_CTRL_USE_IDMAP_SERVICE) && - (nfs_idmap_ctrl & NFS_IDMAP_CTRL_COMPARE_RESULTS)); - unknown = (nfs_idmap_ctrl & NFS_IDMAP_CTRL_UNKNOWN_IS_99) ? 99 : -2; /* * First check if it is just a simple numeric ID string or a special "XXX@" name. @@ -1046,58 +1194,31 @@ nfs4_id2guid(/*const*/ char *id, guid_t *guidp, int isgroup) at = p; p++; } - if (at && !at[1] && !isgroup) - isgroup = 1; /* special "XXX@" names should always be treated as groups */ + if (num) { /* must be numeric ID (or empty) */ - num = *id ? strtol(id, NULL, 10) : unknown; - gp = guidp; - /* Since we are not initilizing guid1 and guid2, skip compare */ - compare = 0; - goto gotnumid; + num = *id ? strtol(id, NULL, 10) : -2; + if (isgroup) + error = kauth_cred_gid2guid((gid_t)num, guidp); + else + error = kauth_cred_uid2guid((uid_t)num, guidp); + nfs4_mapid_log(error, id, isgroup, guidp); + return (error); + } + + /* See if this is a well known NFSv4 name */ + error = nfs4_wkid2sid(id, &sid); + if (!error) { + error = kauth_cred_ntsid2guid(&sid, guidp); + nfs4_mapid_log(error, id, 1, guidp); + return (error); } /* Handle nfs4 domain first */ if (at && at[1]) { - /* Try mapping nfs4 domain */ - char *dsnode, *nfs4domain = at + 1; - size_t otw_domain_len = strnlen(nfs4domain, MAXPATHLEN); - int otw_id_2_at_len = at - id + 1; - - MALLOC(dsnode, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); - if (dsnode) { - /* first try to map nfs4 domain to dsnode for scoped lookups */ - memset(dsnode, 0, MAXPATHLEN); - error = kauth_cred_nfs4domain2dsnode(nfs4domain, dsnode); - if (!error) { - /* Success! Make new id be id@dsnode */ - int dsnode_len = strnlen(dsnode, MAXPATHLEN); - int new_id_len = otw_id_2_at_len + dsnode_len + 1; - - MALLOC(new_id, char*, new_id_len, M_NAMEI, M_WAITOK); - if (new_id) { - (void)strlcpy(new_id, id, otw_id_2_at_len + 1); - (void)strlcpy(new_id + otw_id_2_at_len, dsnode, dsnode_len + 1); - id = new_id; - at = id; - while (*at++ != '@'); - at--; - } - } else { - /* Bummer:-( See if default nfs4 set for unscoped lookup */ - size_t default_domain_len = strnlen(nfs4_domain, MAXPATHLEN); - - if ((otw_domain_len == default_domain_len) && (strncmp(nfs4domain, nfs4_domain, otw_domain_len) == 0)) { - /* Woohoo! We have matching domains, do unscoped lookups */ - *at = '\0'; - } - } - FREE(dsnode, M_NAMEI); - } - - if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS) { - printf("nfs4_id2guid: after domain mapping id is %s\n", id); - } + new_id = nfs4_map_domain(id, &at); + if (new_id) + id = new_id; } /* Now try to do actual id mapping */ @@ -1107,197 +1228,238 @@ nfs4_id2guid(/*const*/ char *id, guid_t *guidp, int isgroup) * * [sigh] this isn't a "pwnam/grnam" it's an NFS ID string! */ - gp = compare ? &guid1 : guidp; if (isgroup) - error = kauth_cred_grnam2guid(id, gp); + error = kauth_cred_grnam2guid(id, guidp); else - error = kauth_cred_pwnam2guid(id, gp); - if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) - printf("nfs4_id2guid: idmap failed for %s %s error %d\n", id, isgroup ? "G" : " ", error); - if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) - printf("nfs4_id2guid: idmap for %s %s got guid " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x\n", - id, isgroup ? "G" : " ", - gp->g_guid[0], gp->g_guid[1], gp->g_guid[2], gp->g_guid[3], - gp->g_guid[4], gp->g_guid[5], gp->g_guid[6], gp->g_guid[7], - gp->g_guid[8], gp->g_guid[9], gp->g_guid[10], gp->g_guid[11], - gp->g_guid[12], gp->g_guid[13], gp->g_guid[14], gp->g_guid[15]); - error1 = error; - } - if (error || compare || !(nfs_idmap_ctrl & NFS_IDMAP_CTRL_USE_IDMAP_SERVICE)) { + error = kauth_cred_pwnam2guid(id, guidp); + nfs4_mapid_log(error, id, isgroup, guidp); + } else { + error = ENOTSUP; + } + + if (error) { /* * fallback path... see if we can come up with an answer ourselves. */ - gp = compare ? &guid2 : guidp; - - if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_WELLKNOWN_IDS) && at && !at[1]) { - /* must be a special ACE "who" ID */ - bzero(&sid, sizeof(sid)); - sid.sid_kind = 1; - sid.sid_authcount = 1; - if (!strcmp(id, "OWNER@")) { - // S-1-3-0 - sid.sid_authority[5] = 3; - sid.sid_authorities[0] = 0; - } else if (!strcmp(id, "GROUP@")) { - // S-1-3-1 - sid.sid_authority[5] = 3; - sid.sid_authorities[0] = 1; - } else if (!strcmp(id, "EVERYONE@")) { - // S-1-1-0 - sid.sid_authority[5] = 1; - sid.sid_authorities[0] = 0; - } else if (!strcmp(id, "INTERACTIVE@")) { - // S-1-5-4 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 4; - } else if (!strcmp(id, "NETWORK@")) { - // S-1-5-2 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 2; - } else if (!strcmp(id, "DIALUP@")) { - // S-1-5-1 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 1; - } else if (!strcmp(id, "BATCH@")) { - // S-1-5-3 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 3; - } else if (!strcmp(id, "ANONYMOUS@")) { - // S-1-5-7 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 7; - } else if (!strcmp(id, "AUTHENTICATED@")) { - // S-1-5-11 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 11; - } else if (!strcmp(id, "SERVICE@")) { - // S-1-5-6 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 6; - } else { - // S-1-0-0 "NOBODY" - sid.sid_authority[5] = 0; - sid.sid_authorities[0] = 0; - } - error = kauth_cred_ntsid2guid(&sid, gp); - } else { - if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_COMMON_IDS) && at) { - /* must be user@domain */ - /* try to identify some well-known IDs */ - if (!strncmp(id, "root@", 5)) - num = 0; - else if (!strncmp(id, "wheel@", 6)) - num = 0; - else if (!strncmp(id, "nobody@", 7)) - num = -2; - else if (!strncmp(id, "nfsnobody@", 10)) - num = -2; - else - num = unknown; - } else if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_COMMON_IDS) && !strcmp(id, "nobody")) { - num = -2; - } else { - num = unknown; - } -gotnumid: - if (isgroup) - error = kauth_cred_gid2guid((gid_t)num, gp); - else - error = kauth_cred_uid2guid((uid_t)num, gp); - } - if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) - printf("nfs4_id2guid: fallback map failed for %s %s error %d\n", id, isgroup ? "G" : " ", error); - if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) - printf("nfs4_id2guid: fallback map for %s %s got guid " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x\n", - id, isgroup ? "G" : " ", - gp->g_guid[0], gp->g_guid[1], gp->g_guid[2], gp->g_guid[3], - gp->g_guid[4], gp->g_guid[5], gp->g_guid[6], gp->g_guid[7], - gp->g_guid[8], gp->g_guid[9], gp->g_guid[10], gp->g_guid[11], - gp->g_guid[12], gp->g_guid[13], gp->g_guid[14], gp->g_guid[15]); - } - - if (compare) { - /* compare the results, log if different */ - if (!error1 && !error) { - if (!kauth_guid_equal(&guid1, &guid2)) - printf("nfs4_id2guid: idmap/fallback results differ for %s %s - " - "idmap %02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x " - "fallback %02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x\n", - id, isgroup ? "G" : " ", - guid1.g_guid[0], guid1.g_guid[1], guid1.g_guid[2], guid1.g_guid[3], - guid1.g_guid[4], guid1.g_guid[5], guid1.g_guid[6], guid1.g_guid[7], - guid1.g_guid[8], guid1.g_guid[9], guid1.g_guid[10], guid1.g_guid[11], - guid1.g_guid[12], guid1.g_guid[13], guid1.g_guid[14], guid1.g_guid[15], - guid2.g_guid[0], guid2.g_guid[1], guid2.g_guid[2], guid2.g_guid[3], - guid2.g_guid[4], guid2.g_guid[5], guid2.g_guid[6], guid2.g_guid[7], - guid2.g_guid[8], guid2.g_guid[9], guid2.g_guid[10], guid2.g_guid[11], - guid2.g_guid[12], guid2.g_guid[13], guid2.g_guid[14], guid2.g_guid[15]); - /* copy idmap result to output guid */ - *guidp = guid1; - } else if (error1 && !error) { - printf("nfs4_id2guid: idmap/fallback results differ for %s %s - " - "idmap error %d " - "fallback %02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x\n", - id, isgroup ? "G" : " ", - error1, - guid2.g_guid[0], guid2.g_guid[1], guid2.g_guid[2], guid2.g_guid[3], - guid2.g_guid[4], guid2.g_guid[5], guid2.g_guid[6], guid2.g_guid[7], - guid2.g_guid[8], guid2.g_guid[9], guid2.g_guid[10], guid2.g_guid[11], - guid2.g_guid[12], guid2.g_guid[13], guid2.g_guid[14], guid2.g_guid[15]); - /* copy fallback result to output guid */ - *guidp = guid2; - } else if (!error1 && error) { - printf("nfs4_id2guid: idmap/fallback results differ for %s %s - " - "idmap %02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x " - "fallback error %d\n", - id, isgroup ? "G" : " ", - guid1.g_guid[0], guid1.g_guid[1], guid1.g_guid[2], guid1.g_guid[3], - guid1.g_guid[4], guid1.g_guid[5], guid1.g_guid[6], guid1.g_guid[7], - guid1.g_guid[8], guid1.g_guid[9], guid1.g_guid[10], guid1.g_guid[11], - guid1.g_guid[12], guid1.g_guid[13], guid1.g_guid[14], guid1.g_guid[15], - error); - /* copy idmap result to output guid */ - *guidp = guid1; - error = 0; - } else { - if (error1 != error) - printf("nfs4_id2guid: idmap/fallback results differ for %s %s - " - "idmap error %d fallback error %d\n", - id, isgroup ? "G" : " ", error1, error); - } + num = nfs4_fallback_name(id, at != NULL); + if (isgroup) + error = kauth_cred_gid2guid((gid_t)num, guidp); + else + error = kauth_cred_uid2guid((uid_t)num, guidp); + nfs4_mapid_log(error, id, isgroup, guidp); } + /* restore @ symbol in case we clobered for unscoped lookup */ if (at && *at == '\0') *at = '@'; /* free mapped domain id string */ - if (id == new_id) - FREE(id, M_NAMEI); + if (new_id) + FREE(new_id, M_TEMP); + + return (error); +} + +/* + * nfs4_sid2wkid: + * mapid a wellknown identity to guid. + * returns well known name for the sid or NULL if sid does not map. + */ +#define MAXWELLKNOWNID 18 + +static const char* +nfs4_sid2wkid(ntsid_t *sp) +{ + if ((sp->sid_kind == 1) && (sp->sid_authcount == 1)) { + /* check if it's one of our well-known ACE WHO names */ + if (sp->sid_authority[5] == 0) { + if (sp->sid_authorities[0] == 0) // S-1-0-0 + return ("nobody@localdomain"); + } else if (sp->sid_authority[5] == 1) { + if (sp->sid_authorities[0] == 0) // S-1-1-0 + return ("EVERYONE@"); + } else if (sp->sid_authority[5] == 3) { + if (sp->sid_authorities[0] == 0) // S-1-3-0 + return ("OWNER@"); + else if (sp->sid_authorities[0] == 1) // S-1-3-1 + return ("GROUP@"); + } else if (sp->sid_authority[5] == 5) { + if (sp->sid_authorities[0] == 1) // S-1-5-1 + return ("DIALUP@"); + else if (sp->sid_authorities[0] == 2) // S-1-5-2 + return ("NETWORK@"); + else if (sp->sid_authorities[0] == 3) // S-1-5-3 + return ("BATCH@"); + else if (sp->sid_authorities[0] == 4) // S-1-5-4 + return ("INTERACTIVE@"); + else if (sp->sid_authorities[0] == 6) // S-1-5-6 + return ("SERVICE@"); + else if (sp->sid_authorities[0] == 7) // S-1-5-7 + return ("ANONYMOUS@"); + else if (sp->sid_authorities[0] == 11) // S-1-5-11 + return ("AUTHENTICATED@"); + } + } + return (NULL); +} + +static void +nfs4_mapguid_log(int error, const char *where, guid_t *gp, int isgroup, const char *idstr) +{ + if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) + printf("nfs4_guid2id: %s idmap failed for " + "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " + "error %d\n", where, + gp->g_guid[0], gp->g_guid[1], gp->g_guid[2], gp->g_guid[3], + gp->g_guid[4], gp->g_guid[5], gp->g_guid[6], gp->g_guid[7], + gp->g_guid[8], gp->g_guid[9], gp->g_guid[10], gp->g_guid[11], + gp->g_guid[12], gp->g_guid[13], gp->g_guid[14], gp->g_guid[15], + isgroup ? "G" : " ", error); + if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) + printf("nfs4_guid2id: %s idmap for " + "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " + "got ID %s\n", where, + gp->g_guid[0], gp->g_guid[1], gp->g_guid[2], gp->g_guid[3], + gp->g_guid[4], gp->g_guid[5], gp->g_guid[6], gp->g_guid[7], + gp->g_guid[8], gp->g_guid[9], gp->g_guid[10], gp->g_guid[11], + gp->g_guid[12], gp->g_guid[13], gp->g_guid[14], gp->g_guid[15], + isgroup ? "G" : " ", idstr); +} + +static int +nfs4_addv4domain(char *id, size_t *idlen) +{ + char *at = NULL, *cp; + int have_domain; + int error = 0; + size_t idsize; + + + if (id == NULL || *id == '\0') + return (EINVAL); + + for (cp = id; *cp != '\0'; cp++) { + if (*cp == '@') { + at = cp; + break; + } + } + + have_domain = (at && at[1] != '\0'); + + if (have_domain) { + char *dsnode = at + 1; + char *nfs4domain; + size_t domain_len; + char *mapped_domain; + + MALLOC_ZONE(nfs4domain, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); + error = kauth_cred_dsnode2nfs4domain(dsnode, nfs4domain); + if (!error) { + domain_len = strnlen(nfs4domain, MAXPATHLEN); + mapped_domain = nfs4domain; + } else { + error = 0; + domain_len = strnlen(nfs4_default_domain, MAXPATHLEN); + mapped_domain = nfs4_default_domain; + } + if (domain_len) { + /* chop off id after the '@' */ + at[1] = '\0'; + /* Add our mapped_domain */ + idsize = strlcat(id, mapped_domain, *idlen); + if (*idlen > idsize) + *idlen = idsize; + else + error = ENOSPC; + } + FREE_ZONE(nfs4domain, MAXPATHLEN, M_NAMEI); + } else if (at == NULL) { + /* + * If we didn't find an 'at' then cp points to the end of id passed in. + * and if we have a nfs4_default_domain set. Try to append the + * default domain if we have root or set ENOSPC. + */ + size_t default_domain_len = strnlen(nfs4_default_domain, MAXPATHLEN); + + if (default_domain_len) { + strlcat(id, "@", *idlen); + idsize = strlcat(id, nfs4_default_domain, *idlen); + if (*idlen > idsize) + *idlen = idsize; + else + error = ENOSPC; + } else { + ; /* Unscoped name otw */ + } + } + + if (!error && nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS) + printf("nfs4_guid2id: id after nfs4 domain map: %s[%zd].\n", id, *idlen); return (error); } +static char * +nfs4_fallback_id(int numid, int isgrp, char *buf, size_t size) +{ + const char *idp = NULL; + + if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_COMMON_IDS)) { + /* map well known uid's to strings */ + if (numid == 0) + idp = isgrp ? "wheel" : "root"; + else if (numid == -2) + idp = "nobody"; + } + if (!idp) { + /* or just use a decimal number string. */ + snprintf(buf, size-1, "%d", numid); + buf[size-1] = '\0'; + } else { + size_t idplen = strlcpy(buf, idp, size); + if (idplen >= size) + return (NULL); + } + + return (buf); +} + /* * Map a VFS guid to an NFSv4 ID string. * * Try to use the ID mapping service... but we may fallback to trying to do it ourselves. */ int -nfs4_guid2id(guid_t *guidp, char *id, int *idlen, int isgroup) +nfs4_guid2id(guid_t *guidp, char *id, size_t *idlen, int isgroup) { - int error1 = 0, error = 0, compare; - int id1len, id2len, len; - char *id1buf, *id1, *at; + int error = 0; + size_t id1len, len; + char *id1buf, *id1; char numbuf[32]; - const char *id2 = NULL; + ntsid_t sid; id1buf = id1 = NULL; - id1len = id2len = 0; - compare = ((nfs_idmap_ctrl & NFS_IDMAP_CTRL_USE_IDMAP_SERVICE) && - (nfs_idmap_ctrl & NFS_IDMAP_CTRL_COMPARE_RESULTS)); + id1len = 0; + + /* + * See if our guid maps to a well known NFSv4 name + */ + error = kauth_cred_guid2ntsid(guidp, &sid); + if (!error) { + const char *wkid = nfs4_sid2wkid(&sid); + if (wkid) { + len = strnlen(wkid, MAXWELLKNOWNID); + strlcpy(id, wkid, *idlen); + error = (len < *idlen) ? 0 : ENOSPC; + *idlen = len; + nfs4_mapguid_log(error, "kauth_cred_guid2ntsid", guidp, 1, id); + return (error); + } + } else { + nfs4_mapguid_log(error, "kauth_cred_guid2ntsid", guidp, isgroup, NULL); + } if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_USE_IDMAP_SERVICE) { /* @@ -1311,10 +1473,9 @@ nfs4_guid2id(guid_t *guidp, char *id, int *idlen, int isgroup) * be at least MAXPATHLEN bytes long even though most if not all ID * strings will be much much shorter than that. */ - if (compare || (*idlen < MAXPATHLEN)) { + + if (*idlen < MAXPATHLEN) { MALLOC_ZONE(id1buf, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); - if (!id1buf) - return (ENOMEM); id1 = id1buf; id1len = MAXPATHLEN; } else { @@ -1322,243 +1483,54 @@ nfs4_guid2id(guid_t *guidp, char *id, int *idlen, int isgroup) id1len = *idlen; } - memset(id1, 0, id1len); if (isgroup) error = kauth_cred_guid2grnam(guidp, id1); else error = kauth_cred_guid2pwnam(guidp, id1); - if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) - printf("nfs4_guid2id: idmap failed for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "error %d\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", error); - if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) - printf("nfs4_guid2id: idmap for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "got ID %s\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", id1); - error1 = error; - if (!error) { - if (compare) { - id1len = strnlen(id1, id1len); - } else if (id1 == id1buf) { - /* copy idmap result to output buffer */ - len = strlcpy(id, id1, *idlen); - if (len >= *idlen) - error = ENOSPC; - else - *idlen = len; - } - } + if (error) + nfs4_mapguid_log(error, "kauth_cred2[pw|gr]nam", guidp, isgroup, id1); + } else { + error = ENOTSUP; } - if (error || compare || !(nfs_idmap_ctrl & NFS_IDMAP_CTRL_USE_IDMAP_SERVICE)) { + + if (error) { /* * fallback path... see if we can come up with an answer ourselves. */ - ntsid_t sid; uid_t uid; - if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_WELLKNOWN_IDS)) { - error = kauth_cred_guid2ntsid(guidp, &sid); - if (!error && (sid.sid_kind == 1) && (sid.sid_authcount == 1)) { - /* check if it's one of our well-known ACE WHO names */ - if (sid.sid_authority[5] == 0) { - if (sid.sid_authorities[0] == 0) // S-1-0-0 - id2 = "nobody@localdomain"; - } else if (sid.sid_authority[5] == 1) { - if (sid.sid_authorities[0] == 0) // S-1-1-0 - id2 = "EVERYONE@"; - } else if (sid.sid_authority[5] == 3) { - if (sid.sid_authorities[0] == 0) // S-1-3-0 - id2 = "OWNER@"; - else if (sid.sid_authorities[0] == 1) // S-1-3-1 - id2 = "GROUP@"; - } else if (sid.sid_authority[5] == 5) { - if (sid.sid_authorities[0] == ntohl(1)) // S-1-5-1 - id2 = "DIALUP@"; - else if (sid.sid_authorities[0] == ntohl(2)) // S-1-5-2 - id2 = "NETWORK@"; - else if (sid.sid_authorities[0] == ntohl(3)) // S-1-5-3 - id2 = "BATCH@"; - else if (sid.sid_authorities[0] == ntohl(4)) // S-1-5-4 - id2 = "INTERACTIVE@"; - else if (sid.sid_authorities[0] == ntohl(6)) // S-1-5-6 - id2 = "SERVICE@"; - else if (sid.sid_authorities[0] == ntohl(7)) // S-1-5-7 - id2 = "ANONYMOUS@"; - else if (sid.sid_authorities[0] == ntohl(11)) // S-1-5-11 - id2 = "AUTHENTICATED@"; - } - } - } - if (!id2) { - /* OK, let's just try mapping it to a UID/GID */ - if (isgroup) - error = kauth_cred_guid2gid(guidp, (gid_t*)&uid); + /* OK, let's just try mapping it to a UID/GID */ + if (isgroup) + error = kauth_cred_guid2gid(guidp, (gid_t*)&uid); + else + error = kauth_cred_guid2uid(guidp, &uid); + if (!error) { + char *fbidp = nfs4_fallback_id(uid, isgroup, numbuf, sizeof(numbuf)); + if (fbidp == NULL) + error = ENOSPC; else - error = kauth_cred_guid2uid(guidp, &uid); - if (!error) { - if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_COMMON_IDS)) { - /* map well known uid's to strings */ - if (uid == 0) - id2 = isgroup ? "wheel@localdomain" : "root@localdomain"; - else if (uid == (uid_t)-2) - id2 = "nobody@localdomain"; - } - if (!id2) { - /* or just use a decimal number string. */ - snprintf(numbuf, sizeof(numbuf), "%d", uid); - id2 = numbuf; - } - } - } - if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) - printf("nfs4_guid2id: fallback map failed for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "error %d\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", error); - if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) - printf("nfs4_guid2id: fallback map for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "got ID %s\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", id2); - if (!error && id2) { - if (compare) { - id2len = strnlen(id2, MAXPATHLEN); - } else { - /* copy fallback result to output buffer */ - len = strlcpy(id, id2, *idlen); - if (len >= *idlen) - error = ENOSPC; - else - *idlen = len; - } + id1 = fbidp; } + } else { + error = nfs4_addv4domain(id1, &id1len); } - if (compare) { - /* compare the results, log if different */ - if (!error1 && !error) { - if ((id1len != id2len) || strncmp(id1, id2, id1len)) - printf("nfs4_guid2id: idmap/fallback results differ for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "idmap %s fallback %s\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", id1, id2); - if (id1 == id1buf) { - /* copy idmap result to output buffer */ - len = strlcpy(id, id1, *idlen); - if (len >= *idlen) - error = ENOSPC; - else - *idlen = len; - } - } else if (error1 && !error) { - printf("nfs4_guid2id: idmap/fallback results differ for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "idmap error %d fallback %s\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", error1, id2); - /* copy fallback result to output buffer */ - len = strlcpy(id, id2, *idlen); + if (!error) { + + if (id1 != id) { + /* copy idmap result to output buffer */ + len = strlcpy(id, id1, *idlen); if (len >= *idlen) error = ENOSPC; else *idlen = len; - } else if (!error1 && error) { - printf("nfs4_guid2id: idmap/fallback results differ for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "idmap %s fallback error %d\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", id1, error); - if (id1 == id1buf) { - /* copy idmap result to output buffer */ - len = strlcpy(id, id1, *idlen); - if (len >= *idlen) - error = ENOSPC; - else - *idlen = len; - } - error = 0; - } else { - if (error1 != error) - printf("nfs4_guid2id: idmap/fallback results differ for %s %s - " - "idmap error %d fallback error %d\n", - id, isgroup ? "G" : " ", error1, error); - } - } - - at = id; - while (at && at[0] != '@' && at[0] != '\0' && at++); - if (at && at[0] == '@' && at[1] != '\0') { - char *dsnode = at + 1; - int id_2_at_len = at - id + 1; - char *nfs4domain, *new_id; - MALLOC(nfs4domain, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); - if (nfs4domain) { - int domain_len; - char *mapped_domain; - memset(nfs4domain, 0, MAXPATHLEN); - error = kauth_cred_dsnode2nfs4domain(dsnode, nfs4domain); - if (!error) { - domain_len = strnlen(nfs4domain, MAXPATHLEN); - mapped_domain = nfs4domain; - } else { - domain_len = strnlen(nfs4_domain, MAXPATHLEN); - mapped_domain = nfs4_domain; - } - if (domain_len) { - MALLOC(new_id, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); - if (new_id) { - strlcpy(new_id, id, id_2_at_len + 1); - strlcpy(new_id + id_2_at_len, mapped_domain, domain_len + 1); - strlcpy(id, new_id, strnlen(new_id, MAXPATHLEN) + 1); - *idlen = strnlen(id, MAXPATHLEN); - FREE(new_id, M_NAMEI); - } - } - FREE(nfs4domain, M_NAMEI); - } - } else if (at && at[0] == '\0') { - int default_domain_len = strnlen(nfs4_domain, MAXPATHLEN); - - if (default_domain_len && MAXPATHLEN - *idlen > default_domain_len) { - at[0] = '@'; - strlcpy(at + 1, nfs4_domain, default_domain_len + 1); - *idlen = strnlen(id, MAXPATHLEN); } } - - if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS) - printf("nfs4_guid2id: id after nfs4 domain map: %s[%d].\n", id, *idlen); + nfs4_mapguid_log(error, "End of routine", guidp, isgroup, id1); if (id1buf) FREE_ZONE(id1buf, MAXPATHLEN, M_NAMEI); + return (error); } @@ -2119,7 +2091,7 @@ nfs4_parsefattr( error = kauth_cred_guid2uid(&nvap->nva_uuuid, &nvap->nva_uid); if (error) { /* unable to get either GUID or UID, set to default */ - nvap->nva_uid = (uid_t)((nfs_idmap_ctrl & NFS_IDMAP_CTRL_UNKNOWN_IS_99) ? 99 : -2); + nvap->nva_uid = (uid_t)(-2); if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS) printf("nfs4_parsefattr: owner %s is no one, no %s?, error %d\n", s, kauth_guid_equal(&nvap->nva_uuuid, &kauth_null_guid) ? "guid" : "uid", @@ -2154,7 +2126,7 @@ nfs4_parsefattr( error = kauth_cred_guid2gid(&nvap->nva_guuid, &nvap->nva_gid); if (error) { /* unable to get either GUID or GID, set to default */ - nvap->nva_gid = (gid_t)((nfs_idmap_ctrl & NFS_IDMAP_CTRL_UNKNOWN_IS_99) ? 99 : -2); + nvap->nva_gid = (gid_t)(-2); if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS) printf("nfs4_parsefattr: group %s is no one, no %s?, error %d\n", s, kauth_guid_equal(&nvap->nva_guuid, &kauth_null_guid) ? "guid" : "gid", @@ -2274,7 +2246,8 @@ nfsmout: int nfsm_chain_add_fattr4_f(struct nfsm_chain *nmc, struct vnode_attr *vap, struct nfsmount *nmp) { - int error = 0, attrbytes, slen, len, i, isgroup; + int error = 0, attrbytes, i, isgroup; + size_t slen, len; uint32_t *pattrbytes, val, acecount;; uint32_t bitmap[NFS_ATTR_BITMAP_LEN]; char sbuf[64], *s; @@ -2314,11 +2287,12 @@ nfsm_chain_add_fattr4_f(struct nfsm_chain *nmc, struct vnode_attr *vap, struct n val = nfs4_ace_vfstype_to_nfstype(val, &error); nfsm_chain_add_32(error, nmc, val); val = nfs4_ace_vfsflags_to_nfsflags(acl->acl_ace[i].ace_flags); + isgroup = (kauth_cred_guid2gid(&acl->acl_ace[i].ace_applicable, &gid) == 0); + val |= (isgroup) ? NFS_ACE_IDENTIFIER_GROUP : 0; nfsm_chain_add_32(error, nmc, val); val = nfs4_ace_vfsrights_to_nfsmask(acl->acl_ace[i].ace_rights); nfsm_chain_add_32(error, nmc, val); len = slen; - isgroup = (kauth_cred_guid2gid(&acl->acl_ace[i].ace_applicable, &gid) == 0); error = nfs4_guid2id(&acl->acl_ace[i].ace_applicable, s, &len, isgroup); if (error == ENOSPC) { if (s != sbuf) { diff --git a/bsd/nfs/nfs4_vnops.c b/bsd/nfs/nfs4_vnops.c index 13e4ad3ff..d0812cfaa 100644 --- a/bsd/nfs/nfs4_vnops.c +++ b/bsd/nfs/nfs4_vnops.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2015 Apple Inc. All rights reserved. + * Copyright (c) 2006-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1401,6 +1401,8 @@ nfs4_vnop_getattr( if (error) return (error); + vap->va_flags |= VA_64BITOBJIDS; + /* copy what we have in nva to *a_vap */ if (VATTR_IS_ACTIVE(vap, va_rdev) && NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_RAWDEV)) { dev_t rdev = makedev(nva.nva_rawdev.specdata1, nva.nva_rawdev.specdata2); diff --git a/bsd/nfs/nfs_bio.c b/bsd/nfs/nfs_bio.c index acaf26c24..d65d98a1a 100644 --- a/bsd/nfs/nfs_bio.c +++ b/bsd/nfs/nfs_bio.c @@ -409,8 +409,8 @@ nfs_buf_upl_setup(struct nfsbuf *bp) */ upl_flags |= UPL_WILL_MODIFY; } - kret = ubc_create_upl(NFSTOV(bp->nb_np), NBOFF(bp), bp->nb_bufsize, - &upl, NULL, upl_flags); + kret = ubc_create_upl_kernel(NFSTOV(bp->nb_np), NBOFF(bp), bp->nb_bufsize, + &upl, NULL, upl_flags, VM_KERN_MEMORY_FILE); if (kret == KERN_INVALID_ARGUMENT) { /* vm object probably doesn't exist any more */ bp->nb_pagelist = NULL; diff --git a/bsd/nfs/nfs_gss.c b/bsd/nfs/nfs_gss.c index 53f4a08f2..0c97dda9c 100644 --- a/bsd/nfs/nfs_gss.c +++ b/bsd/nfs/nfs_gss.c @@ -629,7 +629,7 @@ nfs_gss_clnt_mnt_rele(struct nfsmount *nmp) } } -int nfs_root_steals_ctx = 1; +int nfs_root_steals_ctx = 0; static int nfs_gss_clnt_ctx_find_principal(struct nfsreq *req, uint8_t *principal, uint32_t plen, uint32_t nt) @@ -1834,6 +1834,12 @@ nfs_gss_clnt_gssd_upcall(struct nfsreq *req, struct nfs_gss_clnt_ctx *cp, uint32 cp->gss_clnt_token = NULL; cp->gss_clnt_tokenlen = 0; cp->gss_clnt_proc = RPCSEC_GSS_INIT; + /* Server's handle isn't valid. Don't reuse */ + cp->gss_clnt_handle_len = 0; + if (cp->gss_clnt_handle != NULL) { + FREE(cp->gss_clnt_handle, M_TEMP); + cp->gss_clnt_handle = NULL; + } } NFS_GSS_DBG("Retrycnt = %d nm_etype.count = %d\n", retrycnt, nmp->nm_etype.count); @@ -1879,10 +1885,7 @@ nfs_gss_clnt_gssd_upcall(struct nfsreq *req, struct nfs_gss_clnt_ctx *cp, uint32 nt = cp->gss_clnt_prinnt; } else if (nmp->nm_principal && IS_VALID_CRED(nmp->nm_mcred) && req->r_cred == nmp->nm_mcred) { plen = (uint32_t)strlen(nmp->nm_principal); - MALLOC(principal, uint8_t *, plen, M_TEMP, M_WAITOK | M_ZERO); - if (principal == NULL) - return (ENOMEM); - bcopy(nmp->nm_principal, principal, plen); + principal = (uint8_t *)nmp->nm_principal; cp->gss_clnt_prinnt = nt = GSSD_USER; } else if (nmp->nm_realm) { @@ -1978,6 +1981,12 @@ skip: cp->gss_clnt_major != GSS_S_CONTINUE_NEEDED) { NFS_GSS_DBG("Up call returned error\n"); nfs_gss_clnt_log_error(req, cp, major, minor); + /* Server's handle isn't valid. Don't reuse */ + cp->gss_clnt_handle_len = 0; + if (cp->gss_clnt_handle != NULL) { + FREE(cp->gss_clnt_handle, M_TEMP); + cp->gss_clnt_handle = NULL; + } } if (lucidlen > 0) { @@ -2045,6 +2054,12 @@ out: FREE(cp->gss_clnt_token, M_TEMP); cp->gss_clnt_token = NULL; cp->gss_clnt_tokenlen = 0; + /* Server's handle isn't valid. Don't reuse */ + cp->gss_clnt_handle_len = 0; + if (cp->gss_clnt_handle != NULL) { + FREE(cp->gss_clnt_handle, M_TEMP); + cp->gss_clnt_handle = NULL; + } NFS_GSS_DBG("Up call returned NFSERR_EAUTH"); return (NFSERR_EAUTH); @@ -3728,18 +3743,18 @@ nfs_gss_mach_alloc_buffer(u_char *buf, uint32_t buflen, vm_map_copy_t *addr) tbuflen = vm_map_round_page(buflen, vm_map_page_mask(ipc_kernel_map)); - kr = vm_allocate(ipc_kernel_map, &kmem_buf, tbuflen, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_FILE)); + kr = vm_allocate_kernel(ipc_kernel_map, &kmem_buf, tbuflen, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_FILE); if (kr != 0) { printf("nfs_gss_mach_alloc_buffer: vm_allocate failed\n"); return; } - kr = vm_map_wire(ipc_kernel_map, + kr = vm_map_wire_kernel(ipc_kernel_map, vm_map_trunc_page(kmem_buf, vm_map_page_mask(ipc_kernel_map)), vm_map_round_page(kmem_buf + tbuflen, vm_map_page_mask(ipc_kernel_map)), - VM_PROT_READ|VM_PROT_WRITE|VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE), FALSE); + VM_PROT_READ|VM_PROT_WRITE, VM_KERN_MEMORY_FILE, FALSE); if (kr != 0) { printf("nfs_gss_mach_alloc_buffer: vm_map_wire failed\n"); return; diff --git a/bsd/nfs/nfs_ioctl.h b/bsd/nfs/nfs_ioctl.h index ff4f3eaae..f140e3ea8 100644 --- a/bsd/nfs/nfs_ioctl.h +++ b/bsd/nfs/nfs_ioctl.h @@ -38,12 +38,10 @@ * fsctl (vnop_ioctl) to detroy the callers credentials associated with the vnode's mount */ #define NFS_IOC_DESTROY_CRED _IO('n', 1) -#define NFS_FSCTL_DESTROY_CRED IOCBASECMD(NFS_IOC_DESTROY_CRED) + /* - * fsclt (vnop_ioctl) to set the callers credentials associated with the vnode's mount + * fsctl (vnop_ioctl) to set the callers credentials associated with the vnode's mount */ - - struct nfs_gss_principal { uint32_t princlen; /* length of data */ @@ -62,7 +60,7 @@ struct user_nfs_gss_principal { uint32_t princlen; /* length of data */ uint32_t nametype; /* nametype of data */ - user_addr_t principal; /* principal data in userspace */ + user64_addr_t principal; /* principal data in userspace */ uint32_t flags; /* Returned flags */ }; #endif @@ -72,8 +70,14 @@ struct user_nfs_gss_principal #define NFS_IOC_INVALID_CRED_FLAG 2 /* Found a credential, but its not valid */ #define NFS_IOC_SET_CRED _IOW('n', 2, struct nfs_gss_principal) -#define NFS_FSCTL_SET_CRED IOCBASECMD(NFS_IOC_SET_CRED) #define NFS_IOC_GET_CRED _IOWR('n', 3, struct nfs_gss_principal) -#define NFS_FSCTL_GET_CRED IOCBASECMD(NFS_IOC_GET_CRED) + +#ifdef KERNEL + +#define NFS_IOC_SET_CRED64 _IOW('n', 2, struct user_nfs_gss_principal) + +#define NFS_IOC_GET_CRED64 _IOWR('n', 3, struct user_nfs_gss_principal) +#endif + #endif diff --git a/bsd/nfs/nfs_lock.c b/bsd/nfs/nfs_lock.c index aaf567271..9920f3c89 100644 --- a/bsd/nfs/nfs_lock.c +++ b/bsd/nfs/nfs_lock.c @@ -607,7 +607,7 @@ wait_for_granted: ((lastmsg + nmp->nm_tprintf_delay) < now.tv_sec)) { lck_mtx_unlock(&nmp->nm_lock); lastmsg = now.tv_sec; - nfs_down(nmp, thd, 0, NFSSTA_LOCKTIMEO, "lockd not responding", 0); + nfs_down(nmp, thd, 0, NFSSTA_LOCKTIMEO, "lockd not responding", 1); wentdown = 1; } else lck_mtx_unlock(&nmp->nm_lock); diff --git a/bsd/nfs/nfs_socket.c b/bsd/nfs/nfs_socket.c index 023163518..5ade6666b 100644 --- a/bsd/nfs/nfs_socket.c +++ b/bsd/nfs/nfs_socket.c @@ -3727,7 +3727,7 @@ nfs_request_destroy(struct nfsreq *req) if (!req || !(req->r_flags & R_INITTED)) return; - nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp; + nmp = req->r_nmp; req->r_flags &= ~R_INITTED; if (req->r_lflags & RL_QUEUED) nfs_reqdequeue(req); @@ -3851,7 +3851,7 @@ nfs_request_add_header(struct nfsreq *req) req->r_mhead = NULL; } - nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp; + nmp = req->r_nmp; if (nfs_mount_gone(nmp)) return (ENXIO); @@ -3860,7 +3860,7 @@ nfs_request_add_header(struct nfsreq *req) return (error); req->r_mreqlen = mbuf_pkthdr_len(req->r_mhead); - nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp; + nmp = req->r_nmp; if (nfs_mount_gone(nmp)) return (ENXIO); lck_mtx_lock(&nmp->nm_lock); @@ -3889,7 +3889,7 @@ nfs_request_send(struct nfsreq *req, int wait) lck_mtx_lock(nfs_request_mutex); - nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp; + nmp = req->r_nmp; if (nfs_mount_gone(nmp)) { lck_mtx_unlock(nfs_request_mutex); return (ENXIO); @@ -3963,7 +3963,7 @@ nfs_request_finish( mrep = req->r_nmrep.nmc_mhead; - nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp; + nmp = req->r_nmp; if ((req->r_flags & R_CWND) && nmp) { /* diff --git a/bsd/nfs/nfs_subs.c b/bsd/nfs/nfs_subs.c index d9d6e016b..266710149 100644 --- a/bsd/nfs/nfs_subs.c +++ b/bsd/nfs/nfs_subs.c @@ -3240,7 +3240,7 @@ nfsrv_check_exports_allow_address(mbuf_t nam) { struct nfs_exportfs *nxfs; struct nfs_export *nx; - struct nfs_export_options *nxo; + struct nfs_export_options *nxo = NULL; if (nam == NULL) return (EACCES); diff --git a/bsd/nfs/nfs_syscalls.c b/bsd/nfs/nfs_syscalls.c index d4dead825..2a20ee2ea 100644 --- a/bsd/nfs/nfs_syscalls.c +++ b/bsd/nfs/nfs_syscalls.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -173,7 +173,7 @@ SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, squishy_flags, CTLFLAG_RW | CTLFLA SYSCTL_UINT(_vfs_generic_nfs_client, OID_AUTO, debug_ctl, CTLFLAG_RW | CTLFLAG_LOCKED, &nfs_debug_ctl, 0, ""); SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, readlink_nocache, CTLFLAG_RW | CTLFLAG_LOCKED, &nfs_readlink_nocache, 0, ""); SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, root_steals_gss_context, CTLFLAG_RW | CTLFLAG_LOCKED, &nfs_root_steals_ctx, 0, ""); -SYSCTL_STRING(_vfs_generic_nfs_client, OID_AUTO, default_nfs4domain, CTLFLAG_RW | CTLFLAG_LOCKED, nfs4_domain, sizeof(nfs4_domain), ""); +SYSCTL_STRING(_vfs_generic_nfs_client, OID_AUTO, default_nfs4domain, CTLFLAG_RW | CTLFLAG_LOCKED, nfs4_default_domain, sizeof(nfs4_default_domain), ""); #endif /* NFSCLIENT */ #if NFSSERVER @@ -226,7 +226,7 @@ static int mapid2name(struct nfs_testmapid *map) { int error; - int len = sizeof(map->ntm_name); + size_t len = sizeof(map->ntm_name); if (map->ntm_grpflag) error = kauth_cred_gid2guid((gid_t)map->ntm_id, &map->ntm_guid); @@ -242,12 +242,12 @@ mapid2name(struct nfs_testmapid *map) } - static int nfsclnt_testidmap(proc_t p, user_addr_t argp) { struct nfs_testmapid mapid; int error, coerror; + size_t len = sizeof(mapid.ntm_name); /* Let root make this call. */ error = proc_suser(p); @@ -257,10 +257,22 @@ nfsclnt_testidmap(proc_t p, user_addr_t argp) error = copyin(argp, &mapid, sizeof(mapid)); if (error) return (error); - if (mapid.ntm_name2id) + switch (mapid.ntm_lookup) { + case NTM_NAME2ID: error = mapname2id(&mapid); - else + break; + case NTM_ID2NAME: error = mapid2name(&mapid); + break; + case NTM_NAME2GUID: + error = nfs4_id2guid(mapid.ntm_name, &mapid.ntm_guid, mapid.ntm_grpflag); + break; + case NTM_GUID2NAME: + error = nfs4_guid2id(&mapid.ntm_guid, mapid.ntm_name, &len, mapid.ntm_grpflag); + break; + default: + return (EINVAL); + } coerror = copyout(&mapid, argp, sizeof(mapid)); @@ -581,6 +593,11 @@ out: vnode_put(vp); if (error) return (error); + /* + * At first blush, this may appear to leak a kernel stack + * address, but the copyout() never reaches &nfh.nfh_fhp + * (sizeof(fhandle_t) < sizeof(nfh)). + */ error = copyout((caddr_t)&nfh, uap->fhp, sizeof(fhandle_t)); return (error); } diff --git a/bsd/nfs/nfs_vfsops.c b/bsd/nfs/nfs_vfsops.c index f6524f3bb..3c5f2831a 100644 --- a/bsd/nfs/nfs_vfsops.c +++ b/bsd/nfs/nfs_vfsops.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -150,7 +150,7 @@ uint32_t nfs_open_owner_seqnum = 0; uint32_t nfs_lock_owner_seqnum = 0; thread_call_t nfs4_callback_timer_call; int nfs4_callback_timer_on = 0; -char nfs4_domain[MAXPATHLEN]; +char nfs4_default_domain[MAXPATHLEN]; /* nfsiod */ lck_grp_t *nfsiod_lck_grp; @@ -491,6 +491,51 @@ nfsmout: return (error); } +/* + * Return an NFS volume name from the mntfrom name. + */ +static void +nfs_get_volname(struct mount *mp, char *volname, size_t len) +{ + const char *ptr, *cptr; + const char *mntfrom = mp->mnt_vfsstat.f_mntfromname; + size_t mflen = strnlen(mntfrom, MAXPATHLEN+1); + + if (mflen > MAXPATHLEN || mflen == 0) { + strlcpy(volname, "Bad volname", len); + return; + } + + /* Move back over trailing slashes */ + for (ptr = &mntfrom[mflen-1]; ptr != mntfrom && *ptr == '/'; ptr--) { + mflen--; + } + + /* Find first character after the last slash */ + cptr = ptr = NULL; + for(size_t i = 0; i < mflen; i++) { + if (mntfrom[i] == '/') + ptr = &mntfrom[i+1]; + /* And the first character after the first colon */ + else if (cptr == NULL && mntfrom[i] == ':') + cptr = &mntfrom[i+1]; + } + + /* + * No slash or nothing after the last slash + * use everything past the first colon + */ + if (ptr == NULL || *ptr == '\0') + ptr = cptr; + /* Otherwise use the mntfrom name */ + if (ptr == NULL) + ptr = mntfrom; + + mflen = &mntfrom[mflen] - ptr; + len = mflen+1 < len ? mflen+1 : len; + + strlcpy(volname, ptr, len); +} /* * The NFS VFS_GETATTR function: "statfs"-type information is retrieved @@ -569,6 +614,11 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t ctx) lck_mtx_unlock(&nmp->nm_lock); } + if (VFSATTR_IS_ACTIVE(fsap, f_vol_name)) { + /*%%% IF fail over support is implemented we may need to take nm_lock */ + nfs_get_volname(mp, fsap->f_vol_name, MAXPATHLEN); + VFSATTR_SET_SUPPORTED(fsap, f_vol_name); + } if (VFSATTR_IS_ACTIVE(fsap, f_capabilities)) { u_int32_t caps, valid; nfsnode_t np = nmp->nm_dnp; @@ -748,14 +798,14 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t ctx) if (VFSATTR_IS_ACTIVE(fsap, f_attributes)) { fsap->f_attributes.validattr.commonattr = 0; fsap->f_attributes.validattr.volattr = - ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES; + ATTR_VOL_NAME | ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES; fsap->f_attributes.validattr.dirattr = 0; fsap->f_attributes.validattr.fileattr = 0; fsap->f_attributes.validattr.forkattr = 0; fsap->f_attributes.nativeattr.commonattr = 0; fsap->f_attributes.nativeattr.volattr = - ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES; + ATTR_VOL_NAME | ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES; fsap->f_attributes.nativeattr.dirattr = 0; fsap->f_attributes.nativeattr.fileattr = 0; fsap->f_attributes.nativeattr.forkattr = 0; @@ -1297,7 +1347,7 @@ nfs_mount_diskless_private( mp->mnt_segreadcnt = mp->mnt_segwritecnt = 32; mp->mnt_ioflags = 0; mp->mnt_realrootvp = NULLVP; - mp->mnt_authcache_ttl = CACHED_LOOKUP_RIGHT_TTL; + mp->mnt_authcache_ttl = 0; /* Allways go to our lookup */ mount_lock_init(mp); TAILQ_INIT(&mp->mnt_vnodelist); @@ -2261,7 +2311,7 @@ nocomponents: NFS_CLEAR_ATTRIBUTES(bitmap); NFS4_DEFAULT_ATTRIBUTES(bitmap); /* if no namedattr support or component is ".zfs", clear NFS_FATTR_NAMED_ATTR */ - if (NMFLAG(nmp, NONAMEDATTR) || !strcmp(fspath.np_components[comp], ".zfs")) + if (!NMFLAG(nmp, NAMEDATTR) || !strcmp(fspath.np_components[comp], ".zfs")) NFS_BITMAP_CLR(bitmap, NFS_FATTR_NAMED_ATTR); nfsm_chain_add_bitmap(error, &nmreq, bitmap, NFS_ATTR_BITMAP_LEN); nfsm_chain_build_done(error, &nmreq); @@ -2399,7 +2449,7 @@ nocomponents: gotfh: /* get attrs for mount point root */ - numops = NMFLAG(nmp, NONAMEDATTR) ? 2 : 3; // PUTFH + GETATTR + OPENATTR + numops = NMFLAG(nmp, NAMEDATTR) ? 3 : 2; // PUTFH + GETATTR + OPENATTR nfsm_chain_build_alloc_init(error, &nmreq, 25 * NFSX_UNSIGNED); nfsm_chain_add_compound_header(error, &nmreq, "mount", nmp->nm_minor_vers, numops); numops--; @@ -2410,10 +2460,10 @@ gotfh: NFS_CLEAR_ATTRIBUTES(bitmap); NFS4_DEFAULT_ATTRIBUTES(bitmap); /* if no namedattr support or last component is ".zfs", clear NFS_FATTR_NAMED_ATTR */ - if (NMFLAG(nmp, NONAMEDATTR) || ((fspath.np_compcount > 0) && !strcmp(fspath.np_components[fspath.np_compcount-1], ".zfs"))) + if (!NMFLAG(nmp, NAMEDATTR) || ((fspath.np_compcount > 0) && !strcmp(fspath.np_components[fspath.np_compcount-1], ".zfs"))) NFS_BITMAP_CLR(bitmap, NFS_FATTR_NAMED_ATTR); nfsm_chain_add_bitmap(error, &nmreq, bitmap, NFS_ATTR_BITMAP_LEN); - if (!NMFLAG(nmp, NONAMEDATTR)) { + if (NMFLAG(nmp, NAMEDATTR)) { numops--; nfsm_chain_add_32(error, &nmreq, NFS_OP_OPENATTR); nfsm_chain_add_32(error, &nmreq, 0); @@ -2433,7 +2483,7 @@ gotfh: NFS_CLEAR_ATTRIBUTES(nmp->nm_fsattr.nfsa_bitmap); error = nfs4_parsefattr(&nmrep, &nmp->nm_fsattr, &nvattr, NULL, NULL, NULL); nfsmout_if(error); - if (!NMFLAG(nmp, NONAMEDATTR)) { + if (NMFLAG(nmp, NAMEDATTR)) { nfsm_chain_op_check(error, &nmrep, NFS_OP_OPENATTR); if (error == ENOENT) error = 0; @@ -2700,6 +2750,13 @@ mountnfs( vfs_getnewfsid(mp); nmp->nm_mountp = mp; vfs_setauthopaque(mp); + /* + * Disable cache_lookup_path for NFS. NFS lookup always needs + * to be called to check if the directory attribute cache is + * valid and possibly purge the directory before calling + * cache_lookup. + */ + vfs_setauthcache_ttl(mp, 0); nfs_nhinit_finish(); @@ -3217,7 +3274,7 @@ mountnfs( } else { /* ignore these if not v4 */ NFS_BITMAP_CLR(nmp->nm_flags, NFS_MFLAG_NOCALLBACK); - NFS_BITMAP_CLR(nmp->nm_flags, NFS_MFLAG_NONAMEDATTR); + NFS_BITMAP_CLR(nmp->nm_flags, NFS_MFLAG_NAMEDATTR); NFS_BITMAP_CLR(nmp->nm_flags, NFS_MFLAG_NOACL); NFS_BITMAP_CLR(nmp->nm_flags, NFS_MFLAG_ACLONLY); } @@ -3312,9 +3369,13 @@ mountnfs( * buffers into multiple requests if the buffer size is * larger than the I/O size. */ +#ifndef CONFIG_EMBEDDED iosize = max(nmp->nm_rsize, nmp->nm_wsize); if (iosize < PAGE_SIZE) iosize = PAGE_SIZE; +#else + iosize = PAGE_SIZE; +#endif nmp->nm_biosize = trunc_page_32(iosize); /* For NFSv3 and greater, there is a (relatively) reliable ACCESS call. */ @@ -4634,6 +4695,30 @@ nfs_vfs_quotactl( } #else +static int +nfs_sa_getport(struct sockaddr *sa, int *error) +{ + int port = 0; + + if (sa->sa_family == AF_INET6) + port = ntohs(((struct sockaddr_in6*)sa)->sin6_port); + else if (sa->sa_family == AF_INET) + port = ntohs(((struct sockaddr_in*)sa)->sin_port); + else if (error) + *error = EIO; + + return port; +} + +static void +nfs_sa_setport(struct sockaddr *sa, int port) +{ + if (sa->sa_family == AF_INET6) + ((struct sockaddr_in6*)sa)->sin6_port = htons(port); + else if (sa->sa_family == AF_INET) + ((struct sockaddr_in*)sa)->sin_port = htons(port); +} + int nfs3_getquota(struct nfsmount *nmp, vfs_context_t ctx, uid_t id, int type, struct dqblk *dqb) { @@ -4648,6 +4733,7 @@ nfs3_getquota(struct nfsmount *nmp, vfs_context_t ctx, uid_t id, int type, struc uint32_t val = 0, bsize = 0; struct sockaddr *rqsaddr; struct timeval now; + struct timespec ts = { 1, 0 }; if (!nmp->nm_saddr) return (ENXIO); @@ -4655,38 +4741,91 @@ nfs3_getquota(struct nfsmount *nmp, vfs_context_t ctx, uid_t id, int type, struc if (NMFLAG(nmp, NOQUOTA)) return (ENOTSUP); - if (!nmp->nm_rqsaddr) - MALLOC(nmp->nm_rqsaddr, struct sockaddr *, sizeof(struct sockaddr_storage), M_SONAME, M_WAITOK|M_ZERO); - if (!nmp->nm_rqsaddr) - return (ENOMEM); - rqsaddr = nmp->nm_rqsaddr; - if (rqsaddr->sa_family == AF_INET6) - rqport = ntohs(((struct sockaddr_in6*)rqsaddr)->sin6_port); - else if (rqsaddr->sa_family == AF_INET) - rqport = ntohs(((struct sockaddr_in*)rqsaddr)->sin_port); + /* + * Allocate an address for rquotad if needed + */ + if (!nmp->nm_rqsaddr) { + int need_free = 0; + + MALLOC(rqsaddr, struct sockaddr *, sizeof(struct sockaddr_storage), M_SONAME, M_WAITOK|M_ZERO); + bcopy(nmp->nm_saddr, rqsaddr, min(sizeof(struct sockaddr_storage), nmp->nm_saddr->sa_len)); + /* Set the port to zero, will call rpcbind to get the port below */ + nfs_sa_setport(rqsaddr, 0); + microuptime(&now); + + lck_mtx_lock(&nmp->nm_lock); + if (!nmp->nm_rqsaddr) { + nmp->nm_rqsaddr = rqsaddr; + nmp->nm_rqsaddrstamp = now.tv_sec; + } else { + need_free = 1; + } + lck_mtx_unlock(&nmp->nm_lock); + if (need_free) + FREE(rqsaddr, M_SONAME); + } timeo = NMFLAG(nmp, SOFT) ? 10 : 60; rqproto = IPPROTO_UDP; /* XXX should prefer TCP if mount is TCP */ /* check if we have a recently cached rquota port */ microuptime(&now); - if (!rqport || ((nmp->nm_rqsaddrstamp + 60) >= (uint32_t)now.tv_sec)) { + lck_mtx_lock(&nmp->nm_lock); + rqsaddr = nmp->nm_rqsaddr; + rqport = nfs_sa_getport(rqsaddr, &error); + while (!error && (!rqport || ((nmp->nm_rqsaddrstamp + 60) <= (uint32_t)now.tv_sec))) { + error = nfs_sigintr(nmp, NULL, thd, 1); + if (error) { + lck_mtx_unlock(&nmp->nm_lock); + return (error); + } + if (nmp->nm_state & NFSSTA_RQUOTAINPROG) { + nmp->nm_state |= NFSSTA_WANTRQUOTA; + msleep(&nmp->nm_rqsaddr, &nmp->nm_lock, PZERO-1, "nfswaitrquotaaddr", &ts); + rqport = nfs_sa_getport(rqsaddr, &error); + continue; + } + nmp->nm_state |= NFSSTA_RQUOTAINPROG; + lck_mtx_unlock(&nmp->nm_lock); + /* send portmap request to get rquota port */ - bcopy(nmp->nm_saddr, rqsaddr, min(sizeof(struct sockaddr_storage), nmp->nm_saddr->sa_len)); error = nfs_portmap_lookup(nmp, ctx, rqsaddr, NULL, RPCPROG_RQUOTA, rqvers, rqproto, timeo); if (error) - return (error); - if (rqsaddr->sa_family == AF_INET6) - rqport = ntohs(((struct sockaddr_in6*)rqsaddr)->sin6_port); - else if (rqsaddr->sa_family == AF_INET) - rqport = ntohs(((struct sockaddr_in*)rqsaddr)->sin_port); - else - return (EIO); - if (!rqport) - return (ENOTSUP); + goto out; + rqport = nfs_sa_getport(rqsaddr, &error); + if (error) + goto out; + + if (!rqport) { + /* + * We overload PMAPPORT for the port if rquotad is not + * currently registered or up at the server. In the + * while loop above, port will be set and we will defer + * for a bit. Perhaps the service isn't online yet. + * + * Note that precludes using indirect, but we're not doing + * that here. + */ + rqport = PMAPPORT; + nfs_sa_setport(rqsaddr, rqport); + } microuptime(&now); nmp->nm_rqsaddrstamp = now.tv_sec; + out: + lck_mtx_lock(&nmp->nm_lock); + nmp->nm_state &= ~NFSSTA_RQUOTAINPROG; + if (nmp->nm_state & NFSSTA_WANTRQUOTA) { + nmp->nm_state &= ~NFSSTA_WANTRQUOTA; + wakeup(&nmp->nm_rqsaddr); + } } + lck_mtx_unlock(&nmp->nm_lock); + if (error) + return (error); + + /* Using PMAPPORT for unavailabe rquota service */ + if (rqport == PMAPPORT) + return (ENOTSUP); /* rquota request */ nfsm_chain_null(&nmreq); @@ -5095,7 +5234,7 @@ nfs_mountinfo_assemble(struct nfsmount *nmp, struct xdrbuf *xb) if (nmp->nm_vers >= NFS_VER4) { NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_EPHEMERAL); NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_NOCALLBACK); - NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_NONAMEDATTR); + NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_NAMEDATTR); NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_NOACL); NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_ACLONLY); } @@ -5129,8 +5268,8 @@ nfs_mountinfo_assemble(struct nfsmount *nmp, struct xdrbuf *xb) NFS_BITMAP_SET(mflags, NFS_MFLAG_EPHEMERAL); if (NMFLAG(nmp, NOCALLBACK)) NFS_BITMAP_SET(mflags, NFS_MFLAG_NOCALLBACK); - if (NMFLAG(nmp, NONAMEDATTR)) - NFS_BITMAP_SET(mflags, NFS_MFLAG_NONAMEDATTR); + if (NMFLAG(nmp, NAMEDATTR)) + NFS_BITMAP_SET(mflags, NFS_MFLAG_NAMEDATTR); if (NMFLAG(nmp, NOACL)) NFS_BITMAP_SET(mflags, NFS_MFLAG_NOACL); if (NMFLAG(nmp, ACLONLY)) @@ -5306,7 +5445,9 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, user_addr_t newp, size_t newlen, vfs_context_t ctx) { int error = 0, val; +#ifndef CONFIG_EMBEDDED int softnobrowse; +#endif struct sysctl_req *req = NULL; union union_vfsidctl vc; mount_t mp; @@ -5346,7 +5487,9 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, case VFS_CTL_TIMEO: case VFS_CTL_NOLOCKS: case VFS_CTL_NSTATUS: +#ifndef CONFIG_EMBEDDED case VFS_CTL_QUERY: +#endif req = CAST_DOWN(struct sysctl_req *, oldp); if (req == NULL) { return EFAULT; @@ -5370,6 +5513,10 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, req->newlen = vc.vc32.vc_len; } break; +#if CONFIG_EMBEDDED + case VFS_CTL_QUERY: + return EPERM; +#endif } switch(name[0]) { @@ -5508,6 +5655,7 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, break; /* build exported filesystem path */ + memset(statrec.path, 0, sizeof(statrec.path)); snprintf(statrec.path, sizeof(statrec.path), "%s%s%s", nxfs->nxfs_path, ((nxfs->nxfs_path[1] && nx->nx_path[0]) ? "/" : ""), nx->nx_path); @@ -5562,6 +5710,7 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, LIST_FOREACH(nx, &nxfs->nxfs_exports, nx_next) { /* copy out path */ if (bytes_avail >= sizeof(struct nfs_user_stat_path_rec)) { + memset(upath_rec.path, 0, sizeof(upath_rec.path)); snprintf(upath_rec.path, sizeof(upath_rec.path), "%s%s%s", nxfs->nxfs_path, ((nxfs->nxfs_path[1] && nx->nx_path[0]) ? "/" : ""), nx->nx_path); @@ -5593,6 +5742,7 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, if (bytes_avail >= sizeof(struct nfs_user_stat_user_rec)) { /* prepare a user stat rec for copying out */ ustat_rec.uid = unode->uid; + memset(&ustat_rec.sock, 0, sizeof(ustat_rec.sock)); bcopy(&unode->sock, &ustat_rec.sock, unode->sock.ss_len); ustat_rec.ops = unode->ops; ustat_rec.bytes_read = unode->bytes_read; @@ -5694,6 +5844,7 @@ ustat_skip: lck_mtx_unlock(&nmp->nm_lock); } break; +#ifndef CONFIG_EMBEDDED case VFS_CTL_QUERY: lck_mtx_lock(&nmp->nm_lock); /* XXX don't allow users to know about/disconnect unresponsive, soft, nobrowse mounts */ @@ -5710,6 +5861,7 @@ ustat_skip: lck_mtx_unlock(&nmp->nm_lock); error = SYSCTL_OUT(req, &vq, sizeof(vq)); break; +#endif case VFS_CTL_TIMEO: if (req->oldptr != USER_ADDR_NULL) { lck_mtx_lock(&nmp->nm_lock); diff --git a/bsd/nfs/nfs_vnops.c b/bsd/nfs/nfs_vnops.c index 38653ed5a..4c492ac65 100644 --- a/bsd/nfs/nfs_vnops.c +++ b/bsd/nfs/nfs_vnops.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1731,6 +1731,7 @@ nfs3_vnop_getattr( int error; struct nfs_vattr nva; struct vnode_attr *vap = ap->a_vap; + struct nfsmount *nmp; dev_t rdev; /* @@ -1751,6 +1752,8 @@ nfs3_vnop_getattr( return (error); /* copy nva to *a_vap */ + nmp = VTONMP(ap->a_vp); + vap->va_flags |= nmp ? (nmp->nm_vers > 2 ? VA_64BITOBJIDS : 0) : 0; VATTR_RETURN(vap, va_type, nva.nva_type); VATTR_RETURN(vap, va_mode, nva.nva_mode); rdev = makedev(nva.nva_rawdev.specdata1, nva.nva_rawdev.specdata2); @@ -6696,7 +6699,7 @@ nfsspec_vnop_read( } */ *ap) { nfsnode_t np = VTONFS(ap->a_vp); - struct timeval now; + struct timespec now; int error; /* @@ -6705,9 +6708,9 @@ nfsspec_vnop_read( if ((error = nfs_node_lock(np))) return (error); np->n_flag |= NACC; - microtime(&now); + nanotime(&now); np->n_atim.tv_sec = now.tv_sec; - np->n_atim.tv_nsec = now.tv_usec * 1000; + np->n_atim.tv_nsec = now.tv_nsec; nfs_node_unlock(np); return (VOCALL(spec_vnodeop_p, VOFFSET(vnop_read), ap)); } @@ -6726,7 +6729,7 @@ nfsspec_vnop_write( } */ *ap) { nfsnode_t np = VTONFS(ap->a_vp); - struct timeval now; + struct timespec now; int error; /* @@ -6735,9 +6738,9 @@ nfsspec_vnop_write( if ((error = nfs_node_lock(np))) return (error); np->n_flag |= NUPD; - microtime(&now); + nanotime(&now); np->n_mtim.tv_sec = now.tv_sec; - np->n_mtim.tv_nsec = now.tv_usec * 1000; + np->n_mtim.tv_nsec = now.tv_nsec; nfs_node_unlock(np); return (VOCALL(spec_vnodeop_p, VOFFSET(vnop_write), ap)); } @@ -6804,7 +6807,7 @@ nfsfifo_vnop_read( } */ *ap) { nfsnode_t np = VTONFS(ap->a_vp); - struct timeval now; + struct timespec now; int error; /* @@ -6813,9 +6816,9 @@ nfsfifo_vnop_read( if ((error = nfs_node_lock(np))) return (error); np->n_flag |= NACC; - microtime(&now); + nanotime(&now); np->n_atim.tv_sec = now.tv_sec; - np->n_atim.tv_nsec = now.tv_usec * 1000; + np->n_atim.tv_nsec = now.tv_nsec; nfs_node_unlock(np); return (VOCALL(fifo_vnodeop_p, VOFFSET(vnop_read), ap)); } @@ -6834,7 +6837,7 @@ nfsfifo_vnop_write( } */ *ap) { nfsnode_t np = VTONFS(ap->a_vp); - struct timeval now; + struct timespec now; int error; /* @@ -6843,9 +6846,9 @@ nfsfifo_vnop_write( if ((error = nfs_node_lock(np))) return (error); np->n_flag |= NUPD; - microtime(&now); + nanotime(&now); np->n_mtim.tv_sec = now.tv_sec; - np->n_mtim.tv_nsec = now.tv_usec * 1000; + np->n_mtim.tv_nsec = now.tv_nsec; nfs_node_unlock(np); return (VOCALL(fifo_vnodeop_p, VOFFSET(vnop_write), ap)); } @@ -6867,21 +6870,21 @@ nfsfifo_vnop_close( vnode_t vp = ap->a_vp; nfsnode_t np = VTONFS(vp); struct vnode_attr vattr; - struct timeval now; + struct timespec now; mount_t mp; int error; if ((error = nfs_node_lock(np))) return (error); if (np->n_flag & (NACC | NUPD)) { - microtime(&now); + nanotime(&now); if (np->n_flag & NACC) { np->n_atim.tv_sec = now.tv_sec; - np->n_atim.tv_nsec = now.tv_usec * 1000; + np->n_atim.tv_nsec = now.tv_nsec; } if (np->n_flag & NUPD) { np->n_mtim.tv_sec = now.tv_sec; - np->n_mtim.tv_nsec = now.tv_usec * 1000; + np->n_mtim.tv_nsec = now.tv_nsec; } np->n_flag |= NCHG; if (!vnode_isinuse(vp, 1) && (mp = vnode_mount(vp)) && !vfs_isrdonly(mp)) { @@ -6935,14 +6938,18 @@ nfs_vnop_ioctl( return (EROFS); error = nfs_flush(VTONFS(vp), MNT_WAIT, vfs_context_thread(ctx), 0); break; - case NFS_FSCTL_DESTROY_CRED: + case NFS_IOC_DESTROY_CRED: if (!auth_is_kerberized(mp->nm_auth)) return (ENOTSUP); error = nfs_gss_clnt_ctx_remove(mp, vfs_context_ucred(ctx)); break; - case NFS_FSCTL_SET_CRED: + case NFS_IOC_SET_CRED: + case NFS_IOC_SET_CRED64: if (!auth_is_kerberized(mp->nm_auth)) return (ENOTSUP); + if ((ap->a_command == NFS_IOC_SET_CRED && vfs_context_is64bit(ctx)) || + (ap->a_command == NFS_IOC_SET_CRED64 && !vfs_context_is64bit(ctx))) + return (EINVAL); if (vfs_context_is64bit(ctx)) { gprinc = *(struct user_nfs_gss_principal *)ap->a_data; } else { @@ -6971,9 +6978,13 @@ nfs_vnop_ioctl( NFS_DBG(NFS_FAC_GSS, 7, "Seting credential to principal %s returned %d\n", p, error); FREE(p, M_TEMP); break; - case NFS_FSCTL_GET_CRED: + case NFS_IOC_GET_CRED: + case NFS_IOC_GET_CRED64: if (!auth_is_kerberized(mp->nm_auth)) return (ENOTSUP); + if ((ap->a_command == NFS_IOC_GET_CRED && vfs_context_is64bit(ctx)) || + (ap->a_command == NFS_IOC_GET_CRED64 && !vfs_context_is64bit(ctx))) + return (EINVAL); error = nfs_gss_clnt_ctx_get_principal(mp, ctx, &gprinc); if (error) break; @@ -7906,6 +7917,8 @@ nfs_vnode_notify(nfsnode_t np, uint32_t events) if (!nfs_getattrcache(np, &nvattr, 0)) { vap = &vattr; VATTR_INIT(vap); + + vap->va_flags |= nmp->nm_vers > 2 ? VA_64BITOBJIDS : 0; VATTR_RETURN(vap, va_fsid, vfs_statfs(nmp->nm_mountp)->f_fsid.val[0]); VATTR_RETURN(vap, va_fileid, nvattr.nva_fileid); VATTR_RETURN(vap, va_mode, nvattr.nva_mode); diff --git a/bsd/nfs/nfsmount.h b/bsd/nfs/nfsmount.h index feb205951..78aa778ad 100644 --- a/bsd/nfs/nfsmount.h +++ b/bsd/nfs/nfsmount.h @@ -381,6 +381,8 @@ struct nfsmount { #define NFSSTA_HASWRITEVERF 0x00040000 /* Has write verifier for V3 */ #define NFSSTA_GOTPATHCONF 0x00080000 /* Got the V3 pathconf info */ #define NFSSTA_GOTFSINFO 0x00100000 /* Got the V3 fsinfo */ +#define NFSSTA_WANTRQUOTA 0x00200000 /* Want rquota address */ +#define NFSSTA_RQUOTAINPROG 0x00400000 /* Getting rquota address */ #define NFSSTA_SENDING 0x00800000 /* Sending on socket */ #define NFSSTA_SNDLOCK 0x01000000 /* Send socket lock */ #define NFSSTA_WANTSND 0x02000000 /* Want above */ diff --git a/bsd/pgo/profile_runtime.c b/bsd/pgo/profile_runtime.c index f519a2690..ec24bfffe 100644 --- a/bsd/pgo/profile_runtime.c +++ b/bsd/pgo/profile_runtime.c @@ -26,6 +26,7 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +#include #include #include #include @@ -98,9 +99,8 @@ static int write_buffer(int flags, char *buffer) int kdp_pgo_reset_counters = 0; /* called in debugger context */ -static kern_return_t do_pgo_reset_counters(void *context) +kern_return_t do_pgo_reset_counters() { -#pragma unused(context) #ifdef PROFILE memset(&__pgo_hib_CountersStart, 0, ((uintptr_t)(&__pgo_hib_CountersEnd)) - ((uintptr_t)(&__pgo_hib_CountersStart))); @@ -110,13 +110,27 @@ static kern_return_t do_pgo_reset_counters(void *context) return KERN_SUCCESS; } +static kern_return_t +kextpgo_trap() +{ + return DebuggerTrapWithState(DBOP_RESET_PGO_COUNTERS, NULL, NULL, NULL, 0, FALSE, 0); +} + static kern_return_t pgo_reset_counters() { kern_return_t r; + boolean_t istate; + OSKextResetPgoCountersLock(); + + istate = ml_set_interrupts_enabled(FALSE); + kdp_pgo_reset_counters = 1; - r = DebuggerWithCallback(do_pgo_reset_counters, NULL, FALSE); + r = kextpgo_trap(); + + ml_set_interrupts_enabled(istate); + OSKextResetPgoCountersUnlock(); return r; } @@ -244,7 +258,7 @@ int grab_pgo_data(struct proc *p, goto out; } - MALLOC(buffer, char *, size64, M_TEMP, M_WAITOK); + MALLOC(buffer, char *, size64, M_TEMP, M_WAITOK | M_ZERO); if (!buffer) { err = ENOMEM; goto out; @@ -302,7 +316,7 @@ int grab_pgo_data(struct proc *p, err = EINVAL; goto out; } else { - MALLOC(buffer, char *, size, M_TEMP, M_WAITOK); + MALLOC(buffer, char *, size, M_TEMP, M_WAITOK | M_ZERO); if (!buffer) { err = ENOMEM; goto out; diff --git a/bsd/security/audit/audit.h b/bsd/security/audit/audit.h index 79136bbb7..61e818f60 100644 --- a/bsd/security/audit/audit.h +++ b/bsd/security/audit/audit.h @@ -44,11 +44,6 @@ #if defined(_KERNEL) || defined(KERNEL) -#if CONFIG_MACF -#include -#include -#endif - #include #include diff --git a/bsd/security/audit/audit_bsm.c b/bsd/security/audit/audit_bsm.c index da938d8a1..edebfd61b 100644 --- a/bsd/security/audit/audit_bsm.c +++ b/bsd/security/audit/audit_bsm.c @@ -1256,6 +1256,20 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau) case AUE_UNMOUNT: UPATH1_VNODE1_TOKENS; break; + case AUE_FMOUNT: + if (ARG_IS_VALID(kar, ARG_FD)) { + tok = au_to_arg32(2, "dir fd", ar->ar_arg_fd); + kau_write(rec, tok); + } + if (ARG_IS_VALID(kar, ARG_FFLAGS)) { + tok = au_to_arg32(3, "flags", ar->ar_arg_fflags); + kau_write(rec, tok); + } + if (ARG_IS_VALID(kar, ARG_TEXT)) { + tok = au_to_text(ar->ar_arg_text); + kau_write(rec, tok); + } + break; case AUE_MSGCTL: ar->ar_event = audit_msgctl_to_event(ar->ar_arg_svipc_cmd); @@ -1412,6 +1426,7 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau) case AUE_SYMLINKAT: case AUE_MKDIRAT: case AUE_GETATTRLISTAT: + case AUE_SETATTRLISTAT: if (ARG_IS_VALID(kar, ARG_FD)) { tok = au_to_arg32(1, "dir fd", ar->ar_arg_fd); kau_write(rec, tok); diff --git a/bsd/sys/Makefile b/bsd/sys/Makefile index 366a86188..88c3a51c4 100644 --- a/bsd/sys/Makefile +++ b/bsd/sys/Makefile @@ -19,21 +19,21 @@ DATAFILES = \ appleapiopts.h acct.h aio.h attr.h \ buf.h cdefs.h clonefile.h conf.h \ dir.h dirent.h disk.h dkstat.h dtrace.h dtrace_glue.h dtrace_impl.h \ - errno.h ev.h event.h fasttrap.h fasttrap_isa.h fcntl.h file.h filedesc.h filio.h gmon.h \ + errno.h ev.h event.h fasttrap.h fasttrap_isa.h fcntl.h file.h filedesc.h filio.h fsgetpath.h gmon.h \ ioccom.h ioctl.h \ ioctl_compat.h ipc.h kernel.h kernel_types.h kern_event.h lctx.h loadable_fs.h lock.h lockf.h \ kauth.h kdebug.h kdebug_signpost.h kern_control.h lockstat.h malloc.h \ mbuf.h mman.h mount.h msg.h msgbuf.h netport.h param.h paths.h pipe.h poll.h \ proc.h proc_info.h ptrace.h queue.h quota.h reboot.h resource.h resourcevar.h \ sbuf.h posix_sem.h posix_shm.h random.h sdt.h\ - select.h sem.h semaphore.h shm.h signal.h signalvar.h socket.h socketvar.h sockio.h stat.h stdio.h \ + select.h sem.h semaphore.h shm.h signal.h signalvar.h snapshot.h socket.h socketvar.h sockio.h stat.h stdio.h \ sysctl.h syslimits.h syslog.h sys_domain.h termios.h time.h \ timeb.h times.h tprintf.h trace.h tty.h ttychars.h ttycom.h \ ttydefaults.h ttydev.h types.h ubc.h ucontext.h ucred.h uio.h un.h unistd.h unpcb.h \ user.h utfconv.h utsname.h vadvise.h vcmd.h \ vm.h vmmeter.h vmparam.h vnioctl.h vnode.h vnode_if.h vstat.h wait.h xattr.h \ _select.h _structs.h _types.h _endian.h domain.h protosw.h \ - spawn.h + spawn.h timex.h commpage.h # Installs header file for Apple internal use in user level - # $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders @@ -67,6 +67,7 @@ PRIVATE_DATAFILES = \ kern_overrides.h \ mbuf.h \ mman.h \ + monotonic.h \ persona.h \ priv.h \ proc.h \ @@ -94,7 +95,8 @@ PRIVATE_DATAFILES = \ proc_uuid_policy.h \ priv.h \ pgo.h \ - memory_maintenance.h + memory_maintenance.h \ + commpage.h # Installs header file for kernel extensions - # $(DSTROOT)/System/Library/Frameworks/Kernel.framework/Headers @@ -118,7 +120,7 @@ KERNELFILES = \ kpi_mbuf.h kpi_socket.h kpi_socketfilter.h \ ttycom.h termios.h msg.h \ wait.h \ - spawn.h + spawn.h timex.h commpage.h # The last line was added to export needed headers for the MAC calls # whose source is outside of the xnu/bsd tree. @@ -131,6 +133,7 @@ PRIVATE_KERNELFILES = \ csr.h \ decmpfs.h \ disktab.h \ + eventhandler.h \ fbt.h \ fileport.h \ fsctl.h \ @@ -140,12 +143,14 @@ PRIVATE_KERNELFILES = \ kpi_private.h \ ktrace.h \ mach_swapon.h \ + monotonic.h \ msgbuf.h \ eventvar.h \ persona.h \ proc_info.h \ pthread_shims.h \ quota.h \ + reboot.h \ sem_internal.h \ shm_internal.h \ signalvar.h \ @@ -161,7 +166,8 @@ PRIVATE_KERNELFILES = \ pgo.h \ memory_maintenance.h \ doc_tombstone.h \ - fsevents.h + fsevents.h \ + work_interval.h \ # /usr/include INSTALL_MI_LIST = ${DATAFILES} diff --git a/bsd/sys/_types/Makefile b/bsd/sys/_types/Makefile index 0cf91f657..c64ec4c8a 100644 --- a/bsd/sys/_types/Makefile +++ b/bsd/sys/_types/Makefile @@ -17,6 +17,7 @@ EXPINC_SUBDIRS = DATAFILES = \ _blkcnt_t.h \ _blksize_t.h \ + _caddr_t.h \ _clock_t.h \ _ct_rune_t.h \ _dev_t.h \ @@ -79,16 +80,23 @@ DATAFILES = \ _ucontext.h \ _ucontext64.h \ _uid_t.h \ + _u_char.h \ + _u_int.h \ _u_int16_t.h \ _u_int32_t.h \ _u_int64_t.h \ _u_int8_t.h \ + _u_short.h \ _uintptr_t.h \ _useconds_t.h \ _uuid_t.h \ _va_list.h \ _wchar_t.h \ _wint_t.h \ + _user32_timex.h \ + _user64_timex.h \ + _user32_ntptimeval.h \ + _user64_ntptimeval.h \ # Installs header file for Apple internal use in user level - # $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders diff --git a/bsd/sys/_types/_blkcnt_t.h b/bsd/sys/_types/_blkcnt_t.h index c8a9d3963..30668f8d6 100644 --- a/bsd/sys/_types/_blkcnt_t.h +++ b/bsd/sys/_types/_blkcnt_t.h @@ -27,5 +27,6 @@ */ #ifndef _BLKCNT_T #define _BLKCNT_T +#include /* __darwin_blkcnt_t */ typedef __darwin_blkcnt_t blkcnt_t; #endif /* _BLKCNT_T */ diff --git a/bsd/sys/_types/_blksize_t.h b/bsd/sys/_types/_blksize_t.h index de50f2ca3..a71c373ce 100644 --- a/bsd/sys/_types/_blksize_t.h +++ b/bsd/sys/_types/_blksize_t.h @@ -27,5 +27,6 @@ */ #ifndef _BLKSIZE_T #define _BLKSIZE_T +#include /* __darwin_blksize_t */ typedef __darwin_blksize_t blksize_t; #endif /* _BLKSIZE_T */ diff --git a/bsd/sys/_types/_caddr_t.h b/bsd/sys/_types/_caddr_t.h new file mode 100644 index 000000000..ad1ad5f62 --- /dev/null +++ b/bsd/sys/_types/_caddr_t.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _CADDR_T +#define _CADDR_T +typedef char * caddr_t; +#endif /* _CADDR_T */ diff --git a/bsd/sys/_types/_clock_t.h b/bsd/sys/_types/_clock_t.h index d58801cde..6fcdf6ba9 100644 --- a/bsd/sys/_types/_clock_t.h +++ b/bsd/sys/_types/_clock_t.h @@ -27,5 +27,6 @@ */ #ifndef _CLOCK_T #define _CLOCK_T +#include /* __darwin_clock_t */ typedef __darwin_clock_t clock_t; #endif /* _CLOCK_T */ diff --git a/bsd/sys/_types/_ct_rune_t.h b/bsd/sys/_types/_ct_rune_t.h index 116174cad..ad66d423a 100644 --- a/bsd/sys/_types/_ct_rune_t.h +++ b/bsd/sys/_types/_ct_rune_t.h @@ -28,5 +28,6 @@ #ifndef _CT_RUNE_T #define _CT_RUNE_T +#include /* __darwin_ct_rune_t */ typedef __darwin_ct_rune_t ct_rune_t; #endif /* _CT_RUNE_T */ diff --git a/bsd/sys/_types/_dev_t.h b/bsd/sys/_types/_dev_t.h index cf6d3ad22..8a1e4053f 100644 --- a/bsd/sys/_types/_dev_t.h +++ b/bsd/sys/_types/_dev_t.h @@ -27,5 +27,6 @@ */ #ifndef _DEV_T #define _DEV_T +#include /* __darwin_dev_t */ typedef __darwin_dev_t dev_t; /* device number */ #endif /* _DEV_T */ diff --git a/bsd/sys/_types/_fd_def.h b/bsd/sys/_types/_fd_def.h index 158fb8f16..51c43746c 100644 --- a/bsd/sys/_types/_fd_def.h +++ b/bsd/sys/_types/_fd_def.h @@ -27,6 +27,9 @@ */ #ifndef _FD_SET #define _FD_SET + +#include /* __int32_t */ + /* * Select uses bit masks of file descriptors in longs. These macros * manipulate such bit fields (the filesystem macros use chars). The diff --git a/bsd/sys/_types/_fsblkcnt_t.h b/bsd/sys/_types/_fsblkcnt_t.h index 12e03a537..ac012b146 100644 --- a/bsd/sys/_types/_fsblkcnt_t.h +++ b/bsd/sys/_types/_fsblkcnt_t.h @@ -27,5 +27,6 @@ */ #ifndef _FSBLKCNT_T #define _FSBLKCNT_T +#include /* __darwin_fsblkcnt_t */ typedef __darwin_fsblkcnt_t fsblkcnt_t; #endif /* _FSBLKCNT_T */ diff --git a/bsd/sys/_types/_fsfilcnt_t.h b/bsd/sys/_types/_fsfilcnt_t.h index 9a72eca63..80bfa76ae 100644 --- a/bsd/sys/_types/_fsfilcnt_t.h +++ b/bsd/sys/_types/_fsfilcnt_t.h @@ -27,5 +27,6 @@ */ #ifndef _FSFILCNT_T #define _FSFILCNT_T +#include /* __darwin_fsfilcnt_t */ typedef __darwin_fsfilcnt_t fsfilcnt_t; #endif /* _FSFILCNT_T */ diff --git a/bsd/sys/_types/_fsid_t.h b/bsd/sys/_types/_fsid_t.h index 5532b5715..5806d16ca 100644 --- a/bsd/sys/_types/_fsid_t.h +++ b/bsd/sys/_types/_fsid_t.h @@ -27,5 +27,6 @@ */ #ifndef _FSID_T #define _FSID_T +#include /* int32_t */ typedef struct fsid { int32_t val[2]; } fsid_t; /* file system id type */ #endif /* _FSID_T */ diff --git a/bsd/sys/_types/_fsobj_id_t.h b/bsd/sys/_types/_fsobj_id_t.h index 20e1bcff9..a396cdff9 100644 --- a/bsd/sys/_types/_fsobj_id_t.h +++ b/bsd/sys/_types/_fsobj_id_t.h @@ -28,6 +28,8 @@ #ifndef _FSOBJ_ID_T #define _FSOBJ_ID_T +#include /* u_int32_t */ + typedef struct fsobj_id { u_int32_t fid_objno; u_int32_t fid_generation; diff --git a/bsd/sys/_types/_gid_t.h b/bsd/sys/_types/_gid_t.h index f64f56c62..402f5c219 100644 --- a/bsd/sys/_types/_gid_t.h +++ b/bsd/sys/_types/_gid_t.h @@ -27,5 +27,6 @@ */ #ifndef _GID_T #define _GID_T +#include /* __darwin_gid_t */ typedef __darwin_gid_t gid_t; #endif diff --git a/bsd/sys/_types/_id_t.h b/bsd/sys/_types/_id_t.h index b5a8a2f2c..79cd778da 100644 --- a/bsd/sys/_types/_id_t.h +++ b/bsd/sys/_types/_id_t.h @@ -27,5 +27,6 @@ */ #ifndef _ID_T #define _ID_T +#include /* __darwin_id_t */ typedef __darwin_id_t id_t; /* can hold pid_t, gid_t, or uid_t */ #endif /* _ID_T */ diff --git a/bsd/sys/_types/_in_addr_t.h b/bsd/sys/_types/_in_addr_t.h index a534517c6..aa4956a1c 100644 --- a/bsd/sys/_types/_in_addr_t.h +++ b/bsd/sys/_types/_in_addr_t.h @@ -27,5 +27,6 @@ */ #ifndef _IN_ADDR_T #define _IN_ADDR_T +#include /* __uint32_t */ typedef __uint32_t in_addr_t; /* base type for internet address */ #endif /* _IN_ADDR_T */ diff --git a/bsd/sys/_types/_in_port_t.h b/bsd/sys/_types/_in_port_t.h index cf3da0020..69e719e89 100644 --- a/bsd/sys/_types/_in_port_t.h +++ b/bsd/sys/_types/_in_port_t.h @@ -27,5 +27,6 @@ */ #ifndef _IN_PORT_T #define _IN_PORT_T +#include /* __uint16_t */ typedef __uint16_t in_port_t; #endif /* _IN_PORT_T */ diff --git a/bsd/sys/_types/_ino64_t.h b/bsd/sys/_types/_ino64_t.h index a7ca59e97..effe9f6e6 100644 --- a/bsd/sys/_types/_ino64_t.h +++ b/bsd/sys/_types/_ino64_t.h @@ -27,5 +27,6 @@ */ #ifndef _INO64_T #define _INO64_T +#include /* __darwin_ino64_t */ typedef __darwin_ino64_t ino64_t; /* 64bit inode number */ #endif /* _INO64_T */ diff --git a/bsd/sys/_types/_ino_t.h b/bsd/sys/_types/_ino_t.h index 2bc666f92..721f8646e 100644 --- a/bsd/sys/_types/_ino_t.h +++ b/bsd/sys/_types/_ino_t.h @@ -27,5 +27,6 @@ */ #ifndef _INO_T #define _INO_T +#include /* __darwin_ino_t */ typedef __darwin_ino_t ino_t; /* inode number */ #endif /* _INO_T */ diff --git a/bsd/sys/_types/_intptr_t.h b/bsd/sys/_types/_intptr_t.h index c01f906f5..0e050f7a0 100644 --- a/bsd/sys/_types/_intptr_t.h +++ b/bsd/sys/_types/_intptr_t.h @@ -27,5 +27,7 @@ */ #ifndef _INTPTR_T #define _INTPTR_T +#include /* __darwin_intptr_t */ + typedef __darwin_intptr_t intptr_t; #endif /* _INTPTR_T */ diff --git a/bsd/sys/_types/_iovec_t.h b/bsd/sys/_types/_iovec_t.h index 9aa311d29..6905450ec 100644 --- a/bsd/sys/_types/_iovec_t.h +++ b/bsd/sys/_types/_iovec_t.h @@ -27,6 +27,7 @@ */ #ifndef _STRUCT_IOVEC #define _STRUCT_IOVEC +#include /* size_t */ struct iovec { void * iov_base; /* [XSI] Base address of I/O memory region */ size_t iov_len; /* [XSI] Size of region iov_base points to */ diff --git a/bsd/sys/_types/_key_t.h b/bsd/sys/_types/_key_t.h index 2b5bdbd4b..1d4ca01ed 100644 --- a/bsd/sys/_types/_key_t.h +++ b/bsd/sys/_types/_key_t.h @@ -27,5 +27,6 @@ */ #ifndef _KEY_T #define _KEY_T +#include /* __int32_t */ typedef __int32_t key_t; /* IPC key (for Sys V IPC) */ #endif /* _KEY_T */ diff --git a/bsd/sys/_types/_mach_port_t.h b/bsd/sys/_types/_mach_port_t.h index d2bbae678..8920a37b2 100644 --- a/bsd/sys/_types/_mach_port_t.h +++ b/bsd/sys/_types/_mach_port_t.h @@ -46,5 +46,6 @@ #ifndef _MACH_PORT_T #define _MACH_PORT_T +#include /* __darwin_mach_port_t */ typedef __darwin_mach_port_t mach_port_t; #endif /* _MACH_PORT_T */ diff --git a/bsd/sys/_types/_mbstate_t.h b/bsd/sys/_types/_mbstate_t.h index 790d112a2..0f51de45c 100644 --- a/bsd/sys/_types/_mbstate_t.h +++ b/bsd/sys/_types/_mbstate_t.h @@ -28,5 +28,6 @@ #ifndef _MBSTATE_T #define _MBSTATE_T +#include /* __darwin_mbstate_t */ typedef __darwin_mbstate_t mbstate_t; #endif /* _MBSTATE_T */ diff --git a/bsd/sys/_types/_mode_t.h b/bsd/sys/_types/_mode_t.h index a378b7dcc..c4de010c7 100644 --- a/bsd/sys/_types/_mode_t.h +++ b/bsd/sys/_types/_mode_t.h @@ -27,5 +27,6 @@ */ #ifndef _MODE_T #define _MODE_T +#include /* __darwin_mode_t */ typedef __darwin_mode_t mode_t; #endif /* _MODE_T */ diff --git a/bsd/sys/_types/_nlink_t.h b/bsd/sys/_types/_nlink_t.h index 6b0e8cd32..7d066e178 100644 --- a/bsd/sys/_types/_nlink_t.h +++ b/bsd/sys/_types/_nlink_t.h @@ -27,5 +27,6 @@ */ #ifndef _NLINK_T #define _NLINK_T +#include /* __uint16_t */ typedef __uint16_t nlink_t; /* link count */ #endif /* _NLINK_T */ diff --git a/bsd/sys/_types/_null.h b/bsd/sys/_types/_null.h index 8a32fe0d5..537c10a3f 100644 --- a/bsd/sys/_types/_null.h +++ b/bsd/sys/_types/_null.h @@ -25,6 +25,7 @@ * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#ifndef NULL +#ifndef NULL +#include /* __DARWIN_NULL */ #define NULL __DARWIN_NULL #endif /* NULL */ diff --git a/bsd/sys/_types/_off_t.h b/bsd/sys/_types/_off_t.h index fc6eacad4..205207ea3 100644 --- a/bsd/sys/_types/_off_t.h +++ b/bsd/sys/_types/_off_t.h @@ -27,5 +27,6 @@ */ #ifndef _OFF_T #define _OFF_T +#include /* __darwin_off_t */ typedef __darwin_off_t off_t; #endif /* _OFF_T */ diff --git a/bsd/sys/_types/_pid_t.h b/bsd/sys/_types/_pid_t.h index ea369b218..5050d5278 100644 --- a/bsd/sys/_types/_pid_t.h +++ b/bsd/sys/_types/_pid_t.h @@ -27,5 +27,6 @@ */ #ifndef _PID_T #define _PID_T +#include /* __darwin_pid_t */ typedef __darwin_pid_t pid_t; #endif /* _PID_T */ diff --git a/bsd/sys/_types/_ptrdiff_t.h b/bsd/sys/_types/_ptrdiff_t.h index 2f7344551..40cba6035 100644 --- a/bsd/sys/_types/_ptrdiff_t.h +++ b/bsd/sys/_types/_ptrdiff_t.h @@ -28,5 +28,6 @@ #ifndef _PTRDIFF_T #define _PTRDIFF_T +#include /* __darwin_ptrdiff_t */ typedef __darwin_ptrdiff_t ptrdiff_t; #endif /* _PTRDIFF_T */ diff --git a/bsd/sys/_types/_rsize_t.h b/bsd/sys/_types/_rsize_t.h index 68e18ef71..7150c6693 100644 --- a/bsd/sys/_types/_rsize_t.h +++ b/bsd/sys/_types/_rsize_t.h @@ -27,5 +27,6 @@ */ #ifndef _RSIZE_T #define _RSIZE_T +#include /* __darwin_size_t */ typedef __darwin_size_t rsize_t; #endif /* _RSIZE_T */ diff --git a/bsd/sys/_types/_rune_t.h b/bsd/sys/_types/_rune_t.h index 19a231899..aa9d0470d 100644 --- a/bsd/sys/_types/_rune_t.h +++ b/bsd/sys/_types/_rune_t.h @@ -27,5 +27,6 @@ */ #ifndef _RUNE_T #define _RUNE_T +#include /* __darwin_rune_t */ typedef __darwin_rune_t rune_t; #endif /* _RUNE_T */ diff --git a/bsd/sys/_types/_sa_family_t.h b/bsd/sys/_types/_sa_family_t.h index 3460f2661..ccd168b2c 100644 --- a/bsd/sys/_types/_sa_family_t.h +++ b/bsd/sys/_types/_sa_family_t.h @@ -27,5 +27,6 @@ */ #ifndef _SA_FAMILY_T #define _SA_FAMILY_T +#include /* __uint8_t */ typedef __uint8_t sa_family_t; #endif /* _SA_FAMILY_T */ diff --git a/bsd/sys/_types/_seek_set.h b/bsd/sys/_types/_seek_set.h index e6302b386..6bcdec84e 100644 --- a/bsd/sys/_types/_seek_set.h +++ b/bsd/sys/_types/_seek_set.h @@ -26,6 +26,8 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +#include + /* whence values for lseek(2) */ #ifndef SEEK_SET #define SEEK_SET 0 /* set file offset to offset */ @@ -33,6 +35,7 @@ #define SEEK_END 2 /* set file offset to EOF plus offset */ #endif /* !SEEK_SET */ +#if __DARWIN_C_LEVEL >= __DARWIN_C_FULL #ifndef SEEK_HOLE #define SEEK_HOLE 3 /* set file offset to the start of the next hole greater than or equal to the supplied offset */ #endif @@ -40,3 +43,4 @@ #ifndef SEEK_DATA #define SEEK_DATA 4 /* set file offset to the start of the next non-hole file region greater than or equal to the supplied offset */ #endif +#endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL */ diff --git a/bsd/sys/_types/_sigaltstack.h b/bsd/sys/_types/_sigaltstack.h index 24d23fefb..353cd5b98 100644 --- a/bsd/sys/_types/_sigaltstack.h +++ b/bsd/sys/_types/_sigaltstack.h @@ -28,11 +28,17 @@ /* Structure used in sigaltstack call. */ #ifndef _STRUCT_SIGALTSTACK + +#include /* __DARWIN_UNIX03 */ + #if __DARWIN_UNIX03 #define _STRUCT_SIGALTSTACK struct __darwin_sigaltstack #else /* !__DARWIN_UNIX03 */ #define _STRUCT_SIGALTSTACK struct sigaltstack #endif /* __DARWIN_UNIX03 */ + +#include /* __darwin_size_t */ + _STRUCT_SIGALTSTACK { void *ss_sp; /* signal stack base */ diff --git a/bsd/sys/_types/_sigset_t.h b/bsd/sys/_types/_sigset_t.h index d4e9b6538..6bf670407 100644 --- a/bsd/sys/_types/_sigset_t.h +++ b/bsd/sys/_types/_sigset_t.h @@ -27,5 +27,6 @@ */ #ifndef _SIGSET_T #define _SIGSET_T +#include /* __darwin_sigset_t */ typedef __darwin_sigset_t sigset_t; #endif /* _SIGSET_T */ diff --git a/bsd/sys/_types/_size_t.h b/bsd/sys/_types/_size_t.h index 8346ba897..67786d594 100644 --- a/bsd/sys/_types/_size_t.h +++ b/bsd/sys/_types/_size_t.h @@ -27,5 +27,6 @@ */ #ifndef _SIZE_T #define _SIZE_T +#include /* __darwin_size_t */ typedef __darwin_size_t size_t; #endif /* _SIZE_T */ diff --git a/bsd/sys/_types/_socklen_t.h b/bsd/sys/_types/_socklen_t.h index 110a3fa74..b9354fde5 100644 --- a/bsd/sys/_types/_socklen_t.h +++ b/bsd/sys/_types/_socklen_t.h @@ -27,6 +27,7 @@ */ #ifndef _SOCKLEN_T #define _SOCKLEN_T +#include /* __darwin_socklen_t */ typedef __darwin_socklen_t socklen_t; #endif diff --git a/bsd/sys/_types/_ssize_t.h b/bsd/sys/_types/_ssize_t.h index 636a850d4..fef63730f 100644 --- a/bsd/sys/_types/_ssize_t.h +++ b/bsd/sys/_types/_ssize_t.h @@ -27,5 +27,6 @@ */ #ifndef _SSIZE_T #define _SSIZE_T +#include /* __darwin_ssize_t */ typedef __darwin_ssize_t ssize_t; #endif /* _SSIZE_T */ diff --git a/bsd/sys/_types/_suseconds_t.h b/bsd/sys/_types/_suseconds_t.h index 883143a73..837c4cab1 100644 --- a/bsd/sys/_types/_suseconds_t.h +++ b/bsd/sys/_types/_suseconds_t.h @@ -27,5 +27,6 @@ */ #ifndef _SUSECONDS_T #define _SUSECONDS_T +#include /* __darwin_suseconds_t */ typedef __darwin_suseconds_t suseconds_t; #endif /* _SUSECONDS_T */ diff --git a/bsd/sys/_types/_time_t.h b/bsd/sys/_types/_time_t.h index 19b5f5e1e..ae87acb6f 100644 --- a/bsd/sys/_types/_time_t.h +++ b/bsd/sys/_types/_time_t.h @@ -27,5 +27,6 @@ */ #ifndef _TIME_T #define _TIME_T +#include /* __darwin_time_t */ typedef __darwin_time_t time_t; #endif /* _TIME_T */ diff --git a/bsd/sys/_types/_timespec.h b/bsd/sys/_types/_timespec.h index 73525337d..6837be1ad 100644 --- a/bsd/sys/_types/_timespec.h +++ b/bsd/sys/_types/_timespec.h @@ -27,6 +27,9 @@ */ #ifndef _STRUCT_TIMESPEC #define _STRUCT_TIMESPEC struct timespec + +#include /* __darwin_time_t */ + _STRUCT_TIMESPEC { __darwin_time_t tv_sec; diff --git a/bsd/sys/_types/_timeval.h b/bsd/sys/_types/_timeval.h index 2f73808a4..2f854b9d9 100644 --- a/bsd/sys/_types/_timeval.h +++ b/bsd/sys/_types/_timeval.h @@ -27,6 +27,10 @@ */ #ifndef _STRUCT_TIMEVAL #define _STRUCT_TIMEVAL struct timeval + +#include /* __darwin_time_t */ +#include /* __darwin_suseconds_t */ + _STRUCT_TIMEVAL { __darwin_time_t tv_sec; /* seconds */ diff --git a/bsd/sys/_types/_timeval32.h b/bsd/sys/_types/_timeval32.h index ae5d3fe8b..dbb66d36e 100644 --- a/bsd/sys/_types/_timeval32.h +++ b/bsd/sys/_types/_timeval32.h @@ -27,6 +27,9 @@ */ #ifndef _STRUCT_TIMEVAL32 #define _STRUCT_TIMEVAL32 struct timeval32 + +#include /* __int32_t */ + _STRUCT_TIMEVAL32 { __int32_t tv_sec; /* seconds */ diff --git a/bsd/sys/_types/_timeval64.h b/bsd/sys/_types/_timeval64.h index c14f8338f..58a3255f9 100644 --- a/bsd/sys/_types/_timeval64.h +++ b/bsd/sys/_types/_timeval64.h @@ -28,6 +28,9 @@ #ifndef _STRUCT_TIMEVAL64 #define _STRUCT_TIMEVAL64 + +#include /* __int64_t */ + struct timeval64 { __int64_t tv_sec; /* seconds */ diff --git a/bsd/sys/_types/_u_char.h b/bsd/sys/_types/_u_char.h new file mode 100644 index 000000000..2a8a5b47e --- /dev/null +++ b/bsd/sys/_types/_u_char.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _U_CHAR +#define _U_CHAR +typedef unsigned char u_char; +#endif /* _U_CHAR */ diff --git a/bsd/sys/_types/_u_int.h b/bsd/sys/_types/_u_int.h new file mode 100644 index 000000000..79c36d1b4 --- /dev/null +++ b/bsd/sys/_types/_u_int.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _U_INT +#define _U_INT +typedef unsigned int u_int; +#endif /* _U_INT */ diff --git a/bsd/sys/_types/_u_short.h b/bsd/sys/_types/_u_short.h new file mode 100644 index 000000000..c610d14c8 --- /dev/null +++ b/bsd/sys/_types/_u_short.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _U_SHORT +#define _U_SHORT +typedef unsigned short u_short; +#endif /* _U_SHORT */ diff --git a/bsd/sys/_types/_ucontext.h b/bsd/sys/_types/_ucontext.h index 159ff0a7e..56a520d7a 100644 --- a/bsd/sys/_types/_ucontext.h +++ b/bsd/sys/_types/_ucontext.h @@ -26,11 +26,19 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _STRUCT_UCONTEXT + +#include /* __DARWIN_UNIX03 */ + #if __DARWIN_UNIX03 #define _STRUCT_UCONTEXT struct __darwin_ucontext #else /* !__DARWIN_UNIX03 */ #define _STRUCT_UCONTEXT struct ucontext #endif /* __DARWIN_UNIX03 */ + +#include /* __darwin_size_t */ +#include /* _STRUCT_MCONTEXT */ +#include /* __darwin_sigset_t */ + _STRUCT_UCONTEXT { int uc_onstack; diff --git a/bsd/sys/_types/_ucontext64.h b/bsd/sys/_types/_ucontext64.h index f2a620a19..1befcc9b0 100644 --- a/bsd/sys/_types/_ucontext64.h +++ b/bsd/sys/_types/_ucontext64.h @@ -26,11 +26,19 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _STRUCT_UCONTEXT64 + +#include /* __DARWIN_UNIX03 */ + #if __DARWIN_UNIX03 #define _STRUCT_UCONTEXT64 struct __darwin_ucontext64 #else /* !__DARWIN_UNIX03 */ #define _STRUCT_UCONTEXT64 struct ucontext64 #endif /* __DARWIN_UNIX03 */ + +#include /* __darwin_size_t */ +#include /* _STRUCT_MCONTEXT */ +#include /* __darwin_sigset_t */ + _STRUCT_UCONTEXT64 { int uc_onstack; diff --git a/bsd/sys/_types/_uid_t.h b/bsd/sys/_types/_uid_t.h index 678f7db14..a4ca9cb6f 100644 --- a/bsd/sys/_types/_uid_t.h +++ b/bsd/sys/_types/_uid_t.h @@ -27,5 +27,6 @@ */ #ifndef _UID_T #define _UID_T +#include /* __darwin_uid_t */ typedef __darwin_uid_t uid_t; #endif /* _UID_T */ diff --git a/bsd/sys/_types/_useconds_t.h b/bsd/sys/_types/_useconds_t.h index 780d2364d..751a3748d 100644 --- a/bsd/sys/_types/_useconds_t.h +++ b/bsd/sys/_types/_useconds_t.h @@ -27,5 +27,6 @@ */ #ifndef _USECONDS_T #define _USECONDS_T +#include /* __darwin_useconds_t */ typedef __darwin_useconds_t useconds_t; #endif /* _USECONDS_T */ diff --git a/bsd/sys/_types/_user32_ntptimeval.h b/bsd/sys/_types/_user32_ntptimeval.h new file mode 100644 index 000000000..cb69d5949 --- /dev/null +++ b/bsd/sys/_types/_user32_ntptimeval.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef KERNEL +#ifndef _STRUCT_USER32_NTPTIMEVAL +#define _STRUCT_USER32_NTPTIMEVAL struct user32_ntptimeval +_STRUCT_USER32_NTPTIMEVAL +{ + struct user32_timespec time; + user32_long_t maxerror; + user32_long_t esterror; + user32_long_t tai; + __int32_t time_state; + +}; +#endif /* _STRUCT_USER32_NTPTIMEVAL */ +#endif /* KERNEL */ diff --git a/bsd/sys/_types/_user32_timex.h b/bsd/sys/_types/_user32_timex.h new file mode 100644 index 000000000..5627982d6 --- /dev/null +++ b/bsd/sys/_types/_user32_timex.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef KERNEL +#ifndef _STRUCT_USER32_TIMEX +#define _STRUCT_USER32_TIMEX struct user32_timex +_STRUCT_USER32_TIMEX +{ + u_int32_t modes; + user32_long_t offset; + user32_long_t freq; + user32_long_t maxerror; + user32_long_t esterror; + __int32_t status; + user32_long_t constant; + user32_long_t precision; + user32_long_t tolerance; + + user32_long_t ppsfreq; + user32_long_t jitter; + __int32_t shift; + user32_long_t stabil; + user32_long_t jitcnt; + user32_long_t calcnt; + user32_long_t errcnt; + user32_long_t stbcnt; + +}; +#endif /* _STRUCT_USER32_TIMEX */ +#endif /* KERNEL */ diff --git a/bsd/sys/_types/_user64_ntptimeval.h b/bsd/sys/_types/_user64_ntptimeval.h new file mode 100644 index 000000000..3c3c557fd --- /dev/null +++ b/bsd/sys/_types/_user64_ntptimeval.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef KERNEL +#ifndef _STRUCT_USER64_NTPTIMEVAL +#define _STRUCT_USER64_NTPTIMEVAL struct user64_ntptimeval +_STRUCT_USER64_NTPTIMEVAL +{ + struct user64_timespec time; + user64_long_t maxerror; + user64_long_t esterror; + user64_long_t tai; + __int64_t time_state; + +}; +#endif /* _STRUCT_USER64_NTPTIMEVAL */ +#endif /* KERNEL */ diff --git a/bsd/sys/_types/_user64_timex.h b/bsd/sys/_types/_user64_timex.h new file mode 100644 index 000000000..2547592f4 --- /dev/null +++ b/bsd/sys/_types/_user64_timex.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef KERNEL +#ifndef _STRUCT_USER64_TIMEX +#define _STRUCT_USER64_TIMEX struct user64_timex +_STRUCT_USER64_TIMEX +{ + u_int64_t modes; + user64_long_t offset; + user64_long_t freq; + user64_long_t maxerror; + user64_long_t esterror; + __int64_t status; + user64_long_t constant; + user64_long_t precision; + user64_long_t tolerance; + + user64_long_t ppsfreq; + user64_long_t jitter; + __int64_t shift; + user64_long_t stabil; + user64_long_t jitcnt; + user64_long_t calcnt; + user64_long_t errcnt; + user64_long_t stbcnt; + +}; +#endif /* _STRUCT_USER64_TIMEX */ +#endif /* KERNEL */ diff --git a/bsd/sys/_types/_uuid_t.h b/bsd/sys/_types/_uuid_t.h index b61023892..e459143cd 100644 --- a/bsd/sys/_types/_uuid_t.h +++ b/bsd/sys/_types/_uuid_t.h @@ -27,5 +27,6 @@ */ #ifndef _UUID_T #define _UUID_T +#include /* __darwin_uuid_t */ typedef __darwin_uuid_t uuid_t; #endif /* _UUID_T */ diff --git a/bsd/sys/_types/_va_list.h b/bsd/sys/_types/_va_list.h index c36072a5f..48a2b9969 100644 --- a/bsd/sys/_types/_va_list.h +++ b/bsd/sys/_types/_va_list.h @@ -28,5 +28,6 @@ #ifndef _VA_LIST_T #define _VA_LIST_T +#include /* __darwin_va_list */ typedef __darwin_va_list va_list; #endif /* _VA_LIST_T */ diff --git a/bsd/sys/_types/_wchar_t.h b/bsd/sys/_types/_wchar_t.h index 5a5d56cb7..a452a5fac 100644 --- a/bsd/sys/_types/_wchar_t.h +++ b/bsd/sys/_types/_wchar_t.h @@ -30,6 +30,7 @@ #ifndef __cplusplus #ifndef _WCHAR_T #define _WCHAR_T +#include /* __darwin_wchar_t */ typedef __darwin_wchar_t wchar_t; #endif /* _WCHAR_T */ #endif /* __cplusplus */ diff --git a/bsd/sys/_types/_wint_t.h b/bsd/sys/_types/_wint_t.h index d1bbbad87..66dd7c37f 100644 --- a/bsd/sys/_types/_wint_t.h +++ b/bsd/sys/_types/_wint_t.h @@ -28,5 +28,6 @@ #ifndef _WINT_T #define _WINT_T +#include /* __darwin_wint_t */ typedef __darwin_wint_t wint_t; #endif /* _WINT_T */ diff --git a/bsd/sys/acct.h b/bsd/sys/acct.h index 050ccc3f9..8162eb91f 100644 --- a/bsd/sys/acct.h +++ b/bsd/sys/acct.h @@ -70,6 +70,10 @@ #include #include +#include /* u_int16_t */ +#include /* u_int32_t */ +#include /* uid_t */ + /* * Accounting structures; these use a comp_t type which is a 3 bits base 8 * exponent, 13 bit fraction ``floating point'' number. Units are 1/AHZ diff --git a/bsd/sys/attr.h b/bsd/sys/attr.h index 3437f9cc7..45540a9aa 100644 --- a/bsd/sys/attr.h +++ b/bsd/sys/attr.h @@ -320,6 +320,8 @@ typedef struct vol_capabilities_attr { * * VOL_CAP_INT_CLONE: When set, the volume supports clones. * + * VOL_CAP_INT_SNAPSHOT: When set, the volume supports snapshots. + * * VOL_CAP_INT_RENAME_SWAP: When set, the volume supports swapping * file system objects. * @@ -347,10 +349,7 @@ typedef struct vol_capabilities_attr { #define VOL_CAP_INT_REMOTE_EVENT 0x00008000 #endif /* PRIVATE */ #define VOL_CAP_INT_CLONE 0x00010000 -#ifdef PRIVATE -/* Volume supports snapshots */ #define VOL_CAP_INT_SNAPSHOT 0x00020000 -#endif /* PRIVATE */ #define VOL_CAP_INT_RENAME_SWAP 0x00040000 #define VOL_CAP_INT_RENAME_EXCL 0x00080000 @@ -432,7 +431,7 @@ typedef struct vol_attributes_attr { * * ATTR_CMN_DATA_PROTECT_FLAGS */ -#define ATTR_CMN_SETMASK 0x41C7FF00 +#define ATTR_CMN_SETMASK 0x51C7FF00 #define ATTR_CMN_VOLSETMASK 0x00006700 #define ATTR_VOL_FSTYPE 0x00000001 @@ -506,9 +505,7 @@ typedef struct vol_attributes_attr { /* CMNEXT attributes extend the common attributes, but in the forkattr field */ #define ATTR_CMNEXT_RELPATH 0x00000004 #define ATTR_CMNEXT_PRIVATESIZE 0x00000008 -#ifdef PRIVATE #define ATTR_CMNEXT_LINKID 0x00000010 -#endif /* PRIVATE */ #define ATTR_CMNEXT_VALIDMASK 0x0000001c #define ATTR_CMNEXT_SETMASK 0x00000000 diff --git a/bsd/sys/bitstring.h b/bsd/sys/bitstring.h index f4bb7fa54..3da8f42db 100644 --- a/bsd/sys/bitstring.h +++ b/bsd/sys/bitstring.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Apple Inc. All rights reserved. + * Copyright (c) 2013-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -68,11 +68,11 @@ typedef uint8_t bitstr_t; /* internal macros */ /* byte of the bitstring bit is in */ -#define _bit_byte(bit) \ +#define _bitstr_byte(bit) \ ((bit) >> 3) /* mask for the bit within its byte */ -#define _bit_mask(bit) \ +#define _bitstr_mask(bit) \ (1 << ((bit) & 0x7)) /* external macros */ @@ -85,31 +85,31 @@ typedef uint8_t bitstr_t; ((name)[bitstr_size(nbits)]) /* is bit N of bitstring name set? */ -#define bit_test(name, bit) \ - ((name)[_bit_byte(bit)] & _bit_mask(bit)) +#define bitstr_test(name, bit) \ + ((name)[_bitstr_byte(bit)] & _bitstr_mask(bit)) /* set bit N of bitstring name */ -#define bit_set(name, bit) \ - ((name)[_bit_byte(bit)] |= _bit_mask(bit)) +#define bitstr_set(name, bit) \ + ((name)[_bitstr_byte(bit)] |= _bitstr_mask(bit)) /* set bit N of bitstring name (atomic) */ -#define bit_set_atomic(name, bit) \ - atomic_bitset_8(&((name)[_bit_byte(bit)]), _bit_mask(bit)) +#define bitstr_set_atomic(name, bit) \ + atomic_bitset_8(&((name)[_bitstr_byte(bit)]), _bitstr_mask(bit)) /* clear bit N of bitstring name */ -#define bit_clear(name, bit) \ - ((name)[_bit_byte(bit)] &= ~_bit_mask(bit)) +#define bitstr_clear(name, bit) \ + ((name)[_bitstr_byte(bit)] &= ~_bitstr_mask(bit)) /* clear bit N of bitstring name (atomic) */ -#define bit_clear_atomic(name, bit) \ - atomic_bitclear_8(&((name)[_bit_byte(bit)]), _bit_mask(bit)) +#define bitstr_clear_atomic(name, bit) \ + atomic_bitclear_8(&((name)[_bitstr_byte(bit)]), _bitstr_mask(bit)) /* clear bits start ... stop in bitstring */ -#define bit_nclear(name, start, stop) do { \ +#define bitstr_nclear(name, start, stop) do { \ bitstr_t *_name = (name); \ int _start = (start), _stop = (stop); \ - int _startbyte = _bit_byte(_start); \ - int _stopbyte = _bit_byte(_stop); \ + int _startbyte = _bitstr_byte(_start); \ + int _stopbyte = _bitstr_byte(_stop); \ if (_startbyte == _stopbyte) { \ _name[_startbyte] &= ((0xff >> (8 - (_start & 0x7))) | \ (0xff << ((_stop & 0x7) + 1))); \ @@ -122,11 +122,11 @@ typedef uint8_t bitstr_t; } while (0) /* set bits start ... stop in bitstring */ -#define bit_nset(name, start, stop) do { \ +#define bitstr_nset(name, start, stop) do { \ bitstr_t *_name = (name); \ int _start = (start), _stop = (stop); \ - int _startbyte = _bit_byte(_start); \ - int _stopbyte = _bit_byte(_stop); \ + int _startbyte = _bitstr_byte(_start); \ + int _stopbyte = _bitstr_byte(_stop); \ if (_startbyte == _stopbyte) { \ _name[_startbyte] |= ((0xff << (_start & 0x7)) & \ (0xff >> (7 - (_stop & 0x7)))); \ @@ -139,10 +139,10 @@ typedef uint8_t bitstr_t; } while (0) /* find first bit clear in name */ -#define bit_ffc(name, nbits, value) do { \ +#define bitstr_ffc(name, nbits, value) do { \ bitstr_t *_name = (name); \ int _byte, _nbits = (nbits); \ - int _stopbyte = _bit_byte(_nbits - 1), _value = -1; \ + int _stopbyte = _bitstr_byte(_nbits - 1), _value = -1; \ if (_nbits > 0) \ for (_byte = 0; _byte <= _stopbyte; ++_byte) \ if (_name[_byte] != 0xff) { \ @@ -158,10 +158,10 @@ typedef uint8_t bitstr_t; } while (0) /* find first bit set in name */ -#define bit_ffs(name, nbits, value) do { \ +#define bitstr_ffs(name, nbits, value) do { \ bitstr_t *_name = (name); \ int _byte, _nbits = (nbits); \ - int _stopbyte = _bit_byte(_nbits - 1), _value = -1; \ + int _stopbyte = _bitstr_byte(_nbits - 1), _value = -1; \ if (_nbits > 0) \ for (_byte = 0; _byte <= _stopbyte; ++_byte) \ if (_name[_byte]) { \ diff --git a/bsd/sys/buf_internal.h b/bsd/sys/buf_internal.h index 3b007d99e..23c9ecba6 100644 --- a/bsd/sys/buf_internal.h +++ b/bsd/sys/buf_internal.h @@ -104,6 +104,7 @@ struct buf { LIST_ENTRY(buf) b_vnbufs; /* Buffer's associated vnode. */ TAILQ_ENTRY(buf) b_freelist; /* Free list position if not active. */ int b_timestamp; /* timestamp for queuing operation */ + struct timeval b_timestamp_tv; /* microuptime for disk conditioner */ int b_whichq; /* the free list the buffer belongs to */ volatile uint32_t b_flags; /* B_* flags. */ volatile uint32_t b_lflags; /* BL_BUSY | BL_WANTED flags... protected by buf_mtx */ diff --git a/bsd/sys/cdefs.h b/bsd/sys/cdefs.h index 8137cb3d9..68b4d22d9 100644 --- a/bsd/sys/cdefs.h +++ b/bsd/sys/cdefs.h @@ -498,6 +498,12 @@ #define __DARWIN_ONLY_UNIX_CONFORMANCE 1 #define __DARWIN_ONLY_VERS_1050 1 #endif /* PLATFORM_WatchSimulator */ +#ifdef PLATFORM_BridgeOS +/* Platform: BridgeOS */ +#define __DARWIN_ONLY_64_BIT_INO_T 1 +#define __DARWIN_ONLY_UNIX_CONFORMANCE 1 +#define __DARWIN_ONLY_VERS_1050 1 +#endif /* PLATFORM_BridgeOS */ #ifdef PLATFORM_MacOSX /* Platform: MacOSX */ #define __DARWIN_ONLY_64_BIT_INO_T 0 @@ -849,6 +855,8 @@ */ #if !defined(__sys_cdefs_arch_unknown__) && defined(__i386__) #elif !defined(__sys_cdefs_arch_unknown__) && defined(__x86_64__) +#elif !defined(__sys_cdefs_arch_unknown__) && defined(__arm__) +#elif !defined(__sys_cdefs_arch_unknown__) && defined(__arm64__) #else #error Unsupported architecture #endif diff --git a/bsd/sys/clonefile.h b/bsd/sys/clonefile.h index 17773fd3a..45dfef2bf 100644 --- a/bsd/sys/clonefile.h +++ b/bsd/sys/clonefile.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,7 +30,8 @@ #define _SYS_CLONEFILE_H_ /* Options for clonefile calls */ -#define CLONE_NOFOLLOW 0x0001 /* Don't follow symbolic links */ +#define CLONE_NOFOLLOW 0x0001 /* Don't follow symbolic links */ +#define CLONE_NOOWNERCOPY 0x0002 /* Don't copy ownership information from source */ #ifndef KERNEL diff --git a/bsd/sys/coalition.h b/bsd/sys/coalition.h index cf2811c5b..d3ab93f32 100644 --- a/bsd/sys/coalition.h +++ b/bsd/sys/coalition.h @@ -47,6 +47,8 @@ int coalition_reap(uint64_t cid, uint32_t flags); /* Wrappers around __coalition_info syscall (with proper struct types) */ int coalition_info_resource_usage(uint64_t cid, struct coalition_resource_usage *cru, size_t sz); +int coalition_info_set_name(uint64_t cid, const char *name, size_t size); +int coalition_info_set_efficiency(uint64_t cid, uint64_t flags); #else /* KERNEL */ diff --git a/bsd/sys/codesign.h b/bsd/sys/codesign.h index ca225ce58..15a39365b 100644 --- a/bsd/sys/codesign.h +++ b/bsd/sys/codesign.h @@ -29,36 +29,11 @@ #ifndef _SYS_CODESIGN_H_ #define _SYS_CODESIGN_H_ -/* code signing attributes of a process */ -#define CS_VALID 0x0000001 /* dynamically valid */ -#define CS_ADHOC 0x0000002 /* ad hoc signed */ -#define CS_GET_TASK_ALLOW 0x0000004 /* has get-task-allow entitlement */ -#define CS_INSTALLER 0x0000008 /* has installer entitlement */ - -#define CS_HARD 0x0000100 /* don't load invalid pages */ -#define CS_KILL 0x0000200 /* kill process if it becomes invalid */ -#define CS_CHECK_EXPIRATION 0x0000400 /* force expiration checking */ -#define CS_RESTRICT 0x0000800 /* tell dyld to treat restricted */ -#define CS_ENFORCEMENT 0x0001000 /* require enforcement */ -#define CS_REQUIRE_LV 0x0002000 /* require library validation */ -#define CS_ENTITLEMENTS_VALIDATED 0x0004000 /* code signature permits restricted entitlements */ - -#define CS_ALLOWED_MACHO (CS_ADHOC | CS_HARD | CS_KILL | CS_CHECK_EXPIRATION | CS_RESTRICT | CS_ENFORCEMENT | CS_REQUIRE_LV) - -#define CS_EXEC_SET_HARD 0x0100000 /* set CS_HARD on any exec'ed process */ -#define CS_EXEC_SET_KILL 0x0200000 /* set CS_KILL on any exec'ed process */ -#define CS_EXEC_SET_ENFORCEMENT 0x0400000 /* set CS_ENFORCEMENT on any exec'ed process */ -#define CS_EXEC_SET_INSTALLER 0x0800000 /* set CS_INSTALLER on any exec'ed process */ - -#define CS_KILLED 0x1000000 /* was killed by kernel for invalidity */ -#define CS_DYLD_PLATFORM 0x2000000 /* dyld used to load this is a platform binary */ -#define CS_PLATFORM_BINARY 0x4000000 /* this is a platform binary */ -#define CS_PLATFORM_PATH 0x8000000 /* platform binary by the fact of path (osx only) */ -#define CS_DEBUGGED 0x10000000 /* process is currently or has previously been debugged and allowed to run with invalid pages */ -#define CS_SIGNED 0x20000000 /* process has a signature (may have gone invalid) */ -#define CS_DEV_CODE 0x40000000 /* code is dev signed, cannot be loaded into prod signed code (will go away with rdar://problem/28322552) */ - -#define CS_ENTITLEMENT_FLAGS (CS_GET_TASK_ALLOW | CS_INSTALLER) +#if KERNEL +#include +#else +#include +#endif /* MAC flags used by F_ADDFILESIGS_* */ #define MAC_VNODE_CHECK_DYLD_SIM 0x1 /* tells the MAC framework that dyld-sim is being loaded */ @@ -79,111 +54,9 @@ #define CS_OPS_BLOB 10 /* get codesign blob */ #define CS_OPS_IDENTITY 11 /* get codesign identity */ #define CS_OPS_CLEARINSTALLER 12 /* clear INSTALLER flag */ +#define CS_OPS_CLEARPLATFORM 13 /* clear platform binary status (DEVELOPMENT-only) */ -/* - * Magic numbers used by Code Signing - */ -enum { - CSMAGIC_REQUIREMENT = 0xfade0c00, /* single Requirement blob */ - CSMAGIC_REQUIREMENTS = 0xfade0c01, /* Requirements vector (internal requirements) */ - CSMAGIC_CODEDIRECTORY = 0xfade0c02, /* CodeDirectory blob */ - CSMAGIC_EMBEDDED_SIGNATURE = 0xfade0cc0, /* embedded form of signature data */ - CSMAGIC_EMBEDDED_SIGNATURE_OLD = 0xfade0b02, /* XXX */ - CSMAGIC_EMBEDDED_ENTITLEMENTS = 0xfade7171, /* embedded entitlements */ - CSMAGIC_DETACHED_SIGNATURE = 0xfade0cc1, /* multi-arch collection of embedded signatures */ - CSMAGIC_BLOBWRAPPER = 0xfade0b01, /* CMS Signature, among other things */ - - CS_SUPPORTSSCATTER = 0x20100, - CS_SUPPORTSTEAMID = 0x20200, - - CSSLOT_CODEDIRECTORY = 0, /* slot index for CodeDirectory */ - CSSLOT_INFOSLOT = 1, - CSSLOT_REQUIREMENTS = 2, - CSSLOT_RESOURCEDIR = 3, - CSSLOT_APPLICATION = 4, - CSSLOT_ENTITLEMENTS = 5, - - CSSLOT_ALTERNATE_CODEDIRECTORIES = 0x1000, /* first alternate CodeDirectory, if any */ - CSSLOT_ALTERNATE_CODEDIRECTORY_MAX = 5, /* max number of alternate CD slots */ - CSSLOT_ALTERNATE_CODEDIRECTORY_LIMIT = CSSLOT_ALTERNATE_CODEDIRECTORIES + CSSLOT_ALTERNATE_CODEDIRECTORY_MAX, /* one past the last */ - - CSSLOT_SIGNATURESLOT = 0x10000, /* CMS Signature */ - - CSTYPE_INDEX_REQUIREMENTS = 0x00000002, /* compat with amfi */ - CSTYPE_INDEX_ENTITLEMENTS = 0x00000005, /* compat with amfi */ - - CS_HASHTYPE_SHA1 = 1, - CS_HASHTYPE_SHA256 = 2, - CS_HASHTYPE_SHA256_TRUNCATED = 3, - CS_HASHTYPE_SHA384 = 4, - - CS_SHA1_LEN = 20, - CS_SHA256_TRUNCATED_LEN = 20, - - CS_CDHASH_LEN = 20, /* always - larger hashes are truncated */ - CS_HASH_MAX_SIZE = 48, /* max size of the hash we'll support */ -}; - - -#define KERNEL_HAVE_CS_CODEDIRECTORY 1 -#define KERNEL_CS_CODEDIRECTORY_HAVE_PLATFORM 1 - -/* - * C form of a CodeDirectory. - */ -typedef struct __CodeDirectory { - uint32_t magic; /* magic number (CSMAGIC_CODEDIRECTORY) */ - uint32_t length; /* total length of CodeDirectory blob */ - uint32_t version; /* compatibility version */ - uint32_t flags; /* setup and mode flags */ - uint32_t hashOffset; /* offset of hash slot element at index zero */ - uint32_t identOffset; /* offset of identifier string */ - uint32_t nSpecialSlots; /* number of special hash slots */ - uint32_t nCodeSlots; /* number of ordinary (code) hash slots */ - uint32_t codeLimit; /* limit to main image signature range */ - uint8_t hashSize; /* size of each hash in bytes */ - uint8_t hashType; /* type of hash (cdHashType* constants) */ - uint8_t platform; /* platform identifier; zero if not platform binary */ - uint8_t pageSize; /* log2(page size in bytes); 0 => infinite */ - uint32_t spare2; /* unused (must be zero) */ - /* Version 0x20100 */ - uint32_t scatterOffset; /* offset of optional scatter vector */ - /* Version 0x20200 */ - uint32_t teamOffset; /* offset of optional team identifier */ - /* followed by dynamic content as located by offset fields above */ -} CS_CodeDirectory; - -/* - * Structure of an embedded-signature SuperBlob - */ - -typedef struct __BlobIndex { - uint32_t type; /* type of entry */ - uint32_t offset; /* offset of entry */ -} CS_BlobIndex; - -typedef struct __SC_SuperBlob { - uint32_t magic; /* magic number */ - uint32_t length; /* total length of SuperBlob */ - uint32_t count; /* number of index entries following */ - CS_BlobIndex index[]; /* (count) entries */ - /* followed by Blobs in no particular order as indicated by offsets in index */ -} CS_SuperBlob; - -#define KERNEL_HAVE_CS_GENERICBLOB 1 -typedef struct __SC_GenericBlob { - uint32_t magic; /* magic number */ - uint32_t length; /* total length of blob */ - char data[]; -} CS_GenericBlob; - -typedef struct __SC_Scatter { - uint32_t count; // number of pages; zero for sentinel (only) - uint32_t base; // first page number - uint64_t targetOffset; // offset in target - uint64_t spare; // reserved -} SC_Scatter; - +#define CS_MAX_TEAMID_LEN 64 #ifndef KERNEL @@ -228,7 +101,12 @@ const char * csblob_get_teamid(struct cs_blob *); const char * csblob_get_identity(struct cs_blob *); const uint8_t * csblob_get_cdhash(struct cs_blob *); int csblob_get_platform_binary(struct cs_blob *); -unsigned int csblob_get_flags(struct cs_blob *blob); +unsigned int csblob_get_flags(struct cs_blob *); +uint8_t csblob_get_hashtype(struct cs_blob const *); +unsigned int csblob_get_signer_type(struct cs_blob *); +#if DEVELOPMENT || DEBUG +void csproc_clear_platform_binary(struct proc *); +#endif int csblob_get_entitlements(struct cs_blob *, void **, size_t *); @@ -252,6 +130,8 @@ int csfg_get_path(struct fileglob *, char *, int *); int csfg_get_platform_binary(struct fileglob *); uint8_t * csfg_get_cdhash(struct fileglob *, uint64_t, size_t *); int csfg_get_prod_signed(struct fileglob *); +unsigned int csfg_get_signer_type(struct fileglob *); +unsigned int csproc_get_signer_type(struct proc *); extern int cs_debug; diff --git a/bsd/sys/commpage.h b/bsd/sys/commpage.h new file mode 100644 index 000000000..42bfe61b2 --- /dev/null +++ b/bsd/sys/commpage.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _COMMPAGE_H +#define _COMMPAGE_H + +#ifdef PRIVATE +typedef volatile struct commpage_timeofday_data{ + uint64_t TimeStamp_tick; + uint64_t TimeStamp_sec; + uint64_t TimeStamp_frac; + uint64_t Ticks_scale; + uint64_t Ticks_per_sec; +} new_commpage_timeofday_data_t; +#endif + +#endif diff --git a/bsd/sys/conf.h b/bsd/sys/conf.h index 3b3f0e234..273b690ad 100644 --- a/bsd/sys/conf.h +++ b/bsd/sys/conf.h @@ -200,8 +200,10 @@ struct cdevsw { #ifdef BSD_KERNEL_PRIVATE extern uint64_t cdevsw_flags[]; -#define CDEVSW_SELECT_KQUEUE 0x01 -#define CDEVSW_USE_OFFSET 0x02 +#define CDEVSW_SELECT_KQUEUE 0x01 +#define CDEVSW_USE_OFFSET 0x02 +#define CDEVSW_IS_PTC 0x04 +#define CDEVSW_IS_PTS 0x08 struct thread; diff --git a/bsd/sys/csr.h b/bsd/sys/csr.h index 602b6061b..4c7f51ece 100644 --- a/bsd/sys/csr.h +++ b/bsd/sys/csr.h @@ -49,6 +49,7 @@ typedef uint32_t csr_op_t; #define CSR_ALLOW_UNRESTRICTED_NVRAM (1 << 6) #define CSR_ALLOW_DEVICE_CONFIGURATION (1 << 7) #define CSR_ALLOW_ANY_RECOVERY_OS (1 << 8) +#define CSR_ALLOW_UNAPPROVED_KEXTS (1 << 9) #define CSR_VALID_FLAGS (CSR_ALLOW_UNTRUSTED_KEXTS | \ CSR_ALLOW_UNRESTRICTED_FS | \ @@ -58,7 +59,8 @@ typedef uint32_t csr_op_t; CSR_ALLOW_UNRESTRICTED_DTRACE | \ CSR_ALLOW_UNRESTRICTED_NVRAM | \ CSR_ALLOW_DEVICE_CONFIGURATION | \ - CSR_ALLOW_ANY_RECOVERY_OS) + CSR_ALLOW_ANY_RECOVERY_OS | \ + CSR_ALLOW_UNAPPROVED_KEXTS) #define CSR_ALWAYS_ENFORCED_FLAGS (CSR_ALLOW_DEVICE_CONFIGURATION | CSR_ALLOW_ANY_RECOVERY_OS) diff --git a/bsd/sys/decmpfs.h b/bsd/sys/decmpfs.h index 3638f9fa0..f30a6decc 100644 --- a/bsd/sys/decmpfs.h +++ b/bsd/sys/decmpfs.h @@ -160,7 +160,7 @@ typedef struct { } decmpfs_registration; /* hooks for kexts to call */ -errno_t register_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration); +errno_t register_decmpfs_decompressor(uint32_t compression_type, const decmpfs_registration *registration); errno_t unregister_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration); #endif /* KERNEL */ diff --git a/bsd/sys/disk.h b/bsd/sys/disk.h index a365b5030..f046e7f76 100644 --- a/bsd/sys/disk.h +++ b/bsd/sys/disk.h @@ -331,10 +331,21 @@ typedef struct _dk_cs_unmap { #define _DKIOCCSMAP _IOWR('d', 202, _dk_cs_map_t) // No longer used: _DKIOCCSSETFSVNODE (203) & _DKIOCCSGETFREEBYTES (204) #define _DKIOCCSUNMAP _IOWR('d', 205, _dk_cs_unmap_t) + +typedef enum { + DK_APFS_ONE_DEVICE = 1, + DK_APFS_FUSION +} dk_apfs_flavour_t; + +#define DKIOCGETAPFSFLAVOUR _IOR('d', 91, dk_apfs_flavour_t) + #endif /* PRIVATE */ #endif /* KERNEL */ #ifdef PRIVATE +#if TARGET_OS_EMBEDDED +#define _DKIOCSETSTATIC _IO('d', 84) +#endif /* TARGET_OS_EMBEDDED */ #endif /* PRIVATE */ #endif /* _SYS_DISK_H_ */ diff --git a/bsd/sys/domain.h b/bsd/sys/domain.h index fa77f963b..6c4a1a0b6 100644 --- a/bsd/sys/domain.h +++ b/bsd/sys/domain.h @@ -124,6 +124,9 @@ struct domain { #pragma pack() #ifdef XNU_KERNEL_PRIVATE + +#include + /* * Internal, private and extendable representation of domain. */ diff --git a/bsd/sys/dtrace.h b/bsd/sys/dtrace.h index b44140030..29540c2a3 100644 --- a/bsd/sys/dtrace.h +++ b/bsd/sys/dtrace.h @@ -342,6 +342,10 @@ typedef enum dtrace_probespec { #define DIF_VAR_DISPATCHQADDR 0x0201 /* Apple specific dispatch queue addr */ #define DIF_VAR_MACHTIMESTAMP 0x0202 /* mach_absolute_timestamp() */ #define DIF_VAR_CPU 0x0203 /* cpu number */ +#define DIF_VAR_CPUINSTRS 0x0204 /* cpu instructions */ +#define DIF_VAR_CPUCYCLES 0x0205 /* cpu cycles */ +#define DIF_VAR_VINSTRS 0x0206 /* virtual instructions */ +#define DIF_VAR_VCYCLES 0x0207 /* virtual cycles */ #endif /* __APPLE __ */ #define DIF_SUBR_RAND 0 @@ -2508,6 +2512,9 @@ extern int (*dtrace_return_probe_ptr)(struct regs *); #if defined (__i386__) || defined(__x86_64__) extern int (*dtrace_pid_probe_ptr)(x86_saved_state_t *regs); extern int (*dtrace_return_probe_ptr)(x86_saved_state_t* regs); +#elif defined (__arm__) || defined(__arm64__) +extern int (*dtrace_pid_probe_ptr)(arm_saved_state_t *regs); +extern int (*dtrace_return_probe_ptr)(arm_saved_state_t *regs); #else #error architecture not supported #endif @@ -2579,6 +2586,13 @@ extern void *dtrace_invop_callsite_pre; extern void *dtrace_invop_callsite_post; #endif +#if defined(__arm__) || defined(__arm64__) +extern int dtrace_instr_size(uint32_t instr, int thumb_mode); +extern void dtrace_invop_add(int (*)(uintptr_t, uintptr_t *, uintptr_t)); +extern void dtrace_invop_remove(int (*)(uintptr_t, uintptr_t *, uintptr_t)); +extern void *dtrace_invop_callsite_pre; +extern void *dtrace_invop_callsite_post; +#endif #undef proc_t #endif /* __APPLE__ */ @@ -2617,6 +2631,13 @@ extern void *dtrace_invop_callsite_post; #endif +#if defined(__arm__) || defined(__arm64__) + +#define DTRACE_INVOP_NOP 4 +#define DTRACE_INVOP_RET 5 +#define DTRACE_INVOP_B 6 + +#endif #endif /* __APPLE__ */ diff --git a/bsd/sys/dtrace_glue.h b/bsd/sys/dtrace_glue.h index c5a1ebf6d..d8a65de0c 100644 --- a/bsd/sys/dtrace_glue.h +++ b/bsd/sys/dtrace_glue.h @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -231,6 +232,8 @@ typedef struct modctl { #define MODCTL_HAS_UUID 0x40 // Module has UUID #define MODCTL_FBT_PRIVATE_PROBES_PROVIDED 0x80 // fbt private probes have been provided #define MODCTL_FBT_PROVIDE_PRIVATE_PROBES 0x100 // fbt provider must provide private probes +#define MODCTL_FBT_PROVIDE_BLACKLISTED_PROBES 0x200 // fbt provider must provide blacklisted probes +#define MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED 0x400 // fbt blacklisted probes have been provided /* Simple/singular mod_flags accessors */ #define MOD_IS_MACH_KERNEL(mod) (mod->mod_flags & MODCTL_IS_MACH_KERNEL) @@ -243,10 +246,13 @@ typedef struct modctl { #define MOD_HAS_UUID(mod) (mod->mod_flags & MODCTL_HAS_UUID) #define MOD_FBT_PRIVATE_PROBES_PROVIDED(mod) (mod->mod_flags & MODCTL_FBT_PRIVATE_PROBES_PROVIDED) #define MOD_FBT_PROVIDE_PRIVATE_PROBES(mod) (mod->mod_flags & MODCTL_FBT_PROVIDE_PRIVATE_PROBES) +#define MOD_FBT_BLACKLISTED_PROBES_PROVIDED(mod) (mod->mod_flags & MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED) +#define MOD_FBT_PROVIDE_BLACKLISTED_PROBES(mod) (mod->mod_flags & MODCTL_FBT_PROVIDE_BLACKLISTED_PROBES) /* Compound accessors */ #define MOD_FBT_PRIVATE_PROBES_DONE(mod) (MOD_FBT_PRIVATE_PROBES_PROVIDED(mod) || !MOD_FBT_PROVIDE_PRIVATE_PROBES(mod)) -#define MOD_FBT_DONE(mod) ((MOD_FBT_PROBES_PROVIDED(mod) && MOD_FBT_PRIVATE_PROBES_DONE(mod)) || MOD_FBT_INVALID(mod)) +#define MOD_FBT_BLACKLISTED_PROBES_DONE(mod) (MOD_FBT_BLACKLISTED_PROBES_PROVIDED(mod) || !MOD_FBT_PROVIDE_BLACKLISTED_PROBES(mod)) +#define MOD_FBT_DONE(mod) ((MOD_FBT_PROBES_PROVIDED(mod) && MOD_FBT_PRIVATE_PROBES_DONE(mod) && MOD_FBT_BLACKLISTED_PROBES_DONE(mod)) || MOD_FBT_INVALID(mod)) #define MOD_SDT_DONE(mod) (MOD_SDT_PROBES_PROVIDED(mod) || MOD_SDT_INVALID(mod)) #define MOD_SYMBOLS_DONE(mod) (MOD_FBT_DONE(mod) && MOD_SDT_DONE(mod)) @@ -325,12 +331,6 @@ extern void cyclic_remove_omni(cyclic_id_list_t); extern cyclic_id_t cyclic_timer_add(cyc_handler_t *, cyc_time_t *); extern void cyclic_timer_remove(cyclic_id_t); -/* - * timeout / untimeout (converted to dtrace_timeout / dtrace_untimeout due to name collision) - */ - -thread_call_t dtrace_timeout(void (*func)(void *, void *), void* arg, uint64_t nanos); - /* * ddi */ @@ -505,6 +505,16 @@ extern void vmem_free(vmem_t *vmp, void *vaddr, size_t size); * Atomic */ +static inline uint8_t atomic_or_8(uint8_t *addr, uint8_t mask) +{ + return OSBitOrAtomic8(mask, addr); +} + +static inline uint32_t atomic_and_32( uint32_t *addr, int32_t mask) +{ + return OSBitAndAtomic(mask, addr); +} + static inline uint32_t atomic_add_32( uint32_t *theAddress, int32_t theAmount ) { return OSAddAtomic( theAmount, theAddress ); @@ -515,12 +525,22 @@ static inline void atomic_add_64( uint64_t *theAddress, int64_t theAmount ) { (void)OSAddAtomic64( theAmount, (SInt64 *)theAddress ); } -#endif - -static inline uint32_t atomic_and_32(uint32_t *addr, uint32_t mask) +#elif defined(__arm__) +static inline void atomic_add_64( uint64_t *theAddress, int64_t theAmount ) { - return OSBitAndAtomic(mask, addr); + // FIXME + // atomic_add_64() is at present only called from fasttrap.c to increment + // or decrement a 64bit counter. Narrow to 32bits since arm has + // no convenient 64bit atomic op. + + (void)OSAddAtomic( (int32_t)theAmount, &(((SInt32 *)theAddress)[0])); } +#elif defined (__arm64__) +static inline void atomic_add_64( uint64_t *theAddress, int64_t theAmount ) +{ + (void)OSAddAtomic64( theAmount, (SInt64 *)theAddress ); +} +#endif static inline uint32_t atomic_or_32(uint32_t *addr, uint32_t mask) { @@ -534,12 +554,14 @@ static inline uint32_t atomic_or_32(uint32_t *addr, uint32_t mask) typedef uintptr_t pc_t; typedef uintptr_t greg_t; /* For dtrace_impl.h prototype of dtrace_getfp() */ +#if defined(__arm__) || defined(__arm64__) +#define regs arm_saved_state +#endif extern struct regs *find_user_regs( thread_t thread); extern vm_offset_t dtrace_get_cpu_int_stack_top(void); extern vm_offset_t max_valid_stack_address(void); /* kern/thread.h */ -extern volatile int panicwait; /* kern/debug.c */ -#define panic_quiesce (panicwait) +#define panic_quiesce (panic_active()) #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) @@ -548,6 +570,10 @@ extern int vuprintf(const char *, va_list); extern hrtime_t dtrace_abs_to_nano(uint64_t); __private_extern__ const char * strstr(const char *, const char *); +const void* bsearch(const void*, const void*, size_t, size_t, int (*compar)(const void *, const void *)); + +int dtrace_buffer_copyout(const void*, user_addr_t, vm_size_t); + #define DTRACE_NCLIENTS 32 diff --git a/bsd/sys/dtrace_impl.h b/bsd/sys/dtrace_impl.h index 9229998a3..125293dbf 100644 --- a/bsd/sys/dtrace_impl.h +++ b/bsd/sys/dtrace_impl.h @@ -1343,7 +1343,7 @@ typedef struct dtrace_toxrange { uintptr_t dtt_limit; /* limit of toxic range */ } dtrace_toxrange_t; -extern uint64_t dtrace_getarg(int, int); +extern uint64_t dtrace_getarg(int, int, dtrace_mstate_t*, dtrace_vstate_t*); extern int dtrace_getipl(void); extern uintptr_t dtrace_caller(int); extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t); @@ -1353,6 +1353,9 @@ extern void dtrace_copyinstr(user_addr_t, uintptr_t, size_t, volatile uint16_t * extern void dtrace_copyout(uintptr_t, user_addr_t, size_t, volatile uint16_t *); extern void dtrace_copyoutstr(uintptr_t, user_addr_t, size_t, volatile uint16_t *); extern void dtrace_getpcstack(pc_t *, int, int, uint32_t *); +extern uint64_t dtrace_load64(uintptr_t); +extern int dtrace_canload(uint64_t, size_t, dtrace_mstate_t*, dtrace_vstate_t*); + extern uint64_t dtrace_getreg(struct regs *, uint_t); extern int dtrace_getstackdepth(int); extern void dtrace_getupcstack(uint64_t *, int); diff --git a/bsd/sys/dtrace_ptss.h b/bsd/sys/dtrace_ptss.h index e7f1825dd..e7d8d9b0c 100644 --- a/bsd/sys/dtrace_ptss.h +++ b/bsd/sys/dtrace_ptss.h @@ -79,6 +79,9 @@ extern "C" { struct dtrace_ptss_page_entry { struct dtrace_ptss_page_entry* next; user_addr_t addr; +#if CONFIG_EMBEDDED + user_addr_t write_addr; +#endif }; struct dtrace_ptss_page { diff --git a/bsd/sys/event.h b/bsd/sys/event.h index 14b23450b..f10e15777 100644 --- a/bsd/sys/event.h +++ b/bsd/sys/event.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -60,6 +60,9 @@ #include #include +/* + * Filter types + */ #define EVFILT_READ (-1) #define EVFILT_WRITE (-2) #define EVFILT_AIO (-3) /* attached to aio requests */ @@ -78,8 +81,11 @@ #define EVFILT_MEMORYSTATUS (-14) /* Memorystatus events */ #endif /* PRIVATE */ #define EVFILT_EXCEPT (-15) /* Exception events */ +#ifdef PRIVATE +#define EVFILT_WORKLOOP (-17) /* Workloop events */ +#endif /* PRIVATE */ -#define EVFILT_SYSCOUNT 15 +#define EVFILT_SYSCOUNT 17 #define EVFILT_THREADMARKER EVFILT_SYSCOUNT /* Internal use only */ #pragma pack(4) @@ -125,7 +131,7 @@ struct kevent_internal_s { uint64_t ext[4]; /* filter-specific extensions */ }; -#endif +#endif /* KERNEL_PRIVATE */ #pragma pack() @@ -151,6 +157,12 @@ struct kevent_qos_s { int64_t data; /* filter-specific data */ uint64_t ext[4]; /* filter-specific extensions */ }; + +/* + * Type definition for names/ids of dynamically allocated kqueues. + */ +typedef uint64_t kqueue_id_t; + #endif /* PRIVATE */ #define EV_SET(kevp, a, b, c, d, e, f) do { \ @@ -177,9 +189,9 @@ struct kevent_qos_s { /* kevent system call flags */ -#define KEVENT_FLAG_NONE 0x000 /* no flag value */ -#define KEVENT_FLAG_IMMEDIATE 0x001 /* immediate timeout */ -#define KEVENT_FLAG_ERROR_EVENTS 0x002 /* output events only include change errors */ +#define KEVENT_FLAG_NONE 0x000 /* no flag value */ +#define KEVENT_FLAG_IMMEDIATE 0x001 /* immediate timeout */ +#define KEVENT_FLAG_ERROR_EVENTS 0x002 /* output events only include change errors */ #ifdef PRIVATE @@ -189,42 +201,58 @@ struct kevent_qos_s { * instead. */ -#define KEVENT_FLAG_STACK_EVENTS 0x004 /* output events treated as stack (grows down) */ -#define KEVENT_FLAG_STACK_DATA 0x008 /* output data allocated as stack (grows down) */ -#define KEVENT_FLAG_WORKQ 0x020 /* interact with the default workq kq */ -#define KEVENT_FLAG_WORKQ_MANAGER 0x200 /* current thread is the workq manager */ +#define KEVENT_FLAG_STACK_EVENTS 0x004 /* output events treated as stack (grows down) */ +#define KEVENT_FLAG_STACK_DATA 0x008 /* output data allocated as stack (grows down) */ +#define KEVENT_FLAG_UNBIND_CHECK_FLAGS 0x010 /* check the flags passed to kevent_qos_internal_unbind */ +#define KEVENT_FLAG_WORKQ 0x020 /* interact with the default workq kq */ +#define KEVENT_FLAG_WORKQ_MANAGER 0x200 /* current thread is the workq manager */ +#define KEVENT_FLAG_WORKLOOP 0x400 /* interact with the specified workloop kq */ +#define KEVENT_FLAG_SYNCHRONOUS_BIND 0x800 /* synchronous bind callback */ + +#define KEVENT_FLAG_WORKLOOP_SERVICER_ATTACH 0x8000 /* attach current thread to workloop */ +#define KEVENT_FLAG_WORKLOOP_SERVICER_DETACH 0x10000 /* unbind current thread from workloop */ +#define KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST 0x20000 /* kq lookup by id must exist */ +#define KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST 0x40000 /* kq lookup by id must not exist */ +#define KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD 0x80000 /* do not create workqueue threads for this worloop */ #ifdef XNU_KERNEL_PRIVATE -#define KEVENT_FLAG_LEGACY32 0x040 /* event data in legacy 32-bit format */ -#define KEVENT_FLAG_LEGACY64 0x080 /* event data in legacy 64-bit format */ -#define KEVENT_FLAG_KERNEL 0x100 /* caller is in-kernel */ -#define KEVENT_FLAG_USER (KEVENT_FLAG_IMMEDIATE | KEVENT_FLAG_ERROR_EVENTS | \ - KEVENT_FLAG_STACK_EVENTS | KEVENT_FLAG_STACK_DATA | \ - KEVENT_FLAG_WORKQ) - -/* - * Since some filter ops are not part of the standard sysfilt_ops, we - * use kn_filtid starting from EVFILT_SYSCOUNT to identify these cases. - * This is to let kn_fops() get the correct fops for all cases. +#define KEVENT_FLAG_LEGACY32 0x040 /* event data in legacy 32-bit format */ +#define KEVENT_FLAG_LEGACY64 0x080 /* event data in legacy 64-bit format */ +#define KEVENT_FLAG_KERNEL 0x1000 /* caller is in-kernel */ +#define KEVENT_FLAG_DYNAMIC_KQUEUE 0x2000 /* kqueue is dynamically allocated */ +#define KEVENT_FLAG_WORKLOOP_CANCELED 0x4000 /* workloop bind was cancelled */ + +#define KEVENT_FLAG_USER (KEVENT_FLAG_IMMEDIATE | KEVENT_FLAG_ERROR_EVENTS | \ + KEVENT_FLAG_STACK_EVENTS | KEVENT_FLAG_STACK_DATA | \ + KEVENT_FLAG_WORKQ | KEVENT_FLAG_WORKLOOP | \ + KEVENT_FLAG_WORKLOOP_SERVICER_ATTACH | KEVENT_FLAG_WORKLOOP_SERVICER_DETACH | \ + KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST | KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST | \ + KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD) + +/* + * Since some filter ops are not part of the standard sysfilt_ops, we use + * kn_filtid starting from EVFILT_SYSCOUNT to identify these cases. This is to + * let kn_fops() get the correct fops for all cases. */ -#define EVFILTID_KQREAD (EVFILT_SYSCOUNT) -#define EVFILTID_PIPE_R (EVFILT_SYSCOUNT + 1) -#define EVFILTID_PIPE_W (EVFILT_SYSCOUNT + 2) -#define EVFILTID_PTSD (EVFILT_SYSCOUNT + 3) -#define EVFILTID_SOREAD (EVFILT_SYSCOUNT + 4) -#define EVFILTID_SOWRITE (EVFILT_SYSCOUNT + 5) -#define EVFILTID_SCK (EVFILT_SYSCOUNT + 6) -#define EVFILTID_SOEXCEPT (EVFILT_SYSCOUNT + 7) -#define EVFILTID_SPEC (EVFILT_SYSCOUNT + 8) -#define EVFILTID_BPFREAD (EVFILT_SYSCOUNT + 9) -#define EVFILTID_NECP_FD (EVFILT_SYSCOUNT + 10) -#define EVFILTID_FSEVENT (EVFILT_SYSCOUNT + 13) -#define EVFILTID_VN (EVFILT_SYSCOUNT + 14) - -#define EVFILTID_MAX (EVFILT_SYSCOUNT + 15) - -#endif /* XNU_KERNEL_PRIVATE */ +#define EVFILTID_KQREAD (EVFILT_SYSCOUNT) +#define EVFILTID_PIPE_R (EVFILT_SYSCOUNT + 1) +#define EVFILTID_PIPE_W (EVFILT_SYSCOUNT + 2) +#define EVFILTID_PTSD (EVFILT_SYSCOUNT + 3) +#define EVFILTID_SOREAD (EVFILT_SYSCOUNT + 4) +#define EVFILTID_SOWRITE (EVFILT_SYSCOUNT + 5) +#define EVFILTID_SCK (EVFILT_SYSCOUNT + 6) +#define EVFILTID_SOEXCEPT (EVFILT_SYSCOUNT + 7) +#define EVFILTID_SPEC (EVFILT_SYSCOUNT + 8) +#define EVFILTID_BPFREAD (EVFILT_SYSCOUNT + 9) +#define EVFILTID_NECP_FD (EVFILT_SYSCOUNT + 10) +#define EVFILTID_FSEVENT (EVFILT_SYSCOUNT + 13) +#define EVFILTID_VN (EVFILT_SYSCOUNT + 14) +#define EVFILTID_TTY (EVFILT_SYSCOUNT + 16) + +#define EVFILTID_MAX (EVFILT_SYSCOUNT + 17) + +#endif /* defined(XNU_KERNEL_PRIVATE) */ #define EV_SET_QOS 0 @@ -315,6 +343,82 @@ struct kevent_qos_s { #define NOTE_FFCTRLMASK 0xc0000000 /* mask for operations */ #define NOTE_FFLAGSMASK 0x00ffffff +#ifdef PRIVATE +/* + * data/hint fflags for EVFILT_WORKLOOP, shared with userspace + * + * The ident for thread requests should be the dynamic ID of the workloop + * The ident for each sync waiter must be unique to that waiter [for this workloop] + * + * + * Commands: + * + * @const NOTE_WL_THREAD_REQUEST [in/out] + * The kevent represents asynchronous userspace work and its associated QoS. + * There can only be a single knote with this flag set per workloop. + * + * @const NOTE_WL_SYNC_WAIT [in/out] + * This bit is set when the caller is waiting to become the owner of a workloop. + * If the NOTE_WL_SYNC_WAKE bit is already set then the caller is not blocked, + * else it blocks until it is set. + * + * The QoS field of the knote is used to push on other owners or servicers. + * + * @const NOTE_WL_SYNC_WAKE [in/out] + * Marks the waiter knote as being eligible to become an owner + * This bit can only be set once, trying it again will fail with EALREADY. + * + * + * Flags/Modifiers: + * + * @const NOTE_WL_UPDATE_QOS [in] (only NOTE_WL_THREAD_REQUEST) + * For successful updates (EV_ADD only), learn the new userspace async QoS from + * the kevent qos field. + * + * @const NOTE_WL_END_OWNERSHIP [in] + * If the update is successful (including deletions) or returns ESTALE, and + * the caller thread or the "suspended" thread is currently owning the workloop, + * then ownership is forgotten. + * + * @const NOTE_WL_DISCOVER_OWNER [in] + * If the update is successful (including deletions), learn the owner identity + * from the loaded value during debounce. This requires an address to have been + * filled in the EV_EXTIDX_WL_ADDR ext field, but doesn't require a mask to have + * been set in the EV_EXTIDX_WL_MASK. + * + * @const NOTE_WL_IGNORE_ESTALE [in] + * If the operation would fail with ESTALE, mask the error and pretend the + * update was successful. However the operation itself didn't happen, meaning + * that: + * - attaching a new knote will not happen + * - dropping an existing knote will not happen + * - NOTE_WL_UPDATE_QOS or NOTE_WL_DISCOVER_OWNER will have no effect + * + * This modifier doesn't affect NOTE_WL_END_OWNERSHIP. + */ +#define NOTE_WL_THREAD_REQUEST 0x00000001 +#define NOTE_WL_SYNC_WAIT 0x00000004 +#define NOTE_WL_SYNC_WAKE 0x00000008 +#define NOTE_WL_COMMANDS_MASK 0x0000000f /* Mask of all the [in] commands above */ + +#define NOTE_WL_UPDATE_QOS 0x00000010 +#define NOTE_WL_END_OWNERSHIP 0x00000020 +#define NOTE_WL_UPDATE_OWNER 0 /* ... compatibility define ... */ +#define NOTE_WL_DISCOVER_OWNER 0x00000080 +#define NOTE_WL_IGNORE_ESTALE 0x00000100 +#define NOTE_WL_UPDATES_MASK 0x000001f0 /* Mask of all the [in] updates above */ + +/* + * EVFILT_WORKLOOP ext[] array indexes/meanings. + */ +#define EV_EXTIDX_WL_LANE 0 /* lane identifier [in: sync waiter] + [out: thread request] */ +#define EV_EXTIDX_WL_ADDR 1 /* debounce address [in: NULL==no debounce] */ +#define EV_EXTIDX_WL_MASK 2 /* debounce mask [in] */ +#define EV_EXTIDX_WL_VALUE 3 /* debounce value [in: not current->ESTALE] + [out: new/debounce value] */ +#endif /* PRIVATE */ + /* * data/hint fflags for EVFILT_{READ|WRITE}, shared with userspace * @@ -415,6 +519,7 @@ enum { #define NOTE_MEMORYSTATUS_LOW_SWAP 0x00000008 /* system is in a low-swap state */ #define NOTE_MEMORYSTATUS_PROC_LIMIT_WARN 0x00000010 /* process memory limit has hit a warning state */ #define NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL 0x00000020 /* process memory limit has hit a critical state - soft limit */ +#define NOTE_MEMORYSTATUS_MSL_STATUS 0xf0000000 /* bits used to request change to process MSL status */ #ifdef KERNEL_PRIVATE /* @@ -429,7 +534,8 @@ enum { * Use this mask to protect the kernel private flags. */ #define EVFILT_MEMORYSTATUS_ALL_MASK \ - (NOTE_MEMORYSTATUS_PRESSURE_NORMAL | NOTE_MEMORYSTATUS_PRESSURE_WARN | NOTE_MEMORYSTATUS_PRESSURE_CRITICAL | NOTE_MEMORYSTATUS_LOW_SWAP | NOTE_MEMORYSTATUS_PROC_LIMIT_WARN | NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) + (NOTE_MEMORYSTATUS_PRESSURE_NORMAL | NOTE_MEMORYSTATUS_PRESSURE_WARN | NOTE_MEMORYSTATUS_PRESSURE_CRITICAL | NOTE_MEMORYSTATUS_LOW_SWAP | \ + NOTE_MEMORYSTATUS_PROC_LIMIT_WARN | NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL | NOTE_MEMORYSTATUS_MSL_STATUS) #endif /* KERNEL_PRIVATE */ @@ -453,11 +559,21 @@ typedef enum vm_pressure_level { #define NOTE_USECONDS 0x00000002 /* data is microseconds */ #define NOTE_NSECONDS 0x00000004 /* data is nanoseconds */ #define NOTE_ABSOLUTE 0x00000008 /* absolute timeout */ - /* ... implicit EV_ONESHOT */ + /* ... implicit EV_ONESHOT, timeout uses the gettimeofday epoch */ #define NOTE_LEEWAY 0x00000010 /* ext[1] holds leeway for power aware timers */ #define NOTE_CRITICAL 0x00000020 /* system does minimal timer coalescing */ #define NOTE_BACKGROUND 0x00000040 /* system does maximum timer coalescing */ -#define NOTE_MACH_CONTINUOUS_TIME 0x00000080 /* use continuous time base */ +#define NOTE_MACH_CONTINUOUS_TIME 0x00000080 + /* + * NOTE_MACH_CONTINUOUS_TIME: + * with NOTE_ABSOLUTE: causes the timer to continue to tick across sleep, + * still uses gettimeofday epoch + * with NOTE_MACHTIME and NOTE_ABSOLUTE: uses mach continuous time epoch + * without NOTE_ABSOLUTE (interval timer mode): continues to tick across sleep + */ +#define NOTE_MACHTIME 0x00000100 /* data is mach absolute time units */ + /* timeout uses the mach absolute time epoch */ + #ifdef PRIVATE /* * data/hint fflags for EVFILT_SOCK, shared with userspace. @@ -530,6 +646,8 @@ typedef enum vm_pressure_level { #define NOTE_CHILD 0x00000004 /* am a child process */ +#ifdef PRIVATE +#endif /* PRIVATE */ #ifndef KERNEL /* Temporay solution for BootX to use inode.h till kqueue moves to vfs layer */ @@ -543,6 +661,8 @@ SLIST_HEAD(klist, knote); #ifdef XNU_KERNEL_PRIVATE #include #include +#include /* FREAD, FWRITE */ +#include /* panic */ #ifdef MALLOC_DECLARE MALLOC_DECLARE(M_KQUEUE); @@ -586,12 +706,16 @@ struct knote { struct proc *p_proc; /* proc pointer */ struct ipc_mqueue *p_mqueue; /* pset pointer */ } kn_ptr; - uint64_t kn_req_index:4, /* requested qos index */ - kn_qos_index:4, /* in-use qos index */ - kn_qos_override:4, /* qos override index */ - kn_reserved:4, /* reserved bits */ - kn_filtid:8, /* filter id to index filter ops */ - kn_kq_packed:KNOTE_KQ_BITSIZE; /* packed pointer for kq */ + uint64_t kn_req_index:3, /* requested qos index */ + kn_qos_index:3, /* in-use qos index */ + kn_qos_override:3, /* qos override index */ + kn_qos_sync_override:3, /* qos sync override index */ + kn_vnode_kqok:1, + kn_vnode_use_ofst:1, + kn_qos_override_is_sync:1, /* qos override index is a sync override */ + kn_reserved:1, /* reserved bits */ + kn_filtid:8, /* filter id to index filter ops */ + kn_kq_packed:KNOTE_KQ_BITSIZE; /* packed pointer for kq */ union { void *kn_hook; @@ -634,6 +758,26 @@ static inline void knote_set_kq(struct knote *kn, void *kq) } } +static inline int knote_get_seltype(struct knote *kn) +{ + switch (kn->kn_filter) { + case EVFILT_READ: + return FREAD; + case EVFILT_WRITE: + return FWRITE; + default: + panic("%s(%p): invalid filter %d\n", + __func__, kn, kn->kn_filter); + return 0; + } +} + +static inline void knote_set_error(struct knote *kn, int error) +{ + kn->kn_flags |= EV_ERROR; + kn->kn_data = error; +} + struct filt_process_s { int fp_fd; unsigned int fp_flags; @@ -660,8 +804,20 @@ typedef struct filt_process_s *filt_process_data_t; * * Otherwise the knote is hashed by the ident and has no auto-close behavior. * + * f_adjusts_qos - + * identifies if the filter can adjust its QoS during its lifetime. + * + * Currently, EVFILT_MAACHPORT is the only filter using this facility. + * + * f_needs_boost - + * [OPTIONAL] used by filters to communicate they need to hold a boost + * while holding a usecount on this knote. This is called with the kqlock + * held. + * + * This is only used by EVFILT_WORKLOOP currently. + * * f_attach - - * called to attach the knote to the underlying object that will be delivering events + * called to attach the knote to the underlying object that will be delivering events * through it when EV_ADD is supplied and no existing matching event is found * * provided a knote that is pre-attached to the fd or hashed (see above) but is @@ -679,9 +835,21 @@ typedef struct filt_process_s *filt_process_data_t; * The return value indicates if the knote should already be considered "activated" at * the time of attach (one or more of the interest events has already occured). * + * f_post_attach - + * [OPTIONAL] called after a successful attach, with the kqueue lock held, + * returns lock held, may drop and re-acquire + * + * If this function is non-null, then it indicates that the filter wants + * to perform an action after a successful ATTACH of a knote. + * + * Currently, EVFILT_WORKLOOP is the only filter using this facility. + * + * The return value indicates an error to report to userland. + * + * * f_detach - * called to disassociate the knote from the underlying object delivering events - * the filter should not attempt to deliver events through this knote after this + * the filter should not attempt to deliver events through this knote after this * operation returns control to the kq system. * * f_event - @@ -695,6 +863,22 @@ typedef struct filt_process_s *filt_process_data_t; * The return value indicates if the knote should already be considered "activated" at * the time of attach (one or more of the interest events has already occured). * + * f_drop_and_unlock - + * [OPTIONAL] called with the kqueue locked, and has to unlock + * + * If this function is non-null, then it indicates that the filter + * wants to handle EV_DELETE events. This is necessary if a particular + * filter needs to synchronize knote deletion with its own filter lock. + * Currently, EVFILT_WORKLOOP is the only filter using this facility. + * + * The return value indicates an error during the knote drop, i.e., the + * knote still exists and user space should re-drive the EV_DELETE. + * + * If the return value is ERESTART, kevent_register() is called from + * scratch again (useful to wait for usecounts to drop and then + * reevaluate the relevance of that drop) + * + * * f_process - * called when attempting to deliver triggered events to user-space. * @@ -733,17 +917,35 @@ typedef struct filt_process_s *filt_process_data_t; * explicit indication of just delivering a current event vs delivering * an event with more events still pending. * + * f_touch - + * called to update the knote with new state from the user during EVFILT_ADD/ENABLE/DISABLE + * on an already-attached knote. + * + * f_touch should copy relevant new data from the kevent into the knote. + * (if KN_UDATA_SPECIFIC is not set, you may need to update the udata too) + * + * operator must lock against concurrent f_event and f_process operations. + * + * A return value of 1 indicates that the knote should now be considered 'activated'. + * + * f_touch can set EV_ERROR with specific error in the data field to return an error to the client. + * You should return 1 to indicate that the kevent needs to be activated and processed. + * * f_peek - * For knotes marked KN_STAYACTIVE, indicate if the knote is truly active at * the moment (not used for event delivery, but for status checks). */ struct filterops { - int f_isfd; /* true if ident == filedescriptor */ - int (*f_attach)(struct knote *kn); - void (*f_detach)(struct knote *kn); - int (*f_event)(struct knote *kn, long hint); + bool f_isfd; /* true if ident == filedescriptor */ + bool f_adjusts_qos; /* true if the filter can override the knote */ + bool (*f_needs_boost)(struct kevent_internal_s *kev); + int (*f_attach)(struct knote *kn, struct kevent_internal_s *kev); + int (*f_post_attach)(struct knote *kn, struct kevent_internal_s *kev); + void (*f_detach)(struct knote *kn); + int (*f_event)(struct knote *kn, long hint); int (*f_touch)(struct knote *kn, struct kevent_internal_s *kev); + int (*f_drop_and_unlock)(struct knote *kn, struct kevent_internal_s *kev); int (*f_process)(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); unsigned (*f_peek)(struct knote *kn); }; @@ -759,7 +961,6 @@ extern void klist_init(struct klist *list); #define KNOTE_ATTACH(list, kn) knote_attach(list, kn) #define KNOTE_DETACH(list, kn) knote_detach(list, kn) - extern void knote(struct klist *list, long hint); extern int knote_attach(struct klist *list, struct knote *kn); extern int knote_detach(struct klist *list, struct knote *kn); @@ -769,8 +970,19 @@ extern int knote_unlink_waitq(struct knote *kn, struct waitq *wq); extern void knote_fdclose(struct proc *p, int fd, int force); extern void knote_markstayactive(struct knote *kn); extern void knote_clearstayactive(struct knote *kn); -extern void knote_adjust_qos(struct knote *kn, int qos, int override); -extern struct filterops *knote_fops(struct knote *kn); +extern void knote_adjust_qos(struct knote *kn, int qos, int override, kq_index_t sync_override_index); +extern void knote_adjust_sync_qos(struct knote *kn, kq_index_t sync_qos, boolean_t lock_kq); +extern const struct filterops *knote_fops(struct knote *kn); +extern void knote_set_error(struct knote *kn, int error); + +int kevent_exit_on_workloop_ownership_leak(thread_t thread); +int kevent_proc_copy_uptrs(void *proc, uint64_t *buf, int bufsize); +int kevent_copyout_proc_dynkqids(void *proc, user_addr_t ubuf, + uint32_t ubufsize, int32_t *nkqueues_out); +int kevent_copyout_dynkqinfo(void *proc, kqueue_id_t kq_id, user_addr_t ubuf, + uint32_t ubufsize, int32_t *size_out); +int kevent_copyout_dynkqextinfo(void *proc, kqueue_id_t kq_id, user_addr_t ubuf, + uint32_t ubufsize, int32_t *nknotes_out); #elif defined(KERNEL_PRIVATE) /* !XNU_KERNEL_PRIVATE: kexts still need a klist structure definition */ @@ -792,9 +1004,15 @@ extern int kevent_qos_internal(struct proc *p, int fd, unsigned int flags, int32_t *retval); extern int kevent_qos_internal_bind(struct proc *p, - int fd, thread_t thread, unsigned int flags); + int qos, thread_t thread, unsigned int flags); extern int kevent_qos_internal_unbind(struct proc *p, - int fd, thread_t thread, unsigned int flags); + int qos, thread_t thread, unsigned int flags); + +extern int kevent_id_internal(struct proc *p, kqueue_id_t *id, + user_addr_t changelist, int nchanges, + user_addr_t eventlist, int nevents, + user_addr_t data_out, user_size_t *data_available, + unsigned int flags, int32_t *retval); #endif /* PRIVATE */ #endif /* KERNEL_PRIVATE */ @@ -823,6 +1041,12 @@ int kevent_qos(int kq, struct kevent_qos_s *eventlist, int nevents, void *data_out, size_t *data_available, unsigned int flags); + +int kevent_id(kqueue_id_t id, + const struct kevent_qos_s *changelist, int nchanges, + struct kevent_qos_s *eventlist, int nevents, + void *data_out, size_t *data_available, + unsigned int flags); #endif /* PRIVATE */ __END_DECLS @@ -830,5 +1054,13 @@ __END_DECLS #endif /* KERNEL */ +#ifdef PRIVATE + +/* Flags for pending events notified by kernel via return-to-kernel ast */ +#define R2K_WORKLOOP_PENDING_EVENTS 0x1 +#define R2K_WORKQ_PENDING_EVENTS 0x2 + +#endif /* PRIVATE */ + #endif /* !_SYS_EVENT_H_ */ diff --git a/bsd/sys/eventhandler.h b/bsd/sys/eventhandler.h new file mode 100644 index 000000000..794aa9acd --- /dev/null +++ b/bsd/sys/eventhandler.h @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/*- + * Copyright (c) 1999 Michael Smith + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_EVENTHANDLER_H_ +#define _SYS_EVENTHANDLER_H_ + +#include +#include +#include +#include +#include + +extern int evh_debug; +extern lck_grp_t *el_lock_grp; +extern lck_attr_t *el_lock_attr; +extern struct eventhandler_entry_arg eventhandler_entry_dummy_arg; + +struct eventhandler_lists_ctxt { + TAILQ_HEAD(, eventhandler_list) eventhandler_lists; + int eventhandler_lists_initted; + decl_lck_mtx_data(, eventhandler_mutex); +}; + +struct eventhandler_entry_arg { + union { + /* Generic cookie object reference */ + void *ee_voidptr; + /* Skywalk ids */ + struct { + pid_t ee_fe_pid; + uuid_t ee_fe_uuid; + uuid_t ee_nx_uuid; + } sk_ids; + /* Generic UUID */ + uuid_t ee_uuid; + } ee_arg; +}; + +#define ee_voidptr ee_arg.ee_voidptr +#define ee_fe_pid ee_arg.sk_ids.ee_fe_pid +#define ee_fe_uuid ee_arg.sk_ids.ee_fe_uuid +#define ee_nx_uuid ee_arg.sk_ids.ee_nx_uuid +#define ee_uuid ee_arg.ee_uuid + +struct eventhandler_entry { + TAILQ_ENTRY(eventhandler_entry) ee_link; + int ee_priority; +#define EHE_DEAD_PRIORITY (-1) + struct eventhandler_entry_arg ee_arg; +}; + +struct eventhandler_list { + char *el_name; + int el_flags; +#define EHL_INITTED (1<<0) + u_int el_runcount; + decl_lck_mtx_data(, el_lock); + TAILQ_ENTRY(eventhandler_list) el_link; + TAILQ_HEAD(,eventhandler_entry) el_entries; +}; + +typedef struct eventhandler_entry *eventhandler_tag; + +#define EHL_LOCK_INIT(p) lck_mtx_init(&(p)->el_lock, el_lock_grp, el_lock_attr) +#define EHL_LOCK(p) lck_mtx_lock(&(p)->el_lock) +#define EHL_UNLOCK(p) lck_mtx_unlock(&(p)->el_lock) +#define EHL_LOCK_ASSERT(p, x) LCK_MTX_ASSERT(&(p)->el_lock, x) +#define EHL_LOCK_DESTROY(p) lck_mtx_destroy(&(p)->el_lock, el_lock_grp) + +#define evhlog(x) do { if (evh_debug >= 1) log x; } while (0) + +/* + * Macro to invoke the handlers for a given event. + */ +#define _EVENTHANDLER_INVOKE(name, list, ...) do { \ + struct eventhandler_entry *_ep; \ + struct eventhandler_entry_ ## name *_t; \ + \ + VERIFY((list)->el_flags & EHL_INITTED); \ + EHL_LOCK_ASSERT((list), LCK_MTX_ASSERT_OWNED); \ + (list)->el_runcount++; \ + VERIFY((list)->el_runcount > 0); \ + evhlog((LOG_DEBUG, "eventhandler_invoke(\"" __STRING(name) "\")")); \ + TAILQ_FOREACH(_ep, &((list)->el_entries), ee_link) { \ + if (_ep->ee_priority != EHE_DEAD_PRIORITY) { \ + EHL_UNLOCK((list)); \ + _t = (struct eventhandler_entry_ ## name *)_ep; \ + evhlog((LOG_DEBUG, "eventhandler_invoke: executing %p", \ + VM_KERNEL_UNSLIDE((void *)_t->eh_func))); \ + _t->eh_func(_ep->ee_arg , ## __VA_ARGS__); \ + EHL_LOCK((list)); \ + } \ + } \ + KASSERT((list)->el_runcount > 0, \ + ("eventhandler_invoke: runcount underflow")); \ + (list)->el_runcount--; \ + if ((list)->el_runcount == 0) \ + eventhandler_prune_list(list); \ + EHL_UNLOCK((list)); \ +} while (0) + +/* + * Slow handlers are entirely dynamic; lists are created + * when entries are added to them, and thus have no concept of "owner", + * + * Slow handlers need to be declared, but do not need to be defined. The + * declaration must be in scope wherever the handler is to be invoked. + */ +#define EVENTHANDLER_DECLARE(name, type) \ +struct eventhandler_entry_ ## name \ +{ \ + struct eventhandler_entry ee; \ + type eh_func; \ +}; \ +struct __hack + +/* + * XXX EVENTHANDLER_DEFINE by itself doesn't do much on XNU + * All it does is that it declares the static eventhandler_tag + * and defines an init routine that still needs to called to put the + * event and callback on the list. + */ +#define EVENTHANDLER_DEFINE(evthdlr_ref, name, func, arg, priority) \ + static eventhandler_tag name ## _tag; \ + static void name ## _evh_init(void *ctx) \ + { \ + name ## _tag = EVENTHANDLER_REGISTER(evthdlr_ref, name, func, ctx, \ + priority); \ + } \ + SYSINIT(name ## _evh_init, SI_SUB_CONFIGURE, SI_ORDER_ANY, \ + name ## _evh_init, arg); \ + struct __hack + +#define EVENTHANDLER_INVOKE(evthdlr_ref, name, ...) \ +do { \ + struct eventhandler_list *_el; \ + \ + if ((_el = eventhandler_find_list(evthdlr_ref, #name)) != NULL) \ + _EVENTHANDLER_INVOKE(name, _el , ## __VA_ARGS__); \ +} while (0) + +#define EVENTHANDLER_REGISTER(evthdlr_ref, name, func, arg, priority) \ + eventhandler_register(evthdlr_ref, NULL, #name, func, arg, priority) + +#define EVENTHANDLER_DEREGISTER(evthdlr_ref, name, tag) \ +do { \ + struct eventhandler_list *_el; \ + \ + if ((_el = eventhandler_find_list(evthdlr_ref, #name)) != NULL) \ + eventhandler_deregister(_el, tag); \ +} while(0) + +void eventhandler_init(void); +eventhandler_tag eventhandler_register(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, + struct eventhandler_list *list, const char *name, void *func, struct eventhandler_entry_arg arg, int priority); +void eventhandler_deregister(struct eventhandler_list *list, + eventhandler_tag tag); +struct eventhandler_list *eventhandler_find_list( + struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, const char *name); +void eventhandler_prune_list(struct eventhandler_list *list); +void eventhandler_lists_ctxt_init(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt); +void eventhandler_lists_ctxt_destroy(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt); + +/* Generic priority levels */ +#define EVENTHANDLER_PRI_FIRST 0 +#define EVENTHANDLER_PRI_ANY 10000 +#define EVENTHANDLER_PRI_LAST 20000 + +#endif /* _SYS_EVENTHANDLER_H_ */ diff --git a/bsd/sys/eventvar.h b/bsd/sys/eventvar.h index 8d47aad64..82323625f 100644 --- a/bsd/sys/eventvar.h +++ b/bsd/sys/eventvar.h @@ -63,9 +63,41 @@ #if defined(XNU_KERNEL_PRIVATE) +typedef int (*kevent_callback_t)(struct kqueue *, struct kevent_internal_s *, void *); +typedef void (*kqueue_continue_t)(struct kqueue *, void *, int); + +#include #include +#include #include +/* + * Lock ordering: + * + * The kqueue locking order can follow a few different patterns: + * + * Standard file-based kqueues (from above): + * proc fd lock -> kq lock -> kq-waitq-set lock -> thread lock + * + * WorkQ/WorkLoop kqueues (from above): + * proc fd lock -> kq lock -> kq-request lock -> pthread kext locks -> thread lock + * + * Whenever kqueues interact with source locks, it drops all of its own + * locks in exchange for a use-reference on the knote used to synchronize + * with the source code. When those sources post events from below, they + * have the following lock hierarchy. + * + * Standard file-based kqueues (from below): + * XXX lock -> kq lock -> kq-waitq-set lock -> thread lock + * Standard file-based kqueues with non-kq-aware sources (from below): + * XXX lock -> kq-waitq-set lock -> thread lock + * + * WorkQ/WorkLoop kqueues (from below): + * XXX lock -> kq lock -> kq-request lock -> pthread kext locks -> thread lock + * WorkQ/WorkLoop kqueues with non-kq-aware sources (from below): + * XXX -> kq-waitq-set lock -> kq-request lock -> pthread kext locks -> thread lock + */ + #define KQEXTENT 256 /* linear growth by this amount */ /* @@ -85,17 +117,19 @@ struct kqueue { struct kqtailq kq_queue[1]; /* variable array of kqtailq structs */ }; -#define KQ_SEL 0x001 /* select was recorded for kq */ -#define KQ_SLEEP 0x002 /* thread is waiting for events */ -#define KQ_PROCWAIT 0x004 /* thread waiting for processing */ -#define KQ_KEV32 0x008 /* kq is used with 32-bit events */ -#define KQ_KEV64 0x010 /* kq is used with 64-bit events */ -#define KQ_KEV_QOS 0x020 /* kq events carry QoS info */ -#define KQ_WORKQ 0x040 /* KQ is bould to process workq */ -#define KQ_PROCESSING 0x080 /* KQ is being processed */ -#define KQ_DRAIN 0x100 /* kq is draining */ -#define KQ_WAKEUP 0x200 /* kq awakened while processing */ - +#define KQ_SEL 0x001 /* select was recorded for kq */ +#define KQ_SLEEP 0x002 /* thread is waiting for events */ +#define KQ_PROCWAIT 0x004 /* thread waiting for processing */ +#define KQ_KEV32 0x008 /* kq is used with 32-bit events */ +#define KQ_KEV64 0x010 /* kq is used with 64-bit events */ +#define KQ_KEV_QOS 0x020 /* kq events carry QoS info */ +#define KQ_WORKQ 0x040 /* KQ is bound to process workq */ +#define KQ_WORKLOOP 0x080 /* KQ is part of a workloop */ +#define KQ_PROCESSING 0x100 /* KQ is being processed */ +#define KQ_DRAIN 0x200 /* kq is draining */ +#define KQ_WAKEUP 0x400 /* kq awakened while processing */ +#define KQ_DYNAMIC 0x800 /* kqueue is dynamically managed */ +#define KQ_NO_WQ_THREAD 0x1000 /* kq will not have workqueue threads dynamically created */ /* * kqfile - definition of a typical kqueue opened as a file descriptor * via the kqueue() system call. @@ -119,6 +153,53 @@ struct kqfile { #define QOS_INDEX_KQFILE 0 /* number of qos levels in a file kq */ +struct kqr_bound { + struct kqtailq kqrb_suppressed; /* Per-QoS suppression queues */ + thread_t kqrb_thread; /* thread to satisfy request */ +}; + +/* + * kqrequest - per-QoS thread request status + */ +struct kqrequest { +#if 0 + union { + struct kqr_bound kqru_bound; /* used when thread is bound */ + struct workq_threadreq_s kqru_req; /* used when request oustanding */ + } kqr_u; +#define kqr_suppressed kqr_u.kqru_bound.kqrb_suppressed +#define kqr_thread kqr_u.kqru_bound.kqrb_thread +#define kqr_req kqr_u.kqru_req +#else + struct kqr_bound kqr_bound; /* used when thread is bound */ + struct workq_threadreq_s kqr_req; /* used when request oustanding */ +#define kqr_suppressed kqr_bound.kqrb_suppressed +#define kqr_thread kqr_bound.kqrb_thread +#endif + uint8_t kqr_state; /* KQ/workq interaction state */ + uint8_t kqr_wakeup_indexes; /* QoS/override levels that woke */ + uint16_t kqr_dsync_waiters:13, /* number of dispatch sync waiters */ + kqr_dsync_owner_qos:3; /* Qos override on dispatch sync owner */ + uint16_t kqr_sync_suppress_count; /* number of suppressed sync ipc knotes */ + kq_index_t kqr_stayactive_qos:3, /* max QoS of statyactive knotes */ + kqr_owner_override_is_sync:1, /* sync owner has sync ipc override */ + kqr_override_index:3, /* highest wakeup override index */ + kqr_has_sync_override:1; /* Qos/override at UI is sync ipc override */ + + /* set under both the kqlock and the filt_wllock */ + kq_index_t :0; /* prevent bitfields coalescing */ + kq_index_t kqr_qos_index:4, /* QoS for the thread request */ + kqr_dsync_waiters_qos:4; /* override from dispatch sync waiters */ +}; + + +#define KQR_PROCESSING 0x01 /* requested thread is running the q */ +#define KQR_THREQUESTED 0x02 /* thread has been requested from workq */ +#define KQR_WAKEUP 0x04 /* wakeup called during processing */ +#define KQR_BOUND 0x08 /* servicing thread is bound */ +#define KQR_THOVERCOMMIT 0x20 /* overcommit needed for thread requests */ +#define KQR_DRAIN 0x40 /* cancel initiated - drain fulfill */ +#define KQR_R2K_NOTIF_ARMED 0x80 /* ast notifications armed */ /* * WorkQ kqueues need to request threads to service the triggered * knotes in the queue. These threads are brought up on a @@ -136,17 +217,6 @@ struct kqfile { #define KQWQ_NQOS (KQWQ_QOS_MANAGER + 1) #endif - -/* - * kqrequest - per-QoS thread request status - */ -struct kqrequest { - struct kqtailq kqr_suppressed; /* Per-QoS suppression queues */ - thread_t kqr_thread; /* thread to satisfy request */ - uint8_t kqr_state; /* KQ/workq interaction state */ - uint8_t kqr_override_delta; /* current override delta */ -}; - /* * Workq thread start out a particular effective-requested-QoS, but * additional events processed by the filters may represent @@ -202,21 +272,85 @@ struct kqworkq { #define kqwq_p kqwq_kqueue.kq_p #define kqwq_queue kqwq_kqueue.kq_queue -#define kqwq_req_lock(kqwq) (lck_spin_lock(&kqwq->kqwq_reqlock)) -#define kqwq_req_unlock(kqwq) (lck_spin_unlock(&kqwq->kqwq_reqlock)) -#define kqwq_req_held(kqwq) (lck_spin_held(&kqwq->kqwq_reqlock)) +#define kqwq_req_lock(kqwq) lck_spin_lock(&kqwq->kqwq_reqlock) +#define kqwq_req_unlock(kqwq) lck_spin_unlock(&kqwq->kqwq_reqlock) +#define kqwq_req_held(kqwq) LCK_SPIN_ASSERT(&kqwq->kqwq_reqlock, LCK_ASSERT_OWNED) + +#define KQWQ_THMANAGER 0x10 /* expect manager thread to run the queue */ + +/* + * WorkLoop kqueues need to request a thread to service the triggered + * knotes in the queue. The thread is brought up on a + * effective-requested-QoS basis. Knotes are segregated based on + * that value. Once a request is made, it cannot be undone. If + * events with higher QoS arrive after, they are stored in their + * own queues and an override applied to the original request based + * on the delta between the two QoS values. + */ + +/* + * "Stay-active" knotes are held in a separate bucket that indicates + * special handling required. They are kept separate because the + * wakeups issued to them don't have context to tell us where to go + * to find and process them. All processing of them happens at the + * highest QoS. Unlike WorkQ kqueues, there is no special singular + * "manager thread" for a process. We simply request a servicing + * thread at the higest known QoS when these are woken (or override + * an existing request to that). + */ +#define KQWL_BUCKET_STAYACTIVE (THREAD_QOS_LAST) + +#if !defined(KQWL_NBUCKETS) +#define KQWL_NBUCKETS (KQWL_BUCKET_STAYACTIVE + 1) +#endif + +/* + * kqworkloop - definition of a private kqueue used to coordinate event + * handling for pthread workloops. + * + * Workloops vary from workqs in that only a single thread is ever + * requested to service a workloop at a time. But unlike workqs, + * workloops may be "owned" by user-space threads that are + * synchronously draining an event off the workloop. In those cases, + * any overrides have to be applied to the owner until it relinqueshes + * ownership. + * + * NOTE: "lane" support is TBD. + */ +struct kqworkloop { + struct kqueue kqwl_kqueue; /* queue of events */ + struct kqtailq kqwl_queuecont[KQWL_NBUCKETS-1]; /* continue array of queues */ + struct kqrequest kqwl_request; /* thread request state */ + lck_spin_t kqwl_reqlock; /* kqueue request lock */ + lck_mtx_t kqwl_statelock; /* state/debounce lock */ + thread_t kqwl_owner; /* current [sync] owner thread */ + uint32_t kqwl_retains; /* retain references */ + kqueue_id_t kqwl_dynamicid; /* dynamic identity */ + SLIST_ENTRY(kqworkloop) kqwl_hashlink; /* linkage for search list */ +}; -#define KQWQ_PROCESSING 0x01 /* running the kq in workq mode */ -#define KQWQ_THREQUESTED 0x02 /* thread requested from workq */ -#define KQWQ_THMANAGER 0x04 /* expect manager thread to run the queue */ -#define KQWQ_HOOKCALLED 0x10 /* hook called during processing */ -#define KQWQ_WAKEUP 0x20 /* wakeup called during processing */ +SLIST_HEAD(kqlist, kqworkloop); + +#define kqwl_wqs kqwl_kqueue.kq_wqs +#define kqwl_lock kqwl_kqueue.kq_lock +#define kqwl_state kqwl_kqueue.kq_state +#define kqwl_level kqwl_kqueue.kq_level +#define kqwl_count kqwl_kqueue.kq_count +#define kqwl_p kqwl_kqueue.kq_p +#define kqwl_queue kqwl_kqueue.kq_queue + +#define kqwl_req_lock(kqwl) lck_spin_lock(&kqwl->kqwl_reqlock) +#define kqwl_req_unlock(kqwl) lck_spin_unlock(&kqwl->kqwl_reqlock) +#define kqwl_req_held(kqwl) LCK_SPIN_ASSERT(&kqwl->kqwl_reqlock, LCK_ASSERT_OWNED) + +#define KQ_WORKLOOP_RETAINS_MAX UINT32_MAX + +extern int workloop_fulfill_threadreq(struct proc *p, workq_threadreq_t req, thread_t thread, int flags); extern struct kqueue *kqueue_alloc(struct proc *, unsigned int); extern void kqueue_dealloc(struct kqueue *); -typedef int (*kevent_callback_t)(struct kqueue *, struct kevent_internal_s *, void *); -typedef void (*kqueue_continue_t)(struct kqueue *, void *, int); +extern void knotes_dealloc(struct proc *); extern void kevent_register(struct kqueue *, struct kevent_internal_s *, struct proc *); extern int kqueue_scan(struct kqueue *, kevent_callback_t, kqueue_continue_t, diff --git a/bsd/sys/fasttrap.h b/bsd/sys/fasttrap.h index 3aa0db471..ec2ece46c 100644 --- a/bsd/sys/fasttrap.h +++ b/bsd/sys/fasttrap.h @@ -75,14 +75,21 @@ typedef struct fasttrap_probe_spec { #if defined(__APPLE__) fasttrap_provider_type_t ftps_provider_type; fasttrap_probe_type_t ftps_probe_type; +#if defined(__arm__) || defined(__arm64__) + uint32_t ftps_arch_subinfo; // For any additional per probe architecture specific data +#endif #endif char ftps_func[DTRACE_FUNCNAMELEN]; char ftps_mod[DTRACE_MODNAMELEN]; #if defined(__APPLE__) +#if defined(__arm__) || defined(__arm64__) + // We already have 'padding' from the ftps_arch_subinfo above +#else #if !defined(__LP64__) uint32_t pad; /* Explicit pad to keep ILP32 and LP64 lined up. */ #endif +#endif #endif uint64_t ftps_pc; uint64_t ftps_size; diff --git a/bsd/sys/fasttrap_impl.h b/bsd/sys/fasttrap_impl.h index 439d36864..1ca389cb6 100644 --- a/bsd/sys/fasttrap_impl.h +++ b/bsd/sys/fasttrap_impl.h @@ -97,6 +97,7 @@ typedef struct fasttrap_provider { uint_t ftp_retired; /* mark when retired */ lck_mtx_t ftp_mtx; /* provider lock */ lck_mtx_t ftp_cmtx; /* lock on creating probes */ + uint64_t ftp_pcount; /* probes in provider count */ uint64_t ftp_rcount; /* enabled probes ref count */ uint64_t ftp_ccount; /* consumers creating probes */ uint64_t ftp_mcount; /* meta provider count */ @@ -130,6 +131,7 @@ struct fasttrap_probe { uint8_t *ftp_argmap; /* native to translated args */ uint8_t ftp_nargs; /* translated argument count */ uint8_t ftp_enabled; /* is this probe enabled */ + uint8_t ftp_triggered; char *ftp_xtypes; /* translated types index */ char *ftp_ntypes; /* native types index */ fasttrap_id_tp_t ftp_tps[1]; /* flexible array */ @@ -188,6 +190,13 @@ extern fasttrap_hash_t fasttrap_tpoints; #define FASTTRAP_TPOINTS_INDEX(pid, pc) \ (((pc) / sizeof (fasttrap_instr_t) + (pid)) & fasttrap_tpoints.fth_mask) + +#ifdef CONFIG_EMBEDDED +#define FASTTRAP_ASYNC_REMOVE +#endif + +extern void fasttrap_tracepoint_retire(proc_t *p, fasttrap_tracepoint_t *tp); + /* * Must be implemented by fasttrap_isa.c */ @@ -199,6 +208,9 @@ extern int fasttrap_tracepoint_remove(proc_t *, fasttrap_tracepoint_t *); #if defined(__x86_64__) extern int fasttrap_pid_probe(x86_saved_state_t *regs); extern int fasttrap_return_probe(x86_saved_state_t* regs); +#elif defined(__arm__) || defined(__arm64__) +extern int fasttrap_pid_probe(arm_saved_state_t *rp); +extern int fasttrap_return_probe(arm_saved_state_t *regs); #else #error architecture not supported #endif @@ -206,6 +218,7 @@ extern int fasttrap_return_probe(x86_saved_state_t* regs); extern uint64_t fasttrap_pid_getarg(void *, dtrace_id_t, void *, int, int); extern uint64_t fasttrap_usdt_getarg(void *, dtrace_id_t, void *, int, int); + #ifdef __cplusplus } #endif diff --git a/bsd/sys/fbt.h b/bsd/sys/fbt.h index 4c67eef5c..fdda0b161 100644 --- a/bsd/sys/fbt.h +++ b/bsd/sys/fbt.h @@ -31,6 +31,10 @@ #if defined (__x86_64__) typedef uint8_t machine_inst_t; +#elif defined(__arm__) +typedef uint16_t machine_inst_t; +#elif defined(__arm64__) +typedef uint32_t machine_inst_t; #else #error Unknown Architecture #endif @@ -61,4 +65,11 @@ extern int dtrace_invop(uintptr_t, uintptr_t *, uintptr_t); extern int fbt_invop(uintptr_t, uintptr_t *, uintptr_t); extern void fbt_provide_module(void *, struct modctl *); extern int fbt_enable (void *arg, dtrace_id_t id, void *parg); + +extern int fbt_module_excluded(struct modctl*); +extern int fbt_excluded(const char *); + +extern void fbt_provide_probe(struct modctl *ctl, uintptr_t instr_low, uintptr_t instr_high, char *modname, char* symbol_name, machine_inst_t* symbol_start); + +extern void fbt_provide_module_kernel_syms(struct modctl *ctl); #endif /* _FBT_H */ diff --git a/bsd/sys/fcntl.h b/bsd/sys/fcntl.h index 0e24d78ce..02c868bba 100644 --- a/bsd/sys/fcntl.h +++ b/bsd/sys/fcntl.h @@ -361,7 +361,7 @@ #define F_GETCONFINED 96 /* is-fd-confined? */ #endif -#define F_ADDFILESIGS_RETURN 97 /* Add signature from same file, return end offset in structure on sucess */ +#define F_ADDFILESIGS_RETURN 97 /* Add signature from same file, return end offset in structure on success */ #define F_CHECK_LV 98 /* Check if Library Validation allows this Mach-O file to be mapped into the calling process */ #define F_PUNCHHOLE 99 /* Deallocate a range of the file */ diff --git a/bsd/sys/file_internal.h b/bsd/sys/file_internal.h index 60a47afa5..0b9ed96b2 100644 --- a/bsd/sys/file_internal.h +++ b/bsd/sys/file_internal.h @@ -80,6 +80,8 @@ struct proc; struct uio; struct knote; +struct kevent_internal_s; + #ifdef __APPLE_API_UNSTABLE struct file; @@ -183,7 +185,7 @@ struct fileglob { void *wql, vfs_context_t ctx); int (*fo_close) (struct fileglob *fg, vfs_context_t ctx); int (*fo_kqfilter) (struct fileproc *fp, struct knote *kn, - vfs_context_t ctx); + struct kevent_internal_s *kev, vfs_context_t ctx); int (*fo_drain) (struct fileproc *fp, vfs_context_t ctx); } *fg_ops; off_t fg_offset; @@ -213,7 +215,8 @@ int fo_write(struct fileproc *fp, struct uio *uio, int flags, int fo_ioctl(struct fileproc *fp, u_long com, caddr_t data, vfs_context_t ctx); int fo_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx); int fo_close(struct fileglob *fg, vfs_context_t ctx); -int fo_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx); +int fo_kqfilter(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx); void fileproc_drain(proc_t, struct fileproc *); int fp_tryswap(proc_t, int fd, struct fileproc *nfp); int fp_drop(struct proc *p, int fd, struct fileproc *fp, int locked); diff --git a/bsd/sys/filedesc.h b/bsd/sys/filedesc.h index 41dc190b5..b79440b48 100644 --- a/bsd/sys/filedesc.h +++ b/bsd/sys/filedesc.h @@ -85,10 +85,17 @@ #ifdef BSD_KERNEL_PRIVATE +#include + struct klist; +struct kqlist; struct filedesc { struct fileproc **fd_ofiles; /* file structures for open files */ + lck_mtx_t fd_kqhashlock; /* lock for dynamic kqueue hash */ + u_long fd_kqhashmask; /* size of dynamic kqueue hash */ + struct kqlist *fd_kqhash; /* hash table for dynamic kqueues */ + struct kqueue *fd_wqkqueue; /* the workq kqueue */ char *fd_ofileflags; /* per-process open file flags */ struct vnode *fd_cdir; /* current directory */ struct vnode *fd_rdir; /* root directory */ @@ -96,11 +103,12 @@ struct filedesc { int fd_lastfile; /* high-water mark of fd_ofiles */ int fd_freefile; /* approx. next free file */ u_short fd_cmask; /* mask for file creation */ + int fd_flags; int fd_knlistsize; /* size of knlist */ struct klist *fd_knlist; /* list of attached knotes */ u_long fd_knhashmask; /* size of knhash */ struct klist *fd_knhash; /* hash table for attached knotes */ - int fd_flags; + lck_mtx_t fd_knhashlock; /* lock for hash table for attached knotes */ }; /* @@ -165,7 +173,7 @@ extern int falloc_withalloc(proc_t p, struct fileproc **resultfp, extern struct filedesc *fdcopy(proc_t p, struct vnode *uth_cdir); extern void fdfree(proc_t p); -extern void fdexec(proc_t p, short flags); +extern void fdexec(proc_t p, short flags, int self_exec); #endif /* __APPLE_API_PRIVATE */ #endif /* KERNEL */ diff --git a/bsd/sys/fsctl.h b/bsd/sys/fsctl.h index 5d87fbf7f..8a3624d3b 100644 --- a/bsd/sys/fsctl.h +++ b/bsd/sys/fsctl.h @@ -244,6 +244,15 @@ typedef struct package_ext_info { uint32_t max_width; } package_ext_info; +/* Disk conditioner configuration */ +typedef struct disk_conditioner_info { + int enabled; + uint64_t access_time_usec; // maximum latency until transfer begins + uint64_t read_throughput_mbps; // maximum throughput for reads + uint64_t write_throughput_mbps; // maximum throughput for writes + int is_ssd; // behave like an SSD +} disk_conditioner_info; + #define FSCTL_SYNC_FULLSYNC (1<<0) /* Flush the data fully to disk, if supported by the filesystem */ #define FSCTL_SYNC_WAIT (1<<1) /* Wait for the sync to complete */ @@ -299,18 +308,16 @@ typedef struct package_ext_info { #define FSIOC_FIOSEEKDATA _IOWR('A', 17, off_t) #define FSCTL_FIOSEEKDATA IOCBASECMD(FSIOC_FIOSEEKDATA) -// -// IO commands 16 and 17 are currently unused -// +/* Disk conditioner */ +#define DISK_CONDITIONER_IOC_GET _IOR('A', 18, disk_conditioner_info) +#define DISK_CONDITIONER_FSCTL_GET IOCBASECMD(DISK_CONDITIONER_IOC_GET) +#define DISK_CONDITIONER_IOC_SET _IOW('A', 19, disk_conditioner_info) +#define DISK_CONDITIONER_FSCTL_SET IOCBASECMD(DISK_CONDITIONER_IOC_SET) // // Spotlight and fseventsd use these fsctl()'s to find out // the mount time of a volume and the last time it was -// unmounted. Both HFS and ZFS support these calls. -// -// User space code should pass the "_IOC_" macros while the -// kernel should test for the "_FSCTL_" variant of the macro -// in its vnop_ioctl function. +// unmounted. Both HFS and APFS support these calls. // // NOTE: the values for these defines should _not_ be changed // or else it will break binary compatibility with mds @@ -321,7 +328,6 @@ typedef struct package_ext_info { #define SPOTLIGHT_IOC_GET_LAST_MTIME _IOR('h', 19, u_int32_t) #define SPOTLIGHT_FSCTL_GET_LAST_MTIME IOCBASECMD(SPOTLIGHT_IOC_GET_LAST_MTIME) - #ifndef KERNEL #include diff --git a/bsd/sys/fsevents.h b/bsd/sys/fsevents.h index a9c922557..bf338c6e0 100644 --- a/bsd/sys/fsevents.h +++ b/bsd/sys/fsevents.h @@ -44,8 +44,9 @@ #define FSE_DOCID_CREATED 11 #define FSE_DOCID_CHANGED 12 #define FSE_UNMOUNT_PENDING 13 // iOS-only: client must respond via FSEVENTS_UNMOUNT_PENDING_ACK +#define FSE_CLONE 14 -#define FSE_MAX_EVENTS 14 +#define FSE_MAX_EVENTS 15 #define FSE_ALL_EVENTS 998 #define FSE_EVENTS_DROPPED 999 @@ -99,6 +100,7 @@ #define FSE_MODE_LAST_HLINK (1 << 30) // link count == 0 on a hard-link delete #define FSE_REMOTE_DIR_EVENT (1 << 29) // this is a remotely generated directory-level granularity event #define FSE_TRUNCATED_PATH (1 << 28) // the path for this item had to be truncated +#define FSE_MODE_CLONE (1 << 27) // notification is for a clone // ioctl's on /dev/fsevents typedef struct fsevent_clone_args { diff --git a/bsd/sys/fsgetpath.h b/bsd/sys/fsgetpath.h index bad8b4e1b..da8e53173 100644 --- a/bsd/sys/fsgetpath.h +++ b/bsd/sys/fsgetpath.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -29,23 +29,39 @@ #ifndef _FSGETPATH_H_ #define _FSGETPATH_H_ +#ifndef KERNEL + +#include +#include +#include +#include +#include <_types/_uint64_t.h> +#include + +#ifdef PRIVATE +/* + * These are only included for compatibility with previous header + */ #include #include - #ifdef __APPLE_API_PRIVATE - -#ifndef KERNEL -__BEGIN_DECLS - #include #include +#endif /* __APPLE_API_PRIVATE */ +#endif /* PRIVATE */ + +__BEGIN_DECLS /* * Obtain the full pathname of a file system object by id. - * - * This is a private SPI used by the File Manager. */ -ssize_t fsgetpath(char * __restrict buf, size_t bufsize, fsid_t* fsid, uint64_t objid); +ssize_t fsgetpath(char *, size_t, fsid_t *, uint64_t) __OSX_AVAILABLE(10.13) __IOS_AVAILABLE(11.0) __TVOS_AVAILABLE(11.0) __WATCHOS_AVAILABLE(4.0); + +#ifdef PRIVATE +#include + +#ifdef __APPLE_API_PRIVATE + /* * openbyid_np: open a file given a file system id and a file system object id @@ -65,9 +81,11 @@ ssize_t fsgetpath(char * __restrict buf, size_t bufsize, fsid_t* fsid, uint64_t */ int openbyid_np(fsid_t* fsid, fsobj_id_t* objid, int flags); +#endif /* __APPLE_API_PRIVATE */ +#endif /* PRIVATE */ + __END_DECLS -#endif /* KERNEL */ -#endif /* __APPLE_API_PRIVATE */ +#endif /* KERNEL */ #endif /* !_FSGETPATH_H_ */ diff --git a/bsd/sys/guarded.h b/bsd/sys/guarded.h index 6fa6a7752..ae1ec05f9 100644 --- a/bsd/sys/guarded.h +++ b/bsd/sys/guarded.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -59,12 +59,10 @@ extern ssize_t guarded_pwrite_np(int fd, const guardid_t *guard, const void *buf extern ssize_t guarded_writev_np(int fd, const guardid_t *guard, const struct iovec *iovp, int iovcnt); #endif /* KERNEL */ -/* - * Guard types. - * - * GUARD_TYPE_FD: Guarded file descriptor. - */ -#define GUARD_TYPE_FD 0x2 +#ifndef GUARD_TYPE_FD +/* temporary source compat: use instead */ +#define GUARD_TYPE_FD 0x2 +#endif /* * File descriptor guard flavors. @@ -102,7 +100,7 @@ extern ssize_t guarded_writev_np(int fd, const guardid_t *guard, const struct io * Violating a guard results in an error (EPERM), and potentially * an exception with one or more of the following bits set. */ -enum guard_exception_codes { +enum guard_fd_exception_codes { kGUARD_EXC_CLOSE = 1u << 0, /* close of a guarded fd */ kGUARD_EXC_DUP = 1u << 1, /* dup of a guarded fd */ kGUARD_EXC_NOCLOEXEC = 1u << 2, /* clear close-on-exec */ @@ -112,6 +110,60 @@ enum guard_exception_codes { kGUARD_EXC_WRITE = 1u << 6 /* write on a guarded fd */ }; +/* + * Experimental guarded vnode support + */ +#define VNG_RENAME_TO (1u << 0) +#define VNG_RENAME_FROM (1u << 1) +#define VNG_UNLINK (1u << 2) +#define VNG_WRITE_OTHER (1u << 3) +#define VNG_TRUNC_OTHER (1u << 4) +#define VNG_LINK (1u << 5) +#define VNG_EXCHDATA (1u << 6) + +#define VNG_ALL \ + (VNG_RENAME_TO | VNG_RENAME_FROM | VNG_UNLINK | VNG_LINK | \ + VNG_WRITE_OTHER | VNG_TRUNC_OTHER | VNG_EXCHDATA) + +struct vnguard_set { + int vns_fd; + unsigned vns_attrs; + guardid_t vns_guard; +}; + +#define VNG_SYSC_PING 0 +#define VNG_SYSC_SET_GUARD 1 + +#define VNG_POLICY_NAME "vnguard" + +/* + * Violating a guard may result in an error (EPERM), and potentially + * an exception with one or more of the following bits set. + */ +enum guard_vn_exception_codes { + kGUARD_EXC_RENAME_TO = VNG_RENAME_TO, + kGUARD_EXC_RENAME_FROM = VNG_RENAME_FROM, + kGUARD_EXC_UNLINK = VNG_UNLINK, + kGUARD_EXC_WRITE_OTHER = VNG_WRITE_OTHER, + kGUARD_EXC_TRUNC_OTHER = VNG_TRUNC_OTHER, + kGUARD_EXC_LINK = VNG_LINK, + kGUARD_EXC_EXCHDATA = VNG_EXCHDATA, +}; + +#if defined(KERNEL) + +/* Guard violation behaviors: not all combinations make sense */ + +#define kVNG_POLICY_LOGMSG (1u << 0) +#define kVNG_POLICY_EPERM (1u << 1) +#define kVNG_POLICY_EXC (1u << 2) +#define kVNG_POLICY_EXC_CORPSE (1u << 3) +#define kVNG_POLICY_SIGKILL (1u << 4) + +extern int vnguard_exceptions_active(void); +extern void vnguard_policy_init(void); +#endif /* KERNEL */ + #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ __END_DECLS diff --git a/bsd/sys/imgact.h b/bsd/sys/imgact.h index fc23e70e9..c1fbde252 100644 --- a/bsd/sys/imgact.h +++ b/bsd/sys/imgact.h @@ -119,6 +119,9 @@ struct image_params { void *ip_px_smpx; /* MAC-specific spawn attrs. */ void *ip_px_persona; /* persona args */ void *ip_cs_error; /* codesigning error reason */ + + uint64_t ip_dyld_fsid; + uint64_t ip_dyld_fsobjid; }; /* @@ -134,5 +137,6 @@ struct image_params { #define IMGPF_ALLOW_DATA_EXEC 0x00000040 /* forcibly disallow data execution */ #define IMGPF_VFORK_EXEC 0x00000080 /* vfork followed by exec */ #define IMGPF_EXEC 0x00000100 /* exec */ +#define IMGPF_HIGH_BITS_ASLR 0x00000200 /* randomize high bits of ASLR slide */ #endif /* !_SYS_IMGACT */ diff --git a/bsd/sys/kauth.h b/bsd/sys/kauth.h index dd496f8da..6d46a5afb 100644 --- a/bsd/sys/kauth.h +++ b/bsd/sys/kauth.h @@ -138,6 +138,8 @@ struct kauth_cache_sizes { #define KAUTH_SET_CACHE_SIZES (1<<4) #define KAUTH_CLEAR_CACHES (1<<5) +#define IDENTITYSVC_ENTITLEMENT "com.apple.private.identitysvc" + #ifdef KERNEL /* diff --git a/bsd/sys/kdebug.h b/bsd/sys/kdebug.h index d29e0cd17..0e3988617 100644 --- a/bsd/sys/kdebug.h +++ b/bsd/sys/kdebug.h @@ -35,6 +35,7 @@ #include #include + __BEGIN_DECLS #ifdef __APPLE_API_UNSTABLE @@ -192,6 +193,7 @@ extern void kernel_debug_enter( #define DBG_WORKQUEUE 9 #define DBG_CORESTORAGE 10 #define DBG_CG 11 +#define DBG_MONOTONIC 12 #define DBG_MISC 20 #define DBG_SECURITY 30 #define DBG_DYLD 31 @@ -354,37 +356,45 @@ extern void kdebug_reset(void); #endif /* XNU_KERNEL_PRIVATE */ /* **** The Kernel Debug Sub Classes for Mach (DBG_MACH) **** */ -#define DBG_MACH_EXCP_KTRAP_x86 0x02 /* Kernel Traps on x86 */ -#define DBG_MACH_EXCP_DFLT 0x03 /* Data Translation Fault */ -#define DBG_MACH_EXCP_IFLT 0x04 /* Inst Translation Fault */ -#define DBG_MACH_EXCP_INTR 0x05 /* Interrupts */ -#define DBG_MACH_EXCP_ALNG 0x06 /* Alignment Exception */ -#define DBG_MACH_EXCP_UTRAP_x86 0x07 /* User Traps on x86 */ -#define DBG_MACH_EXCP_FP 0x08 /* FP Unavail */ -#define DBG_MACH_EXCP_DECI 0x09 /* Decrementer Interrupt */ -#define DBG_MACH_CHUD 0x0A /* deprecated name */ -#define DBG_MACH_SIGNPOST 0x0A /* kernel signposts */ -#define DBG_MACH_EXCP_SC 0x0C /* System Calls */ -#define DBG_MACH_EXCP_TRACE 0x0D /* Trace exception */ -#define DBG_MACH_EXCP_EMUL 0x0E /* Instruction emulated */ -#define DBG_MACH_IHDLR 0x10 /* Interrupt Handlers */ -#define DBG_MACH_IPC 0x20 /* Inter Process Comm */ -#define DBG_MACH_RESOURCE 0x25 /* tracing limits, etc */ -#define DBG_MACH_VM 0x30 /* Virtual Memory */ -#define DBG_MACH_LEAKS 0x31 /* alloc/free */ -#define DBG_MACH_WORKINGSET 0x32 /* private subclass for working set related debugging */ -#define DBG_MACH_SCHED 0x40 /* Scheduler */ -#define DBG_MACH_MSGID_INVALID 0x50 /* Messages - invalid */ -#define DBG_MACH_LOCKS 0x60 /* new lock APIs */ -#define DBG_MACH_PMAP 0x70 /* pmap */ -#define DBG_MACH_CLOCK 0x80 /* clock */ -#define DBG_MACH_MP 0x90 /* MP related */ -#define DBG_MACH_VM_PRESSURE 0xA0 /* Memory Pressure Events */ -#define DBG_MACH_STACKSHOT 0xA1 /* Stackshot/Microstackshot subsystem */ -#define DBG_MACH_SFI 0xA2 /* Selective Forced Idle (SFI) */ -#define DBG_MACH_ENERGY_PERF 0xA3 /* Energy/performance resource stats */ -#define DBG_MACH_SYSDIAGNOSE 0xA4 /* sysdiagnose keychord */ -#define DBG_MACH_ZALLOC 0xA5 /* Zone allocator */ +#define DBG_MACH_EXCP_KTRAP_x86 0x02 /* Kernel Traps on x86 */ +#define DBG_MACH_EXCP_DFLT 0x03 /* Data Translation Fault */ +#define DBG_MACH_EXCP_IFLT 0x04 /* Inst Translation Fault */ +#define DBG_MACH_EXCP_INTR 0x05 /* Interrupts */ +#define DBG_MACH_EXCP_ALNG 0x06 /* Alignment Exception */ +#define DBG_MACH_EXCP_UTRAP_x86 0x07 /* User Traps on x86 */ +#define DBG_MACH_EXCP_FP 0x08 /* FP Unavail */ +#define DBG_MACH_EXCP_DECI 0x09 /* Decrementer Interrupt */ +#define DBG_MACH_CHUD 0x0A /* deprecated name */ +#define DBG_MACH_SIGNPOST 0x0A /* kernel signposts */ +#define DBG_MACH_EXCP_SC 0x0C /* System Calls */ +#define DBG_MACH_EXCP_TRACE 0x0D /* Trace exception */ +#define DBG_MACH_EXCP_EMUL 0x0E /* Instruction emulated */ +#define DBG_MACH_IHDLR 0x10 /* Interrupt Handlers */ +#define DBG_MACH_IPC 0x20 /* Inter Process Comm */ +#define DBG_MACH_RESOURCE 0x25 /* tracing limits, etc */ +#define DBG_MACH_VM 0x30 /* Virtual Memory */ +#define DBG_MACH_LEAKS 0x31 /* alloc/free */ +#define DBG_MACH_WORKINGSET 0x32 /* private subclass for working set related debugging */ +#define DBG_MACH_SCHED 0x40 /* Scheduler */ +#define DBG_MACH_MSGID_INVALID 0x50 /* Messages - invalid */ +#define DBG_MACH_LOCKS 0x60 /* new lock APIs */ +#define DBG_MACH_PMAP 0x70 /* pmap */ +#define DBG_MACH_CLOCK 0x80 /* clock */ +#define DBG_MACH_MP 0x90 /* MP related */ +#define DBG_MACH_VM_PRESSURE 0xA0 /* Memory Pressure Events */ +#define DBG_MACH_STACKSHOT 0xA1 /* Stackshot/Microstackshot subsystem */ +#define DBG_MACH_SFI 0xA2 /* Selective Forced Idle (SFI) */ +#define DBG_MACH_ENERGY_PERF 0xA3 /* Energy/performance resource stats */ +#define DBG_MACH_SYSDIAGNOSE 0xA4 /* sysdiagnose */ +#define DBG_MACH_ZALLOC 0xA5 /* Zone allocator */ +#define DBG_MACH_THREAD_GROUP 0xA6 /* Thread groups */ +#define DBG_MACH_COALITION 0xA7 /* Coalitions */ + +/* Interrupt type bits for DBG_MACH_EXCP_INTR */ +#define DBG_INTR_TYPE_UNKNOWN 0x0 /* default/unknown interrupt */ +#define DBG_INTR_TYPE_IPI 0x1 /* interprocessor interrupt */ +#define DBG_INTR_TYPE_TIMER 0x2 /* timer interrupt */ +#define DBG_INTR_TYPE_OTHER 0x3 /* other (usually external) interrupt */ /* Codes for Scheduler (DBG_MACH_SCHED) */ #define MACH_SCHED 0x0 /* Scheduler */ @@ -399,8 +409,8 @@ extern void kdebug_reset(void); #define MACH_IDLE 0x9 /* processor idling */ #define MACH_STACK_DEPTH 0xa /* stack depth at switch */ #define MACH_MOVED 0xb /* did not use original scheduling decision */ -/* unused 0xc */ -/* unused 0xd */ +#define MACH_PSET_LOAD_AVERAGE 0xc +#define MACH_AMP_DEBUG 0xd #define MACH_FAILSAFE 0xe /* tripped fixed-pri/RT failsafe */ #define MACH_BLOCK 0xf /* thread block */ #define MACH_WAIT 0x10 /* thread wait assertion */ @@ -433,6 +443,10 @@ extern void kdebug_reset(void); #define MACH_SCHED_LOAD 0x2d /* load update */ #define MACH_REC_CORES_FAILSAFE 0x2e /* recommended processor failsafe kicked in */ #define MACH_SCHED_QUANTUM_EXPIRED 0x2f /* thread quantum expired */ +#define MACH_EXEC_PROMOTE 0x30 /* Thread promoted by exec boost */ +#define MACH_EXEC_DEMOTE 0x31 /* Thread demoted from exec boost */ +#define MACH_AMP_SIGNAL_SPILL 0x32 /* AMP spill signal sent to cpuid */ +#define MACH_AMP_STEAL 0x33 /* AMP thread stolen or spilled */ /* Variants for MACH_MULTIQ_DEQUEUE */ #define MACH_MULTIQ_BOUND 1 @@ -465,6 +479,21 @@ extern void kdebug_reset(void); #define MACH_IPC_KMSG_INFO 0xa /* Send/Receive info for a kmsg */ #define MACH_IPC_KMSG_LINK 0xb /* link a kernel kmsg pointer to user mach_msg_header_t */ +/* Codes for thread groups (DBG_MACH_THREAD_GROUP) */ +#define MACH_THREAD_GROUP_NEW 0x0 +#define MACH_THREAD_GROUP_FREE 0x1 +#define MACH_THREAD_GROUP_SET 0x2 +#define MACH_THREAD_GROUP_NAME 0x3 +#define MACH_THREAD_GROUP_NAME_FREE 0x4 +#define MACH_THREAD_GROUP_FLAGS 0x5 + +/* Codes for coalitions (DBG_MACH_COALITION) */ +#define MACH_COALITION_NEW 0x0 +#define MACH_COALITION_FREE 0x1 +#define MACH_COALITION_ADOPT 0x2 +#define MACH_COALITION_REMOVE 0x3 +#define MACH_COALITION_THREAD_GROUP_SET 0x4 + /* Codes for pmap (DBG_MACH_PMAP) */ #define PMAP__CREATE 0x0 #define PMAP__DESTROY 0x1 @@ -477,23 +506,26 @@ extern void kdebug_reset(void); #define PMAP__FLUSH_TLBS 0x8 #define PMAP__UPDATE_INTERRUPT 0x9 #define PMAP__ATTRIBUTE_CLEAR 0xa -#define PMAP__REUSABLE 0xb +#define PMAP__REUSABLE 0xb /* This appears to be unused */ #define PMAP__QUERY_RESIDENT 0xc #define PMAP__FLUSH_KERN_TLBS 0xd #define PMAP__FLUSH_DELAYED_TLBS 0xe #define PMAP__FLUSH_TLBS_TO 0xf #define PMAP__FLUSH_EPT 0x10 +#define PMAP__FAST_FAULT 0x11 /* Codes for clock (DBG_MACH_CLOCK) */ #define MACH_EPOCH_CHANGE 0x0 /* wake epoch change */ - /* Codes for Stackshot/Microstackshot (DBG_MACH_STACKSHOT) */ #define MICROSTACKSHOT_RECORD 0x0 #define MICROSTACKSHOT_GATHER 0x1 -/* Codes for sysdiagnose */ -#define SYSDIAGNOSE_NOTIFY_USER 0x0 +/* Codes for sysdiagnose (DBG_MACH_SYSDIAGNOSE) */ +#define SYSDIAGNOSE_NOTIFY_USER 0x0 +#define SYSDIAGNOSE_FULL 0x1 +#define SYSDIAGNOSE_STACKSHOT 0x2 +#define SYSDIAGNOSE_TAILSPIN 0x3 /* Codes for Selective Forced Idle (DBG_MACH_SFI) */ #define SFI_SET_WINDOW 0x0 @@ -597,35 +629,37 @@ extern void kdebug_reset(void); #define DBG_IOGRAPHICS 50 /* Graphics */ #define DBG_HIBERNATE 51 /* hibernation related events */ #define DBG_IOTHUNDERBOLT 52 /* Thunderbolt */ - +#define DBG_BOOTER 53 /* booter related events */ /* Backwards compatibility */ #define DBG_IOPOINTING DBG_IOHID /* OBSOLETE: Use DBG_IOHID instead */ #define DBG_IODISK DBG_IOSTORAGE /* OBSOLETE: Use DBG_IOSTORAGE instead */ /* **** The Kernel Debug Sub Classes for Device Drivers (DBG_DRIVERS) **** */ -#define DBG_DRVSTORAGE 1 /* Storage layers */ -#define DBG_DRVNETWORK 2 /* Network layers */ -#define DBG_DRVKEYBOARD 3 /* Keyboard */ -#define DBG_DRVHID 4 /* HID Devices */ -#define DBG_DRVAUDIO 5 /* Audio */ -#define DBG_DRVSERIAL 7 /* Serial */ -#define DBG_DRVSAM 8 /* SCSI Architecture Model layers */ -#define DBG_DRVPARALLELATA 9 /* Parallel ATA */ -#define DBG_DRVPARALLELSCSI 10 /* Parallel SCSI */ -#define DBG_DRVSATA 11 /* Serial ATA */ -#define DBG_DRVSAS 12 /* SAS */ -#define DBG_DRVFIBRECHANNEL 13 /* FiberChannel */ -#define DBG_DRVUSB 14 /* USB */ -#define DBG_DRVBLUETOOTH 15 /* Bluetooth */ -#define DBG_DRVFIREWIRE 16 /* FireWire */ -#define DBG_DRVINFINIBAND 17 /* Infiniband */ -#define DBG_DRVGRAPHICS 18 /* Graphics */ -#define DBG_DRVSD 19 /* Secure Digital */ -#define DBG_DRVNAND 20 /* NAND drivers and layers */ -#define DBG_SSD 21 /* SSD */ -#define DBG_DRVSPI 22 /* SPI */ -#define DBG_DRVWLAN_802_11 23 /* WLAN 802.11 */ +#define DBG_DRVSTORAGE 1 /* Storage layers */ +#define DBG_DRVNETWORK 2 /* Network layers */ +#define DBG_DRVKEYBOARD 3 /* Keyboard */ +#define DBG_DRVHID 4 /* HID Devices */ +#define DBG_DRVAUDIO 5 /* Audio */ +#define DBG_DRVSERIAL 7 /* Serial */ +#define DBG_DRVSAM 8 /* SCSI Architecture Model layers */ +#define DBG_DRVPARALLELATA 9 /* Parallel ATA */ +#define DBG_DRVPARALLELSCSI 10 /* Parallel SCSI */ +#define DBG_DRVSATA 11 /* Serial ATA */ +#define DBG_DRVSAS 12 /* SAS */ +#define DBG_DRVFIBRECHANNEL 13 /* FiberChannel */ +#define DBG_DRVUSB 14 /* USB */ +#define DBG_DRVBLUETOOTH 15 /* Bluetooth */ +#define DBG_DRVFIREWIRE 16 /* FireWire */ +#define DBG_DRVINFINIBAND 17 /* Infiniband */ +#define DBG_DRVGRAPHICS 18 /* Graphics */ +#define DBG_DRVSD 19 /* Secure Digital */ +#define DBG_DRVNAND 20 /* NAND drivers and layers */ +#define DBG_SSD 21 /* SSD */ +#define DBG_DRVSPI 22 /* SPI */ +#define DBG_DRVWLAN_802_11 23 /* WLAN 802.11 */ +#define DBG_DRVSSM 24 /* System State Manager(AppleSSM) */ +#define DBG_DRVSMC 25 /* System Management Controller */ /* Backwards compatibility */ #define DBG_DRVPOINTING DBG_DRVHID /* OBSOLETE: Use DBG_DRVHID instead */ @@ -638,6 +672,7 @@ extern void kdebug_reset(void); #define DBG_DLIL_PR_FLT 4 /* DLIL Protocol Filter */ #define DBG_DLIL_IF_FLT 5 /* DLIL Interface FIlter */ + /* The Kernel Debug Sub Classes for File System (DBG_FSYSTEM) */ #define DBG_FSRW 0x1 /* reads and writes to the filesystem */ #define DBG_DKRW 0x2 /* reads and writes to the disk */ @@ -670,13 +705,13 @@ extern void kdebug_reset(void); /* The Kernel Debug Sub Classes for BSD */ #define DBG_BSD_PROC 0x01 /* process/signals related */ #define DBG_BSD_MEMSTAT 0x02 /* memorystatus / jetsam operations */ +#define DBG_BSD_KEVENT 0x03 /* kqueue / kevent related */ #define DBG_BSD_EXCP_SC 0x0C /* System Calls */ #define DBG_BSD_AIO 0x0D /* aio (POSIX async IO) */ #define DBG_BSD_SC_EXTENDED_INFO 0x0E /* System Calls, extended info */ #define DBG_BSD_SC_EXTENDED_INFO2 0x0F /* System Calls, extended info */ #define DBG_BSD_KDEBUG_TEST 0xFF /* for testing kdebug */ - /* The Codes for BSD subcode class DBG_BSD_PROC */ #define BSD_PROC_EXIT 1 /* process exit */ #define BSD_PROC_FRCEXIT 2 /* Kernel force termination */ @@ -701,36 +736,66 @@ extern void kdebug_reset(void); #define BSD_MEMSTAT_DO_KILL 13 /* memorystatus kills */ #endif /* PRIVATE */ +/* Codes for BSD subcode class DBG_BSD_KEVENT */ +#define BSD_KEVENT_KQ_PROCESS_BEGIN 1 +#define BSD_KEVENT_KQ_PROCESS_END 2 +#define BSD_KEVENT_KQWQ_PROCESS_BEGIN 3 +#define BSD_KEVENT_KQWQ_PROCESS_END 4 +#define BSD_KEVENT_KQWQ_BIND 5 +#define BSD_KEVENT_KQWQ_UNBIND 6 +#define BSD_KEVENT_KQWQ_THREQUEST 7 +#define BSD_KEVENT_KQWL_PROCESS_BEGIN 8 +#define BSD_KEVENT_KQWL_PROCESS_END 9 +#define BSD_KEVENT_KQWL_THREQUEST 10 +#define BSD_KEVENT_KQWL_THADJUST 11 +#define BSD_KEVENT_KQ_REGISTER 12 +#define BSD_KEVENT_KQWQ_REGISTER 13 +#define BSD_KEVENT_KQWL_REGISTER 14 +#define BSD_KEVENT_KNOTE_ACTIVATE 15 +#define BSD_KEVENT_KQ_PROCESS 16 +#define BSD_KEVENT_KQWQ_PROCESS 17 +#define BSD_KEVENT_KQWL_PROCESS 18 +#define BSD_KEVENT_KQWL_BIND 19 +#define BSD_KEVENT_KQWL_UNBIND 20 +#define BSD_KEVENT_KNOTE_ENABLE 21 + /* The Kernel Debug Sub Classes for DBG_TRACE */ #define DBG_TRACE_DATA 0 #define DBG_TRACE_STRING 1 #define DBG_TRACE_INFO 2 /* The Kernel Debug events: */ -#define TRACE_DATA_NEWTHREAD (TRACEDBG_CODE(DBG_TRACE_DATA, 1)) -#define TRACE_DATA_EXEC (TRACEDBG_CODE(DBG_TRACE_DATA, 2)) -#define TRACE_DATA_THREAD_TERMINATE (TRACEDBG_CODE(DBG_TRACE_DATA, 3)) -#define TRACE_DATA_THREAD_TERMINATE_PID (TRACEDBG_CODE(DBG_TRACE_DATA, 4)) -#define TRACE_STRING_GLOBAL (TRACEDBG_CODE(DBG_TRACE_STRING, 0)) -#define TRACE_STRING_NEWTHREAD (TRACEDBG_CODE(DBG_TRACE_STRING, 1)) -#define TRACE_STRING_EXEC (TRACEDBG_CODE(DBG_TRACE_STRING, 2)) -#define TRACE_STRING_PROC_EXIT (TRACEDBG_CODE(DBG_TRACE_STRING, 3)) -#define TRACE_STRING_THREADNAME (TRACEDBG_CODE(DBG_TRACE_STRING, 4)) -#define TRACE_STRING_THREADNAME_PREV (TRACEDBG_CODE(DBG_TRACE_STRING, 5)) -#define TRACE_PANIC (TRACEDBG_CODE(DBG_TRACE_INFO, 0)) -#define TRACE_TIMESTAMPS (TRACEDBG_CODE(DBG_TRACE_INFO, 1)) -#define TRACE_LOST_EVENTS (TRACEDBG_CODE(DBG_TRACE_INFO, 2)) -#define TRACE_WRITING_EVENTS (TRACEDBG_CODE(DBG_TRACE_INFO, 3)) -#define TRACE_INFO_STRING (TRACEDBG_CODE(DBG_TRACE_INFO, 4)) +#define TRACE_DATA_NEWTHREAD (TRACEDBG_CODE(DBG_TRACE_DATA, 1)) +#define TRACE_DATA_EXEC (TRACEDBG_CODE(DBG_TRACE_DATA, 2)) +#define TRACE_DATA_THREAD_TERMINATE (TRACEDBG_CODE(DBG_TRACE_DATA, 3)) +#define TRACE_DATA_THREAD_TERMINATE_PID (TRACEDBG_CODE(DBG_TRACE_DATA, 4)) +#define TRACE_STRING_GLOBAL (TRACEDBG_CODE(DBG_TRACE_STRING, 0)) +#define TRACE_STRING_NEWTHREAD (TRACEDBG_CODE(DBG_TRACE_STRING, 1)) +#define TRACE_STRING_EXEC (TRACEDBG_CODE(DBG_TRACE_STRING, 2)) +#define TRACE_STRING_PROC_EXIT (TRACEDBG_CODE(DBG_TRACE_STRING, 3)) +#define TRACE_STRING_THREADNAME (TRACEDBG_CODE(DBG_TRACE_STRING, 4)) +#define TRACE_STRING_THREADNAME_PREV (TRACEDBG_CODE(DBG_TRACE_STRING, 5)) +#define TRACE_PANIC (TRACEDBG_CODE(DBG_TRACE_INFO, 0)) +#define TRACE_TIMESTAMPS (TRACEDBG_CODE(DBG_TRACE_INFO, 1)) +#define TRACE_LOST_EVENTS (TRACEDBG_CODE(DBG_TRACE_INFO, 2)) +#define TRACE_WRITING_EVENTS (TRACEDBG_CODE(DBG_TRACE_INFO, 3)) +#define TRACE_INFO_STRING (TRACEDBG_CODE(DBG_TRACE_INFO, 4)) +#define TRACE_RETROGRADE_EVENTS (TRACEDBG_CODE(DBG_TRACE_INFO, 5)) /* The Kernel Debug Sub Classes for DBG_CORESTORAGE */ #define DBG_CS_IO 0 /* The Kernel Debug Sub Classes for DBG_SECURITY */ -#define DBG_SEC_KERNEL 0 /* raw entropy collected by the kernel */ +#define DBG_SEC_KERNEL 0 /* raw entropy collected by the kernel */ +#define DBG_SEC_SANDBOX 1 /* Sub-class codes for CoreGraphics (DBG_CG) are defined in its component. */ +/* The Kernel Debug Sub Classes for DBG_MONOTONIC */ +#define DBG_MT_INSTRS_CYCLES 1 +#define DBG_MT_TMPTH 0xfe +#define DBG_MT_TMPCPU 0xff + /* The Kernel Debug Sub Classes for DBG_MISC */ #define DBG_EVENT 0x10 #define DBG_BUFFER 0x20 @@ -782,6 +847,7 @@ extern void kdebug_reset(void); #define OPEN_THROTTLE_WINDOW 0x1 #define PROCESS_THROTTLED 0x2 #define IO_THROTTLE_DISABLE 0x3 +#define IO_TIER_UPL_MISMATCH 0x4 /* Subclasses for MACH Importance Policies (DBG_IMPORTANCE) */ @@ -796,6 +862,7 @@ extern void kdebug_reset(void); #define IMP_USYNCH_QOS_OVERRIDE 0x1A /* Userspace synchronization applied QoS override to resource owning thread */ #define IMP_DONOR_CHANGE 0x1B /* The iit_donor bit changed */ #define IMP_MAIN_THREAD_QOS 0x1C /* The task's main thread QoS was set */ +#define IMP_SYNC_IPC_QOS 0x1D /* Sync IPC QOS override */ /* DBG_IMPORTANCE subclasses 0x20 - 0x3F reserved for task policy flavors */ /* Codes for IMP_ASSERTION */ @@ -822,6 +889,12 @@ extern void kdebug_reset(void); #define IMP_DONOR_UPDATE_LIVE_DONOR_STATE 0x0 #define IMP_DONOR_INIT_DONOR_STATE 0x1 +/* Code for IMP_SYNC_IPC_QOS */ +#define IMP_SYNC_IPC_QOS_APPLIED 0x0 +#define IMP_SYNC_IPC_QOS_REMOVED 0x1 +#define IMP_SYNC_IPC_QOS_OVERFLOW 0x2 +#define IMP_SYNC_IPC_QOS_UNDERFLOW 0x3 + /* Subclasses for MACH Bank Voucher Attribute Manager (DBG_BANK) */ #define BANK_ACCOUNT_INFO 0x10 /* Trace points related to bank account struct */ #define BANK_TASK_INFO 0x11 /* Trace points related to bank task struct */ @@ -834,6 +907,7 @@ extern void kdebug_reset(void); /* Codes for BANK_ACCOUNT_INFO */ #define BANK_SETTLE_CPU_TIME 0x1 /* Bank ledger(chit) rolled up to tasks. */ #define BANK_SECURE_ORIGINATOR_CHANGED 0x2 /* Secure Originator changed. */ +#define BANK_SETTLE_ENERGY 0x3 /* Bank ledger(energy field) rolled up to tasks. */ /* Codes for ATM_SUBAID_INFO */ #define ATM_MIN_CALLED 0x1 @@ -848,7 +922,8 @@ extern void kdebug_reset(void); #define ATM_VALUE_DIFF_MAILBOX 0x2 /* Kernel Debug Sub Classes for daemons (DBG_DAEMON) */ -#define DBG_DAEMON_COREDUET 0x1 +#define DBG_DAEMON_COREDUET 0x1 +#define DBG_DAEMON_POWERD 0x2 /* Subclasses for the user space allocator */ #define DBG_UMALLOC_EXTERNAL 0x1 @@ -893,6 +968,7 @@ extern void kdebug_reset(void); /* Kernel Debug Macros for specific daemons */ #define COREDUETDBG_CODE(code) DAEMONDBG_CODE(DBG_DAEMON_COREDUET, code) +#define POWERDDBG_CODE(code) DAEMONDBG_CODE(DBG_DAEMON_POWERD, code) /* * To use kdebug in the kernel: @@ -1295,18 +1371,33 @@ extern void kernel_debug_disable(void); struct proc; -extern boolean_t kdebug_debugid_enabled(uint32_t debugid); -extern uint32_t kdebug_commpage_state(void); -extern void kdebug_lookup_gen_events(long *dbg_parms, int dbg_namelen, void *dp, boolean_t lookup); -extern void kdbg_trace_data(struct proc *proc, long *arg_pid); +/* + * Returns false if the debugid is disabled by filters, and true if the + * debugid is allowed to be traced. A debugid may not be traced if the + * typefilter disables its class and subclass, it's outside a range + * check, or if it's not an allowed debugid in a value check. Trace + * system events bypass this check. + */ +boolean_t kdebug_debugid_enabled(uint32_t debugid); + +/* + * Returns true only if the debugid is explicitly enabled by filters. Returns + * false otherwise, including when no filters are active. + */ +boolean_t kdebug_debugid_explicitly_enabled(uint32_t debugid); -extern void kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *arg4); +uint32_t kdebug_commpage_state(void); +void kdebug_lookup_gen_events(long *dbg_parms, int dbg_namelen, void *dp, boolean_t lookup); +void kdbg_trace_data(struct proc *proc, long *arg_pid, long *arg_uniqueid); -extern void kdbg_dump_trace_to_file(const char *); -void kdebug_boot_trace(unsigned int n_events, char *filterdesc); -void kdebug_trace_start(unsigned int n_events, const char *filterdesc, boolean_t need_map); +void kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *arg4); + +void kdbg_dump_trace_to_file(const char *); +void kdebug_init(unsigned int n_events, char *filterdesc); +void kdebug_trace_start(unsigned int n_events, const char *filterdesc, boolean_t at_wake); +void kdebug_free_early_buf(void); struct task; -extern void kdbg_get_task_name(char*, int, struct task *task); +void kdbg_get_task_name(char*, int, struct task *task); boolean_t disable_wrap(uint32_t *old_slowcheck, uint32_t *old_flags); void enable_wrap(uint32_t old_slowcheck, boolean_t lostevents); void release_storage_unit(int cpu, uint32_t storage_unit); @@ -1626,9 +1717,11 @@ int kdbg_write_v3_chunk_to_fd(uint32_t tag, uint32_t sub_tag, uint64_t length, v #define VFS_LOOKUP (FSDBG_CODE(DBG_FSRW,36)) #define VFS_LOOKUP_DONE (FSDBG_CODE(DBG_FSRW,39)) +#if !CONFIG_EMBEDDED #if defined(XNU_KERNEL_PRIVATE) && (DEVELOPMENT || DEBUG) #define KDEBUG_MOJO_TRACE 1 #endif +#endif #endif /* __APPLE_API_PRIVATE */ #endif /* PRIVATE */ diff --git a/bsd/sys/kern_control.h b/bsd/sys/kern_control.h index ba5f6be37..8e1b514bd 100644 --- a/bsd/sys/kern_control.h +++ b/bsd/sys/kern_control.h @@ -36,6 +36,9 @@ #include +#include +#include +#include /* * Define Controller event subclass, and associated events. diff --git a/bsd/sys/kern_memorystatus.h b/bsd/sys/kern_memorystatus.h index b5850a7b9..52bce789c 100644 --- a/bsd/sys/kern_memorystatus.h +++ b/bsd/sys/kern_memorystatus.h @@ -33,6 +33,7 @@ #include #include #include +#include #define MEMORYSTATUS_ENTITLEMENT "com.apple.private.memorystatus" @@ -109,6 +110,10 @@ typedef struct memorystatus_kernel_stats { uint64_t compressions; uint64_t decompressions; uint64_t total_uncompressed_pages_in_compressor; + uint64_t zone_map_size; + uint64_t zone_map_capacity; + uint64_t largest_zone_size; + char largest_zone_name[MACH_ZONE_NAME_MAX_LEN]; } memorystatus_kernel_stats_t; /* @@ -172,20 +177,14 @@ typedef struct memorystatus_freeze_entry { #define kMemorystatusSupportsIdleExit 0x10 #define kMemorystatusDirty 0x20 -/* Cause */ -enum { - kMemorystatusKilled = 1, - kMemorystatusKilledHiwat, - kMemorystatusKilledVnodes, - kMemorystatusKilledVMPageShortage, - kMemorystatusKilledVMThrashing, - kMemorystatusKilledFCThrashing, - kMemorystatusKilledPerProcessLimit, - kMemorystatusKilledDiagnostic, - kMemorystatusKilledIdleExit -}; - -/* Jetsam exit reason definitions */ +/* + * Jetsam exit reason definitions - related to memorystatus + * + * When adding new exit reasons also update: + * JETSAM_REASON_MEMORYSTATUS_MAX + * kMemorystatusKilled... Cause enum + * memorystatus_kill_cause_name[] + */ #define JETSAM_REASON_INVALID 0 #define JETSAM_REASON_GENERIC 1 #define JETSAM_REASON_MEMORY_HIGHWATER 2 @@ -196,12 +195,28 @@ enum { #define JETSAM_REASON_MEMORY_PERPROCESSLIMIT 7 #define JETSAM_REASON_MEMORY_DIAGNOSTIC 8 #define JETSAM_REASON_MEMORY_IDLE_EXIT 9 -#define JETSAM_REASON_CPULIMIT 10 +#define JETSAM_REASON_ZONE_MAP_EXHAUSTION 10 + +#define JETSAM_REASON_MEMORYSTATUS_MAX JETSAM_REASON_ZONE_MAP_EXHAUSTION + +/* + * Jetsam exit reason definitions - not related to memorystatus + */ +#define JETSAM_REASON_CPULIMIT 100 -/* Temporary, to prevent the need for a linked submission of ReportCrash */ -/* Remove when has been integrated */ +/* Cause */ enum { - kMemorystatusKilledVM = kMemorystatusKilledVMPageShortage + kMemorystatusInvalid = JETSAM_REASON_INVALID, + kMemorystatusKilled = JETSAM_REASON_GENERIC, + kMemorystatusKilledHiwat = JETSAM_REASON_MEMORY_HIGHWATER, + kMemorystatusKilledVnodes = JETSAM_REASON_VNODE, + kMemorystatusKilledVMPageShortage = JETSAM_REASON_MEMORY_VMPAGESHORTAGE, + kMemorystatusKilledVMThrashing = JETSAM_REASON_MEMORY_VMTHRASHING, + kMemorystatusKilledFCThrashing = JETSAM_REASON_MEMORY_FCTHRASHING, + kMemorystatusKilledPerProcessLimit = JETSAM_REASON_MEMORY_PERPROCESSLIMIT, + kMemorystatusKilledDiagnostic = JETSAM_REASON_MEMORY_DIAGNOSTIC, + kMemorystatusKilledIdleExit = JETSAM_REASON_MEMORY_IDLE_EXIT, + kMemorystatusKilledZoneMapExhaustion = JETSAM_REASON_ZONE_MAP_EXHAUSTION }; /* Memorystatus control */ @@ -317,12 +332,6 @@ typedef struct memorystatus_memlimit_properties { * Non-fatal limit types are the * - high-water-mark limit * - * P_MEMSTAT_MEMLIMIT_BACKGROUND is translated in posix_spawn as - * the fatal system_wide task limit when active - * non-fatal inactive limit based on limit provided. - * This is necessary for backward compatibility until the - * the flag can be considered obsolete. - * * Processes that opt into dirty tracking are evaluated * based on clean vs dirty state. * dirty ==> active @@ -350,7 +359,7 @@ typedef struct memorystatus_memlimit_properties { #define P_MEMSTAT_FOREGROUND 0x00000100 #define P_MEMSTAT_DIAG_SUSPENDED 0x00000200 #define P_MEMSTAT_PRIOR_THAW 0x00000400 -#define P_MEMSTAT_MEMLIMIT_BACKGROUND 0x00000800 /* Task has a memory limit for when it's in the background. Used for a process' "high water mark".*/ +/* unused 0x00000800 */ #define P_MEMSTAT_INTERNAL 0x00001000 #define P_MEMSTAT_FATAL_MEMLIMIT 0x00002000 /* current fatal state of the process's memlimit */ #define P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL 0x00004000 /* if set, exceeding limit is fatal when the process is active */ @@ -365,7 +374,7 @@ extern void memorystatus_init_at_boot_snapshot(void); extern int memorystatus_add(proc_t p, boolean_t locked); extern int memorystatus_update(proc_t p, int priority, uint64_t user_data, boolean_t effective, boolean_t update_memlimit, int32_t memlimit_active, boolean_t memlimit_active_is_fatal, - int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal, boolean_t memlimit_background); + int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal); extern int memorystatus_remove(proc_t p, boolean_t locked); @@ -399,10 +408,10 @@ void proc_memstat_terminated(proc_t p, boolean_t set); boolean_t memorystatus_proc_is_dirty_unsafe(void *v); #endif /* CONFIG_MEMORYSTATUS */ -#if CONFIG_JETSAM - int memorystatus_get_pressure_status_kdp(void); +#if CONFIG_JETSAM + typedef enum memorystatus_policy { kPolicyDefault = 0x0, kPolicyMoreFree = 0x1, @@ -415,19 +424,19 @@ extern int memorystatus_jetsam_wakeup; extern unsigned int memorystatus_jetsam_running; boolean_t memorystatus_kill_on_VM_page_shortage(boolean_t async); -boolean_t memorystatus_kill_on_VM_thrashing(boolean_t async); boolean_t memorystatus_kill_on_FC_thrashing(boolean_t async); boolean_t memorystatus_kill_on_vnode_limit(void); void jetsam_on_ledger_cpulimit_exceeded(void); -void memorystatus_pages_update(unsigned int pages_avail); +#endif /* CONFIG_JETSAM */ -#else /* CONFIG_JETSAM */ +boolean_t memorystatus_kill_on_zone_map_exhaustion(pid_t pid); +boolean_t memorystatus_kill_on_VM_thrashing(boolean_t async); +void memorystatus_pages_update(unsigned int pages_avail); boolean_t memorystatus_idle_exit_from_VM(void); -#endif /* !CONFIG_JETSAM */ #ifdef CONFIG_FREEZE diff --git a/bsd/sys/kpi_mbuf.h b/bsd/sys/kpi_mbuf.h index 6b5693c00..d877f0974 100644 --- a/bsd/sys/kpi_mbuf.h +++ b/bsd/sys/kpi_mbuf.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -166,6 +166,7 @@ enum { #ifdef KERNEL_PRIVATE MBUF_CSUM_PARTIAL = 0x1000, /* 16-bit 1's complement sum */ MBUF_CSUM_REQ_SUM16 = MBUF_CSUM_PARTIAL, + MBUF_CSUM_REQ_ZERO_INVERT = 0x2000, #endif /* KERNEL_PRIVATE */ MBUF_CSUM_REQ_IP = 0x0001, MBUF_CSUM_REQ_TCP = 0x0002, @@ -283,7 +284,7 @@ __BEGIN_DECLS than one mbuf. In addition, data that is virtually contiguous might not be represented by physically contiguous pages; see further comments in mbuf_data_to_physical. Use mbuf_len to - determine the lenght of data available in this mbuf. If a data + determine the length of data available in this mbuf. If a data structure you want to access stradles two mbufs in a chain, either use mbuf_pullup to get the data contiguous in one mbuf or copy the pieces of data from each mbuf in to a contiguous @@ -953,8 +954,7 @@ extern errno_t mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src); /*! @function mbuf_pkthdr_len @discussion Returns the length as reported by the packet header. - @param mbuf The mbuf containing the packet header with the length to - be changed. + @param mbuf The mbuf containing the packet header @result The length, in bytes, of the packet. */ extern size_t mbuf_pkthdr_len(const mbuf_t mbuf); diff --git a/bsd/sys/kpi_socket.h b/bsd/sys/kpi_socket.h index 6045af624..837611b5c 100644 --- a/bsd/sys/kpi_socket.h +++ b/bsd/sys/kpi_socket.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -105,8 +105,17 @@ typedef void (*sock_evupcall)(socket_t so, void *cookie, u_int32_t event); socket for tracking the connection. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t sock_accept_internal(socket_t so, struct sockaddr *from, int fromlen, + int flags, sock_upcall callback, void *cookie, socket_t *new_so); + +#define sock_accept(so, from, fromlen, flags, callback, cookie, new_so) \ + sock_accept_internal((so), (from), (fromlen), (flags), (callback), \ + (cookie), (new_so)) +#else extern errno_t sock_accept(socket_t so, struct sockaddr *from, int fromlen, int flags, sock_upcall callback, void *cookie, socket_t *new_so); +#endif /* KERNEL_PRIVATE */ /*! @function sock_bind @@ -364,8 +373,17 @@ extern errno_t sock_shutdown(socket_t so, int how); @param new_so Upon success, a reference to the new socket. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t sock_socket_internal(int domain, int type, int protocol, + sock_upcall callback, void *cookie, socket_t *new_so); + +#define sock_socket(domain, type, protocol, callback, cookie, new_so) \ + sock_socket_internal((domain), (type), (protocol), \ + (callback), (cookie), (new_so)) +#else extern errno_t sock_socket(int domain, int type, int protocol, sock_upcall callback, void *cookie, socket_t *new_so); +#endif /* KERNEL_PRIVATE */ /*! @function sock_close @@ -537,6 +555,17 @@ extern errno_t sock_setupcall(socket_t sock, sock_upcall callback, extern errno_t sock_setupcalls(socket_t sock, sock_upcall read_callback, void *read_context, sock_upcall write_callback, void *write_context); +/* + @function sock_setupcalls_locked + @discussion The locked version of sock_setupcalls + @param locked: When sets, indicates that the callbacks expect to be + on a locked socket. Thus, no unlock is done prior to + calling the callback. + */ +extern void sock_setupcalls_locked(socket_t sock, + sock_upcall rcallback, void *rcontext, + sock_upcall wcallback, void *wcontext, int locked); + /* @function sock_catchevents @discussion Set the notifier function to be called when an event @@ -551,6 +580,11 @@ extern errno_t sock_setupcalls(socket_t sock, sock_upcall read_callback, */ extern errno_t sock_catchevents(socket_t sock, sock_evupcall event_callback, void *event_context, u_int32_t event_mask); + +extern void sock_catchevents_locked(socket_t sock, sock_evupcall ecallback, + void *econtext, u_int32_t emask); + + /* @function sock_iskernel @discussion Returns true if the socket was created by the kernel or diff --git a/bsd/sys/kpi_socketfilter.h b/bsd/sys/kpi_socketfilter.h index bb82c5439..d3ac71b96 100644 --- a/bsd/sys/kpi_socketfilter.h +++ b/bsd/sys/kpi_socketfilter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -569,8 +569,16 @@ struct sflt_filter { @param protocol The protocol these filters will be attached to. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t sflt_register_internal(const struct sflt_filter *filter, + int domain, int type, int protocol); + +#define sflt_register(filter, domain, type, protocol) \ + sflt_register_internal((filter), (domain), (type), (protocol)) +#else extern errno_t sflt_register(const struct sflt_filter *filter, int domain, int type, int protocol); +#endif /* KERNEL_PRIVATE */ /*! @function sflt_unregister diff --git a/bsd/sys/ktrace.h b/bsd/sys/ktrace.h index c67c9f6d1..735dc82c3 100644 --- a/bsd/sys/ktrace.h +++ b/bsd/sys/ktrace.h @@ -43,7 +43,11 @@ enum ktrace_state { KTRACE_STATE_BG }; -extern lck_mtx_t *ktrace_lock; +void ktrace_lock(void); +void ktrace_unlock(void); +void ktrace_assert_lock_held(void); +void ktrace_start_single_threaded(void); +void ktrace_end_single_threaded(void); /* * Subsystems that use ktrace to manage ownership. These values are passed as @@ -125,7 +129,7 @@ bool ktrace_background_active(void); * These functions exist for the transition for kperf to allow blessing other * processes. They should not be used by other clients. */ -extern boolean_t ktrace_keep_ownership_on_reset; +extern bool ktrace_keep_ownership_on_reset; extern int ktrace_root_set_owner_allowed; int ktrace_set_owning_pid(int pid); diff --git a/bsd/sys/lctx.h b/bsd/sys/lctx.h index 554176c4a..e5205c632 100644 --- a/bsd/sys/lctx.h +++ b/bsd/sys/lctx.h @@ -5,6 +5,8 @@ #define _SYS_LCTX_H_ #ifndef KERNEL +#include /* errno, ENOSYS */ +#include /* pid_t */ static __inline pid_t getlcid(pid_t pid) { diff --git a/bsd/sys/linker_set.h b/bsd/sys/linker_set.h index 1aea00848..8fd29dbb9 100644 --- a/bsd/sys/linker_set.h +++ b/bsd/sys/linker_set.h @@ -88,6 +88,12 @@ * * void const * __set_SET_sym_SYM __attribute__((section("__DATA,SET"))) = & SYM */ + +/* Wrap entries in a type that can be blacklisted from KASAN */ +struct linker_set_entry { + void *ptr; +} __attribute__((packed)); + #ifdef __LS_VA_STRINGIFY__ # undef __LS_VA_STRINGIFY__ #endif @@ -97,8 +103,8 @@ #define __LS_VA_STRINGIFY(_x...) #_x #define __LS_VA_STRCONCAT(_x,_y) __LS_VA_STRINGIFY(_x,_y) #define __LINKER_MAKE_SET(_set, _sym) \ - /*__unused*/ /*static*/ void const * /*const*/ __set_##_set##_sym_##_sym \ - __attribute__ ((section(__LS_VA_STRCONCAT(__DATA,_set)),used)) = (void *)&_sym + /*__unused*/ /*static*/ const struct linker_set_entry /*const*/ __set_##_set##_sym_##_sym \ + __attribute__ ((section(__LS_VA_STRCONCAT(__DATA,_set)),used)) = { (void *)&_sym } /* the line above is very fragile - if your compiler breaks linker sets, just play around with "static", "const", "used" etc. :-) */ diff --git a/bsd/sys/malloc.h b/bsd/sys/malloc.h index 27a8c2d04..0dd7117f5 100644 --- a/bsd/sys/malloc.h +++ b/bsd/sys/malloc.h @@ -217,8 +217,11 @@ #define M_FD_VN_DATA 122 /* Per fd vnode data */ #define M_FD_DIRBUF 123 /* Directory entries' buffer */ #define M_NETAGENT 124 /* Network Agents */ +#define M_EVENTHANDLER 125 /* Eventhandler */ +#define M_LLTABLE 126 /* Link layer table */ +#define M_NWKWQ 127 /* Network work queue */ -#define M_LAST 125 /* Must be last type + 1 */ +#define M_LAST 128 /* Must be last type + 1 */ #else /* BSD_KERNEL_PRIVATE */ @@ -263,22 +266,23 @@ extern struct kmemstats kmemstats[]; #include -#define MALLOC(space, cast, size, type, flags) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ +#define MALLOC(space, cast, size, type, flags) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ (space) = (cast)__MALLOC(size, type, flags, &site); }) -#define REALLOC(space, cast, addr, size, type, flags) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ + +#define REALLOC(space, cast, addr, size, type, flags) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ (space) = (cast)__REALLOC(addr, size, type, flags, &site); }) -#define _MALLOC(size, type, flags) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ +#define _MALLOC(size, type, flags) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ __MALLOC(size, type, flags, &site); }) -#define _REALLOC(addr, size, type, flags) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ +#define _REALLOC(addr, size, type, flags) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ __REALLOC(addr, size, type, flags, &site); }) -#define _MALLOC_ZONE(size, type, flags) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ +#define _MALLOC_ZONE(size, type, flags) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ __MALLOC_ZONE(size, type, flags, &site); }) #define FREE(addr, type) \ @@ -294,7 +298,7 @@ extern void *__MALLOC( size_t size, int type, int flags, - vm_allocation_site_t *site); + vm_allocation_site_t *site) __attribute__((alloc_size(1))); extern void _FREE( void *addr, @@ -305,7 +309,7 @@ extern void *__REALLOC( size_t size, int type, int flags, - vm_allocation_site_t *site); + vm_allocation_site_t *site) __attribute__((alloc_size(2))); extern void *__MALLOC_ZONE( size_t size, diff --git a/bsd/sys/mbuf.h b/bsd/sys/mbuf.h index 6b585f2b9..828f9f44d 100644 --- a/bsd/sys/mbuf.h +++ b/bsd/sys/mbuf.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999-2015 Apple Inc. All rights reserved. + * Copyright (c) 1999-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -81,6 +81,9 @@ #include #include +#include /* u_int32_t */ +#include /* u_int64_t */ +#include /* u_short */ #ifdef XNU_KERNEL_PRIVATE @@ -127,9 +130,11 @@ /* * Macros for type conversion * mtod(m,t) - convert mbuf pointer to data pointer of correct type + * mtodo(m, o) -- Same as above but with offset 'o' into data. * dtom(x) - convert data pointer within mbuf to mbuf pointer (XXX) */ #define mtod(m, t) ((t)m_mtod(m)) +#define mtodo(m, o) ((void *)(mtod(m, uint8_t *) + (o))) #define dtom(x) m_dtom(x) /* header at beginning of each mbuf: */ @@ -140,7 +145,18 @@ struct m_hdr { int32_t mh_len; /* amount of data in this mbuf */ u_int16_t mh_type; /* type of data in this mbuf */ u_int16_t mh_flags; /* flags; see below */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* This is needed because of how _MLEN is defined and used. Ideally, _MLEN + * should be defined using the offsetof(struct mbuf, M_dat), since there is + * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler + * may (and does in the armv7k case) insert padding between m_hdr and M_dat in + * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced + * in the definition of mbuf. + */ +} __attribute__((aligned(8))); +#else }; +#endif /* * Packet tag structure (see below for details). @@ -199,9 +215,6 @@ struct pf_mtag { u_int16_t pftag_rtableid; /* alternate routing table id */ u_int16_t pftag_tag; u_int16_t pftag_routed; -#if PF_ALTQ - u_int32_t pftag_qid; -#endif /* PF_ALTQ */ #if PF_ECN void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */ #endif /* PF_ECN */ @@ -241,10 +254,12 @@ struct tcp_pktinfo { struct mptcp_pktinfo { u_int64_t mtpi_dsn; /* MPTCP Data Sequence Number */ u_int32_t mtpi_rel_seq; /* Relative Seq Number */ - u_int32_t mtpi_length; /* Length of mapping */ + u_int16_t mtpi_length; /* Length of mapping */ + u_int16_t mtpi_csum; #define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn #define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq #define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length +#define mp_csum proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum }; /* @@ -390,9 +405,6 @@ struct pkthdr { #define bufstatus_if _pkt_bsr.if_data #define bufstatus_sndbuf _pkt_bsr.sndbuf_data }; -#if MEASURE_BW - u_int64_t pkt_bwseq; /* sequence # */ -#endif /* MEASURE_BW */ u_int64_t pkt_timestamp; /* enqueue time */ /* @@ -403,7 +415,8 @@ struct pkthdr { /* * Module private scratch space (32-bit aligned), currently 16-bytes * large. Anything stored here is not guaranteed to survive across - * modules. + * modules. The AQM layer (outbound) uses all 16-bytes for both + * packet scheduling and flow advisory information. */ struct { union { @@ -419,6 +432,11 @@ struct pkthdr { u_int64_t __mpriv64[2]; } __mpriv_u; } pkt_mpriv __attribute__((aligned(4))); +#define pkt_mpriv_hash pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32 +#define pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32 +#define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32 +#define pkt_mpriv_fidx pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32 + u_int32_t redzone; /* red zone */ u_int32_t pkt_compl_callbacks; /* Packet completion callbacks */ }; @@ -480,11 +498,12 @@ struct pkthdr { #define PKTF_TCP_REXMT 0x200000 /* packet is TCP retransmission */ #define PKTF_REASSEMBLED 0x400000 /* Packet was reassembled */ #define PKTF_TX_COMPL_TS_REQ 0x800000 /* tx completion timestamp requested */ -#define PKTF_DRV_TS_VALID 0x1000000 /* driver timestamp is valid */ +#define PKTF_TS_VALID 0x1000000 /* pkt timestamp is valid */ #define PKTF_DRIVER_MTAG 0x2000000 /* driver mbuf tags fields inited */ #define PKTF_NEW_FLOW 0x4000000 /* Data from a new flow */ #define PKTF_START_SEQ 0x8000000 /* valid start sequence */ #define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */ +#define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */ /* flags related to flow control/advisory and identification */ #define PKTF_FLOW_MASK \ @@ -591,6 +610,7 @@ struct mbuf { #define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */ #define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */ #define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */ +#define CSUM_ZERO_INVERT 0x2000 /* invert 0 to -0 (0xffff) */ #define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP) #define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */ @@ -599,7 +619,7 @@ struct mbuf { #define CSUM_TX_FLAGS \ (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \ - CSUM_DATA_VALID | CSUM_PARTIAL) + CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT) #define CSUM_RX_FLAGS \ (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \ @@ -1454,10 +1474,11 @@ __private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *); } while (0) __private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t, - u_int32_t, u_int32_t); + u_int32_t, u_int32_t, u_int32_t); __private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t); -__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *, m_ext_free_func_t, caddr_t); +__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *, + m_ext_free_func_t, caddr_t); __private_extern__ struct ext_ref *m_get_rfa(struct mbuf *); __private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *); __private_extern__ caddr_t m_get_ext_arg(struct mbuf *); diff --git a/bsd/sys/mcache.h b/bsd/sys/mcache.h index 906363384..9bf6ed529 100644 --- a/bsd/sys/mcache.h +++ b/bsd/sys/mcache.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2015 Apple Inc. All rights reserved. + * Copyright (c) 2006-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -50,11 +50,11 @@ extern "C" { #endif /* - * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG build. + * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG/DEVELOPMENT build. */ #define VERIFY(EX) \ ((void)(__probable((EX)) || assfail(#EX, __FILE__, __LINE__))) -#if DEBUG +#if (DEBUG || DEVELOPMENT) #define ASSERT(EX) VERIFY(EX) #else #define ASSERT(EX) ((void)0) @@ -147,6 +147,9 @@ extern "C" { #define atomic_bitset_32(a, n) \ atomic_or_32(a, n) +#define atomic_bitset_32_ov(a, n) \ + atomic_or_32_ov(a, n) + #define atomic_and_8_ov(a, n) \ ((u_int8_t) OSBitAndAtomic8(n, (volatile UInt8 *)a)) @@ -198,6 +201,11 @@ extern "C" { (((uintptr_t)(x)) & ~((uintptr_t)(align) - 1)) #endif /* P2ROUNDDOWN */ +#ifndef P2ALIGN +#define P2ALIGN(x, align) \ + ((uintptr_t)(x) & -((uintptr_t)(align))) +#endif /* P2ALIGN */ + #define MCACHE_FREE_PATTERN 0xdeadbeefdeadbeefULL #define MCACHE_UNINITIALIZED_PATTERN 0xbaddcafebaddcafeULL @@ -382,6 +390,7 @@ __private_extern__ unsigned int mcache_alloc_ext(mcache_t *, mcache_obj_t **, unsigned int, int); __private_extern__ void mcache_free_ext(mcache_t *, mcache_obj_t *); __private_extern__ void mcache_reap(void); +__private_extern__ void mcache_reap_now(mcache_t *, boolean_t); __private_extern__ boolean_t mcache_purge_cache(mcache_t *, boolean_t); __private_extern__ void mcache_waiter_inc(mcache_t *); __private_extern__ void mcache_waiter_dec(mcache_t *); diff --git a/bsd/sys/mman.h b/bsd/sys/mman.h index 06c76abf5..8aba6441f 100644 --- a/bsd/sys/mman.h +++ b/bsd/sys/mman.h @@ -202,6 +202,9 @@ #define MINCORE_MODIFIED 0x4 /* Page has been modified by us */ #define MINCORE_REFERENCED_OTHER 0x8 /* Page has been referenced */ #define MINCORE_MODIFIED_OTHER 0x10 /* Page has been modified */ +#define MINCORE_PAGED_OUT 0x20 /* Page has been paged out */ +#define MINCORE_COPIED 0x40 /* Page has been copied */ +#define MINCORE_ANONYMOUS 0x80 /* Page belongs to an anonymous object */ #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ diff --git a/bsd/sys/monotonic.h b/bsd/sys/monotonic.h new file mode 100644 index 000000000..883b6a0ad --- /dev/null +++ b/bsd/sys/monotonic.h @@ -0,0 +1,149 @@ +#ifndef SYS_MONOTONIC_H +#define SYS_MONOTONIC_H + +#include +#include +#include +#include + +__BEGIN_DECLS + +/* + * XXX These declarations are subject to change at any time. + */ + +struct monotonic_config { + uint64_t event; + uint64_t allowed_ctr_mask; +}; + +union monotonic_ctl_add { + struct { + struct monotonic_config config; + } in; + + struct { + uint32_t ctr; + } out; +}; + +union monotonic_ctl_enable { + struct { + bool enable; + } in; +}; + +union monotonic_ctl_counts { + struct { + uint64_t ctr_mask; + } in; + + struct { + uint64_t counts[1]; + } out; +}; + +#define MT_IOC(x) _IO('m', (x)) + +/* + * FIXME + * + * - Consider a separate IOC for disable -- to avoid the copyin to determine which way to set it. + * + * - Maybe IOC_COUNTS should just return all the enable counters' counts. + */ +enum monotonic_ioc { + MT_IOC_RESET = MT_IOC(0), + MT_IOC_ADD = MT_IOC(1), + MT_IOC_ENABLE = MT_IOC(2), + MT_IOC_COUNTS = MT_IOC(3), +}; + +#undef MT_IOC + +#if XNU_KERNEL_PRIVATE + +#include +#include +#include +#include + +#ifdef MT_CORE_INSTRS +#define COUNTS_INSTRS __counts[MT_CORE_INSTRS] +#else /* defined(MT_CORE_INSTRS) */ +#define COUNTS_INSTRS 0 +#endif /* !defined(MT_CORE_INSTRS) */ + +/* + * MT_KDBG_TMP* macros are meant for temporary (i.e. not checked-in) + * performance investigations. + */ + +/* + * Record the current CPU counters. + * + * Preemption must be disabled. + */ +#define MT_KDBG_TMPCPU_EVT(CODE) \ + KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_TMPCPU, CODE) + +#define MT_KDBG_TMPCPU_(CODE, FUNC) \ + do { \ + if (kdebug_enable && \ + kdebug_debugid_enabled(MT_KDBG_TMPCPU_EVT(CODE))) { \ + uint64_t __counts[MT_CORE_NFIXED]; \ + mt_fixed_counts(__counts); \ + KDBG(MT_KDBG_TMPCPU_EVT(CODE) | (FUNC), COUNTS_INSTRS, \ + __counts[MT_CORE_CYCLES]); \ + } \ + } while (0) + +#define MT_KDBG_TMPCPU(CODE) MT_KDBG_TMPCPU_(CODE, DBG_FUNC_NONE) +#define MT_KDBG_TMPCPU_START(CODE) MT_KDBG_TMPCPU_(CODE, DBG_FUNC_START) +#define MT_KDBG_TMPCPU_END(CODE) MT_KDBG_TMPCPU_(CODE, DBG_FUNC_END) + +/* + * Record the current thread counters. + * + * Interrupts must be disabled. + */ +#define MT_KDBG_TMPTH_EVT(CODE) \ + KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_TMPTH, CODE) + +#define MT_KDBG_TMPTH_(CODE, FUNC) \ + do { \ + if (kdebug_enable && \ + kdebug_debugid_enabled(MT_KDBG_TMPTH_EVT(CODE))) { \ + uint64_t __counts[MT_CORE_NFIXED]; \ + mt_cur_thread_fixed_counts(__counts); \ + KDBG(MT_KDBG_TMPTH_EVT(CODE) | (FUNC), COUNTS_INSTRS, \ + __counts[MT_CORE_CYCLES]); \ + } \ + } while (0) + +#define MT_KDBG_TMPTH(CODE) MT_KDBG_TMPTH_(CODE, DBG_FUNC_NONE) +#define MT_KDBG_TMPTH_START(CODE) MT_KDBG_TMPTH_(CODE, DBG_FUNC_START) +#define MT_KDBG_TMPTH_END(CODE) MT_KDBG_TMPTH_(CODE, DBG_FUNC_END) + +/* maybe provider, bank, group, set, unit, pmu */ + +struct monotonic_dev { + const char *mtd_name; + int (*mtd_init)(void); + int (*mtd_add)(struct monotonic_config *config, uint32_t *ctr_out); + void (*mtd_reset)(void); + void (*mtd_enable)(bool enable); + int (*mtd_read)(uint64_t ctr_mask, uint64_t *counts_out); +}; + +extern const struct monotonic_dev monotonic_devs[]; + +extern lck_grp_t *mt_lock_grp; + +int mt_dev_init(void); + +#endif /* XNU_KERNEL_PRIVATE */ + +__END_DECLS + +#endif /* !defined(SYS_MONOTONIC_H) */ diff --git a/bsd/sys/mount.h b/bsd/sys/mount.h index 70db71af6..8ee2cc256 100644 --- a/bsd/sys/mount.h +++ b/bsd/sys/mount.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -320,7 +320,8 @@ struct vfs_attr { #define MNT_NOUSERXATTR 0x01000000 /* Don't allow user extended attributes */ #define MNT_DEFWRITE 0x02000000 /* filesystem should defer writes */ #define MNT_MULTILABEL 0x04000000 /* MAC support for individual labels */ -#define MNT_NOATIME 0x10000000 /* disable update of file access time */ +#define MNT_NOATIME 0x10000000 /* disable update of file access time */ +#define MNT_SNAPSHOT 0x40000000 /* The mount is a snapshot */ #ifdef BSD_KERNEL_PRIVATE /* #define MNT_IMGSRC_BY_INDEX 0x20000000 see sys/imgsrc.h */ #endif /* BSD_KERNEL_PRIVATE */ @@ -340,7 +341,7 @@ struct vfs_attr { MNT_ROOTFS | MNT_DOVOLFS | MNT_DONTBROWSE | \ MNT_IGNORE_OWNERSHIP | MNT_AUTOMOUNTED | MNT_JOURNALED | \ MNT_NOUSERXATTR | MNT_DEFWRITE | MNT_MULTILABEL | \ - MNT_NOATIME | MNT_CPROTECT) + MNT_NOATIME | MNT_SNAPSHOT | MNT_CPROTECT) /* * External filesystem command modifier flags. * Unmount can use the MNT_FORCE flag. @@ -760,19 +761,16 @@ struct fs_snapshot_mount_args { }; #define VFSIOC_MOUNT_SNAPSHOT _IOW('V', 1, struct fs_snapshot_mount_args) -#define VFSCTL_MOUNT_SNAPSHOT IOCBASECMD(VFSIOC_MOUNT_SNAPSHOT) struct fs_snapshot_revert_args { struct componentname *sr_cnp; }; #define VFSIOC_REVERT_SNAPSHOT _IOW('V', 2, struct fs_snapshot_revert_args) -#define VFSCTL_REVERT_SNAPSHOT IOCBASECMD(VFSIOC_REVERT_SNAPSHOT) struct fs_snapshot_root_args { struct componentname *sr_cnp; }; #define VFSIOC_ROOT_SNAPSHOT _IOW('V', 3, struct fs_snapshot_root_args) -#define VFSCTL_ROOT_SNAPSHOT IOCBASECMD(VFSIOC_ROOT_SNAPSHOT) #endif /* KERNEL */ @@ -1110,7 +1108,7 @@ void vfs_setfsprivate(mount_t mp, void *mntdata); @abstract Get information about filesystem status. @discussion Each filesystem has a struct vfsstatfs associated with it which is updated as events occur; this function returns a pointer to it. Note that the data in the structure will continue to change over time and also that it may - be quite stale of vfs_update_vfsstat has not been called recently. + be quite stale if vfs_update_vfsstat has not been called recently. @param mp Mount for which to get vfsstatfs pointer. @return Pointer to vfsstatfs. */ @@ -1262,6 +1260,13 @@ void vfs_event_signal(fsid_t *fsid, u_int32_t event, intptr_t data); */ void vfs_event_init(void); /* XXX We should not export this */ +/*! + @function vfs_set_root_unmount_cleanly + @abstract This function should be called by the root file system + when it is being mounted if the file system state is consistent. +*/ +void vfs_set_root_unmounted_cleanly(void); + #ifdef KERNEL_PRIVATE int vfs_getbyid(fsid_t *fsid, ino64_t ino, vnode_t *vpp, vfs_context_t ctx); int vfs_getattr(mount_t mp, struct vfs_attr *vfa, vfs_context_t ctx); @@ -1381,10 +1386,14 @@ int getfsstat(struct statfs *, int, int) __DARWIN_INODE64(getfsstat); int getfsstat64(struct statfs64 *, int, int) __OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_5,__MAC_10_6,__IPHONE_NA,__IPHONE_NA); #endif /* !__DARWIN_ONLY_64_BIT_INO_T */ int getmntinfo(struct statfs **, int) __DARWIN_INODE64(getmntinfo); +int getmntinfo_r_np(struct statfs **, int) __DARWIN_INODE64(getmntinfo_r_np) + __OSX_AVAILABLE(10.13) __IOS_AVAILABLE(11.0) + __TVOS_AVAILABLE(11.0) __WATCHOS_AVAILABLE(4.0); #if !__DARWIN_ONLY_64_BIT_INO_T int getmntinfo64(struct statfs64 **, int) __OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_5,__MAC_10_6,__IPHONE_NA,__IPHONE_NA); #endif /* !__DARWIN_ONLY_64_BIT_INO_T */ int mount(const char *, const char *, int, void *); +int fmount(const char *, int, int, void *) __OSX_AVAILABLE(10.13) __IOS_AVAILABLE(11.0) __TVOS_AVAILABLE(11.0) __WATCHOS_AVAILABLE(4.0); int statfs(const char *, struct statfs *) __DARWIN_INODE64(statfs); #if !__DARWIN_ONLY_64_BIT_INO_T int statfs64(const char *, struct statfs64 *) __OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_5,__MAC_10_6,__IPHONE_NA,__IPHONE_NA); diff --git a/bsd/sys/mount_internal.h b/bsd/sys/mount_internal.h index 6393fd712..10ed1f50b 100644 --- a/bsd/sys/mount_internal.h +++ b/bsd/sys/mount_internal.h @@ -196,6 +196,8 @@ struct mount { uint32_t mnt_iobufinuse; + void *mnt_disk_conditioner_info; + lck_mtx_t mnt_iter_lock; /* mutex that protects iteration of vnodes */ }; diff --git a/bsd/sys/munge.h b/bsd/sys/munge.h index 47f07923c..b7e55762e 100644 --- a/bsd/sys/munge.h +++ b/bsd/sys/munge.h @@ -71,6 +71,57 @@ * of uu_arg[] and work our way back to the beginning of the array. */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +int munge_w(const void *regs, void *args); +int munge_ww(const void *regs, void *args); +int munge_www(const void *regs, void *args); +int munge_wwww(const void *regs, void *args); +int munge_wwwww(const void *regs, void *args); +int munge_wwwwww(const void *regs, void *args); +int munge_wwwwwww(const void *regs, void *args); +int munge_wwwwwwww(const void *regs, void *args); +int munge_wl(const void *regs, void *args); +int munge_wwl(const void *regs, void *args); +int munge_wwlw(const void *regs, void *args); +int munge_wwlll(const void *regs, void *args); +int munge_wwllww(const void *regs, void *args); +int munge_wlw(const void *regs, void *args); +int munge_wlww(const void *regs, void *args); +int munge_wlwwwl(const void *regs, void *args); +int munge_wlwwwll(const void *regs, void *args); +int munge_wlwwwllw(const void *regs, void *args); +int munge_wlwwlwlw(const void *regs, void *args); +int munge_wll(const void *regs, void *args); +int munge_wllww(const void *regs, void *args); +int munge_wlll(const void *regs, void *args); +int munge_wllll(const void *regs, void *args); +int munge_wllwwll(const void *regs, void *args); +int munge_wwwlw(const void *regs, void *args); +int munge_wwwlww(const void *regs, void *args); +int munge_wwwl(const void *regs, void *args); +int munge_wwwwlw(const void *regs, void *args); +int munge_wwwwl(const void *regs, void *args); +int munge_wwwwwl(const void *regs, void *args); +int munge_wwwwwlww(const void *regs, void *args); +int munge_wwwwwllw(const void *regs, void *args); +int munge_wwwwwlll(const void *regs, void *args); +int munge_wwwwwwl(const void *regs, void *args); +int munge_wwwwwwlw(const void *regs, void *args); +int munge_wwwwwwll(const void *regs, void *args); +int munge_wsw(const void *regs, void *args); +int munge_wws(const void *regs, void *args); +int munge_wwws(const void *regs, void *args); +int munge_wwwsw(const void *regs, void *args); +int munge_llllll(const void *regs, void *args); +int munge_l(const void *regs, void *args); +int munge_ll(const void *regs, void *args); +int munge_lw(const void *regs, void *args); +int munge_lwww(const void *regs, void *args); +int munge_lwwwwwww(const void *regs, void *args); +int munge_wwlww(const void *regs, void *args); +int munge_wwlwww(const void *regs, void *args); +int munge_wwlwwwl(const void *regs, void *args); +#else void munge_w(void *args); void munge_ww(void *args); void munge_www(void *args); @@ -116,7 +167,9 @@ void munge_l(void *args); void munge_ll(void *args); void munge_lw(void *args); void munge_lwww(void *args); +void munge_lwwwwwww(void *args); void munge_wwlww(void *args); void munge_wwlwww(void *args); void munge_wwlwwwl(void *args); +#endif /* __arm__ && (__BIGGEST_ALIGNMENT__ > 4) */ #endif /* __MUNGE_H__ */ diff --git a/bsd/sys/netport.h b/bsd/sys/netport.h index 74eba9efa..0095d9dda 100644 --- a/bsd/sys/netport.h +++ b/bsd/sys/netport.h @@ -32,6 +32,8 @@ #ifndef _SYS_NETPORT_H_ #define _SYS_NETPORT_H_ +#include <_types/_uint32_t.h> /* uint32_t */ + typedef uint32_t netaddr_t; /* diff --git a/bsd/sys/persona.h b/bsd/sys/persona.h index d0912055f..f1efd66d4 100644 --- a/bsd/sys/persona.h +++ b/bsd/sys/persona.h @@ -225,7 +225,7 @@ struct persona { #define persona_try_lock(persona) lck_mtx_try_lock(&(persona)->pna_lock) #define persona_lock_assert_held(persona) \ - lck_mtx_assert(&(persona)->pna_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(persona)->pna_lock, LCK_MTX_ASSERT_OWNED) #ifdef PERSONA_DEBUG static inline const char *persona_desc(struct persona *persona, int locked) diff --git a/bsd/sys/pgo.h b/bsd/sys/pgo.h index 167b212fa..fcd669b51 100644 --- a/bsd/sys/pgo.h +++ b/bsd/sys/pgo.h @@ -91,4 +91,9 @@ ssize_t grab_pgo_data( #endif +#ifdef XNU_KERNEL_PRIVATE +kern_return_t do_pgo_reset_counters(void); +#endif + + #endif diff --git a/bsd/sys/pipe.h b/bsd/sys/pipe.h index 3437710b2..09b25dbce 100644 --- a/bsd/sys/pipe.h +++ b/bsd/sys/pipe.h @@ -63,6 +63,8 @@ #include /* for TAILQ macros */ #include #include +#include +#include /* * Pipe buffer size, keep moderate in value, pipes take kva space. @@ -167,7 +169,7 @@ struct pipe { #define PIPE_LOCK(pipe) lck_mtx_lock(PIPE_MTX(pipe)) #define PIPE_UNLOCK(pipe) lck_mtx_unlock(PIPE_MTX(pipe)) -#define PIPE_LOCK_ASSERT(pipe, type) lck_mtx_assert(PIPE_MTX(pipe), (type)) +#define PIPE_LOCK_ASSERT(pipe, type) LCK_MTX_ASSERT(PIPE_MTX(pipe), (type)) __BEGIN_DECLS void pipeinit(void); diff --git a/bsd/sys/priv.h b/bsd/sys/priv.h index af9cd806c..58f568b4b 100644 --- a/bsd/sys/priv.h +++ b/bsd/sys/priv.h @@ -92,12 +92,13 @@ #define PRIV_SETPRIORITY_DARWIN_ROLE 1012 /* Allow setpriority(PRIO_DARWIN_ROLE) */ #define PRIV_PACKAGE_EXTENSIONS 1013 /* Push package extension list used by vn_path_package_check() */ #define PRIV_TRIM_ACTIVE_FILE 1014 /* Allow freeing space out from under an active file */ +#define PRIV_PROC_CPUMON_OVERRIDE 1015 /* Allow CPU usage monitor parameters less restrictive than default */ /* * Virtual memory privileges. */ #define PRIV_VM_PRESSURE 6000 /* Check VM pressure. */ -#define PRIV_VM_JETSAM 6001 /* Adjust jetsam configuration. */ +#define PRIV_VM_JETSAM 6001 /* Adjust jetsam configuration. */ #define PRIV_VM_FOOTPRINT_LIMIT 6002 /* Adjust physical footprint limit. */ /* @@ -112,6 +113,9 @@ #define PRIV_NET_PRIVILEGED_NECP_MATCH 10006 /* Privilege verified by Network Extension policies */ #define PRIV_NET_QOSMARKING_POLICY_OVERRIDE 10007 /* Privilege verified by Network Extension policies */ #define PRIV_NET_RESTRICTED_INTCOPROC 10008 /* Access to internal co-processor network interfaces */ + +#define PRIV_NET_PRIVILEGED_MULTIPATH 10009 /* Multipath usage */ +#define PRIV_NET_RESTRICTED_MULTIPATH_EXTENDED 10010 /* Extended multipath (more aggressive on cell) */ /* * IPv4 and IPv6 privileges. */ @@ -121,17 +125,28 @@ /* * VFS privileges */ -#define PRIV_VFS_OPEN_BY_ID 14000 /* Allow calling openbyid_np() */ +#define PRIV_VFS_OPEN_BY_ID 14000 /* Allow calling openbyid_np() */ #define PRIV_VFS_MOVE_DATA_EXTENTS 14001 /* Allow F_MOVEDATAEXTENTS fcntl */ #define PRIV_VFS_SNAPSHOT 14002 /* Allow create/rename/delete of snapshots */ #define PRIV_VFS_SNAPSHOT_REVERT 14003 /* Allow reverting filesystem to a previous snapshot */ +#define PRIV_APFS_EMBED_DRIVER 14100 /* Allow embedding an EFI driver into the APFS container */ +#define PRIV_APFS_FUSION_DEBUG 14101 /* Allow getting internal statistics and controlling the APFS fusion container */ #ifdef KERNEL /* * Privilege check interface. No flags are currently defined for the API. */ +#include #include + +/* + * flags for priv_check_cred + */ +#define PRIVCHECK_DEFAULT_UNPRIVILEGED_FLAG (1) /* Don't grant root privilege by default */ + +__BEGIN_DECLS int priv_check_cred(kauth_cred_t cred, int priv, int flags); +__END_DECLS #endif #endif /* !_SYS_PRIV_H_ */ diff --git a/bsd/sys/proc.h b/bsd/sys/proc.h index 279e9670c..7878ff644 100644 --- a/bsd/sys/proc.h +++ b/bsd/sys/proc.h @@ -76,6 +76,7 @@ #include #include #include +#include #ifdef KERNEL #include #include @@ -303,7 +304,11 @@ pid_t proc_pgrpid(proc_t p); #ifdef KERNEL_PRIVATE // mark a process as being allowed to call vfs_markdependency() void bsd_set_dependency_capable(task_t task); +#ifdef __arm__ +static inline int IS_64BIT_PROCESS(__unused proc_t p) { return 0; } +#else extern int IS_64BIT_PROCESS(proc_t); +#endif /* __arm__ */ extern int tsleep(void *chan, int pri, const char *wmesg, int timo); extern int msleep1(void *chan, lck_mtx_t *mtx, int pri, const char *wmesg, u_int64_t timo); @@ -388,7 +393,11 @@ __BEGIN_DECLS int pid_suspend(int pid); int pid_resume(int pid); - +#if defined(__arm__) || defined(__arm64__) +int pid_hibernate(int pid); +#endif /* defined(__arm__) || defined(__arm64__) */ +int pid_shutdown_sockets(int pid, int level); +int pid_shutdown_networking(int pid, int level); __END_DECLS #endif /* !KERNEL */ diff --git a/bsd/sys/proc_info.h b/bsd/sys/proc_info.h index 8f22d8007..01c7d8b0a 100644 --- a/bsd/sys/proc_info.h +++ b/bsd/sys/proc_info.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005-2016 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2005-2017 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -58,6 +58,7 @@ __BEGIN_DECLS #define PROC_UID_ONLY 4 #define PROC_RUID_ONLY 5 #define PROC_PPID_ONLY 6 +#define PROC_KDBG_ONLY 7 struct proc_bsdinfo { uint32_t pbi_flags; /* 64bit; emulated etc */ @@ -636,6 +637,19 @@ struct kqueue_info { uint32_t rfu_1; /* reserved */ }; +struct kqueue_dyninfo { + struct kqueue_info kqdi_info; + uint64_t kqdi_servicer; + uint64_t kqdi_owner; + uint32_t kqdi_sync_waiters; + uint8_t kqdi_sync_waiter_qos; + uint8_t kqdi_async_qos; + uint16_t kqdi_request_state; + uint8_t kqdi_events_qos; + uint8_t _kqdi_reserved0[7]; + uint64_t _kqdi_reserved1[4]; +}; + /* keep in sync with KQ_* in sys/eventvar.h */ #define PROC_KQUEUE_SELECT 0x01 #define PROC_KQUEUE_SLEEP 0x02 @@ -773,6 +787,12 @@ struct proc_fileportinfo { #define PROC_PIDEXITREASONBASICINFO 25 #define PROC_PIDEXITREASONBASICINFOSIZE (sizeof(struct proc_exitreasonbasicinfo)) +#define PROC_PIDLISTUPTRS 26 +#define PROC_PIDLISTUPTRS_SIZE (sizeof(uint64_t)) + +#define PROC_PIDLISTDYNKQUEUES 27 +#define PROC_PIDLISTDYNKQUEUES_SIZE (sizeof(kqueue_id_t)) + #endif /* Flavors for proc_pidfdinfo */ @@ -805,6 +825,7 @@ struct proc_fileportinfo { #define PROC_PIDFDKQUEUE_EXTINFO 9 #define PROC_PIDFDKQUEUE_EXTINFO_SIZE (sizeof(struct kevent_extinfo)) #define PROC_PIDFDKQUEUE_KNOTES_MAX (1024 * 128) +#define PROC_PIDDYNKQUEUES_MAX (1024 * 128) #endif /* PRIVATE */ @@ -883,19 +904,28 @@ struct proc_fileportinfo { #define PROC_FGHW_VOUCHER_ERROR 98 /* error in voucher / originator callout */ #define PROC_FGHW_ERROR 99 /* syscall parameter/permissions error */ +/* flavors for proc_piddynkqueueinfo */ +#ifdef PRIVATE +#define PROC_PIDDYNKQUEUE_INFO 0 +#define PROC_PIDDYNKQUEUE_INFO_SIZE (sizeof(struct kqueue_dyninfo)) +#define PROC_PIDDYNKQUEUE_EXTINFO 1 +#define PROC_PIDDYNKQUEUE_EXTINFO_SIZE (sizeof(struct kevent_extinfo)) +#endif + /* __proc_info() call numbers */ -#define PROC_INFO_CALL_LISTPIDS 0x1 -#define PROC_INFO_CALL_PIDINFO 0x2 -#define PROC_INFO_CALL_PIDFDINFO 0x3 -#define PROC_INFO_CALL_KERNMSGBUF 0x4 -#define PROC_INFO_CALL_SETCONTROL 0x5 -#define PROC_INFO_CALL_PIDFILEPORTINFO 0x6 -#define PROC_INFO_CALL_TERMINATE 0x7 -#define PROC_INFO_CALL_DIRTYCONTROL 0x8 -#define PROC_INFO_CALL_PIDRUSAGE 0x9 +#define PROC_INFO_CALL_LISTPIDS 0x1 +#define PROC_INFO_CALL_PIDINFO 0x2 +#define PROC_INFO_CALL_PIDFDINFO 0x3 +#define PROC_INFO_CALL_KERNMSGBUF 0x4 +#define PROC_INFO_CALL_SETCONTROL 0x5 +#define PROC_INFO_CALL_PIDFILEPORTINFO 0x6 +#define PROC_INFO_CALL_TERMINATE 0x7 +#define PROC_INFO_CALL_DIRTYCONTROL 0x8 +#define PROC_INFO_CALL_PIDRUSAGE 0x9 #define PROC_INFO_CALL_PIDORIGINATORINFO 0xa -#define PROC_INFO_CALL_LISTCOALITIONS 0xb -#define PROC_INFO_CALL_CANUSEFGHW 0xc +#define PROC_INFO_CALL_LISTCOALITIONS 0xb +#define PROC_INFO_CALL_CANUSEFGHW 0xc +#define PROC_INFO_CALL_PIDDYNKQUEUEINFO 0xd #endif /* PRIVATE */ @@ -921,6 +951,10 @@ extern int pid_kqueue_extinfo(proc_t, struct kqueue * kq, user_addr_t buffer, uint32_t buffersize, int32_t * retval); extern int pid_kqueue_udatainfo(proc_t p, struct kqueue *kq, uint64_t *buf, uint32_t bufsize); +extern int pid_kqueue_listdynamickqueues(proc_t p, user_addr_t ubuf, + uint32_t bufsize, int32_t *retval); +extern int pid_dynamickqueue_extinfo(proc_t p, kqueue_id_t kq_id, + user_addr_t ubuf, uint32_t bufsize, int32_t *retval); extern int fill_procworkqueue(proc_t, struct proc_workqueueinfo *); extern boolean_t workqueue_get_pwq_exceeded(void *v, boolean_t *exceeded_total, boolean_t *exceeded_constrained); diff --git a/bsd/sys/proc_internal.h b/bsd/sys/proc_internal.h index a6bfe6bc3..b6d40d6ee 100644 --- a/bsd/sys/proc_internal.h +++ b/bsd/sys/proc_internal.h @@ -98,7 +98,6 @@ __END_DECLS * PFDL = Process File Desc Lock * PUCL = Process User Credentials Lock * PSL = Process Spin Lock - * PPL = Parent Process Lock (planed for later usage) * LL = List Lock * SL = Session Lock */ @@ -255,6 +254,7 @@ struct proc { pid_t p_oppid; /* Save parent pid during ptrace. XXX */ u_int p_xstat; /* Exit status for wait; also stop signal. */ + uint8_t p_xhighbits; /* Stores the top byte of exit status to avoid truncation*/ #ifdef _PROC_HAS_SCHEDINFO_ /* may need cleanup, not used */ @@ -318,10 +318,6 @@ struct proc { char p_nice; /* Process "nice" value.(PL) */ u_char p_resv1; /* (NU) User-priority based on p_cpu and p_nice. */ -#if CONFIG_MACF - int p_mac_enforce; /* MAC policy enforcement control */ -#endif - // types currently in sys/param.h command_t p_comm; proc_name_t p_name; /* can be changed by the process */ @@ -364,7 +360,6 @@ struct proc { uint32_t p_pth_tsd_offset; /* offset from pthread_t to TSD for new threads */ user_addr_t p_stack_addr_hint; /* stack allocation hint for wq threads */ void * p_wqptr; /* workq ptr */ - struct kqueue * p_wqkqueue; /* private workq kqueue */ struct timeval p_start; /* starting time */ void * p_rcall; @@ -384,6 +379,8 @@ struct proc { #endif /* DIAGNOSTIC */ uint64_t p_dispatchqueue_offset; uint64_t p_dispatchqueue_serialno_offset; + uint64_t p_return_to_kernel_offset; + uint64_t p_mach_thread_self_offset; #if VM_PRESSURE_EVENTS struct timeval vm_pressure_last_notify_tstamp; #endif @@ -409,6 +406,7 @@ struct proc { /* cached proc-specific data required for corpse inspection */ pid_t p_responsible_pid; /* pid resonsible for this process */ + _Atomic uint32_t p_user_faults; /* count the number of user faults generated */ struct os_reason *p_exit_reason; }; @@ -515,7 +513,9 @@ struct proc { /* This packing breaks symmetry with userspace side (struct extern_proc * of proc.h) for the ARMV7K ABI where 64-bit types are 64-bit aligned */ +#if !(__arm__ && (__BIGGEST_ALIGNMENT__ > 4)) #pragma pack(4) +#endif struct user32_extern_proc { union { struct { @@ -655,9 +655,11 @@ extern LIST_HEAD(sesshashhead, session) *sesshashtbl; extern u_long sesshash; extern lck_grp_t * proc_lck_grp; +extern lck_grp_t * proc_fdmlock_grp; +extern lck_grp_t * proc_kqhashlock_grp; +extern lck_grp_t * proc_knhashlock_grp; #if CONFIG_FINE_LOCK_GROUPS extern lck_grp_t * proc_mlock_grp; -extern lck_grp_t * proc_fdmlock_grp; extern lck_grp_t * proc_ucred_mlock_grp; extern lck_grp_t * proc_slock_grp; #endif diff --git a/bsd/sys/process_policy.h b/bsd/sys/process_policy.h index 4aec9e13e..ca679bf2f 100644 --- a/bsd/sys/process_policy.h +++ b/bsd/sys/process_policy.h @@ -65,7 +65,11 @@ __BEGIN_DECLS #define PROC_POLICY_HARDWARE_ACCESS 2 /* access to various hardware */ #define PROC_POLICY_RESOURCE_STARVATION 3 /* behavior on resource starvation */ #define PROC_POLICY_RESOURCE_USAGE 4 /* behavior on resource consumption */ +#if CONFIG_EMBEDDED || TARGET_OS_EMBEDDED +#define PROC_POLICY_APP_LIFECYCLE 5 /* app life cycle management */ +#else /* CONFIG_EMBEDDED */ #define PROC_POLICY_RESERVED 5 /* behavior on resource consumption */ +#endif /* CONFIG_EMBEDDED */ #define PROC_POLICY_APPTYPE 6 /* behavior on resource consumption */ #define PROC_POLICY_BOOST 7 /* importance boost/drop */ @@ -75,7 +79,11 @@ __BEGIN_DECLS #define PROC_POLICY_BG_DISKTHROTTLE 2 /* disk accesses throttled */ #define PROC_POLICY_BG_NETTHROTTLE 4 /* network accesses throttled */ #define PROC_POLICY_BG_GPUDENY 8 /* no access to GPU */ +#if CONFIG_EMBEDDED || TARGET_OS_EMBEDDED +#define PROC_POLICY_BG_ALL 0x0F +#else /* CONFIG_EMBEDDED */ #define PROC_POLICY_BG_ALL 0x07 +#endif /* CONFIG_EMBEDDED */ #define PROC_POLICY_BG_DEFAULT PROC_POLICY_BG_ALL /* sub policies for hardware */ @@ -161,10 +169,20 @@ typedef struct proc_policy_cpuusage_attr { uint64_t ppattr_cpu_attr_deadline; /* 64bit deadline in nsecs */ } proc_policy_cpuusage_attr_t; +#if CONFIG_EMBEDDED || TARGET_OS_EMBEDDED +/* sub policies for app lifecycle management */ +#define PROC_POLICY_APPLIFE_NONE 0 /* does nothing.. */ +#define PROC_POLICY_APPLIFE_STATE 1 /* sets the app to various lifecycle states */ +#define PROC_POLICY_APPLIFE_DEVSTATUS 2 /* notes the device in inactive or short/long term */ +#define PROC_POLICY_APPLIFE_PIDBIND 3 /* a thread is to be bound to another processes app state */ +#endif /* CONFIG_EMBEDDED */ /* sub policies for PROC_POLICY_APPTYPE */ #define PROC_POLICY_APPTYPE_NONE 0 /* does nothing.. */ #define PROC_POLICY_APPTYPE_MODIFY 1 /* sets the app to various lifecycle states */ +#if CONFIG_EMBEDDED || TARGET_OS_EMBEDDED +#define PROC_POLICY_APPTYPE_THREADTHR 2 /* notes the device in inactive or short/long term */ +#endif /* CONFIG_EMBEDDED */ /* exported apptypes for PROC_POLICY_APPTYPE */ #define PROC_POLICY_OSX_APPTYPE_TAL 1 /* TAL-launched app */ diff --git a/bsd/sys/protosw.h b/bsd/sys/protosw.h index c636c19e6..9a9311ee6 100644 --- a/bsd/sys/protosw.h +++ b/bsd/sys/protosw.h @@ -72,6 +72,44 @@ /* XXX: this will go away */ #define PR_SLOWHZ 2 /* 2 slow timeouts per second */ +/* + * The arguments to the ctlinput routine are + * (*protosw[].pr_ctlinput)(cmd, sa, arg); + * where cmd is one of the commands below, sa is a pointer to a sockaddr, + * and arg is a `void *' argument used within a protocol family. + */ +#define PRC_IFDOWN 0 /* interface transition */ +#define PRC_ROUTEDEAD 1 /* select new route if possible ??? */ +#define PRC_IFUP 2 /* interface has come back up */ +#define PRC_QUENCH2 3 /* DEC congestion bit says slow down */ +#define PRC_QUENCH 4 /* some one said to slow down */ +#define PRC_MSGSIZE 5 /* message size forced drop */ +#define PRC_HOSTDEAD 6 /* host appears to be down */ +#define PRC_HOSTUNREACH 7 /* deprecated (use PRC_UNREACH_HOST) */ +#define PRC_UNREACH_NET 8 /* no route to network */ +#define PRC_UNREACH_HOST 9 /* no route to host */ +#define PRC_UNREACH_PROTOCOL 10 /* dst says bad protocol */ +#define PRC_UNREACH_PORT 11 /* bad port # */ +/* was PRC_UNREACH_NEEDFRAG 12 (use PRC_MSGSIZE) */ +#define PRC_UNREACH_SRCFAIL 13 /* source route failed */ +#define PRC_REDIRECT_NET 14 /* net routing redirect */ +#define PRC_REDIRECT_HOST 15 /* host routing redirect */ +#define PRC_REDIRECT_TOSNET 16 /* redirect for type of service & net */ +#define PRC_REDIRECT_TOSHOST 17 /* redirect for tos & host */ +#define PRC_TIMXCEED_INTRANS 18 /* packet lifetime expired in transit */ +#define PRC_TIMXCEED_REASS 19 /* lifetime expired on reass q */ +#define PRC_PARAMPROB 20 /* header incorrect */ +#define PRC_UNREACH_ADMIN_PROHIB 21 /* packet administrativly prohibited */ + +#define PRC_NCMDS 22 + +#define PRC_IS_REDIRECT(cmd) \ + ((cmd) >= PRC_REDIRECT_NET && (cmd) <= PRC_REDIRECT_TOSHOST) + +#ifdef BSD_KERNEL_PRIVATE +#include +#endif + #ifdef KERNEL_PRIVATE #include #include @@ -86,6 +124,7 @@ struct socket; struct sockopt; struct socket_filter; struct uio; +struct ifnet; #ifdef XNU_KERNEL_PRIVATE struct domain_old; #endif /* XNU_KERNEL_PRIVATE */ @@ -118,7 +157,7 @@ struct protosw { int (*pr_output) /* output to protocol (from above) */ (struct mbuf *m, struct socket *so); void (*pr_ctlinput) /* control input (from below) */ - (int, struct sockaddr *, void *); + (int, struct sockaddr *, void *, struct ifnet *); int (*pr_ctloutput) /* control output (from above) */ (struct socket *, struct sockopt *); /* @@ -140,11 +179,11 @@ struct protosw { struct pr_usrreqs *pr_usrreqs; /* supersedes pr_usrreq() */ #endif /* !XNU_KERNEL_PRIVATE */ int (*pr_lock) /* lock function for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); int (*pr_unlock) /* unlock for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); lck_mtx_t *(*pr_getlock) /* retrieve protocol lock */ - (struct socket *so, int locktype); + (struct socket *so, int flags); /* * Implant hooks */ @@ -204,7 +243,7 @@ struct protosw { int (*pr_output) /* output to protocol (from above) */ (struct mbuf *m, struct socket *so); void (*pr_ctlinput) /* control input (from below) */ - (int, struct sockaddr *, void *); + (int, struct sockaddr *, void *, struct ifnet *); int (*pr_ctloutput) /* control output (from above) */ (struct socket *, struct sockopt *); /* @@ -220,17 +259,23 @@ struct protosw { int (*pr_sysctl) /* sysctl for protocol */ (int *, u_int, void *, size_t *, void *, size_t); int (*pr_lock) /* lock function for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); int (*pr_unlock) /* unlock for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); lck_mtx_t *(*pr_getlock) /* retrieve protocol lock */ - (struct socket *so, int locktype); + (struct socket *so, int flags); /* * misc */ TAILQ_HEAD(, socket_filter) pr_filter_head; struct protosw_old *pr_old; }; + +/* + * Values for the flags argument of pr_getlock + */ +#define PR_F_WILLUNLOCK 0x01 /* Will unlock (e.g., msleep) after the pr_getlock call */ + #endif /* XNU_KERNEL_PRIVATE */ /* @@ -268,40 +313,6 @@ struct protosw { #endif /* BSD_KERNEL_PRIVATE */ #ifdef BSD_KERNEL_PRIVATE -/* - * The arguments to the ctlinput routine are - * (*protosw[].pr_ctlinput)(cmd, sa, arg); - * where cmd is one of the commands below, sa is a pointer to a sockaddr, - * and arg is a `void *' argument used within a protocol family. - */ -#define PRC_IFDOWN 0 /* interface transition */ -#define PRC_ROUTEDEAD 1 /* select new route if possible ??? */ -#define PRC_IFUP 2 /* interface has come back up */ -#define PRC_QUENCH2 3 /* DEC congestion bit says slow down */ -#define PRC_QUENCH 4 /* some one said to slow down */ -#define PRC_MSGSIZE 5 /* message size forced drop */ -#define PRC_HOSTDEAD 6 /* host appears to be down */ -#define PRC_HOSTUNREACH 7 /* deprecated (use PRC_UNREACH_HOST) */ -#define PRC_UNREACH_NET 8 /* no route to network */ -#define PRC_UNREACH_HOST 9 /* no route to host */ -#define PRC_UNREACH_PROTOCOL 10 /* dst says bad protocol */ -#define PRC_UNREACH_PORT 11 /* bad port # */ -/* was PRC_UNREACH_NEEDFRAG 12 (use PRC_MSGSIZE) */ -#define PRC_UNREACH_SRCFAIL 13 /* source route failed */ -#define PRC_REDIRECT_NET 14 /* net routing redirect */ -#define PRC_REDIRECT_HOST 15 /* host routing redirect */ -#define PRC_REDIRECT_TOSNET 16 /* redirect for type of service & net */ -#define PRC_REDIRECT_TOSHOST 17 /* redirect for tos & host */ -#define PRC_TIMXCEED_INTRANS 18 /* packet lifetime expired in transit */ -#define PRC_TIMXCEED_REASS 19 /* lifetime expired on reass q */ -#define PRC_PARAMPROB 20 /* header incorrect */ -#define PRC_UNREACH_ADMIN_PROHIB 21 /* packet administrativly prohibited */ - -#define PRC_NCMDS 22 - -#define PRC_IS_REDIRECT(cmd) \ - ((cmd) >= PRC_REDIRECT_NET && (cmd) <= PRC_REDIRECT_TOSHOST) - #ifdef PRCREQUESTS char *prcrequests[] = { "IFDOWN", "ROUTEDEAD", "IFUP", "DEC-BIT-QUENCH2", @@ -383,7 +394,6 @@ char *prurequests[] = { #endif /* PRUREQUESTS */ #endif /* BSD_KERNEL_PRIVATE */ -struct ifnet; struct stat; struct ucred; struct uio; @@ -463,8 +473,6 @@ struct pr_usrreqs { int (*pru_disconnectx)(struct socket *, sae_associd_t, sae_connid_t); int (*pru_listen)(struct socket *, struct proc *); - int (*pru_peeloff)(struct socket *, - sae_associd_t, struct socket **); int (*pru_peeraddr)(struct socket *, struct sockaddr **); int (*pru_rcvd)(struct socket *, int); int (*pru_rcvoob)(struct socket *, struct mbuf *, int); @@ -493,6 +501,12 @@ struct pr_usrreqs { /* Values for pru_flags */ #define PRUF_OLD 0x10000000 /* added via net_add_proto */ +#ifdef BSD_KERNEL_PRIVATE +/* + * For faster access than net_uptime(), bypassing the initialization. + */ +extern u_int64_t _net_uptime; +#endif /* BSD_KERNEL_PRIVATE */ #endif /* XNU_KERNEL_PRIVATE */ __BEGIN_DECLS @@ -511,7 +525,6 @@ extern int pru_connectx_notsupp(struct socket *, struct sockaddr *, extern int pru_disconnectx_notsupp(struct socket *, sae_associd_t, sae_connid_t); extern int pru_socheckopt_null(struct socket *, struct sockopt *); -extern int pru_peeloff_notsupp(struct socket *, sae_associd_t, struct socket **); #endif /* XNU_KERNEL_PRIVATE */ extern int pru_control_notsupp(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p); @@ -555,7 +568,7 @@ extern int net_del_proto(int, int, struct domain *); extern int net_add_proto_old(struct protosw_old *, struct domain_old *); extern int net_del_proto_old(int, int, struct domain_old *); extern void net_update_uptime(void); -extern void net_update_uptime_secs(uint64_t secs); +extern void net_update_uptime_with_time(const struct timeval *); extern u_int64_t net_uptime(void); extern void net_uptime2timeval(struct timeval *); #else diff --git a/bsd/sys/pthread_internal.h b/bsd/sys/pthread_internal.h index 634470f89..1dff9968f 100644 --- a/bsd/sys/pthread_internal.h +++ b/bsd/sys/pthread_internal.h @@ -45,6 +45,7 @@ void workqueue_exit(struct proc *); void pthread_init(void); int thread_qos_from_pthread_priority(unsigned long, unsigned long *); unsigned long pthread_priority_canonicalize(unsigned long priority, boolean_t propagation); +boolean_t workq_thread_has_been_unbound(thread_t th, int qos_class); #endif /* _SYS_PTHREAD_INTERNAL_H_ */ diff --git a/bsd/sys/pthread_shims.h b/bsd/sys/pthread_shims.h index bd3322d42..2256a4a01 100644 --- a/bsd/sys/pthread_shims.h +++ b/bsd/sys/pthread_shims.h @@ -37,21 +37,36 @@ #include #include #include +#include #include #include #include -#include #ifndef PTHREAD_INTERNAL struct uthread; #define M_PROC 41 #endif -#ifdef NEEDS_SCHED_CALL_T +#if !defined(_SCHED_CALL_T_DEFINED) +#define _SCHED_CALL_T_DEFINED typedef void (*sched_call_t)(int type, thread_t thread); #endif typedef struct workq_reqthreads_req_s {unsigned long priority; int count;} *workq_reqthreads_req_t; +typedef struct workq_threadreq_s { void *opaqueptr[2]; uint32_t opaqueint[2];} *workq_threadreq_t; +enum workq_threadreq_type { + WORKQ_THREADREQ_KEVENT = 1, + WORKQ_THREADREQ_WORKLOOP = 2, + WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL = 3, + WORKQ_THREADREQ_REDRIVE = 4, +}; +enum workq_threadreq_op { + WORKQ_THREADREQ_CHANGE_PRI = 1, + WORKQ_THREADREQ_CANCEL = 2, + WORKQ_THREADREQ_CHANGE_PRI_NO_THREAD_CALL = 3, +}; +#define WORKQ_THREADREQ_FLAG_NOEMERGENCY 0x1 + /* * Increment each time new reserved slots are used. When the pthread @@ -65,11 +80,10 @@ typedef const struct pthread_functions_s { /* internal calls, kernel core -> kext */ void (*pthread_init)(void); - int (*fill_procworkqueue)(proc_t p, struct proc_workqueueinfo * pwqinfo); + int (*fill_procworkqueue)(proc_t p, void* pwqinfo); - // UNUSED - TO BE DELETED - void (*workqueue_init_lock)(proc_t p); - void (*workqueue_destroy_lock)(proc_t p); + void (*__unused1)(void); + void (*__unused2)(void); void (*workqueue_exit)(struct proc *p); void (*workqueue_mark_exiting)(struct proc *p); @@ -115,14 +129,51 @@ typedef const struct pthread_functions_s { /* try to get wq flags in debugger context */ uint32_t (*get_pwq_state_kdp)(proc_t p); - unsigned long (*pthread_priority_canonicalize)(unsigned long pthread_priority); + void (*__unused3)(void); unsigned long (*pthread_priority_canonicalize2)(unsigned long pthread_priority, boolean_t propagation); + /* Returns true on success, false on mismatch */ + boolean_t (*workq_thread_has_been_unbound)(thread_t th, int qos_class); + void (*pthread_find_owner)(thread_t thread, struct stackshot_thread_waitinfo *waitinfo); void *(*pthread_get_thread_kwq)(thread_t thread); + /* + * Submits a threadreq to the workq system. + * + * If type is WORKQ_THREADREQ_KEVENT, the semantics are similar to a call + * to workq_reqthreads and the kevent bind function will be called to + * indicate the thread fulfilling the request. The req argument is ignored. + * + * If type is WORKQ_THREADREQ_WORKLOOP, The req argument should point to + * allocated memory of at least the sizeof(workq_threadreq_t). That memory + * is lent to the workq system until workloop_fulfill_threadreq is called + * and passed the pointer, at which point it may be freed. + * + * The properties of the request are passed in the (pthread) priority and flags arguments. + * + * Will return zero upon success or an error value on failure. An error of + * ENOTSUP means the type argument was not understood. + */ + int (*workq_threadreq)(struct proc *p, workq_threadreq_t req, + enum workq_threadreq_type, unsigned long priority, int flags); + + /* + * Modifies an already submitted thread request. + * + * If operation is WORKQ_THREADREQ_CHANGE_PRI, arg1 is the new priority and arg2 is unused. + * + * If operation is WORKQ_THREADREQ_CANCEL, arg1 and arg2 are unused. + * + * Will return zero upon success or an error value on failure. An error of + * ENOTSUP means the operation argument was not understood. + */ + int (*workq_threadreq_modify)(struct proc *t, workq_threadreq_t req, + enum workq_threadreq_op operation, + unsigned long arg1, unsigned long arg2); + /* padding for future */ - void * _pad[90]; + void * _pad[87]; } * pthread_functions_t; typedef const struct pthread_callbacks_s { @@ -142,8 +193,14 @@ typedef const struct pthread_callbacks_s { void (*proc_set_wqthread)(struct proc *t, user_addr_t addr); int (*proc_get_pthsize)(struct proc *t); void (*proc_set_pthsize)(struct proc *t, int size); +#if defined(__arm64__) + unsigned __int128 (*atomic_fetch_add_128_relaxed)(_Atomic unsigned __int128 *ptr, + unsigned __int128 value); + unsigned __int128 (*atomic_load_128_relaxed)(_Atomic unsigned __int128 *ptr); +#else void *unused_was_proc_get_targconc; void *unused_was_proc_set_targconc; +#endif uint64_t (*proc_get_dispatchqueue_offset)(struct proc *t); void (*proc_set_dispatchqueue_offset)(struct proc *t, uint64_t offset); void *unused_was_proc_get_wqlockptr; @@ -178,11 +235,13 @@ typedef const struct pthread_callbacks_s { /* wq functions */ kern_return_t (*thread_set_wq_state32)(thread_t thread, thread_state_t state); +#if !defined(__arm__) kern_return_t (*thread_set_wq_state64)(thread_t thread, thread_state_t state); +#endif /* sched_prim.h */ - void (*thread_exception_return)(); - void (*thread_bootstrap_return)(); + void (*thread_exception_return)(void); + void (*thread_bootstrap_return)(void); /* kern/clock.h */ void (*absolutetime_to_microtime)(uint64_t abstime, clock_sec_t *secs, clock_usec_t *microsecs); @@ -228,6 +287,9 @@ typedef const struct pthread_callbacks_s { /* osfmk//machine_routines.h */ int (*ml_get_max_cpus)(void); + #if defined(__arm__) + uint32_t (*map_is_1gb)(vm_map_t); + #endif /* xnu: struct proc p_dispatchqueue_serialno_offset additions */ uint64_t (*proc_get_dispatchqueue_serialno_offset)(struct proc *p); @@ -243,8 +305,8 @@ typedef const struct pthread_callbacks_s { kern_return_t (*thread_set_tsd_base)(thread_t thread, mach_vm_offset_t tsd_base); int (*proc_usynch_get_requested_thread_qos)(struct uthread *); - void *unused_was_proc_usynch_thread_qos_add_override; - void *unused_was_proc_usynch_thread_qos_remove_override; + uint64_t (*proc_get_mach_thread_self_tsd_offset)(struct proc *p); + void (*proc_set_mach_thread_self_tsd_offset)(struct proc *p, uint64_t mach_thread_self_tsd_offset); kern_return_t (*thread_policy_get)(thread_t t, thread_policy_flavor_t flavor, thread_policy_t info, mach_msg_type_number_t *count, boolean_t *get_default); boolean_t (*qos_main_thread_active)(void); @@ -268,8 +330,24 @@ typedef const struct pthread_callbacks_s { user_addr_t (*proc_get_stack_addr_hint)(struct proc *p); void (*proc_set_stack_addr_hint)(struct proc *p, user_addr_t stack_addr_hint); + uint64_t (*proc_get_return_to_kernel_offset)(struct proc *t); + void (*proc_set_return_to_kernel_offset)(struct proc *t, uint64_t offset); + + /* indicates call is being made synchronously with workq_threadreq call */ +# define WORKLOOP_FULFILL_THREADREQ_SYNC 0x1 +# define WORKLOOP_FULFILL_THREADREQ_CANCEL 0x2 + int (*workloop_fulfill_threadreq)(struct proc *p, workq_threadreq_t req, thread_t thread, int flags); + void (*thread_will_park_or_terminate)(thread_t thread); + + /* For getting maximum parallelism for a given QoS */ + uint32_t (*qos_max_parallelism)(int qos, uint64_t options); + + /* proc_internal.h: struct proc user_stack accessor */ + user_addr_t (*proc_get_user_stack)(struct proc *p); + void (*proc_set_user_stack)(struct proc *p, user_addr_t user_stack); + /* padding for future */ - void* _pad[76]; + void* _pad[69]; } *pthread_callbacks_t; diff --git a/bsd/sys/quota.h b/bsd/sys/quota.h index 35cb6b70a..d0022c0d9 100644 --- a/bsd/sys/quota.h +++ b/bsd/sys/quota.h @@ -69,6 +69,7 @@ #include #include +#include /* u_int32_t */ #ifdef KERNEL_PRIVATE #include #endif diff --git a/bsd/sys/reason.h b/bsd/sys/reason.h index 0952e5e96..2d81e3a41 100644 --- a/bsd/sys/reason.h +++ b/bsd/sys/reason.h @@ -97,22 +97,30 @@ void os_reason_free(os_reason_t cur_reason); #define OS_REASON_REPORTCRASH 12 #define OS_REASON_COREANIMATION 13 #define OS_REASON_AGGREGATED 14 +#define OS_REASON_ASSERTIOND 15 +#define OS_REASON_SKYWALK 16 +#define OS_REASON_SETTINGS 17 +#define OS_REASON_LIBSYSTEM 18 +#define OS_REASON_FOUNDATION 19 +#define OS_REASON_WATCHDOG 20 +#define OS_REASON_METAL 21 /* * Update whenever new OS_REASON namespaces are added. */ -#define OS_REASON_MAX_VALID_NAMESPACE OS_REASON_AGGREGATED +#define OS_REASON_MAX_VALID_NAMESPACE OS_REASON_METAL #define OS_REASON_BUFFER_MAX_SIZE 5120 -#define OS_REASON_FLAG_NO_CRASH_REPORT 0x1 /* Don't create a crash report */ -#define OS_REASON_FLAG_GENERATE_CRASH_REPORT 0x2 /* Create a crash report - the default for userspace requests */ -#define OS_REASON_FLAG_FROM_USERSPACE 0x4 /* Reason created from a userspace syscall */ -#define OS_REASON_FLAG_FAILED_DATA_COPYIN 0x8 /* We failed to copyin data from userspace */ -#define OS_REASON_FLAG_PAYLOAD_TRUNCATED 0x10 /* The payload was truncated because it was longer than allowed */ -#define OS_REASON_FLAG_BAD_PARAMS 0x20 /* Invalid parameters were passed involved with creating this reason */ -#define OS_REASON_FLAG_CONSISTENT_FAILURE 0x40 /* Whatever caused this reason to be created will happen again */ -#define OS_REASON_FLAG_ONE_TIME_FAILURE 0x80 /* Whatever caused this reason to be created was a one time issue */ +#define OS_REASON_FLAG_NO_CRASH_REPORT 0x1 /* Don't create a crash report */ +#define OS_REASON_FLAG_GENERATE_CRASH_REPORT 0x2 /* Create a crash report - the default for userspace requests */ +#define OS_REASON_FLAG_FROM_USERSPACE 0x4 /* Reason created from a userspace syscall */ +#define OS_REASON_FLAG_FAILED_DATA_COPYIN 0x8 /* We failed to copyin data from userspace */ +#define OS_REASON_FLAG_PAYLOAD_TRUNCATED 0x10 /* The payload was truncated because it was longer than allowed */ +#define OS_REASON_FLAG_BAD_PARAMS 0x20 /* Invalid parameters were passed involved with creating this reason */ +#define OS_REASON_FLAG_CONSISTENT_FAILURE 0x40 /* Whatever caused this reason to be created will happen again */ +#define OS_REASON_FLAG_ONE_TIME_FAILURE 0x80 /* Whatever caused this reason to be created was a one time issue */ +#define OS_REASON_FLAG_NO_CRASHED_TID 0x100 /* Don't include the TID that processed the exit in the crash report */ /* * Set of flags that are allowed to be passed from userspace diff --git a/bsd/sys/resource.h b/bsd/sys/resource.h index 1dfb214a5..2f0316c87 100644 --- a/bsd/sys/resource.h +++ b/bsd/sys/resource.h @@ -210,7 +210,8 @@ struct rusage { #define RUSAGE_INFO_V1 1 #define RUSAGE_INFO_V2 2 #define RUSAGE_INFO_V3 3 -#define RUSAGE_INFO_CURRENT RUSAGE_INFO_V3 +#define RUSAGE_INFO_V4 4 +#define RUSAGE_INFO_CURRENT RUSAGE_INFO_V4 typedef void *rusage_info_t; @@ -222,7 +223,7 @@ struct rusage_info_v0 { uint64_t ri_interrupt_wkups; uint64_t ri_pageins; uint64_t ri_wired_size; - uint64_t ri_resident_size; + uint64_t ri_resident_size; uint64_t ri_phys_footprint; uint64_t ri_proc_start_abstime; uint64_t ri_proc_exit_abstime; @@ -236,7 +237,7 @@ struct rusage_info_v1 { uint64_t ri_interrupt_wkups; uint64_t ri_pageins; uint64_t ri_wired_size; - uint64_t ri_resident_size; + uint64_t ri_resident_size; uint64_t ri_phys_footprint; uint64_t ri_proc_start_abstime; uint64_t ri_proc_exit_abstime; @@ -256,7 +257,7 @@ struct rusage_info_v2 { uint64_t ri_interrupt_wkups; uint64_t ri_pageins; uint64_t ri_wired_size; - uint64_t ri_resident_size; + uint64_t ri_resident_size; uint64_t ri_phys_footprint; uint64_t ri_proc_start_abstime; uint64_t ri_proc_exit_abstime; @@ -278,7 +279,7 @@ struct rusage_info_v3 { uint64_t ri_interrupt_wkups; uint64_t ri_pageins; uint64_t ri_wired_size; - uint64_t ri_resident_size; + uint64_t ri_resident_size; uint64_t ri_phys_footprint; uint64_t ri_proc_start_abstime; uint64_t ri_proc_exit_abstime; @@ -301,7 +302,46 @@ struct rusage_info_v3 { uint64_t ri_serviced_system_time; }; -typedef struct rusage_info_v3 rusage_info_current; +struct rusage_info_v4 { + uint8_t ri_uuid[16]; + uint64_t ri_user_time; + uint64_t ri_system_time; + uint64_t ri_pkg_idle_wkups; + uint64_t ri_interrupt_wkups; + uint64_t ri_pageins; + uint64_t ri_wired_size; + uint64_t ri_resident_size; + uint64_t ri_phys_footprint; + uint64_t ri_proc_start_abstime; + uint64_t ri_proc_exit_abstime; + uint64_t ri_child_user_time; + uint64_t ri_child_system_time; + uint64_t ri_child_pkg_idle_wkups; + uint64_t ri_child_interrupt_wkups; + uint64_t ri_child_pageins; + uint64_t ri_child_elapsed_abstime; + uint64_t ri_diskio_bytesread; + uint64_t ri_diskio_byteswritten; + uint64_t ri_cpu_time_qos_default; + uint64_t ri_cpu_time_qos_maintenance; + uint64_t ri_cpu_time_qos_background; + uint64_t ri_cpu_time_qos_utility; + uint64_t ri_cpu_time_qos_legacy; + uint64_t ri_cpu_time_qos_user_initiated; + uint64_t ri_cpu_time_qos_user_interactive; + uint64_t ri_billed_system_time; + uint64_t ri_serviced_system_time; + uint64_t ri_logical_writes; + uint64_t ri_lifetime_max_phys_footprint; + uint64_t ri_instructions; + uint64_t ri_cycles; + uint64_t ri_billed_energy; + uint64_t ri_serviced_energy; + // We're reserving 2 counters for future extension + uint64_t ri_unused[2]; +}; + +typedef struct rusage_info_v4 rusage_info_current; #endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL */ diff --git a/bsd/sys/resourcevar.h b/bsd/sys/resourcevar.h index 57ae8eb7c..6a21d2b6d 100644 --- a/bsd/sys/resourcevar.h +++ b/bsd/sys/resourcevar.h @@ -65,6 +65,7 @@ #define _SYS_RESOURCEVAR_H_ #include +#include /* * Kernel per-process accounting / statistics diff --git a/bsd/sys/sdt_impl.h b/bsd/sys/sdt_impl.h index 7517dd627..a0675b2b1 100644 --- a/bsd/sys/sdt_impl.h +++ b/bsd/sys/sdt_impl.h @@ -77,6 +77,10 @@ extern int sdt_probetab_mask; #if defined(__x86_64__) typedef uint8_t sdt_instr_t; +#elif defined(__arm__) +typedef uint16_t sdt_instr_t; +#elif defined(__arm64__) +typedef uint32_t sdt_instr_t; #else #error Unknown implementation #endif diff --git a/bsd/sys/sem.h b/bsd/sys/sem.h index 67c6064aa..4c19918c5 100644 --- a/bsd/sys/sem.h +++ b/bsd/sys/sem.h @@ -40,6 +40,7 @@ #include #include +#include /* __int32_t */ /* * [XSI] All of the symbols from SHALL be defined diff --git a/bsd/sys/signal.h b/bsd/sys/signal.h index 2483e8db3..817454ab5 100644 --- a/bsd/sys/signal.h +++ b/bsd/sys/signal.h @@ -568,12 +568,12 @@ struct sigstack { * signals delivered on a per-thread basis. */ #define threadmask (sigmask(SIGILL)|sigmask(SIGTRAP)|\ - sigmask(SIGIOT)|sigmask(SIGEMT)|\ + sigmask(SIGABRT)|sigmask(SIGEMT)|\ sigmask(SIGFPE)|sigmask(SIGBUS)|\ sigmask(SIGSEGV)|sigmask(SIGSYS)|\ sigmask(SIGPIPE)|sigmask(SIGKILL)) -#define workq_threadmask (threadmask | sigcantmask | sigmask(SIGPROF)) +#define workq_threadmask ((threadmask | sigcantmask | sigmask(SIGPROF)) & ~sigmask(SIGABRT)) /* * Signals carried across exec. diff --git a/bsd/sys/snapshot.h b/bsd/sys/snapshot.h index 76f115727..3953eab3e 100644 --- a/bsd/sys/snapshot.h +++ b/bsd/sys/snapshot.h @@ -52,7 +52,9 @@ int fs_snapshot_mount(int, const char *, const char *, uint32_t) __OSX_AVAILABLE int fs_snapshot_revert(int, const char *, uint32_t) __OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0) __TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0); +#ifdef PRIVATE int fs_snapshot_root(int, const char *, uint32_t) __OSX_AVAILABLE(10.12.4) __IOS_AVAILABLE(10.3) __TVOS_AVAILABLE(10.3) __WATCHOS_AVAILABLE(3.3); +#endif __END_DECLS diff --git a/bsd/sys/socket.h b/bsd/sys/socket.h index edb1dfdd1..2bdccabf6 100644 --- a/bsd/sys/socket.h +++ b/bsd/sys/socket.h @@ -196,7 +196,7 @@ #endif #ifdef PRIVATE -#define SO_EXECPATH 0x1085 /* Application Firewall Socket option */ +#define SO_EXECPATH 0x1085 /* Application Firewall Socket option */ /* * Traffic service class definitions (lowest to highest): @@ -324,13 +324,11 @@ #define SO_DELEGATED_UUID 0x1108 /* set socket as delegate (uuid_t) */ #define SO_NECP_ATTRIBUTES 0x1109 /* NECP socket attributes (domain, account, etc.) */ #define SO_CFIL_SOCK_ID 0x1110 /* get content filter socket ID (cfil_sock_id_t) */ -#if MPTCP -#define SO_MPTCP_FASTJOIN 0x1111 /* fast join MPTCP */ -#endif /* MPTCP */ +#define SO_NECP_CLIENTUUID 0x1111 /* NECP Client uuid */ #endif /* PRIVATE */ #define SO_NUMRCVPKT 0x1112 /* number of datagrams in receive socket buffer */ #ifdef PRIVATE -#define SO_AWDL_UNRESTRICTED 0x1113 /* try to use AWDL in restricted mode */ +#define SO_AWDL_UNRESTRICTED 0x1113 /* try to use AWDL in restricted mode */ #define SO_EXTENDED_BK_IDLE 0x1114 /* extended time to keep socket idle after app is suspended (int) */ #define SO_MARK_CELLFALLBACK 0x1115 /* Mark as initiated by cell fallback */ #endif /* PRIVATE */ @@ -1269,6 +1267,7 @@ struct so_cinforeq64 { /* valid connection info auxiliary data types */ #define CIAUX_TCP 0x1 /* TCP auxiliary data (conninfo_tcp_t) */ +#define CIAUX_MPTCP 0x2 /* MPTCP auxiliary data (conninfo_mptcp_t) */ /* * Structure for SIOC{S,G}CONNORDER @@ -1425,8 +1424,12 @@ int sendfile(int, int, off_t, off_t *, struct sf_hdtr *, int); #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) void pfctlinput(int, struct sockaddr *); + +__API_AVAILABLE(macosx(10.11), ios(9.0), tvos(9.0), watchos(2.0)) int connectx(int, const sa_endpoints_t *, sae_associd_t, unsigned int, const struct iovec *, unsigned int, size_t *, sae_connid_t *); + +__API_AVAILABLE(macosx(10.11), ios(9.0), tvos(9.0), watchos(2.0)) int disconnectx(int, sae_associd_t, sae_connid_t); #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ __END_DECLS diff --git a/bsd/sys/socketvar.h b/bsd/sys/socketvar.h index adff7ecd7..a8faa3c64 100644 --- a/bsd/sys/socketvar.h +++ b/bsd/sys/socketvar.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -74,6 +74,7 @@ #include #include +#include /* u_quad_t */ #ifdef KERNEL_PRIVATE #include /* for TAILQ macros */ #include /* for struct selinfo */ @@ -222,6 +223,7 @@ struct socket { #define SB_TRIM 0x800 /* Trim the socket buffer */ #define SB_NOCOMPRESS 0x1000 /* do not compress socket buffer */ #define SB_SNDBYTE_CNT 0x2000 /* keep track of snd bytes per interface */ +#define SB_UPCALL_LOCK 0x4000 /* Keep socket locked when doing the upcall */ caddr_t so_tpcb; /* Misc. protocol control block, used by some kexts */ @@ -274,12 +276,10 @@ struct socket { #define SOF_FLOW_DIVERT 0x00800000 /* Flow Divert is enabled */ #define SOF_MP_SUBFLOW 0x01000000 /* is a multipath subflow socket */ #define SOF_MPTCP_TRUE 0x02000000 /* Established e2e MPTCP connection */ -#define SOF_MPTCP_CLIENT 0x04000000 /* Only client starts addtnal flows */ -#define SOF_MP_SEC_SUBFLOW 0x08000000 /* Set up secondary flow */ -#define SOF_MP_TRYFAILOVER 0x10000000 /* Failing subflow */ -#define SOF_DELEGATED 0x20000000 /* on behalf of another process */ -#define SOF_MPTCP_FASTJOIN 0x40000000 /* fast join support */ -#define SOF_CONTENT_FILTER 0x80000000 /* Content filter enabled */ +#define SOF_MP_SEC_SUBFLOW 0x04000000 /* Set up secondary flow */ +#define SOF_MP_TRYFAILOVER 0x08000000 /* Failing subflow */ +#define SOF_DELEGATED 0x10000000 /* on behalf of another process */ +#define SOF_CONTENT_FILTER 0x20000000 /* Content filter enabled */ uint32_t so_upcallusecount; /* number of upcalls in progress */ int so_usecount; /* refcounting of socket use */; @@ -344,6 +344,12 @@ struct socket { #define SOF1_QOSMARKING_POLICY_OVERRIDE 0x00008000 /* Opt-out of QoS marking NECP policy */ #define SOF1_DATA_AUTHENTICATED 0x00010000 /* idempotent data is authenticated */ #define SOF1_ACCEPT_LIST_HELD 0x00020000 /* Another thread is accessing one of the accept lists */ +#define SOF1_CONTENT_FILTER_SKIP 0x00040000 /* Content filter should be skipped, socket is blessed */ +#define SOF1_HAS_NECP_CLIENT_UUID 0x00080000 /* NECP client UUID option set */ +#define SOF1_IN_KERNEL_SOCKET 0x00100000 /* Socket created in kernel via KPI */ +#define SOF1_CONNECT_COUNTED 0x00200000 /* connect() call was counted */ +#define SOF1_DNS_COUNTED 0x00400000 /* socket counted to send DNS queries */ + u_int64_t so_extended_bk_start; }; @@ -441,6 +447,7 @@ struct xsocket { uid_t so_uid; /* XXX */ }; +#if !CONFIG_EMBEDDED struct xsocket64 { u_int32_t xso_len; /* length of this structure */ u_int64_t xso_so; /* makes a convenient handle */ @@ -462,6 +469,7 @@ struct xsocket64 { struct xsockbuf so_snd; uid_t so_uid; /* XXX */ }; +#endif /* !CONFIG_EMBEDDED */ #ifdef PRIVATE #define XSO_SOCKET 0x001 @@ -580,35 +588,32 @@ struct kextcb { #define EXT_NULL 0x0 /* STATE: Not in use */ /* Hints for socket event processing */ -#define SO_FILT_HINT_LOCKED 0x00000001 /* socket is already locked */ -#define SO_FILT_HINT_CONNRESET 0x00000002 /* Reset is received */ +#define SO_FILT_HINT_LOCKED 0x00000001 /* socket is already locked */ +#define SO_FILT_HINT_CONNRESET 0x00000002 /* Reset is received */ #define SO_FILT_HINT_CANTRCVMORE 0x00000004 /* No more data to read */ #define SO_FILT_HINT_CANTSENDMORE 0x00000008 /* Can't write more data */ -#define SO_FILT_HINT_TIMEOUT 0x00000010 /* timeout */ -#define SO_FILT_HINT_NOSRCADDR 0x00000020 /* No src address available */ -#define SO_FILT_HINT_IFDENIED 0x00000040 /* interface denied access */ -#define SO_FILT_HINT_SUSPEND 0x00000080 /* output queue suspended */ -#define SO_FILT_HINT_RESUME 0x00000100 /* output queue resumed */ -#define SO_FILT_HINT_KEEPALIVE 0x00000200 /* TCP Keepalive received */ -#define SO_FILT_HINT_ADAPTIVE_WTIMO 0x00000400 /* TCP adaptive write timeout */ -#define SO_FILT_HINT_ADAPTIVE_RTIMO 0x00000800 /* TCP adaptive read timeout */ -#define SO_FILT_HINT_CONNECTED 0x00001000 /* socket is connected */ +#define SO_FILT_HINT_TIMEOUT 0x00000010 /* timeout */ +#define SO_FILT_HINT_NOSRCADDR 0x00000020 /* No src address available */ +#define SO_FILT_HINT_IFDENIED 0x00000040 /* interface denied access */ +#define SO_FILT_HINT_SUSPEND 0x00000080 /* output queue suspended */ +#define SO_FILT_HINT_RESUME 0x00000100 /* output queue resumed */ +#define SO_FILT_HINT_KEEPALIVE 0x00000200 /* TCP Keepalive received */ +#define SO_FILT_HINT_ADAPTIVE_WTIMO 0x00000400 /* TCP adaptive write timeout */ +#define SO_FILT_HINT_ADAPTIVE_RTIMO 0x00000800 /* TCP adaptive read timeout */ +#define SO_FILT_HINT_CONNECTED 0x00001000 /* socket is connected */ #define SO_FILT_HINT_DISCONNECTED 0x00002000 /* socket is disconnected */ -#define SO_FILT_HINT_CONNINFO_UPDATED 0x00004000 /* updated conninfo avail. */ -#define SO_FILT_HINT_MPFAILOVER 0x00008000 /* multipath failover */ -#define SO_FILT_HINT_MPSTATUS 0x00010000 /* multipath status */ -#define SO_FILT_HINT_MUSTRST 0x00020000 /* must send RST and close */ -#define SO_FILT_HINT_MPFASTJ 0x00040000 /* can do MPTCP fast join */ -#define SO_FILT_HINT_DELETEOK 0x00100000 /* Ok to delete socket */ -#define SO_FILT_HINT_MPCANTRCVMORE 0x00200000 /* MPTCP DFIN Received */ -#define SO_FILT_HINT_NOTIFY_ACK 0x00400000 /* Notify Acknowledgement */ +#define SO_FILT_HINT_CONNINFO_UPDATED 0x00004000 /* updated conninfo avail. */ +#define SO_FILT_HINT_MPFAILOVER 0x00008000 /* multipath failover */ +#define SO_FILT_HINT_MPSTATUS 0x00010000 /* multipath status */ +#define SO_FILT_HINT_MUSTRST 0x00020000 /* must send RST and close */ +#define SO_FILT_HINT_MPCANTRCVMORE 0x00040000 /* MPTCP DFIN Received */ +#define SO_FILT_HINT_NOTIFY_ACK 0x00080000 /* Notify Acknowledgement */ #define SO_FILT_HINT_BITS \ "\020\1LOCKED\2CONNRESET\3CANTRCVMORE\4CANTSENDMORE\5TIMEOUT" \ "\6NOSRCADDR\7IFDENIED\10SUSPEND\11RESUME\12KEEPALIVE\13AWTIMO" \ - "\14ARTIMO\15CONNECTED\16DISCONNECTED\17CONNINFO_UPDATED" \ - "\20MPFAILOVER\21MPSTATUS\22MUSTRST\23MPFASTJ\25DELETEOK" \ - "\26MPCANTRCVMORE\27NOTIFYACK" + "\14ARTIMO\15CONNECTED\16DISCONNECTED\17CONNINFO_UPDATED" \ + "\20MPFAILOVER\21MPSTATUS\22MUSTRST\23MPCANTRCVMORE\24NOTIFYACK" /* Mask for hints that have corresponding kqueue events */ #define SO_FILT_HINT_EV \ @@ -701,9 +706,10 @@ extern int sothrottlelog; extern int sorestrictrecv; extern int sorestrictsend; extern int somaxconn; +extern uint32_t tcp_do_autosendbuf; extern uint32_t tcp_autosndbuf_max; +extern uint32_t tcp_autosndbuf_inc; extern u_int32_t sotcdb; -extern u_int32_t net_io_policy_throttled; extern u_int32_t net_io_policy_log; extern u_int32_t net_io_policy_throttle_best_effort; #if CONFIG_PROC_UUID_POLICY @@ -763,6 +769,7 @@ extern int sopoll(struct socket *so, int events, struct ucred *cred, void *wql); extern int sooptcopyin(struct sockopt *sopt, void *data, size_t len, size_t minlen); extern int sooptcopyout(struct sockopt *sopt, void *data, size_t len); +extern int soopt_cred_check(struct socket *so, int priv, boolean_t allow_root); extern int soreceive(struct socket *so, struct sockaddr **paddr, struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp); extern int soreserve(struct socket *so, u_int32_t sndcc, u_int32_t rcvcc); @@ -853,11 +860,9 @@ extern int soconnectxlocked(struct socket *so, struct sockaddr *src, sae_connid_t *, uint32_t, void *, u_int32_t, uio_t, user_ssize_t *); extern int sodisconnectx(struct socket *so, sae_associd_t, sae_connid_t); extern int sodisconnectxlocked(struct socket *so, sae_associd_t, sae_connid_t); -extern int sopeelofflocked(struct socket *so, sae_associd_t, struct socket **); extern void soevupcall(struct socket *, u_int32_t); /* flags for socreate_internal */ #define SOCF_ASYNC 0x1 /* non-blocking socket */ -#define SOCF_MP_SUBFLOW 0x2 /* multipath subflow socket */ extern int socreate_internal(int dom, struct socket **aso, int type, int proto, struct proc *, uint32_t, struct proc *); extern int socreate(int dom, struct socket **aso, int type, int proto); @@ -882,10 +887,13 @@ extern int soisprivilegedtraffic(struct socket *so); extern int soissrcbackground(struct socket *so); extern int soissrcrealtime(struct socket *so); extern int soissrcbesteffort(struct socket *so); +extern void soclearfastopen(struct socket *so); extern int solisten(struct socket *so, int backlog); extern struct socket *sodropablereq(struct socket *head); -extern int socket_lock(struct socket *so, int refcount); -extern int socket_unlock(struct socket *so, int refcount); +extern void socket_lock(struct socket *so, int refcount); +extern void socket_lock_assert_owned(struct socket *so); +extern int socket_try_lock(struct socket *so); +extern void socket_unlock(struct socket *so, int refcount); extern int sogetaddr_locked(struct socket *, struct sockaddr **, int); extern const char *solockhistory_nr(struct socket *); extern void soevent(struct socket *so, long hint); @@ -899,7 +907,9 @@ extern int soshutdown(struct socket *so, int how); extern int soshutdownlock(struct socket *so, int how); extern int soshutdownlock_final(struct socket *so, int how); extern void sotoxsocket(struct socket *so, struct xsocket *xso); +#if !CONFIG_EMBEDDED extern void sotoxsocket64(struct socket *so, struct xsocket64 *xso); +#endif /* !CONFIG_EMBEDDED */ extern int sosendallatonce(struct socket *so); extern int soreadable(struct socket *so); extern int sowriteable(struct socket *so); @@ -910,7 +920,8 @@ extern int sosendcheck(struct socket *, struct sockaddr *, user_ssize_t, extern int soo_ioctl(struct fileproc *, u_long, caddr_t, vfs_context_t); extern int soo_stat(struct socket *, void *, int); extern int soo_select(struct fileproc *, int, void *, vfs_context_t); -extern int soo_kqfilter(struct fileproc *, struct knote *, vfs_context_t); +extern int soo_kqfilter(struct fileproc *, struct knote *, + struct kevent_internal_s *kev, vfs_context_t); /* Service class flags used for setting service class on a packet */ #define PKT_SCF_IPV6 0x00000001 /* IPv6 packet */ @@ -965,10 +976,11 @@ extern int soopt_mcopyin(struct sockopt *sopt, struct mbuf *m); extern int soopt_mcopyout(struct sockopt *sopt, struct mbuf *m); extern boolean_t so_cache_timer(void); +extern void mptcp_fallback_sbdrop(struct socket *so, struct mbuf *m, int len); extern void mptcp_preproc_sbdrop(struct socket *, struct mbuf *, unsigned int); extern void mptcp_postproc_sbdrop(struct mbuf *, u_int64_t, u_int32_t, u_int32_t); -extern int mptcp_adj_rmap(struct socket *, struct mbuf *); +extern void mptcp_adj_rmap(struct socket *, struct mbuf *, int); extern void netpolicy_post_msg(uint32_t, struct netpolicy_event_data *, uint32_t); diff --git a/bsd/sys/sockio.h b/bsd/sys/sockio.h index f098e7a13..78f3b6a7f 100644 --- a/bsd/sys/sockio.h +++ b/bsd/sys/sockio.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ @@ -151,14 +151,14 @@ #define SIOCGIFBOND _IOWR('i', 71, struct ifreq) /* get bond if config */ #ifdef PRIVATE -/* +/* * temporary control calls to attach/detach IP to/from an ethernet interface */ #define SIOCPROTOATTACH _IOWR('i', 80, struct ifreq) /* attach proto to interface */ #define SIOCPROTODETACH _IOWR('i', 81, struct ifreq) /* detach proto from interface */ #endif /* PRIVATE */ -#define SIOCSIFCAP _IOW('i', 90, struct ifreq) /* set IF features */ +#define SIOCSIFCAP _IOW('i', 90, struct ifreq) /* set IF features */ #define SIOCGIFCAP _IOWR('i', 91, struct ifreq) /* get IF features */ #define SIOCIFCREATE _IOWR('i', 120, struct ifreq) /* create clone if */ @@ -268,7 +268,9 @@ #define SIOCSIFPROBECONNECTIVITY _IOWR('i', 171, struct ifreq) /* Start/Stop probes to check connectivity */ #define SIOCGIFPROBECONNECTIVITY _IOWR('i', 172, struct ifreq) /* check if connectivity probes are enabled */ +#endif /* PRIVATE */ #define SIOCGIFFUNCTIONALTYPE _IOWR('i', 173, struct ifreq) /* get interface functional type */ +#ifdef PRIVATE #define SIOCSIFNETSIGNATURE _IOWR('i', 174, struct if_nsreq) #define SIOCGIFNETSIGNATURE _IOWR('i', 175, struct if_nsreq) @@ -293,11 +295,21 @@ #define SIOCSIFDISABLEOUTPUT _IOWR('i', 187, struct ifreq) #define SIOCGIFAGENTLIST _IOWR('i', 190, struct netagentlist_req) /* Get netagent dump */ + #ifdef BSD_KERNEL_PRIVATE #define SIOCGIFAGENTLIST32 _IOWR('i', 190, struct netagentlist_req32) #define SIOCGIFAGENTLIST64 _IOWR('i', 190, struct netagentlist_req64) #endif /* BSD_KERNEL_PRIVATE */ +#define SIOCSIFLOWINTERNET _IOWR('i', 191, struct ifreq) +#define SIOCGIFLOWINTERNET _IOWR('i', 192, struct ifreq) + +#if INET6 +#define SIOCGIFNAT64PREFIX _IOWR('i', 193, struct if_nat64req) +#define SIOCSIFNAT64PREFIX _IOWR('i', 194, struct if_nat64req) +#endif +#define SIOCGIFNEXUS _IOWR('i', 195, struct if_nexusreq) + #endif /* PRIVATE */ #endif /* !_SYS_SOCKIO_H_ */ diff --git a/bsd/sys/spawn.h b/bsd/sys/spawn.h index edd7020d6..bcf1d6367 100644 --- a/bsd/sys/spawn.h +++ b/bsd/sys/spawn.h @@ -67,6 +67,9 @@ #define _POSIX_SPAWN_ALLOW_DATA_EXEC 0x2000 #endif /* PRIVATE */ #define POSIX_SPAWN_CLOEXEC_DEFAULT 0x4000 +#ifdef PRIVATE +#define _POSIX_SPAWN_HIGH_BITS_ASLR 0x8000 +#endif /* PRIVATE */ /* * Possible values to be set for the process control actions on resource starvation. diff --git a/bsd/sys/stat.h b/bsd/sys/stat.h index 75b8d9322..183fdd207 100644 --- a/bsd/sys/stat.h +++ b/bsd/sys/stat.h @@ -368,7 +368,15 @@ struct user32_stat64 { __uint32_t st_gen; /* file generation number */ __uint32_t st_lspare; /* RESERVED: DO NOT USE! */ __int64_t st_qspare[2]; /* RESERVED: DO NOT USE! */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Applying attributes here causes a mismatch with the user-space struct stat64 + */ +}; +#else } __attribute__((packed,aligned(4))); +#endif extern void munge_user64_stat64(struct stat64 *sbp, struct user64_stat64 *usbp); extern void munge_user32_stat64(struct stat64 *sbp, struct user32_stat64 *usbp); @@ -474,7 +482,10 @@ extern void munge_user32_stat64(struct stat64 *sbp, struct user32_stat64 *usbp); notifications for deletes or renames for files which have UF_TRACKED set. */ #define UF_TRACKED 0x00000040 -/* Bits 0x0080 through 0x4000 are currently undefined. */ +#define UF_DATAVAULT 0x00000080 /* entitlement required for reading */ + /* and writing */ + +/* Bits 0x0100 through 0x4000 are currently undefined. */ #define UF_HIDDEN 0x00008000 /* hint that this item should not be */ /* displayed in a GUI */ /* @@ -485,7 +496,7 @@ extern void munge_user32_stat64(struct stat64 *sbp, struct user32_stat64 *usbp); #define SF_ARCHIVED 0x00010000 /* file is archived */ #define SF_IMMUTABLE 0x00020000 /* file may not be changed */ #define SF_APPEND 0x00040000 /* writes to file may only append */ -#define SF_RESTRICTED 0x00080000 /* restricted access */ +#define SF_RESTRICTED 0x00080000 /* entitlement required for writing */ #define SF_NOUNLINK 0x00100000 /* Item may not be removed, renamed or mounted on */ /* @@ -523,6 +534,13 @@ mode_t umask(mode_t); int fchmodat(int, const char *, mode_t, int) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); int fstatat(int, const char *, struct stat *, int) __DARWIN_INODE64(fstatat) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); int mkdirat(int, const char *, mode_t) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); + +#define UTIME_NOW -1 +#define UTIME_OMIT -2 + +int futimens(int __fd, const struct timespec __times[2]) __API_AVAILABLE(macosx(10.13), ios(11.0), tvos(11.0), watchos(4.0)); +int utimensat(int __fd, const char *__path, const struct timespec __times[2], + int __flag) __API_AVAILABLE(macosx(10.13), ios(11.0), tvos(11.0), watchos(4.0)); #endif #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) diff --git a/bsd/sys/subr_prf.h b/bsd/sys/subr_prf.h index c3408ef9f..966aedf5a 100644 --- a/bsd/sys/subr_prf.h +++ b/bsd/sys/subr_prf.h @@ -83,7 +83,7 @@ #define TOSTR 0x00000008 /* output to string */ #define TOLOGLOCKED 0x00000010 /* output to log (log lock held) */ -extern int prf(const char *fmt, va_list ap, int flags, struct tty *ttyp); +extern int prf(const char *fmt, va_list ap, int flags, struct tty *ttyp) __printflike(1,0); #endif /* __APPLE_API_PRIVATE */ #endif /* KERNEL_PRIVATE */ diff --git a/bsd/sys/sysctl.h b/bsd/sys/sysctl.h index 4ccc92c72..3acef8603 100644 --- a/bsd/sys/sysctl.h +++ b/bsd/sys/sysctl.h @@ -86,6 +86,7 @@ #ifndef XNU_KERNEL_PRIVATE #include #endif + #endif #include #include @@ -372,6 +373,12 @@ __END_DECLS ptr, 0, sysctl_handle_long, "L", descr); \ typedef char _sysctl_##parent##_##name##_size_check[(__builtin_constant_p(ptr) || sizeof(*(ptr)) == sizeof(long)) ? 0 : -1]; +/* Oid for a unsigned long. The pointer must be non NULL. */ +#define SYSCTL_ULONG(parent, nbr, name, access, ptr, descr) \ + SYSCTL_OID(parent, nbr, name, CTLTYPE_INT|access, \ + ptr, 0, sysctl_handle_long, "LU", descr); \ + typedef char _sysctl_##parent##_##name##_size_check[(__builtin_constant_p(ptr) || sizeof(*(ptr)) == sizeof(unsigned long)) ? 0 : -1]; + /* Oid for a quad. The pointer must be non NULL. */ #define SYSCTL_QUAD(parent, nbr, name, access, ptr, descr) \ SYSCTL_OID(parent, nbr, name, CTLTYPE_QUAD|access, \ @@ -414,6 +421,27 @@ SYSCTL_DECL(_user); SYSCTL_DECL(_hw_features); #endif + +#ifndef SYSCTL_SKMEM_UPDATE_FIELD + +#define SYSCTL_SKMEM 0 +#define SYSCTL_SKMEM_UPDATE_FIELD(field, value) +#define SYSCTL_SKMEM_UPDATE_AT_OFFSET(offset, value) +#define SYSCTL_SKMEM_INT(parent, oid, sysctl_name, access, ptr, offset, descr) \ + SYSCTL_INT(parent, oid, sysctl_name, access, ptr, 0, descr) + +#define SYSCTL_SKMEM_TCP_INT(oid, sysctl_name, access, variable_type, \ + variable_name, initial_value, descr) \ + variable_type variable_name = initial_value; \ + SYSCTL_SKMEM_INT(_net_inet_tcp, oid, sysctl_name, access, \ + &variable_name, 0, descr) + +#else /* SYSCTL_SKMEM_UPDATE_FIELD */ +#define SYSCTL_SKMEM 1 +#endif /* SYSCTL_SKMEM_UPDATE_FIELD */ + + + #endif /* KERNEL */ #ifdef XNU_KERNEL_PRIVATE diff --git a/bsd/sys/sysent.h b/bsd/sys/sysent.h index 8182994d3..6d529d6f5 100644 --- a/bsd/sys/sysent.h +++ b/bsd/sys/sysent.h @@ -38,6 +38,8 @@ typedef int32_t sy_call_t(struct proc *, void *, int *); #if CONFIG_REQUIRES_U32_MUNGING typedef void sy_munge_t(void *); +#elif __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +typedef int sy_munge_t(const void *, void *); #endif struct sysent { /* system call table */ diff --git a/bsd/sys/syslog.h b/bsd/sys/syslog.h index 2449ad379..42ff28cc6 100644 --- a/bsd/sys/syslog.h +++ b/bsd/sys/syslog.h @@ -229,7 +229,11 @@ __BEGIN_DECLS void closelog(void); void openlog(const char *, int, int); int setlogmask(int); +#if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __DARWIN_C_LEVEL >= __DARWIN_C_FULL +void syslog(int, const char *, ...) __printflike(2, 3) __not_tail_called __DARWIN_ALIAS_STARTING(__MAC_10_13, __IPHONE_NA, __DARWIN_EXTSN(syslog)); +#else void syslog(int, const char *, ...) __printflike(2, 3) __not_tail_called; +#endif #if __DARWIN_C_LEVEL >= __DARWIN_C_FULL void vsyslog(int, const char *, __darwin_va_list) __printflike(2, 0) __not_tail_called; #endif @@ -323,7 +327,7 @@ __BEGIN_DECLS void log(int, const char *, ...); #ifdef XNU_KERNEL_PRIVATE void logpri(int); -int vaddlog(const char *, va_list); +int vaddlog(const char *, va_list) __printflike(1,0); void logtime(time_t); #endif /* XNU_KERNEL_PRIVATE */ diff --git a/bsd/sys/systm.h b/bsd/sys/systm.h index e346413c8..fb2badf23 100644 --- a/bsd/sys/systm.h +++ b/bsd/sys/systm.h @@ -124,6 +124,9 @@ extern const char copyright[]; /* system copyright */ extern int boothowto; /* reboot flags, from console subsystem */ extern int show_space; extern int minimalboot; +#if CONFIG_EMBEDDED +extern int darkboot; +#endif extern const int nblkdev; /* number of entries in bdevsw */ extern const int nchrdev; /* number of entries in cdevsw */ @@ -157,7 +160,7 @@ void realitexpire(struct proc *); int hzto(struct timeval *tv); void tablefull(const char *); int kvprintf(char const *, void (*)(int, void*), void *, int, - __darwin_va_list); + __darwin_va_list) __printflike(1,0); void uprintf(const char *, ...) __printflike(1,2); int copywithin(void *saddr, void *daddr, size_t len); int64_t fulong(user_addr_t addr); diff --git a/bsd/sys/systrace_args.h b/bsd/sys/systrace_args.h new file mode 100644 index 000000000..5c1ef577e --- /dev/null +++ b/bsd/sys/systrace_args.h @@ -0,0 +1,36 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2017 Apple, Inc. All rights reserved. + * Use is subject to license terms. + */ +#ifndef _SYS_SYSTRACE_ARGS_H +#define _SYS_SYSTRACE_ARGS_H + +#include +#include + +void systrace_args(int sysnum, void *params, uint64_t *uarg); +void systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz); +void systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz); + +#endif /* _SYS_SYSTRACE_ARGS_H */ diff --git a/bsd/sys/time.h b/bsd/sys/time.h index 85e7fe3df..97a536416 100644 --- a/bsd/sys/time.h +++ b/bsd/sys/time.h @@ -193,6 +193,10 @@ struct clockinfo { }; #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ +#ifdef XNU_KERNEL_PRIVATE +#define SETTIME_ENTITLEMENT "com.apple.private.settime" +#endif + #ifdef KERNEL #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) diff --git a/bsd/sys/timex.h b/bsd/sys/timex.h new file mode 100644 index 000000000..5e8a3bfdc --- /dev/null +++ b/bsd/sys/timex.h @@ -0,0 +1,212 @@ +/*- + *********************************************************************** + * * + * Copyright (c) David L. Mills 1993-2001 * + * Copyright (c) Poul-Henning Kamp 2000-2001 * + * * + * Permission to use, copy, modify, and distribute this software and * + * its documentation for any purpose and without fee is hereby * + * granted, provided that the above copyright notice appears in all * + * copies and that both the copyright notice and this permission * + * notice appear in supporting documentation, and that the name * + * University of Delaware not be used in advertising or publicity * + * pertaining to distribution of the software without specific, * + * written prior permission. The University of Delaware makes no * + * representations about the suitability this software for any * + * purpose. It is provided "as is" without express or implied * + * warranty. * + * * + *********************************************************************** + * + * $FreeBSD$ + * + * This header file defines the Network Time Protocol (NTP) interfaces + * for user and daemon application programs. + * + * This file was originally created 17 Sep 93 by David L. Mills, Professor + * of University of Delaware, building on work which had already been ongoing + * for a decade and a half at that point in time. + * + * In 2000 the APIs got a upgrade from microseconds to nanoseconds, + * a joint work between Poul-Henning Kamp and David L. Mills. + * + */ + +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _SYS_TIMEX_H_ +#define _SYS_TIMEX_H_ 1 + +#include + +#define NTP_API 4 /* NTP API version */ + +/* + * The following defines establish the performance envelope of the + * kernel discipline loop. Phase or frequency errors greater than + * NAXPHASE or MAXFREQ are clamped to these maxima. For update intervals + * less than MINSEC, the loop always operates in PLL mode; while, for + * update intervals greater than MAXSEC, the loop always operates in FLL + * mode. Between these two limits the operating mode is selected by the + * STA_FLL bit in the status word. + */ + +#define MAXPHASE 500000000L /* max phase error (ns) */ +#define MAXFREQ 500000L /* max freq error (ns/s) */ +#define MINSEC 256 /* min FLL update interval (s) */ +#define MAXSEC 2048 /* max PLL update interval (s) */ +#define NANOSECOND 1000000000L /* nanoseconds in one second */ +#define SCALE_PPM (65536 / 1000) /* crude ns/s to scaled PPM */ +#define MAXTC 10 /* max time constant */ + +/* Codes for PPS (pulse-per-second) signals or leap seconds are not used but kept + * unchanged and commented for future compatibility. + */ + +/* + * Control mode codes (timex.modes) + */ +#define MOD_OFFSET 0x0001 /* set time offset */ +#define MOD_FREQUENCY 0x0002 /* set frequency offset */ +#define MOD_MAXERROR 0x0004 /* set maximum time error */ +#define MOD_ESTERROR 0x0008 /* set estimated time error */ +#define MOD_STATUS 0x0010 /* set clock status bits */ +#define MOD_TIMECONST 0x0020 /* set PLL time constant */ +#define MOD_PPSMAX 0x0040 /* set PPS maximum averaging time */ +#define MOD_TAI 0x0080 /* set TAI offset */ +#define MOD_MICRO 0x1000 /* select microsecond resolution */ +#define MOD_NANO 0x2000 /* select nanosecond resolution */ +#define MOD_CLKB 0x4000 /* select clock B */ +#define MOD_CLKA 0x8000 /* select clock A */ + +/* + * Status codes (timex.status) + */ +#define STA_PLL 0x0001 /* enable PLL updates (rw) */ +#define STA_PPSFREQ 0x0002 /* enable PPS freq discipline (rw) */ +#define STA_PPSTIME 0x0004 /* enable PPS time discipline (rw) */ +#define STA_FLL 0x0008 /* enable FLL mode (rw) */ +#define STA_INS 0x0010 /* insert leap (rw) */ +#define STA_DEL 0x0020 /* delete leap (rw) */ +#define STA_UNSYNC 0x0040 /* clock unsynchronized (rw) */ +#define STA_FREQHOLD 0x0080 /* hold frequency (rw) */ +#define STA_PPSSIGNAL 0x0100 /* PPS signal present (ro) */ +#define STA_PPSJITTER 0x0200 /* PPS signal jitter exceeded (ro) */ +#define STA_PPSWANDER 0x0400 /* PPS signal wander exceeded (ro) */ +#define STA_PPSERROR 0x0800 /* PPS signal calibration error (ro) */ +#define STA_CLOCKERR 0x1000 /* clock hardware fault (ro) */ +#define STA_NANO 0x2000 /* resolution (0 = us, 1 = ns) (ro) */ +#define STA_MODE 0x4000 /* mode (0 = PLL, 1 = FLL) (ro) */ +#define STA_CLK 0x8000 /* clock source (0 = A, 1 = B) (ro) */ + +#define STA_RONLY (STA_PPSSIGNAL | STA_PPSJITTER | STA_PPSWANDER | \ + STA_PPSERROR | STA_CLOCKERR | STA_NANO | STA_MODE | STA_CLK) + +#define STA_SUPPORTED (STA_PLL | STA_FLL | STA_UNSYNC | STA_FREQHOLD | \ + STA_CLOCKERR | STA_NANO | STA_MODE | STA_CLK) + +/* + * Clock states (ntptimeval.time_state) + */ +#define TIME_OK 0 /* no leap second warning */ +#define TIME_INS 1 /* insert leap second warning */ +#define TIME_DEL 2 /* delete leap second warning */ +#define TIME_OOP 3 /* leap second in progress */ +#define TIME_WAIT 4 /* leap second has occurred */ +#define TIME_ERROR 5 /* error (see status word) */ + +/* + * NTP user interface -- ntp_gettime - used to read kernel clock values + */ +struct ntptimeval { + struct timespec time; /* current time (ns) (ro) */ + long maxerror; /* maximum error (us) (ro) */ + long esterror; /* estimated error (us) (ro) */ + long tai; /* TAI offset */ + int time_state; /* time status */ +}; + +/* + * NTP daemon interface -- ntp_adjtime -- used to discipline CPU clock + * oscillator and control/determine status. + * + * Note: The offset, precision and jitter members are in microseconds if + * STA_NANO is zero and nanoseconds if not. + */ +struct timex { + unsigned int modes; /* clock mode bits (wo) */ + long offset; /* time offset (ns/us) (rw) */ + long freq; /* frequency offset (scaled PPM) (rw) */ + long maxerror; /* maximum error (us) (rw) */ + long esterror; /* estimated error (us) (rw) */ + int status; /* clock status bits (rw) */ + long constant; /* poll interval (log2 s) (rw) */ + long precision; /* clock precision (ns/us) (ro) */ + long tolerance; /* clock frequency tolerance (scaled + * PPM) (ro) */ + /* + * The following read-only structure members are used by + * the PPS signal discipline that is currently not supported. + * They are included for compatibility. + */ + long ppsfreq; /* PPS frequency (scaled PPM) (ro) */ + long jitter; /* PPS jitter (ns/us) (ro) */ + int shift; /* interval duration (s) (shift) (ro) */ + long stabil; /* PPS stability (scaled PPM) (ro) */ + long jitcnt; /* jitter limit exceeded (ro) */ + long calcnt; /* calibration intervals (ro) */ + long errcnt; /* calibration errors (ro) */ + long stbcnt; /* stability limit exceeded (ro) */ +}; + +#ifdef KERNEL +#ifdef XNU_KERNEL_PRIVATE +#include +#include +#include +#include +#include + +int64_t ntp_get_freq(void); +void ntp_update_second(int64_t *adjustment, clock_sec_t secs); +void ntp_init(void); +#endif +#else /* !_KERNEL */ +#include + +__BEGIN_DECLS +#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) +int ntp_adjtime(struct timex *); +int ntp_gettime(struct ntptimeval *); +#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ +__END_DECLS +#endif /* KERNEL */ + + +#endif /* !_SYS_TIMEX_H_ */ diff --git a/bsd/sys/tprintf.h b/bsd/sys/tprintf.h index cf066876d..a9d704a4e 100644 --- a/bsd/sys/tprintf.h +++ b/bsd/sys/tprintf.h @@ -74,7 +74,7 @@ typedef struct session *tpr_t; __BEGIN_DECLS tpr_t tprintf_open(struct proc *); void tprintf_close(tpr_t); -void tprintf(tpr_t, const char *fmt, ...); +void tprintf(tpr_t, const char *fmt, ...) __printflike(2,3); __END_DECLS #endif /* __APPLE_API_UNSTABLE */ diff --git a/bsd/sys/tty.h b/bsd/sys/tty.h index 7fdcd02c7..fb9bcbfd7 100644 --- a/bsd/sys/tty.h +++ b/bsd/sys/tty.h @@ -103,7 +103,6 @@ struct clist { #define TTYCLSIZE 1024 #endif - /* * Per-tty structure. * @@ -331,9 +330,18 @@ void ttyfree(struct tty *); void ttysetpgrphup(struct tty *tp); void ttyclrpgrphup(struct tty *tp); +#ifdef XNU_KERNEL_PRIVATE +extern void ttyhold(struct tty *tp); + +#define TTY_LOCK_OWNED(tp) LCK_MTX_ASSERT(&tp->t_lock, LCK_MTX_ASSERT_OWNED) +#define TTY_LOCK_NOTOWNED(tp) LCK_MTX_ASSERT(&tp->t_lock, LCK_MTX_ASSERT_NOTOWNED) + +#define PTS_MAJOR 4 +#define PTC_MAJOR 5 +#endif /* defined(XNU_KERNEL_PRIVATE) */ + __END_DECLS #endif /* KERNEL */ - #endif /* !_SYS_TTY_H_ */ diff --git a/bsd/sys/types.h b/bsd/sys/types.h index a9fc63938..f11f23ff2 100644 --- a/bsd/sys/types.h +++ b/bsd/sys/types.h @@ -81,9 +81,9 @@ #include #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -typedef unsigned char u_char; -typedef unsigned short u_short; -typedef unsigned int u_int; +#include +#include +#include #ifndef _U_LONG typedef unsigned long u_long; #define _U_LONG @@ -96,7 +96,8 @@ typedef u_int64_t u_quad_t; /* quads */ typedef int64_t quad_t; typedef quad_t * qaddr_t; -typedef char * caddr_t; /* core address */ +#include /* core address */ + typedef int32_t daddr_t; /* disk address */ #include /* device number */ diff --git a/bsd/sys/ubc.h b/bsd/sys/ubc.h index 278c84889..0699b5b03 100644 --- a/bsd/sys/ubc.h +++ b/bsd/sys/ubc.h @@ -129,7 +129,9 @@ cl_direct_read_lock_t *cluster_lock_direct_read(vnode_t vp, lck_rw_type_t exclus void cluster_unlock_direct_read(cl_direct_read_lock_t *lck); /* UPL routines */ +#ifndef XNU_KERNEL_PRIVATE int ubc_create_upl(vnode_t, off_t, int, upl_t *, upl_page_info_t **, int); +#endif /* XNU_KERNEL_PRIVATE */ int ubc_upl_map(upl_t, vm_offset_t *); int ubc_upl_unmap(upl_t); int ubc_upl_commit(upl_t); @@ -147,6 +149,11 @@ errno_t mach_to_bsd_errno(kern_return_t mach_err); #ifdef KERNEL_PRIVATE +int ubc_create_upl_external(vnode_t, off_t, int, upl_t *, upl_page_info_t **, int); +#ifdef XNU_KERNEL_PRIVATE +int ubc_create_upl_kernel(vnode_t, off_t, int, upl_t *, upl_page_info_t **, int, vm_tag_t); +#endif /* XNU_KERNEL_PRIVATE */ + __attribute__((pure)) boolean_t ubc_is_mapped(const struct vnode *, boolean_t *writable); __attribute__((pure)) boolean_t ubc_is_mapped_writable(const struct vnode *); diff --git a/bsd/sys/ubc_internal.h b/bsd/sys/ubc_internal.h index ba3d848c1..3724348ad 100644 --- a/bsd/sys/ubc_internal.h +++ b/bsd/sys/ubc_internal.h @@ -96,6 +96,8 @@ struct cl_writebehind { struct cs_hash; +uint8_t cs_hash_type(struct cs_hash const *); + struct cs_blob { struct cs_blob *csb_next; cpu_type_t csb_cpu_type; @@ -116,6 +118,9 @@ struct cs_blob { const char *csb_teamid; const CS_GenericBlob *csb_entitlements_blob; /* raw blob, subrange of csb_mem_kaddr */ void * csb_entitlements; /* The entitlements as an OSDictionary */ + unsigned int csb_signer_type; + + /* The following two will be replaced by the csb_signer_type. */ unsigned int csb_platform_binary:1; unsigned int csb_platform_path:1; }; diff --git a/bsd/sys/unistd.h b/bsd/sys/unistd.h index b27ab38f8..e373f2feb 100644 --- a/bsd/sys/unistd.h +++ b/bsd/sys/unistd.h @@ -180,11 +180,14 @@ struct accessx_descriptor { #include #include #include <_types/_uint64_t.h> +#include <_types/_uint32_t.h> #include __BEGIN_DECLS -int getattrlistbulk(int, void *, void *, size_t, uint64_t) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); +int getattrlistbulk(int, void *, void *, size_t, uint64_t) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); +int getattrlistat(int, const char *, void *, void *, size_t, unsigned long) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); +int setattrlistat(int, const char *, void *, void *, size_t, uint32_t) __OSX_AVAILABLE(10.13) __IOS_AVAILABLE(11.0) __TVOS_AVAILABLE(11.0) __WATCHOS_AVAILABLE(4.0); __END_DECLS @@ -208,7 +211,6 @@ int linkat(int, const char *, int, const char *, int) __OSX_AVAILABLE_STARTING(_ ssize_t readlinkat(int, const char *, char *, size_t) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); int symlinkat(const char *, int, const char *) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); int unlinkat(int, const char *, int) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); -int getattrlistat(int, const char *, void *, void *, size_t, unsigned long) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); __END_DECLS diff --git a/bsd/sys/unpcb.h b/bsd/sys/unpcb.h index 936e99801..9c66ca69b 100644 --- a/bsd/sys/unpcb.h +++ b/bsd/sys/unpcb.h @@ -67,6 +67,7 @@ #include #include #include +#include /* * Protocol control block for an active @@ -203,6 +204,7 @@ struct xunpcb { u_quad_t xu_alignment_hack; }; +#if !CONFIG_EMBEDDED struct xunpcb64_list_entry { u_int64_t le_next; @@ -236,6 +238,7 @@ struct xunpcb64 { struct xsocket64 xu_socket; }; +#endif /* !CONFIG_EMBEDDED */ #pragma pack() diff --git a/bsd/sys/user.h b/bsd/sys/user.h index 588a5e642..92b235bb9 100644 --- a/bsd/sys/user.h +++ b/bsd/sys/user.h @@ -75,7 +75,9 @@ struct waitq_set; #include #endif #ifdef XNU_KERNEL_PRIVATE +#include #include +#include #include #endif #include /* XXX */ @@ -84,6 +86,7 @@ struct waitq_set; #ifdef KERNEL #ifdef BSD_KERNEL_PRIVATE #include /* for uu_kwe entry */ +#include #endif /* BSD_KERNEL_PRIVATE */ #ifdef __APPLE_API_PRIVATE #include @@ -134,14 +137,13 @@ struct uthread { kevent_callback_t call; /* per-event callback */ kqueue_continue_t cont; /* whole call continuation */ filt_process_data_t process_data; /* needed for filter processing */ - uint8_t servicer_qos_index; /* requested qos index of servicer */ uint64_t deadline; /* computed deadline for operation */ void *data; /* caller's private data */ } ss_kqueue_scan; /* saved state for kevent_scan() */ struct _kevent { struct _kqueue_scan scan; /* space for the generic data */ struct fileproc *fp; /* fileproc we hold iocount on */ - int fd; /* filedescriptor for kq */ + int fd; /* fd for fileproc (if held) */ int eventcount; /* user-level event count */ int eventout; /* number of events output */ struct filt_process_s process_data; /* space for process data fed thru */ @@ -216,8 +218,10 @@ struct uthread { int uu_dupfd; /* fd in fdesc_open/dupfdopen */ int uu_defer_reclaims; - unsigned int uu_kqueue_bound; /* qos index we are bound to service */ - unsigned int uu_kqueue_flags; /* if so, the flags being using */ + struct kqueue *uu_kqueue_bound; /* kqueue we are bound to service */ + unsigned int uu_kqueue_qos_index; /* qos index we are bound to service */ + unsigned int uu_kqueue_flags; /* the flags we are using */ + boolean_t uu_kqueue_override_is_sync; /* sync qos override applied to servicer */ #ifdef JOE_DEBUG int uu_iocount; diff --git a/bsd/sys/vm.h b/bsd/sys/vm.h index a4e3df795..752ef89b5 100644 --- a/bsd/sys/vm.h +++ b/bsd/sys/vm.h @@ -136,6 +136,10 @@ struct user_vmspace { #include #else /* BSD_KERNEL_PRIVATE */ + +#include /* caddr_t */ +#include /* int32_t */ + /* just to keep kinfo_proc happy */ /* NOTE: Pointer fields are size variant for LP64 */ struct vmspace { diff --git a/bsd/sys/vnode.h b/bsd/sys/vnode.h index 0baabcbd7..74e0704e8 100644 --- a/bsd/sys/vnode.h +++ b/bsd/sys/vnode.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -117,7 +117,7 @@ enum vtagtype { */ #define VNODE_READ 0x01 #define VNODE_WRITE 0x02 - +#define VNODE_BLOCKMAP_NO_TRACK 0x04 // APFS Fusion: Do not track this request /* flags for VNOP_ALLOCATE */ @@ -551,6 +551,55 @@ struct vnode_trigger_param { #define VNODE_ATTR_va_private_size (1LL<<43) /* 80000000000 */ #define VNODE_ATTR_BIT(n) (VNODE_ATTR_ ## n) + +/* + * ALL of the attributes. + */ +#define VNODE_ATTR_ALL (VNODE_ATTR_BIT(va_rdev) | \ + VNODE_ATTR_BIT(va_nlink) | \ + VNODE_ATTR_BIT(va_total_size) | \ + VNODE_ATTR_BIT(va_total_alloc) | \ + VNODE_ATTR_BIT(va_data_size) | \ + VNODE_ATTR_BIT(va_data_alloc) | \ + VNODE_ATTR_BIT(va_iosize) | \ + VNODE_ATTR_BIT(va_uid) | \ + VNODE_ATTR_BIT(va_gid) | \ + VNODE_ATTR_BIT(va_mode) | \ + VNODE_ATTR_BIT(va_flags) | \ + VNODE_ATTR_BIT(va_acl) | \ + VNODE_ATTR_BIT(va_create_time) | \ + VNODE_ATTR_BIT(va_access_time) | \ + VNODE_ATTR_BIT(va_modify_time) | \ + VNODE_ATTR_BIT(va_change_time) | \ + VNODE_ATTR_BIT(va_backup_time) | \ + VNODE_ATTR_BIT(va_fileid) | \ + VNODE_ATTR_BIT(va_linkid) | \ + VNODE_ATTR_BIT(va_parentid) | \ + VNODE_ATTR_BIT(va_fsid) | \ + VNODE_ATTR_BIT(va_filerev) | \ + VNODE_ATTR_BIT(va_gen) | \ + VNODE_ATTR_BIT(va_encoding) | \ + VNODE_ATTR_BIT(va_type) | \ + VNODE_ATTR_BIT(va_name) | \ + VNODE_ATTR_BIT(va_uuuid) | \ + VNODE_ATTR_BIT(va_guuid) | \ + VNODE_ATTR_BIT(va_nchildren) | \ + VNODE_ATTR_BIT(va_dirlinkcount) | \ + VNODE_ATTR_BIT(va_addedtime) | \ + VNODE_ATTR_BIT(va_dataprotect_class) | \ + VNODE_ATTR_BIT(va_dataprotect_flags) | \ + VNODE_ATTR_BIT(va_document_id) | \ + VNODE_ATTR_BIT(va_devid) | \ + VNODE_ATTR_BIT(va_objtype) | \ + VNODE_ATTR_BIT(va_objtag) | \ + VNODE_ATTR_BIT(va_user_access) | \ + VNODE_ATTR_BIT(va_finderinfo) | \ + VNODE_ATTR_BIT(va_rsrc_length) | \ + VNODE_ATTR_BIT(va_rsrc_alloc) | \ + VNODE_ATTR_BIT(va_fsid64) | \ + VNODE_ATTR_BIT(va_write_gencount) | \ + VNODE_ATTR_BIT(va_private_size)) + /* * Read-only attributes. */ @@ -570,7 +619,6 @@ struct vnode_trigger_param { VNODE_ATTR_BIT(va_type) | \ VNODE_ATTR_BIT(va_nchildren) | \ VNODE_ATTR_BIT(va_dirlinkcount) | \ - VNODE_ATTR_BIT(va_addedtime) | \ VNODE_ATTR_BIT(va_devid) | \ VNODE_ATTR_BIT(va_objtype) | \ VNODE_ATTR_BIT(va_objtag) | \ @@ -754,7 +802,8 @@ extern int vttoif_tab[]; #define VNODE_READDIR_NAMEMAX 0x0008 /* For extended readdir, try to limit names to NAME_MAX bytes */ /* VNOP_CLONEFILE flags: */ -#define VNODE_CLONEFILE_DEFAULT 0x0000 +#define VNODE_CLONEFILE_DEFAULT 0x0000 +#define VNODE_CLONEFILE_NOOWNERCOPY 0x0001 /* Don't copy ownership information */ #define NULLVP ((struct vnode *)NULL) @@ -1045,7 +1094,6 @@ int vnode_ischr(vnode_t vp); */ int vnode_isswap(vnode_t vp); -#ifdef __APPLE_API_UNSTABLE /*! @function vnode_isnamedstream @abstract Determine if a vnode is a named stream. @@ -1053,7 +1101,6 @@ int vnode_isswap(vnode_t vp); @return Nonzero if the vnode is a named stream, 0 otherwise. */ int vnode_isnamedstream(vnode_t vp); -#endif /*! @function vnode_ismountedon @@ -1342,7 +1389,7 @@ int vfs_context_issignal(vfs_context_t ctx, sigset_t mask); @function vfs_context_suser @abstract Determine if a vfs_context_t corresponds to the superuser. @param ctx Context to examine. - @return Nonzero if context belongs to superuser, 0 otherwise. + @return 0 if context belongs to superuser, EPERM otherwise. */ int vfs_context_suser(vfs_context_t ctx); @@ -1588,6 +1635,21 @@ int vnode_isdyldsharedcache(vnode_t vp); */ int vn_getpath_fsenter(struct vnode *vp, char *pathbuf, int *len); +/*! + @function vn_getpath_fsenter_with_parent + @abstract Attempt to get a vnode's path by entering the file system if needed given a vnode and it's directory vnode. + @discussion Same as vn_getpath_fsenter but is given the directory vnode as well as the target vnode. Used +to get the path from the vnode while performing rename, rmdir, and unlink. This is done to avoid potential +dead lock if another thread is doing a forced unmount. + @param dvp Containing directory vnode. Must be holding an IO count. + @param vp Vnode whose path to get. Must be holding an IO count. + @param pathbuf Buffer in which to store path. + @param len Destination for length of resulting path string. Result will include NULL-terminator in count--that is, "len" + will be strlen(pathbuf) + 1. + @return 0 for success or an error. +*/ +int vn_getpath_fsenter_with_parent(struct vnode *dvp, struct vnode *vp, char *pathbuf, int *len); + #endif /* KERNEL_PRIVATE */ #define VNODE_UPDATE_PARENT 0x01 diff --git a/bsd/sys/vnode_if.h b/bsd/sys/vnode_if.h index 44dd90957..75f5cdd7e 100644 --- a/bsd/sys/vnode_if.h +++ b/bsd/sys/vnode_if.h @@ -1374,6 +1374,7 @@ struct vnop_clonefile_args { vnode_t sdvp, /* source directory vnode pointer (optional) */ mount_t mp, /* mount point of filesystem */ dir_clone_authorizer_op_t vattr_op, /* specific operation requested : setup, authorization or cleanup */ + uint32_t flags, /* needs to have the value passed to a_flags */ vfs_context_t ctx, /* As passed to VNOP */ void *reserved); /* Always NULL */ void *a_reserved; /* Currently unused */ @@ -1690,6 +1691,9 @@ errno_t VNOP_SETLABEL(vnode_t, struct label *, vfs_context_t); enum nsoperation { NS_OPEN, NS_CREATE, NS_DELETE }; +/* a_flags for vnop_getnamedstream_args: */ +#define NS_GETRAWENCRYPTED 0x00000001 + struct vnop_getnamedstream_args { struct vnodeop_desc *a_desc; vnode_t a_vp; @@ -1712,7 +1716,7 @@ struct vnop_getnamedstream_args { @param operation Operation to perform. In HFS and AFP, this parameter is only considered as follows: if the resource fork has not been opened and the operation is not NS_OPEN, fail with ENOATTR. Currently only passed as NS_OPEN by VFS. - @param flags Currently unused. + @param flags Flags used to control getnamedstream behavior. Currently only used for raw-encrypted-requests. @param ctx Context to authenticate for getting named stream. @return 0 for success, else an error code. */ diff --git a/bsd/sys/vnode_internal.h b/bsd/sys/vnode_internal.h index bcc58053b..d06102237 100644 --- a/bsd/sys/vnode_internal.h +++ b/bsd/sys/vnode_internal.h @@ -451,7 +451,7 @@ int vn_authorize_mkdir(vnode_t, struct componentname *, struct vnode_attr *, vfs int vn_authorize_null(vnode_t, struct componentname *, struct vnode_attr *, vfs_context_t, void*); int vnode_attr_authorize_dir_clone(struct vnode_attr *vap, kauth_action_t action, struct vnode_attr *dvap, vnode_t sdvp, mount_t mp, dir_clone_authorizer_op_t vattr_op, - vfs_context_t ctx, void *reserved); + uint32_t flags, vfs_context_t ctx, void *reserved); /* End of authorization subroutines */ #define VN_CREATE_NOAUTH (1<<0) @@ -602,6 +602,8 @@ void vnode_trigger_rearm(vnode_t, vfs_context_t); void vfs_nested_trigger_unmounts(mount_t, int, vfs_context_t); #endif /* CONFIG_TRIGGERS */ +int build_path_with_parent(vnode_t, vnode_t /* parent */, char *, int, int *, int, vfs_context_t); + #endif /* BSD_KERNEL_PRIVATE */ #endif /* !_SYS_VNODE_INTERNAL_H_ */ diff --git a/bsd/sys/work_interval.h b/bsd/sys/work_interval.h index cc9ba9fb7..f7e46ec08 100644 --- a/bsd/sys/work_interval.h +++ b/bsd/sys/work_interval.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,12 +30,17 @@ #define _SYS_WORK_INTERVAL_H #include -#include #include +#include + +#include __BEGIN_DECLS /* + * A work interval is a repeatable unit of work characterized by a + * start, finish, and deadline. + * * Trusted clients with deadline-sensitive work may report information * about the execution of their work using the work interval facility. * This is intended to be a higher-level semantic than realtime scheduling, @@ -97,45 +102,151 @@ __BEGIN_DECLS * Failure to do so will adversely impact system power and performance. * */ + +/* Flags to be passed with work_interval_create() */ + +/* If interval is joinable, create no longer implicitly joins, you must use work_interval_join */ +#define WORK_INTERVAL_FLAG_JOINABLE (0x1) +/* Only threads that join the group are measured together, otherwise the group is the creator's home group */ +#define WORK_INTERVAL_FLAG_GROUP (0x2) + +/* Flags to describe the interval flavor to the performance controller */ +#define WORK_INTERVAL_TYPE_MASK (0xF0000000) +#define WORK_INTERVAL_TYPE_DEFAULT (0x0 << 28) +#define WORK_INTERVAL_TYPE_COREAUDIO (0x1 << 28) +#define WORK_INTERVAL_TYPE_COREANIMATION (0x2 << 28) +#define WORK_INTERVAL_TYPE_LAST (0xF << 28) + #ifndef KERNEL typedef struct work_interval *work_interval_t; -/* Create a new work interval handle (currently for the current thread only). Flags is unused */ -int work_interval_create(work_interval_t *interval_handle, uint32_t flags); +/* + * Create a new work interval handle. + * + * May fail with EALREADY if the current group already has a work interval. + * + * With no flags: + * Auto-joins the work interval to the creating thread + * May only use interval_handle from creating thread + * Data provided affects native thread group + * + * With the JOINABLE flag + * interval_handle is usable by the process + * creating thread does not auto-join + * notifying thread must have joined when notifying + * + * With the GROUP flag + * creates a new thread group to isolate the joined threads from + * the rest of the process for performance controller analysis + * Threads which join the work interval become members of this new group + * + * TODO: Add a name parameter so that clients can name the work interval + * Can also take the thread name from the notifying thread + * + * Requires the 'com.apple.private.kernel.work-interval' entitlement (PRIV_WORK_INTERVAL) + * + * Note that joining a work interval supersedes automatic thread group management via vouchers + */ +int work_interval_create(work_interval_t *interval_handle, uint32_t flags); -/* Notify the power management subsystem that the work for a current interval has completed */ -int work_interval_notify(work_interval_t interval_handle, uint64_t start, uint64_t finish, uint64_t deadline, uint64_t next_start, uint32_t flags); +/* + * Notify the power management subsystem that the work for a current interval has completed + * + * Only the process which created the work interval may notify + */ +int work_interval_notify(work_interval_t interval_handle, + uint64_t start, uint64_t finish, + uint64_t deadline, uint64_t next_start, + uint32_t flags); -/* Notify, with "finish" implicitly set to the current time */ -int work_interval_notify_simple(work_interval_t interval_handle, uint64_t start, uint64_t deadline, uint64_t next_start); +/* + * Notify, with "finish" implicitly set to the current time + * + * Only the process which created the work interval may notify + */ +int work_interval_notify_simple(work_interval_t interval_handle, + uint64_t start, uint64_t deadline, + uint64_t next_start); -/* Deallocate work interval (currently for the current thread only) */ -int work_interval_destroy(work_interval_t interval_handle); +/* + * Deallocate work interval handle + * For non-JOINABLE, also removes thread from work interval + * For JOINABLE, does not remove thread (needs a leave as well) + */ +int work_interval_destroy(work_interval_t interval_handle); -#endif /* KERNEL */ +/* + * Join work interval via work interval handle + * Only allowed if interval is using the joinable and group flags + * + * Supersedes automatic thread group management via vouchers + */ +int work_interval_join(work_interval_t interval_handle); + +/* + * extract Mach send right representing work interval thread group + * Returns a +1 send right ref, which must be deallocated via mach_port_deallocate + * Only allowed if interval is joinable, otherwise returns ENOTSUP + * + * Supersedes automatic thread group management via vouchers + */ +int work_interval_copy_port(work_interval_t interval_handle, mach_port_t *port); + +/* + * Join work interval via Mach send right + * + * Does NOT consume Mach send right, must deallocate with mach_port_deallocate after using + * It's safe to deallocate the right after joining, the thread will stay joined + * + * Can be sent to clients via xpc_dictionary_copy_mach_send, and similar + * + * Supersedes automatic thread group management via vouchers + * + * If the underlying work interval object is terminated then this may return ENOENT + * + */ +int work_interval_join_port(mach_port_t port); + +/* + * Leave the current thread's work interval + */ +int work_interval_leave(void); + +/* TODO: complexity measure */ + +#endif /* !KERNEL */ #if PRIVATE /* Private interface between Libsyscall and xnu */ -#define WORK_INTERVAL_OPERATION_CREATE 0x00000001 /* arg is a uint64_t * that accepts work interval ID as an OUT param */ -#define WORK_INTERVAL_OPERATION_DESTROY 0x00000002 -#define WORK_INTERVAL_OPERATION_NOTIFY 0x00000003 /* arg is a work_interval_notification_t */ +#define WORK_INTERVAL_OPERATION_CREATE 0x00000001 /* deprecated */ +#define WORK_INTERVAL_OPERATION_DESTROY 0x00000002 /* arg is NULL */ +#define WORK_INTERVAL_OPERATION_NOTIFY 0x00000003 /* arg is a work_interval_notification_t */ +#define WORK_INTERVAL_OPERATION_CREATE2 0x00000004 /* arg is a work_interval_create_params */ +#define WORK_INTERVAL_OPERATION_JOIN 0x00000005 /* arg is a port_name */ struct work_interval_notification { uint64_t start; uint64_t finish; uint64_t deadline; uint64_t next_start; - uint32_t flags; - uint32_t unused1; + uint32_t notify_flags; + uint32_t create_flags; }; typedef struct work_interval_notification *work_interval_notification_t; -int __work_interval_ctl(uint32_t operation, uint64_t work_interval_id, void *arg, size_t len); +struct work_interval_create_params { + uint64_t wicp_id; /* out param */ + uint32_t wicp_port; /* out param */ + uint32_t wicp_create_flags; +}; + +int __work_interval_ctl(uint32_t operation, uint64_t work_interval_id, void *arg, size_t len); #endif /* PRIVATE */ __END_DECLS #endif /* _SYS_WORK_INTERVAL_H */ + diff --git a/bsd/vfs/Makefile b/bsd/vfs/Makefile index ee71a736c..d9223229e 100644 --- a/bsd/vfs/Makefile +++ b/bsd/vfs/Makefile @@ -13,7 +13,9 @@ INSTALL_MI_LIST = ${DATAFILES} INSTALL_MI_DIR = vfs -EXPORT_MI_LIST = ${DATAFILES} +INSTALL_KF_MI_LIST = ${DATAFILES} + +EXPORT_MI_LIST = ${DATAFILES} vfs_disk_conditioner.h EXPORT_MI_DIR = vfs diff --git a/bsd/vfs/kpi_vfs.c b/bsd/vfs/kpi_vfs.c index dd3560fed..060866928 100644 --- a/bsd/vfs/kpi_vfs.c +++ b/bsd/vfs/kpi_vfs.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -2168,6 +2168,11 @@ vnode_get_filesec(vnode_t vp, kauth_filesec_t *fsecp, vfs_context_t ctx) /* how many entries would fit? */ fsec_size = KAUTH_FILESEC_COUNT(xsize); + if (fsec_size > KAUTH_ACL_MAX_ENTRIES) { + KAUTH_DEBUG(" ERROR - Bogus (too large) kauth_fiilesec_t: %ld bytes", xsize); + error = 0; + goto out; + } /* get buffer and uio */ if (((fsec = kauth_filesec_alloc(fsec_size)) == NULL) || @@ -2314,6 +2319,7 @@ out: /* * Returns: 0 Success * ENOMEM Not enough space [only if has filesec] + * EINVAL Requested unknown attributes * VNOP_GETATTR: ??? * vnode_get_filesec: ??? * kauth_cred_guid2uid: ??? @@ -2329,6 +2335,12 @@ vnode_getattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx) uid_t nuid; gid_t ngid; + /* + * Reject attempts to fetch unknown attributes. + */ + if (vap->va_active & ~VNODE_ATTR_ALL) + return (EINVAL); + /* don't ask for extended security data if the filesystem doesn't support it */ if (!vfs_extendedsecurity(vnode_mount(vp))) { VATTR_CLEAR_ACTIVE(vap, va_acl); @@ -2555,7 +2567,18 @@ out: int vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx) { - int error, is_perm_change=0; + int error; +#if CONFIG_FSE + uint64_t active; + int is_perm_change = 0; + int is_stat_change = 0; +#endif + + /* + * Reject attempts to set unknown attributes. + */ + if (vap->va_active & ~VNODE_ATTR_ALL) + return (EINVAL); /* * Make sure the filesystem is mounted R/W. @@ -2615,11 +2638,6 @@ vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx) VATTR_CLEAR_ACTIVE(vap, va_gid); } - if ( VATTR_IS_ACTIVE(vap, va_uid) || VATTR_IS_ACTIVE(vap, va_gid) - || VATTR_IS_ACTIVE(vap, va_mode) || VATTR_IS_ACTIVE(vap, va_acl)) { - is_perm_change = 1; - } - /* * Make sure that extended security is enabled if we're going to try * to set any. @@ -2636,23 +2654,55 @@ vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx) vap->va_flags &= (SF_SUPPORTED | UF_SETTABLE); } +#if CONFIG_FSE + /* + * Remember all of the active attributes that we're + * attempting to modify. + */ + active = vap->va_active & ~VNODE_ATTR_RDONLY; +#endif + error = VNOP_SETATTR(vp, vap, ctx); if ((error == 0) && !VATTR_ALL_SUPPORTED(vap)) error = vnode_setattr_fallback(vp, vap, ctx); #if CONFIG_FSE - // only send a stat_changed event if this is more than - // just an access or backup time update - if (error == 0 && (vap->va_active != VNODE_ATTR_BIT(va_access_time)) && (vap->va_active != VNODE_ATTR_BIT(va_backup_time))) { +#define PERMISSION_BITS (VNODE_ATTR_BIT(va_uid) | VNODE_ATTR_BIT(va_uuuid) | \ + VNODE_ATTR_BIT(va_gid) | VNODE_ATTR_BIT(va_guuid) | \ + VNODE_ATTR_BIT(va_mode) | VNODE_ATTR_BIT(va_acl)) + + /* + * Now that we've changed them, decide whether to send an + * FSevent. + */ + if ((active & PERMISSION_BITS) & vap->va_supported) { + is_perm_change = 1; + } else { + /* + * We've already checked the permission bits, and we + * also want to filter out access time / backup time + * changes. + */ + active &= ~(PERMISSION_BITS | + VNODE_ATTR_BIT(va_access_time) | + VNODE_ATTR_BIT(va_backup_time)); + + /* Anything left to notify about? */ + if (active & vap->va_supported) + is_stat_change = 1; + } + + if (error == 0) { if (is_perm_change) { if (need_fsevent(FSE_CHOWN, vp)) { add_fsevent(FSE_CHOWN, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE); } - } else if(need_fsevent(FSE_STAT_CHANGED, vp)) { + } else if (is_stat_change && need_fsevent(FSE_STAT_CHANGED, vp)) { add_fsevent(FSE_STAT_CHANGED, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE); } } +#undef PERMISSION_BITS #endif out: @@ -4013,14 +4063,17 @@ vn_rename(struct vnode *fdvp, struct vnode **fvpp, struct componentname *fcnp, s */ if (_err == 0) { _err = vnode_flags(tdvp, &tdfflags, ctx); - if (_err == 0 && (tdfflags & SF_RESTRICTED)) { - uint32_t fflags; - _err = vnode_flags(*fvpp, &fflags, ctx); - if (_err == 0 && !(fflags & SF_RESTRICTED)) { - struct vnode_attr va; - VATTR_INIT(&va); - VATTR_SET(&va, va_flags, fflags | SF_RESTRICTED); - _err = vnode_setattr(*fvpp, &va, ctx); + if (_err == 0) { + uint32_t inherit_flags = tdfflags & (UF_DATAVAULT | SF_RESTRICTED); + if (inherit_flags) { + uint32_t fflags; + _err = vnode_flags(*fvpp, &fflags, ctx); + if (_err == 0 && fflags != (fflags | inherit_flags)) { + struct vnode_attr va; + VATTR_INIT(&va); + VATTR_SET(&va, va_flags, fflags | inherit_flags); + _err = vnode_setattr(*fvpp, &va, ctx); + } } } } @@ -5383,6 +5436,7 @@ struct vnop_clonefile_args { vnode_t sdvp, /* source directory vnode pointer (optional) */ mount_t mp, /* mount point of filesystem */ dir_clone_authorizer_op_t vattr_op, /* specific operation requested : setup, authorization or cleanup */ + uint32_t flags; /* value passed in a_flags to the VNOP */ vfs_context_t ctx, /* As passed to VNOP */ void *reserved); /* Always NULL */ void *a_reserved; /* Currently unused */ diff --git a/bsd/vfs/vfs_attrlist.c b/bsd/vfs/vfs_attrlist.c index 51eb4756f..4aab3ac01 100644 --- a/bsd/vfs/vfs_attrlist.c +++ b/bsd/vfs/vfs_attrlist.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1995-2016 Apple Inc. All rights reserved. + * Copyright (c) 1995-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -51,6 +51,7 @@ #include #include #include +#include #if CONFIG_MACF #include @@ -269,7 +270,7 @@ attrlist_pack_string(struct _attrlist_buf *ab, const char *source, ssize_t count #define ATTR_PACK8(AB, V) \ do { \ if ((AB.allocated - (AB.fixedcursor - AB.base)) >= 8) { \ - *(uint64_t *)AB.fixedcursor = *(uint64_t *)&V; \ + memcpy(AB.fixedcursor, &V, 8); \ AB.fixedcursor += 8; \ } \ } while (0) @@ -356,8 +357,8 @@ static struct getvolattrlist_attrtab getvolattrlist_vol_tab[] = { {ATTR_VOL_ENCODINGSUSED, 0, sizeof(uint64_t)}, {ATTR_VOL_CAPABILITIES, VFSATTR_BIT(f_capabilities), sizeof(vol_capabilities_attr_t)}, {ATTR_VOL_UUID, VFSATTR_BIT(f_uuid), sizeof(uuid_t)}, - {ATTR_VOL_QUOTA_SIZE, VFSATTR_BIT(f_quota), sizeof(off_t)}, - {ATTR_VOL_RESERVED_SIZE, VFSATTR_BIT(f_reserved), sizeof(off_t)}, + {ATTR_VOL_QUOTA_SIZE, VFSATTR_BIT(f_quota) | VFSATTR_BIT(f_bsize), sizeof(off_t)}, + {ATTR_VOL_RESERVED_SIZE, VFSATTR_BIT(f_reserved) | VFSATTR_BIT(f_bsize), sizeof(off_t)}, {ATTR_VOL_ATTRIBUTES, VFSATTR_BIT(f_attributes), sizeof(vol_attributes_attr_t)}, {ATTR_VOL_INFO, 0, 0}, {0, 0, 0} @@ -1394,6 +1395,21 @@ getvolattrlist(vfs_context_t ctx, vnode_t vp, struct attrlist *alp, vs.f_capabilities.capabilities[VOL_CAPABILITIES_INTERFACES] &= ~VOL_CAP_INT_EXTENDED_SECURITY; } vs.f_capabilities.valid[VOL_CAPABILITIES_INTERFACES] |= VOL_CAP_INT_EXTENDED_SECURITY; + + /* + * if the filesystem doesn't mark either VOL_CAP_FMT_NO_IMMUTABLE_FILES + * or VOL_CAP_FMT_NO_PERMISSIONS as valid, assume they're not supported + */ + if (!(vs.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_NO_IMMUTABLE_FILES)) { + vs.f_capabilities.capabilities[VOL_CAPABILITIES_FORMAT] &= ~VOL_CAP_FMT_NO_IMMUTABLE_FILES; + vs.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] |= VOL_CAP_FMT_NO_IMMUTABLE_FILES; + } + + if (!(vs.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_NO_PERMISSIONS)) { + vs.f_capabilities.capabilities[VOL_CAPABILITIES_FORMAT] &= ~VOL_CAP_FMT_NO_PERMISSIONS; + vs.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] |= VOL_CAP_FMT_NO_PERMISSIONS; + } + ATTR_PACK(&ab, vs.f_capabilities); ab.actual.volattr |= ATTR_VOL_CAPABILITIES; } @@ -2743,7 +2759,7 @@ out: static int getattrlist_internal(vfs_context_t ctx, vnode_t vp, struct attrlist *alp, user_addr_t attributeBuffer, size_t bufferSize, uint64_t options, - enum uio_seg segflg, char* alt_name, struct ucred *file_cred) + enum uio_seg segflg, char* authoritative_name, struct ucred *file_cred) { struct vnode_attr va; kauth_action_t action; @@ -2875,9 +2891,22 @@ getattrlist_internal(vfs_context_t ctx, vnode_t vp, struct attrlist *alp, VFS_DEBUG(ctx, vp, "ATTRLIST - ERROR: cannot allocate va_name buffer"); goto out; } + /* + * If we have an authoritative_name, prefer that name. + * + * N.B. Since authoritative_name implies this is coming from getattrlistbulk, + * we know the name is authoritative. For /dev/fd, we want to use the file + * descriptor as the name not the underlying name of the associate vnode in a + * particular file system. + */ + if (authoritative_name) { + /* Don't ask the file system */ + VATTR_CLEAR_ACTIVE(&va, va_name); + strlcpy(va_name, authoritative_name, MAXPATHLEN); + } } - va.va_name = va_name; + va.va_name = authoritative_name ? NULL : va_name; /* * Call the filesystem. @@ -2907,16 +2936,19 @@ getattrlist_internal(vfs_context_t ctx, vnode_t vp, struct attrlist *alp, #endif /* - * If ATTR_CMN_NAME is not supported by filesystem and the - * caller has provided a name, use that. + * It we ask for the name, i.e., vname is non null and + * we have an authoritative name, then reset va_name is + * active and if needed set va_name is supported. + * * A (buggy) filesystem may change fields which belong * to us. We try to deal with that here as well. */ va.va_active = va_active; - if (alt_name && va_name && - !(VATTR_IS_SUPPORTED(&va, va_name))) { - strlcpy(va_name, alt_name, MAXPATHLEN); - VATTR_SET_SUPPORTED(&va, va_name); + if (authoritative_name && va_name) { + VATTR_SET_ACTIVE(&va, va_name); + if (!(VATTR_IS_SUPPORTED(&va, va_name))) { + VATTR_SET_SUPPORTED(&va, va_name); + } } va.va_name = va_name; } @@ -3680,11 +3712,6 @@ getattrlistbulk(proc_t p, struct getattrlistbulk_args *uap, int32_t *retval) if (uap->options & FSOPT_LIST_SNAPSHOT) { vnode_t snapdvp; - if (!vfs_context_issuser(ctx)) { - error = EPERM; - goto out; - } - if (!vnode_isvroot(dvp)) { error = EINVAL; goto out; @@ -4124,6 +4151,10 @@ setattrlist_internal(vnode_t vp, struct setattrlist_args *uap, proc_t p, vfs_con ATTR_UNPACK(va.va_guuid); VATTR_SET_ACTIVE(&va, va_guuid); } + if (al.commonattr & ATTR_CMN_ADDEDTIME) { + ATTR_UNPACK_TIME(va.va_addedtime, proc_is64); + VATTR_SET_ACTIVE(&va, va_addedtime); + } /* Support setattrlist of data protection class */ if (al.commonattr & ATTR_CMN_DATA_PROTECT_FLAGS) { ATTR_UNPACK(va.va_dataprotect_class); @@ -4310,6 +4341,44 @@ out: return error; } +int +setattrlistat(proc_t p, struct setattrlistat_args *uap, __unused int32_t *retval) +{ + struct setattrlist_args ap; + struct vfs_context *ctx; + struct nameidata nd; + vnode_t vp = NULLVP; + uint32_t nameiflags; + int error; + + ctx = vfs_context_current(); + + AUDIT_ARG(fd, uap->fd); + /* + * Look up the file. + */ + nameiflags = AUDITVNPATH1; + if (!(uap->options & FSOPT_NOFOLLOW)) + nameiflags |= FOLLOW; + NDINIT(&nd, LOOKUP, OP_SETATTR, nameiflags, UIO_USERSPACE, uap->path, ctx); + if ((error = nameiat(&nd, uap->fd)) != 0) + goto out; + vp = nd.ni_vp; + nameidone(&nd); + + ap.path = 0; + ap.alist = uap->alist; + ap.attributeBuffer = uap->attributeBuffer; + ap.bufferSize = uap->bufferSize; + ap.options = uap->options; + + error = setattrlist_internal(vp, &ap, p, ctx); +out: + if (vp) + vnode_put(vp); + return (error); +} + int fsetattrlist(proc_t p, struct fsetattrlist_args *uap, __unused int32_t *retval) { diff --git a/bsd/vfs/vfs_bio.c b/bsd/vfs/vfs_bio.c index cdccdc828..c1019a327 100644 --- a/bsd/vfs/vfs_bio.c +++ b/bsd/vfs/vfs_bio.c @@ -132,6 +132,8 @@ static buf_t buf_create_shadow_internal(buf_t bp, boolean_t force_copy, int bdwrite_internal(buf_t, int); +extern void disk_conditioner_delay(buf_t, int, int, uint64_t); + /* zone allocated buffer headers */ static void bufzoneinit(void); static void bcleanbuf_thread_init(void); @@ -491,10 +493,16 @@ bufattr_markmeta(bufattr_t bap) { } int +#if !CONFIG_EMBEDDED bufattr_delayidlesleep(bufattr_t bap) +#else /* !CONFIG_EMBEDDED */ +bufattr_delayidlesleep(__unused bufattr_t bap) +#endif /* !CONFIG_EMBEDDED */ { +#if !CONFIG_EMBEDDED if ( (bap->ba_flags & BA_DELAYIDLESLEEP) ) return 1; +#endif /* !CONFIG_EMBEDDED */ return 0; } @@ -2629,12 +2637,13 @@ buf_brelse(buf_t bp) if (upl == NULL) { if ( !ISSET(bp->b_flags, B_INVAL)) { - kret = ubc_create_upl(bp->b_vp, + kret = ubc_create_upl_kernel(bp->b_vp, ubc_blktooff(bp->b_vp, bp->b_lblkno), bp->b_bufsize, &upl, NULL, - UPL_PRECIOUS); + UPL_PRECIOUS, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("brelse: Failed to create UPL"); @@ -3034,12 +3043,13 @@ start: case BLK_READ: upl_flags |= UPL_PRECIOUS; if (UBCINFOEXISTS(bp->b_vp) && bp->b_bufsize) { - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, ubc_blktooff(vp, bp->b_lblkno), bp->b_bufsize, &upl, &pl, - upl_flags); + upl_flags, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("Failed to create UPL"); @@ -3183,12 +3193,13 @@ start: f_offset = ubc_blktooff(vp, blkno); upl_flags |= UPL_PRECIOUS; - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, f_offset, bp->b_bufsize, &upl, &pl, - upl_flags); + upl_flags, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("Failed to create UPL"); @@ -3968,6 +3979,8 @@ buf_biodone(buf_t bp) { mount_t mp; struct bufattr *bap; + struct timeval real_elapsed; + uint64_t real_elapsed_usec = 0; KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 387)) | DBG_FUNC_START, bp, bp->b_datap, bp->b_flags, 0, 0); @@ -4035,6 +4048,11 @@ buf_biodone(buf_t bp) buf_kernel_addrperm_addr(bp), (uintptr_t)VM_KERNEL_ADDRPERM(bp->b_vp), bp->b_resid, bp->b_error, 0); } + microuptime(&real_elapsed); + timevalsub(&real_elapsed, &bp->b_timestamp_tv); + real_elapsed_usec = real_elapsed.tv_sec * USEC_PER_SEC + real_elapsed.tv_usec; + disk_conditioner_delay(bp, 1, bp->b_bcount, real_elapsed_usec); + /* * I/O was done, so don't believe * the DIRTY state from VM anymore... @@ -4484,12 +4502,13 @@ brecover_data(buf_t bp) upl_flags |= UPL_WILL_MODIFY; } - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, ubc_blktooff(vp, bp->b_lblkno), bp->b_bufsize, &upl, &pl, - upl_flags); + upl_flags, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("Failed to create UPL"); diff --git a/bsd/vfs/vfs_cache.c b/bsd/vfs/vfs_cache.c index 47cf243ba..b24dbc590 100644 --- a/bsd/vfs/vfs_cache.c +++ b/bsd/vfs/vfs_cache.c @@ -388,12 +388,12 @@ vnode_issubdir(vnode_t vp, vnode_t dvp, int *is_subdir, vfs_context_t ctx) } /* - * This function builds the path to a filename in "buff". The - * length of the buffer *INCLUDING* the trailing zero byte is - * returned in outlen. NOTE: the length includes the trailing - * zero byte and thus the length is one greater than what strlen - * would return. This is important and lots of code elsewhere - * in the kernel assumes this behavior. + * This function builds the path in "buff" from the supplied vnode. + * The length of the buffer *INCLUDING* the trailing zero byte is + * returned in outlen. NOTE: the length includes the trailing zero + * byte and thus the length is one greater than what strlen would + * return. This is important and lots of code elsewhere in the kernel + * assumes this behavior. * * This function can call vnop in file system if the parent vnode * does not exist or when called for hardlinks via volfs path. @@ -410,9 +410,19 @@ vnode_issubdir(vnode_t vp, vnode_t dvp, int *is_subdir, vfs_context_t ctx) * cross over mount points during building the path. * * passed in vp must have a valid io_count reference + * + * If parent vnode is non-NULL it also must have an io count. This + * allows build_path_with_parent to be safely called for operations + * unlink, rmdir and rename that already have io counts on the target + * and the directory. In this way build_path_with_parent does not have + * to try and obtain an additional io count on the parent. Taking an + * io count ont the parent can lead to dead lock if a forced unmount + * occures at the right moment. For a fuller explaination on how this + * can occur see the comment for vn_getpath_with_parent. + * */ int -build_path(vnode_t first_vp, char *buff, int buflen, int *outlen, int flags, vfs_context_t ctx) +build_path_with_parent(vnode_t first_vp, vnode_t parent_vp, char *buff, int buflen, int *outlen, int flags, vfs_context_t ctx) { vnode_t vp, tvp; vnode_t vp_with_iocount; @@ -587,7 +597,7 @@ again: NAME_CACHE_UNLOCK(); - if (vp != first_vp && vp != vp_with_iocount) { + if (vp != first_vp && vp != parent_vp && vp != vp_with_iocount) { if (vp_with_iocount) { vnode_put(vp_with_iocount); vp_with_iocount = NULLVP; @@ -678,7 +688,7 @@ bad_news: NAME_CACHE_UNLOCK(); - if (vp != first_vp && vp != vp_with_iocount) { + if (vp != first_vp && vp != parent_vp && vp != vp_with_iocount) { if (vp_with_iocount) { vnode_put(vp_with_iocount); vp_with_iocount = NULLVP; @@ -745,6 +755,11 @@ out: return (ret); } +int +build_path(vnode_t first_vp, char *buff, int buflen, int *outlen, int flags, vfs_context_t ctx) +{ + return (build_path_with_parent(first_vp, NULL, buff, buflen, outlen, flags, ctx)); +} /* * return NULLVP if vp's parent doesn't @@ -1362,7 +1377,7 @@ skiprsrcfork: #if CONFIG_MACF - /* + /* * Name cache provides authorization caching (see below) * that will short circuit MAC checks in lookup(). * We must perform MAC check here. On denial @@ -1685,7 +1700,7 @@ cache_lookup_locked(vnode_t dvp, struct componentname *cnp) ncpp = NCHHASH(dvp, cnp->cn_hash); LIST_FOREACH(ncp, ncpp, nc_hash) { if ((ncp->nc_dvp == dvp) && (ncp->nc_hashval == hashval)) { - if (memcmp(ncp->nc_name, cnp->cn_nameptr, namelen) == 0 && ncp->nc_name[namelen] == 0) + if (strncmp(ncp->nc_name, cnp->cn_nameptr, namelen) == 0 && ncp->nc_name[namelen] == 0) break; } } @@ -1772,7 +1787,7 @@ relook: ncpp = NCHHASH(dvp, cnp->cn_hash); LIST_FOREACH(ncp, ncpp, nc_hash) { if ((ncp->nc_dvp == dvp) && (ncp->nc_hashval == hashval)) { - if (memcmp(ncp->nc_name, cnp->cn_nameptr, namelen) == 0 && ncp->nc_name[namelen] == 0) + if (strncmp(ncp->nc_name, cnp->cn_nameptr, namelen) == 0 && ncp->nc_name[namelen] == 0) break; } } @@ -2211,7 +2226,7 @@ resize_namecache(u_int newsize) } static void -cache_delete(struct namecache *ncp, int age_entry) +cache_delete(struct namecache *ncp, int free_entry) { NCHSTAT(ncs_deletes); @@ -2232,16 +2247,13 @@ cache_delete(struct namecache *ncp, int age_entry) */ ncp->nc_hash.le_prev = NULL; - if (age_entry) { - /* - * make it the next one available - * for cache_enter's use - */ - TAILQ_REMOVE(&nchead, ncp, nc_entry); - TAILQ_INSERT_HEAD(&nchead, ncp, nc_entry); - } vfs_removename(ncp->nc_name); ncp->nc_name = NULL; + if (free_entry) { + TAILQ_REMOVE(&nchead, ncp, nc_entry); + FREE_ZONE(ncp, sizeof(*ncp), M_CACHE); + numcache--; + } } @@ -2475,7 +2487,7 @@ add_name_internal(const char *name, uint32_t len, u_int hashval, boolean_t need_ lck_mtx_lock_spin(&strcache_mtx_locks[lock_index]); for (entry = head->lh_first; entry != NULL; chain_len++, entry = entry->hash_chain.le_next) { - if (memcmp(entry->str, name, len) == 0 && entry->str[len] == 0) { + if (strncmp(entry->str, name, len) == 0 && entry->str[len] == 0) { entry->refcount++; break; } diff --git a/bsd/vfs/vfs_cluster.c b/bsd/vfs/vfs_cluster.c index 70eecc5ff..df1e9569f 100644 --- a/bsd/vfs/vfs_cluster.c +++ b/bsd/vfs/vfs_cluster.c @@ -98,6 +98,8 @@ #include +#include + #if 0 #undef KERNEL_DEBUG #define KERNEL_DEBUG KERNEL_DEBUG_CONSTANT @@ -266,17 +268,23 @@ int (*bootcache_contains_block)(dev_t device, u_int64_t blkno) = NULL; #define WRITE_BEHIND 1 #define WRITE_BEHIND_SSD 1 +#if CONFIG_EMBEDDED +#define PREFETCH 1 +#define PREFETCH_SSD 1 +uint32_t speculative_prefetch_max = (2048 * 1024); /* maximum bytes in a specluative read-ahead */ +uint32_t speculative_prefetch_max_iosize = (512 * 1024); /* maximum I/O size to use in a specluative read-ahead */ +#else #define PREFETCH 3 #define PREFETCH_SSD 2 uint32_t speculative_prefetch_max = (MAX_UPL_SIZE_BYTES * 3); /* maximum bytes in a specluative read-ahead */ uint32_t speculative_prefetch_max_iosize = (512 * 1024); /* maximum I/O size to use in a specluative read-ahead on SSDs*/ +#endif #define IO_SCALE(vp, base) (vp->v_mount->mnt_ioscale * (base)) #define MAX_CLUSTER_SIZE(vp) (cluster_max_io_size(vp->v_mount, CL_WRITE)) -#define MAX_PREFETCH(vp, size, is_ssd) (size * IO_SCALE(vp, ((is_ssd && !ignore_is_ssd) ? PREFETCH_SSD : PREFETCH))) +#define MAX_PREFETCH(vp, size, is_ssd) (size * IO_SCALE(vp, ((is_ssd) ? PREFETCH_SSD : PREFETCH))) -int ignore_is_ssd = 0; int speculative_reads_disabled = 0; /* @@ -494,8 +502,8 @@ cluster_io_present_in_BC(vnode_t vp, off_t f_offset) size_t io_size; int (*bootcache_check_fn)(dev_t device, u_int64_t blkno) = bootcache_contains_block; - if (bootcache_check_fn) { - if (VNOP_BLOCKMAP(vp, f_offset, PAGE_SIZE, &blkno, &io_size, NULL, VNODE_READ, NULL)) + if (bootcache_check_fn && vp->v_mount && vp->v_mount->mnt_devvp) { + if (VNOP_BLOCKMAP(vp, f_offset, PAGE_SIZE, &blkno, &io_size, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL)) return(0); if (io_size == 0) @@ -1189,7 +1197,7 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no } else { max_cluster_size = MAX_CLUSTER_SIZE(vp); - if ((vp->v_mount->mnt_kern_flag & MNTK_SSD) && !ignore_is_ssd) + if (disk_conditioner_mount_is_ssd(vp->v_mount)) scale = WRITE_THROTTLE_SSD; else scale = WRITE_THROTTLE; @@ -1249,8 +1257,8 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no * Create a UPL to lock the pages in the cache whilst the * write is in progress. */ - ubc_create_upl(vp, f_offset, non_rounded_size, &cached_upl, - NULL, UPL_SET_LITE); + ubc_create_upl_kernel(vp, f_offset, non_rounded_size, &cached_upl, + NULL, UPL_SET_LITE, VM_KERN_MEMORY_FILE); /* * Attach this UPL to the other UPL so that we can find it @@ -1971,7 +1979,7 @@ cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct return; } - max_prefetch = MAX_PREFETCH(vp, cluster_max_io_size(vp->v_mount, CL_READ), (vp->v_mount->mnt_kern_flag & MNTK_SSD)); + max_prefetch = MAX_PREFETCH(vp, cluster_max_io_size(vp->v_mount, CL_READ), disk_conditioner_mount_is_ssd(vp->v_mount)); if (max_prefetch > speculative_prefetch_max) max_prefetch = speculative_prefetch_max; @@ -2516,8 +2524,7 @@ next_dwrite: pages_in_pl = 0; upl_size = upl_needed_size; upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC | - UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE - | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE); + UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE; kret = vm_map_get_upl(map, (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)), @@ -2526,6 +2533,7 @@ next_dwrite: NULL, &pages_in_pl, &upl_flags, + VM_KERN_MEMORY_FILE, force_data_sync); if (kret != KERN_SUCCESS) { @@ -2789,13 +2797,12 @@ next_cwrite: pages_in_pl = 0; upl_size = upl_needed_size; upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC | - UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE - | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE); + UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE; vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map; kret = vm_map_get_upl(map, (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)), - &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0); + &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, VM_KERN_MEMORY_FILE, 0); if (kret != KERN_SUCCESS) { /* @@ -3178,12 +3185,13 @@ cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t old * The UPL_WILL_MODIFY flag lets the UPL subsystem know * that we intend to modify these pages. */ - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, upl_f_offset, upl_size, &upl, &pl, - UPL_SET_LITE | (( uio!=NULL && (uio->uio_flags & UIO_FLAGS_IS_COMPRESSED_FILE)) ? 0 : UPL_WILL_MODIFY)); + UPL_SET_LITE | (( uio!=NULL && (uio->uio_flags & UIO_FLAGS_IS_COMPRESSED_FILE)) ? 0 : UPL_WILL_MODIFY), + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("cluster_write_copy: failed to get pagelist"); @@ -3535,7 +3543,7 @@ check_cluster: n = 0; if (n == 0) { - if (vp->v_mount->mnt_kern_flag & MNTK_SSD) + if (disk_conditioner_mount_is_ssd(vp->v_mount)) n = WRITE_BEHIND_SSD; else n = WRITE_BEHIND; @@ -3777,7 +3785,7 @@ cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t file bflag |= CL_ENCRYPTED; max_io_size = cluster_max_io_size(vp->v_mount, CL_READ); - max_prefetch = MAX_PREFETCH(vp, max_io_size, (vp->v_mount->mnt_kern_flag & MNTK_SSD)); + max_prefetch = MAX_PREFETCH(vp, max_io_size, disk_conditioner_mount_is_ssd(vp->v_mount)); max_rd_size = max_prefetch; last_request_offset = uio->uio_offset + io_req_size; @@ -3974,12 +3982,13 @@ cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t file KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_START, upl, (int)upl_f_offset, upl_size, start_offset, 0); - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, upl_f_offset, upl_size, &upl, &pl, - UPL_FILE_IO | UPL_SET_LITE); + UPL_FILE_IO | UPL_SET_LITE, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("cluster_read_copy: failed to get pagelist"); @@ -4651,8 +4660,7 @@ next_dread: for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) { pages_in_pl = 0; upl_size = upl_needed_size; - upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE - | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE); + upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE; if (no_zero_fill) upl_flags |= UPL_NOZEROFILL; if (force_data_sync) @@ -4660,7 +4668,7 @@ next_dread: kret = vm_map_create_upl(map, (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)), - &upl_size, &upl, NULL, &pages_in_pl, &upl_flags); + &upl_size, &upl, NULL, &pages_in_pl, &upl_flags, VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) { KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END, @@ -4925,8 +4933,7 @@ next_cread: pages_in_pl = 0; upl_size = upl_needed_size; - upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE - | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE); + upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE; KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_START, @@ -4935,7 +4942,7 @@ next_cread: vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map; kret = vm_map_get_upl(map, (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)), - &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0); + &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, VM_KERN_MEMORY_FILE, 0); KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_END, (int)upl_offset, upl_size, io_size, kret, 0); @@ -5103,12 +5110,12 @@ cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t m else upl_size = (u_int32_t)iov_len; - upl_flags = UPL_QUERY_OBJECT_TYPE | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE); + upl_flags = UPL_QUERY_OBJECT_TYPE; vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map; if ((vm_map_get_upl(map, (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)), - &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) { + &upl_size, &upl, NULL, NULL, &upl_flags, VM_KERN_MEMORY_FILE, 0)) != KERN_SUCCESS) { /* * the user app must have passed in an invalid address */ @@ -5177,10 +5184,15 @@ advisory_read_ext(vnode_t vp, off_t filesize, off_t f_offset, int resid, int (*c max_io_size = cluster_max_io_size(vp->v_mount, CL_READ); - if ((vp->v_mount->mnt_kern_flag & MNTK_SSD) && !ignore_is_ssd) { +#if CONFIG_EMBEDDED + if (max_io_size > speculative_prefetch_max_iosize) + max_io_size = speculative_prefetch_max_iosize; +#else + if (disk_conditioner_mount_is_ssd(vp->v_mount)) { if (max_io_size > speculative_prefetch_max_iosize) max_io_size = speculative_prefetch_max_iosize; } +#endif KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_START, (int)f_offset, resid, (int)filesize, 0, 0); @@ -5239,12 +5251,13 @@ advisory_read_ext(vnode_t vp, off_t filesize, off_t f_offset, int resid, int (*c KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_START, upl, (int)upl_f_offset, upl_size, start_offset, 0); - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, upl_f_offset, upl_size, &upl, &pl, - UPL_RET_ONLY_ABSENT | UPL_SET_LITE); + UPL_RET_ONLY_ABSENT | UPL_SET_LITE, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) return(retval); issued_io = 0; @@ -5754,12 +5767,13 @@ cluster_push_now(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags, int (*c else upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE; - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, upl_f_offset, upl_size, &upl, &pl, - upl_flags); + upl_flags, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("cluster_push: failed to get pagelist"); @@ -5988,12 +6002,13 @@ cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t */ upl_flags |= UPL_FILE_IO; } - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE, &upl, &pl, - upl_flags); + upl_flags, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) return(EINVAL); diff --git a/bsd/vfs/vfs_disk_conditioner.c b/bsd/vfs/vfs_disk_conditioner.c new file mode 100644 index 000000000..8cc7237c6 --- /dev/null +++ b/bsd/vfs/vfs_disk_conditioner.c @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2016 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include + +#define DISK_CONDITIONER_SET_ENTITLEMENT "com.apple.private.dmc.set" + +// number of total blocks for a mount +#define BLK_MAX(mp) ((mp->mnt_vfsstat.f_blocks * mp->mnt_vfsstat.f_bsize) / (mp->mnt_devblocksize)) + +// approx. time to spin up an idle HDD +#define DISK_SPINUP_SEC (8) + +// idle period until assumed disk spin down +#define DISK_IDLE_SEC (10 * 60) + +struct _disk_conditioner_info_t { + boolean_t enabled; // if other fields have any effect + uint64_t access_time_usec; // maximum latency before an I/O transfer begins + uint64_t read_throughput_mbps; // throughput of an I/O read + uint64_t write_throughput_mbps; // throughput of an I/O write + boolean_t is_ssd; // behave like an SSD (for both conditioning and affecting behavior in other parts of VFS) + daddr64_t last_blkno; // approx. last transfered block for simulating seek times + struct timeval last_io_timestamp; // the last time an I/O completed +}; + +void disk_conditioner_delay(buf_t, int, int, uint64_t); +void disk_conditioner_unmount(mount_t mp); + +extern void throttle_info_mount_reset_period(mount_t, int isssd); + +static double +weighted_scale_factor(double scale) +{ + // 0 to 1 increasing quickly from 0. This weights smaller blkdiffs higher to add a type of minimum latency + // I would like to use log(10) / 2.0 + 1, but using different approximation due to no math library + // y = (x-1)^3 + 1 + double x_m1 = scale - 1; + return x_m1 * x_m1 * x_m1 + 1; +} + +void +disk_conditioner_delay(buf_t bp, int extents, int total_size, uint64_t already_elapsed_usec) +{ + mount_t mp; + uint64_t delay_usec; + daddr64_t blkdiff; + daddr64_t last_blkno; + double access_time_scale; + struct _disk_conditioner_info_t *info = NULL; + struct timeval elapsed; + struct timeval start; + + mp = buf_vnode(bp)->v_mount; + if (!mp) { + return; + } + + info = mp->mnt_disk_conditioner_info; + if (!info || !info->enabled) { + return; + } + + if (!info->is_ssd) { + // calculate approximate seek time based on difference in block number + last_blkno = info->last_blkno; + blkdiff = bp->b_blkno > last_blkno ? bp->b_blkno - last_blkno : last_blkno - bp->b_blkno; + info->last_blkno = bp->b_blkno + bp->b_bcount; + } else { + blkdiff = BLK_MAX(mp); + } + + // scale access time by (distance in blocks from previous I/O / maximum blocks) + access_time_scale = weighted_scale_factor((double)blkdiff / BLK_MAX(mp)); + // most cases should pass in extents==1 for optimal delay calculation, otherwise just multiply delay by extents + delay_usec = (uint64_t)(((uint64_t)extents * info->access_time_usec) * access_time_scale); + + if (info->read_throughput_mbps && (bp->b_flags & B_READ)) { + delay_usec += (uint64_t)(total_size / ((double)(info->read_throughput_mbps * 1024 * 1024 / 8) / USEC_PER_SEC)); + } else if (info->write_throughput_mbps && !(bp->b_flags & B_READ)) { + delay_usec += (uint64_t)(total_size / ((double)(info->write_throughput_mbps * 1024 * 1024 / 8) / USEC_PER_SEC)); + } + + // try simulating disk spinup based on time since last I/O + if (!info->is_ssd) { + microuptime(&elapsed); + timevalsub(&elapsed, &info->last_io_timestamp); + // avoid this delay right after boot (assuming last_io_timestamp is 0 and disk is already spinning) + if (elapsed.tv_sec > DISK_IDLE_SEC && info->last_io_timestamp.tv_sec != 0) { + delay_usec += DISK_SPINUP_SEC * USEC_PER_SEC; + } + } + + if (delay_usec <= already_elapsed_usec) { + microuptime(&info->last_io_timestamp); + return; + } + + delay_usec -= already_elapsed_usec; + + while (delay_usec) { + microuptime(&start); + delay(delay_usec); + microuptime(&elapsed); + timevalsub(&elapsed, &start); + if (elapsed.tv_sec * USEC_PER_SEC < delay_usec) { + delay_usec -= elapsed.tv_sec * USEC_PER_SEC; + } else { + break; + } + if ((uint64_t)elapsed.tv_usec < delay_usec) { + delay_usec -= elapsed.tv_usec; + } else { + break; + } + } + + microuptime(&info->last_io_timestamp); +} + +int +disk_conditioner_get_info(mount_t mp, disk_conditioner_info *uinfo) +{ + struct _disk_conditioner_info_t *info; + + if (!mp) { + return EINVAL; + } + + info = mp->mnt_disk_conditioner_info; + + if (!info) { + return 0; + } + + uinfo->enabled = info->enabled; + uinfo->access_time_usec = info->access_time_usec; + uinfo->read_throughput_mbps = info->read_throughput_mbps; + uinfo->write_throughput_mbps = info->write_throughput_mbps; + uinfo->is_ssd = info->is_ssd; + + return 0; +} + +int +disk_conditioner_set_info(mount_t mp, disk_conditioner_info *uinfo) +{ + struct _disk_conditioner_info_t *info; + + if (!kauth_cred_issuser(kauth_cred_get()) || !IOTaskHasEntitlement(current_task(), DISK_CONDITIONER_SET_ENTITLEMENT)) { + return EPERM; + } + + if (!mp) { + return EINVAL; + } + + info = mp->mnt_disk_conditioner_info; + if (!info) { + info = mp->mnt_disk_conditioner_info = kalloc(sizeof(struct _disk_conditioner_info_t)); + bzero(info, sizeof(struct _disk_conditioner_info_t)); + } + + info->enabled = uinfo->enabled; + info->access_time_usec = uinfo->access_time_usec; + info->read_throughput_mbps = uinfo->read_throughput_mbps; + info->write_throughput_mbps = uinfo->write_throughput_mbps; + info->is_ssd = uinfo->is_ssd; + microuptime(&info->last_io_timestamp); + + // make sure throttling picks up the new periods + throttle_info_mount_reset_period(mp, info->is_ssd); + + return 0; +} + +void +disk_conditioner_unmount(mount_t mp) +{ + if (!mp->mnt_disk_conditioner_info) { + return; + } + kfree(mp->mnt_disk_conditioner_info, sizeof(struct _disk_conditioner_info_t)); + mp->mnt_disk_conditioner_info = NULL; +} + +boolean_t +disk_conditioner_mount_is_ssd(mount_t mp) +{ + struct _disk_conditioner_info_t *info = mp->mnt_disk_conditioner_info; + + if (!info || !info->enabled) { + return (mp->mnt_kern_flag & MNTK_SSD); + } + + return info->is_ssd; +} diff --git a/bsd/vfs/vfs_disk_conditioner.h b/bsd/vfs/vfs_disk_conditioner.h new file mode 100644 index 000000000..85feb15b5 --- /dev/null +++ b/bsd/vfs/vfs_disk_conditioner.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _VFS_DISK_CONDITIONER_H_ +#define _VFS_DISK_CONDITIONER_H_ + +#ifdef KERNEL_PRIVATE + +#include +int disk_conditioner_get_info(mount_t, disk_conditioner_info *); +int disk_conditioner_set_info(mount_t, disk_conditioner_info *); + +boolean_t disk_conditioner_mount_is_ssd(mount_t); + +#endif /* KERNEL_PRIVATE */ + +#endif /* !_VFS_DISK_CONDITIONER_H_ */ diff --git a/bsd/vfs/vfs_fsevents.c b/bsd/vfs/vfs_fsevents.c index 143f3cc11..f2e6b0bc3 100644 --- a/bsd/vfs/vfs_fsevents.c +++ b/bsd/vfs/vfs_fsevents.c @@ -62,6 +62,7 @@ #include #include +#include typedef struct kfs_event { LIST_ENTRY(kfs_event) kevent_list; @@ -397,7 +398,7 @@ add_fsevent(int type, vfs_context_t ctx, ...) // (as long as it's not an event type that can never be the // same as a previous event) // - if (type != FSE_CREATE_FILE && type != FSE_DELETE && type != FSE_RENAME && type != FSE_EXCHANGE && type != FSE_CHOWN && type != FSE_DOCID_CHANGED && type != FSE_DOCID_CREATED) { + if (type != FSE_CREATE_FILE && type != FSE_DELETE && type != FSE_RENAME && type != FSE_EXCHANGE && type != FSE_CHOWN && type != FSE_DOCID_CHANGED && type != FSE_DOCID_CREATED && type != FSE_CLONE) { void *ptr=NULL; int vid=0, was_str=0, nlen=0; @@ -465,7 +466,7 @@ add_fsevent(int type, vfs_context_t ctx, ...) kfse = zalloc_noblock(event_zone); - if (kfse && (type == FSE_RENAME || type == FSE_EXCHANGE)) { + if (kfse && (type == FSE_RENAME || type == FSE_EXCHANGE || type == FSE_CLONE)) { kfse_dest = zalloc_noblock(event_zone); if (kfse_dest == NULL) { did_alloc = 1; @@ -541,7 +542,7 @@ add_fsevent(int type, vfs_context_t ctx, ...) kfse->type = type; kfse->abstime = now; kfse->pid = p->p_pid; - if (type == FSE_RENAME || type == FSE_EXCHANGE) { + if (type == FSE_RENAME || type == FSE_EXCHANGE || type == FSE_CLONE) { memset(kfse_dest, 0, sizeof(kfs_event)); kfse_dest->refcount = 1; OSBitOrAtomic16(KFSE_BEING_CREATED, &kfse_dest->flags); @@ -1625,6 +1626,63 @@ fmod_watch(fs_event_watcher *watcher, struct uio *uio) void fsevent_unmount(__unused struct mount *mp, __unused vfs_context_t ctx) { +#if CONFIG_EMBEDDED + dev_t dev = mp->mnt_vfsstat.f_fsid.val[0]; + int error, waitcount = 0; + struct timespec ts = {1, 0}; + + // wait for any other pending unmounts to complete + lock_watch_table(); + while (fsevent_unmount_dev != 0) { + error = msleep((caddr_t)&fsevent_unmount_dev, &watch_table_lock, PRIBIO, "fsevent_unmount_wait", &ts); + if (error == EWOULDBLOCK) + error = 0; + if (!error && (++waitcount >= 10)) { + error = EWOULDBLOCK; + printf("timeout waiting to signal unmount pending for dev %d (fsevent_unmount_dev %d)\n", dev, fsevent_unmount_dev); + } + if (error) { + // there's a problem, bail out + unlock_watch_table(); + return; + } + } + if (fs_event_type_watchers[FSE_UNMOUNT_PENDING] == 0) { + // nobody watching for unmount pending events + unlock_watch_table(); + return; + } + // this is now the current unmount pending + fsevent_unmount_dev = dev; + fsevent_unmount_ack_count = fs_event_type_watchers[FSE_UNMOUNT_PENDING]; + unlock_watch_table(); + + // send an event to notify the watcher they need to get off the mount + error = add_fsevent(FSE_UNMOUNT_PENDING, ctx, FSE_ARG_DEV, dev, FSE_ARG_DONE); + + // wait for acknowledgment(s) (give up if it takes too long) + lock_watch_table(); + waitcount = 0; + while (fsevent_unmount_dev == dev) { + error = msleep((caddr_t)&fsevent_unmount_dev, &watch_table_lock, PRIBIO, "fsevent_unmount_pending", &ts); + if (error == EWOULDBLOCK) + error = 0; + if (!error && (++waitcount >= 10)) { + error = EWOULDBLOCK; + printf("unmount pending ack timeout for dev %d\n", dev); + } + if (error) { + // there's a problem, bail out + if (fsevent_unmount_dev == dev) { + fsevent_unmount_dev = 0; + fsevent_unmount_ack_count = 0; + } + wakeup((caddr_t)&fsevent_unmount_dev); + break; + } + } + unlock_watch_table(); +#endif } @@ -1982,24 +2040,25 @@ filt_fsevent_process(struct knote *kn, struct filt_process_s *data, struct keven return res; } -struct filterops fsevent_filtops = { - .f_isfd = 1, - .f_attach = NULL, - .f_detach = filt_fsevent_detach, +SECURITY_READ_ONLY_EARLY(struct filterops) fsevent_filtops = { + .f_isfd = 1, + .f_attach = NULL, + .f_detach = filt_fsevent_detach, .f_event = filt_fsevent, .f_touch = filt_fsevent_touch, .f_process = filt_fsevent_process, }; static int -fseventsf_kqfilter(__unused struct fileproc *fp, __unused struct knote *kn, __unused vfs_context_t ctx) +fseventsf_kqfilter(__unused struct fileproc *fp, __unused struct knote *kn, + __unused struct kevent_internal_s *kev, __unused vfs_context_t ctx) { fsevent_handle *fseh = (struct fsevent_handle *)fp->f_fglob->fg_data; int res; kn->kn_hook = (void*)fseh; kn->kn_hookid = 1; - kn->kn_filtid = EVFILTID_FSEVENT; + kn->kn_filtid = EVFILTID_FSEVENT; lock_watch_table(); @@ -2101,7 +2160,7 @@ parse_buffer_and_add_events(const char *buffer, int bufsize, vfs_context_t ctx, path_len = ptr - path; - if (type != FSE_RENAME && type != FSE_EXCHANGE) { + if (type != FSE_RENAME && type != FSE_EXCHANGE && type != FSE_CLONE) { event_start = ptr; // record where the next event starts err = add_fsevent(type, ctx, FSE_ARG_STRING, path_len, path, FSE_ARG_FINFO, finfo, FSE_ARG_DONE); diff --git a/bsd/vfs/vfs_lookup.c b/bsd/vfs/vfs_lookup.c index 128a8ce04..55b86f9e6 100644 --- a/bsd/vfs/vfs_lookup.c +++ b/bsd/vfs/vfs_lookup.c @@ -182,6 +182,7 @@ namei(struct nameidata *ndp) #if CONFIG_VOLFS int volfs_restarts = 0; #endif + size_t bytes_copied = 0; fdp = p->p_fd; @@ -244,10 +245,10 @@ vnode_recycled: retry_copy: if (UIO_SEG_IS_USER_SPACE(ndp->ni_segflg)) { error = copyinstr(ndp->ni_dirp, cnp->cn_pnbuf, - cnp->cn_pnlen, (size_t *)&ndp->ni_pathlen); + cnp->cn_pnlen, &bytes_copied); } else { error = copystr(CAST_DOWN(void *, ndp->ni_dirp), cnp->cn_pnbuf, - cnp->cn_pnlen, (size_t *)&ndp->ni_pathlen); + cnp->cn_pnlen, &bytes_copied); } if (error == ENAMETOOLONG && !(cnp->cn_flags & HASBUF)) { MALLOC_ZONE(cnp->cn_pnbuf, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK); @@ -258,11 +259,14 @@ retry_copy: cnp->cn_flags |= HASBUF; cnp->cn_pnlen = MAXPATHLEN; + bytes_copied = 0; goto retry_copy; } if (error) goto error_out; + ndp->ni_pathlen = bytes_copied; + bytes_copied = 0; /* * Since the name cache may contain positive entries of @@ -366,6 +370,21 @@ retry_copy: ndp->ni_vp = NULLVP; for (;;) { +#if CONFIG_MACF + /* + * Give MACF policies a chance to reject the lookup + * before performing any filesystem operations. + * This hook is called before resolving the path and + * again each time a symlink is encountered. + * NB: policies receive path information as supplied + * by the caller and thus cannot be trusted. + */ + error = mac_vnode_check_lookup_preflight(ctx, dp, cnp->cn_nameptr, cnp->cn_namelen); + if (error) { + goto error_out; + } +#endif + ndp->ni_startdir = dp; if ( (error = lookup(ndp)) ) { @@ -458,6 +477,7 @@ namei_compound_available(vnode_t dp, struct nameidata *ndp) return 0; } + static int lookup_authorize_search(vnode_t dp, struct componentname *cnp, int dp_authorized_in_cache, vfs_context_t ctx) { @@ -531,6 +551,7 @@ lookup_handle_rsrc_fork(vnode_t dp, struct nameidata *ndp, struct componentname { vnode_t svp = NULLVP; enum nsoperation nsop; + int nsflags; int error; if (dp->v_type != VREG) { @@ -567,8 +588,13 @@ lookup_handle_rsrc_fork(vnode_t dp, struct nameidata *ndp, struct componentname error = EPERM; goto out; } + + nsflags = 0; + if (cnp->cn_flags & CN_RAW_ENCRYPTED) + nsflags |= NS_GETRAWENCRYPTED; + /* Ask the file system for the resource fork. */ - error = vnode_getnamedstream(dp, &svp, XATTR_RESOURCEFORK_NAME, nsop, 0, ctx); + error = vnode_getnamedstream(dp, &svp, XATTR_RESOURCEFORK_NAME, nsop, nsflags, ctx); /* During a create, it OK for stream vnode to be missing. */ if (error == ENOATTR || error == ENOENT) { diff --git a/bsd/vfs/vfs_subr.c b/bsd/vfs/vfs_subr.c index 6b16ca6cb..5acfa82d4 100644 --- a/bsd/vfs/vfs_subr.c +++ b/bsd/vfs/vfs_subr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -110,7 +110,6 @@ #include #include -#include #include #include @@ -127,7 +126,9 @@ #include /* kalloc()/kfree() */ #include /* delay_for_interval() */ #include /* OSAddAtomic() */ +#if !CONFIG_EMBEDDED #include +#endif #ifdef JOE_DEBUG #include @@ -139,6 +140,9 @@ #include #endif +#include +#include + extern lck_grp_t *vnode_lck_grp; extern lck_attr_t *vnode_lck_attr; @@ -173,6 +177,8 @@ extern void memory_object_mark_io_tracking( /* XXX next protptype should be from */ extern int nfs_vinvalbuf(vnode_t, int, vfs_context_t, int); +extern int paniclog_append_noflush(const char *format, ...); + /* XXX next prototytype should be from libsa/stdlib.h> but conflicts libkern */ __private_extern__ void qsort( void * array, @@ -180,10 +186,7 @@ __private_extern__ void qsort( size_t member_size, int (*)(const void *, const void *)); -extern kern_return_t adjust_vm_object_cache(vm_size_t oval, vm_size_t nval); __private_extern__ void vntblinit(void); -__private_extern__ kern_return_t reset_vmobjectcache(unsigned int val1, - unsigned int val2); __private_extern__ int unlink1(vfs_context_t, vnode_t, user_addr_t, enum uio_seg, int); @@ -315,25 +318,6 @@ static int nummounts = 0; ragevnodes--; \ } while(0) - -/* - * vnodetarget hasn't been used in a long time, but - * it was exported for some reason... I'm leaving in - * place for now... it should be deprecated out of the - * exports and removed eventually. - */ -u_int32_t vnodetarget; /* target for vnreclaim() */ -#define VNODE_FREE_TARGET 20 /* Default value for vnodetarget */ - -/* - * We need quite a few vnodes on the free list to sustain the - * rapid stat() the compilation process does, and still benefit from the name - * cache. Having too few vnodes on the free list causes serious disk - * thrashing as we cycle through them. - */ -#define VNODE_FREE_MIN CONFIG_VNODE_FREE_MIN /* freelist should have at least this many */ - - static void async_work_continue(void); /* @@ -350,21 +334,12 @@ vntblinit(void) TAILQ_INIT(&vnode_async_work_list); TAILQ_INIT(&mountlist); - if (!vnodetarget) - vnodetarget = VNODE_FREE_TARGET; - microuptime(&rage_tv); rage_limit = desiredvnodes / 100; if (rage_limit < RAGE_LIMIT_MIN) rage_limit = RAGE_LIMIT_MIN; - /* - * Scale the vm_object_cache to accomodate the vnodes - * we want to cache - */ - (void) adjust_vm_object_cache(0, desiredvnodes - VNODE_FREE_MIN); - /* * create worker threads */ @@ -372,26 +347,6 @@ vntblinit(void) thread_deallocate(thread); } -/* Reset the VM Object Cache with the values passed in */ -__private_extern__ kern_return_t -reset_vmobjectcache(unsigned int val1, unsigned int val2) -{ - vm_size_t oval = val1 - VNODE_FREE_MIN; - vm_size_t nval; - - if (val1 == val2) { - return KERN_SUCCESS; - } - - if(val2 < VNODE_FREE_MIN) - nval = 0; - else - nval = val2 - VNODE_FREE_MIN; - - return(adjust_vm_object_cache(oval, nval)); -} - - /* the timeout is in 10 msecs */ int vnode_waitforwrites(vnode_t vp, int output_target, int slpflag, int slptimeout, const char *msg) { @@ -618,6 +573,7 @@ vnode_iterate_clear(mount_t mp) mp->mnt_lflag &= ~MNT_LITER; } +#if !CONFIG_EMBEDDED #include @@ -629,28 +585,28 @@ struct vnode_iterate_panic_hook { static void vnode_iterate_panic_hook(panic_hook_t *hook_) { - extern int kdb_log(const char *fmt, ...); struct vnode_iterate_panic_hook *hook = (struct vnode_iterate_panic_hook *)hook_; panic_phys_range_t range; uint64_t phys; if (panic_phys_range_before(hook->mp, &phys, &range)) { - kdb_log("mp = %p, phys = %p, prev (%p: %p-%p)\n", + paniclog_append_noflush("mp = %p, phys = %p, prev (%p: %p-%p)\n", hook->mp, phys, range.type, range.phys_start, range.phys_start + range.len); } else { - kdb_log("mp = %p, phys = %p, prev (!)\n", hook->mp, phys); + paniclog_append_noflush("mp = %p, phys = %p, prev (!)\n", hook->mp, phys); } if (panic_phys_range_before(hook->vp, &phys, &range)) { - kdb_log("vp = %p, phys = %p, prev (%p: %p-%p)\n", + paniclog_append_noflush("vp = %p, phys = %p, prev (%p: %p-%p)\n", hook->vp, phys, range.type, range.phys_start, range.phys_start + range.len); } else { - kdb_log("vp = %p, phys = %p, prev (!)\n", hook->vp, phys); + paniclog_append_noflush("vp = %p, phys = %p, prev (!)\n", hook->vp, phys); } panic_dump_mem((void *)(((vm_offset_t)hook->mp -4096) & ~4095), 12288); } +#endif //CONFIG_EMBEDDED int vnode_iterate(mount_t mp, int flags, int (*callout)(struct vnode *, void *), @@ -685,14 +641,18 @@ vnode_iterate(mount_t mp, int flags, int (*callout)(struct vnode *, void *), return(ret); } +#if !CONFIG_EMBEDDED struct vnode_iterate_panic_hook hook; hook.mp = mp; hook.vp = NULL; panic_hook(&hook.hook, vnode_iterate_panic_hook); +#endif /* iterate over all the vnodes */ while (!TAILQ_EMPTY(&mp->mnt_workerqueue)) { vp = TAILQ_FIRST(&mp->mnt_workerqueue); +#if !CONFIG_EMBEDDED hook.vp = vp; +#endif TAILQ_REMOVE(&mp->mnt_workerqueue, vp, v_mntvnodes); TAILQ_INSERT_TAIL(&mp->mnt_vnodelist, vp, v_mntvnodes); vid = vp->v_id; @@ -743,7 +703,9 @@ vnode_iterate(mount_t mp, int flags, int (*callout)(struct vnode *, void *), } out: +#if !CONFIG_EMBEDDED panic_unhook(&hook.hook); +#endif (void)vnode_iterate_reloadq(mp); vnode_iterate_clear(mp); mount_unlock(mp); @@ -1161,12 +1123,14 @@ vfs_mountroot(void) mp->mnt_kern_flag |= MNTK_UNMOUNT_PREFLIGHT; } +#if !CONFIG_EMBEDDED uint32_t speed; - if (MNTK_VIRTUALDEV & mp->mnt_kern_flag) speed = 128; - else if (MNTK_SSD & mp->mnt_kern_flag) speed = 7*256; - else speed = 256; + if (MNTK_VIRTUALDEV & mp->mnt_kern_flag) speed = 128; + else if (disk_conditioner_mount_is_ssd(mp)) speed = 7*256; + else speed = 256; vc_progress_setdiskspeed(speed); +#endif /* * Probe root file system for additional features. */ @@ -1452,7 +1416,6 @@ bdevvp(dev_t dev, vnode_t *vpp) return (0); } - /* * Check to see if the new vnode represents a special device * for which we already have a vnode (either because of @@ -2671,6 +2634,42 @@ vn_getpath_fsenter(struct vnode *vp, char *pathbuf, int *len) return build_path(vp, pathbuf, *len, len, 0, vfs_context_current()); } +/* + * vn_getpath_fsenter_with_parent will reenter the file system to fine the path of the + * vnode. It requires that there are IO counts on both the vnode and the directory vnode. + * + * vn_getpath_fsenter is called by MAC hooks to authorize operations for every thing, but + * unlink, rmdir and rename. For these operation the MAC hook calls vn_getpath. This presents + * problems where if the path can not be found from the name cache, those operations can + * erroneously fail with EPERM even though the call should succeed. When removing or moving + * file system objects with operations such as unlink or rename, those operations need to + * take IO counts on the target and containing directory. Calling vn_getpath_fsenter from a + * MAC hook from these operations during forced unmount operations can lead to dead + * lock. This happens when the operation starts, IO counts are taken on the containing + * directories and targets. Before the MAC hook is called a forced unmount from another + * thread takes place and blocks on the on going operation's directory vnode in vdrain. + * After which, the MAC hook gets called and calls vn_getpath_fsenter. vn_getpath_fsenter + * is called with the understanding that there is an IO count on the target. If in + * build_path the directory vnode is no longer in the cache, then the parent object id via + * vnode_getattr from the target is obtain and used to call VFS_VGET to get the parent + * vnode. The file system's VFS_VGET then looks up by inode in its hash and tries to get + * an IO count. But VFS_VGET "sees" the directory vnode is in vdrain and can block + * depending on which version and how it calls the vnode_get family of interfaces. + * + * N.B. A reasonable interface to use is vnode_getwithvid. This interface was modified to + * call vnode_getiocount with VNODE_DRAINO, so it will happily get an IO count and not + * cause issues, but there is no guarantee that all or any file systems are doing that. + * + * vn_getpath_fsenter_with_parent can enter the file system safely since there is a known + * IO count on the directory vnode by calling build_path_with_parent. + */ + +int +vn_getpath_fsenter_with_parent(struct vnode *dvp, struct vnode *vp, char *pathbuf, int *len) +{ + return build_path_with_parent(vp, dvp, pathbuf, *len, len, 0, vfs_context_current()); +} + int vn_getcdhash(struct vnode *vp, off_t offset, unsigned char *cdhash) { @@ -3347,6 +3346,13 @@ vfs_init_io_attributes(vnode_t devvp, mount_t mp) */ if ((cs_info.flags & DK_CORESTORAGE_PIN_YOUR_METADATA)) mp->mnt_ioflags |= MNT_IOFLAGS_FUSION_DRIVE; + } else { + /* Check for APFS Fusion */ + dk_apfs_flavour_t flavour; + if ((VNOP_IOCTL(devvp, DKIOCGETAPFSFLAVOUR, (caddr_t)&flavour, 0, ctx) == 0) && + (flavour == DK_APFS_FUSION)) { + mp->mnt_ioflags |= MNT_IOFLAGS_FUSION_DRIVE; + } } #if CONFIG_IOSCHED @@ -3625,7 +3631,7 @@ sysctl_vfs_ctlbyfsid(__unused struct sysctl_oid *oidp, void *arg1, int arg2, sfs.f_owner = sp->f_owner; #ifdef NFSCLIENT if (mp->mnt_kern_flag & MNTK_TYPENAME_OVERRIDE) { - strlcpy(&sfs.f_fstypename[0], &mp->fstypename_override[0], MFSTYPENAMELEN); + strlcpy(&sfs.f_fstypename[0], &mp->fstypename_override[0], MFSNAMELEN); } else #endif { @@ -3684,9 +3690,9 @@ sysctl_vfs_ctlbyfsid(__unused struct sysctl_oid *oidp, void *arg1, int arg2, sfs.f_fsid = sp->f_fsid; sfs.f_owner = sp->f_owner; -#ifdef NFS_CLIENT +#ifdef NFSCLIENT if (mp->mnt_kern_flag & MNTK_TYPENAME_OVERRIDE) { - strlcpy(&sfs.f_fstypename[0], &mp->fstypename_override[0], MFSTYPENAMELEN); + strlcpy(&sfs.f_fstypename[0], &mp->fstypename_override[0], MFSNAMELEN); } else #endif { @@ -3708,21 +3714,21 @@ out: return (error); } -static int filt_fsattach(struct knote *kn); +static int filt_fsattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_fsdetach(struct knote *kn); static int filt_fsevent(struct knote *kn, long hint); static int filt_fstouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_fsprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops fs_filtops = { - .f_attach = filt_fsattach, - .f_detach = filt_fsdetach, - .f_event = filt_fsevent, +SECURITY_READ_ONLY_EARLY(struct filterops) fs_filtops = { + .f_attach = filt_fsattach, + .f_detach = filt_fsdetach, + .f_event = filt_fsevent, .f_touch = filt_fstouch, .f_process = filt_fsprocess, }; static int -filt_fsattach(struct knote *kn) +filt_fsattach(struct knote *kn, __unused struct kevent_internal_s *kev) { lck_mtx_lock(fs_klist_lock); KNOTE_ATTACH(&fs_klist, kn); @@ -3908,6 +3914,16 @@ SYSCTL_NODE(_vfs_generic, VFS_CONF, conf, CTLFLAG_RD | CTLFLAG_LOCKED, sysctl_vfs_generic_conf, ""); +/* Indicate that the root file system unmounted cleanly */ +static int vfs_root_unmounted_cleanly = 0; +SYSCTL_INT(_vfs_generic, OID_AUTO, root_unmounted_cleanly, CTLFLAG_RD, &vfs_root_unmounted_cleanly, 0, "Root filesystem was unmounted cleanly"); + +void +vfs_set_root_unmounted_cleanly(void) +{ + vfs_root_unmounted_cleanly = 1; +} + /* * Print vnode state. */ @@ -6371,7 +6387,7 @@ vn_authorize_rmdir(vnode_t dvp, vnode_t vp, struct componentname *cnp, vfs_conte int vnode_attr_authorize_dir_clone(struct vnode_attr *vap, kauth_action_t action, struct vnode_attr *dvap, __unused vnode_t sdvp, mount_t mp, - dir_clone_authorizer_op_t vattr_op, vfs_context_t ctx, + dir_clone_authorizer_op_t vattr_op, uint32_t flags, vfs_context_t ctx, __unused void *reserved) { int error; @@ -6403,8 +6419,9 @@ vnode_attr_authorize_dir_clone(struct vnode_attr *vap, kauth_action_t action, VATTR_WANTED(vap, va_acl); if (dvap) VATTR_WANTED(dvap, va_gid); + } else if (dvap && (flags & VNODE_CLONEFILE_NOOWNERCOPY)) { + VATTR_WANTED(dvap, va_gid); } - return (0); } else if (vattr_op == OP_VATTR_CLEANUP) { return (0); /* Nothing to do for now */ @@ -6420,7 +6437,7 @@ vnode_attr_authorize_dir_clone(struct vnode_attr *vap, kauth_action_t action, * vn_attribute_prepare should be able to accept attributes as well as * vnodes but for now we do this inline. */ - if (!is_suser) { + if (!is_suser || (flags & VNODE_CLONEFILE_NOOWNERCOPY)) { /* * If the filesystem is mounted IGNORE_OWNERSHIP and an explicit * owner is set, that owner takes ownership of all new files. @@ -6454,12 +6471,12 @@ vnode_attr_authorize_dir_clone(struct vnode_attr *vap, kauth_action_t action, /* Inherit SF_RESTRICTED bit from destination directory only */ if (VATTR_IS_ACTIVE(vap, va_flags)) { VATTR_SET(vap, va_flags, - ((vap->va_flags & ~SF_RESTRICTED))); /* Turn off from source */ + ((vap->va_flags & ~(UF_DATAVAULT | SF_RESTRICTED)))); /* Turn off from source */ if (VATTR_IS_ACTIVE(dvap, va_flags)) VATTR_SET(vap, va_flags, - vap->va_flags | (dvap->va_flags & SF_RESTRICTED)); + vap->va_flags | (dvap->va_flags & (UF_DATAVAULT | SF_RESTRICTED))); } else if (VATTR_IS_ACTIVE(dvap, va_flags)) { - VATTR_SET(vap, va_flags, (dvap->va_flags & SF_RESTRICTED)); + VATTR_SET(vap, va_flags, (dvap->va_flags & (UF_DATAVAULT | SF_RESTRICTED))); } return (0); @@ -7959,7 +7976,8 @@ static int vnode_authattr_new_internal(vnode_t dvp, struct vnode_attr *vap, int noauth, uint32_t *defaulted_fieldsp, vfs_context_t ctx) { int error; - int has_priv_suser, ismember, defaulted_owner, defaulted_group, defaulted_mode, inherit_restricted; + int has_priv_suser, ismember, defaulted_owner, defaulted_group, defaulted_mode; + uint32_t inherit_flags; kauth_cred_t cred; guid_t changer; mount_t dmp; @@ -7973,7 +7991,7 @@ vnode_authattr_new_internal(vnode_t dvp, struct vnode_attr *vap, int noauth, uin defaulted_owner = defaulted_group = defaulted_mode = 0; - inherit_restricted = 0; + inherit_flags = 0; /* * Require that the filesystem support extended security to apply any. @@ -8038,9 +8056,8 @@ vnode_authattr_new_internal(vnode_t dvp, struct vnode_attr *vap, int noauth, uin /* Determine if SF_RESTRICTED should be inherited from the parent * directory. */ - if (VATTR_IS_SUPPORTED(&dva, va_flags) && - (dva.va_flags & SF_RESTRICTED)) { - inherit_restricted = 1; + if (VATTR_IS_SUPPORTED(&dva, va_flags)) { + inherit_flags = dva.va_flags & (UF_DATAVAULT | SF_RESTRICTED); } /* default mode is everything, masked with current umask */ @@ -8167,11 +8184,11 @@ vnode_authattr_new_internal(vnode_t dvp, struct vnode_attr *vap, int noauth, uin } } out: - if (inherit_restricted) { + if (inherit_flags) { /* Apply SF_RESTRICTED to the file if its parent directory was * restricted. This is done at the end so that root is not * required if this flag is only set due to inheritance. */ - VATTR_SET(vap, va_flags, (vap->va_flags | SF_RESTRICTED)); + VATTR_SET(vap, va_flags, (vap->va_flags | inherit_flags)); } if (defaulted_fieldsp) { if (defaulted_mode) { @@ -8264,7 +8281,8 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_ VATTR_IS_ACTIVE(vap, va_change_time) || VATTR_IS_ACTIVE(vap, va_modify_time) || VATTR_IS_ACTIVE(vap, va_access_time) || - VATTR_IS_ACTIVE(vap, va_backup_time)) { + VATTR_IS_ACTIVE(vap, va_backup_time) || + VATTR_IS_ACTIVE(vap, va_addedtime)) { VATTR_WANTED(&ova, va_uid); #if 0 /* enable this when we support UUIDs as official owners */ @@ -8333,7 +8351,8 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_ VATTR_IS_ACTIVE(vap, va_change_time) || VATTR_IS_ACTIVE(vap, va_modify_time) || VATTR_IS_ACTIVE(vap, va_access_time) || - VATTR_IS_ACTIVE(vap, va_backup_time)) { + VATTR_IS_ACTIVE(vap, va_backup_time) || + VATTR_IS_ACTIVE(vap, va_addedtime)) { /* * The owner and root may set any timestamps they like, * provided that the file is not immutable. The owner still needs @@ -9130,8 +9149,6 @@ static char *__vpath(vnode_t vp, char *str, int len, int depth) return dst; } -extern int kdb_printf(const char *format, ...) __printflike(1,2); - #define SANE_VNODE_PRINT_LIMIT 5000 void panic_print_vnodes(void) { @@ -9142,7 +9159,7 @@ void panic_print_vnodes(void) char *nm; char vname[257]; - kdb_printf("\n***** VNODES *****\n" + paniclog_append_noflush("\n***** VNODES *****\n" "TYPE UREF ICNT PATH\n"); /* NULL-terminate the path name */ @@ -9154,7 +9171,7 @@ void panic_print_vnodes(void) TAILQ_FOREACH(mnt, &mountlist, mnt_list) { if (!ml_validate_nofault((vm_offset_t)mnt, sizeof(mount_t))) { - kdb_printf("Unable to iterate the mount list %p - encountered an invalid mount pointer %p \n", + paniclog_append_noflush("Unable to iterate the mount list %p - encountered an invalid mount pointer %p \n", &mountlist, mnt); break; } @@ -9162,7 +9179,7 @@ void panic_print_vnodes(void) TAILQ_FOREACH(vp, &mnt->mnt_vnodelist, v_mntvnodes) { if (!ml_validate_nofault((vm_offset_t)vp, sizeof(vnode_t))) { - kdb_printf("Unable to iterate the vnode list %p - encountered an invalid vnode pointer %p \n", + paniclog_append_noflush("Unable to iterate the vnode list %p - encountered an invalid vnode pointer %p \n", &mnt->mnt_vnodelist, vp); break; } @@ -9171,7 +9188,7 @@ void panic_print_vnodes(void) return; type = __vtype(vp->v_type); nm = __vpath(vp, vname, sizeof(vname)-1, 0); - kdb_printf("%s %0d %0d %s\n", + paniclog_append_noflush("%s %0d %0d %s\n", type, vp->v_usecount, vp->v_iocount, nm); } } diff --git a/bsd/vfs/vfs_syscalls.c b/bsd/vfs/vfs_syscalls.c index ee016dac6..788314616 100644 --- a/bsd/vfs/vfs_syscalls.c +++ b/bsd/vfs/vfs_syscalls.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1995-2016 Apple Inc. All rights reserved. + * Copyright (c) 1995-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -107,6 +107,8 @@ #include #include +#include + #include #include @@ -143,6 +145,8 @@ FREE_ZONE((x), MAXPATHLEN, M_NAMEI); #endif /* CONFIG_FSE */ +extern void disk_conditioner_unmount(mount_t mp); + /* struct for checkdirs iteration */ struct cdirargs { vnode_t olddp; @@ -303,6 +307,75 @@ mount(proc_t p, struct mount_args *uap, __unused int32_t *retval) return (__mac_mount(p, &muap, retval)); } +int +fmount(__unused proc_t p, struct fmount_args *uap, __unused int32_t *retval) +{ + struct componentname cn; + vfs_context_t ctx = vfs_context_current(); + size_t dummy = 0; + int error; + int flags = uap->flags; + char fstypename[MFSNAMELEN]; + char *labelstr = NULL; /* regular mount call always sets it to NULL for __mac_mount() */ + vnode_t pvp; + vnode_t vp; + + AUDIT_ARG(fd, uap->fd); + AUDIT_ARG(fflags, flags); + /* fstypename will get audited by mount_common */ + + /* Sanity check the flags */ + if (flags & (MNT_IMGSRC_BY_INDEX|MNT_ROOTFS)) { + return (ENOTSUP); + } + + if (flags & MNT_UNION) { + return (EPERM); + } + + error = copyinstr(uap->type, fstypename, MFSNAMELEN, &dummy); + if (error) { + return (error); + } + + if ((error = file_vnode(uap->fd, &vp)) != 0) { + return (error); + } + + if ((error = vnode_getwithref(vp)) != 0) { + file_drop(uap->fd); + return (error); + } + + pvp = vnode_getparent(vp); + if (pvp == NULL) { + vnode_put(vp); + file_drop(uap->fd); + return (EINVAL); + } + + memset(&cn, 0, sizeof(struct componentname)); + MALLOC(cn.cn_pnbuf, char *, MAXPATHLEN, M_TEMP, M_WAITOK); + cn.cn_pnlen = MAXPATHLEN; + + if((error = vn_getpath(vp, cn.cn_pnbuf, &cn.cn_pnlen)) != 0) { + FREE(cn.cn_pnbuf, M_TEMP); + vnode_put(pvp); + vnode_put(vp); + file_drop(uap->fd); + return (error); + } + + error = mount_common(fstypename, pvp, vp, &cn, uap->data, flags, 0, labelstr, FALSE, ctx); + + FREE(cn.cn_pnbuf, M_TEMP); + vnode_put(pvp); + vnode_put(vp); + file_drop(uap->fd); + + return (error); +} + void vfs_notify_mount(vnode_t pdvp) { @@ -603,6 +676,7 @@ mount_common(char *fstypename, vnode_t pvp, vnode_t vp, vfsp = mp->mnt_vtable; goto update; } + /* * For non-root users, silently enforce MNT_NOSUID and MNT_NODEV, and * MNT_NOEXEC if mount point is already MNT_NOEXEC. @@ -690,6 +764,7 @@ mount_common(char *fstypename, vnode_t pvp, vnode_t vp, #endif /* NFSCLIENT || DEVFS */ update: + /* * Set the mount level flags. */ @@ -713,7 +788,7 @@ update: #if SECURE_KERNEL #if !CONFIG_MNT_SUID /* - * On release builds of iOS based platforms, always enforce NOSUID and NODEV on + * On release builds of iOS based platforms, always enforce NOSUID on * all mounts. We do this here because we can catch update mounts as well as * non-update mounts in this case. */ @@ -1959,6 +2034,9 @@ dounmount(struct mount *mp, int flags, int withref, vfs_context_t ctx) } } + /* free disk_conditioner_info structure for this mount */ + disk_conditioner_unmount(mp); + IOBSDMountChange(mp, kIOMountChangeUnmount); #if CONFIG_TRIGGERS @@ -2400,7 +2478,9 @@ quotactl(proc_t p, struct quotactl_args *uap, __unused int32_t *retval) /* uap->arg is a pointer to a dqblk structure we need to copy out to */ if (error == 0) { if (proc_is64bit(p)) { - struct user_dqblk my_dqblk64 = {.dqb_bhardlimit = 0}; + struct user_dqblk my_dqblk64; + + memset(&my_dqblk64, 0, sizeof(my_dqblk64)); munge_dqblk(&my_dqblk, &my_dqblk64, TRUE); error = copyout((caddr_t)&my_dqblk64, uap->arg, sizeof (my_dqblk64)); } @@ -3532,7 +3612,13 @@ open1(vfs_context_t ctx, struct nameidata *ndp, int uflags, TRUE); } } else if (secluded_for_filecache == 2) { +#if __arm64__ +#define DYLD_SHARED_CACHE_NAME "dyld_shared_cache_arm64" +#elif __arm__ +#define DYLD_SHARED_CACHE_NAME "dyld_shared_cache_armv7" +#else /* not implemented... */ +#endif if (!strncmp(vp->v_name, DYLD_SHARED_CACHE_NAME, strlen(DYLD_SHARED_CACHE_NAME)) || @@ -4918,10 +5004,10 @@ lseek(proc_t p, struct lseek_args *uap, off_t *retval) case L_SET: break; case SEEK_HOLE: - error = VNOP_IOCTL(vp, FSCTL_FIOSEEKHOLE, (caddr_t)&offset, 0, ctx); + error = VNOP_IOCTL(vp, FSIOC_FIOSEEKHOLE, (caddr_t)&offset, 0, ctx); break; case SEEK_DATA: - error = VNOP_IOCTL(vp, FSCTL_FIOSEEKDATA, (caddr_t)&offset, 0, ctx); + error = VNOP_IOCTL(vp, FSIOC_FIOSEEKDATA, (caddr_t)&offset, 0, ctx); break; default: error = EINVAL; @@ -6864,6 +6950,7 @@ clonefile_internal(vnode_t fvp, boolean_t data_read_authorised, int dst_dirfd, uint32_t defaulted; struct vnode_attr va; struct vnode_attr nva; + uint32_t vnop_flags; v_type = vnode_vtype(fvp); switch (v_type) { @@ -6952,6 +7039,7 @@ clonefile_internal(vnode_t fvp, boolean_t data_read_authorised, int dst_dirfd, attr_cleanup = TRUE; } + vnop_flags = VNODE_CLONEFILE_DEFAULT; /* * We've got initial values for all security parameters, * If we are superuser, then we can change owners to be the @@ -6959,22 +7047,24 @@ clonefile_internal(vnode_t fvp, boolean_t data_read_authorised, int dst_dirfd, * WRITE_SECURITY privileges so all other fields can be taken * from source as well. */ - if (vfs_context_issuser(ctx)) { + if (!(flags & CLONE_NOOWNERCOPY) && vfs_context_issuser(ctx)) { if (VATTR_IS_SUPPORTED(&va, va_uid)) VATTR_SET(&nva, va_uid, va.va_uid); if (VATTR_IS_SUPPORTED(&va, va_gid)) VATTR_SET(&nva, va_gid, va.va_gid); + } else { + vnop_flags |= VNODE_CLONEFILE_NOOWNERCOPY; } + if (VATTR_IS_SUPPORTED(&va, va_mode)) VATTR_SET(&nva, va_mode, va.va_mode); if (VATTR_IS_SUPPORTED(&va, va_flags)) { VATTR_SET(&nva, va_flags, - ((va.va_flags & ~SF_RESTRICTED) | /* Turn off from source */ - (nva.va_flags & SF_RESTRICTED))); + ((va.va_flags & ~(UF_DATAVAULT | SF_RESTRICTED)) | /* Turn off from source */ + (nva.va_flags & (UF_DATAVAULT | SF_RESTRICTED)))); } - error = VNOP_CLONEFILE(fvp, tdvp, &tvp, cnp, &nva, - VNODE_CLONEFILE_DEFAULT, ctx); + error = VNOP_CLONEFILE(fvp, tdvp, &tvp, cnp, &nva, vnop_flags, ctx); if (!error && tvp) { int update_flags = 0; @@ -7019,6 +7109,17 @@ clonefile_internal(vnode_t fvp, boolean_t data_read_authorised, int dst_dirfd, } if (need_fsevent(fsevent, tvp)) { + /* + * The following is a sequence of three explicit events. + * A pair of FSE_CLONE events representing the source and destination + * followed by an FSE_CREATE_[FILE | DIR] for the destination. + * fseventsd may coalesce the destination clone and create events + * into a single event resulting in the following sequence for a client + * FSE_CLONE (src) + * FSE_CLONE | FSE_CREATE (dst) + */ + add_fsevent(FSE_CLONE, ctx, FSE_ARG_VNODE, fvp, FSE_ARG_VNODE, tvp, + FSE_ARG_DONE); add_fsevent(fsevent, ctx, FSE_ARG_VNODE, tvp, FSE_ARG_DONE); } @@ -7052,7 +7153,7 @@ clonefileat(__unused proc_t p, struct clonefileat_args *uap, vfs_context_t ctx = vfs_context_current(); /* Check that the flags are valid. */ - if (uap->flags & ~CLONE_NOFOLLOW) + if (uap->flags & ~(CLONE_NOFOLLOW | CLONE_NOOWNERCOPY)) return (EINVAL); AUDIT_ARG(fd, uap->src_dirfd); @@ -7082,6 +7183,10 @@ fclonefileat(__unused proc_t p, struct fclonefileat_args *uap, int error; vfs_context_t ctx = vfs_context_current(); + /* Check that the flags are valid. */ + if (uap->flags & ~(CLONE_NOFOLLOW | CLONE_NOOWNERCOPY)) + return (EINVAL); + AUDIT_ARG(fd, uap->src_fd); error = fp_getfvp(p, uap->src_fd, &fp, &fvp); if (error) @@ -8156,6 +8261,14 @@ rmdir(__unused proc_t p, struct rmdir_args *uap, __unused int32_t *retval) #define DIRENT64_LEN(namlen) \ ((sizeof(struct direntry) + (namlen) - (MAXPATHLEN-1) + 7) & ~7) +/* Get dirent length padded to 4 byte alignment */ +#define DIRENT_LEN(namelen) \ + ((sizeof(struct dirent) + (namelen + 1) - (__DARWIN_MAXNAMLEN + 1) + 3) & ~3) + +/* Get the end of this dirent */ +#define DIRENT_END(dep) \ + (((char *)(dep)) + (dep)->d_reclen - 1) + errno_t vnode_readdir64(struct vnode *vp, struct uio *uio, int flags, int *eofflag, int *numdirent, vfs_context_t ctxp) @@ -8174,9 +8287,13 @@ vnode_readdir64(struct vnode *vp, struct uio *uio, int flags, int *eofflag, int error; /* - * Our kernel buffer needs to be smaller since re-packing - * will expand each dirent. The worse case (when the name - * length is 3) corresponds to a struct direntry size of 32 + * We're here because the underlying file system does not + * support direnties or we mounted denying support so we must + * fall back to dirents and convert them to direntries. + * + * Our kernel buffer needs to be smaller since re-packing will + * expand each dirent. The worse case (when the name length + * is 3 or less) corresponds to a struct direntry size of 32 * bytes (8-byte aligned) and a struct dirent size of 12 bytes * (4-byte aligned). So having a buffer that is 3/8 the size * will prevent us from reading more than we can pack. @@ -8209,6 +8326,15 @@ vnode_readdir64(struct vnode *vp, struct uio *uio, int flags, int *eofflag, while (error == 0 && (char *)dep < ((char *)bufptr + bytesread)) { size_t enbufsize = DIRENT64_LEN(dep->d_namlen); + if (DIRENT_END(dep) > ((char *)bufptr + bytesread) || + DIRENT_LEN(dep->d_namlen) > dep->d_reclen) { + printf("%s: %s: Bad dirent recived from directory %s\n", __func__, + vp->v_mount->mnt_vfsstat.f_mntonname, + vp->v_name ? vp->v_name : ""); + error = EIO; + break; + } + bzero(entry64, enbufsize); /* Convert a dirent to a dirent64. */ entry64->d_ino = dep->d_ino; @@ -9890,6 +10016,52 @@ static int process_namespace_fsctl(nspace_type_t nspace_type, int is64bit, u_int return wait_for_namespace_event(&nhd, nspace_type); } +static unsigned long +fsctl_bogus_command_compat(unsigned long cmd) +{ + + switch (cmd) { + case IOCBASECMD(FSIOC_SYNC_VOLUME): + return (FSIOC_SYNC_VOLUME); + case IOCBASECMD(FSIOC_ROUTEFS_SETROUTEID): + return (FSIOC_ROUTEFS_SETROUTEID); + case IOCBASECMD(FSIOC_SET_PACKAGE_EXTS): + return (FSIOC_SET_PACKAGE_EXTS); + case IOCBASECMD(FSIOC_NAMESPACE_HANDLER_GET): + return (FSIOC_NAMESPACE_HANDLER_GET); + case IOCBASECMD(FSIOC_OLD_SNAPSHOT_HANDLER_GET): + return (FSIOC_OLD_SNAPSHOT_HANDLER_GET); + case IOCBASECMD(FSIOC_SNAPSHOT_HANDLER_GET_EXT): + return (FSIOC_SNAPSHOT_HANDLER_GET_EXT); + case IOCBASECMD(FSIOC_NAMESPACE_HANDLER_UPDATE): + return (FSIOC_NAMESPACE_HANDLER_UPDATE); + case IOCBASECMD(FSIOC_NAMESPACE_HANDLER_UNBLOCK): + return (FSIOC_NAMESPACE_HANDLER_UNBLOCK); + case IOCBASECMD(FSIOC_NAMESPACE_HANDLER_CANCEL): + return (FSIOC_NAMESPACE_HANDLER_CANCEL); + case IOCBASECMD(FSIOC_NAMESPACE_HANDLER_SET_SNAPSHOT_TIME): + return (FSIOC_NAMESPACE_HANDLER_SET_SNAPSHOT_TIME); + case IOCBASECMD(FSIOC_NAMESPACE_ALLOW_DMG_SNAPSHOT_EVENTS): + return (FSIOC_NAMESPACE_ALLOW_DMG_SNAPSHOT_EVENTS); + case IOCBASECMD(FSIOC_SET_FSTYPENAME_OVERRIDE): + return (FSIOC_SET_FSTYPENAME_OVERRIDE); + case IOCBASECMD(DISK_CONDITIONER_IOC_GET): + return (DISK_CONDITIONER_IOC_GET); + case IOCBASECMD(DISK_CONDITIONER_IOC_SET): + return (DISK_CONDITIONER_IOC_SET); + case IOCBASECMD(FSIOC_FIOSEEKHOLE): + return (FSIOC_FIOSEEKHOLE); + case IOCBASECMD(FSIOC_FIOSEEKDATA): + return (FSIOC_FIOSEEKDATA); + case IOCBASECMD(SPOTLIGHT_IOC_GET_MOUNT_TIME): + return (SPOTLIGHT_IOC_GET_MOUNT_TIME); + case IOCBASECMD(SPOTLIGHT_IOC_GET_LAST_MTIME): + return (SPOTLIGHT_IOC_GET_LAST_MTIME); + } + + return (cmd); +} + /* * Make a filesystem-specific control call: */ @@ -9905,6 +10077,8 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long caddr_t data, memp; vnode_t vp = *arg_vp; + cmd = fsctl_bogus_command_compat(cmd); + size = IOCPARM_LEN(cmd); if (size > IOCPARM_MAX) return (EINVAL); @@ -9912,34 +10086,6 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long memp = NULL; - - /* - * ensure the buffer is large enough for underlying calls - */ -#ifndef HFSIOC_GETPATH - typedef char pn_t[MAXPATHLEN]; -#define HFSIOC_GETPATH _IOWR('h', 13, pn_t) -#endif - -#ifndef HFS_GETPATH -#define HFS_GETPATH IOCBASECMD(HFSIOC_GETPATH) -#endif - if (IOCBASECMD(cmd) == HFS_GETPATH) { - /* Round up to MAXPATHLEN regardless of user input */ - size = MAXPATHLEN; - } - else if (vp->v_tag == VT_CIFS) { - /* - * XXX Until fsctl's length encoding can be - * XXX fixed properly. - */ - if (IOCBASECMD(cmd) == _IOWR('z', 19, 0) && size < 1432) { - size = 1432; /* sizeof(struct UniqueSMBShareID) */ - } else if (IOCBASECMD(cmd) == _IOWR('z', 28, 0) && size < 308) { - size = 308; /* sizeof(struct smbDebugTestPB) */ - } - } - if (size > sizeof (stkbuf)) { if ((memp = (caddr_t)kalloc(size)) == 0) return ENOMEM; data = memp; @@ -9980,9 +10126,9 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } /* Check to see if it's a generic command */ - switch (IOCBASECMD(cmd)) { + switch (cmd) { - case FSCTL_SYNC_VOLUME: { + case FSIOC_SYNC_VOLUME: { mount_t mp = vp->v_mount; int arg = *(uint32_t*)data; @@ -10022,7 +10168,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_ROUTEFS_SETROUTEID: { + case FSIOC_ROUTEFS_SETROUTEID: { #if ROUTEFS char routepath[MAXPATHLEN]; size_t len = 0; @@ -10043,7 +10189,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_SET_PACKAGE_EXTS: { + case FSIOC_SET_PACKAGE_EXTS: { user_addr_t ext_strings; uint32_t num_entries; uint32_t max_width; @@ -10075,23 +10221,23 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long break; /* namespace handlers */ - case FSCTL_NAMESPACE_HANDLER_GET: { + case FSIOC_NAMESPACE_HANDLER_GET: { error = process_namespace_fsctl(NSPACE_HANDLER_NSPACE, is64bit, size, data); } break; /* Snapshot handlers */ - case FSCTL_OLD_SNAPSHOT_HANDLER_GET: { + case FSIOC_OLD_SNAPSHOT_HANDLER_GET: { error = process_namespace_fsctl(NSPACE_HANDLER_SNAPSHOT, is64bit, size, data); } break; - case FSCTL_SNAPSHOT_HANDLER_GET_EXT: { + case FSIOC_SNAPSHOT_HANDLER_GET_EXT: { error = process_namespace_fsctl(NSPACE_HANDLER_SNAPSHOT, is64bit, size, data); } break; - case FSCTL_NAMESPACE_HANDLER_UPDATE: { + case FSIOC_NAMESPACE_HANDLER_UPDATE: { uint32_t token, val; int i; @@ -10133,7 +10279,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_NAMESPACE_HANDLER_UNBLOCK: { + case FSIOC_NAMESPACE_HANDLER_UNBLOCK: { uint32_t token, val; int i; @@ -10181,7 +10327,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_NAMESPACE_HANDLER_CANCEL: { + case FSIOC_NAMESPACE_HANDLER_CANCEL: { uint32_t token, val; int i; @@ -10229,7 +10375,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_NAMESPACE_HANDLER_SET_SNAPSHOT_TIME: { + case FSIOC_NAMESPACE_HANDLER_SET_SNAPSHOT_TIME: { if ((error = suser(kauth_cred_get(), &(current_proc()->p_acflag)))) { break; } @@ -10245,7 +10391,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_NAMESPACE_ALLOW_DMG_SNAPSHOT_EVENTS: + case FSIOC_NAMESPACE_ALLOW_DMG_SNAPSHOT_EVENTS: { if ((error = suser(kauth_cred_get(), &(current_proc()->p_acflag)))) { break; @@ -10261,7 +10407,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_SET_FSTYPENAME_OVERRIDE: + case FSIOC_SET_FSTYPENAME_OVERRIDE: { if ((error = suser(kauth_cred_get(), &(current_proc()->p_acflag)))) { break; @@ -10287,9 +10433,19 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; + case DISK_CONDITIONER_IOC_GET: { + error = disk_conditioner_get_info(vp->v_mount, (disk_conditioner_info *)data); + } + break; + + case DISK_CONDITIONER_IOC_SET: { + error = disk_conditioner_set_info(vp->v_mount, (disk_conditioner_info *)data); + } + break; + default: { /* Invoke the filesystem-specific code */ - error = VNOP_IOCTL(vp, IOCBASECMD(cmd), data, options, ctx); + error = VNOP_IOCTL(vp, cmd, data, options, ctx); } } /* end switch stmt */ @@ -10922,10 +11078,7 @@ out: /* * Obtain the full pathname of a file system object by id. - * - * This is a private SPI used by the File Manager. */ -__private_extern__ int fsgetpath(__unused proc_t p, struct fsgetpath_args *uap, user_ssize_t *retval) { @@ -11563,11 +11716,7 @@ snapshot_revert(int dirfd, user_addr_t name, __unused uint32_t flags, #define APFSIOC_REVERT_TO_SNAPSHOT _IOW('J', 1, u_int64_t) #endif -#ifndef APFS_REVERT_TO_SNAPSHOT -#define APFS_REVERT_TO_SNAPSHOT IOCBASECMD(APFSIOC_REVERT_TO_SNAPSHOT) -#endif - - error = VNOP_IOCTL(namend.ni_vp, APFS_REVERT_TO_SNAPSHOT, (caddr_t) NULL, + error = VNOP_IOCTL(namend.ni_vp, APFSIOC_REVERT_TO_SNAPSHOT, (caddr_t) NULL, 0, ctx); vnode_put(namend.ni_vp); @@ -11731,7 +11880,7 @@ snapshot_mount(int dirfd, user_addr_t name, user_addr_t directory, smnt_data.sm_mp = mp; smnt_data.sm_cnp = &snapndp->ni_cnd; error = mount_common(mp->mnt_vfsstat.f_fstypename, pvp, vp, - &dirndp->ni_cnd, CAST_USER_ADDR_T(&smnt_data), 0, + &dirndp->ni_cnd, CAST_USER_ADDR_T(&smnt_data), flags & MNT_DONTBROWSE, KERNEL_MOUNT_SNAPSHOT, NULL, FALSE, ctx); } diff --git a/bsd/vfs/vfs_vnops.c b/bsd/vfs/vfs_vnops.c index 7a1a2b216..797573d75 100644 --- a/bsd/vfs/vfs_vnops.c +++ b/bsd/vfs/vfs_vnops.c @@ -112,18 +112,19 @@ int ubc_setcred(struct vnode *, struct proc *); #endif #include +#include static int vn_closefile(struct fileglob *fp, vfs_context_t ctx); static int vn_ioctl(struct fileproc *fp, u_long com, caddr_t data, - vfs_context_t ctx); + vfs_context_t ctx); static int vn_read(struct fileproc *fp, struct uio *uio, int flags, - vfs_context_t ctx); + vfs_context_t ctx); static int vn_write(struct fileproc *fp, struct uio *uio, int flags, - vfs_context_t ctx); + vfs_context_t ctx); static int vn_select( struct fileproc *fp, int which, void * wql, - vfs_context_t ctx); + vfs_context_t ctx); static int vn_kqfilt_add(struct fileproc *fp, struct knote *kn, - vfs_context_t ctx); + struct kevent_internal_s *kev, vfs_context_t ctx); static void filt_vndetach(struct knote *kn); static int filt_vnode(struct knote *kn, long hint); static int filt_vnode_common(struct knote *kn, vnode_t vp, long hint); @@ -147,10 +148,10 @@ const struct fileops vnops = { static int filt_vntouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_vnprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops vnode_filtops = { - .f_isfd = 1, - .f_attach = NULL, - .f_detach = filt_vndetach, +SECURITY_READ_ONLY_EARLY(struct filterops) vnode_filtops = { + .f_isfd = 1, + .f_attach = NULL, + .f_detach = filt_vndetach, .f_event = filt_vnode, .f_touch = filt_vntouch, .f_process = filt_vnprocess, @@ -1342,7 +1343,7 @@ vn_stat_noauth(struct vnode *vp, void *sbptr, kauth_filesec_t *xsec, int isstat6 }; if (isstat64 != 0) { sb64->st_mode = mode; - sb64->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? (u_int16_t)va.va_nlink : 1; + sb64->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? va.va_nlink > UINT16_MAX ? UINT16_MAX : (u_int16_t)va.va_nlink : 1; sb64->st_uid = va.va_uid; sb64->st_gid = va.va_gid; sb64->st_rdev = va.va_rdev; @@ -1360,7 +1361,7 @@ vn_stat_noauth(struct vnode *vp, void *sbptr, kauth_filesec_t *xsec, int isstat6 sb64->st_blocks = roundup(va.va_total_alloc, 512) / 512; } else { sb->st_mode = mode; - sb->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? (u_int16_t)va.va_nlink : 1; + sb->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? va.va_nlink > UINT16_MAX ? UINT16_MAX : (u_int16_t)va.va_nlink : 1; sb->st_uid = va.va_uid; sb->st_gid = va.va_gid; sb->st_rdev = va.va_rdev; @@ -1686,12 +1687,13 @@ vn_pathconf(vnode_t vp, int name, int32_t *retval, vfs_context_t ctx) } static int -vn_kqfilt_add(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) +vn_kqfilt_add(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx) { struct vnode *vp; int error = 0; int result = 0; - + vp = (struct vnode *)fp->f_fglob->fg_data; /* @@ -1709,7 +1711,7 @@ vn_kqfilt_add(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) } else if (!vnode_isreg(vp)) { if (vnode_ischr(vp)) { - result = spec_kqfilter(vp, kn); + result = spec_kqfilter(vp, kn, kev); if ((kn->kn_flags & EV_ERROR) == 0) { /* claimed by a special device */ vnode_put(vp); diff --git a/bsd/vfs/vfs_xattr.c b/bsd/vfs/vfs_xattr.c index bd38c5f51..b47ec5553 100644 --- a/bsd/vfs/vfs_xattr.c +++ b/bsd/vfs/vfs_xattr.c @@ -376,7 +376,7 @@ xattr_validatename(const char *name) if (name == NULL || name[0] == '\0') { return (EINVAL); } - namelen = strnlen(name, XATTR_MAXNAMELEN); + namelen = strlen(name); if (name[namelen] != '\0') return (ENAMETOOLONG); @@ -407,10 +407,14 @@ vnode_getnamedstream(vnode_t vp, vnode_t *svpp, const char *name, enum nsoperati { int error; - if (vp->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS) + if (vp->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS) { error = VNOP_GETNAMEDSTREAM(vp, svpp, name, op, flags, context); - else - error = default_getnamedstream(vp, svpp, name, op, context); + } else { + if (flags) + error = ENOTSUP; + else + error = default_getnamedstream(vp, svpp, name, op, context); + } if (error == 0) { uint32_t streamflags = VISNAMEDSTREAM; @@ -1602,7 +1606,7 @@ default_getxattr(vnode_t vp, const char *name, uio_t uio, size_t *size, int error; fileflags = FREAD; - if (bcmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) == 0) { + if (strcmp(name, XATTR_RESOURCEFORK_NAME) == 0) { isrsrcfork = 1; /* * Open the file locked (shared) since the Carbon @@ -1623,7 +1627,7 @@ default_getxattr(vnode_t vp, const char *name, uio_t uio, size_t *size, } /* Get the Finder Info. */ - if (bcmp(name, XATTR_FINDERINFO_NAME, sizeof(XATTR_FINDERINFO_NAME)) == 0) { + if (strcmp(name, XATTR_FINDERINFO_NAME) == 0) { if (ainfo.finderinfo == NULL || ainfo.emptyfinderinfo) { error = ENOATTR; @@ -2113,7 +2117,7 @@ default_removexattr(vnode_t vp, const char *name, __unused int options, vfs_cont int error; fileflags = FREAD | FWRITE; - if (bcmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) == 0) { + if (strncmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) == 0) { isrsrcfork = 1; /* * Open the file locked (exclusive) since the Carbon @@ -2140,7 +2144,7 @@ default_removexattr(vnode_t vp, const char *name, __unused int options, vfs_cont ++attrcount; /* Clear the Finder Info. */ - if (bcmp(name, XATTR_FINDERINFO_NAME, sizeof(XATTR_FINDERINFO_NAME)) == 0) { + if (strncmp(name, XATTR_FINDERINFO_NAME, sizeof(XATTR_FINDERINFO_NAME)) == 0) { if (ainfo.finderinfo == NULL || ainfo.emptyfinderinfo) { error = ENOATTR; goto out; @@ -2380,7 +2384,9 @@ default_listxattr(vnode_t vp, uio_t uio, size_t *size, __unused int options, vfs count = ainfo.attrhdr->num_attrs; for (i = 0, entry = ainfo.attr_entry; i < count && ATTR_VALID(entry, ainfo); i++) { if (xattr_protected((const char *)entry->name) || - xattr_validatename((const char *)entry->name) != 0) { + ((entry->namelen < XATTR_MAXNAMELEN) && + (entry->name[entry->namelen] == '\0') && + (xattr_validatename((const char *)entry->name) != 0))) { entry = ATTR_NEXT(entry); continue; } diff --git a/bsd/vm/vm_compressor_backing_file.c b/bsd/vm/vm_compressor_backing_file.c index 9a83330fb..295d023fa 100644 --- a/bsd/vm/vm_compressor_backing_file.c +++ b/bsd/vm/vm_compressor_backing_file.c @@ -182,8 +182,7 @@ vm_swapfile_io(vnode_t vp, uint64_t offset, uint64_t start, int npages, int flag int upl_control_flags = 0; upl_size_t upl_size = 0; - upl_create_flags = UPL_SET_INTERNAL | UPL_SET_LITE - | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK); + upl_create_flags = UPL_SET_INTERNAL | UPL_SET_LITE; #if ENCRYPTED_SWAP upl_control_flags = UPL_IOSYNC | UPL_PAGING_ENCRYPTED; @@ -201,7 +200,8 @@ vm_swapfile_io(vnode_t vp, uint64_t offset, uint64_t start, int npages, int flag &upl, NULL, &count, - &upl_create_flags); + &upl_create_flags, + VM_KERN_MEMORY_OSFMK); if (kr != KERN_SUCCESS || (upl_size != io_size)) { panic("vm_map_create_upl failed with %d\n", kr); @@ -322,17 +322,17 @@ u_int32_t vnode_trim_list (vnode_t vp, struct trim_list *tl, boolean_t route_onl * in each call to ensure that the entire range is covered. */ error = VNOP_BLOCKMAP (vp, current_offset, remaining_length, - &io_blockno, &io_bytecount, NULL, VNODE_READ, NULL); + &io_blockno, &io_bytecount, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) { goto trim_exit; } + if (io_blockno != -1) { + extents[trim_index].offset = (uint64_t) io_blockno * (u_int64_t) blocksize; + extents[trim_index].length = io_bytecount; - extents[trim_index].offset = (uint64_t) io_blockno * (u_int64_t) blocksize; - extents[trim_index].length = io_bytecount; - - trim_index++; - + trim_index++; + } if (trim_index == MAX_BATCH_TO_TRIM) { if (vp->v_mount->mnt_ioflags & MNT_IOFLAGS_CSUNMAP_SUPPORTED) { diff --git a/bsd/vm/vm_unix.c b/bsd/vm/vm_unix.c index 69aeca4ab..8048a6b80 100644 --- a/bsd/vm/vm_unix.c +++ b/bsd/vm/vm_unix.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2010 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -37,9 +37,6 @@ * is included in support of clause 2.2 (b) of the Apple Public License, * Version 2.0. */ - -#include - #include #include @@ -82,6 +79,9 @@ #include #include #include +#if NECP +#include +#endif /* NECP */ #include #include @@ -92,8 +92,6 @@ #include #include -#include - #include #include @@ -101,6 +99,9 @@ #include +#if CONFIG_MACF +#include +#endif int _shared_region_map_and_slide(struct proc*, int, unsigned int, struct shared_file_mapping_np*, uint32_t, user_addr_t, user_addr_t); int shared_region_copyin_mappings(struct proc*, user_addr_t, unsigned int, struct shared_file_mapping_np *); @@ -145,6 +146,22 @@ SYSCTL_INT(_vm, OID_AUTO, region_footprint, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_reg #endif /* DEVELOPMENT || DEBUG */ +#if CONFIG_EMBEDDED + +#if DEVELOPMENT || DEBUG +extern int panic_on_unsigned_execute; +SYSCTL_INT(_vm, OID_AUTO, panic_on_unsigned_execute, CTLFLAG_RW | CTLFLAG_LOCKED, &panic_on_unsigned_execute, 0, ""); +#endif /* DEVELOPMENT || DEBUG */ + +extern int log_executable_mem_entry; +extern int cs_executable_create_upl; +extern int cs_executable_mem_entry; +extern int cs_executable_wire; +SYSCTL_INT(_vm, OID_AUTO, log_executable_mem_entry, CTLFLAG_RD | CTLFLAG_LOCKED, &log_executable_mem_entry, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, cs_executable_create_upl, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_create_upl, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, cs_executable_mem_entry, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_mem_entry, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, cs_executable_wire, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_wire, 0, ""); +#endif /* CONFIG_EMBEDDED */ #if DEVELOPMENT || DEBUG extern int radar_20146450; @@ -156,7 +173,38 @@ SYSCTL_INT(_vm, OID_AUTO, macho_printf, CTLFLAG_RW | CTLFLAG_LOCKED, &macho_prin extern int apple_protect_pager_data_request_debug; SYSCTL_INT(_vm, OID_AUTO, apple_protect_pager_data_request_debug, CTLFLAG_RW | CTLFLAG_LOCKED, &apple_protect_pager_data_request_debug, 0, ""); - +#if __arm__ || __arm64__ +/* These are meant to support the page table accounting unit test. */ +extern unsigned int arm_hardware_page_size; +extern unsigned int arm_pt_desc_size; +extern unsigned int arm_pt_root_size; +extern unsigned int free_page_size_tt_count; +extern unsigned int free_two_page_size_tt_count; +extern unsigned int free_tt_count; +extern unsigned int inuse_user_tteroot_count; +extern unsigned int inuse_kernel_tteroot_count; +extern unsigned int inuse_user_ttepages_count; +extern unsigned int inuse_kernel_ttepages_count; +extern unsigned int inuse_user_ptepages_count; +extern unsigned int inuse_kernel_ptepages_count; +SYSCTL_UINT(_vm, OID_AUTO, native_hw_pagesize, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_hardware_page_size, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, arm_pt_desc_size, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_pt_desc_size, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, arm_pt_root_size, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_pt_root_size, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, free_1page_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_page_size_tt_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, free_2page_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_two_page_size_tt_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, free_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_tt_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, user_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_tteroot_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, kernel_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_tteroot_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, user_tte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_ttepages_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, kernel_tte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_ttepages_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, user_pte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_ptepages_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, kernel_pte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_ptepages_count, 0, ""); +#endif /* __arm__ || __arm64__ */ + +#if __arm64__ +extern int fourk_pager_data_request_debug; +SYSCTL_INT(_vm, OID_AUTO, fourk_pager_data_request_debug, CTLFLAG_RW | CTLFLAG_LOCKED, &fourk_pager_data_request_debug, 0, ""); +#endif /* __arm64__ */ #endif /* DEVELOPMENT || DEBUG */ SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_compressor, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_compressor, 0, ""); @@ -203,6 +251,12 @@ extern int allow_stack_exec, allow_data_exec; SYSCTL_INT(_vm, OID_AUTO, allow_stack_exec, CTLFLAG_RW | CTLFLAG_LOCKED, &allow_stack_exec, 0, ""); SYSCTL_INT(_vm, OID_AUTO, allow_data_exec, CTLFLAG_RW | CTLFLAG_LOCKED, &allow_data_exec, 0, ""); +#if __arm64__ +extern int fourk_binary_compatibility_unsafe; +extern int fourk_binary_compatibility_allow_wx; +SYSCTL_INT(_vm, OID_AUTO, fourk_binary_compatibility_unsafe, CTLFLAG_RW | CTLFLAG_LOCKED, &fourk_binary_compatibility_unsafe, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, fourk_binary_compatibility_allow_wx, CTLFLAG_RW | CTLFLAG_LOCKED, &fourk_binary_compatibility_allow_wx, 0, ""); +#endif /* __arm64__ */ #endif /* DEVELOPMENT || DEBUG */ static const char *prot_values[] = { @@ -242,8 +296,13 @@ int shared_region_unnest_log_count_threshold = 5; * Shared cache path enforcement. */ +#ifndef CONFIG_EMBEDDED static int scdir_enforce = 1; static char scdir_path[] = "/var/db/dyld/"; +#else +static int scdir_enforce = 0; +static char scdir_path[] = "/System/Library/Caches/com.apple.dyld/"; +#endif #ifndef SECURE_KERNEL SYSCTL_INT(_vm, OID_AUTO, enforce_shared_cache_dir, CTLFLAG_RW | CTLFLAG_LOCKED, &scdir_enforce, 0, ""); @@ -325,12 +384,12 @@ vslock( vm_map_t map; map = current_map(); - kret = vm_map_wire(map, + kret = vm_map_wire_kernel(map, vm_map_trunc_page(addr, vm_map_page_mask(map)), vm_map_round_page(addr+len, vm_map_page_mask(map)), - VM_PROT_READ | VM_PROT_WRITE | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_BSD), + VM_PROT_READ | VM_PROT_WRITE, VM_KERN_MEMORY_BSD, FALSE); switch (kret) { @@ -784,6 +843,14 @@ task_for_pid( extmod_statistics_incr_task_for_pid(p->task); sright = (void *) convert_task_to_port(p->task); + + /* Check if the task has been corpsified */ + if (is_corpsetask(p->task)) { + ipc_port_release_send(sright); + error = KERN_FAILURE; + goto tfpout; + } + tret = ipc_port_copyout_send( sright, get_task_ipcspace(current_task())); @@ -921,6 +988,7 @@ pid_suspend(struct proc *p __unused, struct pid_suspend_args *args, int *ret) } target = targetproc->task; +#ifndef CONFIG_EMBEDDED if (target != TASK_NULL) { mach_port_t tfpport; @@ -947,6 +1015,7 @@ pid_suspend(struct proc *p __unused, struct pid_suspend_args *args, int *ret) } } } +#endif task_reference(target); error = task_pidsuspend(target); @@ -1005,6 +1074,7 @@ pid_resume(struct proc *p __unused, struct pid_resume_args *args, int *ret) } target = targetproc->task; +#ifndef CONFIG_EMBEDDED if (target != TASK_NULL) { mach_port_t tfpport; @@ -1031,7 +1101,13 @@ pid_resume(struct proc *p __unused, struct pid_resume_args *args, int *ret) } } } +#endif +#if CONFIG_EMBEDDED +#if SOCKETS + resume_proc_sockets(targetproc); +#endif /* SOCKETS */ +#endif /* CONFIG_EMBEDDED */ task_reference(target); @@ -1062,6 +1138,149 @@ out: return error; } +#if CONFIG_EMBEDDED +/* + * Freeze the specified process (provided in args->pid), or find and freeze a PID. + * When a process is specified, this call is blocking, otherwise we wake up the + * freezer thread and do not block on a process being frozen. + */ +kern_return_t +pid_hibernate(struct proc *p __unused, struct pid_hibernate_args *args, int *ret) +{ + int error = 0; + proc_t targetproc = PROC_NULL; + int pid = args->pid; + +#ifndef CONFIG_FREEZE + #pragma unused(pid) +#else + +#if CONFIG_MACF + error = mac_proc_check_suspend_resume(p, MAC_PROC_CHECK_HIBERNATE); + if (error) { + error = EPERM; + goto out; + } +#endif + + /* + * If a pid has been provided, we obtain the process handle and call task_for_pid_posix_check(). + */ + + if (pid >= 0) { + targetproc = proc_find(pid); + + if (targetproc == PROC_NULL) { + error = ESRCH; + goto out; + } + + if (!task_for_pid_posix_check(targetproc)) { + error = EPERM; + goto out; + } + } + + if (pid == -2) { + vm_pageout_anonymous_pages(); + } else if (pid == -1) { + memorystatus_on_inactivity(targetproc); + } else { + error = memorystatus_freeze_process_sync(targetproc); + } + +out: + +#endif /* CONFIG_FREEZE */ + + if (targetproc != PROC_NULL) + proc_rele(targetproc); + *ret = error; + return error; +} +#endif /* CONFIG_EMBEDDED */ + +#if SOCKETS +static int +shutdown_sockets_callout(proc_t p, void *arg) +{ + struct pid_shutdown_sockets_args *args = arg; + int pid = args->pid; + int level = args->level; + struct filedesc *fdp; + struct fileproc *fp; + int i; + + proc_fdlock(p); + fdp = p->p_fd; + for (i = 0; i < fdp->fd_nfiles; i++) { + fp = fdp->fd_ofiles[i]; + if (fp == NULL || (fdp->fd_ofileflags[i] & UF_RESERVED) != 0) { + continue; + } + if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_SOCKET) { + struct socket *so = (struct socket *)fp->f_fglob->fg_data; + if (p->p_pid == pid || so->last_pid == pid || + ((so->so_flags & SOF_DELEGATED) && so->e_pid == pid)) { + /* Call networking stack with socket and level */ + (void) socket_defunct(p, so, level); + } + } +#if NECP + else if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_NETPOLICY && + p->p_pid == pid) { + necp_defunct_client(p, fp); + } +#endif /* NECP */ + } + proc_fdunlock(p); + + return (PROC_RETURNED); +} + +int +pid_shutdown_sockets(struct proc *p __unused, struct pid_shutdown_sockets_args *args, int *ret) +{ + int error = 0; + proc_t targetproc = PROC_NULL; + int pid = args->pid; + int level = args->level; + + if (level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_SVC && + level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL) { + error = EINVAL; + goto out; + } + +#if CONFIG_MACF + error = mac_proc_check_suspend_resume(p, MAC_PROC_CHECK_SHUTDOWN_SOCKETS); + if (error) { + error = EPERM; + goto out; + } +#endif + + targetproc = proc_find(pid); + if (targetproc == PROC_NULL) { + error = ESRCH; + goto out; + } + + if (!task_for_pid_posix_check(targetproc)) { + error = EPERM; + goto out; + } + + proc_iterate(PROC_ALLPROCLIST | PROC_NOWAITTRANS, shutdown_sockets_callout, args, NULL, NULL); + +out: + if (targetproc != PROC_NULL) + proc_rele(targetproc); + *ret = error; + return error; +} + +#endif /* SOCKETS */ static int sysctl_settfp_policy(__unused struct sysctl_oid *oidp, void *arg1, @@ -1739,9 +1958,8 @@ extern unsigned int vm_pageout_freed_from_cleaned; SYSCTL_UINT(_vm, OID_AUTO, pageout_freed_from_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_freed_from_cleaned, 0, ""); /* counts of pages entering the cleaned queue */ -extern unsigned int vm_pageout_enqueued_cleaned, vm_pageout_enqueued_cleaned_from_inactive_clean, vm_pageout_enqueued_cleaned_from_inactive_dirty; +extern unsigned int vm_pageout_enqueued_cleaned, vm_pageout_enqueued_cleaned_from_inactive_dirty; SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_enqueued_cleaned, 0, ""); /* sum of next two */ -SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned_from_inactive_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_enqueued_cleaned_from_inactive_clean, 0, ""); SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned_from_inactive_dirty, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_enqueued_cleaned_from_inactive_dirty, 0, ""); /* counts of pages leaving the cleaned queue */ @@ -1760,6 +1978,35 @@ extern int64_t vm_prefault_nb_pages, vm_prefault_nb_bailout; SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_pages, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_pages, ""); SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_bailout, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_bailout, ""); +#if defined (__x86_64__) +extern unsigned int vm_clump_promote_threshold; +SYSCTL_UINT(_vm, OID_AUTO, vm_clump_promote_threshold, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_clump_promote_threshold, 0, "clump size threshold for promotes"); +#if DEVELOPMENT || DEBUG +extern unsigned long vm_clump_stats[]; +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats1, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[1], "free page allocations from clump of 1 page"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats2, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[2], "free page allocations from clump of 2 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats3, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[3], "free page allocations from clump of 3 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats4, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[4], "free page allocations from clump of 4 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats5, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[5], "free page allocations from clump of 5 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats6, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[6], "free page allocations from clump of 6 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats7, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[7], "free page allocations from clump of 7 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats8, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[8], "free page allocations from clump of 8 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats9, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[9], "free page allocations from clump of 9 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats10, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[10], "free page allocations from clump of 10 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats11, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[11], "free page allocations from clump of 11 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats12, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[12], "free page allocations from clump of 12 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats13, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[13], "free page allocations from clump of 13 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats14, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[14], "free page allocations from clump of 14 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats15, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[15], "free page allocations from clump of 15 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats16, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[16], "free page allocations from clump of 16 pages"); +extern unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes; +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_alloc, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_allocs, "free page allocations"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_inserts, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_inserts, "free page insertions"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_inrange, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_inrange, "free page insertions that are part of vm_pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_promotes, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_promotes, "pages promoted to head"); +#endif /* if DEVELOPMENT || DEBUG */ +#endif /* #if defined (__x86_64__) */ + #if CONFIG_SECLUDED_MEMORY SYSCTL_UINT(_vm, OID_AUTO, num_tasks_can_use_secluded_mem, CTLFLAG_RD | CTLFLAG_LOCKED, &num_tasks_can_use_secluded_mem, 0, ""); @@ -1782,11 +2029,7 @@ SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_dirty, CTLFLAG_RD | CTLFLA SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_for_iokit, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_for_iokit, 0, ""); SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_for_iokit_success, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_for_iokit_success, 0, ""); -extern uint64_t vm_pageout_freed_from_secluded; -extern uint64_t vm_pageout_secluded_reactivated; extern uint64_t vm_pageout_secluded_burst_count; -SYSCTL_QUAD(_vm, OID_AUTO, pageout_freed_from_secluded, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_freed_from_secluded, ""); -SYSCTL_QUAD(_vm, OID_AUTO, pageout_secluded_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_secluded_reactivated, "Secluded pages reactivated"); /* sum of all reactivated AND busy and nolock (even though those actually get reDEactivated */ SYSCTL_QUAD(_vm, OID_AUTO, pageout_secluded_burst_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_secluded_burst_count, ""); #endif /* CONFIG_SECLUDED_MEMORY */ @@ -1923,3 +2166,44 @@ kas_info(struct proc *p, return 0; #endif /* !SECURE_KERNEL */ } + + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#pragma clang diagnostic ignored "-Wunused-function" + +static void asserts() { + static_assert(sizeof(vm_min_kernel_address) == sizeof(unsigned long)); + static_assert(sizeof(vm_max_kernel_address) == sizeof(unsigned long)); +} + +SYSCTL_ULONG(_vm, OID_AUTO, vm_min_kernel_address, CTLFLAG_RD, (unsigned long *) &vm_min_kernel_address, ""); +SYSCTL_ULONG(_vm, OID_AUTO, vm_max_kernel_address, CTLFLAG_RD, (unsigned long *) &vm_max_kernel_address, ""); +#pragma clang diagnostic pop + +extern uint32_t vm_page_pages; +SYSCTL_UINT(_vm, OID_AUTO, pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pages, 0, ""); + +#if (__arm__ || __arm64__) && (DEVELOPMENT || DEBUG) +extern void pmap_footprint_suspend(vm_map_t map, boolean_t suspend); +static int +sysctl_vm_footprint_suspend SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int error = 0; + int new_value; + + if (req->newptr == USER_ADDR_NULL) { + return 0; + } + error = SYSCTL_IN(req, &new_value, sizeof(int)); + if (error) { + return error; + } + pmap_footprint_suspend(current_map(), new_value); + return 0; +} +SYSCTL_PROC(_vm, OID_AUTO, footprint_suspend, + CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_LOCKED|CTLFLAG_MASKED, + 0, 0, &sysctl_vm_footprint_suspend, "I", ""); +#endif /* (__arm__ || __arm64__) && (DEVELOPMENT || DEBUG) */ diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c index b70717b92..69dad2981 100644 --- a/bsd/vm/vnode_pager.c +++ b/bsd/vm/vnode_pager.c @@ -73,14 +73,13 @@ #include #include -#include - #include #include #include #include +#include void vnode_pager_throttle() @@ -93,13 +92,10 @@ vnode_pager_throttle() throttle_lowpri_io(1); } - boolean_t vnode_pager_isSSD(vnode_t vp) { - if (vp->v_mount->mnt_kern_flag & MNTK_SSD) - return (TRUE); - return (FALSE); + return disk_conditioner_mount_is_ssd(vp->v_mount); } #if CONFIG_IOSCHED @@ -251,7 +247,7 @@ u_int32_t vnode_trim ( * in each call to ensure that the entire range is covered. */ error = VNOP_BLOCKMAP (vp, current_offset, remaining_length, - &io_blockno, &io_bytecount, NULL, VNODE_READ, NULL); + &io_blockno, &io_bytecount, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) { goto trim_exit; @@ -367,7 +363,7 @@ vnode_pageout(struct vnode *vp, else request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY; - if (ubc_create_upl(vp, f_offset, size, &upl, &pl, request_flags) != KERN_SUCCESS) { + if (ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, request_flags, VM_KERN_MEMORY_FILE) != KERN_SUCCESS) { result = PAGER_ERROR; error_ret = EINVAL; goto out; @@ -601,7 +597,7 @@ vnode_pagein( } goto out; } - ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT); + ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT, VM_KERN_MEMORY_FILE); if (upl == (upl_t)NULL) { result = PAGER_ABSENT; diff --git a/config/BSDKernel.arm.exports b/config/BSDKernel.arm.exports new file mode 100644 index 000000000..e4e0cf3fd --- /dev/null +++ b/config/BSDKernel.arm.exports @@ -0,0 +1,16 @@ +_file_vnode +_mbuf_data +_mbuf_len +_mbuf_next +_mbuf_nextpkt +_mbuf_pkthdr_header +_mbuf_pkthdr_len +_mbuf_pkthdr_rcvif +_mbuf_pkthdr_setheader +_mbuf_setlen +_mbuf_setnextpkt +_mbuf_type +_proc_ucred +_rootvnode +_suser +_ubc_setcred diff --git a/config/BSDKernel.arm64.exports b/config/BSDKernel.arm64.exports new file mode 100644 index 000000000..e4e0cf3fd --- /dev/null +++ b/config/BSDKernel.arm64.exports @@ -0,0 +1,16 @@ +_file_vnode +_mbuf_data +_mbuf_len +_mbuf_next +_mbuf_nextpkt +_mbuf_pkthdr_header +_mbuf_pkthdr_len +_mbuf_pkthdr_rcvif +_mbuf_pkthdr_setheader +_mbuf_setlen +_mbuf_setnextpkt +_mbuf_type +_proc_ucred +_rootvnode +_suser +_ubc_setcred diff --git a/config/BSDKernel.exports b/config/BSDKernel.exports index f595bb171..83659ed3c 100644 --- a/config/BSDKernel.exports +++ b/config/BSDKernel.exports @@ -574,7 +574,7 @@ _timevalfix _timevalsub _tvtoabstime _ubc_blktooff -_ubc_create_upl +_ubc_create_upl:_ubc_create_upl_external _ubc_getcred _ubc_getsize _ubc_msync @@ -720,6 +720,7 @@ _vnode_isinuse _vnode_islnk _vnode_ismount _vnode_ismountedon +_vnode_isnamedstream _vnode_isnocache _vnode_isnoreadahead _vnode_israge diff --git a/config/IOKit.arm.exports b/config/IOKit.arm.exports new file mode 100644 index 000000000..f4ee08125 --- /dev/null +++ b/config/IOKit.arm.exports @@ -0,0 +1,309 @@ +_IOPanic +_OSSynchronizeIO +_PE_arm_debug_panic_hook +__Z16IODTFindSlotNameP15IORegistryEntrym +__Z16IODTSetResolvingP15IORegistryEntryPFlmPmS1_EPFvS0_PhS4_S4_E +__Z17IODTGetCellCountsP15IORegistryEntryPmS1_ +__Z22IODTResolveAddressCellP15IORegistryEntryPmS1_S1_ +__Z23IODTFindMatchingEntriesP15IORegistryEntrymPKc +__ZN10IOWorkLoop19workLoopWithOptionsEm +__ZN10IOWorkLoop9sleepGateEPvym +__ZN10IOWorkLoop9sleepGateEPvm +__ZN11IOCatalogue11findDriversEP12OSDictionaryPl +__ZN11IOCatalogue11findDriversEP9IOServicePl +__ZN11IODataQueue11withEntriesEmm +__ZN11IODataQueue12withCapacityEm +__ZN11IODataQueue15initWithEntriesEmm +__ZN11IODataQueue16initWithCapacityEm +__ZN11IODataQueue7enqueueEPvm +__ZN11IOMemoryMap10getAddressEv +__ZN11IOMemoryMap18getPhysicalSegmentEmPm +__ZN11IOMemoryMap19setMemoryDescriptorEP18IOMemoryDescriptory +__ZN11IOMemoryMap7getSizeEv +__ZN11IOMemoryMap8redirectEP18IOMemoryDescriptormm +__ZN11IOMemoryMap8redirectEP18IOMemoryDescriptormy +__ZN12IODMACommand11OutputBig32EPS_NS_9Segment64EPvm +__ZN12IODMACommand11OutputBig64EPS_NS_9Segment64EPvm +__ZN12IODMACommand11synchronizeEm +__ZN12IODMACommand12OutputHost32EPS_NS_9Segment64EPvm +__ZN12IODMACommand12OutputHost64EPS_NS_9Segment64EPvm +__ZN12IODMACommand14OutputLittle32EPS_NS_9Segment64EPvm +__ZN12IODMACommand14OutputLittle64EPS_NS_9Segment64EPvm +__ZN12IODMACommand15genIOVMSegmentsEPFbPS_NS_9Segment64EPvmEPyS2_Pm +__ZN12IODMACommand15genIOVMSegmentsEPyPvPm +__ZN12IODMACommand16createCopyBufferE11IODirectiony +__ZN12IODMACommand17withSpecificationEPFbPS_NS_9Segment64EPvmEPKNS_14SegmentOptionsEjP8IOMapperS2_ +__ZN12IODMACommand21initWithSpecificationEPFbPS_NS_9Segment64EPvmEPKNS_14SegmentOptionsEjP8IOMapperS2_ +__ZN12IODMACommand24prepareWithSpecificationEPFbPS_NS_9Segment64EPvmEPKNS_14SegmentOptionsEjP8IOMapperyybb +__ZN12IODMACommand17withSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOptionsEymP8IOMapperS2_ +__ZN12IODMACommand21initWithSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOptionsEymP8IOMapperS2_ +__ZN12IODMACommand24prepareWithSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOptionsEymP8IOMapperyybb +__ZN12IODMACommand8transferEmyPvy +__ZN12IOUserClient12initWithTaskEP4taskPvm +__ZN12IOUserClient12initWithTaskEP4taskPvmP12OSDictionary +__ZN12IOUserClient15mapClientMemoryEmP4taskmj +__ZN12IOUserClient15sendAsyncResultEPjiPPvm +__ZN12IOUserClient17mapClientMemory64EmP4taskmy +__ZN12IOUserClient17sendAsyncResult64EPyiS0_m +__ZN12IOUserClient19clientMemoryForTypeEmPmPP18IOMemoryDescriptor +__ZN12IOUserClient19setAsyncReference64EPyP8ipc_portyy +__ZN12IOUserClient19setAsyncReference64EPyP8ipc_portyyP4task +__ZN12IOUserClient23getExternalTrapForIndexEm +__ZN12IOUserClient24getNotificationSemaphoreEmPP9semaphore +__ZN12IOUserClient24getTargetAndTrapForIndexEPP9IOServicem +__ZN12IOUserClient24registerNotificationPortEP8ipc_portmm +__ZN12IOUserClient24registerNotificationPortEP8ipc_portmy +__ZN12IOUserClient25getExternalMethodForIndexEm +__ZN12IOUserClient26getTargetAndMethodForIndexEPP9IOServicem +__ZN12IOUserClient28sendAsyncResult64WithOptionsEPyiS0_mm +__ZN12IOUserClient30getExternalAsyncMethodForIndexEm +__ZN12IOUserClient31getAsyncTargetAndMethodForIndexEPP9IOServicem +__ZN13IOCommandGate12commandSleepEPvym +__ZN13IOCommandGate12commandSleepEPvm +__ZN13IOCommandPool11commandPoolEP9IOServiceP10IOWorkLoopm +__ZN13IOCommandPool4initEP9IOServiceP10IOWorkLoopm +__ZN13IOEventSource9sleepGateEPvym +__ZN13IOEventSource9sleepGateEPvm +__ZN13_IOServiceJob8startJobEP9IOServiceim +__ZN14IODeviceMemory12withSubRangeEPS_mm +__ZN14IODeviceMemory13arrayFromListEPNS_11InitElementEm +__ZN14IODeviceMemory9withRangeEmm +__ZN14IOMemoryCursor17withSpecificationEPFvNS_15PhysicalSegmentEPvmEmmm +__ZN14IOMemoryCursor19genPhysicalSegmentsEP18IOMemoryDescriptormPvmmPm +__ZN14IOMemoryCursor21initWithSpecificationEPFvNS_15PhysicalSegmentEPvmEmmm +__ZN14IOPMrootDomain17setSleepSupportedEm +__ZN14IOPMrootDomain19sysPowerDownHandlerEPvS0_mP9IOServiceS0_j +__ZN14IOPMrootDomain20claimSystemWakeEventEP9IOServicemPKcP8OSObject +__ZN14IOPMrootDomain24receivePowerNotificationEm +__ZN14IOPMrootDomain27displayWranglerNotificationEPvS0_mP9IOServiceS0_j +__ZN15IODMAController13getControllerEP9IOServicem +__ZN15IODMAController16notifyDMACommandEP16IODMAEventSourceP12IODMACommandimy +__ZN15IODMAController20createControllerNameEm +__ZN15IODMAController21registerDMAControllerEm +__ZN16IODMAEventSource14dmaEventSourceEP8OSObjectP9IOServicePFvS1_PS_P12IODMACommandimyES8_m +__ZN16IODMAEventSource15startDMACommandEP12IODMACommand11IODirectionmm +__ZN16IODMAEventSource16notifyDMACommandEP12IODMACommandimy +__ZN16IODMAEventSource4initEP8OSObjectP9IOServicePFvS1_PS_P12IODMACommandimyES8_m +__ZN16IORangeAllocator10deallocateEmm +__ZN16IORangeAllocator12allocElementEm +__ZN16IORangeAllocator13allocateRangeEmm +__ZN16IORangeAllocator14deallocElementEm +__ZN16IORangeAllocator28setFragmentCapacityIncrementEm +__ZN16IORangeAllocator4initEmmmm +__ZN16IORangeAllocator8allocateEmPmm +__ZN16IORangeAllocator9withRangeEmmmm +__ZN17IOBigMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvm +__ZN17IOBigMemoryCursor17withSpecificationEmmm +__ZN17IOBigMemoryCursor21initWithSpecificationEmmm +__ZN17IOSharedDataQueue11withEntriesEmm +__ZN17IOSharedDataQueue12getQueueSizeEv +__ZN17IOSharedDataQueue12setQueueSizeEm +__ZN17IOSharedDataQueue12withCapacityEm +__ZN17IOSharedDataQueue16initWithCapacityEm +__ZN17IOSharedDataQueue7dequeueEPvPm +__ZN17IOSharedDataQueue7enqueueEPvm +__ZN18IOMemoryDescriptor10setMappingEP4taskjm +__ZN18IOMemoryDescriptor10withRangesEP14IOVirtualRangem11IODirectionP4taskb +__ZN18IOMemoryDescriptor10writeBytesEmPKvm +__ZN18IOMemoryDescriptor11makeMappingEPS_P4taskjmmm +__ZN18IOMemoryDescriptor11withAddressEPvm11IODirection +__ZN18IOMemoryDescriptor11withAddressEjm11IODirectionP4task +__ZN18IOMemoryDescriptor11withOptionsEPvmmP4taskmP8IOMapper +__ZN18IOMemoryDescriptor12setPurgeableEmPm +__ZN18IOMemoryDescriptor12withSubRangeEPS_mm11IODirection +__ZN18IOMemoryDescriptor13getPageCountsEPmS0_ +__ZN18IOMemoryDescriptor14initWithRangesEP14IOVirtualRangem11IODirectionP4taskb +__ZN18IOMemoryDescriptor15initWithAddressEPvm11IODirection +__ZN18IOMemoryDescriptor15initWithAddressEjm11IODirectionP4task +__ZN18IOMemoryDescriptor15initWithOptionsEPvmmP4taskmP8IOMapper +__ZN18IOMemoryDescriptor16getSourceSegmentEmPm +__ZN18IOMemoryDescriptor16performOperationEmmm +__ZN18IOMemoryDescriptor16withAddressRangeEyymP4task +__ZN18IOMemoryDescriptor17getVirtualSegmentEmPm +__ZN18IOMemoryDescriptor17withAddressRangesEP14IOAddressRangemmP4task +__ZN18IOMemoryDescriptor18getPhysicalSegmentEmPm +__ZN18IOMemoryDescriptor18getPhysicalSegmentEmPmm +__ZN18IOMemoryDescriptor18withPhysicalRangesEP15IOPhysicalRangem11IODirectionb +__ZN18IOMemoryDescriptor19createMappingInTaskEP4taskymyy +__ZN18IOMemoryDescriptor19withPhysicalAddressEmm11IODirection +__ZN18IOMemoryDescriptor20getPhysicalSegment64EmPm +__ZN18IOMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb +__ZN18IOMemoryDescriptor23initWithPhysicalAddressEmm11IODirection +__ZN18IOMemoryDescriptor3mapEP4taskjmmm +__ZN18IOMemoryDescriptor3mapEm +__ZN18IOMemoryDescriptor5doMapEP7_vm_mapPjmmm +__ZN18IOMemoryDescriptor6setTagEm +__ZN18IOMemoryDescriptor7doUnmapEP7_vm_mapjm +__ZN18IOMemoryDescriptor9readBytesEmPvm +__ZN18IORegistryIterator11iterateOverEP15IORegistryEntryPK15IORegistryPlanem +__ZN18IORegistryIterator11iterateOverEPK15IORegistryPlanem +__ZN18IOTimerEventSource10setTimeoutEjyy +__ZN18IOTimerEventSource10setTimeoutEy +__ZN18IOTimerEventSource10setTimeoutE13mach_timespec +__ZN18IOTimerEventSource10setTimeoutEmm +__ZN18IOTimerEventSource10wakeAtTimeEjyy +__ZN18IOTimerEventSource10wakeAtTimeEy +__ZN18IOTimerEventSource10wakeAtTimeE13mach_timespec +__ZN18IOTimerEventSource10wakeAtTimeEmm +__ZN18IOTimerEventSource12setTimeoutMSEm +__ZN18IOTimerEventSource12setTimeoutUSEm +__ZN18IOTimerEventSource12wakeAtTimeMSEm +__ZN18IOTimerEventSource12wakeAtTimeUSEm +__ZN18IOTimerEventSource15setTimeoutTicksEm +__ZN18IOTimerEventSource15wakeAtTimeTicksEm +__ZN20IOLittleMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvm +__ZN20IOLittleMemoryCursor17withSpecificationEmmm +__ZN20IOLittleMemoryCursor21initWithSpecificationEmmm +__ZN20RootDomainUserClient15setPreventativeEmm +__ZN20RootDomainUserClient26getTargetAndMethodForIndexEPP9IOServicem +__ZN21IOInterruptController10initVectorElP17IOInterruptVector +__ZN21IOInterruptController11causeVectorElP17IOInterruptVector +__ZN21IOInterruptController12enableVectorElP17IOInterruptVector +__ZN21IOInterruptController13getVectorTypeElP17IOInterruptVector +__ZN21IOInterruptController17disableVectorHardElP17IOInterruptVector +__ZN21IOInterruptController17vectorCanBeSharedElP17IOInterruptVector +__ZN21IOInterruptController28timeStampInterruptHandlerEndElP17IOInterruptVector +__ZN21IOInterruptController30timeStampInterruptHandlerStartElP17IOInterruptVector +__ZN21IONaturalMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvm +__ZN21IONaturalMemoryCursor17withSpecificationEmmm +__ZN21IONaturalMemoryCursor21initWithSpecificationEmmm +__ZN21IOSubMemoryDescriptor11makeMappingEP18IOMemoryDescriptorP4taskjmmm +__ZN21IOSubMemoryDescriptor12initSubRangeEP18IOMemoryDescriptormm11IODirection +__ZN21IOSubMemoryDescriptor12setPurgeableEmPm +__ZN21IOSubMemoryDescriptor12withSubRangeEP18IOMemoryDescriptormmm +__ZN21IOSubMemoryDescriptor18getPhysicalSegmentEmPmm +__ZN21IOSubMemoryDescriptor7prepareE11IODirection +__ZN21IOSubMemoryDescriptor8completeE11IODirection +__ZN23IOMultiMemoryDescriptor15withDescriptorsEPP18IOMemoryDescriptorm11IODirectionb +__ZN23IOMultiMemoryDescriptor18getPhysicalSegmentEmPmm +__ZN23IOMultiMemoryDescriptor19initWithDescriptorsEPP18IOMemoryDescriptorm11IODirectionb +__ZN23IOMultiMemoryDescriptor7prepareE11IODirection +__ZN23IOMultiMemoryDescriptor8completeE11IODirection +__ZN24IOBufferMemoryDescriptor11appendBytesEPKvj +__ZN24IOBufferMemoryDescriptor11withOptionsEmjj +__ZN24IOBufferMemoryDescriptor12setDirectionE11IODirection +__ZN24IOBufferMemoryDescriptor12withCapacityEj11IODirectionb +__ZN24IOBufferMemoryDescriptor13initWithBytesEPKvj11IODirectionb +__ZN24IOBufferMemoryDescriptor14getBytesNoCopyEjj +__ZN24IOBufferMemoryDescriptor15initWithOptionsEmjj +__ZN24IOBufferMemoryDescriptor15initWithOptionsEmjjP4task +__ZN24IOBufferMemoryDescriptor17getVirtualSegmentEmPm +__ZN24IOBufferMemoryDescriptor17inTaskWithOptionsEP4taskmjj +__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskmyyy +__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskmyy +__ZN24IOBufferMemoryDescriptor9setLengthEj +__ZN24IOBufferMemoryDescriptor9withBytesEPKvj11IODirectionb +__ZN25IOGeneralMemoryDescriptor11setPositionEm +__ZN25IOGeneralMemoryDescriptor11wireVirtualE11IODirection +__ZN25IOGeneralMemoryDescriptor12setPurgeableEmPm +__ZN25IOGeneralMemoryDescriptor13mapIntoKernelEj +__ZN25IOGeneralMemoryDescriptor14initWithRangesEP14IOVirtualRangem11IODirectionP4taskb +__ZN25IOGeneralMemoryDescriptor15initWithAddressEPvm11IODirection +__ZN25IOGeneralMemoryDescriptor15initWithAddressEjm11IODirectionP4task +__ZN25IOGeneralMemoryDescriptor15initWithOptionsEPvmmP4taskmP8IOMapper +__ZN25IOGeneralMemoryDescriptor15unmapFromKernelEv +__ZN25IOGeneralMemoryDescriptor16getSourceSegmentEmPm +__ZN25IOGeneralMemoryDescriptor17getVirtualSegmentEmPm +__ZN25IOGeneralMemoryDescriptor18getPhysicalSegmentEmPm +__ZN25IOGeneralMemoryDescriptor18getPhysicalSegmentEmPmm +__ZN25IOGeneralMemoryDescriptor20getPhysicalSegment64EmPm +__ZN25IOGeneralMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb +__ZN25IOGeneralMemoryDescriptor23initWithPhysicalAddressEmm11IODirection +__ZN25IOGeneralMemoryDescriptor5doMapEP7_vm_mapPjmmm +__ZN25IOGeneralMemoryDescriptor7doUnmapEP7_vm_mapjm +__ZN25IOGeneralMemoryDescriptor7prepareE11IODirection +__ZN25IOGeneralMemoryDescriptor8completeE11IODirection +__ZN29IOInterleavedMemoryDescriptor12withCapacityEm11IODirection +__ZN29IOInterleavedMemoryDescriptor16initWithCapacityEm11IODirection +__ZN29IOInterleavedMemoryDescriptor18getPhysicalSegmentEmPmm +__ZN29IOInterleavedMemoryDescriptor19setMemoryDescriptorEP18IOMemoryDescriptormm +__ZN29IOInterleavedMemoryDescriptor22clearMemoryDescriptorsE11IODirection +__ZN29IOInterleavedMemoryDescriptor7prepareE11IODirection +__ZN29IOInterleavedMemoryDescriptor8completeE11IODirection +__ZN8IOPMprot10gMetaClassE +__ZN8IOPMprot10superClassE +__ZN8IOPMprot9MetaClassC1Ev +__ZN8IOPMprot9MetaClassC2Ev +__ZN8IOPMprot9metaClassE +__ZN8IOPMprotC1EPK11OSMetaClass +__ZN8IOPMprotC1Ev +__ZN8IOPMprotC2EPK11OSMetaClass +__ZN8IOPMprotC2Ev +__ZN8IOPMprotD0Ev +__ZN8IOPMprotD2Ev +__ZN9IOService10adjustBusyEl +__ZN9IOService10handleOpenEPS_mPv +__ZN9IOService10systemWakeEv +__ZN9IOService10youAreRootEv +__ZN9IOService11_adjustBusyEl +__ZN9IOService11handleCloseEPS_m +__ZN9IOService11tellClientsEi +__ZN9IOService12clampPowerOnEm +__ZN9IOService12didTerminateEPS_mPb +__ZN9IOService12requestProbeEm +__ZN9IOService12updateReportEP19IOReportChannelListjPvS2_ +__ZN9IOService12waitForStateEmmP13mach_timespec +__ZN9IOService13getPMworkloopEv +__ZN9IOService13messageClientEmP8OSObjectPvj +__ZN9IOService13newUserClientEP4taskPvmP12OSDictionaryPP12IOUserClient +__ZN9IOService13newUserClientEP4taskPvmPP12IOUserClient +__ZN9IOService13startMatchingEm +__ZN9IOService13waitMatchIdleEm +__ZN9IOService13willTerminateEPS_m +__ZN9IOService14doServiceMatchEm +__ZN9IOService14messageClientsEmPvj +__ZN9IOService14newTemperatureElPS_ +__ZN9IOService14setPowerParentEP17IOPowerConnectionbm +__ZN9IOService15addNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_l +__ZN9IOService15configureReportEP19IOReportChannelListjPvS2_ +__ZN9IOService15nextIdleTimeoutEyyj +__ZN9IOService15registerServiceEm +__ZN9IOService15tellChangeDown1Em +__ZN9IOService15tellChangeDown2Em +__ZN9IOService15terminateClientEPS_m +__ZN9IOService15terminatePhase1Em +__ZN9IOService15terminateWorkerEm +__ZN9IOService16ack_timer_tickedEv +__ZN9IOService16command_receivedEPvS0_S0_S0_ +__ZN9IOService16didYouWakeSystemEv +__ZN9IOService16registerInterestEPK8OSSymbolPFiPvS3_mPS_S3_jES3_S3_ +__ZN9IOService16requestTerminateEPS_m +__ZN9IOService16setCPUSnoopDelayEm +__ZN9IOService18doServiceTerminateEm +__ZN9IOService18matchPropertyTableEP12OSDictionaryPl +__ZN9IOService18requireMaxBusStallEm +__ZN9IOService18settleTimerExpiredEv +__ZN9IOService18systemWillShutdownEm +__ZN9IOService19deliverNotificationEPK8OSSymbolmm +__ZN9IOService22PM_Clamp_Timer_ExpiredEv +__ZN9IOService22powerDomainDidChangeToEmP17IOPowerConnection +__ZN9IOService23acknowledgeNotificationEPvm +__ZN9IOService23addMatchingNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_P10IONotifierES5_S5_l +__ZN9IOService23powerDomainWillChangeToEmP17IOPowerConnection +__ZN9IOService23scheduleTerminatePhase2Em +__ZN9IOService23tellClientsWithResponseEi +__ZN9IOService24PM_idle_timer_expirationEv +__ZN9IOService24mapDeviceMemoryWithIndexEjm +__ZN9IOService26temperatureCriticalForZoneEPS_ +__ZN9IOService27serializedAllowPowerChange2Em +__ZN9IOService28serializedCancelPowerChange2Em +__ZN9IOService4openEPS_mPv +__ZN9IOService5closeEPS_m +__ZN9IOService5probeEPS_Pl +__ZN9IOService6PMfreeEv +__ZN9IOService7messageEmPS_Pv +__ZN9IOService8finalizeEm +__ZN9IOService9terminateEm +__ZNK11IOCatalogue13serializeDataEmP11OSSerialize +__ZNK15IORegistryEntry11getPropertyEPK8OSStringPK15IORegistryPlanem +__ZNK15IORegistryEntry11getPropertyEPK8OSSymbolPK15IORegistryPlanem +__ZNK15IORegistryEntry11getPropertyEPKcPK15IORegistryPlanem +__ZNK15IORegistryEntry12copyPropertyEPK8OSStringPK15IORegistryPlanem +__ZNK15IORegistryEntry12copyPropertyEPK8OSSymbolPK15IORegistryPlanem +__ZNK15IORegistryEntry12copyPropertyEPKcPK15IORegistryPlanem +__ZNK18IOMemoryDescriptor19dmaCommandOperationEmPvj +__ZNK25IOGeneralMemoryDescriptor19dmaCommandOperationEmPvj +__ZNK8IOPMprot12getMetaClassEv +__ZNK8IOPMprot9MetaClass5allocEv +__ZTV8IOPMprot +__ZTVN8IOPMprot9MetaClassE diff --git a/config/IOKit.arm64.exports b/config/IOKit.arm64.exports new file mode 100644 index 000000000..ed271b62c --- /dev/null +++ b/config/IOKit.arm64.exports @@ -0,0 +1,230 @@ +_OSSynchronizeIO +_PE_arm_debug_panic_hook +__Z16IODTFindSlotNameP15IORegistryEntryj +__Z16IODTSetResolvingP15IORegistryEntryPFxjPjS1_EPFvS0_PhS4_S4_E +__Z17IODTGetCellCountsP15IORegistryEntryPjS1_ +__Z22IODTResolveAddressCellP15IORegistryEntryPjPyS2_ +__Z23IODTFindMatchingEntriesP15IORegistryEntryjPKc +__ZN10IOWorkLoop19workLoopWithOptionsEj +__ZN10IOWorkLoop9sleepGateEPvj +__ZN10IOWorkLoop9sleepGateEPvyj +__ZN11IOCatalogue11findDriversEP12OSDictionaryPi +__ZN11IOCatalogue11findDriversEP9IOServicePi +__ZN11IODataQueue11withEntriesEjj +__ZN11IODataQueue12withCapacityEj +__ZN11IODataQueue15initWithEntriesEjj +__ZN11IODataQueue16initWithCapacityEj +__ZN11IODataQueue7enqueueEPvj +__ZN11IOMemoryMap18getPhysicalSegmentEyPyj +__ZN11IOMemoryMap19setMemoryDescriptorEP18IOMemoryDescriptory +__ZN11IOMemoryMap8redirectEP18IOMemoryDescriptorjy +__ZN12IODMACommand11OutputBig32EPS_NS_9Segment64EPvj +__ZN12IODMACommand11OutputBig64EPS_NS_9Segment64EPvj +__ZN12IODMACommand11synchronizeEj +__ZN12IODMACommand12OutputHost32EPS_NS_9Segment64EPvj +__ZN12IODMACommand12OutputHost64EPS_NS_9Segment64EPvj +__ZN12IODMACommand14OutputLittle32EPS_NS_9Segment64EPvj +__ZN12IODMACommand14OutputLittle64EPS_NS_9Segment64EPvj +__ZN12IODMACommand15genIOVMSegmentsEPFbPS_NS_9Segment64EPvjEPyS2_Pj +__ZN12IODMACommand15genIOVMSegmentsEPyPvPj +__ZN12IODMACommand16createCopyBufferEjy +__ZN12IODMACommand17withSpecificationEPFbPS_NS_9Segment64EPvjEPKNS_14SegmentOptionsEjP8IOMapperS2_ +__ZN12IODMACommand21initWithSpecificationEPFbPS_NS_9Segment64EPvjEPKNS_14SegmentOptionsEjP8IOMapperS2_ +__ZN12IODMACommand24prepareWithSpecificationEPFbPS_NS_9Segment64EPvjEPKNS_14SegmentOptionsEjP8IOMapperyybb +__ZN12IODMACommand17withSpecificationEPFbPS_NS_9Segment64EPvjEhyNS_14MappingOptionsEyjP8IOMapperS2_ +__ZN12IODMACommand21initWithSpecificationEPFbPS_NS_9Segment64EPvjEhyNS_14MappingOptionsEyjP8IOMapperS2_ +__ZN12IODMACommand24prepareWithSpecificationEPFbPS_NS_9Segment64EPvjEhyNS_14MappingOptionsEyjP8IOMapperyybb +__ZN12IODMACommand8transferEjyPvy +__ZN12IOUserClient12initWithTaskEP4taskPvj +__ZN12IOUserClient12initWithTaskEP4taskPvjP12OSDictionary +__ZN12IOUserClient15sendAsyncResultEPjiPPvj +__ZN12IOUserClient17mapClientMemory64EjP4taskjy +__ZN12IOUserClient17sendAsyncResult64EPyiS0_j +__ZN12IOUserClient19clientMemoryForTypeEjPjPP18IOMemoryDescriptor +__ZN12IOUserClient19setAsyncReference64EPyP8ipc_portyy +__ZN12IOUserClient19setAsyncReference64EPyP8ipc_portyyP4task +__ZN12IOUserClient23getExternalTrapForIndexEj +__ZN12IOUserClient24getNotificationSemaphoreEjPP9semaphore +__ZN12IOUserClient24getTargetAndTrapForIndexEPP9IOServicej +__ZN12IOUserClient24registerNotificationPortEP8ipc_portjj +__ZN12IOUserClient24registerNotificationPortEP8ipc_portjy +__ZN12IOUserClient25getExternalMethodForIndexEj +__ZN12IOUserClient26getTargetAndMethodForIndexEPP9IOServicej +__ZN12IOUserClient28sendAsyncResult64WithOptionsEPyiS0_jj +__ZN12IOUserClient30getExternalAsyncMethodForIndexEj +__ZN12IOUserClient31getAsyncTargetAndMethodForIndexEPP9IOServicej +__ZN13IOCommandGate12commandSleepEPvj +__ZN13IOCommandGate12commandSleepEPvyj +__ZN13IOCommandPool11commandPoolEP9IOServiceP10IOWorkLoopj +__ZN13IOCommandPool4initEP9IOServiceP10IOWorkLoopj +__ZN13IOEventSource9sleepGateEPvj +__ZN13IOEventSource9sleepGateEPvyj +__ZN13_IOServiceJob8startJobEP9IOServiceij +__ZN14IODeviceMemory12withSubRangeEPS_yy +__ZN14IODeviceMemory13arrayFromListEPNS_11InitElementEj +__ZN14IODeviceMemory9withRangeEyy +__ZN14IOMemoryCursor17withSpecificationEPFvNS_15PhysicalSegmentEPvjEyyy +__ZN14IOMemoryCursor19genPhysicalSegmentsEP18IOMemoryDescriptoryPvjjPy +__ZN14IOMemoryCursor21initWithSpecificationEPFvNS_15PhysicalSegmentEPvjEyyy +__ZN14IOPMrootDomain17setSleepSupportedEj +__ZN14IOPMrootDomain19sysPowerDownHandlerEPvS0_jP9IOServiceS0_m +__ZN14IOPMrootDomain20claimSystemWakeEventEP9IOServicejPKcP8OSObject +__ZN14IOPMrootDomain24receivePowerNotificationEj +__ZN14IOPMrootDomain27displayWranglerNotificationEPvS0_jP9IOServiceS0_m +__ZN15IODMAController13getControllerEP9IOServicej +__ZN15IODMAController16notifyDMACommandEP16IODMAEventSourceP12IODMACommandiyy +__ZN15IODMAController20createControllerNameEj +__ZN15IODMAController21registerDMAControllerEj +__ZN16IODMAEventSource14dmaEventSourceEP8OSObjectP9IOServicePFvS1_PS_P12IODMACommandiyyES8_j +__ZN16IODMAEventSource15startDMACommandEP12IODMACommandjyy +__ZN16IODMAEventSource16notifyDMACommandEP12IODMACommandiyy +__ZN16IODMAEventSource4initEP8OSObjectP9IOServicePFvS1_PS_P12IODMACommandiyyES8_j +__ZN16IORangeAllocator10deallocateEyy +__ZN16IORangeAllocator12allocElementEj +__ZN16IORangeAllocator13allocateRangeEyy +__ZN16IORangeAllocator14deallocElementEj +__ZN16IORangeAllocator28setFragmentCapacityIncrementEj +__ZN16IORangeAllocator4initEyyjj +__ZN16IORangeAllocator8allocateEyPyy +__ZN16IORangeAllocator9withRangeEyyjj +__ZN17IOBigMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvj +__ZN17IOBigMemoryCursor17withSpecificationEyyy +__ZN17IOBigMemoryCursor21initWithSpecificationEyyy +__ZN17IOSharedDataQueue11withEntriesEjj +__ZN17IOSharedDataQueue12getQueueSizeEv +__ZN17IOSharedDataQueue12setQueueSizeEj +__ZN17IOSharedDataQueue12withCapacityEj +__ZN17IOSharedDataQueue16initWithCapacityEj +__ZN17IOSharedDataQueue7dequeueEPvPj +__ZN17IOSharedDataQueue7enqueueEPvj +__ZN18IOMemoryDescriptor10setMappingEP4taskyj +__ZN18IOMemoryDescriptor10writeBytesEyPKvy +__ZN18IOMemoryDescriptor11makeMappingEPS_P4taskyjyy +__ZN18IOMemoryDescriptor11withAddressEPvyj +__ZN18IOMemoryDescriptor11withOptionsEPvjjP4taskjP8IOMapper +__ZN18IOMemoryDescriptor12setPurgeableEjPj +__ZN18IOMemoryDescriptor13getPageCountsEPyS0_ +__ZN18IOMemoryDescriptor15initWithOptionsEPvjjP4taskjP8IOMapper +__ZN18IOMemoryDescriptor16performOperationEjyy +__ZN18IOMemoryDescriptor16withAddressRangeEyyjP4task +__ZN18IOMemoryDescriptor17withAddressRangesEP14IOVirtualRangejjP4task +__ZN18IOMemoryDescriptor19createMappingInTaskEP4taskyjyy +__ZN18IOMemoryDescriptor19withPhysicalAddressEyyj +__ZN18IOMemoryDescriptor3mapEj +__ZN18IOMemoryDescriptor5doMapEP7_vm_mapPyjyy +__ZN18IOMemoryDescriptor6setTagEj +__ZN18IOMemoryDescriptor7doUnmapEP7_vm_mapyy +__ZN18IOMemoryDescriptor9readBytesEyPvy +__ZN18IORegistryIterator11iterateOverEP15IORegistryEntryPK15IORegistryPlanej +__ZN18IORegistryIterator11iterateOverEPK15IORegistryPlanej +__ZN18IOTimerEventSource10setTimeoutEjj +__ZN18IOTimerEventSource10setTimeoutEjyy +__ZN18IOTimerEventSource10setTimeoutEy +__ZN18IOTimerEventSource10wakeAtTimeEjj +__ZN18IOTimerEventSource10wakeAtTimeEjyy +__ZN18IOTimerEventSource10wakeAtTimeEy +__ZN18IOTimerEventSource12setTimeoutMSEj +__ZN18IOTimerEventSource12setTimeoutUSEj +__ZN18IOTimerEventSource12wakeAtTimeMSEj +__ZN18IOTimerEventSource12wakeAtTimeUSEj +__ZN18IOTimerEventSource15setTimeoutTicksEj +__ZN18IOTimerEventSource15wakeAtTimeTicksEj +__ZN20IOLittleMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvj +__ZN20IOLittleMemoryCursor17withSpecificationEyyy +__ZN20IOLittleMemoryCursor21initWithSpecificationEyyy +__ZN20RootDomainUserClient15setPreventativeEjj +__ZN20RootDomainUserClient26getTargetAndMethodForIndexEPP9IOServicej +__ZN21IOInterruptController10initVectorEiP17IOInterruptVector +__ZN21IOInterruptController11causeVectorEiP17IOInterruptVector +__ZN21IOInterruptController12enableVectorEiP17IOInterruptVector +__ZN21IOInterruptController13getVectorTypeEiP17IOInterruptVector +__ZN21IOInterruptController17disableVectorHardEiP17IOInterruptVector +__ZN21IOInterruptController17vectorCanBeSharedEiP17IOInterruptVector +__ZN21IOInterruptController28timeStampInterruptHandlerEndEiP17IOInterruptVector +__ZN21IOInterruptController30timeStampInterruptHandlerStartEiP17IOInterruptVector +__ZN21IONaturalMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvj +__ZN21IONaturalMemoryCursor17withSpecificationEyyy +__ZN21IONaturalMemoryCursor21initWithSpecificationEyyy +__ZN21IOSubMemoryDescriptor11makeMappingEP18IOMemoryDescriptorP4taskyjyy +__ZN21IOSubMemoryDescriptor12initSubRangeEP18IOMemoryDescriptoryyj +__ZN21IOSubMemoryDescriptor12setPurgeableEjPj +__ZN21IOSubMemoryDescriptor12withSubRangeEP18IOMemoryDescriptoryyj +__ZN21IOSubMemoryDescriptor18getPhysicalSegmentEyPyj +__ZN21IOSubMemoryDescriptor7prepareEj +__ZN21IOSubMemoryDescriptor8completeEj +__ZN23IOMultiMemoryDescriptor15withDescriptorsEPP18IOMemoryDescriptorjjb +__ZN23IOMultiMemoryDescriptor19initWithDescriptorsEPP18IOMemoryDescriptorjjb +__ZN23IOMultiMemoryDescriptor7prepareEj +__ZN23IOMultiMemoryDescriptor8completeEj +__ZN24IOBufferMemoryDescriptor11appendBytesEPKvm +__ZN24IOBufferMemoryDescriptor11withOptionsEjmm +__ZN24IOBufferMemoryDescriptor12setDirectionEj +__ZN24IOBufferMemoryDescriptor12withCapacityEmjb +__ZN24IOBufferMemoryDescriptor14getBytesNoCopyEmm +__ZN24IOBufferMemoryDescriptor17inTaskWithOptionsEP4taskjmm +__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskjyyy +__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskjyy +__ZN24IOBufferMemoryDescriptor9setLengthEm +__ZN24IOBufferMemoryDescriptor9withBytesEPKvmjb +__ZN25IOGeneralMemoryDescriptor11wireVirtualEj +__ZN25IOGeneralMemoryDescriptor12setPurgeableEjPj +__ZN25IOGeneralMemoryDescriptor15initWithOptionsEPvjjP4taskjP8IOMapper +__ZN25IOGeneralMemoryDescriptor18getPhysicalSegmentEyPyj +__ZN25IOGeneralMemoryDescriptor5doMapEP7_vm_mapPyjyy +__ZN25IOGeneralMemoryDescriptor7doUnmapEP7_vm_mapyy +__ZN25IOGeneralMemoryDescriptor7prepareEj +__ZN25IOGeneralMemoryDescriptor8completeEj +__ZN29IOInterleavedMemoryDescriptor12withCapacityEyj +__ZN29IOInterleavedMemoryDescriptor16initWithCapacityEyj +__ZN29IOInterleavedMemoryDescriptor19setMemoryDescriptorEP18IOMemoryDescriptoryy +__ZN29IOInterleavedMemoryDescriptor22clearMemoryDescriptorsEj +__ZN29IOInterleavedMemoryDescriptor7prepareEj +__ZN29IOInterleavedMemoryDescriptor8completeEj +__ZN9IOService10adjustBusyEi +__ZN9IOService10handleOpenEPS_jPv +__ZN9IOService11_adjustBusyEi +__ZN9IOService11handleCloseEPS_j +__ZN9IOService12didTerminateEPS_jPb +__ZN9IOService12requestProbeEj +__ZN9IOService12updateReportEP19IOReportChannelListjPvS2_ +__ZN9IOService13messageClientEjP8OSObjectPvm +__ZN9IOService13newUserClientEP4taskPvjP12OSDictionaryPP12IOUserClient +__ZN9IOService13newUserClientEP4taskPvjPP12IOUserClient +__ZN9IOService13startMatchingEj +__ZN9IOService13waitMatchIdleEj +__ZN9IOService13willTerminateEPS_j +__ZN9IOService14doServiceMatchEj +__ZN9IOService14messageClientsEjPvm +__ZN9IOService15addNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_i +__ZN9IOService15configureReportEP19IOReportChannelListjPvS2_ +__ZN9IOService15nextIdleTimeoutEyyj +__ZN9IOService15registerServiceEj +__ZN9IOService15terminateClientEPS_j +__ZN9IOService15terminatePhase1Ej +__ZN9IOService15terminateWorkerEj +__ZN9IOService16registerInterestEPK8OSSymbolPFiPvS3_jPS_S3_mES3_S3_ +__ZN9IOService16requestTerminateEPS_j +__ZN9IOService16setCPUSnoopDelayEj +__ZN9IOService18doServiceTerminateEj +__ZN9IOService18matchPropertyTableEP12OSDictionaryPi +__ZN9IOService18requireMaxBusStallEj +__ZN9IOService18systemWillShutdownEj +__ZN9IOService19deliverNotificationEPK8OSSymboljj +__ZN9IOService23acknowledgeNotificationEPvj +__ZN9IOService23addMatchingNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_P10IONotifierES5_S5_i +__ZN9IOService23scheduleTerminatePhase2Ej +__ZN9IOService24mapDeviceMemoryWithIndexEjj +__ZN9IOService4openEPS_jPv +__ZN9IOService5closeEPS_j +__ZN9IOService5probeEPS_Pi +__ZN9IOService7messageEjPS_Pv +__ZN9IOService8finalizeEj +__ZN9IOService9terminateEj +__ZNK15IORegistryEntry11getPropertyEPK8OSStringPK15IORegistryPlanej +__ZNK15IORegistryEntry11getPropertyEPK8OSSymbolPK15IORegistryPlanej +__ZNK15IORegistryEntry11getPropertyEPKcPK15IORegistryPlanej +__ZNK15IORegistryEntry12copyPropertyEPK8OSStringPK15IORegistryPlanej +__ZNK15IORegistryEntry12copyPropertyEPK8OSSymbolPK15IORegistryPlanej +__ZNK15IORegistryEntry12copyPropertyEPKcPK15IORegistryPlanej +__ZNK18IOMemoryDescriptor19dmaCommandOperationEjPvj +__ZNK25IOGeneralMemoryDescriptor19dmaCommandOperationEjPvj diff --git a/config/IOKit.exports b/config/IOKit.exports index ec0b12790..be6ca8013 100644 --- a/config/IOKit.exports +++ b/config/IOKit.exports @@ -104,7 +104,6 @@ __Z17IODeviceTreeAllocPv __Z17IOServiceOrderingPK15OSMetaClassBaseS1_Pv __Z18IODTCompareNubNamePK15IORegistryEntryP8OSStringPS3_ __Z19printDictionaryKeysP12OSDictionaryPc -__Z20IODTMakeNVDescriptorP15IORegistryEntryP17IONVRAMDescriptor __Z20IODTMatchNubWithKeysP15IORegistryEntryPKc __Z21IODTResolveAddressingP15IORegistryEntryPKcP14IODeviceMemory __Z27IODTInterruptControllerNameP15IORegistryEntry @@ -694,6 +693,7 @@ __ZN18IOMemoryDescriptor18getPhysicalAddressEv __ZN18IOMemoryDescriptor30withPersistentMemoryDescriptorEPS_ __ZN18IOMemoryDescriptor4freeEv __ZN18IOMemoryDescriptor6getTagEv +__ZN18IOMemoryDescriptor8getFlagsEv __ZN18IOMemoryDescriptor8redirectEP4taskb __ZN18IOMemoryDescriptor9MetaClassC1Ev __ZN18IOMemoryDescriptor9MetaClassC2Ev @@ -725,11 +725,14 @@ __ZN18IORegistryIteratorD2Ev __ZN18IOTimerEventSource10gMetaClassE __ZN18IOTimerEventSource10superClassE __ZN18IOTimerEventSource11setWorkLoopEP10IOWorkLoop +__ZN18IOTimerEventSource12checkForWorkEv __ZN18IOTimerEventSource13cancelTimeoutEv __ZN18IOTimerEventSource14setTimeoutFuncEv __ZN18IOTimerEventSource16timerEventSourceEP8OSObjectPFvS1_PS_E +__ZN18IOTimerEventSource16timerEventSourceEjP8OSObjectPFvS1_PS_E __ZN18IOTimerEventSource4freeEv __ZN18IOTimerEventSource4initEP8OSObjectPFvS1_PS_E +__ZN18IOTimerEventSource4initEjP8OSObjectPFvS1_PS_E __ZN18IOTimerEventSource6enableEv __ZN18IOTimerEventSource7disableEv __ZN18IOTimerEventSource7timeoutEPv @@ -810,6 +813,7 @@ __ZN21IOInterruptController16getInterruptTypeEP9IOServiceiPi __ZN21IOInterruptController17registerInterruptEP9IOServiceiPvPFvS2_S2_S2_iES2_ __ZN21IOInterruptController19unregisterInterruptEP9IOServicei __ZN21IOInterruptController26getInterruptHandlerAddressEv +__ZN21IOInterruptController26timeStampSpuriousInterruptEv __ZN21IOInterruptController9MetaClassC1Ev __ZN21IOInterruptController9MetaClassC2Ev __ZN21IOInterruptController9metaClassE @@ -1024,7 +1028,6 @@ __ZN9IOService13addPowerChildEPS_ __ZN9IOService13askChangeDownEm __ZN9IOService13checkResourceEP8OSObject __ZN9IOService13getPowerStateEv -__ZN9IOService13invokeNotiferEP18_IOServiceNotifier __ZN9IOService13matchLocationEPS_ __ZN9IOService13setPowerStateEmPS_ __ZN9IOService14activityTickleEmm @@ -1439,7 +1442,6 @@ _gIODTModelKey _gIODTNWInterruptMappingKey _gIODTNameKey _gIODTPHandleKey -_gIODTPersistKey _gIODTPlane _gIODTRangeKey _gIODTSizeCellKey @@ -1485,7 +1487,7 @@ _gIOServiceKey _gIOServicePlane _gIOTerminatedNotification _gIOUserClientClassKey -_gOFVariables +_gIOWillTerminateNotification _gPlatformInterruptControllerName _registerPrioritySleepWakeInterest _registerSleepWakeInterest diff --git a/config/IOKit.x86_64.exports b/config/IOKit.x86_64.exports index 3aadfffa4..d050685e3 100644 --- a/config/IOKit.x86_64.exports +++ b/config/IOKit.x86_64.exports @@ -257,8 +257,10 @@ __ZN18IOMemoryDescriptor9readBytesEyPvy __ZN18IORegistryIterator11iterateOverEP15IORegistryEntryPK15IORegistryPlanej __ZN18IORegistryIterator11iterateOverEPK15IORegistryPlanej __ZN18IOTimerEventSource10setTimeoutEjj +__ZN18IOTimerEventSource10setTimeoutEjyy __ZN18IOTimerEventSource10setTimeoutEy __ZN18IOTimerEventSource10wakeAtTimeEjj +__ZN18IOTimerEventSource10wakeAtTimeEjyy __ZN18IOTimerEventSource10wakeAtTimeEy __ZN18IOTimerEventSource12setTimeoutMSEj __ZN18IOTimerEventSource12setTimeoutUSEj @@ -266,9 +268,6 @@ __ZN18IOTimerEventSource12wakeAtTimeMSEj __ZN18IOTimerEventSource12wakeAtTimeUSEj __ZN18IOTimerEventSource15setTimeoutTicksEj __ZN18IOTimerEventSource15wakeAtTimeTicksEj -__ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource0Ev -__ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource1Ev -__ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource2Ev __ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource3Ev __ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource4Ev __ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource5Ev @@ -291,6 +290,8 @@ __ZN21IOInterruptController31_RESERVEDIOInterruptController2Ev __ZN21IOInterruptController31_RESERVEDIOInterruptController3Ev __ZN21IOInterruptController31_RESERVEDIOInterruptController4Ev __ZN21IOInterruptController31_RESERVEDIOInterruptController5Ev +__ZN21IOInterruptController28timeStampInterruptHandlerEndEiP17IOInterruptVector +__ZN21IOInterruptController30timeStampInterruptHandlerStartEiP17IOInterruptVector __ZN21IONaturalMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvj __ZN21IONaturalMemoryCursor17withSpecificationEyyy __ZN21IONaturalMemoryCursor21initWithSpecificationEyyy diff --git a/config/Libkern.arm.exports b/config/Libkern.arm.exports new file mode 100644 index 000000000..051d0d07b --- /dev/null +++ b/config/Libkern.arm.exports @@ -0,0 +1,4 @@ +_OSAddAtomic64 +_OSCompareAndSwap64 +__ZN12OSOrderedSet12withCapacityEjPFlPK15OSMetaClassBaseS2_PvES3_ +__ZN12OSOrderedSet16initWithCapacityEjPFlPK15OSMetaClassBaseS2_PvES3_ diff --git a/config/Libkern.arm64.exports b/config/Libkern.arm64.exports new file mode 100644 index 000000000..cc07f5dc2 --- /dev/null +++ b/config/Libkern.arm64.exports @@ -0,0 +1,5 @@ +_OSAddAtomic64 +_OSCompareAndSwap64 +_PAGE_SHIFT_CONST +__ZN12OSOrderedSet12withCapacityEjPFiPK15OSMetaClassBaseS2_PvES3_ +__ZN12OSOrderedSet16initWithCapacityEjPFiPK15OSMetaClassBaseS2_PvES3_ diff --git a/config/Libkern.exports b/config/Libkern.exports index 93dfdd556..6769ed75b 100644 --- a/config/Libkern.exports +++ b/config/Libkern.exports @@ -591,8 +591,17 @@ __Znwm ___bzero ___cxa_pure_virtual ___llvm_profile_runtime +___memcpy_chk +___memmove_chk +___memset_chk ___stack_chk_fail ___stack_chk_guard +___strlcpy_chk +___strlcat_chk +___strncpy_chk +___strncat_chk +___strcpy_chk +___strcat_chk __os_log_default __os_log_internal _adler32 @@ -684,6 +693,7 @@ _memcmp _memcpy _memmove _memset +_memset_s _ml_at_interrupt_context _ml_get_interrupts_enabled _ml_set_interrupts_enabled diff --git a/config/MACFramework.arm.exports b/config/MACFramework.arm.exports new file mode 100644 index 000000000..e69de29bb diff --git a/config/MACFramework.arm64.exports b/config/MACFramework.arm64.exports new file mode 100644 index 000000000..e69de29bb diff --git a/config/MASTER b/config/MASTER index cdacdd750..2fa158923 100644 --- a/config/MASTER +++ b/config/MASTER @@ -72,20 +72,13 @@ options NO_DIRECT_RPC # for untyped mig servers # options LOOP # loopback support # options VLAN # # options BOND # # +options IF_FAKE # # options AH_ALL_CRYPTO # AH all crypto algs # options IPCOMP_ZLIB # IP compression using zlib # options PF # Packet Filter # -options PF_ALTQ # PF ALTQ (Alternate Queueing) # options PF_ECN # PF use ECN marking # options PFLOG # PF log interface # -options PKTSCHED_CBQ # CBQ packet scheduler # -options PKTSCHED_HFSC # H-FSC packet scheduler # -options PKTSCHED_PRIQ # PRIQ packet scheduler # -options PKTSCHED_FAIRQ # FAIRQ packet scheduler # options MEASURE_BW # interface bandwidth measurement # -options CLASSQ_BLUE # BLUE queueing algorithm # -options CLASSQ_RED # RED queueing algorithm # -options CLASSQ_RIO # RIO queueing algorithm # options DUMMYNET # dummynet support # options TRAFFIC_MGT # traffic management support # options MULTICAST # Internet Protocol Class-D $ @@ -193,7 +186,6 @@ options CONFIG_KN_HASHSIZE=20 # # # configurable vfs related resources # CONFIG_VNODES - used to pre allocate vnode related resources -# CONFIG_VNODE_FREE_MIN - mininmum number of free vnodes # CONFIG_NC_HASH - name cache hash table allocation # CONFIG_VFS_NAMES - name strings # @@ -208,12 +200,6 @@ options CONFIG_VNODES=263168 # options CONFIG_VNODES=10240 # options CONFIG_VNODES=750 # -options CONFIG_VNODE_FREE_MIN=500 # -options CONFIG_VNODE_FREE_MIN=300 # -options CONFIG_VNODE_FREE_MIN=200 # -options CONFIG_VNODE_FREE_MIN=100 # -options CONFIG_VNODE_FREE_MIN=75 # - options CONFIG_NC_HASH=5120 # options CONFIG_NC_HASH=4096 # options CONFIG_NC_HASH=2048 # @@ -295,8 +281,12 @@ options CONFIG_MFCTBLSIZ=16 # # # configurable kernel message buffer size # -options CONFIG_MSG_BSIZE=4096 # -options CONFIG_MSG_BSIZE=16384 # +options CONFIG_MSG_BSIZE_REL=4096 # +options CONFIG_MSG_BSIZE_DEV=4096 # +options CONFIG_MSG_BSIZE_REL=16384 # +options CONFIG_MSG_BSIZE_DEV=131072 # +options CONFIG_MSG_BSIZE=CONFIG_MSG_BSIZE_REL # +options CONFIG_MSG_BSIZE=CONFIG_MSG_BSIZE_DEV # # # maximum size of the per-process Mach IPC table @@ -304,14 +294,11 @@ options CONFIG_MSG_BSIZE=16384 # options CONFIG_IPC_TABLE_ENTRIES_STEPS=64 # 137898 entries # options CONFIG_IPC_TABLE_ENTRIES_STEPS=256 # 300714 entries # - # # configurable kernel - use these options to strip strings from panic # and printf calls. -# no_panic_str - saves around 50K of kernel footprint. # no_printf_str - saves around 45K of kernel footprint. # -options CONFIG_NO_PANIC_STRINGS # options CONFIG_NO_PRINTF_STRINGS # options CONFIG_NO_KPRINTF_STRINGS # @@ -399,7 +386,8 @@ options CONFIG_IO_ACCOUNTING # # For now debug is enabled wherever inheritance is # options IMPORTANCE_INHERITANCE # -options IMPORTANCE_DEBUG # +options IMPORTANCE_TRACE # +options IMPORTANCE_DEBUG # options CONFIG_TELEMETRY # @@ -415,6 +403,11 @@ options CONFIG_ECC_LOGGING # # options CONFIG_COREDUMP # +# +# Vnode guards +# +options CONFIG_VNGUARD # + # # Ethernet (ARP) # @@ -465,6 +458,7 @@ pseudo-device systrace 1 init systrace_init # pseudo-device fbt 1 init fbt_init # pseudo-device profile_prvd 1 init profile_init # + # # IOKit configuration options # @@ -525,7 +519,6 @@ options CONFIG_AUDIT # Kernel auditing # # forcibly suspending tasks when the demand exceeds supply. This # option should be on. # -options MACH_RT options TASK_SWAPPER # # @@ -607,6 +600,7 @@ options CONFIG_DTRACE # # options KPERF # options KPC # + options PGO # # MACH_COUNTERS enables code that handles various counters in the system. @@ -716,12 +710,6 @@ options OS_REASON_DEBUG # # options CONFIG_ATM # -# -# Kernel Voucher Attr Manager for BANK -# -options CONFIG_BANK # - - # Group related tasks together into coalitions options CONFIG_COALITIONS # @@ -741,3 +729,9 @@ options VIDEO_CONSOLE # uni-directional output over framebuffer # Syscall options # options CONFIG_REQUIRES_U32_MUNGING # incoming U32 argument structures must be munged to match U64 # + +# +# copyout() instrumentation +# +options COPYOUT_SHIM # Shim for copyout memory analysis via kext # + diff --git a/config/MASTER.arm b/config/MASTER.arm new file mode 100644 index 000000000..b534c3569 --- /dev/null +++ b/config/MASTER.arm @@ -0,0 +1,86 @@ +# +# Mach Operating System +# Copyright (c) 1986 Carnegie-Mellon University +# Copyright 2001-2016 Apple Inc. +# +# All rights reserved. The CMU software License Agreement +# specifies the terms and conditions for use and redistribution. +# +###################################################################### +# +# Master Apple configuration file (see the master machine independent +# configuration file for a description of the file format). +# +###################################################################### +# +# Standard Apple OS Configurations: +# -------- ----- -- --------------- +# +# KERNEL_BASE = [ arm xsmall config_embedded ] +# KERNEL_RELEASE = [ KERNEL_BASE ] +# KERNEL_DEV = [ KERNEL_BASE development mach_assert config_xnupost proc_ref_debug os_reason_debug ] +# KERNEL_DEBUG = [ KERNEL_BASE debug mach_assert config_xnupost config_ltable_stats config_ltable_debug config_waitq_stats config_waitq_debug ] +# BSD_BASE = [ mach_bsd config_workqueue psynch config_proc_uuid_policy ] +# BSD_RELEASE = [ BSD_BASE no_printf_str no_kprintf_str secure_kernel ] +# BSD_DEV = [ BSD_BASE config_imageboot config_coredump pgo config_vnguard ] +# BSD_DEBUG = [ BSD_BASE config_imageboot config_coredump pgo config_vnguard ] +# FILESYS_BASE = [ devfs fifo fs_compression config_protect config_fse routefs quota namedstreams ] +# FILESYS_RELEASE= [ FILESYS_BASE ] +# FILESYS_DEV = [ FILESYS_BASE fdesc ] +# FILESYS_DEBUG = [ FILESYS_BASE fdesc ] +# NFS = [ nfsclient nfsserver ] +# SKYWALK_BASE = [ skywalk config_nexus_user_pipe config_nexus_kernel_pipe config_nexus_monitor config_nexus_flowswitch config_nexus_netif ] +# SKYWALK_RELEASE = [ SKYWALK_BASE ] +# SKYWALK_DEV = [ SKYWALK_BASE ] +# SKYWALK_DEBUG = [ SKYWALK_BASE ] +# NETWORKING = [ inet tcpdrop_synfin bpfilter inet6 ipv6send if_bridge traffic_mgt dummynet ah_all_crypto if_fake ] +# VPN = [ ipsec flow_divert necp content_filter ] +# PF = [ pf ] +# MULTIPATH = [ multipath mptcp ] +# IOKIT_BASE = [ iokit iokitcpp no_kextd no_kernel_hid config_sleep ] +# IOKIT_RELEASE = [ IOKIT_BASE ] +# IOKIT_DEV = [ IOKIT_BASE iokitstats iotracking ] +# IOKIT_DEBUG = [ IOKIT_BASE iokitstats iotracking ] +# LIBKERN_BASE = [ libkerncpp config_kec_fips zlib crypto_sha2 ] +# LIBKERN_RELEASE =[ LIBKERN_BASE ] +# LIBKERN_DEV = [ LIBKERN_BASE iotracking ] +# LIBKERN_DEBUG = [ LIBKERN_BASE iotracking ] +# PERF_DBG_BASE = [ mach_kdp config_serial_kdp kperf kpc MONOTONIC_BASE ] +# PERF_DBG_RELEASE=[ PERF_DBG_BASE ist_kdebug ] +# PERF_DBG_DEV = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging interrupt_masked_debug ] +# PERF_DBG_DEBUG = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging interrupt_masked_debug ] +# MACH_BASE = [ mach slidable vc_progress_white mdebug ipc_debug importance_inheritance config_atm config_coalitions config_library_validation config_iosched config_telemetry config_sysdiagnose ] +# MACH_RELEASE = [ MACH_BASE config_skip_precise_user_kernel_time debugger_for_zone_info ] +# MACH_DEV = [ MACH_BASE task_zone_info config_io_accounting importance_trace ] +# MACH_DEBUG = [ MACH_BASE task_zone_info config_io_accounting importance_trace importance_debug ] +# SCHED_BASE = [ config_sched_traditional config_sched_multiq ] +# SCHED_RELEASE = [ SCHED_BASE ] +# SCHED_DEV = [ SCHED_BASE ] +# SCHED_DEBUG = [ SCHED_BASE config_sched_grrr config_sched_proto ] +# VM_BASE = [ vm_pressure_events jetsam memorystatus config_code_decryption config_cs_validation_bitmap ] +# VM_RELEASE = [ VM_BASE ] +# VM_DEV = [ VM_BASE dynamic_codesigning ] +# VM_DEBUG = [ VM_BASE dynamic_codesigning ] +# SECURITY = [ config_macf ] +# RELEASE = [ KERNEL_RELEASE BSD_RELEASE FILESYS_RELEASE SKYWALK_RELEASE NETWORKING PF MULTIPATH VPN IOKIT_RELEASE LIBKERN_RELEASE PERF_DBG_RELEASE MACH_RELEASE SCHED_RELEASE VM_RELEASE SECURITY ] +# DEVELOPMENT = [ KERNEL_DEV BSD_DEV FILESYS_DEV NFS SKYWALK_DEV NETWORKING PF MULTIPATH VPN IOKIT_DEV LIBKERN_DEV PERF_DBG_DEV MACH_DEV SCHED_DEV VM_DEV SECURITY ] +# DEBUG = [ KERNEL_DEBUG BSD_DEBUG FILESYS_DEBUG SKYWALK_DEBUG NETWORKING PF MULTIPATH VPN IOKIT_DEBUG LIBKERN_DEBUG PERF_DBG_DEBUG MACH_DEBUG SCHED_DEBUG VM_DEBUG SECURITY ] +# +###################################################################### +# +machine "arm" # + +makeoptions OSFMK_MACHINE = "arm" # + +options COUNT_SYSCALLS # count bsd system calls # + +options SLIDABLE=1 # Use PIE-assembly in *.s # +options TRASH_VFP_ON_SAVE # + +options CONFIG_VNODES=1024 # + +options CONFIG_FREEZE_SUSPENDED_MIN=4 # + +options CONFIG_MACH_APPROXIMATE_TIME + +options INTERRUPT_MASKED_DEBUG=1 # # diff --git a/config/MASTER.arm64 b/config/MASTER.arm64 new file mode 100644 index 000000000..ae4eb4903 --- /dev/null +++ b/config/MASTER.arm64 @@ -0,0 +1,92 @@ +# +# Mach Operating System +# Copyright (c) 1986 Carnegie-Mellon University +# Copyright 2001-2016 Apple Inc. +# +# All rights reserved. The CMU software License Agreement +# specifies the terms and conditions for use and redistribution. +# +###################################################################### +# +# Master Apple configuration file (see the master machine independent +# configuration file for a description of the file format). +# +###################################################################### +# +# Standard Apple OS Configurations: +# -------- ----- -- --------------- +# +# KERNEL_BASE = [ arm64 xsmall config_embedded config_requires_u32_munging ] +# KERNEL_RELEASE = [ KERNEL_BASE ] +# KERNEL_DEV = [ KERNEL_BASE development mach_assert config_xnupost proc_ref_debug os_reason_debug pgtrace ] +# KERNEL_DEBUG = [ KERNEL_BASE debug mach_assert config_xnupost config_ltable_stats config_ltable_debug config_waitq_stats config_waitq_debug pgtrace ] +# BSD_BASE = [ mach_bsd config_workqueue psynch config_proc_uuid_policy config_personas ] +# BSD_RELEASE = [ BSD_BASE no_printf_str no_kprintf_str secure_kernel ] +# BSD_DEV = [ BSD_BASE config_imageboot config_coredump pgo config_vnguard ] +# BSD_DEBUG = [ BSD_BASE config_imageboot config_coredump pgo config_vnguard ] +# FILESYS_BASE = [ devfs fifo fs_compression config_protect config_fse routefs quota namedstreams ] +# FILESYS_RELEASE= [ FILESYS_BASE ] +# FILESYS_DEV = [ FILESYS_BASE fdesc ] +# FILESYS_DEBUG = [ FILESYS_BASE fdesc ] +# NFS = [ nfsclient nfsserver ] +# SKYWALK_BASE = [ skywalk config_nexus_user_pipe config_nexus_kernel_pipe config_nexus_monitor config_nexus_flowswitch config_nexus_netif ] +# SKYWALK_RELEASE = [ SKYWALK_BASE ] +# SKYWALK_DEV = [ SKYWALK_BASE ] +# SKYWALK_DEBUG = [ SKYWALK_BASE ] +# NETWORKING = [ inet tcpdrop_synfin bpfilter inet6 ipv6send if_bridge traffic_mgt dummynet ah_all_crypto packet_mangler if_fake ] +# VPN = [ ipsec flow_divert necp content_filter ] +# PF = [ pf ] +# MULTIPATH = [ multipath mptcp ] +# IOKIT_BASE = [ iokit iokitcpp no_kextd no_kernel_hid config_sleep ] +# IOKIT_RELEASE = [ IOKIT_BASE ] +# IOKIT_DEV = [ IOKIT_BASE iokitstats iotracking ] +# IOKIT_DEBUG = [ IOKIT_BASE iokitstats iotracking] +# LIBKERN_BASE = [ libkerncpp config_kec_fips zlib crypto_sha2 ] +# LIBKERN_RELEASE =[ LIBKERN_BASE ] +# LIBKERN_DEV = [ LIBKERN_BASE iotracking ] +# LIBKERN_DEBUG = [ LIBKERN_BASE iotracking ] +# PERF_DBG_BASE = [ mach_kdp config_serial_kdp MONOTONIC_BASE kperf kpc ] +# PERF_DBG_RELEASE=[ PERF_DBG_BASE ist_kdebug ] +# PERF_DBG_DEV = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger interrupt_masked_debug ] +# PERF_DBG_DEBUG = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger interrupt_masked_debug ] +# MACH_BASE = [ mach slidable config_ecc_logging vc_progress_white mdebug ipc_debug importance_inheritance config_atm config_coalitions config_iosched config_library_validation config_sysdiagnose config_telemetry config_mach_bridge_recv_time] +# MACH_RELEASE = [ MACH_BASE config_skip_precise_user_kernel_time debugger_for_zone_info ] +# MACH_DEV = [ MACH_BASE task_zone_info config_io_accounting importance_trace ] +# MACH_DEBUG = [ MACH_BASE task_zone_info config_io_accounting importance_trace importance_debug ] +# SCHED_BASE = [ config_sched_traditional config_sched_multiq config_sched_deferred_ast ] +# SCHED_RELEASE = [ SCHED_BASE ] +# SCHED_DEV = [ SCHED_BASE ] +# SCHED_DEBUG = [ SCHED_BASE config_sched_grrr config_sched_proto ] +# VM_BASE = [ vm_pressure_events jetsam freeze memorystatus config_code_decryption phantom_cache config_secluded_memory config_background_queue config_cs_validation_bitmap] +# VM_RELEASE = [ VM_BASE ] +# VM_DEV = [ VM_BASE dynamic_codesigning ] +# VM_DEBUG = [ VM_BASE dynamic_codesigning ] +# SECURITY = [ config_macf kernel_integrity ] +# RELEASE = [ KERNEL_RELEASE BSD_RELEASE FILESYS_RELEASE SKYWALK_RELEASE NETWORKING PF MULTIPATH VPN IOKIT_RELEASE LIBKERN_RELEASE PERF_DBG_RELEASE MACH_RELEASE SCHED_RELEASE VM_RELEASE SECURITY ] +# DEVELOPMENT = [ KERNEL_DEV BSD_DEV FILESYS_DEV NFS SKYWALK_DEV NETWORKING PF MULTIPATH VPN IOKIT_DEV LIBKERN_DEV PERF_DBG_DEV MACH_DEV SCHED_DEV VM_DEV SECURITY ] +# DEBUG = [ KERNEL_DEBUG BSD_DEBUG FILESYS_DEBUG SKYWALK_DEBUG NETWORKING PF MULTIPATH VPN IOKIT_DEBUG LIBKERN_DEBUG PERF_DBG_DEBUG MACH_DEBUG SCHED_DEBUG VM_DEBUG SECURITY ] +# KASAN = [ DEVELOPMENT ] +# +###################################################################### +# +machine "arm64" # + +makeoptions OSFMK_MACHINE = "arm64" # + +options COUNT_SYSCALLS # count bsd system calls # +options TRASH_VFP_ON_SAVE # +options ALTERNATE_DEBUGGER # + +options CONFIG_VNODES=1024 # + +options CONFIG_FREEZE_SUSPENDED_MIN=4 # + +options CONFIG_MACH_APPROXIMATE_TIME + +options CONFIG_KERNEL_INTEGRITY # + +options INTERRUPT_MASKED_DEBUG=1 # # + +options CONFIG_PGTRACE # +options CONFIG_PGTRACE_NONKEXT # +pseudo-device pgtrace 1 init pgtrace_dev_init # diff --git a/config/MASTER.x86_64 b/config/MASTER.x86_64 index 441be7312..0c8b7a4e5 100644 --- a/config/MASTER.x86_64 +++ b/config/MASTER.x86_64 @@ -1,7 +1,7 @@ # # Mach Operating System # Copyright (c) 1986 Carnegie-Mellon University -# Copyright 2001-2013 Apple Inc. +# Copyright 2001-2016 Apple Inc. # # All rights reserved. The CMU software License Agreement # specifies the terms and conditions for use and redistribution. @@ -20,17 +20,18 @@ # KERNEL_RELEASE = [ KERNEL_BASE ] # KERNEL_DEV = [ KERNEL_BASE development mach_assert config_xnupost proc_ref_debug os_reason_debug ] # KERNEL_DEBUG = [ KERNEL_BASE debug mach_assert config_xnupost config_ltable_stats config_ltable_debug config_waitq_stats config_waitq_debug ] -# BSD = [ mach_bsd sysv_sem sysv_msg sysv_shm config_imageboot config_workqueue psynch config_proc_uuid_policy config_coredump pgo ] +# BSD_BASE = [ mach_bsd sysv_sem sysv_msg sysv_shm config_imageboot config_workqueue psynch config_proc_uuid_policy config_coredump pgo ] +# BSD_RELEASE = [ BSD_BASE ] +# BSD_DEV = [ BSD_BASE config_vnguard ] +# BSD_DEBUG = [ BSD_BASE config_vnguard ] # FILESYS_BASE = [ devfs fdesc config_dev_kmem config_fse quota namedstreams config_protect fifo config_volfs fs_compression config_imgsrc_access config_triggers config_ext_resolver config_searchfs config_appledouble nullfs config_mnt_suid ] # FILESYS_RELEASE= [ FILESYS_BASE ] # FILESYS_DEV = [ FILESYS_BASE ] # FILESYS_DEBUG = [ FILESYS_BASE ] # NFS = [ nfsclient nfsserver ] -# NETWORKING = [ inet inet6 ipv6send tcpdrop_synfin bpfilter dummynet traffic_mgt sendfile ah_all_crypto bond vlan gif stf ifnet_input_chk config_mbuf_jumbo if_bridge ipcomp_zlib MULTIPATH packet_mangler ] +# NETWORKING = [ inet inet6 ipv6send tcpdrop_synfin bpfilter dummynet traffic_mgt sendfile ah_all_crypto bond vlan gif stf ifnet_input_chk config_mbuf_jumbo if_bridge ipcomp_zlib MULTIPATH packet_mangler if_fake ] # VPN = [ ipsec flow_divert necp content_filter ] # PF = [ pf pflog ] -# PKTSCHED = [ pktsched_cbq pktsched_fairq pktsched_hfsc pktsched_priq ] -# CLASSQ = [ classq_blue classq_red classq_rio ] # MULTIPATH = [ multipath mptcp ] # IOKIT_BASE = [ iokit iokitcpp hibernation config_sleep iokitstats hypervisor ] # IOKIT_RELEASE = [ IOKIT_BASE ] @@ -40,20 +41,21 @@ # LIBKERN_RELEASE =[ LIBKERN_BASE ] # LIBKERN_DEV = [ LIBKERN_BASE iotracking ] # LIBKERN_DEBUG = [ LIBKERN_BASE iotracking ] -# PERF_DBG = [ config_dtrace mach_kdp config_serial_kdp kdp_interactive_debugging kperf kpc zleaks config_gzalloc ] -# MACH_BASE = [ mach config_kext_basement mdebug ipc_debug config_mca config_vmx config_mtrr config_lapic config_telemetry importance_inheritance config_atm config_bank config_coalitions hypervisor config_iosched config_sysdiagnose ] +# PERF_DBG = [ config_dtrace mach_kdp config_serial_kdp kdp_interactive_debugging kperf kpc zleaks config_gzalloc MONOTONIC_BASE ] +# MACH_BASE = [ mach config_kext_basement mdebug ipc_debug config_mca config_vmx config_mtrr config_lapic config_telemetry importance_inheritance config_atm config_coalitions hypervisor config_iosched config_sysdiagnose config_mach_bridge_send_time copyout_shim ] # MACH_RELEASE = [ MACH_BASE ] -# MACH_DEV = [ MACH_BASE task_zone_info ] -# MACH_DEBUG = [ MACH_BASE task_zone_info ] +# MACH_DEV = [ MACH_BASE task_zone_info importance_trace ] +# MACH_DEBUG = [ MACH_BASE task_zone_info importance_trace importance_debug ] # SCHED_BASE = [ config_sched_traditional config_sched_multiq config_sched_sfi ] # SCHED_RELEASE = [ SCHED_BASE ] # SCHED_DEV = [ SCHED_BASE ] # SCHED_DEBUG = [ SCHED_BASE config_sched_grrr config_sched_proto ] # VM = [ vm_pressure_events memorystatus dynamic_codesigning config_code_decryption encrypted_swap phantom_cache config_background_queue] # SECURITY = [ config_macf config_audit config_csr ] -# RELEASE = [ KERNEL_RELEASE BSD FILESYS_RELEASE NFS SKYWALK_RELEASE NETWORKING PF VPN IOKIT_RELEASE LIBKERN_RELEASE PERF_DBG MACH_RELEASE SCHED_RELEASE VM SECURITY ] -# DEVELOPMENT = [ KERNEL_DEV BSD FILESYS_DEV NFS SKYWALK_DEV NETWORKING PF VPN IOKIT_DEV LIBKERN_DEV PERF_DBG MACH_DEV SCHED_DEV VM SECURITY ] -# DEBUG = [ KERNEL_DEBUG BSD FILESYS_DEBUG NFS SKYWALK_DEBUG NETWORKING PF VPN IOKIT_DEBUG LIBKERN_DEBUG PERF_DBG MACH_DEBUG SCHED_DEBUG VM SECURITY ] +# RELEASE = [ KERNEL_RELEASE BSD_RELEASE FILESYS_RELEASE NFS SKYWALK_RELEASE NETWORKING PF VPN IOKIT_RELEASE LIBKERN_RELEASE PERF_DBG MACH_RELEASE SCHED_RELEASE VM SECURITY ] +# DEVELOPMENT = [ KERNEL_DEV BSD_DEV FILESYS_DEV NFS SKYWALK_DEV NETWORKING PF VPN IOKIT_DEV LIBKERN_DEV PERF_DBG MACH_DEV SCHED_DEV VM SECURITY ] +# DEBUG = [ KERNEL_DEBUG BSD_DEBUG FILESYS_DEBUG NFS SKYWALK_DEBUG NETWORKING PF VPN IOKIT_DEBUG LIBKERN_DEBUG PERF_DBG MACH_DEBUG SCHED_DEBUG VM SECURITY ] +# KASAN = [ DEVELOPMENT ] # ###################################################################### # diff --git a/config/Mach.arm.exports b/config/Mach.arm.exports new file mode 100644 index 000000000..f5f0e735d --- /dev/null +++ b/config/Mach.arm.exports @@ -0,0 +1,2 @@ +_mach_msg_send_from_kernel +_semaphore_timedwait diff --git a/config/Mach.arm64.exports b/config/Mach.arm64.exports new file mode 100644 index 000000000..cc31a814e --- /dev/null +++ b/config/Mach.arm64.exports @@ -0,0 +1 @@ +_semaphore_timedwait diff --git a/config/Mach.exports b/config/Mach.exports index 09ca16fb4..a33253ff5 100644 --- a/config/Mach.exports +++ b/config/Mach.exports @@ -1,3 +1,4 @@ +_absolutetime_to_continuoustime _absolutetime_to_nanoseconds _assert_wait _assert_wait_deadline @@ -12,6 +13,7 @@ _clock_get_uptime _clock_interval_to_absolutetime_interval _clock_interval_to_deadline _clock_timebase_info +_continuoustime_to_absolutetime _current_task _current_thread _kernel_task @@ -45,6 +47,7 @@ _task_reference _thread_block _thread_block_parameter _thread_call_allocate +_thread_call_allocate_with_options _thread_call_cancel _thread_call_enter _thread_call_enter1 @@ -57,5 +60,7 @@ _thread_reference _thread_terminate _thread_tid _thread_wakeup_prim +_vm_kernel_addrhash:_vm_kernel_addrhash_external +_vm_kernel_addrhide _vm_kernel_addrperm_external _vm_kernel_unslide_or_perm_external diff --git a/config/Makefile b/config/Makefile index d88a78568..36b16f259 100644 --- a/config/Makefile +++ b/config/Makefile @@ -13,6 +13,7 @@ INSTALL_KEXT_DIR = $(DSTROOT)$(INSTALL_EXTENSIONS_DIR) KEXT_PLIST_LIST = \ System.kext/Info.plist \ + System.kext/PlugIns/Kasan.kext/Info.plist \ System.kext/PlugIns/AppleNMI.kext/Info.plist \ System.kext/PlugIns/ApplePlatformFamily.kext/Info.plist \ System.kext/PlugIns/IONVRAMFamily.kext/Info.plist \ @@ -128,12 +129,15 @@ $(DSTROOT)/$(KRESDIR)/$(MD_SUPPORTED_KPI_FILENAME) $(DSTROOT)/$(KRESDIR)/$(MI_SU @echo "$(ColorH)INSTALL$(Color0) $(ColorF)$*$(Color0)" $(_v)$(INSTALL) $(INSTALL_FLAGS) $< $@ +ifneq ($(INSTALL_KASAN_ONLY),1) do_config_install:: $(SYMROOT_INSTALL_KEXT_MACHO_FILES) \ $(SYMROOT_INSTALL_KEXT_PLISTS) \ $(DSTROOT_INSTALL_KEXT_MACHO_FILES) \ $(DSTROOT_INSTALL_KEXT_PLISTS) \ $(DSTROOT)/$(KRESDIR)/$(MD_SUPPORTED_KPI_FILENAME) \ $(DSTROOT)/$(KRESDIR)/$(MI_SUPPORTED_KPI_FILENAME) +endif + $(OBJPATH)/all-kpi.exp: $(EXPORTS_FILES) $(_v)$(SOURCE)/generate_linker_exports.sh $@ $+ diff --git a/config/MasterVersion b/config/MasterVersion index a02b1d1cb..d697dff22 100644 --- a/config/MasterVersion +++ b/config/MasterVersion @@ -1,4 +1,4 @@ -16.7.0 +17.0.0 # The first line of this file contains the master version number for the kernel. # All other instances of the kernel version in xnu are derived from this file. diff --git a/config/Private.arm.exports b/config/Private.arm.exports new file mode 100644 index 000000000..0b393134f --- /dev/null +++ b/config/Private.arm.exports @@ -0,0 +1,21 @@ +__ZN17IONVRAMController* +__ZTV17IONVRAMController +_IOCPURunPlatformActiveActions +_IOCPURunPlatformQuiesceActions +_PE_get_default +_PE_reboot_on_panic +_PE_mark_hwaccess +_ml_arm_sleep +_ml_get_abstime_offset +_ml_get_conttime_offset +_ml_get_wake_timebase +_proc_getcdhash +_cpu_broadcast_xcall +_cpu_xcall +_cpu_number +_enable_kernel_vfp_context +_PE_consistent_debug_register +_ml_static_ptovirt +_ml_static_mfree +_sched_perfcontrol_register_callbacks +_sched_perfcontrol_update_recommended_cores diff --git a/config/Private.arm64.exports b/config/Private.arm64.exports new file mode 100644 index 000000000..ab9007317 --- /dev/null +++ b/config/Private.arm64.exports @@ -0,0 +1,34 @@ +_IOCPURunPlatformActiveActions +_IOCPURunPlatformQuiesceActions +_PE_consistent_debug_register +_PE_get_default +_PE_reboot_on_panic +_PE_mark_hwaccess +__ZN17IONVRAMController* +__ZTV17IONVRAMController +_cpu_broadcast_xcall +_cpu_xcall +_cpu_cluster_id +_cpu_number +_cpu_qos_update_register +_ecc_log_record_event +_ml_arm_sleep +_ml_get_abstime_offset +_ml_get_conttime_offset +_ml_get_wake_timebase +_ml_thread_is64bit +_pe_shmcon_set_child +_proc_getcdhash +_sched_perfcontrol_register_callbacks +_sched_perfcontrol_update_recommended_cores +_sched_perfcontrol_thread_group_recommend +_sched_perfcontrol_update_callback_deadline +_ml_static_ptovirt +_ml_static_mfree +_ex_cb_register +_pgtrace_init +_pgtrace_start +_pgtrace_stop +_pgtrace_active +_pgtrace_add_probe +_pgtrace_clear_probe diff --git a/config/Private.exports b/config/Private.exports index 7d563fe0c..5ce2ff9c6 100644 --- a/config/Private.exports +++ b/config/Private.exports @@ -52,6 +52,8 @@ _cdevsw_setkqueueok _chudxnu_platform_ptr _clalloc _clfree +_cluster_unlock_direct_read +_cluster_lock_direct_read _cons_cinput _convert_port_to_task_suspension_token _convert_task_suspension_token_to_port @@ -67,15 +69,15 @@ _cpx_flush _cpx_free _cpx_has_key _cpx_init -_cpx_is_sep_wrapped_key _cpx_is_composite_key +_cpx_is_sep_wrapped_key _cpx_iv_aes_ctx _cpx_key _cpx_key_len _cpx_max_key_len _cpx_set_aes_iv_key -_cpx_set_is_sep_wrapped_key _cpx_set_is_composite_key +_cpx_set_is_sep_wrapped_key _cpx_set_key_len _cpx_set_use_offset_for_iv _cpx_set_synthetic_offset_for_iv @@ -101,19 +103,23 @@ _csblob_get_addr _csblob_get_base_offset _csblob_get_cdhash _csblob_get_entitlements +_csblob_get_flags +_csblob_get_hashtype _csblob_get_identity _csblob_get_platform_binary -_csblob_get_flags _csblob_get_teamid +_csblob_get_signer_type _csblob_get_size _csfg_get_cdhash _csfg_get_path _csfg_get_platform_binary _csfg_get_prod_signed +_csfg_get_signer_type _csfg_get_teamid _csproc_get_blob _csproc_get_platform_binary _csproc_get_prod_signed +_csproc_get_signer_type _csproc_get_teamid _csvnode_get_blob _csvnode_get_teamid @@ -130,7 +136,9 @@ _gpu_describe _gpu_fceiling_cb_register _gpu_submission_telemetry _hz +_iflt_attach_internal _ifnet_allocate_extended +_ifnet_allocate_internal _ifnet_bandwidths _ifnet_clone_attach _ifnet_clone_detach @@ -176,15 +184,8 @@ _ifnet_set_rcvq_maxlen _ifnet_set_sndq_maxlen _ifnet_start _ifnet_subfamily -_ifnet_transmit_burst_end -_ifnet_transmit_burst_start _ifnet_tx_compl _ifnet_tx_compl_status -_ifnet_set_packetpreamblelen -_ifnet_packetpreamblelen -_ifnet_maxpacketpreamblelen -_ifnet_set_fastlane_capable -_ifnet_get_fastlane_capable _ifnet_get_unsent_bytes _ifnet_get_buffer_status _ifnet_normalise_unsent_data @@ -198,6 +199,8 @@ _io_rate_update_register _ip_gre_output _ip_gre_register_input _ipc_port_release_send +_ipf_addv4_internal +_ipf_addv6_internal _kauth_cred_getgroups _kauth_cred_grnam2guid _kauth_cred_guid2grnam @@ -214,11 +217,18 @@ _kdp_register_link _kdp_set_interface _kdp_unregister_link _kdp_unregister_send_receive +_kern_allocation_get_name +_kern_allocation_name_allocate +_kern_allocation_name_release +_thread_set_allocation_name _kern_asl_msg _kern_asl_msg_va +_kern_coredump_log +_kern_register_coredump_helper _kern_config_is_development _kern_stack_snapshot_with_reason _kernel_debug_string +_kevent_id_internal _kevent_qos_internal _kevent_qos_internal_bind _kevent_qos_internal_unbind @@ -239,13 +249,12 @@ _m_prepend_2 _m_pullup _m_split _m_trailingspace:_mbuf_trailingspace -_mac_proc_set_enforce -_mach_vm_allocate +_mach_vm_allocate:_mach_vm_allocate_external _mach_vm_behavior_set _mach_vm_deallocate -_mach_vm_map +_mach_vm_map:_mach_vm_map_external _mach_vm_protect -_mach_vm_remap +_mach_vm_remap:_mach_vm_remap_external _mbuf_add_drvaux _mbuf_del_drvaux _mbuf_find_drvaux @@ -347,18 +356,21 @@ _sbappendaddr _sbappendrecord _sbflush _sbspace +_sflt_register_internal _soabort _socantrcvmore _socantsendmore +_sock_accept_internal _sock_catchevents _sock_getlistener _sock_gettclassopt +_sock_iskernel _sock_release _sock_retain _sock_settclassopt _sock_setupcall _sock_setupcalls -_sock_iskernel +_sock_socket_internal _sodisconnect _sofree _sofreelastref @@ -378,6 +390,7 @@ _strnstr _sysdiagnose_notify_user _termioschars _thread_call_allocate_with_priority +_thread_call_allocate_with_qos _thread_call_cancel_wait _thread_clear_eager_preempt _thread_dispatchqaddr @@ -432,10 +445,11 @@ _vfs_getattr _vfs_getbyid _vfs_mntlabel _vfs_nativexattrs +_vfs_set_root_unmounted_cleanly _vfs_setcompoundopen _vfs_throttle_mask _vfs_vnodecovered -_vm_fault +_vm_fault:_vm_fault_external _vm_map_copy_copy _vm_map_copy_discard _vm_map_copyin @@ -450,6 +464,7 @@ _vm_map_trunc_page_mask _vm_map_wire_and_extract:_vm_map_wire_and_extract_external _vm_page_wire_count _vn_getpath_fsenter +_vn_getpath_fsenter_with_parent _vn_searchfs_inappropriate_name _vnode_create_empty _vnode_initialize @@ -501,6 +516,7 @@ _throttle_io_will_be_throttled _ubc_is_mapped_writable _ubc_setsize_ex _ubc_upl_range_needed +_upl_get_size _vfs_context_current _vfs_context_issuser _vfs_context_kernel @@ -514,7 +530,6 @@ _vnode_getname_printable _vnode_getfromfd _vnode_isautocandidate _vnode_isfastdevicecandidate -_vnode_isnamedstream _vnode_putname_printable _vnode_setautocandidate _vnode_setdirty @@ -567,6 +582,8 @@ _qf_put _dqfileinit _dqreclaim _zalloc +_zalloc_noblock +_zdestroy _zfree _zinit _zone_change diff --git a/config/Private.x86_64.exports b/config/Private.x86_64.exports index bfe836f99..5341bdbfe 100644 --- a/config/Private.x86_64.exports +++ b/config/Private.x86_64.exports @@ -1,4 +1,6 @@ _IOGetBootKeyStoreData +_IOGetAPFSKeyStoreData +_IOSetAPFSKeyStoreData _SHA256_Final _SHA256_Init _SHA256_Update @@ -52,3 +54,13 @@ _PE_reboot_on_panic _file_vnode _proc_ucred _suser + +#For copyout_shim private KPI +_cos_kernel_unslide +_cos_kernel_reslide +_register_copyout_shim + +#Allow kexts introspect the kernel's layout in memory +_getsegdatafromheader +_getsegbynamefromheader +__mh_execute_header diff --git a/config/System.kext/PlugIns/Kasan.kext/Info.plist b/config/System.kext/PlugIns/Kasan.kext/Info.plist new file mode 100644 index 000000000..69d83ce43 --- /dev/null +++ b/config/System.kext/PlugIns/Kasan.kext/Info.plist @@ -0,0 +1,34 @@ + + + + + CFBundleDevelopmentRegion + English + CFBundleExecutable + Kasan + CFBundleGetInfoString + Kasan Pseudoextension, Apple Computer Inc, ###KERNEL_VERSION_LONG### + CFBundleIdentifier + com.apple.kpi.kasan + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Kasan Pseudoextension + CFBundlePackageType + KEXT + CFBundleShortVersionString + ###KERNEL_VERSION_SHORT### + CFBundleSignature + ???? + CFBundleVersion + ###KERNEL_VERSION_LONG### + OSBundleCompatibleVersion + 8.0.0b1 + OSBundleRequired + Root + OSKernelResource + + OSBundleAllowUserLoad + + + diff --git a/config/Unsupported.arm.exports b/config/Unsupported.arm.exports new file mode 100644 index 000000000..6f33928e1 --- /dev/null +++ b/config/Unsupported.arm.exports @@ -0,0 +1,24 @@ +__ZN9IODTNVRAM17getOWVariableInfoEmPPK8OSSymbolPmS4_ +__ZN9IODTNVRAM19convertObjectToPropEPhPmPK8OSSymbolP8OSObject +__ZN9IODTNVRAM19convertPropToObjectEPhmS0_mPPK8OSSymbolPP8OSObject +__ZN9IODTNVRAM19searchNVRAMPropertyEP17IONVRAMDescriptorPm +__ZN9IODTNVRAM19unescapeBytesToDataEPKhm +_bsd_set_dependency_capable +_clock_get_system_value +_kdp_register_callout +_kdp_set_ip_and_mac_addresses +_logwakeup +_mach_msg_rpc_from_kernel +_mach_msg_send_from_kernel_with_options:_mach_msg_send_from_kernel_with_options_legacy +_ml_stack_remaining +_serial_getc +_serial_init +_serial_putc +_text_crypter_create_hook_set +_vm_map_copyout +_ml_get_cpu_count +_ml_get_boot_cpu_number +_ml_get_cpu_number +_ml_get_max_cpu_number +_ml_dbgwrap_halt_cpu_with_state +_vm_map:_vm_map_external diff --git a/config/Unsupported.arm64.exports b/config/Unsupported.arm64.exports new file mode 100644 index 000000000..38e1a8c0e --- /dev/null +++ b/config/Unsupported.arm64.exports @@ -0,0 +1,40 @@ +__ZN9IODTNVRAM17getOWVariableInfoEjPPK8OSSymbolPjS4_ +__ZN9IODTNVRAM19convertObjectToPropEPhPjPK8OSSymbolP8OSObject +__ZN9IODTNVRAM19convertPropToObjectEPhjS0_jPPK8OSSymbolPP8OSObject +__ZN9IODTNVRAM19searchNVRAMPropertyEP17IONVRAMDescriptorPj +__ZN9IODTNVRAM19unescapeBytesToDataEPKhj +_bsd_set_dependency_capable +_kdp_register_callout +_kdp_set_ip_and_mac_addresses +_logwakeup +_ml_stack_remaining +_serial_getc +_serial_init +_serial_putc +_text_crypter_create_hook_set +_vm_map_copyout +_kpc_register_pm_handler +_kpc_reserve_pm_counters +_kpc_release_pm_counters +_kpc_pm_acknowledge +_kpc_get_running +_kpc_set_running +_kpc_get_cpu_counters +_kpc_get_shadow_counters +_kpc_get_config +_kpc_set_config +_kpc_get_period +_kpc_set_period +_kpc_get_actionid +_kpc_set_actionid +_ml_cpu_signal +_ml_cpu_signal_deferred +_ml_cpu_signal_retract +_ml_get_cpu_count +_ml_get_boot_cpu_number +_ml_get_cpu_number +_ml_get_max_cpu_number +_ml_lockdown_handler_register +_ml_dbgwrap_halt_cpu_with_state +_vm_map:_vm_map_external + diff --git a/config/Unsupported.exports b/config/Unsupported.exports index 70375325a..9840aa96c 100644 --- a/config/Unsupported.exports +++ b/config/Unsupported.exports @@ -82,6 +82,12 @@ _host_get_exception_ports _host_priv_self _hz _ipc_kernel_map +_iflt_attach_internal +_ifnet_allocate_internal +_ifnet_set_fastlane_capable +_ifnet_get_fastlane_capable +_ipf_addv4_internal +_ipf_addv6_internal _kalloc:_kalloc_external _kauth_cred_issuser _kauth_cred_label_update @@ -151,8 +157,11 @@ _putc _rc4_crypt _rc4_init _securelevel +_sflt_register_internal _sha1_hardware_hook _sleep +_sock_accept_internal +_sock_socket_internal _stack_privilege _task_get_special_port _task_resume @@ -164,9 +173,8 @@ _tsleep _ubc_cs_blob_get _vfs_context_current _vfs_update_vfsstat -_vm_allocate +_vm_allocate:_vm_allocate_external _vm_deallocate -_vm_map _vm_map_deallocate _vm_map_unwire _vm_map_wire:_vm_map_wire_external @@ -174,7 +182,6 @@ _set_vm_privilege _vm_protect _vm_region _vm_region_object_create -_vnode_isnamedstream _vnode_tag _vnop_getnamedstream_desc _vnop_kqfilt_add_desc diff --git a/config/Unsupported.x86_64.exports b/config/Unsupported.x86_64.exports index 0f3ed92d1..57c5bb71e 100644 --- a/config/Unsupported.x86_64.exports +++ b/config/Unsupported.x86_64.exports @@ -18,6 +18,7 @@ _dsmos_page_transform_hook _gPEEFIRuntimeServices _gPEEFISystemTable _hibernate_vm_lock +_hibernate_vm_lock_end _hibernate_vm_unlock _kdp_register_callout _kdp_set_ip_and_mac_addresses @@ -42,4 +43,5 @@ _sock_retain _tmrCvt _tsc_get_info _PE_state +_vm_map diff --git a/iokit/IOKit/IOCPU.h b/iokit/IOKit/IOCPU.h index a9ae2e605..25d2398de 100644 --- a/iokit/IOKit/IOCPU.h +++ b/iokit/IOKit/IOCPU.h @@ -51,10 +51,6 @@ enum { kIOCPUStateCount }; -class IOCPUInterruptController; - -extern IOCPUInterruptController *gIOCPUInterruptController; - class IOCPU : public IOService { OSDeclareAbstractStructors(IOCPU); @@ -76,8 +72,6 @@ protected: virtual void setCPUState(UInt32 cpuState); public: - static void initCPUs(void); - virtual bool start(IOService *provider) APPLE_KEXT_OVERRIDE; virtual OSObject *getProperty(const OSSymbol *aKey) const APPLE_KEXT_OVERRIDE; virtual bool setProperty(const OSSymbol *aKey, OSObject *anObject) APPLE_KEXT_OVERRIDE; @@ -116,6 +110,7 @@ extern "C" kern_return_t IOCPURunPlatformQuiesceActions(void); extern "C" kern_return_t IOCPURunPlatformActiveActions(void); extern "C" kern_return_t IOCPURunPlatformHaltRestartActions(uint32_t message); extern "C" kern_return_t IOCPURunPlatformPanicActions(uint32_t message); +extern "C" kern_return_t IOCPURunPlatformPanicSyncAction(void *addr, size_t len); class IOCPUInterruptController : public IOInterruptController { @@ -126,8 +121,8 @@ private: protected: int numCPUs; - IOCPU **cpus; - + int numSources; + struct ExpansionData { }; ExpansionData *reserved; @@ -152,7 +147,8 @@ public: virtual IOReturn handleInterrupt(void *refCon, IOService *nub, int source) APPLE_KEXT_OVERRIDE; - OSMetaClassDeclareReservedUnused(IOCPUInterruptController, 0); + virtual IOReturn initCPUInterruptController(int sources, int cpus); + OSMetaClassDeclareReservedUnused(IOCPUInterruptController, 1); OSMetaClassDeclareReservedUnused(IOCPUInterruptController, 2); OSMetaClassDeclareReservedUnused(IOCPUInterruptController, 3); diff --git a/iokit/IOKit/IODeviceTreeSupport.h b/iokit/IOKit/IODeviceTreeSupport.h index 531202f41..b10c5553c 100644 --- a/iokit/IOKit/IODeviceTreeSupport.h +++ b/iokit/IOKit/IODeviceTreeSupport.h @@ -71,8 +71,13 @@ enum { OSCollectionIterator * IODTFindMatchingEntries( IORegistryEntry * from, IOOptionBits options, const char * keys ); +#if !defined(__arm64__) typedef SInt32 (*IODTCompareAddressCellFunc) (UInt32 cellCount, UInt32 left[], UInt32 right[]); +#else +typedef SInt64 (*IODTCompareAddressCellFunc) + (UInt32 cellCount, UInt32 left[], UInt32 right[]); +#endif typedef void (*IODTNVLocationFunc) (IORegistryEntry * entry, diff --git a/iokit/IOKit/IOEventSource.h b/iokit/IOKit/IOEventSource.h index 66ee9054a..44502a12e 100644 --- a/iokit/IOKit/IOEventSource.h +++ b/iokit/IOKit/IOEventSource.h @@ -122,6 +122,21 @@ protected: Is this event source enabled to deliver requests to the work-loop. */ bool enabled; +#if XNU_KERNEL_PRIVATE + + enum + { + kPassive = 0x0001, + kActive = 0x0002, + }; + uint8_t eventSourceReserved1[1]; + uint16_t flags; +#if __LP64__ + uint8_t eventSourceReserved2[4]; +#endif /* __LP64__ */ + +#endif /* XNU_KERNEL_PRIVATE */ + /*! @var workLoop What is the work-loop for this event source. */ IOWorkLoop *workLoop; @@ -147,9 +162,7 @@ protected: @abstract Primary initialiser for the IOEventSource class. @param owner Owner of this instance of an event source. Used as the first parameter -of the action callout. Owner will generally be an OSObject it doesn't have to -be as no member functions will be called directly in it. It can just be a -refcon for a client routine. +of the action callout. Owner must be an OSObject. @param action Pointer to C call out function. Action is a pointer to a C function that gets called when this event source has outstanding work. It will usually diff --git a/iokit/IOKit/IOHibernatePrivate.h b/iokit/IOKit/IOHibernatePrivate.h index 5fc9cf13e..cf8aa46df 100644 --- a/iokit/IOKit/IOHibernatePrivate.h +++ b/iokit/IOKit/IOHibernatePrivate.h @@ -35,6 +35,9 @@ extern "C" { #ifdef KERNEL #include #include +#include + +extern int kdb_printf(const char *format, ...) __printflike(1,2); #endif #ifndef __IOKIT_IOHIBERNATEPRIVATE_H @@ -308,8 +311,7 @@ void IOCloseDebugDataFile(); IOReturn IOHibernateIOKitSleep(void); IOReturn IOHibernateSystemHasSlept(void); IOReturn IOHibernateSystemWake(void); -IOReturn IOHibernateSystemPostWake(void); -void IOHibernateSystemPostWakeTrim(void * p1, void * p2); +IOReturn IOHibernateSystemPostWake(bool now); uint32_t IOHibernateWasScreenLocked(void); void IOHibernateSetScreenLocked(uint32_t lockState); void IOHibernateSetWakeCapabilities(uint32_t capability); @@ -362,6 +364,10 @@ void hibernate_vm_lock(void); void hibernate_vm_unlock(void); +void +hibernate_vm_lock_end(void); +boolean_t +hibernate_vm_locks_are_safe(void); // mark pages not to be saved, based on VM system accounting void @@ -435,11 +441,15 @@ extern uint8_t gIOHibernateRestoreStack[]; extern uint8_t gIOHibernateRestoreStackEnd[]; extern IOHibernateImageHeader * gIOHibernateCurrentHeader; +#define HIBLOGFROMPANIC(fmt, args...) \ + { if (kernel_debugger_entry_count) { kdb_printf(fmt, ## args); } } + #define HIBLOG(fmt, args...) \ - { kprintf(fmt, ## args); printf(fmt, ## args); } + { if (kernel_debugger_entry_count) { kdb_printf(fmt, ## args); } else { kprintf(fmt, ## args); printf(fmt, ## args); } } #define HIBPRINT(fmt, args...) \ - { kprintf(fmt, ## args); } + { if (kernel_debugger_entry_count) { kdb_printf(fmt, ## args); } else { kprintf(fmt, ## args); } } + #endif /* KERNEL */ diff --git a/iokit/IOKit/IOInterruptAccounting.h b/iokit/IOKit/IOInterruptAccounting.h index 7e03f6bd5..d2715d0b0 100644 --- a/iokit/IOKit/IOInterruptAccounting.h +++ b/iokit/IOKit/IOInterruptAccounting.h @@ -29,6 +29,8 @@ #ifndef __IOKIT_IOINTERRUPTACCOUNTING_H #define __IOKIT_IOINTERRUPTACCOUNTING_H +#include + /* * This header contains definitions that will be needed by userspace clients of the interrupt accounting * mechanisms. diff --git a/iokit/IOKit/IOInterruptController.h b/iokit/IOKit/IOInterruptController.h index d389a79e3..0a634b056 100644 --- a/iokit/IOKit/IOInterruptController.h +++ b/iokit/IOKit/IOInterruptController.h @@ -109,6 +109,16 @@ public: OSMetaClassDeclareReservedUnused(IOInterruptController, 3); OSMetaClassDeclareReservedUnused(IOInterruptController, 4); OSMetaClassDeclareReservedUnused(IOInterruptController, 5); + +public: + // Generic methods (not to be overriden). + + void timeStampSpuriousInterrupt(void); + void timeStampInterruptHandlerStart(IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector); + void timeStampInterruptHandlerEnd(IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector); + +private: + void timeStampInterruptHandlerInternal(bool isStart, IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector); }; diff --git a/iokit/IOKit/IOKernelReportStructs.h b/iokit/IOKit/IOKernelReportStructs.h index 4018f7995..b15a62527 100644 --- a/iokit/IOKit/IOKernelReportStructs.h +++ b/iokit/IOKit/IOKernelReportStructs.h @@ -1,8 +1,8 @@ /* * Copyright (c) 2012-2014 Apple Computer, Inc. All Rights Reserved. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -40,12 +40,10 @@ extern "C" { #endif -#define kIOReportAPIVersion 28 - // Drivers participating in IOReporting can advertise channels by // publishing properties in the I/O Kit registry. Various helper // mechanisms exist to produce correctly-formatted legends. -// 12836893 tracks declaring channels in user space. +// 12836893 tracks advertising channels in user space. #define kIOReportLegendPublicKey "IOReportLegendPublic" // bool #define kIOReportLegendKey "IOReportLegend" // arr #define kIOReportLegendChannelsKey "IOReportChannels" // arr @@ -60,203 +58,7 @@ extern "C" { #define kIOReportChannelIDIdx 0 // required #define kIOReportChannelTypeIdx 1 // required #define kIOReportChannelNameIdx 2 // optional - -// We are currently (internally) limited to 15 (broad!) categories. - - -/* - Units / Scaling Factors - - 1. Implementation Details - 2. Unit Constants (kIOReportUnit...) for clients - - Please file radars if you need more units (IOReporting | X) -*/ - -// 1. Implementation Details -// We are likely to someday support IOReporting data as stored binary data. -// Don't change existing values lest that data become unreadable. - -typedef uint64_t IOReportUnits; -#define __IOR_MAKEUNIT(quantity, scale) \ - (((IOReportUnits)quantity << 56) | (uint64_t)scale) -#define IOREPORT_GETUNIT_QUANTITY(unit) \ - ((IOReportQuantity)((uint64_t)unit >> 56) & 0xff) -#define IOREPORT_GETUNIT_SCALE(unit) \ - ((IOReportScaleFactor)unit & 0x00ffffffffffffff) - -// 8b quantity + 32b const + 8b * 2^10 + 8b * 2^n + 8b cardinal + 8b unused -typedef uint8_t IOReportQuantity; // SI "quantity" is what's measured -typedef uint64_t IOReportScaleFactor; - -// See for a list -// of quantities and their symbols. -enum { - // used by state reports, etc - kIOReportQuantityUndefined = 0, - - kIOReportQuantityTime = 1, // Seconds - kIOReportQuantityPower = 2, // Watts - kIOReportQuantityEnergy = 3, // Joules - kIOReportQuantityCurrent = 4, // Amperes - kIOReportQuantityVoltage = 5, // Volts - kIOReportQuantityCapacitance = 6, // Farad - kIOReportQuantityInductance = 7, // Henry - kIOReportQuantityFrequency = 8, // Hertz - kIOReportQuantityData = 9, // bits/bytes (see scale) - kIOReportQuantityTemperature = 10, // Celsius (not Kelvin :) - - kIOReportQuantityEventCount = 100, - kIOReportQuantityPacketCount = 101 -}; - - -/* A number of units end up with both IEC (2^n) and SI (10^n) scale factors. - For example, the "MB" of a 1.44 MB floppy or a 1024MHz clock. We - thus support separate 2^n and 10^n factors. The exponent encoding - scheme is modeled loosely on single-precision IEEE 754. - */ -#define kIOReportScaleConstMask 0x000000007fffffff // constant ("uint31") -#define kIOReportScaleOneOver (1LL << 31) // 1/constant -#define kIOReportExpBase (-127) // support base^(-n) -#define kIOReportExpZeroOffset -(kIOReportExpBase) // max exponent = 128 -#define kIOReportScaleSIShift 32 // * 10^n -#define kIOReportScaleSIMask 0x000000ff00000000 -#define kIOReportScaleIECShift 40 // * 2^n -#define kIOReportScaleIECMask 0x0000ff0000000000 -#define kIOReportCardinalShift 48 // placeholders -#define kIOReportCardinalMask 0x00ff000000000000 - -/* - Scales are described as a factor times unity: - 1ms = kIOReportScaleMilli * s - - A value expressed in a scaled unit can be scaled to unity via - multiplication by the constant: - 100ms * kIOReportScaleMilli [1e-3] = 0.1s. -*/ - -// SI / decimal -#define kIOReportScalePico ((-12LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleNano ((-9LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleMicro ((-6LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleMilli ((-3LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleUnity 0 // 10^0 = 2^0 = 1 -// unity = 0 is a special case for which we give up exp = -127 -#define kIOReportScaleKilo ((3LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleMega ((6LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleGiga ((9LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleTera ((12LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) - -// IEC / computer / binary -// It's not clear we'll ever use 2^(-n), but 1..2^~120 should suffice. -#define kIOReportScaleBits kIOReportScaleUnity -#define kIOReportScaleBytes ((3LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -// (bytes have to be added to the exponents up front, can't just OR in) -#define kIOReportScaleKibi ((10LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleKiBytes ((13LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleMebi ((20LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleMiBytes ((23LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleGibi ((30LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleGiBytes ((33LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleTebi ((40LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleTiBytes ((43LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -// can't encode more than 2^125 (keeping bits & bytes inside -126..128) -// Also, IOReportScaleValue() is currently limited internally by uint64_t. - - -// Cardinal values, to be filled in appropriately. -// Add values in increasing order. -#define kIOReportScaleMachHWTicks (1LL << kIOReportCardinalShift) -#define kIOReportScaleHWPageSize (2LL << kIOReportCardinalShift) - -// page scales: 2 pages * 4ikB/page = 8096 bytes -#define kIOReportScale4KiB (4 | kIOReportScaleKiBytes) -#define kIOReportScale8KiB (8 | kIOReportScaleKiBytes) - -// Clock frequencies scales (units add seconds). -// 1 GHz ticks are 1 ns: 1000 ticks * 1e-6 = 1e-3s -// The '1' is a no-op, but allows a custom label. -#define kIOReportScale1GHz (1 | kIOReportScaleNano) -// 24MHz ticks are 1/24 of a microsecond: (1/24 * kIOReportScaleMicro [1e-6])s -// So for example, 240 24Mticks * 1/24 * 1e-6 = .00001s [1e-5]s -#define kIOReportScale24MHz (kIOReportScaleOneOver|24 |kIOReportScaleMicro) - -// --- END: implementation details - -// 2. Units Constants -// --- BEGIN: units constants driver writers might use -#define kIOReportUnitNone __IOR_MAKEUNIT(kIOReportQuantityUndefined, \ - kIOReportScaleUnity) - -#define kIOReportUnit_s __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScaleUnity) -#define kIOReportUnit_ms __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScaleMilli) -#define kIOReportUnit_us __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScaleMicro) -#define kIOReportUnit_ns __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScaleNano) - -#define kIOReportUnit_J __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ - kIOReportScaleUnity) -#define kIOReportUnit_mJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ - kIOReportScaleMilli) -#define kIOReportUnit_uJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ - kIOReportScaleMicro) -#define kIOReportUnit_nJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ - kIOReportScaleNano) -#define kIOReportUnit_pJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ - kIOReportScalePico) - -#define kIOReportUnitHWTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScaleMachHWTicks) -#define kIOReportUnit24MHzTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScale24MHz) -#define kIOReportUnit1GHzTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScale1GHz) - -#define kIOReportUnitBits __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleBits) -#define kIOReportUnitBytes __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleBytes) -#define kIOReportUnit_KiB __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleKiBytes) -#define kIOReportUnit_MiB __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleMiBytes) -#define kIOReportUnit_GiB __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleGiBytes) -#define kIOReportUnit_TiB __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleTiBytes) - -#define kIOReportUnitEvents __IOR_MAKEUNIT(kIOReportQuantityEventCount, \ - kIOReportScaleUnity) - -#define kIOReportUnitPackets __IOR_MAKEUNIT(kIOReportQuantityPacketCount, \ - kIOReportScaleUnity) - -// Please file radars if you need more units (IOReporting | X) - -// --- END: unit constants driver writers might use - /* Histogram Segment Configuration Currently supports 2 types of scaling to compute bucket upper bounds, linear or exponential. @@ -280,7 +82,7 @@ typedef struct { uint64_t variance; uint64_t reserved; } __attribute((packed)) IONormDistReportValues; - + #ifdef __cplusplus } #endif diff --git a/iokit/IOKit/IOKernelReporters.h b/iokit/IOKit/IOKernelReporters.h index 5257ca081..bbde3d817 100644 --- a/iokit/IOKit/IOKernelReporters.h +++ b/iokit/IOKit/IOKernelReporters.h @@ -147,7 +147,7 @@ protected: */ virtual bool init(IOService *reportingService, IOReportChannelType channelType, - IOReportUnits unit); + IOReportUnit unit); public: @@ -736,7 +736,7 @@ private: static IOReportLegendEntry* legendWith(OSArray *channelIDs, OSArray *channelNames, IOReportChannelType channelType, - IOReportUnits unit); + IOReportUnit unit); // protected instance variables (want to get rid of these) protected: @@ -761,7 +761,7 @@ protected: // private instance variables private: - IOReportUnits _unit; + IOReportUnit _unit; int _enabled; // 'enabled' if _enabled > 0 @@ -805,7 +805,7 @@ public: */ static IOSimpleReporter* with(IOService *reportingService, IOReportCategories categories, - IOReportUnits unit); + IOReportUnit unit); /*! @function IOSimpleReporter::setValue @abstract Thread safely set a channel's value @@ -860,7 +860,7 @@ protected: */ virtual bool initWith(IOService *reportingService, IOReportCategories categories, - IOReportUnits unit); + IOReportUnit unit); private: @@ -902,7 +902,7 @@ public: static IOStateReporter* with(IOService *reportingService, IOReportCategories categories, int nstates, - IOReportUnits unit = kIOReportUnitHWTicks); + IOReportUnit unit = kIOReportUnitHWTicks); /*! @function IOStateReporter::setStateID @abstract Assign a non-default ID to a state @@ -1300,7 +1300,7 @@ protected: */ virtual bool initWith(IOService *reportingService, IOReportCategories categories, - int16_t nstates, IOReportUnits unit); + int16_t nstates, IOReportUnit unit); /*! @function IOStateReporter::handleSwapPrepare @@ -1482,7 +1482,7 @@ FIXME: need more explanation of the config IOReportCategories categories, uint64_t channelID, const char *channelName, - IOReportUnits unit, + IOReportUnit unit, int nSegments, IOHistogramSegmentConfig *config); @@ -1558,7 +1558,7 @@ protected: IOReportCategories categories, uint64_t channelID, const OSSymbol *channelName, - IOReportUnits unit, + IOReportUnit unit, int nSegments, IOHistogramSegmentConfig *config); diff --git a/iokit/IOKit/IOKitDebug.h b/iokit/IOKit/IOKitDebug.h index 87467f3c9..a6c64b8eb 100644 --- a/iokit/IOKit/IOKitDebug.h +++ b/iokit/IOKit/IOKitDebug.h @@ -81,7 +81,7 @@ enum { // debug aids - change behaviour kIONoFreeObjects = 0x00100000ULL, - kIOLogSynchronous = 0x00200000ULL, // IOLog completes synchronously +// kIOLogSynchronous = 0x00200000ULL, // IOLog completes synchronously -- obsolete kIOTracking = 0x00400000ULL, kIOWaitQuietPanics = 0x00800000ULL, kIOWaitQuietBeforeRoot = 0x01000000ULL, @@ -90,6 +90,29 @@ enum { _kIODebugTopFlag = 0x8000000000000000ULL // force enum to be 64 bits }; +enum { + kIOKitDebugUserOptions = 0 + | kIOLogAttach + | kIOLogProbe + | kIOLogStart + | kIOLogRegister + | kIOLogMatch + | kIOLogConfig + | kIOLogYield + | kIOLogPower + | kIOLogMapping + | kIOLogCatalogue + | kIOLogTracePower + | kIOLogDebugPower + | kOSLogRegistryMods + | kIOLogPMRootDomain + | kOSRegistryModsMode + | kIOLogHibernate + | kIOSleepWakeWdogOff + | kIOKextSpinDump + | kIOWaitQuietPanics +}; + enum { kIOTraceInterrupts = 0x00000001ULL, // Trace primary interrupts kIOTraceWorkLoops = 0x00000002ULL, // Trace workloop activity @@ -143,7 +166,9 @@ struct IOKitDiagnosticsParameters size_t size; uint64_t value; uint32_t options; - uint32_t reserved[3]; + uint32_t tag; + uint32_t zsize; + uint32_t reserved[8]; }; typedef struct IOKitDiagnosticsParameters IOKitDiagnosticsParameters; @@ -166,7 +191,7 @@ struct IOTrackingCallSiteInfo #define kIOWireTrackingName "IOWire" #define kIOMapTrackingName "IOMap" -#if KERNEL && IOTRACKING +#if XNU_KERNEL_PRIVATE && IOTRACKING struct IOTrackingQueue; struct IOTrackingCallSite; @@ -214,7 +239,7 @@ IOTrackingQueue * IOTrackingQueueAlloc(const char * name, uintptr_t btEntry, size_t allocSize, size_t minCaptureSize, uint32_t type, uint32_t numSiteQs); void IOTrackingQueueFree(IOTrackingQueue * head); -void IOTrackingAdd(IOTrackingQueue * head, IOTracking * mem, size_t size, bool address); +void IOTrackingAdd(IOTrackingQueue * head, IOTracking * mem, size_t size, bool address, vm_tag_t tag); void IOTrackingRemove(IOTrackingQueue * head, IOTracking * mem, size_t size); void IOTrackingAddUser(IOTrackingQueue * queue, IOTrackingUser * mem, vm_size_t size); void IOTrackingRemoveUser(IOTrackingQueue * head, IOTrackingUser * tracking); @@ -231,7 +256,7 @@ extern IOTrackingQueue * gIOMallocTracking; extern IOTrackingQueue * gIOWireTracking; extern IOTrackingQueue * gIOMapTracking; -#endif /* KERNEL && IOTRACKING */ +#endif /* XNU_KERNEL_PRIVATE && IOTRACKING */ enum { diff --git a/iokit/IOKit/IOKitKeys.h b/iokit/IOKit/IOKitKeys.h index 240ec58e9..44ed11807 100644 --- a/iokit/IOKit/IOKitKeys.h +++ b/iokit/IOKit/IOKitKeys.h @@ -104,6 +104,7 @@ #define kIOMatchedNotification "IOServiceMatched" #define kIOFirstMatchNotification "IOServiceFirstMatch" #define kIOTerminatedNotification "IOServiceTerminate" +#define kIOWillTerminateNotification "IOServiceWillTerminate" // IOService interest notification types #define kIOGeneralInterest "IOGeneralInterest" diff --git a/iokit/IOKit/IOLib.h b/iokit/IOKit/IOLib.h index 4a8ae78d6..ce3689190 100644 --- a/iokit/IOKit/IOLib.h +++ b/iokit/IOKit/IOLib.h @@ -80,7 +80,7 @@ typedef void (*IOThreadFunc)(void *argument); @param size Size of the memory requested. @result Pointer to the allocated memory, or zero on failure. */ -void * IOMalloc(vm_size_t size); +void * IOMalloc(vm_size_t size) __attribute__((alloc_size(1))); /*! @function IOFree @abstract Frees memory allocated with IOMalloc. @@ -99,7 +99,7 @@ void IOFree(void * address, vm_size_t size); @param alignment Byte count of the alignment for the memory. For example, pass 256 to get memory allocated at an address with bit 0-7 zero. @result Pointer to the allocated memory, or zero on failure. */ -void * IOMallocAligned(vm_size_t size, vm_offset_t alignment); +void * IOMallocAligned(vm_size_t size, vm_offset_t alignment) __attribute__((alloc_size(1))); /*! @function IOFreeAligned @abstract Frees memory allocated with IOMallocAligned. @@ -118,7 +118,7 @@ void IOFreeAligned(void * address, vm_size_t size); @result Virtual address of the allocated memory, or zero on failure. */ void * IOMallocContiguous(vm_size_t size, vm_size_t alignment, - IOPhysicalAddress * physicalAddress) __attribute__((deprecated)); + IOPhysicalAddress * physicalAddress) __attribute__((deprecated)) __attribute__((alloc_size(1))); /*! @function IOFreeContiguous @abstract Deprecated - use IOBufferMemoryDescriptor. Frees memory allocated with IOMallocContiguous. @@ -136,7 +136,7 @@ void IOFreeContiguous(void * address, vm_size_t size) __attribute__((deprecate @param alignment Byte count of the alignment for the memory. For example, pass 256 to get memory allocated at an address with bits 0-7 zero. @result Pointer to the allocated memory, or zero on failure. */ -void * IOMallocPageable(vm_size_t size, vm_size_t alignment); +void * IOMallocPageable(vm_size_t size, vm_size_t alignment) __attribute__((alloc_size(1))); /*! @function IOFreePageable @abstract Frees memory allocated with IOMallocPageable. @@ -309,7 +309,8 @@ __attribute__((format(printf, 1, 2))); @param format A printf() style format string (see printf(3) documentation). @param ap stdarg(3) style variable arguments. */ -void IOLogv(const char *format, va_list ap); +void IOLogv(const char *format, va_list ap) +__attribute__((format(printf, 1, 0))); #ifndef _FN_KPRINTF #define _FN_KPRINTF diff --git a/iokit/IOKit/IOMapper.h b/iokit/IOKit/IOMapper.h index f63f5463a..4fe1ccf43 100644 --- a/iokit/IOKit/IOMapper.h +++ b/iokit/IOKit/IOMapper.h @@ -59,6 +59,7 @@ class IOMapper : public IOService // Give the platform expert access to setMapperRequired(); friend class IOPlatformExpert; friend class IOMemoryDescriptor; + friend class IOGeneralMemoryDescriptor; private: enum SystemMapperState { @@ -69,7 +70,8 @@ private: }; protected: #ifdef XNU_KERNEL_PRIVATE - uint64_t __reservedA[7]; + uint64_t __reservedA[6]; + kern_allocation_name_t fAllocName; uint32_t __reservedB; uint32_t fPageSize; #else diff --git a/iokit/IOKit/IOMemoryDescriptor.h b/iokit/IOKit/IOMemoryDescriptor.h index c284aaa12..0572ef51d 100644 --- a/iokit/IOKit/IOMemoryDescriptor.h +++ b/iokit/IOKit/IOMemoryDescriptor.h @@ -73,18 +73,6 @@ enum IODirection kIODirectionCompleteWithDataValid = 0x00000080, }; - -#if XNU_KERNEL_PRIVATE -enum -{ - // prepare/complete() notify DMA command active - kIODirectionDMACommand = 0x00000100, - kIODirectionDMACommandMask = 0x0001FE00, - kIODirectionDMACommandShift = 9, -}; -#endif - - #ifdef __LP64__ typedef IOOptionBits IODirection; #endif /* __LP64__ */ @@ -124,6 +112,7 @@ enum { #ifdef XNU_KERNEL_PRIVATE kIOMemoryMapCopyOnWrite = 0x00020000, #endif + kIOMemoryRemote = 0x00040000, kIOMemoryThreadSafe = 0x00100000, // Shared with Buffer MD kIOMemoryClearEncrypt = 0x00200000, // Shared with Buffer MD kIOMemoryUseReserve = 0x00800000, // Shared with Buffer MD @@ -217,6 +206,7 @@ enum { kIOMDSetDMAInactive = kIOMDDMAActive, kIOMDAddDMAMapSpec = 0x04000000, kIOMDDMAMap = 0x05000000, + kIOMDDMAUnmap = 0x06000000, kIOMDDMACommandOperationMask = 0xFF000000, }; struct IOMDDMACharacteristics { @@ -286,17 +276,22 @@ protected: #ifdef XNU_KERNEL_PRIVATE public: struct IOMemoryReference * _memRef; + vm_tag_t _kernelTag; + vm_tag_t _userTag; + int16_t _dmaReferences; + uint16_t _internalFlags; + kern_allocation_name_t _mapName; protected: -#else - void * __iomd_reserved5; -#endif +#else /* XNU_KERNEL_PRIVATE */ + void * __iomd_reserved5; + uint16_t __iomd_reserved1[4]; + uintptr_t __iomd_reserved2; +#endif /* XNU_KERNEL_PRIVATE */ -#ifdef __LP64__ - uint64_t __iomd_reserved1; - uint64_t __iomd_reserved2; - uint64_t __iomd_reserved3; - uint64_t __iomd_reserved4; -#else /* !__LP64__ */ + uintptr_t __iomd_reserved3; + uintptr_t __iomd_reserved4; + +#ifndef __LP64__ IODirection _direction; /* use _flags instead */ #endif /* !__LP64__ */ IOByteCount _length; /* length of all ranges */ @@ -399,6 +394,16 @@ typedef IOOptionBits DMACommandOps; uint64_t length, uint64_t * mapAddress, uint64_t * mapLength); + IOReturn dmaUnmap( + IOMapper * mapper, + IODMACommand * command, + uint64_t offset, + uint64_t mapAddress, + uint64_t mapLength); + void dmaMapRecord( + IOMapper * mapper, + IODMACommand * command, + uint64_t mapLength); void setVMTags(vm_tag_t kernelTag, vm_tag_t userTag); vm_tag_t getVMTag(vm_map_t map); @@ -617,6 +622,13 @@ public: virtual IOOptionBits getTag( void ); +/*! @function getFlags + @abstract Accessor to the retrieve the options the memory descriptor was created with. + @discussion Accessor to the retrieve the options the memory descriptor was created with, and flags with its state. These bits are defined by the kIOMemory* enum. + @result The flags bitfield. */ + + uint64_t getFlags(void); + /*! @function readBytes @abstract Copy data from the memory descriptor's buffer to the specified buffer. @discussion This method copies data from the memory descriptor's memory at the given offset, to the caller's buffer. The memory descriptor MUST have the kIODirectionOut direcction bit set and be prepared. kIODirectionOut means that this memory descriptor will be output to an external device, so readBytes is used to get memory into a local buffer for a PIO transfer to the device. diff --git a/iokit/IOKit/IOPlatformExpert.h b/iokit/IOKit/IOPlatformExpert.h index 52e1c366b..936809c7c 100644 --- a/iokit/IOKit/IOPlatformExpert.h +++ b/iokit/IOKit/IOPlatformExpert.h @@ -58,13 +58,16 @@ enum { kPEUPSDelayHaltCPU, kPEPanicRestartCPU, kPEPanicSync, - kPEPagingOff + kPEPagingOff, + kPEPanicBegin, + kPEPanicEnd }; extern int (*PE_halt_restart)(unsigned int type); extern int PEHaltRestart(unsigned int type); // Save the Panic Info. Returns the number of bytes saved. extern UInt32 PESavePanicInfo(UInt8 *buffer, UInt32 length); +extern void PESavePanicInfoAction(void *buffer, size_t length); extern long PEGetGMTTimeOfDay( void ); extern void PESetGMTTimeOfDay( long secs ); diff --git a/iokit/IOKit/IOPolledInterface.h b/iokit/IOKit/IOPolledInterface.h index 3b3a663ec..cd1ba5986 100644 --- a/iokit/IOKit/IOPolledInterface.h +++ b/iokit/IOKit/IOPolledInterface.h @@ -54,7 +54,8 @@ enum enum { kIOPolledWrite = 1, - kIOPolledRead = 2 + kIOPolledRead = 2, + kIOPolledFlush = 3 }; typedef void (*IOPolledCompletionAction)( void * target, @@ -203,6 +204,8 @@ extern __C IOReturn IOPolledFileRead(IOPolledFileIOVars * vars, uint8_t * bytes, IOByteCount size, IOPolledFileCryptVars * cryptvars); +extern __C IOReturn IOPolledFileFlush(IOPolledFileIOVars * vars); + extern __C IOReturn IOPolledFilePollersOpen(IOPolledFileIOVars * vars, uint32_t state, bool abortable); extern __C IOReturn IOPolledFilePollersClose(IOPolledFileIOVars * vars, uint32_t state); diff --git a/iokit/IOKit/IOReportTypes.h b/iokit/IOKit/IOReportTypes.h index 8cd6e7328..3d65c3480 100644 --- a/iokit/IOKit/IOReportTypes.h +++ b/iokit/IOKit/IOReportTypes.h @@ -1,8 +1,8 @@ /* * Copyright (c) 2012-2014 Apple Computer, Inc. All Rights Reserved. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -60,7 +60,7 @@ extern "C" { needed. Groups and subgroups are a more extensible mechanism for aggregating channels produced by different drivers. */ -typedef uint16_t IOReportCategories; +typedef uint16_t IOReportCategories; #define kIOReportCategoryPower (1 << 1) // and energy #define kIOReportCategoryTraffic (1 << 2) // I/O at any level #define kIOReportCategoryPerformance (1 << 3) // e.g. cycles/byte @@ -69,9 +69,8 @@ typedef uint16_t IOReportCategories; #define kIOReportCategoryField (1 << 8) // consider logging // future categories TBD -#define kIOReportCategoryInterrupt (1 << 14) // TBR: 15850269 #define kIOReportCategoryDebug (1 << 15) -#define kIOReportInvalidCategory UINT16_MAX +#define kIOReportInvalidCategory UINT16_MAX // IOReportChannelType.report_format @@ -201,6 +200,210 @@ typedef struct { IOReportElementValues values; } __attribute((packed)) IOReportElement; + + +/* + IOReporting unit type and constants +*/ + +// 1. Mechanism + +// Assume encoded units could be stored in binary format: don't +// change existing values. + +typedef uint64_t IOReportUnit; +typedef uint64_t IOReportUnits; // deprecated typo, please switch +#define __IOR_MAKEUNIT(quantity, scale) \ + (((IOReportUnit)quantity << 56) | (uint64_t)scale) +#define IOREPORT_GETUNIT_QUANTITY(unit) \ + ((IOReportQuantity)((uint64_t)unit >> 56) & 0xff) +#define IOREPORT_GETUNIT_SCALE(unit) \ + ((IOReportScaleFactor)unit & 0x00ffffffffffffff) + +// 8b quantity ID | 32b const val + 8b*2^10 + 8b*2^n | 8b cardinal | 8b unused +typedef uint8_t IOReportQuantity; // SI "quantity" is what's measured +typedef uint64_t IOReportScaleFactor; + +// See for a list +// of quantities and their symbols. +enum { + // used by state reports, etc + kIOReportQuantityUndefined = 0, + + kIOReportQuantityTime = 1, // Seconds + kIOReportQuantityPower = 2, // Watts + kIOReportQuantityEnergy = 3, // Joules + kIOReportQuantityCurrent = 4, // Amperes + kIOReportQuantityVoltage = 5, // Volts + kIOReportQuantityCapacitance = 6, // Farad + kIOReportQuantityInductance = 7, // Henry + kIOReportQuantityFrequency = 8, // Hertz + kIOReportQuantityData = 9, // bits/bytes (see scale) + kIOReportQuantityTemperature = 10, // Celsius (not Kelvin :) + + kIOReportQuantityEventCount = 100, + kIOReportQuantityPacketCount = 101, + kIOReportQuantityCPUInstrs = 102 +}; + + +/* A number of units end up with both IEC (2^n) and SI (10^n) scale factors. + For example, the "MB" of a 1.44 MB floppy or a 1024MHz clock. We + thus support separate 2^n and 10^n factors. The exponent encoding + scheme is modeled loosely on single-precision IEEE 754. + */ +#define kIOReportScaleConstMask 0x000000007fffffff // constant ("uint31") +#define kIOReportScaleOneOver (1LL << 31) // 1/constant +#define kIOReportExpBase (-127) // support base^(-n) +#define kIOReportExpZeroOffset -(kIOReportExpBase) // max exponent = 128 +#define kIOReportScaleSIShift 32 // * 10^n +#define kIOReportScaleSIMask 0x000000ff00000000 +#define kIOReportScaleIECShift 40 // * 2^n +#define kIOReportScaleIECMask 0x0000ff0000000000 +#define kIOReportCardinalShift 48 // placeholders +#define kIOReportCardinalMask 0x00ff000000000000 + + +/* + Scales are described as a factor times unity: + 1ms = kIOReportScaleMilli * s + + A value expressed in a scaled unit can be scaled to unity via + multiplication by the constant: + 100ms * kIOReportScaleMilli [1e-3] = 0.1s. +*/ + +// SI / decimal +#define kIOReportScalePico ((-12LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleNano ((-9LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleMicro ((-6LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleMilli ((-3LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleUnity 0 // 10^0 = 2^0 = 1 +// unity = 0 is a special case for which we give up exp = -127 +#define kIOReportScaleKilo ((3LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleMega ((6LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleGiga ((9LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleTera ((12LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) + +// IEC / computer / binary +// It's not clear we'll ever use 2^(-n), but 1..2^~120 should suffice. +#define kIOReportScaleBits kIOReportScaleUnity +#define kIOReportScaleBytes ((3LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +// (bytes have to be added to the exponents up front, can't just OR in) +#define kIOReportScaleKibi ((10LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleKiBytes ((13LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleMebi ((20LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleMiBytes ((23LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleGibi ((30LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleGiBytes ((33LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleTebi ((40LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleTiBytes ((43LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +// can't encode more than 2^125 (keeping bits & bytes inside -126..128) +// Also, IOReportScaleValue() is currently limited internally by uint64_t. + + +// Cardinal values, to be filled in appropriately. +// Add values in increasing order. +#define kIOReportScaleMachHWTicks (1LL << kIOReportCardinalShift) +#define kIOReportScaleHWPageSize (2LL << kIOReportCardinalShift) + +// page scales: 2 pages * 4ikB/page = 8096 bytes +#define kIOReportScale4KiB (4 | kIOReportScaleKiBytes) +#define kIOReportScale8KiB (8 | kIOReportScaleKiBytes) +#define kIOReportScale16KiB (16 | kIOReportScaleKiBytes) + +// Clock frequency scales (units add seconds). +// 1 GHz ticks are 1 ns: 1000 ticks * 1e-6 = 1e-3s +// This '1' is a no-op for scaling, but allows a custom label. +#define kIOReportScale1GHz (1 | kIOReportScaleNano) +// 24MHz ticks are 1/24 of a microsecond: (1/24 * kIOReportScaleMicro [1e-6])s +// So for example, 240 24Mticks * 1/24 * 1e-6 = .00001s [1e-5]s +#define kIOReportScale24MHz (kIOReportScaleOneOver|24 |kIOReportScaleMicro) + +// --- END: units mechanism + + +// 2. Unit constants +#define kIOReportUnitNone __IOR_MAKEUNIT(kIOReportQuantityUndefined, \ + kIOReportScaleUnity) + +#define kIOReportUnit_s __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScaleUnity) +#define kIOReportUnit_ms __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScaleMilli) +#define kIOReportUnit_us __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScaleMicro) +#define kIOReportUnit_ns __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScaleNano) + +#define kIOReportUnit_J __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ + kIOReportScaleUnity) +#define kIOReportUnit_mJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ + kIOReportScaleMilli) +#define kIOReportUnit_uJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ + kIOReportScaleMicro) +#define kIOReportUnit_nJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ + kIOReportScaleNano) +#define kIOReportUnit_pJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ + kIOReportScalePico) + +#define kIOReportUnitHWTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScaleMachHWTicks) +#define kIOReportUnit24MHzTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScale24MHz) +#define kIOReportUnit1GHzTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScale1GHz) + +#define kIOReportUnitBits __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleBits) +#define kIOReportUnitBytes __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleBytes) +#define kIOReportUnit_KiB __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleKiBytes) +#define kIOReportUnit_MiB __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleMiBytes) +#define kIOReportUnit_GiB __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleGiBytes) +#define kIOReportUnit_TiB __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleTiBytes) + +#define kIOReportUnitEvents __IOR_MAKEUNIT(kIOReportQuantityEventCount, \ + kIOReportScaleUnity) + +#define kIOReportUnitPackets __IOR_MAKEUNIT(kIOReportQuantityPacketCount, \ + kIOReportScaleUnity) + +#define kIOReportUnitInstrs __IOR_MAKEUNIT(kIOReportQuantityCPUInstrs, \ + kIOReportScaleUnity) +#define kIOReportUnit_KI __IOR_MAKEUNIT(kIOReportQuantityCPUInstrs, \ + kIOReportScaleKilo) +#define kIOReportUnit_MI __IOR_MAKEUNIT(kIOReportQuantityCPUInstrs, \ + kIOReportScaleMega) +#define kIOReportUnit_GI __IOR_MAKEUNIT(kIOReportQuantityCPUInstrs, \ + kIOReportScaleGiga) + +// Please file bugs (xnu | IOReporting) for additional units. + +// --- END: unit constants + + #ifdef __cplusplus } #endif diff --git a/iokit/IOKit/IOReturn.h b/iokit/IOKit/IOReturn.h index 464b84a08..048020784 100644 --- a/iokit/IOKit/IOReturn.h +++ b/iokit/IOKit/IOReturn.h @@ -52,14 +52,14 @@ typedef kern_return_t IOReturn; #define sub_iokit_firewire err_sub(2) #define sub_iokit_block_storage err_sub(4) #define sub_iokit_graphics err_sub(5) -#define sub_iokit_networking err_sub(6) +#define sub_iokit_networking err_sub(6) #define sub_iokit_bluetooth err_sub(8) #define sub_iokit_pmu err_sub(9) #define sub_iokit_acpi err_sub(10) #define sub_iokit_smbus err_sub(11) #define sub_iokit_ahci err_sub(12) #define sub_iokit_powermanagement err_sub(13) -#define sub_iokit_hidsystem err_sub(14) +#define sub_iokit_hidsystem err_sub(14) #define sub_iokit_scsi err_sub(16) #define sub_iokit_usbaudio err_sub(17) //#define sub_iokit_pccard err_sub(21) @@ -67,7 +67,12 @@ typedef kern_return_t IOReturn; #define sub_iokit_nvme err_sub(28) #endif #define sub_iokit_thunderbolt err_sub(29) -#define sub_iokit_platform err_sub(0x2A) +#define sub_iokit_graphics_acceleration err_sub(30) +#define sub_iokit_keystore err_sub(31) +#ifdef PRIVATE +#define sub_iokit_smc err_sub(32) +#endif +#define sub_iokit_platform err_sub(0x2A) #define sub_iokit_audio_video err_sub(0x45) #define sub_iokit_baseband err_sub(0x80) #define sub_iokit_HDA err_sub(254) diff --git a/iokit/IOKit/IOService.h b/iokit/IOKit/IOService.h index 35e7ec20e..b45162f87 100644 --- a/iokit/IOKit/IOService.h +++ b/iokit/IOKit/IOService.h @@ -131,6 +131,7 @@ extern const OSSymbol * gIOFirstPublishNotification; extern const OSSymbol * gIOMatchedNotification; extern const OSSymbol * gIOFirstMatchNotification; extern const OSSymbol * gIOTerminatedNotification; +extern const OSSymbol * gIOWillTerminateNotification; extern const OSSymbol * gIOGeneralInterest; extern const OSSymbol * gIOBusyInterest; @@ -639,6 +640,11 @@ public: virtual void unlockForArbitration( void ); +#ifdef XNU_KERNEL_PRIVATE + static uint32_t isLockedForArbitration(IOService * service); +#endif /* XNU_KERNEL_PRIVATE */ + + /*! @function terminateClient @abstract Passes a termination up the stack. @discussion When an IOService object is made inactive the default behavior is to also make any of its clients that have it as their only provider inactive, in this way recursing the termination up the driver stack. This method allows a terminated IOService object to override this behavior. Note the client may also override this behavior by overriding its @link terminate terminate@/link method. @@ -727,7 +733,8 @@ public:
gIOFirstPublishNotification Delivered when an IOService object is registered, but only once per IOService instance. Some IOService objects may be reregistered when their state is changed.
gIOMatchedNotification Delivered when an IOService object has been matched with all client drivers, and they have been probed and started.
gIOFirstMatchNotification Delivered when an IOService object has been matched with all client drivers, but only once per IOService instance. Some IOService objects may be reregistered when their state is changed. -
gIOTerminatedNotification Delivered after an IOService object has been terminated, during its finalize stage. +
gIOWillTerminateNotification Delivered after an IOService object has been terminated, during its finalize stage. Delivered after any matching on the service has finished. +
gIOTerminatedNotification Delivered immediately when an IOService object has been terminated, making it inactive. @param matching A matching dictionary to restrict notifications to only matching IOService objects. The dictionary will be released when the notification is removed, consuming the passed-in reference. @param handler A C function callback to deliver notifications. @param target An instance reference for the callback's use. @@ -750,7 +757,8 @@ public:
gIOFirstPublishNotification Delivered when an IOService object is registered, but only once per IOService instance. Some IOService objects may be reregistered when their state is changed.
gIOMatchedNotification Delivered when an IOService object has been matched with all client drivers, and they have been probed and started.
gIOFirstMatchNotification Delivered when an IOService object has been matched with all client drivers, but only once per IOService instance. Some IOService objects may be reregistered when their state is changed. -
gIOTerminatedNotification Delivered after an IOService object has been terminated, during its finalize stage. +
gIOWillTerminateNotification Delivered after an IOService object has been terminated, during its finalize stage. Delivered after any matching on the service has finished. +
gIOTerminatedNotification Delivered immediately when an IOService object has been terminated, making it inactive. @param matching A matching dictionary to restrict notifications to only matching IOService objects. The dictionary is retained while the notification is installed. (Differs from addNotification). @param handler A C function callback to deliver notifications. @param target An instance reference for the callback's use. @@ -1371,7 +1379,11 @@ private: void deliverNotification( const OSSymbol * type, IOOptionBits orNewState, IOOptionBits andNewState ); - bool invokeNotifer( class _IOServiceNotifier * notify ); + OSArray * copyNotifiers(const OSSymbol * type, + IOOptionBits orNewState, IOOptionBits andNewState); + + bool invokeNotifiers(OSArray ** willSend); + bool invokeNotifier( class _IOServiceNotifier * notify ); APPLE_KEXT_COMPATIBILITY_VIRTUAL void unregisterAllInterest( void ); @@ -1828,7 +1840,7 @@ public: void reset_watchdog_timer( void ); void start_watchdog_timer ( void ); bool stop_watchdog_timer ( void ); - IOReturn registerInterestForNotifer( IONotifier *notify, const OSSymbol * typeOfInterest, + IOReturn registerInterestForNotifier( IONotifier *notify, const OSSymbol * typeOfInterest, IOServiceInterestHandler handler, void * target, void * ref ); static IOWorkLoop * getIOPMWorkloop( void ); diff --git a/iokit/IOKit/IOTimeStamp.h b/iokit/IOKit/IOTimeStamp.h index 1c7cf7e57..955505f73 100644 --- a/iokit/IOKit/IOTimeStamp.h +++ b/iokit/IOKit/IOTimeStamp.h @@ -136,6 +136,7 @@ IOTimeStamp(uintptr_t csc, /* DBG_IOKIT/DBG_IOINTC codes */ #define IOINTC_HANDLER 1 /* 0x05000004 */ +#define IOINTC_SPURIOUS 2 /* 0x05000008 */ /* DBG_IOKIT/DBG_IOWORKLOOP codes */ #define IOWL_CLIENT 1 /* 0x05010004 */ diff --git a/iokit/IOKit/IOTimerEventSource.h b/iokit/IOKit/IOTimerEventSource.h index 538c4991b..49b5f257b 100644 --- a/iokit/IOKit/IOTimerEventSource.h +++ b/iokit/IOKit/IOTimerEventSource.h @@ -47,6 +47,56 @@ __END_DECLS #include #include +/*! + @enum IOTimerEventSource constructor options + @abstract Constants defining behavior of the IOTimerEventSource. + @constant kIOTimerEventSourceOptionsPriorityHigh Importance above everything but realtime. + Thread calls allocated with this priority execute at extremely high priority, + above everything but realtime threads. They are generally executed in serial. + Though they may execute concurrently under some circumstances, no fan-out is implied. + These work items should do very small amounts of work or risk disrupting system + responsiveness. + @constant kIOTimerEventSourceOptionsPriorityKernelHigh Importance higher than most kernel + threads. + @constant kIOTimerEventSourceOptionsPriorityKernel Importance similar to that of normal kernel + threads. + @constant kIOTimerEventSourceOptionsPriorityUser Importance similar to that of normal user threads. + @constant kIOTimerEventSourceOptionsPriorityLow Very low importance. + @constant kIOTimerEventSourceOptionsPriorityWorkLoop Run the callout on the thread of the IOWorkLoop + the event source has been added to. + @constant kIOTimerEventSourceOptionsAllowReenter Allow the callout to be rescheduled and potentially + re-entered, if the IOWorkLoop lock has been released (eg. with commandSleep) during its invocation. + @constant kIOTimerEventSourceOptionsDefault Recommended default options. + */ +enum +{ + kIOTimerEventSourceOptionsPriorityMask = 0x000000ff, + kIOTimerEventSourceOptionsPriorityHigh = 0x00000000, + kIOTimerEventSourceOptionsPriorityKernelHigh = 0x00000001, + kIOTimerEventSourceOptionsPriorityKernel = 0x00000002, + kIOTimerEventSourceOptionsPriorityUser = 0x00000003, + kIOTimerEventSourceOptionsPriorityLow = 0x00000004, + kIOTimerEventSourceOptionsPriorityWorkLoop = 0x000000ff, + + kIOTimerEventSourceOptionsAllowReenter = 0x00000100, + + kIOTimerEventSourceOptionsDefault = kIOTimerEventSourceOptionsPriorityKernelHigh +}; + +#define IOTIMEREVENTSOURCEOPTIONS_DEFINED 1 + +/*! + @enum IOTimerEventSource setTimeout/wakeAtTime options + @abstract Constants defining behavior of a scheduled call from IOTimerEventSource. + @constant kIOTimeOptionsWithLeeway Use the leeway parameter to the call. + @constant kIOTimeOptionsContinuous Use mach_continuous_time() to generate the callback. +*/ +enum +{ + kIOTimeOptionsWithLeeway = 0x00000020, + kIOTimeOptionsContinuous = 0x00000100, +}; + /*! @class IOTimerEventSource : public IOEventSource @abstract Time based event source mechanism. @@ -56,6 +106,7 @@ __END_DECLS

Remember the system doesn't guarantee the accuracy of the callout. It is possible that a higher priority thread is running which will delay the execution of the action routine. In fact the thread will be made runable at the exact requested time, within the accuracy of the CPU's decrementer based interrupt, but the scheduler will then control execution. */ + class IOTimerEventSource : public IOEventSource { OSDeclareDefaultStructors(IOTimerEventSource) @@ -73,6 +124,7 @@ protected: struct ExpansionData { SInt32 calloutGeneration; + SInt32 calloutGenerationSignaled; IOWorkLoop * workLoop; }; @@ -105,11 +157,17 @@ public: @param sender The object that timed out. */ typedef void (*Action)(OSObject *owner, IOTimerEventSource *sender); + static IOTimerEventSource * + timerEventSource(OSObject *owner, Action action = 0); + /*! @function timerEventSource @abstract Allocates and returns an initialized timer instance. + @param options Mask of kIOTimerEventSourceOptions* options. + @param owner The object that that will be passed to the Action callback. + @param action 'C' Function pointer for the callout routine of this event source. */ static IOTimerEventSource * - timerEventSource(OSObject *owner, Action action = 0); + timerEventSource(uint32_t options, OSObject *owner, Action action = 0); /*! @function init @abstract Initializes the timer with an owner, and a handler to call when the timeout expires. @@ -126,6 +184,11 @@ public: @discussion When disable returns the action will not be called until the next time enable(qv) is called. */ virtual void disable() APPLE_KEXT_OVERRIDE; +/*! @function checkForWork + @abstract Pure Virtual member function used by IOWorkLoop for issuing a client calls. + @discussion This function called when the work-loop is ready to check for any work to do and then to call out the owner/action. + @result Return true if this function needs to be called again before all its outstanding events have been processed. */ + virtual bool checkForWork() APPLE_KEXT_OVERRIDE; /*! @function setTimeoutTicks @abstract Setup a callback at after the delay in scheduler ticks. See wakeAtTime(AbsoluteTime). @@ -197,7 +260,7 @@ public: /*! @function wakeAtTime @abstract Setup a callback at this absolute time. - @discussion Starts the timer, which will expire at abstime. After it expires, the timer will call the 'action' registered in the init() function. This timer is not periodic, a further call is needed to reset and restart the timer after it expires. + @discussion Starts the timer, which will expire at abstime. After it expires, the timer will call the 'action' registered in the init() function. This timer is not periodic, a further call is needed to reset and restart the timer after it expires. @param abstime Absolute Time when to wake up, counted in 'decrementer' units and starts at zero when system boots. @result kIOReturnSuccess if everything is fine, kIOReturnNoResources if action hasn't been declared by init or IOEventSource::setAction (qqv). */ virtual IOReturn wakeAtTime(AbsoluteTime abstime); @@ -207,13 +270,36 @@ public: @discussion Clear down any oustanding calls. By the time this function completes it is guaranteed that the action will not be called again. */ virtual void cancelTimeout(); +/*! @function init + @abstract Initializes the timer with an owner, and a handler to call when the timeout expires. + */ + virtual bool init(uint32_t options, OSObject *inOwner, Action inAction); + +/*! @function setTimeout + @abstract Setup a callback at after the delay in decrementer ticks. See wakeAtTime(AbsoluteTime). + @param options see kIOTimeOptionsWithLeeway and kIOTimeOptionsContinuous + @param interval Delay from now to wake up in decrementer ticks. + @param leeway Allowable leeway to wake time, if the kIOTimeOptionsWithLeeway option is set + @result kIOReturnSuccess if everything is fine, kIOReturnNoResources if action hasn't been declared. */ + virtual IOReturn setTimeout(uint32_t options, AbsoluteTime interval, AbsoluteTime leeway); + +/*! @function wakeAtTime + @abstract Setup a callback at this absolute time. + @discussion Starts the timer, which will expire at abstime. After it expires, the timer will call the 'action' registered in the init() function. This timer is not periodic, a further call is needed to reset and restart the timer after it expires. + @param options see kIOTimeOptionsWithLeeway and kIOTimeOptionsContinuous + @param abstime Absolute Time when to wake up, counted in 'decrementer' units and starts at zero when system boots. + @param leeway Allowable leeway to wake time, if the kIOTimeOptionsWithLeeway option is set + @result kIOReturnSuccess if everything is fine, kIOReturnNoResources if action hasn't been declared by init or IOEventSource::setAction (qqv). */ + virtual IOReturn wakeAtTime(uint32_t options, AbsoluteTime abstime, AbsoluteTime leeway); + private: - static void timeoutAndRelease(void *self, void *wl); + static void timeoutAndRelease(void *self, void *c); + static void timeoutSignaled(void *self, void *c); private: - OSMetaClassDeclareReservedUnused(IOTimerEventSource, 0); - OSMetaClassDeclareReservedUnused(IOTimerEventSource, 1); - OSMetaClassDeclareReservedUnused(IOTimerEventSource, 2); + OSMetaClassDeclareReservedUsed(IOTimerEventSource, 0); + OSMetaClassDeclareReservedUsed(IOTimerEventSource, 1); + OSMetaClassDeclareReservedUsed(IOTimerEventSource, 2); OSMetaClassDeclareReservedUnused(IOTimerEventSource, 3); OSMetaClassDeclareReservedUnused(IOTimerEventSource, 4); OSMetaClassDeclareReservedUnused(IOTimerEventSource, 5); diff --git a/iokit/IOKit/IOTypes.h b/iokit/IOKit/IOTypes.h index 6d21a1294..62b5a6b08 100644 --- a/iokit/IOKit/IOTypes.h +++ b/iokit/IOKit/IOTypes.h @@ -192,7 +192,8 @@ enum { kIOWriteThruCache = 2, kIOCopybackCache = 3, kIOWriteCombineCache = 4, - kIOCopybackInnerCache = 5 + kIOCopybackInnerCache = 5, + kIOPostedWrite = 6 }; // IOMemory mapping options @@ -207,6 +208,7 @@ enum { kIOMapCopybackCache = kIOCopybackCache << kIOMapCacheShift, kIOMapWriteCombineCache = kIOWriteCombineCache << kIOMapCacheShift, kIOMapCopybackInnerCache = kIOCopybackInnerCache << kIOMapCacheShift, + kIOMapPostedWrite = kIOPostedWrite << kIOMapCacheShift, kIOMapUserOptionsMask = 0x00000fff, diff --git a/iokit/IOKit/Makefile b/iokit/IOKit/Makefile index 008d33e7e..327365f21 100644 --- a/iokit/IOKit/Makefile +++ b/iokit/IOKit/Makefile @@ -41,19 +41,20 @@ NOT_KF_MI_HEADERS = $(NOT_EXPORT_HEADERS) \ IOCommandQueue.h IOLocksPrivate.h \ IOSyncer.h AppleKeyStoreInterface.h \ IOStatistics.h IOStatisticsPrivate.h \ - IOKernelReporters.h + IOKernelReporters.h \ + IOInterruptAccounting.h # These should be additionally installed in IOKit.framework's public Headers -INSTALL_MI_LIST = IOBSD.h IOKitKeys.h IOKitServer.h IOReturn.h\ - IOSharedLock.h IOTypes.h OSMessageNotification.h\ - IODataQueueShared.h IOMessage.h IOInterruptAccounting.h\ +INSTALL_MI_LIST = IOBSD.h IOKitKeys.h IOKitServer.h IOReturn.h \ + IOSharedLock.h IOTypes.h OSMessageNotification.h \ + IODataQueueShared.h IOMessage.h # These should be additionally installed in IOKit.framework's PrivateHeaders -INSTALL_MI_LCL_LIST = IOKitKeysPrivate.h IOHibernatePrivate.h \ - IOLocksPrivate.h IOStatistics.h \ - AppleKeyStoreInterface.h \ - IOReportTypes.h IOKernelReportStructs.h \ - IOReportMacros.h +INSTALL_MI_LCL_LIST = IOKitKeysPrivate.h IOHibernatePrivate.h \ + IOLocksPrivate.h IOStatistics.h \ + AppleKeyStoreInterface.h \ + IOReportTypes.h IOKernelReportStructs.h \ + IOReportMacros.h IOInterruptAccounting.h INSTALL_MI_DIR = . diff --git a/iokit/IOKit/pwr_mgt/IOPM.h b/iokit/IOKit/pwr_mgt/IOPM.h index 03daffd06..f9ca6ed8a 100644 --- a/iokit/IOKit/pwr_mgt/IOPM.h +++ b/iokit/IOKit/pwr_mgt/IOPM.h @@ -614,6 +614,26 @@ enum { #define kIOPMPSAdapterDetailsPMUConfigurationKey "PMUConfiguration" #define kIOPMPSAdapterDetailsVoltage "AdapterVoltage" +// values for kIOPSPowerAdapterFamilyKey +enum { + kIOPSFamilyCodeDisconnected = 0, + kIOPSFamilyCodeUnsupported = kIOReturnUnsupported, + kIOPSFamilyCodeFirewire = iokit_family_err(sub_iokit_firewire, 0), + kIOPSFamilyCodeUSBHost = iokit_family_err(sub_iokit_usb, 0), + kIOPSFamilyCodeUSBHostSuspended = iokit_family_err(sub_iokit_usb, 1), + kIOPSFamilyCodeUSBDevice = iokit_family_err(sub_iokit_usb, 2), + kIOPSFamilyCodeUSBAdapter = iokit_family_err(sub_iokit_usb, 3), + kIOPSFamilyCodeUSBChargingPortDedicated = iokit_family_err(sub_iokit_usb, 4), + kIOPSFamilyCodeUSBChargingPortDownstream = iokit_family_err(sub_iokit_usb, 5), + kIOPSFamilyCodeUSBChargingPort = iokit_family_err(sub_iokit_usb, 6), + kIOPSFamilyCodeUSBUnknown = iokit_family_err(sub_iokit_usb, 7), + kIOPSFamilyCodeAC = iokit_family_err(sub_iokit_pmu, 0), + kIOPSFamilyCodeExternal = iokit_family_err(sub_iokit_pmu, 1), + kIOPSFamilyCodeExternal2 = iokit_family_err(sub_iokit_pmu, 2), + kIOPSFamilyCodeExternal3 = iokit_family_err(sub_iokit_pmu, 3), + kIOPSFamilyCodeExternal4 = iokit_family_err(sub_iokit_pmu, 4), +}; + // Battery's time remaining estimate is invalid this long (seconds) after a wake #define kIOPMPSInvalidWakeSecondsKey "BatteryInvalidWakeSeconds" diff --git a/iokit/IOKit/pwr_mgt/IOPMPrivate.h b/iokit/IOKit/pwr_mgt/IOPMPrivate.h index 4e498388c..b3f7b3397 100644 --- a/iokit/IOKit/pwr_mgt/IOPMPrivate.h +++ b/iokit/IOKit/pwr_mgt/IOPMPrivate.h @@ -103,6 +103,9 @@ enum { #define kIOPMMessageSystemSleepPreventers \ iokit_family_msg(sub_iokit_powermanagement, 0x430) +#define kIOPMMessageLaunchBootSpinDump \ + iokit_family_msg(sub_iokit_powermanagement, 0x440) + /* @enum SystemSleepReasons * @abstract The potential causes for system sleep as logged in the system event record. */ @@ -623,6 +626,13 @@ enum { */ #define kIOPMAutoPowerOffTimerKey "AutoPowerOff Timer" +/* kIOPMDeepSleepTimerKey + * Key refers to a CFNumberRef that indicates the time in seconds until the + * expiration of the Standby delay period. This value should be used + * to program a wake alarm before system sleep. + */ +#define kIOPMDeepSleepTimerKey "Standby Timer" + /* kIOPMUserWakeAlarmScheduledKey * Key refers to a boolean value that indicates if an user alarm was scheduled * or pending. @@ -706,6 +716,7 @@ struct IOPMSystemSleepPolicyVariables uint32_t hibernateMode; // current hibernate mode uint32_t standbyDelay; // standby delay in seconds + uint32_t standbyTimer; // standby timer in seconds uint32_t poweroffDelay; // auto-poweroff delay in seconds uint32_t scheduledAlarms; // bitmask of scheduled alarm types uint32_t poweroffTimer; // auto-poweroff timer in seconds @@ -788,7 +799,18 @@ enum { kIOPMWakeEventUserPME = 0x00000400, kIOPMWakeEventSleepTimer = 0x00000800, kIOPMWakeEventBatteryLow = 0x00001000, - kIOPMWakeEventDarkPME = 0x00002000 + kIOPMWakeEventDarkPME = 0x00002000, + kIOPMWakeEventWifi = 0x00004000, + kIOPMWakeEventRTCSystem = 0x00008000, // Maintenance RTC wake + kIOPMWakeEventUSBCPlugin = 0x00010000, // USB-C Plugin + kIOPMWakeEventHID = 0x00020000, + kIOPMWakeEventBluetooth = 0x00040000, + kIOPMWakeEventDFR = 0x00080000, + kIOPMWakeEventSD = 0x00100000, // SD card + kIOPMWakeEventLANWake = 0x00200000, // Wake on Lan + kIOPMWakeEventLANPlugin = 0x00400000, // Ethernet media sense + kIOPMWakeEventThunderbolt = 0x00800000, + kIOPMWakeEventRTCUser = 0x01000000, // User requested RTC wake }; /*! diff --git a/iokit/IOKit/pwr_mgt/IOPMlog.h b/iokit/IOKit/pwr_mgt/IOPMlog.h index 351ddad2d..9e93f2379 100644 --- a/iokit/IOKit/pwr_mgt/IOPMlog.h +++ b/iokit/IOKit/pwr_mgt/IOPMlog.h @@ -87,5 +87,7 @@ enum PMLogEnum { kPMLogDrvResponseDelay, // 59 0x050700ec kPMLogPCIDevChangeStart, // 60 0x050700f0 kPMLogPCIDevChangeDone, // 61 0x050700f4 + kPMLogSleepWakeMessage, // 62 0x050700f8 + kPMLogDrvPSChangeDelay, // 63 0x050700fc kIOPMlogLastEvent }; diff --git a/iokit/IOKit/pwr_mgt/RootDomain.h b/iokit/IOKit/pwr_mgt/RootDomain.h index c417c9a34..1e447d199 100644 --- a/iokit/IOKit/pwr_mgt/RootDomain.h +++ b/iokit/IOKit/pwr_mgt/RootDomain.h @@ -44,9 +44,7 @@ class IOPMPowerStateQueue; class RootDomainUserClient; class PMAssertionsTracker; -#define OBFUSCATE(x) \ - (((((uintptr_t)(x)) >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) && (((uintptr_t)(x)) < VM_MAX_KERNEL_ADDRESS)) ? \ - ((void *)(VM_KERNEL_ADDRPERM(x))) : (void *)(x)) +#define OBFUSCATE(x) (void *)VM_KERNEL_UNSLIDE_OR_PERM(x) #endif @@ -490,7 +488,7 @@ public: IOReturn setMaintenanceWakeCalendar( const IOPMCalendarStruct * calendar ); - IOReturn getSystemSleepType( uint32_t * sleepType ); + IOReturn getSystemSleepType(uint32_t * sleepType, uint32_t * standbyTimer); // Handle callbacks from IOService::systemWillShutdown() void acknowledgeSystemWillShutdown( IOService * from ); @@ -511,8 +509,12 @@ public: void kdebugTrace(uint32_t event, uint64_t regId, uintptr_t param1, uintptr_t param2, uintptr_t param3 = 0); - void tracePoint( uint8_t point ); - void traceDetail(uint32_t msgType, uint32_t msgIndex, uintptr_t handler); + void tracePoint(uint8_t point); + void traceDetail(uint32_t msgType, uint32_t msgIndex, uint32_t delay); + void traceDetail(OSObject *notifier); + void traceAckDelay(OSObject *notifier, uint32_t response, uint32_t delay_ms); + + void startSpinDump(uint32_t spindumpKind); bool systemMessageFilter( void * object, void * arg1, void * arg2, void * arg3 ); diff --git a/iokit/Kernel/IOBufferMemoryDescriptor.cpp b/iokit/Kernel/IOBufferMemoryDescriptor.cpp index 5641d3b9e..b3ff13dda 100644 --- a/iokit/Kernel/IOBufferMemoryDescriptor.cpp +++ b/iokit/Kernel/IOBufferMemoryDescriptor.cpp @@ -66,7 +66,8 @@ enum { kInternalFlagPhysical = 0x00000001, kInternalFlagPageSized = 0x00000002, - kInternalFlagPageAllocated = 0x00000004 + kInternalFlagPageAllocated = 0x00000004, + kInternalFlagInit = 0x00000008 }; /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -289,6 +290,11 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask( inTask, iomdOptions, /* System mapper */ 0)) return false; + _internalFlags |= kInternalFlagInit; +#if IOTRACKING + if (!(options & kIOMemoryPageable)) trackingAccumSize(capacity); +#endif /* IOTRACKING */ + // give any system mapper the allocation params if (kIOReturnSuccess != dmaCommandOperation(kIOMDAddDMAMapSpec, &mapSpec, sizeof(mapSpec))) @@ -488,19 +494,26 @@ void IOBufferMemoryDescriptor::free() map->release(); } + if ((options & kIOMemoryPageable) + || (kInternalFlagPageSized & internalFlags)) size = round_page(size); + +#if IOTRACKING + if (!(options & kIOMemoryPageable) + && buffer + && (kInternalFlagInit & _internalFlags)) trackingAccumSize(-size); +#endif /* IOTRACKING */ + /* super::free may unwire - deallocate buffer afterwards */ super::free(); if (options & kIOMemoryPageable) { #if IOALLOCDEBUG - OSAddAtomicLong(-(round_page(size)), &debug_iomallocpageable_size); + OSAddAtomicLong(-size, &debug_iomallocpageable_size); #endif } else if (buffer) { - if (kInternalFlagPageSized & internalFlags) size = round_page(size); - if (kInternalFlagPhysical & internalFlags) { IOKernelFreePhysical((mach_vm_address_t) buffer, size); diff --git a/iokit/Kernel/IOCPU.cpp b/iokit/Kernel/IOCPU.cpp index fac8021d5..afbad6666 100644 --- a/iokit/Kernel/IOCPU.cpp +++ b/iokit/Kernel/IOCPU.cpp @@ -30,10 +30,9 @@ extern "C" { #include #include #include +extern void kperf_kernel_configure(char *); } -#include - #include #include #include @@ -61,13 +60,17 @@ struct iocpu_platform_action_entry const char * name; void * refcon0; void * refcon1; + boolean_t callout_in_progress; struct iocpu_platform_action_entry * alloc_list; }; typedef struct iocpu_platform_action_entry iocpu_platform_action_entry_t; /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#define kBootCPUNumber 0 +static IOLock *gIOCPUsLock; +static OSArray *gIOCPUs; +static const OSSymbol *gIOCPUStateKey; +static OSString *gIOCPUStateNames[kIOCPUStateCount]; enum { @@ -116,7 +119,7 @@ iocpu_remove_platform_action(iocpu_platform_action_entry_t * entry) static kern_return_t iocpu_run_platform_actions(queue_head_t * queue, uint32_t first_priority, uint32_t last_priority, - void * param1, void * param2, void * param3) + void * param1, void * param2, void * param3, boolean_t allow_nested_callouts) { kern_return_t ret = KERN_SUCCESS; kern_return_t result = KERN_SUCCESS; @@ -128,7 +131,16 @@ iocpu_run_platform_actions(queue_head_t * queue, uint32_t first_priority, uint32 if ((pri >= first_priority) && (pri <= last_priority)) { //kprintf("[%p]", next->action); - ret = (*next->action)(next->refcon0, next->refcon1, pri, param1, param2, param3, next->name); + if (!allow_nested_callouts && !next->callout_in_progress) + { + next->callout_in_progress = TRUE; + ret = (*next->action)(next->refcon0, next->refcon1, pri, param1, param2, param3, next->name); + next->callout_in_progress = FALSE; + } + else if (allow_nested_callouts) + { + ret = (*next->action)(next->refcon0, next->refcon1, pri, param1, param2, param3, next->name); + } } if (KERN_SUCCESS == result) result = ret; @@ -142,14 +154,14 @@ extern "C" kern_return_t IOCPURunPlatformQuiesceActions(void) { return (iocpu_run_platform_actions(&gActionQueues[kQueueQuiesce], 0, 0U-1, - NULL, NULL, NULL)); + NULL, NULL, NULL, TRUE)); } extern "C" kern_return_t IOCPURunPlatformActiveActions(void) { return (iocpu_run_platform_actions(&gActionQueues[kQueueActive], 0, 0U-1, - NULL, NULL, NULL)); + NULL, NULL, NULL, TRUE)); } extern "C" kern_return_t @@ -157,15 +169,27 @@ IOCPURunPlatformHaltRestartActions(uint32_t message) { if (!gActionQueues[kQueueHaltRestart].next) return (kIOReturnNotReady); return (iocpu_run_platform_actions(&gActionQueues[kQueueHaltRestart], 0, 0U-1, - (void *)(uintptr_t) message, NULL, NULL)); + (void *)(uintptr_t) message, NULL, NULL, TRUE)); } extern "C" kern_return_t IOCPURunPlatformPanicActions(uint32_t message) { + // Don't allow nested calls of panic actions + if (!gActionQueues[kQueuePanic].next) return (kIOReturnNotReady); + return (iocpu_run_platform_actions(&gActionQueues[kQueuePanic], 0, 0U-1, + (void *)(uintptr_t) message, NULL, NULL, FALSE)); +} + + +extern "C" kern_return_t +IOCPURunPlatformPanicSyncAction(void *addr, size_t len) +{ + // Don't allow nested calls of panic actions if (!gActionQueues[kQueuePanic].next) return (kIOReturnNotReady); return (iocpu_run_platform_actions(&gActionQueues[kQueuePanic], 0, 0U-1, - (void *)(uintptr_t) message, NULL, NULL)); + (void *)(uintptr_t)(kPEPanicSync), addr, (void *)(uintptr_t)len, FALSE)); + } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -232,6 +256,7 @@ IOInstallServicePlatformAction(IOService * service, uint32_t qidx) entry->priority = priority; entry->refcon0 = service; entry->refcon1 = (void *) key; + entry->callout_in_progress = FALSE; iocpu_add_platform_action(queue, entry); } @@ -241,11 +266,25 @@ IOInstallServicePlatformAction(IOService * service, uint32_t qidx) void IOCPUInitialize(void) { + gIOCPUsLock = IOLockAlloc(); + gIOCPUs = OSArray::withCapacity(1); + for (uint32_t qidx = kQueueSleep; qidx < kQueueCount; qidx++) { queue_init(&gActionQueues[qidx]); } + gIOCPUStateKey = OSSymbol::withCStringNoCopy("IOCPUState"); + + gIOCPUStateNames[kIOCPUStateUnregistered] = + OSString::withCStringNoCopy("Unregistered"); + gIOCPUStateNames[kIOCPUStateUninitalized] = + OSString::withCStringNoCopy("Uninitalized"); + gIOCPUStateNames[kIOCPUStateStopped] = + OSString::withCStringNoCopy("Stopped"); + gIOCPUStateNames[kIOCPUStateRunning] = + OSString::withCStringNoCopy("Running"); + gIOPlatformSleepActionKey = gActionSymbols[kQueueSleep] = OSSymbol::withCStringNoCopy(kIOPlatformSleepActionKey); gIOPlatformWakeActionKey = gActionSymbols[kQueueWake] @@ -263,9 +302,13 @@ IOCPUInitialize(void) IOReturn IOInstallServicePlatformActions(IOService * service) { + IOLockLock(gIOCPUsLock); + IOInstallServicePlatformAction(service, kQueueHaltRestart); IOInstallServicePlatformAction(service, kQueuePanic); + IOLockUnlock(gIOCPUsLock); + return (kIOReturnSuccess); } @@ -275,6 +318,8 @@ IORemoveServicePlatformActions(IOService * service) iocpu_platform_action_entry_t * entry; iocpu_platform_action_entry_t * next; + IOLockLock(gIOCPUsLock); + for (uint32_t qidx = kQueueSleep; qidx < kQueueCount; qidx++) { next = (typeof(entry)) queue_first(&gActionQueues[qidx]); @@ -290,6 +335,8 @@ IORemoveServicePlatformActions(IOService * service) } } + IOLockUnlock(gIOCPUsLock); + return (kIOReturnSuccess); } @@ -340,16 +387,49 @@ void PE_cpu_machine_init(cpu_id_t target, boolean_t bootb) { IOCPU *targetCPU = OSDynamicCast(IOCPU, (OSObject *)target); - if (targetCPU) targetCPU->initCPU(bootb); + if (targetCPU) { + targetCPU->initCPU(bootb); +#if defined(__arm__) || defined(__arm64__) + if (!bootb && (targetCPU->getCPUNumber() == (UInt32)master_cpu)) ml_set_is_quiescing(false); +#endif /* defined(__arm__) || defined(__arm64__) */ + } } void PE_cpu_machine_quiesce(cpu_id_t target) { IOCPU *targetCPU = OSDynamicCast(IOCPU, (OSObject *)target); + if (targetCPU) { +#if defined(__arm__) || defined(__arm64__) + if (targetCPU->getCPUNumber() == (UInt32)master_cpu) ml_set_is_quiescing(true); +#endif /* defined(__arm__) || defined(__arm64__) */ + targetCPU->quiesceCPU(); + } +} + +#if defined(__arm__) || defined(__arm64__) +static perfmon_interrupt_handler_func pmi_handler = 0; - if (targetCPU) targetCPU->quiesceCPU(); +kern_return_t PE_cpu_perfmon_interrupt_install_handler(perfmon_interrupt_handler_func handler) +{ + pmi_handler = handler; + + return KERN_SUCCESS; } +void PE_cpu_perfmon_interrupt_enable(cpu_id_t target, boolean_t enable) +{ + IOCPU *targetCPU = OSDynamicCast(IOCPU, (OSObject *)target); + + if (targetCPU) { + if (enable) { + targetCPU->getProvider()->registerInterrupt(1, targetCPU, (IOInterruptAction)pmi_handler, 0); + targetCPU->getProvider()->enableInterrupt(1); + } else { + targetCPU->getProvider()->disableInterrupt(1); + } + } +} +#endif /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -367,10 +447,6 @@ OSMetaClassDefineReservedUnused(IOCPU, 7); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -static OSArray *gIOCPUs; -static const OSSymbol *gIOCPUStateKey; -static OSString *gIOCPUStateNames[kIOCPUStateCount]; - void IOCPUSleepKernel(void) { long cnt, numCPUs; @@ -415,7 +491,7 @@ void IOCPUSleepKernel(void) } iocpu_run_platform_actions(&gActionQueues[kQueueSleep], 0, 0U-1, - NULL, NULL, NULL); + NULL, NULL, NULL, TRUE); rootDomain->tracePoint( kIOPMTracePointSleepCPUs ); @@ -429,7 +505,7 @@ void IOCPUSleepKernel(void) // We make certain that the bootCPU is the last to sleep // We'll skip it for now, and halt it after finishing the // non-boot CPU's. - if (target->getCPUNumber() == kBootCPUNumber) + if (target->getCPUNumber() == (UInt32)master_cpu) { bootCPU = target; } else if (target->getCPUState() == kIOCPUStateRunning) @@ -439,7 +515,7 @@ void IOCPUSleepKernel(void) } assert(bootCPU != NULL); - assert(cpu_number() == 0); + assert(cpu_number() == master_cpu); console_suspend(); @@ -453,7 +529,7 @@ void IOCPUSleepKernel(void) console_resume(); iocpu_run_platform_actions(&gActionQueues[kQueueWake], 0, 0U-1, - NULL, NULL, NULL); + NULL, NULL, NULL, TRUE); iocpu_platform_action_entry_t * entry; for (uint32_t qidx = kQueueSleep; qidx <= kQueueActive; qidx++) @@ -472,47 +548,31 @@ void IOCPUSleepKernel(void) for (cnt = 0; cnt < numCPUs; cnt++) { target = OSDynamicCast(IOCPU, gIOCPUs->getObject(cnt)); - + // Skip the already-woken boot CPU. - if ((target->getCPUNumber() != kBootCPUNumber) - && (target->getCPUState() == kIOCPUStateStopped)) - { - processor_start(target->getMachProcessor()); + if (target->getCPUNumber() != (UInt32)master_cpu) { + if (target->getCPUState() == kIOCPUStateRunning) + panic("Spurious wakeup of cpu %u", (unsigned int)(target->getCPUNumber())); + + if (target->getCPUState() == kIOCPUStateStopped) + processor_start(target->getMachProcessor()); } } } -void IOCPU::initCPUs(void) -{ - if (gIOCPUs == 0) { - gIOCPUs = OSArray::withCapacity(1); - - gIOCPUStateKey = OSSymbol::withCStringNoCopy("IOCPUState"); - - gIOCPUStateNames[kIOCPUStateUnregistered] = - OSString::withCStringNoCopy("Unregistered"); - gIOCPUStateNames[kIOCPUStateUninitalized] = - OSString::withCStringNoCopy("Uninitalized"); - gIOCPUStateNames[kIOCPUStateStopped] = - OSString::withCStringNoCopy("Stopped"); - gIOCPUStateNames[kIOCPUStateRunning] = - OSString::withCStringNoCopy("Running"); - } -} - bool IOCPU::start(IOService *provider) { OSData *busFrequency, *cpuFrequency, *timebaseFrequency; if (!super::start(provider)) return false; - initCPUs(); - _cpuGroup = gIOCPUs; cpuNub = provider; + IOLockLock(gIOCPUsLock); gIOCPUs->setObject(this); - + IOLockUnlock(gIOCPUsLock); + // Correct the bus, cpu and timebase frequencies in the device tree. if (gPEClockFrequencyInfo.bus_frequency_hz < 0x100000000ULL) { busFrequency = OSData::withBytesNoCopy((void *)&gPEClockFrequencyInfo.bus_clock_rate_hz, 4); @@ -658,7 +718,6 @@ processor_t IOCPU::getMachProcessor(void) OSDefineMetaClassAndStructors(IOCPUInterruptController, IOInterruptController); -OSMetaClassDefineReservedUnused(IOCPUInterruptController, 0); OSMetaClassDefineReservedUnused(IOCPUInterruptController, 1); OSMetaClassDefineReservedUnused(IOCPUInterruptController, 2); OSMetaClassDefineReservedUnused(IOCPUInterruptController, 3); @@ -669,28 +728,29 @@ OSMetaClassDefineReservedUnused(IOCPUInterruptController, 5); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - IOReturn IOCPUInterruptController::initCPUInterruptController(int sources) +{ + return initCPUInterruptController(sources, sources); +} + +IOReturn IOCPUInterruptController::initCPUInterruptController(int sources, int cpus) { int cnt; if (!super::init()) return kIOReturnInvalid; - - numCPUs = sources; - - cpus = (IOCPU **)IOMalloc(numCPUs * sizeof(IOCPU *)); - if (cpus == 0) return kIOReturnNoMemory; - bzero(cpus, numCPUs * sizeof(IOCPU *)); - - vectors = (IOInterruptVector *)IOMalloc(numCPUs * sizeof(IOInterruptVector)); + + numSources = sources; + numCPUs = cpus; + + vectors = (IOInterruptVector *)IOMalloc(numSources * sizeof(IOInterruptVector)); if (vectors == 0) return kIOReturnNoMemory; - bzero(vectors, numCPUs * sizeof(IOInterruptVector)); - - // Allocate locks for the - for (cnt = 0; cnt < numCPUs; cnt++) { + bzero(vectors, numSources * sizeof(IOInterruptVector)); + + // Allocate a lock for each vector + for (cnt = 0; cnt < numSources; cnt++) { vectors[cnt].interruptLock = IOLockAlloc(); if (vectors[cnt].interruptLock == NULL) { - for (cnt = 0; cnt < numCPUs; cnt++) { + for (cnt = 0; cnt < numSources; cnt++) { if (vectors[cnt].interruptLock != NULL) IOLockFree(vectors[cnt].interruptLock); } @@ -698,7 +758,20 @@ IOReturn IOCPUInterruptController::initCPUInterruptController(int sources) } } - ml_init_max_cpus(numCPUs); + ml_init_max_cpus(numSources); + +#if KPERF + /* + * kperf allocates based on the number of CPUs and requires them to all be + * accounted for. + */ + boolean_t found_kperf = FALSE; + char kperf_config_str[64]; + found_kperf = PE_parse_boot_arg_str("kperf", kperf_config_str, sizeof(kperf_config_str)); + if (found_kperf && kperf_config_str[0] != '\0') { + kperf_kernel_configure(kperf_config_str); + } +#endif return kIOReturnSuccess; } @@ -724,8 +797,8 @@ void IOCPUInterruptController::setCPUInterruptProperties(IOService *service) return; // Create the interrupt specifer array. - specifier = OSArray::withCapacity(numCPUs); - for (cnt = 0; cnt < numCPUs; cnt++) { + specifier = OSArray::withCapacity(numSources); + for (cnt = 0; cnt < numSources; cnt++) { tmpLong = cnt; tmpData = OSData::withBytes(&tmpLong, sizeof(tmpLong)); specifier->setObject(tmpData); @@ -733,8 +806,8 @@ void IOCPUInterruptController::setCPUInterruptProperties(IOService *service) }; // Create the interrupt controller array. - controller = OSArray::withCapacity(numCPUs); - for (cnt = 0; cnt < numCPUs; cnt++) { + controller = OSArray::withCapacity(numSources); + for (cnt = 0; cnt < numSources; cnt++) { controller->setObject(gPlatformInterruptControllerName); } @@ -750,15 +823,18 @@ void IOCPUInterruptController::enableCPUInterrupt(IOCPU *cpu) IOInterruptHandler handler = OSMemberFunctionCast( IOInterruptHandler, this, &IOCPUInterruptController::handleInterrupt); + assert(numCPUs > 0); + ml_install_interrupt_handler(cpu, cpu->getCPUNumber(), this, handler, 0); - // Ensure that the increment is seen by all processors - OSIncrementAtomic(&enabledCPUs); + IOTakeLock(vectors[0].interruptLock); + ++enabledCPUs; if (enabledCPUs == numCPUs) { - IOService::cpusRunning(); - thread_wakeup(this); - } + IOService::cpusRunning(); + thread_wakeup(this); + } + IOUnlock(vectors[0].interruptLock); } IOReturn IOCPUInterruptController::registerInterrupt(IOService *nub, @@ -768,39 +844,42 @@ IOReturn IOCPUInterruptController::registerInterrupt(IOService *nub, void *refCon) { IOInterruptVector *vector; - - if (source >= numCPUs) return kIOReturnNoResources; - + + if (source >= numSources) return kIOReturnNoResources; + vector = &vectors[source]; - + // Get the lock for this vector. IOTakeLock(vector->interruptLock); - + // Make sure the vector is not in use. if (vector->interruptRegistered) { IOUnlock(vector->interruptLock); return kIOReturnNoResources; } - + // Fill in vector with the client's info. vector->handler = handler; vector->nub = nub; vector->source = source; vector->target = target; vector->refCon = refCon; - + // Get the vector ready. It starts hard disabled. vector->interruptDisabledHard = 1; vector->interruptDisabledSoft = 1; vector->interruptRegistered = 1; - + IOUnlock(vector->interruptLock); - + + IOTakeLock(vectors[0].interruptLock); if (enabledCPUs != numCPUs) { assert_wait(this, THREAD_UNINT); + IOUnlock(vectors[0].interruptLock); thread_block(THREAD_CONTINUE_NULL); - } - + } else + IOUnlock(vectors[0].interruptLock); + return kIOReturnSuccess; } diff --git a/iokit/Kernel/IOCommandGate.cpp b/iokit/Kernel/IOCommandGate.cpp index 2308daf1b..6c1f45767 100644 --- a/iokit/Kernel/IOCommandGate.cpp +++ b/iokit/Kernel/IOCommandGate.cpp @@ -207,7 +207,7 @@ IOReturn IOCommandGate::runAction(Action inAction, bool trace = ( gIOKitTrace & kIOTraceCommandGates ) ? true : false; if (trace) IOTimeStampStartConstant(IODBG_CMDQ(IOCMDQ_ACTION), - VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(inAction), VM_KERNEL_ADDRHIDE(owner)); IOStatisticsActionCall(); @@ -218,7 +218,7 @@ IOReturn IOCommandGate::runAction(Action inAction, *sleepersP -= kSleepersActions; if (trace) IOTimeStampEndConstant(IODBG_CMDQ(IOCMDQ_ACTION), - VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(inAction), VM_KERNEL_ADDRHIDE(owner)); if (kSleepersRemoved == ((kSleepersActionsMask|kSleepersRemoved) & *sleepersP)) { @@ -257,7 +257,7 @@ IOReturn IOCommandGate::attemptAction(Action inAction, if (trace) IOTimeStampStartConstant(IODBG_CMDQ(IOCMDQ_ACTION), - VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(inAction), VM_KERNEL_ADDRHIDE(owner)); IOStatisticsActionCall(); @@ -265,7 +265,7 @@ IOReturn IOCommandGate::attemptAction(Action inAction, if (trace) IOTimeStampEndConstant(IODBG_CMDQ(IOCMDQ_ACTION), - VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(inAction), VM_KERNEL_ADDRHIDE(owner)); } wl->openGate(); diff --git a/iokit/Kernel/IOCommandQueue.cpp b/iokit/Kernel/IOCommandQueue.cpp index 3a184bf94..1b4f3ce23 100644 --- a/iokit/Kernel/IOCommandQueue.cpp +++ b/iokit/Kernel/IOCommandQueue.cpp @@ -166,14 +166,14 @@ bool IOCommandQueue::checkForWork() if (trace) IOTimeStampStartConstant(IODBG_CMDQ(IOCMDQ_ACTION), - (uintptr_t) action, (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(action), VM_KERNEL_ADDRHIDE(owner)); IOStatisticsActionCall(); (*(IOCommandQueueAction) action)(owner, field0, field1, field2, field3); if (trace) IOTimeStampEndConstant(IODBG_CMDQ(IOCMDQ_ACTION), - (uintptr_t) action, (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(action), VM_KERNEL_ADDRHIDE(owner)); return (consumerIndex != producerIndex); } diff --git a/iokit/Kernel/IODMACommand.cpp b/iokit/Kernel/IODMACommand.cpp index 936ae0879..5feadeb14 100644 --- a/iokit/Kernel/IODMACommand.cpp +++ b/iokit/Kernel/IODMACommand.cpp @@ -379,15 +379,7 @@ IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepar fInternalState->fNewMD = true; mem->retain(); fMemory = mem; - if (fMapper) - { -#if IOTRACKING - fInternalState->fTag = IOMemoryTag(kernel_map); - __IODEQUALIFY(IOMemoryDescriptor *, mem)->prepare((IODirection) - (kIODirectionDMACommand | (fInternalState->fTag << kIODirectionDMACommandShift))); - IOTrackingAdd(gIOWireTracking, &fInternalState->fWireTracking, fMemory->getLength(), false); -#endif /* IOTRACKING */ - } + if (!fMapper) mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0); if (autoPrepare) { err = prepare(); if (err) { @@ -407,14 +399,7 @@ IODMACommand::clearMemoryDescriptor(bool autoComplete) if (fMemory) { while (fActive) complete(); - if (fMapper) - { -#if IOTRACKING - __IODEQUALIFY(IOMemoryDescriptor *, fMemory)->complete((IODirection) - (kIODirectionDMACommand | (fInternalState->fTag << kIODirectionDMACommandShift))); - IOTrackingRemove(gIOWireTracking, &fInternalState->fWireTracking, fMemory->getLength()); -#endif /* IOTRACKING */ - } + if (!fMapper) fMemory->dmaCommandOperation(kIOMDSetDMAInactive, this, 0); fMemory->release(); fMemory = 0; } @@ -455,7 +440,7 @@ IODMACommand::segmentOp( IODMACommandInternal * state = target->reserved; - if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperAlloc || !target->fMapper)) + if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperAllocValid || !target->fMapper)) maxPhys = (1ULL << target->fNumAddressBits); else maxPhys = 0; @@ -464,7 +449,6 @@ IODMACommand::segmentOp( address = segment.fIOVMAddr; length = segment.fLength; - assert(address); assert(length); if (!state->fMisaligned) @@ -610,7 +594,8 @@ IODMACommand::walkAll(UInt8 op) op &= ~kWalkPreflight; - state->fDoubleBuffer = (state->fMisaligned || (kWalkDoubleBuffer & op)); + state->fDoubleBuffer = (state->fMisaligned || state->fForceDoubleBuffer); + state->fForceDoubleBuffer = false; if (state->fDoubleBuffer) state->fCopyPageCount = atop_64(round_page(state->fPreparedLength)); @@ -835,6 +820,7 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr state->fNextRemapPage = NULL; state->fCopyMD = 0; state->fLocalMapperAlloc = 0; + state->fLocalMapperAllocValid = false; state->fLocalMapperAllocLength = 0; state->fLocalMapper = (fMapper && (fMapper != IOMapper::gSystem)); @@ -880,8 +866,7 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr mapArgs.fMapSpec.numAddressBits = fNumAddressBits ? fNumAddressBits : 64; mapArgs.fLength = state->fPreparedLength; const IOMemoryDescriptor * md = state->fCopyMD; - if (md) { mapArgs.fOffset = 0; } - else + if (md) { mapArgs.fOffset = 0; } else { md = fMemory; mapArgs.fOffset = state->fPreparedOffset; @@ -892,6 +877,7 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr if (kIOReturnSuccess == ret) { state->fLocalMapperAlloc = mapArgs.fAlloc; + state->fLocalMapperAllocValid = true; state->fLocalMapperAllocLength = mapArgs.fAllocLength; state->fMapContig = mapArgs.fMapContig; } @@ -907,17 +893,21 @@ IODMACommand::complete(bool invalidateCache, bool synchronize) { IODMACommandInternal * state = fInternalState; IOReturn ret = kIOReturnSuccess; + IOMemoryDescriptor * copyMD; if (fActive < 1) return kIOReturnNotReady; if (!--fActive) { + copyMD = state->fCopyMD; + if (copyMD) copyMD->retain(); + if (IS_NONCOHERENT(fMappingOptions) && invalidateCache) { - if (state->fCopyMD) + if (copyMD) { - state->fCopyMD->performOperation(kIOMemoryIncoherentIOFlush, 0, state->fPreparedLength); + copyMD->performOperation(kIOMemoryIncoherentIOFlush, 0, state->fPreparedLength); } else { @@ -933,17 +923,30 @@ IODMACommand::complete(bool invalidateCache, bool synchronize) op |= kWalkSyncIn; ret = walkAll(op); } - if (state->fLocalMapperAlloc) + + if (state->fLocalMapperAllocValid) { - if (state->fLocalMapperAllocLength) + IOMDDMAMapArgs mapArgs; + bzero(&mapArgs, sizeof(mapArgs)); + mapArgs.fMapper = fMapper; + mapArgs.fCommand = this; + mapArgs.fAlloc = state->fLocalMapperAlloc; + mapArgs.fAllocLength = state->fLocalMapperAllocLength; + const IOMemoryDescriptor * md = copyMD; + if (md) { mapArgs.fOffset = 0; } + else { - fMapper->iovmUnmapMemory(getIOMemoryDescriptor(), this, - state->fLocalMapperAlloc, state->fLocalMapperAllocLength); + md = fMemory; + mapArgs.fOffset = state->fPreparedOffset; } + + ret = md->dmaCommandOperation(kIOMDDMAUnmap, &mapArgs, sizeof(mapArgs)); + state->fLocalMapperAlloc = 0; + state->fLocalMapperAllocValid = false; state->fLocalMapperAllocLength = 0; } - + if (copyMD) copyMD->release(); state->fPrepared = false; } @@ -981,14 +984,13 @@ IODMACommand::synchronize(IOOptionBits options) op = 0; if (kForceDoubleBuffer & options) { - if (state->fDoubleBuffer) - return kIOReturnSuccess; - if (state->fCursor) - state->fCursor = false; - else - ret = walkAll(kWalkComplete); + if (state->fDoubleBuffer) return kIOReturnSuccess; + ret = complete(false /* invalidateCache */, true /* synchronize */); + state->fCursor = false; + state->fForceDoubleBuffer = true; + ret = prepare(state->fPreparedOffset, state->fPreparedLength, false /* flushCache */, true /* synchronize */); - op |= kWalkPrepare | kWalkPreflight | kWalkDoubleBuffer; + return (ret); } else if (state->fCursor) return kIOReturnSuccess; @@ -1132,17 +1134,18 @@ IODMACommand::genIOVMSegments(uint32_t op, return kIOReturnOverrun; if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) { - state->fOffset = 0; - state->fIOVMAddr = 0; - internalState->fNextRemapPage = NULL; - internalState->fNewMD = false; - state->fMapped = (0 != fMapper); - mdOp = kIOMDFirstSegment; + state->fOffset = 0; + internalState->fIOVMAddrValid = state->fIOVMAddr = 0; + internalState->fNextRemapPage = NULL; + internalState->fNewMD = false; + state->fMapped = (0 != fMapper); + mdOp = kIOMDFirstSegment; }; UInt32 segIndex = 0; UInt32 numSegments = *numSegmentsP; Segment64 curSeg = { 0, 0 }; + bool curSegValid = false; addr64_t maxPhys; if (fNumAddressBits && (fNumAddressBits < 64)) @@ -1151,17 +1154,17 @@ IODMACommand::genIOVMSegments(uint32_t op, maxPhys = 0; maxPhys--; - while (state->fIOVMAddr || (state->fOffset < memLength)) + while (internalState->fIOVMAddrValid || (state->fOffset < memLength)) { // state = next seg - if (!state->fIOVMAddr) { + if (!internalState->fIOVMAddrValid) { IOReturn rtn; state->fOffset = offset; state->fLength = memLength - offset; - if (internalState->fMapContig && internalState->fLocalMapperAlloc) + if (internalState->fMapContig && internalState->fLocalMapperAllocValid) { state->fIOVMAddr = internalState->fLocalMapperAlloc + offset - internalState->fPreparedOffset; rtn = kIOReturnSuccess; @@ -1193,39 +1196,40 @@ IODMACommand::genIOVMSegments(uint32_t op, if (rtn == kIOReturnSuccess) { - assert(state->fIOVMAddr); + internalState->fIOVMAddrValid = true; assert(state->fLength); - if ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr) { + if (curSegValid && ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr)) { UInt64 length = state->fLength; offset += length; curSeg.fLength += length; - state->fIOVMAddr = 0; + internalState->fIOVMAddrValid = state->fIOVMAddr = 0; } } else if (rtn == kIOReturnOverrun) - state->fIOVMAddr = state->fLength = 0; // At end + internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end else return rtn; } // seg = state, offset = end of seg - if (!curSeg.fIOVMAddr) + if (!curSegValid) { - UInt64 length = state->fLength; - offset += length; - curSeg.fIOVMAddr = state->fIOVMAddr; - curSeg.fLength = length; - state->fIOVMAddr = 0; + UInt64 length = state->fLength; + offset += length; + curSeg.fIOVMAddr = state->fIOVMAddr; + curSeg.fLength = length; + curSegValid = true; + internalState->fIOVMAddrValid = state->fIOVMAddr = 0; } - if (!state->fIOVMAddr) + if (!internalState->fIOVMAddrValid) { // maxPhys if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys) { if (internalState->fCursor) { - curSeg.fIOVMAddr = 0; + curSegValid = curSeg.fIOVMAddr = 0; ret = kIOReturnMessageTooLarge; break; } @@ -1237,6 +1241,7 @@ IODMACommand::genIOVMSegments(uint32_t op, DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength); remain = curSeg.fLength - newLength; state->fIOVMAddr = newLength + curSeg.fIOVMAddr; + internalState->fIOVMAddrValid = true; curSeg.fLength = newLength; state->fLength = remain; offset -= remain; @@ -1264,6 +1269,7 @@ IODMACommand::genIOVMSegments(uint32_t op, curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap)) + (addr & PAGE_MASK); + curSegValid = true; internalState->fNextRemapPage = vm_page_get_next(remap); newLength = PAGE_SIZE - (addr & PAGE_MASK); @@ -1271,6 +1277,7 @@ IODMACommand::genIOVMSegments(uint32_t op, { remain = curSeg.fLength - newLength; state->fIOVMAddr = addr + newLength; + internalState->fIOVMAddrValid = true; curSeg.fLength = newLength; state->fLength = remain; offset -= remain; @@ -1288,6 +1295,7 @@ IODMACommand::genIOVMSegments(uint32_t op, leftover += curSeg.fLength - fMaxSegmentSize; curSeg.fLength = fMaxSegmentSize; state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; + internalState->fIOVMAddrValid = true; } // alignment current length @@ -1298,6 +1306,7 @@ IODMACommand::genIOVMSegments(uint32_t op, leftover += reduce; curSeg.fLength -= reduce; state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; + internalState->fIOVMAddrValid = true; } // alignment next address @@ -1308,6 +1317,7 @@ IODMACommand::genIOVMSegments(uint32_t op, leftover += reduce; curSeg.fLength -= reduce; state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; + internalState->fIOVMAddrValid = true; } if (leftover) @@ -1336,7 +1346,7 @@ IODMACommand::genIOVMSegments(uint32_t op, if (misaligned) { if (misaligned) DEBG("cursor misaligned %qx:%qx\n", curSeg.fIOVMAddr, curSeg.fLength); - curSeg.fIOVMAddr = 0; + curSegValid = curSeg.fIOVMAddr = 0; ret = kIOReturnNotAligned; break; } @@ -1346,23 +1356,23 @@ IODMACommand::genIOVMSegments(uint32_t op, { curSeg.fLength -= (offset - memLength); offset = memLength; - state->fIOVMAddr = state->fLength = 0; // At end + internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end break; } } - if (state->fIOVMAddr) { + if (internalState->fIOVMAddrValid) { if ((segIndex + 1 == numSegments)) break; ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++); - curSeg.fIOVMAddr = 0; + curSegValid = curSeg.fIOVMAddr = 0; if (kIOReturnSuccess != ret) break; } } - if (curSeg.fIOVMAddr) { + if (curSegValid) { ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++); } @@ -1385,7 +1395,7 @@ IODMACommand::clientOutputSegment( if (target->fNumAddressBits && (target->fNumAddressBits < 64) && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits) - && (target->reserved->fLocalMapperAlloc || !target->fMapper)) + && (target->reserved->fLocalMapperAllocValid || !target->fMapper)) { DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength); ret = kIOReturnMessageTooLarge; diff --git a/iokit/Kernel/IODeviceTreeSupport.cpp b/iokit/Kernel/IODeviceTreeSupport.cpp index e115584ce..1a3b426d8 100644 --- a/iokit/Kernel/IODeviceTreeSupport.cpp +++ b/iokit/Kernel/IODeviceTreeSupport.cpp @@ -37,7 +37,11 @@ #include +#if __arm64__ +typedef UInt64 dtptr_t; +#else typedef UInt32 dtptr_t; +#endif #include @@ -55,8 +59,9 @@ int IODTGetDefault(const char *key, void *infoAddr, unsigned int infoSize ); const IORegistryPlane * gIODTPlane; -static OSArray * gIODTPHandles; -static OSArray * gIODTPHandleMap; +static OSArray * gIODTPHandles; +static OSArray * gIODTPHandleMap; +static OSData * gIODTResolvers; const OSSymbol * gIODTNameKey; const OSSymbol * gIODTUnitKey; @@ -80,6 +85,8 @@ const OSSymbol * gIODTNWInterruptMappingKey; OSDictionary * gIODTSharedInterrupts; +static IOLock * gIODTResolversLock; + static IORegistryEntry * MakeReferenceTable( DTEntry dtEntry, bool copy ); static void AddPHandle( IORegistryEntry * regEntry ); static void FreePhysicalMemory( vm_offset_t * range ); @@ -91,7 +98,7 @@ IODeviceTreeAlloc( void * dtTop ) IORegistryEntry * parent; IORegistryEntry * child; IORegistryIterator * regIter; - DTEntryIterator iter; + OpaqueDTEntryIterator iter; DTEntry dtChild; DTEntry mapEntry; OSArray * stack; @@ -135,6 +142,9 @@ IODeviceTreeAlloc( void * dtTop ) gIODTPHandles = OSArray::withCapacity( 1 ); gIODTPHandleMap = OSArray::withCapacity( 1 ); + gIODTResolvers = OSData::withCapacity(16); + + gIODTResolversLock = IOLockAlloc(); gIODTInterruptCellKey = OSSymbol::withCStringNoCopy("#interrupt-cells"); @@ -142,7 +152,7 @@ IODeviceTreeAlloc( void * dtTop ) assert( gIODTDefaultInterruptController && gIODTNWInterruptMappingKey && gIODTAAPLInterruptsKey && gIODTPHandleKey && gIODTInterruptParentKey - && gIODTPHandles && gIODTPHandleMap + && gIODTPHandles && gIODTPHandleMap && gIODTResolvers && gIODTResolversLock && gIODTInterruptCellKey ); @@ -154,21 +164,21 @@ IODeviceTreeAlloc( void * dtTop ) parent = MakeReferenceTable( (DTEntry)dtTop, freeDT ); stack = OSArray::withObjects( (const OSObject **) &parent, 1, 10 ); - DTCreateEntryIterator( (DTEntry)dtTop, &iter ); + DTInitEntryIterator( (DTEntry)dtTop, &iter ); do { parent = (IORegistryEntry *)stack->getObject( stack->getCount() - 1); //parent->release(); stack->removeObject( stack->getCount() - 1); - while( kSuccess == DTIterateEntries( iter, &dtChild) ) { + while( kSuccess == DTIterateEntries( &iter, &dtChild) ) { child = MakeReferenceTable( dtChild, freeDT ); child->attachToParent( parent, gIODTPlane); AddPHandle( child ); - if( kSuccess == DTEnterEntry( iter, dtChild)) { + if( kSuccess == DTEnterEntry( &iter, dtChild)) { stack->setObject( parent); parent = child; } @@ -177,10 +187,10 @@ IODeviceTreeAlloc( void * dtTop ) } } while( stack->getCount() - && (kSuccess == DTExitEntry( iter, &dtChild))); + && (kSuccess == DTExitEntry( &iter, &dtChild))); stack->release(); - DTDisposeEntryIterator( iter); + assert(kSuccess != DTExitEntry(&iter, &dtChild)); // parent is now root of the created tree @@ -338,7 +348,7 @@ MakeReferenceTable( DTEntry dtEntry, bool copy ) const OSSymbol *nameKey; OSData *data; const OSSymbol *sym; - DTPropertyIterator dtIter; + OpaqueDTPropertyIterator dtIter; void *prop; unsigned int propSize; char *name; @@ -354,12 +364,12 @@ MakeReferenceTable( DTEntry dtEntry, bool copy ) } if( regEntry && - (kSuccess == DTCreatePropertyIterator( dtEntry, &dtIter))) { + (kSuccess == DTInitPropertyIterator( dtEntry, &dtIter))) { kernelOnly = (kSuccess == DTGetProperty(dtEntry, "kernel-only", &prop, &propSize)); propTable = regEntry->getPropertyTable(); - while( kSuccess == DTIterateProperties( dtIter, &name)) { + while( kSuccess == DTIterateProperties( &dtIter, &name)) { if( kSuccess != DTGetProperty( dtEntry, name, &prop, &propSize )) continue; @@ -404,7 +414,6 @@ MakeReferenceTable( DTEntry dtEntry, bool copy ) regEntry->setLocation( location ); } } - DTDisposePropertyIterator( dtIter); } return( regEntry); @@ -926,29 +935,57 @@ OSCollectionIterator * IODTFindMatchingEntries( IORegistryEntry * from, struct IODTPersistent { IODTCompareAddressCellFunc compareFunc; - IODTNVLocationFunc locationFunc; }; void IODTSetResolving( IORegistryEntry * regEntry, IODTCompareAddressCellFunc compareFunc, - IODTNVLocationFunc locationFunc ) + IODTNVLocationFunc locationFunc __unused ) { - IODTPersistent persist; - OSData *prop; - - persist.compareFunc = compareFunc; - persist.locationFunc = locationFunc; - prop = OSData::withBytes( &persist, sizeof(persist)); - if( !prop) - return; - - prop->setSerializable(false); - regEntry->setProperty( gIODTPersistKey, prop); - prop->release(); + IODTPersistent persist; + IODTPersistent * entry; + OSNumber * num; + unsigned int index, count; + + IOLockLock(gIODTResolversLock); + + count = (gIODTResolvers->getLength() / sizeof(IODTPersistent)); + entry = (typeof(entry)) gIODTResolvers->getBytesNoCopy(); + for (index = 0; index < count; index++) + { + if (compareFunc == entry->compareFunc) break; + entry++; + } + if (index == count) + { + persist.compareFunc = compareFunc; + if (!gIODTResolvers->appendBytes(&persist, sizeof(IODTPersistent))) panic("IODTSetResolving"); + } + + IOLockUnlock(gIODTResolversLock); + + num = OSNumber::withNumber(index, 32); + regEntry->setProperty(gIODTPersistKey, num); + OSSafeReleaseNULL(num); + return; } -#if defined(__arm__) || defined(__i386__) || defined(__x86_64__) +#if defined(__arm64__) +static SInt64 DefaultCompare( UInt32 cellCount, UInt32 left[], UInt32 right[] ) +{ + SInt64 diff = 0; + + if (cellCount == 2) { + diff = IOPhysical32(left[1], left[0]) - IOPhysical32(right[1], right[0]); + } else if (cellCount == 1) { + diff = ( left[0] - right[0] ); + } else { + panic("DefaultCompare only knows how to handle 1 or 2 cells."); + } + + return diff; +} +#elif defined(__arm__) || defined(__i386__) || defined(__x86_64__) static SInt32 DefaultCompare( UInt32 cellCount, UInt32 left[], UInt32 right[] ) { cellCount--; @@ -965,11 +1002,19 @@ static void AddLengthToCells( UInt32 numCells, UInt32 *cells, UInt64 offset) cells[0] += (UInt32)offset; } else { +#if defined(__arm64__) || defined(__arm__) + UInt64 sum = cells[numCells - 2] + offset; + cells[numCells - 2] = (UInt32)sum; + if (sum > UINT32_MAX) { + cells[numCells - 1] += (UInt32)(sum >> 32); + } +#else UInt64 sum = cells[numCells - 1] + offset; cells[numCells - 1] = (UInt32)sum; if (sum > UINT32_MAX) { cells[numCells - 2] += (UInt32)(sum >> 32); } +#endif } } @@ -978,7 +1023,11 @@ static IOPhysicalAddress CellsValue( UInt32 numCells, UInt32 *cells) if (numCells == 1) { return IOPhysical32( 0, cells[0] ); } else { +#if defined(__arm64__) || defined(arm) + return IOPhysical32( cells[numCells - 1], cells[numCells - 2] ); +#else return IOPhysical32( cells[numCells - 2], cells[numCells - 1] ); +#endif } } @@ -998,12 +1047,15 @@ void IODTGetCellCounts( IORegistryEntry * regEntry, // Range[]: child-addr our-addr child-len // #cells: child ours child -bool IODTResolveAddressCell( IORegistryEntry * regEntry, +bool IODTResolveAddressCell( IORegistryEntry * startEntry, UInt32 cellsIn[], IOPhysicalAddress * phys, IOPhysicalLength * lenOut ) { - IORegistryEntry *parent; - OSData *prop; + IORegistryEntry * parent; + IORegistryEntry * regEntry; + OSData * prop; + OSNumber * num; + unsigned int index, count; // cells in addresses at regEntry UInt32 sizeCells, addressCells; // cells in addresses below regEntry @@ -1023,6 +1075,7 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry, IODTPersistent *persist; IODTCompareAddressCellFunc compare; + regEntry = startEntry; IODTGetCellCounts( regEntry, &childSizeCells, &childAddressCells ); childCells = childAddressCells + childSizeCells; @@ -1039,10 +1092,11 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry, /* end of the road */ *phys = CellsValue( childAddressCells, cell ); *phys += offset; + if (regEntry != startEntry) regEntry->release(); break; } - parent = regEntry->getParentEntry( gIODTPlane ); + parent = regEntry->copyParentEntry( gIODTPlane ); IODTGetCellCounts( parent, &sizeCells, &addressCells ); if( (propLen = prop->getLength())) { @@ -1051,13 +1105,25 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry, range = startRange; endRanges = range + (propLen / sizeof(UInt32)); - prop = (OSData *) regEntry->getProperty( gIODTPersistKey ); - if( prop) { - persist = (IODTPersistent *) prop->getBytesNoCopy(); - compare = persist->compareFunc; - } else if (addressCells == childAddressCells) { + compare = NULL; + num = OSDynamicCast(OSNumber, regEntry->getProperty(gIODTPersistKey)); + if (num) + { + IOLockLock(gIODTResolversLock); + index = num->unsigned32BitValue(); + count = gIODTResolvers->getLength() / sizeof(IODTPersistent); + if (index < count) + { + persist = ((IODTPersistent *) gIODTResolvers->getBytesNoCopy()) + index; + compare = persist->compareFunc; + } + IOLockUnlock(gIODTResolversLock); + } + + if (!compare && (addressCells == childAddressCells)) { compare = DefaultCompare; - } else { + } + if (!compare) { panic("There is no mixed comparison function yet..."); } @@ -1125,6 +1191,7 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry, } /* else zero length range => pass thru to parent */ + if (regEntry != startEntry) regEntry->release(); regEntry = parent; childSizeCells = sizeCells; childAddressCells = addressCells; @@ -1150,107 +1217,45 @@ OSArray * IODTResolveAddressing( IORegistryEntry * regEntry, OSArray *array; IODeviceMemory *range; - parentEntry = regEntry->getParentEntry( gIODTPlane ); - addressProperty = (OSData *) regEntry->getProperty( addressPropertyName ); - if( (0 == addressProperty) || (0 == parentEntry)) - return( 0); - - IODTGetCellCounts( parentEntry, &sizeCells, &addressCells ); - if( 0 == sizeCells) - return( 0); - - cells = sizeCells + addressCells; - reg = (UInt32 *) addressProperty->getBytesNoCopy(); - num = addressProperty->getLength() / (4 * cells); - - array = OSArray::withCapacity( 1 ); - if( 0 == array) - return( 0); - - for( i = 0; i < num; i++) { - if( IODTResolveAddressCell( parentEntry, reg, &phys, &len )) { - range = 0; - if( parent) - range = IODeviceMemory::withSubRange( parent, - phys - parent->getPhysicalSegment(0, 0, kIOMemoryMapperNone), len ); - if( 0 == range) - range = IODeviceMemory::withRange( phys, len ); - if( range) - array->setObject( range ); + array = 0; + do + { + parentEntry = regEntry->copyParentEntry( gIODTPlane ); + addressProperty = (OSData *) regEntry->getProperty( addressPropertyName ); + if( (0 == addressProperty) || (0 == parentEntry)) break; + + IODTGetCellCounts( parentEntry, &sizeCells, &addressCells ); + if( 0 == sizeCells) break; + + cells = sizeCells + addressCells; + reg = (UInt32 *) addressProperty->getBytesNoCopy(); + num = addressProperty->getLength() / (4 * cells); + + array = OSArray::withCapacity( 1 ); + if( 0 == array) break; + + for( i = 0; i < num; i++) { + if( IODTResolveAddressCell( parentEntry, reg, &phys, &len )) { + range = 0; + if( parent) + range = IODeviceMemory::withSubRange( parent, + phys - parent->getPhysicalSegment(0, 0, kIOMemoryMapperNone), len ); + if( 0 == range) + range = IODeviceMemory::withRange( phys, len ); + if( range) + array->setObject( range ); + } + reg += cells; } - reg += cells; - } - - regEntry->setProperty( gIODeviceMemoryKey, array); - array->release(); /* ??? */ - - return( array); -} - -static void IODTGetNVLocation( - IORegistryEntry * parent, - IORegistryEntry * regEntry, - UInt8 * busNum, UInt8 * deviceNum, UInt8 * functionNum ) -{ - OSData *prop; - IODTPersistent *persist; - UInt32 *cell; - - prop = (OSData *) parent->getProperty( gIODTPersistKey ); - if( prop) { - persist = (IODTPersistent *) prop->getBytesNoCopy(); - (*persist->locationFunc)( regEntry, busNum, deviceNum, functionNum ); - } else { - prop = (OSData *) regEntry->getProperty( "reg" ); - *functionNum = 0; - if( prop) { - cell = (UInt32 *) prop->getBytesNoCopy(); - *busNum = 3; - *deviceNum = 0x1f & (cell[ 0 ] >> 24); - } else { - *busNum = 0; - *deviceNum = 0; - } + regEntry->setProperty( gIODeviceMemoryKey, array); + array->release(); /* ??? */ } - return; -} + while (false); -/* - * Try to make the same messed up descriptor as Mac OS - */ - -IOReturn IODTMakeNVDescriptor( IORegistryEntry * regEntry, - IONVRAMDescriptor * hdr ) -{ - IORegistryEntry *parent; - UInt32 level; - UInt32 bridgeDevices; - UInt8 busNum; - UInt8 deviceNum; - UInt8 functionNum; - - hdr->format = 1; - hdr->marker = 0; - - for(level = 0, bridgeDevices = 0; - (parent = regEntry->getParentEntry( gIODTPlane )) && (level < 7); level++ ) { - - IODTGetNVLocation( parent, regEntry, - &busNum, &deviceNum, &functionNum ); - if( level) - bridgeDevices |= ((deviceNum & 0x1f) << ((level - 1) * 5)); - else { - hdr->busNum = busNum; - hdr->deviceNum = deviceNum; - hdr->functionNum = functionNum; - } - regEntry = parent; - } - hdr->bridgeCount = level - 2; - hdr->bridgeDevices = bridgeDevices; + OSSafeReleaseNULL(parentEntry); - return( kIOReturnSuccess ); + return (array); } OSData * IODTFindSlotName( IORegistryEntry * regEntry, UInt32 deviceNumber ) @@ -1266,40 +1271,43 @@ OSData * IODTFindSlotName( IORegistryEntry * regEntry, UInt32 deviceNumber ) UInt32 mask; data = (OSData *) regEntry->getProperty("AAPL,slot-name"); - if( data) - return( data); - parent = regEntry->getParentEntry( gIODTPlane ); - if( !parent) - return( 0 ); - data = OSDynamicCast( OSData, parent->getProperty("slot-names")); - if( !data) - return( 0 ); - if( data->getLength() <= 4) - return( 0 ); - - bits = (UInt32 *) data->getBytesNoCopy(); - mask = *bits; - if( (0 == (mask & (1 << deviceNumber)))) - return( 0 ); - - names = (char *)(bits + 1); - lastName = names + (data->getLength() - 4); - - for( i = 0; (i <= deviceNumber) && (names < lastName); i++ ) { - - if( mask & (1 << i)) { - nlen = 1 + strnlen(names, lastName - names); - if( i == deviceNumber) { - data = OSData::withBytesNoCopy(names, nlen); - if( data) { - regEntry->setProperty("AAPL,slot-name", data); - ret = data; - data->release(); - } - } else - names += nlen; + if (data) return (data); + + do + { + parent = regEntry->copyParentEntry( gIODTPlane ); + if (!parent) break; + + data = OSDynamicCast( OSData, parent->getProperty("slot-names")); + if (!data) break; + if (data->getLength() <= 4) break; + + bits = (UInt32 *) data->getBytesNoCopy(); + mask = *bits; + if ((0 == (mask & (1 << deviceNumber)))) break; + + names = (char *)(bits + 1); + lastName = names + (data->getLength() - 4); + + for( i = 0; (i <= deviceNumber) && (names < lastName); i++ ) { + + if( mask & (1 << i)) { + nlen = 1 + strnlen(names, lastName - names); + if( i == deviceNumber) { + data = OSData::withBytesNoCopy(names, nlen); + if( data) { + regEntry->setProperty("AAPL,slot-name", data); + ret = data; + data->release(); + } + } else + names += nlen; + } } } + while (false); + + OSSafeReleaseNULL(parent); return( ret ); } diff --git a/iokit/Kernel/IOFilterInterruptEventSource.cpp b/iokit/Kernel/IOFilterInterruptEventSource.cpp index 83b6ed6e0..c6f79e91d 100644 --- a/iokit/Kernel/IOFilterInterruptEventSource.cpp +++ b/iokit/Kernel/IOFilterInterruptEventSource.cpp @@ -131,12 +131,12 @@ void IOFilterInterruptEventSource::signalInterrupt() producerCount++; if (trace) - IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); signalWorkAvailable(); if (trace) - IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); } @@ -160,7 +160,7 @@ void IOFilterInterruptEventSource::normalInterruptOccurred if (trace) IOTimeStampStartConstant(IODBG_INTES(IOINTES_FILTER), - VM_KERNEL_UNSLIDE(filterAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_UNSLIDE(filterAction), VM_KERNEL_ADDRHIDE(owner), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (IOInterruptEventSource::reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelTimeIndex)) { @@ -184,7 +184,8 @@ void IOFilterInterruptEventSource::normalInterruptOccurred if (trace) IOTimeStampEndConstant(IODBG_INTES(IOINTES_FILTER), - VM_KERNEL_UNSLIDE(filterAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_ADDRHIDE(filterAction), VM_KERNEL_ADDRHIDE(owner), + VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (filterRes) signalInterrupt(); @@ -200,7 +201,7 @@ void IOFilterInterruptEventSource::disableInterruptOccurred if (trace) IOTimeStampStartConstant(IODBG_INTES(IOINTES_FILTER), - VM_KERNEL_UNSLIDE(filterAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_UNSLIDE(filterAction), VM_KERNEL_ADDRHIDE(owner), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (IOInterruptEventSource::reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelTimeIndex)) { @@ -224,7 +225,7 @@ void IOFilterInterruptEventSource::disableInterruptOccurred if (trace) IOTimeStampEndConstant(IODBG_INTES(IOINTES_FILTER), - VM_KERNEL_UNSLIDE(filterAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_UNSLIDE(filterAction), VM_KERNEL_ADDRHIDE(owner), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (filterRes) { prov->disableInterrupt(source); /* disable the interrupt */ diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp index 953c6d445..66b566ca9 100644 --- a/iokit/Kernel/IOHibernateIO.cpp +++ b/iokit/Kernel/IOHibernateIO.cpp @@ -174,6 +174,7 @@ to restrict I/O ops. #include #include #include +#include extern "C" addr64_t kvtophys(vm_offset_t va); extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); @@ -213,7 +214,7 @@ static IOLock * gDebugImageLock; #endif /* defined(__i386__) || defined(__x86_64__) */ static IOLock * gFSLock; -static uint32_t gFSState; + uint32_t gFSState; static thread_call_t gIOHibernateTrimCalloutEntry; static IOPolledFileIOVars gFileVars; static IOHibernateVars gIOHibernateVars; @@ -235,6 +236,7 @@ enum static IOReturn IOHibernateDone(IOHibernateVars * vars); static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature); static void IOSetBootImageNVRAM(OSData * data); +static void IOHibernateSystemPostWakeTrim(void * p1, void * p2); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -635,20 +637,37 @@ IOHibernateSystemSleep(void) rtcVars.signature[3] = 'L'; rtcVars.revision = 1; bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey)); - if (gIOHibernateBootSignature[0]) + + if (gIOChosenEntry + && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateBootSignatureKey))) + && (sizeof(rtcVars.booterSignature) <= data->getLength())) + { + bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature)); + } + else if (gIOHibernateBootSignature[0]) { char c; uint8_t value = 0; - for (uint32_t i = 0; - (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1)); - i++) + uint32_t in, out, digits; + for (in = out = digits = 0; + (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature)); + in++) { - if (c >= 'a') c -= 'a' - 10; - else if (c >= 'A') c -= 'A' - 10; - else if (c >= '0') c -= '0'; - else continue; + if ((c >= 'a') && (c <= 'f')) c -= 'a' - 10; + else if ((c >= 'A') && (c <= 'F')) c -= 'A' - 10; + else if ((c >= '0') && (c <= '9')) c -= '0'; + else + { + if (c == '=') out = digits = value = 0; + continue; + } value = (value << 4) | c; - if (i & 1) rtcVars.booterSignature[i >> 1] = value; + if (digits & 1) + { + rtcVars.booterSignature[out++] = value; + if (out >= sizeof(rtcVars.booterSignature)) break; + } + digits++; } } data = OSData::withBytes(&rtcVars, sizeof(rtcVars)); @@ -1272,7 +1291,7 @@ IOHibernateDone(IOHibernateVars * vars) return (kIOReturnSuccess); } -void +static void IOHibernateSystemPostWakeTrim(void * p1, void * p2) { // invalidate & close the image file @@ -1293,7 +1312,7 @@ IOHibernateSystemPostWakeTrim(void * p1, void * p2) } IOReturn -IOHibernateSystemPostWake(void) +IOHibernateSystemPostWake(bool now) { gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; IOLockLock(gFSLock); @@ -1301,11 +1320,14 @@ IOHibernateSystemPostWake(void) else if (kFSOpened != gFSState) gFSState = kFSIdle; else { - AbsoluteTime deadline; - gFSState = kFSTrimDelay; - clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline ); - thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline); + if (now) IOHibernateSystemPostWakeTrim(NULL, NULL); + else + { + AbsoluteTime deadline; + clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline ); + thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline); + } } IOLockUnlock(gFSLock); @@ -1478,7 +1500,7 @@ hibernate_write_image(void) if (kIOHibernateModeSleep & gIOHibernateMode) kdebug_enable = save_kdebug_enable; - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START); IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate); restore1Sum = sum1 = sum2 = 0; @@ -1749,6 +1771,16 @@ hibernate_write_image(void) pageCount -= atop_32(segLen); } +#if KASAN + vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext); + + /* no need to save unused shadow pages */ + hibernate_set_page_state(vars->page_list, vars->page_list_wired, + atop_64(shadow_pnext), + shadow_pages_free, + kIOHibernatePageStateFree); +#endif + src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); compressed = src + page_size; scratch = compressed + page_size; @@ -2040,8 +2072,8 @@ hibernate_write_image(void) // should we come back via regular wake, set the state in memory. gIOHibernateState = kIOHibernateStateInactive; - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, - wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted, + wiredPagesClear, dirtyPagesEncrypted); if (kIOReturnSuccess == err) { @@ -2123,6 +2155,11 @@ hibernate_machine_init(void) gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize; gIOHibernateStats->image1Pages = pagesDone; + /* HIBERNATE_stats */ + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart, + gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration, + gIOHibernateStats->trampolineDuration); + HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n", gIOHibernateStats->booterStart, gIOHibernateStats->smcStart, @@ -2403,7 +2440,7 @@ hibernate_machine_init(void) nsec / 1000000ULL, nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ diff --git a/iokit/Kernel/IOHibernateRestoreKernel.c b/iokit/Kernel/IOHibernateRestoreKernel.c index 017d4d4f8..fc5a1b7f2 100644 --- a/iokit/Kernel/IOHibernateRestoreKernel.c +++ b/iokit/Kernel/IOHibernateRestoreKernel.c @@ -73,17 +73,7 @@ extern void acpi_wake_prot_entry(void); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #if defined(__i386__) || defined(__x86_64__) - -#define rdtsc(lo,hi) \ - __asm__ volatile("lfence; rdtsc; lfence" : "=a" (lo), "=d" (hi)) - -static inline uint64_t rdtsc64(void) -{ - uint64_t lo, hi; - rdtsc(lo, hi); - return ((hi) << 32) | (lo); -} - +#include #else static inline uint64_t rdtsc64(void) @@ -409,7 +399,7 @@ store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize, s = *src; d = (uint32_t *)(uintptr_t)dst; - if (!s) bzero((void *) dst, PAGE_SIZE); + if (!s) __nosan_bzero((void *) dst, PAGE_SIZE); else for (i = 0; i < (PAGE_SIZE / sizeof(int32_t)); i++) *d++ = s; } } @@ -465,7 +455,7 @@ hibernate_kernel_entrypoint(uint32_t p1, debug_code(kIOHibernateRestoreCodeImageStart, headerPhys); - memcpy(gIOHibernateCurrentHeader, + __nosan_memcpy(gIOHibernateCurrentHeader, (void *) pal_hib_map(IMAGE_AREA, headerPhys), sizeof(IOHibernateImageHeader)); @@ -638,7 +628,7 @@ hibernate_kernel_entrypoint(uint32_t p1, // alloc new buffer page bufferPage = hibernate_page_list_grab(map, &nextFree); dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage)); - memcpy(dst, src, compressedSize); + __nosan_memcpy(dst, src, compressedSize); } if (copyPageIndex > ((PAGE_SIZE >> 2) - 3)) { diff --git a/iokit/Kernel/IOHistogramReporter.cpp b/iokit/Kernel/IOHistogramReporter.cpp index 929a830f2..4c288ccdb 100644 --- a/iokit/Kernel/IOHistogramReporter.cpp +++ b/iokit/Kernel/IOHistogramReporter.cpp @@ -43,7 +43,7 @@ IOHistogramReporter::with(IOService *reportingService, IOReportCategories categories, uint64_t channelID, const char *channelName, - IOReportUnits unit, + IOReportUnit unit, int nSegments, IOHistogramSegmentConfig *config) { @@ -74,7 +74,7 @@ IOHistogramReporter::initWith(IOService *reportingService, IOReportCategories categories, uint64_t channelID, const OSSymbol *channelName, - IOReportUnits unit, + IOReportUnit unit, int nSegments, IOHistogramSegmentConfig *config) { @@ -272,34 +272,35 @@ IOHistogramReporter::free(void) IOReportLegendEntry* IOHistogramReporter::handleCreateLegend(void) { - OSData *tmpConfigData; - OSDictionary *tmpDict; - IOReportLegendEntry *legendEntry = NULL; + IOReportLegendEntry *rval = NULL, *legendEntry = NULL; + OSData *tmpConfigData = NULL; + OSDictionary *tmpDict; // no refcount legendEntry = super::handleCreateLegend(); + if (!legendEntry) goto finish; - if (legendEntry) { - - PREFL_MEMOP_PANIC(_segmentCount, IOHistogramSegmentConfig); - tmpConfigData = OSData::withBytes(_histogramSegmentsConfig, - (unsigned)_segmentCount * - (unsigned)sizeof(IOHistogramSegmentConfig)); - if (!tmpConfigData) { - legendEntry->release(); - goto finish; - } - - tmpDict = OSDynamicCast(OSDictionary, legendEntry->getObject(kIOReportLegendInfoKey)); - if (!tmpDict) { - legendEntry->release(); - goto finish; - } - - tmpDict->setObject(kIOReportLegendConfigKey, tmpConfigData); - } - + PREFL_MEMOP_PANIC(_segmentCount, IOHistogramSegmentConfig); + tmpConfigData = OSData::withBytes(_histogramSegmentsConfig, + (unsigned)_segmentCount * + sizeof(IOHistogramSegmentConfig)); + if (!tmpConfigData) goto finish; + + tmpDict = OSDynamicCast(OSDictionary, + legendEntry->getObject(kIOReportLegendInfoKey)); + if (!tmpDict) goto finish; + + tmpDict->setObject(kIOReportLegendConfigKey, tmpConfigData); + + // success + rval = legendEntry; + finish: - return legendEntry; + if (tmpConfigData) tmpConfigData->release(); + if (!rval && legendEntry) { + legendEntry->release(); + } + + return rval; } IOReturn diff --git a/iokit/Kernel/IOInterruptAccounting.cpp b/iokit/Kernel/IOInterruptAccounting.cpp index 8130cce23..bfaf153b2 100644 --- a/iokit/Kernel/IOInterruptAccounting.cpp +++ b/iokit/Kernel/IOInterruptAccounting.cpp @@ -30,10 +30,12 @@ #include uint32_t gInterruptAccountingStatisticBitmask = +#if !defined(__arm__) /* Disable timestamps for older ARM platforms; they are expensive. */ IA_GET_ENABLE_BIT(kInterruptAccountingFirstLevelTimeIndex) | IA_GET_ENABLE_BIT(kInterruptAccountingSecondLevelCPUTimeIndex) | IA_GET_ENABLE_BIT(kInterruptAccountingSecondLevelSystemTimeIndex) | +#endif IA_GET_ENABLE_BIT(kInterruptAccountingFirstLevelCountIndex) | IA_GET_ENABLE_BIT(kInterruptAccountingSecondLevelCountIndex); diff --git a/iokit/Kernel/IOInterruptController.cpp b/iokit/Kernel/IOInterruptController.cpp index fb0aa23b0..81f07dcab 100644 --- a/iokit/Kernel/IOInterruptController.cpp +++ b/iokit/Kernel/IOInterruptController.cpp @@ -397,6 +397,54 @@ void IOInterruptController::causeVector(IOInterruptVectorNumber /*vectorNumber*/ { } +void IOInterruptController::timeStampSpuriousInterrupt(void) +{ + uint64_t providerID = 0; + IOService * provider = getProvider(); + + if (provider) { + providerID = provider->getRegistryEntryID(); + } + + IOTimeStampConstant(IODBG_INTC(IOINTC_SPURIOUS), providerID); +} + +void IOInterruptController::timeStampInterruptHandlerInternal(bool isStart, IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector) +{ + uint64_t providerID = 0; + vm_offset_t unslidHandler = 0; + vm_offset_t unslidTarget = 0; + + IOService * provider = getProvider(); + + if (provider) { + providerID = provider->getRegistryEntryID(); + } + + if (vector) { + unslidHandler = VM_KERNEL_UNSLIDE((vm_offset_t)vector->handler); + unslidTarget = VM_KERNEL_UNSLIDE_OR_PERM((vm_offset_t)vector->target); + } + + + if (isStart) { + IOTimeStampStartConstant(IODBG_INTC(IOINTC_HANDLER), (uintptr_t)vectorNumber, (uintptr_t)unslidHandler, + (uintptr_t)unslidTarget, (uintptr_t)providerID); + } else { + IOTimeStampEndConstant(IODBG_INTC(IOINTC_HANDLER), (uintptr_t)vectorNumber, (uintptr_t)unslidHandler, + (uintptr_t)unslidTarget, (uintptr_t)providerID); + } +} + +void IOInterruptController::timeStampInterruptHandlerStart(IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector) +{ + timeStampInterruptHandlerInternal(true, vectorNumber, vector); +} + +void IOInterruptController::timeStampInterruptHandlerEnd(IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector) +{ + timeStampInterruptHandlerInternal(false, vectorNumber, vector); +} /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -691,26 +739,23 @@ IOReturn IOSharedInterruptController::handleInterrupt(void * /*refCon*/, OSMemoryBarrier(); #endif - if (!vector->interruptDisabledSoft) { - - // Call the handler if it exists. - if (vector->interruptRegistered) { - - bool trace = (gIOKitTrace & kIOTraceInterrupts) ? true : false; - - if (trace) - IOTimeStampStartConstant(IODBG_INTC(IOINTC_HANDLER), - (uintptr_t) vectorNumber, (uintptr_t) vector->handler, (uintptr_t)vector->target); - - // Call handler. - vector->handler(vector->target, vector->refCon, vector->nub, vector->source); - - if (trace) - IOTimeStampEndConstant(IODBG_INTC(IOINTC_HANDLER), - (uintptr_t) vectorNumber, (uintptr_t) vector->handler, (uintptr_t)vector->target); - - } - } + if (!vector->interruptDisabledSoft) { + + // Call the handler if it exists. + if (vector->interruptRegistered) { + + bool trace = (gIOKitTrace & kIOTraceInterrupts) ? true : false; + + if (trace) + timeStampInterruptHandlerStart(vectorNumber, vector); + + // Call handler. + vector->handler(vector->target, vector->refCon, vector->nub, vector->source); + + if (trace) + timeStampInterruptHandlerEnd(vectorNumber, vector); + } + } vector->interruptActive = 0; } diff --git a/iokit/Kernel/IOInterruptEventSource.cpp b/iokit/Kernel/IOInterruptEventSource.cpp index 1636405f6..a410de27e 100644 --- a/iokit/Kernel/IOInterruptEventSource.cpp +++ b/iokit/Kernel/IOInterruptEventSource.cpp @@ -308,7 +308,8 @@ bool IOInterruptEventSource::checkForWork() { if (trace) IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION), - VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner), + VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { @@ -341,7 +342,8 @@ bool IOInterruptEventSource::checkForWork() if (trace) IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION), - VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner), + VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); consumerCount = cacheProdCount; if (autoDisable && !explicitDisable) @@ -352,7 +354,8 @@ bool IOInterruptEventSource::checkForWork() { if (trace) IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION), - VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner), + VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { @@ -385,7 +388,8 @@ bool IOInterruptEventSource::checkForWork() if (trace) IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION), - VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner), + VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); consumerCount = cacheProdCount; if (autoDisable && !explicitDisable) @@ -404,7 +408,7 @@ void IOInterruptEventSource::normalInterruptOccurred producerCount++; if (trace) - IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); if (reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) { @@ -415,7 +419,7 @@ void IOInterruptEventSource::normalInterruptOccurred signalWorkAvailable(); if (trace) - IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); } void IOInterruptEventSource::disableInterruptOccurred @@ -429,7 +433,7 @@ void IOInterruptEventSource::disableInterruptOccurred producerCount++; if (trace) - IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); if (reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) { @@ -440,7 +444,7 @@ void IOInterruptEventSource::disableInterruptOccurred signalWorkAvailable(); if (trace) - IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); } void IOInterruptEventSource::interruptOccurred diff --git a/iokit/Kernel/IOKitDebug.cpp b/iokit/Kernel/IOKitDebug.cpp index 22b315da4..0cf42b685 100644 --- a/iokit/Kernel/IOKitDebug.cpp +++ b/iokit/Kernel/IOKitDebug.cpp @@ -62,9 +62,23 @@ SInt64 gIOKitTrace = 0; #define IODEBUG_CTLFLAGS CTLFLAG_RD #endif -SYSCTL_QUAD(_debug, OID_AUTO, iokit, IODEBUG_CTLFLAGS | CTLFLAG_LOCKED, &gIOKitDebug, "boot_arg io"); SYSCTL_QUAD(_debug, OID_AUTO, iotrace, CTLFLAG_RW | CTLFLAG_LOCKED, &gIOKitTrace, "trace io"); +static int +sysctl_debug_iokit +(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) +{ + SInt64 newValue; + int changed, error = sysctl_io_number(req, gIOKitDebug, sizeof(gIOKitDebug), &newValue, &changed); + if (changed) { + gIOKitDebug = ((gIOKitDebug & ~kIOKitDebugUserOptions) | (newValue & kIOKitDebugUserOptions)); + } + return (error); +} + +SYSCTL_PROC(_debug, OID_AUTO, iokit, + CTLTYPE_QUAD | IODEBUG_CTLFLAGS | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, + &gIOKitDebug, 0, sysctl_debug_iokit, "Q", "boot_arg io"); int debug_malloc_size; int debug_iomalloc_size; @@ -245,6 +259,7 @@ struct IOTrackingCallSite IOTrackingQueue * queue; uint32_t crc; + vm_tag_t tag; uint32_t count; size_t size[2]; uintptr_t bt[kIOTrackingCallSiteBTs]; @@ -259,6 +274,7 @@ struct IOTrackingLeaksRef uint32_t zoneSize; uint32_t count; uint32_t found; + uint32_t foundzlen; size_t bytes; }; @@ -500,7 +516,7 @@ IOTrackingRemoveUser(IOTrackingQueue * queue, IOTrackingUser * mem) uint64_t gIOTrackingAddTime; void -IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address) +IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address, vm_tag_t tag) { IOTrackingCallSite * site; uint32_t crc, num; @@ -522,6 +538,7 @@ IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool addre que = &queue->sites[crc % queue->numSiteQs]; queue_iterate(que, site, IOTrackingCallSite *, link) { + if (tag != site->tag) continue; if (crc == site->crc) break; } @@ -534,6 +551,7 @@ IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool addre site->queue = queue; site->crc = crc; site->count = 0; + site->tag = tag; memset(&site->size[0], 0, sizeof(site->size)); bcopy(&bt[1], &site->bt[0], num * sizeof(site->bt[0])); assert(num <= kIOTrackingCallSiteBTs); @@ -587,6 +605,7 @@ IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size) queue->siteCount--; kfree(mem->site, sizeof(IOTrackingCallSite)); } + mem->site = NULL; } IOTRecursiveLockUnlock(&queue->lock); } @@ -608,7 +627,7 @@ IOTrackingAlloc(IOTrackingQueue * queue, uintptr_t address, size_t size) tracking->address = address; tracking->size = size; - IOTrackingAdd(queue, &tracking->tracking, size, true); + IOTrackingAdd(queue, &tracking->tracking, size, true, VM_KERN_MEMORY_NONE); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -790,14 +809,14 @@ IOTrackingLeakScan(void * refcon) uint64_t vaddr, vincr; ppnum_t ppn; uintptr_t ptr, addr, vphysaddr, inst; - size_t size; + size_t size, origsize; uint32_t baseIdx, lim, ptrIdx, count; boolean_t is; AbsoluteTime deadline; - instances = ref->instances; - count = ref->count; - size = ref->zoneSize; + instances = ref->instances; + count = ref->count; + size = origsize = ref->zoneSize; for (deadline = 0, vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS; ; @@ -835,13 +854,14 @@ IOTrackingLeakScan(void * refcon) } else if (kInstanceFlagAddress & inst) { - addr = ~((IOTrackingAddress *)instance)->address; - size = ((IOTrackingAddress *)instance)->size; + addr = ~((IOTrackingAddress *)instance)->address; + origsize = size = ((IOTrackingAddress *)instance)->size; + if (!size) size = 1; } else { - addr = (uintptr_t) (instance + 1); - size = instance->site->queue->allocSize; + addr = (uintptr_t) (instance + 1); + origsize = size = instance->site->queue->allocSize; } if ((ptr >= addr) && (ptr < (addr + size)) @@ -853,6 +873,7 @@ IOTrackingLeakScan(void * refcon) inst |= kInstanceFlagReferenced; instances[baseIdx + (lim >> 1)] = inst; ref->found++; + if (!origsize) ref->foundzlen++; } break; } @@ -948,7 +969,7 @@ IOTrackingLeaks(OSData * data) { ref.bytes = 0; IOTrackingLeakScan(&ref); - IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d\n", idx, ref.bytes / 1024 / 1024, count, ref.found); + IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d (zlen %d)\n", idx, ref.bytes / 1024 / 1024, count, ref.found, ref.foundzlen); if (count <= ref.found) break; } @@ -1022,8 +1043,9 @@ SkipName(uint32_t options, const char * name, size_t namesLen, const char * name /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -kern_return_t +static kern_return_t IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value, + uint32_t intag, uint32_t inzsize, const char * names, size_t namesLen, size_t size, OSObject ** result) { @@ -1119,6 +1141,7 @@ IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value, break; } + case kIOTrackingGetTracking: { if (kIOTrackingQueueTypeMap & queue->type) break; @@ -1135,11 +1158,51 @@ IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value, assert(idx < num); idx++; - if (size && ((site->size[0] + site->size[1]) < size)) continue; + size_t tsize[2]; + uint32_t count = site->count; + tsize[0] = site->size[0]; + tsize[1] = site->size[1]; + + if (intag || inzsize) + { + uintptr_t addr; + vm_size_t size, zoneSize; + vm_tag_t tag; + + if (kIOTrackingQueueTypeAlloc & queue->type) + { + addresses = false; + count = 0; + tsize[0] = tsize[1] = 0; + queue_iterate(&site->instances, instance, IOTracking *, link) + { + if (instance == site->addresses) addresses = true; + + if (addresses) addr = ~((IOTrackingAddress *)instance)->address; + else addr = (uintptr_t) (instance + 1); + + kr = vm_kern_allocation_info(addr, &size, &tag, &zoneSize); + if (KERN_SUCCESS != kr) continue; + + if ((VM_KERN_MEMORY_NONE != intag) && (intag != tag)) continue; + if (inzsize && (inzsize != zoneSize)) continue; + + count++; + tsize[0] += size; + } + } + else + { + if (!intag || inzsize || (intag != site->tag)) continue; + } + } + + if (!count) continue; + if (size && ((tsize[0] + tsize[1]) < size)) continue; - siteInfo.count = site->count; - siteInfo.size[0] = site->size[0]; - siteInfo.size[1] = site->size[1]; + siteInfo.count = count; + siteInfo.size[0] = tsize[0]; + siteInfo.size[1] = tsize[1]; CopyOutKernelBacktrace(site, &siteInfo); data->appendBytes(&siteInfo, sizeof(siteInfo)); @@ -1320,7 +1383,7 @@ IOReturn IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMet namesLen = args->structureInputSize - sizeof(IOKitDiagnosticsParameters); if (namesLen) names = (typeof(names))(params + 1); - ret = IOTrackingDebug(selector, params->options, params->value, names, namesLen, params->size, &result); + ret = IOTrackingDebug(selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result); if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) *args->structureVariableOutputData = result; else if (result) result->release(); diff --git a/iokit/Kernel/IOKitKernelInternal.h b/iokit/Kernel/IOKitKernelInternal.h index 630b39fb5..85507aa65 100644 --- a/iokit/Kernel/IOKitKernelInternal.h +++ b/iokit/Kernel/IOKitKernelInternal.h @@ -141,11 +141,9 @@ struct IODMACommandInternal UInt8 fDoubleBuffer; UInt8 fNewMD; UInt8 fLocalMapper; - - vm_tag_t fTag; -#if IOTRACKING - IOTracking fWireTracking; -#endif /* IOTRACKING */ + UInt8 fLocalMapperAllocValid; + UInt8 fIOVMAddrValid; + UInt8 fForceDoubleBuffer; vm_page_t fCopyPageAlloc; vm_page_t fCopyNext; @@ -217,6 +215,8 @@ extern "C" struct timeval gIOLastWakeTime; extern clock_sec_t gIOConsoleLockTime; +extern bool gCPUsRunning; + extern OSSet * gIORemoveOnReadProperties; extern "C" void IOKitInitializeTime( void ); @@ -230,6 +230,7 @@ extern "C" OSString * IOCopyLogNameForPID(int pid); extern const OSSymbol * gIOCreateEFIDevicePathSymbol; extern "C" void IOSetKeyStoreData(IOMemoryDescriptor * data); +extern "C" void IOSetAPFSKeyStoreData(IOMemoryDescriptor* data); #endif extern const OSSymbol * gAKSGetKey; diff --git a/iokit/Kernel/IOLib.cpp b/iokit/Kernel/IOLib.cpp index 73a0c67a8..c4a63b9dd 100644 --- a/iokit/Kernel/IOLib.cpp +++ b/iokit/Kernel/IOLib.cpp @@ -60,6 +60,7 @@ #include #include #include +#include #if IOKITSTATS @@ -191,7 +192,9 @@ void IOLibInit(void) &gIOKitPageableSpace.maps[0].address, kIOPageableMapSize, TRUE, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IOKIT), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_IOKIT, &gIOKitPageableSpace.maps[0].map); if (ret != KERN_SUCCESS) panic("failed to allocate iokit pageable map\n"); @@ -281,7 +284,7 @@ void * IOMalloc(vm_size_t size) bzero(&hdr->tracking, sizeof(hdr->tracking)); hdr->tracking.address = ~(((uintptr_t) address) + sizeofIOLibMallocHeader); hdr->tracking.size = size; - IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true); + IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); } #endif address = (typeof(address)) (((uintptr_t) address) + sizeofIOLibMallocHeader); @@ -420,7 +423,7 @@ void * IOMallocAligned(vm_size_t size, vm_size_t alignment) bzero(&hdr->tracking, sizeof(hdr->tracking)); hdr->tracking.address = ~address; hdr->tracking.size = size; - IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true); + IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); } #endif } else @@ -525,6 +528,9 @@ IOKernelFreePhysical(mach_vm_address_t address, mach_vm_size_t size) #endif } +#if __arm__ || __arm64__ +extern unsigned long gPhysBase, gPhysSize; +#endif mach_vm_address_t IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxPhys, @@ -560,6 +566,13 @@ IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxP if (!contiguous) { +#if __arm__ || __arm64__ + if (maxPhys >= (mach_vm_address_t)(gPhysBase + gPhysSize)) + { + maxPhys = 0; + } + else +#endif if (maxPhys <= 0xFFFFFFFF) { maxPhys = 0; @@ -613,7 +626,7 @@ IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxP bzero(&hdr->tracking, sizeof(hdr->tracking)); hdr->tracking.address = ~address; hdr->tracking.size = size; - IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true); + IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); } #endif } else @@ -778,7 +791,9 @@ kern_return_t IOIteratePageableMaps(vm_size_t size, &min, segSize, TRUE, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IOKIT), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_IOKIT, &map); if( KERN_SUCCESS != kr) { lck_mtx_unlock( gIOKitPageableSpace.lock ); @@ -1132,12 +1147,7 @@ void IOPause(unsigned nanoseconds) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -static void _iolog_consputc(int ch, void *arg __unused) -{ - cons_putc_locked(ch); -} - -static void _IOLogv(const char *format, va_list ap, void *caller); +static void _IOLogv(const char *format, va_list ap, void *caller) __printflike(1,0); __attribute__((noinline,not_tail_called)) void IOLog(const char *format, ...) @@ -1160,16 +1170,18 @@ void IOLogv(const char *format, va_list ap) void _IOLogv(const char *format, va_list ap, void *caller) { va_list ap2; - - /* Ideally not called at interrupt context or with interrupts disabled. Needs further validate */ - /* assert(TRUE == ml_get_interrupts_enabled()); */ + struct console_printbuf_state info_data; + console_printbuf_state_init(&info_data, TRUE, TRUE); va_copy(ap2, ap); os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, format, ap, caller); - __doprnt(format, ap2, _iolog_consputc, NULL, 16, TRUE); + __doprnt(format, ap2, console_printbuf_putc, &info_data, 16, TRUE); + console_printbuf_clear(&info_data); va_end(ap2); + + assertf(ml_get_interrupts_enabled() || ml_is_quiescing() || debug_mode_active() || !gCPUsRunning, "IOLog called with interrupts disabled"); } #if !__LP64__ diff --git a/iokit/Kernel/IOMapper.cpp b/iokit/Kernel/IOMapper.cpp index 89a00c921..7f944e831 100644 --- a/iokit/Kernel/IOMapper.cpp +++ b/iokit/Kernel/IOMapper.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include "IOKitKernelInternal.h" __BEGIN_DECLS @@ -142,20 +143,16 @@ IOMapper * IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned i OSDictionary * matching; obj = device->copyProperty("iommu-parent"); - if (!obj) - return (NULL); + if (!obj) return (NULL); - if ((mapper = OSDynamicCast(IOMapper, obj))) - return (mapper); + if ((mapper = OSDynamicCast(IOMapper, obj))) goto found; if ((data = OSDynamicCast(OSData, obj))) { - if (index >= data->getLength() / sizeof(UInt32)) - goto done; + if (index >= data->getLength() / sizeof(UInt32)) goto done; data = OSData::withBytesNoCopy((UInt32 *)data->getBytesNoCopy() + index, sizeof(UInt32)); - if (!data) - goto done; + if (!data) goto done; matching = IOService::propertyMatching(gIOMapperIDKey, data); data->release(); @@ -166,12 +163,31 @@ IOMapper * IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned i if (matching) { mapper = OSDynamicCast(IOMapper, IOService::waitForMatchingService(matching)); - matching->release(); + matching->release(); } done: - if (obj) - obj->release(); + if (obj) obj->release(); +found: + if (mapper) + { + if (!mapper->fAllocName) + { + char name[MACH_ZONE_NAME_MAX_LEN]; + char kmodname[KMOD_MAX_NAME]; + vm_tag_t tag; + uint32_t kmodid; + + tag = IOMemoryTag(kernel_map); + if (!(kmodid = vm_tag_get_kext(tag, &kmodname[0], KMOD_MAX_NAME))) + { + snprintf(kmodname, sizeof(kmodname), "%d", tag); + } + snprintf(name, sizeof(name), "%s.DMA.%s", kmodname, device->getName()); + mapper->fAllocName = kern_allocation_name_allocate(name, 16); + } + } + return (mapper); } diff --git a/iokit/Kernel/IOMemoryDescriptor.cpp b/iokit/Kernel/IOMemoryDescriptor.cpp index fa735f3e4..3b59323b8 100644 --- a/iokit/Kernel/IOMemoryDescriptor.cpp +++ b/iokit/Kernel/IOMemoryDescriptor.cpp @@ -104,8 +104,6 @@ static IORecursiveLock * gIOMemoryLock; #define DEBG(fmt, args...) {} #endif -#define IOMD_DEBUG_DMAACTIVE 1 - /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ // Some data structures and accessor macros used by the initWithOptions @@ -142,15 +140,14 @@ struct ioGMDData uint64_t fPreparationID; #if IOTRACKING IOTracking fWireTracking; - struct vm_tag_set fWireTags; - struct vm_tag_set_entry fWireTagsEntries[kMaxWireTags]; #endif /* IOTRACKING */ unsigned int fPageCnt; uint8_t fDMAMapNumAddressBits; - vm_tag_t fAllocTag; unsigned char fDiscontig:1; unsigned char fCompletionError:1; - unsigned char _resv:6; + unsigned char fMappedBaseValid:1; + unsigned char _resv:3; + unsigned char fDMAAccess:2; /* variable length arrays */ upl_page_info_t fPageList[1] @@ -170,6 +167,8 @@ struct ioGMDData #define computeDataSize(p, u) \ (offsetof(ioGMDData, fPageList) + p * sizeof(upl_page_info_t) + u * sizeof(ioPLBlock)) +enum { kIOMemoryHostOrRemote = kIOMemoryHostOnly | kIOMemoryRemote }; + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #define next_page(a) ( trunc_page(a) + PAGE_SIZE ) @@ -274,6 +273,14 @@ purgeableControlBits(IOOptionBits newState, vm_purgable_t * control, int * state err = kIOReturnBadArgument; break; } + + if (*control == VM_PURGABLE_SET_STATE) { + // let VM know this call is from the kernel and is allowed to alter + // the volatility of the memory entry even if it was created with + // MAP_MEM_PURGABLE_KERNEL_ONLY + *control = VM_PURGABLE_SET_STATE_FROM_KERNEL; + } + return (err); } @@ -328,6 +335,10 @@ vmProtForCacheMode(IOOptionBits cacheMode) SET_MAP_MEM(MAP_MEM_INNERWBACK, prot); break; + case kIOPostedWrite: + SET_MAP_MEM(MAP_MEM_POSTED, prot); + break; + case kIODefaultCache: default: SET_MAP_MEM(MAP_MEM_NOOP, prot); @@ -363,6 +374,10 @@ pagerFlagsForCacheMode(IOOptionBits cacheMode) pagerFlags = DEVICE_PAGER_COHERENT; break; + case kIOPostedWrite: + pagerFlags = DEVICE_PAGER_CACHE_INHIB | DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED | DEVICE_PAGER_EARLY_ACK; + break; + case kIODefaultCache: default: pagerFlags = -1U; @@ -517,7 +532,9 @@ IOGeneralMemoryDescriptor::memoryReferenceCreate( pagerFlags = IODefaultCacheBits(nextAddr); if (DEVICE_PAGER_CACHE_INHIB & pagerFlags) { - if (DEVICE_PAGER_GUARDED & pagerFlags) + if (DEVICE_PAGER_EARLY_ACK & pagerFlags) + mode = kIOPostedWrite; + else if (DEVICE_PAGER_GUARDED & pagerFlags) mode = kIOInhibitCache; else mode = kIOWriteCombineCache; @@ -554,7 +571,7 @@ IOGeneralMemoryDescriptor::memoryReferenceCreate( { // IOBufferMemoryDescriptor alloc - set flags for entry + object create prot |= MAP_MEM_NAMED_CREATE; - if (kIOMemoryBufferPurgeable & _flags) prot |= MAP_MEM_PURGABLE; + if (kIOMemoryBufferPurgeable & _flags) prot |= (MAP_MEM_PURGABLE | MAP_MEM_PURGABLE_KERNEL_ONLY); if (kIOMemoryUseReserve & _flags) prot |= MAP_MEM_GRAB_SECLUDED; prot |= VM_PROT_WRITE; @@ -703,8 +720,9 @@ IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref) (vm_map_offset_t) 0, (((ref->options & kIOMapAnywhere) ? VM_FLAGS_ANYWHERE - : VM_FLAGS_FIXED) - | VM_MAKE_TAG(ref->tag)), + : VM_FLAGS_FIXED)), + VM_MAP_KERNEL_FLAGS_NONE, + ref->tag, IPC_PORT_NULL, (memory_object_offset_t) 0, false, /* copy */ @@ -846,6 +864,15 @@ IOGeneralMemoryDescriptor::memoryReferenceMap( } } + /* + * If the memory is associated with a device pager but doesn't have a UPL, + * it will be immediately faulted in through the pager via populateDevicePager(). + * kIOMapPrefault is redundant in that case, so don't try to use it for UPL + * operations. + */ + if ((reserved != NULL) && (reserved->dp.devicePager) && (_memoryEntries == NULL) && (_wireCount != 0)) + options &= ~kIOMapPrefault; + /* * Prefaulting is only possible if we wired the memory earlier. Check the * memory type, and the underlying data. @@ -856,11 +883,9 @@ IOGeneralMemoryDescriptor::memoryReferenceMap( * The memory must have been wired by calling ::prepare(), otherwise * we don't have the UPL. Without UPLs, pages cannot be pre-faulted */ - assert(map != kernel_map); assert(_wireCount != 0); assert(_memoryEntries != NULL); - if ((map == kernel_map) || - (_wireCount == 0) || + if ((_wireCount == 0) || (_memoryEntries == NULL)) { return kIOReturnBadArgument; @@ -930,17 +955,23 @@ IOGeneralMemoryDescriptor::memoryReferenceMap( chunk = entry->size - entryOffset; if (chunk) { + vm_map_kernel_flags_t vmk_flags; + + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_iokit_acct = TRUE; /* iokit accounting */ + if (chunk > remain) chunk = remain; if (options & kIOMapPrefault) { UInt nb_pages = round_page(chunk) / PAGE_SIZE; + err = vm_map_enter_mem_object_prefault(map, &mapAddr, chunk, 0 /* mask */, - (VM_FLAGS_FIXED - | VM_FLAGS_OVERWRITE - | VM_MAKE_TAG(tag) - | VM_FLAGS_IOKIT_ACCT), /* iokit accounting */ + (VM_FLAGS_FIXED + | VM_FLAGS_OVERWRITE), + vmk_flags, + tag, entry->entry, entryOffset, prot, // cur @@ -958,9 +989,9 @@ IOGeneralMemoryDescriptor::memoryReferenceMap( &mapAddr, chunk, 0 /* mask */, (VM_FLAGS_FIXED - | VM_FLAGS_OVERWRITE - | VM_MAKE_TAG(tag) - | VM_FLAGS_IOKIT_ACCT), /* iokit accounting */ + | VM_FLAGS_OVERWRITE), + vmk_flags, + tag, entry->entry, entryOffset, false, // copy @@ -1042,7 +1073,7 @@ IOGeneralMemoryDescriptor::memoryReferenceSetPurgeable( err = purgeableControlBits(newState, &control, &state); if (KERN_SUCCESS != err) break; - err = mach_memory_entry_purgable_control(entries->entry, control, &state); + err = memory_entry_purgeable_control_internal(entries->entry, control, &state); if (KERN_SUCCESS != err) break; err = purgeableStateBits(&state); if (KERN_SUCCESS != err) break; @@ -1469,7 +1500,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, } // Grab the appropriate mapper - if (kIOMemoryHostOnly & options) options |= kIOMemoryMapperNone; + if (kIOMemoryHostOrRemote & options) options |= kIOMemoryMapperNone; if (kIOMemoryMapperNone & options) mapper = 0; // No Mapper else if (mapper == kIOMapperSystem) { @@ -1486,6 +1517,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, _direction = (IODirection) (_flags & kIOMemoryDirectionMask); #endif /* !__LP64__ */ + _dmaReferences = 0; __iomd_reservedA = 0; __iomd_reservedB = 0; _highestPage = 0; @@ -1509,7 +1541,20 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, if (!initMemoryEntries(dataSize, mapper)) return (false); dataP = getDataP(_memoryEntries); dataP->fPageCnt = 0; - + switch (kIOMemoryDirectionMask & options) + { + case kIODirectionOut: + dataP->fDMAAccess = kIODMAMapReadAccess; + break; + case kIODirectionIn: + dataP->fDMAAccess = kIODMAMapWriteAccess; + break; + case kIODirectionNone: + case kIODirectionOutIn: + default: + panic("bad dir for upl 0x%x\n", (int) options); + break; + } // _wireCount++; // UPLs start out life wired _length = count; @@ -1568,7 +1613,9 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, case kIOMemoryTypeVirtual64: case kIOMemoryTypePhysical64: if (count == 1 +#ifndef __arm__ && (((IOAddressRange *) buffers)->address + ((IOAddressRange *) buffers)->length) <= 0x100000000ULL +#endif ) { if (kIOMemoryTypeVirtual64 == type) type = kIOMemoryTypeVirtual; @@ -1604,6 +1651,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, break; } } + _rangesCount = count; // Find starting address within the vector of ranges Ranges vec = _ranges; @@ -1631,10 +1679,11 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, _length = totalLength; _pages = pages; - _rangesCount = count; // Auto-prepare memory at creation time. // Implied completion when descriptor is free-ed + + if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) _wireCount++; // Physical MDs are, by definition, wired else { /* kIOMemoryTypeVirtual | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO */ @@ -1648,6 +1697,12 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, dataP = getDataP(_memoryEntries); dataP->fPageCnt = _pages; + if (((_task != kernel_task) || (kIOMemoryBufferPageable & _flags)) + && (VM_KERN_MEMORY_NONE == _kernelTag)) + { + _kernelTag = IOMemoryTag(kernel_map); + } + if ( (kIOMemoryPersistent & _flags) && !_memRef) { IOReturn @@ -1682,10 +1737,10 @@ void IOGeneralMemoryDescriptor::free() if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) { ioGMDData * dataP; - if (_memoryEntries && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase) + if (_memoryEntries && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBaseValid) { - dataP->fMapper->iovmUnmapMemory(this, NULL, dataP->fMappedBase, dataP->fMappedLength); - dataP->fMappedBase = 0; + dmaUnmap(dataP->fMapper, NULL, 0, dataP->fMappedBase, dataP->fMappedLength); + dataP->fMappedBaseValid = dataP->fMappedBase = 0; } } else @@ -1774,6 +1829,11 @@ IOOptionBits IOMemoryDescriptor::getTag( void ) return( _tag); } +uint64_t IOMemoryDescriptor::getFlags(void) +{ + return (_flags); +} + #ifndef __LP64__ #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" @@ -1810,6 +1870,9 @@ IOByteCount IOMemoryDescriptor::readBytes return 0; } + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (0); + if (kIOMemoryThreadSafe & _flags) LOCK; @@ -1861,6 +1924,9 @@ IOByteCount IOMemoryDescriptor::writeBytes return 0; } + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (0); + if (kIOMemoryThreadSafe & _flags) LOCK; @@ -1961,22 +2027,21 @@ uint64_t IOMemoryDescriptor::getPreparationID( void ) void IOMemoryDescriptor::setVMTags(vm_tag_t kernelTag, vm_tag_t userTag) { - if (!getKernelReserved()) return; - reserved->kernelTag = kernelTag; - reserved->userTag = userTag; + _kernelTag = kernelTag; + _userTag = userTag; } vm_tag_t IOMemoryDescriptor::getVMTag(vm_map_t map) { - if (!reserved - || (VM_KERN_MEMORY_NONE == reserved->kernelTag) - || (VM_KERN_MEMORY_NONE == reserved->userTag)) + if (vm_kernel_map_is_kernel(map)) { - return (IOMemoryTag(map)); + if (VM_KERN_MEMORY_NONE != _kernelTag) return (_kernelTag); } - - if (vm_kernel_map_is_kernel(map)) return (reserved->kernelTag); - return (reserved->userTag); + else + { + if (VM_KERN_MEMORY_NONE != _userTag) return (_userTag); + } + return (IOMemoryTag(map)); } IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const @@ -2015,27 +2080,38 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * && ((dataP->fMappedBase + _length) > (1ULL << dataP->fDMAMapNumAddressBits)); remap |= (dataP->fDMAMapAlignment > page_size); - if (remap || !dataP->fMappedBase) + if (remap || !dataP->fMappedBaseValid) { -// if (dataP->fMappedBase) OSReportWithBacktrace("kIOMDDMAMap whole %d remap %d params %d\n", whole, remap, params); +// if (dataP->fMappedBaseValid) OSReportWithBacktrace("kIOMDDMAMap whole %d remap %d params %d\n", whole, remap, params); err = md->dmaMap(data->fMapper, data->fCommand, &data->fMapSpec, data->fOffset, data->fLength, &data->fAlloc, &data->fAllocLength); - if (keepMap && (kIOReturnSuccess == err) && !dataP->fMappedBase) + if (keepMap && (kIOReturnSuccess == err) && !dataP->fMappedBaseValid) { - dataP->fMappedBase = data->fAlloc; - dataP->fMappedLength = data->fAllocLength; - data->fAllocLength = 0; // IOMD owns the alloc now + dataP->fMappedBase = data->fAlloc; + dataP->fMappedBaseValid = true; + dataP->fMappedLength = data->fAllocLength; + data->fAllocLength = 0; // IOMD owns the alloc now } } else { data->fAlloc = dataP->fMappedBase; data->fAllocLength = 0; // give out IOMD map + md->dmaMapRecord(data->fMapper, data->fCommand, dataP->fMappedLength); } data->fMapContig = !dataP->fDiscontig; } - return (err); } + if (kIOMDDMAUnmap == op) + { + if (dataSize < sizeof(IOMDDMAMapArgs)) + return kIOReturnUnderrun; + IOMDDMAMapArgs * data = (IOMDDMAMapArgs *) vData; + + err = md->dmaUnmap(data->fMapper, data->fCommand, data->fOffset, data->fAlloc, data->fAllocLength); + + return kIOReturnSuccess; + } if (kIOMDAddDMAMapSpec == op) { @@ -2083,9 +2159,24 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * } } - return kIOReturnSuccess; + return kIOReturnSuccess; + } - } else if (kIOMDWalkSegments != op) + else if (kIOMDDMAActive == op) + { + if (params) + { + int16_t prior; + prior = OSAddAtomic16(1, &md->_dmaReferences); + if (!prior) md->_mapName = NULL; + } + else + { + if (md->_dmaReferences) OSAddAtomic16(-1, &md->_dmaReferences); + else panic("_dmaReferences underflow"); + } + } + else if (kIOMDWalkSegments != op) return kIOReturnBadArgument; // Get the next segment @@ -2104,10 +2195,12 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * UInt offset = isP->fIO.fOffset; bool mapped = isP->fIO.fMapped; + if (mapped && (kIOMemoryRemote & _flags)) return (kIOReturnNotAttached); + if (IOMapper::gSystem && mapped && (!(kIOMemoryHostOnly & _flags)) - && (!_memoryEntries || !getDataP(_memoryEntries)->fMappedBase)) -// && (_memoryEntries && !getDataP(_memoryEntries)->fMappedBase)) + && (!_memoryEntries || !getDataP(_memoryEntries)->fMappedBaseValid)) +// && (_memoryEntries && !getDataP(_memoryEntries)->fMappedBaseValid)) { if (!_memoryEntries && !md->initMemoryEntries(computeDataSize(0, 0), kIOMapperWaitSystem)) return (kIOReturnNoMemory); @@ -2121,6 +2214,7 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * mapSpec.alignment = dataP->fDMAMapAlignment; err = md->dmaMap(dataP->fMapper, NULL, &mapSpec, 0, _length, &dataP->fMappedBase, &dataP->fMappedLength); if (kIOReturnSuccess != err) return (err); + dataP->fMappedBaseValid = true; } } @@ -2157,7 +2251,7 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * address = physP[ind - 1].address + len - length; if (true && mapped && _memoryEntries - && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase) + && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBaseValid) { address = dataP->fMappedBase + offset; } @@ -2194,7 +2288,7 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * address = physP[ind - 1].address + len - length; if (true && mapped && _memoryEntries - && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase) + && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBaseValid) { address = dataP->fMappedBase + offset; } @@ -2245,7 +2339,7 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * // If a mapped address is requested and this is a pre-mapped IOPL // then just need to compute an offset relative to the mapped base. - if (mapped && dataP->fMappedBase) { + if (mapped && dataP->fMappedBaseValid) { offset += (ioplInfo.fPageOffset & PAGE_MASK); address = trunc_page_64(dataP->fMappedBase) + ptoa_64(ioplInfo.fMappedPage) + offset; continue; // Done leave do/while(false) now @@ -2368,7 +2462,7 @@ IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *l state->fOffset = offset; state->fLength = _length - offset; - state->fMapped = (0 == (options & kIOMemoryMapperNone)) && !(_flags & kIOMemoryHostOnly); + state->fMapped = (0 == (options & kIOMemoryMapperNone)) && !(_flags & kIOMemoryHostOrRemote); ret = dmaCommandOperation(kIOMDFirstSegment, _state, sizeof(_state)); @@ -2573,8 +2667,19 @@ IOMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt data data->fMapContig = true; err = md->dmaMap(data->fMapper, data->fCommand, &data->fMapSpec, data->fOffset, data->fLength, &data->fAlloc, &data->fAllocLength); + return (err); } + else if (kIOMDDMAUnmap == op) + { + if (dataSize < sizeof(IOMDDMAMapArgs)) + return kIOReturnUnderrun; + IOMDDMAMapArgs * data = (IOMDDMAMapArgs *) vData; + + err = md->dmaUnmap(data->fMapper, data->fCommand, data->fOffset, data->fAlloc, data->fAllocLength); + + return (kIOReturnSuccess); + } else return kIOReturnBadArgument; return kIOReturnSuccess; @@ -2589,6 +2694,9 @@ IOGeneralMemoryDescriptor::setPurgeable( IOOptionBits newState, vm_purgable_t control; int state; + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); + if (_memRef) { err = super::setPurgeable(newState, oldState); @@ -2612,7 +2720,14 @@ IOGeneralMemoryDescriptor::setPurgeable( IOOptionBits newState, break; } else + { curMap = get_task_map(_task); + if (NULL == curMap) + { + err = KERN_INVALID_ARGUMENT; + break; + } + } // can only do one range Ranges vec = _ranges; @@ -2624,7 +2739,7 @@ IOGeneralMemoryDescriptor::setPurgeable( IOOptionBits newState, err = purgeableControlBits(newState, &control, &state); if (kIOReturnSuccess != err) break; - err = mach_vm_purgable_control(curMap, addr, control, &state); + err = vm_map_purgable_control(curMap, addr, control, &state); if (oldState) { if (kIOReturnSuccess == err) @@ -2659,6 +2774,9 @@ IOReturn IOMemoryDescriptor::getPageCounts( IOByteCount * residentPageCount, { IOReturn err = kIOReturnNotReady; + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); + if (kIOMemoryThreadSafe & _flags) LOCK; if (_memRef) err = IOGeneralMemoryDescriptor::memoryReferenceGetPageCounts(_memRef, residentPageCount, dirtyPageCount); else @@ -2680,8 +2798,13 @@ IOReturn IOMemoryDescriptor::getPageCounts( IOByteCount * residentPageCount, } +#if defined(__arm__) || defined(__arm64__) +extern "C" void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count, unsigned int remaining, unsigned int *res); +extern "C" void dcache_incoherent_io_store64(addr64_t pa, unsigned int count, unsigned int remaining, unsigned int *res); +#else /* defined(__arm__) || defined(__arm64__) */ extern "C" void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count); extern "C" void dcache_incoherent_io_store64(addr64_t pa, unsigned int count); +#endif /* defined(__arm__) || defined(__arm64__) */ static void SetEncryptOp(addr64_t pa, unsigned int count) { @@ -2713,15 +2836,41 @@ IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options, IOByteCount remaining; unsigned int res; void (*func)(addr64_t pa, unsigned int count) = 0; +#if defined(__arm__) || defined(__arm64__) + void (*func_ext)(addr64_t pa, unsigned int count, unsigned int remaining, unsigned int *result) = 0; +#endif + + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); switch (options) { case kIOMemoryIncoherentIOFlush: +#if defined(__arm__) || defined(__arm64__) + func_ext = &dcache_incoherent_io_flush64; +#if __ARM_COHERENT_IO__ + func_ext(0, 0, 0, &res); + return kIOReturnSuccess; +#else /* __ARM_COHERENT_IO__ */ + break; +#endif /* __ARM_COHERENT_IO__ */ +#else /* defined(__arm__) || defined(__arm64__) */ func = &dcache_incoherent_io_flush64; break; +#endif /* defined(__arm__) || defined(__arm64__) */ case kIOMemoryIncoherentIOStore: +#if defined(__arm__) || defined(__arm64__) + func_ext = &dcache_incoherent_io_store64; +#if __ARM_COHERENT_IO__ + func_ext(0, 0, 0, &res); + return kIOReturnSuccess; +#else /* __ARM_COHERENT_IO__ */ + break; +#endif /* __ARM_COHERENT_IO__ */ +#else /* defined(__arm__) || defined(__arm64__) */ func = &dcache_incoherent_io_store64; break; +#endif /* defined(__arm__) || defined(__arm64__) */ case kIOMemorySetEncrypted: func = &SetEncryptOp; @@ -2731,8 +2880,13 @@ IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options, break; } +#if defined(__arm__) || defined(__arm64__) + if ((func == 0) && (func_ext == 0)) + return (kIOReturnUnsupported); +#else /* defined(__arm__) || defined(__arm64__) */ if (!func) return (kIOReturnUnsupported); +#endif /* defined(__arm__) || defined(__arm64__) */ if (kIOMemoryThreadSafe & _flags) LOCK; @@ -2753,7 +2907,19 @@ IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options, if (dstLen > remaining) dstLen = remaining; +#if defined(__arm__) || defined(__arm64__) + if (func) + (*func)(dstAddr64, dstLen); + if (func_ext) { + (*func_ext)(dstAddr64, dstLen, remaining, &res); + if (res != 0x0UL) { + remaining = 0; + break; + } + } +#else /* defined(__arm__) || defined(__arm64__) */ (*func)(dstAddr64, dstLen); +#endif /* defined(__arm__) || defined(__arm64__) */ offset += dstLen; remaining -= dstLen; @@ -2774,6 +2940,18 @@ IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options, #define io_kernel_static_start vm_kernel_stext #define io_kernel_static_end vm_kernel_etext +#elif defined(__arm__) || defined(__arm64__) + +extern vm_offset_t static_memory_end; + +#if defined(__arm64__) +#define io_kernel_static_start vm_kext_base +#else /* defined(__arm64__) */ +#define io_kernel_static_start vm_kernel_stext +#endif /* defined(__arm64__) */ + +#define io_kernel_static_end static_memory_end + #else #error io_kernel_static_end is undefined for this architecture #endif @@ -2818,43 +2996,6 @@ io_get_kernel_static_upl( return ((page >= pageCount) ? kIOReturnSuccess : kIOReturnVMError); } -/* - * - */ -#if IOTRACKING -static void -IOMemoryDescriptorUpdateWireOwner(ioGMDData * dataP, OSData * memoryEntries, vm_tag_t tag) -{ - ioPLBlock *ioplList; - UInt ind, count; - vm_tag_t prior; - - count = getNumIOPL(memoryEntries, dataP); - if (!count) return; - ioplList = getIOPLList(dataP); - - if (VM_KERN_MEMORY_NONE == tag) tag = dataP->fAllocTag; - assert(VM_KERN_MEMORY_NONE != tag); - - for (ind = 0; ind < count; ind++) - { - if (!ioplList[ind].fIOPL) continue; - prior = iopl_set_tag(ioplList[ind].fIOPL, tag); - if (VM_KERN_MEMORY_NONE == dataP->fAllocTag) dataP->fAllocTag = prior; -#if 0 - if (tag != prior) - { - char name[2][48]; - vm_tag_get_kext(prior, &name[0][0], sizeof(name[0])); - vm_tag_get_kext(tag, &name[1][0], sizeof(name[1])); - IOLog("switched %48s to %48s\n", name[0], name[1]); - } -#endif - } -} -#endif /* IOTRACKING */ - - IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) { IOOptionBits type = _flags & kIOMemoryTypeMask; @@ -2862,28 +3003,33 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) ioGMDData *dataP; upl_page_info_array_t pageInfo; ppnum_t mapBase; + vm_tag_t tag = VM_KERN_MEMORY_NONE; assert(kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type); if ((kIODirectionOutIn & forDirection) == kIODirectionNone) forDirection = (IODirection) (forDirection | getDirection()); + dataP = getDataP(_memoryEntries); upl_control_flags_t uplFlags; // This Mem Desc's default flags for upl creation switch (kIODirectionOutIn & forDirection) { - case kIODirectionOut: - // Pages do not need to be marked as dirty on commit - uplFlags = UPL_COPYOUT_FROM; - break; + case kIODirectionOut: + // Pages do not need to be marked as dirty on commit + uplFlags = UPL_COPYOUT_FROM; + dataP->fDMAAccess = kIODMAMapReadAccess; + break; - case kIODirectionIn: - default: - uplFlags = 0; // i.e. ~UPL_COPYOUT_FROM - break; - } - dataP = getDataP(_memoryEntries); + case kIODirectionIn: + dataP->fDMAAccess = kIODMAMapWriteAccess; + uplFlags = 0; // i.e. ~UPL_COPYOUT_FROM + break; - if (kIODirectionDMACommand & forDirection) assert(_wireCount); + default: + dataP->fDMAAccess = kIODMAMapReadAccess | kIODMAMapWriteAccess; + uplFlags = 0; // i.e. ~UPL_COPYOUT_FROM + break; + } if (_wireCount) { @@ -2896,11 +3042,13 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) else { IOMapper *mapper; + mapper = dataP->fMapper; - dataP->fMappedBase = 0; + dataP->fMappedBaseValid = dataP->fMappedBase = 0; uplFlags |= UPL_SET_IO_WIRE | UPL_SET_LITE; - uplFlags |= UPL_MEMORY_TAG_MAKE(getVMTag(kernel_map)); + tag = _kernelTag; + if (VM_KERN_MEMORY_NONE == tag) tag = IOMemoryTag(kernel_map); if (kIODirectionPrepareToPhys32 & forDirection) { @@ -3004,7 +3152,8 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) &iopl.fIOPL, baseInfo, &numPageInfo, - &ioplFlags); + &ioplFlags, + tag); } else { assert(theMap); @@ -3014,7 +3163,8 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) &iopl.fIOPL, baseInfo, &numPageInfo, - &ioplFlags); + &ioplFlags, + tag); } if (error != KERN_SUCCESS) goto abortExit; @@ -3071,23 +3221,13 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) } #if IOTRACKING - if (kIOReturnSuccess == error) + if (!(_flags & kIOMemoryAutoPrepare) && (kIOReturnSuccess == error)) { - vm_tag_t tag; - - dataP = getDataP(_memoryEntries); - if (forDirection & kIODirectionDMACommand) tag = (forDirection & kIODirectionDMACommandMask) >> kIODirectionDMACommandShift; - else tag = IOMemoryTag(kernel_map); - - if (!_wireCount) vm_tag_set_init(&dataP->fWireTags, kMaxWireTags); - vm_tag_set_enter(&dataP->fWireTags, kMaxWireTags, tag); - - IOMemoryDescriptorUpdateWireOwner(dataP, _memoryEntries, tag); - if (!_wireCount) - { - //if (!(_flags & kIOMemoryAutoPrepare)) - IOTrackingAdd(gIOWireTracking, &dataP->fWireTracking, ptoa(_pages), false); - } + dataP = getDataP(_memoryEntries); + if (!dataP->fWireTracking.link.next) + { + IOTrackingAdd(gIOWireTracking, &dataP->fWireTracking, ptoa(_pages), false, tag); + } } #endif /* IOTRACKING */ @@ -3145,6 +3285,7 @@ bool IOGeneralMemoryDescriptor::initMemoryEntries(size_t size, IOMapper * mapper dataP->fPreparationID = kIOPreparationIDUnprepared; dataP->fDiscontig = false; dataP->fCompletionError = false; + dataP->fMappedBaseValid = false; return (true); } @@ -3158,16 +3299,83 @@ IOReturn IOMemoryDescriptor::dmaMap( uint64_t * mapAddress, uint64_t * mapLength) { - IOReturn ret; + IOReturn err; uint32_t mapOptions; mapOptions = 0; mapOptions |= kIODMAMapReadAccess; if (!(kIOMemoryPreparedReadOnly & _flags)) mapOptions |= kIODMAMapWriteAccess; - ret = mapper->iovmMapMemory(this, offset, length, mapOptions, + err = mapper->iovmMapMemory(this, offset, length, mapOptions, mapSpec, command, NULL, mapAddress, mapLength); + if (kIOReturnSuccess == err) dmaMapRecord(mapper, command, *mapLength); + + return (err); +} + +void IOMemoryDescriptor::dmaMapRecord( + IOMapper * mapper, + IODMACommand * command, + uint64_t mapLength) +{ + kern_allocation_name_t alloc; + int16_t prior; + + if ((alloc = mapper->fAllocName) /* && mapper != IOMapper::gSystem */) + { + kern_allocation_update_size(mapper->fAllocName, mapLength); + } + + if (!command) return; + prior = OSAddAtomic16(1, &_dmaReferences); + if (!prior) + { + if (alloc && (VM_KERN_MEMORY_NONE != _kernelTag)) + { + _mapName = alloc; + mapLength = _length; + kern_allocation_update_subtotal(alloc, _kernelTag, mapLength); + } + else _mapName = NULL; + } +} + +IOReturn IOMemoryDescriptor::dmaUnmap( + IOMapper * mapper, + IODMACommand * command, + uint64_t offset, + uint64_t mapAddress, + uint64_t mapLength) +{ + IOReturn ret; + kern_allocation_name_t alloc; + kern_allocation_name_t mapName; + int16_t prior; + + mapName = 0; + prior = 0; + if (command) + { + mapName = _mapName; + if (_dmaReferences) prior = OSAddAtomic16(-1, &_dmaReferences); + else panic("_dmaReferences underflow"); + } + + if (!mapLength) return (kIOReturnSuccess); + + ret = mapper->iovmUnmapMemory(this, command, mapAddress, mapLength); + + if ((alloc = mapper->fAllocName)) + { + kern_allocation_update_size(alloc, -mapLength); + if ((1 == prior) && mapName && (VM_KERN_MEMORY_NONE != _kernelTag)) + { + mapLength = _length; + kern_allocation_update_subtotal(mapName, _kernelTag, -mapLength); + } + } + return (ret); } @@ -3186,6 +3394,7 @@ IOReturn IOGeneralMemoryDescriptor::dmaMap( *mapAddress = 0; if (kIOMemoryHostOnly & _flags) return (kIOReturnSuccess); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); if ((type == kIOMemoryTypePhysical) || (type == kIOMemoryTypePhysical64) || offset || (length != _length)) @@ -3217,8 +3426,8 @@ IOReturn IOGeneralMemoryDescriptor::dmaMap( mapOptions |= kIODMAMapPageListFullyOccupied; } - mapOptions |= kIODMAMapReadAccess; - if (!(kIOMemoryPreparedReadOnly & _flags)) mapOptions |= kIODMAMapWriteAccess; + assert(dataP->fDMAAccess); + mapOptions |= dataP->fDMAAccess; // Check for direct device non-paged memory if (ioplList->fFlags & kIOPLOnDevice) mapOptions |= kIODMAMapPhysicallyContiguous; @@ -3231,6 +3440,8 @@ IOReturn IOGeneralMemoryDescriptor::dmaMap( }; err = mapper->iovmMapMemory(this, offset, length, mapOptions, &mapSpec, command, &dmaPageList, mapAddress, mapLength); + + if (kIOReturnSuccess == err) dmaMapRecord(mapper, command, *mapLength); } return (err); @@ -3254,20 +3465,17 @@ IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection) if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) return kIOReturnSuccess; + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); + if (_prepareLock) IOLockLock(_prepareLock); - if (kIODirectionDMACommand & forDirection) - { -#if IOMD_DEBUG_DMAACTIVE - OSIncrementAtomic(&__iomd_reservedA); -#endif /* IOMD_DEBUG_DMAACTIVE */ - } if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) { error = wireVirtual(forDirection); } - if ((kIOReturnSuccess == error) && !(kIODirectionDMACommand & forDirection)) + if (kIOReturnSuccess == error) { if (1 == ++_wireCount) { @@ -3300,6 +3508,9 @@ IOReturn IOGeneralMemoryDescriptor::complete(IODirection forDirection) if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) return kIOReturnSuccess; + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); + if (_prepareLock) IOLockLock(_prepareLock); do { @@ -3308,26 +3519,6 @@ IOReturn IOGeneralMemoryDescriptor::complete(IODirection forDirection) dataP = getDataP(_memoryEntries); if (!dataP) break; -#if IOMD_DEBUG_DMAACTIVE - if (kIODirectionDMACommand & forDirection) - { - if (__iomd_reservedA) OSDecrementAtomic(&__iomd_reservedA); - else panic("kIOMDSetDMAInactive"); - } -#endif /* IOMD_DEBUG_DMAACTIVE */ -#if IOTRACKING - if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) - { - vm_tag_t tag; - - if (forDirection & kIODirectionDMACommand) tag = (forDirection & kIODirectionDMACommandMask) >> kIODirectionDMACommandShift; - else tag = IOMemoryTag(kernel_map); - vm_tag_set_remove(&dataP->fWireTags, kMaxWireTags, tag, &tag); - IOMemoryDescriptorUpdateWireOwner(dataP, _memoryEntries, tag); - } - if (kIODirectionDMACommand & forDirection) break; -#endif /* IOTRACKING */ - if (kIODirectionCompleteWithError & forDirection) dataP->fCompletionError = true; if ((kIOMemoryClearEncrypt & _flags) && (1 == _wireCount)) @@ -3346,30 +3537,28 @@ IOReturn IOGeneralMemoryDescriptor::complete(IODirection forDirection) // kIODirectionCompleteWithDataValid & forDirection if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) { + vm_tag_t tag; + tag = getVMTag(kernel_map); for (ind = 0; ind < count; ind++) { - if (ioplList[ind].fIOPL) iopl_valid_data(ioplList[ind].fIOPL); + if (ioplList[ind].fIOPL) iopl_valid_data(ioplList[ind].fIOPL, tag); } } } else { -#if IOMD_DEBUG_DMAACTIVE - if (__iomd_reservedA) panic("complete() while dma active"); -#endif /* IOMD_DEBUG_DMAACTIVE */ + if (_dmaReferences) panic("complete() while dma active"); - if (dataP->fMappedBase) { - dataP->fMapper->iovmUnmapMemory(this, NULL, dataP->fMappedBase, dataP->fMappedLength); - dataP->fMappedBase = 0; + if (dataP->fMappedBaseValid) { + dmaUnmap(dataP->fMapper, NULL, 0, dataP->fMappedBase, dataP->fMappedLength); + dataP->fMappedBaseValid = dataP->fMappedBase = 0; } - // Only complete iopls that we created which are for TypeVirtual - if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) { #if IOTRACKING - //if (!(_flags & kIOMemoryAutoPrepare)) - { - IOTrackingRemove(gIOWireTracking, &dataP->fWireTracking, ptoa(_pages)); - } + if (dataP->fWireTracking.link.next) IOTrackingRemove(gIOWireTracking, &dataP->fWireTracking, ptoa(_pages)); #endif /* IOTRACKING */ + // Only complete iopls that we created which are for TypeVirtual + if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) + { for (ind = 0; ind < count; ind++) if (ioplList[ind].fIOPL) { if (dataP->fCompletionError) @@ -3385,7 +3574,7 @@ IOReturn IOGeneralMemoryDescriptor::complete(IODirection forDirection) (void) _memoryEntries->initWithBytes(dataP, computeDataSize(0, 0)); // == setLength() dataP->fPreparationID = kIOPreparationIDUnprepared; - dataP->fAllocTag = VM_KERN_MEMORY_NONE; + _flags &= ~kIOMemoryPreparedReadOnly; } } } @@ -3422,6 +3611,9 @@ IOReturn IOGeneralMemoryDescriptor::doMap( if ((offset >= _length) || ((offset + length) > _length)) return( kIOReturnBadArgument ); + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (0); + if (vec.v) getAddrLenForInd(range0Addr, range0Len, type, vec, 0); @@ -3479,12 +3671,11 @@ IOReturn IOGeneralMemoryDescriptor::doMap( size = round_page(mapping->fLength); flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL - | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS - | UPL_MEMORY_TAG_MAKE(getVMTag(kernel_map)); + | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS; if (KERN_SUCCESS != memory_object_iopl_request(_memRef->entries[0].entry, 0, &size, &redirUPL2, NULL, NULL, - &flags)) + &flags, getVMTag(kernel_map))) redirUPL2 = NULL; for (lock_count = 0; @@ -3747,11 +3938,14 @@ IOReturn IOMemoryDescriptor::populateDevicePager( // faulting in later can't take place from interrupt level. if ((addressMap == kernel_map) && !(kIOMemoryRedirected & _flags)) { - vm_fault(addressMap, - (vm_map_offset_t)trunc_page_64(address), - VM_PROT_READ|VM_PROT_WRITE, - FALSE, THREAD_UNINT, NULL, - (vm_map_offset_t)0); + err = vm_fault(addressMap, + (vm_map_offset_t)trunc_page_64(address), + options & kIOMapReadOnly ? VM_PROT_READ : VM_PROT_READ|VM_PROT_WRITE, + FALSE, VM_KERN_MEMORY_NONE, + THREAD_UNINT, NULL, + (vm_map_offset_t)0); + + if (KERN_SUCCESS != err) break; } sourceOffset += segLen - pageOffset; @@ -4103,8 +4297,7 @@ IOReturn IOMemoryMap::wireRange( prot = (kIODirectionOutIn & options); if (prot) { - prot |= VM_PROT_MEMORY_TAG_MAKE(fMemory->getVMTag(kernel_map)); - kr = vm_map_wire(fAddressMap, start, end, prot, FALSE); + kr = vm_map_wire_kernel(fAddressMap, start, end, prot, fMemory->getVMTag(kernel_map), FALSE); } else { @@ -4266,11 +4459,10 @@ IOReturn IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory, { upl_size_t size = round_page(fLength); upl_control_flags_t flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL - | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS - | UPL_MEMORY_TAG_MAKE(fMemory->getVMTag(kernel_map)); + | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS; if (KERN_SUCCESS != memory_object_iopl_request(fMemory->_memRef->entries[0].entry, 0, &size, &fRedirUPL, NULL, NULL, - &flags)) + &flags, fMemory->getVMTag(kernel_map))) fRedirUPL = 0; if (physMem) diff --git a/iokit/Kernel/IONVRAM.cpp b/iokit/Kernel/IONVRAM.cpp index 6a819b459..bf7a07032 100644 --- a/iokit/Kernel/IONVRAM.cpp +++ b/iokit/Kernel/IONVRAM.cpp @@ -139,9 +139,11 @@ void IODTNVRAM::initNVRAMImage(void) // Look through the partitions to find the OF, MacOS partitions. while (currentOffset < kIODTNVRAMImageSize) { currentLength = ((UInt16 *)(_nvramImage + currentOffset))[1] * 16; - + + if (currentLength < 16) break; partitionOffset = currentOffset + 16; partitionLength = currentLength - 16; + if ((partitionOffset + partitionLength) > kIODTNVRAMImageSize) break; if (strncmp((const char *)_nvramImage + currentOffset + 4, kIODTNVRAMOFPartitionName, 12) == 0) { @@ -582,7 +584,7 @@ IOReturn IODTNVRAM::readNVRAMPartition(const OSSymbol *partitionID, IOByteCount length) { OSNumber *partitionOffsetNumber, *partitionLengthNumber; - UInt32 partitionOffset, partitionLength; + UInt32 partitionOffset, partitionLength, end; partitionOffsetNumber = (OSNumber *)_nvramPartitionOffsets->getObject(partitionID); @@ -595,8 +597,8 @@ IOReturn IODTNVRAM::readNVRAMPartition(const OSSymbol *partitionID, partitionOffset = partitionOffsetNumber->unsigned32BitValue(); partitionLength = partitionLengthNumber->unsigned32BitValue(); - if ((buffer == 0) || (length == 0) || - (offset + length > partitionLength)) + if (os_add_overflow(offset, length, &end)) return kIOReturnBadArgument; + if ((buffer == 0) || (length == 0) || (end > partitionLength)) return kIOReturnBadArgument; bcopy(_nvramImage + partitionOffset + offset, buffer, length); @@ -609,7 +611,7 @@ IOReturn IODTNVRAM::writeNVRAMPartition(const OSSymbol *partitionID, IOByteCount length) { OSNumber *partitionOffsetNumber, *partitionLengthNumber; - UInt32 partitionOffset, partitionLength; + UInt32 partitionOffset, partitionLength, end; partitionOffsetNumber = (OSNumber *)_nvramPartitionOffsets->getObject(partitionID); @@ -622,8 +624,8 @@ IOReturn IODTNVRAM::writeNVRAMPartition(const OSSymbol *partitionID, partitionOffset = partitionOffsetNumber->unsigned32BitValue(); partitionLength = partitionLengthNumber->unsigned32BitValue(); - if ((buffer == 0) || (length == 0) || - (offset + length > partitionLength)) + if (os_add_overflow(offset, length, &end)) return kIOReturnBadArgument; + if ((buffer == 0) || (length == 0) || (end > partitionLength)) return kIOReturnBadArgument; bcopy(buffer, _nvramImage + partitionOffset + offset, length); @@ -736,7 +738,6 @@ IOReturn IODTNVRAM::initOFVariables(void) } } - // Create the 'aapl,panic-info' property if needed. if (_piImage != 0) { propDataLength = *(UInt32 *)_piImage; if ((propDataLength != 0) && (propDataLength <= (_piPartitionSize - 4))) { @@ -824,6 +825,7 @@ enum { kOWVariableOffsetString = 17 }; +static const OFVariable gOFVariables[] = { {"little-endian?", kOFVariableTypeBoolean, kOFVariablePermUserRead, 0}, {"real-mode?", kOFVariableTypeBoolean, kOFVariablePermUserRead, 1}, @@ -872,12 +874,22 @@ OFVariable gOFVariables[] = { {"security-password", kOFVariableTypeData, kOFVariablePermRootOnly, -1}, {"boot-image", kOFVariableTypeData, kOFVariablePermUserWrite, -1}, {"com.apple.System.fp-state", kOFVariableTypeData, kOFVariablePermKernelOnly, -1}, +#if CONFIG_EMBEDDED + {"backlight-level", kOFVariableTypeData, kOFVariablePermUserWrite, -1}, + {"com.apple.System.sep.art", kOFVariableTypeData, kOFVariablePermKernelOnly, -1}, + {"com.apple.System.boot-nonce", kOFVariableTypeString, kOFVariablePermKernelOnly, -1}, + {"darkboot", kOFVariableTypeBoolean, kOFVariablePermUserWrite, -1}, + {"acc-mb-ld-lifetime", kOFVariableTypeNumber, kOFVariablePermKernelOnly, -1}, + {"acc-cm-override-charger-count", kOFVariableTypeNumber, kOFVariablePermKernelOnly, -1}, + {"acc-cm-override-count", kOFVariableTypeNumber, kOFVariablePermKernelOnly, -1}, + {"enter-tdm-mode", kOFVariableTypeBoolean, kOFVariablePermUserWrite, -1}, +#endif {0, kOFVariableTypeData, kOFVariablePermUserRead, -1} }; UInt32 IODTNVRAM::getOFVariableType(const OSSymbol *propSymbol) const { - OFVariable *ofVar; + const OFVariable *ofVar; ofVar = gOFVariables; while (1) { @@ -891,7 +903,7 @@ UInt32 IODTNVRAM::getOFVariableType(const OSSymbol *propSymbol) const UInt32 IODTNVRAM::getOFVariablePerm(const OSSymbol *propSymbol) const { - OFVariable *ofVar; + const OFVariable *ofVar; ofVar = gOFVariables; while (1) { @@ -906,7 +918,7 @@ UInt32 IODTNVRAM::getOFVariablePerm(const OSSymbol *propSymbol) const bool IODTNVRAM::getOWVariableInfo(UInt32 variableNumber, const OSSymbol **propSymbol, UInt32 *propType, UInt32 *propOffset) { - OFVariable *ofVar; + const OFVariable *ofVar; ofVar = gOFVariables; while (1) { @@ -999,7 +1011,7 @@ bool IODTNVRAM::convertObjectToProp(UInt8 *buffer, UInt32 *length, const OSSymbol *propSymbol, OSObject *propObject) { const UInt8 *propName; - UInt32 propNameLength, propDataLength; + UInt32 propNameLength, propDataLength, remaining; UInt32 propType, tmpValue; OSBoolean *tmpBoolean = 0; OSNumber *tmpNumber = 0; @@ -1043,29 +1055,30 @@ bool IODTNVRAM::convertObjectToProp(UInt8 *buffer, UInt32 *length, // Copy the property name equal sign. buffer += snprintf((char *)buffer, *length, "%s=", propName); - + remaining = *length - propNameLength - 1; + switch (propType) { case kOFVariableTypeBoolean : if (tmpBoolean->getValue()) { - strlcpy((char *)buffer, "true", *length - propNameLength); + strlcpy((char *)buffer, "true", remaining); } else { - strlcpy((char *)buffer, "false", *length - propNameLength); + strlcpy((char *)buffer, "false", remaining); } break; case kOFVariableTypeNumber : tmpValue = tmpNumber->unsigned32BitValue(); if (tmpValue == 0xFFFFFFFF) { - strlcpy((char *)buffer, "-1", *length - propNameLength); + strlcpy((char *)buffer, "-1", remaining); } else if (tmpValue < 1000) { - snprintf((char *)buffer, *length - propNameLength, "%d", (uint32_t)tmpValue); + snprintf((char *)buffer, remaining, "%d", (uint32_t)tmpValue); } else { - snprintf((char *)buffer, *length - propNameLength, "0x%x", (uint32_t)tmpValue); + snprintf((char *)buffer, remaining, "0x%x", (uint32_t)tmpValue); } break; case kOFVariableTypeString : - strlcpy((char *)buffer, tmpString->getCStringNoCopy(), *length - propNameLength); + strlcpy((char *)buffer, tmpString->getCStringNoCopy(), remaining); break; case kOFVariableTypeData : diff --git a/iokit/Kernel/IOPMrootDomain.cpp b/iokit/Kernel/IOPMrootDomain.cpp index 103a073c3..a28aa990f 100644 --- a/iokit/Kernel/IOPMrootDomain.cpp +++ b/iokit/Kernel/IOPMrootDomain.cpp @@ -57,6 +57,8 @@ #include #include #include +#include +#include #include #include "IOServicePrivate.h" // _IOServiceInterestNotifier @@ -83,14 +85,23 @@ __END_DECLS #define LOG(x...) \ do { kprintf(LOG_PREFIX x); } while (false) +#if DEVELOPMENT #define DLOG(x...) do { \ if (kIOLogPMRootDomain & gIOKitDebug) \ kprintf(LOG_PREFIX x); \ + else \ + os_log(OS_LOG_DEFAULT, LOG_PREFIX x); \ } while (false) +#else +#define DLOG(x...) do { \ + if (kIOLogPMRootDomain & gIOKitDebug) \ + kprintf(LOG_PREFIX x); \ +} while (false) +#endif #define DMSG(x...) do { \ if (kIOLogPMRootDomain & gIOKitDebug) { \ - kprintf(LOG_PREFIX x); IOLog(x); \ + kprintf(LOG_PREFIX x); \ } \ } while (false) @@ -185,7 +196,7 @@ static const OSSymbol *sleepMessagePEFunction = NULL; #define kIOPMSystemCapabilitiesKey "System Capabilities" #define kIORequestWranglerIdleKey "IORequestIdle" -#define kDefaultWranglerIdlePeriod 25 // in milliseconds +#define kDefaultWranglerIdlePeriod 1000 // in milliseconds #define kIOSleepWakeDebugKey "Persistent-memory-note" #define kIOEFIBootRomFailureKey "wake-failure" @@ -307,6 +318,14 @@ static UInt32 gWillShutdown = 0; static UInt32 gPagingOff = 0; static UInt32 gSleepWakeUUIDIsSet = false; static uint32_t gAggressivesState = 0; +static uint32_t gHaltTimeMaxLog; +static uint32_t gHaltTimeMaxPanic; +IOLock * gHaltLogLock; +static char * gHaltLog; +enum { kHaltLogSize = 2048 }; +static size_t gHaltLogPos; +static uint64_t gHaltStartTime; + uuid_string_t bootsessionuuid_string; @@ -349,10 +368,10 @@ enum { kInformableCount = 2 }; -const OSSymbol *gIOPMStatsApplicationResponseTimedOut; -const OSSymbol *gIOPMStatsApplicationResponseCancel; -const OSSymbol *gIOPMStatsApplicationResponseSlow; -const OSSymbol *gIOPMStatsApplicationResponsePrompt; +const OSSymbol *gIOPMStatsResponseTimedOut; +const OSSymbol *gIOPMStatsResponseCancel; +const OSSymbol *gIOPMStatsResponseSlow; +const OSSymbol *gIOPMStatsResponsePrompt; const OSSymbol *gIOPMStatsDriverPSChangeSlow; #define kBadPMFeatureID 0 @@ -560,53 +579,115 @@ static void IOPMRootDomainWillShutdown(void) } } -extern "C" +extern "C" IONotifier * registerSleepWakeInterest(IOServiceInterestHandler handler, void * self, void * ref) { - IONotifier * registerSleepWakeInterest(IOServiceInterestHandler handler, void * self, void * ref) - { - return gRootDomain->registerInterest( gIOGeneralInterest, handler, self, ref ); - } + return gRootDomain->registerInterest( gIOGeneralInterest, handler, self, ref ); +} - IONotifier * registerPrioritySleepWakeInterest(IOServiceInterestHandler handler, void * self, void * ref) - { - return gRootDomain->registerInterest( gIOPriorityPowerStateInterest, handler, self, ref ); - } +extern "C" IONotifier * registerPrioritySleepWakeInterest(IOServiceInterestHandler handler, void * self, void * ref) +{ + return gRootDomain->registerInterest( gIOPriorityPowerStateInterest, handler, self, ref ); +} - IOReturn acknowledgeSleepWakeNotification(void * PMrefcon) - { - return gRootDomain->allowPowerChange ( (unsigned long)PMrefcon ); - } +extern "C" IOReturn acknowledgeSleepWakeNotification(void * PMrefcon) +{ + return gRootDomain->allowPowerChange ( (unsigned long)PMrefcon ); +} - IOReturn vetoSleepWakeNotification(void * PMrefcon) - { - return gRootDomain->cancelPowerChange ( (unsigned long)PMrefcon ); - } +extern "C" IOReturn vetoSleepWakeNotification(void * PMrefcon) +{ + return gRootDomain->cancelPowerChange ( (unsigned long)PMrefcon ); +} - IOReturn rootDomainRestart ( void ) - { - return gRootDomain->restartSystem(); +extern "C" IOReturn rootDomainRestart ( void ) +{ + return gRootDomain->restartSystem(); +} + +extern "C" IOReturn rootDomainShutdown ( void ) +{ + return gRootDomain->shutdownSystem(); +} + +static void halt_log_putc(char c) +{ + if (gHaltLogPos >= (kHaltLogSize - 1)) return; + gHaltLog[gHaltLogPos++] = c; +} + +extern "C" void +_doprnt_log(const char *fmt, + va_list *argp, + void (*putc)(char), + int radix); + +static int +halt_log(const char *fmt, ...) +{ + va_list listp; + + va_start(listp, fmt); + _doprnt_log(fmt, &listp, &halt_log_putc, 16); + va_end(listp); + + return (0); +} + +extern "C" void +halt_log_enter(const char * what, const void * pc, uint64_t time) +{ + uint64_t nano, millis; + + if (!gHaltLog) return; + absolutetime_to_nanoseconds(time, &nano); + millis = nano / 1000000ULL; + if (millis < 100) return; + + IOLockLock(gHaltLogLock); + if (pc) { + halt_log("%s: %qd ms @ 0x%lx, ", what, millis, VM_KERNEL_UNSLIDE(pc)); + OSKext::printKextsInBacktrace((vm_offset_t *) &pc, 1, &halt_log, + OSKext::kPrintKextsLock | OSKext::kPrintKextsUnslide | OSKext::kPrintKextsTerse); + } else { + halt_log("%s: %qd ms\n", what, millis, VM_KERNEL_UNSLIDE(pc)); } + IOLockUnlock(gHaltLogLock); +} - IOReturn rootDomainShutdown ( void ) +extern uint32_t gFSState; + +extern "C" void IOSystemShutdownNotification(void) +{ + uint64_t startTime; + + IOLockLock(gHaltLogLock); + if (!gHaltLog) { - return gRootDomain->shutdownSystem(); + gHaltLog = IONew(char, kHaltLogSize); + gHaltStartTime = mach_absolute_time(); + if (gHaltLog) halt_log_putc('\n'); } + IOLockUnlock(gHaltLogLock); - void IOSystemShutdownNotification(void) - { - IOPMRootDomainWillShutdown(); + startTime = mach_absolute_time(); + IOPMRootDomainWillShutdown(); + halt_log_enter("IOPMRootDomainWillShutdown", 0, mach_absolute_time() - startTime); #if HIBERNATION - IOHibernateSystemPostWake(); + startTime = mach_absolute_time(); + IOHibernateSystemPostWake(true); + halt_log_enter("IOHibernateSystemPostWake", 0, mach_absolute_time() - startTime); +#endif + if (OSCompareAndSwap(0, 1, &gPagingOff)) + { +#if !CONFIG_EMBEDDED + gRootDomain->handlePlatformHaltRestart(kPEPagingOff); #endif - if (OSCompareAndSwap(0, 1, &gPagingOff)) - { - gRootDomain->handlePlatformHaltRestart(kPEPagingOff); - } } - - int sync_internal(void); } + +extern "C" int sync_internal(void); + /* A device is always in the highest power state which satisfies its driver, its policy-maker, and any power children it has, but within the constraint @@ -715,7 +796,7 @@ void IOPMrootDomain::swdDebugSetup( ) swd_DebugImageSetup = TRUE; if (CAP_GAIN(kIOPMSystemCapabilityGraphics) || (CAP_LOSS(kIOPMSystemCapabilityGraphics))) { - IOHibernateSystemPostWakeTrim((void*)1, NULL); + IOHibernateSystemPostWake(true); } IOOpenDebugDataFile(kSleepWakeStackBinFilename, SWD_BUF_SIZE); } @@ -769,7 +850,7 @@ static void disk_sync_callout( thread_call_param_t p0, thread_call_param_t p1 ) else { swdDebugTeardownCallout(p0, NULL); - IOHibernateSystemPostWake(); + IOHibernateSystemPostWake(false); if (gRootDomain) gRootDomain->sleepWakeDebugSaveSpinDumpFile(); @@ -781,16 +862,18 @@ static void disk_sync_callout( thread_call_param_t p0, thread_call_param_t p1 ) } //****************************************************************************** -static UInt32 computeDeltaTimeMS( const AbsoluteTime * startTime ) +static UInt32 computeDeltaTimeMS( const AbsoluteTime * startTime, AbsoluteTime * elapsedTime ) { AbsoluteTime endTime; UInt64 nano = 0; clock_get_uptime(&endTime); - if (CMP_ABSOLUTETIME(&endTime, startTime) > 0) + if (CMP_ABSOLUTETIME(&endTime, startTime) <= 0) *elapsedTime = 0; + else { SUB_ABSOLUTETIME(&endTime, startTime); absolutetime_to_nanoseconds(endTime, &nano); + *elapsedTime = endTime; } return (UInt32)(nano / 1000000ULL); @@ -807,12 +890,12 @@ sysctl_sleepwaketime SYSCTL_HANDLER_ARGS if (p == kernproc) { return sysctl_io_opaque(req, swt, sizeof(*swt), NULL); } else if(proc_is64bit(p)) { - struct user64_timeval t; + struct user64_timeval t = {}; t.tv_sec = swt->tv_sec; t.tv_usec = swt->tv_usec; return sysctl_io_opaque(req, &t, sizeof(t), NULL); } else { - struct user32_timeval t; + struct user32_timeval t = {}; t.tv_sec = swt->tv_sec; t.tv_usec = swt->tv_usec; return sysctl_io_opaque(req, &t, sizeof(t), NULL); @@ -850,7 +933,9 @@ static SYSCTL_PROC(_kern, OID_AUTO, willshutdown, 0, 0, sysctl_willshutdown, "I", ""); extern struct sysctl_oid sysctl__kern_iokittest; +extern struct sysctl_oid sysctl__debug_iokit; +#if !CONFIG_EMBEDDED static int sysctl_progressmeterenable @@ -888,6 +973,7 @@ static SYSCTL_PROC(_kern, OID_AUTO, progressmeter, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, 0, sysctl_progressmeter, "I", ""); +#endif /* !CONFIG_EMBEDDED */ @@ -1002,10 +1088,10 @@ bool IOPMrootDomain::start( IOService * nub ) gIOPMUserTriggeredFullWakeKey = OSSymbol::withCStringNoCopy(kIOPMUserTriggeredFullWakeKey); gIOPMUserIsActiveKey = OSSymbol::withCStringNoCopy(kIOPMUserIsActiveKey); - gIOPMStatsApplicationResponseTimedOut = OSSymbol::withCString(kIOPMStatsResponseTimedOut); - gIOPMStatsApplicationResponseCancel = OSSymbol::withCString(kIOPMStatsResponseCancel); - gIOPMStatsApplicationResponseSlow = OSSymbol::withCString(kIOPMStatsResponseSlow); - gIOPMStatsApplicationResponsePrompt = OSSymbol::withCString(kIOPMStatsResponsePrompt); + gIOPMStatsResponseTimedOut = OSSymbol::withCString(kIOPMStatsResponseTimedOut); + gIOPMStatsResponseCancel = OSSymbol::withCString(kIOPMStatsResponseCancel); + gIOPMStatsResponseSlow = OSSymbol::withCString(kIOPMStatsResponseSlow); + gIOPMStatsResponsePrompt = OSSymbol::withCString(kIOPMStatsResponsePrompt); gIOPMStatsDriverPSChangeSlow = OSSymbol::withCString(kIOPMStatsDriverPSChangeSlow); sleepSupportedPEFunction = OSSymbol::withCString("IOPMSetSleepSupported"); @@ -1034,6 +1120,8 @@ bool IOPMrootDomain::start( IOService * nub ) PE_parse_boot_argn("darkwake", &gDarkWakeFlags, sizeof(gDarkWakeFlags)); PE_parse_boot_argn("noidle", &gNoIdleFlag, sizeof(gNoIdleFlag)); + PE_parse_boot_argn("haltmspanic", &gHaltTimeMaxPanic, sizeof(gHaltTimeMaxPanic)); + PE_parse_boot_argn("haltmslog", &gHaltTimeMaxLog, sizeof(gHaltTimeMaxLog)); queue_init(&aggressivesQueue); aggressivesThreadCall = thread_call_allocate(handleAggressivesFunction, this); @@ -1043,6 +1131,7 @@ bool IOPMrootDomain::start( IOService * nub ) featuresDictLock = IOLockAlloc(); settingsCtrlLock = IOLockAlloc(); wakeEventLock = IOLockAlloc(); + gHaltLogLock = IOLockAlloc(); setPMRootDomain(this); extraSleepTimer = thread_call_allocate( @@ -1228,11 +1317,14 @@ bool IOPMrootDomain::start( IOService * nub ) sysctl_register_oid(&sysctl__kern_waketime); sysctl_register_oid(&sysctl__kern_willshutdown); sysctl_register_oid(&sysctl__kern_iokittest); + sysctl_register_oid(&sysctl__debug_iokit); sysctl_register_oid(&sysctl__hw_targettype); +#if !CONFIG_EMBEDDED sysctl_register_oid(&sysctl__kern_progressmeterenable); sysctl_register_oid(&sysctl__kern_progressmeter); sysctl_register_oid(&sysctl__kern_wakereason); +#endif /* !CONFIG_EMBEDDED */ sysctl_register_oid(&sysctl__kern_consoleoptions); sysctl_register_oid(&sysctl__kern_progressoptions); @@ -1386,6 +1478,7 @@ IOReturn IOPMrootDomain::setProperties( OSObject * props_obj ) setProperty(key, b); } else if (key->isEqualTo(kIOPMDeepSleepDelayKey) || + key->isEqualTo(kIOPMDeepSleepTimerKey) || key->isEqualTo(kIOPMAutoPowerOffDelayKey) || key->isEqualTo(kIOPMAutoPowerOffTimerKey)) { @@ -2191,6 +2284,9 @@ IOReturn IOPMrootDomain::privateSleepSystem( uint32_t sleepReason ) void IOPMrootDomain::powerChangeDone( unsigned long previousPowerState ) { +#if !__i386__ && !__x86_64__ + uint64_t timeSinceReset = 0; +#endif uint64_t now; ASSERT_GATED(); DLOG("PowerChangeDone: %u->%u\n", @@ -2273,7 +2369,7 @@ void IOPMrootDomain::powerChangeDone( unsigned long previousPowerState ) clock_sec_t wakeSecs; clock_usec_t wakeMicrosecs; - clock_initialize_calendar(); + clock_wakeup_calendar(); clock_get_calendar_microtime(&wakeSecs, &wakeMicrosecs); gIOLastWakeTime.tv_sec = wakeSecs; @@ -2438,7 +2534,9 @@ void IOPMrootDomain::powerChangeDone( unsigned long previousPowerState ) } } #else /* !__i386__ && !__x86_64__ */ - kdebugTrace(kPMLogSystemWake, 0, ml_get_wake_timebase() >> 32, ml_get_wake_timebase()); + timeSinceReset = ml_get_time_since_reset(); + + kdebugTrace(kPMLogSystemWake, 0, timeSinceReset >> 32, timeSinceReset); // stay awake for at least 30 seconds wranglerTickled = true; fullWakeReason = kFullWakeReasonLocalUser; @@ -2547,16 +2645,23 @@ bool IOPMrootDomain::updatePreventIdleSleepList( #if defined(__i386__) || defined(__x86_64__) if (addNotRemove && (service == wrangler) && !checkSystemCanSustainFullWake()) { + DLOG("Cannot cancel idle sleep\n"); return false; // do not idle-cancel } #endif - MSG("prevent idle sleep list: %s%c (%u)\n", - service->getName(), - (addNotRemove) ? '+' : '-', newCount); return true; } +//****************************************************************************** +// startSpinDump +//****************************************************************************** + +void IOPMrootDomain::startSpinDump(uint32_t spindumpKind) +{ + messageClients(kIOPMMessageLaunchBootSpinDump, (void *)(uintptr_t)spindumpKind); +} + //****************************************************************************** // preventSystemSleepListUpdate // @@ -2681,21 +2786,23 @@ bool IOPMrootDomain::tellChangeDown( unsigned long stateNum ) tracePoint( kIOPMTracePointSleepPriorityClients ); } - if ((SLEEP_STATE == stateNum) && !ignoreTellChangeDown) - { + if (!ignoreTellChangeDown) { userActivityAtSleep = userActivityCount; - hibernateAborted = false; DLOG("tellChangeDown::userActivityAtSleep %d\n", userActivityAtSleep); - // Direct callout into OSKext so it can disable kext unloads - // during sleep/wake to prevent deadlocks. - OSKextSystemSleepOrWake( kIOMessageSystemWillSleep ); + if (SLEEP_STATE == stateNum) { + hibernateAborted = false; - IOService::updateConsoleUsers(NULL, kIOMessageSystemWillSleep); + // Direct callout into OSKext so it can disable kext unloads + // during sleep/wake to prevent deadlocks. + OSKextSystemSleepOrWake( kIOMessageSystemWillSleep ); - // Two change downs are sent by IOServicePM. Ignore the 2nd. - // But tellClientsWithResponse() must be called for both. - ignoreTellChangeDown = true; + IOService::updateConsoleUsers(NULL, kIOMessageSystemWillSleep); + + // Two change downs are sent by IOServicePM. Ignore the 2nd. + // But tellClientsWithResponse() must be called for both. + ignoreTellChangeDown = true; + } } return super::tellClientsWithResponse( kIOMessageSystemWillSleep ); @@ -3224,7 +3331,9 @@ void IOPMrootDomain::willNotifyPowerChildren( IOPMPowerStateIndex newPowerState tasks_system_suspend(tasksSuspended); clock_interval_to_deadline(10, kSecondScale, &deadline); +#if !CONFIG_EMBEDDED vm_pageout_wait(AbsoluteTime_to_scalar(&deadline)); +#endif /* !CONFIG_EMBEDDED */ } #if HIBERNATION @@ -4012,6 +4121,7 @@ bool IOPMrootDomain::evaluateSystemSleepPolicy( uint32_t standbyDelay = 0; uint32_t powerOffDelay = 0; uint32_t powerOffTimer = 0; + uint32_t standbyTimer = 0; uint32_t mismatch; bool standbyEnabled; bool powerOffEnabled; @@ -4031,9 +4141,11 @@ bool IOPMrootDomain::evaluateSystemSleepPolicy( && (getProperty(kIOPMAutoPowerOffEnabledKey) == kOSBooleanTrue)); if (!getSleepOption(kIOPMAutoPowerOffTimerKey, &powerOffTimer)) powerOffTimer = powerOffDelay; + if (!getSleepOption(kIOPMDeepSleepTimerKey, &standbyTimer)) + standbyTimer = standbyDelay; - DLOG("phase %d, standby %d delay %u, poweroff %d delay %u timer %u, hibernate 0x%x\n", - sleepPhase, standbyEnabled, standbyDelay, + DLOG("phase %d, standby %d delay %u timer %u, poweroff %d delay %u timer %u, hibernate 0x%x\n", + sleepPhase, standbyEnabled, standbyDelay, standbyTimer, powerOffEnabled, powerOffDelay, powerOffTimer, *hibMode); // pmset level overrides @@ -4069,7 +4181,7 @@ bool IOPMrootDomain::evaluateSystemSleepPolicy( currentFactors |= kIOPMSleepFactorACPower; if (lowBatteryCondition) currentFactors |= kIOPMSleepFactorBatteryLow; - if (!standbyDelay) + if (!standbyDelay || !standbyTimer) currentFactors |= kIOPMSleepFactorStandbyNoDelay; if (standbyNixed || !standbyEnabled) currentFactors |= kIOPMSleepFactorStandbyDisabled; @@ -4134,6 +4246,7 @@ bool IOPMrootDomain::evaluateSystemSleepPolicy( gSleepPolicyVars->sleepReason = lastSleepReason; gSleepPolicyVars->sleepPhase = sleepPhase; gSleepPolicyVars->standbyDelay = standbyDelay; + gSleepPolicyVars->standbyTimer = standbyTimer; gSleepPolicyVars->poweroffDelay = powerOffDelay; gSleepPolicyVars->scheduledAlarms = _scheduledAlarms | _userScheduledAlarm; gSleepPolicyVars->poweroffTimer = powerOffTimer; @@ -4332,7 +4445,7 @@ void IOPMrootDomain::evaluateSystemSleepPolicyFinal( void ) if (evaluateSystemSleepPolicy(¶ms, kIOPMSleepPhase2, &hibernateMode)) { if ((kIOPMSleepTypeStandby == params.sleepType) - && gIOHibernateStandbyDisabled + && gIOHibernateStandbyDisabled && gSleepPolicyVars && (!(kIOPMSleepFactorStandbyForced & gSleepPolicyVars->sleepFactors))) { standbyNixed = true; @@ -4446,11 +4559,11 @@ bool IOPMrootDomain::getSleepOption( const char * key, uint32_t * option ) if (optionsProp) optionsProp->release(); - return true; + return ok; } #endif /* HIBERNATION */ -IOReturn IOPMrootDomain::getSystemSleepType( uint32_t * sleepType ) +IOReturn IOPMrootDomain::getSystemSleepType( uint32_t * sleepType, uint32_t * standbyTimer ) { #if HIBERNATION IOPMSystemSleepParameters params; @@ -4463,7 +4576,7 @@ IOReturn IOPMrootDomain::getSystemSleepType( uint32_t * sleepType ) OSMemberFunctionCast(IOWorkLoop::Action, this, &IOPMrootDomain::getSystemSleepType), (OSObject *) this, - (void *) sleepType); + (void *) sleepType, (void *) standbyTimer); return ret; } @@ -4474,6 +4587,11 @@ IOReturn IOPMrootDomain::getSystemSleepType( uint32_t * sleepType ) if (ok) { *sleepType = params.sleepType; + if (!getSleepOption(kIOPMDeepSleepTimerKey, standbyTimer) && + !getSleepOption(kIOPMDeepSleepDelayKey, standbyTimer)) { + DLOG("Standby delay is not set\n"); + *standbyTimer = 0; + } return kIOReturnSuccess; } #endif @@ -4503,7 +4621,7 @@ platformHaltRestartApplier( OSObject * object, void * context ) { IOPowerStateChangeNotification notify; HaltRestartApplierContext * ctx; - AbsoluteTime startTime; + AbsoluteTime startTime, elapsedTime; uint32_t deltaTime; ctx = (HaltRestartApplierContext *) context; @@ -4516,7 +4634,7 @@ platformHaltRestartApplier( OSObject * object, void * context ) clock_get_uptime(&startTime); ctx->RootDomain->messageClient( ctx->MessageType, object, (void *)¬ify ); - deltaTime = computeDeltaTimeMS(&startTime); + deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime); if ((deltaTime > kPMHaltTimeoutMS) || (gIOKitDebug & kIOLogPMRootDomain)) @@ -4531,6 +4649,7 @@ platformHaltRestartApplier( OSObject * object, void * context ) { LOG("%s handler %p took %u ms\n", ctx->LogString, OBFUSCATE(notifier->handler), deltaTime); + halt_log_enter(ctx->LogString, (const void *) notifier->handler, elapsedTime); } } @@ -4548,7 +4667,7 @@ static void quiescePowerTreeCallback( void * target, void * param ) void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type ) { HaltRestartApplierContext ctx; - AbsoluteTime startTime; + AbsoluteTime startTime, elapsedTime; uint32_t deltaTime; memset(&ctx, 0, sizeof(ctx)); @@ -4624,13 +4743,28 @@ void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type ) } } IOLockUnlock(gPMHaltLock); - - deltaTime = computeDeltaTimeMS(&quiesceTime); + deltaTime = computeDeltaTimeMS(&quiesceTime, &elapsedTime); DLOG("PM quiesce took %u ms\n", deltaTime); + halt_log_enter("Quiesce", NULL, elapsedTime); } - deltaTime = computeDeltaTimeMS(&startTime); + deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime); LOG("%s all drivers took %u ms\n", ctx.LogString, deltaTime); + + halt_log_enter(ctx.LogString, NULL, elapsedTime); + if (gHaltLog) gHaltLog[gHaltLogPos] = 0; + + deltaTime = computeDeltaTimeMS(&gHaltStartTime, &elapsedTime); + LOG("%s total %u ms\n", ctx.LogString, deltaTime); + + if (gHaltLog && gHaltTimeMaxLog && (deltaTime >= gHaltTimeMaxLog)) + { + printf("%s total %d ms:%s\n", ctx.LogString, deltaTime, gHaltLog); + } + if (gHaltLog && gHaltTimeMaxPanic && (deltaTime >= gHaltTimeMaxPanic)) + { + panic("%s total %d ms:%s\n", ctx.LogString, deltaTime, gHaltLog); + } } //****************************************************************************** @@ -5132,10 +5266,20 @@ void IOPMrootDomain::handleOurPowerChangeDone( if (!CAP_CURRENT(kIOPMSystemCapabilityGraphics) && CAP_CURRENT(kIOPMSystemCapabilityCPU)) { +#if !CONFIG_EMBEDDED pmPowerStateQueue->submitPowerEvent( kPowerEventPolicyStimulus, (void *) kStimulusDarkWakeReentry, _systemStateGeneration ); +#else + // On embedded, there are no factors that can prolong a + // "darkWake" when a power down is vetoed. We need to + // promote to "fullWake" at least once so that factors + // that prevent idle sleep can assert themselves if required + pmPowerStateQueue->submitPowerEvent( + kPowerEventPolicyStimulus, + (void *) kStimulusDarkWakeActivityTickle); +#endif } // Revert device desire to max. @@ -5604,6 +5748,9 @@ protected: uint32_t ackTimeoutCnt; uint32_t msgType; // Message pending ack + uint64_t uuid0; + uint64_t uuid1; + const OSSymbol *identifier; }; OSDefineMetaClassAndStructors(IOPMServiceInterestNotifier, _IOServiceInterestNotifier) @@ -5633,7 +5780,7 @@ IONotifier * IOPMrootDomain::registerInterest( if (!notifier) return NULL; if (notifier->init()) { - rc = super::registerInterestForNotifer(notifier, typeOfInterest, handler, target, ref); + rc = super::registerInterestForNotifier(notifier, typeOfInterest, handler, target, ref); } if (rc != kIOReturnSuccess) { notifier->release(); @@ -5661,6 +5808,28 @@ IONotifier * IOPMrootDomain::registerInterest( } } + OSData *data = NULL; + uint8_t *uuid = NULL; + OSKext *kext = OSKext::lookupKextWithAddress((vm_address_t)handler); + if (kext) { + data = kext->copyUUID(); + } + if (data && (data->getLength() == sizeof(uuid_t))) { + uuid = (uint8_t *)(data->getBytesNoCopy()); + + notifier->uuid0 = ((uint64_t)(uuid[0]) << 56) | ((uint64_t)(uuid[1]) << 48) | ((uint64_t)(uuid[2]) << 40)| + ((uint64_t)(uuid[3]) << 32) | ((uint64_t)(uuid[4]) << 24) | ((uint64_t)(uuid[5]) << 16) | + ((uint64_t)(uuid[6]) << 8) | (uuid[7]); + notifier->uuid1 = ((uint64_t)(uuid[8]) << 56) | ((uint64_t)(uuid[9]) << 48) | ((uint64_t)(uuid[10]) << 40)| + ((uint64_t)(uuid[11]) << 32) | ((uint64_t)(uuid[12]) << 24) | ((uint64_t)(uuid[13]) << 16) | + ((uint64_t)(uuid[14]) << 8) | (uuid[15]); + + notifier->identifier = kext->getIdentifier(); + + } + if (kext) kext->release(); + if (data) data->release(); + return notifier; } @@ -5737,9 +5906,6 @@ bool IOPMrootDomain::systemMessageFilter( // app has not replied yet, wait for it *((OSObject **) arg3) = kOSBooleanFalse; - if (notifier) { - notifier->msgType = context->messageType; - } } allow = true; @@ -5757,9 +5923,6 @@ bool IOPMrootDomain::systemMessageFilter( if (object == (OSObject *) systemCapabilityNotifier) { allow = true; - if (notifier) { - notifier->msgType = context->messageType; - } break; } @@ -5775,9 +5938,6 @@ bool IOPMrootDomain::systemMessageFilter( if ((object == (OSObject *) systemCapabilityNotifier) && CAP_HIGHEST(kIOPMSystemCapabilityGraphics) && (fullToDarkReason == kIOPMSleepReasonIdle)) { - if (notifier) { - notifier->msgType = context->messageType; - } allow = true; } break; @@ -5805,8 +5965,6 @@ bool IOPMrootDomain::systemMessageFilter( else *((OSObject **) arg3) = kOSBooleanTrue; } - - notifier->msgType = context->messageType; } } else if ((context->notifyType == kNotifyPriority) && @@ -5828,6 +5986,9 @@ bool IOPMrootDomain::systemMessageFilter( _joinedCapabilityClients = 0; } } + if (notifier) { + notifier->msgType = context->messageType; + } return allow; } @@ -6329,20 +6490,22 @@ void IOPMrootDomain::handleDisplayPowerOn( ) void IOPMrootDomain::dispatchPowerEvent( uint32_t event, void * arg0, uint64_t arg1 ) { - DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); ASSERT_GATED(); switch (event) { case kPowerEventFeatureChanged: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); messageClients(kIOPMMessageFeatureChange, this); break; case kPowerEventReceivedPowerNotification: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); handlePowerNotification( (UInt32)(uintptr_t) arg0 ); break; case kPowerEventSystemBootCompleted: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (systemBooting) { systemBooting = false; @@ -6382,6 +6545,7 @@ void IOPMrootDomain::dispatchPowerEvent( break; case kPowerEventSystemShutdown: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (kOSBooleanTrue == (OSBoolean *) arg0) { /* We set systemShutdown = true during shutdown @@ -6396,18 +6560,20 @@ void IOPMrootDomain::dispatchPowerEvent( systemShutdown = true; } else { /* - A shutdown was initiated, but then the shutdown - was cancelled, clearing systemShutdown to false here. - */ + A shutdown was initiated, but then the shutdown + was cancelled, clearing systemShutdown to false here. + */ systemShutdown = false; } break; case kPowerEventUserDisabledSleep: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); userDisabledAllSleep = (kOSBooleanTrue == (OSBoolean *) arg0); break; case kPowerEventRegisterSystemCapabilityClient: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (systemCapabilityNotifier) { systemCapabilityNotifier->release(); @@ -6422,6 +6588,7 @@ void IOPMrootDomain::dispatchPowerEvent( [[clang::fallthrough]]; case kPowerEventRegisterKernelCapabilityClient: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (!_joinedCapabilityClients) _joinedCapabilityClients = OSSet::withCapacity(8); if (arg0) @@ -6437,6 +6604,7 @@ void IOPMrootDomain::dispatchPowerEvent( break; case kPowerEventPolicyStimulus: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (arg0) { int stimulus = (uintptr_t) arg0; @@ -6445,6 +6613,7 @@ void IOPMrootDomain::dispatchPowerEvent( break; case kPowerEventAssertionCreate: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (pmAssertions) { pmAssertions->handleCreateAssertion((OSData *)arg0); } @@ -6452,25 +6621,30 @@ void IOPMrootDomain::dispatchPowerEvent( case kPowerEventAssertionRelease: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (pmAssertions) { pmAssertions->handleReleaseAssertion(arg1); } break; case kPowerEventAssertionSetLevel: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (pmAssertions) { pmAssertions->handleSetAssertionLevel(arg1, (IOPMDriverAssertionLevel)(uintptr_t)arg0); } break; case kPowerEventQueueSleepWakeUUID: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); handleQueueSleepWakeUUID((OSObject *)arg0); break; case kPowerEventPublishSleepWakeUUID: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); handlePublishSleepWakeUUID((bool)arg0); break; case kPowerEventSetDisplayPowerOn: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (!wrangler) break; if (arg1 != 0) { @@ -6655,6 +6829,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMClamshellOpened) { + DLOG("Clamshell opened\n"); // Received clamshel open message from clamshell controlling driver // Update our internal state and tell general interest clients clamshellClosed = false; @@ -6686,6 +6861,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMClamshellClosed) { + DLOG("Clamshell closed\n"); // Received clamshel open message from clamshell controlling driver // Update our internal state and tell general interest clients clamshellClosed = true; @@ -6708,6 +6884,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMSetDesktopMode) { + DLOG("Desktop mode\n"); desktopMode = (0 != (msg & kIOPMSetValue)); msg &= ~(kIOPMSetDesktopMode | kIOPMSetValue); @@ -6761,6 +6938,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMEnableClamshell) { + DLOG("Clamshell enabled\n"); // Re-evaluate the lid state // System should sleep on external display disappearance // in lid closed operation. @@ -6780,6 +6958,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMDisableClamshell) { + DLOG("Clamshell disabled\n"); clamshellDisabled = true; sendClientClamshellNotification(); } @@ -6800,6 +6979,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMPowerButton) { + DLOG("Powerbutton press\n"); if (!wranglerAsleep) { OSString *pbs = OSString::withCString("DisablePowerButtonSleep"); @@ -6835,7 +7015,6 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) uint32_t u32; } flags; - DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); ASSERT_GATED(); flags.u32 = 0; @@ -6843,6 +7022,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) switch (stimulus) { case kStimulusDisplayWranglerSleep: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); if (!wranglerAsleep) { // first transition to wrangler sleep or lower @@ -6851,11 +7031,13 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) break; case kStimulusDisplayWranglerWake: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); displayIdleForDemandSleep = false; wranglerAsleep = false; break; case kStimulusEnterUserActiveState: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); if (_preventUserActive) { DLOG("user active dropped\n"); @@ -6881,6 +7063,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) break; case kStimulusLeaveUserActiveState: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); if (userIsActive) { userIsActive = false; @@ -6895,6 +7078,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) case kStimulusAggressivenessChanged: { + DMSG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); unsigned long minutesToIdleSleep = 0; unsigned long minutesToDisplayDim = 0; unsigned long minutesDelta = 0; @@ -6947,6 +7131,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) } break; case kStimulusDemandSystemSleep: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); displayIdleForDemandSleep = true; if (wrangler && wranglerIdleSettings) { @@ -6963,10 +7148,12 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) break; case kStimulusAllowSystemSleepChanged: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); flags.bit.adjustPowerState = true; break; case kStimulusDarkWakeActivityTickle: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); // arg == true implies real and not self generated wrangler tickle. // Update wake type on PM work loop instead of the tickle thread to // eliminate the possibility of an early tickle clobbering the wake @@ -6990,6 +7177,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) case kStimulusDarkWakeEntry: case kStimulusDarkWakeReentry: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); // Any system transitions since the last dark wake transition // will invalid the stimulus. @@ -7006,6 +7194,10 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) { clock_get_uptime(&userBecameInactiveTime); flags.bit.evaluateDarkWake = true; + if (activitySinceSleep()) { + DLOG("User activity recorded while going to darkwake\n"); + reportUserInput(); + } } // Always accelerate disk spindown while in dark wake, @@ -7017,6 +7209,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) break; case kStimulusDarkWakeEvaluate: + DMSG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); if (systemDarkWake) { flags.bit.evaluateDarkWake = true; @@ -7024,6 +7217,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) break; case kStimulusNoIdleSleepPreventers: + DMSG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); flags.bit.adjustPowerState = true; break; @@ -7438,23 +7632,29 @@ void IOPMrootDomain::pmStatsRecordApplicationResponse( if (object && (notify = OSDynamicCast(IOPMServiceInterestNotifier, object))) { - if (response->isEqualTo(gIOPMStatsApplicationResponseTimedOut)) + if (response->isEqualTo(gIOPMStatsResponseTimedOut)) notify->ackTimeoutCnt++; else notify->ackTimeoutCnt = 0; } - if (response->isEqualTo(gIOPMStatsApplicationResponsePrompt) || + if (response->isEqualTo(gIOPMStatsResponsePrompt) || (_systemTransitionType == kSystemTransitionNone) || (_systemTransitionType == kSystemTransitionNewCapClient)) return; if (response->isEqualTo(gIOPMStatsDriverPSChangeSlow)) { - kdebugTrace(kPMLogDrvResponseDelay, id, messageType, delay_ms); + kdebugTrace(kPMLogDrvPSChangeDelay, id, messageType, delay_ms); } else if (notify) { - kdebugTrace(kPMLogAppResponseDelay, id, notify->msgType, delay_ms); + // User space app or kernel capability client + if (id) { + kdebugTrace(kPMLogAppResponseDelay, id, notify->msgType, delay_ms); + } + else { + kdebugTrace(kPMLogDrvResponseDelay, notify->uuid0, messageType, delay_ms); + } notify->msgType = 0; } @@ -7471,6 +7671,10 @@ void IOPMrootDomain::pmStatsRecordApplicationResponse( msgNum->release(); } + if (!name && notify && notify->identifier) { + name = notify->identifier->getCStringNoCopy(); + } + if (name && (strlen(name) > 0)) { appname = OSSymbol::withCString(name); @@ -7480,8 +7684,11 @@ void IOPMrootDomain::pmStatsRecordApplicationResponse( } } + if (!id && notify) { + id = notify->uuid0; + } if (id != 0) { - pidNum = OSNumber::withNumber(id, 32); + pidNum = OSNumber::withNumber(id, 64); if (pidNum) { responseDescription->setObject(_statsPIDKey, pidNum); pidNum->release(); @@ -7632,14 +7839,57 @@ void IOPMrootDomain::tracePoint( uint8_t point ) pmTracer->tracePoint(point); } -void IOPMrootDomain::traceDetail(uint32_t msgType, uint32_t msgIndex, uintptr_t handler) +void IOPMrootDomain::traceDetail(OSObject *object) { + IOPMServiceInterestNotifier *notifier = OSDynamicCast(IOPMServiceInterestNotifier, object); + if (!notifier) { + DLOG("Unknown notifier\n"); + return; + } + if (!systemBooting) { - uint32_t detail = ((msgIndex & 0xff) << 24) | - ((msgType & 0xfff) << 12) | - (handler & 0xfff); + pmTracer->traceDetail( notifier->uuid0 >> 32 ); + kdebugTrace(kPMLogSleepWakeMessage, pmTracer->getTracePhase(), notifier->msgType, notifier->uuid0, notifier->uuid1); + if (notifier->identifier) { + DLOG("trace point 0x%02x msg 0x%x to %s\n", pmTracer->getTracePhase(), notifier->msgType, + notifier->identifier->getCStringNoCopy()); + } + else { + DLOG("trace point 0x%02x msg 0x%x\n", pmTracer->getTracePhase(), notifier->msgType); + } + } + +} + + +void IOPMrootDomain::traceAckDelay(OSObject *object, uint32_t response, uint32_t delay_ms) +{ + IOPMServiceInterestNotifier *notifier = OSDynamicCast(IOPMServiceInterestNotifier, object); + if (!notifier) { + DLOG("Unknown notifier\n"); + return; + } + + if (!systemBooting) { + kdebugTrace(kPMLogDrvResponseDelay, notifier->uuid0, notifier->uuid1, response, delay_ms); + if (notifier->identifier) { + DLOG("Response from %s took %d ms(response:%d)\n", + notifier->identifier->getCStringNoCopy(), delay_ms, response); + } + else { + DLOG("Response from kext UUID %llx-%llx took %d ms(response:%d)\n", + notifier->uuid0, notifier->uuid1, delay_ms, response); + } + } +} + +void IOPMrootDomain::traceDetail(uint32_t msgType, uint32_t msgIndex, uint32_t delay) +{ + if (!systemBooting) { + uint32_t detail = ((msgType & 0xffff) << 16) | (delay & 0xffff); pmTracer->traceDetail( detail ); - kdebugTrace(kPMLogSleepWakeTracePoint, 0, pmTracer->getTracePhase(), msgType, handler & 0xfff); + kdebugTrace(kPMLogSleepWakeTracePoint, pmTracer->getTracePhase(), msgType, delay); + DLOG("trace point 0x%02x msgType 0x%x detail 0x%08x\n", pmTracer->getTracePhase(), msgType, delay); } } @@ -7940,10 +8190,10 @@ void PMTraceWorker::tracePoint(uint8_t phase) void PMTraceWorker::traceDetail(uint32_t detail) { - + if (detail == traceData32) { + return; + } traceData32 = detail; - DLOG("trace point 0x%02x detail 0x%08x\n", tracePhase, traceData32); - RTC_TRACE(); } @@ -8121,7 +8371,7 @@ void PMHaltWorker::work( PMHaltWorker * me ) { IOService * service; OSSet * inner; - AbsoluteTime startTime; + AbsoluteTime startTime, elapsedTime; UInt32 deltaTime; bool timeout; @@ -8170,7 +8420,7 @@ void PMHaltWorker::work( PMHaltWorker * me ) IOLockUnlock(me->lock); } - deltaTime = computeDeltaTimeMS(&startTime); + deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime); if ((deltaTime > kPMHaltTimeoutMS) || timeout || (gIOKitDebug & kIOLogPMRootDomain)) { @@ -8179,6 +8429,10 @@ void PMHaltWorker::work( PMHaltWorker * me ) "PowerOff" : "Restart", service->getName(), service->getRegistryEntryID(), (uint32_t) deltaTime ); + halt_log_enter( + (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ? "PowerOff" : "Restart", + OSMemberFunctionCast(const void *, service, &IOService::systemWillShutdown), + elapsedTime); } service->release(); @@ -8651,6 +8905,29 @@ void IOPMrootDomain::acceptSystemWakeEvents( bool accept ) else { _acceptSystemWakeEvents = false; +#if CONFIG_EMBEDDED + logWakeReason = gWakeReasonSysctlRegistered; +#if DEVELOPMENT + static int panic_allowed = -1; + + if ((panic_allowed == -1) && + (PE_parse_boot_argn("swd_wakereason_panic", &panic_allowed, sizeof(panic_allowed)) == false)) { + panic_allowed = 1; + } + + if (panic_allowed) { + size_t i = 0; + // Panic if wake reason is null or empty + for (i = 0; (i < strlen(gWakeReasonString)); i++) { + if ((gWakeReasonString[i] != ' ') && (gWakeReasonString[i] != '\t')) + break; + } + if (i >= strlen(gWakeReasonString)) { + panic("Wake reason is empty\n"); + } + } +#endif +#endif } WAKEEVENT_UNLOCK(); @@ -8706,6 +8983,9 @@ void IOPMrootDomain::claimSystemWakeEvent( // Lazy registration until the platform driver stops registering // the same name. gWakeReasonSysctlRegistered = true; +#if CONFIG_EMBEDDED + sysctl_register_oid(&sysctl__kern_wakereason); +#endif } if (_acceptSystemWakeEvents) { @@ -9355,6 +9635,7 @@ void IOPMrootDomain::takeStackshot(bool wdogTrigger, bool isOSXWatchdog, bool is swd_hdr * hdr = NULL; addr64_t data[3]; int wdog_panic = -1; + int stress_rack = -1; int cnt = 0; pid_t pid = 0; kern_return_t kr = KERN_SUCCESS; @@ -9385,9 +9666,10 @@ void IOPMrootDomain::takeStackshot(bool wdogTrigger, bool isOSXWatchdog, bool is if (wdogTrigger) { PE_parse_boot_argn("swd_panic", &wdog_panic, sizeof(wdog_panic)); - if (wdog_panic == 1) { + PE_parse_boot_argn("stress-rack", &stress_rack, sizeof(stress_rack)); + if ((wdog_panic == 1) || (stress_rack == 1)) { // If boot-arg specifies to panic then panic. - panic("Sleep/Wake hang detected\n"); + panic("Sleep/Wake hang detected"); return; } else if (swd_flags & SWD_BOOT_BY_SW_WDOG) { @@ -9395,6 +9677,10 @@ void IOPMrootDomain::takeStackshot(bool wdogTrigger, bool isOSXWatchdog, bool is // then don't trigger again until at least 1 successful sleep & wake. if (!(sleepCnt && (displayWakeCnt || darkWakeCnt))) { IOLog("Shutting down due to repeated Sleep/Wake failures\n"); + if (!tasksSuspended) { + tasksSuspended = TRUE; + tasks_system_suspend(true); + } PEHaltRestart(kPEHaltCPU); return; } @@ -9537,8 +9823,13 @@ exit: gRootDomain->swd_lock = 0; if (wdogTrigger) { - IOLog("Restarting to collect Sleep wake debug logs\n"); - PEHaltRestart(kPERestartCPU); + IOLog("Restarting to collect Sleep wake debug logs\n"); + if (!tasksSuspended) { + tasksSuspended = TRUE; + tasks_system_suspend(true); + } + + PEHaltRestart(kPERestartCPU); } else { logBufMap = sleepWakeDebugRetrieve(); @@ -9846,8 +10137,8 @@ uint32_t IOPMrootDomain::checkForValidDebugData(const char *fname, vfs_context_t vfs_context_ucred(*ctx), (int *) 0, vfs_context_proc(*ctx)); if (rc != 0) { - IOLog("sleepWakeDebugDumpFromFile: Failed to read header size %lu(rc=%d) from %s\n", - round_page(sizeof(IOHibernateImageHeader)), rc, fname); + IOLog("sleepWakeDebugDumpFromFile: Failed to read header size %llu(rc=%d) from %s\n", + mach_vm_round_page(sizeof(IOHibernateImageHeader)), rc, fname); error = SWD_FILEOP_ERROR; goto err; } @@ -9951,15 +10242,15 @@ void IOPMrootDomain::sleepWakeDebugDumpFromFile( ) hdrOffset = ((IOHibernateImageHeader *)tmpBuf)->deviceBlockSize; - DLOG("Reading swd_hdr len 0x%lx offset 0x%lx\n", round_page(sizeof(swd_hdr)), trunc_page(hdrOffset)); + DLOG("Reading swd_hdr len 0x%llx offset 0x%lx\n", mach_vm_round_page(sizeof(swd_hdr)), trunc_page(hdrOffset)); /* Read the sleep/wake debug header(swd_hdr) */ rc = vn_rdwr(UIO_READ, vp, (char *)tmpBuf, round_page(sizeof(swd_hdr)), trunc_page(hdrOffset), UIO_SYSSPACE, IO_SKIP_ENCRYPTION|IO_SYNC|IO_NODELOCKED|IO_UNIT|IO_NOCACHE, vfs_context_ucred(ctx), (int *) 0, vfs_context_proc(ctx)); if (rc != 0) { - DMSG("sleepWakeDebugDumpFromFile: Failed to debug read header size %lu. rc=%d\n", - round_page(sizeof(swd_hdr)), rc); + DMSG("sleepWakeDebugDumpFromFile: Failed to debug read header size %llu. rc=%d\n", + mach_vm_round_page(sizeof(swd_hdr)), rc); swd_flags |= SWD_FILEOP_ERROR; goto exit; } @@ -10236,7 +10527,7 @@ void IOPMrootDomain::sleepWakeDebugTrig(bool restart) (wdog_panic == 0)) { return; } - panic("Sleep/Wake hang detected\n"); + panic("Sleep/Wake hang detected"); return; } } diff --git a/iokit/Kernel/IOPlatformExpert.cpp b/iokit/Kernel/IOPlatformExpert.cpp index c83a71fd5..008afdb78 100644 --- a/iokit/Kernel/IOPlatformExpert.cpp +++ b/iokit/Kernel/IOPlatformExpert.cpp @@ -42,6 +42,7 @@ #include #include +#include #include #include @@ -53,6 +54,16 @@ extern "C" { #include } +#if defined(__x86_64__) +/* + * This will eventually be properly exported in + * ER: Expose coprocessor version (T208/T290) in a kernel/kext header + * although we'll always need to hardcode this here since we won't be able to include whatever + * header this ends up in. + */ +#define kCoprocessorMinVersion 0x00020000 +#endif + void printDictionaryKeys (OSDictionary * inDictionary, char * inMsg); static void getCStringForObject(OSObject *inObj, char *outStr, size_t outStrLen); @@ -99,15 +110,28 @@ bool IOPlatformExpert::start( IOService * provider ) IORangeAllocator * physicalRanges; OSData * busFrequency; uint32_t debugFlags; + +#if defined(__x86_64__) + IORegistryEntry *platform_entry = NULL; + OSData *coprocessor_version_obj = NULL; + uint64_t coprocessor_version = 0; +#endif if (!super::start(provider)) return false; - // Override the mapper present flag is requested by boot arguments. - if (PE_parse_boot_argn("dart", &debugFlags, sizeof (debugFlags)) && (debugFlags == 0)) - removeProperty(kIOPlatformMapperPresentKey); - if (PE_parse_boot_argn("-x", &debugFlags, sizeof (debugFlags))) - removeProperty(kIOPlatformMapperPresentKey); + // Override the mapper present flag is requested by boot arguments, if SIP disabled. +#if CONFIG_CSR + if (csr_check(CSR_ALLOW_UNRESTRICTED_FS) == 0) +#endif /* CONFIG_CSR */ + { + if (PE_parse_boot_argn("dart", &debugFlags, sizeof (debugFlags)) && (debugFlags == 0)) + removeProperty(kIOPlatformMapperPresentKey); +#if DEBUG || DEVELOPMENT + if (PE_parse_boot_argn("-x", &debugFlags, sizeof (debugFlags))) + removeProperty(kIOPlatformMapperPresentKey); +#endif /* DEBUG || DEVELOPMENT */ + } // Register the presence or lack thereof a system // PCI address mapper with the IOMapper class @@ -142,7 +166,21 @@ bool IOPlatformExpert::start( IOService * provider ) serNoString->release(); } } - + +#if defined(__x86_64__) + platform_entry = IORegistryEntry::fromPath(kIODeviceTreePlane ":/efi/platform"); + if (platform_entry != NULL) { + coprocessor_version_obj = OSDynamicCast(OSData, platform_entry->getProperty("apple-coprocessor-version")); + if ((coprocessor_version_obj != NULL) && (coprocessor_version_obj->getLength() <= sizeof(coprocessor_version))) { + memcpy(&coprocessor_version, coprocessor_version_obj->getBytesNoCopy(), coprocessor_version_obj->getLength()); + if (coprocessor_version >= kCoprocessorMinVersion) { + coprocessor_paniclog_flush = TRUE; + } + } + platform_entry->release(); + } +#endif /* defined(__x86_64__) */ + return( configure(provider) ); } @@ -260,9 +298,11 @@ int IOPlatformExpert::haltRestart(unsigned int type) type = kPEHaltCPU; } +#if !CONFIG_EMBEDDED // On ARM kPEPanicRestartCPU is supported in the drivers if (type == kPEPanicRestartCPU) type = kPERestartCPU; +#endif if (PE_halt_restart) return (*PE_halt_restart)(type); else return -1; @@ -738,10 +778,16 @@ static void IOShutdownNotificationsTimedOut( thread_call_param_t p0, thread_call_param_t p1) { +#ifdef CONFIG_EMBEDDED + /* 30 seconds has elapsed - panic */ + panic("Halt/Restart Timed Out"); + +#else /* ! CONFIG_EMBEDDED */ int type = (int)(long)p0; /* 30 seconds has elapsed - resume shutdown */ if(gIOPlatform) gIOPlatform->haltRestart(type); +#endif /* CONFIG_EMBEDDED */ } @@ -783,6 +829,7 @@ int PEHaltRestart(unsigned int type) IORegistryEntry *node; OSData *data; uint32_t timeout = 30; + static boolean_t panic_begin_called = FALSE; if(type == kPEHaltCPU || type == kPERestartCPU || type == kPEUPSDelayHaltCPU) { @@ -797,7 +844,11 @@ int PEHaltRestart(unsigned int type) the timer expires. If the device wants a different timeout, use that value instead of 30 seconds. */ +#if CONFIG_EMBEDDED +#define RESTART_NODE_PATH "/defaults" +#else #define RESTART_NODE_PATH "/chosen" +#endif node = IORegistryEntry::fromPath( RESTART_NODE_PATH, gIODTPlane ); if ( node ) { data = OSDynamicCast( OSData, node->getProperty( "halt-restart-timeout" ) ); @@ -822,6 +873,12 @@ int PEHaltRestart(unsigned int type) } else if(type == kPEPanicRestartCPU || type == kPEPanicSync) { + if (type == kPEPanicRestartCPU) { + // Notify any listeners that we're done collecting + // panic data before we call through to do the restart + IOCPURunPlatformPanicActions(kPEPanicEnd); + } + // Do an initial sync to flush as much panic data as possible, // in case we have a problem in one of the platorm panic handlers. // After running the platform handlers, do a final sync w/ @@ -830,6 +887,15 @@ int PEHaltRestart(unsigned int type) IOCPURunPlatformPanicActions(type); PE_sync_panic_buffers(); } + else if (type == kPEPanicEnd) { + IOCPURunPlatformPanicActions(type); + } else if (type == kPEPanicBegin) { + // Only call the kPEPanicBegin callout once + if (!panic_begin_called) { + panic_begin_called = TRUE; + IOCPURunPlatformPanicActions(type); + } + } if (gIOPlatform) return gIOPlatform->haltRestart(type); else return -1; @@ -841,6 +907,11 @@ UInt32 PESavePanicInfo(UInt8 *buffer, UInt32 length) else return 0; } +void PESavePanicInfoAction(void *buffer, size_t length) +{ + IOCPURunPlatformPanicSyncAction(buffer, length); + return; +} inline static int init_gIOOptionsEntry(void) @@ -1050,6 +1121,41 @@ void IOPlatformExpert::registerNVRAMController(IONVRAMController * caller) OSString * string = 0; uuid_string_t uuid; +#if CONFIG_EMBEDDED + entry = IORegistryEntry::fromPath( "/chosen", gIODTPlane ); + if ( entry ) + { + OSData * data1; + + data1 = OSDynamicCast( OSData, entry->getProperty( "unique-chip-id" ) ); + if ( data1 && data1->getLength( ) == 8 ) + { + OSData * data2; + + data2 = OSDynamicCast( OSData, entry->getProperty( "chip-id" ) ); + if ( data2 && data2->getLength( ) == 4 ) + { + SHA1_CTX context; + uint8_t digest[ SHA_DIGEST_LENGTH ]; + const uuid_t space = { 0xA6, 0xDD, 0x4C, 0xCB, 0xB5, 0xE8, 0x4A, 0xF5, 0xAC, 0xDD, 0xB6, 0xDC, 0x6A, 0x05, 0x42, 0xB8 }; + + SHA1Init( &context ); + SHA1Update( &context, space, sizeof( space ) ); + SHA1Update( &context, data1->getBytesNoCopy( ), data1->getLength( ) ); + SHA1Update( &context, data2->getBytesNoCopy( ), data2->getLength( ) ); + SHA1Final( digest, &context ); + + digest[ 6 ] = ( digest[ 6 ] & 0x0F ) | 0x50; + digest[ 8 ] = ( digest[ 8 ] & 0x3F ) | 0x80; + + uuid_unparse( digest, uuid ); + string = OSString::withCString( uuid ); + } + } + + entry->release( ); + } +#else /* !CONFIG_EMBEDDED */ entry = IORegistryEntry::fromPath( "/efi/platform", gIODTPlane ); if ( entry ) { @@ -1074,6 +1180,7 @@ void IOPlatformExpert::registerNVRAMController(IONVRAMController * caller) entry->release( ); } +#endif /* !CONFIG_EMBEDDED */ if ( string == 0 ) { diff --git a/iokit/Kernel/IOPolledInterface.cpp b/iokit/Kernel/IOPolledInterface.cpp index f0bb31618..2c9dfba68 100644 --- a/iokit/Kernel/IOPolledInterface.cpp +++ b/iokit/Kernel/IOPolledInterface.cpp @@ -336,9 +336,14 @@ IOStartPolledIO(IOPolledFilePollers * vars, poller = (IOPolledInterface *) vars->pollers->getObject(0); err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion); - if (err) - HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err); - + if (err) { + if (kernel_debugger_entry_count) { + HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err); + } else { + HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x", + vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err); + } + } return (err); } @@ -462,6 +467,8 @@ IOCopyMediaForDev(dev_t device) return (result); } +#define APFSMEDIA_GETHIBERKEY "getHiberKey" + static IOReturn IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID, uint8_t * volumeCryptKey, size_t keySize) @@ -478,30 +485,49 @@ IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID, part = IOCopyMediaForDev(block_dev); if (!part) return (kIOReturnNotFound); - err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, - (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL); - if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) + // Try APFS first { -// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy()); + uuid_t volUuid = {0}; + size_t sizeOut = 0; + err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, &keySize, &sizeOut); + if (err == kIOReturnSuccess) { + // No need to create uuid string if it's not requested + if (pKeyUUID) { + uuid_string_t volUuidStr; + uuid_unparse(volUuid, volUuidStr); + *pKeyUUID = OSString::withCString(volUuidStr); + } - if (!sKeyStore) - sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane); - if (sKeyStore) - err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID); - else - err = kIOReturnNoResources; - if (kIOReturnSuccess == err) - err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL); - if (kIOReturnSuccess != err) - IOLog("volume key err 0x%x\n", err); - else - { - if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount; - bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize); - } - bzero(&vek, sizeof(vek)); + part->release(); + return kIOReturnSuccess; + } + } + // Then old CS path + err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, + (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL); + if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) + { +// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy()); + + if (!sKeyStore) + sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane); + if (sKeyStore) + err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID); + else + err = kIOReturnNoResources; + if (kIOReturnSuccess == err) + err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL); + if (kIOReturnSuccess != err) + IOLog("volume key err 0x%x\n", err); + else + { + if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount; + bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize); + } + bzero(&vek, sizeof(vek)); } + part->release(); if (pKeyUUID) *pKeyUUID = keyUUID; @@ -521,7 +547,7 @@ IOPolledFileOpen(const char * filename, IOReturn err = kIOReturnSuccess; IOPolledFileIOVars * vars; _OpenFileContext ctx; - OSData * extentsData; + OSData * extentsData = NULL; OSNumber * num; IOService * part = 0; dev_t block_dev; @@ -642,6 +668,7 @@ IOPolledFileOpen(const char * filename, { HIBLOG("error 0x%x opening polled file\n", err); IOPolledFileClose(&vars, 0, 0, 0, 0, 0); + if (extentsData) extentsData->release(); } if (part) part->release(); @@ -734,6 +761,11 @@ IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position) vars->position = position; + if (position > vars->fileSize) { + HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize); + return kIOReturnNoSpace; + } + while (position >= extentMap->length) { position -= extentMap->length; @@ -760,7 +792,7 @@ IOPolledFileWrite(IOPolledFileIOVars * vars, IOPolledFileCryptVars * cryptvars) { IOReturn err = kIOReturnSuccess; - IOByteCount copy; + IOByteCount copy, original_size = size; bool flush = false; do @@ -844,8 +876,11 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length); err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length); - if (kIOReturnSuccess != err) + if (kIOReturnSuccess != err) { + HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n", + vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err); break; + } vars->pollers->io = true; vars->extentRemaining -= vars->bufferOffset; @@ -879,6 +914,29 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +IOReturn +IOPolledFileFlush(IOPolledFileIOVars * vars) +{ + // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms) + IOReturn err = kIOReturnSuccess; + + err = IOPolledFilePollersIODone(vars->pollers, true); + if (kIOReturnSuccess != err) + return err; + + err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0); + if (kIOReturnSuccess != err) { + HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n", + vars, vars->pollers, err); + return err; + } + vars->pollers->io = true; + + return err; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + IOReturn IOPolledFileRead(IOPolledFileIOVars * vars, uint8_t * bytes, IOByteCount size, diff --git a/iokit/Kernel/IOReporter.cpp b/iokit/Kernel/IOReporter.cpp index 81b6bfb94..dd6ccf764 100644 --- a/iokit/Kernel/IOReporter.cpp +++ b/iokit/Kernel/IOReporter.cpp @@ -161,7 +161,7 @@ finish: bool IOReporter::init(IOService *reportingService, IOReportChannelType channelType, - IOReportUnits unit) + IOReportUnit unit) { bool success = false; @@ -189,7 +189,9 @@ IOReporter::init(IOService *reportingService, _channelType = channelType; // FIXME: need to look up dynamically if (unit == kIOReportUnitHWTicks) { -#if defined(__i386__) || defined(__x86_64__) +#if defined(__arm__) || defined(__arm64__) + unit = kIOReportUnit24MHzTicks; +#elif defined(__i386__) || defined(__x86_64__) // Most, but not all Macs use 1GHz unit = kIOReportUnit1GHzTicks; #else @@ -988,7 +990,7 @@ finish: IOReporter::legendWith(OSArray *channelIDs, OSArray *channelNames, IOReportChannelType channelType, - IOReportUnits unit) + IOReportUnit unit) { unsigned int cnt, chCnt; uint64_t type64; diff --git a/iokit/Kernel/IOService.cpp b/iokit/Kernel/IOService.cpp index 57323e108..40055c5c9 100644 --- a/iokit/Kernel/IOService.cpp +++ b/iokit/Kernel/IOService.cpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -63,7 +64,10 @@ #define LOG kprintf //#define LOG IOLog #define MATCH_DEBUG 0 -#define OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x))) +#define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x))) + +// disabled since lockForArbitration() can be held externally +#define DEBUG_NOTIFIER_LOCKED 0 #include "IOServicePrivate.h" #include "IOKitKernelInternal.h" @@ -115,6 +119,9 @@ const OSSymbol * gIOPathMatchKey; const OSSymbol * gIOMatchCategoryKey; const OSSymbol * gIODefaultMatchCategoryKey; const OSSymbol * gIOMatchedServiceCountKey; +#if !CONFIG_EMBEDDED +const OSSymbol * gIOServiceLegacyMatchingRegistryIDKey; +#endif const OSSymbol * gIOMapperIDKey; const OSSymbol * gIOUserClientClassKey; @@ -148,6 +155,7 @@ const OSSymbol * gIOFirstPublishNotification; const OSSymbol * gIOMatchedNotification; const OSSymbol * gIOFirstMatchNotification; const OSSymbol * gIOTerminatedNotification; +const OSSymbol * gIOWillTerminateNotification; const OSSymbol * gIOGeneralInterest; const OSSymbol * gIOBusyInterest; @@ -179,7 +187,7 @@ static int gOutstandingJobs; static int gNumConfigThreads; static int gNumWaitingThreads; static IOLock * gIOServiceBusyLock; -static bool gCPUsRunning; +bool gCPUsRunning; static thread_t gIOTerminateThread; static UInt32 gIOTerminateWork; @@ -314,6 +322,10 @@ void IOService::initialize( void ) kIODefaultMatchCategoryKey ); gIOMatchedServiceCountKey = OSSymbol::withCStringNoCopy( kIOMatchedServiceCountKey ); +#if !CONFIG_EMBEDDED + gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy( + kIOServiceLegacyMatchingRegistryIDKey ); +#endif gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey ); @@ -356,6 +368,8 @@ void IOService::initialize( void ) kIOFirstMatchNotification ); gIOTerminatedNotification = OSSymbol::withCStringNoCopy( kIOTerminatedNotification ); + gIOWillTerminateNotification = OSSymbol::withCStringNoCopy( + kIOWillTerminateNotification ); gIOServiceKey = OSSymbol::withCStringNoCopy( kIOServiceClass); gIOConsoleLockedKey = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey); @@ -464,6 +478,24 @@ static UInt64 getDebugFlags( OSDictionary * props ) return( debugFlags ); } + +static UInt64 getDebugFlags( IOService * inst ) +{ + OSObject * prop; + OSNumber * debugProp; + UInt64 debugFlags; + + prop = inst->copyProperty(gIOKitDebugKey); + debugProp = OSDynamicCast(OSNumber, prop); + if( debugProp) + debugFlags = debugProp->unsigned64BitValue(); + else + debugFlags = gIOKitDebug; + + OSSafeReleaseNULL(prop); + + return( debugFlags ); +} #endif /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -790,7 +822,7 @@ void IOService::startMatching( IOOptionBits options ) if ( options & kIOServiceAsynchronous ) sync = false; - needConfig = (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigState))) + needConfig = (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning))) && (0 == (__state[0] & kIOServiceInactiveState)); __state[1] |= kIOServiceNeedConfigState; @@ -1549,6 +1581,41 @@ void IOService::unlockForArbitration( void ) IOUnlock( gArbitrationLockQueueLock ); } +uint32_t IOService::isLockedForArbitration(IOService * service) +{ +#if DEBUG_NOTIFIER_LOCKED + uint32_t count; + ArbitrationLockQueueElement * active; + + // lock global access + IOLockLock(gArbitrationLockQueueLock); + + // determine whether this object is already locked (ie. on active queue) + count = 0; + queue_iterate(&gArbitrationLockQueueActive, + active, + ArbitrationLockQueueElement *, + link) + { + if ((active->thread == IOThreadSelf()) + && (!service || (active->service == service))) + { + count += 0x10000; + count += active->count; + } + } + + IOLockUnlock(gArbitrationLockQueueLock); + + return (count); + +#else /* DEBUG_NOTIFIER_LOCKED */ + + return (0); + +#endif /* DEBUG_NOTIFIER_LOCKED */ +} + void IOService::applyToProviders( IOServiceApplierFunction applier, void * context ) { @@ -1625,12 +1692,13 @@ applyToInterestNotifiers(const IORegistryEntry *target, OSObjectApplierFunction applier, void * context ) { - OSArray * copyArray = 0; + OSArray * copyArray = 0; + OSObject * prop; LOCKREADNOTIFY(); - IOCommand *notifyList = - OSDynamicCast( IOCommand, target->getProperty( typeOfInterest )); + prop = target->copyProperty(typeOfInterest); + IOCommand *notifyList = OSDynamicCast(IOCommand, prop); if( notifyList) { copyArray = OSArray::withCapacity(1); @@ -1653,6 +1721,8 @@ applyToInterestNotifiers(const IORegistryEntry *target, (*applier)(next, context); copyArray->release(); } + + OSSafeReleaseNULL(prop); } void IOService::applyToInterested( const OSSymbol * typeOfInterest, @@ -1717,7 +1787,7 @@ IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest, if (!notify) return NULL; if(notify->init()) { - rc = registerInterestForNotifer(notify, typeOfInterest, + rc = registerInterestForNotifier(notify, typeOfInterest, handler, target, ref); } @@ -1729,7 +1799,7 @@ IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest, return( notify ); } -IOReturn IOService::registerInterestForNotifer( IONotifier *svcNotify, const OSSymbol * typeOfInterest, +IOReturn IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest, IOServiceInterestHandler handler, void * target, void * ref ) { IOReturn rc = kIOReturnSuccess; @@ -1806,11 +1876,27 @@ static void cleanInterestList( OSObject * head ) void IOService::unregisterAllInterest( void ) { - cleanInterestList( getProperty( gIOGeneralInterest )); - cleanInterestList( getProperty( gIOBusyInterest )); - cleanInterestList( getProperty( gIOAppPowerStateInterest )); - cleanInterestList( getProperty( gIOPriorityPowerStateInterest )); - cleanInterestList( getProperty( gIOConsoleSecurityInterest )); + OSObject * prop; + + prop = copyProperty(gIOGeneralInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOBusyInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOAppPowerStateInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOPriorityPowerStateInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOConsoleSecurityInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); } /* @@ -2046,7 +2132,10 @@ bool IOService::terminatePhase1( IOOptionBits options ) if( victim == this) startPhase2 = didInactive; if (didInactive) { - victim->deliverNotification( gIOTerminatedNotification, 0, 0xffffffff ); + OSArray * notifiers; + notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff); + victim->invokeNotifiers(¬ifiers); + IOUserClient::destroyUserReferences( victim ); iter = victim->getClientIterator(); @@ -2533,8 +2622,10 @@ void IOService::terminateWorker( IOOptionBits options ) (uintptr_t) victim->__state[1], (uintptr_t) 0); - doPhase2 = (0 == (victim->__state[1] & kIOServiceTermPhase2State)) - && (0 == (victim->__state[1] & kIOServiceConfigState)); + doPhase2 = (0 == (victim->__state[1] & + (kIOServiceTermPhase1State + | kIOServiceTermPhase2State + | kIOServiceConfigState))); if (doPhase2 && (iter = victim->getClientIterator())) { while (doPhase2 && (client = (IOService *) iter->getNextObject())) { @@ -2567,6 +2658,10 @@ void IOService::terminateWorker( IOOptionBits options ) victim, (void *)(uintptr_t) options, NULL ); } + OSArray * notifiers; + notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff); + victim->invokeNotifiers(¬ifiers); + if( 0 == victim->getClient()) { // no clients - will go to finalize @@ -2932,23 +3027,30 @@ static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref) _IOServiceNotifier * notify; OSSymbol * key = (OSSymbol *) ref; OSNumber * offset; + OSObject * prop; + SInt32 result; + prop = 0; + result = kIODefaultProbeScore; if( (dict = OSDynamicCast( OSDictionary, entry))) offset = OSDynamicCast(OSNumber, dict->getObject( key )); else if( (notify = OSDynamicCast( _IOServiceNotifier, entry))) return( notify->priority ); - else if( (service = OSDynamicCast( IOService, entry))) - offset = OSDynamicCast(OSNumber, service->getProperty( key )); + { + prop = service->copyProperty(key); + offset = OSDynamicCast(OSNumber, prop); + } else { assert( false ); offset = 0; } - if( offset) - return( (SInt32) offset->unsigned32BitValue()); - else - return( kIODefaultProbeScore ); + if (offset) result = offset->unsigned32BitValue(); + + OSSafeReleaseNULL(prop); + + return (result); } SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref ) @@ -3003,14 +3105,22 @@ IOService * IOService::getClientWithCategory( const OSSymbol * category ) return (service); } -bool IOService::invokeNotifer( _IOServiceNotifier * notify ) +bool IOService::invokeNotifier( _IOServiceNotifier * notify ) { _IOServiceNotifierInvocation invocation; bool willNotify; bool ret = true; - invocation.thread = current_thread(); +#if DEBUG_NOTIFIER_LOCKED + uint32_t count; + if ((count = isLockedForArbitration(0))) + { + IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count); + panic("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count); + } +#endif /* DEBUG_NOTIFIER_LOCKED */ + LOCKWRITENOTIFY(); willNotify = (0 != (kIOServiceNotifyEnable & notify->state)); @@ -3037,6 +3147,27 @@ bool IOService::invokeNotifer( _IOServiceNotifier * notify ) return( ret ); } +bool IOService::invokeNotifiers(OSArray ** willSend) +{ + OSArray * array; + _IOServiceNotifier * notify; + bool ret = true; + + array = *willSend; + if (!array) return (true); + *willSend = 0; + + for( unsigned int idx = 0; + (notify = (_IOServiceNotifier *) array->getObject(idx)); + idx++) { + ret &= invokeNotifier(notify); + } + array->release(); + + return (ret); +} + + /* * Alloc and probe matching classes, * called on the provider instance @@ -3074,10 +3205,7 @@ void IOService::probeCandidates( OSOrderedSet * matches ) if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) { - lockForArbitration(); - if( 0 == (__state[0] & kIOServiceInactiveState)) - invokeNotifer( notify ); - unlockForArbitration(); + if (0 == (__state[0] & kIOServiceInactiveState)) invokeNotifier( notify ); nextMatch->release(); nextMatch = 0; continue; @@ -3174,7 +3302,7 @@ void IOService::probeCandidates( OSOrderedSet * matches ) if( !symbol) continue; - //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), OBFUSCATE(symbol), OBFUSCATE(props)); + //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props)); // alloc the driver instance inst = (IOService *) OSMetaClass::allocClassWithName( symbol); @@ -3280,7 +3408,7 @@ void IOService::probeCandidates( OSOrderedSet * matches ) startList->removeObject(inst); #if IOMATCHDEBUG - debugFlags = getDebugFlags( inst->getPropertyTable() ); + debugFlags = getDebugFlags( inst ); if( debugFlags & kIOLogStart) { if( started) @@ -3422,11 +3550,15 @@ bool IOService::addNeededResource( const char * key ) OSString * newKey; bool ret; - resourcesProp = getProperty( gIOResourceMatchKey ); + resourcesProp = copyProperty( gIOResourceMatchKey ); + if (!resourcesProp) return(false); newKey = OSString::withCString( key ); - if( (0 == resourcesProp) || (0 == newKey)) + if (!newKey) + { + resourcesProp->release(); return( false); + } set = OSDynamicCast( OSSet, resourcesProp ); if( !set) { @@ -3441,6 +3573,7 @@ bool IOService::addNeededResource( const char * key ) newKey->release(); ret = setProperty( gIOResourceMatchKey, set ); set->release(); + resourcesProp->release(); return( ret ); } @@ -3467,12 +3600,12 @@ bool IOService::checkResource( OSObject * matching ) } if( gIOKitDebug & kIOLogConfig) - LOG("config(%p): stalling %s\n", OBFUSCATE(IOThreadSelf()), getName()); + LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName()); waitForService( table ); if( gIOKitDebug & kIOLogConfig) - LOG("config(%p): waking\n", OBFUSCATE(IOThreadSelf()) ); + LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) ); return( true ); } @@ -3484,7 +3617,7 @@ bool IOService::checkResources( void ) OSIterator * iter; bool ok; - resourcesProp = getProperty( gIOResourceMatchKey ); + resourcesProp = copyProperty( gIOResourceMatchKey ); if( 0 == resourcesProp) return( true ); @@ -3500,6 +3633,8 @@ bool IOService::checkResources( void ) } else ok = checkResource( resourcesProp ); + OSSafeReleaseNULL(resourcesProp); + return( ok ); } @@ -3543,6 +3678,7 @@ void IOService::doServiceMatch( IOOptionBits options ) bool keepGuessing = true; bool reRegistered = true; bool didRegister; + OSArray * notifiers[2] = {0}; // job->nub->deliverNotification( gIOPublishNotification, // kIOServiceRegisteredState, 0xffffffff ); @@ -3556,12 +3692,12 @@ void IOService::doServiceMatch( IOOptionBits options ) lockForArbitration(); if( 0 == (__state[0] & kIOServiceFirstPublishState)) { getMetaClass()->addInstance(this); - deliverNotification( gIOFirstPublishNotification, + notifiers[0] = copyNotifiers(gIOFirstPublishNotification, kIOServiceFirstPublishState, 0xffffffff ); } LOCKREADNOTIFY(); __state[1] &= ~kIOServiceNeedConfigState; - __state[1] |= kIOServiceConfigState; + __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning; didRegister = (0 == (kIOServiceRegisteredState & __state[0])); __state[0] |= kIOServiceRegisteredState; @@ -3583,6 +3719,7 @@ void IOService::doServiceMatch( IOOptionBits options ) UNLOCKNOTIFY(); unlockForArbitration(); + invokeNotifiers(¬ifiers[0]); if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources())) { @@ -3613,15 +3750,22 @@ void IOService::doServiceMatch( IOOptionBits options ) if (resourceKeys) setProperty(gIOResourceMatchedKey, resourceKeys); - deliverNotification( gIOMatchedNotification, - kIOServiceMatchedState, 0xffffffff ); + notifiers[0] = copyNotifiers(gIOMatchedNotification, + kIOServiceMatchedState, 0xffffffff); if( 0 == (__state[0] & kIOServiceFirstMatchState)) - deliverNotification( gIOFirstMatchNotification, - kIOServiceFirstMatchState, 0xffffffff ); + notifiers[1] = copyNotifiers(gIOFirstMatchNotification, + kIOServiceFirstMatchState, 0xffffffff); } + __state[1] &= ~kIOServiceConfigRunning; + unlockForArbitration(); + if (resourceKeys) resourceKeys->release(); + invokeNotifiers(¬ifiers[0]); + invokeNotifiers(¬ifiers[1]); + + lockForArbitration(); __state[1] &= ~kIOServiceConfigState; scheduleTerminatePhase2(); @@ -3785,24 +3929,24 @@ IOReturn IOService::waitForState( UInt32 mask, UInt32 value, return( kIOReturnSuccess ); } -#if NO_KEXTD -#define WAITING_KEXTD false -#else -extern bool gIOKextdClearedBusy; -#define WAITING_KEXTD (false == gIOKextdClearedBusy) -#endif - IOReturn IOService::waitQuiet( uint64_t timeout ) { IOReturn ret; uint32_t loops; - char * string = NULL; - size_t len; + char * string = NULL; + char * panicString = NULL; + size_t len; + size_t panicStringLen; uint64_t time; uint64_t nano; + bool kextdWait; + bool dopanic; + + enum { kTimeoutExtensions = 4 }; time = mach_absolute_time(); - for (loops = 0; loops < 2; loops++) + kextdWait = false; + for (loops = 0; loops < kTimeoutExtensions; loops++) { ret = waitForState( kIOServiceBusyStateMask, 0, timeout ); @@ -3810,15 +3954,13 @@ IOReturn IOService::waitQuiet( uint64_t timeout ) { time = mach_absolute_time() - time; absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano); - IOLog("busy extended ok[%d], (%llds, %llds), kextd wait(%d): %s\n", - loops, timeout / 1000000000ULL, nano / 1000000000ULL, WAITING_KEXTD, - string ? string : ""); + IOLog("busy extended ok[%d], (%llds, %llds)\n", + loops, timeout / 1000000000ULL, nano / 1000000000ULL); break; } else if (kIOReturnTimeout != ret) break; else if (timeout < 41000000000) break; - if (!loops) { IORegistryIterator * iter; OSOrderedSet * set; @@ -3829,17 +3971,23 @@ IOReturn IOService::waitQuiet( uint64_t timeout ) size_t l; len = 256; - string = IONew(char, len); + panicStringLen = 256; + if (!string) string = IONew(char, len); + if (!panicString) panicString = IONew(char, panicStringLen); set = NULL; + kextdWait = OSKext::isWaitingKextd(); iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively); leaves = OSOrderedSet::withCapacity(4); if (iter) set = iter->iterateAll(); - if (string && leaves && set) + if (string && panicString && leaves && set) { + string[0] = panicString[0] = 0; + set->setObject(this); while ((next = (IOService *) set->getLastObject())) { if (next->getBusyState()) { + if (kIOServiceModuleStallState & next->__state[1]) kextdWait = true; leaves->setObject(next); nextParent = next; while ((nextParent = nextParent->getProvider())) @@ -3864,21 +4012,20 @@ IOReturn IOService::waitQuiet( uint64_t timeout ) OSSafeReleaseNULL(set); OSSafeReleaseNULL(iter); } - if (loops && (kIOWaitQuietPanics & gIOKitDebug)) - { - panic("busy timeout[%d], (%llds), kextd wait(%d): %s", - loops, timeout / 1000000000ULL, WAITING_KEXTD, - string ? string : ""); - } - else - { - IOLog("busy timeout[%d], (%llds), kextd wait(%d): %s\n", - loops, timeout / 1000000000ULL, WAITING_KEXTD, - string ? string : ""); - } + + dopanic = ((loops >= (kTimeoutExtensions - 1)) && (kIOWaitQuietPanics & gIOKitDebug)); + snprintf(panicString, panicStringLen, + "%s[%d], (%llds): %s", + kextdWait ? "kextd stall" : "busy timeout", + loops, timeout / 1000000000ULL, + string ? string : ""); + IOLog("%s\n", panicString); + if (dopanic) panic("%s", panicString); + else if (!loops) getPMRootDomain()->startSpinDump(1); } - if (string) IODelete(string, char, 256); + if (string) IODelete(string, char, 256); + if (panicString) IODelete(panicString, char, panicStringLen); return (ret); } @@ -3948,7 +4095,7 @@ void _IOConfigThread::main(void * arg, wait_result_t result) if( gIOKitDebug & kIOLogConfig) LOG("config(%p): starting on %s, %d\n", - OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type); + IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type); switch( job->type) { @@ -3958,7 +4105,7 @@ void _IOConfigThread::main(void * arg, wait_result_t result) default: LOG("config(%p): strange type (%d)\n", - OBFUSCATE(IOThreadSelf()), job->type ); + IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type ); break; } @@ -3982,7 +4129,7 @@ void _IOConfigThread::main(void * arg, wait_result_t result) } while( alive ); if( gIOKitDebug & kIOLogConfig) - LOG("config(%p): terminating\n", OBFUSCATE(IOThreadSelf()) ); + LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) ); self->release(); } @@ -4243,8 +4390,8 @@ OSObject * IOService::copyExistingServices( OSDictionary * matching, OSSerialize * s2 = OSSerialize::withCapacity(128); current->serialize(s1); _current->serialize(s2); - kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", OBFUSCATE(current), - OBFUSCATE(_current), s->text(), s1->text(), s2->text()); + kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current), + IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text()); s1->release(); s2->release(); } @@ -4329,6 +4476,7 @@ IONotifier * IOService::setNotification( if( notify) { notify->handler = handler; notify->target = target; + notify->type = type; notify->matching = matching; matching->retain(); if (handler == &_IOServiceMatchingNotificationHandler) @@ -4386,7 +4534,7 @@ IONotifier * IOService::doInstallNotification( else if (type == gIOFirstMatchNotification) inState = kIOServiceFirstMatchState; - else if( type == gIOTerminatedNotification) + else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification)) inState = 0; else return( 0 ); @@ -4484,14 +4632,14 @@ IONotifier * IOService::addMatchingNotification( if (!ret) return (0); // send notifications for existing set - if (existing) { - - while( (next = (IOService *) existing->getNextObject())) { - - next->lockForArbitration(); + if (existing) + { + while( (next = (IOService *) existing->getNextObject())) + { if( 0 == (next->__state[0] & kIOServiceInactiveState)) - next->invokeNotifer( notify ); - next->unlockForArbitration(); + { + next->invokeNotifier( notify ); + } } existing->release(); } @@ -4596,6 +4744,12 @@ IOService * IOService::waitForService( OSDictionary * matching, void IOService::deliverNotification( const OSSymbol * type, IOOptionBits orNewState, IOOptionBits andNewState ) +{ + panic("deliverNotification"); +} + +OSArray * IOService::copyNotifiers(const OSSymbol * type, + IOOptionBits orNewState, IOOptionBits andNewState ) { _IOServiceNotifier * notify; OSIterator * iter; @@ -4604,7 +4758,8 @@ void IOService::deliverNotification( const OSSymbol * type, lockForArbitration(); if( (0 == (__state[0] & kIOServiceInactiveState)) - || (type == gIOTerminatedNotification)) { + || (type == gIOTerminatedNotification) + || (type == gIOWillTerminateNotification)) { LOCKREADNOTIFY(); @@ -4624,21 +4779,14 @@ void IOService::deliverNotification( const OSSymbol * type, } iter->release(); } - __state[0] = (__state[0] | orNewState) & andNewState; - UNLOCKNOTIFY(); } - if( willSend) { - for( unsigned int idx = 0; - (notify = (_IOServiceNotifier *) willSend->getObject(idx)); - idx++) { - invokeNotifer( notify ); - } - willSend->release(); - } unlockForArbitration(); + + return (willSend); + } IOOptionBits IOService::getState( void ) const @@ -5461,8 +5609,10 @@ bool IOService::matchPassive(OSDictionary * table, uint32_t options) assert( table ); +#if !CONFIG_EMBEDDED OSArray* aliasServiceRegIds = NULL; IOService* foundAlternateService = NULL; +#endif #if MATCH_DEBUG OSDictionary * root = table; @@ -5534,8 +5684,10 @@ bool IOService::matchPassive(OSDictionary * table, uint32_t options) } if(matchParent == true) { +#if !CONFIG_EMBEDDED // check if service has an alias to search its other "parents" if a parent match isn't found - OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, where->getProperty(kIOServiceLegacyMatchingRegistryIDKey)); + OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey); + OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop); if(alternateRegistryID != NULL) { if(aliasServiceRegIds == NULL) { @@ -5543,12 +5695,15 @@ bool IOService::matchPassive(OSDictionary * table, uint32_t options) } aliasServiceRegIds->setObject(alternateRegistryID); } + OSSafeReleaseNULL(prop); +#endif } else { break; } where = where->getProvider(); +#if !CONFIG_EMBEDDED if(where == NULL) { // there were no matching parent services, check to see if there are aliased services that have a matching parent if(aliasServiceRegIds != NULL) { @@ -5570,11 +5725,14 @@ bool IOService::matchPassive(OSDictionary * table, uint32_t options) } } } +#endif } while( where != NULL ); +#if !CONFIG_EMBEDDED OSSafeReleaseNULL(foundAlternateService); OSSafeReleaseNULL(aliasServiceRegIds); +#endif #if MATCH_DEBUG if (where != this) @@ -5596,30 +5754,35 @@ IOReturn IOService::newUserClient( task_t owningTask, void * securityID, { const OSSymbol *userClientClass = 0; IOUserClient *client; + OSObject *prop; OSObject *temp; if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler )) return kIOReturnSuccess; // First try my own properties for a user client class name - temp = getProperty(gIOUserClientClassKey); - if (temp) { - if (OSDynamicCast(OSSymbol, temp)) - userClientClass = (const OSSymbol *) temp; - else if (OSDynamicCast(OSString, temp)) { - userClientClass = OSSymbol::withString((OSString *) temp); + prop = copyProperty(gIOUserClientClassKey); + if (prop) { + if (OSDynamicCast(OSSymbol, prop)) + userClientClass = (const OSSymbol *) prop; + else if (OSDynamicCast(OSString, prop)) { + userClientClass = OSSymbol::withString((OSString *) prop); if (userClientClass) - setProperty(kIOUserClientClassKey, + setProperty(gIOUserClientClassKey, (OSObject *) userClientClass); } } // Didn't find one so lets just bomb out now without further ado. if (!userClientClass) + { + OSSafeReleaseNULL(prop); return kIOReturnUnsupported; + } // This reference is consumed by the IOServiceOpen call temp = OSMetaClass::allocClassWithName(userClientClass); + OSSafeReleaseNULL(prop); if (!temp) return kIOReturnNoMemory; @@ -6303,7 +6466,9 @@ IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistic /* * We now need to add the legend for this reporter to the registry. */ - legend = IOReportLegend::with(OSDynamicCast(OSArray, getProperty(kIOReportLegendKey))); + OSObject * prop = copyProperty(kIOReportLegendKey); + legend = IOReportLegend::with(OSDynamicCast(OSArray, prop)); + OSSafeReleaseNULL(prop); /* * Note that while we compose the subgroup name, we do not need to diff --git a/iokit/Kernel/IOServicePM.cpp b/iokit/Kernel/IOServicePM.cpp index e7fbb0802..6fc90a603 100644 --- a/iokit/Kernel/IOServicePM.cpp +++ b/iokit/Kernel/IOServicePM.cpp @@ -181,7 +181,11 @@ do { \ #define ns_per_us 1000 #define k30Seconds (30*us_per_s) #define k5Seconds ( 5*us_per_s) +#if CONFIG_EMBEDDED +#define kCanSleepMaxTimeReq k5Seconds +#else #define kCanSleepMaxTimeReq k30Seconds +#endif #define kMaxTimeRequested k30Seconds #define kMinAckTimeoutTicks (10*1000000) #define kIOPMTardyAckSPSKey "IOPMTardyAckSetPowerState" @@ -241,6 +245,9 @@ do { \ // use message tracer to log messages longer than (ns): #define LOG_APP_RESPONSE_MSG_TRACER (3 * 1000ULL * 1000ULL * 1000ULL) +// log kext responses longer than (ns): +#define LOG_KEXT_RESPONSE_TIMES (100ULL * 1000ULL * 1000ULL) + enum { kReserveDomainPower = 1 }; @@ -5289,6 +5296,10 @@ bool IOService::ackTimerTick( void ) PM_ERROR("%s::setPowerState(%p, %lu -> %lu) timed out after %d ms\n", fName, OBFUSCATE(this), fCurrentPowerState, fHeadNotePowerState, NS_TO_MS(nsec)); +#if DEBUG && CONFIG_EMBEDDED + panic("%s::setPowerState(%p, %lu -> %lu) timed out after %d ms", + fName, this, fCurrentPowerState, fHeadNotePowerState, NS_TO_MS(nsec)); +#else if (gIOKitDebug & kIOLogDebugPower) { panic("%s::setPowerState(%p, %lu -> %lu) timed out after %d ms", @@ -5299,6 +5310,7 @@ bool IOService::ackTimerTick( void ) // Unblock state machine and pretend driver has acked. done = true; } +#endif } else { // still waiting, set timer again start_ack_timer(); @@ -5772,7 +5784,7 @@ static void logAppTimeouts( OSObject * object, void * arg ) // TODO: record message type if possible IOService::getPMRootDomain()->pmStatsRecordApplicationResponse( - gIOPMStatsApplicationResponseTimedOut, + gIOPMStatsResponseTimedOut, name, 0, (30*1000), pid, object); } @@ -5915,7 +5927,6 @@ bool IOService::tellClientsWithResponse( int messageType ) applyToInterested( gIOGeneralInterest, pmTellClientWithResponse, (void *) &context ); - fNotifyClientArray = context.notifyClients; break; case kNotifyPriority: @@ -5936,7 +5947,6 @@ bool IOService::tellClientsWithResponse( int messageType ) case kNotifyCapabilityChangeApps: applyToInterested( gIOAppPowerStateInterest, pmTellCapabilityAppWithResponse, (void *) &context ); - fNotifyClientArray = context.notifyClients; if(context.messageType == kIOMessageCanSystemSleep) { maxTimeOut = kCanSleepMaxTimeReq; @@ -5954,6 +5964,7 @@ bool IOService::tellClientsWithResponse( int messageType ) pmTellCapabilityClientWithResponse, (void *) &context ); break; } + fNotifyClientArray = context.notifyClients; // do we have to wait for somebody? if ( !checkForDone() ) @@ -6097,6 +6108,8 @@ void IOService::pmTellClientWithResponse( OSObject * object, void * arg ) _IOServiceInterestNotifier * notifier; uint32_t msgIndex, msgRef, msgType; IOReturn retCode; + AbsoluteTime start, end; + uint64_t nsec; if (context->messageFilter && !context->messageFilter(context->us, object, context, 0, 0)) @@ -6137,6 +6150,9 @@ void IOService::pmTellClientWithResponse( OSObject * object, void * arg ) OBFUSCATE(object), OBFUSCATE(notifier->handler)); } + if (0 == context->notifyClients) + context->notifyClients = OSArray::withCapacity( 32 ); + notify.powerRef = (void *)(uintptr_t) msgRef; notify.returnValue = 0; notify.stateNumber = context->stateNumber; @@ -6144,15 +6160,18 @@ void IOService::pmTellClientWithResponse( OSObject * object, void * arg ) if (context->enableTracing && (notifier != 0)) { - getPMRootDomain()->traceDetail(msgType, msgIndex, (uintptr_t) notifier->handler); + getPMRootDomain()->traceDetail(notifier); } + clock_get_uptime(&start); retCode = context->us->messageClient(msgType, object, (void *) ¬ify, sizeof(notify)); + clock_get_uptime(&end); if (kIOReturnSuccess == retCode) { if (0 == notify.returnValue) { OUR_PMLog(kPMLogClientAcknowledge, msgRef, (uintptr_t) object); + context->responseArray->setObject(msgIndex, replied); } else { replied = kOSBooleanFalse; if ( notify.returnValue > context->maxTimeRequested ) @@ -6169,14 +6188,39 @@ void IOService::pmTellClientWithResponse( OSObject * object, void * arg ) else context->maxTimeRequested = notify.returnValue; } + // + // Track time taken to ack, by storing the timestamp of + // callback completion + OSNumber * num; + num = OSNumber::withNumber(AbsoluteTime_to_scalar(&end), sizeof(uint64_t) * 8); + if (num) { + context->responseArray->setObject(msgIndex, num); + num->release(); + } + else { + context->responseArray->setObject(msgIndex, replied); + } } - } else { + + if (context->enableTracing) { + SUB_ABSOLUTETIME(&end, &start); + absolutetime_to_nanoseconds(end, &nsec); + + if ((nsec > LOG_KEXT_RESPONSE_TIMES) || (notify.returnValue != 0)) { + getPMRootDomain()->traceAckDelay(notifier, notify.returnValue/1000, NS_TO_MS(nsec)); + } + } + } + else { // not a client of ours // so we won't be waiting for response OUR_PMLog(kPMLogClientAcknowledge, msgRef, 0); + context->responseArray->setObject(msgIndex, replied); + } + if (context->notifyClients) { + context->notifyClients->setObject(msgIndex, object); } - context->responseArray->setObject(msgIndex, replied); } //********************************************************************************* @@ -6278,6 +6322,8 @@ void IOService::pmTellCapabilityClientWithResponse( _IOServiceInterestNotifier * notifier; uint32_t msgIndex, msgRef, msgType; IOReturn retCode; + AbsoluteTime start, end; + uint64_t nsec; memset(&msgArg, 0, sizeof(msgArg)); if (context->messageFilter && @@ -6295,6 +6341,9 @@ void IOService::pmTellCapabilityClientWithResponse( return; } + if (0 == context->notifyClients) { + context->notifyClients = OSArray::withCapacity( 32 ); + } notifier = OSDynamicCast(_IOServiceInterestNotifier, object); msgType = context->messageType; msgIndex = context->responseArray->getCount(); @@ -6324,11 +6373,13 @@ void IOService::pmTellCapabilityClientWithResponse( if (context->enableTracing && (notifier != 0)) { - getPMRootDomain()->traceDetail(msgType, msgIndex, (uintptr_t) notifier->handler); + getPMRootDomain()->traceDetail(notifier); } + clock_get_uptime(&start); retCode = context->us->messageClient( msgType, object, (void *) &msgArg, sizeof(msgArg)); + clock_get_uptime(&end); if ( kIOReturnSuccess == retCode ) { @@ -6336,6 +6387,7 @@ void IOService::pmTellCapabilityClientWithResponse( { // client doesn't want time to respond OUR_PMLog(kPMLogClientAcknowledge, msgRef, (uintptr_t) object); + context->responseArray->setObject(msgIndex, replied); } else { @@ -6346,14 +6398,35 @@ void IOService::pmTellCapabilityClientWithResponse( { context->maxTimeRequested = kCapabilityClientMaxWait; PM_ERROR("%s: client %p returned %u for %s\n", - context->us->getName(), - notifier ? (void *) OBFUSCATE(notifier->handler) : OBFUSCATE(object), - msgArg.maxWaitForReply, - getIOMessageString(msgType)); + context->us->getName(), + notifier ? (void *) OBFUSCATE(notifier->handler) : OBFUSCATE(object), + msgArg.maxWaitForReply, + getIOMessageString(msgType)); } else context->maxTimeRequested = msgArg.maxWaitForReply; } + + // Track time taken to ack, by storing the timestamp of + // callback completion + OSNumber * num; + num = OSNumber::withNumber(AbsoluteTime_to_scalar(&end), sizeof(uint64_t) * 8); + if (num) { + context->responseArray->setObject(msgIndex, num); + num->release(); + } + else { + context->responseArray->setObject(msgIndex, replied); + } + } + + if (context->enableTracing) { + SUB_ABSOLUTETIME(&end, &start); + absolutetime_to_nanoseconds(end, &nsec); + + if ((nsec > LOG_KEXT_RESPONSE_TIMES) || (msgArg.maxWaitForReply != 0)) { + getPMRootDomain()->traceAckDelay(notifier, msgArg.maxWaitForReply/1000, NS_TO_MS(nsec)); + } } } else @@ -6361,9 +6434,12 @@ void IOService::pmTellCapabilityClientWithResponse( // not a client of ours // so we won't be waiting for response OUR_PMLog(kPMLogClientAcknowledge, msgRef, 0); + context->responseArray->setObject(msgIndex, replied); + } + if (context->notifyClients) { + context->notifyClients->setObject(msgIndex, object); } - context->responseArray->setObject(msgIndex, replied); } //********************************************************************************* @@ -6630,47 +6706,51 @@ bool IOService::responseValid( uint32_t refcon, int pid ) uint64_t nsec; char name[128]; - name[0] = '\0'; - proc_name(pid, name, sizeof(name)); clock_get_uptime(&now); AbsoluteTime_to_scalar(&start) = num->unsigned64BitValue(); SUB_ABSOLUTETIME(&now, &start); absolutetime_to_nanoseconds(now, &nsec); + if (pid != 0) { + name[0] = '\0'; + proc_name(pid, name, sizeof(name)); + + if (nsec > LOG_APP_RESPONSE_TIMES) + { + IOLog("PM response took %d ms (%d, %s)\n", NS_TO_MS(nsec), + pid, name); + } + + + if (nsec > LOG_APP_RESPONSE_MSG_TRACER) + { + // TODO: populate the messageType argument + getPMRootDomain()->pmStatsRecordApplicationResponse( + gIOPMStatsResponseSlow, + name, 0, NS_TO_MS(nsec), pid, object); + } + else + { + getPMRootDomain()->pmStatsRecordApplicationResponse( + gIOPMStatsResponsePrompt, + name, 0, NS_TO_MS(nsec), pid, object); + } + } + else { + getPMRootDomain()->traceAckDelay(object, 0, NS_TO_MS(nsec)); + } + if (kIOLogDebugPower & gIOKitDebug) { PM_LOG("Ack(%u) %u ms\n", (uint32_t) ordinalComponent, NS_TO_MS(nsec)); } - - // > 100 ms - if (nsec > LOG_APP_RESPONSE_TIMES) - { - IOLog("PM response took %d ms (%d, %s)\n", NS_TO_MS(nsec), - pid, name); - } - - if (nsec > LOG_APP_RESPONSE_MSG_TRACER) - { - // TODO: populate the messageType argument - getPMRootDomain()->pmStatsRecordApplicationResponse( - gIOPMStatsApplicationResponseSlow, - name, 0, NS_TO_MS(nsec), pid, object); - } - else - { - getPMRootDomain()->pmStatsRecordApplicationResponse( - gIOPMStatsApplicationResponsePrompt, - name, 0, NS_TO_MS(nsec), pid, object); - } - - theFlag = kOSBooleanFalse; } else if (object) { getPMRootDomain()->pmStatsRecordApplicationResponse( - gIOPMStatsApplicationResponsePrompt, + gIOPMStatsResponsePrompt, 0, 0, 0, pid, object); } @@ -7855,7 +7935,7 @@ bool IOService::actionPMReplyQueue( IOPMRequest * request, IOPMRequestQueue * qu OSString * name = (OSString *) request->fArg2; getPMRootDomain()->pmStatsRecordApplicationResponse( - gIOPMStatsApplicationResponseCancel, + gIOPMStatsResponseCancel, name ? name->getCStringNoCopy() : "", 0, 0, (int)(uintptr_t) request->fArg1, 0); } @@ -8101,7 +8181,9 @@ const char * IOService::getIOMessageString( uint32_t msg ) MSG_ENTRY( kIOMessageSystemWillRestart ), MSG_ENTRY( kIOMessageSystemWillPowerOn ), MSG_ENTRY( kIOMessageSystemCapabilityChange ), - MSG_ENTRY( kIOPMMessageLastCallBeforeSleep ) + MSG_ENTRY( kIOPMMessageLastCallBeforeSleep ), + MSG_ENTRY( kIOMessageSystemPagingOff ), + { 0, NULL } }; return IOFindNameForValue(msg, msgNames); diff --git a/iokit/Kernel/IOServicePMPrivate.h b/iokit/Kernel/IOServicePMPrivate.h index f332c23ee..0dbc58aca 100644 --- a/iokit/Kernel/IOServicePMPrivate.h +++ b/iokit/Kernel/IOServicePMPrivate.h @@ -549,10 +549,10 @@ enum { // PM Statistics & Diagnostics //****************************************************************************** -extern const OSSymbol *gIOPMStatsApplicationResponseTimedOut; -extern const OSSymbol *gIOPMStatsApplicationResponseCancel; -extern const OSSymbol *gIOPMStatsApplicationResponseSlow; -extern const OSSymbol *gIOPMStatsApplicationResponsePrompt; +extern const OSSymbol *gIOPMStatsResponseTimedOut; +extern const OSSymbol *gIOPMStatsResponseCancel; +extern const OSSymbol *gIOPMStatsResponseSlow; +extern const OSSymbol *gIOPMStatsResponsePrompt; extern const OSSymbol *gIOPMStatsDriverPSChangeSlow; //****************************************************************************** diff --git a/iokit/Kernel/IOServicePrivate.h b/iokit/Kernel/IOServicePrivate.h index 4ad23fa1a..2041d0806 100644 --- a/iokit/Kernel/IOServicePrivate.h +++ b/iokit/Kernel/IOServicePrivate.h @@ -64,6 +64,7 @@ enum { kIOServiceRecursing = 0x00100000, kIOServiceNeedWillTerminate = 0x00080000, kIOServiceWaitDetachState = 0x00040000, + kIOServiceConfigRunning = 0x00020000, }; // notify state @@ -88,6 +89,7 @@ public: OSOrderedSet * whence; OSDictionary * matching; + const OSSymbol * type; IOServiceMatchingNotificationHandler handler; IOServiceNotificationHandler compatHandler; void * target; diff --git a/iokit/Kernel/IOSimpleReporter.cpp b/iokit/Kernel/IOSimpleReporter.cpp index 6707bda89..de430f566 100644 --- a/iokit/Kernel/IOSimpleReporter.cpp +++ b/iokit/Kernel/IOSimpleReporter.cpp @@ -37,7 +37,7 @@ OSDefineMetaClassAndStructors(IOSimpleReporter, IOReporter); IOSimpleReporter* IOSimpleReporter::with(IOService *reportingService, IOReportCategories categories, - IOReportUnits unit) + IOReportUnit unit) { IOSimpleReporter *reporter, *rval = NULL; @@ -65,7 +65,7 @@ finish: bool IOSimpleReporter::initWith(IOService *reportingService, IOReportCategories categories, - IOReportUnits unit) + IOReportUnit unit) { // fully specify the channel type for the superclass IOReportChannelType channelType = { diff --git a/iokit/Kernel/IOStartIOKit.cpp b/iokit/Kernel/IOStartIOKit.cpp index f9d9fc018..6eb48b713 100644 --- a/iokit/Kernel/IOStartIOKit.cpp +++ b/iokit/Kernel/IOStartIOKit.cpp @@ -77,7 +77,7 @@ void IOKitInitializeTime( void ) IOService::resourceMatching("IONVRAM"), &t ); #endif - clock_initialize_calendar(); + clock_initialize_calendar(); } void iokit_post_constructor_init(void) @@ -112,9 +112,6 @@ void iokit_post_constructor_init(void) } } -// From -extern int debug_mode; - /***** * Pointer into bootstrap KLD segment for functions never used past startup. */ @@ -137,10 +134,6 @@ void StartIOKit( void * p1, void * p2, void * p3, void * p4 ) // Compat for boot-args gIOKitTrace |= (gIOKitDebug & kIOTraceCompatBootArgs); - // Check for the log synchronous bit set in io - if (gIOKitDebug & kIOLogSynchronous) - debug_mode = true; - if( PE_parse_boot_argn( "pmtimeout", &debugFlags, sizeof (debugFlags) )) gCanSleepTimeout = debugFlags; // diff --git a/iokit/Kernel/IOStateReporter.cpp b/iokit/Kernel/IOStateReporter.cpp index f6e0b7340..e1214dc55 100644 --- a/iokit/Kernel/IOStateReporter.cpp +++ b/iokit/Kernel/IOStateReporter.cpp @@ -40,7 +40,7 @@ IOStateReporter* IOStateReporter::with(IOService *reportingService, IOReportCategories categories, int nstates, - IOReportUnits unit/* = kIOReportUnitHWTicks*/) + IOReportUnit unit/* = kIOReportUnitHWTicks*/) { IOStateReporter *reporter, *rval = NULL; @@ -68,7 +68,7 @@ bool IOStateReporter::initWith(IOService *reportingService, IOReportCategories categories, int16_t nstates, - IOReportUnits unit) + IOReportUnit unit) { bool success = false; diff --git a/iokit/Kernel/IOStatistics.cpp b/iokit/Kernel/IOStatistics.cpp index b67de2511..f3771602a 100644 --- a/iokit/Kernel/IOStatistics.cpp +++ b/iokit/Kernel/IOStatistics.cpp @@ -815,8 +815,11 @@ int IOStatistics::getUserClientStatistics(sysctl_req *req) goto exit; } - SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag)); - + error = SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag)); + if (error) { + goto exit; + } + LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag); buffer = (char*)kalloc(calculatedSize); diff --git a/iokit/Kernel/IOStringFuncs.c b/iokit/Kernel/IOStringFuncs.c index d536b7243..c4f9458fe 100644 --- a/iokit/Kernel/IOStringFuncs.c +++ b/iokit/Kernel/IOStringFuncs.c @@ -86,7 +86,6 @@ long strtol(const char *nptr, char **endptr, int base); unsigned long strtoul(const char *nptr, char **endptr, int base); quad_t strtoq(const char *nptr, char **endptr, int base); u_quad_t strtouq(const char *nptr, char **endptr, int base); -char *strchr(const char *str, int ch); char *strncat(char *s1, const char *s2, unsigned long n); @@ -469,19 +468,6 @@ strtouq(const char *nptr, } -/* - * - */ - -char *strchr(const char *str, int ch) -{ - do { - if (*str == ch) - return(__CAST_AWAY_QUALIFIER(str, const, char *)); - } while (*str++); - return ((char *) 0); -} - /* * */ diff --git a/iokit/Kernel/IOTimerEventSource.cpp b/iokit/Kernel/IOTimerEventSource.cpp index df939da91..bb7acea43 100644 --- a/iokit/Kernel/IOTimerEventSource.cpp +++ b/iokit/Kernel/IOTimerEventSource.cpp @@ -47,9 +47,9 @@ __END_DECLS #define super IOEventSource OSDefineMetaClassAndStructors(IOTimerEventSource, IOEventSource) -OSMetaClassDefineReservedUnused(IOTimerEventSource, 0); -OSMetaClassDefineReservedUnused(IOTimerEventSource, 1); -OSMetaClassDefineReservedUnused(IOTimerEventSource, 2); +OSMetaClassDefineReservedUsed(IOTimerEventSource, 0); +OSMetaClassDefineReservedUsed(IOTimerEventSource, 1); +OSMetaClassDefineReservedUsed(IOTimerEventSource, 2); OSMetaClassDefineReservedUnused(IOTimerEventSource, 3); OSMetaClassDefineReservedUnused(IOTimerEventSource, 4); OSMetaClassDefineReservedUnused(IOTimerEventSource, 5); @@ -95,6 +95,28 @@ do { \ // Timeout handler function. This function is called by the kernel when // the timeout interval expires. // + +static __inline__ void +InvokeAction(IOTimerEventSource::Action action, IOTimerEventSource * ts, + OSObject * owner, IOWorkLoop * workLoop) +{ + bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false; + + if (trace) + IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION), + VM_KERNEL_ADDRHIDE(action), VM_KERNEL_ADDRHIDE(owner)); + + (*action)(owner, ts); + +#if CONFIG_DTRACE + DTRACE_TMR3(iotescallout__expire, Action, action, OSObject, owner, void, workLoop); +#endif + + if (trace) + IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION), + VM_KERNEL_UNSLIDE(action), VM_KERNEL_ADDRHIDE(owner)); +} + void IOTimerEventSource::timeout(void *self) { IOTimerEventSource *me = (IOTimerEventSource *) self; @@ -113,20 +135,7 @@ void IOTimerEventSource::timeout(void *self) doit = (Action) me->action; if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) { - bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false; - - if (trace) - IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION), - VM_KERNEL_UNSLIDE(doit), (uintptr_t) me->owner); - - (*doit)(me->owner, me); -#if CONFIG_DTRACE - DTRACE_TMR3(iotescallout__expire, Action, doit, OSObject, me->owner, void, me->workLoop); -#endif - - if (trace) - IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION), - VM_KERNEL_UNSLIDE(doit), (uintptr_t) me->owner); + InvokeAction(doit, me, me->owner, me->workLoop); } IOStatisticsOpenGate(); wl->openGate(); @@ -155,20 +164,7 @@ void IOTimerEventSource::timeoutAndRelease(void * self, void * c) doit = (Action) me->action; if (doit && (me->reserved->calloutGeneration == count)) { - bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false; - - if (trace) - IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION), - VM_KERNEL_UNSLIDE(doit), (uintptr_t) me->owner); - - (*doit)(me->owner, me); -#if CONFIG_DTRACE - DTRACE_TMR3(iotescallout__expire, Action, doit, OSObject, me->owner, void, me->workLoop); -#endif - - if (trace) - IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION), - VM_KERNEL_UNSLIDE(doit), (uintptr_t) me->owner); + InvokeAction(doit, me, me->owner, me->workLoop); } IOStatisticsOpenGate(); wl->openGate(); @@ -179,13 +175,99 @@ void IOTimerEventSource::timeoutAndRelease(void * self, void * c) me->release(); } +// -- work loop delivery + +bool IOTimerEventSource::checkForWork() +{ + Action doit; + + if (reserved + && (reserved->calloutGenerationSignaled == reserved->calloutGeneration) + && enabled && (doit = (Action) action)) + { + reserved->calloutGenerationSignaled = ~reserved->calloutGeneration; + InvokeAction(doit, this, owner, workLoop); + } + + return false; +} + +void IOTimerEventSource::timeoutSignaled(void * self, void * c) +{ + IOTimerEventSource *me = (IOTimerEventSource *) self; + + me->reserved->calloutGenerationSignaled = (SInt32)(long) c; + if (me->enabled) me->signalWorkAvailable(); +} + +// -- + void IOTimerEventSource::setTimeoutFunc() { + thread_call_priority_t pri; + uint32_t options; + + if (reserved) panic("setTimeoutFunc already %p, %p", this, reserved); + // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used, // not a subclassed implementation reserved = IONew(ExpansionData, 1); - calloutEntry = (void *) thread_call_allocate((thread_call_func_t) &IOTimerEventSource::timeoutAndRelease, - (thread_call_param_t) this); + reserved->calloutGenerationSignaled = ~reserved->calloutGeneration; + options = abstime; + abstime = 0; + + thread_call_options_t tcoptions = 0; + thread_call_func_t func = NULL; + + switch (kIOTimerEventSourceOptionsPriorityMask & options) + { + case kIOTimerEventSourceOptionsPriorityHigh: + pri = THREAD_CALL_PRIORITY_HIGH; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityKernel: + pri = THREAD_CALL_PRIORITY_KERNEL; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityKernelHigh: + pri = THREAD_CALL_PRIORITY_KERNEL_HIGH; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityUser: + pri = THREAD_CALL_PRIORITY_USER; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityLow: + pri = THREAD_CALL_PRIORITY_LOW; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityWorkLoop: + pri = THREAD_CALL_PRIORITY_KERNEL; + tcoptions |= THREAD_CALL_OPTIONS_SIGNAL; + if (kIOTimerEventSourceOptionsAllowReenter & options) break; + func = &IOTimerEventSource::timeoutSignaled; + break; + + default: + break; + } + + assertf(func, "IOTimerEventSource options 0x%x", options); + if (!func) return; // init will fail + + if (THREAD_CALL_OPTIONS_SIGNAL & tcoptions) flags |= kActive; + else flags |= kPassive; + + if (!(kIOTimerEventSourceOptionsAllowReenter & options)) tcoptions |= THREAD_CALL_OPTIONS_ONCE; + + calloutEntry = (void *) thread_call_allocate_with_options(func, + (thread_call_param_t) this, pri, tcoptions); + assert(calloutEntry); } bool IOTimerEventSource::init(OSObject *inOwner, Action inAction) @@ -202,12 +284,18 @@ bool IOTimerEventSource::init(OSObject *inOwner, Action inAction) return true; } +bool IOTimerEventSource::init(uint32_t options, OSObject *inOwner, Action inAction) +{ + abstime = options; + return (init(inOwner, inAction)); +} + IOTimerEventSource * -IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction) +IOTimerEventSource::timerEventSource(uint32_t inOptions, OSObject *inOwner, Action inAction) { IOTimerEventSource *me = new IOTimerEventSource; - if (me && !me->init(inOwner, inAction)) { + if (me && !me->init(inOptions, inOwner, inAction)) { me->release(); return 0; } @@ -215,11 +303,25 @@ IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction) return me; } +#define _thread_call_cancel(tc) ((kActive & flags) ? thread_call_cancel_wait((tc)) : thread_call_cancel((tc))) + +IOTimerEventSource * +IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction) +{ + return (IOTimerEventSource::timerEventSource( + kIOTimerEventSourceOptionsPriorityKernelHigh, + inOwner, inAction)); +} + void IOTimerEventSource::free() { if (calloutEntry) { + __assert_only bool freed; + cancelTimeout(); - thread_call_free((thread_call_t) calloutEntry); + + freed = thread_call_free((thread_call_t) calloutEntry); + assert(freed); } if (reserved) @@ -232,9 +334,9 @@ void IOTimerEventSource::cancelTimeout() { if (reserved) reserved->calloutGeneration++; - bool active = thread_call_cancel((thread_call_t) calloutEntry); + bool active = _thread_call_cancel((thread_call_t) calloutEntry); AbsoluteTime_to_scalar(&abstime) = 0; - if (active && reserved) + if (active && reserved && (kPassive & flags)) { release(); workLoop->release(); @@ -252,9 +354,9 @@ void IOTimerEventSource::disable() { if (reserved) reserved->calloutGeneration++; - bool active = thread_call_cancel((thread_call_t) calloutEntry); + bool active = _thread_call_cancel((thread_call_t) calloutEntry); super::disable(); - if (active && reserved) + if (active && reserved && (kPassive & flags)) { release(); workLoop->release(); @@ -302,11 +404,17 @@ IOReturn IOTimerEventSource::setTimeout(mach_timespec_t interval) IOReturn IOTimerEventSource::setTimeout(AbsoluteTime interval) { AbsoluteTime end; + clock_absolutetime_interval_to_deadline(interval, &end); + return wakeAtTime(end); +} - clock_get_uptime(&end); - ADD_ABSOLUTETIME(&end, &interval); +IOReturn IOTimerEventSource::setTimeout(uint32_t options, + AbsoluteTime abstime, AbsoluteTime leeway) +{ + AbsoluteTime end; + clock_continuoustime_interval_to_deadline(abstime, &end); + return wakeAtTime(options, end, leeway); - return wakeAtTime(end); } IOReturn IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks) @@ -355,6 +463,11 @@ void IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop) } IOReturn IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime) +{ + return wakeAtTime(0, inAbstime, 0); +} + +IOReturn IOTimerEventSource::wakeAtTime(uint32_t options, AbsoluteTime inAbstime, AbsoluteTime leeway) { if (!action) return kIOReturnNoResources; @@ -362,21 +475,33 @@ IOReturn IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime) abstime = inAbstime; if ( enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop ) { + uint32_t tcoptions = 0; + + if (kIOTimeOptionsWithLeeway & options) tcoptions |= THREAD_CALL_DELAY_LEEWAY; + if (kIOTimeOptionsContinuous & options) tcoptions |= THREAD_CALL_CONTINUOUS; + if (reserved) { - retain(); - workLoop->retain(); + if (kPassive & flags) + { + retain(); + workLoop->retain(); + } reserved->workLoop = workLoop; reserved->calloutGeneration++; - if (thread_call_enter1_delayed((thread_call_t) calloutEntry, - (void *)(uintptr_t) reserved->calloutGeneration, inAbstime)) + if (thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry, + (void *)(uintptr_t) reserved->calloutGeneration, inAbstime, leeway, tcoptions) + && (kPassive & flags)) { release(); workLoop->release(); } } else - thread_call_enter_delayed((thread_call_t) calloutEntry, inAbstime); + { + thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry, + NULL, inAbstime, leeway, tcoptions); + } } return kIOReturnSuccess; diff --git a/iokit/Kernel/IOUserClient.cpp b/iokit/Kernel/IOUserClient.cpp index 985e3b201..b16a516bb 100644 --- a/iokit/Kernel/IOUserClient.cpp +++ b/iokit/Kernel/IOUserClient.cpp @@ -647,6 +647,7 @@ public: void * reference, vm_size_t referenceSize, bool clientIs64 ); virtual void free() APPLE_KEXT_OVERRIDE; + void invalidatePort(void); static bool _handler( void * target, void * ref, IOService * newService, IONotifier * notifier ); @@ -680,6 +681,7 @@ public: bool clientIs64 ); virtual void free() APPLE_KEXT_OVERRIDE; + void invalidatePort(void); static IOReturn _handler( void * target, void * ref, UInt32 messageType, IOService * provider, @@ -785,6 +787,11 @@ bool IOServiceUserNotification::init( mach_port_t port, natural_t type, return( true ); } +void IOServiceUserNotification::invalidatePort(void) +{ + if (pingMsg) pingMsg->msgHdr.msgh_remote_port = MACH_PORT_NULL; +} + void IOServiceUserNotification::free( void ) { PingMsg * _pingMsg; @@ -941,6 +948,11 @@ bool IOServiceMessageUserNotification::init( mach_port_t port, natural_t type, return( true ); } +void IOServiceMessageUserNotification::invalidatePort(void) +{ + if (pingMsg) pingMsg->msgHdr.msgh_remote_port = MACH_PORT_NULL; +} + void IOServiceMessageUserNotification::free( void ) { PingMsg * _pingMsg; @@ -1922,38 +1934,36 @@ kern_return_t is_io_object_get_superclass( io_name_t obj_name, io_name_t class_name) { - const OSMetaClass* my_obj = NULL; - const OSMetaClass* superclass = NULL; - const OSSymbol *my_name = NULL; - const char *my_cstr = NULL; + IOReturn ret; + const OSMetaClass * meta; + const OSMetaClass * super; + const OSSymbol * name; + const char * cstr; - if (!obj_name || !class_name) - return (kIOReturnBadArgument); + if (!obj_name || !class_name) return (kIOReturnBadArgument); + if (master_port != master_device_port) return( kIOReturnNotPrivileged); - if( master_port != master_device_port) - return( kIOReturnNotPrivileged); + ret = kIOReturnNotFound; + meta = 0; + do + { + name = OSSymbol::withCString(obj_name); + if (!name) break; + meta = OSMetaClass::copyMetaClassWithName(name); + if (!meta) break; + super = meta->getSuperClass(); + if (!super) break; + cstr = super->getClassName(); + if (!cstr) break; + strlcpy(class_name, cstr, sizeof(io_name_t)); + ret = kIOReturnSuccess; + } + while (false); - my_name = OSSymbol::withCString(obj_name); - - if (my_name) { - my_obj = OSMetaClass::getMetaClassWithName(my_name); - my_name->release(); - } - if (my_obj) { - superclass = my_obj->getSuperClass(); - } - - if (!superclass) { - return( kIOReturnNotFound ); - } + OSSafeReleaseNULL(name); + if (meta) meta->releaseMetaClass(); - my_cstr = superclass->getClassName(); - - if (my_cstr) { - strlcpy(class_name, my_cstr, sizeof(io_name_t)); - return( kIOReturnSuccess ); - } - return (kIOReturnNotFound); + return (ret); } /* Routine io_object_get_bundle_identifier */ @@ -1962,38 +1972,36 @@ kern_return_t is_io_object_get_bundle_identifier( io_name_t obj_name, io_name_t bundle_name) { - const OSMetaClass* my_obj = NULL; - const OSSymbol *my_name = NULL; - const OSSymbol *identifier = NULL; - const char *my_cstr = NULL; + IOReturn ret; + const OSMetaClass * meta; + const OSSymbol * name; + const OSSymbol * identifier; + const char * cstr; - if (!obj_name || !bundle_name) - return (kIOReturnBadArgument); + if (!obj_name || !bundle_name) return (kIOReturnBadArgument); + if (master_port != master_device_port) return( kIOReturnNotPrivileged); - if( master_port != master_device_port) - return( kIOReturnNotPrivileged); - - my_name = OSSymbol::withCString(obj_name); - - if (my_name) { - my_obj = OSMetaClass::getMetaClassWithName(my_name); - my_name->release(); - } + ret = kIOReturnNotFound; + meta = 0; + do + { + name = OSSymbol::withCString(obj_name); + if (!name) break; + meta = OSMetaClass::copyMetaClassWithName(name); + if (!meta) break; + identifier = meta->getKmodName(); + if (!identifier) break; + cstr = identifier->getCStringNoCopy(); + if (!cstr) break; + strlcpy(bundle_name, identifier->getCStringNoCopy(), sizeof(io_name_t)); + ret = kIOReturnSuccess; + } + while (false); - if (my_obj) { - identifier = my_obj->getKmodName(); - } - if (!identifier) { - return( kIOReturnNotFound ); - } - - my_cstr = identifier->getCStringNoCopy(); - if (my_cstr) { - strlcpy(bundle_name, identifier->getCStringNoCopy(), sizeof(io_name_t)); - return( kIOReturnSuccess ); - } + OSSafeReleaseNULL(name); + if (meta) meta->releaseMetaClass(); - return (kIOReturnBadArgument); + return (ret); } /* Routine io_object_conforms_to */ @@ -2328,7 +2336,8 @@ static kern_return_t internal_io_service_add_notification( else if( (sym == gIOMatchedNotification) || (sym == gIOFirstMatchNotification)) userMsgType = kIOServiceMatchedNotificationType; - else if( sym == gIOTerminatedNotification) + else if ((sym == gIOTerminatedNotification) + || (sym == gIOWillTerminateNotification)) userMsgType = kIOServiceTerminatedNotificationType; else userMsgType = kLastIOKitNotificationType; @@ -2337,7 +2346,6 @@ static kern_return_t internal_io_service_add_notification( if( userNotify && !userNotify->init( port, userMsgType, reference, referenceSize, client64)) { - iokit_release_port_send(port); userNotify->release(); userNotify = 0; } @@ -2355,6 +2363,13 @@ static kern_return_t internal_io_service_add_notification( } while( false ); + if ((kIOReturnSuccess != err) && userNotify) + { + userNotify->invalidatePort(); + userNotify->release(); + userNotify = 0; + } + if( sym) sym->release(); if( dict) @@ -2530,7 +2545,6 @@ static kern_return_t internal_io_service_add_interest_notification( reference, referenceSize, kIOUserNotifyMaxMessageSize, client64 )) { - iokit_release_port_send(port); userNotify->release(); userNotify = 0; } @@ -2550,6 +2564,13 @@ static kern_return_t internal_io_service_add_interest_notification( } while( false ); + if ((kIOReturnSuccess != err) && userNotify) + { + userNotify->invalidatePort(); + userNotify->release(); + userNotify = 0; + } + return( err ); } @@ -3974,6 +3995,8 @@ kern_return_t is_io_connect_async_method args.asyncReference = reference; args.asyncReferenceCount = referenceCnt; + args.structureVariableOutputData = 0; + args.scalarInput = scalar_input; args.scalarInputCount = scalar_inputCnt; args.structureInput = inband_input; @@ -5120,7 +5143,7 @@ kern_return_t is_io_catalog_get_data( vm_size_t size; size = s->getLength(); - kr = vm_allocate(kernel_map, &data, size, VM_FLAGS_ANYWHERE); + kr = vm_allocate_kernel(kernel_map, &data, size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IOKIT); if ( kr == kIOReturnSuccess ) { bcopy(s->text(), (void *)data, size); kr = vm_map_copyin(kernel_map, (vm_map_address_t)data, diff --git a/iokit/Kernel/IOWorkLoop.cpp b/iokit/Kernel/IOWorkLoop.cpp index e3896d38b..a5eb85181 100644 --- a/iokit/Kernel/IOWorkLoop.cpp +++ b/iokit/Kernel/IOWorkLoop.cpp @@ -224,7 +224,7 @@ void IOWorkLoop::free() is = IOSimpleLockLockDisableInterrupt(workToDoLock); SETP(&fFlags, kLoopTerminate); - thread_wakeup_one((void *) &workToDo); + thread_wakeup_thread((void *) &workToDo, workThread); IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); openGate(); @@ -350,7 +350,7 @@ void IOWorkLoop::disableAllInterrupts() const goto abort; if (traceWL) - IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); + IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), VM_KERNEL_ADDRHIDE(this)); bool more; do { @@ -363,12 +363,12 @@ void IOWorkLoop::disableAllInterrupts() const for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) { if (traceES) - IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); + IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(evnt)); more |= evnt->checkForWork(); if (traceES) - IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); + IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(evnt)); if (ISSETP(&fFlags, kLoopTerminate)) goto abort; @@ -382,7 +382,7 @@ void IOWorkLoop::disableAllInterrupts() const res = true; if (traceWL) - IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); + IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), VM_KERNEL_ADDRHIDE(this)); abort: openGate(); @@ -445,7 +445,7 @@ void IOWorkLoop::signalWorkAvailable() if (workToDoLock) { IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); workToDo = true; - thread_wakeup_one((void *) &workToDo); + thread_wakeup_thread((void *) &workToDo, workThread); IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); } } @@ -633,7 +633,10 @@ IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource) * checkForWork() here. We're just testing to see if it's the same or not. * */ - if (controlG) { + + if (IOEventSource::kPassive & inEventSource->flags) result = false; + else if (IOEventSource::kActive & inEventSource->flags) result = true; + else if (controlG) { void * ptr1; void * ptr2; diff --git a/iokit/Kernel/RootDomainUserClient.cpp b/iokit/Kernel/RootDomainUserClient.cpp index ecd0cc1f3..7a909d998 100644 --- a/iokit/Kernel/RootDomainUserClient.cpp +++ b/iokit/Kernel/RootDomainUserClient.cpp @@ -198,7 +198,7 @@ IOReturn RootDomainUserClient::secureSetUserAssertionLevels( } IOReturn RootDomainUserClient::secureGetSystemSleepType( - uint32_t *outSleepType) + uint32_t *outSleepType, uint32_t *sleepTimer) { int admin_priv = 0; IOReturn ret; @@ -207,7 +207,7 @@ IOReturn RootDomainUserClient::secureGetSystemSleepType( admin_priv = (kIOReturnSuccess == ret); if (admin_priv && fOwner) { - ret = fOwner->getSystemSleepType(outSleepType); + ret = fOwner->getSystemSleepType(outSleepType, sleepTimer); } else { ret = kIOReturnNotPrivileged; } @@ -333,10 +333,11 @@ IOReturn RootDomainUserClient::externalMethod( break; case kPMGetSystemSleepType: - if (1 == arguments->scalarOutputCount) + if (2 == arguments->scalarOutputCount) { ret = this->secureGetSystemSleepType( - (uint32_t *) &arguments->scalarOutput[0]); + (uint32_t *) &arguments->scalarOutput[0], + (uint32_t *) &arguments->scalarOutput[1]); } break; diff --git a/iokit/Kernel/RootDomainUserClient.h b/iokit/Kernel/RootDomainUserClient.h index b37f71029..f4b815bce 100644 --- a/iokit/Kernel/RootDomainUserClient.h +++ b/iokit/Kernel/RootDomainUserClient.h @@ -67,7 +67,7 @@ private: IOReturn secureSetUserAssertionLevels(uint32_t assertionBitfield); - IOReturn secureGetSystemSleepType( uint32_t *sleepType ); + IOReturn secureGetSystemSleepType( uint32_t *sleepType, uint32_t *sleepTimer); public: diff --git a/iokit/Kernel/i386/IOKeyStoreHelper.cpp b/iokit/Kernel/i386/IOKeyStoreHelper.cpp index a1d41b8d0..17ebea802 100644 --- a/iokit/Kernel/i386/IOKeyStoreHelper.cpp +++ b/iokit/Kernel/i386/IOKeyStoreHelper.cpp @@ -47,6 +47,13 @@ IOGetBootKeyStoreData(void); void IOSetKeyStoreData(IOMemoryDescriptor * data); +// APFS +static volatile UInt32 apfsKeyFetched = 0; +static IOMemoryDescriptor* apfsKeyData = NULL; + +IOMemoryDescriptor* IOGetAPFSKeyStoreData(); +void IOSetAPFSKeyStoreData(IOMemoryDescriptor* data); + __END_DECLS #if 1 @@ -102,3 +109,52 @@ IOGetBootKeyStoreData(void) return memoryDescriptor; } + +// APFS volume key fetcher + +// Store in-memory key (could be used by IOHibernateDone) +void +IOSetAPFSKeyStoreData(IOMemoryDescriptor* data) +{ + // Do not allow re-fetching of the boot_args key by passing NULL here. + if (data != NULL) + { + apfsKeyData = data; + apfsKeyFetched = 0; + } +} + +// Retrieve any key we may have (stored in boot_args or by Hibernate) +IOMemoryDescriptor* +IOGetAPFSKeyStoreData() +{ + // Check if someone got the key before us + if (!OSCompareAndSwap(0, 1, &apfsKeyFetched)) + return NULL; + + // Do we have in-memory key? + if (apfsKeyData) + { + IOMemoryDescriptor* data = apfsKeyData; + apfsKeyData = NULL; + return data; + } + + // Looks like there was no in-memory key and it's the first call - try boot_args + boot_args* args = (boot_args*)PE_state.bootArgs; + + DEBG("%s: data at address %u size %u\n", __func__, args->apfsDataStart, args->apfsDataSize); + if (args->apfsDataStart == 0) + return NULL; + + // We have the key in the boot_args, create IOMemoryDescriptor for the blob + IOAddressRange ranges; + ranges.address = args->apfsDataStart; + ranges.length = args->apfsDataSize; + + const IOOptionBits options = kIODirectionInOut | kIOMemoryTypePhysical64 | kIOMemoryMapperNone; + + IOMemoryDescriptor* memoryDescriptor = IOMemoryDescriptor::withOptions(&ranges, 1, 0, NULL, options); + DEBG("%s: memory descriptor %p\n", __func__, memoryDescriptor); + return memoryDescriptor; +} diff --git a/iokit/Tests/TestIOMemoryDescriptor.cpp b/iokit/Tests/TestIOMemoryDescriptor.cpp index 59ce35546..11780e5f2 100644 --- a/iokit/Tests/TestIOMemoryDescriptor.cpp +++ b/iokit/Tests/TestIOMemoryDescriptor.cpp @@ -110,6 +110,204 @@ static int IOMultMemoryDescriptorTest(int newValue) return (0); } + + +// +static int +IODMACommandForceDoubleBufferTest(int newValue) +{ + IOReturn ret; + IOBufferMemoryDescriptor * bmd; + IODMACommand * dma; + uint32_t dir, data; + IODMACommand::SegmentOptions segOptions = + { + .fStructSize = sizeof(segOptions), + .fNumAddressBits = 64, + .fMaxSegmentSize = 0x2000, + .fMaxTransferSize = 128*1024, + .fAlignment = 1, + .fAlignmentLength = 1, + .fAlignmentInternalSegments = 1 + }; + IODMACommand::Segment64 segments[1]; + UInt32 numSegments; + UInt64 dmaOffset; + + + for (dir = kIODirectionIn; ; dir++) + { + bmd = IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task, + dir | kIOMemoryPageable, ptoa(8)); + assert(bmd); + + ((uint32_t*) bmd->getBytesNoCopy())[0] = 0x53535300 | dir; + + ret = bmd->prepare((IODirection) dir); + assert(kIOReturnSuccess == ret); + + dma = IODMACommand::withSpecification(kIODMACommandOutputHost64, &segOptions, + kIODMAMapOptionMapped, + NULL, NULL); + assert(dma); + ret = dma->setMemoryDescriptor(bmd, true); + assert(kIOReturnSuccess == ret); + + ret = dma->synchronize(IODMACommand::kForceDoubleBuffer | kIODirectionOut); + assert(kIOReturnSuccess == ret); + + dmaOffset = 0; + numSegments = 1; + ret = dma->gen64IOVMSegments(&dmaOffset, &segments[0], &numSegments); + assert(kIOReturnSuccess == ret); + assert(1 == numSegments); + + if (kIODirectionOut & dir) + { + data = ((uint32_t*) bmd->getBytesNoCopy())[0]; + assertf((0x53535300 | dir) == data, "mismatch 0x%x", data); + } + if (kIODirectionIn & dir) + { + IOMappedWrite32(segments[0].fIOVMAddr, 0x11223300 | dir); + } + + ret = dma->clearMemoryDescriptor(true); + assert(kIOReturnSuccess == ret); + dma->release(); + + bmd->complete((IODirection) dir); + + if (kIODirectionIn & dir) + { + data = ((uint32_t*) bmd->getBytesNoCopy())[0]; + assertf((0x11223300 | dir) == data, "mismatch 0x%x", data); + } + + bmd->release(); + + if (dir == kIODirectionInOut) break; + } + + return (0); +} + +// +static int +IOMemoryRemoteTest(int newValue) +{ + IOReturn ret; + IOMemoryDescriptor * md; + IOByteCount offset, length; + addr64_t addr; + uint32_t idx; + + IODMACommand * dma; + IODMACommand::SegmentOptions segOptions = + { + .fStructSize = sizeof(segOptions), + .fNumAddressBits = 64, + .fMaxSegmentSize = 0x2000, + .fMaxTransferSize = 128*1024, + .fAlignment = 1, + .fAlignmentLength = 1, + .fAlignmentInternalSegments = 1 + }; + IODMACommand::Segment64 segments[1]; + UInt32 numSegments; + UInt64 dmaOffset; + + IOAddressRange ranges[2] = { + { 0x1234567890123456ULL, 0x1000 }, { 0x5432109876543210, 0x2000 }, + }; + + md = IOMemoryDescriptor::withAddressRanges(&ranges[0], 2, kIODirectionOutIn|kIOMemoryRemote, TASK_NULL); + assert(md); + +// md->map(); +// md->readBytes(0, &idx, sizeof(idx)); + + ret = md->prepare(kIODirectionOutIn); + assert(kIOReturnSuccess == ret); + + printf("remote md flags 0x%qx, r %d\n", + md->getFlags(), (0 != (kIOMemoryRemote & md->getFlags()))); + + for (offset = 0, idx = 0; true; offset += length, idx++) + { + addr = md->getPhysicalSegment(offset, &length, 0); + if (!length) break; + assert(idx < 2); + assert(addr == ranges[idx].address); + assert(length == ranges[idx].length); + } + assert(offset == md->getLength()); + + dma = IODMACommand::withSpecification(kIODMACommandOutputHost64, &segOptions, + kIODMAMapOptionUnmapped | kIODMAMapOptionIterateOnly, + NULL, NULL); + assert(dma); + ret = dma->setMemoryDescriptor(md, true); + assert(kIOReturnSuccess == ret); + + for (dmaOffset = 0, idx = 0; dmaOffset < md->getLength(); idx++) + { + numSegments = 1; + ret = dma->gen64IOVMSegments(&dmaOffset, &segments[0], &numSegments); + assert(kIOReturnSuccess == ret); + assert(1 == numSegments); + assert(idx < 2); + assert(segments[0].fIOVMAddr == ranges[idx].address); + assert(segments[0].fLength == ranges[idx].length); + } + assert(dmaOffset == md->getLength()); + + ret = dma->clearMemoryDescriptor(true); + assert(kIOReturnSuccess == ret); + dma->release(); + md->complete(kIODirectionOutIn); + md->release(); + + return (0); +} + +static IOReturn +IOMemoryPrefaultTest(uint32_t options) +{ + IOBufferMemoryDescriptor * bmd; + IOMemoryMap * map; + IOReturn kr; + uint32_t data; + uint32_t * p; + IOSimpleLock * lock; + + lock = IOSimpleLockAlloc(); + assert(lock); + + bmd = IOBufferMemoryDescriptor::inTaskWithOptions(current_task(), + kIODirectionOutIn | kIOMemoryPageable, ptoa(8)); + assert(bmd); + kr = bmd->prepare(); + assert(KERN_SUCCESS == kr); + + map = bmd->map(kIOMapPrefault); + assert(map); + + p = (typeof(p)) map->getVirtualAddress(); + IOSimpleLockLock(lock); + data = p[0]; + IOSimpleLockUnlock(lock); + + IOLog("IOMemoryPrefaultTest %d\n", data); + + map->release(); + bmd->release(); + IOSimpleLockFree(lock); + + return (kIOReturnSuccess); +} + + // static IOReturn ZeroLengthTest(int newValue) @@ -159,7 +357,6 @@ IODirectionPrepareNoZeroFillTest(int newValue) return (0); } - // static IOReturn IOMemoryMapTest(uint32_t options) @@ -236,6 +433,31 @@ IOMemoryMapCopyOnWriteTest(int newValue) return (0); } +static int +AllocationNameTest(int newValue) +{ + IOMemoryDescriptor * bmd; + kern_allocation_name_t name, prior; + + name = kern_allocation_name_allocate("com.apple.iokit.test", 0); + assert(name); + + prior = thread_set_allocation_name(name); + + bmd = IOBufferMemoryDescriptor::inTaskWithOptions(TASK_NULL, + kIODirectionOutIn | kIOMemoryPageable | kIOMemoryKernelUserShared, + ptoa(13)); + assert(bmd); + bmd->prepare(); + + thread_set_allocation_name(prior); + kern_allocation_name_release(name); + + if (newValue != 7) bmd->release(); + + return (0); +} + int IOMemoryDescriptorTest(int newValue) { int result; @@ -466,6 +688,12 @@ int IOMemoryDescriptorTest(int newValue) } #endif + result = IODMACommandForceDoubleBufferTest(newValue); + if (result) return (result); + + result = AllocationNameTest(newValue); + if (result) return (result); + result = IOMemoryMapCopyOnWriteTest(newValue); if (result) return (result); @@ -481,6 +709,12 @@ int IOMemoryDescriptorTest(int newValue) result = BadFixedAllocTest(newValue); if (result) return (result); + result = IOMemoryRemoteTest(newValue); + if (result) return (result); + + result = IOMemoryPrefaultTest(newValue); + if (result) return (result); + IOGeneralMemoryDescriptor * md; vm_offset_t data[2]; vm_size_t bsize = 16*1024*1024; @@ -488,7 +722,7 @@ int IOMemoryDescriptorTest(int newValue) kern_return_t kr; data[0] = data[1] = 0; - kr = vm_allocate(kernel_map, &data[0], bsize, VM_FLAGS_ANYWHERE); + kr = vm_allocate_kernel(kernel_map, &data[0], bsize, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IOKIT); assert(KERN_SUCCESS == kr); vm_inherit(kernel_map, data[0] + ptoa(1), ptoa(1), VM_INHERIT_NONE); diff --git a/iokit/Tests/Tests.cpp b/iokit/Tests/Tests.cpp index bc2d05b69..6336b67ac 100644 --- a/iokit/Tests/Tests.cpp +++ b/iokit/Tests/Tests.cpp @@ -177,6 +177,48 @@ #include #include "Tests.h" +#include +#include + +#if DEVELOPMENT || DEBUG + +static uint64_t gIOWorkLoopTestDeadline; + +static void +TESAction(OSObject * owner, IOTimerEventSource * tes) +{ + if (mach_absolute_time() < gIOWorkLoopTestDeadline) tes->setTimeout(1, kMicrosecondScale); +} + +static int +IOWorkLoopTest(int newValue) +{ + IOReturn err; + uint32_t idx; + IOWorkLoop * wl; + IOTimerEventSource * tes; + + wl = IOWorkLoop::workLoop(); + assert(wl); + tes = IOTimerEventSource::timerEventSource(kIOTimerEventSourceOptionsPriorityWorkLoop, wl, &TESAction); + assert(tes); + err = wl->addEventSource(tes); + assert(kIOReturnSuccess == err); + clock_interval_to_deadline(2000, kMillisecondScale, &gIOWorkLoopTestDeadline); + for (idx = 0; mach_absolute_time() < gIOWorkLoopTestDeadline; idx++) + { + tes->setTimeout(idx & 1023, kNanosecondScale); + } + tes->cancelTimeout(); + wl->removeEventSource(tes); + tes->release(); + wl->release(); + + return (0); +} + +#endif /* DEVELOPMENT || DEBUG */ + static int sysctl_iokittest(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { @@ -194,7 +236,13 @@ sysctl_iokittest(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused data->release(); } - if (changed && newValue) error = IOMemoryDescriptorTest(newValue); + if (changed && newValue) + { + error = IOWorkLoopTest(newValue); + assert(KERN_SUCCESS == error); + error = IOMemoryDescriptorTest(newValue); + assert(KERN_SUCCESS == error); + } #endif /* DEVELOPMENT || DEBUG */ return (error); diff --git a/iokit/bsddev/DINetBootHook.cpp b/iokit/bsddev/DINetBootHook.cpp index 62865b0a1..6ca295a81 100644 --- a/iokit/bsddev/DINetBootHook.cpp +++ b/iokit/bsddev/DINetBootHook.cpp @@ -140,7 +140,7 @@ extern "C" { dev_p <- device number generated from major/minor numbers Comments: */ -int di_root_image(const char *path, char devname[], dev_t *dev_p) +int di_root_image(const char *path, char *devname, size_t devsz, dev_t *dev_p) { IOReturn res = 0; IOService * controller = 0; @@ -196,8 +196,7 @@ int di_root_image(const char *path, char devname[], dev_t *dev_p) myDevName = OSDynamicCast(OSString, controller->getProperty(kDIRootImageDevNameKey)); if (myDevName) { - /* rootdevice is 16 chars in bsd_init.c */ - strlcpy(devname, myDevName->getCStringNoCopy(), 16); + strlcpy(devname, myDevName->getCStringNoCopy(), devsz); } else { IOLog("could not get %s\n", kDIRootImageDevNameKey); res = kIOReturnError; @@ -216,6 +215,66 @@ NoIOHDIXController: return res; } +int +di_root_ramfile_buf(void *buf, size_t bufsz, char *devname, size_t devsz, dev_t *dev_p) +{ + IOReturn res = 0; + IOService *controller = 0; + OSNumber *myResult = 0; + OSString *myDevName = 0; + OSNumber *myDevT = 0; + IOMemoryDescriptor *mem = 0; + + mem = IOMemoryDescriptor::withAddress(buf, bufsz, kIODirectionInOut); + assert(mem); + + controller = di_load_controller(); + if (controller) { + /* attach the image */ + controller->setProperty(kDIRootRamFileKey, mem); + controller->release(); + } else { + res = kIOReturnNotFound; + goto out; + } + + myResult = OSDynamicCast(OSNumber, controller->getProperty(kDIRootImageResultKey)); + res = kIOReturnError; + if (myResult) { + res = myResult->unsigned32BitValue(); + } + + if (res) { + IOLog("%s is 0x%08X/%d\n", kDIRootImageResultKey, res, res); + goto out; + } + + myDevT = OSDynamicCast(OSNumber, controller->getProperty(kDIRootImageDevTKey)); + if (myDevT) + *dev_p = myDevT->unsigned32BitValue(); + else { + IOLog("could not get %s\n", kDIRootImageDevTKey); + res = kIOReturnError; + goto out; + } + + myDevName = OSDynamicCast(OSString, controller->getProperty(kDIRootImageDevNameKey)); + if (myDevName) { + strlcpy(devname, myDevName->getCStringNoCopy(), devsz); + } else { + IOLog("could not get %s\n", kDIRootImageDevNameKey); + res = kIOReturnError; + goto out; + } + +out: + if (res) { + OSSafeReleaseNULL(mem); + } + + return res; +} + void di_root_ramfile( IORegistryEntry * entry ) { OSData * data; diff --git a/iokit/bsddev/DINetBootHook.h b/iokit/bsddev/DINetBootHook.h index 2f44361b0..172679524 100644 --- a/iokit/bsddev/DINetBootHook.h +++ b/iokit/bsddev/DINetBootHook.h @@ -94,8 +94,9 @@ extern "C" { dev_p <- combination of major/minor node Comments: */ -int di_root_image(const char *path, char devname[], dev_t *dev_p); +int di_root_image(const char *path, char *devname, size_t devsz, dev_t *dev_p); void di_root_ramfile( IORegistryEntry * entry ); +int di_root_ramfile_buf(void *buf, size_t bufsz, char *devname, size_t devsz, dev_t *dev_p); #ifdef __cplusplus }; diff --git a/iokit/bsddev/IOKitBSDInit.cpp b/iokit/bsddev/IOKitBSDInit.cpp index a35aa5495..f68eebd1e 100644 --- a/iokit/bsddev/IOKitBSDInit.cpp +++ b/iokit/bsddev/IOKitBSDInit.cpp @@ -58,7 +58,13 @@ extern int mdevgetrange(int devid, uint64_t *base, uint64_t *size); extern void di_root_ramfile(IORegistryEntry * entry); -#if DEVELOPMENT +#if CONFIG_EMBEDDED +#define IOPOLLED_COREFILE (CONFIG_KDP_INTERACTIVE_DEBUGGING) +#define kIOCoreDumpPath "/private/var/vm/kernelcore" +#define kIOCoreDumpSize 350ULL*1024ULL*1024ULL +// leave free space on volume: +#define kIOCoreDumpFreeSize 350ULL*1024ULL*1024ULL +#elif DEVELOPMENT #define IOPOLLED_COREFILE 1 // no sizing #define kIOCoreDumpSize 0ULL @@ -657,6 +663,37 @@ bool IORamDiskBSDRoot(void) void IOSecureBSDRoot(const char * rootName) { +#if CONFIG_EMBEDDED + int tmpInt; + IOReturn result; + IOPlatformExpert *pe; + OSDictionary *matching; + const OSSymbol *functionName = OSSymbol::withCStringNoCopy("SecureRootName"); + + matching = IOService::serviceMatching("IOPlatformExpert"); + assert(matching); + pe = (IOPlatformExpert *) IOService::waitForMatchingService(matching, 30ULL * kSecondScale); + matching->release(); + assert(pe); + // Returns kIOReturnNotPrivileged is the root device is not secure. + // Returns kIOReturnUnsupported if "SecureRootName" is not implemented. + result = pe->callPlatformFunction(functionName, false, (void *)rootName, (void *)0, (void *)0, (void *)0); + functionName->release(); + OSSafeReleaseNULL(pe); + + if (result == kIOReturnNotPrivileged) { + mdevremoveall(); + } else if (result == kIOReturnSuccess) { + // If we are booting with a secure root, and we have the right + // boot-arg, we will want to panic on exception triage. This + // behavior is intended as a debug aid (we can look at why an + // exception occured in the kernel debugger). + if (PE_parse_boot_argn("-panic_on_exception_triage", &tmpInt, sizeof(tmpInt))) { + panic_on_exception_triage = 1; + } + } + +#endif // CONFIG_EMBEDDED } void * @@ -848,6 +885,43 @@ IOBSDMountChange(struct mount * mp, uint32_t op) } while (false); +#if CONFIG_EMBEDDED + uint64_t flags; + char path[128]; + int pathLen; + vnode_t vn; + int result; + + switch (op) + { + case kIOMountChangeMount: + case kIOMountChangeDidResize: + + if (gIOPolledCoreFileVars) break; + flags = vfs_flags(mp); + if (MNT_RDONLY & flags) break; + if (!(MNT_LOCAL & flags)) break; + + vn = vfs_vnodecovered(mp); + if (!vn) break; + pathLen = sizeof(path); + result = vn_getpath(vn, &path[0], &pathLen); + vnode_put(vn); + if (0 != result) break; + if (!pathLen) break; + if (0 != bcmp(path, kIOCoreDumpPath, pathLen - 1)) break; + IOOpenPolledCoreFile(kIOCoreDumpPath); + break; + + case kIOMountChangeUnmount: + case kIOMountChangeWillResize: + if (gIOPolledCoreFileVars && (mp == kern_file_mount(gIOPolledCoreFileVars->fileRef))) + { + IOClosePolledCoreFile(); + } + break; + } +#endif /* CONFIG_EMBEDDED */ #endif /* IOPOLLED_COREFILE */ } diff --git a/iokit/conf/Makefile.arm b/iokit/conf/Makefile.arm new file mode 100644 index 000000000..184148ea4 --- /dev/null +++ b/iokit/conf/Makefile.arm @@ -0,0 +1,18 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +IODMACommand.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IODataQueue.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOMemoryDescriptor.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IONVRAM.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOPMrootDomain.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOSharedDataQueue.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOUserClient.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align + +# Files that must go in the __HIB segment: +HIB_FILES= + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/iokit/conf/Makefile.arm64 b/iokit/conf/Makefile.arm64 new file mode 100644 index 000000000..184148ea4 --- /dev/null +++ b/iokit/conf/Makefile.arm64 @@ -0,0 +1,18 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +IODMACommand.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IODataQueue.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOMemoryDescriptor.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IONVRAM.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOPMrootDomain.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOSharedDataQueue.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOUserClient.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align + +# Files that must go in the __HIB segment: +HIB_FILES= + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/iokit/conf/Makefile.template b/iokit/conf/Makefile.template index 4777a86b0..c29d59de6 100644 --- a/iokit/conf/Makefile.template +++ b/iokit/conf/Makefile.template @@ -18,6 +18,7 @@ include $(MakeInc_def) # CFLAGS+= -include meta_features.h -DDRIVER_PRIVATE \ -DIOKIT_KERNEL_PRIVATE -DIOMATCHDEBUG=1 -DIOALLOCDEBUG=1 +SFLAGS+= -include meta_features.h #-DIOKITDEBUG=-1 CWARNFLAGS = $(CWARNFLAGS_STD) -Wno-unused-parameter diff --git a/iokit/conf/files.arm b/iokit/conf/files.arm new file mode 100644 index 000000000..7269e46f8 --- /dev/null +++ b/iokit/conf/files.arm @@ -0,0 +1,4 @@ +iokit/Families/IONVRAM/IONVRAMController.cpp optional iokitcpp + +# Power Domains +iokit/Kernel/IOPMrootDomain.cpp optional iokitcpp diff --git a/iokit/conf/files.arm64 b/iokit/conf/files.arm64 new file mode 100644 index 000000000..7269e46f8 --- /dev/null +++ b/iokit/conf/files.arm64 @@ -0,0 +1,4 @@ +iokit/Families/IONVRAM/IONVRAMController.cpp optional iokitcpp + +# Power Domains +iokit/Kernel/IOPMrootDomain.cpp optional iokitcpp diff --git a/libkdd/kcdata.h b/libkdd/kcdata.h deleted file mode 120000 index f5573542b..000000000 --- a/libkdd/kcdata.h +++ /dev/null @@ -1 +0,0 @@ -./osfmk/kern/kcdata.h \ No newline at end of file diff --git a/libkdd/kcdata.h b/libkdd/kcdata.h new file mode 100644 index 000000000..741c7f864 --- /dev/null +++ b/libkdd/kcdata.h @@ -0,0 +1,1139 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + + +/* + * + * THE KCDATA MANIFESTO + * + * Kcdata is a self-describing data serialization format. It is meant to get + * nested data structures out of xnu with minimum fuss, but also for that data + * to be easy to parse. It is also meant to allow us to add new fields and + * evolve the data format without breaking old parsers. + * + * Kcdata is a permanent data format suitable for long-term storage including + * in files. It is very important that we continue to be able to parse old + * versions of kcdata-based formats. To this end, there are several + * invariants you MUST MAINTAIN if you alter this file. + * + * * None of the magic numbers should ever be a byteswap of themselves or + * of any of the other magic numbers. + * + * * Never remove any type. + * + * * All kcdata structs must be packed, and must exclusively use fixed-size + * types. + * + * * Never change the definition of any type, except to add new fields to + * the end. + * + * * If you do add new fields to the end of a type, do not actually change + * the definition of the old structure. Instead, define a new structure + * with the new fields. See thread_snapshot_v3 as an example. This + * provides source compatibility for old readers, and also documents where + * the potential size cutoffs are. + * + * * If you change libkdd, or kcdata.py run the unit tests under libkdd. + * + * * If you add a type or extend an existing one, add a sample test to + * libkdd/tests so future changes to libkdd will always parse your struct + * correctly. + * + * For example to add a field to this: + * + * struct foobar { + * uint32_t baz; + * uint32_t quux; + * } __attribute__ ((packed)); + * + * Make it look like this: + * + * struct foobar { + * uint32_t baz; + * uint32_t quux; + * ///////// end version 1 of foobar. sizeof(struct foobar) was 8 //////// + * uint32_t frozzle; + * } __attribute__ ((packed)); + * + * If you are parsing kcdata formats, you MUST + * + * * Check the length field of each struct, including array elements. If the + * struct is longer than you expect, you must ignore the extra data. + * + * * Ignore any data types you do not understand. + * + * Additionally, we want to be as forward compatible as we can. Meaning old + * tools should still be able to use new data whenever possible. To this end, + * you should: + * + * * Try not to add new versions of types that supplant old ones. Instead + * extend the length of existing types or add supplemental types. + * + * * Try not to remove information from existing kcdata formats, unless + * removal was explicitly asked for. For example it is fine to add a + * stackshot flag to remove unwanted information, but you should not + * remove it from the default stackshot if the new flag is absent. + * + * * (TBD) If you do break old readers by removing information or + * supplanting old structs, then increase the major version number. + * + * + * + * The following is a description of the kcdata format. + * + * + * The format for data is setup in a generic format as follows + * + * Layout of data structure: + * + * | 8 - bytes | + * | type = MAGIC | LENGTH | + * | 0 | + * | type | size | + * | flags | + * | data | + * |___________data____________| + * | type | size | + * | flags | + * |___________data____________| + * | type = END | size=0 | + * | 0 | + * + * + * The type field describes what kind of data is passed. For example type = TASK_CRASHINFO_UUID means the following data is a uuid. + * These types need to be defined in task_corpses.h for easy consumption by userspace inspection tools. + * + * Some range of types is reserved for special types like ints, longs etc. A cool new functionality made possible with this + * extensible data format is that kernel can decide to put more information as required without requiring user space tools to + * re-compile to be compatible. The case of rusage struct versions could be introduced without breaking existing tools. + * + * Feature description: Generic data with description + * ------------------- + * Further more generic data with description is very much possible now. For example + * + * - kcdata_add_uint64_with_description(cdatainfo, 0x700, "NUM MACH PORTS"); + * - and more functions that allow adding description. + * The userspace tools can then look at the description and print the data even if they are not compiled with knowledge of the field apriori. + * + * Example data: + * 0000 57 f1 ad de 00 00 00 00 00 00 00 00 00 00 00 00 W............... + * 0010 01 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 ........0....... + * 0020 50 49 44 00 00 00 00 00 00 00 00 00 00 00 00 00 PID............. + * 0030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + * 0040 9c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + * 0050 01 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 ........0....... + * 0060 50 41 52 45 4e 54 20 50 49 44 00 00 00 00 00 00 PARENT PID...... + * 0070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + * 0080 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + * 0090 ed 58 91 f1 + * + * Feature description: Container markers for compound data + * ------------------ + * If a given kernel data type is complex and requires adding multiple optional fields inside a container + * object for a consumer to understand arbitrary data, we package it using container markers. + * + * For example, the stackshot code gathers information and describes the state of a given task with respect + * to many subsystems. It includes data such as io stats, vm counters, process names/flags and syscall counts. + * + * kcdata_add_container_marker(kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN, STACKSHOT_KCCONTAINER_TASK, task_uniqueid); + * // add multiple data, or add__with_description()s here + * + * kcdata_add_container_marker(kcdata_p, KCDATA_TYPE_CONTAINER_END, STACKSHOT_KCCONTAINER_TASK, task_uniqueid); + * + * Feature description: Custom Data formats on demand + * -------------------- + * With the self describing nature of format, the kernel provider can describe a data type (uniquely identified by a number) and use + * it in the buffer for sending data. The consumer can parse the type information and have knowledge of describing incoming data. + * Following is an example of how we can describe a kernel specific struct sample_disk_io_stats in buffer. + * + * struct sample_disk_io_stats { + * uint64_t disk_reads_count; + * uint64_t disk_reads_size; + * uint64_t io_priority_count[4]; + * uint64_t io_priority_size; + * } __attribute__ ((packed)); + * + * + * struct kcdata_subtype_descriptor disk_io_stats_def[] = { + * {KCS_SUBTYPE_FLAGS_NONE, KC_ST_UINT64, 0 * sizeof(uint64_t), sizeof(uint64_t), "disk_reads_count"}, + * {KCS_SUBTYPE_FLAGS_NONE, KC_ST_UINT64, 1 * sizeof(uint64_t), sizeof(uint64_t), "disk_reads_size"}, + * {KCS_SUBTYPE_FLAGS_ARRAY, KC_ST_UINT64, 2 * sizeof(uint64_t), KCS_SUBTYPE_PACK_SIZE(4, sizeof(uint64_t)), "io_priority_count"}, + * {KCS_SUBTYPE_FLAGS_ARRAY, KC_ST_UINT64, (2 + 4) * sizeof(uint64_t), sizeof(uint64_t), "io_priority_size"}, + * }; + * + * Now you can add this custom type definition into the buffer as + * kcdata_add_type_definition(kcdata_p, KCTYPE_SAMPLE_DISK_IO_STATS, "sample_disk_io_stats", + * &disk_io_stats_def[0], sizeof(disk_io_stats_def)/sizeof(struct kcdata_subtype_descriptor)); + * + */ + + +#ifndef _KCDATA_H_ +#define _KCDATA_H_ + +#include +#include +#include + +#define KCDATA_DESC_MAXLEN 32 /* including NULL byte at end */ + +#define KCDATA_FLAGS_STRUCT_PADDING_MASK 0xf +#define KCDATA_FLAGS_STRUCT_HAS_PADDING 0x80 + +/* + * kcdata aligns elements to 16 byte boundaries. + */ +#define KCDATA_ALIGNMENT_SIZE 0x10 + +struct kcdata_item { + uint32_t type; + uint32_t size; /* len(data) */ + /* flags. + * + * For structures: + * padding = flags & 0xf + * has_padding = (flags & 0x80) >> 7 + * + * has_padding is needed to disambiguate cases such as + * thread_snapshot_v2 and thread_snapshot_v3. Their + * respective sizes are 0x68 and 0x70, and thread_snapshot_v2 + * was emmitted by old kernels *before* we started recording + * padding. Since legacy thread_snapsht_v2 and modern + * thread_snapshot_v3 will both record 0 for the padding + * flags, we need some other bit which will be nonzero in the + * flags to disambiguate. + * + * This is why we hardcode a special case for + * STACKSHOT_KCTYPE_THREAD_SNAPSHOT into the iterator + * functions below. There is only a finite number of such + * hardcodings which will ever be needed. They can occur + * when: + * + * * We have a legacy structure that predates padding flags + * + * * which we want to extend without changing the kcdata type + * + * * by only so many bytes as would fit in the space that + * was previously unused padding. + * + * For containers: + * container_id = flags + * + * For arrays: + * element_count = flags & UINT32_MAX + * element_type = (flags >> 32) & UINT32_MAX + */ + uint64_t flags; + char data[]; /* must be at the end */ +}; + +typedef struct kcdata_item * kcdata_item_t; + +enum KCDATA_SUBTYPE_TYPES { KC_ST_CHAR = 1, KC_ST_INT8, KC_ST_UINT8, KC_ST_INT16, KC_ST_UINT16, KC_ST_INT32, KC_ST_UINT32, KC_ST_INT64, KC_ST_UINT64 }; +typedef enum KCDATA_SUBTYPE_TYPES kctype_subtype_t; + +/* + * A subtype description structure that defines + * how a compound data is laid out in memory. This + * provides on the fly definition of types and consumption + * by the parser. + */ +struct kcdata_subtype_descriptor { + uint8_t kcs_flags; +#define KCS_SUBTYPE_FLAGS_NONE 0x0 +#define KCS_SUBTYPE_FLAGS_ARRAY 0x1 +/* Force struct type even if only one element. + * + * Normally a kcdata_type_definition is treated as a structure if it has + * more than one subtype descriptor. Otherwise it is treated as a simple + * type. For example libkdd will represent a simple integer 42 as simply + * 42, but it will represent a structure containing an integer 42 as + * {"field_name": 42}.. + * + * If a kcdata_type_definition has only single subtype, then it will be + * treated as a structure iff KCS_SUBTYPE_FLAGS_STRUCT is set. If it has + * multiple subtypes, it will always be treated as a structure. + * + * KCS_SUBTYPE_FLAGS_MERGE has the opposite effect. If this flag is used then + * even if there are multiple elements, they will all be treated as individual + * properties of the parent dictionary. + */ +#define KCS_SUBTYPE_FLAGS_STRUCT 0x2 /* force struct type even if only one element */ +#define KCS_SUBTYPE_FLAGS_MERGE 0x4 /* treat as multiple elements of parents instead of struct */ + uint8_t kcs_elem_type; /* restricted to kctype_subtype_t */ + uint16_t kcs_elem_offset; /* offset in struct where data is found */ + uint32_t kcs_elem_size; /* size of element (or) packed state for array type */ + char kcs_name[KCDATA_DESC_MAXLEN]; /* max 31 bytes for name of field */ +}; + +typedef struct kcdata_subtype_descriptor * kcdata_subtype_descriptor_t; + +/* + * In case of array of basic c types in kctype_subtype_t, + * size is packed in lower 16 bits and + * count is packed in upper 16 bits of kcs_elem_size field. + */ +#define KCS_SUBTYPE_PACK_SIZE(e_count, e_size) (((e_count)&0xffffu) << 16 | ((e_size)&0xffffu)) + +static inline uint32_t +kcs_get_elem_size(kcdata_subtype_descriptor_t d) +{ + if (d->kcs_flags & KCS_SUBTYPE_FLAGS_ARRAY) { + /* size is composed as ((count &0xffff)<<16 | (elem_size & 0xffff)) */ + return (uint32_t)((d->kcs_elem_size & 0xffff) * ((d->kcs_elem_size & 0xffff0000)>>16)); + } + return d->kcs_elem_size; +} + +static inline uint32_t +kcs_get_elem_count(kcdata_subtype_descriptor_t d) +{ + if (d->kcs_flags & KCS_SUBTYPE_FLAGS_ARRAY) + return (d->kcs_elem_size >> 16) & 0xffff; + return 1; +} + +static inline int +kcs_set_elem_size(kcdata_subtype_descriptor_t d, uint32_t size, uint32_t count) +{ + if (count > 1) { + /* means we are setting up an array */ + if (size > 0xffff || count > 0xffff) + return -1; //invalid argument + d->kcs_elem_size = ((count & 0xffff) << 16 | (size & 0xffff)); + } + else + { + d->kcs_elem_size = size; + } + return 0; +} + +struct kcdata_type_definition { + uint32_t kct_type_identifier; + uint32_t kct_num_elements; + char kct_name[KCDATA_DESC_MAXLEN]; + struct kcdata_subtype_descriptor kct_elements[]; +}; + + +/* chunk type definitions. 0 - 0x7ff are reserved and defined here + * NOTE: Please update kcdata/libkdd/kcdtypes.c if you make any changes + * in STACKSHOT_KCTYPE_* types. + */ + +/* + * Types with description value. + * these will have KCDATA_DESC_MAXLEN-1 length string description + * and rest of kcdata_iter_size() - KCDATA_DESC_MAXLEN bytes as data + */ +#define KCDATA_TYPE_INVALID 0x0u +#define KCDATA_TYPE_STRING_DESC 0x1u +#define KCDATA_TYPE_UINT32_DESC 0x2u +#define KCDATA_TYPE_UINT64_DESC 0x3u +#define KCDATA_TYPE_INT32_DESC 0x4u +#define KCDATA_TYPE_INT64_DESC 0x5u +#define KCDATA_TYPE_BINDATA_DESC 0x6u + +/* + * Compound type definitions + */ +#define KCDATA_TYPE_ARRAY 0x11u /* Array of data OBSOLETE DONT USE THIS*/ +#define KCDATA_TYPE_TYPEDEFINTION 0x12u /* Meta type that describes a type on the fly. */ +#define KCDATA_TYPE_CONTAINER_BEGIN \ + 0x13u /* Container type which has corresponding CONTAINER_END header. \ + * KCDATA_TYPE_CONTAINER_BEGIN has type in the data segment. \ + * Both headers have (uint64_t) ID for matching up nested data. \ + */ +#define KCDATA_TYPE_CONTAINER_END 0x14u + +#define KCDATA_TYPE_ARRAY_PAD0 0x20u /* Array of data with 0 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD1 0x21u /* Array of data with 1 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD2 0x22u /* Array of data with 2 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD3 0x23u /* Array of data with 3 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD4 0x24u /* Array of data with 4 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD5 0x25u /* Array of data with 5 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD6 0x26u /* Array of data with 6 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD7 0x27u /* Array of data with 7 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD8 0x28u /* Array of data with 8 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD9 0x29u /* Array of data with 9 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADa 0x2au /* Array of data with a byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADb 0x2bu /* Array of data with b byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADc 0x2cu /* Array of data with c byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADd 0x2du /* Array of data with d byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADe 0x2eu /* Array of data with e byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADf 0x2fu /* Array of data with f byte of padding*/ + +/* + * Generic data types that are most commonly used + */ +#define KCDATA_TYPE_LIBRARY_LOADINFO 0x30u /* struct dyld_uuid_info_32 */ +#define KCDATA_TYPE_LIBRARY_LOADINFO64 0x31u /* struct dyld_uuid_info_64 */ +#define KCDATA_TYPE_TIMEBASE 0x32u /* struct mach_timebase_info */ +#define KCDATA_TYPE_MACH_ABSOLUTE_TIME 0x33u /* uint64_t */ +#define KCDATA_TYPE_TIMEVAL 0x34u /* struct timeval64 */ +#define KCDATA_TYPE_USECS_SINCE_EPOCH 0x35u /* time in usecs uint64_t */ +#define KCDATA_TYPE_PID 0x36u /* int32_t */ +#define KCDATA_TYPE_PROCNAME 0x37u /* char * */ +#define KCDATA_TYPE_NESTED_KCDATA 0x38u /* nested kcdata buffer */ + +#define KCDATA_TYPE_BUFFER_END 0xF19158EDu + +/* MAGIC numbers defined for each class of chunked data + * + * To future-proof against big-endian arches, make sure none of these magic + * numbers are byteswaps of each other + */ + +#define KCDATA_BUFFER_BEGIN_CRASHINFO 0xDEADF157u /* owner: corpses/task_corpse.h */ + /* type-range: 0x800 - 0x8ff */ +#define KCDATA_BUFFER_BEGIN_STACKSHOT 0x59a25807u /* owner: sys/stackshot.h */ + /* type-range: 0x900 - 0x93f */ +#define KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT 0xDE17A59Au /* owner: sys/stackshot.h */ + /* type-range: 0x940 - 0x9ff */ +#define KCDATA_BUFFER_BEGIN_OS_REASON 0x53A20900u /* owner: sys/reason.h */ + /* type-range: 0x1000-0x103f */ +#define KCDATA_BUFFER_BEGIN_XNUPOST_CONFIG 0x1e21c09fu /* owner: osfmk/tests/kernel_tests.c */ + /* type-range: 0x1040-0x105f */ + +/* next type range number available 0x1060 */ +/**************** definitions for XNUPOST *********************/ +#define XNUPOST_KCTYPE_TESTCONFIG 0x1040 + +/**************** definitions for stackshot *********************/ + +/* This value must always match IO_NUM_PRIORITIES defined in thread_info.h */ +#define STACKSHOT_IO_NUM_PRIORITIES 4 +/* This value must always match MAXTHREADNAMESIZE used in bsd */ +#define STACKSHOT_MAX_THREAD_NAME_SIZE 64 + +/* + * NOTE: Please update kcdata/libkdd/kcdtypes.c if you make any changes + * in STACKSHOT_KCTYPE_* types. + */ +#define STACKSHOT_KCTYPE_IOSTATS 0x901u /* io_stats_snapshot */ +#define STACKSHOT_KCTYPE_GLOBAL_MEM_STATS 0x902u /* struct mem_and_io_snapshot */ +#define STACKSHOT_KCCONTAINER_TASK 0x903u +#define STACKSHOT_KCCONTAINER_THREAD 0x904u +#define STACKSHOT_KCTYPE_TASK_SNAPSHOT 0x905u /* task_snapshot_v2 */ +#define STACKSHOT_KCTYPE_THREAD_SNAPSHOT 0x906u /* thread_snapshot_v2, thread_snapshot_v3 */ +#define STACKSHOT_KCTYPE_DONATING_PIDS 0x907u /* int[] */ +#define STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO 0x908u /* same as KCDATA_TYPE_LIBRARY_LOADINFO64 */ +#define STACKSHOT_KCTYPE_THREAD_NAME 0x909u /* char[] */ +#define STACKSHOT_KCTYPE_KERN_STACKFRAME 0x90Au /* struct stack_snapshot_frame32 */ +#define STACKSHOT_KCTYPE_KERN_STACKFRAME64 0x90Bu /* struct stack_snapshot_frame64 */ +#define STACKSHOT_KCTYPE_USER_STACKFRAME 0x90Cu /* struct stack_snapshot_frame32 */ +#define STACKSHOT_KCTYPE_USER_STACKFRAME64 0x90Du /* struct stack_snapshot_frame64 */ +#define STACKSHOT_KCTYPE_BOOTARGS 0x90Eu /* boot args string */ +#define STACKSHOT_KCTYPE_OSVERSION 0x90Fu /* os version string */ +#define STACKSHOT_KCTYPE_KERN_PAGE_SIZE 0x910u /* kernel page size in uint32_t */ +#define STACKSHOT_KCTYPE_JETSAM_LEVEL 0x911u /* jetsam level in uint32_t */ +#define STACKSHOT_KCTYPE_DELTA_SINCE_TIMESTAMP 0x912u /* timestamp used for the delta stackshot */ + +#define STACKSHOT_KCTYPE_TASK_DELTA_SNAPSHOT 0x940u /* task_delta_snapshot_v2 */ +#define STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT 0x941u /* thread_delta_snapshot_v2 */ + +#define STACKSHOT_KCTYPE_KERN_STACKLR 0x913u /* uint32_t */ +#define STACKSHOT_KCTYPE_KERN_STACKLR64 0x914u /* uint64_t */ +#define STACKSHOT_KCTYPE_USER_STACKLR 0x915u /* uint32_t */ +#define STACKSHOT_KCTYPE_USER_STACKLR64 0x916u /* uint64_t */ +#define STACKSHOT_KCTYPE_NONRUNNABLE_TIDS 0x917u /* uint64_t */ +#define STACKSHOT_KCTYPE_NONRUNNABLE_TASKS 0x918u /* uint64_t */ +#define STACKSHOT_KCTYPE_CPU_TIMES 0x919u /* struct stackshot_cpu_times */ +#define STACKSHOT_KCTYPE_STACKSHOT_DURATION 0x91au /* struct stackshot_duration */ +#define STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS 0x91bu /* struct stackshot_fault_stats */ +#define STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO 0x91cu /* kernelcache UUID -- same as KCDATA_TYPE_LIBRARY_LOADINFO64 */ +#define STACKSHOT_KCTYPE_THREAD_WAITINFO 0x91du /* struct stackshot_thread_waitinfo */ +#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot */ +#define STACKSHOT_KCTYPE_THREAD_GROUP 0x91fu /* uint64_t */ +#define STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT 0x920u /* struct jetsam_coalition_snapshot */ +#define STACKSHOT_KCTYPE_JETSAM_COALITION 0x921u /* uint64_t */ +#define STACKSHOT_KCTYPE_INSTRS_CYCLES 0x923u /* struct instrs_cycles_snapshot */ + +#define STACKSHOT_KCTYPE_THREAD_POLICY_VERSION 0x922u /* THREAD_POLICY_INTERNAL_STRUCT_VERSION in uint32 */ + +struct stack_snapshot_frame32 { + uint32_t lr; + uint32_t sp; +}; + +struct stack_snapshot_frame64 { + uint64_t lr; + uint64_t sp; +}; + +struct dyld_uuid_info_32 { + uint32_t imageLoadAddress; /* base address image is mapped at */ + uuid_t imageUUID; +}; + +struct dyld_uuid_info_64 { + uint64_t imageLoadAddress; /* XXX image slide */ + uuid_t imageUUID; +}; + +struct dyld_uuid_info_64_v2 { + uint64_t imageLoadAddress; /* XXX image slide */ + uuid_t imageUUID; + /* end of version 1 of dyld_uuid_info_64. sizeof v1 was 24 */ + uint64_t imageSlidBaseAddress; /* slid base address of image */ +}; + +struct user32_dyld_uuid_info { + uint32_t imageLoadAddress; /* base address image is mapped into */ + uuid_t imageUUID; /* UUID of image */ +}; + +struct user64_dyld_uuid_info { + uint64_t imageLoadAddress; /* base address image is mapped into */ + uuid_t imageUUID; /* UUID of image */ +}; + +enum task_snapshot_flags { + kTaskRsrcFlagged = 0x4, // In the EXC_RESOURCE danger zone? + kTerminatedSnapshot = 0x8, + kPidSuspended = 0x10, // true for suspended task + kFrozen = 0x20, // true for hibernated task (along with pidsuspended) + kTaskDarwinBG = 0x40, + kTaskExtDarwinBG = 0x80, + kTaskVisVisible = 0x100, + kTaskVisNonvisible = 0x200, + kTaskIsForeground = 0x400, + kTaskIsBoosted = 0x800, + kTaskIsSuppressed = 0x1000, + kTaskIsTimerThrottled = 0x2000, /* deprecated */ + kTaskIsImpDonor = 0x4000, + kTaskIsLiveImpDonor = 0x8000, + kTaskIsDirty = 0x10000, + kTaskWqExceededConstrainedThreadLimit = 0x20000, + kTaskWqExceededTotalThreadLimit = 0x40000, + kTaskWqFlagsAvailable = 0x80000, + kTaskUUIDInfoFaultedIn = 0x100000, /* successfully faulted in some UUID info */ + kTaskUUIDInfoMissing = 0x200000, /* some UUID info was paged out */ + kTaskUUIDInfoTriedFault = 0x400000, /* tried to fault in UUID info */ + kTaskSharedRegionInfoUnavailable = 0x800000, /* shared region info unavailable */ +}; + +enum thread_snapshot_flags { + kHasDispatchSerial = 0x4, + kStacksPCOnly = 0x8, /* Stack traces have no frame pointers. */ + kThreadDarwinBG = 0x10, /* Thread is darwinbg */ + kThreadIOPassive = 0x20, /* Thread uses passive IO */ + kThreadSuspended = 0x40, /* Thread is suspended */ + kThreadTruncatedBT = 0x80, /* Unmapped pages caused truncated backtrace */ + kGlobalForcedIdle = 0x100, /* Thread performs global forced idle */ + kThreadFaultedBT = 0x200, /* Some thread stack pages were faulted in as part of BT */ + kThreadTriedFaultBT = 0x400, /* We tried to fault in thread stack pages as part of BT */ + kThreadOnCore = 0x800, /* Thread was on-core when we entered debugger context */ + kThreadIdleWorker = 0x1000, /* Thread is an idle libpthread worker thread */ + kThreadMain = 0x2000, /* Thread is the main thread */ +}; + +struct mem_and_io_snapshot { + uint32_t snapshot_magic; + uint32_t free_pages; + uint32_t active_pages; + uint32_t inactive_pages; + uint32_t purgeable_pages; + uint32_t wired_pages; + uint32_t speculative_pages; + uint32_t throttled_pages; + uint32_t filebacked_pages; + uint32_t compressions; + uint32_t decompressions; + uint32_t compressor_size; + int32_t busy_buffer_count; + uint32_t pages_wanted; + uint32_t pages_reclaimed; + uint8_t pages_wanted_reclaimed_valid; // did mach_vm_pressure_monitor succeed? +} __attribute__((packed)); + +/* SS_TH_* macros are for ths_state */ +#define SS_TH_WAIT 0x01 /* queued for waiting */ +#define SS_TH_SUSP 0x02 /* stopped or requested to stop */ +#define SS_TH_RUN 0x04 /* running or on runq */ +#define SS_TH_UNINT 0x08 /* waiting uninteruptibly */ +#define SS_TH_TERMINATE 0x10 /* halted at termination */ +#define SS_TH_TERMINATE2 0x20 /* added to termination queue */ +#define SS_TH_IDLE 0x80 /* idling processor */ + +struct thread_snapshot_v2 { + uint64_t ths_thread_id; + uint64_t ths_wait_event; + uint64_t ths_continuation; + uint64_t ths_total_syscalls; + uint64_t ths_voucher_identifier; + uint64_t ths_dqserialnum; + uint64_t ths_user_time; + uint64_t ths_sys_time; + uint64_t ths_ss_flags; + uint64_t ths_last_run_time; + uint64_t ths_last_made_runnable_time; + uint32_t ths_state; + uint32_t ths_sched_flags; + int16_t ths_base_priority; + int16_t ths_sched_priority; + uint8_t ths_eqos; + uint8_t ths_rqos; + uint8_t ths_rqos_override; + uint8_t ths_io_tier; +} __attribute__((packed)); + +struct thread_snapshot_v3 { + uint64_t ths_thread_id; + uint64_t ths_wait_event; + uint64_t ths_continuation; + uint64_t ths_total_syscalls; + uint64_t ths_voucher_identifier; + uint64_t ths_dqserialnum; + uint64_t ths_user_time; + uint64_t ths_sys_time; + uint64_t ths_ss_flags; + uint64_t ths_last_run_time; + uint64_t ths_last_made_runnable_time; + uint32_t ths_state; + uint32_t ths_sched_flags; + int16_t ths_base_priority; + int16_t ths_sched_priority; + uint8_t ths_eqos; + uint8_t ths_rqos; + uint8_t ths_rqos_override; + uint8_t ths_io_tier; + uint64_t ths_thread_t; +} __attribute__((packed)); + + +struct thread_snapshot_v4 { + uint64_t ths_thread_id; + uint64_t ths_wait_event; + uint64_t ths_continuation; + uint64_t ths_total_syscalls; + uint64_t ths_voucher_identifier; + uint64_t ths_dqserialnum; + uint64_t ths_user_time; + uint64_t ths_sys_time; + uint64_t ths_ss_flags; + uint64_t ths_last_run_time; + uint64_t ths_last_made_runnable_time; + uint32_t ths_state; + uint32_t ths_sched_flags; + int16_t ths_base_priority; + int16_t ths_sched_priority; + uint8_t ths_eqos; + uint8_t ths_rqos; + uint8_t ths_rqos_override; + uint8_t ths_io_tier; + uint64_t ths_thread_t; + uint64_t ths_requested_policy; + uint64_t ths_effective_policy; +} __attribute__((packed)); + + +struct thread_group_snapshot { + uint64_t tgs_id; + char tgs_name[16]; +} __attribute__((packed)); + +enum coalition_flags { + kCoalitionTermRequested = 0x1, + kCoalitionTerminated = 0x2, + kCoalitionReaped = 0x4, + kCoalitionPrivileged = 0x8, +}; + +struct jetsam_coalition_snapshot { + uint64_t jcs_id; + uint64_t jcs_flags; + uint64_t jcs_thread_group; + uint64_t jcs_leader_task_uniqueid; +} __attribute__((packed)); + +struct instrs_cycles_snapshot { + uint64_t ics_instructions; + uint64_t ics_cycles; +} __attribute__((packed)); + +struct thread_delta_snapshot_v2 { + uint64_t tds_thread_id; + uint64_t tds_voucher_identifier; + uint64_t tds_ss_flags; + uint64_t tds_last_made_runnable_time; + uint32_t tds_state; + uint32_t tds_sched_flags; + int16_t tds_base_priority; + int16_t tds_sched_priority; + uint8_t tds_eqos; + uint8_t tds_rqos; + uint8_t tds_rqos_override; + uint8_t tds_io_tier; +} __attribute__ ((packed)); + +struct io_stats_snapshot +{ + /* + * I/O Statistics + * XXX: These fields must be together. + */ + uint64_t ss_disk_reads_count; + uint64_t ss_disk_reads_size; + uint64_t ss_disk_writes_count; + uint64_t ss_disk_writes_size; + uint64_t ss_io_priority_count[STACKSHOT_IO_NUM_PRIORITIES]; + uint64_t ss_io_priority_size[STACKSHOT_IO_NUM_PRIORITIES]; + uint64_t ss_paging_count; + uint64_t ss_paging_size; + uint64_t ss_non_paging_count; + uint64_t ss_non_paging_size; + uint64_t ss_data_count; + uint64_t ss_data_size; + uint64_t ss_metadata_count; + uint64_t ss_metadata_size; + /* XXX: I/O Statistics end */ + +} __attribute__ ((packed)); + +struct task_snapshot_v2 { + uint64_t ts_unique_pid; + uint64_t ts_ss_flags; + uint64_t ts_user_time_in_terminated_threads; + uint64_t ts_system_time_in_terminated_threads; + uint64_t ts_p_start_sec; + uint64_t ts_task_size; + uint64_t ts_max_resident_size; + uint32_t ts_suspend_count; + uint32_t ts_faults; + uint32_t ts_pageins; + uint32_t ts_cow_faults; + uint32_t ts_was_throttled; + uint32_t ts_did_throttle; + uint32_t ts_latency_qos; + int32_t ts_pid; + char ts_p_comm[32]; +} __attribute__ ((packed)); + +struct task_delta_snapshot_v2 { + uint64_t tds_unique_pid; + uint64_t tds_ss_flags; + uint64_t tds_user_time_in_terminated_threads; + uint64_t tds_system_time_in_terminated_threads; + uint64_t tds_task_size; + uint64_t tds_max_resident_size; + uint32_t tds_suspend_count; + uint32_t tds_faults; + uint32_t tds_pageins; + uint32_t tds_cow_faults; + uint32_t tds_was_throttled; + uint32_t tds_did_throttle; + uint32_t tds_latency_qos; +} __attribute__ ((packed)); + +struct stackshot_cpu_times { + uint64_t user_usec; + uint64_t system_usec; +} __attribute__((packed)); + +struct stackshot_duration { + uint64_t stackshot_duration; + uint64_t stackshot_duration_outer; +} __attribute__((packed)); + +struct stackshot_fault_stats { + uint32_t sfs_pages_faulted_in; /* number of pages faulted in using KDP fault path */ + uint64_t sfs_time_spent_faulting; /* MATUs spent faulting */ + uint64_t sfs_system_max_fault_time; /* MATUs fault time limit per stackshot */ + uint8_t sfs_stopped_faulting; /* we stopped decompressing because we hit the limit */ +} __attribute__((packed)); + +typedef struct stackshot_thread_waitinfo { + uint64_t owner; /* The thread that owns the object */ + uint64_t waiter; /* The thread that's waiting on the object */ + uint64_t context; /* A context uniquely identifying the object */ + uint8_t wait_type; /* The type of object that the thread is waiting on */ +} __attribute__((packed)) thread_waitinfo_t; + +#define STACKSHOT_WAITOWNER_KERNEL (UINT64_MAX - 1) +#define STACKSHOT_WAITOWNER_PORT_LOCKED (UINT64_MAX - 2) +#define STACKSHOT_WAITOWNER_PSET_LOCKED (UINT64_MAX - 3) +#define STACKSHOT_WAITOWNER_INTRANSIT (UINT64_MAX - 4) +#define STACKSHOT_WAITOWNER_MTXSPIN (UINT64_MAX - 5) +#define STACKSHOT_WAITOWNER_THREQUESTED (UINT64_MAX - 6) /* workloop waiting for a new worker thread */ +#define STACKSHOT_WAITOWNER_SUSPENDED (UINT64_MAX - 7) /* workloop is suspended */ + + +/**************** definitions for crashinfo *********************/ + +/* + * NOTE: Please update kcdata/libkdd/kcdtypes.c if you make any changes + * in TASK_CRASHINFO_* types. + */ + +/* FIXME some of these types aren't clean (fixed width, packed, and defined *here*) */ + +#define TASK_CRASHINFO_BEGIN KCDATA_BUFFER_BEGIN_CRASHINFO +#define TASK_CRASHINFO_STRING_DESC KCDATA_TYPE_STRING_DESC +#define TASK_CRASHINFO_UINT32_DESC KCDATA_TYPE_UINT32_DESC +#define TASK_CRASHINFO_UINT64_DESC KCDATA_TYPE_UINT64_DESC + +#define TASK_CRASHINFO_EXTMODINFO 0x801 +#define TASK_CRASHINFO_BSDINFOWITHUNIQID 0x802 /* struct proc_uniqidentifierinfo */ +#define TASK_CRASHINFO_TASKDYLD_INFO 0x803 +#define TASK_CRASHINFO_UUID 0x804 +#define TASK_CRASHINFO_PID 0x805 +#define TASK_CRASHINFO_PPID 0x806 +#define TASK_CRASHINFO_RUSAGE 0x807 /* struct rusage DEPRECATED do not use. + This struct has longs in it */ +#define TASK_CRASHINFO_RUSAGE_INFO 0x808 /* struct rusage_info_v3 from resource.h */ +#define TASK_CRASHINFO_PROC_NAME 0x809 /* char * */ +#define TASK_CRASHINFO_PROC_STARTTIME 0x80B /* struct timeval64 */ +#define TASK_CRASHINFO_USERSTACK 0x80C /* uint64_t */ +#define TASK_CRASHINFO_ARGSLEN 0x80D +#define TASK_CRASHINFO_EXCEPTION_CODES 0x80E /* mach_exception_data_t */ +#define TASK_CRASHINFO_PROC_PATH 0x80F /* string of len MAXPATHLEN */ +#define TASK_CRASHINFO_PROC_CSFLAGS 0x810 /* uint32_t */ +#define TASK_CRASHINFO_PROC_STATUS 0x811 /* char */ +#define TASK_CRASHINFO_UID 0x812 /* uid_t */ +#define TASK_CRASHINFO_GID 0x813 /* gid_t */ +#define TASK_CRASHINFO_PROC_ARGC 0x814 /* int */ +#define TASK_CRASHINFO_PROC_FLAGS 0x815 /* unsigned int */ +#define TASK_CRASHINFO_CPUTYPE 0x816 /* cpu_type_t */ +#define TASK_CRASHINFO_WORKQUEUEINFO 0x817 /* struct proc_workqueueinfo */ +#define TASK_CRASHINFO_RESPONSIBLE_PID 0x818 /* pid_t */ +#define TASK_CRASHINFO_DIRTY_FLAGS 0x819 /* int */ +#define TASK_CRASHINFO_CRASHED_THREADID 0x81A /* uint64_t */ +#define TASK_CRASHINFO_COALITION_ID 0x81B /* uint64_t */ +#define TASK_CRASHINFO_UDATA_PTRS 0x81C /* uint64_t */ +#define TASK_CRASHINFO_MEMORY_LIMIT 0x81D /* uint64_t */ + +#define TASK_CRASHINFO_END KCDATA_TYPE_BUFFER_END + +/**************** definitions for os reasons *********************/ + +#define EXIT_REASON_SNAPSHOT 0x1001 +#define EXIT_REASON_USER_DESC 0x1002 /* string description of reason */ +#define EXIT_REASON_USER_PAYLOAD 0x1003 /* user payload data */ +#define EXIT_REASON_CODESIGNING_INFO 0x1004 +#define EXIT_REASON_WORKLOOP_ID 0x1005 +#define EXIT_REASON_DISPATCH_QUEUE_NO 0x1006 + +struct exit_reason_snapshot { + uint32_t ers_namespace; + uint64_t ers_code; + /* end of version 1 of exit_reason_snapshot. sizeof v1 was 12 */ + uint64_t ers_flags; +} __attribute__((packed)); + +#define EXIT_REASON_CODESIG_PATH_MAX 1024 + +struct codesigning_exit_reason_info { + uint64_t ceri_virt_addr; + uint64_t ceri_file_offset; + char ceri_pathname[EXIT_REASON_CODESIG_PATH_MAX]; + char ceri_filename[EXIT_REASON_CODESIG_PATH_MAX]; + uint64_t ceri_codesig_modtime_secs; + uint64_t ceri_codesig_modtime_nsecs; + uint64_t ceri_page_modtime_secs; + uint64_t ceri_page_modtime_nsecs; + uint8_t ceri_path_truncated; + uint8_t ceri_object_codesigned; + uint8_t ceri_page_codesig_validated; + uint8_t ceri_page_codesig_tainted; + uint8_t ceri_page_codesig_nx; + uint8_t ceri_page_wpmapped; + uint8_t ceri_page_slid; + uint8_t ceri_page_dirty; + uint32_t ceri_page_shadow_depth; +} __attribute__((packed)); + +#define EXIT_REASON_USER_DESC_MAX_LEN 1024 +#define EXIT_REASON_PAYLOAD_MAX_LEN 2048 +/**************** safe iterators *********************/ + +typedef struct kcdata_iter { + kcdata_item_t item; + void *end; +} kcdata_iter_t; + + +static inline +kcdata_iter_t kcdata_iter(void *buffer, unsigned long size) { + kcdata_iter_t iter; + iter.item = (kcdata_item_t) buffer; + iter.end = (void*) (((uintptr_t)buffer) + size); + return iter; +} + +static inline +kcdata_iter_t kcdata_iter_unsafe(void *buffer) __attribute__((deprecated)); + +static inline +kcdata_iter_t kcdata_iter_unsafe(void *buffer) { + kcdata_iter_t iter; + iter.item = (kcdata_item_t) buffer; + iter.end = (void*) (uintptr_t) ~0; + return iter; +} + +static const kcdata_iter_t kcdata_invalid_iter = { .item = 0, .end = 0 }; + +static inline +int kcdata_iter_valid(kcdata_iter_t iter) { + return + ( (uintptr_t)iter.item + sizeof(struct kcdata_item) <= (uintptr_t)iter.end ) && + ( (uintptr_t)iter.item + sizeof(struct kcdata_item) + iter.item->size <= (uintptr_t)iter.end); +} + + +static inline +kcdata_iter_t kcdata_iter_next(kcdata_iter_t iter) { + iter.item = (kcdata_item_t) (((uintptr_t)iter.item) + sizeof(struct kcdata_item) + (iter.item->size)); + return iter; +} + +static inline uint32_t +kcdata_iter_type(kcdata_iter_t iter) +{ + if ((iter.item->type & ~0xfu) == KCDATA_TYPE_ARRAY_PAD0) + return KCDATA_TYPE_ARRAY; + else + return iter.item->type; +} + +static inline uint32_t +kcdata_calc_padding(uint32_t size) +{ + /* calculate number of bits to add to size to get something divisible by 16 */ + return (-size) & 0xf; +} + +static inline uint32_t +kcdata_flags_get_padding(uint64_t flags) +{ + return flags & KCDATA_FLAGS_STRUCT_PADDING_MASK; +} + +/* see comment above about has_padding */ +static inline int +kcdata_iter_is_legacy_item(kcdata_iter_t iter, uint32_t legacy_size) +{ + uint32_t legacy_size_padded = legacy_size + kcdata_calc_padding(legacy_size); + return (iter.item->size == legacy_size_padded && + (iter.item->flags & (KCDATA_FLAGS_STRUCT_PADDING_MASK | KCDATA_FLAGS_STRUCT_HAS_PADDING)) == 0); + +} + +static inline uint32_t +kcdata_iter_size(kcdata_iter_t iter) +{ + uint32_t legacy_size = 0; + + switch (kcdata_iter_type(iter)) { + case KCDATA_TYPE_ARRAY: + case KCDATA_TYPE_CONTAINER_BEGIN: + return iter.item->size; + case STACKSHOT_KCTYPE_THREAD_SNAPSHOT: { + legacy_size = sizeof(struct thread_snapshot_v2); + if (kcdata_iter_is_legacy_item(iter, legacy_size)) { + return legacy_size; + } + + goto not_legacy; + } + case STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO: { + legacy_size = sizeof(struct dyld_uuid_info_64); + if (kcdata_iter_is_legacy_item(iter, legacy_size)) { + return legacy_size; + } + + goto not_legacy; + } +not_legacy: + default: + if (iter.item->size < kcdata_flags_get_padding(iter.item->flags)) + return 0; + else + return iter.item->size - kcdata_flags_get_padding(iter.item->flags); + } +} + +static inline uint64_t +kcdata_iter_flags(kcdata_iter_t iter) +{ + return iter.item->flags; +} + +static inline +void * kcdata_iter_payload(kcdata_iter_t iter) { + return &iter.item->data; +} + + +static inline +uint32_t kcdata_iter_array_elem_type(kcdata_iter_t iter) { + return (iter.item->flags >> 32) & UINT32_MAX; +} + +static inline +uint32_t kcdata_iter_array_elem_count(kcdata_iter_t iter) { + return (iter.item->flags) & UINT32_MAX; +} + +/* KCDATA_TYPE_ARRAY is ambiguous about the size of the array elements. Size is + * calculated as total_size / elements_count, but total size got padded out to a + * 16 byte alignment. New kernels will generate KCDATA_TYPE_ARRAY_PAD* instead + * to explicitly tell us how much padding was used. Here we have a fixed, never + * to be altered list of the sizes of array elements that were used before I + * discovered this issue. If you find a KCDATA_TYPE_ARRAY that is not one of + * these types, treat it as invalid data. */ + +static inline +uint32_t +kcdata_iter_array_size_switch(kcdata_iter_t iter) { + switch(kcdata_iter_array_elem_type(iter)) { + case KCDATA_TYPE_LIBRARY_LOADINFO: + return sizeof(struct dyld_uuid_info_32); + case KCDATA_TYPE_LIBRARY_LOADINFO64: + return sizeof(struct dyld_uuid_info_64); + case STACKSHOT_KCTYPE_KERN_STACKFRAME: + case STACKSHOT_KCTYPE_USER_STACKFRAME: + return sizeof(struct stack_snapshot_frame32); + case STACKSHOT_KCTYPE_KERN_STACKFRAME64: + case STACKSHOT_KCTYPE_USER_STACKFRAME64: + return sizeof(struct stack_snapshot_frame64); + case STACKSHOT_KCTYPE_DONATING_PIDS: + return sizeof(int32_t); + case STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT: + return sizeof(struct thread_delta_snapshot_v2); + // This one is only here to make some unit tests work. It should be OK to + // remove. + case TASK_CRASHINFO_CRASHED_THREADID: + return sizeof(uint64_t); + default: + return 0; + } +} + +static inline +int kcdata_iter_array_valid(kcdata_iter_t iter) { + if (!kcdata_iter_valid(iter)) + return 0; + if (kcdata_iter_type(iter) != KCDATA_TYPE_ARRAY) + return 0; + if (kcdata_iter_array_elem_count(iter) == 0) + return iter.item->size == 0; + if (iter.item->type == KCDATA_TYPE_ARRAY) { + uint32_t elem_size = kcdata_iter_array_size_switch(iter); + if (elem_size == 0) + return 0; + /* sizes get aligned to the nearest 16. */ + return + kcdata_iter_array_elem_count(iter) <= iter.item->size / elem_size && + iter.item->size % kcdata_iter_array_elem_count(iter) < 16; + } else { + return + (iter.item->type & 0xf) <= iter.item->size && + kcdata_iter_array_elem_count(iter) <= iter.item->size - (iter.item->type & 0xf) && + (iter.item->size - (iter.item->type & 0xf)) % kcdata_iter_array_elem_count(iter) == 0; + } +} + + +static inline +uint32_t kcdata_iter_array_elem_size(kcdata_iter_t iter) { + if (iter.item->type == KCDATA_TYPE_ARRAY) + return kcdata_iter_array_size_switch(iter); + if (kcdata_iter_array_elem_count(iter) == 0) + return 0; + return (iter.item->size - (iter.item->type & 0xf)) / kcdata_iter_array_elem_count(iter); +} + +static inline +int kcdata_iter_container_valid(kcdata_iter_t iter) { + return + kcdata_iter_valid(iter) && + kcdata_iter_type(iter) == KCDATA_TYPE_CONTAINER_BEGIN && + iter.item->size >= sizeof(uint32_t); +} + +static inline +uint32_t kcdata_iter_container_type(kcdata_iter_t iter) { + return * (uint32_t *) kcdata_iter_payload(iter); +} + +static inline +uint64_t kcdata_iter_container_id(kcdata_iter_t iter) { + return iter.item->flags; +} + + +#define KCDATA_ITER_FOREACH(iter) for(; kcdata_iter_valid(iter) && iter.item->type != KCDATA_TYPE_BUFFER_END; iter = kcdata_iter_next(iter)) +#define KCDATA_ITER_FOREACH_FAILED(iter) (!kcdata_iter_valid(iter) || (iter).item->type != KCDATA_TYPE_BUFFER_END) + +static inline +kcdata_iter_t +kcdata_iter_find_type(kcdata_iter_t iter, uint32_t type) +{ + KCDATA_ITER_FOREACH(iter) + { + if (kcdata_iter_type(iter) == type) + return iter; + } + return kcdata_invalid_iter; +} + +static inline +int kcdata_iter_data_with_desc_valid(kcdata_iter_t iter, uint32_t minsize) { + return + kcdata_iter_valid(iter) && + kcdata_iter_size(iter) >= KCDATA_DESC_MAXLEN + minsize && + ((char*)kcdata_iter_payload(iter))[KCDATA_DESC_MAXLEN-1] == 0; +} + +static inline +char *kcdata_iter_string(kcdata_iter_t iter, uint32_t offset) { + if (offset > kcdata_iter_size(iter)) { + return NULL; + } + uint32_t maxlen = kcdata_iter_size(iter) - offset; + char *s = ((char*)kcdata_iter_payload(iter)) + offset; + if (strnlen(s, maxlen) < maxlen) { + return s; + } else { + return NULL; + } +} + +static inline void kcdata_iter_get_data_with_desc(kcdata_iter_t iter, char **desc_ptr, void **data_ptr, uint32_t *size_ptr) { + if (desc_ptr) + *desc_ptr = (char *)kcdata_iter_payload(iter); + if (data_ptr) + *data_ptr = (void *)((uintptr_t)kcdata_iter_payload(iter) + KCDATA_DESC_MAXLEN); + if (size_ptr) + *size_ptr = kcdata_iter_size(iter) - KCDATA_DESC_MAXLEN; +} + +#endif diff --git a/libkdd/kcdtypes.c b/libkdd/kcdtypes.c index 6b1ac415e..8fe7aee23 100644 --- a/libkdd/kcdtypes.c +++ b/libkdd/kcdtypes.c @@ -325,6 +325,8 @@ kcdata_get_typedescription(unsigned type_id, uint8_t * buffer, uint32_t buffer_s _SUBTYPE(KC_ST_UINT8, struct thread_snapshot_v3, ths_rqos_override); _SUBTYPE(KC_ST_UINT8, struct thread_snapshot_v3, ths_io_tier); _SUBTYPE(KC_ST_UINT64, struct thread_snapshot_v3, ths_thread_t); + _SUBTYPE(KC_ST_UINT64, struct thread_snapshot_v4, ths_requested_policy); + _SUBTYPE(KC_ST_UINT64, struct thread_snapshot_v4, ths_effective_policy); setup_type_definition(retval, type_id, i, "thread_snapshot"); break; @@ -442,6 +444,13 @@ kcdata_get_typedescription(unsigned type_id, uint8_t * buffer, uint32_t buffer_s break; } + case STACKSHOT_KCTYPE_THREAD_POLICY_VERSION: { + i = 0; + setup_subtype_description(&subtypes[i++], KC_ST_UINT32, 0, "thread_policy_version"); + setup_type_definition(retval, type_id, i, "thread_policy_version"); + break; + } + case STACKSHOT_KCTYPE_JETSAM_LEVEL: { i = 0; setup_subtype_description(&subtypes[i++], KC_ST_UINT32, 0, "jetsam_level"); @@ -563,6 +572,46 @@ kcdata_get_typedescription(unsigned type_id, uint8_t * buffer, uint32_t buffer_s break; } + case STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT: { + i = 0; + _SUBTYPE(KC_ST_UINT64, struct thread_group_snapshot, tgs_id); + _SUBTYPE_ARRAY(KC_ST_CHAR, struct thread_group_snapshot, tgs_name, 16); + setup_type_definition(retval, type_id, i, "thread_group_snapshot"); + break; + } + + case STACKSHOT_KCTYPE_THREAD_GROUP: { + i = 0; + setup_subtype_description(&subtypes[i++], KC_ST_UINT64, 0, "thread_group"); + setup_type_definition(retval, type_id, i, "thread_group"); + break; + }; + + case STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT: { + i = 0; + _SUBTYPE(KC_ST_UINT64, struct jetsam_coalition_snapshot, jcs_id); + _SUBTYPE(KC_ST_UINT64, struct jetsam_coalition_snapshot, jcs_flags); + _SUBTYPE(KC_ST_UINT64, struct jetsam_coalition_snapshot, jcs_thread_group); + _SUBTYPE(KC_ST_UINT64, struct jetsam_coalition_snapshot, jcs_leader_task_uniqueid); + setup_type_definition(retval, type_id, i, "jetsam_coalition_snapshot"); + break; + } + + case STACKSHOT_KCTYPE_JETSAM_COALITION: { + i = 0; + setup_subtype_description(&subtypes[i++], KC_ST_UINT64, 0, "jetsam_coalition"); + setup_type_definition(retval, type_id, i, "jetsam_coalition"); + break; + }; + + case STACKSHOT_KCTYPE_INSTRS_CYCLES: { + i = 0; + _SUBTYPE(KC_ST_UINT64, struct instrs_cycles_snapshot, ics_instructions); + _SUBTYPE(KC_ST_UINT64, struct instrs_cycles_snapshot, ics_cycles); + setup_type_definition(retval, type_id, i, "instrs_cycles_snapshot"); + break; + } + case TASK_CRASHINFO_PROC_STARTTIME: { i = 0; _SUBTYPE(KC_ST_INT64, struct timeval64, tv_sec); @@ -736,6 +785,20 @@ kcdata_get_typedescription(unsigned type_id, uint8_t * buffer, uint32_t buffer_s break; + case EXIT_REASON_WORKLOOP_ID: { + i = 0; + setup_subtype_description(&subtypes[i++], KC_ST_UINT64, 0, "exit_reason_workloop_id"); + setup_type_definition(retval, type_id, i, "exit_reason_workloop_id"); + break; + } + + case EXIT_REASON_DISPATCH_QUEUE_NO: { + i = 0; + setup_subtype_description(&subtypes[i++], KC_ST_UINT64, 0, "exit_reason_dispatch_queue_no"); + setup_type_definition(retval, type_id, i, "exit_reason_dispatch_queue_no"); + break; + } + } default: diff --git a/libkdd/kdd.xcodeproj/project.pbxproj b/libkdd/kdd.xcodeproj/project.pbxproj index 7e55392bf..5a2b63714 100644 --- a/libkdd/kdd.xcodeproj/project.pbxproj +++ b/libkdd/kdd.xcodeproj/project.pbxproj @@ -23,6 +23,8 @@ 0860F87B1BFC3857007E1301 /* stackshot-sample-tailspin-2.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 0860F8791BFC3845007E1301 /* stackshot-sample-tailspin-2.plist.gz */; }; 086395B51BF5655D005ED913 /* kdd_main.m in Sources */ = {isa = PBXBuildFile; fileRef = 086395B41BF5655D005ED913 /* kdd_main.m */; }; 086395B91BF565A2005ED913 /* libkdd.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C91C93C71ACB58B700119B60 /* libkdd.a */; }; + 088C36E01EF323C300ABB2E0 /* stackshot-sample-thread-policy in Resources */ = {isa = PBXBuildFile; fileRef = 088C36DF1EF323AE00ABB2E0 /* stackshot-sample-thread-policy */; }; + 088C36E11EF323C300ABB2E0 /* stackshot-sample-thread-policy.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 088C36DE1EF323AE00ABB2E0 /* stackshot-sample-thread-policy.plist.gz */; }; 08A4C94C1C4701B800D5F010 /* KCDEmbeddedBufferDescription.m in Sources */ = {isa = PBXBuildFile; fileRef = 08A4C94B1C4701B800D5F010 /* KCDEmbeddedBufferDescription.m */; }; 08A4C94F1C470F1C00D5F010 /* nested-sample in Resources */ = {isa = PBXBuildFile; fileRef = 08A4C94D1C470F0900D5F010 /* nested-sample */; }; 08A4C9501C470F1C00D5F010 /* nested-sample.plist in Resources */ = {isa = PBXBuildFile; fileRef = 08A4C94E1C470F0900D5F010 /* nested-sample.plist */; }; @@ -58,6 +60,10 @@ 13DBA26A1CAB1BA000227EB2 /* stackshot-sample-sharedcachev2 in Resources */ = {isa = PBXBuildFile; fileRef = 13DBA2691CAB1B9C00227EB2 /* stackshot-sample-sharedcachev2 */; }; 13F3DA9C1C7C1BEE00ACFFCC /* corpse-twr-sample-v2 in Resources */ = {isa = PBXBuildFile; fileRef = 13F3DA9B1C7C1BE700ACFFCC /* corpse-twr-sample-v2 */; }; 13F3DA9E1C7C1C6600ACFFCC /* corpse-twr-sample-v2.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 13F3DA9D1C7C1C6000ACFFCC /* corpse-twr-sample-v2.plist.gz */; }; + 1862B0341E7A083F0005ADF4 /* stackshot-sample-thread-groups in Resources */ = {isa = PBXBuildFile; fileRef = 1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */; }; + 1862B0351E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */; }; + 18E592981E9451A20018612A /* stackshot-sample-coalitions in Resources */ = {isa = PBXBuildFile; fileRef = 18E592961E9451A20018612A /* stackshot-sample-coalitions */; }; + 18E592991E9451A20018612A /* stackshot-sample-coalitions.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */; }; C91C93CB1ACB58B700119B60 /* kdd.h in Headers */ = {isa = PBXBuildFile; fileRef = C91C93CA1ACB58B700119B60 /* kdd.h */; settings = {ATTRIBUTES = (Private, ); }; }; C91C93CD1ACB58B700119B60 /* kdd.m in Sources */ = {isa = PBXBuildFile; fileRef = C91C93CC1ACB58B700119B60 /* kdd.m */; }; C91C93E41ACB598700119B60 /* KCDBasicTypeDescription.h in Headers */ = {isa = PBXBuildFile; fileRef = C91C93E01ACB598700119B60 /* KCDBasicTypeDescription.h */; }; @@ -67,6 +73,8 @@ C9C5C68C1ACDAFDB00BE0E5E /* kcdtypes.c in Sources */ = {isa = PBXBuildFile; fileRef = C9C5C68B1ACDAFDB00BE0E5E /* kcdtypes.c */; }; C9D7B53F1D1B41D700F1019D /* xnupost_testconfig-sample.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = C9D7B53D1D1B41D700F1019D /* xnupost_testconfig-sample.plist.gz */; }; C9D7B5401D1B41D700F1019D /* xnupost_testconfig-sample in Resources */ = {isa = PBXBuildFile; fileRef = C9D7B53E1D1B41D700F1019D /* xnupost_testconfig-sample */; }; + C9DCEF011F01C3810000BD02 /* stackshot-sample-instrs-cycles in Resources */ = {isa = PBXBuildFile; fileRef = C9DCEF001F01C3790000BD02 /* stackshot-sample-instrs-cycles */; }; + C9DCEF021F01C3810000BD02 /* stackshot-sample-instrs-cycles.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = C9DCEEFF1F01C3790000BD02 /* stackshot-sample-instrs-cycles.plist.gz */; }; C9DE39141ACB5A540020F4A3 /* kcdata_core.m in Sources */ = {isa = PBXBuildFile; fileRef = C9DE39131ACB5A540020F4A3 /* kcdata_core.m */; }; /* End PBXBuildFile section */ @@ -117,6 +125,8 @@ 0860F8791BFC3845007E1301 /* stackshot-sample-tailspin-2.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; name = "stackshot-sample-tailspin-2.plist.gz"; path = "tests/stackshot-sample-tailspin-2.plist.gz"; sourceTree = SOURCE_ROOT; }; 086395B21BF5655D005ED913 /* kdd */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = kdd; sourceTree = BUILT_PRODUCTS_DIR; }; 086395B41BF5655D005ED913 /* kdd_main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = kdd_main.m; sourceTree = ""; }; + 088C36DE1EF323AE00ABB2E0 /* stackshot-sample-thread-policy.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-thread-policy.plist.gz"; sourceTree = ""; }; + 088C36DF1EF323AE00ABB2E0 /* stackshot-sample-thread-policy */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-thread-policy"; sourceTree = ""; }; 08A4C94A1C47019E00D5F010 /* KCDEmbeddedBufferDescription.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = KCDEmbeddedBufferDescription.h; sourceTree = ""; }; 08A4C94B1C4701B800D5F010 /* KCDEmbeddedBufferDescription.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = KCDEmbeddedBufferDescription.m; sourceTree = ""; }; 08A4C94D1C470F0900D5F010 /* nested-sample */ = {isa = PBXFileReference; lastKnownFileType = file; name = "nested-sample"; path = "tests/nested-sample"; sourceTree = SOURCE_ROOT; }; @@ -153,6 +163,10 @@ 13EADC171C4DCDA100468D97 /* test-twr-sample.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; name = "test-twr-sample.plist.gz"; path = "tests/test-twr-sample.plist.gz"; sourceTree = SOURCE_ROOT; }; 13F3DA9B1C7C1BE700ACFFCC /* corpse-twr-sample-v2 */ = {isa = PBXFileReference; lastKnownFileType = file; name = "corpse-twr-sample-v2"; path = "tests/corpse-twr-sample-v2"; sourceTree = SOURCE_ROOT; }; 13F3DA9D1C7C1C6000ACFFCC /* corpse-twr-sample-v2.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; name = "corpse-twr-sample-v2.plist.gz"; path = "tests/corpse-twr-sample-v2.plist.gz"; sourceTree = SOURCE_ROOT; }; + 1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-thread-groups"; sourceTree = ""; }; + 1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-thread-groups.plist.gz"; sourceTree = ""; }; + 18E592961E9451A20018612A /* stackshot-sample-coalitions */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-coalitions"; sourceTree = ""; }; + 18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-coalitions.plist.gz"; sourceTree = ""; }; C91C93C71ACB58B700119B60 /* libkdd.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libkdd.a; sourceTree = BUILT_PRODUCTS_DIR; }; C91C93CA1ACB58B700119B60 /* kdd.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = kdd.h; sourceTree = ""; }; C91C93CC1ACB58B700119B60 /* kdd.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = kdd.m; sourceTree = ""; }; @@ -163,6 +177,8 @@ C9C5C68B1ACDAFDB00BE0E5E /* kcdtypes.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = kcdtypes.c; sourceTree = ""; }; C9D7B53D1D1B41D700F1019D /* xnupost_testconfig-sample.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "xnupost_testconfig-sample.plist.gz"; sourceTree = ""; }; C9D7B53E1D1B41D700F1019D /* xnupost_testconfig-sample */ = {isa = PBXFileReference; lastKnownFileType = file; path = "xnupost_testconfig-sample"; sourceTree = ""; }; + C9DCEEFF1F01C3790000BD02 /* stackshot-sample-instrs-cycles.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-instrs-cycles.plist.gz"; sourceTree = ""; }; + C9DCEF001F01C3790000BD02 /* stackshot-sample-instrs-cycles */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-instrs-cycles"; sourceTree = ""; }; C9DE39131ACB5A540020F4A3 /* kcdata_core.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = kcdata_core.m; sourceTree = ""; }; /* End PBXFileReference section */ @@ -198,6 +214,14 @@ 08603F351BF69EDE007D3784 /* tests */ = { isa = PBXGroup; children = ( + C9DCEF001F01C3790000BD02 /* stackshot-sample-instrs-cycles */, + C9DCEEFF1F01C3790000BD02 /* stackshot-sample-instrs-cycles.plist.gz */, + 088C36DF1EF323AE00ABB2E0 /* stackshot-sample-thread-policy */, + 088C36DE1EF323AE00ABB2E0 /* stackshot-sample-thread-policy.plist.gz */, + 18E592961E9451A20018612A /* stackshot-sample-coalitions */, + 18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */, + 1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */, + 1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */, C9D7B53D1D1B41D700F1019D /* xnupost_testconfig-sample.plist.gz */, C9D7B53E1D1B41D700F1019D /* xnupost_testconfig-sample */, 04C64AC91D25C43400C6C781 /* stackshot-with-waitinfo */, @@ -374,12 +398,12 @@ isa = PBXProject; attributes = { LastSwiftUpdateCheck = 0730; - LastUpgradeCheck = 0730; + LastUpgradeCheck = 0830; ORGANIZATIONNAME = "Vishal Patel"; TargetAttributes = { 08603F331BF69EDE007D3784 = { CreatedOnToolsVersion = 7.3; - LastSwiftMigration = 0800; + LastSwiftMigration = 0830; }; 086395B11BF5655D005ED913 = { CreatedOnToolsVersion = 7.3; @@ -413,9 +437,14 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( + C9DCEF011F01C3810000BD02 /* stackshot-sample-instrs-cycles in Resources */, + C9DCEF021F01C3810000BD02 /* stackshot-sample-instrs-cycles.plist.gz in Resources */, + 088C36E01EF323C300ABB2E0 /* stackshot-sample-thread-policy in Resources */, + 088C36E11EF323C300ABB2E0 /* stackshot-sample-thread-policy.plist.gz in Resources */, 045F7F131D2ADE8000B4808B /* stackshot-with-waitinfo.plist.gz in Resources */, 045F7F121D2ADE7C00B4808B /* stackshot-with-waitinfo in Resources */, 08A4C94F1C470F1C00D5F010 /* nested-sample in Resources */, + 1862B0341E7A083F0005ADF4 /* stackshot-sample-thread-groups in Resources */, 08A4C9501C470F1C00D5F010 /* nested-sample.plist in Resources */, 13D6C5D21C4DDDBE005E617C /* test-twr-sample in Resources */, 13D6C5D01C4DDDB6005E617C /* corpse-twr-sample in Resources */, @@ -440,9 +469,11 @@ 13CC08441CB97F8D00EA6069 /* stackshot-fault-stats in Resources */, 13F3DA9C1C7C1BEE00ACFFCC /* corpse-twr-sample-v2 in Resources */, 13D6C5D31C4DDE0D005E617C /* test-twr-sample.plist.gz in Resources */, + 1862B0351E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz in Resources */, 1368F0851C87E06A00940FC6 /* exitreason-codesigning.plist.gz in Resources */, 08C9D83D1BFFF8E100DF6C05 /* exitreason-sample in Resources */, 08C9D83E1BFFF8E100DF6C05 /* exitreason-sample.plist.gz in Resources */, + 18E592981E9451A20018612A /* stackshot-sample-coalitions in Resources */, 08B4808B1BF9474A00B4AAE0 /* corpse-sample in Resources */, 13D6C5D11C4DDDB8005E617C /* corpse-twr-sample.plist.gz in Resources */, 08B4808C1BF9474A00B4AAE0 /* corpse-sample.plist.gz in Resources */, @@ -458,6 +489,7 @@ 08B4807B1BF8297500B4AAE0 /* stackshot-sample-old-arrays.plist.gz in Resources */, 0843EE941BF6BAC100CD4150 /* stackshot-sample.plist.gz in Resources */, 0843EE921BF6AFC600CD4150 /* stackshot-sample in Resources */, + 18E592991E9451A20018612A /* stackshot-sample-coalitions.plist.gz in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -518,13 +550,12 @@ ENABLE_TESTABILITY = YES; INFOPLIST_FILE = tests/Info.plist; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/../Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 10.11; PRODUCT_BUNDLE_IDENTIFIER = apple.com.Tests; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SWIFT_OBJC_BRIDGING_HEADER = tests/kdd_bridge.h; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; - SWIFT_VERSION = 2.3; + SWIFT_VERSION = 3.0; }; name = Debug; }; @@ -536,12 +567,11 @@ COMBINE_HIDPI_IMAGES = YES; INFOPLIST_FILE = tests/Info.plist; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/../Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 10.11; PRODUCT_BUNDLE_IDENTIFIER = apple.com.Tests; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SWIFT_OBJC_BRIDGING_HEADER = tests/kdd_bridge.h; - SWIFT_VERSION = 2.3; + SWIFT_VERSION = 3.0; }; name = Release; }; diff --git a/libkdd/tests/Tests.swift b/libkdd/tests/Tests.swift index e2225f35a..cd7f46ea0 100644 --- a/libkdd/tests/Tests.swift +++ b/libkdd/tests/Tests.swift @@ -20,61 +20,60 @@ import Foundation // Swift's bridging to uuid_t is awkward. -func nsuuid2uuid_t(nsuuid : NSUUID) -> uuid_t { +func nsuuid2uuid_t(_ nsuuid : NSUUID) -> uuid_t { let dat = nsuuid2array(nsuuid) return nsarray2uuid(dat) } -func nsarray2uuid(x : AnyObject) -> uuid_t { - let a = x as! NSArray - return uuid_t(UInt8(a[0] as! Int), - UInt8(a[1] as! Int), - UInt8(a[2] as! Int), - UInt8(a[3] as! Int), - UInt8(a[4] as! Int), - UInt8(a[5] as! Int), - UInt8(a[6] as! Int), - UInt8(a[7] as! Int), - UInt8(a[8] as! Int), - UInt8(a[9] as! Int), - UInt8(a[10] as! Int), - UInt8(a[11] as! Int), - UInt8(a[12] as! Int), - UInt8(a[13] as! Int), - UInt8(a[14] as! Int), - UInt8(a[15] as! Int)) +func nsarray2uuid(_ a : [Int]) -> uuid_t { + return uuid_t(UInt8(a[0]), + UInt8(a[1]), + UInt8(a[2]), + UInt8(a[3]), + UInt8(a[4]), + UInt8(a[5]), + UInt8(a[6]), + UInt8(a[7]), + UInt8(a[8]), + UInt8(a[9]), + UInt8(a[10]), + UInt8(a[11]), + UInt8(a[12]), + UInt8(a[13]), + UInt8(a[14]), + UInt8(a[15])) } -func nsuuid2array(uuid : NSUUID) -> [Int] { +func nsuuid2array(_ uuid: NSUUID) -> [Int] { var ret = [Int]() - let ptr = UnsafeMutablePointer.alloc(16) + let ptr = UnsafeMutablePointer.allocate(capacity: 16) - defer { ptr.dealloc(16) } + defer { ptr.deallocate(capacity:16) } - uuid.getUUIDBytes(ptr) + uuid.getBytes(ptr) for i in 0..<16 { ret.append(Int(ptr[i])) } return ret } -func decompress(data:NSData) throws -> NSData { +func decompress(_ data:NSData) throws -> NSData { var stream = z_stream(next_in: nil, avail_in: 0, total_in: 0, next_out: nil, avail_out: 0, total_out: 0, msg: nil, state: nil, zalloc: nil, zfree: nil, opaque: nil, data_type: 0, adler: 0, reserved: 0) let bufsize : Int = 1000 - let buffer = UnsafeMutablePointer.alloc(bufsize) - defer { buffer.dealloc(bufsize) } + let buffer = UnsafeMutablePointer.allocate(capacity: bufsize) + defer { buffer.deallocate(capacity:bufsize) } let output = NSMutableData() stream.next_out = buffer stream.avail_out = UInt32(bufsize) - stream.next_in = UnsafeMutablePointer(data.bytes) + stream.next_in = UnsafeMutablePointer(mutating:data.bytes.assumingMemoryBound(to:Bytef.self)) stream.avail_in = UInt32(data.length) - inflateInit2_(&stream, 16+MAX_WBITS, ZLIB_VERSION, Int32(sizeof(z_stream))) + inflateInit2_(&stream, 16+MAX_WBITS, ZLIB_VERSION, Int32(MemoryLayout.size)) while (true) { let z = inflate(&stream, Z_NO_FLUSH); if (z == Z_OK || z == Z_STREAM_END) { - output.appendBytes(buffer, length: bufsize - Int(stream.avail_out)) + output.append(buffer, length: bufsize - Int(stream.avail_out)) stream.avail_out = UInt32(bufsize) stream.next_out = buffer if (z == Z_STREAM_END) { @@ -87,6 +86,12 @@ func decompress(data:NSData) throws -> NSData { } +extension Dictionary { + func value(forKeyPath s:String) -> Any? { + return (self as NSDictionary).value(forKeyPath:s) + } +} + class Tests: XCTestCase { @@ -100,9 +105,9 @@ class Tests: XCTestCase { super.tearDown() } - func parseBuffer(buffer:NSData) throws -> NSDictionary { + func parseBuffer(_ buffer:NSData) throws -> [AnyHashable:Any] { var error : NSError? - guard let dict = parseKCDataBuffer(UnsafeMutablePointer(buffer.bytes), UInt32(buffer.length), &error) + guard let dict = parseKCDataBuffer(UnsafeMutablePointer(mutating:buffer.bytes.assumingMemoryBound(to:UInt8.self)), UInt32(buffer.length), &error) else { XCTAssert(error != nil) throw error! @@ -110,7 +115,7 @@ class Tests: XCTestCase { return dict } - func testPaddingFlags(pad : Int) { + func testPaddingFlags(_ pad : Int) { let buffer = NSMutableData(capacity:1000)! var item = kcdata_item() @@ -118,22 +123,22 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_LIBRARY_LOADINFO) item.flags = UInt64(pad) - item.size = UInt32(sizeof(dyld_uuid_info_32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! var payload = dyld_uuid_info_32(imageLoadAddress: 42, imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32)) + buffer.append(&payload, length:MemoryLayout.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } @@ -143,8 +148,8 @@ class Tests: XCTestCase { uuidarray.removeLast() } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageLoadAddress"] == 42) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageUUID"] == uuidarray) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageLoadAddress") as? Int == 42) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageUUID") as! [Int] == uuidarray) } func testPaddingFlags() { @@ -152,7 +157,6 @@ class Tests: XCTestCase { testPaddingFlags(i) } } - func testBootArgs() { let s = "hello, I am some boot args" @@ -163,23 +167,22 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(STACKSHOT_KCTYPE_BOOTARGS) item.flags = 0 item.size = UInt32(s.utf8.count + 1) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - s.nulTerminatedUTF8.withUnsafeBufferPointer({ - buffer.appendBytes($0.baseAddress, length:s.utf8.count + 1) + buffer.append(&item, length: MemoryLayout.size) + s.utf8CString.withUnsafeBufferPointer({ + buffer.append($0.baseAddress!, length:s.utf8.count + 1) }) - item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["boot_args"] == s) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.boot_args") as? String == s) } func testBootArgsMissingNul() { @@ -192,20 +195,20 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(STACKSHOT_KCTYPE_BOOTARGS) item.flags = 0 item.size = UInt32(s.utf8.count) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - s.nulTerminatedUTF8.withUnsafeBufferPointer({ - buffer.appendBytes($0.baseAddress, length:s.utf8.count) + buffer.append(&item, length: MemoryLayout.size) + s.utf8CString.withUnsafeBufferPointer({ + buffer.append($0.baseAddress!, length:s.utf8.count) }) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -218,28 +221,28 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_LIBRARY_LOADINFO) item.flags = 0 - item.size = UInt32(sizeof(dyld_uuid_info_32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! var payload = dyld_uuid_info_32(imageLoadAddress: 42, imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32)) + buffer.append(&payload, length:MemoryLayout.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageLoadAddress"] == 42) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageUUID"] == nsuuid2array(uuid)) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageLoadAddress") as? Int == 42) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageUUID") as! [Int] == nsuuid2array(uuid)) } func testLoadInfoWrongSize() { @@ -252,29 +255,29 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_LIBRARY_LOADINFO) item.flags = 0 - item.size = UInt32(sizeof(dyld_uuid_info_32)) - 1 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) - 1 + buffer.append(&item, length: MemoryLayout.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! var payload = dyld_uuid_info_32(imageLoadAddress: 42, imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32) - 1) + buffer.append(&payload, length:MemoryLayout.size - 1) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageLoadAddress"] == 42) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageLoadAddress") as? Int == 42) var uuidarray = nsuuid2array(uuid) uuidarray.removeLast() - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageUUID"] == uuidarray) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageUUID") as! [Int] == uuidarray) } func testLoadInfoWayWrongSize() { @@ -287,27 +290,26 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_LIBRARY_LOADINFO) item.flags = 0 - item.size = UInt32(sizeof(dyld_uuid_info_32)) - 16 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) - 16 + buffer.append(&item, length: MemoryLayout.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! var payload = dyld_uuid_info_32(imageLoadAddress: 42, imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32) - 16) + buffer.append(&payload, length:MemoryLayout.size - 16) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageLoadAddress"] == 42) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageUUID"] == nil) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageLoadAddress") as? Int == 42) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageUUID") == nil) } func testLoadInfoPreposterousWrongSize() { @@ -320,25 +322,25 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_LIBRARY_LOADINFO) item.flags = 0 item.size = UInt32(1) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) var payload = UInt8(42) - buffer.appendBytes(&payload, length:1) + buffer.append(&payload, length:1) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageLoadAddress"] == nil) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageUUID"] == nil) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageLoadAddress") == nil) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageUUID") == nil) } @@ -349,43 +351,43 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_ARRAY_PAD0) + UInt32(pad) item.flags = UInt64(STACKSHOT_KCTYPE_DONATING_PIDS) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt32) + pad) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout.size + pad) + buffer.append(&item, length: MemoryLayout.size) for i in 0...size) } for i in 0...size) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["donating_pids"]??.count == n) + XCTAssert((dict.value(forKeyPath:"kcdata_crashinfo.donating_pids") as! [Any]).count == n) for i in 0...size) item.type = UInt32(KCDATA_TYPE_ARRAY_PAD0) item.flags = UInt64(KCDATA_TYPE_LIBRARY_LOADINFO) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(dyld_uuid_info_32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! for i in 0...size) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??.count == n) + XCTAssert((dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as! [Any]).count == n) for i in 0...size) item.type = UInt32(KCDATA_TYPE_ARRAY_PAD0) item.flags = UInt64(KCDATA_TYPE_LIBRARY_LOADINFO) << 32 | UInt64(n) - item.size = UInt32(n * (sizeof(dyld_uuid_info_32) - wrong)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * (MemoryLayout.size - wrong)) + buffer.append(&item, length: MemoryLayout.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! for i in 0...size-wrong) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout.size) var uuidarray = nsuuid2array(uuid) uuidarray.removeLast() guard let dict = try? self.parseBuffer(buffer) - else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??.count == n) + else { XCTFail(); return; } + XCTAssert((dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as! [Any]).count == n) for i in 0...size) item.type = UInt32(KCDATA_TYPE_ARRAY_PAD0) item.flags = UInt64(KCDATA_TYPE_LIBRARY_LOADINFO) << 32 | UInt64(n) - item.size = UInt32(n * (sizeof(dyld_uuid_info_32) - wrong)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * (MemoryLayout.size - wrong)) + buffer.append(&item, length: MemoryLayout.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! for i in 0...size-wrong) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) + guard let dict = try? self.parseBuffer(buffer) - else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??.count == n) + else { XCTFail(); return; } + XCTAssert((dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as! [Any]).count == n) for i in 0...size) item.type = UInt32(KCDATA_TYPE_ARRAY_PAD0) item.flags = UInt64(KCDATA_TYPE_LIBRARY_LOADINFO) << 32 | UInt64(n) - item.size = UInt32(n * (sizeof(dyld_uuid_info_32) - wrong)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * (MemoryLayout.size - wrong)) + buffer.append(&item, length: MemoryLayout.size) for i in 0...size) + guard let dict = try? self.parseBuffer(buffer) - else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??.count == n) + else { XCTFail(); return; } + XCTAssert((dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as! [Any]).count == n) for i in 0...size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) let buffer2 = NSMutableData(capacity:1000)! item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer2.appendBytes(&item, length: sizeof(kcdata_item)) + buffer2.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_NESTED_KCDATA) item.flags = 0 item.size = UInt32(buffer.length) - buffer2.appendBytes(&item, length: sizeof(kcdata_item)) - buffer2.appendData(buffer) + buffer2.append(&item, length: MemoryLayout.size) + buffer2.append(buffer as Data) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer2.appendBytes(&item, length: sizeof(kcdata_item)) + buffer2.append(&item, length: MemoryLayout.size) guard let dict2 = try? self.parseBuffer(buffer2) else { XCTFail(); return; } - XCTAssert(dict2["kcdata_crashinfo"]?["kcdata_crashinfo"]??["crashed_threadid"] == 42) + XCTAssert(dict2.value(forKeyPath:"kcdata_crashinfo.kcdata_crashinfo.crashed_threadid") as? Int == 42) } @@ -610,27 +630,27 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]!["crashed_threadid"] == 42) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") as? Int == 42) } - + func testRepeatedKey() { // test a repeated item of the same key causes error @@ -642,28 +662,28 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -679,40 +699,39 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_BEGIN) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload64 = 42 - buffer.appendBytes(&payload64, length:sizeof(UInt64)) + buffer.append(&payload64, length:MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_END) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) - else { XCTFail(); return; } - - XCTAssert(dict["kcdata_crashinfo"]?["task_snapshots"]??["0"]??["crashed_threadid"] == 42) + else { XCTFail(); return; } + XCTAssert(dict.value(forKeyPath: "kcdata_crashinfo.task_snapshots.0.crashed_threadid") as? Int == 42) } func testRepeatedContainer() { @@ -727,55 +746,55 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_BEGIN) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload64 = 42 - buffer.appendBytes(&payload64, length:sizeof(UInt64)) + buffer.append(&payload64, length:MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_END) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_BEGIN) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload64 = 42 - buffer.appendBytes(&payload64, length:sizeof(UInt64)) + buffer.append(&payload64, length:MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_END) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -791,26 +810,26 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_BEGIN) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload64 = 42 - buffer.appendBytes(&payload64, length:sizeof(UInt64)) + buffer.append(&payload64, length:MemoryLayout.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -825,21 +844,21 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_BEGIN) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) payload64 = 42 - buffer.appendBytes(&payload64, length:sizeof(UInt64)) + buffer.append(&payload64, length:MemoryLayout.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -854,15 +873,15 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -876,20 +895,20 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 item.size = 99999 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -905,41 +924,41 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) for i in 0...size) } item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout.size) + buffer.append(&item, length: MemoryLayout.size) for i in 0...size) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return } - XCTAssert( 2*n == dict["kcdata_crashinfo"]!["crashed_threadid"]!!.count) + XCTAssert( 2*n == (dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") as! [Any]).count) for i in 0..<2*n { let x = dict["kcdata_crashinfo"] as? NSDictionary let y = x?["crashed_threadid"] as? NSArray - XCTAssert((y?[i]) as? NSObject == i % n) + XCTAssert((y?[i]) as? Int == i % n) } } @@ -951,46 +970,47 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout.size) + item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt64) + pad) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout.size + pad) + buffer.append(&item, length: MemoryLayout.size) for i in 0...size) } for _ in 0...size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - - XCTAssert( n == dict["kcdata_crashinfo"]?["crashed_threadid"]??.count) + + XCTAssert( n == (dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") as! [Any]).count) + for i in 0...size) + item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) item.size = UInt32(4) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout.size) + var payload : UInt32 = UInt32(42) - buffer.appendBytes(&payload, length:sizeof(UInt32)) - + buffer.append(&payload, length:MemoryLayout.size) + item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } - + func testReadThreadidArrayWrongSize5() { /// if the count is bigger than the buffer, parsing will just fail - + let n = 5 - + let buffer = NSMutableData(capacity:1000)! - + var item = kcdata_item() - + item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout.size) + item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) item.size = UInt32(4) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout.size) + var payload : UInt32 = UInt32(42) - buffer.appendBytes(&payload, length:sizeof(UInt32)) - + buffer.append(&payload, length:MemoryLayout.size) + item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout.size) + XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } - + func testReadThreadidArrayPaddedSize() { // test that we can tolerate a little padding at the end of an array let n = 5 @@ -1067,33 +1087,33 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt64)) + 1 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout.size) + 1 + buffer.append(&item, length: MemoryLayout.size) for i in 0...size) } var payload : UInt8 = 0 - buffer.appendBytes(&payload, length:1) + buffer.append(&payload, length:1) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert( n == dict["kcdata_crashinfo"]?["crashed_threadid"]??.count) + XCTAssert( n == (dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") as! [Any]).count) for i in 0...size) item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt64)) + 15 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout.size) + 15 + buffer.append(&item, length: MemoryLayout.size) for i in 0...size) } - for i in 0..<15 { - i; + for _ in 0..<15 { var payload : UInt8 = 0 - buffer.appendBytes(&payload, length:1) + buffer.append(&payload, length:1) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert( n == dict["kcdata_crashinfo"]?["crashed_threadid"]??.count) + XCTAssert( n == (dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") as! [Any]).count) for i in 0...size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 item.size = size - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:Int(size)) + buffer.append(&payload, length:Int(size)) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["crashed_threadid"] == nil) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") == nil) } func testReadThreadidWrongSize0() { - self.testReadThreadidWrongSize(0) + self.testReadThreadidWrongSize(size: 0) } func testReadThreadidWrongSize7() { - self.testReadThreadidWrongSize(7) + self.testReadThreadidWrongSize(size: 7) } - func dataWithResource(name:String) -> NSData? { - guard let filename = NSBundle(forClass: self.classForCoder).pathForResource(name, ofType: nil) + func dataWithResource(_ name:String) -> NSData? { + guard let filename = Bundle(for: self.classForCoder).path(forResource: name, ofType: nil) else { return nil } return NSData(contentsOfFile:filename)! } - - func testSampleStackshot(name : String) { + + func testSampleStackshot(_ name : String) { // check that we agree with sample file guard let sampledata = self.dataWithResource(name) else { XCTFail(); return } var dict : NSDictionary? - dict = try? self.parseBuffer(sampledata) + dict = try? self.parseBuffer(sampledata) as NSDictionary if (dict == nil) { - if let decoded = NSData(base64EncodedData: sampledata, options:.IgnoreUnknownCharacters) { - dict = try? self.parseBuffer(decoded) + if let decoded = NSData(base64Encoded: sampledata as Data, options:.ignoreUnknownCharacters) { + dict = try? self.parseBuffer(decoded) as NSDictionary } } if (dict == nil) { if let decompressed = try? decompress(sampledata) { - dict = try? self.parseBuffer(decompressed) + dict = try? self.parseBuffer(decompressed) as NSDictionary } } @@ -1214,9 +1233,9 @@ class Tests: XCTestCase { self.dataWithResource(name + ".plist") else {XCTFail(); return} - var dict2 = try? NSPropertyListSerialization.propertyListWithData(plistdata, options: NSPropertyListReadOptions.Immutable, format: nil) + var dict2 = try? PropertyListSerialization.propertyList(from: plistdata as Data, options: [], format: nil) if dict2 == nil { - dict2 = try? NSPropertyListSerialization.propertyListWithData(decompress(plistdata), options: .Immutable, format: nil) + dict2 = try? PropertyListSerialization.propertyList(from:decompress(plistdata) as Data, options:[], format: nil) } XCTAssert(dict2 != nil) @@ -1227,19 +1246,19 @@ class Tests: XCTestCase { #if os(OSX) - let kcdatapy = NSBundle(forClass: self.classForCoder).pathForResource("kcdata.py", ofType: nil) + let kcdatapy = Bundle(for: self.classForCoder).path(forResource: "kcdata.py", ofType: nil) - let task = NSTask() + let task = Process() task.launchPath = kcdatapy task.arguments = ["-p", - NSBundle(forClass:self.classForCoder).pathForResource(name, ofType: nil)!] - let pipe = NSPipe() + Bundle(for:self.classForCoder).path(forResource: name, ofType: nil)!] + let pipe = Pipe() task.standardOutput = pipe task.launch() let data = pipe.fileHandleForReading.readDataToEndOfFile() - guard let dict3 = try? NSPropertyListSerialization.propertyListWithData(data, options: .Immutable, format: nil) as? NSDictionary + guard let dict3 = try? PropertyListSerialization.propertyList(from:data, options:[], format: nil) as? NSDictionary else { XCTFail(); return } XCTAssert(dict == dict3) @@ -1282,7 +1301,7 @@ class Tests: XCTestCase { func testSampleExitReason() { self.testSampleStackshot("exitreason-sample") } - + func testSampleThreadT() { self.testSampleStackshot("stackshot-sample-ths-thread-t") } @@ -1315,6 +1334,14 @@ class Tests: XCTestCase { self.testSampleStackshot("exitreason-codesigning") } + func testSampleThreadGroups() { + self.testSampleStackshot("stackshot-sample-thread-groups") + } + + func testSampleCoalitions() { + self.testSampleStackshot("stackshot-sample-coalitions") + } + func testStackshotSharedcacheV2() { self.testSampleStackshot("stackshot-sample-sharedcachev2") } @@ -1335,6 +1362,14 @@ class Tests: XCTestCase { self.testSampleStackshot("stackshot-with-waitinfo") } + func testStackshotWithThreadPolicy() { + self.testSampleStackshot("stackshot-sample-thread-policy") + } + + func testStackshotWithInstrsCycles() { + self.testSampleStackshot("stackshot-sample-instrs-cycles") + } + func testTrivial() { } } diff --git a/libkdd/tests/stackshot-sample-coalitions b/libkdd/tests/stackshot-sample-coalitions new file mode 100644 index 0000000000000000000000000000000000000000..3f837018bd1341c006d37507a784ce7e75c1b8fb GIT binary patch literal 14448 zcmb_j50n(enJ@k=e*|0+0SW4eh|0mT3kxj&beCO_l?7H<{uy=YndzBrccy#n?%8G4 z#EAkT2Lx6`jEE}|K%OFjL^Nl3+@0RL^F@=3#)qDd#~C$wzDV%iJzg&HF85XSS3NVo zg+MM>@AXvwe!s7)s;j>Gs=9l(@5()^1f`t(q12O(aym9iK*{rsSgY6K_|2BxoJbi- zFW~iWuSk!QKT+yhKdwvWfA;!}71!74{AxOp*Ztdxt)(*WMOfCqfepM4&o|S})C}J| zU;VN^nSNEjWUIXXivB{B64vdPn&f{y3k@Qu^hjXXu)Z@s9SS#LZ|nIlFvKMP+3q<~5sk!f|6}+O+-VxaAqqlsUU1G9hbwM#5}vRjq9H%{6|+ zGhNF_&7K@-kNdMHR7Avpt+L2Tbd=Je`}bRPF8jQH>AOmw4Ej5deYl4xG4R88&AkWG$`OGy!FG)#!Yltt)T*%q1l1zES>-rbmDqNEs# zK7g@Yv>yQWeE|;yjO7gmoo=ZpsZJ_ju)ZM$nmoM>@KC_m9_q@XG^hv(F#<5QcO>YS z10Dq!@8??3uK*kYjQ2Yl^eX|60gU}I7WAtC-wYW0YaHmq0gnfa{aFq=_NSbGh3OlG zfX*ucV}DNt{TjekfN?x#fPNj|S%4#eZv}ik;A+4)o^wFI0dO5)9PfFc-w1dPVdqJ-Nyc#h2 zdlYoJ-V~;IA)s@d4++radQ_NN3IUzte7O(wX@G6O(*b7y<9u>Kp9$CpjPtDx^x1&d z0LJ?eh6?i;BN!20h|Mj^Z60bYXPqXjPrdx=yL&Y0F3cqBj~pS-UJxq#TL-( z0e1pk0C+oKj3+xlUkG?7V2n541AQ^zU4Stj?FRi$z+HebUhM&WDd4?;F`n%MeHq{b zfHB@31bqeImjGitJOsLW_$lBH7o5vejF(41|0duc0>*gyD(LqBeho0j+aH784EPPe zs{tPeYydt17~}OnfF1+We*$a*{%61mz;6Ri0{$uB7QpWUwgCSeFs>KB0KFCP z`+!q`KLDHtd=jt?_#?m$;Ew_0dUXo)`vIQ@>;nD-um|{4z&_y50cQdK25=kTF95d# z{w?4&fX@K#0DKnk1AzYn@PmN=6YxWT&jFUt--YS-g@De#4fqd$9|rt2;2hxp0sI}n zqDL|DNHG*WL4OqV-k`4qy#(}ip!WlPJ?H~Ke+=}2pl<+uFzAniUJCj~(1(Ejx1bLN zeG}-zK;I1d2++5HJ`(irf<6lLt)O2E`Zmxbpm%~k8uTYX9|QV!(8q%QBB^t(WR1@yZ?KMeZ4p#K1L6Z9jXCqX|7dK&asL3cnu2KxP=zXtlvgnwQtgsk`4 z{&_L@4v@#Xm?(XoNV-ne=8a}gHZkHx#<$wS?Pah0rA24Gi%=f);Zv$Nw$W?&nk4-Q(UlLgPHf* zEk72rLR_wS(@&?vUJKs!ONqMJw`^(%rz<#5Q#(x~9r8moFYnhi(~cipbEW33@KzNw zlzG{2ZDz`eS$>Bs>?-D4Oe2+Vz4}z3l$c{Dtb%bF&h{CnoqISSMlf&q#ZM|DnV0p3 zjVJCK%%{nv{AN4hF&@QvtJx7X3hKRv`64GF?po#(M%qes*p5&2#0+0t$9!EjooO^| zOSdY*yl18gnhkZcD&58ebW`U z1hhz+zF)Z49>;u5CX+Jj>uShF8353u8xdNmCXAVMFKvT*%UtaS`69HP@2ko)bLvPRAti`P(K@f zP_$yEGjFAhARsZG!8ykPC(T#Q40+o(Xkxo*!=@|6EX`AUQdYjHvzeFU;uchVEAzgU z3Hwq~w=o|}IoWu|u(X={LR8#9x-_<%9i`-DmWj>nKBNY(yn9sOzgC;5?A#Ue0+h_~N zL{jsaPy1=pP0Em)Q6Zw9`DiL@`i>KZEd74a@=f}X?Z(T38y5|1U+bjPwKVl)8)aG; zWI{y@J$Z>+r1?bg1X#>`ERk`|M1BYwnV0i1>d*qk=}yk+U`Bc6L#Vlps zXfN)!Cg!F8>lh%ZWz5@#O)F25Y6(Kna^^FwMcpY@FfaWxPF<(fmAV|W#Z1vmVklN= zzM%Y+kqzI|ku#^fjHR~&;+t%rN$8RT;%??$KjxYlhg>?CeD^T#nX#xqFkdU@#R9V<)I7BEjXW`KT`@j;5YUeyrH9bkJF- z+J8GaFRjD#O<}Qp&<_+SIR@RwJk7f%ihb0ZGMxKDzaO(yw3QtfxLk_uX`PZQZRmAr z=4q9ql{4uYbcLjBZ4dtAFrS}COKA4ZSAWAHrMM9mtt{_n`xq^nyw)iL?O^$G)SU~x z{;%M5VTlp9oP~zaGiqt={38F%a{2WO8=Pb^>;};m@^y{!k4PvO*fFz4v@`E@q%*#g z4ja3Md5X%Z4&RDtK}RTRf3U>!SUwPzW5?oV!pNqAK7Ek+BLAdue2Dou6t9{bM-PYi z7V}BdDe#i~dnjH9x$83GVYY9v;;u=PSZ+UQ=@&WXjd*-fuyod|Aom^Sqn494X^Trs zwwm*gFrTq9W~yML`KT_(O9d+w57b(1PgAR~q3f7Wq!Vh1q!uY=z2+%2q&woK+~i2* zG3L`Wkuq5?+!1eJ-jMr~m_fnbPz62Cd^B9q0%D^shc@kn3$T7a6f<1D)4*^O+tUIf zx9;IBY-V1@LURoTK8}c6m~YE(wH5PS=7V{r(0G5^eN_- z%B>`y%WNl~*QQLv&Sn}ZNYn1OQ9R8&jY~e55U1bMw42eWRD^V#2)5d@=P65CG`UH9*s-k^V(kK<+@3+BkH8&b2s7{gB&33)|^9g5t2w%xS}%WJoqeh6IbEgBbpKazExn=tO9;z2 zl;i8P9Ye9V^QyjQ>F-H5->8`&*Ui{s{%et-`6bgoS0P0HEvihKzmDDr^rp6a)%o}D z&8dC+!K=Byb6m>S*Mo7_g+}P4nhwbhBik{z4m&8z+BGZ4>qe(kvM5{CLk0!4r8$ub#h@TU+dVmNjrGQ zUbb{jS(-N66b(JorZqbmSa<(L|G5)PB802y(EhWU{paJIJ(XXK*c0TZou&=)Ubefp ztNirAo0NX~#UMYhJ;*QSKSQYunRLJE@#uSrQbaia^t!Hjov*40JyKD6JjS2A z;~Gg^XgoeV)8j8354i=-KbPwIF~{+U4KKF1nc!+VE~Dcwna_=fHmG@Bwe8ZziHcY0 zQpZd8yB@C_E^)lHovu&E%MqG@e8m;TgdVSt9;!I+{E_ierjFV_y??%V`C4&t;;-%B z5&kp%WSdU9L4KimpsT&q{?+}Y`*#q<0hulqpR`?Iou3akX#(hYf z!u2nVE;$)NyQfg^E(Od5uG8k>^%tG)HjEH# zq(gSc|0Ujj{G9p!vmWwz=NXsV&kLjdK@^T;@=PdZbymSk4^|K@Ams&tBa`~ zax+iKn|S{5xsBi4cl_f==Z&ueXs=qBBjv=WpkZQ2T^YJA?f}7w;2DR_S8% zP5ZCD&+h+Ev|F{{RR5OAziXS`@WPH8H%@HZM34Ww*Gk`$?U8@OBiGXkS>uUwYTx_$ z>-$fBG3$D5Dd_T;hM_lR7uAAWMr*DsFkKYix<=2ud8uk5{3RF77FqIC86J*U2` zuY6<2!<&B7xMw$z literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-coalitions.plist.gz b/libkdd/tests/stackshot-sample-coalitions.plist.gz new file mode 100644 index 0000000000000000000000000000000000000000..110c456fde759a0f13ce0286c31c0819ff5d5c07 GIT binary patch literal 6794 zcmZXYWmHsew8jaMQc4<00U2QEmXuDVyBUxeMjE6$hVDi>1?diH2?>!JLO?(m2I(5& zj(^u(>#lXboOeC%@7>RS&OYbMIrLB*oMp;i256XW?v~b$Zto!OKsQV9dnXXk8e-{W z?`{uqcH?^QWbfvVhA=huf{7#t9A1+xca7}mGW(`XxExG)eq&x0rM0^_NFh+`=Nst~ zR9-s1y}@EOS3-M)RXO2Wem-~jwv80m@2cvk@M!1H`l*wrESwbxZLK?a0sj$8?vhn9 zbb_dc>^^wY_S_&ld%Lge8rL4j$BC*X7}|!hktPHLyC+A9s?1XVSC#mplt_XLWmLldzm7=D zsYs!Ek8W@BB#P3xMr%xq$Wg!i>*s0bqZL8zTv4rx?` zZ)86iGbm#-$E?JEouEimq>0rPh#h*II6kmp!ie%iFR$cOkmF*ueD?tm)NZQRK(ryy))t}K3^m;&qq-^ zx@g<*r4aA(B~MUm+5w#(soU~x9uqOMhg39?3-_YV6ZY&V={(YO1Ug@#7?U5PCb3*| zbK6(Zw69UFa8VNE>p5d6O(&`mj=9nSzNQP%+4hYsADti8U*!gL{`tR3b%h~o)&^zG z)=935ubmZJ(Mgb4fm3v3q~oB5svyJttImvc%Ks|A@V{;Sfb6cQ5WM<0BxEwF=1}&M zuWAZm+{_o@Y<=Wu`hQLO2$B{liXIv3%w!bbcmCJ}|680(N}{B@N7 zn0pcqnfA!kRJj%*RllN8D*QlfL!5*UR#bgz4z8vC|7DIz; z)?+laWQ8}j{8pPn_<&vJB39$sP?ThiVBF4Y;^BP~873N~;tuP#v?M4ZT_qn`v=Oql zq9h=6<4Fm40lM>i`#s>rG0(){`n#-NV%{elm%_FukeY+_%)Q?FlR#@$G{rHrLyI!h zf_`Yhimo_@fl{ai^N?X|wzSwGWhpkwQmMU7=&lr>E&C*R|K>&}-uFkEu%X8_T10^%S=KjtFxo3I9A?3rI`(R0aElLYQCNoz4_`iJjlaL7g6yoBN|M51sr&ga44*KScYlYUIDFU;mKr zKSckp3Wc&gN}Tq&%4}%?c^%M7Lkd9x)UZSgl0CiH-&P4=~H`;Te|asySpz20p5)Z@=i z%Nv=bzu7M6VSC63BM04|ndM7HA8B0?#^`D|4nc-qIOGYB1{w=-58{{PyM;rIlq;AjU}V z=i~JES8ut@JS;A8L;Oqu^Gl{i-Ht}t+y;VAt{=y|a;-ndm?16MR2P;#&Gfo*5yQn_ zMk6`fHEo?^6LQCn9r2s?8RRiO8;jWOTy($0QVJfuPiCDhThorVICDsE*}8cnk6pda zF)6!eVAN}7ykM;yvFOjf<8(+tmgU&{rh`DWUtojQaa;wP&{rZI>WTKNoDQ*UH`wk> z+MZPMX3c&uuIuR@n%N(?iefAhr~B^*q5+*SZT&Qbfe6lU0MGF8Qd(H zS@O`|9-)+4?VtXI9Lw_OxU!2^sPL+Bs+G@UOwHv4gap}}fb;yA!ts0WS%UY7krZz9 z0|~N5jI#ojV|b6mGI1#fd)~rcYZX#FzT(uc1)YM=q@GmhFvK~z&%rmenIkr}xlPKc zsXCvu4?*^WYw#wfS=R8I=mJMI3!#?ALO@0{8C3Z?ys+j@l3nXuT$bx-DH%_8W3X%6 zKc(sHb@J?)St9{GErKHXRMM-!fhsp(CKaLsmrT<@UO@L1I=qw8d7hs;Hebjq8{^)U zr^JawxaPYHc#ja58xxjTS9W``&=no%*U%mR_n)5N zne9eM&nx0LCHW$Pz9-=G?gf2mOXN1(`%zOKu^d${8B}3c665i~d+62Wp)(g~9^<^-7H=^8n3ED9fi?D^f3 zL7i0kgHZa#7U2x^F8vX3UW?8Gu6g`q1v}+II5@ng^k{;P-O{|e>U;7J*Ye-;kN_`Q zv?0&Afd`KH5mP!`S&bRfueKhLT^WH(rf_Xh^#r%&eIMpB@T@6Wcji+G0fHNen|OZ9b^h4(aOGjIBw_sArPQiBM+m z@MHrW!rldv{w_22Du|N#Sud|o@D9v~wXX?y6+;P2wnrQ-OYj)#{JO-AkE9257mYwl z7T0{JksU^4wHnywkp%#z6eFgOaa$f`*<~Nu=PpKT-LX7`syyLh{%M-st<+Grk1xGv z;#Sk^lv<&x4P!>im@Ovs^sTa~Z_kT5QNnG$*;%F@<6F%td_qsb zm+MRHYbCfqqo-L%OL$d_RQ?sER@@v@zVd&zLq*lKguLK1MlVg(E$et*s0upf4P!1+ zFV*g{3kW3$eqPEWcNnL2L^Jeh&u4GZR_G@GOcKYrj!;va${r6c9Vx0VE0QlV#C$9@ z+uZ?G)o!nRJUok6yjYFkpEN z$EHklIUYp$RAgW-FG((ngw%=oTf>cd@Z;FmM%{YM^!&AYj+j`x4am-XEeK1jCvamq z)udNWU39nwGyHkk?sNFvGmpqfztqDF{ln0`+tkDCP4B`#4vy1T?5i@xu~hL$j@6zk zMq};N!Y;61`XR?sEuOmhYh!@(@~|WESM@(EfLvR|PuQL1I_=`O@EP|!|K}rl)gf=6 zT?Q@|?g7QCg2pnnzY8vS$X*9jXt0Qll);EWi_RatPDPCuRw zDB&(=Jk zHPaX{Ih5PURVclr%46^E+?F#qwo99N`_}QYtf#Iu?tt8A5He-k)z$J zL#_d3{v7$p(@|@4`mPapjK7eYZn4!=jO=>Li&DD4f*6T~b52-3hiQ<6l-bX4vyX2v z4)yDkIWC4fZ3A5qZy$?S9^rejk-X#=S26{iXGy0$}>8k)%G2X575{Iw0$i?sJ5#@f^5R2<*s)-D!g4!56E)+&3`FUz>7x(!>S^5J1`!t~lSq zxXybuT~A$qxE3W^#w-jM_53}NMg=E~y->7j>fu|>3U+=*jjaLrK zthklSXAy_%4N>#fz(4CnqwV><^2D<(l$wU^kFIL8Vs?`vdUeM!eQ?GYpXFbWs9 znmgl&y$2=uZTg%z;n_Vc(TrUUX6TZgmxHN^+K5rF)BLy==5&+0{mAdwod>W|p1y~V z4C4oj7`qaCtrkCWpxjnjkY|={s$sxDCe~m^#@UKB!j18XVY5CFI0wVTEvJQ7X?$2( z5hjjpAXjoVXE5NjWMjrY$|F3~mV1|x{Jdx{Z~9Z2&r*niGakeTFW5Jh5sbhItr-SuIj@2q6sLH=!k0Km98#(-ycaC);w&` zAD$20Y(RepBUM|JG9gO*0*ZOTvtrNz|B%X*C_Z@@lZ;Jgh0zyZy5(vuA!5A!gF>qW zP1a-?Ole-lAC=*Z$E@zNmQCA^=T4JHKTEM#fQm4JPg` z@~_0)dYET@r0eNT+m?6TFu|dgXSFLg-k$xp+;mrHzk>l2yz3buS9h~_g~f9$5$}Fb zE+x0H9t$F$6?tgCLmC}FbaCT8l25ZyE^aCJ=wxRbT@b=*g*BB?8EaQ{+!D<60Q$+A zx`A-Vod_;y1NzUmv#ZuHxk>J;3~U~Au@OP@(=~wp`XBC}f`fRLGUbk8_>7yET4L?MflLsAZ6~1r{T;3{ zzR!Mg%_1IT#)U3s5yP*Gzrm^Gx^--!WTa2kt{NZU5>FCkZ>v_EKAz2G8G@0lVp(&% z{_Ty1W6S5%!6YLH%7+}d;7OF3Yk_6SO=ugKs=a#4&_c=wWEPRPGdU{eDXRY2! zq#Btj=|(d`!~7YrswmduPC5ZJY#@9}qD-i+S<``fgrq%f9O<8bU>1R`(Tep2su|wC z5{T#Vkn~j0Lu2C7=CnSvo7Vw^5t3&W8G7K(Q~~0DfK9XmDlyffUh>6r&3Nk$RJ>x+ zHsHwC%@YJ0rmzfBlVpV$2VB4x*6p=<3+SzSl)WyTchTxDxfv#e<=eFnZy!?DhV+?% zJeA#w#vQoeRFxn(Kc{ihokMKXl`>)RX`C4> zYcLtTH^XpFktvO=H+D;^mwV0X6g=ZzDs`N3*_=TqXV#wr77PqumaGSAHz5(Xzj6iq z)=88YA;!aB?#_1@&0VDBa%Ye0XGH1izT@|+`DqVs?NOb&8$Pp@43k@>JYuNm0&~U% z#)RIQr6ahBOnHVn!-D)=+$_bV9NUO7zqt32UgWRcQKK<-Sm60S`1U$Jyi%S$;qR-b zgca>;%<^ z<&!dZZrNAJ7zxWuJw~TtVOD|<_b+rZY5Zm}1o1fL*QfXFZUi53S?k^*o#dy+Nic3p zPFJnL-1fL9%j(2`%&OdbFVIG1z}7wwy4m6-@;gOpQCyUb-OSe>r%aOlP6La>t`E?r z2>_D6M?)t5F*W`}byE@5v^}r&q?CO_Tr$??Doyo02S#YeaL2>=t?Q?c?3NnV*%)ge zBj#(<*Vl8{dqFM+2|DV{4$>MRYSo8@EKci-8de5G&gL3cRQiZe`u(mwCUb6MT=N5O zW$@9Y!Ik+B7ODFuvn|IL@R*jfS8kjZYQjUUG>L9)s4(hm2)OOSPky=ES%b@|MA{wF z&-E7t3C(Rbefi|lUyQdf(ez2>ZXh&{4@(V?XKMis$18I1_o5SASzylFUe3mvF07@8 z);Ph3=Qg#_lXT1HqpW!-@-Rl|AtT^~_r9EpWvJ>#>wC=H8NM7v>^ZA07WpS3y@&N# zD72yL+tNPXYn!+!^v@JipavrKoT3%mwCgs}UOX3zB_Y&*C2u3KZG9SmOV2;u2T{Vy*jZ ze(*}B_LZ7Xng(dv!Yi?7!fCZKSLsheaSW*G-*|TPmhR~WtkhR} z%1_OIzb~oMK)qKnzIj}J;^s18z_=-Z_+fh2PBBpt_N!lO_EcMs8EJ<{9t%;U^F*sy z;7KdDdm`{E-o+fxgogg_4*LbBkUwP*f7RJ$~hf)%)oqk z%^xW;9j?)_?h!67CE5F~HEJ(J59D`l;3m9m(U}i#5VbJW3&gxE+$?|4GhF^Gbx;3; Mcbojo4;}4)0Q}5Q+yDRo literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-instrs-cycles b/libkdd/tests/stackshot-sample-instrs-cycles new file mode 100644 index 0000000000000000000000000000000000000000..bca394042264d2b630c6fafe2e5b134f9e051bd7 GIT binary patch literal 625 zcmV-%0*?J3iwFpvhEZ7n19Nm?V{3D0Z*(nlVQp}1Wi4rLb98cZEn|6OY-Mu*V~ut<|%lm z7UiYpD1@aJ6=&w>DHxjT8R!{UDTJq{D0r3TDHs_7g{+KCtxPNw0$f5Aj0_CTtrc<; zOG>ON@=A40Oe{@wQ!dnwy$e5}%S;oS2l8YHO&F zlA2VSZfj6sl5Ak205i{s0pfpH_|`(h90rWh)WH4l~!m^o49S)uU(vk#_@(E+=8wu0W^^n8N>nk)|p!sTFQ$e`%~nOU5Z znp!}FdzGQ-rvZmN%sJkV*51%=UAfbKpiyW91LGw38dU2QsHZ~yq z?8z;2%*>i?&9hvM{ThMdVEbVDVd+2$&AzqD5PH5IgvMPS<(HOVDNxYMv$qiwKO*x1 L<`az&Fa-brYP1-x literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-instrs-cycles.plist.gz b/libkdd/tests/stackshot-sample-instrs-cycles.plist.gz new file mode 100644 index 0000000000000000000000000000000000000000..b35b02a3166bf9995fd9d4b084ad2fb45f27886d GIT binary patch literal 1630 zcmV-k2BG;MiwFpP5=&VC19Nm?V{3D0Z*(nlVQp}1Wi4rLb98cZEn|6OY-MvUaBOLF zbO4=Id2AI$7~h#a3W754QIVst>wyOfeWkCo6e|T=6)2^a7I?H}-R@5Nw%xb;x-(l^ z6rrNlTM+?~OHiW5s2EL*ffy5G)EEt~96{w!6HN^Mfl-4;JZAPiS|El$+`s1g%{Slo z`_64mJ2s)QSO!2Hp-$5>sG$>Tn4Ki)(UWMDD9K9Ru;G`C71gbKnDhdQeI1P+x?)I{ z!wPDUPMx?$n&XU8Bp}!%Ip1P1$waHiUKlK{L9B&(P&}UOEtPd3j0VztE0n!CPuD)?cjuhiIY?{VW*hA#>RP7 z%tLstVFAyvEFTl*G(UtEJGM2KVZUR-==^0n7pf5Ys# z>n@&A9gVSFLVco}y{3c7U@jOHn)ESK8_Z72dC59B<;I()R^0MG$xkbc6-_UW$4h3E zmdz}mbu+3%OVMZO6ZAIPhGw93r~;Lu4pfbbQ7!6=f*qB@Xh9G_A3F?G&QaCnwmHa| zIc7o_M_}@jlgzDE4aGA`OAb}vcKaRk7J4*crh~8~OncM?fn6jAydC`PNa$_8!JZGy zUr?haLYTKoQ*HwIIT9f!QGV^Bx;yI^H#9C;x-7B0skvo^ewWc|TDVQM)b^y^(do3O zShd^)O_JQ;a7IM8x|mVha9EqLW(sdvy(~TL+n#UJ-WCq!sPk5jaM>{k)&0N?V`~l{ z^^#D?DTFh4)bmIpb1x#D9xhD+UEz&QIXZp?hP zv5OTPUTN4=XS=|q2!!qN*2oE8BQbyn>A^NGEgdsd$n;i6lbGi%lk-kzCsbLKvki6;@d|A7blAA0zaM<09qi6@_0 z|MUj+8Fiz!X*27hXEjyb^4xN^x%f(AL|Q1X%Sye?2pr1R8?PwEuGy=z3@5lAyH2IJ zTHxAU0nV{05l4hkU>kTiDLcJTuC4yiP3 zK|T{j{yO)Q91j27hjzP;$(IXWuSvcpzUtQn~`yJBjQ#wv-~ zM)Xab;u)}P!rQ;1NV9Nj&|VSiiJP&w7O50;)Kz8pD{WU>5dIFdCD~tqR#ZAIHZ4}J zEyb2LKXA3_MGPu0E-84+yCDk>H>K0blHt2Ke7tV<54P~*SlIY(BhwE50BZNu9Y@tE0g?oh!!kfZ-LPq!z zKo9|wK_Qq6YJdSukOVHEpd0jn)nF~?2kXI$U?2D#8~_90FgOa1f$yLQ3t%CvfK{*- z8n6dG3Ll40!u4t+caf|r6xL-UV9v6R-L}{cnMJkgPNXsNFxzajmv-E5a!mO~`Cj=!Ii;Lde$R{N cmFB&gon{DfWBt5v-j}!cUjbccR1gUO0GKu%nE(I) literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-thread-groups b/libkdd/tests/stackshot-sample-thread-groups new file mode 100644 index 0000000000000000000000000000000000000000..91eb05c6ae9f403e2e6b25a7544aba24b6e20e7f GIT binary patch literal 6784 zcmbW5dypJO8NmCF+9RWG%k7Dxdh}rFH@+Mn2 z9NJ?j-Gq3UCoLJ3&KF4}g3^T|Nu`QP{VU6T<-S#=i-cbqc3IsC0!-oI*~uQWD1 zSz6lHzw(??8i-hEsaH)$nJBC36=x2wyKvpc&10kMHceIr$46HTR3^{r>t9|_;V_tg zE7WgA=A{)8)l}k%P(-@o%am6GvAVy+!C=c3<|~ezh3?8mBlqPx)5aJCTMrv$Z`oYZaUiKAb6xY9Ul$@SB}^OuaLI~l9b7`Y!Yt&13Y8nP2%EG6d$ zRAY>EsDgXe6O~2iS>zAg=dqSkf<_eIiI~af7Au7^y#dS{sCvh3=VPBOV9<3_uq1tzzQcp^3`&K4pMg)cdbnG%T=Mz2;&g^`ld3#_uStXgPc2aZL~wTC5_9gj6LZ>}V;_aP6#rF12#QMfw+c~i8jyfE+kk#{)> zuoIEjcqoH*q%_Rqam`LbKAeT|7>^`rRYIPMpr{Tz8F{Fxu)oslwyC{;6~BBX$AJL^0a|P zo2|qO*pq7~+LE)8%aFT(g!NUJV;oQ+jyfmjk>;?mlaNO+l6{Og=o83}s;iM3`zM8o zKaO0>IJc!i=OXt4mH9E3XnwW^x%1~1Y|DnP;7Axm9&2lrDb2R zA>_PXPek3iz9qxRH%|@@C1ATD6-#h8whnnUqm^=CJ@Pp5Ic!v{F?Jr7B0iO`i9s8X zhZ=Tb-GtoOhKLd5)gTjEsXSzzk6da27n#H_yBB90kq@ab9D*%obTnzyWz0ql+N8iX zA@^!=B5EB|Z$=Im0}W&H1sg*ibh=t{0rIHUyj zG<4unqE;~RVmc`Cdt!L!Jq%aOa~;~v~C%uZqz_!NeMy-ZDzFBuF7( zVJx7dd&um&^F`Zh!0sMPSdH{C_mDtV6YlH`ThAt+cRLCNosNHwgI8~068k&-CqPSx z8l#F`fgJYTB*Z=lWhQKE)fjoF(XhWC( zXDEMUbfc=*b2G3hg{iMbUKgrB88i2Pld5^dcJqw5d4|p7cF1PxA4tdwUfPM6N z>L2l-t))i>`U3TbOLg&pydHTitl1+KX13%8;t;~ZwlB=($F$stJeIKt3KwGfUX0;@ zbP`<0o2mb-b-x99$|Jw(t{Rqn5qZetG|tlehJGu_NtC(j=TqE<+{9)v4S^Wf=62+8 zf2O0?lI_SdSScIDFvi`19PE(GHt0*pOht6RHYzOkG z&L@E-UqNnO%2Qx{8QHjc7xEPJFMg-&MDF6xh`Z&&z51)jhXTQ)EFMcl47cJj_BG_@ zzP%Pcz^Y(x*v0H_Mz z;X4}d&hCVI47s^iKrE}Oz4P7<=-6 z`@<*25c>}D1a>r?nFc+DJk9ELSo(bO?;>~Si@nX-w(n8@SsxxxBR3!7D$S~?mq_>_ zg8cybp#A=m@G%D)Z_5b#A<2QQ-U}a>5Sw~i;GdL^g2EB-^dOjXAho!B9@J-`Nb!pL z_24fLOHa7t$l{36U;N5mtzNhmN9=wkZ!Ux5?E(RRL9zgIBNOgS5$r>_dYiBG*!&ok zk1G~>K%e^n%piPaE*AQpvZ~#50M_qw!%&-up9H@U%mteqzV$3C|2UZ6tp5y(G4*09L1hg* z#*BwT+mCg?zn5dvELuf3+yK~FFme{_84P8y6v|Mm+#Mhr@}89^I(nlZuz~H zKihuo!^qN3P6@I;c;>)k?6>BZ&VFN$yk_m!c;e^Qey?Nwtw&rPy4&<_?M3fR_Il=B z+Kc+_J*L}U71|%92b-(ty;*D;5 zVg0?@i{6{;_58cE7xmlM+ikD6F&B#u>V7}6*C{`FXM0s1+P}T%dxWIEt-W5G8yQc` z{R`{w)n4@8X1=ZPnC;XL4`Y8Tn|U6;;qM2%eDAr>?!2;f;^WOTYHasgOYBZKX8qXq z%a*NwaL?rNd+u)Rfsw}={_2Vi*Ihnu<&U0z_UgSarrV!`RX@(~N&Wn)-^6F%w6kZ& n8SVSGy-4xE%7XUeF@DTI*`%~&Gxnd;T>g)XxBt_X-2eXq=#6?S literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-thread-groups.plist.gz b/libkdd/tests/stackshot-sample-thread-groups.plist.gz new file mode 100644 index 0000000000000000000000000000000000000000..d4ebd8f86aaf272067461bebb6cff575a97159bd GIT binary patch literal 4312 zcmV;}5GU^+iwFo3;>lP519Nm?V{3D0Z*(nlVQp}1Wi51Qa%Ev;EoX9Xb#QYoaBOLF zbO7yL-E!MFvc8^u3Z2~B=0B;NBwLQ1J>!~W#+5j`b*@wp2}xL!1P1`EXrF!o%CbcP z1O>ZONv1M2RaQg+-2l43ZZx{VSAYLfm6JQnIHC3H7yhaDVuI_O7NlOje)0L{A1B$1 zzrXq0tN*$D00SQSs_beJdZ{|7iQGIi+HT*plxa|MH3pCdc$9=QCU{D9dq$ zYcVSbNAnWD_9v)Xkk=>v^;; znxQ$TWh?MZkjl_B4nl95MNtq8v{%jMh~pWN)AHDUH=WyG7PzL>!^6kk!(OM`-_~t~ znc=s75cgR)>n$1O_sgy=*(G9YQcr&CrfKre?XM<&D#gZoIr$eZCLa)+__6%|W$eF< zyvg|87eRlcopYPuPasA=l?>Dpaj~|j~c9VI2tSuv7 zA%=@tF6Ri(icML}N{R~WERjks;d}Y3^P*swbHgMgY3v1FW|+3F@u!j$Z{=cGI*6jF z9GV{oLw$Vye090k5HW_fANQ6b?Mu_%qre}0GW^I-rbCaW@n9owkcl`MWFj2wE*z#5 zc*DaTWWx7`TMcrN1%uuB!~DgEdUW=%Y6n{KfP1r|Wk~eY<$;ILOLb+}Th6G24Q4*K-SP`!v(Gs*h_pHAZfrgf%xr=*QinS_>i{x zdrI=~fYC;{Z^v@(35U2^QS==XNRIUti^|eqVb{GI+`Otu@Mb4`_&tb?O|Y5N^8HqW*htB zWF>2O`jvz&%Yq~T!M7l=DOYS8Z)!z@&?XmCTfa7`h^>g4Sr;#}aWsO9hRNQ?c^mQt zR`}BWmflmVUiwpgL|;STQa_EOFoF>{tLFN?bp@w3GZEV?X5d*n_N@qKUBA$7sI7+0 z+R_G{cB7~5BzjL1>r>yxut}7M1+MpGD=D;dHVGFb-xs!h$=rITeAdMwa@y8nU##}L zJZ%bkEqgot-WW13O!Y0xV{0P4TZU1=Ez~6*Xo_j*z`{fnM^jzudVQ=J5!l8>qRWV> zgF!CRBnk~4?5+>FS`8yqcCs-r(NpjJi7}wWuxE43qeOQx4usCYur_EjziJ`menAI^ zXYwqkHVs03eT{rqSfCMc_d^})wdR8hoq35j4K$$Ibx_OAYJ--fUfMWFbqi_eV2}rC zek0%4(SO&+I0J`q9j)y!?Q|L^S)#=v&JH%Fy`8NMpM+A&^=yIKQgE||2bxc?Ygv$D zCRU6JQ4Z+kP)i%VEY-J+yLP4A;Me(P_9rzrHp7T)SfMWx$~Y-hFn4%*1Gz;@Q!|jr zUK%-BRn=>H9}O%~Tj#6dv{2Dp+NhZEex$J1hqN3_GJ$h54^cKv{4Dkcr(FerNo}5h zXn5}SgDE|NLk*-n_G2mgt>PdGvOyMk%Ntx579|*5sry)t3OB0|kie`WwOn>3YHmGk zm0%D`rcp9vJ!RvzCV#frebQQvnVETJ<|CAja}tM%*@Vg-*N8u8nVA_a{;0vhN||v( zsLOx|QRIa^Dw?yMH*Gh<12233cfGJ&dm!po!?SJK&7!4UG7TcrmOp zFxWoBl#LBltoW=u9ogCK0meg<3%=_hr?nt;+Yi?s$XymiNh;UKi+#^a!z?hyGb7!R0o58G!0f*ITbfDlMUefYoudhshzkkT1)j0CuGN8C|FmjgF48ddrg1Dc z1U9z1&Sk&0E@RoJ>@7)6``}$i>}k`Yn$74PW=vY4Crl2{R}x1wM0RTZ9O*QfgMG{6 zO|7j}Y<_4VZS6MPwqUj)->YyR!Fzj8&$LH8Xf6gJNXeX(x(vOZKW8nS?30k#nY5@jfj{@H>W;gX8bhtv&0ZpGC*}V zyzcmCwqQszF9%VW9t3YYC=W)nAH9vUNNpSpC_k+ z{^@%B#axfrc;4>rdPHg`6ehOeALV)^#SD}4WSJN_zehcOM~Np6Q1O^q2hPHu-y zlgFHnATj3cd{dVr$qYG*Ka$HKp(P6qhnwHj<>;6Sjj19(tjm$AL*Vb+;|N0i?DvOo zIc)Ii=yLdxpQdqS%8feSv-^3CdiuW0=h`||y%er}&X1<6l|qitol6B(032hW!Ud7L1}2?Cfa%vNm`A8DcxTNy0BNnza#W+$UyfMSFgnK& z9Jt~c;+#mYqZSavT>))DN-RfyCmk|~mDegt#^~o%+ObxwAZWV+9A6WWgHUz`IxMOR zz=&wI3}b|)*_1nb5HWYfxDH|%uBZSZ?~HKrFGdvvX;*lw2n4d}7cU%PR8*#YF1Ohb z^j&dng;5Ej*jgR*IH=SM0^vvKms5;TZs-~WN42E2Q6nO90IIHVqC^a0w9Z(fY9?m& zf&rfNV%!0ir3@_z&qB0dt)42wpDYDY62kQTAALs zsV*TV-4#eGI+vcAinKw%N$Bhv(S=Z(k+vxih`rWt6dK`x)Y2y|LFV2S)li;e&MAYi zn%d#{sJ68bR9!)B8;=zmwFM+OM0mp$+>n&FTdWSpo*n0K1vsIC($XcD!VW;)72KqQ zc3aSOU7V_B41fbyh+ATz4q~27*A?Ts>kh9j&*k<{#a#e6aQ2$GTy_ZOT+105^_+4D zs;Te$^Z)|3M{ zQQ9%xF=6w$T6uusYp0Ckigw$mLSL@#n}lG%72}jA1wq#pp0W?s8yU>sW-2%Dy*MUKvQN!y~4RuzJ{E4pkqsB5Gm=yVCdfGf^40>WKS zTwSPzobRSYo8t7>;Ix2HGiNtyxny_=vn@|taqf?s^9v>o%LQ!+#AZdm?RdGC}%nOJZgnmo=5&KY(NL%)lT!TWw3IuIecd5oJTs9EuoVlWz+!d6(ArP*y zcIWOf!Ob}Z*pO*?@Z)B)mdm}AS`8A%*HTJHy|guipQ2qKH%b;{D>8LI!c6bD+);AeO)t;XZWa{|~EkQt>N*mpXODICT+iMV1T+3L*c_c~zIcfBJ}d zL+b)U{|qmw8N(mmvXWz{ptkS10^o&;0R4tH`3jNx8e|Tf_lYX4uOxGZKTONWR`WsTz&S%~Q|ATVUQ!OifGhZY#0_Pl3r0hbcTT0a_!VeF?>nc` zUwwE_mrFp*bDYOwEI0Z^faH!X*$ zfp&$shEY`m4no^Gbk|~mzsOA#h=Fs?{$F8`Jb3-RonwU6MMK@u3C!V*?_38Up%sL# zE2OCd1GwFja}j`&x7(VorMSaHO`OXBB&@=ezdNV@-+`3AJ171tY{{uK;p!b<5a)!x zbLu~rY7TVGgDb#o1N2whQbvfL$|;G)E+Zkua{8^rK61PiWkr^=4R{lYb37=4$Pzz~qKtR{IE@3IZRtzv&mN0kBI2R^> z7OmhZy{K41v+VARb2=GY3J&Mt)KC> z%dEXX0M}&r&SeUG!|#h*A;M(N$ExQ8>2o_v{?{*Hd_RbC-%2)WZ)4Z#j z79vdRTrcr(Ds`en0D=SOdWpwLIB%K@2o_wykKsE4l?disIKkWHQaYI+v|Vv;n~ec> zGUr@9!9ng)@SW=?)SZcxw{xB|(ttpYb49~>7fT@${E66nzJg%ES@-#>IFa8jaecBL z+}!P5cHDKS-l)K5oYXm<;fCgquPxsX@2Ho&r_vKrFH-OEwd?rtZ&MO3Kl~TUeKbyl Gl>h)`PGWcf literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-thread-policy b/libkdd/tests/stackshot-sample-thread-policy new file mode 100644 index 0000000000000000000000000000000000000000..e74ba80691a8f8918ef67d30dc357a1950eef681 GIT binary patch literal 1274 zcmV{OOdDkwe+w-XVY5JSgJf_nSp+lS3KTX7&MY7xxRDL8 z#>jE&9Z)FO+7CW-$vS0|WXn>cQ!~HVhS?U2l3=1{TSkl{(||golEptnjoGqf3+jTH zP4T_=_eZ@V=911tT#28pK2rKSJY&`~kU7fN27<-t!X( z*2DOE7{4(E087;Kjqpx`>-o^WlwbuG#;z*o;JO*wPeZ&P+QV@95HZ(?z5$oQHOf9k zc5`jA&+qamoUNjWFXAh?mhK?;co6D#&Q@Ak%2(R$TysqeSHjyWsyMee;FA@nEJ&iF zh?Rp%u&}IbYk6TsVOMFVBuczku|+DBT)x0i@%Gk+rkY~Uo}S{G+P2!Joy||w?r0Gl zTgrvKVA-(B7ZKqkp&Anu7N2)OA%oX?bkb4AyS(TwoL zkB&T~;)k=_o>%eK&FyHOLFiBH$luax|05Rw9~`i~Fi8&715r8MVo-5*OGT30FS|Y7 zK3PFlu8Gu9PtXAC8I9WCRTKGQX+Wd+G#yCGBM4@QJrl%=llpbk;cvVl)*+1J%QfAB z9Ma5GFGBhpwNKnedT%n(y@#~ts{MV2q&JfOUhl{INh1BF`=NfX|8g{R@Veo|gJ zaz1H2KUO`A`xV)SFdyLgQr{B~K;P$HTKkncM-4|FM92!uR|Yk3xS}U%VGH@#pQ!gS zdD^Lg><=Z4mX3nE9}dN(#o>K4YL;~LnXB3HDYFul@<91(kL6DjmOtvZZtnrq--%ft z2*Lipz|@sW@Uz{x|C37_5Ylqa=Y2c99^B1e9=we3rCJvu{LwdX#(o4We{T?skhx^} z%Q%4fvtL-8)#H!$qn+tTB+h~@oAC1MwpYR^GXV%5X zAJ5ds^7lS9gz<@VQGE*MKSSe~JU#S_mz-n^Ox1>8r5_6 zmoq0@c9t9+D<4^SwQAl0n!iT--fAkjk@Beh=CPT_-paZ#-b(Y)r{^&Jjl7L5m#$!3 k`Tqe=Frf1XeJcK4w)~WX`MKeE`S)o1e`Xy@XRr(a02fZ3KL7v# literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-thread-policy.plist.gz b/libkdd/tests/stackshot-sample-thread-policy.plist.gz new file mode 100644 index 0000000000000000000000000000000000000000..bcd465fa9216e9bcb2039b116efdd9678d6f31ce GIT binary patch literal 2631 zcmV-N3b^$jiwFQ_??PDs1MOUGQ`|Tb{+#>@FW+JGE|~+iHVapkJ8qY%g{`|!E@K-B%GF1%@$R<(C^^Xc;QuinAoX!Of;nkA#r`SrPX zb@};^YftJJjV^vW@D9dxJv|!3m?mZYd?}S3N`a%ejt}H8 zbL~BIIIAAdNh z>Qc_>?6FQysaMM9SyEM}z>AaH=Kd6Xg(UGF;%P7m>u4P0;Uqata&R(Q$SpNLBxRmt z!8E*2f-3!A@-jf0*U5cSo)O@)lhN{DOSwrXwT8DRp{*dk8@OH!lmHN*Q}e5Qljl9)Hc7LqdFZ z$Pe%FT^z!py3l=baa_tk(Rhdp;d7rur19!Z{EsY+KgqQ+tAGS>Oa+4k zQXeyuqMMt~=N;*=EW`PaFPab;NOuMi!uZHHK0*{3A7SGo$bHiz3XS@PF6`)|#3%*K zvof1*wgNJ%%ZaUy|RVS0S`pY6}5Jjq39aaxPm50-I z*vgdiu#;n}CyR=zFB<)1>`JXIBi%?L6cWOrQUNSQU^sL~o4|^eZq^KQqyKam==}__ zX%;q7r7ZL+!!VQkCDJAenQc_1c=VH6^>(+O+Kn1!eQ#Ks#R%DqQc88NY<%cej$Psn z`$+x~Kr`Cr@kY7L7u&8o!Zx)w58Fe+njF}+tmWb}kwt!bGSQpvN% zh+bxQtDG9sozOEsQ=|Q5o*HNOqfTS7GHboTSV(9JH2MFMhg@jtuZ!IT0x%?YM%yPz z9j=LaD160%inwvLeTW6*#*oV;3X}|0av)pQ0jA7Xqrhl`Bo9b%br)UgSZjbEQP-KlQ&8_DDQg*#`$)lR zOR-Gi-#mH_z7%>{qpg$htE^{L8Yg*eSy~1F2HZIORC;SN>4`2VcNR8zG>(trm4A3S**iTdIOpSWI*<*MlXmFK|-I& zt!PtClRUO0fpMi4RHtqDwCqY!x=EVLWp886l+Rq&H(!MWOO^9 zlWgw3rb zGn?W7aChJ95!hxB^}X6A!GsRCwQtXSX%Y4%tXC0Xk2KiPo8D*>g0pU&b`0kzY5{OP zpzEZuxq&&0<%U&xW8=+e&M^dy zc+NclZHUg%3Ja!lx5L2(O9ZbUUW35Z;x(w<>&jO>u7cT_$ua~DV{1DOO9fGp*J@AXb_<5erCzH_ z2oQvlEK=nVp{uG)B;Ki7sA6|9!$R9FTOuj@y)0*fgSwV0cZ1jW(m*;Ot;2xGO^ihY$Ak-WD-ekPW8O<)O0jFh)UJhM+EPM*S^kEA5c^ zm=RrI@1nhJ^a~`TOpA7UUbLJtw7PM74D&_|h(L}s-7!ZBggD~@>w4SYo&*^)*?|Q? z2nke9!n+C_xsYkEcjiKJqg5Npn{y#dOf1%&4Iw5Zc=;Bb$i@xe_Ot?P54B#G6)6{t zG|#u60pbu+~v3(_9ST__I;D2 z_IOpoFr}H=RkbHcKORY105GCOyRIw=YcK9>PLkyI&l$wlD&a(u2t~wd(>j*4K$cW{ znd#klk`NdYBB&GM-SZ@^_TA}G3;W)P9zkDw!^wpkK@F~ZL+D^j;fI8ZDJ9qTQUODn()$*zRO@Ywb-f}Xh#z7>JPz&X17fU2T0qu*pv!o{q4#N|&cPQm z;x89UM2u&e_-F=jS5I8h=df7vFqwv9Rim011s%KP%7wu`Du_I(%i4SCVqT{58r z5$z|XU3d}XSbFk(c+n0Q?X7#qCb#V^N9c}MOep8tOG%sZBCtCzqH-~9UfG%#G0i!V z;|&x$F9N#K`MSJ_x5P-Z@wneYu}(oQKh=Kzem82g>m3uwWO=lNf-^O8yQgBMMi?Tn zp4{xI(ViN8$JA(kP{a_m_Wk$N=m(@mGyDboU9OnS@ZxJxBfz!arLieBvIgMV^P(B9 z6vm6Q?@5g`U;eYFMy}em-xuV_jXFPfzBjD@4PuK}pO?o#`EtIM89ghO3l_ETxc%O} pWLiXHRciK0dT~006D&3b6nH literal 0 HcmV?d00001 diff --git a/libkern/OSKextLib.cpp b/libkern/OSKextLib.cpp index 00264ecf3..5b9ee7b4a 100644 --- a/libkern/OSKextLib.cpp +++ b/libkern/OSKextLib.cpp @@ -439,10 +439,10 @@ void kext_dump_panic_lists(int (*printf_func)(const char * fmt, ...)) void kmod_panic_dump(vm_offset_t * addr, unsigned int cnt) { - extern int kdb_printf(const char *format, ...) __printflike(1,2); + extern int paniclog_append_noflush(const char *format, ...) __printflike(1,2); + + OSKext::printKextsInBacktrace(addr, cnt, &paniclog_append_noflush, 0); - OSKext::printKextsInBacktrace(addr, cnt, &kdb_printf, - /* takeLock? */ false, false); return; } @@ -455,7 +455,9 @@ kmod_dump_log( unsigned int cnt, boolean_t doUnslide) { - OSKext::printKextsInBacktrace(addr, cnt, &printf, /* lock? */ true, doUnslide); + uint32_t flags = OSKext::kPrintKextsLock; + if (doUnslide) flags |= OSKext::kPrintKextsUnslide; + OSKext::printKextsInBacktrace(addr, cnt, &printf, flags); } void * diff --git a/libkern/c++/OSData.cpp b/libkern/c++/OSData.cpp index a542ee603..fda3dd7c9 100644 --- a/libkern/c++/OSData.cpp +++ b/libkern/c++/OSData.cpp @@ -76,8 +76,8 @@ bool OSData::initWithCapacity(unsigned int inCapacity) if (inCapacity < page_size) data = (void *) kalloc_container(inCapacity); else { kern_return_t kr; - inCapacity = round_page_32(inCapacity); - kr = kmem_alloc(kernel_map, (vm_offset_t *)&data, inCapacity, IOMemoryTag(kernel_map)); + if (round_page_overflow(inCapacity, &inCapacity)) kr = KERN_RESOURCE_SHORTAGE; + else kr = kmem_alloc(kernel_map, (vm_offset_t *)&data, inCapacity, IOMemoryTag(kernel_map)); if (KERN_SUCCESS != kr) data = NULL; } if (!data) diff --git a/libkern/c++/OSKext.cpp b/libkern/c++/OSKext.cpp index d30dbe5b4..797fd38a2 100644 --- a/libkern/c++/OSKext.cpp +++ b/libkern/c++/OSKext.cpp @@ -71,6 +71,8 @@ extern "C" { #include #include +#include + #if PRAGMA_MARK #pragma mark External & Internal Function Protos #endif @@ -121,6 +123,9 @@ static void * GetAppleTEXTHashForKext(OSKext * theKext, OSDictionary *theInfoDic #define VM_MAPPED_KEXTS 1 #define KASLR_KEXT_DEBUG 0 #define KASLR_IOREG_DEBUG 0 +#elif __arm__ || __arm64__ +#define VM_MAPPED_KEXTS 0 +#define KASLR_KEXT_DEBUG 0 #else #error Unsupported architecture #endif @@ -314,7 +319,7 @@ kmod_info_t g_kernel_kmod_info = { /* version */ "0", // filled in in OSKext::initialize() /* reference_count */ -1, // never adjusted; kernel never unloads /* reference_list */ NULL, - /* address */ NULL, + /* address */ 0, /* size */ 0, // filled in in OSKext::initialize() /* hdr_size */ 0, /* start */ 0, @@ -681,6 +686,10 @@ OSKext::initialize(void) } PE_parse_boot_argn("keepsyms", &sKeepSymbols, sizeof(sKeepSymbols)); +#if KASAN_DYNAMIC_BLACKLIST + /* needed for function lookup */ + sKeepSymbols = true; +#endif /* Set up an OSKext instance to represent the kernel itself. */ @@ -812,6 +821,11 @@ OSKext::removeKextBootstrap(void) kernel_segment_command_t * seg_to_remove = NULL; +#if __arm__ || __arm64__ + const char * dt_segment_name = NULL; + void * segment_paddress = NULL; + int segment_size = 0; +#endif /* This must be the very first thing done by this function. */ @@ -854,7 +868,21 @@ OSKext::removeKextBootstrap(void) OSRuntimeUnloadCPPForSegment(seg_to_remove); } -#if __i386__ || __x86_64__ +#if __arm__ || __arm64__ +#if !(defined(KERNEL_INTEGRITY_KTRR)) + /* Free the memory that was set up by bootx. + */ + dt_segment_name = "Kernel-__KLD"; + if (0 == IODTGetLoaderInfo(dt_segment_name, &segment_paddress, &segment_size)) { + /* We cannot free this with KTRR enabled, as we cannot + * update the permissions on the KLD range this late + * in the boot process. + */ + IODTFreeLoaderInfo(dt_segment_name, (void *)segment_paddress, + (int)segment_size); + } +#endif /* !(defined(KERNEL_INTEGRITY_KTRR)) */ +#elif __i386__ || __x86_64__ /* On x86, use the mapping data from the segment load command to * unload KLD directly. * This may invalidate any assumptions about "avail_start" @@ -890,6 +918,9 @@ OSKext::removeKextBootstrap(void) * managed memory, then copy the segment back in. */ #if CONFIG_KXLD +#if (__arm__ || __arm64__) +#error CONFIG_KXLD not expected for this arch +#endif if (!sKeepSymbols) { kern_return_t mem_result; void *seg_copy = NULL; @@ -930,6 +961,8 @@ OSKext::removeKextBootstrap(void) &seg_offset, seg_length, /* mask */ 0, VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, (ipc_port_t)NULL, (vm_object_offset_t) 0, /* copy */ FALSE, @@ -956,7 +989,9 @@ OSKext::removeKextBootstrap(void) kmem_free(kernel_map, seg_copy_offset, seg_length); } #else /* we are not CONFIG_KXLD */ +#if !(__arm__ || __arm64__) #error CONFIG_KXLD is expected for this arch +#endif /***** * Dump the LINKEDIT segment, unless keepsyms is set. @@ -3427,8 +3462,20 @@ bool OSKext::isKextWithIdentifierLoaded(const char * kextIdentifier) OSReturn OSKext::removeKext( OSKext * aKext, +#if CONFIG_EMBEDDED + __unused +#endif bool terminateServicesAndRemovePersonalitiesFlag) { +#if CONFIG_EMBEDDED + OSKextLog(aKext, + kOSKextLogErrorLevel | + kOSKextLogKextBookkeepingFlag, + "removeKext() called for %s, not supported on embedded", + aKext->getIdentifier() ? aKext->getIdentifierCString() : "unknown kext"); + + return kOSReturnSuccess; +#else /* CONFIG_EMBEDDED */ OSReturn result = kOSKextReturnInUse; OSKext * checkKext = NULL; // do not release @@ -3524,6 +3571,7 @@ OSKext::removeKext( finish: IORecursiveLockUnlock(sKextLock); return result; +#endif /* CONFIG_EMBEDDED */ } /********************************************************************* @@ -4112,9 +4160,30 @@ finish: /********************************************************************* *********************************************************************/ +#if defined (__arm__) +#include +#endif #if defined (__x86_64__) #define ARCHNAME "x86_64" +#elif defined (__arm64__) +#define ARCHNAME "arm64" +#elif defined (__arm__) + +#if defined (__ARM_ARCH_7S__) +#define ARCHNAME "armv7s" +#elif defined (__ARM_ARCH_7F__) +#define ARCHNAME "armv7f" +#elif defined (__ARM_ARCH_7K__) +#define ARCHNAME "armv7k" +#elif defined (_ARM_ARCH_7) /* umbrella for all remaining */ +#define ARCHNAME "armv7" +#elif defined (_ARM_ARCH_6) /* umbrella for all armv6 */ +#define ARCHNAME "armv6" +#endif + +#elif defined (__arm64__) +#define ARCHNAME "arm64" #else #error architecture not supported #endif @@ -4700,6 +4769,7 @@ OSKext::load( } bzero(account, sizeof(*account)); account->loadTag = kmod_info->id; + account->site.refcount = 0; account->site.flags = VM_TAG_KMOD; account->kext = this; @@ -5404,6 +5474,21 @@ register_kmod: goto finish; } +#if KASAN + kasan_load_kext((vm_offset_t)linkedExecutable->getBytesNoCopy(), + linkedExecutable->getLength(), getIdentifierCString()); +#else + if (lookupSection(KASAN_GLOBAL_SEGNAME, KASAN_GLOBAL_SECTNAME)) { + OSKextLog(this, + kOSKextLogErrorLevel | kOSKextLogLoadFlag, + "KASAN: cannot load KASAN-ified kext %s on a non-KASAN kernel\n", + getIdentifierCString() + ); + result = KERN_FAILURE; + goto finish; + } +#endif + result = kOSReturnSuccess; finish: @@ -5679,7 +5764,40 @@ OSKext::unregisterWithDTrace(void) * called only by loadExecutable() *********************************************************************/ #if !VM_MAPPED_KEXTS +#if defined(__arm__) || defined(__arm64__) +static inline kern_return_t +OSKext_protect( + vm_map_t map, + vm_map_offset_t start, + vm_map_offset_t end, + vm_prot_t new_prot, + boolean_t set_max) +{ +#pragma unused(map) + assert(map == kernel_map); // we can handle KEXTs arising from the PRELINK segment and no others + assert(start <= end); + if (start >= end) + return KERN_SUCCESS; // Punt segments of length zero (e.g., headers) or less (i.e., blunders) + else if (set_max) + return KERN_SUCCESS; // Punt set_max, as there's no mechanism to record that state + else + return ml_static_protect(start, end - start, new_prot); +} + +static inline kern_return_t +OSKext_wire( + vm_map_t map, + vm_map_offset_t start, + vm_map_offset_t end, + vm_prot_t access_type, + boolean_t user_wire) +{ +#pragma unused(map,start,end,access_type,user_wire) + return KERN_SUCCESS; // No-op as PRELINK kexts are cemented into physical memory at boot +} +#else #error Unrecognized architecture +#endif #else static inline kern_return_t OSKext_protect( @@ -5703,7 +5821,7 @@ OSKext_wire( vm_prot_t access_type, boolean_t user_wire) { - return vm_map_wire(map, start, end, access_type | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_KEXT), user_wire); + return vm_map_wire_kernel(map, start, end, access_type, VM_KERN_MEMORY_KEXT, user_wire); } #endif @@ -5754,6 +5872,11 @@ OSKext::setVMAttributes(bool protect, bool wire) seg = firstsegfromheader((kernel_mach_header_t *)kmod_info->address); while (seg) { +#if __arm__ + /* We build all ARM kexts, so we can ensure they are aligned */ + assert((seg->vmaddr & PAGE_MASK) == 0); + assert((seg->vmsize & PAGE_MASK) == 0); +#endif start = round_page(seg->vmaddr); end = trunc_page(seg->vmaddr + seg->vmsize); @@ -6332,7 +6455,17 @@ OSKext::unload(void) goto finish; } - if (hasOSMetaClassInstances()) { + if (!isLoaded()) { + result = kOSReturnSuccess; + goto finish; + } + + if (isKernelComponent()) { + result = kOSKextReturnInvalidArgument; + goto finish; + } + + if (metaClasses && !OSMetaClass::removeClasses(metaClasses)) { OSKextLog(this, kOSKextLogErrorLevel | kOSKextLogLoadFlag | kOSKextLogKextBookkeepingFlag, @@ -6343,16 +6476,6 @@ OSKext::unload(void) result = kOSKextReturnInUse; goto finish; } - - if (!isLoaded()) { - result = kOSReturnSuccess; - goto finish; - } - - if (isKernelComponent()) { - result = kOSKextReturnInvalidArgument; - goto finish; - } /* Note that the kext is unloading before running any code that * might be in the kext (request callbacks, module stop function). @@ -6474,6 +6597,10 @@ OSKext::unload(void) /* Unwire and free the linked executable. */ if (linkedExecutable) { +#if KASAN + kasan_unload_kext((vm_offset_t)linkedExecutable->getBytesNoCopy(), linkedExecutable->getLength()); +#endif + #if VM_MAPPED_KEXTS if (!isInterface()) { kernel_segment_command_t *seg = NULL; @@ -8234,7 +8361,6 @@ size_t OSKextPgoMetadataSize(OSKext *kext) return position; } - int OSKextGrabPgoDataLocked(OSKext *kext, bool metadata, uuid_t instance_uuid, @@ -8242,7 +8368,6 @@ int OSKextGrabPgoDataLocked(OSKext *kext, char *pBuffer, uint64_t bufferSize) { - int err = 0; kernel_section_t *sect_prf_data = NULL; @@ -8767,6 +8892,7 @@ OSKext::copyInfo(OSArray * infoKeys) segp->filesize = 0; } } + #if 0 OSKextLog(/* kext */ NULL, kOSKextLogErrorLevel | @@ -8788,7 +8914,7 @@ OSKext::copyInfo(OSArray * infoKeys) } #endif segp->vmaddr = VM_KERNEL_UNSLIDE(segp->vmaddr); - + for (secp = firstsect(segp); secp != NULL; secp = nextsect(segp, secp)) { secp->addr = VM_KERNEL_UNSLIDE(secp->addr); } @@ -9472,6 +9598,17 @@ finish: return result; } + +/********************************************************************* +* Busy timeout triage +*********************************************************************/ +/* static */ +bool +OSKext::isWaitingKextd(void) +{ + return sRequestCallbackRecords && sRequestCallbackRecords->getCount(); +} + /********************************************************************* * Assumes sKextLock is held. *********************************************************************/ @@ -10741,15 +10878,14 @@ OSKext::printKextsInBacktrace( vm_offset_t * addr, unsigned int cnt, int (* printf_func)(const char *fmt, ...), - bool lockFlag, - bool doUnslide) + uint32_t flags) { addr64_t summary_page = 0; addr64_t last_summary_page = 0; bool found_kmod = false; u_int i = 0; - if (lockFlag) { + if (kPrintKextsLock & flags) { if (!sKextSummariesLock) return; IOLockLock(sKextSummariesLock); } @@ -10782,15 +10918,17 @@ OSKext::printKextsInBacktrace( } if (!found_kmod) { - (*printf_func)(" Kernel Extensions in backtrace:\n"); + if (!(kPrintKextsTerse & flags)) { + (*printf_func)(" Kernel Extensions in backtrace:\n"); + } found_kmod = true; } - printSummary(summary, printf_func, doUnslide); + printSummary(summary, printf_func, flags); } finish: - if (lockFlag) { + if (kPrintKextsLock & flags) { IOLockUnlock(sKextSummariesLock); } @@ -10919,7 +11057,7 @@ static void findSummaryUUID( void OSKext::printSummary( OSKextLoadedKextSummary * summary, int (* printf_func)(const char *fmt, ...), - bool doUnslide) + uint32_t flags) { kmod_reference_t * kmod_ref = NULL; uuid_string_t uuid; @@ -10931,15 +11069,18 @@ void OSKext::printSummary( } (void) uuid_unparse(summary->uuid, uuid); - if (doUnslide) { + if (kPrintKextsUnslide & flags) { tmpAddr = VM_KERNEL_UNSLIDE(summary->address); } else { tmpAddr = summary->address; } - (*printf_func)(" %s(%s)[%s]@0x%llx->0x%llx\n", + (*printf_func)("%s%s(%s)[%s]@0x%llx->0x%llx\n", + (kPrintKextsTerse & flags) ? "" : " ", summary->name, version, uuid, tmpAddr, tmpAddr + summary->size - 1); + + if (kPrintKextsTerse & flags) return; /* print dependency info */ for (kmod_ref = (kmod_reference_t *) summary->reference_list; @@ -10950,7 +11091,7 @@ void OSKext::printSummary( if (pmap_find_phys(kernel_pmap, (addr64_t)((uintptr_t)kmod_ref)) == 0) { (*printf_func)(" kmod dependency scan stopped " "due to missing dependency page: %p\n", - doUnslide ? (void *)VM_KERNEL_UNSLIDE(kmod_ref) : kmod_ref); + (kPrintKextsUnslide & flags) ? (void *)VM_KERNEL_UNSLIDE(kmod_ref) : kmod_ref); break; } rinfo = kmod_ref->info; @@ -10958,7 +11099,7 @@ void OSKext::printSummary( if (pmap_find_phys(kernel_pmap, (addr64_t)((uintptr_t)rinfo)) == 0) { (*printf_func)(" kmod dependency scan stopped " "due to missing kmod page: %p\n", - doUnslide ? (void *)VM_KERNEL_UNSLIDE(rinfo) : rinfo); + (kPrintKextsUnslide & flags) ? (void *)VM_KERNEL_UNSLIDE(rinfo) : rinfo); break; } @@ -10969,7 +11110,7 @@ void OSKext::printSummary( /* locate UUID in gLoadedKextSummaries */ findSummaryUUID(rinfo->id, uuid); - if (doUnslide) { + if (kPrintKextsUnslide & flags) { tmpAddr = VM_KERNEL_UNSLIDE(rinfo->address); } else { @@ -11597,11 +11738,14 @@ OSKextGetAllocationSiteForCaller(uintptr_t address) { OSKextActiveAccount * active; vm_allocation_site_t * site; + vm_allocation_site_t * releasesite; + uint32_t baseIdx; uint32_t lim; IOSimpleLockLock(sKextAccountsLock); - site = NULL; + site = releasesite = NULL; + // bsearch sKextAccounts list for (baseIdx = 0, lim = sKextAccountsCount; lim; lim >>= 1) { @@ -11609,7 +11753,7 @@ OSKextGetAllocationSiteForCaller(uintptr_t address) if ((address >= active->address) && (address < active->address_end)) { site = &active->account->site; - if (!site->tag) vm_tag_alloc_locked(site); + if (!site->tag) vm_tag_alloc_locked(site, &releasesite); break; } else if (address > active->address) @@ -11621,12 +11765,13 @@ OSKextGetAllocationSiteForCaller(uintptr_t address) // else move left } IOSimpleLockUnlock(sKextAccountsLock); + if (releasesite) kern_allocation_name_release(releasesite); return (site); } extern "C" uint32_t -OSKextGetKmodIDForSite(vm_allocation_site_t * site, char * name, vm_size_t namelen) +OSKextGetKmodIDForSite(const vm_allocation_site_t * site, char * name, vm_size_t namelen) { OSKextAccount * account = (typeof(account)) site; const char * kname; diff --git a/libkern/c++/OSMetaClass.cpp b/libkern/c++/OSMetaClass.cpp index 5e25aa5fd..3d7c2f6e4 100644 --- a/libkern/c++/OSMetaClass.cpp +++ b/libkern/c++/OSMetaClass.cpp @@ -110,6 +110,7 @@ IOLock * sStalledClassesLock = NULL; struct ExpansionData { OSOrderedSet * instances; OSKext * kext; + uint32_t retain; #if IOTRACKING IOTrackingQueue * tracking; #endif @@ -656,7 +657,11 @@ OSMetaClass::postModLoad(void * loadHandle) /* Log this error here so we can include the class name. * xxx - we should look up the other kext that defines the class */ +#if CONFIG_EMBEDDED + panic( +#else OSKextLog(myKext, kOSMetaClassLogSpec, +#endif /* CONFIG_EMBEDDED */ "OSMetaClass: Kext %s class %s is a duplicate;" "kext %s already has a class by that name.", sStalled->kextIdentifier, (const char *)me->className, @@ -944,6 +949,43 @@ OSMetaClass::considerUnloads() OSKext::considerUnloads(); } +/********************************************************************* +*********************************************************************/ +bool +OSMetaClass::removeClasses(OSCollection * metaClasses) +{ + OSCollectionIterator * classIterator; + OSMetaClass * checkClass; + bool result; + + classIterator = OSCollectionIterator::withCollection(metaClasses); + if (!classIterator) return (false); + + IOLockLock(sAllClassesLock); + + result = false; + do + { + while ((checkClass = (OSMetaClass *)classIterator->getNextObject()) + && !checkClass->getInstanceCount() + && !checkClass->reserved->retain) {} + if (checkClass) break; + classIterator->reset(); + while ((checkClass = (OSMetaClass *)classIterator->getNextObject())) + { + sAllClassesDict->removeObject(checkClass->className); + } + result = true; + } + while (false); + + IOLockUnlock(sAllClassesLock); + OSSafeReleaseNULL(classIterator); + + return (result); +} + + /********************************************************************* *********************************************************************/ const OSMetaClass * @@ -964,17 +1006,48 @@ OSMetaClass::getMetaClassWithName(const OSSymbol * name) return retMeta; } +/********************************************************************* +*********************************************************************/ +const OSMetaClass * +OSMetaClass::copyMetaClassWithName(const OSSymbol * name) +{ + const OSMetaClass * meta; + + if (!name) return (0); + + meta = 0; + IOLockLock(sAllClassesLock); + if (sAllClassesDict) { + meta = (OSMetaClass *) sAllClassesDict->getObject(name); + if (meta) OSIncrementAtomic(&meta->reserved->retain); + } + IOLockUnlock(sAllClassesLock); + + return (meta); +} + +/********************************************************************* +*********************************************************************/ +void +OSMetaClass::releaseMetaClass() const +{ + OSDecrementAtomic(&reserved->retain); +} + /********************************************************************* *********************************************************************/ OSObject * OSMetaClass::allocClassWithName(const OSSymbol * name) { - OSObject * result = 0; - - const OSMetaClass * const meta = getMetaClassWithName(name); + const OSMetaClass * meta; + OSObject * result; - if (meta) { + result = 0; + meta = copyMetaClassWithName(name); + if (meta) + { result = meta->alloc(); + meta->releaseMetaClass(); } return result; @@ -1246,7 +1319,7 @@ void OSMetaClass::trackedInstance(OSObject * instance) const { IOTracking * mem = (typeof(mem)) instance; mem--; - return (IOTrackingAdd(reserved->tracking, mem, classSize, false)); + return (IOTrackingAdd(reserved->tracking, mem, classSize, false, VM_KERN_MEMORY_NONE)); } void OSMetaClass::trackedFree(OSObject * instance) const diff --git a/libkern/c++/OSUnserializeXML.cpp b/libkern/c++/OSUnserializeXML.cpp index 7df203dc7..e44bee927 100644 --- a/libkern/c++/OSUnserializeXML.cpp +++ b/libkern/c++/OSUnserializeXML.cpp @@ -159,7 +159,8 @@ #include #include -#define MAX_OBJECTS 65535 +#define MAX_OBJECTS 131071 +#define MAX_REFED_OBJECTS 65535 #define YYSTYPE object_t * #define YYPARSE_PARAM state @@ -192,6 +193,7 @@ typedef struct parser_state { OSString **errorString; // parse error with line OSObject *parsedObject; // resultant object of parsed text int parsedObjectCount; + int retrievedObjectCount; } parser_state_t; #define STATE ((parser_state_t *)state) @@ -1632,6 +1634,11 @@ yyreduce: #line 246 "OSUnserializeXML.y" { (yyval) = retrieveObject(STATE, (yyvsp[(1) - (1)])->idref); if ((yyval)) { + STATE->retrievedObjectCount++; + if (STATE->retrievedObjectCount > MAX_REFED_OBJECTS) { + yyerror("maximum object reference count"); + YYERROR; + } (yyval)->object->retain(); } else { yyerror("forward reference detected"); @@ -2835,6 +2842,7 @@ OSUnserializeXML(const char *buffer, OSString **errorString) state->errorString = errorString; state->parsedObject = 0; state->parsedObjectCount = 0; + state->retrievedObjectCount = 0; (void)yyparse((void *)state); diff --git a/libkern/c++/Tests/TestSerialization/test1/test1_main.cpp b/libkern/c++/Tests/TestSerialization/test1/test1_main.cpp old mode 100755 new mode 100644 diff --git a/libkern/conf/Makefile.arm b/libkern/conf/Makefile.arm new file mode 100644 index 000000000..04938ae54 --- /dev/null +++ b/libkern/conf/Makefile.arm @@ -0,0 +1,20 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# The following files cast opaque pointers to more specific +# structures +OBJS_NO_CAST_ALIGN = kxld_kext.o kxld_reloc.o kxld_sect.o kxld_seg.o \ + kxld_state.o kxld_sym.o kxld_symtab.o kxld_util.o \ + kxld_srcversion.o kxld_splitinfolc.o kxld_uuid.o kxld_vtable.o uuid.o + +$(foreach file,$(OBJS_NO_CAST_ALIGN),$(eval $(call add_perfile_cflags,$(file),-Wno-cast-align))) + +OSKext.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align -Wno-error=shadow +OSMetaClass.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +OSKextLib.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +OSUnserialize.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/libkern/conf/Makefile.arm64 b/libkern/conf/Makefile.arm64 new file mode 100644 index 000000000..04938ae54 --- /dev/null +++ b/libkern/conf/Makefile.arm64 @@ -0,0 +1,20 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# The following files cast opaque pointers to more specific +# structures +OBJS_NO_CAST_ALIGN = kxld_kext.o kxld_reloc.o kxld_sect.o kxld_seg.o \ + kxld_state.o kxld_sym.o kxld_symtab.o kxld_util.o \ + kxld_srcversion.o kxld_splitinfolc.o kxld_uuid.o kxld_vtable.o uuid.o + +$(foreach file,$(OBJS_NO_CAST_ALIGN),$(eval $(call add_perfile_cflags,$(file),-Wno-cast-align))) + +OSKext.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align -Wno-error=shadow +OSMetaClass.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +OSKextLib.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +OSUnserialize.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/libkern/conf/Makefile.template b/libkern/conf/Makefile.template index f434fc023..5291268bc 100644 --- a/libkern/conf/Makefile.template +++ b/libkern/conf/Makefile.template @@ -17,6 +17,7 @@ include $(MakeInc_def) # CFLAGS # CFLAGS+= -include meta_features.h -DLIBKERN_KERNEL_PRIVATE -DOSALLOCDEBUG=1 +SFLAGS+= -include meta_features.h # Objects that don't want -Wcast-align warning (8474835) OSKextLib.cpo_CXXWARNFLAGS_ADD = -Wno-cast-align diff --git a/libkern/conf/files b/libkern/conf/files index 6f4f78550..5867b9373 100644 --- a/libkern/conf/files +++ b/libkern/conf/files @@ -61,7 +61,6 @@ libkern/zlib/adler32.c optional zlib libkern/zlib/compress.c optional zlib libkern/zlib/crc32.c optional zlib libkern/zlib/deflate.c optional zlib -#libkern/zlib/gzio.c not needed for kernel optional zlib libkern/zlib/infback.c optional zlib libkern/zlib/inffast.c optional zlib libkern/zlib/inflate.c optional zlib @@ -79,6 +78,7 @@ libkern/crypto/corecrypto_aes.c optional crypto libkern/crypto/corecrypto_aesxts.c optional crypto libkern/crypto/corecrypto_rand.c optional crypto libkern/crypto/corecrypto_rsa.c optional crypto +libkern/crypto/corecrypto_chacha20poly1305.c optional crypto libkern/stack_protector.c standard diff --git a/libkern/conf/files.arm b/libkern/conf/files.arm new file mode 100644 index 000000000..28a08b01f --- /dev/null +++ b/libkern/conf/files.arm @@ -0,0 +1,3 @@ +libkern/zlib/arm/inffastS.s optional zlib +libkern/zlib/arm/adler32vec.s optional zlib + diff --git a/libkern/crypto/corecrypto_chacha20poly1305.c b/libkern/crypto/corecrypto_chacha20poly1305.c new file mode 100644 index 000000000..8957b0708 --- /dev/null +++ b/libkern/crypto/corecrypto_chacha20poly1305.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include + +static ccchacha20poly1305_fns_t fns(void) +{ + return g_crypto_funcs->ccchacha20poly1305_fns; +} + +static const struct ccchacha20poly1305_info *info(void) +{ + return fns()->info(); +} + +int chacha20poly1305_init(chacha20poly1305_ctx *ctx, const uint8_t *key) +{ + return fns()->init(info(), ctx, key); +} + +int chacha20poly1305_reset(chacha20poly1305_ctx *ctx) +{ + return fns()->reset(info(), ctx); +} + +int chacha20poly1305_setnonce(chacha20poly1305_ctx *ctx, const uint8_t *nonce) +{ + return fns()->setnonce(info(), ctx, nonce); +} + +int chacha20poly1305_incnonce(chacha20poly1305_ctx *ctx, uint8_t *nonce) +{ + return fns()->incnonce(info(), ctx, nonce); +} + +int chacha20poly1305_aad(chacha20poly1305_ctx *ctx, size_t nbytes, const void *aad) +{ + return fns()->aad(info(), ctx, nbytes, aad); +} + +int chacha20poly1305_encrypt(chacha20poly1305_ctx *ctx, size_t nbytes, const void *ptext, void *ctext) +{ + return fns()->encrypt(info(), ctx, nbytes, ptext, ctext); +} + +int chacha20poly1305_finalize(chacha20poly1305_ctx *ctx, uint8_t *tag) +{ + return fns()->finalize(info(), ctx, tag); +} + +int chacha20poly1305_decrypt(chacha20poly1305_ctx *ctx, size_t nbytes, const void *ctext, void *ptext) +{ + return fns()->decrypt(info(), ctx, nbytes, ctext, ptext); +} + +int chacha20poly1305_verify(chacha20poly1305_ctx *ctx, const uint8_t *tag) +{ + return fns()->verify(info(), ctx, tag); +} diff --git a/libkern/firehose/chunk_private.h b/libkern/firehose/chunk_private.h index ca5fe069a..b4fbcd74a 100644 --- a/libkern/firehose/chunk_private.h +++ b/libkern/firehose/chunk_private.h @@ -52,7 +52,8 @@ typedef union { uint8_t fcp_stream; uint8_t fcp_flag_full : 1; uint8_t fcp_flag_io : 1; - uint8_t _fcp_flag_unused : 6; + uint8_t fcp_quarantined : 1; + uint8_t _fcp_flag_unused : 5; }; } firehose_chunk_pos_u; diff --git a/libkern/firehose/firehose_types_private.h b/libkern/firehose/firehose_types_private.h index 785ac177d..ea1f91279 100644 --- a/libkern/firehose/firehose_types_private.h +++ b/libkern/firehose/firehose_types_private.h @@ -91,6 +91,7 @@ OS_ENUM(firehose_tracepoint_namespace, uint8_t, firehose_tracepoint_namespace_trace = 0x03, firehose_tracepoint_namespace_log = 0x04, firehose_tracepoint_namespace_metadata = 0x05, + firehose_tracepoint_namespace_signpost = 0x06, ); /*! @@ -130,13 +131,6 @@ OS_ENUM(firehose_tracepoint_flags, uint16_t, _firehose_tracepoint_flags_pc_style__unused6 = 0x0006 << 1, _firehose_tracepoint_flags_pc_style__unused7 = 0x0007 << 1, _firehose_tracepoint_flags_base_has_unique_pid = 0x0010, - - _firehose_tracepoint_flags_base_main_executable __deprecated = - _firehose_tracepoint_flags_pc_style_main_exe, - _firehose_tracepoint_flags_base_shared_cache __deprecated = - _firehose_tracepoint_flags_pc_style_shared_cache, - _firehose_tracepoint_flags_base_caller_pc __deprecated = - _firehose_tracepoint_flags_pc_style_absolute, ); /*! @@ -207,8 +201,8 @@ OS_ENUM(_firehose_tracepoint_type_log, firehose_tracepoint_type_t, OS_ENUM(_firehose_tracepoint_flags_log, uint16_t, _firehose_tracepoint_flags_log_has_private_data = 0x0100, _firehose_tracepoint_flags_log_has_subsystem = 0x0200, - _firehose_tracepoint_flags_log_has_rules = 0x0400, - _firehose_tracepoint_flags_log_has_oversize = 0x0800, + _firehose_tracepoint_flags_log_has_rules = 0x0400, + _firehose_tracepoint_flags_log_has_oversize = 0x0800, ); /*! @@ -223,6 +217,36 @@ OS_ENUM(_firehose_tracepoint_type_metadata, firehose_tracepoint_type_t, _firehose_tracepoint_type_metadata_kext = 0x03, ); +/*! + * @enum firehose_tracepoint_type_signpost_t + * + * @abstract + * Types of Log tracepoints (namespace signpost). + */ +OS_ENUM(_firehose_tracepoint_type_signpost, firehose_tracepoint_type_t, + _firehose_tracepoint_type_signpost_event = 0x00, + _firehose_tracepoint_type_signpost_interval_begin = 0x01, + _firehose_tracepoint_type_signpost_interval_end = 0x02, + + _firehose_tracepoint_type_signpost_scope_mask = 0xc0, + _firehose_tracepoint_type_signpost_scope_thread = 0x40, + _firehose_tracepoint_type_signpost_scope_process = 0x80, + _firehose_tracepoint_type_signpost_scope_system = 0xc0, +); + +/*! + * @enum firehose_tracepoint_flags_signpost_t + * + * @abstract + * Flags for Log tracepoints (namespace signpost). + */ +OS_ENUM(_firehose_tracepoint_flags_signpost, uint16_t, + _firehose_tracepoint_flags_signpost_has_private_data = 0x0100, + _firehose_tracepoint_flags_signpost_has_subsystem = 0x0200, + _firehose_tracepoint_flags_signpost_has_rules = 0x0400, + _firehose_tracepoint_flags_signpost_has_oversize = 0x0800, +); + /* MIG firehose push reply structure */ typedef struct firehose_push_reply_s { uint64_t fpr_mem_flushed_pos; diff --git a/libkern/gen/OSDebug.cpp b/libkern/gen/OSDebug.cpp index 305cfc3cb..f11263631 100644 --- a/libkern/gen/OSDebug.cpp +++ b/libkern/gen/OSDebug.cpp @@ -56,6 +56,11 @@ extern boolean_t doprnt_hide_pointers; extern void kmod_dump_log(vm_offset_t *addr, unsigned int cnt, boolean_t doUnslide); extern addr64_t kvtophys(vm_offset_t va); +#if __arm__ +extern int copyinframe(vm_address_t fp, char *frame); +#elif defined(__arm64__) +extern int copyinframe(vm_address_t fp, char *frame, boolean_t is64bit); +#endif __END_DECLS @@ -96,7 +101,7 @@ void OSReportWithBacktrace(const char *str, ...) { char buf[128]; - void *bt[9]; + void *bt[9] = {}; const unsigned cnt = sizeof(bt) / sizeof(bt[0]); va_list listp; @@ -217,6 +222,42 @@ pad: for ( ; frame_index < maxAddrs; frame_index++) bt[frame_index] = (void *) 0; +#elif __arm__ || __arm64__ + uint32_t i = 0; + uintptr_t frameb[2]; + uintptr_t fp = 0; + + // get the current frame pointer for this thread +#if defined(__arm__) +#define OSBacktraceFrameAlignOK(x) (((x) & 0x3) == 0) + __asm__ volatile("mov %0,r7" : "=r" (fp)); +#elif defined(__arm64__) +#define OSBacktraceFrameAlignOK(x) (((x) & 0xf) == 0) + __asm__ volatile("mov %0, fp" : "=r" (fp)); +#else +#error Unknown architecture. +#endif + + // now crawl up the stack recording the link value of each frame + do { + // check bounds + if ((fp == 0) || (!OSBacktraceFrameAlignOK(fp)) || (fp > VM_MAX_KERNEL_ADDRESS) || (fp < VM_MIN_KERNEL_AND_KEXT_ADDRESS)) { + break; + } + // safely read frame +#ifdef __arm64__ + if (copyinframe(fp, (char*)frameb, TRUE) != 0) { +#else + if (copyinframe(fp, (char*)frameb) != 0) { +#endif + break; + } + + // No need to use copyin as this is always a kernel address, see check above + bt[i] = (void*)frameb[1]; // link register + fp = frameb[0]; + } while (++i < maxAddrs); + frame= i; #else #error arch #endif diff --git a/libkern/kmod/libkmodtest/libkmodtest.h b/libkern/kmod/libkmodtest/libkmodtest.h index cd0eb4401..3ce25ed12 100644 --- a/libkern/kmod/libkmodtest/libkmodtest.h +++ b/libkern/kmod/libkmodtest/libkmodtest.h @@ -36,4 +36,4 @@ class testlibkmod : public IOService { IOService *provider, SInt32 *score ); -}; \ No newline at end of file +}; diff --git a/libkern/kxld/kxld_object.c b/libkern/kxld/kxld_object.c index f8fc526bc..3413a2ce7 100644 --- a/libkern/kxld/kxld_object.c +++ b/libkern/kxld/kxld_object.c @@ -48,6 +48,7 @@ #include #include #include +#include #define DEBUG_ASSERT_COMPONENT_NAME_STRING "kxld" #include @@ -384,6 +385,14 @@ get_target_machine_info(KXLDObject *object, cpu_type_t cputype __unused, object->cpusubtype = CPU_SUBTYPE_X86_64_ALL; #endif return KERN_SUCCESS; +#elif defined(__arm__) + object->cputype = CPU_TYPE_ARM; + object->cpusubtype = CPU_SUBTYPE_ARM_ALL; + return KERN_SUCCESS; +#elif defined(__arm64__) + object->cputype = CPU_TYPE_ARM64; + object->cpusubtype = CPU_SUBTYPE_ARM64_ALL; + return KERN_SUCCESS; #else kxld_log(kKxldLogLinking, kKxldLogErr, kKxldLogArchNotSupported, _mh_execute_header->cputype); @@ -495,8 +504,10 @@ get_macho_slice_for_arch(KXLDObject *object, u_char *file, u_long size) if (fat->magic == FAT_MAGIC) { struct fat_arch *arch = NULL; + u_long arch_size; + boolean_t ovr = os_mul_and_add_overflow(fat->nfat_arch, sizeof(*archs), sizeof(*fat), &arch_size); - require_action(size >= (sizeof(*fat) + (fat->nfat_arch * sizeof(*archs))), + require_action(!ovr && size >= arch_size, finish, rval=KERN_FAILURE; kxld_log(kKxldLogLinking, kKxldLogErr, kKxldLogTruncatedMachO)); @@ -569,6 +580,7 @@ init_from_final_linked_image(KXLDObject *object, u_int *filetype_out, struct symtab_command *symtab_hdr = NULL; struct uuid_command *uuid_hdr = NULL; struct version_min_command *versionmin_hdr = NULL; + struct build_version_command *build_version_hdr = NULL; struct source_version_command *source_version_hdr = NULL; u_long base_offset = 0; u_long offset = 0; @@ -702,6 +714,10 @@ init_from_final_linked_image(KXLDObject *object, u_int *filetype_out, versionmin_hdr = (struct version_min_command *) cmd_hdr; kxld_versionmin_init_from_macho(&object->versionmin, versionmin_hdr); break; + case LC_BUILD_VERSION: + build_version_hdr = (struct build_version_command *)cmd_hdr; + kxld_versionmin_init_from_build_cmd(&object->versionmin, build_version_hdr); + break; case LC_SOURCE_VERSION: source_version_hdr = (struct source_version_command *) (void *) cmd_hdr; kxld_srcversion_init_from_macho(&object->srcversion, source_version_hdr); @@ -734,6 +750,9 @@ init_from_final_linked_image(KXLDObject *object, u_int *filetype_out, kxld_splitinfolc_init_from_macho(&object->splitinfolc, split_info_hdr); } break; + case LC_NOTE: + /* binary blob of data */ + break; case LC_CODE_SIGNATURE: case LC_DYLD_INFO: case LC_DYLD_INFO_ONLY: @@ -1052,6 +1071,15 @@ init_from_object(KXLDObject *object) case LC_DYLIB_CODE_SIGN_DRS: /* Various metadata that might be stored in the linkedit segment */ break; + case LC_NOTE: + /* bag-of-bits carried with the binary: ignore */ + break; + case LC_BUILD_VERSION: + /* should be able to ignore build version commands */ + kxld_log(kKxldLogLinking, kKxldLogWarn, + "Ignoring LC_BUILD_VERSION (%u) in MH_OBJECT kext: (platform:%d)", + cmd_hdr->cmd, ((struct build_version_command *)cmd_hdr)->platform); + break; case LC_VERSION_MIN_MACOSX: case LC_VERSION_MIN_IPHONEOS: case LC_VERSION_MIN_TVOS: @@ -1180,7 +1208,7 @@ get_macho_header_size(const KXLDObject *object) } if (object->versionmin.has_versionmin) { - header_size += kxld_versionmin_get_macho_header_size(); + header_size += kxld_versionmin_get_macho_header_size(&object->versionmin); } if (object->srcversion.has_srcversion) { diff --git a/libkern/kxld/kxld_util.c b/libkern/kxld/kxld_util.c index eee5dee72..480a454c4 100644 --- a/libkern/kxld/kxld_util.c +++ b/libkern/kxld/kxld_util.c @@ -784,7 +784,8 @@ kxld_strstr(const char *s, const char *find) #if KERNEL char c, sc; size_t len; - + if (!s || !find) + return s; if ((c = *find++) != 0) { len = strlen(find); do { diff --git a/libkern/kxld/kxld_util.h b/libkern/kxld/kxld_util.h index 5b55b4d8b..d8be6faef 100644 --- a/libkern/kxld/kxld_util.h +++ b/libkern/kxld/kxld_util.h @@ -196,7 +196,7 @@ boolean_t kxld_is_32_bit(cpu_type_t) __attribute__((const, visibility("hidden"))); const char * kxld_strstr(const char *s, const char *find) - __attribute__((pure, nonnull, visibility("hidden"))); + __attribute__((pure, visibility("hidden"))); /******************************************************************************* * Debugging diff --git a/libkern/kxld/kxld_versionmin.c b/libkern/kxld/kxld_versionmin.c index abbfaed6a..36c22203f 100644 --- a/libkern/kxld/kxld_versionmin.c +++ b/libkern/kxld/kxld_versionmin.c @@ -63,6 +63,31 @@ kxld_versionmin_init_from_macho(KXLDversionmin *versionmin, struct version_min_c versionmin->has_versionmin = TRUE; } +void +kxld_versionmin_init_from_build_cmd(KXLDversionmin *versionmin, struct build_version_command *src) +{ + check(versionmin); + check(src); + switch (src->platform) { + case PLATFORM_MACOS: + versionmin->platform = kKxldVersionMinMacOSX; + break; + case PLATFORM_IOS: + versionmin->platform = kKxldVersionMiniPhoneOS; + break; + case PLATFORM_TVOS: + versionmin->platform = kKxldVersionMinAppleTVOS; + break; + case PLATFORM_WATCHOS: + versionmin->platform = kKxldVersionMinWatchOS; + break; + default: + return; + } + versionmin->version = src->minos; + versionmin->has_versionmin = TRUE; +} + /******************************************************************************* *******************************************************************************/ void @@ -74,8 +99,9 @@ kxld_versionmin_clear(KXLDversionmin *versionmin) /******************************************************************************* *******************************************************************************/ u_long -kxld_versionmin_get_macho_header_size(void) +kxld_versionmin_get_macho_header_size(__unused const KXLDversionmin *versionmin) { + /* TODO: eventually we can just use struct build_version_command */ return sizeof(struct version_min_command); } @@ -92,6 +118,7 @@ kxld_versionmin_export_macho(const KXLDversionmin *versionmin, u_char *buf, check(buf); check(header_offset); + require_action(sizeof(*versionminhdr) <= header_size - *header_offset, finish, rval=KERN_FAILURE); versionminhdr = (struct version_min_command *) ((void *) (buf + *header_offset)); @@ -111,6 +138,8 @@ kxld_versionmin_export_macho(const KXLDversionmin *versionmin, u_char *buf, case kKxldVersionMinWatchOS: versionminhdr->cmd = LC_VERSION_MIN_WATCHOS; break; + default: + goto finish; } versionminhdr->cmdsize = (uint32_t) sizeof(*versionminhdr); versionminhdr->version = versionmin->version; diff --git a/libkern/kxld/kxld_versionmin.h b/libkern/kxld/kxld_versionmin.h index ff9c02124..8b3df067e 100644 --- a/libkern/kxld/kxld_versionmin.h +++ b/libkern/kxld/kxld_versionmin.h @@ -58,6 +58,9 @@ struct kxld_versionmin { void kxld_versionmin_init_from_macho(KXLDversionmin *versionmin, struct version_min_command *src) __attribute__((nonnull, visibility("hidden"))); +void kxld_versionmin_init_from_build_cmd(KXLDversionmin *versionmin, struct build_version_command *src) + __attribute__((nonnull, visibility("hidden"))); + void kxld_versionmin_clear(KXLDversionmin *versionmin) __attribute__((nonnull, visibility("hidden"))); @@ -65,7 +68,7 @@ void kxld_versionmin_clear(KXLDversionmin *versionmin) * Accessors *******************************************************************************/ -u_long kxld_versionmin_get_macho_header_size(void) +u_long kxld_versionmin_get_macho_header_size(const KXLDversionmin *versionmin) __attribute__((pure, visibility("hidden"))); kern_return_t diff --git a/libkern/kxld/tests/loadtest.py b/libkern/kxld/tests/loadtest.py old mode 100644 new mode 100755 diff --git a/libkern/libkern/OSAtomic.h b/libkern/libkern/OSAtomic.h index f4a2a6736..76b945443 100644 --- a/libkern/libkern/OSAtomic.h +++ b/libkern/libkern/OSAtomic.h @@ -676,13 +676,21 @@ extern void OSSpinLockUnlock(volatile OSSpinLock * lock); * @discussion * The OSSynchronizeIO routine ensures orderly load and store operations to noncached memory mapped I/O devices. It executes the eieio instruction on PowerPC processors. */ +#if defined(__arm__) || defined(__arm64__) +extern void OSSynchronizeIO(void); +#else static __inline__ void OSSynchronizeIO(void) { } +#endif #if defined(KERNEL_PRIVATE) -#if defined(__i386__) || defined(__x86_64__) +#if defined(__arm__) || defined(__arm64__) +static inline void OSMemoryBarrier(void) { + __asm__ volatile("dmb ish" ::: "memory"); +} +#elif defined(__i386__) || defined(__x86_64__) #if defined(XNU_KERNEL_PRIVATE) static inline void OSMemoryBarrier(void) { __asm__ volatile("mfence" ::: "memory"); diff --git a/libkern/libkern/OSByteOrder.h b/libkern/libkern/OSByteOrder.h index 8ae2c33b8..2a1d1da5d 100644 --- a/libkern/libkern/OSByteOrder.h +++ b/libkern/libkern/OSByteOrder.h @@ -41,6 +41,8 @@ #if (defined(__i386__) || defined(__x86_64__)) #include +#elif defined (__arm__) || defined(__arm64__) +#include #else #include #endif diff --git a/libkern/libkern/OSKextLib.h b/libkern/libkern/OSKextLib.h index 4c863af70..a08218c2e 100644 --- a/libkern/libkern/OSKextLib.h +++ b/libkern/libkern/OSKextLib.h @@ -238,6 +238,12 @@ __BEGIN_DECLS */ #define kOSKextReturnStopping libkern_kext_err(0x1a) +/*! + * @define kOSKextReturnSystemPolicy + * @abstract The kext was prevented from loading due to system policy. + */ +#define kOSKextReturnSystemPolicy libkern_kext_err(0x1b) + #if PRAGMA_MARK #pragma mark - /********************************************************************/ @@ -914,7 +920,7 @@ OSKextGrabPgoData(uuid_t uuid, * Call this function before trapping into the debugger to call OSKextResetPgoCounters. */ void -OSKextResetPgoCountersLock(); +OSKextResetPgoCountersLock(void); /*! * @function OSKextResetPgoCountersUnlock @@ -923,7 +929,7 @@ OSKextResetPgoCountersLock(); * Call this function after trapping into the debugger to call OSKextResetPgoCounters. */ void -OSKextResetPgoCountersUnlock(); +OSKextResetPgoCountersUnlock(void); /*! * @function OSKextResetPgoCounters @@ -932,7 +938,7 @@ OSKextResetPgoCountersUnlock(); * context, while holding OSKextResetPgoCountersLock(). */ void -OSKextResetPgoCounters(); +OSKextResetPgoCounters(void); #if PRAGMA_MARK diff --git a/libkern/libkern/OSKextLibPrivate.h b/libkern/libkern/OSKextLibPrivate.h index 2557007ea..fd08744ed 100644 --- a/libkern/libkern/OSKextLibPrivate.h +++ b/libkern/libkern/OSKextLibPrivate.h @@ -926,7 +926,7 @@ void OSKextLoadedKextSummariesUpdated(void); #ifdef XNU_KERNEL_PRIVATE extern const vm_allocation_site_t * OSKextGetAllocationSiteForCaller(uintptr_t address); -extern uint32_t OSKextGetKmodIDForSite(vm_allocation_site_t * site, +extern uint32_t OSKextGetKmodIDForSite(const vm_allocation_site_t * site, char * name, vm_size_t namelen); extern void OSKextFreeSite(vm_allocation_site_t * site); diff --git a/libkern/libkern/OSMalloc.h b/libkern/libkern/OSMalloc.h index 04a5ba6f2..2d14ce94d 100644 --- a/libkern/libkern/OSMalloc.h +++ b/libkern/libkern/OSMalloc.h @@ -200,7 +200,7 @@ extern void OSMalloc_Tagfree(OSMallocTag tag); */ extern void * OSMalloc( uint32_t size, - OSMallocTag tag); + OSMallocTag tag) __attribute__((alloc_size(1))); /*! @@ -211,7 +211,7 @@ extern void * OSMalloc( */ extern void * OSMalloc_nowait( uint32_t size, - OSMallocTag tag); + OSMallocTag tag) __attribute__((alloc_size(1))); /*! @@ -241,7 +241,7 @@ extern void * OSMalloc_nowait( */ extern void * OSMalloc_noblock( uint32_t size, - OSMallocTag tag); + OSMallocTag tag) __attribute__((alloc_size(1))); /*! diff --git a/libkern/libkern/_OSByteOrder.h b/libkern/libkern/_OSByteOrder.h index 3ceec32eb..5ffcba282 100644 --- a/libkern/libkern/_OSByteOrder.h +++ b/libkern/libkern/_OSByteOrder.h @@ -66,6 +66,9 @@ #include #endif +#if defined (__arm__) || defined(__arm64__) +#include +#endif #define __DARWIN_OSSwapInt16(x) \ diff --git a/libkern/libkern/arm/Makefile b/libkern/libkern/arm/Makefile new file mode 100644 index 000000000..acfa028c8 --- /dev/null +++ b/libkern/libkern/arm/Makefile @@ -0,0 +1,21 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +DATAFILES = \ + OSByteOrder.h + +INSTALL_MD_LIST = ${DATAFILES} + +INSTALL_MD_DIR = libkern/arm + +EXPORT_MD_LIST = ${DATAFILES} + +EXPORT_MD_DIR = libkern/arm + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/libkern/libkern/arm/OSByteOrder.h b/libkern/libkern/arm/OSByteOrder.h new file mode 100644 index 000000000..81279a1f8 --- /dev/null +++ b/libkern/libkern/arm/OSByteOrder.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 1999-2007 Apple Inc. All rights reserved. + */ + +#ifndef _OS_OSBYTEORDERARM_H +#define _OS_OSBYTEORDERARM_H + +#include +#include /* for _ARM_ARCH_6 */ +#include + +/* Generic byte swapping functions. */ + +OS_INLINE +uint16_t +_OSSwapInt16( + uint16_t data +) +{ + /* Reduces to 'rev16' with clang */ + return (uint16_t)(data << 8 | data >> 8); +} + +OS_INLINE +uint32_t +_OSSwapInt32( + uint32_t data +) +{ +#if defined(__llvm__) + data = __builtin_bswap32(data); +#else + /* This actually generates the best code */ + data = (((data ^ (data >> 16 | (data << 16))) & 0xFF00FFFF) >> 8) ^ (data >> 8 | data << 24); +#endif + + return data; +} + +OS_INLINE +uint64_t +_OSSwapInt64( + uint64_t data +) +{ +#if defined(__llvm__) + return __builtin_bswap64(data); +#else + union { + uint64_t ull; + uint32_t ul[2]; + } u; + + /* This actually generates the best code */ + u.ul[0] = (uint32_t)(data >> 32); + u.ul[1] = (uint32_t)(data & 0xffffffff); + u.ul[0] = _OSSwapInt32(u.ul[0]); + u.ul[1] = _OSSwapInt32(u.ul[1]); + return u.ull; +#endif +} + +/* Functions for byte reversed loads. */ + +OS_INLINE +uint16_t +OSReadSwapInt16( + const volatile void * base, + uintptr_t offset +) +{ + uint16_t result; + + result = *(volatile uint16_t *)((volatile uintptr_t)base + offset); + return _OSSwapInt16(result); +} + +OS_INLINE +uint32_t +OSReadSwapInt32( + const volatile void * base, + uintptr_t offset +) +{ + uint32_t result; + + result = *(volatile uint32_t *)((volatile uintptr_t)base + offset); + return _OSSwapInt32(result); +} + +OS_INLINE +uint64_t +OSReadSwapInt64( + const volatile void * base, + uintptr_t offset +) +{ + volatile uint32_t * inp; + union ullc { + uint64_t ull; + uint32_t ul[2]; + } outv; + + inp = (volatile uint32_t *)((volatile uintptr_t)base + offset); + outv.ul[0] = inp[1]; + outv.ul[1] = inp[0]; + outv.ul[0] = _OSSwapInt32(outv.ul[0]); + outv.ul[1] = _OSSwapInt32(outv.ul[1]); + return outv.ull; +} + +/* Functions for byte reversed stores. */ + +OS_INLINE +void +OSWriteSwapInt16( + volatile void * base, + uintptr_t offset, + uint16_t data +) +{ + *(volatile uint16_t *)((volatile uintptr_t)base + offset) = _OSSwapInt16(data); +} + +OS_INLINE +void +OSWriteSwapInt32( + volatile void * base, + uintptr_t offset, + uint32_t data +) +{ + *(volatile uint32_t *)((volatile uintptr_t)base + offset) = _OSSwapInt32(data); +} + +OS_INLINE +void +OSWriteSwapInt64( + volatile void * base, + uintptr_t offset, + uint64_t data +) +{ + *(volatile uint64_t *)((volatile uintptr_t)base + offset) = _OSSwapInt64(data); +} + +#endif /* ! _OS_OSBYTEORDERARM_H */ diff --git a/libkern/libkern/c++/OSData.h b/libkern/libkern/c++/OSData.h index e25079f80..031a056a3 100644 --- a/libkern/libkern/c++/OSData.h +++ b/libkern/libkern/c++/OSData.h @@ -391,7 +391,7 @@ public: * * @abstract * Deallocates or releases any resources - * used by the OSDictionary instance. + * used by the OSData instance. * * @discussion * This function should not be called directly; diff --git a/libkern/libkern/c++/OSKext.h b/libkern/libkern/c++/OSKext.h index 4d4cea4ec..2fc70eb1c 100644 --- a/libkern/libkern/c++/OSKext.h +++ b/libkern/libkern/c++/OSKext.h @@ -528,12 +528,18 @@ private: /* panic() support. */ +public: + enum { + kPrintKextsLock = 0x01, + kPrintKextsUnslide = 0x02, + kPrintKextsTerse = 0x04 + }; static void printKextsInBacktrace( vm_offset_t * addr, unsigned int cnt, int (* printf_func)(const char *fmt, ...), - bool lockFlag, - bool doUnslide); + uint32_t flags); +private: static OSKextLoadedKextSummary *summaryForAddress(const uintptr_t addr); static void *kextForAddress(const void *addr); static boolean_t summaryIsInBacktrace( @@ -543,7 +549,7 @@ private: static void printSummary( OSKextLoadedKextSummary * summary, int (* printf_func)(const char *fmt, ...), - bool doUnslide); + uint32_t flags); static int saveLoadedKextPanicListTyped( const char * prefix, @@ -625,6 +631,8 @@ public: OSCollectionIterator * theIterator); static void createExcludeListFromPrelinkInfo(OSArray * theInfoArray); + static bool isWaitingKextd(void); + virtual bool setAutounloadEnabled(bool flag); virtual const OSSymbol * getIdentifier(void); diff --git a/libkern/libkern/c++/OSMetaClass.h b/libkern/libkern/c++/OSMetaClass.h index 4660b240e..322e5238a 100644 --- a/libkern/libkern/c++/OSMetaClass.h +++ b/libkern/libkern/c++/OSMetaClass.h @@ -41,7 +41,8 @@ class OSDictionary; class OSSerialize; #ifdef XNU_KERNEL_PRIVATE class OSOrderedSet; -#endif +class OSCollection; +#endif /* XNU_KERNEL_PRIVATE */ /*! @@ -59,14 +60,22 @@ class OSOrderedSet; #ifdef XNU_KERNEL_PRIVATE +#ifdef CONFIG_EMBEDDED +#define APPLE_KEXT_VTABLE_PADDING 0 +#else /* CONFIG_EMBEDDED */ /*! @parseOnly */ #define APPLE_KEXT_VTABLE_PADDING 1 +#endif /* CONFIG_EMBEDDED */ #else /* XNU_KERNEL_PRIVATE */ #include +#if TARGET_OS_EMBEDDED +#define APPLE_KEXT_VTABLE_PADDING 0 +#else /* TARGET_OS_EMBEDDED */ /*! @parseOnly */ #define APPLE_KEXT_VTABLE_PADDING 1 +#endif /* TARGET_OS_EMBEDDED */ #endif /* XNU_KERNEL_PRIVATE */ @@ -75,6 +84,8 @@ class OSOrderedSet; #if defined(__LP64__) /*! @parseOnly */ #define APPLE_KEXT_LEGACY_ABI 0 +#elif defined(__arm__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2)) +#define APPLE_KEXT_LEGACY_ABI 0 #else #define APPLE_KEXT_LEGACY_ABI 1 #endif @@ -346,7 +357,43 @@ _ptmf2ptf(const OSMetaClassBase *self, void (OSMetaClassBase::*func)(void)) } #else /* !APPLE_KEXT_LEGACY_ABI */ -#if defined(__i386__) || defined(__x86_64__) +#if defined(__arm__) || defined(__arm64__) +typedef long int ptrdiff_t; +/* + * Ugly reverse engineered ABI. Where does it come from? Nobody knows. + * gcc 4.2-built ARM kernel panics with multiple inheritance (no, really) + */ +static inline _ptf_t +_ptmf2ptf(const OSMetaClassBase *self, void (OSMetaClassBase::*func)(void)) +{ + struct ptmf_t { + _ptf_t fPFN; + ptrdiff_t delta; + }; + union { + void (OSMetaClassBase::*fIn)(void); + struct ptmf_t pTMF; + } map; + + + map.fIn = func; + + if (map.pTMF.delta & 1) { + // virtual + union { + const OSMetaClassBase *fObj; + _ptf_t **vtablep; + } u; + u.fObj = self; + + // Virtual member function so dereference table + return *(_ptf_t *)(((uintptr_t)*u.vtablep) + (uintptr_t)map.pTMF.fPFN); + } else { + // Not virtual, i.e. plain member func + return map.pTMF.fPFN; + } +} +#elif defined(__i386__) || defined(__x86_64__) // Slightly less arcane and slightly less evil code to do // the same for kexts compiled with the standard Itanium C++ @@ -926,13 +973,43 @@ public: * @abstract * Look up a metaclass in the run-time type information system. * - * @param name The name of the desired class's metaclass. + * @param name The name of the desired class's metaclass. * * @result * A pointer to the metaclass object if found, NULL otherwise. */ static const OSMetaClass * getMetaClassWithName(const OSSymbol * name); +#if XNU_KERNEL_PRIVATE + + /*! + * @function copyMetaClassWithName + * + * @abstract + * Look up a metaclass in the run-time type information system. + * + * @param name The name of the desired class's metaclass. + * + * @result + * A pointer to the metaclass object if found, NULL otherwise. + * The metaclass will be protected from unloading until releaseMetaClass() + * is called. + */ + static const OSMetaClass * copyMetaClassWithName(const OSSymbol * name); + /*! + * @function releaseMetaClass + * + * @abstract + * Releases reference obtained from copyMetaClassWithName(). + * + * @discussion + * The metaclass will be protected from unloading until releaseMetaClass() + * is called. + */ + void releaseMetaClass() const; + +#endif /* XNU_KERNEL_PRIVATE */ + protected: /*! * @function retain @@ -1263,6 +1340,9 @@ public: */ static void considerUnloads(); +#if XNU_KERNEL_PRIVATE + static bool removeClasses(OSCollection * metaClasses); +#endif /* XNU_KERNEL_PRIVATE */ /*! * @function allocClassWithName @@ -1569,7 +1649,7 @@ private: OSMetaClassInstanceApplierFunction applier, void * context); public: -#endif +#endif /* XNU_KERNEL_PRIVATE */ /* Not to be included in headerdoc. * @@ -2086,8 +2166,8 @@ public: void trackedFree(OSObject * instance) const; void trackedAccumSize(OSObject * instance, size_t size) const; struct IOTrackingQueue * getTracking() const; -#endif -#endif +#endif /* IOTRACKING */ +#endif /* XNU_KERNEL_PRIVATE */ private: // Obsolete APIs diff --git a/libkern/libkern/crypto/Makefile b/libkern/libkern/crypto/Makefile index e701850eb..0fdcb6258 100644 --- a/libkern/libkern/crypto/Makefile +++ b/libkern/libkern/crypto/Makefile @@ -8,7 +8,7 @@ include $(MakeInc_def) DATAFILES = md5.h sha1.h -PRIVATE_DATAFILES = register_crypto.h sha2.h des.h aes.h aesxts.h rand.h rsa.h +PRIVATE_DATAFILES = register_crypto.h sha2.h des.h aes.h aesxts.h rand.h rsa.h chacha20poly1305.h INSTALL_KF_MI_LIST = ${DATAFILES} diff --git a/libkern/libkern/crypto/chacha20poly1305.h b/libkern/libkern/crypto/chacha20poly1305.h new file mode 100644 index 000000000..598f59746 --- /dev/null +++ b/libkern/libkern/crypto/chacha20poly1305.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _CHACHA20POLY1305_H +#define _CHACHA20POLY1305_H + +#if defined(__cplusplus) +extern "C" +{ +#endif + +#include + +typedef ccchacha20poly1305_ctx chacha20poly1305_ctx; + +int chacha20poly1305_init(chacha20poly1305_ctx *ctx, const uint8_t *key); +int chacha20poly1305_reset(chacha20poly1305_ctx *ctx); +int chacha20poly1305_setnonce(chacha20poly1305_ctx *ctx, const uint8_t *nonce); +int chacha20poly1305_incnonce(chacha20poly1305_ctx *ctx, uint8_t *nonce); +int chacha20poly1305_aad(chacha20poly1305_ctx *ctx, size_t nbytes, const void *aad); +int chacha20poly1305_encrypt(chacha20poly1305_ctx *ctx, size_t nbytes, const void *ptext, void *ctext); +int chacha20poly1305_finalize(chacha20poly1305_ctx *ctx, uint8_t *tag); +int chacha20poly1305_decrypt(chacha20poly1305_ctx *ctx, size_t nbytes, const void *ctext, void *ptext); +int chacha20poly1305_verify(chacha20poly1305_ctx *ctx, const uint8_t *tag); + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/libkern/libkern/crypto/register_crypto.h b/libkern/libkern/crypto/register_crypto.h index 6041ebb31..a29592700 100644 --- a/libkern/libkern/crypto/register_crypto.h +++ b/libkern/libkern/crypto/register_crypto.h @@ -39,6 +39,7 @@ extern "C" { #include #include #include +#include /* Function types */ @@ -68,7 +69,19 @@ typedef int (*ccgcm_init_with_iv_fn_t)(const struct ccmode_gcm *mode, ccgcm_ctx size_t key_nbytes, const void *key, const void *iv); typedef int (*ccgcm_inc_iv_fn_t)(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, void *iv); - + +typedef const struct ccchacha20poly1305_fns { + const struct ccchacha20poly1305_info *(*info)(void); + int (*init)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *key); + int (*reset)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx); + int (*setnonce)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *nonce); + int (*incnonce)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, uint8_t *nonce); + int (*aad)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *aad); + int (*encrypt)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *ptext, void *ctext); + int (*finalize)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, uint8_t *tag); + int (*decrypt)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *ctext, void *ptext); + int (*verify)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *tag); +} *ccchacha20poly1305_fns_t; /* pbkdf2 */ typedef void (*ccpbkdf2_hmac_fn_t)(const struct ccdigest_info *di, @@ -130,6 +143,7 @@ typedef struct crypto_functions { const struct ccmode_ecb *ccaes_ecb_decrypt; const struct ccmode_cbc *ccaes_cbc_encrypt; const struct ccmode_cbc *ccaes_cbc_decrypt; + const struct ccmode_ctr *ccaes_ctr_crypt; const struct ccmode_xts *ccaes_xts_encrypt; const struct ccmode_xts *ccaes_xts_decrypt; const struct ccmode_gcm *ccaes_gcm_encrypt; @@ -137,6 +151,8 @@ typedef struct crypto_functions { ccgcm_init_with_iv_fn_t ccgcm_init_with_iv_fn; ccgcm_inc_iv_fn_t ccgcm_inc_iv_fn; + + ccchacha20poly1305_fns_t ccchacha20poly1305_fns; /* DES, ecb and cbc */ const struct ccmode_ecb *ccdes_ecb_encrypt; diff --git a/libkern/net/inet_aton.c b/libkern/net/inet_aton.c index 9c0d94ae3..ff518373e 100644 --- a/libkern/net/inet_aton.c +++ b/libkern/net/inet_aton.c @@ -41,7 +41,7 @@ int inet_aton(const char *cp, struct in_addr *addr) { u_long parts[4]; - in_addr_t val; + in_addr_t val = 0; const char *c; char *endptr; int gotend, n; diff --git a/libkern/os/Makefile b/libkern/os/Makefile index 80a933fb9..390b9b861 100644 --- a/libkern/os/Makefile +++ b/libkern/os/Makefile @@ -16,9 +16,11 @@ KERNELFILES = \ overflow.h PRIVATE_KERNELFILES = \ - object_private.h + object_private.h \ + reason_private.h -PRIVATE_DATAFILES = +PRIVATE_DATAFILES = \ + reason_private.h INSTALL_MI_LIST = \ overflow.h diff --git a/libkern/os/log.c b/libkern/os/log.c index a2638fee5..a019b7bd9 100644 --- a/libkern/os/log.c +++ b/libkern/os/log.c @@ -72,7 +72,12 @@ uint32_t oslog_s_error_count = 0; uint32_t oslog_s_metadata_msgcount = 0; static bool oslog_boot_done = false; -extern boolean_t oslog_early_boot_complete; +extern boolean_t early_boot_complete; + +#ifdef XNU_KERNEL_PRIVATE +bool startup_serial_logging_active = true; +uint64_t startup_serial_num_procs = 300; +#endif /* XNU_KERNEL_PRIVATE */ // XXX firehose_tracepoint_id_t @@ -166,9 +171,15 @@ _os_log_with_args_internal(os_log_t oslog, os_log_type_t type, if (format[0] == '\0') { return; } - /* cf. r24974766 & r25201228*/ - safe = (!oslog_early_boot_complete || oslog_is_safe()); - logging = (!(logging_config & ATM_TRACE_DISABLE) || !(logging_config & ATM_TRACE_OFF)); + + /* early boot can log to dmesg for later replay (27307943) */ + safe = (!early_boot_complete || oslog_is_safe()); + + if (logging_config & ATM_TRACE_DISABLE || logging_config & ATM_TRACE_OFF) { + logging = false; + } else { + logging = true; + } if (oslog != &_os_log_replay) { _os_log_to_msgbuf_internal(format, args, safe, logging); diff --git a/libkern/os/log.h b/libkern/os/log.h index a26a129a1..1da91a8f3 100644 --- a/libkern/os/log.h +++ b/libkern/os/log.h @@ -48,7 +48,12 @@ __BEGIN_DECLS extern void *__dso_handle; -OS_ALWAYS_INLINE static inline void _os_log_verify_format_str(__unused const char *msg, ...) __attribute__((format(os_trace, 1, 2))); +#ifdef XNU_KERNEL_PRIVATE +extern bool startup_serial_logging_active; +extern uint64_t startup_serial_num_procs; +#endif /* XNU_KERNEL_PRIVATE */ + +OS_ALWAYS_INLINE static inline void _os_log_verify_format_str(__unused const char *msg, ...) __attribute__((format(os_log, 1, 2))); OS_ALWAYS_INLINE static inline void _os_log_verify_format_str(__unused const char *msg, ...) { /* placeholder */ } #if OS_OBJECT_USE_OBJC @@ -485,6 +490,13 @@ os_log_debug_enabled(os_log_t log); __asm__(""); /* avoid tailcall */ \ }) +#ifdef XNU_KERNEL_PRIVATE +#define os_log_with_startup_serial(log, format, ...) __extension__({ \ + if (startup_serial_logging_active) { printf(format, ##__VA_ARGS__); } \ + else { os_log(log, format, ##__VA_ARGS__); } \ +}) +#endif /* XNU_KERNEL_PRIVATE */ + /*! * @function _os_log_internal * diff --git a/libkern/os/overflow.h b/libkern/os/overflow.h index 8d0fd9949..05ddbef39 100644 --- a/libkern/os/overflow.h +++ b/libkern/os/overflow.h @@ -131,6 +131,15 @@ __os_warn_unused(__const bool x) _s | _t; \ })) +/* os_mul3_overflow(a, b, c) -> (a * b * c) */ +#define os_mul3_overflow(a, b, c, res) __os_warn_unused(__extension__({ \ + __typeof(*(res)) _tmp; \ + bool _s, _t; \ + _s = os_mul_overflow((a), (b), &_tmp); \ + _t = os_mul_overflow((c), _tmp, (res)); \ + _s | _t; \ +})) + /* os_add_and_mul_overflow(a, b, x) -> (a + b)*x */ #define os_add_and_mul_overflow(a, b, x, res) __os_warn_unused(__extension__({ \ __typeof(*(res)) _tmp; \ @@ -149,4 +158,6 @@ __os_warn_unused(__const bool x) _s | _t; \ })) +#define os_convert_overflow(a, res) os_add_overflow((a), 0, (res)) + #endif /* _OS_OVERFLOW_H */ diff --git a/libkern/os/reason_private.h b/libkern/os/reason_private.h new file mode 100644 index 000000000..477bceeed --- /dev/null +++ b/libkern/os/reason_private.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef OS_REASON_PRIVATE_H +#define OS_REASON_PRIVATE_H + +#include +#include + +/* Codes in the OS_REASON_LIBSYSTEM namespace */ + +OS_ENUM(os_reason_libsystem_code, uint64_t, + OS_REASON_LIBSYSTEM_CODE_WORKLOOP_OWNERSHIP_LEAK = 1, + OS_REASON_LIBSYSTEM_CODE_FAULT = 2, /* generated by os_log_fault */ +); + +#ifndef KERNEL + +/* + * similar to abort_with_payload, but for faults. + * + * [EBUSY] too many corpses are being generated at the moment + * [EQFULL] the process used all its user fault quota + * [ENOTSUP] generating simulated abort with reason is disabled + * [EPERM] generating simulated abort with reason for this namespace is not turned on + */ +int +os_fault_with_payload(uint32_t reason_namespace, uint64_t reason_code, + void *payload, uint32_t payload_size, const char *reason_string, + uint64_t reason_flags); + +#endif // !KERNEL + +#endif // OS_REASON_PRIVATE_H diff --git a/libkern/zlib/gzio.c b/libkern/zlib/gzio.c deleted file mode 100644 index 9c87cdd0f..000000000 --- a/libkern/zlib/gzio.c +++ /dev/null @@ -1,1031 +0,0 @@ -/* - * Copyright (c) 2008-2016 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ -/* gzio.c -- IO on .gz files - * Copyright (C) 1995-2005 Jean-loup Gailly. - * For conditions of distribution and use, see copyright notice in zlib.h - * - * Compile this file with -DNO_GZCOMPRESS to avoid the compression code. - */ - -/* @(#) $Id$ */ - -#include - -#include "zutil.h" - -#ifdef NO_DEFLATE /* for compatibility with old definition */ -# define NO_GZCOMPRESS -#endif - -#ifndef NO_DUMMY_DECL -struct internal_state {int dummy;}; /* for buggy compilers */ -#endif - -#ifndef Z_BUFSIZE -# ifdef MAXSEG_64K -# define Z_BUFSIZE 4096 /* minimize memory usage for 16-bit DOS */ -# else -# define Z_BUFSIZE 16384 -# endif -#endif -#ifndef Z_PRINTF_BUFSIZE -# define Z_PRINTF_BUFSIZE 4096 -#endif - -#ifdef __MVS__ -# pragma map (fdopen , "\174\174FDOPEN") - FILE *fdopen(int, const char *); -#endif - -#ifndef STDC -extern voidp malloc OF((uInt size)); -extern void free OF((voidpf ptr)); -#endif - -#define ALLOC(size) malloc(size) -#define TRYFREE(p) {if (p) free(p);} - -static int const gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */ - -/* gzip flag byte */ -#define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */ -#define HEAD_CRC 0x02 /* bit 1 set: header CRC present */ -#define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ -#define ORIG_NAME 0x08 /* bit 3 set: original file name present */ -#define COMMENT 0x10 /* bit 4 set: file comment present */ -#define RESERVED 0xE0 /* bits 5..7: reserved */ - -typedef struct gz_stream { - z_stream stream; - int z_err; /* error code for last stream operation */ - int z_eof; /* set if end of input file */ - FILE *file; /* .gz file */ - Byte *inbuf; /* input buffer */ - Byte *outbuf; /* output buffer */ - uLong crc; /* crc32 of uncompressed data */ - char *msg; /* error message */ - char *path; /* path name for debugging only */ - int transparent; /* 1 if input file is not a .gz file */ - char mode; /* 'w' or 'r' */ - z_off_t start; /* start of compressed data in file (header skipped) */ - z_off_t in; /* bytes into deflate or inflate */ - z_off_t out; /* bytes out of deflate or inflate */ - int back; /* one character push-back */ - int last; /* true if push-back is last character */ -} gz_stream; - - -local gzFile gz_open OF((const char *path, const char *mode, int fd)); -local int do_flush OF((gzFile file, int flush)); -local int get_byte OF((gz_stream *s)); -local void check_header OF((gz_stream *s)); -local int destroy OF((gz_stream *s)); -local void putLong OF((FILE *file, uLong x)); -local uLong getLong OF((gz_stream *s)); - -/* =========================================================================== - Opens a gzip (.gz) file for reading or writing. The mode parameter - is as in fopen ("rb" or "wb"). The file is given either by file descriptor - or path name (if fd == -1). - gz_open returns NULL if the file could not be opened or if there was - insufficient memory to allocate the (de)compression state; errno - can be checked to distinguish the two cases (if errno is zero, the - zlib error is Z_MEM_ERROR). -*/ -local gzFile -gz_open(const char *path, const char *mode, int fd) -{ - int err; - int level = Z_DEFAULT_COMPRESSION; /* compression level */ - int strategy = Z_DEFAULT_STRATEGY; /* compression strategy */ - char *p = (char*)mode; - gz_stream *s; - char fmode[80]; /* copy of mode, without the compression level */ - char *m = fmode; - - if (!path || !mode) return Z_NULL; - - s = (gz_stream *)ALLOC(sizeof(gz_stream)); - if (!s) return Z_NULL; - - s->stream.zalloc = (alloc_func)0; - s->stream.zfree = (free_func)0; - s->stream.opaque = (voidpf)0; - s->stream.next_in = s->inbuf = Z_NULL; - s->stream.next_out = s->outbuf = Z_NULL; - s->stream.avail_in = s->stream.avail_out = 0; - s->file = NULL; - s->z_err = Z_OK; - s->z_eof = 0; - s->in = 0; - s->out = 0; - s->back = EOF; - s->crc = z_crc32(0L, Z_NULL, 0); - s->msg = NULL; - s->transparent = 0; - - s->path = (char*)ALLOC(strlen(path)+1); - if (s->path == NULL) { - return destroy(s), (gzFile)Z_NULL; - } - strcpy(s->path, path); /* do this early for debugging */ - - s->mode = '\0'; - do { - if (*p == 'r') s->mode = 'r'; - if (*p == 'w' || *p == 'a') s->mode = 'w'; - if (*p >= '0' && *p <= '9') { - level = *p - '0'; - } else if (*p == 'f') { - strategy = Z_FILTERED; - } else if (*p == 'h') { - strategy = Z_HUFFMAN_ONLY; - } else if (*p == 'R') { - strategy = Z_RLE; - } else { - *m++ = *p; /* copy the mode */ - } - } while (*p++ && m != fmode + sizeof(fmode)); - if (s->mode == '\0') return destroy(s), (gzFile)Z_NULL; - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - err = Z_STREAM_ERROR; -#else - err = deflateInit2(&(s->stream), level, - Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL, strategy); - /* windowBits is passed < 0 to suppress zlib header */ - - s->stream.next_out = s->outbuf = (Byte*)ALLOC(Z_BUFSIZE); -#endif - if (err != Z_OK || s->outbuf == Z_NULL) { - return destroy(s), (gzFile)Z_NULL; - } - } else { - s->stream.next_in = s->inbuf = (Byte*)ALLOC(Z_BUFSIZE); - - err = inflateInit2(&(s->stream), -MAX_WBITS); - /* windowBits is passed < 0 to tell that there is no zlib header. - * Note that in this case inflate *requires* an extra "dummy" byte - * after the compressed stream in order to complete decompression and - * return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are - * present after the compressed stream. - */ - if (err != Z_OK || s->inbuf == Z_NULL) { - return destroy(s), (gzFile)Z_NULL; - } - } - s->stream.avail_out = Z_BUFSIZE; - - errno = 0; - s->file = fd < 0 ? F_OPEN(path, fmode) : (FILE*)fdopen(fd, fmode); - - if (s->file == NULL) { - return destroy(s), (gzFile)Z_NULL; - } - if (s->mode == 'w') { - /* Write a very simple .gz header: - */ - fprintf(s->file, "%c%c%c%c%c%c%c%c%c%c", gz_magic[0], gz_magic[1], - Z_DEFLATED, 0 /*flags*/, 0,0,0,0 /*time*/, 0 /*xflags*/, OS_CODE); - s->start = 10L; - /* We use 10L instead of ftell(s->file) to because ftell causes an - * fflush on some systems. This version of the library doesn't use - * start anyway in write mode, so this initialization is not - * necessary. - */ - } else { - check_header(s); /* skip the .gz header */ - s->start = ftell(s->file) - s->stream.avail_in; - } - - return (gzFile)s; -} - -/* =========================================================================== - Opens a gzip (.gz) file for reading or writing. -*/ -gzFile ZEXPORT -gzopen(const char *path, const char *mode) -{ - return gz_open (path, mode, -1); -} - -/* =========================================================================== - Associate a gzFile with the file descriptor fd. fd is not dup'ed here - to mimic the behavio(u)r of fdopen. -*/ -gzFile ZEXPORT -gzdopen(int fd, const char *mode) -{ - char name[46]; /* allow for up to 128-bit integers */ - - if (fd < 0) return (gzFile)Z_NULL; - sprintf(name, "", fd); /* for debugging */ - - return gz_open (name, mode, fd); -} - -/* =========================================================================== - * Update the compression level and strategy - */ -int ZEXPORT -gzsetparams(gzFile file, int level, int strategy) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - /* Make room to allow flushing */ - if (s->stream.avail_out == 0) { - - s->stream.next_out = s->outbuf; - if (fwrite(s->outbuf, 1, Z_BUFSIZE, s->file) != Z_BUFSIZE) { - s->z_err = Z_ERRNO; - } - s->stream.avail_out = Z_BUFSIZE; - } - - return deflateParams (&(s->stream), level, strategy); -} - -/* =========================================================================== - Read a byte from a gz_stream; update next_in and avail_in. Return EOF - for end of file. - IN assertion: the stream s has been sucessfully opened for reading. -*/ -local int -get_byte(gz_stream *s) -{ - if (s->z_eof) return EOF; - if (s->stream.avail_in == 0) { - errno = 0; - s->stream.avail_in = (uInt)fread(s->inbuf, 1, Z_BUFSIZE, s->file); - if (s->stream.avail_in == 0) { - s->z_eof = 1; - if (ferror(s->file)) s->z_err = Z_ERRNO; - return EOF; - } - s->stream.next_in = s->inbuf; - } - s->stream.avail_in--; - return *(s->stream.next_in)++; -} - -/* =========================================================================== - Check the gzip header of a gz_stream opened for reading. Set the stream - mode to transparent if the gzip magic header is not present; set s->err - to Z_DATA_ERROR if the magic header is present but the rest of the header - is incorrect. - IN assertion: the stream s has already been created sucessfully; - s->stream.avail_in is zero for the first time, but may be non-zero - for concatenated .gz files. -*/ -local void -check_header(gz_stream *s) -{ - int method; /* method byte */ - int flags; /* flags byte */ - uInt len; - int c; - - /* Assure two bytes in the buffer so we can peek ahead -- handle case - where first byte of header is at the end of the buffer after the last - gzip segment */ - len = s->stream.avail_in; - if (len < 2) { - if (len) s->inbuf[0] = s->stream.next_in[0]; - errno = 0; - len = (uInt)fread(s->inbuf + len, 1, Z_BUFSIZE >> len, s->file); - if (len == 0 && ferror(s->file)) s->z_err = Z_ERRNO; - s->stream.avail_in += len; - s->stream.next_in = s->inbuf; - if (s->stream.avail_in < 2) { - s->transparent = s->stream.avail_in; - return; - } - } - - /* Peek ahead to check the gzip magic header */ - if (s->stream.next_in[0] != gz_magic[0] || - s->stream.next_in[1] != gz_magic[1]) { - s->transparent = 1; - return; - } - s->stream.avail_in -= 2; - s->stream.next_in += 2; - - /* Check the rest of the gzip header */ - method = get_byte(s); - flags = get_byte(s); - if (method != Z_DEFLATED || (flags & RESERVED) != 0) { - s->z_err = Z_DATA_ERROR; - return; - } - - /* Discard time, xflags and OS code: */ - for (len = 0; len < 6; len++) (void)get_byte(s); - - if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */ - len = (uInt)get_byte(s); - len += ((uInt)get_byte(s))<<8; - /* len is garbage if EOF but the loop below will quit anyway */ - while (len-- != 0 && get_byte(s) != EOF) ; - } - if ((flags & ORIG_NAME) != 0) { /* skip the original file name */ - while ((c = get_byte(s)) != 0 && c != EOF) ; - } - if ((flags & COMMENT) != 0) { /* skip the .gz file comment */ - while ((c = get_byte(s)) != 0 && c != EOF) ; - } - if ((flags & HEAD_CRC) != 0) { /* skip the header crc */ - for (len = 0; len < 2; len++) (void)get_byte(s); - } - s->z_err = s->z_eof ? Z_DATA_ERROR : Z_OK; -} - - /* =========================================================================== - * Cleanup then free the given gz_stream. Return a zlib error code. - Try freeing in the reverse order of allocations. - */ -local int -destroy(gz_stream *s) -{ - int err = Z_OK; - - if (!s) return Z_STREAM_ERROR; - - TRYFREE(s->msg); - - if (s->stream.state != NULL) { - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - err = Z_STREAM_ERROR; -#else - err = deflateEnd(&(s->stream)); -#endif - } else if (s->mode == 'r') { - err = inflateEnd(&(s->stream)); - } - } - if (s->file != NULL && fclose(s->file)) { -#ifdef ESPIPE - if (errno != ESPIPE) /* fclose is broken for pipes in HP/UX */ -#endif - err = Z_ERRNO; - } - if (s->z_err < 0) err = s->z_err; - - TRYFREE(s->inbuf); - TRYFREE(s->outbuf); - TRYFREE(s->path); - TRYFREE(s); - return err; -} - -/* =========================================================================== - Reads the given number of uncompressed bytes from the compressed file. - gzread returns the number of bytes actually read (0 for end of file). -*/ -int ZEXPORT -gzread(gzFile file, voidp buf, unsigned len) -{ - gz_stream *s = (gz_stream*)file; - Bytef *start = (Bytef*)buf; /* starting point for crc computation */ - Byte *next_out; /* == stream.next_out but not forced far (for MSDOS) */ - - if (s == NULL || s->mode != 'r') return Z_STREAM_ERROR; - - if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO) return -1; - if (s->z_err == Z_STREAM_END) return 0; /* EOF */ - - next_out = (Byte*)buf; - s->stream.next_out = (Bytef*)buf; - s->stream.avail_out = len; - - if (s->stream.avail_out && s->back != EOF) { - *next_out++ = s->back; - s->stream.next_out++; - s->stream.avail_out--; - s->back = EOF; - s->out++; - start++; - if (s->last) { - s->z_err = Z_STREAM_END; - return 1; - } - } - - while (s->stream.avail_out != 0) { - - if (s->transparent) { - /* Copy first the lookahead bytes: */ - uInt n = s->stream.avail_in; - if (n > s->stream.avail_out) n = s->stream.avail_out; - if (n > 0) { - zmemcpy(s->stream.next_out, s->stream.next_in, n); - next_out += n; - s->stream.next_out = next_out; - s->stream.next_in += n; - s->stream.avail_out -= n; - s->stream.avail_in -= n; - } - if (s->stream.avail_out > 0) { - s->stream.avail_out -= - (uInt)fread(next_out, 1, s->stream.avail_out, s->file); - } - len -= s->stream.avail_out; - s->in += len; - s->out += len; - if (len == 0) s->z_eof = 1; - return (int)len; - } - if (s->stream.avail_in == 0 && !s->z_eof) { - - errno = 0; - s->stream.avail_in = (uInt)fread(s->inbuf, 1, Z_BUFSIZE, s->file); - if (s->stream.avail_in == 0) { - s->z_eof = 1; - if (ferror(s->file)) { - s->z_err = Z_ERRNO; - break; - } - } - s->stream.next_in = s->inbuf; - } - s->in += s->stream.avail_in; - s->out += s->stream.avail_out; - s->z_err = inflate(&(s->stream), Z_NO_FLUSH); - s->in -= s->stream.avail_in; - s->out -= s->stream.avail_out; - - if (s->z_err == Z_STREAM_END) { - /* Check CRC and original size */ - s->crc = z_crc32(s->crc, start, (uInt)(s->stream.next_out - start)); - start = s->stream.next_out; - - if (getLong(s) != s->crc) { - s->z_err = Z_DATA_ERROR; - } else { - (void)getLong(s); - /* The uncompressed length returned by above getlong() may be - * different from s->out in case of concatenated .gz files. - * Check for such files: - */ - check_header(s); - if (s->z_err == Z_OK) { - inflateReset(&(s->stream)); - s->crc = z_crc32(0L, Z_NULL, 0); - } - } - } - if (s->z_err != Z_OK || s->z_eof) break; - } - s->crc = z_crc32(s->crc, start, (uInt)(s->stream.next_out - start)); - - if (len == s->stream.avail_out && - (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO)) - return -1; - return (int)(len - s->stream.avail_out); -} - - -/* =========================================================================== - Reads one byte from the compressed file. gzgetc returns this byte - or -1 in case of end of file or error. -*/ -int ZEXPORT -gzgetc(gzFile file) -{ - unsigned char c; - - return gzread(file, &c, 1) == 1 ? c : -1; -} - - -/* =========================================================================== - Push one byte back onto the stream. -*/ -int ZEXPORT -gzungetc(int c, gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r' || c == EOF || s->back != EOF) return EOF; - s->back = c; - s->out--; - s->last = (s->z_err == Z_STREAM_END); - if (s->last) s->z_err = Z_OK; - s->z_eof = 0; - return c; -} - - -/* =========================================================================== - Reads bytes from the compressed file until len-1 characters are - read, or a newline character is read and transferred to buf, or an - end-of-file condition is encountered. The string is then terminated - with a null character. - gzgets returns buf, or Z_NULL in case of error. - - The current implementation is not optimized at all. -*/ -char * ZEXPORT -gzgets(gzFile file, char *buf, int len) -{ - char *b = buf; - if (buf == Z_NULL || len <= 0) return Z_NULL; - - while (--len > 0 && gzread(file, buf, 1) == 1 && *buf++ != '\n') ; - *buf = '\0'; - return b == buf && len > 0 ? Z_NULL : b; -} - - -#ifndef NO_GZCOMPRESS -/* =========================================================================== - Writes the given number of uncompressed bytes into the compressed file. - gzwrite returns the number of bytes actually written (0 in case of error). -*/ -int ZEXPORT -gzwrite(gzFile file, voidpc buf, unsigned len) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - s->stream.next_in = (Bytef*)buf; - s->stream.avail_in = len; - - while (s->stream.avail_in != 0) { - - if (s->stream.avail_out == 0) { - - s->stream.next_out = s->outbuf; - if (fwrite(s->outbuf, 1, Z_BUFSIZE, s->file) != Z_BUFSIZE) { - s->z_err = Z_ERRNO; - break; - } - s->stream.avail_out = Z_BUFSIZE; - } - s->in += s->stream.avail_in; - s->out += s->stream.avail_out; - s->z_err = deflate(&(s->stream), Z_NO_FLUSH); - s->in -= s->stream.avail_in; - s->out -= s->stream.avail_out; - if (s->z_err != Z_OK) break; - } - s->crc = z_crc32(s->crc, (const Bytef *)buf, len); - - return (int)(len - s->stream.avail_in); -} - - -/* =========================================================================== - Converts, formats, and writes the args to the compressed file under - control of the format string, as in fprintf. gzprintf returns the number of - uncompressed bytes actually written (0 in case of error). -*/ -#ifdef STDC -#include - -int ZEXPORTVA -gzprintf(gzFile file, const char *format, /* args */ ...) -{ - char buf[Z_PRINTF_BUFSIZE]; - va_list va; - int len; - - buf[sizeof(buf) - 1] = 0; - va_start(va, format); -#ifdef NO_vsnprintf -# ifdef HAS_vsprintf_void - (void)vsprintf(buf, format, va); - va_end(va); - for (len = 0; len < sizeof(buf); len++) - if (buf[len] == 0) break; -# else - len = vsprintf(buf, format, va); - va_end(va); -# endif -#else -# ifdef HAS_vsnprintf_void - (void)vsnprintf(buf, sizeof(buf), format, va); - va_end(va); - len = strlen(buf); -# else - len = vsnprintf(buf, sizeof(buf), format, va); - va_end(va); -# endif -#endif - if (len <= 0 || len >= (int)sizeof(buf) || buf[sizeof(buf) - 1] != 0) - return 0; - return gzwrite(file, buf, (unsigned)len); -} -#else /* not ANSI C */ - -int ZEXPORTVA -gzprintf(gzFile file, const char *format, int a1, int a2, int a3, int a4, - int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, - int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20) -{ - char buf[Z_PRINTF_BUFSIZE]; - int len; - - buf[sizeof(buf) - 1] = 0; -#ifdef NO_snprintf -# ifdef HAS_sprintf_void - sprintf(buf, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); - for (len = 0; len < sizeof(buf); len++) - if (buf[len] == 0) break; -# else - len = sprintf(buf, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); -# endif -#else -# ifdef HAS_snprintf_void - snprintf(buf, sizeof(buf), format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); - len = strlen(buf); -# else - len = snprintf(buf, sizeof(buf), format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); -# endif -#endif - if (len <= 0 || len >= sizeof(buf) || buf[sizeof(buf) - 1] != 0) - return 0; - return gzwrite(file, buf, len); -} -#endif - -/* =========================================================================== - Writes c, converted to an unsigned char, into the compressed file. - gzputc returns the value that was written, or -1 in case of error. -*/ -int ZEXPORT -gzputc(gzFile file, int c) -{ - unsigned char cc = (unsigned char) c; /* required for big endian systems */ - - return gzwrite(file, &cc, 1) == 1 ? (int)cc : -1; -} - - -/* =========================================================================== - Writes the given null-terminated string to the compressed file, excluding - the terminating null character. - gzputs returns the number of characters written, or -1 in case of error. -*/ -int ZEXPORT -gzputs(gzFile file, const char *s) -{ - return gzwrite(file, (char*)s, (unsigned)strlen(s)); -} - - -/* =========================================================================== - Flushes all pending output into the compressed file. The parameter - flush is as in the deflate() function. -*/ -local int -do_flush(gzFile file, int flush) -{ - uInt len; - int done = 0; - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - s->stream.avail_in = 0; /* should be zero already anyway */ - - for (;;) { - len = Z_BUFSIZE - s->stream.avail_out; - - if (len != 0) { - if ((uInt)fwrite(s->outbuf, 1, len, s->file) != len) { - s->z_err = Z_ERRNO; - return Z_ERRNO; - } - s->stream.next_out = s->outbuf; - s->stream.avail_out = Z_BUFSIZE; - } - if (done) break; - s->out += s->stream.avail_out; - s->z_err = deflate(&(s->stream), flush); - s->out -= s->stream.avail_out; - - /* Ignore the second of two consecutive flushes: */ - if (len == 0 && s->z_err == Z_BUF_ERROR) s->z_err = Z_OK; - - /* deflate has finished flushing only when it hasn't used up - * all the available space in the output buffer: - */ - done = (s->stream.avail_out != 0 || s->z_err == Z_STREAM_END); - - if (s->z_err != Z_OK && s->z_err != Z_STREAM_END) break; - } - return s->z_err == Z_STREAM_END ? Z_OK : s->z_err; -} - -int ZEXPORT -gzflush(gzFile file, int flush) -{ - gz_stream *s = (gz_stream*)file; - int err = do_flush (file, flush); - - if (err) return err; - fflush(s->file); - return s->z_err == Z_STREAM_END ? Z_OK : s->z_err; -} -#endif /* NO_GZCOMPRESS */ - -/* =========================================================================== - Sets the starting position for the next gzread or gzwrite on the given - compressed file. The offset represents a number of bytes in the - gzseek returns the resulting offset location as measured in bytes from - the beginning of the uncompressed stream, or -1 in case of error. - SEEK_END is not implemented, returns error. - In this version of the library, gzseek can be extremely slow. -*/ -z_off_t ZEXPORT -gzseek(gzFile file, z_off_t offset, int whence) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || whence == SEEK_END || - s->z_err == Z_ERRNO || s->z_err == Z_DATA_ERROR) { - return -1L; - } - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - return -1L; -#else - if (whence == SEEK_SET) { - offset -= s->in; - } - if (offset < 0) return -1L; - - /* At this point, offset is the number of zero bytes to write. */ - if (s->inbuf == Z_NULL) { - s->inbuf = (Byte*)ALLOC(Z_BUFSIZE); /* for seeking */ - if (s->inbuf == Z_NULL) return -1L; - zmemzero(s->inbuf, Z_BUFSIZE); - } - while (offset > 0) { - uInt size = Z_BUFSIZE; - if (offset < Z_BUFSIZE) size = (uInt)offset; - - size = gzwrite(file, s->inbuf, size); - if (size == 0) return -1L; - - offset -= size; - } - return s->in; -#endif - } - /* Rest of function is for reading only */ - - /* compute absolute position */ - if (whence == SEEK_CUR) { - offset += s->out; - } - if (offset < 0) return -1L; - - if (s->transparent) { - /* map to fseek */ - s->back = EOF; - s->stream.avail_in = 0; - s->stream.next_in = s->inbuf; - if (fseek(s->file, offset, SEEK_SET) < 0) return -1L; - - s->in = s->out = offset; - return offset; - } - - /* For a negative seek, rewind and use positive seek */ - if (offset >= s->out) { - offset -= s->out; - } else if (gzrewind(file) < 0) { - return -1L; - } - /* offset is now the number of bytes to skip. */ - - if (offset != 0 && s->outbuf == Z_NULL) { - s->outbuf = (Byte*)ALLOC(Z_BUFSIZE); - if (s->outbuf == Z_NULL) return -1L; - } - if (offset && s->back != EOF) { - s->back = EOF; - s->out++; - offset--; - if (s->last) s->z_err = Z_STREAM_END; - } - while (offset > 0) { - int size = Z_BUFSIZE; - if (offset < Z_BUFSIZE) size = (int)offset; - - size = gzread(file, s->outbuf, (uInt)size); - if (size <= 0) return -1L; - offset -= size; - } - return s->out; -} - -/* =========================================================================== - Rewinds input file. -*/ -int ZEXPORT -gzrewind(gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r') return -1; - - s->z_err = Z_OK; - s->z_eof = 0; - s->back = EOF; - s->stream.avail_in = 0; - s->stream.next_in = s->inbuf; - s->crc = z_crc32(0L, Z_NULL, 0); - if (!s->transparent) (void)inflateReset(&s->stream); - s->in = 0; - s->out = 0; - return fseek(s->file, s->start, SEEK_SET); -} - -/* =========================================================================== - Returns the starting position for the next gzread or gzwrite on the - given compressed file. This position represents a number of bytes in the - uncompressed data stream. -*/ -z_off_t ZEXPORT -gztell(gzFile file) -{ - return gzseek(file, 0L, SEEK_CUR); -} - -/* =========================================================================== - Returns 1 when EOF has previously been detected reading the given - input stream, otherwise zero. -*/ -int ZEXPORT -gzeof(gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - /* With concatenated compressed files that can have embedded - * crc trailers, z_eof is no longer the only/best indicator of EOF - * on a gz_stream. Handle end-of-stream error explicitly here. - */ - if (s == NULL || s->mode != 'r') return 0; - if (s->z_eof) return 1; - return s->z_err == Z_STREAM_END; -} - -/* =========================================================================== - Returns 1 if reading and doing so transparently, otherwise zero. -*/ -int ZEXPORT -gzdirect(gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r') return 0; - return s->transparent; -} - -/* =========================================================================== - Outputs a long in LSB order to the given file -*/ -local void -putLong(FILE *file, uLong x) -{ - int n; - for (n = 0; n < 4; n++) { - fputc((int)(x & 0xff), file); - x >>= 8; - } -} - -/* =========================================================================== - Reads a long in LSB order from the given gz_stream. Sets z_err in case - of error. -*/ -local uLong -getLong(gz_stream *s) -{ - uLong x = (uLong)get_byte(s); - int c; - - x += ((uLong)get_byte(s))<<8; - x += ((uLong)get_byte(s))<<16; - c = get_byte(s); - if (c == EOF) s->z_err = Z_DATA_ERROR; - x += ((uLong)c)<<24; - return x; -} - -/* =========================================================================== - Flushes all pending output if necessary, closes the compressed file - and deallocates all the (de)compression state. -*/ -int ZEXPORT -gzclose(gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL) return Z_STREAM_ERROR; - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - return Z_STREAM_ERROR; -#else - if (do_flush (file, Z_FINISH) != Z_OK) - return destroy((gz_stream*)file); - - putLong (s->file, s->crc); - putLong (s->file, (uLong)(s->in & 0xffffffff)); -#endif - } - return destroy((gz_stream*)file); -} - -#ifdef STDC -# define zstrerror(errnum) strerror(errnum) -#else -# define zstrerror(errnum) "" -#endif - -/* =========================================================================== - Returns the error message for the last error which occurred on the - given compressed file. errnum is set to zlib error number. If an - error occurred in the file system and not in the compression library, - errnum is set to Z_ERRNO and the application may consult errno - to get the exact error code. -*/ -const char * ZEXPORT -gzerror(gzFile file, int *errnum) -{ - char *m; - gz_stream *s = (gz_stream*)file; - - if (s == NULL) { - *errnum = Z_STREAM_ERROR; - return (const char*)ERR_MSG(Z_STREAM_ERROR); - } - *errnum = s->z_err; - if (*errnum == Z_OK) return (const char*)""; - - m = (char*)(*errnum == Z_ERRNO ? zstrerror(errno) : s->stream.msg); - - if (m == NULL || *m == '\0') m = (char*)ERR_MSG(s->z_err); - - TRYFREE(s->msg); - s->msg = (char*)ALLOC(strlen(s->path) + strlen(m) + 3); - if (s->msg == Z_NULL) return (const char*)ERR_MSG(Z_MEM_ERROR); - strcpy(s->msg, s->path); - strcat(s->msg, ": "); - strcat(s->msg, m); - return (const char*)s->msg; -} - -/* =========================================================================== - Clear the error and end-of-file flags, and do the same for the real file. -*/ -void ZEXPORT -gzclearerr(gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL) return; - if (s->z_err != Z_STREAM_END) s->z_err = Z_OK; - s->z_eof = 0; - clearerr(s->file); -} diff --git a/libsa/bootstrap.cpp b/libsa/bootstrap.cpp index 1e60c8ccf..80a7508f3 100644 --- a/libsa/bootstrap.cpp +++ b/libsa/bootstrap.cpp @@ -30,6 +30,9 @@ extern "C" { #include #include +#if CONFIG_EMBEDDED +extern uuid_t kernelcache_uuid; +#endif } #include @@ -243,6 +246,9 @@ KLDBootstrap::readPrelinkedExtensions( OSDictionary * prelinkInfoDict = NULL; // do not release OSString * errorString = NULL; // must release OSKext * theKernel = NULL; // must release +#if CONFIG_EMBEDDED + OSData * kernelcacheUUID = NULL; // do not release +#endif kernel_segment_command_t * prelinkTextSegment = NULL; // see code kernel_segment_command_t * prelinkInfoSegment = NULL; // see code @@ -368,6 +374,19 @@ KLDBootstrap::readPrelinkedExtensions( ramDiskBoot = IORamDiskBSDRoot(); #endif /* NO_KEXTD */ +#if CONFIG_EMBEDDED + /* Copy in the kernelcache UUID */ + kernelcacheUUID = OSDynamicCast(OSData, + prelinkInfoDict->getObject(kPrelinkInfoKCIDKey)); + if (!kernelcacheUUID) { + bzero(&kernelcache_uuid, sizeof(kernelcache_uuid)); + } else if (kernelcacheUUID->getLength() != sizeof(kernelcache_uuid)) { + panic("kernelcacheUUID length is %d, expected %lu", kernelcacheUUID->getLength(), + sizeof(kernelcache_uuid)); + } else { + memcpy((void *)&kernelcache_uuid, (void *)kernelcacheUUID->getBytesNoCopy(), kernelcacheUUID->getLength()); + } +#endif /* CONFIG_EMBEDDED */ infoDictArray = OSDynamicCast(OSArray, prelinkInfoDict->getObject(kPrelinkInfoDictionaryKey)); @@ -383,7 +402,9 @@ KLDBootstrap::readPrelinkedExtensions( /* Create dictionary of excluded kexts */ +#ifndef CONFIG_EMBEDDED OSKext::createExcludeListFromPrelinkInfo(infoDictArray); +#endif /* Create OSKext objects for each info dictionary. */ for (i = 0; i < infoDictArray->getCount(); ++i) { @@ -432,7 +453,13 @@ KLDBootstrap::readPrelinkedExtensions( OSNumber *lengthNum = OSDynamicCast(OSNumber, infoDict->getObject(kPrelinkExecutableSizeKey)); if (addressNum && lengthNum) { +#if __arm__ || __arm64__ + vm_offset_t data = (vm_offset_t) ((addressNum->unsigned64BitValue()) + vm_kernel_slide); + vm_size_t length = (vm_size_t) (lengthNum->unsigned32BitValue()); + ml_static_mfree(data, length); +#else #error Pick the right way to free prelinked data on this arch +#endif } infoDictArray->removeObject(i--); @@ -605,7 +632,9 @@ KLDBootstrap::readBooterExtensions(void) /* Create dictionary of excluded kexts */ +#ifndef CONFIG_EMBEDDED OSKext::createExcludeListFromBooterData(propertyDict, keyIterator); +#endif keyIterator->reset(); while ( ( deviceTreeName = diff --git a/libsa/conf/Makefile.arm b/libsa/conf/Makefile.arm new file mode 100644 index 000000000..1c1cef911 --- /dev/null +++ b/libsa/conf/Makefile.arm @@ -0,0 +1,10 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# Bootstrap __KLD files must be Mach-O for "setsegname" +$(foreach file,$(OBJS),$(eval $(file)_CFLAGS_ADD += $(CFLAGS_NOLTO_FLAG))) + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/libsa/conf/Makefile.arm64 b/libsa/conf/Makefile.arm64 new file mode 100644 index 000000000..1c1cef911 --- /dev/null +++ b/libsa/conf/Makefile.arm64 @@ -0,0 +1,10 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# Bootstrap __KLD files must be Mach-O for "setsegname" +$(foreach file,$(OBJS),$(eval $(file)_CFLAGS_ADD += $(CFLAGS_NOLTO_FLAG))) + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/libsa/conf/Makefile.template b/libsa/conf/Makefile.template index ebfd0d2a8..bc570dde5 100644 --- a/libsa/conf/Makefile.template +++ b/libsa/conf/Makefile.template @@ -17,6 +17,7 @@ include $(MakeInc_def) # CFLAGS # CFLAGS+= -include meta_features.h -DLIBSA_KERNEL_PRIVATE +SFLAGS+= -include meta_features.h # # Directories for mig generated files diff --git a/libsa/conf/files.arm b/libsa/conf/files.arm new file mode 100644 index 000000000..e69de29bb diff --git a/libsa/conf/files.arm64 b/libsa/conf/files.arm64 new file mode 100644 index 000000000..e69de29bb diff --git a/libsa/lastkerneldataconst.c b/libsa/lastkerneldataconst.c index 9b8db0b51..580756f0f 100644 --- a/libsa/lastkerneldataconst.c +++ b/libsa/lastkerneldataconst.c @@ -38,7 +38,12 @@ * alignment and no straight forward way to specify section ordering. */ +#if defined(__arm64__) +/* PAGE_SIZE on ARM64 is an expression derived from a non-const global variable */ +#define PAD_SIZE PAGE_MAX_SIZE +#else #define PAD_SIZE PAGE_SIZE +#endif static const uint8_t __attribute__((section("__DATA,__const"))) data_const_padding[PAD_SIZE] = {[0 ... PAD_SIZE-1] = 0xFF}; const vm_offset_t __attribute__((section("__DATA,__data"))) _lastkerneldataconst = (vm_offset_t)&data_const_padding[0]; diff --git a/libsyscall/Libsyscall.xcconfig b/libsyscall/Libsyscall.xcconfig index 181fe1f4e..105571937 100644 --- a/libsyscall/Libsyscall.xcconfig +++ b/libsyscall/Libsyscall.xcconfig @@ -1,7 +1,7 @@ #include "/Makefiles/CoreOS/Xcode/BSD.xcconfig" BUILD_VARIANTS = normal -SUPPORTED_PLATFORMS = macosx iphoneos iphoneosnano tvos appletvos watchos +SUPPORTED_PLATFORMS = macosx iphoneos iphoneosnano tvos appletvos watchos bridgeos ONLY_ACTIVE_ARCH = NO DEAD_CODE_STRIPPING = YES DEBUG_INFORMATION_FORMAT = dwarf-with-dsym @@ -20,6 +20,7 @@ OTHER_CFLAGS[sdk=iphoneos*] = $(inherited) -DNO_SYSCALL_LEGACY OTHER_CFLAGS[sdk=watchos*] = $(inherited) -DNO_SYSCALL_LEGACY OTHER_CFLAGS[sdk=tvos*] = $(inherited) -DNO_SYSCALL_LEGACY OTHER_CFLAGS[sdk=appletvos*] = $(inherited) -DNO_SYSCALL_LEGACY +OTHER_CFLAGS[sdk=bridgeos*] = $(inherited) -DNO_SYSCALL_LEGACY GCC_PREPROCESSOR_DEFINITIONS = CF_OPEN_SOURCE CF_EXCLUDE_CSTD_HEADERS DEBUG _FORTIFY_SOURCE=0 HEADER_SEARCH_PATHS = $(PROJECT_DIR)/mach $(PROJECT_DIR)/os $(PROJECT_DIR)/wrappers $(PROJECT_DIR)/wrappers/string $(PROJECT_DIR)/wrappers/libproc $(PROJECT_DIR)/wrappers/libproc/spawn $(BUILT_PRODUCTS_DIR)/internal_hdr/include $(BUILT_PRODUCTS_DIR)/mig_hdr/local/include $(BUILT_PRODUCTS_DIR)/mig_hdr/include $(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders WARNING_CFLAGS = -Wmost @@ -27,11 +28,12 @@ GCC_TREAT_WARNINGS_AS_ERRORS = YES GCC_WARN_ABOUT_MISSING_NEWLINE = YES CODE_SIGN_IDENTITY = - DYLIB_CURRENT_VERSION = $(RC_ProjectSourceVersion) -DYLIB_LDFLAGS = -umbrella System -all_load +DYLIB_LDFLAGS = -umbrella System -all_load -lCrashReporterClient DYLIB_LDFLAGS[sdk=iphoneos*] = $(inherited) -Wl,-sectalign,__DATA,__data,1000 DYLIB_LDFLAGS[sdk=watchos*] = $(inherited) -Wl,-sectalign,__DATA,__data,1000 DYLIB_LDFLAGS[sdk=tvos*] = $(inherited) -Wl,-sectalign,__DATA,__data,1000 DYLIB_LDFLAGS[sdk=appletvos*] = $(inherited) -Wl,-sectalign,__DATA,__data,1000 +DYLIB_LDFLAGS[sdk=bridgeos*] = $(inherited) -Wl,-sectalign,__DATA,__data,1000 OTHER_LDFLAGS = INSTALLHDRS_SCRIPT_PHASE = YES INSTALLHDRS_COPY_PHASE = YES diff --git a/libsyscall/Libsyscall.xcodeproj/project.pbxproj b/libsyscall/Libsyscall.xcodeproj/project.pbxproj index 1f54c0158..8175500a0 100644 --- a/libsyscall/Libsyscall.xcodeproj/project.pbxproj +++ b/libsyscall/Libsyscall.xcodeproj/project.pbxproj @@ -111,6 +111,11 @@ 3F538F891A659C5600B37EFD /* persona.c in Sources */ = {isa = PBXBuildFile; fileRef = 3F538F881A659C5600B37EFD /* persona.c */; }; 401BB71A1BCAE57B005080D3 /* os_channel.c in Sources */ = {isa = PBXBuildFile; fileRef = 401BB7161BCAE539005080D3 /* os_channel.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; 401BB71C1BCAE57B005080D3 /* os_nexus.c in Sources */ = {isa = PBXBuildFile; fileRef = 401BB7181BCAE539005080D3 /* os_nexus.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; + 402AF43F1E5CD88600F1A4B9 /* cpu_in_cksum_gen.c in Sources */ = {isa = PBXBuildFile; fileRef = 402AF43E1E5CD88100F1A4B9 /* cpu_in_cksum_gen.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; + 403C7CEE1E1F4E4400D6FEEF /* os_packet.c in Sources */ = {isa = PBXBuildFile; fileRef = 405FA3381E0C669D007D66EA /* os_packet.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; + 406E0B721E4ACD2000295EA3 /* cpu_copy_in_cksum.s in Sources */ = {isa = PBXBuildFile; fileRef = 40DD162F1E4ACCAA003297CC /* cpu_copy_in_cksum.s */; }; + 409A78321E4EB3E300E0699B /* cpu_in_cksum.s in Sources */ = {isa = PBXBuildFile; fileRef = 409A78301E4EB3D900E0699B /* cpu_in_cksum.s */; }; + 40DF0F741E5CD7BB0035A864 /* cpu_copy_in_cksum_gen.c in Sources */ = {isa = PBXBuildFile; fileRef = 40DF0F731E5CD7B30035A864 /* cpu_copy_in_cksum_gen.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; 435F3CAA1B06B7BA005ED9EF /* work_interval.c in Sources */ = {isa = PBXBuildFile; fileRef = 435F3CA91B06B7BA005ED9EF /* work_interval.c */; }; 467DAFD4157E8AF200CE68F0 /* guarded_open_np.c in Sources */ = {isa = PBXBuildFile; fileRef = 467DAFD3157E8AF200CE68F0 /* guarded_open_np.c */; }; 4BDD5F1D1891AB2F004BF300 /* mach_approximate_time.c in Sources */ = {isa = PBXBuildFile; fileRef = 4BDD5F1B1891AB2F004BF300 /* mach_approximate_time.c */; }; @@ -123,6 +128,7 @@ 74F3290B18EB269400B2B70E /* vm_page_size.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = 7466C923170CB99B004557CC /* vm_page_size.h */; }; 7AE28FDF18AC41B1006A5626 /* csr.c in Sources */ = {isa = PBXBuildFile; fileRef = 7AE28FDE18AC41B1006A5626 /* csr.c */; }; 9002401118FC9A7F00D73BFA /* renamex.c in Sources */ = {isa = PBXBuildFile; fileRef = 906AA2D018F74CD1001C681A /* renamex.c */; }; + 92197BAF1EAD8F2C003994B9 /* utimensat.c in Sources */ = {isa = PBXBuildFile; fileRef = 92197BAE1EAD8DF2003994B9 /* utimensat.c */; }; 925559921CBC23C300E527CE /* mach_boottime.c in Sources */ = {isa = PBXBuildFile; fileRef = 925559911CBBBBB300E527CE /* mach_boottime.c */; }; 928336A11B83ED9100873B90 /* thread_register_state.c in Sources */ = {isa = PBXBuildFile; fileRef = 928336A01B83ED7800873B90 /* thread_register_state.c */; }; 9299E14A1B841E74005B7350 /* thread_state.h in Headers */ = {isa = PBXBuildFile; fileRef = 928336A21B8412C100873B90 /* thread_state.h */; }; @@ -130,6 +136,11 @@ 929FD46F1C5711DB0087B9C8 /* mach_timebase_info.c in Sources */ = {isa = PBXBuildFile; fileRef = 929FD46E1C5711CF0087B9C8 /* mach_timebase_info.c */; }; 978228281B8678DC008385AC /* pselect-darwinext.c in Sources */ = {isa = PBXBuildFile; fileRef = 978228271B8678CB008385AC /* pselect-darwinext.c */; }; 978228291B8678DF008385AC /* pselect-darwinext-cancel.c in Sources */ = {isa = PBXBuildFile; fileRef = 978228261B8678C2008385AC /* pselect-darwinext-cancel.c */; }; + 9CCF28271E68E993002EE6CD /* pid_shutdown_networking.c in Sources */ = {isa = PBXBuildFile; fileRef = 9CCF28261E68E993002EE6CD /* pid_shutdown_networking.c */; }; + A50845861DDA69AC0041C0E0 /* thread_self_restrict.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */; }; + A50845871DDA69C90041C0E0 /* thread_self_restrict.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */; }; + A50BD52F1DDA548F006622C8 /* thread_self_restrict.h in Headers */ = {isa = PBXBuildFile; fileRef = A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */; }; + A50BD5301DDA5500006622C8 /* thread_self_restrict.h in Headers */ = {isa = PBXBuildFile; fileRef = A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */; }; A59CB95616669EFB00B064B3 /* stack_logging_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = A59CB95516669DB700B064B3 /* stack_logging_internal.h */; }; A59CB9581666A1A200B064B3 /* munmap.c in Sources */ = {isa = PBXBuildFile; fileRef = A59CB9571666A1A200B064B3 /* munmap.c */; }; BA0D9FB1199031AD007E8A73 /* kdebug_trace.c in Sources */ = {isa = PBXBuildFile; fileRef = BA0D9FB0199031AD007E8A73 /* kdebug_trace.c */; }; @@ -239,7 +250,6 @@ E453AF3917013F1B00F2C94C /* spawn_private.h in Headers */ = {isa = PBXBuildFile; fileRef = E4D45C3E16FB20970002AF25 /* spawn_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; E453AF3A17013F4C00F2C94C /* stack_logging_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = A59CB95516669DB700B064B3 /* stack_logging_internal.h */; }; E4D45C2416F856900002AF25 /* __commpage_gettimeofday.c in Sources */ = {isa = PBXBuildFile; fileRef = E4D45C2116F856900002AF25 /* __commpage_gettimeofday.c */; }; - E4D45C2516F856900002AF25 /* __commpage_gettimeofday.s in Sources */ = {isa = PBXBuildFile; fileRef = E4D45C2216F856900002AF25 /* __commpage_gettimeofday.s */; }; E4D45C2616F856900002AF25 /* mach_absolute_time.s in Sources */ = {isa = PBXBuildFile; fileRef = E4D45C2316F856900002AF25 /* mach_absolute_time.s */; }; E4D45C2E16F868ED0002AF25 /* libproc.c in Sources */ = {isa = PBXBuildFile; fileRef = E4D45C2A16F868ED0002AF25 /* libproc.c */; }; E4D45C2F16F868ED0002AF25 /* libproc.h in Headers */ = {isa = PBXBuildFile; fileRef = E4D45C2B16F868ED0002AF25 /* libproc.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -377,6 +387,7 @@ dstPath = "$(OS_PRIVATE_HEADERS_FOLDER_PATH)"; dstSubfolderSpec = 0; files = ( + A50845871DDA69C90041C0E0 /* thread_self_restrict.h in CopyFiles */, C9FD8508166D6BD400963B73 /* tsd.h in CopyFiles */, ); runOnlyForDeploymentPostprocessing = 1; @@ -387,6 +398,7 @@ dstPath = "$(OS_PRIVATE_HEADERS_FOLDER_PATH)"; dstSubfolderSpec = 0; files = ( + A50845861DDA69AC0041C0E0 /* thread_self_restrict.h in CopyFiles */, C9A3D6EB1672AD1000A5CAA3 /* tsd.h in CopyFiles */, ); runOnlyForDeploymentPostprocessing = 1; @@ -484,6 +496,11 @@ 3F538F881A659C5600B37EFD /* persona.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = persona.c; sourceTree = ""; }; 401BB7161BCAE539005080D3 /* os_channel.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = os_channel.c; path = skywalk/os_channel.c; sourceTree = ""; }; 401BB7181BCAE539005080D3 /* os_nexus.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = os_nexus.c; path = skywalk/os_nexus.c; sourceTree = ""; }; + 402AF43E1E5CD88100F1A4B9 /* cpu_in_cksum_gen.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = cpu_in_cksum_gen.c; path = skywalk/cpu_in_cksum_gen.c; sourceTree = ""; }; + 405FA3381E0C669D007D66EA /* os_packet.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = os_packet.c; path = skywalk/os_packet.c; sourceTree = ""; }; + 409A78301E4EB3D900E0699B /* cpu_in_cksum.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; name = cpu_in_cksum.s; path = skywalk/cpu_in_cksum.s; sourceTree = ""; }; + 40DD162F1E4ACCAA003297CC /* cpu_copy_in_cksum.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; name = cpu_copy_in_cksum.s; path = skywalk/cpu_copy_in_cksum.s; sourceTree = ""; }; + 40DF0F731E5CD7B30035A864 /* cpu_copy_in_cksum_gen.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = cpu_copy_in_cksum_gen.c; path = skywalk/cpu_copy_in_cksum_gen.c; sourceTree = ""; }; 435F3CA91B06B7BA005ED9EF /* work_interval.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = work_interval.c; sourceTree = ""; }; 467DAFD3157E8AF200CE68F0 /* guarded_open_np.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = guarded_open_np.c; sourceTree = ""; }; 4BDD5F1B1891AB2F004BF300 /* mach_approximate_time.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = mach_approximate_time.c; sourceTree = ""; }; @@ -493,12 +510,15 @@ 7466C923170CB99B004557CC /* vm_page_size.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = vm_page_size.h; sourceTree = ""; }; 7AE28FDE18AC41B1006A5626 /* csr.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = csr.c; sourceTree = ""; }; 906AA2D018F74CD1001C681A /* renamex.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = renamex.c; sourceTree = ""; }; + 92197BAE1EAD8DF2003994B9 /* utimensat.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = utimensat.c; sourceTree = ""; }; 925559911CBBBBB300E527CE /* mach_boottime.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = mach_boottime.c; sourceTree = ""; }; 928336A01B83ED7800873B90 /* thread_register_state.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = thread_register_state.c; sourceTree = ""; }; 928336A21B8412C100873B90 /* thread_state.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = thread_state.h; sourceTree = ""; }; 929FD46E1C5711CF0087B9C8 /* mach_timebase_info.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = mach_timebase_info.c; sourceTree = ""; }; 978228261B8678C2008385AC /* pselect-darwinext-cancel.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "pselect-darwinext-cancel.c"; sourceTree = ""; }; 978228271B8678CB008385AC /* pselect-darwinext.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "pselect-darwinext.c"; sourceTree = ""; }; + 9CCF28261E68E993002EE6CD /* pid_shutdown_networking.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pid_shutdown_networking.c; sourceTree = ""; }; + A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = thread_self_restrict.h; sourceTree = ""; }; A59CB95516669DB700B064B3 /* stack_logging_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = stack_logging_internal.h; sourceTree = ""; }; A59CB9571666A1A200B064B3 /* munmap.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = munmap.c; sourceTree = ""; }; BA0D9FB0199031AD007E8A73 /* kdebug_trace.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = kdebug_trace.c; sourceTree = ""; }; @@ -521,7 +541,6 @@ C99A4F4E1305B1B70054B7B7 /* __get_cpu_capabilities.s */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.asm; path = __get_cpu_capabilities.s; sourceTree = ""; }; C99A4F511305B43F0054B7B7 /* init_cpu_capabilities.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = init_cpu_capabilities.c; sourceTree = ""; }; C9C1824F15338C0B00933F23 /* alloc_once.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = alloc_once.c; sourceTree = ""; }; - C9D9BCBF114B00600000D8B9 /* .open_source_exclude */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = .open_source_exclude; sourceTree = ""; }; C9D9BCC5114B00600000D8B9 /* clock_priv.defs */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.mig; path = clock_priv.defs; sourceTree = ""; }; C9D9BCC6114B00600000D8B9 /* clock_reply.defs */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.mig; path = clock_reply.defs; sourceTree = ""; }; C9D9BCC7114B00600000D8B9 /* clock_sleep.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = clock_sleep.c; sourceTree = ""; }; @@ -588,7 +607,6 @@ E4216C301822D404006F2632 /* mach_voucher.defs */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.mig; path = mach_voucher.defs; sourceTree = ""; }; E453AF341700FD3C00F2C94C /* getiopolicy_np.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = getiopolicy_np.c; sourceTree = ""; }; E4D45C2116F856900002AF25 /* __commpage_gettimeofday.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = __commpage_gettimeofday.c; sourceTree = ""; }; - E4D45C2216F856900002AF25 /* __commpage_gettimeofday.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; path = __commpage_gettimeofday.s; sourceTree = ""; }; E4D45C2316F856900002AF25 /* mach_absolute_time.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; path = mach_absolute_time.s; sourceTree = ""; }; E4D45C2A16F868ED0002AF25 /* libproc.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = libproc.c; sourceTree = ""; }; E4D45C2B16F868ED0002AF25 /* libproc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = libproc.h; sourceTree = ""; }; @@ -649,7 +667,6 @@ 08FB7795FE84155DC02AAC07 /* mach */ = { isa = PBXGroup; children = ( - C9D9BCBE114B00600000D8B9 /* arm */, 247A08FF11F8E18000E4693F /* abort.h */, C9D9BCC5114B00600000D8B9 /* clock_priv.defs */, C9D9BCC6114B00600000D8B9 /* clock_reply.defs */, @@ -751,14 +768,20 @@ isa = PBXGroup; children = ( E4D45C2116F856900002AF25 /* __commpage_gettimeofday.c */, + C99A4F4E1305B1B70054B7B7 /* __get_cpu_capabilities.s */, + 24A7C5CB11FF973C007669EB /* _errno.h */, 24E47824120881DF009A384D /* _libc_funcptr.c */, 247A08B311F8B05900E4693F /* _libkernel_init.c */, + 247A08B211F8B05900E4693F /* _libkernel_init.h */, + 248BA04A121C8EE4008C073F /* cancelable */, FB50F1B315AB7DE700F814BA /* carbon_delete.c */, E214BDC71C2E34E200CEE8A3 /* clonefile.c */, 2BA88DCB1810A3CE00EB63F6 /* coalition.c */, 7AE28FDE18AC41B1006A5626 /* csr.c */, E2A0F3331C3B17D100A11F8A /* fs_snapshot.c */, + C6C40121174154D9000AE69F /* gethostuuid_private.h */, C6C4012017415384000AE69F /* gethostuuid.c */, + C639F0E41741C09A00A39F47 /* gethostuuid.h */, E453AF341700FD3C00F2C94C /* getiopolicy_np.c */, 72B1E6EC190723DB00FB3FA2 /* guarded_open_dprotected_np.c */, 467DAFD3157E8AF200CE68F0 /* guarded_open_np.c */, @@ -766,13 +789,18 @@ 248BA07F121DA36B008C073F /* ioctl.c */, BA0D9FB0199031AD007E8A73 /* kdebug_trace.c */, 248BA081121DA4F3008C073F /* kill.c */, + 24A7C6951200AF8A007669EB /* legacy */, + E4D45C2916F868ED0002AF25 /* libproc */, + E4D45C2316F856900002AF25 /* mach_absolute_time.s */, 4BDD5F1B1891AB2F004BF300 /* mach_approximate_time.c */, + 4BDD5F1C1891AB2F004BF300 /* mach_approximate_time.s */, 925559911CBBBBB300E527CE /* mach_boottime.c */, 72FB18801B437F7A00181A5B /* mach_continuous_time.c */, 14FE60EB1B7D3BED00ACB44C /* mach_get_times.c */, 929FD46E1C5711CF0087B9C8 /* mach_timebase_info.c */, 030B179A135377B400DAD1F0 /* open_dprotected_np.c */, 3F538F881A659C5600B37EFD /* persona.c */, + 9CCF28261E68E993002EE6CD /* pid_shutdown_networking.c */, C6BEE9171806840200D25AAB /* posix_sem_obsolete.c */, BA9973461C3B4C8A00B14D8C /* quota_obsolete.c */, 24B8C2611237F53900D36CC3 /* remove-counter.c */, @@ -784,28 +812,18 @@ C962B16B18DBA2C80031244A /* setpriority.c */, C6460B7B182025DF00F73CCA /* sfi.c */, 24B223B3121DFF12007DAEDE /* sigsuspend-base.c */, + 401BB7141BCAE523005080D3 /* skywalk */, + E4D45C3B16FB20970002AF25 /* spawn */, 13B598931A142F5900DB2D5A /* stackshot.c */, + E4D7E55216F8776300F92D8D /* string */, 13D932CB1C7B9DE600158FA1 /* terminate_with_reason.c */, 928336A01B83ED7800873B90 /* thread_register_state.c */, + 2419382912135FE1003CDE41 /* unix03 */, 248AA962122C7B2A0085F5B1 /* unlink.c */, 29A59AE5183B110C00E8B896 /* unlinkat.c */, - 435F3CA91B06B7BA005ED9EF /* work_interval.c */, - 24A7C5CB11FF973C007669EB /* _errno.h */, - 247A08B211F8B05900E4693F /* _libkernel_init.h */, - C6C40121174154D9000AE69F /* gethostuuid_private.h */, - C639F0E41741C09A00A39F47 /* gethostuuid.h */, - E4D45C2216F856900002AF25 /* __commpage_gettimeofday.s */, - C99A4F4E1305B1B70054B7B7 /* __get_cpu_capabilities.s */, - E4D45C2316F856900002AF25 /* mach_absolute_time.s */, - 4BDD5F1C1891AB2F004BF300 /* mach_approximate_time.s */, + 92197BAE1EAD8DF2003994B9 /* utimensat.c */, 374A36E214748EE400AAF39D /* varargs_wrappers.s */, - 248BA04A121C8EE4008C073F /* cancelable */, - 24A7C6951200AF8A007669EB /* legacy */, - E4D45C2916F868ED0002AF25 /* libproc */, - 401BB7141BCAE523005080D3 /* skywalk */, - E4D45C3B16FB20970002AF25 /* spawn */, - E4D7E55216F8776300F92D8D /* string */, - 2419382912135FE1003CDE41 /* unix03 */, + 435F3CA91B06B7BA005ED9EF /* work_interval.c */, ); path = wrappers; sourceTree = ""; @@ -943,6 +961,11 @@ 401BB7141BCAE523005080D3 /* skywalk */ = { isa = PBXGroup; children = ( + 405FA3381E0C669D007D66EA /* os_packet.c */, + 40DD162F1E4ACCAA003297CC /* cpu_copy_in_cksum.s */, + 409A78301E4EB3D900E0699B /* cpu_in_cksum.s */, + 40DF0F731E5CD7B30035A864 /* cpu_copy_in_cksum_gen.c */, + 402AF43E1E5CD88100F1A4B9 /* cpu_in_cksum_gen.c */, 401BB7161BCAE539005080D3 /* os_channel.c */, 401BB7181BCAE539005080D3 /* os_nexus.c */, ); @@ -964,18 +987,11 @@ children = ( C9C1824F15338C0B00933F23 /* alloc_once.c */, C9EE57F51669673D00337E4B /* tsd.h */, + A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */, ); path = os; sourceTree = ""; }; - C9D9BCBE114B00600000D8B9 /* arm */ = { - isa = PBXGroup; - children = ( - C9D9BCBF114B00600000D8B9 /* .open_source_exclude */, - ); - path = arm; - sourceTree = ""; - }; C9D9BCD8114B00600000D8B9 /* mach */ = { isa = PBXGroup; children = ( @@ -1061,6 +1077,7 @@ C6D3EFBC16542C510052CF30 /* mach_interface.h in Headers */, C6D3EFBD16542C510052CF30 /* port_obj.h in Headers */, C6D3EFBE16542C510052CF30 /* sync.h in Headers */, + A50BD52F1DDA548F006622C8 /* thread_self_restrict.h in Headers */, C6D3EFC116542C510052CF30 /* vm_task.h in Headers */, C6D3EFC216542C510052CF30 /* key_defs.h in Headers */, C6D3EFC316542C510052CF30 /* ls_defs.h in Headers */, @@ -1095,6 +1112,7 @@ C9D9BD29114B00600000D8B9 /* mach_interface.h in Headers */, C9D9BD2B114B00600000D8B9 /* port_obj.h in Headers */, C9D9BD2C114B00600000D8B9 /* sync.h in Headers */, + A50BD5301DDA5500006622C8 /* thread_self_restrict.h in Headers */, C9D9BD2F114B00600000D8B9 /* vm_task.h in Headers */, C9D9BD50114B00600000D8B9 /* key_defs.h in Headers */, C9D9BD51114B00600000D8B9 /* ls_defs.h in Headers */, @@ -1283,12 +1301,14 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 403C7CEE1E1F4E4400D6FEEF /* os_packet.c in Sources */, E214BDC81C2E358300CEE8A3 /* clonefile.c in Sources */, C9D9BD19114B00600000D8B9 /* clock_priv.defs in Sources */, C9D9BD1A114B00600000D8B9 /* clock_reply.defs in Sources */, C9D9BD1C114B00600000D8B9 /* clock.defs in Sources */, C9D9BD22114B00600000D8B9 /* exc.defs in Sources */, C9D9BD30114B00600000D8B9 /* host_priv.defs in Sources */, + 409A78321E4EB3E300E0699B /* cpu_in_cksum.s in Sources */, C9D9BD31114B00600000D8B9 /* host_security.defs in Sources */, C9D9BD35114B00600000D8B9 /* lock_set.defs in Sources */, C9D9BD38114B00600000D8B9 /* mach_host.defs in Sources */, @@ -1319,7 +1339,6 @@ C9D9BD3C114B00600000D8B9 /* mach_msg.c in Sources */, C9D9BD3E114B00600000D8B9 /* mach_traps.s in Sources */, C9D9BD41114B00600000D8B9 /* mig_allocate.c in Sources */, - E4D45C2516F856900002AF25 /* __commpage_gettimeofday.s in Sources */, C9D9BD42114B00600000D8B9 /* mig_deallocate.c in Sources */, BA9973471C3B4C9A00B14D8C /* quota_obsolete.c in Sources */, E4D45C2416F856900002AF25 /* __commpage_gettimeofday.c in Sources */, @@ -1348,6 +1367,7 @@ 24A7C5C211FF8DA6007669EB /* lchown.c in Sources */, 24A7C5C311FF8DA6007669EB /* listen.c in Sources */, 24A7C5C411FF8DA6007669EB /* recvfrom.c in Sources */, + 92197BAF1EAD8F2C003994B9 /* utimensat.c in Sources */, C962B16E18DBB43F0031244A /* thread_act.c in Sources */, 24A7C5C511FF8DA6007669EB /* recvmsg.c in Sources */, 24A7C5C611FF8DA6007669EB /* sendmsg.c in Sources */, @@ -1373,6 +1393,7 @@ C6BEE9181806840200D25AAB /* posix_sem_obsolete.c in Sources */, 248BA082121DA4F3008C073F /* kill.c in Sources */, 248BA085121DA5E4008C073F /* kill.c in Sources */, + 9CCF28271E68E993002EE6CD /* pid_shutdown_networking.c in Sources */, 2BA88DCC1810A3CE00EB63F6 /* coalition.c in Sources */, 248BA087121DA72D008C073F /* mmap.c in Sources */, 7AE28FDF18AC41B1006A5626 /* csr.c in Sources */, @@ -1398,10 +1419,13 @@ 248AA965122C7C330085F5B1 /* rmdir.c in Sources */, 435F3CAA1B06B7BA005ED9EF /* work_interval.c in Sources */, 248AA967122C7CDA0085F5B1 /* rename.c in Sources */, + 406E0B721E4ACD2000295EA3 /* cpu_copy_in_cksum.s in Sources */, 24B8C2621237F53900D36CC3 /* remove-counter.c in Sources */, C99A4F501305B2BD0054B7B7 /* __get_cpu_capabilities.s in Sources */, 978228291B8678DF008385AC /* pselect-darwinext-cancel.c in Sources */, + 40DF0F741E5CD7BB0035A864 /* cpu_copy_in_cksum_gen.c in Sources */, C99A4F531305B43F0054B7B7 /* init_cpu_capabilities.c in Sources */, + 402AF43F1E5CD88600F1A4B9 /* cpu_in_cksum_gen.c in Sources */, 030B179B135377B400DAD1F0 /* open_dprotected_np.c in Sources */, E4D45C3116F868ED0002AF25 /* proc_listpidspath.c in Sources */, 374A36E314748F1300AAF39D /* varargs_wrappers.s in Sources */, @@ -1480,6 +1504,7 @@ DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; MAP_PLATFORM = "$(MAP_PLATFORM_$(PLATFORM_NAME))"; MAP_PLATFORM_appletvos = iPhoneOS; + MAP_PLATFORM_bridgeos = iPhoneOS; MAP_PLATFORM_iphoneos = iPhoneOS; MAP_PLATFORM_iphoneosnano = iPhoneOS; MAP_PLATFORM_macosx = MacOSX; diff --git a/libsyscall/Platforms/iPhoneOS/arm/syscall.map b/libsyscall/Platforms/iPhoneOS/arm/syscall.map new file mode 100644 index 000000000..2466d41b2 --- /dev/null +++ b/libsyscall/Platforms/iPhoneOS/arm/syscall.map @@ -0,0 +1,77 @@ +_accept$NOCANCEL ___accept_nocancel +_aio_suspend$NOCANCEL ___aio_suspend_nocancel +_close$NOCANCEL ___close_nocancel +_connect$NOCANCEL ___connect_nocancel +_fstat ___fstat64 +_fstat64 +_fstatat ___fstatat64 +_fstatat64 +_fstatfs ___fstatfs64 +_fstatfs64 +_fstatx_np ___fstatx64_np +_fstatx64_np +_fsync$NOCANCEL ___fsync_nocancel +_getfsstat ___getfsstat64 +_getfsstat64 +_getmntinfo ___getmntinfo64 +_getmntinfo64 +_lstat ___lstat64 +_lstat64 +_lstatx_np ___lstatx64_np +_lstatx64_np +_msgrcv$NOCANCEL ___msgrcv_nocancel +_msgsnd$NOCANCEL ___msgsnd_nocancel +_msync$NOCANCEL ___msync_nocancel +_msgsys ___msgsys +_open$NOCANCEL ___open_nocancel +_openat$NOCANCEL ___openat_nocancel +_poll$NOCANCEL ___poll_nocancel +_pread$NOCANCEL ___pread_nocancel +_pwrite$NOCANCEL ___pwrite_nocancel +_read$NOCANCEL ___read_nocancel +_readv$NOCANCEL ___readv_nocancel +_recvfrom$NOCANCEL ___recvfrom_nocancel +_recvmsg$NOCANCEL ___recvmsg_nocancel +_select$DARWIN_EXTSN ___select +_select$DARWIN_EXTSN$NOCANCEL ___select_nocancel +_sem_wait$NOCANCEL ___sem_wait_nocancel +_semsys ___semsys +_sendmsg$NOCANCEL ___sendmsg_nocancel +_sendto$NOCANCEL ___sendto_nocancel +_stat ___stat64 +_stat64 +_statfs ___statfs64 +_statfs64 +_statx_np ___statx64_np +_statx64_np +_waitid$NOCANCEL ___waitid_nocancel +_write$NOCANCEL ___write_nocancel +_writev$NOCANCEL ___writev_nocancel + +_accept ___accept +_bind ___bind +_getattrlist ___getattrlist +_getpeername ___getpeername +_getsockname ___getsockname +_lchown ___lchown +_listen ___listen +_recvfrom ___recvfrom +_recvmsg ___recvmsg +_sendmsg ___sendmsg +_sendto ___sendto +_setattrlist ___setattrlist +_socketpair ___socketpair + +_mprotect ___mprotect +_setregid ___setregid +_setreuid ___setreuid +_open ___open +_openat ___openat +_connect ___connect +_msync ___msync +_sem_open ___sem_open +_semctl ___semctl +_msgctl ___msgctl +_shmctl ___shmctl +_shmsys ___shmsys +_shm_open ___shm_open diff --git a/libsyscall/Platforms/iPhoneOS/arm64/syscall.map b/libsyscall/Platforms/iPhoneOS/arm64/syscall.map new file mode 100644 index 000000000..20eb08fae --- /dev/null +++ b/libsyscall/Platforms/iPhoneOS/arm64/syscall.map @@ -0,0 +1,67 @@ +_accept$NOCANCEL ___accept_nocancel +_aio_suspend$NOCANCEL ___aio_suspend_nocancel +_close$NOCANCEL ___close_nocancel +_connect$NOCANCEL ___connect_nocancel +_fstat ___fstat64 +_fstat64 +_fstatat ___fstatat64 +_fstatat64 +_fstatfs ___fstatfs64 +_fstatfs64 +_fstatx_np ___fstatx64_np +_fstatx64_np +_fsync$NOCANCEL ___fsync_nocancel +_getfsstat ___getfsstat64 +_getfsstat64 +_getmntinfo ___getmntinfo64 +_getmntinfo64 +_lstat ___lstat64 +_lstat64 +_lstatx_np ___lstatx64_np +_lstatx64_np +_msgrcv$NOCANCEL ___msgrcv_nocancel +_msgsnd$NOCANCEL ___msgsnd_nocancel +_msync$NOCANCEL ___msync_nocancel +_poll$NOCANCEL ___poll_nocancel +_pread$NOCANCEL ___pread_nocancel +_pwrite$NOCANCEL ___pwrite_nocancel +_read$NOCANCEL ___read_nocancel +_readv$NOCANCEL ___readv_nocancel +_recvfrom$NOCANCEL ___recvfrom_nocancel +_recvmsg$NOCANCEL ___recvmsg_nocancel +_select$DARWIN_EXTSN ___select +_select$DARWIN_EXTSN$NOCANCEL ___select_nocancel +_sem_wait$NOCANCEL ___sem_wait_nocancel +_sendmsg$NOCANCEL ___sendmsg_nocancel +_sendto$NOCANCEL ___sendto_nocancel +_stat ___stat64 +_stat64 +_statfs ___statfs64 +_statfs64 +_statx_np ___statx64_np +_statx64_np +_waitid$NOCANCEL ___waitid_nocancel +_write$NOCANCEL ___write_nocancel +_writev$NOCANCEL ___writev_nocancel + +_accept ___accept +_bind ___bind +_getattrlist ___getattrlist +_getpeername ___getpeername +_getsockname ___getsockname +_lchown ___lchown +_listen ___listen +_recvfrom ___recvfrom +_recvmsg ___recvmsg +_sendmsg ___sendmsg +_sendto ___sendto +_setattrlist ___setattrlist +_socketpair ___socketpair + +_mprotect ___mprotect +_setregid ___setregid +_setreuid ___setreuid +_connect ___connect +_msync ___msync +_msgctl ___msgctl +_shmctl ___shmctl diff --git a/libsyscall/custom/SYS.h b/libsyscall/custom/SYS.h index 2c66bbf4d..ff93f852a 100644 --- a/libsyscall/custom/SYS.h +++ b/libsyscall/custom/SYS.h @@ -162,6 +162,317 @@ LEAF(pseudo, 0) ;\ PSEUDO(pseudo, name, nargs, cerror) ;\ ret +#elif defined(__arm__) + +#include +#include + +/* + * ARM system call interface: + * + * swi 0x80 + * args: r0-r6 + * return code: r0 + * on error, carry bit is set in the psr, otherwise carry bit is cleared. + */ + +/* + * Macros. + */ + +/* + * until we update the architecture project, these live here + */ + +#if defined(__DYNAMIC__) +#define MI_GET_ADDRESS(reg,var) \ + ldr reg, 4f ;\ +3: ldr reg, [pc, reg] ;\ + b 5f ;\ +4: .long 6f - (3b + 8) ;\ +5: ;\ + .non_lazy_symbol_pointer ;\ +6: ;\ + .indirect_symbol var ;\ + .long 0 ;\ + .text ;\ + .align 2 +#else +#define MI_GET_ADDRESS(reg,var) \ + ldr reg, 3f ;\ + b 4f ;\ +3: .long var ;\ +4: +#endif + +#if defined(__DYNAMIC__) +#define MI_BRANCH_EXTERNAL(var) \ + .globl var ;\ + MI_GET_ADDRESS(ip, var) ;\ + bx ip +#else +#define MI_BRANCH_EXTERNAL(var) ;\ + .globl var ;\ + b var +#endif + +#if defined(__DYNAMIC__) +#define MI_CALL_EXTERNAL(var) \ + .globl var ;\ + MI_GET_ADDRESS(ip,var) ;\ + blx ip +#else +#define MI_CALL_EXTERNAL(var) \ + .globl var ;\ + bl var +#endif + +#define MI_ENTRY_POINT(name) \ + .align 2 ;\ + .globl name ;\ + .text ;\ +name: + +/* load the syscall number into r12 and trap */ +#define DO_SYSCALL(num) \ + .if (((num) & 0xff) == (num)) ;\ + mov r12, #(num) ;\ + .elseif (((num) & 0x3fc) == (num)) ;\ + mov r12, #(num) ;\ + .else ;\ + mov r12, #((num) & 0xffffff00) /* top half of the syscall number */ ;\ + orr r12, r12, #((num) & 0xff) /* bottom half */ ;\ + .endif ;\ + swi #SWI_SYSCALL + +/* simple syscalls (0 to 4 args) */ +#define SYSCALL_0to4(name, cerror) \ + MI_ENTRY_POINT(_##name) ;\ + DO_SYSCALL(SYS_##name) ;\ + bxcc lr /* return if carry is clear (no error) */ ; \ +1: MI_BRANCH_EXTERNAL(_##cerror) + +/* syscalls with 5 args is different, because of the single arg register load */ +#define SYSCALL_5(name, cerror) \ + MI_ENTRY_POINT(_##name) ;\ + mov ip, sp /* save a pointer to the args */ ; \ + stmfd sp!, { r4-r5 } /* save r4-r5 */ ;\ + ldr r4, [ip] /* load 5th arg */ ; \ + DO_SYSCALL(SYS_##name) ;\ + ldmfd sp!, { r4-r5 } /* restore r4-r5 */ ; \ + bxcc lr /* return if carry is clear (no error) */ ; \ +1: MI_BRANCH_EXTERNAL(_##cerror) + +/* syscalls with 6 to 12 args. kernel may have to read from stack */ +#define SYSCALL_6to12(name, save_regs, arg_regs, cerror) \ + MI_ENTRY_POINT(_##name) ;\ + mov ip, sp /* save a pointer to the args */ ; \ + stmfd sp!, { save_regs } /* callee saved regs */ ;\ + ldmia ip, { arg_regs } /* load arg regs */ ; \ + DO_SYSCALL(SYS_##name) ;\ + ldmfd sp!, { save_regs } /* restore callee saved regs */ ; \ + bxcc lr /* return if carry is clear (no error) */ ; \ +1: MI_BRANCH_EXTERNAL(_##cerror) + +#define COMMA , + +#if __BIGGEST_ALIGNMENT__ > 4 + +/* For the armv7k ABI, the alignment requirements may add padding. So we + * let the kernel figure it out and push extra on the stack to avoid un-needed + * copy-ins */ + + /* We'll also use r8 for moving arguments */ + +#define SYSCALL_0(name) SYSCALL_0to4(name) +#define SYSCALL_1(name) SYSCALL_0to4(name) +#define SYSCALL_2(name) SYSCALL_0to4(name) +#define SYSCALL_3(name) SYSCALL_0to4(name) +#define SYSCALL_4(name) SYSCALL_6to12(name, r4-r5, r4-r5) +#undef SYSCALL_5 +#define SYSCALL_5(name) SYSCALL_6to12(name, r4-r5, r4-r5) +#define SYSCALL_6(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8) +#define SYSCALL_7(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8) +#define SYSCALL_8(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8) +#define SYSCALL_12(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8) + +#else // !(__BIGGEST_ALIGNMENT__ > 4) (the normal arm32 ABI case) + +#define SYSCALL_0(name) SYSCALL_0to4(name) +#define SYSCALL_1(name) SYSCALL_0to4(name) +#define SYSCALL_2(name) SYSCALL_0to4(name) +#define SYSCALL_3(name) SYSCALL_0to4(name) +#define SYSCALL_4(name) SYSCALL_0to4(name) +/* SYSCALL_5 declared above */ +#define SYSCALL_6(name) SYSCALL_6to12(name, r4-r5, r4-r5) +#define SYSCALL_7(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6) +#define SYSCALL_8(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6) /* 8th on stack */ +#define SYSCALL_12(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6) /* 8th-12th on stack */ + +#endif // __BIGGEST_ALIGNMENT__ > 4 + +/* select the appropriate syscall code, based on the number of arguments */ +#ifndef __SYSCALL_32BIT_ARG_BYTES +#define SYSCALL(name, nargs, cerror) SYSCALL_##nargs(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_##nargs(name, cerror) +#else +#if __SYSCALL_32BIT_ARG_BYTES < 20 +#define SYSCALL(name, nargs, cerror) SYSCALL_0to4(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_0to4(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 20 +#define SYSCALL(name, nargs, cerror) SYSCALL_5(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_5(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 24 +#define SYSCALL(name, nargs, cerror) SYSCALL_6(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_6(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 28 +#define SYSCALL(name, nargs, cerror) SYSCALL_7(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_7(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 32 +#define SYSCALL(name, nargs, cerror) SYSCALL_8(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_8(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 36 +#define SYSCALL(name, nargs, cerror) SYSCALL_8(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_8(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 44 +#define SYSCALL(name, nargs, cerror) SYSCALL_8(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_8(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 48 +#define SYSCALL(name, nargs, cerror) SYSCALL_12(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_12(name, cerror) +#endif +#endif + +#define SYSCALL_NONAME_0to4(name, cerror) \ + DO_SYSCALL(SYS_##name) ;\ + bcc 1f /* branch if carry bit is clear (no error) */ ; \ + MI_BRANCH_EXTERNAL(_##cerror) /* call cerror */ ; \ +1: + +#define SYSCALL_NONAME_5(name, cerror) \ + mov ip, sp /* save a pointer to the args */ ; \ + stmfd sp!, { r4-r5 } /* save r4-r5 */ ;\ + ldr r4, [ip] /* load 5th arg */ ; \ + DO_SYSCALL(SYS_##name) ;\ + ldmfd sp!, { r4-r5 } /* restore r4-r7 */ ; \ + bcc 1f /* branch if carry bit is clear (no error) */ ; \ + MI_BRANCH_EXTERNAL(_##cerror) /* call cerror */ ; \ +1: + +#define SYSCALL_NONAME_6to12(name, save_regs, arg_regs, cerror) \ + mov ip, sp /* save a pointer to the args */ ; \ + stmfd sp!, { save_regs } /* callee save regs */ ;\ + ldmia ip, { arg_regs } /* load arguments */ ; \ + DO_SYSCALL(SYS_##name) ;\ + ldmfd sp!, { save_regs } /* restore callee saved regs */ ; \ + bcc 1f /* branch if carry bit is clear (no error) */ ; \ + MI_BRANCH_EXTERNAL(_##cerror) /* call cerror */ ; \ +1: + + +#if __BIGGEST_ALIGNMENT__ > 4 + +/* For the armv7k ABI, the alignment requirements may add padding. So we + * let the kernel figure it out and push extra on the stack to avoid un-needed + * copy-ins. We are relying on arguments that aren't in registers starting + * 32 bytes from sp. We also use r8 like in the mach case. */ + +#define SYSCALL_NONAME_0(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_1(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_2(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_3(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_4(name, cerror) SYSCALL_NONAME_6to12(name, r4-r5, r4-r5, cerror) +#undef SYSCALL_NONAME_5 +#define SYSCALL_NONAME_5(name, cerror) SYSCALL_NONAME_6to12(name, r4-r5, r4-r5, cerror) +#define SYSCALL_NONAME_6(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8, cerror) +#define SYSCALL_NONAME_7(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8, cerror) +#define SYSCALL_NONAME_8(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8, cerror) +#define SYSCALL_NONAME_12(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8, cerror) + +#else // !(__BIGGEST_ALIGNMENT__ > 4) (the normal arm32 ABI case) + +#define SYSCALL_NONAME_0(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_1(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_2(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_3(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_4(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +/* SYSCALL_NONAME_5 declared above */ +#define SYSCALL_NONAME_6(name, cerror) SYSCALL_NONAME_6to12(name, r4-r5, r4-r5, cerror) +#define SYSCALL_NONAME_7(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6, cerror) +#define SYSCALL_NONAME_8(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6, cerror) +#define SYSCALL_NONAME_12(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6, cerror) + +#endif // __BIGGEST_ALIGNMENT__ > 4 + +#define PSEUDO(pseudo, name, nargs, cerror) \ + .globl pseudo ;\ + .text ;\ + .align 2 ;\ +pseudo: ;\ + SYSCALL_NONAME(name, nargs, cerror) + +#define __SYSCALL2(pseudo, name, nargs, cerror) \ + PSEUDO(pseudo, name, nargs, cerror) ;\ + bx lr + +#define __SYSCALL(pseudo, name, nargs) \ + PSEUDO(pseudo, name, nargs, cerror) ;\ + bx lr + +#elif defined(__arm64__) + +#include +#include +#include + +/* + * ARM64 system call interface: + * + * TBD + */ + +#define DO_SYSCALL(num, cerror) \ + mov x16, #(num) %%\ + svc #SWI_SYSCALL %%\ + b.cc 2f %%\ + PUSH_FRAME %%\ + bl _##cerror %%\ + POP_FRAME %%\ +2: + +#define MI_GET_ADDRESS(reg,var) \ + adrp reg, var@page %%\ + add reg, reg, var@pageoff %% + +#define MI_CALL_EXTERNAL(sym) \ + .globl sym %% \ + bl sym + +#define SYSCALL_NONAME(name, nargs, cerror) \ + DO_SYSCALL(SYS_##name, cerror) %% \ +1: + +#define MI_ENTRY_POINT(name) \ + .text %% \ + .align 2 %% \ + .globl name %% \ +name: + +#define PSEUDO(pseudo, name, nargs, cerror) \ + .text %% \ + .align 2 %% \ + .globl pseudo %% \ + pseudo: %% \ + SYSCALL_NONAME(name, nargs, cerror) + +#define __SYSCALL(pseudo, name, nargs) \ + PSEUDO(pseudo, name, nargs, cerror) %% \ + ret + +#define __SYSCALL2(pseudo, name, nargs, cerror) \ + PSEUDO(pseudo, name, nargs, cerror) %% \ + ret + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__fork.s b/libsyscall/custom/__fork.s index 5857ab2ad..dc517a1a2 100644 --- a/libsyscall/custom/__fork.s +++ b/libsyscall/custom/__fork.s @@ -100,6 +100,59 @@ L2: addq $24, %rsp // restore the stack ret +#elif defined(__arm__) + +MI_ENTRY_POINT(___fork) + stmfd sp!, {r4, r7, lr} + add r7, sp, #4 + + mov r1, #1 // prime results + mov r12, #SYS_fork + swi #SWI_SYSCALL // make the syscall + bcs Lbotch // error? + + cmp r1, #0 // parent (r1=0) or child(r1=1) + beq Lparent + + //child here... + MI_GET_ADDRESS(r3, __current_pid) + mov r0, #0 + str r0, [r3] // clear cached pid in child + ldmfd sp!, {r4, r7, pc} + +Lbotch: + MI_CALL_EXTERNAL(_cerror) // jump here on error + mov r0,#-1 // set the error + // fall thru +Lparent: + ldmfd sp!, {r4, r7, pc} // pop and return + +#elif defined(__arm64__) + +#include + +MI_ENTRY_POINT(___fork) + PUSH_FRAME + // ARM moves a 1 in to r1 here, but I can't see why. + mov x16, #SYS_fork // Syscall code + svc #SWI_SYSCALL // Trap to kernel + b.cs Lbotch // Carry bit indicates failure + cbz x1, Lparent // x1 == 0 indicates that we are the parent + + // Child + MI_GET_ADDRESS(x9, __current_pid) // Get address of cached "current pid" + mov w0, #0 + str w0, [x9] // Clear cached current pid + POP_FRAME // And done + ret + +Lbotch: + MI_CALL_EXTERNAL(_cerror) // Handle error + mov w0, #-1 // Return value is -1 +Lparent: + POP_FRAME // Return + ret + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__getpid.s b/libsyscall/custom/__getpid.s index 2768d9b82..a048f48aa 100644 --- a/libsyscall/custom/__getpid.s +++ b/libsyscall/custom/__getpid.s @@ -97,6 +97,77 @@ LEAF(___getpid, 0) movl %edx, %eax ret +#elif defined(__arm__) + +#include + + .data + .globl __current_pid + .align 2 +__current_pid: + /* Cached pid. Possible values: + * 0: no value cached + * > 0: cached PID of current process + * < 0: negative number of vforks in progress + * INT_MIN: for pre-ARMv6, "looking" value (0x80000000) + */ + .long 0 + +MI_ENTRY_POINT(___getpid) + ldr r3, L__current_pid +L1: add r3, pc, r3 // r3 = &__current_pid + ldr r0, [r3] // get the cached pid + cmp r0, #0 + bxgt lr // if positive, return it + + SYSCALL_NONAME(getpid, 0, cerror_nocancel) + +#ifdef _ARM_ARCH_6 + ldrex r2, [r3] // see if we can cache it + cmp r2, #0 // we can't if there are any... + bxlt lr // ...vforks in progress + strex r2, r0, [r3] // ignore conflicts +#else + mov r1, #0x80000000 // load "looking" value + swp r2, r1, [r3] // look at the value, lock others out + cmp r2, r1 // anyone else trying to look? + bxeq lr // yes, so return immediately/ + cmp r2, #0 // see if we can cache it + streq r0, [r3] // if zero, we can + strne r2, [r3] // otherwise restore previous value +#endif + + bx lr + +L__current_pid: + .long __current_pid - (L1+8) + +#elif defined(__arm64__) + .data + .globl __current_pid + .align 2 +__current_pid: + /* cached pid. possible values: + * 0: no value cached + * > 0: cached pid of current process + * < 0: negative number of vforks in progress + * int_min: for pre-armv6, "looking" value (0x80000000) + */ + .long 0 + +MI_ENTRY_POINT(___getpid) + MI_GET_ADDRESS(x9, __current_pid) // Get address of cached value + ldr w0, [x9] // Load it + cmp w0, #0 // See if there's a cached value + b.ls L_notcached // If not, make syscall + ret // Else, we're done +L_notcached: + SYSCALL_NONAME(getpid, 0, cerror_nocancel) + ldxr w10, [x9] // Exclusive load + cbnz w10, L_done // Unless unset, don't even try + stxr wzr, w0, [x9] // Try to store, but don't care if we fail (someone will win, or not) +L_done: + ret // Done #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__gettimeofday.s b/libsyscall/custom/__gettimeofday.s index 8712a2094..47aada592 100644 --- a/libsyscall/custom/__gettimeofday.s +++ b/libsyscall/custom/__gettimeofday.s @@ -91,6 +91,30 @@ LABEL(___gettimeofday) 2: ret +#elif defined(__arm__) + +__SYSCALL2(___gettimeofday_with_mach, gettimeofday, 3, cerror_nocancel) + +.text +.align 2 +.globl ___gettimeofday +___gettimeofday: + mov r2, #0x0 + SYSCALL_NONAME(gettimeofday, 3, cerror_nocancel) + bx lr + +#elif defined(__arm64__) + +__SYSCALL2(___gettimeofday_with_mach, gettimeofday, 3, cerror_nocancel) + +.text +.align 2 +.globl ___gettimeofday +___gettimeofday: + movz x2, #0x0 + SYSCALL_NONAME(gettimeofday, 3, cerror_nocancel) + ret + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__kdebug_trace_string.s b/libsyscall/custom/__kdebug_trace_string.s index e3543e6bf..81cf375b5 100644 --- a/libsyscall/custom/__kdebug_trace_string.s +++ b/libsyscall/custom/__kdebug_trace_string.s @@ -36,6 +36,14 @@ __SYSCALL(___kdebug_trace_string, kdebug_trace_string, 3) __SYSCALL_INT(___kdebug_trace_string, kdebug_trace_string, 3) +#elif defined(__arm__) + +__SYSCALL(___kdebug_trace_string, kdebug_trace_string, 4) + +#elif defined(__arm64__) + +__SYSCALL(___kdebug_trace_string, kdebug_trace_string, 3) + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__lseek.s b/libsyscall/custom/__lseek.s index b051cc5a4..7c7f41eb2 100644 --- a/libsyscall/custom/__lseek.s +++ b/libsyscall/custom/__lseek.s @@ -36,6 +36,14 @@ __SYSCALL(___lseek, lseek, 3) __SYSCALL_INT(___lseek, lseek, 3) +#elif defined(__arm__) + +__SYSCALL(___lseek, lseek, 4) + +#elif defined(__arm64__) + +__SYSCALL(___lseek, lseek, 3) + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__pipe.s b/libsyscall/custom/__pipe.s index d375dddbd..0c527d5ea 100644 --- a/libsyscall/custom/__pipe.s +++ b/libsyscall/custom/__pipe.s @@ -46,6 +46,25 @@ PSEUDO(___pipe, pipe, 0, cerror_nocancel) xorl %eax, %eax ret +#elif defined(__arm__) + +MI_ENTRY_POINT(___pipe) + mov r3,r0 // save fildes across syscall + SYSCALL_NONAME(pipe, 0, cerror_nocancel) + str r0, [r3, #0] + str r1, [r3, #4] + mov r0,#0 + bx lr + +#elif defined(__arm64__) + +MI_ENTRY_POINT(___pipe) + mov x9, x0 // Stash FD array + SYSCALL_NONAME(pipe, 0, cerror_nocancel) + stp w0, w1, [x9] // Save results + mov x0, #0 // Success + ret // Done + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__ptrace.s b/libsyscall/custom/__ptrace.s index bdcbec9fb..607a26e87 100644 --- a/libsyscall/custom/__ptrace.s +++ b/libsyscall/custom/__ptrace.s @@ -50,6 +50,25 @@ LEAF(___ptrace, 0) UNIX_SYSCALL_NONAME(ptrace, 4, cerror) ret +#elif defined(__arm__) + +MI_ENTRY_POINT(___ptrace) + MI_GET_ADDRESS(ip,_errno) + str r8, [sp, #-4]! + mov r8, #0 + str r8, [ip] + ldr r8, [sp], #4 + SYSCALL_NONAME(ptrace, 4, cerror) + bx lr + +#elif defined(__arm64__) + +MI_ENTRY_POINT(___ptrace) + MI_GET_ADDRESS(x9,_errno) + str wzr, [x9] + SYSCALL_NONAME(ptrace, 4, cerror) + ret + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__sigaltstack.s b/libsyscall/custom/__sigaltstack.s index d5f1803ff..7c4fceaf5 100644 --- a/libsyscall/custom/__sigaltstack.s +++ b/libsyscall/custom/__sigaltstack.s @@ -36,6 +36,14 @@ __SYSCALL(___sigaltstack, sigaltstack, 3) __SYSCALL_INT(___sigaltstack, sigaltstack, 3) +#elif defined(__arm__) + +__SYSCALL(___sigaltstack, sigaltstack, 3) + +#elif defined(__arm64__) + +__SYSCALL(___sigaltstack, sigaltstack, 3) + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__sigreturn.s b/libsyscall/custom/__sigreturn.s index 16d5be4fc..a6a24404e 100644 --- a/libsyscall/custom/__sigreturn.s +++ b/libsyscall/custom/__sigreturn.s @@ -36,6 +36,14 @@ __SYSCALL(___sigreturn, sigreturn, 2) __SYSCALL_INT(___sigreturn, sigreturn, 2) +#elif defined(__arm__) + +__SYSCALL(___sigreturn, sigreturn, 2) + +#elif defined(__arm64__) + +__SYSCALL(___sigreturn, sigreturn, 2) + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__syscall.s b/libsyscall/custom/__syscall.s index f00894425..81e49f11a 100644 --- a/libsyscall/custom/__syscall.s +++ b/libsyscall/custom/__syscall.s @@ -50,6 +50,24 @@ END(___syscall) // that value anyway. __SYSCALL(___syscall, syscall, 0); +#elif defined(__arm__) + +__SYSCALL(___syscall, syscall, 7) + +#elif defined(__arm64__) + +/* + * Ignore nominal number of arguments: just pop from stack and let the kernel + * interpret. + */ +#include +MI_ENTRY_POINT(___syscall) + ldp x1, x2, [sp] + ldp x3, x4, [sp, #16] + ldp x5, x6, [sp, #32] + ldr x7, [sp, #48] + DO_SYSCALL(SYS_syscall, cerror) + ret #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__thread_selfid.s b/libsyscall/custom/__thread_selfid.s index d84a5305f..86c17638a 100644 --- a/libsyscall/custom/__thread_selfid.s +++ b/libsyscall/custom/__thread_selfid.s @@ -36,4 +36,12 @@ __SYSCALL(___thread_selfid, thread_selfid, 0) __SYSCALL_INT(___thread_selfid, thread_selfid, 0) +#elif defined(__arm__) + +__SYSCALL(___thread_selfid, thread_selfid, 0) + +#elif defined(__arm64__) + +__SYSCALL(___thread_selfid, thread_selfid, 0) + #endif diff --git a/libsyscall/custom/__thread_selfusage.s b/libsyscall/custom/__thread_selfusage.s index 064c5bad9..ec83854e4 100644 --- a/libsyscall/custom/__thread_selfusage.s +++ b/libsyscall/custom/__thread_selfusage.s @@ -36,4 +36,12 @@ __SYSCALL(___thread_selfusage, thread_selfusage, 0) __SYSCALL_INT(___thread_selfusage, thread_selfusage, 0) +#elif defined(__arm__) + +__SYSCALL(___thread_selfusage, thread_selfusage, 0) + +#elif defined(__arm64__) + +__SYSCALL(___thread_selfusage, thread_selfusage, 0) + #endif diff --git a/libsyscall/custom/__vfork.s b/libsyscall/custom/__vfork.s index 8449d25e4..65a781efd 100644 --- a/libsyscall/custom/__vfork.s +++ b/libsyscall/custom/__vfork.s @@ -128,6 +128,107 @@ L2: addl $1, (%rdx) jmp *%rdi +#elif defined(__arm__) + +#include + + .globl cerror + MI_ENTRY_POINT(___vfork) + + MI_GET_ADDRESS(r3, __current_pid) // get address of __current_pid +#ifdef _ARM_ARCH_6 +L0: + ldrex r1, [r3] + subs r1, r1, #1 // if __current_pid <= 0, decrement it + movpl r1, #-1 // otherwise put -1 in there + strex r2, r1, [r3] + cmp r2, #0 + bne L0 +#else + mov r2, #0x80000000 // load "looking" value +L0: + swp r1, r2, [r3] // look at the value, lock others out + cmp r1, r2 // anyone else trying to look? + beq L0 // yes, so wait our turn + subs r1, r1, #1 // if __current_pid <= 0, decrement it + movpl r1, #-1 // otherwise put -1 in there + str r1, [r3] +#endif + + mov r1, #1 // prime results + mov r12, #SYS_vfork + swi #SWI_SYSCALL // make the syscall + bcs Lbotch // error? + cmp r1, #0 // parent (r1=0) or child(r1=1) + beq Lparent + + //child here... + mov r0, #0 + bx lr // return + +Lbotch: + MI_CALL_EXTERNAL(_cerror) // jump here on error + mov r0,#-1 // set the error + // reload values clobbered by cerror (so we can treat them as live in Lparent) + MI_GET_ADDRESS(r3, __current_pid) // get address of __current_pid +#ifndef _ARM_ARCH_6 + mov r2, #0x80000000 // load "looking" value +#endif + // fall thru + +Lparent: +#ifdef _ARM_ARCH_6 + ldrex r1, [r3] + add r1, r1, #1 // we're back, decrement vfork count + strex r2, r1, [r3] + cmp r2, #0 + bne Lparent +#else + swp r1, r2, [r3] // look at the value, lock others out + cmp r1, r2 // anyone else trying to look? + beq Lparent // yes, so wait our turn + add r1, r1, #1 // we're back, decrement vfork count + str r1, [r3] +#endif + + bx lr // return + +#elif defined(__arm64__) + + MI_ENTRY_POINT(___vfork) + + MI_GET_ADDRESS(x9, __current_pid) +Ltry_set_vfork: + ldxr w10, [x9] // Get old current pid value (exclusive) + mov w11, #-1 // Will be -1 if current value is positive + subs w10, w10, #1 // Subtract one + csel w12, w11, w10, pl // If >= 0, set to -1, else set to (current - 1) + stxr w13, w12, [x9] // Attempt exclusive store to current pid + cbnz w13, Ltry_set_vfork // If store failed, retry + + // ARM sets r1 to 1 here. I don't see why. + mov w16, #SYS_vfork // Set syscall code + svc #SWI_SYSCALL + b.cs Lbotch + cbz w1, Lparent + + // Child + mov w0, #0 + ret + + // Error case +Lbotch: + bl _cerror // Update errno + mov w0, #-1 // Set return value + MI_GET_ADDRESS(x9, __current_pid) // Reload current pid address + // Fall through +Lparent: + ldxr w10, [x9] // Exclusive load current pid value + add w10, w10, #1 // Increment (i.e. decrement vfork count) + stxr w11, w10, [x9] // Attempt exclusive store of updated vfork count + cbnz w11, Lparent // If exclusive store failed, retry + ret // Done, return + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/custom.s b/libsyscall/custom/custom.s index 56a95cf5b..de2e0c6bd 100644 --- a/libsyscall/custom/custom.s +++ b/libsyscall/custom/custom.s @@ -122,6 +122,26 @@ __thread_set_tsd_base: MACHDEP_SYSCALL_TRAP ret +#elif defined(__arm__) + + .align 2 + .globl __thread_set_tsd_base +__thread_set_tsd_base: + mov r3, #2 + mov r12, #0x80000000 + swi #SWI_SYSCALL + bx lr + +#elif defined(__arm64__) + + .align 2 + .globl __thread_set_tsd_base +__thread_set_tsd_base: + mov x3, #2 + mov x16, #0x80000000 + svc #SWI_SYSCALL + ret + #else #error unknown architecture #endif diff --git a/libsyscall/mach/err_libkern.sub b/libsyscall/mach/err_libkern.sub index f419d04fa..f865b9d7c 100644 --- a/libsyscall/mach/err_libkern.sub +++ b/libsyscall/mach/err_libkern.sub @@ -89,6 +89,7 @@ static const char * const err_codes_libkern_kext[] = { "(libkern/kext) kext is in use or retained (cannot unload)", /* 0x18 */ "(libkern/kext) kext request timed out", /* 0x19 */ "(libkern/kext) kext is stopping and cannot issue requests", /* 0x1a */ + "(libkern/kext) system policy prevents loading", /* 0x1b */ }; /* libkern is err_system(0x37) */ diff --git a/libsyscall/mach/host.c b/libsyscall/mach/host.c index a781a09ad..651d148d8 100644 --- a/libsyscall/mach/host.c +++ b/libsyscall/mach/host.c @@ -46,16 +46,33 @@ kern_return_t host_get_multiuser_config_flags(host_t host __unused, uint32_t *multiuser_flags) { +#if TARGET_OS_EMBEDDED + volatile uint32_t *multiuser_flag_address = (volatile uint32_t *)(uintptr_t)(_COMM_PAGE_MULTIUSER_CONFIG); + *multiuser_flags = *multiuser_flag_address; + return KERN_SUCCESS; +#else (void)multiuser_flags; return KERN_NOT_SUPPORTED; +#endif } kern_return_t host_check_multiuser_mode(host_t host __unused, uint32_t *multiuser_mode) { +#if TARGET_OS_EMBEDDED + uint32_t multiuser_flags; + kern_return_t kr; + + kr = host_get_multiuser_config_flags(host, &multiuser_flags); + if (kr != KERN_SUCCESS) + return kr; + *multiuser_mode = (multiuser_flags & kIsMultiUserDevice) == kIsMultiUserDevice; + return KERN_SUCCESS; +#else (void)multiuser_mode; return KERN_NOT_SUPPORTED; +#endif } extern kern_return_t diff --git a/libsyscall/mach/mach_init.c b/libsyscall/mach/mach_init.c index 90a42ceb5..338f7c95b 100644 --- a/libsyscall/mach/mach_init.c +++ b/libsyscall/mach/mach_init.c @@ -115,6 +115,12 @@ _mach_fork_child(void) return 0; } +#if defined(__arm__) || defined(__arm64__) +#if !defined(_COMM_PAGE_USER_PAGE_SHIFT_64) && defined(_COMM_PAGE_UNUSED0) +#define _COMM_PAGE_USER_PAGE_SHIFT_32 (_COMM_PAGE_UNUSED0) +#define _COMM_PAGE_USER_PAGE_SHIFT_64 (_COMM_PAGE_UNUSED0+1) +#endif +#endif void mach_init_doit(void) @@ -136,7 +142,13 @@ mach_init_doit(void) } if (vm_page_shift == 0) { +#if defined(__arm64__) + vm_page_shift = *(uint8_t*) _COMM_PAGE_USER_PAGE_SHIFT_64; +#elif defined(__arm__) + vm_page_shift = *(uint8_t*) _COMM_PAGE_USER_PAGE_SHIFT_32; +#else vm_page_shift = vm_kernel_page_shift; +#endif vm_page_size = 1 << vm_page_shift; vm_page_mask = vm_page_size - 1; } diff --git a/libsyscall/mach/string.h b/libsyscall/mach/string.h index b3c00458e..7d668126a 100644 --- a/libsyscall/mach/string.h +++ b/libsyscall/mach/string.h @@ -39,8 +39,8 @@ // of Libc's string.h (which no one should be using bar MIG) in order // to override their use of memcpy. -int _mach_snprintf(char *buffer, int length, const char *fmt, ...); -int _mach_vsnprintf(char *buffer, int length, const char *fmt, va_list ap); +int _mach_snprintf(char *buffer, int length, const char *fmt, ...) __printflike(3, 4); +int _mach_vsnprintf(char *buffer, int length, const char *fmt, va_list ap) __printflike(3, 0); // Actually in memcpy.c but MIG likes to include string.h diff --git a/libsyscall/os/tsd.h b/libsyscall/os/tsd.h index 0e064b954..d49087f14 100644 --- a/libsyscall/os/tsd.h +++ b/libsyscall/os/tsd.h @@ -38,15 +38,35 @@ #define __TSD_THREAD_SELF 0 #define __TSD_ERRNO 1 #define __TSD_MIG_REPLY 2 +#define __TSD_MACH_THREAD_SELF 3 +#define __TSD_THREAD_QOS_CLASS 4 +#define __TSD_RETURN_TO_KERNEL 5 +/* slot 6 is reserved for Windows/WINE compatibility reasons */ #define __TSD_SEMAPHORE_CACHE 9 +#ifdef __arm__ +#include +#endif __attribute__((always_inline)) static __inline__ unsigned int _os_cpu_number(void) { - /* Not yet implemented */ - return 0; +#if defined(__arm__) && defined(_ARM_ARCH_6) + uintptr_t p; + __asm__("mrc p15, 0, %[p], c13, c0, 3" : [p] "=&r" (p)); + return (unsigned int)(p & 0x3ul); +#elif defined(__arm64__) + uint64_t p; + __asm__("mrs %[p], TPIDRRO_EL0" : [p] "=&r" (p)); + return (unsigned int)p & 0x7; +#elif defined(__x86_64__) || defined(__i386__) + struct { uintptr_t p1, p2; } p; + __asm__("sidt %[p]" : [p] "=&m" (p)); + return (unsigned int)(p.p1 & 0xfff); +#else +#error _os_cpu_number not implemented on this architecture +#endif } #if defined(__i386__) || defined(__x86_64__) @@ -84,6 +104,28 @@ _os_tsd_set_direct(unsigned long slot, void *val) } #endif +#elif defined(__arm__) || defined(__arm64__) + +__attribute__((always_inline, pure)) +static __inline__ void** +_os_tsd_get_base(void) +{ +#if defined(__arm__) && defined(_ARM_ARCH_6) + uintptr_t tsd; + __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r" (tsd)); + tsd &= ~0x3ul; /* lower 2-bits contain CPU number */ +#elif defined(__arm__) && defined(_ARM_ARCH_5) + register uintptr_t tsd asm ("r9"); +#elif defined(__arm64__) + uint64_t tsd; + __asm__("mrs %0, TPIDRRO_EL0" : "=r" (tsd)); + tsd &= ~0x7ull; +#endif + + return (void**)(uintptr_t)tsd; +} +#define _os_tsd_get_base() _os_tsd_get_base() + #else #error _os_tsd_get_base not implemented on this architecture #endif diff --git a/libsyscall/wrappers/__commpage_gettimeofday.c b/libsyscall/wrappers/__commpage_gettimeofday.c index 4967a2f8d..0ebfc5318 100644 --- a/libsyscall/wrappers/__commpage_gettimeofday.c +++ b/libsyscall/wrappers/__commpage_gettimeofday.c @@ -24,56 +24,93 @@ #include #include #include +#include +#include int __commpage_gettimeofday(struct timeval *); __attribute__((visibility("hidden"))) int __commpage_gettimeofday_internal(struct timeval *tp, uint64_t *tbr_out); -#if defined(__x86_64__) || defined(__i386__) +int +__commpage_gettimeofday(struct timeval *tp) +{ + return __commpage_gettimeofday_internal(tp, NULL); +} -// XXX: must be kept in sync with __commpage_gettimeofday.s int __commpage_gettimeofday_internal(struct timeval *tp, uint64_t *tbr_out) { - volatile uint32_t *gtod_generation_p = _COMM_PAGE_GTOD_GENERATION; - volatile uint64_t *gtod_sec_base_p = _COMM_PAGE_GTOD_SEC_BASE; - volatile uint64_t *gtod_ns_base_p = _COMM_PAGE_GTOD_NS_BASE; - - uint64_t tbr, gen, tod_secs, tod_nsecs, elapsed; - while(1) { - gen = *gtod_generation_p; - tbr = mach_absolute_time(); - tod_secs = *gtod_sec_base_p; - tod_nsecs = *gtod_ns_base_p; - uint64_t gen2 = *gtod_generation_p; - if(__builtin_expect(gen, gen2) == gen2) - break; + uint64_t now, over; + uint64_t delta,frac; + uint64_t TimeStamp_tick; + uint64_t TimeStamp_sec; + uint64_t TimeStamp_frac; + uint64_t Tick_scale; + uint64_t Ticks_per_sec; + + volatile uint64_t *gtod_TimeStamp_tick_p; + volatile uint64_t *gtod_TimeStamp_sec_p; + volatile uint64_t *gtod_TimeStamp_frac_p; + volatile uint64_t *gtod_Ticks_scale_p; + volatile uint64_t *gtod_Ticks_per_sec_p; + + new_commpage_timeofday_data_t *commpage_timeofday_datap; + + commpage_timeofday_datap = (new_commpage_timeofday_data_t *)_COMM_PAGE_NEWTIMEOFDAY_DATA; + + gtod_TimeStamp_tick_p = &commpage_timeofday_datap->TimeStamp_tick; + gtod_TimeStamp_sec_p = &commpage_timeofday_datap->TimeStamp_sec; + gtod_TimeStamp_frac_p = &commpage_timeofday_datap->TimeStamp_frac; + gtod_Ticks_scale_p = &commpage_timeofday_datap->Ticks_scale; + gtod_Ticks_per_sec_p = &commpage_timeofday_datap->Ticks_per_sec; + + do { + TimeStamp_tick = *gtod_TimeStamp_tick_p; + TimeStamp_sec = *gtod_TimeStamp_sec_p; + TimeStamp_frac = *gtod_TimeStamp_frac_p; + Tick_scale = *gtod_Ticks_scale_p; + Ticks_per_sec = *gtod_Ticks_per_sec_p; + + /* + * This call contains an instruction barrier which will ensure that the + * second read of the abs time isn't speculated above the reads of the + * other values above + */ + now = mach_absolute_time(); + } while (TimeStamp_tick != *gtod_TimeStamp_tick_p); + + if (TimeStamp_tick == 0) + return(1); + + delta = now - TimeStamp_tick; + + /* If more than one second force a syscall */ + if (delta >= Ticks_per_sec) + return(1); + + tp->tv_sec = TimeStamp_sec; + + over = multi_overflow(Tick_scale, delta); + if(over){ + tp->tv_sec += over; } - if (gen == 0) return KERN_FAILURE; - elapsed = tbr - tod_nsecs; - - unsigned long secs; - uint32_t nsec; -#if defined(__x86_64__) - secs = elapsed/NSEC_PER_SEC; - nsec = elapsed % NSEC_PER_SEC; -#elif defined(__i386__) - uint32_t secs1, secs2; - secs1 = elapsed >> 32; - secs2 = elapsed; - __asm__ ( - "divl %4" - : "=a" (secs), "=d" (nsec) - : "0" (secs2), "1" (secs1), "rm" (NSEC_PER_SEC) - ); -#endif /* __i386 or __x86_64__ */ - tp->tv_sec = tod_secs + secs; - tp->tv_usec = nsec / NSEC_PER_USEC; - - if (tbr_out) *tbr_out = tbr; - - return KERN_SUCCESS; -} -#endif + /* Sum scale*delta to TimeStamp_frac, if it overflows increment sec */ + frac = TimeStamp_frac; + frac += Tick_scale * delta; + if( TimeStamp_frac > frac ) + tp->tv_sec++; + + /* + * Convert frac (64 bit frac of a sec) to usec + * usec = frac * USEC_PER_SEC / 2^64 + */ + tp->tv_usec = ((uint64_t)1000000 * (uint32_t)(frac >> 32)) >> 32; + + if (tbr_out) { + *tbr_out = now; + } + + return(0); +} diff --git a/libsyscall/wrappers/__commpage_gettimeofday.s b/libsyscall/wrappers/__commpage_gettimeofday.s deleted file mode 100644 index da920f28b..000000000 --- a/libsyscall/wrappers/__commpage_gettimeofday.s +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2003-2007 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -#include -#include - -#define NSEC_PER_SEC 1000*1000*1000 -#define NSEC_PER_USEC 1000 - -#if defined(__i386__) - - .align 4 - .globl ___commpage_gettimeofday -___commpage_gettimeofday: - push %ebp - mov %esp,%ebp - push %esi - push %ebx -0: - movl _COMM_PAGE_GTOD_GENERATION,%esi /* get generation (0 if disabled) */ - testl %esi,%esi /* disabled? */ - jz 4f - - call _mach_absolute_time /* get nanotime in %edx:%eax */ - - sub _COMM_PAGE_GTOD_NS_BASE,%eax - sbb _COMM_PAGE_GTOD_NS_BASE+4,%edx - mov _COMM_PAGE_GTOD_SEC_BASE,%ebx /* load all the data before checking generation */ - mov $ NSEC_PER_SEC,%ecx - - cmpl _COMM_PAGE_GTOD_GENERATION,%esi /* has time data changed out from under us? */ - jne 0b - - div %ecx - add %eax,%ebx - - mov $ NSEC_PER_USEC,%ecx - mov %edx,%eax - xor %edx,%edx - div %ecx - - mov 8(%ebp),%ecx - mov %ebx,(%ecx) - mov %eax,4(%ecx) - xor %eax,%eax -3: - pop %ebx - pop %esi - pop %ebp - ret -4: /* fail */ - movl $1,%eax - jmp 3b - -#elif defined(__x86_64__) - - .align 4, 0x90 - .globl ___commpage_gettimeofday -___commpage_gettimeofday: -// %rdi = ptr to timeval - pushq %rbp // set up a frame for backtraces - pushq %r12 // push callee-saved registers we want to use - pushq %r13 - pushq %r14 - subq $8, %rsp - movq %rsp,%rbp - movq %rdi,%r12 // save ptr to timeval - movq $(_COMM_PAGE_TIME_DATA_START),%r13 -0: - movl _GTOD_GENERATION(%r13),%r14d // get generation (0 if disabled) - testl %r14d,%r14d // disabled? - jz 4f - - call _mach_absolute_time // get %rax <- nanotime() - - movl _GTOD_SEC_BASE(%r13),%r8d // get _COMM_PAGE_TIMESTAMP - subq _GTOD_NS_BASE(%r13),%rax // generate nanoseconds since timestamp - cmpl _GTOD_GENERATION(%r13),%r14d // has data changed out from under us? - jne 0b - - movl $ NSEC_PER_SEC,%ecx - movq %rax,%rdx - shrq $32,%rdx // get high half of delta in %edx - divl %ecx // %eax <- seconds since timestamp, %edx <- nanoseconds - addl %eax,%r8d // add seconds elapsed to timestamp seconds - - movl $ NSEC_PER_USEC,%ecx - movl %edx,%eax - xorl %edx,%edx - divl %ecx // divide residual ns by 1000 to get residual us in %eax - - movq %r8,(%r12) // store 64-bit seconds into timeval - movl %eax,8(%r12) // store 32-bit useconds into timeval - xorl %eax,%eax // return 0 for success -3: - addq $8, %rsp - popq %r14 - popq %r13 - popq %r12 - popq %rbp - ret -4: // fail - movl $1,%eax - jmp 3b - -#endif diff --git a/libsyscall/wrappers/__get_cpu_capabilities.s b/libsyscall/wrappers/__get_cpu_capabilities.s index de177986a..86b0ee2a7 100644 --- a/libsyscall/wrappers/__get_cpu_capabilities.s +++ b/libsyscall/wrappers/__get_cpu_capabilities.s @@ -47,6 +47,31 @@ __get_cpu_capabilities: movl _COMM_PAGE_CPU_CAPABILITIES64+4, %edx ret +#elif defined(__arm__) + + .text + .align 2 + .globl __get_cpu_capabilities +__get_cpu_capabilities: + mov r0, #(_COMM_PAGE_CPU_CAPABILITIES & 0x000000ff) + orr r0, r0, #(_COMM_PAGE_CPU_CAPABILITIES & 0x0000ff00) + orr r0, r0, #(_COMM_PAGE_CPU_CAPABILITIES & 0x00ff0000) + orr r0, r0, #(_COMM_PAGE_CPU_CAPABILITIES & 0xff000000) + ldr r0, [r0] + bx lr + +#elif defined(__arm64__) + + .text + .align 2 + .globl __get_cpu_capabilities +__get_cpu_capabilities: + ldr x0, Lcommpage_cc_addr + ldr w0, [x0] + ret +Lcommpage_cc_addr: +.quad _COMM_PAGE_CPU_CAPABILITIES + #else #error Unsupported architecture #endif diff --git a/libsyscall/wrappers/_libkernel_init.c b/libsyscall/wrappers/_libkernel_init.c index 16d7e1917..3eb67853d 100644 --- a/libsyscall/wrappers/_libkernel_init.c +++ b/libsyscall/wrappers/_libkernel_init.c @@ -26,10 +26,20 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +#include +#include +#include +#include #include "_libkernel_init.h" extern int mach_init(void); +#if TARGET_OS_OSX +__attribute__((visibility("default"))) +extern bool _os_xbs_chrooted; +bool _os_xbs_chrooted; +#endif + /* dlsym() funcptr is for legacy support in exc_catcher */ void* (*_dlsym)(void*, const char*) __attribute__((visibility("hidden"))); diff --git a/libsyscall/wrappers/coalition.c b/libsyscall/wrappers/coalition.c index 92a0bda27..627da2261 100644 --- a/libsyscall/wrappers/coalition.c +++ b/libsyscall/wrappers/coalition.c @@ -50,3 +50,14 @@ int coalition_info_resource_usage(uint64_t cid, struct coalition_resource_usage { return __coalition_info(COALITION_INFO_RESOURCE_USAGE, &cid, cru, &sz); } + +int coalition_info_set_name(uint64_t cid, const char *name, size_t size) +{ + return __coalition_info(COALITION_INFO_SET_NAME, &cid, (void *)name, &size); +} + +int coalition_info_set_efficiency(uint64_t cid, uint64_t flags) +{ + size_t size = sizeof(flags); + return __coalition_info(COALITION_INFO_SET_EFFICIENCY, &cid, (void *)&flags, &size); +} diff --git a/libsyscall/wrappers/init_cpu_capabilities.c b/libsyscall/wrappers/init_cpu_capabilities.c index 7eecac6bf..8414c1dfa 100644 --- a/libsyscall/wrappers/init_cpu_capabilities.c +++ b/libsyscall/wrappers/init_cpu_capabilities.c @@ -38,4 +38,16 @@ _init_cpu_capabilities( void ) _cpu_capabilities = _get_cpu_capabilities(); } +#elif defined(__arm__) || defined(__arm64__) + +extern int _get_cpu_capabilities(void); + +int _cpu_capabilities = 0; +int _cpu_has_altivec = 0; // DEPRECATED: use _cpu_capabilities instead + +void +_init_cpu_capabilities( void ) +{ +} + #endif diff --git a/libsyscall/wrappers/libproc/libproc.c b/libsyscall/wrappers/libproc/libproc.c index cc0321e2d..255664d9c 100644 --- a/libsyscall/wrappers/libproc/libproc.c +++ b/libsyscall/wrappers/libproc/libproc.c @@ -170,6 +170,17 @@ proc_pidfileportinfo(int pid, uint32_t fileport, int flavor, void *buffer, int b return (retval); } +int +proc_piddynkqueueinfo(int pid, int flavor, kqueue_id_t kq_id, void *buffer, int buffersize) +{ + int ret; + + if ((ret = __proc_info(PROC_INFO_CALL_PIDDYNKQUEUEINFO, pid, flavor, (uint64_t)kq_id, buffer, buffersize)) == -1) { + return 0; + } + + return ret; +} int proc_name(int pid, void * buffer, uint32_t buffersize) @@ -199,7 +210,7 @@ proc_name(int pid, void * buffer, uint32_t buffersize) int proc_regionfilename(int pid, uint64_t address, void * buffer, uint32_t buffersize) { - int retval = 0, len; + int retval; struct proc_regionwithpathinfo reginfo; if (buffersize < MAXPATHLEN) { @@ -209,17 +220,9 @@ proc_regionfilename(int pid, uint64_t address, void * buffer, uint32_t buffersiz retval = proc_pidinfo(pid, PROC_PIDREGIONPATHINFO, (uint64_t)address, ®info, sizeof(struct proc_regionwithpathinfo)); if (retval != -1) { - len = (int)strlen(®info.prp_vip.vip_path[0]); - if (len != 0) { - if (len > MAXPATHLEN) - len = MAXPATHLEN; - bcopy(®info.prp_vip.vip_path[0], buffer, len); - return(len); - } - return(0); + return ((int)(strlcpy(buffer, reginfo.prp_vip.vip_path, MAXPATHLEN))); } - return(0); - + return(0); } int @@ -557,90 +560,67 @@ proc_disable_wakemon(pid_t pid) int proc_list_uptrs(int pid, uint64_t *buf, uint32_t bufsz) { - int i, j; - int nfds, nkns; - int count = 0; - int knote_max = 4096; /* arbitrary starting point */ - - /* if buffer is empty, this call simply counts the knotes */ - if (bufsz > 0 && buf == NULL) { - errno = EFAULT; - return -1; - } + return __proc_info(PROC_INFO_CALL_PIDINFO, pid, PROC_PIDLISTUPTRS, 0, + buf, bufsz); +} - /* get the list of FDs for this process */ - struct proc_fdinfo fdlist[OPEN_MAX+1]; - nfds = proc_pidinfo(pid, PROC_PIDLISTFDS, 0, &fdlist[1], OPEN_MAX*sizeof(struct proc_fdinfo)); - if (nfds < 0 || nfds > OPEN_MAX) { - return -1; - } +int +proc_list_dynkqueueids(int pid, kqueue_id_t *buf, uint32_t bufsz) +{ + return __proc_info(PROC_INFO_CALL_PIDINFO, pid, PROC_PIDLISTDYNKQUEUES, 0, + buf, bufsz); +} - /* Add FD -1, the implicit workq kqueue */ - fdlist[0].proc_fd = -1; - fdlist[0].proc_fdtype = PROX_FDTYPE_KQUEUE; - nfds++; - struct kevent_extinfo *kqext = malloc(knote_max * sizeof(struct kevent_extinfo)); - if (!kqext) { - errno = ENOMEM; - return -1; - } +int +proc_setcpu_percentage(pid_t pid, int action, int percentage) +{ + proc_policy_cpuusage_attr_t attr; - for (i = 0; i < nfds; i++) { - if (fdlist[i].proc_fdtype != PROX_FDTYPE_KQUEUE) { - continue; - } + bzero(&attr, sizeof(proc_policy_cpuusage_attr_t)); + attr.ppattr_cpu_attr = action; + attr.ppattr_cpu_percentage = percentage; + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_RESOURCE_USAGE, PROC_POLICY_RUSAGE_CPU, (proc_policy_attribute_t*)&attr, pid, (uint64_t)0) != -1) + return(0); + else + return(errno); +} - again: - nkns = __proc_info(PROC_INFO_CALL_PIDFDINFO, pid, PROC_PIDFDKQUEUE_EXTINFO, - (uint64_t)fdlist[i].proc_fd, kqext, knote_max * sizeof(struct kevent_extinfo)); - if (nkns < 0) { - if (errno == EBADF) { - /* the FD table can change after enumerating the FDs */ - errno = EAGAIN; - } - free(kqext); - return -1; - } +int +proc_clear_cpulimits(pid_t pid) +{ + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_RESTORE, PROC_POLICY_RESOURCE_USAGE, PROC_POLICY_RUSAGE_CPU, NULL, pid, (uint64_t)0) != -1) + return(0); + else + return(errno); +} - if (nkns > knote_max) { - /* there are more knotes than we requested - try again with a - * larger buffer */ - free(kqext); - knote_max = nkns + 32; /* small margin in case of extra knotes */ - kqext = malloc(knote_max * sizeof(struct kevent_extinfo)); - if (!kqext) { - errno = ENOMEM; - return -1; - } - goto again; - } +#if TARGET_OS_EMBEDDED - for (j = 0; j < nkns; j++) { - if (kqext[j].kqext_kev.udata == 0) { - continue; - } +int +proc_setcpu_deadline(pid_t pid, int action, uint64_t deadline) +{ + proc_policy_cpuusage_attr_t attr; - if (bufsz >= sizeof(uint64_t)) { - *buf++ = kqext[j].kqext_kev.udata; - bufsz -= sizeof(uint64_t); - } - count++; - } - } + bzero(&attr, sizeof(proc_policy_cpuusage_attr_t)); + attr.ppattr_cpu_attr = action; + attr.ppattr_cpu_attr_deadline = deadline; + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_RESOURCE_USAGE, PROC_POLICY_RUSAGE_CPU, (proc_policy_attribute_t*)&attr, pid, (uint64_t)0) != -1) + return(0); + else + return(errno); - free(kqext); - return count; } -int -proc_setcpu_percentage(pid_t pid, int action, int percentage) +int +proc_setcpu_percentage_withdeadline(pid_t pid, int action, int percentage, uint64_t deadline) { proc_policy_cpuusage_attr_t attr; bzero(&attr, sizeof(proc_policy_cpuusage_attr_t)); attr.ppattr_cpu_attr = action; attr.ppattr_cpu_percentage = percentage; + attr.ppattr_cpu_attr_deadline = deadline; if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_RESOURCE_USAGE, PROC_POLICY_RUSAGE_CPU, (proc_policy_attribute_t*)&attr, pid, (uint64_t)0) != -1) return(0); else @@ -648,14 +628,87 @@ proc_setcpu_percentage(pid_t pid, int action, int percentage) } int -proc_clear_cpulimits(pid_t pid) +proc_appstate(int pid, int * appstatep) { - if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_RESTORE, PROC_POLICY_RESOURCE_USAGE, PROC_POLICY_RUSAGE_CPU, NULL, pid, (uint64_t)0) != -1) + int state; + + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_GET, PROC_POLICY_APP_LIFECYCLE, PROC_POLICY_APPLIFE_STATE, (proc_policy_attribute_t*)&state, pid, (uint64_t)0) != -1) { + if (appstatep != NULL) + *appstatep = state; + return(0); + } else + return(errno); + +} + +int +proc_setappstate(int pid, int appstate) +{ + int state = appstate; + + switch (state) { + case PROC_APPSTATE_NONE: + case PROC_APPSTATE_ACTIVE: + case PROC_APPSTATE_INACTIVE: + case PROC_APPSTATE_BACKGROUND: + case PROC_APPSTATE_NONUI: + break; + default: + return(EINVAL); + } + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_APP_LIFECYCLE, PROC_POLICY_APPLIFE_STATE, (proc_policy_attribute_t*)&state, pid, (uint64_t)0) != -1) return(0); else return(errno); } +int +proc_devstatusnotify(int devicestatus) +{ + int state = devicestatus; + + switch (devicestatus) { + case PROC_DEVSTATUS_SHORTTERM: + case PROC_DEVSTATUS_LONGTERM: + break; + default: + return(EINVAL); + } + + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_APP_LIFECYCLE, PROC_POLICY_APPLIFE_DEVSTATUS, (proc_policy_attribute_t*)&state, getpid(), (uint64_t)0) != -1) { + return(0); + } else + return(errno); + +} + +int +proc_pidbind(int pid, uint64_t threadid, int bind) +{ + int state = bind; + pid_t passpid = pid; + + switch (bind) { + case PROC_PIDBIND_CLEAR: + passpid = getpid(); /* ignore pid on clear */ + break; + case PROC_PIDBIND_SET: + break; + default: + return(EINVAL); + } + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_APP_LIFECYCLE, PROC_POLICY_APPLIFE_PIDBIND, (proc_policy_attribute_t*)&state, passpid, threadid) != -1) + return(0); + else + return(errno); +} + +int +proc_can_use_foreground_hw(int pid, uint32_t *reason) +{ + return __proc_info(PROC_INFO_CALL_CANUSEFGHW, pid, 0, NULL, reason, sizeof(*reason)); +} +#endif /* TARGET_OS_EMBEDDED */ /* Donate importance to adaptive processes from this process */ @@ -664,11 +717,19 @@ proc_donate_importance_boost() { int rval; +#if TARGET_OS_EMBEDDED + rval = __process_policy(PROC_POLICY_SCOPE_PROCESS, + PROC_POLICY_ACTION_ENABLE, + PROC_POLICY_APPTYPE, + PROC_POLICY_IOS_DONATEIMP, + NULL, getpid(), (uint64_t)0); +#else /* TARGET_OS_EMBEDDED */ rval = __process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_SET, PROC_POLICY_BOOST, PROC_POLICY_IMP_DONATION, NULL, getpid(), 0); +#endif /* TARGET_OS_EMBEDDED */ if (rval == 0) return (0); @@ -808,6 +869,7 @@ proc_denap_assertion_complete(uint64_t assertion_token) return proc_importance_assertion_complete(assertion_token); } +#if !TARGET_OS_EMBEDDED int proc_clear_vmpressure(pid_t pid) @@ -903,6 +965,7 @@ proc_suppress(__unused pid_t pid, __unused uint64_t *generation) #endif /* !TARGET_IPHONE_SIMULATOR */ +#endif /* !TARGET_OS_EMBEDDED */ diff --git a/libsyscall/wrappers/libproc/libproc.h b/libsyscall/wrappers/libproc/libproc.h index 27633ffa4..3cfcfdc70 100644 --- a/libsyscall/wrappers/libproc/libproc.h +++ b/libsyscall/wrappers/libproc/libproc.h @@ -138,6 +138,10 @@ int proc_terminate(pid_t pid, int *sig); * failure and errno set appropriately. */ int proc_list_uptrs(pid_t pid, uint64_t *buffer, uint32_t buffersize); + +int proc_list_dynkqueueids(int pid, kqueue_id_t *buf, uint32_t bufsz); +int proc_piddynkqueueinfo(int pid, int flavor, kqueue_id_t kq_id, void *buffer, + int buffersize); #endif /* PRIVATE */ __END_DECLS diff --git a/libsyscall/wrappers/libproc/libproc_internal.h b/libsyscall/wrappers/libproc/libproc_internal.h index 7169b7eb4..f18366427 100644 --- a/libsyscall/wrappers/libproc/libproc_internal.h +++ b/libsyscall/wrappers/libproc/libproc_internal.h @@ -41,6 +41,52 @@ int proc_clear_cpulimits(pid_t pid) __OSX_AVAILABLE_STARTING(__MAC_10_12_2, __IP /* CPU limits, applies to current thread only. 0% unsets limit */ int proc_setthread_cpupercent(uint8_t percentage, uint32_t ms_refill) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_5_0); +#if TARGET_OS_EMBEDDED + +/* CPU monitor action, continued */ +#define PROC_SETCPU_ACTION_SUSPEND 2 +#define PROC_SETCPU_ACTION_TERMINATE 3 +#define PROC_SETCPU_ACTION_NOTIFY 4 + +int proc_setcpu_deadline(pid_t pid, int action, uint64_t deadline) __OSX_AVAILABLE_STARTING(__MAC_NA, __IPHONE_5_0); +int proc_setcpu_percentage_withdeadline(pid_t pid, int action, int percentage, uint64_t deadline) __OSX_AVAILABLE_STARTING(__MAC_NA, __IPHONE_5_0); + +#define PROC_APPSTATE_NONE 0 +#define PROC_APPSTATE_ACTIVE 1 +#define PROC_APPSTATE_BACKGROUND 2 +#define PROC_APPSTATE_NONUI 3 +#define PROC_APPSTATE_INACTIVE 4 + +int proc_setappstate(int pid, int appstate); +int proc_appstate(int pid, int * appstatep); + +#define PROC_DEVSTATUS_SHORTTERM 1 +#define PROC_DEVSTATUS_LONGTERM 2 + +int proc_devstatusnotify(int devicestatus); + +#define PROC_PIDBIND_CLEAR 0 +#define PROC_PIDBIND_SET 1 +int proc_pidbind(int pid, uint64_t threadid, int bind); + +/* + * High level check to see if a process is allowed to use HW + * resources reserved for foreground applications. + * Returns: + * 1 if the PID is allowed + * 0 if the PID is NOT allowed + * <0 on error + * + * When 0 is returned, 'reason' is set to indicate why + * the pid is not allowed to use foreground-only hardware. + * Reasons returned by the kernel are found in + * + * When <0 is returned, errno indicates the reason + * for the failure. + */ +int proc_can_use_foreground_hw(int pid, uint32_t *reason); + +#else /* TARGET_OS_EMBEDDED */ /* resume the process suspend due to low VM resource */ int proc_clear_vmpressure(pid_t pid); @@ -67,6 +113,7 @@ int proc_clear_delayidlesleep(void); int proc_disable_apptype(pid_t pid, int apptype); int proc_enable_apptype(pid_t pid, int apptype); +#endif /* TARGET_OS_EMBEDDED */ /* mark process as importance donating */ int proc_donate_importance_boost(void); diff --git a/libsyscall/wrappers/mach_absolute_time.s b/libsyscall/wrappers/mach_absolute_time.s index 603504a9a..cf93161b9 100644 --- a/libsyscall/wrappers/mach_absolute_time.s +++ b/libsyscall/wrappers/mach_absolute_time.s @@ -150,6 +150,130 @@ _mach_absolute_time: popq %rbp ret +#elif defined(__arm__) + +#include + +/* + * If userspace access to the timebase is supported (indicated through the commpage), + * directly reads the timebase and uses it and the current timebase offset (also in + * the commpage, and updated whenever the system wakes from sleep) to construct the + * current time value; otherwise, traps to the kernel to handle this. + * + * If we do this in user mode, there are two cases where we may need to redrive the + * read. We do 3 reads (high-low-high) to the timebase, because we only have a + * 32-bit interface to it (despite the use of mrrc). If the high bits change, we + * need to reread the register (as our returned value could otherwise be off by + * 2^32 mach absolute time units). + * + * We do two reads of the offset, before and after the register reads. If the offset + * changes, we have gone to sleep in the midst of doing a read. This case should be + * exceedingly rare, but could result in a terribly inaccurate result, so we need + * to get a fresh timebase value. + */ + .text + .align 2 + .globl _mach_absolute_time +_mach_absolute_time: + movw ip, #((_COMM_PAGE_TIMEBASE_OFFSET) & 0x0000FFFF) + movt ip, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 16) & 0x0000FFFF) + ldrb r0, [ip, #((_COMM_PAGE_USER_TIMEBASE) - (_COMM_PAGE_TIMEBASE_OFFSET))] + cmp r0, #0 // Are userspace reads supported? + beq _mach_absolute_time_kernel // If not, go to the kernel + isb // Prevent speculation on CNTPCT across calls + // (see ARMV7C.b section B8.1.2, ARMv8 section D6.1.2) + push {r4, r5, r7, lr} // Push a frame + add r7, sp, #8 +L_mach_absolute_time_user: + ldr r4, [ip] // Load offset low bits + ldr r5, [ip, #4] // Load offset high bits + mrrc p15, 0, r3, r1, c14 // Read timebase high to r1 + mrrc p15, 0, r0, r3, c14 // Read timebase low to r0 + mrrc p15, 0, r3, r2, c14 // Read timebase high to r2 + cmp r1, r2 // Did the high bits change? + bne L_mach_absolute_time_user // Loop if timebase high changed + ldr r2, [ip] // Load offset low bits + ldr r3, [ip, #4] // Load offset high bits + eor r4, r2 // Compare our offset values... + eor r5, r3 + orrs r5, r4 + bne L_mach_absolute_time_user // If they changed, try again + adds r0, r0, r2 // Construct mach_absolute_time + adcs r1, r1, r3 + pop {r4, r5, r7, pc} // Pop the frame + + .text + .align 2 + .globl _mach_absolute_time_kernel +_mach_absolute_time_kernel: + mov r12, #-3 // Load the magic MAT number + swi #SWI_SYSCALL + bx lr + + .text + .align 2 + .globl _mach_continuous_time_kernel +_mach_continuous_time_kernel: + mov r12, #-4 // Load the magic MCT number + swi #SWI_SYSCALL + bx lr + +#elif defined(__arm64__) + +#include + +/* + * If userspace access to the timebase is supported (indicated through the commpage), + * directly reads the timebase and uses it and the current timebase offset (also in + * the commpage, and updated whenever the system wakes from sleep) to construct the + * current time value; otherwise, traps to the kernel to handle this. + * + * If we do this in user mode, we do two reads of the offset, before and after we + * read the register. If the offset changes, we have gone to sleep in the midst of + * doing a read. This case should be exceedingly rare, but could result in a terribly + * inaccurate result, so we need to get a fresh timebase value. + * + * Note that the commpage address construction expects our top 2 bytes to be 0xFFFF. + * If this changes (i.e, we significantly relocate the commpage), this logic will need + * to change as well (use 4 movk instructions rather than cheating with the movn). + */ + .text + .align 2 + .globl _mach_absolute_time +_mach_absolute_time: + movn x3, #(~((_COMM_PAGE_TIMEBASE_OFFSET) >> 32) & 0x000000000000FFFF), lsl #32 + movk x3, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 16) & 0x000000000000FFFF), lsl #16 + movk x3, #((_COMM_PAGE_TIMEBASE_OFFSET) & 0x000000000000FFFF) + ldrb w2, [x3, #((_COMM_PAGE_USER_TIMEBASE) - (_COMM_PAGE_TIMEBASE_OFFSET))] + cmp x2, #0 // Are userspace reads supported? + b.eq _mach_absolute_time_kernel // If not, go to the kernel + isb // Prevent speculation on CNTPCT across calls + // (see ARMV7C.b section B8.1.2, ARMv8 section D6.1.2) +L_mach_absolute_time_user: + ldr x1, [x3] // Load the offset + mrs x0, CNTPCT_EL0 // Read the timebase + ldr x2, [x3] // Load the offset + cmp x1, x2 // Compare our offset values... + b.ne L_mach_absolute_time_user // If they changed, try again + add x0, x0, x1 // Construct mach_absolute_time + ret + + .text + .align 2 + .globl _mach_absolute_time_kernel +_mach_absolute_time_kernel: + mov w16, #-3 // Load the magic MAT number + svc #SWI_SYSCALL + ret + + .text + .align 2 + .globl _mach_continuous_time_kernel +_mach_continuous_time_kernel: + mov w16, #-4 // Load the magic MCT number + svc #SWI_SYSCALL + ret + #else #error Unsupported architecture #endif diff --git a/libsyscall/wrappers/mach_approximate_time.s b/libsyscall/wrappers/mach_approximate_time.s index 7ef3336f7..f1e6ed871 100644 --- a/libsyscall/wrappers/mach_approximate_time.s +++ b/libsyscall/wrappers/mach_approximate_time.s @@ -29,7 +29,45 @@ #include #include -#if defined(__i386__) +#if defined(__arm__) + + .text + .align 2 + .globl _mach_approximate_time +_mach_approximate_time: + + movw r0, #((_COMM_PAGE_APPROX_TIME_SUPPORTED>>0)&0x0FFFF) + movt r0, #((_COMM_PAGE_APPROX_TIME_SUPPORTED>>16)&0x0FFFF) + ldrb r0, [r0] // load COMM_PAGE_APPROX_TIME_SUPPORTED + cmp r0, #1 // check if approx time is supported + + bne _mach_absolute_time // if not supported, fall through to + // absolute_time + + movw r2, #((_COMM_PAGE_APPROX_TIME>>0)&0x0FFFF) + movt r2, #((_COMM_PAGE_APPROX_TIME>>16)&0x0FFFF) + + // at this point, r2->COMM_PAGE_APPROX_TIME, which is a 64-bit value. + // Since this is a 32-bit architecture, and the commpage is readonly, + // there is no "guaranteed" atomic way to read all 64-bits with + // hardware protection. Even the ldrd instruction is not guaranteed to + // complete atomically. The solution is to use a 32-bit high/low/high + // read with a consistency check on the high bits. To further + // complicate things, reading the same location in memory back to back + // could trigger a predictive read, which would defeat the purpose of + // doing the consistency check so we insert a data memory barrier to + // prevent this. +_consistency_check: + ldr r1, [r2,#4] // load high + ldr r0, [r2] // load low + dsb // avoid predictive reads that could + // be invalid if interrupted + ldr r3, [r2,#4] // load high again + cmp r1, r3 // check that high1 == high2 + bne _consistency_check // try again if not + bx lr + +#elif defined(__i386__) .text .align 4, 0x90 diff --git a/libsyscall/wrappers/mach_continuous_time.c b/libsyscall/wrappers/mach_continuous_time.c index 4f20664f8..61b996de7 100644 --- a/libsyscall/wrappers/mach_continuous_time.c +++ b/libsyscall/wrappers/mach_continuous_time.c @@ -36,7 +36,9 @@ _mach_continuous_time_base(void) volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE; uint64_t read1, read2; read1 = *base_ptr; -#if defined(__i386__) +#if defined(__arm__) + __asm__ volatile("dsb sy" ::: "memory"); +#elif defined(__i386__) __asm__ volatile("lfence" ::: "memory"); #else #error "unsupported arch" @@ -51,6 +53,21 @@ _mach_continuous_time_base(void) #endif // 64-bit } +__attribute__((visibility("hidden"))) +kern_return_t +_mach_continuous_hwclock(uint64_t *cont_time __unused) +{ +#if defined(__arm64__) + uint8_t cont_hwclock = *((uint8_t*)_COMM_PAGE_CONT_HWCLOCK); + uint64_t timebase; + if (cont_hwclock) { + __asm__ volatile("isb\n" "mrs %0, CNTPCT_EL0" : "=r"(timebase)); + *cont_time = timebase; + return KERN_SUCCESS; + } +#endif + return KERN_NOT_SUPPORTED; +} __attribute__((visibility("hidden"))) kern_return_t @@ -63,6 +80,13 @@ _mach_continuous_time(uint64_t* absolute_time, uint64_t* cont_time) do { read1 = *base_ptr; absolute = mach_absolute_time(); +#if defined(__arm__) || defined(__arm64__) + /* + * mach_absolute_time() contains an instruction barrier which will + * prevent the speculation of read2 above this point, so we don't + * need another barrier here. + */ +#endif read2 = *base_ptr; } while (__builtin_expect((read1 != read2), 0)); @@ -76,17 +100,18 @@ uint64_t mach_continuous_time(void) { uint64_t cont_time; - _mach_continuous_time(NULL, &cont_time); + if (_mach_continuous_hwclock(&cont_time) != KERN_SUCCESS) + _mach_continuous_time(NULL, &cont_time); return cont_time; } uint64_t mach_continuous_approximate_time(void) { - /* - * No retry loop here because if we use a slightly too old timebase that's - * okay, we are approximate time anyway. - */ - volatile register uint64_t time_base = _mach_continuous_time_base(); - return time_base + mach_approximate_time(); + /* + * No retry loop here because if we use a slightly too old timebase that's + * okay, we are approximate time anyway. + */ + volatile register uint64_t time_base = _mach_continuous_time_base(); + return time_base + mach_approximate_time(); } diff --git a/libsyscall/wrappers/mach_get_times.c b/libsyscall/wrappers/mach_get_times.c index 37ddfa9fd..b078c8eb8 100644 --- a/libsyscall/wrappers/mach_get_times.c +++ b/libsyscall/wrappers/mach_get_times.c @@ -63,8 +63,12 @@ mach_get_times(uint64_t* absolute_time, uint64_t* cont_time, struct timespec *tp if (__gettimeofday_with_mach(&tv, NULL, &tbr) < 0) { return KERN_FAILURE; } else if (tbr == 0) { +#if !TARGET_OS_EMBEDDED // On an old kernel, likely chroot'ed. (remove next year) tbr = mach_absolute_time(); +#else + __builtin_trap(); +#endif } } diff --git a/libsyscall/wrappers/pid_shutdown_networking.c b/libsyscall/wrappers/pid_shutdown_networking.c new file mode 100644 index 000000000..7d96044be --- /dev/null +++ b/libsyscall/wrappers/pid_shutdown_networking.c @@ -0,0 +1,33 @@ +/* + * Copyright (c) 20017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#include + +/* + * Wrapper for pid_shutdown_sockets. + */ +int +pid_shutdown_networking(int pid, int level) +{ + return pid_shutdown_sockets(pid, level); +} diff --git a/libsyscall/wrappers/quota_obsolete.c b/libsyscall/wrappers/quota_obsolete.c index 435339b2c..aa566c0c6 100644 --- a/libsyscall/wrappers/quota_obsolete.c +++ b/libsyscall/wrappers/quota_obsolete.c @@ -25,6 +25,7 @@ #include #include +#if !TARGET_OS_EMBEDDED /* * system call stubs are no longer generated for these from * syscalls.master. Instead, provide simple stubs here. @@ -42,3 +43,4 @@ int setquota(void) { return kill(getpid(), SIGSYS); } +#endif /* !TARGET_OS_EMBEDDED */ diff --git a/libsyscall/wrappers/remove-counter.c b/libsyscall/wrappers/remove-counter.c index fe41f2757..f1757a654 100644 --- a/libsyscall/wrappers/remove-counter.c +++ b/libsyscall/wrappers/remove-counter.c @@ -22,6 +22,9 @@ */ #include +#if defined(__arm__) +#include +#endif #if defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) static int64_t __remove_counter = 0; @@ -31,11 +34,19 @@ static int32_t __remove_counter = 0; __uint64_t __get_remove_counter(void) { +#if defined(__arm__) && !defined(_ARM_ARCH_6) + return __remove_counter; +#else return __sync_add_and_fetch(&__remove_counter, 0); +#endif } void __inc_remove_counter(void) { +#if defined(__arm__) && !defined(_ARM_ARCH_6) + __remove_counter++; +#else __sync_add_and_fetch(&__remove_counter, 1); +#endif } diff --git a/libsyscall/wrappers/spawn/posix_spawn.c b/libsyscall/wrappers/spawn/posix_spawn.c index be3e94cea..69002dc0b 100644 --- a/libsyscall/wrappers/spawn/posix_spawn.c +++ b/libsyscall/wrappers/spawn/posix_spawn.c @@ -1331,6 +1331,41 @@ posix_spawnattr_getcpumonitor(posix_spawnattr_t * __restrict attr, return (0); } +#if TARGET_OS_EMBEDDED +/* + * posix_spawnattr_setjetsam + * + * Description: Set jetsam attributes for the spawn attribute object + * referred to by 'attr'. + * + * Parameters: flags The flags value to set + * priority Relative jetsam priority + * memlimit Value in megabytes; a memory footprint + * above this level may result in termination. + * Implies both active and inactive limits. + * + * Returns: 0 Success + * + * Note: to be deprecated (not available on desktop) + * + */ +int +posix_spawnattr_setjetsam(posix_spawnattr_t * __restrict attr, + short flags, int priority, int memlimit) +{ + short flags_ext = flags; + + if (flags & POSIX_SPAWN_JETSAM_MEMLIMIT_FATAL) { + flags_ext |= POSIX_SPAWN_JETSAM_MEMLIMIT_ACTIVE_FATAL; + flags_ext |= POSIX_SPAWN_JETSAM_MEMLIMIT_INACTIVE_FATAL; + } else { + flags_ext &= ~POSIX_SPAWN_JETSAM_MEMLIMIT_ACTIVE_FATAL; + flags_ext &= ~POSIX_SPAWN_JETSAM_MEMLIMIT_INACTIVE_FATAL; + } + + return (posix_spawnattr_setjetsam_ext(attr, flags_ext, priority, memlimit, memlimit)); +} +#endif /* TARGET_OS_EMBEDDED */ /* * posix_spawnattr_setjetsam_ext diff --git a/libsyscall/wrappers/spawn/spawn_private.h b/libsyscall/wrappers/spawn/spawn_private.h index ec7f50fb6..bebd58e60 100644 --- a/libsyscall/wrappers/spawn/spawn_private.h +++ b/libsyscall/wrappers/spawn/spawn_private.h @@ -40,6 +40,10 @@ int posix_spawnattr_setcpumonitor(posix_spawnattr_t * __restrict, uint64_t, uint int posix_spawnattr_getcpumonitor(posix_spawnattr_t * __restrict, uint64_t *, uint64_t *) __OSX_AVAILABLE_STARTING(__MAC_10_8, __IPHONE_6_0); int posix_spawnattr_setcpumonitor_default(posix_spawnattr_t * __restrict) __OSX_AVAILABLE_STARTING(__MAC_10_9, __IPHONE_6_0); +#if TARGET_OS_EMBEDDED +int posix_spawnattr_setjetsam(posix_spawnattr_t * __restrict attr, + short flags, int priority, int memlimit) __OSX_AVAILABLE_STARTING(__MAC_NA, __IPHONE_5_0); +#endif /* TARGET_OS_EMBEDDED */ int posix_spawnattr_setjetsam_ext(posix_spawnattr_t * __restrict attr, short flags, int priority, int memlimit_active, int memlimit_inactive) __OSX_AVAILABLE_STARTING(__MAC_10_11, __IPHONE_9_0); diff --git a/libsyscall/wrappers/thread_register_state.c b/libsyscall/wrappers/thread_register_state.c index 2fa478328..e1181d251 100644 --- a/libsyscall/wrappers/thread_register_state.c +++ b/libsyscall/wrappers/thread_register_state.c @@ -44,6 +44,14 @@ thread_get_register_pointer_values(thread_t thread, uintptr_t *sp, size_t *lengt x86_thread_state64_t state = {}; thread_state_flavor_t flavor = x86_THREAD_STATE64; mach_msg_type_number_t count = x86_THREAD_STATE64_COUNT; +#elif defined(__arm__) + arm_thread_state_t state = {}; + thread_state_flavor_t flavor = ARM_THREAD_STATE; + mach_msg_type_number_t count = ARM_THREAD_STATE_COUNT; +#elif defined(__arm64__) + arm_thread_state64_t state = {}; + thread_state_flavor_t flavor = ARM_THREAD_STATE64; + mach_msg_type_number_t count = ARM_THREAD_STATE64_COUNT; #else #error thread_get_register_pointer_values not defined for this architecture #endif @@ -76,7 +84,12 @@ thread_get_register_pointer_values(thread_t thread, uintptr_t *sp, size_t *lengt push_register_value(state.__esi); push_register_value(state.__ebp); #elif defined(__x86_64__) - if (sp) *sp = state.__rsp - 128 /* redzone */; + if (sp) { + if (state.__rsp > 128) + *sp = state.__rsp - 128 /* redzone */; + else + *sp = 0; + } push_register_value(state.__rip); @@ -94,6 +107,29 @@ thread_get_register_pointer_values(thread_t thread, uintptr_t *sp, size_t *lengt push_register_value(state.__r13); push_register_value(state.__r14); push_register_value(state.__r15); +#elif defined(__arm__) + if (sp) *sp = state.__sp; + + push_register_value(state.__pc); + push_register_value(state.__lr); + + for (int i = 0; i < 13; i++){ + push_register_value(state.__r[i]); + } +#elif defined(__arm64__) + if (sp) { + if (state.__sp > 128) + *sp = state.__sp - 128 /* redzone */; + else + *sp = 0; + } + + push_register_value(state.__pc); + push_register_value(state.__lr); + + for (int i = 0; i < 29; i++){ + push_register_value(state.__x[i]); + } #else #error thread_get_register_pointer_values not defined for this architecture #endif diff --git a/libsyscall/wrappers/utimensat.c b/libsyscall/wrappers/utimensat.c new file mode 100644 index 000000000..6deaf45a3 --- /dev/null +++ b/libsyscall/wrappers/utimensat.c @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2006, 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include + +extern int __gettimeofday(struct timeval *, struct timezone *); +extern int __commpage_gettimeofday(struct timeval *); + +static struct timespec times_now[2] = { + { .tv_nsec = UTIME_NOW }, + { .tv_nsec = UTIME_NOW } +}; + +/* + * Resolve any UTIME_NOW or UTIME_OMIT and return the attributes buffer and + * attributes to pass. Assumes times_in is writable. + */ +static int +prepare_times_array_and_attrs(struct timespec times_in[2], + struct timespec times_out[2], size_t *times_out_size) +{ + if (times_in[0].tv_nsec == UTIME_OMIT && + times_in[1].tv_nsec == UTIME_OMIT) { + return 0; + } + + if (times_in[0].tv_nsec == UTIME_NOW || + times_in[1].tv_nsec == UTIME_NOW) { + struct timespec now = {}; + { + /* + * TODO: Replace with nanosecond time when available + */ + struct timeval tv; + if (__commpage_gettimeofday(&tv) != 0) { + __gettimeofday(&tv, NULL); + } + TIMEVAL_TO_TIMESPEC(&tv, &now); + } + + if (times_in[0].tv_nsec == UTIME_NOW) { + times_in[0] = now; + } + if (times_in[1].tv_nsec == UTIME_NOW) { + times_in[1] = now; + } + } + + int attrs = 0; + *times_out_size = 0; + struct timespec *times_cursor = times_out; + if (times_in[1].tv_nsec != UTIME_OMIT) { + attrs |= ATTR_CMN_MODTIME; + *times_cursor++ = times_in[1]; + *times_out_size += sizeof(struct timespec); + } + if (times_in[0].tv_nsec != UTIME_OMIT) { + attrs |= ATTR_CMN_ACCTIME; + *times_cursor = times_in[0]; + *times_out_size += sizeof(struct timespec); + } + return attrs; +} + +int +futimens(int fd, const struct timespec _times_in[2]) +{ + struct timespec times_in[2]; + if (_times_in != NULL) { + memcpy(×_in, _times_in, sizeof(times_in)); + } else { + memcpy(×_in, times_now, sizeof(times_in)); + } + + size_t attrbuf_size = 0; + struct timespec times_out[2] = {}; + struct attrlist a = { + .bitmapcount = ATTR_BIT_MAP_COUNT + }; + a.commonattr = prepare_times_array_and_attrs(times_in, times_out, &attrbuf_size); + + return fsetattrlist(fd, &a, ×_out, attrbuf_size, 0); +} + +int +utimensat(int fd, const char *path, const struct timespec _times_in[2], int flags) +{ + struct timespec times_in[2]; + if (_times_in != NULL) { + memcpy(×_in, _times_in, sizeof(times_in)); + } else { + memcpy(×_in, times_now, sizeof(times_in)); + } + + size_t attrbuf_size = 0; + struct timespec times_out[2] = {}; + struct attrlist a = { + .bitmapcount = ATTR_BIT_MAP_COUNT + }; + a.commonattr = prepare_times_array_and_attrs(times_in, times_out, &attrbuf_size); + + int flags_out = 0; + if (flags & AT_SYMLINK_NOFOLLOW) { + flags_out |= FSOPT_NOFOLLOW; + } + + return setattrlistat(fd, path, &a, ×_out, attrbuf_size, flags_out); +} diff --git a/libsyscall/wrappers/varargs_wrappers.s b/libsyscall/wrappers/varargs_wrappers.s index bc6d6c3a4..6a22a5395 100644 --- a/libsyscall/wrappers/varargs_wrappers.s +++ b/libsyscall/wrappers/varargs_wrappers.s @@ -21,3 +21,127 @@ * @APPLE_LICENSE_HEADER_END@ */ +#ifdef __arm64__ + +#include "../custom/SYS.h" +#include + +/* + * Stubs are to handle the ARM64 ABI for variadic functions' + * not matching the ABI used by the system call handler. + */ + +/* + * sem_t* sem_open(const char *name, int oflag, ...); + * sem_t* __sem_open(const char *name, int oflag, int mode, int value); + */ +MI_ENTRY_POINT(_sem_open) + PUSH_FRAME + ldp x2, x3, [fp, #16] + MI_CALL_EXTERNAL(___sem_open) + POP_FRAME + ret + +/* + * int open(const char *name, int oflag, ...); + * int __open(const char *name, int oflag, int mode, int value); + */ +MI_ENTRY_POINT(_open) + PUSH_FRAME + ldr x2, [fp, #16] + MI_CALL_EXTERNAL(___open) + POP_FRAME + ret + +/* + * int open_nocancel(const char *name, int oflag, ...); + * int __open_nocancel(const char *name, int oflag, int mode); + */ +MI_ENTRY_POINT(_open$NOCANCEL) + PUSH_FRAME + ldr x2, [fp, #16] + MI_CALL_EXTERNAL(___open_nocancel) + POP_FRAME + ret + +/* + * int openat(int fd,const char *name, int oflag, ...); + * int __openat(int fd, const char *name, int oflag, int mode, int value); + */ +MI_ENTRY_POINT(_openat) + PUSH_FRAME + ldr x3, [fp, #16] + MI_CALL_EXTERNAL(___openat) + POP_FRAME + ret + +/* + * int openat_nocancel(int fd, const char *name, int oflag, ...); + * int __openat_nocancel(int fd, const char *name, int oflag, int mode); + */ +MI_ENTRY_POINT(_openat$NOCANCEL) + PUSH_FRAME + ldr x3, [fp, #16] + MI_CALL_EXTERNAL(___openat_nocancel) + POP_FRAME + ret + +/* + * int shm_open(const char *, int, ...); + * int __shm_open(const char*, int oflag, int mode); + */ +MI_ENTRY_POINT(_shm_open) + PUSH_FRAME + ldr x2, [fp, #16] + MI_CALL_EXTERNAL(___shm_open) + POP_FRAME + ret + +/* + * int msgsys(int, ...); + * int __msgsys(int which, int a2, int a3, int a4, int a5); + */ +MI_ENTRY_POINT(_msgsys) + PUSH_FRAME + ldp x1, x2, [fp, #16] + ldp x3, x4, [fp, #32] + MI_CALL_EXTERNAL(___msgsys) + POP_FRAME + ret + +/* + * int semsys(int, ...); + * int __semsys(int which, int a2, int a3, int a4, int a5); + */ +MI_ENTRY_POINT(_semsys) + PUSH_FRAME + ldp x1, x2, [fp, #16] + ldp x3, x4, [fp, #32] + MI_CALL_EXTERNAL(___semsys) + POP_FRAME + ret + +/* + * int semctl(int, int, int, ...); + * int __semctl(int semid, int semnum, int cmd, semun_t arg); + */ + MI_ENTRY_POINT(_semctl) + PUSH_FRAME + ldr x3, [fp, #16] + MI_CALL_EXTERNAL(___semctl) + POP_FRAME + ret + +/* + * int shmsys(int, ...); + * int __shmsys(int which, int a2, int a3, int a4); + */ + MI_ENTRY_POINT(_shmsys) + PUSH_FRAME + ldp x1, x2, [fp, #16] + ldr x3, [fp, #32] + MI_CALL_EXTERNAL(___shmsys) + POP_FRAME + ret + +#endif /* defined(__arm64__) */ diff --git a/libsyscall/wrappers/work_interval.c b/libsyscall/wrappers/work_interval.c index 29dd2ad61..96ca9ef83 100644 --- a/libsyscall/wrappers/work_interval.c +++ b/libsyscall/wrappers/work_interval.c @@ -23,26 +23,41 @@ #include #include #include + +#include #include +#include + #include #include struct work_interval { uint64_t thread_id; uint64_t work_interval_id; + uint32_t create_flags; + mach_port_t wi_port; }; extern uint64_t __thread_selfid(void); -/* Create a new work interval handle (currently for the current thread only). Flags is unused */ +/* Create a new work interval handle (currently for the current thread only). */ int -work_interval_create(work_interval_t *interval_handle, uint32_t flags __unused) +work_interval_create(work_interval_t *interval_handle, uint32_t create_flags) { int ret; - uint64_t work_interval_id; work_interval_t handle; - ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_CREATE, 0, &work_interval_id, sizeof(work_interval_id)); + if (interval_handle == NULL) { + errno = EINVAL; + return -1; + } + + struct work_interval_create_params create_params = { + .wicp_create_flags = create_flags, + }; + + ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_CREATE2, 0, + &create_params, sizeof(create_params)); if (ret == -1) { return ret; } @@ -54,14 +69,18 @@ work_interval_create(work_interval_t *interval_handle, uint32_t flags __unused) } handle->thread_id = __thread_selfid(); - handle->work_interval_id = work_interval_id; + handle->work_interval_id = create_params.wicp_id; + handle->create_flags = create_params.wicp_create_flags; + handle->wi_port = create_params.wicp_port; *interval_handle = handle; return 0; } int -work_interval_notify(work_interval_t interval_handle, uint64_t start, uint64_t finish, uint64_t deadline, uint64_t next_start, uint32_t flags) +work_interval_notify(work_interval_t interval_handle, uint64_t start, + uint64_t finish, uint64_t deadline, uint64_t next_start, + uint32_t notify_flags) { int ret; uint64_t work_interval_id; @@ -70,8 +89,7 @@ work_interval_notify(work_interval_t interval_handle, uint64_t start, uint64_t f .finish = finish, .deadline = deadline, .next_start = next_start, - .flags = flags, - .unused1 = 0 + .notify_flags = notify_flags }; if (interval_handle == NULL) { @@ -79,35 +97,156 @@ work_interval_notify(work_interval_t interval_handle, uint64_t start, uint64_t f return -1; } + notification.create_flags = interval_handle->create_flags; work_interval_id = interval_handle->work_interval_id; - ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_NOTIFY, work_interval_id, ¬ification, sizeof(notification)); + ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_NOTIFY, work_interval_id, + ¬ification, sizeof(notification)); return ret; } int -work_interval_notify_simple(work_interval_t interval_handle, uint64_t start, uint64_t deadline, uint64_t next_start) +work_interval_notify_simple(work_interval_t interval_handle, uint64_t start, + uint64_t deadline, uint64_t next_start) { - return work_interval_notify(interval_handle, start, mach_absolute_time(), deadline, next_start, 0); + return work_interval_notify(interval_handle, start, mach_absolute_time(), + deadline, next_start, 0); } int work_interval_destroy(work_interval_t interval_handle) { - int ret, saved_errno; - uint64_t work_interval_id; + if (interval_handle == NULL) { + errno = EINVAL; + return -1; + } + + if (interval_handle->create_flags & WORK_INTERVAL_FLAG_JOINABLE) { + mach_port_t wi_port = interval_handle->wi_port; + + /* + * A joinable work interval's lifetime is tied to the port lifetime. + * When the last port reference is destroyed, the work interval + * is destroyed via no-senders notification. + * + * Note however that after destroy it can no longer be notified + * because the userspace token is gone. + * + * Additionally, this function does not cause the thread to un-join + * the interval. + */ + kern_return_t kr = mach_port_deallocate(mach_task_self(), wi_port); + + if (kr != KERN_SUCCESS) { + /* + * If the deallocate fails, then someone got their port + * lifecycle wrong and over-released a port right. + * + * Return an error so the client can assert on this, + * and still find the port name in the interval handle. + */ + errno = EINVAL; + return -1; + } + + interval_handle->wi_port = MACH_PORT_NULL; + interval_handle->work_interval_id = 0; + + free(interval_handle); + return 0; + } else { + uint64_t work_interval_id = interval_handle->work_interval_id; + + int ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_DESTROY, + work_interval_id, NULL, 0); + + interval_handle->work_interval_id = 0; + + int saved_errno = errno; + free(interval_handle); + errno = saved_errno; + return ret; + } +} +int +work_interval_join(work_interval_t interval_handle) +{ if (interval_handle == NULL) { errno = EINVAL; return -1; } - work_interval_id = interval_handle->work_interval_id; + if ((interval_handle->create_flags & WORK_INTERVAL_FLAG_JOINABLE) == 0) { + errno = EINVAL; + return -1; + } - ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_DESTROY, work_interval_id, NULL, 0); - saved_errno = errno; - free(interval_handle); - errno = saved_errno; + mach_port_t wi_port = interval_handle->wi_port; - return ret; + if (!MACH_PORT_VALID(wi_port)) { + errno = EINVAL; + return -1; + } + + return work_interval_join_port(wi_port); } + +int +work_interval_join_port(mach_port_t port) +{ + if (port == MACH_PORT_NULL) { + errno = EINVAL; + return -1; + } + + return __work_interval_ctl(WORK_INTERVAL_OPERATION_JOIN, + (uint64_t)port, NULL, 0); +} + +int +work_interval_leave(void) +{ + return __work_interval_ctl(WORK_INTERVAL_OPERATION_JOIN, + (uint64_t)MACH_PORT_NULL, NULL, 0); +} + +int +work_interval_copy_port(work_interval_t interval_handle, mach_port_t *port) +{ + if (port == NULL) { + errno = EINVAL; + return -1; + } + + if (interval_handle == NULL) { + *port = MACH_PORT_NULL; + errno = EINVAL; + return -1; + } + + if ((interval_handle->create_flags & WORK_INTERVAL_FLAG_JOINABLE) == 0) { + *port = MACH_PORT_NULL; + errno = EINVAL; + return -1; + } + + mach_port_t wi_port = interval_handle->wi_port; + + kern_return_t kr = mach_port_mod_refs(mach_task_self(), wi_port, + MACH_PORT_RIGHT_SEND, 1); + + if (kr != KERN_SUCCESS) { + *port = MACH_PORT_NULL; + errno = EINVAL; + return -1; + } + + *port = wi_port; + + return 0; +} + + + + diff --git a/libsyscall/xcodescripts/create-syscalls.pl b/libsyscall/xcodescripts/create-syscalls.pl index 54514f454..81dfc8a8a 100755 --- a/libsyscall/xcodescripts/create-syscalls.pl +++ b/libsyscall/xcodescripts/create-syscalls.pl @@ -222,6 +222,7 @@ sub checkForCustomStubs { foreach my $subarch (@Architectures) { (my $arch = $subarch) =~ s/arm(v.*)/arm/; $arch =~ s/x86_64(.*)/x86_64/; + $arch =~ s/arm64(.*)/arm64/; $$sym{aliases}{$arch} = [] unless $$sym{aliases}{$arch}; push(@{$$sym{aliases}{$arch}}, $$sym{asm_sym}); } @@ -244,6 +245,7 @@ sub readAliases { for my $arch (@Architectures) { (my $new_arch = $arch) =~ s/arm(v.*)/arm/g; $new_arch =~ s/x86_64(.*)/x86_64/g; + $new_arch =~ s/arm64(.*)/arm64/g; push(@a, $new_arch) unless grep { $_ eq $new_arch } @a; } @@ -302,6 +304,7 @@ sub writeStubForSymbol { for my $subarch (@Architectures) { (my $arch = $subarch) =~ s/arm(v.*)/arm/; $arch =~ s/x86_64(.*)/x86_64/; + $arch =~ s/arm64(.*)/arm64/; push(@conditions, "defined(__${arch}__)") unless grep { $_ eq $arch } @{$$symbol{except}}; } @@ -334,6 +337,7 @@ sub writeAliasesForSymbol { foreach my $subarch (@Architectures) { (my $arch = $subarch) =~ s/arm(v.*)/arm/; $arch =~ s/x86_64(.*)/x86_64/; + $arch =~ s/arm64(.*)/arm64/; next unless scalar($$symbol{aliases}{$arch}); diff --git a/libsyscall/xcodescripts/mach_install_mig.sh b/libsyscall/xcodescripts/mach_install_mig.sh index 7457fa195..94cc8dbb7 100755 --- a/libsyscall/xcodescripts/mach_install_mig.sh +++ b/libsyscall/xcodescripts/mach_install_mig.sh @@ -42,17 +42,17 @@ MACH_HEADER_DST="$BUILT_PRODUCTS_DIR/mig_hdr/include/mach" # from old Libsystem makefiles MACHINE_ARCH=`echo $ARCHS | cut -d' ' -f 1` -if [[ ( "$MACHINE_ARCH" = "arm64" || "$MACHINE_ARCH" = "x86_64" || "$MACHINE_ARCH" = "x86_64h" ) && `echo $ARCHS | wc -w` -gt 1 ]] +if [[ ( "$MACHINE_ARCH" =~ ^"arm64" || "$MACHINE_ARCH" =~ ^"x86_64" ) && `echo $ARCHS | wc -w` -gt 1 ]] then # MACHINE_ARCH needs to be a 32-bit arch to generate vm_map_internal.h correctly. MACHINE_ARCH=`echo $ARCHS | cut -d' ' -f 2` - if [[ ( "$MACHINE_ARCH" = "arm64" || "$MACHINE_ARCH" = "x86_64" || "$MACHINE_ARCH" = "x86_64h" ) && `echo $ARCHS | wc -w` -gt 1 ]] + if [[ ( "$MACHINE_ARCH" =~ ^"arm64" || "$MACHINE_ARCH" =~ ^"x86_64" ) && `echo $ARCHS | wc -w` -gt 1 ]] then # MACHINE_ARCH needs to be a 32-bit arch to generate vm_map_internal.h correctly. MACHINE_ARCH=`echo $ARCHS | cut -d' ' -f 3` fi fi -if [[ ( "$MACHINE_ARCH" = "arm64" ) ]] +if [[ ( "$MACHINE_ARCH" =~ ^"arm64" ) ]] then # MACHINE_ARCH *really* needs to be a 32-bit arch to generate vm_map_internal.h correctly, even if there are no 32-bit targets. MACHINE_ARCH="armv7" @@ -88,7 +88,7 @@ MIGS_PRIVATE="" MIGS_DUAL_PUBLIC_PRIVATE="" -if ( echo {iphone,tv,appletv,watch}{os,simulator} iphone{osnano,nanosimulator} | grep -wFq "$PLATFORM_NAME" ) +if ( echo {iphone,tv,appletv,watch,bridge}{os,simulator} iphone{osnano,nanosimulator} | grep -wFq "$PLATFORM_NAME" ) then MIGS_PRIVATE="mach_vm.defs" else diff --git a/makedefs/MakeInc.cmd b/makedefs/MakeInc.cmd index 0619a3324..a70a2d815 100644 --- a/makedefs/MakeInc.cmd +++ b/makedefs/MakeInc.cmd @@ -118,7 +118,7 @@ endif # # Platform options # -SUPPORTED_EMBEDDED_PLATFORMS := iPhoneOS iPhoneOSNano tvOS AppleTVOS WatchOS +SUPPORTED_EMBEDDED_PLATFORMS := iPhoneOS iPhoneOSNano tvOS AppleTVOS WatchOS BridgeOS SUPPORTED_SIMULATOR_PLATFORMS := iPhoneSimulator iPhoneNanoSimulator tvSimulator AppleTVSimulator WatchSimulator SUPPORTED_PLATFORMS := MacOSX $(SUPPORTED_SIMULATOR_PLATFORMS) $(SUPPORTED_EMBEDDED_PLATFORMS) @@ -127,7 +127,7 @@ ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) ifeq ($(EMBEDDED_DEVICE_MAP),) export EMBEDDED_DEVICE_MAP := $(shell $(XCRUN) -sdk $(SDKROOT) -find embedded_device_map) endif -EDM_DBPATH = $(PLATFORMPATH)/usr/local/standalone/firmware/device_map.db +EDM_DBPATH ?= $(PLATFORMPATH)/usr/local/standalone/firmware/device_map.db endif # Scripts or tools we build ourselves diff --git a/makedefs/MakeInc.def b/makedefs/MakeInc.def index 9edf2627a..4eca357c9 100644 --- a/makedefs/MakeInc.def +++ b/makedefs/MakeInc.def @@ -1,6 +1,6 @@ # -*- mode: makefile;-*- # -# Copyright (C) 1999-2016 Apple Inc. All rights reserved. +# Copyright (C) 1999-2017 Apple Inc. All rights reserved. # # MakeInc.def contains global definitions for building, # linking, and installing files. @@ -14,7 +14,7 @@ SUPPORTED_ARCH_CONFIGS := X86_64 X86_64H # # Kernel Configuration options # -SUPPORTED_KERNEL_CONFIGS = RELEASE DEVELOPMENT DEBUG PROFILE +SUPPORTED_KERNEL_CONFIGS = RELEASE DEVELOPMENT DEBUG PROFILE KASAN # # Machine Configuration options @@ -43,7 +43,7 @@ endif # # Component List # -COMPONENT_LIST = osfmk bsd libkern iokit pexpert libsa security +COMPONENT_LIST = osfmk bsd libkern iokit pexpert libsa security san COMPONENT = $(if $(word 2,$(subst /, ,$(RELATIVE_SOURCE_PATH))),$(word 2,$(subst /, ,$(RELATIVE_SOURCE_PATH))),$(firstword $(subst /, ,$(RELATIVE_SOURCE_PATH)))) COMPONENT_IMPORT_LIST = $(filter-out $(COMPONENT),$(COMPONENT_LIST)) @@ -59,6 +59,8 @@ else ifeq ($(PLATFORM),tvOS) DEPLOYMENT_TARGET_FLAGS = -mtvos-version-min=$(SDKVERSION) else ifeq ($(PLATFORM),AppleTVOS) DEPLOYMENT_TARGET_FLAGS = -mtvos-version-min=$(SDKVERSION) +else ifeq ($(PLATFORM),BridgeOS) + DEPLOYMENT_TARGET_FLAGS = -mbridgeos-version-min=$(SDKVERSION) else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) DEPLOYMENT_TARGET_FLAGS = -miphoneos-version-min=$(SDKVERSION) else ifneq ($(filter $(SUPPORTED_SIMULATOR_PLATFORMS),$(PLATFORM)),) @@ -89,11 +91,29 @@ GENASSYM_KCC = $(CC) # Compiler warning flags # -CWARNFLAGS_STD = \ - -Weverything -Werror -Wextra -Wstrict-prototypes \ - -Wmissing-prototypes -Wpointer-arith -Wreturn-type -Wcast-qual \ - -Wwrite-strings -Wswitch -Wshadow -Wcast-align -Wchar-subscripts \ - -Winline -Wnested-externs -Wredundant-decls -Wextra-tokens \ +USE_WERROR := 1 +ifneq ($(BUILD_WERROR),) +USE_WERROR := $(BUILD_WERROR) +endif + +ifeq ($(USE_WERROR),1) +WERROR := -Werror +endif + +# Shared C/C++ warning flags +WARNFLAGS_STD := \ + -Weverything \ + -Wextra \ + $(WERROR) \ + -Wpointer-arith \ + -Wreturn-type \ + -Wcast-qual \ + -Wwrite-strings \ + -Wswitch \ + -Wcast-align \ + -Wchar-subscripts \ + -Wredundant-decls \ + -Wextra-tokens \ -Wunreachable-code \ -Wno-assign-enum \ -Wno-bad-function-cast \ @@ -101,6 +121,15 @@ CWARNFLAGS_STD = \ -Wno-c++-compat \ -Wno-conditional-uninitialized \ -Wno-conversion \ + -Wnull-conversion \ + -Wstring-conversion \ + -Wliteral-conversion \ + -Wnon-literal-null-conversion \ + -Wint-conversion \ + -Wenum-conversion \ + -Wfloat-conversion \ + -Wconstant-conversion \ + -Wpointer-bool-conversion \ -Wno-covered-switch-default \ -Wno-disabled-macro-expansion \ -Wno-documentation-unknown-command \ @@ -122,6 +151,14 @@ CWARNFLAGS_STD = \ -Wno-vla \ -Wno-zero-length-array +CWARNFLAGS_STD = \ + $(WARNFLAGS_STD) \ + -Wstrict-prototypes \ + -Wmissing-prototypes \ + -Wshadow \ + -Winline \ + -Wnested-externs + # Can be overridden in Makefile.template or Makefile.$arch export CWARNFLAGS ?= $(CWARNFLAGS_STD) @@ -130,40 +167,11 @@ $(1)_CWARNFLAGS_ADD += $2 endef CXXWARNFLAGS_STD = \ - -Weverything -Werror -Wextra -Wpointer-arith -Wreturn-type \ - -Wcast-qual -Wwrite-strings -Wswitch -Wcast-align -Wchar-subscripts \ - -Wredundant-decls -Wextra-tokens \ - -Wunreachable-code \ - -Wno-assign-enum \ - -Wno-bad-function-cast \ - -Wno-c++98-compat \ + $(WARNFLAGS_STD) \ -Wno-c++98-compat-pedantic \ - -Wno-c++-compat \ - -Wno-conditional-uninitialized \ - -Wno-conversion \ - -Wno-covered-switch-default \ - -Wno-disabled-macro-expansion \ - -Wno-documentation-unknown-command \ -Wno-exit-time-destructors \ - -Wno-format-non-iso \ - -Wno-format-nonliteral \ -Wno-global-constructors \ - -Wno-reserved-id-macro \ - -Wno-language-extension-token \ - -Wno-missing-variable-declarations \ - -Wno-old-style-cast \ - -Wno-packed \ - -Wno-padded \ - -Wno-partial-availability \ - -Wno-pedantic \ - -Wno-shift-sign-overflow \ - -Wno-switch-enum \ - -Wno-undef \ - -Wno-unused-macros \ - -Wno-used-but-marked-unused \ - -Wno-variadic-macros \ - -Wno-vla \ - -Wno-zero-length-array + -Wno-old-style-cast # overloaded-virtual warnings are non-fatal (9000888) CXXWARNFLAGS_STD += -Wno-error=overloaded-virtual @@ -186,8 +194,8 @@ ARCH_FLAGS_X86_64H = -arch x86_64h # # Default CFLAGS # -ifdef RC_CFLAGS -OTHER_CFLAGS = $(subst $(addprefix -arch ,$(RC_ARCHS)),,$(RC_CFLAGS)) +ifdef RC_NONARCH_CFLAGS +OTHER_CFLAGS = $(RC_NONARCH_CLFAGS) endif # @@ -198,7 +206,7 @@ DSYMKGMACROSDIR = Contents/Resources DSYMLLDBMACROSDIR = Contents/Resources/Python DSYMDWARFDIR = Contents/Resources/DWARF -DEBUG_CFLAGS := -gdwarf-2 +DEBUG_CFLAGS := -g BUILD_DSYM := 1 # @@ -212,6 +220,7 @@ CFLAGS_GEN = $(DEBUG_CFLAGS) -nostdinc \ CFLAGS_RELEASE = CFLAGS_DEVELOPMENT = CFLAGS_DEBUG = +CFLAGS_KASAN = $(CFLAGS_DEVELOPMENT) CFLAGS_PROFILE = -pg CFLAGS_X86_64 = -Dx86_64 -DX86_64 -D__X86_64__ -DLP64 \ @@ -222,12 +231,14 @@ CFLAGS_X86_64H = $(CFLAGS_X86_64) CFLAGS_RELEASEX86_64 = -O2 CFLAGS_DEVELOPMENTX86_64 = -O2 +CFLAGS_KASANX86_64 = $(CFLAGS_DEVELOPMENTX86_64) # No space optimization for the DEBUG kernel for the benefit of gdb: CFLAGS_DEBUGX86_64 = -O0 CFLAGS_PROFILEX86_64 = -O2 CFLAGS_RELEASEX86_64H = -O2 CFLAGS_DEVELOPMENTX86_64H = -O2 +CFLAGS_KASANX86_64H = $(CFLAGS_DEVELOPMENTX86_64H) # No space optimization for the DEBUG kernel for the benefit of gdb: CFLAGS_DEBUGX86_64H = -O0 CFLAGS_PROFILEX86_64H = -O2 @@ -238,6 +249,30 @@ CFLAGS_DEBUGARM = -O0 CFLAGS_PROFILEARM = -O2 + +# +# KASAN support +# + + +ifeq ($(CURRENT_KERNEL_CONFIG),KASAN) +KASAN = 1 +endif + +ifeq ($(KASAN),1) + +BUILD_LTO = 0 +KASAN_SHIFT_X86_64=0xdffffe1000000000 +KASAN_SHIFT_X86_64H=$(KASAN_SHIFT_X86_64) +KASAN_SHIFT=$($(addsuffix $(CURRENT_ARCH_CONFIG),KASAN_SHIFT_)) +KASAN_BLACKLIST=$(OBJROOT)/san/kasan-blacklist-$(CURRENT_ARCH_CONFIG_LC) +CFLAGS_GEN += -DKASAN=1 -DKASAN_SHIFT=$(KASAN_SHIFT) -fsanitize=address \ + -mllvm -asan-globals-live-support \ + -mllvm -asan-mapping-offset=$(KASAN_SHIFT) \ + -fsanitize-blacklist=$(KASAN_BLACKLIST) + +endif + CFLAGS = $(CFLAGS_GEN) \ $($(addsuffix $(CURRENT_MACHINE_CONFIG),MACHINE_FLAGS_$(CURRENT_ARCH_CONFIG)_)) \ $($(addsuffix $(CURRENT_ARCH_CONFIG),ARCH_FLAGS_)) \ @@ -273,6 +308,7 @@ SFLAGS_GEN = -D__ASSEMBLER__ -DASSEMBLER $(OTHER_CFLAGS) SFLAGS_RELEASE = SFLAGS_DEVELOPMENT = +SFLAGS_KASAN = $(SFLAGS_DEVELOPMENT) -DKASAN=1 SFLAGS_DEBUG = SFLAGS_PROFILE = @@ -323,7 +359,8 @@ LDFLAGS_KERNEL_GEN = \ LDFLAGS_KERNEL_SDK = -L$(SDKROOT)/usr/local/lib/kernel -lfirehose_kernel LDFLAGS_KERNEL_RELEASE = -LDFLAGS_KERNEL_DEVELOPMENT = +LDFLAGS_KERNEL_DEVELOPMENT = +LDFLAGS_KERNEL_KASAN = $(LDFLAGS_KERNEL_DEVELOPMENT) LDFLAGS_KERNEL_DEBUG = LDFLAGS_KERNEL_PROFILE = @@ -362,17 +399,29 @@ LDFLAGS_KERNEL_RELEASEX86_64 = \ -Wl,-no_zero_fill_sections \ $(LDFLAGS_NOSTRIP_FLAG) +ifeq ($(KASAN),1) +LDFLAGS_KERNEL_RELEASEX86_64 += \ + -Wl,-sectalign,__HIB,__cstring,0x1000 \ + -Wl,-sectalign,__HIB,__asan_globals,0x1000 \ + -Wl,-sectalign,__HIB,__asan_liveness,0x1000 \ + -Wl,-sectalign,__HIB,__mod_term_func,0x1000 \ + -Wl,-rename_section,__HIB,__mod_init_func,__NULL,__mod_init_func \ + -Wl,-rename_section,__HIB,__eh_frame,__NULL,__eh_frame +endif + # Define KERNEL_BASE_OFFSET so known at compile time: CFLAGS_X86_64 += -DKERNEL_BASE_OFFSET=$(KERNEL_BASE_OFFSET) CFLAGS_X86_64H += -DKERNEL_BASE_OFFSET=$(KERNEL_BASE_OFFSET) LDFLAGS_KERNEL_DEBUGX86_64 = $(LDFLAGS_KERNEL_RELEASEX86_64) LDFLAGS_KERNEL_DEVELOPMENTX86_64 = $(LDFLAGS_KERNEL_RELEASEX86_64) +LDFLAGS_KERNEL_KASANX86_64 = $(LDFLAGS_KERNEL_RELEASEX86_64) LDFLAGS_KERNEL_PROFILEX86_64 = $(LDFLAGS_KERNEL_RELEASEX86_64) LDFLAGS_KERNEL_RELEASEX86_64H = $(LDFLAGS_KERNEL_RELEASEX86_64) LDFLAGS_KERNEL_DEBUGX86_64H = $(LDFLAGS_KERNEL_RELEASEX86_64H) LDFLAGS_KERNEL_DEVELOPMENTX86_64H = $(LDFLAGS_KERNEL_RELEASEX86_64H) +LDFLAGS_KERNEL_KASANX86_64H = $(LDFLAGS_KERNEL_RELEASEX86_64H) LDFLAGS_KERNEL_PROFILEX86_64H = $(LDFLAGS_KERNEL_RELEASEX86_64H) @@ -458,6 +507,7 @@ endif LTO_ENABLED_RELEASE = 1 LTO_ENABLED_DEVELOPMENT = 1 LTO_ENABLED_DEBUG = 0 +LTO_ENABLED_KASAN = 0 ifneq ($(BUILD_LTO),) USE_LTO = $(BUILD_LTO) @@ -553,6 +603,7 @@ EXPDIR = EXPORT_HDRS/$(COMPONENT) # STRIP_FLAGS_RELEASE = -S -x STRIP_FLAGS_DEVELOPMENT = -S +STRIP_FLAGS_KASAN = $(STRIP_FLAGS_DEVELOPMENT) STRIP_FLAGS_DEBUG = -S STRIP_FLAGS_PROFILE = -S -x @@ -577,7 +628,7 @@ MANDIR = /usr/share/man # # DEBUG alias location # -DEVELOPER_EXTRAS_DIR = /AppleInternal/CoreOS/xnu_debug +DEVELOPER_EXTRAS_DIR = /AppleInternal/CoreOS/xnu_$(CURRENT_KERNEL_CONFIG_LC) # # mach_kernel install location @@ -643,6 +694,14 @@ INSTALL_KERNEL_DIR := $(DEVELOPER_EXTRAS_DIR) INSTALL_KERNEL_SYM_DIR := $(DEVELOPER_EXTRAS_DIR) INSTALL_KERNEL_SYM_TO_KDK = 1 INSTALL_XNU_DEBUG_FILES = 1 +else ifeq ($(RC_ProjectName),xnu_kasan) +ifeq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) +# MacOS +INSTALL_KERNEL_DIR := $(DEVELOPER_EXTRAS_DIR) +INSTALL_KERNEL_SYM_DIR := $(DEVELOPER_EXTRAS_DIR) +endif +INSTALL_KERNEL_SYM_TO_KDK = 1 +INSTALL_KASAN_ONLY = 1 else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) INSTALL_KERNEL_SYM_TO_KDK = 1 USE_BINARY_PLIST = 1 diff --git a/makedefs/MakeInc.top b/makedefs/MakeInc.top index c19e8e59f..c552c108a 100644 --- a/makedefs/MakeInc.top +++ b/makedefs/MakeInc.top @@ -56,8 +56,12 @@ endif # Kernel Configuration options # +DEFAULT_PRODUCT_CONFIGS := + ifeq ($(RC_ProjectName),xnu_debug) override DEFAULT_KERNEL_CONFIG := DEBUG +else ifeq ($(RC_ProjectName),xnu_kasan) +override KERNEL_CONFIGS := KASAN else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) override DEFAULT_KERNEL_CONFIG := DEVELOPMENT else ifeq ($(PLATFORM),MacOSX) @@ -71,6 +75,11 @@ ifndef KERNEL_CONFIGS KERNEL_CONFIGS := DEFAULT endif +# If PRODUCT_CONFIGS is specified it should override default +ifndef PRODUCT_CONFIGS +PRODUCT_CONFIGS := $(DEFAULT_PRODUCT_CONFIGS) +endif + # # Machine Configuration options # @@ -158,7 +167,7 @@ TARGET_CONFIGS_ALIASES_UC := $(strip $(shell printf "%s" "$(TARGET_CONFIGS_ALIAS # TARGET_CONFIGS is unwieldy for use in Makefiles. Convert them to # "build configurations" which are tuples joined by "^". For # example, "RELEASE I386 DEFAULT DEVELOPMENT ARM DEFAULT" becomes -# "RELEASE^I386^NONE DEVELOPMENT^ARM^S5L8920X", which can be looped +# "RELEASE^I386^NONE DEVELOPMENT^ARM^T8002", which can be looped # over trivially. PRIMARY_BUILD_CONFIGS is the first config # for each architecture, used primarily for machine-dependent recursion. @@ -340,6 +349,9 @@ exporthdrs_md: build_exporthdrs_md_bootstrap .PHONY: installhdrs installhdrs_mi installhdrs_md ifeq ($(RC_ProjectName),xnu_debug) +installhdrs: + @: +else ifeq ($(RC_ProjectName),xnu_kasan) installhdrs: @: else @@ -469,6 +481,8 @@ final_touch_config_timestamps: config_install_bootstrap ifeq ($(RC_ProjectName),xnu_debug) install: install_kernels +else ifeq ($(RC_ProjectName),xnu_kasan) +install: install_config install_kernels else ifeq ($(RC_ProjectName),xnu_headers_Sim) install: installhdrs else diff --git a/osfmk/arm/Makefile b/osfmk/arm/Makefile new file mode 100644 index 000000000..8fd552a10 --- /dev/null +++ b/osfmk/arm/Makefile @@ -0,0 +1,49 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +ARM_HEADER_FILES = \ + arch.h \ + atomic.h \ + cpu_number.h \ + cpu_capabilities.h \ + cpuid.h \ + cpuid_internal.h \ + dbgwrap.h \ + io_map_entries.h \ + lock.h \ + locks.h \ + machine_cpu.h \ + machine_cpuid.h \ + machine_routines.h \ + pal_routines.h \ + proc_reg.h \ + smp.h \ + thread.h \ + simple_lock.h + + +INSTALL_MD_DIR = arm + +INSTALL_MD_LCL_LIST = arch.h cpu_capabilities.h + +INSTALL_MD_LIST = arch.h + +INSTALL_KF_MD_LIST = $(ARM_HEADER_FILES) + +INSTALL_KF_MD_LCL_LIST = machine_kpc.h monotonic.h $(ARM_HEADER_FILES) + +EXPORT_MD_LIST = \ + caches_internal.h \ + machine_kpc.h \ + monotonic.h \ + ${ARM_HEADER_FILES} + +EXPORT_MD_DIR = arm + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/osfmk/arm/WKdmCompress_new.s b/osfmk/arm/WKdmCompress_new.s new file mode 100644 index 000000000..7b5f5a195 --- /dev/null +++ b/osfmk/arm/WKdmCompress_new.s @@ -0,0 +1,710 @@ +/* + * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains armv7 hand optimized implementation of WKdm memory page compressor. + + int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); + + input : + src_buf : address of input page (length = 1024 words) + dest_buf : address of output buffer (may not be 16-byte aligned) + scratch : a 16-byte aligned 4k bytes scratch memory provided by the caller, + bytes_budget : a given byte target in compression + + output : + + if the input buffer can be compressed within the given byte budget, the dest_buf is written with compressed data and the function returns with number of bytes for the compressed data + o.w., the function returns -1 to signal that the input data can not be compressed with the given byte budget. + During the scan and tag process, each word that can not be compressed will be written to dest_buf, followed by a 12-bytes header + 256-bytes tag area. + When the functions returns -1, dest_buf is filled with all those words that can not be compressed and should be considered undefined. + The worst-case scenario is that all words can not be compressed. Hence, the minimum size requirement for dest_buf should be 12+256+4096 = 4364 bytes to prevent from memory fault. + + The 4th argument bytes_budget is the target compress budget in bytes. + Should the input page can be compressed within the budget, the compressed data is written to *dest_buf, and the function returns the number of compressed bytes. + Otherwise, the function returns -1 (to signal to the caller that the page can not be compressed). + + WKdm Compression algorithm is briefly stated as follows: + + There is a dynamically updated dictionary consisting of 16 words. Each dictionary word is initialized to 1 at the point of entry to the function. + For a nonzero input word x, its 8-bits (10-bits scaled up) is used to determine a corresponding word from the dictionary, represented by dict_index (4-bits) and dict_word (32-bits). + a. k = (x>>10)&255; // 8-bit hash table index + b. dict_index = hashTable[k]; // 4-bit dictionary index, hashTable[] is fixed + c. dict_word = dictionary[dict_index]; // 32-bit dictionary word, dictionary[] is dynamically updated + + Each input word x is classified/tagged into 4 classes : + 0 : x = 0 + 1 : (x>>10) == (dict_word>>10), bits 10:31 of the input word match a dictionary word + 2 : (x>>10) != (dict_word>>10), the above condition (22 higher bits matched) is not met, meaning a dictionary miss + 3 : (x == dict_word), the exact input word is in the dictionary + + For each class, different numbers of bits are needed for the decompressor to reproduce the original input word. + 0 : 2-bits tag (32->2 compression) + 1 : 2-bits tag + 4-bits dict_index + 10-bits lower bits (32->16 compression) + 2 : 2-bits tag + 32-bits new word (32->34 expansion) + 3 : 2-bits tag + 4-bits dict_index (32->6 compression) + + It is obvious now that WKdm compress algorithm works well for pages where there are lots of zero words (32->2) and/or there are freqeunt repeats of some word patterns (32->6). + + the output bit stream (*dest_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + the header is actually of 3 words that specify the ending offset (in 32-bit words) from the start of the bit stream of c,d,e, respectively. + Note that there might be padding bits in d (if the number of dict_indices does not divide by 8), and there are 2/12/22 padding bits for packing 3/2/1 low 10-bits in a 32-bit word. + + + The WKdm compress algorithm 1st runs a scan and classification pass, tagging and write unpacked data into temporary buffers. It follows by packing those data into the output buffer. + + The temp buffers are + + uint8_t tempTagsArray[1024]; // temporary saving for tags before final packing + uint8_t tempQPosArray[1024]; // temporary saving for dict_indices before final packing + uint16_t tempLowBitsArray[1024]; // temporary saving for partially matched lower 10 bits before final packing + + Since the new words (that can not matched fully or partially to the dictionary) are stored right after the header and the tags section and need no packing, we directly write them to + the destination buffer. + + uint32_t *new_word = dest_buf+3+64; // 3 words for header, 64 words for tags, new words come right after the tags. + + Now since we are given a byte budget for this compressor, we can monitor the byte usage on the fly in the scanning and tagging pass. + + byte_count = bytes_budget - 12 - 256; // header + tags + + whenever an input word is classified as class + + 2 : byte_count -= 4; + + in 4-bit/10-bit packing, we can also return -1 when byte_budget <=0; + + Note : since there might be extra padding bits for class 1 and 3, it is complicated to track this padding bits on the fly. To compromise, we change class 1 to + + without showing the bit budget management, the pseudo code is given as follows: + + uint8_t *tags=tempTagsArray; + uint8_t *dict=tempQPosArray; + uint8_t *partial=tempLowBitsArray; + + for (i=0;i<1024;i++) { + x = *src_buf++; + if (x == 0) { // zero, 2-bits tag + *tags++ = 0; + } else { + + // find dict_index and dict_word from x + k = (x>>10)&255; + dict_index = hashTable[k]; + dict_word = dictionary[dict_index]; + + if (dict_word == x) { // exactly match + // 2-bits tag + 4-bits table index + *tags++ = 3; + *dict++ = dict_index; + } else if (((x^dict_word)>>10)==0) { // 22 higher bits matched + // 2-bits tag + 4-bits table index + 10-bits lower partial + *tags++ = 1; + *dict++ = dict_index; + *partial++ = x &0x3ff; + dictionary[dict_index] = x; + } else { // not matched + // 2-bits tag + 32-bits new word + *tags++ = 2; + *new_word++ = x; + dictionary[dict_index] = x; + } + } + } + + after this classification/tagging pass is completed, the 3 temp buffers are packed into the output *dest_buf: + + 1. 1024 tags are packed into 256 bytes right after the 12-bytes header + 2. dictionary indices (4-bits each) are packed into are right after the new words section + 3. 3 low 10-bits are packed into a 32-bit word, this is after the dictionary indices section. + + cclee, 11/9/12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 +*/ + .text + .align 4 + + // int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); + +.globl _WKdm_compress_new +_WKdm_compress_new: + +/* + ------------------------- symbolizing register use ----------------------------------- +*/ + + #define src_buf r0 + #define next_input_word r0 + #define dest_buf r1 + #define scratch r2 + #define dictionary sp + #define byte_count r3 + + #define next_tag r12 + + #define remaining r4 + #define next_full_patt r5 + #define dict_location r6 + #define next_qp r8 + #define hashTable r9 + #define next_low_bits r10 + #define eax r11 + #define ecx r12 + #define edx lr + #define rdi r6 + + #define tempTagsArray scratch + #define R11 r0 // only safe to use between phase-1 and phase-2 + #define R13 r4 // only safe to use between phase-1 and phase-2 +/* + ------------------------- allocate scratch memory for local use -------------------------------------- + + need 256*4 (tempTagsArray) + 256*4 (tempQPosArray) + 1024*2 (tempLowBitsArray) + total 4096 bytes + [scratch,#0] : tempTagsArray + [scratch,#1024] : tempQPosArray + [scratch,#2048] : tempLowBitsArray + + [sp,#0] : dictionary + +*/ + + #define TagsArray_offset 0 + #define QPosArray_offset 1024 + #define LowBitsArray_offset 2048 + + #define SV_RETURN 0 // return value when SV, ZV page is found + #define MZV_MAGIC 17185 // magic value used to identify MZV page encoding + #define CHKPT_BYTES 416 // for early aborts: checkpoint after processing this many bytes. Must be in range [4..4096] + #define CHKPT_WORDS (CHKPT_BYTES/4) // checkpoint bytes in words + #define CHKPT_TAG_BYTES (CHKPT_BYTES/16) // size of the tags for CHKPT_BYTES of data + #define CHKPT_SHRUNK_BYTES 426 // for early aborts: max size of compressed stream to allow further processing .. + // .. to disable early aborts, set CHKPT_SHRUNK_BYTES to 4096 +#if CHKPT_BYTES > 4096 + #error CHKPT_BYTES must be <= 4096 +#endif +#if CHKPT_BYTES < 4 + #error CHKPT_BYTES must be >= 4 +#endif + + push {r7,lr} + mov r7, sp + push {r4-r6,r8-r11} + +#if KERNEL + sub sp, sp, #32 + vst1.64 {q0,q1}, [sp] +#endif + + sub sp, sp, #(64+24) // reserve stack space for temps + dictionary + +/* + ----- set up registers and initialize WKdm dictionary ---------- +*/ + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO + // THIS IS NEEDED TO EFFICIENTLY DETECT SINGLE VALUE PAGES + mov eax, #0 + + mov next_tag, scratch // &tempTagsArray[0] + vdup.32 q0, eax + + add next_qp, scratch, #QPosArray_offset // next_qp + mov lr, sp + mov remaining, #(CHKPT_WORDS) // remaining input words .. initially set to checkpoint + vst1.64 {q0}, [lr]! + add next_full_patt, dest_buf, #268 // dest_buf + [TAGS_AREA_OFFSET + (4096 / 16)]*4 + vst1.64 {q0}, [lr]! + vst1.64 {q0}, [lr]! + add next_low_bits, scratch, #LowBitsArray_offset // &tempLowBitsArray[0] + vst1.64 {q0}, [lr]! + +#if defined(KERNEL) && !SLIDABLE + adr hashTable, L_table + ldr hashTable, [hashTable] +#else + ldr hashTable, L_table +L_table0: + ldr hashTable, [pc, hashTable] +#endif + +#define EARLYCHECK 0 +#define NORMAL 1 + +#define mode [sp, #64] +#define start_next_full_patt [sp, #68] +#define start_next_input_word [sp, #72] +#define start_next_low_bits [sp, #76] +#define byte_budget [sp, #80] + + mov edx, #EARLYCHECK + str edx, mode // indicate we are yet to evaluate the early aborts + str next_full_patt, start_next_full_patt // remember the start of next_full_patt + str next_input_word, start_next_input_word // remember the start of next_input_word + str next_low_bits, start_next_low_bits // remember the start of next_low_bits + str byte_count, byte_budget // remember the byte budget + + sub byte_count, byte_count, #(12+256) // byte_count - header bytes - tags bytes + b L_scan_loop + + .align 4, 0x90 +L_RECORD_ZERO: + /* we've just detected a zero input word in edx */ + strb edx, [next_tag], #1 // *next_tag++ = ZERO; + subs remaining, remaining, #1 // remaining input words + ble CHECKPOINT // if remaining = 0, break + + /* WKdm compress scan/tag loop */ +L_scan_loop: + ldr edx, [next_input_word], #4 + cmp edx, #0 + beq L_RECORD_ZERO // if (input_word==0) RECORD_ZERO + + /* + now the input word edx is nonzero, we next find the corresponding dictionary word (eax) and dict_location + */ + and eax, edx, #(0xff<<10) // part of input_word for hash table index + lsr eax, eax, #10 // 8-bit index to the Hash Table + ldrb eax, [hashTable, eax] // HASH_TO_DICT_BYTE_OFFSET(input_word) + add dict_location, dictionary, eax // ((char*) dictionary) + HASH_TO_DICT_BYTE_OFFSET(input_word)); + ldr eax, [dictionary, eax] // dict_word = *dict_location; + cmp eax, edx // dict_word vs input_word + beq L_RECORD_EXACT // if identical, RECORD_EXACT + + eor eax, eax, edx + lsrs eax, eax, #10 // HIGH_BITS(dict_word) + beq L_RECORD_PARTIAL // if identical, RECORD_PARTIAL + +L_RECORD_MISS: +/* + if we are here, the input word can not be derived from the dictionary, + we write the input word as a new word, + and update the dictionary with this new word +*/ + + subs byte_count, byte_count, #4 + ble L_budgetExhausted // o.w., return -1 to signal this page is not compressable + str edx, [next_full_patt], #4 // *next_full_patt++ = input_word; + mov eax, #2 + str edx, [dict_location] // *dict_location = input_word + strb eax, [next_tag], #1 // *next_tag++ = 2 for miss + subs remaining, remaining, #1 // remaining input words + bgt L_scan_loop // if bit_count>0, go on the scan/tag pass, + b CHECKPOINT + +L_done_search: + + // SET_QPOS_AREA_START(dest_buf,next_full_patt); + sub eax, next_full_patt, dest_buf // next_full_patt - dest_buf + lsr eax, eax, #2 // offset in 4-bytes + str eax, [dest_buf] // dest_buf[0] = next_full_patt - dest_buf + + + /* -------------------------- packing 1024 tags into 256 bytes ----------------------------------------*/ + // boundary_tmp = WK_pack_2bits(tempTagsArray, (WK_word *) next_tag, dest_buf + HEADER_SIZE_IN_WORDS); + + add rdi, dest_buf, #12 // dest_buf + mov eax, scratch // &tempTagsArray[0] + sub edx, next_tag, scratch // this should be 1024 + + vld1.64 {q0,q1}, [eax,:128]! + subs edx, edx, #32 // pre-decrement by 32 +L_pack_2bits: + subs edx, edx, #32 + vshl.i64 d1, d1, #4 + vshl.i64 d3, d3, #4 + vorr d0, d0, d1 + vorr d2, d2, d3 + vshr.u64 d1, d0, #30 + vshr.u64 d3, d2, #30 + vorr d0, d0, d1 + vorr d2, d2, d3 + vzip.32 d0, d2 + vst1.64 {d0}, [rdi]! + vld1.64 {q0,q1}, [eax,:128]! + bgt L_pack_2bits + vshl.i64 d1, d1, #4 + vshl.i64 d3, d3, #4 + vorr d0, d0, d1 + vorr d2, d2, d3 + vshr.u64 d1, d0, #30 + vshr.u64 d3, d2, #30 + vorr d0, d0, d1 + vorr d2, d2, d3 + vzip.32 d0, d2 + vst1.64 {d0}, [rdi] + + + /* --------------------------------- packing 4-bits dict indices into dest_buf ---------------------------------- */ + + /* 1st, round up number of 4-bits dict_indices to a multiple of 8 and fill in 0 if needed */ + add ecx, scratch, #QPosArray_offset // tempQPosArray + sub eax, next_qp, ecx // eax = num_bytes_to_pack = next_qp - (char *) tempQPosArray; + add eax, eax, #7 // num_bytes_to_pack+7 + lsr eax, eax, #3 // num_packed_words = (num_bytes_to_pack + 7) >> 3 + subs byte_count, byte_count, eax, lsl #2 // byte_count -= 4 * packed_words + blt L_budgetExhausted // o.w., return -1 to signal this page is not compressable + add ecx, ecx, eax, lsl #3 // endQPosArray = tempQPosArray + 2*num_source_words + cmp ecx, next_qp // endQPosArray vs next_qp + bls L16 // if (next_qp >= endQPosArray) skip the following zero paddings + sub eax, ecx, next_qp + mov edx, #0 + tst eax, #4 + beq 1f + str edx, [next_qp], #4 +1: tst eax, #2 + beq 1f + strh edx, [next_qp], #2 +1: tst eax, #1 + beq 1f + strb edx, [next_qp], #1 +1: +L16: + add edx, scratch, #QPosArray_offset // tempQPosArray + mov rdi, next_full_patt // next_full_patt + cmp ecx, edx // endQPosArray vs tempQPosArray + ldr eax, [dest_buf] + bls L20 // if (endQPosArray <= tempQPosArray) skip the following + + /* packing 4-bits dict indices into dest_buf */ +L_pack_4bits: + vld1.64 {d0}, [edx,:64]! // src_next[1]:src_next[0] + vshr.u64 d1, d0, #28 // (src_next[1] << 4) + vorr d0, d0, d1 // src_next[0] | (src_next[1] << 4) + cmp ecx, edx // source_end vs src_next + vstr s0, [rdi] + add rdi, rdi, #4 + bhi L_pack_4bits // while (src_next < source_end) repeat the loop + + /* --------------------------- packing 3 10-bits low bits into a 32-bit word in dest_buf[] ----------------------------------------- */ + // SET_LOW_BITS_AREA_START(dest_buf,boundary_tmp); + sub eax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in words +L20: + str eax, [dest_buf,#4] // dest_buf[1] = boundary_tmp - dest_buf + + add ecx, scratch, #LowBitsArray_offset // tempLowBitsArray + sub edx, next_low_bits, ecx // next_low_bits - tempLowBitsArray (in bytes) + lsr edx, edx, #1 // num_tenbits_to_pack (in half-words) + subs edx, edx, #3 // pre-decrement num_tenbits_to_pack by 3 + blt 1f // if num_tenbits_to_pack < 3, skip the following loop +0: + subs byte_count, byte_count, #4 // byte_count -= 4 + ble L_budgetExhausted // o.w., return -1 to signal this page is not compressable + ldr r4,[ecx, #2] // w2:6bits:w1 + ldrh r0,[ecx], #6 // w0 + uxth r5, r4, ror #16 // w2 + uxth r4, r4 // w1 + orr r0, r0, r4, lsl #10 // w1:w0 + subs edx, edx, #3 // num_tenbits_to_pack-=3 + orr r0, r0, r5, lsl #20 // w2:w1:w0 + str r0, [rdi], #4 // pack w0,w1,w2 into 1 dest_buf word + bge 0b // if no less than 3 elements, back to loop head + +1: adds edx, edx, #3 // post-increment num_tenbits_to_pack by 3 + beq 3f // if num_tenbits_to_pack is a multiple of 3, skip the following + subs byte_count, byte_count, #4 // byte_count -= 4 + ble L_budgetExhausted // o.w., return -1 to signal this page is not compressable + ldrh eax,[ecx] // w0 + subs edx, edx, #1 // num_tenbits_to_pack-- + beq 2f // + ldrh edx, [ecx, #2] // w1 + orr eax, eax, edx, lsl #10 // w0 | (w1<<10) + +2: str eax, [rdi], #4 // write the final dest_buf word + +3: sub eax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in terms of words + str eax, [dest_buf, #8] // SET_LOW_BITS_AREA_END(dest_buf,boundary_tmp) + lsl r0, eax, #2 // boundary_tmp - dest_buf in terms of bytes + +L_done: + // restore registers and return + + add sp, sp, #(64+24) // skip memory for temps + dictionary +#if KERNEL + vld1.64 {q0,q1}, [sp]! +#endif + pop {r4-r6,r8-r11} + pop {r7,pc} + + .align 4 +L_budgetExhausted: + mov r0, #-1 + b L_done + + + .align 4,0x90 +L_RECORD_EXACT: +/* + we have an exact match of the input word to its corresponding dictionary word + write tag/dict_index to the temorary buffers +*/ + sub edx, dict_location, dictionary // dict_location - dictionary + mov eax, #3 + lsr edx, edx, #2 // divide by 4 for word offset + strb eax, [next_tag], #1 // *next_tag++ = 3 for exact + strb edx, [next_qp], #1 // *next_qp = word offset (4-bit) + subs remaining, remaining, #1 // remaining input words + bgt L_scan_loop // if remaining>0, go on the scan/tag pass, + b CHECKPOINT // if remaining = 0, break + + .align 4,0x90 +L_RECORD_PARTIAL: +/* + we have a partial (high 22-bits) match of the input word to its corresponding dictionary word + write tag/dict_index/low 10 bits to the temorary buffers +*/ + sub eax, dict_location, dictionary // dict_location - dictionary + str edx, [dict_location] // *dict_location = input_word; + lsr eax, eax, #2 // offset in 32-bit word + lsl edx, edx, #22 + strb eax, [next_qp], #1 // update *next_qp++ + mov eax, #1 + lsr edx, edx, #22 // lower 10 bits + strb eax, [next_tag], #1 // *next_tag++ = 1 for partial matched + strh edx, [next_low_bits], #2 // save next_low_bits++ + subs remaining, remaining, #1 // remaining input words + bgt L_scan_loop // if remaining>0, go on the scan/tag pass, + +CHECKPOINT: + ldr eax, mode // load the mode + cmp eax, #EARLYCHECK + beq L_check_compression_ratio // early abort check + +L_check_zero_page: + + ldr eax, start_next_full_patt // check if any dictionary misses in page + cmp eax, next_full_patt + bne L_check_single_value_page + + add eax, scratch, #QPosArray_offset // get start_next_qp + cmp eax, next_qp // check if any partial or exact dictionary matches + + moveq r0, #SV_RETURN // Magic return value + beq L_done + +L_check_single_value_page: + + ldr eax, start_next_full_patt // get # dictionary misses + sub eax, next_full_patt, eax + lsr eax, eax, #2 + + add R11, scratch, #QPosArray_offset // get start_next_qp + sub R11, next_qp, R11 // get # dictionary hits (exact + partial) + + ldr R13, start_next_low_bits + sub R13, next_low_bits, R13 // get # dictionary partial hits + lsrs R13, R13, #1 + + // Single value page if one of the follwoing is true: + // partial == 0 AND hits == 1023 AND miss == 1 AND tag[0] == 2 (i.e. miss) + // partial == 1 AND hits == 1024 AND tag[0] == 1 (i.e. partial) + // + bne 1f // were there 0 partial hits? + + mov edx, #1023 + cmp R11, edx // were there 1023 dictionary hits + bne 1f + + cmp eax, #1 // was there exacly 1 dictionary miss? + bne 1f + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #2 // was the very 1st tag a miss? + beq L_is_single_value_page + +1: + cmp R13, #1 // was there 1 partial hit? + bne L_check_mostly_zero + + mov edx, #1024 + cmp R11, edx // were there 1024 dictionary hits + bne L_check_mostly_zero + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #1 // was the very 1st tag a partial? + bne L_is_single_value_page + +L_is_single_value_page: + + moveq r0, #SV_RETURN // Magic return value + beq L_done + +L_check_mostly_zero: + // how much space will the sparse packer take? + add eax, eax, R11 // eax += (next_qp - start_next_qp) + mov edx, #6 + mov R11, #4 + mla R11, eax, edx, R11 // R11 = eax * 6 (i.e. 4 byte word + 2 byte offset) + 4 byte for header + + ldr eax, start_next_low_bits + sub eax, next_low_bits, eax // get bytes consumed by lower-10 bits + mov edx, #1365 + mul eax, eax, edx + + ldr edx, start_next_full_patt + sub edx, next_full_patt, edx // get bytes consumed by dictionary misses + add eax, edx, eax, lsr #11 // eax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + add edx, scratch, #QPosArray_offset // get start_next_qp + sub edx, next_qp, edx + add eax, eax, edx, lsr #1 // eax += (next_qp - start_next_qp)/2 + mov edx, #(12+256) + add eax, eax, edx // rax += bytes taken by the header + tags + + cmp eax, R11 // is the default packer the better option? + blt L_done_search + + ldr edx, byte_budget + cmp R11, edx // can the sparse packer fit into the given budget? + bgt L_budgetExhausted + +L_sparse_packer: + + mov edx, #MZV_MAGIC + str edx, [dest_buf], #4 // header to indicate a sparse packer + + ldr R13, start_next_input_word // get the starting address of src + mov edx, #0 + mov ecx, #4096 + +1: + ldm R13!, {r2, r3, r5, r6, r7, r8, r9, r10} + + teq r2, #0 + teqeq r3, #0 + teqeq r5, #0 + teqeq r6, #0 + teqeq r7, #0 + teqeq r8, #0 + teqeq r9, #0 + teqeq r10, #0 + + bne 2f + subs ecx, ecx, #32 + add edx, edx, #32 // 16 more bytes have been processed + bne 1b + mov r0, R11 // store the size of the compressed stream + b L_done + +2: + teq r2, #0 + strne r2, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r3, #0 + strne r3, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r5, #0 + strne r5, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r6, #0 + strne r6, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r7, #0 + strne r7, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r8, #0 + strne r8, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r9, #0 + strne r9, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r10, #0 + strne r10, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + subs ecx, ecx, #32 + bne 1b + mov r0, R11 // store the size of the compressed stream + b L_done + +L_check_compression_ratio: + + mov eax, #NORMAL + str eax, mode + mov remaining, #(1024 - CHKPT_WORDS) // remaining input words to process + cmp remaining, #0 + beq CHECKPOINT // if there are no remaining words to process + + ldr eax, start_next_low_bits + sub eax, next_low_bits, eax // get bytes consumed by lower-10 bits + mov edx, #1365 + mul eax, eax, edx + + ldr edx, start_next_full_patt + sub edx, next_full_patt, edx // get bytes consumed by dictionary misses + add eax, edx, eax, lsr #11 // eax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + add edx, scratch, #QPosArray_offset // get start_next_qp + sub edx, next_qp, edx + add eax, eax, edx, lsr #1 // eax += (next_qp - start_next_qp)/2 + mov edx, #(CHKPT_SHRUNK_BYTES - CHKPT_TAG_BYTES) + subs eax, eax, edx // eax += CHKPT_TAG_BYTES; eax -= CHKPT_SHRUNK_BYTES + bgt L_budgetExhausted // if eax is > 0, we need to early abort + b L_scan_loop // we are done + + +#if defined(KERNEL) && !SLIDABLE + .align 2 +L_table: + .long _hashLookupTable_new +#else + .align 2 +L_table: + .long L_Tab$non_lazy_ptr-(L_table0+8) + + .section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers + .align 2 +L_Tab$non_lazy_ptr: + .indirect_symbol _hashLookupTable_new + .long 0 +#endif + diff --git a/osfmk/arm/WKdmData_new.s b/osfmk/arm/WKdmData_new.s new file mode 100644 index 000000000..081059ab2 --- /dev/null +++ b/osfmk/arm/WKdmData_new.s @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + + .const + .align 4 +.globl _hashLookupTable_new +_hashLookupTable_new: + .byte 0 + .byte 52 + .byte 8 + .byte 56 + .byte 16 + .byte 12 + .byte 28 + .byte 20 + .byte 4 + .byte 36 + .byte 48 + .byte 24 + .byte 44 + .byte 40 + .byte 32 + .byte 60 + .byte 8 + .byte 12 + .byte 28 + .byte 20 + .byte 4 + .byte 60 + .byte 16 + .byte 36 + .byte 24 + .byte 48 + .byte 44 + .byte 32 + .byte 52 + .byte 56 + .byte 40 + .byte 12 + .byte 8 + .byte 48 + .byte 16 + .byte 52 + .byte 60 + .byte 28 + .byte 56 + .byte 32 + .byte 20 + .byte 24 + .byte 36 + .byte 40 + .byte 44 + .byte 4 + .byte 8 + .byte 40 + .byte 60 + .byte 32 + .byte 20 + .byte 44 + .byte 4 + .byte 36 + .byte 52 + .byte 24 + .byte 16 + .byte 56 + .byte 48 + .byte 12 + .byte 28 + .byte 16 + .byte 8 + .byte 40 + .byte 36 + .byte 28 + .byte 32 + .byte 12 + .byte 4 + .byte 44 + .byte 52 + .byte 20 + .byte 24 + .byte 48 + .byte 60 + .byte 56 + .byte 40 + .byte 48 + .byte 8 + .byte 32 + .byte 28 + .byte 36 + .byte 4 + .byte 44 + .byte 20 + .byte 56 + .byte 60 + .byte 24 + .byte 52 + .byte 16 + .byte 12 + .byte 12 + .byte 4 + .byte 48 + .byte 20 + .byte 8 + .byte 52 + .byte 16 + .byte 60 + .byte 24 + .byte 36 + .byte 44 + .byte 28 + .byte 56 + .byte 40 + .byte 32 + .byte 36 + .byte 20 + .byte 24 + .byte 60 + .byte 40 + .byte 44 + .byte 52 + .byte 16 + .byte 32 + .byte 4 + .byte 48 + .byte 8 + .byte 28 + .byte 56 + .byte 12 + .byte 28 + .byte 32 + .byte 40 + .byte 52 + .byte 36 + .byte 16 + .byte 20 + .byte 48 + .byte 8 + .byte 4 + .byte 60 + .byte 24 + .byte 56 + .byte 44 + .byte 12 + .byte 8 + .byte 36 + .byte 24 + .byte 28 + .byte 16 + .byte 60 + .byte 20 + .byte 56 + .byte 32 + .byte 40 + .byte 48 + .byte 12 + .byte 4 + .byte 44 + .byte 52 + .byte 44 + .byte 40 + .byte 12 + .byte 56 + .byte 8 + .byte 36 + .byte 24 + .byte 60 + .byte 28 + .byte 48 + .byte 4 + .byte 32 + .byte 20 + .byte 16 + .byte 52 + .byte 60 + .byte 12 + .byte 24 + .byte 36 + .byte 8 + .byte 4 + .byte 16 + .byte 56 + .byte 48 + .byte 44 + .byte 40 + .byte 52 + .byte 32 + .byte 20 + .byte 28 + .byte 32 + .byte 12 + .byte 36 + .byte 28 + .byte 24 + .byte 56 + .byte 40 + .byte 16 + .byte 52 + .byte 44 + .byte 4 + .byte 20 + .byte 60 + .byte 8 + .byte 48 + .byte 48 + .byte 52 + .byte 12 + .byte 20 + .byte 32 + .byte 44 + .byte 36 + .byte 28 + .byte 4 + .byte 40 + .byte 24 + .byte 8 + .byte 56 + .byte 60 + .byte 16 + .byte 36 + .byte 32 + .byte 8 + .byte 40 + .byte 4 + .byte 52 + .byte 24 + .byte 44 + .byte 20 + .byte 12 + .byte 28 + .byte 48 + .byte 56 + .byte 16 + .byte 60 + .byte 4 + .byte 52 + .byte 60 + .byte 48 + .byte 20 + .byte 16 + .byte 56 + .byte 44 + .byte 24 + .byte 8 + .byte 40 + .byte 12 + .byte 32 + .byte 28 + .byte 36 + .byte 24 + .byte 32 + .byte 12 + .byte 4 + .byte 20 + .byte 16 + .byte 60 + .byte 36 + .byte 28 + .byte 8 + .byte 52 + .byte 40 + .byte 48 + .byte 44 + .byte 56 + diff --git a/osfmk/arm/WKdmDecompress_new.s b/osfmk/arm/WKdmDecompress_new.s new file mode 100644 index 000000000..2dbbd4502 --- /dev/null +++ b/osfmk/arm/WKdmDecompress_new.s @@ -0,0 +1,427 @@ +/* + * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains armv7 hand optimized implementation of WKdm memory page decompressor. + + void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, __unused__ unsigned int words); + + input : + src_buf : address of input compressed data buffer + dest_buf : address of output decompressed buffer + scratch : a 16-byte aligned 4k bytes scratch memory provided by the caller + words : this argument is not used in the implementation + + output : + + the input buffer is decompressed and the dest_buf is written with decompressed data. + + Am algorithm description of the WKdm compress and bit stream format can be found in the WKdm Compress armv7 assembly code WKdmCompress.s + + The bit stream (*src_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + where the header (of 3 words) specifies the ending boundaries (in 32-bit words) from the start of the bit stream of c,d,e, respectively. + + The decompressor 1st unpacking the bit stream component b/d/e into temorary buffers. Then it sequentially decodes the decompressed word as follows + + for (i=0;i<1024;i++) { + tag = *next_tag++ + switch (tag) { + case 0 : *dest_buf++ = 0; break; + case 1 : dict_word = dictionary[*dict_index]; dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; break; + case 2 : x = *new_word++; k = (x>>10)&255; k = hashTable[k]; dictionary[k] = *dest_buf++ = x; break; + case 3 : *dest_buf++ = dictionary[*dict_index++]; break; + } + + cclee, 11/9/12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 + +*/ + + #define MZV_MAGIC 17185 // magic value used to identify MZV page encoding + + #define ZERO 0 + #define PARTIAL_MATCH 1 + #define MISS_TAG 2 + #define MATCH 3 + + .text + .syntax unified + .align 4 + + // void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes); + + .globl _WKdm_decompress_new +_WKdm_decompress_new: + + /* + -------- symbolizing registers -------- + the armv7 code was ported from x86_64 so we name some registers that are used as temp variables with x86_64 register names. + */ + + #define src_buf r0 + #define dest_buf r1 + #define scratch r2 + #define eax r3 + #define ebx r4 + #define hashTable r4 + #define ecx r5 + #define edx r6 + #define n_bytes r8 + #define next_tag r12 + #define tags_counter lr + #define dictionary sp + #define v0 q0 + #define v1 q1 + #define v2 q2 + #define v3 q3 + #define v4 q4 + #define v5 q5 + + // and scratch memory for local variables + + // [sp,#0] : dictionary + // [scratch,#0] : tempTagsArray was 64 + // [scratch,#1024] : tempQPosArray was 1088 + // [scratch,#2048] : tempLowBitsArray was 2112 + + push {r7, lr} + mov r7, sp + push {r4-r6,r8-r11} +#if KERNEL + sub ecx, sp, #96 + sub sp, sp, #96 + vst1.64 {q0,q1},[ecx]! + vst1.64 {q2,q3},[ecx]! + vst1.64 {q4,q5},[ecx]! +#endif + sub sp, sp, #64 // allocate for dictionary + + mov n_bytes, r3 // save the n_bytes passed as function args + ldr eax, [src_buf] // read the 1st word from the header + mov ecx, #MZV_MAGIC + cmp eax, ecx // is the alternate packer used (i.e. is MZV page)? + bne L_default_decompressor // default decompressor was used + + // Mostly Zero Page Handling... + // { + add src_buf, src_buf, 4 // skip the header + mov eax, dest_buf + mov ecx, #4096 // number of bytes to zero out + mov r9, #0 + mov r10, #0 + mov r11, #0 + mov r12, #0 +1: + subs ecx, ecx, #64 + stmia eax!, {r9-r12} + stmia eax!, {r9-r12} + stmia eax!, {r9-r12} + stmia eax!, {r9-r12} + bne 1b + + mov r12, #4 // current byte position in src to read from +2: + ldr eax, [src_buf], #4 // get the word + ldrh edx, [src_buf], #2 // get the index + str eax, [dest_buf, edx] // store non-0 word in the destination buffer + add r12, r12, #6 // 6 more bytes processed + cmp r12, n_bytes // finished processing all the bytes? + bne 2b + b L_done + // } + +L_default_decompressor: + + /* + ---------------------- set up registers and PRELOAD_DICTIONARY --------------------------------- + */ + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO TO MIRROR THE COMPRESSOR + vmov.i32 q0, #0 + mov r8, sp + adr ebx, _table_2bits + vst1.64 {q0}, [r8]! + add r10, src_buf, #268 // TAGS_AREA_END + vst1.64 {q0}, [r8]! + add eax, src_buf, #12 // TAGS_AREA_START + vst1.64 {q0}, [r8]! + mov ecx, scratch // tempTagsArray + vst1.64 {q0}, [r8]! + vld1.64 {q0,q1},[ebx,:128] + + + // WK_unpack_2bits(TAGS_AREA_START(src_buf), TAGS_AREA_END(src_buf), tempTagsArray); +/* + unpacking 16 2-bit tags (from a 32-bit word) into 16 bytes + for arm64, this can be done by + 1. read the input 32-bit word into GPR w + 2. duplicate GPR into 4 elements in a vector register v0 + 3. ushl.4s vd, v0, vshift where vshift = {0, -2, -4, -6} + 4. and.4s vd, vd, vmask where vmask = 0x03030303030303030303030303030303 +*/ + +L_WK_unpack_2bits: + vld1.64 {v5}, [eax]! // read 4 32-bit words for 64 2-bit tags + vdup.32 v2, d10[0] // duplicate to 4 elements + vdup.32 v3, d10[1] // duplicate to 4 elements + vdup.32 v4, d11[0] // duplicate to 4 elements + vdup.32 v5, d11[1] // duplicate to 4 elements + vshl.u32 v2, v2, v0 // v0 = {0, -2, -4, -6} + vshl.u32 v3, v3, v0 // v0 = {0, -2, -4, -6} + vshl.u32 v4, v4, v0 // v0 = {0, -2, -4, -6} + vshl.u32 v5, v5, v0 // v0 = {0, -2, -4, -6} + vand v2, v2, v1 // v1 = {3,3,...,3} + vand v3, v3, v1 // v1 = {3,3,...,3} + vand v4, v4, v1 // v1 = {3,3,...,3} + vand v5, v5, v1 // v1 = {3,3,...,3} + vst1.64 {v2,v3}, [ecx,:128]! // write 64 tags into tempTagsArray + cmp r10, eax // TAGS_AREA_END vs TAGS_AREA_START + vst1.64 {v4,v5}, [ecx,:128]! // write 64 tags into tempTagsArray + bhi L_WK_unpack_2bits // if not reach TAGS_AREA_END, repeat L_WK_unpack_2bits + + + // WK_unpack_4bits(QPOS_AREA_START(src_buf), QPOS_AREA_END(src_buf), tempQPosArray); + + ldm src_buf, {r8,r9} // WKdm header qpos start and end + adr ebx, _table_4bits + subs r12, r9, r8 // r12 = (QPOS_AREA_END - QPOS_AREA_START)/4 + add r8, src_buf, r8, lsl #2 // QPOS_AREA_START + add r9, src_buf, r9, lsl #2 // QPOS_AREA_END + bls 1f // if QPOS_AREA_END <= QPOS_AREA_START, skip L_WK_unpack_4bits + add ecx, scratch, #1024 // tempQPosArray + vld1.64 {v0,v1},[ebx,:128] + + subs r12, r12, #1 + bls 2f // do loop of 2 only if w14 >= 5 +L_WK_unpack_4bits: + vld1.64 {d4}, [r8]! // read a 32-bit word for 8 4-bit positions + subs r12, r12, #2 + vmov d5, d4 + vzip.32 d4, d5 + vshl.u32 v2, v2, v0 // v0 = {0, -4, 0, -4} + vand v2, v2, v1 // v1 = {15,15,...,15} + vst1.64 {q2}, [ecx,:128]! + bhi L_WK_unpack_4bits +2: + adds r12, r12, #1 + ble 1f + + ldr r12, [r8], #4 // read a 32-bit word for 8 4-bit positions + vdup.32 d4, r12 // duplicate to 2 elements + vshl.u32 v2, v2, v0 // v0 = {0, -4} + vand v2, v2, v1 // v1 = {15,15,...,15} + vst1.64 {d4}, [ecx,:64]! // write 16 tags into tempTagsArray + +1: + + // WK_unpack_3_tenbits(LOW_BITS_AREA_START(src_buf), LOW_BITS_AREA_END(src_buf), tempLowBitsArray); + + ldr eax, [src_buf,#8] // LOW_BITS_AREA_END offset + add r8, src_buf, eax, lsl #2 // LOW_BITS_AREA_END + cmp r8, r9 // LOW_BITS_AREA_START vs LOW_BITS_AREA_END + add ecx, scratch, #2048 // tempLowBitsArray + add edx, scratch, #4096 // last tenbits + bls 1f // if START>=END, skip L_WK_unpack_3_tenbits + + adr ebx, _table_10bits + vld1.64 {v0,v1},[ebx,:128] + + mov r11, #0x03ff +L_WK_unpack_3_tenbits: + ldr r12, [r9], #4 // read a 32-bit word for 3 low 10-bits + and lr, r11, r12 + strh lr, [ecx], #2 + cmp ecx, edx + and lr, r11, r12, lsr #10 + beq 1f + strh lr, [ecx], #2 + and lr, r11, r12, lsr #20 + strh lr, [ecx], #2 + + cmp r8, r9 // LOW_BITS_AREA_START vs LOW_BITS_AREA_END + bhi L_WK_unpack_3_tenbits // repeat loop if LOW_BITS_AREA_END > next_word + +1: + /* + set up before going to the main decompress loop + */ + + mov next_tag, scratch // tempTagsArray + add r8, scratch, #1024 // next_qpos + add r11, scratch, #2048 // tempLowBitsArray +#if defined(KERNEL) && !SLIDABLE + adr hashTable, L_table + ldr hashTable, [hashTable] +#else + ldr hashTable, L_table +L_table0: + ldr hashTable, [pc, hashTable] +#endif + mov tags_counter, #1024 // tags_counter + + b L_next + + .align 4,0x90 +L_ZERO_TAG: + /* + we can only get here if w9 = 0, meaning this is a zero tag + *dest_buf++ = 0; + */ + subs tags_counter,tags_counter,#1 // tags_counter-- + str r9, [dest_buf], #4 // *dest_buf++ = 0 + ble L_done // if next_tag >= tag_area_end, we're done + + /* WKdm decompress main loop */ +L_next: + ldrb r9, [next_tag], #1 // new tag + cmp r9, #0 + beq L_ZERO_TAG + cmp r9, #2 // partial match tag ? + beq L_MISS_TAG + bgt L_EXACT_TAG + +L_PARTIAL_TAG: + /* + this is a partial match: + dict_word = dictionary[*dict_index]; + dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; + */ + ldrb edx, [r8], #1 // qpos = *next_qpos++ + ldrh ecx, [r11], #2 // lower 10-bits from *next_low_bits++ + ldr eax, [dictionary, edx, lsl #2] // read dictionary word + subs tags_counter,tags_counter,#1 // tags_counter-- + lsr eax, eax, #10 // clear lower 10 bits + orr eax, ecx, eax, lsl #10 // pad the lower 10-bits from *next_low_bits + str eax, [dictionary,edx,lsl #2] // *dict_location = newly formed word + str eax, [dest_buf], #4 // *dest_buf++ = newly formed word + bgt L_next // repeat loop until next_tag==tag_area_end + +L_done: + + add sp, sp, #64 // deallocate for dictionary + + // release stack memory, restore registers, and return +#if KERNEL + vld1.64 {q0,q1},[sp]! + vld1.64 {q2,q3},[sp]! + vld1.64 {q4,q5},[sp]! +#endif + pop {r4-r6,r8-r11} + pop {r7,pc} + + .align 4,0x90 +L_MISS_TAG: + /* + this is a dictionary miss. + x = *new_word++; + k = (x>>10)&255; + k = hashTable[k]; + dictionary[k] = *dest_buf++ = x; + */ + subs tags_counter,tags_counter,#1 // tags_counter-- + ldr eax, [r10], #4 // w = *next_full_patt++ + lsr edx, eax, #10 // w>>10 + str eax, [dest_buf], #4 // *dest_buf++ = word + and edx, edx, #0x0ff // 8-bit hash table index + ldrb edx, [ebx, edx] // qpos + str eax, [dictionary,edx] // dictionary[qpos] = word + bgt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + .align 4,0x90 + +L_EXACT_TAG: + /* + this is an exact match; + *dest_buf++ = dictionary[*dict_index++]; + */ + + ldrb eax, [r8], #1 // qpos = *next_qpos++ + subs tags_counter,tags_counter,#1 // tags_counter-- + ldr eax, [dictionary,eax,lsl #2] // w = dictionary[qpos] + str eax, [dest_buf], #4 // *dest_buf++ = w + bgt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + + .align 4 + +_table_2bits: + .word 0 + .word -2 + .word -4 + .word -6 + .word 0x03030303 + .word 0x03030303 + .word 0x03030303 + .word 0x03030303 + +_table_4bits: + .word 0 + .word -4 + .word 0 + .word -4 + .word 0x0f0f0f0f + .word 0x0f0f0f0f + .word 0x0f0f0f0f + .word 0x0f0f0f0f + +_table_10bits: + .word 0 + .word -10 + .word -20 + .word 0 + .word 1023 + .word 1023 + .word 1023 + .word 0 + + +#if defined(KERNEL) && !SLIDABLE + .align 2 +L_table: + .long _hashLookupTable_new +#else + .align 2 +L_table: + .long L_Tab$non_lazy_ptr-(L_table0+8) + + .section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers + .align 2 +L_Tab$non_lazy_ptr: + .indirect_symbol _hashLookupTable_new + .long 0 +#endif + diff --git a/osfmk/arm/arch.h b/osfmk/arm/arch.h new file mode 100644 index 000000000..8c38de577 --- /dev/null +++ b/osfmk/arm/arch.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_ARCH_H +#define _ARM_ARCH_H + +/* Collect the __ARM_ARCH_*__ compiler flags into something easier to use. */ +#if defined (__ARM_ARCH_7A__) || defined (__ARM_ARCH_7S__) || defined (__ARM_ARCH_7F__) || defined (__ARM_ARCH_7K__) +#define _ARM_ARCH_7 +#endif + +#if defined (_ARM_ARCH_7) || defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6ZK__) +#define _ARM_ARCH_6K +#endif + +#if defined (_ARM_ARCH_7) || defined (__ARM_ARCH_6Z__) || defined (__ARM_ARCH_6ZK__) +#define _ARM_ARCH_6Z +#endif + +#if defined (__ARM_ARCH_6__) || defined (__ARM_ARCH_6J__) || \ + defined (_ARM_ARCH_6Z) || defined (_ARM_ARCH_6K) +#define _ARM_ARCH_6 +#endif + +#if defined (_ARM_ARCH_6) || defined (__ARM_ARCH_5E__) || \ + defined (__ARM_ARCH_5TE__) || defined (__ARM_ARCH_5TEJ__) +#define _ARM_ARCH_5E +#endif + +#if defined (_ARM_ARCH_5E) || defined (__ARM_ARCH_5__) || \ + defined (__ARM_ARCH_5T__) +#define _ARM_ARCH_5 +#endif + +#if defined (_ARM_ARCH_5) || defined (__ARM_ARCH_4T__) +#define _ARM_ARCH_4T +#endif + +#if defined (_ARM_ARCH_4T) || defined (__ARM_ARCH_4__) +#define _ARM_ARCH_4 +#endif + +#endif diff --git a/osfmk/arm/arm_init.c b/osfmk/arm/arm_init.c new file mode 100644 index 000000000..a9bb6d407 --- /dev/null +++ b/osfmk/arm/arm_init.c @@ -0,0 +1,531 @@ +/* + * Copyright (c) 2007-2009 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +/* ARM64_TODO unify boot.h */ +#if __arm64__ +#include +#elif __arm__ +#include +#else +#error Unsupported arch +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include + +#if CONFIG_TELEMETRY +#include +#endif +#if MONOTONIC +#include +#endif /* MONOTONIC */ + +extern void patch_low_glo(void); +extern int serial_init(void); +extern void sleep_token_buffer_init(void); + +extern vm_offset_t intstack_top; +extern vm_offset_t fiqstack_top; +#if __arm64__ +extern vm_offset_t excepstack_top; +#endif + +extern const char version[]; +extern const char version_variant[]; +extern int disableConsoleOutput; + +#if __ARM_PAN_AVAILABLE__ +SECURITY_READ_ONLY_LATE(boolean_t) arm_pan_enabled = FALSE; /* PAN support on Hurricane and newer HW */ +#endif + +int pc_trace_buf[PC_TRACE_BUF_SIZE] = {0}; +int pc_trace_cnt = PC_TRACE_BUF_SIZE; +int debug_task; + +boolean_t up_style_idle_exit = 0; + + + +#if INTERRUPT_MASKED_DEBUG +boolean_t interrupt_masked_debug = 1; +uint64_t interrupt_masked_timeout = 0xd0000; +#endif + +boot_args const_boot_args __attribute__((section("__DATA, __const"))); +boot_args *BootArgs __attribute__((section("__DATA, __const"))); + +unsigned int arm_diag; +#ifdef APPLETYPHOON +static unsigned cpus_defeatures = 0x0; +extern void cpu_defeatures_set(unsigned int); +#endif + +#if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__ +extern volatile boolean_t arm64_stall_sleep; +#endif + +extern boolean_t force_immediate_debug_halt; + +#define MIN_LOW_GLO_MASK (0x144) + +/* + * Forward definition + */ +void arm_init(boot_args * args); + +#if __arm64__ +unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */ +#endif /* __arm64__ */ + + +/* + * Routine: arm_init + * Function: + */ +void +arm_init( + boot_args *args) +{ + unsigned int maxmem; + uint32_t memsize; + uint64_t xmaxmem; + thread_t thread; + processor_t my_master_proc; + + /* If kernel integrity is supported, use a constant copy of the boot args. */ + const_boot_args = *args; + BootArgs = &const_boot_args; + + cpu_data_init(&BootCpuData); + + PE_init_platform(FALSE, args); /* Get platform expert set up */ + +#if __arm64__ + { + unsigned int tmp_16k = 0; + +#ifdef XXXX + /* + * Select the advertised kernel page size; without the boot-arg + * we default to the hardware page size for the current platform. + */ + if (PE_parse_boot_argn("-vm16k", &tmp_16k, sizeof(tmp_16k))) + PAGE_SHIFT_CONST = PAGE_MAX_SHIFT; + else + PAGE_SHIFT_CONST = ARM_PGSHIFT; +#else + /* + * Select the advertised kernel page size; with the boot-arg + * use to the hardware page size for the current platform. + */ + int radar_20804515 = 1; /* default: new mode */ + PE_parse_boot_argn("radar_20804515", &radar_20804515, sizeof(radar_20804515)); + if (radar_20804515) { + if (args->memSize > 1ULL*1024*1024*1024) { + /* + * arm64 device with > 1GB of RAM: + * kernel uses 16KB pages. + */ + PAGE_SHIFT_CONST = PAGE_MAX_SHIFT; + } else { + /* + * arm64 device with <= 1GB of RAM: + * kernel uses hardware page size + * (4KB for H6/H7, 16KB for H8+). + */ + PAGE_SHIFT_CONST = ARM_PGSHIFT; + } + /* 32-bit apps always see 16KB page size */ + page_shift_user32 = PAGE_MAX_SHIFT; + } else { + /* kernel page size: */ + if (PE_parse_boot_argn("-use_hwpagesize", &tmp_16k, sizeof(tmp_16k))) + PAGE_SHIFT_CONST = ARM_PGSHIFT; + else + PAGE_SHIFT_CONST = PAGE_MAX_SHIFT; + /* old mode: 32-bit apps see same page size as kernel */ + page_shift_user32 = PAGE_SHIFT_CONST; + } +#endif +#ifdef APPLETYPHOON + if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) { + if ((cpus_defeatures & 0xF) != 0) + cpu_defeatures_set(cpus_defeatures & 0xF); + } +#endif + } +#endif + + ml_parse_cpu_topology(); + + master_cpu = ml_get_boot_cpu_number(); + assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number()); + + BootCpuData.cpu_number = (unsigned short)master_cpu; +#if __arm__ + BootCpuData.cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable; +#endif + BootCpuData.intstack_top = (vm_offset_t) & intstack_top; + BootCpuData.istackptr = BootCpuData.intstack_top; + BootCpuData.fiqstack_top = (vm_offset_t) & fiqstack_top; + BootCpuData.fiqstackptr = BootCpuData.fiqstack_top; +#if __arm64__ + BootCpuData.excepstack_top = (vm_offset_t) & excepstack_top; + BootCpuData.excepstackptr = BootCpuData.excepstack_top; +#endif + BootCpuData.cpu_processor = cpu_processor_alloc(TRUE); + BootCpuData.cpu_console_buf = (void *)NULL; + CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData; + CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase) + + ((uintptr_t)&BootCpuData + - (uintptr_t)(args->virtBase))); + + thread_bootstrap(); + thread = current_thread(); + /* + * Preemption is enabled for this thread so that it can lock mutexes without + * tripping the preemption check. In reality scheduling is not enabled until + * this thread completes, and there are no other threads to switch to, so + * preemption level is not really meaningful for the bootstrap thread. + */ + thread->machine.preemption_count = 0; + thread->machine.CpuDatap = &BootCpuData; +#if __arm__ && __ARM_USER_PROTECT__ + { + unsigned int ttbr0_val, ttbr1_val, ttbcr_val; + __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val)); + thread->machine.uptw_ttb = ttbr0_val; + thread->machine.kptw_ttb = ttbr1_val; + thread->machine.uptw_ttc = ttbcr_val; + } +#endif + BootCpuData.cpu_processor->processor_data.kernel_timer = &thread->system_timer; + BootCpuData.cpu_processor->processor_data.thread_timer = &thread->system_timer; + + cpu_bootstrap(); + + rtclock_early_init(); + + kernel_early_bootstrap(); + + cpu_init(); + + EntropyData.index_ptr = EntropyData.buffer; + + processor_bootstrap(); + my_master_proc = master_processor; + + (void)PE_parse_boot_argn("diag", &arm_diag, sizeof (arm_diag)); + + if (PE_parse_boot_argn("maxmem", &maxmem, sizeof (maxmem))) + xmaxmem = (uint64_t) maxmem *(1024 * 1024); + else if (PE_get_default("hw.memsize", &memsize, sizeof (memsize))) + xmaxmem = (uint64_t) memsize; + else + xmaxmem = 0; + + if (PE_parse_boot_argn("up_style_idle_exit", &up_style_idle_exit, sizeof(up_style_idle_exit))) { + up_style_idle_exit = 1; + } +#if INTERRUPT_MASKED_DEBUG + int wdt_boot_arg = 0; + /* Disable if WDT is disabled or no_interrupt_mask_debug in boot-args */ + if (PE_parse_boot_argn("no_interrupt_masked_debug", &interrupt_masked_debug, + sizeof(interrupt_masked_debug)) || (PE_parse_boot_argn("wdt", &wdt_boot_arg, + sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1))) { + interrupt_masked_debug = 0; + } + + PE_parse_boot_argn("interrupt_masked_debug_timeout", &interrupt_masked_timeout, sizeof(interrupt_masked_timeout)); +#endif + + + + PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt)); + +#if __ARM_PAN_AVAILABLE__ +#if (DEVELOPMENT || DEBUG) + boolean_t pan; + if (!PE_parse_boot_argn("-pmap_smap_disable", &pan, sizeof(pan))) { + arm_pan_enabled = TRUE; + __builtin_arm_wsr("pan", 1); + set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED); + } +#else + arm_pan_enabled = TRUE; + __builtin_arm_wsr("pan", 1); + /* SCTLR_EL1.SPAN is clear on RELEASE */ +#endif +#endif /* __ARM_PAN_AVAILABLE__ */ + + arm_vm_init(xmaxmem, args); + + uint32_t debugmode; + if (PE_parse_boot_argn("debug", &debugmode, sizeof(debugmode)) && + ((debugmode & MIN_LOW_GLO_MASK) == MIN_LOW_GLO_MASK)) + patch_low_glo(); + + printf_init(); + panic_init(); +#if __arm64__ && WITH_CLASSIC_S2R + sleep_token_buffer_init(); +#endif + + PE_consistent_debug_inherit(); + + /* setup debugging output if one has been chosen */ + PE_init_kprintf(FALSE); + + kprintf("kprintf initialized\n"); + + serialmode = 0; /* Assume normal keyboard and console */ + if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) { /* Do we want a serial + * keyboard and/or + * console? */ + kprintf("Serial mode specified: %08X\n", serialmode); + int force_sync = serialmode & SERIALMODE_SYNCDRAIN; + if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) { + if (force_sync) { + serialmode |= SERIALMODE_SYNCDRAIN; + kprintf( + "WARNING: Forcing uart driver to output synchronously." + "printf()s/IOLogs will impact kernel performance.\n" + "You are advised to avoid using 'drain_uart_sync' boot-arg.\n"); + } + } + } + if (kern_feature_override(KF_SERIAL_OVRD)) { + serialmode = 0; + } + + if (serialmode & SERIALMODE_OUTPUT) { /* Start serial if requested */ + (void)switch_to_serial_console(); /* Switch into serial mode */ + disableConsoleOutput = FALSE; /* Allow printfs to happen */ + } + PE_create_console(); + + /* setup console output */ + PE_init_printf(FALSE); + +#if __arm64__ +#if DEBUG + dump_kva_space(); +#endif +#endif + + cpu_machine_idle_init(TRUE); + +#if (__ARM_ARCH__ == 7) + if (arm_diag & 0x8000) + set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC); +#endif + + PE_init_platform(TRUE, &BootCpuData); + cpu_timebase_init(TRUE); + fiq_context_init(TRUE); + + + /* + * Initialize the stack protector for all future calls + * to C code. Since kernel_bootstrap() eventually + * switches stack context without returning through this + * function, we do not risk failing the check even though + * we mutate the guard word during execution. + */ + __stack_chk_guard = (unsigned long)early_random(); + /* Zero a byte of the protector to guard + * against string vulnerabilities + */ + __stack_chk_guard &= ~(0xFFULL << 8); + machine_startup(args); +} + +/* + * Routine: arm_init_cpu + * Function: + * Re-initialize CPU when coming out of reset + */ + +void +arm_init_cpu( + cpu_data_t *cpu_data_ptr) +{ +#if __ARM_PAN_AVAILABLE__ +#if (DEVELOPMENT || DEBUG) + if (arm_pan_enabled) { + __builtin_arm_wsr("pan", 1); + set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED); + } +#else + __builtin_arm_wsr("pan", 1); + /* SCTLR_EL1.SPAN is clear on RELEASE */ +#endif +#endif + + cpu_data_ptr->cpu_flags &= ~SleepState; +#if __ARM_SMP__ && defined(ARMA7) + cpu_data_ptr->cpu_CLW_active = 1; +#endif + + machine_set_current_thread(cpu_data_ptr->cpu_active_thread); + +#if __arm64__ + /* Enable asynchronous exceptions */ + __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF); +#endif + + cpu_machine_idle_init(FALSE); + + cpu_init(); + +#if (__ARM_ARCH__ == 7) + if (arm_diag & 0x8000) + set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC); +#endif +#ifdef APPLETYPHOON + if ((cpus_defeatures & (0xF << 4*cpu_data_ptr->cpu_number)) != 0) + cpu_defeatures_set((cpus_defeatures >> 4*cpu_data_ptr->cpu_number) & 0xF); +#endif + /* Initialize the timebase before serial_init, as some serial + * drivers use mach_absolute_time() to implement rate control + */ + cpu_timebase_init(FALSE); + + if (cpu_data_ptr == &BootCpuData) { +#if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__ + /* + * Prevent CPUs from going into deep sleep until all + * CPUs are ready to do so. + */ + arm64_stall_sleep = TRUE; +#endif + serial_init(); + PE_init_platform(TRUE, NULL); + commpage_update_timebase(); + } + + fiq_context_init(TRUE); + cpu_data_ptr->rtcPop = EndOfAllTime; + timer_resync_deadlines(); + +#if DEVELOPMENT || DEBUG + PE_arm_debug_enable_trace(); +#endif + + kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_processor->cpu_id); + + if (cpu_data_ptr == &BootCpuData) { +#if CONFIG_TELEMETRY + bootprofile_wake_from_sleep(); +#endif /* CONFIG_TELEMETRY */ +#if MONOTONIC && defined(__arm64__) + mt_wake(); +#endif /* MONOTONIC && defined(__arm64__) */ + } + + slave_main(NULL); +} + +/* + * Routine: arm_init_idle_cpu + * Function: + */ +void __attribute__((noreturn)) +arm_init_idle_cpu( + cpu_data_t *cpu_data_ptr) +{ +#if __ARM_PAN_AVAILABLE__ +#if (DEVELOPMENT || DEBUG) + if (arm_pan_enabled) { + __builtin_arm_wsr("pan", 1); + set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED); + } +#else + __builtin_arm_wsr("pan", 1); + /* SCTLR_EL1.SPAN is clear on RELEASE */ +#endif +#endif +#if __ARM_SMP__ && defined(ARMA7) + cpu_data_ptr->cpu_CLW_active = 1; +#endif + + machine_set_current_thread(cpu_data_ptr->cpu_active_thread); + +#if __arm64__ + /* Enable asynchronous exceptions */ + __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF); +#endif + +#if (__ARM_ARCH__ == 7) + if (arm_diag & 0x8000) + set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC); +#endif +#ifdef APPLETYPHOON + if ((cpus_defeatures & (0xF << 4*cpu_data_ptr->cpu_number)) != 0) + cpu_defeatures_set((cpus_defeatures >> 4*cpu_data_ptr->cpu_number) & 0xF); +#endif + + fiq_context_init(FALSE); + + cpu_idle_exit(); +} diff --git a/osfmk/arm/arm_timer.c b/osfmk/arm/arm_timer.c new file mode 100644 index 000000000..a38ff056d --- /dev/null +++ b/osfmk/arm/arm_timer.c @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * @APPLE_FREE_COPYRIGHT@ + */ +/* + * File: etimer.c + * Purpose: Routines for handling the machine independent + * event timer. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +/* + * Event timer interrupt. + * + * XXX a drawback of this implementation is that events serviced earlier must not set deadlines + * that occur before the entire chain completes. + * + * XXX a better implementation would use a set of generic callouts and iterate over them + */ +void +timer_intr(__unused int inuser, __unused uint64_t iaddr) +{ + uint64_t abstime, new_idle_timeout_ticks; + rtclock_timer_t *mytimer; + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = getCpuDatap(); + mytimer = &cpu_data_ptr->rtclock_timer; /* Point to the event timer */ + abstime = mach_absolute_time(); /* Get the time now */ + + /* is it time for an idle timer event? */ + if ((cpu_data_ptr->idle_timer_deadline > 0) && (cpu_data_ptr->idle_timer_deadline <= abstime)) { + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + new_idle_timeout_ticks = 0x0ULL; + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, MACHDBG_CODE(DBG_MACH_EXCP_DECI, 3) | DBG_FUNC_START, 0, 0, 0, 0, 0); + ((idle_timer_t)cpu_data_ptr->idle_timer_notify)(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, MACHDBG_CODE(DBG_MACH_EXCP_DECI, 3) | DBG_FUNC_END, 0, 0, 0, 0, 0); + + /* if a new idle timeout was requested set the new idle timer deadline */ + if (new_idle_timeout_ticks != 0x0ULL) { + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } + + abstime = mach_absolute_time(); /* Get the time again since we ran a bit */ + } + + /* has a pending clock timer expired? */ + if (mytimer->deadline <= abstime) { /* Have we expired the + * deadline? */ + mytimer->has_expired = TRUE; /* Remember that we popped */ + mytimer->deadline = EndOfAllTime; /* Set timer request to + * the end of all time + * in case we have no + * more events */ + mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime); + mytimer->has_expired = FALSE; + abstime = mach_absolute_time(); /* Get the time again since we ran a bit */ + } + + uint64_t quantum_deadline = cpu_data_ptr->quantum_timer_deadline; + /* is it the quantum timer expiration? */ + if ((quantum_deadline <= abstime) && (quantum_deadline > 0)) { + cpu_data_ptr->quantum_timer_deadline = 0; + quantum_timer_expire(abstime); + } + + /* Force reload our next deadline */ + cpu_data_ptr->rtcPop = EndOfAllTime; + /* schedule our next deadline */ + timer_resync_deadlines(); +} + +/* + * Set the clock deadline + */ +void +timer_set_deadline(uint64_t deadline) +{ + rtclock_timer_t *mytimer; + spl_t s; + cpu_data_t *cpu_data_ptr; + + s = splclock(); /* no interruptions */ + cpu_data_ptr = getCpuDatap(); + + mytimer = &cpu_data_ptr->rtclock_timer; /* Point to the timer itself */ + mytimer->deadline = deadline; /* Set the new expiration time */ + + timer_resync_deadlines(); + + splx(s); +} + +void +quantum_timer_set_deadline(uint64_t deadline) +{ + cpu_data_t *cpu_data_ptr; + + /* We should've only come into this path with interrupts disabled */ + assert(ml_get_interrupts_enabled() == FALSE); + + cpu_data_ptr = getCpuDatap(); + cpu_data_ptr->quantum_timer_deadline = deadline; + timer_resync_deadlines(); +} + +/* + * Re-evaluate the outstanding deadlines and select the most proximate. + * + * Should be called at splclock. + */ +void +timer_resync_deadlines(void) +{ + uint64_t deadline; + rtclock_timer_t *mytimer; + spl_t s = splclock(); /* No interruptions please */ + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = getCpuDatap(); + + deadline = 0; + + /* if we have a clock timer set sooner, pop on that */ + mytimer = &cpu_data_ptr->rtclock_timer; /* Point to the timer itself */ + if ((!mytimer->has_expired) && (mytimer->deadline > 0)) + deadline = mytimer->deadline; + + /* if we have a idle timer event coming up, how about that? */ + if ((cpu_data_ptr->idle_timer_deadline > 0) + && (cpu_data_ptr->idle_timer_deadline < deadline)) + deadline = cpu_data_ptr->idle_timer_deadline; + + /* If we have the quantum timer setup, check that */ + if ((cpu_data_ptr->quantum_timer_deadline > 0) + && (cpu_data_ptr->quantum_timer_deadline < deadline)) + deadline = cpu_data_ptr->quantum_timer_deadline; + + if ((deadline == EndOfAllTime) + || ((deadline > 0) && (cpu_data_ptr->rtcPop != deadline))) { + int decr; + + decr = setPop(deadline); + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1) | DBG_FUNC_NONE, + decr, 2, 0, 0, 0); + } + splx(s); +} + + +boolean_t +timer_resort_threshold(__unused uint64_t skew) { + return FALSE; +} + +mpqueue_head_t * +timer_queue_assign( + uint64_t deadline) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + mpqueue_head_t *queue; + + if (cpu_data_ptr->cpu_running) { + queue = &cpu_data_ptr->rtclock_timer.queue; + + if (deadline < cpu_data_ptr->rtclock_timer.deadline) + timer_set_deadline(deadline); + } + else + queue = &cpu_datap(master_cpu)->rtclock_timer.queue; + + return (queue); +} + +void +timer_queue_cancel( + mpqueue_head_t *queue, + uint64_t deadline, + uint64_t new_deadline) +{ + if (queue == &getCpuDatap()->rtclock_timer.queue) { + if (deadline < new_deadline) + timer_set_deadline(new_deadline); + } +} + +mpqueue_head_t * +timer_queue_cpu(int cpu) +{ + return &cpu_datap(cpu)->rtclock_timer.queue; +} + +void +timer_call_cpu(int cpu, void (*fn)(void *), void *arg) +{ + cpu_signal(cpu_datap(cpu), SIGPxcall, (void *) fn, arg); +} + +void +timer_call_nosync_cpu(int cpu, void (*fn)(void *), void *arg) +{ + /* XXX Needs error checking and retry */ + cpu_signal(cpu_datap(cpu), SIGPxcall, (void *) fn, arg); +} + + +static timer_coalescing_priority_params_ns_t tcoal_prio_params_init = +{ + .idle_entry_timer_processing_hdeadline_threshold_ns = 5000ULL * NSEC_PER_USEC, + .interrupt_timer_coalescing_ilat_threshold_ns = 30ULL * NSEC_PER_USEC, + .timer_resort_threshold_ns = 50 * NSEC_PER_MSEC, + .timer_coalesce_rt_shift = 0, + .timer_coalesce_bg_shift = -5, + .timer_coalesce_kt_shift = 3, + .timer_coalesce_fp_shift = 3, + .timer_coalesce_ts_shift = 3, + .timer_coalesce_rt_ns_max = 0ULL, + .timer_coalesce_bg_ns_max = 100 * NSEC_PER_MSEC, + .timer_coalesce_kt_ns_max = 1 * NSEC_PER_MSEC, + .timer_coalesce_fp_ns_max = 1 * NSEC_PER_MSEC, + .timer_coalesce_ts_ns_max = 1 * NSEC_PER_MSEC, + .latency_qos_scale = {3, 2, 1, -2, -15, -15}, + .latency_qos_ns_max ={1 * NSEC_PER_MSEC, 5 * NSEC_PER_MSEC, 20 * NSEC_PER_MSEC, + 75 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC}, + .latency_tier_rate_limited = {FALSE, FALSE, FALSE, FALSE, TRUE, TRUE}, +}; +timer_coalescing_priority_params_ns_t * timer_call_get_priority_params(void) +{ + return &tcoal_prio_params_init; +} diff --git a/osfmk/arm/arm_vm_init.c b/osfmk/arm/arm_vm_init.c new file mode 100644 index 000000000..07bcfb9b2 --- /dev/null +++ b/osfmk/arm/arm_vm_init.c @@ -0,0 +1,537 @@ +/* + * Copyright (c) 2007-2008 Apple Inc. All rights reserved. + * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include + +/* + * Denotes the end of xnu. + */ +extern void *last_kernel_symbol; + +/* + * KASLR parameters + */ +vm_offset_t vm_kernel_base; +vm_offset_t vm_kernel_top; +vm_offset_t vm_kernel_stext; +vm_offset_t vm_kernel_etext; +vm_offset_t vm_kernel_slide; +vm_offset_t vm_kernel_slid_base; +vm_offset_t vm_kernel_slid_top; +vm_offset_t vm_kext_base; +vm_offset_t vm_kext_top; +vm_offset_t vm_prelink_stext; +vm_offset_t vm_prelink_etext; +vm_offset_t vm_prelink_sinfo; +vm_offset_t vm_prelink_einfo; +vm_offset_t vm_slinkedit; +vm_offset_t vm_elinkedit; +vm_offset_t vm_prelink_sdata; +vm_offset_t vm_prelink_edata; + +unsigned long gVirtBase, gPhysBase, gPhysSize; /* Used by */ + +vm_offset_t mem_size; /* Size of actual physical memory present + * minus any performance buffer and possibly + * limited by mem_limit in bytes */ +uint64_t mem_actual; /* The "One True" physical memory size + * actually, it's the highest physical + * address + 1 */ +uint64_t max_mem; /* Size of physical memory (bytes), adjusted + * by maxmem */ +uint64_t sane_size; /* Memory size to use for defaults + * calculations */ +addr64_t vm_last_addr = VM_MAX_KERNEL_ADDRESS; /* Highest kernel + * virtual address known + * to the VM system */ + +static vm_offset_t segTEXTB; +static unsigned long segSizeTEXT; +static vm_offset_t segDATAB; +static unsigned long segSizeDATA; +static vm_offset_t segLINKB; +static unsigned long segSizeLINK; +static vm_offset_t segKLDB; +static unsigned long segSizeKLD; +static vm_offset_t segLASTB; +static unsigned long segSizeLAST; +static vm_offset_t sectCONSTB; +static unsigned long sectSizeCONST; + +vm_offset_t segPRELINKTEXTB; +unsigned long segSizePRELINKTEXT; +vm_offset_t segPRELINKINFOB; +unsigned long segSizePRELINKINFO; + +static kernel_segment_command_t *segDATA; +static boolean_t doconstro = TRUE; + +vm_offset_t end_kern, etext, sdata, edata; + +/* + * Bootstrap the system enough to run with virtual memory. + * Map the kernel's code and data, and allocate the system page table. + * Page_size must already be set. + * + * Parameters: + * first_avail: first available physical page - + * after kernel page tables + * avail_start: PA of first physical page + * avail_end : PA of last physical page + */ +vm_offset_t first_avail; +vm_offset_t static_memory_end; +pmap_paddr_t avail_start, avail_end; + +#define MEM_SIZE_MAX 0x40000000 + +extern vm_offset_t ExceptionVectorsBase; /* the code we want to load there */ + +/* The translation tables have to be 16KB aligned */ +#define round_x_table(x) \ + (((pmap_paddr_t)(x) + (ARM_PGBYTES<<2) - 1) & ~((ARM_PGBYTES<<2) - 1)) + + +static void +arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va, + int pte_prot_APX, int pte_prot_XN) +{ + if (va & ARM_TT_L1_PT_OFFMASK) { /* ragged edge hanging over a ARM_TT_L1_PT_SIZE boundary */ + va &= (~ARM_TT_L1_PT_OFFMASK); + tt_entry_t *tte = &cpu_tte[ttenum(va)]; + tt_entry_t tmplate = *tte; + pmap_paddr_t pa; + pt_entry_t *ppte, ptmp; + unsigned int i; + + pa = va - gVirtBase + gPhysBase; + + if (ARM_TTE_TYPE_TABLE == (tmplate & ARM_TTE_TYPE_MASK)) { + /* pick up the existing page table. */ + ppte = (pt_entry_t *)phystokv((tmplate & ARM_TTE_TABLE_MASK)); + } else { + /* TTE must be reincarnated COARSE. */ + ppte = (pt_entry_t *)phystokv(avail_start); + avail_start += ARM_PGBYTES; + + pmap_init_pte_static_page(kernel_pmap, ppte, pa); + + for (i = 0; i < 4; ++i) + tte[i] = pa_to_tte(kvtophys((vm_offset_t)ppte) + (i * 0x400)) | ARM_TTE_TYPE_TABLE; + } + + /* Apply the desired protections to the specified page range */ + for (i = 0; i < (ARM_PGBYTES / sizeof(*ppte)); i++) { + if (start <= va && va < _end) { + + ptmp = pa | ARM_PTE_AF | ARM_PTE_SH | ARM_PTE_TYPE; + ptmp = ptmp | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); + ptmp = ptmp | ARM_PTE_AP(pte_prot_APX); + if (pte_prot_XN) + ptmp = ptmp | ARM_PTE_NX; + + ppte[i] = ptmp; + } + + va += ARM_PGBYTES; + pa += ARM_PGBYTES; + } + } +} + +static void +arm_vm_page_granular_prot(vm_offset_t start, unsigned long size, + int tte_prot_XN, int pte_prot_APX, int pte_prot_XN, int forceCoarse) +{ + vm_offset_t _end = start + size; + vm_offset_t align_start = (start + ARM_TT_L1_PT_OFFMASK) & ~ARM_TT_L1_PT_OFFMASK; + vm_offset_t align_end = _end & ~ARM_TT_L1_PT_OFFMASK; + + arm_vm_page_granular_helper(start, _end, start, pte_prot_APX, pte_prot_XN); + + while (align_start < align_end) { + if (forceCoarse) { + arm_vm_page_granular_helper(align_start, align_end, align_start + 1, + pte_prot_APX, pte_prot_XN); + } else { + tt_entry_t *tte = &cpu_tte[ttenum(align_start)]; + for (int i = 0; i < 4; ++i) { + tt_entry_t tmplate = tte[i]; + + tmplate = (tmplate & ~ARM_TTE_BLOCK_APMASK) | ARM_TTE_BLOCK_AP(pte_prot_APX); + tmplate = (tmplate & ~ARM_TTE_BLOCK_NX_MASK); + if (tte_prot_XN) + tmplate = tmplate | ARM_TTE_BLOCK_NX; + + tte[i] = tmplate; + } + } + align_start += ARM_TT_L1_PT_SIZE; + } + + arm_vm_page_granular_helper(start, _end, _end, pte_prot_APX, pte_prot_XN); +} + +static inline void +arm_vm_page_granular_RNX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 1, AP_RONA, 1, forceCoarse); +} + +static inline void +arm_vm_page_granular_ROX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 0, AP_RONA, 0, forceCoarse); +} + +static inline void +arm_vm_page_granular_RWNX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 1, AP_RWNA, 1, forceCoarse); +} + +static inline void +arm_vm_page_granular_RWX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 0, AP_RWNA, 0, forceCoarse); +} + +void +arm_vm_prot_init(boot_args * args) +{ +#if __ARM_PTE_PHYSMAP__ + boolean_t force_coarse_physmap = TRUE; +#else + boolean_t force_coarse_physmap = FALSE; +#endif + /* + * Enforce W^X protections on segments that have been identified so far. This will be + * further refined for each KEXT's TEXT and DATA segments in readPrelinkedExtensions() + */ + + /* + * Protection on kernel text is loose here to allow shenanigans early on (e.g. copying exception vectors) + * and storing an address into "error_buffer" (see arm_init.c) !?! + * These protections are tightened in arm_vm_prot_finalize() + */ + arm_vm_page_granular_RWX(gVirtBase, segSizeTEXT + (segTEXTB - gVirtBase), FALSE); + + if (doconstro) { + /* + * We map __DATA with 3 calls, so that the __const section can have its + * protections changed independently of the rest of the __DATA segment. + */ + arm_vm_page_granular_RWNX(segDATAB, sectCONSTB - segDATAB, FALSE); + arm_vm_page_granular_RNX(sectCONSTB, sectSizeCONST, FALSE); + arm_vm_page_granular_RWNX(sectCONSTB + sectSizeCONST, (segDATAB + segSizeDATA) - (sectCONSTB + sectSizeCONST), FALSE); + } else { + /* If we aren't protecting const, just map DATA as a single blob. */ + arm_vm_page_granular_RWNX(segDATAB, segSizeDATA, FALSE); + } + + arm_vm_page_granular_ROX(segKLDB, segSizeKLD, force_coarse_physmap); + arm_vm_page_granular_RWNX(segLINKB, segSizeLINK, force_coarse_physmap); + arm_vm_page_granular_RWNX(segLASTB, segSizeLAST, FALSE); // __LAST may be empty, but we cannot assume this + arm_vm_page_granular_RWNX(segPRELINKTEXTB, segSizePRELINKTEXT, TRUE); // Refined in OSKext::readPrelinkedExtensions + arm_vm_page_granular_RWNX(segPRELINKTEXTB + segSizePRELINKTEXT, + end_kern - (segPRELINKTEXTB + segSizePRELINKTEXT), force_coarse_physmap); // PreLinkInfoDictionary + arm_vm_page_granular_RWNX(end_kern, phystokv(args->topOfKernelData) - end_kern, force_coarse_physmap); // Device Tree, RAM Disk (if present), bootArgs + arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData), ARM_PGBYTES * 8, FALSE); // boot_tte, cpu_tte + + /* + * FIXME: Any page table pages that arm_vm_page_granular_* created with ROX entries in the range + * phystokv(args->topOfKernelData) to phystokv(prot_avail_start) should themselves be + * write protected in the static mapping of that range. + * [Page table pages whose page table entries grant execute (X) privileges should themselves be + * marked read-only. This aims to thwart attacks that replace the X entries with vectors to evil code + * (relying on some thread of execution to eventually arrive at what previously was a trusted routine).] + */ + arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 8, ARM_PGBYTES, FALSE); /* Excess physMem over 1MB */ + arm_vm_page_granular_RWX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 9, ARM_PGBYTES, FALSE); /* refined in finalize */ + + /* Map the remainder of xnu owned memory. */ + arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 10, + static_memory_end - (phystokv(args->topOfKernelData) + ARM_PGBYTES * 10), force_coarse_physmap); /* rest of physmem */ + + /* + * Special case write protection for the mapping of ExceptionVectorsBase (EVB) at 0xFFFF0000. + * Recall that start.s handcrafted a page table page for EVB mapping + */ + pmap_paddr_t p = (pmap_paddr_t)(args->topOfKernelData) + (ARM_PGBYTES * 9); + pt_entry_t *ppte = (pt_entry_t *)phystokv(p); + + int idx = (HIGH_EXC_VECTORS & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT; + pt_entry_t ptmp = ppte[idx]; + + ptmp = (ptmp & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA); + + ppte[idx] = ptmp; +} + +void +arm_vm_prot_finalize(boot_args * args) +{ + /* + * Naively we could have: + * arm_vm_page_granular_ROX(segTEXTB, segSizeTEXT, FALSE); + * but, at present, that would miss a 1Mb boundary at the beginning of the segment and + * so would force a (wasteful) coarse page (e.g. when gVirtBase is 0x80000000, segTEXTB is 0x80001000). + */ + arm_vm_page_granular_ROX(gVirtBase, segSizeTEXT + (segTEXTB - gVirtBase), FALSE); + + arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 9, ARM_PGBYTES, FALSE); /* commpage, EVB */ + +#ifndef __ARM_L1_PTW__ + FlushPoC_Dcache(); +#endif + flush_mmu_tlb(); +} + +void +arm_vm_init(uint64_t memory_size, boot_args * args) +{ + vm_map_address_t va, off, off_end; + tt_entry_t *tte, *tte_limit; + pmap_paddr_t boot_ttep; + tt_entry_t *boot_tte; + uint32_t mem_segments; + kernel_section_t *sectDCONST; + + /* + * Get the virtual and physical memory base from boot_args. + */ + gVirtBase = args->virtBase; + gPhysBase = args->physBase; + gPhysSize = args->memSize; + mem_size = args->memSize; + if ((memory_size != 0) && (mem_size > memory_size)) + mem_size = memory_size; + if (mem_size > MEM_SIZE_MAX ) + mem_size = MEM_SIZE_MAX; + static_memory_end = gVirtBase + mem_size; + + /* Calculate the nubmer of ~256MB segments of memory */ + mem_segments = (mem_size + 0x0FFFFFFF) >> 28; + + /* + * Copy the boot mmu tt to create system mmu tt. + * System mmu tt start after the boot mmu tt. + * Determine translation table base virtual address: - aligned at end + * of executable. + */ + boot_ttep = args->topOfKernelData; + boot_tte = (tt_entry_t *) phystokv(boot_ttep); + + cpu_ttep = boot_ttep + ARM_PGBYTES * 4; + cpu_tte = (tt_entry_t *) phystokv(cpu_ttep); + + bcopy(boot_tte, cpu_tte, ARM_PGBYTES * 4); + + /* + * Clear out any V==P mappings that may have been established in e.g. start.s + */ + tte = &cpu_tte[ttenum(gPhysBase)]; + tte_limit = &cpu_tte[ttenum(gPhysBase + gPhysSize)]; + + /* Hands off [gVirtBase, gVirtBase + gPhysSize) please. */ + if (gPhysBase < gVirtBase) { + if (gPhysBase + gPhysSize > gVirtBase) + tte_limit = &cpu_tte[ttenum(gVirtBase)]; + } else { + if (gPhysBase < gVirtBase + gPhysSize) + tte = &cpu_tte[ttenum(gVirtBase + gPhysSize)]; + } + + while (tte < tte_limit) { + *tte = ARM_TTE_TYPE_FAULT; + tte++; + } + + /* Skip 6 pages (four L1 + two L2 entries) */ + avail_start = cpu_ttep + ARM_PGBYTES * 6; + avail_end = gPhysBase + mem_size; + + /* + * Now retrieve addresses for end, edata, and etext + * from MACH-O headers for the currently running 32 bit kernel. + */ + segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT", &segSizeTEXT); + segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA", &segSizeDATA); + segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LINKEDIT", &segSizeLINK); + segKLDB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLD", &segSizeKLD); + segLASTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LAST", &segSizeLAST); + segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_TEXT", &segSizePRELINKTEXT); + segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_INFO", &segSizePRELINKINFO); + + etext = (vm_offset_t) segTEXTB + segSizeTEXT; + sdata = (vm_offset_t) segDATAB; + edata = (vm_offset_t) segDATAB + segSizeDATA; + end_kern = round_page(getlastaddr()); /* Force end to next page */ + + /* + * Special handling for the __DATA,__const *section*. + * A page of padding named lastkerneldataconst is at the end of the __DATA,__const + * so we can safely truncate the size. __DATA,__const is also aligned, but + * just in case we will round that to a page, too. + */ + segDATA = getsegbynamefromheader(&_mh_execute_header, "__DATA"); + sectDCONST = getsectbynamefromheader(&_mh_execute_header, "__DATA", "__const"); + sectCONSTB = sectDCONST->addr; + sectSizeCONST = sectDCONST->size; + +#if !SECURE_KERNEL + /* doconstro is true by default, but we allow a boot-arg to disable it */ + (void) PE_parse_boot_argn("dataconstro", &doconstro, sizeof(doconstro)); +#endif + + if (doconstro) { + extern vm_offset_t _lastkerneldataconst; + extern vm_size_t _lastkerneldataconst_padsize; + vm_offset_t sdataconst = sectCONSTB; + + /* this should already be aligned, but so that we can protect we round */ + sectCONSTB = round_page(sectCONSTB); + + /* make sure lastkerneldataconst is really last and the right size */ + if ((_lastkerneldataconst == sdataconst + sectSizeCONST - _lastkerneldataconst_padsize) && + (_lastkerneldataconst_padsize >= PAGE_SIZE)) { + sectSizeCONST = trunc_page(sectSizeCONST); + } else { + /* otherwise see if next section is aligned then protect up to it */ + kernel_section_t *next_sect = nextsect(segDATA, sectDCONST); + + if (next_sect && ((next_sect->addr & PAGE_MASK) == 0)) { + sectSizeCONST = next_sect->addr - sectCONSTB; + } else { + /* lastly just go ahead and truncate so we try to protect something */ + sectSizeCONST = trunc_page(sectSizeCONST); + } + } + + /* sanity check */ + if ((sectSizeCONST == 0) || (sectCONSTB < sdata) || (sectCONSTB + sectSizeCONST) >= edata) { + doconstro = FALSE; + } + } + + vm_set_page_size(); + +#ifndef __ARM_L1_PTW__ + FlushPoC_Dcache(); +#endif + set_mmu_ttb(cpu_ttep); + set_mmu_ttb_alternate(cpu_ttep); + flush_mmu_tlb(); +#if __arm__ && __ARM_USER_PROTECT__ + { + unsigned int ttbr0_val, ttbr1_val, ttbcr_val; + thread_t thread = current_thread(); + + __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val)); + thread->machine.uptw_ttb = ttbr0_val; + thread->machine.kptw_ttb = ttbr1_val; + thread->machine.uptw_ttc = ttbcr_val; + } +#endif + vm_prelink_stext = segPRELINKTEXTB; + vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT; + vm_prelink_sinfo = segPRELINKINFOB; + vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO; + vm_slinkedit = segLINKB; + vm_elinkedit = segLINKB + segSizeLINK; + + sane_size = mem_size - (avail_start - gPhysBase); + max_mem = mem_size; + vm_kernel_slide = gVirtBase-0x80000000; + vm_kernel_stext = segTEXTB; + vm_kernel_etext = segTEXTB + segSizeTEXT; + vm_kernel_base = gVirtBase; + vm_kernel_top = (vm_offset_t) &last_kernel_symbol; + vm_kext_base = segPRELINKTEXTB; + vm_kext_top = vm_kext_base + segSizePRELINKTEXT; + vm_kernel_slid_base = segTEXTB; + vm_kernel_slid_top = vm_kext_top; + + pmap_bootstrap((gVirtBase+MEM_SIZE_MAX+0x3FFFFF) & 0xFFC00000); + + arm_vm_prot_init(args); + + /* + * To avoid recursing while trying to init the vm_page and object * mechanisms, + * pre-initialize kernel pmap page table pages to cover this address range: + * 2MB + FrameBuffer size + 3MB for each 256MB segment + */ + off_end = (2 + (mem_segments * 3)) << 20; + off_end += (unsigned int) round_page(args->Video.v_height * args->Video.v_rowBytes); + + for (off = 0, va = (gVirtBase+MEM_SIZE_MAX+0x3FFFFF) & 0xFFC00000; off < off_end; off += ARM_TT_L1_PT_SIZE) { + pt_entry_t *ptp; + pmap_paddr_t ptp_phys; + + ptp = (pt_entry_t *) phystokv(avail_start); + ptp_phys = (pmap_paddr_t)avail_start; + avail_start += ARM_PGBYTES; + pmap_init_pte_page(kernel_pmap, ptp, va + off, 2, TRUE); + tte = &cpu_tte[ttenum(va + off)]; + *tte = pa_to_tte((ptp_phys )) | ARM_TTE_TYPE_TABLE;; + *(tte+1) = pa_to_tte((ptp_phys + 0x400)) | ARM_TTE_TYPE_TABLE;; + *(tte+2) = pa_to_tte((ptp_phys + 0x800)) | ARM_TTE_TYPE_TABLE;; + *(tte+3) = pa_to_tte((ptp_phys + 0xC00)) | ARM_TTE_TYPE_TABLE;; + } + + avail_start = (avail_start + PAGE_MASK) & ~PAGE_MASK; + + first_avail = avail_start; + patch_low_glo_static_region(args->topOfKernelData, avail_start - args->topOfKernelData); +} + diff --git a/osfmk/arm/asm.h b/osfmk/arm/asm.h new file mode 100644 index 000000000..f27a8763a --- /dev/null +++ b/osfmk/arm/asm.h @@ -0,0 +1,320 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#ifndef _ARM_ASM_H_ +#define _ARM_ASM_H_ + +#include + +#define FRAME pushl %ebp; movl %esp, %ebp +#define EMARF leave + + +/* There is another definition of ALIGN for .c sources */ +#ifdef ASSEMBLER +#define ALIGN 2 +#endif /* ASSEMBLER */ + +#ifndef FALIGN +#define FALIGN ALIGN +#endif + +#define LB(x,n) n +#if __STDC__ +#ifndef __NO_UNDERSCORES__ +#define LCL(x) L ## x +#define EXT(x) _ ## x +#define LEXT(x) _ ## x ## : +#else +#define LCL(x) .L ## x +#define EXT(x) x +#define LEXT(x) x ## : +#endif +#define LBc(x,n) n ## : +#define LBb(x,n) n ## b +#define LBf(x,n) n ## f +#else /* __STDC__ */ +#ifndef __NO_UNDERSCORES__ +#define LCL(x) L/**/x +#define EXT(x) _/**/x +#define LEXT(x) _/**/x/**/: +#else /* __NO_UNDERSCORES__ */ +#define LCL(x) .L/**/x +#define EXT(x) x +#define LEXT(x) x/**/: +#endif /* __NO_UNDERSCORES__ */ +#define LBc(x,n) n/**/: +#define LBb(x,n) n/**/b +#define LBf(x,n) n/**/f +#endif /* __STDC__ */ + +#define String .asciz +#define Value .word +#define Times(a,b) (a*b) +#define Divide(a,b) (a/b) + +#if 0 /* TOTOJK */ +#ifdef __ELF__ +#define ELF_FUNC(x) .type x,@function +#define ELF_DATA(x) .type x,@object +#define ELF_SIZE(x,s) .size x,s +#else +#define ELF_FUNC(x) +#define ELF_DATA(x) +#define ELF_SIZE(x,s) +#endif +#else +#define ELF_FUNC(x) +#define ELF_DATA(x) +#define ELF_SIZE(x,s) +#endif /* TODOJK */ + +#define Entry(x) .globl EXT(x); ELF_FUNC(EXT(x)); .align FALIGN; LEXT(x) +#define ENTRY(x) Entry(x) MCOUNT +#define ENTRY2(x,y) .globl EXT(x); .globl EXT(y); \ + ELF_FUNC(EXT(x)); ELF_FUNC(EXT(y)); \ + .align FALIGN; LEXT(x); LEXT(y) \ + MCOUNT +#if __STDC__ +#define ASENTRY(x) .globl x; .align FALIGN; x ## : ELF_FUNC(x) MCOUNT +#else +#define ASENTRY(x) .globl x; .align FALIGN; x: ELF_FUNC(x) MCOUNT +#endif /* __STDC__ */ + +#define DATA(x) .globl EXT(x); ELF_DATA(EXT(x)); .align ALIGN; LEXT(x) + +#define End(x) ELF_SIZE(x,.-x) +#define END(x) End(EXT(x)) +#define ENDDATA(x) END(x) +#define Enddata(x) End(x) + +#ifdef ASSEMBLER + +#define MCOUNT + +#else /* NOT ASSEMBLER */ + +/* These defines are here for .c files that wish to reference global symbols + * within __asm__ statements. + */ +#ifndef __NO_UNDERSCORES__ +#define CC_SYM_PREFIX "_" +#else +#define CC_SYM_PREFIX "" +#endif /* __NO_UNDERSCORES__ */ +#endif /* ASSEMBLER */ + +#ifdef ASSEMBLER + +#if defined (_ARM_ARCH_4T) +# define RET bx lr +# define RETeq bxeq lr +# define RETne bxne lr +# ifdef __STDC__ +# define RETc(c) bx##c lr +# else +# define RETc(c) bx/**/c lr +# endif +#else +# define RET mov pc, lr +# define RETeq moveq pc, lr +# define RETne movne pc, lr +# ifdef __STDC__ +# define RETc(c) mov##c pc, lr +# else +# define RETc(c) mov/**/c pc, lr +# endif +#endif + +#if defined (__thumb__) +/* Provide a PI mechanism for thumb branching. */ +# define BRANCH_EXTERN(x) ldr pc, [pc, #-4] ; \ + .long EXT(x) +#else +# define BRANCH_EXTERN(x) b EXT(x) +#endif + +/* + * arg0: Register for thread pointer + */ +.macro READ_THREAD + mrc p15, 0, $0, c13, c0, 4 /* Read TPIDRPRW */ +.endmacro + + +/* Macros for loading up addresses that are external to the .s file. + * LOAD_ADDR: loads the address for (label) into (reg). Not safe for + * loading to the PC. + * LOAD_ADDR_PC: Variant for loading to the PC; load the address of (label) + * into the pc. + * LOAD_ADDR_GEN_DEF: The general definition needed to support loading + * a label address. + * + * Usage: For any label accessed, we require one (and only one) instance + * of LOAD_ADDR_GEN_DEF(label). + * + * Example: + * LOAD_ADDR(r0, arm_init) + * LOAD_ADDR(lr, arm_init_cpu) + * LOAD_ADDR_PC(arm_init) + * ... + * + * LOAD_ADDR_GEN_DEF(arm_init) + * LOAD_ADDR_GEN_DEF(arm_init_cpu) + */ + +#if SLIDABLE +/* Definitions for a position dependent kernel using non-lazy pointers. + */ + +/* TODO: Make this work with thumb .s files. */ +#define PC_INC 0x8 + +/* We need wrapper macros in order to ensure that __LINE__ is expanded. + * + * There is some small potential for duplicate labels here, but because + * we do not export the generated labels, it should not be an issue. + */ + +#define GLUE_LABEL_GUTS(label, tag) L_##label##_##tag##_glue +#define GLUE_LABEL(label, tag) GLUE_LABEL_GUTS(label, tag) + +#define LOAD_ADDR(reg, label) \ + movw reg, :lower16:(label##$non_lazy_ptr - (GLUE_LABEL(label, __LINE__) + PC_INC)) ; \ + movt reg, :upper16:(label##$non_lazy_ptr - (GLUE_LABEL(label, __LINE__) + PC_INC)) ; \ +GLUE_LABEL(label, __LINE__): ; \ + ldr reg, [pc, reg] + +/* Designed with the understanding that directly branching to thumb code + * is unreliable; this should allow for dealing with __thumb__ in + * assembly; the non-thumb variant still needs to provide the glue label + * to avoid failing to build on undefined symbols. + * + * TODO: Make this actually use a scratch register; this macro is convenient + * for translating (ldr pc, [?]) to a slidable format without the risk of + * clobbering registers, but it is also wasteful. + */ +#if defined(__thumb__) +#define LOAD_ADDR_PC(label) \ + stmfd sp!, { r0 } ; \ + stmfd sp!, { r0 } ; \ + LOAD_ADDR(r0, label) ; \ + str r0, [sp, #4] ; \ + ldmfd sp!, { r0 } ; \ + ldmfd sp!, { pc } +#else +#define LOAD_ADDR_PC(label) \ + b EXT(label) +#endif + +#define LOAD_ADDR_GEN_DEF(label) \ + .section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers ; \ + .align 2 ; \ +label##$non_lazy_ptr: ; \ + .indirect_symbol EXT(label) ; \ + .long 0 + +#else /* !SLIDABLE */ + +/* Definitions for a position dependent kernel */ +#define LOAD_ADDR(reg, label) \ + ldr reg, L_##label + +#if defined(__thumb__) +#define LOAD_ADDR_PC(label) \ + ldr pc, L_##label +#else +#define LOAD_ADDR_PC(label) \ + b EXT(label) +#endif + +#define LOAD_ADDR_GEN_DEF(label) \ + .text ; \ + .align 2 ; \ +L_##label: ; \ + .long EXT(label) + +#endif /* SLIDABLE */ + +/* The linker can deal with branching from ARM to thumb in unconditional + * branches, but not in conditional branches. To support this in our + * assembly (which allows us to build xnu without -mno-thumb), use the + * following macros for branching conditionally to external symbols. + * These macros are used just like the corresponding conditional branch + * instructions. + */ + +#define SHIM_LABEL_GUTS(line_num) L_cond_extern_##line_num##_shim +#define SHIM_LABEL(line_num) SHIM_LABEL_GUTS(line_num) + +#define COND_EXTERN_BEQ(label) \ + bne SHIM_LABEL(__LINE__) ; \ + b EXT(label) ; \ +SHIM_LABEL(__LINE__): + +#define COND_EXTERN_BLNE(label) \ + beq SHIM_LABEL(__LINE__) ; \ + bl EXT(label) ; \ +SHIM_LABEL(__LINE__): + +#define COND_EXTERN_BLGT(label) \ + ble SHIM_LABEL(__LINE__) ; \ + bl EXT(label) ; \ +SHIM_LABEL(__LINE__): + +#endif /* ASSEMBLER */ + +#endif /* _ARM_ASM_H_ */ diff --git a/osfmk/arm/atomic.h b/osfmk/arm/atomic.h new file mode 100644 index 000000000..3da426b3d --- /dev/null +++ b/osfmk/arm/atomic.h @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _ARM_ATOMIC_H_ +#define _ARM_ATOMIC_H_ + +#include + +// Parameter for __builtin_arm_dmb +#define DMB_NSH 0x7 +#define DMB_ISHLD 0x9 +#define DMB_ISHST 0xa +#define DMB_ISH 0xb +#define DMB_SY 0xf + +// Parameter for __builtin_arm_dsb +#define DSB_NSH 0x7 +#define DSB_ISHLD 0x9 +#define DSB_ISHST 0xa +#define DSB_ISH 0xb +#define DSB_SY 0xf + +// Parameter for __builtin_arm_isb +#define ISB_SY 0xf + +#if __SMP__ + +#define memory_order_consume_smp memory_order_consume +#define memory_order_acquire_smp memory_order_acquire +#define memory_order_release_smp memory_order_release +#define memory_order_acq_rel_smp memory_order_acq_rel +#define memory_order_seq_cst_smp memory_order_seq_cst + +#else + +#define memory_order_consume_smp memory_order_relaxed +#define memory_order_acquire_smp memory_order_relaxed +#define memory_order_release_smp memory_order_relaxed +#define memory_order_acq_rel_smp memory_order_relaxed +#define memory_order_seq_cst_smp memory_order_relaxed + +#endif + +/* + * Atomic operations functions + * + * These static functions are designed for inlining + * It is expected that the memory_order arguments are + * known at compile time. This collapses these + * functions into a simple atomic operation + */ + +static inline boolean_t +memory_order_has_acquire(enum memory_order ord) +{ + switch (ord) { + case memory_order_consume: + case memory_order_acquire: + case memory_order_acq_rel: + case memory_order_seq_cst: + return TRUE; + default: + return FALSE; + } +} + +static inline boolean_t +memory_order_has_release(enum memory_order ord) +{ + switch (ord) { + case memory_order_release: + case memory_order_acq_rel: + case memory_order_seq_cst: + return TRUE; + default: + return FALSE; + } +} + +#ifdef ATOMIC_PRIVATE + +#define clear_exclusive() __builtin_arm_clrex() + +__unused static uint32_t +load_exclusive32(uint32_t *target, enum memory_order ord) +{ + uint32_t value; + +#if __arm__ + if (memory_order_has_release(ord)) { + // Pre-load release barrier + atomic_thread_fence(memory_order_release); + } + value = __builtin_arm_ldrex(target); +#else + if (memory_order_has_acquire(ord)) + value = __builtin_arm_ldaex(target); // ldaxr + else + value = __builtin_arm_ldrex(target); // ldxr +#endif // __arm__ + return value; +} + +__unused static boolean_t +store_exclusive32(uint32_t *target, uint32_t value, enum memory_order ord) +{ + boolean_t err; + +#if __arm__ + err = __builtin_arm_strex(value, target); + if (memory_order_has_acquire(ord)) { + // Post-store acquire barrier + atomic_thread_fence(memory_order_acquire); + } +#else + if (memory_order_has_release(ord)) + err = __builtin_arm_stlex(value, target); // stlxr + else + err = __builtin_arm_strex(value, target); // stxr +#endif // __arm__ + return !err; +} + +__unused static uintptr_t +load_exclusive(uintptr_t *target, enum memory_order ord) +{ +#if !__LP64__ + return load_exclusive32((uint32_t *)target, ord); +#else + uintptr_t value; + + if (memory_order_has_acquire(ord)) + value = __builtin_arm_ldaex(target); // ldaxr + else + value = __builtin_arm_ldrex(target); // ldxr + return value; +#endif // __arm__ +} + +__unused static boolean_t +store_exclusive(uintptr_t *target, uintptr_t value, enum memory_order ord) +{ +#if !__LP64__ + return store_exclusive32((uint32_t *)target, value, ord); +#else + boolean_t err; + + if (memory_order_has_release(ord)) + err = __builtin_arm_stlex(value, target); // stlxr + else + err = __builtin_arm_strex(value, target); // stxr + return !err; +#endif +} + +__unused static boolean_t +atomic_compare_exchange(uintptr_t *target, uintptr_t oldval, uintptr_t newval, + enum memory_order orig_ord, boolean_t wait) +{ + enum memory_order ord = orig_ord; + uintptr_t value; + + +#if __arm__ + ord = memory_order_relaxed; + if (memory_order_has_release(orig_ord)) { + atomic_thread_fence(memory_order_release); + } +#endif + do { + value = load_exclusive(target, ord); + if (value != oldval) { + if (wait) + wait_for_event(); // Wait with monitor held + else + clear_exclusive(); // Clear exclusive monitor + return FALSE; + } + } while (!store_exclusive(target, newval, ord)); +#if __arm__ + if (memory_order_has_acquire(orig_ord)) { + atomic_thread_fence(memory_order_acquire); + } +#endif + return TRUE; +} + +#endif // ATOMIC_PRIVATE + +#if __arm__ +#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ + boolean_t _result = FALSE; uint32_t _err = 0; \ + typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \ + for (;;) { \ + ov = __builtin_arm_ldrex(_p); \ + __VA_ARGS__; \ + if (!_err && memory_order_has_release(memory_order_##m)) { \ + /* only done for the first loop iteration */ \ + atomic_thread_fence(memory_order_release); \ + } \ + _err = __builtin_arm_strex(nv, _p); \ + if (__builtin_expect(!_err, 1)) { \ + if (memory_order_has_acquire(memory_order_##m)) { \ + atomic_thread_fence(memory_order_acquire); \ + } \ + _result = TRUE; \ + break; \ + } \ + } \ + _result; \ + }) +#else +#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ + boolean_t _result = FALSE; \ + typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \ + do { \ + if (memory_order_has_acquire(memory_order_##m)) { \ + ov = __builtin_arm_ldaex(_p); \ + } else { \ + ov = __builtin_arm_ldrex(_p); \ + } \ + __VA_ARGS__; \ + if (memory_order_has_release(memory_order_##m)) { \ + _result = !__builtin_arm_stlex(nv, _p); \ + } else { \ + _result = !__builtin_arm_strex(nv, _p); \ + } \ + } while (__builtin_expect(!_result, 0)); \ + _result; \ + }) +#endif + +#define os_atomic_rmw_loop_give_up(expr) \ + ({ __builtin_arm_clrex(); expr; __builtin_trap(); }) + +#endif // _ARM_ATOMIC_H_ diff --git a/osfmk/arm/bcopy.s b/osfmk/arm/bcopy.s new file mode 100644 index 000000000..013232499 --- /dev/null +++ b/osfmk/arm/bcopy.s @@ -0,0 +1,402 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include + +.syntax unified +.text +.align 2 + + .globl _ovbcopy + .globl _memcpy + .globl _bcopy + .globl _memmove + +_bcopy: /* void bcopy(const void *src, void *dest, size_t len); */ +_ovbcopy: + mov r3, r0 + mov r0, r1 + mov r1, r3 + +_memcpy: /* void *memcpy(void *dest, const void *src, size_t len); */ +_memmove: /* void *memmove(void *dest, const void *src, size_t len); */ + /* check for zero len or if the pointers are the same */ + cmp r2, #0 + cmpne r0, r1 + bxeq lr + + /* save r0 (return value), r4 (scratch), and r5 (scratch) */ + stmfd sp!, { r0, r4, r5, r7, lr } + add r7, sp, #12 + + /* check for overlap. r3 <- distance between src & dest */ + subhs r3, r0, r1 + sublo r3, r1, r0 + cmp r3, r2 /* if distance(src, dest) < len, we have overlap */ + blo Loverlap + +Lnormalforwardcopy: + /* are src and dest dissimilarly word aligned? */ + mov r12, r0, lsl #30 + cmp r12, r1, lsl #30 + bne Lnonwordaligned_forward + + /* if len < 64, do a quick forward copy */ + cmp r2, #64 + blt Lsmallforwardcopy + + /* check for 16 byte src/dest unalignment */ + tst r0, #0xf + bne Lsimilarlyunaligned + + /* check for 32 byte dest unalignment */ + tst r0, #(1<<4) + bne Lunaligned_32 + +Lmorethan64_aligned: + /* save some more registers to use in the copy */ + stmfd sp!, { r6, r8, r10, r11 } + + /* pre-subtract 64 from the len counter to avoid an extra compare in the loop */ + sub r2, r2, #64 + +L64loop: + /* copy 64 bytes at a time */ + ldmia r1!, { r3, r4, r5, r6, r8, r10, r11, r12 } + pld [r1, #32] + stmia r0!, { r3, r4, r5, r6, r8, r10, r11, r12 } + ldmia r1!, { r3, r4, r5, r6, r8, r10, r11, r12 } + subs r2, r2, #64 + pld [r1, #32] + stmia r0!, { r3, r4, r5, r6, r8, r10, r11, r12 } + bge L64loop + + /* restore the scratch registers we just saved */ + ldmfd sp!, { r6, r8, r10, r11 } + + /* fix up the len counter (previously subtracted an extra 64 from it) and test for completion */ + adds r2, r2, #64 + beq Lexit + +Llessthan64_aligned: + /* copy 16 bytes at a time until we have < 16 bytes */ + cmp r2, #16 + ldmiage r1!, { r3, r4, r5, r12 } + stmiage r0!, { r3, r4, r5, r12 } + subsge r2, r2, #16 + bgt Llessthan64_aligned + beq Lexit + +Llessthan16_aligned: + mov r2, r2, lsl #28 + msr cpsr_f, r2 + + ldmiami r1!, { r2, r3 } + ldreq r4, [r1], #4 + ldrhcs r5, [r1], #2 + ldrbvs r12, [r1], #1 + + stmiami r0!, { r2, r3 } + streq r4, [r0], #4 + strhcs r5, [r0], #2 + strbvs r12, [r0], #1 + b Lexit + +Lsimilarlyunaligned: + /* both src and dest are unaligned in similar ways, align to dest on 32 byte boundary */ + mov r12, r0, lsl #28 + rsb r12, r12, #0 + msr cpsr_f, r12 + + ldrbvs r3, [r1], #1 + ldrhcs r4, [r1], #2 + ldreq r5, [r1], #4 + + strbvs r3, [r0], #1 + strhcs r4, [r0], #2 + streq r5, [r0], #4 + + ldmiami r1!, { r3, r4 } + stmiami r0!, { r3, r4 } + + subs r2, r2, r12, lsr #28 + beq Lexit + +Lunaligned_32: + /* bring up to dest 32 byte alignment */ + tst r0, #(1 << 4) + ldmiane r1!, { r3, r4, r5, r12 } + stmiane r0!, { r3, r4, r5, r12 } + subne r2, r2, #16 + + /* we should now be aligned, see what copy method we should use */ + cmp r2, #64 + bge Lmorethan64_aligned + b Llessthan64_aligned + +Lbytewise2: + /* copy 2 bytes at a time */ + subs r2, r2, #2 + + ldrb r3, [r1], #1 + ldrbpl r4, [r1], #1 + + strb r3, [r0], #1 + strbpl r4, [r0], #1 + + bhi Lbytewise2 + b Lexit + +Lbytewise: + /* simple bytewise forward copy */ + ldrb r3, [r1], #1 + subs r2, r2, #1 + strb r3, [r0], #1 + bne Lbytewise + b Lexit + +Lsmallforwardcopy: + /* src and dest are word aligned similarly, less than 64 bytes to copy */ + cmp r2, #4 + blt Lbytewise2 + + /* bytewise copy until word aligned */ + tst r1, #3 +Lwordalignloop: + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + subne r2, r2, #1 + tstne r1, #3 + bne Lwordalignloop + + cmp r2, #16 + bge Llessthan64_aligned + blt Llessthan16_aligned + +Loverlap: + /* src and dest overlap in some way, len > 0 */ + cmp r0, r1 /* if dest > src */ + bhi Loverlap_srclower + +Loverlap_destlower: + /* dest < src, see if we can still do a fast forward copy or fallback to slow forward copy */ + cmp r3, #64 + bge Lnormalforwardcopy /* overlap is greater than one stride of the copy, use normal copy */ + + cmp r3, #2 + bge Lbytewise2 + b Lbytewise + + /* the following routines deal with having to copy in the reverse direction */ +Loverlap_srclower: + /* src < dest, with overlap */ + + /* src += len; dest += len; */ + add r0, r0, r2 + add r1, r1, r2 + + /* we have to copy in reverse no matter what, test if we can we use a large block reverse copy */ + cmp r2, #64 /* less than 64 bytes to copy? */ + cmpgt r3, #64 /* less than 64 bytes of nonoverlap? */ + blt Lbytewise_reverse + + /* test of src and dest are nonword aligned differently */ + mov r3, r0, lsl #30 + cmp r3, r1, lsl #30 + bne Lbytewise_reverse + + /* test if src and dest are non word aligned or dest is non 16 byte aligned */ + tst r0, #0xf + bne Lunaligned_reverse_similarly + + /* test for dest 32 byte alignment */ + tst r0, #(1<<4) + bne Lunaligned_32_reverse_similarly + + /* 64 byte reverse block copy, src and dest aligned */ +Lmorethan64_aligned_reverse: + /* save some more registers to use in the copy */ + stmfd sp!, { r6, r8, r10, r11 } + + /* pre-subtract 64 from the len counter to avoid an extra compare in the loop */ + sub r2, r2, #64 + +L64loop_reverse: + /* copy 64 bytes at a time */ + ldmdb r1!, { r3, r4, r5, r6, r8, r10, r11, r12 } +#if ARCH_ARMv5 || ARCH_ARMv5e || ARCH_ARMv6 + pld [r1, #-32] +#endif + stmdb r0!, { r3, r4, r5, r6, r8, r10, r11, r12 } + ldmdb r1!, { r3, r4, r5, r6, r8, r10, r11, r12 } + subs r2, r2, #64 + pld [r1, #-32] + stmdb r0!, { r3, r4, r5, r6, r8, r10, r11, r12 } + bge L64loop_reverse + + /* restore the scratch registers we just saved */ + ldmfd sp!, { r6, r8, r10, r11 } + + /* fix up the len counter (previously subtracted an extra 64 from it) and test for completion */ + adds r2, r2, #64 + beq Lexit + +Lbytewise_reverse: + ldrb r3, [r1, #-1]! + strb r3, [r0, #-1]! + subs r2, r2, #1 + bne Lbytewise_reverse + b Lexit + +Lunaligned_reverse_similarly: + /* both src and dest are unaligned in similar ways, align to dest on 32 byte boundary */ + mov r12, r0, lsl #28 + msr cpsr_f, r12 + + ldrbvs r3, [r1, #-1]! + ldrhcs r4, [r1, #-2]! + ldreq r5, [r1, #-4]! + + strbvs r3, [r0, #-1]! + strhcs r4, [r0, #-2]! + streq r5, [r0, #-4]! + + ldmdbmi r1!, { r3, r4 } + stmdbmi r0!, { r3, r4 } + + subs r2, r2, r12, lsr #28 + beq Lexit + +Lunaligned_32_reverse_similarly: + /* bring up to dest 32 byte alignment */ + tst r0, #(1 << 4) + ldmdbne r1!, { r3, r4, r5, r12 } + stmdbne r0!, { r3, r4, r5, r12 } + subne r2, r2, #16 + + /* we should now be aligned, see what copy method we should use */ + cmp r2, #64 + bge Lmorethan64_aligned_reverse + b Lbytewise_reverse + + /* the following routines deal with non word aligned copies */ +Lnonwordaligned_forward: + cmp r2, #8 + blt Lbytewise2 /* not worth the effort with less than 24 bytes total */ + + /* bytewise copy until src word aligned */ + tst r1, #3 +Lwordalignloop2: + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + subne r2, r2, #1 + tstne r1, #3 + bne Lwordalignloop2 + + /* figure out how the src and dest are unaligned */ + and r3, r0, #3 + cmp r3, #2 + blt Lalign1_forward + beq Lalign2_forward + bgt Lalign3_forward + +Lalign1_forward: + /* the dest pointer is 1 byte off from src */ + mov r12, r2, lsr #2 /* number of words we should copy */ + sub r0, r0, #1 + + /* prime the copy */ + ldrb r4, [r0] /* load D[7:0] */ + +Lalign1_forward_loop: + ldr r3, [r1], #4 /* load S */ + orr r4, r4, r3, lsl #8 /* D[31:8] = S[24:0] */ + str r4, [r0], #4 /* save D */ + mov r4, r3, lsr #24 /* D[7:0] = S[31:25] */ + subs r12, r12, #1 + bne Lalign1_forward_loop + + /* finish the copy off */ + strb r4, [r0], #1 /* save D[7:0] */ + + ands r2, r2, #3 + beq Lexit + b Lbytewise2 + +Lalign2_forward: + /* the dest pointer is 2 bytes off from src */ + mov r12, r2, lsr #2 /* number of words we should copy */ + sub r0, r0, #2 + + /* prime the copy */ + ldrh r4, [r0] /* load D[15:0] */ + +Lalign2_forward_loop: + ldr r3, [r1], #4 /* load S */ + orr r4, r4, r3, lsl #16 /* D[31:16] = S[15:0] */ + str r4, [r0], #4 /* save D */ + mov r4, r3, lsr #16 /* D[15:0] = S[31:15] */ + subs r12, r12, #1 + bne Lalign2_forward_loop + + /* finish the copy off */ + strh r4, [r0], #2 /* save D[15:0] */ + + ands r2, r2, #3 + beq Lexit + b Lbytewise2 + +Lalign3_forward: + /* the dest pointer is 3 bytes off from src */ + mov r12, r2, lsr #2 /* number of words we should copy */ + sub r0, r0, #3 + + /* prime the copy */ + ldr r4, [r0] + and r4, r4, #0x00ffffff /* load D[24:0] */ + +Lalign3_forward_loop: + ldr r3, [r1], #4 /* load S */ + orr r4, r4, r3, lsl #24 /* D[31:25] = S[7:0] */ + str r4, [r0], #4 /* save D */ + mov r4, r3, lsr #8 /* D[24:0] = S[31:8] */ + subs r12, r12, #1 + bne Lalign3_forward_loop + + /* finish the copy off */ + strh r4, [r0], #2 /* save D[15:0] */ + mov r4, r4, lsr #16 + strb r4, [r0], #1 /* save D[23:16] */ + + ands r2, r2, #3 + beq Lexit + b Lbytewise2 + +Lexit: + ldmfd sp!, { r0, r4, r5, r7, pc } + diff --git a/osfmk/arm/bsd_arm.c b/osfmk/arm/bsd_arm.c new file mode 100644 index 000000000..5845d01de --- /dev/null +++ b/osfmk/arm/bsd_arm.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef MACH_BSD +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +kern_return_t +thread_setsinglestep(__unused thread_t thread, __unused int on) +{ + return (KERN_FAILURE); /* XXX TODO */ +} + +#if CONFIG_DTRACE + +vm_offset_t dtrace_get_cpu_int_stack_top(void); + +vm_offset_t +dtrace_get_cpu_int_stack_top(void) +{ + return getCpuDatap()->intstack_top; +} +#endif /* CONFIG_DTRACE */ + +#endif /* MACH_BSD */ diff --git a/osfmk/arm/bzero.s b/osfmk/arm/bzero.s new file mode 100644 index 000000000..3ea2e2288 --- /dev/null +++ b/osfmk/arm/bzero.s @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include + +#include + +/* + * A reasonably well-optimized bzero/memset. Should work equally well on arm11 and arm9 based + * cores. + * + * The algorithm is to align the destination pointer on a 32 byte boundary and then + * blast data 64 bytes at a time, in two stores of 32 bytes per loop. + */ + .syntax unified + .text + .align 2 + +/* + * void *secure_memset(void * addr, int pattern, size_t length) + * + * It is important that this function remains defined in assembly to avoid + * compiler optimizations. + */ +ENTRY(secure_memset) +/* void *memset(void *ptr, int c, size_t len); */ +ENTRY(memset) + /* move len into r1, unpack c into r2 */ + mov r3, r2 + and r1, r1, #0xff + orr r1, r1, r1, lsl #8 + orr r2, r1, r1, lsl #16 + mov r1, r3 + b Lbzeroengine + +/* void bzero(void *ptr, size_t len); */ +ENTRY2(bzero,__bzero) + /* zero out r2 so we can be just like memset(0) */ + mov r2, #0 + +Lbzeroengine: + /* move the base pointer into r12 and leave r0 alone so that we return the original pointer */ + mov r12, r0 + + /* copy r2 into r3 for 64-bit stores */ + mov r3, r2 + + /* check for zero len */ + cmp r1, #0 + bxeq lr + + /* fall back to a bytewise store for less than 32 bytes */ + cmp r1, #32 + blt L_bytewise + + /* check for 32 byte unaligned ptr */ + tst r12, #0x1f + bne L_unaligned + + /* make sure we have more than 64 bytes to zero */ + cmp r1, #64 + blt L_lessthan64aligned + + /* >= 64 bytes of len, 32 byte aligned */ +L_64ormorealigned: + + /* we need some registers, avoid r7 (frame pointer) and r9 (thread register) */ + stmfd sp!, { r4-r6, r8, r10-r11 } + mov r4, r2 + mov r5, r2 + mov r6, r2 + mov r8, r2 + mov r10, r2 + mov r11, r2 + + /* pre-subtract 64 from the len to avoid an extra compare in the loop */ + sub r1, r1, #64 + +L_64loop: + stmia r12!, { r2-r6, r8, r10-r11 } + subs r1, r1, #64 + stmia r12!, { r2-r6, r8, r10-r11 } + bge L_64loop + + /* restore the saved regs */ + ldmfd sp!, { r4-r6, r8, r10-r11 } + + /* check for completion (had previously subtracted an extra 64 from len) */ + adds r1, r1, #64 + bxeq lr + +L_lessthan64aligned: + /* do we have 16 or more bytes left */ + cmp r1, #16 + stmiage r12!, { r2-r3 } + stmiage r12!, { r2-r3 } + subsge r1, r1, #16 + bgt L_lessthan64aligned + bxeq lr + +L_lessthan16aligned: + /* store 0 to 15 bytes */ + mov r1, r1, lsl #28 /* move the remaining len bits [3:0] to the flags area of cpsr */ + msr cpsr_f, r1 + + stmiami r12!, { r2-r3 } /* n is set, store 8 bytes */ + streq r2, [r12], #4 /* z is set, store 4 bytes */ + strhcs r2, [r12], #2 /* c is set, store 2 bytes */ + strbvs r2, [r12], #1 /* v is set, store 1 byte */ + bx lr + +L_bytewise: + /* bytewise copy, 2 bytes at a time, alignment not guaranteed */ + subs r1, r1, #2 + strb r2, [r12], #1 + strbpl r2, [r12], #1 + bhi L_bytewise + bx lr + +L_unaligned: + /* unaligned on 32 byte boundary, store 1-15 bytes until we're 16 byte aligned */ + mov r3, r12, lsl #28 + rsb r3, r3, #0x00000000 + msr cpsr_f, r3 + + strbvs r2, [r12], #1 /* v is set, unaligned in the 1s column */ + strhcs r2, [r12], #2 /* c is set, unaligned in the 2s column */ + streq r2, [r12], #4 /* z is set, unaligned in the 4s column */ + strmi r2, [r12], #4 /* n is set, unaligned in the 8s column */ + strmi r2, [r12], #4 + + subs r1, r1, r3, lsr #28 + bxeq lr + + /* we had previously trashed r3, restore it */ + mov r3, r2 + + /* now make sure we're 32 byte aligned */ + tst r12, #(1 << 4) + stmiane r12!, { r2-r3 } + stmiane r12!, { r2-r3 } + subsne r1, r1, #16 + + /* we're now aligned, check for >= 64 bytes left */ + cmp r1, #64 + bge L_64ormorealigned + b L_lessthan64aligned + diff --git a/osfmk/arm/caches.c b/osfmk/arm/caches.c new file mode 100644 index 000000000..5f37e202d --- /dev/null +++ b/osfmk/arm/caches.c @@ -0,0 +1,753 @@ +/* + * Copyright (c) 2010 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +/* + * dcache_incoherent_io_flush64() dcache_incoherent_io_store64() result info + */ +#define LWOpDone 1 +#define BWOpDone 3 + +#ifndef __ARM_COHERENT_IO__ + +extern boolean_t up_style_idle_exit; + +void +flush_dcache( + vm_offset_t addr, + unsigned length, + boolean_t phys) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (phys) { + unsigned int paddr; + unsigned int vaddr; + + paddr = CAST_DOWN(unsigned int, addr); + if (!isphysmem(paddr)) + return; + vaddr = (unsigned int)phystokv(paddr); + FlushPoC_DcacheRegion( (vm_offset_t) vaddr, length); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheCleanFlushRegion, (unsigned int) paddr, length); + return; + } + if (cpu_data_ptr->cpu_cache_dispatch == (cache_dispatch_t) NULL) { + FlushPoC_DcacheRegion( (vm_offset_t) addr, length); + } else { + addr64_t paddr; + uint32_t count; + + while (length > 0) { + count = PAGE_SIZE - (addr & PAGE_MASK); + if (count > length) + count = length; + FlushPoC_DcacheRegion( (vm_offset_t) addr, count); + paddr = kvtophys(addr); + if (paddr) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheCleanFlushRegion, (unsigned int) paddr, count); + addr += count; + length -= count; + } + } + return; +} + +void +clean_dcache( + vm_offset_t addr, + unsigned length, + boolean_t phys) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (phys) { + unsigned int paddr; + unsigned int vaddr; + + paddr = CAST_DOWN(unsigned int, addr); + if (!isphysmem(paddr)) + return; + + vaddr = (unsigned int)phystokv(paddr); + CleanPoC_DcacheRegion( (vm_offset_t) vaddr, length); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheCleanRegion, paddr, length); + return; + } + + if (cpu_data_ptr->cpu_cache_dispatch == (cache_dispatch_t) NULL) { + CleanPoC_DcacheRegion( (vm_offset_t) addr, length); + } else { + addr64_t paddr; + uint32_t count; + + while (length > 0) { + count = PAGE_SIZE - (addr & PAGE_MASK); + if (count > length) + count = length; + CleanPoC_DcacheRegion( (vm_offset_t) addr, count); + paddr = kvtophys(addr); + if (paddr) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheCleanRegion, (unsigned int) paddr, count); + addr += count; + length -= count; + } + } + return; +} + +void +flush_dcache_syscall( + vm_offset_t va, + unsigned length) +{ + if ((cache_info()->c_bulksize_op !=0) && (length >= (cache_info()->c_bulksize_op))) { +#if __ARM_SMP__ && defined(ARMA7) + cache_xcall(LWFlush); +#else + FlushPoC_Dcache(); + if (getCpuDatap()->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) getCpuDatap()->cpu_cache_dispatch) ( getCpuDatap()->cpu_id, CacheCleanFlush, 0x0UL , 0x0UL); +#endif + } else { + FlushPoC_DcacheRegion( (vm_offset_t) va, length); + } + return; +} + +void +dcache_incoherent_io_flush64( + addr64_t pa, + unsigned int size, + unsigned int remaining, + unsigned int *res) +{ + unsigned int vaddr; + unsigned int paddr = CAST_DOWN(unsigned int, pa); + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if ((cache_info()->c_bulksize_op !=0) && (remaining >= (cache_info()->c_bulksize_op))) { +#if __ARM_SMP__ && defined (ARMA7) + cache_xcall(LWFlush); +#else + FlushPoC_Dcache(); + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( cpu_data_ptr->cpu_id, CacheCleanFlush, 0x0UL , 0x0UL); +#endif + *res = BWOpDone; + } else { + if (isphysmem(paddr)) { + vaddr = (unsigned int)phystokv(pa); + { + FlushPoC_DcacheRegion( (vm_offset_t) vaddr, size); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) (cpu_data_ptr->cpu_id, CacheCleanFlushRegion, (unsigned int) pa, size); + } + } else { + /* slow path - pa isn't in the vtop region. Flush one page at a time via cpu_copywindows */ + unsigned int wimg_bits, index; + uint32_t count; + + mp_disable_preemption(); + + while (size > 0) { + count = PAGE_SIZE - (paddr & PAGE_MASK); + if (count > size) + count = size; + + wimg_bits = pmap_cache_attributes((paddr >> PAGE_SHIFT)); + index = pmap_map_cpu_windows_copy((paddr >> PAGE_SHIFT), VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | (paddr & PAGE_MASK); + + CleanPoC_DcacheRegion( (vm_offset_t) vaddr, count); + + pmap_unmap_cpu_windows_copy(index); + + paddr += count; + size -= count; + } + + mp_enable_preemption(); + } + } + + return; +} + +void +dcache_incoherent_io_store64( + addr64_t pa, + unsigned int size, + unsigned int remaining, + unsigned int *res) +{ + unsigned int vaddr; + unsigned int paddr = CAST_DOWN(unsigned int, pa); + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (isphysmem(paddr)) { + unsigned int wimg_bits = pmap_cache_attributes(paddr >> PAGE_SHIFT); + if ((wimg_bits == VM_WIMG_IO) || (wimg_bits == VM_WIMG_WCOMB)) { + return; + } + } + + if ((cache_info()->c_bulksize_op !=0) && (remaining >= (cache_info()->c_bulksize_op))) { +#if __ARM_SMP__ && defined (ARMA7) + cache_xcall(LWClean); + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( cpu_data_ptr->cpu_id, CacheClean, 0x0UL , 0x0UL); +#else + CleanPoC_Dcache(); + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( cpu_data_ptr->cpu_id, CacheClean, 0x0UL , 0x0UL); +#endif + *res = BWOpDone; + } else { + if (isphysmem(paddr)) { + vaddr = (unsigned int)phystokv(pa); + { + CleanPoC_DcacheRegion( (vm_offset_t) vaddr, size); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) (cpu_data_ptr->cpu_id, CacheCleanRegion, (unsigned int) pa, size); + } + } else { + /* slow path - pa isn't in the vtop region. Flush one page at a time via cpu_copywindows */ + unsigned int wimg_bits, index; + uint32_t count; + + mp_disable_preemption(); + + while (size > 0) { + count = PAGE_SIZE - (paddr & PAGE_MASK); + if (count > size) + count = size; + + wimg_bits = pmap_cache_attributes((paddr >> PAGE_SHIFT)); + index = pmap_map_cpu_windows_copy((paddr >> PAGE_SHIFT), VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | (paddr & PAGE_MASK); + + CleanPoC_DcacheRegion( (vm_offset_t) vaddr, count); + + pmap_unmap_cpu_windows_copy(index); + + paddr += count; + size -= count; + } + + mp_enable_preemption(); + } + } + + return; +} + +void +cache_sync_page( + ppnum_t pp +) +{ + pmap_paddr_t paddr = ptoa(pp); + + if (isphysmem(paddr)) { + vm_offset_t vaddr = phystokv(paddr); + + CleanPoU_DcacheRegion(vaddr, PAGE_SIZE); +#ifdef __ARM_IC_NOALIAS_ICACHE__ + InvalidatePoU_IcacheRegion(vaddr, PAGE_SIZE); +#else + InvalidatePoU_Icache(); +#endif + } else { + FlushPoC_Dcache(); + InvalidatePoU_Icache(); + }; +} + +void +platform_cache_init( + void) +{ + cache_info_t *cpuid_cache_info; + unsigned int cache_size = 0x0UL; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + cpuid_cache_info = cache_info(); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) { + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheControl, CacheControlEnable, 0x0UL); + + if ( cpuid_cache_info->c_l2size == 0x0 ) { + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheConfig, CacheConfigSize , (unsigned int)&cache_size); + cpuid_cache_info->c_l2size = cache_size; + } + } + +} + +void +platform_cache_flush( + void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + FlushPoC_Dcache(); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheCleanFlush, 0x0UL , 0x0UL); +} + +void +platform_cache_clean( + void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + CleanPoC_Dcache(); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheClean, 0x0UL , 0x0UL); +} + +void +platform_cache_shutdown( + void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + CleanPoC_Dcache(); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheShutdown, 0x0UL , 0x0UL); +} + +void +platform_cache_disable(void) +{ + uint32_t sctlr_value = 0; + + /* Disable dcache allocation. */ + __asm__ volatile("mrc p15, 0, %0, c1, c0, 0" + : "=r"(sctlr_value)); + + sctlr_value &= ~SCTLR_DCACHE; + + __asm__ volatile("mcr p15, 0, %0, c1, c0, 0\n" + "isb" + :: "r"(sctlr_value)); + +} + +void +platform_cache_idle_enter( + void) +{ +#if __ARM_SMP__ + platform_cache_disable(); + + /* + * If we're only using a single CPU, just write back any + * dirty cachelines. We can avoid doing housekeeping + * on CPU data that would normally be modified by other + * CPUs. + */ + if (up_style_idle_exit && (real_ncpus == 1)) + CleanPoU_Dcache(); + else { + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + FlushPoU_Dcache(); + + cpu_data_ptr->cpu_CLW_active = 0; + __asm__ volatile("dmb ish"); + cpu_data_ptr->cpu_CLWFlush_req = 0; + cpu_data_ptr->cpu_CLWClean_req = 0; + CleanPoC_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t)); + } +#else + CleanPoU_Dcache(); +#endif + +#if defined (__ARM_SMP__) && defined (ARMA7) + uint32_t actlr_value = 0; + + /* Leave the coherency domain */ + __asm__ volatile("clrex\n" + "mrc p15, 0, %0, c1, c0, 1\n" + : "=r"(actlr_value)); + + actlr_value &= ~0x40; + + __asm__ volatile("mcr p15, 0, %0, c1, c0, 1\n" + /* Ensures any pending fwd request gets serviced and ends up */ + "dsb\n" + /* Forces the processor to re-fetch, so any pending fwd request gets into the core */ + "isb\n" + /* Ensures the second possible pending fwd request ends up. */ + "dsb\n" + :: "r"(actlr_value)); +#endif +} + +void +platform_cache_idle_exit( + void) +{ +#if defined (ARMA7) + uint32_t actlr_value = 0; + + /* Flush L1 caches and TLB before rejoining the coherency domain */ + FlushPoU_Dcache(); + /* + * If we're only using a single CPU, we can avoid flushing the + * I-cache or the TLB, as neither program text nor pagetables + * should have been changed during the idle period. We still + * want to flush the D-cache to PoU (above), as memory contents + * may have been changed by DMA. + */ + if (!up_style_idle_exit || (real_ncpus > 1)) { + InvalidatePoU_Icache(); + flush_core_tlb(); + } + + /* Rejoin the coherency domain */ + __asm__ volatile("mrc p15, 0, %0, c1, c0, 1\n" + : "=r"(actlr_value)); + + actlr_value |= 0x40; + + __asm__ volatile("mcr p15, 0, %0, c1, c0, 1\n" + "isb\n" + :: "r"(actlr_value)); + +#if __ARM_SMP__ + uint32_t sctlr_value = 0; + + /* Enable dcache allocation. */ + __asm__ volatile("mrc p15, 0, %0, c1, c0, 0\n" + : "=r"(sctlr_value)); + + sctlr_value |= SCTLR_DCACHE; + + __asm__ volatile("mcr p15, 0, %0, c1, c0, 0\n" + "isb" + :: "r"(sctlr_value)); + getCpuDatap()->cpu_CLW_active = 1; +#endif +#endif +} + +boolean_t +platform_cache_batch_wimg( + __unused unsigned int new_wimg, + __unused unsigned int size + ) +{ + boolean_t do_cache_op = FALSE; + + if ((cache_info()->c_bulksize_op != 0) && (size >= (cache_info()->c_bulksize_op))) do_cache_op = TRUE; + + return do_cache_op; +} + +void +platform_cache_flush_wimg( + __unused unsigned int new_wimg +) +{ +#if __ARM_SMP__ && defined (ARMA7) + cache_xcall(LWFlush); +#else + FlushPoC_Dcache(); + if (getCpuDatap()->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) getCpuDatap()->cpu_cache_dispatch) ( getCpuDatap()->cpu_id, CacheCleanFlush, 0x0UL , 0x0UL); +#endif +} + +#if __ARM_SMP__ && defined(ARMA7) +void +cache_xcall_handler(unsigned int op) +{ + cpu_data_t *cdp; + uint64_t abstime; + + cdp = getCpuDatap(); + + if ((op == LWFlush) && (cdp->cpu_CLWFlush_req > cdp->cpu_CLWFlush_last)) { + FlushPoU_Dcache(); + abstime = ml_get_timebase(); + cdp->cpu_CLWFlush_last = abstime; + cdp->cpu_CLWClean_last = abstime; + } else if ((op == LWClean) && (cdp->cpu_CLWClean_req > cdp->cpu_CLWClean_last)) { + CleanPoU_Dcache(); + abstime = ml_get_timebase(); + cdp->cpu_CLWClean_last = abstime; + } +} + + +void +cache_xcall(unsigned int op) +{ + boolean_t intr; + cpu_data_t *cdp; + cpu_data_t *target_cdp; + unsigned int cpu; + unsigned int signal; + uint64_t abstime; + + intr = ml_set_interrupts_enabled(FALSE); + cdp = getCpuDatap(); + abstime = ml_get_timebase(); + if (op == LWClean) + signal = SIGPLWClean; + else + signal = SIGPLWFlush; + + for (cpu=0; cpu < MAX_CPUS; cpu++) { + + target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + if(target_cdp == (cpu_data_t *)NULL) + break; + + if (target_cdp->cpu_CLW_active == 0) + continue; + + if (op == LWFlush) + target_cdp->cpu_CLWFlush_req = abstime; + else if (op == LWClean) + target_cdp->cpu_CLWClean_req = abstime; + __asm__ volatile("dmb ish"); + if (target_cdp->cpu_CLW_active == 0) { + if (op == LWFlush) + target_cdp->cpu_CLWFlush_req = 0x0ULL; + else if (op == LWClean) + target_cdp->cpu_CLWClean_req = 0x0ULL; + continue; + } + + if (target_cdp == cdp) + continue; + + if(KERN_SUCCESS != cpu_signal(target_cdp, signal, (void *)NULL, NULL)) { + if (op == LWFlush) + target_cdp->cpu_CLWFlush_req = 0x0ULL; + else if (op == LWClean) + target_cdp->cpu_CLWClean_req = 0x0ULL; + } + if (cpu == real_ncpus) + break; + } + + cache_xcall_handler (op); + + (void) ml_set_interrupts_enabled(intr); + + for (cpu=0; cpu < MAX_CPUS; cpu++) { + + target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + if(target_cdp == (cpu_data_t *)NULL) + break; + + if (target_cdp == cdp) + continue; + + if (op == LWFlush) + while ((target_cdp->cpu_CLWFlush_req != 0x0ULL) && (target_cdp->cpu_CLWFlush_last < abstime)); + else if (op == LWClean) + while ((target_cdp->cpu_CLWClean_req != 0x0ULL ) && (target_cdp->cpu_CLWClean_last < abstime)); + + if (cpu == real_ncpus) + break; + } + + if (op == LWFlush) + FlushPoC_Dcache(); + else if (op == LWClean) + CleanPoC_Dcache(); +} +#endif + + +#else /* __ARM_COHERENT_IO__ */ + +void +flush_dcache( + __unused vm_offset_t addr, + __unused unsigned length, + __unused boolean_t phys) +{ + __asm__ volatile ("dsb sy"); +} + +void +clean_dcache( + __unused vm_offset_t addr, + __unused unsigned length, + __unused boolean_t phys) +{ + __asm__ volatile ("dsb sy"); +} + +void +flush_dcache_syscall( + __unused vm_offset_t va, + __unused unsigned length) +{ + __asm__ volatile ("dsb sy"); +} + +void +dcache_incoherent_io_flush64( + __unused addr64_t pa, + __unused unsigned int size, + __unused unsigned int remaining, + __unused unsigned int *res) +{ + __asm__ volatile ("dsb sy"); + *res = LWOpDone; + return; +} + +void +dcache_incoherent_io_store64( + __unused addr64_t pa, + __unused unsigned int size, + __unused unsigned int remaining, + __unused unsigned int *res) +{ + __asm__ volatile ("dsb sy"); + *res = LWOpDone; + return; +} + +void +cache_sync_page( + ppnum_t pp +) +{ + pmap_paddr_t paddr = ptoa(pp); + + if (isphysmem(paddr)) { + vm_offset_t vaddr = phystokv(paddr); + +#ifdef __ARM_IC_NOALIAS_ICACHE__ + InvalidatePoU_IcacheRegion(vaddr, PAGE_SIZE); +#else + InvalidatePoU_Icache(); +#endif + } +} + +void +platform_cache_init( + void) +{ +} + +void +platform_cache_flush( + void) +{ +} + +void +platform_cache_clean( + void) +{ +} + +void +platform_cache_shutdown( + void) +{ +} + +void +platform_cache_idle_enter( + void) +{ +} + +void +platform_cache_idle_exit( + void) +{ +} + +boolean_t +platform_cache_batch_wimg( + __unused unsigned int new_wimg, + __unused unsigned int size + ) +{ + return TRUE; +} + +void +platform_cache_flush_wimg( + __unused unsigned int new_wimg) +{ +} + +#endif /* __ARM_COHERENT_IO__ */ diff --git a/osfmk/arm/caches_asm.s b/osfmk/arm/caches_asm.s new file mode 100644 index 000000000..b4e6a94c8 --- /dev/null +++ b/osfmk/arm/caches_asm.s @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2010 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include "assym.s" + + +/* + * void invalidate_mmu_cache(void) + * + * Invalidate d-cache and i-cache + */ + .text + .align 2 + .globl EXT(invalidate_mmu_cache) +LEXT(invalidate_mmu_cache) + mov r0, #0 + mcr p15, 0, r0, c7, c7, 0 // Invalidate caches + bx lr + +/* + * void invalidate_mmu_dcache(void) + * + * Invalidate d-cache + */ + .text + .align 2 + .globl EXT(invalidate_mmu_dcache) +LEXT(invalidate_mmu_dcache) + mov r0, #0 + mcr p15, 0, r0, c7, c6, 0 // Invalidate dcache + bx lr + +/* + * void invalidate_mmu_dcache_region(vm_offset_t va, unsigned length) + * + * Invalidate d-cache region + */ + .text + .align 2 + .globl EXT(invalidate_mmu_dcache_region) +LEXT(invalidate_mmu_dcache_region) + and r2, r0, #((1< + +#include + +extern void flush_dcache_syscall( vm_offset_t addr, unsigned length); + +#ifdef MACH_KERNEL_PRIVATE +extern void flush_dcache(vm_offset_t addr, unsigned count, int phys); +extern void flush_dcache64(addr64_t addr, unsigned count, int phys); +extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys); +extern void invalidate_icache64(addr64_t addr, unsigned cnt, int phys); + +#if __ARM_SMP__ && defined(ARMA7) +#define LWFlush 1 +#define LWClean 2 +extern void cache_xcall(unsigned int op); +extern void cache_xcall_handler(unsigned int op); +#endif +#endif +extern void clean_dcache(vm_offset_t addr, unsigned count, int phys); +extern void clean_dcache64(addr64_t addr, unsigned count, int phys); + +extern void CleanPoC_Dcache(void); +extern void CleanPoU_Dcache(void); + +/* + * May not actually perform a flush on platforms + * where AP caches are snooped by all agents on SoC. + * + * This is the one you need unless you really know what + * you're doing. + */ +extern void CleanPoC_DcacheRegion(vm_offset_t va, unsigned length); + +/* + * Always actually flushes the cache, even on platforms + * where AP caches are snooped by all agents. You + * probably don't need to use this. Intended for use in + * panic save routine (where caches will be yanked by reset + * and coherency doesn't help). + */ +extern void CleanPoC_DcacheRegion_Force(vm_offset_t va, unsigned length); + +extern void CleanPoU_DcacheRegion(vm_offset_t va, unsigned length); + +extern void FlushPoC_Dcache(void); +extern void FlushPoU_Dcache(void); +extern void FlushPoC_DcacheRegion(vm_offset_t va, unsigned length); + +#ifdef __arm__ +extern void invalidate_mmu_cache(void); +extern void invalidate_mmu_dcache(void); +extern void invalidate_mmu_dcache_region(vm_offset_t va, unsigned length); +#endif + +extern void InvalidatePoU_Icache(void); +extern void InvalidatePoU_IcacheRegion(vm_offset_t va, unsigned length); + +extern void cache_sync_page(ppnum_t pp); + +extern void platform_cache_init(void); +extern void platform_cache_idle_enter(void); +extern void platform_cache_idle_exit(void); +extern void platform_cache_flush(void); +extern boolean_t platform_cache_batch_wimg(unsigned int new_wimg, unsigned int size); +extern void platform_cache_flush_wimg(unsigned int new_wimg); +extern void platform_cache_clean(void); +extern void platform_cache_shutdown(void); +extern void platform_cache_disable(void); + +#endif /* #ifndef _ARM_CACHES_INTERNAL */ diff --git a/osfmk/arm/commpage/commpage.c b/osfmk/arm/commpage/commpage.c new file mode 100644 index 000000000..520a140d0 --- /dev/null +++ b/osfmk/arm/commpage/commpage.c @@ -0,0 +1,432 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * @APPLE_FREE_COPYRIGHT@ + */ +/* + * File: arm/commpage/commpage.c + * Purpose: Set up and export a RO/RW page + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for cpuid_info() & cache_info() */ +#include +#include +#include + +#include + +#if CONFIG_ATM +#include +#endif + +static void commpage_init_cpu_capabilities( void ); +static int commpage_cpus( void ); + +vm_address_t commPagePtr=0; +vm_address_t sharedpage_rw_addr = 0; +uint32_t _cpu_capabilities = 0; + +extern int gARMv81Atomics; /* For sysctl access from BSD side */ + +void +commpage_populate( + void) +{ + uint16_t c2; + int cpufamily; + + sharedpage_rw_addr = pmap_create_sharedpage(); + commPagePtr = (vm_address_t)_COMM_PAGE_BASE_ADDRESS; + + *((uint16_t*)(_COMM_PAGE_VERSION+_COMM_PAGE_RW_OFFSET)) = (uint16_t) _COMM_PAGE_THIS_VERSION; + + commpage_init_cpu_capabilities(); + commpage_set_timestamp(0, 0, 0, 0, 0); + + if (_cpu_capabilities & kCache32) + c2 = 32; + else if (_cpu_capabilities & kCache64) + c2 = 64; + else if (_cpu_capabilities & kCache128) + c2 = 128; + else + c2 = 0; + + *((uint16_t*)(_COMM_PAGE_CACHE_LINESIZE+_COMM_PAGE_RW_OFFSET)) = c2; + *((uint32_t*)(_COMM_PAGE_SPIN_COUNT+_COMM_PAGE_RW_OFFSET)) = 1; + + commpage_update_active_cpus(); + cpufamily = cpuid_get_cpufamily(); + + /* machine_info valid after ml_get_max_cpus() */ + *((uint8_t*)(_COMM_PAGE_PHYSICAL_CPUS+_COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.physical_cpu_max; + *((uint8_t*)(_COMM_PAGE_LOGICAL_CPUS+_COMM_PAGE_RW_OFFSET))= (uint8_t) machine_info.logical_cpu_max; + *((uint64_t*)(_COMM_PAGE_MEMORY_SIZE+_COMM_PAGE_RW_OFFSET)) = machine_info.max_mem; + *((uint32_t*)(_COMM_PAGE_CPUFAMILY+_COMM_PAGE_RW_OFFSET)) = (uint32_t)cpufamily; + *((uint32_t*)(_COMM_PAGE_DEV_FIRM+_COMM_PAGE_RW_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL); + *((uint8_t*)(_COMM_PAGE_USER_TIMEBASE+_COMM_PAGE_RW_OFFSET)) = user_timebase_allowed(); + *((uint8_t*)(_COMM_PAGE_CONT_HWCLOCK+_COMM_PAGE_RW_OFFSET)) = user_cont_hwclock_allowed(); + *((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT+_COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift; + +#if __arm64__ + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift_user32; + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT; +#elif (__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS) + /* enforce 16KB alignment for watch targets with new ABI */ + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT; + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT; +#else /* __arm64__ */ + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT; + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT; +#endif /* __arm64__ */ + + commpage_update_timebase(); + commpage_update_mach_continuous_time(0); + + clock_sec_t secs; + clock_usec_t microsecs; + clock_get_boottime_microtime(&secs, µsecs); + commpage_update_boottime(secs * USEC_PER_SEC + microsecs); + + /* + * set commpage approximate time to zero for initialization. + * scheduler shall populate correct value before running user thread + */ + *((uint64_t *)(_COMM_PAGE_APPROX_TIME+ _COMM_PAGE_RW_OFFSET)) = 0; +#ifdef CONFIG_MACH_APPROXIMATE_TIME + *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED+_COMM_PAGE_RW_OFFSET)) = 1; +#else + *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED+_COMM_PAGE_RW_OFFSET)) = 0; +#endif + + commpage_update_kdebug_state(); + +#if CONFIG_ATM + commpage_update_atm_diagnostic_config(atm_get_diagnostic_config()); +#endif + +} + +struct mu { + uint64_t m; // magic number + int32_t a; // add indicator + int32_t s; // shift amount +}; + +void +commpage_set_timestamp( + uint64_t tbr, + uint64_t secs, + uint64_t frac, + uint64_t scale, + uint64_t tick_per_sec) +{ + new_commpage_timeofday_data_t *commpage_timeofday_datap; + + if (commPagePtr == 0) + return; + + commpage_timeofday_datap = (new_commpage_timeofday_data_t *)(_COMM_PAGE_NEWTIMEOFDAY_DATA+_COMM_PAGE_RW_OFFSET); + + commpage_timeofday_datap->TimeStamp_tick = 0x0ULL; + +#if (__ARM_ARCH__ >= 7) + __asm__ volatile("dmb ish"); +#endif + commpage_timeofday_datap->TimeStamp_sec = secs; + commpage_timeofday_datap->TimeStamp_frac = frac; + commpage_timeofday_datap->Ticks_scale = scale; + commpage_timeofday_datap->Ticks_per_sec = tick_per_sec; + +#if (__ARM_ARCH__ >= 7) + __asm__ volatile("dmb ish"); +#endif + commpage_timeofday_datap->TimeStamp_tick = tbr; +} + +/* + * Update _COMM_PAGE_MEMORY_PRESSURE. Called periodically from vm's compute_memory_pressure() + */ + +void +commpage_set_memory_pressure( + unsigned int pressure ) +{ + if (commPagePtr == 0) + return; + *((uint32_t *)(_COMM_PAGE_MEMORY_PRESSURE+_COMM_PAGE_RW_OFFSET)) = pressure; +} + +/* + * Update _COMM_PAGE_SPIN_COUNT. We might want to reduce when running on a battery, etc. + */ + +void +commpage_set_spin_count( + unsigned int count ) +{ + if (count == 0) /* we test for 0 after decrement, not before */ + count = 1; + + if (commPagePtr == 0) + return; + *((uint32_t *)(_COMM_PAGE_SPIN_COUNT+_COMM_PAGE_RW_OFFSET)) = count; +} + +/* + * Determine number of CPUs on this system. + */ +static int +commpage_cpus( void ) +{ + int cpus; + + cpus = ml_get_max_cpus(); // NB: this call can block + + if (cpus == 0) + panic("commpage cpus==0"); + if (cpus > 0xFF) + cpus = 0xFF; + + return cpus; +} + +/* + * Initialize _cpu_capabilities vector + */ +static void +commpage_init_cpu_capabilities( void ) +{ + uint32_t bits; + int cpus; + ml_cpu_info_t cpu_info; + + bits = 0; + ml_cpu_get_info(&cpu_info); + + switch (cpu_info.cache_line_size) { + case 128: + bits |= kCache128; + break; + case 64: + bits |= kCache64; + break; + case 32: + bits |= kCache32; + break; + default: + break; + } + cpus = commpage_cpus(); + + if (cpus == 1) + bits |= kUP; + + bits |= (cpus << kNumCPUsShift); + + bits |= kFastThreadLocalStorage; // TPIDRURO for TLS +#if __ARM_VFP__ + bits |= kHasVfp; +#if (__ARM_VFP__ >= 3) + bits |= kHasNeon; + +#if defined(__arm64__) + bits |= kHasNeonHPFP; +#else + boolean_t intr = ml_set_interrupts_enabled(FALSE); + unsigned int mvfr1 = get_mvfr1(); + + if (mvfr1 & MVFR_ASIMD_HPFP) + bits |= kHasNeonHPFP; + (void) ml_set_interrupts_enabled(intr); +#endif +#endif +#endif +#if defined(__arm64__) + bits |= kHasFMA; +#endif +#if __ARM_ENABLE_WFE_ +#ifdef __arm64__ + if (arm64_wfe_allowed()) { + bits |= kHasEvent; + } +#else + bits |= kHasEvent; +#endif +#endif +#if __ARM_V8_CRYPTO_EXTENSIONS__ + bits |= kHasARMv8Crypto; +#endif +#ifdef __arm64__ + if ((__builtin_arm_rsr64("ID_AA64ISAR0_EL1") & ID_AA64ISAR0_EL1_ATOMIC_MASK) == ID_AA64ISAR0_EL1_ATOMIC_8_1) { + bits |= kHasARMv81Atomics; + gARMv81Atomics = 1; + } +#endif + _cpu_capabilities = bits; + + *((uint32_t *)(_COMM_PAGE_CPU_CAPABILITIES+_COMM_PAGE_RW_OFFSET)) = _cpu_capabilities; +} + +/* + * Updated every time a logical CPU goes offline/online + */ +void +commpage_update_active_cpus(void) +{ + if (!commPagePtr) + return; + *((uint8_t *)(_COMM_PAGE_ACTIVE_CPUS+_COMM_PAGE_RW_OFFSET)) = processor_avail_count; +} + +/* + * Update the commpage bits for mach_absolute_time and mach_continuous_time (for userspace) + */ +void +commpage_update_timebase(void) +{ + if (commPagePtr) { + *((uint64_t*)(_COMM_PAGE_TIMEBASE_OFFSET+_COMM_PAGE_RW_OFFSET)) = rtclock_base_abstime; + } +} + +/* + * Update the commpage with current kdebug state. This currently has bits for + * global trace state, and typefilter enablement. It is likely additional state + * will be tracked in the future. + * + * INVARIANT: This value will always be 0 if global tracing is disabled. This + * allows simple guard tests of "if (*_COMM_PAGE_KDEBUG_ENABLE) { ... }" + */ +void +commpage_update_kdebug_state(void) +{ + if (commPagePtr) + *((volatile uint32_t*)(_COMM_PAGE_KDEBUG_ENABLE+_COMM_PAGE_RW_OFFSET)) = kdebug_commpage_state(); +} + +/* Ditto for atm_diagnostic_config */ +void +commpage_update_atm_diagnostic_config(uint32_t diagnostic_config) +{ + if (commPagePtr) + *((volatile uint32_t*)(_COMM_PAGE_ATM_DIAGNOSTIC_CONFIG+_COMM_PAGE_RW_OFFSET)) = diagnostic_config; +} + +/* + * Update the commpage data with the state of multiuser mode for + * this device. Allowing various services in userspace to avoid + * IPC in the (more common) non-multiuser environment. + */ +void +commpage_update_multiuser_config(uint32_t multiuser_config) +{ + if (commPagePtr) + *((volatile uint32_t *)(_COMM_PAGE_MULTIUSER_CONFIG+_COMM_PAGE_RW_OFFSET)) = multiuser_config; +} + +/* + * update the commpage data for + * last known value of mach_absolute_time() + */ + +void +commpage_update_mach_approximate_time(uint64_t abstime) +{ +#ifdef CONFIG_MACH_APPROXIMATE_TIME + uintptr_t approx_time_base = (uintptr_t)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET); + uint64_t saved_data; + + if (commPagePtr) { + saved_data = atomic_load_explicit((_Atomic uint64_t *)approx_time_base, + memory_order_relaxed); + if (saved_data < abstime) { + /* ignoring the success/fail return value assuming that + * if the value has been updated since we last read it, + * "someone" has a newer timestamp than us and ours is + * now invalid. */ + atomic_compare_exchange_strong_explicit((_Atomic uint64_t *)approx_time_base, + &saved_data, abstime, memory_order_relaxed, memory_order_relaxed); + } + } +#else +#pragma unused (abstime) +#endif +} + +/* + * update the commpage data's total system sleep time for + * userspace call to mach_continuous_time() + */ +void +commpage_update_mach_continuous_time(uint64_t sleeptime) +{ + if (commPagePtr) { +#ifdef __arm64__ + *((uint64_t *)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = sleeptime; +#else + uint64_t *c_time_base = (uint64_t *)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET); + uint64_t old; + do { + old = *c_time_base; + } while(!OSCompareAndSwap64(old, sleeptime, c_time_base)); +#endif /* __arm64__ */ + } +} + +/* + * update the commpage's value for the boot time + */ +void +commpage_update_boottime(uint64_t value) +{ + if (commPagePtr) { +#ifdef __arm64__ + *((uint64_t *)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET)) = value; +#else + uint64_t *cp = (uint64_t *)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET); + uint64_t old_value; + do { + old_value = *cp; + } while (!OSCompareAndSwap64(old_value, value, cp)); +#endif /* __arm64__ */ + } +} diff --git a/osfmk/arm/commpage/commpage.h b/osfmk/arm/commpage/commpage.h new file mode 100644 index 000000000..69ac59cf7 --- /dev/null +++ b/osfmk/arm/commpage/commpage.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2003-2008 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _ARM_COMMPAGE_H +#define _ARM_COMMPAGE_H + +#ifndef __ASSEMBLER__ +#include +#endif /* __ASSEMBLER__ */ + +extern void commpage_set_timestamp(uint64_t tbr, uint64_t secs, uint64_t frac, uint64_t scale, uint64_t tick_per_sec); +#define commpage_disable_timestamp() commpage_set_timestamp( 0, 0, 0, 0, 0 ); +extern void commpage_set_memory_pressure( unsigned int pressure ); +extern void commpage_update_active_cpus(void); +extern void commpage_set_spin_count(unsigned int count); +extern void commpage_update_timebase(void); +extern void commpage_update_mach_approximate_time(uint64_t); +extern void commpage_update_kdebug_state(void); +extern void commpage_update_atm_diagnostic_config(uint32_t); +extern void commpage_update_mach_continuous_time(uint64_t sleeptime); +extern void commpage_update_multiuser_config(uint32_t); +extern void commpage_update_boottime(uint64_t boottime_usec); + +#endif /* _ARM_COMMPAGE_H */ diff --git a/osfmk/arm/commpage/commpage_sigs.h b/osfmk/arm/commpage/commpage_sigs.h new file mode 100644 index 000000000..d392097a1 --- /dev/null +++ b/osfmk/arm/commpage/commpage_sigs.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#define BSWAP_32(x) \ + (((x) << 24) & 0xff000000) | \ + (((x) << 8) & 0x00ff0000) | \ + (((x) >> 8) & 0x0000ff00) | \ + (((x) >> 24) & 0x000000ff) + +#define BSWAP_32_OFFSET(x) \ + BSWAP_32(x + _COMM_PAGE_SIGS_OFFSET) + +#define COMMPAGE_SIGS_BEGIN \ +.const_data ; \ +.align 2 ; \ +.private_extern _commpage_sigs_begin ; \ +_commpage_sigs_begin: + +#define COMMPAGE_SIGS_DONE \ +.private_extern _commpage_sigs_end ; \ +_commpage_sigs_end: ; \ + +#define COMMPAGE_SIG_START(x) \ +.private_extern _commpage_sig ## x ; \ +_commpage_sig ## x ## : ; \ + .long BSWAP_32(0x14400000) ; \ + .long BSWAP_32(0x00000001) ; \ + .asciz # x ; \ + .align 2 ; \ + .long BSWAP_32(0x14400000) + +#define COMMPAGE_SIG_END(x) \ + .long BSWAP_32(0x4e800020) ; \ + .long BSWAP_32(0x14400000) ; \ + .long BSWAP_32(0x00000000) ; \ + .asciz # x ; \ + .align 2 ; \ + .long BSWAP_32(0x14400000) + +#define OBJCRTP_SIG_START(x) COMMPAGE_SIG_START(x) + +#define OBJCRTP_SIG_END(x) \ + .long BSWAP_32(0x14400000) ; \ + .long BSWAP_32(0x00000000) ; \ + .asciz # x ; \ + .align 2 ; \ + .long BSWAP_32(0x14400000) + +#define OBJCRTP_SIG_CALL_SUBJECT(x) \ + .long BSWAP_32(0x14400002) ; \ + .long BSWAP_32(0x00000000) ; \ + .long BSWAP_32(0x00040000) ; \ + .long BSWAP_32(0x00000000) ; \ + .asciz # x ; \ + .align 2 ; \ + .long BSWAP_32(0x14400002) + +#define ARG(n) \ + ((((n * 2) + 6) << 20) + 4) + +#define COMMPAGE_SIG_ARG(n) \ + .long BSWAP_32(0x14400001) ; \ + .long BSWAP_32(ARG(n)) ; \ + .long BSWAP_32(0x14400001) + +#define COMMPAGE_SIG_CALL(x, n) \ + .long BSWAP_32(0x14400002) ; \ + .long BSWAP_32(n) ; \ + .long BSWAP_32(0x00000000) ; \ + .long BSWAP_32(0x00000000) ; \ + .asciz # x ; \ + .align 2 ; \ + .long BSWAP_32(0x14400002) + +#define COMMPAGE_SIG_CALL_VOID(x) \ + COMMPAGE_SIG_CALL(x, 0) + +#define COMMPAGE_SIG_CALL_RET0(x) \ + COMMPAGE_SIG_CALL(x, ARG(0)) + +#define COMMPAGE_SIG_CALL_RET1(x) \ + COMMPAGE_SIG_CALL(x, ARG(1)) + +#define COMMPAGE_FAST_TRAP(x) \ + .long BSWAP_32(0x14400005) ; \ + .long BSWAP_32_OFFSET(x) ; \ + .long BSWAP_32(0x14400005) diff --git a/osfmk/arm/conf.c b/osfmk/arm/conf.c new file mode 100644 index 000000000..a9da87124 --- /dev/null +++ b/osfmk/arm/conf.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * @APPLE_FREE_COPYRIGHT@ + */ +/* + * Mach Operating System Copyright (c) 1991,1990,1989 Carnegie Mellon + * University All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright notice + * and this permission notice appear in all copies of the software, + * derivative works or modified versions, and any portions thereof, and that + * both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" CONDITION. + * CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ANY DAMAGES + * WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science Carnegie Mellon University Pittsburgh PA + * 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon the + * rights to redistribute these changes. + */ +/* + * */ + +#include +#include +#include + +/* + * Clock device subsystem configuration. The clock_list[] + * table contains the clock structures for all clocks in + * the system. + */ + +extern const struct clock_ops sysclk_ops, calend_ops; + +/* + * List of clock devices. + */ +SECURITY_READ_ONLY_LATE(struct clock) clock_list[] = { + + /* SYSTEM_CLOCK */ + {&sysclk_ops, 0, 0}, + + /* CALENDAR_CLOCK */ + {&calend_ops, 0, 0}, +}; +int clock_count = sizeof(clock_list) / sizeof(clock_list[0]); diff --git a/osfmk/arm/cpu.c b/osfmk/arm/cpu.c new file mode 100644 index 000000000..46cfcddb7 --- /dev/null +++ b/osfmk/arm/cpu.c @@ -0,0 +1,604 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * File: arm/cpu.c + * + * cpu specific routines + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if KPC +#include +#endif + +extern unsigned int resume_idle_cpu; +extern unsigned int start_cpu; + +unsigned int start_cpu_paddr; + +extern boolean_t idle_enable; +extern unsigned int real_ncpus; +extern uint64_t wake_abstime; + +extern void* wfi_inst; +unsigned wfi_fast = 1; +unsigned patch_to_nop = 0xe1a00000; + +void *LowExceptionVectorsAddr; +#define IOS_STATE (((vm_offset_t)LowExceptionVectorsAddr + 0x80)) +#define IOS_STATE_SIZE (0x08UL) +static const uint8_t suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'}; +static const uint8_t running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'}; + +/* + * Routine: cpu_bootstrap + * Function: + */ +void +cpu_bootstrap(void) +{ +} + + +/* + * Routine: cpu_sleep + * Function: + */ +void +cpu_sleep(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + pmap_switch_user_ttb(kernel_pmap); + cpu_data_ptr->cpu_active_thread = current_thread(); + cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr; + cpu_data_ptr->cpu_flags |= SleepState; + cpu_data_ptr->cpu_user_debug = NULL; + + CleanPoC_Dcache(); + + PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id); + +} + +_Atomic uint32_t cpu_idle_count = 0; + +/* + * Routine: cpu_idle + * Function: + */ +void __attribute__((noreturn)) +cpu_idle(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop; + + if ((!idle_enable) || (cpu_data_ptr->cpu_signal & SIGPdisabled)) + Idle_load_context(); + if (!SetIdlePop()) + Idle_load_context(); + lastPop = cpu_data_ptr->rtcPop; + + pmap_switch_user_ttb(kernel_pmap); + cpu_data_ptr->cpu_active_thread = current_thread(); + if (cpu_data_ptr->cpu_user_debug) + arm_debug_set(NULL); + cpu_data_ptr->cpu_user_debug = NULL; + + if (cpu_data_ptr->cpu_idle_notify) + ((processor_idle_t) cpu_data_ptr->cpu_idle_notify) (cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks); + + if (cpu_data_ptr->idle_timer_notify != 0) { + if (new_idle_timeout_ticks == 0x0ULL) { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } else { + /* set the new idle timeout */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } + timer_resync_deadlines(); + if (cpu_data_ptr->rtcPop != lastPop) + SetIdlePop(); + } + +#if KPC + kpc_idle(); +#endif + + platform_cache_idle_enter(); + cpu_idle_wfi((boolean_t) wfi_fast); + platform_cache_idle_exit(); + + ClearIdlePop(TRUE); + cpu_idle_exit(); +} + +/* + * Routine: cpu_idle_exit + * Function: + */ +void +cpu_idle_exit(void) +{ + uint64_t new_idle_timeout_ticks = 0x0ULL; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + +#if KPC + kpc_idle_exit(); +#endif + + + pmap_set_pmap(cpu_data_ptr->cpu_active_thread->map->pmap, current_thread()); + + if (cpu_data_ptr->cpu_idle_notify) + ((processor_idle_t) cpu_data_ptr->cpu_idle_notify) (cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks); + + if (cpu_data_ptr->idle_timer_notify != 0) { + if (new_idle_timeout_ticks == 0x0ULL) { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } else { + /* set the new idle timeout */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } + timer_resync_deadlines(); + } + + Idle_load_context(); +} + +void +cpu_init(void) +{ + cpu_data_t *cdp = getCpuDatap(); + arm_cpu_info_t *cpu_info_p; + + if (cdp->cpu_type != CPU_TYPE_ARM) { + + cdp->cpu_type = CPU_TYPE_ARM; + + timer_call_queue_init(&cdp->rtclock_timer.queue); + cdp->rtclock_timer.deadline = EndOfAllTime; + + if (cdp == &BootCpuData) { + do_cpuid(); + do_cacheid(); + do_mvfpid(); + } else { + /* + * We initialize non-boot CPUs here; the boot CPU is + * dealt with as part of pmap_bootstrap. + */ + pmap_cpu_data_init(); + } + /* ARM_SMP: Assuming identical cpu */ + do_debugid(); + + cpu_info_p = cpuid_info(); + + /* switch based on CPU's reported architecture */ + switch (cpu_info_p->arm_info.arm_arch) { + case CPU_ARCH_ARMv4T: + case CPU_ARCH_ARMv5T: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V4T; + break; + case CPU_ARCH_ARMv5TE: + case CPU_ARCH_ARMv5TEJ: + if (cpu_info_p->arm_info.arm_implementor == CPU_VID_INTEL) + cdp->cpu_subtype = CPU_SUBTYPE_ARM_XSCALE; + else + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V5TEJ; + break; + case CPU_ARCH_ARMv6: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V6; + break; + case CPU_ARCH_ARMv7: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7; + break; + case CPU_ARCH_ARMv7f: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7F; + break; + case CPU_ARCH_ARMv7s: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7S; + break; + case CPU_ARCH_ARMv7k: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7K; + break; + default: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_ALL; + break; + } + + cdp->cpu_threadtype = CPU_THREADTYPE_NONE; + } + cdp->cpu_stat.irq_ex_cnt_wake = 0; + cdp->cpu_stat.ipi_cnt_wake = 0; + cdp->cpu_stat.timer_cnt_wake = 0; + cdp->cpu_running = TRUE; + cdp->cpu_sleep_token_last = cdp->cpu_sleep_token; + cdp->cpu_sleep_token = 0x0UL; + +} + +cpu_data_t * +cpu_data_alloc(boolean_t is_boot_cpu) +{ + cpu_data_t *cpu_data_ptr = NULL; + + if (is_boot_cpu) + cpu_data_ptr = &BootCpuData; + else { + void *irq_stack = NULL; + void *fiq_stack = NULL; + + if ((kmem_alloc(kernel_map, (vm_offset_t *)&cpu_data_ptr, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU)) != KERN_SUCCESS) + goto cpu_data_alloc_error; + + bzero((void *)cpu_data_ptr, sizeof(cpu_data_t)); + + if ((irq_stack = kalloc(INTSTACK_SIZE)) == 0) + goto cpu_data_alloc_error; +#if __BIGGEST_ALIGNMENT__ + /* force 16-byte alignment */ + if ((uint32_t)irq_stack & 0x0F) + irq_stack = (void *)((uint32_t)irq_stack + (0x10 - ((uint32_t)irq_stack & 0x0F))); +#endif + cpu_data_ptr->intstack_top = (vm_offset_t)irq_stack + INTSTACK_SIZE ; + cpu_data_ptr->istackptr = cpu_data_ptr->intstack_top; + + if ((fiq_stack = kalloc(PAGE_SIZE)) == 0) + goto cpu_data_alloc_error; +#if __BIGGEST_ALIGNMENT__ + /* force 16-byte alignment */ + if ((uint32_t)fiq_stack & 0x0F) + fiq_stack = (void *)((uint32_t)fiq_stack + (0x10 - ((uint32_t)fiq_stack & 0x0F))); +#endif + cpu_data_ptr->fiqstack_top = (vm_offset_t)fiq_stack + PAGE_SIZE ; + cpu_data_ptr->fiqstackptr = cpu_data_ptr->fiqstack_top; + } + + cpu_data_ptr->cpu_processor = cpu_processor_alloc(is_boot_cpu); + if (cpu_data_ptr->cpu_processor == (struct processor *)NULL) + goto cpu_data_alloc_error; + + return cpu_data_ptr; + +cpu_data_alloc_error: + panic("cpu_data_alloc() failed\n"); + return (cpu_data_t *)NULL; +} + + +void +cpu_data_free(cpu_data_t *cpu_data_ptr) +{ + if (cpu_data_ptr == &BootCpuData) + return; + + cpu_processor_free( cpu_data_ptr->cpu_processor); + kfree( (void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE); + kfree( (void *)(cpu_data_ptr->fiqstack_top - PAGE_SIZE), PAGE_SIZE); + kmem_free(kernel_map, (vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t)); +} + +void +cpu_data_init(cpu_data_t *cpu_data_ptr) +{ + uint32_t i = 0; + + cpu_data_ptr->cpu_flags = 0; +#if __arm__ + cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable; +#endif + cpu_data_ptr->interrupts_enabled = 0; + cpu_data_ptr->cpu_int_state = 0; + cpu_data_ptr->cpu_pending_ast = AST_NONE; + cpu_data_ptr->cpu_cache_dispatch = (void *) 0; + cpu_data_ptr->rtcPop = EndOfAllTime; + cpu_data_ptr->rtclock_datap = &RTClockData; + cpu_data_ptr->cpu_user_debug = NULL; + cpu_data_ptr->cpu_base_timebase_low = 0; + cpu_data_ptr->cpu_base_timebase_high = 0; + cpu_data_ptr->cpu_idle_notify = (void *) 0; + cpu_data_ptr->cpu_idle_latency = 0x0ULL; + cpu_data_ptr->cpu_idle_pop = 0x0ULL; + cpu_data_ptr->cpu_reset_type = 0x0UL; + cpu_data_ptr->cpu_reset_handler = 0x0UL; + cpu_data_ptr->cpu_reset_assist = 0x0UL; + cpu_data_ptr->cpu_regmap_paddr = 0x0ULL; + cpu_data_ptr->cpu_phys_id = 0x0UL; + cpu_data_ptr->cpu_l2_access_penalty = 0; + cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP; + cpu_data_ptr->cpu_cluster_id = 0; + cpu_data_ptr->cpu_l2_id = 0; + cpu_data_ptr->cpu_l2_size = 0; + cpu_data_ptr->cpu_l3_id = 0; + cpu_data_ptr->cpu_l3_size = 0; + + cpu_data_ptr->cpu_signal = SIGPdisabled; + +#if DEBUG || DEVELOPMENT + cpu_data_ptr->failed_xcall = NULL; + cpu_data_ptr->failed_signal = 0; + cpu_data_ptr->failed_signal_count = 0; +#endif + + cpu_data_ptr->cpu_get_fiq_handler = NULL; + cpu_data_ptr->cpu_tbd_hardware_addr = NULL; + cpu_data_ptr->cpu_tbd_hardware_val = NULL; + cpu_data_ptr->cpu_get_decrementer_func = NULL; + cpu_data_ptr->cpu_set_decrementer_func = NULL; + cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; + cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL; + cpu_data_ptr->cpu_xcall_p0 = NULL; + cpu_data_ptr->cpu_xcall_p1 = NULL; + +#if __ARM_SMP__ && defined(ARMA7) + cpu_data_ptr->cpu_CLWFlush_req = 0x0ULL; + cpu_data_ptr->cpu_CLWFlush_last = 0x0ULL; + cpu_data_ptr->cpu_CLWClean_req = 0x0ULL; + cpu_data_ptr->cpu_CLWClean_last = 0x0ULL; + cpu_data_ptr->cpu_CLW_active = 0x1UL; +#endif + + pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data; + + pmap_cpu_data_ptr->cpu_user_pmap = (struct pmap *) NULL; + pmap_cpu_data_ptr->cpu_user_pmap_stamp = 0; + pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM; + + for (i = 0; i < (sizeof(pmap_cpu_data_ptr->cpu_asid_high_bits) / sizeof(*pmap_cpu_data_ptr->cpu_asid_high_bits)); i++) { + pmap_cpu_data_ptr->cpu_asid_high_bits[i] = 0; + } + cpu_data_ptr->halt_status = CPU_NOT_HALTED; +} + +kern_return_t +cpu_data_register(cpu_data_t *cpu_data_ptr) +{ + int cpu; + + cpu = OSIncrementAtomic((SInt32*)&real_ncpus); + if (real_ncpus > MAX_CPUS) { + return KERN_FAILURE; + } + + cpu_data_ptr->cpu_number = cpu; + CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr; + CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys( (vm_offset_t)cpu_data_ptr); + return KERN_SUCCESS; +} + +kern_return_t +cpu_start(int cpu) +{ + kprintf("cpu_start() cpu: %d\n", cpu); + if (cpu == cpu_number()) { + cpu_machine_init(); + return KERN_SUCCESS; + } else { +#if __ARM_SMP__ + cpu_data_t *cpu_data_ptr; + thread_t first_thread; + + cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr; + cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr; + + cpu_data_ptr->cpu_pmap_cpu_data.cpu_user_pmap = NULL; + + if (cpu_data_ptr->cpu_processor->next_thread != THREAD_NULL) + first_thread = cpu_data_ptr->cpu_processor->next_thread; + else + first_thread = cpu_data_ptr->cpu_processor->idle_thread; + cpu_data_ptr->cpu_active_thread = first_thread; + first_thread->machine.CpuDatap = cpu_data_ptr; + + flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE); + flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); + (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL); + return KERN_SUCCESS; +#else + return KERN_FAILURE; +#endif + } +} + +void +cpu_timebase_init(boolean_t from_boot __unused) +{ + cpu_data_t *cdp = getCpuDatap(); + + if (cdp->cpu_get_fiq_handler == NULL) { + cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler; + cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer; + cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer; + cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr; + cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val; + } + cdp->cpu_decrementer = 0x7FFFFFFFUL; + cdp->cpu_timebase_low = 0x0UL; + cdp->cpu_timebase_high = 0x0UL; + +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) + /* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit. */ + cdp->cpu_base_timebase_low = rtclock_base_abstime_low; + cdp->cpu_base_timebase_high = rtclock_base_abstime_high; +#else + *((uint64_t *) & cdp->cpu_base_timebase_low) = rtclock_base_abstime; +#endif +} + + +__attribute__((noreturn)) +void +ml_arm_sleep(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (cpu_data_ptr == &BootCpuData) { + cpu_data_t *target_cdp; + unsigned int cpu; + + for (cpu=0; cpu < MAX_CPUS; cpu++) { + target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + if(target_cdp == (cpu_data_t *)NULL) + break; + + if (target_cdp == cpu_data_ptr) + continue; + + while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH); + } + + /* Now that the other cores have entered the sleep path, set + * the abstime fixup we'll use when we resume.*/ + rtclock_base_abstime = ml_get_timebase(); + wake_abstime = rtclock_base_abstime; + + } else { + platform_cache_disable(); + CleanPoU_Dcache(); + } + cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; +#if __ARM_SMP__ && defined(ARMA7) + cpu_data_ptr->cpu_CLWFlush_req = 0; + cpu_data_ptr->cpu_CLWClean_req = 0; + __builtin_arm_dmb(DMB_ISH); + cpu_data_ptr->cpu_CLW_active = 0; +#endif + if (cpu_data_ptr == &BootCpuData) { + platform_cache_disable(); + platform_cache_shutdown(); + bcopy((const void *)suspend_signature, (void *)(IOS_STATE), IOS_STATE_SIZE); + } else + CleanPoC_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t)); + + __builtin_arm_dsb(DSB_SY); + while (TRUE) { +#if __ARM_ENABLE_WFE_ + __builtin_arm_wfe(); +#endif + } /* Spin */ +} + +void +cpu_machine_idle_init(boolean_t from_boot) +{ + static const unsigned int *BootArgs_paddr = (unsigned int *)NULL; + static const unsigned int *CpuDataEntries_paddr = (unsigned int *)NULL; + static unsigned int resume_idle_cpu_paddr = (unsigned int )NULL; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (from_boot) { + unsigned int jtag = 0; + unsigned int wfi; + + + if (PE_parse_boot_argn("jtag", &jtag, sizeof (jtag))) { + if (jtag != 0) + idle_enable = FALSE; + else + idle_enable = TRUE; + } else + idle_enable = TRUE; + + if (!PE_parse_boot_argn("wfi", &wfi, sizeof (wfi))) + wfi = 1; + + if (wfi == 0) + bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&patch_to_nop), + (addr64_t)ml_static_vtop((vm_offset_t)&wfi_inst), sizeof(unsigned)); + if (wfi == 2) + wfi_fast = 0; + + LowExceptionVectorsAddr = (void *)ml_io_map(ml_vtophys((vm_offset_t)gPhysBase), PAGE_SIZE); + + /* Copy Exception Vectors low, but don't touch the sleep token */ + bcopy((void *)&ExceptionLowVectorsBase, (void *)LowExceptionVectorsAddr, 0x90); + bcopy(((void *)(((vm_offset_t)&ExceptionLowVectorsBase) + 0xA0)), ((void *)(((vm_offset_t)LowExceptionVectorsAddr) + 0xA0)), ARM_PGBYTES - 0xA0); + + start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu); + + BootArgs_paddr = (unsigned int *)ml_static_vtop((vm_offset_t)BootArgs); + bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&BootArgs_paddr), + (addr64_t)((unsigned int)(gPhysBase) + + ((unsigned int)&(ResetHandlerData.boot_args) - (unsigned int)&ExceptionLowVectorsBase)), + 4); + + CpuDataEntries_paddr = (unsigned int *)ml_static_vtop((vm_offset_t)CpuDataEntries); + bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&CpuDataEntries_paddr), + (addr64_t)((unsigned int)(gPhysBase) + + ((unsigned int)&(ResetHandlerData.cpu_data_entries) - (unsigned int)&ExceptionLowVectorsBase)), + 4); + + CleanPoC_DcacheRegion((vm_offset_t) phystokv((char *) (gPhysBase)), PAGE_SIZE); + + resume_idle_cpu_paddr = (unsigned int)ml_static_vtop((vm_offset_t)&resume_idle_cpu); + + } + + if (cpu_data_ptr == &BootCpuData) { + bcopy(((const void *)running_signature), (void *)(IOS_STATE), IOS_STATE_SIZE); + }; + + cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr; + clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); +} + +void +machine_track_platform_idle(boolean_t entry) +{ + if (entry) + (void)__c11_atomic_fetch_add(&cpu_idle_count, 1, __ATOMIC_RELAXED); + else + (void)__c11_atomic_fetch_sub(&cpu_idle_count, 1, __ATOMIC_RELAXED); +} + diff --git a/osfmk/arm/cpu_affinity.h b/osfmk/arm/cpu_affinity.h new file mode 100644 index 000000000..531428736 --- /dev/null +++ b/osfmk/arm/cpu_affinity.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef KERNEL_PRIVATE +#ifndef _ARM_CPU_AFFINITY_H_ +#define _ARM_CPU_AFFINITY_H_ + +static inline int +ml_get_max_affinity_sets(void) +{ + return 0; +} + +static inline processor_set_t +ml_affinity_to_pset(__unused int affinity_num) +{ + return PROCESSOR_SET_NULL; +} + +#endif /* _ARM_CPU_AFFINITY_H_ */ +#endif /* KERNEL_PRIVATE */ diff --git a/osfmk/arm/cpu_capabilities.h b/osfmk/arm/cpu_capabilities.h new file mode 100644 index 000000000..f1d16dde6 --- /dev/null +++ b/osfmk/arm/cpu_capabilities.h @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef PRIVATE + +#ifndef _ARM_CPU_CAPABILITIES_H +#define _ARM_CPU_CAPABILITIES_H + +#ifndef __ASSEMBLER__ +#include +#ifdef KERNEL_PRIVATE +#include +#endif +#endif + +/* + * This is the authoritative way to determine from user mode what + * implementation-specific processor features are available. + * This API only supported for Apple internal use. + * + */ + +/* + * Bit definitions for _cpu_capabilities: + */ +#define kCache32 0x00000010 // cache line size is 32 bytes +#define kCache64 0x00000020 // cache line size is 64 bytes +#define kCache128 0x00000040 // cache line size is 128 bytes +#define kFastThreadLocalStorage 0x00000080 // TLS ptr is kept in a user-mode-readable register +#define kHasNeon 0x00000100 // Advanced SIMD is supported +#define kHasNeonHPFP 0x00000200 // Advanced SIMD half-precision +#define kHasVfp 0x00000400 // VFP is supported +#define kHasEvent 0x00001000 // WFE/SVE and period event wakeup +#define kHasFMA 0x00002000 // Fused multiply add is supported +#define kUP 0x00008000 // set if (kNumCPUs == 1) +#define kNumCPUs 0x00FF0000 // number of CPUs (see _NumCPUs() below) +#define kHasARMv8Crypto 0x01000000 // Optional ARMv8 Crypto extensions +#define kHasARMv81Atomics 0x02000000 // ARMv8.1 Atomic instructions supported + +#define kNumCPUsShift 16 // see _NumCPUs() below + + /* + * Bit definitions for multiuser_config: + */ +#define kIsMultiUserDevice 0x80000000 // this device is in multiuser mode +#define kMultiUserCurrentUserMask 0x7fffffff // the current user UID of the multiuser device + +#ifndef __ASSEMBLER__ +#include + +extern int _get_cpu_capabilities( void ); + +__inline static +int _NumCPUs( void ) +{ + return (_get_cpu_capabilities() & kNumCPUs) >> kNumCPUsShift; +} + +typedef struct { + volatile uint64_t TimeBase; + volatile uint32_t TimeStamp_sec; + volatile uint32_t TimeStamp_usec; + volatile uint32_t TimeBaseTicks_per_sec; + volatile uint32_t TimeBaseTicks_per_usec; + volatile uint64_t TimeBase_magic; + volatile uint32_t TimeBase_add; + volatile uint32_t TimeBase_shift; +} commpage_timeofday_data_t; + +#endif /* __ASSEMBLER__ */ + + +/* + * The shared kernel/user "comm page(s)": + */ + +#if defined(__arm64__) + +#define _COMM_PAGE64_BASE_ADDRESS (0xfffffff0001fc000ULL) /* Just below the kernel, safely in TTBR1 */ +#define _COMM_PRIV_PAGE64_BASE_ADDRESS (_COMM_PAGE64_BASE_ADDRESS - (PAGE_SIZE)) /* Privileged RO in kernel mode */ + +#define _COMM_PAGE64_AREA_LENGTH (_COMM_PAGE32_AREA_LENGTH) +#define _COMM_PAGE64_AREA_USED (-1) + +// macro to change a user comm page address to one that is accessible from privileged mode +// we can no longer access user memory in privileged mode once PAN is enabled +#define _COMM_PAGE_PRIV(_addr_) ((_addr_) - (_COMM_PAGE_START_ADDRESS) + (_COMM_PRIV_PAGE64_BASE_ADDRESS)) + +#ifdef KERNEL_PRIVATE +extern vm_address_t sharedpage_rw_addr; +#define _COMM_PAGE_RW_OFFSET (0) +#define _COMM_PAGE_AREA_LENGTH (PAGE_SIZE) + +#define _COMM_PAGE_BASE_ADDRESS (sharedpage_rw_addr) +#define _COMM_PAGE_START_ADDRESS (sharedpage_rw_addr) +#else +#define _COMM_PAGE_AREA_LENGTH (4096) + +#define _COMM_PAGE_BASE_ADDRESS _COMM_PAGE64_BASE_ADDRESS +#define _COMM_PAGE_START_ADDRESS _COMM_PAGE64_BASE_ADDRESS +#endif + +#elif defined(__arm__) + +#define _COMM_PAGE64_BASE_ADDRESS (-1) +#define _COMM_PAGE64_AREA_LENGTH (-1) +#define _COMM_PAGE64_AREA_USED (-1) + +// macro to change a user comm page address to one that is accessible from privileged mode +// this macro is stubbed as PAN is not available on AARCH32, +// but this may still be required for compatibility +#define _COMM_PAGE_PRIV(_addr_) (_addr_) + +#ifdef KERNEL_PRIVATE +extern vm_address_t sharedpage_rw_addr; +#define _COMM_PAGE_RW_OFFSET (sharedpage_rw_addr-_COMM_PAGE_BASE_ADDRESS) +#define _COMM_PAGE_AREA_LENGTH (PAGE_SIZE) +#else +#define _COMM_PAGE_AREA_LENGTH (4096) +#endif + +#define _COMM_PAGE_BASE_ADDRESS _COMM_PAGE32_BASE_ADDRESS +#define _COMM_PAGE_START_ADDRESS _COMM_PAGE32_BASE_ADDRESS + +#else +#error Unknown architecture. +#endif + +#define _COMM_PAGE32_BASE_ADDRESS (0xFFFF4000) /* Must be outside of normal map bounds */ +#define _COMM_PAGE32_AREA_LENGTH (_COMM_PAGE_AREA_LENGTH) + +#define _COMM_PAGE_TEXT_START (-1) +#define _COMM_PAGE32_TEXT_START (-1) +#define _COMM_PAGE64_TEXT_START (-1) +#define _COMM_PAGE_PFZ_START_OFFSET (-1) +#define _COMM_PAGE_PFZ_END_OFFSET (-1) + +#define _COMM_PAGE32_OBJC_SIZE 0ULL +#define _COMM_PAGE32_OBJC_BASE 0ULL +#define _COMM_PAGE64_OBJC_SIZE 0ULL +#define _COMM_PAGE64_OBJC_BASE 0ULL + +/* + * data in the comm pages + * apply _COMM_PAGE_PRIV macro to use these in privileged mode + */ +#define _COMM_PAGE_SIGNATURE (_COMM_PAGE_START_ADDRESS+0x000) // first few bytes are a signature +#define _COMM_PAGE_VERSION (_COMM_PAGE_START_ADDRESS+0x01E) // 16-bit version# +#define _COMM_PAGE_THIS_VERSION 3 // version of the commarea format + +#define _COMM_PAGE_CPU_CAPABILITIES (_COMM_PAGE_START_ADDRESS+0x020) // uint32_t _cpu_capabilities +#define _COMM_PAGE_NCPUS (_COMM_PAGE_START_ADDRESS+0x022) // uint8_t number of configured CPUs +#define _COMM_PAGE_USER_PAGE_SHIFT_32 (_COMM_PAGE_START_ADDRESS+0x024) // VM page shift for 32-bit processes +#define _COMM_PAGE_USER_PAGE_SHIFT_64 (_COMM_PAGE_START_ADDRESS+0x025) // VM page shift for 64-bit processes +#define _COMM_PAGE_CACHE_LINESIZE (_COMM_PAGE_START_ADDRESS+0x026) // uint16_t cache line size +#define _COMM_PAGE_SCHED_GEN (_COMM_PAGE_START_ADDRESS+0x028) // uint32_t scheduler generation number (count of pre-emptions) +#define _COMM_PAGE_SPIN_COUNT (_COMM_PAGE_START_ADDRESS+0x02C) // uint32_t max spin count for mutex's +#define _COMM_PAGE_MEMORY_PRESSURE (_COMM_PAGE_START_ADDRESS+0x030) // uint32_t copy of vm_memory_pressure +#define _COMM_PAGE_ACTIVE_CPUS (_COMM_PAGE_START_ADDRESS+0x034) // uint8_t number of active CPUs (hw.activecpu) +#define _COMM_PAGE_PHYSICAL_CPUS (_COMM_PAGE_START_ADDRESS+0x035) // uint8_t number of physical CPUs (hw.physicalcpu_max) +#define _COMM_PAGE_LOGICAL_CPUS (_COMM_PAGE_START_ADDRESS+0x036) // uint8_t number of logical CPUs (hw.logicalcpu_max) +#define _COMM_PAGE_KERNEL_PAGE_SHIFT (_COMM_PAGE_START_ADDRESS+0x037) // uint8_t kernel vm page shift */ +#define _COMM_PAGE_MEMORY_SIZE (_COMM_PAGE_START_ADDRESS+0x038) // uint64_t max memory size */ +#define _COMM_PAGE_TIMEOFDAY_DATA (_COMM_PAGE_START_ADDRESS+0x040) // used by gettimeofday(). Currently, sizeof(commpage_timeofday_data_t) = 40. A new struct is used on gettimeofday but space is reserved on the commpage for compatibility +#define _COMM_PAGE_CPUFAMILY (_COMM_PAGE_START_ADDRESS+0x080) // used by memcpy() resolver +#define _COMM_PAGE_DEV_FIRM (_COMM_PAGE_START_ADDRESS+0x084) // uint32_t handle on PE_i_can_has_debugger +#define _COMM_PAGE_TIMEBASE_OFFSET (_COMM_PAGE_START_ADDRESS+0x088) // uint64_t timebase offset for constructing mach_absolute_time() +#define _COMM_PAGE_USER_TIMEBASE (_COMM_PAGE_START_ADDRESS+0x090) // uint8_t is userspace mach_absolute_time supported (can read the timebase) +#define _COMM_PAGE_CONT_HWCLOCK (_COMM_PAGE_START_ADDRESS+0x091) // uint8_t is always-on hardware clock present for mach_continuous_time() +#define _COMM_PAGE_UNUSED0 (_COMM_PAGE_START_ADDRESS+0x092) // 6 unused bytes +#define _COMM_PAGE_CONT_TIMEBASE (_COMM_PAGE_START_ADDRESS+0x098) // uint64_t base for mach_continuous_time() +#define _COMM_PAGE_BOOTTIME_USEC (_COMM_PAGE_START_ADDRESS+0x0A0) // uint64_t boottime in microseconds + +// aligning to 64byte for cacheline size +#define _COMM_PAGE_APPROX_TIME (_COMM_PAGE_START_ADDRESS+0x0C0) // uint64_t last known mach_absolute_time() +#define _COMM_PAGE_APPROX_TIME_SUPPORTED (_COMM_PAGE_START_ADDRESS+0x0C8) // uint8_t is mach_approximate_time supported +#define _COMM_PAGE_UNUSED1 (_COMM_PAGE_START_ADDRESS+0x0C9) // 55 unused bytes, align next mutable value to a separate cache line + +#define _COMM_PAGE_KDEBUG_ENABLE (_COMM_PAGE_START_ADDRESS+0x100) // uint32_t export kdebug status bits to userspace +#define _COMM_PAGE_ATM_DIAGNOSTIC_CONFIG (_COMM_PAGE_START_ADDRESS+0x104) // uint32_t export "atm_diagnostic_config" to userspace +#define _COMM_PAGE_MULTIUSER_CONFIG (_COMM_PAGE_START_ADDRESS+0x108) // uint32_t export "multiuser_config" to userspace + + +#define _COMM_PAGE_NEWTIMEOFDAY_DATA (_COMM_PAGE_START_ADDRESS+0x120) // used by gettimeofday(). Currently, sizeof(new_commpage_timeofday_data_t) = 40. +#define _COMM_PAGE_END (_COMM_PAGE_START_ADDRESS+0x1000) // end of common page + +#endif /* _ARM_CPU_CAPABILITIES_H */ +#endif /* PRIVATE */ diff --git a/osfmk/arm/cpu_common.c b/osfmk/arm/cpu_common.c new file mode 100644 index 000000000..ac41435d4 --- /dev/null +++ b/osfmk/arm/cpu_common.c @@ -0,0 +1,577 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * File: arm/cpu_common.c + * + * cpu routines common to all supported arm variants + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if KPERF +void kperf_signal_handler(unsigned int cpu_number); +#endif + +struct processor BootProcessor; + +unsigned int real_ncpus = 1; +boolean_t idle_enable = FALSE; +uint64_t wake_abstime=0x0ULL; + + +cpu_data_t * +cpu_datap(int cpu) +{ + assert(cpu < MAX_CPUS); + return (CpuDataEntries[cpu].cpu_data_vaddr); +} + +kern_return_t +cpu_control(int slot_num, + processor_info_t info, + unsigned int count) +{ + printf("cpu_control(%d,%p,%d) not implemented\n", + slot_num, info, count); + return (KERN_FAILURE); +} + +kern_return_t +cpu_info_count(processor_flavor_t flavor, + unsigned int *count) +{ + + switch (flavor) { + case PROCESSOR_CPU_STAT: + *count = PROCESSOR_CPU_STAT_COUNT; + return (KERN_SUCCESS); + + default: + *count = 0; + return (KERN_FAILURE); + } +} + +kern_return_t +cpu_info(processor_flavor_t flavor, + int slot_num, + processor_info_t info, + unsigned int *count) +{ + switch (flavor) { + case PROCESSOR_CPU_STAT: + { + processor_cpu_stat_t cpu_stat; + cpu_data_t *cpu_data_ptr = CpuDataEntries[slot_num].cpu_data_vaddr; + + if (*count < PROCESSOR_CPU_STAT_COUNT) + return (KERN_FAILURE); + + cpu_stat = (processor_cpu_stat_t) info; + cpu_stat->irq_ex_cnt = cpu_data_ptr->cpu_stat.irq_ex_cnt; + cpu_stat->ipi_cnt = cpu_data_ptr->cpu_stat.ipi_cnt; + cpu_stat->timer_cnt = cpu_data_ptr->cpu_stat.timer_cnt; + cpu_stat->undef_ex_cnt = cpu_data_ptr->cpu_stat.undef_ex_cnt; + cpu_stat->unaligned_cnt = cpu_data_ptr->cpu_stat.unaligned_cnt; + cpu_stat->vfp_cnt = cpu_data_ptr->cpu_stat.vfp_cnt; + cpu_stat->vfp_shortv_cnt = 0; + cpu_stat->data_ex_cnt = cpu_data_ptr->cpu_stat.data_ex_cnt; + cpu_stat->instr_ex_cnt = cpu_data_ptr->cpu_stat.instr_ex_cnt; + + *count = PROCESSOR_CPU_STAT_COUNT; + + return (KERN_SUCCESS); + } + + default: + return (KERN_FAILURE); + } +} + +/* + * Routine: cpu_doshutdown + * Function: + */ +void +cpu_doshutdown(void (*doshutdown) (processor_t), + processor_t processor) +{ + doshutdown(processor); +} + +/* + * Routine: cpu_idle_tickle + * + */ +void +cpu_idle_tickle(void) +{ + boolean_t intr; + cpu_data_t *cpu_data_ptr; + uint64_t new_idle_timeout_ticks = 0x0ULL; + + intr = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + if (cpu_data_ptr->idle_timer_notify != (void *)NULL) { + ((idle_timer_t)cpu_data_ptr->idle_timer_notify)(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks); + if (new_idle_timeout_ticks != 0x0ULL) { + /* if a new idle timeout was requested set the new idle timer deadline */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } else { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } + timer_resync_deadlines(); + } + (void) ml_set_interrupts_enabled(intr); +} + +static void +cpu_handle_xcall(cpu_data_t *cpu_data_ptr) +{ + broadcastFunc xfunc; + void *xparam; + + __c11_atomic_thread_fence(memory_order_acquire_smp); + /* Come back around if cpu_signal_internal is running on another CPU and has just + * added SIGPxcall to the pending mask, but hasn't yet assigned the call params.*/ + if (cpu_data_ptr->cpu_xcall_p0 != NULL && cpu_data_ptr->cpu_xcall_p1 != NULL) { + xfunc = cpu_data_ptr->cpu_xcall_p0; + xparam = cpu_data_ptr->cpu_xcall_p1; + cpu_data_ptr->cpu_xcall_p0 = NULL; + cpu_data_ptr->cpu_xcall_p1 = NULL; + __c11_atomic_thread_fence(memory_order_acq_rel_smp); + hw_atomic_and_noret(&cpu_data_ptr->cpu_signal, ~SIGPxcall); + xfunc(xparam); + } + +} + +unsigned int +cpu_broadcast_xcall(uint32_t *synch, + boolean_t self_xcall, + broadcastFunc func, + void *parm) +{ + boolean_t intr; + cpu_data_t *cpu_data_ptr; + cpu_data_t *target_cpu_datap; + unsigned int failsig; + int cpu; + int max_cpu; + + intr = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + failsig = 0; + + if (synch != NULL) { + *synch = real_ncpus; + assert_wait((event_t)synch, THREAD_UNINT); + } + + max_cpu = ml_get_max_cpu_number(); + for (cpu=0; cpu <= max_cpu; cpu++) { + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) + continue; + + if(KERN_SUCCESS != cpu_signal(target_cpu_datap, SIGPxcall, (void *)func, parm)) { + failsig++; + } + } + + + if (self_xcall) { + func(parm); + } + + (void) ml_set_interrupts_enabled(intr); + + if (synch != NULL) { + if (hw_atomic_sub(synch, (!self_xcall)? failsig+1 : failsig) == 0) + clear_wait(current_thread(), THREAD_AWAKENED); + else + thread_block(THREAD_CONTINUE_NULL); + } + + if (!self_xcall) + return (real_ncpus - failsig - 1); + else + return (real_ncpus - failsig); +} + +kern_return_t +cpu_xcall(int cpu_number, broadcastFunc func, void *param) +{ + cpu_data_t *target_cpu_datap; + + if ((cpu_number < 0) || (cpu_number > ml_get_max_cpu_number())) + return KERN_INVALID_ARGUMENT; + + target_cpu_datap = (cpu_data_t*)CpuDataEntries[cpu_number].cpu_data_vaddr; + if (target_cpu_datap == NULL) + return KERN_INVALID_ARGUMENT; + + return cpu_signal(target_cpu_datap, SIGPxcall, (void*)func, param); +} + +static kern_return_t +cpu_signal_internal(cpu_data_t *target_proc, + unsigned int signal, + void *p0, + void *p1, + boolean_t defer) +{ + unsigned int Check_SIGPdisabled; + int current_signals; + Boolean swap_success; + boolean_t interruptible = ml_set_interrupts_enabled(FALSE); + cpu_data_t *current_proc = getCpuDatap(); + + /* We'll mandate that only IPIs meant to kick a core out of idle may ever be deferred. */ + if (defer) { + assert(signal == SIGPnop); + } + + if (current_proc != target_proc) + Check_SIGPdisabled = SIGPdisabled; + else + Check_SIGPdisabled = 0; + + if (signal == SIGPxcall) { + do { + current_signals = target_proc->cpu_signal; + if ((current_signals & SIGPdisabled) == SIGPdisabled) { +#if DEBUG || DEVELOPMENT + target_proc->failed_signal = SIGPxcall; + target_proc->failed_xcall = p0; + OSIncrementAtomicLong(&target_proc->failed_signal_count); +#endif + ml_set_interrupts_enabled(interruptible); + return KERN_FAILURE; + } + swap_success = OSCompareAndSwap(current_signals & (~SIGPxcall), current_signals | SIGPxcall, + &target_proc->cpu_signal); + + /* Drain pending xcalls on this cpu; the CPU we're trying to xcall may in turn + * be trying to xcall us. Since we have interrupts disabled that can deadlock, + * so break the deadlock by draining pending xcalls. */ + if (!swap_success && (current_proc->cpu_signal & SIGPxcall)) + cpu_handle_xcall(current_proc); + + } while (!swap_success); + + target_proc->cpu_xcall_p0 = p0; + target_proc->cpu_xcall_p1 = p1; + } else { + do { + current_signals = target_proc->cpu_signal; + if ((Check_SIGPdisabled !=0 ) && (current_signals & Check_SIGPdisabled) == SIGPdisabled) { +#if DEBUG || DEVELOPMENT + target_proc->failed_signal = signal; + OSIncrementAtomicLong(&target_proc->failed_signal_count); +#endif + ml_set_interrupts_enabled(interruptible); + return KERN_FAILURE; + } + + swap_success = OSCompareAndSwap(current_signals, current_signals | signal, + &target_proc->cpu_signal); + } while (!swap_success); + } + + /* + * Issue DSB here to guarantee: 1) prior stores to pending signal mask and xcall params + * will be visible to other cores when the IPI is dispatched, and 2) subsequent + * instructions to signal the other cores will not execute until after the barrier. + * DMB would be sufficient to guarantee 1) but not 2). + */ + __builtin_arm_dsb(DSB_ISH); + + if (!(target_proc->cpu_signal & SIGPdisabled)) { + if (defer) { + PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id); + } else { + PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id); + } + } + + ml_set_interrupts_enabled(interruptible); + return (KERN_SUCCESS); +} + +kern_return_t +cpu_signal(cpu_data_t *target_proc, + unsigned int signal, + void *p0, + void *p1) +{ + return cpu_signal_internal(target_proc, signal, p0, p1, FALSE); +} + +kern_return_t +cpu_signal_deferred(cpu_data_t *target_proc) +{ + return cpu_signal_internal(target_proc, SIGPnop, NULL, NULL, TRUE); +} + +void +cpu_signal_cancel(cpu_data_t *target_proc) +{ + /* TODO: Should we care about the state of a core as far as squashing deferred IPIs goes? */ + if (!(target_proc->cpu_signal & SIGPdisabled)) { + PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id); + } +} + +void +cpu_signal_handler(void) +{ + cpu_signal_handler_internal(FALSE); +} + +void +cpu_signal_handler_internal(boolean_t disable_signal) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + unsigned int cpu_signal; + + + cpu_data_ptr->cpu_stat.ipi_cnt++; + cpu_data_ptr->cpu_stat.ipi_cnt_wake++; + + SCHED_STATS_IPI(current_processor()); + + cpu_signal = hw_atomic_or(&cpu_data_ptr->cpu_signal, 0); + + if ((!(cpu_signal & SIGPdisabled)) && (disable_signal == TRUE)) + (void)hw_atomic_or(&cpu_data_ptr->cpu_signal, SIGPdisabled); + else if ((cpu_signal & SIGPdisabled) && (disable_signal == FALSE)) + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPdisabled); + + while (cpu_signal & ~SIGPdisabled) { + if (cpu_signal & SIGPdec) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPdec); + rtclock_intr(FALSE); + } + if (cpu_signal & SIGPchud) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPchud); + chudxnu_cpu_signal_handler(); + } +#if KPERF + if (cpu_signal & SIGPkptimer) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPkptimer); + kperf_signal_handler((unsigned int)cpu_data_ptr->cpu_number); + } +#endif + if (cpu_signal & SIGPxcall) { + cpu_handle_xcall(cpu_data_ptr); + } + if (cpu_signal & SIGPast) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPast); + ast_check(cpu_data_ptr->cpu_processor); + } + if (cpu_signal & SIGPdebug) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPdebug); + DebuggerXCall(cpu_data_ptr->cpu_int_state); + } +#if __ARM_SMP__ && defined(ARMA7) + if (cpu_signal & SIGPLWFlush) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPLWFlush); + cache_xcall_handler(LWFlush); + } + if (cpu_signal & SIGPLWClean) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPLWClean); + cache_xcall_handler(LWClean); + } +#endif + + cpu_signal = hw_atomic_or(&cpu_data_ptr->cpu_signal, 0); + } +} + +void +cpu_exit_wait(int cpu) +{ + if ( cpu != master_cpu) { + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr; + while (!((*(volatile unsigned int*)&cpu_data_ptr->cpu_sleep_token) == ARM_CPU_ON_SLEEP_PATH)) {}; + } +} + +void +cpu_machine_init(void) +{ + static boolean_t started = FALSE; + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = getCpuDatap(); + started = ((cpu_data_ptr->cpu_flags & StartedState) == StartedState); + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + platform_cache_init(); + PE_cpu_machine_init(cpu_data_ptr->cpu_id, !started); + cpu_data_ptr->cpu_flags |= StartedState; + ml_init_interrupt(); +} + +processor_t +cpu_processor_alloc(boolean_t is_boot_cpu) +{ + processor_t proc; + + if (is_boot_cpu) + return &BootProcessor; + + proc = kalloc(sizeof(*proc)); + if (!proc) + return NULL; + + bzero((void *) proc, sizeof(*proc)); + return proc; +} + +void +cpu_processor_free(processor_t proc) +{ + if (proc != NULL && proc != &BootProcessor) + kfree((void *) proc, sizeof(*proc)); +} + +processor_t +current_processor(void) +{ + return getCpuDatap()->cpu_processor; +} + +processor_t +cpu_to_processor(int cpu) +{ + cpu_data_t *cpu_data = cpu_datap(cpu); + if (cpu_data != NULL) + return cpu_data->cpu_processor; + else + return NULL; +} + +cpu_data_t * +processor_to_cpu_datap(processor_t processor) +{ + cpu_data_t *target_cpu_datap; + + assert(processor->cpu_id < MAX_CPUS); + assert(CpuDataEntries[processor->cpu_id].cpu_data_vaddr != NULL); + + target_cpu_datap = (cpu_data_t*)CpuDataEntries[processor->cpu_id].cpu_data_vaddr; + assert(target_cpu_datap->cpu_processor == processor); + + return target_cpu_datap; +} + +ast_t * +ast_pending(void) +{ + return (&getCpuDatap()->cpu_pending_ast); +} + +cpu_type_t +slot_type(int slot_num) +{ + return (cpu_datap(slot_num)->cpu_type); +} + +cpu_subtype_t +slot_subtype(int slot_num) +{ + return (cpu_datap(slot_num)->cpu_subtype); +} + +cpu_threadtype_t +slot_threadtype(int slot_num) +{ + return (cpu_datap(slot_num)->cpu_threadtype); +} + +cpu_type_t +cpu_type(void) +{ + return (getCpuDatap()->cpu_type); +} + +cpu_subtype_t +cpu_subtype(void) +{ + return (getCpuDatap()->cpu_subtype); +} + +cpu_threadtype_t +cpu_threadtype(void) +{ + return (getCpuDatap()->cpu_threadtype); +} + +int +cpu_number(void) +{ + return (getCpuDatap()->cpu_number); +} + +uint64_t +ml_get_wake_timebase(void) +{ + return wake_abstime; +} + diff --git a/osfmk/arm/cpu_data.h b/osfmk/arm/cpu_data.h new file mode 100644 index 000000000..f35121e35 --- /dev/null +++ b/osfmk/arm/cpu_data.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + * + */ + +#ifndef ARM_CPU_DATA +#define ARM_CPU_DATA + +#ifdef MACH_KERNEL_PRIVATE + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +#define current_thread() current_thread_fast() + +static inline thread_t current_thread_fast(void) +{ + thread_t result; +#if defined(__arm64__) + __asm__ volatile("mrs %0, TPIDR_EL1" : "=r" (result)); +#else + result = (thread_t)__builtin_arm_mrc(15, 0, 13, 0, 4); // TPIDRPRW +#endif + return result; +} + +#if defined(__arm64__) + +static inline vm_offset_t exception_stack_pointer(void) +{ + vm_offset_t result = 0; + __asm__ volatile( + "msr SPSel, #1 \n" + "mov %0, sp \n" + "msr SPSel, #0 \n" + : "=r" (result)); + + return result; +} + +#endif /* defined(__arm64__) */ + +#define getCpuDatap() current_thread()->machine.CpuDatap +#define current_cpu_datap() getCpuDatap() + +extern int get_preemption_level(void); +extern void _enable_preemption_no_check(void); + +#define enable_preemption_no_check() _enable_preemption_no_check() +#define mp_disable_preemption() _disable_preemption() +#define mp_enable_preemption() _enable_preemption() +#define mp_enable_preemption_no_check() _enable_preemption_no_check() + +#endif /* MACH_KERNEL_PRIVATE */ + +#endif /* ARM_CPU_DATA */ diff --git a/osfmk/arm/cpu_data_internal.h b/osfmk/arm/cpu_data_internal.h new file mode 100644 index 000000000..264e7ed96 --- /dev/null +++ b/osfmk/arm/cpu_data_internal.h @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + * + */ + +#ifndef ARM_CPU_DATA_INTERNAL +#define ARM_CPU_DATA_INTERNAL + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if MONOTONIC +#include +#endif /* MONOTONIC */ + +#define NSEC_PER_HZ (NSEC_PER_SEC / 100) + +typedef struct reset_handler_data { + vm_offset_t assist_reset_handler; /* Assist handler phys address */ + vm_offset_t cpu_data_entries; /* CpuDataEntries phys address */ +#if !__arm64__ + vm_offset_t boot_args; /* BootArgs phys address */ +#endif +} reset_handler_data_t; + +extern reset_handler_data_t ResetHandlerData; + +#if __ARM_SMP__ +#ifdef CPU_COUNT +#define MAX_CPUS CPU_COUNT +#else +#define MAX_CPUS 2 +#endif +#else +#define MAX_CPUS 1 +#endif + +#define CPUWINDOWS_MAX 4 +#ifdef __arm__ +#define CPUWINDOWS_BASE 0xFFF00000UL +#else +#define CPUWINDOWS_BASE_MASK 0xFFFFFFFFFFF00000UL +#define CPUWINDOWS_BASE (VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) +#endif + +typedef struct cpu_data_entry { + void *cpu_data_paddr; /* Cpu data physical address */ + struct cpu_data *cpu_data_vaddr; /* Cpu data virtual address */ +#if __arm__ + uint32_t cpu_data_offset_8; + uint32_t cpu_data_offset_12; +#elif __arm64__ +#else +#error Check cpu_data_entry padding for this architecture +#endif +} cpu_data_entry_t; + + +typedef struct rtclock_timer { + mpqueue_head_t queue; + uint64_t deadline; + uint32_t is_set:1, + has_expired:1, + :0; +} rtclock_timer_t; + +typedef struct { + uint32_t irq_ex_cnt; + uint32_t irq_ex_cnt_wake; + uint32_t ipi_cnt; + uint32_t ipi_cnt_wake; + uint32_t timer_cnt; + uint32_t timer_cnt_wake; + uint32_t undef_ex_cnt; + uint32_t unaligned_cnt; + uint32_t vfp_cnt; + uint32_t data_ex_cnt; + uint32_t instr_ex_cnt; +} cpu_stat_t; + +typedef struct cpu_data +{ + unsigned short cpu_number; + unsigned short cpu_flags; + vm_offset_t istackptr; + vm_offset_t intstack_top; + vm_offset_t fiqstackptr; + vm_offset_t fiqstack_top; +#if __arm64__ + vm_offset_t excepstackptr; + vm_offset_t excepstack_top; + boolean_t cluster_master; +#endif + boolean_t interrupts_enabled; + thread_t cpu_active_thread; + vm_offset_t cpu_active_stack; + unsigned int cpu_ident; + cpu_id_t cpu_id; + unsigned volatile int cpu_signal; +#if DEBUG || DEVELOPMENT + void *failed_xcall; + unsigned int failed_signal; + volatile long failed_signal_count; +#endif + void *cpu_cache_dispatch; + ast_t cpu_pending_ast; + struct processor *cpu_processor; + int cpu_type; + int cpu_subtype; + int cpu_threadtype; + int cpu_running; + +#ifdef __LP64__ + uint64_t cpu_base_timebase; + uint64_t cpu_timebase; +#else + union { + struct { + uint32_t low; + uint32_t high; + } split; + struct { + uint64_t val; + } raw; + } cbtb; +#define cpu_base_timebase_low cbtb.split.low +#define cpu_base_timebase_high cbtb.split.high + + union { + struct { + uint32_t low; + uint32_t high; + } split; + struct { + uint64_t val; + } raw; + } ctb; +#define cpu_timebase_low ctb.split.low +#define cpu_timebase_high ctb.split.high +#endif + + uint32_t cpu_decrementer; + void *cpu_get_decrementer_func; + void *cpu_set_decrementer_func; + void *cpu_get_fiq_handler; + + void *cpu_tbd_hardware_addr; + void *cpu_tbd_hardware_val; + + void *cpu_console_buf; + void *cpu_chud; + + void *cpu_idle_notify; + uint64_t cpu_idle_latency; + uint64_t cpu_idle_pop; + +#if __arm__ + vm_offset_t cpu_exc_vectors; +#endif + vm_offset_t cpu_reset_handler; + uint32_t cpu_reset_type; + uintptr_t cpu_reset_assist; + + void *cpu_int_state; + IOInterruptHandler interrupt_handler; + void *interrupt_nub; + unsigned int interrupt_source; + void *interrupt_target; + void *interrupt_refCon; + + void *idle_timer_notify; + void *idle_timer_refcon; + uint64_t idle_timer_deadline; + + uint64_t quantum_timer_deadline; + uint64_t rtcPop; + rtclock_timer_t rtclock_timer; + struct _rtclock_data_ *rtclock_datap; + + arm_debug_state_t *cpu_user_debug; /* Current debug state */ + vm_offset_t cpu_debug_interface_map; + + volatile int debugger_active; + + void *cpu_xcall_p0; + void *cpu_xcall_p1; + +#if __ARM_SMP__ && defined(ARMA7) + volatile uint32_t cpu_CLW_active; + volatile uint64_t cpu_CLWFlush_req; + volatile uint64_t cpu_CLWFlush_last; + volatile uint64_t cpu_CLWClean_req; + volatile uint64_t cpu_CLWClean_last; +#endif + + +#if __arm64__ + vm_offset_t coresight_base[CORESIGHT_REGIONS]; +#endif + + /* CCC ARMv8 registers */ + uint64_t cpu_regmap_paddr; + + uint32_t cpu_phys_id; + uint32_t cpu_l2_access_penalty; + void *platform_error_handler; + + int cpu_mcount_off; + + #define ARM_CPU_ON_SLEEP_PATH 0x50535553UL + volatile unsigned int cpu_sleep_token; + unsigned int cpu_sleep_token_last; + + cpu_stat_t cpu_stat; + + volatile int PAB_active; /* Tells the console if we are dumping backtraces */ + +#if KPC + /* double-buffered performance counter data */ + uint64_t *cpu_kpc_buf[2]; + /* PMC shadow and reload value buffers */ + uint64_t *cpu_kpc_shadow; + uint64_t *cpu_kpc_reload; +#endif +#if MONOTONIC + struct mt_cpu cpu_monotonic; +#endif /* MONOTONIC */ + struct prngContext *cpu_prng; + cluster_type_t cpu_cluster_type; + uint32_t cpu_cluster_id; + uint32_t cpu_l2_id; + uint32_t cpu_l2_size; + uint32_t cpu_l3_id; + uint32_t cpu_l3_size; + + struct pmap_cpu_data cpu_pmap_cpu_data; + dbgwrap_thread_state_t halt_state; + enum { + CPU_NOT_HALTED = 0, + CPU_HALTED, + CPU_HALTED_WITH_STATE + } halt_status; +} cpu_data_t; + +/* + * cpu_flags + */ +#define SleepState 0x0800 +#define StartedState 0x1000 + +extern cpu_data_entry_t CpuDataEntries[MAX_CPUS]; +extern cpu_data_t BootCpuData; +extern boot_args *BootArgs; + +#if __arm__ +extern unsigned int *ExceptionLowVectorsBase; +extern unsigned int *ExceptionVectorsTable; +#elif __arm64__ +extern unsigned int LowResetVectorBase; +extern unsigned int LowResetVectorEnd; +#if WITH_CLASSIC_S2R +extern uint8_t SleepToken[8]; +#endif +extern unsigned int LowExceptionVectorBase; +#else +#error Unknown arch +#endif + +extern cpu_data_t *cpu_datap(int cpu); +extern cpu_data_t *cpu_data_alloc(boolean_t is_boot); +extern void cpu_data_init(cpu_data_t *cpu_data_ptr); +extern void cpu_data_free(cpu_data_t *cpu_data_ptr); +extern kern_return_t cpu_data_register(cpu_data_t *cpu_data_ptr); +extern cpu_data_t *processor_to_cpu_datap( processor_t processor); + +#if __arm64__ +typedef struct sysreg_restore +{ + uint64_t tcr_el1; +} sysreg_restore_t; + +extern sysreg_restore_t sysreg_restore; +#endif /* __arm64__ */ + +#endif /* ARM_CPU_DATA_INTERNAL */ diff --git a/osfmk/arm/cpu_internal.h b/osfmk/arm/cpu_internal.h new file mode 100644 index 000000000..227863b4c --- /dev/null +++ b/osfmk/arm/cpu_internal.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +#ifndef _ARM_CPU_INTERNAL_H_ +#define _ARM_CPU_INTERNAL_H_ + + +#include +#include + +extern void cpu_bootstrap( + void); + +extern void cpu_init( + void); + +extern void cpu_timebase_init(boolean_t from_boot); + +extern kern_return_t cpu_signal( + cpu_data_t *target, + unsigned int signal, + void *p0, + void *p1); + +extern kern_return_t cpu_signal_deferred( + cpu_data_t *target); + +extern void cpu_signal_cancel( + cpu_data_t *target); + +#define SIGPnop 0x00000000U /* Send IPI with no service */ +#define SIGPdec 0x00000001U /* Request decremeter service */ +#define SIGPchud 0x00000002U /* CHUD CPU Signal request types */ +#define SIGPxcall 0x00000004U /* Call a function on a processor */ +#define SIGPast 0x00000008U /* Request AST check */ +#define SIGPdebug 0x00000010U /* Request Debug call */ +#define SIGPLWFlush 0x00000020UL /* Request LWFlush call */ +#define SIGPLWClean 0x00000040UL /* Request LWClean call */ +#define SIGPkptimer 0x00000100U /* Request kperf timer */ + +#define SIGPdisabled 0x80000000U /* Signal disabled */ + +extern void * chudxnu_cpu_alloc( + boolean_t boot_processor); + +extern void chudxnu_cpu_free( + void *per_proc_chud); + +extern unsigned int real_ncpus; + + +#endif /* _ARM_CPU_INTERNAL_H_ */ diff --git a/osfmk/arm/cpu_number.h b/osfmk/arm/cpu_number.h new file mode 100644 index 000000000..d781fa067 --- /dev/null +++ b/osfmk/arm/cpu_number.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +/* + */ + +/* + * Machine-dependent definitions for cpu identification. + * + */ +#ifdef KERNEL_PRIVATE + +#ifndef _ARM_CPU_NUMBER_H_ +#define _ARM_CPU_NUMBER_H_ + +#include + +__BEGIN_DECLS + +extern int cpu_number(void); +extern int cpu_cluster_id(void); + +__END_DECLS + +#endif /* _ARM_CPU_NUMBER_H_ */ + +#endif /* KERNEL_PRIVATE */ diff --git a/osfmk/arm/cpuid.c b/osfmk/arm/cpuid.c new file mode 100644 index 000000000..2782475e2 --- /dev/null +++ b/osfmk/arm/cpuid.c @@ -0,0 +1,314 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#include +#include +#include +#include +#include "proc_reg.h" + +#include + +/* Temporary types to aid decoding, + * Everything in Little Endian */ + +typedef struct { + uint32_t + + Ctype1:3, /* 2:0 */ + Ctype2:3, /* 5:3 */ + Ctype3:3, /* 8:6 */ + Ctypes:15, /* 6:23 - Don't Care */ + LoC:3, /* 26-24 - Level of Coherency */ + LoU:3, /* 29:27 - Level of Unification */ + RAZ:2; /* 31:30 - Read-As-Zero */ +} arm_cache_clidr_t; + +typedef union { + arm_cache_clidr_t bits; + uint32_t value; +} arm_cache_clidr_info_t; + + +typedef struct { + uint32_t + + LineSize:3, /* 2:0 - Number of words in cache line */ + Assoc:10, /* 12:3 - Associativity of cache */ + NumSets:15, /* 27:13 - Number of sets in cache */ + c_type:4; /* 31:28 - Cache type */ +} arm_cache_ccsidr_t; + + +typedef union { + arm_cache_ccsidr_t bits; + uint32_t value; +} arm_cache_ccsidr_info_t; + +/* Statics */ + +static SECURITY_READ_ONLY_LATE(arm_cpu_info_t) cpuid_cpu_info; +static SECURITY_READ_ONLY_LATE(cache_info_t) cpuid_cache_info; + +/* Code */ + +__private_extern__ +void +do_cpuid(void) +{ + cpuid_cpu_info.value = machine_read_midr(); +#if (__ARM_ARCH__ == 8) + + cpuid_cpu_info.arm_info.arm_arch = CPU_ARCH_ARMv8; + +#elif (__ARM_ARCH__ == 7) + #ifdef __ARM_SUB_ARCH__ + cpuid_cpu_info.arm_info.arm_arch = __ARM_SUB_ARCH__; + #else + cpuid_cpu_info.arm_info.arm_arch = CPU_ARCH_ARMv7; + #endif +#else + /* 1176 architecture lives in the extended feature register */ + if (cpuid_cpu_info.arm_info.arm_arch == CPU_ARCH_EXTENDED) { + arm_isa_feat1_reg isa = machine_read_isa_feat1(); + + /* + * if isa feature register 1 [15:12] == 0x2, this chip + * supports sign extention instructions, which indicate ARMv6 + */ + if (isa.field.sign_zero_ext_support == 0x2) { + cpuid_cpu_info.arm_info.arm_arch = CPU_ARCH_ARMv6; + } + } +#endif +} + +arm_cpu_info_t * +cpuid_info(void) +{ + return &cpuid_cpu_info; +} + +int +cpuid_get_cpufamily(void) +{ + int cpufamily = 0; + + switch (cpuid_info()->arm_info.arm_implementor) { + case CPU_VID_ARM: + switch (cpuid_info()->arm_info.arm_part) { + case CPU_PART_CORTEXA9: + cpufamily = CPUFAMILY_ARM_14; + break; + case CPU_PART_CORTEXA8: + cpufamily = CPUFAMILY_ARM_13; + break; + case CPU_PART_CORTEXA7: + cpufamily = CPUFAMILY_ARM_15; + break; + case CPU_PART_1136JFS: + case CPU_PART_1176JZFS: + cpufamily = CPUFAMILY_ARM_11; + break; + case CPU_PART_926EJS: + case CPU_PART_920T: + cpufamily = CPUFAMILY_ARM_9; + break; + default: + cpufamily = CPUFAMILY_UNKNOWN; + break; + } + break; + + case CPU_VID_INTEL: + cpufamily = CPUFAMILY_ARM_XSCALE; + break; + + case CPU_VID_APPLE: + switch (cpuid_info()->arm_info.arm_part) { + case CPU_PART_SWIFT: + cpufamily = CPUFAMILY_ARM_SWIFT; + break; + case CPU_PART_CYCLONE: + cpufamily = CPUFAMILY_ARM_CYCLONE; + break; + case CPU_PART_TYPHOON: + case CPU_PART_TYPHOON_CAPRI: + cpufamily = CPUFAMILY_ARM_TYPHOON; + break; + case CPU_PART_TWISTER: + case CPU_PART_TWISTER_ELBA_MALTA: + cpufamily = CPUFAMILY_ARM_TWISTER; + break; + case CPU_PART_HURRICANE: + case CPU_PART_HURRICANE_MYST: + cpufamily = CPUFAMILY_ARM_HURRICANE; + break; + default: + cpufamily = CPUFAMILY_UNKNOWN; + break; + } + break; + + default: + cpufamily = CPUFAMILY_UNKNOWN; + break; + } + + return cpufamily; +} + +void +do_debugid(void) +{ + machine_do_debugid(); +} + +arm_debug_info_t * +arm_debug_info(void) +{ + return machine_arm_debug_info(); +} + +void +do_mvfpid(void) +{ + return machine_do_mvfpid(); +} + +arm_mvfp_info_t +*arm_mvfp_info(void) +{ + return machine_arm_mvfp_info(); +} + +void +do_cacheid(void) +{ + arm_cache_clidr_info_t arm_cache_clidr_info; + arm_cache_ccsidr_info_t arm_cache_ccsidr_info; + + arm_cache_clidr_info.value = machine_read_clidr(); + + + /* Select L1 data/unified cache */ + + machine_write_csselr(CSSELR_L1, CSSELR_DATA_UNIFIED); + arm_cache_ccsidr_info.value = machine_read_ccsidr(); + + cpuid_cache_info.c_unified = (arm_cache_clidr_info.bits.Ctype1 == 0x4) ? 1 : 0; + + switch (arm_cache_ccsidr_info.bits.c_type) { + case 0x1: + cpuid_cache_info.c_type = CACHE_WRITE_ALLOCATION; + break; + case 0x2: + cpuid_cache_info.c_type = CACHE_READ_ALLOCATION; + break; + case 0x4: + cpuid_cache_info.c_type = CACHE_WRITE_BACK; + break; + case 0x8: + cpuid_cache_info.c_type = CACHE_WRITE_THROUGH; + break; + default: + cpuid_cache_info.c_type = CACHE_UNKNOWN; + } + + cpuid_cache_info.c_linesz = 4 * (1<<(arm_cache_ccsidr_info.bits.LineSize + 2)); + cpuid_cache_info.c_assoc = (arm_cache_ccsidr_info.bits.Assoc + 1); + + /* I cache size */ + cpuid_cache_info.c_isize = (arm_cache_ccsidr_info.bits.NumSets + 1) * cpuid_cache_info.c_linesz * cpuid_cache_info.c_assoc; + + /* D cache size */ + cpuid_cache_info.c_dsize = (arm_cache_ccsidr_info.bits.NumSets + 1) * cpuid_cache_info.c_linesz * cpuid_cache_info.c_assoc; + + + if ((arm_cache_clidr_info.bits.Ctype3 == 0x4) || + (arm_cache_clidr_info.bits.Ctype2 == 0x4) || (arm_cache_clidr_info.bits.Ctype2 == 0x2)) { + + if (arm_cache_clidr_info.bits.Ctype3 == 0x4) { + /* Select L3 (LLC) if the SoC is new enough to have that. + * This will be the second-level cache for the highest-performing ACC. */ + machine_write_csselr(CSSELR_L3, CSSELR_DATA_UNIFIED); + } else { + /* Select L2 data cache */ + machine_write_csselr(CSSELR_L2, CSSELR_DATA_UNIFIED); + } + arm_cache_ccsidr_info.value = machine_read_ccsidr(); + + cpuid_cache_info.c_linesz = 4 * (1<<(arm_cache_ccsidr_info.bits.LineSize + 2)); + cpuid_cache_info.c_assoc = (arm_cache_ccsidr_info.bits.Assoc + 1); + cpuid_cache_info.c_l2size = (arm_cache_ccsidr_info.bits.NumSets + 1) * cpuid_cache_info.c_linesz * cpuid_cache_info.c_assoc; + cpuid_cache_info.c_inner_cache_size = cpuid_cache_info.c_dsize; + cpuid_cache_info.c_bulksize_op = cpuid_cache_info.c_l2size; + + /* capri has a 2MB L2 cache unlike every other SoC up to this + * point with a 1MB L2 cache, so to get the same performance + * gain from coloring, we have to double the number of colors. + * Note that in general (and in fact as it's implemented in + * i386/cpuid.c), the number of colors is calculated as the + * cache line size * the number of sets divided by the page + * size. Also note that for H8 devices and up, the page size + * will be 16k instead of 4, which will reduce the number of + * colors required. Thus, this is really a temporary solution + * for capri specifically that we may want to generalize later: + * + * TODO: Are there any special considerations for our unusual + * cache geometries (3MB)? + */ + vm_cache_geometry_colors = ((arm_cache_ccsidr_info.bits.NumSets + 1) * cpuid_cache_info.c_linesz) / PAGE_SIZE; + kprintf(" vm_cache_geometry_colors: %d\n", vm_cache_geometry_colors); + } else { + cpuid_cache_info.c_l2size = 0; + + cpuid_cache_info.c_inner_cache_size = cpuid_cache_info.c_dsize; + cpuid_cache_info.c_bulksize_op = cpuid_cache_info.c_dsize; + } + + kprintf("%s() - %u bytes %s cache (I:%u D:%u (%s)), %u-way assoc, %u bytes/line\n", + __FUNCTION__, + cpuid_cache_info.c_dsize + cpuid_cache_info.c_isize, + ((cpuid_cache_info.c_type == CACHE_WRITE_BACK) ? "WB" : + (cpuid_cache_info.c_type == CACHE_WRITE_THROUGH ? "WT" : "Unknown")), + cpuid_cache_info.c_isize, + cpuid_cache_info.c_dsize, + (cpuid_cache_info.c_unified) ? "unified" : "separate", + cpuid_cache_info.c_assoc, + cpuid_cache_info.c_linesz); +} + +cache_info_t * +cache_info(void) +{ + return &cpuid_cache_info; +} diff --git a/osfmk/arm/cpuid.h b/osfmk/arm/cpuid.h new file mode 100644 index 000000000..07778404b --- /dev/null +++ b/osfmk/arm/cpuid.h @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* + * ARM CPU identification + */ + +#ifndef _MACHINE_CPUID_H_ +#define _MACHINE_CPUID_H_ + +#include +#include +#include + +typedef struct { +uint32_t arm_rev : 4, /* 00:03 revision number */ + arm_part : 12, /* 04:15 primary part number */ + arm_arch : 4, /* 16:19 architecture */ + arm_variant : 4, /* 20:23 variant */ + arm_implementor : 8; /* 24:31 implementor (0x41) */ +} arm_cpuid_bits_t; + +typedef union { + arm_cpuid_bits_t arm_info; /* ARM9xx, ARM11xx, and later processors */ + uint32_t value; +} arm_cpu_info_t; + +/* Implementor codes */ +#define CPU_VID_ARM 0x41 // ARM Limited +#define CPU_VID_DEC 0x44 // Digital Equipment Corporation +#define CPU_VID_MOTOROLA 0x4D // Motorola - Freescale Semiconductor Inc. +#define CPU_VID_MARVELL 0x56 // Marvell Semiconductor Inc. +#define CPU_VID_INTEL 0x69 // Intel ARM parts. +#define CPU_VID_APPLE 0x61 // Apple Inc. + + +/* ARM Architecture Codes */ + +#define CPU_ARCH_ARMv4 0x1 /* ARMv4 */ +#define CPU_ARCH_ARMv4T 0x2 /* ARMv4 + Thumb */ +#define CPU_ARCH_ARMv5 0x3 /* ARMv5 */ +#define CPU_ARCH_ARMv5T 0x4 /* ARMv5 + Thumb */ +#define CPU_ARCH_ARMv5TE 0x5 /* ARMv5 + Thumb + Extensions(?) */ +#define CPU_ARCH_ARMv5TEJ 0x6 /* ARMv5 + Thumb + Extensions(?) + //Jazelle(?) XXX */ +#define CPU_ARCH_ARMv6 0x7 /* ARMv6 */ +#define CPU_ARCH_ARMv7 0x8 /* ARMv7 */ +#define CPU_ARCH_ARMv7f 0x9 /* ARMv7 for Cortex A9 */ +#define CPU_ARCH_ARMv7s 0xa /* ARMv7 for Swift */ +#define CPU_ARCH_ARMv7k 0xb /* ARMv7 for Cortex A7 */ + +#define CPU_ARCH_ARMv8 0xc /* Subtype for CPU_TYPE_ARM64 */ + + +/* special code indicating we need to look somewhere else for the architecture version */ +#define CPU_ARCH_EXTENDED 0xF + +/* ARM Part Numbers */ +/* + * XXX: ARM Todo + * Fill out these part numbers more completely + */ + +/* ARM9 (ARMv4T architecture) */ +#define CPU_PART_920T 0x920 +#define CPU_PART_926EJS 0x926 /* ARM926EJ-S */ + +/* ARM11 (ARMv6 architecture) */ +#define CPU_PART_1136JFS 0xB36 /* ARM1136JF-S or ARM1136J-S */ +#define CPU_PART_1176JZFS 0xB76 /* ARM1176JZF-S */ + +/* G1 (ARMv7 architecture) */ +#define CPU_PART_CORTEXA5 0xC05 + +/* M7 (ARMv7 architecture) */ +#define CPU_PART_CORTEXA7 0xC07 + +/* H2 H3 (ARMv7 architecture) */ +#define CPU_PART_CORTEXA8 0xC08 + +/* H4 (ARMv7 architecture) */ +#define CPU_PART_CORTEXA9 0xC09 + +/* H5 (SWIFT architecture) */ +#define CPU_PART_SWIFT 0x0 + +/* H6 (ARMv8 architecture) */ +#define CPU_PART_CYCLONE 0x1 + +/* H7 (ARMv8 architecture) */ +#define CPU_PART_TYPHOON 0x2 + +/* H7G (ARMv8 architecture) */ +#define CPU_PART_TYPHOON_CAPRI 0x3 + +/* H8 (ARMv8 architecture) */ +#define CPU_PART_TWISTER 0x4 + +/* H8G H8M (ARMv8 architecture) */ +#define CPU_PART_TWISTER_ELBA_MALTA 0x5 + +/* H9 (ARMv8 architecture) */ +#define CPU_PART_HURRICANE 0x6 + +/* H9G (ARMv8 architecture) */ +#define CPU_PART_HURRICANE_MYST 0x7 + + +/* Cache type identification */ + +/* Supported Cache Types */ +typedef enum { + CACHE_WRITE_THROUGH, + CACHE_WRITE_BACK, + CACHE_READ_ALLOCATION, + CACHE_WRITE_ALLOCATION, + CACHE_UNKNOWN +} cache_type_t; + +typedef struct { + boolean_t c_unified; /* unified I & D cache? */ + uint32_t c_isize; /* in Bytes (ARM caches can be 0.5 KB) */ + boolean_t c_i_ppage; /* protected page restriction for I cache + * (see B6-11 in ARM DDI 0100I document). */ + uint32_t c_dsize; /* in Bytes (ARM caches can be 0.5 KB) */ + boolean_t c_d_ppage; /* protected page restriction for I cache + * (see B6-11 in ARM DDI 0100I document). */ + cache_type_t c_type; /* WB or WT */ + uint32_t c_linesz; /* number of bytes */ + uint32_t c_assoc; /* n-way associativity */ + uint32_t c_l2size; /* L2 size, if present */ + uint32_t c_bulksize_op; /* bulk operation size limit. 0 if disabled */ + uint32_t c_inner_cache_size; /* inner dache size */ +} cache_info_t; + +typedef struct { + uint32_t + + RB:4, /* 3:0 - 32x64-bit media register bank supported: 0x2 */ + SP:4, /* 7:4 - Single precision supported in VFPv3: 0x2 */ + DP:4, /* 8:11 - Double precision supported in VFPv3: 0x2 */ + TE:4, /* 12-15 - Only untrapped exception handling can be selected: 0x0 */ + D:4, /* 19:16 - VFP hardware divide supported: 0x1 */ + SR:4, /* 23:20 - VFP hardware square root supported: 0x1 */ + SV:4, /* 27:24 - VFP short vector supported: 0x1 */ + RM:4; /* 31:28 - All VFP rounding modes supported: 0x1 */ +} arm_mvfr0_t; + +typedef union { + arm_mvfr0_t bits; + uint32_t value; +} arm_mvfr0_info_t; + +typedef struct { + uint32_t + + FZ:4, /* 3:0 - Full denormal arithmetic supported for VFP: 0x1 */ + DN:4, /* 7:4 - Propagation of NaN values supported for VFP: 0x1 */ + LS:4, /* 11:8 - Load/store instructions supported for NEON: 0x1 */ + I:4, /* 15:12 - Integer instructions supported for NEON: 0x1 */ + SP:4, /* 19:16 - Single precision floating-point instructions supported for NEON: 0x1 */ + HPFP:4, /* 23:20 - Half precision floating-point instructions supported */ + RSVP:8; /* 31:24 - Reserved */ +} arm_mvfr1_t; + +typedef union { + arm_mvfr1_t bits; + uint32_t value; +} arm_mvfr1_info_t; + +typedef struct { + uint32_t neon; + uint32_t neon_hpfp; +} arm_mvfp_info_t; + +#ifdef __cplusplus +extern "C" { +#endif + +extern void do_cpuid(void); +extern arm_cpu_info_t *cpuid_info(void); +extern int cpuid_get_cpufamily(void); + +extern void do_debugid(void); +extern arm_debug_info_t *arm_debug_info(void); + +extern void do_cacheid(void); +extern cache_info_t *cache_info(void); + +extern void do_mvfpid(void); +extern arm_mvfp_info_t *arm_mvfp_info(void); + +#ifdef __cplusplus +} +#endif + +#endif // _MACHINE_CPUID_H_ diff --git a/osfmk/arm/cpuid_internal.h b/osfmk/arm/cpuid_internal.h new file mode 100644 index 000000000..9778d117b --- /dev/null +++ b/osfmk/arm/cpuid_internal.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2011 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _ARM_CPUID_INTERNAL_H_ +#define _ARM_CPUID_INTERNAL_H_ + +void machine_do_debugid(void); +arm_debug_info_t *machine_arm_debug_info(void); + +void machine_do_mvfpid(void); +arm_mvfp_info_t *machine_arm_mvfp_info(void); + +uint32_t machine_read_midr(void); +uint32_t machine_read_clidr(void); +uint32_t machine_read_ccsidr(void); + +typedef enum { + CSSELR_L1 = 0x0, + CSSELR_L2 = 0x2, + CSSELR_L3 = 0x4 +} csselr_cache_level; + +typedef enum { + CSSELR_DATA_UNIFIED = 0x0, + CSSELR_INSTR = 0x1 +} csselr_cache_type; + +void machine_write_csselr(csselr_cache_level level, csselr_cache_type type); + +#endif /* _ARM_CPUID_INTERNAL_H_ */ diff --git a/osfmk/arm/cswitch.s b/osfmk/arm/cswitch.s new file mode 100644 index 000000000..7c3812dd0 --- /dev/null +++ b/osfmk/arm/cswitch.s @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include "assym.s" + +/* + * save_vfp_registers + * + * Expects a pointer to the VFP save area in r3; saves the callee-saved registers to that save area. + * Clobbers r2 and r3. + */ +.macro save_vfp_registers +#if __ARM_VFP__ + fmrx r2, fpscr // Get the current FPSCR... + str r2, [r3, VSS_FPSCR] // ...and save it to the save area + add r3, r3, #64 // Only s16-s31 are callee-saved +#if (__ARM_VFP__ >= 3) + vstmia.64 r3!, {d8-d11} + vstmia.64 r3!, {d12-d15} +#else + fstmias r3!, {s16-s31} +#endif /* __ARM_VFP__ >= 3 */ +#endif /* __ARM_VFP__ */ +.endmacro + +/* + * load_vfp_registers + * + * Expects a pointer to the VFP save area in r3; loads the callee-saved registers from that save area. + * Clobbers r2 and r3. + */ +.macro load_vfp_registers +#if __ARM_VFP__ + add r2, r3, #64 // Only s16-s31 are callee-saved +#if (__ARM_VFP__ >= 3) + vldmia.64 r2!, {d8-d11} + vldmia.64 r2!, {d12-d15} +#else + fldmias r2!, {s16-s31} +#endif /* __ARM_VFP__ >= 3 */ + ldr r3, [r3, VSS_FPSCR] // Get our saved FPSCR value... + fmxr fpscr, r3 // ...and restore it +#endif /* __ARM_VFP__ */ +.endmacro + +/* + * void machine_load_context(thread_t thread) + * + * Load the context for the first thread to run on a + * cpu, and go. + */ + .syntax unified + .text + .align 2 + .globl EXT(machine_load_context) + +LEXT(machine_load_context) + mcr p15, 0, r0, c13, c0, 4 // Write TPIDRPRW + ldr r1, [r0, TH_CTH_SELF] + mrc p15, 0, r2, c13, c0, 3 // Read TPIDRURO + and r2, r2, #3 // Extract cpu number + orr r1, r1, r2 // + mcr p15, 0, r1, c13, c0, 3 // Write TPIDRURO + ldr r1, [r0, TH_CTH_DATA] + mcr p15, 0, r1, c13, c0, 2 // Write TPIDRURW + mov r7, #0 // Clear frame pointer + ldr r3, [r0, TH_KSTACKPTR] // Get kernel stack top + mov r0, #0 // no param + add r3, r3, SS_R4 + ldmia r3!, {r4-r14} // Load thread status + bx lr // Return + +/* + * void Call_continuation( void (*continuation)(void), + * void *param, + * wait_result_t wresult, + * vm_offset_t stack_ptr) + */ + .text + .align 5 + .globl EXT(Call_continuation) + +LEXT(Call_continuation) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr sp, [r9, TH_KSTACKPTR] // Set stack pointer + mov r7, #0 // Clear frame pointer + mov r6,r0 // Load continuation + mov r0,r1 // Set first parameter + mov r1,r2 // Set wait result arg + blx r6 // Branch to continuation + mrc p15, 0, r0, c13, c0, 4 // Read TPIDRPRW + LOAD_ADDR_PC(thread_terminate) + b . // Not reach + + +/* + * thread_t Switch_context(thread_t old, + * void (*cont)(void), + * thread_t new) + */ + .text + .align 5 + .globl EXT(Switch_context) + +LEXT(Switch_context) + teq r1, #0 // Test if blocking on continuaton + bne switch_threads // No need to save GPR/NEON state if we are +#if __ARM_VFP__ + mov r1, r2 // r2 will be clobbered by the save, so preserve it + add r3, r0, ACT_KVFP // Get the kernel VFP save area for the old thread... + save_vfp_registers // ...and save our VFP state to it + mov r2, r1 // Restore r2 (the new thread pointer) +#endif /* __ARM_VFP__ */ + ldr r3, [r0, TH_KSTACKPTR] // Get old kernel stack top + add r3, r3, SS_R4 + stmia r3!, {r4-r14} // Save general registers to pcb +switch_threads: + ldr r3, [r2, TH_KSTACKPTR] // get kernel stack top + mcr p15, 0, r2, c13, c0, 4 // Write TPIDRPRW + ldr r6, [r2, TH_CTH_SELF] + mrc p15, 0, r5, c13, c0, 3 // Read TPIDRURO + and r5, r5, #3 // Extract cpu number + orr r6, r6, r5 + mcr p15, 0, r6, c13, c0, 3 // Write TPIDRURO + ldr r6, [r2, TH_CTH_DATA] + mcr p15, 0, r6, c13, c0, 2 // Write TPIDRURW +load_reg: + add r3, r3, SS_R4 + ldmia r3!, {r4-r14} // Restore new thread status +#if __ARM_VFP__ + add r3, r2, ACT_KVFP // Get the kernel VFP save area for the new thread... + load_vfp_registers // ...and load the saved state +#endif /* __ARM_VFP__ */ + bx lr // Return + +/* + * thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor) + * + */ + .text + .align 5 + .globl EXT(Shutdown_context) + +LEXT(Shutdown_context) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#if __ARM_VFP__ + add r3, r9, ACT_KVFP // Get the kernel VFP save area for the current thread... + save_vfp_registers // ...and save our VFP state to it +#endif + ldr r3, [r9, TH_KSTACKPTR] // Get kernel stack top + add r3, r3, SS_R4 + stmia r3!, {r4-r14} // Save general registers to pcb + cpsid if // Disable FIQ IRQ + + ldr r12, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [r12, CPU_ISTACKPTR] // Switch to interrupt stack + LOAD_ADDR_PC(cpu_doshutdown) + +/* + * thread_t Idle_context(void) + * + */ + .text + .align 5 + .globl EXT(Idle_context) + +LEXT(Idle_context) + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#if __ARM_VFP__ + add r3, r9, ACT_KVFP // Get the kernel VFP save area for the current thread... + save_vfp_registers // ...and save our VFP state to it +#endif + ldr r3, [r9, TH_KSTACKPTR] // Get kernel stack top + add r3, r3, SS_R4 + stmia r3!, {r4-r14} // Save general registers to pcb + + ldr r12, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [r12, CPU_ISTACKPTR] // Switch to interrupt stack + LOAD_ADDR_PC(cpu_idle) + +/* + * thread_t Idle_context(void) + * + */ + .text + .align 5 + .globl EXT(Idle_load_context) + +LEXT(Idle_load_context) + + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r3, [r12, TH_KSTACKPTR] // Get kernel stack top + add r3, r3, SS_R4 + ldmia r3!, {r4-r14} // Restore new thread status +#if __ARM_VFP__ + add r3, r9, ACT_KVFP // Get the kernel VFP save area for the current thread... + load_vfp_registers // ...and load the saved state +#endif + bx lr // Return + +/* + * void vfp_save(struct arm_vfpsaved_state *vfp_ss) + */ + .text + .align 2 + .globl EXT(vfp_save) + +LEXT(vfp_save) +#if __ARM_VFP__ + fmrx r1, fpscr // Get the current FPSCR... + str r1, [r0, VSS_FPSCR] // ...and save it to the save area +#if (__ARM_VFP__ >= 3) + vstmia.64 r0!, {d0-d3} // Save vfp registers + vstmia.64 r0!, {d4-d7} + vstmia.64 r0!, {d8-d11} + vstmia.64 r0!, {d12-d15} + vstmia.64 r0!, {d16-d19} + vstmia.64 r0!, {d20-d23} + vstmia.64 r0!, {d24-d27} + vstmia.64 r0!, {d28-d31} +#else + fstmias r0!, {s0-s31} // Save vfp registers +#endif +#endif /* __ARM_VFP__ */ + bx lr // Return + +/* + * void vfp_load(struct arm_vfpsaved_state *vfp_ss) + * + * Loads the state in vfp_ss into the VFP registers. + */ + .text + .align 2 + .globl EXT(vfp_load) +LEXT(vfp_load) +#if __ARM_VFP__ + /* r0: vfp_ss, r1: unused, r2: unused, r3: unused */ + mov r1, r0 +#if (__ARM_VFP__ >= 3) + vldmia.64 r0!, {d0-d3} // Restore vfp registers + vldmia.64 r0!, {d4-d7} + vldmia.64 r0!, {d8-d11} + vldmia.64 r0!, {d12-d15} + vldmia.64 r0!, {d16-d19} + vldmia.64 r0!, {d20-d23} + vldmia.64 r0!, {d24-d27} + vldmia.64 r0!, {d28-d31} +#else + fldmias r0!, {s0-s31} // Restore vfp registers +#endif /* __ARM_VFP__ >= 3 */ + ldr r1, [r1, VSS_FPSCR] // Get fpscr from the save state... + fmxr fpscr, r1 // ...and load it into the register +#endif /* __ARM_VFP__ */ + bx lr // Return + +#include "globals_asm.h" + +LOAD_ADDR_GEN_DEF(thread_terminate) +LOAD_ADDR_GEN_DEF(cpu_doshutdown) +LOAD_ADDR_GEN_DEF(cpu_idle) + +/* vim: set ts=4: */ + diff --git a/osfmk/arm/data.s b/osfmk/arm/data.s new file mode 100644 index 000000000..b82b50339 --- /dev/null +++ b/osfmk/arm/data.s @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2007-2009 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include "assym.s" + +#if defined(__arm__) +#include "globals_asm.h" +#elif defined(__arm64__) +/* We're fine, use adrp, add */ +#else +#error Unknown architecture. +#endif + + + .section __DATA, __data // Aligned data + +#if __arm64__ + /* + * Exception stack; this is above the interrupt stack so we don't squash the interrupt + * stack on an exception. + */ + .global EXT(excepstack) +LEXT(excepstack) + .space (4096) + .globl EXT(excepstack_top) +LEXT(excepstack_top) +#endif + + /* IRQ stack */ + .globl EXT(intstack) // Boot processor IRQ stack +LEXT(intstack) + .space (4*4096) + .globl EXT(intstack_top) +LEXT(intstack_top) + + + .align 12 // Page aligned Section + + .globl EXT(fiqstack) // Boot processor FIQ stack +LEXT(fiqstack) + .space (4096) // One page size + .globl EXT(fiqstack_top) // Boot processor FIQ stack top +LEXT(fiqstack_top) + + .globl EXT(CpuDataEntries) + .align 12 // Page aligned +LEXT(CpuDataEntries) // Cpu Data Entry Array + .space (cdeSize_NUM*MAX_CPUS_NUM),0 // (filled with 0s) + + .globl EXT(BootCpuData) + .align 12 // Page aligned +LEXT(BootCpuData) // Per cpu data area + .space cdSize_NUM,0 // (filled with 0s) + + .align 3 // unsigned long long aligned Section + .globl EXT(RTClockData) +LEXT(RTClockData) // Real Time clock area + .space RTCLOCKDataSize_NUM,0 // (filled with 0s) + +#if TRASH_VFP_ON_SAVE + .align 4 + .globl EXT(vfptrash_data) +LEXT(vfptrash_data) + .fill 64, 4, 0xca55e77e +#endif + +// Must align to 16K here, due to + .global EXT(kd_early_buffer) + .align 14 +LEXT(kd_early_buffer) // space for kdebug's early event buffer + .space 16*1024,0 + +#if __arm64__ + .section __DATA, __const + +#if defined(KERNEL_INTEGRITY_KTRR) +/* reserve space for read only page tables */ + .align 14 +LEXT(ropagetable_begin) + .space 16*16*1024,0 +#else +LEXT(ropagetable_begin) +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + +LEXT(ropagetable_end) + + .globl EXT(ropagetable_begin) + .globl EXT(ropagetable_end) +#endif /* __arm64__ */ + +/* vim: set ts=4: */ diff --git a/osfmk/arm/dbgwrap.c b/osfmk/arm/dbgwrap.c new file mode 100644 index 000000000..73aa8b658 --- /dev/null +++ b/osfmk/arm/dbgwrap.c @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include + +boolean_t +ml_dbgwrap_cpu_is_halted(int cpu_index __unused) +{ + return FALSE; +} + +dbgwrap_status_t +ml_dbgwrap_wait_cpu_halted(int cpu_index __unused, uint64_t timeout_ns __unused) +{ + return DBGWRAP_ERR_UNSUPPORTED; +} + +dbgwrap_status_t +ml_dbgwrap_halt_cpu(int cpu_index __unused, uint64_t timeout_ns __unused) +{ + return DBGWRAP_ERR_UNSUPPORTED; +} + +dbgwrap_status_t +ml_dbgwrap_halt_cpu_with_state(int cpu_index __unused, uint64_t timeout_ns __unused, dbgwrap_thread_state_t *state __unused) +{ + return DBGWRAP_ERR_UNSUPPORTED; +} + diff --git a/osfmk/arm/dbgwrap.h b/osfmk/arm/dbgwrap.h new file mode 100644 index 000000000..940346719 --- /dev/null +++ b/osfmk/arm/dbgwrap.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#pragma once + +#include +#include + +__BEGIN_DECLS + +#if defined(__arm64__) +typedef arm_thread_state64_t __attribute__((aligned(16))) dbgwrap_thread_state_t; +#else +typedef arm_thread_state32_t dbgwrap_thread_state_t; +#endif + +typedef enum { + DBGWRAP_ERR_SELF_HALT = -6, + DBGWRAP_ERR_UNSUPPORTED = -5, + DBGWRAP_ERR_INPROGRESS = -4, + DBGWRAP_ERR_INSTR_ERROR = -3, + DBGWRAP_ERR_INSTR_TIMEOUT = -2, + DBGWRAP_ERR_HALT_TIMEOUT = -1, + DBGWRAP_SUCCESS = 0, + DBGWRAP_WARN_ALREADY_HALTED, + DBGWRAP_WARN_CPU_OFFLINE +} dbgwrap_status_t; + +boolean_t ml_dbgwrap_cpu_is_halted(int cpu_index); + +dbgwrap_status_t ml_dbgwrap_wait_cpu_halted(int cpu_index, uint64_t timeout_ns); + +dbgwrap_status_t ml_dbgwrap_halt_cpu(int cpu_index, uint64_t timeout_ns); + +dbgwrap_status_t ml_dbgwrap_halt_cpu_with_state(int cpu_index, uint64_t timeout_ns, dbgwrap_thread_state_t *state); + +__END_DECLS + diff --git a/osfmk/arm/exception.h b/osfmk/arm/exception.h new file mode 100644 index 000000000..bafe40b7d --- /dev/null +++ b/osfmk/arm/exception.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* CMU_ENDHIST */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +/* + */ + +/* + * ARM Exception Info + */ +#ifndef _ARM_EXCEPTION_H_ +#define _ARM_EXCEPTION_H_ + +#define VECT_RESET 0x0 +#define VECT_UNDEF_INST 0x4 +#define VECT_SWI 0x8 +#define VECT_PREFECT_ABT 0xC +#define VECT_DATA_ABT 0x10 +#define VECT_IRQ 0x18 +#define VECT_FIQ 0x1C +/* can put actual code for FIQ here, avoiding extra fetch */ + + +#endif /* _ARM_EXCEPTION_H_ */ diff --git a/osfmk/arm/genassym.c b/osfmk/arm/genassym.c new file mode 100644 index 000000000..4f3c1b8ba --- /dev/null +++ b/osfmk/arm/genassym.c @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#include + +#include + +/* + * Pass field offsets to assembly code. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if CONFIG_DTRACE +#define NEED_DTRACE_DEFS +#include <../bsd/sys/lockstat.h> +#endif /* CONFIG_DTRACE */ + +/* + * genassym.c is used to produce an + * assembly file which, intermingled with unuseful assembly code, + * has all the necessary definitions emitted. This assembly file is + * then postprocessed with sed to extract only these definitions + * and thus the final assyms.s is created. + * + * This convoluted means is necessary since the structure alignment + * and packing may be different between the host machine and the + * target so we are forced into using the cross compiler to generate + * the values, but we cannot run anything on the target machine. + */ + +#define DECLARE(SYM,VAL) \ + __asm("DEFINITION__define__" SYM ":\t .ascii \"%0\"" : : "n" ((u_int)(VAL))) + + +int main( + int argc, + char ** argv); + +int +main( + int argc, + char **argv) +{ + + DECLARE("T_PREFETCH_ABT", T_PREFETCH_ABT); + DECLARE("T_DATA_ABT", T_DATA_ABT); + + DECLARE("AST_URGENT", AST_URGENT); + DECLARE("AST_PREEMPTION", AST_PREEMPTION); + + DECLARE("TH_RECOVER", offsetof(struct thread, recover)); + DECLARE("TH_CONTINUATION", offsetof(struct thread, continuation)); + DECLARE("TH_KERNEL_STACK", offsetof(struct thread, kernel_stack)); + DECLARE("TH_KSTACKPTR", offsetof(struct thread, machine.kstackptr)); + DECLARE("TH_UTHREAD", offsetof(struct thread, uthread)); + + DECLARE("TASK_MACH_EXC_PORT", + offsetof(struct task, exc_actions[EXC_MACH_SYSCALL].port)); + + /* These fields are being added on demand */ + DECLARE("ACT_TASK", offsetof(struct thread, task)); + DECLARE("ACT_PCBDATA", offsetof(struct thread, machine.PcbData)); +#if __ARM_VFP__ + DECLARE("ACT_UVFP", offsetof(struct thread, machine.uVFPdata)); + DECLARE("ACT_KVFP", offsetof(struct thread, machine.kVFPdata)); +#endif + DECLARE("TH_CTH_SELF", offsetof(struct thread, machine.cthread_self)); + DECLARE("TH_CTH_DATA", offsetof(struct thread, machine.cthread_data)); + DECLARE("ACT_PCBDATA_PC", offsetof(struct thread, machine.PcbData.pc)); + DECLARE("ACT_PCBDATA_R0", offsetof(struct thread, machine.PcbData.r[0])); + DECLARE("ACT_PREEMPT_CNT", offsetof(struct thread, machine.preemption_count)); + DECLARE("ACT_CPUDATAP", offsetof(struct thread, machine.CpuDatap)); + DECLARE("ACT_MAP", offsetof(struct thread, map)); +#if __ARM_USER_PROTECT__ + DECLARE("ACT_UPTW_TTC", offsetof(struct thread, machine.uptw_ttc)); + DECLARE("ACT_UPTW_TTB", offsetof(struct thread, machine.uptw_ttb)); + DECLARE("ACT_KPTW_TTB", offsetof(struct thread, machine.kptw_ttb)); + DECLARE("ACT_ASID", offsetof(struct thread, machine.asid)); +#endif + DECLARE("ACT_DEBUGDATA", offsetof(struct thread, machine.DebugData)); + DECLARE("TH_IOTIER_OVERRIDE", offsetof(struct thread, iotier_override)); + DECLARE("TH_RWLOCK_CNT", offsetof(struct thread, rwlock_count)); + DECLARE("TH_SCHED_FLAGS", offsetof(struct thread, sched_flags)); + DECLARE("TH_SFLAG_RW_PROMOTED", TH_SFLAG_RW_PROMOTED); + + DECLARE("TH_MACH_SYSCALLS", offsetof(struct thread, syscalls_mach)); + DECLARE("TH_UNIX_SYSCALLS", offsetof(struct thread, syscalls_unix)); + DECLARE("TASK_BSD_INFO", offsetof(struct task, bsd_info)); + + DECLARE("MACH_TRAP_TABLE_COUNT", MACH_TRAP_TABLE_COUNT); + DECLARE("MACH_TRAP_TABLE_ENTRY_SIZE", sizeof(mach_trap_t)); + + DECLARE("MAP_PMAP", offsetof(struct _vm_map, pmap)); + + DECLARE("SS_SIZE", sizeof(struct arm_saved_state)); + DECLARE("SS_LR", offsetof(struct arm_saved_state, lr)); + DECLARE("SS_CPSR", offsetof(struct arm_saved_state, cpsr)); + DECLARE("SS_PC", offsetof(struct arm_saved_state, pc)); + DECLARE("SS_R0", offsetof(struct arm_saved_state, r[0])); + DECLARE("SS_R4", offsetof(struct arm_saved_state, r[4])); + DECLARE("SS_R9", offsetof(struct arm_saved_state, r[9])); + DECLARE("SS_R12", offsetof(struct arm_saved_state, r[12])); + DECLARE("SS_SP", offsetof(struct arm_saved_state, sp)); + DECLARE("SS_STATUS", offsetof(struct arm_saved_state, fsr)); + DECLARE("SS_VADDR", offsetof(struct arm_saved_state, far)); + DECLARE("SS_EXC", offsetof(struct arm_saved_state, exception)); + +#if __ARM_VFP__ + DECLARE("VSS_SIZE", sizeof(struct arm_vfpsaved_state)); + DECLARE("VSS_FPSCR", offsetof(struct arm_vfpsaved_state, fpscr)); + DECLARE("VSS_FPEXC", offsetof(struct arm_vfpsaved_state, fpexc)); + + DECLARE("EXC_CTX_SIZE", sizeof(struct arm_saved_state) + + sizeof(struct arm_vfpsaved_state) + + VFPSAVE_ALIGN); + DECLARE("VSS_ALIGN", VFPSAVE_ALIGN); +#else + DECLARE("EXC_CTX_SIZE", sizeof(struct arm_saved_state)); +#endif + + + DECLARE("PGBYTES", ARM_PGBYTES); + DECLARE("PGSHIFT", ARM_PGSHIFT); + DECLARE("PGMASK", ARM_PGMASK); + + DECLARE("VM_MIN_ADDRESS", VM_MIN_ADDRESS); + DECLARE("VM_MAX_ADDRESS", VM_MAX_ADDRESS); + DECLARE("KERNELBASE", VM_MIN_KERNEL_ADDRESS); + DECLARE("KERNEL_STACK_SIZE", KERNEL_STACK_SIZE); + + DECLARE("KERN_INVALID_ADDRESS", KERN_INVALID_ADDRESS); + + DECLARE("MAX_CPUS", MAX_CPUS); + + DECLARE("cdeSize", + sizeof(struct cpu_data_entry)); + + DECLARE("cdSize", + sizeof(struct cpu_data)); + + DECLARE("CPU_ACTIVE_THREAD", + offsetof(cpu_data_t, cpu_active_thread)); + DECLARE("CPU_ACTIVE_STACK", + offsetof(cpu_data_t, cpu_active_stack)); + DECLARE("CPU_ISTACKPTR", + offsetof(cpu_data_t, istackptr)); + DECLARE("CPU_INTSTACK_TOP", + offsetof(cpu_data_t, intstack_top)); + DECLARE("CPU_FIQSTACKPTR", + offsetof(cpu_data_t, fiqstackptr)); + DECLARE("CPU_FIQSTACK_TOP", + offsetof(cpu_data_t, fiqstack_top)); + DECLARE("CPU_NUMBER_GS", + offsetof(cpu_data_t,cpu_number)); + DECLARE("CPU_IDENT", + offsetof(cpu_data_t,cpu_ident)); + DECLARE("CPU_RUNNING", + offsetof(cpu_data_t,cpu_running)); + DECLARE("CPU_MCOUNT_OFF", + offsetof(cpu_data_t,cpu_mcount_off)); + DECLARE("CPU_PENDING_AST", + offsetof(cpu_data_t,cpu_pending_ast)); + DECLARE("CPU_PROCESSOR", + offsetof(cpu_data_t,cpu_processor)); + DECLARE("CPU_CACHE_DISPATCH", + offsetof(cpu_data_t,cpu_cache_dispatch)); + DECLARE("CPU_BASE_TIMEBASE_LOW", + offsetof(cpu_data_t,cpu_base_timebase_low)); + DECLARE("CPU_BASE_TIMEBASE_HIGH", + offsetof(cpu_data_t,cpu_base_timebase_high)); + DECLARE("CPU_TIMEBASE_LOW", + offsetof(cpu_data_t,cpu_timebase_low)); + DECLARE("CPU_TIMEBASE_HIGH", + offsetof(cpu_data_t,cpu_timebase_high)); + DECLARE("CPU_DECREMENTER", + offsetof(cpu_data_t,cpu_decrementer)); + DECLARE("CPU_GET_DECREMENTER_FUNC", + offsetof(cpu_data_t,cpu_get_decrementer_func)); + DECLARE("CPU_SET_DECREMENTER_FUNC", + offsetof(cpu_data_t,cpu_set_decrementer_func)); + DECLARE("CPU_GET_FIQ_HANDLER", + offsetof(cpu_data_t,cpu_get_fiq_handler)); + DECLARE("CPU_TBD_HARDWARE_ADDR", + offsetof(cpu_data_t,cpu_tbd_hardware_addr)); + DECLARE("CPU_TBD_HARDWARE_VAL", + offsetof(cpu_data_t,cpu_tbd_hardware_val)); + DECLARE("CPU_INT_STATE", + offsetof(cpu_data_t,cpu_int_state)); + DECLARE("INTERRUPT_HANDLER", + offsetof(cpu_data_t,interrupt_handler)); + DECLARE("INTERRUPT_TARGET", + offsetof(cpu_data_t,interrupt_target)); + DECLARE("INTERRUPT_REFCON", + offsetof(cpu_data_t,interrupt_refCon)); + DECLARE("INTERRUPT_NUB", + offsetof(cpu_data_t,interrupt_nub)); + DECLARE("INTERRUPT_SOURCE", + offsetof(cpu_data_t,interrupt_source)); + DECLARE("CPU_USER_DEBUG", + offsetof(cpu_data_t, cpu_user_debug)); + DECLARE("CPU_STAT_IRQ", + offsetof(cpu_data_t, cpu_stat.irq_ex_cnt)); + DECLARE("CPU_STAT_IRQ_WAKE", + offsetof(cpu_data_t, cpu_stat.irq_ex_cnt_wake)); + DECLARE("CPU_RESET_HANDLER", + offsetof(cpu_data_t, cpu_reset_handler)); + DECLARE("CPU_RESET_ASSIST", + offsetof(cpu_data_t, cpu_reset_assist)); + DECLARE("RTCLOCK_DATAP", + offsetof(cpu_data_t, rtclock_datap)); +#ifdef __arm__ + DECLARE("CPU_EXC_VECTORS", + offsetof(cpu_data_t, cpu_exc_vectors)); +#endif + + DECLARE("RTCLOCKDataSize", + sizeof(rtclock_data_t)); + DECLARE("RTCLOCK_ADJ_ABSTIME_LOW", + offsetof(rtclock_data_t, rtc_adj.abstime_val.low)); + DECLARE("RTCLOCK_ADJ_ABSTIME_HIGH", + offsetof(rtclock_data_t, rtc_adj.abstime_val.high)); + DECLARE("RTCLOCK_BASE_ABSTIME_LOW", + offsetof(rtclock_data_t, rtc_base.abstime_val.low)); + DECLARE("RTCLOCK_BASE_ABSTIME_HIGH", + offsetof(rtclock_data_t, rtc_base.abstime_val.high)); + DECLARE("RTCLOCK_TB_FUNC", + offsetof(rtclock_data_t, rtc_timebase_func)); + DECLARE("RTCLOCK_TB_ADDR", + offsetof(rtclock_data_t, rtc_timebase_addr)); + DECLARE("RTCLOCK_TB_VAL", + offsetof(rtclock_data_t, rtc_timebase_val)); + + DECLARE("SIGPdec", SIGPdec); + + DECLARE("rhdSize", + sizeof(struct reset_handler_data)); + + DECLARE("CPU_DATA_ENTRIES", offsetof(struct reset_handler_data, cpu_data_entries)); + DECLARE("BOOT_ARGS", offsetof(struct reset_handler_data, boot_args)); + DECLARE("ASSIST_RESET_HANDLER", offsetof(struct reset_handler_data, assist_reset_handler)); + + DECLARE("CPU_DATA_PADDR", offsetof(struct cpu_data_entry, cpu_data_paddr)); + + + DECLARE("INTSTACK_SIZE", INTSTACK_SIZE); + + /* values from kern/timer.h */ + DECLARE("TIMER_LOW", + offsetof(struct timer, low_bits)); + DECLARE("TIMER_HIGH", + offsetof(struct timer, high_bits)); + DECLARE("TIMER_HIGHCHK", + offsetof(struct timer, high_bits_check)); + DECLARE("TIMER_TSTAMP", + offsetof(struct timer, tstamp)); + DECLARE("THREAD_TIMER", + offsetof(struct processor, processor_data.thread_timer)); + DECLARE("KERNEL_TIMER", + offsetof(struct processor, processor_data.kernel_timer)); + DECLARE("SYSTEM_STATE", + offsetof(struct processor, processor_data.system_state)); + DECLARE("USER_STATE", + offsetof(struct processor, processor_data.user_state)); + DECLARE("CURRENT_STATE", + offsetof(struct processor, processor_data.current_state)); + + DECLARE("SYSTEM_TIMER", + offsetof(struct thread, system_timer)); + DECLARE("USER_TIMER", + offsetof(struct thread, user_timer)); + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + DECLARE("PRECISE_USER_KERNEL_TIME", + offsetof(struct thread, precise_user_kernel_time)); +#endif + + DECLARE("BA_VIRT_BASE", + offsetof(struct boot_args, virtBase)); + DECLARE("BA_PHYS_BASE", + offsetof(struct boot_args, physBase)); + DECLARE("BA_MEM_SIZE", + offsetof(struct boot_args, memSize)); + DECLARE("BA_TOP_OF_KERNEL_DATA", + offsetof(struct boot_args, topOfKernelData)); + + DECLARE("ENTROPY_INDEX_PTR", + offsetof(entropy_data_t, index_ptr)); + DECLARE("ENTROPY_BUFFER", + offsetof(entropy_data_t, buffer)); + DECLARE("ENTROPY_DATA_SIZE", sizeof(struct entropy_data)); + + return (0); +} diff --git a/osfmk/arm/globals_asm.h b/osfmk/arm/globals_asm.h new file mode 100644 index 000000000..62fe102b9 --- /dev/null +++ b/osfmk/arm/globals_asm.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +LOAD_ADDR_GEN_DEF(ExceptionVectorsBase) +LOAD_ADDR_GEN_DEF(intstack_top) +LOAD_ADDR_GEN_DEF(fiqstack_top) +LOAD_ADDR_GEN_DEF(gVirtBase) +LOAD_ADDR_GEN_DEF(gPhysBase) +LOAD_ADDR_GEN_DEF(gPhysSize) +LOAD_ADDR_GEN_DEF(EntropyData) +LOAD_ADDR_GEN_DEF(kdebug_enable) +#if CONFIG_TELEMETRY +LOAD_ADDR_GEN_DEF(telemetry_needs_record) +#endif + diff --git a/osfmk/arm/hw_lock_types.h b/osfmk/arm/hw_lock_types.h new file mode 100644 index 000000000..342445c33 --- /dev/null +++ b/osfmk/arm/hw_lock_types.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (C) 1998 Apple Computer + * All Rights Reserved + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#ifndef _ARM_HW_LOCK_TYPES_H_ +#define _ARM_HW_LOCK_TYPES_H_ + +struct hslock { + uintptr_t lock_data; +}; + +typedef struct hslock hw_lock_data_t, *hw_lock_t; + +#define hw_lock_addr(hwl) (&((hwl).lock_data)) + +#endif /* _ARM_HW_LOCK_TYPES_H_ */ diff --git a/osfmk/arm/io_map.c b/osfmk/arm/io_map.c new file mode 100644 index 000000000..2aa718001 --- /dev/null +++ b/osfmk/arm/io_map.c @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright notice + * and this permission notice appear in all copies of the software, + * derivative works or modified versions, and any portions thereof, and that + * both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" CONDITION. + * CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ANY DAMAGES + * WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science Carnegie Mellon University Pittsburgh PA + * 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon the + * rights to redistribute these changes. + */ +/* + */ + +#include +#include +#include +#include +#include +#include + +extern vm_offset_t virtual_space_start; /* Next available kernel VA */ + +/* + * Allocate and map memory for devices that may need to be mapped before + * Mach VM is running. + */ +vm_offset_t +io_map(vm_map_offset_t phys_addr, vm_size_t size, unsigned int flags) +{ + vm_offset_t start, start_offset; + + start_offset = phys_addr & PAGE_MASK; + size += start_offset; + phys_addr -= start_offset; + + if (kernel_map == VM_MAP_NULL) { + /* + * VM is not initialized. Grab memory. + */ + start = virtual_space_start; + virtual_space_start += round_page(size); + + assert(flags == VM_WIMG_WCOMB || flags == VM_WIMG_IO); + + if (flags == VM_WIMG_WCOMB) { + (void) pmap_map_bd_with_options(start, phys_addr, phys_addr + round_page(size), + VM_PROT_READ | VM_PROT_WRITE, PMAP_MAP_BD_WCOMB); + } else { + (void) pmap_map_bd(start, phys_addr, phys_addr + round_page(size), + VM_PROT_READ | VM_PROT_WRITE); + } + } else { + (void) kmem_alloc_pageable(kernel_map, &start, round_page(size), VM_KERN_MEMORY_IOKIT); + (void) pmap_map(start, phys_addr, phys_addr + round_page(size), + VM_PROT_READ | VM_PROT_WRITE, flags); + } +#if KASAN + kasan_notify_address(start + start_offset, size); +#endif + return (start + start_offset); +} + +/* just wrap this since io_map handles it */ + +vm_offset_t +io_map_spec(vm_map_offset_t phys_addr, vm_size_t size, unsigned int flags) +{ + return (io_map(phys_addr, size, flags)); +} diff --git a/osfmk/arm/io_map_entries.h b/osfmk/arm/io_map_entries.h new file mode 100644 index 000000000..1a96d9764 --- /dev/null +++ b/osfmk/arm/io_map_entries.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +#ifdef KERNEL_PRIVATE + +#ifndef _ARM_IO_MAP_ENTRIES +#define _ARM_IO_MAP_ENTRIES + +#include + +#ifdef __APPLE_API_PRIVATE +extern vm_offset_t io_map( + vm_map_offset_t phys_addr, + vm_size_t size, + unsigned int flags); +extern vm_offset_t io_map_spec(vm_map_offset_t phys_addr, vm_size_t size, unsigned int flags); +#endif /* __APPLE_API_PRIVATE */ + +#endif /* _ARM_IO_MAP_ENTRIES */ + +#endif /* KERNEL_PRIVATE */ + diff --git a/osfmk/arm/kpc_arm.c b/osfmk/arm/kpc_arm.c new file mode 100644 index 000000000..11a544584 --- /dev/null +++ b/osfmk/arm/kpc_arm.c @@ -0,0 +1,986 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef ARMA7 +/* PMU v2 based implementation for A7 */ +static uint32_t saved_PMXEVTYPER[MAX_CPUS][KPC_ARM_TOTAL_COUNT]; +static uint32_t saved_PMCNTENSET[MAX_CPUS]; +static uint64_t saved_counter[MAX_CPUS][KPC_ARM_TOTAL_COUNT]; +static uint32_t saved_PMOVSR[MAX_CPUS]; + +static uint32_t kpc_configured = 0; +static uint32_t kpc_xcall_sync; +static uint64_t kpc_running_cfg_pmc_mask = 0; +static uint32_t kpc_running_classes = 0; +static uint32_t kpc_reload_sync; +static uint32_t kpc_enabled_counters = 0; + +static int first_time = 1; + +/* Private */ + +static boolean_t +enable_counter(uint32_t counter) +{ + boolean_t enabled; + uint32_t PMCNTENSET; + /* Cycle counter is MSB; configurable counters reside in LSBs */ + uint32_t mask = (counter == 0) ? (1 << 31) : (1 << (counter - 1)); + + /* Enabled? */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 1;" : "=r" (PMCNTENSET)); + + enabled = (PMCNTENSET & mask); + if (!enabled) { + /* Counter interrupt enable (PMINTENSET) */ + __asm__ volatile("mcr p15, 0, %0, c9, c14, 1;" : : "r" (mask)); + + /* Individual counter enable set (PMCNTENSET) */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 1;" : : "r" (mask)); + + kpc_enabled_counters++; + + /* 1st enabled counter? Set the master enable bit in PMCR */ + if (kpc_enabled_counters == 1) { + uint32_t PMCR = 1; + __asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR)); + } + } + + return enabled; +} + +static boolean_t +disable_counter(uint32_t counter) +{ + boolean_t enabled; + uint32_t PMCNTENCLR; + /* Cycle counter is MSB; configurable counters reside in LSBs */ + uint32_t mask = (counter == 0) ? (1 << 31) : (1 << (counter - 1)); + + /* Enabled? */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 2;" : "=r" (PMCNTENCLR)); + + enabled = (PMCNTENCLR & mask); + if (enabled) { + /* Individual counter enable clear (PMCNTENCLR) */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 2;" : : "r" (mask)); + + /* Counter interrupt disable (PMINTENCLR) */ + __asm__ volatile("mcr p15, 0, %0, c9, c14, 2;" : : "r" (mask)); + + kpc_enabled_counters--; + + /* Last enabled counter? Clear the master enable bit in PMCR */ + if (kpc_enabled_counters == 0) { + uint32_t PMCR = 0; + __asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR)); + } + } + + return enabled; +} + +static uint64_t +read_counter(uint32_t counter) +{ + uint32_t low = 0; + + switch (counter) { + case 0: + /* Fixed counter */ + __asm__ volatile("mrc p15, 0, %0, c9, c13, 0;" : "=r" (low)); + break; + case 1: + case 2: + case 3: + case 4: + /* Configurable. Set PMSELR... */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1)); + /* ...then read PMXEVCNTR */ + __asm__ volatile("mrc p15, 0, %0, c9, c13, 2;" : "=r" (low)); + break; + default: + /* ??? */ + break; + } + + return (uint64_t)low; +} + +static void +write_counter(uint32_t counter, uint64_t value) +{ + uint32_t low = value & 0xFFFFFFFF; + + switch (counter) { + case 0: + /* Fixed counter */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 0;" : : "r" (low)); + break; + case 1: + case 2: + case 3: + case 4: + /* Configurable. Set PMSELR... */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1)); + /* ...then write PMXEVCNTR */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 2;" : : "r" (low)); + break; + default: + /* ??? */ + break; + } +} + +static uint64_t +kpc_reload_counter(int ctr) +{ + uint64_t old = read_counter(ctr); + write_counter(ctr, FIXED_RELOAD(ctr)); + return old; +} + +static void +set_running_fixed(boolean_t on) +{ + int i; + boolean_t enabled; + int n = KPC_ARM_FIXED_COUNT; + + enabled = ml_set_interrupts_enabled(FALSE); + + for( i = 0; i < n; i++ ) { + if (on) { + enable_counter(i); + } else { + disable_counter(i); + } + } + + ml_set_interrupts_enabled(enabled); +} + +static void +set_running_configurable(uint64_t target_mask, uint64_t state_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + boolean_t enabled; + + enabled = ml_set_interrupts_enabled(FALSE); + + for (uint32_t i = 0; i < cfg_count; ++i) { + if (((1ULL << i) & target_mask) == 0) + continue; + assert(kpc_controls_counter(offset + i)); + + if ((1ULL << i) & state_mask) { + enable_counter(offset + i); + } else { + disable_counter(offset + i); + } + } + + ml_set_interrupts_enabled(enabled); +} + +void kpc_pmi_handler(cpu_id_t source); +void +kpc_pmi_handler(cpu_id_t source) +{ + uint64_t extra; + int ctr; + int enabled; + + enabled = ml_set_interrupts_enabled(FALSE); + + /* The pmi must be delivered to the CPU that generated it */ + if (source != getCpuDatap()->interrupt_nub) { + panic("pmi from IOCPU %p delivered to IOCPU %p", source, getCpuDatap()->interrupt_nub); + } + + for (ctr = 0; + ctr < (KPC_ARM_FIXED_COUNT + KPC_ARM_CONFIGURABLE_COUNT); + ctr++) + { + uint32_t PMOVSR; + uint32_t mask; + + /* check the counter for overflow */ + if (ctr == 0) { + mask = 1 << 31; + } else { + mask = 1 << (ctr - 1); + } + + /* read PMOVSR */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR)); + + if (PMOVSR & mask) { + extra = kpc_reload_counter(ctr); + + FIXED_SHADOW(ctr) + += (kpc_fixed_max() - FIXED_RELOAD(ctr) + 1 /* wrap */) + extra; + + if (FIXED_ACTIONID(ctr)) + kpc_sample_kperf(FIXED_ACTIONID(ctr)); + + /* clear PMOVSR bit */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 3;" : : "r" (mask)); + } + } + + ml_set_interrupts_enabled(enabled); +} + +static void +kpc_set_running_xcall( void *vstate ) +{ + struct kpc_running_remote *mp_config = (struct kpc_running_remote*) vstate; + assert(mp_config); + + if (kpc_controls_fixed_counters()) + set_running_fixed(mp_config->classes & KPC_CLASS_FIXED_MASK); + + set_running_configurable(mp_config->cfg_target_mask, + mp_config->cfg_state_mask); + + if (hw_atomic_sub(&kpc_xcall_sync, 1) == 0) { + thread_wakeup((event_t) &kpc_xcall_sync); + } +} + +static uint64_t +get_counter_config(uint32_t counter) +{ + uint32_t config = 0; + + switch (counter) { + case 0: + /* Fixed counter accessed via top bit... */ + counter = 31; + /* Write PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter)); + /* Read PMXEVTYPER */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : "=r" (config)); + break; + case 1: + case 2: + case 3: + case 4: + /* Offset */ + counter -= 1; + /* Write PMSELR.SEL to select the configurable counter */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter)); + /* Read PMXEVTYPER to get the config */ + __asm__ volatile("mrc p15, 0, %0, c9, c13, 1;" : "=r" (config)); + break; + default: + break; + } + + return config; +} + +static void +set_counter_config(uint32_t counter, uint64_t config) +{ + switch (counter) { + case 0: + /* Write PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31)); + /* Write PMXEVTYPER */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (config & 0xFFFFFFFF)); + break; + case 1: + case 2: + case 3: + case 4: + /* Write PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1)); + /* Write PMXEVTYPER */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (config & 0xFFFFFFFF)); + break; + default: + break; + } +} + +/* Common */ + +void +kpc_arch_init(void) +{ + uint32_t PMCR; + uint32_t event_counters; + + /* read PMOVSR and determine the number of event counters */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 0;" : "=r" (PMCR)); + event_counters = (PMCR >> 11) & 0x1F; + + assert(event_counters >= KPC_ARM_CONFIGURABLE_COUNT); +} + +uint32_t +kpc_get_classes(void) +{ + return KPC_CLASS_FIXED_MASK | KPC_CLASS_CONFIGURABLE_MASK; +} + +uint32_t +kpc_fixed_count(void) +{ + return KPC_ARM_FIXED_COUNT; +} + +uint32_t +kpc_configurable_count(void) +{ + return KPC_ARM_CONFIGURABLE_COUNT; +} + +uint32_t +kpc_fixed_config_count(void) +{ + return KPC_ARM_FIXED_COUNT; +} + +uint32_t +kpc_configurable_config_count(uint64_t pmc_mask) +{ + assert(kpc_popcount(pmc_mask) <= kpc_configurable_count()); + return kpc_popcount(pmc_mask); +} + +int +kpc_get_fixed_config(kpc_config_t *configv) +{ + configv[0] = get_counter_config(0); + return 0; +} + +uint64_t +kpc_fixed_max(void) +{ + return (1ULL << KPC_ARM_COUNTER_WIDTH) - 1; +} + +uint64_t +kpc_configurable_max(void) +{ + return (1ULL << KPC_ARM_COUNTER_WIDTH) - 1; +} + +int +kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + + assert(counterv); + + for (uint32_t i = 0; i < cfg_count; ++i) { + uint32_t PMOVSR; + uint32_t mask; + uint64_t ctr; + + if (((1ULL << i) & pmc_mask) == 0) + continue; + ctr = read_counter(i + offset); + + /* check the counter for overflow */ + mask = 1 << i; + + /* read PMOVSR */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR)); + + if (PMOVSR & mask) { + ctr = CONFIGURABLE_SHADOW(i) + + (kpc_configurable_max() - CONFIGURABLE_RELOAD(i) + 1 /* Wrap */) + + ctr; + } else { + ctr = CONFIGURABLE_SHADOW(i) + + (ctr - CONFIGURABLE_RELOAD(i)); + } + + *counterv++ = ctr; + } + + return 0; +} + +int +kpc_get_fixed_counters(uint64_t *counterv) +{ + uint32_t PMOVSR; + uint32_t mask; + uint64_t ctr; + + /* check the counter for overflow */ + mask = 1 << 31; + + /* read PMOVSR */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR)); + + ctr = read_counter(0); + + if (PMOVSR & mask) { + ctr = FIXED_SHADOW(0) + + (kpc_fixed_max() - FIXED_RELOAD(0) + 1 /* Wrap */) + + (ctr & 0xFFFFFFFF); + } else { + ctr = FIXED_SHADOW(0) + + (ctr - FIXED_RELOAD(0)); + } + + counterv[0] = ctr; + + return 0; +} +boolean_t +kpc_is_running_fixed(void) +{ + return (kpc_running_classes & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK; +} + +boolean_t +kpc_is_running_configurable(uint64_t pmc_mask) +{ + assert(kpc_popcount(pmc_mask) <= kpc_configurable_count()); + return ((kpc_running_classes & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK) && + ((kpc_running_cfg_pmc_mask & pmc_mask) == pmc_mask); +} + +int +kpc_set_running_arch(struct kpc_running_remote *mp_config) +{ + unsigned int cpu; + + assert(mp_config); + + if (first_time) { + kprintf( "kpc: setting PMI handler\n" ); + PE_cpu_perfmon_interrupt_install_handler(kpc_pmi_handler); + for (cpu = 0; cpu < real_ncpus; cpu++) + PE_cpu_perfmon_interrupt_enable(cpu_datap(cpu)->cpu_id, + TRUE); + first_time = 0; + } + + /* dispatch to all CPUs */ + cpu_broadcast_xcall(&kpc_xcall_sync, TRUE, kpc_set_running_xcall, + mp_config); + + kpc_running_cfg_pmc_mask = mp_config->cfg_state_mask; + kpc_running_classes = mp_config->classes; + kpc_configured = 1; + + return 0; +} + +static void +save_regs(void) +{ + int i; + int cpuid = current_processor()->cpu_id; + uint32_t PMCR = 0; + + __asm__ volatile("dmb ish"); + + /* Clear master enable */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR)); + + /* Save individual enable state */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 1;" : "=r" (saved_PMCNTENSET[cpuid])); + + /* Save PMOVSR */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (saved_PMOVSR[cpuid])); + + /* Select fixed counter with PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31)); + /* Read PMXEVTYPER */ + __asm__ volatile("mrc p15, 0, %0, c9, c13, 1;" : "=r" (saved_PMXEVTYPER[cpuid][0])); + + /* Save configurable event selections */ + for (i = 0; i < 4; i++) { + /* Select counter with PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (i)); + /* Read PMXEVTYPER */ + __asm__ volatile("mrc p15, 0, %0, c9, c13, 1;" : "=r" (saved_PMXEVTYPER[cpuid][i + 1])); + } + + /* Finally, save count for each counter */ + for (i=0; i < 5; i++) { + saved_counter[cpuid][i] = read_counter(i); + } +} + +static void +restore_regs(void) +{ + int i; + int cpuid = current_processor()->cpu_id; + uint64_t extra; + uint32_t PMCR = 1; + + /* Restore counter values */ + for (i = 0; i < 5; i++) { + /* did we overflow? if so handle it now since we won't get a pmi */ + uint32_t mask; + + /* check the counter for overflow */ + if (i == 0) { + mask = 1 << 31; + } else { + mask = 1 << (i - 1); + } + + if (saved_PMOVSR[cpuid] & mask) { + extra = kpc_reload_counter(i); + + /* + * CONFIGURABLE_* directly follows FIXED, so we can simply + * increment the index here. Although it's ugly. + */ + FIXED_SHADOW(i) + += (kpc_fixed_max() - FIXED_RELOAD(i) + 1 /* Wrap */) + extra; + + if (FIXED_ACTIONID(i)) + kpc_sample_kperf(FIXED_ACTIONID(i)); + } else { + write_counter(i, saved_counter[cpuid][i]); + } + } + + /* Restore configuration - first, the fixed... */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31)); + /* Write PMXEVTYPER */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (saved_PMXEVTYPER[cpuid][0])); + + /* ...then the configurable */ + for (i = 0; i < 4; i++) { + /* Select counter with PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (i)); + /* Write PMXEVTYPER */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (saved_PMXEVTYPER[cpuid][i + 1])); + } + + /* Restore enable state */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 1;" : : "r" (saved_PMCNTENSET[cpuid])); + + /* Counter master re-enable */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR)); +} + +static void +kpc_set_reload_xcall(void *vmp_config) +{ + struct kpc_config_remote *mp_config = vmp_config; + uint32_t classes = 0, count = 0, offset = kpc_fixed_count(); + uint64_t *new_period = NULL, max = kpc_configurable_max(); + boolean_t enabled; + + assert(mp_config); + assert(mp_config->configv); + classes = mp_config->classes; + new_period = mp_config->configv; + + enabled = ml_set_interrupts_enabled(FALSE); + + if ((classes & KPC_CLASS_FIXED_MASK) && kpc_controls_fixed_counters()) { + /* update shadow counters */ + kpc_get_fixed_counters(&FIXED_SHADOW(0)); + + /* set the new period */ + count = kpc_fixed_count(); + for (uint32_t i = 0; i < count; ++i) { + if (*new_period == 0) + *new_period = kpc_fixed_max(); + FIXED_RELOAD(i) = max - *new_period; + /* reload the counter if possible */ + kpc_reload_counter(i); + /* next period value */ + new_period++; + } + } + + if (classes & KPC_CLASS_CONFIGURABLE_MASK) { + /* + * Update _all_ shadow counters, this cannot be done for only + * selected PMCs. Otherwise, we would corrupt the configurable + * shadow buffer since the PMCs are muxed according to the pmc + * mask. + */ + uint64_t all_cfg_mask = (1ULL << kpc_configurable_count()) - 1; + kpc_get_configurable_counters(&CONFIGURABLE_SHADOW(0), all_cfg_mask); + + /* set the new period */ + count = kpc_configurable_count(); + for (uint32_t i = 0; i < count; ++i) { + /* ignore the counter */ + if (((1ULL << i) & mp_config->pmc_mask) == 0) + continue; + if (*new_period == 0) + *new_period = kpc_configurable_max(); + CONFIGURABLE_RELOAD(i) = max - *new_period; + /* reload the counter */ + kpc_reload_counter(offset + i); + /* next period value */ + new_period++; + } + } + + ml_set_interrupts_enabled(enabled); + + if (hw_atomic_sub(&kpc_reload_sync, 1) == 0) + thread_wakeup((event_t) &kpc_reload_sync); +} + + +int +kpc_set_period_arch(struct kpc_config_remote *mp_config) +{ + /* dispatch to all CPUs */ + cpu_broadcast_xcall(&kpc_reload_sync, TRUE, kpc_set_reload_xcall, mp_config); + + kpc_configured = 1; + + return 0; +} + +int +kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + + assert(configv); + + for (uint32_t i = 0; i < cfg_count; ++i) + if ((1ULL << i) & pmc_mask) + *configv++ = get_counter_config(i + offset); + + return 0; +} + +static int +kpc_set_configurable_config(kpc_config_t *configv, uint64_t pmc_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + boolean_t enabled; + + assert(configv); + + enabled = ml_set_interrupts_enabled(FALSE); + + for (uint32_t i = 0; i < cfg_count; ++i) { + if (((1ULL << i) & pmc_mask) == 0) + continue; + assert(kpc_controls_counter(i + offset)); + + set_counter_config(i + offset, *configv++); + } + + ml_set_interrupts_enabled(enabled); + + return 0; +} + +static uint32_t kpc_config_sync; +static void +kpc_set_config_xcall(void *vmp_config) +{ + struct kpc_config_remote *mp_config = vmp_config; + kpc_config_t *new_config = NULL; + uint32_t classes = 0ULL; + + assert(mp_config); + assert(mp_config->configv); + classes = mp_config->classes; + new_config = mp_config->configv; + + if (classes & KPC_CLASS_CONFIGURABLE_MASK) { + kpc_set_configurable_config(new_config, mp_config->pmc_mask); + new_config += kpc_popcount(mp_config->pmc_mask); + } + + if (hw_atomic_sub(&kpc_config_sync, 1) == 0) + thread_wakeup((event_t) &kpc_config_sync); +} + +int +kpc_set_config_arch(struct kpc_config_remote *mp_config) +{ + /* dispatch to all CPUs */ + cpu_broadcast_xcall(&kpc_config_sync, TRUE, kpc_set_config_xcall, mp_config); + + kpc_configured = 1; + + return 0; +} + +void +kpc_idle(void) +{ + if (kpc_configured) + save_regs(); +} + +void +kpc_idle_exit(void) +{ + if (kpc_configured) + restore_regs(); +} + +static uint32_t kpc_xread_sync; +static void +kpc_get_curcpu_counters_xcall(void *args) +{ + struct kpc_get_counters_remote *handler = args; + int offset=0, r=0; + + assert(handler); + assert(handler->buf); + + offset = cpu_number() * handler->buf_stride; + r = kpc_get_curcpu_counters(handler->classes, NULL, &handler->buf[offset]); + + /* number of counters added by this CPU, needs to be atomic */ + hw_atomic_add(&(handler->nb_counters), r); + + if (hw_atomic_sub(&kpc_xread_sync, 1) == 0) + thread_wakeup((event_t) &kpc_xread_sync); +} + +int +kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf) +{ + int enabled = 0; + + struct kpc_get_counters_remote hdl = { + .classes = classes, .nb_counters = 0, + .buf_stride = kpc_get_counter_count(classes), + .buf = buf + }; + + assert(buf); + + enabled = ml_set_interrupts_enabled(FALSE); + + if (curcpu) + *curcpu = current_processor()->cpu_id; + cpu_broadcast_xcall(&kpc_xread_sync, TRUE, kpc_get_curcpu_counters_xcall, &hdl); + + ml_set_interrupts_enabled(enabled); + + return hdl.nb_counters; +} + +int +kpc_get_pmu_version(void) +{ + return KPC_PMU_ARM_V2; +} + +int +kpc_set_sw_inc( uint32_t mask ) +{ + /* Only works with the configurable counters set to count the increment event (0x0) */ + + /* Write to PMSWINC */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 4;" : : "r" (mask)); + + return 0; +} + +#else /* !ARMA7 */ + +/* no kpc */ + +void +kpc_arch_init(void) +{ + /* No-op */ +} + +uint32_t +kpc_get_classes(void) +{ + return 0; +} + +uint32_t +kpc_fixed_count(void) +{ + return 0; +} + +uint32_t +kpc_configurable_count(void) +{ + return 0; +} + +uint32_t +kpc_fixed_config_count(void) +{ + return 0; +} + +uint32_t +kpc_configurable_config_count(uint64_t pmc_mask __unused) +{ + return 0; +} + +int +kpc_get_fixed_config(kpc_config_t *configv __unused) +{ + return 0; +} + +uint64_t +kpc_fixed_max(void) +{ + return 0; +} + +uint64_t +kpc_configurable_max(void) +{ + return 0; +} + +int +kpc_get_configurable_config(kpc_config_t *configv __unused, uint64_t pmc_mask __unused) +{ + return ENOTSUP; +} + +int +kpc_get_configurable_counters(uint64_t *counterv __unused, uint64_t pmc_mask __unused) +{ + return ENOTSUP; +} + +int +kpc_get_fixed_counters(uint64_t *counterv __unused) +{ + return 0; +} + +boolean_t +kpc_is_running_fixed(void) +{ + return FALSE; +} + +boolean_t +kpc_is_running_configurable(uint64_t pmc_mask __unused) +{ + return FALSE; +} + +int +kpc_set_running_arch(struct kpc_running_remote *mp_config __unused) +{ + return ENOTSUP; +} + +int +kpc_set_period_arch(struct kpc_config_remote *mp_config __unused) +{ + return ENOTSUP; +} + +int +kpc_set_config_arch(struct kpc_config_remote *mp_config __unused) +{ + return ENOTSUP; +} + +void +kpc_idle(void) +{ + // do nothing +} + +void +kpc_idle_exit(void) +{ + // do nothing +} + +int +kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf) +{ +#pragma unused(classes) +#pragma unused(curcpu) +#pragma unused(buf) + + return 0; +} + +int +kpc_set_sw_inc( uint32_t mask __unused ) +{ + return ENOTSUP; +} + +int +kpc_get_pmu_version(void) +{ + return KPC_PMU_ERROR; +} + +#endif + +/* + * RAWPMU isn't implemented for any of the 32-bit ARMs. + */ + +uint32_t +kpc_rawpmu_config_count(void) +{ + return 0; +} + +int +kpc_get_rawpmu_config(__unused kpc_config_t *configv) +{ + return 0; +} diff --git a/osfmk/arm/lock.h b/osfmk/arm/lock.h new file mode 100644 index 000000000..943a5f345 --- /dev/null +++ b/osfmk/arm/lock.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (C) 1998 Apple Computer + * All Rights Reserved + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#ifdef KERNEL_PRIVATE + +#ifndef _ARM_LOCK_H_ +#define _ARM_LOCK_H_ + +#warning This header is deprecated. Use instead. + +#endif /* _ARM_LOCK_H_ */ + +#endif /* KERNEL_PRIVATE */ diff --git a/osfmk/arm/locks.h b/osfmk/arm/locks.h new file mode 100644 index 000000000..3a58a7fcc --- /dev/null +++ b/osfmk/arm/locks.h @@ -0,0 +1,332 @@ +/* + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _ARM_LOCKS_H_ +#define _ARM_LOCKS_H_ + +#include +#ifdef MACH_KERNEL_PRIVATE +#include +#endif + + +#ifdef MACH_KERNEL_PRIVATE + +extern unsigned int LcksOpts; + +#define enaLkDeb 0x00000001 /* Request debug in default attribute */ +#define enaLkStat 0x00000002 /* Request statistic in default attribute */ +#define disLkRWPrio 0x00000004 /* Disable RW lock priority promotion */ + +#define disLkType 0x80000000 /* Disable type checking */ +#define disLktypeb 0 +#define disLkThread 0x40000000 /* Disable ownership checking */ +#define disLkThreadb 1 +#define enaLkExtStck 0x20000000 /* Enable extended backtrace */ +#define enaLkExtStckb 2 +#define disLkMyLck 0x10000000 /* Disable recursive lock dectection */ +#define disLkMyLckb 3 + +#endif + +#ifdef MACH_KERNEL_PRIVATE +typedef struct { + struct hslock hwlock; + uintptr_t type; +} lck_spin_t; + +#define lck_spin_data hwlock.lock_data + +#define LCK_SPIN_TAG_DESTROYED 0xdead /* lock marked as Destroyed */ + +#define LCK_SPIN_TYPE 0x00000011 + +#else +#ifdef KERNEL_PRIVATE + +typedef struct { + uintptr_t opaque[2]; +} lck_spin_t; + +#else +typedef struct __lck_spin_t__ lck_spin_t; +#endif // KERNEL_PRIVATE +#endif // MACH_KERNEL_PRIVATE + +#ifdef MACH_KERNEL_PRIVATE +typedef struct _lck_mtx_ { + union { + + uintptr_t lck_mtx_data; /* Thread pointer plus lock bits */ + uintptr_t lck_mtx_tag; /* Tag for type */ + }; /* arm: 4 arm64: 8 */ + union { + struct { + uint16_t lck_mtx_waiters;/* Number of waiters */ + uint8_t lck_mtx_pri; /* Priority to inherit */ + uint8_t lck_mtx_type; /* Type */ + }; + struct { + struct _lck_mtx_ext_ *lck_mtx_ptr; /* Indirect pointer */ + }; + }; /* arm: 4 arm64: 8 */ +} lck_mtx_t; /* arm: 8 arm64: 16 */ + +/* Shared between mutex and read-write locks */ +#define LCK_ILOCK_BIT 0 +#define ARM_LCK_WAITERS_BIT 1 +#define LCK_ILOCK (1 << LCK_ILOCK_BIT) +#define ARM_LCK_WAITERS (1 << ARM_LCK_WAITERS_BIT) + +#define LCK_MTX_TYPE 0x22 /* lock type */ + +#define LCK_MTX_TAG_INDIRECT 0x00001007 /* lock marked as Indirect */ +#define LCK_MTX_TAG_DESTROYED 0x00002007 /* lock marked as Destroyed */ + +#define LCK_FRAMES_MAX 8 + +extern uint64_t MutexSpin; + +typedef struct { + unsigned int type; + vm_offset_t stack[LCK_FRAMES_MAX]; + vm_offset_t thread; +} lck_mtx_deb_t; + +#define MUTEX_TAG 0x4d4d + +typedef struct { + unsigned int lck_mtx_stat_data; +} lck_mtx_stat_t; + +typedef struct _lck_mtx_ext_ { + lck_mtx_t lck_mtx; /* arm: 12 arm64: 24 */ + struct _lck_grp_ *lck_mtx_grp; /* arm: 4 arm64: 8 */ + unsigned int lck_mtx_attr; /* arm: 4 arm64: 4 */ + lck_mtx_stat_t lck_mtx_stat; /* arm: 4 arm64: 4 */ + lck_mtx_deb_t lck_mtx_deb; /* arm: 40 arm64: 80 */ +} lck_mtx_ext_t; /* arm: 64 arm64: 120 */ + +#define LCK_MTX_ATTR_DEBUG 0x1 +#define LCK_MTX_ATTR_DEBUGb 31 +#define LCK_MTX_ATTR_STAT 0x2 +#define LCK_MTX_ATTR_STATb 30 + +#define LCK_MTX_EVENT(lck) ((event_t)(((unsigned int*)(lck))+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int)))) +#define LCK_EVENT_TO_MUTEX(event) ((lck_mtx_t *)(uintptr_t)(((unsigned int *)(event)) - ((sizeof(lck_mtx_t)-1)/sizeof(unsigned int)))) + +#else +#ifdef KERNEL_PRIVATE +typedef struct { + uintptr_t opaque[2]; +} lck_mtx_t; + +typedef struct { +#if defined(__arm64__) + unsigned long opaque[16]; +#else /* __arm__ */ + unsigned int opaque[16]; +#endif +} lck_mtx_ext_t; + +#else +typedef struct __lck_mtx_t__ lck_mtx_t; +#endif +#endif + +#ifdef MACH_KERNEL_PRIVATE + +typedef union { + struct { + uint16_t shared_count; /* No. of shared granted request */ + uint16_t interlock: 1, /* Interlock */ + priv_excl: 1, /* priority for Writer */ + want_upgrade: 1, /* Read-to-write upgrade waiting */ + want_excl: 1, /* Writer is waiting, or locked for write */ + r_waiting: 1, /* Someone is sleeping on lock */ + w_waiting: 1, /* Writer is sleeping on lock */ + can_sleep: 1, /* Can attempts to lock go to sleep? */ + _pad2: 8, /* padding */ + tag_valid: 1; /* Field is actually a tag, not a bitfield */ +#if __arm64__ + uint32_t _pad4; +#endif + }; + struct { + uint32_t data; /* Single word version of bitfields and shared count */ +#if __arm64__ + uint32_t lck_rw_pad4; +#endif + }; +} lck_rw_word_t; + +typedef struct { + lck_rw_word_t word; + thread_t lck_rw_owner; +} lck_rw_t; /* arm: 8 arm64: 16 */ + +#define lck_rw_shared_count word.shared_count +#define lck_rw_interlock word.interlock +#define lck_rw_priv_excl word.priv_excl +#define lck_rw_want_upgrade word.want_upgrade +#define lck_rw_want_excl word.want_excl +#define lck_r_waiting word.r_waiting +#define lck_w_waiting word.w_waiting +#define lck_rw_can_sleep word.can_sleep +#define lck_rw_data word.data +// tag and data reference the same memory. When the tag_valid bit is set, +// the data word should be treated as a tag instead of a bitfield. +#define lck_rw_tag_valid word.tag_valid +#define lck_rw_tag word.data + +#define LCK_RW_SHARED_READER_OFFSET 0 +#define LCK_RW_INTERLOCK_BIT 16 +#define LCK_RW_PRIV_EXCL_BIT 17 +#define LCK_RW_WANT_UPGRADE_BIT 18 +#define LCK_RW_WANT_EXCL_BIT 19 +#define LCK_RW_R_WAITING_BIT 20 +#define LCK_RW_W_WAITING_BIT 21 +#define LCK_RW_CAN_SLEEP_BIT 22 +// 23-30 +#define LCK_RW_TAG_VALID_BIT 31 + +#define LCK_RW_INTERLOCK (1 << LCK_RW_INTERLOCK_BIT) +#define LCK_RW_R_WAITING (1 << LCK_RW_R_WAITING_BIT) +#define LCK_RW_W_WAITING (1 << LCK_RW_W_WAITING_BIT) +#define LCK_RW_WANT_UPGRADE (1 << LCK_RW_WANT_UPGRADE_BIT) +#define LCK_RW_WANT_EXCL (1 << LCK_RW_WANT_EXCL_BIT) +#define LCK_RW_TAG_VALID (1 << LCK_RW_TAG_VALID_BIT) +#define LCK_RW_PRIV_EXCL (1 << LCK_RW_PRIV_EXCL_BIT) +#define LCK_RW_SHARED_MASK (0xffff << LCK_RW_SHARED_READER_OFFSET) +#define LCK_RW_SHARED_READER (0x1 << LCK_RW_SHARED_READER_OFFSET) + +#define LCK_RW_TAG_DESTROYED ((LCK_RW_TAG_VALID | 0xdddddeadu)) /* lock marked as Destroyed */ + +#define LCK_RW_WRITER_EVENT(lck) (event_t)((uintptr_t)(lck)+1) +#define LCK_RW_READER_EVENT(lck) (event_t)((uintptr_t)(lck)+2) +#define WRITE_EVENT_TO_RWLOCK(event) ((lck_rw_t *)((uintptr_t)(event)-1)) +#define READ_EVENT_TO_RWLOCK(event) ((lck_rw_t *)((uintptr_t)(event)-2)) + +#if __ARM_ENABLE_WFE_ + +#define wait_for_event() __builtin_arm_wfe() +#if __arm__ +#define set_event() do{__builtin_arm_dsb(DSB_ISHST);__builtin_arm_sev();}while(0) +#define LOCK_SNOOP_SPINS 4 +#else +#define set_event() do{}while(0) // arm64 sev is implicit in stlxr +#define LOCK_SNOOP_SPINS 0x300 +#endif + +#else + +#define wait_for_event() __builtin_arm_clrex() +#define set_event() do{}while(0) +#define LOCK_SNOOP_SPINS 0x300 + +#endif // __ARM_ENABLE_WFE_ + +#if LOCK_PRIVATE + +#define LOCK_PANIC_TIMEOUT 0xc00000 // 12.5 m ticks = 250ms with 24MHz OSC + +#define LOCK_TRY_DISABLE_INT 1 // Disable interrupts for a quick acquire attempt + +#define PLATFORM_LCK_ILOCK LCK_ILOCK + + +/* + * Lock state to thread pointer + * Clear the bottom bits + */ +#define LCK_MTX_STATE_TO_THREAD(s) (thread_t)(s & ~(LCK_ILOCK | ARM_LCK_WAITERS)) +/* + * Thread pointer to lock state + * arm thread pointers are aligned such that the bottom two bits are clear + */ +#define LCK_MTX_THREAD_TO_STATE(t) ((uintptr_t)t) +/* + * Thread pointer mask + */ +#define LCK_MTX_THREAD_MASK (~(uintptr_t)(LCK_ILOCK | ARM_LCK_WAITERS)) + +#define disable_preemption_for_thread(t) ((volatile thread_t)t)->machine.preemption_count++ + + +__unused static void disable_interrupts_noread(void) +{ +#if __arm__ + __asm__ volatile ("cpsid if" ::: "memory"); // Mask IRQ FIQ +#else + __builtin_arm_wsr64("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF)); // Mask IRQ FIQ +#endif +} + +__unused static inline long get_interrupts(void) +{ + long state; + +#if __arm__ + __asm__ volatile ("mrs %[state], cpsr" :[state] "=r" (state)); // Read cpsr +#else + state = __builtin_arm_rsr64("DAIF"); // Read interrupt state +#endif + return state; +} + +__unused static inline long disable_interrupts(void) +{ + long state; + + state = get_interrupts(); // Get previous state + disable_interrupts_noread(); // Disable + return state; +} + +__unused static inline void restore_interrupts(long state) +{ +#if __arm__ + __asm__ volatile ("msr cpsr, %[state]" :: [state] "r" (state) : "cc", "memory"); // Restore CPSR +#elif __arm64__ + __builtin_arm_wsr64("DAIF", state); // Restore masks +#endif +} + +#endif // LOCK_PRIVATE + +#else +#ifdef KERNEL_PRIVATE +typedef struct { + uintptr_t opaque[2]; +} lck_rw_t; +#else +typedef struct __lck_rw_t__ lck_rw_t; +#endif +#endif + +#endif /* _ARM_LOCKS_H_ */ diff --git a/osfmk/arm/locks_arm.c b/osfmk/arm/locks_arm.c new file mode 100644 index 000000000..941ec38fc --- /dev/null +++ b/osfmk/arm/locks_arm.c @@ -0,0 +1,2882 @@ +/* + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System Copyright (c) 1991,1990,1989,1988,1987 Carnegie + * Mellon University All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright notice + * and this permission notice appear in all copies of the software, + * derivative works or modified versions, and any portions thereof, and that + * both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" CONDITION. + * CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ANY DAMAGES + * WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science Carnegie Mellon University Pittsburgh PA + * 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon the + * rights to redistribute these changes. + */ +/* + * File: kern/lock.c + * Author: Avadis Tevanian, Jr., Michael Wayne Young + * Date: 1985 + * + * Locking primitives implementation + */ + +#define ATOMIC_PRIVATE 1 +#define LOCK_PRIVATE 1 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +/* + * We need only enough declarations from the BSD-side to be able to + * test if our probe is active, and to call __dtrace_probe(). Setting + * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in. + */ +#if CONFIG_DTRACE +#define NEED_DTRACE_DEFS +#include <../bsd/sys/lockstat.h> + +#define DTRACE_RW_SHARED 0x0 //reader +#define DTRACE_RW_EXCL 0x1 //writer +#define DTRACE_NO_FLAG 0x0 //not applicable + +#endif /* CONFIG_DTRACE */ + +#define LCK_RW_LCK_EXCLUSIVE_CODE 0x100 +#define LCK_RW_LCK_EXCLUSIVE1_CODE 0x101 +#define LCK_RW_LCK_SHARED_CODE 0x102 +#define LCK_RW_LCK_SH_TO_EX_CODE 0x103 +#define LCK_RW_LCK_SH_TO_EX1_CODE 0x104 +#define LCK_RW_LCK_EX_TO_SH_CODE 0x105 + + +#define ANY_LOCK_DEBUG (USLOCK_DEBUG || LOCK_DEBUG || MUTEX_DEBUG) + +// Panic in tests that check lock usage correctness +// These are undesirable when in a panic or a debugger is runnning. +#define LOCK_CORRECTNESS_PANIC() (kernel_debugger_entry_count == 0) + +unsigned int LcksOpts = 0; + +#if CONFIG_DTRACE && __SMP__ +extern uint64_t dtrace_spin_threshold; +#endif + +/* Forwards */ + + +#if USLOCK_DEBUG +/* + * Perform simple lock checks. + */ +int uslock_check = 1; +int max_lock_loops = 100000000; +decl_simple_lock_data(extern, printf_lock) +decl_simple_lock_data(extern, panic_lock) +#endif /* USLOCK_DEBUG */ + +extern unsigned int not_in_kdp; + +/* + * We often want to know the addresses of the callers + * of the various lock routines. However, this information + * is only used for debugging and statistics. + */ +typedef void *pc_t; +#define INVALID_PC ((void *) VM_MAX_KERNEL_ADDRESS) +#define INVALID_THREAD ((void *) VM_MAX_KERNEL_ADDRESS) + +#ifdef lint +/* + * Eliminate lint complaints about unused local pc variables. + */ +#define OBTAIN_PC(pc,l) ++pc +#else /* lint */ +#define OBTAIN_PC(pc,l) +#endif /* lint */ + + +/* + * Portable lock package implementation of usimple_locks. + */ + +#if USLOCK_DEBUG +#define USLDBG(stmt) stmt + void usld_lock_init(usimple_lock_t, unsigned short); + void usld_lock_pre(usimple_lock_t, pc_t); + void usld_lock_post(usimple_lock_t, pc_t); + void usld_unlock(usimple_lock_t, pc_t); + void usld_lock_try_pre(usimple_lock_t, pc_t); + void usld_lock_try_post(usimple_lock_t, pc_t); + int usld_lock_common_checks(usimple_lock_t, const char *); +#else /* USLOCK_DEBUG */ +#define USLDBG(stmt) +#endif /* USLOCK_DEBUG */ + +/* + * Owner thread pointer when lock held in spin mode + */ +#define LCK_MTX_SPIN_TAG 0xfffffff0 + + +#define interlock_lock(lock) hw_lock_bit ((hw_lock_bit_t*)(&(lock)->lck_mtx_data), LCK_ILOCK_BIT) +#define interlock_try(lock) hw_lock_bit_try((hw_lock_bit_t*)(&(lock)->lck_mtx_data), LCK_ILOCK_BIT) +#define interlock_unlock(lock) hw_unlock_bit ((hw_lock_bit_t*)(&(lock)->lck_mtx_data), LCK_ILOCK_BIT) +#define lck_rw_ilk_lock(lock) hw_lock_bit ((hw_lock_bit_t*)(&(lock)->lck_rw_tag), LCK_RW_INTERLOCK_BIT) +#define lck_rw_ilk_unlock(lock) hw_unlock_bit((hw_lock_bit_t*)(&(lock)->lck_rw_tag), LCK_RW_INTERLOCK_BIT) + +#define memory_barrier() __c11_atomic_thread_fence(memory_order_acq_rel_smp) +#define load_memory_barrier() __c11_atomic_thread_fence(memory_order_acquire_smp) +#define store_memory_barrier() __c11_atomic_thread_fence(memory_order_release_smp) + +// Enforce program order of loads and stores. +#define ordered_load(target, type) \ + __c11_atomic_load((_Atomic type *)(target), memory_order_relaxed) +#define ordered_store(target, type, value) \ + __c11_atomic_store((_Atomic type *)(target), value, memory_order_relaxed) + +#define ordered_load_mtx(lock) ordered_load(&(lock)->lck_mtx_data, uintptr_t) +#define ordered_store_mtx(lock, value) ordered_store(&(lock)->lck_mtx_data, uintptr_t, (value)) +#define ordered_load_rw(lock) ordered_load(&(lock)->lck_rw_data, uint32_t) +#define ordered_store_rw(lock, value) ordered_store(&(lock)->lck_rw_data, uint32_t, (value)) +#define ordered_load_rw_owner(lock) ordered_load(&(lock)->lck_rw_owner, thread_t) +#define ordered_store_rw_owner(lock, value) ordered_store(&(lock)->lck_rw_owner, thread_t, (value)) +#define ordered_load_hw(lock) ordered_load(&(lock)->lock_data, uintptr_t) +#define ordered_store_hw(lock, value) ordered_store(&(lock)->lock_data, uintptr_t, (value)) +#define ordered_load_bit(lock) ordered_load((lock), uint32_t) +#define ordered_store_bit(lock, value) ordered_store((lock), uint32_t, (value)) + + +// Prevent the compiler from reordering memory operations around this +#define compiler_memory_fence() __asm__ volatile ("" ::: "memory") + +#define LOCK_PANIC_TIMEOUT 0xc00000 +#define NOINLINE __attribute__((noinline)) + + +#if __arm__ +#define interrupts_disabled(mask) (mask & PSR_INTMASK) +#else +#define interrupts_disabled(mask) (mask & DAIF_IRQF) +#endif + + +#if __arm__ +#define enable_fiq() __asm__ volatile ("cpsie f" ::: "memory"); +#define enable_interrupts() __asm__ volatile ("cpsie if" ::: "memory"); +#endif + +/* + * Forward declarations + */ + +static void lck_rw_lock_shared_gen(lck_rw_t *lck); +static void lck_rw_lock_exclusive_gen(lck_rw_t *lck); +static boolean_t lck_rw_lock_shared_to_exclusive_success(lck_rw_t *lck); +static boolean_t lck_rw_lock_shared_to_exclusive_failure(lck_rw_t *lck, uint32_t prior_lock_state); +static void lck_rw_lock_exclusive_to_shared_gen(lck_rw_t *lck, uint32_t prior_lock_state); +static lck_rw_type_t lck_rw_done_gen(lck_rw_t *lck, uint32_t prior_lock_state); +void lck_rw_clear_promotions_x86(thread_t thread); +static boolean_t lck_rw_grab(lck_rw_t *lock, int mode, boolean_t wait); + +/* + * atomic exchange API is a low level abstraction of the operations + * to atomically read, modify, and write a pointer. This abstraction works + * for both Intel and ARMv8.1 compare and exchange atomic instructions as + * well as the ARM exclusive instructions. + * + * atomic_exchange_begin() - begin exchange and retrieve current value + * atomic_exchange_complete() - conclude an exchange + * atomic_exchange_abort() - cancel an exchange started with atomic_exchange_begin() + */ +static uint32_t +atomic_exchange_begin32(uint32_t *target, uint32_t *previous, enum memory_order ord) +{ + uint32_t val; + + val = load_exclusive32(target, ord); + *previous = val; + return val; +} + +static boolean_t +atomic_exchange_complete32(uint32_t *target, uint32_t previous, uint32_t newval, enum memory_order ord) +{ + (void)previous; // Previous not needed, monitor is held + return store_exclusive32(target, newval, ord); +} + +static void +atomic_exchange_abort(void) +{ + clear_exclusive(); +} + +static boolean_t +atomic_test_and_set32(uint32_t *target, uint32_t test_mask, uint32_t set_mask, enum memory_order ord, boolean_t wait) +{ + uint32_t value, prev; + + for ( ; ; ) { + value = atomic_exchange_begin32(target, &prev, ord); + if (value & test_mask) { + if (wait) + wait_for_event(); // Wait with monitor held + else + atomic_exchange_abort(); // Clear exclusive monitor + return FALSE; + } + value |= set_mask; + if (atomic_exchange_complete32(target, prev, value, ord)) + return TRUE; + } +} + +void _disable_preemption(void) +{ + thread_t thread = current_thread(); + unsigned int count; + + count = thread->machine.preemption_count + 1; + ordered_store(&thread->machine.preemption_count, unsigned int, count); +} + +void _enable_preemption(void) +{ + thread_t thread = current_thread(); + long state; + unsigned int count; +#if __arm__ +#define INTERRUPT_MASK PSR_IRQF +#else // __arm__ +#define INTERRUPT_MASK DAIF_IRQF +#endif // __arm__ + + count = thread->machine.preemption_count; + if (count == 0) + panic("Preemption count negative"); // Count will go negative when released + count--; + if (count > 0) + goto update_count; // Preemption is still disabled, just update + state = get_interrupts(); // Get interrupt state + if (state & INTERRUPT_MASK) + goto update_count; // Interrupts are already masked, can't take AST here + + disable_interrupts_noread(); // Disable interrupts + ordered_store(&thread->machine.preemption_count, unsigned int, count); + if (thread->machine.CpuDatap->cpu_pending_ast & AST_URGENT) { +#if __arm__ +#if __ARM_USER_PROTECT__ + uintptr_t up = arm_user_protect_begin(thread); +#endif // __ARM_USER_PROTECT__ + enable_fiq(); +#endif // __arm__ + ast_taken_kernel(); // Handle urgent AST +#if __arm__ +#if __ARM_USER_PROTECT__ + arm_user_protect_end(thread, up, TRUE); +#endif // __ARM_USER_PROTECT__ + enable_interrupts(); + return; // Return early on arm only due to FIQ enabling +#endif // __arm__ + } + restore_interrupts(state); // Enable interrupts + return; + +update_count: + ordered_store(&thread->machine.preemption_count, unsigned int, count); + return; +} + +int get_preemption_level(void) +{ + return current_thread()->machine.preemption_count; +} + +/* Forward declarations for unexported functions that are used externally */ +void hw_lock_bit(hw_lock_bit_t *lock, unsigned int bit); +void hw_unlock_bit(hw_lock_bit_t *lock, unsigned int bit); + +#if __SMP__ +static unsigned int +hw_lock_bit_to_contended(hw_lock_bit_t *lock, uint32_t mask, uint32_t timeout); +#endif + +unsigned int +hw_lock_bit_to(hw_lock_bit_t *lock, unsigned int bit, uint32_t timeout) +{ + unsigned int success = 0; + uint32_t mask = (1 << bit); +#if !__SMP__ + uint32_t state; +#endif + + _disable_preemption(); +#if __SMP__ + if (__improbable(!atomic_test_and_set32(lock, mask, mask, memory_order_acquire, FALSE))) + success = hw_lock_bit_to_contended(lock, mask, timeout); + else + success = 1; +#else // __SMP__ + (void)timeout; + state = ordered_load_bit(lock); + if (!(mask & state)) { + ordered_store_bit(lock, state | mask); + success = 1; + } +#endif // __SMP__ + +#if CONFIG_DTRACE + if (success) + LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_ACQUIRE, lock, bit); +#endif + + return success; +} + +#if __SMP__ +static unsigned int NOINLINE +hw_lock_bit_to_contended(hw_lock_bit_t *lock, uint32_t mask, uint32_t timeout) +{ + uint64_t end = 0; + int i; +#if CONFIG_DTRACE + uint64_t begin; + boolean_t dtrace_enabled = lockstat_probemap[LS_LCK_SPIN_LOCK_SPIN] != 0; + if (__improbable(dtrace_enabled)) + begin = mach_absolute_time(); +#endif + for ( ; ; ) { + for (i = 0; i < LOCK_SNOOP_SPINS; i++) { + // Always load-exclusive before wfe + // This grabs the monitor and wakes up on a release event + if (atomic_test_and_set32(lock, mask, mask, memory_order_acquire, TRUE)) { + goto end; + } + } + if (end == 0) + end = ml_get_timebase() + timeout; + else if (ml_get_timebase() >= end) + break; + } + return 0; +end: +#if CONFIG_DTRACE + if (__improbable(dtrace_enabled)) { + uint64_t spintime = mach_absolute_time() - begin; + if (spintime > dtrace_spin_threshold) + LOCKSTAT_RECORD2(LS_LCK_SPIN_LOCK_SPIN, lock, spintime, mask); + } +#endif + return 1; +} +#endif // __SMP__ + +void +hw_lock_bit(hw_lock_bit_t *lock, unsigned int bit) +{ + if (hw_lock_bit_to(lock, bit, LOCK_PANIC_TIMEOUT)) + return; +#if __SMP__ + panic("hw_lock_bit(): timed out (%p)", lock); +#else + panic("hw_lock_bit(): interlock held (%p)", lock); +#endif +} + +unsigned int +hw_lock_bit_try(hw_lock_bit_t *lock, unsigned int bit) +{ + long intmask; + uint32_t mask = (1 << bit); +#if !__SMP__ + uint32_t state; +#endif + boolean_t success = FALSE; + + intmask = disable_interrupts(); +#if __SMP__ + // TODO: consider weak (non-looping) atomic test-and-set + success = atomic_test_and_set32(lock, mask, mask, memory_order_acquire, FALSE); +#else + state = ordered_load_bit(lock); + if (!(mask & state)) { + ordered_store_bit(lock, state | mask); + success = TRUE; + } +#endif // __SMP__ + if (success) + disable_preemption(); + restore_interrupts(intmask); + +#if CONFIG_DTRACE + if (success) + LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_ACQUIRE, lock, bit); +#endif + + return success; +} + +/* + * Routine: hw_unlock_bit + * + * Release spin-lock. The second parameter is the bit number to test and set. + * Decrement the preemption level. + */ +void +hw_unlock_bit(hw_lock_bit_t *lock, unsigned int bit) +{ + uint32_t mask = (1 << bit); +#if !__SMP__ + uint32_t state; +#endif + +#if __SMP__ + __c11_atomic_fetch_and((_Atomic uint32_t *)lock, ~mask, memory_order_release); + set_event(); +#else // __SMP__ + state = ordered_load_bit(lock); + ordered_store_bit(lock, state & ~mask); +#endif // __SMP__ +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_SPIN_UNLOCK_RELEASE, lock, bit); +#endif + enable_preemption(); +} + + +/* + * Routine: lck_spin_alloc_init + */ +lck_spin_t * +lck_spin_alloc_init( + lck_grp_t * grp, + lck_attr_t * attr) +{ + lck_spin_t *lck; + + if ((lck = (lck_spin_t *) kalloc(sizeof(lck_spin_t))) != 0) + lck_spin_init(lck, grp, attr); + + return (lck); +} + +/* + * Routine: lck_spin_free + */ +void +lck_spin_free( + lck_spin_t * lck, + lck_grp_t * grp) +{ + lck_spin_destroy(lck, grp); + kfree((void *) lck, sizeof(lck_spin_t)); +} + +/* + * Routine: lck_spin_init + */ +void +lck_spin_init( + lck_spin_t * lck, + lck_grp_t * grp, + __unused lck_attr_t * attr) +{ + hw_lock_init(&lck->hwlock); + lck->type = LCK_SPIN_TYPE; + lck_grp_reference(grp); + lck_grp_lckcnt_incr(grp, LCK_TYPE_SPIN); + store_memory_barrier(); +} + +/* + * arm_usimple_lock is a lck_spin_t without a group or attributes + */ +void inline +arm_usimple_lock_init(simple_lock_t lck, __unused unsigned short initial_value) +{ + lck->type = LCK_SPIN_TYPE; + hw_lock_init(&lck->hwlock); + store_memory_barrier(); +} + + +/* + * Routine: lck_spin_lock + */ +void +lck_spin_lock(lck_spin_t *lock) +{ +#if DEVELOPMENT || DEBUG + if (lock->type != LCK_SPIN_TYPE) + panic("Invalid spinlock %p", lock); +#endif // DEVELOPMENT || DEBUG + hw_lock_lock(&lock->hwlock); +} + +/* + * Routine: lck_spin_try_lock + */ +int +lck_spin_try_lock(lck_spin_t *lock) +{ + return hw_lock_try(&lock->hwlock); +} + +/* + * Routine: lck_spin_unlock + */ +void +lck_spin_unlock(lck_spin_t *lock) +{ +#if DEVELOPMENT || DEBUG + if ((LCK_MTX_STATE_TO_THREAD(lock->lck_spin_data) != current_thread()) && LOCK_CORRECTNESS_PANIC()) + panic("Spinlock not owned by thread %p = %lx", lock, lock->lck_spin_data); + if (lock->type != LCK_SPIN_TYPE) + panic("Invalid spinlock type %p", lock); +#endif // DEVELOPMENT || DEBUG + hw_lock_unlock(&lock->hwlock); +} + +/* + * Routine: lck_spin_destroy + */ +void +lck_spin_destroy( + lck_spin_t * lck, + lck_grp_t * grp) +{ + if (lck->lck_spin_data == LCK_SPIN_TAG_DESTROYED) + return; + lck->lck_spin_data = LCK_SPIN_TAG_DESTROYED; + lck_grp_lckcnt_decr(grp, LCK_TYPE_SPIN); + lck_grp_deallocate(grp); +} + +/* + * Routine: kdp_lck_spin_is_acquired + * NOT SAFE: To be used only by kernel debugger to avoid deadlock. + */ +boolean_t +kdp_lck_spin_is_acquired(lck_spin_t *lck) { + if (not_in_kdp) { + panic("panic: spinlock acquired check done outside of kernel debugger"); + } + return ((lck->lck_spin_data & ~LCK_SPIN_TAG_DESTROYED) != 0) ? TRUE:FALSE; +} + +/* + * Initialize a usimple_lock. + * + * No change in preemption state. + */ +void +usimple_lock_init( + usimple_lock_t l, + unsigned short tag) +{ +#ifndef MACHINE_SIMPLE_LOCK + USLDBG(usld_lock_init(l, tag)); + hw_lock_init(&l->lck_spin_data); +#else + simple_lock_init((simple_lock_t) l, tag); +#endif +} + + +/* + * Acquire a usimple_lock. + * + * Returns with preemption disabled. Note + * that the hw_lock routines are responsible for + * maintaining preemption state. + */ +void +usimple_lock( + usimple_lock_t l) +{ +#ifndef MACHINE_SIMPLE_LOCK + pc_t pc; + + OBTAIN_PC(pc, l); + USLDBG(usld_lock_pre(l, pc)); + + if (!hw_lock_to(&l->lck_spin_data, LockTimeOut)) /* Try to get the lock + * with a timeout */ + panic("simple lock deadlock detection - l=%p, cpu=%d, ret=%p", &l, cpu_number(), pc); + + USLDBG(usld_lock_post(l, pc)); +#else + simple_lock((simple_lock_t) l); +#endif +} + + +extern void sync(void); + +/* + * Release a usimple_lock. + * + * Returns with preemption enabled. Note + * that the hw_lock routines are responsible for + * maintaining preemption state. + */ +void +usimple_unlock( + usimple_lock_t l) +{ +#ifndef MACHINE_SIMPLE_LOCK + pc_t pc; + + OBTAIN_PC(pc, l); + USLDBG(usld_unlock(l, pc)); + sync(); + hw_lock_unlock(&l->lck_spin_data); +#else + simple_unlock((simple_lock_t) l); +#endif +} + + +/* + * Conditionally acquire a usimple_lock. + * + * On success, returns with preemption disabled. + * On failure, returns with preemption in the same state + * as when first invoked. Note that the hw_lock routines + * are responsible for maintaining preemption state. + * + * XXX No stats are gathered on a miss; I preserved this + * behavior from the original assembly-language code, but + * doesn't it make sense to log misses? XXX + */ +unsigned int +usimple_lock_try( + usimple_lock_t l) +{ +#ifndef MACHINE_SIMPLE_LOCK + pc_t pc; + unsigned int success; + + OBTAIN_PC(pc, l); + USLDBG(usld_lock_try_pre(l, pc)); + if ((success = hw_lock_try(&l->lck_spin_data))) { + USLDBG(usld_lock_try_post(l, pc)); + } + return success; +#else + return (simple_lock_try((simple_lock_t) l)); +#endif +} + +#if USLOCK_DEBUG +/* + * States of a usimple_lock. The default when initializing + * a usimple_lock is setting it up for debug checking. + */ +#define USLOCK_CHECKED 0x0001 /* lock is being checked */ +#define USLOCK_TAKEN 0x0002 /* lock has been taken */ +#define USLOCK_INIT 0xBAA0 /* lock has been initialized */ +#define USLOCK_INITIALIZED (USLOCK_INIT|USLOCK_CHECKED) +#define USLOCK_CHECKING(l) (uslock_check && \ + ((l)->debug.state & USLOCK_CHECKED)) + +/* + * Trace activities of a particularly interesting lock. + */ +void usl_trace(usimple_lock_t, int, pc_t, const char *); + + +/* + * Initialize the debugging information contained + * in a usimple_lock. + */ +void +usld_lock_init( + usimple_lock_t l, + __unused unsigned short tag) +{ + if (l == USIMPLE_LOCK_NULL) + panic("lock initialization: null lock pointer"); + l->lock_type = USLOCK_TAG; + l->debug.state = uslock_check ? USLOCK_INITIALIZED : 0; + l->debug.lock_cpu = l->debug.unlock_cpu = 0; + l->debug.lock_pc = l->debug.unlock_pc = INVALID_PC; + l->debug.lock_thread = l->debug.unlock_thread = INVALID_THREAD; + l->debug.duration[0] = l->debug.duration[1] = 0; + l->debug.unlock_cpu = l->debug.unlock_cpu = 0; + l->debug.unlock_pc = l->debug.unlock_pc = INVALID_PC; + l->debug.unlock_thread = l->debug.unlock_thread = INVALID_THREAD; +} + + +/* + * These checks apply to all usimple_locks, not just + * those with USLOCK_CHECKED turned on. + */ +int +usld_lock_common_checks( + usimple_lock_t l, + const char *caller) +{ + if (l == USIMPLE_LOCK_NULL) + panic("%s: null lock pointer", caller); + if (l->lock_type != USLOCK_TAG) + panic("%s: 0x%x is not a usimple lock", caller, (integer_t) l); + if (!(l->debug.state & USLOCK_INIT)) + panic("%s: 0x%x is not an initialized lock", + caller, (integer_t) l); + return USLOCK_CHECKING(l); +} + + +/* + * Debug checks on a usimple_lock just before attempting + * to acquire it. + */ +/* ARGSUSED */ +void +usld_lock_pre( + usimple_lock_t l, + pc_t pc) +{ + const char *caller = "usimple_lock"; + + + if (!usld_lock_common_checks(l, caller)) + return; + + /* + * Note that we have a weird case where we are getting a lock when we are] + * in the process of putting the system to sleep. We are running with no + * current threads, therefore we can't tell if we are trying to retake a lock + * we have or someone on the other processor has it. Therefore we just + * ignore this test if the locking thread is 0. + */ + + if ((l->debug.state & USLOCK_TAKEN) && l->debug.lock_thread && + l->debug.lock_thread == (void *) current_thread()) { + printf("%s: lock 0x%x already locked (at %p) by", + caller, (integer_t) l, l->debug.lock_pc); + printf(" current thread %p (new attempt at pc %p)\n", + l->debug.lock_thread, pc); + panic("%s", caller); + } + mp_disable_preemption(); + usl_trace(l, cpu_number(), pc, caller); + mp_enable_preemption(); +} + + +/* + * Debug checks on a usimple_lock just after acquiring it. + * + * Pre-emption has been disabled at this point, + * so we are safe in using cpu_number. + */ +void +usld_lock_post( + usimple_lock_t l, + pc_t pc) +{ + int mycpu; + const char *caller = "successful usimple_lock"; + + + if (!usld_lock_common_checks(l, caller)) + return; + + if (!((l->debug.state & ~USLOCK_TAKEN) == USLOCK_INITIALIZED)) + panic("%s: lock 0x%x became uninitialized", + caller, (integer_t) l); + if ((l->debug.state & USLOCK_TAKEN)) + panic("%s: lock 0x%x became TAKEN by someone else", + caller, (integer_t) l); + + mycpu = cpu_number(); + l->debug.lock_thread = (void *) current_thread(); + l->debug.state |= USLOCK_TAKEN; + l->debug.lock_pc = pc; + l->debug.lock_cpu = mycpu; + + usl_trace(l, mycpu, pc, caller); +} + + +/* + * Debug checks on a usimple_lock just before + * releasing it. Note that the caller has not + * yet released the hardware lock. + * + * Preemption is still disabled, so there's + * no problem using cpu_number. + */ +void +usld_unlock( + usimple_lock_t l, + pc_t pc) +{ + int mycpu; + const char *caller = "usimple_unlock"; + + + if (!usld_lock_common_checks(l, caller)) + return; + + mycpu = cpu_number(); + + if (!(l->debug.state & USLOCK_TAKEN)) + panic("%s: lock 0x%x hasn't been taken", + caller, (integer_t) l); + if (l->debug.lock_thread != (void *) current_thread()) + panic("%s: unlocking lock 0x%x, owned by thread %p", + caller, (integer_t) l, l->debug.lock_thread); + if (l->debug.lock_cpu != mycpu) { + printf("%s: unlocking lock 0x%x on cpu 0x%x", + caller, (integer_t) l, mycpu); + printf(" (acquired on cpu 0x%x)\n", l->debug.lock_cpu); + panic("%s", caller); + } + usl_trace(l, mycpu, pc, caller); + + l->debug.unlock_thread = l->debug.lock_thread; + l->debug.lock_thread = INVALID_PC; + l->debug.state &= ~USLOCK_TAKEN; + l->debug.unlock_pc = pc; + l->debug.unlock_cpu = mycpu; +} + + +/* + * Debug checks on a usimple_lock just before + * attempting to acquire it. + * + * Preemption isn't guaranteed to be disabled. + */ +void +usld_lock_try_pre( + usimple_lock_t l, + pc_t pc) +{ + const char *caller = "usimple_lock_try"; + + if (!usld_lock_common_checks(l, caller)) + return; + mp_disable_preemption(); + usl_trace(l, cpu_number(), pc, caller); + mp_enable_preemption(); +} + + +/* + * Debug checks on a usimple_lock just after + * successfully attempting to acquire it. + * + * Preemption has been disabled by the + * lock acquisition attempt, so it's safe + * to use cpu_number. + */ +void +usld_lock_try_post( + usimple_lock_t l, + pc_t pc) +{ + int mycpu; + const char *caller = "successful usimple_lock_try"; + + if (!usld_lock_common_checks(l, caller)) + return; + + if (!((l->debug.state & ~USLOCK_TAKEN) == USLOCK_INITIALIZED)) + panic("%s: lock 0x%x became uninitialized", + caller, (integer_t) l); + if ((l->debug.state & USLOCK_TAKEN)) + panic("%s: lock 0x%x became TAKEN by someone else", + caller, (integer_t) l); + + mycpu = cpu_number(); + l->debug.lock_thread = (void *) current_thread(); + l->debug.state |= USLOCK_TAKEN; + l->debug.lock_pc = pc; + l->debug.lock_cpu = mycpu; + + usl_trace(l, mycpu, pc, caller); +} + + +/* + * For very special cases, set traced_lock to point to a + * specific lock of interest. The result is a series of + * XPRs showing lock operations on that lock. The lock_seq + * value is used to show the order of those operations. + */ +usimple_lock_t traced_lock; +unsigned int lock_seq; + +void +usl_trace( + usimple_lock_t l, + int mycpu, + pc_t pc, + const char *op_name) +{ + if (traced_lock == l) { + XPR(XPR_SLOCK, + "seq %d, cpu %d, %s @ %x\n", + (integer_t) lock_seq, (integer_t) mycpu, + (integer_t) op_name, (integer_t) pc, 0); + lock_seq++; + } +} + + +#endif /* USLOCK_DEBUG */ + +/* + * The C portion of the shared/exclusive locks package. + */ + +/* + * compute the deadline to spin against when + * waiting for a change of state on a lck_rw_t + */ +#if __SMP__ +static inline uint64_t +lck_rw_deadline_for_spin(lck_rw_t *lck) +{ + lck_rw_word_t word; + + word.data = ordered_load_rw(lck); + if (word.can_sleep) { + if (word.r_waiting || word.w_waiting || (word.shared_count > machine_info.max_cpus)) { + /* + * there are already threads waiting on this lock... this + * implies that they have spun beyond their deadlines waiting for + * the desired state to show up so we will not bother spinning at this time... + * or + * the current number of threads sharing this lock exceeds our capacity to run them + * concurrently and since all states we're going to spin for require the rw_shared_count + * to be at 0, we'll not bother spinning since the latency for this to happen is + * unpredictable... + */ + return (mach_absolute_time()); + } + return (mach_absolute_time() + MutexSpin); + } else + return (mach_absolute_time() + (100000LL * 1000000000LL)); +} +#endif // __SMP__ + +static boolean_t +lck_rw_drain_status(lck_rw_t *lock, uint32_t status_mask, boolean_t wait __unused) +{ +#if __SMP__ + uint64_t deadline = 0; + uint32_t data; + + if (wait) + deadline = lck_rw_deadline_for_spin(lock); + + for ( ; ; ) { + data = load_exclusive32(&lock->lck_rw_data, memory_order_acquire_smp); + if ((data & status_mask) == 0) + break; + if (wait) + wait_for_event(); + else + clear_exclusive(); + if (!wait || (mach_absolute_time() >= deadline)) + return FALSE; + } + clear_exclusive(); + return TRUE; +#else + uint32_t data; + + data = ordered_load_rw(lock); + if ((data & status_mask) == 0) + return TRUE; + else + return FALSE; +#endif // __SMP__ +} + +/* + * Spin while interlock is held. + */ +static inline void +lck_rw_interlock_spin(lck_rw_t *lock) +{ +#if __SMP__ + uint32_t data; + + for ( ; ; ) { + data = load_exclusive32(&lock->lck_rw_data, memory_order_relaxed); + if (data & LCK_RW_INTERLOCK) + wait_for_event(); + else { + clear_exclusive(); + return; + } + } +#else + panic("lck_rw_interlock_spin(): Interlock locked %p %x", lock, lock->lck_rw_data); +#endif +} + +/* + * We disable interrupts while holding the RW interlock to prevent an + * interrupt from exacerbating hold time. + * Hence, local helper functions lck_interlock_lock()/lck_interlock_unlock(). + */ +static inline boolean_t +lck_interlock_lock(lck_rw_t *lck) +{ + boolean_t istate; + + istate = ml_set_interrupts_enabled(FALSE); + lck_rw_ilk_lock(lck); + return istate; +} + +static inline void +lck_interlock_unlock(lck_rw_t *lck, boolean_t istate) +{ + lck_rw_ilk_unlock(lck); + ml_set_interrupts_enabled(istate); +} + + +#define LCK_RW_GRAB_WANT 0 +#define LCK_RW_GRAB_SHARED 1 + +static boolean_t +lck_rw_grab(lck_rw_t *lock, int mode, boolean_t wait) +{ + uint64_t deadline = 0; + uint32_t data, prev; + boolean_t do_exch; + +#if __SMP__ + if (wait) + deadline = lck_rw_deadline_for_spin(lock); +#else + wait = FALSE; // Don't spin on UP systems +#endif + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; + } + do_exch = FALSE; + if (mode == LCK_RW_GRAB_WANT) { + if ((data & LCK_RW_WANT_EXCL) == 0) { + data |= LCK_RW_WANT_EXCL; + do_exch = TRUE; + } + } else { // LCK_RW_GRAB_SHARED + if (((data & (LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE)) == 0) || + (((data & LCK_RW_SHARED_MASK)) && ((data & LCK_RW_PRIV_EXCL) == 0))) { + data += LCK_RW_SHARED_READER; + do_exch = TRUE; + } + } + if (do_exch) { + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + return TRUE; + } else { + if (wait) // Non-waiting + wait_for_event(); + else + atomic_exchange_abort(); + if (!wait || (mach_absolute_time() >= deadline)) + return FALSE; + } + } +} + + +/* + * Routine: lck_rw_alloc_init + */ +lck_rw_t * +lck_rw_alloc_init( + lck_grp_t *grp, + lck_attr_t *attr) +{ + lck_rw_t *lck; + + if ((lck = (lck_rw_t *)kalloc(sizeof(lck_rw_t))) != 0) + lck_rw_init(lck, grp, attr); + + return lck; +} + +/* + * Routine: lck_rw_free + */ +void +lck_rw_free( + lck_rw_t *lck, + lck_grp_t *grp) +{ + lck_rw_destroy(lck, grp); + kfree(lck, sizeof(lck_rw_t)); +} + +/* + * Routine: lck_rw_init + */ +void +lck_rw_init( + lck_rw_t *lck, + lck_grp_t *grp, + lck_attr_t *attr) +{ + if (attr == LCK_ATTR_NULL) + attr = &LockDefaultLckAttr; + memset(lck, 0, sizeof(lck_rw_t)); + lck->lck_rw_can_sleep = TRUE; + if ((attr->lck_attr_val & LCK_ATTR_RW_SHARED_PRIORITY) == 0) + lck->lck_rw_priv_excl = TRUE; + + lck_grp_reference(grp); + lck_grp_lckcnt_incr(grp, LCK_TYPE_RW); +} + + +/* + * Routine: lck_rw_destroy + */ +void +lck_rw_destroy( + lck_rw_t *lck, + lck_grp_t *grp) +{ + if (lck->lck_rw_tag == LCK_RW_TAG_DESTROYED) + return; +#if MACH_LDEBUG + lck_rw_assert(lck, LCK_RW_ASSERT_NOTHELD); +#endif + lck->lck_rw_tag = LCK_RW_TAG_DESTROYED; + lck_grp_lckcnt_decr(grp, LCK_TYPE_RW); + lck_grp_deallocate(grp); + return; +} + +/* + * Routine: lck_rw_lock + */ +void +lck_rw_lock( + lck_rw_t *lck, + lck_rw_type_t lck_rw_type) +{ + if (lck_rw_type == LCK_RW_TYPE_SHARED) + lck_rw_lock_shared(lck); + else if (lck_rw_type == LCK_RW_TYPE_EXCLUSIVE) + lck_rw_lock_exclusive(lck); + else + panic("lck_rw_lock(): Invalid RW lock type: %x", lck_rw_type); +} + +/* + * Routine: lck_rw_lock_exclusive + */ +void +lck_rw_lock_exclusive(lck_rw_t *lock) +{ + thread_t thread = current_thread(); + + thread->rwlock_count++; + if (atomic_test_and_set32(&lock->lck_rw_data, + (LCK_RW_SHARED_MASK | LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE | LCK_RW_INTERLOCK), + LCK_RW_WANT_EXCL, memory_order_acquire_smp, FALSE)) { +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, lock, DTRACE_RW_EXCL); +#endif /* CONFIG_DTRACE */ + } else + lck_rw_lock_exclusive_gen(lock); +#if MACH_ASSERT + thread_t owner = ordered_load_rw_owner(lock); + assertf(owner == THREAD_NULL, "state=0x%x, owner=%p", ordered_load_rw(lock), owner); +#endif + ordered_store_rw_owner(lock, thread); +} + +/* + * Routine: lck_rw_lock_shared + */ +void +lck_rw_lock_shared(lck_rw_t *lock) +{ + uint32_t data, prev; + + current_thread()->rwlock_count++; + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_acquire_smp); + if (data & (LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE | LCK_RW_INTERLOCK)) { + atomic_exchange_abort(); + lck_rw_lock_shared_gen(lock); + break; + } + data += LCK_RW_SHARED_READER; + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + break; + cpu_pause(); + } +#if MACH_ASSERT + thread_t owner = ordered_load_rw_owner(lock); + assertf(owner == THREAD_NULL, "state=0x%x, owner=%p", ordered_load_rw(lock), owner); +#endif +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, lock, DTRACE_RW_SHARED); +#endif /* CONFIG_DTRACE */ + return; +} + +/* + * Routine: lck_rw_lock_shared_to_exclusive + */ +boolean_t +lck_rw_lock_shared_to_exclusive(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; + } + if (data & LCK_RW_WANT_UPGRADE) { + data -= LCK_RW_SHARED_READER; + if ((data & LCK_RW_SHARED_MASK) == 0) /* we were the last reader */ + data &= ~(LCK_RW_W_WAITING); /* so clear the wait indicator */ + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + return lck_rw_lock_shared_to_exclusive_failure(lock, prev); + } else { + data |= LCK_RW_WANT_UPGRADE; /* ask for WANT_UPGRADE */ + data -= LCK_RW_SHARED_READER; /* and shed our read count */ + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + break; + } + cpu_pause(); + } + /* we now own the WANT_UPGRADE */ + if (data & LCK_RW_SHARED_MASK) /* check to see if all of the readers are drained */ + lck_rw_lock_shared_to_exclusive_success(lock); /* if not, we need to go wait */ +#if MACH_ASSERT + thread_t owner = ordered_load_rw_owner(lock); + assertf(owner == THREAD_NULL, "state=0x%x, owner=%p", ordered_load_rw(lock), owner); +#endif + ordered_store_rw_owner(lock, current_thread()); +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, lock, 0); +#endif /* CONFIG_DTRACE */ + return TRUE; +} + + +/* + * Routine: lck_rw_lock_shared_to_exclusive_failure + * Function: + * Fast path code has already dropped our read + * count and determined that someone else owns 'lck_rw_want_upgrade' + * if 'lck_rw_shared_count' == 0, its also already dropped 'lck_w_waiting' + * all we need to do here is determine if a wakeup is needed + */ +static boolean_t +lck_rw_lock_shared_to_exclusive_failure( + lck_rw_t *lck, + uint32_t prior_lock_state) +{ + thread_t thread = current_thread(); + uint32_t rwlock_count; + + /* Check if dropping the lock means that we need to unpromote */ + rwlock_count = thread->rwlock_count--; +#if MACH_LDEBUG + if (rwlock_count == 0) { + panic("rw lock count underflow for thread %p", thread); + } +#endif + if ((prior_lock_state & LCK_RW_W_WAITING) && + ((prior_lock_state & LCK_RW_SHARED_MASK) == LCK_RW_SHARED_READER)) { + /* + * Someone else has requested upgrade. + * Since we've released the read lock, wake + * him up if he's blocked waiting + */ + thread_wakeup(LCK_RW_WRITER_EVENT(lck)); + } + + if ((rwlock_count == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { + /* sched_flags checked without lock, but will be rechecked while clearing */ + lck_rw_clear_promotion(thread); + } + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | DBG_FUNC_NONE, + VM_KERNEL_UNSLIDE_OR_PERM(lck), lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, 0, 0); + + return (FALSE); +} + +/* + * Routine: lck_rw_lock_shared_to_exclusive_success + * Function: + * assembly fast path code has already dropped our read + * count and successfully acquired 'lck_rw_want_upgrade' + * we just need to wait for the rest of the readers to drain + * and then we can return as the exclusive holder of this lock + */ +static boolean_t +lck_rw_lock_shared_to_exclusive_success( + lck_rw_t *lock) +{ + __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lock); + int slept = 0; + lck_rw_word_t word; + wait_result_t res; + boolean_t istate; + boolean_t not_shared; + +#if CONFIG_DTRACE + uint64_t wait_interval = 0; + int readers_at_sleep = 0; + boolean_t dtrace_ls_initialized = FALSE; + boolean_t dtrace_rwl_shared_to_excl_spin, dtrace_rwl_shared_to_excl_block, dtrace_ls_enabled = FALSE; +#endif + + while (!lck_rw_drain_status(lock, LCK_RW_SHARED_MASK, FALSE)) { + + word.data = ordered_load_rw(lock); +#if CONFIG_DTRACE + if (dtrace_ls_initialized == FALSE) { + dtrace_ls_initialized = TRUE; + dtrace_rwl_shared_to_excl_spin = (lockstat_probemap[LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN] != 0); + dtrace_rwl_shared_to_excl_block = (lockstat_probemap[LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK] != 0); + dtrace_ls_enabled = dtrace_rwl_shared_to_excl_spin || dtrace_rwl_shared_to_excl_block; + if (dtrace_ls_enabled) { + /* + * Either sleeping or spinning is happening, + * start a timing of our delay interval now. + */ + readers_at_sleep = word.shared_count; + wait_interval = mach_absolute_time(); + } + } +#endif + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_SPIN_CODE) | DBG_FUNC_START, + trace_lck, word.shared_count, 0, 0, 0); + + not_shared = lck_rw_drain_status(lock, LCK_RW_SHARED_MASK, TRUE); + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_SPIN_CODE) | DBG_FUNC_END, + trace_lck, lock->lck_rw_shared_count, 0, 0, 0); + + if (not_shared) + break; + + /* + * if we get here, the spin deadline in lck_rw_wait_on_status() + * has expired w/o the rw_shared_count having drained to 0 + * check to see if we're allowed to do a thread_block + */ + if (word.can_sleep) { + + istate = lck_interlock_lock(lock); + + word.data = ordered_load_rw(lock); + if (word.shared_count != 0) { + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_WAIT_CODE) | DBG_FUNC_START, + trace_lck, word.shared_count, 0, 0, 0); + + word.w_waiting = 1; + ordered_store_rw(lock, word.data); + + thread_set_pending_block_hint(current_thread(), kThreadWaitKernelRWLockUpgrade); + res = assert_wait(LCK_RW_WRITER_EVENT(lock), THREAD_UNINT); + lck_interlock_unlock(lock, istate); + + if (res == THREAD_WAITING) { + res = thread_block(THREAD_CONTINUE_NULL); + slept++; + } + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_WAIT_CODE) | DBG_FUNC_END, + trace_lck, res, slept, 0, 0); + } else { + lck_interlock_unlock(lock, istate); + break; + } + } + } +#if CONFIG_DTRACE + /* + * We infer whether we took the sleep/spin path above by checking readers_at_sleep. + */ + if (dtrace_ls_enabled == TRUE) { + if (slept == 0) { + LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, lock, mach_absolute_time() - wait_interval, 0); + } else { + LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, lock, + mach_absolute_time() - wait_interval, 1, + (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep); + } + } + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, lock, 1); +#endif + return (TRUE); +} + + +/* + * Routine: lck_rw_lock_exclusive_to_shared + */ + +void lck_rw_lock_exclusive_to_shared(lck_rw_t *lock) +{ + uint32_t data, prev; + + assertf(lock->lck_rw_owner == current_thread(), "state=0x%x, owner=%p", lock->lck_rw_data, lock->lck_rw_owner); + ordered_store_rw_owner(lock, THREAD_NULL); + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_release_smp); + if (data & LCK_RW_INTERLOCK) { +#if __SMP__ + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); /* wait for interlock to clear */ + continue; +#else + panic("lck_rw_lock_exclusive_to_shared(): Interlock locked (%p): %x", lock, data); +#endif // __SMP__ + } + data += LCK_RW_SHARED_READER; + if (data & LCK_RW_WANT_UPGRADE) + data &= ~(LCK_RW_WANT_UPGRADE); + else + data &= ~(LCK_RW_WANT_EXCL); + if (!((prev & LCK_RW_W_WAITING) && (prev & LCK_RW_PRIV_EXCL))) + data &= ~(LCK_RW_W_WAITING); + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_release_smp)) + break; + cpu_pause(); + } + return lck_rw_lock_exclusive_to_shared_gen(lock, prev); +} + +/* + * Routine: lck_rw_lock_exclusive_to_shared_gen + * Function: + * Fast path has already dropped + * our exclusive state and bumped lck_rw_shared_count + * all we need to do here is determine if anyone + * needs to be awakened. + */ +static void +lck_rw_lock_exclusive_to_shared_gen( + lck_rw_t *lck, + uint32_t prior_lock_state) +{ + __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lck); + lck_rw_word_t fake_lck; + + /* + * prior_lock state is a snapshot of the 1st word of the + * lock in question... we'll fake up a pointer to it + * and carefully not access anything beyond whats defined + * in the first word of a lck_rw_t + */ + fake_lck.data = prior_lock_state; + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_TO_SH_CODE) | DBG_FUNC_START, + trace_lck, fake_lck->want_excl, fake_lck->want_upgrade, 0, 0); + + /* + * don't wake up anyone waiting to take the lock exclusively + * since we hold a read count... when the read count drops to 0, + * the writers will be woken. + * + * wake up any waiting readers if we don't have any writers waiting, + * or the lock is NOT marked as rw_priv_excl (writers have privilege) + */ + if (!(fake_lck.priv_excl && fake_lck.w_waiting) && fake_lck.r_waiting) + thread_wakeup(LCK_RW_READER_EVENT(lck)); + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_TO_SH_CODE) | DBG_FUNC_END, + trace_lck, lck->lck_rw_want_excl, lck->lck_rw_want_upgrade, lck->lck_rw_shared_count, 0); + +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, lck, 0); +#endif +} + + +/* + * Routine: lck_rw_try_lock + */ +boolean_t +lck_rw_try_lock( + lck_rw_t *lck, + lck_rw_type_t lck_rw_type) +{ + if (lck_rw_type == LCK_RW_TYPE_SHARED) + return lck_rw_try_lock_shared(lck); + else if (lck_rw_type == LCK_RW_TYPE_EXCLUSIVE) + return lck_rw_try_lock_exclusive(lck); + else + panic("lck_rw_try_lock(): Invalid rw lock type: %x", lck_rw_type); + return FALSE; +} + +/* + * Routine: lck_rw_try_lock_shared + */ + +boolean_t lck_rw_try_lock_shared(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { +#if __SMP__ + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; +#else + panic("lck_rw_try_lock_shared(): Interlock locked (%p): %x", lock, data); +#endif + } + if (data & (LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE)) { + atomic_exchange_abort(); + return FALSE; /* lock is busy */ + } + data += LCK_RW_SHARED_READER; /* Increment reader refcount */ + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + break; + cpu_pause(); + } +#if MACH_ASSERT + thread_t owner = ordered_load_rw_owner(lock); + assertf(owner == THREAD_NULL, "state=0x%x, owner=%p", ordered_load_rw(lock), owner); +#endif + current_thread()->rwlock_count++; +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, lock, DTRACE_RW_SHARED); +#endif /* CONFIG_DTRACE */ + return TRUE; +} + + +/* + * Routine: lck_rw_try_lock_exclusive + */ + +boolean_t lck_rw_try_lock_exclusive(lck_rw_t *lock) +{ + uint32_t data, prev; + thread_t thread; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { +#if __SMP__ + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; +#else + panic("lck_rw_try_lock_exclusive(): Interlock locked (%p): %x", lock, data); +#endif + } + if (data & (LCK_RW_SHARED_MASK | LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE)) { + atomic_exchange_abort(); + return FALSE; + } + data |= LCK_RW_WANT_EXCL; + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + break; + cpu_pause(); + } + thread = current_thread(); + thread->rwlock_count++; +#if MACH_ASSERT + thread_t owner = ordered_load_rw_owner(lock); + assertf(owner == THREAD_NULL, "state=0x%x, owner=%p", ordered_load_rw(lock), owner); +#endif + ordered_store_rw_owner(lock, thread); +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, lock, DTRACE_RW_EXCL); +#endif /* CONFIG_DTRACE */ + return TRUE; +} + + +/* + * Routine: lck_rw_unlock + */ +void +lck_rw_unlock( + lck_rw_t *lck, + lck_rw_type_t lck_rw_type) +{ + if (lck_rw_type == LCK_RW_TYPE_SHARED) + lck_rw_unlock_shared(lck); + else if (lck_rw_type == LCK_RW_TYPE_EXCLUSIVE) + lck_rw_unlock_exclusive(lck); + else + panic("lck_rw_unlock(): Invalid RW lock type: %d", lck_rw_type); +} + + +/* + * Routine: lck_rw_unlock_shared + */ +void +lck_rw_unlock_shared( + lck_rw_t *lck) +{ + lck_rw_type_t ret; + + assertf(lck->lck_rw_owner == THREAD_NULL, "state=0x%x, owner=%p", lck->lck_rw_data, lck->lck_rw_owner); + assertf(lck->lck_rw_shared_count > 0, "shared_count=0x%x", lck->lck_rw_shared_count); + ret = lck_rw_done(lck); + + if (ret != LCK_RW_TYPE_SHARED) + panic("lck_rw_unlock_shared(): lock %p held in mode: %d", lck, ret); +} + + +/* + * Routine: lck_rw_unlock_exclusive + */ +void +lck_rw_unlock_exclusive( + lck_rw_t *lck) +{ + lck_rw_type_t ret; + + assertf(lck->lck_rw_owner == current_thread(), "state=0x%x, owner=%p", lck->lck_rw_data, lck->lck_rw_owner); + ret = lck_rw_done(lck); + + if (ret != LCK_RW_TYPE_EXCLUSIVE) + panic("lck_rw_unlock_exclusive(): lock %p held in mode: %d", lck, ret); +} + + +/* + * Routine: lck_rw_lock_exclusive_gen + */ +static void +lck_rw_lock_exclusive_gen( + lck_rw_t *lock) +{ + __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lock); + lck_rw_word_t word; + int slept = 0; + boolean_t gotlock = 0; + boolean_t not_shared_or_upgrade = 0; + wait_result_t res = 0; + boolean_t istate; + +#if CONFIG_DTRACE + boolean_t dtrace_ls_initialized = FALSE; + boolean_t dtrace_rwl_excl_spin, dtrace_rwl_excl_block, dtrace_ls_enabled= FALSE; + uint64_t wait_interval = 0; + int readers_at_sleep = 0; +#endif + + /* + * Try to acquire the lck_rw_want_excl bit. + */ + while (!lck_rw_grab(lock, LCK_RW_GRAB_WANT, FALSE)) { + +#if CONFIG_DTRACE + if (dtrace_ls_initialized == FALSE) { + dtrace_ls_initialized = TRUE; + dtrace_rwl_excl_spin = (lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] != 0); + dtrace_rwl_excl_block = (lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK] != 0); + dtrace_ls_enabled = dtrace_rwl_excl_spin || dtrace_rwl_excl_block; + if (dtrace_ls_enabled) { + /* + * Either sleeping or spinning is happening, + * start a timing of our delay interval now. + */ + readers_at_sleep = lock->lck_rw_shared_count; + wait_interval = mach_absolute_time(); + } + } +#endif + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_WRITER_SPIN_CODE) | DBG_FUNC_START, trace_lck, 0, 0, 0, 0); + + gotlock = lck_rw_grab(lock, LCK_RW_GRAB_WANT, TRUE); + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_WRITER_SPIN_CODE) | DBG_FUNC_END, trace_lck, 0, 0, gotlock, 0); + + if (gotlock) + break; + /* + * if we get here, the deadline has expired w/o us + * being able to grab the lock exclusively + * check to see if we're allowed to do a thread_block + */ + word.data = ordered_load_rw(lock); + if (word.can_sleep) { + + istate = lck_interlock_lock(lock); + word.data = ordered_load_rw(lock); + + if (word.want_excl) { + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_WRITER_WAIT_CODE) | DBG_FUNC_START, trace_lck, 0, 0, 0, 0); + + word.w_waiting = 1; + ordered_store_rw(lock, word.data); + + thread_set_pending_block_hint(current_thread(), kThreadWaitKernelRWLockWrite); + res = assert_wait(LCK_RW_WRITER_EVENT(lock), THREAD_UNINT); + lck_interlock_unlock(lock, istate); + + if (res == THREAD_WAITING) { + res = thread_block(THREAD_CONTINUE_NULL); + slept++; + } + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_WRITER_WAIT_CODE) | DBG_FUNC_END, trace_lck, res, slept, 0, 0); + } else { + word.want_excl = 1; + ordered_store_rw(lock, word.data); + lck_interlock_unlock(lock, istate); + break; + } + } + } + /* + * Wait for readers (and upgrades) to finish... + */ + while (!lck_rw_drain_status(lock, LCK_RW_SHARED_MASK | LCK_RW_WANT_UPGRADE, FALSE)) { + +#if CONFIG_DTRACE + /* + * Either sleeping or spinning is happening, start + * a timing of our delay interval now. If we set it + * to -1 we don't have accurate data so we cannot later + * decide to record a dtrace spin or sleep event. + */ + if (dtrace_ls_initialized == FALSE) { + dtrace_ls_initialized = TRUE; + dtrace_rwl_excl_spin = (lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] != 0); + dtrace_rwl_excl_block = (lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK] != 0); + dtrace_ls_enabled = dtrace_rwl_excl_spin || dtrace_rwl_excl_block; + if (dtrace_ls_enabled) { + /* + * Either sleeping or spinning is happening, + * start a timing of our delay interval now. + */ + readers_at_sleep = lock->lck_rw_shared_count; + wait_interval = mach_absolute_time(); + } + } +#endif + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_READER_SPIN_CODE) | DBG_FUNC_START, trace_lck, 0, 0, 0, 0); + + not_shared_or_upgrade = lck_rw_drain_status(lock, LCK_RW_SHARED_MASK | LCK_RW_WANT_UPGRADE, TRUE); + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_READER_SPIN_CODE) | DBG_FUNC_END, trace_lck, 0, 0, not_shared_or_upgrade, 0); + + if (not_shared_or_upgrade) + break; + /* + * if we get here, the deadline has expired w/o us + * being able to grab the lock exclusively + * check to see if we're allowed to do a thread_block + */ + word.data = ordered_load_rw(lock); + if (word.can_sleep) { + + istate = lck_interlock_lock(lock); + word.data = ordered_load_rw(lock); + + if (word.shared_count != 0 || word.want_upgrade) { + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_READER_WAIT_CODE) | DBG_FUNC_START, trace_lck, 0, 0, 0, 0); + + word.w_waiting = 1; + ordered_store_rw(lock, word.data); + + thread_set_pending_block_hint(current_thread(), kThreadWaitKernelRWLockWrite); + res = assert_wait(LCK_RW_WRITER_EVENT(lock), THREAD_UNINT); + lck_interlock_unlock(lock, istate); + + if (res == THREAD_WAITING) { + res = thread_block(THREAD_CONTINUE_NULL); + slept++; + } + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_READER_WAIT_CODE) | DBG_FUNC_END, trace_lck, res, slept, 0, 0); + } else { + lck_interlock_unlock(lock, istate); + /* + * must own the lock now, since we checked for + * readers or upgrade owner behind the interlock + * no need for a call to 'lck_rw_drain_status' + */ + break; + } + } + } + +#if CONFIG_DTRACE + /* + * Decide what latencies we suffered that are Dtrace events. + * If we have set wait_interval, then we either spun or slept. + * At least we get out from under the interlock before we record + * which is the best we can do here to minimize the impact + * of the tracing. + * If we have set wait_interval to -1, then dtrace was not enabled when we + * started sleeping/spinning so we don't record this event. + */ + if (dtrace_ls_enabled == TRUE) { + if (slept == 0) { + LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_EXCL_SPIN, lock, + mach_absolute_time() - wait_interval, 1); + } else { + /* + * For the blocking case, we also record if when we blocked + * it was held for read or write, and how many readers. + * Notice that above we recorded this before we dropped + * the interlock so the count is accurate. + */ + LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_EXCL_BLOCK, lock, + mach_absolute_time() - wait_interval, 1, + (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep); + } + } + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, lock, 1); +#endif /* CONFIG_DTRACE */ +} + +/* + * Routine: lck_rw_done + */ + +lck_rw_type_t lck_rw_done(lck_rw_t *lock) +{ + uint32_t data, prev; + boolean_t once = FALSE; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_release_smp); + if (data & LCK_RW_INTERLOCK) { /* wait for interlock to clear */ +#if __SMP__ + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; +#else + panic("lck_rw_done(): Interlock locked (%p): %x", lock, data); +#endif // __SMP__ + } + if (data & LCK_RW_SHARED_MASK) { /* lock is held shared */ + assertf(lock->lck_rw_owner == THREAD_NULL, "state=0x%x, owner=%p", lock->lck_rw_data, lock->lck_rw_owner); + data -= LCK_RW_SHARED_READER; + if ((data & LCK_RW_SHARED_MASK) == 0) /* if reader count has now gone to 0, check for waiters */ + goto check_waiters; + } else { /* if reader count == 0, must be exclusive lock */ + if (data & LCK_RW_WANT_UPGRADE) { + data &= ~(LCK_RW_WANT_UPGRADE); + } else { + if (data & LCK_RW_WANT_EXCL) + data &= ~(LCK_RW_WANT_EXCL); + else /* lock is not 'owned', panic */ + panic("Releasing non-exclusive RW lock without a reader refcount!"); + } + if (!once) { + // Only check for holder and clear it once + assertf(lock->lck_rw_owner == current_thread(), "state=0x%x, owner=%p", lock->lck_rw_data, lock->lck_rw_owner); + ordered_store_rw_owner(lock, THREAD_NULL); + once = TRUE; + } +check_waiters: + /* + * test the original values to match what + * lck_rw_done_gen is going to do to determine + * which wakeups need to happen... + * + * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting) + */ + if (prev & LCK_RW_W_WAITING) { + data &= ~(LCK_RW_W_WAITING); + if ((prev & LCK_RW_PRIV_EXCL) == 0) + data &= ~(LCK_RW_R_WAITING); + } else + data &= ~(LCK_RW_R_WAITING); + } + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_release_smp)) + break; + cpu_pause(); + } + return lck_rw_done_gen(lock, prev); +} + +/* + * Routine: lck_rw_done_gen + * + * called from the assembly language wrapper... + * prior_lock_state is the value in the 1st + * word of the lock at the time of a successful + * atomic compare and exchange with the new value... + * it represents the state of the lock before we + * decremented the rw_shared_count or cleared either + * rw_want_upgrade or rw_want_write and + * the lck_x_waiting bits... since the wrapper + * routine has already changed the state atomically, + * we just need to decide if we should + * wake up anyone and what value to return... we do + * this by examining the state of the lock before + * we changed it + */ +static lck_rw_type_t +lck_rw_done_gen( + lck_rw_t *lck, + uint32_t prior_lock_state) +{ + lck_rw_word_t fake_lck; + lck_rw_type_t lock_type; + thread_t thread; + uint32_t rwlock_count; + + /* + * prior_lock state is a snapshot of the 1st word of the + * lock in question... we'll fake up a pointer to it + * and carefully not access anything beyond whats defined + * in the first word of a lck_rw_t + */ + fake_lck.data = prior_lock_state; + + if (fake_lck.shared_count <= 1) { + if (fake_lck.w_waiting) + thread_wakeup(LCK_RW_WRITER_EVENT(lck)); + + if (!(fake_lck.priv_excl && fake_lck.w_waiting) && fake_lck.r_waiting) + thread_wakeup(LCK_RW_READER_EVENT(lck)); + } + if (fake_lck.shared_count) + lock_type = LCK_RW_TYPE_SHARED; + else + lock_type = LCK_RW_TYPE_EXCLUSIVE; + + /* Check if dropping the lock means that we need to unpromote */ + thread = current_thread(); + rwlock_count = thread->rwlock_count--; +#if MACH_LDEBUG + if (rwlock_count == 0) + panic("rw lock count underflow for thread %p", thread); +#endif + if ((rwlock_count == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { + /* sched_flags checked without lock, but will be rechecked while clearing */ + lck_rw_clear_promotion(thread); + } +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_DONE_RELEASE, lck, lock_type == LCK_RW_TYPE_SHARED ? 0 : 1); +#endif + return lock_type; +} + +/* + * Routine: lck_rw_lock_shared_gen + * Function: + * Fast path code has determined that this lock + * is held exclusively... this is where we spin/block + * until we can acquire the lock in the shared mode + */ +static void +lck_rw_lock_shared_gen( + lck_rw_t *lck) +{ + __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lck); + lck_rw_word_t word; + boolean_t gotlock = 0; + int slept = 0; + wait_result_t res = 0; + boolean_t istate; + +#if CONFIG_DTRACE + uint64_t wait_interval = 0; + int readers_at_sleep = 0; + boolean_t dtrace_ls_initialized = FALSE; + boolean_t dtrace_rwl_shared_spin, dtrace_rwl_shared_block, dtrace_ls_enabled = FALSE; +#endif /* CONFIG_DTRACE */ + + while ( !lck_rw_grab(lck, LCK_RW_GRAB_SHARED, FALSE)) { + +#if CONFIG_DTRACE + if (dtrace_ls_initialized == FALSE) { + dtrace_ls_initialized = TRUE; + dtrace_rwl_shared_spin = (lockstat_probemap[LS_LCK_RW_LOCK_SHARED_SPIN] != 0); + dtrace_rwl_shared_block = (lockstat_probemap[LS_LCK_RW_LOCK_SHARED_BLOCK] != 0); + dtrace_ls_enabled = dtrace_rwl_shared_spin || dtrace_rwl_shared_block; + if (dtrace_ls_enabled) { + /* + * Either sleeping or spinning is happening, + * start a timing of our delay interval now. + */ + readers_at_sleep = lck->lck_rw_shared_count; + wait_interval = mach_absolute_time(); + } + } +#endif + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_SPIN_CODE) | DBG_FUNC_START, + trace_lck, lck->lck_rw_want_excl, lck->lck_rw_want_upgrade, 0, 0); + + gotlock = lck_rw_grab(lck, LCK_RW_GRAB_SHARED, TRUE); + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_SPIN_CODE) | DBG_FUNC_END, + trace_lck, lck->lck_rw_want_excl, lck->lck_rw_want_upgrade, gotlock, 0); + + if (gotlock) + break; + /* + * if we get here, the deadline has expired w/o us + * being able to grab the lock for read + * check to see if we're allowed to do a thread_block + */ + if (lck->lck_rw_can_sleep) { + + istate = lck_interlock_lock(lck); + + word.data = ordered_load_rw(lck); + if ((word.want_excl || word.want_upgrade) && + ((word.shared_count == 0) || word.priv_excl)) { + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_WAIT_CODE) | DBG_FUNC_START, + trace_lck, word.want_excl, word.want_upgrade, 0, 0); + + word.r_waiting = 1; + ordered_store_rw(lck, word.data); + + thread_set_pending_block_hint(current_thread(), kThreadWaitKernelRWLockRead); + res = assert_wait(LCK_RW_READER_EVENT(lck), THREAD_UNINT); + lck_interlock_unlock(lck, istate); + + if (res == THREAD_WAITING) { + res = thread_block(THREAD_CONTINUE_NULL); + slept++; + } + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_WAIT_CODE) | DBG_FUNC_END, + trace_lck, res, slept, 0, 0); + } else { + word.shared_count++; + ordered_store_rw(lck, word.data); + lck_interlock_unlock(lck, istate); + break; + } + } + } + +#if CONFIG_DTRACE + if (dtrace_ls_enabled == TRUE) { + if (slept == 0) { + LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_SPIN, lck, mach_absolute_time() - wait_interval, 0); + } else { + LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_BLOCK, lck, + mach_absolute_time() - wait_interval, 0, + (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep); + } + } + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, lck, 0); +#endif /* CONFIG_DTRACE */ +} + + +void +lck_rw_assert( + lck_rw_t *lck, + unsigned int type) +{ + switch (type) { + case LCK_RW_ASSERT_SHARED: + if ((lck->lck_rw_shared_count != 0) && + (lck->lck_rw_owner == THREAD_NULL)) { + return; + } + break; + case LCK_RW_ASSERT_EXCLUSIVE: + if ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) && + (lck->lck_rw_shared_count == 0) && + (lck->lck_rw_owner == current_thread())) { + return; + } + break; + case LCK_RW_ASSERT_HELD: + if (lck->lck_rw_shared_count != 0) + return; // Held shared + if ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) && + (lck->lck_rw_owner == current_thread())) { + return; // Held exclusive + } + break; + case LCK_RW_ASSERT_NOTHELD: + if ((lck->lck_rw_shared_count == 0) && + !(lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) && + (lck->lck_rw_owner == THREAD_NULL)) { + return; + } + break; + default: + break; + } + panic("rw lock (%p)%s held (mode=%u)", lck, (type == LCK_RW_ASSERT_NOTHELD ? "" : " not"), type); +} + + +/* + * Routine: kdp_lck_rw_lock_is_acquired_exclusive + * NOT SAFE: To be used only by kernel debugger to avoid deadlock. + */ +boolean_t +kdp_lck_rw_lock_is_acquired_exclusive(lck_rw_t *lck) { + if (not_in_kdp) { + panic("panic: rw lock exclusive check done outside of kernel debugger"); + } + return ((lck->lck_rw_want_upgrade || lck->lck_rw_want_excl) && (lck->lck_rw_shared_count == 0)) ? TRUE : FALSE; +} + +/* + * The C portion of the mutex package. These routines are only invoked + * if the optimized assembler routines can't do the work. + */ + +/* + * Forward declaration + */ + +void +lck_mtx_ext_init( + lck_mtx_ext_t * lck, + lck_grp_t * grp, + lck_attr_t * attr); + +/* + * Routine: lck_mtx_alloc_init + */ +lck_mtx_t * +lck_mtx_alloc_init( + lck_grp_t * grp, + lck_attr_t * attr) +{ + lck_mtx_t *lck; + + if ((lck = (lck_mtx_t *) kalloc(sizeof(lck_mtx_t))) != 0) + lck_mtx_init(lck, grp, attr); + + return (lck); +} + +/* + * Routine: lck_mtx_free + */ +void +lck_mtx_free( + lck_mtx_t * lck, + lck_grp_t * grp) +{ + lck_mtx_destroy(lck, grp); + kfree((void *) lck, sizeof(lck_mtx_t)); +} + +/* + * Routine: lck_mtx_init + */ +void +lck_mtx_init( + lck_mtx_t * lck, + lck_grp_t * grp, + lck_attr_t * attr) +{ +#ifdef BER_XXX + lck_mtx_ext_t *lck_ext; +#endif + lck_attr_t *lck_attr; + + if (attr != LCK_ATTR_NULL) + lck_attr = attr; + else + lck_attr = &LockDefaultLckAttr; + +#ifdef BER_XXX + if ((lck_attr->lck_attr_val) & LCK_ATTR_DEBUG) { + if ((lck_ext = (lck_mtx_ext_t *) kalloc(sizeof(lck_mtx_ext_t))) != 0) { + lck_mtx_ext_init(lck_ext, grp, lck_attr); + lck->lck_mtx_tag = LCK_MTX_TAG_INDIRECT; + lck->lck_mtx_ptr = lck_ext; + lck->lck_mtx_type = LCK_MTX_TYPE; + } + } else +#endif + { + lck->lck_mtx_ptr = NULL; // Clear any padding in the union fields below + lck->lck_mtx_waiters = 0; + lck->lck_mtx_pri = 0; + lck->lck_mtx_type = LCK_MTX_TYPE; + ordered_store_mtx(lck, 0); + } + lck_grp_reference(grp); + lck_grp_lckcnt_incr(grp, LCK_TYPE_MTX); +} + +/* + * Routine: lck_mtx_init_ext + */ +void +lck_mtx_init_ext( + lck_mtx_t * lck, + lck_mtx_ext_t * lck_ext, + lck_grp_t * grp, + lck_attr_t * attr) +{ + lck_attr_t *lck_attr; + + if (attr != LCK_ATTR_NULL) + lck_attr = attr; + else + lck_attr = &LockDefaultLckAttr; + + if ((lck_attr->lck_attr_val) & LCK_ATTR_DEBUG) { + lck_mtx_ext_init(lck_ext, grp, lck_attr); + lck->lck_mtx_tag = LCK_MTX_TAG_INDIRECT; + lck->lck_mtx_ptr = lck_ext; + lck->lck_mtx_type = LCK_MTX_TYPE; + } else { + lck->lck_mtx_waiters = 0; + lck->lck_mtx_pri = 0; + lck->lck_mtx_type = LCK_MTX_TYPE; + ordered_store_mtx(lck, 0); + } + lck_grp_reference(grp); + lck_grp_lckcnt_incr(grp, LCK_TYPE_MTX); +} + +/* + * Routine: lck_mtx_ext_init + */ +void +lck_mtx_ext_init( + lck_mtx_ext_t * lck, + lck_grp_t * grp, + lck_attr_t * attr) +{ + bzero((void *) lck, sizeof(lck_mtx_ext_t)); + + lck->lck_mtx.lck_mtx_type = LCK_MTX_TYPE; + + if ((attr->lck_attr_val) & LCK_ATTR_DEBUG) { + lck->lck_mtx_deb.type = MUTEX_TAG; + lck->lck_mtx_attr |= LCK_MTX_ATTR_DEBUG; + } + lck->lck_mtx_grp = grp; + + if (grp->lck_grp_attr & LCK_GRP_ATTR_STAT) + lck->lck_mtx_attr |= LCK_MTX_ATTR_STAT; +} + +/* The slow versions */ +static void lck_mtx_lock_contended(lck_mtx_t *lock, thread_t thread, boolean_t interlocked); +static boolean_t lck_mtx_try_lock_contended(lck_mtx_t *lock, thread_t thread); +static void lck_mtx_unlock_contended(lck_mtx_t *lock, thread_t thread, boolean_t interlocked); + +/* + * Routine: lck_mtx_verify + * + * Verify if a mutex is valid + */ +static inline void +lck_mtx_verify(lck_mtx_t *lock) +{ + if (lock->lck_mtx_type != LCK_MTX_TYPE) + panic("Invalid mutex %p", lock); +#if DEVELOPMENT || DEBUG + if (lock->lck_mtx_tag == LCK_MTX_TAG_DESTROYED) + panic("Mutex destroyed %p", lock); +#endif /* DEVELOPMENT || DEBUG */ +} + +/* + * Routine: lck_mtx_check_preemption + * + * Verify preemption is enabled when attempting to acquire a mutex. + */ + +static inline void +lck_mtx_check_preemption(lck_mtx_t *lock) +{ +#if DEVELOPMENT || DEBUG + int pl = get_preemption_level(); + + if (pl != 0) + panic("Attempt to take mutex with preemption disabled. Lock=%p, level=%d", lock, pl); +#else + (void)lock; +#endif +} + +/* + * Routine: lck_mtx_lock + */ +void +lck_mtx_lock(lck_mtx_t *lock) +{ + thread_t thread; + + lck_mtx_verify(lock); + lck_mtx_check_preemption(lock); + thread = current_thread(); + if (atomic_compare_exchange(&lock->lck_mtx_data, 0, LCK_MTX_THREAD_TO_STATE(thread), + memory_order_acquire_smp, FALSE)) { +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, lock, 0); +#endif /* CONFIG_DTRACE */ + return; + } + lck_mtx_lock_contended(lock, thread, FALSE); +} + +/* + This is the slow version of mutex locking. + */ +static void NOINLINE +lck_mtx_lock_contended(lck_mtx_t *lock, thread_t thread, boolean_t interlocked) +{ + thread_t holding_thread; + uintptr_t state; + int waiters; + + if (interlocked) + goto interlock_held; + + for ( ; ; ) { + if (atomic_compare_exchange(&lock->lck_mtx_data, 0, LCK_MTX_THREAD_TO_STATE(thread), + memory_order_acquire_smp, FALSE)) + return; + interlock_lock(lock); +interlock_held: + state = ordered_load_mtx(lock); + holding_thread = LCK_MTX_STATE_TO_THREAD(state); + if (holding_thread == NULL) + break; + ordered_store_mtx(lock, (state | LCK_ILOCK | ARM_LCK_WAITERS)); // Set waiters bit and wait + lck_mtx_lock_wait(lock, holding_thread); + } + waiters = lck_mtx_lock_acquire(lock); + state = LCK_MTX_THREAD_TO_STATE(thread); + if (waiters != 0) + state |= ARM_LCK_WAITERS; +#if __SMP__ + state |= LCK_ILOCK; // Preserve interlock + ordered_store_mtx(lock, state); // Set ownership + interlock_unlock(lock); // Release interlock, enable preemption +#else + ordered_store_mtx(lock, state); // Set ownership + enable_preemption(); +#endif + load_memory_barrier(); + +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, lock, 0); +#endif /* CONFIG_DTRACE */ +} + +/* + * Common code for mutex locking as spinlock + */ +static inline void +lck_mtx_lock_spin_internal(lck_mtx_t *lock, boolean_t allow_held_as_mutex) +{ + uintptr_t state; + + interlock_lock(lock); + state = ordered_load_mtx(lock); + if (LCK_MTX_STATE_TO_THREAD(state)) { + if (allow_held_as_mutex) + lck_mtx_lock_contended(lock, current_thread(), TRUE); + else + // "Always" variants can never block. If the lock is held and blocking is not allowed + // then someone is mixing always and non-always calls on the same lock, which is + // forbidden. + panic("Attempting to block on a lock taken as spin-always %p", lock); + return; + } + state &= ARM_LCK_WAITERS; // Preserve waiters bit + state |= (LCK_MTX_SPIN_TAG | LCK_ILOCK); // Add spin tag and maintain interlock + ordered_store_mtx(lock, state); + load_memory_barrier(); + +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_SPIN_ACQUIRE, lock, 0); +#endif /* CONFIG_DTRACE */ +} + +/* + * Routine: lck_mtx_lock_spin + */ +void +lck_mtx_lock_spin(lck_mtx_t *lock) +{ + lck_mtx_check_preemption(lock); + lck_mtx_lock_spin_internal(lock, TRUE); +} + +/* + * Routine: lck_mtx_lock_spin_always + */ +void +lck_mtx_lock_spin_always(lck_mtx_t *lock) +{ + lck_mtx_lock_spin_internal(lock, FALSE); +} + +/* + * Routine: lck_mtx_try_lock + */ +boolean_t +lck_mtx_try_lock(lck_mtx_t *lock) +{ + thread_t thread = current_thread(); + + lck_mtx_verify(lock); + if (atomic_compare_exchange(&lock->lck_mtx_data, 0, LCK_MTX_THREAD_TO_STATE(thread), + memory_order_acquire_smp, FALSE)) { +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_TRY_LOCK_ACQUIRE, lock, 0); +#endif /* CONFIG_DTRACE */ + return TRUE; + } + return lck_mtx_try_lock_contended(lock, thread); +} + +static boolean_t NOINLINE +lck_mtx_try_lock_contended(lck_mtx_t *lock, thread_t thread) +{ + thread_t holding_thread; + uintptr_t state; + int waiters; + +#if __SMP__ + interlock_lock(lock); + state = ordered_load_mtx(lock); + holding_thread = LCK_MTX_STATE_TO_THREAD(state); + if (holding_thread) { + interlock_unlock(lock); + return FALSE; + } +#else + disable_preemption_for_thread(thread); + state = ordered_load_mtx(lock); + if (state & LCK_ILOCK) + panic("Unexpected interlock set (%p)", lock); + holding_thread = LCK_MTX_STATE_TO_THREAD(state); + if (holding_thread) { + enable_preemption(); + return FALSE; + } + state |= LCK_ILOCK; + ordered_store_mtx(lock, state); +#endif // __SMP__ + waiters = lck_mtx_lock_acquire(lock); + state = LCK_MTX_THREAD_TO_STATE(thread); + if (waiters != 0) + state |= ARM_LCK_WAITERS; +#if __SMP__ + state |= LCK_ILOCK; // Preserve interlock + ordered_store_mtx(lock, state); // Set ownership + interlock_unlock(lock); // Release interlock, enable preemption +#else + ordered_store_mtx(lock, state); // Set ownership + enable_preemption(); +#endif + load_memory_barrier(); + return TRUE; +} + +static inline boolean_t +lck_mtx_try_lock_spin_internal(lck_mtx_t *lock, boolean_t allow_held_as_mutex) +{ + uintptr_t state; + + if (!interlock_try(lock)) + return FALSE; + state = ordered_load_mtx(lock); + if(LCK_MTX_STATE_TO_THREAD(state)) { + // Lock is held as mutex + if (allow_held_as_mutex) + interlock_unlock(lock); + else + // "Always" variants can never block. If the lock is held as a normal mutex + // then someone is mixing always and non-always calls on the same lock, which is + // forbidden. + panic("Spin-mutex held as full mutex %p", lock); + return FALSE; + } + state &= ARM_LCK_WAITERS; // Preserve waiters bit + state |= (LCK_MTX_SPIN_TAG | LCK_ILOCK); // Add spin tag and maintain interlock + ordered_store_mtx(lock, state); + load_memory_barrier(); + +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, lock, 0); +#endif /* CONFIG_DTRACE */ + return TRUE; +} + +/* + * Routine: lck_mtx_try_lock_spin + */ +boolean_t +lck_mtx_try_lock_spin(lck_mtx_t *lock) +{ + return lck_mtx_try_lock_spin_internal(lock, TRUE); +} + +/* + * Routine: lck_mtx_try_lock_spin_always + */ +boolean_t +lck_mtx_try_lock_spin_always(lck_mtx_t *lock) +{ + return lck_mtx_try_lock_spin_internal(lock, FALSE); +} + + + +/* + * Routine: lck_mtx_unlock + */ +void +lck_mtx_unlock(lck_mtx_t *lock) +{ + thread_t thread = current_thread(); + uintptr_t state; + boolean_t ilk_held = FALSE; + + lck_mtx_verify(lock); + + state = ordered_load_mtx(lock); + if (state & LCK_ILOCK) { + if(LCK_MTX_STATE_TO_THREAD(state) == (thread_t)LCK_MTX_SPIN_TAG) + ilk_held = TRUE; // Interlock is held by (presumably) this thread + goto slow_case; + } + // Locked as a mutex + if (atomic_compare_exchange(&lock->lck_mtx_data, LCK_MTX_THREAD_TO_STATE(thread), 0, + memory_order_release_smp, FALSE)) { +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, lock, 0); +#endif /* CONFIG_DTRACE */ + return; + } +slow_case: + lck_mtx_unlock_contended(lock, thread, ilk_held); +} + +static void NOINLINE +lck_mtx_unlock_contended(lck_mtx_t *lock, thread_t thread, boolean_t ilk_held) +{ + uintptr_t state; + + if (ilk_held) { + state = ordered_load_mtx(lock); + } else { +#if __SMP__ + interlock_lock(lock); + state = ordered_load_mtx(lock); + if (thread != LCK_MTX_STATE_TO_THREAD(state)) + panic("lck_mtx_unlock(): Attempt to release lock not owned by thread (%p)", lock); +#else + disable_preemption_for_thread(thread); + state = ordered_load_mtx(lock); + if (state & LCK_ILOCK) + panic("lck_mtx_unlock(): Unexpected interlock set (%p)", lock); + if (thread != LCK_MTX_STATE_TO_THREAD(state)) + panic("lck_mtx_unlock(): Attempt to release lock not owned by thread (%p)", lock); + state |= LCK_ILOCK; + ordered_store_mtx(lock, state); +#endif + } + if (state & ARM_LCK_WAITERS) { + lck_mtx_unlock_wakeup(lock, thread); + state = ordered_load_mtx(lock); + } else { + assertf(lock->lck_mtx_pri == 0, "pri=0x%x", lock->lck_mtx_pri); + } + state &= ARM_LCK_WAITERS; // Retain waiters bit +#if __SMP__ + state |= LCK_ILOCK; + ordered_store_mtx(lock, state); + interlock_unlock(lock); +#else + ordered_store_mtx(lock, state); + enable_preemption(); +#endif + +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, lock, 0); +#endif /* CONFIG_DTRACE */ +} + +/* + * Routine: lck_mtx_assert + */ +void +lck_mtx_assert(lck_mtx_t *lock, unsigned int type) +{ + thread_t thread, holder; + uintptr_t state; + + state = ordered_load_mtx(lock); + holder = LCK_MTX_STATE_TO_THREAD(state); + if (holder == (thread_t)LCK_MTX_SPIN_TAG) { + // Lock is held in spin mode, owner is unknown. + return; // Punt + } + thread = current_thread(); + if (type == LCK_MTX_ASSERT_OWNED) { + if (thread != holder) + panic("lck_mtx_assert(): mutex (%p) owned", lock); + } else if (type == LCK_MTX_ASSERT_NOTOWNED) { + if (thread == holder) + panic("lck_mtx_assert(): mutex (%p) not owned", lock); + } else + panic("lck_mtx_assert(): invalid arg (%u)", type); +} + +/* + * Routine: lck_mtx_ilk_unlock + */ +boolean_t +lck_mtx_ilk_unlock(lck_mtx_t *lock) +{ + interlock_unlock(lock); + return TRUE; +} + +/* + * Routine: lck_mtx_convert_spin + * + * Convert a mutex held for spin into a held full mutex + */ +void +lck_mtx_convert_spin(lck_mtx_t *lock) +{ + thread_t thread = current_thread(); + uintptr_t state; + int waiters; + + state = ordered_load_mtx(lock); + if (LCK_MTX_STATE_TO_THREAD(state) == thread) + return; // Already owned as mutex, return + if ((state & LCK_ILOCK) == 0 || (LCK_MTX_STATE_TO_THREAD(state) != (thread_t)LCK_MTX_SPIN_TAG)) + panic("lck_mtx_convert_spin: Not held as spinlock (%p)", lock); + state &= ~(LCK_MTX_THREAD_MASK); // Clear the spin tag + ordered_store_mtx(lock, state); + waiters = lck_mtx_lock_acquire(lock); // Acquire to manage priority boosts + state = LCK_MTX_THREAD_TO_STATE(thread); + if (waiters != 0) + state |= ARM_LCK_WAITERS; +#if __SMP__ + state |= LCK_ILOCK; + ordered_store_mtx(lock, state); // Set ownership + interlock_unlock(lock); // Release interlock, enable preemption +#else + ordered_store_mtx(lock, state); // Set ownership + enable_preemption(); +#endif +} + + +/* + * Routine: lck_mtx_destroy + */ +void +lck_mtx_destroy( + lck_mtx_t * lck, + lck_grp_t * grp) +{ + if (lck->lck_mtx_type != LCK_MTX_TYPE) + panic("Destroying invalid mutex %p", lck); + if (lck->lck_mtx_tag == LCK_MTX_TAG_DESTROYED) + panic("Destroying previously destroyed lock %p", lck); + lck_mtx_assert(lck, LCK_MTX_ASSERT_NOTOWNED); + lck->lck_mtx_tag = LCK_MTX_TAG_DESTROYED; + lck_grp_lckcnt_decr(grp, LCK_TYPE_MTX); + lck_grp_deallocate(grp); + return; +} + +/* + * Routine: lck_spin_assert + */ +void +lck_spin_assert(lck_spin_t *lock, unsigned int type) +{ + thread_t thread, holder; + uintptr_t state; + + if (lock->type != LCK_SPIN_TYPE) + panic("Invalid spinlock %p", lock); + + state = lock->lck_spin_data; + holder = (thread_t)(state & ~LCK_ILOCK); + thread = current_thread(); + if (type == LCK_ASSERT_OWNED) { + if (holder == 0) + panic("Lock not owned %p = %lx", lock, state); + if (holder != thread) + panic("Lock not owned by current thread %p = %lx", lock, state); + if ((state & LCK_ILOCK) == 0) + panic("Lock bit not set %p = %lx", lock, state); + } else if (type == LCK_ASSERT_NOTOWNED) { + if (holder != 0) { + if (holder == thread) + panic("Lock owned by current thread %p = %lx", lock, state); + else + panic("Lock %p owned by thread %p", lock, holder); + } + if (state & LCK_ILOCK) + panic("Lock bit set %p = %lx", lock, state); + } else + panic("lck_spin_assert(): invalid arg (%u)", type); +} + +boolean_t +lck_rw_lock_yield_shared(lck_rw_t *lck, boolean_t force_yield) +{ + lck_rw_word_t word; + + lck_rw_assert(lck, LCK_RW_ASSERT_SHARED); + + word.data = ordered_load_rw(lck); + if (word.want_excl || word.want_upgrade || force_yield) { + lck_rw_unlock_shared(lck); + mutex_pause(2); + lck_rw_lock_shared(lck); + return TRUE; + } + + return FALSE; +} + +/* + * Routine: kdp_lck_mtx_lock_spin_is_acquired + * NOT SAFE: To be used only by kernel debugger to avoid deadlock. + */ +boolean_t +kdp_lck_mtx_lock_spin_is_acquired(lck_mtx_t *lck) +{ + uintptr_t state; + + if (not_in_kdp) { + panic("panic: spinlock acquired check done outside of kernel debugger"); + } + state = ordered_load_mtx(lck); + if (state == LCK_MTX_TAG_DESTROYED) + return FALSE; + if (LCK_MTX_STATE_TO_THREAD(state) || (state & LCK_ILOCK)) + return TRUE; + return FALSE; +} + +void +kdp_lck_mtx_find_owner(__unused struct waitq * waitq, event64_t event, thread_waitinfo_t * waitinfo) +{ + lck_mtx_t * mutex = LCK_EVENT_TO_MUTEX(event); + waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(mutex); + uintptr_t state = ordered_load_mtx(mutex); + thread_t holder = LCK_MTX_STATE_TO_THREAD(state); + if ((uintptr_t)holder == (uintptr_t)LCK_MTX_SPIN_TAG) { + waitinfo->owner = STACKSHOT_WAITOWNER_MTXSPIN; + } else { + assertf(state != (uintptr_t)LCK_MTX_TAG_DESTROYED, "state=0x%llx", (uint64_t)state); + assertf(state != (uintptr_t)LCK_MTX_TAG_INDIRECT, "state=0x%llx", (uint64_t)state); + waitinfo->owner = thread_tid(holder); + } +} + +void +kdp_rwlck_find_owner(__unused struct waitq * waitq, event64_t event, thread_waitinfo_t * waitinfo) +{ + lck_rw_t *rwlck = NULL; + switch(waitinfo->wait_type) { + case kThreadWaitKernelRWLockRead: + rwlck = READ_EVENT_TO_RWLOCK(event); + break; + case kThreadWaitKernelRWLockWrite: + case kThreadWaitKernelRWLockUpgrade: + rwlck = WRITE_EVENT_TO_RWLOCK(event); + break; + default: + panic("%s was called with an invalid blocking type", __FUNCTION__); + break; + } + waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(rwlck); + waitinfo->owner = thread_tid(rwlck->lck_rw_owner); +} diff --git a/osfmk/arm/locore.s b/osfmk/arm/locore.s new file mode 100644 index 000000000..5af8c2a79 --- /dev/null +++ b/osfmk/arm/locore.s @@ -0,0 +1,2041 @@ +/* + * Copyright (c) 2007-2011 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "assym.s" + +#define TRACE_SYSCALL 0 + +/* + * Copied to low physical memory in arm_init, + * so the kernel must be linked virtually at + * 0xc0001000 or higher to leave space for it. + */ + .syntax unified + .text + .align 12 + .globl EXT(ExceptionLowVectorsBase) + +LEXT(ExceptionLowVectorsBase) + adr pc, Lreset_low_vector + b . // Undef + b . // SWI + b . // Prefetch Abort + b . // Data Abort + b . // Address Exception + b . // IRQ + b . // FIQ/DEC +LEXT(ResetPrivateData) + .space (480),0 // (filled with 0s) + // ExceptionLowVectorsBase + 0x200 +Lreset_low_vector: + adr r4, EXT(ResetHandlerData) + ldr r0, [r4, ASSIST_RESET_HANDLER] + movs r0, r0 + blxne r0 + adr r4, EXT(ResetHandlerData) + ldr r1, [r4, CPU_DATA_ENTRIES] + ldr r1, [r1, CPU_DATA_PADDR] + ldr r5, [r1, CPU_RESET_ASSIST] + movs r5, r5 + blxne r5 + adr r4, EXT(ResetHandlerData) + ldr r0, [r4, BOOT_ARGS] + ldr r1, [r4, CPU_DATA_ENTRIES] +#if __ARM_SMP__ +#if defined(ARMA7) + // physical cpu number is stored in MPIDR Affinity level 0 + mrc p15, 0, r6, c0, c0, 5 // Read MPIDR + and r6, r6, #0xFF // Extract Affinity level 0 +#else +#error missing Who Am I implementation +#endif +#else + mov r6, #0 +#endif /* __ARM_SMP__ */ + // physical cpu number matches cpu number +//#if cdeSize != 16 +//#error cpu_data_entry is not 16bytes in size +//#endif + lsl r6, r6, #4 // Get CpuDataEntry offset + add r1, r1, r6 // Get cpu_data_entry pointer + ldr r1, [r1, CPU_DATA_PADDR] + ldr r5, [r1, CPU_RESET_HANDLER] + movs r5, r5 + blxne r5 // Branch to cpu reset handler + b . // Unexpected reset + .globl EXT(ResetHandlerData) +LEXT(ResetHandlerData) + .space (rhdSize_NUM),0 // (filled with 0s) + + + .globl EXT(ExceptionLowVectorsEnd) +LEXT(ExceptionLowVectorsEnd) + + .text + .align 12 + .globl EXT(ExceptionVectorsBase) + +LEXT(ExceptionVectorsBase) + + adr pc, Lexc_reset_vector + adr pc, Lexc_undefined_inst_vector + adr pc, Lexc_swi_vector + adr pc, Lexc_prefetch_abort_vector + adr pc, Lexc_data_abort_vector + adr pc, Lexc_address_exception_vector + adr pc, Lexc_irq_vector +#if __ARM_TIME__ + adr pc, Lexc_decirq_vector +#else /* ! __ARM_TIME__ */ + mov pc, r9 +#endif /* __ARM_TIME__ */ + +Lexc_reset_vector: + b . + .long 0x0 + .long 0x0 + .long 0x0 +Lexc_undefined_inst_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #4] // Branch to exception handler +Lexc_swi_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #8] // Branch to exception handler +Lexc_prefetch_abort_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #0xC] // Branch to exception handler +Lexc_data_abort_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #0x10] // Branch to exception handler +Lexc_address_exception_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #0x14] // Branch to exception handler +Lexc_irq_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #0x18] // Branch to exception handler +#if __ARM_TIME__ +Lexc_decirq_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #0x1C] // Branch to exception handler +#else /* ! __ARM_TIME__ */ + .long 0x0 + .long 0x0 + .long 0x0 + .long 0x0 +#endif /* __ARM_TIME__ */ + + .fill 984, 4, 0 // Push to the 4KB page boundary + + .globl EXT(ExceptionVectorsEnd) +LEXT(ExceptionVectorsEnd) + + +/* + * Targets for the exception vectors; we patch these during boot (to allow + * for position independent code without complicating the vectors; see start.s). + */ + .globl EXT(ExceptionVectorsTable) +LEXT(ExceptionVectorsTable) +Lreset_vector: + .long 0x0 +Lundefined_inst_vector: + .long 0x0 +Lswi_vector: + .long 0x0 +Lprefetch_abort_vector: + .long 0x0 +Ldata_abort_vector: + .long 0x0 +Laddress_exception_vector: + .long 0x0 +Lirq_vector: + .long 0x0 +Ldecirq_vector: + .long 0x0 + + +/* + * First Level Exception Handlers + */ + .text + .align 2 + .globl EXT(fleh_reset) +LEXT(fleh_reset) + b . // Never return + +/* + * First Level Exception Handler for Undefined Instruction. + */ + .text + .align 2 + .globl EXT(fleh_undef) + +LEXT(fleh_undef) + mrs sp, spsr // Check the previous mode + tst sp, #PSR_TF // Is it Thumb? + subeq lr, lr, #4 + subne lr, lr, #2 + tst sp, #0x0f // Is it from user? + bne undef_from_kernel + +undef_from_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get current thread PCB pointer + + stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB + mov r7, #0 // Zero the frame pointer + nop + + mov r0, sp // Store arm_saved_state pointer + // for argument + + str lr, [sp, SS_PC] // Save user mode pc register + + mrs r4, spsr + str r4, [sp, SS_CPSR] // Save user mode cpsr + + mrs r4, cpsr // Read cpsr + cpsid i, #PSR_SVC_MODE + mrs r3, cpsr // Read cpsr + msr spsr_cxsf, r3 // Set spsr(svc mode cpsr) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + and r0, r4, #PSR_MODE_MASK // Extract current mode + cmp r0, #PSR_UND_MODE // Check undef mode + bne EXT(ExceptionVectorPanic) + + mvn r0, #0 + str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#endif + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r3, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r3 // And shove it into FPSCR + add r1, r9, ACT_UVFP // Reload the pointer to the save state + add r0, r9, ACT_PCBDATA // Reload the VFP save state argument +#else + mov r1, #0 // Clear the VFP save state argument + add r0, r9, ACT_PCBDATA // Reload arm_saved_state pointer +#endif + + bl EXT(sleh_undef) // Call second level handler + // sleh will enable interrupt + b load_and_go_user + +undef_from_kernel: + mrs sp, cpsr // Read cpsr + and sp, sp, #PSR_MODE_MASK // Extract current mode + cmp sp, #PSR_UND_MODE // Check undef mode + movne r0, sp + bne EXT(ExceptionVectorPanic) + mrs sp, spsr // Check the previous mode + + /* + * We have a kernel stack already, and I will use it to save contexts + * IRQ is disabled + */ + +#if CONFIG_DTRACE + /* + * See if we came here from IRQ or SVC mode, and go back to that mode + */ + + and sp, sp, #PSR_MODE_MASK + cmp sp, #PSR_IRQ_MODE + bne undef_from_kernel_svc + + cpsid i, #PSR_IRQ_MODE + b handle_undef +#endif + +undef_from_kernel_svc: + cpsid i, #PSR_SVC_MODE + +handle_undef: +#if CONFIG_DTRACE + // We need a frame for backtracing. The LR here is the LR of supervisor mode, not the location where the exception + // took place. We'll store that later after we switch to undef mode and pull out the LR from there. + + // This frame is consumed by fbt_invop. Any changes with the size or location of this frame will probably require + // changes in fbt_invop also. + stmfd sp!, { r7, lr } +#endif + + sub sp, sp, EXC_CTX_SIZE // Reserve for arm_saved_state + + stmia sp, {r0-r12} // Save on supervisor mode stack + str lr, [sp, SS_LR] + +#if CONFIG_DTRACE + add r7, sp, EXC_CTX_SIZE // Save frame pointer +#endif + + mov ip, sp // Stack transfer + + cpsid i, #PSR_UND_MODE + + str lr, [ip, SS_PC] // Save complete + mrs r4, spsr + str r4, [ip, SS_CPSR] + +#if CONFIG_DTRACE + /* + * Go back to previous mode for mode specific regs + */ + and r4, r4, #PSR_MODE_MASK + cmp r4, #PSR_IRQ_MODE + bne handle_undef_from_svc + + cpsid i, #PSR_IRQ_MODE + b handle_undef2 +#endif + +handle_undef_from_svc: + cpsid i, #PSR_SVC_MODE + +handle_undef2: + +/* + sp - stack pointer + ip - stack pointer + r7 - frame pointer state + */ + + +#if CONFIG_DTRACE + ldr r0, [ip, SS_PC] // Get the exception pc to store later +#endif + + add ip, ip, EXC_CTX_SIZE // Send stack pointer to debugger +#if CONFIG_DTRACE + str r0, [ip, #4] + add ip, ip, #8 +#endif + str ip, [sp, SS_SP] // for accessing local variable +#if CONFIG_DTRACE + sub ip, ip, #8 +#endif + sub ip, ip, EXC_CTX_SIZE + +#if __ARM_VFP__ + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + mov r5, r0 // Stash the save area in another register + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r1, r5 // Load the VFP save area argument + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#else + mov r1, #0 // Clear the facility context argument +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r3, r10 + beq 1f + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 +1: + mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r0, sp // Argument + +/* + * For armv7k ABI, the stack needs to be 16-byte aligned + */ +#if __BIGGEST_ALIGNMENT__ > 4 + and r1, sp, #0x0F // sp mod 16-bytes + cmp r1, #4 // need space for the sp on the stack + addlt r1, r1, #0x10 // make room if needed, but keep stack aligned + mov r2, sp // get current sp + sub sp, sp, r1 // align stack + str r2, [sp] // store previous sp on stack +#endif + + bl EXT(sleh_undef) // Call second level handler + +#if __BIGGEST_ALIGNMENT__ > 4 + ldr sp, [sp] // restore stack +#endif + +#if __ARM_USER_PROTECT__ + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r10, r0 + beq 1f + ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb + cmp r10, r0 + beq 1f + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + ldr r11, [r9, ACT_ASID] // Load thread asid +1: + mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR + isb +#endif + b load_and_go_sys + + +/* + * First Level Exception Handler for Software Interrupt + * + * We assert that only user level can use the "SWI" instruction for a system + * call on development kernels, and assume it's true on release. + * + * System call number is stored in r12. + * System call arguments are stored in r0 to r6 and r8 (we skip r7) + * + */ + .text + .align 5 + .globl EXT(fleh_swi) + +LEXT(fleh_swi) + cpsid i, #PSR_ABT_MODE + mov sp, ip // Save ip + cpsid i, #PSR_SVC_MODE + mrs ip, spsr // Check the previous mode + tst ip, #0x0f + cpsid i, #PSR_ABT_MODE + mov ip, sp // Restore ip + cpsid i, #PSR_SVC_MODE + beq swi_from_user + +/* Only user mode can use SWI. Panic if the kernel tries. */ +swi_from_kernel: + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + adr r0, L_kernel_swi_panic_str // Load panic messages and panic() + blx EXT(panic) + b . + +swi_from_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + + + /* Check for special mach_absolute_time trap value. + * This is intended to be a super-lightweight call to ml_get_timebase(), which + * is handrolled assembly and does not use the stack, thus not requiring us to setup a kernel stack. */ + cmp r12, #-3 + beq fleh_swi_trap_tb + stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB + mov r7, #0 // Zero the frame pointer + nop + mov r8, sp // Store arm_saved_state pointer + add sp, sp, SS_PC + srsia sp, #PSR_SVC_MODE + mrs r3, cpsr // Read cpsr + msr spsr_cxsf, r3 // Set spsr(svc mode cpsr) + sub r9, sp, ACT_PCBDATA_PC + + ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack + mov r11, r12 // save the syscall vector in a nontrashed register + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + + mvn r0, #0 + str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling SWI from userspace + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + add r8, r9, ACT_PCBDATA // Reload arm_saved_state pointer +#endif + ldr r10, [r9, ACT_TASK] // Load the current task + + /* enable interrupts */ + cpsie i // Enable IRQ + + cmp r11, #-4 // Special value for mach_continuous_time + beq fleh_swi_trap_mct + + cmp r11, #0x80000000 + beq fleh_swi_trap +fleh_swi_trap_ret: + +#if TRACE_SYSCALL + /* trace the syscall */ + mov r0, r8 + bl EXT(syscall_trace) +#endif + + bl EXT(mach_kauth_cred_uthread_update) + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + /* unix syscall? */ + rsbs r5, r11, #0 // make the syscall positive (if negative) + ble fleh_swi_unix // positive syscalls are unix (note reverse logic here) + +fleh_swi_mach: + /* note that mach_syscall_trace can modify r9, so increment the thread + * syscall count before the call : */ + ldr r2, [r9, TH_MACH_SYSCALLS] + add r2, r2, #1 + str r2, [r9, TH_MACH_SYSCALLS] + + LOAD_ADDR(r1, mach_trap_table) // load mach_trap_table +#if MACH_TRAP_TABLE_ENTRY_SIZE_NUM == 12 + add r11, r5, r5, lsl #1 // syscall * 3 + add r6, r1, r11, lsl #2 // trap_table + syscall * 12 +#elif MACH_TRAP_TABLE_ENTRY_SIZE_NUM == 16 + add r6, r1, r5, lsl #4 // trap_table + syscall * 16 +#elif MACH_TRAP_TABLE_ENTRY_SIZE_NUM == 20 + add r11, r5, r5, lsl #2 // syscall * 5 + add r6, r1, r11, lsl #2 // trap_table + syscall * 20 +#else +#error mach_trap_t size unhandled (see MACH_TRAP_TABLE_ENTRY_SIZE)! +#endif + +#ifndef NO_KDEBUG + LOAD_ADDR(r4, kdebug_enable) + ldr r4, [r4] + movs r4, r4 + movne r0, r8 // ready the reg state pointer as an arg to the call + movne r1, r5 // syscall number as 2nd arg + COND_EXTERN_BLNE(mach_syscall_trace) +#endif + adr lr, fleh_swi_exit // any calls from here on out will return to our exit path + cmp r5, MACH_TRAP_TABLE_COUNT // check syscall number range + bge fleh_swi_mach_error + +/* + * For arm32 ABI where 64-bit types are aligned to even registers and + * 64-bits on stack, we need to unpack registers differently. So + * we use the mungers for marshalling in arguments from user space. + * Currently this is just ARMv7k. + */ +#if __BIGGEST_ALIGNMENT__ > 4 + sub sp, #0x40 // allocate buffer and keep stack 128-bit aligned + // it should be big enough for all syscall arguments + ldr r11, [r6, #8] // get mach_trap_table[call_number].mach_trap_arg_munge32 + teq r11, #0 // check if we have a munger + moveq r0, #0 + movne r0, r8 // ready the reg state pointer as an arg to the call + movne r1, sp // stack will hold arguments buffer + blxne r11 // call munger to get arguments from userspace + adr lr, fleh_swi_exit // any calls from here on out will return to our exit path + teq r0, #0 + bne fleh_swi_mach_error // exit if the munger returned non-zero status +#endif + + ldr r1, [r6, #4] // load the syscall vector + + LOAD_ADDR(r2, kern_invalid) // test to make sure the trap is not kern_invalid + teq r1, r2 + beq fleh_swi_mach_error + +#if __BIGGEST_ALIGNMENT__ > 4 + mov r0, sp // argument buffer on stack + bx r1 // call the syscall handler +#else + mov r0, r8 // ready the reg state pointer as an arg to the call + bx r1 // call the syscall handler +#endif + +fleh_swi_exit64: + str r1, [r8, #4] // top of 64-bit return +fleh_swi_exit: + str r0, [r8] // save the return value +#ifndef NO_KDEBUG + movs r4, r4 + movne r1, r5 + COND_EXTERN_BLNE(mach_syscall_trace_exit) +#endif +#if TRACE_SYSCALL + bl EXT(syscall_trace_exit) +#endif + + mov r0, #1 + bl EXT(throttle_lowpri_io) // throttle_lowpri_io(1); + + bl EXT(thread_exception_return) + b . + +fleh_swi_mach_error: + mov r0, #EXC_SYSCALL + sub r1, sp, #4 + mov r2, #1 + bl EXT(exception_triage) + b . + + .align 5 +fleh_swi_unix: + ldr r1, [r9, TH_UNIX_SYSCALLS] + mov r0, r8 // reg state structure is arg + add r1, r1, #1 + str r1, [r9, TH_UNIX_SYSCALLS] + mov r1, r9 // current thread in arg1 + ldr r2, [r9, TH_UTHREAD] // current uthread in arg2 + ldr r3, [r10, TASK_BSD_INFO] // current proc in arg3 + bl EXT(unix_syscall) + b . + +fleh_swi_trap: + ldmia r8, {r0-r3} + cmp r3, #3 + addls pc, pc, r3, LSL#2 + b fleh_swi_trap_ret + b icache_invalidate_trap + b dcache_flush_trap + b thread_set_cthread_trap + b thread_get_cthread_trap + +icache_invalidate_trap: + add r3, r0, r1 + cmp r3, VM_MAX_ADDRESS + subhi r3, r3, #1< 4 + and r1, sp, #0x0F // sp mod 16-bytes + cmp r1, #4 // need space for the sp on the stack + addlt r1, r1, #0x10 // make room if needed, but keep stack aligned + mov r2, sp // get current sp + sub sp, sp, r1 // align stack + str r2, [sp] // store previous sp on stack +#endif + + mov r1, T_PREFETCH_ABT // Pass abort type + bl EXT(sleh_abort) // Call second level handler + +#if __BIGGEST_ALIGNMENT__ > 4 + ldr sp, [sp] // restore stack +#endif + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#if __ARM_USER_PROTECT__ + ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r10, r0 + beq 1f + ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb + cmp r10, r0 + beq 1f + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + ldr r11, [r9, ACT_ASID] // Load thread asid +1: + mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR + isb +#endif + + b load_and_go_sys + + +/* + * First Level Exception Handler for Data Abort + */ + .text + .align 2 + .globl EXT(fleh_dataabt) + +LEXT(fleh_dataabt) + sub lr, lr, #8 + + mrs sp, spsr // For check the previous mode + tst sp, #0x0f // Is it from kernel? + bne dataabt_from_kernel + +dataabt_from_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + + stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB + mov r7, #0 // Zero the frame pointer + nop + + mov r0, sp // Store arm_saved_state pointer + // For argument + + str lr, [sp, SS_PC] // Save user mode pc register + + mrs r4, spsr + str r4, [sp, SS_CPSR] // Save user mode cpsr + + mrc p15, 0, r5, c5, c0 // Read Fault Status + mrc p15, 0, r6, c6, c0 // Read Fault Address + str r5, [sp, SS_STATUS] // Save fault status register to pcb + str r6, [sp, SS_VADDR] // Save fault address to pcb + + mrs r4, cpsr // Read cpsr + cpsid i, #PSR_SVC_MODE + mrs r3, cpsr // Read cpsr + msr spsr_cxsf, r3 // Set spsr(svc mode cpsr) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r3, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r3 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + and r0, r4, #PSR_MODE_MASK // Extract current mode + cmp r0, #PSR_ABT_MODE // Check abort mode + bne EXT(ExceptionVectorPanic) + + mvn r0, #0 + str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#endif + + add r0, r9, ACT_PCBDATA // Reload arm_saved_state pointer + mov r1, T_DATA_ABT // Pass abort type + bl EXT(sleh_abort) // Call second level handler + // Sleh will enable irq + b load_and_go_user + +dataabt_from_kernel: + mrs sp, cpsr // Read cpsr + and sp, sp, #PSR_MODE_MASK // Extract current mode + cmp sp, #PSR_ABT_MODE // Check abort mode + movne r0, sp + bne EXT(ExceptionVectorPanic) + mrs sp, spsr // Check the previous mode + + /* + * We have a kernel stack already, and I will use it to save contexts: + * ------------------ + * | VFP saved state | + * |------------------| + * | ARM saved state | + * SP ------------------ + * + * IRQ is disabled + */ + cpsid i, #PSR_SVC_MODE + + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif + + mov ip, sp + + cpsid i, #PSR_ABT_MODE + + str lr, [ip, SS_PC] + mrs r4, spsr + str r4, [ip, SS_CPSR] + + cpsid i, #PSR_SVC_MODE + +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r3, r10 + beq 1f + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 +1: + mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mrc p15, 0, r5, c5, c0 // Read Fault Status + mrc p15, 0, r6, c6, c0 // Read Fault Address + str r5, [sp, SS_STATUS] // Save fault status register to pcb + str r6, [sp, SS_VADDR] // Save fault address to pcb + + mov r0, sp // Argument + +/* + * For armv7k ABI, the stack needs to be 16-byte aligned + */ +#if __BIGGEST_ALIGNMENT__ > 4 + and r1, sp, #0x0F // sp mod 16-bytes + cmp r1, #4 // need space for the sp on the stack + addlt r1, r1, #0x10 // make room if needed, but keep stack aligned + mov r2, sp // get current sp + sub sp, sp, r1 // align stack + str r2, [sp] // store previous sp on stack +#endif + + mov r1, T_DATA_ABT // Pass abort type + bl EXT(sleh_abort) // Call second level handler + +#if __BIGGEST_ALIGNMENT__ > 4 + ldr sp, [sp] // restore stack (removed align padding) +#endif + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#if __ARM_USER_PROTECT__ + ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r10, r0 + beq 1f + ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb + cmp r10, r0 + beq 1f + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + ldr r11, [r9, ACT_ASID] // Load thread asid +1: + mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR + isb +#endif + +load_and_go_sys: + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + + ldr r4, [sp, SS_CPSR] // Load saved cpsr + tst r4, #PSR_IRQF // Test IRQ set + bne lags1 // Branch if IRQ disabled + + cpsid i // Disable IRQ + ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count + movs r2, r2 // Test if null + ldr r8, [r9, ACT_CPUDATAP] // Get current cpu + bne lags1 // Branch if count not null + ldr r5, [r8, CPU_PENDING_AST] // Get ASTs + ands r5, r5, AST_URGENT // Get the requests we do honor + beq lags1 // Branch if no ASTs +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r3, r10 + beq 1f + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 +1: + mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + ldr lr, [sp, SS_LR] // Restore the link register + stmfd sp!, {r7, lr} // Push a fake frame + + /* TODO: Should this be setting r7? I think so. */ + mov r7, sp // Set the frame pointer + +#if __BIGGEST_ALIGNMENT__ > 4 + and r2, sp, #0x0F // sp mod 16-bytes + cmp r2, #4 // need space for the sp on the stack + addlt r2, r2, #0x10 // make room if needed, but keep stack aligned + mov r3, sp // get current sp + sub sp, sp, r2 // align stack + str r3, [sp] // store previous sp on stack +#endif + + bl EXT(ast_taken_kernel) // Handle AST_URGENT + +#if __BIGGEST_ALIGNMENT__ > 4 + ldr sp, [sp] +#endif + + + ldmfd sp!, {r7, lr} // Pop the fake frame + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + ldr r8, [r9, ACT_CPUDATAP] // Get current cpu +#if __ARM_USER_PROTECT__ + ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r10, r0 + beq 1f + ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb + cmp r10, r0 + beq 1f + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + ldr r11, [r9, ACT_ASID] // Load thread asid +1: + mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR + isb +#endif +lags1: + ldr lr, [sp, SS_LR] + + mov ip, sp // Save pointer to contexts for abort mode + ldr sp, [ip, SS_SP] // Restore stack pointer + + cpsid if, #PSR_ABT_MODE + + mov sp, ip + + ldr r4, [sp, SS_CPSR] + msr spsr_cxsf, r4 // Restore spsr + + clrex // clear exclusive memory tag +#if __ARM_ENABLE_WFE_ + sev +#endif + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_load) // Load the desired VFP state from the stack +#endif + + ldr lr, [sp, SS_PC] // Restore lr + + ldmia sp, {r0-r12} // Restore other registers + + movs pc, lr // Return to sys (svc, irq, fiq) + +/* + * First Level Exception Handler for address exception + * Not supported + */ + .text + .align 2 + .globl EXT(fleh_addrexc) + +LEXT(fleh_addrexc) + b . + + +/* + * First Level Exception Handler for IRQ + * Current mode : IRQ + * IRQ and FIQ are always disabled while running in FIQ handler + * We do not permit nested interrupt. + * + * Saving area: from user : PCB. + * from kernel : interrupt stack. + */ + + .text + .align 2 + .globl EXT(fleh_irq) + +LEXT(fleh_irq) + sub lr, lr, #4 + + cpsie a // Re-enable async aborts + + mrs sp, spsr + tst sp, #0x0f // From user? or kernel? + bne fleh_irq_kernel + +fleh_irq_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + stmia sp, {r0-r12, sp, lr}^ + mov r7, #0 // Zero the frame pointer + nop + str lr, [sp, SS_PC] + mrs r4, spsr + str r4, [sp, SS_CPSR] + mov r5, sp // Saved context in r5 + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr r6, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [r6, CPU_ISTACKPTR] // Set interrupt stack + cpsid i, #PSR_SVC_MODE + ldr sp, [r9, TH_KSTACKPTR] // Set kernel stack + cpsid i, #PSR_IRQ_MODE + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#endif +#if CONFIG_TELEMETRY + LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested... + mov r0, #1 + ldr r2, [r2] + movs r2, r2 + beq 1f + bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread... + mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW +1: +#endif + + b fleh_irq_handler + +fleh_irq_kernel: + cpsid i, #PSR_SVC_MODE + + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r5, sp // Saved context in r5 + + cpsid i, #PSR_IRQ_MODE + + str lr, [r5, SS_PC] // Save LR as the return PC + mrs r4, spsr + str r4, [r5, SS_CPSR] // Save the cpsr of the interrupted mode + + ldr sp, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [sp, CPU_ISTACKPTR] // Set interrupt stack + +#if CONFIG_TELEMETRY + LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested... + mov r0, #0 + ldr r2, [r2] + movs r2, r2 + beq 1f + bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread... + mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW +1: +#endif + +fleh_irq_handler: + ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count + add r2, r2, #1 // Increment count + str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count +#ifndef NO_KDEBUG + LOAD_ADDR(r8, kdebug_enable) + ldr r8, [r8] + movs r8, r8 + movne r0, r5 + COND_EXTERN_BLNE(interrupt_trace) +#endif + bl EXT(interrupt_stats) // Record interrupt statistics + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + ldr r4, [r9, ACT_CPUDATAP] // Get current cpu + str r5, [r4, CPU_INT_STATE] // Saved context in cpu_int_state + ldr r3, [r4, CPU_STAT_IRQ] // Get IRQ count + add r3, r3, #1 // Increment count + str r3, [r4, CPU_STAT_IRQ] // Update IRQ count + ldr r3, [r4, CPU_STAT_IRQ_WAKE] // Get post-wake IRQ count + add r3, r3, #1 // Increment count + str r3, [r4, CPU_STAT_IRQ_WAKE] // Update post-wake IRQ count + ldr r0, [r4, INTERRUPT_TARGET] + ldr r1, [r4, INTERRUPT_REFCON] + ldr r2, [r4, INTERRUPT_NUB] + ldr r3, [r4, INTERRUPT_SOURCE] + ldr r5, [r4, INTERRUPT_HANDLER] // Call second level exception handler + blx r5 +#ifndef NO_KDEBUG + movs r8, r8 + COND_EXTERN_BLNE(interrupt_trace_exit) +#endif + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + bl EXT(ml_get_timebase) // get current timebase + LOAD_ADDR(r3, EntropyData) + ldr r2, [r3, ENTROPY_INDEX_PTR] + add r1, r3, ENTROPY_DATA_SIZE + add r2, r2, #4 + cmp r2, r1 + addge r2, r3, ENTROPY_BUFFER + ldr r4, [r2] + eor r0, r0, r4, ROR #9 + str r0, [r2] // Update gEntropie + str r2, [r3, ENTROPY_INDEX_PTR] + +return_from_irq: + mov r5, #0 + ldr r4, [r9, ACT_CPUDATAP] // Get current cpu + str r5, [r4, CPU_INT_STATE] // Clear cpu_int_state + ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count +#if MACH_ASSERT + cmp r2, #0 // verify positive count + bgt 1f + push {r7, lr} + mov r7, sp + adr r0, L_preemption_count_zero_str + blx EXT(panic) + b . +1: +#endif + sub r2, r2, #1 // Decrement count + str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count + + mrs r0, spsr // For check the previous mode + + cpsid i, #PSR_SVC_MODE + + tst r0, #0x0f // Check if the previous is from user + ldreq sp, [r9, TH_KSTACKPTR] // ...If so, reload the kernel stack pointer + beq load_and_go_user // ...and return + +#if __ARM_USER_PROTECT__ + ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r10, r0 + beq 1f + ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb + cmp r10, r0 + beq 1f + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + ldr r11, [r9, ACT_ASID] // Load thread asid +1: + mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR + isb +#endif + b load_and_go_sys + + .align 2 +L_preemption_count_zero_str: + .ascii "locore.s: preemption count is zero \000" + .align 2 +/* + * First Level Exception Handler for DEC + * Current mode : IRQ + * IRQ and FIQ are always disabled while running in FIQ handler + * We do not permit nested interrupt. + * + * Saving area: from user : PCB. + * from kernel : interrupt stack. + */ + + .text + .align 2 + .globl EXT(fleh_decirq) + +LEXT(fleh_decirq) + sub lr, lr, #4 + + cpsie af // Re-enable async aborts/FIQ + + mrs sp, spsr + tst sp, #0x0f // From user? or kernel? + bne fleh_decirq_kernel + +fleh_decirq_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + stmia sp, {r0-r12, sp, lr}^ + mov r7, #0 // Zero the frame pointer + nop + str lr, [sp, SS_PC] + mrs r4, spsr + str r4, [sp, SS_CPSR] + mov r5, sp // Saved context in r5 + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr r6, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [r6, CPU_ISTACKPTR] // Set interrupt stack + cpsid i, #PSR_SVC_MODE + ldr sp, [r9, TH_KSTACKPTR] // Set kernel stack + cpsid i, #PSR_IRQ_MODE + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#endif +#if CONFIG_TELEMETRY + LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested... + mov r0, #1 + ldr r2, [r2] + movs r2, r2 + beq 1f + bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread... + mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW +1: +#endif + + b fleh_decirq_handler + +fleh_decirq_kernel: + cpsid i, #PSR_SVC_MODE + + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r5, sp // Saved context in r5 + + cpsid i, #PSR_IRQ_MODE + + str lr, [r5, SS_PC] // Save LR as the return PC + mrs r4, spsr + str r4, [r5, SS_CPSR] // Save the cpsr of the interrupted mode + + ldr sp, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [sp, CPU_ISTACKPTR] // Set interrupt stack + +#if CONFIG_TELEMETRY + LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested... + mov r0, #0 + ldr r2, [r2] + movs r2, r2 + beq 1f + bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread... + mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW +1: +#endif + +fleh_decirq_handler: + ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count + add r2, r2, #1 // Increment count + str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count + ldr r2, [r9, ACT_CPUDATAP] // Get current cpu + str r5, [r2, CPU_INT_STATE] // Saved context in cpu_int_state + ldr r3, [r2, CPU_STAT_IRQ] // Get IRQ count + add r3, r3, #1 // Increment count + str r3, [r2, CPU_STAT_IRQ] // Update IRQ count + ldr r3, [r2, CPU_STAT_IRQ_WAKE] // Get post-wake IRQ count + add r3, r3, #1 // Increment count + str r3, [r2, CPU_STAT_IRQ_WAKE] // Update post-wake IRQ count +#ifndef NO_KDEBUG + LOAD_ADDR(r4, kdebug_enable) + ldr r4, [r4] + movs r4, r4 + movne r0, r5 // Pass saved context + COND_EXTERN_BLNE(interrupt_trace) +#endif + bl EXT(interrupt_stats) // Record interrupt statistics + mov r0, #0 + bl EXT(rtclock_intr) // Call second level exception handler +#ifndef NO_KDEBUG + movs r4, r4 + COND_EXTERN_BLNE(interrupt_trace_exit) +#endif + + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + + b return_from_irq + + +/* + * First Level Exception Handler for FIQ + * Current mode : FIQ + * IRQ and FIQ are always disabled while running in FIQ handler + * We do not permit nested interrupt. + * + * Saving area: from user : PCB. + * from kernel : interrupt stack. + * + * We have 7 added shadow registers in FIQ mode for fast services. + * So only we have to save is just 8 general registers and LR. + * But if the current thread was running on user mode before the FIQ interrupt, + * All user registers be saved for ast handler routine. + */ + .text + .align 2 + .globl EXT(fleh_fiq_generic) + +LEXT(fleh_fiq_generic) + str r11, [r10] // Clear the FIQ source + + ldr r13, [r8, CPU_TIMEBASE_LOW] // Load TBL + adds r13, r13, #1 // Increment TBL + str r13, [r8, CPU_TIMEBASE_LOW] // Store TBL + ldreq r13, [r8, CPU_TIMEBASE_HIGH] // Load TBU + addeq r13, r13, #1 // Increment TBU + streq r13, [r8, CPU_TIMEBASE_HIGH] // Store TBU + subs r12, r12, #1 // Decrement, DEC + str r12, [r8, CPU_DECREMENTER] // Store DEC + subspl pc, lr, #4 // Return unless DEC < 0 + b EXT(fleh_dec) + + .text + .align 2 + .globl EXT(fleh_dec) +LEXT(fleh_dec) + mrs sp, spsr // Get the spsr + sub lr, lr, #4 + tst sp, #0x0f // From user? or kernel? + bne 2f + + /* From user */ + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + + stmia sp, {r0-r12, sp, lr}^ + mov r7, #0 // Zero the frame pointer + nop + str lr, [sp, SS_PC] + + mrs r4, spsr + str r4, [sp, SS_CPSR] + mov r5, sp + sub sp, sp, ACT_PCBDATA // Get User PCB + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu + ldr sp, [sp, CPU_ISTACKPTR] // Set interrupt stack + mov r6, sp + cpsid i, #PSR_SVC_MODE + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr sp, [r9, TH_KSTACKPTR] // Set kernel stack + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r0, #1 // Mark this as coming from user context + b 4f + +2: + /* From kernel */ + tst sp, #PSR_IRQF // Test for IRQ masked + bne 3f // We're on the cpu_signal path + + cpsid if, #PSR_SVC_MODE + + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r5, sp // Saved context in r5 + + cpsid if, #PSR_FIQ_MODE + + mrc p15, 0, r1, c13, c0, 4 // Read TPIDRPRW + + str lr, [r5, SS_PC] // Save LR as the return PC + mrs r4, spsr + str r4, [r5, SS_CPSR] // Save the cpsr of the interrupted mode + + ldr r6, [r1, ACT_CPUDATAP] // Get current cpu + ldr r6, [r6, CPU_ISTACKPTR] // Set interrupt stack + + mov r0, #0 // Mark this as coming from kernel context + b 4f + +3: + /* cpu_signal path */ + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu + ldr sp, [sp, CPU_FIQSTACKPTR] // Set fiq stack + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + str lr, [sp, SS_PC] + mrs r4, spsr + str r4, [sp, SS_CPSR] + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r0, r8 // Get current cpu in arg 0 + mov r1, SIGPdec // Decrementer signal in arg1 + mov r2, #0 + mov r3, #0 + bl EXT(cpu_signal) // Call cpu_signal + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_load) // Load the desired VFP state from the stack +#endif + + clrex // clear exclusive memory tag +#if __ARM_ENABLE_WFE_ + sev +#endif +#if __ARM_USER_PROTECT__ + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + mcr p15, 0, r11, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + ldr lr, [sp, SS_PC] + ldmia sp, {r0-r12} // Restore saved registers + movs pc, lr // Return from fiq + +4: + cpsid i, #PSR_IRQ_MODE + cpsie f + mov sp, r6 // Restore the stack pointer + msr spsr_cxsf, r4 // Restore the spsr + ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count + add r2, r2, #1 // Increment count + str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count + ldr r4, [r9, ACT_CPUDATAP] // Get current cpu + str r5, [r4, CPU_INT_STATE] + ldr r3, [r4, CPU_STAT_IRQ] // Get IRQ count + add r3, r3, #1 // Increment count + str r3, [r4, CPU_STAT_IRQ] // Update IRQ count + ldr r3, [r4, CPU_STAT_IRQ_WAKE] // Get post-wake IRQ count + add r3, r3, #1 // Increment count + str r3, [r4, CPU_STAT_IRQ_WAKE] // Update post-wake IRQ count +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + movs r0, r0 + beq 5f + mov r8, r0 // Stash our "from_user" boolean value + bl EXT(timer_state_event_user_to_kernel) + mov r0, r8 // Restore our "from_user" value + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +5: +#endif +#if CONFIG_TELEMETRY + LOAD_ADDR(r4, telemetry_needs_record) // Check if a telemetry record was requested... + ldr r4, [r4] + movs r4, r4 + beq 6f + bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread... + mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW +6: +#endif + +#ifndef NO_KDEBUG + LOAD_ADDR(r4, kdebug_enable) + ldr r4, [r4] + movs r4, r4 + ldrne r1, [r9, ACT_CPUDATAP] // Get current cpu + ldrne r0, [r1, CPU_INT_STATE] + COND_EXTERN_BLNE(interrupt_trace) +#endif + bl EXT(interrupt_stats) // Record interrupt statistics + mov r0, #0 + bl EXT(rtclock_intr) // Call second level exception handler +#ifndef NO_KDEBUG + movs r4, r4 + COND_EXTERN_BLNE(interrupt_trace_exit) +#endif + + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + + b return_from_irq + +/* + * void thread_syscall_return(kern_return_t r0) + * + */ + .text + .align 2 + .globl EXT(thread_syscall_return) + +LEXT(thread_syscall_return) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + add r1, r9, ACT_PCBDATA // Get User PCB + str r0, [r1, SS_R0] // set return value +#ifndef NO_KDEBUG + LOAD_ADDR(r4, kdebug_enable) + ldr r4, [r4] + movs r4, r4 + beq load_and_go_user + ldr r12, [r1, SS_R12] // Load syscall number + rsbs r1, r12, #0 // make the syscall positive (if negative) + COND_EXTERN_BLGT(mach_syscall_trace_exit) +#endif + b load_and_go_user + +/* + * void thread_exception_return(void) + * void thread_bootstrap_return(void) + * + */ + .text + .globl EXT(thread_exception_return) + .globl EXT(thread_bootstrap_return) + +LEXT(thread_bootstrap_return) +#if CONFIG_DTRACE + bl EXT(dtrace_thread_bootstrap) +#endif + // Fall through + +LEXT(thread_exception_return) + +load_and_go_user: +/* + * Restore user mode states and go back to user mode + */ + cpsid i // Disable irq + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + + mvn r0, #0 + str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before returning to user + + ldr r8, [r9, ACT_CPUDATAP] // Get current cpu + ldr r5, [r8, CPU_PENDING_AST] // Get ASTs + cmp r5, #0 // Test if ASTs pending + beq return_to_user_now // Branch if no ASTs + +#if __BIGGEST_ALIGNMENT__ > 4 + and r2, sp, #0x0F // sp mod 16-bytes + cmp r2, #4 // need space for the sp on the stack + addlt r2, r2, #0x10 // make room if needed, but keep stack aligned + mov r3, sp // get current sp + sub sp, sp, r2 // align stack + str r3, [sp] // store previous sp on stack +#endif + + bl EXT(ast_taken_user) // Handle all ASTs (may continue via thread_exception_return) + +#if __BIGGEST_ALIGNMENT__ > 4 + ldr sp, [sp] // Restore the stack pointer +#endif + + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + b load_and_go_user // Loop back + +return_to_user_now: + +#if MACH_ASSERT +/* + * Assert that the preemption level is zero prior to the return to user space + */ + ldr r1, [r9, ACT_PREEMPT_CNT] // Load preemption count + movs r1, r1 // Test + beq 0f // Continue if zero, or... + adr r0, L_lagu_panic_str // Load the panic string... + blx EXT(panic) // Finally, panic +0: + ldr r2, [r9, TH_RWLOCK_CNT] // Load RW lock count + movs r2, r2 // Test + beq 0f // Continue if zero, or... + adr r0, L_lagu_rwlock_cnt_panic_str // Load the panic string... + mov r1, r9 // Thread argument for panic string + blx EXT(panic) // Finally, panic +#endif + +0: +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_kernel_to_user) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr r8, [r9, ACT_CPUDATAP] // Get current cpu data +#endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */ +#if __ARM_DEBUG__ >= 6 + ldr r0, [r9, ACT_DEBUGDATA] + ldr r6, [r8, CPU_USER_DEBUG] + cmp r0, r6 // test if debug registers need to be changed + beq 1f + bl EXT(arm_debug_set) // argument is already in r0 + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +1: +#endif +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_load) // Load the desired VFP state from ACT_UVFP +#endif + add r0, r9, ACT_PCBDATA // Get User PCB + ldr r4, [r0, SS_CPSR] // Get saved cpsr + and r3, r4, #PSR_MODE_MASK // Extract current mode + cmp r3, #PSR_USER_MODE // Check user mode + movne r0, r3 + bne EXT(ExceptionVectorPanic) + + msr spsr_cxsf, r4 // Restore spsr(user mode cpsr) + mov sp, r0 // Get User PCB + + clrex // clear exclusive memory tag +#if __ARM_ENABLE_WFE_ + sev +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_UPTW_TTB] // Load thread ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + ldr r2, [r9, ACT_ASID] // Load thread asid + mcr p15, 0, r2, c13, c0, 1 + isb +#endif + ldr lr, [sp, SS_PC] // Restore user mode pc + ldmia sp, {r0-r12, sp, lr}^ // Restore the other user mode registers + nop // Hardware problem + movs pc, lr // Return to user + + .align 2 +L_lagu_panic_str: + .asciz "load_and_go_user: preemption_level %d" + .align 2 + + .align 2 +L_lagu_rwlock_cnt_panic_str: + .asciz "load_and_go_user: RW lock count not 0 on thread %p (%u)" + .align 2 + + .align 2 +L_evimpanic_str: + .ascii "Exception Vector: Illegal Mode: 0x%08X\n\000" + .align 2 + + .text + .align 2 + .globl EXT(ExceptionVectorPanic) + +LEXT(ExceptionVectorPanic) + cpsid i, #PSR_SVC_MODE + mov r1, r0 + adr r0, L_evimpanic_str + blx EXT(panic) + b . + +#include "globals_asm.h" + +LOAD_ADDR_GEN_DEF(mach_trap_table) +LOAD_ADDR_GEN_DEF(kern_invalid) + +/* vim: set ts=4: */ diff --git a/osfmk/arm/loose_ends.c b/osfmk/arm/loose_ends.c new file mode 100644 index 000000000..46aeec6da --- /dev/null +++ b/osfmk/arm/loose_ends.c @@ -0,0 +1,665 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !MACH_KDP +#include +#endif /* !MACH_KDP */ +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#define INT_SIZE (BYTE_SIZE * sizeof (int)) + + +void +bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes) +{ + unsigned int src_index; + unsigned int dst_index; + vm_offset_t src_offset; + vm_offset_t dst_offset; + unsigned int cpu_num; + unsigned int wimg_bits_src, wimg_bits_dst; + ppnum_t pn_src = (src >> PAGE_SHIFT); + ppnum_t pn_dst = (dst >> PAGE_SHIFT); + + wimg_bits_src = pmap_cache_attributes(pn_src); + wimg_bits_dst = pmap_cache_attributes(pn_dst); + + if (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)) && + ((wimg_bits_src & VM_WIMG_MASK) == VM_WIMG_DEFAULT) && + ((wimg_bits_dst & VM_WIMG_MASK) == VM_WIMG_DEFAULT)) { + /* Fast path - dst is writable and both source and destination have default attributes */ + bcopy((char *)phystokv((pmap_paddr_t) src), (char *)phystokv((pmap_paddr_t) dst), bytes); + return; + } + + src_offset = src & PAGE_MASK; + dst_offset = dst & PAGE_MASK; + + if ((src_offset + bytes) > PAGE_SIZE || (dst_offset + bytes) > PAGE_SIZE) + panic("bcopy extends beyond copy windows"); + + mp_disable_preemption(); + cpu_num = cpu_number(); + src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src); + dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ|VM_PROT_WRITE, wimg_bits_dst); + + bcopy((char *)(pmap_cpu_windows_copy_addr(cpu_num, src_index)+src_offset), + (char *)(pmap_cpu_windows_copy_addr(cpu_num, dst_index)+dst_offset), + bytes); + + pmap_unmap_cpu_windows_copy(src_index); + pmap_unmap_cpu_windows_copy(dst_index); + mp_enable_preemption(); +} + +void +bzero_phys_nc(addr64_t src64, vm_size_t bytes) +{ + bzero_phys(src64, bytes); +} + +/* Zero bytes starting at a physical address */ +void +bzero_phys(addr64_t src, vm_size_t bytes) +{ + unsigned int wimg_bits; + ppnum_t pn = (src >> PAGE_SHIFT); + + wimg_bits = pmap_cache_attributes(pn); + if ((wimg_bits & VM_WIMG_MASK) == VM_WIMG_DEFAULT) { + /* Fast path - default attributes */ + bzero((char *)phystokv((pmap_paddr_t) src), bytes); + } else { + mp_disable_preemption(); + + unsigned int cpu_num = cpu_number(); + + while (bytes > 0) { + vm_offset_t offset = src & PAGE_MASK; + uint32_t count = PAGE_SIZE - offset; + + if (count > bytes) + count = bytes; + + unsigned int index = pmap_map_cpu_windows_copy(src >> PAGE_SHIFT, VM_PROT_READ | VM_PROT_WRITE, wimg_bits); + + bzero((char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset), count); + + pmap_unmap_cpu_windows_copy(index); + + src += count; + bytes -= count; + } + + mp_enable_preemption(); + } +} + +/* + * Read data from a physical address. + */ + + +static unsigned int +ml_phys_read_data(pmap_paddr_t paddr, int size) +{ + unsigned int index; + unsigned int result; + unsigned int wimg_bits; + ppnum_t pn = (paddr >> PAGE_SHIFT); + unsigned char s1; + unsigned short s2; + vm_offset_t copywindow_vaddr = 0; + + mp_disable_preemption(); + wimg_bits = pmap_cache_attributes(pn); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits); + copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);; + + switch (size) { + case 1: + s1 = *(volatile unsigned char *)(copywindow_vaddr); + result = s1; + break; + case 2: + s2 = *(volatile unsigned short *)(copywindow_vaddr); + result = s2; + break; + case 4: + default: + result = *(volatile unsigned int *)(copywindow_vaddr); + break; + } + + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); + + return result; +} + +static unsigned long long +ml_phys_read_long_long(pmap_paddr_t paddr) +{ + unsigned int index; + unsigned int result; + unsigned int wimg_bits; + ppnum_t pn = (paddr >> PAGE_SHIFT); + + mp_disable_preemption(); + wimg_bits = pmap_cache_attributes(pn); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits); + + result = *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index) + | ((uint32_t)paddr & PAGE_MASK)); + + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); + + return result; +} + +unsigned int ml_phys_read( vm_offset_t paddr) +{ + return ml_phys_read_data((pmap_paddr_t)paddr, 4); +} + +unsigned int ml_phys_read_word(vm_offset_t paddr) { + + return ml_phys_read_data((pmap_paddr_t)paddr, 4); +} + +unsigned int ml_phys_read_64(addr64_t paddr64) +{ + return ml_phys_read_data((pmap_paddr_t)paddr64, 4); +} + +unsigned int ml_phys_read_word_64(addr64_t paddr64) +{ + return ml_phys_read_data((pmap_paddr_t)paddr64, 4); +} + +unsigned int ml_phys_read_half(vm_offset_t paddr) +{ + return ml_phys_read_data((pmap_paddr_t)paddr, 2); +} + +unsigned int ml_phys_read_half_64(addr64_t paddr64) +{ + return ml_phys_read_data((pmap_paddr_t)paddr64, 2); +} + +unsigned int ml_phys_read_byte(vm_offset_t paddr) +{ + return ml_phys_read_data((pmap_paddr_t)paddr, 1); +} + +unsigned int ml_phys_read_byte_64(addr64_t paddr64) +{ + return ml_phys_read_data((pmap_paddr_t)paddr64, 1); +} + +unsigned long long ml_phys_read_double(vm_offset_t paddr) +{ + return ml_phys_read_long_long((pmap_paddr_t)paddr); +} + +unsigned long long ml_phys_read_double_64(addr64_t paddr64) +{ + return ml_phys_read_long_long((pmap_paddr_t)paddr64); +} + + + +/* + * Write data to a physical address. + */ + +static void +ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size) +{ + unsigned int index; + unsigned int wimg_bits; + ppnum_t pn = (paddr >> PAGE_SHIFT); + vm_offset_t copywindow_vaddr = 0; + + mp_disable_preemption(); + wimg_bits = pmap_cache_attributes(pn); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t) paddr & PAGE_MASK); + + switch (size) { + case 1: + *(volatile unsigned char *)(copywindow_vaddr) = (unsigned char)data; + break; + case 2: + *(volatile unsigned short *)(copywindow_vaddr) = (unsigned short)data; + break; + case 4: + default: + *(volatile unsigned int *)(copywindow_vaddr) = (uint32_t)data; + break; + } + + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); +} + +static void +ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data) +{ + unsigned int index; + unsigned int wimg_bits; + ppnum_t pn = (paddr >> PAGE_SHIFT); + + mp_disable_preemption(); + wimg_bits = pmap_cache_attributes(pn); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + + *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index) + | ((uint32_t)paddr & PAGE_MASK)) = data; + + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); +} + + + +void ml_phys_write_byte(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 1); +} + +void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 1); +} + +void ml_phys_write_half(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 2); +} + +void ml_phys_write_half_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 2); +} + +void ml_phys_write(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 4); +} + +void ml_phys_write_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); +} + +void ml_phys_write_word(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 4); +} + +void ml_phys_write_word_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); +} + +void ml_phys_write_double(vm_offset_t paddr, unsigned long long data) +{ + ml_phys_write_long_long((pmap_paddr_t)paddr, data); +} + +void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) +{ + ml_phys_write_long_long((pmap_paddr_t)paddr64, data); +} + + +/* + * Set indicated bit in bit string. + */ +void +setbit(int bitno, int *s) +{ + s[bitno / INT_SIZE] |= 1 << (bitno % INT_SIZE); +} + +/* + * Clear indicated bit in bit string. + */ +void +clrbit(int bitno, int *s) +{ + s[bitno / INT_SIZE] &= ~(1 << (bitno % INT_SIZE)); +} + +/* + * Test if indicated bit is set in bit string. + */ +int +testbit(int bitno, int *s) +{ + return s[bitno / INT_SIZE] & (1 << (bitno % INT_SIZE)); +} + +/* + * Find first bit set in bit string. + */ +int +ffsbit(int *s) +{ + int offset; + + for (offset = 0; !*s; offset += INT_SIZE, ++s); + return offset + __builtin_ctz(*s); +} + +int +ffs(unsigned int mask) +{ + if (mask == 0) + return 0; + + /* + * NOTE: cannot use __builtin_ffs because it generates a call to + * 'ffs' + */ + return 1 + __builtin_ctz(mask); +} + +int +ffsll(unsigned long long mask) +{ + if (mask == 0) + return 0; + + /* + * NOTE: cannot use __builtin_ffsll because it generates a call to + * 'ffsll' + */ + return 1 + __builtin_ctzll(mask); +} + +/* + * Find last bit set in bit string. + */ +int +fls(unsigned int mask) +{ + if (mask == 0) + return 0; + + return (sizeof (mask) << 3) - __builtin_clz(mask); +} + +int +flsll(unsigned long long mask) +{ + if (mask == 0) + return 0; + + return (sizeof (mask) << 3) - __builtin_clzll(mask); +} + +int +bcmp( + const void *pa, + const void *pb, + size_t len) +{ + const char *a = (const char *) pa; + const char *b = (const char *) pb; + + if (len == 0) + return 0; + + do + if (*a++ != *b++) + break; + while (--len); + + return len; +} + +int +memcmp(const void *s1, const void *s2, size_t n) +{ + if (n != 0) { + const unsigned char *p1 = s1, *p2 = s2; + + do { + if (*p1++ != *p2++) + return (*--p1 - *--p2); + } while (--n != 0); + } + return (0); +} + +kern_return_t +copypv(addr64_t source, addr64_t sink, unsigned int size, int which) +{ + kern_return_t retval = KERN_SUCCESS; + void *from, *to; + unsigned int from_wimg_bits, to_wimg_bits; + + from = CAST_DOWN(void *, source); + to = CAST_DOWN(void *, sink); + + if ((which & (cppvPsrc | cppvPsnk)) == 0) /* Make sure that only + * one is virtual */ + panic("copypv: no more than 1 parameter may be virtual\n"); /* Not allowed */ + + if (which & cppvPsrc) + from = (void *)phystokv(from); + if (which & cppvPsnk) + to = (void *)phystokv(to); + + if ((which & (cppvPsrc | cppvKmap)) == 0) /* Source is virtual in + * current map */ + retval = copyin((user_addr_t) from, to, size); + else if ((which & (cppvPsnk | cppvKmap)) == 0) /* Sink is virtual in + * current map */ + retval = copyout(from, (user_addr_t) to, size); + else /* both addresses are physical or kernel map */ + bcopy(from, to, size); + + if (which & cppvFsrc) { + flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc)); + } else if (which & cppvPsrc) { + from_wimg_bits = pmap_cache_attributes(source >> PAGE_SHIFT); + if ((from_wimg_bits != VM_WIMG_COPYBACK) && (from_wimg_bits != VM_WIMG_WTHRU)) + flush_dcache64(source, size, TRUE); + } + + if (which & cppvFsnk) { + flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk)); + } else if (which & cppvPsnk) { + to_wimg_bits = pmap_cache_attributes(sink >> PAGE_SHIFT); + if (to_wimg_bits != VM_WIMG_COPYBACK) + flush_dcache64(sink, size, TRUE); + } + return retval; +} + +/* + * Copy sizes bigger than this value will cause a kernel panic. + * + * Yes, this is an arbitrary fixed limit, but it's almost certainly + * a programming error to be copying more than this amount between + * user and wired kernel memory in a single invocation on this + * platform. + */ +const int copysize_limit_panic = (64 * 1024 * 1024); + +/* + * Validate the arguments to copy{in,out} on this platform. + * + * Called when nbytes is "large" e.g. more than a page. Such sizes are + * infrequent, and very large sizes are likely indications of attempts + * to exploit kernel programming errors (bugs). + */ +static int +copy_validate(const user_addr_t user_addr, + uintptr_t kernel_addr, vm_size_t nbytes) +{ + uintptr_t kernel_addr_last = kernel_addr + nbytes; + + if (kernel_addr < VM_MIN_KERNEL_ADDRESS || + kernel_addr > VM_MAX_KERNEL_ADDRESS || + kernel_addr_last < kernel_addr || + kernel_addr_last > VM_MAX_KERNEL_ADDRESS) + panic("%s(%p, %p, %u) - kaddr not in kernel", __func__, + (void *)user_addr, (void *)kernel_addr, nbytes); + + user_addr_t user_addr_last = user_addr + nbytes; + + if (user_addr_last < user_addr || + user_addr_last > VM_MIN_KERNEL_ADDRESS) + return (EFAULT); + + if (__improbable(nbytes > copysize_limit_panic)) + panic("%s(%p, %p, %u) - transfer too large", __func__, + (void *)user_addr, (void *)kernel_addr, nbytes); + + return (0); +} + +int +copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes) +{ + return (copy_validate(ua, ka, nbytes)); +} + +int +copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes) +{ + return (copy_validate(ua, ka, nbytes)); +} + +#if MACH_ASSERT + +extern int copyinframe(vm_address_t fp, char *frame); + +/* + * Machine-dependent routine to fill in an array with up to callstack_max + * levels of return pc information. + */ +void +machine_callstack( + uintptr_t * buf, + vm_size_t callstack_max) +{ + /* Captures the USER call stack */ + uint32_t i=0; + uint32_t frame[2]; + + struct arm_saved_state* state = find_user_regs(current_thread()); + + if (!state) { + while (ipc; + frame[0] = state->r[7]; + + while (i: Reduce print noise + // kprintf("ml_thread_policy() unimplemented\n"); +} + +#if !MACH_KDP +void +kdp_register_callout(kdp_callout_fn_t fn, void *arg) +{ +#pragma unused(fn,arg) +} +#endif diff --git a/osfmk/arm/lowglobals.h b/osfmk/arm/lowglobals.h new file mode 100644 index 000000000..447a28b9a --- /dev/null +++ b/osfmk/arm/lowglobals.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Header files for the Low Memory Globals (lg) + */ +#ifndef _LOW_MEMORY_GLOBALS_H_ +#define _LOW_MEMORY_GLOBALS_H_ + +#ifndef __arm__ +#error Wrong architecture - this file is meant for arm +#endif + +#define LOWGLO_LAYOUT_MAGIC 0xC0DEC0DE + +#pragma pack(4) /* Make sure the structure stays as we defined it */ +typedef struct lowglo { + unsigned char lgVerCode[8]; /* 0xffff1000 System verification code */ + uint32_t lgZero[2]; /* 0xffff1008 Double constant 0 */ + uint32_t lgStext; /* 0xffff1010 Start of kernel text */ + uint32_t lgRsv014[2]; /* 0xffff1014 Reserved */ + uint32_t lgVersion; /* 0xffff101C Pointer to kernel version string */ + uint32_t lgRsv020[216]; /* 0xffff1020 Reserved */ + uint32_t lgKmodptr; /* 0xffff1380 Pointer to kmod, debugging aid */ + uint32_t lgTransOff; /* 0xffff1384 Pointer to kdp_trans_off, debugging aid */ + uint32_t lgRsv388[3]; /* 0xffff1388 Reserved */ + uint32_t lgOSVersion; /* 0xffff1394 Pointer to OS version string */ + uint32_t lgRsv398; /* 0xffff1398 Reserved */ + uint32_t lgRebootFlag; /* 0xffff139C Pointer to debugger reboot trigger */ + uint32_t lgManualPktAddr; /* 0xffff13A0 Pointer to manual packet structure */ + uint32_t lgRsv3A4; /* 0xffff13A4 Reserved */ + uint32_t lgPmapMemQ; /* 0xffff13A8 Pointer to PMAP memory queue */ + uint32_t lgPmapMemPageOffset;/* 0xffff13AC Offset of physical page member in vm_page_with_ppnum_t */ + uint32_t lgPmapMemChainOffset;/*0xffff13B0 Offset of listq in vm_page_t or vm_page_with_ppnum_t */ + uint32_t lgStaticAddr; /* 0xffff13B4 Static allocation address */ + uint32_t lgStaticSize; /* 0xffff13B8 Static allocation size */ + uint32_t lgLayoutMajorVersion; /* 0xffff13BC Lowglo layout major version */ + uint32_t lgLayoutMagic; /* 0xffff13C0 Magic value evaluated to determine if lgLayoutVersion is valid */ + uint32_t lgPmapMemStartAddr; /* 0xffff13C4 Pointer to start of vm_page_t array */ + uint32_t lgPmapMemEndAddr; /* 0xffff13C8 Pointer to end of vm_page_t array */ + uint32_t lgPmapMemPagesize; /* 0xffff13CC size of vm_page_t */ + uint32_t lgPmapMemFirstppnum; /* 0xffff13D0 physical page number of the first vm_page_t in the array */ + uint32_t lgLayoutMinorVersion; /* 0xffff13D4 Lowglo layout minor version */ + uint32_t lgPageShift; /* 0xffff13D8 Number of shifts from page number to size */ +} lowglo; +#pragma pack() + +extern lowglo lowGlo; + +void patch_low_glo(void); +void patch_low_glo_static_region(uint32_t address, uint32_t size); +void patch_low_glo_vm_page_info(void *, void *, uint32_t); + +#endif /* _LOW_MEMORY_GLOBALS_H_ */ diff --git a/osfmk/arm/lowmem_vectors.c b/osfmk/arm/lowmem_vectors.c new file mode 100644 index 000000000..f710eb65f --- /dev/null +++ b/osfmk/arm/lowmem_vectors.c @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include + +extern vm_offset_t vm_kernel_stext; +extern void *version; +extern void *kmod; +extern void *kdp_trans_off; +extern void *osversion; +extern void *flag_kdp_trigger_reboot; +extern void *manual_pkt; +extern struct vm_object pmap_object_store; /* store pt pages */ + +lowglo lowGlo __attribute__ ((aligned(PAGE_MAX_SIZE))) = { + .lgVerCode = { 'O','c','t','o','p','u','s',' ' }, + // Increment the major version for changes that break the current Astris + // usage of lowGlo values + // Increment the minor version for changes that provide additonal info/function + // but does not break current usage + .lgLayoutMajorVersion = 3, + .lgLayoutMinorVersion = 0, + .lgLayoutMagic = LOWGLO_LAYOUT_MAGIC, + .lgVersion = (uint32_t)&version, + .lgKmodptr = (uint32_t)&kmod, + .lgPageShift = PAGE_SHIFT, +#if MACH_KDP + .lgTransOff = (uint32_t)&kdp_trans_off, +#endif + .lgOSVersion = (uint32_t)&osversion, +#if MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING + .lgRebootFlag = (uint32_t)&flag_kdp_trigger_reboot, + .lgManualPktAddr = (uint32_t)&manual_pkt, +#endif + .lgPmapMemQ = (uint32_t)&(pmap_object_store.memq), + .lgPmapMemPageOffset = offsetof(struct vm_page_with_ppnum, phys_page), + .lgPmapMemChainOffset = offsetof(struct vm_page, listq), + .lgPmapMemPagesize = (uint32_t)sizeof(struct vm_page), + + .lgPmapMemStartAddr = -1, + .lgPmapMemEndAddr = -1, + .lgPmapMemFirstppnum = -1 +}; + +void patch_low_glo(void) +{ + lowGlo.lgStext = (uint32_t)vm_kernel_stext; +} + +void patch_low_glo_static_region(uint32_t address, uint32_t size) +{ + lowGlo.lgStaticAddr = address; + lowGlo.lgStaticSize = size; +} + + +void patch_low_glo_vm_page_info(void * start_addr, void * end_addr, uint32_t first_ppnum) +{ + lowGlo.lgPmapMemStartAddr = (uint32_t)start_addr; + lowGlo.lgPmapMemEndAddr = (uint32_t)end_addr; + lowGlo.lgPmapMemFirstppnum = first_ppnum; +} diff --git a/osfmk/arm/lz4_decode_armv7NEON.s b/osfmk/arm/lz4_decode_armv7NEON.s new file mode 100644 index 000000000..f9faa2a55 --- /dev/null +++ b/osfmk/arm/lz4_decode_armv7NEON.s @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#if LZ4_ENABLE_ASSEMBLY_DECODE_ARMV7 + +/* + + int64_t lz4_decode_asm( + uint8_t ** dst_ptr, *dst_ptr points to next output byte to write + uint8_t * dst_begin, points to first valid output byte we can access, dst_begin <= dst + uint8_t * dst_end, "relaxed" end of output buffer (see below) + const uint8_t ** src_ptr, *src_ptr points to next input byte to read + const uint8_t * src_end) "relaxed" end of input buffer (see below) + + We test the position of the pointers only to ensure we don't access past src_end/dst_end + some fixed constant. + We never read before dst_begin. + + Return 0 on success, -1 on failure + On output, (*src_ptr,*dst_ptr) receives the last position in both buffers corresponding to the beginning of a LZ4 instruction. + +*/ + +.globl _lz4_decode_asm + +#define dst r0 // arg0 +#define dst_begin r1 // arg1 +#define dst_end r2 // arg2 +#define src r3 // arg3 + +#define src_end r4 // arg4 + +#define n_matches r5 +#define n_literals r10 +#define copy_src r11 // match/literal copy source +#define copy_dst r8 // match/literal copy destination + +#define aux1 r9 + +#define match_distance r6 + +#define match_permtable r12 +#define match_disttable lr + +#define dst_good [sp, #0] +#define src_good [sp, #4] + +.macro establish_frame + ldr ip, [sp, #0] // read src_end + push {r4-r7, lr} // Save registers + add r7, sp, #12 // Establish stack frame + push {r8-r11} + mov src_end, ip + push {r0, r3} // save dst/src + sub sp, sp, #4+16 // 4 for 16-byte stack alignment, extra 16-bytes for local +.endm + +.macro clear_frame_and_return + add sp, sp, #12+16 // skip r0/r3 + pop {r8-r11} + pop {r4-r7,pc} +.endm + +// copy_1x16 SOURCE_ADDR DESTINATION_ADDR +// Copy 16 bytes, clobber: q0 +.macro copy_1x16 + vld1.8 {q0}, [$0] + vst1.8 {q0}, [$1] +.endm + +// copy_1x16_and_increment SOURCE_ADDR DESTINATION_ADDR +// Copy 16 bytes, and increment both addresses by 16, clobber: q0 +.macro copy_1x16_and_increment + vld1.8 {q0}, [$0]! + vst1.8 {q0}, [$1]! +.endm + +// copy_2x16_and_increment SOURCE_ADDR DESTINATION_ADDR +// Copy 2 times 16 bytes, and increment both addresses by 32, clobber: q0 +.macro copy_2x16_and_increment + vld1.8 {q0}, [$0]! + vst1.8 {q0}, [$1]! + vld1.8 {q0}, [$0]! + vst1.8 {q0}, [$1]! +.endm + +// copy_1x32_and_increment SOURCE_ADDR DESTINATION_ADDR +// Copy 32 bytes, and increment both addresses by 32, clobber: q0,q1 +.macro copy_1x32_and_increment + vld1.8 {q0,q1}, [$0]! + vst1.8 {q0,q1}, [$1]! +.endm + +// If we don't branch, src < src_end after this +.macro check_src_end + cmp src,src_end + bhs L_done // extremely unlikely, DONE when src >= src_end +.endm + +// If we don't branch, dst < dst_end after this +.macro check_dst_end + cmp dst,dst_end + bhs L_done // extremely unlikely, DONE when dst >= dst_end +.endm + +.text +.syntax unified +.thumb +.thumb_func _lz4_decode_asm +.p2align 1 +_lz4_decode_asm: + establish_frame + ldr src,[src] // src = *src_ptr + ldr dst,[dst] // dst = *dst_ptr + + adr match_permtable,L_match_permtable + adr match_disttable,L_match_disttable + +L_decode_command: + // Keep last known good positions in both streams + str dst, dst_good + str src, src_good + + // Check limits + check_src_end + check_dst_end + + // Decode 1-byte command + ldrb aux1,[src],#1 // read command byte LLLLMMMM + lsr n_literals,aux1,#4 // 0000LLLL. n_literals is now 0..15 + and n_matches,aux1,#0xf // 0000MMMM. n_matches is now 0..15 + add n_matches,n_matches,#4 // n_matches is now 4..19 + + // Test number of literals (do not test if n_literals==0, because branch prediction fails on it) + cmp n_literals,#14 + bls L_copy_short_literal // 96% likely: n_literals in 0..14 + // continue to decode_long_literal + + // the number of literals is encoded on more bytes, we need to decode them +L_decode_long_literal: + check_src_end // required here, since we may loop an arbitrarily high number of times + ldrb aux1,[src],#1 + add n_literals,n_literals,aux1 + cmp aux1,#255 + beq L_decode_long_literal // extremely unlikely + // continue to copy_long_literal + + // Copy literals, n_literals >= 15 +L_copy_long_literal: + mov copy_src,src // literal copy origin + mov copy_dst,dst // literal copy destination + add src,src,n_literals + add dst,dst,n_literals + check_src_end // required here, since n_literals can be arbitrarily high + check_dst_end + + // fixed + loop + copy_1x32_and_increment copy_src,copy_dst +L_copy_long_literal_loop: + copy_1x32_and_increment copy_src,copy_dst + cmp dst,copy_dst + bhi L_copy_long_literal_loop // first test occurs after 64 bytes have been copied, and is unlikely to loop back + b L_expand_match + + // Copy literals, n_literals <= 14: copy 16 bytes +L_copy_short_literal: + copy_1x16 src,dst + add src,src,n_literals + add dst,dst,n_literals + // continue to expand match + +L_expand_match: + + // Decode match distance + ldrh match_distance,[src],#2 // 16-bit distance + cbz match_distance,L_fail // distance == 0 is invalid + sub copy_src,dst,match_distance // copy_src is the match copy source + cmp copy_src,dst_begin + blo L_fail // copy_src < dst_begin: FAIL + mov copy_dst,dst // copy_dst is the match copy destination + add dst,dst,n_matches // dst is updated to be the byte after the match; n_matches <= 19 here + + // Do we need to decode a long match? + cmp n_matches,#19 + beq L_decode_long_match // unlikely, n_matches >= 19 encoded on more bytes + cmp n_matches,#16 + bhi L_long_match // unlikely, n_matches == 17 or 18 + // continue to short match (most probable case) + + // Copy match, n_matches <= 16 +L_short_match: + cmp match_distance,#15 + bls L_copy_short_match_small_distance + + // Copy match, n_matches <= 16, match_distance >= 16: copy 16 bytes + copy_1x16 copy_src,copy_dst + b L_decode_command + +L_fail: +mov aux1,#-1 // FAIL +b L_exit + +L_done: +mov aux1,#0 // OK +// continue to L_exit + +L_exit: + +ldr dst,[sp, #20] // get back src_ptr,dst_ptr from stack +ldr src,[sp, #24] // get back src_ptr,dst_ptr from stack +ldr ip, src_good +ldr lr, dst_good +str ip,[src] // *src_ptr = src_good +str lr,[dst] // *dst_ptr = dst_good + +mov r0,aux1 // x0 = return value +clear_frame_and_return + + // Copy match, n_matches <= 16, match_distance < 16: + // load shuffle table, and permute to replicate the pattern on 16 bytes +L_copy_short_match_small_distance: + vld1.8 {q0},[copy_src] + add aux1,match_permtable,match_distance,lsl #5 // index in table + vld1.8 {q1},[aux1] // load only permutation for the low 16 bytes + vtbl.8 d4, {q0}, d2 // low 16 bytes of pattern + vtbl.8 d5, {q0}, d3 // low 16 bytes of pattern + vst1.8 {q2},[copy_dst] + b L_decode_command + + // n_matches == 19: the number of matches in encoded on more bytes, we need to decode them +L_decode_long_match: + check_src_end // required here, since we may loop an arbitrarily high number of times + ldrb aux1,[src],#1 + add dst,dst,aux1 + cmp aux1,#255 + beq L_decode_long_match // very unlikely + check_dst_end // required here, since dst was incremented by a arbitrarily high value + // continue to long_match + + // n_matches > 16 +L_long_match: + cmp match_distance,#31 + bhi L_copy_long_match_32 + cmp match_distance,#15 + bhi L_copy_long_match_16 + + // Copy match, n_matches >= 16, match_distance < 16: + // load shuffle table, and permute to replicate the pattern on 32 bytes +L_copy_long_match_small_distance: + vld1.8 {q1}, [copy_src] // 16 pattern bytes + add aux1,match_permtable,match_distance,lsl #5 // index in table + vld1.8 {q2-q3}, [aux1] // load 32-byte permutation + + vtbl.8 d4, {q1}, d4 // low 16 bytes of pattern + vtbl.8 d5, {q1}, d5 // low 16 bytes of pattern + vtbl.8 d6, {q1}, d6 // low 16 bytes of pattern + vtbl.8 d7, {q1}, d7 // low 16 bytes of pattern + + ldrb aux1,[match_disttable,match_distance] // valid pattern length in aux1 + // fixed + vst1.8 {q2-q3},[copy_dst] + add copy_dst,copy_dst,aux1 +L_copy_long_match_small_distance_loop: + // loop + vst1.8 {q2-q3},[copy_dst] + add copy_dst,copy_dst,aux1 + vst1.8 {q2-q3},[copy_dst] + add copy_dst,copy_dst,aux1 + cmp dst,copy_dst + bhi L_copy_long_match_small_distance_loop + b L_decode_command + + // Copy match, n_matches >= 16, match_distance >= 32 +L_copy_long_match_32: + // fixed + loop + copy_1x16_and_increment copy_src,copy_dst +L_copy_long_match_32_loop: + copy_1x32_and_increment copy_src,copy_dst + cmp dst,copy_dst + bhi L_copy_long_match_32_loop + b L_decode_command + + // Copy match, n_matches >= 16, match_distance >= 16 +L_copy_long_match_16: + // fixed + loop + copy_1x16_and_increment copy_src,copy_dst +L_copy_long_match_16_loop: + copy_2x16_and_increment copy_src,copy_dst + cmp dst,copy_dst + bhi L_copy_long_match_16_loop + b L_decode_command + + +// permutation tables for short distance matches, 32 byte result, for match_distance = 0 to 15 +// value(d)[i] = i%d for i = 0..31 +.p2align 6 +L_match_permtable: +.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 // 0 +.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 // 1 +.byte 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 // 2 +.byte 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1 // 3 +.byte 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 // 4 +.byte 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1 // 5 +.byte 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1 // 6 +.byte 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3 // 7 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 // 8 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4 // 9 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1 // 10 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 // 11 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11, 0, 1, 2, 3, 4, 5, 6, 7 // 12 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12, 0, 1, 2, 3, 4, 5 // 13 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13, 0, 1, 2, 3 // 14 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, 0, 1 // 15 + +// valid repeating pattern size, for each match_distance = 0 to 15 +// value(d) = 32 - (32%d), is the largest a multiple of d <= 32 +.p2align 6 +L_match_disttable: +.byte 32,32,32,30 // 0 .. 3 +.byte 16,30,30,28 // 4 .. 7 +.byte 16,27,30,22 // 8 .. 11 +.byte 24,26,28,30 // 12 .. 15 + +#endif // LZ4_ENABLE_ASSEMBLY_DECODE_ARMV7 diff --git a/osfmk/arm/lz4_encode_armv7.s b/osfmk/arm/lz4_encode_armv7.s new file mode 100644 index 000000000..370386791 --- /dev/null +++ b/osfmk/arm/lz4_encode_armv7.s @@ -0,0 +1,429 @@ +/* + * Copyright (c) 2016-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#if LZ4_ENABLE_ASSEMBLY_ENCODE_ARMV7 + +/* void lz4_encode_2gb(uint8_t ** dst_ptr, + size_t dst_size, + const uint8_t ** src_ptr, + const uint8_t * src_begin, + size_t src_size, + lz4_hash_entry_t hash_table[LZ4_COMPRESS_HASH_ENTRIES], + int skip_final_literals) */ + +.globl _lz4_encode_2gb +.syntax unified + +#define dst_ptr r0 +#define dst_end r1 +#define src_ptr r2 +#define src_beg r3 +#define src_end r4 +#define table r5 +#define mch_ptr r6 +#define mch_dis r8 +#define mch_len r9 +#define mch_ref r10 + +#define margin 128 + +.macro establish_frame + push {r4-r7, lr} + add r7, sp, #12 + push {r8-r11} + ldrd r4, r5, [sp, #36] + push {r0, r2} + ldr dst_ptr, [r0] + ldr src_ptr, [r2] + subs r1, r1, margin // subtract safety margin from dst_size + bls L_done + add dst_end, dst_ptr, r1 // dst end - margin + sub r4, r4, margin // subtract safety margin from src_size (src_size < margin is detected by check on mch_ptr in match_candidate_loop). + add src_end, src_ptr, r4 // src end - margin. + vmov.i8 q1, #255 // vector of all 1s used to emit +.endm + +.macro clear_frame_and_return + pop {r1, r3} + str dst_ptr, [r1] + str src_ptr, [r3] + pop {r8-r11} + pop {r4-r7, pc} +.endm + +.p2align 4 +_lz4_encode_2gb: + establish_frame +L_next_match: + // Start searching for the next match, starting at the end of the last one. + // [We begin with mch_ptr = src_ptr - 1 because we pre-increment mch_ptr + // within the search loop itself]. Load the hash magic number in lr, and + // zero out mch_len (when we find a match, its length will initially be + // four, but we actually work with the match length minus four at all times). + ldr lr, L_hash + sub mch_ptr, src_ptr, #1 + +L_match_candidate_loop: + // If mch_ptr >= src_end, we are near the end of the source buffer (remember + // that we subtracted margin from src_end, so we are *not* actually past the + // end just yet). + cmp mch_ptr, src_end + bhs L_trailing_literal + + // Load the four-byte word starting at mch_ptr, and get the address of the + // corresponding row of the hash table. + ldr r9, [mch_ptr, #1]! + sub r8, mch_ptr, src_beg + mul r12, r9, lr + mvn r10, #0x7 + and r12, r10, r12, lsr #17 // byte offset of table entry. + + // Load offset and word from hash table row, then update with the new offset + // and word that we just computed. + ldrd r10,r11, [table, r12] + strd r8, r9, [table, r12] + + // At this point, we only know that the hashes of the words match; check to + // see if the words themselves match. If not, move on to the next candidate. + cmp r9, r11 + bne L_match_candidate_loop + + // It's not enough for the words to match; the match distance must also be + // in the representable range (i.e. less than 0x10000). + sub mch_dis, r8, r10 + add mch_ref, src_beg, r10 + cmp mch_dis, #0x10000 + bhs L_match_candidate_loop + + // We have found a match; registers at this point are as follows: + // + // register symbolic name meaning + // r0 dst_ptr pointer into destination buffer where the + // match information will be stored. + // r1 dst_end pointer to the end of the destination buffer, + // less margin. + // r2 src_ptr pointer to the byte after the last match that + // we found, or to the point from which we + // started searching if this is the first match. + // r3 src_beg pointer to the actual start of the buffer. + // r4 src_end pointer to the end of the source buffer, less + // margin. + // r5 table address of hash table. + // r6 mch_ptr pointer to match. + // r8 mch_dis match distance ("D") + // r9 mch_len length of match less four ("M") + // r10 mch_ref pointer to match reference. + // r11 - + // r12 - + // lr - + // + // Attempt to grow the match backwards (typically we only grow backwards by + // a byte or two, if at all, so we use a byte-by-byte scan). + eor mch_len, mch_len +0:cmp mch_ref, src_beg + cmpne mch_ptr, src_ptr + beq 1f + ldrb r11, [mch_ref, #-1] + ldrb r12, [mch_ptr, #-1] + cmp r11, r12 + bne 1f + sub mch_ref, #1 + sub mch_ptr, #1 + add mch_len, #1 + b 0b + + // Now that we have the start of the match, we can compute the literal + // length. Then advance the mch and ref pointers to the end of the match + // and its reference. Because mch_len is the real match length minus four, + // we actually advance to four before the end of the match, but our loop + // to grow the matches uses pre-incremented loads with writeback, so this + // works out correctly. +#define lit_len lr +1:sub lit_len, mch_ptr, src_ptr + add mch_ptr, mch_len + add mch_ref, mch_len + + // Now attempt to grow the match forwards. This is much more common, and + // there is a safety margin at the end of the buffer, so we grow forwards + // in four-byte chunks. +0:ldr r11, [mch_ptr, #4]! + ldr r12, [mch_ref, #4]! + eors r11, r12 + bne 1f + add mch_len, #4 + cmp mch_ptr, src_end + blo 0b + b L_emit_match + // At least one of the bytes in the last comparison did not match. Identify + // which byte had the mismatch and compute the final length (less four). +1:rev r11, r11 + clz r11, r11 + add mch_len, r11, lsr #3 + +L_emit_match: + // Time to emit what we've found! + // + // register symbolic name meaning + // r0 dst_ptr pointer into destination buffer where the + // match information will be stored. + // r1 dst_end pointer to the end of the destination buffer, + // less margin. + // r2 src_ptr pointer to the byte after the last match that + // we found, or to the point from which we + // started searching if this is the first match. + // r3 src_beg pointer to the actual start of the buffer. + // r4 src_end pointer to the end of the source buffer, less + // margin. + // r5 table address of hash table. + // r6 - + // r8 mch_dis match distance ("D") + // r9 mch_len length of match ("M") + // r10 - + // r11 - + // r12 - + // lr lit_len literal length ("L") + // q1 vector of all ones + // + // Synthesize control byte under the assumption that L and M are both less + // than 15, jumping out of the fast path if one of them is not. + cmp lit_len, #15 + orr r10, mch_len, lit_len, lsl #4 + cmplo mch_len, #15 + bhs L_emit_careful + // L and M are both less than 15, which means (a) we use the most compact + // encoding for the match and (b) we do not need to do a bounds check on + // the destination buffer before writing, only before continuing our search. + // Store the command byte. + strb r10, [dst_ptr], #1 + // Copy literal. + vld1.8 q0, [src_ptr] + add src_ptr, lit_len + vst1.8 q0, [dst_ptr] + add dst_ptr, lit_len + // Restore "true" match length before updating src_ptr. + add mch_len, #4 + // Store match distance (D) and update the source pointer. + strh r8, [dst_ptr], #2 + add src_ptr, mch_len + // If we're not into the safety margin of the destination buffer, look for + // another match. + cmp dst_ptr, dst_end + blo L_next_match + // If we *are* into the safety margin of the destination buffer, we're done + // encoding this block; update the source and destination pointers and + // return. +L_done: + clear_frame_and_return + +// Constant island +L_hash: .long 2654435761 +L_magic: .long 0x80808081 + +L_emit_careful: + // Either L or M is >= 15, which means that we don't get to use the compact + // encoding, and that we need to do extra bounds checks while writing. + // + // register symbolic name meaning + // r0 dst_ptr pointer into destination buffer where the + // match information will be stored. + // r1 dst_end pointer to the end of the destination buffer, + // less margin. + // r2 src_ptr pointer to the byte after the last match that + // we found, or to the point from which we + // started searching if this is the first match. + // r3 src_beg pointer to the actual start of the buffer. + // r4 src_end pointer to the end of the source buffer, less + // margin. + // r5 table address of hash table. + // r6 - + // r8 mch_dis match distance ("D") + // r9 mch_len length of match ("M") less four + // r10 - + // r11 - + // r12 - + // lr lit_len literal length ("L") + // q1 vector of all ones + // + // Start by creating the low 4 bits of the control word; M if M < 15, 0xf + // otherwise. We also load 0x80808081, which is the magic number for + // division by 255; this will be required later on. + ldr r12, L_magic + cmp mch_len, #15 + mov r10, mch_len + movhs r10, #0x0f + subs r6, lit_len, #15 + bhs L_large_L + // M is large, but L is < 15. This means we can use the simple approach + // for copying the literal with no bounds checks. + orr r10, lit_len, lsl #4 + strb r10, [dst_ptr], #1 + // Copy literal. + vld1.8 q0, [src_ptr] + add src_ptr, lit_len + vst1.8 q0, [dst_ptr] + add dst_ptr, lit_len + // Store match distance (D). + strh r8, [dst_ptr], #2 + sub r6, mch_len, #15 + b L_large_M + +L_large_L: + // L is large, M may or may not be. We need to encode extra literal length + // bytes and we need to do bounds checks while store both those byte and the + // literal itself. + orr r10, #0xf0 + strb r10, [dst_ptr], #1 + // How many extra literal bytes do we need to store? We need to store + // (L - 15)/255 extra literal bytes of 0xff, plus one more byte that is + // (L - 15)%255. Get these quantities via magic number multiplication: + // (L - 15)*0x80808081 >> (32 + 7) + umull r10, r11, r6, r12 + mov r12, #255 + lsr r10, r11, #7 // (L - 15) / 255 + mls r11, r10, r12, r6 // (L - 15) % 255 + ldr r12, L_magic // may need magic number again for M. + // Compute address dst_ptr will have after storing all 0xff bytes, and + // check that we won't exceed dst_end in doing so. + add r10, dst_ptr, r10 + cmp r10, dst_end + bhs L_done + // There's enough space for all the 0xff bytes, so go ahead and store them. +0:vst1.8 q1, [dst_ptr]! + cmp dst_ptr, r10 + blo 0b + // Store the (L - 15) % 255 byte. + strb r11, [r10], #1 + // Compute the address we'll have reached after storing all literal bytes. + // If that passes dst_end, we're done. + add dst_ptr, r10, lit_len + cmp dst_ptr, dst_end + bhs L_done + // Copy the literal. +0:vld1.8 q0, [src_ptr]! + vst1.8 q0, [r10]! + subs r6, r10, dst_ptr + blo 0b + // Fixup src_ptr, store match distance (D), and check whether or not M is + // bigger than 14. If not, go find the next match. + strh r8, [dst_ptr], #2 + sub src_ptr, r6 + subs r6, mch_len, #15 + bhs L_large_M + // M is small, so we're all done; we just need to update the source pointer + // and we can go look for the next match. + add mch_len, #4 + add src_ptr, mch_len + b L_next_match + +L_large_M: + // Just like with large L, we split (M - 15) into (M - 15) / 255 and + // (M - 15) % 255 via magic number multiply. + umull r10, r11, r6, r12 + mov r12, #255 + lsr r10, r11, #7 // (M - 15) / 255 + mls r11, r10, r12, r6 // (M - 15) % 255 + // Compute address dst_ptr will have after storing all 0xff bytes, and + // check that we won't exceed dst_end in doing so. + add r10, dst_ptr, r10 + cmp r10, dst_end + bhs L_done + // There's enough space for all the 0xff bytes, so go ahead and store them. +0:vst1.8 q1, [dst_ptr]! + cmp dst_ptr, r10 + blo 0b + // Store final M % 255 byte, update dst_ptr and src_ptr, and look for next + // match. + strb r11, [r10] + add mch_len, #4 + add dst_ptr, r10, #1 + add src_ptr, mch_len + b L_next_match + +L_trailing_literal: + // Check if skip_final_literals is set. + ldr r5, [sp, #52] + tst r5, r5 + bne L_done + // Emit a trailing literal that covers the remainder of the source buffer, + // if we can do so without exceeding the bounds of the destination buffer. + add src_end, margin + sub lit_len, src_end, src_ptr + subs r6, lit_len, #15 + bhs L_trailing_literal_long + lsl r10, lit_len, #4 + strb r10, [dst_ptr], #1 + vld1.8 q0, [src_ptr] + mov src_ptr, src_end + vst1.8 q0, [dst_ptr] + add dst_ptr, lit_len + b L_done + +L_trailing_literal_long: + ldr r12, L_magic + mov r10, #0xf0 + add dst_end, margin + strb r10, [dst_ptr], #1 + umull r10, r11, r6, r12 + mov r12, #255 + lsr r10, r11, #7 // (L - 15) / 255 + mls r11, r10, r12, r6 // (L - 15) % 255 + // We want to write out lit_len + (L - 15)/255 + 1 bytes. Check if we have + // space for all of them. + add r10, dst_ptr + add r12, r10, lit_len + cmp r12, dst_end + bhs L_done + // We have enough space, so go ahead and write them all out. Because we + // know that we have enough space, and that the literal is at least 15 bytes, + // we can write the block of 0xffs using vector stores, even without a + // safety margin. +0:vst1.8 q1, [dst_ptr]! + cmp dst_ptr, r10 + blo 0b + // Store the (L - 15) % 255 byte. + strb r11, [r10], #1 + mov dst_ptr, r10 + // Now store the literal itself; here we need to actually be somewhat + // careful to ensure that we don't write past the end of the destination + // buffer or read past the end of the source buffer. + subs lit_len, #16 + blo 1f +0:vld1.8 q0, [src_ptr]! + subs lit_len, #16 + vst1.8 q0, [dst_ptr]! + bhs 0b +1:adds lit_len, #16 + beq L_done +2:ldrb r6, [src_ptr], #1 + subs lit_len, #1 + strb r6, [dst_ptr], #1 + bne 2b + b L_done + +#endif diff --git a/osfmk/arm/machdep_call.c b/osfmk/arm/machdep_call.c new file mode 100644 index 000000000..b77c3c7ce --- /dev/null +++ b/osfmk/arm/machdep_call.c @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + * Machine dependent kernel calls. + * + * HISTORY + * + * 17 June 1992 ? at NeXT + * Created. + */ + +#include +#include +#include +#if __arm64__ +#include +#endif + +extern kern_return_t kern_invalid(void); + +uintptr_t +get_tpidrro(void) +{ + uintptr_t uthread; +#if __arm__ + uthread = __builtin_arm_mrc(15, 0, 13, 0, 3); // TPIDRURO +#else + __asm__ volatile("mrs %0, TPIDRRO_EL0" : "=r" (uthread)); +#endif + return uthread; +} + +void +set_tpidrro(uintptr_t uthread) +{ +#if __arm__ + __builtin_arm_mcr(15, 0, uthread, 13, 0, 3); // TPIDRURO +#else + __asm__ volatile("msr TPIDRRO_EL0, %0" : : "r" (uthread)); +#endif +} + +kern_return_t +thread_set_cthread_self(vm_address_t self) +{ + return machine_thread_set_tsd_base(current_thread(), self); +} + +vm_address_t +thread_get_cthread_self(void) +{ + uintptr_t self; + + self = get_tpidrro(); +#if __arm__ + self &= ~3; + assert( self == current_thread()->machine.cthread_self); + return ((kern_return_t) current_thread()->machine.cthread_self); +#else + self &= MACHDEP_CTHREAD_MASK; + assert( self == current_thread()->machine.cthread_self); + return self; +#endif +} + diff --git a/osfmk/arm/machdep_call.h b/osfmk/arm/machdep_call.h new file mode 100644 index 000000000..1f7f1606f --- /dev/null +++ b/osfmk/arm/machdep_call.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + * Machine dependent kernel call table defines. + * + * HISTORY + * + * 17 June 1992 ? at NeXT + * Created. + */ + +typedef union { + kern_return_t (*args_0)(void); + kern_return_t (*args_1)(vm_address_t); + kern_return_t (*args_2)(vm_address_t,vm_address_t); + kern_return_t (*args_3)(vm_address_t,vm_address_t,vm_address_t); + kern_return_t (*args_4)(vm_address_t, vm_address_t,vm_address_t,vm_address_t); + kern_return_t (*args_var)(vm_address_t,...); +} machdep_call_routine_t; + +#define MACHDEP_CALL_ROUTINE(func,args) \ + { { .args_ ## args = func }, args } + +typedef struct { + machdep_call_routine_t routine; + int nargs; +} machdep_call_t; + +extern const machdep_call_t machdep_call_table[]; +extern int machdep_call_count; + +extern vm_address_t thread_get_cthread_self(void); +extern kern_return_t thread_set_cthread_self(vm_address_t); + +// Read and write raw TPIDRURO / TPIDRRO_EL0 +uintptr_t get_tpidrro(void); +void set_tpidrro(uintptr_t); + diff --git a/osfmk/arm/machine_cpu.h b/osfmk/arm/machine_cpu.h new file mode 100644 index 000000000..64d795e70 --- /dev/null +++ b/osfmk/arm/machine_cpu.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_MACHINE_CPU_H_ +#define _ARM_MACHINE_CPU_H_ + +#include +#include +#include +#include +#include + +extern void cpu_machine_init(void); + +extern kern_return_t cpu_register(int *slot_nump); + +extern void cpu_signal_handler(void); +extern void cpu_signal_handler_internal(boolean_t disable_signal); + +extern void cpu_doshutdown(void (*doshutdown)(processor_t), processor_t processor); + +extern void cpu_idle(void); +extern void cpu_idle_exit(void) __attribute__((noreturn)); +extern void cpu_idle_tickle(void); + +extern void cpu_machine_idle_init(boolean_t from_boot); + +extern void arm_init_cpu(cpu_data_t *args); + +extern void arm_init_idle_cpu(cpu_data_t *args); + +extern void init_cpu_timebase(boolean_t enable_fiq); + +#define cpu_pause() do {} while (0) /* Not for this architecture */ + +#endif /* _ARM_MACHINE_CPU_H_ */ diff --git a/osfmk/arm/machine_cpuid.c b/osfmk/arm/machine_cpuid.c new file mode 100644 index 000000000..232c6e64a --- /dev/null +++ b/osfmk/arm/machine_cpuid.c @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include +#include +#include + +static arm_mvfp_info_t cpuid_mvfp_info; +static arm_debug_info_t cpuid_debug_info; + +uint32_t +machine_read_midr(void) +{ +#if __arm__ + uint32_t midr = __builtin_arm_mrc(15,0,0,0,0); +#else + uint64_t midr; + __asm__ volatile("mrs %0, MIDR_EL1" : "=r" (midr)); +#endif + return (uint32_t)midr; +} + +uint32_t +machine_read_clidr(void) +{ +#if __arm__ + uint32_t clidr = __builtin_arm_mrc(15,1,0,0,1); +#else + uint64_t clidr; + __asm__ volatile("mrs %0, CLIDR_EL1" : "=r" (clidr)); +#endif + return (uint32_t)clidr; +} + +uint32_t +machine_read_ccsidr(void) +{ +#if __arm__ + uint32_t ccsidr = __builtin_arm_mrc(15,1,0,0,0); +#else + uint64_t ccsidr; + __asm__ volatile("mrs %0, CCSIDR_EL1" : "=r" (ccsidr)); +#endif + return (uint32_t)ccsidr; +} + +#if __arm__ +arm_isa_feat1_reg +machine_read_isa_feat1(void) +{ + arm_isa_feat1_reg isa; + isa.value = __builtin_arm_mrc(15,0,0,2,1); + return isa; +} +#endif // __arm__ + +void +machine_write_csselr(csselr_cache_level level, csselr_cache_type type) +{ +#if __arm__ + uint32_t csselr = (level | type); + __builtin_arm_mcr(15,2,csselr,0,0,0); +#else + uint64_t csselr = (level | type); + __asm__ volatile("msr CSSELR_EL1, %0" : : "r" (csselr)); +#endif + __builtin_arm_isb(ISB_SY); +} + +void +machine_do_debugid(void) +{ +#if __arm__ + arm_cpuid_id_dfr0 id_dfr0; + arm_debug_dbgdidr dbgdidr; + + /* read CPUID ID_DFR0 */ + id_dfr0.value = __builtin_arm_mrc(15,0,0,1,2); + /* read DBGDIDR */ + dbgdidr.value = __builtin_arm_mrc(14,0,0,0,0); + + cpuid_debug_info.coprocessor_core_debug = id_dfr0.debug_feature.coprocessor_core_debug != 0; + cpuid_debug_info.memory_mapped_core_debug = (id_dfr0.debug_feature.memory_mapped_core_debug != 0) + && (getCpuDatap()->cpu_debug_interface_map != 0); + + if (cpuid_debug_info.coprocessor_core_debug || cpuid_debug_info.memory_mapped_core_debug) { + cpuid_debug_info.num_watchpoint_pairs = dbgdidr.debug_id.wrps + 1; + cpuid_debug_info.num_breakpoint_pairs = dbgdidr.debug_id.brps + 1; + } +#else + arm_cpuid_id_aa64dfr0_el1 id_dfr0; + + /* read ID_AA64DFR0_EL1 */ + __asm__ volatile("mrs %0, ID_AA64DFR0_EL1" : "=r"(id_dfr0.value)); + + if (id_dfr0.debug_feature.debug_arch_version) { + cpuid_debug_info.num_watchpoint_pairs = id_dfr0.debug_feature.wrps + 1; + cpuid_debug_info.num_breakpoint_pairs = id_dfr0.debug_feature.brps + 1; + } +#endif +} + +arm_debug_info_t * +machine_arm_debug_info(void) +{ + return &cpuid_debug_info; +} + +void +machine_do_mvfpid() +{ + arm_mvfr0_info_t arm_mvfr0_info; + arm_mvfr1_info_t arm_mvfr1_info; + uint64_t tmp; + +#if __arm__ + (void)tmp; + __asm__ volatile("vmrs %0, mvfr0":"=r"(arm_mvfr0_info.value)); + __asm__ volatile("vmrs %0, mvfr1":"=r"(arm_mvfr1_info.value)); +#else + __asm__ volatile("mrs %0, MVFR0_EL1":"=r"(tmp)); + arm_mvfr0_info.value = (uint32_t)tmp; + + __asm__ volatile("mrs %0, MVFR1_EL1":"=r"(tmp)); + arm_mvfr1_info.value = (uint32_t)tmp; +#endif + + cpuid_mvfp_info.neon = arm_mvfr1_info.bits.SP; + cpuid_mvfp_info.neon_hpfp = arm_mvfr1_info.bits.HPFP; +} + +arm_mvfp_info_t * +machine_arm_mvfp_info(void) +{ + return &cpuid_mvfp_info; +} + diff --git a/osfmk/arm/machine_cpuid.h b/osfmk/arm/machine_cpuid.h new file mode 100644 index 000000000..e50ac9302 --- /dev/null +++ b/osfmk/arm/machine_cpuid.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_MACHINE_CPUID_H_ +#define _ARM_MACHINE_CPUID_H_ + +/* CPU feature identification */ + +typedef struct { +uint32_t arm_32bit_isa : 4, + arm_thumb_ver : 4, + arm_jazelle : 4, + arm_thumb2 : 4, + reserved : 16; +} arm_feature_bits_t; + +typedef union { + arm_feature_bits_t field; + uint32_t value; +} arm_feature0_reg_t; + +// Register 0, subtype 21: Instruction Set Features #1 +typedef struct +{ + uint32_t endianness_support : 4; + uint32_t exception_1_support : 4; + uint32_t exception_2_support : 4; + uint32_t sign_zero_ext_support : 4; + uint32_t if_then_support : 4; + uint32_t immediate_support : 4; + uint32_t interworking_support : 4; + uint32_t jazelle_support : 4; +} +syscp_ID_instructions_feat_1_reg; + +typedef union { + uint32_t value; + syscp_ID_instructions_feat_1_reg field; +} arm_isa_feat1_reg; + +arm_isa_feat1_reg machine_read_isa_feat1(void); + +/* Debug identification */ + +/* ID_DFR0 */ +typedef union { + struct { + uint32_t coprocessor_core_debug : 4, + coprocessor_secure_debug : 4, + memory_mapped_core_debug : 4, + coprocessor_trace_debug : 4, + memory_mapped_trace_debug : 4, + microcontroller_debug : 4; + } debug_feature; + uint32_t value; +} arm_cpuid_id_dfr0; + +/* DBGDIDR */ +typedef union { + struct { + uint32_t revision : 4, + variant : 4, + : 4, + se_imp : 1, + pcsr_imp : 1, + nsuhd_imp : 1, + : 1, + version : 4, + ctx_cmps : 4, + brps : 4, + wrps : 4; + } debug_id; + uint32_t value; +} arm_debug_dbgdidr; + +typedef struct { + boolean_t memory_mapped_core_debug; + boolean_t coprocessor_core_debug; + uint32_t num_watchpoint_pairs; + uint32_t num_breakpoint_pairs; +} arm_debug_info_t; + +#endif /* _ARM_MACHINE_CPUID_H_ */ diff --git a/osfmk/arm/machine_kpc.h b/osfmk/arm/machine_kpc.h new file mode 100644 index 000000000..ec7aed315 --- /dev/null +++ b/osfmk/arm/machine_kpc.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _MACHINE_ARM_KPC_H +#define _MACHINE_ARM_KPC_H + +#ifdef ARMA7 + +#define KPC_ARM_FIXED_COUNT 1 +#define KPC_ARM_CONFIGURABLE_COUNT 4 + +#define KPC_ARM_TOTAL_COUNT (KPC_ARM_FIXED_COUNT + KPC_ARM_CONFIGURABLE_COUNT) + +#define KPC_ARM_COUNTER_WIDTH 32 + +#else + +#define KPC_ARM_FIXED_COUNT 2 +#define KPC_ARM_CONFIGURABLE_COUNT 6 + +#define KPC_ARM_COUNTER_WIDTH 39 +#define KPC_ARM_COUNTER_MASK ((1ull << KPC_ARM_COUNTER_WIDTH) - 1) +#define KPC_ARM_COUNTER_OVF_BIT (39) +#define KPC_ARM_COUNTER_OVF_MASK (1ull << KPC_ARM_COUNTER_OVF_BIT) + +#endif + +typedef uint64_t kpc_config_t; + +/* Size to the maximum number of counters we could read from every class in one go */ +#define KPC_MAX_COUNTERS (KPC_ARM_FIXED_COUNT + KPC_ARM_CONFIGURABLE_COUNT + 1) + +/* arm32 uses fixed counter shadows */ +#define FIXED_COUNTER_SHADOW (1) + +#endif /* _MACHINE_ARM_KPC_H */ diff --git a/osfmk/arm/machine_routines.c b/osfmk/arm/machine_routines.c new file mode 100644 index 000000000..b86bd643b --- /dev/null +++ b/osfmk/arm/machine_routines.c @@ -0,0 +1,1176 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#if KPC +#include +#endif + +static int max_cpus_initialized = 0; +#define MAX_CPUS_SET 0x1 +#define MAX_CPUS_WAIT 0x2 + +static unsigned int avail_cpus = 0; + +uint32_t LockTimeOut; +uint32_t LockTimeOutUsec; +uint64_t MutexSpin; +boolean_t is_clock_configured = FALSE; + +extern int mach_assert; +extern volatile uint32_t debug_enabled; +SECURITY_READ_ONLY_LATE(unsigned int) debug_boot_arg; + +void machine_conf(void); + +void +machine_startup(__unused boot_args * args) +{ + int boot_arg; + +#if MACH_KDP + if (PE_parse_boot_argn("debug", &debug_boot_arg, sizeof (debug_boot_arg)) && + debug_enabled) { +#if DEVELOPMENT || DEBUG + if (debug_boot_arg & DB_HALT) + halt_in_debugger = 1; +#endif + if (debug_boot_arg & DB_NMI) + panicDebugging = TRUE; + } else { + debug_boot_arg = 0; + } +#endif + + PE_parse_boot_argn("assert", &mach_assert, sizeof (mach_assert)); + + if (PE_parse_boot_argn("preempt", &boot_arg, sizeof (boot_arg))) { + default_preemption_rate = boot_arg; + } + if (PE_parse_boot_argn("bg_preempt", &boot_arg, sizeof (boot_arg))) { + default_bg_preemption_rate = boot_arg; + } + + machine_conf(); + + /* + * Kick off the kernel bootstrap. + */ + kernel_bootstrap(); + /* NOTREACHED */ +} + +char * +machine_boot_info( + __unused char *buf, + __unused vm_size_t size) +{ + return (PE_boot_args()); +} + +void +machine_conf(void) +{ + machine_info.memory_size = mem_size; +} + +void +machine_init(void) +{ + debug_log_init(); + clock_config(); + is_clock_configured = TRUE; + if (debug_enabled) + pmap_map_globals(); +} + +void +slave_machine_init(__unused void *param) +{ + cpu_machine_init(); /* Initialize the processor */ + clock_init(); /* Init the clock */ +} + +/* + * Routine: machine_processor_shutdown + * Function: + */ +thread_t +machine_processor_shutdown( + __unused thread_t thread, + void (*doshutdown) (processor_t), + processor_t processor) +{ + return (Shutdown_context(doshutdown, processor)); +} + +/* + * Routine: ml_init_max_cpus + * Function: + */ +void +ml_init_max_cpus(unsigned int max_cpus) +{ + boolean_t current_state; + + current_state = ml_set_interrupts_enabled(FALSE); + if (max_cpus_initialized != MAX_CPUS_SET) { + machine_info.max_cpus = max_cpus; + machine_info.physical_cpu_max = max_cpus; + machine_info.logical_cpu_max = max_cpus; + if (max_cpus_initialized == MAX_CPUS_WAIT) + thread_wakeup((event_t) & max_cpus_initialized); + max_cpus_initialized = MAX_CPUS_SET; + } + (void) ml_set_interrupts_enabled(current_state); +} + +/* + * Routine: ml_get_max_cpus + * Function: + */ +unsigned int +ml_get_max_cpus(void) +{ + boolean_t current_state; + + current_state = ml_set_interrupts_enabled(FALSE); + if (max_cpus_initialized != MAX_CPUS_SET) { + max_cpus_initialized = MAX_CPUS_WAIT; + assert_wait((event_t) & max_cpus_initialized, THREAD_UNINT); + (void) thread_block(THREAD_CONTINUE_NULL); + } + (void) ml_set_interrupts_enabled(current_state); + return (machine_info.max_cpus); +} + +/* + * Routine: ml_init_lock_timeout + * Function: + */ +void +ml_init_lock_timeout(void) +{ + uint64_t abstime; + uint64_t mtxspin; + uint64_t default_timeout_ns = NSEC_PER_SEC>>2; + uint32_t slto; + + if (PE_parse_boot_argn("slto_us", &slto, sizeof (slto))) + default_timeout_ns = slto * NSEC_PER_USEC; + + nanoseconds_to_absolutetime(default_timeout_ns, &abstime); + LockTimeOutUsec = (uint32_t)(abstime / NSEC_PER_USEC); + LockTimeOut = (uint32_t)abstime; + + if (PE_parse_boot_argn("mtxspin", &mtxspin, sizeof (mtxspin))) { + if (mtxspin > USEC_PER_SEC>>4) + mtxspin = USEC_PER_SEC>>4; + nanoseconds_to_absolutetime(mtxspin*NSEC_PER_USEC, &abstime); + } else { + nanoseconds_to_absolutetime(10*NSEC_PER_USEC, &abstime); + } + MutexSpin = abstime; +} + +/* + * This is called from the machine-independent routine cpu_up() + * to perform machine-dependent info updates. + */ +void +ml_cpu_up(void) +{ + hw_atomic_add(&machine_info.physical_cpu, 1); + hw_atomic_add(&machine_info.logical_cpu, 1); +} + +/* + * This is called from the machine-independent routine cpu_down() + * to perform machine-dependent info updates. + */ +void +ml_cpu_down(void) +{ + cpu_data_t *cpu_data_ptr; + + hw_atomic_sub(&machine_info.physical_cpu, 1); + hw_atomic_sub(&machine_info.logical_cpu, 1); + + /* + * If we want to deal with outstanding IPIs, we need to + * do relatively early in the processor_doshutdown path, + * as we pend decrementer interrupts using the IPI + * mechanism if we cannot immediately service them (if + * IRQ is masked). Do so now. + * + * We aren't on the interrupt stack here; would it make + * more sense to disable signaling and then enable + * interrupts? It might be a bit cleaner. + */ + cpu_data_ptr = getCpuDatap(); + cpu_data_ptr->cpu_running = FALSE; + + cpu_signal_handler_internal(TRUE); +} + +/* + * Routine: ml_cpu_get_info + * Function: + */ +void +ml_cpu_get_info(ml_cpu_info_t * ml_cpu_info) +{ + cache_info_t *cpuid_cache_info; + + cpuid_cache_info = cache_info(); + ml_cpu_info->vector_unit = 0; + ml_cpu_info->cache_line_size = cpuid_cache_info->c_linesz; + ml_cpu_info->l1_icache_size = cpuid_cache_info->c_isize; + ml_cpu_info->l1_dcache_size = cpuid_cache_info->c_dsize; + +#if (__ARM_ARCH__ >= 7) + ml_cpu_info->l2_settings = 1; + ml_cpu_info->l2_cache_size = cpuid_cache_info->c_l2size; +#else + ml_cpu_info->l2_settings = 0; + ml_cpu_info->l2_cache_size = 0xFFFFFFFF; +#endif + ml_cpu_info->l3_settings = 0; + ml_cpu_info->l3_cache_size = 0xFFFFFFFF; +} + +unsigned int +ml_get_machine_mem(void) +{ + return (machine_info.memory_size); +} + +/* Return max offset */ +vm_map_offset_t +ml_get_max_offset( + boolean_t is64, + unsigned int option) +{ + unsigned int pmap_max_offset_option = 0; + + switch (option) { + case MACHINE_MAX_OFFSET_DEFAULT: + pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_DEFAULT; + break; + case MACHINE_MAX_OFFSET_MIN: + pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_MIN; + break; + case MACHINE_MAX_OFFSET_MAX: + pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_MAX; + break; + case MACHINE_MAX_OFFSET_DEVICE: + pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_DEVICE; + break; + default: + panic("ml_get_max_offset(): Illegal option 0x%x\n", option); + break; + } + return pmap_max_offset(is64, pmap_max_offset_option); +} + +boolean_t +ml_wants_panic_trap_to_debugger(void) +{ + return FALSE; +} + +void +ml_panic_trap_to_debugger(__unused const char *panic_format_str, + __unused va_list *panic_args, + __unused unsigned int reason, + __unused void *ctx, + __unused uint64_t panic_options_mask, + __unused unsigned long panic_caller) +{ + return; +} + +__attribute__((noreturn)) +void +halt_all_cpus(boolean_t reboot) +{ + if (reboot) { + printf("MACH Reboot\n"); + PEHaltRestart(kPERestartCPU); + } else { + printf("CPU halted\n"); + PEHaltRestart(kPEHaltCPU); + } + while (1); +} + +__attribute__((noreturn)) +void +halt_cpu(void) +{ + halt_all_cpus(FALSE); +} + +/* + * Routine: machine_signal_idle + * Function: + */ +void +machine_signal_idle( + processor_t processor) +{ + cpu_signal(processor_to_cpu_datap(processor), SIGPnop, (void *)NULL, (void *)NULL); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0); +} + +void +machine_signal_idle_deferred( + processor_t processor) +{ + cpu_signal_deferred(processor_to_cpu_datap(processor)); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_DEFERRED_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0); +} + +void +machine_signal_idle_cancel( + processor_t processor) +{ + cpu_signal_cancel(processor_to_cpu_datap(processor)); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_CANCEL_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0); +} + +/* + * Routine: ml_install_interrupt_handler + * Function: Initialize Interrupt Handler + */ +void +ml_install_interrupt_handler( + void *nub, + int source, + void *target, + IOInterruptHandler handler, + void *refCon) +{ + cpu_data_t *cpu_data_ptr; + boolean_t current_state; + + current_state = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + cpu_data_ptr->interrupt_nub = nub; + cpu_data_ptr->interrupt_source = source; + cpu_data_ptr->interrupt_target = target; + cpu_data_ptr->interrupt_handler = handler; + cpu_data_ptr->interrupt_refCon = refCon; + + cpu_data_ptr->interrupts_enabled = TRUE; + (void) ml_set_interrupts_enabled(current_state); + + initialize_screen(NULL, kPEAcquireScreen); +} + +/* + * Routine: ml_init_interrupt + * Function: Initialize Interrupts + */ +void +ml_init_interrupt(void) +{ +} + +/* + * Routine: ml_init_timebase + * Function: register and setup Timebase, Decremeter services + */ +void ml_init_timebase( + void *args, + tbd_ops_t tbd_funcs, + vm_offset_t int_address, + vm_offset_t int_value) +{ + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = (cpu_data_t *)args; + + if ((cpu_data_ptr == &BootCpuData) + && (rtclock_timebase_func.tbd_fiq_handler == (void *)NULL)) { + rtclock_timebase_func = *tbd_funcs; + rtclock_timebase_addr = int_address; + rtclock_timebase_val = int_value; + } +} + +void +ml_parse_cpu_topology(void) +{ + DTEntry entry, child; + OpaqueDTEntryIterator iter; + uint32_t cpu_boot_arg; + int err; + + err = DTLookupEntry(NULL, "/cpus", &entry); + assert(err == kSuccess); + + err = DTInitEntryIterator(entry, &iter); + assert(err == kSuccess); + + while (kSuccess == DTIterateEntries(&iter, &child)) { + +#if MACH_ASSERT + unsigned int propSize; + void *prop = NULL; + if (avail_cpus == 0) { + if (kSuccess != DTGetProperty(child, "state", &prop, &propSize)) + panic("unable to retrieve state for cpu %u", avail_cpus); + + if (strncmp((char*)prop, "running", propSize) != 0) + panic("cpu 0 has not been marked as running!"); + } + assert(kSuccess == DTGetProperty(child, "reg", &prop, &propSize)); + assert(avail_cpus == *((uint32_t*)prop)); +#endif + ++avail_cpus; + } + + cpu_boot_arg = avail_cpus; + if (PE_parse_boot_argn("cpus", &cpu_boot_arg, sizeof(cpu_boot_arg)) && + (avail_cpus > cpu_boot_arg)) + avail_cpus = cpu_boot_arg; + + if (avail_cpus == 0) + panic("No cpus found!"); +} + +unsigned int +ml_get_cpu_count(void) +{ + return avail_cpus; +} + +int +ml_get_boot_cpu_number(void) +{ + return 0; +} + +cluster_type_t +ml_get_boot_cluster(void) +{ + return CLUSTER_TYPE_SMP; +} + +int +ml_get_cpu_number(uint32_t phys_id) +{ + return (int)phys_id; +} + +int +ml_get_max_cpu_number(void) +{ + return avail_cpus - 1; +} + +kern_return_t +ml_processor_register( + ml_processor_info_t * in_processor_info, + processor_t * processor_out, + ipi_handler_t * ipi_handler) +{ + cpu_data_t *this_cpu_datap; + boolean_t is_boot_cpu; + + if (in_processor_info->phys_id >= MAX_CPUS) { + /* + * The physical CPU ID indicates that we have more CPUs than + * this xnu build support. This probably means we have an + * incorrect board configuration. + * + * TODO: Should this just return a failure instead? A panic + * is simply a convenient way to catch bugs in the pexpert + * headers. + */ + panic("phys_id %u is too large for MAX_CPUS (%u)", in_processor_info->phys_id, MAX_CPUS); + } + + /* Fail the registration if the number of CPUs has been limited by boot-arg. */ + if ((in_processor_info->phys_id >= avail_cpus) || + (in_processor_info->log_id > (uint32_t)ml_get_max_cpu_number())) + return KERN_FAILURE; + + if (in_processor_info->log_id != (uint32_t)ml_get_boot_cpu_number()) { + is_boot_cpu = FALSE; + this_cpu_datap = cpu_data_alloc(FALSE); + cpu_data_init(this_cpu_datap); + } else { + this_cpu_datap = &BootCpuData; + is_boot_cpu = TRUE; + } + + this_cpu_datap->cpu_id = in_processor_info->cpu_id; + + this_cpu_datap->cpu_chud = chudxnu_cpu_alloc(is_boot_cpu); + if (this_cpu_datap->cpu_chud == (void *)NULL) + goto processor_register_error; + this_cpu_datap->cpu_console_buf = console_cpu_alloc(is_boot_cpu); + if (this_cpu_datap->cpu_console_buf == (void *)(NULL)) + goto processor_register_error; + + if (!is_boot_cpu) { + if (cpu_data_register(this_cpu_datap) != KERN_SUCCESS) + goto processor_register_error; + } + + this_cpu_datap->cpu_idle_notify = (void *) in_processor_info->processor_idle; + this_cpu_datap->cpu_cache_dispatch = in_processor_info->platform_cache_dispatch; + nanoseconds_to_absolutetime((uint64_t) in_processor_info->powergate_latency, &this_cpu_datap->cpu_idle_latency); + this_cpu_datap->cpu_reset_assist = kvtophys(in_processor_info->powergate_stub_addr); + + this_cpu_datap->idle_timer_notify = (void *) in_processor_info->idle_timer; + this_cpu_datap->idle_timer_refcon = in_processor_info->idle_timer_refcon; + + this_cpu_datap->platform_error_handler = (void *) in_processor_info->platform_error_handler; + this_cpu_datap->cpu_regmap_paddr = in_processor_info->regmap_paddr; + this_cpu_datap->cpu_phys_id = in_processor_info->phys_id; + this_cpu_datap->cpu_l2_access_penalty = in_processor_info->l2_access_penalty; + + if (!is_boot_cpu) { + processor_init((struct processor *)this_cpu_datap->cpu_processor, + this_cpu_datap->cpu_number, processor_pset(master_processor)); + + if (this_cpu_datap->cpu_l2_access_penalty) { + /* + * Cores that have a non-zero L2 access penalty compared + * to the boot processor should be de-prioritized by the + * scheduler, so that threads use the cores with better L2 + * preferentially. + */ + processor_set_primary(this_cpu_datap->cpu_processor, + master_processor); + } + } + + *processor_out = this_cpu_datap->cpu_processor; + *ipi_handler = cpu_signal_handler; + if (in_processor_info->idle_tickle != (idle_tickle_t *) NULL) + *in_processor_info->idle_tickle = (idle_tickle_t) cpu_idle_tickle; + +#if KPC + if (kpc_register_cpu(this_cpu_datap) != TRUE) + goto processor_register_error; +#endif + + if (!is_boot_cpu) + prng_cpu_init(this_cpu_datap->cpu_number); + + return KERN_SUCCESS; + +processor_register_error: +#if KPC + kpc_unregister_cpu(this_cpu_datap); +#endif + if (this_cpu_datap->cpu_chud != (void *)NULL) + chudxnu_cpu_free(this_cpu_datap->cpu_chud); + if (!is_boot_cpu) + cpu_data_free(this_cpu_datap); + return KERN_FAILURE; +} + +void +ml_init_arm_debug_interface( + void * in_cpu_datap, + vm_offset_t virt_address) +{ + ((cpu_data_t *)in_cpu_datap)->cpu_debug_interface_map = virt_address; + do_debugid(); +} + +/* + * Routine: init_ast_check + * Function: + */ +void +init_ast_check( + __unused processor_t processor) +{ +} + +/* + * Routine: cause_ast_check + * Function: + */ +void +cause_ast_check( + processor_t processor) +{ + if (current_processor() != processor) { + cpu_signal(processor_to_cpu_datap(processor), SIGPast, (void *)NULL, (void *)NULL); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_AST), processor->cpu_id, 1 /* ast */, 0, 0, 0); + } +} + + +/* + * Routine: ml_at_interrupt_context + * Function: Check if running at interrupt context + */ +boolean_t +ml_at_interrupt_context(void) +{ + vm_offset_t stack_ptr; + vm_offset_t intstack_top_ptr; + + __asm__ volatile("mov %0, sp\n":"=r"(stack_ptr)); + intstack_top_ptr = getCpuDatap()->intstack_top; + return ((stack_ptr < intstack_top_ptr) && (stack_ptr > intstack_top_ptr - INTSTACK_SIZE)); +} + +extern uint32_t cpu_idle_count; + +void ml_get_power_state(boolean_t *icp, boolean_t *pidlep) { + *icp = ml_at_interrupt_context(); + *pidlep = (cpu_idle_count == real_ncpus); +} + +/* + * Routine: ml_cause_interrupt + * Function: Generate a fake interrupt + */ +void +ml_cause_interrupt(void) +{ + return; /* BS_XXX */ +} + +/* Map memory map IO space */ +vm_offset_t +ml_io_map( + vm_offset_t phys_addr, + vm_size_t size) +{ + return (io_map(phys_addr, size, VM_WIMG_IO)); +} + +vm_offset_t +ml_io_map_wcomb( + vm_offset_t phys_addr, + vm_size_t size) +{ + return (io_map(phys_addr, size, VM_WIMG_WCOMB)); +} + +/* boot memory allocation */ +vm_offset_t +ml_static_malloc( + __unused vm_size_t size) +{ + return ((vm_offset_t) NULL); +} + +vm_map_address_t +ml_map_high_window( + vm_offset_t phys_addr, + vm_size_t len) +{ + return pmap_map_high_window_bd(phys_addr, len, VM_PROT_READ | VM_PROT_WRITE); +} + +vm_offset_t +ml_static_ptovirt( + vm_offset_t paddr) +{ + return phystokv(paddr); +} + +vm_offset_t +ml_static_vtop( + vm_offset_t vaddr) +{ + if (((vm_address_t)(vaddr) - gVirtBase) >= gPhysSize) + panic("ml_static_ptovirt(): illegal vaddr: %p\n", (void*)vaddr); + return ((vm_address_t)(vaddr) - gVirtBase + gPhysBase); +} + + +kern_return_t +ml_static_protect( + vm_offset_t vaddr, /* kernel virtual address */ + vm_size_t size, + vm_prot_t new_prot) +{ + pt_entry_t arm_prot = 0; + pt_entry_t arm_block_prot = 0; + vm_offset_t vaddr_cur; + ppnum_t ppn; + kern_return_t result = KERN_SUCCESS; + + if (vaddr < VM_MIN_KERNEL_ADDRESS) + return KERN_FAILURE; + + assert((vaddr & (ARM_PGBYTES - 1)) == 0); /* must be page aligned */ + + if ((new_prot & VM_PROT_WRITE) && (new_prot & VM_PROT_EXECUTE)) { + panic("ml_static_protect(): WX request on %p", (void *) vaddr); + } + + /* Set up the protection bits, and block bits so we can validate block mappings. */ + if (new_prot & VM_PROT_WRITE) { + arm_prot |= ARM_PTE_AP(AP_RWNA); + arm_block_prot |= ARM_TTE_BLOCK_AP(AP_RWNA); + } else { + arm_prot |= ARM_PTE_AP(AP_RONA); + arm_block_prot |= ARM_TTE_BLOCK_AP(AP_RONA); + } + + if (!(new_prot & VM_PROT_EXECUTE)) { + arm_prot |= ARM_PTE_NX; + arm_block_prot |= ARM_TTE_BLOCK_NX; + } + + for (vaddr_cur = vaddr; + vaddr_cur < ((vaddr + size) & ~ARM_PGMASK); + vaddr_cur += ARM_PGBYTES) { + ppn = pmap_find_phys(kernel_pmap, vaddr_cur); + if (ppn != (vm_offset_t) NULL) { + tt_entry_t *ttp = &kernel_pmap->tte[ttenum(vaddr_cur)]; + tt_entry_t tte = *ttp; + + if ((tte & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE) { + if (((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) && + ((tte & (ARM_TTE_BLOCK_APMASK | ARM_TTE_BLOCK_NX_MASK)) == arm_block_prot)) { + /* + * We can support ml_static_protect on a block mapping if the mapping already has + * the desired protections. We still want to run checks on a per-page basis. + */ + continue; + } + + result = KERN_FAILURE; + break; + } + + pt_entry_t *pte_p = (pt_entry_t *) ttetokv(tte) + ptenum(vaddr_cur); + pt_entry_t ptmp = *pte_p; + + ptmp = (ptmp & ~(ARM_PTE_APMASK | ARM_PTE_NX_MASK)) | arm_prot; + *pte_p = ptmp; +#ifndef __ARM_L1_PTW__ + FlushPoC_DcacheRegion((vm_offset_t) pte_p, sizeof(*pte_p)); +#endif + } + } + + if (vaddr_cur > vaddr) + flush_mmu_tlb_region(vaddr, (vm_size_t)(vaddr_cur - vaddr)); + + return result; +} + +/* + * Routine: ml_static_mfree + * Function: + */ +void +ml_static_mfree( + vm_offset_t vaddr, + vm_size_t size) +{ + vm_offset_t vaddr_cur; + ppnum_t ppn; + uint32_t freed_pages = 0; + + /* It is acceptable (if bad) to fail to free. */ + if (vaddr < VM_MIN_KERNEL_ADDRESS) + return; + + assert((vaddr & (PAGE_SIZE - 1)) == 0); /* must be page aligned */ + + for (vaddr_cur = vaddr; + vaddr_cur < trunc_page_32(vaddr + size); + vaddr_cur += PAGE_SIZE) { + ppn = pmap_find_phys(kernel_pmap, vaddr_cur); + if (ppn != (vm_offset_t) NULL) { + /* + * It is not acceptable to fail to update the protections on a page + * we will release to the VM. We need to either panic or continue. + * For now, we'll panic (to help flag if there is memory we can + * reclaim). + */ + if (ml_static_protect(vaddr_cur, PAGE_SIZE, VM_PROT_WRITE | VM_PROT_READ) != KERN_SUCCESS) { + panic("Failed ml_static_mfree on %p", (void *) vaddr_cur); + } +#if 0 + /* + * Must NOT tear down the "V==P" mapping for vaddr_cur as the zone alias scheme + * relies on the persistence of these mappings for all time. + */ + // pmap_remove(kernel_pmap, (addr64_t) vaddr_cur, (addr64_t) (vaddr_cur + PAGE_SIZE)); +#endif + vm_page_create(ppn, (ppn + 1)); + freed_pages++; + } + } + vm_page_lockspin_queues(); + vm_page_wire_count -= freed_pages; + vm_page_wire_count_initial -= freed_pages; + vm_page_unlock_queues(); +#if DEBUG + kprintf("ml_static_mfree: Released 0x%x pages at VA %p, size:0x%llx, last ppn: 0x%x\n", freed_pages, (void *)vaddr, (uint64_t)size, ppn); +#endif +} + + +/* virtual to physical on wired pages */ +vm_offset_t +ml_vtophys(vm_offset_t vaddr) +{ + return kvtophys(vaddr); +} + +/* + * Routine: ml_nofault_copy + * Function: Perform a physical mode copy if the source and destination have + * valid translations in the kernel pmap. If translations are present, they are + * assumed to be wired; e.g., no attempt is made to guarantee that the + * translations obtained remain valid for the duration of the copy process. + */ +vm_size_t +ml_nofault_copy(vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size) +{ + addr64_t cur_phys_dst, cur_phys_src; + uint32_t count, nbytes = 0; + + while (size > 0) { + if (!(cur_phys_src = kvtophys(virtsrc))) + break; + if (!(cur_phys_dst = kvtophys(virtdst))) + break; + if (!pmap_valid_address(trunc_page_64(cur_phys_dst)) || + !pmap_valid_address(trunc_page_64(cur_phys_src))) + break; + count = PAGE_SIZE - (cur_phys_src & PAGE_MASK); + if (count > (PAGE_SIZE - (cur_phys_dst & PAGE_MASK))) + count = PAGE_SIZE - (cur_phys_dst & PAGE_MASK); + if (count > size) + count = size; + + bcopy_phys(cur_phys_src, cur_phys_dst, count); + + nbytes += count; + virtsrc += count; + virtdst += count; + size -= count; + } + + return nbytes; +} + +/* + * Routine: ml_validate_nofault + * Function: Validate that ths address range has a valid translations + * in the kernel pmap. If translations are present, they are + * assumed to be wired; i.e. no attempt is made to guarantee + * that the translation persist after the check. + * Returns: TRUE if the range is mapped and will not cause a fault, + * FALSE otherwise. + */ + +boolean_t ml_validate_nofault( + vm_offset_t virtsrc, vm_size_t size) +{ + addr64_t cur_phys_src; + uint32_t count; + + while (size > 0) { + if (!(cur_phys_src = kvtophys(virtsrc))) + return FALSE; + if (!pmap_valid_address(trunc_page_64(cur_phys_src))) + return FALSE; + count = (uint32_t)(PAGE_SIZE - (cur_phys_src & PAGE_MASK)); + if (count > size) + count = (uint32_t)size; + + virtsrc += count; + size -= count; + } + + return TRUE; +} + +void +ml_get_bouncepool_info(vm_offset_t * phys_addr, vm_size_t * size) +{ + *phys_addr = 0; + *size = 0; +} + +/* + * Stubs for CPU Stepper + */ +void +active_rt_threads(__unused boolean_t active) +{ +} + +void +thread_tell_urgency(__unused int urgency, + __unused uint64_t rt_period, + __unused uint64_t rt_deadline, + __unused uint64_t sched_latency, + __unused thread_t nthread) +{ +} + +void +machine_run_count(__unused uint32_t count) +{ +} + +processor_t +machine_choose_processor(__unused processor_set_t pset, processor_t processor) +{ + return (processor); +} + +vm_offset_t +ml_stack_remaining(void) +{ + uintptr_t local = (uintptr_t) &local; + + if (ml_at_interrupt_context()) { + return (local - (getCpuDatap()->intstack_top - INTSTACK_SIZE)); + } else { + return (local - current_thread()->kernel_stack); + } +} + +boolean_t machine_timeout_suspended(void) { + return FALSE; +} + +kern_return_t +ml_interrupt_prewarm(__unused uint64_t deadline) +{ + return KERN_FAILURE; +} + +uint64_t +ml_get_hwclock(void) +{ + uint64_t high_first = 0; + uint64_t high_second = 0; + uint64_t low = 0; + + __builtin_arm_isb(ISB_SY); + + do { + high_first = __builtin_arm_mrrc(15, 0, 14) >> 32; + low = __builtin_arm_mrrc(15, 0, 14) & 0xFFFFFFFFULL; + high_second = __builtin_arm_mrrc(15, 0, 14) >> 32; + } while (high_first != high_second); + + return (high_first << 32) | (low); +} + +boolean_t +ml_delay_should_spin(uint64_t interval) +{ + cpu_data_t *cdp = getCpuDatap(); + + if (cdp->cpu_idle_latency) { + return (interval < cdp->cpu_idle_latency) ? TRUE : FALSE; + } else { + /* + * Early boot, latency is unknown. Err on the side of blocking, + * which should always be safe, even if slow + */ + return FALSE; + } +} + +boolean_t ml_thread_is64bit(thread_t thread) +{ + return (thread_is_64bit(thread)); +} + +void ml_timer_evaluate(void) { +} + +boolean_t +ml_timer_forced_evaluation(void) { + return FALSE; +} + +uint64_t +ml_energy_stat(__unused thread_t t) { + return 0; +} + + +void +ml_gpu_stat_update(__unused uint64_t gpu_ns_delta) { +#if CONFIG_EMBEDDED + /* + * For now: update the resource coalition stats of the + * current thread's coalition + */ + task_coalition_update_gpu_stats(current_task(), gpu_ns_delta); +#endif +} + +uint64_t +ml_gpu_stat(__unused thread_t t) { + return 0; +} + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME +static void +timer_state_event(boolean_t switch_to_kernel) +{ + thread_t thread = current_thread(); + if (!thread->precise_user_kernel_time) return; + + processor_data_t *pd = &getCpuDatap()->cpu_processor->processor_data; + uint64_t now = ml_get_timebase(); + + timer_stop(pd->current_state, now); + pd->current_state = (switch_to_kernel) ? &pd->system_state : &pd->user_state; + timer_start(pd->current_state, now); + + timer_stop(pd->thread_timer, now); + pd->thread_timer = (switch_to_kernel) ? &thread->system_timer : &thread->user_timer; + timer_start(pd->thread_timer, now); +} + +void +timer_state_event_user_to_kernel(void) +{ + timer_state_event(TRUE); +} + +void +timer_state_event_kernel_to_user(void) +{ + timer_state_event(FALSE); +} +#endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */ + +boolean_t +user_cont_hwclock_allowed(void) +{ + return FALSE; +} + +boolean_t +user_timebase_allowed(void) +{ +#if __ARM_TIME__ + return TRUE; +#else + return FALSE; +#endif +} + +/* + * The following are required for parts of the kernel + * that cannot resolve these functions as inlines: + */ +extern thread_t current_act(void); +thread_t +current_act(void) +{ + return current_thread_fast(); +} + +#undef current_thread +extern thread_t current_thread(void); +thread_t +current_thread(void) +{ + return current_thread_fast(); +} + +#if __ARM_USER_PROTECT__ +uintptr_t +arm_user_protect_begin(thread_t thread) +{ + uintptr_t ttbr0, asid = 0; // kernel asid + + ttbr0 = __builtin_arm_mrc(15,0,2,0,0); // Get TTBR0 + if (ttbr0 != thread->machine.kptw_ttb) { + __builtin_arm_mcr(15,0,thread->machine.kptw_ttb,2,0,0); // Set TTBR0 + __builtin_arm_mcr(15,0,asid,13,0,1); // Set CONTEXTIDR + __builtin_arm_isb(ISB_SY); + } + return ttbr0; +} + +void +arm_user_protect_end(thread_t thread, uintptr_t ttbr0, boolean_t disable_interrupts) +{ + if ((ttbr0 != thread->machine.kptw_ttb) && (thread->machine.uptw_ttb != thread->machine.kptw_ttb)) { + if (disable_interrupts) + __asm__ volatile ("cpsid if" ::: "memory"); // Disable FIQ/IRQ + __builtin_arm_mcr(15,0,thread->machine.uptw_ttb,2,0,0); // Set TTBR0 + __builtin_arm_mcr(15,0,thread->machine.asid,13,0,1); // Set CONTEXTIDR with thread asid + __builtin_arm_dsb(DSB_ISH); + __builtin_arm_isb(ISB_SY); + } +} +#endif // __ARM_USER_PROTECT__ + +void ml_task_set_rop_pid(__unused task_t task, __unused task_t parent_task, __unused boolean_t inherit) +{ + return; +} diff --git a/osfmk/arm/machine_routines.h b/osfmk/arm/machine_routines.h new file mode 100644 index 000000000..3510649b4 --- /dev/null +++ b/osfmk/arm/machine_routines.h @@ -0,0 +1,884 @@ +/* + * Copyright (c) 2007-2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#ifndef _ARM_MACHINE_ROUTINES_H_ +#define _ARM_MACHINE_ROUTINES_H_ + +#include +#include +#include +#include + +#include +#include + +#include + +__BEGIN_DECLS + +/* Interrupt handling */ + +void ml_cpu_signal(unsigned int cpu_id); +void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs); +uint64_t ml_cpu_signal_deferred_get_timer(void); +void ml_cpu_signal_deferred(unsigned int cpu_id); +void ml_cpu_signal_retract(unsigned int cpu_id); + +/* Initialize Interrupts */ +void ml_init_interrupt(void); + +/* Get Interrupts Enabled */ +boolean_t ml_get_interrupts_enabled(void); + +/* Set Interrupts Enabled */ +boolean_t ml_set_interrupts_enabled(boolean_t enable); + +/* Check if running at interrupt context */ +boolean_t ml_at_interrupt_context(void); + +/* Generate a fake interrupt */ +void ml_cause_interrupt(void); + +/* Clear interrupt spin debug state for thread */ +#if INTERRUPT_MASKED_DEBUG +void ml_spin_debug_reset(thread_t thread); +void ml_spin_debug_clear(thread_t thread); +void ml_spin_debug_clear_self(void); +void ml_check_interrupts_disabled_duration(thread_t thread); +#endif + +#ifdef XNU_KERNEL_PRIVATE +extern boolean_t ml_is_quiescing(void); +extern void ml_set_is_quiescing(boolean_t); +extern uint64_t ml_get_booter_memory_size(void); +#endif + +/* Type for the Time Base Enable function */ +typedef void (*time_base_enable_t)(cpu_id_t cpu_id, boolean_t enable); +#if MACH_KERNEL_PRIVATE +/* Type for the Processor Cache Dispatch function */ +typedef void (*cache_dispatch_t)(cpu_id_t cpu_id, unsigned int select, unsigned int param0, unsigned int param1); +#endif + +#define CacheConfig 0x00000000UL +#define CacheControl 0x00000001UL +#define CacheClean 0x00000002UL +#define CacheCleanRegion 0x00000003UL +#define CacheCleanFlush 0x00000004UL +#define CacheCleanFlushRegion 0x00000005UL +#define CacheShutdown 0x00000006UL + +#define CacheControlEnable 0x00000000UL + +#define CacheConfigCCSIDR 0x00000001UL +#define CacheConfigSize 0x00000100UL + +/* Type for the Processor Idle function */ +typedef void (*processor_idle_t)(cpu_id_t cpu_id, boolean_t enter, uint64_t *new_timeout_ticks); + +/* Type for the Idle Tickle function */ +typedef void (*idle_tickle_t)(void); + +/* Type for the Idle Timer function */ +typedef void (*idle_timer_t)(void *refcon, uint64_t *new_timeout_ticks); + +/* Type for the IPI Hander */ +typedef void (*ipi_handler_t)(void); + +/* Type for the Lockdown Hander */ +typedef void (*lockdown_handler_t)(void *); + +/* Type for the Platform specific Error Handler */ +typedef void (*platform_error_handler_t)(void *refcon, vm_offset_t fault_addr); + +/* + * The exception callback (ex_cb) module allows kernel drivers to + * register and receive callbacks for exceptions, and indicate + * actions to be taken by the platform kernel + * Currently this is supported for ARM64 but extending support for ARM32 + * should be straightforward + */ + +/* Supported exception classes for callbacks */ +typedef enum +{ + EXCB_CLASS_ILLEGAL_INSTR_SET, + EXCB_CLASS_MAX // this must be last +} +ex_cb_class_t; + +/* Actions indicated by callbacks to be taken by platform kernel */ +typedef enum +{ + EXCB_ACTION_RERUN, // re-run the faulting instruction + EXCB_ACTION_NONE, // continue normal exception handling +} +ex_cb_action_t; + +/* + * Exception state + * We cannot use a private kernel data structure such as arm_saved_state_t + * The CPSR and ESR are not clobbered when the callback function is invoked so + * those registers can be examined by the callback function; + * the same is done in the platform error handlers + */ +typedef struct +{ + vm_offset_t far; +} +ex_cb_state_t; + +/* callback type definition */ +typedef ex_cb_action_t (*ex_cb_t) ( + ex_cb_class_t cb_class, + void *refcon,// provided at registration + const ex_cb_state_t *state // exception state + ); + +/* + * Callback registration + * Currently we support only one registered callback per class but + * it should be possible to support more callbacks + */ +kern_return_t ex_cb_register( + ex_cb_class_t cb_class, + ex_cb_t cb, + void *refcon ); + +/* + * Called internally by platform kernel to invoke the registered callback for class + */ +ex_cb_action_t ex_cb_invoke( + ex_cb_class_t cb_class, + vm_offset_t far); + + +void ml_parse_cpu_topology(void); + +unsigned int ml_get_cpu_count(void); + +int ml_get_boot_cpu_number(void); + +int ml_get_cpu_number(uint32_t phys_id); + +int ml_get_max_cpu_number(void); + +/* Struct for ml_cpu_get_info */ +struct ml_cpu_info { + unsigned long vector_unit; + unsigned long cache_line_size; + unsigned long l1_icache_size; + unsigned long l1_dcache_size; + unsigned long l2_settings; + unsigned long l2_cache_size; + unsigned long l3_settings; + unsigned long l3_cache_size; +}; +typedef struct ml_cpu_info ml_cpu_info_t; + +typedef enum { + CLUSTER_TYPE_SMP, +} cluster_type_t; + +cluster_type_t ml_get_boot_cluster(void); + +/* Struct for ml_processor_register */ +struct ml_processor_info { + cpu_id_t cpu_id; + vm_offset_t start_paddr; + boolean_t supports_nap; + void *platform_cache_dispatch; + time_base_enable_t time_base_enable; + processor_idle_t processor_idle; + idle_tickle_t *idle_tickle; + idle_timer_t idle_timer; + void *idle_timer_refcon; + vm_offset_t powergate_stub_addr; + uint32_t powergate_stub_length; + uint32_t powergate_latency; + platform_error_handler_t platform_error_handler; + uint64_t regmap_paddr; + uint32_t phys_id; + uint32_t log_id; + uint32_t l2_access_penalty; + uint32_t cluster_id; + cluster_type_t cluster_type; + uint32_t l2_cache_id; + uint32_t l2_cache_size; + uint32_t l3_cache_id; + uint32_t l3_cache_size; +}; +typedef struct ml_processor_info ml_processor_info_t; + +#if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) +/* Struct for ml_init_timebase */ +struct tbd_ops { + void (*tbd_fiq_handler)(void); + uint32_t (*tbd_get_decrementer)(void); + void (*tbd_set_decrementer)(uint32_t dec_value); +}; +typedef struct tbd_ops *tbd_ops_t; +typedef struct tbd_ops tbd_ops_data_t; +#endif + +/* Register a processor */ +kern_return_t ml_processor_register( + ml_processor_info_t *ml_processor_info, + processor_t *processor, + ipi_handler_t *ipi_handler); + +/* Register a lockdown handler */ +kern_return_t ml_lockdown_handler_register(lockdown_handler_t, void *); + +#if XNU_KERNEL_PRIVATE +void ml_lockdown_init(void); + +/* Check if the machine layer wants to intercept a panic call */ +boolean_t ml_wants_panic_trap_to_debugger(void); + +/* Machine layer routine for intercepting panics */ +void ml_panic_trap_to_debugger(const char *panic_format_str, + va_list *panic_args, + unsigned int reason, + void *ctx, + uint64_t panic_options_mask, + unsigned long panic_caller); +#endif /* XNU_KERNEL_PRIVATE */ + +/* Initialize Interrupts */ +void ml_install_interrupt_handler( + void *nub, + int source, + void *target, + IOInterruptHandler handler, + void *refCon); + +vm_offset_t +ml_static_vtop( + vm_offset_t); + +vm_offset_t +ml_static_ptovirt( + vm_offset_t); + +/* Offset required to obtain absolute time value from tick counter */ +uint64_t ml_get_abstime_offset(void); + +/* Offset required to obtain continuous time value from tick counter */ +uint64_t ml_get_conttime_offset(void); + +#ifdef __APPLE_API_UNSTABLE +/* PCI config cycle probing */ +boolean_t ml_probe_read( + vm_offset_t paddr, + unsigned int *val); +boolean_t ml_probe_read_64( + addr64_t paddr, + unsigned int *val); + +/* Read physical address byte */ +unsigned int ml_phys_read_byte( + vm_offset_t paddr); +unsigned int ml_phys_read_byte_64( + addr64_t paddr); + +/* Read physical address half word */ +unsigned int ml_phys_read_half( + vm_offset_t paddr); +unsigned int ml_phys_read_half_64( + addr64_t paddr); + +/* Read physical address word*/ +unsigned int ml_phys_read( + vm_offset_t paddr); +unsigned int ml_phys_read_64( + addr64_t paddr); +unsigned int ml_phys_read_word( + vm_offset_t paddr); +unsigned int ml_phys_read_word_64( + addr64_t paddr); + +unsigned long long ml_io_read(uintptr_t iovaddr, int iovsz); +unsigned int ml_io_read8(uintptr_t iovaddr); +unsigned int ml_io_read16(uintptr_t iovaddr); +unsigned int ml_io_read32(uintptr_t iovaddr); +unsigned long long ml_io_read64(uintptr_t iovaddr); + +/* Read physical address double word */ +unsigned long long ml_phys_read_double( + vm_offset_t paddr); +unsigned long long ml_phys_read_double_64( + addr64_t paddr); + +/* Write physical address byte */ +void ml_phys_write_byte( + vm_offset_t paddr, unsigned int data); +void ml_phys_write_byte_64( + addr64_t paddr, unsigned int data); + +/* Write physical address half word */ +void ml_phys_write_half( + vm_offset_t paddr, unsigned int data); +void ml_phys_write_half_64( + addr64_t paddr, unsigned int data); + +/* Write physical address word */ +void ml_phys_write( + vm_offset_t paddr, unsigned int data); +void ml_phys_write_64( + addr64_t paddr, unsigned int data); +void ml_phys_write_word( + vm_offset_t paddr, unsigned int data); +void ml_phys_write_word_64( + addr64_t paddr, unsigned int data); + +/* Write physical address double word */ +void ml_phys_write_double( + vm_offset_t paddr, unsigned long long data); +void ml_phys_write_double_64( + addr64_t paddr, unsigned long long data); + +void ml_static_mfree( + vm_offset_t, + vm_size_t); + +kern_return_t +ml_static_protect( + vm_offset_t start, + vm_size_t size, + vm_prot_t new_prot); + +/* virtual to physical on wired pages */ +vm_offset_t ml_vtophys( + vm_offset_t vaddr); + +/* Get processor info */ +void ml_cpu_get_info(ml_cpu_info_t *ml_cpu_info); + +#endif /* __APPLE_API_UNSTABLE */ + +#ifdef __APPLE_API_PRIVATE +#ifdef XNU_KERNEL_PRIVATE +vm_size_t ml_nofault_copy( + vm_offset_t virtsrc, + vm_offset_t virtdst, + vm_size_t size); +boolean_t ml_validate_nofault( + vm_offset_t virtsrc, vm_size_t size); +#endif /* XNU_KERNEL_PRIVATE */ +#if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) +/* IO memory map services */ + +/* Map memory map IO space */ +vm_offset_t ml_io_map( + vm_offset_t phys_addr, + vm_size_t size); + +vm_offset_t ml_io_map_wcomb( + vm_offset_t phys_addr, + vm_size_t size); + +void ml_get_bouncepool_info( + vm_offset_t *phys_addr, + vm_size_t *size); + +vm_map_address_t ml_map_high_window( + vm_offset_t phys_addr, + vm_size_t len); + +/* boot memory allocation */ +vm_offset_t ml_static_malloc( + vm_size_t size); + +void ml_init_timebase( + void *args, + tbd_ops_t tbd_funcs, + vm_offset_t int_address, + vm_offset_t int_value); + +uint64_t ml_get_timebase(void); + +void ml_init_lock_timeout(void); + +boolean_t ml_delay_should_spin(uint64_t interval); + +uint32_t ml_get_decrementer(void); + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME +void timer_state_event_user_to_kernel(void); +void timer_state_event_kernel_to_user(void); +#endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */ + +uint64_t ml_get_hwclock(void); + +#ifdef __arm64__ +boolean_t ml_get_timer_pending(void); +#endif + +void platform_syscall( + struct arm_saved_state *); + +void ml_set_decrementer( + uint32_t dec_value); + +boolean_t is_user_contex( + void); + +void ml_init_arm_debug_interface(void *args, vm_offset_t virt_address); + +/* These calls are only valid if __ARM_USER_PROTECT__ is defined */ +uintptr_t arm_user_protect_begin( + thread_t thread); + +void arm_user_protect_end( + thread_t thread, + uintptr_t up, + boolean_t disable_interrupts); + +#endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */ + +/* Zero bytes starting at a physical address */ +void bzero_phys( + addr64_t phys_address, + vm_size_t length); + +void bzero_phys_nc(addr64_t src64, vm_size_t bytes); + +void ml_thread_policy( + thread_t thread, + unsigned policy_id, + unsigned policy_info); + +#define MACHINE_GROUP 0x00000001 +#define MACHINE_NETWORK_GROUP 0x10000000 +#define MACHINE_NETWORK_WORKLOOP 0x00000001 +#define MACHINE_NETWORK_NETISR 0x00000002 + +/* Initialize the maximum number of CPUs */ +void ml_init_max_cpus( + unsigned int max_cpus); + +/* Return the maximum number of CPUs set by ml_init_max_cpus() */ +unsigned int ml_get_max_cpus( + void); + +/* Return the maximum memory size */ +unsigned int ml_get_machine_mem(void); + +#ifdef XNU_KERNEL_PRIVATE +/* Return max offset */ +vm_map_offset_t ml_get_max_offset( + boolean_t is64, + unsigned int option); +#define MACHINE_MAX_OFFSET_DEFAULT 0x01 +#define MACHINE_MAX_OFFSET_MIN 0x02 +#define MACHINE_MAX_OFFSET_MAX 0x04 +#define MACHINE_MAX_OFFSET_DEVICE 0x08 +#endif + +extern void ml_cpu_up(void); +extern void ml_cpu_down(void); +extern void ml_arm_sleep(void); + +extern uint64_t ml_get_wake_timebase(void); +extern uint64_t ml_get_conttime_wake_time(void); + +/* Time since the system was reset (as part of boot/wake) */ +uint64_t ml_get_time_since_reset(void); + +#ifdef XNU_KERNEL_PRIVATE +/* Just a stub on ARM */ +extern kern_return_t ml_interrupt_prewarm(uint64_t deadline); +#define TCOAL_DEBUG(x, a, b, c, d, e) do { } while(0) +#endif /* XNU_KERNEL_PRIVATE */ + +/* Bytes available on current stack */ +vm_offset_t ml_stack_remaining(void); + +#ifdef MACH_KERNEL_PRIVATE +uint32_t get_fpscr(void); +void set_fpscr(uint32_t); + +extern void init_vfp(void); +extern boolean_t get_vfp_enabled(void); +#if (__ARM_VFP__ >= 3) +extern unsigned int get_mvfr0(void); +extern unsigned int get_mvfr1(void); +#endif +extern void arm_debug_set_cp14(arm_debug_state_t *debug_state); +extern void fiq_context_init(boolean_t enable_fiq); + +extern void reenable_async_aborts(void); +extern void cpu_idle_wfi(boolean_t wfi_fast); + +#ifdef MONITOR +#define MONITOR_SET_ENTRY 0x800 /* Set kernel entry point from monitor */ +#define MONITOR_LOCKDOWN 0x801 /* Enforce kernel text/rodata integrity */ +unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1, + uintptr_t arg2, uintptr_t arg3); +#endif /* MONITOR */ + +#if defined(KERNEL_INTEGRITY_KTRR) +void rorgn_stash_range(void); +void rorgn_lockdown(void); +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + +#endif /* MACH_KERNEL_PRIVATE */ + +extern uint32_t arm_debug_read_dscr(void); + +extern int set_be_bit(void); +extern int clr_be_bit(void); +extern int be_tracing(void); + +typedef void (*broadcastFunc) (void *); +unsigned int cpu_broadcast_xcall(uint32_t *, boolean_t, broadcastFunc, void *); +kern_return_t cpu_xcall(int, broadcastFunc, void *); + +#ifdef KERNEL_PRIVATE + +/* Interface to be used by the perf. controller to register a callback, in a + * single-threaded fashion. The callback will receive notifications of + * processor performance quality-of-service changes from the scheduler. + */ + +#ifdef __arm64__ +typedef void (*cpu_qos_update_t)(int throughput_qos, uint64_t qos_param1, uint64_t qos_param2); +void cpu_qos_update_register(cpu_qos_update_t); +#endif /* __arm64__ */ + +struct going_on_core { + uint64_t thread_id; + uint16_t qos_class; + uint16_t urgency; /* XCPM compatibility */ + uint32_t is_32_bit : 1; /* uses 32-bit ISA/register state in userspace (which may differ from address space size) */ + uint32_t is_kernel_thread : 1; + uint64_t thread_group_id; + void *thread_group_data; + uint64_t scheduling_latency; /* absolute time between when thread was made runnable and this ctx switch */ + uint64_t start_time; + uint64_t scheduling_latency_at_same_basepri; + uint32_t energy_estimate_nj; /* return: In nanojoules */ + /* smaller of the time between last change to base priority and ctx switch and scheduling_latency */ +}; +typedef struct going_on_core *going_on_core_t; + +struct going_off_core { + uint64_t thread_id; + uint32_t energy_estimate_nj; /* return: In nanojoules */ + uint32_t reserved; + uint64_t end_time; + uint64_t thread_group_id; + void *thread_group_data; +}; +typedef struct going_off_core *going_off_core_t; + +struct thread_group_data { + uint64_t thread_group_id; + void *thread_group_data; + uint32_t thread_group_size; + uint32_t thread_group_flags; +}; +typedef struct thread_group_data *thread_group_data_t; + +struct perfcontrol_max_runnable_latency { + uint64_t max_scheduling_latencies[4 /* THREAD_URGENCY_MAX */]; +}; +typedef struct perfcontrol_max_runnable_latency *perfcontrol_max_runnable_latency_t; + +struct perfcontrol_work_interval { + uint64_t thread_id; + uint16_t qos_class; + uint16_t urgency; + uint32_t flags; // notify + uint64_t work_interval_id; + uint64_t start; + uint64_t finish; + uint64_t deadline; + uint64_t next_start; + uint64_t thread_group_id; + void *thread_group_data; + uint32_t create_flags; +}; +typedef struct perfcontrol_work_interval *perfcontrol_work_interval_t; + + +/* + * Structure to export per-CPU counters as part of the CLPC callout. + * Contains only the fixed CPU counters (instructions and cycles); CLPC + * would call back into XNU to get the configurable counters if needed. + */ +struct perfcontrol_cpu_counters { + uint64_t instructions; + uint64_t cycles; +}; + +/* + * Structure used to pass information about a thread to CLPC + */ +struct perfcontrol_thread_data { + /* + * Energy estimate (return value) + * The field is populated by CLPC and used to update the + * energy estimate of the thread + */ + uint32_t energy_estimate_nj; + /* Perfcontrol class for thread */ + perfcontrol_class_t perfctl_class; + /* Thread ID for the thread */ + uint64_t thread_id; + /* Thread Group ID */ + uint64_t thread_group_id; + /* + * Scheduling latency for threads at the same base priority. + * Calculated by the scheduler and passed into CLPC. The field is + * populated only in the thread_data structure for the thread + * going on-core. + */ + uint64_t scheduling_latency_at_same_basepri; + /* Thread Group data pointer */ + void *thread_group_data; + /* perfctl state pointer */ + void *perfctl_state; +}; + +/* + * All callouts from the scheduler are executed with interrupts + * disabled. Callouts should be implemented in C with minimal + * abstractions, and only use KPI exported by the mach/libkern + * symbolset, restricted to routines like spinlocks and atomic + * operations and scheduler routines as noted below. Spinlocks that + * are used to synchronize data in the perfcontrol_state_t should only + * ever be acquired with interrupts disabled, to avoid deadlocks where + * an quantum expiration timer interrupt attempts to perform a callout + * that attempts to lock a spinlock that is already held. + */ + +/* + * When a processor is switching between two threads (after the + * scheduler has chosen a new thread), the low-level platform layer + * will call this routine, which should perform required timestamps, + * MMIO register reads, or other state switching. No scheduler locks + * are held during this callout. + * + * This function is called with interrupts ENABLED. + */ +typedef void (*sched_perfcontrol_context_switch_t)(perfcontrol_state_t, perfcontrol_state_t); + +/* + * Once the processor has switched to the new thread, the offcore + * callout will indicate the old thread that is no longer being + * run. The thread's scheduler lock is held, so it will not begin + * running on another processor (in the case of preemption where it + * remains runnable) until it completes. If the "thread_terminating" + * boolean is TRUE, this will be the last callout for this thread_id. + */ +typedef void (*sched_perfcontrol_offcore_t)(perfcontrol_state_t, going_off_core_t /* populated by callee */, boolean_t); + +/* + * After the offcore callout and after the old thread can potentially + * start running on another processor, the oncore callout will be + * called with the thread's scheduler lock held. The oncore callout is + * also called any time one of the parameters in the going_on_core_t + * structure changes, like priority/QoS changes, and quantum + * expiration, so the callout must not assume callouts are paired with + * offcore callouts. + */ +typedef void (*sched_perfcontrol_oncore_t)(perfcontrol_state_t, going_on_core_t); + +/* + * Periodically (on hundreds of ms scale), the scheduler will perform + * maintenance and report the maximum latency for runnable (but not currently + * running) threads for each urgency class. + */ +typedef void (*sched_perfcontrol_max_runnable_latency_t)(perfcontrol_max_runnable_latency_t); + +/* + * When the kernel receives information about work intervals from userland, + * it is passed along using this callback. No locks are held, although the state + * object will not go away during the callout. + */ +typedef void (*sched_perfcontrol_work_interval_notify_t)(perfcontrol_state_t, perfcontrol_work_interval_t); + +/* + * These callbacks are used when thread groups are added, removed or properties + * updated. + * No blocking allocations (or anything else blocking) are allowed inside these + * callbacks. No locks allowed in these callbacks as well since the kernel might + * be holding the thread/task locks. + */ +typedef void (*sched_perfcontrol_thread_group_init_t)(thread_group_data_t); +typedef void (*sched_perfcontrol_thread_group_deinit_t)(thread_group_data_t); +typedef void (*sched_perfcontrol_thread_group_flags_update_t)(thread_group_data_t); + +/* + * Sometime after the timeout set by sched_perfcontrol_update_callback_deadline has passed, + * this function will be called, passing the timeout deadline that was previously armed as an argument. + * + * This is called inside context-switch/quantum-interrupt context and must follow the safety rules for that context. + */ +typedef void (*sched_perfcontrol_deadline_passed_t)(uint64_t deadline); + +/* + * Context Switch Callout + * + * Parameters: + * event - The perfcontrol_event for this callout + * cpu_id - The CPU doing the context switch + * timestamp - The timestamp for the context switch + * flags - Flags for other relevant information + * offcore - perfcontrol_data structure for thread going off-core + * oncore - perfcontrol_data structure for thread going on-core + * cpu_counters - perfcontrol_cpu_counters for the CPU doing the switch + */ +typedef void (*sched_perfcontrol_csw_t)( + perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, + struct perfcontrol_thread_data *offcore, struct perfcontrol_thread_data *oncore, + struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused); + + +/* + * Thread State Update Callout + * + * Parameters: + * event - The perfcontrol_event for this callout + * cpu_id - The CPU doing the state update + * timestamp - The timestamp for the state update + * flags - Flags for other relevant information + * thr_data - perfcontrol_data structure for the thread being updated + */ +typedef void (*sched_perfcontrol_state_update_t)( + perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, + struct perfcontrol_thread_data *thr_data, __unused void *unused); + + +/* + * Callers should always use the CURRENT version so that the kernel can detect both older + * and newer structure layouts. New callbacks should always be added at the end of the + * structure, and xnu should expect existing source recompiled against newer headers + * to pass NULL for unimplemented callbacks. Pass NULL as the as the callbacks parameter + * to reset callbacks to their default in-kernel values. + */ + +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_0 (0) /* up-to oncore */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_1 (1) /* up-to max_runnable_latency */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_2 (2) /* up-to work_interval_notify */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_3 (3) /* up-to thread_group_deinit */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_4 (4) /* up-to deadline_passed */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_5 (5) /* up-to state_update */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_6 (6) /* up-to thread_group_flags_update */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT SCHED_PERFCONTROL_CALLBACKS_VERSION_6 + +struct sched_perfcontrol_callbacks { + unsigned long version; /* Use SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT */ + sched_perfcontrol_offcore_t offcore; + sched_perfcontrol_context_switch_t context_switch; + sched_perfcontrol_oncore_t oncore; + sched_perfcontrol_max_runnable_latency_t max_runnable_latency; + sched_perfcontrol_work_interval_notify_t work_interval_notify; + sched_perfcontrol_thread_group_init_t thread_group_init; + sched_perfcontrol_thread_group_deinit_t thread_group_deinit; + sched_perfcontrol_deadline_passed_t deadline_passed; + sched_perfcontrol_csw_t csw; + sched_perfcontrol_state_update_t state_update; + sched_perfcontrol_thread_group_flags_update_t thread_group_flags_update; +}; +typedef struct sched_perfcontrol_callbacks *sched_perfcontrol_callbacks_t; + +extern void sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state); + +/* + * Update the scheduler with the set of cores that should be used to dispatch new threads. + * Non-recommended cores can still be used to field interrupts or run bound threads. + * This should be called with interrupts enabled and no scheduler locks held. + */ +#define ALL_CORES_RECOMMENDED (~(uint32_t)0) + +extern void sched_perfcontrol_update_recommended_cores(uint32_t recommended_cores); +extern void sched_perfcontrol_thread_group_recommend(void *data, cluster_type_t recommendation); + +/* + * Update the deadline after which sched_perfcontrol_deadline_passed will be called. + * Returns TRUE if it successfully canceled a previously set callback, + * and FALSE if it did not (i.e. one wasn't set, or callback already fired / is in flight). + * The callback is automatically canceled when it fires, and does not repeat unless rearmed. + * + * This 'timer' executes as the scheduler switches between threads, on a non-idle core + * + * There can be only one outstanding timer globally. + */ +extern boolean_t sched_perfcontrol_update_callback_deadline(uint64_t deadline); + +typedef enum perfcontrol_callout_type { + PERFCONTROL_CALLOUT_ON_CORE, + PERFCONTROL_CALLOUT_OFF_CORE, + PERFCONTROL_CALLOUT_CONTEXT, + PERFCONTROL_CALLOUT_STATE_UPDATE, + /* Add other callout types here */ + PERFCONTROL_CALLOUT_MAX +} perfcontrol_callout_type_t; + +typedef enum perfcontrol_callout_stat { + PERFCONTROL_STAT_INSTRS, + PERFCONTROL_STAT_CYCLES, + /* Add other stat types here */ + PERFCONTROL_STAT_MAX +} perfcontrol_callout_stat_t; + +uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type, + perfcontrol_callout_stat_t stat); + + +#endif /* KERNEL_PRIVATE */ + +boolean_t machine_timeout_suspended(void); +void ml_get_power_state(boolean_t *, boolean_t *); + +boolean_t user_cont_hwclock_allowed(void); +boolean_t user_timebase_allowed(void); +boolean_t ml_thread_is64bit(thread_t thread); +void ml_task_set_rop_pid(task_t task, task_t parent_task, boolean_t inherit); + +#ifdef __arm64__ +void ml_set_align_checking(void); +boolean_t arm64_wfe_allowed(void); +#endif /* __arm64__ */ + +void ml_timer_evaluate(void); +boolean_t ml_timer_forced_evaluation(void); +uint64_t ml_energy_stat(thread_t); +void ml_gpu_stat_update(uint64_t); +uint64_t ml_gpu_stat(thread_t); +#endif /* __APPLE_API_PRIVATE */ + +__END_DECLS + +#endif /* _ARM_MACHINE_ROUTINES_H_ */ diff --git a/osfmk/arm/machine_routines_asm.s b/osfmk/arm/machine_routines_asm.s new file mode 100644 index 000000000..b64d28373 --- /dev/null +++ b/osfmk/arm/machine_routines_asm.s @@ -0,0 +1,1131 @@ +/* + * Copyright (c) 2007-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include "assym.s" + + .align 2 + .globl EXT(machine_set_current_thread) +LEXT(machine_set_current_thread) + mcr p15, 0, r0, c13, c0, 4 // Write TPIDRPRW + ldr r1, [r0, TH_CTH_SELF] + mrc p15, 0, r2, c13, c0, 3 // Read TPIDRURO + and r2, r2, #3 // Extract cpu number + orr r1, r1, r2 // + mcr p15, 0, r1, c13, c0, 3 // Write TPIDRURO + ldr r1, [r0, TH_CTH_DATA] + mcr p15, 0, r1, c13, c0, 2 // Write TPIDRURW + bx lr + +/* + * void machine_idle(void) + */ + .text + .align 2 + .globl EXT(machine_idle) +LEXT(machine_idle) + cpsid if // Disable FIQ IRQ + mov ip, lr + bl EXT(Idle_context) + mov lr, ip + cpsie if // Enable FIQ IRQ + bx lr + +/* + * void cpu_idle_wfi(boolean_t wfi_fast): + * cpu_idle is the only function that should call this. + */ + .text + .align 2 + .globl EXT(cpu_idle_wfi) +LEXT(cpu_idle_wfi) + mov r1, #32 + mov r2, #1200 + cmp r0, #0 + beq 3f + mov r1, #1 + b 2f + .align 5 +1: + add r0, r0, #1 + mov r1, r2 +2: + +/* + * We export the address of the WFI instruction so that it can be patched; this will be + * ugly from a debugging perspective. + */ + +#if (__ARM_ARCH__ >= 7) + dsb + .globl EXT(wfi_inst) +LEXT(wfi_inst) + wfi +#else + mcr p15, 0, r0, c7, c10, 4 + .globl EXT(wfi_inst) +LEXT(wfi_inst) + mcr p15, 0, r0, c7, c0, 4 +#endif +3: + subs r1, r1, #1 + bne 3b + nop + nop + nop + nop + nop + cmp r0, #0 + beq 1b + bx lr + + .align 2 + .globl EXT(timer_grab) +LEXT(timer_grab) +0: + ldr r2, [r0, TIMER_HIGH] + ldr r3, [r0, TIMER_LOW] +#if __ARM_SMP__ + dmb ish // dmb ish +#endif + ldr r1, [r0, TIMER_HIGHCHK] + cmp r1, r2 + bne 0b + mov r0, r3 + bx lr + + .align 2 + .globl EXT(timer_update) +LEXT(timer_update) + str r1, [r0, TIMER_HIGHCHK] +#if __ARM_SMP__ + dmb ish // dmb ish +#endif + str r2, [r0, TIMER_LOW] +#if __ARM_SMP__ + dmb ish // dmb ish +#endif + str r1, [r0, TIMER_HIGH] + bx lr + + .align 2 + .globl EXT(get_vfp_enabled) +LEXT(get_vfp_enabled) +#if __ARM_VFP__ + fmrx r0, fpexc + and r1, r0, #FPEXC_EN // Extact vfp enable previous state + mov r0, r1, LSR #FPEXC_EN_BIT // Return 1 if enabled, 0 if disabled +#else + mov r0, #0 // return false +#endif + bx lr + +/* This is no longer useful (but is exported, so this may require kext cleanup). */ + .align 2 + .globl EXT(enable_kernel_vfp_context) +LEXT(enable_kernel_vfp_context) + bx lr + +/* uint32_t get_fpscr(void): + * Returns the current state of the FPSCR register. + */ + .align 2 + .globl EXT(get_fpscr) +LEXT(get_fpscr) +#if __ARM_VFP__ + fmrx r0, fpscr +#endif + bx lr + .align 2 + .globl EXT(set_fpscr) +/* void set_fpscr(uint32_t value): + * Set the FPSCR register. + */ +LEXT(set_fpscr) +#if __ARM_VFP__ + fmxr fpscr, r0 +#else + mov r0, #0 +#endif + bx lr + +#if (__ARM_VFP__ >= 3) + .align 2 + .globl EXT(get_mvfr0) +LEXT(get_mvfr0) + vmrs r0, mvfr0 + bx lr + .globl EXT(get_mvfr1) +LEXT(get_mvfr1) + vmrs r0, mvfr1 + bx lr +#endif + +/* + * void OSSynchronizeIO(void) + */ + .text + .align 2 + .globl EXT(OSSynchronizeIO) +LEXT(OSSynchronizeIO) + .align 2 + dsb + bx lr + +/* + * void flush_mmu_tlb(void) + * + * Flush all TLBs + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb) +LEXT(flush_mmu_tlb) + mov r0, #0 +#if __ARM_SMP__ + mcr p15, 0, r0, c8, c3, 0 // Invalidate Inner Shareable entire TLBs +#else + mcr p15, 0, r0, c8, c7, 0 // Invalidate entire TLB +#endif + dsb ish + isb + bx lr + +/* + * void flush_core_tlb(void) + * + * Flush core TLB + */ + .text + .align 2 + .globl EXT(flush_core_tlb) +LEXT(flush_core_tlb) + mov r0, #0 + mcr p15, 0, r0, c8, c7, 0 // Invalidate entire TLB + dsb ish + isb + bx lr + +/* + * void flush_mmu_tlb_entry(uint32_t) + * + * Flush TLB entry + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_entry) +LEXT(flush_mmu_tlb_entry) +#if __ARM_SMP__ + mcr p15, 0, r0, c8, c3, 1 // Invalidate TLB Inner Shareableentry +#else + mcr p15, 0, r0, c8, c7, 1 // Invalidate TLB entry +#endif + dsb ish + isb + bx lr + +/* + * void flush_mmu_tlb_entries(uint32_t, uint32_t) + * + * Flush TLB entries + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_entries) +LEXT(flush_mmu_tlb_entries) +1: +#if __ARM_SMP__ + mcr p15, 0, r0, c8, c3, 1 // Invalidate TLB Inner Shareable entry +#else + mcr p15, 0, r0, c8, c7, 1 // Invalidate TLB entry +#endif + add r0, r0, ARM_PGBYTES // Increment to the next page + cmp r0, r1 // Loop if current address < end address + blt 1b + dsb ish // Synchronize + isb + bx lr + + +/* + * void flush_mmu_tlb_mva_entries(uint32_t) + * + * Flush TLB entries for mva + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_mva_entries) +LEXT(flush_mmu_tlb_mva_entries) +#if __ARM_SMP__ + mcr p15, 0, r0, c8, c3, 3 // Invalidate TLB Inner Shareable entries by mva +#else + mcr p15, 0, r0, c8, c7, 3 // Invalidate TLB Inner Shareable entries by mva +#endif + dsb ish + isb + bx lr + +/* + * void flush_mmu_tlb_asid(uint32_t) + * + * Flush TLB entriesfor requested asid + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_asid) +LEXT(flush_mmu_tlb_asid) +#if __ARM_SMP__ + mcr p15, 0, r0, c8, c3, 2 // Invalidate TLB Inner Shareable entries by asid +#else + mcr p15, 0, r0, c8, c7, 2 // Invalidate TLB entries by asid +#endif + dsb ish + isb + bx lr + +/* + * void flush_core_tlb_asid(uint32_t) + * + * Flush TLB entries for core for requested asid + */ + .text + .align 2 + .globl EXT(flush_core_tlb_asid) +LEXT(flush_core_tlb_asid) + mcr p15, 0, r0, c8, c7, 2 // Invalidate TLB entries by asid + dsb ish + isb + bx lr + +/* + * Set MMU Translation Table Base + */ + .text + .align 2 + .globl EXT(set_mmu_ttb) +LEXT(set_mmu_ttb) + orr r0, r0, #(TTBR_SETUP & 0xFF) // Setup PTWs memory attribute + orr r0, r0, #(TTBR_SETUP & 0xFF00) // Setup PTWs memory attribute + mcr p15, 0, r0, c2, c0, 0 // write r0 to translation table 0 + dsb ish + isb + bx lr + +/* + * Set MMU Translation Table Base Alternate + */ + .text + .align 2 + .globl EXT(set_mmu_ttb_alternate) +LEXT(set_mmu_ttb_alternate) + orr r0, r0, #(TTBR_SETUP & 0xFF) // Setup PTWs memory attribute + orr r0, r0, #(TTBR_SETUP & 0xFF00) // Setup PTWs memory attribute + mcr p15, 0, r0, c2, c0, 1 // write r0 to translation table 1 + dsb ish + isb + bx lr + +/* + * Set MMU Translation Table Base + */ + .text + .align 2 + .globl EXT(get_mmu_ttb) +LEXT(get_mmu_ttb) + mrc p15, 0, r0, c2, c0, 0 // translation table to r0 + isb + bx lr + +/* + * get MMU control register + */ + .text + .align 2 + .globl EXT(get_aux_control) +LEXT(get_aux_control) + mrc p15, 0, r0, c1, c0, 1 // read aux control into r0 + bx lr // return old bits in r0 + +/* + * set MMU control register + */ + .text + .align 2 + .globl EXT(set_aux_control) +LEXT(set_aux_control) + mcr p15, 0, r0, c1, c0, 1 // write r0 back to aux control + isb + bx lr + + +/* + * get MMU control register + */ + .text + .align 2 + .globl EXT(get_mmu_control) +LEXT(get_mmu_control) + mrc p15, 0, r0, c1, c0, 0 // read mmu control into r0 + bx lr // return old bits in r0 + +/* + * set MMU control register + */ + .text + .align 2 + .globl EXT(set_mmu_control) +LEXT(set_mmu_control) + mcr p15, 0, r0, c1, c0, 0 // write r0 back to mmu control + isb + bx lr + +/* + * MMU kernel virtual to physical address translation + */ + .text + .align 2 + .globl EXT(mmu_kvtop) +LEXT(mmu_kvtop) + mrs r3, cpsr // Read cpsr + cpsid if // Disable FIQ IRQ + mov r1, r0 + mcr p15, 0, r1, c7, c8, 0 // Write V2PCWPR + isb + mrc p15, 0, r0, c7, c4, 0 // Read PAR + ands r2, r0, #0x1 // Test conversion aborted + bne mmu_kvtophys_fail + ands r2, r0, #0x2 // Test super section + mvnne r2, #0xFF000000 + moveq r2, #0x000000FF + orreq r2, r2, #0x00000F00 + bics r0, r0, r2 // Clear lower bits + beq mmu_kvtophys_fail + and r1, r1, r2 + orr r0, r0, r1 + b mmu_kvtophys_ret +mmu_kvtophys_fail: + mov r0, #0 +mmu_kvtophys_ret: + msr cpsr, r3 // Restore cpsr + bx lr + +/* + * MMU user virtual to physical address translation + */ + .text + .align 2 + .globl EXT(mmu_uvtop) +LEXT(mmu_uvtop) + mrs r3, cpsr // Read cpsr + cpsid if // Disable FIQ IRQ + mov r1, r0 + mcr p15, 0, r1, c7, c8, 2 // Write V2PCWUR + isb + mrc p15, 0, r0, c7, c4, 0 // Read PAR + ands r2, r0, #0x1 // Test conversion aborted + bne mmu_uvtophys_fail + ands r2, r0, #0x2 // Test super section + mvnne r2, #0xFF000000 + moveq r2, #0x000000FF + orreq r2, r2, #0x00000F00 + bics r0, r0, r2 // Clear lower bits + beq mmu_uvtophys_fail + and r1, r1, r2 + orr r0, r0, r1 + b mmu_uvtophys_ret +mmu_uvtophys_fail: + mov r0, #0 +mmu_uvtophys_ret: + msr cpsr, r3 // Restore cpsr + bx lr + +/* + * MMU kernel virtual to physical address preflight write access + */ + .text + .align 2 + .globl EXT(mmu_kvtop_wpreflight) +LEXT(mmu_kvtop_wpreflight) + mrs r3, cpsr // Read cpsr + cpsid if // Disable FIQ IRQ + mov r1, r0 + mcr p15, 0, r1, c7, c8, 1 // Write V2PCWPW + isb + mrc p15, 0, r0, c7, c4, 0 // Read PAR + ands r2, r0, #0x1 // Test conversion aborted + bne mmu_kvtophys_wpreflight_fail + ands r2, r0, #0x2 // Test super section + mvnne r2, #0xFF000000 + moveq r2, #0x000000FF + orreq r2, r2, #0x00000F00 + bics r0, r0, r2 // Clear lower bits + beq mmu_kvtophys_wpreflight_fail // Sanity check: successful access must deliver zero low bits + and r1, r1, r2 + orr r0, r0, r1 + b mmu_kvtophys_wpreflight_ret +mmu_kvtophys_wpreflight_fail: + mov r0, #0 +mmu_kvtophys_wpreflight_ret: + msr cpsr, r3 // Restore cpsr + bx lr + +/* + * set context id register + */ +/* + * set context id register + */ + .text + .align 2 + .globl EXT(set_context_id) +LEXT(set_context_id) + mcr p15, 0, r0, c13, c0, 1 + isb + bx lr + +#define COPYIO_HEADER(rUser, kLabel) \ + /* test for zero len */ ;\ + cmp r2, #0 ;\ + moveq r0, #0 ;\ + bxeq lr ;\ + /* test user_addr, user_addr+len to see if it's in kernel space */ ;\ + add r12, rUser, r2 ;\ + cmp r12, KERNELBASE ;\ + bhs kLabel ;\ + cmp r12, rUser ;\ + bcc kLabel + +#define COPYIO_VALIDATE(NAME, SIZE) \ + /* branch around for small sizes */ ;\ + cmp r2, #(SIZE) ;\ + bls L##NAME##_validate_done ;\ + /* call NAME_validate to check the arguments */ ;\ + push {r0, r1, r2, r7, lr} ;\ + add r7, sp, #12 ;\ + blx EXT(NAME##_validate) ;\ + cmp r0, #0 ;\ + addne sp, #12 ;\ + popne {r7, pc} ;\ + pop {r0, r1, r2, r7, lr} ;\ +L##NAME##_validate_done: + +#define COPYIO_SET_RECOVER() \ + /* set recovery address */ ;\ + stmfd sp!, { r4, r5, r6 } ;\ + adr r3, copyio_error ;\ + mrc p15, 0, r12, c13, c0, 4 ;\ + ldr r4, [r12, TH_RECOVER] ;\ + str r3, [r12, TH_RECOVER] + +#if __ARM_USER_PROTECT__ +#define COPYIO_MAP_USER() \ + /* disable interrupts to prevent expansion to 2GB at L1 ;\ + * between loading ttep and storing it in ttbr0.*/ ;\ + mrs r5, cpsr ;\ + cpsid if ;\ + ldr r3, [r12, ACT_UPTW_TTB] ;\ + mcr p15, 0, r3, c2, c0, 0 ;\ + msr cpsr, r5 ;\ + ldr r3, [r12, ACT_ASID] ;\ + mcr p15, 0, r3, c13, c0, 1 ;\ + isb +#else +#define COPYIO_MAP_USER() +#endif + +#define COPYIO_HEADER_KERN() ;\ + /* test for zero len */ ;\ + cmp r2, #0 ;\ + moveq r0, #0 ;\ + bxeq lr + +.macro COPYIO_BODY + /* if len is less than 16 bytes, just do a simple copy */ + cmp r2, #16 + blt L$0_bytewise + /* test for src and dest of the same word alignment */ + orr r3, r0, r1 + tst r3, #3 + bne L$0_bytewise +L$0_wordwise: + sub r2, r2, #16 +L$0_wordwise_loop: + /* 16 bytes at a time */ + ldmia r0!, { r3, r5, r6, r12 } + stmia r1!, { r3, r5, r6, r12 } + subs r2, r2, #16 + bge L$0_wordwise_loop + /* fixup the len and test for completion */ + adds r2, r2, #16 + beq L$0_noerror +L$0_bytewise: + /* copy 2 bytes at a time */ + subs r2, r2, #2 + ldrb r3, [r0], #1 + ldrbpl r12, [r0], #1 + strb r3, [r1], #1 + strbpl r12, [r1], #1 + bhi L$0_bytewise +L$0_noerror: + mov r0, #0 +.endmacro + +#if __ARM_USER_PROTECT__ +#define COPYIO_UNMAP_USER() \ + mrc p15, 0, r12, c13, c0, 4 ;\ + ldr r3, [r12, ACT_KPTW_TTB] ;\ + mcr p15, 0, r3, c2, c0, 0 ;\ + mov r3, #0 ;\ + mcr p15, 0, r3, c13, c0, 1 ;\ + isb +#else +#define COPYIO_UNMAP_USER() \ + mrc p15, 0, r12, c13, c0, 4 +#endif + +#define COPYIO_RESTORE_RECOVER() \ + /* restore the recovery address */ ;\ + str r4, [r12, TH_RECOVER] ;\ + ldmfd sp!, { r4, r5, r6 } + +/* + * int copyinstr( + * const user_addr_t user_addr, + * char *kernel_addr, + * vm_size_t max, + * vm_size_t *actual) + */ + .text + .align 2 + .globl EXT(copyinstr) +LEXT(copyinstr) + stmfd sp!, { r4, r5, r6 } + + mov r6, r3 + add r3, r0, r2 // user_addr + max + cmp r3, KERNELBASE // Check KERNELBASE < user_addr + max + bhs copyinstr_param_error // Drop out if it is + cmp r3, r0 // Check we're copying from user space + bcc copyinstr_param_error // Drop out if we aren't + adr r3, copyinstr_error // Get address for recover + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r4, [r12, TH_RECOVER] ;\ + str r3, [r12, TH_RECOVER] + COPYIO_MAP_USER() + mov r12, #0 // Number of bytes copied so far + cmp r2, #0 + beq copyinstr_too_long +copyinstr_loop: + ldrb r3, [r0], #1 // Load a byte from the source (user) + strb r3, [r1], #1 // Store a byte to the destination (kernel) + add r12, r12, #1 + cmp r3, #0 + beq copyinstr_done + cmp r12, r2 // Room to copy more bytes? + bne copyinstr_loop +// +// Ran out of space in the destination buffer, so return ENAMETOOLONG. +// +copyinstr_too_long: + mov r3, #ENAMETOOLONG +copyinstr_done: +// +// When we get here, we have finished copying the string. We came here from +// either the "beq copyinstr_done" above, in which case r4 == 0 (which is also +// the function result for success), or falling through from copyinstr_too_long, +// in which case r4 == ENAMETOOLONG. +// + str r12, [r6] // Save the count for actual + mov r0, r3 // Return error code from r3 +copyinstr_exit: + COPYIO_UNMAP_USER() + str r4, [r12, TH_RECOVER] +copyinstr_exit2: + ldmfd sp!, { r4, r5, r6 } + bx lr + +copyinstr_error: + /* set error, exit routine */ + mov r0, #EFAULT + b copyinstr_exit + +copyinstr_param_error: + /* set error, exit routine */ + mov r0, #EFAULT + b copyinstr_exit2 + +/* + * int copyin(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes) + */ + .text + .align 2 + .globl EXT(copyin) +LEXT(copyin) + COPYIO_HEADER(r0,copyio_kernel) + COPYIO_VALIDATE(copyin,4096) + COPYIO_SET_RECOVER() + COPYIO_MAP_USER() + COPYIO_BODY copyin + COPYIO_UNMAP_USER() + COPYIO_RESTORE_RECOVER() + bx lr + +/* + * int copyout(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes) + */ + .text + .align 2 + .globl EXT(copyout) +LEXT(copyout) + COPYIO_HEADER(r1,copyio_kernel) + COPYIO_VALIDATE(copyout,4096) + COPYIO_SET_RECOVER() + COPYIO_MAP_USER() + COPYIO_BODY copyout + COPYIO_UNMAP_USER() + COPYIO_RESTORE_RECOVER() + bx lr + + +/* + * int copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_size_t nbytes) + */ + .text + .align 2 + .globl EXT(copyin_word) +LEXT(copyin_word) + cmp r2, #4 // Test if size is 4 or 8 + cmpne r2, #8 + bne L_copyin_invalid + sub r3, r2, #1 + tst r0, r3 // Test alignment of user address + bne L_copyin_invalid + + COPYIO_HEADER(r0,L_copyin_word_fault) + COPYIO_SET_RECOVER() + COPYIO_MAP_USER() + + mov r3, #0 // Clear high register + cmp r2, #4 // If size is 4 + ldreq r2, [r0] // Load word from user + ldrdne r2, r3, [r0] // Else Load double word from user + stm r1, {r2, r3} // Store to kernel_addr + mov r0, #0 // Success + + COPYIO_UNMAP_USER() + COPYIO_RESTORE_RECOVER() + bx lr +L_copyin_invalid: + mov r0, #EINVAL + bx lr +L_copyin_word_fault: + mov r0, #EFAULT + bx lr + + +copyio_error: + mov r0, #EFAULT + COPYIO_UNMAP_USER() + str r4, [r12, TH_RECOVER] + ldmfd sp!, { r4, r5, r6 } + bx lr + +/* + * int copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes) + */ + .text + .align 2 + .globl EXT(copyin_kern) +LEXT(copyin_kern) + COPYIO_HEADER_KERN() + b bypass_check + +/* + * int copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes) + */ + .text + .align 2 + .globl EXT(copyout_kern) +LEXT(copyout_kern) + COPYIO_HEADER_KERN() + b bypass_check + +copyio_kernel_error: + mov r0, #EFAULT + bx lr + +copyio_kernel: + /* if (current_thread()->map->pmap != kernel_pmap) return EFAULT */ + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r3, [r12, ACT_MAP] + ldr r3, [r3, MAP_PMAP] + LOAD_ADDR(ip, kernel_pmap_store) + cmp r3, ip + bne copyio_kernel_error + +bypass_check: + stmfd sp!, { r5, r6 } + COPYIO_BODY copyio_kernel + ldmfd sp!, { r5, r6 } + bx lr + +/* + * int copyinframe(const vm_address_t frame_addr, char *kernel_addr) + * + * Safely copy eight bytes (the fixed top of an ARM frame) from + * either user or kernel memory. + */ + .text + .align 2 + .globl EXT(copyinframe) +LEXT(copyinframe) + COPYIO_SET_RECOVER() + COPYIO_MAP_USER() + ldmia r0, {r2, r3} + stmia r1, {r2, r3} + b Lcopyin_noerror + +/* + * uint32_t arm_debug_read_dscr(void) + */ + .text + .align 2 + .globl EXT(arm_debug_read_dscr) +LEXT(arm_debug_read_dscr) +#if __ARM_DEBUG__ >= 6 + mrc p14, 0, r0, c0, c1 +#else + mov r0, #0 +#endif + bx lr + +/* + * void arm_debug_set_cp14(arm_debug_state_t *debug_state) + * + * Set debug registers to match the current thread state + * (NULL to disable). Assume 6 breakpoints and 2 + * watchpoints, since that has been the case in all cores + * thus far. + */ + .text + .align 2 + .globl EXT(arm_debug_set_cp14) +LEXT(arm_debug_set_cp14) +#if __ARM_DEBUG__ >= 6 + mrc p15, 0, r1, c13, c0, 4 // Read TPIDRPRW + ldr r2, [r1, ACT_CPUDATAP] // Get current cpu + str r0, [r2, CPU_USER_DEBUG] // Set current user debug + + // Lock the debug registers + movw ip, #0xCE55 + movt ip, #0xC5AC + mcr p14, 0, ip, c1, c0, 4 + + // enable monitor mode (needed to set and use debug registers) + mrc p14, 0, ip, c0, c1, 0 + orr ip, ip, #0x8000 // set MDBGen = 1 +#if __ARM_DEBUG__ >= 7 + mcr p14, 0, ip, c0, c2, 2 +#else + mcr p14, 0, ip, c0, c1, 0 +#endif + // first turn off all breakpoints/watchpoints + mov r1, #0 + mcr p14, 0, r1, c0, c0, 5 // BCR0 + mcr p14, 0, r1, c0, c1, 5 // BCR1 + mcr p14, 0, r1, c0, c2, 5 // BCR2 + mcr p14, 0, r1, c0, c3, 5 // BCR3 + mcr p14, 0, r1, c0, c4, 5 // BCR4 + mcr p14, 0, r1, c0, c5, 5 // BCR5 + mcr p14, 0, r1, c0, c0, 7 // WCR0 + mcr p14, 0, r1, c0, c1, 7 // WCR1 + // if (debug_state == NULL) disable monitor mode and return; + cmp r0, #0 + biceq ip, ip, #0x8000 // set MDBGen = 0 +#if __ARM_DEBUG__ >= 7 + mcreq p14, 0, ip, c0, c2, 2 +#else + mcreq p14, 0, ip, c0, c1, 0 +#endif + bxeq lr + ldmia r0!, {r1, r2, r3, ip} + mcr p14, 0, r1, c0, c0, 4 // BVR0 + mcr p14, 0, r2, c0, c1, 4 // BVR1 + mcr p14, 0, r3, c0, c2, 4 // BVR2 + mcr p14, 0, ip, c0, c3, 4 // BVR3 + ldmia r0!, {r1, r2} + mcr p14, 0, r1, c0, c4, 4 // BVR4 + mcr p14, 0, r2, c0, c5, 4 // BVR5 + add r0, r0, #40 // advance to bcr[0] + ldmia r0!, {r1, r2, r3, ip} + mcr p14, 0, r1, c0, c0, 5 // BCR0 + mcr p14, 0, r2, c0, c1, 5 // BCR1 + mcr p14, 0, r3, c0, c2, 5 // BCR2 + mcr p14, 0, ip, c0, c3, 5 // BCR3 + ldmia r0!, {r1, r2} + mcr p14, 0, r1, c0, c4, 5 // BCR4 + mcr p14, 0, r2, c0, c5, 5 // BCR5 + add r0, r0, #40 // advance to wvr[0] + ldmia r0!, {r1, r2} + mcr p14, 0, r1, c0, c0, 6 // WVR0 + mcr p14, 0, r2, c0, c1, 6 // WVR1 + add r0, r0, #56 // advance to wcr[0] + ldmia r0!, {r1, r2} + mcr p14, 0, r1, c0, c0, 7 // WCR0 + mcr p14, 0, r2, c0, c1, 7 // WCR1 + + // Unlock debug registers + mov ip, #0 + mcr p14, 0, ip, c1, c0, 4 +#endif + bx lr + +/* + * void fiq_context_init(boolean_t enable_fiq) + */ + .text + .align 2 + .globl EXT(fiq_context_init) +LEXT(fiq_context_init) + mrs r3, cpsr // Save current CPSR + cmp r0, #0 // Test enable_fiq + bicne r3, r3, #PSR_FIQF // Enable FIQ if not FALSE + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r2, [r12, ACT_CPUDATAP] // Get current cpu data + +#if __ARM_TIME__ + /* Despite the fact that we use the physical timebase + * register as the basis for time on our platforms, we + * end up using the virtual timer in order to manage + * deadlines. This is due to the fact that for our + * current platforms, the interrupt generated by the + * physical timer is not hooked up to anything, and is + * therefore dropped on the floor. Therefore, for + * timers to function they MUST be based on the virtual + * timer. + */ + + mov r0, #1 // Enable Timer + mcr p15, 0, r0, c14, c3, 1 // Write to CNTV_CTL + + /* Enable USER access to the physical timebase (PL0PCTEN). + * The rationale for providing access to the physical + * timebase being that the virtual timebase is broken for + * some platforms. Maintaining the offset ourselves isn't + * expensive, so mandate that the userspace implementation + * do timebase_phys+offset rather than trying to propogate + * all of the informaiton about what works up to USER. + */ + mcr p15, 0, r0, c14, c1, 0 // Set CNTKCTL.PL0PCTEN (CNTKCTL[0]) + +#else /* ! __ARM_TIME__ */ + msr cpsr_c, #(PSR_FIQ_MODE|PSR_FIQF|PSR_IRQF) // Change mode to FIQ with FIQ/IRQ disabled + mov r8, r2 // Load the BootCPUData address + ldr r9, [r2, CPU_GET_FIQ_HANDLER] // Load fiq function address + ldr r10, [r2, CPU_TBD_HARDWARE_ADDR] // Load the hardware address + ldr r11, [r2, CPU_TBD_HARDWARE_VAL] // Load the hardware value +#endif /* __ARM_TIME__ */ + + msr cpsr_c, r3 // Restore saved CPSR + bx lr + +/* + * void reenable_async_aborts(void) + */ + .text + .align 2 + .globl EXT(reenable_async_aborts) +LEXT(reenable_async_aborts) + cpsie a // Re-enable async aborts + bx lr + +/* + * uint64_t ml_get_timebase(void) + */ + .text + .align 2 + .globl EXT(ml_get_timebase) +LEXT(ml_get_timebase) + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r3, [r12, ACT_CPUDATAP] // Get current cpu data +#if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ + isb // Required by ARMV7C.b section B8.1.2, ARMv8 section D6.1.2. +1: + mrrc p15, 0, r3, r1, c14 // Read the Time Base (CNTPCT), high => r1 + mrrc p15, 0, r0, r3, c14 // Read the Time Base (CNTPCT), low => r0 + mrrc p15, 0, r3, r2, c14 // Read the Time Base (CNTPCT), high => r2 + cmp r1, r2 + bne 1b // Loop until both high values are the same + + ldr r3, [r12, ACT_CPUDATAP] // Get current cpu data + ldr r2, [r3, CPU_BASE_TIMEBASE_LOW] // Add in the offset to + adds r0, r0, r2 // convert to + ldr r2, [r3, CPU_BASE_TIMEBASE_HIGH] // mach_absolute_time + adc r1, r1, r2 // +#else /* ! __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ */ +1: + ldr r2, [r3, CPU_TIMEBASE_HIGH] // Get the saved TBU value + ldr r0, [r3, CPU_TIMEBASE_LOW] // Get the saved TBL value + ldr r1, [r3, CPU_TIMEBASE_HIGH] // Get the saved TBU value + cmp r1, r2 // Make sure TB has not rolled over + bne 1b +#endif /* __ARM_TIME__ */ + bx lr // return + + +/* + * uint32_t ml_get_decrementer(void) + */ + .text + .align 2 + .globl EXT(ml_get_decrementer) +LEXT(ml_get_decrementer) + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r3, [r12, ACT_CPUDATAP] // Get current cpu data + ldr r2, [r3, CPU_GET_DECREMENTER_FUNC] // Get get_decrementer_func + cmp r2, #0 + bxne r2 // Call it if there is one +#if __ARM_TIME__ + mrc p15, 0, r0, c14, c3, 0 // Read the Decrementer (CNTV_TVAL) +#else + ldr r0, [r3, CPU_DECREMENTER] // Get the saved dec value +#endif + bx lr // return + + +/* + * void ml_set_decrementer(uint32_t dec_value) + */ + .text + .align 2 + .globl EXT(ml_set_decrementer) +LEXT(ml_set_decrementer) + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r3, [r12, ACT_CPUDATAP] // Get current cpu data + ldr r2, [r3, CPU_SET_DECREMENTER_FUNC] // Get set_decrementer_func + cmp r2, #0 + bxne r2 // Call it if there is one +#if __ARM_TIME__ + str r0, [r3, CPU_DECREMENTER] // Save the new dec value + mcr p15, 0, r0, c14, c3, 0 // Write the Decrementer (CNTV_TVAL) +#else + mrs r2, cpsr // Save current CPSR + msr cpsr_c, #(PSR_FIQ_MODE|PSR_FIQF|PSR_IRQF) // Change mode to FIQ with FIQ/IRQ disabled. + mov r12, r0 // Set the DEC value + str r12, [r8, CPU_DECREMENTER] // Store DEC + msr cpsr_c, r2 // Restore saved CPSR +#endif + bx lr + + +/* + * boolean_t ml_get_interrupts_enabled(void) + */ + .text + .align 2 + .globl EXT(ml_get_interrupts_enabled) +LEXT(ml_get_interrupts_enabled) + mrs r2, cpsr + mov r0, #1 + bic r0, r0, r2, lsr #PSR_IRQFb + bx lr + +/* + * Platform Specific Timebase & Decrementer Functions + * + */ + +#if defined(ARM_BOARD_CLASS_S7002) + .text + .align 2 + .globl EXT(fleh_fiq_s7002) +LEXT(fleh_fiq_s7002) + str r11, [r10, #PMGR_INTERVAL_TMR_CTL_OFFSET] // Clear the decrementer interrupt + mvn r13, #0 + str r13, [r8, CPU_DECREMENTER] + b EXT(fleh_dec) + + .text + .align 2 + .globl EXT(s7002_get_decrementer) +LEXT(s7002_get_decrementer) + ldr ip, [r3, CPU_TBD_HARDWARE_ADDR] // Get the hardware address + add ip, ip, #PMGR_INTERVAL_TMR_OFFSET + ldr r0, [ip] // Get the Decrementer + bx lr + + .text + .align 2 + .globl EXT(s7002_set_decrementer) +LEXT(s7002_set_decrementer) + str r0, [r3, CPU_DECREMENTER] // Save the new dec value + ldr ip, [r3, CPU_TBD_HARDWARE_ADDR] // Get the hardware address + str r0, [ip, #PMGR_INTERVAL_TMR_OFFSET] // Store the new Decrementer + bx lr +#endif /* defined(ARM_BOARD_CLASS_S7002) */ + +#if defined(ARM_BOARD_CLASS_T8002) + .text + .align 2 + .globl EXT(fleh_fiq_t8002) +LEXT(fleh_fiq_t8002) + mov r13, #kAICTmrIntStat + str r11, [r10, r13] // Clear the decrementer interrupt + mvn r13, #0 + str r13, [r8, CPU_DECREMENTER] + b EXT(fleh_dec) + + .text + .align 2 + .globl EXT(t8002_get_decrementer) +LEXT(t8002_get_decrementer) + ldr ip, [r3, CPU_TBD_HARDWARE_ADDR] // Get the hardware address + mov r0, #kAICTmrCnt + add ip, ip, r0 + ldr r0, [ip] // Get the Decrementer + bx lr + + .text + .align 2 + .globl EXT(t8002_set_decrementer) +LEXT(t8002_set_decrementer) + str r0, [r3, CPU_DECREMENTER] // Save the new dec value + ldr ip, [r3, CPU_TBD_HARDWARE_ADDR] // Get the hardware address + mov r5, #kAICTmrCnt + str r0, [ip, r5] // Store the new Decrementer + bx lr +#endif /* defined(ARM_BOARD_CLASS_T8002) */ + +LOAD_ADDR_GEN_DEF(kernel_pmap_store) + +#include "globals_asm.h" + +/* vim: set ts=4: */ diff --git a/osfmk/arm/machine_routines_common.c b/osfmk/arm/machine_routines_common.c new file mode 100644 index 000000000..764fb97e0 --- /dev/null +++ b/osfmk/arm/machine_routines_common.c @@ -0,0 +1,614 @@ +/* + * Copyright (c) 2007-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if MONOTONIC +#include +#include +#endif /* MONOTONIC */ + +#include + +#if INTERRUPT_MASKED_DEBUG +extern boolean_t interrupt_masked_debug; +extern uint64_t interrupt_masked_timeout; +#endif + +extern uint64_t mach_absolutetime_asleep; + +static void +sched_perfcontrol_oncore_default(perfcontrol_state_t new_thread_state __unused, going_on_core_t on __unused) +{ +} + +static void +sched_perfcontrol_switch_default(perfcontrol_state_t old_thread_state __unused, perfcontrol_state_t new_thread_state __unused) +{ +} + +static void +sched_perfcontrol_offcore_default(perfcontrol_state_t old_thread_state __unused, going_off_core_t off __unused, boolean_t thread_terminating __unused) +{ +} + +static void +sched_perfcontrol_thread_group_default(thread_group_data_t data __unused) +{ +} + +static void +sched_perfcontrol_max_runnable_latency_default(perfcontrol_max_runnable_latency_t latencies __unused) +{ +} + +static void +sched_perfcontrol_work_interval_notify_default(perfcontrol_state_t thread_state __unused, perfcontrol_work_interval_t work_interval __unused) +{ +} + +static void +sched_perfcontrol_deadline_passed_default(__unused uint64_t deadline) +{ +} + +static void +sched_perfcontrol_csw_default( + __unused perfcontrol_event event, __unused uint32_t cpu_id, __unused uint64_t timestamp, + __unused uint32_t flags, __unused struct perfcontrol_thread_data *offcore, + __unused struct perfcontrol_thread_data *oncore, + __unused struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused) +{ +} + +static void +sched_perfcontrol_state_update_default( + __unused perfcontrol_event event, __unused uint32_t cpu_id, __unused uint64_t timestamp, + __unused uint32_t flags, __unused struct perfcontrol_thread_data *thr_data, + __unused void *unused) +{ +} + +sched_perfcontrol_offcore_t sched_perfcontrol_offcore = sched_perfcontrol_offcore_default; +sched_perfcontrol_context_switch_t sched_perfcontrol_switch = sched_perfcontrol_switch_default; +sched_perfcontrol_oncore_t sched_perfcontrol_oncore = sched_perfcontrol_oncore_default; +sched_perfcontrol_thread_group_init_t sched_perfcontrol_thread_group_init = sched_perfcontrol_thread_group_default; +sched_perfcontrol_thread_group_deinit_t sched_perfcontrol_thread_group_deinit = sched_perfcontrol_thread_group_default; +sched_perfcontrol_thread_group_flags_update_t sched_perfcontrol_thread_group_flags_update = sched_perfcontrol_thread_group_default; +sched_perfcontrol_max_runnable_latency_t sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default; +sched_perfcontrol_work_interval_notify_t sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default; +sched_perfcontrol_deadline_passed_t sched_perfcontrol_deadline_passed = sched_perfcontrol_deadline_passed_default; +sched_perfcontrol_csw_t sched_perfcontrol_csw = sched_perfcontrol_csw_default; +sched_perfcontrol_state_update_t sched_perfcontrol_state_update = sched_perfcontrol_state_update_default; + +void +sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state) +{ + assert(callbacks == NULL || callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_2); + + if (size_of_state > sizeof(struct perfcontrol_state)) { + panic("%s: Invalid required state size %lu", __FUNCTION__, size_of_state); + } + + if (callbacks) { + + + if (callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_5) { + if (callbacks->csw != NULL) { + sched_perfcontrol_csw = callbacks->csw; + } else { + sched_perfcontrol_csw = sched_perfcontrol_csw_default; + } + + if (callbacks->state_update != NULL) { + sched_perfcontrol_state_update = callbacks->state_update; + } else { + sched_perfcontrol_state_update = sched_perfcontrol_state_update_default; + } + } + + if (callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_4) { + if (callbacks->deadline_passed != NULL) { + sched_perfcontrol_deadline_passed = callbacks->deadline_passed; + } else { + sched_perfcontrol_deadline_passed = sched_perfcontrol_deadline_passed_default; + } + } + + if (callbacks->offcore != NULL) { + sched_perfcontrol_offcore = callbacks->offcore; + } else { + sched_perfcontrol_offcore = sched_perfcontrol_offcore_default; + } + + if (callbacks->context_switch != NULL) { + sched_perfcontrol_switch = callbacks->context_switch; + } else { + sched_perfcontrol_switch = sched_perfcontrol_switch_default; + } + + if (callbacks->oncore != NULL) { + sched_perfcontrol_oncore = callbacks->oncore; + } else { + sched_perfcontrol_oncore = sched_perfcontrol_oncore_default; + } + + if (callbacks->max_runnable_latency != NULL) { + sched_perfcontrol_max_runnable_latency = callbacks->max_runnable_latency; + } else { + sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default; + } + + if (callbacks->work_interval_notify != NULL) { + sched_perfcontrol_work_interval_notify = callbacks->work_interval_notify; + } else { + sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default; + } + } else { + /* reset to defaults */ + sched_perfcontrol_offcore = sched_perfcontrol_offcore_default; + sched_perfcontrol_switch = sched_perfcontrol_switch_default; + sched_perfcontrol_oncore = sched_perfcontrol_oncore_default; + sched_perfcontrol_thread_group_init = sched_perfcontrol_thread_group_default; + sched_perfcontrol_thread_group_deinit = sched_perfcontrol_thread_group_default; + sched_perfcontrol_thread_group_flags_update = sched_perfcontrol_thread_group_default; + sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default; + sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default; + sched_perfcontrol_csw = sched_perfcontrol_csw_default; + sched_perfcontrol_state_update = sched_perfcontrol_state_update_default; + } +} + + +static void +machine_switch_populate_perfcontrol_thread_data(struct perfcontrol_thread_data *data, + thread_t thread, + uint64_t same_pri_latency) +{ + bzero(data, sizeof(struct perfcontrol_thread_data)); + data->perfctl_class = thread_get_perfcontrol_class(thread); + data->energy_estimate_nj = 0; + data->thread_id = thread->thread_id; + data->scheduling_latency_at_same_basepri = same_pri_latency; + data->perfctl_state = FIND_PERFCONTROL_STATE(thread); +} + +static void +machine_switch_populate_perfcontrol_cpu_counters(struct perfcontrol_cpu_counters *cpu_counters) +{ +#if MONOTONIC + mt_perfcontrol(&cpu_counters->instructions, &cpu_counters->cycles); +#else /* MONOTONIC */ + cpu_counters->instructions = 0; + cpu_counters->cycles = 0; +#endif /* !MONOTONIC */ +} + +int perfcontrol_callout_stats_enabled = 0; +static _Atomic uint64_t perfcontrol_callout_stats[PERFCONTROL_CALLOUT_MAX][PERFCONTROL_STAT_MAX]; +static _Atomic uint64_t perfcontrol_callout_count[PERFCONTROL_CALLOUT_MAX]; + +#if MONOTONIC +static inline +bool perfcontrol_callout_counters_begin(uint64_t *counters) +{ + if (!perfcontrol_callout_stats_enabled) + return false; + mt_fixed_counts(counters); + return true; +} + +static inline +void perfcontrol_callout_counters_end(uint64_t *start_counters, + perfcontrol_callout_type_t type) +{ + uint64_t end_counters[MT_CORE_NFIXED]; + mt_fixed_counts(end_counters); + atomic_fetch_add_explicit(&perfcontrol_callout_stats[type][PERFCONTROL_STAT_CYCLES], + end_counters[MT_CORE_CYCLES] - start_counters[MT_CORE_CYCLES], memory_order_relaxed); +#ifdef MT_CORE_INSTRS + atomic_fetch_add_explicit(&perfcontrol_callout_stats[type][PERFCONTROL_STAT_INSTRS], + end_counters[MT_CORE_INSTRS] - start_counters[MT_CORE_INSTRS], memory_order_relaxed); +#endif /* defined(MT_CORE_INSTRS) */ + atomic_fetch_add_explicit(&perfcontrol_callout_count[type], 1, memory_order_relaxed); +} +#endif /* MONOTONIC */ + +uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type, + perfcontrol_callout_stat_t stat) +{ + if (!perfcontrol_callout_stats_enabled) + return 0; + return (perfcontrol_callout_stats[type][stat] / perfcontrol_callout_count[type]); +} + +void +machine_switch_perfcontrol_context(perfcontrol_event event, + uint64_t timestamp, + uint32_t flags, + uint64_t new_thread_same_pri_latency, + thread_t old, + thread_t new) +{ + if (sched_perfcontrol_switch != sched_perfcontrol_switch_default) { + perfcontrol_state_t old_perfcontrol_state = FIND_PERFCONTROL_STATE(old); + perfcontrol_state_t new_perfcontrol_state = FIND_PERFCONTROL_STATE(new); + sched_perfcontrol_switch(old_perfcontrol_state, new_perfcontrol_state); + } + + if (sched_perfcontrol_csw != sched_perfcontrol_csw_default) { + uint32_t cpu_id = (uint32_t)cpu_number(); + struct perfcontrol_cpu_counters cpu_counters; + struct perfcontrol_thread_data offcore, oncore; + machine_switch_populate_perfcontrol_thread_data(&offcore, old, 0); + machine_switch_populate_perfcontrol_thread_data(&oncore, new, + new_thread_same_pri_latency); + machine_switch_populate_perfcontrol_cpu_counters(&cpu_counters); + +#if MONOTONIC + uint64_t counters[MT_CORE_NFIXED]; + bool ctrs_enabled = perfcontrol_callout_counters_begin(counters); +#endif /* MONOTONIC */ + sched_perfcontrol_csw(event, cpu_id, timestamp, flags, + &offcore, &oncore, &cpu_counters, NULL); +#if MONOTONIC + if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_CONTEXT); +#endif /* MONOTONIC */ + +#if __arm64__ + old->machine.energy_estimate_nj += offcore.energy_estimate_nj; + new->machine.energy_estimate_nj += oncore.energy_estimate_nj; +#endif + } +} + +void +machine_switch_perfcontrol_state_update(perfcontrol_event event, + uint64_t timestamp, + uint32_t flags, + thread_t thread) +{ + if (sched_perfcontrol_state_update == sched_perfcontrol_state_update_default) + return; + uint32_t cpu_id = (uint32_t)cpu_number(); + struct perfcontrol_thread_data data; + machine_switch_populate_perfcontrol_thread_data(&data, thread, 0); + +#if MONOTONIC + uint64_t counters[MT_CORE_NFIXED]; + bool ctrs_enabled = perfcontrol_callout_counters_begin(counters); +#endif /* MONOTONIC */ + sched_perfcontrol_state_update(event, cpu_id, timestamp, flags, + &data, NULL); +#if MONOTONIC + if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_STATE_UPDATE); +#endif /* MONOTONIC */ + +#if __arm64__ + thread->machine.energy_estimate_nj += data.energy_estimate_nj; +#endif +} + +void +machine_thread_going_on_core(thread_t new_thread, + int urgency, + uint64_t sched_latency, + uint64_t same_pri_latency, + uint64_t timestamp) +{ + + if (sched_perfcontrol_oncore == sched_perfcontrol_oncore_default) + return; + struct going_on_core on_core; + perfcontrol_state_t state = FIND_PERFCONTROL_STATE(new_thread); + + on_core.thread_id = new_thread->thread_id; + on_core.energy_estimate_nj = 0; + on_core.qos_class = proc_get_effective_thread_policy(new_thread, TASK_POLICY_QOS); + on_core.urgency = urgency; + on_core.is_32_bit = thread_is_64bit(new_thread) ? FALSE : TRUE; + on_core.is_kernel_thread = new_thread->task == kernel_task; + on_core.scheduling_latency = sched_latency; + on_core.start_time = timestamp; + on_core.scheduling_latency_at_same_basepri = same_pri_latency; + +#if MONOTONIC + uint64_t counters[MT_CORE_NFIXED]; + bool ctrs_enabled = perfcontrol_callout_counters_begin(counters); +#endif /* MONOTONIC */ + sched_perfcontrol_oncore(state, &on_core); +#if MONOTONIC + if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_ON_CORE); +#endif /* MONOTONIC */ + +#if __arm64__ + new_thread->machine.energy_estimate_nj += on_core.energy_estimate_nj; +#endif +} + +void +machine_thread_going_off_core(thread_t old_thread, boolean_t thread_terminating, uint64_t last_dispatch) +{ + if (sched_perfcontrol_offcore == sched_perfcontrol_offcore_default) + return; + struct going_off_core off_core; + perfcontrol_state_t state = FIND_PERFCONTROL_STATE(old_thread); + + off_core.thread_id = old_thread->thread_id; + off_core.energy_estimate_nj = 0; + off_core.end_time = last_dispatch; + +#if MONOTONIC + uint64_t counters[MT_CORE_NFIXED]; + bool ctrs_enabled = perfcontrol_callout_counters_begin(counters); +#endif /* MONOTONIC */ + sched_perfcontrol_offcore(state, &off_core, thread_terminating); +#if MONOTONIC + if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_OFF_CORE); +#endif /* MONOTONIC */ + +#if __arm64__ + old_thread->machine.energy_estimate_nj += off_core.energy_estimate_nj; +#endif +} + + +void +machine_max_runnable_latency(uint64_t bg_max_latency, + uint64_t default_max_latency, + uint64_t realtime_max_latency) +{ + if (sched_perfcontrol_max_runnable_latency == sched_perfcontrol_max_runnable_latency_default) + return; + struct perfcontrol_max_runnable_latency latencies = { + .max_scheduling_latencies = { + [THREAD_URGENCY_NONE] = 0, + [THREAD_URGENCY_BACKGROUND] = bg_max_latency, + [THREAD_URGENCY_NORMAL] = default_max_latency, + [THREAD_URGENCY_REAL_TIME] = realtime_max_latency + } + }; + + sched_perfcontrol_max_runnable_latency(&latencies); +} + +void +machine_work_interval_notify(thread_t thread, + struct kern_work_interval_args* kwi_args) +{ + if (sched_perfcontrol_work_interval_notify == sched_perfcontrol_work_interval_notify_default) + return; + perfcontrol_state_t state = FIND_PERFCONTROL_STATE(thread); + struct perfcontrol_work_interval work_interval = { + .thread_id = thread->thread_id, + .qos_class = proc_get_effective_thread_policy(thread, TASK_POLICY_QOS), + .urgency = kwi_args->urgency, + .flags = kwi_args->notify_flags, + .work_interval_id = kwi_args->work_interval_id, + .start = kwi_args->start, + .finish = kwi_args->finish, + .deadline = kwi_args->deadline, + .next_start = kwi_args->next_start, + .create_flags = kwi_args->create_flags, + }; + sched_perfcontrol_work_interval_notify(state, &work_interval); +} + +void +machine_perfcontrol_deadline_passed(uint64_t deadline) +{ + if (sched_perfcontrol_deadline_passed != sched_perfcontrol_deadline_passed_default) + sched_perfcontrol_deadline_passed(deadline); +} + +#if INTERRUPT_MASKED_DEBUG +/* + * ml_spin_debug_reset() + * Reset the timestamp on a thread that has been unscheduled + * to avoid false alarms. Alarm will go off if interrupts are held + * disabled for too long, starting from now. + */ +void +ml_spin_debug_reset(thread_t thread) +{ + thread->machine.intmask_timestamp = mach_absolute_time(); +} + +/* + * ml_spin_debug_clear() + * Clear the timestamp on a thread that has been unscheduled + * to avoid false alarms + */ +void +ml_spin_debug_clear(thread_t thread) +{ + thread->machine.intmask_timestamp = 0; +} + +/* + * ml_spin_debug_clear_self() + * Clear the timestamp on the current thread to prevent + * false alarms + */ +void +ml_spin_debug_clear_self() +{ + ml_spin_debug_clear(current_thread()); +} + +void +ml_check_interrupts_disabled_duration(thread_t thread) +{ + uint64_t start; + uint64_t now; + + start = thread->machine.intmask_timestamp; + if (start != 0) { + now = mach_absolute_time(); + + if ((now - start) > interrupt_masked_timeout) { + mach_timebase_info_data_t timebase; + clock_timebase_info(&timebase); + +#ifndef KASAN + /* + * Disable the actual panic for KASAN due to the overhead of KASAN itself, leave the rest of the + * mechanism enabled so that KASAN can catch any bugs in the mechanism itself. + */ + panic("Interrupts held disabled for %llu nanoseconds", (((now - start) * timebase.numer)/timebase.denom)); +#endif + } + } + + return; +} +#endif // INTERRUPT_MASKED_DEBUG + + +boolean_t +ml_set_interrupts_enabled(boolean_t enable) +{ + thread_t thread; + uint64_t state; + +#if __arm__ +#define INTERRUPT_MASK PSR_IRQF + state = __builtin_arm_rsr("cpsr"); +#else +#define INTERRUPT_MASK DAIF_IRQF + state = __builtin_arm_rsr("DAIF"); +#endif + if (enable) { +#if INTERRUPT_MASKED_DEBUG + if (interrupt_masked_debug && (state & INTERRUPT_MASK)) { + // Interrupts are currently masked, we will enable them (after finishing this check) + thread = current_thread(); + ml_check_interrupts_disabled_duration(thread); + thread->machine.intmask_timestamp = 0; + } +#endif // INTERRUPT_MASKED_DEBUG + if (get_preemption_level() == 0) { + thread = current_thread(); + while (thread->machine.CpuDatap->cpu_pending_ast & AST_URGENT) { +#if __ARM_USER_PROTECT__ + uintptr_t up = arm_user_protect_begin(thread); +#endif + ast_taken_kernel(); +#if __ARM_USER_PROTECT__ + arm_user_protect_end(thread, up, FALSE); +#endif + } + } +#if __arm__ + __asm__ volatile ("cpsie if" ::: "memory"); // Enable IRQ FIQ +#else + __builtin_arm_wsr("DAIFClr", (DAIFSC_IRQF | DAIFSC_FIQF)); +#endif + } else { +#if __arm__ + __asm__ volatile ("cpsid if" ::: "memory"); // Mask IRQ FIQ +#else + __builtin_arm_wsr("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF)); +#endif +#if INTERRUPT_MASKED_DEBUG + if (interrupt_masked_debug && ((state & INTERRUPT_MASK) == 0)) { + // Interrupts were enabled, we just masked them + current_thread()->machine.intmask_timestamp = mach_absolute_time(); + } +#endif + } + return ((state & INTERRUPT_MASK) == 0); +} + +static boolean_t ml_quiescing; + +void ml_set_is_quiescing(boolean_t quiescing) +{ + assert(FALSE == ml_get_interrupts_enabled()); + ml_quiescing = quiescing; +} + +boolean_t ml_is_quiescing(void) +{ + assert(FALSE == ml_get_interrupts_enabled()); + return (ml_quiescing); +} + +uint64_t ml_get_booter_memory_size(void) +{ + enum { kRoundSize = 512*1024*1024ULL }; + uint64_t size; + size = BootArgs->memSizeActual; + if (!size) + { + size = BootArgs->memSize; + size = (size + kRoundSize - 1) & ~(kRoundSize - 1); + size -= BootArgs->memSize; + } + return (size); +} + +uint64_t +ml_get_abstime_offset(void) +{ + return rtclock_base_abstime; +} + +uint64_t +ml_get_conttime_offset(void) +{ + return (rtclock_base_abstime + mach_absolutetime_asleep); +} + +uint64_t +ml_get_time_since_reset(void) +{ + /* The timebase resets across S2R, so just return the raw value. */ + return ml_get_hwclock(); +} + +uint64_t +ml_get_conttime_wake_time(void) +{ + /* The wake time is simply our continuous time offset. */ + return ml_get_conttime_offset(); +} + diff --git a/osfmk/arm/machine_task.c b/osfmk/arm/machine_task.c new file mode 100644 index 000000000..517f4fa1a --- /dev/null +++ b/osfmk/arm/machine_task.c @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#include +#include +#include + +extern zone_t ads_zone; + +kern_return_t +machine_task_set_state( + task_t task, + int flavor, + thread_state_t state, + mach_msg_type_number_t state_count) +{ + switch (flavor) { + case ARM_DEBUG_STATE: + { + arm_debug_state_t *tstate = (arm_debug_state_t *) state; + + if (state_count != ARM_DEBUG_STATE_COUNT) { + return KERN_INVALID_ARGUMENT; + } + + if (task->task_debug == NULL) { + task->task_debug = zalloc(ads_zone); + if (task->task_debug == NULL) + return KERN_FAILURE; + } + + copy_debug_state(tstate, (arm_debug_state_t*) task->task_debug, FALSE); + + return KERN_SUCCESS; + } + case THREAD_STATE_NONE: /* Using this flavor to clear task_debug */ + { + if (task->task_debug != NULL) { + zfree(ads_zone, task->task_debug); + task->task_debug = NULL; + + return KERN_SUCCESS; + } + return KERN_FAILURE; + } + default: + { + return KERN_INVALID_ARGUMENT; + } + } + + return KERN_FAILURE; +} + +kern_return_t +machine_task_get_state(task_t task, + int flavor, + thread_state_t state, + mach_msg_type_number_t *state_count) +{ + switch (flavor) { + case ARM_DEBUG_STATE: + { + arm_debug_state_t *tstate = (arm_debug_state_t *) state; + + if (*state_count != ARM_DEBUG_STATE_COUNT) { + return KERN_INVALID_ARGUMENT; + } + + if (task->task_debug == NULL) { + bzero(state, sizeof(*tstate)); + } else { + copy_debug_state((arm_debug_state_t*) task->task_debug, tstate, FALSE); /* FALSE OR TRUE doesn't matter since we are ignoring it for arm */ + } + + return KERN_SUCCESS; + } + default: + { + return KERN_INVALID_ARGUMENT; + } + + } + return KERN_FAILURE; +} + +void +machine_task_terminate(task_t task) +{ + if (task) { + void *task_debug; + + task_debug = task->task_debug; + if (task_debug != NULL) { + task->task_debug = NULL; + zfree(ads_zone, task_debug); + } + } +} + + +kern_return_t +machine_thread_inherit_taskwide( + thread_t thread, + task_t parent_task) +{ + if (parent_task->task_debug) { + int flavor; + mach_msg_type_number_t count; + + flavor = ARM_DEBUG_STATE; + count = ARM_DEBUG_STATE_COUNT; + + return machine_thread_set_state(thread, flavor, parent_task->task_debug, count); + } + + return KERN_SUCCESS; +} + + +void +machine_task_init(__unused task_t new_task, + __unused task_t parent_task, + __unused boolean_t memory_inherit) +{ +} diff --git a/osfmk/arm/machlimits.h b/osfmk/arm/machlimits.h new file mode 100644 index 000000000..0ab749b6b --- /dev/null +++ b/osfmk/arm/machlimits.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * HISTORY + * + * Revision 1.1.1.1 1998/09/22 21:05:41 wsanchez + * Import of Mac OS X kernel (~semeria) + * + * Revision 1.1.1.1 1998/03/07 02:26:02 wsanchez + * Import of OSF Mach kernel (~mburg) + * + * Revision 1.1.2.1 1996/12/09 16:55:05 stephen + * nmklinux_1.0b3_shared into pmk1.1 + * New file based on hp_pa + * [1996/12/09 11:09:22 stephen] + * + * $EndLog$ + */ +/* + * Copyright (c) 1988 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms are permitted + * provided that the above copyright notice and this paragraph are + * duplicated in all such forms and that any documentation, + * advertising materials, and other materials related to such + * distribution and use acknowledge that the software was developed + * by the University of California, Berkeley. The name of the + * University may not be used to endorse or promote products derived + * from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + * + * @(#)machlimits.h 7.1 (Berkeley) 2/15/89 + */ +#ifndef _MACH_MACHLIMITS_H_ +#define _MACH_MACHLIMITS_H_ + +#define CHAR_BIT 8 /* number of bits in a char */ + +#define SCHAR_MAX 127 /* max value for a signed char */ +#define SCHAR_MIN (-128) /* min value for a signed char */ + +#define UCHAR_MAX 255U /* max value for an unsigned char */ +#define CHAR_MAX 127 /* max value for a char */ +#define CHAR_MIN (-128) /* min value for a char */ + +#define USHRT_MAX 65535U /* max value for an unsigned short */ +#define SHRT_MAX 32767 /* max value for a short */ +#define SHRT_MIN (-32768) /* min value for a short */ + +#define UINT_MAX 0xFFFFFFFFU /* max value for an unsigned int */ +#define INT_MAX 2147483647 /* max value for an int */ +#define INT_MIN (-2147483647-1) /* min value for an int */ + +#ifdef __LP64__ +#define ULONG_MAX 0xffffffffffffffffUL /* max unsigned long */ +#define LONG_MAX 0x7fffffffffffffffL /* max signed long */ +#define LONG_MIN (-0x7fffffffffffffffL-1)/* min signed long */ +#else /* !__LP64__ */ +#define ULONG_MAX 0xffffffffUL /* max value for an unsigned long */ +#define LONG_MAX 2147483647L /* max value for a long */ +#define LONG_MIN (-2147483647L-1) /* min value for a long */ +#endif /* __LP64__ */ + +/* Must be at least two, for internationalization (NLS/KJI) */ +#define MB_LEN_MAX 4 /* multibyte characters */ + +#endif /* _MACH_MACHLIMITS_H_ */ diff --git a/osfmk/arm/machparam.h b/osfmk/arm/machparam.h new file mode 100644 index 000000000..4d5ec30de --- /dev/null +++ b/osfmk/arm/machparam.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +/* + * Machine-dependent SPL definitions. + * + * SPLs are true functions on i386, defined elsewhere. + */ + diff --git a/osfmk/arm/misc_protos.h b/osfmk/arm/misc_protos.h new file mode 100644 index 000000000..416bcb2a3 --- /dev/null +++ b/osfmk/arm/misc_protos.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#ifndef _ARM_MISC_PROTOS_H_ +#define _ARM_MISC_PROTOS_H_ + +#include + +extern processor_t cpu_processor_alloc(boolean_t is_boot_cpu); +extern void cpu_processor_free(processor_t proc); + +extern void machine_startup(__unused boot_args *args) __attribute__((noinline)); +extern void machine_lockdown_preflight(void); +extern void machine_lockdown(void); +extern void arm_vm_init(uint64_t memory_size, boot_args *args); +extern void arm_vm_prot_init(boot_args *args); +extern void arm_vm_prot_finalize(boot_args *args); + + +extern kern_return_t DebuggerXCallEnter(boolean_t); +extern void DebuggerXCallReturn(void); + +#if __arm64__ && DEBUG +extern void dump_kva_space(void); +#endif + +extern void Load_context(thread_t); +extern void Idle_load_context(void) __attribute__((noreturn)); +extern thread_t Switch_context(thread_t, thread_continue_t, thread_t); +extern thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor); +extern void Call_continuation(thread_continue_t, void *, wait_result_t, vm_offset_t); + +extern void DebuggerCall(unsigned int reason, void *ctx); +extern void DebuggerXCall(void *ctx); + +extern int _copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t max, vm_size_t *actual); +extern int copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes); +extern int copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes); + +extern void bcopy_phys(addr64_t from, addr64_t to, vm_size_t nbytes); + +extern void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count, unsigned int remaining, unsigned int *res); +extern void dcache_incoherent_io_store64(addr64_t pa, unsigned int count, unsigned int remaining, unsigned int *res); + +#if defined(__arm__) +extern void copy_debug_state(arm_debug_state_t *src, arm_debug_state_t *target, __unused boolean_t all); +#elif defined(__arm64__) +extern void copy_legacy_debug_state(arm_legacy_debug_state_t *src, arm_legacy_debug_state_t *target, __unused boolean_t all); +extern void copy_debug_state32(arm_debug_state32_t *src, arm_debug_state32_t *target, __unused boolean_t all); +extern void copy_debug_state64(arm_debug_state64_t *src, arm_debug_state64_t *target, __unused boolean_t all); + +extern boolean_t debug_legacy_state_is_valid(arm_legacy_debug_state_t *ds); +extern boolean_t debug_state_is_valid32(arm_debug_state32_t *ds); +extern boolean_t debug_state_is_valid64(arm_debug_state64_t *ds); + +extern int copyio_check_user_addr(user_addr_t user_addr, vm_size_t nbytes); +extern int _emulate_swp(user_addr_t addr, uint32_t newval, uint32_t *oldval); +extern int _emulate_swpb(user_addr_t addr, uint8_t newval, uint32_t *oldval); + +/* Top-Byte-Ignore */ +extern boolean_t user_tbi; +#define TBI_MASK 0xff00000000000000 +#define user_tbi_enabled() (user_tbi) +#define tbi_clear(addr) ((addr) & ~(TBI_MASK)) + +#else +#error Unknown architecture. +#endif + +#endif /* _ARM_MISC_PROTOS_H_ */ diff --git a/osfmk/arm/model_dep.c b/osfmk/arm/model_dep.c new file mode 100644 index 000000000..ab930d549 --- /dev/null +++ b/osfmk/arm/model_dep.c @@ -0,0 +1,868 @@ +/* + * Copyright (c) 2007-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#if ALTERNATE_DEBUGGER +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include /* for btop */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#if MACH_KDP +void kdp_trap(unsigned int, struct arm_saved_state *); +#endif + +extern kern_return_t do_stackshot(void *); +extern void kdp_snapshot_preflight(int pid, void *tracebuf, + uint32_t tracebuf_size, uint32_t flags, + kcdata_descriptor_t data_p, + boolean_t enable_faulting); +extern int kdp_stack_snapshot_bytes_traced(void); + +/* + * Increment the PANICLOG_VERSION if you change the format of the panic + * log in any way. + */ +#define PANICLOG_VERSION 8 +static struct kcdata_descriptor kc_panic_data; + +extern char firmware_version[]; +extern volatile uint32_t debug_enabled; +extern unsigned int not_in_kdp; + +extern int copyinframe(vm_address_t fp, uint32_t * frame); +extern void kdp_callouts(kdp_event_t event); + +/* #include */ +#define MAXCOMLEN 16 +extern int proc_pid(void *p); +extern void proc_name_kdp(task_t, char *, int); + +extern const char version[]; +extern char osversion[]; +extern uint8_t gPlatformECID[8]; +extern uint32_t gPlatformMemoryID; + +extern uint64_t last_hwaccess_thread; + +/*Choosing the size for gTargetTypeBuffer as 8 and size for gModelTypeBuffer as 32 + since the target name and model name typically doesn't exceed this size */ +extern char gTargetTypeBuffer[8]; +extern char gModelTypeBuffer[32]; + +decl_simple_lock_data(extern,clock_lock) +extern struct timeval gIOLastSleepTime; +extern struct timeval gIOLastWakeTime; +extern boolean_t is_clock_configured; +extern uuid_t kernelcache_uuid; + +/* Definitions for frame pointers */ +#define FP_ALIGNMENT_MASK ((uint32_t)(0x3)) +#define FP_LR_OFFSET ((uint32_t)4) +#define FP_LR_OFFSET64 ((uint32_t)8) +#define FP_MAX_NUM_TO_EVALUATE (50) + +/* Timeout (in nanoseconds) for all processors responding to debug crosscall */ +#define DEBUG_ACK_TIMEOUT ((uint64_t) 10000000) + +/* Forward functions definitions */ +void panic_display_times(void) ; +void panic_print_symbol_name(vm_address_t search); + + +/* Global variables */ +static uint32_t panic_bt_depth; +boolean_t PanicInfoSaved = FALSE; +boolean_t force_immediate_debug_halt = FALSE; +unsigned int debug_ack_timeout_count = 0; +volatile unsigned int debugger_sync = 0; +volatile unsigned int mp_kdp_trap = 0; /* CPUs signalled by the debug CPU will spin on this */ +unsigned int DebugContextCount = 0; + +// Convenient macros to easily validate one or more pointers if +// they have defined types +#define VALIDATE_PTR(ptr) \ + validate_ptr((vm_offset_t)(ptr), sizeof(*(ptr)), #ptr) + +#define VALIDATE_PTR_2(ptr0, ptr1) \ + VALIDATE_PTR(ptr0) && VALIDATE_PTR(ptr1) + +#define VALIDATE_PTR_3(ptr0, ptr1, ptr2) \ + VALIDATE_PTR_2(ptr0, ptr1) && VALIDATE_PTR(ptr2) + +#define VALIDATE_PTR_4(ptr0, ptr1, ptr2, ptr3) \ + VALIDATE_PTR_2(ptr0, ptr1) && VALIDATE_PTR_2(ptr2, ptr3) + +#define GET_MACRO(_1,_2,_3,_4,NAME,...) NAME + +#define VALIDATE_PTR_LIST(...) GET_MACRO(__VA_ARGS__, VALIDATE_PTR_4, VALIDATE_PTR_3, VALIDATE_PTR_2, VALIDATE_PTR)(__VA_ARGS__) + +/* + * Evaluate if a pointer is valid + * Print a message if pointer is invalid + */ +static boolean_t validate_ptr( + vm_offset_t ptr, vm_size_t size, const char * ptr_name) +{ + if (ptr) { + if (ml_validate_nofault(ptr, size)) { + return TRUE; + } else { + paniclog_append_noflush("Invalid %s pointer: %p size: %d\n", + ptr_name, (void *)ptr, (int)size); + return FALSE; + } + } else { + paniclog_append_noflush("NULL %s pointer\n", ptr_name); + return FALSE; + } +} + +/* + * Backtrace a single frame. + */ +static void +print_one_backtrace(pmap_t pmap, vm_offset_t topfp, const char *cur_marker, + boolean_t is_64_bit) +{ + int i = 0; + addr64_t lr; + addr64_t fp; + addr64_t fp_for_ppn; + ppnum_t ppn; + boolean_t dump_kernel_stack; + + fp = topfp; + fp_for_ppn = 0; + ppn = (ppnum_t)NULL; + + if (fp >= VM_MIN_KERNEL_ADDRESS) + dump_kernel_stack = TRUE; + else + dump_kernel_stack = FALSE; + + do { + if ((fp == 0) || ((fp & FP_ALIGNMENT_MASK) != 0)) + break; + if (dump_kernel_stack && ((fp < VM_MIN_KERNEL_ADDRESS) || (fp > VM_MAX_KERNEL_ADDRESS))) + break; + if ((!dump_kernel_stack) && (fp >=VM_MIN_KERNEL_ADDRESS)) + break; + + /* + * Check to see if current address will result in a different + * ppn than previously computed (to avoid recomputation) via + * (addr) ^ fp_for_ppn) >> PAGE_SHIFT) + */ + if ((((fp + FP_LR_OFFSET) ^ fp_for_ppn) >> PAGE_SHIFT) != 0x0U) { + ppn = pmap_find_phys(pmap, fp + FP_LR_OFFSET); + fp_for_ppn = fp + (is_64_bit ? FP_LR_OFFSET64 : FP_LR_OFFSET); + } + if (ppn != (ppnum_t)NULL) { + if (is_64_bit) { + lr = ml_phys_read_double_64(((((vm_offset_t)ppn) << PAGE_SHIFT)) | ((fp + FP_LR_OFFSET64) & PAGE_MASK)); + } else { + lr = ml_phys_read_word(((((vm_offset_t)ppn) << PAGE_SHIFT)) | ((fp + FP_LR_OFFSET) & PAGE_MASK)); + } + } else { + if (is_64_bit) { + paniclog_append_noflush("%s\t Could not read LR from frame at 0x%016llx\n", cur_marker, fp + FP_LR_OFFSET64); + } else { + paniclog_append_noflush("%s\t Could not read LR from frame at 0x%08x\n", cur_marker, (uint32_t)(fp + FP_LR_OFFSET)); + } + break; + } + if (((fp ^ fp_for_ppn) >> PAGE_SHIFT) != 0x0U) { + ppn = pmap_find_phys(pmap, fp); + fp_for_ppn = fp; + } + if (ppn != (ppnum_t)NULL) { + if (is_64_bit) { + fp = ml_phys_read_double_64(((((vm_offset_t)ppn) << PAGE_SHIFT)) | (fp & PAGE_MASK)); + } else { + fp = ml_phys_read_word(((((vm_offset_t)ppn) << PAGE_SHIFT)) | (fp & PAGE_MASK)); + } + } else { + if (is_64_bit) { + paniclog_append_noflush("%s\t Could not read FP from frame at 0x%016llx\n", cur_marker, fp); + } else { + paniclog_append_noflush("%s\t Could not read FP from frame at 0x%08x\n", cur_marker, (uint32_t)fp); + } + break; + } + + if (lr) { + if (is_64_bit) { + paniclog_append_noflush("%s\t lr: 0x%016llx fp: 0x%016llx\n", cur_marker, lr, fp); + } else { + paniclog_append_noflush("%s\t lr: 0x%08x fp: 0x%08x\n", cur_marker, (uint32_t)lr, (uint32_t)fp); + } + } + } while ((++i < FP_MAX_NUM_TO_EVALUATE) && (fp != topfp)); +} + +#define SANE_TASK_LIMIT 256 +#define TOP_RUNNABLE_LIMIT 5 +#define PANICLOG_UUID_BUF_SIZE 256 + +extern void panic_print_vnodes(void); + +static void +do_print_all_backtraces( + const char *message) +{ + int logversion = PANICLOG_VERSION; + thread_t cur_thread = current_thread(); + uintptr_t cur_fp; + task_t task; + int i; + size_t index; + int print_vnodes = 0; + const char *nohilite_thread_marker="\t"; + + /* end_marker_bytes set to 200 for printing END marker + stackshot summary info always */ + int bytes_traced = 0, bytes_remaining = 0, end_marker_bytes = 200; + uint64_t bytes_used = 0ULL; + int err = 0; + char *stackshot_begin_loc = NULL; + +#if defined(__arm__) + __asm__ volatile("mov %0, r7":"=r"(cur_fp)); +#elif defined(__arm64__) + __asm__ volatile("add %0, xzr, fp":"=r"(cur_fp)); +#else +#error Unknown architecture. +#endif + if (panic_bt_depth != 0) + return; + panic_bt_depth++; + + /* Truncate panic string to 1200 bytes -- WDT log can be ~1100 bytes */ + paniclog_append_noflush("Debugger message: %.1200s\n", message); + if (debug_enabled) { + paniclog_append_noflush("Device: %s\n", + ('\0' != gTargetTypeBuffer[0]) ? gTargetTypeBuffer : "Not set yet"); + paniclog_append_noflush("Hardware Model: %s\n", + ('\0' != gModelTypeBuffer[0]) ? gModelTypeBuffer:"Not set yet"); + paniclog_append_noflush("ECID: %02X%02X%02X%02X%02X%02X%02X%02X\n", gPlatformECID[7], + gPlatformECID[6], gPlatformECID[5], gPlatformECID[4], gPlatformECID[3], + gPlatformECID[2], gPlatformECID[1], gPlatformECID[0]); + if (last_hwaccess_thread) { + paniclog_append_noflush("AppleHWAccess Thread: 0x%llx\n", last_hwaccess_thread); + } + } + paniclog_append_noflush("Memory ID: 0x%x\n", gPlatformMemoryID); + paniclog_append_noflush("OS version: %.256s\n", + ('\0' != osversion[0]) ? osversion : "Not set yet"); + paniclog_append_noflush("Kernel version: %.512s\n", version); + paniclog_append_noflush("KernelCache UUID: "); + for (index = 0; index < sizeof(uuid_t); index++) { + paniclog_append_noflush("%02X", kernelcache_uuid[index]); + } + paniclog_append_noflush("\n"); + + paniclog_append_noflush("iBoot version: %.128s\n", firmware_version); + paniclog_append_noflush("secure boot?: %s\n", debug_enabled ? "NO": "YES"); + paniclog_append_noflush("Paniclog version: %d\n", logversion); + + panic_display_kernel_aslr(); + panic_display_times(); + panic_display_zprint(); +#if CONFIG_ZLEAKS + panic_display_ztrace(); +#endif /* CONFIG_ZLEAKS */ +#if CONFIG_ECC_LOGGING + panic_display_ecc_errors(); +#endif /* CONFIG_ECC_LOGGING */ + + // Just print threads with high CPU usage for WDT timeouts + if (strncmp(message, "WDT timeout", 11) == 0) { + thread_t top_runnable[5] = {0}; + thread_t thread; + int total_cpu_usage = 0; + + print_vnodes = 1; + + + for (thread = (thread_t)queue_first(&threads); + VALIDATE_PTR(thread) && !queue_end(&threads, (queue_entry_t)thread); + thread = (thread_t)queue_next(&thread->threads)) { + + total_cpu_usage += thread->cpu_usage; + + // Look for the 5 runnable threads with highest priority + if (thread->state & TH_RUN) { + int k; + thread_t comparison_thread = thread; + + for (k = 0; k < TOP_RUNNABLE_LIMIT; k++) { + if (top_runnable[k] == 0) { + top_runnable[k] = comparison_thread; + break; + } else if (comparison_thread->sched_pri > top_runnable[k]->sched_pri) { + thread_t temp = top_runnable[k]; + top_runnable[k] = comparison_thread; + comparison_thread = temp; + } // if comparison thread has higher priority than previously saved thread + } // loop through highest priority runnable threads + } // Check if thread is runnable + } // Loop through all threads + + // Print the relevant info for each thread identified + paniclog_append_noflush("Total cpu_usage: %d\n", total_cpu_usage); + paniclog_append_noflush("Thread task pri cpu_usage\n"); + + for (i = 0; i < TOP_RUNNABLE_LIMIT; i++) { + + if (top_runnable[i] && VALIDATE_PTR(top_runnable[i]->task) && + validate_ptr((vm_offset_t)top_runnable[i]->task->bsd_info, 1, "bsd_info")) { + + char name[MAXCOMLEN + 1]; + proc_name_kdp(top_runnable[i]->task, name, sizeof(name)); + paniclog_append_noflush("%p %s %d %d\n", + top_runnable[i], name, top_runnable[i]->sched_pri, top_runnable[i]->cpu_usage); + } + } // Loop through highest priority runnable threads + paniclog_append_noflush("\n"); + } // Check if message is "WDT timeout" + + // print current task info + if (VALIDATE_PTR_LIST(cur_thread, cur_thread->task)) { + + task = cur_thread->task; + + if (VALIDATE_PTR_LIST(task->map, task->map->pmap)) { + paniclog_append_noflush("Panicked task %p: %d pages, %d threads: ", + task, task->map->pmap->stats.resident_count, task->thread_count); + } else { + paniclog_append_noflush("Panicked task %p: %d threads: ", + task, task->thread_count); + } + + if (validate_ptr((vm_offset_t)task->bsd_info, 1, "bsd_info")) { + char name[MAXCOMLEN + 1]; + int pid = proc_pid(task->bsd_info); + proc_name_kdp(task, name, sizeof(name)); + paniclog_append_noflush("pid %d: %s", pid, name); + } else { + paniclog_append_noflush("unknown task"); + } + + paniclog_append_noflush("\n"); + } + + if (cur_fp < VM_MAX_KERNEL_ADDRESS) { + paniclog_append_noflush("Panicked thread: %p, backtrace: 0x%llx, tid: %llu\n", + cur_thread, (addr64_t)cur_fp, thread_tid(cur_thread)); +#if __LP64__ + print_one_backtrace(kernel_pmap, cur_fp, nohilite_thread_marker, TRUE); +#else + print_one_backtrace(kernel_pmap, cur_fp, nohilite_thread_marker, FALSE); +#endif + } else { + paniclog_append_noflush("Could not print panicked thread backtrace:" + "frame pointer outside kernel vm.\n"); + } + + paniclog_append_noflush("\n"); + panic_info->eph_panic_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_panic_log_offset; + + if (debug_ack_timeout_count) { + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_DEBUGGERSYNC; + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("!! debugger synchronization failed, no stackshot !!\n"); + } else if (stackshot_active()) { + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_NESTED; + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("!! panicked during stackshot, skipping panic stackshot !!\n"); + } else { + /* Align the stackshot buffer to an 8-byte address (especially important for armv7k devices) */ + debug_buf_ptr += (8 - ((uintptr_t)debug_buf_ptr % 8)); + stackshot_begin_loc = debug_buf_ptr; + + bytes_remaining = debug_buf_size - (unsigned int)((uintptr_t)stackshot_begin_loc - (uintptr_t)debug_buf_base); + err = kcdata_memory_static_init(&kc_panic_data, (mach_vm_address_t)debug_buf_ptr, + KCDATA_BUFFER_BEGIN_STACKSHOT, bytes_remaining - end_marker_bytes, + KCFLAG_USE_MEMCOPY); + if (err == KERN_SUCCESS) { + kdp_snapshot_preflight(-1, stackshot_begin_loc, bytes_remaining - end_marker_bytes, + (STACKSHOT_GET_GLOBAL_MEM_STATS | STACKSHOT_SAVE_LOADINFO | STACKSHOT_KCDATA_FORMAT | + STACKSHOT_ENABLE_BT_FAULTING | STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_FROM_PANIC | + STACKSHOT_NO_IO_STATS | STACKSHOT_THREAD_WAITINFO), &kc_panic_data, 0); + err = do_stackshot(NULL); + bytes_traced = kdp_stack_snapshot_bytes_traced(); + if (bytes_traced > 0 && !err) { + debug_buf_ptr += bytes_traced; + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_SUCCEEDED; + panic_info->eph_stackshot_offset = PE_get_offset_into_panic_region(stackshot_begin_loc); + panic_info->eph_stackshot_len = bytes_traced; + + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("\n** Stackshot Succeeded ** Bytes Traced %d **\n", bytes_traced); + } else { + bytes_used = kcdata_memory_get_used_bytes(&kc_panic_data); + if (bytes_used > 0) { + /* Zero out the stackshot data */ + bzero(stackshot_begin_loc, bytes_used); + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_INCOMPLETE; + + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("\n** Stackshot Incomplete ** Bytes Filled %llu **\n", bytes_used); + } else { + bzero(stackshot_begin_loc, bytes_used); + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR; + + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("\n!! Stackshot Failed !! Bytes Traced %d, err %d\n", bytes_traced, err); + } + } + } else { + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR; + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("\n!! Stackshot Failed !!\nkcdata_memory_static_init returned %d", err); + } + } + + assert(panic_info->eph_other_log_offset != 0); + + if (print_vnodes != 0) + panic_print_vnodes(); + + panic_bt_depth--; +} + +/* + * Entry to print_all_backtraces is serialized by the debugger lock + */ +static void +print_all_backtraces(const char *message) +{ + unsigned int initial_not_in_kdp = not_in_kdp; + + cpu_data_t * cpu_data_ptr = getCpuDatap(); + + assert(cpu_data_ptr->PAB_active == FALSE); + cpu_data_ptr->PAB_active = TRUE; + + /* + * Because print all backtraces uses the pmap routines, it needs to + * avoid taking pmap locks. Right now, this is conditionalized on + * not_in_kdp. + */ + not_in_kdp = 0; + do_print_all_backtraces(message); + + not_in_kdp = initial_not_in_kdp; + + cpu_data_ptr->PAB_active = FALSE; +} + +void +panic_display_times() +{ + if (kdp_clock_is_locked()) { + paniclog_append_noflush("Warning: clock is locked. Can't get time\n"); + return; + } + + if ((is_clock_configured) && (simple_lock_try(&clock_lock))) { + clock_sec_t secs, boot_secs; + clock_usec_t usecs, boot_usecs; + + simple_unlock(&clock_lock); + + clock_get_calendar_microtime(&secs, &usecs); + clock_get_boottime_microtime(&boot_secs, &boot_usecs); + + paniclog_append_noflush("Epoch Time: sec usec\n"); + paniclog_append_noflush(" Boot : 0x%08x 0x%08x\n", (unsigned int)boot_secs, (unsigned int)boot_usecs); + paniclog_append_noflush(" Sleep : 0x%08x 0x%08x\n", (unsigned int)gIOLastSleepTime.tv_sec, (unsigned int)gIOLastSleepTime.tv_usec); + paniclog_append_noflush(" Wake : 0x%08x 0x%08x\n", (unsigned int)gIOLastWakeTime.tv_sec, (unsigned int)gIOLastWakeTime.tv_usec); + paniclog_append_noflush(" Calendar: 0x%08x 0x%08x\n\n", (unsigned int)secs, (unsigned int)usecs); + } +} + +void panic_print_symbol_name(vm_address_t search) +{ +#pragma unused(search) + // empty stub. Really only used on x86_64. + return; +} + +void +SavePanicInfo( + const char *message, __unused uint64_t panic_options) +{ + + /* This should be initialized by the time we get here */ + assert(panic_info->eph_panic_log_offset != 0); + + if (panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) { + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_BUTTON_RESET_PANIC; + } + + if (panic_options & DEBUGGER_OPTION_COPROC_INITIATED_PANIC) { + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC; + } + + /* + * On newer targets, panic data is stored directly into the iBoot panic region. + * If we re-enter SavePanicInfo (e.g. on a double panic) on such a target, update the + * panic CRC so that iBoot can hopefully find *something* useful in the panic region. + */ + if (PanicInfoSaved && (debug_buf_base >= (char*)gPanicBase) && (debug_buf_base < (char*)gPanicBase + gPanicSize)) { + unsigned int pi_size = (unsigned int)(debug_buf_ptr - gPanicBase); + PE_save_buffer_to_vram((unsigned char*)gPanicBase, &pi_size); + PE_sync_panic_buffers(); // extra precaution; panic path likely isn't reliable if we're here + } + + if (PanicInfoSaved || (debug_buf_size == 0)) + return; + + PanicInfoSaved = TRUE; + + print_all_backtraces(message); + + assert(panic_info->eph_panic_log_len != 0); + panic_info->eph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_other_log_offset; + + PEHaltRestart(kPEPanicSync); + + /* + * Notifies registered IOPlatformPanicAction callbacks + * (which includes one to disable the memcache) and flushes + * the buffer contents from the cache + */ + paniclog_flush(); +} + +void +paniclog_flush() +{ + unsigned int panicbuf_length = 0; + + panicbuf_length = (unsigned int)(debug_buf_ptr - gPanicBase); + if (!panicbuf_length) + return; + + /* + * Updates the log length of the last part of the panic log. + */ + panic_info->eph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_other_log_offset; + + /* + * Updates the metadata at the beginning of the panic buffer, + * updates the CRC. + */ + PE_save_buffer_to_vram((unsigned char *)gPanicBase, &panicbuf_length); + + /* + * This is currently unused by platform KEXTs on embedded but is + * kept for compatibility with the published IOKit interfaces. + */ + PESavePanicInfo((unsigned char *)gPanicBase, panicbuf_length); + + PE_sync_panic_buffers(); +} + +/* + * @function DebuggerXCallEnter + * + * @abstract IPI other cores so this core can run in a single-threaded context. + * + * @discussion This function should be called with the debugger lock held. It + * signals the other cores to go into a busy loop so this core can run in a + * single-threaded context and inspect kernel memory. + * + * @param proceed_on_sync_failure If true, then go ahead and try to debug even + * if we can't synch with the other cores. This is inherently unsafe and should + * only be used if the kernel is going down in flames anyway. + * + * @result returns KERN_OPERATION_TIMED_OUT if synchronization times out and + * proceed_on_sync_failure is false. + */ +kern_return_t +DebuggerXCallEnter( + boolean_t proceed_on_sync_failure) +{ + uint64_t max_mabs_time, current_mabs_time; + int cpu; + int max_cpu; + cpu_data_t *target_cpu_datap; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + /* Check for nested debugger entry. */ + cpu_data_ptr->debugger_active++; + if (cpu_data_ptr->debugger_active != 1) + return KERN_SUCCESS; + + /* + * If debugger_sync is not 0, someone responded excessively late to the last + * debug request (we zero the sync variable in the return function). Zero it + * again here. This should prevent us from getting out of sync (heh) and + * timing out on every entry to the debugger if we timeout once. + */ + + debugger_sync = 0; + mp_kdp_trap = 1; + + /* + * We need a barrier here to ensure CPUs see mp_kdp_trap and spin when responding + * to the signal. + */ + __builtin_arm_dmb(DMB_ISH); + + /* + * Try to signal all CPUs (except ourselves, of course). Use debugger_sync to + * synchronize with every CPU that we appeared to signal successfully (cpu_signal + * is not synchronous). + */ + bool cpu_signal_failed = false; + max_cpu = ml_get_max_cpu_number(); + + boolean_t immediate_halt = FALSE; + if (proceed_on_sync_failure && force_immediate_debug_halt) + immediate_halt = TRUE; + + if (!immediate_halt) { + for (cpu=0; cpu <= max_cpu; cpu++) { + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) + continue; + + if(KERN_SUCCESS == cpu_signal(target_cpu_datap, SIGPdebug, (void *)NULL, NULL)) { + (void)hw_atomic_add(&debugger_sync, 1); + } else { + cpu_signal_failed = true; + kprintf("cpu_signal failed in DebuggerXCallEnter\n"); + } + } + + nanoseconds_to_absolutetime(DEBUG_ACK_TIMEOUT, &max_mabs_time); + current_mabs_time = mach_absolute_time(); + max_mabs_time += current_mabs_time; + assert(max_mabs_time > current_mabs_time); + + /* + * Wait for DEBUG_ACK_TIMEOUT ns for a response from everyone we IPI'd. If we + * timeout, that is simply too bad; we don't have a true NMI, and one CPU may be + * uninterruptibly spinning on someone else. The best we can hope for is that + * all other CPUs have either responded or are spinning in a context that is + * debugger safe. + */ + while ((debugger_sync != 0) && (current_mabs_time < max_mabs_time)) + current_mabs_time = mach_absolute_time(); + + } + + if (cpu_signal_failed && !proceed_on_sync_failure) { + DebuggerXCallReturn(); + return KERN_FAILURE; + } else if (immediate_halt || (current_mabs_time >= max_mabs_time)) { + /* + * For the moment, we're aiming for a timeout that the user shouldn't notice, + * but will be sufficient to let the other core respond. + */ + __builtin_arm_dmb(DMB_ISH); + for (cpu=0; cpu <= max_cpu; cpu++) { + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) + continue; + if (!(target_cpu_datap->cpu_signal & SIGPdebug) && !immediate_halt) + continue; + if (proceed_on_sync_failure) { + paniclog_append_noflush("Attempting to forcibly halt cpu %d\n", cpu); + dbgwrap_status_t halt_status = ml_dbgwrap_halt_cpu(cpu, 0); + if (halt_status < 0) + paniclog_append_noflush("Unable to halt cpu %d: %d\n", cpu, halt_status); + else { + if (halt_status > 0) + paniclog_append_noflush("cpu %d halted with warning %d\n", cpu, halt_status); + target_cpu_datap->halt_status = CPU_HALTED; + } + } else + kprintf("Debugger synch pending on cpu %d\n", cpu); + } + if (proceed_on_sync_failure) { + for (cpu = 0; cpu <= max_cpu; cpu++) { + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr) || + (target_cpu_datap->halt_status == CPU_NOT_HALTED)) + continue; + dbgwrap_status_t halt_status = ml_dbgwrap_halt_cpu_with_state(cpu, + NSEC_PER_SEC, &target_cpu_datap->halt_state); + if ((halt_status < 0) || (halt_status == DBGWRAP_WARN_CPU_OFFLINE)) + paniclog_append_noflush("Unable to obtain state for cpu %d: %d\n", cpu, halt_status); + else + target_cpu_datap->halt_status = CPU_HALTED_WITH_STATE; + } + if (immediate_halt) + paniclog_append_noflush("Immediate halt requested on all cores\n"); + else + paniclog_append_noflush("Debugger synchronization timed out; waited %llu nanoseconds\n", DEBUG_ACK_TIMEOUT); + debug_ack_timeout_count++; + return KERN_SUCCESS; + } else { + DebuggerXCallReturn(); + return KERN_OPERATION_TIMED_OUT; + } + } else { + return KERN_SUCCESS; + } +} + +/* + * @function DebuggerXCallReturn + * + * @abstract Resume normal multicore operation after DebuggerXCallEnter() + * + * @discussion This function should be called with debugger lock held. + */ +void +DebuggerXCallReturn( + void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + cpu_data_ptr->debugger_active--; + if (cpu_data_ptr->debugger_active != 0) + return; + + mp_kdp_trap = 0; + debugger_sync = 0; + + /* Do we need a barrier here? */ + __builtin_arm_dmb(DMB_ISH); +} + +void +DebuggerXCall( + void *ctx) +{ + boolean_t save_context = FALSE; + vm_offset_t kstackptr = 0; + arm_saved_state_t *regs = (arm_saved_state_t *) ctx; + + if (regs != NULL) { +#if defined(__arm64__) + save_context = PSR64_IS_KERNEL(get_saved_state_cpsr(regs)); +#else + save_context = PSR_IS_KERNEL(regs->cpsr); +#endif + } + + kstackptr = current_thread()->machine.kstackptr; + arm_saved_state_t *state = (arm_saved_state_t *)kstackptr; + + if (save_context) { + /* Save the interrupted context before acknowledging the signal */ + *state = *regs; + } else if (regs) { + /* zero old state so machine_trace_thread knows not to backtrace it */ + set_saved_state_fp(state, 0); + set_saved_state_pc(state, 0); + set_saved_state_lr(state, 0); + set_saved_state_sp(state, 0); + } + + (void)hw_atomic_sub(&debugger_sync, 1); + __builtin_arm_dmb(DMB_ISH); + while (mp_kdp_trap); + + /* Any cleanup for our pushed context should go here */ +} + + +void +DebuggerCall( + unsigned int reason, + void *ctx) +{ +#if !MACH_KDP +#pragma unused(reason,ctx) +#endif /* !MACH_KDP */ + +#if ALTERNATE_DEBUGGER + alternate_debugger_enter(); +#endif + +#if MACH_KDP + kdp_trap(reason, (struct arm_saved_state *)ctx); +#else + /* TODO: decide what to do if no debugger config */ +#endif +} + + diff --git a/osfmk/arm/monotonic.h b/osfmk/arm/monotonic.h new file mode 100644 index 000000000..9b638bfbe --- /dev/null +++ b/osfmk/arm/monotonic.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef ARM_MONOTONIC_H +#define ARM_MONOTONIC_H + +#define MT_NDEVS 0 + +#define MT_CORE_NFIXED 1 +#define MT_CORE_CYCLES 0 +#define MT_CORE_MAXVAL UINT32_MAX + +#endif /* !defined(ARM_MONOTONIC_H) */ diff --git a/osfmk/arm/monotonic_arm.c b/osfmk/arm/monotonic_arm.c new file mode 100644 index 000000000..a5b071631 --- /dev/null +++ b/osfmk/arm/monotonic_arm.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include + +bool mt_core_supported = false; + +void +mt_init(void) +{ +} + +uint64_t +mt_core_snap(__unused unsigned int ctr) +{ + return 0; +} + +struct mt_cpu * +mt_cur_cpu(void) +{ + return &getCpuDatap()->cpu_monotonic; +} + +const struct monotonic_dev monotonic_devs[0]; diff --git a/osfmk/arm/pal_routines.c b/osfmk/arm/pal_routines.c new file mode 100644 index 000000000..535e3cc80 --- /dev/null +++ b/osfmk/arm/pal_routines.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2009 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + * file: pal_routines.c + * Platform Abstraction Layer routines for ARM + */ + + +#include +#include +#include + +/* Serial routines */ +int +pal_serial_init(void) +{ + return serial_init(); +} + +void +pal_serial_putc_nocr(char c) +{ + serial_putc(c); +} + +void +pal_serial_putc(char c) +{ + serial_putc(c); +} + +int +pal_serial_getc(void) +{ + return serial_getc(); +} diff --git a/osfmk/arm/pal_routines.h b/osfmk/arm/pal_routines.h new file mode 100644 index 000000000..f100f6e99 --- /dev/null +++ b/osfmk/arm/pal_routines.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2009 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_PAL_ROUTINES_H +#define _ARM_PAL_ROUTINES_H + +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +#ifdef XNU_KERNEL_PRIVATE + +/* No-op */ +#define pal_dbg_set_task_name( x ) do { } while(0) + +#define pal_ast_check(thread) +#define pal_thread_terminate_self(t) + +/* serial / debug output routines */ +extern int pal_serial_init(void); +extern void pal_serial_putc(char a); +extern void pal_serial_putc_nocr(char a); +extern int pal_serial_getc(void); + +#define panic_display_pal_info() do { } while(0) +#define pal_kernel_announce() do { } while(0) + +#endif /* XNU_KERNEL_PRIVATE */ + +/* Allows us to set a property on the IOResources object. Unused on ARM. */ +static inline void +pal_get_resource_property(const char **property_name, + int *property_value) +{ + *property_name = 0; + (void) property_value; +} + +#if defined(__cplusplus) +} +#endif + +#endif /* _ARM_PAL_ROUTINES_H */ diff --git a/osfmk/arm/pcb.c b/osfmk/arm/pcb.c new file mode 100644 index 000000000..2d06b559e --- /dev/null +++ b/osfmk/arm/pcb.c @@ -0,0 +1,406 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +extern int debug_task; + +zone_t ads_zone; /* zone for debug_state area */ + +/* + * Routine: consider_machine_collect + * + */ +void +consider_machine_collect(void) +{ + pmap_gc(); +} + +/* + * Routine: consider_machine_adjust + * + */ +void +consider_machine_adjust(void) +{ +} + +/* + * Routine: machine_switch_context + * + */ +thread_t +machine_switch_context( + thread_t old, + thread_continue_t continuation, + thread_t new) +{ + thread_t retval; + cpu_data_t *cpu_data_ptr; + +#define machine_switch_context_kprintf(x...) /* kprintf("machine_switch_con + * text: " x) */ + + cpu_data_ptr = getCpuDatap(); + if (old == new) + panic("machine_switch_context"); + + kpc_off_cpu(old); + + pmap_set_pmap(new->map->pmap, new); + + new->machine.CpuDatap = cpu_data_ptr; + + machine_switch_context_kprintf("old= %x contination = %x new = %x\n", old, continuation, new); + retval = Switch_context(old, continuation, new); + assert(retval != NULL); + + return retval; +} + +/* + * Routine: machine_thread_create + * + */ +kern_return_t +machine_thread_create( + thread_t thread, +#if !__ARM_USER_PROTECT__ + __unused +#endif + task_t task) +{ + +#define machine_thread_create_kprintf(x...) /* kprintf("machine_thread_create: " x) */ + + machine_thread_create_kprintf("thread = %x\n", thread); + + if (current_thread() != thread) { + thread->machine.CpuDatap = (cpu_data_t *)0; + } + thread->machine.preemption_count = 0; + thread->machine.cthread_self = 0; + thread->machine.cthread_data = 0; +#if __ARM_USER_PROTECT__ + { + struct pmap *new_pmap = vm_map_pmap(task->map); + + thread->machine.kptw_ttb = ((unsigned int) kernel_pmap->ttep) | TTBR_SETUP; + thread->machine.asid = new_pmap->asid; + if (new_pmap->tte_index_max == NTTES) { + thread->machine.uptw_ttc = 2; + thread->machine.uptw_ttb = ((unsigned int) new_pmap->ttep) | TTBR_SETUP; + } else { + thread->machine.uptw_ttc = 1; + thread->machine.uptw_ttb = ((unsigned int) new_pmap->ttep ) | TTBR_SETUP; + } + } +#endif + machine_thread_state_initialize(thread); + + return (KERN_SUCCESS); +} + +/* + * Routine: machine_thread_destroy + * + */ +void +machine_thread_destroy( + thread_t thread) +{ + + if (thread->machine.DebugData != NULL) { + if (thread->machine.DebugData == getCpuDatap()->cpu_user_debug) + arm_debug_set(NULL); + zfree(ads_zone, thread->machine.DebugData); + } +} + + +/* + * Routine: machine_thread_init + * + */ +void +machine_thread_init(void) +{ + ads_zone = zinit(sizeof(arm_debug_state_t), + THREAD_CHUNK * (sizeof(arm_debug_state_t)), + THREAD_CHUNK * (sizeof(arm_debug_state_t)), + "arm debug state"); +} + + +/* + * Routine: get_useraddr + * + */ +user_addr_t +get_useraddr() +{ + return (current_thread()->machine.PcbData.pc); +} + +/* + * Routine: machine_stack_detach + * + */ +vm_offset_t +machine_stack_detach( + thread_t thread) +{ + vm_offset_t stack; + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DETACH), + (uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0); + + stack = thread->kernel_stack; + thread->kernel_stack = 0; + thread->machine.kstackptr = 0; + + return (stack); +} + + +/* + * Routine: machine_stack_attach + * + */ +void +machine_stack_attach( + thread_t thread, + vm_offset_t stack) +{ + struct arm_saved_state *savestate; + +#define machine_stack_attach_kprintf(x...) /* kprintf("machine_stack_attach: " x) */ + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_ATTACH), + (uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0); + + thread->kernel_stack = stack; + thread->machine.kstackptr = stack + kernel_stack_size - sizeof(struct thread_kernel_state); + thread_initialize_kernel_state(thread); + savestate = (struct arm_saved_state *) thread->machine.kstackptr; + + savestate->lr = (uint32_t) thread_continue; + savestate->sp = thread->machine.kstackptr; + savestate->r[7] = 0x0UL; + savestate->r[9] = (uint32_t) NULL; + savestate->cpsr = PSR_SVC_MODE | PSR_INTMASK; + machine_stack_attach_kprintf("thread = %x pc = %x, sp = %x\n", thread, savestate->lr, savestate->sp); +} + + +/* + * Routine: machine_stack_handoff + * + */ +void +machine_stack_handoff( + thread_t old, + thread_t new) +{ + vm_offset_t stack; + cpu_data_t *cpu_data_ptr; + + kpc_off_cpu(old); + + stack = machine_stack_detach(old); + cpu_data_ptr = getCpuDatap(); + new->kernel_stack = stack; + new->machine.kstackptr = stack + kernel_stack_size - sizeof(struct thread_kernel_state); + if (stack == old->reserved_stack) { + assert(new->reserved_stack); + old->reserved_stack = new->reserved_stack; + new->reserved_stack = stack; + } + + pmap_set_pmap(new->map->pmap, new); + new->machine.CpuDatap = cpu_data_ptr; + machine_set_current_thread(new); + thread_initialize_kernel_state(new); + + return; +} + + +/* + * Routine: call_continuation + * + */ +void +call_continuation( + thread_continue_t continuation, + void *parameter, + wait_result_t wresult) +{ +#define call_continuation_kprintf(x...) /* kprintf("call_continuation_kprintf: + * " x) */ + + call_continuation_kprintf("thread = %x continuation = %x, stack = %x\n", current_thread(), continuation, current_thread()->machine.kstackptr); + Call_continuation(continuation, parameter, wresult, current_thread()->machine.kstackptr); +} + +void arm_debug_set(arm_debug_state_t *debug_state) +{ + /* If this CPU supports the memory-mapped debug interface, use it, otherwise + * attempt the Extended CP14 interface. The two routines need to be kept in sync, + * functionality-wise. + */ + struct cpu_data *cpu_data_ptr; + arm_debug_info_t *debug_info = arm_debug_info(); + boolean_t intr; + + intr = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + // Set current user debug + cpu_data_ptr->cpu_user_debug = debug_state; + + if (debug_info->memory_mapped_core_debug) { + int i; + uintptr_t debug_map = cpu_data_ptr->cpu_debug_interface_map; + + // unlock debug registers + *(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY; + + // read DBGPRSR to clear the sticky power-down bit (necessary to access debug registers) + *(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGPRSR); + + // enable monitor mode (needed to set and use debug registers) + *(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGDSCR) |= ARM_DBGDSCR_MDBGEN; + + // first turn off all breakpoints/watchpoints + for (i = 0; i < 16; i++) { + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBCR))[i] = 0; + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWCR))[i] = 0; + } + + // if (debug_state == NULL) disable monitor mode + if (debug_state == NULL) { + *(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGDSCR) &= ~ARM_DBGDSCR_MDBGEN; + } else { + for (i = 0; i < 16; i++) { + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBVR))[i] = debug_state->bvr[i]; + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBCR))[i] = debug_state->bcr[i]; + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWVR))[i] = debug_state->wvr[i]; + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWCR))[i] = debug_state->wcr[i]; + } + } + + // lock debug registers + *(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGLAR) = 0; + + } else if (debug_info->coprocessor_core_debug) { + arm_debug_set_cp14(debug_state); + } + + (void) ml_set_interrupts_enabled(intr); + + return; +} + +/* + * Duplicate one arm_debug_state_t to another. "all" parameter + * is ignored in the case of ARM -- Is this the right assumption? + */ +void +copy_debug_state( + arm_debug_state_t *src, + arm_debug_state_t *target, + __unused boolean_t all) +{ + bcopy(src, target, sizeof(arm_debug_state_t)); +} + +kern_return_t +machine_thread_set_tsd_base( + thread_t thread, + mach_vm_offset_t tsd_base) +{ + + if (thread->task == kernel_task) { + return KERN_INVALID_ARGUMENT; + } + + if (tsd_base & 0x3) { + return KERN_INVALID_ARGUMENT; + } + + if (tsd_base > UINT32_MAX) + tsd_base = 0ULL; + + thread->machine.cthread_self = tsd_base; + + /* For current thread, make the TSD base active immediately */ + if (thread == current_thread()) { + + mp_disable_preemption(); + __asm__ volatile( + "mrc p15, 0, r6, c13, c0, 3\n" + "and r6, r6, #3\n" + "orr r6, r6, %0\n" + "mcr p15, 0, r6, c13, c0, 3\n" + : /* output */ + : "r"((uint32_t)tsd_base) /* input */ + : "r6" /* clobbered register */ + ); + mp_enable_preemption(); + + } + + return KERN_SUCCESS; +} diff --git a/osfmk/arm/pmap.c b/osfmk/arm/pmap.c new file mode 100644 index 000000000..327ac9b9f --- /dev/null +++ b/osfmk/arm/pmap.c @@ -0,0 +1,10555 @@ +/* + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if (__ARM_VMSA__ > 7) +#include +#include +#if CONFIG_PGTRACE +#include +#include +#if CONFIG_PGTRACE_NONKEXT +#include +#endif // CONFIG_PGTRACE_NONKEXT +#endif +#endif + +#include + +#include + +#if DEVELOPMENT || DEBUG +#define PMAP_FOOTPRINT_SUSPENDED(pmap) ((pmap)->footprint_suspended) +#else /* DEVELOPMENT || DEBUG */ +#define PMAP_FOOTPRINT_SUSPENDED(pmap) (FALSE) +#endif /* DEVELOPMENT || DEBUG */ + + + +#if DEVELOPMENT || DEBUG +int panic_on_unsigned_execute = 0; +#endif /* DEVELOPMENT || DEBUG */ + + +/* Virtual memory region for early allocation */ +#if (__ARM_VMSA__ == 7) +#define VREGION1_START (VM_HIGH_KERNEL_WINDOW & ~ARM_TT_L1_PT_OFFMASK) +#else +#define VREGION1_HIGH_WINDOW (PE_EARLY_BOOT_VA) +#define VREGION1_START ((VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - VREGION1_HIGH_WINDOW) +#endif +#define VREGION1_SIZE (trunc_page(VM_MAX_KERNEL_ADDRESS - (VREGION1_START))) + +extern unsigned int not_in_kdp; + +extern vm_offset_t first_avail; + +extern pmap_paddr_t avail_start; +extern pmap_paddr_t avail_end; + +extern vm_offset_t virtual_space_start; /* Next available kernel VA */ +extern vm_offset_t virtual_space_end; /* End of kernel address space */ + +extern int hard_maxproc; + +#if (__ARM_VMSA__ > 7) +/* The number of address bits one TTBR can cover. */ +#define PGTABLE_ADDR_BITS (64ULL - T0SZ_BOOT) + +/* + * The bounds on our TTBRs. These are for sanity checking that + * an address is accessible by a TTBR before we attempt to map it. + */ +#define ARM64_TTBR0_MIN_ADDR (0ULL) +#define ARM64_TTBR0_MAX_ADDR (0ULL + (1ULL << PGTABLE_ADDR_BITS) - 1) +#define ARM64_TTBR1_MIN_ADDR (0ULL - (1ULL << PGTABLE_ADDR_BITS)) +#define ARM64_TTBR1_MAX_ADDR (~0ULL) + +/* The level of the root of a page table. */ +const uint64_t arm64_root_pgtable_level = (3 - ((PGTABLE_ADDR_BITS - 1 - ARM_PGSHIFT) / (ARM_PGSHIFT - TTE_SHIFT))); + +/* The number of entries in the root TT of a page table. */ +const uint64_t arm64_root_pgtable_num_ttes = (2 << ((PGTABLE_ADDR_BITS - 1 - ARM_PGSHIFT) % (ARM_PGSHIFT - TTE_SHIFT))); +#else +const uint64_t arm64_root_pgtable_level = 0; +const uint64_t arm64_root_pgtable_num_ttes = 0; +#endif + +struct pmap kernel_pmap_store MARK_AS_PMAP_DATA; +SECURITY_READ_ONLY_LATE(pmap_t) kernel_pmap = &kernel_pmap_store; + +struct vm_object pmap_object_store __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); /* store pt pages */ +vm_object_t pmap_object = &pmap_object_store; + +static struct zone *pmap_zone; /* zone of pmap structures */ + +decl_simple_lock_data(, pmaps_lock MARK_AS_PMAP_DATA) +unsigned int pmap_stamp MARK_AS_PMAP_DATA; +queue_head_t map_pmap_list MARK_AS_PMAP_DATA; + +queue_head_t tt_pmap_list MARK_AS_PMAP_DATA; +unsigned int tt_pmap_count MARK_AS_PMAP_DATA; +unsigned int tt_pmap_max MARK_AS_PMAP_DATA; + +decl_simple_lock_data(, pt_pages_lock MARK_AS_PMAP_DATA) +queue_head_t pt_page_list MARK_AS_PMAP_DATA; /* pt page ptd entries list */ + +decl_simple_lock_data(, pmap_pages_lock MARK_AS_PMAP_DATA) + +typedef struct page_free_entry { + struct page_free_entry *next; +} page_free_entry_t; + +#define PAGE_FREE_ENTRY_NULL ((page_free_entry_t *) 0) + +page_free_entry_t *pmap_pages_reclaim_list MARK_AS_PMAP_DATA; /* Reclaimed pt page list */ +unsigned int pmap_pages_request_count MARK_AS_PMAP_DATA; /* Pending requests to reclaim pt page */ +unsigned long long pmap_pages_request_acum MARK_AS_PMAP_DATA; + + +typedef struct tt_free_entry { + struct tt_free_entry *next; +} tt_free_entry_t; + +#define TT_FREE_ENTRY_NULL ((tt_free_entry_t *) 0) + +tt_free_entry_t *free_page_size_tt_list MARK_AS_PMAP_DATA; +unsigned int free_page_size_tt_count MARK_AS_PMAP_DATA; +unsigned int free_page_size_tt_max MARK_AS_PMAP_DATA; +#define FREE_PAGE_SIZE_TT_MAX 4 +tt_free_entry_t *free_two_page_size_tt_list MARK_AS_PMAP_DATA; +unsigned int free_two_page_size_tt_count MARK_AS_PMAP_DATA; +unsigned int free_two_page_size_tt_max MARK_AS_PMAP_DATA; +#define FREE_TWO_PAGE_SIZE_TT_MAX 4 +tt_free_entry_t *free_tt_list MARK_AS_PMAP_DATA; +unsigned int free_tt_count MARK_AS_PMAP_DATA; +unsigned int free_tt_max MARK_AS_PMAP_DATA; + +#define TT_FREE_ENTRY_NULL ((tt_free_entry_t *) 0) + +boolean_t pmap_gc_allowed MARK_AS_PMAP_DATA = TRUE; +boolean_t pmap_gc_forced MARK_AS_PMAP_DATA = FALSE; +boolean_t pmap_gc_allowed_by_time_throttle = TRUE; + +unsigned int inuse_user_ttepages_count MARK_AS_PMAP_DATA = 0; /* non-root, non-leaf user pagetable pages, in units of PAGE_SIZE */ +unsigned int inuse_user_ptepages_count MARK_AS_PMAP_DATA = 0; /* leaf user pagetable pages, in units of PAGE_SIZE */ +unsigned int inuse_user_tteroot_count MARK_AS_PMAP_DATA = 0; /* root user pagetables, in units of PMAP_ROOT_ALLOC_SIZE */ +unsigned int inuse_kernel_ttepages_count MARK_AS_PMAP_DATA = 0; /* non-root, non-leaf kernel pagetable pages, in units of PAGE_SIZE */ +unsigned int inuse_kernel_ptepages_count MARK_AS_PMAP_DATA = 0; /* leaf kernel pagetable pages, in units of PAGE_SIZE */ +unsigned int inuse_kernel_tteroot_count MARK_AS_PMAP_DATA = 0; /* root kernel pagetables, in units of PMAP_ROOT_ALLOC_SIZE */ +unsigned int inuse_pmap_pages_count = 0; /* debugging */ + +SECURITY_READ_ONLY_LATE(tt_entry_t *) invalid_tte = 0; +SECURITY_READ_ONLY_LATE(pmap_paddr_t) invalid_ttep = 0; + +SECURITY_READ_ONLY_LATE(tt_entry_t *) cpu_tte = 0; /* set by arm_vm_init() - keep out of bss */ +SECURITY_READ_ONLY_LATE(pmap_paddr_t) cpu_ttep = 0; /* set by arm_vm_init() - phys tte addr */ + +#if DEVELOPMENT || DEBUG +int nx_enabled = 1; /* enable no-execute protection */ +int allow_data_exec = 0; /* No apps may execute data */ +int allow_stack_exec = 0; /* No apps may execute from the stack */ +#else /* DEVELOPMENT || DEBUG */ +const int nx_enabled = 1; /* enable no-execute protection */ +const int allow_data_exec = 0; /* No apps may execute data */ +const int allow_stack_exec = 0; /* No apps may execute from the stack */ +#endif /* DEVELOPMENT || DEBUG */ + +/* + * pv_entry_t - structure to track the active mappings for a given page + */ +typedef struct pv_entry { + struct pv_entry *pve_next; /* next alias */ + pt_entry_t *pve_ptep; /* page table entry */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Since pt_desc is 64-bit aligned and we cast often from pv_entry to + * pt_desc. + */ +} __attribute__ ((aligned(8))) pv_entry_t; +#else +} pv_entry_t; +#endif + +#define PV_ENTRY_NULL ((pv_entry_t *) 0) + +/* + * PMAP LEDGERS: + * We use the least significant bit of the "pve_next" pointer in a "pv_entry" + * as a marker for pages mapped through an "alternate accounting" mapping. + * These macros set, clear and test for this marker and extract the actual + * value of the "pve_next" pointer. + */ +#define PVE_NEXT_ALTACCT ((uintptr_t) 0x1) +#define PVE_NEXT_SET_ALTACCT(pve_next_p) \ + *(pve_next_p) = (struct pv_entry *) (((uintptr_t) *(pve_next_p)) | \ + PVE_NEXT_ALTACCT) +#define PVE_NEXT_CLR_ALTACCT(pve_next_p) \ + *(pve_next_p) = (struct pv_entry *) (((uintptr_t) *(pve_next_p)) & \ + ~PVE_NEXT_ALTACCT) +#define PVE_NEXT_IS_ALTACCT(pve_next) \ + ((((uintptr_t) (pve_next)) & PVE_NEXT_ALTACCT) ? TRUE : FALSE) +#define PVE_NEXT_PTR(pve_next) \ + ((struct pv_entry *)(((uintptr_t) (pve_next)) & \ + ~PVE_NEXT_ALTACCT)) +#if MACH_ASSERT +static void pmap_check_ledgers(pmap_t pmap); +#else +static inline void pmap_check_ledgers(__unused pmap_t pmap) {} +#endif /* MACH_ASSERT */ + +SECURITY_READ_ONLY_LATE(pv_entry_t **) pv_head_table; /* array of pv entry pointers */ + +pv_entry_t *pv_free_list MARK_AS_PMAP_DATA; +pv_entry_t *pv_kern_free_list MARK_AS_PMAP_DATA; +decl_simple_lock_data(,pv_free_list_lock MARK_AS_PMAP_DATA) +decl_simple_lock_data(,pv_kern_free_list_lock MARK_AS_PMAP_DATA) + +decl_simple_lock_data(,phys_backup_lock) + +/* + * pt_desc - structure to keep info on page assigned to page tables + */ +#if (__ARM_VMSA__ == 7) +#define PT_INDEX_MAX 1 +#else +#if (ARM_PGSHIFT == 14) +#define PT_INDEX_MAX 1 +#else +#define PT_INDEX_MAX 4 +#endif +#endif + +#define PT_DESC_REFCOUNT 0x4000U + +typedef struct pt_desc { + queue_chain_t pt_page; + struct { + unsigned short refcnt; + unsigned short wiredcnt; + } pt_cnt[PT_INDEX_MAX]; + struct pmap *pmap; + struct { + vm_offset_t va; + } pt_map[PT_INDEX_MAX]; +} pt_desc_t; + + +#define PTD_ENTRY_NULL ((pt_desc_t *) 0) + +SECURITY_READ_ONLY_LATE(pt_desc_t *) ptd_root_table; + +pt_desc_t *ptd_free_list MARK_AS_PMAP_DATA = PTD_ENTRY_NULL; +SECURITY_READ_ONLY_LATE(boolean_t) ptd_preboot = TRUE; +unsigned int ptd_free_count MARK_AS_PMAP_DATA = 0; +decl_simple_lock_data(,ptd_free_list_lock MARK_AS_PMAP_DATA) + +/* + * physical page attribute + */ +typedef u_int16_t pp_attr_t; + +#define PP_ATTR_WIMG_MASK 0x003F +#define PP_ATTR_WIMG(x) ((x) & PP_ATTR_WIMG_MASK) + +#define PP_ATTR_REFERENCED 0x0040 +#define PP_ATTR_MODIFIED 0x0080 + +#define PP_ATTR_INTERNAL 0x0100 +#define PP_ATTR_REUSABLE 0x0200 +#define PP_ATTR_ALTACCT 0x0400 +#define PP_ATTR_NOENCRYPT 0x0800 + +#define PP_ATTR_REFFAULT 0x1000 +#define PP_ATTR_MODFAULT 0x2000 + + +SECURITY_READ_ONLY_LATE(pp_attr_t*) pp_attr_table; + + +typedef uint8_t io_attr_t; + +#define IO_ATTR_WIMG_MASK 0x3F +#define IO_ATTR_WIMG(x) ((x) & IO_ATTR_WIMG_MASK) + +SECURITY_READ_ONLY_LATE(io_attr_t*) io_attr_table; + +SECURITY_READ_ONLY_LATE(pmap_paddr_t) vm_first_phys = (pmap_paddr_t) 0; +SECURITY_READ_ONLY_LATE(pmap_paddr_t) vm_last_phys = (pmap_paddr_t) 0; + +SECURITY_READ_ONLY_LATE(pmap_paddr_t) io_rgn_start = 0; +SECURITY_READ_ONLY_LATE(pmap_paddr_t) io_rgn_end = 0; +SECURITY_READ_ONLY_LATE(uint32_t) io_rgn_granule = 0; + +SECURITY_READ_ONLY_LATE(boolean_t) pmap_initialized = FALSE; /* Has pmap_init completed? */ + +SECURITY_READ_ONLY_LATE(uint64_t) pmap_nesting_size_min; +SECURITY_READ_ONLY_LATE(uint64_t) pmap_nesting_size_max; + +SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm_pmap_max_offset_default = 0x0; +#if defined(__arm64__) +SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm64_pmap_max_offset_default = 0x0; +#endif + +/* free address spaces (1 means free) */ +static uint32_t asid_bitmap[MAX_ASID / (sizeof(uint32_t) * NBBY)] MARK_AS_PMAP_DATA; + +#if (__ARM_VMSA__ > 7) +SECURITY_READ_ONLY_LATE(pmap_t) u32_sharedpage_pmap; +#endif + + +#define pa_index(pa) \ + (atop((pa) - vm_first_phys)) + +#define pai_to_pvh(pai) \ + (&pv_head_table[pai]) + +#define pa_valid(x) \ + ((x) >= vm_first_phys && (x) < vm_last_phys) + +/* PTE Define Macros */ + +#define pte_is_wired(pte) \ + (((pte) & ARM_PTE_WIRED_MASK) == ARM_PTE_WIRED) + +#define pte_set_wired(ptep, wired) \ + do { \ + SInt16 *ptd_wiredcnt_ptr; \ + ptd_wiredcnt_ptr = (SInt16 *)&(ptep_get_ptd(ptep)->pt_cnt[ARM_PT_DESC_INDEX(ptep)].wiredcnt); \ + if (wired) { \ + *ptep |= ARM_PTE_WIRED; \ + OSAddAtomic16(1, ptd_wiredcnt_ptr); \ + } else { \ + *ptep &= ~ARM_PTE_WIRED; \ + OSAddAtomic16(-1, ptd_wiredcnt_ptr); \ + } \ + } while(0) + +#define pte_is_ffr(pte) \ + (((pte) & ARM_PTE_WRITEABLE) == ARM_PTE_WRITEABLE) + +#define pte_set_ffr(pte, ffr) \ + do { \ + if (ffr) { \ + pte |= ARM_PTE_WRITEABLE; \ + } else { \ + pte &= ~ARM_PTE_WRITEABLE; \ + } \ + } while(0) + +/* PVE Define Macros */ + +#define pve_next(pve) \ + ((pve)->pve_next) + +#define pve_link_field(pve) \ + (&pve_next(pve)) + +#define pve_link(pp, e) \ + ((pve_next(e) = pve_next(pp)), (pve_next(pp) = (e))) + +#define pve_unlink(pp, e) \ + (pve_next(pp) = pve_next(e)) + +/* bits held in the ptep pointer field */ + +#define pve_get_ptep(pve) \ + ((pve)->pve_ptep) + +#define pve_set_ptep(pve, ptep_new) \ + do { \ + (pve)->pve_ptep = (ptep_new); \ + } while (0) + +/* PTEP Define Macros */ + +#if (__ARM_VMSA__ == 7) + +#define ARM_PT_DESC_INDEX_MASK 0x00000 +#define ARM_PT_DESC_INDEX_SHIFT 0 + + /* + * mask for page descriptor index: 4MB per page table + */ +#define ARM_TT_PT_INDEX_MASK 0xfffU /* mask for page descriptor index: 4MB per page table */ + + /* + * Shift value used for reconstructing the virtual address for a PTE. + */ +#define ARM_TT_PT_ADDR_SHIFT (10U) + +#define ARM_PT_DESC_INDEX(ptep) \ + (((unsigned)(ptep) & ARM_PT_DESC_INDEX_MASK) >> ARM_PT_DESC_INDEX_SHIFT) + +#define ptep_get_ptd(ptep) \ + ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)(ptep) - gVirtBase + gPhysBase))))) & PVH_LIST_MASK)) + +#define ptep_get_va(ptep) \ + ((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~0xFFF) - gVirtBase + gPhysBase))))))->pt_map[ARM_PT_DESC_INDEX(ptep)].va)+ ((((unsigned)(ptep)) & ARM_TT_PT_INDEX_MASK)<pmap)) + + +#else + +#if (ARM_PGSHIFT == 12) +#define ARM_PT_DESC_INDEX_MASK ((PAGE_SHIFT_CONST == ARM_PGSHIFT )? 0x00000ULL : 0x03000ULL) +#define ARM_PT_DESC_INDEX_SHIFT ((PAGE_SHIFT_CONST == ARM_PGSHIFT )? 0 : 12) + /* + * mask for page descriptor index: 2MB per page table + */ +#define ARM_TT_PT_INDEX_MASK (0x0fffULL) + /* + * Shift value used for reconstructing the virtual address for a PTE. + */ +#define ARM_TT_PT_ADDR_SHIFT (9ULL) + + /* TODO: Give this a better name/documentation than "other" */ +#define ARM_TT_PT_OTHER_MASK (0x0fffULL) + +#else + +#define ARM_PT_DESC_INDEX_MASK (0x00000) +#define ARM_PT_DESC_INDEX_SHIFT (0) + /* + * mask for page descriptor index: 32MB per page table + */ +#define ARM_TT_PT_INDEX_MASK (0x3fffULL) + /* + * Shift value used for reconstructing the virtual address for a PTE. + */ +#define ARM_TT_PT_ADDR_SHIFT (11ULL) + + /* TODO: Give this a better name/documentation than "other" */ +#define ARM_TT_PT_OTHER_MASK (0x3fffULL) +#endif + +#define ARM_PT_DESC_INDEX(ptep) \ + (((unsigned)(ptep) & ARM_PT_DESC_INDEX_MASK) >> ARM_PT_DESC_INDEX_SHIFT) + + +#define ptep_get_ptd(ptep) \ + ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)(ptep) - gVirtBase + gPhysBase))))) & PVH_LIST_MASK)) + +#define ptep_get_va(ptep) \ + ((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~ARM_TT_PT_OTHER_MASK) - gVirtBase + gPhysBase))))))->pt_map[ARM_PT_DESC_INDEX(ptep)].va)+ ((((unsigned)(ptep)) & ARM_TT_PT_INDEX_MASK)<pmap)) + +#endif + + +/* PVH Define Macros */ + +/* pvhead type */ +#define PVH_TYPE_NULL 0x0UL +#define PVH_TYPE_PVEP 0x1UL +#define PVH_TYPE_PTEP 0x2UL +#define PVH_TYPE_PTDP 0x3UL + +#define PVH_TYPE_MASK (0x3UL) +#define PVH_LIST_MASK (~PVH_TYPE_MASK) + +#if (__ARM_VMSA__ == 7) +#define pvh_set_bits(h, b) \ + do { \ + while (!OSCompareAndSwap(*(vm_offset_t *)(h), *(vm_offset_t *)(h) | (b), (vm_offset_t *)(h))); \ + } while (0) + +#define pvh_clear_bits(h, b) \ + do { \ + while (!OSCompareAndSwap(*(vm_offset_t *)(h), *(vm_offset_t *)(h) & ~(b), (vm_offset_t *)(h))); \ + } while (0) +#else +#define pvh_set_bits(h, b) \ + do { \ + while (!OSCompareAndSwap64(*(vm_offset_t *)(h), *(vm_offset_t *)(h) | ((int64_t)b), (vm_offset_t *)(h))); \ + } while (0) + +#define pvh_clear_bits(h, b) \ + do { \ + while (!OSCompareAndSwap64(*(vm_offset_t *)(h), *(vm_offset_t *)(h) & ~((int64_t)b), (vm_offset_t *)(h))); \ + } while (0) +#endif + +#define pvh_test_type(h, b) \ + ((*(vm_offset_t *)(h) & (PVH_TYPE_MASK)) == (b)) + +#define pvh_ptep(h) \ + ((pt_entry_t *)(*(vm_offset_t *)(h) & PVH_LIST_MASK)) + +#define pvh_list(h) \ + ((pv_entry_t *)(*(vm_offset_t *)(h) & PVH_LIST_MASK)) + +#define pvh_bits(h) \ + (*(vm_offset_t *)(h) & PVH_TYPE_MASK) + +#if (__ARM_VMSA__ == 7) +#define pvh_update_head(h, e, t) \ + do { \ + while (!OSCompareAndSwap(*(vm_offset_t *)(h), (vm_offset_t)(e) | (t), (vm_offset_t *)(h))); \ + } while (0) +#else +#define pvh_update_head(h, e, t) \ + do { \ + while (!OSCompareAndSwap64(*(vm_offset_t *)(h), (vm_offset_t)(e) | (t), (vm_offset_t *)(h))); \ + } while (0) +#endif + +#define pvh_add(h, e) \ + do { \ + assert(!pvh_test_type((h), PVH_TYPE_PTEP)); \ + pve_next(e) = pvh_list(h); \ + pvh_update_head((h), (e), PVH_TYPE_PVEP); \ + } while (0) + +#define pvh_remove(h, p, e) \ + do { \ + assert(!PVE_NEXT_IS_ALTACCT(pve_next((e)))); \ + if ((p) == (h)) { \ + if (PVE_NEXT_PTR(pve_next((e))) == PV_ENTRY_NULL) { \ + pvh_update_head((h), PV_ENTRY_NULL, PVH_TYPE_NULL); \ + } else { \ + pvh_update_head((h), PVE_NEXT_PTR(pve_next((e))), PVH_TYPE_PVEP); \ + } \ + } else { \ + /* \ + * PMAP LEDGERS: \ + * preserve the "alternate accounting" bit \ + * when updating "p" (the previous entry's \ + * "pve_next"). \ + */ \ + boolean_t __is_altacct; \ + __is_altacct = PVE_NEXT_IS_ALTACCT(*(p)); \ + *(p) = PVE_NEXT_PTR(pve_next((e))); \ + if (__is_altacct) { \ + PVE_NEXT_SET_ALTACCT((p)); \ + } else { \ + PVE_NEXT_CLR_ALTACCT((p)); \ + } \ + } \ + } while (0) + + +/* PPATTR Define Macros */ + +#define ppattr_set_bits(h, b) \ + do { \ + while (!OSCompareAndSwap16(*(pp_attr_t *)(h), *(pp_attr_t *)(h) | (b), (pp_attr_t *)(h))); \ + } while (0) + +#define ppattr_clear_bits(h, b) \ + do { \ + while (!OSCompareAndSwap16(*(pp_attr_t *)(h), *(pp_attr_t *)(h) & ~(b), (pp_attr_t *)(h))); \ + } while (0) + +#define ppattr_test_bits(h, b) \ + ((*(pp_attr_t *)(h) & (b)) == (b)) + +#define pa_set_bits(x, b) \ + do { \ + if (pa_valid(x)) \ + ppattr_set_bits(&pp_attr_table[pa_index(x)], \ + (b)); \ + } while (0) + +#define pa_test_bits(x, b) \ + (pa_valid(x) ? ppattr_test_bits(&pp_attr_table[pa_index(x)],\ + (b)) : FALSE) + +#define pa_clear_bits(x, b) \ + do { \ + if (pa_valid(x)) \ + ppattr_clear_bits(&pp_attr_table[pa_index(x)], \ + (b)); \ + } while (0) + +#define pa_set_modify(x) \ + pa_set_bits(x, PP_ATTR_MODIFIED) + +#define pa_clear_modify(x) \ + pa_clear_bits(x, PP_ATTR_MODIFIED) + +#define pa_set_reference(x) \ + pa_set_bits(x, PP_ATTR_REFERENCED) + +#define pa_clear_reference(x) \ + pa_clear_bits(x, PP_ATTR_REFERENCED) + + +#define IS_INTERNAL_PAGE(pai) \ + ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL) +#define SET_INTERNAL_PAGE(pai) \ + ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL) +#define CLR_INTERNAL_PAGE(pai) \ + ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL) + +#define IS_REUSABLE_PAGE(pai) \ + ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE) +#define SET_REUSABLE_PAGE(pai) \ + ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE) +#define CLR_REUSABLE_PAGE(pai) \ + ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE) + +#define IS_ALTACCT_PAGE(pai, pve_p) \ + (((pve_p) == NULL) \ + ? ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT) \ + : PVE_NEXT_IS_ALTACCT(pve_next((pve_p)))) +#define SET_ALTACCT_PAGE(pai, pve_p) \ + if ((pve_p) == NULL) { \ + ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT); \ + } else { \ + PVE_NEXT_SET_ALTACCT(&pve_next((pve_p))); \ + } +#define CLR_ALTACCT_PAGE(pai, pve_p) \ + if ((pve_p) == NULL) { \ + ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT);\ + } else { \ + PVE_NEXT_CLR_ALTACCT(&pve_next((pve_p))); \ + } + +#define IS_REFFAULT_PAGE(pai) \ + ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT) +#define SET_REFFAULT_PAGE(pai) \ + ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT) +#define CLR_REFFAULT_PAGE(pai) \ + ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT) + +#define IS_MODFAULT_PAGE(pai) \ + ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT) +#define SET_MODFAULT_PAGE(pai) \ + ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT) +#define CLR_MODFAULT_PAGE(pai) \ + ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT) + + +#if (__ARM_VMSA__ == 7) + +#define tte_index(pmap, addr) \ + ttenum((addr)) + +#define tte_get_ptd(tte) \ + ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)((tte) & ~PAGE_MASK)))))) & PVH_LIST_MASK)) + +#else + +#define tt0_index(pmap, addr) \ + (((addr) & ARM_TT_L0_INDEX_MASK) >> ARM_TT_L0_SHIFT) + +#define tt1_index(pmap, addr) \ + (((addr) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT) + +#define tt2_index(pmap, addr) \ + (((addr) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT) + +#define tt3_index(pmap, addr) \ + (((addr) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT) + +#define tte_index(pmap, addr) \ + (((addr) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT) + +#define tte_get_ptd(tte) \ + ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)((tte) & ~PAGE_MASK)))))) & PVH_LIST_MASK)) + +#endif + +/* + * Lock on pmap system + */ + +#define PMAP_LOCK_INIT(pmap) { \ + simple_lock_init(&(pmap)->lock, 0); \ + } + +#define PMAP_LOCK(pmap) { \ + simple_lock(&(pmap)->lock); \ +} + +#define PMAP_UNLOCK(pmap) { \ + simple_unlock(&(pmap)->lock); \ +} + +#if MACH_ASSERT +#define PMAP_ASSERT_LOCKED(pmap) { \ + simple_lock_assert(&(pmap)->lock, LCK_ASSERT_OWNED); \ +} +#else +#define PMAP_ASSERT_LOCKED(pmap) +#endif + +/* + * Each entry in the pv_head_table is locked by a bit in the + * pv lock array, which is stored in the region preceding pv_head_table. + * The lock bits are accessed by the physical address of the page they lock. + */ +#define LOCK_PVH(index) { \ + hw_lock_bit((hw_lock_bit_t *) \ + ((unsigned int*)pv_head_table)-1-(index>>5), \ + (index&0x1F)); \ + } + +#define UNLOCK_PVH(index) { \ + hw_unlock_bit((hw_lock_bit_t *) \ + ((unsigned int*)pv_head_table)-1-(index>>5), \ + (index&0x1F)); \ + } + +#define ASSERT_PVH_LOCKED(index) { \ + assert(*(((unsigned int*)pv_head_table)-1-(index>>5)) & (1 << (index & 0x1F))); \ +} + +#define PMAP_UPDATE_TLBS(pmap, s, e) { \ + flush_mmu_tlb_region_asid(s, (unsigned)(e - s), pmap); \ +} + +#ifdef __ARM_L1_PTW__ + +#define FLUSH_PTE_RANGE(spte, epte) \ + __asm__ volatile("dsb ish"); + +#define FLUSH_PTE(pte_p) \ + __asm__ volatile("dsb ish"); + +#else + +#define FLUSH_PTE_RANGE(spte, epte) \ + CleanPoU_DcacheRegion((vm_offset_t)spte, \ + (vm_offset_t)epte - (vm_offset_t)spte); + +#define FLUSH_PTE(pte_p) \ + CleanPoU_DcacheRegion((vm_offset_t)pte_p, sizeof(pt_entry_t)); +#endif + +#define WRITE_PTE(pte_p, pte_entry) \ + __unreachable_ok_push \ + if (TEST_PAGE_RATIO_4) { \ + do { \ + if (((unsigned)(pte_p)) & 0x1f) panic("WRITE_PTE\n"); \ + if (((pte_entry) & ~ARM_PTE_COMPRESSED_MASK) == ARM_PTE_EMPTY) { \ + *(pte_p) = (pte_entry); \ + *((pte_p)+1) = (pte_entry); \ + *((pte_p)+2) = (pte_entry); \ + *((pte_p)+3) = (pte_entry); \ + } else { \ + *(pte_p) = (pte_entry); \ + *((pte_p)+1) = (pte_entry) | 0x1000; \ + *((pte_p)+2) = (pte_entry) | 0x2000; \ + *((pte_p)+3) = (pte_entry) | 0x3000; \ + } \ + FLUSH_PTE_RANGE((pte_p),((pte_p)+4)); \ + } while(0); \ + } else { \ + do { \ + *(pte_p) = (pte_entry); \ + FLUSH_PTE(pte_p); \ + } while(0); \ + } \ + __unreachable_ok_pop + +#define WRITE_PTE_FAST(pte_p, pte_entry) \ + __unreachable_ok_push \ + if (TEST_PAGE_RATIO_4) { \ + if (((unsigned)(pte_p)) & 0x1f) panic("WRITE_PTE\n"); \ + if (((pte_entry) & ~ARM_PTE_COMPRESSED_MASK) == ARM_PTE_EMPTY) { \ + *(pte_p) = (pte_entry); \ + *((pte_p)+1) = (pte_entry); \ + *((pte_p)+2) = (pte_entry); \ + *((pte_p)+3) = (pte_entry); \ + } else { \ + *(pte_p) = (pte_entry); \ + *((pte_p)+1) = (pte_entry) | 0x1000; \ + *((pte_p)+2) = (pte_entry) | 0x2000; \ + *((pte_p)+3) = (pte_entry) | 0x3000; \ + } \ + } else { \ + *(pte_p) = (pte_entry); \ + } \ + __unreachable_ok_pop + + +/* + * Other useful macros. + */ +#define current_pmap() \ + (vm_map_pmap(current_thread()->map)) + +#define PMAP_IS_VALID(x) (TRUE) + +#ifdef PMAP_TRACES +unsigned int pmap_trace = 0; + +#define PMAP_TRACE(...) \ + if (pmap_trace) { \ + KDBG_RELEASE(__VA_ARGS__); \ + } +#else +#define PMAP_TRACE(...) KDBG_DEBUG(__VA_ARGS__) +#endif + +#define PMAP_TRACE_CONSTANT(...) KDBG_RELEASE(__VA_ARGS__) + +/* + * Internal function prototypes (forward declarations). + */ + +static void pv_init( + void); + +static boolean_t pv_alloc( + pmap_t pmap, + unsigned int pai, + pv_entry_t **pvepp); + +static void pv_free( + pv_entry_t *pvep); + +static void pv_list_free( + pv_entry_t *pvehp, + pv_entry_t *pvetp, + unsigned int cnt); + +static void ptd_bootstrap( + pt_desc_t *ptdp, unsigned int ptd_cnt); + +static pt_desc_t *ptd_alloc( + pmap_t pmap); + +static void ptd_deallocate( + pt_desc_t *ptdp); + +static void ptd_init( + pt_desc_t *ptdp, pmap_t pmap, vm_map_address_t va, unsigned int ttlevel, pt_entry_t * pte_p); + +static void pmap_zone_init( + void); + +static void pmap_set_reference( + ppnum_t pn); + +ppnum_t pmap_vtophys( + pmap_t pmap, addr64_t va); + +void pmap_switch_user_ttb( + pmap_t pmap); + +static void flush_mmu_tlb_region_asid( + vm_offset_t va, unsigned length, pmap_t pmap); + +static kern_return_t pmap_expand( + pmap_t, vm_map_address_t, unsigned int options, unsigned int level); + +static int pmap_remove_range( + pmap_t, vm_map_address_t, pt_entry_t *, pt_entry_t *, uint32_t *); + +static int pmap_remove_range_options( + pmap_t, vm_map_address_t, pt_entry_t *, pt_entry_t *, uint32_t *, int); + +static tt_entry_t *pmap_tt1_allocate( + pmap_t, vm_size_t, unsigned int); + +#define PMAP_TT_ALLOCATE_NOWAIT 0x1 + +static void pmap_tt1_deallocate( + pmap_t, tt_entry_t *, vm_size_t, unsigned int); + +#define PMAP_TT_DEALLOCATE_NOBLOCK 0x1 + +static kern_return_t pmap_tt_allocate( + pmap_t, tt_entry_t **, unsigned int, unsigned int); + +#define PMAP_TT_ALLOCATE_NOWAIT 0x1 + +static void pmap_tte_deallocate( + pmap_t, tt_entry_t *, unsigned int); + +#define PMAP_TT_L1_LEVEL 0x1 +#define PMAP_TT_L2_LEVEL 0x2 +#define PMAP_TT_L3_LEVEL 0x3 +#if (__ARM_VMSA__ == 7) +#define PMAP_TT_MAX_LEVEL PMAP_TT_L2_LEVEL +#else +#define PMAP_TT_MAX_LEVEL PMAP_TT_L3_LEVEL +#endif + +#ifdef __ARM64_PMAP_SUBPAGE_L1__ +#if (__ARM_VMSA__ <= 7) +#error This is not supported for old-style page tables +#endif +#define PMAP_ROOT_ALLOC_SIZE (((ARM_TT_L1_INDEX_MASK >> ARM_TT_L1_SHIFT) + 1) * sizeof(tt_entry_t)) +#else +#define PMAP_ROOT_ALLOC_SIZE (ARM_PGBYTES) +#endif + +const unsigned int arm_hardware_page_size = ARM_PGBYTES; +const unsigned int arm_pt_desc_size = sizeof(pt_desc_t); +const unsigned int arm_pt_root_size = PMAP_ROOT_ALLOC_SIZE; + +#define PMAP_TT_DEALLOCATE_NOBLOCK 0x1 + +void pmap_init_pte_page_internal( + pmap_t, pt_entry_t *, vm_offset_t, unsigned int , pt_desc_t **); + + +#if (__ARM_VMSA__ > 7) + +static inline tt_entry_t *pmap_tt1e( + pmap_t, vm_map_address_t); + +static inline tt_entry_t *pmap_tt2e( + pmap_t, vm_map_address_t); + +static inline pt_entry_t *pmap_tt3e( + pmap_t, vm_map_address_t); + +static void pmap_unmap_sharedpage32( + pmap_t pmap); + +static void pmap_sharedpage_flush_32_to_64( + void); + +static boolean_t + pmap_is_64bit(pmap_t); + + +#endif +static inline tt_entry_t *pmap_tte( + pmap_t, vm_map_address_t); + +static inline pt_entry_t *pmap_pte( + pmap_t, vm_map_address_t); + +static void pmap_update_cache_attributes_locked( + ppnum_t, unsigned); + +boolean_t arm_clear_fast_fault( + ppnum_t ppnum, + vm_prot_t fault_type); + +static pmap_paddr_t pmap_pages_reclaim( + void); + +static kern_return_t pmap_pages_alloc( + pmap_paddr_t *pa, + unsigned size, + unsigned option); + +#define PMAP_PAGES_ALLOCATE_NOWAIT 0x1 +#define PMAP_PAGES_RECLAIM_NOWAIT 0x2 + +static void pmap_pages_free( + pmap_paddr_t pa, + unsigned size); + + +#define PMAP_SUPPORT_PROTOTYPES(__return_type, __function_name, __function_args, __function_index) \ + static __return_type __function_name##_internal __function_args; + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +arm_fast_fault, (pmap_t pmap, + vm_map_address_t va, + vm_prot_t fault_type, + boolean_t from_user), ARM_FAST_FAULT_INDEX); + + +PMAP_SUPPORT_PROTOTYPES( +boolean_t, +arm_force_fast_fault, (ppnum_t ppnum, + vm_prot_t allow_mode, + int options), ARM_FORCE_FAST_FAULT_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +mapping_free_prime, (void), MAPPING_FREE_PRIME_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +mapping_replenish, (void), MAPPING_REPLENISH_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +boolean_t, +pmap_batch_set_cache_attributes, (ppnum_t pn, + unsigned int cacheattr, + unsigned int page_cnt, + unsigned int page_index, + boolean_t doit, + unsigned int *res), PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_change_wiring, (pmap_t pmap, + vm_map_address_t v, + boolean_t wired), PMAP_CHANGE_WIRING_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +pmap_t, +pmap_create, (ledger_t ledger, + vm_map_size_t size, + boolean_t is_64bit), PMAP_CREATE_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_destroy, (pmap_t pmap), PMAP_DESTROY_INDEX); + + + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +pmap_enter_options, (pmap_t pmap, + vm_map_address_t v, + ppnum_t pn, + vm_prot_t prot, + vm_prot_t fault_type, + unsigned int flags, + boolean_t wired, + unsigned int options), PMAP_ENTER_OPTIONS_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +vm_offset_t, +pmap_extract, (pmap_t pmap, + vm_map_address_t va), PMAP_EXTRACT_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +ppnum_t, +pmap_find_phys, (pmap_t pmap, + addr64_t va), PMAP_FIND_PHYS_INDEX); + +#if (__ARM_VMSA__ > 7) +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_insert_sharedpage, (pmap_t pmap), PMAP_INSERT_SHAREDPAGE_INDEX); +#endif + + +PMAP_SUPPORT_PROTOTYPES( +boolean_t, +pmap_is_empty, (pmap_t pmap, + vm_map_offset_t va_start, + vm_map_offset_t va_end), PMAP_IS_EMPTY_INDEX); + + +PMAP_SUPPORT_PROTOTYPES( +unsigned int, +pmap_map_cpu_windows_copy, (ppnum_t pn, + vm_prot_t prot, + unsigned int wimg_bits), PMAP_MAP_CPU_WINDOWS_COPY_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +pmap_nest, (pmap_t grand, + pmap_t subord, + addr64_t vstart, + addr64_t nstart, + uint64_t size), PMAP_NEST_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_page_protect_options, (ppnum_t ppnum, + vm_prot_t prot, + unsigned int options), PMAP_PAGE_PROTECT_OPTIONS_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_protect_options, (pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + vm_prot_t prot, + unsigned int options, + void *args), PMAP_PROTECT_OPTIONS_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +pmap_query_page_info, (pmap_t pmap, + vm_map_offset_t va, + int *disp_p), PMAP_QUERY_PAGE_INFO_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +boolean_t, +pmap_query_resident, (pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + mach_vm_size_t *resident_bytes_p, + mach_vm_size_t *compressed_bytes_p), PMAP_QUERY_RESIDENT_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_reference, (pmap_t pmap), PMAP_REFERENCE_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +int, +pmap_remove_options, (pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + int options), PMAP_REMOVE_OPTIONS_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +pmap_return, (boolean_t do_panic, + boolean_t do_recurse), PMAP_RETURN_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_set_cache_attributes, (ppnum_t pn, + unsigned int cacheattr), PMAP_SET_CACHE_ATTRIBUTES_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_set_nested, (pmap_t pmap), PMAP_SET_NESTED_INDEX); + +#if MACH_ASSERT +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_set_process, (pmap_t pmap, + int pid, + char *procname), PMAP_SET_PROCESS_INDEX); +#endif + + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_unmap_cpu_windows_copy, (unsigned int index), PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +pmap_unnest_options, (pmap_t grand, + addr64_t vaddr, + uint64_t size, + unsigned int option), PMAP_UNNEST_OPTIONS_INDEX); + + +PMAP_SUPPORT_PROTOTYPES( +void, +phys_attribute_set, (ppnum_t pn, + unsigned int bits), PHYS_ATTRIBUTE_SET_INDEX); + + +PMAP_SUPPORT_PROTOTYPES( +void, +phys_attribute_clear, (ppnum_t pn, + unsigned int bits, + int options, + void *arg), PHYS_ATTRIBUTE_CLEAR_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_switch, (pmap_t pmap), PMAP_SWITCH_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_switch_user_ttb, (pmap_t pmap), PMAP_SWITCH_USER_TTB_INDEX); + + + +void pmap_footprint_suspend(vm_map_t map, + boolean_t suspend); +PMAP_SUPPORT_PROTOTYPES( + void, + pmap_footprint_suspend, (vm_map_t map, + boolean_t suspend), + PMAP_FOOTPRINT_SUSPEND_INDEX); + +#if CONFIG_PGTRACE +boolean_t pgtrace_enabled = 0; + +typedef struct { + queue_chain_t chain; + + /* + pmap - pmap for below addresses + ova - original va page address + cva - clone va addresses for pre, target and post pages + cva_spte - clone saved ptes + range - trace range in this map + cloned - has been cloned or not + */ + pmap_t pmap; + vm_map_offset_t ova; + vm_map_offset_t cva[3]; + pt_entry_t cva_spte[3]; + struct { + pmap_paddr_t start; + pmap_paddr_t end; + } range; + bool cloned; +} pmap_pgtrace_map_t; + +static void pmap_pgtrace_init(void); +static bool pmap_pgtrace_enter_clone(pmap_t pmap, vm_map_offset_t va_page, vm_map_offset_t start, vm_map_offset_t end); +static void pmap_pgtrace_remove_clone(pmap_t pmap, pmap_paddr_t pa_page, vm_map_offset_t va_page); +static void pmap_pgtrace_remove_all_clone(pmap_paddr_t pa); +#endif + +#if (__ARM_VMSA__ > 7) +/* + * The low global vector page is mapped at a fixed alias. + * Since the page size is 16k for H8 and newer we map the globals to a 16k + * aligned address. Readers of the globals (e.g. lldb, panic server) need + * to check both addresses anyway for backward compatibility. So for now + * we leave H6 and H7 where they were. + */ +#if (ARM_PGSHIFT == 14) +#define LOWGLOBAL_ALIAS (LOW_GLOBAL_BASE_ADDRESS + 0x4000) +#else +#define LOWGLOBAL_ALIAS (LOW_GLOBAL_BASE_ADDRESS + 0x2000) +#endif + +#else +#define LOWGLOBAL_ALIAS (0xFFFF1000) +#endif + +long long alloc_tteroot_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL; +long long alloc_ttepages_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL; +long long alloc_ptepages_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL; +long long alloc_pmap_pages_count __attribute__((aligned(8))) = 0LL; + +int pt_fake_zone_index = -1; /* index of pmap fake zone */ + + + +/* + * Allocates and initializes a per-CPU data structure for the pmap. + */ +static void +pmap_cpu_data_init_internal(unsigned int cpu_number) +{ + pmap_cpu_data_t * pmap_cpu_data = NULL; + + pmap_cpu_data = pmap_get_cpu_data(); + pmap_cpu_data->cpu_number = cpu_number; +} + +void +pmap_cpu_data_init(void) +{ + pmap_cpu_data_init_internal(cpu_number()); +} + +static void +pmap_cpu_data_array_init(void) +{ + + pmap_cpu_data_init(); +} + +pmap_cpu_data_t * +pmap_get_cpu_data(void) +{ + pmap_cpu_data_t * pmap_cpu_data = NULL; + + pmap_cpu_data = &getCpuDatap()->cpu_pmap_cpu_data; + + return pmap_cpu_data; +} + + +/* TODO */ +pmap_paddr_t +pmap_pages_reclaim( + void) +{ + boolean_t found_page; + unsigned i; + pt_desc_t *ptdp; + + + /* + * pmap_pages_reclaim() is returning a page by freeing an active pt page. + * To be eligible, a pt page is assigned to a user pmap. It doesn't have any wired pte + * entry and it contains at least one valid pte entry. + * + * In a loop, check for a page in the reclaimed pt page list. + * if one is present, unlink that page and return the physical page address. + * Otherwise, scan the pt page list for an eligible pt page to reclaim. + * If found, invoke pmap_remove_range() on its pmap and address range then + * deallocates that pt page. This will end up adding the pt page to the + * reclaimed pt page list. + * If no eligible page were found in the pt page list, panic. + */ + + simple_lock(&pmap_pages_lock); + pmap_pages_request_count++; + pmap_pages_request_acum++; + + while (1) { + + if (pmap_pages_reclaim_list != (page_free_entry_t *)NULL) { + page_free_entry_t *page_entry; + + page_entry = pmap_pages_reclaim_list; + pmap_pages_reclaim_list = pmap_pages_reclaim_list->next; + simple_unlock(&pmap_pages_lock); + + return((pmap_paddr_t)ml_static_vtop((vm_offset_t)page_entry)); + } + + simple_unlock(&pmap_pages_lock); + + simple_lock(&pt_pages_lock); + ptdp = (pt_desc_t *)queue_first(&pt_page_list); + found_page = FALSE; + + while (!queue_end(&pt_page_list, (queue_entry_t)ptdp)) { + if ((ptdp->pmap != kernel_pmap) + && (ptdp->pmap->nested == FALSE) + && (simple_lock_try(&ptdp->pmap->lock))) { + + unsigned refcnt_acc = 0; + unsigned wiredcnt_acc = 0; + + for (i = 0 ; i < PT_INDEX_MAX ; i++) { + if (ptdp->pt_cnt[i].refcnt & PT_DESC_REFCOUNT) { + /* Do not attempt to free a page that contains an L2 table + * or is currently being operated on by pmap_enter(), + * which can drop the pmap lock. */ + refcnt_acc = 0; + break; + } + refcnt_acc += ptdp->pt_cnt[i].refcnt; + wiredcnt_acc += ptdp->pt_cnt[i].wiredcnt; + } + if ((wiredcnt_acc == 0) && (refcnt_acc != 0)) { + found_page = TRUE; + /* Leave ptdp->pmap locked here. We're about to reclaim + * a tt page from it, so we don't want anyone else messing + * with it while we do that. */ + break; + } + simple_unlock(&ptdp->pmap->lock); + } + ptdp = (pt_desc_t *)queue_next((queue_t)ptdp); + } + if (!found_page) { + panic("pmap_pages_reclaim(): No eligible page in pt_page_list\n"); + } else { + int remove_count = 0; + vm_map_address_t va; + pmap_t pmap; + pt_entry_t *bpte, *epte; + pt_entry_t *pte_p; + tt_entry_t *tte_p; + uint32_t rmv_spte=0; + + simple_unlock(&pt_pages_lock); + pmap = ptdp->pmap; + PMAP_ASSERT_LOCKED(pmap); // pmap lock should be held from loop above + for (i = 0 ; i < PT_INDEX_MAX ; i++) { + va = ptdp->pt_map[i].va; + + tte_p = pmap_tte(pmap, va); + if ((tte_p != (tt_entry_t *) NULL) + && ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE)) { + +#if (__ARM_VMSA__ == 7) + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte = &pte_p[ptenum(va)]; + epte = bpte + PAGE_SIZE/sizeof(pt_entry_t); +#else + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte = &pte_p[tt3_index(pmap, va)]; + epte = bpte + PAGE_SIZE/sizeof(pt_entry_t); +#endif + /* + * Use PMAP_OPTIONS_REMOVE to clear any + * "compressed" markers and update the + * "compressed" counter in pmap->stats. + * This means that we lose accounting for + * any compressed pages in this range + * but the alternative is to not be able + * to account for their future decompression, + * which could cause the counter to drift + * more and more. + */ + remove_count += pmap_remove_range_options( + pmap, va, bpte, epte, + &rmv_spte, PMAP_OPTIONS_REMOVE); + if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt != 0) + panic("pmap_pages_reclaim(): ptdp %p, count %d\n", ptdp, ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt); +#if (__ARM_VMSA__ == 7) + pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L1_LEVEL); + flush_mmu_tlb_entry((va & ~ARM_TT_L1_PT_OFFMASK) | (pmap->asid & 0xff)); + flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + ARM_TT_L1_SIZE) | (pmap->asid & 0xff)); + flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + 2*ARM_TT_L1_SIZE)| (pmap->asid & 0xff)); + flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + 3*ARM_TT_L1_SIZE)| (pmap->asid & 0xff)); +#else + pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L2_LEVEL); + flush_mmu_tlb_entry(tlbi_addr(va & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); +#endif + + if (remove_count > 0) { +#if (__ARM_VMSA__ == 7) + PMAP_UPDATE_TLBS(pmap, va, va+4*ARM_TT_L1_SIZE); +#else + PMAP_UPDATE_TLBS(pmap, va, va+ARM_TT_L2_SIZE); +#endif + } + } + } + // Undo the lock we grabbed when we found ptdp above + PMAP_UNLOCK(pmap); + } + simple_lock(&pmap_pages_lock); + } +} + + +static kern_return_t +pmap_pages_alloc( + pmap_paddr_t *pa, + unsigned size, + unsigned option) +{ + vm_page_t m = VM_PAGE_NULL, m_prev; + + if(option & PMAP_PAGES_RECLAIM_NOWAIT) { + assert(size == PAGE_SIZE); + *pa = pmap_pages_reclaim(); + return KERN_SUCCESS; + } + if (size == PAGE_SIZE) { + while ((m = vm_page_grab()) == VM_PAGE_NULL) { + if(option & PMAP_PAGES_ALLOCATE_NOWAIT) { + return KERN_RESOURCE_SHORTAGE; + } + + VM_PAGE_WAIT(); + } + vm_page_lock_queues(); + vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE); + vm_page_unlock_queues(); + } + if (size == 2*PAGE_SIZE) { + while (cpm_allocate(size, &m, 0, 1, TRUE, 0) != KERN_SUCCESS) { + if(option & PMAP_PAGES_ALLOCATE_NOWAIT) + return KERN_RESOURCE_SHORTAGE; + + VM_PAGE_WAIT(); + } + } + + *pa = (pmap_paddr_t)ptoa(VM_PAGE_GET_PHYS_PAGE(m)); + + vm_object_lock(pmap_object); + while (m != VM_PAGE_NULL) { + vm_page_insert_wired(m, pmap_object, (vm_object_offset_t) ((ptoa(VM_PAGE_GET_PHYS_PAGE(m))) - gPhysBase), VM_KERN_MEMORY_PTE); + m_prev = m; + m = NEXT_PAGE(m_prev); + *(NEXT_PAGE_PTR(m_prev)) = VM_PAGE_NULL; + } + vm_object_unlock(pmap_object); + + OSAddAtomic(size>>PAGE_SHIFT, &inuse_pmap_pages_count); + OSAddAtomic64(size>>PAGE_SHIFT, &alloc_pmap_pages_count); + + return KERN_SUCCESS; +} + + +static void +pmap_pages_free( + pmap_paddr_t pa, + unsigned size) +{ + simple_lock(&pmap_pages_lock); + + if (pmap_pages_request_count != 0) { + page_free_entry_t *page_entry; + + pmap_pages_request_count--; + page_entry = (page_free_entry_t *)phystokv(pa); + page_entry->next = pmap_pages_reclaim_list; + pmap_pages_reclaim_list = page_entry; + simple_unlock(&pmap_pages_lock); + + return; + } + + simple_unlock(&pmap_pages_lock); + + vm_page_t m; + pmap_paddr_t pa_max; + + OSAddAtomic(-(size>>PAGE_SHIFT), &inuse_pmap_pages_count); + + for (pa_max = pa + size; pa < pa_max; pa = pa + PAGE_SIZE) { + vm_object_lock(pmap_object); + m = vm_page_lookup(pmap_object, (pa - gPhysBase)); + assert(m != VM_PAGE_NULL); + assert(VM_PAGE_WIRED(m)); + vm_page_lock_queues(); + vm_page_free(m); + vm_page_unlock_queues(); + vm_object_unlock(pmap_object); + } +} + +static inline void +PMAP_ZINFO_PALLOC( + pmap_t pmap, int bytes) +{ + pmap_ledger_credit(pmap, task_ledgers.tkm_private, bytes); +} + +static inline void +PMAP_ZINFO_PFREE( + pmap_t pmap, + int bytes) +{ + pmap_ledger_debit(pmap, task_ledgers.tkm_private, bytes); +} + +static inline void +pmap_tt_ledger_credit( + pmap_t pmap, + vm_size_t size) +{ + if (pmap != kernel_pmap) { + pmap_ledger_credit(pmap, task_ledgers.phys_footprint, size); + pmap_ledger_credit(pmap, task_ledgers.page_table, size); + } +} + +static inline void +pmap_tt_ledger_debit( + pmap_t pmap, + vm_size_t size) +{ + if (pmap != kernel_pmap) { + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, size); + pmap_ledger_debit(pmap, task_ledgers.page_table, size); + } +} + +static unsigned int +alloc_asid( + void) +{ + unsigned int asid_bitmap_index; + + simple_lock(&pmaps_lock); + for (asid_bitmap_index = 0; asid_bitmap_index < (MAX_ASID / (sizeof(uint32_t) * NBBY)); asid_bitmap_index++) { + unsigned int temp = ffs(asid_bitmap[asid_bitmap_index]); + if (temp > 0) { + temp -= 1; + asid_bitmap[asid_bitmap_index] &= ~(1 << temp); + simple_unlock(&pmaps_lock); + + /* We should never vend out physical ASID 0 through this method. */ + assert(((asid_bitmap_index * sizeof(uint32_t) * NBBY + temp) % ARM_MAX_ASID) != 0); + + return (asid_bitmap_index * sizeof(uint32_t) * NBBY + temp); + } + } + simple_unlock(&pmaps_lock); + /* + * ToDo: Add code to deal with pmap with no asid panic for now. Not + * an issue with the small config process hard limit + */ + panic("alloc_asid(): out of ASID number"); + return MAX_ASID; +} + +static void +free_asid( + int asid) +{ + /* Don't free up any alias of physical ASID 0. */ + assert((asid % ARM_MAX_ASID) != 0); + + simple_lock(&pmaps_lock); + setbit(asid, (int *) asid_bitmap); + simple_unlock(&pmaps_lock); +} + +#define PV_LOW_WATER_MARK_DEFAULT 0x200 +#define PV_KERN_LOW_WATER_MARK_DEFAULT 0x200 +#define PV_ALLOC_CHUNK_INITIAL 0x200 +#define PV_KERN_ALLOC_CHUNK_INITIAL 0x200 +#define PV_ALLOC_INITIAL_TARGET (PV_ALLOC_CHUNK_INITIAL * 5) +#define PV_KERN_ALLOC_INITIAL_TARGET (PV_KERN_ALLOC_CHUNK_INITIAL) + + +uint32_t pv_free_count MARK_AS_PMAP_DATA = 0; +uint32_t pv_page_count MARK_AS_PMAP_DATA = 0; +uint32_t pv_kern_free_count MARK_AS_PMAP_DATA = 0; + +uint32_t pv_low_water_mark MARK_AS_PMAP_DATA; +uint32_t pv_kern_low_water_mark MARK_AS_PMAP_DATA; +uint32_t pv_alloc_chunk MARK_AS_PMAP_DATA; +uint32_t pv_kern_alloc_chunk MARK_AS_PMAP_DATA; + +thread_t mapping_replenish_thread; +event_t mapping_replenish_event; +event_t pmap_user_pv_throttle_event; +volatile uint32_t mappingrecurse = 0; + +uint64_t pmap_pv_throttle_stat; +uint64_t pmap_pv_throttled_waiters; + +unsigned pmap_mapping_thread_wakeups; +unsigned pmap_kernel_reserve_replenish_stat MARK_AS_PMAP_DATA; +unsigned pmap_user_reserve_replenish_stat MARK_AS_PMAP_DATA; +unsigned pmap_kern_reserve_alloc_stat MARK_AS_PMAP_DATA; + + +static void +pv_init( + void) +{ + simple_lock_init(&pv_free_list_lock, 0); + simple_lock_init(&pv_kern_free_list_lock, 0); + pv_free_list = PV_ENTRY_NULL; + pv_free_count = 0x0U; + pv_kern_free_list = PV_ENTRY_NULL; + pv_kern_free_count = 0x0U; +} + +static inline void PV_ALLOC(pv_entry_t **pv_ep); +static inline void PV_KERN_ALLOC(pv_entry_t **pv_e); +static inline void PV_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt); +static inline void PV_KERN_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt); + +static inline void pmap_pv_throttle(pmap_t p); + +static boolean_t +pv_alloc( + pmap_t pmap, + unsigned int pai, + pv_entry_t **pvepp) +{ + PMAP_ASSERT_LOCKED(pmap); + ASSERT_PVH_LOCKED(pai); + PV_ALLOC(pvepp); + if (PV_ENTRY_NULL == *pvepp) { + + if (kernel_pmap == pmap) { + + PV_KERN_ALLOC(pvepp); + + if (PV_ENTRY_NULL == *pvepp) { + pv_entry_t *pv_e; + pv_entry_t *pv_eh; + pv_entry_t *pv_et; + int pv_cnt; + unsigned j; + pmap_paddr_t pa; + kern_return_t ret; + + UNLOCK_PVH(pai); + PMAP_UNLOCK(pmap); + + ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT); + + if (ret == KERN_RESOURCE_SHORTAGE) { + ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_RECLAIM_NOWAIT); + } + + if (ret != KERN_SUCCESS) { + panic("%s: failed to alloc page for kernel, ret=%d, " + "pmap=%p, pai=%u, pvepp=%p", + __FUNCTION__, ret, + pmap, pai, pvepp); + } + + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + *pvepp = pv_e; + pv_e++; + + for (j = 1; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt); + PMAP_LOCK(pmap); + LOCK_PVH(pai); + return FALSE; + } + } else { + UNLOCK_PVH(pai); + PMAP_UNLOCK(pmap); + pmap_pv_throttle(pmap); + { + pv_entry_t *pv_e; + pv_entry_t *pv_eh; + pv_entry_t *pv_et; + int pv_cnt; + unsigned j; + pmap_paddr_t pa; + kern_return_t ret; + + ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0); + + if (ret != KERN_SUCCESS) { + panic("%s: failed to alloc page, ret=%d, " + "pmap=%p, pai=%u, pvepp=%p", + __FUNCTION__, ret, + pmap, pai, pvepp); + } + + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + *pvepp = pv_e; + pv_e++; + + for (j = 1; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + PV_FREE_LIST(pv_eh, pv_et, pv_cnt); + } + PMAP_LOCK(pmap); + LOCK_PVH(pai); + return FALSE; + } + } + assert(PV_ENTRY_NULL != *pvepp); + return TRUE; +} + +static void +pv_free( + pv_entry_t *pvep) +{ + PV_FREE_LIST(pvep, pvep, 1); +} + +static void +pv_list_free( + pv_entry_t *pvehp, + pv_entry_t *pvetp, + unsigned int cnt) +{ + PV_FREE_LIST(pvehp, pvetp, cnt); +} + + + +static inline void PV_ALLOC(pv_entry_t **pv_ep) { + assert(*pv_ep == PV_ENTRY_NULL); + simple_lock(&pv_free_list_lock); + /* + * If the kernel reserved pool is low, let non-kernel mappings allocate + * synchronously, possibly subject to a throttle. + */ + if ((pv_kern_free_count >= pv_kern_low_water_mark) && ((*pv_ep = pv_free_list) != 0)) { + pv_free_list = (pv_entry_t *)(*pv_ep)->pve_next; + (*pv_ep)->pve_next = PV_ENTRY_NULL; + pv_free_count--; + } + + simple_unlock(&pv_free_list_lock); + + if ((pv_free_count < pv_low_water_mark) || (pv_kern_free_count < pv_kern_low_water_mark)) { + if (!mappingrecurse && hw_compare_and_store(0,1, &mappingrecurse)) + thread_wakeup(&mapping_replenish_event); + } +} + +static inline void PV_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt) { + simple_lock(&pv_free_list_lock); + pv_et->pve_next = (pv_entry_t *)pv_free_list; + pv_free_list = pv_eh; + pv_free_count += pv_cnt; + simple_unlock(&pv_free_list_lock); +} + +static inline void PV_KERN_ALLOC(pv_entry_t **pv_e) { + assert(*pv_e == PV_ENTRY_NULL); + simple_lock(&pv_kern_free_list_lock); + + if ((*pv_e = pv_kern_free_list) != 0) { + pv_kern_free_list = (pv_entry_t *)(*pv_e)->pve_next; + (*pv_e)->pve_next = PV_ENTRY_NULL; + pv_kern_free_count--; + pmap_kern_reserve_alloc_stat++; + } + + simple_unlock(&pv_kern_free_list_lock); + + if (pv_kern_free_count < pv_kern_low_water_mark) { + if (!mappingrecurse && hw_compare_and_store(0,1, &mappingrecurse)) { + thread_wakeup(&mapping_replenish_event); + } + } +} + +static inline void PV_KERN_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt) { + simple_lock(&pv_kern_free_list_lock); + pv_et->pve_next = pv_kern_free_list; + pv_kern_free_list = pv_eh; + pv_kern_free_count += pv_cnt; + simple_unlock(&pv_kern_free_list_lock); +} + +static inline void pmap_pv_throttle(__unused pmap_t p) { + assert(p != kernel_pmap); + /* Apply throttle on non-kernel mappings */ + if (pv_kern_free_count < (pv_kern_low_water_mark / 2)) { + pmap_pv_throttle_stat++; + /* This doesn't need to be strictly accurate, merely a hint + * to eliminate the timeout when the reserve is replenished. + */ + pmap_pv_throttled_waiters++; + assert_wait_timeout(&pmap_user_pv_throttle_event, THREAD_UNINT, 1, 1000 * NSEC_PER_USEC); + thread_block(THREAD_CONTINUE_NULL); + } +} + +/* + * Creates a target number of free pv_entry_t objects for the kernel free list + * and the general free list. + */ +static kern_return_t +mapping_free_prime_internal(void) +{ + unsigned j; + pmap_paddr_t pa; + kern_return_t ret; + pv_entry_t *pv_e; + pv_entry_t *pv_eh; + pv_entry_t *pv_et; + int pv_cnt; + int alloc_options = 0; + int needed_pv_cnt = 0; + int target_pv_free_cnt = 0; + + SECURITY_READ_ONLY_LATE(static boolean_t) mapping_free_prime_internal_called = FALSE; + SECURITY_READ_ONLY_LATE(static boolean_t) mapping_free_prime_internal_done = FALSE; + + if (mapping_free_prime_internal_done) { + return KERN_FAILURE; + } + + if (!mapping_free_prime_internal_called) { + mapping_free_prime_internal_called = TRUE; + + pv_low_water_mark = PV_LOW_WATER_MARK_DEFAULT; + + /* Alterable via sysctl */ + pv_kern_low_water_mark = PV_KERN_LOW_WATER_MARK_DEFAULT; + + pv_kern_alloc_chunk = PV_KERN_ALLOC_CHUNK_INITIAL; + pv_alloc_chunk = PV_ALLOC_CHUNK_INITIAL; + } + + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + target_pv_free_cnt = PV_ALLOC_INITIAL_TARGET; + + /* + * We don't take the lock to read pv_free_count, as we should not be + * invoking this from a multithreaded context. + */ + needed_pv_cnt = target_pv_free_cnt - pv_free_count; + + if (needed_pv_cnt > target_pv_free_cnt) { + needed_pv_cnt = 0; + } + + while (pv_cnt < needed_pv_cnt) { + ret = pmap_pages_alloc(&pa, PAGE_SIZE, alloc_options); + + assert(ret == KERN_SUCCESS); + + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + + for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + } + + if (pv_cnt) { + PV_FREE_LIST(pv_eh, pv_et, pv_cnt); + } + + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + target_pv_free_cnt = PV_KERN_ALLOC_INITIAL_TARGET; + + /* + * We don't take the lock to read pv_kern_free_count, as we should not + * be invoking this from a multithreaded context. + */ + needed_pv_cnt = target_pv_free_cnt - pv_kern_free_count; + + if (needed_pv_cnt > target_pv_free_cnt) { + needed_pv_cnt = 0; + } + + while (pv_cnt < needed_pv_cnt) { + + ret = pmap_pages_alloc(&pa, PAGE_SIZE, alloc_options); + + assert(ret == KERN_SUCCESS); + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + + for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + } + + if (pv_cnt) { + PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt); + } + + mapping_free_prime_internal_done = TRUE; + return KERN_SUCCESS; +} + +void +mapping_free_prime(void) +{ + kern_return_t kr = KERN_FAILURE; + + kr = mapping_free_prime_internal(); + + if (kr != KERN_SUCCESS) { + panic("%s: failed, kr=%d", __FUNCTION__, kr); + } +} + +void mapping_replenish(void); + +void mapping_adjust(void) { + kern_return_t mres; + + mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread); + if (mres != KERN_SUCCESS) { + panic("pmap: mapping_replenish thread creation failed"); + } + thread_deallocate(mapping_replenish_thread); +} + +/* + * Fills the kernel and general PV free lists back up to their low watermarks. + */ +static kern_return_t +mapping_replenish_internal(void) +{ + pv_entry_t *pv_e; + pv_entry_t *pv_eh; + pv_entry_t *pv_et; + int pv_cnt; + unsigned j; + pmap_paddr_t pa; + kern_return_t ret = KERN_SUCCESS; + + while (pv_kern_free_count < pv_kern_low_water_mark) { + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + + ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0); + assert(ret == KERN_SUCCESS); + + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + + for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + pmap_kernel_reserve_replenish_stat += pv_cnt; + PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt); + } + + while (pv_free_count < pv_low_water_mark) { + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + + ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0); + assert(ret == KERN_SUCCESS); + + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + + for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + pmap_user_reserve_replenish_stat += pv_cnt; + PV_FREE_LIST(pv_eh, pv_et, pv_cnt); + } + + return ret; +} + +/* + * Continuation function that keeps the PV free lists from running out of free + * elements. + */ +__attribute__((noreturn)) +void +mapping_replenish(void) +{ + kern_return_t kr; + + /* We qualify for VM privileges...*/ + current_thread()->options |= TH_OPT_VMPRIV; + + for (;;) { + kr = mapping_replenish_internal(); + + if (kr != KERN_SUCCESS) { + panic("%s: failed, kr=%d", __FUNCTION__, kr); + } + + /* + * Wake threads throttled while the kernel reserve was being replenished. + */ + if (pmap_pv_throttled_waiters) { + pmap_pv_throttled_waiters = 0; + thread_wakeup(&pmap_user_pv_throttle_event); + } + + /* Check if the kernel pool has been depleted since the + * first pass, to reduce refill latency. + */ + if (pv_kern_free_count < pv_kern_low_water_mark) + continue; + /* Block sans continuation to avoid yielding kernel stack */ + assert_wait(&mapping_replenish_event, THREAD_UNINT); + mappingrecurse = 0; + thread_block(THREAD_CONTINUE_NULL); + pmap_mapping_thread_wakeups++; + } +} + + +static void +ptd_bootstrap( + pt_desc_t *ptdp, + unsigned int ptd_cnt) +{ + simple_lock_init(&ptd_free_list_lock, 0); + while (ptd_cnt != 0) { + (*(void **)ptdp) = (void *)ptd_free_list; + ptd_free_list = ptdp; + ptdp++; + ptd_cnt--; + ptd_free_count++; + } + ptd_preboot = FALSE; +} + +static pt_desc_t +*ptd_alloc( + pmap_t pmap) +{ + pt_desc_t *ptdp; + unsigned i; + + if (!ptd_preboot) + simple_lock(&ptd_free_list_lock); + + if (ptd_free_count == 0) { + unsigned int ptd_cnt; + pt_desc_t *ptdp_next; + + if (ptd_preboot) { + ptdp = (pt_desc_t *)avail_start; + avail_start += ARM_PGBYTES; + ptdp_next = ptdp; + ptd_cnt = ARM_PGBYTES/sizeof(pt_desc_t); + } else { + pmap_paddr_t pa; + kern_return_t ret; + + simple_unlock(&ptd_free_list_lock); + + if (pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT) != KERN_SUCCESS) { + ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_RECLAIM_NOWAIT); + assert(ret == KERN_SUCCESS); + } + ptdp = (pt_desc_t *)phystokv(pa); + + simple_lock(&ptd_free_list_lock); + ptdp_next = ptdp; + ptd_cnt = PAGE_SIZE/sizeof(pt_desc_t); + } + + while (ptd_cnt != 0) { + (*(void **)ptdp_next) = (void *)ptd_free_list; + ptd_free_list = ptdp_next; + ptdp_next++; + ptd_cnt--; + ptd_free_count++; + } + } + + if ((ptdp = ptd_free_list) != PTD_ENTRY_NULL) { + ptd_free_list = (pt_desc_t *)(*(void **)ptdp); + ptd_free_count--; + } else { + panic("out of ptd entry\n"); + } + + if (!ptd_preboot) + simple_unlock(&ptd_free_list_lock); + + ptdp->pt_page.next = NULL; + ptdp->pt_page.prev = NULL; + ptdp->pmap = pmap; + + for (i = 0 ; i < PT_INDEX_MAX ; i++) { + ptdp->pt_map[i].va = 0; + ptdp->pt_cnt[i].refcnt = 0; + ptdp->pt_cnt[i].wiredcnt = 0; + } + simple_lock(&pt_pages_lock); + queue_enter(&pt_page_list, ptdp, pt_desc_t *, pt_page); + simple_unlock(&pt_pages_lock); + + pmap_tt_ledger_credit(pmap, sizeof(*ptdp)); + + return(ptdp); +} + +static void +ptd_deallocate( + pt_desc_t *ptdp) +{ + unsigned i; + pmap_t pmap = ptdp->pmap; + + if (ptd_preboot) { + panic("ptd_deallocate(): early boot\n"); + } + for (i = 0 ; i < PT_INDEX_MAX ; i++) { + if (ptdp->pt_cnt[i].refcnt != 0) + panic("ptd_deallocate(): ptdp=%p refcnt=0x%x \n", ptdp, ptdp->pt_cnt[i].refcnt); + } + + if (ptdp->pt_page.next != NULL) { + simple_lock(&pt_pages_lock); + queue_remove(&pt_page_list, ptdp, pt_desc_t *, pt_page); + simple_unlock(&pt_pages_lock); + } + simple_lock(&ptd_free_list_lock); + (*(void **)ptdp) = (void *)ptd_free_list; + ptd_free_list = (pt_desc_t *)ptdp; + ptd_free_count++; + simple_unlock(&ptd_free_list_lock); + pmap_tt_ledger_debit(pmap, sizeof(*ptdp)); +} + +static void +ptd_init( + pt_desc_t *ptdp, + pmap_t pmap, + vm_map_address_t va, + unsigned int level, + pt_entry_t *pte_p) +{ + if (ptdp->pmap != pmap) + panic("ptd_init(): pmap mismatch\n"); + +#if (__ARM_VMSA__ == 7) + assert(level == 2); + ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~(ARM_TT_L1_PT_OFFMASK); +#else + if (level == 3) { + ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~ARM_TT_L2_OFFMASK ; + } else if (level == 2) + ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~ARM_TT_L1_OFFMASK ; +#endif + if (level < PMAP_TT_MAX_LEVEL) + ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt = PT_DESC_REFCOUNT; + +} + + +boolean_t +pmap_valid_address( + pmap_paddr_t addr) +{ + return pa_valid(addr); +} + +#if (__ARM_VMSA__ == 7) + +/* + * Given an offset and a map, compute the address of the + * corresponding translation table entry. + */ +static inline tt_entry_t * +pmap_tte(pmap_t pmap, + vm_map_address_t addr) +{ + if (!(tte_index(pmap, addr) < pmap->tte_index_max)) + return (tt_entry_t *)NULL; + return (&pmap->tte[tte_index(pmap, addr)]); +} + + +/* + * Given an offset and a map, compute the address of the + * pte. If the address is invalid with respect to the map + * then PT_ENTRY_NULL is returned (and the map may need to grow). + * + * This is only used internally. + */ +static inline pt_entry_t * +pmap_pte( + pmap_t pmap, + vm_map_address_t addr) +{ + pt_entry_t *ptp; + tt_entry_t *ttp; + tt_entry_t tte; + + ttp = pmap_tte(pmap, addr); + if (ttp == (tt_entry_t *)NULL) + return (PT_ENTRY_NULL); + tte = *ttp; + #if MACH_ASSERT + if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) + panic("Attempt to demote L1 block: pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte); + #endif + if ((tte & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE) + return (PT_ENTRY_NULL); + ptp = (pt_entry_t *) ttetokv(tte) + ptenum(addr); + return (ptp); +} + +#else + +/* + * Given an offset and a map, compute the address of level 1 translation table entry. + * If the tranlation is invalid then PT_ENTRY_NULL is returned. + */ +static inline tt_entry_t * +pmap_tt1e(pmap_t pmap, + vm_map_address_t addr) +{ +#if __ARM64_TWO_LEVEL_PMAP__ +#pragma unused(pmap, addr) + panic("pmap_tt1e called on a two level pmap"); + return (NULL); +#else + return (&pmap->tte[tt1_index(pmap, addr)]); +#endif +} + +/* + * Given an offset and a map, compute the address of level 2 translation table entry. + * If the tranlation is invalid then PT_ENTRY_NULL is returned. + */ +static inline tt_entry_t * +pmap_tt2e(pmap_t pmap, + vm_map_address_t addr) +{ +#if __ARM64_TWO_LEVEL_PMAP__ + return (&pmap->tte[tt2_index(pmap, addr)]); +#else + tt_entry_t *ttp; + tt_entry_t tte; + + ttp = pmap_tt1e(pmap, addr); + tte = *ttp; + #if MACH_ASSERT + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID)) + panic("Attempt to demote L1 block (?!): pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte); + #endif + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) + return (PT_ENTRY_NULL); + + ttp = &((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, addr)]; + return ((tt_entry_t *)ttp); +#endif +} + + +/* + * Given an offset and a map, compute the address of level 3 translation table entry. + * If the tranlation is invalid then PT_ENTRY_NULL is returned. + */ +static inline pt_entry_t * +pmap_tt3e( + pmap_t pmap, + vm_map_address_t addr) +{ + pt_entry_t *ptp; + tt_entry_t *ttp; + tt_entry_t tte; + + /* Level 0 currently unused */ +#if __ARM64_TWO_LEVEL_PMAP__ + ttp = pmap_tt2e(pmap, addr); + tte = *ttp; +#else + /* Get first-level (1GB) entry */ + ttp = pmap_tt1e(pmap, addr); + tte = *ttp; + #if MACH_ASSERT + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID)) + panic("Attempt to demote L1 block (?!): pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte); + #endif + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) + return (PT_ENTRY_NULL); + + tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, addr)]; +#endif +#if MACH_ASSERT + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID)) + panic("Attempt to demote L2 block: pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte); +#endif + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) { + return (PT_ENTRY_NULL); + } + + /* Get third-level (4KB) entry */ + ptp = &(((pt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, addr)]); + return (ptp); +} + + +static inline tt_entry_t * +pmap_tte( + pmap_t pmap, + vm_map_address_t addr) +{ + return(pmap_tt2e(pmap, addr)); +} + + +static inline pt_entry_t * +pmap_pte( + pmap_t pmap, + vm_map_address_t addr) +{ + return(pmap_tt3e(pmap, addr)); +} + +#endif + + +/* + * Map memory at initialization. The physical addresses being + * mapped are not managed and are never unmapped. + * + * For now, VM is already on, we only need to map the + * specified memory. + */ +vm_map_address_t +pmap_map( + vm_map_address_t virt, + vm_offset_t start, + vm_offset_t end, + vm_prot_t prot, + unsigned int flags) +{ + kern_return_t kr; + vm_size_t ps; + + ps = PAGE_SIZE; + while (start < end) { + kr = pmap_enter(kernel_pmap, virt, (ppnum_t)atop(start), + prot, VM_PROT_NONE, flags, FALSE); + + if (kr != KERN_SUCCESS) { + panic("%s: failed pmap_enter, " + "virt=%p, start_addr=%p, end_addr=%p, prot=%#x, flags=%#x", + __FUNCTION__, + (void *) virt, (void *) start, (void *) end, prot, flags); + } + + virt += ps; + start += ps; + } + return (virt); +} + +vm_map_address_t +pmap_map_bd_with_options( + vm_map_address_t virt, + vm_offset_t start, + vm_offset_t end, + vm_prot_t prot, + int32_t options) +{ + pt_entry_t tmplate; + pt_entry_t *ptep; + vm_map_address_t vaddr; + vm_offset_t paddr; + pt_entry_t mem_attr; + + switch (options & PMAP_MAP_BD_MASK) { + case PMAP_MAP_BD_WCOMB: + mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITECOMB); +#if (__ARM_VMSA__ > 7) + mem_attr |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + mem_attr |= ARM_PTE_SH; +#endif + break; + case PMAP_MAP_BD_POSTED: + mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED); + break; + default: + mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE); + break; + } + + tmplate = pa_to_pte(start) | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) | + mem_attr | ARM_PTE_TYPE | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF; + + vaddr = virt; + paddr = start; + while (paddr < end) { + + ptep = pmap_pte(kernel_pmap, vaddr); + if (ptep == PT_ENTRY_NULL) { + panic("pmap_map_bd"); + } + assert(!ARM_PTE_IS_COMPRESSED(*ptep)); + WRITE_PTE(ptep, tmplate); + + pte_increment_pa(tmplate); + vaddr += PAGE_SIZE; + paddr += PAGE_SIZE; + } + + if (end >= start) + flush_mmu_tlb_region(virt, (unsigned)(end - start)); + + return (vaddr); +} + +/* + * Back-door routine for mapping kernel VM at initialization. + * Useful for mapping memory outside the range + * [vm_first_phys, vm_last_phys] (i.e., devices). + * Otherwise like pmap_map. + */ +vm_map_address_t +pmap_map_bd( + vm_map_address_t virt, + vm_offset_t start, + vm_offset_t end, + vm_prot_t prot) +{ + pt_entry_t tmplate; + pt_entry_t *ptep; + vm_map_address_t vaddr; + vm_offset_t paddr; + + /* not cacheable and not buffered */ + tmplate = pa_to_pte(start) + | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX + | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) + | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE); + + vaddr = virt; + paddr = start; + while (paddr < end) { + + ptep = pmap_pte(kernel_pmap, vaddr); + if (ptep == PT_ENTRY_NULL) { + panic("pmap_map_bd"); + } + assert(!ARM_PTE_IS_COMPRESSED(*ptep)); + WRITE_PTE(ptep, tmplate); + + pte_increment_pa(tmplate); + vaddr += PAGE_SIZE; + paddr += PAGE_SIZE; + } + + if (end >= start) + flush_mmu_tlb_region(virt, (unsigned)(end - start)); + + return (vaddr); +} + +/* + * Back-door routine for mapping kernel VM at initialization. + * Useful for mapping memory specific physical addresses in early + * boot (i.e., before kernel_map is initialized). + * + * Maps are in the VM_HIGH_KERNEL_WINDOW area. + */ + +vm_map_address_t +pmap_map_high_window_bd( + vm_offset_t pa_start, + vm_size_t len, + vm_prot_t prot) +{ + pt_entry_t *ptep, pte; +#if (__ARM_VMSA__ == 7) + vm_map_address_t va_start = VM_HIGH_KERNEL_WINDOW; + vm_map_address_t va_max = VM_MAX_KERNEL_ADDRESS; +#else + vm_map_address_t va_start = VREGION1_START; + vm_map_address_t va_max = VREGION1_START + VREGION1_SIZE; +#endif + vm_map_address_t va_end; + vm_map_address_t va; + vm_size_t offset; + + offset = pa_start & PAGE_MASK; + pa_start -= offset; + len += offset; + + if (len > (va_max - va_start)) { + panic("pmap_map_high_window_bd: area too large\n"); + } + +scan: + for ( ; va_start < va_max; va_start += PAGE_SIZE) { + ptep = pmap_pte(kernel_pmap, va_start); + assert(!ARM_PTE_IS_COMPRESSED(*ptep)); + if (*ptep == ARM_PTE_TYPE_FAULT) + break; + } + if (va_start > va_max) { + panic("pmap_map_high_window_bd: insufficient pages\n"); + } + + for (va_end = va_start + PAGE_SIZE; va_end < va_start + len; va_end += PAGE_SIZE) { + ptep = pmap_pte(kernel_pmap, va_end); + assert(!ARM_PTE_IS_COMPRESSED(*ptep)); + if (*ptep != ARM_PTE_TYPE_FAULT) { + va_start = va_end + PAGE_SIZE; + goto scan; + } + } + + for (va = va_start; va < va_end; va += PAGE_SIZE, pa_start += PAGE_SIZE) { + ptep = pmap_pte(kernel_pmap, va); + pte = pa_to_pte(pa_start) + | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX + | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) + | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + WRITE_PTE(ptep, pte); + } + PMAP_UPDATE_TLBS(kernel_pmap, va_start, va_start + len); +#if KASAN + kasan_notify_address(va_start, len); +#endif + return va_start; +} + +#define PMAP_ALIGN(addr, align) ((addr) + ((align) - 1) & ~((align) - 1)) + +typedef struct pmap_io_range +{ + uint64_t addr; + uint32_t len; + uint32_t wimg; +} __attribute__((packed)) pmap_io_range_t; + +static unsigned int +pmap_compute_io_rgns(void) +{ + DTEntry entry; + pmap_io_range_t *ranges; + void *prop = NULL; + int err; + unsigned int prop_size; + + err = DTLookupEntry(NULL, "/defaults", &entry); + assert(err == kSuccess); + + if (kSuccess != DTGetProperty(entry, "pmap-io-granule", &prop, &prop_size)) + return 0; + + io_rgn_granule = *((uint32_t*)prop); + + if (kSuccess != DTGetProperty(entry, "pmap-io-ranges", &prop, &prop_size)) + return 0; + + if ((io_rgn_granule == 0) || (io_rgn_granule & PAGE_MASK)) + panic("pmap I/O region granularity is not page-aligned!\n"); + + ranges = prop; + for (unsigned int i = 0; i < (prop_size / sizeof(*ranges)); ++i) { + if ((i == 0) || (ranges[i].addr < io_rgn_start)) + io_rgn_start = ranges[i].addr; + if ((i == 0) || ((ranges[i].addr + ranges[i].len) > io_rgn_end)) + io_rgn_end = ranges[i].addr + ranges[i].len; + } + + if (io_rgn_start & PAGE_MASK) + panic("pmap I/O region start is not page-aligned!\n"); + + if (io_rgn_end & PAGE_MASK) + panic("pmap I/O region end is not page-aligned!\n"); + + if (((io_rgn_start < gPhysBase) && (io_rgn_end >= gPhysBase)) || + ((io_rgn_start < avail_end) && (io_rgn_end >= avail_end))) + panic("pmap I/O region overlaps physical memory!\n"); + + return (unsigned int)((io_rgn_end - io_rgn_start) / io_rgn_granule); +} + +static void +pmap_load_io_rgns(void) +{ + DTEntry entry; + pmap_io_range_t *ranges; + void *prop = NULL; + int err; + unsigned int prop_size; + + if (io_rgn_granule == 0) + return; + + err = DTLookupEntry(NULL, "/defaults", &entry); + assert(err == kSuccess); + + err = DTGetProperty(entry, "pmap-io-ranges", &prop, &prop_size); + assert(err == kSuccess); + + ranges = prop; + for (unsigned int i = 0; i < (prop_size / sizeof(*ranges)); ++i) { + if ((ranges[i].addr - io_rgn_start) % io_rgn_granule) + panic("pmap I/O region %d is not aligned to I/O granularity!\n", i); + if (ranges[i].len % io_rgn_granule) + panic("pmap I/O region %d size is not a multiple of I/O granularity!\n", i); + for (uint32_t offs = 0; offs < ranges[i].len; offs += io_rgn_granule) { + io_attr_table[(ranges[i].addr + offs - io_rgn_start) / io_rgn_granule] = + IO_ATTR_WIMG(ranges[i].wimg); + } + } +} + + +/* + * Bootstrap the system enough to run with virtual memory. + * + * The early VM initialization code has already allocated + * the first CPU's translation table and made entries for + * all the one-to-one mappings to be found there. + * + * We must set up the kernel pmap structures, the + * physical-to-virtual translation lookup tables for the + * physical memory to be managed (between avail_start and + * avail_end). + + * Map the kernel's code and data, and allocate the system page table. + * Page_size must already be set. + * + * Parameters: + * first_avail first available physical page - + * after kernel page tables + * avail_start PA of first managed physical page + * avail_end PA of last managed physical page + */ + +void +pmap_bootstrap( + vm_offset_t vstart) +{ + pmap_paddr_t pmap_struct_start; + vm_size_t pv_head_size; + vm_size_t pv_lock_table_size; + vm_size_t ptd_root_table_size; + vm_size_t pp_attr_table_size; + vm_size_t io_attr_table_size; + unsigned int niorgns; + unsigned int npages; + unsigned int i; + vm_map_offset_t maxoffset; + + +#ifdef PMAP_TRACES + if (PE_parse_boot_argn("-pmap_trace", &pmap_trace, sizeof (pmap_trace))) { + kprintf("Kernel traces for pmap operations enabled\n"); + } +#endif + + /* + * Initialize the kernel pmap. + */ + pmap_stamp = 1; + kernel_pmap->tte = cpu_tte; + kernel_pmap->ttep = cpu_ttep; +#if (__ARM_VMSA__ > 7) + kernel_pmap->min = ARM64_TTBR1_MIN_ADDR; +#else + kernel_pmap->min = VM_MIN_KERNEL_AND_KEXT_ADDRESS; +#endif + kernel_pmap->max = VM_MAX_KERNEL_ADDRESS; + kernel_pmap->wired = 0; + kernel_pmap->ref_count = 1; + kernel_pmap->gc_status = 0; + kernel_pmap->nx_enabled = TRUE; +#ifdef __arm64__ + kernel_pmap->is_64bit = TRUE; +#else + kernel_pmap->is_64bit = FALSE; +#endif + kernel_pmap->stamp = hw_atomic_add(&pmap_stamp, 1); + + kernel_pmap->nested_region_grand_addr = 0x0ULL; + kernel_pmap->nested_region_subord_addr = 0x0ULL; + kernel_pmap->nested_region_size = 0x0ULL; + kernel_pmap->nested_region_asid_bitmap = NULL; + kernel_pmap->nested_region_asid_bitmap_size = 0x0UL; + +#if (__ARM_VMSA__ == 7) + kernel_pmap->tte_index_max = 4*NTTES; +#else + kernel_pmap->tte_index_max = (ARM_PGBYTES / sizeof(tt_entry_t)); +#endif + kernel_pmap->prev_tte = (tt_entry_t *) NULL; + kernel_pmap->cpu_ref = 0; + + PMAP_LOCK_INIT(kernel_pmap); +#if (__ARM_VMSA__ == 7) + simple_lock_init(&kernel_pmap->tt1_lock, 0); +#endif + memset((void *) &kernel_pmap->stats, 0, sizeof(kernel_pmap->stats)); + + /* allocate space for and initialize the bookkeeping structures */ + niorgns = pmap_compute_io_rgns(); + npages = (unsigned int)atop(mem_size); + pp_attr_table_size = npages * sizeof(pp_attr_t); + io_attr_table_size = niorgns * sizeof(io_attr_t); + pv_lock_table_size = npages; + pv_head_size = round_page(sizeof(pv_entry_t *) * npages); +#if (__ARM_VMSA__ == 7) + ptd_root_table_size = sizeof(pt_desc_t) * (1<<((mem_size>>30)+12)); +#else + ptd_root_table_size = sizeof(pt_desc_t) * (1<<((mem_size>>30)+13)); +#endif + + pmap_struct_start = avail_start; + + pp_attr_table = (pp_attr_t *) phystokv(avail_start); + avail_start = PMAP_ALIGN(avail_start + pp_attr_table_size, __alignof(pp_attr_t)); + io_attr_table = (io_attr_t *) phystokv(avail_start); + avail_start = PMAP_ALIGN(avail_start + io_attr_table_size + pv_lock_table_size, __alignof(pv_entry_t*)); + pv_head_table = (pv_entry_t **) phystokv(avail_start); + avail_start = PMAP_ALIGN(avail_start + pv_head_size, __alignof(pt_desc_t)); + ptd_root_table = (pt_desc_t *)phystokv(avail_start); + avail_start = round_page(avail_start + ptd_root_table_size); + + memset((char *)phystokv(pmap_struct_start), 0, avail_start - pmap_struct_start); + + pmap_load_io_rgns(); + ptd_bootstrap(ptd_root_table, (unsigned int)(ptd_root_table_size/sizeof(pt_desc_t))); + + pmap_cpu_data_array_init(); + + vm_first_phys = gPhysBase; + vm_last_phys = trunc_page(avail_end); + + simple_lock_init(&pmaps_lock, 0); + queue_init(&map_pmap_list); + queue_enter(&map_pmap_list, kernel_pmap, pmap_t, pmaps); + queue_init(&tt_pmap_list); + tt_pmap_count = 0; + tt_pmap_max = 0; + free_page_size_tt_list = TT_FREE_ENTRY_NULL; + free_page_size_tt_count = 0; + free_page_size_tt_max = 0; + free_two_page_size_tt_list = TT_FREE_ENTRY_NULL; + free_two_page_size_tt_count = 0; + free_two_page_size_tt_max = 0; + free_tt_list = TT_FREE_ENTRY_NULL; + free_tt_count = 0; + free_tt_max = 0; + + simple_lock_init(&pt_pages_lock, 0); + queue_init(&pt_page_list); + + simple_lock_init(&pmap_pages_lock, 0); + pmap_pages_request_count = 0; + pmap_pages_request_acum = 0; + pmap_pages_reclaim_list = PAGE_FREE_ENTRY_NULL; + + virtual_space_start = vstart; + virtual_space_end = VM_MAX_KERNEL_ADDRESS; + + /* mark all the address spaces in use */ + for (i = 0; i < MAX_ASID / (sizeof(uint32_t) * NBBY); i++) + asid_bitmap[i] = 0xffffffff; + + /* + * The kernel gets ASID 0, and all aliases of it. This is + * important because ASID 0 is global; if we vend ASID 0 + * out to a user pmap, those translations will show up in + * other processes through the TLB. + */ + for (i = 0; i < MAX_ASID; i += ARM_MAX_ASID) { + asid_bitmap[i / (sizeof(uint32_t) * NBBY)] &= ~(1 << (i % (sizeof(uint32_t) * NBBY))); + } + + kernel_pmap->asid = 0; + kernel_pmap->vasid = 0; + + if (PE_parse_boot_argn("arm_maxoffset", &maxoffset, sizeof (maxoffset))) { + maxoffset = trunc_page(maxoffset); + if ((maxoffset >= pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_MIN)) + && (maxoffset <= pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_MAX))) { + arm_pmap_max_offset_default = maxoffset; + } + } +#if defined(__arm64__) + if (PE_parse_boot_argn("arm64_maxoffset", &maxoffset, sizeof (maxoffset))) { + maxoffset = trunc_page(maxoffset); + if ((maxoffset >= pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_MIN)) + && (maxoffset <= pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_MAX))) { + arm64_pmap_max_offset_default = maxoffset; + } + } +#endif + +#if DEVELOPMENT || DEBUG + PE_parse_boot_argn("panic_on_unsigned_execute", &panic_on_unsigned_execute, sizeof (panic_on_unsigned_execute)); +#endif /* DEVELOPMENT || DEBUG */ + + pmap_nesting_size_min = ARM_NESTING_SIZE_MIN; + pmap_nesting_size_max = ARM_NESTING_SIZE_MAX; + + simple_lock_init(&phys_backup_lock, 0); +} + + +void +pmap_virtual_space( + vm_offset_t *startp, + vm_offset_t *endp +) +{ + *startp = virtual_space_start; + *endp = virtual_space_end; +} + + +boolean_t +pmap_virtual_region( + unsigned int region_select, + vm_map_offset_t *startp, + vm_map_size_t *size +) +{ + boolean_t ret = FALSE; +#if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__ + if (region_select == 0) { + /* + * In this config, the bootstrap mappings should occupy their own L2 + * TTs, as they should be immutable after boot. Having the associated + * TTEs and PTEs in their own pages allows us to lock down those pages, + * while allowing the rest of the kernel address range to be remapped. + */ +#if (__ARM_VMSA__ > 7) + *startp = LOW_GLOBAL_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK; +#else +#error Unsupported configuration +#endif + *size = ((VM_MAX_KERNEL_ADDRESS - *startp) & ~PAGE_MASK); + ret = TRUE; + } +#else +#if (__ARM_VMSA__ > 7) + unsigned long low_global_vr_mask = 0; + vm_map_size_t low_global_vr_size = 0; +#endif + + if (region_select == 0) { +#if (__ARM_VMSA__ == 7) + *startp = gVirtBase & 0xFFC00000; + *size = ((virtual_space_start-(gVirtBase & 0xFFC00000)) + ~0xFFC00000) & 0xFFC00000; +#else + /* Round to avoid overlapping with the V=P area; round to at least the L2 block size. */ + if (!TEST_PAGE_SIZE_4K) { + *startp = gVirtBase & 0xFFFFFFFFFE000000; + *size = ((virtual_space_start-(gVirtBase & 0xFFFFFFFFFE000000)) + ~0xFFFFFFFFFE000000) & 0xFFFFFFFFFE000000; + } else { + *startp = gVirtBase & 0xFFFFFFFFFF800000; + *size = ((virtual_space_start-(gVirtBase & 0xFFFFFFFFFF800000)) + ~0xFFFFFFFFFF800000) & 0xFFFFFFFFFF800000; + } +#endif + ret = TRUE; + } + if (region_select == 1) { + *startp = VREGION1_START; + *size = VREGION1_SIZE; + ret = TRUE; + } +#if (__ARM_VMSA__ > 7) + /* We need to reserve a range that is at least the size of an L2 block mapping for the low globals */ + if (!TEST_PAGE_SIZE_4K) { + low_global_vr_mask = 0xFFFFFFFFFE000000; + low_global_vr_size = 0x2000000; + } else { + low_global_vr_mask = 0xFFFFFFFFFF800000; + low_global_vr_size = 0x800000; + } + + if (((gVirtBase & low_global_vr_mask) != LOW_GLOBAL_BASE_ADDRESS) && (region_select == 2)) { + *startp = LOW_GLOBAL_BASE_ADDRESS; + *size = low_global_vr_size; + ret = TRUE; + } + + if (region_select == 3) { + /* In this config, we allow the bootstrap mappings to occupy the same + * page table pages as the heap. + */ + *startp = VM_MIN_KERNEL_ADDRESS; + *size = LOW_GLOBAL_BASE_ADDRESS - *startp; + ret = TRUE; + } +#endif +#endif + return ret; +} + +unsigned int +pmap_free_pages( + void) +{ + return (unsigned int)atop(avail_end - first_avail); +} + + +boolean_t +pmap_next_page_hi( + ppnum_t * pnum) +{ + return pmap_next_page(pnum); +} + + +boolean_t +pmap_next_page( + ppnum_t *pnum) +{ + if (first_avail != avail_end) { + *pnum = (ppnum_t)atop(first_avail); + first_avail += PAGE_SIZE; + return TRUE; + } + return FALSE; +} + + +/* + * Initialize the pmap module. + * Called by vm_init, to initialize any structures that the pmap + * system needs to map virtual memory. + */ +void +pmap_init( + void) +{ + /* + * Protect page zero in the kernel map. + * (can be overruled by permanent transltion + * table entries at page zero - see arm_vm_init). + */ + vm_protect(kernel_map, 0, PAGE_SIZE, TRUE, VM_PROT_NONE); + + pmap_initialized = TRUE; + + pmap_zone_init(); + + + /* + * Initialize the pmap object (for tracking the vm_page_t + * structures for pages we allocate to be page tables in + * pmap_expand(). + */ + _vm_object_allocate(mem_size, pmap_object); + pmap_object->copy_strategy = MEMORY_OBJECT_COPY_NONE; + + pv_init(); + + /* + * The value of hard_maxproc may have been scaled, make sure + * it is still less than the value of MAX_ASID. + */ + assert(hard_maxproc < MAX_ASID); + +#if CONFIG_PGTRACE + pmap_pgtrace_init(); +#endif +} + +boolean_t +pmap_verify_free( + ppnum_t ppnum) +{ + pv_entry_t **pv_h; + int pai; + boolean_t result = TRUE; + pmap_paddr_t phys = ptoa(ppnum); + + assert(phys != vm_page_fictitious_addr); + + if (!pa_valid(phys)) + return (FALSE); + + pai = (int)pa_index(phys); + pv_h = pai_to_pvh(pai); + + result = (pvh_list(pv_h) == PV_ENTRY_NULL); + + return (result); +} + + +/* + * Initialize zones used by pmap. + */ +static void +pmap_zone_init( + void) +{ + /* + * Create the zone of physical maps + * and the physical-to-virtual entries. + */ + pmap_zone = zinit((vm_size_t) sizeof(struct pmap), (vm_size_t) sizeof(struct pmap)*256, + PAGE_SIZE, "pmap"); +} + + +/* + * Create and return a physical map. + * + * If the size specified for the map + * is zero, the map is an actual physical + * map, and may be referenced by the + * hardware. + * + * If the size specified is non-zero, + * the map will be used in software only, and + * is bounded by that size. + */ +static pmap_t +pmap_create_internal( + ledger_t ledger, + vm_map_size_t size, + boolean_t is_64bit) +{ + unsigned i; + pmap_t p; + + /* + * A software use-only map doesn't even need a pmap. + */ + if (size != 0) { + return (PMAP_NULL); + } + + + /* + * Allocate a pmap struct from the pmap_zone. Then allocate + * the translation table of the right size for the pmap. + */ + if ((p = (pmap_t) zalloc(pmap_zone)) == PMAP_NULL) + return (PMAP_NULL); + + if (is_64bit) { + p->min = MACH_VM_MIN_ADDRESS; + p->max = MACH_VM_MAX_ADDRESS; + } else { + p->min = VM_MIN_ADDRESS; + p->max = VM_MAX_ADDRESS; + } + + p->wired = 0; + p->ref_count = 1; + p->gc_status = 0; + p->stamp = hw_atomic_add(&pmap_stamp, 1); + p->nx_enabled = TRUE; + p->is_64bit = is_64bit; + p->nested = FALSE; + p->nested_pmap = PMAP_NULL; + + + ledger_reference(ledger); + p->ledger = ledger; + + PMAP_LOCK_INIT(p); +#if (__ARM_VMSA__ == 7) + simple_lock_init(&p->tt1_lock, 0); +#endif + memset((void *) &p->stats, 0, sizeof(p->stats)); + + p->tt_entry_free = (tt_entry_t *)0; + + p->tte = pmap_tt1_allocate(p, PMAP_ROOT_ALLOC_SIZE, 0); + p->ttep = ml_static_vtop((vm_offset_t)p->tte); + +#if (__ARM_VMSA__ == 7) + p->tte_index_max = NTTES; +#else + p->tte_index_max = (PMAP_ROOT_ALLOC_SIZE / sizeof(tt_entry_t)); +#endif + p->prev_tte = (tt_entry_t *) NULL; + p->cpu_ref = 0; + + /* nullify the translation table */ + for (i = 0; i < p->tte_index_max; i++) + p->tte[i] = ARM_TTE_TYPE_FAULT; + +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) (p->tte), PMAP_ROOT_ALLOC_SIZE); +#else + __asm__ volatile("dsb ish"); +#endif + /* assign a asid */ + p->vasid = alloc_asid(); + p->asid = p->vasid % ARM_MAX_ASID; + + /* + * initialize the rest of the structure + */ + p->nested_region_grand_addr = 0x0ULL; + p->nested_region_subord_addr = 0x0ULL; + p->nested_region_size = 0x0ULL; + p->nested_region_asid_bitmap = NULL; + p->nested_region_asid_bitmap_size = 0x0UL; + +#if MACH_ASSERT + p->pmap_pid = 0; + strlcpy(p->pmap_procname, "", sizeof (p->pmap_procname)); +#endif /* MACH_ASSERT */ +#if DEVELOPMENT || DEBUG + p->footprint_suspended = FALSE; + p->footprint_was_suspended = FALSE; +#endif /* DEVELOPMENT || DEBUG */ + + simple_lock(&pmaps_lock); + queue_enter(&map_pmap_list, p, pmap_t, pmaps); + simple_unlock(&pmaps_lock); + + return (p); +} + +pmap_t +pmap_create( + ledger_t ledger, + vm_map_size_t size, + boolean_t is_64bit) +{ + pmap_t pmap; + + PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, size, is_64bit); + + pmap = pmap_create_internal(ledger, size, is_64bit); + + PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_END, + VM_KERNEL_ADDRHIDE(pmap)); + + return pmap; +} + +#if MACH_ASSERT +static void +pmap_set_process_internal( + __unused pmap_t pmap, + __unused int pid, + __unused char *procname) +{ +#if MACH_ASSERT + if (pmap == NULL) { + return; + } + + pmap->pmap_pid = pid; + strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname)); +#endif +} +#endif + +#if MACH_ASSERT +void +pmap_set_process( + pmap_t pmap, + int pid, + char *procname) +{ + pmap_set_process_internal(pmap, pid, procname); +} + +/* + * We maintain stats and ledgers so that a task's physical footprint is: + * phys_footprint = ((internal - alternate_accounting) + * + (internal_compressed - alternate_accounting_compressed) + * + iokit_mapped + * + purgeable_nonvolatile + * + purgeable_nonvolatile_compressed + * + page_table) + * where "alternate_accounting" includes "iokit" and "purgeable" memory. + */ + +struct { + uint64_t num_pmaps_checked; + + int phys_footprint_over; + ledger_amount_t phys_footprint_over_total; + ledger_amount_t phys_footprint_over_max; + int phys_footprint_under; + ledger_amount_t phys_footprint_under_total; + ledger_amount_t phys_footprint_under_max; + + int internal_over; + ledger_amount_t internal_over_total; + ledger_amount_t internal_over_max; + int internal_under; + ledger_amount_t internal_under_total; + ledger_amount_t internal_under_max; + + int internal_compressed_over; + ledger_amount_t internal_compressed_over_total; + ledger_amount_t internal_compressed_over_max; + int internal_compressed_under; + ledger_amount_t internal_compressed_under_total; + ledger_amount_t internal_compressed_under_max; + + int iokit_mapped_over; + ledger_amount_t iokit_mapped_over_total; + ledger_amount_t iokit_mapped_over_max; + int iokit_mapped_under; + ledger_amount_t iokit_mapped_under_total; + ledger_amount_t iokit_mapped_under_max; + + int alternate_accounting_over; + ledger_amount_t alternate_accounting_over_total; + ledger_amount_t alternate_accounting_over_max; + int alternate_accounting_under; + ledger_amount_t alternate_accounting_under_total; + ledger_amount_t alternate_accounting_under_max; + + int alternate_accounting_compressed_over; + ledger_amount_t alternate_accounting_compressed_over_total; + ledger_amount_t alternate_accounting_compressed_over_max; + int alternate_accounting_compressed_under; + ledger_amount_t alternate_accounting_compressed_under_total; + ledger_amount_t alternate_accounting_compressed_under_max; + + int page_table_over; + ledger_amount_t page_table_over_total; + ledger_amount_t page_table_over_max; + int page_table_under; + ledger_amount_t page_table_under_total; + ledger_amount_t page_table_under_max; + + int purgeable_volatile_over; + ledger_amount_t purgeable_volatile_over_total; + ledger_amount_t purgeable_volatile_over_max; + int purgeable_volatile_under; + ledger_amount_t purgeable_volatile_under_total; + ledger_amount_t purgeable_volatile_under_max; + + int purgeable_nonvolatile_over; + ledger_amount_t purgeable_nonvolatile_over_total; + ledger_amount_t purgeable_nonvolatile_over_max; + int purgeable_nonvolatile_under; + ledger_amount_t purgeable_nonvolatile_under_total; + ledger_amount_t purgeable_nonvolatile_under_max; + + int purgeable_volatile_compressed_over; + ledger_amount_t purgeable_volatile_compressed_over_total; + ledger_amount_t purgeable_volatile_compressed_over_max; + int purgeable_volatile_compressed_under; + ledger_amount_t purgeable_volatile_compressed_under_total; + ledger_amount_t purgeable_volatile_compressed_under_max; + + int purgeable_nonvolatile_compressed_over; + ledger_amount_t purgeable_nonvolatile_compressed_over_total; + ledger_amount_t purgeable_nonvolatile_compressed_over_max; + int purgeable_nonvolatile_compressed_under; + ledger_amount_t purgeable_nonvolatile_compressed_under_total; + ledger_amount_t purgeable_nonvolatile_compressed_under_max; +} pmap_ledgers_drift; +#endif /* MACH_ASSERT */ + +/* + * Retire the given physical map from service. + * Should only be called if the map contains + * no valid mappings. + */ +static void +pmap_destroy_internal( + pmap_t pmap) +{ +#if (__ARM_VMSA__ == 7) + pt_entry_t *ttep; + unsigned int i; + pmap_t tmp_pmap, tt_pmap; + queue_head_t tmp_pmap_list; + + queue_init(&tmp_pmap_list); + simple_lock(&pmaps_lock); + tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&tt_pmap_list)); + while (!queue_end(&tt_pmap_list, (queue_entry_t)tt_pmap)) { + if (tt_pmap->cpu_ref == 0 ) { + tmp_pmap = tt_pmap; + tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tmp_pmap->pmaps)); + queue_remove(&tt_pmap_list, tmp_pmap, pmap_t, pmaps); + tt_pmap_count--; + queue_enter(&tmp_pmap_list, tmp_pmap, pmap_t, pmaps); + } else { + tmp_pmap = tt_pmap; + tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tmp_pmap->pmaps)); + } + } + simple_unlock(&pmaps_lock); + + tmp_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&tmp_pmap_list)); + while (!queue_end(&tmp_pmap_list, (queue_entry_t)tmp_pmap)) { + tt_pmap = tmp_pmap; + tmp_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tt_pmap->pmaps)); + queue_remove(&tmp_pmap_list, tt_pmap, pmap_t, pmaps); + if (tt_pmap->tte) { + pmap_tt1_deallocate(pmap, tt_pmap->tte, tt_pmap->tte_index_max*sizeof(tt_entry_t), 0); + tt_pmap->tte = (tt_entry_t *) NULL; + tt_pmap->ttep = 0; + tt_pmap->tte_index_max = 0; + } + if (tt_pmap->prev_tte) { + pmap_tt1_deallocate(pmap, tt_pmap->prev_tte, PMAP_ROOT_ALLOC_SIZE, 0); + tt_pmap->prev_tte = (tt_entry_t *) NULL; + } + assert((tt_free_entry_t*)pmap->tt_entry_free == NULL); + free_asid(tt_pmap->vasid); + + pmap_check_ledgers(tt_pmap); + ledger_dereference(tt_pmap->ledger); + + zfree(pmap_zone, tt_pmap); + } + + if (pmap == PMAP_NULL) + return; + + if (hw_atomic_sub(&pmap->ref_count, 1) != 0) + return; + + simple_lock(&pmaps_lock); + + while (pmap->gc_status & PMAP_GC_INFLIGHT) { + pmap->gc_status |= PMAP_GC_WAIT; + assert_wait((event_t) & pmap->gc_status, THREAD_UNINT); + simple_unlock(&pmaps_lock); + (void) thread_block(THREAD_CONTINUE_NULL); + simple_lock(&pmaps_lock); + + } + + queue_remove(&map_pmap_list, pmap, pmap_t, pmaps); + simple_unlock(&pmaps_lock); + + /* + * Free the memory maps, then the + * pmap structure. + */ + PMAP_LOCK(pmap); + for (i = 0; i < pmap->tte_index_max; i++) { + ttep = &pmap->tte[i]; + if ((*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + pmap_tte_deallocate(pmap, ttep, PMAP_TT_L1_LEVEL); + flush_mmu_tlb_entry((i<asid & 0xff)); + } + } + PMAP_UNLOCK(pmap); + + if (pmap->cpu_ref == 0) { + if (pmap->tte) { + pmap_tt1_deallocate(pmap, pmap->tte, pmap->tte_index_max*sizeof(tt_entry_t), 0); + pmap->tte = (tt_entry_t *) NULL; + pmap->ttep = 0; + pmap->tte_index_max = 0; + } + if (pmap->prev_tte) { + pmap_tt1_deallocate(pmap, pmap->prev_tte, PMAP_ROOT_ALLOC_SIZE, 0); + pmap->prev_tte = (tt_entry_t *) NULL; + } + assert((tt_free_entry_t*)pmap->tt_entry_free == NULL); + + /* return its asid to the pool */ + free_asid(pmap->vasid); + pmap_check_ledgers(pmap); + + ledger_dereference(pmap->ledger); + if (pmap->nested_region_asid_bitmap) + kfree(pmap->nested_region_asid_bitmap, pmap->nested_region_asid_bitmap_size*sizeof(unsigned int)); + zfree(pmap_zone, pmap); + } else { + simple_lock(&pmaps_lock); + queue_enter(&tt_pmap_list, pmap, pmap_t, pmaps); + tt_pmap_count++; + if (tt_pmap_count > tt_pmap_max) + tt_pmap_max = tt_pmap_count; + simple_unlock(&pmaps_lock); + } +#else + pt_entry_t *ttep; + pmap_paddr_t pa; + vm_map_address_t c; + + if (pmap == PMAP_NULL) { + return; + } + + if (!pmap->is_64bit) + pmap_unmap_sharedpage32(pmap); + + if (hw_atomic_sub(&pmap->ref_count, 1) == 0) { + + simple_lock(&pmaps_lock); + while (pmap->gc_status & PMAP_GC_INFLIGHT) { + pmap->gc_status |= PMAP_GC_WAIT; + assert_wait((event_t) & pmap->gc_status, THREAD_UNINT); + simple_unlock(&pmaps_lock); + (void) thread_block(THREAD_CONTINUE_NULL); + simple_lock(&pmaps_lock); + } + queue_remove(&map_pmap_list, pmap, pmap_t, pmaps); + simple_unlock(&pmaps_lock); + + /* + * Free the memory maps, then the + * pmap structure. + */ + for (c = pmap->min; c < pmap->max; c += ARM_TT_L2_SIZE) { + ttep = pmap_tt2e(pmap, c); + if ((ttep != PT_ENTRY_NULL) && (*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + PMAP_LOCK(pmap); + pmap_tte_deallocate(pmap, ttep, PMAP_TT_L2_LEVEL); + PMAP_UNLOCK(pmap); + flush_mmu_tlb_entry(tlbi_addr(c) | tlbi_asid(pmap->asid)); + } + } +#if !__ARM64_TWO_LEVEL_PMAP__ + for (c = pmap->min; c < pmap->max; c += ARM_TT_L1_SIZE) { + ttep = pmap_tt1e(pmap, c); + if ((ttep != PT_ENTRY_NULL) && (*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + PMAP_LOCK(pmap); + pmap_tte_deallocate(pmap, ttep, PMAP_TT_L1_LEVEL); + PMAP_UNLOCK(pmap); + } + } +#endif + + if (pmap->tte) { + pa = pmap->ttep; + pmap_tt1_deallocate(pmap, (tt_entry_t *)phystokv(pa), PMAP_ROOT_ALLOC_SIZE, 0); + } + + + assert((tt_free_entry_t*)pmap->tt_entry_free == NULL); + + flush_mmu_tlb_asid((uint64_t)(pmap->asid) << TLBI_ASID_SHIFT); + free_asid(pmap->vasid); + + if (pmap->nested_region_asid_bitmap) { + kfree(pmap->nested_region_asid_bitmap, pmap->nested_region_asid_bitmap_size*sizeof(unsigned int)); + } + + pmap_check_ledgers(pmap); + ledger_dereference(pmap->ledger); + + zfree(pmap_zone, pmap); + } + +#endif +} + +void +pmap_destroy( + pmap_t pmap) +{ + PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap)); + + pmap_destroy_internal(pmap); + + PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END); +} + + +/* + * Add a reference to the specified pmap. + */ +static void +pmap_reference_internal( + pmap_t pmap) +{ + if (pmap != PMAP_NULL) { + (void) hw_atomic_add(&pmap->ref_count, 1); + } +} + +void +pmap_reference( + pmap_t pmap) +{ + pmap_reference_internal(pmap); +} + +static tt_entry_t * +pmap_tt1_allocate( + pmap_t pmap, + vm_size_t size, + unsigned option) +{ + tt_entry_t *tt1; + tt_free_entry_t *tt1_free; + pmap_paddr_t pa; + vm_address_t va; + vm_address_t va_end; + kern_return_t ret; + + simple_lock(&pmaps_lock); + if ((size == PAGE_SIZE) && (free_page_size_tt_count != 0)) { + free_page_size_tt_count--; + tt1 = (tt_entry_t *)free_page_size_tt_list; + free_page_size_tt_list = ((tt_free_entry_t *)tt1)->next; + simple_unlock(&pmaps_lock); + pmap_tt_ledger_credit(pmap, size); + return (tt_entry_t *)tt1; + }; + if ((size == 2*PAGE_SIZE) && (free_two_page_size_tt_count != 0)) { + free_two_page_size_tt_count--; + tt1 = (tt_entry_t *)free_two_page_size_tt_list; + free_two_page_size_tt_list = ((tt_free_entry_t *)tt1)->next; + simple_unlock(&pmaps_lock); + pmap_tt_ledger_credit(pmap, size); + return (tt_entry_t *)tt1; + }; + if (free_tt_count != 0) { + free_tt_count--; + tt1 = (tt_entry_t *)free_tt_list; + free_tt_list = (tt_free_entry_t *)((tt_free_entry_t *)tt1)->next; + simple_unlock(&pmaps_lock); + pmap_tt_ledger_credit(pmap, size); + return (tt_entry_t *)tt1; + } + + simple_unlock(&pmaps_lock); + + ret = pmap_pages_alloc(&pa, (unsigned)((size < PAGE_SIZE)? PAGE_SIZE : size), ((option & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)); + + if(ret == KERN_RESOURCE_SHORTAGE) + return (tt_entry_t *)0; + + + if (size < PAGE_SIZE) { + simple_lock(&pmaps_lock); + + for (va_end = phystokv(pa) + PAGE_SIZE, va = phystokv(pa) + size; va < va_end; va = va+size) { + tt1_free = (tt_free_entry_t *)va; + tt1_free->next = free_tt_list; + free_tt_list = tt1_free; + free_tt_count++; + } + if (free_tt_count > free_tt_max) + free_tt_max = free_tt_count; + + simple_unlock(&pmaps_lock); + } + + /* Always report root allocations in units of PMAP_ROOT_ALLOC_SIZE, which can be obtained by sysctl arm_pt_root_size. + * Depending on the device, this can vary between 512b and 16K. */ + OSAddAtomic((uint32_t)(size / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count)); + OSAddAtomic64(size / PMAP_ROOT_ALLOC_SIZE, &alloc_tteroot_count); + pmap_tt_ledger_credit(pmap, size); + + return (tt_entry_t *) phystokv(pa); +} + +static void +pmap_tt1_deallocate( + pmap_t pmap, + tt_entry_t *tt, + vm_size_t size, + unsigned option) +{ + tt_free_entry_t *tt_entry; + + tt_entry = (tt_free_entry_t *)tt; + if (not_in_kdp) + simple_lock(&pmaps_lock); + + if (size < PAGE_SIZE) { + free_tt_count++; + if (free_tt_count > free_tt_max) + free_tt_max = free_tt_count; + tt_entry->next = free_tt_list; + free_tt_list = tt_entry; + } + + if (size == PAGE_SIZE) { + free_page_size_tt_count++; + if (free_page_size_tt_count > free_page_size_tt_max) + free_page_size_tt_max = free_page_size_tt_count; + tt_entry->next = free_page_size_tt_list; + free_page_size_tt_list = tt_entry; + } + + if (size == 2*PAGE_SIZE) { + free_two_page_size_tt_count++; + if (free_two_page_size_tt_count > free_two_page_size_tt_max) + free_two_page_size_tt_max = free_two_page_size_tt_count; + tt_entry->next = free_two_page_size_tt_list; + free_two_page_size_tt_list = tt_entry; + } + + if ((option & PMAP_TT_DEALLOCATE_NOBLOCK) || (!not_in_kdp)) { + if (not_in_kdp) + simple_unlock(&pmaps_lock); + pmap_tt_ledger_debit(pmap, size); + return; + } + + while (free_page_size_tt_count > FREE_PAGE_SIZE_TT_MAX) { + + free_page_size_tt_count--; + tt = (tt_entry_t *)free_page_size_tt_list; + free_page_size_tt_list = ((tt_free_entry_t *)tt)->next; + + simple_unlock(&pmaps_lock); + + pmap_pages_free(ml_static_vtop((vm_offset_t)tt), PAGE_SIZE); + + OSAddAtomic(-(int32_t)(PAGE_SIZE / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count)); + + simple_lock(&pmaps_lock); + } + + while (free_two_page_size_tt_count > FREE_TWO_PAGE_SIZE_TT_MAX) { + free_two_page_size_tt_count--; + tt = (tt_entry_t *)free_two_page_size_tt_list; + free_two_page_size_tt_list = ((tt_free_entry_t *)tt)->next; + + simple_unlock(&pmaps_lock); + + pmap_pages_free(ml_static_vtop((vm_offset_t)tt), 2*PAGE_SIZE); + + OSAddAtomic(-2 * (int32_t)(PAGE_SIZE / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count)); + + simple_lock(&pmaps_lock); + } + simple_unlock(&pmaps_lock); + pmap_tt_ledger_debit(pmap, size); +} + +static kern_return_t +pmap_tt_allocate( + pmap_t pmap, + tt_entry_t **ttp, + unsigned int level, + unsigned int options) +{ + pmap_paddr_t pa; + *ttp = NULL; + + PMAP_LOCK(pmap); + if ((tt_free_entry_t *)pmap->tt_entry_free != NULL) { + tt_free_entry_t *tt_free_next; + + tt_free_next = ((tt_free_entry_t *)pmap->tt_entry_free)->next; + *ttp = (tt_entry_t *)pmap->tt_entry_free; + pmap->tt_entry_free = (tt_entry_t *)tt_free_next; + } + PMAP_UNLOCK(pmap); + + if (*ttp == NULL) { + pt_desc_t *ptdp; + + /* + * Allocate a VM page for the level x page table entries. + */ + while (pmap_pages_alloc(&pa, PAGE_SIZE, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) { + if(options & PMAP_OPTIONS_NOWAIT) { + return KERN_RESOURCE_SHORTAGE; + } + VM_PAGE_WAIT(); + } + + if (level < PMAP_TT_MAX_LEVEL) { + OSAddAtomic64(1, &alloc_ttepages_count); + OSAddAtomic(1, (pmap == kernel_pmap ? &inuse_kernel_ttepages_count : &inuse_user_ttepages_count)); + } else { + OSAddAtomic64(1, &alloc_ptepages_count); + OSAddAtomic(1, (pmap == kernel_pmap ? &inuse_kernel_ptepages_count : &inuse_user_ptepages_count)); + } + + pmap_tt_ledger_credit(pmap, PAGE_SIZE); + + PMAP_ZINFO_PALLOC(pmap, PAGE_SIZE); + + ptdp = ptd_alloc(pmap); + *(pt_desc_t **)pai_to_pvh(pa_index(pa)) = ptdp; + + __unreachable_ok_push + if (TEST_PAGE_RATIO_4) { + vm_address_t va; + vm_address_t va_end; + + PMAP_LOCK(pmap); + + for (va_end = phystokv(pa) + PAGE_SIZE, va = phystokv(pa) + ARM_PGBYTES; va < va_end; va = va+ARM_PGBYTES) { + ((tt_free_entry_t *)va)->next = (tt_free_entry_t *)pmap->tt_entry_free; + pmap->tt_entry_free = (tt_entry_t *)va; + } + PMAP_UNLOCK(pmap); + } + __unreachable_ok_pop + + *ttp = (tt_entry_t *)phystokv(pa); + } + + + return KERN_SUCCESS; +} + + +static void +pmap_tt_deallocate( + pmap_t pmap, + tt_entry_t *ttp, + unsigned int level) +{ + pt_desc_t *ptdp; + unsigned pt_acc_cnt; + unsigned i, max_pt_index = PAGE_RATIO; + vm_offset_t free_page=0; + + PMAP_LOCK(pmap); + + ptdp = ptep_get_ptd((vm_offset_t)ttp); + + if (level < PMAP_TT_MAX_LEVEL) { + + if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt == PT_DESC_REFCOUNT) + ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt = 0; + } + + ptdp->pt_map[ARM_PT_DESC_INDEX(ttp)].va = 0; + + if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt != 0) + panic("pmap_tt_deallocate(): ptdp %p, count %d\n", ptdp, ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt); + + for (i = 0, pt_acc_cnt = 0 ; i < max_pt_index ; i++) + pt_acc_cnt += ptdp->pt_cnt[i].refcnt; + + if (pt_acc_cnt == 0) { + tt_free_entry_t *tt_free_list = (tt_free_entry_t *)&pmap->tt_entry_free; + unsigned pt_free_entry_cnt = 1; + + while (pt_free_entry_cnt < max_pt_index && tt_free_list) { + tt_free_entry_t *tt_free_list_next; + + tt_free_list_next = tt_free_list->next; + if ((((vm_offset_t)tt_free_list_next) - ((vm_offset_t)ttp & ~PAGE_MASK)) < PAGE_SIZE) { + pt_free_entry_cnt++; + } + tt_free_list = tt_free_list_next; + } + if (pt_free_entry_cnt == max_pt_index) { + tt_free_entry_t *tt_free_list_cur; + + free_page = (vm_offset_t)ttp & ~PAGE_MASK; + tt_free_list = (tt_free_entry_t *)&pmap->tt_entry_free; + tt_free_list_cur = (tt_free_entry_t *)&pmap->tt_entry_free; + + while (tt_free_list_cur) { + tt_free_entry_t *tt_free_list_next; + + tt_free_list_next = tt_free_list_cur->next; + if ((((vm_offset_t)tt_free_list_next) - free_page) < PAGE_SIZE) { + tt_free_list->next = tt_free_list_next->next; + } else { + tt_free_list = tt_free_list_next; + } + tt_free_list_cur = tt_free_list_next; + } + } else { + ((tt_free_entry_t *)ttp)->next = (tt_free_entry_t *)pmap->tt_entry_free; + pmap->tt_entry_free = ttp; + } + } else { + ((tt_free_entry_t *)ttp)->next = (tt_free_entry_t *)pmap->tt_entry_free; + pmap->tt_entry_free = ttp; + } + + PMAP_UNLOCK(pmap); + + if (free_page != 0) { + + ptd_deallocate(ptep_get_ptd((vm_offset_t)free_page)); + *(pt_desc_t **)pai_to_pvh(pa_index(ml_static_vtop(free_page))) = NULL; + pmap_pages_free(ml_static_vtop(free_page), PAGE_SIZE); + if (level < PMAP_TT_MAX_LEVEL) + OSAddAtomic(-1, (pmap == kernel_pmap ? &inuse_kernel_ttepages_count : &inuse_user_ttepages_count)); + else + OSAddAtomic(-1, (pmap == kernel_pmap ? &inuse_kernel_ptepages_count : &inuse_user_ptepages_count)); + PMAP_ZINFO_PFREE(pmap, PAGE_SIZE); + pmap_tt_ledger_debit(pmap, PAGE_SIZE); + } +} + +static void +pmap_tte_deallocate( + pmap_t pmap, + tt_entry_t *ttep, + unsigned int level) +{ + pmap_paddr_t pa; + tt_entry_t tte; + + PMAP_ASSERT_LOCKED(pmap); + + tte = *ttep; + + if (tte == 0) { + panic("pmap_tte_deallocate(): null tt_entry ttep==%p\n", ttep); + } + +#if MACH_ASSERT + if (tte_get_ptd(tte)->pmap != pmap) { + panic("pmap_tte_deallocate(): ptd=%p ptd->pmap=%p pmap=%p \n", + tte_get_ptd(tte), tte_get_ptd(tte)->pmap, pmap); + } +#endif + if (((level+1) == PMAP_TT_MAX_LEVEL) && (tte_get_ptd(tte)->pt_cnt[ARM_PT_DESC_INDEX(ttetokv(*ttep))].refcnt != 0)) { + panic("pmap_tte_deallocate(): pmap=%p ttep=%p ptd=%p refcnt=0x%x \n", pmap, ttep, + tte_get_ptd(tte), (tte_get_ptd(tte)->pt_cnt[ARM_PT_DESC_INDEX(ttetokv(*ttep))].refcnt)); + } + +#if (__ARM_VMSA__ == 7) + { + tt_entry_t *ttep_4M = (tt_entry_t *) ((vm_offset_t)ttep & 0xFFFFFFF0); + unsigned i; + + for (i = 0; i<4; i++, ttep_4M++) + *ttep_4M = (tt_entry_t) 0; + } +#else + *ttep = (tt_entry_t) 0; +#endif + +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) ttep, sizeof(tt_entry_t)); +#else + __asm__ volatile("dsb ish"); +#endif + if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { +#if MACH_ASSERT + { + pt_entry_t *pte_p = ((pt_entry_t *) (ttetokv(tte) & ~ARM_PGMASK)); + unsigned i; + + for (i = 0; i < (ARM_PGBYTES / sizeof(*pte_p)); i++,pte_p++) { + if (ARM_PTE_IS_COMPRESSED(*pte_p)) { + panic("pmap_tte_deallocate: tte=0x%llx pmap=%p, pte_p=%p, pte=0x%llx compressed\n", + (uint64_t)tte, pmap, pte_p, (uint64_t)(*pte_p)); + } else if (((*pte_p) & ARM_PTE_TYPE_MASK) != ARM_PTE_TYPE_FAULT) { + panic("pmap_tte_deallocate: tte=0x%llx pmap=%p, pte_p=%p, pte=0x%llx\n", + (uint64_t)tte, pmap, pte_p, (uint64_t)(*pte_p)); + } + } + } +#endif + PMAP_UNLOCK(pmap); + + /* Clear any page offset: we mean to free the whole page, but armv7 TTEs may only be + * aligned on 1K boundaries. We clear the surrounding "chunk" of 4 TTEs above. */ + pa = tte_to_pa(tte) & ~ARM_PGMASK; + pmap_tt_deallocate(pmap, (tt_entry_t *) phystokv(pa), level+1); + PMAP_LOCK(pmap); + } +} + +/* + * Remove a range of hardware page-table entries. + * The entries given are the first (inclusive) + * and last (exclusive) entries for the VM pages. + * The virtual address is the va for the first pte. + * + * The pmap must be locked. + * If the pmap is not the kernel pmap, the range must lie + * entirely within one pte-page. This is NOT checked. + * Assumes that the pte-page exists. + * + * Returns the number of PTE changed, and sets *rmv_cnt + * to the number of SPTE changed. + */ +static int +pmap_remove_range( + pmap_t pmap, + vm_map_address_t va, + pt_entry_t *bpte, + pt_entry_t *epte, + uint32_t *rmv_cnt) +{ + return pmap_remove_range_options(pmap, va, bpte, epte, rmv_cnt, + PMAP_OPTIONS_REMOVE); +} + +#if MACH_ASSERT +int num_reusable_mismatch = 0; +#endif /* MACH_ASSERT */ + +static int +pmap_remove_range_options( + pmap_t pmap, + vm_map_address_t va, + pt_entry_t *bpte, + pt_entry_t *epte, + uint32_t *rmv_cnt, + int options) +{ + pt_entry_t *cpte; + int num_removed, num_unwired; + int num_pte_changed; + int pai = 0; + pmap_paddr_t pa; + int num_external, num_internal, num_reusable; + int num_alt_internal; + uint64_t num_compressed, num_alt_compressed; + + PMAP_ASSERT_LOCKED(pmap); + + num_removed = 0; + num_unwired = 0; + num_pte_changed = 0; + num_external = 0; + num_internal = 0; + num_reusable = 0; + num_compressed = 0; + num_alt_internal = 0; + num_alt_compressed = 0; + + for (cpte = bpte; cpte < epte; + cpte += PAGE_SIZE/ARM_PGBYTES, va += PAGE_SIZE) { + pv_entry_t **pv_h, **pve_pp; + pv_entry_t *pve_p; + pt_entry_t spte; + boolean_t managed=FALSE; + + spte = *cpte; + +#if CONFIG_PGTRACE + if (pgtrace_enabled) { + pmap_pgtrace_remove_clone(pmap, pte_to_pa(spte), va); + } +#endif + + while (!managed) { + if (pmap != kernel_pmap && + (options & PMAP_OPTIONS_REMOVE) && + (ARM_PTE_IS_COMPRESSED(spte))) { + /* + * "pmap" must be locked at this point, + * so this should not race with another + * pmap_remove_range() or pmap_enter(). + */ + + /* one less "compressed"... */ + num_compressed++; + if (spte & ARM_PTE_COMPRESSED_ALT) { + /* ... but it used to be "ALTACCT" */ + num_alt_compressed++; + } + + /* clear marker */ + WRITE_PTE_FAST(cpte, ARM_PTE_TYPE_FAULT); + /* + * "refcnt" also accounts for + * our "compressed" markers, + * so let's update it here. + */ + if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(cpte)->pt_cnt[ARM_PT_DESC_INDEX(cpte)].refcnt)) <= 0) + panic("pmap_remove_range_options: over-release of ptdp %p for pte %p\n", ptep_get_ptd(cpte), cpte); + spte = *cpte; + } + /* + * It may be possible for the pte to transition from managed + * to unmanaged in this timeframe; for now, elide the assert. + * We should break out as a consequence of checking pa_valid. + */ + //assert(!ARM_PTE_IS_COMPRESSED(spte)); + pa = pte_to_pa(spte); + if (!pa_valid(pa)) { + break; + } + pai = (int)pa_index(pa); + LOCK_PVH(pai); + spte = *cpte; + pa = pte_to_pa(spte); + if (pai == (int)pa_index(pa)) { + managed =TRUE; + break; // Leave pai locked as we will unlock it after we free the PV entry + } + UNLOCK_PVH(pai); + } + + if (ARM_PTE_IS_COMPRESSED(*cpte)) { + /* + * There used to be a valid mapping here but it + * has already been removed when the page was + * sent to the VM compressor, so nothing left to + * remove now... + */ + continue; + } + + /* remove the translation, do not flush the TLB */ + if (*cpte != ARM_PTE_TYPE_FAULT) { + assert(!ARM_PTE_IS_COMPRESSED(*cpte)); +#if MACH_ASSERT + if (managed && (pmap != kernel_pmap) && (ptep_get_va(cpte) != va)) { + panic("pmap_remove_range_options(): cpte=%p ptd=%p pte=0x%llx va=0x%llx\n", + cpte, ptep_get_ptd(cpte), (uint64_t)*cpte, (uint64_t)va); + } +#endif + WRITE_PTE_FAST(cpte, ARM_PTE_TYPE_FAULT); + num_pte_changed++; + } + + if ((spte != ARM_PTE_TYPE_FAULT) && + (pmap != kernel_pmap)) { + assert(!ARM_PTE_IS_COMPRESSED(spte)); + if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(cpte)->pt_cnt[ARM_PT_DESC_INDEX(cpte)].refcnt)) <= 0) + panic("pmap_remove_range_options: over-release of ptdp %p for pte %p\n", ptep_get_ptd(cpte), cpte); + if(rmv_cnt) (*rmv_cnt)++; + } + + if (pte_is_wired(spte)) { + pte_set_wired(cpte, 0); + num_unwired++; + } + /* + * if not managed, we're done + */ + if (!managed) + continue; + /* + * find and remove the mapping from the chain for this + * physical address. + */ + ASSERT_PVH_LOCKED(pai); // Should have been locked when we found the managed PTE above + pv_h = pai_to_pvh(pai); + + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + if (__builtin_expect((cpte != pvh_ptep(pv_h)), 0)) + panic("pmap_remove_range(): cpte=%p (0x%llx) does not match pv_h=%p (%p)\n", cpte, (uint64_t)spte, pv_h, pvh_ptep(pv_h)); + if (IS_ALTACCT_PAGE(pai, PV_ENTRY_NULL)) { + assert(IS_INTERNAL_PAGE(pai)); + num_internal++; + num_alt_internal++; + CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL); + } else if (IS_INTERNAL_PAGE(pai)) { + if (IS_REUSABLE_PAGE(pai)) { + num_reusable++; + } else { + num_internal++; + } + } else { + num_external++; + } + pvh_update_head(pv_h, PV_ENTRY_NULL, PVH_TYPE_NULL); + } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + + pve_pp = pv_h; + pve_p = pvh_list(pv_h); + + while (pve_p != PV_ENTRY_NULL && + (pve_get_ptep(pve_p) != cpte)) { + pve_pp = pve_link_field(pve_p); + pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + } + + if (__builtin_expect((pve_p == PV_ENTRY_NULL), 0)) { + UNLOCK_PVH(pai); + panic("pmap_remove_range(): cpte=%p (0x%llx) not in pv_h=%p\n", cpte, (uint64_t)spte, pv_h); + } + +#if MACH_ASSERT + if (kern_feature_override(KF_PMAPV_OVRD) == FALSE) { + pv_entry_t *check_pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + while (check_pve_p != PV_ENTRY_NULL) { + if (pve_get_ptep(check_pve_p) == cpte) { + panic("pmap_remove_range(): duplicate pve entry cpte=%p pmap=%p, pv_h=%p, pve_p=%p, pte=0x%llx, va=0x%llx\n", + cpte, pmap, pv_h, pve_p, (uint64_t)spte, (uint64_t)va); + } + check_pve_p = PVE_NEXT_PTR(pve_next(check_pve_p)); + } + } +#endif + + if (IS_ALTACCT_PAGE(pai, pve_p)) { + assert(IS_INTERNAL_PAGE(pai)); + num_internal++; + num_alt_internal++; + CLR_ALTACCT_PAGE(pai, pve_p); + } else if (IS_INTERNAL_PAGE(pai)) { + if (IS_REUSABLE_PAGE(pai)) { + num_reusable++; + } else { + num_internal++; + } + } else { + num_external++; + } + + pvh_remove(pv_h, pve_pp, pve_p) ; + pv_free(pve_p); + } else { + panic("pmap_remove_range(): unexpected PV head %p, cpte=%p pmap=%p pv_h=%p pte=0x%llx va=0x%llx\n", + *pv_h, cpte, pmap, pv_h, (uint64_t)spte, (uint64_t)va); + } + + UNLOCK_PVH(pai); + num_removed++; + } + + /* + * Update the counts + */ + OSAddAtomic(-num_removed, (SInt32 *) &pmap->stats.resident_count); + pmap_ledger_debit(pmap, task_ledgers.phys_mem, machine_ptob(num_removed)); + + if (pmap != kernel_pmap) { + /* sanity checks... */ +#if MACH_ASSERT + if (pmap->stats.internal < num_internal) { + if ((pmap->stats.internal + pmap->stats.reusable) == + (num_internal + num_reusable)) { + num_reusable_mismatch++; + printf("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d\n", + pmap, + (uint64_t) va, + bpte, + epte, + options, + num_internal, + num_removed, + num_unwired, + num_external, + num_reusable, + num_compressed, + num_alt_internal, + num_alt_compressed, + num_pte_changed, + pmap->stats.internal, + pmap->stats.reusable); + /* slight mismatch: fix it... */ + num_internal = pmap->stats.internal; + num_reusable = pmap->stats.reusable; + } else { + panic("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d", + pmap, + (uint64_t) va, + bpte, + epte, + options, + num_internal, + num_removed, + num_unwired, + num_external, + num_reusable, + num_compressed, + num_alt_internal, + num_alt_compressed, + num_pte_changed, + pmap->stats.internal, + pmap->stats.reusable); + } + } +#endif /* MACH_ASSERT */ + assertf(pmap->stats.external >= num_external, + "pmap=%p num_external=%d stats.external=%d", + pmap, num_external, pmap->stats.external); + assertf(pmap->stats.internal >= num_internal, + "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d", + pmap, + num_internal, pmap->stats.internal, + num_reusable, pmap->stats.reusable); + assertf(pmap->stats.reusable >= num_reusable, + "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d", + pmap, + num_internal, pmap->stats.internal, + num_reusable, pmap->stats.reusable); + assertf(pmap->stats.compressed >= num_compressed, + "pmap=%p num_compressed=%lld num_alt_compressed=%lld stats.compressed=%lld", + pmap, num_compressed, num_alt_compressed, + pmap->stats.compressed); + + /* update pmap stats... */ + OSAddAtomic(-num_unwired, (SInt32 *) &pmap->stats.wired_count); + if (num_external) + OSAddAtomic(-num_external, &pmap->stats.external); + if (num_internal) + OSAddAtomic(-num_internal, &pmap->stats.internal); + if (num_reusable) + OSAddAtomic(-num_reusable, &pmap->stats.reusable); + if (num_compressed) + OSAddAtomic64(-num_compressed, &pmap->stats.compressed); + /* ... and ledgers */ + pmap_ledger_debit(pmap, task_ledgers.wired_mem, machine_ptob(num_unwired)); + pmap_ledger_debit(pmap, task_ledgers.internal, machine_ptob(num_internal)); + pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, machine_ptob(num_alt_internal)); + pmap_ledger_debit(pmap, task_ledgers.alternate_accounting_compressed, machine_ptob(num_alt_compressed)); + pmap_ledger_debit(pmap, task_ledgers.internal_compressed, machine_ptob(num_compressed)); + /* make needed adjustments to phys_footprint */ + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, + machine_ptob((num_internal - + num_alt_internal) + + (num_compressed - + num_alt_compressed))); + } + + /* flush the ptable entries we have written */ + if (num_pte_changed > 0) + FLUSH_PTE_RANGE(bpte, epte); + + return num_pte_changed; +} + + +/* + * Remove the given range of addresses + * from the specified map. + * + * It is assumed that the start and end are properly + * rounded to the hardware page size. + */ +void +pmap_remove( + pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end) +{ + pmap_remove_options(pmap, start, end, PMAP_OPTIONS_REMOVE); +} + +static int +pmap_remove_options_internal(pmap_t pmap, +vm_map_address_t start, +vm_map_address_t end, +int options) +{ + int remove_count = 0; + pt_entry_t *bpte, *epte; + pt_entry_t *pte_p; + tt_entry_t *tte_p; + uint32_t rmv_spte=0; + + PMAP_LOCK(pmap); + + tte_p = pmap_tte(pmap, start); + + if (tte_p == (tt_entry_t *) NULL) { + goto done; + } + + if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte = &pte_p[ptenum(start)]; + epte = bpte + ((end - start) >> ARM_TT_LEAF_SHIFT); + + remove_count += pmap_remove_range_options(pmap, start, bpte, epte, + &rmv_spte, options); + +#if (__ARM_VMSA__ == 7) + if (rmv_spte && (ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt == 0) && + (pmap != kernel_pmap) && (pmap->nested == FALSE)) { + pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L1_LEVEL); + flush_mmu_tlb_entry((start & ~ARM_TT_L1_OFFMASK) | (pmap->asid & 0xff)); + } +#else + if (rmv_spte && (ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt == 0) && + (pmap != kernel_pmap) && (pmap->nested == FALSE)) { + pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L2_LEVEL); + flush_mmu_tlb_entry(tlbi_addr(start & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); + } +#endif + } + +done: + PMAP_UNLOCK(pmap); + + return remove_count; +} + +void +pmap_remove_options( + pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + int options) +{ + int remove_count = 0; + vm_map_address_t va; + + if (pmap == PMAP_NULL) + return; + + PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(start), + VM_KERNEL_ADDRHIDE(end)); + +#if MACH_ASSERT + if ((start|end) & PAGE_MASK) { + panic("pmap_remove_options() pmap %p start 0x%llx end 0x%llx\n", + pmap, (uint64_t)start, (uint64_t)end); + } + if ((end < start) || (start < pmap->min) || (end > pmap->max)) { + panic("pmap_remove_options(): invalid address range, pmap=%p, start=0x%llx, end=0x%llx\n", + pmap, (uint64_t)start, (uint64_t)end); + } +#endif + + /* + * Invalidate the translation buffer first + */ + va = start; + while (va < end) { + vm_map_address_t l; + +#if (__ARM_VMSA__ == 7) + l = ((va + ARM_TT_L1_SIZE) & ~ARM_TT_L1_OFFMASK); +#else + l = ((va + ARM_TT_L2_SIZE) & ~ARM_TT_L2_OFFMASK); +#endif + if (l > end) + l = end; + + remove_count += pmap_remove_options_internal(pmap, va, l, options); + + va = l; + } + + + if (remove_count > 0) + PMAP_UPDATE_TLBS(pmap, start, end); + + PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END); +} + + +/* + * Remove phys addr if mapped in specified map + */ +void +pmap_remove_some_phys( + __unused pmap_t map, + __unused ppnum_t pn) +{ + /* Implement to support working set code */ +} + + +void +pmap_set_pmap( + pmap_t pmap, +#if !__ARM_USER_PROTECT__ + __unused +#endif + thread_t thread) +{ + pmap_switch(pmap); +#if __ARM_USER_PROTECT__ + if (pmap->tte_index_max == NTTES) { + thread->machine.uptw_ttc = 2; + thread->machine.uptw_ttb = ((unsigned int) pmap->ttep) | TTBR_SETUP; + } else { + thread->machine.uptw_ttc = 1; \ + thread->machine.uptw_ttb = ((unsigned int) pmap->ttep ) | TTBR_SETUP; + } + thread->machine.asid = pmap->asid; +#endif +} + +static void +pmap_flush_core_tlb_asid(pmap_t pmap) +{ +#if (__ARM_VMSA__ == 7) + flush_core_tlb_asid(pmap->asid); +#else + flush_core_tlb_asid(((uint64_t) pmap->asid) << TLBI_ASID_SHIFT); +#endif +} + +static void +pmap_switch_internal( + pmap_t pmap) +{ + pmap_cpu_data_t *cpu_data_ptr = pmap_get_cpu_data(); + uint32_t last_asid_high_bits, asid_high_bits; + pmap_t cur_pmap; + pmap_t cur_user_pmap; + boolean_t do_asid_flush = FALSE; + +#if (__ARM_VMSA__ == 7) + if (not_in_kdp) + simple_lock(&pmap->tt1_lock); +#endif + + cur_pmap = current_pmap(); + cur_user_pmap = cpu_data_ptr->cpu_user_pmap; + + /* Paranoia. */ + assert(pmap->asid < (sizeof(cpu_data_ptr->cpu_asid_high_bits) / sizeof(*cpu_data_ptr->cpu_asid_high_bits))); + + /* Extract the "virtual" bits of the ASIDs (which could cause us to alias). */ + asid_high_bits = pmap->vasid >> ARM_ASID_SHIFT; + last_asid_high_bits = (uint32_t) cpu_data_ptr->cpu_asid_high_bits[pmap->asid]; + + if (asid_high_bits != last_asid_high_bits) { + /* + * If the virtual ASID of the new pmap does not match the virtual ASID + * last seen on this CPU for the physical ASID (that was a mouthful), + * then this switch runs the risk of aliasing. We need to flush the + * TLB for this phyiscal ASID in this case. + */ + cpu_data_ptr->cpu_asid_high_bits[pmap->asid] = (uint8_t) asid_high_bits; + do_asid_flush = TRUE; + } + + if ((cur_user_pmap == cur_pmap) && (cur_pmap == pmap)) { + if (cpu_data_ptr->cpu_user_pmap_stamp == pmap->stamp) { + pmap_switch_user_ttb_internal(pmap); + +#if (__ARM_VMSA__ == 7) + if (not_in_kdp) + simple_unlock(&pmap->tt1_lock); +#endif + + if (do_asid_flush) { + pmap_flush_core_tlb_asid(pmap); + } + + return; + } else + cur_user_pmap = NULL; + } else if ((cur_user_pmap == pmap) && (cpu_data_ptr->cpu_user_pmap_stamp != pmap->stamp)) + cur_user_pmap = NULL; + + pmap_switch_user_ttb_internal(pmap); + + if (do_asid_flush) { + pmap_flush_core_tlb_asid(pmap); + } + +#if (__ARM_VMSA__ == 7) + if (not_in_kdp) + simple_unlock(&pmap->tt1_lock); +#else + if (pmap != kernel_pmap) { + + if (cur_user_pmap != PMAP_NULL) { + /* + * We have a low-address global mapping for the commpage + * for 32-bit processes; flush it if we switch to a 64-bot + * process. + */ + if (pmap_is_64bit(pmap) && !pmap_is_64bit(cur_user_pmap)) { + pmap_sharedpage_flush_32_to_64(); + } + + } else + flush_core_tlb(); + } +#endif +} + +void +pmap_switch( + pmap_t pmap) +{ + pmap_switch_internal(pmap); +} + +void +pmap_page_protect( + ppnum_t ppnum, + vm_prot_t prot) +{ + pmap_page_protect_options(ppnum, prot, 0, NULL); +} + +/* + * Routine: pmap_page_protect_options + * + * Function: + * Lower the permission for all mappings to a given + * page. + */ +static void +pmap_page_protect_options_internal( + ppnum_t ppnum, + vm_prot_t prot, + unsigned int options) +{ + pmap_paddr_t phys = ptoa(ppnum); + pv_entry_t **pv_h; + pv_entry_t *pve_p; + pv_entry_t *pveh_p; + pv_entry_t *pvet_p; + pt_entry_t *pte_p; + int pai; + boolean_t remove; + boolean_t set_NX; + unsigned int pvh_cnt = 0; + + assert(ppnum != vm_page_fictitious_addr); + + /* Only work with managed pages. */ + if (!pa_valid(phys)) { + return; + } + + /* + * Determine the new protection. + */ + switch (prot) { + case VM_PROT_ALL: + return; /* nothing to do */ + case VM_PROT_READ: + case VM_PROT_READ | VM_PROT_EXECUTE: + remove = FALSE; + break; + default: + remove = TRUE; + break; + } + + pai = (int)pa_index(phys); + LOCK_PVH(pai); + pv_h = pai_to_pvh(pai); + + pte_p = PT_ENTRY_NULL; + pve_p = PV_ENTRY_NULL; + pveh_p = PV_ENTRY_NULL; + pvet_p = PV_ENTRY_NULL; + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + pte_p = pvh_ptep(pv_h); + } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + pve_p = pvh_list(pv_h); + pveh_p = pve_p; + } + + while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) { + vm_map_address_t va; + pmap_t pmap; + pt_entry_t tmplate; + boolean_t update = FALSE; + + if (pve_p != PV_ENTRY_NULL) + pte_p = pve_get_ptep(pve_p); + + pmap = ptep_get_pmap(pte_p); + va = ptep_get_va(pte_p); + + if (pte_p == PT_ENTRY_NULL) { + panic("pmap_page_protect: pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, va=0x%llx ppnum: 0x%x\n", + pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)va, ppnum); + } else if ((pmap == NULL) || (atop(pte_to_pa(*pte_p)) != ppnum)) { +#if MACH_ASSERT + if (kern_feature_override(KF_PMAPV_OVRD) == FALSE) { + + pv_entry_t *check_pve_p = pveh_p; + while (check_pve_p != PV_ENTRY_NULL) { + if ((check_pve_p != pve_p) && (pve_get_ptep(check_pve_p) == pte_p)) { + panic("pmap_page_protect: duplicate pve entry pte_p=%p pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, pte=0x%llx, va=0x%llx ppnum: 0x%x\n", + pte_p, pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)*pte_p, (uint64_t)va, ppnum); + } + check_pve_p = PVE_NEXT_PTR(pve_next(check_pve_p)); + } + } +#endif + panic("pmap_page_protect: bad pve entry pte_p=%p pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, pte=0x%llx, va=0x%llx ppnum: 0x%x\n", + pte_p, pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)*pte_p, (uint64_t)va, ppnum); + } + +#if DEVELOPMENT || DEBUG + if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled) +#else + if ((prot & VM_PROT_EXECUTE)) +#endif + set_NX = FALSE; + else + set_NX = TRUE; + + /* Remove the mapping if new protection is NONE */ + if (remove) { + boolean_t is_altacct = FALSE; + + if (IS_ALTACCT_PAGE(pai, pve_p)) { + is_altacct = TRUE; + } else { + is_altacct = FALSE; + } + + if (pte_is_wired(*pte_p)) { + pte_set_wired(pte_p, 0); + if (pmap != kernel_pmap) { + pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE); + OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count); + } + } + + if (*pte_p != ARM_PTE_TYPE_FAULT && + pmap != kernel_pmap && + (options & PMAP_OPTIONS_COMPRESSOR) && + IS_INTERNAL_PAGE(pai)) { + assert(!ARM_PTE_IS_COMPRESSED(*pte_p)); + /* mark this PTE as having been "compressed" */ + tmplate = ARM_PTE_COMPRESSED; + if (is_altacct) { + tmplate |= ARM_PTE_COMPRESSED_ALT; + is_altacct = TRUE; + } + } else { + tmplate = ARM_PTE_TYPE_FAULT; + } + + if ((*pte_p != ARM_PTE_TYPE_FAULT) && + tmplate == ARM_PTE_TYPE_FAULT && + (pmap != kernel_pmap)) { + if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt)) <= 0) + panic("pmap_page_protect_options(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p); + } + + if (*pte_p != tmplate) { + WRITE_PTE(pte_p, tmplate); + update = TRUE; + } + pvh_cnt++; + pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE); + OSAddAtomic(-1, (SInt32 *) &pmap->stats.resident_count); + +#if MACH_ASSERT + /* + * We only ever compress internal pages. + */ + if (options & PMAP_OPTIONS_COMPRESSOR) { + assert(IS_INTERNAL_PAGE(pai)); + } +#endif + + if (pmap != kernel_pmap) { + if (IS_REUSABLE_PAGE(pai) && + IS_INTERNAL_PAGE(pai) && + !is_altacct) { + assert(pmap->stats.reusable > 0); + OSAddAtomic(-1, &pmap->stats.reusable); + } else if (IS_INTERNAL_PAGE(pai)) { + assert(pmap->stats.internal > 0); + OSAddAtomic(-1, &pmap->stats.internal); + } else { + assert(pmap->stats.external > 0); + OSAddAtomic(-1, &pmap->stats.external); + } + if ((options & PMAP_OPTIONS_COMPRESSOR) && + IS_INTERNAL_PAGE(pai)) { + /* adjust "compressed" stats */ + OSAddAtomic64(+1, &pmap->stats.compressed); + PMAP_STATS_PEAK(pmap->stats.compressed); + pmap->stats.compressed_lifetime++; + } + + if (IS_ALTACCT_PAGE(pai, pve_p)) { + assert(IS_INTERNAL_PAGE(pai)); + pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE); + pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE); + if (options & PMAP_OPTIONS_COMPRESSOR) { + pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE); + pmap_ledger_credit(pmap, task_ledgers.alternate_accounting_compressed, PAGE_SIZE); + } + + /* + * Cleanup our marker before + * we free this pv_entry. + */ + CLR_ALTACCT_PAGE(pai, pve_p); + + } else if (IS_REUSABLE_PAGE(pai)) { + assert(IS_INTERNAL_PAGE(pai)); + if (options & PMAP_OPTIONS_COMPRESSOR) { + pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE); + /* was not in footprint, but is now */ + pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } + + } else if (IS_INTERNAL_PAGE(pai)) { + pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE); + + /* + * Update all stats related to physical footprint, which only + * deals with internal pages. + */ + if (options & PMAP_OPTIONS_COMPRESSOR) { + /* + * This removal is only being done so we can send this page to + * the compressor; therefore it mustn't affect total task footprint. + */ + pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE); + } else { + /* + * This internal page isn't going to the compressor, so adjust stats to keep + * phys_footprint up to date. + */ + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } + } else { + /* external page: no impact on ledgers */ + } + } + + if (pve_p != PV_ENTRY_NULL) { + assert(pve_next(pve_p) == PVE_NEXT_PTR(pve_next(pve_p))); + } + + } else { + pt_entry_t spte; + + spte = *pte_p; + + if (pmap == kernel_pmap) + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA)); + else + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO)); + + pte_set_ffr(tmplate, 0); + +#if (__ARM_VMSA__ == 7) + if (set_NX) { + tmplate |= ARM_PTE_NX; + } else { + /* + * While the naive implementation of this would serve to add execute + * permission, this is not how the VM uses this interface, or how + * x86_64 implements it. So ignore requests to add execute permissions. + */ +#if 0 + tmplate &= ~ARM_PTE_NX; +#else + ; +#endif + } +#else + if (set_NX) + tmplate |= ARM_PTE_NX | ARM_PTE_PNX; + else { + /* + * While the naive implementation of this would serve to add execute + * permission, this is not how the VM uses this interface, or how + * x86_64 implements it. So ignore requests to add execute permissions. + */ +#if 0 + if (pmap == kernel_pmap) { + tmplate &= ~ARM_PTE_PNX; + tmplate |= ARM_PTE_NX; + } else { + tmplate &= ~ARM_PTE_NX; + tmplate |= ARM_PTE_PNX; + } +#else + ; +#endif + } +#endif + + + if (*pte_p != ARM_PTE_TYPE_FAULT && + !ARM_PTE_IS_COMPRESSED(*pte_p) && + *pte_p != tmplate) { + WRITE_PTE(pte_p, tmplate); + update = TRUE; + } + } + + /* Invalidate TLBs for all CPUs using it */ + if (update) + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); + + pte_p = PT_ENTRY_NULL; + pvet_p = pve_p; + if (pve_p != PV_ENTRY_NULL) { + pvet_p = pve_p; + if (remove) { + assert(pve_next(pve_p) == PVE_NEXT_PTR(pve_next(pve_p))); + } + pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + } + } + + /* if we removed a bunch of entries, take care of them now */ + if (remove) { + pvh_update_head(pv_h, PV_ENTRY_NULL, PVH_TYPE_NULL); + } + + UNLOCK_PVH(pai); + + if (remove && (pveh_p != PV_ENTRY_NULL)) { + pv_list_free(pveh_p, pvet_p, pvh_cnt); + } +} + +void +pmap_page_protect_options( + ppnum_t ppnum, + vm_prot_t prot, + unsigned int options, + __unused void *arg) +{ + pmap_paddr_t phys = ptoa(ppnum); + + assert(ppnum != vm_page_fictitious_addr); + + /* Only work with managed pages. */ + if (!pa_valid(phys)) + return; + + /* + * Determine the new protection. + */ + if (prot == VM_PROT_ALL) { + return; /* nothing to do */ + } + + PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, ppnum, prot); + + pmap_page_protect_options_internal(ppnum, prot, options); + + PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END); +} + +/* + * Indicates if the pmap layer enforces some additional restrictions on the + * given set of protections. + */ +bool pmap_has_prot_policy(__unused vm_prot_t prot) +{ + return FALSE; +} + +/* + * Set the physical protection on the + * specified range of this map as requested. + * VERY IMPORTANT: Will not increase permissions. + * VERY IMPORTANT: Only pmap_enter() is allowed to grant permissions. + */ +void +pmap_protect( + pmap_t pmap, + vm_map_address_t b, + vm_map_address_t e, + vm_prot_t prot) +{ + pmap_protect_options(pmap, b, e, prot, 0, NULL); +} + +static void +pmap_protect_options_internal(pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + vm_prot_t prot, + unsigned int options, + __unused void *args) +{ + tt_entry_t *tte_p; + pt_entry_t *bpte_p, *epte_p; + pt_entry_t *pte_p; + boolean_t set_NX = TRUE; +#if (__ARM_VMSA__ > 7) + boolean_t set_XO = FALSE; +#endif + boolean_t should_have_removed = FALSE; + +#ifndef __ARM_IC_NOALIAS_ICACHE__ + boolean_t InvalidatePoU_Icache_Done = FALSE; +#endif + +#if DEVELOPMENT || DEBUG + if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) { + if ((prot & VM_PROT_ALL) == VM_PROT_NONE) { + should_have_removed = TRUE; + } + } else +#endif + { + /* Determine the new protection. */ + switch (prot) { +#if (__ARM_VMSA__ > 7) + case VM_PROT_EXECUTE: + set_XO = TRUE; + /* fall through */ +#endif + case VM_PROT_READ: + case VM_PROT_READ | VM_PROT_EXECUTE: + break; + case VM_PROT_READ | VM_PROT_WRITE: + case VM_PROT_ALL: + return; /* nothing to do */ + default: + should_have_removed = TRUE; + } + } + + if (should_have_removed) { + panic("%s: should have been a remove operation, " + "pmap=%p, start=%p, end=%p, prot=%#x, options=%#x, args=%p", + __FUNCTION__, + pmap, (void *)start, (void *)end, prot, options, args); + } + +#if DEVELOPMENT || DEBUG + if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled) +#else + if ((prot & VM_PROT_EXECUTE)) +#endif + { + set_NX = FALSE; + } else { + set_NX = TRUE; + } + + PMAP_LOCK(pmap); + tte_p = pmap_tte(pmap, start); + + if ((tte_p != (tt_entry_t *) NULL) && (*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + bpte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte_p = &bpte_p[ptenum(start)]; + epte_p = bpte_p + arm_atop(end - start); + pte_p = bpte_p; + + for (pte_p = bpte_p; + pte_p < epte_p; + pte_p += PAGE_SIZE/ARM_PGBYTES) { + pt_entry_t spte; +#if DEVELOPMENT || DEBUG + boolean_t force_write = FALSE; +#endif + + spte = *pte_p; + + if ((spte == ARM_PTE_TYPE_FAULT) || + ARM_PTE_IS_COMPRESSED(spte)) { + continue; + } + + pmap_paddr_t pa; + int pai=0; + boolean_t managed=FALSE; + + while (!managed) { + /* + * It may be possible for the pte to transition from managed + * to unmanaged in this timeframe; for now, elide the assert. + * We should break out as a consequence of checking pa_valid. + */ + // assert(!ARM_PTE_IS_COMPRESSED(spte)); + pa = pte_to_pa(spte); + if (!pa_valid(pa)) + break; + pai = (int)pa_index(pa); + LOCK_PVH(pai); + spte = *pte_p; + pa = pte_to_pa(spte); + if (pai == (int)pa_index(pa)) { + managed =TRUE; + break; // Leave the PVH locked as we will unlock it after we free the PTE + } + UNLOCK_PVH(pai); + } + + if ((spte == ARM_PTE_TYPE_FAULT) || + ARM_PTE_IS_COMPRESSED(spte)) { + continue; + } + + pt_entry_t tmplate; + + if (pmap == kernel_pmap) { +#if DEVELOPMENT || DEBUG + if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) && (prot & VM_PROT_WRITE)) { + force_write = TRUE; + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWNA)); + } else +#endif + { + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA)); + } + } else { +#if DEVELOPMENT || DEBUG + if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) && (prot & VM_PROT_WRITE)) { + force_write = TRUE; + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWRW)); + } else +#endif + { + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO)); + } + } + + /* + * XXX Removing "NX" would + * grant "execute" access + * immediately, bypassing any + * checks VM might want to do + * in its soft fault path. + * pmap_protect() and co. are + * not allowed to increase + * access permissions. + */ +#if (__ARM_VMSA__ == 7) + if (set_NX) + tmplate |= ARM_PTE_NX; + else { + /* do NOT clear "NX"! */ + } +#else + if (set_NX) + tmplate |= ARM_PTE_NX | ARM_PTE_PNX; + else { + if (pmap == kernel_pmap) { + /* + * TODO: Run CS/Monitor checks here; + * should we be clearing PNX here? Is + * this just for dtrace? + */ + tmplate &= ~ARM_PTE_PNX; + tmplate |= ARM_PTE_NX; + } else { + /* do NOT clear "NX"! */ + tmplate |= ARM_PTE_PNX; + if (set_XO) { + tmplate &= ~ARM_PTE_APMASK; + tmplate |= ARM_PTE_AP(AP_RONA); + } + } + } +#endif + +#if DEVELOPMENT || DEBUG + if (force_write) { + /* + * TODO: Run CS/Monitor checks here. + */ + if (managed) { + /* + * We are marking the page as writable, + * so we consider it to be modified and + * referenced. + */ + pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED); + tmplate |= ARM_PTE_AF; + + if (IS_REFFAULT_PAGE(pai)) { + CLR_REFFAULT_PAGE(pai); + } + + if (IS_MODFAULT_PAGE(pai)) { + CLR_MODFAULT_PAGE(pai); + } + } + } else if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) { + /* + * An immediate request for anything other than + * write should still mark the page as + * referenced if managed. + */ + if (managed) { + pa_set_bits(pa, PP_ATTR_REFERENCED); + tmplate |= ARM_PTE_AF; + + if (IS_REFFAULT_PAGE(pai)) { + CLR_REFFAULT_PAGE(pai); + } + } + } +#endif + + /* We do not expect to write fast fault the entry. */ + pte_set_ffr(tmplate, 0); + + /* TODO: Doesn't this need to worry about PNX? */ + if (((spte & ARM_PTE_NX) == ARM_PTE_NX) && (prot & VM_PROT_EXECUTE)) { + CleanPoU_DcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE); +#ifdef __ARM_IC_NOALIAS_ICACHE__ + InvalidatePoU_IcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE); +#else + if (!InvalidatePoU_Icache_Done) { + InvalidatePoU_Icache(); + InvalidatePoU_Icache_Done = TRUE; + } +#endif + } + + WRITE_PTE_FAST(pte_p, tmplate); + + if (managed) { + ASSERT_PVH_LOCKED(pai); + UNLOCK_PVH(pai); + } + } + + FLUSH_PTE_RANGE(bpte_p, epte_p); + PMAP_UPDATE_TLBS(pmap, start, end); + } + + PMAP_UNLOCK(pmap); +} + +void +pmap_protect_options( + pmap_t pmap, + vm_map_address_t b, + vm_map_address_t e, + vm_prot_t prot, + unsigned int options, + __unused void *args) +{ + vm_map_address_t l, beg; + + if ((b|e) & PAGE_MASK) { + panic("pmap_protect_options() pmap %p start 0x%llx end 0x%llx\n", + pmap, (uint64_t)b, (uint64_t)e); + } + +#if DEVELOPMENT || DEBUG + if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) { + if ((prot & VM_PROT_ALL) == VM_PROT_NONE) { + pmap_remove_options(pmap, b, e, options); + return; + } + } else +#endif + { + /* Determine the new protection. */ + switch (prot) { + case VM_PROT_EXECUTE: + case VM_PROT_READ: + case VM_PROT_READ | VM_PROT_EXECUTE: + break; + case VM_PROT_READ | VM_PROT_WRITE: + case VM_PROT_ALL: + return; /* nothing to do */ + default: + pmap_remove_options(pmap, b, e, options); + return; + } + } + + PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(b), + VM_KERNEL_ADDRHIDE(e)); + + beg = b; + + while (beg < e) { + l = ((beg + ARM_TT_TWIG_SIZE) & ~ARM_TT_TWIG_OFFMASK); + + if (l > e) + l = e; + + pmap_protect_options_internal(pmap, beg, l, prot, options, args); + + beg = l; + } + + PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END); +} + +/* Map a (possibly) autogenned block */ +kern_return_t +pmap_map_block( + pmap_t pmap, + addr64_t va, + ppnum_t pa, + uint32_t size, + vm_prot_t prot, + int attr, + __unused unsigned int flags) +{ + kern_return_t kr; + addr64_t original_va = va; + uint32_t page; + + for (page = 0; page < size; page++) { + kr = pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE); + + if (kr != KERN_SUCCESS) { + /* + * This will panic for now, as it is unclear that + * removing the mappings is correct. + */ + panic("%s: failed pmap_enter, " + "pmap=%p, va=%#llx, pa=%u, size=%u, prot=%#x, flags=%#x", + __FUNCTION__, + pmap, va, pa, size, prot, flags); + + pmap_remove(pmap, original_va, va - original_va); + return kr; + } + + va += PAGE_SIZE; + pa++; + } + + return KERN_SUCCESS; +} + +/* + * Insert the given physical page (p) at + * the specified virtual address (v) in the + * target physical map with the protection requested. + * + * If specified, the page will be wired down, meaning + * that the related pte can not be reclaimed. + * + * NB: This is the only routine which MAY NOT lazy-evaluate + * or lose information. That is, this routine must actually + * insert this page into the given map eventually (must make + * forward progress eventually. + */ +kern_return_t +pmap_enter( + pmap_t pmap, + vm_map_address_t v, + ppnum_t pn, + vm_prot_t prot, + vm_prot_t fault_type, + unsigned int flags, + boolean_t wired) +{ + return pmap_enter_options(pmap, v, pn, prot, fault_type, flags, wired, 0, NULL); +} + + +static inline void pmap_enter_pte(pmap_t pmap, pt_entry_t *pte_p, pt_entry_t pte, vm_map_address_t v) +{ + if (pmap != kernel_pmap && ((pte & ARM_PTE_WIRED) != (*pte_p & ARM_PTE_WIRED))) + { + SInt16 *ptd_wiredcnt_ptr = (SInt16 *)&(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].wiredcnt); + if (pte & ARM_PTE_WIRED) { + OSAddAtomic16(1, ptd_wiredcnt_ptr); + pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE); + OSAddAtomic(1, (SInt32 *) &pmap->stats.wired_count); + } else { + OSAddAtomic16(-1, ptd_wiredcnt_ptr); + pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE); + OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count); + } + } + if (*pte_p != ARM_PTE_TYPE_FAULT && + !ARM_PTE_IS_COMPRESSED(*pte_p)) { + WRITE_PTE(pte_p, pte); + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); + } else { + WRITE_PTE(pte_p, pte); + __asm__ volatile("isb"); + } +} + +static pt_entry_t +wimg_to_pte(unsigned int wimg) +{ + pt_entry_t pte; + + switch (wimg & (VM_WIMG_MASK)) { + case VM_WIMG_IO: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE); + pte |= ARM_PTE_NX | ARM_PTE_PNX; + break; + case VM_WIMG_POSTED: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED); + pte |= ARM_PTE_NX | ARM_PTE_PNX; + break; + case VM_WIMG_WCOMB: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITECOMB); + pte |= ARM_PTE_NX | ARM_PTE_PNX; + break; + case VM_WIMG_WTHRU: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITETHRU); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + break; + case VM_WIMG_COPYBACK: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + break; + case VM_WIMG_INNERWBACK: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_INNERWRITEBACK); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_INNER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + break; + default: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + } + + return pte; +} + +static kern_return_t +pmap_enter_options_internal( + pmap_t pmap, + vm_map_address_t v, + ppnum_t pn, + vm_prot_t prot, + vm_prot_t fault_type, + unsigned int flags, + boolean_t wired, + unsigned int options) +{ + pmap_paddr_t pa = ptoa(pn); + pt_entry_t pte; + pt_entry_t spte; + pt_entry_t *pte_p; + pv_entry_t *pve_p; + boolean_t set_NX; + boolean_t set_XO = FALSE; + boolean_t refcnt_updated; + unsigned int wimg_bits; + boolean_t was_compressed, was_alt_compressed; + + if ((v) & PAGE_MASK) { + panic("pmap_enter_options() pmap %p v 0x%llx\n", + pmap, (uint64_t)v); + } + + if ((prot & VM_PROT_EXECUTE) && (prot & VM_PROT_WRITE) && (pmap == kernel_pmap)) { + panic("pmap_enter_options(): WX request on kernel_pmap"); + } + +#if DEVELOPMENT || DEBUG + if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled) +#else + if ((prot & VM_PROT_EXECUTE)) +#endif + set_NX = FALSE; + else + set_NX = TRUE; + +#if (__ARM_VMSA__ > 7) + if (prot == VM_PROT_EXECUTE) { + set_XO = TRUE; + } +#endif + + assert(pn != vm_page_fictitious_addr); + + refcnt_updated = FALSE; + pve_p = PV_ENTRY_NULL; + was_compressed = FALSE; + was_alt_compressed = FALSE; + + PMAP_LOCK(pmap); + + /* + * Expand pmap to include this pte. Assume that + * pmap is always expanded to include enough hardware + * pages to map one VM page. + */ + while ((pte_p = pmap_pte(pmap, v)) == PT_ENTRY_NULL) { + /* Must unlock to expand the pmap. */ + PMAP_UNLOCK(pmap); + + kern_return_t kr=pmap_expand(pmap, v, options, PMAP_TT_MAX_LEVEL); + + if(kr) { + return kr; + } + + PMAP_LOCK(pmap); + } + + if (options & PMAP_OPTIONS_NOENTER) { + PMAP_UNLOCK(pmap); + return KERN_SUCCESS; + } + +Pmap_enter_retry: + + spte = *pte_p; + + if (ARM_PTE_IS_COMPRESSED(spte)) { + /* + * "pmap" should be locked at this point, so this should + * not race with another pmap_enter() or pmap_remove_range(). + */ + assert(pmap != kernel_pmap); + + /* one less "compressed" */ + OSAddAtomic64(-1, &pmap->stats.compressed); + pmap_ledger_debit(pmap, task_ledgers.internal_compressed, + PAGE_SIZE); + + was_compressed = TRUE; + if (spte & ARM_PTE_COMPRESSED_ALT) { + was_alt_compressed = TRUE; + pmap_ledger_debit( + pmap, + task_ledgers.alternate_accounting_compressed, + PAGE_SIZE); + } else { + /* was part of the footprint */ + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } + + /* clear "compressed" marker */ + /* XXX is it necessary since we're about to overwrite it ? */ + WRITE_PTE_FAST(pte_p, ARM_PTE_TYPE_FAULT); + spte = ARM_PTE_TYPE_FAULT; + + /* + * We're replacing a "compressed" marker with a valid PTE, + * so no change for "refcnt". + */ + refcnt_updated = TRUE; + } + + if ((spte != ARM_PTE_TYPE_FAULT) && (pte_to_pa(spte) != pa)) { + pmap_remove_range(pmap, v, pte_p, pte_p + 1, 0); + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); + } + + pte = pa_to_pte(pa) | ARM_PTE_TYPE; + + /* Don't bother tracking wiring for kernel PTEs. We use ARM_PTE_WIRED to track + * wired memory statistics for user pmaps, but kernel PTEs are assumed + * to be wired in nearly all cases. For VM layer functionality, the wired + * count in vm_page_t is sufficient. */ + if (wired && pmap != kernel_pmap) + pte |= ARM_PTE_WIRED; + +#if (__ARM_VMSA__ == 7) + if (set_NX) + pte |= ARM_PTE_NX; +#else + if (set_NX) + pte |= ARM_PTE_NX | ARM_PTE_PNX; + else { + if (pmap == kernel_pmap) { + pte |= ARM_PTE_NX; + } else { + pte |= ARM_PTE_PNX; + } + } +#endif + + if ((flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT))) + wimg_bits = (flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT)); + else + wimg_bits = pmap_cache_attributes(pn); + + pte |= wimg_to_pte(wimg_bits); + + if (pmap == kernel_pmap) { + if (prot & VM_PROT_WRITE) { + pte |= ARM_PTE_AP(AP_RWNA); + pa_set_bits(pa, PP_ATTR_MODIFIED | PP_ATTR_REFERENCED); + } else { + pte |= ARM_PTE_AP(AP_RONA); + pa_set_bits(pa, PP_ATTR_REFERENCED); + } +#if (__ARM_VMSA__ == 7) + if ((_COMM_PAGE_BASE_ADDRESS <= v) && (v < _COMM_PAGE_BASE_ADDRESS + _COMM_PAGE_AREA_LENGTH)) + pte = (pte & ~(ARM_PTE_APMASK)) | ARM_PTE_AP(AP_RORO); +#endif + } else { + if (!(pmap->nested)) { + pte |= ARM_PTE_NG; + } else if ((pmap->nested_region_asid_bitmap) + && (v >= pmap->nested_region_subord_addr) + && (v < (pmap->nested_region_subord_addr+pmap->nested_region_size))) { + + unsigned int index = (unsigned int)((v - pmap->nested_region_subord_addr) >> ARM_TT_TWIG_SHIFT); + + if ((pmap->nested_region_asid_bitmap) + && testbit(index, (int *)pmap->nested_region_asid_bitmap)) + pte |= ARM_PTE_NG; + } +#if MACH_ASSERT + if (pmap->nested_pmap != NULL) { + vm_map_address_t nest_vaddr; + pt_entry_t *nest_pte_p; + + nest_vaddr = v - pmap->nested_region_grand_addr + pmap->nested_region_subord_addr; + + if ((nest_vaddr >= pmap->nested_region_subord_addr) + && (nest_vaddr < (pmap->nested_region_subord_addr+pmap->nested_region_size)) + && ((nest_pte_p = pmap_pte(pmap->nested_pmap, nest_vaddr)) != PT_ENTRY_NULL) + && (*nest_pte_p != ARM_PTE_TYPE_FAULT) + && (!ARM_PTE_IS_COMPRESSED(*nest_pte_p)) + && (((*nest_pte_p) & ARM_PTE_NG) != ARM_PTE_NG)) { + unsigned int index = (unsigned int)((v - pmap->nested_region_subord_addr) >> ARM_TT_TWIG_SHIFT); + + if ((pmap->nested_pmap->nested_region_asid_bitmap) + && !testbit(index, (int *)pmap->nested_pmap->nested_region_asid_bitmap)) { + + panic("pmap_enter(): Global attribute conflict nest_pte_p=%p pmap=%p v=0x%llx spte=0x%llx \n", + nest_pte_p, pmap, (uint64_t)v, (uint64_t)*nest_pte_p); + } + } + + } +#endif + if (prot & VM_PROT_WRITE) { + + if (pa_valid(pa) && (!pa_test_bits(pa, PP_ATTR_MODIFIED))) { + if (fault_type & VM_PROT_WRITE) { + if (set_XO) + pte |= ARM_PTE_AP(AP_RWNA); + else + pte |= ARM_PTE_AP(AP_RWRW); + pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED); + } else { + if (set_XO) + pte |= ARM_PTE_AP(AP_RONA); + else + pte |= ARM_PTE_AP(AP_RORO); + pa_set_bits(pa, PP_ATTR_REFERENCED); + pte_set_ffr(pte, 1); + } + } else { + if (set_XO) + pte |= ARM_PTE_AP(AP_RWNA); + else + pte |= ARM_PTE_AP(AP_RWRW); + pa_set_bits(pa, PP_ATTR_REFERENCED); + } + } else { + + if (set_XO) + pte |= ARM_PTE_AP(AP_RONA); + else + pte |= ARM_PTE_AP(AP_RORO); + pa_set_bits(pa, PP_ATTR_REFERENCED); + } + } + + pte |= ARM_PTE_AF; + + volatile uint16_t *refcnt = NULL; + if (pmap != kernel_pmap) { + refcnt = &(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt); + /* Mark the PT page active to keep it from being reclaimed. We need this because + * we may drop the PVH and pmap locks later in pmap_enter() if we need to allocate + * a new PV entry. Note that setting this high bit (0x4000) can temporarily + * prevent the refcount underflow checks in pmap_page_protect() and pmap_remove() from + * working. If an underflow should happen during this window, we'll instead get a + * refcount along the lines of 0x3FFF, which will produce a later panic on non-zero + * refcount in pmap_pages_reclaim() or pmap_tt_deallocate(). */ + OSBitOrAtomic16(PT_DESC_REFCOUNT, refcnt); + if (!refcnt_updated) { + OSAddAtomic16(1, (volatile int16_t*)refcnt); + refcnt_updated = TRUE; + } + } + + if (pa_valid(pa)) { + pv_entry_t **pv_h; + int pai; + boolean_t is_altacct, is_internal; + + is_internal = FALSE; + is_altacct = FALSE; + + pai = (int)pa_index(pa); + pv_h = pai_to_pvh(pai); + + LOCK_PVH(pai); +Pmap_enter_loop: + + if (pte == *pte_p) { + /* + * This pmap_enter operation has been completed by another thread + * undo refcnt on pt and return + */ + if (refcnt != NULL) { + assert(refcnt_updated); + if (OSAddAtomic16(-1, (volatile int16_t*)refcnt) <= (int16_t)PT_DESC_REFCOUNT) + panic("pmap_enter(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p); + } + UNLOCK_PVH(pai); + goto Pmap_enter_return; + } else if (pte_to_pa(*pte_p) == pa) { + if (refcnt != NULL) { + assert(refcnt_updated); + if (OSAddAtomic16(-1, (volatile int16_t*)refcnt) <= (int16_t)PT_DESC_REFCOUNT) + panic("pmap_enter(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p); + } + pmap_enter_pte(pmap, pte_p, pte, v); + UNLOCK_PVH(pai); + goto Pmap_enter_return; + } else if (*pte_p != ARM_PTE_TYPE_FAULT) { + /* + * pte has been modified by another thread + * hold refcnt on pt and retry pmap_enter operation + */ + UNLOCK_PVH(pai); + goto Pmap_enter_retry; + } + if (pvh_test_type(pv_h, PVH_TYPE_NULL)) { + pvh_update_head(pv_h, pte_p, PVH_TYPE_PTEP); + /* 1st mapping: see what kind of page it is */ + if (options & PMAP_OPTIONS_INTERNAL) { + SET_INTERNAL_PAGE(pai); + } else { + CLR_INTERNAL_PAGE(pai); + } + if ((options & PMAP_OPTIONS_INTERNAL) && + (options & PMAP_OPTIONS_REUSABLE)) { + SET_REUSABLE_PAGE(pai); + } else { + CLR_REUSABLE_PAGE(pai); + } + if (pmap != kernel_pmap && + ((options & PMAP_OPTIONS_ALT_ACCT) || + PMAP_FOOTPRINT_SUSPENDED(pmap)) && + IS_INTERNAL_PAGE(pai)) { + /* + * Make a note to ourselves that this mapping is using alternative + * accounting. We'll need this in order to know which ledger to + * debit when the mapping is removed. + * + * The altacct bit must be set while the pv head is locked. Defer + * the ledger accounting until after we've dropped the lock. + */ + SET_ALTACCT_PAGE(pai, PV_ENTRY_NULL); + is_altacct = TRUE; + } else { + CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL); + } + } else { + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + pt_entry_t *pte1_p; + + /* + * convert pvh list from PVH_TYPE_PTEP to PVH_TYPE_PVEP + */ + pte1_p = pvh_ptep(pv_h); + if((pve_p == PV_ENTRY_NULL) && (!pv_alloc(pmap, pai, &pve_p))) { + goto Pmap_enter_loop; + } + pve_set_ptep(pve_p, pte1_p); + pve_p->pve_next = PV_ENTRY_NULL; + + if (IS_ALTACCT_PAGE(pai, PV_ENTRY_NULL)) { + /* + * transfer "altacct" from + * pp_attr to this pve + */ + CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL); + SET_ALTACCT_PAGE(pai, pve_p); + } + pvh_update_head(pv_h, pve_p, PVH_TYPE_PVEP); + pve_p = PV_ENTRY_NULL; + } + /* + * Set up pv_entry for this new mapping and then + * add it to the list for this physical page. + */ + if((pve_p == PV_ENTRY_NULL) && (!pv_alloc(pmap, pai, &pve_p))) { + goto Pmap_enter_loop; + } + pve_set_ptep(pve_p, pte_p); + pve_p->pve_next = PV_ENTRY_NULL; + + pvh_add(pv_h, pve_p); + + if (pmap != kernel_pmap && + ((options & PMAP_OPTIONS_ALT_ACCT) || + PMAP_FOOTPRINT_SUSPENDED(pmap)) && + IS_INTERNAL_PAGE(pai)) { + /* + * Make a note to ourselves that this + * mapping is using alternative + * accounting. We'll need this in order + * to know which ledger to debit when + * the mapping is removed. + * + * The altacct bit must be set while + * the pv head is locked. Defer the + * ledger accounting until after we've + * dropped the lock. + */ + SET_ALTACCT_PAGE(pai, pve_p); + is_altacct = TRUE; + } + + pve_p = PV_ENTRY_NULL; + } + + pmap_enter_pte(pmap, pte_p, pte, v); + + if (pmap != kernel_pmap) { + if (IS_REUSABLE_PAGE(pai) && + !is_altacct) { + assert(IS_INTERNAL_PAGE(pai)); + OSAddAtomic(+1, &pmap->stats.reusable); + PMAP_STATS_PEAK(pmap->stats.reusable); + } else if (IS_INTERNAL_PAGE(pai)) { + OSAddAtomic(+1, &pmap->stats.internal); + PMAP_STATS_PEAK(pmap->stats.internal); + is_internal = TRUE; + } else { + OSAddAtomic(+1, &pmap->stats.external); + PMAP_STATS_PEAK(pmap->stats.external); + } + } + + UNLOCK_PVH(pai); + + if (pmap != kernel_pmap) { + pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE); + + if (is_internal) { + /* + * Make corresponding adjustments to + * phys_footprint statistics. + */ + pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE); + if (is_altacct) { + /* + * If this page is internal and + * in an IOKit region, credit + * the task's total count of + * dirty, internal IOKit pages. + * It should *not* count towards + * the task's total physical + * memory footprint, because + * this entire region was + * already billed to the task + * at the time the mapping was + * created. + * + * Put another way, this is + * internal++ and + * alternate_accounting++, so + * net effect on phys_footprint + * is 0. That means: don't + * touch phys_footprint here. + */ + pmap_ledger_credit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE); + } else { + pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } + } + } + + OSAddAtomic(1, (SInt32 *) &pmap->stats.resident_count); + if (pmap->stats.resident_count > pmap->stats.resident_max) + pmap->stats.resident_max = pmap->stats.resident_count; + } else { + pmap_enter_pte(pmap, pte_p, pte, v); + } + +Pmap_enter_return: + +#if CONFIG_PGTRACE + if (pgtrace_enabled) { + // Clone and invalidate original mapping if eligible + for (int i = 0; i < PAGE_RATIO; i++) { + pmap_pgtrace_enter_clone(pmap, v + ARM_PGBYTES*i, 0, 0); + } + } +#endif + + if (pve_p != PV_ENTRY_NULL) + pv_free(pve_p); + + if (refcnt != NULL) + OSBitAndAtomic16(~PT_DESC_REFCOUNT, refcnt); // clear active marker + PMAP_UNLOCK(pmap); + + return KERN_SUCCESS; +} + +kern_return_t +pmap_enter_options( + pmap_t pmap, + vm_map_address_t v, + ppnum_t pn, + vm_prot_t prot, + vm_prot_t fault_type, + unsigned int flags, + boolean_t wired, + unsigned int options, + __unused void *arg) +{ + kern_return_t kr = KERN_FAILURE; + + PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(v), pn, prot); + + kr = pmap_enter_options_internal(pmap, v, pn, prot, fault_type, flags, wired, options); + + PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, kr); + + return kr; +} + +/* + * Routine: pmap_change_wiring + * Function: Change the wiring attribute for a map/virtual-address + * pair. + * In/out conditions: + * The mapping must already exist in the pmap. + */ +static void +pmap_change_wiring_internal( + pmap_t pmap, + vm_map_address_t v, + boolean_t wired) +{ + pt_entry_t *pte_p; + pmap_paddr_t pa; + + /* Don't bother tracking wiring for kernel PTEs. We use ARM_PTE_WIRED to track + * wired memory statistics for user pmaps, but kernel PTEs are assumed + * to be wired in nearly all cases. For VM layer functionality, the wired + * count in vm_page_t is sufficient. */ + if (pmap == kernel_pmap) { + return; + } + + PMAP_LOCK(pmap); + pte_p = pmap_pte(pmap, v); + assert(pte_p != PT_ENTRY_NULL); + pa = pte_to_pa(*pte_p); + if (pa_valid(pa)) + LOCK_PVH((int)pa_index(pa)); + + if (wired && !pte_is_wired(*pte_p)) { + pte_set_wired(pte_p, wired); + OSAddAtomic(+1, (SInt32 *) &pmap->stats.wired_count); + pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE); + } else if (!wired && pte_is_wired(*pte_p)) { + assert(pmap->stats.wired_count >= 1); + pte_set_wired(pte_p, wired); + OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count); + pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE); + } + + if (pa_valid(pa)) + UNLOCK_PVH((int)pa_index(pa)); + + PMAP_UNLOCK(pmap); +} + +void +pmap_change_wiring( + pmap_t pmap, + vm_map_address_t v, + boolean_t wired) +{ + pmap_change_wiring_internal(pmap, v, wired); +} + +static ppnum_t +pmap_find_phys_internal( + pmap_t pmap, + addr64_t va) +{ + ppnum_t ppn=0; + + if (pmap != kernel_pmap) { + PMAP_LOCK(pmap); + } + + ppn = pmap_vtophys(pmap, va); + + if (pmap != kernel_pmap) { + PMAP_UNLOCK(pmap); + } + + return ppn; +} + +ppnum_t +pmap_find_phys( + pmap_t pmap, + addr64_t va) +{ + pmap_paddr_t pa=0; + + if (pmap == kernel_pmap) + pa = mmu_kvtop(va); + else if ((current_thread()->map) && (pmap == vm_map_pmap(current_thread()->map))) + pa = mmu_uvtop(va); + + if (pa) return (ppnum_t)(pa >> PAGE_SHIFT); + + if (not_in_kdp) { + return pmap_find_phys_internal(pmap, va); + } else { + return pmap_vtophys(pmap, va); + } +} + +pmap_paddr_t +kvtophys( + vm_offset_t va) +{ + pmap_paddr_t pa; + + pa = mmu_kvtop(va); + if (pa) return pa; + pa = ((pmap_paddr_t)pmap_vtophys(kernel_pmap, va)) << PAGE_SHIFT; + if (pa) + pa |= (va & PAGE_MASK); + + return ((pmap_paddr_t)pa); +} + +ppnum_t +pmap_vtophys( + pmap_t pmap, + addr64_t va) +{ + if ((va < pmap->min) || (va >= pmap->max)) { + return 0; + } + +#if (__ARM_VMSA__ == 7) + tt_entry_t *tte_p, tte; + pt_entry_t *pte_p; + ppnum_t ppn; + + tte_p = pmap_tte(pmap, va); + if (tte_p == (tt_entry_t *) NULL) + return (ppnum_t) 0; + + tte = *tte_p; + if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + pte_p = (pt_entry_t *) ttetokv(tte) + ptenum(va); + ppn = (ppnum_t) atop(pte_to_pa(*pte_p) | (va & ARM_PGMASK)); +#if DEVELOPMENT || DEBUG + if (ppn != 0 && + ARM_PTE_IS_COMPRESSED(*pte_p)) { + panic("pmap_vtophys(%p,0x%llx): compressed pte_p=%p 0x%llx with ppn=0x%x\n", + pmap, va, pte_p, (uint64_t) (*pte_p), ppn); + } +#endif /* DEVELOPMENT || DEBUG */ + } else if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) + if ((tte & ARM_TTE_BLOCK_SUPER) == ARM_TTE_BLOCK_SUPER) + ppn = (ppnum_t) atop(suptte_to_pa(tte) | (va & ARM_TT_L1_SUPER_OFFMASK)); + else + ppn = (ppnum_t) atop(sectte_to_pa(tte) | (va & ARM_TT_L1_BLOCK_OFFMASK)); + else + ppn = 0; +#else + tt_entry_t *ttp; + tt_entry_t tte; + ppnum_t ppn=0; + + /* Level 0 currently unused */ + +#if __ARM64_TWO_LEVEL_PMAP__ + /* We have no L1 entry; go straight to the L2 entry */ + ttp = pmap_tt2e(pmap, va); + tte = *ttp; +#else + /* Get first-level (1GB) entry */ + ttp = pmap_tt1e(pmap, va); + tte = *ttp; + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) + return (ppn); + + tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, va)]; +#endif + if ((tte & ARM_TTE_VALID) != (ARM_TTE_VALID)) + return (ppn); + + if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) { + ppn = (ppnum_t) atop((tte & ARM_TTE_BLOCK_L2_MASK)| (va & ARM_TT_L2_OFFMASK)); + return(ppn); + } + tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, va)]; + ppn = (ppnum_t) atop((tte & ARM_PTE_MASK)| (va & ARM_TT_L3_OFFMASK)); +#endif + + return ppn; +} + +static vm_offset_t +pmap_extract_internal( + pmap_t pmap, + vm_map_address_t va) +{ + pmap_paddr_t pa=0; + ppnum_t ppn=0; + + if (pmap == NULL) { + return 0; + } + + PMAP_LOCK(pmap); + + ppn = pmap_vtophys(pmap, va); + + if (ppn != 0) + pa = ptoa(ppn)| ((va) & PAGE_MASK); + + PMAP_UNLOCK(pmap); + + return pa; +} + +/* + * Routine: pmap_extract + * Function: + * Extract the physical page address associated + * with the given map/virtual_address pair. + * + */ +vm_offset_t +pmap_extract( + pmap_t pmap, + vm_map_address_t va) +{ + pmap_paddr_t pa=0; + + if (pmap == kernel_pmap) + pa = mmu_kvtop(va); + else if (pmap == vm_map_pmap(current_thread()->map)) + pa = mmu_uvtop(va); + + if (pa) return pa; + + return pmap_extract_internal(pmap, va); +} + +/* + * pmap_init_pte_page - Initialize a page table page. + */ +void +pmap_init_pte_page( + pmap_t pmap, + pt_entry_t *pte_p, + vm_offset_t va, + unsigned int ttlevel, + boolean_t alloc_ptd) +{ + pt_desc_t *ptdp; + + ptdp = *(pt_desc_t **)pai_to_pvh(pa_index((((vm_offset_t)pte_p) - gVirtBase + gPhysBase))); + + if (ptdp == NULL) { + if (alloc_ptd) { + /* + * This path should only be invoked from arm_vm_init. If we are emulating 16KB pages + * on 4KB hardware, we may already have allocated a page table descriptor for a + * bootstrap request, so we check for an existing PTD here. + */ + ptdp = ptd_alloc(pmap); + *(pt_desc_t **)pai_to_pvh(pa_index((((vm_offset_t)pte_p) - gVirtBase + gPhysBase))) = ptdp; + } else { + panic("pmap_init_pte_page(): pte_p %p\n", pte_p); + } + } + + pmap_init_pte_page_internal(pmap, pte_p, va, ttlevel, &ptdp); +} + +/* + * pmap_init_pte_page_internal - Initialize page table page and page table descriptor + */ +void +pmap_init_pte_page_internal( + pmap_t pmap, + pt_entry_t *pte_p, + vm_offset_t va, + unsigned int ttlevel, + pt_desc_t **ptdp) +{ + bzero(pte_p, ARM_PGBYTES); + // below barrier ensures the page zeroing is visible to PTW before + // it is linked to the PTE of previous level + __asm__ volatile("DMB ST" : : : "memory"); + ptd_init(*ptdp, pmap, va, ttlevel, pte_p); +} + +/* + * pmap_init_pte_static_page - for static mappings to a known contiguous range of pa's + * Called from arm_vm_init(). + */ +void +pmap_init_pte_static_page( + __unused pmap_t pmap, + pt_entry_t * pte_p, + pmap_paddr_t pa) +{ +#if (__ARM_VMSA__ == 7) + unsigned int i; + pt_entry_t *pte_cur; + + for (i = 0, pte_cur = pte_p; + i < (ARM_PGBYTES / sizeof(*pte_p)); + i++, pa += PAGE_SIZE) { + if (pa >= avail_end) { + /* We don't want to map memory xnu does not own through this routine. */ + break; + } + + *pte_cur = pa_to_pte(pa) + | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_SH | ARM_PTE_AP(AP_RONA) + | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); + pte_cur++; + } +#else + unsigned int i; + pt_entry_t *pte_cur; + pt_entry_t template; + + template = ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_AP(AP_RONA) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) | ARM_PTE_NX; + + for (i = 0, pte_cur = pte_p; + i < (ARM_PGBYTES / sizeof(*pte_p)); + i++, pa += PAGE_SIZE) { + if (pa >= avail_end) { + /* We don't want to map memory xnu does not own through this routine. */ + break; + } + + /* TEST_PAGE_RATIO_4 may be pre-processor defined to 0 */ + __unreachable_ok_push + if (TEST_PAGE_RATIO_4) { + *pte_cur = pa_to_pte(pa) | template; + *(pte_cur+1) = pa_to_pte(pa+0x1000) | template; + *(pte_cur+2) = pa_to_pte(pa+0x2000) | template; + *(pte_cur+3) = pa_to_pte(pa+0x3000) | template; + pte_cur += 4; + } else { + *pte_cur = pa_to_pte(pa) | template; + pte_cur++; + } + __unreachable_ok_pop + } +#endif + bzero(pte_cur, ARM_PGBYTES - ((vm_offset_t)pte_cur - (vm_offset_t)pte_p)); +} + + +/* + * Routine: pmap_expand + * + * Expands a pmap to be able to map the specified virtual address. + * + * Allocates new memory for the default (COARSE) translation table + * entry, initializes all the pte entries to ARM_PTE_TYPE_FAULT and + * also allocates space for the corresponding pv entries. + * + * Nothing should be locked. + */ +static kern_return_t +pmap_expand( + pmap_t pmap, + vm_map_address_t v, + unsigned int options, + unsigned int level) +{ +#if (__ARM_VMSA__ == 7) + vm_offset_t pa; + tt_entry_t *tte_p; + tt_entry_t *tt_p; + unsigned int i; + + + while (tte_index(pmap, v) >= pmap->tte_index_max) { + tte_p = pmap_tt1_allocate(pmap, 2*ARM_PGBYTES, ((options & PMAP_OPTIONS_NOWAIT)? PMAP_TT_ALLOCATE_NOWAIT : 0)); + if (tte_p == (tt_entry_t *)0) + return KERN_RESOURCE_SHORTAGE; + + PMAP_LOCK(pmap); + if (pmap->tte_index_max > NTTES) { + pmap_tt1_deallocate(pmap, tte_p, 2*ARM_PGBYTES, PMAP_TT_DEALLOCATE_NOBLOCK); + PMAP_UNLOCK(pmap); + break; + } + + simple_lock(&pmap->tt1_lock); + for (i = 0; i < pmap->tte_index_max; i++) + tte_p[i] = pmap->tte[i]; + for (i = NTTES; i < 2*NTTES; i++) + tte_p[i] = ARM_TTE_TYPE_FAULT; + + pmap->prev_tte = pmap->tte; + pmap->tte = tte_p; + pmap->ttep = ml_static_vtop((vm_offset_t)pmap->tte); +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) pmap->tte, 2*NTTES * sizeof(tt_entry_t)); +#else + __builtin_arm_dsb(DSB_ISH); +#endif + pmap->tte_index_max = 2*NTTES; + pmap->stamp = hw_atomic_add(&pmap_stamp, 1); + + for (i = 0; i < NTTES; i++) + pmap->prev_tte[i] = ARM_TTE_TYPE_FAULT; +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) pmap->prev_tte, NTTES * sizeof(tt_entry_t)); +#else + __builtin_arm_dsb(DSB_ISH); +#endif + + simple_unlock(&pmap->tt1_lock); + PMAP_UNLOCK(pmap); + pmap_set_pmap(pmap, current_thread()); + + } + + if (level == 1) + return (KERN_SUCCESS); + + { + tt_entry_t *tte_next_p; + + PMAP_LOCK(pmap); + pa = 0; + if (pmap_pte(pmap, v) != PT_ENTRY_NULL) { + PMAP_UNLOCK(pmap); + return (KERN_SUCCESS); + } + tte_p = &pmap->tte[ttenum(v & ~ARM_TT_L1_PT_OFFMASK)]; + for (i = 0, tte_next_p = tte_p; i<4; i++) { + if (tte_to_pa(*tte_next_p)) { + pa = tte_to_pa(*tte_next_p); + break; + } + tte_next_p++; + } + pa = pa & ~PAGE_MASK; + if (pa) { + tte_p = &pmap->tte[ttenum(v)]; + *tte_p = pa_to_tte(pa) | (((v >> ARM_TT_L1_SHIFT) & 0x3) << 10) | ARM_TTE_TYPE_TABLE; +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) tte_p, sizeof(tt_entry_t)); +#endif + PMAP_UNLOCK(pmap); + return (KERN_SUCCESS); + } + PMAP_UNLOCK(pmap); + } + v = v & ~ARM_TT_L1_PT_OFFMASK; + + + while (pmap_pte(pmap, v) == PT_ENTRY_NULL) { + /* + * Allocate a VM page for the level 2 page table entries. + */ + while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L2_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) { + if(options & PMAP_OPTIONS_NOWAIT) { + return KERN_RESOURCE_SHORTAGE; + } + VM_PAGE_WAIT(); + } + + PMAP_LOCK(pmap); + /* + * See if someone else expanded us first + */ + if (pmap_pte(pmap, v) == PT_ENTRY_NULL) { + tt_entry_t *tte_next_p; + + pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v, PMAP_TT_L2_LEVEL, FALSE); + pa = kvtophys((vm_offset_t)tt_p); +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE); +#endif + tte_p = &pmap->tte[ttenum(v)]; + for (i = 0, tte_next_p = tte_p; i<4; i++) { + *tte_next_p = pa_to_tte(pa) | ARM_TTE_TYPE_TABLE; + tte_next_p++; + pa = pa +0x400; + } +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) tte_p, 4*sizeof(tt_entry_t)); +#endif + pa = 0x0ULL; + tt_p = (tt_entry_t *)NULL; + } + PMAP_UNLOCK(pmap); + if (tt_p != (tt_entry_t *)NULL) { + pmap_tt_deallocate(pmap, tt_p, PMAP_TT_L2_LEVEL); + tt_p = (tt_entry_t *)NULL; + } + } + return (KERN_SUCCESS); +#else + pmap_paddr_t pa; +#if __ARM64_TWO_LEVEL_PMAP__ + /* If we are using a two level page table, we'll start at L2. */ + unsigned int ttlevel = 2; +#else + /* Otherwise, we start at L1 (we use 3 levels by default). */ + unsigned int ttlevel = 1; +#endif + tt_entry_t *tte_p; + tt_entry_t *tt_p; + + pa = 0x0ULL; + tt_p = (tt_entry_t *)NULL; + + for (; ttlevel < level; ttlevel++) { + + PMAP_LOCK(pmap); + + if (ttlevel == 1) { + if ((pmap_tt2e(pmap, v) == PT_ENTRY_NULL)) { + PMAP_UNLOCK(pmap); + while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L2_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) { + if(options & PMAP_OPTIONS_NOWAIT) { + return KERN_RESOURCE_SHORTAGE; + } + VM_PAGE_WAIT(); + } + PMAP_LOCK(pmap); + if ((pmap_tt2e(pmap, v) == PT_ENTRY_NULL)) { + pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v, PMAP_TT_L2_LEVEL, FALSE); + pa = kvtophys((vm_offset_t)tt_p); + tte_p = pmap_tt1e( pmap, v); + *tte_p = (pa & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID; + pa = 0x0ULL; + tt_p = (tt_entry_t *)NULL; + if ((pmap == kernel_pmap) && (VM_MIN_KERNEL_ADDRESS < 0x00000000FFFFFFFFULL)) + current_pmap()->tte[v>>ARM_TT_L1_SHIFT] = kernel_pmap->tte[v>>ARM_TT_L1_SHIFT]; + } + + } + } else if (ttlevel == 2) { + if (pmap_tt3e(pmap, v) == PT_ENTRY_NULL) { + PMAP_UNLOCK(pmap); + while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L3_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) { + if(options & PMAP_OPTIONS_NOWAIT) { + return KERN_RESOURCE_SHORTAGE; + } + VM_PAGE_WAIT(); + } + PMAP_LOCK(pmap); + if ((pmap_tt3e(pmap, v) == PT_ENTRY_NULL)) { + pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v , PMAP_TT_L3_LEVEL, FALSE); + pa = kvtophys((vm_offset_t)tt_p); + tte_p = pmap_tt2e( pmap, v); + *tte_p = (pa & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID; + pa = 0x0ULL; + tt_p = (tt_entry_t *)NULL; + } + } + } + + PMAP_UNLOCK(pmap); + + if (tt_p != (tt_entry_t *)NULL) { + pmap_tt_deallocate(pmap, tt_p, ttlevel+1); + tt_p = (tt_entry_t *)NULL; + } + } + + return (KERN_SUCCESS); +#endif +} + +/* + * Routine: pmap_collect + * Function: + * Garbage collects the physical map system for + * pages which are no longer used. + * Success need not be guaranteed -- that is, there + * may well be pages which are not referenced, but + * others may be collected. + */ +void +pmap_collect(pmap_t pmap) +{ + if (pmap == PMAP_NULL) + return; + +#if 0 + PMAP_LOCK(pmap); + if ((pmap->nested == FALSE) && (pmap != kernel_pmap)) { + /* TODO: Scan for vm page assigned to top level page tables with no reference */ + } + PMAP_UNLOCK(pmap); +#endif + + return; +} + +/* + * Routine: pmap_gc + * Function: + * Pmap garbage collection + * Called by the pageout daemon when pages are scarce. + * + */ +void +pmap_gc( + void) +{ + pmap_t pmap, pmap_next; + boolean_t gc_wait; + + if (pmap_gc_allowed && + (pmap_gc_allowed_by_time_throttle || + pmap_gc_forced)) { + pmap_gc_forced = FALSE; + pmap_gc_allowed_by_time_throttle = FALSE; + simple_lock(&pmaps_lock); + pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&map_pmap_list)); + while (!queue_end(&map_pmap_list, (queue_entry_t)pmap)) { + if (!(pmap->gc_status & PMAP_GC_INFLIGHT)) + pmap->gc_status |= PMAP_GC_INFLIGHT; + simple_unlock(&pmaps_lock); + + pmap_collect(pmap); + + simple_lock(&pmaps_lock); + gc_wait = (pmap->gc_status & PMAP_GC_WAIT); + pmap->gc_status &= ~(PMAP_GC_INFLIGHT|PMAP_GC_WAIT); + pmap_next = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&pmap->pmaps)); + if (gc_wait) { + if (!queue_end(&map_pmap_list, (queue_entry_t)pmap_next)) + pmap_next->gc_status |= PMAP_GC_INFLIGHT; + simple_unlock(&pmaps_lock); + thread_wakeup((event_t) & pmap->gc_status); + simple_lock(&pmaps_lock); + } + pmap = pmap_next; + } + simple_unlock(&pmaps_lock); + } +} + +/* + * Called by the VM to reclaim pages that we can reclaim quickly and cheaply. + */ +void +pmap_release_pages_fast(void) +{ +} + +/* + * By default, don't attempt pmap GC more frequently + * than once / 1 minutes. + */ + +void +compute_pmap_gc_throttle( + void *arg __unused) +{ + pmap_gc_allowed_by_time_throttle = TRUE; +} + +/* + * pmap_attribute_cache_sync(vm_offset_t pa) + * + * Invalidates all of the instruction cache on a physical page and + * pushes any dirty data from the data cache for the same physical page + */ + +kern_return_t +pmap_attribute_cache_sync( + ppnum_t pp, + vm_size_t size, + __unused vm_machine_attribute_t attribute, + __unused vm_machine_attribute_val_t * value) +{ + if (size > PAGE_SIZE) { + panic("pmap_attribute_cache_sync size: 0x%llx\n", (uint64_t)size); + } else + cache_sync_page(pp); + + return KERN_SUCCESS; +} + +/* + * pmap_sync_page_data_phys(ppnum_t pp) + * + * Invalidates all of the instruction cache on a physical page and + * pushes any dirty data from the data cache for the same physical page + */ +void +pmap_sync_page_data_phys( + ppnum_t pp) +{ + cache_sync_page(pp); +} + +/* + * pmap_sync_page_attributes_phys(ppnum_t pp) + * + * Write back and invalidate all cachelines on a physical page. + */ +void +pmap_sync_page_attributes_phys( + ppnum_t pp) +{ + flush_dcache((vm_offset_t) (pp << PAGE_SHIFT), PAGE_SIZE, TRUE); +} + +#if CONFIG_COREDUMP +/* temporary workaround */ +boolean_t +coredumpok( + vm_map_t map, + vm_offset_t va) +{ + pt_entry_t *pte_p; + pt_entry_t spte; + + pte_p = pmap_pte(map->pmap, va); + if (0 == pte_p) + return FALSE; + spte = *pte_p; + return ((spte & ARM_PTE_ATTRINDXMASK) == ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT)); +} +#endif + +void +fillPage( + ppnum_t pn, + unsigned int fill) +{ + unsigned int *addr; + int count; + + addr = (unsigned int *) phystokv(ptoa(pn)); + count = PAGE_SIZE / sizeof(unsigned int); + while (count--) + *addr++ = fill; +} + +extern void mapping_set_mod(ppnum_t pn); + +void +mapping_set_mod( + ppnum_t pn) +{ + pmap_set_modify(pn); +} + +extern void mapping_set_ref(ppnum_t pn); + +void +mapping_set_ref( + ppnum_t pn) +{ + pmap_set_reference(pn); +} + +/* + * Clear specified attribute bits. + * + * Try to force an arm_fast_fault() for all mappings of + * the page - to force attributes to be set again at fault time. + * If the forcing succeeds, clear the cached bits at the head. + * Otherwise, something must have been wired, so leave the cached + * attributes alone. + */ +static void +phys_attribute_clear_internal( + ppnum_t pn, + unsigned int bits, + int options, + void *arg) +{ + pmap_paddr_t pa = ptoa(pn); + vm_prot_t allow_mode = VM_PROT_ALL; + + + if ((bits & PP_ATTR_MODIFIED) && + (options & PMAP_OPTIONS_NOFLUSH) && + (arg == NULL)) { + panic("phys_attribute_clear(0x%x,0x%x,0x%x,%p): " + "should not clear 'modified' without flushing TLBs\n", + pn, bits, options, arg); + } + + assert(pn != vm_page_fictitious_addr); + if (bits & PP_ATTR_REFERENCED) + allow_mode &= ~(VM_PROT_READ | VM_PROT_EXECUTE); + if (bits & PP_ATTR_MODIFIED) + allow_mode &= ~VM_PROT_WRITE; + + if (bits == PP_ATTR_NOENCRYPT) { + /* + * We short circuit this case; it should not need to + * invoke arm_force_fast_fault, so just clear and + * return. On ARM, this bit is just a debugging aid. + */ + pa_clear_bits(pa, bits); + return; + } + + if (arm_force_fast_fault_internal(pn, allow_mode, options)) + pa_clear_bits(pa, bits); + return; +} + +static void +phys_attribute_clear( + ppnum_t pn, + unsigned int bits, + int options, + void *arg) +{ + /* + * Do we really want this tracepoint? It will be extremely chatty. + * Also, should we have a corresponding trace point for the set path? + */ + PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, pn, bits); + + phys_attribute_clear_internal(pn, bits, options, arg); + + PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END); +} + +/* + * Set specified attribute bits. + * + * Set cached value in the pv head because we have + * no per-mapping hardware support for referenced and + * modify bits. + */ +static void +phys_attribute_set_internal( + ppnum_t pn, + unsigned int bits) +{ + pmap_paddr_t pa = ptoa(pn); + assert(pn != vm_page_fictitious_addr); + + + pa_set_bits(pa, bits); + + return; +} + +static void +phys_attribute_set( + ppnum_t pn, + unsigned int bits) +{ + phys_attribute_set_internal(pn, bits); +} + + +/* + * Check specified attribute bits. + * + * use the software cached bits (since no hw support). + */ +static boolean_t +phys_attribute_test( + ppnum_t pn, + unsigned int bits) +{ + pmap_paddr_t pa = ptoa(pn); + assert(pn != vm_page_fictitious_addr); + return pa_test_bits(pa, bits); +} + + +/* + * Set the modify/reference bits on the specified physical page. + */ +void +pmap_set_modify(ppnum_t pn) +{ + phys_attribute_set(pn, PP_ATTR_MODIFIED); +} + + +/* + * Clear the modify bits on the specified physical page. + */ +void +pmap_clear_modify( + ppnum_t pn) +{ + phys_attribute_clear(pn, PP_ATTR_MODIFIED, 0, NULL); +} + + +/* + * pmap_is_modified: + * + * Return whether or not the specified physical page is modified + * by any physical maps. + */ +boolean_t +pmap_is_modified( + ppnum_t pn) +{ + return phys_attribute_test(pn, PP_ATTR_MODIFIED); +} + + +/* + * Set the reference bit on the specified physical page. + */ +static void +pmap_set_reference( + ppnum_t pn) +{ + phys_attribute_set(pn, PP_ATTR_REFERENCED); +} + +/* + * Clear the reference bits on the specified physical page. + */ +void +pmap_clear_reference( + ppnum_t pn) +{ + phys_attribute_clear(pn, PP_ATTR_REFERENCED, 0, NULL); +} + + +/* + * pmap_is_referenced: + * + * Return whether or not the specified physical page is referenced + * by any physical maps. + */ +boolean_t +pmap_is_referenced( + ppnum_t pn) +{ + return phys_attribute_test(pn, PP_ATTR_REFERENCED); +} + +/* + * pmap_get_refmod(phys) + * returns the referenced and modified bits of the specified + * physical page. + */ +unsigned int +pmap_get_refmod( + ppnum_t pn) +{ + return (((phys_attribute_test(pn, PP_ATTR_MODIFIED)) ? VM_MEM_MODIFIED : 0) + | ((phys_attribute_test(pn, PP_ATTR_REFERENCED)) ? VM_MEM_REFERENCED : 0)); +} + +/* + * pmap_clear_refmod(phys, mask) + * clears the referenced and modified bits as specified by the mask + * of the specified physical page. + */ +void +pmap_clear_refmod_options( + ppnum_t pn, + unsigned int mask, + unsigned int options, + void *arg) +{ + unsigned int bits; + + bits = ((mask & VM_MEM_MODIFIED) ? PP_ATTR_MODIFIED : 0) | + ((mask & VM_MEM_REFERENCED) ? PP_ATTR_REFERENCED : 0); + phys_attribute_clear(pn, bits, options, arg); +} + +void +pmap_clear_refmod( + ppnum_t pn, + unsigned int mask) +{ + pmap_clear_refmod_options(pn, mask, 0, NULL); +} + +unsigned int +pmap_disconnect_options( + ppnum_t pn, + unsigned int options, + void *arg) +{ + if ((options & PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED)) { + /* + * On ARM, the "modified" bit is managed by software, so + * we know up-front if the physical page is "modified", + * without having to scan all the PTEs pointing to it. + * The caller should have made the VM page "busy" so noone + * should be able to establish any new mapping and "modify" + * the page behind us. + */ + if (pmap_is_modified(pn)) { + /* + * The page has been modified and will be sent to + * the VM compressor. + */ + options |= PMAP_OPTIONS_COMPRESSOR; + } else { + /* + * The page hasn't been modified and will be freed + * instead of compressed. + */ + } + } + + /* disconnect the page */ + pmap_page_protect_options(pn, 0, options, arg); + + /* return ref/chg status */ + return (pmap_get_refmod(pn)); +} + +/* + * Routine: + * pmap_disconnect + * + * Function: + * Disconnect all mappings for this page and return reference and change status + * in generic format. + * + */ +unsigned int +pmap_disconnect( + ppnum_t pn) +{ + pmap_page_protect(pn, 0); /* disconnect the page */ + return (pmap_get_refmod(pn)); /* return ref/chg status */ +} + +boolean_t +pmap_has_managed_page(ppnum_t first, ppnum_t last) +{ + if (ptoa(first) >= vm_last_phys) return (FALSE); + if (ptoa(last) < vm_first_phys) return (FALSE); + + return (TRUE); +} + +/* + * The state maintained by the noencrypt functions is used as a + * debugging aid on ARM. This incurs some overhead on the part + * of the caller. A special case check in phys_attribute_clear + * (the most expensive path) currently minimizes this overhead, + * but stubbing these functions out on RELEASE kernels yields + * further wins. + */ +boolean_t +pmap_is_noencrypt( + ppnum_t pn) +{ +#if DEVELOPMENT || DEBUG + boolean_t result = FALSE; + + if (!pa_valid(ptoa(pn))) return FALSE; + + result = (phys_attribute_test(pn, PP_ATTR_NOENCRYPT)); + + return result; +#else +#pragma unused(pn) + return FALSE; +#endif +} + +void +pmap_set_noencrypt( + ppnum_t pn) +{ +#if DEVELOPMENT || DEBUG + if (!pa_valid(ptoa(pn))) return; + + phys_attribute_set(pn, PP_ATTR_NOENCRYPT); +#else +#pragma unused(pn) +#endif +} + +void +pmap_clear_noencrypt( + ppnum_t pn) +{ +#if DEVELOPMENT || DEBUG + if (!pa_valid(ptoa(pn))) return; + + phys_attribute_clear(pn, PP_ATTR_NOENCRYPT, 0, NULL); +#else +#pragma unused(pn) +#endif +} + + +void +pmap_lock_phys_page(ppnum_t pn) +{ + int pai; + pmap_paddr_t phys = ptoa(pn); + + if (pa_valid(phys)) { + pai = (int)pa_index(phys); + LOCK_PVH(pai); + } else + simple_lock(&phys_backup_lock); +} + + +void +pmap_unlock_phys_page(ppnum_t pn) +{ + int pai; + pmap_paddr_t phys = ptoa(pn); + + if (pa_valid(phys)) { + pai = (int)pa_index(phys); + UNLOCK_PVH(pai); + } else + simple_unlock(&phys_backup_lock); +} + +static void +pmap_switch_user_ttb_internal( + pmap_t pmap) +{ +#if (__ARM_VMSA__ == 7) + pmap_cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = pmap_get_cpu_data(); + + if ((cpu_data_ptr->cpu_user_pmap != PMAP_NULL) + && (cpu_data_ptr->cpu_user_pmap != kernel_pmap)) { + unsigned int c; + + c = hw_atomic_sub((volatile uint32_t *)&cpu_data_ptr->cpu_user_pmap->cpu_ref, 1); + if ((c == 0) && (cpu_data_ptr->cpu_user_pmap->prev_tte != 0)) { + /* We saved off the old 1-page tt1 in pmap_expand() in case other cores were still using it. + * Now that the user pmap's cpu_ref is 0, we should be able to safely free it.*/ + tt_entry_t *tt_entry; + + tt_entry = cpu_data_ptr->cpu_user_pmap->prev_tte; + cpu_data_ptr->cpu_user_pmap->prev_tte = (tt_entry_t *) NULL; + pmap_tt1_deallocate(cpu_data_ptr->cpu_user_pmap, tt_entry, ARM_PGBYTES, PMAP_TT_DEALLOCATE_NOBLOCK); + } + } + cpu_data_ptr->cpu_user_pmap = pmap; + cpu_data_ptr->cpu_user_pmap_stamp = pmap->stamp; + (void) hw_atomic_add((volatile uint32_t *)&pmap->cpu_ref, 1); + +#if MACH_ASSERT && __ARM_USER_PROTECT__ + { + unsigned int ttbr0_val, ttbr1_val; + __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val)); + if (ttbr0_val != ttbr1_val) { + panic("Misaligned ttbr0 %08X\n", ttbr0_val); + } + } +#endif + if (pmap->tte_index_max == NTTES) { + /* Setting TTBCR.N for TTBR0 TTBR1 boundary at 0x40000000 */ + __asm__ volatile("mcr p15,0,%0,c2,c0,2" : : "r"(2)); + __asm__ volatile("isb"); +#if !__ARM_USER_PROTECT__ + set_mmu_ttb(pmap->ttep); +#endif + } else { +#if !__ARM_USER_PROTECT__ + set_mmu_ttb(pmap->ttep); +#endif + /* Setting TTBCR.N for TTBR0 TTBR1 boundary at 0x80000000 */ + __asm__ volatile("mcr p15,0,%0,c2,c0,2" : : "r"(1)); + __asm__ volatile("isb"); +#if MACH_ASSERT && __ARM_USER_PROTECT__ + if (pmap->ttep & 0x1000) { + panic("Misaligned ttbr0 %08X\n", pmap->ttep); + } +#endif + } + +#if !__ARM_USER_PROTECT__ + set_context_id(pmap->asid); +#endif +#else + + pmap_get_cpu_data()->cpu_user_pmap = pmap; + pmap_get_cpu_data()->cpu_user_pmap_stamp = pmap->stamp; + +#if !__arm64__ + set_context_id(pmap->asid); /* Not required */ +#endif + if (pmap == kernel_pmap) { + set_mmu_ttb(invalid_ttep & TTBR_BADDR_MASK); + } else { + set_mmu_ttb((pmap->ttep & TTBR_BADDR_MASK)|(((uint64_t)pmap->asid) << TTBR_ASID_SHIFT)); + } +#endif +} + +void +pmap_switch_user_ttb( + pmap_t pmap) +{ + pmap_switch_user_ttb_internal(pmap); +} + +/* + * Try to "intuit" whether we need to raise a VM_PROT_WRITE fault + * for the given address when a "swp" instruction raised the fault. + * We have to look at the existing pte for the address to see + * if it needs to get bumped, or just added. If just added, do it + * as a read-only mapping first (this could result in extra faults - + * but better that than extra copy-on-write evaluations). + */ + +#if (__ARM_VMSA__ == 7) +boolean_t +arm_swap_readable_type( + vm_map_address_t addr, + unsigned int spsr) +{ + int ap; + pt_entry_t spte; + pt_entry_t *ptep; + + ptep = pmap_pte(current_pmap(), addr); + if (ptep == PT_ENTRY_NULL) + return (FALSE); + + spte = *ptep; + if (spte == ARM_PTE_TYPE_FAULT || + ARM_PTE_IS_COMPRESSED(spte)) + return (FALSE); + + /* get the access permission bitmaps */ + /* (all subpages should be the same) */ + ap = (spte & ARM_PTE_APMASK); + + if (spsr & 0xf) { /* Supervisor mode */ + panic("arm_swap_readable_type supv"); + return TRUE; + } else { /* User mode */ + if ((ap == ARM_PTE_AP(AP_RWRW)) || (ap == ARM_PTE_AP(AP_RORO))) + return (FALSE); + else + return (TRUE); + } +} +#endif + +/* + * Routine: arm_force_fast_fault + * + * Function: + * Force all mappings for this page to fault according + * to the access modes allowed, so we can gather ref/modify + * bits again. + */ +static boolean_t +arm_force_fast_fault_internal( + ppnum_t ppnum, + vm_prot_t allow_mode, + int options) +{ + pmap_paddr_t phys = ptoa(ppnum); + pv_entry_t *pve_p; + pt_entry_t *pte_p; + int pai; + boolean_t result; + pv_entry_t **pv_h; + boolean_t is_reusable, is_internal; + boolean_t ref_fault; + boolean_t mod_fault; + + assert(ppnum != vm_page_fictitious_addr); + + if (!pa_valid(phys)) { + return FALSE; /* Not a managed page. */ + } + + result = TRUE; + ref_fault = FALSE; + mod_fault = FALSE; + pai = (int)pa_index(phys); + LOCK_PVH(pai); + pv_h = pai_to_pvh(pai); + + pte_p = PT_ENTRY_NULL; + pve_p = PV_ENTRY_NULL; + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + pte_p = pvh_ptep(pv_h); + } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + pve_p = pvh_list(pv_h); + } + + is_reusable = IS_REUSABLE_PAGE(pai); + is_internal = IS_INTERNAL_PAGE(pai); + + while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) { + vm_map_address_t va; + pt_entry_t spte; + pt_entry_t tmplate; + pmap_t pmap; + boolean_t update_pte; + + if (pve_p != PV_ENTRY_NULL) + pte_p = pve_get_ptep(pve_p); + + if (pte_p == PT_ENTRY_NULL) { + panic("pte_p is NULL: pve_p=%p ppnum=0x%x\n", pve_p, ppnum); + } + if (*pte_p == ARM_PTE_EMPTY) { + panic("pte is NULL: pte_p=%p ppnum=0x%x\n", pte_p, ppnum); + } + if (ARM_PTE_IS_COMPRESSED(*pte_p)) { + panic("pte is COMPRESSED: pte_p=%p ppnum=0x%x\n", pte_p, ppnum); + } + + pmap = ptep_get_pmap(pte_p); + va = ptep_get_va(pte_p); + + assert(va >= pmap->min && va < pmap->max); + + if (pte_is_wired(*pte_p) || pmap == kernel_pmap) { + result = FALSE; + break; + } + + spte = *pte_p; + tmplate = spte; + update_pte = FALSE; + + if ((allow_mode & VM_PROT_READ) != VM_PROT_READ) { + /* read protection sets the pte to fault */ + tmplate = tmplate & ~ARM_PTE_AF; + update_pte = TRUE; + ref_fault = TRUE; + } + if ((allow_mode & VM_PROT_WRITE) != VM_PROT_WRITE) { + /* take away write permission if set */ + if (pmap == kernel_pmap) { + if ((tmplate & ARM_PTE_APMASK) == ARM_PTE_AP(AP_RWNA)) { + tmplate = ((tmplate & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA)); + } + } else { + if ((tmplate & ARM_PTE_APMASK) == ARM_PTE_AP(AP_RWRW)) { + tmplate = ((tmplate & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO)); + } + } + + pte_set_ffr(tmplate, 1); + update_pte = TRUE; + mod_fault = TRUE; + } + + + if (update_pte) { + if (*pte_p != ARM_PTE_TYPE_FAULT && + !ARM_PTE_IS_COMPRESSED(*pte_p)) { + WRITE_PTE(pte_p, tmplate); + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); + } else { + WRITE_PTE(pte_p, tmplate); + __asm__ volatile("isb"); + } + } + + /* update pmap stats and ledgers */ + if (IS_ALTACCT_PAGE(pai, pve_p)) { + /* + * We do not track "reusable" status for + * "alternate accounting" mappings. + */ + } else if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) && + is_reusable && + is_internal && + pmap != kernel_pmap) { + /* one less "reusable" */ + assert(pmap->stats.reusable > 0); + OSAddAtomic(-1, &pmap->stats.reusable); + /* one more "internal" */ + OSAddAtomic(+1, &pmap->stats.internal); + PMAP_STATS_PEAK(pmap->stats.internal); + assert(pmap->stats.internal > 0); + pmap_ledger_credit(pmap, + task_ledgers.internal, + machine_ptob(1)); + assert(!IS_ALTACCT_PAGE(pai, pve_p)); + assert(IS_INTERNAL_PAGE(pai)); + pmap_ledger_credit(pmap, + task_ledgers.phys_footprint, + machine_ptob(1)); + + /* + * Avoid the cost of another trap to handle the fast + * fault when we next write to this page: let's just + * handle that now since we already have all the + * necessary information. + */ + { + arm_clear_fast_fault(ppnum, VM_PROT_WRITE); + } + } else if ((options & PMAP_OPTIONS_SET_REUSABLE) && + !is_reusable && + is_internal && + pmap != kernel_pmap) { + /* one more "reusable" */ + OSAddAtomic(+1, &pmap->stats.reusable); + PMAP_STATS_PEAK(pmap->stats.reusable); + assert(pmap->stats.reusable > 0); + /* one less "internal" */ + assert(pmap->stats.internal > 0); + OSAddAtomic(-1, &pmap->stats.internal); + pmap_ledger_debit(pmap, + task_ledgers.internal, + machine_ptob(1)); + assert(!IS_ALTACCT_PAGE(pai, pve_p)); + assert(IS_INTERNAL_PAGE(pai)); + pmap_ledger_debit(pmap, + task_ledgers.phys_footprint, + machine_ptob(1)); + } + + pte_p = PT_ENTRY_NULL; + if (pve_p != PV_ENTRY_NULL) + pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + } + + /* update global "reusable" status for this page */ + if (is_internal) { + if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) && + is_reusable) { + CLR_REUSABLE_PAGE(pai); + } else if ((options & PMAP_OPTIONS_SET_REUSABLE) && + !is_reusable) { + SET_REUSABLE_PAGE(pai); + } + } + + if (mod_fault) { + SET_MODFAULT_PAGE(pai); + } + if (ref_fault) { + SET_REFFAULT_PAGE(pai); + } + + UNLOCK_PVH(pai); + return result; +} + +boolean_t +arm_force_fast_fault( + ppnum_t ppnum, + vm_prot_t allow_mode, + int options, + __unused void *arg) +{ + pmap_paddr_t phys = ptoa(ppnum); + + assert(ppnum != vm_page_fictitious_addr); + + if (!pa_valid(phys)) { + return FALSE; /* Not a managed page. */ + } + + return arm_force_fast_fault_internal(ppnum, allow_mode, options); +} + +/* + * Routine: arm_clear_fast_fault + * + * Function: + * Clear pending force fault for all mappings for this page based on + * the observed fault type, update ref/modify bits. + */ +boolean_t +arm_clear_fast_fault( + ppnum_t ppnum, + vm_prot_t fault_type) +{ + pmap_paddr_t pa = ptoa(ppnum); + pv_entry_t *pve_p; + pt_entry_t *pte_p; + int pai; + boolean_t result; + pv_entry_t **pv_h; + + assert(ppnum != vm_page_fictitious_addr); + + if (!pa_valid(pa)) { + return FALSE; /* Not a managed page. */ + } + + result = FALSE; + pai = (int)pa_index(pa); + ASSERT_PVH_LOCKED(pai); + pv_h = pai_to_pvh(pai); + + pte_p = PT_ENTRY_NULL; + pve_p = PV_ENTRY_NULL; + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + pte_p = pvh_ptep(pv_h); + } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + pve_p = pvh_list(pv_h); + } + + while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) { + vm_map_address_t va; + pt_entry_t spte; + pt_entry_t tmplate; + pmap_t pmap; + + if (pve_p != PV_ENTRY_NULL) + pte_p = pve_get_ptep(pve_p); + + if (pte_p == PT_ENTRY_NULL) { + panic("pte_p is NULL: pve_p=%p ppnum=0x%x\n", pve_p, ppnum); + } + if (*pte_p == ARM_PTE_EMPTY) { + panic("pte is NULL: pte_p=%p ppnum=0x%x\n", pte_p, ppnum); + } + + pmap = ptep_get_pmap(pte_p); + va = ptep_get_va(pte_p); + + assert(va >= pmap->min && va < pmap->max); + + spte = *pte_p; + tmplate = spte; + + if ((fault_type & VM_PROT_WRITE) && (pte_is_ffr(spte))) { + { + if (pmap == kernel_pmap) + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWNA)); + else + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWRW)); + } + + tmplate |= ARM_PTE_AF; + + pte_set_ffr(tmplate, 0); + pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED); + + } else if ((fault_type & VM_PROT_READ) && ((spte & ARM_PTE_AF) != ARM_PTE_AF)) { + tmplate = spte | ARM_PTE_AF; + + { + pa_set_bits(pa, PP_ATTR_REFERENCED); + } + } + + + if (spte != tmplate) { + if (spte != ARM_PTE_TYPE_FAULT) { + WRITE_PTE(pte_p, tmplate); + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); + } else { + WRITE_PTE(pte_p, tmplate); + __asm__ volatile("isb"); + } + result = TRUE; + } + + pte_p = PT_ENTRY_NULL; + if (pve_p != PV_ENTRY_NULL) + pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + } + return result; +} + +/* + * Determine if the fault was induced by software tracking of + * modify/reference bits. If so, re-enable the mapping (and set + * the appropriate bits). + * + * Returns KERN_SUCCESS if the fault was induced and was + * successfully handled. + * + * Returns KERN_FAILURE if the fault was not induced and + * the function was unable to deal with it. + * + * Returns KERN_PROTECTION_FAILURE if the pmap layer explictly + * disallows this type of access. + */ +static kern_return_t +arm_fast_fault_internal( + pmap_t pmap, + vm_map_address_t va, + vm_prot_t fault_type, + __unused boolean_t from_user) +{ + kern_return_t result = KERN_FAILURE; + pt_entry_t *ptep; + pt_entry_t spte = ARM_PTE_TYPE_FAULT; + int pai; + pmap_paddr_t pa; + + PMAP_LOCK(pmap); + + /* + * If the entry doesn't exist, is completely invalid, or is already + * valid, we can't fix it here. + */ + + ptep = pmap_pte(pmap, va); + if (ptep != PT_ENTRY_NULL) { + spte = *ptep; + + pa = pte_to_pa(spte); + + if ((spte == ARM_PTE_TYPE_FAULT) || + ARM_PTE_IS_COMPRESSED(spte) || + (!pa_valid(pa))) { + PMAP_UNLOCK(pmap); + return result; + } + + pai = (int)pa_index(pa); + LOCK_PVH(pai); + } else { + PMAP_UNLOCK(pmap); + return result; + } + + + if ((IS_REFFAULT_PAGE(pai)) || + ((fault_type & VM_PROT_WRITE) && IS_MODFAULT_PAGE(pai))) { + /* + * An attempted access will always clear ref/mod fault state, as + * appropriate for the fault type. arm_clear_fast_fault will + * update the associated PTEs for the page as appropriate; if + * any PTEs are updated, we redrive the access. If the mapping + * does not actually allow for the attempted access, the + * following fault will (hopefully) fail to update any PTEs, and + * thus cause arm_fast_fault to decide that it failed to handle + * the fault. + */ + if (IS_REFFAULT_PAGE(pai)) { + CLR_REFFAULT_PAGE(pai); + } + if ( (fault_type & VM_PROT_WRITE) && IS_MODFAULT_PAGE(pai)) { + CLR_MODFAULT_PAGE(pai); + } + + if (arm_clear_fast_fault((ppnum_t)atop(pa),fault_type)) { + /* + * Should this preserve KERN_PROTECTION_FAILURE? The + * cost of not doing so is a another fault in a case + * that should already result in an exception. + */ + result = KERN_SUCCESS; + } + } + + UNLOCK_PVH(pai); + PMAP_UNLOCK(pmap); + return result; +} + +kern_return_t +arm_fast_fault( + pmap_t pmap, + vm_map_address_t va, + vm_prot_t fault_type, + __unused boolean_t from_user) +{ + kern_return_t result = KERN_FAILURE; + + if (va < pmap->min || va >= pmap->max) + return result; + + PMAP_TRACE(PMAP_CODE(PMAP__FAST_FAULT) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(va), fault_type, + from_user); + +#if (__ARM_VMSA__ == 7) + if (pmap != kernel_pmap) { + pmap_cpu_data_t *cpu_data_ptr = pmap_get_cpu_data(); + pmap_t cur_pmap; + pmap_t cur_user_pmap; + + cur_pmap = current_pmap(); + cur_user_pmap = cpu_data_ptr->cpu_user_pmap; + + if ((cur_user_pmap == cur_pmap) && (cur_pmap == pmap)) { + if (cpu_data_ptr->cpu_user_pmap_stamp != pmap->stamp) { + pmap_set_pmap(pmap, current_thread()); + result = KERN_SUCCESS; + goto done; + } + } + } +#endif + + result = arm_fast_fault_internal(pmap, va, fault_type, from_user); + +#if (__ARM_VMSA__ == 7) +done: +#endif + + PMAP_TRACE(PMAP_CODE(PMAP__FAST_FAULT) | DBG_FUNC_END, result); + + return result; +} + +void +pmap_copy_page( + ppnum_t psrc, + ppnum_t pdst) +{ + bcopy_phys((addr64_t) (ptoa(psrc)), + (addr64_t) (ptoa(pdst)), + PAGE_SIZE); +} + + +/* + * pmap_copy_page copies the specified (machine independent) pages. + */ +void +pmap_copy_part_page( + ppnum_t psrc, + vm_offset_t src_offset, + ppnum_t pdst, + vm_offset_t dst_offset, + vm_size_t len) +{ + bcopy_phys((addr64_t) (ptoa(psrc) + src_offset), + (addr64_t) (ptoa(pdst) + dst_offset), + len); +} + + +/* + * pmap_zero_page zeros the specified (machine independent) page. + */ +void +pmap_zero_page( + ppnum_t pn) +{ + assert(pn != vm_page_fictitious_addr); + bzero_phys((addr64_t) ptoa(pn), PAGE_SIZE); +} + +/* + * pmap_zero_part_page + * zeros the specified (machine independent) part of a page. + */ +void +pmap_zero_part_page( + ppnum_t pn, + vm_offset_t offset, + vm_size_t len) +{ + assert(pn != vm_page_fictitious_addr); + assert(offset + len <= PAGE_SIZE); + bzero_phys((addr64_t) (ptoa(pn) + offset), len); +} + + +/* + * nop in current arm implementation + */ +void +inval_copy_windows( + __unused thread_t t) +{ +} + +void +pmap_map_globals( + void) +{ + pt_entry_t *ptep, pte; + + ptep = pmap_pte(kernel_pmap, LOWGLOBAL_ALIAS); + assert(ptep != PT_ENTRY_NULL); + assert(*ptep == ARM_PTE_EMPTY); + + pte = pa_to_pte(ml_static_vtop((vm_offset_t)&lowGlo)) | AP_RONA | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF | ARM_PTE_TYPE; + pte |= ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + *ptep = pte; + FLUSH_PTE_RANGE(ptep,(ptep+1)); + PMAP_UPDATE_TLBS(kernel_pmap, LOWGLOBAL_ALIAS, LOWGLOBAL_ALIAS + PAGE_SIZE); +} + +vm_offset_t +pmap_cpu_windows_copy_addr(int cpu_num, unsigned int index) +{ + return (vm_offset_t)(CPUWINDOWS_BASE + (PAGE_SIZE * ((CPUWINDOWS_MAX * cpu_num) + index))); +} + +static unsigned int +pmap_map_cpu_windows_copy_internal( + ppnum_t pn, + vm_prot_t prot, + unsigned int wimg_bits) +{ + pt_entry_t *ptep = NULL, pte; + unsigned int cpu_num; + unsigned int i; + vm_offset_t cpu_copywindow_vaddr = 0; + + cpu_num = pmap_get_cpu_data()->cpu_number; + + for (i = 0; icpu_number; + + cpu_copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_num, index); + __asm__ volatile("dsb sy"); + ptep = pmap_pte(kernel_pmap, cpu_copywindow_vaddr); + WRITE_PTE(ptep, ARM_PTE_TYPE_FAULT); + PMAP_UPDATE_TLBS(kernel_pmap, cpu_copywindow_vaddr, cpu_copywindow_vaddr + PAGE_SIZE); +} + +void +pmap_unmap_cpu_windows_copy( + unsigned int index) +{ + return pmap_unmap_cpu_windows_copy_internal(index); +} + +/* + * Marked a pmap has nested + */ +static void +pmap_set_nested_internal( + pmap_t pmap) +{ + pmap->nested = TRUE; +} + +void +pmap_set_nested( + pmap_t pmap) +{ + pmap_set_nested_internal(pmap); +} + +/* + * kern_return_t pmap_nest(grand, subord, vstart, size) + * + * grand = the pmap that we will nest subord into + * subord = the pmap that goes into the grand + * vstart = start of range in pmap to be inserted + * nstart = start of range in pmap nested pmap + * size = Size of nest area (up to 16TB) + * + * Inserts a pmap into another. This is used to implement shared segments. + * + */ + +static kern_return_t +pmap_nest_internal( + pmap_t grand, + pmap_t subord, + addr64_t vstart, + addr64_t nstart, + uint64_t size) +{ + kern_return_t kr = KERN_FAILURE; + vm_map_offset_t vaddr, nvaddr; + tt_entry_t *stte_p; + tt_entry_t *gtte_p; + unsigned int i; + unsigned int num_tte; + unsigned int nested_region_asid_bitmap_size; + unsigned int* nested_region_asid_bitmap; + int expand_options = 0; + + +#if (__ARM_VMSA__ == 7) + if (((size|vstart|nstart) & ARM_TT_L1_PT_OFFMASK) != 0x0ULL) { + return KERN_INVALID_VALUE; /* Nest 4MB region */ + } +#else + if (((size|vstart|nstart) & (ARM_TT_L2_OFFMASK)) != 0x0ULL) { + panic("pmap_nest() pmap %p has a nested pmap 0x%llx, 0x%llx, 0x%llx\n", grand, vstart, nstart, size); + } +#endif + + if ((grand->nested_pmap != PMAP_NULL) && (grand->nested_pmap != subord)) { + panic("pmap_nest() pmap %p has a nested pmap\n", grand); + } + + if (subord->nested_region_asid_bitmap == NULL) { + nested_region_asid_bitmap_size = (unsigned int)(size>>ARM_TT_TWIG_SHIFT)/(sizeof(unsigned int)*NBBY); + + nested_region_asid_bitmap = kalloc(nested_region_asid_bitmap_size*sizeof(unsigned int)); + bzero(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int)); + + PMAP_LOCK(subord); + if (subord->nested_region_asid_bitmap == NULL) { + subord->nested_region_asid_bitmap = nested_region_asid_bitmap; + subord->nested_region_asid_bitmap_size = nested_region_asid_bitmap_size; + subord->nested_region_subord_addr = nstart; + subord->nested_region_size = (mach_vm_offset_t) size; + nested_region_asid_bitmap = NULL; + } + PMAP_UNLOCK(subord); + if (nested_region_asid_bitmap != NULL) { + kfree(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int)); + } + } + if ((subord->nested_region_subord_addr + subord->nested_region_size) < (nstart+size)) { + uint64_t new_size; + unsigned int new_nested_region_asid_bitmap_size; + unsigned int* new_nested_region_asid_bitmap; + + nested_region_asid_bitmap = NULL; + nested_region_asid_bitmap_size = 0; + new_size = nstart + size - subord->nested_region_subord_addr; + + /* We explicitly add 1 to the bitmap allocation size in order to avoid issues with truncation. */ + new_nested_region_asid_bitmap_size = (unsigned int)((new_size>>ARM_TT_TWIG_SHIFT)/(sizeof(unsigned int)*NBBY)) + 1; + + new_nested_region_asid_bitmap = kalloc(new_nested_region_asid_bitmap_size*sizeof(unsigned int)); + PMAP_LOCK(subord); + if (subord->nested_region_size < new_size) { + bzero(new_nested_region_asid_bitmap, new_nested_region_asid_bitmap_size*sizeof(unsigned int)); + bcopy(subord->nested_region_asid_bitmap, new_nested_region_asid_bitmap, subord->nested_region_asid_bitmap_size); + nested_region_asid_bitmap_size = subord->nested_region_asid_bitmap_size; + nested_region_asid_bitmap = subord->nested_region_asid_bitmap; + subord->nested_region_asid_bitmap = new_nested_region_asid_bitmap; + subord->nested_region_asid_bitmap_size = new_nested_region_asid_bitmap_size; + subord->nested_region_size = new_size; + new_nested_region_asid_bitmap = NULL; + } + PMAP_UNLOCK(subord); + if (nested_region_asid_bitmap != NULL) + kfree(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int)); + if (new_nested_region_asid_bitmap != NULL) + kfree(new_nested_region_asid_bitmap, new_nested_region_asid_bitmap_size*sizeof(unsigned int)); + } + + PMAP_LOCK(subord); + if (grand->nested_pmap == PMAP_NULL) { + grand->nested_pmap = subord; + grand->nested_region_grand_addr = vstart; + grand->nested_region_subord_addr = nstart; + grand->nested_region_size = (mach_vm_offset_t) size; + } else { + if ((grand->nested_region_grand_addr > vstart)) { + panic("pmap_nest() pmap %p : attempt to nest outside the nested region\n", grand); + } + else if ((grand->nested_region_grand_addr + grand->nested_region_size) < (vstart+size)) { + grand->nested_region_size = (mach_vm_offset_t)(vstart - grand->nested_region_grand_addr + size); + } + } + +#if (__ARM_VMSA__ == 7) + nvaddr = (vm_map_offset_t) nstart; + vaddr = (vm_map_offset_t) vstart; + num_tte = size >> ARM_TT_L1_SHIFT; + + for (i = 0; i < num_tte; i++) { + stte_p = pmap_tte(subord, nvaddr); + if ((stte_p == (tt_entry_t *)NULL) || (((*stte_p) & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE)) { + PMAP_UNLOCK(subord); + kr = pmap_expand(subord, nvaddr, expand_options, PMAP_TT_L2_LEVEL); + + if (kr != KERN_SUCCESS) { + PMAP_LOCK(grand); + goto done; + } + + PMAP_LOCK(subord); + } + PMAP_UNLOCK(subord); + PMAP_LOCK(grand); + stte_p = pmap_tte(grand, vaddr); + if (stte_p == (tt_entry_t *)NULL) { + PMAP_UNLOCK(grand); + kr = pmap_expand(grand, vaddr, expand_options, PMAP_TT_L1_LEVEL); + + if (kr != KERN_SUCCESS) { + PMAP_LOCK(grand); + goto done; + } + } else { + PMAP_UNLOCK(grand); + kr = KERN_SUCCESS; + } + PMAP_LOCK(subord); + + + nvaddr += ARM_TT_L1_SIZE; + vaddr += ARM_TT_L1_SIZE; + } + +#else + nvaddr = (vm_map_offset_t) nstart; + num_tte = (unsigned int)(size >> ARM_TT_L2_SHIFT); + + for (i = 0; i < num_tte; i++) { + stte_p = pmap_tt2e(subord, nvaddr); + if (stte_p == PT_ENTRY_NULL || *stte_p == ARM_TTE_EMPTY) { + PMAP_UNLOCK(subord); + kr = pmap_expand(subord, nvaddr, expand_options, PMAP_TT_L3_LEVEL); + + if (kr != KERN_SUCCESS) { + PMAP_LOCK(grand); + goto done; + } + + PMAP_LOCK(subord); + } + nvaddr += ARM_TT_L2_SIZE; + } +#endif + PMAP_UNLOCK(subord); + + /* + * copy tte's from subord pmap into grand pmap + */ + + PMAP_LOCK(grand); + nvaddr = (vm_map_offset_t) nstart; + vaddr = (vm_map_offset_t) vstart; + + +#if (__ARM_VMSA__ == 7) + for (i = 0; i < num_tte; i++) { + + stte_p = pmap_tte(subord, nvaddr); + gtte_p = pmap_tte(grand, vaddr); + *gtte_p = *stte_p; + + nvaddr += ARM_TT_L1_SIZE; + vaddr += ARM_TT_L1_SIZE; + } +#else + for (i = 0; i < num_tte; i++) { + + stte_p = pmap_tt2e(subord, nstart); + gtte_p = pmap_tt2e(grand, vaddr); + if (gtte_p == PT_ENTRY_NULL) { + PMAP_UNLOCK(grand); + kr = pmap_expand(grand, vaddr, expand_options, PMAP_TT_L2_LEVEL); + PMAP_LOCK(grand); + + if (kr != KERN_SUCCESS) { + goto done; + } + + gtte_p = pmap_tt2e(grand, vaddr); + } + *gtte_p = *stte_p; + vaddr += ARM_TT_L2_SIZE; + nstart += ARM_TT_L2_SIZE; + } +#endif + + kr = KERN_SUCCESS; +done: + +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) pmap_tte(grand, vstart), num_tte * sizeof(tt_entry_t)); +#endif + +#if (__ARM_VMSA__ > 7) + /* + * check for overflow on LP64 arch + */ + assert((size & 0xFFFFFFFF00000000ULL) == 0); +#endif + PMAP_UPDATE_TLBS(grand, vstart, vstart + size); + + PMAP_UNLOCK(grand); + return kr; +} + +kern_return_t pmap_nest( + pmap_t grand, + pmap_t subord, + addr64_t vstart, + addr64_t nstart, + uint64_t size) +{ + kern_return_t kr = KERN_FAILURE; + + PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(subord), + VM_KERNEL_ADDRHIDE(vstart)); + + kr = pmap_nest_internal(grand, subord, vstart, nstart, size); + + PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, kr); + + return kr; +} + +/* + * kern_return_t pmap_unnest(grand, vaddr) + * + * grand = the pmap that we will nest subord into + * vaddr = start of range in pmap to be unnested + * size = size of range in pmap to be unnested + * + */ + +kern_return_t +pmap_unnest( + pmap_t grand, + addr64_t vaddr, + uint64_t size) +{ + return(pmap_unnest_options(grand, vaddr, size, 0)); +} + +static kern_return_t +pmap_unnest_options_internal( + pmap_t grand, + addr64_t vaddr, + uint64_t size, + unsigned int option) +{ + vm_map_offset_t start; + vm_map_offset_t addr; + tt_entry_t *tte_p; + unsigned int current_index; + unsigned int start_index; + unsigned int max_index; + unsigned int num_tte; + unsigned int i; + +#if (__ARM_VMSA__ == 7) + if (((size|vaddr) & ARM_TT_L1_PT_OFFMASK) != 0x0ULL) { + panic("pmap_unnest(): unaligned request\n"); + } +#else + if (((size|vaddr) & ARM_TT_L2_OFFMASK) != 0x0ULL) { + panic("pmap_unnest(): unaligned request\n"); + } +#endif + + if ((option & PMAP_UNNEST_CLEAN) == 0) + { + PMAP_LOCK(grand->nested_pmap); + + start = vaddr - grand->nested_region_grand_addr + grand->nested_region_subord_addr ; + start_index = (unsigned int)((vaddr - grand->nested_region_grand_addr) >> ARM_TT_TWIG_SHIFT); + max_index = (unsigned int)(start_index + (size >> ARM_TT_TWIG_SHIFT)); + num_tte = (unsigned int)(size >> ARM_TT_TWIG_SHIFT); + + if (size > grand->nested_region_size) { + panic("pmap_unnest() pmap %p %llu, %llu\n", grand, size, (uint64_t)grand->nested_region_size); + } + + for (current_index = start_index, addr = start; current_index < max_index; current_index++) { + pt_entry_t *bpte, *epte, *cpte; + + + if(!testbit(current_index, (int *)grand->nested_pmap->nested_region_asid_bitmap)) { + + setbit(current_index, (int *)grand->nested_pmap->nested_region_asid_bitmap); + bpte = pmap_pte(grand->nested_pmap, addr); + epte = bpte + (ARM_TT_LEAF_INDEX_MASK>>ARM_TT_LEAF_SHIFT); + + for (cpte = bpte; cpte <= epte; cpte++) { + pmap_paddr_t pa; + int pai=0; + boolean_t managed=FALSE; + pt_entry_t spte; + + if ((*cpte != ARM_PTE_TYPE_FAULT) + && (!ARM_PTE_IS_COMPRESSED(*cpte))) { + + spte = *cpte; + while (!managed) { + pa = pte_to_pa(spte); + if (!pa_valid(pa)) + break; + pai = (int)pa_index(pa); + LOCK_PVH(pai); + spte = *cpte; + pa = pte_to_pa(spte); + if (pai == (int)pa_index(pa)) { + managed =TRUE; + break; // Leave the PVH locked as we'll unlock it after we update the PTE + } + UNLOCK_PVH(pai); + } + + if (((spte & ARM_PTE_NG) != ARM_PTE_NG)) { + + WRITE_PTE(cpte, (spte | ARM_PTE_NG)); + } + + if (managed) + { + ASSERT_PVH_LOCKED(pai); + UNLOCK_PVH(pai); + } + } + } + } + + addr += ARM_TT_TWIG_SIZE; + +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) pmap_pte(grand->nested_pmap, start), num_tte * sizeof(tt_entry_t)); +#endif + PMAP_UPDATE_TLBS(grand->nested_pmap, start, start + size); + } + + PMAP_UNLOCK(grand->nested_pmap); + } + + PMAP_LOCK(grand); + + /* + * invalidate all pdes for segment at vaddr in pmap grand + */ + start = vaddr; + addr = vaddr; + + num_tte = (unsigned int)(size >> ARM_TT_TWIG_SHIFT); + + for (i = 0; i < num_tte; i++) { + tte_p = pmap_tte(grand, addr); + *tte_p = ARM_TTE_TYPE_FAULT; + + addr += ARM_TT_TWIG_SIZE; + } + +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) pmap_tte(grand, start), num_tte * sizeof(tt_entry_t)); +#endif + PMAP_UPDATE_TLBS(grand, start, start + size); + + PMAP_UNLOCK(grand); + + return KERN_SUCCESS; +} + +kern_return_t +pmap_unnest_options( + pmap_t grand, + addr64_t vaddr, + uint64_t size, + unsigned int option) +{ + kern_return_t kr = KERN_FAILURE; + + PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(vaddr)); + + kr = pmap_unnest_options_internal(grand, vaddr, size, option); + + PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, kr); + + return kr; +} + +boolean_t +pmap_adjust_unnest_parameters( + __unused pmap_t p, + __unused vm_map_offset_t *s, + __unused vm_map_offset_t *e) +{ + return TRUE; /* to get to log_unnest_badness()... */ +} + +/* + * disable no-execute capability on + * the specified pmap + */ +#if DEVELOPMENT || DEBUG +void +pmap_disable_NX( + pmap_t pmap) +{ + pmap->nx_enabled = FALSE; +} +#else +void +pmap_disable_NX( + __unused pmap_t pmap) +{ +} +#endif + +void +pt_fake_zone_init( + int zone_index) +{ + pt_fake_zone_index = zone_index; +} + +void +pt_fake_zone_info( + int *count, + vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size, + uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct) +{ + *count = inuse_pmap_pages_count; + *cur_size = PAGE_SIZE * (inuse_pmap_pages_count); + *max_size = PAGE_SIZE * (inuse_pmap_pages_count + vm_page_inactive_count + vm_page_active_count + vm_page_free_count); + *elem_size = PAGE_SIZE; + *alloc_size = PAGE_SIZE; + *sum_size = (alloc_pmap_pages_count) * PAGE_SIZE; + + *collectable = 1; + *exhaustable = 0; + *caller_acct = 1; +} + +/* + * flush a range of hardware TLB entries. + * NOTE: assumes the smallest TLB entry in use will be for + * an ARM small page (4K). + */ + +#define ARM_FULL_TLB_FLUSH_THRESHOLD 64 +#define ARM64_FULL_TLB_FLUSH_THRESHOLD 256 + +static void +flush_mmu_tlb_region_asid( + vm_offset_t va, + unsigned length, + pmap_t pmap) +{ +#if (__ARM_VMSA__ == 7) + vm_offset_t end = va + length; + uint32_t asid; + + asid = pmap->asid; + + if (length / ARM_SMALL_PAGE_SIZE > ARM_FULL_TLB_FLUSH_THRESHOLD) { + boolean_t flush_all = FALSE; + + if ((asid == 0) || (pmap->nested == TRUE)) + flush_all = TRUE; + if (flush_all) + flush_mmu_tlb(); + else + flush_mmu_tlb_asid(asid); + + return; + } + if (pmap->nested == TRUE) { +#if !__ARM_MP_EXT__ + flush_mmu_tlb(); +#else + va = arm_trunc_page(va); + while (va < end) { + flush_mmu_tlb_mva_entries(va); + va += ARM_SMALL_PAGE_SIZE; + } +#endif + return; + } + va = arm_trunc_page(va) | (asid & 0xff); + flush_mmu_tlb_entries(va, end); + +#else + vm_offset_t end = va + length; + uint32_t asid; + + asid = pmap->asid; + + if ((length >> ARM_TT_L3_SHIFT) > ARM64_FULL_TLB_FLUSH_THRESHOLD) { + boolean_t flush_all = FALSE; + + if ((asid == 0) || (pmap->nested == TRUE)) + flush_all = TRUE; + if (flush_all) + flush_mmu_tlb(); + else + flush_mmu_tlb_asid((uint64_t)asid << TLBI_ASID_SHIFT); + return; + } + va = tlbi_asid(asid) | tlbi_addr(va); + end = tlbi_asid(asid) | tlbi_addr(end); + if (pmap->nested == TRUE) { + flush_mmu_tlb_allentries(va, end); + } else { + flush_mmu_tlb_entries(va, end); + } + +#endif +} + +void +flush_mmu_tlb_region( + vm_offset_t va, + unsigned length) +{ + flush_mmu_tlb_region_asid(va, length, kernel_pmap); +} + +unsigned int +pmap_cache_attributes( + ppnum_t pn) +{ + pmap_paddr_t paddr; + int pai; + unsigned int result; + pp_attr_t pp_attr_current; + + paddr = ptoa(pn); + + if ((paddr >= io_rgn_start) && (paddr < io_rgn_end)) { + unsigned int attr = IO_ATTR_WIMG(io_attr_table[(paddr - io_rgn_start) / io_rgn_granule]); + if (attr) + return attr; + else + return (VM_WIMG_IO); + } + + + if (!pmap_initialized) { + if ((paddr >= gPhysBase) && (paddr < gPhysBase+gPhysSize)) + return (VM_WIMG_DEFAULT); + else + return (VM_WIMG_IO); + } + + + if (!pa_valid(paddr)) + return (VM_WIMG_IO); + + result = VM_WIMG_DEFAULT; + + pai = (int)pa_index(paddr); + + pp_attr_current = pp_attr_table[pai]; + if (pp_attr_current & PP_ATTR_WIMG_MASK) + result = pp_attr_current & PP_ATTR_WIMG_MASK; + return result; +} + +static boolean_t +pmap_batch_set_cache_attributes_internal( + ppnum_t pn, + unsigned int cacheattr, + unsigned int page_cnt, + unsigned int page_index, + boolean_t doit, + unsigned int *res) +{ + pmap_paddr_t paddr; + int pai; + pp_attr_t pp_attr_current; + pp_attr_t pp_attr_template; + unsigned int wimg_bits_prev, wimg_bits_new; + + if (cacheattr & VM_WIMG_USE_DEFAULT) + cacheattr = VM_WIMG_DEFAULT; + + if ((doit == FALSE) && (*res == 0)) { + *res = page_cnt; + if (platform_cache_batch_wimg(cacheattr & (VM_WIMG_MASK), page_cnt<asid)); +#elif (ARM_PGSHIFT == 14) + flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); +#endif + + PMAP_UNLOCK(pmap); +} + +static void +pmap_sharedpage_flush_32_to_64( + void) +{ + flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE); +} + +static void +pmap_unmap_sharedpage32( + pmap_t pmap) +{ + pt_entry_t *ttep; + +#if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE +#error We assume a single page. +#endif + +#if (ARM_PGSHIFT == 12) +#if __ARM64_TWO_LEVEL_PMAP__ +#error A two level page table with a page shift of 12 is not currently supported +#endif + ttep = pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS); + if (ttep == NULL) { + return; + } + + /* It had better be mapped to the shared page */ + if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt1e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS)) { + panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__); + } +#elif (ARM_PGSHIFT == 14) + ttep = pmap_tt2e(pmap, _COMM_PAGE32_BASE_ADDRESS); + if (ttep == NULL) { + return; + } + + /* It had better be mapped to the shared page */ + if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt2e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS)) { + panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__); + } +#endif + + *ttep = ARM_TTE_EMPTY; + flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE); + +#if (ARM_PGSHIFT == 12) +#if __ARM64_TWO_LEVEL_PMAP__ +#error A two level page table with a page shift of 12 is not currently supported +#endif + flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid)); +#elif (ARM_PGSHIFT == 14) + flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); +#endif +} + +void +pmap_insert_sharedpage( + pmap_t pmap) +{ + pmap_insert_sharedpage_internal(pmap); +} + +static boolean_t +pmap_is_64bit( + pmap_t pmap) +{ + return (pmap->is_64bit); +} + +#endif + +/* ARMTODO -- an implementation that accounts for + * holes in the physical map, if any. + */ +boolean_t +pmap_valid_page( + ppnum_t pn) { + return pa_valid(ptoa(pn)); +} + +static boolean_t +pmap_is_empty_internal( + pmap_t pmap, + vm_map_offset_t va_start, + vm_map_offset_t va_end) +{ + vm_map_offset_t block_start, block_end; + tt_entry_t *tte_p; + + if (pmap == NULL) { + return TRUE; + } + + if ((pmap != kernel_pmap) && (not_in_kdp)) { + PMAP_LOCK(pmap); + } + +#if (__ARM_VMSA__ == 7) + if (tte_index(pmap, va_end) >= pmap->tte_index_max) { + if ((pmap != kernel_pmap) && (not_in_kdp)) { + PMAP_UNLOCK(pmap); + } + return TRUE; + } + + block_start = va_start; + tte_p = pmap_tte(pmap, block_start); + while (block_start < va_end) { + block_end = (block_start + ARM_TT_L1_SIZE) & ~(ARM_TT_L1_OFFMASK); + if (block_end > va_end) + block_end = va_end; + + if ((*tte_p & ARM_TTE_TYPE_MASK) != 0) { + vm_map_offset_t offset; + ppnum_t phys_page = 0; + + for (offset = block_start; + offset < block_end; + offset += ARM_PGBYTES) { + // This does a pmap_find_phys() lookup but assumes lock is held + phys_page = pmap_vtophys(pmap, offset); + if (phys_page) { + if ((pmap != kernel_pmap) && (not_in_kdp)) { + PMAP_UNLOCK(pmap); + } + return FALSE; + } + } + } + + block_start = block_end; + tte_p++; + } +#else + block_start = va_start; + + while (block_start < va_end) { + pt_entry_t *bpte_p, *epte_p; + pt_entry_t *pte_p; + + block_end = (block_start + ARM_TT_L2_SIZE) & ~ARM_TT_L2_OFFMASK; + if (block_end > va_end) + block_end = va_end; + + tte_p = pmap_tt2e(pmap, block_start); + if ((tte_p != PT_ENTRY_NULL) + && ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE)) { + + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte_p = &pte_p[tt3_index(pmap, block_start)]; + epte_p = bpte_p + (((block_end - block_start) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT); + + for (pte_p = bpte_p; pte_p < epte_p; pte_p++) { + if (*pte_p != ARM_PTE_EMPTY) { + if ((pmap != kernel_pmap) && (not_in_kdp)) { + PMAP_UNLOCK(pmap); + } + return FALSE; + } + } + } + block_start = block_end; + } +#endif + + if ((pmap != kernel_pmap) && (not_in_kdp)) { + PMAP_UNLOCK(pmap); + } + + return TRUE; +} + +boolean_t +pmap_is_empty( + pmap_t pmap, + vm_map_offset_t va_start, + vm_map_offset_t va_end) +{ + return pmap_is_empty_internal(pmap, va_start, va_end); +} + +vm_map_offset_t pmap_max_offset( + boolean_t is64 __unused, + unsigned int option) +{ + vm_map_offset_t max_offset_ret = 0; + +#if defined(__arm64__) + assert (is64); + vm_map_offset_t min_max_offset = SHARED_REGION_BASE_ARM64 + SHARED_REGION_SIZE_ARM64 + 0x20000000; // end of shared region + 512MB for various purposes + if (option == ARM_PMAP_MAX_OFFSET_DEFAULT) { + max_offset_ret = arm64_pmap_max_offset_default; + } else if (option == ARM_PMAP_MAX_OFFSET_MIN) { + max_offset_ret = min_max_offset; + } else if (option == ARM_PMAP_MAX_OFFSET_MAX) { + max_offset_ret = MACH_VM_MAX_ADDRESS; + } else if (option == ARM_PMAP_MAX_OFFSET_DEVICE) { + if (arm64_pmap_max_offset_default) { + max_offset_ret = arm64_pmap_max_offset_default; + } else if (max_mem > 0xC0000000) { + max_offset_ret = 0x0000000318000000ULL; // Max offset is 12.375GB for devices with > 3GB of memory + } else if (max_mem > 0x40000000) { + max_offset_ret = 0x0000000218000000ULL; // Max offset is 8.375GB for devices with > 1GB and <= 3GB of memory + } else { + max_offset_ret = min_max_offset; + } + } else if (option == ARM_PMAP_MAX_OFFSET_JUMBO) { + max_offset_ret = 0x0000000518000000ULL; // Max offset is 20.375GB for pmaps with special "jumbo" blessing + } else { + panic("pmap_max_offset illegal option 0x%x\n", option); + } + + assert(max_offset_ret >= min_max_offset); + return max_offset_ret; +#else + if (option == ARM_PMAP_MAX_OFFSET_DEFAULT) { + max_offset_ret = arm_pmap_max_offset_default; + } else if (option == ARM_PMAP_MAX_OFFSET_MIN) { + max_offset_ret = 0x66000000; + } else if (option == ARM_PMAP_MAX_OFFSET_MAX) { + max_offset_ret = VM_MAX_ADDRESS; + } else if (option == ARM_PMAP_MAX_OFFSET_DEVICE) { + if (arm_pmap_max_offset_default) { + max_offset_ret = arm_pmap_max_offset_default; + } else if (max_mem > 0x20000000) { + max_offset_ret = 0x80000000; + } else { + max_offset_ret = 0x66000000; + } + } else { + panic("pmap_max_offset illegal option 0x%x\n", option); + } + + return max_offset_ret; +#endif +} + +#if CONFIG_DTRACE +/* + * Constrain DTrace copyin/copyout actions + */ +extern kern_return_t dtrace_copyio_preflight(addr64_t); +extern kern_return_t dtrace_copyio_postflight(addr64_t); + +kern_return_t dtrace_copyio_preflight( + __unused addr64_t va) +{ + if (current_map() == kernel_map) + return KERN_FAILURE; + else + return KERN_SUCCESS; +} + +kern_return_t dtrace_copyio_postflight( + __unused addr64_t va) +{ + return KERN_SUCCESS; +} +#endif /* CONFIG_DTRACE */ + + +void +pmap_flush_context_init(__unused pmap_flush_context *pfc) +{ +} + + +void +pmap_flush( + __unused pmap_flush_context *cpus_to_flush) +{ + /* not implemented yet */ + return; +} + +static boolean_t +pmap_query_resident_internal( + pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + mach_vm_size_t *resident_bytes_p, + mach_vm_size_t *compressed_bytes_p) +{ + mach_vm_size_t resident_bytes = 0; + mach_vm_size_t compressed_bytes = 0; + + pt_entry_t *bpte, *epte; + pt_entry_t *pte_p; + tt_entry_t *tte_p; + + if (pmap == NULL) { + return FALSE; + } + + /* Ensure that this request is valid, and addresses exactly one TTE. */ + assert(!(start % ARM_PGBYTES)); + assert(!(end % ARM_PGBYTES)); + assert(end >= start); + assert((end - start) <= (PTE_PGENTRIES * ARM_PGBYTES)); + + PMAP_LOCK(pmap); + tte_p = pmap_tte(pmap, start); + if (tte_p == (tt_entry_t *) NULL) { + PMAP_UNLOCK(pmap); + return FALSE; + } + if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + +#if (__ARM_VMSA__ == 7) + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte = &pte_p[ptenum(start)]; + epte = bpte + atop(end - start); +#else + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte = &pte_p[tt3_index(pmap, start)]; + epte = bpte + ((end - start) >> ARM_TT_L3_SHIFT); +#endif + + for (; bpte < epte; bpte++) { + if (ARM_PTE_IS_COMPRESSED(*bpte)) { + compressed_bytes += ARM_PGBYTES; + } else if (pa_valid(pte_to_pa(*bpte))) { + resident_bytes += ARM_PGBYTES; + } + } + } + PMAP_UNLOCK(pmap); + + if (compressed_bytes_p) { + *compressed_bytes_p += compressed_bytes; + } + + if (resident_bytes_p) { + *resident_bytes_p += resident_bytes; + } + + return TRUE; +} + +mach_vm_size_t +pmap_query_resident( + pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + mach_vm_size_t *compressed_bytes_p) +{ + mach_vm_size_t resident_bytes; + mach_vm_size_t compressed_bytes; + vm_map_address_t va; + + + if (pmap == PMAP_NULL) { + if (compressed_bytes_p) { + *compressed_bytes_p = 0; + } + return 0; + } + + resident_bytes = 0; + compressed_bytes = 0; + + PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(start), + VM_KERNEL_ADDRHIDE(end)); + + va = start; + while (va < end) { + vm_map_address_t l; + + l = ((va + ARM_TT_TWIG_SIZE) & ~ARM_TT_TWIG_OFFMASK); + + if (l > end) + l = end; + if (!pmap_query_resident_internal(pmap, va, l, &resident_bytes, compressed_bytes_p)) { + break; + } + + va = l; + } + + if (compressed_bytes_p) { + *compressed_bytes_p = compressed_bytes; + } + + PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END, + resident_bytes); + + return resident_bytes; +} + +#if MACH_ASSERT +extern int pmap_ledgers_panic; +static void +pmap_check_ledgers( + pmap_t pmap) +{ + ledger_amount_t bal; + int pid; + char *procname; + boolean_t do_panic; + + if (pmap->pmap_pid == 0) { + /* + * This pmap was not or is no longer fully associated + * with a task (e.g. the old pmap after a fork()/exec() or + * spawn()). Its "ledger" still points at a task that is + * now using a different (and active) address space, so + * we can't check that all the pmap ledgers are balanced here. + * + * If the "pid" is set, that means that we went through + * pmap_set_process() in task_terminate_internal(), so + * this task's ledger should not have been re-used and + * all the pmap ledgers should be back to 0. + */ + return; + } + + do_panic = FALSE; + pid = pmap->pmap_pid; + procname = pmap->pmap_procname; + + pmap_ledgers_drift.num_pmaps_checked++; + + ledger_get_balance(pmap->ledger, + task_ledgers.phys_footprint, + &bal); + if (bal != 0) { +#if DEVELOPMENT || DEBUG +// if (!pmap->footprint_was_suspended) +#endif /* DEVELOPMENT || DEBUG */ + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"phys_footprint\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.phys_footprint_over++; + pmap_ledgers_drift.phys_footprint_over_total += bal; + if (bal > pmap_ledgers_drift.phys_footprint_over_max) { + pmap_ledgers_drift.phys_footprint_over_max = bal; + } + } else { + pmap_ledgers_drift.phys_footprint_under++; + pmap_ledgers_drift.phys_footprint_under_total += bal; + if (bal < pmap_ledgers_drift.phys_footprint_under_max) { + pmap_ledgers_drift.phys_footprint_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.internal, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"internal\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.internal_over++; + pmap_ledgers_drift.internal_over_total += bal; + if (bal > pmap_ledgers_drift.internal_over_max) { + pmap_ledgers_drift.internal_over_max = bal; + } + } else { + pmap_ledgers_drift.internal_under++; + pmap_ledgers_drift.internal_under_total += bal; + if (bal < pmap_ledgers_drift.internal_under_max) { + pmap_ledgers_drift.internal_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.internal_compressed, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"internal_compressed\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.internal_compressed_over++; + pmap_ledgers_drift.internal_compressed_over_total += bal; + if (bal > pmap_ledgers_drift.internal_compressed_over_max) { + pmap_ledgers_drift.internal_compressed_over_max = bal; + } + } else { + pmap_ledgers_drift.internal_compressed_under++; + pmap_ledgers_drift.internal_compressed_under_total += bal; + if (bal < pmap_ledgers_drift.internal_compressed_under_max) { + pmap_ledgers_drift.internal_compressed_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.iokit_mapped, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"iokit_mapped\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.iokit_mapped_over++; + pmap_ledgers_drift.iokit_mapped_over_total += bal; + if (bal > pmap_ledgers_drift.iokit_mapped_over_max) { + pmap_ledgers_drift.iokit_mapped_over_max = bal; + } + } else { + pmap_ledgers_drift.iokit_mapped_under++; + pmap_ledgers_drift.iokit_mapped_under_total += bal; + if (bal < pmap_ledgers_drift.iokit_mapped_under_max) { + pmap_ledgers_drift.iokit_mapped_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.alternate_accounting, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"alternate_accounting\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.alternate_accounting_over++; + pmap_ledgers_drift.alternate_accounting_over_total += bal; + if (bal > pmap_ledgers_drift.alternate_accounting_over_max) { + pmap_ledgers_drift.alternate_accounting_over_max = bal; + } + } else { + pmap_ledgers_drift.alternate_accounting_under++; + pmap_ledgers_drift.alternate_accounting_under_total += bal; + if (bal < pmap_ledgers_drift.alternate_accounting_under_max) { + pmap_ledgers_drift.alternate_accounting_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.alternate_accounting_compressed, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"alternate_accounting_compressed\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.alternate_accounting_compressed_over++; + pmap_ledgers_drift.alternate_accounting_compressed_over_total += bal; + if (bal > pmap_ledgers_drift.alternate_accounting_compressed_over_max) { + pmap_ledgers_drift.alternate_accounting_compressed_over_max = bal; + } + } else { + pmap_ledgers_drift.alternate_accounting_compressed_under++; + pmap_ledgers_drift.alternate_accounting_compressed_under_total += bal; + if (bal < pmap_ledgers_drift.alternate_accounting_compressed_under_max) { + pmap_ledgers_drift.alternate_accounting_compressed_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.page_table, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"page_table\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.page_table_over++; + pmap_ledgers_drift.page_table_over_total += bal; + if (bal > pmap_ledgers_drift.page_table_over_max) { + pmap_ledgers_drift.page_table_over_max = bal; + } + } else { + pmap_ledgers_drift.page_table_under++; + pmap_ledgers_drift.page_table_under_total += bal; + if (bal < pmap_ledgers_drift.page_table_under_max) { + pmap_ledgers_drift.page_table_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.purgeable_volatile, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"purgeable_volatile\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.purgeable_volatile_over++; + pmap_ledgers_drift.purgeable_volatile_over_total += bal; + if (bal > pmap_ledgers_drift.purgeable_volatile_over_max) { + pmap_ledgers_drift.purgeable_volatile_over_max = bal; + } + } else { + pmap_ledgers_drift.purgeable_volatile_under++; + pmap_ledgers_drift.purgeable_volatile_under_total += bal; + if (bal < pmap_ledgers_drift.purgeable_volatile_under_max) { + pmap_ledgers_drift.purgeable_volatile_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.purgeable_nonvolatile, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"purgeable_nonvolatile\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.purgeable_nonvolatile_over++; + pmap_ledgers_drift.purgeable_nonvolatile_over_total += bal; + if (bal > pmap_ledgers_drift.purgeable_nonvolatile_over_max) { + pmap_ledgers_drift.purgeable_nonvolatile_over_max = bal; + } + } else { + pmap_ledgers_drift.purgeable_nonvolatile_under++; + pmap_ledgers_drift.purgeable_nonvolatile_under_total += bal; + if (bal < pmap_ledgers_drift.purgeable_nonvolatile_under_max) { + pmap_ledgers_drift.purgeable_nonvolatile_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.purgeable_volatile_compressed, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"purgeable_volatile_compressed\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.purgeable_volatile_compressed_over++; + pmap_ledgers_drift.purgeable_volatile_compressed_over_total += bal; + if (bal > pmap_ledgers_drift.purgeable_volatile_compressed_over_max) { + pmap_ledgers_drift.purgeable_volatile_compressed_over_max = bal; + } + } else { + pmap_ledgers_drift.purgeable_volatile_compressed_under++; + pmap_ledgers_drift.purgeable_volatile_compressed_under_total += bal; + if (bal < pmap_ledgers_drift.purgeable_volatile_compressed_under_max) { + pmap_ledgers_drift.purgeable_volatile_compressed_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.purgeable_nonvolatile_compressed, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"purgeable_nonvolatile_compressed\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.purgeable_nonvolatile_compressed_over++; + pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_total += bal; + if (bal > pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max) { + pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max = bal; + } + } else { + pmap_ledgers_drift.purgeable_nonvolatile_compressed_under++; + pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_total += bal; + if (bal < pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max) { + pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max = bal; + } + } + } + + if (do_panic) { + if (pmap_ledgers_panic) { + panic("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n", + pmap, pid, procname); + } else { + printf("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n", + pmap, pid, procname); + } + } + + assert(pmap->stats.resident_count == 0); +#if 00 + assert(pmap->stats.wired_count == 0); +#endif + assert(pmap->stats.device == 0); + assert(pmap->stats.internal == 0); + assert(pmap->stats.external == 0); + assert(pmap->stats.reusable == 0); + assert(pmap->stats.compressed == 0); +} +#endif /* MACH_ASSERT */ + +void pmap_advise_pagezero_range(__unused pmap_t p, __unused uint64_t a) { +} + + +#if CONFIG_PGTRACE +#define PROF_START uint64_t t, nanot;\ + t = mach_absolute_time(); + +#define PROF_END absolutetime_to_nanoseconds(mach_absolute_time()-t, &nanot);\ + kprintf("%s: took %llu ns\n", __func__, nanot); + +#define PMAP_PGTRACE_LOCK(p) \ + do { \ + *(p) = ml_set_interrupts_enabled(false); \ + if (simple_lock_try(&(pmap_pgtrace.lock))) break; \ + ml_set_interrupts_enabled(*(p)); \ + } while (true) + +#define PMAP_PGTRACE_UNLOCK(p) \ + do { \ + simple_unlock(&(pmap_pgtrace.lock)); \ + ml_set_interrupts_enabled(*(p)); \ + } while (0) + +#define PGTRACE_WRITE_PTE(pte_p, pte_entry) \ + do { \ + *(pte_p) = (pte_entry); \ + FLUSH_PTE(pte_p); \ + } while (0) + +#define PGTRACE_MAX_MAP 16 // maximum supported va to same pa + +typedef enum { + UNDEFINED, + PA_UNDEFINED, + VA_UNDEFINED, + DEFINED +} pmap_pgtrace_page_state_t; + +typedef struct { + queue_chain_t chain; + + /* + pa - pa + maps - list of va maps to upper pa + map_pool - map pool + map_waste - waste can + state - state + */ + pmap_paddr_t pa; + queue_head_t maps; + queue_head_t map_pool; + queue_head_t map_waste; + pmap_pgtrace_page_state_t state; +} pmap_pgtrace_page_t; + +static struct { + /* + pages - list of tracing page info + */ + queue_head_t pages; + decl_simple_lock_data(, lock); +} pmap_pgtrace = {}; + +static void pmap_pgtrace_init(void) +{ + queue_init(&(pmap_pgtrace.pages)); + simple_lock_init(&(pmap_pgtrace.lock), 0); + + boolean_t enabled; + + if (PE_parse_boot_argn("pgtrace", &enabled, sizeof(enabled))) { + pgtrace_enabled = enabled; + } +} + +// find a page with given pa - pmap_pgtrace should be locked +inline static pmap_pgtrace_page_t *pmap_pgtrace_find_page(pmap_paddr_t pa) +{ + queue_head_t *q = &(pmap_pgtrace.pages); + pmap_pgtrace_page_t *p; + + queue_iterate(q, p, pmap_pgtrace_page_t *, chain) { + if (p->state == UNDEFINED) { + continue; + } + if (p->state == PA_UNDEFINED) { + continue; + } + if (p->pa == pa) { + return p; + } + } + + return NULL; +} + +// enter clone of given pmap, va page and range - pmap should be locked +static bool pmap_pgtrace_enter_clone(pmap_t pmap, vm_map_offset_t va_page, vm_map_offset_t start, vm_map_offset_t end) +{ + bool ints; + queue_head_t *q = &(pmap_pgtrace.pages); + pmap_paddr_t pa_page; + pt_entry_t *ptep, *cptep; + pmap_pgtrace_page_t *p; + bool found = false; + + PMAP_ASSERT_LOCKED(pmap); + assert(va_page == arm_trunc_page(va_page)); + + PMAP_PGTRACE_LOCK(&ints); + + ptep = pmap_pte(pmap, va_page); + + // target pte should exist + if (!ptep || !(*ptep & ARM_PTE_TYPE_VALID)) { + PMAP_PGTRACE_UNLOCK(&ints); + return false; + } + + queue_head_t *mapq; + queue_head_t *mappool; + pmap_pgtrace_map_t *map = NULL; + + pa_page = pte_to_pa(*ptep); + + // find if we have a page info defined for this + queue_iterate(q, p, pmap_pgtrace_page_t *, chain) { + mapq = &(p->maps); + mappool = &(p->map_pool); + + switch (p->state) { + case PA_UNDEFINED: + queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) { + if (map->cloned == false && map->pmap == pmap && map->ova == va_page) { + p->pa = pa_page; + map->range.start = start; + map->range.end = end; + found = true; + break; + } + } + break; + + case VA_UNDEFINED: + if (p->pa != pa_page) { + break; + } + queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) { + if (map->cloned == false) { + map->pmap = pmap; + map->ova = va_page; + map->range.start = start; + map->range.end = end; + found = true; + break; + } + } + break; + + case DEFINED: + if (p->pa != pa_page) { + break; + } + queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) { + if (map->cloned == true && map->pmap == pmap && map->ova == va_page) { + kprintf("%s: skip existing mapping at va=%llx\n", __func__, va_page); + break; + } else if (map->cloned == true && map->pmap == kernel_pmap && map->cva[1] == va_page) { + kprintf("%s: skip clone mapping at va=%llx\n", __func__, va_page); + break; + } else if (map->cloned == false && map->pmap == pmap && map->ova == va_page) { + // range should be already defined as well + found = true; + break; + } + } + break; + + default: + panic("invalid state p->state=%x\n", p->state); + } + + if (found == true) { + break; + } + } + + // do not clone if no page info found + if (found == false) { + PMAP_PGTRACE_UNLOCK(&ints); + return false; + } + + // copy pre, target and post ptes to clone ptes + for (int i = 0; i < 3; i++) { + ptep = pmap_pte(pmap, va_page + (i-1)*ARM_PGBYTES); + cptep = pmap_pte(kernel_pmap, map->cva[i]); + assert(cptep != NULL); + if (ptep == NULL) { + PGTRACE_WRITE_PTE(cptep, (pt_entry_t)NULL); + } else { + PGTRACE_WRITE_PTE(cptep, *ptep); + } + PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES); + } + + // get ptes for original and clone + ptep = pmap_pte(pmap, va_page); + cptep = pmap_pte(kernel_pmap, map->cva[1]); + + // invalidate original pte and mark it as a pgtrace page + PGTRACE_WRITE_PTE(ptep, (*ptep | ARM_PTE_PGTRACE) & ~ARM_PTE_TYPE_VALID); + PMAP_UPDATE_TLBS(pmap, map->ova, map->ova+ARM_PGBYTES); + + map->cloned = true; + p->state = DEFINED; + + kprintf("%s: pa_page=%llx va_page=%llx cva[1]=%llx pmap=%p ptep=%p cptep=%p\n", __func__, pa_page, va_page, map->cva[1], pmap, ptep, cptep); + + PMAP_PGTRACE_UNLOCK(&ints); + + return true; +} + +// This function removes trace bit and validate pte if applicable. Pmap must be locked. +static void pmap_pgtrace_remove_clone(pmap_t pmap, pmap_paddr_t pa, vm_map_offset_t va) +{ + bool ints, found = false; + pmap_pgtrace_page_t *p; + pt_entry_t *ptep; + + PMAP_PGTRACE_LOCK(&ints); + + // we must have this page info + p = pmap_pgtrace_find_page(pa); + if (p == NULL) { + goto unlock_exit; + } + + // find matching map + queue_head_t *mapq = &(p->maps); + queue_head_t *mappool = &(p->map_pool); + pmap_pgtrace_map_t *map; + + queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) { + if (map->pmap == pmap && map->ova == va) { + found = true; + break; + } + } + + if (!found) { + goto unlock_exit; + } + + if (map->cloned == true) { + // Restore back the pte to original state + ptep = pmap_pte(pmap, map->ova); + assert(ptep); + PGTRACE_WRITE_PTE(ptep, *ptep | ARM_PTE_TYPE_VALID); + PMAP_UPDATE_TLBS(pmap, va, va+ARM_PGBYTES); + + // revert clone pages + for (int i = 0; i < 3; i++) { + ptep = pmap_pte(kernel_pmap, map->cva[i]); + assert(ptep != NULL); + PGTRACE_WRITE_PTE(ptep, map->cva_spte[i]); + PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES); + } + } + + queue_remove(mapq, map, pmap_pgtrace_map_t *, chain); + map->pmap = NULL; + map->ova = (vm_map_offset_t)NULL; + map->cloned = false; + queue_enter_first(mappool, map, pmap_pgtrace_map_t *, chain); + + kprintf("%s: p=%p pa=%llx va=%llx\n", __func__, p, pa, va); + +unlock_exit: + PMAP_PGTRACE_UNLOCK(&ints); +} + +// remove all clones of given pa - pmap must be locked +static void pmap_pgtrace_remove_all_clone(pmap_paddr_t pa) +{ + bool ints; + pmap_pgtrace_page_t *p; + pt_entry_t *ptep; + + PMAP_PGTRACE_LOCK(&ints); + + // we must have this page info + p = pmap_pgtrace_find_page(pa); + if (p == NULL) { + PMAP_PGTRACE_UNLOCK(&ints); + return; + } + + queue_head_t *mapq = &(p->maps); + queue_head_t *mappool = &(p->map_pool); + queue_head_t *mapwaste = &(p->map_waste); + pmap_pgtrace_map_t *map; + + // move maps to waste + while (!queue_empty(mapq)) { + queue_remove_first(mapq, map, pmap_pgtrace_map_t *, chain); + queue_enter_first(mapwaste, map, pmap_pgtrace_map_t*, chain); + } + + PMAP_PGTRACE_UNLOCK(&ints); + + // sanitize maps in waste + queue_iterate(mapwaste, map, pmap_pgtrace_map_t *, chain) { + if (map->cloned == true) { + PMAP_LOCK(map->pmap); + + // restore back original pte + ptep = pmap_pte(map->pmap, map->ova); + assert(ptep); + PGTRACE_WRITE_PTE(ptep, *ptep | ARM_PTE_TYPE_VALID); + PMAP_UPDATE_TLBS(map->pmap, map->ova, map->ova+ARM_PGBYTES); + + // revert clone ptes + for (int i = 0; i < 3; i++) { + ptep = pmap_pte(kernel_pmap, map->cva[i]); + assert(ptep != NULL); + PGTRACE_WRITE_PTE(ptep, map->cva_spte[i]); + PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES); + } + + PMAP_UNLOCK(map->pmap); + } + + map->pmap = NULL; + map->ova = (vm_map_offset_t)NULL; + map->cloned = false; + } + + PMAP_PGTRACE_LOCK(&ints); + + // recycle maps back to map_pool + while (!queue_empty(mapwaste)) { + queue_remove_first(mapwaste, map, pmap_pgtrace_map_t *, chain); + queue_enter_first(mappool, map, pmap_pgtrace_map_t*, chain); + } + + PMAP_PGTRACE_UNLOCK(&ints); +} + +inline static void pmap_pgtrace_get_search_space(pmap_t pmap, vm_map_offset_t *startp, vm_map_offset_t *endp) +{ + uint64_t tsz; + vm_map_offset_t end; + + if (pmap == kernel_pmap) { + tsz = (get_tcr() >> TCR_T1SZ_SHIFT) & TCR_TSZ_MASK; + *startp = MAX(VM_MIN_KERNEL_ADDRESS, (UINT64_MAX >> (64-tsz)) << (64-tsz)); + *endp = VM_MAX_KERNEL_ADDRESS; + } else { + tsz = (get_tcr() >> TCR_T0SZ_SHIFT) & TCR_TSZ_MASK; + if (tsz == 64) { + end = 0; + } else { + end = ((uint64_t)1 << (64-tsz)) - 1; + } + + *startp = 0; + *endp = end; + } + + assert(*endp > *startp); + + return; +} + +// has pa mapped in given pmap? then clone it +static uint64_t pmap_pgtrace_clone_from_pa(pmap_t pmap, pmap_paddr_t pa, vm_map_offset_t start_offset, vm_map_offset_t end_offset) { + uint64_t ret = 0; + vm_map_offset_t min, max; + vm_map_offset_t cur_page, end_page; + pt_entry_t *ptep; + tt_entry_t *ttep; + tt_entry_t tte; + + pmap_pgtrace_get_search_space(pmap, &min, &max); + + cur_page = arm_trunc_page(min); + end_page = arm_trunc_page(max); + while (cur_page <= end_page) { + vm_map_offset_t add = 0; + + PMAP_LOCK(pmap); + + // skip uninterested space + if (pmap == kernel_pmap && + ((vm_kernel_base <= cur_page && cur_page < vm_kernel_top) || + (vm_kext_base <= cur_page && cur_page < vm_kext_top))) { + add = ARM_PGBYTES; + goto unlock_continue; + } + +#if __ARM64_TWO_LEVEL_PMAP__ + // check whether we can skip l2 + ttep = pmap_tt2e(pmap, cur_page); + assert(ttep); + tte = *ttep; +#else + // check whether we can skip l1 + ttep = pmap_tt1e(pmap, cur_page); + assert(ttep); + tte = *ttep; + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) { + add = ARM_TT_L1_SIZE; + goto unlock_continue; + } + + // how about l2 + tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, cur_page)]; +#endif + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) { + add = ARM_TT_L2_SIZE; + goto unlock_continue; + } + + // ptep finally + ptep = &(((pt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, cur_page)]); + if (ptep == PT_ENTRY_NULL) { + add = ARM_TT_L3_SIZE; + goto unlock_continue; + } + + if (arm_trunc_page(pa) == pte_to_pa(*ptep)) { + if (pmap_pgtrace_enter_clone(pmap, cur_page, start_offset, end_offset) == true) { + ret++; + } + } + + add = ARM_PGBYTES; + +unlock_continue: + PMAP_UNLOCK(pmap); + + //overflow + if (cur_page + add < cur_page) { + break; + } + + cur_page += add; + } + + + return ret; +} + +// search pv table and clone vas of given pa +static uint64_t pmap_pgtrace_clone_from_pvtable(pmap_paddr_t pa, vm_map_offset_t start_offset, vm_map_offset_t end_offset) +{ + uint64_t ret = 0; + unsigned long pai; + pv_entry_t **pvh; + pt_entry_t *ptep; + pmap_t pmap; + + typedef struct { + queue_chain_t chain; + pmap_t pmap; + vm_map_offset_t va; + } pmap_va_t; + + queue_head_t pmapvaq; + pmap_va_t *pmapva; + + queue_init(&pmapvaq); + + pai = pa_index(pa); + LOCK_PVH(pai); + pvh = pai_to_pvh(pai); + + // collect pmap/va pair from pvh + if (pvh_test_type(pvh, PVH_TYPE_PTEP)) { + ptep = pvh_ptep(pvh); + pmap = ptep_get_pmap(ptep); + + pmapva = (pmap_va_t *)kalloc(sizeof(pmap_va_t)); + pmapva->pmap = pmap; + pmapva->va = ptep_get_va(ptep); + + queue_enter_first(&pmapvaq, pmapva, pmap_va_t *, chain); + + } else if (pvh_test_type(pvh, PVH_TYPE_PVEP)) { + pv_entry_t *pvep; + + pvep = pvh_list(pvh); + while (pvep) { + ptep = pve_get_ptep(pvep); + pmap = ptep_get_pmap(ptep); + + pmapva = (pmap_va_t *)kalloc(sizeof(pmap_va_t)); + pmapva->pmap = pmap; + pmapva->va = ptep_get_va(ptep); + + queue_enter_first(&pmapvaq, pmapva, pmap_va_t *, chain); + + pvep = PVE_NEXT_PTR(pve_next(pvep)); + } + } + + UNLOCK_PVH(pai); + + // clone them while making sure mapping still exists + queue_iterate(&pmapvaq, pmapva, pmap_va_t *, chain) { + PMAP_LOCK(pmapva->pmap); + ptep = pmap_pte(pmapva->pmap, pmapva->va); + if (pte_to_pa(*ptep) == pa) { + if (pmap_pgtrace_enter_clone(pmapva->pmap, pmapva->va, start_offset, end_offset) == true) { + ret++; + } + } + PMAP_UNLOCK(pmapva->pmap); + + kfree(pmapva, sizeof(pmap_va_t)); + } + + return ret; +} + +// allocate a page info +static pmap_pgtrace_page_t *pmap_pgtrace_alloc_page(void) +{ + pmap_pgtrace_page_t *p; + queue_head_t *mapq; + queue_head_t *mappool; + queue_head_t *mapwaste; + pmap_pgtrace_map_t *map; + + p = kalloc(sizeof(pmap_pgtrace_page_t)); + assert(p); + + p->state = UNDEFINED; + + mapq = &(p->maps); + mappool = &(p->map_pool); + mapwaste = &(p->map_waste); + queue_init(mapq); + queue_init(mappool); + queue_init(mapwaste); + + for (int i = 0; i < PGTRACE_MAX_MAP; i++) { + vm_map_offset_t newcva; + pt_entry_t *cptep; + kern_return_t kr; + vm_map_entry_t entry; + + // get a clone va + vm_object_reference(kernel_object); + kr = vm_map_find_space(kernel_map, &newcva, vm_map_round_page(3*ARM_PGBYTES, PAGE_MASK), 0, 0, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_DIAG, &entry); + if (kr != KERN_SUCCESS) { + panic("%s VM couldn't find any space kr=%d\n", __func__, kr); + } + VME_OBJECT_SET(entry, kernel_object); + VME_OFFSET_SET(entry, newcva); + vm_map_unlock(kernel_map); + + // fill default clone page info and add to pool + map = kalloc(sizeof(pmap_pgtrace_map_t)); + for (int j = 0; j < 3; j ++) { + vm_map_offset_t addr = newcva + j * ARM_PGBYTES; + + // pre-expand pmap while preemption enabled + kr = pmap_expand(kernel_pmap, addr, 0, PMAP_TT_MAX_LEVEL); + if (kr != KERN_SUCCESS) { + panic("%s: pmap_expand(kernel_pmap, addr=%llx) returns kr=%d\n", __func__, addr, kr); + } + + cptep = pmap_pte(kernel_pmap, addr); + assert(cptep != NULL); + + map->cva[j] = addr; + map->cva_spte[j] = *cptep; + } + map->range.start = map->range.end = 0; + map->cloned = false; + queue_enter_first(mappool, map, pmap_pgtrace_map_t *, chain); + } + + return p; +} + +// free a page info +static void pmap_pgtrace_free_page(pmap_pgtrace_page_t *p) +{ + queue_head_t *mapq; + queue_head_t *mappool; + queue_head_t *mapwaste; + pmap_pgtrace_map_t *map; + + assert(p); + + mapq = &(p->maps); + mappool = &(p->map_pool); + mapwaste = &(p->map_waste); + + while (!queue_empty(mapq)) { + queue_remove_first(mapq, map, pmap_pgtrace_map_t *, chain); + kfree(map, sizeof(pmap_pgtrace_map_t)); + } + + while (!queue_empty(mappool)) { + queue_remove_first(mappool, map, pmap_pgtrace_map_t *, chain); + kfree(map, sizeof(pmap_pgtrace_map_t)); + } + + while (!queue_empty(mapwaste)) { + queue_remove_first(mapwaste, map, pmap_pgtrace_map_t *, chain); + kfree(map, sizeof(pmap_pgtrace_map_t)); + } + + kfree(p, sizeof(pmap_pgtrace_page_t)); +} + +// construct page infos with the given address range +int pmap_pgtrace_add_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end) +{ + int ret = 0; + pt_entry_t *ptep; + queue_head_t *q = &(pmap_pgtrace.pages); + bool ints; + vm_map_offset_t cur_page, end_page; + + if (start > end) { + kprintf("%s: invalid start=%llx > end=%llx\n", __func__, start, end); + return -1; + } + + PROF_START + + // add each page in given range + cur_page = arm_trunc_page(start); + end_page = arm_trunc_page(end); + while (cur_page <= end_page) { + pmap_paddr_t pa_page = 0; + uint64_t num_cloned = 0; + pmap_pgtrace_page_t *p = NULL, *newp; + bool free_newp = true; + pmap_pgtrace_page_state_t state; + + // do all allocations outside of spinlocks + newp = pmap_pgtrace_alloc_page(); + + // keep lock orders in pmap, kernel_pmap and pgtrace lock + if (pmap != NULL) { + PMAP_LOCK(pmap); + } + if (pmap != kernel_pmap) { + PMAP_LOCK(kernel_pmap); + } + + // addresses are physical if pmap is null + if (pmap == NULL) { + ptep = NULL; + pa_page = cur_page; + state = VA_UNDEFINED; + } else { + ptep = pmap_pte(pmap, cur_page); + if (ptep != NULL) { + pa_page = pte_to_pa(*ptep); + state = DEFINED; + } else { + state = PA_UNDEFINED; + } + } + + // search if we have a page info already + PMAP_PGTRACE_LOCK(&ints); + if (state != PA_UNDEFINED) { + p = pmap_pgtrace_find_page(pa_page); + } + + // add pre-allocated page info if nothing found + if (p == NULL) { + queue_enter_first(q, newp, pmap_pgtrace_page_t *, chain); + p = newp; + free_newp = false; + } + + // now p points what we want + p->state = state; + + queue_head_t *mapq = &(p->maps); + queue_head_t *mappool = &(p->map_pool); + pmap_pgtrace_map_t *map; + vm_map_offset_t start_offset, end_offset; + + // calculate trace offsets in the page + if (cur_page > start) { + start_offset = 0; + } else { + start_offset = start-cur_page; + } + if (cur_page == end_page) { + end_offset = end-end_page; + } else { + end_offset = ARM_PGBYTES-1; + } + + kprintf("%s: pmap=%p cur_page=%llx ptep=%p state=%d start_offset=%llx end_offset=%llx\n", __func__, pmap, cur_page, ptep, state, start_offset, end_offset); + + // fill map info + assert(!queue_empty(mappool)); + queue_remove_first(mappool, map, pmap_pgtrace_map_t *, chain); + if (p->state == PA_UNDEFINED) { + map->pmap = pmap; + map->ova = cur_page; + map->range.start = start_offset; + map->range.end = end_offset; + } else if (p->state == VA_UNDEFINED) { + p->pa = pa_page; + map->range.start = start_offset; + map->range.end = end_offset; + } else if (p->state == DEFINED) { + p->pa = pa_page; + map->pmap = pmap; + map->ova = cur_page; + map->range.start = start_offset; + map->range.end = end_offset; + } else { + panic("invalid p->state=%d\n", p->state); + } + + // not cloned yet + map->cloned = false; + queue_enter(mapq, map, pmap_pgtrace_map_t *, chain); + + // unlock locks + PMAP_PGTRACE_UNLOCK(&ints); + if (pmap != kernel_pmap) { + PMAP_UNLOCK(kernel_pmap); + } + if (pmap != NULL) { + PMAP_UNLOCK(pmap); + } + + // now clone it + if (pa_valid(pa_page)) { + num_cloned = pmap_pgtrace_clone_from_pvtable(pa_page, start_offset, end_offset); + } + if (pmap == NULL) { + num_cloned += pmap_pgtrace_clone_from_pa(kernel_pmap, pa_page, start_offset, end_offset); + } else { + num_cloned += pmap_pgtrace_clone_from_pa(pmap, pa_page, start_offset, end_offset); + } + + // free pre-allocations if we didn't add it to the q + if (free_newp) { + pmap_pgtrace_free_page(newp); + } + + if (num_cloned == 0) { + kprintf("%s: no mapping found for pa_page=%llx but will be added when a page entered\n", __func__, pa_page); + } + + ret += num_cloned; + + // overflow + if (cur_page + ARM_PGBYTES < cur_page) { + break; + } else { + cur_page += ARM_PGBYTES; + } + } + + PROF_END + + return ret; +} + +// delete page infos for given address range +int pmap_pgtrace_delete_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end) +{ + int ret = 0; + bool ints; + queue_head_t *q = &(pmap_pgtrace.pages); + pmap_pgtrace_page_t *p; + vm_map_offset_t cur_page, end_page; + + kprintf("%s start=%llx end=%llx\n", __func__, start, end); + + PROF_START + + pt_entry_t *ptep; + pmap_paddr_t pa_page; + + // remove page info from start to end + cur_page = arm_trunc_page(start); + end_page = arm_trunc_page(end); + while (cur_page <= end_page) { + p = NULL; + + if (pmap == NULL) { + pa_page = cur_page; + } else { + PMAP_LOCK(pmap); + ptep = pmap_pte(pmap, cur_page); + if (ptep == NULL) { + PMAP_UNLOCK(pmap); + goto cont; + } + pa_page = pte_to_pa(*ptep); + PMAP_UNLOCK(pmap); + } + + // remove all clones and validate + pmap_pgtrace_remove_all_clone(pa_page); + + // find page info and delete + PMAP_PGTRACE_LOCK(&ints); + p = pmap_pgtrace_find_page(pa_page); + if (p != NULL) { + queue_remove(q, p, pmap_pgtrace_page_t *, chain); + ret++; + } + PMAP_PGTRACE_UNLOCK(&ints); + + // free outside of locks + if (p != NULL) { + pmap_pgtrace_free_page(p); + } + +cont: + // overflow + if (cur_page + ARM_PGBYTES < cur_page) { + break; + } else { + cur_page += ARM_PGBYTES; + } + } + + PROF_END + + return ret; +} + +kern_return_t pmap_pgtrace_fault(pmap_t pmap, vm_map_offset_t va, arm_saved_state_t *ss) +{ + pt_entry_t *ptep; + pgtrace_run_result_t res; + pmap_pgtrace_page_t *p; + bool ints, found = false; + pmap_paddr_t pa; + + // Quick check if we are interested + ptep = pmap_pte(pmap, va); + if (!ptep || !(*ptep & ARM_PTE_PGTRACE)) { + return KERN_FAILURE; + } + + PMAP_PGTRACE_LOCK(&ints); + + // Check again since access is serialized + ptep = pmap_pte(pmap, va); + if (!ptep || !(*ptep & ARM_PTE_PGTRACE)) { + PMAP_PGTRACE_UNLOCK(&ints); + return KERN_FAILURE; + + } else if ((*ptep & ARM_PTE_TYPE_VALID) == ARM_PTE_TYPE_VALID) { + // Somehow this cpu's tlb has not updated + kprintf("%s Somehow this cpu's tlb has not updated?\n", __func__); + PMAP_UPDATE_TLBS(pmap, va, va+ARM_PGBYTES); + + PMAP_PGTRACE_UNLOCK(&ints); + return KERN_SUCCESS; + } + + // Find if this pa is what we are tracing + pa = pte_to_pa(*ptep); + + p = pmap_pgtrace_find_page(arm_trunc_page(pa)); + if (p == NULL) { + panic("%s Can't find va=%llx pa=%llx from tracing pages\n", __func__, va, pa); + } + + // find if pmap and va are also matching + queue_head_t *mapq = &(p->maps); + queue_head_t *mapwaste = &(p->map_waste); + pmap_pgtrace_map_t *map; + + queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) { + if (map->pmap == pmap && map->ova == arm_trunc_page(va)) { + found = true; + break; + } + } + + // if not found, search map waste as they are still valid + if (!found) { + queue_iterate(mapwaste, map, pmap_pgtrace_map_t *, chain) { + if (map->pmap == pmap && map->ova == arm_trunc_page(va)) { + found = true; + break; + } + } + } + + if (!found) { + panic("%s Can't find va=%llx pa=%llx from tracing pages\n", __func__, va, pa); + } + + // Decode and run it on the clone map + bzero(&res, sizeof(res)); + pgtrace_decode_and_run(*(uint32_t *)get_saved_state_pc(ss), // instruction + va, map->cva, // fault va and clone page vas + ss, &res); + + // write a log if in range + vm_map_offset_t offset = va - map->ova; + if (map->range.start <= offset && offset <= map->range.end) { + pgtrace_write_log(res); + } + + PMAP_PGTRACE_UNLOCK(&ints); + + // Return to next instruction + set_saved_state_pc(ss, get_saved_state_pc(ss) + sizeof(uint32_t)); + + return KERN_SUCCESS; +} +#endif + +boolean_t +pmap_enforces_execute_only( +#if (__ARM_VMSA__ == 7) + __unused +#endif + pmap_t pmap) +{ +#if (__ARM_VMSA__ > 7) + return (pmap != kernel_pmap); +#else + return FALSE; +#endif +} + +void +pmap_set_jit_entitled( + __unused pmap_t pmap) +{ + return; +} + +static kern_return_t +pmap_query_page_info_internal( + pmap_t pmap, + vm_map_offset_t va, + int *disp_p) +{ + int disp; + pmap_paddr_t pa; + int pai; + pt_entry_t *pte; + pv_entry_t **pv_h, *pve_p; + + if (pmap == PMAP_NULL || pmap == kernel_pmap) { + *disp_p = 0; + return KERN_INVALID_ARGUMENT; + } + + disp = 0; + + PMAP_LOCK(pmap); + + pte = pmap_pte(pmap, va); + if (pte == PT_ENTRY_NULL) { + goto done; + } + + pa = pte_to_pa(*pte); + if (pa == 0) { + if (ARM_PTE_IS_COMPRESSED(*pte)) { + disp |= PMAP_QUERY_PAGE_COMPRESSED; + if (*pte & ARM_PTE_COMPRESSED_ALT) { + disp |= PMAP_QUERY_PAGE_COMPRESSED_ALTACCT; + } + } + } else { + disp |= PMAP_QUERY_PAGE_PRESENT; + pai = (int) pa_index(pa); + if (!pa_valid(pa)) { + goto done; + } + LOCK_PVH(pai); + pv_h = pai_to_pvh(pai); + pve_p = PV_ENTRY_NULL; + if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + pve_p = pvh_list(pv_h); + while (pve_p != PV_ENTRY_NULL && + pve_get_ptep(pve_p) != pte) { + pve_p = pvh_list(pv_h); + } + } + if (IS_ALTACCT_PAGE(pai, pve_p)) { + disp |= PMAP_QUERY_PAGE_ALTACCT; + } else if (IS_REUSABLE_PAGE(pai)) { + disp |= PMAP_QUERY_PAGE_REUSABLE; + } else if (IS_INTERNAL_PAGE(pai)) { + disp |= PMAP_QUERY_PAGE_INTERNAL; + } + UNLOCK_PVH(pai); + } + +done: + PMAP_UNLOCK(pmap); + *disp_p = disp; + return KERN_SUCCESS; +} + +kern_return_t +pmap_query_page_info( + pmap_t pmap, + vm_map_offset_t va, + int *disp_p) +{ + return pmap_query_page_info_internal(pmap, va, disp_p); +} + +kern_return_t +pmap_return_internal(__unused boolean_t do_panic, __unused boolean_t do_recurse) +{ + + return KERN_SUCCESS; +} + +kern_return_t +pmap_return(boolean_t do_panic, boolean_t do_recurse) +{ + return pmap_return_internal(do_panic, do_recurse); +} + +static void +pmap_footprint_suspend_internal( + vm_map_t map, + boolean_t suspend) +{ +#if DEVELOPMENT || DEBUG + if (suspend) { + map->pmap->footprint_suspended = TRUE; + map->pmap->footprint_was_suspended = TRUE; + } else { + map->pmap->footprint_suspended = FALSE; + } +#else /* DEVELOPMENT || DEBUG */ + (void) map; + (void) suspend; +#endif /* DEVELOPMENT || DEBUG */ +} +void +pmap_footprint_suspend( + vm_map_t map, + boolean_t suspend) +{ + pmap_footprint_suspend_internal(map, suspend); +} diff --git a/osfmk/arm/pmap.h b/osfmk/arm/pmap.h new file mode 100644 index 000000000..349ebc17f --- /dev/null +++ b/osfmk/arm/pmap.h @@ -0,0 +1,516 @@ +/* + * + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_PMAP_H_ +#define _ARM_PMAP_H_ 1 + +#include + +#include +#if defined(__arm64__) +#include +#endif + +/* + * Machine-dependent structures for the physical map module. + */ + +#ifndef ASSEMBLER + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Base address for low globals. */ +#define LOW_GLOBAL_BASE_ADDRESS 0xfffffff000000000ULL + +/* + * This indicates (roughly) where there is free space for the VM + * to use for the heap; this does not need to be precise. + */ +#if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__ +#define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS +#else +#define KERNEL_PMAP_HEAP_RANGE_START LOW_GLOBAL_BASE_ADDRESS +#endif + +#if defined(__arm64__) + +typedef uint64_t tt_entry_t; /* translation table entry type */ +#define TT_ENTRY_NULL ((tt_entry_t *) 0) + +typedef uint64_t pt_entry_t; /* page table entry type */ +#define PT_ENTRY_NULL ((pt_entry_t *) 0) + +typedef uint64_t pmap_paddr_t; /* physical address (not ppnum_t) */ + +#elif defined(__arm__) + +typedef uint32_t tt_entry_t; /* translation table entry type */ +#define PT_ENTRY_NULL ((pt_entry_t *) 0) + +typedef uint32_t pt_entry_t; /* page table entry type */ +#define TT_ENTRY_NULL ((tt_entry_t *) 0) + +typedef uint32_t pmap_paddr_t; /* physical address (not ppnum_t) */ + +#else +#error unknown arch +#endif + + +/* superpages */ +#define SUPERPAGE_NBASEPAGES 1 /* No superpages support */ + +/* + * Convert addresses to pages and vice versa. + * No rounding is used. + */ +#define arm_atop(x) (((vm_map_address_t)(x)) >> ARM_PGSHIFT) +#define arm_ptoa(x) (((vm_map_address_t)(x)) << ARM_PGSHIFT) + +/* + * Round off or truncate to the nearest page. These will work + * for either addresses or counts. (i.e. 1 byte rounds to 1 page + * bytes. + */ +#define arm_round_page(x) \ + ((((vm_map_address_t)(x)) + ARM_PGMASK) & ~ARM_PGMASK) +#define arm_trunc_page(x) (((vm_map_address_t)(x)) & ~ARM_PGMASK) + +/* Convert address offset to page table index */ +#define ptenum(a) ((((a) & ARM_TT_LEAF_INDEX_MASK) >> ARM_TT_LEAF_SHIFT)) + +/* + * For setups where the kernel page size does not match the hardware + * page size (assumably, the kernel page size must be a multiple of + * the hardware page size), we will need to determine what the page + * ratio is. + */ +#define PAGE_RATIO ((1 << PAGE_SHIFT) >> ARM_PGSHIFT) +#define TEST_PAGE_RATIO_4 (PAGE_RATIO == 4) + +#if (__ARM_VMSA__ <= 7) +#define NTTES (ARM_PGBYTES / sizeof(tt_entry_t)) +#define NPTES ((ARM_PGBYTES/4) /sizeof(pt_entry_t)) +#else +#define NTTES (ARM_PGBYTES / sizeof(tt_entry_t)) +#define NPTES (ARM_PGBYTES / sizeof(pt_entry_t)) +#endif + +extern void flush_mmu_tlb(void); +extern void flush_core_tlb(void); +#if defined(__arm64__) +extern void flush_mmu_tlb_allentries(uint64_t, uint64_t); +extern void flush_mmu_tlb_entry(uint64_t); +extern void flush_mmu_tlb_entries(uint64_t, uint64_t); +extern void flush_mmu_tlb_asid(uint64_t); +extern void flush_core_tlb_asid(uint64_t); +/* + * TLBI appers to only deal in 4KB page addresses, so give + * it an explicit shift of 12. + */ +#define TLBI_ADDR_SIZE 44 +#define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1) +#define TLBI_ADDR_SHIFT (12) +#define tlbi_addr(x) (((x) >> TLBI_ADDR_SHIFT) & TLBI_ADDR_MASK) + +#define TLBI_ASID_SHIFT 48 +#define TLBI_ASID_SIZE 16 +#define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1) << TLBI_ASID_SHIFT) +#define tlbi_asid(x) (((uint64_t)x << TLBI_ASID_SHIFT) & TLBI_ASID_MASK) +#else +extern void flush_mmu_tlb_entry(uint32_t); +extern void flush_mmu_tlb_entries(uint32_t, uint32_t); +extern void flush_mmu_tlb_mva_entries(uint32_t); +extern void flush_mmu_tlb_asid(uint32_t); +extern void flush_core_tlb_asid(uint32_t); +#endif +extern void flush_mmu_tlb_region(vm_offset_t va, unsigned length); + +#if defined(__arm64__) +extern uint64_t get_mmu_control(void); +extern void set_mmu_control(uint64_t); +extern uint64_t get_aux_control(void); +extern void set_aux_control(uint64_t); +extern void set_mmu_ttb(uint64_t); +extern void set_mmu_ttb_alternate(uint64_t); +extern uint64_t get_tcr(void); +extern void set_tcr(uint64_t); +#else +extern uint32_t get_mmu_control(void); +extern void set_mmu_control(uint32_t); +extern uint32_t get_aux_control(void); +extern void set_aux_control(uint32_t); +extern void set_mmu_ttb(pmap_paddr_t); +extern void set_mmu_ttb_alternate(pmap_paddr_t); +extern void set_context_id(uint32_t); +#endif + +extern pmap_paddr_t get_mmu_ttb(void); +extern pmap_paddr_t mmu_kvtop(vm_offset_t va); +extern pmap_paddr_t mmu_kvtop_wpreflight(vm_offset_t va); +extern pmap_paddr_t mmu_uvtop(vm_offset_t va); + +#if (__ARM_VMSA__ <= 7) +/* Convert address offset to translation table index */ +#define ttenum(a) ((a) >> ARM_TT_L1_SHIFT) + +/* Convert translation table index to user virtual address */ +#define tteitova(a) ((a) << ARM_TT_L1_SHIFT) + +#define pa_to_suptte(a) ((a) & ARM_TTE_SUPER_L1_MASK) +#define suptte_to_pa(p) ((p) & ARM_TTE_SUPER_L1_MASK) + +#define pa_to_sectte(a) ((a) & ARM_TTE_BLOCK_L1_MASK) +#define sectte_to_pa(p) ((p) & ARM_TTE_BLOCK_L1_MASK) + +#define pa_to_tte(a) ((a) & ARM_TTE_TABLE_MASK) +#define tte_to_pa(p) ((p) & ARM_TTE_TABLE_MASK) + +#define pa_to_pte(a) ((a) & ARM_PTE_PAGE_MASK) +#define pte_to_pa(p) ((p) & ARM_PTE_PAGE_MASK) +#define pte_increment_pa(p) ((p) += ptoa(1)) + +#define ARM_NESTING_SIZE_MIN ((PAGE_SIZE/0x1000)*4*ARM_TT_L1_SIZE) +#define ARM_NESTING_SIZE_MAX ((256*ARM_TT_L1_SIZE)) + +#else + +/* Convert address offset to translation table index */ +#define ttel0num(a) ((a & ARM_TTE_L0_MASK) >> ARM_TT_L0_SHIFT) +#define ttel1num(a) ((a & ARM_TTE_L1_MASK) >> ARM_TT_L1_SHIFT) +#define ttel2num(a) ((a & ARM_TTE_L2_MASK) >> ARM_TT_L2_SHIFT) + +#define pa_to_tte(a) ((a) & ARM_TTE_TABLE_MASK) +#define tte_to_pa(p) ((p) & ARM_TTE_TABLE_MASK) + +#define pa_to_pte(a) ((a) & ARM_PTE_MASK) +#define pte_to_pa(p) ((p) & ARM_PTE_MASK) +#define pte_to_ap(p) (((p) & ARM_PTE_APMASK) >> ARM_PTE_APSHIFT) +#define pte_increment_pa(p) ((p) += ptoa(1)) + +#define ARM_NESTING_SIZE_MIN ((PAGE_SIZE/ARM_PGBYTES)*ARM_TT_L2_SIZE) +#define ARM_NESTING_SIZE_MAX (0x0000000010000000ULL) + +#define TLBFLUSH_SIZE (ARM_TTE_MAX/((sizeof(unsigned int))*BYTE_SIZE)) + +#endif /* __ARM_VMSA__ <= 7 */ + +#define PMAP_GC_INFLIGHT 1 +#define PMAP_GC_WAIT 2 + +/* + * Convert translation/page table entry to kernel virtual address + */ +#define ttetokv(a) (phystokv(tte_to_pa(a))) +#define ptetokv(a) (phystokv(pte_to_pa(a))) + +struct pmap { + tt_entry_t *tte; /* translation table entries */ + pmap_paddr_t ttep; /* translation table physical */ + vm_map_address_t min; /* min address in pmap */ + vm_map_address_t max; /* max address in pmap */ + unsigned int asid; /* address space id */ + unsigned int vasid; /* Virtual address space id */ + unsigned int stamp; /* creation stamp */ + unsigned int wired; /* wired bits */ + volatile uint32_t ref_count; /* pmap reference count */ + unsigned int cpu_ref; /* number of cpus using pmap */ + unsigned int gc_status; /* gc status */ + ledger_t ledger; /* ledger tracking phys mappings */ + decl_simple_lock_data(,lock) /* lock on map */ + struct pmap_statistics stats; /* map statistics */ + queue_chain_t pmaps; /* global list of pmaps */ + tt_entry_t *tt_entry_free; /* free translation table entries */ + tt_entry_t *prev_tte; /* previous translation table */ + unsigned int tte_index_max; /* max tte index in translation table entries */ + boolean_t nx_enabled; /* no execute */ + boolean_t nested; /* is nested */ + boolean_t is_64bit; /* is 64bit */ + struct pmap *nested_pmap; /* nested pmap */ + vm_map_address_t nested_region_grand_addr; + vm_map_address_t nested_region_subord_addr; + vm_map_offset_t nested_region_size; + unsigned int *nested_region_asid_bitmap; + unsigned int nested_region_asid_bitmap_size; + +#if (__ARM_VMSA__ <= 7) + decl_simple_lock_data(,tt1_lock) /* lock on tt1 */ +#endif +#if MACH_ASSERT + int pmap_pid; + char pmap_procname[17]; +#endif /* MACH_ASSERT */ +#if DEVELOPMENT || DEBUG + boolean_t footprint_suspended; + boolean_t footprint_was_suspended; +#endif /* DEVELOPMENT || DEBUG */ +}; + +/* typedef struct pmap *pmap_t; */ +#define PMAP_NULL ((pmap_t) 0) + + +/* + * WIMG control + */ +#define VM_MEM_INNER 0x10 +#define VM_MEM_EARLY_ACK 0x20 + +#define VM_WIMG_DEFAULT (VM_MEM_COHERENT) +#define VM_WIMG_COPYBACK (VM_MEM_COHERENT) +#define VM_WIMG_INNERWBACK (VM_MEM_COHERENT | VM_MEM_INNER) +#define VM_WIMG_IO (VM_MEM_COHERENT | VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED) +#define VM_WIMG_POSTED (VM_MEM_COHERENT | VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED | VM_MEM_EARLY_ACK) +#define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED) +#define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT) + + +#if VM_DEBUG +extern int pmap_list_resident_pages( + pmap_t pmap, + vm_offset_t *listp, + int space + ); +#else /* #if VM_DEBUG */ +#define pmap_list_resident_pages(pmap, listp, space) (0) +#endif /* #if VM_DEBUG */ + +extern int copysafe(vm_map_address_t from, vm_map_address_t to, uint32_t cnt, int type, uint32_t *bytes_copied); + +/* globals shared between arm_vm_init and pmap */ +extern tt_entry_t *cpu_tte; /* first CPUs translation table (shared with kernel pmap) */ +extern pmap_paddr_t cpu_ttep; /* physical translation table addr */ + +#if __arm64__ +extern void *ropagetable_begin; +extern void *ropagetable_end; +#endif + +#if __arm64__ +extern tt_entry_t *invalid_tte; /* global invalid translation table */ +extern pmap_paddr_t invalid_ttep; /* physical invalid translation table addr */ +#endif + +#define PMAP_CONTEXT(pmap, thread) + +/* + * platform dependent Prototypes + */ +extern void pmap_switch_user_ttb(pmap_t pmap); +extern void pmap_bootstrap(vm_offset_t); +extern vm_map_address_t pmap_ptov(pmap_t, ppnum_t); +extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va); +extern void pmap_set_pmap(pmap_t pmap, thread_t thread); +extern void pmap_collect(pmap_t pmap); +extern void pmap_gc(void); +#if defined(__arm64__) +extern vm_offset_t pmap_extract(pmap_t pmap, vm_map_offset_t va); +#endif + +/* + * Interfaces implemented as macros. + */ + +#define PMAP_SWITCH_USER(th, new_map, my_cpu) { \ + th->map = new_map; \ + pmap_set_pmap(vm_map_pmap(new_map), th); \ +} + +#define pmap_kernel() \ + (kernel_pmap) + +#define pmap_compressed(pmap) \ + ((pmap)->stats.compressed) + +#define pmap_resident_count(pmap) \ + ((pmap)->stats.resident_count) + +#define pmap_resident_max(pmap) \ + ((pmap)->stats.resident_max) + +#define MACRO_NOOP + +#define pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) \ + MACRO_NOOP + +#define pmap_pageable(pmap, start, end, pageable) \ + MACRO_NOOP + +#define pmap_kernel_va(VA) \ + (((VA) >= VM_MIN_KERNEL_ADDRESS) && ((VA) <= VM_MAX_KERNEL_ADDRESS)) + +#define pmap_attribute(pmap,addr,size,attr,value) \ + (KERN_INVALID_ADDRESS) + +#define copyinmsg(from, to, cnt) \ + copyin(from, to, cnt) + +#define copyoutmsg(from, to, cnt) \ + copyout(from, to, cnt) + +extern pmap_paddr_t kvtophys(vm_offset_t va); + +extern vm_map_address_t pmap_map(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot, unsigned int flags); +extern vm_map_address_t pmap_map_high_window_bd( vm_offset_t pa, vm_size_t len, vm_prot_t prot); +extern kern_return_t pmap_map_block(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags); +extern void pmap_map_globals(void); + +#define PMAP_MAP_BD_DEVICE 0x1 +#define PMAP_MAP_BD_WCOMB 0x2 +#define PMAP_MAP_BD_POSTED 0x3 +#define PMAP_MAP_BD_MASK 0x3 + +extern vm_map_address_t pmap_map_bd_with_options(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot, int32_t options); +extern vm_map_address_t pmap_map_bd(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot); + +extern void pmap_init_pte_page(pmap_t, pt_entry_t *, vm_offset_t, unsigned int ttlevel, boolean_t alloc_ptd); +extern void pmap_init_pte_static_page(pmap_t, pt_entry_t *, pmap_paddr_t); + +extern boolean_t pmap_valid_address(pmap_paddr_t addr); +extern void pmap_disable_NX(pmap_t pmap); +extern void pmap_set_nested(pmap_t pmap); +extern vm_map_address_t pmap_create_sharedpage(void); +extern void pmap_insert_sharedpage(pmap_t pmap); +extern void pmap_protect_sharedpage(void); + +extern vm_offset_t pmap_cpu_windows_copy_addr(int cpu_num, unsigned int index); +extern unsigned int pmap_map_cpu_windows_copy(ppnum_t pn, vm_prot_t prot, unsigned int wimg_bits); +extern void pmap_unmap_cpu_windows_copy(unsigned int index); + +extern void pt_fake_zone_init(int); +extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, + uint64_t *, int *, int *, int *); + +extern boolean_t pmap_valid_page(ppnum_t pn); + +#define MACHINE_PMAP_IS_EMPTY 1 +extern boolean_t pmap_is_empty(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end); + +#define ARM_PMAP_MAX_OFFSET_DEFAULT 0x01 +#define ARM_PMAP_MAX_OFFSET_MIN 0x02 +#define ARM_PMAP_MAX_OFFSET_MAX 0x04 +#define ARM_PMAP_MAX_OFFSET_DEVICE 0x08 +#define ARM_PMAP_MAX_OFFSET_JUMBO 0x10 + +#define ASID_SHIFT (11) /* Shift for the maximum virtual ASID value (2048) */ +#define MAX_ASID (1 << ASID_SHIFT) /* Max supported ASIDs (can be virtual) */ +#define ARM_ASID_SHIFT (8) /* Shift for the maximum ARM ASID value (256) */ +#define ARM_MAX_ASID (1 << ARM_ASID_SHIFT) /* Max ASIDs supported by the hardware */ +#define ASID_VIRT_BITS (ASID_SHIFT - ARM_ASID_SHIFT) /* The number of virtual bits in a virtaul ASID */ +#define NBBY 8 + +extern vm_map_offset_t pmap_max_offset(boolean_t is64, unsigned int option); + +boolean_t pmap_virtual_region(unsigned int region_select, vm_map_offset_t *startp, vm_map_size_t *size); + +boolean_t pmap_enforces_execute_only(pmap_t pmap); + +/* pmap dispatch indices */ +#define ARM_FAST_FAULT_INDEX 0 +#define ARM_FORCE_FAST_FAULT_INDEX 1 +#define MAPPING_FREE_PRIME_INDEX 2 +#define MAPPING_REPLENISH_INDEX 3 +#define PHYS_ATTRIBUTE_CLEAR_INDEX 4 +#define PHYS_ATTRIBUTE_SET_INDEX 5 +#define PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX 6 +#define PMAP_CHANGE_WIRING_INDEX 7 +#define PMAP_CREATE_INDEX 8 +#define PMAP_DESTROY_INDEX 9 +#define PMAP_ENTER_OPTIONS_INDEX 10 +#define PMAP_EXTRACT_INDEX 11 +#define PMAP_FIND_PHYS_INDEX 12 +#define PMAP_INSERT_SHAREDPAGE_INDEX 13 +#define PMAP_IS_EMPTY_INDEX 14 +#define PMAP_MAP_CPU_WINDOWS_COPY_INDEX 15 +#define PMAP_MARK_PAGE_AS_PMAP_PAGE_INDEX 16 +#define PMAP_NEST_INDEX 17 +#define PMAP_PAGE_PROTECT_OPTIONS_INDEX 18 +#define PMAP_PROTECT_OPTIONS_INDEX 19 +#define PMAP_QUERY_PAGE_INFO_INDEX 20 +#define PMAP_QUERY_RESIDENT_INDEX 21 +#define PMAP_REFERENCE_INDEX 22 +#define PMAP_REMOVE_OPTIONS_INDEX 23 +#define PMAP_RETURN_INDEX 24 +#define PMAP_SET_CACHE_ATTRIBUTES_INDEX 25 +#define PMAP_SET_NESTED_INDEX 26 +#define PMAP_SET_PROCESS_INDEX 27 +#define PMAP_SWITCH_INDEX 28 +#define PMAP_SWITCH_USER_TTB_INDEX 29 +#define PMAP_UNHINT_KV_ADDR_INDEX 30 +#define PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX 31 +#define PMAP_UNNEST_OPTIONS_INDEX 32 +#define PMAP_FOOTPRINT_SUSPEND_INDEX 33 +#define PMAP_CPU_DATA_INIT_INDEX 34 +#define PMAP_RELEASE_PAGES_TO_KERNEL_INDEX 35 + +#define MAX_PMAP_INDEX 36 + +#define PMAP_INVALID_CPU_NUM (~0U) + +struct pmap_cpu_data { + pmap_t cpu_user_pmap; + unsigned int cpu_number; + unsigned int cpu_user_pmap_stamp; + + /* + * This supports overloading of ARM ASIDs by the pmap. The field needs + * to be wide enough to cover all the virtual bits in a virtual ASID. + * With 256 physical ASIDs, 8-bit fields let us support up to 65536 + * Virtual ASIDs, minus all that would map on to 0 (as 0 is a global + * ASID). + * + * If we were to use bitfield shenanigans here, we could save a bit of + * memory by only having enough bits to support MAX_ASID. However, such + * an implementation would be more error prone. + */ + uint8_t cpu_asid_high_bits[ARM_MAX_ASID]; +}; + +typedef struct pmap_cpu_data pmap_cpu_data_t; + +/* Initialize the pmap per-CPU data for the current CPU. */ +extern void pmap_cpu_data_init(void); + +/* Get the pmap per-CPU data for the current CPU. */ +extern pmap_cpu_data_t * pmap_get_cpu_data(void); + +#define MARK_AS_PMAP_TEXT +#define MARK_AS_PMAP_DATA + +extern kern_return_t pmap_return(boolean_t do_panic, boolean_t do_recurse); + +#endif /* #ifndef ASSEMBLER */ + +#endif /* #ifndef _ARM_PMAP_H_ */ diff --git a/osfmk/arm/proc_reg.h b/osfmk/arm/proc_reg.h new file mode 100644 index 000000000..ca58e18ae --- /dev/null +++ b/osfmk/arm/proc_reg.h @@ -0,0 +1,1084 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* CMU_ENDHIST */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +/* + */ + +/* + * Processor registers for ARM + */ +#ifndef _ARM_PROC_REG_H_ +#define _ARM_PROC_REG_H_ + +#if defined (__arm64__) +#include +#elif defined (__arm__) +#include +#endif + +#if defined (ARMA7) +#define __ARM_ARCH__ 7 +#define __ARM_SUB_ARCH__ CPU_ARCH_ARMv7k +#define __ARM_VMSA__ 7 +#define __ARM_VFP__ 3 +#if defined(__XNU_UP__) +#define __ARM_SMP__ 0 +#else +#define __ARM_SMP__ 1 +/* For SMP kernels, force physical aperture to be mapped at PTE level so that its mappings + * can be updated to reflect cache attribute changes on alias mappings. This prevents + * prefetched physical aperture cachelines from becoming dirty in L1 due to a write to + * an uncached alias mapping on the same core. Subsequent uncached writes from another + * core may not snoop this line, and the dirty line may end up being evicted later to + * effectively overwrite the uncached writes from other cores. */ +#define __ARM_PTE_PHYSMAP__ 1 +#endif +/* __ARMA7_SMP__ controls whether we are consistent with the A7 MP_CORE spec; needed because entities other than + * the xnu-managed processors may need to snoop our cache operations. + */ +#define __ARMA7_SMP__ 1 +#define __ARM_COHERENT_CACHE__ 1 +#define __ARM_L1_PTW__ 1 +#define __ARM_DEBUG__ 7 +#define __ARM_USER_PROTECT__ 1 +#define __ARM_TIME_TIMEBASE_ONLY__ 1 + +#elif defined (APPLECYCLONE) +#define __ARM_ARCH__ 8 +#define __ARM_VMSA__ 8 +#define __ARM_SMP__ 1 +#define __ARM_VFP__ 4 +#define __ARM_COHERENT_CACHE__ 1 +#define __ARM_COHERENT_IO__ 1 +#define __ARM_IC_NOALIAS_ICACHE__ 1 +#define __ARM_L1_PTW__ 1 +#define __ARM_DEBUG__ 7 +#define __ARM_ENABLE_SWAP__ 1 +#define __ARM_V8_CRYPTO_EXTENSIONS__ 1 +#define __ARM64_PMAP_SUBPAGE_L1__ 1 + +#elif defined (APPLETYPHOON) +#define __ARM_ARCH__ 8 +#define __ARM_VMSA__ 8 +#define __ARM_SMP__ 1 +#define __ARM_VFP__ 4 +#define __ARM_COHERENT_CACHE__ 1 +#define __ARM_COHERENT_IO__ 1 +#define __ARM_IC_NOALIAS_ICACHE__ 1 +#define __ARM_L1_PTW__ 1 +#define __ARM_DEBUG__ 7 +#define __ARM_ENABLE_SWAP__ 1 +#define __ARM_V8_CRYPTO_EXTENSIONS__ 1 +#define __ARM64_PMAP_SUBPAGE_L1__ 1 + +#elif defined (APPLETWISTER) +#define __ARM_ARCH__ 8 +#define __ARM_VMSA__ 8 +#define __ARM_SMP__ 1 +#define __ARM_VFP__ 4 +#define __ARM_COHERENT_CACHE__ 1 +#define __ARM_COHERENT_IO__ 1 +#define __ARM_IC_NOALIAS_ICACHE__ 1 +#define __ARM_L1_PTW__ 1 +#define __ARM_DEBUG__ 7 +#define __ARM_ENABLE_SWAP__ 1 +#define __ARM_V8_CRYPTO_EXTENSIONS__ 1 +#define __ARM_16K_PG__ 1 +#define __ARM64_TWO_LEVEL_PMAP__ 1 + +#elif defined (APPLEHURRICANE) +#define __ARM_ARCH__ 8 +#define __ARM_VMSA__ 8 +#define __ARM_SMP__ 1 +#define __ARM_VFP__ 4 +#define __ARM_COHERENT_CACHE__ 1 +#define __ARM_COHERENT_IO__ 1 +#define __ARM_IC_NOALIAS_ICACHE__ 1 +#define __ARM_L1_PTW__ 1 +#define __ARM_DEBUG__ 7 +#define __ARM_ENABLE_SWAP__ 1 +#define __ARM_V8_CRYPTO_EXTENSIONS__ 1 +#define __ARM_16K_PG__ 1 +#define __ARM64_PMAP_SUBPAGE_L1__ 1 +#define __ARM_GLOBAL_SLEEP_BIT__ 1 +#define __ARM_PAN_AVAILABLE__ 1 + +#else +#error processor not supported +#endif + +#if defined(ARM_BOARD_WFE_TIMEOUT_NS) +#define __ARM_ENABLE_WFE_ 1 +#else +#define __ARM_ENABLE_WFE_ 0 +#endif + +#define CONFIG_THREAD_GROUPS 0 + + +#ifdef XNU_KERNEL_PRIVATE + +#if __ARM_VFP__ +#define ARM_VFP_DEBUG 0 +#endif + +#endif + + + +/* + * FSR registers + * + * CPSR: Current Program Status Register + * SPSR: Saved Program Status Registers + * + * 31 30 29 28 27 24 19 16 9 8 7 6 5 4 0 + * +-----------------------------------------------------------+ + * | N| Z| C| V| Q|...| J|...|GE[3:0]|...| E| A| I| F| T| MODE | + * +-----------------------------------------------------------+ + */ + +/* + * Flags + */ +#define PSR_NF 0x80000000 /* Negative/Less than */ +#define PSR_ZF 0x40000000 /* Zero */ +#define PSR_CF 0x20000000 /* Carry/Borrow/Extend */ +#define PSR_VF 0x10000000 /* Overflow */ +#define PSR_QF 0x08000000 /* saturation flag (QADD ARMv5) */ + +/* + * Modified execution mode flags + */ +#define PSR_JF 0x01000000 /* Jazelle flag (BXJ ARMv5) */ +#define PSR_EF 0x00000200 /* mixed-endian flag (SETEND ARMv6) */ +#define PSR_AF 0x00000100 /* precise abort flag (ARMv6) */ +#define PSR_TF 0x00000020 /* thumb flag (BX ARMv4T) */ +#define PSR_TFb 5 /* thumb flag (BX ARMv4T) */ + +/* + * Interrupts + */ +#define PSR_IRQFb 7 /* IRQ : 0 = IRQ enable */ +#define PSR_IRQF 0x00000080 /* IRQ : 0 = IRQ enable */ +#define PSR_FIQF 0x00000040 /* FIQ : 0 = FIQ enable */ + +/* + * CPU mode + */ +#define PSR_USER_MODE 0x00000010 /* User mode */ +#define PSR_FIQ_MODE 0x00000011 /* FIQ mode */ +#define PSR_IRQ_MODE 0x00000012 /* IRQ mode */ +#define PSR_SVC_MODE 0x00000013 /* Supervisor mode */ +#define PSR_ABT_MODE 0x00000017 /* Abort mode */ +#define PSR_UND_MODE 0x0000001B /* Undefined mode */ + +#define PSR_MODE_MASK 0x0000001F +#define PSR_IS_KERNEL(psr) (((psr) & PSR_MODE_MASK) != PSR_USER_MODE) +#define PSR_IS_USER(psr) (((psr) & PSR_MODE_MASK) == PSR_USER_MODE) + +#define PSR_USERDFLT PSR_USER_MODE +#define PSR_USER_MASK (PSR_AF | PSR_IRQF | PSR_FIQF | PSR_MODE_MASK) +#define PSR_USER_SET PSR_USER_MODE + +#define PSR_INTMASK PSR_IRQF /* Interrupt disable */ + +/* + * FPEXC: Floating-Point Exception Register + */ + +#define FPEXC_EX 0x80000000 /* Exception status */ +#define FPEXC_EX_BIT 31 +#define FPEXC_EN 0x40000000 /* VFP : 1 = EN enable */ +#define FPEXC_EN_BIT 30 + + +/* + * FPSCR: Floating-point Status and Control Register + */ + +#define FPSCR_DN 0x02000000 /* Default NaN */ +#define FPSCR_FZ 0x01000000 /* Flush to zero */ + +#define FPSCR_DEFAULT FPSCR_DN | FPSCR_FZ + + +/* + * FSR registers + * + * IFSR: Instruction Fault Status Register + * DFSR: Data Fault Status Register + */ +#define FSR_ALIGN 0x00000001 /* Alignment */ +#define FSR_DEBUG 0x00000002 /* Debug (watch/break) */ +#define FSR_ICFAULT 0x00000004 /* Fault on instruction cache maintenance */ +#define FSR_SFAULT 0x00000005 /* Translation Section */ +#define FSR_PFAULT 0x00000007 /* Translation Page */ +#define FSR_SACCESS 0x00000003 /* Section access */ +#define FSR_PACCESS 0x00000006 /* Page Access */ +#define FSR_SDOM 0x00000009 /* Domain Section */ +#define FSR_PDOM 0x0000000B /* Domain Page */ +#define FSR_SPERM 0x0000000D /* Permission Section */ +#define FSR_PPERM 0x0000000F /* Permission Page */ +#define FSR_EXT 0x00001000 /* External (Implementation Defined Classification) */ + +#define FSR_MASK 0x0000040F /* Valid bits */ +#define FSR_ALIGN_MASK 0x0000040D /* Valid bits to check align */ + +#define DFSR_WRITE 0x00000800 /* write data abort fault */ + +#if defined (ARMA7) || defined (APPLE_ARM64_ARCH_FAMILY) + +#define TEST_FSR_VMFAULT(status) \ + (((status) == FSR_PFAULT) \ + || ((status) == FSR_PPERM) \ + || ((status) == FSR_SFAULT) \ + || ((status) == FSR_SPERM) \ + || ((status) == FSR_ICFAULT) \ + || ((status) == FSR_SACCESS) \ + || ((status) == FSR_PACCESS)) + +#else + +#error Incompatible CPU type configured + +#endif + +/* + * Cache configuration + */ + +#if defined (ARMA7) + +/* I-Cache */ +#define MMU_I_CLINE 5 /* cache line size as 1<>1)&0x1)<>2)&0x1)<> PTE_SHIFT) /* number of ptes per page */ + +#define ARM_PTE_EMPTY 0x00000000 /* unasigned - invalid entry */ + +/* markers for (invalid) PTE for a page sent to compressor */ +#define ARM_PTE_COMPRESSED ARM_PTE_TEX1 /* compressed... */ +#define ARM_PTE_COMPRESSED_ALT ARM_PTE_TEX2 /* ... and was "alt_acct" */ +#define ARM_PTE_COMPRESSED_MASK (ARM_PTE_COMPRESSED | ARM_PTE_COMPRESSED_ALT) +#define ARM_PTE_IS_COMPRESSED(x) \ + ((((x) & 0x3) == 0) && /* PTE is not valid... */ \ + ((x) & ARM_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \ + ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \ + (panic("compressed PTE %p 0x%x has extra bits 0x%x: corrupted?", \ + &(x), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE))) + +#define ARM_PTE_TYPE_FAULT 0x00000000 /* fault entry type */ +#define ARM_PTE_TYPE 0x00000002 /* small page entry type */ +#define ARM_PTE_TYPE_MASK 0x00000002 /* mask to get pte type */ + +#define ARM_PTE_NG_MASK 0x00000800 /* mask to determine notGlobal bit */ +#define ARM_PTE_NG 0x00000800 /* value for a per-process mapping */ + +#define ARM_PTE_SHSHIFT 10 +#define ARM_PTE_SH_MASK 0x00000400 /* shared (SMP) mapping mask */ +#define ARM_PTE_SH 0x00000400 /* shared (SMP) mapping */ + +#define ARM_PTE_CBSHIFT 2 +#define ARM_PTE_CB(x) ((x)<>1)&0x1)<>2)&0x1)< + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#if __arm64__ +#include +#elif __arm__ +#include +#else +#error Unsupported arch +#endif +#include + +#include +#include + +#include + +#define MAX_TIMEBASE_TRIES 10 + +int rtclock_init(void); + +static int +deadline_to_decrementer(uint64_t deadline, + uint64_t now); +static void +timebase_callback(struct timebase_freq_t * freq); + +#if DEVELOPMENT || DEBUG +uint32_t absolute_time_validation = 1; +#endif + +/* + * Configure the real-time clock device at boot + */ +void +rtclock_early_init(void) +{ + PE_register_timebase_callback(timebase_callback); +#if DEVELOPMENT || DEBUG + uint32_t tmp_mv = 1; + if (kern_feature_override(KF_MATV_OVRD)) { + absolute_time_validation = 0; + } + if (PE_parse_boot_argn("timebase_validation", &tmp_mv, sizeof(tmp_mv))) { + if (tmp_mv == 0) { + absolute_time_validation = 0; + } + } +#endif +} + +static void +timebase_callback(struct timebase_freq_t * freq) +{ + unsigned long numer, denom; + uint64_t t64_1, t64_2; + uint32_t divisor; + + if (freq->timebase_den < 1 || freq->timebase_den > 4 || + freq->timebase_num < freq->timebase_den) + panic("rtclock timebase_callback: invalid constant %ld / %ld", + freq->timebase_num, freq->timebase_den); + + denom = freq->timebase_num; + numer = freq->timebase_den * NSEC_PER_SEC; + // reduce by the greatest common denominator to minimize overflow + if (numer > denom) { + t64_1 = numer; + t64_2 = denom; + } else { + t64_1 = denom; + t64_2 = numer; + } + while (t64_2 != 0) { + uint64_t temp = t64_2; + t64_2 = t64_1 % t64_2; + t64_1 = temp; + } + numer /= t64_1; + denom /= t64_1; + + rtclock_timebase_const.numer = (uint32_t)numer; + rtclock_timebase_const.denom = (uint32_t)denom; + divisor = (uint32_t)(freq->timebase_num / freq->timebase_den); + + rtclock_sec_divisor = divisor; + rtclock_usec_divisor = divisor / USEC_PER_SEC; +} + +/* + * Initialize the system clock device for the current cpu + */ +int +rtclock_init(void) +{ + uint64_t abstime; + cpu_data_t * cdp; + + clock_timebase_init(); + ml_init_lock_timeout(); + + cdp = getCpuDatap(); + + abstime = mach_absolute_time(); + cdp->rtcPop = EndOfAllTime; /* Init Pop time */ + timer_resync_deadlines(); /* Start the timers going */ + + return (1); +} + +uint64_t +mach_absolute_time(void) +{ +#if DEVELOPMENT || DEBUG + if (__improbable(absolute_time_validation == 1)) { + static volatile uint64_t s_last_absolute_time = 0; + uint64_t new_absolute_time, old_absolute_time; + int attempts = 0; + + /* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time + * completes before the timebase read. Were the load to complete after the + * timebase read, there would be a window for another CPU to update + * s_last_absolute_time and leave us in an inconsistent state. Consider the + * following interleaving: + * + * Let s_last_absolute_time = t0 + * CPU0: Read timebase at t1 + * CPU1: Read timebase at t2 + * CPU1: Update s_last_absolute_time to t2 + * CPU0: Load completes + * CPU0: Update s_last_absolute_time to t1 + * + * This would cause the assertion to fail even though time did not go + * backwards. Thus, we use a dsb to guarantee completion of the load before + * the timebase read. + */ + do { + attempts++; + old_absolute_time = s_last_absolute_time; + +#if __arm64__ + __asm__ volatile("dsb ld" ::: "memory"); +#else + OSSynchronizeIO(); // See osfmk/arm64/rtclock.c +#endif + + new_absolute_time = ml_get_timebase(); + } while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time)); + + if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) { + panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)\n", + new_absolute_time, old_absolute_time); + } + return new_absolute_time; + } else { + return ml_get_timebase(); + } +#else + return ml_get_timebase(); +#endif +} + +uint64_t +mach_approximate_time(void) +{ +#if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__ + /* Hardware supports a fast timestamp, so grab it without asserting monotonicity */ + return ml_get_timebase(); +#else + processor_t processor; + uint64_t approx_time; + + disable_preemption(); + processor = current_processor(); + approx_time = processor->last_dispatch; + enable_preemption(); + + return approx_time; +#endif +} + +void +clock_get_system_microtime(clock_sec_t * secs, + clock_usec_t * microsecs) +{ + absolutetime_to_microtime(mach_absolute_time(), secs, microsecs); +} + +void +clock_get_system_nanotime(clock_sec_t * secs, + clock_nsec_t * nanosecs) +{ + uint64_t abstime; + uint64_t t64; + + abstime = mach_absolute_time(); + *secs = (t64 = abstime / rtclock_sec_divisor); + abstime -= (t64 * rtclock_sec_divisor); + + *nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor); +} + +void +clock_gettimeofday_set_commpage(uint64_t abstime, + uint64_t sec, + uint64_t frac, + uint64_t scale, + uint64_t tick_per_sec) +{ + commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec); +} + +void +clock_timebase_info(mach_timebase_info_t info) +{ + *info = rtclock_timebase_const; +} + +/* + * Real-time clock device interrupt. + */ +void +rtclock_intr(__unused unsigned int is_user_context) +{ + uint64_t abstime; + cpu_data_t * cdp; + struct arm_saved_state * regs; + unsigned int user_mode; + uintptr_t pc; + + cdp = getCpuDatap(); + + cdp->cpu_stat.timer_cnt++; + cdp->cpu_stat.timer_cnt_wake++; + SCHED_STATS_TIMER_POP(current_processor()); + + assert(!ml_get_interrupts_enabled()); + + abstime = mach_absolute_time(); + + if (cdp->cpu_idle_pop != 0x0ULL) { + if (( cdp->rtcPop-abstime) < cdp->cpu_idle_latency) { + cdp->cpu_idle_pop = 0x0ULL; + while (abstime < cdp->rtcPop) + abstime = mach_absolute_time(); + } else { + ClearIdlePop(FALSE); + } + } + + if ((regs = cdp->cpu_int_state)) { + pc = get_saved_state_pc(regs); + +#if __arm64__ + user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs)); +#else + user_mode = (regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE; +#endif + } else { + pc = 0; + user_mode = 0; + } + if (abstime >= cdp->rtcPop) { + /* Log the interrupt service latency (-ve value expected by tool) */ + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE, + -(abstime - cdp->rtcPop), + user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode, 0, 0); + } + + /* call the generic etimer */ + timer_intr(user_mode, pc); +} + +static int +deadline_to_decrementer(uint64_t deadline, + uint64_t now) +{ + uint64_t delt; + + if (deadline <= now) + return DECREMENTER_MIN; + else { + delt = deadline - now; + + return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN); + } +} + +/* + * Request a decrementer pop + */ +int +setPop(uint64_t time) +{ + int delay_time; + uint64_t current_time; + cpu_data_t * cdp; + + cdp = getCpuDatap(); + current_time = mach_absolute_time(); + + delay_time = deadline_to_decrementer(time, current_time); + cdp->rtcPop = delay_time + current_time; + + ml_set_decrementer((uint32_t) delay_time); + + return (delay_time); +} + +/* + * Request decrementer Idle Pop. Return true if set + */ +boolean_t +SetIdlePop(void) +{ + int delay_time; + uint64_t time; + uint64_t current_time; + cpu_data_t * cdp; + + cdp = getCpuDatap(); + current_time = mach_absolute_time(); + + if (((cdp->rtcPop < current_time) || + (cdp->rtcPop - current_time) < cdp->cpu_idle_latency)) + return FALSE; + + time = cdp->rtcPop - cdp->cpu_idle_latency; + + delay_time = deadline_to_decrementer(time, current_time); + cdp->cpu_idle_pop = delay_time + current_time; + ml_set_decrementer((uint32_t) delay_time); + + return TRUE; +} + +/* + * Clear decrementer Idle Pop + */ +void +ClearIdlePop( + boolean_t wfi) +{ +#if !__arm64__ +#pragma unused(wfi) +#endif + cpu_data_t * cdp; + + cdp = getCpuDatap(); + cdp->cpu_idle_pop = 0x0ULL; + +#if __arm64__ + /* + * Don't update the HW timer if there's a pending + * interrupt (we can lose interrupt assertion); + * we want to take the interrupt right now and update + * the deadline from the handler). + * + * ARM64_TODO: consider this more carefully. + */ + if (!(wfi && ml_get_timer_pending())) +#endif + { + setPop(cdp->rtcPop); + } +} + +void +absolutetime_to_microtime(uint64_t abstime, + clock_sec_t * secs, + clock_usec_t * microsecs) +{ + uint64_t t64; + + *secs = t64 = abstime / rtclock_sec_divisor; + abstime -= (t64 * rtclock_sec_divisor); + + *microsecs = (uint32_t)(abstime / rtclock_usec_divisor); +} + +void +absolutetime_to_nanoseconds(uint64_t abstime, + uint64_t * result) +{ + uint64_t t64; + + *result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC; + abstime -= (t64 * rtclock_sec_divisor); + *result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor; +} + +void +nanoseconds_to_absolutetime(uint64_t nanosecs, + uint64_t * result) +{ + uint64_t t64; + + *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor; + nanosecs -= (t64 * NSEC_PER_SEC); + *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; +} + +void +nanotime_to_absolutetime(clock_sec_t secs, + clock_nsec_t nanosecs, + uint64_t * result) +{ + *result = ((uint64_t) secs * rtclock_sec_divisor) + + ((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; +} + +void +clock_interval_to_absolutetime_interval(uint32_t interval, + uint32_t scale_factor, + uint64_t * result) +{ + uint64_t nanosecs = (uint64_t) interval * scale_factor; + uint64_t t64; + + *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor; + nanosecs -= (t64 * NSEC_PER_SEC); + *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; +} + +void +machine_delay_until(uint64_t interval, + uint64_t deadline) +{ +#pragma unused(interval) + uint64_t now; + + do { +#if __ARM_ENABLE_WFE_ +#if __arm64__ + if (arm64_wfe_allowed()) +#endif /* __arm64__ */ + { + __builtin_arm_wfe(); + } +#endif /* __ARM_ENABLE_WFE_ */ + + now = mach_absolute_time(); + } while (now < deadline); +} diff --git a/osfmk/arm/rtclock.h b/osfmk/arm/rtclock.h new file mode 100644 index 000000000..fb051b2ab --- /dev/null +++ b/osfmk/arm/rtclock.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * @APPLE_FREE_COPYRIGHT@ + */ + +#ifndef _ARM_RTCLOCK_H_ +#define _ARM_RTCLOCK_H_ + +#include +#include +#include +#include + +#define EndOfAllTime 0xFFFFFFFFFFFFFFFFULL +#define DECREMENTER_MAX 0x7FFFFFFFUL +#define DECREMENTER_MIN 0xAUL + +typedef struct _rtclock_data_ { + uint32_t rtc_sec_divisor; + uint32_t rtc_usec_divisor; + mach_timebase_info_data_t rtc_timebase_const; + union { + uint64_t abstime; + struct { + uint32_t low; + uint32_t high; + } abstime_val; + } rtc_base; + union { + uint64_t abstime; + struct { + uint32_t low; + uint32_t high; + } abstime_val; + } rtc_adj; + tbd_ops_data_t rtc_timebase_func; + + /* Only needed for AIC manipulation */ + vm_offset_t rtc_timebase_addr; + vm_offset_t rtc_timebase_val; + +} rtclock_data_t; + +extern rtclock_data_t RTClockData; +#define rtclock_sec_divisor RTClockData.rtc_sec_divisor +#define rtclock_usec_divisor RTClockData.rtc_usec_divisor +#define rtclock_timebase_const RTClockData.rtc_timebase_const +#define rtclock_base_abstime RTClockData.rtc_base.abstime +#define rtclock_base_abstime_low RTClockData.rtc_base.abstime_val.low +#define rtclock_base_abstime_high RTClockData.rtc_base.abstime_val.high +#define rtclock_adj_abstime RTClockData.rtc_adj.abstime +#define rtclock_adj_abstime_low RTClockData.rtc_adj.abstime_val.low +#define rtclock_adj_abstime_high RTClockData.rtc_adj.abstime_val.high +#define rtclock_timebase_func RTClockData.rtc_timebase_func + +/* Only needed for AIC manipulation */ +#define rtclock_timebase_addr RTClockData.rtc_timebase_addr +#define rtclock_timebase_val RTClockData.rtc_timebase_val + +extern uint64_t arm_timer_slop_max; + +extern void rtclock_intr(unsigned int); +extern boolean_t SetIdlePop(void); + +extern void ClearIdlePop(boolean_t); +extern void rtclock_early_init(void); + +#endif /* _ARM_RTCLOCK_H_ */ diff --git a/osfmk/arm/sched_param.h b/osfmk/arm/sched_param.h new file mode 100644 index 000000000..a3d20dc47 --- /dev/null +++ b/osfmk/arm/sched_param.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +/* + */ + +/* + * Scheduler parameters. + */ + +#ifndef _ARM_SCHED_PARAM_H_ +#define _ARM_SCHED_PARAM_H_ + +#endif /* _ARM_SCHED_PARAM_H_ */ diff --git a/osfmk/arm/setjmp.h b/osfmk/arm/setjmp.h new file mode 100644 index 000000000..a3a2f5ead --- /dev/null +++ b/osfmk/arm/setjmp.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +/* + * Setjmp/longjmp buffer for ARM. + */ +#ifndef _ARM_SETJMP_H_ +#define _ARM_SETJMP_H_ + +typedef struct jmp_buf { + int jmp_buf[28]; +} jmp_buf_t; + +#endif /* _ARM_SETJMP_H_ */ diff --git a/osfmk/arm/simple_lock.h b/osfmk/arm/simple_lock.h new file mode 100644 index 000000000..0fb708ede --- /dev/null +++ b/osfmk/arm/simple_lock.h @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#ifdef KERNEL_PRIVATE + +#ifndef _ARM_SIMPLE_LOCK_TYPES_H_ +#define _ARM_SIMPLE_LOCK_TYPES_H_ + +#ifdef KERNEL_PRIVATE +#include +#include + +#include +#ifdef MACH_KERNEL_PRIVATE +#include +#include +#include +#endif + +#ifdef MACH_KERNEL_PRIVATE + +typedef uint32_t hw_lock_bit_t; + +extern void hw_lock_bit( + hw_lock_bit_t *, + unsigned int); + +extern void hw_unlock_bit( + hw_lock_bit_t *, + unsigned int); + +extern unsigned int hw_lock_bit_try( + hw_lock_bit_t *, + unsigned int); + +extern unsigned int hw_lock_bit_to( + hw_lock_bit_t *, + unsigned int, + uint32_t); + +#define hw_lock_bit_held(l,b) (((*(l))&(1< + +#define __SMP__ __ARM_SMP__ +#define __AMP__ __ARM_AMP__ + +#endif /* _ARM_SMP_H_ */ diff --git a/osfmk/arm/start.s b/osfmk/arm/start.s new file mode 100644 index 000000000..ced8c0d3e --- /dev/null +++ b/osfmk/arm/start.s @@ -0,0 +1,434 @@ +/* + * Copyright (c) 2007-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include "assym.s" + + .text + .align 12 + + .align 2 + .globl EXT(resume_idle_cpu) +LEXT(resume_idle_cpu) + // r0 set to BootArgs phys address + // r1 set to cpu data phys address + LOAD_ADDR(lr, arm_init_idle_cpu) + b L_start_cpu_0 + + .globl EXT(start_cpu) +LEXT(start_cpu) + // r0 set to BootArgs phys address + // r1 set to cpu data phys address + LOAD_ADDR(lr, arm_init_cpu) + b L_start_cpu_0 + +L_start_cpu_0: + cpsid if // Disable IRQ FIQ + + // Turn on L1 I-Cache, Branch prediction early + mcr p15, 0, r11, c7, c5, 0 // invalidate the icache + isb // before moving on + mrc p15, 0, r11, c1, c0, 0 // read mmu control into r11 + orr r11, r11, #(SCTLR_ICACHE | SCTLR_PREDIC) // enable i-cache, b-prediction + mcr p15, 0, r11, c1, c0, 0 // set mmu control + dsb // ensure mmu settings are inplace + isb // before moving on + + // Get the kernel's phys & virt addr, and size from BootArgs + ldr r8, [r0, BA_PHYS_BASE] // Get the phys base in r8 + ldr r9, [r0, BA_VIRT_BASE] // Get the virt base in r9 + ldr r10, [r0, BA_MEM_SIZE] // Get the mem size in r10 + + // Set the base of the translation table into the MMU + ldr r4, [r0, BA_TOP_OF_KERNEL_DATA] // Get the top of kernel data + orr r5, r4, #(TTBR_SETUP & 0x00FF) // Setup PTWs memory attribute + orr r5, r5, #(TTBR_SETUP & 0xFF00) // Setup PTWs memory attribute + mcr p15, 0, r5, c2, c0, 0 // write kernel to translation table base 0 + mcr p15, 0, r5, c2, c0, 1 // also to translation table base 1 + mov r5, #TTBCR_N_1GB_TTB0 // identify the split between 0 and 1 + mcr p15, 0, r5, c2, c0, 2 // and set up the translation control reg + ldr r2, [r1, CPU_NUMBER_GS] // Get cpu number + mcr p15, 0, r2, c13, c0, 3 // Write TPIDRURO + ldr sp, [r1, CPU_INTSTACK_TOP] // Get interrupt stack top + sub sp, sp, SS_SIZE // Set stack pointer + sub r0, r1, r8 // Convert to virtual address + add r0, r0, r9 + b join_start + + .align 2 + .globl EXT(_start) +LEXT(_start) + // r0 has the boot-args pointer + // r1 set to zero + mov r1, #0 + LOAD_ADDR(lr, arm_init) + cpsid if // Disable IRQ FIQ + + // Turn on L1 I-Cache, Branch prediction early + mcr p15, 0, r11, c7, c5, 0 // invalidate the icache + isb // before moving on + mrc p15, 0, r11, c1, c0, 0 // read mmu control into r11 + orr r11, r11, #(SCTLR_ICACHE | SCTLR_PREDIC) // enable i-cache, b-prediction + mcr p15, 0, r11, c1, c0, 0 // set mmu control + dsb // ensure mmu settings are inplace + isb // before moving on + + // Get the kernel's phys & virt addr, and size from boot_args. + ldr r8, [r0, BA_PHYS_BASE] // Get the phys base in r8 + ldr r9, [r0, BA_VIRT_BASE] // Get the virt base in r9 + ldr r10, [r0, BA_MEM_SIZE] // Get the mem size in r10 + +#define LOAD_PHYS_ADDR(reg, label) \ + LOAD_ADDR(reg, label); \ + sub reg, reg, r9; \ + add reg, reg, r8 + + // Take this opportunity to patch the targets for the exception vectors + LOAD_ADDR(r4, fleh_reset) + LOAD_PHYS_ADDR(r5, ExceptionVectorsTable) + str r4, [r5] + LOAD_ADDR(r4, fleh_undef) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_swi) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_prefabt) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_dataabt) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_addrexc) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_irq) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_decirq) + add r5, #4 + str r4, [r5] + + // arm_init_tramp is sensitive, so for the moment, take the opportunity to store the + // virtual address locally, so that we don't run into issues retrieving it later. + // This is a pretty miserable solution, but it should be enough for the moment + LOAD_ADDR(r4, arm_init_tramp) + adr r5, arm_init_tramp_addr + str r4, [r5] + +#undef LOAD_PHYS_ADDR + + // Set the base of the translation table into the MMU + ldr r4, [r0, BA_TOP_OF_KERNEL_DATA] // Get the top of kernel data + orr r5, r4, #(TTBR_SETUP & 0x00FF) // Setup PTWs memory attribute + orr r5, r5, #(TTBR_SETUP & 0xFF00) // Setup PTWs memory attribute + mcr p15, 0, r5, c2, c0, 0 // write kernel to translation table base 0 + mcr p15, 0, r5, c2, c0, 1 // also to translation table base 1 + mov r5, #TTBCR_N_1GB_TTB0 // identify the split between 0 and 1 + mcr p15, 0, r5, c2, c0, 2 // and set up the translation control reg + + // Mark the entries invalid in the 4 page trampoline translation table + // Mark the entries invalid in the 4 page CPU translation table + // Mark the entries invalid in the one page table for the final 1MB (if used) + // Mark the entries invalid in the one page table for HIGH_EXC_VECTORS + mov r5, r4 // local copy of base + mov r11, #ARM_TTE_TYPE_FAULT // invalid entry template + mov r2, PGBYTES >> 2 // number of ttes/page + add r2, r2, r2, LSL #2 // 8 ttes + 2 ptes to clear. Multiply by 5... + mov r2, r2, LSL #1 // ...then multiply by 2 +invalidate_tte: + str r11, [r5] // store the invalid tte + add r5, r5, #4 // increment tte pointer + subs r2, r2, #1 // decrement count + bne invalidate_tte + + // create default section tte template + mov r6, #ARM_TTE_TYPE_BLOCK // use block mapping entries + mov r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xFF) + orr r7, r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xFF00) + orr r7, r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xF0000) + orr r6, r6, r7 // with default cache attrs + mov r7, #ARM_TTE_BLOCK_AP(AP_RWNA) // Set kernel rw, user no access + orr r7, r7, #(ARM_TTE_BLOCK_AP(AP_RWNA) & 0xFF00) + orr r7, r7, #(ARM_TTE_BLOCK_AP(AP_RWNA) & 0xF0000) + orr r6, r6, r7 // Set RWNA protection + + orr r6, r6, #ARM_TTE_BLOCK_AF // Set access protection + orr r6, r6, #ARM_TTE_BLOCK_SH // Set shareability + + // Set up the V=P mapping for the 1 MB section around the current pc + lsr r7, pc, #ARM_TT_L1_SHIFT // Extract tte index for pc addr + add r5, r4, r7, LSL #2 // convert tte index to tte pointer + lsl r7, r7, #ARM_TT_L1_SHIFT // Truncate pc to 1MB aligned addr + orr r11, r7, r6 // make tte entry value + str r11, [r5] // store tte + + // Set up the virtual mapping for the kernel using 1Mb direct section TTE entries + mov r7, r8 // Save original phys base + add r5, r4, r9, LSR #ARM_TT_L1_SHIFT-2 // convert vaddr to tte pointer + mov r3, #ARM_TT_L1_SIZE // set 1MB boundary + +mapveqp: + cmp r3, r10 // Check if we're beyond the last 1MB section + bgt mapveqpL2 // If so, a coarse entry is required + + orr r11, r7, r6 // make tte entry value + str r11, [r5], #4 // store tte and move to next + add r7, r7, #ARM_TT_L1_SIZE // move to next phys addr + subs r10, r10, #ARM_TT_L1_SIZE // subtract tte size + bne mapveqp + b doneveqp // end is 1MB aligned, and we're done + +mapveqpL2: + // The end is not 1MB aligned, so steal a page and set up L2 entries within + + // Coarse entry first + add r6, r4, PGBYTES * 8 // add L2 offset + mov r11, r6 + + orr r6, #ARM_TTE_TYPE_TABLE // coarse entry + + str r6, [r5] // store coarse tte entry + + // Fill in the L2 entries + mov r5, r11 + + // create pte template + mov r2, #ARM_PTE_TYPE // default pte type + orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff) // with default cache attrs + orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff00) + orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff) // with default cache attrs + orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff00) + orr r2, r2, #ARM_PTE_AF // Set access + orr r2, r2, #ARM_PTE_SH // Set shareability + +storepte: + orr r11, r7, r2 // make pte entry value + str r11, [r5], #4 // store pte and move to next + add r7, r7, PGBYTES // move to next phys addr + subs r10, r10, PGBYTES // subtract pte size + bne storepte + +doneveqp: + // Insert page table page for high address exception vectors into translation table + mov r5, #0xff000000 // part of virt HIGH_EXC_VECTORS (HACK!) + orr r5, r5, #0x00ff0000 // rest of virt HIGH_EXC_VECTORS (HACK!) + mov r5, r5, LSR #ARM_TT_L1_SHIFT // convert virt addr to index + add r5, r4, r5, LSL #2 // convert to tte pointer + + add r6, r4, PGBYTES * 9 // get page table base (past 4 + 4 + 1 tte/pte pages) + mov r7, #(ARM_TTE_TABLE_MASK & 0xFFFF) // ARM_TTE_TABLE_MASK low halfword + movt r7, #(ARM_TTE_TABLE_MASK >> 16) // ARM_TTE_TABLE_MASK top halfword + and r11, r6, r7 // apply mask + orr r11, r11, #ARM_TTE_TYPE_TABLE // mark it as a coarse page table + str r11, [r5] // store tte entry for page table + + // create pte template + mov r2, #ARM_PTE_TYPE // pte type + orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0x00ff) // default cache attrs + orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff00) + orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0x00ff) // set RWNA protection + orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff00) + orr r2, r2, #ARM_PTE_AF // Set access + orr r2, r2, #ARM_PTE_SH // Set shareability + + // Now initialize the page table entry for the exception vectors + mov r5, #0xff000000 // part of HIGH_EXC_VECTORS + orr r5, r5, #0x00ff0000 // rest of HIGH_EXC_VECTORS + mov r7, #(ARM_TT_L2_INDEX_MASK & 0xFFFF) // ARM_TT_L2_INDEX_MASK low halfword + movt r7, #(ARM_TT_L2_INDEX_MASK >> 16) // ARM_TT_L2_INDEX_MASK top halfword + and r5, r5, r7 // mask for getting index + mov r5, r5, LSR #ARM_TT_L2_SHIFT // get page table index + add r5, r6, r5, LSL #2 // convert to pte pointer + + LOAD_ADDR(r11, ExceptionVectorsBase) // get address of vectors addr + sub r11, r11, r9 // convert to physical address + add r11, r11, r8 + + mov r7, #(ARM_PTE_PAGE_MASK & 0xFFFF) // ARM_PTE_PAGE_MASK low halfword + movt r7, #(ARM_PTE_PAGE_MASK >> 16) // ARM_PTE_PAGE_MASK top halfword + and r11, r11, r7 // insert masked address into pte + orr r11, r11, r2 // add template bits + str r11, [r5] // store pte by base and index + + // clean the dcache + mov r11, #0 +cleanflushway: +cleanflushline: + mcr p15, 0, r11, c7, c14, 2 // cleanflush dcache line by way/set + add r11, r11, #1 << MMU_I7SET // increment set index + tst r11, #1 << (MMU_NSET + MMU_I7SET) // look for overflow + beq cleanflushline + bic r11, r11, #1 << (MMU_NSET + MMU_I7SET) // clear set overflow + adds r11, r11, #1 << MMU_I7WAY // increment way + bcc cleanflushway // loop + +#if __ARM_L2CACHE__ + // Invalidate L2 cache + mov r11, #2 +invall2flushway: +invall2flushline: + mcr p15, 0, r11, c7, c14, 2 // Invalidate dcache line by way/set + add r11, r11, #1 << L2_I7SET // increment set index + tst r11, #1 << (L2_NSET + L2_I7SET) // look for overflow + beq invall2flushline + bic r11, r11, #1 << (L2_NSET + L2_I7SET) // clear set overflow + adds r11, r11, #1 << L2_I7WAY // increment way + bcc invall2flushway // loop + +#endif + + mov r11, #0 + mcr p15, 0, r11, c13, c0, 3 // Write TPIDRURO + LOAD_ADDR(sp, intstack_top) // Get interrupt stack top + sub sp, sp, SS_SIZE // Set stack pointer + sub r0, r0, r8 // Convert to virtual address + add r0, r0, r9 + +join_start: + // kernel page table is setup + // lr set to return handler function virtual address + // r0 set to return handler argument virtual address + // sp set to interrupt context stack virtual address + + // Cpu specific configuration + +#ifdef ARMA7 +#if __ARMA7_SMP__ + mrc p15, 0, r11, c1, c0, 1 + orr r11, r11, #(1<<6) // SMP + mcr p15, 0, r11, c1, c0, 1 + isb +#endif +#endif + + mrs r11, cpsr // Get cpsr + bic r11, #0x100 // Allow async aborts + msr cpsr_x, r11 // Update cpsr + + mov r11, #0 + mcr p15, 0, r11, c8, c7, 0 // invalidate all TLB entries + mcr p15, 0, r11, c7, c5, 0 // invalidate the icache + + // set DACR + mov r11, #(ARM_DAC_SETUP & 0xFFFF) // ARM_DAC_SETUP low halfword + movt r11, #(ARM_DAC_SETUP >> 16) // ARM_DAC_SETUP top halfword + mcr p15, 0, r11, c3, c0, 0 // write to dac register + + // Set PRRR + mov r11, #(PRRR_SETUP & 0xFFFF) // PRRR_SETUP low halfword + movt r11, #(PRRR_SETUP >> 16) // PRRR_SETUP top halfword + mcr p15, 0, r11, c10,c2,0 // write to PRRR register + + // Set NMRR + mov r11, #(NMRR_SETUP & 0xFFFF) // NMRR_SETUP low halfword + movt r11, #(NMRR_SETUP >> 16) // NMRR_SETUP top halfword + mcr p15, 0, r11, c10,c2,1 // write to NMRR register + + // set SCTLR + mrc p15, 0, r11, c1, c0, 0 // read system control + + bic r11, r11, #SCTLR_ALIGN // force off alignment exceptions + mov r7, #(SCTLR_AFE|SCTLR_TRE) // Access flag, TEX remap + orr r7, r7, #(SCTLR_HIGHVEC | SCTLR_ICACHE | SCTLR_PREDIC) + orr r7, r7, #(SCTLR_DCACHE | SCTLR_ENABLE) +#if (__ARM_ENABLE_SWAP__ == 1) + orr r7, r7, #SCTLR_SW // SWP/SWPB Enable +#endif + orr r11, r11, r7 // or in the default settings + mcr p15, 0, r11, c1, c0, 0 // set mmu control + + dsb // ensure mmu settings are inplace + isb // before moving on + +#if __ARM_VFP__ + // Initialize the VFP coprocessors. + mrc p15, 0, r2, c1, c0, 2 // read coprocessor control register + mov r3, #15 // 0xF + orr r2, r2, r3, LSL #20 // enable 10 and 11 + mcr p15, 0, r2, c1, c0, 2 // write coprocessor control register + isb +#endif /* __ARM_VFP__ */ + + // Running virtual. Prepare to call init code + cmp r1, #0 // Test if invoked from start + beq join_start_1 // Branch if yes + ldr r7, arm_init_tramp_addr // Load trampoline address + bx r7 // Branch to virtual trampoline address + + // Loading the virtual address for arm_init_tramp is a rather ugly + // problem. There is probably a better solution, but for the moment, + // patch the address in locally so that loading it is trivial +arm_init_tramp_addr: + .long 0 + .globl EXT(arm_init_tramp) +LEXT(arm_init_tramp) + mrc p15, 0, r5, c2, c0, 0 // Read to translation table base 0 + add r5, r5, PGBYTES * 4 // get kernel page table base (past 4 boot tte pages) + mcr p15, 0, r5, c2, c0, 0 // write kernel to translation table base 0 + mcr p15, 0, r5, c2, c0, 1 // also to translation table base 1 + isb + mov r5, #0 + mcr p15, 0, r5, c8, c7, 0 // Flush all TLB entries + dsb // ensure mmu settings are inplace + isb // before moving on + +join_start_1: +#if __ARM_VFP__ + // Enable VFP for the bootstrap thread context. + // VFP is enabled for the arm_init path as we may + // execute VFP code before we can handle an undef. + fmrx r2, fpexc // get fpexc + orr r2, #FPEXC_EN // set the enable bit + fmxr fpexc, r2 // set fpexc + mov r2, #FPSCR_DEFAULT // set default fpscr + fmxr fpscr, r2 // set fpscr +#endif /* __ARM_VFP__ */ + + mov r7, #0 // Set stack frame 0 + bx lr + +LOAD_ADDR_GEN_DEF(arm_init) +LOAD_ADDR_GEN_DEF(arm_init_cpu) +LOAD_ADDR_GEN_DEF(arm_init_idle_cpu) +LOAD_ADDR_GEN_DEF(arm_init_tramp) +LOAD_ADDR_GEN_DEF(fleh_reset) +LOAD_ADDR_GEN_DEF(ExceptionVectorsTable) +LOAD_ADDR_GEN_DEF(fleh_undef) +LOAD_ADDR_GEN_DEF(fleh_swi) +LOAD_ADDR_GEN_DEF(fleh_prefabt) +LOAD_ADDR_GEN_DEF(fleh_dataabt) +LOAD_ADDR_GEN_DEF(fleh_addrexc) +LOAD_ADDR_GEN_DEF(fleh_irq) +LOAD_ADDR_GEN_DEF(fleh_decirq) + +#include "globals_asm.h" + +/* vim: set ts=4: */ diff --git a/osfmk/arm/status.c b/osfmk/arm/status.c new file mode 100644 index 000000000..94128c4bf --- /dev/null +++ b/osfmk/arm/status.c @@ -0,0 +1,873 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct arm_vfpv2_state +{ + __uint32_t __r[32]; + __uint32_t __fpscr; + +}; + +typedef struct arm_vfpv2_state arm_vfpv2_state_t; + +#define ARM_VFPV2_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_vfpv2_state_t)/sizeof(uint32_t))) + + +/* + * Forward definitions + */ +void + thread_set_child(thread_t child, int pid); + +void + thread_set_parent(thread_t parent, int pid); + +/* + * Maps state flavor to number of words in the state: + */ +/* __private_extern__ */ +unsigned int _MachineStateCount[] = { + /* FLAVOR_LIST */ 0, + ARM_THREAD_STATE_COUNT, + ARM_VFP_STATE_COUNT, + ARM_EXCEPTION_STATE_COUNT, + ARM_DEBUG_STATE_COUNT +}; + +extern zone_t ads_zone; + +/* + * Routine: machine_thread_get_state + * + */ +kern_return_t +machine_thread_get_state( + thread_t thread, + thread_flavor_t flavor, + thread_state_t tstate, + mach_msg_type_number_t * count) +{ + +#define machine_thread_get_state_kprintf(x...) /* kprintf("machine_thread_get + * _state: " x) */ + + switch (flavor) { + case THREAD_STATE_FLAVOR_LIST: + if (*count < 4) + return (KERN_INVALID_ARGUMENT); + + tstate[0] = ARM_THREAD_STATE; + tstate[1] = ARM_VFP_STATE; + tstate[2] = ARM_EXCEPTION_STATE; + tstate[3] = ARM_DEBUG_STATE; + *count = 4; + break; + + case ARM_THREAD_STATE:{ + struct arm_thread_state *state; + struct arm_saved_state *saved_state; + arm_unified_thread_state_t *unified_state; + + unsigned int i; + if (*count < ARM_THREAD_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (*count == ARM_UNIFIED_THREAD_STATE_COUNT) { + unified_state = (arm_unified_thread_state_t *) tstate; + state = &unified_state->ts_32; + unified_state->ash.flavor = ARM_THREAD_STATE32; + unified_state->ash.count = ARM_THREAD_STATE32_COUNT; + } else { + state = (struct arm_thread_state *) tstate; + } + saved_state = &thread->machine.PcbData; + + state->sp = saved_state->sp; + state->lr = saved_state->lr; + state->pc = saved_state->pc; + state->cpsr = saved_state->cpsr; + for (i = 0; i < 13; i++) + state->r[i] = saved_state->r[i]; + machine_thread_get_state_kprintf("machine_thread_get_state: pc 0x%x r0 0x%x sp 0x%x\n", + state->pc, state->r[0], state->sp); + + if (*count != ARM_UNIFIED_THREAD_STATE_COUNT) { + *count = ARM_THREAD_STATE_COUNT; + } + break; + } + case ARM_EXCEPTION_STATE:{ + struct arm_exception_state *state; + struct arm_saved_state *saved_state; + + if (*count < ARM_EXCEPTION_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + state = (struct arm_exception_state *) tstate; + saved_state = &thread->machine.PcbData; + + state->exception = saved_state->exception; + state->fsr = saved_state->fsr; + state->far = saved_state->far; + + *count = ARM_EXCEPTION_STATE_COUNT; + break; + } + case ARM_VFP_STATE:{ +#if __ARM_VFP__ + struct arm_vfp_state *state; + struct arm_vfpsaved_state *saved_state; + unsigned int i; + unsigned int max; + + if (*count < ARM_VFP_STATE_COUNT) { + if (*count < ARM_VFPV2_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + else + *count = ARM_VFPV2_STATE_COUNT; + } + + if (*count == ARM_VFPV2_STATE_COUNT) + max = 32; + else + max = 64; + + state = (struct arm_vfp_state *) tstate; + saved_state = find_user_vfp(thread); + + state->fpscr = saved_state->fpscr; + for (i = 0; i < max; i++) + state->r[i] = saved_state->r[i]; + +#endif + break; + } + case ARM_DEBUG_STATE:{ + arm_debug_state_t *state; + arm_debug_state_t *thread_state; + + if (*count < ARM_DEBUG_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + state = (arm_debug_state_t *) tstate; + thread_state = find_debug_state(thread); + + if (thread_state == NULL) + bzero(state, sizeof(arm_debug_state_t)); + else + bcopy(thread_state, state, sizeof(arm_debug_state_t)); + + *count = ARM_DEBUG_STATE_COUNT; + break; + } + + default: + return (KERN_INVALID_ARGUMENT); + } + return (KERN_SUCCESS); +} + + +/* + * Routine: machine_thread_get_kern_state + * + */ +kern_return_t +machine_thread_get_kern_state( + thread_t thread, + thread_flavor_t flavor, + thread_state_t tstate, + mach_msg_type_number_t * count) +{ + +#define machine_thread_get_kern_state_kprintf(x...) /* kprintf("machine_threa + * d_get_kern_state: " + * x) */ + + /* + * This works only for an interrupted kernel thread + */ + if (thread != current_thread() || getCpuDatap()->cpu_int_state == NULL) + return KERN_FAILURE; + + switch (flavor) { + case ARM_THREAD_STATE:{ + struct arm_thread_state *state; + struct arm_saved_state *saved_state; + unsigned int i; + if (*count < ARM_THREAD_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + state = (struct arm_thread_state *) tstate; + saved_state = getCpuDatap()->cpu_int_state; + + state->sp = saved_state->sp; + state->lr = saved_state->lr; + state->pc = saved_state->pc; + state->cpsr = saved_state->cpsr; + for (i = 0; i < 13; i++) + state->r[i] = saved_state->r[i]; + machine_thread_get_kern_state_kprintf("machine_thread_get_state: pc 0x%x r0 0x%x sp 0x%x\n", + state->pc, state->r[0], state->sp); + *count = ARM_THREAD_STATE_COUNT; + break; + } + default: + return (KERN_INVALID_ARGUMENT); + } + return (KERN_SUCCESS); +} + +extern long long arm_debug_get(void); + +/* + * Routine: machine_thread_set_state + * + */ +kern_return_t +machine_thread_set_state( + thread_t thread, + thread_flavor_t flavor, + thread_state_t tstate, + mach_msg_type_number_t count) +{ + +#define machine_thread_set_state_kprintf(x...) /* kprintf("machine_thread_set + * _state: " x) */ + + switch (flavor) { + case ARM_THREAD_STATE:{ + struct arm_thread_state *state; + struct arm_saved_state *saved_state; + arm_unified_thread_state_t *unified_state; + int old_psr; + + if (count < ARM_THREAD_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (count == ARM_UNIFIED_THREAD_STATE_COUNT) { + unified_state = (arm_unified_thread_state_t *) tstate; + state = &unified_state->ts_32; + } else { + state = (struct arm_thread_state *) tstate; + } + saved_state = &thread->machine.PcbData; + old_psr = saved_state->cpsr; + memcpy((char *) saved_state, (char *) state, sizeof(*state)); + /* + * do not allow privileged bits of the PSR to be + * changed + */ + saved_state->cpsr = (saved_state->cpsr & ~PSR_USER_MASK) | (old_psr & PSR_USER_MASK); + + machine_thread_set_state_kprintf("machine_thread_set_state: pc 0x%x r0 0x%x sp 0x%x\n", + state->pc, state->r[0], state->sp); + break; + } + case ARM_VFP_STATE:{ +#if __ARM_VFP__ + struct arm_vfp_state *state; + struct arm_vfpsaved_state *saved_state; + unsigned int i; + unsigned int max; + + if (count < ARM_VFP_STATE_COUNT) { + if (count < ARM_VFPV2_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + else + count = ARM_VFPV2_STATE_COUNT; + } + + if (count == ARM_VFPV2_STATE_COUNT) + max = 32; + else + max = 64; + + state = (struct arm_vfp_state *) tstate; + saved_state = find_user_vfp(thread); + + saved_state->fpscr = state->fpscr; + for (i = 0; i < max; i++) + saved_state->r[i] = state->r[i]; + +#endif + break; + } + case ARM_EXCEPTION_STATE:{ + + if (count < ARM_EXCEPTION_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + break; + } + case ARM_DEBUG_STATE:{ + arm_debug_state_t *state; + arm_debug_state_t *thread_state; + boolean_t enabled = FALSE; + unsigned int i; + + if (count < ARM_DEBUG_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + state = (arm_debug_state_t *) tstate; + thread_state = find_debug_state(thread); + + if (count < ARM_DEBUG_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + for (i = 0; i < 16; i++) { + /* do not allow context IDs to be set */ + if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA) + || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED) + || ((state->wcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA) + || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) { + return KERN_PROTECTION_FAILURE; + } + if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) + || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) { + enabled = TRUE; + } + } + + if (!enabled) { + if (thread_state != NULL) + { + void *pTmp = thread->machine.DebugData; + thread->machine.DebugData = NULL; + zfree(ads_zone, pTmp); + } + } + else + { + if (thread_state == NULL) + thread_state = zalloc(ads_zone); + + for (i = 0; i < 16; i++) { + /* set appropriate priviledge; mask out unknown bits */ + thread_state->bcr[i] = (state->bcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK + | ARM_DBGBCR_MATCH_MASK + | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK + | ARM_DBG_CR_ENABLE_MASK)) + | ARM_DBGBCR_TYPE_IVA + | ARM_DBG_CR_LINKED_UNLINKED + | ARM_DBG_CR_SECURITY_STATE_BOTH + | ARM_DBG_CR_MODE_CONTROL_USER; + thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK; + thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK + | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK + | ARM_DBGWCR_ACCESS_CONTROL_MASK + | ARM_DBG_CR_ENABLE_MASK)) + | ARM_DBG_CR_LINKED_UNLINKED + | ARM_DBG_CR_SECURITY_STATE_BOTH + | ARM_DBG_CR_MODE_CONTROL_USER; + thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK; + } + + if (thread->machine.DebugData == NULL) + thread->machine.DebugData = thread_state; + } + + if (thread == current_thread()) { + arm_debug_set(thread_state); + } + + break; + } + + default: + return (KERN_INVALID_ARGUMENT); + } + return (KERN_SUCCESS); +} + +/* + * Routine: machine_thread_state_initialize + * + */ +kern_return_t +machine_thread_state_initialize( + thread_t thread) +{ + struct arm_saved_state *savestate; + + savestate = (struct arm_saved_state *) & thread->machine.PcbData; + bzero((char *) savestate, sizeof(struct arm_saved_state)); + savestate->cpsr = PSR_USERDFLT; + +#if __ARM_VFP__ + vfp_state_initialize(&thread->machine.uVFPdata); + vfp_state_initialize(&thread->machine.kVFPdata); +#endif + + thread->machine.DebugData = NULL; + + return KERN_SUCCESS; +} + +#if __ARM_VFP__ +void +vfp_state_initialize(struct arm_vfpsaved_state *vfp_state) +{ + /* Set default VFP state to RunFast mode: + * + * - flush-to-zero mode + * - default NaN mode + * - no enabled exceptions + * + * On the VFP11, this allows the use of floating point without + * trapping to support code, which we do not provide. With + * the Cortex-A8, this allows the use of the (much faster) NFP + * pipeline for single-precision operations. + */ + + bzero(vfp_state, sizeof(*vfp_state)); + vfp_state->fpscr = FPSCR_DEFAULT; +} +#endif /* __ARM_VFP__ */ + + +/* + * Routine: machine_thread_dup + * + */ +kern_return_t +machine_thread_dup( + thread_t self, + thread_t target) +{ + struct arm_saved_state *self_saved_state; + struct arm_saved_state *target_saved_state; + +#if __ARM_VFP__ + struct arm_vfpsaved_state *self_vfp_state; + struct arm_vfpsaved_state *target_vfp_state; +#endif + + target->machine.cthread_self = self->machine.cthread_self; + target->machine.cthread_data = self->machine.cthread_data; + + self_saved_state = &self->machine.PcbData; + target_saved_state = &target->machine.PcbData; + bcopy(self_saved_state, target_saved_state, sizeof(struct arm_saved_state)); + +#if __ARM_VFP__ + self_vfp_state = &self->machine.uVFPdata; + target_vfp_state = &target->machine.uVFPdata; + bcopy(self_vfp_state, target_vfp_state, sizeof(struct arm_vfpsaved_state)); +#endif + + return (KERN_SUCCESS); +} + +/* + * Routine: get_user_regs + * + */ +struct arm_saved_state * +get_user_regs( + thread_t thread) +{ + return (&thread->machine.PcbData); +} + +/* + * Routine: find_user_regs + * + */ +struct arm_saved_state * +find_user_regs( + thread_t thread) +{ + return get_user_regs(thread); +} + +/* + * Routine: find_kern_regs + * + */ +struct arm_saved_state * +find_kern_regs( + thread_t thread) +{ + /* + * This works only for an interrupted kernel thread + */ + if (thread != current_thread() || getCpuDatap()->cpu_int_state == NULL) + return ((struct arm_saved_state *) NULL); + else + return (getCpuDatap()->cpu_int_state); + +} + +#if __ARM_VFP__ +/* + * Find the user state floating point context. If there is no user state context, + * we just return a 0. + */ + +struct arm_vfpsaved_state * +find_user_vfp( + thread_t thread) +{ + return &thread->machine.uVFPdata; +} +#endif /* __ARM_VFP__ */ + +arm_debug_state_t * +find_debug_state( + thread_t thread) +{ + return thread->machine.DebugData; +} + +/* + * Routine: thread_userstack + * + */ +kern_return_t +thread_userstack( + __unused thread_t thread, + int flavor, + thread_state_t tstate, + unsigned int count, + mach_vm_offset_t * user_stack, + int *customstack, + __unused boolean_t is64bit +) +{ + + switch (flavor) { + case ARM_THREAD_STATE: + { + struct arm_thread_state *state; + + + if (count < ARM_THREAD_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (customstack) + *customstack = 0; + state = (struct arm_thread_state *) tstate; + + if (state->sp) { + *user_stack = CAST_USER_ADDR_T(state->sp); + if (customstack) + *customstack = 1; + } else { + *user_stack = CAST_USER_ADDR_T(USRSTACK); + } + } + break; + + default: + return (KERN_INVALID_ARGUMENT); + } + + return (KERN_SUCCESS); +} + +/* + * thread_userstackdefault: + * + * Return the default stack location for the + * thread, if otherwise unknown. + */ +kern_return_t +thread_userstackdefault( + mach_vm_offset_t *default_user_stack, + boolean_t is64bit __unused) +{ + *default_user_stack = USRSTACK; + + return (KERN_SUCCESS); +} + +/* + * Routine: thread_setuserstack + * + */ +void +thread_setuserstack(thread_t thread, mach_vm_address_t user_stack) +{ + struct arm_saved_state *sv; + +#define thread_setuserstack_kprintf(x...) /* kprintf("thread_setuserstac + * k: " x) */ + + sv = get_user_regs(thread); + + sv->sp = user_stack; + + thread_setuserstack_kprintf("stack %x\n", sv->sp); + + return; +} + +/* + * Routine: thread_adjuserstack + * + */ +uint64_t +thread_adjuserstack(thread_t thread, int adjust) +{ + struct arm_saved_state *sv; + + sv = get_user_regs(thread); + + sv->sp += adjust; + + return sv->sp; +} + +/* + * Routine: thread_setentrypoint + * + */ +void +thread_setentrypoint(thread_t thread, mach_vm_offset_t entry) +{ + struct arm_saved_state *sv; + +#define thread_setentrypoint_kprintf(x...) /* kprintf("thread_setentrypoi + * nt: " x) */ + + sv = get_user_regs(thread); + + sv->pc = entry; + + thread_setentrypoint_kprintf("entry %x\n", sv->pc); + + return; +} + +/* + * Routine: thread_entrypoint + * + */ +kern_return_t +thread_entrypoint( + __unused thread_t thread, + int flavor, + thread_state_t tstate, + __unused unsigned int count, + mach_vm_offset_t * entry_point +) +{ + switch (flavor) { + case ARM_THREAD_STATE: + { + struct arm_thread_state *state; + + state = (struct arm_thread_state *) tstate; + + /* + * If a valid entry point is specified, use it. + */ + if (state->pc) { + *entry_point = CAST_USER_ADDR_T(state->pc); + } else { + *entry_point = CAST_USER_ADDR_T(VM_MIN_ADDRESS); + } + } + break; + + default: + return (KERN_INVALID_ARGUMENT); + } + + return (KERN_SUCCESS); +} + + +/* + * Routine: thread_set_child + * + */ +void +thread_set_child( + thread_t child, + int pid) +{ + struct arm_saved_state *child_state; + + child_state = get_user_regs(child); + + child_state->r[0] = (uint_t) pid; + child_state->r[1] = 1ULL; +} + + +/* + * Routine: thread_set_parent + * + */ +void +thread_set_parent( + thread_t parent, + int pid) +{ + struct arm_saved_state *parent_state; + + parent_state = get_user_regs(parent); + + parent_state->r[0] = pid; + parent_state->r[1] = 0; +} + + +struct arm_act_context { + struct arm_saved_state ss; +#if __ARM_VFP__ + struct arm_vfpsaved_state vfps; +#endif +}; + +/* + * Routine: act_thread_csave + * + */ +void * +act_thread_csave(void) +{ + struct arm_act_context *ic; + kern_return_t kret; + unsigned int val; + + ic = (struct arm_act_context *) kalloc(sizeof(struct arm_act_context)); + + if (ic == (struct arm_act_context *) NULL) + return ((void *) 0); + + val = ARM_THREAD_STATE_COUNT; + kret = machine_thread_get_state(current_thread(), + ARM_THREAD_STATE, + (thread_state_t) & ic->ss, + &val); + if (kret != KERN_SUCCESS) { + kfree(ic, sizeof(struct arm_act_context)); + return ((void *) 0); + } +#if __ARM_VFP__ + val = ARM_VFP_STATE_COUNT; + kret = machine_thread_get_state(current_thread(), + ARM_VFP_STATE, + (thread_state_t) & ic->vfps, + &val); + if (kret != KERN_SUCCESS) { + kfree(ic, sizeof(struct arm_act_context)); + return ((void *) 0); + } +#endif + return (ic); +} + +/* + * Routine: act_thread_catt + * + */ +void +act_thread_catt(void *ctx) +{ + struct arm_act_context *ic; + kern_return_t kret; + + ic = (struct arm_act_context *) ctx; + + if (ic == (struct arm_act_context *) NULL) + return; + + kret = machine_thread_set_state(current_thread(), + ARM_THREAD_STATE, + (thread_state_t) & ic->ss, + ARM_THREAD_STATE_COUNT); + if (kret != KERN_SUCCESS) + goto out; + +#if __ARM_VFP__ + kret = machine_thread_set_state(current_thread(), + ARM_VFP_STATE, + (thread_state_t) & ic->vfps, + ARM_VFP_STATE_COUNT); + if (kret != KERN_SUCCESS) + goto out; +#endif +out: + kfree(ic, sizeof(struct arm_act_context)); +} + +/* + * Routine: act_thread_catt + * + */ +void +act_thread_cfree(void *ctx) +{ + kfree(ctx, sizeof(struct arm_act_context)); +} + +kern_return_t +thread_set_wq_state32(thread_t thread, thread_state_t tstate) +{ + arm_thread_state_t *state; + struct arm_saved_state *saved_state; + thread_t curth = current_thread(); + spl_t s=0; + + saved_state = &thread->machine.PcbData; + state = (arm_thread_state_t *)tstate; + + if (curth != thread) { + s = splsched(); + thread_lock(thread); + } + + /* + * do not zero saved_state, it can be concurrently accessed + * and zero is not a valid state for some of the registers, + * like sp. + */ + thread_state32_to_saved_state(state, saved_state); + saved_state->cpsr = PSR_USERDFLT; + + if (curth != thread) { + thread_unlock(thread); + splx(s); + } + + return KERN_SUCCESS; +} diff --git a/osfmk/arm/status_shared.c b/osfmk/arm/status_shared.c new file mode 100644 index 000000000..42d7a1235 --- /dev/null +++ b/osfmk/arm/status_shared.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Copy values from saved_state to ts32. + */ +void +saved_state_to_thread_state32(const arm_saved_state_t *saved_state, arm_thread_state32_t *ts32) +{ + uint32_t i; + + assert(is_saved_state32(saved_state)); + + ts32->lr = (uint32_t)get_saved_state_lr(saved_state); + ts32->sp = (uint32_t)get_saved_state_sp(saved_state); + ts32->pc = (uint32_t)get_saved_state_pc(saved_state); + ts32->cpsr = get_saved_state_cpsr(saved_state); + for (i = 0; i < 13; i++) + ts32->r[i] = (uint32_t)get_saved_state_reg(saved_state, i); +} + +/* + * Copy values from ts32 to saved_state. + */ +void +thread_state32_to_saved_state(const arm_thread_state32_t *ts32, arm_saved_state_t *saved_state) +{ + uint32_t i; + + assert(is_saved_state32(saved_state)); + + set_saved_state_lr(saved_state, ts32->lr); + set_saved_state_sp(saved_state, ts32->sp); + set_saved_state_pc(saved_state, ts32->pc); + +#if defined(__arm64__) + set_saved_state_cpsr(saved_state, (ts32->cpsr & ~PSR64_MODE_MASK) | PSR64_MODE_RW_32); +#elif defined(__arm__) + set_saved_state_cpsr(saved_state, (ts32->cpsr & ~PSR_USER_MASK) | (ts32->cpsr & PSR_USER_MASK)); +#else +#error Unknown architecture. +#endif + + for (i = 0; i < 13; i++) + set_saved_state_reg(saved_state, i, ts32->r[i]); +} + + diff --git a/osfmk/arm/strlcpy.c b/osfmk/arm/strlcpy.c new file mode 100644 index 000000000..e06f4173b --- /dev/null +++ b/osfmk/arm/strlcpy.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 Apple, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include "string.h" + +#undef strlcpy +size_t +strlcpy(char * dst, const char * src, size_t maxlen) { + const size_t srclen = strlen(src); + if (srclen + 1 < maxlen) { + memcpy(dst, src, srclen + 1); + } else if (maxlen != 0) { + memcpy(dst, src, maxlen - 1); + dst[maxlen-1] = '\0'; + } + return srclen; +} diff --git a/osfmk/arm/strlen.s b/osfmk/arm/strlen.s new file mode 100644 index 000000000..05d779709 --- /dev/null +++ b/osfmk/arm/strlen.s @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2011 Apple, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +.syntax unified +.code 32 +.globl _strlen + +#define addr r0 +#define word r1 +#define temp r2 +#define mask r3 +#define save ip +#define indx r0 + +.macro IfWordDoesntContainNUL_SetZ +#if defined _ARM_ARCH_6 +// In each word of the string, we check for NUL bytes via a saturating +// unsigned subtraction of each byte from 0x1. The result of this is +// non-zero if and only if the corresponding byte in the string is NUL. +// Simply using a TST instruction checks all four bytes for NULs in one +// go. + uqsub8 temp, mask, word + tst temp, temp +#else +// If we're on armv5, we do not have the uqsub8 instruction, so we need +// to use a different test for NUL. Instead, we compute: +// +// byte - 0x1 & ~byte +// +// and test the high-order bit. If it is set, then byte is NUL. Just +// as with the other test, this can be applied simultaneously to all +// bytes in a word. + sub temp, word, mask + bic temp, temp, word + tst temp, mask, lsl #7 +#endif +.endm + +.text +.align 4 +.long 0x0 // padding +.long 0x01010101 // mask for use in finding NULs +_strlen: +// Establish stack frame, load mask that we will use to find NUL bytes, +// and set aside a copy of the pointer to the string. + push {r7,lr} + mov r7, sp + ldr mask, (_strlen-4) + add save, addr, #4 + +// Load the aligned word that contains the start of the string, then OR +// 0x01 into any bytes that preceed the start of the string to prevent +// false positives when we check for NUL bytes. + and temp, addr, #3 + bic addr, addr, #3 + lsl temp, temp, #3 + ldr word, [addr], #4 + rsb temp, temp, #32 + orr word, word, mask, lsr temp + +// Check if the string ends in the first word. If so, don't load any +// more of the string; instead jump directly to the cleanup code. + IfWordDoesntContainNUL_SetZ + bne 1f + +.align 4 +// Load one word of the string on each iteration, and check it for NUL +// bytes. If a NUL is found, fall through into the cleanup code. +0: ldr word, [addr], #4 + IfWordDoesntContainNUL_SetZ + beq 0b + +// The last word that we loaded contained a NUL. Subtracting the saved +// pointer from the current pointer gives us the number of bytes from +// the start of the string to the word containing the NUL. +1: sub indx, addr, save +#if defined _ARM_ARCH_6 +// To that we add the index of the first NUL byte in the word, computed +// using REV and CLZ followed by a shift. + rev temp, temp + clz temp, temp + add indx, indx, temp, lsr #3 +#else +// armv5 does not have the REV instruction, so instead we find the +// index of the NUL byte in word with a linear search. + tst word, #0x000000ff + addne indx, #1 + tstne word, #0x0000ff00 + addne indx, #1 + tstne word, #0x00ff0000 + addne indx, #1 +#endif + pop {r7,pc} diff --git a/osfmk/arm/strncmp.s b/osfmk/arm/strncmp.s new file mode 100644 index 000000000..a58185565 --- /dev/null +++ b/osfmk/arm/strncmp.s @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2010, 2011 Apple, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +.text +.syntax unified +.code 32 +.globl _strncmp +// int strncmp(const char *s1, const char *s2, size_t n); +// +// Returns zero if the two NUL-terminated strings s1 and s2 are equal up to +// n characters. Otherwise, returns the difference between the first two +// characters that do not match, interpreted as unsigned integers. + +#define ESTABLISH_FRAME \ + push {r4-r7,lr} ;\ + add r7, sp, #12 ;\ + push {r8,r10} +#define CLEAR_FRAME \ + pop {r8,r10} ;\ + pop {r4-r7,lr} + +.align 3 +.long 0, 0x01010101 +_strncmp: +// If n < 16, jump straight to the byte-by-byte comparison loop. + cmp r2, #16 + blo L_byteCompareLoop +// Load a character from each string and advance the pointers. If the loaded +// characters are unequal or NUL, return their difference. +0: ldrb r3, [r0],#1 + ldrb ip, [r1],#1 + sub r2, #1 + cmp r3, #1 + cmphs r3, ip + bne L_earlyReturn +// If the address of the next character from s1 does not have word alignment, +// continue with the character-by-character comparison. Otherwise, fall +// through into the word-by-word comparison path. + tst r0, #3 + bne 0b + +// We have not encountered a NUL or a mismatch, and s1 has word alignment. +// Establish a frame, since we're going to need additional registers anyway. + ESTABLISH_FRAME + ldr lr, (_strncmp-4) + +// Word align s2, and place the remainder in r10. Compute the right- and +// left-shifts to extract each word that we will compare to the other source +// from the aligned words that we load: +// +// aligned s2 to be loaded on next iteration +// | "true" s2 | +// v v v +// +---+---+---+---+ +---+---+---+---+ +// | 0 | 1 | 2 | 3 | | 4 | 5 | 6 | 7 | +// +---+---+---+---+ +---+---+---+---+ +// ^-----------------^ +// to be compared on next iteration + and r10, r1, #3 + bic r1, r1, #3 + mov r10, r10, lsl #3 + rsb r6, r10,#32 + +// Subtract the number of bytes of the initial word load from s2 that will +// actually be used from n. + sub r2, r2, r6, lsr #3 + +// Load the first aligned word of s2. OR 0x01 into any bytes that preceed the +// "true s2", to prevent our check for NUL from generating a false positive. +// Then check for NUL, and jump to the byte-by-byte comparison loop after +// unwinding the pointers if we enounter one. + ldr r8, [r1],#4 + orr r8, r8, lr, lsr r6 + sub r3, r8, lr + bic r3, r3, r8 + tst r3, lr, lsl #7 + mov r5, r8, lsr r10 + bne L_unwindLoopPreload + +.align 3 +L_wordCompareLoop: +// If n < 4, abort the word compare loop before we load any more data. + subs r2, r2, #4 + blo L_nIsLessThanFour +// Load the next aligned word of s2 and check if it contains any NUL bytes. +// Load the next aligned word of s1, and extract the corresponding bytes from +// the two words of s2 loaded in this and the previous iteration of the loop. +// Compare these two words. +// If no NUL or mismatched words have been encountered, continue the loop. + ldr r8, [r1],#4 +#if defined _ARM_ARCH_6 + uqsub8 r3, lr, r8 + tst r3, r3 + ldr ip, [r0],#4 +#else + sub r3, r8, lr + bic r3, r3, r8 + ldr ip, [r0],#4 + tst r3, lr, lsl #7 +#endif + orr r4, r5, r8, lsl r6 + cmpeq ip, r4 + mov r5, r8, lsr r10 + beq L_wordCompareLoop + +// Either we have encountered a NUL, or we have found a mismatch between s1 +// and s2. Unwind the pointers and use a byte-by-byte comparison loop. + sub r0, r0, #4 + sub r1, r1, #4 +L_nIsLessThanFour: + add r2, r2, #4 +L_unwindLoopPreload: + sub r1, r1, r6, lsr #3 + add r2, r2, r6, lsr #3 + CLEAR_FRAME + +L_byteCompareLoop: +// If n-- == 0, we have exhausted the allowed number of comparisons, and need +// to return zero without additional loads. + subs r2, r2, #1 + movlo r0, #0 + bxlo lr +// Load a character from each string and advance the pointers. If the loaded +// characters are unequal or NUL, return their difference. + ldrb r3, [r0],#1 + ldrb ip, [r1],#1 + cmp r3, #1 + cmpcs r3, ip + beq L_byteCompareLoop + +L_earlyReturn: +// Return the difference between the last two characters loaded. + sub r0, r3, ip + bx lr diff --git a/osfmk/arm/strncpy.c b/osfmk/arm/strncpy.c new file mode 100644 index 000000000..5ee1847a6 --- /dev/null +++ b/osfmk/arm/strncpy.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 Apple, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include "string.h" + +#undef strncpy +char * +strncpy(char * dst, const char * src, size_t maxlen) { + const size_t srclen = strnlen(src, maxlen); + if (srclen < maxlen) { + memcpy(dst, src, srclen); + memset(dst+srclen, 0, maxlen - srclen); + } else { + memcpy(dst, src, maxlen); + } + return dst; +} diff --git a/osfmk/arm/strnlen.s b/osfmk/arm/strnlen.s new file mode 100644 index 000000000..49788471f --- /dev/null +++ b/osfmk/arm/strnlen.s @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2011 Apple, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +.syntax unified +.code 32 +.globl _strnlen + +#define addr r0 +#define maxl r1 +#define temp r2 +#define mask r3 +#define save ip +#define word lr +#define byte lr +#define indx r0 + +.macro IfHS_and_WordDoesntContainNUL_SetZ +#if defined _ARM_ARCH_6 +// In each word of the string, we check for NUL bytes via a saturating +// unsigned subtraction of each byte from 0x1. The result of this is +// non-zero if and only if the corresponding byte in the string is NUL. +// Simply using a TST instruction checks all four bytes for NULs in one +// go. + uqsub8 temp, mask, word + tsths temp, temp +#else +// If we're on armv5, we do not have the uqsub8 instruction, so we need +// to use a different test for NUL. Instead, we compute: +// +// byte - 0x1 & ~byte +// +// and test the high-order bit. If it is set, then byte is NUL. Just +// as with the other test, this can be applied simultaneously to all +// bytes in a word. + sub temp, word, mask + bic temp, temp, word + tsths temp, mask, lsl #7 +#endif +.endm + +.text +.align 3 +.long 0x0 // padding +.long 0x01010101 // mask for use in finding NULs +_strnlen: +// Establish stack frame, load mask that we will use to find NUL bytes, +// and set aside a copy of the pointer to the string. Subtract 4 from +// the maxlen, and jump into a byte-by-byte search if this requires a +// borrow, as we cannot use a word-by-word search in that case. + push {r7,lr} + mov r7, sp + ldr mask, (_strnlen-4) + add save, addr, #4 + subs maxl, maxl, #4 + blo L_bytewiseSearch + +// Load the aligned word that contains the start of the string, then OR +// 0x01 into any bytes that preceed the start to prevent false positives +// when we check for NUL bytes. Additionally, add the number of unused +// bytes to maxlen. + and temp, addr, #3 + bic addr, addr, #3 + add maxl, maxl, temp + lsl temp, temp, #3 + ldr word, [addr], #4 + rsb temp, temp, #32 + orr word, word, mask, lsr temp + + subs maxl, maxl, #4 + IfHS_and_WordDoesntContainNUL_SetZ + bne 1f + +.align 4 +0: ldr word, [addr], #4 + subs maxl, maxl, #4 + IfHS_and_WordDoesntContainNUL_SetZ + beq 0b + +.align 4 +// Either the last word that we loaded contained a NUL, or we will +// exceed maxlen before we finish the next word in the string. Determine +// which case we are in by repeating the check for NUL, and branch if +// there was not a NUL byte. Padding ensures that we don't have two +// branches in a single 16-byte fetch group, as this interferes with +// branch prediction on Swift. +1: tst temp, temp + beq L_bytewiseSearch + +// The last word that we loaded contained a NUL. Subtracting the saved +// pointer from the current pointer gives us the number of bytes from +// the start of the string to the word containing the NUL. + sub indx, addr, save +#if defined _ARM_ARCH_6 +// To that we add the index of the first NUL byte in the word, computed +// using REV and CLZ followed by a shift. + rev temp, temp + clz temp, temp + add indx, indx, temp, lsr #3 +#else +// armv5 does not have the REV instruction, so instead we find the +// index of the NUL byte in word with a linear search. + tst word, #0x000000ff + addne indx, #1 + tstne word, #0x0000ff00 + addne indx, #1 + tstne word, #0x00ff0000 + addne indx, #1 +#endif + pop {r7,pc} + +.align 4 +L_bytewiseSearch: +// Restore maxlen (the last thing that happened before we branched here +// was that we subtracted 4 from maxlen), and adjust the saved string +// pointer. Then we do a simple byte-by-byte search until we either +// reach the end of the string or maxlen reaches zero, at which point +// the length to return is simply the difference between the current +// and saved pointers. + adds maxl, maxl, #4 + sub save, save, #4 + beq 1f +0: ldrb byte, [addr] + cmp byte, #0 + addhi addr, #1 + subshi maxl, #1 + bhi 0b +1: sub indx, addr, save + pop {r7,pc} diff --git a/osfmk/arm/task.h b/osfmk/arm/task.h new file mode 100644 index 000000000..e545bd36c --- /dev/null +++ b/osfmk/arm/task.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +/* + * Machine dependant task fields + */ + +#define MACHINE_TASK \ + void* task_debug; + + diff --git a/osfmk/arm/thread.h b/osfmk/arm/thread.h new file mode 100644 index 000000000..e270512a2 --- /dev/null +++ b/osfmk/arm/thread.h @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +#ifndef _ARM_THREAD_H_ +#define _ARM_THREAD_H_ + +#include +#include +#include +#include + +#ifdef MACH_KERNEL_PRIVATE +#include +#include +#endif + +#if __ARM_VFP__ + +#define VFPSAVE_ALIGN 16 +#define VFPSAVE_ATTRIB __attribute__ ((aligned (VFPSAVE_ALIGN))) +#define THREAD_ALIGN VFPSAVE_ALIGN + +/* + * vector floating point saved state + */ +struct arm_vfpsaved_state { + uint32_t r[64]; + uint32_t fpscr; + uint32_t fpexc; +}; +#endif + +struct perfcontrol_state { + uint64_t opaque[8] __attribute__((aligned(8))); +}; + +/* + * Maps state flavor to number of words in the state: + */ +extern unsigned int _MachineStateCount[]; + +#ifdef MACH_KERNEL_PRIVATE +#if __arm64__ +typedef arm_context_t machine_thread_kernel_state; +#else +typedef struct arm_saved_state machine_thread_kernel_state; +#endif +#include + +struct machine_thread { +#if __arm64__ + arm_context_t *contextData; /* allocated user context */ + arm_saved_state_t *upcb; /* pointer to user GPR state */ + arm_neon_saved_state_t *uNeon; /* pointer to user VFP state */ +#elif __arm__ + struct arm_saved_state PcbData; +#if __ARM_VFP__ + struct arm_vfpsaved_state uVFPdata VFPSAVE_ATTRIB; + struct arm_vfpsaved_state kVFPdata VFPSAVE_ATTRIB; +#endif /* __ARM_VFP__ */ + +#else +#error Unknown arch +#endif +#if __ARM_USER_PROTECT__ + unsigned int uptw_ttc; + unsigned int uptw_ttb; + unsigned int kptw_ttb; + unsigned int asid; +#endif + + vm_offset_t kstackptr; /* top of kernel stack */ + struct cpu_data *CpuDatap; /* current per cpu data */ + unsigned int preemption_count; /* preemption count */ + + arm_debug_state_t *DebugData; + mach_vm_address_t cthread_self; /* for use of cthread package */ + mach_vm_address_t cthread_data; /* for use of cthread package */ + + struct perfcontrol_state perfctrl_state; +#if __arm64__ + uint64_t energy_estimate_nj; +#endif + +#if INTERRUPT_MASKED_DEBUG + uint64_t intmask_timestamp; /* timestamp of when interrupts were masked */ +#endif +}; +#endif + +extern struct arm_saved_state *get_user_regs(thread_t); +extern struct arm_saved_state *find_user_regs(thread_t); +extern struct arm_saved_state *find_kern_regs(thread_t); +extern struct arm_vfpsaved_state *find_user_vfp(thread_t); +#if defined(__arm__) +extern arm_debug_state_t *find_debug_state(thread_t); +#elif defined(__arm64__) +extern arm_debug_state32_t *find_debug_state32(thread_t); +extern arm_debug_state64_t *find_debug_state64(thread_t); +extern arm_neon_saved_state_t *get_user_neon_regs(thread_t); +#else +#error unknown arch +#endif + +#define FIND_PERFCONTROL_STATE(th) (&th->machine.perfctrl_state) + +#ifdef MACH_KERNEL_PRIVATE +#if __ARM_VFP__ +extern void vfp_state_initialize(struct arm_vfpsaved_state *vfp_state); +extern void vfp_save(struct arm_vfpsaved_state *vfp_ss); +extern void vfp_load(struct arm_vfpsaved_state *vfp_ss); +extern void toss_live_vfp(void *vfp_fc); +#endif /* __ARM_VFP__ */ +extern void arm_debug_set(arm_debug_state_t *debug_state); +#if defined(__arm64__) +extern void arm_debug_set32(arm_debug_state_t *debug_state); +extern void arm_debug_set64(arm_debug_state_t *debug_state); + +kern_return_t handle_get_arm_thread_state( + thread_state_t tstate, + mach_msg_type_number_t * count, + const arm_saved_state_t *saved_state); +kern_return_t handle_get_arm32_thread_state( + thread_state_t tstate, + mach_msg_type_number_t * count, + const arm_saved_state_t *saved_state); +kern_return_t handle_get_arm64_thread_state( + thread_state_t tstate, + mach_msg_type_number_t * count, + const arm_saved_state_t *saved_state); + +kern_return_t handle_set_arm_thread_state( + const thread_state_t tstate, + mach_msg_type_number_t count, + arm_saved_state_t *saved_state); +kern_return_t handle_set_arm32_thread_state( + const thread_state_t tstate, + mach_msg_type_number_t count, + arm_saved_state_t *saved_state); +kern_return_t handle_set_arm64_thread_state( + const thread_state_t tstate, + mach_msg_type_number_t count, + arm_saved_state_t *saved_state); +#endif +#endif /* MACH_KERNEL_PRIVATE */ + +extern void *act_thread_csave(void); +extern void act_thread_catt(void *ctx); +extern void act_thread_cfree(void *ctx); + +/* + * Return address of the function that called current function, given + * address of the first parameter of current function. + */ +#define GET_RETURN_PC(addr) (((vm_offset_t *)0)) + +/* + * Defining this indicates that MD code will supply an exception() + * routine, conformant with kern/exception.c (dependency alert!) + * but which does wonderfully fast, machine-dependent magic. + */ +#define MACHINE_FAST_EXCEPTION 1 + +#endif /* _ARM_THREAD_H_ */ diff --git a/osfmk/arm/trap.c b/osfmk/arm/trap.c new file mode 100644 index 000000000..617c652ab --- /dev/null +++ b/osfmk/arm/trap.c @@ -0,0 +1,897 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * External function prototypes. + */ +#include +#include +#include + + +#if CONFIG_DTRACE +extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs, unsigned int instr); +extern boolean_t dtrace_tally_fault(user_addr_t); + +/* Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy and paste the trap instructions + over from that file. Need to keep these in sync! */ +#define FASTTRAP_ARM_INSTR 0xe7ffdefc +#define FASTTRAP_THUMB_INSTR 0xdefc + +#define FASTTRAP_ARM_RET_INSTR 0xe7ffdefb +#define FASTTRAP_THUMB_RET_INSTR 0xdefb + +/* See */ +perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */ +#endif + +#define COPYIN(dst, src, size) \ + ((regs->cpsr & PSR_MODE_MASK) != PSR_USER_MODE) ? \ + copyin_kern(dst, src, size) \ + : \ + copyin(dst, src, size) + +#define COPYOUT(src, dst, size) \ + ((regs->cpsr & PSR_MODE_MASK) != PSR_USER_MODE) ? \ + copyout_kern(src, dst, size) \ + : \ + copyout(src, dst, size) + +/* Second-level exception handlers forward declarations */ +void sleh_undef(struct arm_saved_state *, struct arm_vfpsaved_state *); +void sleh_abort(struct arm_saved_state *, int); +static kern_return_t sleh_alignment(struct arm_saved_state *); +static void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *regs); + + +volatile perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */ + +int sleh_alignment_count = 0; +int trap_on_alignment_fault = 0; + +/* + * Routine: sleh_undef + * Function: Second level exception handler for undefined exception + */ + +void +sleh_undef(struct arm_saved_state * regs, struct arm_vfpsaved_state * vfp_ss __unused) +{ + exception_type_t exception = EXC_BAD_INSTRUCTION; + mach_exception_data_type_t code[2] = {EXC_ARM_UNDEFINED}; + mach_msg_type_number_t codeCnt = 2; + thread_t thread = current_thread(); + vm_offset_t recover; + + recover = thread->recover; + thread->recover = 0; + + getCpuDatap()->cpu_stat.undef_ex_cnt++; + + /* Inherit the interrupt masks from previous */ + if (!(regs->cpsr & PSR_INTMASK)) + ml_set_interrupts_enabled(TRUE); + +#if CONFIG_DTRACE + if (tempDTraceTrapHook) { + if (tempDTraceTrapHook(exception, regs, 0, 0) == KERN_SUCCESS) { + /* + * If it succeeds, we are done... + */ + goto exit; + } + } + + /* Check to see if we've hit a userland probe */ + if ((regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE) { + if (regs->cpsr & PSR_TF) { + uint16_t instr; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&instr,(vm_size_t)(sizeof(uint16_t))) != KERN_SUCCESS) + goto exit; + + if (instr == FASTTRAP_THUMB_INSTR || instr == FASTTRAP_THUMB_RET_INSTR) { + if (dtrace_user_probe(regs, instr) == KERN_SUCCESS) + /* If it succeeds, we are done... */ + goto exit; + } + } else { + uint32_t instr; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&instr,(vm_size_t)(sizeof(uint32_t))) != KERN_SUCCESS) + goto exit; + + if (instr == FASTTRAP_ARM_INSTR || instr == FASTTRAP_ARM_RET_INSTR) { + if (dtrace_user_probe(regs, instr) == KERN_SUCCESS) + /* If it succeeds, we are done... */ + goto exit; + } + } + } +#endif /* CONFIG_DTRACE */ + + + if (regs->cpsr & PSR_TF) { + unsigned short instr; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&instr,(vm_size_t)(sizeof(unsigned short))) != KERN_SUCCESS) + goto exit; + + if (IS_THUMB32(instr)) { + unsigned int instr32; + + instr32 = (instr<<16); + + if(COPYIN((user_addr_t)(((unsigned short *) (regs->pc))+1), (char *)&instr,(vm_size_t)(sizeof(unsigned short))) != KERN_SUCCESS) + goto exit; + + instr32 |= instr; + code[1] = instr32; + +#if __ARM_VFP__ + if (IS_THUMB_VFP(instr32)) { + /* We no longer manage FPEXC beyond bootstrap, so verify that VFP is still enabled. */ + if (!get_vfp_enabled()) + panic("VFP was disabled (thumb); VFP should always be enabled"); + } +#endif + } else { + /* I don't believe we have any 16 bit VFP instructions, so just set code[1]. */ + code[1] = instr; + + if (IS_THUMB_GDB_TRAP(instr)) { + exception = EXC_BREAKPOINT; + code[0] = EXC_ARM_BREAKPOINT; + } + } + } else { + uint32_t instr; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&instr,(vm_size_t)(sizeof(uint32_t))) != KERN_SUCCESS) + goto exit; + + code[1] = instr; +#if __ARM_VFP__ + if (IS_ARM_VFP(instr)) { + /* We no longer manage FPEXC beyond bootstrap, so verify that VFP is still enabled. */ + if (!get_vfp_enabled()) + panic("VFP was disabled (arm); VFP should always be enabled"); + } +#endif + + if (IS_ARM_GDB_TRAP(instr)) { + exception = EXC_BREAKPOINT; + code[0] = EXC_ARM_BREAKPOINT; + } + } + + if (!((regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE)) { + boolean_t intr; + + intr = ml_set_interrupts_enabled(FALSE); + + if (exception == EXC_BREAKPOINT) { + /* Save off the context here (so that the debug logic + * can see the original state of this thread). + */ + vm_offset_t kstackptr = current_thread()->machine.kstackptr; + *((arm_saved_state_t *) kstackptr) = *regs; + + DebuggerCall(exception, regs); + (void) ml_set_interrupts_enabled(intr); + goto exit; + } + panic_context(exception, (void *)regs, "undefined kernel instruction\n" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + } else { + exception_triage(exception, code, codeCnt); + /* NOTREACHED */ + } + +exit: + if (recover) + thread->recover = recover; +} + +/* + * Routine: sleh_abort + * Function: Second level exception handler for abort(Pref/Data) + */ + +void +sleh_abort(struct arm_saved_state * regs, int type) +{ + int status; + int debug_status=0; + int spsr; + int exc; + mach_exception_data_type_t codes[2]; + vm_map_t map; + vm_map_address_t vaddr; + vm_map_address_t fault_addr; + vm_prot_t fault_type; + kern_return_t result; + vm_offset_t recover; + thread_t thread = current_thread(); + boolean_t intr; + + recover = thread->recover; + thread->recover = 0; + + status = regs->fsr & FSR_MASK; + spsr = regs->cpsr; + + /* The DSFR/IFSR.ExT bit indicates "IMPLEMENTATION DEFINED" classification. + * Allow a platform-level error handler to decode it. + */ + if ((regs->fsr) & FSR_EXT) { + cpu_data_t *cdp = getCpuDatap(); + + if (cdp->platform_error_handler != (platform_error_handler_t) NULL) { + (*(platform_error_handler_t)cdp->platform_error_handler) (cdp->cpu_id, 0); + /* If a platform error handler is registered, expect it to panic, not fall through */ + panic("Unexpected return from platform_error_handler"); + } + } + + /* Done with asynchronous handling; re-enable here so that subsequent aborts are taken as early as possible. */ + reenable_async_aborts(); + + if (ml_at_interrupt_context()) + panic_with_thread_kernel_state("sleh_abort at interrupt context", regs); + + fault_addr = vaddr = regs->far; + + if (type == T_DATA_ABT) { + getCpuDatap()->cpu_stat.data_ex_cnt++; + } else { /* T_PREFETCH_ABT */ + getCpuDatap()->cpu_stat.instr_ex_cnt++; + fault_type = VM_PROT_READ | VM_PROT_EXECUTE; + } + + if (status == FSR_DEBUG) + debug_status = arm_debug_read_dscr() & ARM_DBGDSCR_MOE_MASK; + + /* Inherit the interrupt masks from previous */ + if (!(spsr & PSR_INTMASK)) + ml_set_interrupts_enabled(TRUE); + + if (type == T_DATA_ABT) { + /* + * Now that interrupts are reenabled, we can perform any needed + * copyin operations. + * + * Because we have reenabled interrupts, any instruction copy + * must be a copyin, even on UP systems. + */ + + if (regs->fsr & DFSR_WRITE) { + fault_type = (VM_PROT_READ | VM_PROT_WRITE); + /* Cache operations report faults as write access, change these to read access */ + /* Cache operations are invoked from arm mode for now */ + if (!(regs->cpsr & PSR_TF)) { + unsigned int ins; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&ins,(vm_size_t)(sizeof(unsigned int))) != KERN_SUCCESS) + goto exit; + + if (arm_mcr_cp15(ins) || arm_mcrr_cp15(ins)) + fault_type = VM_PROT_READ; + } + } else { + fault_type = VM_PROT_READ; + /* + * DFSR is not getting the "write" bit set + * when a swp instruction is encountered (even when it is + * a write fault. + */ + if (!(regs->cpsr & PSR_TF)) { + unsigned int ins; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&ins,(vm_size_t)(sizeof(unsigned int))) != KERN_SUCCESS) + goto exit; + + if ((ins & ARM_SWP_MASK) == ARM_SWP) + fault_type = VM_PROT_WRITE; + } + } + } + + if ((spsr & PSR_MODE_MASK) != PSR_USER_MODE) { + /* Fault in kernel mode */ + + if ((status == FSR_DEBUG) + && ((debug_status == ARM_DBGDSCR_MOE_ASYNC_WATCHPOINT) || (debug_status == ARM_DBGDSCR_MOE_SYNC_WATCHPOINT)) + && (recover != 0) && (getCpuDatap()->cpu_user_debug != 0)) { + /* If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to + * abort. Turn off watchpoints and keep going; we'll turn them back on in load_and_go_user. + */ + arm_debug_set(NULL); + goto exit; + } + + if ((type == T_PREFETCH_ABT) || (status == FSR_DEBUG)) { + + intr = ml_set_interrupts_enabled(FALSE); + if (status == FSR_DEBUG) { + DebuggerCall(EXC_BREAKPOINT, regs); + (void) ml_set_interrupts_enabled(intr); + goto exit; + } + panic_context(EXC_BAD_ACCESS, (void*)regs, "sleh_abort: prefetch abort in kernel mode: fault_addr=0x%x\n" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + fault_addr, + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + } else if (TEST_FSR_VMFAULT(status)) { + +#if CONFIG_DTRACE + if (thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */ + if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */ + /* Point to next instruction */ + regs->pc += ((regs->cpsr & PSR_TF) && !IS_THUMB32(*((uint16_t*) (regs->pc)))) ? 2 : 4; + goto exit; + } else { + intr = ml_set_interrupts_enabled(FALSE); + panic_context(EXC_BAD_ACCESS, (void *)regs, "Unexpected page fault under dtrace_probe" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + goto exit; + } + } +#endif + + if (VM_KERNEL_ADDRESS(vaddr) || thread == THREAD_NULL) + map = kernel_map; + else + map = thread->map; + + /* check to see if it is just a pmap ref/modify fault */ + result = arm_fast_fault(map->pmap, trunc_page(fault_addr), fault_type, FALSE); + if (result == KERN_SUCCESS) + goto exit; + + /* + * We have to "fault" the page in. + */ + result = vm_fault(map, fault_addr, + fault_type, + FALSE /* change_wiring */, VM_KERN_MEMORY_NONE, + (map == kernel_map) ? THREAD_UNINT : THREAD_ABORTSAFE, NULL, 0); + + if (result == KERN_SUCCESS) { + goto exit; + } else { + /* + * If we have a recover handler, invoke it now. + */ + if (recover != 0) { + regs->pc = (register_t) (recover & ~0x1); + regs->cpsr = (regs->cpsr & ~PSR_TF) | ((recover & 0x1) << PSR_TFb); + goto exit; + } + } + } else if ((status & FSR_ALIGN_MASK) == FSR_ALIGN) { + result = sleh_alignment(regs); + if (result == KERN_SUCCESS) { + goto exit; + } else { + intr = ml_set_interrupts_enabled(FALSE); + + panic_context(EXC_BAD_ACCESS, (void *)regs, "unaligned kernel data access: pc=0x%08x fault_addr=0x%x\n" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + regs->pc, fault_addr, + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + goto exit; + } + + } + intr = ml_set_interrupts_enabled(FALSE); + + panic_context(EXC_BAD_ACCESS, (void *)regs, "kernel abort type %d: fault_type=0x%x, fault_addr=0x%x\n" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + type, fault_type, fault_addr, + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + goto exit; + } + /* Fault in user mode */ + + if (TEST_FSR_VMFAULT(status)) { + map = thread->map; + +#if CONFIG_DTRACE + if (thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */ + if (dtrace_tally_fault(fault_addr)) { /* Should a user mode fault under dtrace be ignored? */ + if (recover) { + regs->pc = recover; + } else { + intr = ml_set_interrupts_enabled(FALSE); + + panic_context(EXC_BAD_ACCESS, (void *)regs, "copyin/out has no recovery point" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + } + goto exit; + } else { + intr = ml_set_interrupts_enabled(FALSE); + + panic_context(EXC_BAD_ACCESS, (void*)regs, "Unexpected UMW page fault under dtrace_probe" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + goto exit; + } + } +#endif + + /* check to see if it is just a pmap ref/modify fault */ + result = arm_fast_fault(map->pmap, trunc_page(fault_addr), fault_type, TRUE); + if (result != KERN_SUCCESS) { + /* + * We have to "fault" the page in. + */ + result = vm_fault(map, fault_addr, fault_type, + FALSE /* change_wiring */, VM_KERN_MEMORY_NONE, + THREAD_ABORTSAFE, NULL, 0); + } + if (result == KERN_SUCCESS || result == KERN_ABORTED) { + goto exception_return; + } + exc = EXC_BAD_ACCESS; + codes[0] = result; + } else if ((status & FSR_ALIGN_MASK) == FSR_ALIGN) { + if (sleh_alignment(regs) == KERN_SUCCESS) { + goto exception_return; + } + exc = EXC_BAD_ACCESS; + codes[0] = EXC_ARM_DA_ALIGN; + } else if (status == FSR_DEBUG) { + exc = EXC_BREAKPOINT; + codes[0] = EXC_ARM_DA_DEBUG; + } else if ((status == FSR_SDOM) || (status == FSR_PDOM)) { + exc = EXC_BAD_ACCESS; + codes[0] = KERN_INVALID_ADDRESS; + } else { + exc = EXC_BAD_ACCESS; + codes[0] = KERN_FAILURE; + } + + codes[1] = vaddr; + exception_triage(exc, codes, 2); + /* NOTREACHED */ + +exception_return: + if (recover) + thread->recover = recover; + thread_exception_return(); + /* NOTREACHED */ + +exit: + if (recover) + thread->recover = recover; + return; +} + + +/* + * Routine: sleh_alignment + * Function: Second level exception handler for alignment data fault + */ + +static kern_return_t +sleh_alignment(struct arm_saved_state * regs) +{ + unsigned int status; + unsigned int ins; + unsigned int rd_index; + unsigned int base_index; + unsigned int paddr; + void *src; + unsigned int reg_list; + unsigned int pre; + unsigned int up; + unsigned int write_back; + kern_return_t rc = KERN_SUCCESS; + + getCpuDatap()->cpu_stat.unaligned_cnt++; + + /* Do not try to emulate in modified execution states */ + if (regs->cpsr & (PSR_EF | PSR_JF)) + return KERN_NOT_SUPPORTED; + + /* Disallow emulation of kernel instructions */ + if ((regs->cpsr & PSR_MODE_MASK) != PSR_USER_MODE) + return KERN_NOT_SUPPORTED; + + +#define ALIGN_THRESHOLD 1024 + if ((sleh_alignment_count++ & (ALIGN_THRESHOLD - 1)) == + (ALIGN_THRESHOLD - 1)) + kprintf("sleh_alignment: %d more alignment faults: %d total\n", + ALIGN_THRESHOLD, sleh_alignment_count); + + if ((trap_on_alignment_fault != 0) + && (sleh_alignment_count % trap_on_alignment_fault == 0)) + return KERN_NOT_SUPPORTED; + + status = regs->fsr; + paddr = regs->far; + + if (regs->cpsr & PSR_TF) { + unsigned short ins16; + + /* Get aborted instruction */ +#if __ARM_SMP__ || __ARM_USER_PROTECT__ + if(COPYIN((user_addr_t)(regs->pc), (char *)&ins16,(vm_size_t)(sizeof(uint16_t))) != KERN_SUCCESS) { + /* Failed to fetch instruction, return success to re-drive the exception */ + return KERN_SUCCESS; + } +#else + ins16 = *(unsigned short *) (regs->pc); +#endif + + /* + * Map multi-word Thumb loads and stores to their ARM + * equivalents. + * Don't worry about single-word instructions, since those are + * handled in hardware. + */ + + reg_list = ins16 & 0xff; + if (reg_list == 0) + return KERN_NOT_SUPPORTED; + + if (((ins16 & THUMB_STR_1_MASK) == THUMB_LDMIA) || + ((ins16 & THUMB_STR_1_MASK) == THUMB_STMIA)) { + base_index = (ins16 >> 8) & 0x7; + ins = 0xE8800000 | (base_index << 16) | reg_list; + if ((ins16 & THUMB_STR_1_MASK) == THUMB_LDMIA) + ins |= (1 << 20); + if (((ins16 & THUMB_STR_1_MASK) == THUMB_STMIA) || + !(reg_list & (1 << base_index))) + ins |= (1 << 21); + } else if ((ins16 & THUMB_PUSH_MASK) == THUMB_POP) { + unsigned int r = (ins16 >> 8) & 1; + ins = 0xE8BD0000 | (r << 15) | reg_list; + } else if ((ins16 & THUMB_PUSH_MASK) == THUMB_PUSH) { + unsigned int r = (ins16 >> 8) & 1; + ins = 0xE92D0000 | (r << 14) | reg_list; + } else { + return KERN_NOT_SUPPORTED; + } + } else { + /* Get aborted instruction */ +#if __ARM_SMP__ || __ARM_USER_PROTECT__ + if(COPYIN((user_addr_t)(regs->pc), (char *)&ins,(vm_size_t)(sizeof(unsigned int))) != KERN_SUCCESS) { + /* Failed to fetch instruction, return success to re-drive the exception */ + return KERN_SUCCESS; + } +#else + ins = *(unsigned int *) (regs->pc); +#endif + } + + /* Don't try to emulate unconditional instructions */ + if ((ins & 0xF0000000) == 0xF0000000) + return KERN_NOT_SUPPORTED; + + pre = (ins >> 24) & 1; + up = (ins >> 23) & 1; + reg_list = ins & 0xffff; + write_back = (ins >> 21) & 1; + base_index = (ins >> 16) & 0xf; + + if ((ins & ARM_BLK_MASK) == ARM_STM) { /* STM or LDM */ + int reg_count = 0; + int waddr; + + for (rd_index = 0; rd_index < 16; rd_index++) { + if (reg_list & (1 << rd_index)) + reg_count++; + } + + paddr = regs->r[base_index]; + + switch (ins & (ARM_POST_INDEXING | ARM_INCREMENT)) { + /* Increment after */ + case ARM_INCREMENT: + waddr = paddr + reg_count * 4; + break; + + /* Increment before */ + case ARM_POST_INDEXING | ARM_INCREMENT: + waddr = paddr + reg_count * 4; + paddr += 4; + break; + + /* Decrement after */ + case 0: + waddr = paddr - reg_count * 4; + paddr = waddr + 4; + break; + + /* Decrement before */ + case ARM_POST_INDEXING: + waddr = paddr - reg_count * 4; + paddr = waddr; + break; + + default: + waddr = 0; + } + + for (rd_index = 0; rd_index < 16; rd_index++) { + if (reg_list & (1 << rd_index)) { + src = ®s->r[rd_index]; + + if ((ins & (1 << 20)) == 0) /* STM */ + rc = COPYOUT(src, paddr, 4); + else /* LDM */ + rc = COPYIN(paddr, src, 4); + + if (rc != KERN_SUCCESS) + break; + + paddr += 4; + } + } + + paddr = waddr; + } else { + rc = 1; + } + + if (rc == KERN_SUCCESS) { + if (regs->cpsr & PSR_TF) + regs->pc += 2; + else + regs->pc += 4; + + if (write_back) + regs->r[base_index] = paddr; + } + return (rc); +} + + +#ifndef NO_KDEBUG +/* XXX quell warnings */ +void syscall_trace(struct arm_saved_state * regs); +void syscall_trace_exit(unsigned int, unsigned int); +void mach_syscall_trace(struct arm_saved_state * regs, unsigned int call_number); +void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number); +void interrupt_trace(struct arm_saved_state * regs); +void interrupt_trace_exit(void); + +/* called from the fleh_swi handler, if TRACE_SYSCALL is enabled */ +void +syscall_trace( + struct arm_saved_state * regs) +{ + kprintf("syscall: %d\n", regs->r[12]); +} + +void +syscall_trace_exit( + unsigned int r0, + unsigned int r1) +{ + kprintf("syscall exit: 0x%x 0x%x\n", r0, r1); +} + +void +mach_syscall_trace( + struct arm_saved_state * regs, + unsigned int call_number) +{ + int i, argc; + int kdarg[3] = {0, 0, 0}; + + argc = mach_trap_table[call_number].mach_trap_arg_count; + + if (argc > 3) + argc = 3; + + for (i = 0; i < argc; i++) + kdarg[i] = (int) regs->r[i]; + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_START, + kdarg[0], kdarg[1], kdarg[2], 0, 0); + +} + +void +mach_syscall_trace_exit( + unsigned int retval, + unsigned int call_number) +{ + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_END, + retval, 0, 0, 0, 0); +} + +void +interrupt_trace( + struct arm_saved_state * regs) +{ +#define UMODE(rp) (((rp)->cpsr & PSR_MODE_MASK) == PSR_USER_MODE) + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START, + 0, UMODE(regs) ? regs->pc : VM_KERNEL_UNSLIDE(regs->pc), + UMODE(regs), 0, 0); +} + +void +interrupt_trace_exit( + void) +{ + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END, + 0, 0, 0, 0, 0); +} +#endif + +/* XXX quell warnings */ +void interrupt_stats(void); + +/* This is called from locore.s directly. We only update per-processor interrupt counters in this function */ +void +interrupt_stats(void) +{ + SCHED_STATS_INTERRUPT(current_processor()); +} + +static void +panic_with_thread_kernel_state(const char *msg, struct arm_saved_state *regs) +{ + panic_context(0, (void*)regs, "%s (saved state:%p)\n" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + msg, regs, + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + +} diff --git a/osfmk/arm/trap.h b/osfmk/arm/trap.h new file mode 100644 index 000000000..fe15c2733 --- /dev/null +++ b/osfmk/arm/trap.h @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +#ifndef _ARM_TRAP_H_ +#define _ARM_TRAP_H_ + +/* + * Hardware trap vectors for ARM. + */ + +#define T_RESET 0 +#define T_UNDEF 1 +#define T_SWI 2 +#define T_PREFETCH_ABT 3 +#define T_DATA_ABT 4 +#define T_IRQ 6 +#define T_FIQ 7 +#define T_PMU 8 + + +#define TRAP_NAMES "reset", "undefined instruction", "software interrupt", \ + "prefetch abort", "data abort", "irq interrupt", \ + "fast interrupt", "perfmon" + +/* + * Page-fault trap codes. + */ +#define T_PF_PROT 0x1 /* protection violation */ +#define T_PF_WRITE 0x2 /* write access */ +#define T_PF_USER 0x4 /* from user state */ + +#if !defined(ASSEMBLER) && defined(MACH_KERNEL) + +#include + +#define GDB_TRAP_INSTR1 0xe7ffdefe +#define GDB_TRAP_INSTR2 0xe7ffdeff + +#define ARM_GDB_INSTR1 GDB_TRAP_INSTR1 +#define ARM_GDB_INSTR2 GDB_TRAP_INSTR2 + +#define IS_ARM_GDB_TRAP(op) \ + (((op) == ARM_GDB_INSTR1) || ((op) == ARM_GDB_INSTR2)) + +#define THUMB_GDB_INSTR1 (GDB_TRAP_INSTR1 & 0xFFFF) +#define THUMB_GDB_INSTR2 (GDB_TRAP_INSTR2 & 0xFFFF) + +#define IS_THUMB_GDB_TRAP(op) \ + (((op) == THUMB_GDB_INSTR1) || ((op) == THUMB_GDB_INSTR2)) + + +#define ARM_STR 0x04000000 /* STR */ +#define ARM_STRH 0x000000B0 /* STRH */ +#define ARM_STRH_MASK 0x0E1000F0 /* STRH MASK */ +#define ARM_SDX_MASK 0x0C100000 /* SINGLE DATA TRANSFER */ +#define ARM_SNGL_DX_MASK 0x0C000000 /* SINGLE DATA TRANSFER MASK */ +#define ARM_SDX 0x04000000 + +#define ARM_STM 0x08000000 /* STM */ +#define ARM_BDX_MASK 0x0E100000 /* BLOCK DATA TRANSFER */ +#define ARM_BLK_MASK 0x0E000000 /* BLOCK DATA TRANSFER */ +#define ARM_BDX 0x08000000 /* BLOCK DATA TRANSFER */ + +#define ARM_WRITE_BACK 0x00200000 +#define ARM_BASE_REG 0x000F0000 +#define ARM_INCREMENT 0x00800000 + +#define ARM_STC 0x0C000000 /* STC */ +#define ARM_CDX_MASK ARM_BDX_MASK /* COPROCESSOR DATA TRANSFER */ +#define ARM_CBLK_MASK ARM_BLK_MASK +#define ARM_CDX 0x0C000000 /* COPROCESSOR DATA TRANSFER */ + +#define ARM_SWP 0x01000090 /* SWP */ +#define ARM_SWP_MASK 0x0FB00FF0 /* SWP */ + +#define ARM_POST_INDEXING 0x01000000 +#define ARM_IMMEDIATE 0x02000000 +#define ARM_LSL 0 +#define ARM_LSR 1 +#define ARM_ASR 2 +#define ARM_ROR 3 + +#define MCR_MASK 0x0F100F10 +#define MCR_CP15 0x0E000F10 +#define MCRR_MASK 0x0FF00F00 +#define MCRR_CP15 0x0C400F00 + +#define arm_mcr_cp15(op) (((op)&MCR_MASK) == 0x0E000F10) +#define arm_mcrr_cp15(op) (((op)&0x0FF00F00) == 0x0C400F00) + +#define IS_THUMB32(op) ( \ + (((op) & 0xE000) == 0xE000) && (((op) & 0x1800) != 0x0000)) + +#define THUMB_LDR_1_MASK 0x8800 /* (1) forms of LD* instructions */ +#define THUMB_STR_1_MASK 0xF800 /* (1) forms of ST* instructions */ +#define THUMB_STR_2_MASK 0xFE00 /* (2) forms of ST* instructions */ +#define THUMB_STR_3_MASK 0xF800 /* (3) forms of ST* instructions */ +#define THUMB_PUSH_MASK 0xFE00 /* PUSH instruction */ + +#define THUMB_LDRH_1 0x8800 /* LDRH(1) */ +#define THUMB_STMIA 0xC000 /* STMIA */ +#define THUMB_STR_1 0x6000 /* STR(1) */ +#define THUMB_STR_2 0x5000 /* STR(2) */ +#define THUMB_STR_3 0x9000 /* STR(3) */ +#define THUMB_STRB_1 0x7000 /* STRB(1) */ +#define THUMB_STRB_2 0x5400 /* STRB(2) */ +#define THUMB_STRH_1 0x8000 /* STRH(1) */ +#define THUMB_STRH_2 0x5200 /* STRH(2) */ +#define THUMB_PUSH 0xB400 /* PUSH */ +#define THUMB_LDMIA 0xC800 /* LDMIA */ +#define THUMB_POP 0xBC00 /* POP */ + + +/* + * Shifts, masks, and other values for load/store multiple decoding; largely needed for + * supporting misaligned accesses. + */ +#define THUMB_STR_1_BASE_OFFSET 8 /* Offset of the base register field */ +#define THUMB_PUSH_EXTRA_OFFSET 8 /* Offset of the "extra" register field */ +#define ARM_STM_BASE_OFFSET 16 /* Offset of the base register field */ +#define ARM_STM_LOAD_OFFSET 20 /* Offset of the load flag */ +#define ARM_STM_WBACK_OFFSET 21 /* Offset of the writeback flag */ +#define ARM_STM_INCR_OFFSET 23 /* Offset of the increment flag */ +#define ARM_STM_BEFORE_OFFSET 24 /* Offset of the pre-index flag */ +#define ARM_REG_LIST_LR_OFFSET 14 /* Offset of LR in the register list */ +#define ARM_REG_LIST_PC_OFFSET 15 /* Offset of PC in the register list */ + +#define THUMB_STR_REG_LIST_MASK 0x000000FF /* Offset of the reg list is 0 */ +#define THUMB_STR_1_BASE_MASK 0x00000700 +#define THUMB_PUSH_EXTRA_MASK 0x00000100 +#define ARM_STM_REG_LIST_MASK 0x0000FFFF /* Offset of the reg list is 0 */ +#define ARM_STM_BASE_MASK 0x000F0000 +#define ARM_STM_LOAD_MASK 0x00100000 +#define ARM_STM_WBACK_MASK 0x00200000 +#define ARM_STM_INCR_MASK 0x00800000 +#define ARM_STM_BEFORE_MASK 0x01000000 +#define ARM_COND_MASK 0xF0000000 /* Mask for the condition code */ + +#define ARM_COND_UNCOND 0xF0000000 /* Instruction does not support condition codes */ + +#define ARM_SIMD_MASK0 0xFE000000 +#define ARM_SIMD_CODE0 0xF2000000 + +#define ARM_VFP_MASK0 0x0F000E10 +#define ARM_VFP_CODE0 0x0E000A00 + +#define ARM_SIMD_VFP_MASK0 0x0E000E00 +#define ARM_SIMD_VFP_CODE0 0x0C000A00 +#define ARM_SIMD_VFP_MASK1 0xFF100000 +#define ARM_SIMD_VFP_CODE1 0xF4000000 +#define ARM_SIMD_VFP_MASK2 0x0F000E10 +#define ARM_SIMD_VFP_CODE2 0x0E000A10 +#define ARM_SIMD_VFP_MASK3 0x0FE00E00 +#define ARM_SIMD_VFP_CODE3 0x0C400A00 + +#define IS_ARM_VFP(op) ( \ + (((op) & ARM_SIMD_MASK0) == ARM_SIMD_CODE0) \ + ||(((op) & ARM_VFP_MASK0) == ARM_VFP_CODE0) \ + ||(((op) & ARM_SIMD_VFP_MASK0) == ARM_SIMD_VFP_CODE0) \ + ||(((op) & ARM_SIMD_VFP_MASK1) == ARM_SIMD_VFP_CODE1) \ + ||(((op) & ARM_SIMD_VFP_MASK2) == ARM_SIMD_VFP_CODE2) \ + || (((op) & ARM_SIMD_VFP_MASK3) == ARM_SIMD_VFP_CODE3)) + +#define THUMB_SIMD_MASK0 0xEF000000 +#define THUMB_SIMD_CODE0 0xEF000000 + +#define THUMB_VFP_MASK0 0xEF000E10 +#define THUMB_VFP_CODE0 0xEE000A00 + +#define THUMB_SIMD_VFP_MASK0 0xEE000E00 +#define THUMB_SIMD_VFP_CODE0 0xEC000A00 +#define THUMB_SIMD_VFP_MASK1 0xFF100000 +#define THUMB_SIMD_VFP_CODE1 0xF9000000 +#define THUMB_SIMD_VFP_MASK2 0xEF000E10 +#define THUMB_SIMD_VFP_CODE2 0xEE000A10 +#define THUMB_SIMD_VFP_MASK3 0xEFE00E00 +#define THUMB_SIMD_VFP_CODE3 0xEC400A00 + +#define IS_THUMB_VFP(op) ( \ + (((op) & THUMB_SIMD_MASK0) == THUMB_SIMD_CODE0 ) \ + || (((op) & THUMB_VFP_MASK0) == THUMB_VFP_CODE0 ) \ + || (((op) & THUMB_SIMD_VFP_MASK0) == THUMB_SIMD_VFP_CODE0 ) \ + || (((op) & THUMB_SIMD_VFP_MASK1) == THUMB_SIMD_VFP_CODE1 ) \ + || (((op) & THUMB_SIMD_VFP_MASK2) == THUMB_SIMD_VFP_CODE2 ) \ + || (((op) & THUMB_SIMD_VFP_MASK3) == THUMB_SIMD_VFP_CODE3)) + +extern boolean_t arm_swap_readable_type(vm_map_address_t, unsigned int /* spsr */); +extern boolean_t arm_force_fast_fault(ppnum_t, vm_prot_t, int, void *); +extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, boolean_t); + +/* + * Determines if the aborted instruction is read or write operation + */ +#define arm_fault_type(op,spsr,vaddr) \ + (((((op)&ARM_CDX_MASK) == ARM_STC) || \ + (((op)&ARM_STRH_MASK) == ARM_STRH) || \ + (((op)&ARM_BDX_MASK) == ARM_STM) || \ + (((op)&ARM_SDX_MASK) == ARM_STR) || \ + ((((op)&ARM_SWP_MASK) == ARM_SWP) && \ + arm_swap_readable_type(vaddr,spsr))) ? \ + (VM_PROT_WRITE|VM_PROT_READ) : (VM_PROT_READ)) + +#define thumb_fault_type(op,spsr,vaddr) \ + (((((op)&THUMB_STR_1_MASK) == THUMB_STMIA) || \ + (((op)&THUMB_STR_1_MASK) == THUMB_STR_1) || \ + (((op)&THUMB_STR_2_MASK) == THUMB_STR_2) || \ + (((op)&THUMB_STR_3_MASK) == THUMB_STR_3) || \ + (((op)&THUMB_STR_1_MASK) == THUMB_STRB_1) || \ + (((op)&THUMB_STR_2_MASK) == THUMB_STRB_2) || \ + (((op)&THUMB_STR_1_MASK) == THUMB_STRH_1) || \ + (((op)&THUMB_STR_2_MASK) == THUMB_STRH_2) || \ + (((op)&THUMB_PUSH_MASK) == THUMB_PUSH)) ? \ + (VM_PROT_WRITE|VM_PROT_READ) : (VM_PROT_READ)) + +typedef kern_return_t (*perfCallback)( + int trapno, + struct arm_saved_state *ss, + uintptr_t *, + int); + +typedef kern_return_t (*perfASTCallback)(ast_t reasons, ast_t *myast); + +extern volatile perfCallback perfTrapHook; +extern volatile perfASTCallback perfASTHook; +extern volatile perfCallback perfIntHook; + +#endif /* !ASSEMBLER && MACH_KERNEL */ + +#endif /* _ARM_TRAP_H_ */ diff --git a/osfmk/arm/vm_tuning.h b/osfmk/arm/vm_tuning.h new file mode 100644 index 000000000..728de1775 --- /dev/null +++ b/osfmk/arm/vm_tuning.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ +/* + * File: arm/vm_tuning.h + * + * VM tuning parameters for arm (without reference bits). + */ + +#ifndef _ARM_VM_TUNING_H_ +#define _ARM_VM_TUNING_H_ + +#endif /* _ARM_VM_TUNING_H_ */ diff --git a/osfmk/arm/xpr.h b/osfmk/arm/xpr.h new file mode 100644 index 000000000..b6151ddca --- /dev/null +++ b/osfmk/arm/xpr.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* + * Machine dependent module for the XPR tracing facility. + */ + +#define XPR_TIMESTAMP (0) diff --git a/osfmk/arm64/Makefile b/osfmk/arm64/Makefile new file mode 100644 index 000000000..ad75e8a11 --- /dev/null +++ b/osfmk/arm64/Makefile @@ -0,0 +1,31 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +ARM_HEADER_FILES = \ + lowglobals.h \ + machine_cpuid.h \ + machine_machdep.h \ + proc_reg.h + +INSTALL_MD_DIR = arm64 + +INSTALL_MD_LCL_LIST = + +INSTALL_MD_LIST = + +INSTALL_KF_MD_LIST = $(ARM_HEADER_FILES) + +INSTALL_KF_MD_LCL_LIST = machine_kpc.h monotonic.h pgtrace.h $(ARM_HEADER_FILES) + +EXPORT_MD_LIST = machine_cpuid.h machine_kpc.h monotonic.h proc_reg.h pgtrace.h + + +EXPORT_MD_DIR = arm64 + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/osfmk/arm64/WKdmCompress_16k.s b/osfmk/arm64/WKdmCompress_16k.s new file mode 100644 index 000000000..7588c9f40 --- /dev/null +++ b/osfmk/arm64/WKdmCompress_16k.s @@ -0,0 +1,634 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains arm64 hand optimized implementation of WKdm memory page compressor. + + int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); + + input : + src_buf : address of input page (length = 1024 words) + dest_buf : address of output buffer (may not be 16-byte aligned) + scratch : a 16-byte aligned 4k bytes scratch memory provided by the caller, + bytes_budget : a given byte target in compression + + output : + + if the input buffer can be compressed within the given byte budget, the dest_buf is written with compressed data and the function returns with number of bytes for the compressed data + o.w., the function returns -1 to signal that the input data can not be compressed with the given byte budget. + During the scan and tag process, each word that can not be compressed will be written to dest_buf, followed by a 12-bytes header + 256-bytes tag area. + When the functions returns -1, dest_buf is filled with all those words that can not be compressed and should be considered undefined. + The worst-case scenario is that all words can not be compressed. Hence, the minimum size requirement for dest_buf should be 12+256+4096 = 4364 bytes to prevent from memory fault. + + The 4th argument bytes_budget is the target compress budget in bytes. + Should the input page can be compressed within the budget, the compressed data is written to *dest_buf, and the function returns the number of compressed bytes. + Otherwise, the function returns -1 (to signal to the caller that the page can not be compressed). + + WKdm Compression algorithm is briefly stated as follows: + + There is a dynamically updated dictionary consisting of 16 words. Each dictionary word is initialized to 1 at the point of entry to the function. + For a nonzero input word x, its 8-bits (10-bits scaled up) is used to determine a corresponding word from the dictionary, represented by dict_index (4-bits) and dict_word (32-bits). + a. k = (x>>10)&255; // 8-bit hash table index + b. dict_index = hashTable[k]; // 4-bit dictionary index, hashTable[] is fixed + c. dict_word = dictionary[dict_index]; // 32-bit dictionary word, dictionary[] is dynamically updated + + Each input word x is classified/tagged into 4 classes : + 0 : x = 0 + 1 : (x>>10) == (dict_word>>10), bits 10:31 of the input word match a dictionary word + 2 : (x>>10) != (dict_word>>10), the above condition (22 higher bits matched) is not met, meaning a dictionary miss + 3 : (x == dict_word), the exact input word is in the dictionary + + For each class, different numbers of bits are needed for the decompressor to reproduce the original input word. + 0 : 2-bits tag (32->2 compression) + 1 : 2-bits tag + 4-bits dict_index + 10-bits lower bits (32->16 compression) + 2 : 2-bits tag + 32-bits new word (32->34 expansion) + 3 : 2-bits tag + 4-bits dict_index (32->6 compression) + + It is obvious now that WKdm compress algorithm works well for pages where there are lots of zero words (32->2) and/or there are freqeunt repeats of some word patterns (32->6). + + the output bit stream (*dest_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + the header is actually of 3 words that specify the ending offset (in 32-bit words) from the start of the bit stream of c,d,e, respectively. + Note that there might be padding bits in d (if the number of dict_indices does not divide by 8), and there are 2/12/22 padding bits for packing 3/2/1 low 10-bits in a 32-bit word. + + + The WKdm compress algorithm 1st runs a scan and classification pass, tagging and write unpacked data into temporary buffers. It follows by packing those data into the output buffer. + + The temp buffers are + + uint8_t tempTagsArray[1024]; // temporary saving for tags before final packing + uint8_t tempQPosArray[1024]; // temporary saving for dict_indices before final packing + uint16_t tempLowBitsArray[1024]; // temporary saving for partially matched lower 10 bits before final packing + + Since the new words (that can not matched fully or partially to the dictionary) are stored right after the header and the tags section and need no packing, we directly write them to + the destination buffer. + + uint32_t *new_word = dest_buf+3+64; // 3 words for header, 64 words for tags, new words come right after the tags. + + Now since we are given a byte budget for this compressor, we can monitor the byte (or bit) usage on the fly in the scanning and tagging pass. + + byte_count -= 12 + 256; // bit budget minus header and tags + + whenever an input word is classified as class + + 2 : byte_count -= 4; + + the compress function can early exit (return -1) should the page can not be compressed with the given byte budget (i.e., byte_count <= 0). + + without showing the bit budget management, the pseudo code is given as follows: + + uint8_t *tags=tempTagsArray; + uint8_t *dict=tempQPosArray; + uint8_t *partial=tempLowBitsArray; + + for (i=0;i<1024;i++) { + x = *src_buf++; + if (x == 0) { // zero, 2-bits tag + *tags++ = 0; + } else { + + // find dict_index and dict_word from x + k = (x>>10)&255; + dict_index = hashTable[k]; + dict_word = dictionary[dict_index]; + + if (dict_word == x) { // exactly match + // 2-bits tag + 4-bits table index + *tags++ = 3; + *dict++ = dict_index; + } else if (((x^dict_word)>>10)==0) { // 22 higher bits matched + // 2-bits tag + 4-bits table index + 10-bits lower partial + *tags++ = 1; + *dict++ = dict_index; + *partial++ = x &0x3ff; + dictionary[dict_index] = x; + } else { // not matched + // 2-bits tag + 32-bits new word + *tags++ = 2; + *new_word++ = x; + dictionary[dict_index] = x; + } + } + } + + after this classification/tagging pass is completed, the 3 temp buffers are packed into the output *dest_buf: + + 1. 1024 tags are packed into 256 bytes right after the 12-bytes header + 2. dictionary indices (4-bits each) are packed into are right after the new words section + 3. 3 low 10-bits are packed into a 32-bit word, this is after the dictionary indices section. + + cclee, 11/9/12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 +*/ + +#define PAGES_SIZE_IN_KBYTES 16 + +#ifndef PAGES_SIZE_IN_KBYTES +#define PAGES_SIZE_IN_KBYTES 4 +#endif + +#if !((PAGES_SIZE_IN_KBYTES==4) || (PAGES_SIZE_IN_KBYTES==16)) +#error "Only PAGES_SIZE_IN_KBYTES = 4 or 16 is supported" +#endif + + + .text + .align 4 + +/* + int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); +*/ + +.globl _WKdm_compress_16k +_WKdm_compress_16k: + +/* + ------------------------- symbolizing register use ----------------------------------- +*/ + #define src_buf x0 + #define next_input_word x0 + #define dest_buf x1 + #define scratch x2 + #define byte_count x3 + #define next_tag x4 + #define tempTagsArray x2 // scratch + #define dictionary x5 + #define remaining x6 + #define next_full_patt x7 + #define dict_location x8 + #define wdict_location w8 + #define next_qp x9 + #define hashTable x10 + #define tempQPosArray x11 + #define next_low_bits x12 + +/* + this arm64 assembly code is ported from x86_64 assembly code, + therefore need such symbolization to quickly reuse the x86_64 assembly code + for these intermediate/temporary register use +*/ + #define rax x13 + #define eax w13 + #define rcx x14 + #define ecx w14 + #define rdx x15 + #define edx w15 + #define rdi x0 /* after some point, x0/rdi becomes free other usage */ + + +/* + ------------------------- scratch memory -------------------------------------- + + need 16*4 (dictionary) + 256*4 (tempTagsArray) + 256*4 (tempQPosArray) + 1024*4 (tempLowBitsArray) + total 6208 bytes + [sp,#0] : dictionary + [scratch,#0] : tempTagsArray + [scratch,#1024] : tempQPosArray + [scratch,#2048] : tempLowBitsArray +*/ + +#define scale (PAGES_SIZE_IN_KBYTES/4) + +#define SV_RETURN 0 // return value when SV, ZV page is found +#define MZV_MAGIC 17185 // magic value used to identify MZV page encoding +#define CHKPT_BYTES 416 // for early aborts: checkpoint after processing this many bytes. Must be in range [4..4096] +#define CHKPT_WORDS (CHKPT_BYTES/4) // checkpoint bytes in words +#define CHKPT_TAG_BYTES (CHKPT_BYTES/16) // size of the tags for CHKPT_BYTES of data +#define CHKPT_SHRUNK_BYTES 426 // for early aborts: max size of compressed stream to allow further processing .. + // .. to disable early aborts, set CHKPT_SHRUNK_BYTES to 4096 +#if CHKPT_BYTES > 4096 + #error CHKPT_BYTES must be <= 4096 +#endif +#if CHKPT_BYTES < 4 + #error CHKPT_BYTES must be >= 4 +#endif + +#if KERNEL + sub sp, sp, #64 + st1.4s {v0,v1,v2,v3},[sp] +#endif + + sub sp, sp, #64 // allocate for dictionary + mov dictionary, sp // use x5 to point to sp, so we can use sub xd, xn, sp + + sub sp, sp, #64 // allocate space for saving callee-saved registers + mov x15, sp + stp x20, x21, [x15, #0] // save x20, x21 + stp x22, x23, [x15, #16] // save x22, x23 + stp x24, x25, [x15, #32] // save x24, x25 + stp x26, x27, [x15, #48] // save x26, x27 + +/* + ------- entwined statck space allocation, registers set up, and PRELOAD_DICTIONARY ------------------- +*/ + + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO + // THIS IS NEEDED TO EFFICIENTLY DETECT SINGLE VALUE PAGES + mov next_tag, tempTagsArray // &tempTagsArray[0] + add next_qp, scratch, #(1024*scale) // next_qp + mov remaining, #(CHKPT_WORDS*scale) // remaining input words .. initially set to checkpoint + add next_full_patt, dest_buf, #(12+256*scale) // dest_buf + [TAGS_AREA_OFFSET + (num_input_words / 16)]*4 + sub byte_count, byte_count, #(12+256*scale) // bit_count - header - tags + add next_low_bits, scratch, #(2048*scale) // &tempLowBitsArray[0] + stp xzr, xzr, [dictionary, #0] // initialize dictionary + adrp hashTable, _hashLookupTable@GOTPAGE + stp xzr, xzr, [dictionary, #16] // initialize dictionary + stp xzr, xzr, [dictionary, #32] // initialize dictionary + ldr hashTable, [hashTable, _hashLookupTable@GOTPAGEOFF] + stp xzr, xzr, [dictionary, #48] // initialize dictionary + +#define EARLYCHECK 0 +#define NORMAL 1 + +#define mode w20 +#define start_next_full_patt x21 +#define start_next_input_word x22 +#define start_next_low_bits x23 +#define r11 x24 +#define r13 x25 +#define byte_budget x26 +#define start_next_qp tempQPosArray + + add tempQPosArray, scratch, #(1024*scale) // &tempQPosArray[0] + mov mode, EARLYCHECK // indicate we are yet to evaluate the early aborts + mov start_next_full_patt, next_full_patt // remember the start of next_full_patt + mov start_next_input_word, next_input_word // remember the start of next_input_word + mov start_next_low_bits, next_low_bits // remember the start of next_low_bit + add byte_budget, byte_count, #(12+256*scale) // remember the byte budget + + b L_loop + + .align 4, 0x90 + + /* we've just detected a zero input word in edx */ +L_RECORD_ZERO: + strb edx, [next_tag], #1 // *next_tag++ = ZERO; edx is used as input word, and if we are here edx = 0 + subs remaining, remaining, #1 // remaing--; + b.le CHECKPOINT // if remaining = 0, break + + /* -------------- scan/tag pass loop ------------------------- */ +L_loop: + + /* load new input word to edx */ + ldr edx, [next_input_word], #4 + cbz edx, L_RECORD_ZERO // if (input_word==0) RECORD_ZERO + + /* + now the input word edx is nonzero, we next find the corresponding dictionary word (eax) and dict_location + */ + ubfm eax, edx, #10, #17 + ldrb wdict_location, [hashTable, rax] // HASH_TO_DICT_BYTE_OFFSET(input_word) + ldr eax, [dictionary, dict_location] // dict_word = *dict_location; + + /* detect whether we match input to its corresponding dictionary word */ + eor eax, eax, edx // dict_word vs input_word + cbz eax, L_RECORD_EXACT // if identical, RECORD_EXACT + lsr eax, eax, #10 // HIGH_BITS(dict_word^input_word) + cbz eax, L_RECORD_PARTIAL // if identical, RECORD_PARTIAL + +L_RECORD_MISS: +/* + if we are here, the input word can not be derived from the dictionary, + we write the input word as a new word, + and update the dictionary with this new word +*/ + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + str edx, [next_full_patt], #4 // *next_full_patt++ = input_word; + mov eax, #2 // tag for MISS + subs remaining, remaining, #1 // remaing--; + str edx, [dictionary, dict_location] // *dict_location = input_word + strb eax, [next_tag], #1 // *next_tag++ = 2 for miss + b.gt L_loop // // if remaining > 0, repeat + b CHECKPOINT + +L_done_search: + + // SET_QPOS_AREA_START(dest_buf,next_full_patt); + /* 1st word in dest_buf header = 4-byte offset (from start) of end of new word section */ + + sub rax, next_full_patt, dest_buf // next_full_patt - dest_buf + lsr eax, eax, #2 // offset in 4-bytes + str eax, [dest_buf] // dest_buf[0] = next_full_patt - dest_buf + + /* -------------------------- packing 1024 tags into 256 bytes ----------------------------------------*/ + // boundary_tmp = WK_pack_2bits(tempTagsArray, (WK_word *) next_tag, dest_buf + HEADER_SIZE_IN_WORDS); + + add rdi, dest_buf, #12 // dest_buf + mov rcx, tempTagsArray // &tempTagsArray[0] + +L_pack_2bits: + ld1.2s {v0,v1,v2,v3},[rcx],#32 + + shl.2d v1,v1,#4 + shl.2d v3,v3,#4 + + orr.8b v0, v0, v1 + orr.8b v2, v2, v3 + + ushr.2d v1, v0, #30 + ushr.2d v3, v2, #30 + + orr.8b v0, v0, v1 + orr.8b v2, v2, v3 + + zip1.2s v0, v0, v2 + st1.2s {v0},[rdi],#8 + cmp next_tag, rcx + b.hi L_pack_2bits + + /* --------------------------------- packing 4-bits dict indices into dest_buf ---------------------------------- */ + + /* 1st, round up number of 4-bits dict_indices to a multiple of 8 and fill in 0 if needed */ + sub rax, next_qp, tempQPosArray // eax = num_bytes_to_pack = next_qp - (char *) tempQPosArray; + add eax, eax, #7 // num_bytes_to_pack+7 + lsr eax, eax, #3 // num_packed_words = (num_bytes_to_pack + 7) >> 3 + add rcx, tempQPosArray, rax, lsl #3 // endQPosArray = tempQPosArray + 2*num_source_words + lsl rax, rax, #2 + subs byte_count, byte_count, rax + b.lt L_budgetExhausted + + cmp rcx, next_qp // endQPosArray vs next_qp + b.ls 2f // if (next_qp >= endQPosArray) skip the following zero paddings + sub rax, rcx, next_qp + mov edx, #0 + tst eax, #4 + b.eq 1f + str edx, [next_qp], #4 +1: tst eax, #2 + b.eq 1f + strh edx, [next_qp], #2 +1: tst eax, #1 + b.eq 2f + strb edx, [next_qp], #1 +2: + mov rdi, next_full_patt // next_full_patt + cmp rcx, tempQPosArray // endQPosArray vs tempQPosArray + ldr eax, [dest_buf] + b.ls L20 // if (endQPosArray <= tempQPosArray) skip the following + mov rdx, tempQPosArray // tempQPosArray + + /* packing 4-bits dict indices into dest_buf */ +L_pack_4bits: + ldr rax, [rdx], #8 // src_next[1]:src_next[0] + orr rax, rax, rax, lsr #28 // eax = src_next[0] | (src_next[1] << 4) + cmp rcx, rdx // source_end vs src_next + str eax, [rdi], #4 // *dest_next++ = temp; + b.hi L_pack_4bits // while (src_next < source_end) repeat the loop + + // SET_LOW_BITS_AREA_START(dest_buf,boundary_tmp); + sub rax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in words +L20: + str eax, [dest_buf,#4] // dest_buf[1] = boundary_tmp - dest_buf + + + + /* --------------------------- packing 3 10-bits low bits into a 32-bit word in dest_buf[] ----------------------------------------- */ + + add rcx, scratch, #(2048*scale) // tempLowBitsArray + sub rdx, next_low_bits, rcx // next_low_bits - tempLowBitsArray (in bytes) + lsr rdx, rdx, #1 // num_tenbits_to_pack (in half-words) + subs edx, edx, #3 // pre-decrement num_tenbits_to_pack by 3 + b.lt 1f // if num_tenbits_to_pack < 3, skip the following loop +0: + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + subs edx, edx, #3 // num_tenbits_to_pack-=3 + ldr rax, [rcx], #6 + bfm rax, rax, #58, #9 // pack 1st toward 2nd + bfm rax, rax, #58, #25 // pack 1st/2nd toward 3rd + lsr rax, rax, #12 + str eax, [rdi], #4 // pack w0,w1,w2 into 1 dest_buf word + b.ge 0b // if no less than 3 elements, back to loop head + +1: adds edx, edx, #3 // post-increment num_tenbits_to_pack by 3 + b.eq 3f // if num_tenbits_to_pack is a multiple of 3, skip the following + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + ldrh eax,[rcx] // w0 + subs edx, edx, #1 // num_tenbits_to_pack-- + b.eq 2f // + ldrh edx, [rcx, #2] // w1 + orr eax, eax, edx, lsl #10 // w0 | (w1<<10) + +2: str eax, [rdi], #4 // write the final dest_buf word + +3: sub rax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in terms of words + str eax, [dest_buf, #8] // SET_LOW_BITS_AREA_END(dest_buf,boundary_tmp) + lsl w0, eax, #2 // boundary_tmp - dest_buf in terms of bytes + +L_done: + + // restore registers and return + mov x15, sp + ldp x20, x21, [x15, #0] // restore x20, x21 + ldp x22, x23, [x15, #16] // restore x22, x23 + ldp x24, x25, [x15, #32] // restore x24, x25 + ldp x26, x27, [x15, #48] // restore x26, x27 + add sp, sp, #128 // deallocate for dictionary + saved register space + +#if KERNEL + ld1.4s {v0,v1,v2,v3},[sp],#64 +#endif + ret lr + + .align 4 +L_budgetExhausted: + mov x0, #-1 + b L_done + + + .align 4,0x90 +L_RECORD_EXACT: +/* + we have an exact match of the input word to its corresponding dictionary word + write tag/dict_index to the temorary buffers +*/ + mov eax, #3 + lsr w14, wdict_location, #2 // divide by 4 for word offset + subs remaining, remaining, #1 // remaing--; + strb eax, [next_tag], #1 // *next_tag++ = 3 for exact + strb w14, [next_qp], #1 // *next_qp = word offset (4-bit) + b.gt L_loop + b CHECKPOINT // if remaining = 0, break + + .align 4,0x90 +L_RECORD_PARTIAL: +/* + we have a partial (high 22-bits) match of the input word to its corresponding dictionary word + write tag/dict_index/low 10 bits to the temorary buffers +*/ + mov ecx, #1 + strb ecx, [next_tag], #1 // *next_tag++ = 1 for partial matched + str edx, [dictionary, dict_location] // *dict_location = input_word; + subs remaining, remaining, #1 // remaing--; + lsr eax, wdict_location, #2 // offset in 32-bit word + and edx, edx, #1023 // lower 10 bits + strb eax, [next_qp], #1 // update *next_qp++ + strh edx, [next_low_bits], #2 // save next_low_bits++ + b.gt L_loop + +CHECKPOINT: + + cbz mode, L_check_compression_ratio // if this this an early abort check.. + +L_check_zero_page: + + cmp start_next_full_patt, next_full_patt // check if any dictionary misses in page + b.ne L_check_single_value_page + + cmp start_next_qp, next_qp // check if any partial or exact dictionary matches + b.ne L_check_single_value_page + + mov x0, #SV_RETURN // Magic return value + b L_done + +L_check_single_value_page: + + sub rax, next_full_patt, start_next_full_patt // get # dictionary misses + lsr rax, rax, #2 + + sub r11, next_qp, start_next_qp // get # dictionary hits (exact + partial) + + sub r13, next_low_bits, start_next_low_bits // get # dictionary partial hits + lsr r13, r13, #1 + + // Single value page if one of the follwoing is true: + // partial == 0 AND hits == 1023(for 4K page) AND miss == 1 AND tag[0] == 2 (i.e. miss) + // partial == 1 AND hits == 1024(for 4K page) AND tag[0] == 1 (i.e. partial) + // + cbnz r13, 1f // were there 0 partial hits? + + cmp r11, #(256*PAGES_SIZE_IN_KBYTES - 1) // were there 1023 dictionary hits + b.ne 1f + + cmp rax, #1 // was there exacly 1 dictionary miss? + b.ne 1f + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #2 // was the very 1st tag a miss? + b.eq L_is_single_value_page + +1: + cmp r13, #1 // was there 1 partial hit? + b.ne L_check_mostly_zero + + cmp r11, #(256*PAGES_SIZE_IN_KBYTES) // were there 1024 dictionary hits + b.ne L_check_mostly_zero + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #1 // was the very 1st tag a partial? + b.ne L_check_mostly_zero + +L_is_single_value_page: + + mov x0, #SV_RETURN // Magic return value + b L_done + +L_check_mostly_zero: + // how much space will the sparse packer take? + add rax, rax, r11 // rax += (next_qp - start_next_qp) + mov rdx, #6 + mov rcx, #4 + madd r11, rax, rdx, rcx // r11 = rax * 6 (i.e. 4 byte word + 2 byte offset) + 4 byte for header + + sub rax, next_low_bits, start_next_low_bits // get bytes consumed by lower-10 bits + mov rdx, #1365 + mul rax, rax, rdx + + sub rdx, next_full_patt, start_next_full_patt // get bytes consumed by dictionary misses + add rax, rdx, rax, lsr #11 // rax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + sub rdx, next_qp, start_next_qp + add rax, rax, rdx, lsr #1 // rax += (next_qp - start_next_qp)/2 + add rax, rax, #(12+256*scale) // rax += bytes taken by the header + tags + + cmp rax, r11 // is the default packer the better option? + b.lt L_done_search + + cmp r11, byte_budget // can the sparse packer fit into the given budget? + b.gt L_budgetExhausted + +L_sparse_packer: + mov edx, #MZV_MAGIC + str edx, [dest_buf], #4 // header to indicate a sparse packer + + mov rdx, #0 // rdx = byte offset in src of non-0 word +1: + ldr rax, [start_next_input_word, rdx] // rax = read dword + cbnz rax, 5f // is dword != 0 +3: + add rdx, rdx, #8 // 8 more bytes have been processed +4: + cmp rdx, #(4096*scale) // has the entire page been processed + b.ne 1b + mov x0, r11 // store the size of the compressed stream + b L_done + +5: + cbz eax, 6f // is lower word == 0 + str eax, [dest_buf], #4 // store the non-0 word in the dest buffer + strh edx, [dest_buf], #2 // store the byte index +6: + lsr rax, rax, 32 // get the upper word into position + cbz eax, 3b // is dword == 0 + add rdx, rdx, #4 + str eax, [dest_buf], #4 // store the non-0 word in the dest buffer + strh edx, [dest_buf], #2 // store the byte index + add rdx, rdx, #4 + b 4b + +L_check_compression_ratio: + + mov mode, NORMAL + mov remaining, #((1024 - CHKPT_WORDS)*scale) // remaining input words to process + cbz remaining, CHECKPOINT // if there are no remaining words to process + + sub rax, next_low_bits, start_next_low_bits // get bytes consumed by lower-10 bits + mov rdx, #1365 + mul rax, rax, rdx + + sub rdx, next_full_patt, start_next_full_patt // get bytes consumed by dictionary misses + add rax, rdx, rax, lsr #11 // rax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + sub rdx, next_qp, start_next_qp + add rax, rax, rdx, lsr #1 // rax += (next_qp - start_next_qp)/2 + subs rax, rax, #((CHKPT_SHRUNK_BYTES - CHKPT_TAG_BYTES)*scale) + // rax += CHKPT_TAG_BYTES; rax -= CHKPT_SHRUNK_BYTES + + b.gt L_budgetExhausted // if rax is > 0, we need to early abort + b L_loop // we are done diff --git a/osfmk/arm64/WKdmCompress_4k.s b/osfmk/arm64/WKdmCompress_4k.s new file mode 100644 index 000000000..2ac438768 --- /dev/null +++ b/osfmk/arm64/WKdmCompress_4k.s @@ -0,0 +1,632 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains arm64 hand optimized implementation of WKdm memory page compressor. + + int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); + + input : + src_buf : address of input page (length = 1024 words) + dest_buf : address of output buffer (may not be 16-byte aligned) + scratch : a 16-byte aligned 4k bytes scratch memory provided by the caller, + bytes_budget : a given byte target in compression + + output : + + if the input buffer can be compressed within the given byte budget, the dest_buf is written with compressed data and the function returns with number of bytes for the compressed data + o.w., the function returns -1 to signal that the input data can not be compressed with the given byte budget. + During the scan and tag process, each word that can not be compressed will be written to dest_buf, followed by a 12-bytes header + 256-bytes tag area. + When the functions returns -1, dest_buf is filled with all those words that can not be compressed and should be considered undefined. + The worst-case scenario is that all words can not be compressed. Hence, the minimum size requirement for dest_buf should be 12+256+4096 = 4364 bytes to prevent from memory fault. + + The 4th argument bytes_budget is the target compress budget in bytes. + Should the input page can be compressed within the budget, the compressed data is written to *dest_buf, and the function returns the number of compressed bytes. + Otherwise, the function returns -1 (to signal to the caller that the page can not be compressed). + + WKdm Compression algorithm is briefly stated as follows: + + There is a dynamically updated dictionary consisting of 16 words. Each dictionary word is initialized to 1 at the point of entry to the function. + For a nonzero input word x, its 8-bits (10-bits scaled up) is used to determine a corresponding word from the dictionary, represented by dict_index (4-bits) and dict_word (32-bits). + a. k = (x>>10)&255; // 8-bit hash table index + b. dict_index = hashTable[k]; // 4-bit dictionary index, hashTable[] is fixed + c. dict_word = dictionary[dict_index]; // 32-bit dictionary word, dictionary[] is dynamically updated + + Each input word x is classified/tagged into 4 classes : + 0 : x = 0 + 1 : (x>>10) == (dict_word>>10), bits 10:31 of the input word match a dictionary word + 2 : (x>>10) != (dict_word>>10), the above condition (22 higher bits matched) is not met, meaning a dictionary miss + 3 : (x == dict_word), the exact input word is in the dictionary + + For each class, different numbers of bits are needed for the decompressor to reproduce the original input word. + 0 : 2-bits tag (32->2 compression) + 1 : 2-bits tag + 4-bits dict_index + 10-bits lower bits (32->16 compression) + 2 : 2-bits tag + 32-bits new word (32->34 expansion) + 3 : 2-bits tag + 4-bits dict_index (32->6 compression) + + It is obvious now that WKdm compress algorithm works well for pages where there are lots of zero words (32->2) and/or there are freqeunt repeats of some word patterns (32->6). + + the output bit stream (*dest_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + the header is actually of 3 words that specify the ending offset (in 32-bit words) from the start of the bit stream of c,d,e, respectively. + Note that there might be padding bits in d (if the number of dict_indices does not divide by 8), and there are 2/12/22 padding bits for packing 3/2/1 low 10-bits in a 32-bit word. + + + The WKdm compress algorithm 1st runs a scan and classification pass, tagging and write unpacked data into temporary buffers. It follows by packing those data into the output buffer. + + The temp buffers are + + uint8_t tempTagsArray[1024]; // temporary saving for tags before final packing + uint8_t tempQPosArray[1024]; // temporary saving for dict_indices before final packing + uint16_t tempLowBitsArray[1024]; // temporary saving for partially matched lower 10 bits before final packing + + Since the new words (that can not matched fully or partially to the dictionary) are stored right after the header and the tags section and need no packing, we directly write them to + the destination buffer. + + uint32_t *new_word = dest_buf+3+64; // 3 words for header, 64 words for tags, new words come right after the tags. + + Now since we are given a byte budget for this compressor, we can monitor the byte (or bit) usage on the fly in the scanning and tagging pass. + + byte_count -= 12 + 256; // bit budget minus header and tags + + whenever an input word is classified as class + + 2 : byte_count -= 4; + + the compress function can early exit (return -1) should the page can not be compressed with the given byte budget (i.e., byte_count <= 0). + + without showing the bit budget management, the pseudo code is given as follows: + + uint8_t *tags=tempTagsArray; + uint8_t *dict=tempQPosArray; + uint8_t *partial=tempLowBitsArray; + + for (i=0;i<1024;i++) { + x = *src_buf++; + if (x == 0) { // zero, 2-bits tag + *tags++ = 0; + } else { + + // find dict_index and dict_word from x + k = (x>>10)&255; + dict_index = hashTable[k]; + dict_word = dictionary[dict_index]; + + if (dict_word == x) { // exactly match + // 2-bits tag + 4-bits table index + *tags++ = 3; + *dict++ = dict_index; + } else if (((x^dict_word)>>10)==0) { // 22 higher bits matched + // 2-bits tag + 4-bits table index + 10-bits lower partial + *tags++ = 1; + *dict++ = dict_index; + *partial++ = x &0x3ff; + dictionary[dict_index] = x; + } else { // not matched + // 2-bits tag + 32-bits new word + *tags++ = 2; + *new_word++ = x; + dictionary[dict_index] = x; + } + } + } + + after this classification/tagging pass is completed, the 3 temp buffers are packed into the output *dest_buf: + + 1. 1024 tags are packed into 256 bytes right after the 12-bytes header + 2. dictionary indices (4-bits each) are packed into are right after the new words section + 3. 3 low 10-bits are packed into a 32-bit word, this is after the dictionary indices section. + + cclee, 11/9/12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 +*/ + +#ifndef PAGES_SIZE_IN_KBYTES +#define PAGES_SIZE_IN_KBYTES 4 +#endif + +#if !((PAGES_SIZE_IN_KBYTES==4) || (PAGES_SIZE_IN_KBYTES==16)) +#error "Only PAGES_SIZE_IN_KBYTES = 4 or 16 is supported" +#endif + + + .text + .align 4 + +/* + int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); +*/ + +.globl _WKdm_compress_4k +_WKdm_compress_4k: + +/* + ------------------------- symbolizing register use ----------------------------------- +*/ + #define src_buf x0 + #define next_input_word x0 + #define dest_buf x1 + #define scratch x2 + #define byte_count x3 + #define next_tag x4 + #define tempTagsArray x2 // scratch + #define dictionary x5 + #define remaining x6 + #define next_full_patt x7 + #define dict_location x8 + #define wdict_location w8 + #define next_qp x9 + #define hashTable x10 + #define tempQPosArray x11 + #define next_low_bits x12 + +/* + this arm64 assembly code is ported from x86_64 assembly code, + therefore need such symbolization to quickly reuse the x86_64 assembly code + for these intermediate/temporary register use +*/ + #define rax x13 + #define eax w13 + #define rcx x14 + #define ecx w14 + #define rdx x15 + #define edx w15 + #define rdi x0 /* after some point, x0/rdi becomes free other usage */ + + +/* + ------------------------- scratch memory -------------------------------------- + + need 16*4 (dictionary) + 256*4 (tempTagsArray) + 256*4 (tempQPosArray) + 1024*4 (tempLowBitsArray) + total 6208 bytes + [sp,#0] : dictionary + [scratch,#0] : tempTagsArray + [scratch,#1024] : tempQPosArray + [scratch,#2048] : tempLowBitsArray +*/ + +#define scale (PAGES_SIZE_IN_KBYTES/4) + +#define SV_RETURN 0 // return value when SV, ZV page is found +#define MZV_MAGIC 17185 // magic value used to identify MZV page encoding +#define CHKPT_BYTES 416 // for early aborts: checkpoint after processing this many bytes. Must be in range [4..4096] +#define CHKPT_WORDS (CHKPT_BYTES/4) // checkpoint bytes in words +#define CHKPT_TAG_BYTES (CHKPT_BYTES/16) // size of the tags for CHKPT_BYTES of data +#define CHKPT_SHRUNK_BYTES 426 // for early aborts: max size of compressed stream to allow further processing .. + // .. to disable early aborts, set CHKPT_SHRUNK_BYTES to 4096 +#if CHKPT_BYTES > 4096 + #error CHKPT_BYTES must be <= 4096 +#endif +#if CHKPT_BYTES < 4 + #error CHKPT_BYTES must be >= 4 +#endif + +#if KERNEL + sub sp, sp, #64 + st1.4s {v0,v1,v2,v3},[sp] +#endif + + sub sp, sp, #64 // allocate for dictionary + mov dictionary, sp // use x5 to point to sp, so we can use sub xd, xn, sp + + sub sp, sp, #64 // allocate space for saving callee-saved registers + mov x15, sp + stp x20, x21, [x15, #0] // save x20, x21 + stp x22, x23, [x15, #16] // save x22, x23 + stp x24, x25, [x15, #32] // save x24, x25 + stp x26, x27, [x15, #48] // save x26, x27 + +/* + ------- entwined statck space allocation, registers set up, and PRELOAD_DICTIONARY ------------------- +*/ + + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO + // THIS IS NEEDED TO EFFICIENTLY DETECT SINGLE VALUE PAGES + mov next_tag, tempTagsArray // &tempTagsArray[0] + add next_qp, scratch, #(1024*scale) // next_qp + mov remaining, #(CHKPT_WORDS*scale) // remaining input words .. initially set to checkpoint + add next_full_patt, dest_buf, #(12+256*scale) // dest_buf + [TAGS_AREA_OFFSET + (num_input_words / 16)]*4 + sub byte_count, byte_count, #(12+256*scale) // bit_count - header - tags + add next_low_bits, scratch, #(2048*scale) // &tempLowBitsArray[0] + stp xzr, xzr, [dictionary, #0] // initialize dictionary + adrp hashTable, _hashLookupTable@GOTPAGE + stp xzr, xzr, [dictionary, #16] // initialize dictionary + stp xzr, xzr, [dictionary, #32] // initialize dictionary + ldr hashTable, [hashTable, _hashLookupTable@GOTPAGEOFF] + stp xzr, xzr, [dictionary, #48] // initialize dictionary + +#define EARLYCHECK 0 +#define NORMAL 1 + +#define mode w20 +#define start_next_full_patt x21 +#define start_next_input_word x22 +#define start_next_low_bits x23 +#define r11 x24 +#define r13 x25 +#define byte_budget x26 +#define start_next_qp tempQPosArray + + add tempQPosArray, scratch, #(1024*scale) // &tempQPosArray[0] + mov mode, EARLYCHECK // indicate we are yet to evaluate the early aborts + mov start_next_full_patt, next_full_patt // remember the start of next_full_patt + mov start_next_input_word, next_input_word // remember the start of next_input_word + mov start_next_low_bits, next_low_bits // remember the start of next_low_bit + add byte_budget, byte_count, #(12+256*scale) // remember the byte budget + + b L_loop + + .align 4, 0x90 + + /* we've just detected a zero input word in edx */ +L_RECORD_ZERO: + strb edx, [next_tag], #1 // *next_tag++ = ZERO; edx is used as input word, and if we are here edx = 0 + subs remaining, remaining, #1 // remaing--; + b.le CHECKPOINT // if remaining = 0, break + + /* -------------- scan/tag pass loop ------------------------- */ +L_loop: + + /* load new input word to edx */ + ldr edx, [next_input_word], #4 + cbz edx, L_RECORD_ZERO // if (input_word==0) RECORD_ZERO + + /* + now the input word edx is nonzero, we next find the corresponding dictionary word (eax) and dict_location + */ + ubfm eax, edx, #10, #17 + ldrb wdict_location, [hashTable, rax] // HASH_TO_DICT_BYTE_OFFSET(input_word) + ldr eax, [dictionary, dict_location] // dict_word = *dict_location; + + /* detect whether we match input to its corresponding dictionary word */ + eor eax, eax, edx // dict_word vs input_word + cbz eax, L_RECORD_EXACT // if identical, RECORD_EXACT + lsr eax, eax, #10 // HIGH_BITS(dict_word^input_word) + cbz eax, L_RECORD_PARTIAL // if identical, RECORD_PARTIAL + +L_RECORD_MISS: +/* + if we are here, the input word can not be derived from the dictionary, + we write the input word as a new word, + and update the dictionary with this new word +*/ + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + str edx, [next_full_patt], #4 // *next_full_patt++ = input_word; + mov eax, #2 // tag for MISS + subs remaining, remaining, #1 // remaing--; + str edx, [dictionary, dict_location] // *dict_location = input_word + strb eax, [next_tag], #1 // *next_tag++ = 2 for miss + b.gt L_loop // // if remaining > 0, repeat + b CHECKPOINT + +L_done_search: + + // SET_QPOS_AREA_START(dest_buf,next_full_patt); + /* 1st word in dest_buf header = 4-byte offset (from start) of end of new word section */ + + sub rax, next_full_patt, dest_buf // next_full_patt - dest_buf + lsr eax, eax, #2 // offset in 4-bytes + str eax, [dest_buf] // dest_buf[0] = next_full_patt - dest_buf + + /* -------------------------- packing 1024 tags into 256 bytes ----------------------------------------*/ + // boundary_tmp = WK_pack_2bits(tempTagsArray, (WK_word *) next_tag, dest_buf + HEADER_SIZE_IN_WORDS); + + add rdi, dest_buf, #12 // dest_buf + mov rcx, tempTagsArray // &tempTagsArray[0] + +L_pack_2bits: + ld1.2s {v0,v1,v2,v3},[rcx],#32 + + shl.2d v1,v1,#4 + shl.2d v3,v3,#4 + + orr.8b v0, v0, v1 + orr.8b v2, v2, v3 + + ushr.2d v1, v0, #30 + ushr.2d v3, v2, #30 + + orr.8b v0, v0, v1 + orr.8b v2, v2, v3 + + zip1.2s v0, v0, v2 + st1.2s {v0},[rdi],#8 + cmp next_tag, rcx + b.hi L_pack_2bits + + /* --------------------------------- packing 4-bits dict indices into dest_buf ---------------------------------- */ + + /* 1st, round up number of 4-bits dict_indices to a multiple of 8 and fill in 0 if needed */ + sub rax, next_qp, tempQPosArray // eax = num_bytes_to_pack = next_qp - (char *) tempQPosArray; + add eax, eax, #7 // num_bytes_to_pack+7 + lsr eax, eax, #3 // num_packed_words = (num_bytes_to_pack + 7) >> 3 + add rcx, tempQPosArray, rax, lsl #3 // endQPosArray = tempQPosArray + 2*num_source_words + lsl rax, rax, #2 + subs byte_count, byte_count, rax + b.lt L_budgetExhausted + + cmp rcx, next_qp // endQPosArray vs next_qp + b.ls 2f // if (next_qp >= endQPosArray) skip the following zero paddings + sub rax, rcx, next_qp + mov edx, #0 + tst eax, #4 + b.eq 1f + str edx, [next_qp], #4 +1: tst eax, #2 + b.eq 1f + strh edx, [next_qp], #2 +1: tst eax, #1 + b.eq 2f + strb edx, [next_qp], #1 +2: + mov rdi, next_full_patt // next_full_patt + cmp rcx, tempQPosArray // endQPosArray vs tempQPosArray + ldr eax, [dest_buf] + b.ls L20 // if (endQPosArray <= tempQPosArray) skip the following + mov rdx, tempQPosArray // tempQPosArray + + /* packing 4-bits dict indices into dest_buf */ +L_pack_4bits: + ldr rax, [rdx], #8 // src_next[1]:src_next[0] + orr rax, rax, rax, lsr #28 // eax = src_next[0] | (src_next[1] << 4) + cmp rcx, rdx // source_end vs src_next + str eax, [rdi], #4 // *dest_next++ = temp; + b.hi L_pack_4bits // while (src_next < source_end) repeat the loop + + // SET_LOW_BITS_AREA_START(dest_buf,boundary_tmp); + sub rax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in words +L20: + str eax, [dest_buf,#4] // dest_buf[1] = boundary_tmp - dest_buf + + + + /* --------------------------- packing 3 10-bits low bits into a 32-bit word in dest_buf[] ----------------------------------------- */ + + add rcx, scratch, #(2048*scale) // tempLowBitsArray + sub rdx, next_low_bits, rcx // next_low_bits - tempLowBitsArray (in bytes) + lsr rdx, rdx, #1 // num_tenbits_to_pack (in half-words) + subs edx, edx, #3 // pre-decrement num_tenbits_to_pack by 3 + b.lt 1f // if num_tenbits_to_pack < 3, skip the following loop +0: + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + subs edx, edx, #3 // num_tenbits_to_pack-=3 + ldr rax, [rcx], #6 + bfm rax, rax, #58, #9 // pack 1st toward 2nd + bfm rax, rax, #58, #25 // pack 1st/2nd toward 3rd + lsr rax, rax, #12 + str eax, [rdi], #4 // pack w0,w1,w2 into 1 dest_buf word + b.ge 0b // if no less than 3 elements, back to loop head + +1: adds edx, edx, #3 // post-increment num_tenbits_to_pack by 3 + b.eq 3f // if num_tenbits_to_pack is a multiple of 3, skip the following + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + ldrh eax,[rcx] // w0 + subs edx, edx, #1 // num_tenbits_to_pack-- + b.eq 2f // + ldrh edx, [rcx, #2] // w1 + orr eax, eax, edx, lsl #10 // w0 | (w1<<10) + +2: str eax, [rdi], #4 // write the final dest_buf word + +3: sub rax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in terms of words + str eax, [dest_buf, #8] // SET_LOW_BITS_AREA_END(dest_buf,boundary_tmp) + lsl w0, eax, #2 // boundary_tmp - dest_buf in terms of bytes + +L_done: + + // restore registers and return + mov x15, sp + ldp x20, x21, [x15, #0] // restore x20, x21 + ldp x22, x23, [x15, #16] // restore x22, x23 + ldp x24, x25, [x15, #32] // restore x24, x25 + ldp x26, x27, [x15, #48] // restore x26, x27 + add sp, sp, #128 // deallocate for dictionary + saved register space + +#if KERNEL + ld1.4s {v0,v1,v2,v3},[sp],#64 +#endif + ret lr + + .align 4 +L_budgetExhausted: + mov x0, #-1 + b L_done + + + .align 4,0x90 +L_RECORD_EXACT: +/* + we have an exact match of the input word to its corresponding dictionary word + write tag/dict_index to the temorary buffers +*/ + mov eax, #3 + lsr w14, wdict_location, #2 // divide by 4 for word offset + subs remaining, remaining, #1 // remaing--; + strb eax, [next_tag], #1 // *next_tag++ = 3 for exact + strb w14, [next_qp], #1 // *next_qp = word offset (4-bit) + b.gt L_loop + b CHECKPOINT // if remaining = 0, break + + .align 4,0x90 +L_RECORD_PARTIAL: +/* + we have a partial (high 22-bits) match of the input word to its corresponding dictionary word + write tag/dict_index/low 10 bits to the temorary buffers +*/ + mov ecx, #1 + strb ecx, [next_tag], #1 // *next_tag++ = 1 for partial matched + str edx, [dictionary, dict_location] // *dict_location = input_word; + subs remaining, remaining, #1 // remaing--; + lsr eax, wdict_location, #2 // offset in 32-bit word + and edx, edx, #1023 // lower 10 bits + strb eax, [next_qp], #1 // update *next_qp++ + strh edx, [next_low_bits], #2 // save next_low_bits++ + b.gt L_loop + +CHECKPOINT: + + cbz mode, L_check_compression_ratio // if this this an early abort check.. + +L_check_zero_page: + + cmp start_next_full_patt, next_full_patt // check if any dictionary misses in page + b.ne L_check_single_value_page + + cmp start_next_qp, next_qp // check if any partial or exact dictionary matches + b.ne L_check_single_value_page + + mov x0, #SV_RETURN // Magic return value + b L_done + +L_check_single_value_page: + + sub rax, next_full_patt, start_next_full_patt // get # dictionary misses + lsr rax, rax, #2 + + sub r11, next_qp, start_next_qp // get # dictionary hits (exact + partial) + + sub r13, next_low_bits, start_next_low_bits // get # dictionary partial hits + lsr r13, r13, #1 + + // Single value page if one of the follwoing is true: + // partial == 0 AND hits == 1023(for 4K page) AND miss == 1 AND tag[0] == 2 (i.e. miss) + // partial == 1 AND hits == 1024(for 4K page) AND tag[0] == 1 (i.e. partial) + // + cbnz r13, 1f // were there 0 partial hits? + + cmp r11, #(256*PAGES_SIZE_IN_KBYTES - 1) // were there 1023 dictionary hits + b.ne 1f + + cmp rax, #1 // was there exacly 1 dictionary miss? + b.ne 1f + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #2 // was the very 1st tag a miss? + b.eq L_is_single_value_page + +1: + cmp r13, #1 // was there 1 partial hit? + b.ne L_check_mostly_zero + + cmp r11, #(256*PAGES_SIZE_IN_KBYTES) // were there 1024 dictionary hits + b.ne L_check_mostly_zero + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #1 // was the very 1st tag a partial? + b.ne L_check_mostly_zero + +L_is_single_value_page: + + mov x0, #SV_RETURN // Magic return value + b L_done + +L_check_mostly_zero: + // how much space will the sparse packer take? + add rax, rax, r11 // rax += (next_qp - start_next_qp) + mov rdx, #6 + mov rcx, #4 + madd r11, rax, rdx, rcx // r11 = rax * 6 (i.e. 4 byte word + 2 byte offset) + 4 byte for header + + sub rax, next_low_bits, start_next_low_bits // get bytes consumed by lower-10 bits + mov rdx, #1365 + mul rax, rax, rdx + + sub rdx, next_full_patt, start_next_full_patt // get bytes consumed by dictionary misses + add rax, rdx, rax, lsr #11 // rax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + sub rdx, next_qp, start_next_qp + add rax, rax, rdx, lsr #1 // rax += (next_qp - start_next_qp)/2 + add rax, rax, #(12+256*scale) // rax += bytes taken by the header + tags + + cmp rax, r11 // is the default packer the better option? + b.lt L_done_search + + cmp r11, byte_budget // can the sparse packer fit into the given budget? + b.gt L_budgetExhausted + +L_sparse_packer: + mov edx, #MZV_MAGIC + str edx, [dest_buf], #4 // header to indicate a sparse packer + + mov rdx, #0 // rdx = byte offset in src of non-0 word +1: + ldr rax, [start_next_input_word, rdx] // rax = read dword + cbnz rax, 5f // is dword != 0 +3: + add rdx, rdx, #8 // 8 more bytes have been processed +4: + cmp rdx, #(4096*scale) // has the entire page been processed + b.ne 1b + mov x0, r11 // store the size of the compressed stream + b L_done + +5: + cbz eax, 6f // is lower word == 0 + str eax, [dest_buf], #4 // store the non-0 word in the dest buffer + strh edx, [dest_buf], #2 // store the byte index +6: + lsr rax, rax, 32 // get the upper word into position + cbz eax, 3b // is dword == 0 + add rdx, rdx, #4 + str eax, [dest_buf], #4 // store the non-0 word in the dest buffer + strh edx, [dest_buf], #2 // store the byte index + add rdx, rdx, #4 + b 4b + +L_check_compression_ratio: + + mov mode, NORMAL + mov remaining, #((1024 - CHKPT_WORDS)*scale) // remaining input words to process + cbz remaining, CHECKPOINT // if there are no remaining words to process + + sub rax, next_low_bits, start_next_low_bits // get bytes consumed by lower-10 bits + mov rdx, #1365 + mul rax, rax, rdx + + sub rdx, next_full_patt, start_next_full_patt // get bytes consumed by dictionary misses + add rax, rdx, rax, lsr #11 // rax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + sub rdx, next_qp, start_next_qp + add rax, rax, rdx, lsr #1 // rax += (next_qp - start_next_qp)/2 + subs rax, rax, #((CHKPT_SHRUNK_BYTES - CHKPT_TAG_BYTES)*scale) + // rax += CHKPT_TAG_BYTES; rax -= CHKPT_SHRUNK_BYTES + + b.gt L_budgetExhausted // if rax is > 0, we need to early abort + b L_loop // we are done diff --git a/osfmk/arm64/WKdmData.s b/osfmk/arm64/WKdmData.s new file mode 100644 index 000000000..2f2736bbe --- /dev/null +++ b/osfmk/arm64/WKdmData.s @@ -0,0 +1,330 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + + +.globl _hashLookupTable + .const + .align 5 +_hashLookupTable: + .byte 0 + .byte 52 + .byte 8 + .byte 56 + .byte 16 + .byte 12 + .byte 28 + .byte 20 + .byte 4 + .byte 36 + .byte 48 + .byte 24 + .byte 44 + .byte 40 + .byte 32 + .byte 60 + .byte 8 + .byte 12 + .byte 28 + .byte 20 + .byte 4 + .byte 60 + .byte 16 + .byte 36 + .byte 24 + .byte 48 + .byte 44 + .byte 32 + .byte 52 + .byte 56 + .byte 40 + .byte 12 + .byte 8 + .byte 48 + .byte 16 + .byte 52 + .byte 60 + .byte 28 + .byte 56 + .byte 32 + .byte 20 + .byte 24 + .byte 36 + .byte 40 + .byte 44 + .byte 4 + .byte 8 + .byte 40 + .byte 60 + .byte 32 + .byte 20 + .byte 44 + .byte 4 + .byte 36 + .byte 52 + .byte 24 + .byte 16 + .byte 56 + .byte 48 + .byte 12 + .byte 28 + .byte 16 + .byte 8 + .byte 40 + .byte 36 + .byte 28 + .byte 32 + .byte 12 + .byte 4 + .byte 44 + .byte 52 + .byte 20 + .byte 24 + .byte 48 + .byte 60 + .byte 56 + .byte 40 + .byte 48 + .byte 8 + .byte 32 + .byte 28 + .byte 36 + .byte 4 + .byte 44 + .byte 20 + .byte 56 + .byte 60 + .byte 24 + .byte 52 + .byte 16 + .byte 12 + .byte 12 + .byte 4 + .byte 48 + .byte 20 + .byte 8 + .byte 52 + .byte 16 + .byte 60 + .byte 24 + .byte 36 + .byte 44 + .byte 28 + .byte 56 + .byte 40 + .byte 32 + .byte 36 + .byte 20 + .byte 24 + .byte 60 + .byte 40 + .byte 44 + .byte 52 + .byte 16 + .byte 32 + .byte 4 + .byte 48 + .byte 8 + .byte 28 + .byte 56 + .byte 12 + .byte 28 + .byte 32 + .byte 40 + .byte 52 + .byte 36 + .byte 16 + .byte 20 + .byte 48 + .byte 8 + .byte 4 + .byte 60 + .byte 24 + .byte 56 + .byte 44 + .byte 12 + .byte 8 + .byte 36 + .byte 24 + .byte 28 + .byte 16 + .byte 60 + .byte 20 + .byte 56 + .byte 32 + .byte 40 + .byte 48 + .byte 12 + .byte 4 + .byte 44 + .byte 52 + .byte 44 + .byte 40 + .byte 12 + .byte 56 + .byte 8 + .byte 36 + .byte 24 + .byte 60 + .byte 28 + .byte 48 + .byte 4 + .byte 32 + .byte 20 + .byte 16 + .byte 52 + .byte 60 + .byte 12 + .byte 24 + .byte 36 + .byte 8 + .byte 4 + .byte 16 + .byte 56 + .byte 48 + .byte 44 + .byte 40 + .byte 52 + .byte 32 + .byte 20 + .byte 28 + .byte 32 + .byte 12 + .byte 36 + .byte 28 + .byte 24 + .byte 56 + .byte 40 + .byte 16 + .byte 52 + .byte 44 + .byte 4 + .byte 20 + .byte 60 + .byte 8 + .byte 48 + .byte 48 + .byte 52 + .byte 12 + .byte 20 + .byte 32 + .byte 44 + .byte 36 + .byte 28 + .byte 4 + .byte 40 + .byte 24 + .byte 8 + .byte 56 + .byte 60 + .byte 16 + .byte 36 + .byte 32 + .byte 8 + .byte 40 + .byte 4 + .byte 52 + .byte 24 + .byte 44 + .byte 20 + .byte 12 + .byte 28 + .byte 48 + .byte 56 + .byte 16 + .byte 60 + .byte 4 + .byte 52 + .byte 60 + .byte 48 + .byte 20 + .byte 16 + .byte 56 + .byte 44 + .byte 24 + .byte 8 + .byte 40 + .byte 12 + .byte 32 + .byte 28 + .byte 36 + .byte 24 + .byte 32 + .byte 12 + .byte 4 + .byte 20 + .byte 16 + .byte 60 + .byte 36 + .byte 28 + .byte 8 + .byte 52 + .byte 40 + .byte 48 + .byte 44 + .byte 56 + + .globl _table_2bits +_table_2bits: + .word 0 + .word -2 + .word -4 + .word -6 + .word 0x03030303 + .word 0x03030303 + .word 0x03030303 + .word 0x03030303 + + .globl _table_4bits +_table_4bits: + .word 0 + .word -4 + .word 0 + .word -4 + .word 0x0f0f0f0f + .word 0x0f0f0f0f + .word 0x0f0f0f0f + .word 0x0f0f0f0f + + .globl _table_10bits +_table_10bits: + .word 6 + .word 0 + .word 6 + .word 0 + .word 0 + .word -20 + .word 0 + .word -20 + .word (1023<<16) + .word 0 + .word (1023<<16) + .word 0 + .word 1023 + .word 1023 + .word 1023 + .word 1023 diff --git a/osfmk/arm64/WKdmDecompress_16k.s b/osfmk/arm64/WKdmDecompress_16k.s new file mode 100644 index 000000000..b3def749b --- /dev/null +++ b/osfmk/arm64/WKdmDecompress_16k.s @@ -0,0 +1,428 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains arm64 hand optimized implementation of WKdm memory page decompressor. + + void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word *scratch, __unused__ unsigned int words); + + input : + src_buf : address of input compressed data buffer + dest_buf : address of output decompressed buffer + scratch : an 8-k bytes scratch mempro provided by the caller + words : this argument is used by the mostly-zero-value decoder + + output : + + the input buffer is decompressed and the dest_buf is written with decompressed data. + + Am algorithm description of the WKdm compress and bit stream format can be found in the WKdm Compress arm64 assembly code WKdmCompress.s + + The bit stream (*src_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + where the header (of 3 words) specifies the ending boundaries (in 32-bit words) of the bit stream of c,d,e, respectively. + + The decompressor 1st unpacking the bit stream component b/d/e into temorary buffers. Then it sequentially decodes the decompressed word as follows + + for (i=0;i<1024;i++) { + tag = *next_tag++ + switch (tag) { + case 0 : *dest_buf++ = 0; break; + case 1 : dict_word = dictionary[*dict_index]; dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; break; + case 2 : x = *new_word++; k = (x>>10)&255; k = hashTable[k]; dictionary[k] = *dest_buf++ = x; break; + case 3 : *dest_buf++ = dictionary[*dict_index++]; break; + } + + cclee, Nov 9, '12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 +*/ + +#define PAGES_SIZE_IN_KBYTES 16 +#define MZV_MAGIC 17185 // magic value used to identify MZV page encoding + +#ifndef PAGES_SIZE_IN_KBYTES +#define PAGES_SIZE_IN_KBYTES 4 +#endif + +#if !((PAGES_SIZE_IN_KBYTES==4) || (PAGES_SIZE_IN_KBYTES==16)) +#error "Only PAGES_SIZE_IN_KBYTES = 4 or 16 is supported" +#endif + +#define scale (PAGES_SIZE_IN_KBYTES/4) + + + .align 4 + .text + +/* + void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes); +*/ + + .globl _WKdm_decompress_16k +_WKdm_decompress_16k: + + /* + -------- symbolizing registers -------- + the arm64 code was ported from x86_64 so we name some registers that are used as temp variables with x86_64 register names. + */ + + #define src_buf x0 + #define dest_buf x1 + #define scratch x2 + #define n_bytes x3 + #define dictionary sp + #define rax x13 + #define eax w13 + #define rbx x4 + #define ebx w4 + #define rcx x5 + #define ecx w5 + #define rdx x6 + #define edx w6 + #define tags_counter x7 + #define next_tag x12 + #define r8 x8 + #define r9 x9 + #define r10 x10 + #define r11 x11 + #define r12 x12 + + /* + + ------ scratch memory for local variables --------- + + [sp,#0] : dictionary + [scratch,#0] : tempTagsArray + [scratch,#1024] : tempQPosArray + [scratch,#2048] : tempLowBitsArray + + */ + +#if KERNEL + sub rax, sp, #96 + sub sp, sp, #96 + st1.4s {v0,v1,v2},[rax],#48 + st1.4s {v3,v4,v5},[rax],#48 +#endif + + sub sp, sp, #64 + + ldr eax, [src_buf] // read the 1st word from the header + mov ecx, #MZV_MAGIC + cmp eax, ecx // is the alternate packer used (i.e. is MZV page)? + b.ne L_default_decompressor // default decompressor was used + + // Mostly Zero Page Handling... + // { + add src_buf, src_buf, 4 // skip the header + mov rax, dest_buf + mov rcx, #(PAGES_SIZE_IN_KBYTES*1024) // number of bytes to zero out +1: + dc zva, rax // zero 64 bytes. since dest_buf is a page, it will be 4096 or 16384 byte aligned + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + subs rcx, rcx, #256 + b.ne 1b + + mov r12, #4 // current byte position in src to read from + mov rdx, #0 +2: + ldr eax, [src_buf], #4 // get the word + ldrh edx, [src_buf], #2 // get the index + str eax, [dest_buf, rdx] // store non-0 word in the destination buffer + add r12, r12, #6 // 6 more bytes processed + cmp r12, n_bytes // finished processing all the bytes? + b.ne 2b + b L_done + // } + +L_default_decompressor: + + /* + ---------------------- set up registers and PRELOAD_DICTIONARY --------------------------------- + */ + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO TO MIRROR THE COMPRESSOR + adrp rbx, _table_2bits@GOTPAGE + stp xzr, xzr, [dictionary, #0] + add r10, src_buf, #(12+256*scale) // TAGS_AREA_END + stp xzr, xzr, [dictionary, #16] + add rax, src_buf, #12 // TAGS_AREA_START + ldr rbx, [rbx, _table_2bits@GOTPAGEOFF] + stp xzr, xzr, [dictionary, #32] + mov rcx, scratch // tempTagsArray + stp xzr, xzr, [dictionary, #48] + ld1.4s {v0,v1},[rbx] + + + /* + ------------------------------ unpacking bit stream ---------------------------------- + */ + + // WK_unpack_2bits(TAGS_AREA_START(src_buf), TAGS_AREA_END(src_buf), tempTagsArray); +/* + unpacking 16 2-bit tags (from a 32-bit word) into 16 bytes + for arm64, this can be done by + 1. read the input 32-bit word into GPR w + 2. duplicate GPR into 4 elements in a vector register v0 + 3. ushl.4s vd, v0, vshift where vshift = {0, -2, -4, -6} + 4. and.4s vd, vd, vmask where vmask = 0x03030303030303030303030303030303 +*/ + +L_WK_unpack_2bits: + ldr q5, [rax], #16 // read 4 32-bit words for 64 2-bit tags + dup.4s v2, v5[0] // duplicate to 4 elements + dup.4s v3, v5[1] // duplicate to 4 elements + dup.4s v4, v5[2] // duplicate to 4 elements + dup.4s v5, v5[3] // duplicate to 4 elements + ushl.4s v2, v2, v0 // v1 = {0, -2, -4, -6} + ushl.4s v3, v3, v0 // v1 = {0, -2, -4, -6} + ushl.4s v4, v4, v0 // v1 = {0, -2, -4, -6} + ushl.4s v5, v5, v0 // v1 = {0, -2, -4, -6} + and.16b v2, v2, v1 // v2 = {3,3,...,3} + and.16b v3, v3, v1 // v2 = {3,3,...,3} + and.16b v4, v4, v1 // v2 = {3,3,...,3} + and.16b v5, v5, v1 // v2 = {3,3,...,3} + cmp r10, rax // TAGS_AREA_END vs TAGS_AREA_START + st1.4s {v2,v3,v4,v5}, [rcx], #64 // write 64 tags into tempTagsArray + b.hi L_WK_unpack_2bits // if not reach TAGS_AREA_END, repeat L_WK_unpack_2bits + + + // WK_unpack_4bits(QPOS_AREA_START(src_buf), QPOS_AREA_END(src_buf), tempQPosArray); + + ldp w8, w9, [src_buf] // WKdm header qpos start and end + adrp rbx, _table_4bits@GOTPAGE + subs x14, r9, r8 // x14 = (QPOS_AREA_END - QPOS_AREA_START)/4 + add r8, src_buf, r8, lsl #2 // QPOS_AREA_START + add r9, src_buf, r9, lsl #2 // QPOS_AREA_END + + b.ls 1f // if QPOS_AREA_END <= QPOS_AREA_START, skip L_WK_unpack_4bits + ldr rbx, [rbx, _table_4bits@GOTPAGEOFF] + add rcx, scratch, #(1024*scale) // tempQPosArray + ld1.4s {v0,v1},[rbx] + subs w14, w14, #1 + b.ls 2f // do loop of 2 only if w14 >= 5 +L_WK_unpack_4bits: + ldr d2, [r8], #8 // read a 32-bit word for 8 4-bit positions + subs w14, w14, #2 + zip1.4s v2, v2, v2 + ushl.4s v2, v2, v0 // v1 = {0, -4, 0, -4} + and.16b v2, v2, v1 // v2 = {15,15,...,15} + str q2, [rcx], #16 + b.hi L_WK_unpack_4bits +2: + adds w14, w14, #1 + b.le 1f + + ldr s3, [r8], #4 // read a 32-bit word for 8 4-bit positions + dup.2s v2, v3[0] // duplicate to 2 elements + ushl.2s v2, v2, v0 // v1 = {0, -4} + and.8b v2, v2, v1 // v2 = {15,15,...,15} + str d2, [rcx], #8 // write 16 tags into tempTagsArray + +1: + + // WK_unpack_3_tenbits(LOW_BITS_AREA_START(src_buf), LOW_BITS_AREA_END(src_buf), tempLowBitsArray); + + ldr eax, [src_buf,#8] // LOW_BITS_AREA_END offset + add r8, src_buf, rax, lsl #2 // LOW_BITS_AREA_END + add rcx, scratch, #(2048*scale) // tempLowBitsArray +#if (scale==1) + add r11, scratch, #(4096*scale-2) // final tenbits for the rare case +#else + add r11, scratch, #(4096*scale) // final tenbits for the rare case + sub r11, r11, #2 +#endif + subs r8, r8, r9 // LOW_BITS_AREA_START vs LOW_BITS_AREA_END + b.ls 1f // if START>=END, skip L_WK_unpack_3_tenbits + + adrp rbx, _table_10bits@GOTPAGE + ldr rbx, [rbx, _table_10bits@GOTPAGEOFF] + ld1.4s {v0,v1,v2,v3},[rbx] + + /* + a very rare case : 1024 tenbits, 1023 + 1 -> 341 + final 1 that is padded with 2 zeros + since the scratch memory is 4k (2k for this section), we need to pay attention to the last case + so we don't overwrite to the scratch memory + + we 1st do a single 3_tenbits, followed by 2x_3_tenbits loop, and detect whether the last 3_tenbits + hits the raee case + */ +#if 1 + subs r8, r8, #4 // pre-decrement by 8 + ldr s4, [r9], #4 // read 32-bit words for 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data + str d4, [rcx], #6 // write 3 low 10-bits + b.eq 1f +#endif + + subs r8, r8, #8 // pre-decrement by 8 + b.lt L_WK_unpack_3_tenbits + +L_WK_unpack_2x_3_tenbits: + ldr d4, [r9], #8 // read 2 32-bit words for a pair of 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data + ins v5.d[0], v4.d[1] + str d4, [rcx], #6 // write 3 low 10-bits + str d5, [rcx], #6 // write 3 low 10-bits + + subs r8, r8, #8 + b.ge L_WK_unpack_2x_3_tenbits // repeat loop if LOW_BITS_AREA_END > next_word + + tst r8, #4 + b.eq 1f + +L_WK_unpack_3_tenbits: + ldr s4, [r9] // read 32-bit words for 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data +#if 0 + str d4, [rcx] // write 3 low 10-bits +#else + cmp rcx, r11 + b.eq 2f + str d4, [rcx] // write 3 low 10-bits + b 1f +2: + str h4, [rcx] // write final 1 low 10-bits +#endif +1: + + /* + set up before going to the main decompress loop + */ + mov next_tag, scratch // tempTagsArray + add r8, scratch, #(1024*scale) // next_qpos + add r11, scratch, #(2048*scale) // tempLowBitsArray + adrp rbx, _hashLookupTable@GOTPAGE + mov tags_counter, #(1024*scale) // tag_area_end + ldr rbx, [rbx, _hashLookupTable@GOTPAGEOFF] + + b L_next + + .align 4,0x90 +L_ZERO_TAG: + /* + we can only get here if w9 = 0, meaning this is a zero tag + *dest_buf++ = 0; + */ + str w9, [dest_buf], #4 // *dest_buf++ = 0 + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.ls L_done // if next_tag >= tag_area_end, we're done + + /* WKdm decompress main loop */ +L_next: + ldrb w9, [next_tag], #1 // new tag + cbz w9, L_ZERO_TAG + cmp w9, #2 // partial match tag ? + b.eq L_MISS_TAG + b.gt L_EXACT_TAG + +L_PARTIAL_TAG: + /* + this is a partial match: + dict_word = dictionary[*dict_index]; + dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; + */ + + ldrb edx, [r8], #1 // qpos = *next_qpos++ + ldrh ecx, [r11], #2 // lower 10-bits from *next_low_bits++ + ldr eax, [dictionary, rdx, lsl #2] // read dictionary word + bfm eax, ecx, #0, #9 // pad the lower 10-bits from *next_low_bits + str eax, [dictionary,rdx,lsl #2] // *dict_location = newly formed word + str eax, [dest_buf], #4 // *dest_buf++ = newly formed word + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat loop until next_tag==tag_area_end + +L_done: + + // release stack memory, restore registers, and return + add sp, sp, #64 // deallocate for dictionary +#if KERNEL + ld1.4s {v0,v1,v2},[sp],#48 + ld1.4s {v3,v4,v5},[sp],#48 +#endif + ret lr + + .align 4,0x90 +L_MISS_TAG: + /* + this is a dictionary miss. + x = *new_word++; + k = (x>>10)&255; + k = hashTable[k]; + dictionary[k] = *dest_buf++ = x; + */ + ldr eax, [r10], #4 // w = *next_full_patt++ + ubfm edx, eax, #10, #17 // 8-bit hash table index + str eax, [dest_buf], #4 // *dest_buf++ = word + ldrb edx, [rbx, rdx] // qpos + str eax, [dictionary,rdx] // dictionary[qpos] = word + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + .align 4,0x90 +L_EXACT_TAG: + /* + this is an exact match; + *dest_buf++ = dictionary[*dict_index++]; + */ + ldrb eax, [r8], #1 // qpos = *next_qpos++ + ldr eax, [dictionary,rax,lsl #2] // w = dictionary[qpos] + str eax, [dest_buf], #4 // *dest_buf++ = w + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + diff --git a/osfmk/arm64/WKdmDecompress_4k.s b/osfmk/arm64/WKdmDecompress_4k.s new file mode 100644 index 000000000..1b399ea33 --- /dev/null +++ b/osfmk/arm64/WKdmDecompress_4k.s @@ -0,0 +1,428 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains arm64 hand optimized implementation of WKdm memory page decompressor. + + void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word *scratch, __unused__ unsigned int words); + + input : + src_buf : address of input compressed data buffer + dest_buf : address of output decompressed buffer + scratch : an 8-k bytes scratch mempro provided by the caller + words : this argument is not used in the implementation + (The 4th argument is, in fact, used by the Mostly Zero Value decoder) + + output : + + the input buffer is decompressed and the dest_buf is written with decompressed data. + + Am algorithm description of the WKdm compress and bit stream format can be found in the WKdm Compress arm64 assembly code WKdmCompress.s + + The bit stream (*src_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + where the header (of 3 words) specifies the ending boundaries (in 32-bit words) of the bit stream of c,d,e, respectively. + + The decompressor 1st unpacking the bit stream component b/d/e into temorary buffers. Then it sequentially decodes the decompressed word as follows + + for (i=0;i<1024;i++) { + tag = *next_tag++ + switch (tag) { + case 0 : *dest_buf++ = 0; break; + case 1 : dict_word = dictionary[*dict_index]; dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; break; + case 2 : x = *new_word++; k = (x>>10)&255; k = hashTable[k]; dictionary[k] = *dest_buf++ = x; break; + case 3 : *dest_buf++ = dictionary[*dict_index++]; break; + } + + cclee, Nov 9, '12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 +*/ + +#define MZV_MAGIC 17185 // magic value used to identify MZV page encoding + +#ifndef PAGES_SIZE_IN_KBYTES +#define PAGES_SIZE_IN_KBYTES 4 +#endif + +#if !((PAGES_SIZE_IN_KBYTES==4) || (PAGES_SIZE_IN_KBYTES==16)) +#error "Only PAGES_SIZE_IN_KBYTES = 4 or 16 is supported" +#endif + +#define scale (PAGES_SIZE_IN_KBYTES/4) + + + .align 4 + .text + +/* + void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes); +*/ + + .globl _WKdm_decompress_4k +_WKdm_decompress_4k: + + /* + -------- symbolizing registers -------- + the arm64 code was ported from x86_64 so we name some registers that are used as temp variables with x86_64 register names. + */ + + #define src_buf x0 + #define dest_buf x1 + #define scratch x2 + #define n_bytes x3 + #define dictionary sp + #define rax x13 + #define eax w13 + #define rbx x4 + #define ebx w4 + #define rcx x5 + #define ecx w5 + #define rdx x6 + #define edx w6 + #define tags_counter x7 + #define next_tag x12 + #define r8 x8 + #define r9 x9 + #define r10 x10 + #define r11 x11 + #define r12 x12 + + /* + + ------ scratch memory for local variables --------- + + [sp,#0] : dictionary + [scratch,#0] : tempTagsArray + [scratch,#1024] : tempQPosArray + [scratch,#2048] : tempLowBitsArray + + */ + +#if KERNEL + sub rax, sp, #96 + sub sp, sp, #96 + st1.4s {v0,v1,v2},[rax],#48 + st1.4s {v3,v4,v5},[rax],#48 +#endif + + sub sp, sp, #64 + + ldr eax, [src_buf] // read the 1st word from the header + mov ecx, #MZV_MAGIC + cmp eax, ecx // is the alternate packer used (i.e. is MZV page)? + b.ne L_default_decompressor // default decompressor was used + + // Mostly Zero Page Handling... + // { + add src_buf, src_buf, 4 // skip the header + mov rax, dest_buf + mov rcx, #(PAGES_SIZE_IN_KBYTES*1024) // number of bytes to zero out +1: + dc zva, rax // zero 64 bytes. since dest_buf is a page, it will be 4096 or 16384 byte aligned + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + subs rcx, rcx, #256 + b.ne 1b + + mov r12, #4 // current byte position in src to read from + mov rdx, #0 +2: + ldr eax, [src_buf], #4 // get the word + ldrh edx, [src_buf], #2 // get the index + str eax, [dest_buf, rdx] // store non-0 word in the destination buffer + add r12, r12, #6 // 6 more bytes processed + cmp r12, n_bytes // finished processing all the bytes? + b.ne 2b + b L_done + // } + +L_default_decompressor: + + /* + ---------------------- set up registers and PRELOAD_DICTIONARY --------------------------------- + */ + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO TO MIRROR THE COMPRESSOR + adrp rbx, _table_2bits@GOTPAGE + stp xzr, xzr, [dictionary, #0] + add r10, src_buf, #(12+256*scale) // TAGS_AREA_END + stp xzr, xzr, [dictionary, #16] + add rax, src_buf, #12 // TAGS_AREA_START + ldr rbx, [rbx, _table_2bits@GOTPAGEOFF] + stp xzr, xzr, [dictionary, #32] + mov rcx, scratch // tempTagsArray + stp xzr, xzr, [dictionary, #48] + ld1.4s {v0,v1},[rbx] + + + /* + ------------------------------ unpacking bit stream ---------------------------------- + */ + + // WK_unpack_2bits(TAGS_AREA_START(src_buf), TAGS_AREA_END(src_buf), tempTagsArray); +/* + unpacking 16 2-bit tags (from a 32-bit word) into 16 bytes + for arm64, this can be done by + 1. read the input 32-bit word into GPR w + 2. duplicate GPR into 4 elements in a vector register v0 + 3. ushl.4s vd, v0, vshift where vshift = {0, -2, -4, -6} + 4. and.4s vd, vd, vmask where vmask = 0x03030303030303030303030303030303 +*/ + +L_WK_unpack_2bits: + ldr q5, [rax], #16 // read 4 32-bit words for 64 2-bit tags + dup.4s v2, v5[0] // duplicate to 4 elements + dup.4s v3, v5[1] // duplicate to 4 elements + dup.4s v4, v5[2] // duplicate to 4 elements + dup.4s v5, v5[3] // duplicate to 4 elements + ushl.4s v2, v2, v0 // v1 = {0, -2, -4, -6} + ushl.4s v3, v3, v0 // v1 = {0, -2, -4, -6} + ushl.4s v4, v4, v0 // v1 = {0, -2, -4, -6} + ushl.4s v5, v5, v0 // v1 = {0, -2, -4, -6} + and.16b v2, v2, v1 // v2 = {3,3,...,3} + and.16b v3, v3, v1 // v2 = {3,3,...,3} + and.16b v4, v4, v1 // v2 = {3,3,...,3} + and.16b v5, v5, v1 // v2 = {3,3,...,3} + cmp r10, rax // TAGS_AREA_END vs TAGS_AREA_START + st1.4s {v2,v3,v4,v5}, [rcx], #64 // write 64 tags into tempTagsArray + b.hi L_WK_unpack_2bits // if not reach TAGS_AREA_END, repeat L_WK_unpack_2bits + + + // WK_unpack_4bits(QPOS_AREA_START(src_buf), QPOS_AREA_END(src_buf), tempQPosArray); + + ldp w8, w9, [src_buf] // WKdm header qpos start and end + adrp rbx, _table_4bits@GOTPAGE + subs x14, r9, r8 // x14 = (QPOS_AREA_END - QPOS_AREA_START)/4 + add r8, src_buf, r8, lsl #2 // QPOS_AREA_START + add r9, src_buf, r9, lsl #2 // QPOS_AREA_END + + b.ls 1f // if QPOS_AREA_END <= QPOS_AREA_START, skip L_WK_unpack_4bits + ldr rbx, [rbx, _table_4bits@GOTPAGEOFF] + add rcx, scratch, #(1024*scale) // tempQPosArray + ld1.4s {v0,v1},[rbx] + subs w14, w14, #1 + b.ls 2f // do loop of 2 only if w14 >= 5 +L_WK_unpack_4bits: + ldr d2, [r8], #8 // read a 32-bit word for 8 4-bit positions + subs w14, w14, #2 + zip1.4s v2, v2, v2 + ushl.4s v2, v2, v0 // v1 = {0, -4, 0, -4} + and.16b v2, v2, v1 // v2 = {15,15,...,15} + str q2, [rcx], #16 + b.hi L_WK_unpack_4bits +2: + adds w14, w14, #1 + b.le 1f + + ldr s3, [r8], #4 // read a 32-bit word for 8 4-bit positions + dup.2s v2, v3[0] // duplicate to 2 elements + ushl.2s v2, v2, v0 // v1 = {0, -4} + and.8b v2, v2, v1 // v2 = {15,15,...,15} + str d2, [rcx], #8 // write 16 tags into tempTagsArray + +1: + + // WK_unpack_3_tenbits(LOW_BITS_AREA_START(src_buf), LOW_BITS_AREA_END(src_buf), tempLowBitsArray); + + ldr eax, [src_buf,#8] // LOW_BITS_AREA_END offset + add r8, src_buf, rax, lsl #2 // LOW_BITS_AREA_END + add rcx, scratch, #(2048*scale) // tempLowBitsArray +#if (scale==1) + add r11, scratch, #(4096*scale-2) // final tenbits for the rare case +#else + add r11, scratch, #(4096*scale) // final tenbits for the rare case + sub r11, r11, #2 +#endif + subs r8, r8, r9 // LOW_BITS_AREA_START vs LOW_BITS_AREA_END + b.ls 1f // if START>=END, skip L_WK_unpack_3_tenbits + + adrp rbx, _table_10bits@GOTPAGE + ldr rbx, [rbx, _table_10bits@GOTPAGEOFF] + ld1.4s {v0,v1,v2,v3},[rbx] + + /* + a very rare case : 1024 tenbits, 1023 + 1 -> 341 + final 1 that is padded with 2 zeros + since the scratch memory is 4k (2k for this section), we need to pay attention to the last case + so we don't overwrite to the scratch memory + + we 1st do a single 3_tenbits, followed by 2x_3_tenbits loop, and detect whether the last 3_tenbits + hits the raee case + */ +#if 1 + subs r8, r8, #4 // pre-decrement by 8 + ldr s4, [r9], #4 // read 32-bit words for 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data + str d4, [rcx], #6 // write 3 low 10-bits + b.eq 1f +#endif + + subs r8, r8, #8 // pre-decrement by 8 + b.lt L_WK_unpack_3_tenbits + +L_WK_unpack_2x_3_tenbits: + ldr d4, [r9], #8 // read 2 32-bit words for a pair of 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data + ins v5.d[0], v4.d[1] + str d4, [rcx], #6 // write 3 low 10-bits + str d5, [rcx], #6 // write 3 low 10-bits + + subs r8, r8, #8 + b.ge L_WK_unpack_2x_3_tenbits // repeat loop if LOW_BITS_AREA_END > next_word + + tst r8, #4 + b.eq 1f + +L_WK_unpack_3_tenbits: + ldr s4, [r9] // read 32-bit words for 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data +#if 0 + str d4, [rcx] // write 3 low 10-bits +#else + cmp rcx, r11 + b.eq 2f + str d4, [rcx] // write 3 low 10-bits + b 1f +2: + str h4, [rcx] // write final 1 low 10-bits +#endif +1: + + /* + set up before going to the main decompress loop + */ + mov next_tag, scratch // tempTagsArray + add r8, scratch, #(1024*scale) // next_qpos + add r11, scratch, #(2048*scale) // tempLowBitsArray + adrp rbx, _hashLookupTable@GOTPAGE + mov tags_counter, #(1024*scale) // tag_area_end + ldr rbx, [rbx, _hashLookupTable@GOTPAGEOFF] + + b L_next + + .align 4,0x90 +L_ZERO_TAG: + /* + we can only get here if w9 = 0, meaning this is a zero tag + *dest_buf++ = 0; + */ + str w9, [dest_buf], #4 // *dest_buf++ = 0 + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.ls L_done // if next_tag >= tag_area_end, we're done + + /* WKdm decompress main loop */ +L_next: + ldrb w9, [next_tag], #1 // new tag + cbz w9, L_ZERO_TAG + cmp w9, #2 // partial match tag ? + b.eq L_MISS_TAG + b.gt L_EXACT_TAG + +L_PARTIAL_TAG: + /* + this is a partial match: + dict_word = dictionary[*dict_index]; + dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; + */ + + ldrb edx, [r8], #1 // qpos = *next_qpos++ + ldrh ecx, [r11], #2 // lower 10-bits from *next_low_bits++ + ldr eax, [dictionary, rdx, lsl #2] // read dictionary word + bfm eax, ecx, #0, #9 // pad the lower 10-bits from *next_low_bits + str eax, [dictionary,rdx,lsl #2] // *dict_location = newly formed word + str eax, [dest_buf], #4 // *dest_buf++ = newly formed word + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat loop until next_tag==tag_area_end + +L_done: + + // release stack memory, restore registers, and return + add sp, sp, #64 // deallocate for dictionary +#if KERNEL + ld1.4s {v0,v1,v2},[sp],#48 + ld1.4s {v3,v4,v5},[sp],#48 +#endif + ret lr + + .align 4,0x90 +L_MISS_TAG: + /* + this is a dictionary miss. + x = *new_word++; + k = (x>>10)&255; + k = hashTable[k]; + dictionary[k] = *dest_buf++ = x; + */ + ldr eax, [r10], #4 // w = *next_full_patt++ + ubfm edx, eax, #10, #17 // 8-bit hash table index + str eax, [dest_buf], #4 // *dest_buf++ = word + ldrb edx, [rbx, rdx] // qpos + str eax, [dictionary,rdx] // dictionary[qpos] = word + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + .align 4,0x90 +L_EXACT_TAG: + /* + this is an exact match; + *dest_buf++ = dictionary[*dict_index++]; + */ + ldrb eax, [r8], #1 // qpos = *next_qpos++ + ldr eax, [dictionary,rax,lsl #2] // w = dictionary[qpos] + str eax, [dest_buf], #4 // *dest_buf++ = w + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + diff --git a/osfmk/arm64/alternate_debugger.c b/osfmk/arm64/alternate_debugger.c new file mode 100644 index 000000000..0fa80fe36 --- /dev/null +++ b/osfmk/arm64/alternate_debugger.c @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#if ALTERNATE_DEBUGGER + +/* + +The alternate debugger feature is enabled by setting the boot arg "alternate_debugger_init" +to the size of memory that should be set aside for the debugger. The boot arg +"alternate_debugger_init_pages" is used to allocate more vmpages that the alternate debugger +may use to do additional VA->PA mappings. The boot-arg "alternate_debugger_pause_for_load_at_boot" +will halt the system so that the debugger can be loaded early in the boot cycle -- once the +alternate debugger code is loaded, a register must be set to a 1 to continue the boot process. + +Here's an example: +nvram boot-arg="alternate_debugger_init=0x800000 alternate_debugger_init_pages=0x8000 alternate_debugger_pause_for_load_at_boot=1" + +The low memory global lgAltDebugger will contain the address of the allocated memory for +the alternate debugger. On arm64, the address of this low memory global is 0xffffff8000002048. + +At any point after the low memory global is non-zero, Astris may be used to halt the cpu +and load the alternate debugger: + +If no alternate debugger is given, but alternate_debugger_init has been specified, and the +kernel debugger is entered, the string ">MT<" is printed and normal processing continues. + +Anytime the alternate debugger is entered, the osversion string is modified to start with "ALT" +so that panic reports can clearly indicated that some kernel poking may have occurred, and +the panic should be weighted accordingly. + +*/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void kprintf(const char *fmt, ...); + + +static mach_vm_address_t alt_code; +static mach_vm_size_t alt_size; +static mach_vm_address_t alt_pages; +static mach_vm_size_t alt_pages_size; + +typedef void (*t_putc_fn)(char c); +typedef void (*t_call_altdbg_fn)(mach_vm_size_t size, mach_vm_address_t pages, mach_vm_size_t pages_size, t_putc_fn putc_address ); + +// used as a temporary alternate debugger until another is loaded +extern void alternate_debugger_just_return(__unused mach_vm_size_t size, __unused mach_vm_address_t pages, __unused mach_vm_size_t pages_size, t_putc_fn putc_address); +extern void *alternate_debugger_just_return_end; + +// public entry to the alternate debugger +void alternate_debugger_enter(void) +{ + if ( alt_code != 0 ) { + disable_preemption(); + + printf("########## Going to call ALTERNATE DEBUGGER\n"); + + // make sure it isn't in the cache + assert((alt_size & 0xFFFFFFFF00000000) == 0); + flush_dcache(alt_code, (unsigned int)alt_size, 0); + + // set the code to execute + pmap_protect(kernel_map->pmap, alt_code, alt_code+alt_size, VM_PROT_READ|VM_PROT_EXECUTE); + + // black-spot the OS version for any panic reports that occur because of entering the alternate debugger + if ( *osversion ) { + memcpy(osversion, "ALT", 3); // Version set, stomp on the begining of it + } else { + strncpy(osversion, "ALT - Version Not Set Yet", OSVERSIZE); + } + + kprintf("########## Calling ALTERNATE DEBUGGER (size %lld, pages 0x%llx, pages_size 0x%llx, putc %p\n", alt_size, alt_pages, alt_pages_size, &consdebug_putc_unbuffered); + ((t_call_altdbg_fn)alt_code)(alt_size, alt_pages, alt_pages_size, &consdebug_putc_unbuffered); + kprintf("########## Returned from calling ALTERNATE DEBUGGER\n"); + + enable_preemption(); + } +} + +// public entry to check boot args and init accordingly +void alternate_debugger_init(void) +{ + // use the alternate debugger + if( PE_parse_boot_argn("alternate_debugger_init", (void*)&alt_size, sizeof(alt_size)) ) + { + vm_offset_t alt_va = 0; + + kprintf("########## ALTERNATE_DEBUGGER\n"); + + PE_parse_boot_argn("alternate_debugger_init_pages", (void*)&alt_pages_size, sizeof(alt_pages_size)); + + alt_size = vm_map_round_page(alt_size, + VM_MAP_PAGE_MASK(kernel_map)); + alt_pages_size = vm_map_round_page(alt_pages_size, + VM_MAP_PAGE_MASK(kernel_map)); + + kern_return_t kr = KERN_SUCCESS; + kr = kmem_alloc_contig(kernel_map, &alt_va, alt_size, VM_MAP_PAGE_MASK(kernel_map), 0, 0, KMA_NOPAGEWAIT | KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_DIAG); + if( kr != KERN_SUCCESS) + { + kprintf("########## ALTERNATE_DEBUGGER FAILED kmem_alloc_contig with %d\n", kr); + alt_va = 0; + } + else { + if ( alt_pages_size ) { + alt_pages = (vm_offset_t) kalloc((vm_size_t) alt_pages_size); + } + } + + kprintf("########## Initializing ALTERNATE DEBUGGER : [alloc size 0x%llx @0x%lx] [pages_size 0x%llx @0x%llx] -- lowmem pointer at %p\n", + alt_size, alt_va, alt_pages_size, alt_pages, &lowGlo.lgAltDebugger ); + + if ( alt_va ) { + uintptr_t just_return_size = (uintptr_t)&alternate_debugger_just_return_end - (uintptr_t)&alternate_debugger_just_return; + assert(just_return_size <= alt_size); // alt_size is page-rounded, just_return_size should be much less than a page. + // install a simple return vector + memcpy((void*)alt_va, &alternate_debugger_just_return, just_return_size); + + // code is ready, enable the pointers to it + lowGlo.lgAltDebugger = alt_code = alt_va; + +#if 1 + // DEBUG for BRING-UP testing + unsigned int alt_init_test; + if(PE_parse_boot_argn("alternate_debugger_pause_for_load_at_boot", &alt_init_test, sizeof(alt_init_test)) ) { + + // debug!! + kprintf("########## Waiting for ALTERNATE DEBUGGER to load (in file %s).... to continue, set register to 1", __FILE__ ); + volatile int ii = 0; + while(!ii) + ; + kprintf("\n"); + alternate_debugger_enter(); + } +#endif + } + } +} + +#endif /* ALTERNATE_DEBUGGER */ diff --git a/osfmk/arm64/alternate_debugger.h b/osfmk/arm64/alternate_debugger.h new file mode 100644 index 000000000..22be4c09e --- /dev/null +++ b/osfmk/arm64/alternate_debugger.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ALTERNATE_DEBUGGER_H_ +#define _ALTERNATE_DEBUGGER_H_ + +#if ALTERNATE_DEBUGGER + +#include + +__BEGIN_DECLS + +void alternate_debugger_init(void); +void alternate_debugger_enter(void); + +__END_DECLS + +#endif /* ALTERNATE_DEBUGGER */ + +#endif /* _ALTERNATE_DEBUGGER_H_ */ + diff --git a/osfmk/arm64/alternate_debugger_asm.s b/osfmk/arm64/alternate_debugger_asm.s new file mode 100644 index 000000000..97f381063 --- /dev/null +++ b/osfmk/arm64/alternate_debugger_asm.s @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include + +#if ALTERNATE_DEBUGGER + .text +/* void alternate_debugger_just_return(__unused mach_vm_size_t size, __unused mach_vm_address_t pages, __unused mach_vm_size_t pages_size, t_putc_fn putc_address) */ + .align 2 + .globl EXT(alternate_debugger_just_return) +LEXT(alternate_debugger_just_return) + sub sp, sp, #0x20 + stp x29, x30, [sp, #0x10] + add x29, sp, #0x10 + str x3, [sp, #0x8] + mov w0, #0xa + mov x1, x3 + blr x1 // (*putc_address)('\n'); + orr w0, wzr, #0x3e + ldr x1, [sp, #0x8] + blr x1 // (*putc_address)('>'); + mov w0, #0x4d + ldr x1, [sp, #0x8] + blr x1 // (*putc_address)('M'); + mov w0, #0x54 + ldr x1, [sp, #0x8] + blr x1 // (*putc_address)('T'); + orr w0, wzr, #0x3c + ldr x1, [sp, #0x8] + blr x1 // (*putc_address)('<'); + mov w0, #0xa + ldr x1, [sp, #0x8] + ldp x29, x30, [sp, #0x10] + add sp, sp, #0x20 + br x1 // (*putc_address)('\n'); + .align 2 + .globl EXT(alternate_debugger_just_return_end) +LEXT(alternate_debugger_just_return_end) + +#endif /* ALTERNATE_DEBUGGER */ diff --git a/osfmk/arm64/arm_vm_init.c b/osfmk/arm64/arm_vm_init.c new file mode 100644 index 000000000..dc0cb7430 --- /dev/null +++ b/osfmk/arm64/arm_vm_init.c @@ -0,0 +1,1203 @@ +/* + * Copyright (c) 2007-2011 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#if KASAN +extern vm_offset_t shadow_pbase; +extern vm_offset_t shadow_ptop; +extern vm_offset_t physmap_vbase; +extern vm_offset_t physmap_vtop; +#endif + +/* + * Denotes the end of xnu. + */ +extern void *last_kernel_symbol; + +/* + * KASLR parameters + */ +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_base; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_top; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kext_base; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kext_top; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_stext; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_etext; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slide; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slid_base; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slid_top; + +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_stext; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_etext; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_sdata; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_edata; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_sinfo; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_einfo; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_slinkedit; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_elinkedit; + +/* Used by */ +SECURITY_READ_ONLY_LATE(unsigned long) gVirtBase; +SECURITY_READ_ONLY_LATE(unsigned long) gPhysBase; +SECURITY_READ_ONLY_LATE(unsigned long) gPhysSize; + + +/* + * NOTE: mem_size is bogus on large memory machines. + * We will pin it to 0x80000000 if there is more than 2 GB + * This is left only for compatibility and max_mem should be used. + */ +vm_offset_t mem_size; /* Size of actual physical memory present + * minus any performance buffer and possibly + * limited by mem_limit in bytes */ +uint64_t mem_actual; /* The "One True" physical memory size + * actually, it's the highest physical + * address + 1 */ +uint64_t max_mem; /* Size of physical memory (bytes), adjusted + * by maxmem */ +uint64_t sane_size; /* Memory size to use for defaults + * calculations */ +/* This no longer appears to be used; kill it? */ +addr64_t vm_last_addr = VM_MAX_KERNEL_ADDRESS; /* Highest kernel + * virtual address known + * to the VM system */ + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segTEXTB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeTEXT; + + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segDATACONSTB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeDATACONST; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segTEXTEXECB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeTEXTEXEC; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segDATAB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeDATA; + + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segLINKB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeLINK; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segKLDB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKLD; +SECURITY_READ_ONLY_LATE(static vm_offset_t) segLASTB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeLAST; + +SECURITY_READ_ONLY_LATE(vm_offset_t) segPRELINKTEXTB; +SECURITY_READ_ONLY_LATE(unsigned long) segSizePRELINKTEXT; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKTEXTEXECB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKTEXTEXEC; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKDATACONSTB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKDATACONST; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPRELINKDATAB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePRELINKDATA; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKLLVMCOVB = 0; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKLLVMCOV = 0; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKLINKEDITB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKLINKEDIT; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPRELINKINFOB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePRELINKINFO; + +SECURITY_READ_ONLY_LATE(static boolean_t) use_contiguous_hint = TRUE; + +SECURITY_READ_ONLY_LATE(unsigned) PAGE_SHIFT_CONST; + +SECURITY_READ_ONLY_LATE(vm_offset_t) end_kern; +SECURITY_READ_ONLY_LATE(vm_offset_t) etext; +SECURITY_READ_ONLY_LATE(vm_offset_t) sdata; +SECURITY_READ_ONLY_LATE(vm_offset_t) edata; + +vm_offset_t alloc_ptpage(boolean_t map_static); +SECURITY_READ_ONLY_LATE(vm_offset_t) ropage_next; + +/* + * Bootstrap the system enough to run with virtual memory. + * Map the kernel's code and data, and allocate the system page table. + * Page_size must already be set. + * + * Parameters: + * first_avail: first available physical page - + * after kernel page tables + * avail_start: PA of first physical page + * avail_end: PA of last physical page + */ +SECURITY_READ_ONLY_LATE(vm_offset_t) first_avail; +SECURITY_READ_ONLY_LATE(vm_offset_t) static_memory_end; +SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_start; +SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_end; + +#define MEM_SIZE_MAX 0x100000000ULL + +#if defined(KERNEL_INTEGRITY_KTRR) +#if __ARM64_TWO_LEVEL_PMAP__ +/* We could support this configuration, but it adds memory overhead. */ +#error This configuration is not supported +#endif +#endif + +/* + * This rounds the given address up to the nearest boundary for a PTE contiguous + * hint. + */ +static vm_offset_t +round_up_pte_hint_address(vm_offset_t address) +{ + vm_offset_t hint_size = ARM_PTE_SIZE << ARM_PTE_HINT_ENTRIES_SHIFT; + return ((address + (hint_size - 1)) & ~(hint_size - 1)); +} + +/* allocate a page for a page table: we support static and dynamic mappings. + * + * returns a virtual address for the allocated page + * + * for static mappings, we allocate from the region ropagetable_begin to ro_pagetable_end-1, + * which is defined in the DATA_CONST segment and will be protected RNX when vm_prot_finalize runs. + * + * for dynamic mappings, we allocate from avail_start, which should remain RWNX. + */ + +vm_offset_t alloc_ptpage(boolean_t map_static) { + vm_offset_t vaddr; + +#if !(defined(KERNEL_INTEGRITY_KTRR)) + map_static = FALSE; +#endif + + if (!ropage_next) { + ropage_next = (vm_offset_t)&ropagetable_begin; + } + + if (map_static) { + assert(ropage_next < (vm_offset_t)&ropagetable_end); + + vaddr = ropage_next; + ropage_next += ARM_PGBYTES; + + return vaddr; + } else { + vaddr = phystokv(avail_start); + avail_start += ARM_PGBYTES; + + return vaddr; + } +} + +#if DEBUG + +void dump_kva_l2(vm_offset_t tt_base, tt_entry_t *tt, int indent, uint64_t *rosz_out, uint64_t *rwsz_out); + +void dump_kva_l2(vm_offset_t tt_base, tt_entry_t *tt, int indent, uint64_t *rosz_out, uint64_t *rwsz_out) { + unsigned int i; + boolean_t cur_ro, prev_ro = 0; + int start_entry = -1; + tt_entry_t cur, prev = 0; + pmap_paddr_t robegin = kvtophys((vm_offset_t)&ropagetable_begin); + pmap_paddr_t roend = kvtophys((vm_offset_t)&ropagetable_end); + boolean_t tt_static = kvtophys((vm_offset_t)tt) >= robegin && + kvtophys((vm_offset_t)tt) < roend; + + for(i=0; i= robegin && cur < roend); + } else { + cur_ro = 0; + } + + if ((cur == 0 && prev != 0) || (cur_ro != prev_ro && prev != 0)) { // falling edge + uintptr_t start,end,sz; + + start = (uintptr_t)start_entry << ARM_TT_L2_SHIFT; + start += tt_base; + end = ((uintptr_t)i << ARM_TT_L2_SHIFT) - 1; + end += tt_base; + + sz = end - start + 1; + printf("%*s0x%08x_%08x-0x%08x_%08x %s (%luMB)\n", + indent*4, "", + (uint32_t)(start >> 32),(uint32_t)start, + (uint32_t)(end >> 32),(uint32_t)end, + prev_ro ? "Static " : "Dynamic", + (sz >> 20)); + + if (prev_ro) { + *rosz_out += sz; + } else { + *rwsz_out += sz; + } + } + + if ((prev == 0 && cur != 0) || cur_ro != prev_ro) { // rising edge: set start + start_entry = i; + } + + prev = cur; + prev_ro = cur_ro; + } +} + +void dump_kva_space() { + uint64_t tot_rosz=0, tot_rwsz=0; + int ro_ptpages, rw_ptpages; + pmap_paddr_t robegin = kvtophys((vm_offset_t)&ropagetable_begin); + pmap_paddr_t roend = kvtophys((vm_offset_t)&ropagetable_end); + boolean_t root_static = kvtophys((vm_offset_t)cpu_tte) >= robegin && + kvtophys((vm_offset_t)cpu_tte) < roend; + uint64_t kva_base = ~((1ULL << (64 - T1SZ_BOOT)) - 1); + + printf("Root page table: %s\n", root_static ? "Static" : "Dynamic"); + +#if !__ARM64_TWO_LEVEL_PMAP__ + for(unsigned int i=0; i= robegin && cur < roend; + + printf("0x%08x_%08x-0x%08x_%08x %s\n", + (uint32_t)(start >> 32),(uint32_t)start, + (uint32_t)(end >> 32),(uint32_t)end, + cur_ro ? "Static " : "Dynamic"); + + dump_kva_l2(start, (tt_entry_t*)phystokv(cur), 1, &rosz, &rwsz); + tot_rosz += rosz; + tot_rwsz += rwsz; + } +#else + dump_kva_l2(kva_base, cpu_tte, 0, &tot_rosz, &tot_rwsz); +#endif /* !_ARM64_TWO_LEVEL_PMAP__ */ + + printf("L2 Address space mapped: Static %lluMB Dynamic %lluMB Total %lluMB\n", + tot_rosz >> 20, + tot_rwsz >> 20, + (tot_rosz >> 20) + (tot_rwsz >> 20)); + + ro_ptpages = (int)((ropage_next - (vm_offset_t)&ropagetable_begin) >> ARM_PGSHIFT); + rw_ptpages = (int)(lowGlo.lgStaticSize >> ARM_PGSHIFT); + printf("Pages used: static %d dynamic %d\n", ro_ptpages, rw_ptpages); +} + +#endif /* DEBUG */ + +#if defined(KERNEL_INTEGRITY_KTRR) +extern void bootstrap_instructions; + +/* + * arm_replace_identity_map takes the V=P map that we construct in start.s + * and repurposes it in order to have it map only the page we need in order + * to turn on the MMU. This prevents us from running into issues where + * KTRR will cause us to fault on executable block mappings that cross the + * KTRR boundary. + */ +static void arm_replace_identity_map(boot_args * args) +{ + vm_offset_t addr; + pmap_paddr_t paddr; + +#if !__ARM64_TWO_LEVEL_PMAP__ + pmap_paddr_t l1_ptp_phys = 0; + tt_entry_t *l1_ptp_virt = NULL; + tt_entry_t *tte1 = NULL; +#endif + pmap_paddr_t l2_ptp_phys = 0; + tt_entry_t *l2_ptp_virt = NULL; + tt_entry_t *tte2 = NULL; + pmap_paddr_t l3_ptp_phys = 0; + pt_entry_t *l3_ptp_virt = NULL; + pt_entry_t *ptep = NULL; + + addr = ((vm_offset_t)&bootstrap_instructions) & ~ARM_PGMASK; + paddr = kvtophys(addr); + + /* + * The V=P page tables (at the time this comment was written) start + * after the last bit of kernel data, and consist of 1 to 2 pages. + * Grab references to those pages, and allocate an L3 page. + */ +#if !__ARM64_TWO_LEVEL_PMAP__ + l1_ptp_phys = args->topOfKernelData; + l1_ptp_virt = (tt_entry_t *)phystokv(l1_ptp_phys); + tte1 = &l1_ptp_virt[(((paddr) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)]; + + l2_ptp_phys = l1_ptp_phys + ARM_PGBYTES; +#else + l2_ptp_phys = args->topOfKernelData; +#endif + l2_ptp_virt = (tt_entry_t *)phystokv(l2_ptp_phys); + tte2 = &l2_ptp_virt[(((paddr) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; + + l3_ptp_virt = (pt_entry_t *)alloc_ptpage(FALSE); + l3_ptp_phys = kvtophys((vm_offset_t)l3_ptp_virt); + ptep = &l3_ptp_virt[(((paddr) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)]; + + /* + * Replace the large V=P mapping with a mapping that provides only the + * mappings needed to turn on the MMU. + */ +#if !__ARM64_TWO_LEVEL_PMAP__ + bzero(l1_ptp_virt, ARM_PGBYTES); + *tte1 = ARM_TTE_BOOT_TABLE | (l2_ptp_phys & ARM_TTE_TABLE_MASK); +#endif + bzero(l2_ptp_virt, ARM_PGBYTES); + *tte2 = ARM_TTE_BOOT_TABLE | (l3_ptp_phys & ARM_TTE_TABLE_MASK); + + *ptep = (paddr & ARM_PTE_MASK) | + ARM_PTE_TYPE_VALID | + ARM_PTE_SH(SH_OUTER_MEMORY) | + ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | + ARM_PTE_AF | + ARM_PTE_AP(AP_RONA) | + ARM_PTE_NX; +} +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + +/* + * arm_vm_page_granular_helper updates protections at the L3 level. It will (if + * neccessary) allocate a page for the L3 table and update the corresponding L2 + * entry. Then, it will iterate over the L3 table, updating protections as necessary. + * This expects to be invoked on a L2 entry or sub L2 entry granularity, so this should + * not be invoked from a context that does not do L2 iteration separately (basically, + * don't call this except from arm_vm_page_granular_prot). + */ +static void +arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va, + int pte_prot_APX, int pte_prot_XN, int forceCoarse, + pt_entry_t **deferred_pte, pt_entry_t *deferred_ptmp) +{ + if (va & ARM_TT_L2_OFFMASK) { /* ragged edge hanging over a ARM_TT_L2_SIZE boundary */ +#if __ARM64_TWO_LEVEL_PMAP__ + tt_entry_t *tte2; +#else + tt_entry_t *tte1, *tte2; +#endif + tt_entry_t tmplate; + pmap_paddr_t pa; + pt_entry_t *ppte, *recursive_pte = NULL, ptmp, recursive_ptmp = 0; + addr64_t ppte_phys; + unsigned i; + + va &= ~ARM_TT_L2_OFFMASK; + pa = va - gVirtBase + gPhysBase; + +#if __ARM64_TWO_LEVEL_PMAP__ + tte2 = &cpu_tte[(((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; +#else + tte1 = &cpu_tte[(((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)]; + tte2 = &((tt_entry_t*) phystokv((*tte1) & ARM_TTE_TABLE_MASK))[(((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; +#endif + + tmplate = *tte2; + + if (ARM_TTE_TYPE_TABLE == (tmplate & ARM_TTE_TYPE_MASK)) { + /* pick up the existing page table. */ + ppte = (pt_entry_t *)phystokv((tmplate & ARM_TTE_TABLE_MASK)); + } else { + // TTE must be reincarnated COARSE. + ppte = (pt_entry_t*)alloc_ptpage(TRUE); + ppte_phys = kvtophys((vm_offset_t)ppte); + + pmap_init_pte_static_page(kernel_pmap, ppte, pa); + + *tte2 = pa_to_tte(ppte_phys) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID; + } + + /* Apply the desired protections to the specified page range */ + for (i = 0; i <= (ARM_TT_L3_INDEX_MASK>>ARM_TT_L3_SHIFT); i++) { + if ((start <= va) && (va < _end)) { + + ptmp = pa | ARM_PTE_AF | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_TYPE; + ptmp = ptmp | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); + ptmp = ptmp | ARM_PTE_AP(pte_prot_APX); + ptmp = ptmp | ARM_PTE_NX; + + if (pte_prot_XN) { + ptmp = ptmp | ARM_PTE_PNX; + } + + /* + * If we can, apply the contiguous hint to this range. The hint is + * applicable if we are not trying to create per-page mappings and + * if the current address falls within a hint-sized range that will + * be fully covered by this mapping request. + */ + if ((va >= round_up_pte_hint_address(start)) && (round_up_pte_hint_address(va + 1) < _end) && + !forceCoarse && use_contiguous_hint) { + ptmp |= ARM_PTE_HINT; + } + + if ((pt_entry_t*)(phystokv(pa)) == ppte) { + assert(recursive_pte == NULL); + /* This assert should be reenabled as part of rdar://problem/30149465 */ + assert(!forceCoarse); + recursive_pte = &ppte[i]; + recursive_ptmp = ptmp; + } else if ((deferred_pte != NULL) && (&ppte[i] == &recursive_pte[1])) { + assert(*deferred_pte == NULL); + assert(deferred_ptmp != NULL); + *deferred_pte = &ppte[i]; + *deferred_ptmp = ptmp; + } else { + ppte[i] = ptmp; + } + } + + va += ARM_PGBYTES; + pa += ARM_PGBYTES; + } + if (recursive_pte != NULL) + *recursive_pte = recursive_ptmp; + } +} + +/* + * arm_vm_page_granular_prot updates protections by iterating over the L2 entries and + * changing them. If a particular chunk necessitates L3 entries (for reasons of + * alignment or length, or an explicit request that the entry be fully expanded), we + * hand off to arm_vm_page_granular_helper to deal with the L3 chunk of the logic. + * + * Note that counterintuitively a forceCoarse request is a request to expand the entries + * out to L3, i.e. to make *finer* grained mappings. That comes from historical arm32 + * nomenclature in which the 4K granule is "coarse" vs. the 1K "fine" granule (which we + * don't use). + */ +static void +arm_vm_page_granular_prot(vm_offset_t start, unsigned long size, + int tte_prot_XN, int pte_prot_APX, int pte_prot_XN, int forceCoarse) +{ + pt_entry_t *deferred_pte = NULL, deferred_ptmp = 0; + vm_offset_t _end = start + size; + vm_offset_t align_start = (start + ARM_TT_L2_OFFMASK) & ~ARM_TT_L2_OFFMASK; + + if (size == 0x0UL) + return; + + if (align_start > _end) { + arm_vm_page_granular_helper(start, _end, start, pte_prot_APX, pte_prot_XN, forceCoarse, NULL, NULL); + return; + } + + arm_vm_page_granular_helper(start, align_start, start, pte_prot_APX, pte_prot_XN, forceCoarse, &deferred_pte, &deferred_ptmp); + + while ((_end - align_start) >= ARM_TT_L2_SIZE) { + if (forceCoarse) + arm_vm_page_granular_helper(align_start, align_start+ARM_TT_L2_SIZE, align_start + 1, + pte_prot_APX, pte_prot_XN, forceCoarse, NULL, NULL); + else { +#if __ARM64_TWO_LEVEL_PMAP__ + tt_entry_t *tte2; +#else + tt_entry_t *tte1, *tte2; +#endif + tt_entry_t tmplate; + +#if __ARM64_TWO_LEVEL_PMAP__ + tte2 = &cpu_tte[((align_start & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; +#else + tte1 = &cpu_tte[((align_start & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)]; + tte2 = &((tt_entry_t*) phystokv((*tte1) & ARM_TTE_TABLE_MASK))[((align_start & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; +#endif + + tmplate = *tte2; + + tmplate = (tmplate & ~ARM_TTE_BLOCK_APMASK) | ARM_TTE_BLOCK_AP(pte_prot_APX); + tmplate = tmplate | ARM_TTE_BLOCK_NX; + if (tte_prot_XN) + tmplate = tmplate | ARM_TTE_BLOCK_PNX; + + *tte2 = tmplate; + } + align_start += ARM_TT_L2_SIZE; + } + + if (align_start < _end) + arm_vm_page_granular_helper(align_start, _end, _end, pte_prot_APX, pte_prot_XN, forceCoarse, &deferred_pte, &deferred_ptmp); + + if (deferred_pte != NULL) + *deferred_pte = deferred_ptmp; +} + +static inline void +arm_vm_page_granular_RNX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 1, AP_RONA, 1, forceCoarse); +} + +static inline void +arm_vm_page_granular_ROX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 0, AP_RONA, 0, forceCoarse); +} + +static inline void +arm_vm_page_granular_RWNX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 1, AP_RWNA, 1, forceCoarse); +} + +static inline void +arm_vm_page_granular_RWX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 0, AP_RWNA, 0, forceCoarse); +} + +void +arm_vm_prot_init(boot_args * args) +{ + /* + * Enforce W^X protections on sections that have been identified so far. This will be + * further refined for each KEXT's TEXT and DATA segments in readPrelinkedExtensions() + */ + bool use_small_page_mappings = FALSE; + + /* + * First off, we'll create mappings for any physical memory preceeding the kernel TEXT. + * This is memory that we want to give to the VM; this will be accomplished through an + * ml_static_mfree call in arm_vm_prot_finalize. This allows the pmap/vm bootstrap + * routines to assume they will have a physically contiguous chunk of memory to deal + * with during bootstrap, while reclaiming this memory later. + */ + arm_vm_page_granular_RWNX(gVirtBase, segPRELINKTEXTB - gVirtBase, use_small_page_mappings); // Memory for the VM + + /* Map coalesced kext TEXT segment RWNX for now */ + arm_vm_page_granular_RWNX(segPRELINKTEXTB, segSizePRELINKTEXT, FALSE); // Refined in OSKext::readPrelinkedExtensions + + /* Map coalesced kext DATA_CONST segment RWNX (could be empty) */ + arm_vm_page_granular_RWNX(segPLKDATACONSTB, segSizePLKDATACONST, FALSE); // Refined in OSKext::readPrelinkedExtensions + + /* Map coalesced kext TEXT_EXEC segment RWX (could be empty) */ + arm_vm_page_granular_ROX(segPLKTEXTEXECB, segSizePLKTEXTEXEC, FALSE); // Refined in OSKext::readPrelinkedExtensions + + /* if new segments not present, set space between PRELINK_TEXT and xnu TEXT to RWNX + * otherwise we no longer expecting any space between the coalesced kext read only segments and xnu rosegments + */ + if (!segSizePLKDATACONST && !segSizePLKTEXTEXEC) { + arm_vm_page_granular_RWNX(segPRELINKTEXTB + segSizePRELINKTEXT, segTEXTB - (segPRELINKTEXTB + segSizePRELINKTEXT), FALSE); + } else { + /* + * If we have the new segments, we should still protect the gap between kext + * read-only pages and kernel read-only pages, in the event that this gap + * exists. + */ + if ((segPLKDATACONSTB + segSizePLKDATACONST) < segTEXTB) { + arm_vm_page_granular_RWNX(segPLKDATACONSTB + segSizePLKDATACONST, segTEXTB - (segPLKDATACONSTB + segSizePLKDATACONST), FALSE); + } + } + + /* + * Protection on kernel text is loose here to allow shenanigans early on. These + * protections are tightened in arm_vm_prot_finalize(). This is necessary because + * we currently patch LowResetVectorBase in cpu.c. + * + * TEXT segment contains mach headers and other non-executable data. This will become RONX later. + */ + arm_vm_page_granular_RNX(segTEXTB, segSizeTEXT, FALSE); + + /* Can DATACONST start out and stay RNX? + * NO, stuff in this segment gets modified during startup (viz. mac_policy_init()/mac_policy_list) + * Make RNX in prot_finalize + */ + arm_vm_page_granular_RWNX(segDATACONSTB, segSizeDATACONST, FALSE); + + /* TEXTEXEC contains read only executable code: becomes ROX in prot_finalize */ + arm_vm_page_granular_RWX(segTEXTEXECB, segSizeTEXTEXEC, FALSE); + + + /* DATA segment will remain RWNX */ + arm_vm_page_granular_RWNX(segDATAB, segSizeDATA, FALSE); + + arm_vm_page_granular_ROX(segKLDB, segSizeKLD, FALSE); + arm_vm_page_granular_RWNX(segLINKB, segSizeLINK, FALSE); + arm_vm_page_granular_ROX(segLASTB, segSizeLAST, FALSE); // __LAST may be empty, but we cannot assume this + + arm_vm_page_granular_RWNX(segPRELINKDATAB, segSizePRELINKDATA, FALSE); // Prelink __DATA for kexts (RW data) + + if (segSizePLKLLVMCOV > 0) + arm_vm_page_granular_RWNX(segPLKLLVMCOVB, segSizePLKLLVMCOV, FALSE); // LLVM code coverage data + + arm_vm_page_granular_RWNX(segPLKLINKEDITB, segSizePLKLINKEDIT, use_small_page_mappings); // Coalesced kext LINKEDIT segment + + arm_vm_page_granular_RWNX(segPRELINKINFOB, segSizePRELINKINFO, FALSE); /* PreLinkInfoDictionary */ + arm_vm_page_granular_RWNX(end_kern, phystokv(args->topOfKernelData) - end_kern, use_small_page_mappings); /* Device Tree, RAM Disk (if present), bootArgs */ + + /* + * This is offset by 4 pages to make room for the boot page tables; we could probably + * include them in the overall mapping, but we'll be paranoid for now. + */ + vm_offset_t extra = 0; +#if KASAN + /* add the KASAN stolen memory to the physmap */ + extra = shadow_ptop - shadow_pbase; + + /* record the extent of the physmap */ + physmap_vbase = phystokv(args->topOfKernelData) + ARM_PGBYTES * 4; + physmap_vtop = static_memory_end; +#endif + arm_vm_page_granular_RNX(phystokv(args->topOfKernelData), ARM_PGBYTES * 4, FALSE); // Boot page tables; they should not be mutable. + arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 4, + extra + static_memory_end - ((phystokv(args->topOfKernelData) + ARM_PGBYTES * 4)), use_small_page_mappings); // rest of physmem +} + +void +arm_vm_prot_finalize(boot_args * args) +{ +#pragma unused(args) + /* + * At this point, we are far enough along in the boot process that it will be + * safe to free up all of the memory preceeding the kernel. It may in fact + * be safe to do this earlier. + * + * This keeps the memory in the V-to-P mapping, but advertises it to the VM + * as usable. + */ + + /* + * if old style PRELINK segment exists, free memory before it, and after it before XNU text + * otherwise we're dealing with a new style kernel cache, so we should just free the + * memory before PRELINK_TEXT segment, since the rest of the KEXT read only data segments + * should be immediately followed by XNU's TEXT segment + */ + + ml_static_mfree(gVirtBase, segPRELINKTEXTB - gVirtBase); + + if (!segSizePLKDATACONST && !segSizePLKTEXTEXEC) { + /* If new segments not present, PRELINK_TEXT is not dynamically sized, free DRAM between it and xnu TEXT */ + ml_static_mfree(segPRELINKTEXTB + segSizePRELINKTEXT, segTEXTB - (segPRELINKTEXTB + segSizePRELINKTEXT)); + } + + /* + * LowResetVectorBase patching should be done by now, so tighten executable + * protections. + */ + arm_vm_page_granular_ROX(segTEXTEXECB, segSizeTEXTEXEC, FALSE); + + /* tighten permissions on kext read only data and code */ + if (segSizePLKDATACONST && segSizePLKTEXTEXEC) { + arm_vm_page_granular_RNX(segPRELINKTEXTB, segSizePRELINKTEXT, FALSE); + arm_vm_page_granular_ROX(segPLKTEXTEXECB, segSizePLKTEXTEXEC, FALSE); + arm_vm_page_granular_RNX(segPLKDATACONSTB, segSizePLKDATACONST, FALSE); + } + +#if defined(KERNEL_INTEGRITY_KTRR) + /* + * __LAST,__pinst should no longer be executable. + */ + arm_vm_page_granular_RNX(segLASTB, segSizeLAST, FALSE); + + /* + * Must wait until all other region permissions are set before locking down DATA_CONST + * as the kernel static page tables live in DATA_CONST on KTRR enabled systems + * and will become immutable. + */ +#endif + arm_vm_page_granular_RNX(segDATACONSTB, segSizeDATACONST, FALSE); + +#ifndef __ARM_L1_PTW__ + FlushPoC_Dcache(); +#endif + flush_mmu_tlb(); +} + +#define TBI_USER 0x1 +#define TBI_KERNEL 0x2 + +boolean_t user_tbi = TRUE; + +/* + * TBI (top-byte ignore) is an ARMv8 feature for ignoring the top 8 bits of + * address accesses. It can be enabled separately for TTBR0 (user) and + * TTBR1 (kernel). We enable it by default for user only, but allow both + * to be controlled by the 'tbi' boot-arg. + */ +static void +set_tbi(void) +{ + uint64_t old_tcr, new_tcr; + int tbi = 0; + + if (PE_parse_boot_argn("tbi", &tbi, sizeof(tbi))) + user_tbi = ((tbi & TBI_USER) == TBI_USER); + old_tcr = new_tcr = get_tcr(); + new_tcr |= (user_tbi) ? TCR_TBI0_TOPBYTE_IGNORED : 0; + new_tcr |= (tbi & TBI_KERNEL) ? TCR_TBI1_TOPBYTE_IGNORED : 0; + + if (old_tcr != new_tcr) { + set_tcr(new_tcr); + sysreg_restore.tcr_el1 = new_tcr; + } +} + +void +arm_vm_init(uint64_t memory_size, boot_args * args) +{ +#if !__ARM64_TWO_LEVEL_PMAP__ + vm_map_address_t va_l1, va_l1_end; + pmap_paddr_t pa_l1; + tt_entry_t *cpu_l1_tte; +#else + /* + * If we are using two level page tables, rather than the + * 3 level page tables that xnu defaults to for ARM64, + * then a great deal of the code in this path becomes + * redundant. As a result, most of the logic having to + * do with L1 pages will be excluded from such + * configurations in this function. + */ +#endif + vm_map_address_t va_l2, va_l2_end; + pmap_paddr_t pa_l2; + tt_entry_t *cpu_l2_tte; + pmap_paddr_t boot_ttep; + tt_entry_t *boot_tte; + uint64_t mem_segments; + vm_offset_t ptpage_vaddr; + + + /* + * Get the virtual and physical memory base from boot_args. + */ + gVirtBase = args->virtBase; + gPhysBase = args->physBase; + gPhysSize = args->memSize; + mem_size = args->memSize; + if ((memory_size != 0) && (mem_size > memory_size)) + mem_size = memory_size; + if (mem_size > MEM_SIZE_MAX ) + mem_size = MEM_SIZE_MAX; + static_memory_end = gVirtBase + mem_size; + + boot_ttep = args->topOfKernelData; + boot_tte = (tt_entry_t *) phystokv(boot_ttep); + + /* + * Four pages: + * TTBR0 L1, TTBR0 L2 - 1:1 bootstrap mapping. + * TTBR1 L1, TTBR1 L2 - kernel mapping + */ + avail_start = boot_ttep + 4*ARM_PGBYTES; + +#if defined(KERNEL_INTEGRITY_KTRR) + arm_replace_identity_map(args); +#endif + + /* Initialize invalid tte page */ + invalid_tte = (tt_entry_t *)alloc_ptpage(TRUE); + invalid_ttep = kvtophys((vm_offset_t)invalid_tte); + bzero(invalid_tte, ARM_PGBYTES); + + /* + * Initialize l1 page table page + */ +#if __ARM64_TWO_LEVEL_PMAP__ + /* + * If we're using a two level page table, we still need to + * set the cpu_ttep to avail_start, as this will be the root + * of our page table regardless of how many levels we are + * using. + */ +#endif + cpu_tte = (tt_entry_t *)alloc_ptpage(TRUE); + cpu_ttep = kvtophys((vm_offset_t)cpu_tte); + bzero(cpu_tte, ARM_PGBYTES); + + avail_end = gPhysBase + mem_size; + + /* + * Initialize l1 and l2 page table pages : + * map physical memory at the kernel base virtual address + * cover the kernel dynamic address range section + * + * the so called physical aperture should be statically mapped + */ + +#if !__ARM64_TWO_LEVEL_PMAP__ + pa_l1 = gPhysBase; + va_l1 = gVirtBase; + va_l1_end = gVirtBase + mem_size; +#if KASAN + /* add the KASAN stolen memory to the physmap */ + va_l1_end = gVirtBase + (shadow_ptop - gPhysBase); +#endif + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + + while (va_l1 < va_l1_end) { + tt_entry_t *new_tte = (tt_entry_t *)alloc_ptpage(TRUE); + /* Allocate a page and setup L1 Table TTE in L1 */ + *cpu_l1_tte = (kvtophys((vm_offset_t)new_tte) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID; + bzero((void *)new_tte, ARM_PGBYTES); + + va_l2 = va_l1; + + if (((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE) < va_l1) { + /* If this is the last L1 entry, it must cover the last mapping. */ + va_l2_end = va_l1_end; + } else { + va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE, va_l1_end); + } + + pa_l2 = pa_l1; + cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l1 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#else + va_l2 = gVirtBase; + va_l2_end = gVirtBase + mem_size; + pa_l2 = gPhysBase; + cpu_l2_tte = cpu_tte + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); + +#if KASAN + /* add the KASAN stolen memory to the physmap */ + va_l2_end = gVirtBase + (shadow_ptop - gPhysBase); +#endif + +#endif + + while (va_l2 < va_l2_end) { + /* Set up L2 Block TTE in L2 */ + *cpu_l2_tte = (pa_l2 & ARM_TTE_BLOCK_L2_MASK) | ARM_TTE_TYPE_BLOCK + | ARM_TTE_VALID | ARM_TTE_BLOCK_AF + | ARM_TTE_BLOCK_AP(AP_RWNA) | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) + | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK); + va_l2 += ARM_TT_L2_SIZE; + pa_l2 += ARM_TT_L2_SIZE; + cpu_l2_tte++; + } +#if !__ARM64_TWO_LEVEL_PMAP__ + cpu_l1_tte++; + va_l1 = va_l2; + pa_l1 = pa_l2; + } +#endif + + /* + * Now retrieve addresses for end, edata, and etext from MACH-O headers + */ + segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_TEXT", &segSizePRELINKTEXT); + segPLKDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_DATA_CONST", &segSizePLKDATACONST); + segPLKTEXTEXECB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_TEXT_EXEC", &segSizePLKTEXTEXEC); + segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT", &segSizeTEXT); + segDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA_CONST", &segSizeDATACONST); + segTEXTEXECB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT_EXEC", &segSizeTEXTEXEC); + segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA", &segSizeDATA); + segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LINKEDIT", &segSizeLINK); + segKLDB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLD", &segSizeKLD); + segPRELINKDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_DATA", &segSizePRELINKDATA); + segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_INFO", &segSizePRELINKINFO); + segPLKLLVMCOVB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_LLVM_COV", &segSizePLKLLVMCOV); + segPLKLINKEDITB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_LINKEDIT", &segSizePLKLINKEDIT); + segLASTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LAST", &segSizeLAST); + + (void) PE_parse_boot_argn("use_contiguous_hint", &use_contiguous_hint, sizeof(use_contiguous_hint)); + assert(segSizePRELINKTEXT < 0x03000000); /* 23355738 */ + + /* if one of the new segments is present, the other one better be as well */ + if (segSizePLKDATACONST || segSizePLKTEXTEXEC) { + assert(segSizePLKDATACONST && segSizePLKTEXTEXEC); + } + + etext = (vm_offset_t) segTEXTB + segSizeTEXT; + sdata = (vm_offset_t) segDATAB; + edata = (vm_offset_t) segDATAB + segSizeDATA; + end_kern = round_page(getlastaddr()); /* Force end to next page */ + + vm_set_page_size(); + + vm_kernel_base = segTEXTB; + vm_kernel_top = (vm_offset_t) &last_kernel_symbol; + vm_kext_base = segPRELINKTEXTB; + vm_kext_top = vm_kext_base + segSizePRELINKTEXT; + + vm_prelink_stext = segPRELINKTEXTB; + if (!segSizePLKTEXTEXEC && !segSizePLKDATACONST) { + vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT; + } else { + vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT + segSizePLKDATACONST + segSizePLKTEXTEXEC; + } + vm_prelink_sinfo = segPRELINKINFOB; + vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO; + vm_slinkedit = segLINKB; + vm_elinkedit = segLINKB + segSizeLINK; + + vm_prelink_sdata = segPRELINKDATAB; + vm_prelink_edata = segPRELINKDATAB + segSizePRELINKDATA; + + arm_vm_prot_init(args); + + + /* + * Initialize the page tables for the low globals: + * cover this address range: + * LOW_GLOBAL_BASE_ADDRESS + 2MB + */ +#if __ARM64_TWO_LEVEL_PMAP__ + va_l2 = LOW_GLOBAL_BASE_ADDRESS; + cpu_l2_tte = cpu_tte + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#else + va_l1 = va_l2 = LOW_GLOBAL_BASE_ADDRESS; + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#endif + ptpage_vaddr = alloc_ptpage(TRUE); + *cpu_l2_tte = (kvtophys(ptpage_vaddr) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN; + bzero((void *)ptpage_vaddr, ARM_PGBYTES); + + /* + * Initialize l2 page table pages : + * cover this address range: + * KERNEL_DYNAMIC_ADDR - VM_MAX_KERNEL_ADDRESS + */ +#if !__ARM64_TWO_LEVEL_PMAP__ + va_l1 = (gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL; + va_l1_end = VM_MAX_KERNEL_ADDRESS; + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + + while (va_l1 < va_l1_end) { + if (*cpu_l1_tte == ARM_TTE_EMPTY) { + /* Allocate a page and setup L1 Table TTE in L1 */ + ptpage_vaddr = alloc_ptpage(TRUE); + *cpu_l1_tte = (kvtophys(ptpage_vaddr) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN; + bzero((void *)ptpage_vaddr, ARM_PGBYTES); + } + + if ((va_l1 + ARM_TT_L1_SIZE) < va_l1) { + /* If this is the last L1 entry, it must cover the last mapping. */ + break; + } + + va_l1 += ARM_TT_L1_SIZE; + cpu_l1_tte++; + } +#endif + +#if KASAN + kasan_init(); +#endif + + set_mmu_ttb(invalid_ttep & TTBR_BADDR_MASK); + set_mmu_ttb_alternate(cpu_ttep & TTBR_BADDR_MASK); + set_tbi(); + flush_mmu_tlb(); + + /* + * TODO: We're hardcoding the expected virtual TEXT base here; + * that gives us an ugly dependency on a linker argument in + * the make files. Clean this up, so we don't hardcode it + * twice; this is nothing but trouble. + */ + sane_size = mem_size - (avail_start - gPhysBase); + max_mem = mem_size; + vm_kernel_slid_base = segPRELINKTEXTB; + vm_kernel_slid_top = vm_prelink_einfo; + vm_kernel_slide = segTEXTB-0xfffffff007004000; + vm_kernel_stext = segTEXTB; + assert(segDATACONSTB == segTEXTB + segSizeTEXT); + assert(segTEXTEXECB == segDATACONSTB + segSizeDATACONST); + vm_kernel_etext = segTEXTB + segSizeTEXT + segSizeDATACONST + segSizeTEXTEXEC; + + pmap_bootstrap((gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL); + + /* + * Initialize l3 page table pages : + * cover this address range: + * 2MB + FrameBuffer size + 10MB for each 256MB segment + */ + + mem_segments = (mem_size + 0x0FFFFFFF) >> 28; + +#if !__ARM64_TWO_LEVEL_PMAP__ + va_l1 = (gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL; + va_l1_end = va_l1 + ((2 + (mem_segments * 10)) << 20); + va_l1_end += round_page(args->Video.v_height * args->Video.v_rowBytes); + va_l1_end = (va_l1_end + 0x00000000007FFFFFULL) & 0xFFFFFFFFFF800000ULL; + + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + + while (va_l1 < va_l1_end) { + + va_l2 = va_l1; + + if (((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE) < va_l1) { + /* If this is the last L1 entry, it must cover the last mapping. */ + va_l2_end = va_l1_end; + } else { + va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE, va_l1_end); + } + + cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#else + va_l2 = (gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL; + va_l2_end = va_l2 + ((2 + (mem_segments * 10)) << 20); + va_l2_end += round_page(args->Video.v_height * args->Video.v_rowBytes); + va_l2_end = (va_l2_end + 0x00000000007FFFFFULL) & 0xFFFFFFFFFF800000ULL; + cpu_l2_tte = cpu_tte + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#endif + + while (va_l2 < va_l2_end) { + pt_entry_t * ptp; + pmap_paddr_t ptp_phys; + + /* Allocate a page and setup L3 Table TTE in L2 */ + ptp = (pt_entry_t *) alloc_ptpage(FALSE); + ptp_phys = (pmap_paddr_t)kvtophys((vm_offset_t)ptp); + + pmap_init_pte_page(kernel_pmap, ptp, va_l2, 3, TRUE); + + *cpu_l2_tte = (pa_to_tte (ptp_phys)) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN; + + va_l2 += ARM_TT_L2_SIZE; + cpu_l2_tte++; + }; +#if !__ARM64_TWO_LEVEL_PMAP__ + va_l1 = va_l2_end; + cpu_l1_tte++; + } +#endif + + /* + * Initialize l3 page table pages : + * cover this address range: + * (VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - VM_MAX_KERNEL_ADDRESS + */ +#if !__ARM64_TWO_LEVEL_PMAP__ + va_l1 = VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK; + va_l1_end = VM_MAX_KERNEL_ADDRESS; + + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + + while (va_l1 < va_l1_end) { + + va_l2 = va_l1; + + if (((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE) < va_l1) { + /* If this is the last L1 entry, it must cover the last mapping. */ + va_l2_end = va_l1_end; + } else { + va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE, va_l1_end); + } + + cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#else + va_l2 = VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK; + va_l2_end = VM_MAX_KERNEL_ADDRESS; + cpu_l2_tte = cpu_tte + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#endif + + while (va_l2 < va_l2_end) { + pt_entry_t * ptp; + pmap_paddr_t ptp_phys; + + /* Allocate a page and setup L3 Table TTE in L2 */ + ptp = (pt_entry_t *) alloc_ptpage(FALSE); + ptp_phys = (pmap_paddr_t)kvtophys((vm_offset_t)ptp); + + pmap_init_pte_page(kernel_pmap, ptp, va_l2, 3, TRUE); + + *cpu_l2_tte = (pa_to_tte (ptp_phys)) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN; + + va_l2 += ARM_TT_L2_SIZE; + cpu_l2_tte++; + }; +#if !__ARM64_TWO_LEVEL_PMAP__ + va_l1 = va_l2_end; + cpu_l1_tte++; + } +#endif + +#if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__ + /* + * In this configuration, the bootstrap mappings (arm_vm_init) and + * the heap mappings occupy separate L1 regions. Explicitly set up + * the heap L1 allocations here. + */ + va_l1 = VM_MIN_KERNEL_ADDRESS & ~ARM_TT_L1_OFFMASK; + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + + while ((va_l1 >= (VM_MIN_KERNEL_ADDRESS & ~ARM_TT_L1_OFFMASK)) && (va_l1 < VM_MAX_KERNEL_ADDRESS)) { + /* + * If the L1 entry has not yet been allocated, allocate it + * now and treat it as a heap table. + */ + if (*cpu_l1_tte == ARM_TTE_EMPTY) { + tt_entry_t *new_tte = (tt_entry_t*)alloc_ptpage(FALSE); + bzero(new_tte, ARM_PGBYTES); + *cpu_l1_tte = (kvtophys((vm_offset_t)new_tte) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN; + } + + cpu_l1_tte++; + va_l1 += ARM_TT_L1_SIZE; + } +#endif + + /* + * Adjust avail_start so that the range that the VM owns + * starts on a PAGE_SIZE aligned boundary. + */ + avail_start = (avail_start + PAGE_MASK) & ~PAGE_MASK; + + + first_avail = avail_start; + patch_low_glo_static_region(args->topOfKernelData, avail_start - args->topOfKernelData); +} + diff --git a/osfmk/arm64/asm.h b/osfmk/arm64/asm.h new file mode 100644 index 000000000..f756f22ae --- /dev/null +++ b/osfmk/arm64/asm.h @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#ifndef _ARM_ASM_H_ +#define _ARM_ASM_H_ + +#include + +#ifndef __arm64__ +#error Why are we including this? +#endif + +/* There is another definition of ALIGN for .c sources */ +#ifdef __ASSEMBLER__ +#define ALIGN 2 +#endif /* ASSEMBLER */ + +#ifndef FALIGN +#define FALIGN ALIGN +#endif + +#define LB(x,n) n +#if __STDC__ +#ifndef __NO_UNDERSCORES__ +#define LCL(x) L ## x +#define EXT(x) _ ## x +#define LEXT(x) _ ## x ## : +#else +#define LCL(x) .L ## x +#define EXT(x) x +#define LEXT(x) x ## : +#endif +#define LBc(x,n) n ## : +#define LBb(x,n) n ## b +#define LBf(x,n) n ## f +#else /* __STDC__ */ +#ifndef __NO_UNDERSCORES__ +#define LCL(x) L/**/x +#define EXT(x) _/**/x +#define LEXT(x) _/**/x/**/: +#else /* __NO_UNDERSCORES__ */ +#define LCL(x) .L/**/x +#define EXT(x) x +#define LEXT(x) x/**/: +#endif /* __NO_UNDERSCORES__ */ +#define LBc(x,n) n/**/: +#define LBb(x,n) n/**/b +#define LBf(x,n) n/**/f +#endif /* __STDC__ */ + +#define String .asciz +#define Value .word +#define Times(a,b) (a*b) +#define Divide(a,b) (a/b) + +#ifdef __ASSEMBLER__ +#if MACH_KDB +#include +/* + * This pseudo-assembler line is added so that there will be at least + * one N_SO entry in the symbol stable to define the current file name. + */ +#endif /* MACH_KDB */ + +/* + * Multiline macros must use .macro syntax for now, + * as there is no ARM64 statement separator. + */ +.macro ENTRY + .align FALIGN + .globl _$0 + _$0 : +.endmacro + +.macro ENTRY2 + .align FALIGN + .globl _$0 + .globl _$1 + _$0 : + _$1 : +.endmacro + +.macro READ_THREAD + mrs $0, TPIDR_EL1 +.endmacro + +.macro BRANCH_EXTERN + b _$0 +.endmacro + +.macro CALL_EXTERN + bl _$0 +.endmacro + +.macro MOV64 + movk $0, #((($1) >> 48) & 0x000000000000FFFF), lsl #48 + movk $0, #((($1) >> 32) & 0x000000000000FFFF), lsl #32 + movk $0, #((($1) >> 16) & 0x000000000000FFFF), lsl #16 + movk $0, #((($1) >> 00) & 0x000000000000FFFF), lsl #00 +.endmacro + +#define PUSH_FRAME \ + stp fp, lr, [sp, #-16]! %% \ + mov fp, sp %% + +#define POP_FRAME \ + mov sp, fp %% \ + ldp fp, lr, [sp], #16 %% + +#define EXT(x) _ ## x + +#ifdef XNU_KERNEL_PRIVATE +.macro PANIC_UNIMPLEMENTED + bl _panic_unimplemented +.endmacro +#endif + +#else /* NOT __ASSEMBLER__ */ + +/* These defines are here for .c files that wish to reference global symbols + * within __asm__ statements. + */ +#ifndef __NO_UNDERSCORES__ +#define CC_SYM_PREFIX "_" +#else +#define CC_SYM_PREFIX "" +#endif /* __NO_UNDERSCORES__ */ +#endif /* __ASSEMBLER__ */ + +#ifdef __ASSEMBLER__ + +# define BRANCH_EXTERN(x) b EXT(x) + +#endif /* __ASSEMBLER__ */ + +#endif /* _ARM_ASM_H_ */ diff --git a/osfmk/arm64/bcopy.s b/osfmk/arm64/bcopy.s new file mode 100644 index 000000000..01f33d61e --- /dev/null +++ b/osfmk/arm64/bcopy.s @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2012 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + * + * This file implements the following functions for the arm64 architecture. + * + * void bcopy(const void * source, + * void * destination, + * size_t length); + * + * void *memmove(void * destination, + * const void * source, + * size_t n); + * + * void *memcpy(void * restrict destination, + * const void * restrict source, + * size_t n); + * + * All copy n successive bytes from source to destination. Memmove and memcpy + * return destination, whereas bcopy has no return value. Copying takes place + * as if it were through a temporary buffer -- after return destination + * contains exactly the bytes from source, even if the buffers overlap (this is + * not required of memcpy by the C standard; its behavior is undefined if the + * buffers overlap, but we are holding ourselves to the historical behavior of + * this function on MacOS). + */ + +#include "asm.h" + +.globl _bcopy +.globl _ovbcopy +.globl _memcpy +.globl _memmove + +/***************************************************************************** + * Macros * + *****************************************************************************/ + +#define kSmallCopy 64 + +/***************************************************************************** + * Entrypoints * + *****************************************************************************/ + +.text +.align 5 +_bcopy: +_ovbcopy: +// Translate bcopy into memcpy by swapping the first and second arguments. + mov x3, x0 + mov x0, x1 + mov x1, x3 + +.align 4 +_memcpy: +_memmove: +// Our preference is to copy the data in ascending address order, but if the +// buffers overlap such that the beginning of the destination buffer aliases +// the end of the source buffer, we need to copy in descending address order +// instead to preserve the memmove semantics. We detect this case with the +// test: +// +// destination - source < length (unsigned compare) +// +// If the address of the source buffer is higher than the address of the +// destination buffer, this arithmetic can overflow, but the overflowed value +// can only be smaller than length if the buffers do not overlap, so we don't +// need to worry about false positives due to the overflow (they happen, but +// only in cases where copying in either order is correct). + PUSH_FRAME + sub x3, x0, x1 + cmp x3, x2 + b.cc L_reverse + mov x3, x0 // copy destination pointer + cmp x2, #(kSmallCopy) + b.cc L_forwardSmallCopy + +/***************************************************************************** + * Forward large copy * + *****************************************************************************/ + +// Load the first 32 bytes from src, and compute the number of bytes to the +// first 32-byte aligned location in dst. Even though we are going to copy +// 32 bytes, only those preceeding that 32-byte location "count" towards +// reducing the length of the buffer or advancing the pointers. We will need +// to issue the first load from the advanced src pointer BEFORE the store to +// the unmodified dst pointer. + add x3, x3, #32 + and x3, x3, #-32 // aligned dst + ldp x12,x13,[x1] + ldp x14,x15,[x1, #16] + sub x5, x3, x0 // bytes between original dst and aligned dst + add x1, x1, x5 // update src pointer + +// At this point, data in the following registers is in flight: +// +// x0 original dst pointer +// x1 corresponding location in src buffer. +// x2 length from aligned location in dst to end of buffer. This is +// guaranteed to be >= (64 - 32). +// x3 aligned location in dst buffer. +// x12:x15 first 32 bytes of src buffer. +// +// We now load 32 bytes from x1, and store 32 bytes from x12:x15 to x3. The +// store *may* overlap the first 32 bytes of the load, so in order to get +// correct memmove semantics, the first 32 byte load must occur before the +// store. +// +// After loading these 32 bytes, we advance x1, and decrement the length by +// 64. If the remaining length of the buffer was less than 64, then we jump +// directly to the cleanup path. + ldp x8, x9, [x1] + ldp x10,x11,[x1, #16] + add x1, x1, #32 + sub x2, x2, x5 // update length + stp x12,x13,[x0] // initial unaligned store + stp x14,x15,[x0, #16] // initial unaligned store + subs x2, x2, #64 + b.ls L_forwardCleanup + +L_forwardCopyLoop: +// Main copy loop: +// +// 1. store the 32 bytes loaded in the previous loop iteration +// 2. advance the destination pointer +// 3. load the next 32 bytes +// 4. advance the source pointer +// 5. subtract 32 from the length +// +// The loop is terminated when 32 or fewer bytes remain to be loaded. Those +// trailing 1-32 bytes will be copied in the loop cleanup. + stnp x8, x9, [x3] + stnp x10,x11,[x3, #16] + add x3, x3, #32 + ldnp x8, x9, [x1] + ldnp x10,x11,[x1, #16] + add x1, x1, #32 + subs x2, x2, #32 + b.hi L_forwardCopyLoop + +L_forwardCleanup: +// There are 32 bytes in x8-x11 that were loaded in the previous loop +// iteration, which need to be stored to [x3,x3+32). In addition, between +// 0 and 32 more bytes need to be copied from x1 to x3 + 32. The exact +// number of bytes to copy is x2 + 32. Instead of using smaller conditional +// copies, we simply copy 32 unaligned bytes from x1+x2 to 64+x3+x2. +// This copy may overlap with the first store, so the loads must come before +// the store of the data from the previous loop iteration. + add x1, x1, x2 + ldp x12,x13,[x1] + ldp x14,x15,[x1, #16] + stp x8, x9, [x3] + stp x10,x11,[x3, #16] + add x3, x3, x2 + stp x12,x13,[x3, #32] + stp x14,x15,[x3, #48] + POP_FRAME + ret + +/***************************************************************************** + * forward small copy * + *****************************************************************************/ + +// Copy one quadword at a time until less than 8 bytes remain to be copied. +// At the point of entry to L_forwardSmallCopy, the "calling convention" +// is as follows: +// +// x0 pointer to first byte of destination +// x1 pointer to first byte of source +// x2 length of buffers +// x3 pointer to first byte of destination +0: ldr x6, [x1],#8 + str x6, [x3],#8 +L_forwardSmallCopy: + subs x2, x2, #8 + b.cs 0b + adds x2, x2, #8 + b.eq 2f +1: ldrb w6, [x1],#1 + strb w6, [x3],#1 + subs x2, x2, #1 + b.ne 1b +2: POP_FRAME + ret + +/***************************************************************************** + * Reverse copy engines * + *****************************************************************************/ + +// The reverse copy engines are identical in every way to the forward copy +// engines, except in that they do everything backwards. For this reason, they +// are somewhat more sparsely commented than the forward copy loops. I have +// tried to only comment things that might be somewhat surprising in how they +// differ from the forward implementation. +// +// The one important thing to note is that (almost without fail), x1 and x3 +// will point to ONE BYTE BEYOND the "right-hand edge" of the active buffer +// throughout these copy loops. They are initially advanced to that position +// in the L_reverse jump island. Because of this, whereas the forward copy +// loops generally follow a "copy data, then advance pointers" scheme, in the +// reverse copy loops, we advance the pointers, then copy the data. + +L_reverse: +// As a minor optimization, we early out if dst == src. + cbz x3, L_return +// advance both pointers to the ends of their respective buffers before +// jumping into the appropriate reverse copy loop. + add x4, x0, x2 + add x1, x1, x2 + cmp x2, #(kSmallCopy) + b.cc L_reverseSmallCopy + +/***************************************************************************** + * Reverse large copy * + *****************************************************************************/ + + ldp x12,x13,[x1, #-16] + ldp x14,x15,[x1, #-32] + sub x3, x4, #1 // In the forward copy, we used dst+32 & -32 + and x3, x3, #-32 // to find an aligned location in the dest + sub x5, x4, x3 // buffer. Here we use dst-1 & -32 instead, + sub x1, x1, x5 // because we are going backwards. + sub x2, x2, x5 + ldp x8, x9, [x1, #-16] + ldp x10,x11,[x1, #-32] + stp x12,x13,[x4, #-16] + stp x14,x15,[x4, #-32] + sub x1, x1, #32 + subs x2, x2, #64 + b.ls L_reverseCleanup + +L_reverseCopyLoop: + stnp x8, x9, [x3, #-16] + stnp x10,x11,[x3, #-32] + sub x3, x3, #32 + ldnp x8, x9, [x1, #-16] + ldnp x10,x11,[x1, #-32] + sub x1, x1, #32 + subs x2, x2, #32 + b.hi L_reverseCopyLoop + +L_reverseCleanup: + sub x1, x1, x2 + ldp x12,x13,[x1, #-16] + ldp x14,x15,[x1, #-32] + stp x8, x9, [x3, #-16] + stp x10,x11,[x3, #-32] + stp x12,x13,[x0, #16] // In the forward copy, we need to compute the + stp x14,x15,[x0] // address of these stores, but here we already + POP_FRAME // have a pointer to the start of the buffer. + ret + +/***************************************************************************** + * reverse small copy * + *****************************************************************************/ + +0: ldr x6, [x1,#-8]! + str x6, [x4,#-8]! +L_reverseSmallCopy: + subs x2, x2, #8 + b.cs 0b + adds x2, x2, #8 + b.eq 2f +1: ldrb w6, [x1,#-1]! + strb w6, [x4,#-1]! + subs x2, x2, #1 + b.ne 1b +2: POP_FRAME + ret + +L_return: + POP_FRAME + ret diff --git a/osfmk/arm64/bsd_arm64.c b/osfmk/arm64/bsd_arm64.c new file mode 100644 index 000000000..726b12bd3 --- /dev/null +++ b/osfmk/arm64/bsd_arm64.c @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifdef MACH_BSD +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +extern void throttle_lowpri_io(int); +void mach_syscall(struct arm_saved_state*); +typedef kern_return_t (*mach_call_t)(void *); + +struct mach_call_args { + syscall_arg_t arg1; + syscall_arg_t arg2; + syscall_arg_t arg3; + syscall_arg_t arg4; + syscall_arg_t arg5; + syscall_arg_t arg6; + syscall_arg_t arg7; + syscall_arg_t arg8; + syscall_arg_t arg9; +}; + +static void +arm_set_mach_syscall_ret(struct arm_saved_state *state, int retval) +{ + if (is_saved_state32(state)) { + saved_state32(state)->r[0] = retval; + } else { + saved_state64(state)->x[0] = retval; + } +} + +static kern_return_t +arm_get_mach_syscall_args(struct arm_saved_state *state, struct mach_call_args *dest, const mach_trap_t *trapp) +{ + uint32_t reg_count; + + if (is_saved_state32(state)) { + /* The trap table entry defines the number of 32-bit words to be copied in from userspace. */ + reg_count = trapp->mach_trap_u32_words; + + /* + * We get 7 contiguous words; r0-r6, hop over r7 + * (frame pointer), optionally r8 + */ + if (reg_count <= 7) { + bcopy((char*)saved_state32(state), (char*)dest, sizeof(uint32_t) * reg_count); + } else if (reg_count <= 9) { + bcopy((char*)saved_state32(state), (char*)dest, sizeof(uint32_t) * 7); + bcopy((char*)&saved_state32(state)->r[8], ((char*)dest) + sizeof(uint32_t) * 7, + reg_count - 7); + } else { + panic("Trap with %d words of args? We only support 9.", reg_count); + } + +#if CONFIG_REQUIRES_U32_MUNGING + trapp->mach_trap_arg_munge32(dest); +#else +#error U32 mach traps on ARM64 kernel requires munging +#endif + } else { + assert(is_saved_state64(state)); + bcopy((char*)saved_state64(state), (char*)dest, trapp->mach_trap_arg_count * sizeof(uint64_t)); + } + + return KERN_SUCCESS; +} + +kern_return_t +thread_setsinglestep(__unused thread_t thread, __unused int on) +{ + return (KERN_FAILURE); /* XXX TODO */ +} + +#if CONFIG_DTRACE + +vm_offset_t dtrace_get_cpu_int_stack_top(void); + +vm_offset_t +dtrace_get_cpu_int_stack_top(void) +{ + return getCpuDatap()->intstack_top; +} +#endif /* CONFIG_DTRACE */ +extern const char *mach_syscall_name_table[]; + +/* ARM64_TODO: remove this. still TODO?*/ +extern struct proc* current_proc(void); +extern int proc_pid(struct proc*); + +void +mach_syscall(struct arm_saved_state *state) +{ + kern_return_t retval; + mach_call_t mach_call; + struct mach_call_args args = { 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + int call_number = get_saved_state_svc_number(state); + int64_t exc_code; + int argc; + + struct uthread *ut = get_bsdthread_info(current_thread()); + uthread_reset_proc_refcount(ut); + + assert(call_number < 0); /* Otherwise it would be a Unix syscall */ + call_number = -call_number; + + if (call_number >= MACH_TRAP_TABLE_COUNT) { + goto bad; + } + + DEBUG_KPRINT_SYSCALL_MACH( + "mach_syscall: code=%d(%s) (pid %d, tid %lld)\n", + call_number, mach_syscall_name_table[call_number], + proc_pid(current_proc()), thread_tid(current_thread())); + +#if DEBUG_TRACE + kprintf("mach_syscall(0x%08x) code=%d\n", state, call_number); +#endif + + mach_call = (mach_call_t)mach_trap_table[call_number].mach_trap_function; + + if (mach_call == (mach_call_t)kern_invalid) { + DEBUG_KPRINT_SYSCALL_MACH( + "mach_syscall: kern_invalid 0x%x\n", call_number); + goto bad; + } + + argc = mach_trap_table[call_number].mach_trap_arg_count; + if (argc) { + retval = arm_get_mach_syscall_args(state, &args, &mach_trap_table[call_number]); + if (retval != KERN_SUCCESS) { + arm_set_mach_syscall_ret(state, retval); + + DEBUG_KPRINT_SYSCALL_MACH( + "mach_syscall: retval=0x%x\n", retval); + return; + } + } + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_START, + args.arg1, args.arg2, args.arg3, args.arg4, 0); + + retval = mach_call(&args); + + DEBUG_KPRINT_SYSCALL_MACH("mach_syscall: retval=0x%x (pid %d, tid %lld)\n", retval, + proc_pid(current_proc()), thread_tid(current_thread())); + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_SC,(call_number)) | DBG_FUNC_END, + retval, 0, 0, 0, 0); + + arm_set_mach_syscall_ret(state, retval); + + throttle_lowpri_io(1); + +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_get_kernel_state(current_thread())->allocation_name; + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + +#if PROC_REF_DEBUG + if (__improbable(uthread_get_proc_refcount(ut) != 0)) { + panic("system call returned with uu_proc_refcount != 0"); + } +#endif + + return; + +bad: + exc_code = call_number; + exception_triage(EXC_SYSCALL, &exc_code, 1); + /* NOTREACHED */ + panic("Returned from exception_triage()?\n"); +} +#endif /* MACH_BSD */ diff --git a/osfmk/arm64/bzero.s b/osfmk/arm64/bzero.s new file mode 100644 index 000000000..c2f084e47 --- /dev/null +++ b/osfmk/arm64/bzero.s @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2012 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + * + * This file implements the following functions for the arm64 architecture: + * + * void bzero(void *buffer, size_t length); + * void __bzero(void *buffer, size_t length); + * void *memset(void *buffer, int value, size_t length); + * + * The first two zero-fill a buffer. The third fills the buffer with the low + * byte of its second argument. + */ + +#include "asm.h" + +.globl _bzero +.globl ___bzero +.globl _memset +.globl _secure_memset + +/***************************************************************************** + * bzero entrypoint * + *****************************************************************************/ + +.text +.align 4 +_bzero: +___bzero: + PUSH_FRAME + mov x2, x1 + eor x1, x1, x1 + mov x3, x0 + cmp x2, #128 + b.cc L_memsetSmall + +/***************************************************************************** + * Large buffer zero engine * + *****************************************************************************/ + +L_bzeroLarge: +// Write the first 64 bytes of the buffer without regard to alignment, then +// advance x3 to point to a cacheline-aligned location within the buffer, and +// decrement the length accordingly. + stp x1, x1, [x0] + stp x1, x1, [x0, #16] + stp x1, x1, [x0, #32] + stp x1, x1, [x0, #48] + add x3, x0, #64 + and x3, x3, #-64 + add x2, x2, x0 // end of buffer + add x4, x3, #64 // end of first cacheline to zero + subs x2, x2, x4 // if the end of the buffer comes first, jump + b.ls 1f // directly to the cleanup pass. +0: dc zva, x3 // zero cacheline + add x3, x3, #64 // increment pointer + subs x2, x2, #64 // decrement length + b.hi 0b +1: add x3, x3, x2 // back up pointer to (end of buffer) - 64. + stp x1, x1, [x3] // and store 64 bytes to reach end of buffer. + stp x1, x1, [x3, #16] + stp x1, x1, [x3, #32] + stp x1, x1, [x3, #48] + POP_FRAME + ret + +/***************************************************************************** + * memset entrypoint * + *****************************************************************************/ + +.align 4 +/* + * It is important that secure_memset remains defined in assembly to avoid + * compiler optimizations. + */ +_secure_memset: +_memset: + PUSH_FRAME + and x1, x1, #0xff + orr x3, xzr,#0x0101010101010101 + mul x1, x1, x3 + mov x3, x0 + cmp x2, #64 + b.cc L_memsetSmall + +/***************************************************************************** + * Large buffer store engine * + *****************************************************************************/ + +L_memsetLarge: +// Write the first 64 bytes of the buffer without regard to alignment, then +// advance x3 to point to an aligned location within the buffer, and +// decrement the length accordingly. + stp x1, x1, [x0] + add x3, x0, #16 + and x3, x3, #-16 + add x2, x2, x0 // end of buffer + add x4, x3, #64 // end of first aligned 64-byte store + subs x2, x2, x4 // if the end of the buffer comes first, jump + b.ls 1f // directly to the cleanup store. +0: stnp x1, x1, [x3] + stnp x1, x1, [x3, #16] + stnp x1, x1, [x3, #32] + stnp x1, x1, [x3, #48] + add x3, x3, #64 + subs x2, x2, #64 + b.hi 0b +1: add x3, x3, x2 // back up pointer to (end of buffer) - 64. + stp x1, x1, [x3] + stp x1, x1, [x3, #16] + stp x1, x1, [x3, #32] + stp x1, x1, [x3, #48] + POP_FRAME + ret + +/***************************************************************************** + * Small buffer store engine * + *****************************************************************************/ + +0: str x1, [x3],#8 +L_memsetSmall: + subs x2, x2, #8 + b.cs 0b + adds x2, x2, #8 + b.eq 2f +1: strb w1, [x3],#1 + subs x2, x2, #1 + b.ne 1b +2: POP_FRAME + ret diff --git a/osfmk/arm64/caches_asm.s b/osfmk/arm64/caches_asm.s new file mode 100644 index 000000000..463d39067 --- /dev/null +++ b/osfmk/arm64/caches_asm.s @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2010-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include "assym.s" + +/* + * void InvalidatePoU_Icache(void) + * + * Invalidate i-cache + */ + .text + .align 2 + .globl EXT(InvalidatePoU_Icache) + .globl EXT(invalidate_mmu_icache) +LEXT(InvalidatePoU_Icache) +LEXT(invalidate_mmu_icache) + ic ialluis // Invalidate icache + dsb sy + isb sy + ret + +/* + * void InvalidatePoU_IcacheRegion(vm_offset_t va, unsigned length) + * + * Invalidate icache region + */ + .text + .align 2 + .globl EXT(InvalidatePoU_IcacheRegion) +LEXT(InvalidatePoU_IcacheRegion) + mov x9, #((1< +#include +#include +#include +#include +#include +#include + +extern int _bcopyin(const char *src, char *dst, vm_size_t len); +extern int _bcopyinstr(const char *src, char *dst, vm_size_t max, vm_size_t *actual); +extern int _bcopyout(const char *src, char *dst, vm_size_t len); +extern int _copyin_word(const char *src, uint64_t *dst, vm_size_t len); + +extern pmap_t kernel_pmap; +extern boolean_t arm_pan_enabled; + +typedef enum copyio_type { + COPYIO_IN, + COPYIO_IN_WORD, + COPYIO_INSTR, + COPYIO_OUT, +} copyio_type_t; + +int +copyio_check_user_addr(user_addr_t user_addr, vm_size_t nbytes) +{ + if (nbytes && (user_addr + nbytes <= user_addr)) + return EFAULT; + + if ((user_addr + nbytes) > vm_map_max(current_thread()->map)) + return EFAULT; + + return 0; +} + +static inline void +user_access_enable(void) +{ +#if __ARM_PAN_AVAILABLE__ + if (arm_pan_enabled) { + __builtin_arm_wsr("pan", 0); + } +#endif /* __ARM_PAN_AVAILABLE__ */ +} + +static inline void +user_access_disable(void) +{ +#if __ARM_PAN_AVAILABLE__ + if (arm_pan_enabled) { + __builtin_arm_wsr("pan", 1); + } +#endif /* __ARM_PAN_AVAILABLE__ */ +} + +static int +copyio(copyio_type_t copytype, const char *src, char *dst, + vm_size_t nbytes, vm_size_t *lencopied) +{ + int result = 0; + vm_size_t bytes_copied = 0; + + /* Reject TBI addresses */ + if (copytype == COPYIO_OUT) { + if ((uintptr_t)dst & TBI_MASK) + return EINVAL; + } else { + if ((uintptr_t)src & TBI_MASK) + return EINVAL; + } + + if (!nbytes) { + return 0; + } + +#if KASAN + /* For user copies, asan-check the kernel-side buffer */ + if (copytype == COPYIO_IN || copytype == COPYIO_INSTR || copytype == COPYIO_IN_WORD) { + __asan_storeN((uintptr_t)dst, nbytes); + } else if (copytype == COPYIO_OUT) { + __asan_loadN((uintptr_t)src, nbytes); + } +#endif + + user_access_enable(); + + /* Select copy routines based on direction: + * COPYIO_IN - Use unprivileged loads to read from user address + * COPYIO_OUT - Use unprivleged stores to write to user address + */ + + switch (copytype) { + case COPYIO_IN: + result = _bcopyin(src, dst, nbytes); + break; + case COPYIO_INSTR: + result = _bcopyinstr(src, dst, nbytes, &bytes_copied); + if (result != EFAULT) { + *lencopied = bytes_copied; + } + break; + case COPYIO_IN_WORD: + result = _copyin_word(src, (uint64_t *)(uintptr_t)dst, nbytes); + break; + case COPYIO_OUT: + result = _bcopyout(src, dst, nbytes); + break; + default: + result = EINVAL; + } + + user_access_disable(); + return result; +} + +int +copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes) +{ + bcopy((const char*)(uintptr_t)user_addr, kernel_addr, nbytes); + + return 0; +} + +int +copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes) +{ + bcopy(kernel_addr, (char *)(uintptr_t)user_addr, nbytes); + + return 0; +} + +int +copyin(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes) +{ + int result; + + if (user_addr >= VM_MIN_KERNEL_ADDRESS || user_addr + nbytes >= VM_MIN_KERNEL_ADDRESS) { + if (current_thread()->map->pmap == kernel_pmap) + return copyin_kern(user_addr, kernel_addr, nbytes); + else + return EFAULT; + } + + if (nbytes >= 4096) { + result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes); + if (result) return result; + } + + result = copyio_check_user_addr(user_addr, nbytes); + + if (result) return result; + + return copyio(COPYIO_IN, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, NULL); +} + +/* + * copyin_word + * Read an aligned value from userspace as a single memory transaction. + * This function supports userspace synchronization features + */ +int +copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_size_t nbytes) +{ + int result; + + /* Verify sizes */ + if ((nbytes != 4) && (nbytes != 8)) + return EINVAL; + + /* Test alignment */ + if (user_addr & (nbytes - 1)) + return EINVAL; + + /* Address must be user */ + if (user_addr >= VM_MIN_KERNEL_ADDRESS || user_addr + nbytes >= VM_MIN_KERNEL_ADDRESS) + return EFAULT; + + result = copyio_check_user_addr(user_addr, nbytes); + if (result) + return result; + + return copyio(COPYIO_IN_WORD, (const char *)user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL); +} + +int +copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied) +{ + int result; + + if (user_addr >= VM_MIN_KERNEL_ADDRESS || user_addr + nbytes >= VM_MIN_KERNEL_ADDRESS) { + return EFAULT; + } + + result = copyio_check_user_addr(user_addr, nbytes); + + if (result) return result; + + if (!nbytes) { + return ENAMETOOLONG; + } + + return copyio(COPYIO_INSTR, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, lencopied); +} + +int +copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes) +{ + int result; + + if (user_addr >= VM_MIN_KERNEL_ADDRESS || user_addr + nbytes >= VM_MIN_KERNEL_ADDRESS) { + if (current_thread()->map->pmap == kernel_pmap) + return copyout_kern(kernel_addr, user_addr, nbytes); + else + return EFAULT; + } + + if (nbytes >= 4096) { + result = copyout_validate((uintptr_t)kernel_addr, user_addr, nbytes); + if (result) return result; + } + + result = copyio_check_user_addr(user_addr, nbytes); + + if (result) return result; + + return copyio(COPYIO_OUT, kernel_addr, (char *)(uintptr_t)user_addr, nbytes, NULL); +} + + +/* + * Copy sizes bigger than this value will cause a kernel panic. + * + * Yes, this is an arbitrary fixed limit, but it's almost certainly + * a programming error to be copying more than this amount between + * user and wired kernel memory in a single invocation on this + * platform. + */ +const int copysize_limit_panic = (64 * 1024 * 1024); + +/* + * Validate the arguments to copy{in,out} on this platform. + * + * Called when nbytes is "large" e.g. more than a page. Such sizes are + * infrequent, and very large sizes are likely indications of attempts + * to exploit kernel programming errors (bugs). + */ +static int +copy_validate(const user_addr_t user_addr, + uintptr_t kernel_addr, vm_size_t nbytes) +{ + uintptr_t kernel_addr_last = kernel_addr + nbytes; + + if (kernel_addr < VM_MIN_KERNEL_ADDRESS || + kernel_addr > VM_MAX_KERNEL_ADDRESS || + kernel_addr_last < kernel_addr || + kernel_addr_last > VM_MAX_KERNEL_ADDRESS) + panic("%s(%p, %p, %lu) - kaddr not in kernel", __func__, + (void *)user_addr, (void *)kernel_addr, nbytes); + + user_addr_t user_addr_last = user_addr + nbytes; + + if (user_addr_last < user_addr || user_addr_last > VM_MIN_KERNEL_ADDRESS) + return (EFAULT); + + if (__improbable(nbytes > copysize_limit_panic)) + panic("%s(%p, %p, %lu) - transfer too large", __func__, + (void *)user_addr, (void *)kernel_addr, nbytes); + + return (0); +} + +int +copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes) +{ + return (copy_validate(ua, ka, nbytes)); +} + +int +copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes) +{ + return (copy_validate(ua, ka, nbytes)); +} + diff --git a/osfmk/arm64/cpu.c b/osfmk/arm64/cpu.c new file mode 100644 index 000000000..6e0d5ed52 --- /dev/null +++ b/osfmk/arm64/cpu.c @@ -0,0 +1,864 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * File: arm64/cpu.c + * + * cpu specific routines + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#if KPC +#include +#endif + +#if MONOTONIC +#include +#endif /* MONOTONIC */ + +extern boolean_t idle_enable; +extern uint64_t wake_abstime; + +#if WITH_CLASSIC_S2R +void sleep_token_buffer_init(void); +#endif + + +extern uintptr_t resume_idle_cpu; +extern uintptr_t start_cpu; + +extern void __attribute__((noreturn)) arm64_prepare_for_sleep(void); +extern void arm64_force_wfi_clock_gate(void); +#if (defined(APPLECYCLONE) || defined(APPLETYPHOON)) +// CPU1 Stuck in WFIWT Because of MMU Prefetch +extern void cyclone_typhoon_prepare_for_wfi(void); +extern void cyclone_typhoon_return_from_wfi(void); +#endif + + +vm_address_t start_cpu_paddr; + +sysreg_restore_t sysreg_restore __attribute__((section("__DATA, __const"))) = { + .tcr_el1 = TCR_EL1_BOOT, +}; + + +// wfi - wfi mode +// 0 : disabled +// 1 : normal +// 2 : overhead simulation (delay & flags) +static int wfi = 1; + +#if DEVELOPMENT || DEBUG + +// wfi_flags +// 1 << 0 : flush L1s +// 1 << 1 : flush TLBs +static int wfi_flags = 0; + +// wfi_delay - delay ticks after wfi exit +static uint64_t wfi_delay = 0; + +#endif /* DEVELOPMENT || DEBUG */ + +#if __ARM_GLOBAL_SLEEP_BIT__ +volatile boolean_t arm64_stall_sleep = TRUE; +#endif + +#if WITH_CLASSIC_S2R +/* + * These must be aligned to avoid issues with calling bcopy_phys on them before + * we are done with pmap initialization. + */ +static const uint8_t __attribute__ ((aligned(8))) suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'}; +static const uint8_t __attribute__ ((aligned(8))) running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'}; +#endif + +#if WITH_CLASSIC_S2R +static vm_offset_t sleepTokenBuffer = (vm_offset_t)NULL; +#endif +static boolean_t coresight_debug_enabled = FALSE; + + +static void +configure_coresight_registers(cpu_data_t *cdp) +{ + uint64_t addr; + int i; + + assert(cdp); + + /* + * ARMv8 coresight registers are optional. If the device tree did not + * provide cpu_regmap_paddr, assume that coresight registers are not + * supported. + */ + if (cdp->cpu_regmap_paddr) { + for (i = 0; i < CORESIGHT_REGIONS; ++i) { + /* Skip CTI; these registers are debug-only (they are + * not present on production hardware), and there is + * at least one known Cyclone errata involving CTI + * (rdar://12802966). We have no known clients that + * need the kernel to unlock CTI, so it is safer + * to avoid doing the access. + */ + if (i == CORESIGHT_CTI) + continue; + /* Skip debug-only registers on production chips */ + if (((i == CORESIGHT_ED) || (i == CORESIGHT_UTT)) && !coresight_debug_enabled) + continue; + + if (!cdp->coresight_base[i]) { + addr = cdp->cpu_regmap_paddr + CORESIGHT_OFFSET(i); + cdp->coresight_base[i] = (vm_offset_t)ml_io_map(addr, CORESIGHT_SIZE); + + /* + * At this point, failing to io map the + * registers is considered as an error. + */ + if (!cdp->coresight_base[i]) { + panic("unable to ml_io_map coresight regions"); + } + } + /* Unlock EDLAR, CTILAR, PMLAR */ + if (i != CORESIGHT_UTT) + *(volatile uint32_t *)(cdp->coresight_base[i] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY; + } + } +} + + +/* + * Routine: cpu_bootstrap + * Function: + */ +void +cpu_bootstrap(void) +{ +} + +/* + * Routine: cpu_sleep + * Function: + */ +void +cpu_sleep(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + pmap_switch_user_ttb(kernel_pmap); + cpu_data_ptr->cpu_active_thread = current_thread(); + cpu_data_ptr->cpu_reset_handler = (uintptr_t) start_cpu_paddr; + cpu_data_ptr->cpu_flags |= SleepState; + cpu_data_ptr->cpu_user_debug = NULL; +#if KPC + kpc_idle(); +#endif /* KPC */ +#if MONOTONIC + mt_cpu_down(cpu_data_ptr); +#endif /* MONOTONIC */ + + CleanPoC_Dcache(); + + PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id); + +} + +/* + * Routine: cpu_idle + * Function: + */ +void __attribute__((noreturn)) +cpu_idle(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop; + + if ((!idle_enable) || (cpu_data_ptr->cpu_signal & SIGPdisabled)) + Idle_load_context(); + if (!SetIdlePop()) + Idle_load_context(); + lastPop = cpu_data_ptr->rtcPop; + + pmap_switch_user_ttb(kernel_pmap); + cpu_data_ptr->cpu_active_thread = current_thread(); + if (cpu_data_ptr->cpu_user_debug) + arm_debug_set(NULL); + cpu_data_ptr->cpu_user_debug = NULL; + + if (cpu_data_ptr->cpu_idle_notify) + ((processor_idle_t) cpu_data_ptr->cpu_idle_notify) (cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks); + + if (cpu_data_ptr->idle_timer_notify != 0) { + if (new_idle_timeout_ticks == 0x0ULL) { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } else { + /* set the new idle timeout */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } + timer_resync_deadlines(); + if (cpu_data_ptr->rtcPop != lastPop) + SetIdlePop(); + } + +#if KPC + kpc_idle(); +#endif +#if MONOTONIC + mt_cpu_idle(cpu_data_ptr); +#endif /* MONOTONIC */ + + if (wfi) { + platform_cache_idle_enter(); + +#if DEVELOPMENT || DEBUG + // When simulating wfi overhead, + // force wfi to clock gating only + if (wfi == 2) { + arm64_force_wfi_clock_gate(); + } +#endif /* DEVELOPMENT || DEBUG */ + +#if defined(APPLECYCLONE) || defined(APPLETYPHOON) + // CPU1 Stuck in WFIWT Because of MMU Prefetch + cyclone_typhoon_prepare_for_wfi(); +#endif + __builtin_arm_dsb(DSB_SY); + __builtin_arm_wfi(); + +#if defined(APPLECYCLONE) || defined(APPLETYPHOON) + // CPU1 Stuck in WFIWT Because of MMU Prefetch + cyclone_typhoon_return_from_wfi(); +#endif + +#if DEVELOPMENT || DEBUG + // Handle wfi overhead simulation + if (wfi == 2) { + uint64_t deadline; + + // Calculate wfi delay deadline + clock_absolutetime_interval_to_deadline(wfi_delay, &deadline); + + // Flush L1 caches + if ((wfi_flags & 1) != 0) { + InvalidatePoU_Icache(); + FlushPoC_Dcache(); + } + + // Flush TLBs + if ((wfi_flags & 2) != 0) { + flush_core_tlb(); + } + + // Wait for the ballance of the wfi delay + clock_delay_until(deadline); + } +#endif /* DEVELOPMENT || DEBUG */ + + platform_cache_idle_exit(); + } + + ClearIdlePop(TRUE); + + cpu_idle_exit(); +} + +/* + * Routine: cpu_idle_exit + * Function: + */ +void +cpu_idle_exit(void) +{ + uint64_t new_idle_timeout_ticks = 0x0ULL; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + assert(exception_stack_pointer() != 0); + + /* Back from WFI, unlock OSLAR and EDLAR. */ + configure_coresight_registers(cpu_data_ptr); + +#if KPC + kpc_idle_exit(); +#endif + +#if MONOTONIC + mt_cpu_run(cpu_data_ptr); +#endif /* MONOTONIC */ + + pmap_switch_user_ttb(cpu_data_ptr->cpu_active_thread->map->pmap); + + if (cpu_data_ptr->cpu_idle_notify) + ((processor_idle_t) cpu_data_ptr->cpu_idle_notify) (cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks); + + if (cpu_data_ptr->idle_timer_notify != 0) { + if (new_idle_timeout_ticks == 0x0ULL) { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } else { + /* set the new idle timeout */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } + timer_resync_deadlines(); + } + + Idle_load_context(); +} + +void +cpu_init(void) +{ + cpu_data_t *cdp = getCpuDatap(); + arm_cpu_info_t *cpu_info_p; + + assert(exception_stack_pointer() != 0); + + if (cdp->cpu_type != CPU_TYPE_ARM64) { + + cdp->cpu_type = CPU_TYPE_ARM64; + + timer_call_queue_init(&cdp->rtclock_timer.queue); + cdp->rtclock_timer.deadline = EndOfAllTime; + + if (cdp == &BootCpuData) { + do_cpuid(); + do_cacheid(); + do_mvfpid(); + } else { + /* + * We initialize non-boot CPUs here; the boot CPU is + * dealt with as part of pmap_bootstrap. + */ + pmap_cpu_data_init(); + } + /* ARM_SMP: Assuming identical cpu */ + do_debugid(); + + cpu_info_p = cpuid_info(); + + /* switch based on CPU's reported architecture */ + switch (cpu_info_p->arm_info.arm_arch) { + case CPU_ARCH_ARMv8: + cdp->cpu_subtype = CPU_SUBTYPE_ARM64_V8; + break; + default: + //cdp->cpu_subtype = CPU_SUBTYPE_ARM64_ALL; + /* this panic doesn't work this early in startup */ + panic("Unknown CPU subtype..."); + break; + } + + cdp->cpu_threadtype = CPU_THREADTYPE_NONE; + } + cdp->cpu_stat.irq_ex_cnt_wake = 0; + cdp->cpu_stat.ipi_cnt_wake = 0; + cdp->cpu_stat.timer_cnt_wake = 0; + cdp->cpu_running = TRUE; + cdp->cpu_sleep_token_last = cdp->cpu_sleep_token; + cdp->cpu_sleep_token = 0x0UL; +#if KPC + kpc_idle_exit(); +#endif /* KPC */ +#if MONOTONIC + mt_cpu_up(cdp); +#endif /* MONOTONIC */ +} + +cpu_data_t * +cpu_data_alloc(boolean_t is_boot_cpu) +{ + cpu_data_t *cpu_data_ptr = NULL; + + if (is_boot_cpu) + cpu_data_ptr = &BootCpuData; + else { + void *irq_stack = NULL; + void *exc_stack = NULL; + void *fiq_stack = NULL; + + if ((kmem_alloc(kernel_map, (vm_offset_t *)&cpu_data_ptr, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU)) != KERN_SUCCESS) + goto cpu_data_alloc_error; + + bzero((void *)cpu_data_ptr, sizeof(cpu_data_t)); + + if ((irq_stack = kalloc(INTSTACK_SIZE)) == 0) + goto cpu_data_alloc_error; + cpu_data_ptr->intstack_top = (vm_offset_t)irq_stack + INTSTACK_SIZE ; + cpu_data_ptr->istackptr = cpu_data_ptr->intstack_top; + + if ((exc_stack = kalloc(PAGE_SIZE)) == 0) + goto cpu_data_alloc_error; + cpu_data_ptr->excepstack_top = (vm_offset_t)exc_stack + PAGE_SIZE ; + cpu_data_ptr->excepstackptr = cpu_data_ptr->excepstack_top; + + if ((fiq_stack = kalloc(PAGE_SIZE)) == 0) + goto cpu_data_alloc_error; + cpu_data_ptr->fiqstack_top = (vm_offset_t)fiq_stack + PAGE_SIZE ; + cpu_data_ptr->fiqstackptr = cpu_data_ptr->fiqstack_top; + } + + cpu_data_ptr->cpu_processor = cpu_processor_alloc(is_boot_cpu); + if (cpu_data_ptr->cpu_processor == (struct processor *)NULL) + goto cpu_data_alloc_error; + + return cpu_data_ptr; + +cpu_data_alloc_error: + panic("cpu_data_alloc() failed\n"); + return (cpu_data_t *)NULL; +} + + +void +cpu_data_free(cpu_data_t *cpu_data_ptr) +{ + if (cpu_data_ptr == &BootCpuData) + return; + + cpu_processor_free( cpu_data_ptr->cpu_processor); + kfree( (void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE); + kfree( (void *)(cpu_data_ptr->fiqstack_top - PAGE_SIZE), PAGE_SIZE); + kmem_free(kernel_map, (vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t)); +} + +void +cpu_data_init(cpu_data_t *cpu_data_ptr) +{ + uint32_t i; + + cpu_data_ptr->cpu_flags = 0; + cpu_data_ptr->interrupts_enabled = 0; + cpu_data_ptr->cpu_int_state = 0; + cpu_data_ptr->cpu_pending_ast = AST_NONE; + cpu_data_ptr->cpu_cache_dispatch = (void *) 0; + cpu_data_ptr->rtcPop = EndOfAllTime; + cpu_data_ptr->rtclock_datap = &RTClockData; + cpu_data_ptr->cpu_user_debug = NULL; + + + cpu_data_ptr->cpu_base_timebase = 0; + cpu_data_ptr->cpu_idle_notify = (void *) 0; + cpu_data_ptr->cpu_idle_latency = 0x0ULL; + cpu_data_ptr->cpu_idle_pop = 0x0ULL; + cpu_data_ptr->cpu_reset_type = 0x0UL; + cpu_data_ptr->cpu_reset_handler = 0x0UL; + cpu_data_ptr->cpu_reset_assist = 0x0UL; + cpu_data_ptr->cpu_regmap_paddr = 0x0ULL; + cpu_data_ptr->cpu_phys_id = 0x0UL; + cpu_data_ptr->cpu_l2_access_penalty = 0; + cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP; + cpu_data_ptr->cpu_cluster_id = 0; + cpu_data_ptr->cpu_l2_id = 0; + cpu_data_ptr->cpu_l2_size = 0; + cpu_data_ptr->cpu_l3_id = 0; + cpu_data_ptr->cpu_l3_size = 0; + + cpu_data_ptr->cpu_signal = SIGPdisabled; + +#if DEBUG || DEVELOPMENT + cpu_data_ptr->failed_xcall = NULL; + cpu_data_ptr->failed_signal = 0; + cpu_data_ptr->failed_signal_count = 0; +#endif + + cpu_data_ptr->cpu_get_fiq_handler = NULL; + cpu_data_ptr->cpu_tbd_hardware_addr = NULL; + cpu_data_ptr->cpu_tbd_hardware_val = NULL; + cpu_data_ptr->cpu_get_decrementer_func = NULL; + cpu_data_ptr->cpu_set_decrementer_func = NULL; + cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; + cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL; + cpu_data_ptr->cpu_xcall_p0 = NULL; + cpu_data_ptr->cpu_xcall_p1 = NULL; + + for (i = 0; i < CORESIGHT_REGIONS; ++i) { + cpu_data_ptr->coresight_base[i] = 0; + } + + pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data; + + pmap_cpu_data_ptr->cpu_user_pmap = (struct pmap *) NULL; + pmap_cpu_data_ptr->cpu_user_pmap_stamp = 0; + pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM; + + for (i = 0; i < (sizeof(pmap_cpu_data_ptr->cpu_asid_high_bits) / sizeof(*pmap_cpu_data_ptr->cpu_asid_high_bits)); i++) { + pmap_cpu_data_ptr->cpu_asid_high_bits[i] = 0; + } + cpu_data_ptr->halt_status = CPU_NOT_HALTED; +} + +kern_return_t +cpu_data_register(cpu_data_t *cpu_data_ptr) +{ + int cpu = cpu_data_ptr->cpu_number; + +#if KASAN + for (int i = 0; i < CPUWINDOWS_MAX; i++) { + kasan_notify_address_nopoison(pmap_cpu_windows_copy_addr(cpu, i), PAGE_SIZE); + } +#endif + + CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr; + CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys( (vm_offset_t)cpu_data_ptr); + return KERN_SUCCESS; + +} + +kern_return_t +cpu_start(int cpu) +{ + cpu_data_t *cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr; + + kprintf("cpu_start() cpu: %d\n", cpu); + + if (cpu == cpu_number()) { + cpu_machine_init(); + configure_coresight_registers(cpu_data_ptr); + } else { + thread_t first_thread; + + cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr; + + cpu_data_ptr->cpu_pmap_cpu_data.cpu_user_pmap = NULL; + + if (cpu_data_ptr->cpu_processor->next_thread != THREAD_NULL) + first_thread = cpu_data_ptr->cpu_processor->next_thread; + else + first_thread = cpu_data_ptr->cpu_processor->idle_thread; + cpu_data_ptr->cpu_active_thread = first_thread; + first_thread->machine.CpuDatap = cpu_data_ptr; + + configure_coresight_registers(cpu_data_ptr); + + flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE); + flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); + (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL); + } + + return KERN_SUCCESS; +} + + +void +cpu_timebase_init(boolean_t from_boot) +{ + cpu_data_t *cdp = getCpuDatap(); + + if (cdp->cpu_get_fiq_handler == NULL) { + cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler; + cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer; + cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer; + cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr; + cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val; + } + + if (!from_boot && (cdp == &BootCpuData)) { + /* + * When we wake from sleep, we have no guarantee about the state + * of the hardware timebase. It may have kept ticking across sleep, or + * it may have reset. + * + * To deal with this, we calculate an offset to the clock that will + * produce a timebase value wake_abstime at the point the boot + * CPU calls cpu_timebase_init on wake. + * + * This ensures that mach_absolute_time() stops ticking across sleep. + */ + rtclock_base_abstime = wake_abstime - ml_get_hwclock(); + } + + cdp->cpu_decrementer = 0x7FFFFFFFUL; + cdp->cpu_timebase = 0x0UL; + cdp->cpu_base_timebase = rtclock_base_abstime; +} + +int +cpu_cluster_id(void) +{ + return (getCpuDatap()->cpu_cluster_id); +} + +__attribute__((noreturn)) +void +ml_arm_sleep(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (cpu_data_ptr == &BootCpuData) { + cpu_data_t *target_cdp; + int cpu; + int max_cpu; + + max_cpu = ml_get_max_cpu_number(); + for (cpu=0; cpu <= max_cpu; cpu++) { + target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cdp == NULL) || (target_cdp == cpu_data_ptr)) + continue; + + while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH); + } + + /* + * Now that the other cores have entered the sleep path, set + * the abstime value we'll use when we resume. + */ + wake_abstime = ml_get_timebase(); + } else { + CleanPoU_Dcache(); + } + + cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; + + if (cpu_data_ptr == &BootCpuData) { +#if WITH_CLASSIC_S2R + // Classic suspend to RAM writes the suspend signature into the + // sleep token buffer so that iBoot knows that it's on the warm + // boot (wake) path (as opposed to the cold boot path). Newer SoC + // do not go through SecureROM/iBoot on the warm boot path. The + // reconfig engine script brings the CPU out of reset at the kernel's + // reset vector which points to the warm boot initialization code. + if(sleepTokenBuffer != (vm_offset_t) NULL) { + platform_cache_shutdown(); + bcopy((const void *)suspend_signature, (void *)sleepTokenBuffer, sizeof(SleepToken)); + } + else { + panic("No sleep token buffer"); + } +#endif + +#if __ARM_GLOBAL_SLEEP_BIT__ + /* Allow other CPUs to go to sleep. */ + arm64_stall_sleep = FALSE; + __builtin_arm_dmb(DMB_ISH); +#endif + + /* Architectural debug state: : + * Grab debug lock EDLAR and clear bit 0 in EDPRCR, + * tell debugger to not prevent power gating . + */ + if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) { + *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY; + *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0; + } + +#if MONOTONIC + mt_sleep(); +#endif /* MONOTONIC */ + /* ARM64-specific preparation */ + arm64_prepare_for_sleep(); + } else { +#if __ARM_GLOBAL_SLEEP_BIT__ + /* + * With the exception of the CPU revisions listed above, our ARM64 CPUs have a + * global register to manage entering deep sleep, as opposed to a per-CPU + * register. We cannot update this register until all CPUs are ready to enter + * deep sleep, because if a CPU executes WFI outside of the deep sleep context + * (by idling), it will hang (due to the side effects of enabling deep sleep), + * which can hang the sleep process or cause memory corruption on wake. + * + * To avoid these issues, we'll stall on this global value, which CPU0 will + * manage. + */ + while (arm64_stall_sleep) { + __builtin_arm_wfe(); + } +#endif + CleanPoU_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t)); + + /* Architectural debug state: : + * Grab debug lock EDLAR and clear bit 0 in EDPRCR, + * tell debugger to not prevent power gating . + */ + if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) { + *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY; + *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0; + } + + /* ARM64-specific preparation */ + arm64_prepare_for_sleep(); + } +} + +void +cpu_machine_idle_init(boolean_t from_boot) +{ + static vm_address_t resume_idle_cpu_paddr = (vm_address_t)NULL; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (from_boot) { + unsigned long jtag = 0; + int wfi_tmp = 1; + uint32_t production = 1; + DTEntry entry; + + if (PE_parse_boot_argn("jtag", &jtag, sizeof (jtag))) { + if (jtag != 0) + idle_enable = FALSE; + else + idle_enable = TRUE; + } else + idle_enable = TRUE; + + PE_parse_boot_argn("wfi", &wfi_tmp, sizeof (wfi_tmp)); + + // bits 7..0 give the wfi type + switch (wfi_tmp & 0xff) { + case 0 : + // disable wfi + wfi = 0; + break; + +#if DEVELOPMENT || DEBUG + case 2 : + // wfi overhead simulation + // 31..16 - wfi delay is us + // 15..8 - flags + // 7..0 - 2 + wfi = 2; + wfi_flags = (wfi_tmp >> 8) & 0xFF; + nanoseconds_to_absolutetime(((wfi_tmp >> 16) & 0xFFFF) * NSEC_PER_MSEC, &wfi_delay); + break; +#endif /* DEVELOPMENT || DEBUG */ + + case 1 : + default : + // do nothing + break; + } + + ResetHandlerData.assist_reset_handler = 0; + ResetHandlerData.cpu_data_entries = ml_static_vtop((vm_offset_t)CpuDataEntries); + +#ifdef MONITOR + monitor_call(MONITOR_SET_ENTRY, (uintptr_t)ml_static_vtop((vm_offset_t)&LowResetVectorBase), 0, 0); +#elif !defined(NO_MONITOR) +#error MONITOR undefined, WFI power gating may not operate correctly +#endif /* MONITOR */ + + // Determine if we are on production or debug chip + if (kSuccess == DTLookupEntry(NULL, "/chosen", &entry)) { + unsigned int size; + void *prop; + + if (kSuccess == DTGetProperty(entry, "effective-production-status-ap", &prop, &size)) + if (size == 4) + bcopy(prop, &production, size); + } + if (!production) { +#if defined(APPLE_ARM64_ARCH_FAMILY) + // Enable coresight debug registers on debug-fused chips + coresight_debug_enabled = TRUE; +#endif + } + + start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu); + resume_idle_cpu_paddr = ml_static_vtop((vm_offset_t)&resume_idle_cpu); + } + +#if WITH_CLASSIC_S2R + if (cpu_data_ptr == &BootCpuData) { + static addr64_t SleepToken_low_paddr = (addr64_t)NULL; + if (sleepTokenBuffer != (vm_offset_t) NULL) { + SleepToken_low_paddr = ml_vtophys(sleepTokenBuffer); + } + else { + panic("No sleep token buffer"); + } + + bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)running_signature), + SleepToken_low_paddr, sizeof(SleepToken)); + flush_dcache((vm_offset_t)SleepToken, sizeof(SleepToken), TRUE); + }; +#endif + + cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr; + clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); +} + +_Atomic uint32_t cpu_idle_count = 0; + +void +machine_track_platform_idle(boolean_t entry) +{ + if (entry) + (void)__c11_atomic_fetch_add(&cpu_idle_count, 1, __ATOMIC_RELAXED); + else + (void)__c11_atomic_fetch_sub(&cpu_idle_count, 1, __ATOMIC_RELAXED); +} + +#if WITH_CLASSIC_S2R +void +sleep_token_buffer_init(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + DTEntry entry; + size_t size; + void **prop; + + if ((cpu_data_ptr == &BootCpuData) && (sleepTokenBuffer == (vm_offset_t) NULL)) { + /* Find the stpage node in the device tree */ + if (kSuccess != DTLookupEntry(0, "stram", &entry)) + return; + + if (kSuccess != DTGetProperty(entry, "reg", (void **)&prop, (unsigned int *)&size)) + return; + + /* Map the page into the kernel space */ + sleepTokenBuffer = ml_io_map(((vm_offset_t *)prop)[0], ((vm_size_t *)prop)[1]); + } +} +#endif + diff --git a/osfmk/arm64/cswitch.s b/osfmk/arm64/cswitch.s new file mode 100644 index 000000000..e3a0cb317 --- /dev/null +++ b/osfmk/arm64/cswitch.s @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include +#include "assym.s" + +/* + * save_general_registers + * + * Saves variable registers to kernel PCB. + * arg0 - thread_kernel_state pointer + * arg1 - Scratch register + */ + +.macro save_general_registers +/* AAPCS-64 Page 14 + * + * A subroutine invocation must preserve the contents of the registers r19-r29 + * and SP. We also save IP0 and IP1, as machine_idle uses IP0 for saving the LR. + */ + stp x16, x17, [$0, SS64_X16] + stp x19, x20, [$0, SS64_X19] + stp x21, x22, [$0, SS64_X21] + stp x23, x24, [$0, SS64_X23] + stp x25, x26, [$0, SS64_X25] + stp x27, x28, [$0, SS64_X27] + stp fp, lr, [$0, SS64_FP] + mov $1, sp + str $1, [$0, SS64_SP] + +/* AAPCS-64 Page 14 + * + * Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine + * calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved + * (or should be preserved by the caller). + */ + str d8, [$0, NS64_D8] + str d9, [$0, NS64_D9] + str d10,[$0, NS64_D10] + str d11,[$0, NS64_D11] + str d12,[$0, NS64_D12] + str d13,[$0, NS64_D13] + str d14,[$0, NS64_D14] + str d15,[$0, NS64_D15] +.endmacro + +/* + * load_general_registers + * + * Loads variable registers from kernel PCB. + * arg0 - thread_kernel_state pointer + * arg1 - Scratch register + */ +.macro load_general_registers + ldp x16, x17, [$0, SS64_X16] + ldp x19, x20, [$0, SS64_X19] + ldp x21, x22, [$0, SS64_X21] + ldp x23, x24, [$0, SS64_X23] + ldp x25, x26, [$0, SS64_X25] + ldp x27, x28, [$0, SS64_X27] + ldp fp, lr, [$0, SS64_FP] + ldr $1, [$0, SS64_SP] + mov sp, $1 + + ldr d8, [$0, NS64_D8] + ldr d9, [$0, NS64_D9] + ldr d10,[$0, NS64_D10] + ldr d11,[$0, NS64_D11] + ldr d12,[$0, NS64_D12] + ldr d13,[$0, NS64_D13] + ldr d14,[$0, NS64_D14] + ldr d15,[$0, NS64_D15] +.endmacro + +/* + * set_thread_registers + * + * Updates thread registers during context switch + * arg0 - New thread pointer + * arg1 - Scratch register + * arg2 - Scratch register + */ +.macro set_thread_registers + msr TPIDR_EL1, $0 // Write new thread pointer to TPIDR_EL1 + ldr $1, [$0, TH_CTH_SELF] // Get cthread pointer + mrs $2, TPIDRRO_EL0 // Extract cpu number from TPIDRRO_EL0 + and $2, $2, #(MACHDEP_CPUNUM_MASK) + orr $2, $1, $2 // Save new cthread/cpu to TPIDRRO_EL0 + msr TPIDRRO_EL0, $2 + ldr $1, [$0, TH_CTH_DATA] // Get new cthread data pointer + msr TPIDR_EL0, $1 // Save data pointer to TPIDRRW_EL0 + /* ARM64_TODO Reserve x18 until we decide what to do with it */ + mov x18, $1 // ... and trash reserved x18 +.endmacro + + +/* + * void machine_load_context(thread_t thread) + * + * Load the context for the first thread to run on a + * cpu, and go. + */ + .text + .align 2 + .globl EXT(machine_load_context) + +LEXT(machine_load_context) + set_thread_registers x0, x1, x2 + ldr x1, [x0, TH_KSTACKPTR] // Get top of kernel stack + load_general_registers x1, x2 + mov x0, xzr // Clear argument to thread_continue + ret + +/* + * void Call_continuation( void (*continuation)(void), + * void *param, + * wait_result_t wresult, + * vm_offset_t stack_ptr) + */ + .text + .align 5 + .globl EXT(Call_continuation) + +LEXT(Call_continuation) + mrs x4, TPIDR_EL1 // Get the current thread pointer + + /* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */ + ldr x5, [x4, TH_KSTACKPTR] // Get the top of the kernel stack + mov sp, x5 // Set stack pointer + + mov fp, xzr // Clear the frame pointer + mov x4, x0 // Load the continuation + mov x0, x1 // Set the first parameter + mov x1, x2 // Set the wait result arg + blr x4 // Branch to the continuation + mrs x0, TPIDR_EL1 // Get the current thread pointer + b EXT(thread_terminate) // Kill the thread + + +/* + * thread_t Switch_context(thread_t old, + * void (*cont)(void), + * thread_t new) + */ + .text + .align 5 + .globl EXT(Switch_context) + +LEXT(Switch_context) + cbnz x1, Lswitch_threads // Skip saving old state if blocking on continuation + ldr x3, [x0, TH_KSTACKPTR] // Get the old kernel stack top + save_general_registers x3, x4 +Lswitch_threads: + set_thread_registers x2, x3, x4 + ldr x3, [x2, TH_KSTACKPTR] + load_general_registers x3, x4 + ret + +/* + * thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor) + * + */ + .text + .align 5 + .globl EXT(Shutdown_context) + +LEXT(Shutdown_context) + mrs x10, TPIDR_EL1 // Get thread pointer + ldr x11, [x10, TH_KSTACKPTR] // Get the top of the kernel stack + save_general_registers x11, x12 + msr DAIFSet, #(DAIFSC_FIQF | DAIFSC_IRQF) // Disable interrupts + ldr x11, [x10, ACT_CPUDATAP] // Get current cpu + ldr x12, [x11, CPU_ISTACKPTR] // Switch to interrupt stack + mov sp, x12 + b EXT(cpu_doshutdown) + + +/* + * thread_t Idle_context(void) + * + */ + .text + .align 5 + .globl EXT(Idle_context) + +LEXT(Idle_context) + mrs x0, TPIDR_EL1 // Get thread pointer + ldr x1, [x0, TH_KSTACKPTR] // Get the top of the kernel stack + save_general_registers x1, x2 + ldr x1, [x0, ACT_CPUDATAP] // Get current cpu + ldr x2, [x1, CPU_ISTACKPTR] // Switch to interrupt stack + mov sp, x2 + b EXT(cpu_idle) + +/* + * thread_t Idle_context(void) + * + */ + .text + .align 5 + .globl EXT(Idle_load_context) + +LEXT(Idle_load_context) + mrs x0, TPIDR_EL1 // Get thread pointer + ldr x1, [x0, TH_KSTACKPTR] // Get the top of the kernel stack + load_general_registers x1, x2 + ret + + .align 2 + .globl EXT(machine_set_current_thread) +LEXT(machine_set_current_thread) + set_thread_registers x0, x1, x2 + ret diff --git a/osfmk/arm64/dbgwrap.c b/osfmk/arm64/dbgwrap.c new file mode 100644 index 000000000..282432450 --- /dev/null +++ b/osfmk/arm64/dbgwrap.c @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include + +#define DBGWRAP_REG_OFFSET 0 +#define DBGWRAP_DBGHALT (1ULL << 31) +#define DBGWRAP_DBGACK (1ULL << 28) + +#define EDDTRRX_REG_OFFSET 0x80 +#define EDITR_REG_OFFSET 0x84 +#define EDSCR_REG_OFFSET 0x88 +#define EDSCR_TXFULL (1ULL << 29) +#define EDSCR_ITE (1ULL << 24) +#define EDSCR_MA (1ULL << 20) +#define EDSCR_ERR (1ULL << 6) +#define EDDTRTX_REG_OFFSET 0x8C +#define EDRCR_REG_OFFSET 0x90 +#define EDRCR_CSE (1ULL << 2) +#define EDPRSR_REG_OFFSET 0x314 +#define EDPRSR_OSLK (1ULL << 5) + +#define MAX_EDITR_RETRIES 16 + +/* Older SoCs require 32-bit accesses for DBGWRAP; + * newer ones require 64-bit accesses. */ +#ifdef HAS_32BIT_DBGWRAP +typedef uint32_t dbgwrap_reg_t; +#else +typedef uint64_t dbgwrap_reg_t; +#endif + +#if DEVELOPMENT || DEBUG +#define MAX_STUFFED_INSTRS 64 +uint32_t stuffed_instrs[MAX_STUFFED_INSTRS]; +volatile uint32_t stuffed_instr_count = 0; +#endif + +static volatile uint32_t halt_from_cpu = (uint32_t)-1; + +boolean_t +ml_dbgwrap_cpu_is_halted(int cpu_index) +{ + cpu_data_t *cdp = cpu_datap(cpu_index); + if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) + return FALSE; + + return ((*(volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET) & DBGWRAP_DBGACK) != 0); +} + +dbgwrap_status_t +ml_dbgwrap_wait_cpu_halted(int cpu_index, uint64_t timeout_ns) +{ + cpu_data_t *cdp = cpu_datap(cpu_index); + if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) + return DBGWRAP_ERR_UNSUPPORTED; + + volatile dbgwrap_reg_t *dbgWrapReg = (volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET); + + uint64_t interval; + nanoseconds_to_absolutetime(timeout_ns, &interval); + uint64_t deadline = mach_absolute_time() + interval; + while (!(*dbgWrapReg & DBGWRAP_DBGACK)) { + if (mach_absolute_time() > deadline) + return DBGWRAP_ERR_HALT_TIMEOUT; + } + + return DBGWRAP_SUCCESS; +} + +dbgwrap_status_t +ml_dbgwrap_halt_cpu(int cpu_index, uint64_t timeout_ns) +{ + cpu_data_t *cdp = cpu_datap(cpu_index); + if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) + return DBGWRAP_ERR_UNSUPPORTED; + + /* Only one cpu is allowed to initiate the halt sequence, to prevent cpus from cross-halting + * each other. The first cpu to request a halt may then halt any and all other cpus besides itself. */ + int curcpu = cpu_number(); + if (cpu_index == curcpu) + return DBGWRAP_ERR_SELF_HALT; + + if (!hw_compare_and_store((uint32_t)-1, (unsigned int)curcpu, &halt_from_cpu) && + (halt_from_cpu != (uint32_t)curcpu)) + return DBGWRAP_ERR_INPROGRESS; + + volatile dbgwrap_reg_t *dbgWrapReg = (volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET); + + if (ml_dbgwrap_cpu_is_halted(cpu_index)) + return DBGWRAP_WARN_ALREADY_HALTED; + + /* Clear all other writable bits besides dbgHalt; none of the power-down or reset bits must be set. */ + *dbgWrapReg = DBGWRAP_DBGHALT; + + if (timeout_ns != 0) { + dbgwrap_status_t stat = ml_dbgwrap_wait_cpu_halted(cpu_index, timeout_ns); + return stat; + } + else + return DBGWRAP_SUCCESS; +} + +static void +ml_dbgwrap_stuff_instr(cpu_data_t *cdp, uint32_t instr, uint64_t timeout_ns, dbgwrap_status_t *status) +{ + if (*status < 0) + return; + + volatile uint32_t *editr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDITR_REG_OFFSET); + volatile uint32_t *edscr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDSCR_REG_OFFSET); + volatile uint32_t *edrcr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDRCR_REG_OFFSET); + + int retries = 0; + + uint64_t interval; + nanoseconds_to_absolutetime(timeout_ns, &interval); + uint64_t deadline = mach_absolute_time() + interval; + +#if DEVELOPMENT || DEBUG + uint32_t stuffed_instr_index = hw_atomic_add(&stuffed_instr_count, 1); + stuffed_instrs[(stuffed_instr_index - 1) % MAX_STUFFED_INSTRS] = instr; +#endif + + do { + *editr = instr; + volatile uint32_t edscr_val; + while (!((edscr_val = *edscr) & EDSCR_ITE)) { + if (mach_absolute_time() > deadline) { + *status = DBGWRAP_ERR_INSTR_TIMEOUT; + return; + } + if (edscr_val & EDSCR_ERR) + break; + } + if (edscr_val & EDSCR_ERR) { + /* If memory access mode was enable by a debugger, clear it. + * This will cause ERR to be set on any attempt to use EDITR. */ + if (edscr_val & EDSCR_MA) + *edscr = edscr_val & ~EDSCR_MA; + *edrcr = EDRCR_CSE; + ++retries; + } else + break; + } while (retries < MAX_EDITR_RETRIES); + + if (retries >= MAX_EDITR_RETRIES) { + *status = DBGWRAP_ERR_INSTR_ERROR; + return; + } +} + +static uint64_t +ml_dbgwrap_read_dtr(cpu_data_t *cdp, uint64_t timeout_ns, dbgwrap_status_t *status) +{ + if (*status < 0) + return 0; + + uint64_t interval; + nanoseconds_to_absolutetime(timeout_ns, &interval); + uint64_t deadline = mach_absolute_time() + interval; + + /* Per armv8 debug spec, writes to DBGDTR_EL0 on target cpu will set EDSCR.TXFull, + * with bits 63:32 available in EDDTRRX and bits 31:0 availabe in EDDTRTX. */ + volatile uint32_t *edscr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDSCR_REG_OFFSET); + + while (!(*edscr & EDSCR_TXFULL)) { + if (*edscr & EDSCR_ERR) { + *status = DBGWRAP_ERR_INSTR_ERROR; + return 0; + } + if (mach_absolute_time() > deadline) { + *status = DBGWRAP_ERR_INSTR_TIMEOUT; + return 0; + } + } + + uint32_t dtrrx = *((volatile uint32_t*)(cdp->coresight_base[CORESIGHT_ED] + EDDTRRX_REG_OFFSET)); + uint32_t dtrtx = *((volatile uint32_t*)(cdp->coresight_base[CORESIGHT_ED] + EDDTRTX_REG_OFFSET)); + + return (((uint64_t)dtrrx << 32) | dtrtx); +} + +dbgwrap_status_t +ml_dbgwrap_halt_cpu_with_state(int cpu_index, uint64_t timeout_ns, dbgwrap_thread_state_t *state) +{ + cpu_data_t *cdp = cpu_datap(cpu_index); + if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_ED] == 0)) + return DBGWRAP_ERR_UNSUPPORTED; + + /* Ensure memory-mapped coresight registers can be written */ + *((volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR)) = ARM_DBG_LOCK_ACCESS_KEY; + + dbgwrap_status_t status = ml_dbgwrap_halt_cpu(cpu_index, timeout_ns); + + /* A core that is not fully powered (e.g. idling in wfi) can still be halted; the dbgwrap + * register and certain coresight registers such EDPRSR are in the always-on domain. + * However, EDSCR/EDITR are not in the always-on domain and will generate a parity abort + * on read. EDPRSR can be safely read in all cases, and the OS lock defaults to being set + * but we clear it first thing, so use that to detect the offline state. */ + if (*((volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDPRSR_REG_OFFSET)) & EDPRSR_OSLK) { + bzero(state, sizeof(*state)); + return DBGWRAP_WARN_CPU_OFFLINE; + } + + uint32_t instr; + + for (unsigned int i = 0; i < (sizeof(state->x) / sizeof(state->x[0])); ++i) { + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | i; // msr DBGDTR0, x + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->x[i] = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + } + + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 29; // msr DBGDTR0, fp + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->fp = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 30; // msr DBGDTR0, lr + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->lr = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + + /* Stack pointer (x31) can't be used as a register operand for msr; register 31 is treated as xzr + * rather than sp when used as the transfer operand there. Instead, load sp into a GPR + * we've already saved off and then store that register in the DTR. I've chosen x18 + * as the temporary GPR since it's reserved by the arm64 ABI and unused by xnu, so overwriting + * it poses the least risk of causing trouble for external debuggers. */ + + instr = (0x91U << 24) | (31 << 5) | 18; // mov x18, sp + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18 + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->sp = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + + /* reading PC (e.g. through adr) is undefined in debug state. Instead use DLR_EL0, + * which contains PC at time of entry into debug state.*/ + + instr = (0xD53U << 20) | (1 << 19) | (3 << 16) | (4 << 12) | (5 << 8) | (1 << 5) | 18; // mrs x18, DLR_EL0 + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18 + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->pc = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + + /* reading CPSR is undefined in debug state. Instead use DSPSR_EL0, + * which contains CPSR at time of entry into debug state.*/ + instr = (0xD53U << 20) | (1 << 19) | (3 << 16) | (4 << 12) | (5 << 8) | 18; // mrs x18, DSPSR_EL0 + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18 + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->cpsr = (uint32_t)ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + + return status; +} + + diff --git a/osfmk/arm64/genassym.c b/osfmk/arm64/genassym.c new file mode 100644 index 000000000..5c06aabce --- /dev/null +++ b/osfmk/arm64/genassym.c @@ -0,0 +1,430 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#include + +#include + +/* + * Pass field offsets to assembly code. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if CONFIG_DTRACE +#define NEED_DTRACE_DEFS +#include <../bsd/sys/lockstat.h> +#endif /* CONFIG_DTRACE */ + +/* + * genassym.c is used to produce an + * assembly file which, intermingled with unuseful assembly code, + * has all the necessary definitions emitted. This assembly file is + * then postprocessed with sed to extract only these definitions + * and thus the final assyms.s is created. + * + * This convoluted means is necessary since the structure alignment + * and packing may be different between the host machine and the + * target so we are forced into using the cross compiler to generate + * the values, but we cannot run anything on the target machine. + */ + +#define DECLARE(SYM,VAL) \ + __asm("DEFINITION__define__" SYM ":\t .ascii \"%0\"" : : "n" ((u_long)(VAL))) + + +int main( + int argc, + char ** argv); + +int +main( + int argc, + char **argv) +{ + + DECLARE("T_PREFETCH_ABT", T_PREFETCH_ABT); + DECLARE("T_DATA_ABT", T_DATA_ABT); + + DECLARE("AST_URGENT", AST_URGENT); + DECLARE("AST_PREEMPTION", AST_PREEMPTION); + + DECLARE("TH_RECOVER", offsetof(struct thread, recover)); + DECLARE("TH_CONTINUATION", offsetof(struct thread, continuation)); + DECLARE("TH_KERNEL_STACK", offsetof(struct thread, kernel_stack)); + DECLARE("TH_KSTACKPTR", offsetof(struct thread, machine.kstackptr)); + DECLARE("THREAD_UTHREAD", offsetof(struct thread, uthread)); + + DECLARE("TASK_MACH_EXC_PORT", + offsetof(struct task, exc_actions[EXC_MACH_SYSCALL].port)); + + /* These fields are being added on demand */ + DECLARE("ACT_TASK", offsetof(struct thread, task)); + DECLARE("ACT_CONTEXT", offsetof(struct thread, machine.contextData)); + DECLARE("ACT_UPCB", offsetof(struct thread, machine.upcb)); +// DECLARE("ACT_PCBDATA", offsetof(struct thread, machine.contextData.ss)); + DECLARE("ACT_UNEON", offsetof(struct thread, machine.uNeon)); +// DECLARE("ACT_NEONDATA", offsetof(struct thread, machine.contextData.ns)); + DECLARE("TH_CTH_SELF", offsetof(struct thread, machine.cthread_self)); + DECLARE("TH_CTH_DATA", offsetof(struct thread, machine.cthread_data)); + DECLARE("ACT_PREEMPT_CNT", offsetof(struct thread, machine.preemption_count)); + DECLARE("ACT_CPUDATAP", offsetof(struct thread, machine.CpuDatap)); + DECLARE("ACT_MAP", offsetof(struct thread, map)); + DECLARE("ACT_DEBUGDATA", offsetof(struct thread, machine.DebugData)); + DECLARE("TH_IOTIER_OVERRIDE", offsetof(struct thread, iotier_override)); + DECLARE("TH_RWLOCK_CNT", offsetof(struct thread, rwlock_count)); + DECLARE("TH_SCHED_FLAGS", offsetof(struct thread, sched_flags)); + DECLARE("TH_SFLAG_RW_PROMOTED_BIT", TH_SFLAG_RW_PROMOTED_BIT); + + DECLARE("TH_MACH_SYSCALLS", offsetof(struct thread, syscalls_mach)); + DECLARE("TH_UNIX_SYSCALLS", offsetof(struct thread, syscalls_unix)); + DECLARE("TASK_BSD_INFO", offsetof(struct task, bsd_info)); + + DECLARE("MACH_TRAP_TABLE_COUNT", MACH_TRAP_TABLE_COUNT); + DECLARE("MACH_TRAP_TABLE_ENTRY_SIZE", sizeof(mach_trap_t)); + + DECLARE("MAP_PMAP", offsetof(struct _vm_map, pmap)); + + DECLARE("ARM_CONTEXT_SIZE", sizeof(arm_context_t)); + + DECLARE("CONTEXT_SS", offsetof(arm_context_t, ss)); + DECLARE("SS_FLAVOR", offsetof(arm_context_t, ss.ash.flavor)); + DECLARE("ARM_SAVED_STATE32", ARM_SAVED_STATE32); + DECLARE("ARM_SAVED_STATE64", ARM_SAVED_STATE64); + DECLARE("ARM_SAVED_STATE64_COUNT", ARM_SAVED_STATE64_COUNT); + + DECLARE("SS32_W0", offsetof(arm_context_t, ss.ss_32.r[0])); + DECLARE("SS32_W2", offsetof(arm_context_t, ss.ss_32.r[2])); + DECLARE("SS32_W4", offsetof(arm_context_t, ss.ss_32.r[4])); + DECLARE("SS32_W6", offsetof(arm_context_t, ss.ss_32.r[6])); + DECLARE("SS32_W8", offsetof(arm_context_t, ss.ss_32.r[8])); + DECLARE("SS32_W10", offsetof(arm_context_t, ss.ss_32.r[10])); + DECLARE("SS32_W12", offsetof(arm_context_t, ss.ss_32.r[12])); + DECLARE("SS32_SP", offsetof(arm_context_t, ss.ss_32.sp)); + DECLARE("SS32_LR", offsetof(arm_context_t, ss.ss_32.lr)); + DECLARE("SS32_PC", offsetof(arm_context_t, ss.ss_32.pc)); + DECLARE("SS32_CPSR", offsetof(arm_context_t, ss.ss_32.cpsr)); + DECLARE("SS32_VADDR", offsetof(arm_context_t, ss.ss_32.far)); + DECLARE("SS32_STATUS", offsetof(arm_context_t, ss.ss_32.esr)); + + DECLARE("SS64_X0", offsetof(arm_context_t, ss.ss_64.x[0])); + DECLARE("SS64_X2", offsetof(arm_context_t, ss.ss_64.x[2])); + DECLARE("SS64_X4", offsetof(arm_context_t, ss.ss_64.x[4])); + DECLARE("SS64_X6", offsetof(arm_context_t, ss.ss_64.x[6])); + DECLARE("SS64_X8", offsetof(arm_context_t, ss.ss_64.x[8])); + DECLARE("SS64_X10", offsetof(arm_context_t, ss.ss_64.x[10])); + DECLARE("SS64_X12", offsetof(arm_context_t, ss.ss_64.x[12])); + DECLARE("SS64_X14", offsetof(arm_context_t, ss.ss_64.x[14])); + DECLARE("SS64_X16", offsetof(arm_context_t, ss.ss_64.x[16])); + DECLARE("SS64_X18", offsetof(arm_context_t, ss.ss_64.x[18])); + DECLARE("SS64_X19", offsetof(arm_context_t, ss.ss_64.x[19])); + DECLARE("SS64_X20", offsetof(arm_context_t, ss.ss_64.x[20])); + DECLARE("SS64_X21", offsetof(arm_context_t, ss.ss_64.x[21])); + DECLARE("SS64_X22", offsetof(arm_context_t, ss.ss_64.x[22])); + DECLARE("SS64_X23", offsetof(arm_context_t, ss.ss_64.x[23])); + DECLARE("SS64_X24", offsetof(arm_context_t, ss.ss_64.x[24])); + DECLARE("SS64_X25", offsetof(arm_context_t, ss.ss_64.x[25])); + DECLARE("SS64_X26", offsetof(arm_context_t, ss.ss_64.x[26])); + DECLARE("SS64_X27", offsetof(arm_context_t, ss.ss_64.x[27])); + DECLARE("SS64_X28", offsetof(arm_context_t, ss.ss_64.x[28])); + DECLARE("SS64_FP", offsetof(arm_context_t, ss.ss_64.fp)); + DECLARE("SS64_LR", offsetof(arm_context_t, ss.ss_64.lr)); + DECLARE("SS64_SP", offsetof(arm_context_t, ss.ss_64.sp)); + DECLARE("SS64_PC", offsetof(arm_context_t, ss.ss_64.pc)); + DECLARE("SS64_CPSR", offsetof(arm_context_t, ss.ss_64.cpsr)); + DECLARE("SS64_FAR", offsetof(arm_context_t, ss.ss_64.far)); + DECLARE("SS64_ESR", offsetof(arm_context_t, ss.ss_64.esr)); + + DECLARE("CONTEXT_NS", offsetof(arm_context_t, ns)); + DECLARE("NS_FLAVOR", offsetof(arm_context_t, ns.nsh.flavor)); + DECLARE("NS_COUNT", offsetof(arm_context_t, ns.nsh.count)); + DECLARE("ARM_NEON_SAVED_STATE32", ARM_NEON_SAVED_STATE32); + DECLARE("ARM_NEON_SAVED_STATE64", ARM_NEON_SAVED_STATE64); + DECLARE("ARM_NEON_SAVED_STATE64_COUNT", ARM_NEON_SAVED_STATE64_COUNT); + + DECLARE("NS32_Q0", offsetof(arm_context_t, ns.ns_32.v.q[0])); + DECLARE("NS32_Q2", offsetof(arm_context_t, ns.ns_32.v.q[2])); + DECLARE("NS32_Q4", offsetof(arm_context_t, ns.ns_32.v.q[4])); + DECLARE("NS32_Q6", offsetof(arm_context_t, ns.ns_32.v.q[6])); + DECLARE("NS32_Q8", offsetof(arm_context_t, ns.ns_32.v.q[8])); + DECLARE("NS32_Q10", offsetof(arm_context_t, ns.ns_32.v.q[10])); + DECLARE("NS32_Q12", offsetof(arm_context_t, ns.ns_32.v.q[12])); + DECLARE("NS32_Q14", offsetof(arm_context_t, ns.ns_32.v.q[14])); + DECLARE("NS32_FPSR", offsetof(arm_context_t, ns.ns_32.fpsr)); + DECLARE("NS32_FPCR", offsetof(arm_context_t, ns.ns_32.fpcr)); + + DECLARE("NS64_D8", offsetof(arm_context_t, ns.ns_64.v.d[8])); + DECLARE("NS64_D9", offsetof(arm_context_t, ns.ns_64.v.d[9])); + DECLARE("NS64_D10", offsetof(arm_context_t, ns.ns_64.v.d[10])); + DECLARE("NS64_D11", offsetof(arm_context_t, ns.ns_64.v.d[11])); + DECLARE("NS64_D12", offsetof(arm_context_t, ns.ns_64.v.d[12])); + DECLARE("NS64_D13", offsetof(arm_context_t, ns.ns_64.v.d[13])); + DECLARE("NS64_D14", offsetof(arm_context_t, ns.ns_64.v.d[14])); + DECLARE("NS64_D15", offsetof(arm_context_t, ns.ns_64.v.d[15])); + + DECLARE("NS64_Q0", offsetof(arm_context_t, ns.ns_64.v.q[0])); + DECLARE("NS64_Q2", offsetof(arm_context_t, ns.ns_64.v.q[2])); + DECLARE("NS64_Q4", offsetof(arm_context_t, ns.ns_64.v.q[4])); + DECLARE("NS64_Q6", offsetof(arm_context_t, ns.ns_64.v.q[6])); + DECLARE("NS64_Q8", offsetof(arm_context_t, ns.ns_64.v.q[8])); + DECLARE("NS64_Q10", offsetof(arm_context_t, ns.ns_64.v.q[10])); + DECLARE("NS64_Q12", offsetof(arm_context_t, ns.ns_64.v.q[12])); + DECLARE("NS64_Q14", offsetof(arm_context_t, ns.ns_64.v.q[14])); + DECLARE("NS64_Q16", offsetof(arm_context_t, ns.ns_64.v.q[16])); + DECLARE("NS64_Q18", offsetof(arm_context_t, ns.ns_64.v.q[18])); + DECLARE("NS64_Q20", offsetof(arm_context_t, ns.ns_64.v.q[20])); + DECLARE("NS64_Q22", offsetof(arm_context_t, ns.ns_64.v.q[22])); + DECLARE("NS64_Q24", offsetof(arm_context_t, ns.ns_64.v.q[24])); + DECLARE("NS64_Q26", offsetof(arm_context_t, ns.ns_64.v.q[26])); + DECLARE("NS64_Q28", offsetof(arm_context_t, ns.ns_64.v.q[28])); + DECLARE("NS64_Q30", offsetof(arm_context_t, ns.ns_64.v.q[30])); + DECLARE("NS64_FPSR", offsetof(arm_context_t, ns.ns_64.fpsr)); + DECLARE("NS64_FPCR", offsetof(arm_context_t, ns.ns_64.fpcr)); + + DECLARE("PGBYTES", ARM_PGBYTES); + DECLARE("PGSHIFT", ARM_PGSHIFT); + DECLARE("PGMASK", ARM_PGMASK); + + + DECLARE("VM_MIN_ADDRESS", VM_MIN_ADDRESS); + DECLARE("VM_MAX_ADDRESS", VM_MAX_ADDRESS); + DECLARE("VM_MIN_KERNEL_ADDRESS", VM_MIN_KERNEL_ADDRESS); + DECLARE("VM_MAX_KERNEL_ADDRESS", VM_MAX_KERNEL_ADDRESS); + DECLARE("KERNELBASE", VM_MIN_KERNEL_ADDRESS); + DECLARE("KERNEL_STACK_SIZE", KERNEL_STACK_SIZE); + DECLARE("TBI_MASK", TBI_MASK); + + DECLARE("KERN_INVALID_ADDRESS", KERN_INVALID_ADDRESS); + + + DECLARE("MAX_CPUS", MAX_CPUS); + + DECLARE("cdeSize", + sizeof(struct cpu_data_entry)); + + DECLARE("cdSize", + sizeof(struct cpu_data)); + + DECLARE("CPU_ACTIVE_THREAD", + offsetof(cpu_data_t, cpu_active_thread)); + DECLARE("CPU_ACTIVE_STACK", + offsetof(cpu_data_t, cpu_active_stack)); + DECLARE("CPU_ISTACKPTR", + offsetof(cpu_data_t, istackptr)); + DECLARE("CPU_INTSTACK_TOP", + offsetof(cpu_data_t, intstack_top)); + DECLARE("CPU_EXCEPSTACKPTR", + offsetof(cpu_data_t, excepstackptr)); + DECLARE("CPU_EXCEPSTACK_TOP", + offsetof(cpu_data_t, excepstack_top)); + DECLARE("CPU_FIQSTACKPTR", + offsetof(cpu_data_t, fiqstackptr)); + DECLARE("CPU_FIQSTACK_TOP", + offsetof(cpu_data_t, fiqstack_top)); + DECLARE("CPU_NUMBER_GS", + offsetof(cpu_data_t,cpu_number)); + DECLARE("CPU_IDENT", + offsetof(cpu_data_t,cpu_ident)); + DECLARE("CPU_RUNNING", + offsetof(cpu_data_t,cpu_running)); + DECLARE("CPU_MCOUNT_OFF", + offsetof(cpu_data_t,cpu_mcount_off)); + DECLARE("CPU_PENDING_AST", + offsetof(cpu_data_t,cpu_pending_ast)); + DECLARE("CPU_PROCESSOR", + offsetof(cpu_data_t,cpu_processor)); + DECLARE("CPU_CACHE_DISPATCH", + offsetof(cpu_data_t,cpu_cache_dispatch)); + DECLARE("CPU_BASE_TIMEBASE", + offsetof(cpu_data_t,cpu_base_timebase)); + DECLARE("CPU_DECREMENTER", + offsetof(cpu_data_t,cpu_decrementer)); + DECLARE("CPU_GET_DECREMENTER_FUNC", + offsetof(cpu_data_t,cpu_get_decrementer_func)); + DECLARE("CPU_SET_DECREMENTER_FUNC", + offsetof(cpu_data_t,cpu_set_decrementer_func)); + DECLARE("CPU_GET_FIQ_HANDLER", + offsetof(cpu_data_t,cpu_get_fiq_handler)); + DECLARE("CPU_TBD_HARDWARE_ADDR", + offsetof(cpu_data_t,cpu_tbd_hardware_addr)); + DECLARE("CPU_TBD_HARDWARE_VAL", + offsetof(cpu_data_t,cpu_tbd_hardware_val)); + DECLARE("CPU_INT_STATE", + offsetof(cpu_data_t,cpu_int_state)); + DECLARE("INTERRUPT_HANDLER", + offsetof(cpu_data_t,interrupt_handler)); + DECLARE("INTERRUPT_TARGET", + offsetof(cpu_data_t,interrupt_target)); + DECLARE("INTERRUPT_REFCON", + offsetof(cpu_data_t,interrupt_refCon)); + DECLARE("INTERRUPT_NUB", + offsetof(cpu_data_t,interrupt_nub)); + DECLARE("INTERRUPT_SOURCE", + offsetof(cpu_data_t,interrupt_source)); + DECLARE("CPU_USER_DEBUG", + offsetof(cpu_data_t, cpu_user_debug)); + DECLARE("CPU_STAT_IRQ", + offsetof(cpu_data_t, cpu_stat.irq_ex_cnt)); + DECLARE("CPU_STAT_IRQ_WAKE", + offsetof(cpu_data_t, cpu_stat.irq_ex_cnt_wake)); + DECLARE("CPU_RESET_HANDLER", + offsetof(cpu_data_t, cpu_reset_handler)); + DECLARE("CPU_RESET_ASSIST", + offsetof(cpu_data_t, cpu_reset_assist)); + DECLARE("CPU_REGMAP_PADDR", + offsetof(cpu_data_t, cpu_regmap_paddr)); + DECLARE("CPU_PHYS_ID", + offsetof(cpu_data_t, cpu_phys_id)); + DECLARE("RTCLOCK_DATAP", + offsetof(cpu_data_t, rtclock_datap)); + + DECLARE("RTCLOCKDataSize", + sizeof(rtclock_data_t)); + DECLARE("RTCLOCK_ADJ_ABSTIME_LOW", + offsetof(rtclock_data_t, rtc_adj.abstime_val.low)); + DECLARE("RTCLOCK_ADJ_ABSTIME_HIGH", + offsetof(rtclock_data_t, rtc_adj.abstime_val.high)); + DECLARE("RTCLOCK_BASE_ABSTIME_LOW", + offsetof(rtclock_data_t, rtc_base.abstime_val.low)); + DECLARE("RTCLOCK_BASE_ABSTIME_HIGH", + offsetof(rtclock_data_t, rtc_base.abstime_val.high)); + + DECLARE("SIGPdec", SIGPdec); + + DECLARE("rhdSize", + sizeof(struct reset_handler_data)); +#if WITH_CLASSIC_S2R || !__arm64__ + DECLARE("stSize", + sizeof(SleepToken)); +#endif + + DECLARE("CPU_DATA_ENTRIES", offsetof(struct reset_handler_data, cpu_data_entries)); + DECLARE("ASSIST_RESET_HANDLER", offsetof(struct reset_handler_data, assist_reset_handler)); + + DECLARE("CPU_DATA_PADDR", offsetof(struct cpu_data_entry, cpu_data_paddr)); + + + DECLARE("INTSTACK_SIZE", INTSTACK_SIZE); + + DECLARE("TIMER_TSTAMP", + offsetof(struct timer, tstamp)); + DECLARE("THREAD_TIMER", + offsetof(struct processor, processor_data.thread_timer)); + DECLARE("KERNEL_TIMER", + offsetof(struct processor, processor_data.kernel_timer)); + DECLARE("SYSTEM_STATE", + offsetof(struct processor, processor_data.system_state)); + DECLARE("USER_STATE", + offsetof(struct processor, processor_data.user_state)); + DECLARE("CURRENT_STATE", + offsetof(struct processor, processor_data.current_state)); + + DECLARE("SYSTEM_TIMER", + offsetof(struct thread, system_timer)); + DECLARE("USER_TIMER", + offsetof(struct thread, user_timer)); + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + DECLARE("PRECISE_USER_KERNEL_TIME", + offsetof(struct thread, precise_user_kernel_time)); +#endif + + DECLARE("BA_VIRT_BASE", + offsetof(struct boot_args, virtBase)); + DECLARE("BA_PHYS_BASE", + offsetof(struct boot_args, physBase)); + DECLARE("BA_MEM_SIZE", + offsetof(struct boot_args, memSize)); + DECLARE("BA_TOP_OF_KERNEL_DATA", + offsetof(struct boot_args, topOfKernelData)); + DECLARE("BA_DEVICE_TREE", + offsetof(struct boot_args, deviceTreeP)); + DECLARE("BA_DEVICE_TREE_LENGTH", + offsetof(struct boot_args, deviceTreeLength)); + + DECLARE("ENTROPY_INDEX_PTR", + offsetof(entropy_data_t, index_ptr)); + DECLARE("ENTROPY_BUFFER", + offsetof(entropy_data_t, buffer)); + DECLARE("ENTROPY_DATA_SIZE", sizeof(struct entropy_data)); + + DECLARE("SR_RESTORE_TCR_EL1", offsetof(struct sysreg_restore, tcr_el1)); + + + return (0); +} diff --git a/osfmk/arm64/kpc.c b/osfmk/arm64/kpc.c new file mode 100644 index 000000000..b1eae91fe --- /dev/null +++ b/osfmk/arm64/kpc.c @@ -0,0 +1,1135 @@ +/* + * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if MONOTONIC +#include +#endif /* MONOTONIC */ + +/* + * PMCs 8 and 9 were added to Hurricane and to maintain the existing bit + * positions of the other PMCs, their configuration bits start at position 32. + */ +#define PMCR_PMC_8_9_OFFSET (32) +#define PMCR_PMC_8_9_SHIFT(PMC) (((PMC) - 8) + PMCR_PMC_8_9_OFFSET) +#define PMCR_PMC_SHIFT(PMC) (((PMC) <= 7) ? (PMC) : \ + PMCR_PMC_8_9_SHIFT(PMC)) + +/* + * PMCR0 controls enabling, interrupts, and overflow of performance counters. + */ + +/* PMC is enabled */ +#define PMCR0_PMC_ENABLE_MASK(PMC) (UINT64_C(0x1) << PMCR_PMC_SHIFT(PMC)) +#define PMCR0_PMC_DISABLE_MASK(PMC) (~PMCR0_PMC_ENABLE_MASK(PMC)) + +/* how interrupts are generated on PMIs */ +#define PMCR0_INTGEN_SHIFT (8) +#define PMCR0_INTGEN_MASK (UINT64_C(0x7) << PMCR0_INTGEN_SHIFT) +#define PMCR0_INTGEN_OFF (UINT64_C(0) << PMCR0_INTGEN_SHIFT) +#define PMCR0_INTGEN_PMI (UINT64_C(1) << PMCR0_INTGEN_SHIFT) +#define PMCR0_INTGEN_AIC (UINT64_C(2) << PMCR0_INTGEN_SHIFT) +#define PMCR0_INTGEN_DBG_HLT (UINT64_C(3) << PMCR0_INTGEN_SHIFT) +#define PMCR0_INTGEN_FIQ (UINT64_C(4) << PMCR0_INTGEN_SHIFT) + +/* 10 unused */ + +/* set by hardware if PMI was generated */ +#define PMCR0_PMAI_SHIFT (11) +#define PMCR0_PMAI_MASK (UINT64_C(1) << PMCR0_PMAI_SHIFT) + +/* overflow on a PMC generates an interrupt */ +#define PMCR0_PMI_OFFSET (12) +#define PMCR0_PMI_SHIFT(PMC) (PMCR0_PMI_OFFSET + PMCR_PMC_SHIFT(PMC)) +#define PMCR0_PMI_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR0_PMI_SHIFT(PMC)) +#define PMCR0_PMI_DISABLE_MASK(PMC) (~PMCR0_PMI_ENABLE_MASK(PMC)) + +/* disable counting when a PMI is signaled (except for AIC interrupts) */ +#define PMCR0_DISCNT_SHIFT (20) +#define PMCR0_DISCNT_ENABLE_MASK (UINT64_C(1) << PMCR0_DISCNT_SHIFT) +#define PMCR0_DISCNT_DISABLE_MASK (~PMCR0_DISCNT_ENABLE_MASK) + +/* 21 unused */ + +/* block PMIs until ERET retires */ +#define PMCR0_WFRFE_SHIFT (22) +#define PMCR0_WFRFE_ENABLE_MASK (UINT64_C(1) << PMCR0_WFRE_SHIFT) +#define PMCR0_WFRFE_DISABLE_MASK (~PMCR0_WFRFE_ENABLE_MASK) + +/* count global L2C events */ +#define PMCR0_L2CGLOBAL_SHIFT (23) +#define PMCR0_L2CGLOBAL_ENABLE_MASK (UINT64_C(1) << PMCR0_L2CGLOBAL_SHIFT) +#define PMCR0_L2CGLOBAL_DISABLE_MASK (~PMCR0_L2CGLOBAL_ENABLE_MASK) + +/* allow user mode access to configuration registers */ +#define PMCR0_USEREN_SHIFT (30) +#define PMCR0_USEREN_ENABLE_MASK (UINT64_C(1) << PMCR0_USEREN_SHIFT) +#define PMCR0_USEREN_DISABLE_MASK (~PMCR0_USEREN_ENABLE_MASK) + +/* force the CPMU clocks in case of a clocking bug */ +#define PMCR0_CLKEN_SHIFT (31) +#define PMCR0_CLKEN_ENABLE_MASK (UINT64_C(1) << PMCR0_USEREN_SHIFT) +#define PMCR0_CLKEN_DISABLE_MASK (~PMCR0_CLKEN_ENABLE_MASK) + +/* 32 - 44 mirror the low bits for PMCs 8 and 9 */ + +/* PMCR1 enables counters in different processor modes */ + +#define PMCR1_EL0_A32_OFFSET (0) +#define PMCR1_EL0_A64_OFFSET (8) +#define PMCR1_EL1_A64_OFFSET (16) +#define PMCR1_EL3_A64_OFFSET (24) + +#define PMCR1_EL0_A32_SHIFT(PMC) (PMCR1_EL0_A32_OFFSET + PMCR_PMC_SHIFT(PMC)) +#define PMCR1_EL0_A64_SHIFT(PMC) (PMCR1_EL0_A64_OFFSET + PMCR_PMC_SHIFT(PMC)) +#define PMCR1_EL1_A64_SHIFT(PMC) (PMCR1_EL1_A64_OFFSET + PMCR_PMC_SHIFT(PMC)) +#define PMCR1_EL3_A64_SHIFT(PMC) (PMCR1_EL0_A64_OFFSET + PMCR_PMC_SHIFT(PMC)) + +#define PMCR1_EL0_A32_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL0_A32_SHIFT(PMC)) +#define PMCR1_EL0_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL0_A64_SHIFT(PMC)) +#define PMCR1_EL1_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL1_A64_SHIFT(PMC)) +/* PMCR1_EL3_A64 is not supported on PMCs 8 and 9 */ +#if NO_MONITOR +#define PMCR1_EL3_A64_ENABLE_MASK(PMC) UINT64_C(0) +#else +#define PMCR1_EL3_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL3_A64_SHIFT(PMC)) +#endif + +#define PMCR1_EL_ALL_ENABLE_MASK(PMC) (PMCR1_EL0_A32_ENABLE_MASK(PMC) | \ + PMCR1_EL0_A64_ENABLE_MASK(PMC) | \ + PMCR1_EL1_A64_ENABLE_MASK(PMC) | \ + PMCR1_EL3_A64_ENABLE_MASK(PMC)) +#define PMCR1_EL_ALL_DISABLE_MASK(PMC) (~PMCR1_EL_ALL_ENABLE_MASK(PMC)) + +/* PMESR0 and PMESR1 are event selection registers */ + +/* PMESR0 selects which event is counted on PMCs 2, 3, 4, and 5 */ +/* PMESR1 selects which event is counted on PMCs 6, 7, 8, and 9 */ + +#define PMESR_PMC_WIDTH (8) +#define PMESR_PMC_MASK (UINT8_MAX) +#define PMESR_SHIFT(PMC, OFF) (8 * ((PMC) - (OFF))) +#define PMESR_EVT_MASK(PMC, OFF) (PMESR_PMC_MASK << PMESR_SHIFT(PMC, OFF)) +#define PMESR_EVT_CLEAR(PMC, OFF) (~PMESR_EVT_MASK(PMC, OFF)) + +#define PMESR_EVT_DECODE(PMESR, PMC, OFF) \ + (((PMESR) >> PMESR_SHIFT(PMC, OFF)) & PMESR_PMC_MASK) +#define PMESR_EVT_ENCODE(EVT, PMC, OFF) \ + (((EVT) & PMESR_PMC_MASK) << PMESR_SHIFT(PMC, OFF)) + +/* system registers in the CPMU */ + +#define SREG_PMCR0 "S3_1_c15_c0_0" +#define SREG_PMCR1 "S3_1_c15_c1_0" +#define SREG_PMCR2 "S3_1_c15_c2_0" +#define SREG_PMCR3 "S3_1_c15_c3_0" +#define SREG_PMCR4 "S3_1_c15_c4_0" +#define SREG_PMESR0 "S3_1_c15_c5_0" +#define SREG_PMESR1 "S3_1_c15_c6_0" +#define SREG_PMSR "S3_1_c15_c13_0" +#define SREG_OPMAT0 "S3_1_c15_c7_0" +#define SREG_OPMAT1 "S3_1_c15_c8_0" +#define SREG_OPMSK0 "S3_1_c15_c9_0" +#define SREG_OPMSK1 "S3_1_c15_c10_0" + +#define SREG_PMC0 "S3_2_c15_c0_0" +#define SREG_PMC1 "S3_2_c15_c1_0" +#define SREG_PMC2 "S3_2_c15_c2_0" +#define SREG_PMC3 "S3_2_c15_c3_0" +#define SREG_PMC4 "S3_2_c15_c4_0" +#define SREG_PMC5 "S3_2_c15_c5_0" +#define SREG_PMC6 "S3_2_c15_c6_0" +#define SREG_PMC7 "S3_2_c15_c7_0" +#define SREG_PMC8 "S3_2_c15_c9_0" +#define SREG_PMC9 "S3_2_c15_c10_0" + +#if !defined(APPLECYCLONE) +#define SREG_PMMMAP "S3_2_c15_c15_0" +#define SREG_PMTRHLD2 "S3_2_c15_c14_0" +#define SREG_PMTRHLD4 "S3_2_c15_c13_0" +#define SREG_PMTRHLD6 "S3_2_c15_c12_0" +#endif + +/* + * The low 8 bits of a configuration words select the event to program on + * PMESR{0,1}. Bits 16-19 are mapped to PMCR1 bits. + */ +#define CFGWORD_EL0A32EN_MASK (0x10000) +#define CFGWORD_EL0A64EN_MASK (0x20000) +#define CFGWORD_EL1EN_MASK (0x40000) +#define CFGWORD_EL3EN_MASK (0x80000) +#define CFGWORD_ALLMODES_MASK (0xf0000) + +/* ACC offsets for PIO */ +#define ACC_CPMU_PMC0_OFFSET (0x200) +#define ACC_CPMU_PMC8_OFFSET (0x280) + +/* + * Macros for reading and writing system registers. + * + * SR must be one of the SREG_* defines above. + */ +#define SREG_WRITE(SR, V) __asm__ volatile("msr " SR ", %0 ; isb" : : "r"(V)) +#define SREG_READ(SR) ({ uint64_t VAL; \ + __asm__ volatile("mrs %0, " SR : "=r"(VAL)); \ + VAL; }) + +/* + * Configuration registers that can be controlled by RAWPMU: + * + * All: PMCR2-4, OPMAT0-1, OPMSK0-1. + * Typhoon/Twister/Hurricane: PMMMAP, PMTRHLD2/4/6. + */ +#if defined(APPLECYCLONE) +#define RAWPMU_CONFIG_COUNT 7 +#else +#define RAWPMU_CONFIG_COUNT 11 +#endif + +/* TODO: allocate dynamically */ +static uint64_t saved_PMCR[MAX_CPUS][2]; +static uint64_t saved_PMESR[MAX_CPUS][2]; +static uint64_t saved_RAWPMU[MAX_CPUS][RAWPMU_CONFIG_COUNT]; +static uint64_t saved_counter[MAX_CPUS][KPC_MAX_COUNTERS]; +static uint64_t kpc_running_cfg_pmc_mask = 0; +static uint32_t kpc_running_classes = 0; +static uint32_t kpc_configured = 0; + +static int first_time = 1; + +/* + * The whitelist is disabled by default on development/debug kernel. This can + * be changed via the kpc.disable_whitelist sysctl. The whitelist is enabled on + * release kernel and cannot be disabled. + */ +#if DEVELOPMENT || DEBUG +static boolean_t whitelist_disabled = TRUE; +#else +static boolean_t whitelist_disabled = FALSE; +#endif + +/* List of counter events that are allowed externally */ +static kpc_config_t whitelist[] = { + 0, /* NO_EVENT */ + +#if defined(APPLECYCLONE) + 0x02, /* CORE_CYCLE */ + 0x19, /* BIU_UPSTREAM_CYCLE */ + 0x1a, /* BIU_DOWNSTREAM_CYCLE */ + 0x22, /* L2C_AGENT_LD */ + 0x23, /* L2C_AGENT_LD_MISS */ + 0x24, /* L2C_AGENT_ST */ + 0x25, /* L2C_AGENT_ST_MISS */ + 0x78, /* INST_A32 */ + 0x79, /* INST_THUMB */ + 0x7a, /* INST_A64 */ + 0x7b, /* INST_BRANCH */ + 0xb4, /* SYNC_DC_LOAD_MISS */ + 0xb5, /* SYNC_DC_STORE_MISS */ + 0xb6, /* SYNC_DTLB_MISS */ + 0xb9, /* SYNC_ST_HIT_YNGR_LD */ + 0xc0, /* SYNC_BR_ANY_MISP */ + 0xce, /* FED_IC_MISS_DEM */ + 0xcf, /* FED_ITLB_MISS */ + +#elif defined(APPLETYPHOON) + 0x02, /* CORE_CYCLE */ + 0x13, /* BIU_UPSTREAM_CYCLE */ + 0x14, /* BIU_DOWNSTREAM_CYCLE */ + 0x1a, /* L2C_AGENT_LD */ + 0x1b, /* L2C_AGENT_LD_MISS */ + 0x1c, /* L2C_AGENT_ST */ + 0x1d, /* L2C_AGENT_ST_MISS */ + 0x8a, /* INST_A32 */ + 0x8b, /* INST_THUMB */ + 0x8c, /* INST_A64 */ + 0x8d, /* INST_BRANCH */ + 0xbf, /* SYNC_DC_LOAD_MISS */ + 0xc0, /* SYNC_DC_STORE_MISS */ + 0xc1, /* SYNC_DTLB_MISS */ + 0xc4, /* SYNC_ST_HIT_YNGR_LD */ + 0xcb, /* SYNC_BR_ANY_MISP */ + 0xd3, /* FED_IC_MISS_DEM */ + 0xd4, /* FED_ITLB_MISS */ + +#elif defined(APPLETWISTER) || defined(APPLEHURRICANE) + 0x02, /* CORE_CYCLE */ + 0x1a, /* L2C_AGENT_LD */ + 0x1b, /* L2C_AGENT_LD_MISS */ + 0x1c, /* L2C_AGENT_ST */ + 0x1d, /* L2C_AGENT_ST_MISS */ + 0x8a, /* INST_A32 */ + 0x8b, /* INST_THUMB */ + 0x8c, /* INST_A64 */ + 0x8d, /* INST_BRANCH */ + 0xbf, /* SYNC_DC_LOAD_MISS */ + 0xc0, /* SYNC_DC_STORE_MISS */ + 0xc1, /* SYNC_DTLB_MISS */ + 0xc4, /* SYNC_ST_HIT_YNGR_LD */ + 0xcb, /* SYNC_BR_ANY_MISP */ + 0xd3, /* FED_IC_MISS_DEM */ + 0xd4, /* FED_ITLB_MISS */ + +#else + /* An unknown CPU gets a trivial { NO_EVENT } whitelist. */ +#endif +}; +#define WHITELIST_COUNT (sizeof(whitelist)/sizeof(*whitelist)) + +static boolean_t +config_in_whitelist(kpc_config_t cfg) +{ + unsigned int i; + + for (i = 0; i < WHITELIST_COUNT; i++) { + if (cfg == whitelist[i]) { + return TRUE; + } + } + + return FALSE; +} + +#ifdef KPC_DEBUG +static void dump_regs(void) +{ + uint64_t val; + kprintf("PMCR0 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR0)); + kprintf("PMCR1 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR1)); + kprintf("PMCR2 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR2)); + kprintf("PMCR3 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR3)); + kprintf("PMCR4 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR4)); + kprintf("PMESR0 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMESR0)); + kprintf("PMESR1 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMESR1)); + + kprintf("PMC0 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC0)); + kprintf("PMC1 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC1)); + kprintf("PMC2 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC2)); + kprintf("PMC3 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC3)); + kprintf("PMC4 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC4)); + kprintf("PMC5 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC5)); + kprintf("PMC6 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC6)); + kprintf("PMC7 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC7)); + +#if (KPC_ARM64_CONFIGURABLE_COUNT > 6) + kprintf("PMC8 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC8)); + kprintf("PMC9 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC9)); +#endif +} +#endif + +static boolean_t +enable_counter(uint32_t counter) +{ + int cpuid = cpu_number(); + uint64_t pmcr0 = 0, intgen_type; + boolean_t counter_running, pmi_enabled, intgen_correct, enabled; + + pmcr0 = SREG_READ(SREG_PMCR0) | 0x3 /* leave the fixed counters enabled for monotonic */; + + counter_running = (pmcr0 & PMCR0_PMC_ENABLE_MASK(counter)) != 0; + pmi_enabled = (pmcr0 & PMCR0_PMI_ENABLE_MASK(counter)) != 0; + + /* TODO this should use the PMI path rather than AIC for the interrupt + * as it is faster + */ + intgen_type = PMCR0_INTGEN_AIC; + intgen_correct = (pmcr0 & PMCR0_INTGEN_MASK) == intgen_type; + + enabled = counter_running && pmi_enabled && intgen_correct; + + if (!enabled) { + pmcr0 |= PMCR0_PMC_ENABLE_MASK(counter); + pmcr0 |= PMCR0_PMI_ENABLE_MASK(counter); + pmcr0 &= ~PMCR0_INTGEN_MASK; + pmcr0 |= intgen_type; + + SREG_WRITE(SREG_PMCR0, pmcr0); + } + + saved_PMCR[cpuid][0] = pmcr0; + return enabled; +} + +static boolean_t +disable_counter(uint32_t counter) +{ + uint64_t pmcr0; + boolean_t enabled; + int cpuid = cpu_number(); + + if (counter < 2) { + return true; + } + + pmcr0 = SREG_READ(SREG_PMCR0) | 0x3; + enabled = (pmcr0 & PMCR0_PMC_ENABLE_MASK(counter)) != 0; + + if (enabled) { + pmcr0 &= PMCR0_PMC_DISABLE_MASK(counter); + SREG_WRITE(SREG_PMCR0, pmcr0); + } + + saved_PMCR[cpuid][0] = pmcr0; + return enabled; +} + +/* + * Enable counter in processor modes determined by configuration word. + */ +static void +set_modes(uint32_t counter, kpc_config_t cfgword) +{ + uint64_t bits = 0; + int cpuid = cpu_number(); + + if (cfgword & CFGWORD_EL0A32EN_MASK) { + bits |= PMCR1_EL0_A32_ENABLE_MASK(counter); + } + if (cfgword & CFGWORD_EL0A64EN_MASK) { + bits |= PMCR1_EL0_A64_ENABLE_MASK(counter); + } + if (cfgword & CFGWORD_EL1EN_MASK) { + bits |= PMCR1_EL1_A64_ENABLE_MASK(counter); + } +#if !NO_MONITOR + if (cfgword & CFGWORD_EL3EN_MASK) { + bits |= PMCR1_EL3_A64_ENABLE_MASK(counter); + } +#endif + + /* + * Backwards compatibility: Writing a non-zero configuration word with + * all zeros in bits 16-19 is interpreted as enabling in all modes. + * This matches the behavior when the PMCR1 bits weren't exposed. + */ + if (bits == 0 && cfgword != 0) { + bits = PMCR1_EL_ALL_ENABLE_MASK(counter); + } + + uint64_t pmcr1 = SREG_READ(SREG_PMCR1); + pmcr1 &= PMCR1_EL_ALL_DISABLE_MASK(counter); + pmcr1 |= bits; + pmcr1 |= 0x30303; /* monotonic compatibility */ + SREG_WRITE(SREG_PMCR1, pmcr1); + saved_PMCR[cpuid][1] = pmcr1; +} + +static uint64_t +read_counter(uint32_t counter) +{ + switch (counter) { + // case 0: return SREG_READ(SREG_PMC0); + // case 1: return SREG_READ(SREG_PMC1); + case 2: return SREG_READ(SREG_PMC2); + case 3: return SREG_READ(SREG_PMC3); + case 4: return SREG_READ(SREG_PMC4); + case 5: return SREG_READ(SREG_PMC5); + case 6: return SREG_READ(SREG_PMC6); + case 7: return SREG_READ(SREG_PMC7); +#if (KPC_ARM64_CONFIGURABLE_COUNT > 6) + case 8: return SREG_READ(SREG_PMC8); + case 9: return SREG_READ(SREG_PMC9); +#endif + default: return 0; + } +} + +static void +write_counter(uint32_t counter, uint64_t value) +{ + switch (counter) { + // case 0: SREG_WRITE(SREG_PMC0, value); break; + // case 1: SREG_WRITE(SREG_PMC1, value); break; + case 2: SREG_WRITE(SREG_PMC2, value); break; + case 3: SREG_WRITE(SREG_PMC3, value); break; + case 4: SREG_WRITE(SREG_PMC4, value); break; + case 5: SREG_WRITE(SREG_PMC5, value); break; + case 6: SREG_WRITE(SREG_PMC6, value); break; + case 7: SREG_WRITE(SREG_PMC7, value); break; +#if (KPC_ARM64_CONFIGURABLE_COUNT > 6) + case 8: SREG_WRITE(SREG_PMC8, value); break; + case 9: SREG_WRITE(SREG_PMC9, value); break; +#endif + default: break; + } +} + +uint32_t +kpc_rawpmu_config_count(void) +{ + return RAWPMU_CONFIG_COUNT; +} + +int +kpc_get_rawpmu_config(kpc_config_t *configv) +{ + configv[0] = SREG_READ(SREG_PMCR2); + configv[1] = SREG_READ(SREG_PMCR3); + configv[2] = SREG_READ(SREG_PMCR4); + configv[3] = SREG_READ(SREG_OPMAT0); + configv[4] = SREG_READ(SREG_OPMAT1); + configv[5] = SREG_READ(SREG_OPMSK0); + configv[6] = SREG_READ(SREG_OPMSK1); +#if RAWPMU_CONFIG_COUNT > 7 + configv[7] = SREG_READ(SREG_PMMMAP); + configv[8] = SREG_READ(SREG_PMTRHLD2); + configv[9] = SREG_READ(SREG_PMTRHLD4); + configv[10] = SREG_READ(SREG_PMTRHLD6); +#endif + return 0; +} + +static int +kpc_set_rawpmu_config(kpc_config_t *configv) +{ + SREG_WRITE(SREG_PMCR2, configv[0]); + SREG_WRITE(SREG_PMCR3, configv[1]); + SREG_WRITE(SREG_PMCR4, configv[2]); + SREG_WRITE(SREG_OPMAT0, configv[3]); + SREG_WRITE(SREG_OPMAT1, configv[4]); + SREG_WRITE(SREG_OPMSK0, configv[5]); + SREG_WRITE(SREG_OPMSK1, configv[6]); +#if RAWPMU_CONFIG_COUNT > 7 + SREG_WRITE(SREG_PMMMAP, configv[7]); + SREG_WRITE(SREG_PMTRHLD2, configv[8]); + SREG_WRITE(SREG_PMTRHLD4, configv[9]); + SREG_WRITE(SREG_PMTRHLD6, configv[10]); +#endif + return 0; +} + +static void +save_regs(void) +{ + int cpuid = cpu_number(); + + __asm__ volatile("dmb ish"); + + assert(ml_get_interrupts_enabled() == FALSE); + + /* Save current PMCR0/1 values. PMCR2-4 are in the RAWPMU set. */ + saved_PMCR[cpuid][0] = SREG_READ(SREG_PMCR0) | 0x3; + + /* Save event selections. */ + saved_PMESR[cpuid][0] = SREG_READ(SREG_PMESR0); + saved_PMESR[cpuid][1] = SREG_READ(SREG_PMESR1); + + kpc_get_rawpmu_config(saved_RAWPMU[cpuid]); + + /* Disable the counters. */ + // SREG_WRITE(SREG_PMCR0, clear); + + /* Finally, save state for each counter*/ + for (int i = 2; i < KPC_ARM64_PMC_COUNT; i++) { + saved_counter[cpuid][i] = read_counter(i); + } +} + +static void +restore_regs(void) +{ + int cpuid = cpu_number(); + + /* Restore PMESR values. */ + SREG_WRITE(SREG_PMESR0, saved_PMESR[cpuid][0]); + SREG_WRITE(SREG_PMESR1, saved_PMESR[cpuid][1]); + + kpc_set_rawpmu_config(saved_RAWPMU[cpuid]); + + /* Restore counter values */ + for (int i = 2; i < KPC_ARM64_PMC_COUNT; i++) { + write_counter(i, saved_counter[cpuid][i]); + } + + /* Restore PMCR0/1 values (with PMCR0 last to enable). */ + SREG_WRITE(SREG_PMCR1, saved_PMCR[cpuid][1] | 0x30303); + SREG_WRITE(SREG_PMCR0, saved_PMCR[cpuid][0] | 0x3); +} + +static uint64_t +get_counter_config(uint32_t counter) +{ + uint64_t pmesr; + + switch (counter) { + case 2: /* FALLTHROUGH */ + case 3: /* FALLTHROUGH */ + case 4: /* FALLTHROUGH */ + case 5: + pmesr = PMESR_EVT_DECODE(SREG_READ(SREG_PMESR0), counter, 2); + break; + case 6: /* FALLTHROUGH */ + case 7: +#if (KPC_ARM64_CONFIGURABLE_COUNT > 6) + /* FALLTHROUGH */ + case 8: /* FALLTHROUGH */ + case 9: +#endif + pmesr = PMESR_EVT_DECODE(SREG_READ(SREG_PMESR1), counter, 6); + break; + default: + pmesr = 0; + break; + } + + kpc_config_t config = pmesr; + + uint64_t pmcr1 = SREG_READ(SREG_PMCR1); + + if (pmcr1 & PMCR1_EL0_A32_ENABLE_MASK(counter)) { + config |= CFGWORD_EL0A32EN_MASK; + } + if (pmcr1 & PMCR1_EL0_A64_ENABLE_MASK(counter)) { + config |= CFGWORD_EL0A64EN_MASK; + } + if (pmcr1 & PMCR1_EL1_A64_ENABLE_MASK(counter)) { + config |= CFGWORD_EL1EN_MASK; +#if NO_MONITOR + config |= CFGWORD_EL3EN_MASK; +#endif + } +#if !NO_MONITOR + if (pmcr1 & PMCR1_EL3_A64_ENABLE_MASK(counter)) { + config |= CFGWORD_EL3EN_MASK; + } +#endif + + return config; +} + +static void +set_counter_config(uint32_t counter, uint64_t config) +{ + int cpuid = cpu_number(); + uint64_t pmesr = 0; + + switch (counter) { + case 2: /* FALLTHROUGH */ + case 3: /* FALLTHROUGH */ + case 4: /* FALLTHROUGH */ + case 5: + pmesr = SREG_READ(SREG_PMESR0); + pmesr &= PMESR_EVT_CLEAR(counter, 2); + pmesr |= PMESR_EVT_ENCODE(config, counter, 2); + SREG_WRITE(SREG_PMESR0, pmesr); + saved_PMESR[cpuid][0] = pmesr; + break; + + case 6: /* FALLTHROUGH */ + case 7: +#if KPC_ARM64_CONFIGURABLE_COUNT > 6 + /* FALLTHROUGH */ + case 8: /* FALLTHROUGH */ + case 9: +#endif + pmesr = SREG_READ(SREG_PMESR1); + pmesr &= PMESR_EVT_CLEAR(counter, 6); + pmesr |= PMESR_EVT_ENCODE(config, counter, 6); + SREG_WRITE(SREG_PMESR1, pmesr); + saved_PMESR[cpuid][1] = pmesr; + break; + default: + break; + } + + set_modes(counter, config); +} + +/* internal functions */ + +void +kpc_arch_init(void) +{ +} + +boolean_t +kpc_is_running_fixed(void) +{ + return (kpc_running_classes & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK; +} + +boolean_t +kpc_is_running_configurable(uint64_t pmc_mask) +{ + assert(kpc_popcount(pmc_mask) <= kpc_configurable_count()); + return ((kpc_running_classes & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK) && + ((kpc_running_cfg_pmc_mask & pmc_mask) == pmc_mask); +} + +uint32_t +kpc_fixed_count(void) +{ + return KPC_ARM64_FIXED_COUNT; +} + +uint32_t +kpc_configurable_count(void) +{ + return KPC_ARM64_CONFIGURABLE_COUNT; +} + +uint32_t +kpc_fixed_config_count(void) +{ + return 0; +} + +uint32_t +kpc_configurable_config_count(uint64_t pmc_mask) +{ + assert(kpc_popcount(pmc_mask) <= kpc_configurable_count()); + return kpc_popcount(pmc_mask); +} + +int +kpc_get_fixed_config(kpc_config_t *configv __unused) +{ + return 0; +} + +uint64_t +kpc_fixed_max(void) +{ + return (1ULL << KPC_ARM64_COUNTER_WIDTH) - 1; +} + +uint64_t +kpc_configurable_max(void) +{ + return (1ULL << KPC_ARM64_COUNTER_WIDTH) - 1; +} + +static void +set_running_configurable(uint64_t target_mask, uint64_t state_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + boolean_t enabled; + + enabled = ml_set_interrupts_enabled(FALSE); + + for (uint32_t i = 0; i < cfg_count; ++i) { + if (((1ULL << i) & target_mask) == 0) + continue; + assert(kpc_controls_counter(offset + i)); + + if ((1ULL << i) & state_mask) { + enable_counter(offset + i); + } else { + disable_counter(offset + i); + } + } + + ml_set_interrupts_enabled(enabled); +} + +static uint32_t kpc_xcall_sync; +static void +kpc_set_running_xcall( void *vstate ) +{ + struct kpc_running_remote *mp_config = (struct kpc_running_remote*) vstate; + assert(mp_config); + + set_running_configurable(mp_config->cfg_target_mask, + mp_config->cfg_state_mask); + + if (hw_atomic_sub(&kpc_xcall_sync, 1) == 0) + thread_wakeup((event_t) &kpc_xcall_sync); +} + +static uint32_t kpc_xread_sync; +static void +kpc_get_curcpu_counters_xcall(void *args) +{ + struct kpc_get_counters_remote *handler = args; + + assert(handler != NULL); + assert(handler->buf != NULL); + + int offset = cpu_number() * handler->buf_stride; + int r = kpc_get_curcpu_counters(handler->classes, NULL, &handler->buf[offset]); + + /* number of counters added by this CPU, needs to be atomic */ + hw_atomic_add(&(handler->nb_counters), r); + + if (hw_atomic_sub(&kpc_xread_sync, 1) == 0) { + thread_wakeup((event_t) &kpc_xread_sync); + } +} + +int +kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf) +{ + assert(buf != NULL); + + int enabled = ml_set_interrupts_enabled(FALSE); + + /* grab counters and CPU number as close as possible */ + if (curcpu) { + *curcpu = current_processor()->cpu_id; + } + + struct kpc_get_counters_remote hdl = { + .classes = classes, + .nb_counters = 0, + .buf = buf, + .buf_stride = kpc_get_counter_count(classes) + }; + + cpu_broadcast_xcall(&kpc_xread_sync, TRUE, kpc_get_curcpu_counters_xcall, &hdl); + int offset = hdl.nb_counters; + + (void)ml_set_interrupts_enabled(enabled); + + return offset; +} + +int +kpc_get_fixed_counters(uint64_t *counterv) +{ +#if MONOTONIC + mt_fixed_counts(counterv); + return 0; +#else /* MONOTONIC */ +#pragma unused(counterv) + return ENOTSUP; +#endif /* !MONOTONIC */ +} + +int +kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + uint64_t ctr = 0ULL; + + assert(counterv); + + for (uint32_t i = 0; i < cfg_count; ++i) { + if (((1ULL << i) & pmc_mask) == 0) + continue; + ctr = read_counter(i + offset); + + if (ctr & KPC_ARM64_COUNTER_OVF_MASK) { + ctr = CONFIGURABLE_SHADOW(i) + + (kpc_configurable_max() - CONFIGURABLE_RELOAD(i) + 1 /* Wrap */) + + (ctr & KPC_ARM64_COUNTER_MASK); + } else { + ctr = CONFIGURABLE_SHADOW(i) + + (ctr - CONFIGURABLE_RELOAD(i)); + } + + *counterv++ = ctr; + } + + return 0; +} + +int +kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + + assert(configv); + + for (uint32_t i = 0; i < cfg_count; ++i) + if ((1ULL << i) & pmc_mask) + *configv++ = get_counter_config(i + offset); + return 0; +} + +static int +kpc_set_configurable_config(kpc_config_t *configv, uint64_t pmc_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + boolean_t enabled; + + assert(configv); + + enabled = ml_set_interrupts_enabled(FALSE); + + for (uint32_t i = 0; i < cfg_count; ++i) { + if (((1ULL << i) & pmc_mask) == 0) + continue; + assert(kpc_controls_counter(i + offset)); + + set_counter_config(i + offset, *configv++); + } + + ml_set_interrupts_enabled(enabled); + + return 0; +} + +static uint32_t kpc_config_sync; +static void +kpc_set_config_xcall(void *vmp_config) +{ + struct kpc_config_remote *mp_config = vmp_config; + kpc_config_t *new_config = NULL; + uint32_t classes = 0ULL; + + assert(mp_config); + assert(mp_config->configv); + classes = mp_config->classes; + new_config = mp_config->configv; + + if (classes & KPC_CLASS_CONFIGURABLE_MASK) { + kpc_set_configurable_config(new_config, mp_config->pmc_mask); + new_config += kpc_popcount(mp_config->pmc_mask); + } + + if (classes & KPC_CLASS_RAWPMU_MASK) { + kpc_set_rawpmu_config(new_config); + new_config += RAWPMU_CONFIG_COUNT; + } + + if (hw_atomic_sub(&kpc_config_sync, 1) == 0) + thread_wakeup((event_t) &kpc_config_sync); +} + +static uint64_t +kpc_reload_counter(uint32_t ctr) +{ + assert(ctr < (kpc_configurable_count() + kpc_fixed_count())); + + /* don't reload counters reserved for power management */ + if (!kpc_controls_counter(ctr)) + return 0ULL; + + uint64_t old = read_counter(ctr); + write_counter(ctr, FIXED_RELOAD(ctr)); + return old & KPC_ARM64_COUNTER_MASK; +} + +static uint32_t kpc_reload_sync; +static void +kpc_set_reload_xcall(void *vmp_config) +{ + struct kpc_config_remote *mp_config = vmp_config; + uint32_t classes = 0, count = 0, offset = kpc_fixed_count(); + uint64_t *new_period = NULL, max = kpc_configurable_max(); + boolean_t enabled; + + assert(mp_config); + assert(mp_config->configv); + classes = mp_config->classes; + new_period = mp_config->configv; + + enabled = ml_set_interrupts_enabled(FALSE); + + if (classes & KPC_CLASS_CONFIGURABLE_MASK) { + /* + * Update _all_ shadow counters, this cannot be done for only + * selected PMCs. Otherwise, we would corrupt the configurable + * shadow buffer since the PMCs are muxed according to the pmc + * mask. + */ + uint64_t all_cfg_mask = (1ULL << kpc_configurable_count()) - 1; + kpc_get_configurable_counters(&CONFIGURABLE_SHADOW(0), all_cfg_mask); + + /* set the new period */ + count = kpc_configurable_count(); + for (uint32_t i = 0; i < count; ++i) { + /* ignore the counter */ + if (((1ULL << i) & mp_config->pmc_mask) == 0) + continue; + if (*new_period == 0) + *new_period = kpc_configurable_max(); + CONFIGURABLE_RELOAD(i) = max - *new_period; + /* reload the counter */ + kpc_reload_counter(offset + i); + /* next period value */ + new_period++; + } + } + + ml_set_interrupts_enabled(enabled); + + if (hw_atomic_sub(&kpc_reload_sync, 1) == 0) + thread_wakeup((event_t) &kpc_reload_sync); +} + +void kpc_pmi_handler(cpu_id_t source); +void +kpc_pmi_handler(cpu_id_t source __unused) +{ + uint64_t PMSR, extra; + int ctr; + int enabled; + + enabled = ml_set_interrupts_enabled(FALSE); + + /* The pmi must be delivered to the CPU that generated it */ + if (source != getCpuDatap()->interrupt_nub) { + panic("pmi from IOCPU %p delivered to IOCPU %p", source, getCpuDatap()->interrupt_nub); + } + + /* Get the PMSR which has the overflow bits for all the counters */ + __asm__ volatile("mrs %0, S3_1_c15_c13_0" : "=r"(PMSR)); + + for (ctr = 0; ctr < (KPC_ARM64_FIXED_COUNT + KPC_ARM64_CONFIGURABLE_COUNT); ctr++) { + if ((1ull << ctr) & PMSR) { + if (ctr < 2) { +#if MONOTONIC + mt_cpu_pmi(getCpuDatap(), PMSR); +#endif /* MONOTONIC */ + } else { + extra = kpc_reload_counter(ctr); + + FIXED_SHADOW(ctr) + += (kpc_fixed_max() - FIXED_RELOAD(ctr) + 1 /* Wrap */) + extra; + + if (FIXED_ACTIONID(ctr)) + kpc_sample_kperf(FIXED_ACTIONID(ctr)); + } + } + } + + ml_set_interrupts_enabled(enabled); +} + +uint32_t +kpc_get_classes(void) +{ + return KPC_CLASS_FIXED_MASK | KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_RAWPMU_MASK; +} + +int +kpc_set_running_arch(struct kpc_running_remote *mp_config) +{ + int cpu; + + assert(mp_config); + + if (first_time) { + PE_cpu_perfmon_interrupt_install_handler(kpc_pmi_handler); + int max_cpu = ml_get_max_cpu_number(); + for (cpu = 0; cpu <= max_cpu; cpu++) { + cpu_data_t *target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + if (target_cpu_datap != NULL) + PE_cpu_perfmon_interrupt_enable(target_cpu_datap->cpu_id, TRUE); + } + first_time = 0; + } + + /* dispatch to all CPUs */ + cpu_broadcast_xcall(&kpc_xcall_sync, TRUE, kpc_set_running_xcall, mp_config); + + kpc_running_cfg_pmc_mask = mp_config->cfg_state_mask; + kpc_running_classes = mp_config->classes; + kpc_configured = 1; + + return 0; +} + +int +kpc_set_period_arch(struct kpc_config_remote *mp_config) +{ + assert(mp_config); + + /* dispatch to all CPUs */ + cpu_broadcast_xcall(&kpc_reload_sync, TRUE, kpc_set_reload_xcall, mp_config); + + kpc_configured = 1; + + return 0; +} + +int +kpc_set_config_arch(struct kpc_config_remote *mp_config) +{ + uint32_t count = kpc_popcount(mp_config->pmc_mask); + + assert(mp_config); + assert(mp_config->configv); + + /* check config against whitelist for external devs */ + for (uint32_t i = 0; i < count; ++i) { + if (!whitelist_disabled && !config_in_whitelist(mp_config->configv[i])) { + return EPERM; + } + } + + /* dispatch to all CPUs */ + cpu_broadcast_xcall(&kpc_config_sync, TRUE, kpc_set_config_xcall, mp_config); + + kpc_configured = 1; + + return 0; +} + +void +kpc_idle(void) +{ + if (kpc_configured) { + save_regs(); + } +} + +void +kpc_idle_exit(void) +{ + if (kpc_configured) { + restore_regs(); + } +} + +int +kpc_set_sw_inc( uint32_t mask __unused ) +{ + return ENOTSUP; +} + +int +kpc_disable_whitelist( int val ) +{ + whitelist_disabled = val; + return 0; +} + +int +kpc_get_whitelist_disabled( void ) +{ + return whitelist_disabled; +} + +int +kpc_get_pmu_version(void) +{ + return KPC_PMU_ARM_APPLE; +} diff --git a/osfmk/arm64/locore.s b/osfmk/arm64/locore.s new file mode 100644 index 000000000..4f8b9e9ad --- /dev/null +++ b/osfmk/arm64/locore.s @@ -0,0 +1,868 @@ +/* + * Copyright (c) 2011-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include "assym.s" + + +/* + * INIT_SAVED_STATE_FLAVORS + * + * Initializes the saved state flavors of a new saved state structure + * arg0 - saved state pointer + * arg1 - 32-bit scratch reg + * arg2 - 32-bit scratch reg + */ +.macro INIT_SAVED_STATE_FLAVORS + mov $1, ARM_SAVED_STATE64 // Set saved state to 64-bit flavor + mov $2, ARM_SAVED_STATE64_COUNT + stp $1, $2, [$0, SS_FLAVOR] + mov $1, ARM_NEON_SAVED_STATE64 // Set neon state to 64-bit flavor + str $1, [$0, NS_FLAVOR] + mov $1, ARM_NEON_SAVED_STATE64_COUNT + str $1, [$0, NS_COUNT] +.endmacro + +.macro EL1_SP0_VECTOR + msr SPSel, #0 // Switch to SP0 + sub sp, sp, ARM_CONTEXT_SIZE // Create exception frame + stp x0, x1, [sp, SS64_X0] // Save x0, x1 to exception frame + add x0, sp, ARM_CONTEXT_SIZE // Calculate the original stack pointer + str x0, [sp, SS64_SP] // Save stack pointer to exception frame + stp fp, lr, [sp, SS64_FP] // Save fp and lr to exception frame + INIT_SAVED_STATE_FLAVORS sp, w0, w1 + mov x0, sp // Copy saved state pointer to x0 +.endmacro + +/* + * SPILL_REGISTERS + * + * Spills the current set of registers (excluding x0 and x1) to the specified + * save area. + * x0 - Address of the save area + */ +.macro SPILL_REGISTERS + stp x2, x3, [x0, SS64_X2] // Save remaining GPRs + stp x4, x5, [x0, SS64_X4] + stp x6, x7, [x0, SS64_X6] + stp x8, x9, [x0, SS64_X8] + stp x10, x11, [x0, SS64_X10] + stp x12, x13, [x0, SS64_X12] + stp x14, x15, [x0, SS64_X14] + stp x16, x17, [x0, SS64_X16] + stp x18, x19, [x0, SS64_X18] + stp x20, x21, [x0, SS64_X20] + stp x22, x23, [x0, SS64_X22] + stp x24, x25, [x0, SS64_X24] + stp x26, x27, [x0, SS64_X26] + str x28, [x0, SS64_X28] + + /* Save arm_neon_saved_state64 */ + + stp q0, q1, [x0, NS64_Q0] + stp q2, q3, [x0, NS64_Q2] + stp q4, q5, [x0, NS64_Q4] + stp q6, q7, [x0, NS64_Q6] + stp q8, q9, [x0, NS64_Q8] + stp q10, q11, [x0, NS64_Q10] + stp q12, q13, [x0, NS64_Q12] + stp q14, q15, [x0, NS64_Q14] + stp q16, q17, [x0, NS64_Q16] + stp q18, q19, [x0, NS64_Q18] + stp q20, q21, [x0, NS64_Q20] + stp q22, q23, [x0, NS64_Q22] + stp q24, q25, [x0, NS64_Q24] + stp q26, q27, [x0, NS64_Q26] + stp q28, q29, [x0, NS64_Q28] + stp q30, q31, [x0, NS64_Q30] + + mrs lr, ELR_EL1 // Get exception link register + mrs x23, SPSR_EL1 // Load CPSR into var reg x23 + mrs x24, FPSR + mrs x25, FPCR + + str lr, [x0, SS64_PC] // Save ELR to PCB + str w23, [x0, SS64_CPSR] // Save CPSR to PCB + str w24, [x0, NS64_FPSR] + str w25, [x0, NS64_FPCR] + + mrs x20, FAR_EL1 + mrs x21, ESR_EL1 + str x20, [x0, SS64_FAR] + str w21, [x0, SS64_ESR] +.endmacro + + +#define CBF_DISABLE 0 +#define CBF_ENABLE 1 + +.macro COMPARE_BRANCH_FUSION +#if defined(APPLE_ARM64_ARCH_FAMILY) + mrs $1, ARM64_REG_HID1 + .if $0 == CBF_DISABLE + orr $1, $1, ARM64_REG_HID1_disCmpBrFusion + .else + mov $2, ARM64_REG_HID1_disCmpBrFusion + bic $1, $1, $2 + .endif + msr ARM64_REG_HID1, $1 + .if $0 == CBF_DISABLE + isb sy + .endif +#endif +.endmacro + + .text + .align 12 + .globl EXT(ExceptionVectorsBase) +LEXT(ExceptionVectorsBase) +Lel1_sp0_synchronous_vector: + sub sp, sp, ARM_CONTEXT_SIZE // Make space on the exception stack + stp x0, x1, [sp, SS64_X0] // Save x0, x1 to the stack + mrs x1, ESR_EL1 // Get the exception syndrome + /* If the stack pointer is corrupt, it will manifest either as a data abort + * (syndrome 0x25) or a misaligned pointer (syndrome 0x26). We can check + * these quickly by testing bit 5 of the exception class. + */ + tbz x1, #(5 + ESR_EC_SHIFT), Lkernel_stack_valid + mrs x0, SP_EL0 // Get SP_EL0 + stp fp, lr, [sp, SS64_FP] // Save fp, lr to the stack + str x0, [sp, SS64_SP] // Save sp to the stack + bl check_kernel_stack + ldp fp, lr, [sp, SS64_FP] // Restore fp, lr +Lkernel_stack_valid: + ldp x0, x1, [sp, SS64_X0] // Restore x0, x1 + add sp, sp, ARM_CONTEXT_SIZE // Restore SP1 + EL1_SP0_VECTOR + adrp x1, fleh_synchronous@page // Load address for fleh + add x1, x1, fleh_synchronous@pageoff + b fleh_dispatch64 + + .text + .align 7 +Lel1_sp0_irq_vector: + EL1_SP0_VECTOR + mrs x1, TPIDR_EL1 + ldr x1, [x1, ACT_CPUDATAP] + ldr x1, [x1, CPU_ISTACKPTR] + mov sp, x1 + adrp x1, fleh_irq@page // Load address for fleh + add x1, x1, fleh_irq@pageoff + b fleh_dispatch64 + + .text + .align 7 +Lel1_sp0_fiq_vector: + // ARM64_TODO write optimized decrementer + EL1_SP0_VECTOR + mrs x1, TPIDR_EL1 + ldr x1, [x1, ACT_CPUDATAP] + ldr x1, [x1, CPU_ISTACKPTR] + mov sp, x1 + adrp x1, fleh_fiq@page // Load address for fleh + add x1, x1, fleh_fiq@pageoff + b fleh_dispatch64 + + .text + .align 7 +Lel1_sp0_serror_vector: + EL1_SP0_VECTOR + adrp x1, fleh_serror@page // Load address for fleh + add x1, x1, fleh_serror@pageoff + b fleh_dispatch64 + +.macro EL1_SP1_VECTOR + sub sp, sp, ARM_CONTEXT_SIZE // Create exception frame + stp x0, x1, [sp, SS64_X0] // Save x0, x1 to exception frame + add x0, sp, ARM_CONTEXT_SIZE // Calculate the original stack pointer + str x0, [sp, SS64_SP] // Save stack pointer to exception frame + INIT_SAVED_STATE_FLAVORS sp, w0, w1 + stp fp, lr, [sp, SS64_FP] // Save fp and lr to exception frame + mov x0, sp // Copy saved state pointer to x0 +.endmacro + + .text + .align 7 +Lel1_sp1_synchronous_vector: +#if defined(KERNEL_INTEGRITY_KTRR) + b check_ktrr_sctlr_trap +Lel1_sp1_synchronous_vector_continue: +#endif + EL1_SP1_VECTOR + adrp x1, fleh_synchronous_sp1@page + add x1, x1, fleh_synchronous_sp1@pageoff + b fleh_dispatch64 + + .text + .align 7 +Lel1_sp1_irq_vector: + EL1_SP1_VECTOR + adrp x1, fleh_irq_sp1@page + add x1, x1, fleh_irq_sp1@pageoff + b fleh_dispatch64 + + .text + .align 7 +Lel1_sp1_fiq_vector: + EL1_SP1_VECTOR + adrp x1, fleh_fiq_sp1@page + add x1, x1, fleh_fiq_sp1@pageoff + b fleh_dispatch64 + + .text + .align 7 +Lel1_sp1_serror_vector: + EL1_SP1_VECTOR + adrp x1, fleh_serror_sp1@page + add x1, x1, fleh_serror_sp1@pageoff + b fleh_dispatch64 + +.macro EL0_64_VECTOR + stp x0, x1, [sp, #-16]! // Save x0 and x1 to the exception stack + mrs x0, TPIDR_EL1 // Load the thread register + mrs x1, SP_EL0 // Load the user stack pointer + add x0, x0, ACT_CONTEXT // Calculate where we store the user context pointer + ldr x0, [x0] // Load the user context pointer + str x1, [x0, SS64_SP] // Store the user stack pointer in the user PCB + msr SP_EL0, x0 // Copy the user PCB pointer to SP0 + ldp x0, x1, [sp], #16 // Restore x0 and x1 from the exception stack + msr SPSel, #0 // Switch to SP0 + stp x0, x1, [sp, SS64_X0] // Save x0, x1 to the user PCB + stp fp, lr, [sp, SS64_FP] // Save fp and lr to the user PCB + mov fp, xzr // Clear the fp and lr for the + mov lr, xzr // debugger stack frame + mov x0, sp // Copy the user PCB pointer to x0 +.endmacro + + .text + .align 7 +Lel0_synchronous_vector_64: + EL0_64_VECTOR + mrs x1, TPIDR_EL1 // Load the thread register + ldr x1, [x1, TH_KSTACKPTR] // Load the top of the kernel stack to x1 + mov sp, x1 // Set the stack pointer to the kernel stack + adrp x1, fleh_synchronous@page // Load address for fleh + add x1, x1, fleh_synchronous@pageoff + b fleh_dispatch64 + + .text + .align 7 +Lel0_irq_vector_64: + EL0_64_VECTOR + mrs x1, TPIDR_EL1 + ldr x1, [x1, ACT_CPUDATAP] + ldr x1, [x1, CPU_ISTACKPTR] + mov sp, x1 // Set the stack pointer to the kernel stack + adrp x1, fleh_irq@page // load address for fleh + add x1, x1, fleh_irq@pageoff + b fleh_dispatch64 + + .text + .align 7 +Lel0_fiq_vector_64: + EL0_64_VECTOR + mrs x1, TPIDR_EL1 + ldr x1, [x1, ACT_CPUDATAP] + ldr x1, [x1, CPU_ISTACKPTR] + mov sp, x1 // Set the stack pointer to the kernel stack + adrp x1, fleh_fiq@page // load address for fleh + add x1, x1, fleh_fiq@pageoff + b fleh_dispatch64 + + .text + .align 7 +Lel0_serror_vector_64: + EL0_64_VECTOR + mrs x1, TPIDR_EL1 // Load the thread register + ldr x1, [x1, TH_KSTACKPTR] // Load the top of the kernel stack to x1 + mov sp, x1 // Set the stack pointer to the kernel stack + adrp x1, fleh_serror@page // load address for fleh + add x1, x1, fleh_serror@pageoff + b fleh_dispatch64 + + /* Fill out the rest of the page */ + .align 12 + +/********************************* + * END OF EXCEPTION VECTORS PAGE * + *********************************/ + + +/* + * check_kernel_stack + * + * Verifies that the kernel stack is aligned and mapped within an expected + * stack address range. Note: happens before saving registers (in case we can't + * save to kernel stack). + * + * Expects: + * {x0, x1, sp} - saved + * x0 - SP_EL0 + * x1 - Exception syndrome + * sp - Saved state + */ + .text + .align 2 +check_kernel_stack: + stp x2, x3, [sp, SS64_X2] // Save {x2-x3} + and x1, x1, #ESR_EC_MASK // Mask the exception class + mov x2, #(ESR_EC_SP_ALIGN << ESR_EC_SHIFT) + cmp x1, x2 // If we have a stack alignment exception + b.eq Lcorrupt_stack // ...the stack is definitely corrupted + mov x2, #(ESR_EC_DABORT_EL1 << ESR_EC_SHIFT) + cmp x1, x2 // If we have a data abort, we need to + b.ne Lvalid_stack // ...validate the stack pointer + mrs x1, TPIDR_EL1 // Get thread pointer +Ltest_kstack: + ldr x2, [x1, TH_KSTACKPTR] // Get top of kernel stack + sub x3, x2, KERNEL_STACK_SIZE // Find bottom of kernel stack + cmp x0, x2 // if (SP_EL0 >= kstack top) + b.ge Ltest_istack // jump to istack test + cmp x0, x3 // if (SP_EL0 > kstack bottom) + b.gt Lvalid_stack // stack pointer valid +Ltest_istack: + ldr x1, [x1, ACT_CPUDATAP] // Load the cpu data ptr + ldr x2, [x1, CPU_INTSTACK_TOP] // Get top of istack + sub x3, x2, PGBYTES // Find bottom of istack + cmp x0, x2 // if (SP_EL0 >= istack top) + b.ge Ltest_fiqstack // jump to fiqstack test + cmp x0, x3 // if (SP_EL0 > istack bottom) + b.gt Lvalid_stack // stack pointer valid +Ltest_fiqstack: + ldr x2, [x1, CPU_FIQSTACK_TOP] // Get top of fiqstack + sub x3, x2, PGBYTES // Find bottom of fiqstack + cmp x0, x2 // if (SP_EL0 >= fiqstack top) + b.ge Lcorrupt_stack // corrupt stack pointer + cmp x0, x3 // if (SP_EL0 > fiqstack bottom) + b.gt Lvalid_stack // stack pointer valid +Lcorrupt_stack: + INIT_SAVED_STATE_FLAVORS sp, w0, w1 + mov x0, sp // Copy exception frame pointer to x0 + adrp x1, fleh_invalid_stack@page // Load address for fleh + add x1, x1, fleh_invalid_stack@pageoff // fleh_dispatch64 will save register state before we get there + ldp x2, x3, [sp, SS64_X2] // Restore {x2-x3} + b fleh_dispatch64 +Lvalid_stack: + ldp x2, x3, [sp, SS64_X2] // Restore {x2-x3} + ret + +#if defined(KERNEL_INTEGRITY_KTRR) + .text + .align 2 +check_ktrr_sctlr_trap: +/* We may abort on an instruction fetch on reset when enabling the MMU by + * writing SCTLR_EL1 because the page containing the privileged instruction is + * not executable at EL1 (due to KTRR). The abort happens only on SP1 which + * would otherwise panic unconditionally. Check for the condition and return + * safe execution to the caller on behalf of the faulting function. + * + * Expected register state: + * x22 - Kernel virtual base + * x23 - Kernel physical base + */ + sub sp, sp, ARM_CONTEXT_SIZE // Make some space on the stack + stp x0, x1, [sp, SS64_X0] // Stash x0, x1 + mrs x0, ESR_EL1 // Check ESR for instr. fetch abort + and x0, x0, #0xffffffffffffffc0 // Mask off ESR.ISS.IFSC + movz w1, #0x8600, lsl #16 + movk w1, #0x0000 + cmp x0, x1 + mrs x0, ELR_EL1 // Check for expected abort address + adrp x1, _pinst_set_sctlr_trap_addr@page + add x1, x1, _pinst_set_sctlr_trap_addr@pageoff + sub x1, x1, x22 // Convert to physical address + add x1, x1, x23 + ccmp x0, x1, #0, eq + ldp x0, x1, [sp, SS64_X0] // Restore x0, x1 + add sp, sp, ARM_CONTEXT_SIZE // Clean up stack + b.ne Lel1_sp1_synchronous_vector_continue + msr ELR_EL1, lr // Return to caller + eret +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + +/* 64-bit first level exception handler dispatcher. + * Completes register context saving and branches to FLEH. + * Expects: + * {x0, x1, fp, lr, sp} - saved + * x0 - arm_context_t + * x1 - address of FLEH + * fp - previous stack frame if EL1 + * lr - unused + * sp - kernel stack + */ + .text + .align 2 +fleh_dispatch64: + /* Save arm_saved_state64 */ + SPILL_REGISTERS + + /* If exception is from userspace, zero lr */ + ldr w21, [x0, SS64_CPSR] + and x21, x21, #(PSR64_MODE_EL_MASK) + cmp x21, #(PSR64_MODE_EL0) + bne 1f + mov lr, #0 +1: + + mov x21, x0 // Copy arm_context_t pointer to x21 + mov x22, x1 // Copy handler routine to x22 + + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + tst x23, PSR64_MODE_EL_MASK // If any EL MODE bits are set, we're coming from + b.ne 1f // kernel mode, so skip precise time update + PUSH_FRAME + bl EXT(timer_state_event_user_to_kernel) + POP_FRAME + mov x0, x21 // Reload arm_context_t pointer +1: +#endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */ + + /* Dispatch to FLEH */ + + br x22 + + + .text + .align 2 +fleh_synchronous: + mrs x1, ESR_EL1 // Load exception syndrome + mrs x2, FAR_EL1 // Load fault address + + /* At this point, the LR contains the value of ELR_EL1. In the case of an + * instruction prefetch abort, this will be the faulting pc, which we know + * to be invalid. This will prevent us from backtracing through the + * exception if we put it in our stack frame, so we load the LR from the + * exception saved state instead. + */ + and w3, w1, #(ESR_EC_MASK) + lsr w3, w3, #(ESR_EC_SHIFT) + mov w4, #(ESR_EC_IABORT_EL1) + cmp w3, w4 + b.eq Lfleh_sync_load_lr +Lvalid_link_register: + + PUSH_FRAME + bl EXT(sleh_synchronous) + POP_FRAME + + + b exception_return_dispatch + +Lfleh_sync_load_lr: + ldr lr, [x0, SS64_LR] + b Lvalid_link_register + +/* Shared prologue code for fleh_irq and fleh_fiq. + * Does any interrupt booking we may want to do + * before invoking the handler proper. + * Expects: + * x0 - arm_context_t + * x23 - CPSR + * fp - Undefined live value (we may push a frame) + * lr - Undefined live value (we may push a frame) + * sp - Interrupt stack for the current CPU + */ +.macro BEGIN_INTERRUPT_HANDLER + mrs x22, TPIDR_EL1 + ldr x23, [x22, ACT_CPUDATAP] // Get current cpu + /* Update IRQ count */ + ldr w1, [x23, CPU_STAT_IRQ] + add w1, w1, #1 // Increment count + str w1, [x23, CPU_STAT_IRQ] // Update IRQ count + ldr w1, [x23, CPU_STAT_IRQ_WAKE] + add w1, w1, #1 // Increment count + str w1, [x23, CPU_STAT_IRQ_WAKE] // Update post-wake IRQ count + /* Increment preempt count */ + ldr w1, [x22, ACT_PREEMPT_CNT] + add w1, w1, #1 + str w1, [x22, ACT_PREEMPT_CNT] + /* Store context in int state */ + str x0, [x23, CPU_INT_STATE] // Saved context in cpu_int_state +.endmacro + +/* Shared epilogue code for fleh_irq and fleh_fiq. + * Cleans up after the prologue, and may do a bit more + * bookkeeping (kdebug related). + * Expects: + * x22 - Live TPIDR_EL1 value (thread address) + * x23 - Address of the current CPU data structure + * w24 - 0 if kdebug is disbled, nonzero otherwise + * fp - Undefined live value (we may push a frame) + * lr - Undefined live value (we may push a frame) + * sp - Interrupt stack for the current CPU + */ +.macro END_INTERRUPT_HANDLER + /* Clear int context */ + str xzr, [x23, CPU_INT_STATE] + /* Decrement preempt count */ + ldr w0, [x22, ACT_PREEMPT_CNT] + cbnz w0, 1f // Detect underflow + b preempt_underflow +1: + sub w0, w0, #1 + str w0, [x22, ACT_PREEMPT_CNT] + /* Switch back to kernel stack */ + ldr x0, [x22, TH_KSTACKPTR] + mov sp, x0 +.endmacro + + .text + .align 2 +fleh_irq: + BEGIN_INTERRUPT_HANDLER + PUSH_FRAME + bl EXT(sleh_irq) + POP_FRAME + END_INTERRUPT_HANDLER + + + b exception_return_dispatch + + .text + .align 2 + .global EXT(fleh_fiq_generic) +LEXT(fleh_fiq_generic) + PANIC_UNIMPLEMENTED + + .text + .align 2 +fleh_fiq: + BEGIN_INTERRUPT_HANDLER + PUSH_FRAME + bl EXT(sleh_fiq) + POP_FRAME + END_INTERRUPT_HANDLER + + + b exception_return_dispatch + + .text + .align 2 +fleh_serror: + mrs x1, ESR_EL1 // Load exception syndrome + mrs x2, FAR_EL1 // Load fault address + + PUSH_FRAME + bl EXT(sleh_serror) + POP_FRAME + + + b exception_return_dispatch + +/* + * Register state saved before we get here. + */ + .text + .align 2 +fleh_invalid_stack: + mrs x1, ESR_EL1 // Load exception syndrome + str x1, [x0, SS64_ESR] + mrs x2, FAR_EL1 // Load fault address + str x2, [x0, SS64_FAR] + PUSH_FRAME + bl EXT(sleh_invalid_stack) // Shouldn't return! + b . + + .text + .align 2 +fleh_synchronous_sp1: + mrs x1, ESR_EL1 // Load exception syndrome + str x1, [x0, SS64_ESR] + mrs x2, FAR_EL1 // Load fault address + str x2, [x0, SS64_FAR] + PUSH_FRAME + bl EXT(sleh_synchronous_sp1) + b . + + .text + .align 2 +fleh_irq_sp1: + mov x1, x0 + adr x0, Lsp1_irq_str + b EXT(panic_with_thread_kernel_state) +Lsp1_irq_str: + .asciz "IRQ exception taken while SP1 selected" + + .text + .align 2 +fleh_fiq_sp1: + mov x1, x0 + adr x0, Lsp1_fiq_str + b EXT(panic_with_thread_kernel_state) +Lsp1_fiq_str: + .asciz "FIQ exception taken while SP1 selected" + + .text + .align 2 +fleh_serror_sp1: + mov x1, x0 + adr x0, Lsp1_serror_str + b EXT(panic_with_thread_kernel_state) +Lsp1_serror_str: + .asciz "Asynchronous exception taken while SP1 selected" + + .text + .align 2 +exception_return_dispatch: + ldr w0, [x21, SS_FLAVOR] // x0 = (threadIs64Bit) ? ss_64.cpsr : ss_32.cpsr + cmp x0, ARM_SAVED_STATE64 + ldr w1, [x21, SS64_CPSR] + ldr w2, [x21, SS32_CPSR] + csel w0, w1, w2, eq + tbnz w0, PSR64_MODE_EL_SHIFT, return_to_kernel // Test for low bit of EL, return to kernel if set + b return_to_user + + .text + .align 2 +return_to_kernel: + tbnz w0, #DAIF_IRQF_SHIFT, Lkernel_skip_ast_taken // Skip AST check if IRQ disabled + msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable interrupts + mrs x0, TPIDR_EL1 // Load thread pointer + ldr w1, [x0, ACT_PREEMPT_CNT] // Load preemption count + cbnz x1, Lkernel_skip_ast_taken // If preemption disabled, skip AST check + ldr x1, [x0, ACT_CPUDATAP] // Get current CPU data pointer + ldr x2, [x1, CPU_PENDING_AST] // Get ASTs + tst x2, AST_URGENT // If no urgent ASTs, skip ast_taken + b.eq Lkernel_skip_ast_taken + mov sp, x21 // Switch to thread stack for preemption + PUSH_FRAME + bl EXT(ast_taken_kernel) // Handle AST_URGENT + POP_FRAME +Lkernel_skip_ast_taken: + b exception_return + + .text + .globl EXT(thread_bootstrap_return) +LEXT(thread_bootstrap_return) +#if CONFIG_DTRACE + bl EXT(dtrace_thread_bootstrap) +#endif + b EXT(thread_exception_return) + + .text + .globl EXT(thread_exception_return) +LEXT(thread_exception_return) + mrs x0, TPIDR_EL1 + add x21, x0, ACT_CONTEXT + ldr x21, [x21] + + // + // Fall Through to return_to_user from thread_exception_return. + // Note that if we move return_to_user or insert a new routine + // below thread_exception_return, the latter will need to change. + // + .text +return_to_user: +check_user_asts: + msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable interrupts + mrs x3, TPIDR_EL1 // Load thread pointer + + movn w2, #0 + str w2, [x3, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before returning to user + + ldr w0, [x3, TH_RWLOCK_CNT] + cbz w0, 1f // Detect unbalance RW lock/unlock + b rwlock_count_notzero +1: + + ldr x4, [x3, ACT_CPUDATAP] // Get current CPU data pointer + ldr x0, [x4, CPU_PENDING_AST] // Get ASTs + cbnz x0, user_take_ast // If pending ASTs, go service them + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + PUSH_FRAME + bl EXT(timer_state_event_kernel_to_user) + POP_FRAME + mrs x3, TPIDR_EL1 // Reload thread pointer +#endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */ + +#if (CONFIG_KERNEL_INTEGRITY && KERNEL_INTEGRITY_WT) + /* Watchtower + * + * Here we attempt to enable NEON access for EL0. If the last entry into the + * kernel from user-space was due to an IRQ, the monitor will have disabled + * NEON for EL0 _and_ access to CPACR_EL1 from EL1 (1). This forces xnu to + * check in with the monitor in order to reenable NEON for EL0 in exchange + * for routing IRQs through the monitor (2). This way the monitor will + * always 'own' either IRQs or EL0 NEON. + * + * If Watchtower is disabled or we did not enter the kernel through an IRQ + * (e.g. FIQ or syscall) this is a no-op, otherwise we will trap to EL3 + * here. + * + * EL0 user ________ IRQ ______ + * EL1 xnu \ ______________________ CPACR_EL1 __/ + * EL3 monitor \_/ \___/ + * + * (1) (2) + */ + + mov x0, #(CPACR_FPEN_ENABLE) + msr CPACR_EL1, x0 +#endif + + /* Establish this thread's debug state as the live state on the selected CPU. */ + ldr x4, [x3, ACT_CPUDATAP] // Get current CPU data pointer + ldr x1, [x4, CPU_USER_DEBUG] // Get Debug context + ldr x0, [x3, ACT_DEBUGDATA] + orr x1, x1, x0 // Thread debug state and live debug state both NULL? + cbnz x1, user_set_debug_state_and_return // If one or the other non-null, go set debug state + + // + // Fall through from return_to_user to exception_return. + // Note that if we move exception_return or add a new routine below + // return_to_user, the latter will have to change. + // + + +exception_return: + msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable interrupts + mrs x3, TPIDR_EL1 // Load thread pointer + mov sp, x21 // Reload the pcb pointer + + /* ARM64_TODO Reserve x18 until we decide what to do with it */ + ldr x0, [x3, TH_CTH_DATA] // Load cthread data pointer + str x0, [sp, SS64_X18] // and use it to trash x18 + +Lexception_return_restore_registers: + /* Restore special register state */ + ldr x0, [sp, SS64_PC] // Get the return address + ldr w1, [sp, SS64_CPSR] // Get the return CPSR + ldr w2, [sp, NS64_FPSR] + ldr w3, [sp, NS64_FPCR] + + msr ELR_EL1, x0 // Load the return address into ELR + msr SPSR_EL1, x1 // Load the return CPSR into SPSR + msr FPSR, x2 + msr FPCR, x3 // Synchronized by ERET + + mov x0, sp // x0 = &pcb + + /* Restore arm_neon_saved_state64 */ + ldp q0, q1, [x0, NS64_Q0] + ldp q2, q3, [x0, NS64_Q2] + ldp q4, q5, [x0, NS64_Q4] + ldp q6, q7, [x0, NS64_Q6] + ldp q8, q9, [x0, NS64_Q8] + ldp q10, q11, [x0, NS64_Q10] + ldp q12, q13, [x0, NS64_Q12] + ldp q14, q15, [x0, NS64_Q14] + ldp q16, q17, [x0, NS64_Q16] + ldp q18, q19, [x0, NS64_Q18] + ldp q20, q21, [x0, NS64_Q20] + ldp q22, q23, [x0, NS64_Q22] + ldp q24, q25, [x0, NS64_Q24] + ldp q26, q27, [x0, NS64_Q26] + ldp q28, q29, [x0, NS64_Q28] + ldp q30, q31, [x0, NS64_Q30] + + /* Restore arm_saved_state64 */ + + // Skip x0, x1 - we're using them + ldp x2, x3, [x0, SS64_X2] + ldp x4, x5, [x0, SS64_X4] + ldp x6, x7, [x0, SS64_X6] + ldp x8, x9, [x0, SS64_X8] + ldp x10, x11, [x0, SS64_X10] + ldp x12, x13, [x0, SS64_X12] + ldp x14, x15, [x0, SS64_X14] + ldp x16, x17, [x0, SS64_X16] + ldp x18, x19, [x0, SS64_X18] + ldp x20, x21, [x0, SS64_X20] + ldp x22, x23, [x0, SS64_X22] + ldp x24, x25, [x0, SS64_X24] + ldp x26, x27, [x0, SS64_X26] + ldr x28, [x0, SS64_X28] + ldp fp, lr, [x0, SS64_FP] + + // Restore stack pointer and our last two GPRs + ldr x1, [x0, SS64_SP] + mov sp, x1 + ldp x0, x1, [x0, SS64_X0] // Restore the GPRs + + eret + +user_take_ast: + PUSH_FRAME + bl EXT(ast_taken_user) // Handle all ASTs, may return via continuation + POP_FRAME + mrs x3, TPIDR_EL1 // Reload thread pointer + b check_user_asts // Now try again + +user_set_debug_state_and_return: + ldr x4, [x3, ACT_CPUDATAP] // Get current CPU data pointer + isb // Synchronize context + PUSH_FRAME + bl EXT(arm_debug_set) // Establish thread debug state in live regs + POP_FRAME + isb + mrs x3, TPIDR_EL1 // Reload thread pointer + b exception_return // And continue + + .text + .align 2 +preempt_underflow: + mrs x0, TPIDR_EL1 + str x0, [sp, #-16]! // We'll print thread pointer + adr x0, L_underflow_str // Format string + CALL_EXTERN panic // Game over + +L_underflow_str: + .asciz "Preemption count negative on thread %p" +.align 2 + + .text + .align 2 +rwlock_count_notzero: + mrs x0, TPIDR_EL1 + str x0, [sp, #-16]! // We'll print thread pointer + ldr w0, [x0, TH_RWLOCK_CNT] + str w0, [sp, #8] + adr x0, L_rwlock_count_notzero_str // Format string + CALL_EXTERN panic // Game over + +L_rwlock_count_notzero_str: + .asciz "RW lock count not 0 on thread %p (%u)" +.align 2 + + .text + .align 2 + .globl EXT(ml_panic_trap_to_debugger) +LEXT(ml_panic_trap_to_debugger) + ret + +/* ARM64_TODO Is globals_asm.h needed? */ +//#include "globals_asm.h" + +/* vim: set ts=4: */ diff --git a/osfmk/arm64/loose_ends.c b/osfmk/arm64/loose_ends.c new file mode 100644 index 000000000..1eec53104 --- /dev/null +++ b/osfmk/arm64/loose_ends.c @@ -0,0 +1,699 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !MACH_KDP +#include +#endif /* !MACH_KDP */ +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#define INT_SIZE (BYTE_SIZE * sizeof (int)) + +void +bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes) +{ + unsigned int src_index; + unsigned int dst_index; + vm_offset_t src_offset; + vm_offset_t dst_offset; + unsigned int wimg_bits_src, wimg_bits_dst; + unsigned int cpu_num = 0; + ppnum_t pn_src = (ppnum_t)(src >> PAGE_SHIFT); + ppnum_t pn_dst = (ppnum_t)(dst >> PAGE_SHIFT); + +#ifdef __ARM_COHERENT_IO__ + if (pmap_valid_address(src) && + pmap_valid_address(dst) && + (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)))) { + bcopy((char *)phystokv((pmap_paddr_t) src), (char *)phystokv((pmap_paddr_t) dst), bytes); + return; + } +#endif + + wimg_bits_src = pmap_cache_attributes(pn_src); + wimg_bits_dst = pmap_cache_attributes(pn_dst); + +#ifndef __ARM_COHERENT_IO__ + if (((wimg_bits_src & VM_WIMG_MASK) == VM_WIMG_DEFAULT) && + ((wimg_bits_dst & VM_WIMG_MASK) == VM_WIMG_DEFAULT) && + (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)))) { + /* Fast path - dst is writable and both source and destination have default attributes */ + bcopy((char *)phystokv((pmap_paddr_t) src), (char *)phystokv((pmap_paddr_t) dst), bytes); + return; + } +#endif + + src_offset = src & PAGE_MASK; + dst_offset = dst & PAGE_MASK; + + if ((src_offset + bytes) > PAGE_SIZE || (dst_offset + bytes) > PAGE_SIZE) + panic("bcopy extends beyond copy windows"); + + mp_disable_preemption(); + cpu_num = cpu_number(); + src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src); + dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ|VM_PROT_WRITE, wimg_bits_dst); + + bcopy((char *)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset), + (char *)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset), + bytes); + + pmap_unmap_cpu_windows_copy(src_index); + pmap_unmap_cpu_windows_copy(dst_index); + mp_enable_preemption(); +} + +void +bzero_phys_nc(addr64_t src64, vm_size_t bytes) +{ + bzero_phys(src64, bytes); +} + +/* Zero bytes starting at a physical address */ +void +bzero_phys(addr64_t src, vm_size_t bytes) +{ + unsigned int wimg_bits; + unsigned int cpu_num = cpu_number(); + ppnum_t pn = (ppnum_t)(src >> PAGE_SHIFT); + +#ifdef __ARM_COHERENT_IO__ + if (pmap_valid_address(src)) { + bzero((char *)phystokv((pmap_paddr_t) src), bytes); + return; + } +#endif + + wimg_bits = pmap_cache_attributes(pn); + +#ifndef __ARM_COHERENT_IO__ + if ((wimg_bits & VM_WIMG_MASK) == VM_WIMG_DEFAULT) { + /* Fast path - default attributes */ + bzero((char *)phystokv((pmap_paddr_t) src), bytes); + return; + } +#endif + + mp_disable_preemption(); + cpu_num = cpu_number(); + + while (bytes > 0) { + vm_offset_t offset = src & PAGE_MASK; + uint64_t count = PAGE_SIZE - offset; + + if (count > bytes) + count = bytes; + + pn = (ppnum_t)(src >> PAGE_SHIFT); + + unsigned int index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits); + + bzero((char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset), count); + + pmap_unmap_cpu_windows_copy(index); + + src += count; + bytes -= count; + } + + mp_enable_preemption(); +} + +/* + * Read data from a physical address. + */ + + +static unsigned long long +ml_phys_read_data(pmap_paddr_t paddr, int size) +{ + unsigned int index; + unsigned int wimg_bits; + ppnum_t pn = (ppnum_t)(paddr >> PAGE_SHIFT); + unsigned long long result = 0; + vm_offset_t copywindow_vaddr = 0; + unsigned char s1; + unsigned short s2; + unsigned int s4; + +#ifdef __ARM_COHERENT_IO__ + if (pmap_valid_address(paddr)) { + switch (size) { + case 1: + s1 = *(volatile unsigned char *)phystokv(paddr); + result = s1; + break; + case 2: + s2 = *(volatile unsigned short *)phystokv(paddr); + result = s2; + break; + case 4: + s4 = *(volatile unsigned int *)phystokv(paddr); + result = s4; + break; + case 8: + result = *(volatile unsigned long long *)phystokv(paddr); + break; + default: + panic("Invalid size %d for ml_phys_read_data\n", size); + break; + } + return result; + } +#endif + + mp_disable_preemption(); + wimg_bits = pmap_cache_attributes(pn); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits); + copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK); + + switch (size) { + case 1: + s1 = *(volatile unsigned char *)copywindow_vaddr; + result = s1; + break; + case 2: + s2 = *(volatile unsigned short *)copywindow_vaddr; + result = s2; + break; + case 4: + s4 = *(volatile unsigned int *)copywindow_vaddr; + result = s4; + break; + case 8: + result = *(volatile unsigned long long*)copywindow_vaddr; + break; + default: + panic("Invalid size %d for ml_phys_read_data\n", size); + break; + + } + + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); + + return result; +} + +unsigned int ml_phys_read( vm_offset_t paddr) +{ + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4); +} + +unsigned int ml_phys_read_word(vm_offset_t paddr) { + + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4); +} + +unsigned int ml_phys_read_64(addr64_t paddr64) +{ + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4); +} + +unsigned int ml_phys_read_word_64(addr64_t paddr64) +{ + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4); +} + +unsigned int ml_phys_read_half(vm_offset_t paddr) +{ + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 2); +} + +unsigned int ml_phys_read_half_64(addr64_t paddr64) +{ + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 2); +} + +unsigned int ml_phys_read_byte(vm_offset_t paddr) +{ + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 1); +} + +unsigned int ml_phys_read_byte_64(addr64_t paddr64) +{ + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 1); +} + +unsigned long long ml_phys_read_double(vm_offset_t paddr) +{ + return ml_phys_read_data((pmap_paddr_t)paddr, 8); +} + +unsigned long long ml_phys_read_double_64(addr64_t paddr64) +{ + return ml_phys_read_data((pmap_paddr_t)paddr64, 8); +} + + + +/* + * Write data to a physical address. + */ + +static void +ml_phys_write_data(pmap_paddr_t paddr, unsigned long long data, int size) +{ + unsigned int index; + unsigned int wimg_bits; + ppnum_t pn = (ppnum_t)(paddr >> PAGE_SHIFT); + vm_offset_t copywindow_vaddr = 0; + +#ifdef __ARM_COHERENT_IO__ + if (pmap_valid_address(paddr)) { + switch (size) { + case 1: + *(volatile unsigned char *)phystokv(paddr) = (unsigned char)data; + return; + case 2: + *(volatile unsigned short *)phystokv(paddr) = (unsigned short)data; + return; + case 4: + *(volatile unsigned int *)phystokv(paddr) = (unsigned int)data; + return; + case 8: + *(volatile unsigned long long *)phystokv(paddr) = data; + return; + default: + panic("Invalid size %d for ml_phys_write_data\n", size); + } + } +#endif + + mp_disable_preemption(); + wimg_bits = pmap_cache_attributes(pn); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK); + + switch (size) { + case 1: + *(volatile unsigned char *)(copywindow_vaddr) = + (unsigned char)data; + break; + case 2: + *(volatile unsigned short *)(copywindow_vaddr) = + (unsigned short)data; + break; + case 4: + *(volatile unsigned int *)(copywindow_vaddr) = + (uint32_t)data; + break; + case 8: + *(volatile unsigned long long *)(copywindow_vaddr) = + (unsigned long long)data; + break; + default: + panic("Invalid size %d for ml_phys_write_data\n", size); + break; + } + + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); +} + +void ml_phys_write_byte(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 1); +} + +void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 1); +} + +void ml_phys_write_half(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 2); +} + +void ml_phys_write_half_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 2); +} + +void ml_phys_write(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 4); +} + +void ml_phys_write_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); +} + +void ml_phys_write_word(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 4); +} + +void ml_phys_write_word_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); +} + +void ml_phys_write_double(vm_offset_t paddr, unsigned long long data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 8); +} + +void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 8); +} + + +/* + * Set indicated bit in bit string. + */ +void +setbit(int bitno, int *s) +{ + s[bitno / INT_SIZE] |= 1 << (bitno % INT_SIZE); +} + +/* + * Clear indicated bit in bit string. + */ +void +clrbit(int bitno, int *s) +{ + s[bitno / INT_SIZE] &= ~(1 << (bitno % INT_SIZE)); +} + +/* + * Test if indicated bit is set in bit string. + */ +int +testbit(int bitno, int *s) +{ + return s[bitno / INT_SIZE] & (1 << (bitno % INT_SIZE)); +} + +/* + * Find first bit set in bit string. + */ +int +ffsbit(int *s) +{ + int offset; + + for (offset = 0; !*s; offset += INT_SIZE, ++s); + return offset + __builtin_ctz(*s); +} + +int +ffs(unsigned int mask) +{ + if (mask == 0) + return 0; + + /* + * NOTE: cannot use __builtin_ffs because it generates a call to + * 'ffs' + */ + return 1 + __builtin_ctz(mask); +} + +int +ffsll(unsigned long long mask) +{ + if (mask == 0) + return 0; + + /* + * NOTE: cannot use __builtin_ffsll because it generates a call to + * 'ffsll' + */ + return 1 + __builtin_ctzll(mask); +} + +/* + * Find last bit set in bit string. + */ +int +fls(unsigned int mask) +{ + if (mask == 0) + return 0; + + return (sizeof (mask) << 3) - __builtin_clz(mask); +} + +int +flsll(unsigned long long mask) +{ + if (mask == 0) + return 0; + + return (sizeof (mask) << 3) - __builtin_clzll(mask); +} + +#undef bcmp +int +bcmp( + const void *pa, + const void *pb, + size_t len) +{ + const char *a = (const char *) pa; + const char *b = (const char *) pb; + + if (len == 0) + return 0; + + do + if (*a++ != *b++) + break; + while (--len); + + /* + * Check for the overflow case but continue to handle the non-overflow + * case the same way just in case someone is using the return value + * as more than zero/non-zero + */ + if ((len & 0xFFFFFFFF00000000ULL) && !(len & 0x00000000FFFFFFFFULL)) + return 0xFFFFFFFFL; + else + return (int)len; +} + +#undef memcmp +int +memcmp(const void *s1, const void *s2, size_t n) +{ + if (n != 0) { + const unsigned char *p1 = s1, *p2 = s2; + + do { + if (*p1++ != *p2++) + return (*--p1 - *--p2); + } while (--n != 0); + } + return (0); +} + +kern_return_t +copypv(addr64_t source, addr64_t sink, unsigned int size, int which) +{ + kern_return_t retval = KERN_SUCCESS; + void *from, *to; +#ifndef __ARM_COHERENT_IO__ + unsigned int from_wimg_bits, to_wimg_bits; +#endif + + from = CAST_DOWN(void *, source); + to = CAST_DOWN(void *, sink); + + if ((which & (cppvPsrc | cppvPsnk)) == 0) /* Make sure that only + * one is virtual */ + panic("copypv: no more than 1 parameter may be virtual\n"); /* Not allowed */ + + if (which & cppvPsrc) + from = (void *)phystokv(from); + if (which & cppvPsnk) + to = (void *)phystokv(to); + + if ((which & (cppvPsrc | cppvKmap)) == 0) /* Source is virtual in + * current map */ + retval = copyin((user_addr_t) from, to, size); + else if ((which & (cppvPsnk | cppvKmap)) == 0) /* Sink is virtual in + * current map */ + retval = copyout(from, (user_addr_t) to, size); + else /* both addresses are physical or kernel map */ + bcopy(from, to, size); + +#ifndef __ARM_COHERENT_IO__ + if (which & cppvFsrc) { + flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc)); + } else if (which & cppvPsrc) { + from_wimg_bits = pmap_cache_attributes(source >> PAGE_SHIFT); + if ((from_wimg_bits != VM_WIMG_COPYBACK) && (from_wimg_bits != VM_WIMG_WTHRU)) + flush_dcache64(source, size, TRUE); + } + + if (which & cppvFsnk) { + flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk)); + } else if (which & cppvPsnk) { + to_wimg_bits = pmap_cache_attributes(sink >> PAGE_SHIFT); + if (to_wimg_bits != VM_WIMG_COPYBACK) + flush_dcache64(sink, size, TRUE); + } +#endif + return retval; +} + + +#if MACH_ASSERT + +extern int copyinframe(vm_address_t fp, char *frame, boolean_t is64bit); + +/* + * Machine-dependent routine to fill in an array with up to callstack_max + * levels of return pc information. + */ +void +machine_callstack( + uintptr_t * buf, + vm_size_t callstack_max) +{ + /* Captures the USER call stack */ + uint32_t i=0; + + struct arm_saved_state *state = find_user_regs(current_thread()); + + if (!state) { + while (i: Reduce print noise + // kprintf("ml_thread_policy() unimplemented\n"); +} + +void +panic_unimplemented() +{ + panic("Not yet implemented."); +} + +/* ARM64_TODO */ +void abort(void); + +void +abort() +{ + panic("Abort."); +} + + +#if !MACH_KDP +void +kdp_register_callout(kdp_callout_fn_t fn, void *arg) +{ +#pragma unused(fn,arg) +} +#endif + diff --git a/osfmk/arm64/lowglobals.h b/osfmk/arm64/lowglobals.h new file mode 100644 index 000000000..c4d0c3df0 --- /dev/null +++ b/osfmk/arm64/lowglobals.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Header files for the Low Memory Globals (lg) + */ +#ifndef _LOW_MEMORY_GLOBALS_H_ +#define _LOW_MEMORY_GLOBALS_H_ + +#include +#include +#include +#include + +#ifndef __arm64__ +#error Wrong architecture - this file is meant for arm64 +#endif + +#define LOWGLO_LAYOUT_MAGIC 0xC0DEC0DE + +/* + * This structure is bound to lowmem_vectors.c. Make sure changes here are + * reflected there as well. + */ + +#pragma pack(8) /* Make sure the structure stays as we defined it */ +typedef struct lowglo { + unsigned char lgVerCode[8]; /* 0xffffff8000002000 System verification code */ + uint64_t lgZero; /* 0xffffff8000002008 Constant 0 */ + uint64_t lgStext; /* 0xffffff8000002010 Start of kernel text */ + uint64_t lgVersion; /* 0xffffff8000002018 Pointer to kernel version string */ + uint64_t lgOSVersion; /* 0xffffff8000002020 Pointer to OS version string */ + uint64_t lgKmodptr; /* 0xffffff8000002028 Pointer to kmod, debugging aid */ + uint64_t lgTransOff; /* 0xffffff8000002030 Pointer to kdp_trans_off, debugging aid */ + uint64_t lgRebootFlag; /* 0xffffff8000002038 Pointer to debugger reboot trigger */ + uint64_t lgManualPktAddr; /* 0xffffff8000002040 Pointer to manual packet structure */ + uint64_t lgAltDebugger; /* 0xffffff8000002048 Pointer to reserved space for alternate kernel debugger */ + uint64_t lgPmapMemQ; /* 0xffffff8000002050 Pointer to PMAP memory queue */ + uint64_t lgPmapMemPageOffset;/* 0xffffff8000002058 Offset of physical page member in vm_page_t or vm_page_with_ppnum_t */ + uint64_t lgPmapMemChainOffset;/*0xffffff8000002060 Offset of listq in vm_page_t or vm_page_with_ppnum_t */ + uint64_t lgStaticAddr; /* 0xffffff8000002068 Static allocation address */ + uint64_t lgStaticSize; /* 0xffffff8000002070 Static allocation size */ + uint64_t lgLayoutMajorVersion; /* 0xffffff8000002078 Lowglo major layout version */ + uint64_t lgLayoutMagic; /* 0xffffff8000002080 Magic value evaluated to determine if lgLayoutVersion is valid */ + uint64_t lgPmapMemStartAddr; /* 0xffffff8000002088 Pointer to start of vm_page_t array */ + uint64_t lgPmapMemEndAddr; /* 0xffffff8000002090 Pointer to end of vm_page_t array */ + uint64_t lgPmapMemPagesize; /* 0xffffff8000002098 size of vm_page_t */ + uint64_t lgPmapMemFromArrayMask; /* 0xffffff80000020A0 Mask to indicate page is from vm_page_t array */ + uint64_t lgPmapMemFirstppnum; /* 0xffffff80000020A8 physical page number of the first vm_page_t in the array */ + uint64_t lgPmapMemPackedShift; /* 0xffffff80000020B0 alignment of packed pointer */ + uint64_t lgPmapMemPackedBaseAddr;/* 0xffffff80000020B8 base address of that packed pointers are relative to */ + uint64_t lgLayoutMinorVersion; /* 0xffffff80000020C0 Lowglo minor layout version */ + uint64_t lgPageShift; /* 0xffffff80000020C8 number of shifts from page number to size */ +} lowglo; +#pragma pack() + +extern lowglo lowGlo; + +void patch_low_glo(void); +void patch_low_glo_static_region(uint64_t address, uint64_t size); +void patch_low_glo_vm_page_info(void *, void *, uint32_t); + +#endif /* _LOW_MEMORY_GLOBALS_H_ */ diff --git a/osfmk/arm64/lowmem_vectors.c b/osfmk/arm64/lowmem_vectors.c new file mode 100644 index 000000000..a04ef6d76 --- /dev/null +++ b/osfmk/arm64/lowmem_vectors.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include + +/* + * On arm64, the low globals get mapped low via machine_init() during kernel + * bootstrap. + */ + +extern vm_offset_t vm_kernel_stext; +extern void *version; +extern void *kmod; +extern void *kdp_trans_off; +extern void *osversion; +extern void *flag_kdp_trigger_reboot; +extern void *manual_pkt; +extern struct vm_object pmap_object_store; /* store pt pages */ + +lowglo lowGlo __attribute__ ((aligned(PAGE_MAX_SIZE))) = { + // Increment the major version for changes that break the current Astris + // usage of lowGlo values + // Increment the minor version for changes that provide additonal info/function + // but does not break current usage + .lgLayoutMajorVersion = 3, + .lgLayoutMinorVersion = 0, + .lgLayoutMagic = LOWGLO_LAYOUT_MAGIC, + .lgVerCode = { 'K','r','a','k','e','n',' ',' ' }, + .lgZero = 0, + .lgStext = 0, // To be filled in below + .lgVersion = (uint64_t) &version, + .lgOSVersion = (uint64_t) &osversion, + .lgKmodptr = (uint64_t) &kmod, +#if MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING + .lgTransOff = (uint64_t) &kdp_trans_off, + .lgRebootFlag = (uint64_t) &flag_kdp_trigger_reboot, + .lgManualPktAddr = (uint64_t) &manual_pkt, +#endif + .lgPmapMemQ = (uint64_t)&(pmap_object_store.memq), + .lgPmapMemPageOffset = offsetof(struct vm_page_with_ppnum, phys_page), + .lgPmapMemChainOffset = offsetof(struct vm_page, listq), + .lgPmapMemPagesize = (uint64_t)sizeof(struct vm_page), + .lgPmapMemFromArrayMask = VM_PACKED_FROM_VM_PAGES_ARRAY, + .lgPmapMemPackedShift = VM_PACKED_POINTER_SHIFT, + .lgPmapMemPackedBaseAddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS, + .lgPmapMemStartAddr = -1, + .lgPmapMemEndAddr = -1, + .lgPmapMemFirstppnum = -1 +}; + +void patch_low_glo(void) +{ + lowGlo.lgStext = (uint64_t)vm_kernel_stext; + lowGlo.lgPageShift = PAGE_SHIFT; +} + +void patch_low_glo_static_region(uint64_t address, uint64_t size) +{ + lowGlo.lgStaticAddr = address; + lowGlo.lgStaticSize = size; +} + + +void patch_low_glo_vm_page_info(void * start_addr, void * end_addr, uint32_t first_ppnum) +{ + lowGlo.lgPmapMemStartAddr = (uint64_t)start_addr; + lowGlo.lgPmapMemEndAddr = (uint64_t)end_addr; + lowGlo.lgPmapMemFirstppnum = first_ppnum; +} diff --git a/osfmk/arm64/lz4_decode_arm64.s b/osfmk/arm64/lz4_decode_arm64.s new file mode 100644 index 000000000..2c7353465 --- /dev/null +++ b/osfmk/arm64/lz4_decode_arm64.s @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#if LZ4_ENABLE_ASSEMBLY_DECODE_ARM64 + +/* + + int64_t lz4_decode_asm( + uint8_t ** dst_ptr, *dst_ptr points to next output byte to write + uint8_t * dst_begin, points to first valid output byte we can access, dst_begin <= dst + uint8_t * dst_end, "relaxed" end of output buffer (see below) + const uint8_t ** src_ptr, *src_ptr points to next input byte to read + const uint8_t * src_end) "relaxed" end of input buffer (see below) + + We test the position of the pointers only to ensure we don't access past src_end/dst_end + some fixed constant. + We never read before dst_begin. + + Return 0 on success, -1 on failure + On output, (*src_ptr,*dst_ptr) receives the last position in both buffers corresponding to the beginning of a LZ4 instruction. + +*/ + +.globl _lz4_decode_asm + +#define dst x0 // arg0 +#define dst_begin x1 // arg1 +#define dst_end x2 // arg2 +#define src x3 // arg3 +#define src_end x4 // arg4 + +#define w_n_matches w5 // lower 32 bits of n_matches +#define n_matches x5 +#define n_literals x6 +#define copy_src x7 // match/literal copy source +#define copy_dst x8 // match/literal copy destination + +#define w_aux1 w9 // lower 32 bits of aux1 +#define aux1 x9 +#define aux2 x10 + +#define w_match_distance w11 // lower 32 bits of match_distance +#define match_distance x11 + +#define match_permtable x12 +#define match_disttable x13 + +#define dst_good x19 +#define src_good x20 + +.macro establish_frame + stp fp, lr, [sp, #-16]! + mov fp, sp +.endm + +.macro clear_frame_and_return + ldp fp, lr, [sp], #16 + ret lr +.endm + +// copy_1x16 SOURCE_ADDR DESTINATION_ADDR +// Copy 16 bytes, clobber: q0 +.macro copy_1x16 + ldr q0,[$0] + str q0,[$1] +.endm + +// copy_1x16_and_increment SOURCE_ADDR DESTINATION_ADDR +// Copy 16 bytes, and increment both addresses by 16, clobber: q0 +.macro copy_1x16_and_increment + ldr q0,[$0],#16 + str q0,[$1],#16 +.endm + +// copy_2x16_and_increment SOURCE_ADDR DESTINATION_ADDR +// Copy 2 times 16 bytes, and increment both addresses by 32, clobber: q0 +.macro copy_2x16_and_increment + ldr q0,[$0],#16 + str q0,[$1],#16 + ldr q0,[$0],#16 + str q0,[$1],#16 +.endm + +// copy_1x32_and_increment SOURCE_ADDR DESTINATION_ADDR +// Copy 32 bytes, and increment both addresses by 32, clobber: q0,q1 +.macro copy_1x32_and_increment + ldp q0,q1,[$0],#32 + stp q0,q1,[$1],#32 +.endm + +// If we don't branch, src < src_end after this +.macro check_src_end + cmp src,src_end + b.hs L_done // extremely unlikely, DONE when src >= src_end +.endm + +// If we don't branch, dst < dst_end after this +.macro check_dst_end + cmp dst,dst_end + b.hs L_done // extremely unlikely, DONE when dst >= dst_end +.endm + +.text +.p2align 4 +_lz4_decode_asm: + establish_frame + stp x19,x20,[sp,#-16]! // need to preserve these + stp src,dst,[sp,#-16]! // save src_ptr,dst_ptr on stack + ldr src,[src] // src = *src_ptr + ldr dst,[dst] // dst = *dst_ptr + adr match_permtable,L_match_permtable + adr match_disttable,L_match_disttable + +L_decode_command: + // Keep last known good positions in both streams + mov dst_good,dst + mov src_good,src + + // Check limits + check_src_end + check_dst_end + + // Decode 1-byte command + ldrb w_aux1,[src],#1 // read command byte LLLLMMMM + lsr n_literals,aux1,#4 // 0000LLLL. n_literals is now 0..15 + and n_matches,aux1,#0xf // 0000MMMM. n_matches is now 0..15 + add n_matches,n_matches,#4 // n_matches is now 4..19 + + // Test number of literals (do not test if n_literals==0, because branch prediction fails on it) + cmp n_literals,#14 + b.ls L_copy_short_literal // 96% likely: n_literals in 0..14 + // continue to decode_long_literal + + // the number of literals is encoded on more bytes, we need to decode them +L_decode_long_literal: + check_src_end // required here, since we may loop an arbitrarily high number of times + ldrb w_aux1,[src],#1 + add n_literals,n_literals,aux1 + cmp aux1,#255 + b.eq L_decode_long_literal // extremely unlikely + // continue to copy_long_literal + + // Copy literals, n_literals >= 15 +L_copy_long_literal: + mov copy_src,src // literal copy origin + mov copy_dst,dst // literal copy destination + add src,src,n_literals + add dst,dst,n_literals + check_src_end // required here, since n_literals can be arbitrarily high + check_dst_end + + // fixed + loop + copy_1x32_and_increment copy_src,copy_dst +L_copy_long_literal_loop: + copy_1x32_and_increment copy_src,copy_dst + cmp dst,copy_dst + b.hi L_copy_long_literal_loop // first test occurs after 64 bytes have been copied, and is unlikely to loop back + b L_expand_match + + // Copy literals, n_literals <= 14: copy 16 bytes +L_copy_short_literal: + copy_1x16 src,dst + add src,src,n_literals + add dst,dst,n_literals + // continue to expand match + +L_expand_match: + + // Decode match distance + ldrh w_match_distance,[src],#2 // 16-bit distance + cbz match_distance,L_fail // distance == 0 is invalid + sub copy_src,dst,match_distance // copy_src is the match copy source + cmp copy_src,dst_begin + b.lo L_fail // copy_src < dst_begin: FAIL + mov copy_dst,dst // copy_dst is the match copy destination + add dst,dst,n_matches // dst is updated to be the byte after the match; n_matches <= 19 here + + // Do we need to decode a long match? + cmp n_matches,#19 + b.eq L_decode_long_match // unlikely, n_matches >= 19 encoded on more bytes + cmp n_matches,#16 + b.hi L_long_match // unlikely, n_matches == 17 or 18 + // continue to short match (most probable case) + + // Copy match, n_matches <= 16 +L_short_match: + cmp match_distance,#15 + b.ls L_copy_short_match_small_distance + + // Copy match, n_matches <= 16, match_distance >= 16: copy 16 bytes + copy_1x16 copy_src,copy_dst + b L_decode_command + + // Copy match, n_matches <= 16, match_distance < 16: + // load shuffle table, and permute to replicate the pattern on 16 bytes +L_copy_short_match_small_distance: + ldr q0,[copy_src] + add aux1,match_permtable,match_distance,lsl #5 // index in table + ldr q1,[aux1] // load only permutation for the low 16 bytes + tbl v0.16b,{v0.16b},v1.16b // low 16 bytes of pattern + str q0,[copy_dst] + b L_decode_command + + // n_matches == 19: the number of matches in encoded on more bytes, we need to decode them +L_decode_long_match: + check_src_end // required here, since we may loop an arbitrarily high number of times + ldrb w_aux1,[src],#1 + add dst,dst,aux1 + cmp aux1,#255 + b.eq L_decode_long_match // very unlikely + check_dst_end // required here, since dst was incremented by a arbitrarily high value + // continue to long_match + + // n_matches > 16 +L_long_match: + cmp match_distance,#31 + b.hi L_copy_long_match_32 + cmp match_distance,#15 + b.hi L_copy_long_match_16 + + // Copy match, n_matches >= 16, match_distance < 16: + // load shuffle table, and permute to replicate the pattern on 32 bytes +L_copy_long_match_small_distance: + ldr q1,[copy_src] // 16 pattern bytes + add aux1,match_permtable,match_distance,lsl #5 // index in table + ldp q2,q3,[aux1] // load 32-byte permutation + tbl v0.16b,{v1.16b},v2.16b // low 16 bytes of pattern in q0 + tbl v1.16b,{v1.16b},v3.16b // high 16 bytes of pattern in q1 + ldrb w_aux1,[match_disttable,match_distance] // valid pattern length in aux1 + // fixed + stp q0,q1,[copy_dst] + add copy_dst,copy_dst,aux1 +L_copy_long_match_small_distance_loop: + // loop + stp q0,q1,[copy_dst] + add copy_dst,copy_dst,aux1 + stp q0,q1,[copy_dst] + add copy_dst,copy_dst,aux1 + cmp dst,copy_dst + b.hi L_copy_long_match_small_distance_loop + b L_decode_command + + // Copy match, n_matches >= 16, match_distance >= 32 +L_copy_long_match_32: + // fixed + loop + copy_1x16_and_increment copy_src,copy_dst +L_copy_long_match_32_loop: + copy_1x32_and_increment copy_src,copy_dst + cmp dst,copy_dst + b.hi L_copy_long_match_32_loop + b L_decode_command + + // Copy match, n_matches >= 16, match_distance >= 16 +L_copy_long_match_16: + // fixed + loop + copy_1x16_and_increment copy_src,copy_dst +L_copy_long_match_16_loop: + copy_2x16_and_increment copy_src,copy_dst + cmp dst,copy_dst + b.hi L_copy_long_match_16_loop + b L_decode_command + +L_fail: + mov aux1,#-1 // FAIL + b L_exit + +L_done: + mov aux1,#0 // OK + // continue to L_exit + +L_exit: + ldp src,dst,[sp],#16 // get back src_ptr,dst_ptr from stack + str src_good,[src] // *src_ptr = src_good + str dst_good,[dst] // *dst_ptr = dst_good + mov x0,aux1 // x0 = return value + ldp x19,x20,[sp],#16 // restore + clear_frame_and_return + +// permutation tables for short distance matches, 32 byte result, for match_distance = 0 to 15 +// value(d)[i] = i%d for i = 0..31 +.p2align 6 +L_match_permtable: +.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 // 0 +.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 // 1 +.byte 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 // 2 +.byte 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1 // 3 +.byte 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 // 4 +.byte 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1 // 5 +.byte 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1 // 6 +.byte 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3 // 7 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 // 8 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4 // 9 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1 // 10 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 // 11 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11, 0, 1, 2, 3, 4, 5, 6, 7 // 12 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12, 0, 1, 2, 3, 4, 5 // 13 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13, 0, 1, 2, 3 // 14 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, 0, 1 // 15 + +// valid repeating pattern size, for each match_distance = 0 to 15 +// value(d) = 32 - (32%d), is the largest a multiple of d <= 32 +.p2align 6 +L_match_disttable: +.byte 32,32,32,30 // 0 .. 3 +.byte 16,30,30,28 // 4 .. 7 +.byte 16,27,30,22 // 8 .. 11 +.byte 24,26,28,30 // 12 .. 15 + +#endif // LZ4_ENABLE_ASSEMBLY_DECODE_ARM64 diff --git a/osfmk/arm64/lz4_encode_arm64.s b/osfmk/arm64/lz4_encode_arm64.s new file mode 100644 index 000000000..bf94a8536 --- /dev/null +++ b/osfmk/arm64/lz4_encode_arm64.s @@ -0,0 +1,406 @@ +/* + * Copyright (c) 2016-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include + +#if LZ4_ENABLE_ASSEMBLY_ENCODE_ARM64 + +/* void lz4_encode_2gb(uint8_t ** dst_ptr, + size_t dst_size, + const uint8_t ** src_ptr, + const uint8_t * src_begin, + size_t src_size, + lz4_hash_entry_t hash_table[LZ4_COMPRESS_HASH_ENTRIES], + int skip_final_literals) */ + +.globl _lz4_encode_2gb + +#define dst_ptr x0 +#define dst_size x1 +#define src_ptr x2 +#define src_begin x3 +#define src_size x4 +#define hash_table x5 +#define skip_final_literals x6 + +.text +.p2align 4 +_lz4_encode_2gb: + + // esteblish frame + stp fp, lr, [sp, #-16]! + mov fp, sp + + stp x19, x20, [sp, #-16]! + stp x21, x22, [sp, #-16]! + stp x23, x24, [sp, #-16]! + stp x25, x26, [sp, #-16]! + stp x27, x28, [sp, #-16]! + + // constant registers + adr x7, L_constant + ldr w28, [x7, #4] // x28 = 0x80808081 (magic number to cmopute 1/255) + ldr w7, [x7] // x7 = LZ4_COMPRESS_HASH_MULTIPLY + mov x27, #-1 // x27 = 0xffffffffffffffff + dup.4s v1, w27 // q1 = {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff} + + + // x9 - is current dst + // x10 - dst_end - safety_margin + ldr x9, [x0] // dst + add x10, x9, x1 // dst_end + sub x10, x10, #LZ4_GOFAST_SAFETY_MARGIN // dst_end - safety_margin + cmp x10, x9 // if dst_size < safety_margin abort + b.lt L_done + + // x11 - is current src + // x12 - is src_end - safety margin + ldr x11, [x2] // src + add x12, x11, x4 // src_end + sub x12, x12, #LZ4_GOFAST_SAFETY_MARGIN // src_end - safety_margin + cmp x12, x11 // if src_size < safety_margin skip to trailing_literals + b.lt L_trailing_literals + + + // this block search for the next available match + // set match_begin to current src (which is also where last match ended) +L_search_next_available_match: + mov x13, x11 // match_begin = src + sub x14, x13, x3 // match_postion = match_begin - src_begin + + // compute hash value for the next 5 "quads" + // hash distance need to be 0 < D < 0x10000 + +L_hash_match: + ldr x15, [x13] // match_first_4_bytes + umull x20, w7, w15 // match_bytes * LZ4_COMPRESS_HASH_MULTIPLY + lsr w20, w20, #LZ4_COMPRESS_HASH_SHIFT // use LZ4_COMPRESS_HASH_BITS MSbits as index + add x20, x5, x20, lsl #3 // hash_table_entry ptr (hash + 8*index) + + ldp w19, w22, [x20] // read entry values (w19 - pos, w22 - 4 bytes at pos) + stp w14, w15, [x20] // write entry values (w14 - current pos, w15 - current 4 bytes) + + add x26, x14, #1 // next_match pos + lsr x25, x15, #8 // next_match_first_4_bytes + umull x21, w7, w25 // match_bytes * LZ4_COMPRESS_HASH_MULTIPLY + lsr w21, w21, #LZ4_COMPRESS_HASH_SHIFT // use LZ4_COMPRESS_HASH_BITS MSbits as index + add x21, x5, x21, lsl #3 // hash_table_entry ptr (hash + 8*index) + + ldp w23, w24, [x21] // read entry values (w23 - pos, w24 - 4 bytes at pos) + stp w26, w25, [x21] // write entry values (w26 - next pos, w25 - next 4 bytes) + + cmp w15, w22 + b.ne L_try_next_match_0 // compare the 4 bytes to see if there is a match + sub w19, w14, w19 // x19 - match_dist (current_pos - match_pos) + cmp w19, #0x10000 + ccmp w19, #0, #0xf, lo + b.eq L_try_next_match_0 // verify the 0 < dist < 0x10000 + b L_found_valid_match + +L_try_next_match_0: + add x13, x13, #1 + add x14, x14, #1 + + add x26, x14, #1 // next_match pos + lsr x15, x15, #16 // next_match_first_4_bytes + umull x20, w7, w15 // match_bytes * LZ4_COMPRESS_HASH_MULTIPLY + lsr w20, w20, #LZ4_COMPRESS_HASH_SHIFT // use LZ4_COMPRESS_HASH_BITS MSbits as index + add x20, x5, x20, lsl #3 // hash_table_entry ptr (hash + 8*index) + + ldp w21, w22, [x20] // read entry values (w19 - pos, w22 - 4 bytes at pos) + stp w26, w15, [x20] // write entry values (w14 - current pos, w15 - current 4 bytes) + + cmp w25, w24 + b.ne L_try_next_match_1 // compare the 4 bytes to see if there is a match + sub w19, w14, w23 // x19 - match_dist (current_pos - match_pos) + cmp w19, #0x10000 + ccmp w19, #0, #0xf, lo + b.eq L_try_next_match_1 // verify the 0 < dist < 0x10000 + b L_found_valid_match + +L_try_next_match_1: + add x13, x13, #1 + add x14, x14, #1 + + add x26, x14, #1 // next_match pos + lsr x25, x15, #8 // next_match_first_4_bytes + umull x20, w7, w25 // match_bytes * LZ4_COMPRESS_HASH_MULTIPLY + lsr w20, w20, #LZ4_COMPRESS_HASH_SHIFT // use LZ4_COMPRESS_HASH_BITS MSbits as index + add x20, x5, x20, lsl #3 // hash_table_entry ptr (hash + 8*index) + + ldp w23, w24, [x20] // read entry values (w23 - pos, w24 - 4 bytes at pos) + stp w26, w25, [x20] // write entry values (w26 - next pos, w25 - next 4 bytes) + + cmp w15, w22 + b.ne L_try_next_match_2 // compare the 4 bytes to see if there is a match + sub w19, w14, w21 // x19 - match_dist (current_pos - match_pos) + cmp w19, #0x10000 + ccmp w19, #0, #0xf, lo + b.eq L_try_next_match_2 // verify the 0 < dist < 0x10000 + b L_found_valid_match + +L_try_next_match_2: + add x13, x13, #1 + add x14, x14, #1 + + add x26, x14, #1 // next_match pos + lsr x15, x15, #16 // next_match_first_4_bytes + umull x20, w7, w15 // match_bytes * LZ4_COMPRESS_HASH_MULTIPLY + lsr w20, w20, #LZ4_COMPRESS_HASH_SHIFT // use LZ4_COMPRESS_HASH_BITS MSbits as index + add x20, x5, x20, lsl #3 // hash_table_entry ptr (hash + 8*index) + + ldp w21, w22, [x20] // read entry values (w19 - pos, w22 - 4 bytes at pos) + stp w26, w15, [x20] // write entry values (w14 - current pos, w15 - current 4 bytes) + + cmp w25, w24 + b.ne L_try_next_match_3 // compare the 4 bytes to see if there is a match + sub w19, w14, w23 // x19 - match_dist (current_pos - match_pos) + cmp w19, #0x10000 + ccmp w19, #0, #0xf, lo + b.eq L_try_next_match_3 // verify the 0 < dist < 0x10000 + b L_found_valid_match + +L_try_next_match_3: + add x13, x13, #1 + add x14, x14, #1 + + cmp w15, w22 + b.ne L_try_next_matchs // compare the 4 bytes to see if there is a match + sub w19, w14, w21 // x19 - match_dist (current_pos - match_pos) + cmp w19, #0x10000 + ccmp w19, #0, #0xf, lo + b.eq L_try_next_matchs // verify the 0 < dist < 0x10000 + b L_found_valid_match + + // this block exapnd the valid match as much as possible + // first it try to expand the match forward + // next it try to expand the match backword +L_found_valid_match: + add x20, x13, #4 // match_end = match_begin+4 (already confirmd the first 4 bytes) + sub x21, x20, x19 // ref_end = match_end - dist +L_found_valid_match_expand_forward_loop: + ldr x22, [x20], #8 // load match_current_8_bytes (safe to load becasue of safety margin) + ldr x23, [x21], #8 // load ref_current_8_bytes + cmp x22, x23 + b.ne L_found_valid_match_expand_forward_partial + cmp x20, x12 // check if match_end reached src_end + b.lo L_found_valid_match_expand_forward_loop + b L_found_valid_match_expand_backward +L_found_valid_match_expand_forward_partial: + sub x20, x20, #8 // revert match_end by 8 and compute actual match of current 8 bytes + eor x22, x22, x23 // compare the bits using xor + rbit x22, x22 // revert the bits to use clz (the none equivalent bytes would have at least 1 set bit) + clz x22, x22 // after the revrse for every equal prefix byte clz would count 8 + add x20, x20, x22, lsr #3 // add the actual number of matching bytes is (clz result)>>3 +L_found_valid_match_expand_backward: + sub x15, x13, x19 // ref_begin = match_begin - dist +L_found_valid_match_expand_backward_loop: + cmp x13, x11 // check if match_begin reached src (previous match end) + ccmp x15, x3, #0xd, gt // check if ref_begin reached src_begin + b.le L_found_valid_match_emit_match + ldrb w22, [x13, #-1]! // load match_current_8_bytes (safe to load becasue of safety margin) + ldrb w23, [x15, #-1]! // load ref_current_8_bytes + cmp w22, w23 + b.eq L_found_valid_match_expand_backward_loop + add x13, x13, #1 // revert x13, last compare didn't match + + // this block write the match into dst + // it write the ML token [extra L tokens] [literals] <2byte dist> [extar M tokens] + // it update src & dst positions and progress to L_search_next_available_match +L_found_valid_match_emit_match: + sub x21, x20, x13 // match_length - match_end - match_begin + sub x21, x21, #4 // match_length - 4 (first 4 bytes are guaranteed) + sub x22, x13, x11 // literals_length = match_begin - src // compute + sub x26, x10, x9 // dst_remaining_space = dst_end - dst + sub x26, x26, x22 // dst_remaining_space -= literals_length + subs x26, x26, #3 // dst_remaining_space -= 2_dist_bytes + L/M_token + b.lo L_done // exit if dst isn't sufficent + + and x23, x21, #0xf // store M 4 LSbits + add x23, x23, x22, lsl #4 // add L 4 LSbits + add x15, x9, #1 // tmp_dst = dst + 1 + cmp x22, #15 // if L >= 15 need to write more L tokens + b.lo L_found_valid_match_copy_literals + orr x23, x23, #0xf0 // update L/M token to be 0xfM + sub x24, x22, #15 // reduce 15 from number_of_literals + sub x26, x26, #1 // check if there is space for the extra L token + b.lo L_done + cmp x24, #255 // check if need to compute number of 255 tokens + b.lo L_found_valid_match_skip_L_255_tokens + umull x25, w24, w28 // x25 - (literals_to_token * 1_DIV_255_magic_number) + lsr x25, x25, #39 // x25 - number_of_255_tokens = (literals_to_token * 1_DIV_255_magic_number)>>39 + subs x26, x26, x25 // check if there is sufficent space for the 255_tokens + b.lo L_done + mov x13, #255 + umsubl x24, w25, w13, x24 // x24 - value_of_remainder_token = literals_to_token - (number_of_255_tokens*255) +L_found_valid_match_L_255_tokens_loop: + str q1, [x15], #16 // store 16 255 tokens into dst_tmp. safe to store because dst has safety_margin + subs x25, x25, #16 // check if there are any 255 token left after current 16 + b.hi L_found_valid_match_L_255_tokens_loop + add x15, x15, x25 // revert tmp_dst if written too many 255 tokens. +L_found_valid_match_skip_L_255_tokens: + strb w24, [x15], #1 // write last L token +L_found_valid_match_copy_literals: + ldr q0, [x11], #16 // load current 16 literals. (safe becasue src_end has safety margin) + str q0, [x15], #16 // store current 16 literals. (safe becasue dst_end has safety margin) + subs x22, x22, #16 + b.gt L_found_valid_match_copy_literals + add x15, x15, x22 // revert tmp_dst if written too many literals + strh w19, [x15], #2 // store dist bytes + cmp x21, #15 // if M >= 15 need to write more M tokens + b.lo L_found_valid_match_finish_writing_match + orr x23, x23, #0xf // update L/M token to be 0xLf + sub x24, x21, #15 // reduce 15 from match_length + sub x26, x26, #1 // check if there is space for the extra M token + b.lo L_done + cmp x24, #255 // check if need to compute number of 255 tokens + b.lo L_found_valid_match_skip_M_255_tokens + umull x25, w24, w28 // x25 - (match_length * 1_DIV_255_magic_number) + lsr x25, x25, #39 // x25 - number_of_255_tokens = (match_length * 1_DIV_255_magic_number)>>39 + subs x26, x26, x25 // check if there is sufficent space for the 255_tokens + b.lo L_done + mov x13, #255 + umsubl x24, w25, w13, x24 // x24 - value_of_remainder_token = literals_to_token - (match_length*255) +L_found_valid_match_M_255_tokens_loop: + str q1, [x15], #16 // store 16 255 tokens into dst_tmp. safe to store because dst has safety_margin + subs x25, x25, #16 // check if there are any 255 token left after current 16 + b.hi L_found_valid_match_M_255_tokens_loop + add x15, x15, x25 // revert tmp_dst if written too many 255 tokens. +L_found_valid_match_skip_M_255_tokens: + strb w24, [x15], #1 // write last M token +L_found_valid_match_finish_writing_match: + strb w23, [x9] // store first token of match in dst + mov x9, x15 // update dst to last postion written + mov x11, x20 // update src to match_end (last byte that was encoded) + cmp x11, x12 // check if src reached src_end + ccmp x9, x10, #9, lt // check if dst reached dst_end + b.ge L_trailing_literals + b L_search_next_available_match + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + // attempted to hash three quad values from the end of each emited match + // this eneded up being slower and less compression (???) + // this block set match_begin and pos for next hash search and + // compute the hash values for the last 3 bytes of currently emited match + // only need to comute these hash becasue other "quads" were hashed when the original + // data was read. + +L_try_next_matchs: + add x13, x13, #1 // move to next match + add x14, x14, #1 // update next match pos + cmp x13, x12 // check match_begin didn't reach src_end + b.lo L_hash_match + +L_trailing_literals: + // unless skip_final_literals is set + // write the trailing bytes as literals + // traliing bytes include the whole src (with the safty margin) + // need to verify whole dst (withthe safty margin) has sufficent space + + tst x6, x6 + b.ne L_done // if skip_final_literals is set skip writing them + + add x12, x12, #LZ4_GOFAST_SAFETY_MARGIN // add safety_margin + subs x13, x12, x11 // remaining_src + b.eq L_done // finish if there are 0 trailing literals + + add x10, x10, #LZ4_GOFAST_SAFETY_MARGIN // add safety_margin + sub x14, x10, x9 // remaining dst (dst_end - dst) + sub x14, x14, #1 // 1 byte is needed at least to write literals token + subs x14, x14, x13 // finish if dst can't contain all remaining literals + 1 literals token + b.le L_done // (need to verify that it has room for literals tokens + + cmp x13, #15 + b.lt L_trailing_literals_store_less_than_15_literals + subs x14, x14, #1 // 1-extra byte is needed for literals tokens + b.mi L_done + mov w15, #0xf0 + strb w15, [x9], #1 // write literals first token (Important !!! if 255 tokens exist but dst isn't sufficent need to revert dst by 1) + sub x15, x13, #15 + cmp x15, #255 + b.lo L_trailing_literals_no_255_tokens + umull x19, w15, w28 // x19 - (literals_to_token * 1_DIV_255_magic_number) + lsr x19, x19, #39 // x19 - number_of_255_tokens = (literals_to_token * 1_DIV_255_magic_number)>>39 + subs x14, x14, x19 + b.mi L_revert_x9_and_done + mov x26, #255 + umsubl x15, w26, w19, x15 // x15 - value_of_remainder_token = literals_to_token - (number_of_255_tokens*255) +L_tariling_literals_write_16_255_tokens: + str q1, [x9], #16 // store 16 255 tokens each iteration (this is safe becasue there is space for 15 or more literals + remainder token) + subs x19, x19, #16 + b.gt L_tariling_literals_write_16_255_tokens + add x9, x9, x19 // fixes dst to actual number of tokens (x19 might not be a mulitple of 16) +L_trailing_literals_no_255_tokens: + strb w15, [x9], #1 // store remainder_token + lsr x14, x13, #4 // check if there are more than 16 literals left to be written + tst x14, x14 + b.eq L_trailing_literals_copy_less_than_16_literals +L_trailing_literals_copy_16_literals: + ldr q0, [x11], #16 // load current_16_literals + str q0, [ x9], #16 // *dst16++ = current_16_literals + subs x14, x14, #1 + b.gt L_trailing_literals_copy_16_literals + cmp x11, x12 + b.lo L_trailing_literals_copy_less_than_16_literals + b L_done + +L_trailing_literals_store_less_than_15_literals: + lsl x14, x13, #4 // literals_only_token is 0xL0 (where L is 4 bits) + strb w14, [x9], #1 // *dst++ = literals_only_token +L_trailing_literals_copy_less_than_16_literals: + ldrb w13, [x11], #1 // load current_literal + strb w13, [ x9], #1 // *dst++ = current_literal + cmp x11, x12 + b.lo L_trailing_literals_copy_less_than_16_literals + + // this block upadte dst & src pointers and remove frame +L_done: + str x9, [x0] + str x11, [x2] + + ldp x27, x28, [sp], #16 + ldp x25, x26, [sp], #16 + ldp x23, x24, [sp], #16 + ldp x21, x22, [sp], #16 + ldp x19, x20, [sp], #16 + + // clear frame + ldp fp, lr, [sp], #16 + ret lr + +L_revert_x9_and_done: + sub x9, x9, #1 + b L_done + +.p2align 2 +L_constant: +.long LZ4_COMPRESS_HASH_MULTIPLY +.long 0x80808081 + +#endif + diff --git a/osfmk/arm64/machine_cpuid.h b/osfmk/arm64/machine_cpuid.h new file mode 100644 index 000000000..63eff5f7b --- /dev/null +++ b/osfmk/arm64/machine_cpuid.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM64_MACHINE_CPUID_H_ +#define _ARM64_MACHINE_CPUID_H_ + +typedef struct { +uint64_t el0_not_implemented : 1, + el0_aarch64_only : 1, + el0_aarch32_and_64 : 1, + el1_not_implemented : 1, + el1_aarch64_only : 1, + el1_aarch32_and_64 : 1, + el2_not_implemented : 1, + el2_aarch64_only : 1, + el2_aarch32_and_64 : 1, + el3_not_implemented : 1, + el3_aarch64_only : 1, + el3_aarch32_and_64 : 1, + reserved : 52; +} arm_feature_bits_t; + +/* Debug identification */ + +/* ID_AA64DFR0_EL1 */ +typedef union { + struct { + uint64_t debug_arch_version : 4, + trace_extn_version : 4, + perf_extn_version : 4, + brps : 4, + reserved0 : 4, + wrps : 4, + reserved1 : 4, + ctx_cmps : 4, + reserved32 : 32; + } debug_feature; + uint64_t value; +} arm_cpuid_id_aa64dfr0_el1; + +typedef struct { + uint32_t num_watchpoint_pairs; + uint32_t num_breakpoint_pairs; +} arm_debug_info_t; + +#endif /* _MACHINE_CPUID_H_ */ diff --git a/osfmk/arm64/machine_kpc.h b/osfmk/arm64/machine_kpc.h new file mode 100644 index 000000000..1ba778dc7 --- /dev/null +++ b/osfmk/arm64/machine_kpc.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _MACHINE_ARM64_KPC_H +#define _MACHINE_ARM64_KPC_H + +#include + +typedef uint64_t kpc_config_t; + +#define KPC_ARM64_FIXED_COUNT (2) +#if NO_MONITOR +/* Addition of 2 counters to the SoC happens to coincide with removal of + * EL3 monitor. If this changes again in the future, consider moving + * counter config to per-SoC headers. */ +#define KPC_ARM64_CONFIGURABLE_COUNT (8) +#else +#define KPC_ARM64_CONFIGURABLE_COUNT (6) +#endif + +#define KPC_ARM64_COUNTER_WIDTH (47) +#define KPC_ARM64_COUNTER_MASK ((UINT64_C(1) << KPC_ARM64_COUNTER_WIDTH) - 1) +#define KPC_ARM64_COUNTER_OVF_BIT (47) +#define KPC_ARM64_COUNTER_OVF_MASK (UINT64_C(1) << KPC_ARM64_COUNTER_OVF_BIT) + +/* arm64 uses fixed counter shadows */ +#define FIXED_COUNTER_SHADOW (1) + +#define KPC_ARM64_PMC_COUNT (KPC_ARM64_FIXED_COUNT + KPC_ARM64_CONFIGURABLE_COUNT) + +/* Size to the maximum number of counters we could read from every class in one go */ +#define KPC_MAX_COUNTERS (KPC_ARM64_FIXED_COUNT + KPC_ARM64_CONFIGURABLE_COUNT + 1) + +#endif /* _MACHINE_ARM64_KPC_H */ diff --git a/osfmk/arm64/machine_machdep.h b/osfmk/arm64/machine_machdep.h new file mode 100644 index 000000000..cc8470ba3 --- /dev/null +++ b/osfmk/arm64/machine_machdep.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _MACHDEP_INTERNAL_H_ +#define _MACHDEP_INTERNAL_H_ + +/* We cache the current cthread pointer in the high bits of TPIDRRO_EL0 and + * the current CPU number in the low bits. The cthread pointer must be aligned + * sufficiently that the maximum CPU number will fit. + * + * NOTE: Keep this in sync with libsyscall/os/tsd.h, specifically _os_cpu_number() + */ + +#define MACHDEP_CTHREAD_ALIGNMENT (1 << 3) +#define MACHDEP_CPUNUM_MASK (MACHDEP_CTHREAD_ALIGNMENT - 1) +#define MACHDEP_CTHREAD_MASK (~MACHDEP_CPUNUM_MASK) + +#endif /* _MACHDEP_INTERNAL_H_ */ diff --git a/osfmk/arm64/machine_routines.c b/osfmk/arm64/machine_routines.c new file mode 100644 index 000000000..c4e6ba138 --- /dev/null +++ b/osfmk/arm64/machine_routines.c @@ -0,0 +1,2048 @@ +/* + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#if defined(KERNEL_INTEGRITY_KTRR) +#include +#endif + +#if KPC +#include +#endif + + +static int max_cpus_initialized = 0; +#define MAX_CPUS_SET 0x1 +#define MAX_CPUS_WAIT 0x2 + +uint32_t LockTimeOut; +uint32_t LockTimeOutUsec; +uint64_t MutexSpin; +boolean_t is_clock_configured = FALSE; + +extern int mach_assert; +extern volatile uint32_t debug_enabled; +SECURITY_READ_ONLY_LATE(unsigned int) debug_boot_arg; + + +void machine_conf(void); + +thread_t Idle_context(void); + +static uint32_t cpu_phys_ids[MAX_CPUS] = {[0 ... MAX_CPUS - 1] = (uint32_t)-1}; +static unsigned int avail_cpus = 0; +static int boot_cpu = -1; +static int max_cpu_number = 0; +cluster_type_t boot_cluster = CLUSTER_TYPE_SMP; + +lockdown_handler_t lockdown_handler; +void *lockdown_this; +lck_mtx_t lockdown_handler_lck; +lck_grp_t *lockdown_handler_grp; +int lockdown_done; + +void ml_lockdown_init(void); +void ml_lockdown_run_handler(void); +uint32_t get_arm_cpu_version(void); + + +void ml_cpu_signal(unsigned int cpu_id __unused) +{ + panic("Platform does not support ACC Fast IPI"); +} + +void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs) { + (void)nanosecs; + panic("Platform does not support ACC Fast IPI"); +} + +uint64_t ml_cpu_signal_deferred_get_timer() { + return 0; +} + +void ml_cpu_signal_deferred(unsigned int cpu_id __unused) +{ + panic("Platform does not support ACC Fast IPI deferral"); +} + +void ml_cpu_signal_retract(unsigned int cpu_id __unused) +{ + panic("Platform does not support ACC Fast IPI retraction"); +} + +void machine_idle(void) +{ + __asm__ volatile ("msr DAIFSet, %[mask]" ::[mask] "i" (DAIFSC_IRQF | DAIFSC_FIQF)); + Idle_context(); + __asm__ volatile ("msr DAIFClr, %[mask]" ::[mask] "i" (DAIFSC_IRQF | DAIFSC_FIQF)); +} + +void init_vfp(void) +{ + return; +} + +boolean_t get_vfp_enabled(void) +{ + return TRUE; +} + +void OSSynchronizeIO(void) +{ + __builtin_arm_dsb(DSB_SY); +} + +uint64_t get_aux_control(void) +{ + uint64_t value; + + MRS(value, "ACTLR_EL1"); + return value; +} + +uint64_t get_mmu_control(void) +{ + uint64_t value; + + MRS(value, "SCTLR_EL1"); + return value; +} + +uint64_t get_tcr(void) +{ + uint64_t value; + + MRS(value, "TCR_EL1"); + return value; +} + +boolean_t ml_get_interrupts_enabled(void) +{ + uint64_t value; + + MRS(value, "DAIF"); + if (value & DAIF_IRQF) + return FALSE; + return TRUE; +} + +pmap_paddr_t get_mmu_ttb(void) +{ + pmap_paddr_t value; + + MRS(value, "TTBR0_EL1"); + return value; +} + +MARK_AS_PMAP_TEXT +void set_mmu_ttb(pmap_paddr_t value) +{ + __builtin_arm_dsb(DSB_ISH); + MSR("TTBR0_EL1", value); + __builtin_arm_isb(ISB_SY); +} + +static uint32_t get_midr_el1(void) +{ + uint64_t value; + + MRS(value, "MIDR_EL1"); + + /* This is a 32-bit register. */ + return (uint32_t) value; +} + +uint32_t get_arm_cpu_version(void) +{ + uint32_t value = get_midr_el1(); + + /* Compose the register values into 8 bits; variant[7:4], revision[3:0]. */ + return ((value & MIDR_EL1_REV_MASK) >> MIDR_EL1_REV_SHIFT) | ((value & MIDR_EL1_VAR_MASK) >> (MIDR_EL1_VAR_SHIFT - 4)); +} + +/* + * user_cont_hwclock_allowed() + * + * Indicates whether we allow EL0 to read the physical timebase (CNTPCT_EL0) + * as a continuous time source (e.g. from mach_continuous_time) + */ +boolean_t user_cont_hwclock_allowed(void) +{ + return FALSE; +} + +/* + * user_timebase_allowed() + * + * Indicates whether we allow EL0 to read the physical timebase (CNTPCT_EL0). + */ +boolean_t user_timebase_allowed(void) +{ + return TRUE; +} + +boolean_t arm64_wfe_allowed(void) +{ + return TRUE; +} + +#if defined(KERNEL_INTEGRITY_KTRR) + +uint64_t rorgn_begin __attribute__((section("__DATA, __const"))) = 0; +uint64_t rorgn_end __attribute__((section("__DATA, __const"))) = 0; +vm_offset_t amcc_base; + +static void assert_unlocked(void); +static void assert_amcc_cache_disabled(void); +static void lock_amcc(void); +static void lock_mmu(uint64_t begin, uint64_t end); + +void rorgn_stash_range(void) +{ + +#if DEVELOPMENT || DEBUG + boolean_t rorgn_disable = FALSE; + + PE_parse_boot_argn("-unsafe_kernel_text", &rorgn_disable, sizeof(rorgn_disable)); + + if (rorgn_disable) { + /* take early out if boot arg present, don't query any machine registers to avoid + * dependency on amcc DT entry + */ + return; + } +#endif + + /* Get the AMC values, and stash them into rorgn_begin, rorgn_end. */ + +#if defined(KERNEL_INTEGRITY_KTRR) + uint64_t soc_base = 0; + DTEntry entryP = NULL; + uintptr_t *reg_prop = NULL; + uint32_t prop_size = 0; + int rc; + + soc_base = pe_arm_get_soc_base_phys(); + rc = DTFindEntry("name", "mcc", &entryP); + assert(rc == kSuccess); + rc = DTGetProperty(entryP, "reg", (void **)®_prop, &prop_size); + assert(rc == kSuccess); + amcc_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1)); +#else +#error "KERNEL_INTEGRITY config error" +#endif + +#if defined(KERNEL_INTEGRITY_KTRR) + assert(rRORGNENDADDR > rRORGNBASEADDR); + rorgn_begin = (rRORGNBASEADDR << ARM_PGSHIFT) + gPhysBase; + rorgn_end = (rRORGNENDADDR << ARM_PGSHIFT) + gPhysBase; +#else +#error KERNEL_INTEGRITY config error +#endif /* defined (KERNEL_INTEGRITY_KTRR) */ +} + +static void assert_unlocked() { + uint64_t ktrr_lock = 0; + uint32_t rorgn_lock = 0; + + assert(amcc_base); +#if defined(KERNEL_INTEGRITY_KTRR) + rorgn_lock = rRORGNLOCK; + ktrr_lock = __builtin_arm_rsr64(ARM64_REG_KTRR_LOCK_EL1); +#else +#error KERNEL_INTEGRITY config error +#endif /* defined(KERNEL_INTEGRITY_KTRR) */ + + assert(!ktrr_lock); + assert(!rorgn_lock); +} + +static void lock_amcc() { +#if defined(KERNEL_INTEGRITY_KTRR) + rRORGNLOCK = 1; + __builtin_arm_isb(ISB_SY); +#else +#error KERNEL_INTEGRITY config error +#endif +} + +static void lock_mmu(uint64_t begin, uint64_t end) { + +#if defined(KERNEL_INTEGRITY_KTRR) + + __builtin_arm_wsr64(ARM64_REG_KTRR_LOWER_EL1, begin); + __builtin_arm_wsr64(ARM64_REG_KTRR_UPPER_EL1, end); + __builtin_arm_wsr64(ARM64_REG_KTRR_LOCK_EL1, 1ULL); + + /* flush TLB */ + + __builtin_arm_isb(ISB_SY); + flush_mmu_tlb(); + +#else +#error KERNEL_INTEGRITY config error +#endif + +} + +static void assert_amcc_cache_disabled() { +#if defined(KERNEL_INTEGRITY_KTRR) + assert((rMCCGEN & 1) == 0); /* assert M$ disabled or LLC clean will be unreliable */ +#else +#error KERNEL_INTEGRITY config error +#endif +} + +/* + * void rorgn_lockdown(void) + * + * Lock the MMU and AMCC RORegion within lower and upper boundaries if not already locked + * + * [ ] - ensure this is being called ASAP on secondary CPUs: KTRR programming and lockdown handled in + * start.s:start_cpu() for subsequent wake/resume of all cores + */ +void rorgn_lockdown(void) +{ + vm_offset_t ktrr_begin, ktrr_end; + unsigned long plt_segsz, last_segsz; + +#if DEVELOPMENT || DEBUG + boolean_t ktrr_disable = FALSE; + + PE_parse_boot_argn("-unsafe_kernel_text", &ktrr_disable, sizeof(ktrr_disable)); + + if (ktrr_disable) { + /* + * take early out if boot arg present, since we may not have amcc DT entry present + * we can't assert that iboot hasn't programmed the RO region lockdown registers + */ + goto out; + } +#endif /* DEVELOPMENT || DEBUG */ + + assert_unlocked(); + + /* [x] - Use final method of determining all kernel text range or expect crashes */ + + ktrr_begin = (uint64_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_TEXT", &plt_segsz); + assert(ktrr_begin && gVirtBase && gPhysBase); + + ktrr_begin = kvtophys(ktrr_begin); + + /* __LAST is not part of the MMU KTRR region (it is however part of the AMCC KTRR region) */ + ktrr_end = (uint64_t) getsegdatafromheader(&_mh_execute_header, "__LAST", &last_segsz); + ktrr_end = (kvtophys(ktrr_end) - 1) & ~PAGE_MASK; + + /* ensure that iboot and xnu agree on the ktrr range */ + assert(rorgn_begin == ktrr_begin && rorgn_end == (ktrr_end + last_segsz)); + /* assert that __LAST segment containing privileged insns is only a single page */ + assert(last_segsz == PAGE_SIZE); + +#if DEBUG + printf("KTRR Begin: %p End: %p, setting lockdown\n", (void *)ktrr_begin, (void *)ktrr_end); +#endif + + /* [x] - ensure all in flight writes are flushed to AMCC before enabling RO Region Lock */ + + assert_amcc_cache_disabled(); + + CleanPoC_DcacheRegion_Force(phystokv(ktrr_begin), + (unsigned)((ktrr_end + last_segsz) - ktrr_begin + PAGE_MASK)); + + lock_amcc(); + + lock_mmu(ktrr_begin, ktrr_end); + +#if DEVELOPMENT || DEBUG +out: +#endif + + /* now we can run lockdown handler */ + ml_lockdown_run_handler(); +} + +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + +void +machine_startup(__unused boot_args * args) +{ + int boot_arg; + + +#if MACH_KDP + if (PE_parse_boot_argn("debug", &debug_boot_arg, sizeof (debug_boot_arg)) && + debug_enabled) { + if (debug_boot_arg & DB_HALT) + halt_in_debugger = 1; + if (debug_boot_arg & DB_NMI) + panicDebugging = TRUE; + } else { + debug_boot_arg = 0; + } + +#endif + + PE_parse_boot_argn("assert", &mach_assert, sizeof (mach_assert)); + + if (PE_parse_boot_argn("preempt", &boot_arg, sizeof (boot_arg))) { + default_preemption_rate = boot_arg; + } + if (PE_parse_boot_argn("bg_preempt", &boot_arg, sizeof (boot_arg))) { + default_bg_preemption_rate = boot_arg; + } + + machine_conf(); + + /* + * Kick off the kernel bootstrap. + */ + kernel_bootstrap(); + /* NOTREACHED */ +} + +void machine_lockdown_preflight(void) +{ +#if CONFIG_KERNEL_INTEGRITY + +#if defined(KERNEL_INTEGRITY_KTRR) + rorgn_stash_range(); +#endif + +#endif +} + +void machine_lockdown(void) +{ +#if CONFIG_KERNEL_INTEGRITY +#if KERNEL_INTEGRITY_WT + /* Watchtower + * + * Notify the monitor about the completion of early kernel bootstrap. + * From this point forward it will enforce the integrity of kernel text, + * rodata and page tables. + */ + +#ifdef MONITOR + monitor_call(MONITOR_LOCKDOWN, 0, 0, 0); +#endif +#endif /* KERNEL_INTEGRITY_WT */ + + +#if defined(KERNEL_INTEGRITY_KTRR) + /* KTRR + * + * Lock physical KTRR region. KTRR region is read-only. Memory outside + * the region is not executable at EL1. + */ + + rorgn_lockdown(); +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + + +#endif /* CONFIG_KERNEL_INTEGRITY */ +} + +char * +machine_boot_info( + __unused char *buf, + __unused vm_size_t size) +{ + return (PE_boot_args()); +} + +void +machine_conf(void) +{ + /* + * This is known to be inaccurate. mem_size should always be capped at 2 GB + */ + machine_info.memory_size = (uint32_t)mem_size; +} + +void +machine_init(void) +{ + debug_log_init(); + clock_config(); + is_clock_configured = TRUE; + if (debug_enabled) + pmap_map_globals(); +} + +void +slave_machine_init(__unused void *param) +{ + cpu_machine_init(); /* Initialize the processor */ + clock_init(); /* Init the clock */ +} + +/* + * Routine: machine_processor_shutdown + * Function: + */ +thread_t +machine_processor_shutdown( + __unused thread_t thread, + void (*doshutdown) (processor_t), + processor_t processor) +{ + return (Shutdown_context(doshutdown, processor)); +} + +/* + * Routine: ml_init_max_cpus + * Function: + */ +void +ml_init_max_cpus(unsigned int max_cpus) +{ + boolean_t current_state; + + current_state = ml_set_interrupts_enabled(FALSE); + if (max_cpus_initialized != MAX_CPUS_SET) { + machine_info.max_cpus = max_cpus; + machine_info.physical_cpu_max = max_cpus; + machine_info.logical_cpu_max = max_cpus; + if (max_cpus_initialized == MAX_CPUS_WAIT) + thread_wakeup((event_t) & max_cpus_initialized); + max_cpus_initialized = MAX_CPUS_SET; + } + (void) ml_set_interrupts_enabled(current_state); +} + +/* + * Routine: ml_get_max_cpus + * Function: + */ +unsigned int +ml_get_max_cpus(void) +{ + boolean_t current_state; + + current_state = ml_set_interrupts_enabled(FALSE); + if (max_cpus_initialized != MAX_CPUS_SET) { + max_cpus_initialized = MAX_CPUS_WAIT; + assert_wait((event_t) & max_cpus_initialized, THREAD_UNINT); + (void) thread_block(THREAD_CONTINUE_NULL); + } + (void) ml_set_interrupts_enabled(current_state); + return (machine_info.max_cpus); +} + +/* + * Routine: ml_init_lock_timeout + * Function: + */ +void +ml_init_lock_timeout(void) +{ + uint64_t abstime; + uint64_t mtxspin; + uint64_t default_timeout_ns = NSEC_PER_SEC>>2; + uint32_t slto; + + if (PE_parse_boot_argn("slto_us", &slto, sizeof (slto))) + default_timeout_ns = slto * NSEC_PER_USEC; + + nanoseconds_to_absolutetime(default_timeout_ns, &abstime); + LockTimeOutUsec = (uint32_t)(abstime / NSEC_PER_USEC); + LockTimeOut = (uint32_t)abstime; + + if (PE_parse_boot_argn("mtxspin", &mtxspin, sizeof (mtxspin))) { + if (mtxspin > USEC_PER_SEC>>4) + mtxspin = USEC_PER_SEC>>4; + nanoseconds_to_absolutetime(mtxspin*NSEC_PER_USEC, &abstime); + } else { + nanoseconds_to_absolutetime(10*NSEC_PER_USEC, &abstime); + } + MutexSpin = abstime; +} + +/* + * This is called from the machine-independent routine cpu_up() + * to perform machine-dependent info updates. + */ +void +ml_cpu_up(void) +{ + hw_atomic_add(&machine_info.physical_cpu, 1); + hw_atomic_add(&machine_info.logical_cpu, 1); +} + +/* + * This is called from the machine-independent routine cpu_down() + * to perform machine-dependent info updates. + */ +void +ml_cpu_down(void) +{ + cpu_data_t *cpu_data_ptr; + + hw_atomic_sub(&machine_info.physical_cpu, 1); + hw_atomic_sub(&machine_info.logical_cpu, 1); + + /* + * If we want to deal with outstanding IPIs, we need to + * do relatively early in the processor_doshutdown path, + * as we pend decrementer interrupts using the IPI + * mechanism if we cannot immediately service them (if + * IRQ is masked). Do so now. + * + * We aren't on the interrupt stack here; would it make + * more sense to disable signaling and then enable + * interrupts? It might be a bit cleaner. + */ + cpu_data_ptr = getCpuDatap(); + cpu_data_ptr->cpu_running = FALSE; + cpu_signal_handler_internal(TRUE); +} + +/* + * Routine: ml_cpu_get_info + * Function: + */ +void +ml_cpu_get_info(ml_cpu_info_t * ml_cpu_info) +{ + cache_info_t *cpuid_cache_info; + + cpuid_cache_info = cache_info(); + ml_cpu_info->vector_unit = 0; + ml_cpu_info->cache_line_size = cpuid_cache_info->c_linesz; + ml_cpu_info->l1_icache_size = cpuid_cache_info->c_isize; + ml_cpu_info->l1_dcache_size = cpuid_cache_info->c_dsize; + +#if (__ARM_ARCH__ >= 7) + ml_cpu_info->l2_settings = 1; + ml_cpu_info->l2_cache_size = cpuid_cache_info->c_l2size; +#else + ml_cpu_info->l2_settings = 0; + ml_cpu_info->l2_cache_size = 0xFFFFFFFF; +#endif + ml_cpu_info->l3_settings = 0; + ml_cpu_info->l3_cache_size = 0xFFFFFFFF; +} + +unsigned int +ml_get_machine_mem(void) +{ + return (machine_info.memory_size); +} + +__attribute__((noreturn)) +void +halt_all_cpus(boolean_t reboot) +{ + if (reboot) { + printf("MACH Reboot\n"); + PEHaltRestart(kPERestartCPU); + } else { + printf("CPU halted\n"); + PEHaltRestart(kPEHaltCPU); + } + while (1); +} + +__attribute__((noreturn)) +void +halt_cpu(void) +{ + halt_all_cpus(FALSE); +} + +/* + * Routine: machine_signal_idle + * Function: + */ +void +machine_signal_idle( + processor_t processor) +{ + cpu_signal(processor_to_cpu_datap(processor), SIGPnop, (void *)NULL, (void *)NULL); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0); +} + +void +machine_signal_idle_deferred( + processor_t processor) +{ + cpu_signal_deferred(processor_to_cpu_datap(processor)); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_DEFERRED_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0); +} + +void +machine_signal_idle_cancel( + processor_t processor) +{ + cpu_signal_cancel(processor_to_cpu_datap(processor)); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_CANCEL_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0); +} + +/* + * Routine: ml_install_interrupt_handler + * Function: Initialize Interrupt Handler + */ +void +ml_install_interrupt_handler( + void *nub, + int source, + void *target, + IOInterruptHandler handler, + void *refCon) +{ + cpu_data_t *cpu_data_ptr; + boolean_t current_state; + + current_state = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + cpu_data_ptr->interrupt_nub = nub; + cpu_data_ptr->interrupt_source = source; + cpu_data_ptr->interrupt_target = target; + cpu_data_ptr->interrupt_handler = handler; + cpu_data_ptr->interrupt_refCon = refCon; + + cpu_data_ptr->interrupts_enabled = TRUE; + (void) ml_set_interrupts_enabled(current_state); + + initialize_screen(NULL, kPEAcquireScreen); +} + +/* + * Routine: ml_init_interrupt + * Function: Initialize Interrupts + */ +void +ml_init_interrupt(void) +{ +} + +/* + * Routine: ml_init_timebase + * Function: register and setup Timebase, Decremeter services + */ +void ml_init_timebase( + void *args, + tbd_ops_t tbd_funcs, + vm_offset_t int_address, + vm_offset_t int_value __unused) +{ + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = (cpu_data_t *)args; + + if ((cpu_data_ptr == &BootCpuData) + && (rtclock_timebase_func.tbd_fiq_handler == (void *)NULL)) { + rtclock_timebase_func = *tbd_funcs; + rtclock_timebase_addr = int_address; + } +} + +void +ml_parse_cpu_topology(void) +{ + DTEntry entry, child __unused; + OpaqueDTEntryIterator iter; + uint32_t cpu_boot_arg; + int err; + + cpu_boot_arg = MAX_CPUS; + + PE_parse_boot_argn("cpus", &cpu_boot_arg, sizeof(cpu_boot_arg)); + + err = DTLookupEntry(NULL, "/cpus", &entry); + assert(err == kSuccess); + + err = DTInitEntryIterator(entry, &iter); + assert(err == kSuccess); + + while (kSuccess == DTIterateEntries(&iter, &child)) { + unsigned int propSize; + void *prop = NULL; + int cpu_id = avail_cpus++; + + if (kSuccess == DTGetProperty(child, "cpu-id", &prop, &propSize)) + cpu_id = *((int32_t*)prop); + + assert(cpu_id < MAX_CPUS); + assert(cpu_phys_ids[cpu_id] == (uint32_t)-1); + + if (boot_cpu == -1) { + if (kSuccess != DTGetProperty(child, "state", &prop, &propSize)) + panic("unable to retrieve state for cpu %d", cpu_id); + + if (strncmp((char*)prop, "running", propSize) == 0) { + boot_cpu = cpu_id; + } + } + if (kSuccess != DTGetProperty(child, "reg", &prop, &propSize)) + panic("unable to retrieve physical ID for cpu %d", cpu_id); + + cpu_phys_ids[cpu_id] = *((uint32_t*)prop); + + if ((cpu_id > max_cpu_number) && ((cpu_id == boot_cpu) || (avail_cpus <= cpu_boot_arg))) + max_cpu_number = cpu_id; + } + + if (avail_cpus > cpu_boot_arg) + avail_cpus = cpu_boot_arg; + + if (avail_cpus == 0) + panic("No cpus found!"); + + if (boot_cpu == -1) + panic("unable to determine boot cpu!"); +} + +unsigned int +ml_get_cpu_count(void) +{ + return avail_cpus; +} + +int +ml_get_boot_cpu_number(void) +{ + return boot_cpu; +} + +cluster_type_t +ml_get_boot_cluster(void) +{ + return boot_cluster; +} + +int +ml_get_cpu_number(uint32_t phys_id) +{ + for (int log_id = 0; log_id <= ml_get_max_cpu_number(); ++log_id) { + if (cpu_phys_ids[log_id] == phys_id) + return log_id; + } + return -1; +} + +int +ml_get_max_cpu_number(void) +{ + return max_cpu_number; +} + + +void ml_lockdown_init() { + lockdown_handler_grp = lck_grp_alloc_init("lockdown_handler", NULL); + assert(lockdown_handler_grp != NULL); + + lck_mtx_init(&lockdown_handler_lck, lockdown_handler_grp, NULL); +} + +kern_return_t +ml_lockdown_handler_register(lockdown_handler_t f, void *this) +{ + if (lockdown_handler || !f) { + return KERN_FAILURE; + } + + lck_mtx_lock(&lockdown_handler_lck); + lockdown_handler = f; + lockdown_this = this; + +#if !(defined(KERNEL_INTEGRITY_KTRR)) + lockdown_done=1; + lockdown_handler(this); +#else + if (lockdown_done) { + lockdown_handler(this); + } +#endif + lck_mtx_unlock(&lockdown_handler_lck); + + return KERN_SUCCESS; +} + +void ml_lockdown_run_handler() { + lck_mtx_lock(&lockdown_handler_lck); + assert(!lockdown_done); + + lockdown_done = 1; + if (lockdown_handler) { + lockdown_handler(lockdown_this); + } + lck_mtx_unlock(&lockdown_handler_lck); +} + +kern_return_t +ml_processor_register( + ml_processor_info_t * in_processor_info, + processor_t * processor_out, + ipi_handler_t * ipi_handler) +{ + cpu_data_t *this_cpu_datap; + processor_set_t pset; + boolean_t is_boot_cpu; + static unsigned int reg_cpu_count = 0; + + if (in_processor_info->log_id > (uint32_t)ml_get_max_cpu_number()) + return KERN_FAILURE; + + if ((unsigned int)OSIncrementAtomic((SInt32*)®_cpu_count) >= avail_cpus) + return KERN_FAILURE; + + if (in_processor_info->log_id != (uint32_t)ml_get_boot_cpu_number()) { + is_boot_cpu = FALSE; + this_cpu_datap = cpu_data_alloc(FALSE); + cpu_data_init(this_cpu_datap); + } else { + this_cpu_datap = &BootCpuData; + is_boot_cpu = TRUE; + } + + assert(in_processor_info->log_id < MAX_CPUS); + + this_cpu_datap->cpu_id = in_processor_info->cpu_id; + + this_cpu_datap->cpu_chud = chudxnu_cpu_alloc(is_boot_cpu); + if (this_cpu_datap->cpu_chud == (void *)NULL) + goto processor_register_error; + this_cpu_datap->cpu_console_buf = console_cpu_alloc(is_boot_cpu); + if (this_cpu_datap->cpu_console_buf == (void *)(NULL)) + goto processor_register_error; + + if (!is_boot_cpu) { + this_cpu_datap->cpu_number = in_processor_info->log_id; + + if (cpu_data_register(this_cpu_datap) != KERN_SUCCESS) + goto processor_register_error; + } + + this_cpu_datap->cpu_idle_notify = (void *) in_processor_info->processor_idle; + this_cpu_datap->cpu_cache_dispatch = in_processor_info->platform_cache_dispatch; + nanoseconds_to_absolutetime((uint64_t) in_processor_info->powergate_latency, &this_cpu_datap->cpu_idle_latency); + this_cpu_datap->cpu_reset_assist = kvtophys(in_processor_info->powergate_stub_addr); + + this_cpu_datap->idle_timer_notify = (void *) in_processor_info->idle_timer; + this_cpu_datap->idle_timer_refcon = in_processor_info->idle_timer_refcon; + + this_cpu_datap->platform_error_handler = (void *) in_processor_info->platform_error_handler; + this_cpu_datap->cpu_regmap_paddr = in_processor_info->regmap_paddr; + this_cpu_datap->cpu_phys_id = in_processor_info->phys_id; + this_cpu_datap->cpu_l2_access_penalty = in_processor_info->l2_access_penalty; + + this_cpu_datap->cpu_cluster_type = in_processor_info->cluster_type; + this_cpu_datap->cpu_cluster_id = in_processor_info->cluster_id; + this_cpu_datap->cpu_l2_id = in_processor_info->l2_cache_id; + this_cpu_datap->cpu_l2_size = in_processor_info->l2_cache_size; + this_cpu_datap->cpu_l3_id = in_processor_info->l3_cache_id; + this_cpu_datap->cpu_l3_size = in_processor_info->l3_cache_size; + + this_cpu_datap->cluster_master = is_boot_cpu; + + pset = pset_find(in_processor_info->cluster_id, processor_pset(master_processor)); + assert(pset != NULL); + kprintf("%s>cpu_id %p cluster_id %d cpu_number %d is type %d\n", __FUNCTION__, in_processor_info->cpu_id, in_processor_info->cluster_id, this_cpu_datap->cpu_number, in_processor_info->cluster_type); + + if (!is_boot_cpu) { + processor_init((struct processor *)this_cpu_datap->cpu_processor, + this_cpu_datap->cpu_number, pset); + + if (this_cpu_datap->cpu_l2_access_penalty) { + /* + * Cores that have a non-zero L2 access penalty compared + * to the boot processor should be de-prioritized by the + * scheduler, so that threads use the cores with better L2 + * preferentially. + */ + processor_set_primary(this_cpu_datap->cpu_processor, + master_processor); + } + } + + *processor_out = this_cpu_datap->cpu_processor; + *ipi_handler = cpu_signal_handler; + if (in_processor_info->idle_tickle != (idle_tickle_t *) NULL) + *in_processor_info->idle_tickle = (idle_tickle_t) cpu_idle_tickle; + +#if KPC + if (kpc_register_cpu(this_cpu_datap) != TRUE) + goto processor_register_error; +#endif + + if (!is_boot_cpu) { + prng_cpu_init(this_cpu_datap->cpu_number); + // now let next CPU register itself + OSIncrementAtomic((SInt32*)&real_ncpus); + } + + return KERN_SUCCESS; + +processor_register_error: +#if KPC + kpc_unregister_cpu(this_cpu_datap); +#endif + if (this_cpu_datap->cpu_chud != (void *)NULL) + chudxnu_cpu_free(this_cpu_datap->cpu_chud); + if (!is_boot_cpu) + cpu_data_free(this_cpu_datap); + + return KERN_FAILURE; +} + +void +ml_init_arm_debug_interface( + void * in_cpu_datap, + vm_offset_t virt_address) +{ + ((cpu_data_t *)in_cpu_datap)->cpu_debug_interface_map = virt_address; + do_debugid(); +} + +/* + * Routine: init_ast_check + * Function: + */ +void +init_ast_check( + __unused processor_t processor) +{ +} + +/* + * Routine: cause_ast_check + * Function: + */ +void +cause_ast_check( + processor_t processor) +{ + if (current_processor() != processor) { + cpu_signal(processor_to_cpu_datap(processor), SIGPast, (void *)NULL, (void *)NULL); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_AST), processor->cpu_id, 1 /* ast */, 0, 0, 0); + } +} + + +/* + * Routine: ml_at_interrupt_context + * Function: Check if running at interrupt context + */ +boolean_t +ml_at_interrupt_context(void) +{ + unsigned int local; + vm_offset_t intstack_top_ptr; + + intstack_top_ptr = getCpuDatap()->intstack_top; + return (((vm_offset_t)(&local) < intstack_top_ptr) && ((vm_offset_t)(&local) > (intstack_top_ptr - INTSTACK_SIZE))); +} +extern uint32_t cpu_idle_count; + +void ml_get_power_state(boolean_t *icp, boolean_t *pidlep) { + *icp = ml_at_interrupt_context(); + *pidlep = (cpu_idle_count == real_ncpus); +} + +/* + * Routine: ml_cause_interrupt + * Function: Generate a fake interrupt + */ +void +ml_cause_interrupt(void) +{ + return; /* BS_XXX */ +} + +/* Map memory map IO space */ +vm_offset_t +ml_io_map( + vm_offset_t phys_addr, + vm_size_t size) +{ + return (io_map(phys_addr, size, VM_WIMG_IO)); +} + +vm_offset_t +ml_io_map_wcomb( + vm_offset_t phys_addr, + vm_size_t size) +{ + return (io_map(phys_addr, size, VM_WIMG_WCOMB)); +} + +/* boot memory allocation */ +vm_offset_t +ml_static_malloc( + __unused vm_size_t size) +{ + return ((vm_offset_t) NULL); +} + +vm_map_address_t +ml_map_high_window( + vm_offset_t phys_addr, + vm_size_t len) +{ + return pmap_map_high_window_bd(phys_addr, len, VM_PROT_READ | VM_PROT_WRITE); +} + +vm_offset_t +ml_static_ptovirt( + vm_offset_t paddr) +{ + return phystokv(paddr); +} + +vm_offset_t +ml_static_vtop( + vm_offset_t vaddr) +{ + if (((vm_address_t)(vaddr) - gVirtBase) >= gPhysSize) + panic("ml_static_ptovirt(): illegal vaddr: %p\n", (void*)vaddr); + return ((vm_address_t)(vaddr) - gVirtBase + gPhysBase); +} + +kern_return_t +ml_static_protect( + vm_offset_t vaddr, /* kernel virtual address */ + vm_size_t size, + vm_prot_t new_prot) +{ + pt_entry_t arm_prot = 0; + pt_entry_t arm_block_prot = 0; + vm_offset_t vaddr_cur; + ppnum_t ppn; + kern_return_t result = KERN_SUCCESS; + + if (vaddr < VM_MIN_KERNEL_ADDRESS) { + panic("ml_static_protect(): %p < %p", (void *) vaddr, (void *) VM_MIN_KERNEL_ADDRESS); + return KERN_FAILURE; + } + + assert((vaddr & (PAGE_SIZE - 1)) == 0); /* must be page aligned */ + + if ((new_prot & VM_PROT_WRITE) && (new_prot & VM_PROT_EXECUTE)) { + panic("ml_static_protect(): WX request on %p", (void *) vaddr); + } + + /* Set up the protection bits, and block bits so we can validate block mappings. */ + if (new_prot & VM_PROT_WRITE) { + arm_prot |= ARM_PTE_AP(AP_RWNA); + arm_block_prot |= ARM_TTE_BLOCK_AP(AP_RWNA); + } else { + arm_prot |= ARM_PTE_AP(AP_RONA); + arm_block_prot |= ARM_TTE_BLOCK_AP(AP_RONA); + } + + arm_prot |= ARM_PTE_NX; + arm_block_prot |= ARM_TTE_BLOCK_NX; + + if (!(new_prot & VM_PROT_EXECUTE)) { + arm_prot |= ARM_PTE_PNX; + arm_block_prot |= ARM_TTE_BLOCK_PNX; + } + + for (vaddr_cur = vaddr; + vaddr_cur < trunc_page_64(vaddr + size); + vaddr_cur += PAGE_SIZE) { + ppn = pmap_find_phys(kernel_pmap, vaddr_cur); + if (ppn != (vm_offset_t) NULL) { +#if __ARM64_TWO_LEVEL_PMAP__ + tt_entry_t *tte2; +#else + tt_entry_t *tte1, *tte2; +#endif + pt_entry_t *pte_p; + pt_entry_t ptmp; + + +#if __ARM64_TWO_LEVEL_PMAP__ + tte2 = &kernel_pmap->tte[(((vaddr_cur) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; +#else + tte1 = &kernel_pmap->tte[(((vaddr_cur) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)]; + tte2 = &((tt_entry_t*) phystokv((*tte1) & ARM_TTE_TABLE_MASK))[(((vaddr_cur) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; +#endif + + if (((*tte2) & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE) { + if ((((*tte2) & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) && + ((*tte2 & (ARM_TTE_BLOCK_NXMASK | ARM_TTE_BLOCK_PNXMASK | ARM_TTE_BLOCK_APMASK)) == arm_block_prot)) { + /* + * We can support ml_static_protect on a block mapping if the mapping already has + * the desired protections. We still want to run checks on a per-page basis. + */ + continue; + } + + result = KERN_FAILURE; + break; + } + + pte_p = (pt_entry_t *)&((tt_entry_t*)(phystokv((*tte2) & ARM_TTE_TABLE_MASK)))[(((vaddr_cur) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)]; + ptmp = *pte_p; + + if ((ptmp & ARM_PTE_HINT_MASK) && ((ptmp & (ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)) != arm_prot)) { + /* + * The contiguous hint is similar to a block mapping for ml_static_protect; if the existing + * protections do not match the desired protections, then we will fail (as we cannot update + * this mapping without updating other mappings as well). + */ + result = KERN_FAILURE; + break; + } + + __unreachable_ok_push + if (TEST_PAGE_RATIO_4) { + { + unsigned int i; + pt_entry_t *ptep_iter; + + ptep_iter = pte_p; + for (i=0; i<4; i++, ptep_iter++) { + /* Note that there is a hole in the HINT sanity checking here. */ + ptmp = *ptep_iter; + + /* We only need to update the page tables if the protections do not match. */ + if ((ptmp & (ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)) != arm_prot) { + ptmp = (ptmp & ~(ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)) | arm_prot; + *ptep_iter = ptmp; + } + } + } +#ifndef __ARM_L1_PTW__ + FlushPoC_DcacheRegion( trunc_page_32(pte_p), 4*sizeof(*pte_p)); +#endif + } else { + ptmp = *pte_p; + + /* We only need to update the page tables if the protections do not match. */ + if ((ptmp & (ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)) != arm_prot) { + ptmp = (ptmp & ~(ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)) | arm_prot; + *pte_p = ptmp; + } + +#ifndef __ARM_L1_PTW__ + FlushPoC_DcacheRegion( trunc_page_32(pte_p), sizeof(*pte_p)); +#endif + } + __unreachable_ok_pop + } + } + + if (vaddr_cur > vaddr) { + assert(((vaddr_cur - vaddr) & 0xFFFFFFFF00000000ULL) == 0); + flush_mmu_tlb_region(vaddr, (uint32_t)(vaddr_cur - vaddr)); + } + + + return result; +} + +/* + * Routine: ml_static_mfree + * Function: + */ +void +ml_static_mfree( + vm_offset_t vaddr, + vm_size_t size) +{ + vm_offset_t vaddr_cur; + ppnum_t ppn; + uint32_t freed_pages = 0; + + /* It is acceptable (if bad) to fail to free. */ + if (vaddr < VM_MIN_KERNEL_ADDRESS) + return; + + assert((vaddr & (PAGE_SIZE - 1)) == 0); /* must be page aligned */ + + for (vaddr_cur = vaddr; + vaddr_cur < trunc_page_64(vaddr + size); + vaddr_cur += PAGE_SIZE) { + + ppn = pmap_find_phys(kernel_pmap, vaddr_cur); + if (ppn != (vm_offset_t) NULL) { + /* + * It is not acceptable to fail to update the protections on a page + * we will release to the VM. We need to either panic or continue. + * For now, we'll panic (to help flag if there is memory we can + * reclaim). + */ + if (ml_static_protect(vaddr_cur, PAGE_SIZE, VM_PROT_WRITE | VM_PROT_READ) != KERN_SUCCESS) { + panic("Failed ml_static_mfree on %p", (void *) vaddr_cur); + } + +#if 0 + /* + * Must NOT tear down the "V==P" mapping for vaddr_cur as the zone alias scheme + * relies on the persistence of these mappings for all time. + */ + // pmap_remove(kernel_pmap, (addr64_t) vaddr_cur, (addr64_t) (vaddr_cur + PAGE_SIZE)); +#endif + + vm_page_create(ppn, (ppn + 1)); + freed_pages++; + } + } + vm_page_lockspin_queues(); + vm_page_wire_count -= freed_pages; + vm_page_wire_count_initial -= freed_pages; + vm_page_unlock_queues(); +#if DEBUG + kprintf("ml_static_mfree: Released 0x%x pages at VA %p, size:0x%llx, last ppn: 0x%x\n", freed_pages, (void *)vaddr, (uint64_t)size, ppn); +#endif +} + + +/* virtual to physical on wired pages */ +vm_offset_t +ml_vtophys(vm_offset_t vaddr) +{ + return kvtophys(vaddr); +} + +/* + * Routine: ml_nofault_copy + * Function: Perform a physical mode copy if the source and destination have + * valid translations in the kernel pmap. If translations are present, they are + * assumed to be wired; e.g., no attempt is made to guarantee that the + * translations obtained remain valid for the duration of the copy process. + */ +vm_size_t +ml_nofault_copy(vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size) +{ + addr64_t cur_phys_dst, cur_phys_src; + vm_size_t count, nbytes = 0; + + while (size > 0) { + if (!(cur_phys_src = kvtophys(virtsrc))) + break; + if (!(cur_phys_dst = kvtophys(virtdst))) + break; + if (!pmap_valid_address(trunc_page_64(cur_phys_dst)) || + !pmap_valid_address(trunc_page_64(cur_phys_src))) + break; + count = PAGE_SIZE - (cur_phys_src & PAGE_MASK); + if (count > (PAGE_SIZE - (cur_phys_dst & PAGE_MASK))) + count = PAGE_SIZE - (cur_phys_dst & PAGE_MASK); + if (count > size) + count = size; + + bcopy_phys(cur_phys_src, cur_phys_dst, count); + + nbytes += count; + virtsrc += count; + virtdst += count; + size -= count; + } + + return nbytes; +} + +/* + * Routine: ml_validate_nofault + * Function: Validate that ths address range has a valid translations + * in the kernel pmap. If translations are present, they are + * assumed to be wired; i.e. no attempt is made to guarantee + * that the translation persist after the check. + * Returns: TRUE if the range is mapped and will not cause a fault, + * FALSE otherwise. + */ + +boolean_t ml_validate_nofault( + vm_offset_t virtsrc, vm_size_t size) +{ + addr64_t cur_phys_src; + uint32_t count; + + while (size > 0) { + if (!(cur_phys_src = kvtophys(virtsrc))) + return FALSE; + if (!pmap_valid_address(trunc_page_64(cur_phys_src))) + return FALSE; + count = (uint32_t)(PAGE_SIZE - (cur_phys_src & PAGE_MASK)); + if (count > size) + count = (uint32_t)size; + + virtsrc += count; + size -= count; + } + + return TRUE; +} + +void +ml_get_bouncepool_info(vm_offset_t * phys_addr, vm_size_t * size) +{ + *phys_addr = 0; + *size = 0; +} + +void +active_rt_threads(__unused boolean_t active) +{ +} + +static void cpu_qos_cb_default(__unused int urgency, __unused uint64_t qos_param1, __unused uint64_t qos_param2) { + return; +} + +cpu_qos_update_t cpu_qos_update = cpu_qos_cb_default; + +void cpu_qos_update_register(cpu_qos_update_t cpu_qos_cb) { + if (cpu_qos_cb != NULL) { + cpu_qos_update = cpu_qos_cb; + } else { + cpu_qos_update = cpu_qos_cb_default; + } +} + +void +thread_tell_urgency(int urgency, uint64_t rt_period, uint64_t rt_deadline, uint64_t sched_latency __unused, __unused thread_t nthread) +{ + SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_START, urgency, rt_period, rt_deadline, sched_latency, 0); + + cpu_qos_update(urgency, rt_period, rt_deadline); + + SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_END, urgency, rt_period, rt_deadline, 0, 0); +} + +void +machine_run_count(__unused uint32_t count) +{ +} + +processor_t +machine_choose_processor(__unused processor_set_t pset, processor_t processor) +{ + return (processor); +} + +vm_offset_t +ml_stack_remaining(void) +{ + uintptr_t local = (uintptr_t) &local; + + if (ml_at_interrupt_context()) { + return (local - (getCpuDatap()->intstack_top - INTSTACK_SIZE)); + } else { + return (local - current_thread()->kernel_stack); + } +} + +#if KASAN +vm_offset_t ml_stack_base(void); +vm_size_t ml_stack_size(void); + +vm_offset_t +ml_stack_base(void) +{ + if (ml_at_interrupt_context()) { + return getCpuDatap()->intstack_top - INTSTACK_SIZE; + } else { + return current_thread()->kernel_stack; + } +} +vm_size_t +ml_stack_size(void) +{ + if (ml_at_interrupt_context()) { + return INTSTACK_SIZE; + } else { + return kernel_stack_size; + } +} +#endif + +boolean_t machine_timeout_suspended(void) { + return FALSE; +} + +kern_return_t +ml_interrupt_prewarm(__unused uint64_t deadline) +{ + return KERN_FAILURE; +} + +/* + * Assumes fiq, irq disabled. + */ +void +ml_set_decrementer(uint32_t dec_value) +{ + cpu_data_t *cdp = getCpuDatap(); + + assert(ml_get_interrupts_enabled() == FALSE); + cdp->cpu_decrementer = dec_value; + + if (cdp->cpu_set_decrementer_func) { + ((void (*)(uint32_t))cdp->cpu_set_decrementer_func)(dec_value); + } else { + __asm__ volatile("msr CNTP_TVAL_EL0, %0" : : "r"((uint64_t)dec_value)); + } +} + +uint64_t ml_get_hwclock() +{ + uint64_t timebase; + + // ISB required by ARMV7C.b section B8.1.2 & ARMv8 section D6.1.2 + // "Reads of CNTPCT[_EL0] can occur speculatively and out of order relative + // to other instructions executed on the same processor." + __asm__ volatile("isb\n" + "mrs %0, CNTPCT_EL0" + : "=r"(timebase)); + + return timebase; +} + +uint64_t +ml_get_timebase() +{ + return (ml_get_hwclock() + getCpuDatap()->cpu_base_timebase); +} + +uint32_t +ml_get_decrementer() +{ + cpu_data_t *cdp = getCpuDatap(); + uint32_t dec; + + assert(ml_get_interrupts_enabled() == FALSE); + + if (cdp->cpu_get_decrementer_func) { + dec = ((uint32_t (*)(void))cdp->cpu_get_decrementer_func)(); + } else { + uint64_t wide_val; + + __asm__ volatile("mrs %0, CNTP_TVAL_EL0" : "=r"(wide_val)); + dec = (uint32_t)wide_val; + assert(wide_val == (uint64_t)dec); + } + + return dec; +} + +boolean_t +ml_get_timer_pending() +{ + uint64_t cntp_ctl; + + __asm__ volatile("mrs %0, CNTP_CTL_EL0" : "=r"(cntp_ctl)); + return ((cntp_ctl & CNTP_CTL_EL0_ISTATUS) != 0) ? TRUE : FALSE; +} + +boolean_t +ml_wants_panic_trap_to_debugger(void) +{ + boolean_t result = FALSE; + return result; +} + +static void +cache_trap_error(thread_t thread, vm_map_address_t fault_addr) +{ + mach_exception_data_type_t exc_data[2]; + arm_saved_state_t *regs = get_user_regs(thread); + + set_saved_state_far(regs, fault_addr); + + exc_data[0] = KERN_INVALID_ADDRESS; + exc_data[1] = fault_addr; + + exception_triage(EXC_BAD_ACCESS, exc_data, 2); +} + +static void +cache_trap_recover() +{ + vm_map_address_t fault_addr; + + __asm__ volatile("mrs %0, FAR_EL1" : "=r"(fault_addr)); + + cache_trap_error(current_thread(), fault_addr); +} + +static void +dcache_flush_trap(vm_map_address_t start, vm_map_size_t size) +{ + vm_map_address_t end = start + size; + thread_t thread = current_thread(); + vm_offset_t old_recover = thread->recover; + + /* Check bounds */ + if (task_has_64BitAddr(current_task())) { + if (end > MACH_VM_MAX_ADDRESS) { + cache_trap_error(thread, end & ((1 << ARM64_CLINE_SHIFT) - 1)); + } + } else { + if (end > VM_MAX_ADDRESS) { + cache_trap_error(thread, end & ((1 << ARM64_CLINE_SHIFT) - 1)); + } + } + + if (start > end) { + cache_trap_error(thread, start & ((1 << ARM64_CLINE_SHIFT) - 1)); + } + + /* Set recovery function */ + thread->recover = (vm_address_t)cache_trap_recover; + +#if defined(APPLE_ARM64_ARCH_FAMILY) + /* + * We're coherent on Apple ARM64 CPUs, so this could be a nop. However, + * if the region given us is bad, it would be good to catch it and + * crash, ergo we still do the flush. + */ + assert((size & 0xFFFFFFFF00000000ULL) == 0); + FlushPoC_DcacheRegion(start, (uint32_t)size); +#else +#error "Make sure you don't need to xcall." +#endif + + /* Restore recovery function */ + thread->recover = old_recover; + + /* Return (caller does exception return) */ +} + +static void +icache_invalidate_trap(vm_map_address_t start, vm_map_size_t size) +{ + vm_map_address_t end = start + size; + thread_t thread = current_thread(); + vm_offset_t old_recover = thread->recover; + + /* Check bounds */ + if (task_has_64BitAddr(current_task())) { + if (end > MACH_VM_MAX_ADDRESS) { + cache_trap_error(thread, end & ((1 << ARM64_CLINE_SHIFT) - 1)); + } + } else { + if (end > VM_MAX_ADDRESS) { + cache_trap_error(thread, end & ((1 << ARM64_CLINE_SHIFT) - 1)); + } + } + + if (start > end) { + cache_trap_error(thread, start & ((1 << ARM64_CLINE_SHIFT) - 1)); + } + + /* Set recovery function */ + thread->recover = (vm_address_t)cache_trap_recover; + +#if defined(APPLE_ARM64_ARCH_FAMILY) + /* Clean dcache to unification, except we're coherent on Apple ARM64 CPUs */ +#else +#error Make sure not cleaning is right for this platform! +#endif + + /* Invalidate iCache to point of unification */ + assert((size & 0xFFFFFFFF00000000ULL) == 0); + InvalidatePoU_IcacheRegion(start, (uint32_t)size); + + /* Restore recovery function */ + thread->recover = old_recover; + + /* Return (caller does exception return) */ +} + +__attribute__((noreturn)) +void +platform_syscall(arm_saved_state_t *state) +{ + uint32_t code; + +#define platform_syscall_kprintf(x...) /* kprintf("platform_syscall: " x) */ + + code = (uint32_t)get_saved_state_reg(state, 3); + switch (code) { + case 0: + /* I-Cache flush */ + platform_syscall_kprintf("icache flush requested.\n"); + icache_invalidate_trap(get_saved_state_reg(state, 0), get_saved_state_reg(state, 1)); + break; + case 1: + /* D-Cache flush */ + platform_syscall_kprintf("dcache flush requested.\n"); + dcache_flush_trap(get_saved_state_reg(state, 0), get_saved_state_reg(state, 1)); + break; + case 2: + /* set cthread */ + platform_syscall_kprintf("set cthread self.\n"); + thread_set_cthread_self(get_saved_state_reg(state, 0)); + break; + case 3: + /* get cthread */ + platform_syscall_kprintf("get cthread self.\n"); + set_saved_state_reg(state, 0, thread_get_cthread_self()); + break; + default: + platform_syscall_kprintf("unknown: %d\n", code); + break; + } + + thread_exception_return(); +} + +static void +_enable_timebase_event_stream(uint32_t bit_index) +{ + uint64_t cntkctl; /* One wants to use 32 bits, but "mrs" prefers it this way */ + + if (bit_index >= 64) { + panic("%s: invalid bit index (%u)", __FUNCTION__, bit_index); + } + + __asm__ volatile ("mrs %0, CNTKCTL_EL1" : "=r"(cntkctl)); + + cntkctl |= (bit_index << CNTKCTL_EL1_EVENTI_SHIFT); + cntkctl |= CNTKCTL_EL1_EVNTEN; + cntkctl |= CNTKCTL_EL1_EVENTDIR; /* 1->0; why not? */ + + /* + * If the SOC supports it (and it isn't broken), enable + * EL0 access to the physical timebase register. + */ + if (user_timebase_allowed()) { + cntkctl |= CNTKCTL_EL1_PL0PCTEN; + } + + __asm__ volatile ("msr CNTKCTL_EL1, %0" : : "r"(cntkctl)); +} + +/* + * Turn timer on, unmask that interrupt. + */ +static void +_enable_virtual_timer(void) +{ + uint64_t cntvctl = CNTP_CTL_EL0_ENABLE; /* One wants to use 32 bits, but "mrs" prefers it this way */ + + __asm__ volatile ("msr CNTP_CTL_EL0, %0" : : "r"(cntvctl)); +} + +void +fiq_context_init(boolean_t enable_fiq __unused) +{ +#if defined(APPLE_ARM64_ARCH_FAMILY) + /* Could fill in our own ops here, if we needed them */ + uint64_t ticks_per_sec, ticks_per_event, events_per_sec; + uint32_t bit_index; + + ticks_per_sec = gPEClockFrequencyInfo.timebase_frequency_hz; +#if defined(ARM_BOARD_WFE_TIMEOUT_NS) + events_per_sec = 1000000000 / ARM_BOARD_WFE_TIMEOUT_NS; +#else + /* Default to 1usec (or as close as we can get) */ + events_per_sec = 1000000; +#endif + ticks_per_event = ticks_per_sec / events_per_sec; + bit_index = flsll(ticks_per_event) - 1; /* Highest bit set */ + + /* Round up to power of two */ + if ((ticks_per_event & ((1 << bit_index) - 1)) != 0) { + bit_index++; + } + + /* + * The timer can only trigger on rising or falling edge, + * not both; we don't care which we trigger on, but we + * do need to adjust which bit we are interested in to + * account for this. + */ + if (bit_index != 0) + bit_index--; + + _enable_timebase_event_stream(bit_index); +#else +#error Need a board configuration. +#endif + + /* Interrupts still disabled. */ + assert(ml_get_interrupts_enabled() == FALSE); + _enable_virtual_timer(); +} + +/* + * ARM64_TODO: remove me (just a convenience while we don't have crashreporter) + */ +extern int copyinframe(vm_address_t, char *, boolean_t); +size_t _OSUserBacktrace(char *buffer, size_t bufsize); + +size_t _OSUserBacktrace(char *buffer, size_t bufsize) +{ + thread_t thread = current_thread(); + boolean_t is64bit = thread_is_64bit(thread); + size_t trace_size_bytes = 0, lr_size; + vm_address_t frame_addr; // Should really by mach_vm_offset_t... + + if (bufsize < 8) { + return 0; + } + + if (get_threadtask(thread) == kernel_task) { + panic("%s: Should never be called from a kernel thread.", __FUNCTION__); + } + + frame_addr = get_saved_state_fp(thread->machine.upcb); + if (is64bit) { + uint64_t frame[2]; + lr_size = sizeof(frame[1]); + + *((uint64_t*)buffer) = get_saved_state_pc(thread->machine.upcb); + trace_size_bytes = lr_size; + + while (trace_size_bytes + lr_size < bufsize) { + if (!(frame_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS)) { + break; + } + + if (0 != copyinframe(frame_addr, (char*)frame, TRUE)) { + break; + } + + *((uint64_t*)(buffer + trace_size_bytes)) = frame[1]; /* lr */ + frame_addr = frame[0]; + trace_size_bytes += lr_size; + + if (frame[0] == 0x0ULL) { + break; + } + } + } else { + uint32_t frame[2]; + lr_size = sizeof(frame[1]); + + *((uint32_t*)buffer) = (uint32_t)get_saved_state_pc(thread->machine.upcb); + trace_size_bytes = lr_size; + + while (trace_size_bytes + lr_size < bufsize) { + if (!(frame_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS)) { + break; + } + + if (0 != copyinframe(frame_addr, (char*)frame, FALSE)) { + break; + } + + *((uint32_t*)(buffer + trace_size_bytes)) = frame[1]; /* lr */ + frame_addr = frame[0]; + trace_size_bytes += lr_size; + + if (frame[0] == 0x0ULL) { + break; + } + } + } + + return trace_size_bytes; +} + +boolean_t +ml_delay_should_spin(uint64_t interval) +{ + cpu_data_t *cdp = getCpuDatap(); + + if (cdp->cpu_idle_latency) { + return (interval < cdp->cpu_idle_latency) ? TRUE : FALSE; + } else { + /* + * Early boot, latency is unknown. Err on the side of blocking, + * which should always be safe, even if slow + */ + return FALSE; + } +} + +boolean_t ml_thread_is64bit(thread_t thread) { + return (thread_is_64bit(thread)); +} + +void ml_timer_evaluate(void) { +} + +boolean_t +ml_timer_forced_evaluation(void) { + return FALSE; +} + +uint64_t +ml_energy_stat(thread_t t) { + return t->machine.energy_estimate_nj; +} + + +void +ml_gpu_stat_update(__unused uint64_t gpu_ns_delta) { +#if CONFIG_EMBEDDED + /* + * For now: update the resource coalition stats of the + * current thread's coalition + */ + task_coalition_update_gpu_stats(current_task(), gpu_ns_delta); +#endif +} + +uint64_t +ml_gpu_stat(__unused thread_t t) { + return 0; +} + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME +static void +timer_state_event(boolean_t switch_to_kernel) +{ + thread_t thread = current_thread(); + if (!thread->precise_user_kernel_time) return; + + processor_data_t *pd = &getCpuDatap()->cpu_processor->processor_data; + uint64_t now = ml_get_timebase(); + + timer_stop(pd->current_state, now); + pd->current_state = (switch_to_kernel) ? &pd->system_state : &pd->user_state; + timer_start(pd->current_state, now); + + timer_stop(pd->thread_timer, now); + pd->thread_timer = (switch_to_kernel) ? &thread->system_timer : &thread->user_timer; + timer_start(pd->thread_timer, now); +} + +void +timer_state_event_user_to_kernel(void) +{ + timer_state_event(TRUE); +} + +void +timer_state_event_kernel_to_user(void) +{ + timer_state_event(FALSE); +} +#endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */ + +/* + * The following are required for parts of the kernel + * that cannot resolve these functions as inlines: + */ +extern thread_t current_act(void); +thread_t +current_act(void) +{ + return current_thread_fast(); +} + +#undef current_thread +extern thread_t current_thread(void); +thread_t +current_thread(void) +{ + return current_thread_fast(); +} + +typedef struct +{ + ex_cb_t cb; + void *refcon; +} +ex_cb_info_t; + +ex_cb_info_t ex_cb_info[EXCB_CLASS_MAX]; + +/* + * Callback registration + * Currently we support only one registered callback per class but + * it should be possible to support more callbacks + */ +kern_return_t ex_cb_register( + ex_cb_class_t cb_class, + ex_cb_t cb, + void *refcon) +{ + ex_cb_info_t *pInfo = &ex_cb_info[cb_class]; + + if ((NULL == cb) || (cb_class >= EXCB_CLASS_MAX)) + { + return KERN_INVALID_VALUE; + } + + if (NULL == pInfo->cb) + { + pInfo->cb = cb; + pInfo->refcon = refcon; + return KERN_SUCCESS; + } + return KERN_FAILURE; +} + +/* + * Called internally by platform kernel to invoke the registered callback for class + */ +ex_cb_action_t ex_cb_invoke( + ex_cb_class_t cb_class, + vm_offset_t far) +{ + ex_cb_info_t *pInfo = &ex_cb_info[cb_class]; + ex_cb_state_t state = {far}; + + if (cb_class >= EXCB_CLASS_MAX) + { + panic("Invalid exception callback class 0x%x\n", cb_class); + } + + if (pInfo->cb) + { + return pInfo->cb(cb_class, pInfo->refcon, &state); + } + return EXCB_ACTION_NONE; +} + diff --git a/osfmk/arm64/machine_routines_asm.s b/osfmk/arm64/machine_routines_asm.s new file mode 100644 index 000000000..915dffe57 --- /dev/null +++ b/osfmk/arm64/machine_routines_asm.s @@ -0,0 +1,970 @@ +/* + * Copyright (c) 2007-2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include "assym.s" + + +/* uint32_t get_fpscr(void): + * Returns (FPSR | FPCR). + */ + .align 2 + .globl EXT(get_fpscr) +LEXT(get_fpscr) +#if __ARM_VFP__ + mrs x1, FPSR // Grab FPSR + mov x4, #(FPSR_MASK & 0xFFFF) + mov x5, #(FPSR_MASK & 0xFFFF0000) + orr x0, x4, x5 + and x1, x1, x0 // Be paranoid, and clear bits we expect to + // be clear + mrs x2, FPCR // Grab FPCR + mov x4, #(FPCR_MASK & 0xFFFF) + mov x5, #(FPCR_MASK & 0xFFFF0000) + orr x0, x4, x5 + and x2, x2, x0 // Be paranoid, and clear bits we expect to + // be clear + orr x0, x1, x2 // OR them to get FPSCR equivalent state +#else + mov x0, #0 +#endif + ret + .align 2 + .globl EXT(set_fpscr) +/* void set_fpscr(uint32_t value): + * Set the FPCR and FPSR registers, based on the given value; a + * noteworthy point is that unlike 32-bit mode, 64-bit mode FPSR + * and FPCR are not responsible for condition codes. + */ +LEXT(set_fpscr) +#if __ARM_VFP__ + mov x4, #(FPSR_MASK & 0xFFFF) + mov x5, #(FPSR_MASK & 0xFFFF0000) + orr x1, x4, x5 + and x1, x1, x0 // Clear the bits that don't apply to FPSR + mov x4, #(FPCR_MASK & 0xFFFF) + mov x5, #(FPCR_MASK & 0xFFFF0000) + orr x2, x4, x5 + and x2, x2, x0 // Clear the bits that don't apply to FPCR + msr FPSR, x1 // Write FPCR + msr FPCR, x2 // Write FPSR + dsb ish // FPCR requires synchronization +#endif + ret + +#if (__ARM_VFP__ >= 3) + .align 2 + .globl EXT(get_mvfr0) +LEXT(get_mvfr0) + mrs x0, MVFR0_EL1 + ret + + .globl EXT(get_mvfr1) +LEXT(get_mvfr1) + mrs x0, MVFR1_EL1 + ret + +#endif + +/* + * void flush_mmu_tlb(void) + * + * Flush all TLBs + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb) +LEXT(flush_mmu_tlb) + tlbi vmalle1is + dsb ish + isb sy + ret + +/* + * void flush_core_tlb(void) + * + * Flush core TLB + */ + .text + .align 2 + .globl EXT(flush_core_tlb) +LEXT(flush_core_tlb) + tlbi vmalle1 + dsb ish + isb sy + ret + +/* + * void flush_mmu_tlb_allentries(uint64_t, uint64_t) + * + * Flush TLB entries + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_allentries) +LEXT(flush_mmu_tlb_allentries) +#if __ARM_16K_PG__ + and x0, x0, #~0x3 + + /* + * The code below is not necessarily correct. From an overview of + * the client code, the expected contract for TLB flushes is that + * we will expand from an "address, length" pair to "start address, + * end address" in the course of a TLB flush. This suggests that + * a flush for "X, X+4" is actually only asking for a flush of a + * single 16KB page. At the same time, we'd like to be prepared + * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page + * number to a 16KB page boundary. This should deal correctly with + * unaligned inputs. + * + * If our expecations about client behavior are wrong however, this + * will lead to occasional TLB corruption on platforms with 16KB + * pages. + */ + add x1, x1, #0x3 + and x1, x1, #~0x3 +#endif + +1: + tlbi vaae1is, x0 + add x0, x0, #(ARM_PGBYTES / 4096) // Units are 4KB pages, as defined by the ISA + cmp x0, x1 + b.lt 1b + dsb ish + isb sy + ret + +/* + * void flush_mmu_tlb_entry(uint64_t) + * + * Flush TLB entry + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_entry) +LEXT(flush_mmu_tlb_entry) + tlbi vae1is, x0 + dsb ish + isb sy + ret + +/* + * void flush_mmu_tlb_entries(uint64_t, uint64_t) + * + * Flush TLB entries + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_entries) +LEXT(flush_mmu_tlb_entries) +#if __ARM_16K_PG__ + and x0, x0, #~0x3 + + /* + * The code below is not necessarily correct. From an overview of + * the client code, the expected contract for TLB flushes is that + * we will expand from an "address, length" pair to "start address, + * end address" in the course of a TLB flush. This suggests that + * a flush for "X, X+4" is actually only asking for a flush of a + * single 16KB page. At the same time, we'd like to be prepared + * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page + * number to a 16KB page boundary. This should deal correctly with + * unaligned inputs. + * + * If our expecations about client behavior are wrong however, this + * will lead to occasional TLB corruption on platforms with 16KB + * pages. + */ + add x1, x1, #0x3 + and x1, x1, #~0x3 +#endif + +1: + tlbi vae1is, x0 + add x0, x0, #(ARM_PGBYTES / 4096) // Units are pages + cmp x0, x1 + b.lt 1b + dsb ish + isb sy + ret + +/* + * void flush_mmu_tlb_asid(uint64_t) + * + * Flush TLB entriesfor requested asid + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_asid) +LEXT(flush_mmu_tlb_asid) + tlbi aside1is, x0 + dsb ish + isb sy + ret + +/* + * void flush_core_tlb_asid(uint64_t) + * + * Flush TLB entries for core for requested asid + */ + .text + .align 2 + .globl EXT(flush_core_tlb_asid) +LEXT(flush_core_tlb_asid) + tlbi aside1, x0 + dsb ish + isb sy + ret + +/* + * Set MMU Translation Table Base Alternate + */ + .text + .align 2 + .globl EXT(set_mmu_ttb_alternate) +LEXT(set_mmu_ttb_alternate) + dsb sy +#if defined(KERNEL_INTEGRITY_KTRR) + mov x1, lr + bl EXT(pinst_set_ttbr1) + mov lr, x1 +#else + msr TTBR1_EL1, x0 +#endif /* defined(KERNEL_INTEGRITY_KTRR) */ + isb sy + ret + +/* + * set AUX control register + */ + .text + .align 2 + .globl EXT(set_aux_control) +LEXT(set_aux_control) + msr ACTLR_EL1, x0 + // Synchronize system + dsb sy + isb sy + ret + +#if (DEVELOPMENT || DEBUG) +/* + * set MMU control register + */ + .text + .align 2 + .globl EXT(set_mmu_control) +LEXT(set_mmu_control) + msr SCTLR_EL1, x0 + dsb sy + isb sy + ret +#endif + + +/* + * set translation control register + */ + .text + .align 2 + .globl EXT(set_tcr) +LEXT(set_tcr) +#if defined(APPLE_ARM64_ARCH_FAMILY) + // Assert that T0Z is always equal to T1Z + eor x1, x0, x0, lsr #(TCR_T1SZ_SHIFT - TCR_T0SZ_SHIFT) + and x1, x1, #(TCR_TSZ_MASK << TCR_T0SZ_SHIFT) + cbnz x1, L_set_tcr_panic +#if defined(KERNEL_INTEGRITY_KTRR) + mov x1, lr + bl _pinst_set_tcr + mov lr, x1 +#else + msr TCR_EL1, x0 +#endif /* defined(KERNEL_INTRITY_KTRR) */ + isb sy + ret + +L_set_tcr_panic: + PUSH_FRAME + sub sp, sp, #16 + str x0, [sp] + adr x0, L_set_tcr_panic_str + BRANCH_EXTERN panic + +L_set_locked_reg_panic: + PUSH_FRAME + sub sp, sp, #16 + str x0, [sp] + adr x0, L_set_locked_reg_panic_str + BRANCH_EXTERN panic + b . + +L_set_tcr_panic_str: + .asciz "set_tcr: t0sz, t1sz not equal (%llx)\n" + + +L_set_locked_reg_panic_str: + .asciz "attempt to set locked register: (%llx)\n" +#else +#if defined(KERNEL_INTEGRITY_KTRR) + mov x1, lr + bl _pinst_set_tcr + mov lr, x1 +#else + msr TCR_EL1, x0 +#endif + isb sy + ret +#endif // defined(APPLE_ARM64_ARCH_FAMILY) + +/* + * MMU kernel virtual to physical address translation + */ + .text + .align 2 + .globl EXT(mmu_kvtop) +LEXT(mmu_kvtop) + mrs x2, DAIF // Load current DAIF + msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ + at s1e1r, x0 // Translation Stage 1 EL1 + mrs x1, PAR_EL1 // Read result + msr DAIF, x2 // Restore interrupt state + tbnz x1, #0, L_mmu_kvtop_invalid // Test Translation not valid + bfm x1, x0, #0, #11 // Add page offset + and x0, x1, #0x0000ffffffffffff // Clear non-address bits + ret +L_mmu_kvtop_invalid: + mov x0, xzr // Return invalid + ret + +/* + * MMU user virtual to physical address translation + */ + .text + .align 2 + .globl EXT(mmu_uvtop) +LEXT(mmu_uvtop) + lsr x8, x0, #56 // Extract top byte + cbnz x8, L_mmu_uvtop_invalid // Tagged pointers are invalid + mrs x2, DAIF // Load current DAIF + msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ + at s1e0r, x0 // Translation Stage 1 EL0 + mrs x1, PAR_EL1 // Read result + msr DAIF, x2 // Restore interrupt state + tbnz x1, #0, L_mmu_uvtop_invalid // Test Translation not valid + bfm x1, x0, #0, #11 // Add page offset + and x0, x1, #0x0000ffffffffffff // Clear non-address bits + ret +L_mmu_uvtop_invalid: + mov x0, xzr // Return invalid + ret + +/* + * MMU kernel virtual to physical address preflight write access + */ + .text + .align 2 + .globl EXT(mmu_kvtop_wpreflight) +LEXT(mmu_kvtop_wpreflight) + mrs x2, DAIF // Load current DAIF + msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ + at s1e1w, x0 // Translation Stage 1 EL1 + mrs x1, PAR_EL1 // Read result + msr DAIF, x2 // Restore interrupt state + tbnz x1, #0, L_mmu_kvtop_wpreflight_invalid // Test Translation not valid + bfm x1, x0, #0, #11 // Add page offset + and x0, x1, #0x0000ffffffffffff // Clear non-address bits + ret +L_mmu_kvtop_wpreflight_invalid: + mov x0, xzr // Return invalid + ret + +/* + * SET_RECOVERY_HANDLER + * + * Sets up a page fault recovery handler + * + * arg0 - persisted thread pointer + * arg1 - persisted recovery handler + * arg2 - scratch reg + * arg3 - recovery label + */ +.macro SET_RECOVERY_HANDLER + mrs $0, TPIDR_EL1 // Load thread pointer + ldr $1, [$0, TH_RECOVER] // Save previous recovery handler + adrp $2, $3@page // Load the recovery handler address + add $2, $2, $3@pageoff + str $2, [$0, TH_RECOVER] // Set new recovery handler +.endmacro + +/* + * CLEAR_RECOVERY_HANDLER + * + * Clears page fault handler set by SET_RECOVERY_HANDLER + * + * arg0 - thread pointer saved by SET_RECOVERY_HANDLER + * arg1 - old recovery handler saved by SET_RECOVERY_HANDLER + */ +.macro CLEAR_RECOVERY_HANDLER + str $1, [$0, TH_RECOVER] // Restore the previous recovery handler +.endmacro + + + .text + .align 2 +copyio_error: + CLEAR_RECOVERY_HANDLER x10, x11 + mov x0, #EFAULT // Return an EFAULT error + POP_FRAME + ret + +/* + * int _bcopyin(const char *src, char *dst, vm_size_t len) + */ + .text + .align 2 + .globl EXT(_bcopyin) +LEXT(_bcopyin) + PUSH_FRAME + SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + /* If len is less than 16 bytes, just do a bytewise copy */ + cmp x2, #16 + b.lt 2f + sub x2, x2, #16 +1: + /* 16 bytes at a time */ + ldp x3, x4, [x0], #16 + stp x3, x4, [x1], #16 + subs x2, x2, #16 + b.ge 1b + /* Fixup the len and test for completion */ + adds x2, x2, #16 + b.eq 3f +2: /* Bytewise */ + subs x2, x2, #1 + ldrb w3, [x0], #1 + strb w3, [x1], #1 + b.hi 2b +3: + CLEAR_RECOVERY_HANDLER x10, x11 + mov x0, xzr + POP_FRAME + ret + +/* + * int _copyin_word(const char *src, uint64_t *dst, vm_size_t len) + */ + .text + .align 2 + .globl EXT(_copyin_word) +LEXT(_copyin_word) + PUSH_FRAME + SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + cmp x2, #4 + b.eq L_copyin_word_4 + cmp x2, #8 + b.eq L_copyin_word_8 + mov x0, EINVAL + b L_copying_exit +L_copyin_word_4: + ldr w8, [x0] + b L_copyin_word_store +L_copyin_word_8: + ldr x8, [x0] +L_copyin_word_store: + str x8, [x1] + mov x0, xzr + CLEAR_RECOVERY_HANDLER x10, x11 +L_copying_exit: + POP_FRAME + ret + + +/* + * int _bcopyout(const char *src, char *dst, vm_size_t len) + */ + .text + .align 2 + .globl EXT(_bcopyout) +LEXT(_bcopyout) + PUSH_FRAME + SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + /* If len is less than 16 bytes, just do a bytewise copy */ + cmp x2, #16 + b.lt 2f + sub x2, x2, #16 +1: + /* 16 bytes at a time */ + ldp x3, x4, [x0], #16 + stp x3, x4, [x1], #16 + subs x2, x2, #16 + b.ge 1b + /* Fixup the len and test for completion */ + adds x2, x2, #16 + b.eq 3f +2: /* Bytewise */ + subs x2, x2, #1 + ldrb w3, [x0], #1 + strb w3, [x1], #1 + b.hi 2b +3: + CLEAR_RECOVERY_HANDLER x10, x11 + mov x0, xzr + POP_FRAME + ret + +/* + * int _bcopyinstr( + * const user_addr_t user_addr, + * char *kernel_addr, + * vm_size_t max, + * vm_size_t *actual) + */ + .text + .align 2 + .globl EXT(_bcopyinstr) +LEXT(_bcopyinstr) + PUSH_FRAME + adr x4, Lcopyinstr_error // Get address for recover + mrs x10, TPIDR_EL1 // Get thread pointer + ldr x11, [x10, TH_RECOVER] // Save previous recover + str x4, [x10, TH_RECOVER] // Store new recover + mov x4, xzr // x4 - total bytes copied +Lcopyinstr_loop: + ldrb w5, [x0], #1 // Load a byte from the user source + strb w5, [x1], #1 // Store a byte to the kernel dest + add x4, x4, #1 // Increment bytes copied + cbz x5, Lcopyinstr_done // If this byte is null, we're done + cmp x4, x2 // If we're out of space, return an error + b.ne Lcopyinstr_loop +Lcopyinstr_too_long: + mov x5, #ENAMETOOLONG // Set current byte to error code for later return +Lcopyinstr_done: + str x4, [x3] // Return number of bytes copied + mov x0, x5 // Set error code (0 on success, ENAMETOOLONG on failure) + b Lcopyinstr_exit +Lcopyinstr_error: + mov x0, #EFAULT // Return EFAULT on error +Lcopyinstr_exit: + str x11, [x10, TH_RECOVER] // Restore old recover + POP_FRAME + ret + +/* + * int copyinframe(const vm_address_t frame_addr, char *kernel_addr, bool is64bit) + * + * Safely copy sixteen bytes (the fixed top of an ARM64 frame) from + * either user or kernel memory, or 8 bytes (AArch32) from user only. + * + * x0 : address of frame to copy. + * x1 : kernel address at which to store data. + * w2 : whether to copy an AArch32 or AArch64 frame. + * x3 : temp + * x5 : temp (kernel virtual base) + * x9 : temp + * x10 : thread pointer (set by SET_RECOVERY_HANDLER) + * x11 : old recovery function (set by SET_RECOVERY_HANDLER) + * x12, x13 : backtrace data + * + */ + .text + .align 2 + .globl EXT(copyinframe) +LEXT(copyinframe) + PUSH_FRAME + SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + cbnz w2, Lcopyinframe64 // Check frame size + adrp x5, EXT(gVirtBase)@page // For 32-bit frame, make sure we're not trying to copy from kernel + add x5, x5, EXT(gVirtBase)@pageoff + ldr x5, [x5] + cmp x5, x0 // See if address is in kernel virtual range + b.hi Lcopyinframe32 // If below kernel virtual range, proceed. + mov w0, #EFAULT // Should never have a 32-bit frame in kernel virtual range + b Lcopyinframe_done + +Lcopyinframe32: + ldr x12, [x0] // Copy 8 bytes + str x12, [x1] + mov w0, #0 // Success + b Lcopyinframe_done + +Lcopyinframe64: + mov x3, VM_MIN_KERNEL_ADDRESS // Check if kernel address + orr x9, x0, TBI_MASK // Hide tags in address comparison + cmp x9, x3 // If in kernel address range, skip tag test + b.hs Lcopyinframe_valid + tst x0, TBI_MASK // Detect tagged pointers + b.eq Lcopyinframe_valid + mov w0, #EFAULT // Tagged address, fail + b Lcopyinframe_done +Lcopyinframe_valid: + ldp x12, x13, [x0] // Copy 16 bytes + stp x12, x13, [x1] + mov w0, #0 // Success + +Lcopyinframe_done: + CLEAR_RECOVERY_HANDLER x10, x11 + POP_FRAME + ret + + +/* + * int _emulate_swp(user_addr_t addr, uint32_t newval, uint32_t *oldval) + * + * Securely emulates the swp instruction removed from armv8. + * Returns true on success. + * Returns false if the user address is not user accessible. + * + * x0 : address to swap + * x1 : new value to store + * x2 : address to save old value + * x3 : scratch reg + * x10 : thread pointer (set by SET_RECOVERY_HANDLER) + * x11 : old recovery handler (set by SET_RECOVERY_HANDLER) + * x12 : interrupt state + * x13 : return value + */ + .text + .align 2 + .globl EXT(_emulate_swp) +LEXT(_emulate_swp) + PUSH_FRAME + SET_RECOVERY_HANDLER x10, x11, x3, swp_error + + // Perform swap +Lswp_try: + ldxr w3, [x0] // Load data at target address + stxr w4, w1, [x0] // Store new value to target address + cbnz w4, Lswp_try // Retry if store failed + str w3, [x2] // Save old value + mov x13, #1 // Set successful return value + +Lswp_exit: + mov x0, x13 // Set return value + CLEAR_RECOVERY_HANDLER x10, x11 + POP_FRAME + ret + +/* + * int _emulate_swpb(user_addr_t addr, uint32_t newval, uint32_t *oldval) + * + * Securely emulates the swpb instruction removed from armv8. + * Returns true on success. + * Returns false if the user address is not user accessible. + * + * x0 : address to swap + * x1 : new value to store + * x2 : address to save old value + * x3 : scratch reg + * x10 : thread pointer (set by SET_RECOVERY_HANDLER) + * x11 : old recovery handler (set by SET_RECOVERY_HANDLER) + * x12 : interrupt state + * x13 : return value + */ + .text + .align 2 + .globl EXT(_emulate_swpb) +LEXT(_emulate_swpb) + PUSH_FRAME + SET_RECOVERY_HANDLER x10, x11, x3, swp_error + + // Perform swap +Lswpb_try: + ldxrb w3, [x0] // Load data at target address + stxrb w4, w1, [x0] // Store new value to target address + cbnz w4, Lswp_try // Retry if store failed + str w3, [x2] // Save old value + mov x13, #1 // Set successful return value + +Lswpb_exit: + mov x0, x13 // Set return value + CLEAR_RECOVERY_HANDLER x10, x11 + POP_FRAME + ret + + .text + .align 2 +swp_error: + mov x0, xzr // Return false + CLEAR_RECOVERY_HANDLER x10, x11 + POP_FRAME + ret + +/* + * uint32_t arm_debug_read_dscr(void) + */ + .text + .align 2 + .globl EXT(arm_debug_read_dscr) +LEXT(arm_debug_read_dscr) + PANIC_UNIMPLEMENTED + +/* + * void arm_debug_set_cp14(arm_debug_state_t *debug_state) + * + * Set debug registers to match the current thread state + * (NULL to disable). Assume 6 breakpoints and 2 + * watchpoints, since that has been the case in all cores + * thus far. + */ + .text + .align 2 + .globl EXT(arm_debug_set_cp14) +LEXT(arm_debug_set_cp14) + PANIC_UNIMPLEMENTED + + +#if defined(APPLE_ARM64_ARCH_FAMILY) +/* + * Note: still have to ISB before executing wfi! + */ + .text + .align 2 + .globl EXT(arm64_prepare_for_sleep) +LEXT(arm64_prepare_for_sleep) + PUSH_FRAME + +#if defined(APPLECYCLONE) || defined(APPLETYPHOON) + // CPU1 Stuck in WFIWT Because of MMU Prefetch + mrs x0, ARM64_REG_HID2 // Read HID2 + orr x0, x0, #(ARM64_REG_HID2_disMMUmtlbPrefetch) // Set HID.DisableMTLBPrefetch + msr ARM64_REG_HID2, x0 // Write HID2 + dsb sy + isb sy +#endif + +#if __ARM_GLOBAL_SLEEP_BIT__ + // Enable deep sleep + mrs x1, ARM64_REG_ACC_OVRD + orr x1, x1, #(ARM64_REG_ACC_OVRD_enDeepSleep) + and x1, x1, #(~(ARM64_REG_ACC_OVRD_disL2Flush4AccSlp_mask)) + orr x1, x1, #( ARM64_REG_ACC_OVRD_disL2Flush4AccSlp_deepsleep) + and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnSRM_mask)) + orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2PwrDnSRM_deepsleep) + and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2TrDnLnk_mask)) + orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2TrDnLnk_deepsleep) + and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnCPM_mask)) + orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2PwrDnCPM_deepsleep) + msr ARM64_REG_ACC_OVRD, x1 + + +#else + // Enable deep sleep + mov x1, ARM64_REG_CYC_CFG_deepSleep + msr ARM64_REG_CYC_CFG, x1 +#endif + // Set "OK to power down" () + mrs x0, ARM64_REG_CYC_OVRD + orr x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_down) + msr ARM64_REG_CYC_OVRD, x0 + +Lwfi_inst: + dsb sy + isb sy + wfi + b Lwfi_inst + +/* + * Force WFI to use clock gating only + * + */ + .text + .align 2 + .globl EXT(arm64_force_wfi_clock_gate) +LEXT(arm64_force_wfi_clock_gate) + PUSH_FRAME + + mrs x0, ARM64_REG_CYC_OVRD + orr x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_up) + msr ARM64_REG_CYC_OVRD, x0 + + POP_FRAME + ret + + + +#if defined(APPLECYCLONE) || defined(APPLETYPHOON) + + .text + .align 2 + .globl EXT(cyclone_typhoon_prepare_for_wfi) + +LEXT(cyclone_typhoon_prepare_for_wfi) + PUSH_FRAME + + // CPU1 Stuck in WFIWT Because of MMU Prefetch + mrs x0, ARM64_REG_HID2 // Read HID2 + orr x0, x0, #(ARM64_REG_HID2_disMMUmtlbPrefetch) // Set HID.DisableMTLBPrefetch + msr ARM64_REG_HID2, x0 // Write HID2 + dsb sy + isb sy + + POP_FRAME + ret + + + .text + .align 2 + .globl EXT(cyclone_typhoon_return_from_wfi) +LEXT(cyclone_typhoon_return_from_wfi) + PUSH_FRAME + + // CPU1 Stuck in WFIWT Because of MMU Prefetch + mrs x0, ARM64_REG_HID2 // Read HID2 + mov x1, #(ARM64_REG_HID2_disMMUmtlbPrefetch) // + bic x0, x0, x1 // Clear HID.DisableMTLBPrefetchMTLBPrefetch + msr ARM64_REG_HID2, x0 // Write HID2 + dsb sy + isb sy + + POP_FRAME + ret +#endif + +#ifdef APPLETYPHOON + +#define HID0_DEFEATURES_1 0x0000a0c000064010ULL +#define HID1_DEFEATURES_1 0x000000004005bf20ULL +#define HID2_DEFEATURES_1 0x0000000000102074ULL +#define HID3_DEFEATURES_1 0x0000000000400003ULL +#define HID4_DEFEATURES_1 0x83ff00e100000268ULL +#define HID7_DEFEATURES_1 0x000000000000000eULL + +#define HID0_DEFEATURES_2 0x0000a1c000020010ULL +#define HID1_DEFEATURES_2 0x000000000005d720ULL +#define HID2_DEFEATURES_2 0x0000000000002074ULL +#define HID3_DEFEATURES_2 0x0000000000400001ULL +#define HID4_DEFEATURES_2 0x8390000200000208ULL +#define HID7_DEFEATURES_2 0x0000000000000000ULL + +/* + arg0 = target register + arg1 = 64-bit constant +*/ +.macro LOAD_UINT64 + movz $0, #(($1 >> 48) & 0xffff), lsl #48 + movk $0, #(($1 >> 32) & 0xffff), lsl #32 + movk $0, #(($1 >> 16) & 0xffff), lsl #16 + movk $0, #(($1) & 0xffff) +.endmacro + + .text + .align 2 + .globl EXT(cpu_defeatures_set) +LEXT(cpu_defeatures_set) + PUSH_FRAME + cmp x0, #2 + b.eq cpu_defeatures_set_2 + cmp x0, #1 + b.ne cpu_defeatures_set_ret + LOAD_UINT64 x1, HID0_DEFEATURES_1 + mrs x0, ARM64_REG_HID0 + orr x0, x0, x1 + msr ARM64_REG_HID0, x0 + LOAD_UINT64 x1, HID1_DEFEATURES_1 + mrs x0, ARM64_REG_HID1 + orr x0, x0, x1 + msr ARM64_REG_HID1, x0 + LOAD_UINT64 x1, HID2_DEFEATURES_1 + mrs x0, ARM64_REG_HID2 + orr x0, x0, x1 + msr ARM64_REG_HID2, x0 + LOAD_UINT64 x1, HID3_DEFEATURES_1 + mrs x0, ARM64_REG_HID3 + orr x0, x0, x1 + msr ARM64_REG_HID3, x0 + LOAD_UINT64 x1, HID4_DEFEATURES_1 + mrs x0, ARM64_REG_HID4 + orr x0, x0, x1 + msr ARM64_REG_HID4, x0 + LOAD_UINT64 x1, HID7_DEFEATURES_1 + mrs x0, ARM64_REG_HID7 + orr x0, x0, x1 + msr ARM64_REG_HID7, x0 + dsb sy + isb sy + b cpu_defeatures_set_ret +cpu_defeatures_set_2: + LOAD_UINT64 x1, HID0_DEFEATURES_2 + mrs x0, ARM64_REG_HID0 + orr x0, x0, x1 + msr ARM64_REG_HID0, x0 + LOAD_UINT64 x1, HID1_DEFEATURES_2 + mrs x0, ARM64_REG_HID1 + orr x0, x0, x1 + msr ARM64_REG_HID1, x0 + LOAD_UINT64 x1, HID2_DEFEATURES_2 + mrs x0, ARM64_REG_HID2 + orr x0, x0, x1 + msr ARM64_REG_HID2, x0 + LOAD_UINT64 x1, HID3_DEFEATURES_2 + mrs x0, ARM64_REG_HID3 + orr x0, x0, x1 + msr ARM64_REG_HID3, x0 + LOAD_UINT64 x1, HID4_DEFEATURES_2 + mrs x0, ARM64_REG_HID4 + orr x0, x0, x1 + msr ARM64_REG_HID4, x0 + LOAD_UINT64 x1, HID7_DEFEATURES_2 + mrs x0, ARM64_REG_HID7 + orr x0, x0, x1 + msr ARM64_REG_HID7, x0 + dsb sy + isb sy + b cpu_defeatures_set_ret +cpu_defeatures_set_ret: + POP_FRAME + ret +#endif + +#endif + +#ifdef MONITOR +/* + * unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1, + uintptr_t arg2, uintptr_t arg3) + * + * Call the EL3 monitor with 4 arguments in registers + * The monitor interface maintains the same ABI as the C function call standard. Callee-saved + * registers are preserved, temporary registers are not. Parameters and results are passed in + * the usual manner. + */ + .text + .align 2 + .globl EXT(monitor_call) +LEXT(monitor_call) + smc 0x11 + ret +#endif + +/* vim: set sw=4 ts=4: */ diff --git a/osfmk/arm64/machine_task.c b/osfmk/arm64/machine_task.c new file mode 100644 index 000000000..d9efa1cad --- /dev/null +++ b/osfmk/arm64/machine_task.c @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#include +#include +#include + +extern zone_t ads_zone; + +kern_return_t +machine_task_set_state( + task_t task, + int flavor, + thread_state_t state, + mach_msg_type_number_t state_count) +{ + switch (flavor) { + case ARM_DEBUG_STATE: + { + arm_legacy_debug_state_t *tstate = (arm_legacy_debug_state_t *) state; + if (task_has_64BitAddr(task) || + (state_count != ARM_LEGACY_DEBUG_STATE_COUNT) || + (!debug_legacy_state_is_valid(tstate))) { + return KERN_INVALID_ARGUMENT; + } + + if (task->task_debug == NULL) { + task->task_debug = zalloc(ads_zone); + if (task->task_debug == NULL) + return KERN_FAILURE; + } + + copy_legacy_debug_state(tstate, (arm_legacy_debug_state_t *) task->task_debug, FALSE); /* FALSE OR TRUE doesn't matter since we are ignoring it for arm */ + + return KERN_SUCCESS; + } + case ARM_DEBUG_STATE32: + { + arm_debug_state32_t *tstate = (arm_debug_state32_t *) state; + if (task_has_64BitAddr(task) || + (state_count != ARM_DEBUG_STATE32_COUNT) || + (!debug_state_is_valid32(tstate))) { + return KERN_INVALID_ARGUMENT; + } + + if (task->task_debug == NULL) { + task->task_debug = zalloc(ads_zone); + if (task->task_debug == NULL) + return KERN_FAILURE; + } + + copy_debug_state32(tstate, (arm_debug_state32_t *) task->task_debug, FALSE); /* FALSE OR TRUE doesn't matter since we are ignoring it for arm */ + + return KERN_SUCCESS; + } + case ARM_DEBUG_STATE64: + { + arm_debug_state64_t *tstate = (arm_debug_state64_t *) state; + + if ((!task_has_64BitAddr(task)) || + (state_count != ARM_DEBUG_STATE64_COUNT) || + (!debug_state_is_valid64(tstate))) { + return KERN_INVALID_ARGUMENT; + } + + if (task->task_debug == NULL) { + task->task_debug = zalloc(ads_zone); + if (task->task_debug == NULL) + return KERN_FAILURE; + } + + copy_debug_state64(tstate, (arm_debug_state64_t *) task->task_debug, FALSE); /* FALSE OR TRUE doesn't matter since we are ignoring it for arm */ + + return KERN_SUCCESS; + } + case THREAD_STATE_NONE: /* Using this flavor to clear task_debug */ + { + if (task->task_debug != NULL) { + zfree(ads_zone, task->task_debug); + task->task_debug = NULL; + + return KERN_SUCCESS; + } + return KERN_FAILURE; + } + default: + { + return KERN_INVALID_ARGUMENT; + } + } + + return KERN_FAILURE; +} + +kern_return_t +machine_task_get_state(task_t task, + int flavor, + thread_state_t state, + mach_msg_type_number_t *state_count) +{ + switch (flavor) { + case ARM_DEBUG_STATE: + { + arm_legacy_debug_state_t *tstate = (arm_legacy_debug_state_t *) state; + + if (task_has_64BitAddr(task) || (*state_count != ARM_LEGACY_DEBUG_STATE_COUNT)) { + return KERN_INVALID_ARGUMENT; + } + + if (task->task_debug == NULL) { + bzero(state, sizeof(*tstate)); + } else { + copy_legacy_debug_state((arm_legacy_debug_state_t*) task->task_debug, tstate, FALSE); /* FALSE OR TRUE doesn't matter since we are ignoring it for arm */ + } + + return KERN_SUCCESS; + } + case ARM_DEBUG_STATE32: + { + arm_debug_state32_t *tstate = (arm_debug_state32_t *) state; + + if (task_has_64BitAddr(task) || (*state_count != ARM_DEBUG_STATE32_COUNT)) { + return KERN_INVALID_ARGUMENT; + } + + if (task->task_debug == NULL) { + bzero(state, sizeof(*tstate)); + } else { + copy_debug_state32((arm_debug_state32_t*) task->task_debug, tstate, FALSE); /* FALSE OR TRUE doesn't matter since we are ignoring it for arm */ + } + + return KERN_SUCCESS; + } + case ARM_DEBUG_STATE64: + { + arm_debug_state64_t *tstate = (arm_debug_state64_t *) state; + + if ((!task_has_64BitAddr(task)) || (*state_count != ARM_DEBUG_STATE64_COUNT)) { + return KERN_INVALID_ARGUMENT; + } + + if (task->task_debug == NULL) { + bzero(state, sizeof(*tstate)); + } else { + copy_debug_state64((arm_debug_state64_t*) task->task_debug, tstate, FALSE); /* FALSE OR TRUE doesn't matter since we are ignoring it for arm */ + } + + return KERN_SUCCESS; + } + default: + { + return KERN_INVALID_ARGUMENT; + } + + } + return KERN_FAILURE; +} + +void +machine_task_terminate(task_t task) +{ + if (task) { + void *task_debug; + + task_debug = task->task_debug; + if (task_debug != NULL) { + task->task_debug = NULL; + zfree(ads_zone, task_debug); + } + } +} + + +kern_return_t +machine_thread_inherit_taskwide( + thread_t thread, + task_t parent_task) +{ + if (parent_task->task_debug) { + int flavor; + mach_msg_type_number_t count; + + flavor = task_has_64BitAddr(parent_task) ? ARM_DEBUG_STATE64 : ARM_DEBUG_STATE32; + count = task_has_64BitAddr(parent_task) ? ARM_DEBUG_STATE64_COUNT : ARM_DEBUG_STATE32_COUNT; + + return machine_thread_set_state(thread, flavor, parent_task->task_debug, count); + } + + return KERN_SUCCESS; +} + + +void +machine_task_init(__unused task_t new_task, + __unused task_t parent_task, + __unused boolean_t memory_inherit) +{ +} diff --git a/osfmk/arm64/monotonic.h b/osfmk/arm64/monotonic.h new file mode 100644 index 000000000..1cc446028 --- /dev/null +++ b/osfmk/arm64/monotonic.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef ARM64_MONOTONIC_H +#define ARM64_MONOTONIC_H + +#include + +#if KERNEL_PRIVATE + +#include + +#define MT_NDEVS 1 + +#define MT_CORE_CYCLES 0 +#define MT_CORE_INSTRS 1 +#define MT_CORE_NFIXED 2 +#define MT_CORE_MAXVAL ((UINT64_C(1) << 48) - 1) + +#endif /* KERNEL_PRIVATE */ + +#if MACH_KERNEL_PRIVATE + +#include + + +static inline void +mt_fiq(void) +{ +} + +#endif /* MACH_KERNEL_PRIVATE */ + +#endif /* !defined(ARM64_MONOTONIC_H) */ diff --git a/osfmk/arm64/monotonic_arm64.c b/osfmk/arm64/monotonic_arm64.c new file mode 100644 index 000000000..4a1563ea7 --- /dev/null +++ b/osfmk/arm64/monotonic_arm64.c @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include /* static_assert, assert */ +#include /* panic */ +#include +#include /* CHAR_BIT */ +#include +#include +#include +#include +#include +#include +#include + +#pragma mark core counters + +bool mt_core_supported = true; +void mt_fiq_internal(uint64_t upmsr); + +/* + * PMC[0-1] are the 48-bit fixed counters -- PMC0 is cycles and PMC1 is + * instructions (see arm64/monotonic.h). + * + * PMC2+ are currently handled by kpc. + */ + +#define PMC0 "s3_2_c15_c0_0" +#define PMC1 "s3_2_c15_c1_0" +#define PMC2 "s3_2_c15_c2_0" +#define PMC3 "s3_2_c15_c3_0" +#define PMC4 "s3_2_c15_c4_0" +#define PMC5 "s3_2_c15_c5_0" +#define PMC6 "s3_2_c15_c6_0" +#define PMC7 "s3_2_c15_c7_0" +#define PMC8 "s3_2_c15_c9_0" +#define PMC9 "s3_2_c15_c10_0" + +#define CYCLES 0 +#define INSTRS 1 + +/* + * PMC0's offset into a core's PIO range. + * + * This allows cores to remotely query another core's counters. + */ + +#define PIO_PMC0_OFFSET (0x200) + +/* + * The offset of the counter in the configuration registers. Post-Hurricane + * devices have additional counters that need a larger shift than the original + * counters. + * + * XXX For now, just support the lower-numbered counters. + */ +#define CTR_POS(CTR) (CTR) + +/* + * PMCR0 is the main control register for the performance monitor. It + * controls whether the counters are enabled, how they deliver interrupts, and + * other features. + */ + +#define PMCR0 "s3_1_c15_c0_0" + +#define PMCR0_CTR_EN(CTR) (UINT64_C(1) << CTR_POS(CTR)) +#define PMCR0_FIXED_EN (PMCR0_CTR_EN(CYCLES) | PMCR0_CTR_EN(INSTRS)) +/* how interrupts are delivered on a PMI */ +enum { + PMCR0_INTGEN_OFF = 0, + PMCR0_INTGEN_PMI = 1, + PMCR0_INTGEN_AIC = 2, + PMCR0_INTGEN_HALT = 3, + PMCR0_INTGEN_FIQ = 4, +}; +#define PMCR0_INTGEN_SET(INT) ((uint64_t)(INT) << 8) +/* use AIC for backwards compatibility with kpc */ +#define PMCR0_INTGEN_INIT PMCR0_INTGEN_SET(PMCR0_INTGEN_AIC) +/* set by hardware if a PMI was delivered */ +#define PMCR0_PMAI (UINT64_C(1) << 11) +#define PMCR0_PMI_EN(CTR) (UINT64_C(1) << (12 + CTR_POS(CTR))) +/* fixed counters are always counting XXX probably need to just set this to all true */ +#define PMCR0_PMI_INIT (PMCR0_PMI_EN(CYCLES) | PMCR0_PMI_EN(INSTRS)) +/* disable counting on a PMI (except for AIC interrupts) */ +#define PMCR0_DISCNT_EN (UINT64_C(1) << 20) +/* block PMIs until ERET retires */ +#define PMCR0_WFRFE_EN (UINT64_C(1) << 22) +/* count global (not just core-local) L2C events */ +#define PMCR0_L2CGLOBAL_EN (UINT64_C(1) << 23) +/* user mode access to configuration registers */ +#define PMCR0_USEREN_EN (UINT64_C(1) << 30) + +/* XXX this needs to be synchronized with kpc... */ +#define PMCR0_INIT (PMCR0_INTGEN_INIT | PMCR0_PMI_INIT | PMCR0_DISCNT_EN) + +/* + * PMCR1 controls which execution modes count events. + */ + +#define PMCR1 "s3_1_c15_c1_0" + +#define PMCR1_EL0A32_EN(CTR) (UINT64_C(1) << (0 + CTR_POS(CTR))) +#define PMCR1_EL0A64_EN(CTR) (UINT64_C(1) << (8 + CTR_POS(CTR))) +#define PMCR1_EL1A64_EN(CTR) (UINT64_C(1) << (16 + CTR_POS(CTR))) +/* PMCR1_EL3A64 is not supported on systems with no monitor */ +#if defined(APPLEHURRICANE) +#define PMCR1_EL3A64_EN(CTR) UINT64_C(0) +#else +#define PMCR1_EL3A64_EN(CTR) (UINT64_C(1) << (24 + CTR_POS(CTR))) +#endif +#define PMCR1_ALL_EN(CTR) (PMCR1_EL0A32_EN(CTR) | PMCR1_EL0A64_EN(CTR) | \ + PMCR1_EL1A64_EN(CTR) | PMCR1_EL3A64_EN(CTR)) + +/* fixed counters always count in all modes */ +#define PMCR1_INIT (PMCR1_ALL_EN(CYCLES) | PMCR1_ALL_EN(INSTRS)) + +static inline void +core_init_execution_modes(void) +{ + uint64_t pmcr1; + + pmcr1 = __builtin_arm_rsr64(PMCR1); + pmcr1 |= PMCR1_INIT; + __builtin_arm_wsr64(PMCR1, pmcr1); +} + +/* + * PMSR reports the overflow status of all counters. + */ + +#define PMSR "s3_1_c15_c13_0" + +#define PMSR_OVF(CTR) (UINT64_C(1) << (CTR)) + +/* + * PMCR2 controls watchpoint registers. + * + * PMCR3 controls breakpoints and address matching. + * + * PMCR4 controls opcode matching. + */ + +#define PMCR2 "s3_1_c15_c2_0" +#define PMCR3 "s3_1_c15_c3_0" +#define PMCR4 "s3_1_c15_c4_0" + +/* + * PMCR_AFFINITY does ??? XXX. + */ + +#define PMCR_AFFINITY "s3_1_c15_c11_0" + +void +mt_init(void) +{ +} + +static int +core_init(void) +{ + /* the dev node interface to the core counters is still unsupported */ + return ENOTSUP; +} + +struct mt_cpu * +mt_cur_cpu(void) +{ + return &getCpuDatap()->cpu_monotonic; +} + +uint64_t +mt_core_snap(unsigned int ctr) +{ + switch (ctr) { + case 0: + return __builtin_arm_rsr64(PMC0); + case 1: + return __builtin_arm_rsr64(PMC1); + default: + panic("monotonic: invalid core counter read: %u", ctr); + __builtin_trap(); + } +} + +void +mt_core_set_snap(unsigned int ctr, uint64_t count) +{ + switch (ctr) { + case 0: + __builtin_arm_wsr64(PMC0, count); + break; + case 1: + __builtin_arm_wsr64(PMC1, count); + break; + default: + panic("monotonic: invalid core counter %u write %llu", ctr, count); + __builtin_trap(); + } +} + +static void +core_set_enabled(void) +{ + uint64_t pmcr0; + + pmcr0 = __builtin_arm_rsr64(PMCR0); + pmcr0 |= PMCR0_INIT | PMCR0_FIXED_EN; + __builtin_arm_wsr64(PMCR0, pmcr0); +} + +static void +core_idle(__unused cpu_data_t *cpu) +{ + assert(cpu != NULL); + assert(ml_get_interrupts_enabled() == FALSE); + +#if DEBUG + uint64_t pmcr0 = __builtin_arm_rsr64(PMCR0); + if ((pmcr0 & PMCR0_FIXED_EN) == 0) { + panic("monotonic: counters disabled while idling, pmcr0 = 0x%llx\n", pmcr0); + } + uint64_t pmcr1 = __builtin_arm_rsr64(PMCR1); + if ((pmcr1 & PMCR1_INIT) == 0) { + panic("monotonic: counter modes disabled while idling, pmcr1 = 0x%llx\n", pmcr1); + } +#endif /* DEBUG */ + + /* disable counters before updating */ + __builtin_arm_wsr64(PMCR0, PMCR0_INIT); + + mt_update_fixed_counts(); +} + +static void +core_run(cpu_data_t *cpu) +{ + uint64_t pmcr0; + struct mt_cpu *mtc; + + assert(cpu != NULL); + assert(ml_get_interrupts_enabled() == FALSE); + + mtc = &cpu->cpu_monotonic; + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + mt_core_set_snap(i, mtc->mtc_snaps[i]); + } + + /* re-enable the counters */ + core_init_execution_modes(); + + pmcr0 = __builtin_arm_rsr64(PMCR0); + pmcr0 |= PMCR0_INIT | PMCR0_FIXED_EN; + __builtin_arm_wsr64(PMCR0, pmcr0); +} + +static void +core_up(__unused cpu_data_t *cpu) +{ + assert(ml_get_interrupts_enabled() == FALSE); + + core_init_execution_modes(); +} + +#pragma mark uncore counters + + +static void +uncore_sleep(void) +{ +} + +static void +uncore_wake(void) +{ +} + +static void +uncore_fiq(uint64_t upmsr) +{ +#pragma unused(upmsr) +} + +#pragma mark common hooks + +void +mt_cpu_idle(cpu_data_t *cpu) +{ + core_idle(cpu); +} + +void +mt_cpu_run(cpu_data_t *cpu) +{ + core_run(cpu); +} + +void +mt_cpu_down(cpu_data_t *cpu) +{ + mt_cpu_idle(cpu); +} + +void +mt_cpu_up(cpu_data_t *cpu) +{ + core_up(cpu); + mt_cpu_run(cpu); +} + +void +mt_sleep(void) +{ + uncore_sleep(); +} + +void +mt_wake(void) +{ + uncore_wake(); +} + +void +mt_cpu_pmi(cpu_data_t *cpu, uint64_t pmsr) +{ + bool found_overflow = false; + + assert(cpu != NULL); + assert(ml_get_interrupts_enabled() == FALSE); + + (void)atomic_fetch_add_explicit(&mt_pmis, 1, memory_order_relaxed); + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + if (pmsr & PMSR_OVF(i)) { + mt_cpu_update_count(cpu, i); + mt_core_set_snap(i, 0); + found_overflow = true; + } + } + + assert(found_overflow); + core_set_enabled(); +} + +void +mt_fiq_internal(uint64_t upmsr) +{ + uncore_fiq(upmsr); +} + +#pragma mark dev nodes + +const struct monotonic_dev monotonic_devs[] = { + [0] = { + .mtd_name = "monotonic/core", + .mtd_init = core_init, + }, +}; + +static_assert( + (sizeof(monotonic_devs) / sizeof(monotonic_devs[0])) == MT_NDEVS, + "MT_NDEVS macro should be same as the length of monotonic_devs"); diff --git a/osfmk/arm64/pcb.c b/osfmk/arm64/pcb.c new file mode 100644 index 000000000..55d5fe4d9 --- /dev/null +++ b/osfmk/arm64/pcb.c @@ -0,0 +1,878 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if MONOTONIC +#include +#endif /* MONOTONIC */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#define USER_SS_ZONE_ALLOC_SIZE (0x4000) + +extern int debug_task; + +zone_t ads_zone; /* zone for debug_state area */ +zone_t user_ss_zone; /* zone for user arm_context_t allocations */ + +/* + * Routine: consider_machine_collect + * + */ +void +consider_machine_collect(void) +{ + pmap_gc(); +} + +/* + * Routine: consider_machine_adjust + * + */ +void +consider_machine_adjust(void) +{ +} + +/* + * Routine: machine_switch_context + * + */ +thread_t +machine_switch_context( + thread_t old, + thread_continue_t continuation, + thread_t new) +{ + thread_t retval; + pmap_t new_pmap; + cpu_data_t *cpu_data_ptr; + +#define machine_switch_context_kprintf(x...) /* kprintf("machine_switch_con + * text: " x) */ + + cpu_data_ptr = getCpuDatap(); + if (old == new) + panic("machine_switch_context"); + + kpc_off_cpu(old); + + + new_pmap = new->map->pmap; + if (old->map->pmap != new_pmap) + pmap_switch(new_pmap); + + new->machine.CpuDatap = cpu_data_ptr; + + machine_switch_context_kprintf("old= %x contination = %x new = %x\n", old, continuation, new); + + retval = Switch_context(old, continuation, new); + assert(retval != NULL); + + return retval; +} + +/* + * Routine: machine_thread_create + * + */ +kern_return_t +machine_thread_create( + thread_t thread, + task_t task) +{ + arm_context_t *thread_user_ss = NULL; + kern_return_t result = KERN_SUCCESS; + +#define machine_thread_create_kprintf(x...) /* kprintf("machine_thread_create: " x) */ + + machine_thread_create_kprintf("thread = %x\n", thread); + + if (current_thread() != thread) { + thread->machine.CpuDatap = (cpu_data_t *)0; + } + thread->machine.preemption_count = 0; + thread->machine.cthread_self = 0; + thread->machine.cthread_data = 0; + + + if (task != kernel_task) { + /* If this isn't a kernel thread, we'll have userspace state. */ + thread->machine.contextData = (arm_context_t *)zalloc(user_ss_zone); + + if (!thread->machine.contextData) { + return KERN_FAILURE; + } + + thread->machine.upcb = &thread->machine.contextData->ss; + thread->machine.uNeon = &thread->machine.contextData->ns; + + if (task_has_64BitAddr(task)) { + thread->machine.upcb->ash.flavor = ARM_SAVED_STATE64; + thread->machine.upcb->ash.count = ARM_SAVED_STATE64_COUNT; + thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE64; + thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE64_COUNT; + } else { + thread->machine.upcb->ash.flavor = ARM_SAVED_STATE32; + thread->machine.upcb->ash.count = ARM_SAVED_STATE32_COUNT; + thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32; + thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT; + } + } else { + thread->machine.upcb = NULL; + thread->machine.uNeon = NULL; + thread->machine.contextData = NULL; + } + + bzero(&thread->machine.perfctrl_state, sizeof(thread->machine.perfctrl_state)); + + result = machine_thread_state_initialize(thread); + + if (result != KERN_SUCCESS) { + thread_user_ss = thread->machine.contextData; + thread->machine.upcb = NULL; + thread->machine.uNeon = NULL; + thread->machine.contextData = NULL; + zfree(user_ss_zone, thread_user_ss); + } + + return result; +} + +/* + * Routine: machine_thread_destroy + * + */ +void +machine_thread_destroy( + thread_t thread) +{ + arm_context_t *thread_user_ss; + + if (thread->machine.contextData) { + /* Disassociate the user save state from the thread before we free it. */ + thread_user_ss = thread->machine.contextData; + thread->machine.upcb = NULL; + thread->machine.uNeon = NULL; + thread->machine.contextData = NULL; + zfree(user_ss_zone, thread_user_ss); + } + + if (thread->machine.DebugData != NULL) { + if (thread->machine.DebugData == getCpuDatap()->cpu_user_debug) { + arm_debug_set(NULL); + } + + zfree(ads_zone, thread->machine.DebugData); + } +} + + +/* + * Routine: machine_thread_init + * + */ +void +machine_thread_init(void) +{ + ads_zone = zinit(sizeof(arm_debug_state_t), + THREAD_CHUNK * (sizeof(arm_debug_state_t)), + THREAD_CHUNK * (sizeof(arm_debug_state_t)), + "arm debug state"); + + /* + * Create a zone for the user save state. At the time this zone was created, + * the user save state was 848 bytes, and the matching kalloc zone was 1024 + * bytes, which would result in significant amounts of wasted space if we + * simply used kalloc to allocate the user saved state. + * + * 0x4000 has been chosen as the allocation size, as it results in 272 bytes + * of wasted space per chunk, which should correspond to 19 allocations. + */ + user_ss_zone = zinit(sizeof(arm_context_t), + CONFIG_THREAD_MAX * (sizeof(arm_context_t)), + USER_SS_ZONE_ALLOC_SIZE, + "user save state"); +} + + +/* + * Routine: get_useraddr + * + */ +user_addr_t +get_useraddr() +{ + return (get_saved_state_pc(current_thread()->machine.upcb)); +} + +/* + * Routine: machine_stack_detach + * + */ +vm_offset_t +machine_stack_detach( + thread_t thread) +{ + vm_offset_t stack; + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DETACH), + (uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0); + + stack = thread->kernel_stack; + thread->kernel_stack = 0; + thread->machine.kstackptr = 0; + + return (stack); +} + + +/* + * Routine: machine_stack_attach + * + */ +void +machine_stack_attach( + thread_t thread, + vm_offset_t stack) +{ + struct arm_context *context; + struct arm_saved_state64 *savestate; + +#define machine_stack_attach_kprintf(x...) /* kprintf("machine_stack_attach: " x) */ + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_ATTACH), + (uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0); + + thread->kernel_stack = stack; + thread->machine.kstackptr = stack + kernel_stack_size - sizeof(struct thread_kernel_state); + thread_initialize_kernel_state(thread); + + machine_stack_attach_kprintf("kstackptr: %lx\n", (vm_address_t)thread->machine.kstackptr); + + context = &((thread_kernel_state_t) thread->machine.kstackptr)->machine; + savestate = saved_state64(&context->ss); + savestate->fp = 0; + savestate->lr = (uintptr_t)thread_continue; + savestate->sp = thread->machine.kstackptr; + savestate->cpsr = PSR64_KERNEL_DEFAULT; + machine_stack_attach_kprintf("thread = %x pc = %x, sp = %x\n", thread, savestate->lr, savestate->sp); +} + + +/* + * Routine: machine_stack_handoff + * + */ +void +machine_stack_handoff( + thread_t old, + thread_t new) +{ + vm_offset_t stack; + pmap_t new_pmap; + cpu_data_t *cpu_data_ptr; + + kpc_off_cpu(old); + + stack = machine_stack_detach(old); + cpu_data_ptr = getCpuDatap(); + new->kernel_stack = stack; + new->machine.kstackptr = stack + kernel_stack_size - sizeof(struct thread_kernel_state); + if (stack == old->reserved_stack) { + assert(new->reserved_stack); + old->reserved_stack = new->reserved_stack; + new->reserved_stack = stack; + } + + + new_pmap = new->map->pmap; + if (old->map->pmap != new_pmap) + pmap_switch(new_pmap); + + new->machine.CpuDatap = cpu_data_ptr; + machine_set_current_thread(new); + thread_initialize_kernel_state(new); + + return; +} + + +/* + * Routine: call_continuation + * + */ +void +call_continuation( + thread_continue_t continuation, + void *parameter, + wait_result_t wresult) +{ +#define call_continuation_kprintf(x...) /* kprintf("call_continuation_kprintf:" x) */ + + call_continuation_kprintf("thread = %p continuation = %p, stack = %p\n", current_thread(), continuation, current_thread()->machine.kstackptr); + Call_continuation(continuation, parameter, wresult, current_thread()->machine.kstackptr); +} + +void arm_debug_set32(arm_debug_state_t *debug_state) +{ + struct cpu_data *cpu_data_ptr; + arm_debug_info_t *debug_info = arm_debug_info(); + volatile uint64_t state; + boolean_t intr, set_mde = 0; + arm_debug_state_t off_state; + uint32_t i; + + intr = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + // Set current user debug + cpu_data_ptr->cpu_user_debug = debug_state; + + if (NULL == debug_state) { + bzero(&off_state, sizeof(off_state)); + debug_state = &off_state; + } + + switch (debug_info->num_breakpoint_pairs) { + case 16: + __asm__ volatile("msr DBGBVR15_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[15])); + __asm__ volatile("msr DBGBCR15_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[15])); + case 15: + __asm__ volatile("msr DBGBVR14_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[14])); + __asm__ volatile("msr DBGBCR14_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[14])); + case 14: + __asm__ volatile("msr DBGBVR13_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[13])); + __asm__ volatile("msr DBGBCR13_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[13])); + case 13: + __asm__ volatile("msr DBGBVR12_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[12])); + __asm__ volatile("msr DBGBCR12_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[12])); + case 12: + __asm__ volatile("msr DBGBVR11_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[11])); + __asm__ volatile("msr DBGBCR11_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[11])); + case 11: + __asm__ volatile("msr DBGBVR10_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[10])); + __asm__ volatile("msr DBGBCR10_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[10])); + case 10: + __asm__ volatile("msr DBGBVR9_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[9])); + __asm__ volatile("msr DBGBCR9_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[9])); + case 9: + __asm__ volatile("msr DBGBVR8_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[8])); + __asm__ volatile("msr DBGBCR8_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[8])); + case 8: + __asm__ volatile("msr DBGBVR7_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[7])); + __asm__ volatile("msr DBGBCR7_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[7])); + case 7: + __asm__ volatile("msr DBGBVR6_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[6])); + __asm__ volatile("msr DBGBCR6_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[6])); + case 6: + __asm__ volatile("msr DBGBVR5_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[5])); + __asm__ volatile("msr DBGBCR5_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[5])); + case 5: + __asm__ volatile("msr DBGBVR4_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[4])); + __asm__ volatile("msr DBGBCR4_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[4])); + case 4: + __asm__ volatile("msr DBGBVR3_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[3])); + __asm__ volatile("msr DBGBCR3_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[3])); + case 3: + __asm__ volatile("msr DBGBVR2_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[2])); + __asm__ volatile("msr DBGBCR2_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[2])); + case 2: + __asm__ volatile("msr DBGBVR1_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[1])); + __asm__ volatile("msr DBGBCR1_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[1])); + case 1: + __asm__ volatile("msr DBGBVR0_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bvr[0])); + __asm__ volatile("msr DBGBCR0_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.bcr[0])); + default: + break; + } + + switch (debug_info->num_watchpoint_pairs) { + case 16: + __asm__ volatile("msr DBGWVR15_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[15])); + __asm__ volatile("msr DBGWCR15_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[15])); + case 15: + __asm__ volatile("msr DBGWVR14_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[14])); + __asm__ volatile("msr DBGWCR14_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[14])); + case 14: + __asm__ volatile("msr DBGWVR13_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[13])); + __asm__ volatile("msr DBGWCR13_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[13])); + case 13: + __asm__ volatile("msr DBGWVR12_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[12])); + __asm__ volatile("msr DBGWCR12_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[12])); + case 12: + __asm__ volatile("msr DBGWVR11_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[11])); + __asm__ volatile("msr DBGWCR11_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[11])); + case 11: + __asm__ volatile("msr DBGWVR10_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[10])); + __asm__ volatile("msr DBGWCR10_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[10])); + case 10: + __asm__ volatile("msr DBGWVR9_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[9])); + __asm__ volatile("msr DBGWCR9_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[9])); + case 9: + __asm__ volatile("msr DBGWVR8_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[8])); + __asm__ volatile("msr DBGWCR8_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[8])); + case 8: + __asm__ volatile("msr DBGWVR7_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[7])); + __asm__ volatile("msr DBGWCR7_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[7])); + case 7: + __asm__ volatile("msr DBGWVR6_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[6])); + __asm__ volatile("msr DBGWCR6_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[6])); + case 6: + __asm__ volatile("msr DBGWVR5_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[5])); + __asm__ volatile("msr DBGWCR5_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[5])); + case 5: + __asm__ volatile("msr DBGWVR4_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[4])); + __asm__ volatile("msr DBGWCR4_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[4])); + case 4: + __asm__ volatile("msr DBGWVR3_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[3])); + __asm__ volatile("msr DBGWCR3_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[3])); + case 3: + __asm__ volatile("msr DBGWVR2_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[2])); + __asm__ volatile("msr DBGWCR2_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[2])); + case 2: + __asm__ volatile("msr DBGWVR1_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[1])); + __asm__ volatile("msr DBGWCR1_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[1])); + case 1: + __asm__ volatile("msr DBGWVR0_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wvr[0])); + __asm__ volatile("msr DBGWCR0_EL1, %0" : : "r"((uint64_t)debug_state->uds.ds32.wcr[0])); + default: + break; + } + + for (i = 0; i < debug_info->num_breakpoint_pairs; i++) { + if (0 != debug_state->uds.ds32.bcr[i]) { + set_mde = 1; + break; + } + } + + for (i = 0; i < debug_info->num_watchpoint_pairs; i++) { + if (0 != debug_state->uds.ds32.wcr[i]) { + set_mde = 1; + break; + } + } + + /* + * Breakpoint/Watchpoint Enable + */ + if (set_mde) { + + __asm__ volatile("mrs %0, MDSCR_EL1" : "=r"(state)); + state |= 0x8000; // MDSCR_EL1[MDE] + __asm__ volatile("msr MDSCR_EL1, %0" : : "r"(state)); + + } else { + + __asm__ volatile("mrs %0, MDSCR_EL1" : "=r"(state)); + state &= ~0x8000; + __asm__ volatile("msr MDSCR_EL1, %0" : : "r"(state)); + + } + + /* + * Software debug single step enable + */ + if (debug_state->uds.ds32.mdscr_el1 & 0x1) { + + __asm__ volatile("mrs %0, MDSCR_EL1" : "=r"(state)); + state = (state & ~0x8000) | 0x1; // ~MDE | SS : no brk/watch while single stepping (which we've set) + __asm__ volatile("msr MDSCR_EL1, %0" : : "r"(state)); + + set_saved_state_cpsr((current_thread()->machine.upcb), + get_saved_state_cpsr((current_thread()->machine.upcb)) | PSR64_SS); + + } else { + + __asm__ volatile("mrs %0, MDSCR_EL1" : "=r"(state)); + state &= ~0x1; + __asm__ volatile("msr MDSCR_EL1, %0" : : "r"(state)); + +#if SINGLE_STEP_RETIRE_ERRATA + // Workaround for radar 20619637 + __builtin_arm_isb(ISB_SY); +#endif + } + + (void) ml_set_interrupts_enabled(intr); + + return; +} + +void arm_debug_set64(arm_debug_state_t *debug_state) +{ + struct cpu_data *cpu_data_ptr; + arm_debug_info_t *debug_info = arm_debug_info(); + volatile uint64_t state; + boolean_t intr, set_mde = 0; + arm_debug_state_t off_state; + uint32_t i; + + intr = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + // Set current user debug + cpu_data_ptr->cpu_user_debug = debug_state; + + if (NULL == debug_state) { + bzero(&off_state, sizeof(off_state)); + debug_state = &off_state; + } + + switch (debug_info->num_breakpoint_pairs) { + case 16: + __asm__ volatile("msr DBGBVR15_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[15])); + __asm__ volatile("msr DBGBCR15_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[15])); + case 15: + __asm__ volatile("msr DBGBVR14_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[14])); + __asm__ volatile("msr DBGBCR14_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[14])); + case 14: + __asm__ volatile("msr DBGBVR13_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[13])); + __asm__ volatile("msr DBGBCR13_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[13])); + case 13: + __asm__ volatile("msr DBGBVR12_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[12])); + __asm__ volatile("msr DBGBCR12_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[12])); + case 12: + __asm__ volatile("msr DBGBVR11_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[11])); + __asm__ volatile("msr DBGBCR11_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[11])); + case 11: + __asm__ volatile("msr DBGBVR10_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[10])); + __asm__ volatile("msr DBGBCR10_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[10])); + case 10: + __asm__ volatile("msr DBGBVR9_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[9])); + __asm__ volatile("msr DBGBCR9_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[9])); + case 9: + __asm__ volatile("msr DBGBVR8_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[8])); + __asm__ volatile("msr DBGBCR8_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[8])); + case 8: + __asm__ volatile("msr DBGBVR7_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[7])); + __asm__ volatile("msr DBGBCR7_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[7])); + case 7: + __asm__ volatile("msr DBGBVR6_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[6])); + __asm__ volatile("msr DBGBCR6_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[6])); + case 6: + __asm__ volatile("msr DBGBVR5_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[5])); + __asm__ volatile("msr DBGBCR5_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[5])); + case 5: + __asm__ volatile("msr DBGBVR4_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[4])); + __asm__ volatile("msr DBGBCR4_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[4])); + case 4: + __asm__ volatile("msr DBGBVR3_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[3])); + __asm__ volatile("msr DBGBCR3_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[3])); + case 3: + __asm__ volatile("msr DBGBVR2_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[2])); + __asm__ volatile("msr DBGBCR2_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[2])); + case 2: + __asm__ volatile("msr DBGBVR1_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[1])); + __asm__ volatile("msr DBGBCR1_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[1])); + case 1: + __asm__ volatile("msr DBGBVR0_EL1, %0" : : "r"(debug_state->uds.ds64.bvr[0])); + __asm__ volatile("msr DBGBCR0_EL1, %0" : : "r"(debug_state->uds.ds64.bcr[0])); + default: + break; + } + + switch (debug_info->num_watchpoint_pairs) { + case 16: + __asm__ volatile("msr DBGWVR15_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[15])); + __asm__ volatile("msr DBGWCR15_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[15])); + case 15: + __asm__ volatile("msr DBGWVR14_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[14])); + __asm__ volatile("msr DBGWCR14_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[14])); + case 14: + __asm__ volatile("msr DBGWVR13_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[13])); + __asm__ volatile("msr DBGWCR13_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[13])); + case 13: + __asm__ volatile("msr DBGWVR12_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[12])); + __asm__ volatile("msr DBGWCR12_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[12])); + case 12: + __asm__ volatile("msr DBGWVR11_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[11])); + __asm__ volatile("msr DBGWCR11_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[11])); + case 11: + __asm__ volatile("msr DBGWVR10_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[10])); + __asm__ volatile("msr DBGWCR10_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[10])); + case 10: + __asm__ volatile("msr DBGWVR9_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[9])); + __asm__ volatile("msr DBGWCR9_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[9])); + case 9: + __asm__ volatile("msr DBGWVR8_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[8])); + __asm__ volatile("msr DBGWCR8_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[8])); + case 8: + __asm__ volatile("msr DBGWVR7_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[7])); + __asm__ volatile("msr DBGWCR7_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[7])); + case 7: + __asm__ volatile("msr DBGWVR6_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[6])); + __asm__ volatile("msr DBGWCR6_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[6])); + case 6: + __asm__ volatile("msr DBGWVR5_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[5])); + __asm__ volatile("msr DBGWCR5_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[5])); + case 5: + __asm__ volatile("msr DBGWVR4_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[4])); + __asm__ volatile("msr DBGWCR4_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[4])); + case 4: + __asm__ volatile("msr DBGWVR3_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[3])); + __asm__ volatile("msr DBGWCR3_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[3])); + case 3: + __asm__ volatile("msr DBGWVR2_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[2])); + __asm__ volatile("msr DBGWCR2_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[2])); + case 2: + __asm__ volatile("msr DBGWVR1_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[1])); + __asm__ volatile("msr DBGWCR1_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[1])); + case 1: + __asm__ volatile("msr DBGWVR0_EL1, %0" : : "r"(debug_state->uds.ds64.wvr[0])); + __asm__ volatile("msr DBGWCR0_EL1, %0" : : "r"(debug_state->uds.ds64.wcr[0])); + default: + break; + } + + for (i = 0; i < debug_info->num_breakpoint_pairs; i++) { + if (0 != debug_state->uds.ds64.bcr[i]) { + set_mde = 1; + break; + } + } + + for (i = 0; i < debug_info->num_watchpoint_pairs; i++) { + if (0 != debug_state->uds.ds64.wcr[i]) { + set_mde = 1; + break; + } + } + + /* + * Breakpoint/Watchpoint Enable + */ + if (set_mde) { + + __asm__ volatile("mrs %0, MDSCR_EL1" : "=r"(state)); + state |= 0x8000; // MDSCR_EL1[MDE] + __asm__ volatile("msr MDSCR_EL1, %0" : : "r"(state)); + + } + + /* + * Software debug single step enable + */ + if (debug_state->uds.ds64.mdscr_el1 & 0x1) { + + __asm__ volatile("mrs %0, MDSCR_EL1" : "=r"(state)); + state = (state & ~0x8000) | 0x1; // ~MDE | SS : no brk/watch while single stepping (which we've set) + __asm__ volatile("msr MDSCR_EL1, %0" : : "r"(state)); + + set_saved_state_cpsr((current_thread()->machine.upcb), + get_saved_state_cpsr((current_thread()->machine.upcb)) | PSR64_SS); + + } else { + + __asm__ volatile("mrs %0, MDSCR_EL1" : "=r"(state)); + state &= ~0x1; + __asm__ volatile("msr MDSCR_EL1, %0" : : "r"(state)); + +#if SINGLE_STEP_RETIRE_ERRATA + // Workaround for radar 20619637 + __builtin_arm_isb(ISB_SY); +#endif + } + + (void) ml_set_interrupts_enabled(intr); + + return; +} + +void arm_debug_set(arm_debug_state_t *debug_state) +{ + if (debug_state) { + switch (debug_state->dsh.flavor) { + case ARM_DEBUG_STATE32: + arm_debug_set32(debug_state); + break; + case ARM_DEBUG_STATE64: + arm_debug_set64(debug_state); + break; + default: + panic("arm_debug_set"); + break; + } + } else { + if (thread_is_64bit(current_thread())) + arm_debug_set64(debug_state); + else + arm_debug_set32(debug_state); + } +} + +#define VM_MAX_ADDRESS32 ((vm_address_t) 0x80000000) +boolean_t +debug_legacy_state_is_valid(arm_legacy_debug_state_t *debug_state) +{ + arm_debug_info_t *debug_info = arm_debug_info(); + uint32_t i; + for (i = 0; i < debug_info->num_breakpoint_pairs; i++) { + if (0 != debug_state->bcr[i] && VM_MAX_ADDRESS32 <= debug_state->bvr[i]) + return FALSE; + } + + for (i = 0; i < debug_info->num_watchpoint_pairs; i++) { + if (0 != debug_state->wcr[i] && VM_MAX_ADDRESS32 <= debug_state->wvr[i]) + return FALSE; + } + return TRUE; +} + +boolean_t +debug_state_is_valid32(arm_debug_state32_t *debug_state) +{ + arm_debug_info_t *debug_info = arm_debug_info(); + uint32_t i; + for (i = 0; i < debug_info->num_breakpoint_pairs; i++) { + if (0 != debug_state->bcr[i] && VM_MAX_ADDRESS32 <= debug_state->bvr[i]) + return FALSE; + } + + for (i = 0; i < debug_info->num_watchpoint_pairs; i++) { + if (0 != debug_state->wcr[i] && VM_MAX_ADDRESS32 <= debug_state->wvr[i]) + return FALSE; + } + return TRUE; +} + +boolean_t +debug_state_is_valid64(arm_debug_state64_t *debug_state) +{ + arm_debug_info_t *debug_info = arm_debug_info(); + uint32_t i; + for (i = 0; i < debug_info->num_breakpoint_pairs; i++) { + if (0 != debug_state->bcr[i] && MACH_VM_MAX_ADDRESS <= debug_state->bvr[i]) + return FALSE; + } + + for (i = 0; i < debug_info->num_watchpoint_pairs; i++) { + if (0 != debug_state->wcr[i] && MACH_VM_MAX_ADDRESS <= debug_state->wvr[i]) + return FALSE; + } + return TRUE; +} + +/* + * Duplicate one arm_debug_state_t to another. "all" parameter + * is ignored in the case of ARM -- Is this the right assumption? + */ +void +copy_legacy_debug_state( + arm_legacy_debug_state_t *src, + arm_legacy_debug_state_t *target, + __unused boolean_t all) +{ + bcopy(src, target, sizeof(arm_legacy_debug_state_t)); +} + +void +copy_debug_state32( + arm_debug_state32_t *src, + arm_debug_state32_t *target, + __unused boolean_t all) +{ + bcopy(src, target, sizeof(arm_debug_state32_t)); +} + +void +copy_debug_state64( + arm_debug_state64_t *src, + arm_debug_state64_t *target, + __unused boolean_t all) +{ + bcopy(src, target, sizeof(arm_debug_state64_t)); +} + +kern_return_t +machine_thread_set_tsd_base( + thread_t thread, + mach_vm_offset_t tsd_base) +{ + + if (thread->task == kernel_task) { + return KERN_INVALID_ARGUMENT; + } + + if (tsd_base & MACHDEP_CPUNUM_MASK) { + return KERN_INVALID_ARGUMENT; + } + + if (thread_is_64bit(thread)) { + if (tsd_base > vm_map_max(thread->map)) + tsd_base = 0ULL; + } else { + if (tsd_base > UINT32_MAX) + tsd_base = 0ULL; + } + + thread->machine.cthread_self = tsd_base; + + /* For current thread, make the TSD base active immediately */ + if (thread == current_thread()) { + uint64_t cpunum, tpidrro_el0; + + mp_disable_preemption(); + tpidrro_el0 = get_tpidrro(); + cpunum = tpidrro_el0 & (MACHDEP_CPUNUM_MASK); + set_tpidrro(tsd_base | cpunum); + mp_enable_preemption(); + + } + + return KERN_SUCCESS; +} diff --git a/osfmk/arm64/pgtrace.c b/osfmk/arm64/pgtrace.c new file mode 100644 index 000000000..8bb15a959 --- /dev/null +++ b/osfmk/arm64/pgtrace.c @@ -0,0 +1,594 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#if CONFIG_PGTRACE +#include +#include +#include +#include +#include +#include + +typedef struct { + queue_chain_t chain; + + pmap_t pmap; + vm_offset_t start; + vm_offset_t end; +} probe_t; + +#if CONFIG_PGTRACE_NONKEXT +#include "pgtrace_decoder.h" + +//-------------------------------------------- +// Macros +// +#define RBUF_DEFAULT_SIZE 1024 +#define RBUF_IDX(idx, mask) ((idx) & (mask)) +#define MSG_MAX 130 + +//-------------------------------------------- +// Types +// +typedef uint8_t RWLOCK; + +typedef struct { + uint64_t id; + pgtrace_run_result_t res; + void *stack[PGTRACE_STACK_DEPTH]; +} log_t; + +//-------------------------------------------- +// Statics +// +static struct { + log_t *logs; // Protect + uint32_t size; // Protect + uint64_t rdidx, wridx; // Protect + decl_simple_lock_data(, loglock); + + uint64_t id; + uint32_t option; + uint32_t enabled; + uint32_t bytes; + + queue_head_t probes; // Protect + + lck_grp_t *lock_grp; + lck_grp_attr_t *lock_grp_attr; + lck_attr_t *lock_attr; + lck_mtx_t probelock; +} pgtrace = {}; + +//-------------------------------------------- +// Globals +// +void pgtrace_init(void) +{ + simple_lock_init(&pgtrace.loglock, 0); + + pgtrace.lock_attr = lck_attr_alloc_init(); + pgtrace.lock_grp_attr = lck_grp_attr_alloc_init(); + pgtrace.lock_grp = lck_grp_alloc_init("pgtrace_lock", pgtrace.lock_grp_attr); + + lck_mtx_init(&pgtrace.probelock, pgtrace.lock_grp, pgtrace.lock_attr); + + queue_init(&pgtrace.probes); + + pgtrace.size = RBUF_DEFAULT_SIZE; + pgtrace.logs = kalloc(RBUF_DEFAULT_SIZE * sizeof(log_t)); +} + +void pgtrace_clear_probe(void) +{ + probe_t *p, *next; + queue_head_t *q = &pgtrace.probes; + + lck_mtx_lock(&pgtrace.probelock); + + p = (probe_t *)queue_first(q); + while (!queue_end(q, (queue_entry_t)p)) { + next = (probe_t *)queue_next(&(p->chain)); + + queue_remove(q, p, probe_t *, chain); + kfree(p, sizeof(probe_t)); + + p = next; + } + + lck_mtx_unlock(&pgtrace.probelock); + + return; +} + +int pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end) +{ + probe_t *p; + queue_head_t *q = &pgtrace.probes; + + if (start > end) { + kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end); + return -1; + } + + p = kalloc(sizeof(probe_t)); + p->start = start; + p->end = end; + if (thread == NULL) { + p->pmap = NULL; + } else { + p->pmap = vm_map_pmap(thread->map); + } + + lck_mtx_lock(&pgtrace.probelock); + queue_enter(q, p, probe_t *, chain); + lck_mtx_unlock(&pgtrace.probelock); + + return 0; +} + +void pgtrace_start(void) +{ + probe_t *p; + queue_head_t *q = &pgtrace.probes; + + kprintf("%s\n", __func__); + + if (pgtrace.enabled) { + return; + } + + pgtrace.enabled = 1; + + lck_mtx_lock(&pgtrace.probelock); + + queue_iterate(q, p, probe_t *, chain) { + pmap_pgtrace_add_page(p->pmap, p->start, p->end); + } + + lck_mtx_unlock(&pgtrace.probelock); + + return; +} + +void pgtrace_stop(void) +{ + probe_t *p; + queue_head_t *q = &pgtrace.probes; + + kprintf("%s\n", __func__); + + lck_mtx_lock(&pgtrace.probelock); + + queue_iterate(q, p, probe_t *, chain) { + pmap_pgtrace_delete_page(p->pmap, p->start, p->end); + } + + lck_mtx_unlock(&pgtrace.probelock); + + pgtrace.enabled = 0; +} + +uint32_t pgtrace_get_size(void) +{ + return pgtrace.size; +} + +bool pgtrace_set_size(uint32_t size) +{ + log_t *old_buf, *new_buf; + uint32_t old_size, new_size = 1; + + // round up to next power of 2 + while (size > new_size) { + new_size <<= 1; + if (new_size > 0x100000) { + // over million entries + kprintf("%s: size=%x new_size=%x is too big\n", __func__, size, new_size); + return false; + } + } + + new_buf = kalloc(new_size * sizeof(log_t)); + if (new_buf == NULL) { + kprintf("%s: can't allocate new_size=%x\n entries", __func__, new_size); + return false; + } + + pgtrace_stop(); + + simple_lock(&pgtrace.loglock); + old_buf = pgtrace.logs; + old_size = pgtrace.size; + pgtrace.logs = new_buf; + pgtrace.size = new_size; + pgtrace.rdidx = pgtrace.wridx = 0; + simple_unlock(&pgtrace.loglock); + + if (old_buf) { + kfree(old_buf, old_size * sizeof(log_t)); + } + + return true; +} + +void pgtrace_clear_trace(void) +{ + simple_lock(&pgtrace.loglock); + pgtrace.rdidx = pgtrace.wridx = 0; + simple_unlock(&pgtrace.loglock); +} + +boolean_t pgtrace_active(void) +{ + return (pgtrace.enabled > 0); +} + +uint32_t pgtrace_get_option(void) +{ + return pgtrace.option; +} + +void pgtrace_set_option(uint32_t option) +{ + pgtrace.option = option; +} + +// pgtrace_write_log() is in interrupt disabled context +void pgtrace_write_log(pgtrace_run_result_t res) +{ + uint8_t i; + log_t log = {}; + const char *rwmap[] = { "R", "W", "PREFETCH" }; + + log.id = pgtrace.id++; + log.res = res; + + if (pgtrace.option & PGTRACE_OPTION_KPRINTF) { + char msg[MSG_MAX]; + char *p; + + p = msg; + + snprintf(p, MSG_MAX, "%llu %s ", res.rr_time, rwmap[res.rr_rw]); + p += strlen(p); + + for (i = 0; i < res.rr_num; i++) { + snprintf(p, MSG_MAX-(p-msg), "%lx=%llx ", res.rr_addrdata[i].ad_addr, res.rr_addrdata[i].ad_data); + p += strlen(p); + } + + kprintf("%s %s\n", __func__, msg); + } + + if (pgtrace.option & PGTRACE_OPTION_STACK) { + OSBacktrace(log.stack, PGTRACE_STACK_DEPTH); + } + + pgtrace.bytes += sizeof(log); + + simple_lock(&pgtrace.loglock); + + pgtrace.logs[RBUF_IDX(pgtrace.wridx, pgtrace.size-1)] = log; + + // Advance rdidx if ring is full + if (RBUF_IDX(pgtrace.wridx, pgtrace.size-1) == RBUF_IDX(pgtrace.rdidx, pgtrace.size-1) && + (pgtrace.wridx != pgtrace.rdidx)) { + pgtrace.rdidx++; + } + pgtrace.wridx++; + + // Signal if ring was empty + if (pgtrace.wridx == (pgtrace.rdidx + 1)) { + thread_wakeup(pgtrace.logs); + } + + simple_unlock(&pgtrace.loglock); + + return; +} + +// pgtrace_read_log() is in user thread +int64_t pgtrace_read_log(uint8_t *buf, uint32_t size) +{ + int total, front, back; + boolean_t ints; + wait_result_t wr; + + if (pgtrace.enabled == FALSE) { + return -EINVAL; + } + + total = size / sizeof(log_t); + + // Check if buf is too small + if (buf && total == 0) { + return -EINVAL; + } + + ints = ml_set_interrupts_enabled(FALSE); + simple_lock(&pgtrace.loglock); + + // Wait if ring is empty + if (pgtrace.rdidx == pgtrace.wridx) { + assert_wait(pgtrace.logs, THREAD_ABORTSAFE); + + simple_unlock(&pgtrace.loglock); + ml_set_interrupts_enabled(ints); + + wr = thread_block(NULL); + if (wr != THREAD_AWAKENED) { + return -EINTR; + } + + ints = ml_set_interrupts_enabled(FALSE); + simple_lock(&pgtrace.loglock); + } + + // Trim the size + if ((pgtrace.rdidx + total) > pgtrace.wridx) { + total = (int)(pgtrace.wridx - pgtrace.rdidx); + } + + // Copy front + if ((RBUF_IDX(pgtrace.rdidx, pgtrace.size-1) + total) >= pgtrace.size) { + front = pgtrace.size - RBUF_IDX(pgtrace.rdidx, pgtrace.size-1); + } else { + front = total; + } + + memcpy(buf, &(pgtrace.logs[RBUF_IDX(pgtrace.rdidx, pgtrace.size-1)]), front*sizeof(log_t)); + + // Copy back if any + back = total-front; + if (back) { + buf += front * sizeof(log_t); + memcpy(buf, pgtrace.logs, back*sizeof(log_t)); + } + + pgtrace.rdidx += total; + + simple_unlock(&pgtrace.loglock); + ml_set_interrupts_enabled(ints); + + return total*sizeof(log_t); +} + +int pgtrace_get_stats(pgtrace_stats_t *stats) +{ + if (!stats) { + return -1; + } + + stats->stat_logger.sl_bytes = pgtrace.bytes; + pgtrace_decoder_get_stats(stats); + + return 0; +} + +#else // CONFIG_PGTRACE_NONKEXT + +static struct { + bool active; + decoder_t *decoder; + logger_t *logger; + queue_head_t probes; + + lck_grp_t *lock_grp; + lck_grp_attr_t *lock_grp_attr; + lck_attr_t *lock_attr; + lck_mtx_t probelock; +} pgtrace = {}; + +//------------------------------------ +// functions for pmap fault handler +// - pgtrace_decode_and_run +// - pgtrace_write_log +//------------------------------------ +int pgtrace_decode_and_run(uint32_t inst, vm_offset_t fva, vm_map_offset_t *cva_page, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + vm_offset_t pa, cva; + pgtrace_instruction_info_t info; + vm_offset_t cva_front_page = cva_page[0]; + vm_offset_t cva_cur_page = cva_page[1]; + + pgtrace.decoder->decode(inst, ss, &info); + + if (info.addr == fva) { + cva = cva_cur_page + (fva & ARM_PGMASK); + } else { + // which means a front page is not a tracing page + cva = cva_front_page + (fva & ARM_PGMASK); + } + + pa = mmu_kvtop(cva); + if (!pa) { + panic("%s: invalid address cva=%lx fva=%lx info.addr=%lx inst=%x", __func__, cva, fva, info.addr, inst); + } + + absolutetime_to_nanoseconds(mach_absolute_time(), &res->rr_time); + + pgtrace.decoder->run(inst, pa, cva, ss, res); + + return 0; +} + +int pgtrace_write_log(pgtrace_run_result_t res) +{ + pgtrace.logger->write(res); + return 0; +} + +//------------------------------------ +// functions for kext +// - pgtrace_init +// - pgtrace_add_probe +// - pgtrace_clear_probe +// - pgtrace_start +// - pgtrace_stop +// - pgtrace_active +//------------------------------------ +int pgtrace_init(decoder_t *decoder, logger_t *logger) +{ + kprintf("%s decoder=%p logger=%p\n", __func__, decoder, logger); + + assert(decoder && logger); + + if (decoder->magic != 0xfeedface || logger->magic != 0xfeedface || + strcmp(decoder->arch, "arm64") != 0 || strcmp(logger->arch, "arm64") != 0) { + kprintf("%s:wrong decoder/logger magic=%llx/%llx arch=%s/%s", __func__, decoder->magic, logger->magic, decoder->arch, logger->arch); + return EINVAL; + } + + pgtrace.lock_attr = lck_attr_alloc_init(); + pgtrace.lock_grp_attr = lck_grp_attr_alloc_init(); + pgtrace.lock_grp = lck_grp_alloc_init("pgtrace_lock", pgtrace.lock_grp_attr); + + lck_mtx_init(&pgtrace.probelock, pgtrace.lock_grp, pgtrace.lock_attr); + + queue_init(&pgtrace.probes); + pgtrace.decoder = decoder; + pgtrace.logger = logger; + + return 0; +} + +int pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end) +{ + probe_t *p; + queue_head_t *q = &pgtrace.probes; + + kprintf("%s start=%lx end=%lx\n", __func__, start, end); + + if (start > end) { + kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end); + return -1; + } + + p = kalloc(sizeof(probe_t)); + p->start = start; + p->end = end; + if (thread == NULL) { + p->pmap = NULL; + } else { + p->pmap = vm_map_pmap(thread->map); + } + + lck_mtx_lock(&pgtrace.probelock); + queue_enter(q, p, probe_t *, chain); + lck_mtx_unlock(&pgtrace.probelock); + + return 0; +} + +void pgtrace_clear_probe(void) +{ + probe_t *p, *next; + queue_head_t *q = &pgtrace.probes; + + kprintf("%s\n", __func__); + + lck_mtx_lock(&pgtrace.probelock); + + p = (probe_t *)queue_first(q); + while (!queue_end(q, (queue_entry_t)p)) { + next = (probe_t *)queue_next(&(p->chain)); + + queue_remove(q, p, probe_t *, chain); + kfree(p, sizeof(probe_t)); + + p = next; + } + + lck_mtx_unlock(&pgtrace.probelock); + + return; +} + +void pgtrace_start(void) +{ + probe_t *p; + queue_head_t *q = &pgtrace.probes; + + kprintf("%s\n", __func__); + + if (pgtrace.active == true) { + return; + } + + pgtrace.active = true; + + lck_mtx_lock(&pgtrace.probelock); + + queue_iterate(q, p, probe_t *, chain) { + pmap_pgtrace_add_page(p->pmap, p->start, p->end); + } + + lck_mtx_unlock(&pgtrace.probelock); + + return; +} + +void pgtrace_stop(void) +{ + probe_t *p; + queue_head_t *q = &pgtrace.probes; + + kprintf("%s\n", __func__); + + lck_mtx_lock(&pgtrace.probelock); + + queue_iterate(q, p, probe_t *, chain) { + pmap_pgtrace_delete_page(p->pmap, p->start, p->end); + } + + lck_mtx_unlock(&pgtrace.probelock); + + pgtrace.active = false; +} + +bool pgtrace_active(void) +{ + return pgtrace.active; +} +#endif // CONFIG_PGTRACE_NONKEXT +#else +// empty funcs for release kernel +extern void pgtrace_stop(void); +extern void pgtrace_start(void); +extern void pgtrace_clear_probe(void); +extern void pgtrace_add_probe(void); +extern void pgtrace_init(void); +extern void pgtrace_active(void); +void pgtrace_stop(void) {} +void pgtrace_start(void) {} +void pgtrace_clear_probe(void) {} +void pgtrace_add_probe(void) {} +void pgtrace_init(void) {} +void pgtrace_active(void) {} +#endif diff --git a/osfmk/arm64/pgtrace.h b/osfmk/arm64/pgtrace.h new file mode 100644 index 000000000..bbee25de9 --- /dev/null +++ b/osfmk/arm64/pgtrace.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#pragma once + +#ifdef CONFIG_PGTRACE +#include +#include +#include + +#define RR_NUM_MAX 2 +#define PGTRACE_STACK_DEPTH 8 + +typedef enum { + PGTRACE_RW_LOAD, + PGTRACE_RW_STORE, + PGTRACE_RW_PREFETCH +} pgtrace_rw_t; + +typedef struct { + vm_offset_t ad_addr; + uint64_t ad_data; +} pgtrace_addr_data_t; + +typedef struct { + uint64_t rr_time; + pgtrace_rw_t rr_rw; + uint8_t rr_num; + pgtrace_addr_data_t rr_addrdata[RR_NUM_MAX]; +} pgtrace_run_result_t; + +#ifdef CONFIG_PGTRACE_NONKEXT +#ifdef XNU_KERNEL_PRIVATE +#define PGTRACE_OPTION_KPRINTF 0x1 +#define PGTRACE_OPTION_STACK 0x2 +#define PGTRACE_OPTION_SPIN 0x4 + +typedef struct { + struct { + uint32_t sl_bytes; + } stat_logger; + + struct { + uint64_t sd_ldr; + uint64_t sd_str; + uint64_t sd_ldrs; + uint64_t sd_ldtr; + uint64_t sd_sttr; + uint64_t sd_ldtrs; + uint64_t sd_ldp; + uint64_t sd_stp; + uint64_t sd_ldpsw; + uint64_t sd_prfm; + + uint64_t sd_c335; + uint64_t sd_c336; + uint64_t sd_c337; + uint64_t sd_c338; + uint64_t sd_c339; + uint64_t sd_c3310; + uint64_t sd_c3311; + uint64_t sd_c3312; + uint64_t sd_c3313; + uint64_t sd_c3314; + uint64_t sd_c3315; + uint64_t sd_c3316; + } stat_decoder; +} pgtrace_stats_t; + +void pgtrace_init(void); +int pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end); +void pgtrace_clear_probe(void); +void pgtrace_start(void); +void pgtrace_stop(void); +uint32_t pgtrace_get_size(void); +bool pgtrace_set_size(uint32_t); +void pgtrace_clear_trace(void); +boolean_t pgtrace_active(void); +uint32_t pgtrace_get_option(void); +void pgtrace_set_option(uint32_t option); +int64_t pgtrace_read_log(uint8_t *buf, uint32_t size); +void pgtrace_write_log(pgtrace_run_result_t res); +int pgtrace_get_stats(pgtrace_stats_t *stats); +#endif +#else // CONFIG_PGTRACE_NONKEXT +#ifdef __cplusplus +extern "C" { +#endif +typedef struct { + vm_offset_t addr; + uint64_t bytes; +} pgtrace_instruction_info_t; + +typedef struct { + uint64_t id; + pgtrace_run_result_t res; + void *stack[PGTRACE_STACK_DEPTH]; +} log_t; + +typedef int (*run_func_t)(uint32_t inst, vm_offset_t pa, vm_offset_t va, void *ss, pgtrace_run_result_t *res); +typedef bool (*decode_func_t)(uint32_t inst, void *ss, pgtrace_instruction_info_t *info); +typedef void (*write_func_t)(pgtrace_run_result_t res); + +typedef struct { + uint64_t magic; + char *arch; + char *desc; + decode_func_t decode; + run_func_t run; +} decoder_t; + +typedef struct { + uint64_t magic; + char *arch; + char *desc; + write_func_t write; +} logger_t; + +//------------------------------------ +// for pmap fault handler +//------------------------------------ +int pgtrace_decode_and_run(uint32_t inst, vm_offset_t fva, vm_map_offset_t *cva_page, arm_saved_state_t *ss, pgtrace_run_result_t *res); +int pgtrace_write_log(pgtrace_run_result_t res); + +//------------------------------------ +// for kext +//------------------------------------ +int pgtrace_init(decoder_t *decoder, logger_t *logger); +int pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end); +void pgtrace_clear_probe(void); +void pgtrace_start(void); +void pgtrace_stop(void); +bool pgtrace_active(void); +#ifdef __cplusplus +} +#endif +#endif // CONFIG_PGTRACE_NONKEXT +#endif diff --git a/osfmk/arm64/pgtrace_decoder.c b/osfmk/arm64/pgtrace_decoder.c new file mode 100644 index 000000000..98471cbd6 --- /dev/null +++ b/osfmk/arm64/pgtrace_decoder.c @@ -0,0 +1,1551 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#if CONFIG_PGTRACE +#include +#include +#include +#include +#include "pgtrace_decoder.h" + +//------------------------------------------------------------------- +// Macros +// +#define DBG 1 +#if DBG == 1 +#define INLINE __attribute__((noinline)) +#else +#define INLINE inline +#endif + +#define BITS(v, msb, lsb) ((v) << (31-msb) >> (31-msb) >> (lsb)) +#define READ_GPR_X(ss, n, v) { \ + if (__builtin_expect(n < 31, 1)) (v) = (ss)->ss_64.x[(n)]; \ + else if (n == 31) (v) = 0; \ + else { panic("Invalid GPR x%d", n); __builtin_unreachable(); } \ +} +#define READ_GPR_W(ss, n, v) { \ + if (__builtin_expect(n < 31, 1)) (v) = *(uint32_t*)&((ss)->ss_64.x[(n)]); \ + else if (n == 31) (v) = 0; \ + else { panic("Invalid GPR w%d", n); __builtin_unreachable(); } \ +} +#define WRITE_GPR_X(ss, n, v) { \ + if (__builtin_expect(n < 31, 1)) (ss)->ss_64.x[(n)] = (v); \ + else if (n == 31) {} \ + else { panic("Invalid GPR x%d", n); __builtin_unreachable(); } \ +} +#define WRITE_GPR_W(ss, n, v) { \ + if (__builtin_expect(n < 31, 1)) *(uint32_t*)&((ss)->ss_64.x[(n)]) = (v); \ + else if (n == 31) {} \ + else { panic("Invalid GPR w%d", n); __builtin_unreachable(); } \ +} +#define SIGN_EXTEND_64(val, width) (((int64_t)(val) << (64 - (width)) >> (64 - (width)))) +#define ZERO_EXTEND_64(val, width) (((uint64_t)(val) << (64 - (width))) >> (64 - (width))) + +//------------------------------------------------------------------- +// Types +// +typedef int (*run_t)(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); + +typedef struct { + vm_offset_t addr; + uint64_t bytes; +} instruction_info_t; + +typedef bool (*get_info_t)(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); + +typedef struct { + uint32_t mask; + uint32_t value; + run_t run; + get_info_t get_info; +} type_entry_t; + +//------------------------------------------------------------------- +// Statics +// +static int run_simd(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c335(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c336(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c337(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c338(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c339(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c3310(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c3311(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c3312(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c3313(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c3314(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c3315(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static int run_c3316(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res); +static bool get_info_simd(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c335(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c336(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c337(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c338(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c339(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c3310(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c3311(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c3312(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c3313(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c3314(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c3315(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); +static bool get_info_c3316(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info); + +// Table from ARM DDI 0487A.a C3.3 +static type_entry_t typetbl[] = { + { 0x3f000000, 0x08000000, run_c336, get_info_c336 }, // Load/store exclusive + { 0x3b000000, 0x18000000, run_c335, get_info_c335 }, // Load register (literal) + { 0x3b800000, 0x28000000, run_c337, get_info_c337 }, // Load/store no-allocate pair (offset) + { 0x3b800000, 0x28800000, run_c3315, get_info_c3315 }, // Load/store register pair (post-indexed) + { 0x3b800000, 0x29000000, run_c3314, get_info_c3314 }, // Load/store register pair (offset) + { 0x3b800000, 0x29800000, run_c3316, get_info_c3316 }, // Load/store register pair (pre-indexed) + { 0x3b200c00, 0x38000000, run_c3312, get_info_c3312 }, // Load/store register (unscaled immediate) + { 0x3b200c00, 0x38000400, run_c338, get_info_c338 }, // Load/store register (immediate post-indexed) + { 0x3b200c00, 0x38000800, run_c3311, get_info_c3311 }, // Load/store register (unprivileged) + { 0x3b200c00, 0x38000c00, run_c339, get_info_c339 }, // Load/store register (immediate pre-indexed) + { 0x3b200c00, 0x38200800, run_c3310, get_info_c3310 }, // Load/store register (register offset) + { 0x3b000000, 0x39000000, run_c3313, get_info_c3313 }, // Load/store register (unsigned immediate) + + { 0xbfbf0000, 0x0c000000, run_simd, get_info_simd }, // AdvSIMD load/store multiple structures + { 0xbfa00000, 0x0c800000, run_simd, get_info_simd }, // AdvSIMD load/store multiple structures (post-indexed) + { 0xbf980000, 0x0d000000, run_simd, get_info_simd }, // AdvSIMD load/store single structure + { 0xbf800000, 0x0d800000, run_simd, get_info_simd } // AdvSIMD load/store single structure (post-indexed) +}; + +static pgtrace_stats_t stats; + +INLINE static void do_str(uint8_t size, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t wt; + uint64_t xt; + + res->rr_rw = PGTRACE_RW_STORE; + + if (size == 8) { + READ_GPR_X(ss, Rt, xt); + res->rr_addrdata[0].ad_data = xt; + } else { + READ_GPR_W(ss, Rt, wt); + res->rr_addrdata[0].ad_data = wt; + } + + if (size == 1) __asm__ volatile("strb %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va)); + else if (size == 2) __asm__ volatile("strh %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va)); + else if (size == 4) __asm__ volatile("str %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va)); + else if (size == 8) __asm__ volatile("str %x[xt], [%[va]]\n" :: [xt] "r"(xt), [va] "r"(va)); + else panic("%s Invalid size %d\n", __func__, size); + + stats.stat_decoder.sd_str++; +} + +INLINE static void do_ldr(uint8_t size, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t wt; + uint64_t xt; + + res->rr_rw = PGTRACE_RW_LOAD; + + if (size == 1) __asm__ volatile("ldrb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + else if (size == 2) __asm__ volatile("ldrh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + else if (size == 4) __asm__ volatile("ldr %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + else if (size == 8) __asm__ volatile("ldr %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + else panic("%s Invalid size %d\n", __func__, size); + + if (size == 8) { + WRITE_GPR_X(ss, Rt, xt); + res->rr_addrdata[0].ad_data = xt; + } else { + WRITE_GPR_W(ss, Rt, wt); + res->rr_addrdata[0].ad_data = wt; + } + + stats.stat_decoder.sd_ldr++; +} + +INLINE static void do_stp(uint8_t size, uint8_t Rt, uint8_t Rt2, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t wt1, wt2; + uint64_t xt1, xt2; + + if (size == 4) { + READ_GPR_W(ss, Rt, wt1); + READ_GPR_W(ss, Rt2, wt2); + __asm__ volatile("stp %w[wt1], %w[wt2], [%[va]]\n" :: [wt1] "r"(wt1), [wt2] "r"(wt2), [va] "r"(va)); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[1].ad_addr = va+sizeof(wt1); + res->rr_addrdata[0].ad_data = wt1; + res->rr_addrdata[1].ad_data = wt2; + } else if (size == 8) { + READ_GPR_X(ss, Rt, xt1); + READ_GPR_X(ss, Rt2, xt2); + __asm__ volatile("stp %x[xt1], %x[xt2], [%[va]]\n" :: [xt1] "r"(xt1), [xt2] "r"(xt2), [va] "r"(va)); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[1].ad_addr = va+sizeof(xt1); + res->rr_addrdata[0].ad_data = xt1; + res->rr_addrdata[1].ad_data = xt2; + } else panic("%s Invalid size %d\n", __func__, size); + + stats.stat_decoder.sd_stp++; +} + +INLINE static void do_ldp(uint8_t size, uint8_t Rt, uint8_t Rt2, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t wt1, wt2; + uint64_t xt1, xt2; + + if (size == 4) { + __asm__ volatile("ldp %w[wt1], %w[wt2], [%[va]]\n" : [wt1] "=r"(wt1), [wt2] "=r"(wt2) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt1); + WRITE_GPR_W(ss, Rt2, wt2); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[1].ad_addr = va+sizeof(wt1); + res->rr_addrdata[0].ad_data = wt1; + res->rr_addrdata[1].ad_data = wt2; + } else if (size == 8) { + __asm__ volatile("ldp %x[xt1], %x[xt2], [%[va]]\n" : [xt1] "=r"(xt1), [xt2] "=r"(xt2) : [va] "r"(va)); + WRITE_GPR_X(ss, Rt, xt1); + WRITE_GPR_X(ss, Rt2, xt2); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[1].ad_addr = va+sizeof(xt1); + res->rr_addrdata[0].ad_data = xt1; + res->rr_addrdata[1].ad_data = xt2; + } else panic("%s Invalid size %d\n", __func__, size); + + stats.stat_decoder.sd_ldp++; +} + +INLINE static void do_ldpsw(uint8_t Rt, uint8_t Rt2, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint64_t xt1, xt2; + + __asm__ volatile("ldpsw %x[xt1], %x[xt2], [%[va]]\n" : [xt1] "=r"(xt1), [xt2] "=r"(xt2) : [va] "r"(va)); + WRITE_GPR_X(ss, Rt, xt1); + WRITE_GPR_X(ss, Rt2, xt2); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[1].ad_addr = va+sizeof(uint32_t); + res->rr_addrdata[0].ad_data = xt1; + res->rr_addrdata[1].ad_data = xt2; + + stats.stat_decoder.sd_ldpsw++; +} + +INLINE static void do_ldrs(uint8_t size, uint8_t extsize, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t wt; + uint64_t xt; + + res->rr_rw = PGTRACE_RW_LOAD; + + if (size == 1 && extsize == 4) __asm__ volatile("ldrsb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + else if (size == 1 && extsize == 8) __asm__ volatile("ldrsb %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + else if (size == 2 && extsize == 4) __asm__ volatile("ldrsh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + else if (size == 2 && extsize == 8) __asm__ volatile("ldrsh %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + else if (size == 4 && extsize == 8) __asm__ volatile("ldrsw %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + else panic("%s Invalid size %d extsize=%d\n", __func__, size, extsize); + + if (extsize == 8) { + WRITE_GPR_X(ss, Rt, xt); + res->rr_addrdata[0].ad_data = xt; + } else { + WRITE_GPR_W(ss, Rt, wt); + res->rr_addrdata[0].ad_data = wt; + } + + stats.stat_decoder.sd_ldrs++; +} + +INLINE static void do_ldtrs(uint8_t size, uint8_t extsize, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t wt; + uint64_t xt; + + res->rr_rw = PGTRACE_RW_LOAD; + + if (size == 1 && extsize == 4) __asm__ volatile("ldtrsb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + else if (size == 1 && extsize == 8) __asm__ volatile("ldtrsb %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + else if (size == 2 && extsize == 4) __asm__ volatile("ldtrsh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + else if (size == 2 && extsize == 8) __asm__ volatile("ldtrsh %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + else if (size == 4 && extsize == 8) __asm__ volatile("ldtrsw %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + else panic("%s Invalid size %d extsize=%d\n", __func__, size, extsize); + + if (extsize == 8) { + WRITE_GPR_X(ss, Rt, xt); + res->rr_addrdata[0].ad_data = xt; + } else { + WRITE_GPR_W(ss, Rt, wt); + res->rr_addrdata[0].ad_data = wt; + } + + stats.stat_decoder.sd_ldtrs++; +} + +INLINE static void do_ldtr(uint8_t size, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t wt; + uint64_t xt; + + res->rr_rw = PGTRACE_RW_LOAD; + + if (size == 1) __asm__ volatile("ldtrb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + else if (size == 2) __asm__ volatile("ldtrh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + else if (size == 4) __asm__ volatile("ldtr %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + else if (size == 8) __asm__ volatile("ldtr %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + else panic("%s Invalid size %d\n", __func__, size); + + if (size == 8) { + WRITE_GPR_X(ss, Rt, xt); + res->rr_addrdata[0].ad_data = xt; + } else { + WRITE_GPR_W(ss, Rt, wt); + res->rr_addrdata[0].ad_data = wt; + } + + stats.stat_decoder.sd_ldtr++; +} + +INLINE static void do_sttr(uint8_t size, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t wt; + uint64_t xt; + + res->rr_rw = PGTRACE_RW_STORE; + + if (size == 8) { + READ_GPR_X(ss, Rt, xt); + res->rr_addrdata[0].ad_data = xt; + } else { + READ_GPR_W(ss, Rt, wt); + res->rr_addrdata[0].ad_data = wt; + } + + if (size == 1) __asm__ volatile("sttrb %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va)); + else if (size == 2) __asm__ volatile("sttrh %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va)); + else if (size == 4) __asm__ volatile("sttr %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va)); + else if (size == 8) __asm__ volatile("sttr %x[xt], [%[va]]\n" :: [xt] "r"(xt), [va] "r"(va)); + else panic("%s Invalid size %d\n", __func__, size); + + stats.stat_decoder.sd_sttr++; +} + +INLINE static void do_prfm(uint8_t Rt, vm_offset_t va, pgtrace_run_result_t *res) +{ + if (Rt == 0) __asm__ volatile("prfm pldl1keep, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 1) __asm__ volatile("prfm pldl1strm, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 2) __asm__ volatile("prfm pldl2keep, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 3) __asm__ volatile("prfm pldl2strm, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 4) __asm__ volatile("prfm pldl3keep, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 5) __asm__ volatile("prfm pldl3strm, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 6) __asm__ volatile("prfm #6, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 7) __asm__ volatile("prfm #7, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 8) __asm__ volatile("prfm #8, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 9) __asm__ volatile("prfm #9, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 10) __asm__ volatile("prfm #10, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 11) __asm__ volatile("prfm #11, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 12) __asm__ volatile("prfm #12, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 13) __asm__ volatile("prfm #13, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 14) __asm__ volatile("prfm #14, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 15) __asm__ volatile("prfm #15, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 16) __asm__ volatile("prfm pstl1keep, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 17) __asm__ volatile("prfm pstl1strm, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 18) __asm__ volatile("prfm pstl2keep, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 19) __asm__ volatile("prfm pstl2strm, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 20) __asm__ volatile("prfm pstl3keep, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 21) __asm__ volatile("prfm pstl3strm, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 22) __asm__ volatile("prfm #22, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 23) __asm__ volatile("prfm #23, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 24) __asm__ volatile("prfm #24, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 25) __asm__ volatile("prfm #25, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 26) __asm__ volatile("prfm #26, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 27) __asm__ volatile("prfm #27, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 28) __asm__ volatile("prfm #28, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 29) __asm__ volatile("prfm #29, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 30) __asm__ volatile("prfm #30, [%[va]]\n" : : [va] "r"(va)); + else if (Rt == 31) __asm__ volatile("prfm #31, [%[va]]\n" : : [va] "r"(va)); + else panic("%s Invalid Rt %d\n", __func__, Rt); + + res->rr_num = 0; + res->rr_rw = PGTRACE_RW_PREFETCH; + + stats.stat_decoder.sd_prfm++; +} + +#define CANNOTDECODE(msg, inst) do {\ + panic("%s: " msg " inst=%x not supported yet\n", __func__, inst);\ +} while (0) + +static int run_simd(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ +#pragma unused(pa,va,ss,res) + CANNOTDECODE("simd", inst); + return 0; +} + +static int run_c335(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t opc = BITS(inst, 31, 30), + v = BITS(inst, 26, 26), + Rt = BITS(inst, 4, 0); + uint8_t fields = (opc << 1) | v; + + res->rr_num = 1; + res->rr_addrdata[0].ad_addr = pa; + + if (fields == 0) do_ldr(4, Rt, va, ss, res); + else if ((fields == 1) || + (fields == 3) || + (fields == 5)) CANNOTDECODE("simd", inst); + else if (fields == 2) do_ldr(8, Rt, va, ss, res); + else if (fields == 4) do_ldrs(4, 8, Rt, va, ss, res); + else if (fields == 6) do_prfm(Rt, va, res); + else CANNOTDECODE("unknown", inst); + + stats.stat_decoder.sd_c335++; + + return 0; +} + +static int run_c336(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t ws, wt, wt1, wt2; + uint64_t xt, xt1, xt2; + uint32_t size = BITS(inst, 31, 30), + o2 = BITS(inst, 23, 23), + L = BITS(inst, 22, 22), + o1 = BITS(inst, 21, 21), + Rs = BITS(inst, 20, 16), + o0 = BITS(inst, 15, 15), + Rt2 = BITS(inst, 14, 10), + Rt = BITS(inst, 4, 0); + uint8_t fields = (size << 4) | (o2 << 3) | (L << 2) | (o1 << 1) | o0; + + kprintf("%s Load/store exclusive on device memory???n", __func__); + + res->rr_num = 1; + res->rr_addrdata[0].ad_addr = pa; + + switch (fields) { + case 0: + READ_GPR_W(ss, Rt, wt); + __asm__ volatile("stxrb %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = wt; + break; + case 1: + READ_GPR_W(ss, Rt, wt); + __asm__ volatile("stlxrb %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = wt; + break; + case 4: + __asm__ volatile("ldxrb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = wt; + break; + case 5: + __asm__ volatile("ldaxrb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = wt; + break; + case 9: + READ_GPR_W(ss, Rt, wt); + __asm__ volatile("stlrb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = wt; + break; + case 0xd: + __asm__ volatile("ldarb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x10: + READ_GPR_W(ss, Rt, wt); + __asm__ volatile("stxrh %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x11: + READ_GPR_W(ss, Rt, wt); + __asm__ volatile("stlxrh %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x14: + __asm__ volatile("ldxrh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x15: + __asm__ volatile("ldaxrh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x19: + READ_GPR_W(ss, Rt, wt); + __asm__ volatile("stlrh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x1d: + __asm__ volatile("ldarh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x20: + READ_GPR_W(ss, Rt, wt); + __asm__ volatile("stxr %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x21: + READ_GPR_W(ss, Rt, wt); + __asm__ volatile("stlxr %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x22: + READ_GPR_W(ss, Rt, wt1); + READ_GPR_W(ss, Rt2, wt2); + __asm__ volatile("stxp %w[ws], %w[wt1], %w[wt2], [%[va]]\n" : [ws] "=r"(ws) : [wt1] "r"(wt1), [wt2] "r"(wt2), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = va; + res->rr_addrdata[1].ad_addr = va+sizeof(wt1); + res->rr_addrdata[0].ad_data = wt1; + res->rr_addrdata[1].ad_data = wt2; + break; + case 0x23: + READ_GPR_W(ss, Rt, wt1); + READ_GPR_W(ss, Rt2, wt2); + __asm__ volatile("stlxp %w[ws], %w[wt1], %w[wt2], [%[va]]\n" : [ws] "=r"(ws) : [wt1] "r"(wt1), [wt2] "r"(wt2), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = va; + res->rr_addrdata[1].ad_addr = va+sizeof(wt1); + res->rr_addrdata[0].ad_data = wt1; + res->rr_addrdata[1].ad_data = wt2; + break; + case 0x24: + __asm__ volatile("ldxr %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x25: + __asm__ volatile("ldaxr %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x26: + __asm__ volatile("ldxp %w[wt1], %w[wt2], [%[va]]\n" : [wt1] "=r"(wt1), [wt2] "=r"(wt2) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt1); + WRITE_GPR_W(ss, Rt2, wt2); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = va; + res->rr_addrdata[1].ad_addr = va+sizeof(wt1); + res->rr_addrdata[0].ad_data = wt1; + res->rr_addrdata[1].ad_data = wt2; + break; + case 0x27: + __asm__ volatile("ldaxp %w[wt1], %w[wt2], [%[va]]\n" : [wt1] "=r"(wt1), [wt2] "=r"(wt2) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt1); + WRITE_GPR_W(ss, Rt2, wt2); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = va; + res->rr_addrdata[1].ad_addr = va+sizeof(wt1); + res->rr_addrdata[0].ad_data = wt1; + res->rr_addrdata[1].ad_data = wt2; + break; + case 0x29: + READ_GPR_W(ss, Rt, wt); + __asm__ volatile("stlr %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x2d: + __asm__ volatile("ldar %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = wt; + break; + case 0x30: + READ_GPR_X(ss, Rt, xt); + __asm__ volatile("stxr %w[ws], %[xt], [%[va]]\n" : [ws] "=r"(ws) : [xt] "r"(xt), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = xt; + break; + case 0x31: + READ_GPR_X(ss, Rt, xt); + __asm__ volatile("stlxr %w[ws], %[xt], [%[va]]\n" : [ws] "=r"(ws) : [xt] "r"(xt), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = xt; + break; + case 0x32: + READ_GPR_X(ss, Rt, xt1); + READ_GPR_X(ss, Rt2, xt2); + __asm__ volatile("stxp %w[ws], %[xt1], %[xt2], [%[va]]\n" : [ws] "=r"(ws) : [xt1] "r"(xt1), [xt2] "r"(xt2), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = va; + res->rr_addrdata[1].ad_addr = va+sizeof(xt1); + res->rr_addrdata[0].ad_data = xt1; + res->rr_addrdata[1].ad_data = xt2; + break; + case 0x33: + READ_GPR_X(ss, Rt, xt1); + READ_GPR_X(ss, Rt2, xt2); + __asm__ volatile("stlxp %w[ws], %[xt1], %[xt2], [%[va]]\n" : [ws] "=r"(ws) : [xt1] "r"(xt1), [xt2] "r"(xt2), [va] "r"(va)); + WRITE_GPR_W(ss, Rs, ws); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = va; + res->rr_addrdata[1].ad_addr = va+sizeof(xt1); + res->rr_addrdata[0].ad_data = xt1; + res->rr_addrdata[1].ad_data = xt2; + break; + case 0x34: + __asm__ volatile("ldxr %[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + WRITE_GPR_X(ss, Rt, xt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = xt; + break; + case 0x35: + __asm__ volatile("ldaxr %[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + WRITE_GPR_X(ss, Rt, xt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = xt; + break; + case 0x36: + __asm__ volatile("ldxp %[xt1], %[xt2], [%[va]]\n" : [xt1] "=r"(xt1), [xt2] "=r"(xt2) : [va] "r"(va)); + WRITE_GPR_X(ss, Rt, xt1); + WRITE_GPR_X(ss, Rt2, xt2); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = va; + res->rr_addrdata[1].ad_addr = va+sizeof(xt1); + res->rr_addrdata[0].ad_data = xt1; + res->rr_addrdata[0].ad_data = xt2; + break; + case 0x37: + __asm__ volatile("ldaxp %[xt1], %[xt2], [%[va]]\n" : [xt1] "=r"(xt1), [xt2] "=r"(xt2) : [va] "r"(va)); + WRITE_GPR_X(ss, Rt, xt1); + WRITE_GPR_X(ss, Rt2, xt2); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = va; + res->rr_addrdata[1].ad_addr = va+sizeof(xt1); + res->rr_addrdata[0].ad_data = xt1; + res->rr_addrdata[0].ad_data = xt2; + break; + case 0x39: + READ_GPR_X(ss, Rt, xt); + __asm__ volatile("stlr %[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_addrdata[0].ad_data = xt; + break; + case 0x3d: + __asm__ volatile("ldar %[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va)); + WRITE_GPR_X(ss, Rt, xt); + res->rr_rw = PGTRACE_RW_LOAD; + res->rr_addrdata[0].ad_data = xt; + break; + default: + CANNOTDECODE("unknown", inst); + } + + stats.stat_decoder.sd_c336++; + + return 0; +} + +static int run_c337(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t wt1, wt2; + uint64_t xt1, xt2; + uint32_t opc = BITS(inst, 31, 30), + V = BITS(inst, 26, 26), + L = BITS(inst, 22, 22), + Rt = BITS(inst, 4, 0), + Rt2 = BITS(inst, 14, 10); + uint8_t fields = (opc << 2) | (V << 1) | L; + + switch (fields) { + case 0: + READ_GPR_W(ss, Rt, wt1); + READ_GPR_W(ss, Rt2, wt2); + __asm__ volatile("stnp %w[wt1], %w[wt2], [%[va]]\n" :: [wt1] "r"(wt1), [wt2] "r"(wt2), [va] "r"(va)); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = pa; + res->rr_addrdata[1].ad_addr = pa+sizeof(wt1); + res->rr_addrdata[0].ad_data = wt1; + res->rr_addrdata[1].ad_data = wt2; + break; + case 1: + __asm__ volatile("ldnp %w[wt1], %w[wt2], [%[va]]\n" : [wt1] "=r"(wt1), [wt2] "=r"(wt2) : [va] "r"(va)); + WRITE_GPR_W(ss, Rt, wt1); + WRITE_GPR_W(ss, Rt2, wt2); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = pa; + res->rr_addrdata[1].ad_addr = pa+sizeof(wt1); + res->rr_addrdata[0].ad_data = wt1; + res->rr_addrdata[1].ad_data = wt2; + break; + case 2: + case 3: + case 6: + case 7: + case 10: + case 11: + CANNOTDECODE("simd", inst); + case 8: + READ_GPR_X(ss, Rt, xt1); + READ_GPR_X(ss, Rt2, xt2); + __asm__ volatile("stnp %x[xt1], %x[xt2], [%[va]]\n" :: [xt1] "r"(xt1), [xt2] "r"(xt2), [va] "r"(va)); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = pa; + res->rr_addrdata[1].ad_addr = pa+sizeof(xt1); + res->rr_addrdata[0].ad_data = xt1; + res->rr_addrdata[1].ad_data = xt2; + break; + case 9: + __asm__ volatile("ldnp %x[xt1], %x[xt2], [%[va]]\n" : [xt1] "=r"(xt1), [xt2] "=r"(xt2) : [va] "r"(va)); + WRITE_GPR_X(ss, Rt, xt1); + WRITE_GPR_X(ss, Rt2, xt2); + res->rr_rw = PGTRACE_RW_STORE; + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = pa; + res->rr_addrdata[1].ad_addr = pa+sizeof(xt1); + res->rr_addrdata[0].ad_data = xt1; + res->rr_addrdata[1].ad_data = xt2; + break; + default: + CANNOTDECODE("simd", inst); + } + + stats.stat_decoder.sd_c337++; + + return 0; +} + +static int run_c338(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t size = BITS(inst, 31, 30), + V = BITS(inst, 26, 26), + opc = BITS(inst, 23, 22), + Rt = BITS(inst, 4, 0); + uint8_t fields = (size << 3) | (V << 2) | opc; + + res->rr_num = 1; + res->rr_addrdata[0].ad_addr = pa; + + if (fields == 0) do_str(1, Rt, va, ss, res); + else if (fields == 1) do_ldr(1, Rt, va, ss, res); + else if (fields == 2) do_ldrs(1, 8, Rt, va, ss, res); + else if (fields == 3) do_ldrs(1, 4, Rt, va, ss, res); + else if ((fields == 4) || + (fields == 5) || + (fields == 6) || + (fields == 7) || + (fields == 12) || + (fields == 13) || + (fields == 0x14) || + (fields == 0x15) || + (fields == 0x1c) || + (fields == 0x1d)) CANNOTDECODE("simd", inst); + else if (fields == 8) do_str(2, Rt, va, ss, res); + else if (fields == 9) do_ldr(2, Rt, va, ss, res); + else if (fields == 10) do_ldrs(2, 8, Rt, va, ss, res); + else if (fields == 11) do_ldrs(2, 4, Rt, va, ss, res); + else if (fields == 0x10) do_str(4, Rt, va, ss, res); + else if (fields == 0x11) do_ldr(4, Rt, va, ss, res); + else if (fields == 0x12) do_ldrs(4, 8, Rt, va, ss, res); + else if (fields == 0x18) do_str(8, Rt, va, ss, res); + else if (fields == 0x19) do_ldr(8, Rt, va, ss, res); + else CANNOTDECODE("unknown", inst); + + stats.stat_decoder.sd_c338++; + + return 0; +} + +static int run_c339(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t size = BITS(inst, 31, 30), + V = BITS(inst, 26, 26), + opc = BITS(inst, 23, 22), + Rt = BITS(inst, 4, 0); + uint8_t fields = (size << 3) | (V << 2) | opc; + + res->rr_num = 1; + res->rr_addrdata[0].ad_addr = pa; + + if (fields == 0) do_str(1, Rt, va, ss, res); + else if (fields == 1) do_ldr(1, Rt, va, ss, res); + else if (fields == 2) do_ldrs(1, 8, Rt, va, ss, res); + else if (fields == 3) do_ldrs(1, 4, Rt, va, ss, res); + else if ((fields == 4) || + (fields == 5) || + (fields == 6) || + (fields == 7) || + (fields == 12) || + (fields == 13) || + (fields == 0x14) || + (fields == 0x15) || + (fields == 0x1c) || + (fields == 0x1d)) CANNOTDECODE("simd", inst); + else if (fields == 8) do_str(2, Rt, va, ss, res); + else if (fields == 9) do_ldr(2, Rt, va, ss, res); + else if (fields == 10) do_ldrs(2, 8, Rt, va, ss, res); + else if (fields == 11) do_ldrs(2, 4, Rt, va, ss, res); + else if (fields == 0x10) do_str(4, Rt, va, ss, res); + else if (fields == 0x11) do_ldr(4, Rt, va, ss, res); + else if (fields == 0x12) do_ldrs(4, 8, Rt, va, ss, res); + else if (fields == 0x18) do_str(8, Rt, va, ss, res); + else if (fields == 0x19) do_ldr(8, Rt, va, ss, res); + else CANNOTDECODE("unknown", inst); + + stats.stat_decoder.sd_c339++; + + return 0; +} + +static int run_c3310(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t size = BITS(inst, 31, 30), + V = BITS(inst, 26, 26), + opc = BITS(inst, 23, 22), + Rt = BITS(inst, 4, 0); + uint8_t fields = (size << 3) | (V << 2) | opc; + + res->rr_num = 1; + res->rr_addrdata[0].ad_addr = pa; + + if (fields == 0) do_str(1, Rt, va, ss, res); + else if (fields == 1) do_ldr(1, Rt, va, ss, res); + else if (fields == 2) do_ldrs(1, 8, Rt, va, ss, res); + else if (fields == 3) do_ldrs(1, 4, Rt, va, ss, res); + else if ((fields == 4) || + (fields == 5) || + (fields == 6) || + (fields == 7) || + (fields == 12) || + (fields == 13) || + (fields == 0x14) || + (fields == 0x15) || + (fields == 0x1c) || + (fields == 0x1d)) CANNOTDECODE("simd", inst); + else if (fields == 8) do_str(2, Rt, va, ss, res); + else if (fields == 9) do_ldr(2, Rt, va, ss, res); + else if (fields == 10) do_ldrs(2, 8, Rt, va, ss, res); + else if (fields == 11) do_ldrs(2, 4, Rt, va, ss, res); + else if (fields == 0x10) do_str(4, Rt, va, ss, res); + else if (fields == 0x11) do_ldr(4, Rt, va, ss, res); + else if (fields == 0x12) do_ldrs(4, 8, Rt, va, ss, res); + else if (fields == 0x18) do_str(8, Rt, va, ss, res); + else if (fields == 0x19) do_ldr(8, Rt, va, ss, res); + else if (fields == 0x1a) do_prfm(Rt, va, res); + else CANNOTDECODE("unknown", inst); + + stats.stat_decoder.sd_c3310++; + + return 0; +} + +static int run_c3311(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t size = BITS(inst, 31, 30), + V = BITS(inst, 26, 26), + opc = BITS(inst, 23, 22), + Rt = BITS(inst, 4, 0); + uint8_t fields = (size << 3) | (V << 2) | opc; + + res->rr_num = 1; + res->rr_addrdata[0].ad_addr = pa; + + if (fields == 0) do_sttr(1, Rt, va, ss, res); + else if (fields == 1) do_ldtr(1, Rt, va, ss, res); + else if (fields == 2) do_ldtrs(1, 8, Rt, va, ss, res); + else if (fields == 3) do_ldtrs(1, 4, Rt, va, ss, res); + else if (fields == 8) do_sttr(2, Rt, va, ss, res); + else if (fields == 9) do_ldtr(2, Rt, va, ss, res); + else if (fields == 10) do_ldtrs(2, 8, Rt, va, ss, res); + else if (fields == 11) do_ldtrs(2, 4, Rt, va, ss, res); + else if (fields == 0x10) do_sttr(4, Rt, va, ss, res); + else if (fields == 0x11) do_ldtr(4, Rt, va, ss, res); + else if (fields == 0x12) do_ldtrs(4, 8, Rt, va, ss, res); + else if (fields == 0x18) do_sttr(8, Rt, va, ss, res); + else if (fields == 0x19) do_ldtr(8, Rt, va, ss, res); + else CANNOTDECODE("unknown", inst); + + stats.stat_decoder.sd_c3311++; + + return 0; +} + +static int run_c3312(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t size = BITS(inst, 31, 30), + V = BITS(inst, 26, 26), + opc = BITS(inst, 23, 22), + Rt = BITS(inst, 4, 0); + uint8_t fields = (size << 3) | (V << 2) | opc; + + res->rr_num = 1; + res->rr_addrdata[0].ad_addr = pa; + + if (fields == 0) do_str(1, Rt, va, ss, res); + else if (fields == 1) do_ldr(1, Rt, va, ss, res); + else if (fields == 2) do_ldrs(1, 8, Rt, va, ss, res); + else if (fields == 3) do_ldrs(1, 4, Rt, va, ss, res); + else if ((fields == 4) || + (fields == 5) || + (fields == 6) || + (fields == 7) || + (fields == 12) || + (fields == 13) || + (fields == 0x14) || + (fields == 0x15) || + (fields == 0x1c) || + (fields == 0x1d)) CANNOTDECODE("simd", inst); + else if (fields == 8) do_str(2, Rt, va, ss, res); + else if (fields == 9) do_ldr(2, Rt, va, ss, res); + else if (fields == 10) do_ldrs(2, 8, Rt, va, ss, res); + else if (fields == 11) do_ldrs(2, 4, Rt, va, ss, res); + else if (fields == 0x10) do_str(4, Rt, va, ss, res); + else if (fields == 0x11) do_ldr(4, Rt, va, ss, res); + else if (fields == 0x12) do_ldrs(4, 8, Rt, va, ss, res); + else if (fields == 0x18) do_str(8, Rt, va, ss, res); + else if (fields == 0x19) do_ldr(8, Rt, va, ss, res); + else if (fields == 0x1a) do_prfm(Rt, va, res); + else CANNOTDECODE("unknown", inst); + + stats.stat_decoder.sd_c3312++; + + return 0; +} + +static int run_c3313(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t size = BITS(inst, 31, 30), + V = BITS(inst, 26, 26), + opc = BITS(inst, 23, 22), + Rt = BITS(inst, 4, 0); + uint8_t fields = (size << 3) | (V << 2) | opc; + + res->rr_num = 1; + res->rr_addrdata[0].ad_addr = pa; + + if (fields == 0) do_str(1, Rt, va, ss, res); + else if (fields == 1) do_ldr(1, Rt, va, ss, res); + else if (fields == 2) do_ldrs(1, 8, Rt, va, ss, res); + else if (fields == 3) do_ldrs(1, 4, Rt, va, ss, res); + else if ((fields == 4) || + (fields == 5) || + (fields == 6) || + (fields == 7) || + (fields == 12) || + (fields == 13) || + (fields == 0x14) || + (fields == 0x15) || + (fields == 0x1c) || + (fields == 0x1d)) CANNOTDECODE("simd", inst); + else if (fields == 8) do_str(2, Rt, va, ss, res); + else if (fields == 9) do_ldr(2, Rt, va, ss, res); + else if (fields == 10) do_ldrs(2, 8, Rt, va, ss, res); + else if (fields == 11) do_ldrs(2, 4, Rt, va, ss, res); + else if (fields == 0x10) do_str(4, Rt, va, ss, res); + else if (fields == 0x11) do_ldr(4, Rt, va, ss, res); + else if (fields == 0x12) do_ldrs(4, 8, Rt, va, ss, res); + else if (fields == 0x18) do_str(8, Rt, va, ss, res); + else if (fields == 0x19) do_ldr(8, Rt, va, ss, res); + else if (fields == 0x1a) do_prfm(Rt, va, res); + else CANNOTDECODE("unknown", inst); + + stats.stat_decoder.sd_c3313++; + + return 0; +} + +static int run_c3314(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t opc = BITS(inst, 31, 30), + V = BITS(inst, 26, 26), + L = BITS(inst, 22, 22), + Rt = BITS(inst, 4, 0), + Rt2 = BITS(inst, 14, 10); + uint8_t fields = (opc << 2) | (V << 1) | L; + + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = pa; + + if (fields == 0) do_stp(4, Rt, Rt2, va, ss, res); + else if (fields == 1) do_ldp(4, Rt, Rt2, va, ss, res); + else if ((fields == 2) || + (fields == 3) || + (fields == 6) || + (fields == 7) || + (fields == 10) || + (fields == 11)) CANNOTDECODE("simd", inst); + else if (fields == 5) do_ldpsw(Rt, Rt2, va, ss, res); + else if (fields == 8) do_stp(8, Rt, Rt2, va, ss, res); + else if (fields == 9) do_ldp(8, Rt, Rt2, va, ss, res); + else CANNOTDECODE("unknown", inst); + + stats.stat_decoder.sd_c3314++; + + return 0; +} + +static int run_c3315(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t opc = BITS(inst, 31, 30), + V = BITS(inst, 26, 26), + L = BITS(inst, 22, 22), + Rt = BITS(inst, 4, 0), + Rt2 = BITS(inst, 14, 10); + uint8_t fields = (opc << 2) | (V << 1) | L; + + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = pa; + + if (fields == 0) do_stp(4, Rt, Rt2, va, ss, res); + else if (fields == 1) do_ldp(4, Rt, Rt2, va, ss, res); + else if ((fields == 2) || + (fields == 3) || + (fields == 6) || + (fields == 7) || + (fields == 10) || + (fields == 11)) CANNOTDECODE("simd", inst); + else if (fields == 5) do_ldpsw(Rt, Rt2, va, ss, res); + else if (fields == 8) do_stp(8, Rt, Rt2, va, ss, res); + else if (fields == 9) do_ldp(8, Rt, Rt2, va, ss, res); + else CANNOTDECODE("unknown", inst); + + stats.stat_decoder.sd_c3315++; + + return 0; +} + +static int run_c3316(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint32_t opc = BITS(inst, 31, 30), + V = BITS(inst, 26, 26), + L = BITS(inst, 22, 22), + Rt = BITS(inst, 4, 0), + Rt2 = BITS(inst, 14, 10); + uint8_t fields = (opc << 2) | (V << 1) | L; + + res->rr_num = 2; + res->rr_addrdata[0].ad_addr = pa; + + if (fields == 0) do_stp(4, Rt, Rt2, va, ss, res); + else if (fields == 1) do_ldp(4, Rt, Rt2, va, ss, res); + else if ((fields == 2) || + (fields == 3) || + (fields == 6) || + (fields == 7) || + (fields == 10) || + (fields == 11)) CANNOTDECODE("simd", inst); + else if (fields == 5) do_ldpsw(Rt, Rt2, va, ss, res); + else if (fields == 8) do_stp(8, Rt, Rt2, va, ss, res); + else if (fields == 9) do_ldp(8, Rt, Rt2, va, ss, res); + else CANNOTDECODE("unknown", inst); + + stats.stat_decoder.sd_c3316++; + + return 0; +} + +static bool get_info_simd(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ +#pragma unused(inst, ss, info) + CANNOTDECODE("simd", inst); + return false; +} + +// load register (literal) +static bool get_info_c335(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t opc = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t imm19 = BITS(inst, 23, 5); + uint32_t fields = (opc << 1) | V; + uint8_t scale; + + if (__builtin_expect(fields > 6, false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + assert(fields <= 6); + + if (V == 1) { + scale = 2 + opc; + } else { + switch (opc) { + case 0 ... 1: + scale = 2 + opc; + break; + case 2: + scale = 2; + break; + default: + CANNOTDECODE("invalid", inst); + return false; + } + } + + info->bytes = 1 << scale; + info->addr = ss->ss_64.pc + (SIGN_EXTEND_64(imm19, 19) << 2); + + return true; +} + +// load/store exclusive +static bool get_info_c336(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t size = BITS(inst, 31, 30); + uint32_t o2 = BITS(inst, 23, 23); + uint32_t L = BITS(inst, 22, 22); + uint32_t o1 = BITS(inst, 21, 21); + uint32_t o0 = BITS(inst, 15, 15); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (size << 4) | (o2 << 3) | (L << 2) | (o1 << 1) | o0; + + if (__builtin_expect((2 <= fields && fields <= 3) || + (6 <= fields && fields <= 8) || + (10 <= fields && fields <= 12) || + (14 <= fields && fields <= 15) || + (18 <= fields && fields <= 19) || + (22 <= fields && fields <= 24) || + (26 <= fields && fields <= 28) || + (30 <= fields && fields <= 31) || + (40 == fields) || + (42 <= fields && fields <= 44) || + (46 <= fields && fields <= 47) || + (56 == fields) || + (58 <= fields && fields <= 60) || + (62 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + info->bytes = (1 << size) << o1; + info->addr = ss->ss_64.x[Rn]; + + return true; +} + +// load/store no-allocate pair (offset) +bool get_info_c337(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t opc = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t L = BITS(inst, 22, 22); + uint32_t imm7 = BITS(inst, 21, 15); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (opc << 2) | (V << 1) | L; + uint8_t scale; + + if (__builtin_expect((4 <= fields && fields <= 5) || + (12 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + if (V == 1) { + scale = opc + 2; + } else { + scale = BITS(opc, 1, 1) + 2; + } + + // double since it's pair + info->bytes = 2 * (1 << scale); + info->addr = ss->ss_64.x[Rn] + (SIGN_EXTEND_64(imm7, 7) << scale); + + return true; +} + +// load/store reigster (immediate post-indexed) +static bool get_info_c338(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t size = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t opc = BITS(inst, 23, 22); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (size << 3) | (V << 2) | opc; + uint8_t scale; + + if (__builtin_expect((14 <= fields && fields <= 15) || + (19 == fields) || + (22 <= fields && fields <= 23) || + (26 <= fields && fields <= 27) || + (30 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + if (V == 1) { + scale = BITS(opc, 1, 1) << 2 | size; + } else { + scale = size; + } + + info->bytes = 1 << scale; + // post-indexed + info->addr = ss->ss_64.x[Rn]; + + return true; +} + +// load/store register (immediate pre-indexed) +static bool get_info_c339(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t size = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t opc = BITS(inst, 23, 22); + uint32_t imm9 = BITS(inst, 20, 12); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (size << 3) | (V << 2) | opc; + uint8_t scale; + + if (__builtin_expect((14 <= fields && fields <= 15) || + (19 == fields) || + (22 <= fields && fields <= 23) || + (26 <= fields && fields <= 27) || + (30 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + if (V == 1) { + scale = BITS(opc, 1, 1) << 2 | size; + } else { + scale = size; + } + + info->bytes = 1 << scale; + info->addr = ss->ss_64.x[Rn] + SIGN_EXTEND_64(imm9, 9); + + return true; +} + +// load/store register (register offset) +static bool get_info_c3310(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t size = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t opc = BITS(inst, 23, 22); + uint32_t Rm = BITS(inst, 20, 16); + uint32_t option = BITS(inst, 15, 13); + uint32_t S = BITS(inst, 12, 12); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (size << 3) | (V << 2) | opc; + uint32_t scale; + + if (__builtin_expect((14 <= fields && fields <= 15) || + (19 == fields) || + (22 <= fields && fields <= 23) || + (27 == fields) || + (30 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + if (V == 1) { + scale = BITS(opc, 1, 1) | size; + } else { + scale = size; + } + + info->bytes = 1 << scale; + + uint64_t m = ss->ss_64.x[Rm]; + uint8_t shift = (S == 1 ? scale : 0); + + switch (option) { + case 0 ... 3: + info->addr = ss->ss_64.x[Rn] + (ZERO_EXTEND_64(m, 8 << option) << shift); + break; + case 4 ... 7: + info->addr = ss->ss_64.x[Rn] + (SIGN_EXTEND_64(m, 8 << BITS(option, 1, 0)) << shift); + break; + default: + CANNOTDECODE("invalid", inst); + return false; + } + + return true; +} + +// load/store register (unprivileged) +static bool get_info_c3311(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t size = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t opc = BITS(inst, 23, 22); + uint32_t imm9 = BITS(inst, 20, 12); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (size << 3) | (V << 2) | opc; + + if (__builtin_expect((4 <= fields && fields <= 7) || + (12 <= fields && fields <= 15) || + (19 <= fields && fields <= 23) || + (26 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + info->bytes = 1 << size; + info->addr = ss->ss_64.x[Rn] + SIGN_EXTEND_64(imm9, 9); + + return true; +} + +// load/store register (unscaled immediate) +static bool get_info_c3312(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t size = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t opc = BITS(inst, 23, 22); + uint32_t imm9 = BITS(inst, 20, 12); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (size << 3) | (V << 2) | opc; + uint32_t scale; + + if (__builtin_expect((14 <= fields && fields <= 15) || + (19 == fields) || + (22 <= fields && fields <= 23) || + (27 == fields) || + (30 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + if (V == 1) { + scale = BITS(opc, 1, 1) << 2 | size; + } else { + scale = size; + } + + info->bytes = 1 < scale; + info->addr = ss->ss_64.x[Rn] + SIGN_EXTEND_64(imm9, 9); + + return true; +} + +// load/store register (unsigned immediate) +bool get_info_c3313(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t size = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t opc = BITS(inst, 23, 22); + uint32_t imm12 = BITS(inst, 21, 10); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (size << 3) | (V << 2) | opc; + uint32_t scale; + + if (__builtin_expect((14 <= fields && fields <= 15) || + (19 == fields) || + (22 <= fields && fields <= 23) || + (27 == fields) || + (30 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + if (V == 1) { + scale = BITS(opc, 1, 1) << 2 | size; + } else { + scale = size; + } + + info->bytes = 1 << scale; + info->addr = ss->ss_64.x[Rn] + (ZERO_EXTEND_64(imm12, 12) << scale); + + return true; +} + +// load/store register pair (offset) +static bool get_info_c3314(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t opc = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t L = BITS(inst, 22, 22); + uint32_t imm7 = BITS(inst, 21, 15); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (opc << 2) | (V << 1) | L; + uint8_t scale = 2 + (opc >> 1); + + if (__builtin_expect((4 == fields) || + (12 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + if (V == 1) { + scale = 2 + opc; + } else { + scale = 2 + BITS(opc, 1, 1); + } + + info->bytes = 2 * (1 << scale); + info->addr = ss->ss_64.x[Rn] + (SIGN_EXTEND_64(imm7, 7) << scale); + + return true; +} + +// load/store register pair (post-indexed) +static bool get_info_c3315(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t opc = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t L = BITS(inst, 22, 22); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (opc << 2) | (V << 1) | L; + uint8_t scale = 2 + (opc >> 1); + + if (__builtin_expect((4 == fields) || + (12 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + if (V == 1) { + scale = 2 + opc; + } else { + scale = 2 + BITS(opc, 1, 1); + } + + info->bytes = 2 * (1 << scale); + // post-indexed + info->addr = ss->ss_64.x[Rn]; + + return true; +} + +// load/store register pair (pre-indexed) +static bool get_info_c3316(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info) +{ + uint32_t opc = BITS(inst, 31, 30); + uint32_t V = BITS(inst, 26, 26); + uint32_t L = BITS(inst, 22, 22); + uint32_t imm7 = BITS(inst, 21, 15); + uint32_t Rn = BITS(inst, 9, 5); + uint32_t fields = (opc << 2) | (V << 1) | L; + uint8_t scale = 2 + (opc >> 1); + + if (__builtin_expect((4 == fields) || + (12 <= fields), false)) { + CANNOTDECODE("invalid", inst); + return false; + } + + if (V == 1) { + scale = 2 + opc; + } else { + scale = 2 + BITS(opc, 1, 1); + } + + info->bytes = 2 * (1 << scale); + info->addr = ss->ss_64.x[Rn] + (SIGN_EXTEND_64(imm7, 7) << scale); + + return true; +} + + +//------------------------------------------------------------------- +// Globals +// +int pgtrace_decode_and_run(uint32_t inst, vm_offset_t fva, vm_map_offset_t *cva_page, arm_saved_state_t *ss, pgtrace_run_result_t *res) +{ + uint8_t len = sizeof(typetbl)/sizeof(type_entry_t); + run_t run = NULL; + get_info_t get_info = NULL; + vm_offset_t pa, cva; + vm_offset_t cva_front_page = cva_page[0]; + vm_offset_t cva_cur_page = cva_page[1]; + instruction_info_t info; + + for (uint8_t i = 0; i < len; i++) { + if ((typetbl[i].mask & inst) == typetbl[i].value) { + run = typetbl[i].run; + get_info = typetbl[i].get_info; + break; + } + } + + assert(run != NULL && get_info != NULL); + + get_info(inst, ss, &info); + + if (info.addr == fva) { + cva = cva_cur_page + (fva & ARM_PGMASK); + } else { + // which means a front page is not a tracing page + cva = cva_front_page + (fva & ARM_PGMASK); + } + + pa = mmu_kvtop(cva); + if (!pa) { + panic("%s: invalid address cva=%lx fva=%lx info.addr=%lx inst=%x", __func__, cva, fva, info.addr, inst); + } + + absolutetime_to_nanoseconds(mach_absolute_time(), &res->rr_time); + run(inst, pa, cva, ss, res); + + return 0; +} + +void pgtrace_decoder_get_stats(pgtrace_stats_t *s) +{ + memcpy((void *)&(s->stat_decoder), &(stats.stat_decoder), sizeof(stats.stat_decoder)); +} +#endif diff --git a/osfmk/arm64/pgtrace_decoder.h b/osfmk/arm64/pgtrace_decoder.h new file mode 100644 index 000000000..e5c4b5c77 --- /dev/null +++ b/osfmk/arm64/pgtrace_decoder.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#pragma once + +#if CONFIG_PGTRACE +#include +#include +#include +#include "pgtrace.h" + +int pgtrace_decode_and_run(uint32_t inst, vm_offset_t va, vm_map_offset_t *cva, arm_saved_state_t *ss, pgtrace_run_result_t *res); +void pgtrace_decoder_get_stats(pgtrace_stats_t *stats); +#endif + diff --git a/osfmk/arm64/pinst.s b/osfmk/arm64/pinst.s new file mode 100644 index 000000000..740a63915 --- /dev/null +++ b/osfmk/arm64/pinst.s @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include + +/* + * Compare two instructions with constant, spin on mismatch. + * arg0 - Constant scratch register + * arg1 - Instruction address scratch register + * arg2 - Instruction location + * arg3 - Instruction constant + */ +.macro check_instruction + // construct 64-bit constant inline to make sure it is non-executable + movz $0, #(($3 >> 48) & 0xffff), lsl #48 + movk $0, #(($3 >> 32) & 0xffff), lsl #32 + movk $0, #(($3 >> 16) & 0xffff), lsl #16 + movk $0, #(($3) & 0xffff) + // fetch instructions from "untrusted" memory + adrp $1, $2@page + add $1, $1, $2@pageoff + ldr $1, [$1] + // spin forever if we do not find what we expect + cmp $0, $1 + b.ne . +.endmacro + +#if defined(KERNEL_INTEGRITY_KTRR) + +/* AMCC only KTRR protected text, non-executable once the MMU is enabled */ + .text + .section __LAST,__pinst + .align 2 + +__pinst_set_ttbr1: + msr TTBR1_EL1, x0 + ret + +__pinst_set_vbar: + msr VBAR_EL1, x0 + ret + +__pinst_set_tcr: + msr TCR_EL1, x0 + ret + + .globl _pinst_set_sctlr_trap_addr +__pinst_set_sctlr: + msr SCTLR_EL1, x0 +_pinst_set_sctlr_trap_addr: + ret + + +/* MMU and AMCC KTRR protected text */ + .text + .section __TEXT_EXEC,__text + .align 2 + + .globl _pinst_set_ttbr1 +_pinst_set_ttbr1: + check_instruction x2, x3, __pinst_set_ttbr1, 0xd65f03c0d5182020 + b __pinst_set_ttbr1 + + .globl _pinst_set_vbar +_pinst_set_vbar: + check_instruction x2, x3, __pinst_set_vbar, 0xd65f03c0d518c000 + b __pinst_set_vbar + + .globl _pinst_set_tcr +_pinst_set_tcr: + check_instruction x2, x3, __pinst_set_tcr, 0xd65f03c0d5182040 + b __pinst_set_tcr + + .globl _pinst_set_sctlr +_pinst_set_sctlr: + check_instruction x2, x3, __pinst_set_sctlr, 0xd65f03c0d5181000 + b __pinst_set_sctlr + +#endif /* defined(KERNEL_INTEGRITY_KTRR) */ + +#if defined(KERNEL_INTEGRITY_KTRR) + + .text + .section __LAST,__pinst + .align 2 + +__pinst_spsel_1: + msr SPSel, #1 + ret + + .text + .section __TEXT_EXEC,__text + .align 2 + + .globl _pinst_spsel_1 +_pinst_spsel_1: + check_instruction x2, x3, __pinst_spsel_1, 0xd65f03c0d50041bf + b __pinst_spsel_1 + +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + diff --git a/osfmk/arm64/platform_tests.c b/osfmk/arm64/platform_tests.c new file mode 100644 index 000000000..0c20f8a02 --- /dev/null +++ b/osfmk/arm64/platform_tests.c @@ -0,0 +1,1087 @@ +/* + * Copyright (c) 2011 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System Copyright (c) 1991,1990,1989,1988,1987 Carnegie + * Mellon University All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright notice + * and this permission notice appear in all copies of the software, + * derivative works or modified versions, and any portions thereof, and that + * both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" CONDITION. + * CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ANY DAMAGES + * WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science Carnegie Mellon University Pittsburgh PA + * 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon the + * rights to redistribute these changes. + */ + +#include + +#define LOCK_PRIVATE 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if MACH_KDB +#include +#include +#include +#include +#endif /* MACH_KDB */ + +#include +#include +#include +#include + +extern boolean_t arm_pan_enabled; +kern_return_t arm64_lock_test(void); +kern_return_t arm64_munger_test(void); +kern_return_t ex_cb_test(void); +kern_return_t arm64_pan_test(void); + +// exception handler ignores this fault address during PAN test +#if __ARM_PAN_AVAILABLE__ +vm_offset_t pan_test_addr; +#endif + +#include +#define LOCK_TEST_ITERATIONS 50 +static hw_lock_data_t lt_hw_lock; +static lck_spin_t lt_lck_spin_t; +static lck_mtx_t lt_mtx; +static lck_rw_t lt_rwlock; +static volatile uint32_t lt_counter = 0; +static volatile int lt_spinvolatile; +static volatile uint32_t lt_max_holders = 0; +static volatile uint32_t lt_upgrade_holders = 0; +static volatile uint32_t lt_max_upgrade_holders = 0; +static volatile uint32_t lt_num_holders = 0; +static volatile uint32_t lt_done_threads; +static volatile uint32_t lt_target_done_threads; +static volatile uint32_t lt_cpu_bind_id = 0; + +static void +lt_note_another_blocking_lock_holder() +{ + hw_lock_lock(<_hw_lock); + lt_num_holders++; + lt_max_holders = (lt_max_holders < lt_num_holders) ? lt_num_holders : lt_max_holders; + hw_lock_unlock(<_hw_lock); +} + +static void +lt_note_blocking_lock_release() +{ + hw_lock_lock(<_hw_lock); + lt_num_holders--; + hw_lock_unlock(<_hw_lock); +} + +static void +lt_spin_a_little_bit() +{ + uint32_t i; + + for (i = 0; i < 10000; i++) { + lt_spinvolatile++; + } +} + +static void +lt_sleep_a_little_bit() +{ + delay(100); +} + +static void +lt_grab_mutex() +{ + lck_mtx_lock(<_mtx); + lt_note_another_blocking_lock_holder(); + lt_sleep_a_little_bit(); + lt_counter++; + lt_note_blocking_lock_release(); + lck_mtx_unlock(<_mtx); +} + +static void +lt_grab_mutex_with_try() +{ + while(0 == lck_mtx_try_lock(<_mtx)); + lt_note_another_blocking_lock_holder(); + lt_sleep_a_little_bit(); + lt_counter++; + lt_note_blocking_lock_release(); + lck_mtx_unlock(<_mtx); + +} + +static void +lt_grab_rw_exclusive() +{ + lck_rw_lock_exclusive(<_rwlock); + lt_note_another_blocking_lock_holder(); + lt_sleep_a_little_bit(); + lt_counter++; + lt_note_blocking_lock_release(); + lck_rw_done(<_rwlock); +} + +static void +lt_grab_rw_exclusive_with_try() +{ + while(0 == lck_rw_try_lock_exclusive(<_rwlock)) { + lt_sleep_a_little_bit(); + } + + lt_note_another_blocking_lock_holder(); + lt_sleep_a_little_bit(); + lt_counter++; + lt_note_blocking_lock_release(); + lck_rw_done(<_rwlock); +} + +/* Disabled until lt_grab_rw_shared() is fixed (rdar://30685840) +static void +lt_grab_rw_shared() +{ + lck_rw_lock_shared(<_rwlock); + lt_counter++; + + lt_note_another_blocking_lock_holder(); + lt_sleep_a_little_bit(); + lt_note_blocking_lock_release(); + + lck_rw_done(<_rwlock); +} +*/ + +/* Disabled until lt_grab_rw_shared_with_try() is fixed (rdar://30685840) +static void +lt_grab_rw_shared_with_try() +{ + while(0 == lck_rw_try_lock_shared(<_rwlock)); + lt_counter++; + + lt_note_another_blocking_lock_holder(); + lt_sleep_a_little_bit(); + lt_note_blocking_lock_release(); + + lck_rw_done(<_rwlock); +} +*/ + +static void +lt_upgrade_downgrade_rw() +{ + boolean_t upgraded, success; + + success = lck_rw_try_lock_shared(<_rwlock); + if (!success) { + lck_rw_lock_shared(<_rwlock); + } + + lt_note_another_blocking_lock_holder(); + lt_sleep_a_little_bit(); + lt_note_blocking_lock_release(); + + upgraded = lck_rw_lock_shared_to_exclusive(<_rwlock); + if (!upgraded) { + success = lck_rw_try_lock_exclusive(<_rwlock); + + if (!success) { + lck_rw_lock_exclusive(<_rwlock); + } + } + + lt_upgrade_holders++; + if (lt_upgrade_holders > lt_max_upgrade_holders) { + lt_max_upgrade_holders = lt_upgrade_holders; + } + + lt_counter++; + lt_sleep_a_little_bit(); + + lt_upgrade_holders--; + + lck_rw_lock_exclusive_to_shared(<_rwlock); + + lt_spin_a_little_bit(); + lck_rw_done(<_rwlock); +} + +const int limit = 1000000; +static int lt_stress_local_counters[MAX_CPUS]; + +static void +lt_stress_hw_lock() +{ + int local_counter = 0; + + uint cpuid = current_processor()->cpu_id; + + kprintf("%s>cpu %d starting\n", __FUNCTION__, cpuid); + + hw_lock_lock(<_hw_lock); + lt_counter++; + local_counter++; + hw_lock_unlock(<_hw_lock); + + while (lt_counter < lt_target_done_threads) { + ; + } + + kprintf("%s>cpu %d started\n", __FUNCTION__, cpuid); + + while (lt_counter < limit) { + spl_t s = splsched(); + hw_lock_lock(<_hw_lock); + if (lt_counter < limit) { + lt_counter++; + local_counter++; + } + hw_lock_unlock(<_hw_lock); + splx(s); + } + + lt_stress_local_counters[cpuid] = local_counter; + + kprintf("%s>final counter %d cpu %d incremented the counter %d times\n", __FUNCTION__, lt_counter, cpuid, local_counter); +} + +static void +lt_grab_hw_lock() +{ + hw_lock_lock(<_hw_lock); + lt_counter++; + lt_spin_a_little_bit(); + hw_lock_unlock(<_hw_lock); +} + +static void +lt_grab_hw_lock_with_try() +{ + while(0 == hw_lock_try(<_hw_lock)); + lt_counter++; + lt_spin_a_little_bit(); + hw_lock_unlock(<_hw_lock); +} + +static void +lt_grab_hw_lock_with_to() +{ + while(0 == hw_lock_to(<_hw_lock, LockTimeOut)) + mp_enable_preemption(); + lt_counter++; + lt_spin_a_little_bit(); + hw_lock_unlock(<_hw_lock); +} + +static void +lt_grab_spin_lock() +{ + lck_spin_lock(<_lck_spin_t); + lt_counter++; + lt_spin_a_little_bit(); + lck_spin_unlock(<_lck_spin_t); +} + +static void +lt_grab_spin_lock_with_try() +{ + while(0 == lck_spin_try_lock(<_lck_spin_t)); + lt_counter++; + lt_spin_a_little_bit(); + lck_spin_unlock(<_lck_spin_t); +} + +static volatile boolean_t lt_thread_lock_grabbed; +static volatile boolean_t lt_thread_lock_success; + +static void +lt_reset() +{ + lt_counter = 0; + lt_max_holders = 0; + lt_num_holders = 0; + lt_max_upgrade_holders = 0; + lt_upgrade_holders = 0; + lt_done_threads = 0; + lt_target_done_threads = 0; + lt_cpu_bind_id = 0; + + OSMemoryBarrier(); +} + +static void +lt_trylock_hw_lock_with_to() +{ + OSMemoryBarrier(); + while (!lt_thread_lock_grabbed) { + lt_sleep_a_little_bit(); + OSMemoryBarrier(); + } + lt_thread_lock_success = hw_lock_to(<_hw_lock, 100); + OSMemoryBarrier(); + mp_enable_preemption(); +} + +static void +lt_trylock_spin_try_lock() +{ + OSMemoryBarrier(); + while (!lt_thread_lock_grabbed) { + lt_sleep_a_little_bit(); + OSMemoryBarrier(); + } + lt_thread_lock_success = lck_spin_try_lock(<_lck_spin_t); + OSMemoryBarrier(); +} + +static void +lt_trylock_thread(void *arg, wait_result_t wres __unused) +{ + void (*func)(void) = (void(*)(void))arg; + + func(); + + OSIncrementAtomic((volatile SInt32*) <_done_threads); +} + +static void +lt_start_trylock_thread(thread_continue_t func) +{ + thread_t thread; + kern_return_t kr; + + kr = kernel_thread_start(lt_trylock_thread, func, &thread); + assert(kr == KERN_SUCCESS); + + thread_deallocate(thread); +} + +static void +lt_wait_for_lock_test_threads() +{ + OSMemoryBarrier(); + /* Spin to reduce dependencies */ + while (lt_done_threads < lt_target_done_threads) { + lt_sleep_a_little_bit(); + OSMemoryBarrier(); + } + OSMemoryBarrier(); +} + +static kern_return_t +lt_test_trylocks() +{ + boolean_t success; + + /* + * First mtx try lock succeeds, second fails. + */ + success = lck_mtx_try_lock(<_mtx); + T_ASSERT_NOTNULL(success, "First mtx try lock"); + success = lck_mtx_try_lock(<_mtx); + T_ASSERT_NULL(success, "Second mtx try lock for a locked mtx"); + lck_mtx_unlock(<_mtx); + + /* + * After regular grab, can't try lock. + */ + lck_mtx_lock(<_mtx); + success = lck_mtx_try_lock(<_mtx); + T_ASSERT_NULL(success, "try lock should fail after regular lck_mtx_lock"); + lck_mtx_unlock(<_mtx); + + /* + * Two shared try locks on a previously unheld rwlock suceed, and a + * subsequent exclusive attempt fails. + */ + success = lck_rw_try_lock_shared(<_rwlock); + T_ASSERT_NOTNULL(success, "Two shared try locks on a previously unheld rwlock should succeed"); + success = lck_rw_try_lock_shared(<_rwlock); + T_ASSERT_NOTNULL(success, "Two shared try locks on a previously unheld rwlock should succeed"); + success = lck_rw_try_lock_exclusive(<_rwlock); + T_ASSERT_NULL(success, "exclusive lock attempt on previously held lock should fail"); + lck_rw_done(<_rwlock); + lck_rw_done(<_rwlock); + + /* + * After regular shared grab, can trylock + * for shared but not for exclusive. + */ + lck_rw_lock_shared(<_rwlock); + success = lck_rw_try_lock_shared(<_rwlock); + T_ASSERT_NOTNULL(success, "After regular shared grab another shared try lock should succeed."); + success = lck_rw_try_lock_exclusive(<_rwlock); + T_ASSERT_NULL(success, "After regular shared grab an exclusive lock attempt should fail."); + lck_rw_done(<_rwlock); + lck_rw_done(<_rwlock); + + /* + * An exclusive try lock succeeds, subsequent shared and exclusive + * attempts fail. + */ + success = lck_rw_try_lock_exclusive(<_rwlock); + T_ASSERT_NOTNULL(success, "An exclusive try lock should succeed"); + success = lck_rw_try_lock_shared(<_rwlock); + T_ASSERT_NULL(success, "try lock in shared mode attempt after an exclusive grab should fail"); + success = lck_rw_try_lock_exclusive(<_rwlock); + T_ASSERT_NULL(success, "try lock in exclusive mode attempt after an exclusive grab should fail"); + lck_rw_done(<_rwlock); + + /* + * After regular exclusive grab, neither kind of trylock succeeds. + */ + lck_rw_lock_exclusive(<_rwlock); + success = lck_rw_try_lock_shared(<_rwlock); + T_ASSERT_NULL(success, "After regular exclusive grab, shared trylock should not succeed"); + success = lck_rw_try_lock_exclusive(<_rwlock); + T_ASSERT_NULL(success, "After regular exclusive grab, exclusive trylock should not succeed"); + lck_rw_done(<_rwlock); + + /* + * First spin lock attempts succeed, second attempts fail. + */ + success = hw_lock_try(<_hw_lock); + T_ASSERT_NOTNULL(success, "First spin lock attempts should succeed"); + success = hw_lock_try(<_hw_lock); + T_ASSERT_NULL(success, "Second attempt to spin lock should fail"); + hw_lock_unlock(<_hw_lock); + + hw_lock_lock(<_hw_lock); + success = hw_lock_try(<_hw_lock); + T_ASSERT_NULL(success, "After taking spin lock, trylock attempt should fail"); + hw_lock_unlock(<_hw_lock); + + lt_reset(); + lt_thread_lock_grabbed = false; + lt_thread_lock_success = true; + lt_target_done_threads = 1; + OSMemoryBarrier(); + lt_start_trylock_thread(lt_trylock_hw_lock_with_to); + success = hw_lock_to(<_hw_lock, 100); + T_ASSERT_NOTNULL(success, "First spin lock with timeout should succeed"); + OSIncrementAtomic((volatile SInt32*)<_thread_lock_grabbed); + lt_wait_for_lock_test_threads(); + T_ASSERT_NULL(lt_thread_lock_success, "Second spin lock with timeout should fail and timeout"); + hw_lock_unlock(<_hw_lock); + + lt_reset(); + lt_thread_lock_grabbed = false; + lt_thread_lock_success = true; + lt_target_done_threads = 1; + OSMemoryBarrier(); + lt_start_trylock_thread(lt_trylock_hw_lock_with_to); + hw_lock_lock(<_hw_lock); + OSIncrementAtomic((volatile SInt32*)<_thread_lock_grabbed); + lt_wait_for_lock_test_threads(); + T_ASSERT_NULL(lt_thread_lock_success, "after taking a spin lock, lock attempt with timeout should fail"); + hw_lock_unlock(<_hw_lock); + + success = lck_spin_try_lock(<_lck_spin_t); + T_ASSERT_NOTNULL(success, "spin trylock of previously unheld lock should succeed"); + success = lck_spin_try_lock(<_lck_spin_t); + T_ASSERT_NULL(success, "spin trylock attempt of previously held lock (with trylock) should fail"); + lck_spin_unlock(<_lck_spin_t); + + lt_reset(); + lt_thread_lock_grabbed = false; + lt_thread_lock_success = true; + lt_target_done_threads = 1; + lt_start_trylock_thread(lt_trylock_spin_try_lock); + lck_spin_lock(<_lck_spin_t); + OSIncrementAtomic((volatile SInt32*)<_thread_lock_grabbed); + lt_wait_for_lock_test_threads(); + T_ASSERT_NULL(lt_thread_lock_success, "spin trylock attempt of previously held lock should fail"); + lck_spin_unlock(<_lck_spin_t); + + return KERN_SUCCESS; +} + +static void +lt_thread(void *arg, wait_result_t wres __unused) +{ + void (*func)(void) = (void(*)(void)) arg; + uint32_t i; + + for (i = 0; i < LOCK_TEST_ITERATIONS; i++) { + func(); + } + + OSIncrementAtomic((volatile SInt32*) <_done_threads); +} + +static void +lt_bound_thread(void *arg, wait_result_t wres __unused) +{ + void (*func)(void) = (void(*)(void)) arg; + + int cpuid = OSIncrementAtomic((volatile SInt32 *)<_cpu_bind_id); + + processor_t processor = processor_list; + while ((processor != NULL) && (processor->cpu_id != cpuid)) { + processor = processor->processor_list; + } + + if (processor != NULL) { + thread_bind(processor); + } + + thread_block(THREAD_CONTINUE_NULL); + + func(); + + OSIncrementAtomic((volatile SInt32*) <_done_threads); +} + +static void +lt_start_lock_thread(thread_continue_t func) +{ + thread_t thread; + kern_return_t kr; + + kr = kernel_thread_start(lt_thread, func, &thread); + assert(kr == KERN_SUCCESS); + + thread_deallocate(thread); +} + + +static void +lt_start_lock_thread_bound(thread_continue_t func) +{ + thread_t thread; + kern_return_t kr; + + kr = kernel_thread_start(lt_bound_thread, func, &thread); + assert(kr == KERN_SUCCESS); + + thread_deallocate(thread); +} + +static kern_return_t +lt_test_locks() +{ + kern_return_t kr = KERN_SUCCESS; + lck_grp_attr_t *lga = lck_grp_attr_alloc_init(); + lck_grp_t *lg = lck_grp_alloc_init("lock test", lga); + + lck_mtx_init(<_mtx, lg, LCK_ATTR_NULL); + lck_rw_init(<_rwlock, lg, LCK_ATTR_NULL); + lck_spin_init(<_lck_spin_t, lg, LCK_ATTR_NULL); + hw_lock_init(<_hw_lock); + + T_LOG("Testing locks."); + + /* Try locks (custom) */ + lt_reset(); + + T_LOG("Running try lock test."); + kr = lt_test_trylocks(); + T_EXPECT_NULL(kr, "try lock test failed."); + + /* Uncontended mutex */ + T_LOG("Running uncontended mutex test."); + lt_reset(); + lt_target_done_threads = 1; + lt_start_lock_thread(lt_grab_mutex); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL); + + /* Contended mutex:try locks*/ + T_LOG("Running contended mutex test."); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_mutex); + lt_start_lock_thread(lt_grab_mutex); + lt_start_lock_thread(lt_grab_mutex); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL); + + /* Contended mutex: try locks*/ + T_LOG("Running contended mutex trylock test."); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_mutex_with_try); + lt_start_lock_thread(lt_grab_mutex_with_try); + lt_start_lock_thread(lt_grab_mutex_with_try); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL); + + /* Uncontended exclusive rwlock */ + T_LOG("Running uncontended exclusive rwlock test."); + lt_reset(); + lt_target_done_threads = 1; + lt_start_lock_thread(lt_grab_rw_exclusive); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL); + + /* Uncontended shared rwlock */ + + /* Disabled until lt_grab_rw_shared() is fixed (rdar://30685840) + T_LOG("Running uncontended shared rwlock test."); + lt_reset(); + lt_target_done_threads = 1; + lt_start_lock_thread(lt_grab_rw_shared); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL); + */ + + /* Contended exclusive rwlock */ + T_LOG("Running contended exclusive rwlock test."); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_rw_exclusive); + lt_start_lock_thread(lt_grab_rw_exclusive); + lt_start_lock_thread(lt_grab_rw_exclusive); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL); + + /* One shared, two exclusive */ + /* Disabled until lt_grab_rw_shared() is fixed (rdar://30685840) + T_LOG("Running test with one shared and two exclusive rw lock threads."); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_rw_shared); + lt_start_lock_thread(lt_grab_rw_exclusive); + lt_start_lock_thread(lt_grab_rw_exclusive); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL); + */ + + /* Four shared */ + /* Disabled until lt_grab_rw_shared() is fixed (rdar://30685840) + T_LOG("Running test with four shared holders."); + lt_reset(); + lt_target_done_threads = 4; + lt_start_lock_thread(lt_grab_rw_shared); + lt_start_lock_thread(lt_grab_rw_shared); + lt_start_lock_thread(lt_grab_rw_shared); + lt_start_lock_thread(lt_grab_rw_shared); + lt_wait_for_lock_test_threads(); + T_EXPECT_LE_UINT(lt_max_holders, 4, NULL); + */ + + /* Three doing upgrades and downgrades */ + T_LOG("Running test with threads upgrading and downgrading."); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_upgrade_downgrade_rw); + lt_start_lock_thread(lt_upgrade_downgrade_rw); + lt_start_lock_thread(lt_upgrade_downgrade_rw); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_LE_UINT(lt_max_holders, 3, NULL); + T_EXPECT_EQ_UINT(lt_max_upgrade_holders, 1, NULL); + + /* Uncontended - exclusive trylocks */ + T_LOG("Running test with single thread doing exclusive rwlock trylocks."); + lt_reset(); + lt_target_done_threads = 1; + lt_start_lock_thread(lt_grab_rw_exclusive_with_try); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL); + + /* Uncontended - shared trylocks */ + /* Disabled until lt_grab_rw_shared_with_try() is fixed (rdar://30685840) + T_LOG("Running test with single thread doing shared rwlock trylocks."); + lt_reset(); + lt_target_done_threads = 1; + lt_start_lock_thread(lt_grab_rw_shared_with_try); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL); + */ + + /* Three doing exclusive trylocks */ + T_LOG("Running test with threads doing exclusive rwlock trylocks."); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_rw_exclusive_with_try); + lt_start_lock_thread(lt_grab_rw_exclusive_with_try); + lt_start_lock_thread(lt_grab_rw_exclusive_with_try); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_EQ_UINT(lt_max_holders, 1, NULL); + + /* Three doing shared trylocks */ + /* Disabled until lt_grab_rw_shared_with_try() is fixed (rdar://30685840) + T_LOG("Running test with threads doing shared rwlock trylocks."); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_rw_shared_with_try); + lt_start_lock_thread(lt_grab_rw_shared_with_try); + lt_start_lock_thread(lt_grab_rw_shared_with_try); + lt_wait_for_lock_test_threads(); + T_EXPECT_LE_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_LE_UINT(lt_max_holders, 3, NULL); + */ + + /* Three doing various trylocks */ + /* Disabled until lt_grab_rw_shared_with_try() is fixed (rdar://30685840) + T_LOG("Running test with threads doing mixed rwlock trylocks."); + lt_reset(); + lt_target_done_threads = 4; + lt_start_lock_thread(lt_grab_rw_shared_with_try); + lt_start_lock_thread(lt_grab_rw_shared_with_try); + lt_start_lock_thread(lt_grab_rw_exclusive_with_try); + lt_start_lock_thread(lt_grab_rw_exclusive_with_try); + lt_wait_for_lock_test_threads(); + T_EXPECT_LE_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + T_EXPECT_LE_UINT(lt_max_holders, 2, NULL); + */ + + /* HW locks */ + T_LOG("Running test with hw_lock_lock()"); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_hw_lock); + lt_start_lock_thread(lt_grab_hw_lock); + lt_start_lock_thread(lt_grab_hw_lock); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + + /* HW locks stress test */ + T_LOG("Running HW locks stress test with hw_lock_lock()"); + extern unsigned int real_ncpus; + lt_reset(); + lt_target_done_threads = real_ncpus; + for (processor_t processor = processor_list; processor != NULL; processor = processor->processor_list) { + lt_start_lock_thread_bound(lt_stress_hw_lock); + } + lt_wait_for_lock_test_threads(); + bool starvation = false; + uint total_local_count = 0; + for (processor_t processor = processor_list; processor != NULL; processor = processor->processor_list) { + starvation = starvation || (lt_stress_local_counters[processor->cpu_id] < 10); + total_local_count += lt_stress_local_counters[processor->cpu_id]; + } + if (total_local_count != lt_counter) { + T_FAIL("Lock failure\n"); + } else if (starvation) { + T_FAIL("Lock starvation found\n"); + } else { + T_PASS("HW locks stress test with hw_lock_lock()"); + } + + + /* HW locks: trylocks */ + T_LOG("Running test with hw_lock_try()"); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_hw_lock_with_try); + lt_start_lock_thread(lt_grab_hw_lock_with_try); + lt_start_lock_thread(lt_grab_hw_lock_with_try); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + + /* HW locks: with timeout */ + T_LOG("Running test with hw_lock_to()"); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_hw_lock_with_to); + lt_start_lock_thread(lt_grab_hw_lock_with_to); + lt_start_lock_thread(lt_grab_hw_lock_with_to); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + + /* Spin locks */ + T_LOG("Running test with lck_spin_lock()"); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_spin_lock); + lt_start_lock_thread(lt_grab_spin_lock); + lt_start_lock_thread(lt_grab_spin_lock); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + + /* Spin locks: trylocks */ + T_LOG("Running test with lck_spin_try_lock()"); + lt_reset(); + lt_target_done_threads = 3; + lt_start_lock_thread(lt_grab_spin_lock_with_try); + lt_start_lock_thread(lt_grab_spin_lock_with_try); + lt_start_lock_thread(lt_grab_spin_lock_with_try); + lt_wait_for_lock_test_threads(); + T_EXPECT_EQ_UINT(lt_counter, LOCK_TEST_ITERATIONS * lt_target_done_threads, NULL); + + return KERN_SUCCESS; +} + +#define MT_MAX_ARGS 8 +#define MT_INITIAL_VALUE 0xfeedbeef +#define MT_W_VAL (0x00000000feedbeefULL) /* Drop in zeros */ +#define MT_S_VAL (0xfffffffffeedbeefULL) /* High bit is 1, so sign-extends as negative */ +#define MT_L_VAL (((uint64_t)MT_INITIAL_VALUE) | (((uint64_t)MT_INITIAL_VALUE) << 32)) /* Two back-to-back */ + +typedef void (*sy_munge_t)(void*); + +#define MT_FUNC(x) #x, x +struct munger_test { + const char *mt_name; + sy_munge_t mt_func; + uint32_t mt_in_words; + uint32_t mt_nout; + uint64_t mt_expected[MT_MAX_ARGS]; +} munger_tests[] = { + {MT_FUNC(munge_w), 1, 1, {MT_W_VAL}}, + {MT_FUNC(munge_ww), 2, 2, {MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_www), 3, 3, {MT_W_VAL, MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwww), 4, 4, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwwww), 5, 5, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwwwww), 6, 6, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwwwwww), 7, 7, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwwwwwww), 8, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wl), 3, 2, {MT_W_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wwl), 4, 3, {MT_W_VAL, MT_W_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wwlll), 8, 5, {MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wlw), 4, 3, {MT_W_VAL, MT_L_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wlwwwll), 10, 7, {MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wlwwwllw), 11, 8, {MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wlwwlwlw), 11, 8, {MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wll), 5, 3, {MT_W_VAL, MT_L_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wlll), 7, 4, {MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wllwwll), 11, 7, {MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wwwlw), 6, 5, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwwlww), 7, 6, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwwl), 5, 4, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wwwwlw), 7, 6, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwwwl), 6, 5, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wwwwwl), 7, 6, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wwwwwlww), 9, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwwwwllw), 10, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwwwwlll), 11, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wwwwwwl), 8, 7, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wwwwwwlw), 9, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wwwwwwll), 10, 8, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wsw), 3, 3, {MT_W_VAL, MT_S_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wws), 3, 3, {MT_W_VAL, MT_W_VAL, MT_S_VAL}}, + {MT_FUNC(munge_wwwsw), 5, 5, {MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_S_VAL, MT_W_VAL}}, + {MT_FUNC(munge_llllll), 12, 6, {MT_L_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL, MT_L_VAL}}, + {MT_FUNC(munge_l), 2, 1, {MT_L_VAL}}, + {MT_FUNC(munge_lw), 3, 2, {MT_L_VAL, MT_W_VAL}}, + {MT_FUNC(munge_lwww), 5, 4, {MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_lwwwwwww), 9, 8, {MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL}}, + {MT_FUNC(munge_wlwwwl), 8, 6, {MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}}, + {MT_FUNC(munge_wwlwwwl), 9, 7, {MT_W_VAL, MT_W_VAL, MT_L_VAL, MT_W_VAL, MT_W_VAL, MT_W_VAL, MT_L_VAL}} +}; + +#define MT_TEST_COUNT (sizeof(munger_tests) / sizeof(struct munger_test)) + +static void +mt_reset(uint32_t in_words, size_t total_size, uint32_t *data) +{ + uint32_t i; + + for (i = 0; i < in_words; i++) { + data[i] = MT_INITIAL_VALUE; + } + + if (in_words * sizeof(uint32_t) < total_size) { + bzero(&data[in_words], total_size - in_words * sizeof(uint32_t)); + } +} + +static void +mt_test_mungers() +{ + uint64_t data[MT_MAX_ARGS]; + uint32_t i, j; + + for (i = 0; i < MT_TEST_COUNT; i++) { + struct munger_test *test = &munger_tests[i]; + int pass = 1; + + T_LOG("Testing %s", test->mt_name); + + mt_reset(test->mt_in_words, sizeof(data), (uint32_t*)data); + test->mt_func(data); + + for (j = 0; j < test->mt_nout; j++) { + if (data[j] != test->mt_expected[j]) { + T_FAIL("Index %d: expected %llx, got %llx.", j, test->mt_expected[j], data[j]); + pass = 0; + } + } + if (pass) { + T_PASS(test->mt_name); + } + } +} + +/* Exception Callback Test */ +static ex_cb_action_t excb_test_action( + ex_cb_class_t cb_class, + void *refcon, + const ex_cb_state_t *state + ) +{ + ex_cb_state_t *context = (ex_cb_state_t *)refcon; + + if ((NULL == refcon) || (NULL == state)) + { + return EXCB_ACTION_TEST_FAIL; + } + + context->far = state->far; + + switch (cb_class) + { + case EXCB_CLASS_TEST1: + return EXCB_ACTION_RERUN; + case EXCB_CLASS_TEST2: + return EXCB_ACTION_NONE; + default: + return EXCB_ACTION_TEST_FAIL; + } +} + + +kern_return_t +ex_cb_test() +{ + const vm_offset_t far1 = 0xdead0001; + const vm_offset_t far2 = 0xdead0002; + kern_return_t kr; + ex_cb_state_t test_context_1 = {0xdeadbeef}; + ex_cb_state_t test_context_2 = {0xdeadbeef}; + ex_cb_action_t action; + + T_LOG("Testing Exception Callback."); + + T_LOG("Running registration test."); + + kr = ex_cb_register(EXCB_CLASS_TEST1, &excb_test_action, &test_context_1); + T_ASSERT(KERN_SUCCESS == kr, "First registration of TEST1 exception callback"); + kr = ex_cb_register(EXCB_CLASS_TEST2, &excb_test_action, &test_context_2); + T_ASSERT(KERN_SUCCESS == kr, "First registration of TEST2 exception callback"); + + kr = ex_cb_register(EXCB_CLASS_TEST2, &excb_test_action, &test_context_2); + T_ASSERT(KERN_SUCCESS != kr, "Second registration of TEST2 exception callback"); + kr = ex_cb_register(EXCB_CLASS_TEST1, &excb_test_action, &test_context_1); + T_ASSERT(KERN_SUCCESS != kr, "Second registration of TEST1 exception callback"); + + T_LOG("Running invocation test."); + + action = ex_cb_invoke(EXCB_CLASS_TEST1, far1); + T_ASSERT(EXCB_ACTION_RERUN == action, NULL); + T_ASSERT(far1 == test_context_1.far, NULL); + + action = ex_cb_invoke(EXCB_CLASS_TEST2, far2); + T_ASSERT(EXCB_ACTION_NONE == action, NULL); + T_ASSERT(far2 == test_context_2.far, NULL); + + action = ex_cb_invoke(EXCB_CLASS_TEST3, 0); + T_ASSERT(EXCB_ACTION_NONE == action, NULL); + + return KERN_SUCCESS; +} + +#if __ARM_PAN_AVAILABLE__ +kern_return_t +arm64_pan_test() +{ + unsigned long last_pan_config; + vm_offset_t priv_addr = _COMM_PAGE_SIGNATURE; + + T_LOG("Testing PAN."); + + last_pan_config = __builtin_arm_rsr("pan"); + if (!last_pan_config) { + T_ASSERT(!arm_pan_enabled, "PAN is not enabled even though it is configured to be"); + __builtin_arm_wsr("pan", 1); + } + + T_ASSERT(__builtin_arm_rsr("pan") != 0, NULL); + + // convert priv_addr to one that is accessible from user mode + pan_test_addr = priv_addr + _COMM_PAGE64_BASE_ADDRESS - + _COMM_PAGE_START_ADDRESS; + + // Below should trigger a PAN exception as pan_test_addr is accessible + // in user mode + // The exception handler, upon recognizing the fault address is pan_test_addr, + // will disable PAN and rerun this instruction successfully + T_ASSERT(*(char *)pan_test_addr == *(char *)priv_addr, NULL); + pan_test_addr = 0; + + T_ASSERT(__builtin_arm_rsr("pan") == 0, NULL); + + // restore previous PAN config value + if (last_pan_config) + __builtin_arm_wsr("pan", 1); + + return KERN_SUCCESS; +} +#endif + + +kern_return_t +arm64_lock_test() +{ + return lt_test_locks(); +} + +kern_return_t +arm64_munger_test() +{ + mt_test_mungers(); + return 0; +} + diff --git a/osfmk/arm64/proc_reg.h b/osfmk/arm64/proc_reg.h new file mode 100644 index 000000000..370ed6347 --- /dev/null +++ b/osfmk/arm64/proc_reg.h @@ -0,0 +1,1401 @@ +/* + * Copyright (c) 2007-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Processor registers for ARM64 + */ +#ifndef _ARM64_PROC_REG_H_ +#define _ARM64_PROC_REG_H_ + +#include + +/* + * 64-bit Program Status Register (PSR64) + * + * 31 27 23 22 21 20 19 10 9 5 4 0 + * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+ + * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0| M | + * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+ + * + * where: + * NZCV Comparison flags + * PAN Privileged Access Never + * SS Single step + * IL Illegal state + * DAIF Interrupt masks + * M Mode field + */ + +#define PSR64_NZCV_SHIFT 28 +#define PSR64_NZCV_MASK (1 << PSR64_NZCV_SHIFT) + +#define PSR64_N_SHIFT 31 +#define PSR64_N (1 << PSR64_N_SHIFT) + +#define PSR64_Z_SHIFT 30 +#define PSR64_Z (1 << PSR64_Z_SHIFT) + +#define PSR64_C_SHIFT 29 +#define PSR64_C (1 << PSR64_C_SHIFT) + +#define PSR64_V_SHIFT 28 +#define PSR64_V (1 << PSR64_V_SHIFT) + +#define PSR64_PAN_SHIFT 22 +#define PSR64_PAN (1 << PSR64_PAN_SHIFT) + +#define PSR64_SS_SHIFT 21 +#define PSR64_SS (1 << PSR64_SS_SHIFT) + +#define PSR64_IL_SHIFT 20 +#define PSR64_IL (1 << PSR64_IL_SHIFT) + +/* + * msr DAIF, Xn and mrs Xn, DAIF transfer into + * and out of bits 9:6 + */ +#define DAIF_DEBUG_SHIFT 9 +#define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT) + +#define DAIF_ASYNC_SHIFT 8 +#define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT) + +#define DAIF_IRQF_SHIFT 7 +#define DAIF_IRQF (1 << DAIF_IRQF_SHIFT) + +#define DAIF_FIQF_SHIFT 6 +#define DAIF_FIQF (1 << DAIF_FIQF_SHIFT) + +#define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) +#define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) + +#define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF)) + +/* + * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer + * from bits 3:0. + */ +#define DAIFSC_DEBUGF (1 << 3) +#define DAIFSC_ASYNCF (1 << 2) +#define DAIFSC_IRQF (1 << 1) +#define DAIFSC_FIQF (1 << 0) +#define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) +#define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) + +/* + * ARM64_TODO: unify with ARM? + */ +#define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */ + +#define PSR64_MODE_MASK 0x1F + +#define PSR64_MODE_USER32_THUMB 0x20 + +#define PSR64_MODE_RW_SHIFT 4 +#define PSR64_MODE_RW_64 0 +#define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT) + +#define PSR64_MODE_EL_SHIFT 2 +#define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT) +#define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT) +#define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT) +#define PSR64_MODE_EL0 0 + +#define PSR64_MODE_SPX 0x1 +#define PSR64_MODE_SP0 0 + +#define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0) +#define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0) +#define PSR64_KERNEL_DEFAULT (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0) + +#define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL1) +#define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0) + +#define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32)) +#define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32)) + + + +/* + * System Control Register (SCTLR) + */ + +#define SCTLR_RESERVED ((3 << 28) | (1 << 22) | (1 << 20) | (1 << 11)) + +// 31 PACIA_ENABLED AddPACIA and AuthIA functions enabled +#define SCTLR_PACIA_ENABLED (1 << 31) +// 30 PACIB_ENABLED AddPACIB and AuthIB functions enabled +#define SCTLR_PACIB_ENABLED (1 << 30) +// 29:28 RES1 11 +// 27 PACDA_ENABLED AddPACDA and AuthDA functions enabled +#define SCTLR_PACDA_ENABLED (1 << 27) + +// 26 UCI User Cache Instructions +#define SCTLR_UCI_ENABLED (1 << 26) + +// 25 EE Exception Endianness +#define SCTLR_EE_BIG_ENDIAN (1 << 25) + +// 24 E0E EL0 Endianness +#define SCTLR_E0E_BIG_ENDIAN (1 << 24) + +// 23 SPAN Set PAN +#define SCTLR_PAN_UNCHANGED (1 << 23) + +// 22 RES1 1 +// 21 RES0 0 +// 20 RES1 1 + +// 19 WXN Writeable implies eXecute Never +#define SCTLR_WXN_ENABLED (1 << 19) + +// 18 nTWE Not trap WFE from EL0 +#define SCTLR_nTWE_WFE_ENABLED (1 << 18) + +// 17 RES0 0 + +// 16 nTWI Not trap WFI from EL0 +#define SCTRL_nTWI_WFI_ENABLED (1 << 16) + +// 15 UCT User Cache Type register (CTR_EL0) +#define SCTLR_UCT_ENABLED (1 << 15) + +// 14 DZE User Data Cache Zero (DC ZVA) +#define SCTLR_DZE_ENABLED (1 << 14) + +// 13 RES0 0 + +// 12 I Instruction cache enable +#define SCTLR_I_ENABLED (1 << 12) + +// 11 RES1 1 +// 10 RES0 0 + +// 9 UMA User Mask Access +#define SCTLR_UMA_ENABLED (1 << 9) + +// 8 SED SETEND Disable +#define SCTLR_SED_DISABLED (1 << 8) + +// 7 ITD IT Disable +#define SCTLR_ITD_DISABLED (1 << 7) + +// 6 RES0 0 + +// 5 CP15BEN CP15 Barrier ENable +#define SCTLR_CP15BEN_ENABLED (1 << 5) + +// 4 SA0 Stack Alignment check for EL0 +#define SCTLR_SA0_ENABLED (1 << 4) + +// 3 SA Stack Alignment check +#define SCTLR_SA_ENABLED (1 << 3) + +// 2 C Cache enable +#define SCTLR_C_ENABLED (1 << 2) + +// 1 A Alignment check +#define SCTLR_A_ENABLED (1 << 1) + +// 0 M MMU enable +#define SCTLR_M_ENABLED (1 << 0) + +#define SCTLR_PAC_DEFAULT 0 + +#define SCTLR_EL1_DEFAULT (SCTLR_PAC_DEFAULT | SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \ + SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED | \ + SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_PAN_UNCHANGED | \ + SCTLR_C_ENABLED | SCTLR_M_ENABLED) + + + +/* + * Coprocessor Access Control Register (CPACR) + * + * 31 28 27 22 21 20 19 0 + * +---+---+------+------+--------------------+ + * |000|TTA|000000| FPEN |00000000000000000000| + * +---+---+------+------+--------------------+ + * + * where: + * TTA Trace trap + * FPEN Floating point enable + */ +#define CPACR_TTA_SHIFT 28 +#define CPACR_TTA (1 << CPACR_TTA_SHIFT) + +#define CPACR_FPEN_SHIFT 20 +#define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT) +#define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT) + +/* + * FPSR: Floating Point Status Register + * + * 31 30 29 28 27 26 7 6 4 3 2 1 0 + * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ + * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC| + * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ + */ + +#define FPSR_N_SHIFT 31 +#define FPSR_Z_SHIFT 30 +#define FPSR_C_SHIFT 29 +#define FPSR_V_SHIFT 28 +#define FPSR_QC_SHIFT 27 +#define FPSR_IDC_SHIFT 7 +#define FPSR_IXC_SHIFT 4 +#define FPSR_UFC_SHIFT 3 +#define FPSR_OFC_SHIFT 2 +#define FPSR_DZC_SHIFT 1 +#define FPSR_IOC_SHIFT 0 +#define FPSR_N (1 << FPSR_N_SHIFT) +#define FPSR_Z (1 << FPSR_Z_SHIFT) +#define FPSR_C (1 << FPSR_C_SHIFT) +#define FPSR_V (1 << FPSR_V_SHIFT) +#define FPSR_QC (1 << FPSR_QC_SHIFT) +#define FPSR_IDC (1 << FPSR_IDC_SHIFT) +#define FPSR_IXC (1 << FPSR_IXC_SHIFT) +#define FPSR_UFC (1 << FPSR_UFC_SHIFT) +#define FPSR_OFC (1 << FPSR_OFC_SHIFT) +#define FPSR_DZC (1 << FPSR_DZC_SHIFT) +#define FPSR_IOC (1 << FPSR_IOC_SHIFT) + +/* + * A mask for all for all of the bits that are not RAZ for FPSR; this + * is primarily for converting between a 32-bit view of NEON state + * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). + */ +#define FPSR_MASK (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | \ + FPSR_IDC | FPSR_IXC | FPSR_UFC | FPSR_OFC | \ + FPSR_DZC | FPSR_IOC) + +/* + * FPCR: Floating Point Control Register + * + * 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0 + * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ + * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000| + * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ + */ + +#define FPCR_AHP_SHIFT 26 +#define FPCR_DN_SHIFT 25 +#define FPCR_FZ_SHIFT 24 +#define FPCR_RMODE_SHIFT 22 +#define FPCR_STRIDE_SHIFT 20 +#define FPCR_LEN_SHIFT 16 +#define FPCR_IDE_SHIFT 15 +#define FPCR_IXE_SHIFT 12 +#define FPCR_UFE_SHIFT 11 +#define FPCR_OFE_SHIFT 10 +#define FPCR_DZE_SHIFT 9 +#define FPCR_IOE_SHIFT 8 +#define FPCR_AHP (1 << FPCR_AHP_SHIFT) +#define FPCR_DN (1 << FPCR_DN_SHIFT) +#define FPCR_FZ (1 << FPCR_FZ_SHIFT) +#define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT) +#define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT) +#define FPCR_LEN (0x7 << FPCR_LEN_SHIFT) +#define FPCR_IDE (1 << FPCR_IDE_SHIFT) +#define FPCR_IXE (1 << FPCR_IXE_SHIFT) +#define FPCR_UFE (1 << FPCR_UFE_SHIFT) +#define FPCR_OFE (1 << FPCR_OFE_SHIFT) +#define FPCR_DZE (1 << FPCR_DZE_SHIFT) +#define FPCR_IOE (1 << FPCR_IOE_SHIFT) +#define FPCR_DEFAULT (FPCR_DN) +#define FPCR_DEFAULT_32 (FPCR_DN|FPCR_FZ) + +/* + * A mask for all for all of the bits that are not RAZ for FPCR; this + * is primarily for converting between a 32-bit view of NEON state + * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). + */ +#define FPCR_MASK (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | \ + FPCR_STRIDE | FPCR_LEN | FPCR_IDE | FPCR_IXE | \ + FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE) + +/* + * Translation Control Register (TCR) + * + * Legacy: + * + * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 + * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ + * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| + * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ + * + * Current (with 16KB granule support): + * + * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 + * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ + * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| + * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ + * + * TBI1 Top Byte Ignored for TTBR1 region + * TBI0 Top Byte Ignored for TTBR0 region + * AS ASID Size + * IPS Physical Address Size limit + * TG1 Granule Size for TTBR1 region + * SH1 Shareability for TTBR1 region + * ORGN1 Outer Cacheability for TTBR1 region + * IRGN1 Inner Cacheability for TTBR1 region + * EPD1 Translation table walk disable for TTBR1 + * A1 ASID selection from TTBR1 enable + * T1SZ Virtual address size for TTBR1 + * TG0 Granule Size for TTBR0 region + * SH0 Shareability for TTBR0 region + * ORGN0 Outer Cacheability for TTBR0 region + * IRGN0 Inner Cacheability for TTBR0 region + * T0SZ Virtual address size for TTBR0 + */ + +#define TCR_T0SZ_SHIFT 0ULL +#define TCR_TSZ_BITS 6ULL +#define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL) + +#define TCR_IRGN0_SHIFT 8ULL +#define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT) +#define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT) +#define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT) +#define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT) + +#define TCR_ORGN0_SHIFT 10ULL +#define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT) +#define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT) +#define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT) +#define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT) + +#define TCR_SH0_SHIFT 12ULL +#define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT) +#define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT) +#define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT) + +#define TCR_TG0_GRANULE_SHIFT (14ULL) + +#define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT) +#define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT) +#define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT) + +#if __ARM_16K_PG__ +#define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB) +#else +#define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB) +#endif + +#define TCR_T1SZ_SHIFT 16ULL + +#define TCR_A1_ASID1 (1ULL << 22ULL) +#define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL) + +#define TCR_IRGN1_SHIFT 24ULL +#define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT) +#define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT) +#define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT) +#define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT) + +#define TCR_ORGN1_SHIFT 26ULL +#define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT) +#define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT) +#define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT) +#define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT) + +#define TCR_SH1_SHIFT 28ULL +#define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT) +#define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT) +#define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT) + +#define TCR_TG1_GRANULE_SHIFT 30ULL + +#define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT) +#define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT) +#define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT) + +#if __ARM_16K_PG__ +#define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB) +#else +#define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB) +#endif + +#define TCR_IPS_SHIFT 32ULL +#define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT) +#define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT) +#define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT) +#define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT) +#define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT) +#define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT) + +#define TCR_AS_16BIT_ASID (1ULL << 36) +#define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37) +#define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38) + +/* + * Multiprocessor Affinity Register (MPIDR_EL1) + * + * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+ + * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0| + * +---------------------------------+--+-----+--+-----+----+----+ + * + * where + * U Uniprocessor + * MT Multi-threading at lowest affinity level + * Aff2 "1" - PCORE, "0" - ECORE + * Aff1 Cluster ID + * Aff0 CPU ID + */ +#define MPIDR_PNE_SHIFT 16 // pcore not ecore +#define MPIDR_PNE (1 << MPIDR_PNE_SHIFT) +#define MPIDR_AFF0_MASK 0xFF +#define MPIDR_AFF1_MASK 0xFF00 +#define MPIDR_AFF2_MASK 0xFF0000 + +/* + * We currently use a 3 level page table (rather than the full 4 + * level page table). As a result, we do not have the full 48-bits + * of address space per TTBR (although the 16KB granule size lets us + * get very close). + */ +#if __ARM64_TWO_LEVEL_PMAP__ && !__ARM_16K_PG__ +#error ARM64 does not currently support a 2 level page table with 4KB pages +#endif /* __ARM64_TWO_LEVEL_PMAP__ */ + +/* + * TXSZ indicates the size of the range a TTBR covers. Currently, + * we support the following: + * + * 4KB pages, full page L1: 39 bit range. + * 4KB pages, sub-page L1: 36 bit range. + * 16KB pages, full page L1: 47 bit range. + * 16KB pages, sub-page L1: 37 bit range. + * 16KB pages, two level page tables: 36 bit range. + */ +#ifdef __ARM_16K_PG__ +#if __ARM64_TWO_LEVEL_PMAP__ +#define T0SZ_BOOT 28ULL +#elif __ARM64_PMAP_SUBPAGE_L1__ +#define T0SZ_BOOT 27ULL +#else /* __ARM64_TWO_LEVEL_PMAP__ */ +#define T0SZ_BOOT 17ULL +#endif /* __ARM64_TWO_LEVEL_PMAP__ */ +#else /* __ARM_16K_PG__ */ +#if __ARM64_PMAP_SUBPAGE_L1__ +#define T0SZ_BOOT 28ULL +#else /* __ARM64_PMAP_SUBPAGE_L1__ */ +#define T0SZ_BOOT 25ULL +#endif /* __ARM64_PMAP_SUBPAGE_L1__ */ +#endif /* __ARM_16K_PG__ */ + +#if defined(APPLE_ARM64_ARCH_FAMILY) +/* T0SZ must be the same as T1SZ */ +#define T1SZ_BOOT T0SZ_BOOT +#else /* defined(APPLE_ARM64_ARCH_FAMILY) */ +#ifdef __ARM_16K_PG__ +#if __ARM64_TWO_LEVEL_PMAP__ +#define T1SZ_BOOT 28ULL +#elif __ARM64_PMAP_SUBPAGE_L1__ +#define T1SZ_BOOT 27ULL +#else /* __ARM64_TWO_LEVEL_PMAP__ */ +#define T1SZ_BOOT 17ULL +#endif /* __ARM64_TWO_LEVEL_PMAP__ */ +#else /* __ARM_16K_PG__ */ +#if __ARM64_PMAP_SUBPAGE_L1__ +#define T1SZ_BOOT 28ULL +#else /* __ARM64_PMAP_SUBPAGE_L1__ */ +#define T1SZ_BOOT 25ULL +#endif /*__ARM64_PMAP_SUBPAGE_L1__*/ +#endif /* __ARM_16K_PG__ */ +#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ + +#define TCR_EL1_BOOT (TCR_IPS_40BITS | \ + TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE) |\ + TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | TCR_IRGN1_WRITEBACK | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG1_GRANULE_SIZE)) + +/* + * Translation Table Base Register (TTBR) + * + * 63 48 47 x x-1 0 + * +--------+------------------+------+ + * | ASID | Base Address | zero | + * +--------+------------------+------+ + * + */ +#define TTBR_ASID_SHIFT 48 +#define TTBR_ASID_MASK 0xffff000000000000 + +#define TTBR_BADDR_MASK 0x0000ffffffffffff + +/* + * Memory Attribute Indirection Register + * + * 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0 + * +-------+-------+-------+-------+-------+-------+-------+-------+ + * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 | + * +-------+-------+-------+-------+-------+-------+-------+-------+ + * + */ + +#define MAIR_ATTR_SHIFT(x) (8*(x)) + +/* Strongly ordered or device memory attributes */ +#define MAIR_OUTER_STRONGLY_ORDERED 0x0 +#define MAIR_OUTER_DEVICE 0x0 + +#define MAIR_INNER_STRONGLY_ORDERED 0x0 +#define MAIR_INNER_DEVICE 0x4 + +/* Normal memory attributes */ +#define MAIR_OUTER_NON_CACHEABLE 0x40 +#define MAIR_OUTER_WRITE_THROUGH 0x80 +#define MAIR_OUTER_WRITE_BACK 0xc0 + +#define MAIR_INNER_NON_CACHEABLE 0x4 +#define MAIR_INNER_WRITE_THROUGH 0x8 +#define MAIR_INNER_WRITE_BACK 0xc + +/* Allocate policy for cacheable memory */ +#define MAIR_OUTER_WRITE_ALLOCATE 0x10 +#define MAIR_OUTER_READ_ALLOCATE 0x20 + +#define MAIR_INNER_WRITE_ALLOCATE 0x1 +#define MAIR_INNER_READ_ALLOCATE 0x2 + +/* Memory Atribute Encoding */ + +/* Device memory types: + G (gathering): multiple reads/writes can be combined + R (reordering): reads or writes may reach device out of program order + E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes) +*/ +#define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */ +#define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */ +#define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */ +#define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */ +#define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */ +#define MAIR_INNERWRITEBACK 0x4F /* Normal Memory, Outer Non-Cacheable, Inner Write-back */ + + +/* + * ARM 4-level Page Table support - 2*1024TB (2^48) of address space + */ + + +/* + * Memory Attribute Index + */ +#define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled */ +#define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes */ +#define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled */ +#define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer */ +#define CACHE_ATTRINDX_INNERWRITEBACK 0x4 /* inner cache enabled, buffer enabled, write allocate */ +#define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes */ +#define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK + +/* + * Access protection bit values (TTEs and PTEs) + */ +#define AP_RWNA 0x0 /* priv=read-write, user=no-access */ +#define AP_RWRW 0x1 /* priv=read-write, user=read-write */ +#define AP_RONA 0x2 /* priv=read-only, user=no-access */ +#define AP_RORO 0x3 /* priv=read-only, user=read-only */ +#define AP_MASK 0x3 /* mask to find ap bits */ + +/* + * Shareability attributes + */ +#define SH_NONE 0x0 /* Non shareable */ +#define SH_NONE 0x0 /* Device shareable */ +#define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */ +#define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */ +#define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */ + + +/* + * ARM Page Granule + */ +#ifdef __ARM_16K_PG__ +#define ARM_PGSHIFT 14 +#else +#define ARM_PGSHIFT 12 +#endif +#define ARM_PGBYTES (1 << ARM_PGSHIFT) +#define ARM_PGMASK (ARM_PGBYTES-1) + + +/* + * L0 Translation table + * + * 4KB granule size: + * Each translation table is 4KB + * 512 64-bit entries of 512GB (2^39) of address space. + * Covers 256TB (2^48) of address space. + * + * 16KB granule size: + * Each translation table is 16KB + * 2 64-bit entries of 128TB (2^47) of address space. + * Covers 256TB (2^48) of address space. + */ + +#ifdef __ARM_16K_PG__ +#define ARM_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */ +#define ARM_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */ +#define ARM_TT_L0_SHIFT 47 /* page descriptor shift */ +#define ARM_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */ +#else +#define ARM_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */ +#define ARM_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */ +#define ARM_TT_L0_SHIFT 39 /* page descriptor shift */ +#define ARM_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */ +#endif + +/* + * L1 Translation table + * + * 4KB granule size: + * Each translation table is 4KB + * 512 64-bit entries of 1GB (2^30) of address space. + * Covers 512GB (2^39) of address space. + * + * 16KB granule size: + * Each translation table is 16KB + * 2048 64-bit entries of 64GB (2^36) of address space. + * Covers 128TB (2^47) of address space. + */ + +#ifdef __ARM_16K_PG__ +#define ARM_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */ +#define ARM_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */ +#define ARM_TT_L1_SHIFT 36 /* page descriptor shift */ +#ifdef __ARM64_PMAP_SUBPAGE_L1__ +/* This config supports 128GB per TTBR. */ +#define ARM_TT_L1_INDEX_MASK 0x0000001000000000ULL /* mask for getting index into L1 table from virtual address */ +#else +#define ARM_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */ +#endif +#else +#define ARM_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */ +#define ARM_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */ +#define ARM_TT_L1_SHIFT 30 /* page descriptor shift */ +#ifdef __ARM64_PMAP_SUBPAGE_L1__ +/* This config supports 64GB per TTBR. */ +#define ARM_TT_L1_INDEX_MASK 0x0000000fc0000000ULL /* mask for getting index into L1 table from virtual address */ +#else +#define ARM_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */ +#endif +#endif + +/* + * L2 Translation table + * + * 4KB granule size: + * Each translation table is 4KB + * 512 64-bit entries of 2MB (2^21) of address space. + * Covers 1GB (2^30) of address space. + * + * 16KB granule size: + * Each translation table is 16KB + * 2048 64-bit entries of 32MB (2^25) of address space. + * Covers 64GB (2^36) of address space. + */ + +#ifdef __ARM_16K_PG__ +#define ARM_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */ +#define ARM_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */ +#define ARM_TT_L2_SHIFT 25 /* page descriptor shift */ +#define ARM_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */ +#else +#define ARM_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */ +#define ARM_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */ +#define ARM_TT_L2_SHIFT 21 /* page descriptor shift */ +#define ARM_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */ +#endif + +/* + * L3 Translation table + * + * 4KB granule size: + * Each translation table is 4KB + * 512 64-bit entries of 4KB (2^12) of address space. + * Covers 2MB (2^21) of address space. + * + * 16KB granule size: + * Each translation table is 16KB + * 2048 64-bit entries of 16KB (2^14) of address space. + * Covers 32MB (2^25) of address space. + */ + +#ifdef __ARM_16K_PG__ +#define ARM_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ +#define ARM_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */ +#define ARM_TT_L3_SHIFT 14 /* page descriptor shift */ +#define ARM_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */ +#else +#define ARM_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ +#define ARM_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */ +#define ARM_TT_L3_SHIFT 12 /* page descriptor shift */ +#define ARM_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */ +#endif + +/* + * Convenience definitions for: + * ARM_TT_LEAF: The last level of the configured page table format. + * ARM_TT_TWIG: The second to last level of the configured page table format. + * + * My apologies to any botanists who may be reading this. + */ +#define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE +#define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK +#define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT +#define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK + +#define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE +#define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK +#define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT +#define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK + +/* + * 4KB granule size: + * + * Level 0 Translation Table Entry + * + * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * + * Level 1 Translation Table Entry + * + * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * + * Level 1 Translation Block Entry + * + * 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0 + * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ + * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| + * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ + * + * Level 2 Translation Table Entry + * + * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * + * Level 2 Translation Block Entry + * + * 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0 + * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ + * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| + * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ + * + * 16KB granule size: + * + * Level 0 Translation Table Entry + * + * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * + * Level 1 Translation Table Entry + * + * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * + * Level 2 Translation Table Entry + * + * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| + * +--+-----+--+---+-------+------+----------------------+-------+-+-+ + * + * Level 2 Translation Block Entry + * + * 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0 + * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ + * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| + * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ + * + * where: + * 'nG' notGlobal bit + * 'SH' Shareability field + * 'AP' access protection + * 'XN' eXecute Never bit + * 'PXN' Privilege eXecute Never bit + * 'NS' Non-Secure bit + * 'HINT' 16 entry continuguous output hint + * 'AttrIdx' Memory Attribute Index + */ + +#define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */ +#ifdef __ARM_16K_PG__ +#define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */ +#else +#define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */ +#endif + +#define ARM_TTE_MAX (TTE_PGENTRIES) + +#define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */ +#define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */ + +#define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */ + +#define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ +#define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */ +#define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */ +#define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL +#define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ + +#ifdef __ARM_16K_PG__ +/* Note that L0/L1 block entries are disallowed for the 16KB granule size; what are we doing with these? */ +#define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */ +#define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */ +#define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */ +#define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */ +#define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */ +#define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */ +#else +#define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */ +#define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */ +#define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */ +#define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */ +#define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */ +#define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */ +#endif + +#define ARM_TTE_BLOCK_APSHIFT 6 +#define ARM_TTE_BLOCK_AP(x) ((x)<> PTE_SHIFT) /* number of ptes per page */ +#else +#define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */ +#endif + +#define ARM_PTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */ + +/* markers for (invalid) PTE for a page sent to compressor */ +#define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */ +#define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */ +#define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL +#define ARM_PTE_IS_COMPRESSED(x) \ + ((((x) & 0x3) == 0) && /* PTE is not valid... */ \ + ((x) & ARM_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \ + ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \ + (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \ + &(x), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE))) + +#define ARM_PTE_TYPE 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ +#define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ +#define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */ +#define ARM_PTE_TYPE_MASK 0x0000000000000002ULL /* mask to get pte type */ + +#ifdef __ARM_16K_PG__ +/* TODO: What does the shift mean here? */ +#define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFC000ULL /* mask for 16KB page */ +#else +#define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* mask for 4KB page */ +#define ARM_PTE_PAGE_SHIFT 12 /* page shift for 4KB page */ +#endif + +#define ARM_PTE_AP(x) ((x) << 6) /* access protections */ +#define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */ +#define ARM_PTE_EXTRACT_AP(x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */ + +#define ARM_PTE_ATTRINDX(x) ((x) << 2) /* memory attributes index */ +#define ARM_PTE_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ + +#define ARM_PTE_SH(x) ((x) << 8) /* access shared */ +#define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */ + +#define ARM_PTE_AF 0x0000000000000400ULL /* value for access */ +#define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */ + +#define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */ +#define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ + +#define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */ +#define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ + +#define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */ +#define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */ + +#if __ARM_16K_PG__ +#define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */ +#define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */ +#define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */ +#define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */ +#else +#define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */ +#define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */ +#define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */ +#define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */ +#endif + +#define ARM_PTE_PNX 0x0020000000000000ULL /* value for no execute */ +#define ARM_PTE_PNXMASK 0x0020000000000000ULL /* no execute mask */ + +#define ARM_PTE_NX 0x0040000000000000ULL /* value for privilege no execute bit */ +#define ARM_PTE_NXMASK 0x0040000000000000ULL /* privilege execute mask */ + +#define ARM_PTE_WIRED 0x0080000000000000ULL /* value for software wired bit */ +#define ARM_PTE_WIRED_MASK 0x0080000000000000ULL /* software wired mask */ + +#define ARM_PTE_WRITEABLE 0x0100000000000000ULL /* value for software writeable bit */ +#define ARM_PTE_WRITEABLE_MASK 0x0100000000000000ULL /* software writeable mask */ + +#if CONFIG_PGTRACE +#define ARM_PTE_PGTRACE 0x0200000000000000ULL /* value for software trace bit */ +#define ARM_PTE_PGTRACE_MASK 0x0200000000000000ULL /* software trace mask */ +#endif + +#define ARM_PTE_BOOT_PAGE (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) \ + | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF) + +/* + * Exception Syndrome Register + * + * 31 26 25 24 0 + * +------+--+------------------+ + * | EC |IL| ISS | + * +------+--+------------------+ + * + * EC - Exception Class + * IL - Instruction Length + * ISS- Instruction Specific Syndrome + * + * Note: The ISS can have many forms. These are defined separately below. + */ + +#define ESR_EC_SHIFT 26 +#define ESR_EC_MASK (0x3F << ESR_EC_SHIFT) +#define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT) + +#define ESR_IL_SHIFT 25 +#define ESR_IL (1 << ESR_IL_SHIFT) + +#define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL)) + +#define ESR_ISS_MASK 0x01FFFFFF +#define ESR_ISS(x) (x & ESR_ISS_MASK) + +#ifdef __ASSEMBLER__ +/* Define only the classes we need to test in the exception vectors. */ +#define ESR_EC_IABORT_EL1 0x21 +#define ESR_EC_DABORT_EL1 0x25 +#define ESR_EC_SP_ALIGN 0x26 +#else +typedef enum { + ESR_EC_UNCATEGORIZED = 0x00, + ESR_EC_WFI_WFE = 0x01, + ESR_EC_MCR_MRC_CP15_TRAP = 0x03, + ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04, + ESR_EC_MCR_MRC_CP14_TRAP = 0x05, + ESR_EC_LDC_STC_CP14_TRAP = 0x06, + ESR_EC_TRAP_SIMD_FP = 0x07, + ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c, + ESR_EC_ILLEGAL_INSTR_SET = 0x0e, + ESR_EC_SVC_32 = 0x11, + ESR_EC_SVC_64 = 0x15, + ESR_EC_MSR_TRAP = 0x18, + ESR_EC_IABORT_EL0 = 0x20, + ESR_EC_IABORT_EL1 = 0x21, + ESR_EC_PC_ALIGN = 0x22, + ESR_EC_DABORT_EL0 = 0x24, + ESR_EC_DABORT_EL1 = 0x25, + ESR_EC_SP_ALIGN = 0x26, + ESR_EC_FLOATING_POINT_32 = 0x28, + ESR_EC_FLOATING_POINT_64 = 0x2C, + ESR_EC_BKPT_REG_MATCH_EL0 = 0x30, // Breakpoint Debug event taken to the EL from a lower EL. + ESR_EC_BKPT_REG_MATCH_EL1 = 0x31, // Breakpoint Debug event taken to the EL from the EL. + ESR_EC_SW_STEP_DEBUG_EL0 = 0x32, // Software Step Debug event taken to the EL from a lower EL. + ESR_EC_SW_STEP_DEBUG_EL1 = 0x33, // Software Step Debug event taken to the EL from the EL. + ESR_EC_WATCHPT_MATCH_EL0 = 0x34, // Watchpoint Debug event taken to the EL from a lower EL. + ESR_EC_WATCHPT_MATCH_EL1 = 0x35, // Watchpoint Debug event taken to the EL from the EL. + ESR_EC_BKPT_AARCH32 = 0x38, + ESR_EC_BRK_AARCH64 = 0x3C +} esr_exception_class_t; + +typedef enum { + FSC_TRANSLATION_FAULT_L0 = 0x04, + FSC_TRANSLATION_FAULT_L1 = 0x05, + FSC_TRANSLATION_FAULT_L2 = 0x06, + FSC_TRANSLATION_FAULT_L3 = 0x07, + FSC_ACCESS_FLAG_FAULT_L1 = 0x09, + FSC_ACCESS_FLAG_FAULT_L2 = 0x0A, + FSC_ACCESS_FLAG_FAULT_L3 = 0x0B, + FSC_PERMISSION_FAULT_L1 = 0x0D, + FSC_PERMISSION_FAULT_L2 = 0x0E, + FSC_PERMISSION_FAULT_L3 = 0x0F, + FSC_SYNC_EXT_ABORT = 0x10, + FSC_ASYNC_EXT_ABORT = 0x11, + FSC_SYNC_EXT_ABORT_TT_L1 = 0x15, + FSC_SYNC_EXT_ABORT_TT_L2 = 0x16, + FSC_SYNC_EXT_ABORT_TT_L3 = 0x17, + FSC_SYNC_PARITY = 0x18, + FSC_ASYNC_PARITY = 0x19, + FSC_SYNC_PARITY_TT_L1 = 0x1D, + FSC_SYNC_PARITY_TT_L2 = 0x1E, + FSC_SYNC_PARITY_TT_L3 = 0x1F, + FSC_ALIGNMENT_FAULT = 0x21, + FSC_DEBUG_FAULT = 0x22 +} fault_status_t; +#endif /* ASSEMBLER */ + +/* + * Software step debug event ISS (EL1) + * 24 23 6 5 0 + * +---+-----------------+--+------+ + * |ISV|00000000000000000|EX| IFSC | + * +---+-----------------+--+------+ + * + * where: + * ISV Instruction syndrome valid + * EX Exclusive access + * IFSC Instruction Fault Status Code + */ + +#define ISS_SSDE_ISV_SHIFT 24 +#define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT) + +#define ISS_SSDE_EX_SHIFT 6 +#define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT) + +#define ISS_SSDE_FSC_MASK 0x3F +#define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK) + +/* + * Instruction Abort ISS (EL1) + * 24 10 9 5 0 + * +---------------+--+---+------+ + * |000000000000000|EA|000| IFSC | + * +---------------+--+---+------+ + * + * where: + * EA External Abort type + * IFSC Instruction Fault Status Code + */ + +#define ISS_IA_EA_SHIFT 9 +#define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT) + +#define ISS_IA_FSC_MASK 0x3F +#define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK) + + +/* + * Data Abort ISS (EL1) + * + * 24 9 8 7 6 5 0 + * +---------------+--+--+-+---+----+ + * |000000000000000|EA|CM|0|WnR|DFSC| + * +---------------+--+--+-+---+----+ + * + * where: + * EA External Abort type + * CM Cache Maintenance operation + * WnR Write not Read + * DFSC Data Fault Status Code + */ +#define ISS_DA_EA_SHIFT 9 +#define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT) + +#define ISS_DA_CM_SHIFT 8 +#define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT) + +#define ISS_DA_WNR_SHIFT 6 +#define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT) + +#define ISS_DA_FSC_MASK 0x3F +#define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK) + +/* + * Physical Address Register (EL1) + */ +#define PAR_F_SHIFT 0 +#define PAR_F (0x1 << PAR_F_SHIFT) + +#define PLATFORM_SYSCALL_TRAP_NO 0x80000000 + +#define ARM64_SYSCALL_CODE_REG_NUM (16) + +#define ARM64_CLINE_SHIFT 6 + +#if defined(APPLE_ARM64_ARCH_FAMILY) +#define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */ +#define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */ +#endif + +/* + * Timer definitions. + */ +#define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */ +#define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */ +#define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */ +#define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */ +#define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */ +#define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */ +#define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to physical timebase + frequency reg enabled */ +#define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to virtual timebase + frequency reg enabled */ + +#define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */ +#define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */ +#define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */ + +#define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS +#define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED +#define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE + +/* + * At present all other uses of ARM_DBG_* are shared bit compatibly with the 32bit definitons. + * (cf. osfmk/arm/proc_reg.h) + */ +#define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */ + +#define MIDR_EL1_REV_SHIFT 0 +#define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT) +#define MIDR_EL1_PNUM_SHIFT 4 +#define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT) +#define MIDR_EL1_ARCH_SHIFT 16 +#define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT) +#define MIDR_EL1_VAR_SHIFT 20 +#define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT) +#define MIDR_EL1_IMP_SHIFT 24 +#define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT) + +/* + * CoreSight debug registers + */ +#define CORESIGHT_ED 0 +#define CORESIGHT_CTI 1 +#define CORESIGHT_PMU 2 +#define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */ + +#define CORESIGHT_OFFSET(x) ((x) * 0x10000) +#define CORESIGHT_REGIONS 4 +#define CORESIGHT_SIZE 0x1000 + + +/* + * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0 + * + * 63 24 23 20 19 16 15 12 11 8 7 4 3 0 + * +----------+--------+------+------+------+-----+------+ + * | reserved | atomic |crc32 | sha2 | sha1 | aes | res0 | + * +----------+--------+------+------+------+-----+------+ + */ + +#define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20 +#define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) +#define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) + +#define ID_AA64ISAR0_EL1_CRC32_OFFSET 16 +#define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET) +#define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET) + +#define ID_AA64ISAR0_EL1_SHA2_OFFSET 12 +#define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET) +#define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) + +#define ID_AA64ISAR0_EL1_SHA1_OFFSET 8 +#define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET) +#define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET) + +#define ID_AA64ISAR0_EL1_AES_OFFSET 4 +#define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET) +#define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET) +#define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET) + + + +#ifdef __ASSEMBLER__ + +/* + * Compute CPU version: + * Version is constructed as [4 bits of MIDR variant]:[4 bits of MIDR revision] + * + * Where the "variant" is the major number and the "revision" is the minor number. + * + * For example: + * Cyclone A0 is variant 0, revision 0, i.e. 0. + * Cyclone B0 is variant 1, revision 0, i.e. 0x10 + * $0 - register to place value in + */ +.macro GET_MIDR_CPU_VERSION + mrs $0, MIDR_EL1 // Read MIDR_EL1 for CPUID + bfi $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #4 // move bits 3:0 (revision) to 19:16 (below variant) to get values adjacent + ubfx $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #8 // And extract the concatenated bitstring to beginning of register +.endmacro + +/* + * To apply a workaround for CPU versions less than a given value + * (e.g. earlier than when a fix arrived) + * + * $0 - scratch register1 + * $1 - version at which to stop applying workaround + * $2 - label to branch to (at end of workaround) + */ +.macro SKIP_IF_CPU_VERSION_GREATER_OR_EQUAL + GET_MIDR_CPU_VERSION $0 + cmp $0, $1 + b.pl $2 // Unsigned "greater or equal" +.endmacro + +/* + * To apply a workaround for CPU versions greater than a given value + * (e.g. starting when a bug was introduced) + * + * $0 - scratch register1 + * $1 - version at which to stop applying workaround + * $2 - label to branch to (at end of workaround) + */ +.macro SKIP_IF_CPU_VERSION_LESS_THAN + GET_MIDR_CPU_VERSION $0 + cmp $0, $1 + b.mi $2 // Unsigned "strictly less than" +.endmacro + +#endif /* __ASSEMBLER__ */ + +#define MSR(reg,src) __asm__ volatile ("msr " reg ", %0" :: "r" (src)) +#define MRS(dest,reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest)) + + +#endif /* _ARM64_PROC_REG_H_ */ diff --git a/osfmk/arm64/sleh.c b/osfmk/arm64/sleh.c new file mode 100644 index 000000000..ba7484a83 --- /dev/null +++ b/osfmk/arm64/sleh.c @@ -0,0 +1,1456 @@ +/* + * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include /* for IS_ARM_GDB_TRAP() et al */ +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include + +#include + +#include +#if CONFIG_TELEMETRY +#include +#endif + +#include + +#ifndef __arm64__ +#error Should only be compiling for arm64. +#endif + +#define TEST_CONTEXT32_SANITY(context) \ + (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \ + context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT) + +#define TEST_CONTEXT64_SANITY(context) \ + (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \ + context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT) + +#define ASSERT_CONTEXT_SANITY(context) \ + assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context)) + + +#define COPYIN(src, dst, size) \ + (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \ + copyin_kern(src, dst, size) \ + : \ + copyin(src, dst, size) + +#define COPYOUT(src, dst, size) \ + (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \ + copyout_kern(src, dst, size) \ + : \ + copyout(src, dst, size) + +// Below is for concatenating a string param to a string literal +#define STR1(x) #x +#define STR(x) STR1(x) + +void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss); + +void sleh_synchronous_sp1(arm_context_t *, uint32_t, vm_offset_t); +void sleh_synchronous(arm_context_t *, uint32_t, vm_offset_t); +void sleh_irq(arm_saved_state_t *); +void sleh_fiq(arm_saved_state_t *); +void sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far); +void sleh_invalid_stack(arm_context_t *context, uint32_t esr, vm_offset_t far); + +static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type); +static void sleh_interrupt_handler_epilogue(void); + +static void handle_svc(arm_saved_state_t *); +static void handle_mach_absolute_time_trap(arm_saved_state_t *); +static void handle_mach_continuous_time_trap(arm_saved_state_t *); + +static void handle_msr_trap(arm_saved_state_t *state, uint32_t iss); + +extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, boolean_t); + +static void handle_uncategorized(arm_saved_state_t *, boolean_t); +static void handle_breakpoint(arm_saved_state_t *); + +typedef void(*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *); +static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *); +static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *); + +static int is_vm_fault(fault_status_t); +static int is_alignment_fault(fault_status_t); + +typedef void(*abort_handler_t)(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, vm_offset_t); +static void handle_user_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, vm_offset_t); +static void handle_kernel_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, vm_offset_t); + +static void handle_pc_align(arm_saved_state_t *ss); +static void handle_sp_align(arm_saved_state_t *ss); +static void handle_sw_step_debug(arm_saved_state_t *ss); +static void handle_wf_trap(arm_saved_state_t *ss); + +static void handle_watchpoint(vm_offset_t fault_addr); + +static void handle_abort(arm_saved_state_t *, uint32_t, vm_offset_t, vm_offset_t, abort_inspector_t, abort_handler_t); + +static void handle_user_trapped_instruction32(arm_saved_state_t *, uint32_t esr); + +static void handle_simd_trap(arm_saved_state_t *, uint32_t esr); + +extern void mach_kauth_cred_uthread_update(void); +void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number); + +struct uthread; +struct proc; + +extern void +unix_syscall(struct arm_saved_state * regs, thread_t thread_act, + struct uthread * uthread, struct proc * proc); + +extern void +mach_syscall(struct arm_saved_state*); + +volatile perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */ + +#if CONFIG_DTRACE +extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs); +extern boolean_t dtrace_tally_fault(user_addr_t); + +/* Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy and paste the trap instructions + over from that file. Need to keep these in sync! */ +#define FASTTRAP_ARM32_INSTR 0xe7ffdefc +#define FASTTRAP_THUMB32_INSTR 0xdefc +#define FASTTRAP_ARM64_INSTR 0xe7eeee7e + +#define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb +#define FASTTRAP_THUMB32_RET_INSTR 0xdefb +#define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d + +/* See */ +perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */ +#endif + +#if CONFIG_PGTRACE +extern boolean_t pgtrace_enabled; +#endif + +#if __ARM_PAN_AVAILABLE__ +extern boolean_t arm_pan_enabled; +#endif + +#if defined(APPLECYCLONE) +#define CPU_NAME "Cyclone" +#elif defined(APPLETYPHOON) +#define CPU_NAME "Typhoon" +#elif defined(APPLETWISTER) +#define CPU_NAME "Twister" +#elif defined(APPLEHURRICANE) +#define CPU_NAME "Hurricane" +#else +#define CPU_NAME "Unknown" +#endif + +#if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT)) +#define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400) +#define ESR_WT_REASON(esr) ((esr) & 0xff) + +#define WT_REASON_NONE 0 +#define WT_REASON_INTEGRITY_FAIL 1 +#define WT_REASON_BAD_SYSCALL 2 +#define WT_REASON_NOT_LOCKED 3 +#define WT_REASON_ALREADY_LOCKED 4 +#define WT_REASON_SW_REQ 5 +#define WT_REASON_PT_INVALID 6 +#define WT_REASON_PT_VIOLATION 7 +#define WT_REASON_REG_VIOLATION 8 +#endif + + +static inline unsigned +__ror(unsigned value, unsigned shift) +{ + return (((unsigned)(value) >> (unsigned)(shift)) | + (unsigned)(value) << ((unsigned)(sizeof(unsigned) * CHAR_BIT) - (unsigned)(shift))); +} + +static void +arm64_implementation_specific_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far) +{ +#if defined(APPLE_ARM64_ARCH_FAMILY) + uint64_t fed_err_sts, mmu_err_sts, lsu_err_sts; +#if defined(NO_ECORE) + uint64_t l2c_err_sts, l2c_err_adr, l2c_err_inf; + + mmu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_MMU_ERR_STS)); + l2c_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_STS)); + l2c_err_adr = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_ADR)); + l2c_err_inf = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_INF)); + lsu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_LSU_ERR_STS)); + fed_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_FED_ERR_STS)); + + panic_plain("Unhandled " CPU_NAME + " implementation specific error. state=%p esr=%#x far=%p\n" + "\tlsu_err_sts:%p, fed_err_sts:%p, mmu_err_sts:%p\n" + "\tl2c_err_sts:%p, l2c_err_adr:%p, l2c_err_inf:%p\n", + state, esr, (void *)far, + (void *)lsu_err_sts, (void *)fed_err_sts, (void *)mmu_err_sts, + (void *)l2c_err_sts, (void *)l2c_err_adr, (void *)l2c_err_inf); + +#elif defined(HAS_MIGSTS) + uint64_t l2c_err_sts, l2c_err_adr, l2c_err_inf, mpidr, migsts; + + mpidr = __builtin_arm_rsr64("MPIDR_EL1"); + migsts = __builtin_arm_rsr64(STR(ARM64_REG_MIGSTS_EL1)); + mmu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_MMU_ERR_STS)); + l2c_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_STS)); + l2c_err_adr = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_ADR)); + l2c_err_inf = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_INF)); + lsu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_LSU_ERR_STS)); + fed_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_FED_ERR_STS)); + + panic_plain("Unhandled " CPU_NAME + " implementation specific error. state=%p esr=%#x far=%p p-core?%d migsts=%p\n" + "\tlsu_err_sts:%p, fed_err_sts:%p, mmu_err_sts:%p\n" + "\tl2c_err_sts:%p, l2c_err_adr:%p, l2c_err_inf:%p\n", + state, esr, (void *)far, !!(mpidr & MPIDR_PNE), (void *)migsts, + (void *)lsu_err_sts, (void *)fed_err_sts, (void *)mmu_err_sts, + (void *)l2c_err_sts, (void *)l2c_err_adr, (void *)l2c_err_inf); +#else // !defined(NO_ECORE) && !defined(HAS_MIGSTS) + uint64_t llc_err_sts, llc_err_adr, llc_err_inf, mpidr; + + mpidr = __builtin_arm_rsr64("MPIDR_EL1"); + + if (mpidr & MPIDR_PNE) { + mmu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_MMU_ERR_STS)); + lsu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_LSU_ERR_STS)); + fed_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_FED_ERR_STS)); + } else { + mmu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_E_MMU_ERR_STS)); + lsu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_E_LSU_ERR_STS)); + fed_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_E_FED_ERR_STS)); + } + + llc_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_STS)); + llc_err_adr = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_ADR)); + llc_err_inf = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_INF)); + + panic_plain("Unhandled " CPU_NAME + " implementation specific error. state=%p esr=%#x far=%p p-core?%d\n" + "\tlsu_err_sts:%p, fed_err_sts:%p, mmu_err_sts:%p\n" + "\tllc_err_sts:%p, llc_err_adr:%p, llc_err_inf:%p\n", + state, esr, (void *)far, !!(mpidr & MPIDR_PNE), + (void *)lsu_err_sts, (void *)fed_err_sts, (void *)mmu_err_sts, + (void *)llc_err_sts, (void *)llc_err_adr, (void *)llc_err_inf); +#endif +#else // !defined(APPLE_ARM64_ARCH_FAMILY) + panic_plain("Unhandled implementation specific error\n"); +#endif +} + +#if CONFIG_KERNEL_INTEGRITY +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +static void +kernel_integrity_error_handler(uint32_t esr, vm_offset_t far) { +#if defined(KERNEL_INTEGRITY_WT) +#if (DEVELOPMENT || DEBUG) + if (ESR_WT_SERROR(esr)) { + switch (ESR_WT_REASON(esr)) { + case WT_REASON_INTEGRITY_FAIL: + panic_plain("Kernel integrity, violation in frame 0x%016lx.", far); + case WT_REASON_BAD_SYSCALL: + panic_plain("Kernel integrity, bad syscall."); + case WT_REASON_NOT_LOCKED: + panic_plain("Kernel integrity, not locked."); + case WT_REASON_ALREADY_LOCKED: + panic_plain("Kernel integrity, already locked."); + case WT_REASON_SW_REQ: + panic_plain("Kernel integrity, software request."); + case WT_REASON_PT_INVALID: + panic_plain("Kernel integrity, encountered invalid TTE/PTE while " + "walking 0x%016lx.", far); + case WT_REASON_PT_VIOLATION: + panic_plain("Kernel integrity, violation in mapping 0x%016lx.", + far); + case WT_REASON_REG_VIOLATION: + panic_plain("Kernel integrity, violation in system register %d.", + (unsigned) far); + default: + panic_plain("Kernel integrity, unknown (esr=0x%08x).", esr); + } + } +#else + if (ESR_WT_SERROR(esr)) { + panic_plain("SError esr: 0x%08x far: 0x%016lx.", esr, far); + } +#endif +#endif +} +#pragma clang diagnostic pop +#endif + +static void +arm64_platform_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far) +{ + cpu_data_t *cdp = getCpuDatap(); + +#if CONFIG_KERNEL_INTEGRITY + kernel_integrity_error_handler(esr, far); +#endif + + if (cdp->platform_error_handler != (platform_error_handler_t) NULL) + (*(platform_error_handler_t)cdp->platform_error_handler) (cdp->cpu_id, far); + else + arm64_implementation_specific_error(state, esr, far); +} + +void +panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) +{ + boolean_t ss_valid; + + ss_valid = is_saved_state64(ss); + arm_saved_state64_t *state = saved_state64(ss); + + panic_plain("%s (saved state: %p%s)\n" + "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n" + "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n" + "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n" + "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n" + "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n" + "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n" + "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n" + "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n" + "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%08x far: 0x%016llx\n", + msg, ss, (ss_valid ? "" : " INVALID"), + state->x[0], state->x[1], state->x[2], state->x[3], + state->x[4], state->x[5], state->x[6], state->x[7], + state->x[8], state->x[9], state->x[10], state->x[11], + state->x[12], state->x[13], state->x[14], state->x[15], + state->x[16], state->x[17], state->x[18], state->x[19], + state->x[20], state->x[21], state->x[22], state->x[23], + state->x[24], state->x[25], state->x[26], state->x[27], + state->x[28], state->fp, state->lr, state->sp, + state->pc, state->cpsr, state->esr, state->far); +} + + +void +sleh_synchronous_sp1(arm_context_t *context, uint32_t esr, vm_offset_t far __unused) +{ + esr_exception_class_t class = ESR_EC(esr); + arm_saved_state_t *state = &context->ss; + + switch (class) { + case ESR_EC_UNCATEGORIZED: + { + uint32_t instr = *((uint32_t*)get_saved_state_pc(state)); + if (IS_ARM_GDB_TRAP(instr)) + DebuggerCall(EXC_BREAKPOINT, state); + // Intentionally fall through to panic if we return from the debugger + } + default: + panic_with_thread_kernel_state("Synchronous exception taken while SP1 selected", state); + } +} + +void +sleh_synchronous(arm_context_t *context, uint32_t esr, vm_offset_t far) +{ + esr_exception_class_t class = ESR_EC(esr); + arm_saved_state_t *state = &context->ss; + vm_offset_t recover = 0; + thread_t thread = current_thread(); + + ASSERT_CONTEXT_SANITY(context); + + /* Don't run exception handler with recover handler set in case of double fault */ + if (thread->recover) { + recover = thread->recover; + thread->recover = (vm_offset_t)NULL; + } + + /* Inherit the interrupt masks from previous context */ + if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state))) + ml_set_interrupts_enabled(TRUE); + + switch (class) { + case ESR_EC_SVC_64: + if (!is_saved_state64(state) || !PSR64_IS_USER(get_saved_state_cpsr(state))) { + panic("Invalid SVC_64 context"); + } + + handle_svc(state); + break; + + case ESR_EC_DABORT_EL0: + handle_abort(state, esr, far, recover, inspect_data_abort, handle_user_abort); + assert(0); /* Unreachable */ + + case ESR_EC_MSR_TRAP: + handle_msr_trap(state, ESR_ISS(esr)); + break; + + case ESR_EC_IABORT_EL0: + handle_abort(state, esr, far, recover, inspect_instruction_abort, handle_user_abort); + assert(0); /* Unreachable */ + + case ESR_EC_IABORT_EL1: + panic("Kernel instruction fetch abort: pc=%p iss=0x%x far=%p. Note: the faulting frame may be missing in the backtrace.", + (void *)get_saved_state_pc(state), ESR_ISS(esr), (void*)far); + + case ESR_EC_PC_ALIGN: + handle_pc_align(state); + assert(0); /* Unreachable */ + break; + + case ESR_EC_DABORT_EL1: + handle_abort(state, esr, far, recover, inspect_data_abort, handle_kernel_abort); + break; + + case ESR_EC_UNCATEGORIZED: + assert(!ESR_ISS(esr)); + + handle_uncategorized(&context->ss, ESR_INSTR_IS_2BYTES(esr)); + /* TODO: Uncomment this after stackshot uses a brk instruction + * rather than an undefined instruction, as stackshot is the + * only case where we want to return to the first-level handler. + */ + //assert(0); /* Unreachable */ + break; + + case ESR_EC_SP_ALIGN: + handle_sp_align(state); + assert(0); /* Unreachable */ + break; + + case ESR_EC_BKPT_AARCH32: + handle_breakpoint(state); + assert(0); /* Unreachable */ + break; + + case ESR_EC_BRK_AARCH64: + if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { + + kprintf("Breakpoint instruction exception from kernel. Hanging here (by design).\n"); + for (;;); + + __unreachable_ok_push + DebuggerCall(EXC_BREAKPOINT, &context->ss); + break; + __unreachable_ok_pop + } else { + handle_breakpoint(state); + assert(0); /* Unreachable */ + } + + case ESR_EC_BKPT_REG_MATCH_EL0: + if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { + handle_breakpoint(state); + assert(0); /* Unreachable */ + } + panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p", + class, state, class, esr, (void *)far); + assert(0); /* Unreachable */ + break; + + case ESR_EC_BKPT_REG_MATCH_EL1: + if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { + kprintf("Hardware Breakpoint Debug exception from kernel. Hanging here (by design).\n"); + for (;;); + + __unreachable_ok_push + DebuggerCall(EXC_BREAKPOINT, &context->ss); + break; + __unreachable_ok_pop + } + panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p", + class, state, class, esr, (void *)far); + assert(0); /* Unreachable */ + break; + + case ESR_EC_SW_STEP_DEBUG_EL0: + if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { + handle_sw_step_debug(state); + assert(0); /* Unreachable */ + } + panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p", + class, state, class, esr, (void *)far); + assert(0); /* Unreachable */ + break; + + case ESR_EC_SW_STEP_DEBUG_EL1: + if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { + kprintf("Software Step Debug exception from kernel. Hanging here (by design).\n"); + for (;;); + + __unreachable_ok_push + DebuggerCall(EXC_BREAKPOINT, &context->ss); + break; + __unreachable_ok_pop + } + panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p", + class, state, class, esr, (void *)far); + assert(0); /* Unreachable */ + break; + + case ESR_EC_WATCHPT_MATCH_EL0: + if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { + handle_watchpoint(far); + assert(0); /* Unreachable */ + } + panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p", + class, state, class, esr, (void *)far); + assert(0); /* Unreachable */ + break; + + case ESR_EC_WATCHPT_MATCH_EL1: + /* + * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to + * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception.. + */ + if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { + arm_debug_set(NULL); + break; /* return to first level handler */ + } + panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p", + class, state, class, esr, (void *)far); + assert(0); /* Unreachable */ + break; + + case ESR_EC_TRAP_SIMD_FP: + handle_simd_trap(state, esr); + assert(0); + break; + + case ESR_EC_ILLEGAL_INSTR_SET: + if (EXCB_ACTION_RERUN != + ex_cb_invoke(EXCB_CLASS_ILLEGAL_INSTR_SET, far)) { + // instruction is not re-executed + panic("Illegal instruction set exception. state=%p class=%u esr=%u far=%p spsr=0x%x", + state, class, esr, (void *)far, get_saved_state_cpsr(state)); + assert(0); + } + // must clear this fault in PSR to re-run + set_saved_state_cpsr(state, get_saved_state_cpsr(state) & (~PSR64_IL)); + break; + + case ESR_EC_MCR_MRC_CP15_TRAP: + case ESR_EC_MCRR_MRRC_CP15_TRAP: + case ESR_EC_MCR_MRC_CP14_TRAP: + case ESR_EC_LDC_STC_CP14_TRAP: + case ESR_EC_MCRR_MRRC_CP14_TRAP: + handle_user_trapped_instruction32(state, esr); + assert(0); + break; + + case ESR_EC_WFI_WFE: + // Use of WFI or WFE instruction when they have been disabled for EL0 + handle_wf_trap(state); + assert(0); /* Unreachable */ + break; + + default: + panic("Unsupported synchronous exception. state=%p class=%u esr=%u far=%p", + state, class, esr, (void *)far); + assert(0); /* Unreachable */ + break; + } + + if (recover) + thread->recover = recover; +} + +/* + * Uncategorized exceptions are a catch-all for general execution errors. + * ARM64_TODO: For now, we assume this is for undefined instruction exceptions. + */ +static void +handle_uncategorized(arm_saved_state_t *state, boolean_t instrLen2) +{ + exception_type_t exception = EXC_BAD_INSTRUCTION; + mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; + mach_msg_type_number_t numcodes = 2; + uint32_t instr; + + if (instrLen2) { + uint16_t instr16; + COPYIN(get_saved_state_pc(state), (char *)&instr16, sizeof(instr16)); + + instr = instr16; + } else { + COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); + } + +#if CONFIG_DTRACE + if (tempDTraceTrapHook && (tempDTraceTrapHook(exception, state, 0, 0) == KERN_SUCCESS)) { + return; + } + + if (PSR64_IS_USER64(get_saved_state_cpsr(state))) { + /* + * For a 64bit user process, we care about all 4 bytes of the + * instr. + */ + if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) { + if (dtrace_user_probe(state) == KERN_SUCCESS) + return; + } + } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) { + /* + * For a 32bit user process, we check for thumb mode, in + * which case we only care about a 2 byte instruction length. + * For non-thumb mode, we care about all 4 bytes of the instructin. + */ + if (get_saved_state_cpsr(state) & PSR64_MODE_USER32_THUMB) { + if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) || + ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) { + if (dtrace_user_probe(state) == KERN_SUCCESS) { + return; + } + } + } else { + if ((instr == FASTTRAP_ARM32_INSTR) || + (instr == FASTTRAP_ARM32_RET_INSTR)) { + if (dtrace_user_probe(state) == KERN_SUCCESS) { + return; + } + } + } + } + +#endif /* CONFIG_DTRACE */ + + if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { + if (IS_ARM_GDB_TRAP(instr)) { + boolean_t interrupt_state; + vm_offset_t kstackptr; + exception = EXC_BREAKPOINT; + + interrupt_state = ml_set_interrupts_enabled(FALSE); + + /* Save off the context here (so that the debug logic + * can see the original state of this thread). + */ + kstackptr = (vm_offset_t) current_thread()->machine.kstackptr; + if (kstackptr) { + ((thread_kernel_state_t) kstackptr)->machine.ss = *state; + } + + /* Hop into the debugger (typically either due to a + * fatal exception, an explicit panic, or a stackshot + * request. + */ + DebuggerCall(exception, state); + + (void) ml_set_interrupts_enabled(interrupt_state); + return; + } else { + panic("Undefined kernel instruction: pc=%p instr=%x\n", (void*)get_saved_state_pc(state), instr); + } + } + + /* + * Check for GDB breakpoint via illegal opcode. + */ + if (instrLen2) { + if (IS_THUMB_GDB_TRAP(instr)) { + exception = EXC_BREAKPOINT; + codes[0] = EXC_ARM_BREAKPOINT; + codes[1] = instr; + } else { + codes[1] = instr; + } + } else { + if (IS_ARM_GDB_TRAP(instr)) { + exception = EXC_BREAKPOINT; + codes[0] = EXC_ARM_BREAKPOINT; + codes[1] = instr; + } else if (IS_THUMB_GDB_TRAP((instr & 0xFFFF))) { + exception = EXC_BREAKPOINT; + codes[0] = EXC_ARM_BREAKPOINT; + codes[1] = instr & 0xFFFF; + } else if (IS_THUMB_GDB_TRAP((instr >> 16))) { + exception = EXC_BREAKPOINT; + codes[0] = EXC_ARM_BREAKPOINT; + codes[1] = instr >> 16; + } else { + codes[1] = instr; + } + } + + exception_triage(exception, codes, numcodes); + assert(0); /* NOTREACHED */ +} + +static void +handle_breakpoint(arm_saved_state_t *state) +{ + exception_type_t exception = EXC_BREAKPOINT; + mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT}; + mach_msg_type_number_t numcodes = 2; + + codes[1] = get_saved_state_pc(state); + exception_triage(exception, codes, numcodes); + assert(0); /* NOTREACHED */ +} + +static void +handle_watchpoint(vm_offset_t fault_addr) +{ + exception_type_t exception = EXC_BREAKPOINT; + mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG}; + mach_msg_type_number_t numcodes = 2; + + codes[1] = fault_addr; + exception_triage(exception, codes, numcodes); + assert(0); /* NOTREACHED */ +} + +static void +handle_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr, vm_offset_t recover, + abort_inspector_t inspect_abort, abort_handler_t handler) +{ + fault_status_t fault_code; + vm_prot_t fault_type; + + inspect_abort(ESR_ISS(esr), &fault_code, &fault_type); + handler(state, esr, fault_addr, fault_code, fault_type, recover); +} + +static void +inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type) +{ + getCpuDatap()->cpu_stat.instr_ex_cnt++; + *fault_code = ISS_IA_FSC(iss); + *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE); +} + +static void +inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type) +{ + getCpuDatap()->cpu_stat.data_ex_cnt++; + *fault_code = ISS_DA_FSC(iss); + + /* Cache operations report faults as write access. Change these to read access. */ + if ((iss & ISS_DA_WNR) && !(iss & ISS_DA_CM)) { + *fault_type = (VM_PROT_READ | VM_PROT_WRITE); + } else { + *fault_type = (VM_PROT_READ); + } +} + +static void +handle_pc_align(arm_saved_state_t *ss) +{ + exception_type_t exc; + mach_exception_data_type_t codes[2]; + mach_msg_type_number_t numcodes = 2; + + if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) { + panic_with_thread_kernel_state("PC alignment exception from kernel.", ss); + } + + exc = EXC_BAD_ACCESS; + codes[0] = EXC_ARM_DA_ALIGN; + codes[1] = get_saved_state_pc(ss); + + exception_triage(exc, codes, numcodes); + assert(0); /* NOTREACHED */ +} + +static void +handle_sp_align(arm_saved_state_t *ss) +{ + exception_type_t exc; + mach_exception_data_type_t codes[2]; + mach_msg_type_number_t numcodes = 2; + + if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) { + panic_with_thread_kernel_state("SP alignment exception from kernel.", ss); + } + + exc = EXC_BAD_ACCESS; + codes[0] = EXC_ARM_SP_ALIGN; + codes[1] = get_saved_state_sp(ss); + + exception_triage(exc, codes, numcodes); + assert(0); /* NOTREACHED */ +} + +static void +handle_wf_trap(arm_saved_state_t *ss) +{ + exception_type_t exc; + mach_exception_data_type_t codes[2]; + mach_msg_type_number_t numcodes = 2; + + exc = EXC_BAD_INSTRUCTION; + codes[0] = EXC_ARM_UNDEFINED; + codes[1] = get_saved_state_sp(ss); + + exception_triage(exc, codes, numcodes); + assert(0); /* NOTREACHED */ +} + + +static void +handle_sw_step_debug(arm_saved_state_t *state) +{ + thread_t thread = current_thread(); + exception_type_t exc; + mach_exception_data_type_t codes[2]; + mach_msg_type_number_t numcodes = 2; + + if (!PSR64_IS_USER(get_saved_state_cpsr(state))) { + panic_with_thread_kernel_state("SW_STEP_DEBUG exception from kernel.", state); + } + + // Disable single step and unmask interrupts (in the saved state, anticipating next exception return) + if (thread->machine.DebugData != NULL) { + thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1; + } else { + panic_with_thread_kernel_state("SW_STEP_DEBUG exception thread DebugData is NULL.", state); + } + + set_saved_state_cpsr((thread->machine.upcb), + get_saved_state_cpsr((thread->machine.upcb)) & ~(PSR64_SS | DAIF_IRQF | DAIF_FIQF)); + + // Special encoding for gdb single step event on ARM + exc = EXC_BREAKPOINT; + codes[0] = 1; + codes[1] = 0; + + exception_triage(exc, codes, numcodes); + assert(0); /* NOTREACHED */ +} + +static int +is_vm_fault(fault_status_t status) +{ + switch (status) { + case FSC_TRANSLATION_FAULT_L0: + case FSC_TRANSLATION_FAULT_L1: + case FSC_TRANSLATION_FAULT_L2: + case FSC_TRANSLATION_FAULT_L3: + case FSC_ACCESS_FLAG_FAULT_L1: + case FSC_ACCESS_FLAG_FAULT_L2: + case FSC_ACCESS_FLAG_FAULT_L3: + case FSC_PERMISSION_FAULT_L1: + case FSC_PERMISSION_FAULT_L2: + case FSC_PERMISSION_FAULT_L3: + return TRUE; + default: + return FALSE; + } +} + +#if __ARM_PAN_AVAILABLE__ +static int +is_permission_fault(fault_status_t status) +{ + switch (status) { + case FSC_PERMISSION_FAULT_L1: + case FSC_PERMISSION_FAULT_L2: + case FSC_PERMISSION_FAULT_L3: + return TRUE; + default: + return FALSE; + } +} +#endif + +static int +is_alignment_fault(fault_status_t status) +{ + return (status == FSC_ALIGNMENT_FAULT); +} + +static int +is_parity_error(fault_status_t status) +{ + switch (status) { + case FSC_SYNC_PARITY: + case FSC_ASYNC_PARITY: + case FSC_SYNC_PARITY_TT_L1: + case FSC_SYNC_PARITY_TT_L2: + case FSC_SYNC_PARITY_TT_L3: + return TRUE; + default: + return FALSE; + } +} + +static void +handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr, + fault_status_t fault_code, vm_prot_t fault_type, vm_offset_t recover) +{ + exception_type_t exc = EXC_BAD_ACCESS; + mach_exception_data_type_t codes[2]; + mach_msg_type_number_t numcodes = 2; + thread_t thread = current_thread(); + + (void)esr; + (void)state; + + if (ml_at_interrupt_context()) + panic_with_thread_kernel_state("Apparently on interrupt stack when taking user abort!\n", state); + + thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */ + + if (is_vm_fault(fault_code)) { + kern_return_t result; + vm_map_t map = thread->map; + vm_offset_t vm_fault_addr = fault_addr; + + assert(map != kernel_map); + + if (!(fault_type & VM_PROT_EXECUTE) && user_tbi_enabled()) + vm_fault_addr = tbi_clear(fault_addr); + +#if CONFIG_DTRACE + if (thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */ + if (dtrace_tally_fault(vm_fault_addr)) { /* Should a user mode fault under dtrace be ignored? */ + if (recover) { + set_saved_state_pc(state, recover); + } else { + boolean_t intr = ml_set_interrupts_enabled(FALSE); + panic_with_thread_kernel_state("copyin/out has no recovery point", state); + (void) ml_set_interrupts_enabled(intr); + } + return; + } else { + boolean_t intr = ml_set_interrupts_enabled(FALSE); + panic_with_thread_kernel_state("Unexpected UMW page fault under dtrace_probe", state); + (void) ml_set_interrupts_enabled(intr); + return; + } + } +#else + (void)recover; +#endif + +#if CONFIG_PGTRACE + if (pgtrace_enabled) { + /* Check to see if trace bit is set */ + result = pmap_pgtrace_fault(map->pmap, fault_addr, state); + if (result == KERN_SUCCESS) return; + } +#endif + + /* check to see if it is just a pmap ref/modify fault */ + result = arm_fast_fault(map->pmap, trunc_page(vm_fault_addr), fault_type, TRUE); + if (result != KERN_SUCCESS) { + + { + /* We have to fault the page in */ + result = vm_fault(map, vm_fault_addr, fault_type, + /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE, + /* caller_pmap */ NULL, /* caller_pmap_addr */ 0); + } + } + if (result == KERN_SUCCESS || result == KERN_ABORTED) { + thread_exception_return(); + /* NOTREACHED */ + } + + codes[0] = result; + } else if (is_alignment_fault(fault_code)) { + codes[0] = EXC_ARM_DA_ALIGN; + } else if (is_parity_error(fault_code)) { +#if defined(APPLE_ARM64_ARCH_FAMILY) + if (fault_code == FSC_SYNC_PARITY) { + arm64_platform_error(state, esr, fault_addr); + thread_exception_return(); + /* NOTREACHED */ + } +#else + panic("User parity error."); +#endif + } else { + codes[0] = KERN_FAILURE; + } + + codes[1] = fault_addr; + exception_triage(exc, codes, numcodes); + assert(0); /* NOTREACHED */ +} + +#if __ARM_PAN_AVAILABLE__ +static int +is_pan_fault(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr, fault_status_t fault_code) +{ + // PAN (Privileged Access Never) fault occurs for data read/write in EL1 to + // virtual address that is readable/writeable from both EL1 and EL0 + + // To check for PAN fault, we evaluate if the following conditions are true: + // 1. This is a permission fault + // 2. PAN is enabled + // 3. AT instruction (on which PAN has no effect) on the same faulting address + // succeeds + + vm_offset_t pa; + + if (!(is_permission_fault(fault_code) && get_saved_state_cpsr(state) & PSR64_PAN)) { + return FALSE; + } + + if (esr & ISS_DA_WNR) { + pa = mmu_kvtop_wpreflight(fault_addr); + } else { + pa = mmu_kvtop(fault_addr); + } + return (pa)? TRUE: FALSE; +} +#endif + +static void +handle_kernel_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr, + fault_status_t fault_code, vm_prot_t fault_type, vm_offset_t recover) +{ + thread_t thread = current_thread(); + (void)esr; + +#if CONFIG_DTRACE + if (is_vm_fault(fault_code) && thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */ + if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */ + /* + * Point to next instruction, or recovery handler if set. + */ + if (recover) { + set_saved_state_pc(state, recover); + } else { + set_saved_state_pc(state, get_saved_state_pc(state) + 4); + } + return; + } else { + boolean_t intr = ml_set_interrupts_enabled(FALSE); + panic_with_thread_kernel_state("Unexpected page fault under dtrace_probe", state); + (void) ml_set_interrupts_enabled(intr); + return; + } + } +#endif + +#if !CONFIG_PGTRACE /* This will be moved next to pgtrace fault evaluation */ + if (ml_at_interrupt_context()) + panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state); +#endif + + if (is_vm_fault(fault_code)) { + kern_return_t result; + vm_map_t map; + int interruptible; + + if (fault_addr >= gVirtBase && fault_addr < (gVirtBase+gPhysSize)) { + panic_with_thread_kernel_state("Unexpected fault in kernel static region\n",state); + } + + if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL) { + map = kernel_map; + interruptible = THREAD_UNINT; + } else { + map = thread->map; + interruptible = THREAD_ABORTSAFE; + } + +#if CONFIG_PGTRACE + if (pgtrace_enabled) { + /* Check to see if trace bit is set */ + result = pmap_pgtrace_fault(map->pmap, fault_addr, state); + if (result == KERN_SUCCESS) return; + } + + if (ml_at_interrupt_context()) + panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state); +#endif + + /* check to see if it is just a pmap ref/modify fault */ + result = arm_fast_fault(map->pmap, trunc_page(fault_addr), fault_type, FALSE); + if (result == KERN_SUCCESS) return; + + { + /* + * We have to "fault" the page in. + */ + result = vm_fault(map, fault_addr, fault_type, + /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible, + /* caller_pmap */ NULL, /* caller_pmap_addr */ 0); + } + + if (result == KERN_SUCCESS) return; + + /* + * If we have a recover handler, invoke it now. + */ + if (recover) { + set_saved_state_pc(state, recover); + return; + } + +#if __ARM_PAN_AVAILABLE__ + if (is_pan_fault(state, esr, fault_addr, fault_code)) { + panic_with_thread_kernel_state("Privileged access never abort.", state); + } +#endif + +#if CONFIG_PGTRACE + } else if (ml_at_interrupt_context()) { + panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state); +#endif + } else if (is_alignment_fault(fault_code)) { + panic_with_thread_kernel_state("Unaligned kernel data abort.", state); + } else if (is_parity_error(fault_code)) { +#if defined(APPLE_ARM64_ARCH_FAMILY) + if (fault_code == FSC_SYNC_PARITY) { + arm64_platform_error(state, esr, fault_addr); + return; + } +#else + panic_with_thread_kernel_state("Kernel parity error.", state); +#endif + } else { + kprintf("Unclassified kernel abort (fault_code=0x%x)\n", fault_code); + } + + panic_with_thread_kernel_state("Kernel data abort.", state); +} + +extern void syscall_trace(struct arm_saved_state * regs); + +static void +handle_svc(arm_saved_state_t *state) +{ + int trap_no = get_saved_state_svc_number(state); + thread_t thread = current_thread(); + struct proc *p; + +#define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */ + +#define TRACE_SYSCALL 1 +#if TRACE_SYSCALL + syscall_trace(state); +#endif + + thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */ + + if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) { + platform_syscall(state); + panic("Returned from platform_syscall()?"); + } + + mach_kauth_cred_uthread_update(); + + if (trap_no < 0) { + if (trap_no == -3) { + handle_mach_absolute_time_trap(state); + return; + } else if (trap_no == -4) { + handle_mach_continuous_time_trap(state); + return; + } + + /* Counting perhaps better in the handler, but this is how it's been done */ + thread->syscalls_mach++; + mach_syscall(state); + } else { + /* Counting perhaps better in the handler, but this is how it's been done */ + thread->syscalls_unix++; + p = get_bsdthreadtask_info(thread); + + assert(p); + + unix_syscall(state, thread, (struct uthread*)thread->uthread, p); + } +} + +static void +handle_mach_absolute_time_trap(arm_saved_state_t *state) +{ + uint64_t now = mach_absolute_time(); + saved_state64(state)->x[0] = now; +} + +static void +handle_mach_continuous_time_trap(arm_saved_state_t *state) +{ + uint64_t now = mach_continuous_time(); + saved_state64(state)->x[0] = now; +} + +static void +handle_msr_trap(arm_saved_state_t *state, uint32_t iss) +{ + exception_type_t exception = EXC_BAD_INSTRUCTION; + mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; + mach_msg_type_number_t numcodes = 2; + uint32_t instr; + + (void)iss; + + if (!is_saved_state64(state)) { + panic("MSR/MRS trap (EC 0x%x) from 32-bit state\n", ESR_EC_MSR_TRAP); + } + + if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { + panic("MSR/MRS trap (EC 0x%x) from kernel\n", ESR_EC_MSR_TRAP); + } + + COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); + codes[1] = instr; + + exception_triage(exception, codes, numcodes); +} + +static void +handle_user_trapped_instruction32(arm_saved_state_t *state, uint32_t esr) +{ + exception_type_t exception = EXC_BAD_INSTRUCTION; + mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; + mach_msg_type_number_t numcodes = 2; + uint32_t instr; + + if (is_saved_state64(state)) { + panic("ESR (0x%x) for instruction trapped from U32, but saved state is 64-bit.", esr); + } + + if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { + panic("ESR (0x%x) for instruction trapped from U32, actually came from kernel?", esr); + } + + COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); + codes[1] = instr; + + exception_triage(exception, codes, numcodes); +} + +static void +handle_simd_trap(arm_saved_state_t *state, uint32_t esr) +{ + exception_type_t exception = EXC_BAD_INSTRUCTION; + mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; + mach_msg_type_number_t numcodes = 2; + uint32_t instr; + + if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { + panic("ESR (0x%x) for SIMD trap from userland, actually came from kernel?", esr); + } + + COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); + codes[1] = instr; + + exception_triage(exception, codes, numcodes); +} + +void +sleh_irq(arm_saved_state_t *state) +{ + uint64_t timestamp = 0; + uint32_t old_entropy_data = 0; + uint32_t * old_entropy_data_ptr = NULL; + uint32_t * new_entropy_data_ptr = NULL; + cpu_data_t * cdp = getCpuDatap(); + + sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER); + + /* Run the registered interrupt handler. */ + cdp->interrupt_handler(cdp->interrupt_target, + cdp->interrupt_refCon, + cdp->interrupt_nub, + cdp->interrupt_source); + + /* We use interrupt timing as an entropy source. */ + timestamp = ml_get_timebase(); + + /* + * The buffer index is subject to races, but as these races should only + * result in multiple CPUs updating the same location, the end result + * should be that noise gets written into the entropy buffer. As this + * is the entire point of the entropy buffer, we will not worry about + * these races for now. + */ + old_entropy_data_ptr = EntropyData.index_ptr; + new_entropy_data_ptr = old_entropy_data_ptr + 1; + + if (new_entropy_data_ptr >= &EntropyData.buffer[ENTROPY_BUFFER_SIZE]) { + new_entropy_data_ptr = EntropyData.buffer; + } + + EntropyData.index_ptr = new_entropy_data_ptr; + + /* Mix the timestamp data and the old data together. */ + old_entropy_data = *old_entropy_data_ptr; + *old_entropy_data_ptr = (uint32_t)timestamp ^ __ror(old_entropy_data, 9); + + sleh_interrupt_handler_epilogue(); +} + +void +sleh_fiq(arm_saved_state_t *state) +{ + unsigned int type = DBG_INTR_TYPE_UNKNOWN; + if (ml_get_timer_pending()) { + type = DBG_INTR_TYPE_TIMER; + } + + sleh_interrupt_handler_prologue(state, type); + + { + /* + * We don't know that this is a timer, but we don't have insight into + * the other interrupts that go down this path. + */ + + + cpu_data_t *cdp = getCpuDatap(); + + cdp->cpu_decrementer = -1; /* Large */ + + /* + * ARM64_TODO: whether we're coming from userland is ignored right now. + * We can easily thread it through, but not bothering for the + * moment (AArch32 doesn't either). + */ + rtclock_intr(TRUE); + } + + sleh_interrupt_handler_epilogue(); +} + +void +sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far) +{ + arm_saved_state_t *state = &context->ss; + + ASSERT_CONTEXT_SANITY(context); + arm64_platform_error(state, esr, far); +} + +void +mach_syscall_trace_exit( + unsigned int retval, + unsigned int call_number) +{ + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_END, + retval, 0, 0, 0, 0); +} + +__attribute__((noreturn)) +void +thread_syscall_return(kern_return_t error) +{ + thread_t thread; + struct arm_saved_state *state; + + thread = current_thread(); + state = get_user_regs(thread); + + assert(is_saved_state64(state)); + saved_state64(state)->x[0] = error; + +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_get_kernel_state(thread)->allocation_name; + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + + if (kdebug_enable) { + /* Invert syscall number (negative for a mach syscall) */ + mach_syscall_trace_exit(error, (-1) * get_saved_state_svc_number(state)); + } + + thread_exception_return(); +} + +void +syscall_trace( + struct arm_saved_state * regs __unused) +{ + /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */ +} + +static void +sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type) +{ + uint64_t is_user = PSR64_IS_USER(get_saved_state_cpsr(state)); + + uint64_t pc = is_user ? get_saved_state_pc(state) : + VM_KERNEL_UNSLIDE(get_saved_state_pc(state)); + + KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START, + 0, pc, is_user, type); + +#if CONFIG_TELEMETRY + if (telemetry_needs_record) { + telemetry_mark_curthread((boolean_t)is_user); + } +#endif /* CONFIG_TELEMETRY */ +} + +static void +sleh_interrupt_handler_epilogue(void) +{ + KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END); +} + +void +sleh_invalid_stack(arm_context_t *context, uint32_t esr __unused, vm_offset_t far __unused) +{ + thread_t thread = current_thread(); + vm_offset_t kernel_stack_bottom, sp; + + sp = get_saved_state_sp(&context->ss); + kernel_stack_bottom = round_page(thread->machine.kstackptr) - KERNEL_STACK_SIZE; + + if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) { + panic_with_thread_kernel_state("Invalid kernel stack pointer (probable overflow).", &context->ss); + } + + panic_with_thread_kernel_state("Invalid kernel stack pointer (probable corruption).", &context->ss); +} + diff --git a/osfmk/arm64/start.s b/osfmk/arm64/start.s new file mode 100644 index 000000000..2776c8f46 --- /dev/null +++ b/osfmk/arm64/start.s @@ -0,0 +1,898 @@ +/* + * Copyright (c) 2007-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "assym.s" + + +.macro MSR_VBAR_EL1_X0 +#if defined(KERNEL_INTEGRITY_KTRR) + mov x1, lr + bl EXT(pinst_set_vbar) + mov lr, x1 +#else + msr VBAR_EL1, x0 +#endif +.endmacro + +.macro MSR_TCR_EL1_X1 +#if defined(KERNEL_INTEGRITY_KTRR) + mov x0, x1 + mov x1, lr + bl _pinst_set_tcr + mov lr, x1 +#else + msr TCR_EL1, x1 +#endif +.endmacro + +.macro MSR_TTBR1_EL1_X0 +#if defined(KERNEL_INTEGRITY_KTRR) + mov x1, lr + bl _pinst_set_ttbr1 + mov lr, x1 +#else + msr TTBR1_EL1, x0 +#endif +.endmacro + +.macro MSR_SCTLR_EL1_X0 +#if defined(KERNEL_INTEGRITY_KTRR) + mov x1, lr + + // This may abort, do so on SP1 + bl _pinst_spsel_1 + + bl _pinst_set_sctlr + msr SPSel, #0 // Back to SP0 + mov lr, x1 +#else + msr SCTLR_EL1, x0 +#endif /* defined(KERNEL_INTEGRITY_KTRR) */ +.endmacro + +/* + * Checks the reset handler for global and CPU-specific reset-assist functions, + * then jumps to the reset handler with boot args and cpu data. This is copied + * to the first physical page during CPU bootstrap (see cpu.c). + * + * Variables: + * x19 - Reset handler data pointer + * x20 - Boot args pointer + * x21 - CPU data pointer + */ + .text + .align 12 + .globl EXT(LowResetVectorBase) +LEXT(LowResetVectorBase) + // Preserve x0 for start_first_cpu, if called + + // Unlock the core for debugging + msr OSLAR_EL1, xzr + +#if !(defined(KERNEL_INTEGRITY_KTRR)) + // Set low reset vector before attempting any loads + adrp x0, EXT(LowExceptionVectorBase)@page + add x0, x0, EXT(LowExceptionVectorBase)@pageoff + msr VBAR_EL1, x0 +#endif + + +#if defined(KERNEL_INTEGRITY_KTRR) + /* + * Set KTRR registers immediately after wake/resume + * + * During power on reset, XNU stashed the kernel text region range values + * into __DATA,__const which should be protected by AMCC RoRgn at this point. + * Read this data and program/lock KTRR registers accordingly. + * If either values are zero, we're debugging kernel so skip programming KTRR. + */ + + // load stashed rorgn_begin + adrp x17, EXT(rorgn_begin)@page + add x17, x17, EXT(rorgn_begin)@pageoff + ldr x17, [x17] + // if rorgn_begin is zero, we're debugging. skip enabling ktrr + cbz x17, 1f + + // load stashed rorgn_end + adrp x19, EXT(rorgn_end)@page + add x19, x19, EXT(rorgn_end)@pageoff + ldr x19, [x19] + cbz x19, 1f + + // program and lock down KTRR + // subtract one page from rorgn_end to make pinst insns NX + msr ARM64_REG_KTRR_LOWER_EL1, x17 + sub x19, x19, #(1 << (ARM_PTE_SHIFT-12)), lsl #12 + msr ARM64_REG_KTRR_UPPER_EL1, x19 + mov x17, #1 + msr ARM64_REG_KTRR_LOCK_EL1, x17 + +1: +#endif /* defined(KERNEL_INTEGRITY_KTRR) */ + + // Process reset handlers + adrp x19, EXT(ResetHandlerData)@page // Get address of the reset handler data + add x19, x19, EXT(ResetHandlerData)@pageoff + mrs x15, MPIDR_EL1 // Load MPIDR to get CPU number + and x0, x15, #0xFF // CPU number is in MPIDR Affinity Level 0 + ldr x1, [x19, CPU_DATA_ENTRIES] // Load start of data entries + add x3, x1, MAX_CPUS * 16 // end addr of data entries = start + (16 * MAX_CPUS) +Lcheck_cpu_data_entry: + ldr x21, [x1, CPU_DATA_PADDR] // Load physical CPU data address + cbz x21, Lnext_cpu_data_entry + ldr w2, [x21, CPU_PHYS_ID] // Load ccc cpu phys id + cmp x0, x2 // Compare cpu data phys cpu and MPIDR_EL1 phys cpu + b.eq Lfound_cpu_data_entry // Branch if match +Lnext_cpu_data_entry: + add x1, x1, #16 // Increment to the next cpu data entry + cmp x1, x3 + b.eq Lskip_cpu_reset_handler // Not found + b Lcheck_cpu_data_entry // loop +Lfound_cpu_data_entry: + adrp x20, EXT(const_boot_args)@page + add x20, x20, EXT(const_boot_args)@pageoff + ldr x0, [x21, CPU_RESET_HANDLER] // Call CPU reset handler + cbz x0, Lskip_cpu_reset_handler + + // Validate that our handler is one of the two expected handlers + adrp x2, EXT(resume_idle_cpu)@page + add x2, x2, EXT(resume_idle_cpu)@pageoff + cmp x0, x2 + beq 1f + adrp x2, EXT(start_cpu)@page + add x2, x2, EXT(start_cpu)@pageoff + cmp x0, x2 + bne Lskip_cpu_reset_handler +1: + + + + blr x0 +Lskip_cpu_reset_handler: + b . // Hang if the handler is NULL or returns + + .align 3 + .globl EXT(ResetHandlerData) +LEXT(ResetHandlerData) + .space (rhdSize_NUM),0 // (filled with 0s) + + .align 3 + .global EXT(LowResetVectorEnd) +LEXT(LowResetVectorEnd) + .global EXT(SleepToken) +#if WITH_CLASSIC_S2R +LEXT(SleepToken) + .space (stSize_NUM),0 +#endif + + +/* + * __start trampoline is located at a position relative to LowResetVectorBase + * so that iBoot can compute the reset vector position to set IORVBAR using + * only the kernel entry point. Reset vector = (__start & ~0xfff) + */ + .align 3 + .globl EXT(_start) +LEXT(_start) + b EXT(start_first_cpu) + + +/* + * Provides an early-boot exception vector so that the processor will spin + * and preserve exception information (e.g., ELR_EL1) when early CPU bootstrap + * code triggers an exception. This is copied to the second physical page + * during CPU bootstrap (see cpu.c). + */ + .align 12, 0 + .global EXT(LowExceptionVectorBase) +LEXT(LowExceptionVectorBase) + /* EL1 SP 0 */ + b . + .align 7 + b . + .align 7 + b . + .align 7 + b . + /* EL1 SP1 */ + .align 7 + b . + .align 7 + b . + .align 7 + b . + .align 7 + b . + /* EL0 64 */ + .align 7 + b . + .align 7 + b . + .align 7 + b . + .align 7 + b . + /* EL0 32 */ + .align 7 + b . + .align 7 + b . + .align 7 + b . + .align 7 + b . + .align 12, 0 + +#if defined(KERNEL_INTEGRITY_KTRR) +/* + * Provide a global symbol so that we can narrow the V=P mapping to cover + * this page during arm_vm_init. + */ +.align ARM_PGSHIFT +.globl EXT(bootstrap_instructions) +LEXT(bootstrap_instructions) +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + .align 2 + .globl EXT(resume_idle_cpu) +LEXT(resume_idle_cpu) + adrp lr, EXT(arm_init_idle_cpu)@page + add lr, lr, EXT(arm_init_idle_cpu)@pageoff + b start_cpu + + .align 2 + .globl EXT(start_cpu) +LEXT(start_cpu) + adrp lr, EXT(arm_init_cpu)@page + add lr, lr, EXT(arm_init_cpu)@pageoff + b start_cpu + + .align 2 +start_cpu: +#if defined(KERNEL_INTEGRITY_KTRR) + // This is done right away in reset vector for pre-KTRR devices + // Set low reset vector now that we are in the KTRR-free zone + adrp x0, EXT(LowExceptionVectorBase)@page + add x0, x0, EXT(LowExceptionVectorBase)@pageoff + MSR_VBAR_EL1_X0 +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + + // x20 set to BootArgs phys address + // x21 set to cpu data phys address + msr DAIFSet, #(DAIFSC_ALL) // Disable all interrupts + + // Get the kernel memory parameters from the boot args + ldr x22, [x20, BA_VIRT_BASE] // Get the kernel virt base + ldr x23, [x20, BA_PHYS_BASE] // Get the kernel phys base + ldr x24, [x20, BA_MEM_SIZE] // Get the physical memory size + ldr x25, [x20, BA_TOP_OF_KERNEL_DATA] // Get the top of the kernel data + + // Set TPIDRRO_EL0 with the CPU number + ldr x0, [x21, CPU_NUMBER_GS] + msr TPIDRRO_EL0, x0 + + // Set the exception stack pointer + ldr x0, [x21, CPU_EXCEPSTACK_TOP] + + + // Set SP_EL1 to exception stack +#if defined(KERNEL_INTEGRITY_KTRR) + mov x1, lr + bl _pinst_spsel_1 + mov lr, x1 +#else + msr SPSel, #1 +#endif + mov sp, x0 + + // Set the interrupt stack pointer + ldr x0, [x21, CPU_INTSTACK_TOP] + msr SPSel, #0 + mov sp, x0 + + // Convert lr to KVA + add lr, lr, x22 + sub lr, lr, x23 + + b common_start + +/* + * create_l1_table_entry + * + * Given a virtual address, creates a table entry in an L1 translation table + * to point to an L2 translation table. + * arg0 - Virtual address + * arg1 - L1 table address + * arg2 - L2 table address + * arg3 - Scratch register + * arg4 - Scratch register + * arg5 - Scratch register + */ +.macro create_l1_table_entry + and $3, $0, #(ARM_TT_L1_INDEX_MASK) + lsr $3, $3, #(ARM_TT_L1_SHIFT) // Get index in L1 table for L2 table + lsl $3, $3, #(TTE_SHIFT) // Convert index into pointer offset + add $3, $1, $3 // Get L1 entry pointer + mov $4, #(ARM_TTE_BOOT_TABLE) // Get L1 table entry template + and $5, $2, #(ARM_TTE_TABLE_MASK) // Get address bits of L2 table + orr $5, $4, $5 // Create table entry for L2 table + str $5, [$3] // Write entry to L1 table +.endmacro + +/* + * create_l2_block_entries + * + * Given base virtual and physical addresses, creates consecutive block entries + * in an L2 translation table. + * arg0 - Virtual address + * arg1 - Physical address + * arg2 - L2 table address + * arg3 - Number of entries + * arg4 - Scratch register + * arg5 - Scratch register + * arg6 - Scratch register + * arg7 - Scratch register + */ +.macro create_l2_block_entries + and $4, $0, #(ARM_TT_L2_INDEX_MASK) + lsr $4, $4, #(ARM_TTE_BLOCK_L2_SHIFT) // Get index in L2 table for block entry + lsl $4, $4, #(TTE_SHIFT) // Convert index into pointer offset + add $4, $2, $4 // Get L2 entry pointer + mov $5, #(ARM_TTE_BOOT_BLOCK) // Get L2 block entry template + and $6, $1, #(ARM_TTE_BLOCK_L2_MASK) // Get address bits of block mapping + orr $6, $5, $6 + mov $5, $3 + mov $7, #(ARM_TT_L2_SIZE) +1: + str $6, [$4], #(1 << TTE_SHIFT) // Write entry to L2 table and advance + add $6, $6, $7 // Increment the output address + subs $5, $5, #1 // Decrement the number of entries + b.ne 1b +.endmacro + +/* + * _start_first_cpu + * Cold boot init routine. Called from __start + * x0 - Boot args + */ + .align 2 + .globl EXT(start_first_cpu) +LEXT(start_first_cpu) + + // Unlock the core for debugging + msr OSLAR_EL1, xzr + mov x20, x0 + mov x21, xzr + + // Set low reset vector before attempting any loads + adrp x0, EXT(LowExceptionVectorBase)@page + add x0, x0, EXT(LowExceptionVectorBase)@pageoff + MSR_VBAR_EL1_X0 + + + + // Get the kernel memory parameters from the boot args + ldr x22, [x20, BA_VIRT_BASE] // Get the kernel virt base + ldr x23, [x20, BA_PHYS_BASE] // Get the kernel phys base + ldr x24, [x20, BA_MEM_SIZE] // Get the physical memory size + ldr x25, [x20, BA_TOP_OF_KERNEL_DATA] // Get the top of the kernel data + + // Set CPU number to 0 + msr TPIDRRO_EL0, x21 + + // Set up exception stack pointer + adrp x0, EXT(excepstack_top)@page // Load top of exception stack + add x0, x0, EXT(excepstack_top)@pageoff + add x0, x0, x22 // Convert to KVA + sub x0, x0, x23 + + // Set SP_EL1 to exception stack +#if defined(KERNEL_INTEGRITY_KTRR) + bl _pinst_spsel_1 +#else + msr SPSel, #1 +#endif + + mov sp, x0 + + // Set up interrupt stack pointer + adrp x0, EXT(intstack_top)@page // Load top of irq stack + add x0, x0, EXT(intstack_top)@pageoff + add x0, x0, x22 // Convert to KVA + sub x0, x0, x23 + msr SPSel, #0 // Set SP_EL0 to interrupt stack + mov sp, x0 + + // Load address to the C init routine into link register + adrp lr, EXT(arm_init)@page + add lr, lr, EXT(arm_init)@pageoff + add lr, lr, x22 // Convert to KVA + sub lr, lr, x23 + + /* + * Set up the bootstrap page tables with a single block entry for the V=P + * mapping, a single block entry for the trampolined kernel address (KVA), + * and all else invalid. This requires four pages: + * Page 1 - V=P L1 table + * Page 2 - V=P L2 table + * Page 3 - KVA L1 table + * Page 4 - KVA L2 table + */ +#if __ARM64_TWO_LEVEL_PMAP__ + /* + * If we are using a two level scheme, we don't need the L1 entries, so: + * Page 1 - V=P L2 table + * Page 2 - KVA L2 table + */ +#endif + + // Invalidate all entries in the bootstrap page tables + mov x0, #(ARM_TTE_EMPTY) // Load invalid entry template + mov x1, x25 // Start at top of kernel + mov x2, #(TTE_PGENTRIES) // Load number of entries per page +#if __ARM64_TWO_LEVEL_PMAP__ + lsl x2, x2, #1 // Shift by 1 for num entries on 2 pages +#else + lsl x2, x2, #2 // Shift by 2 for num entries on 4 pages +#endif + sub x2, x2, #1 // Subtract one to terminate on last entry +Linvalidate_bootstrap: // do { + str x0, [x1], #(1 << TTE_SHIFT) // Invalidate and advance + subs x2, x2, #1 // entries-- + b.ne Linvalidate_bootstrap // } while (entries != 0) + + /* Load addresses for page table construction macros + * x0 - Physical base (used to identify V=P section to set up) + * x1 - V=P L1 table base + * x2 - V=P L2 table base + * x3 - KVA L1 table base + * x4 - KVA L2 table base + * x5 - Mem size in entries (up to 1GB) + */ + + /* + * In order to reclaim memory on targets where TZ0 (or some other entity) + * must be located at the base of memory, iBoot may set the virtual and + * physical base addresses to immediately follow whatever lies at the + * base of physical memory. + * + * If the base address belongs to TZ0, it may be dangerous for xnu to map + * it (as it may be prefetched, despite being technically inaccessible). + * In order to avoid this issue while keeping the mapping code simple, we + * may continue to use block mappings, but we will only map xnu's mach + * header to the end of memory. + * + * Given that iBoot guarantees that the unslid kernelcache base address + * will begin on an L2 boundary, this should prevent us from accidentally + * mapping TZ0. + */ + adrp x0, EXT(_mh_execute_header)@page // Use xnu's mach header as the start address + add x0, x0, EXT(_mh_execute_header)@pageoff +#if __ARM64_TWO_LEVEL_PMAP__ + /* + * We don't need the L1 entries in this case, so skip them. + */ + mov x2, x25 // Load V=P L2 table address + add x4, x2, PGBYTES // Load KVA L2 table address +#else + mov x1, x25 // Load V=P L1 table address + add x2, x1, PGBYTES // Load V=P L2 table address + add x3, x2, PGBYTES // Load KVA L1 table address + add x4, x3, PGBYTES // Load KVA L2 table address +#endif + /* + * We must adjust the amount we wish to map in order to account for the + * memory preceeding xnu's mach header. + */ + sub x5, x0, x23 // Map from the mach header up to the end of our memory + sub x5, x24, x5 + lsr x5, x5, #(ARM_TT_L2_SHIFT) + mov x6, #(TTE_PGENTRIES) // Load number of L2 entries per page + cmp x5, x6 // If memsize requires more than 1 page of entries + csel x5, x5, x6, lt // ... round down to a single page (first 1GB) + +#if !__ARM64_TWO_LEVEL_PMAP__ + /* Create entry for L2 table in V=P L1 table + * create_l1_table_entry(V=P, L1 table, L2 table, scratch1, scratch2, scratch3) + */ + create_l1_table_entry x0, x1, x2, x10, x11, x12 +#endif + + /* Create block entry in V=P L2 table + * create_l2_block_entries(V=P virt, V=P phys, L2 table, num_ents, scratch1, scratch2, scratch3) + */ + create_l2_block_entries x0, x0, x2, x5, x10, x11, x12, x13 + +#if !__ARM64_TWO_LEVEL_PMAP__ + /* Create entry for L2 table in KVA L1 table + * create_l1_table_entry(virt_base, L1 table, L2 table, scratch1, scratch2, scratch3) + */ + create_l1_table_entry x22, x3, x4, x10, x11, x12 +#endif + + /* Create block entries in KVA L2 table + * create_l2_block_entries(virt_base, phys_base, L2 table, num_ents, scratch1, scratch2, scratch3) + */ + create_l2_block_entries x22, x23, x4, x5, x10, x11, x12, x13 + + /* Ensure TTEs are visible */ + dsb ish + + b common_start + +/* + * Begin common CPU initialization + * + * Regster state: + * x20 - PA of boot args + * x21 - zero on cold boot, PA of cpu data on warm reset + * x22 - Kernel virtual base + * x23 - Kernel physical base + * x24 - Physical memory size + * x25 - PA of the end of the kernl + * lr - KVA of C init routine + * sp - SP_EL0 selected + * + * SP_EL0 - KVA of CPU's interrupt stack + * SP_EL1 - KVA of CPU's exception stack + * TPIDRRO_EL0 - CPU number + */ +common_start: + // Set the translation control register. + adrp x0, EXT(sysreg_restore)@page // Load TCR value from the system register restore structure + add x0, x0, EXT(sysreg_restore)@pageoff + ldr x1, [x0, SR_RESTORE_TCR_EL1] + MSR_TCR_EL1_X1 + + /* Set up translation table base registers. + * TTBR0 - V=P table @ top of kernel + * TTBR1 - KVA table @ top of kernel + 2 pages + */ +#if defined(KERNEL_INTEGRITY_KTRR) + /* Note that for KTRR configurations, the V=P map will be modified by + * arm_vm_init.c. + */ +#endif + and x0, x25, #(TTBR_BADDR_MASK) + msr TTBR0_EL1, x0 +#if __ARM64_TWO_LEVEL_PMAP__ + /* + * If we're using a two level pmap, we'll only need a + * single page per bootstrap pmap. + */ + mov x12, #1 +#else + /* + * If we're using a three level pmap, we'll need two + * pages per bootstrap pmap. + */ + mov x12, #2 +#endif + add x0, x25, x12, lsl PGSHIFT + and x0, x0, #(TTBR_BADDR_MASK) + MSR_TTBR1_EL1_X0 + + // Set up MAIR attr0 for normal memory, attr1 for device memory + mov x0, xzr + mov x1, #(MAIR_WRITEBACK << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITEBACK)) + orr x0, x0, x1 + mov x1, #(MAIR_INNERWRITEBACK << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_INNERWRITEBACK)) + orr x0, x0, x1 + mov x1, #(MAIR_DISABLE << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_DISABLE)) + orr x0, x0, x1 + mov x1, #(MAIR_WRITETHRU << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITETHRU)) + orr x0, x0, x1 + mov x1, #(MAIR_WRITECOMB << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITECOMB)) + orr x0, x0, x1 + mov x1, #(MAIR_POSTED << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_POSTED)) + orr x0, x0, x1 + msr MAIR_EL1, x0 + + // Disable interrupts + msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) + +#if defined(APPLEHURRICANE) + + // Increase Snoop reservation in EDB to reduce starvation risk + // Needs to be done before MMU is enabled + mrs x12, ARM64_REG_HID5 + and x12, x12, (~ARM64_REG_HID5_CrdEdbSnpRsvd_mask) + orr x12, x12, ARM64_REG_HID5_CrdEdbSnpRsvd_VALUE + msr ARM64_REG_HID5, x12 + +#endif + + +#ifndef __ARM_IC_NOALIAS_ICACHE__ + /* Invalidate the TLB and icache on systems that do not guarantee that the + * caches are invalidated on reset. + */ + tlbi vmalle1 + ic iallu +#endif + + /* If x21 is not 0, then this is either the start_cpu path or + * the resume_idle_cpu path. cpu_ttep should already be + * populated, so just switch to the kernel_pmap now. + */ + + cbz x21, 1f + adrp x0, EXT(cpu_ttep)@page + add x0, x0, EXT(cpu_ttep)@pageoff + ldr x0, [x0] + MSR_TTBR1_EL1_X0 +1: + + // Set up the exception vectors + adrp x0, EXT(ExceptionVectorsBase)@page // Load exception vectors base address + add x0, x0, EXT(ExceptionVectorsBase)@pageoff + add x0, x0, x22 // Convert exception vector address to KVA + sub x0, x0, x23 + MSR_VBAR_EL1_X0 + + + // Enable caches and MMU + mov x0, #(SCTLR_EL1_DEFAULT & 0xFFFF) + mov x1, #(SCTLR_EL1_DEFAULT & 0xFFFF0000) + orr x0, x0, x1 + MSR_SCTLR_EL1_X0 + isb sy + +#if (!CONFIG_KERNEL_INTEGRITY || (CONFIG_KERNEL_INTEGRITY && !defined(KERNEL_INTEGRITY_WT))) + /* Watchtower + * + * If we have a Watchtower monitor it will setup CPACR_EL1 for us, touching + * it here would trap to EL3. + */ + + // Enable NEON + mov x0, #(CPACR_FPEN_ENABLE) + msr CPACR_EL1, x0 +#endif + + // Clear thread pointer + mov x0, #0 + msr TPIDR_EL1, x0 // Set thread register + +#if defined(APPLECYCLONE) || defined(APPLETYPHOON) + // + // Cyclone/Typhoon-Specific initialization + // For tunable summary, see Alcatraz/H6: Confirm Cyclone CPU tunables have been set + // + + // + // Disable LSP flush with context switch to work around bug in LSP + // that can cause Cyclone to wedge when CONTEXTIDR is written. + // Innsbruck11A175: panic(cpu 0 caller 0xffffff800024e30c): "wait queue deadlock - wq=0xffffff805a7a63c0, cpu=0\n" + // + + mrs x12, ARM64_REG_HID0 + orr x12, x12, ARM64_REG_HID0_LoopBuffDisb + msr ARM64_REG_HID0, x12 + + mrs x12, ARM64_REG_HID1 + orr x12, x12, ARM64_REG_HID1_rccDisStallInactiveIexCtl +#if defined(APPLECYCLONE) + orr x12, x12, ARM64_REG_HID1_disLspFlushWithContextSwitch +#endif + msr ARM64_REG_HID1, x12 + + mrs x12, ARM64_REG_HID3 + orr x12, x12, ARM64_REG_HID3_DisXmonSnpEvictTriggerL2StarvationMode + msr ARM64_REG_HID3, x12 + + // Do not disable cache ops -- XNU's cache operations already are no-op'ed for Cyclone, but explicit _Force variants are provided + // for when we really do need the L2 cache to be cleaned: Innsbruck11A416: Panic logs not preserved on h6 +/* + mrs x12, ARM64_REG_HID4 + orr x12, x12, ARM64_REG_HID4_DisDcMVAOps + orr x12, x12, ARM64_REG_HID4_DisDcSWL2Ops + msr ARM64_REG_HID4, x12 +*/ + + mrs x12, ARM64_REG_HID5 + and x12, x12, (~ARM64_REG_HID5_DisHwpLd) + and x12, x12, (~ARM64_REG_HID5_DisHwpSt) + msr ARM64_REG_HID5, x12 + + // Change the default memcache data set ID from 0 to 15 for all agents + mrs x12, ARM64_REG_HID8 + orr x12, x12, (ARM64_REG_HID8_DataSetID0_VALUE | ARM64_REG_HID8_DataSetID1_VALUE) +#if ARM64_BOARD_CONFIG_T7001 + orr x12, x12, ARM64_REG_HID8_DataSetID2_VALUE +#endif // ARM64_BOARD_CONFIG_T7001 + msr ARM64_REG_HID8, x12 + isb sy +#endif // APPLECYCLONE || APPLETYPHOON + +#if defined(APPLETWISTER) + mrs x12, ARM64_REG_HID11 + and x12, x12, (~ARM64_REG_HID11_DisFillC1BubOpt) + msr ARM64_REG_HID11, x12 + + // Change the default memcache data set ID from 0 to 15 for all agents + mrs x12, ARM64_REG_HID8 + orr x12, x12, (ARM64_REG_HID8_DataSetID0_VALUE | ARM64_REG_HID8_DataSetID1_VALUE) + orr x12, x12, (ARM64_REG_HID8_DataSetID2_VALUE | ARM64_REG_HID8_DataSetID3_VALUE) + msr ARM64_REG_HID8, x12 + + // Use 4-cycle MUL latency to avoid denormal stalls + mrs x12, ARM64_REG_HID7 + orr x12, x12, #ARM64_REG_HID7_disNexFastFmul + msr ARM64_REG_HID7, x12 + + // disable reporting of TLB-multi-hit-error + // + mrs x12, ARM64_REG_LSU_ERR_STS + and x12, x12, (~ARM64_REG_LSU_ERR_STS_L1DTlbMultiHitEN) + msr ARM64_REG_LSU_ERR_STS, x12 + + isb sy +#endif // APPLETWISTER + +#if defined(APPLEHURRICANE) + + // IC prefetch configuration + // + mrs x12, ARM64_REG_HID0 + and x12, x12, (~ARM64_REG_HID0_ICPrefDepth_bmsk) + orr x12, x12, (1 << ARM64_REG_HID0_ICPrefDepth_bshift) + orr x12, x12, ARM64_REG_HID0_ICPrefLimitOneBrn + msr ARM64_REG_HID0, x12 + + // disable reporting of TLB-multi-hit-error + // + mrs x12, ARM64_REG_LSU_ERR_CTL + and x12, x12, (~ARM64_REG_LSU_ERR_CTL_L1DTlbMultiHitEN) + msr ARM64_REG_LSU_ERR_CTL, x12 + + // disable crypto fusion across decode groups + // + mrs x12, ARM64_REG_HID1 + orr x12, x12, ARM64_REG_HID1_disAESFuseAcrossGrp + msr ARM64_REG_HID1, x12 + +#if defined(ARM64_BOARD_CONFIG_T8011) + // Clear DisDcZvaCmdOnly + // Per Myst A0/B0 tunables document + // https://seg-docs.csg.apple.com/projects/myst//release/UserManual/tunables_a0/ACC.html + // Myst: Confirm ACC Per-CPU Tunables + mrs x12, ARM64_REG_HID3 + and x12, x12, ~ARM64_REG_HID3_DisDcZvaCmdOnly + msr ARM64_REG_HID3, x12 + + mrs x12, ARM64_REG_EHID3 + and x12, x12, ~ARM64_REG_EHID3_DisDcZvaCmdOnly + msr ARM64_REG_EHID3, x12 +#endif /* defined(ARM64_BOARD_CONFIG_T8011) */ + +#endif // APPLEHURRICANE + + + // If x21 != 0, we're doing a warm reset, so we need to trampoline to the kernel pmap. + cbnz x21, Ltrampoline + + // Set KVA of boot args as first arg + add x0, x20, x22 + sub x0, x0, x23 + +#if KASAN + mov x20, x0 + mov x21, lr + + // x0: boot args + // x1: KVA page table phys base + mrs x1, TTBR1_EL1 + bl _kasan_bootstrap + + mov x0, x20 + mov lr, x21 +#endif + + // Return to arm_init() + ret + +Ltrampoline: + // Load VA of the trampoline + adrp x0, arm_init_tramp@page + add x0, x0, arm_init_tramp@pageoff + add x0, x0, x22 + sub x0, x0, x23 + + // Branch to the trampoline + br x0 + +/* + * V=P to KVA trampoline. + * x0 - KVA of cpu data pointer + */ + .text + .align 2 +arm_init_tramp: + /* On a warm boot, the full kernel translation table is initialized in + * addition to the bootstrap tables. The layout is as follows: + * + * +--Top of Memory--+ + * ... + * | | + * | Primary Kernel | + * | Trans. Table | + * | | + * +--Top + 5 pages--+ + * | | + * | Invalid Table | + * | | + * +--Top + 4 pages--+ + * | | + * | KVA Table | + * | | + * +--Top + 2 pages--+ + * | | + * | V=P Table | + * | | + * +--Top of Kernel--+ + * | | + * | Kernel Mach-O | + * | | + * ... + * +---Kernel Base---+ + */ + + + adrp x0, EXT(invalid_ttep)@page + add x0, x0, EXT(invalid_ttep)@pageoff + ldr x0, [x0] + + msr TTBR0_EL1, x0 + + // Convert CPU data PA to VA and set as first argument + add x0, x21, x22 + sub x0, x0, x23 + mov x1, #0 + + // Make sure that the TLB flush happens after the registers are set! + isb sy + + // Synchronize system for TTBR updates + tlbi vmalle1 + dsb sy + isb sy + + /* Return to arm_init() */ + ret + +//#include "globals_asm.h" + +/* vim: set ts=4: */ diff --git a/osfmk/arm64/status.c b/osfmk/arm64/status.c new file mode 100644 index 000000000..cf2d66cd8 --- /dev/null +++ b/osfmk/arm64/status.c @@ -0,0 +1,1455 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct arm_vfpv2_state +{ + __uint32_t __r[32]; + __uint32_t __fpscr; + +}; + +typedef struct arm_vfpv2_state arm_vfpv2_state_t; + +#define ARM_VFPV2_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_vfpv2_state_t)/sizeof(uint32_t))) + +/* + * Forward definitions + */ +void thread_set_child(thread_t child, int pid); +void thread_set_parent(thread_t parent, int pid); + +/* + * Maps state flavor to number of words in the state: + */ +/* __private_extern__ */ +unsigned int _MachineStateCount[] = { + /* FLAVOR_LIST */ 0, + ARM_UNIFIED_THREAD_STATE_COUNT, + ARM_VFP_STATE_COUNT, + ARM_EXCEPTION_STATE_COUNT, + ARM_DEBUG_STATE_COUNT, + /* THREAD_STATE_NONE (legacy) */ 0, + ARM_THREAD_STATE64_COUNT, + ARM_EXCEPTION_STATE64_COUNT, + /* THREAD_STATE_LAST (legacy) */ 0, + ARM_THREAD_STATE32_COUNT, + /* UNALLOCATED */ 0, + /* UNALLOCATED */ 0, + /* UNALLOCATED */ 0, + /* UNALLOCATED */ 0, + ARM_DEBUG_STATE32_COUNT, + ARM_DEBUG_STATE64_COUNT, + ARM_NEON_STATE_COUNT, + ARM_NEON_STATE64_COUNT, + /* UNALLOCATED */ 0, + /* UNALLOCATED */ 0, + /* ARM_SAVED_STATE32_COUNT */ 0, + /* ARM_SAVED_STATE64_COUNT */ 0, + /* ARM_NEON_SAVED_STATE32_COUNT */ 0, + /* ARM_NEON_SAVED_STATE64_COUNT */ 0, +}; + +extern zone_t ads_zone; + +#if __arm64__ +/* + * Copy values from saved_state to ts64. + */ +void +saved_state_to_thread_state64(const arm_saved_state_t *saved_state, arm_thread_state64_t *ts64) +{ + uint32_t i; + + assert(is_saved_state64(saved_state)); + + ts64->fp = get_saved_state_fp(saved_state); + ts64->lr = get_saved_state_lr(saved_state); + ts64->sp = get_saved_state_sp(saved_state); + ts64->pc = get_saved_state_pc(saved_state); + ts64->cpsr = get_saved_state_cpsr(saved_state); + for (i = 0; i < 29; i++) + ts64->x[i] = get_saved_state_reg(saved_state, i); +} + +/* + * Copy values from ts64 to saved_state + */ +void +thread_state64_to_saved_state(const arm_thread_state64_t *ts64, arm_saved_state_t *saved_state) +{ + uint32_t i; + + assert(is_saved_state64(saved_state)); + + set_saved_state_fp(saved_state, ts64->fp); + set_saved_state_lr(saved_state, ts64->lr); + set_saved_state_sp(saved_state, ts64->sp); + set_saved_state_pc(saved_state, ts64->pc); + set_saved_state_cpsr(saved_state, (ts64->cpsr & ~PSR64_MODE_MASK) | PSR64_MODE_RW_64); + for (i = 0; i < 29; i++) + set_saved_state_reg(saved_state, i, ts64->x[i]); +} +#endif + +kern_return_t +handle_get_arm32_thread_state( + thread_state_t tstate, + mach_msg_type_number_t * count, + const arm_saved_state_t *saved_state) +{ + if (*count < ARM_THREAD_STATE32_COUNT) + return (KERN_INVALID_ARGUMENT); + if (!is_saved_state32(saved_state)) + return (KERN_INVALID_ARGUMENT); + + (void)saved_state_to_thread_state32(saved_state, (arm_thread_state32_t *)tstate); + *count = ARM_THREAD_STATE32_COUNT; + return KERN_SUCCESS; +} + +kern_return_t +handle_get_arm64_thread_state( + thread_state_t tstate, + mach_msg_type_number_t * count, + const arm_saved_state_t *saved_state) +{ + if (*count < ARM_THREAD_STATE64_COUNT) + return (KERN_INVALID_ARGUMENT); + if (!is_saved_state64(saved_state)) + return (KERN_INVALID_ARGUMENT); + + (void)saved_state_to_thread_state64(saved_state, (arm_thread_state64_t *)tstate); + *count = ARM_THREAD_STATE64_COUNT; + return KERN_SUCCESS; +} + + +kern_return_t +handle_get_arm_thread_state( + thread_state_t tstate, + mach_msg_type_number_t * count, + const arm_saved_state_t *saved_state) +{ + /* In an arm64 world, this flavor can be used to retrieve the thread + * state of a 32-bit or 64-bit thread into a unified structure, but we + * need to support legacy clients who are only aware of 32-bit, so + * check the count to see what the client is expecting. + */ + if (*count < ARM_UNIFIED_THREAD_STATE_COUNT) { + return handle_get_arm32_thread_state(tstate, count, saved_state); + } + + arm_unified_thread_state_t *unified_state = (arm_unified_thread_state_t *) tstate; + bzero(unified_state, sizeof(*unified_state)); +#if __arm64__ + if (is_saved_state64(saved_state)) { + unified_state->ash.flavor = ARM_THREAD_STATE64; + unified_state->ash.count = ARM_THREAD_STATE64_COUNT; + (void)saved_state_to_thread_state64(saved_state, thread_state64(unified_state)); + } else +#endif + { + unified_state->ash.flavor = ARM_THREAD_STATE32; + unified_state->ash.count = ARM_THREAD_STATE32_COUNT; + (void)saved_state_to_thread_state32(saved_state, thread_state32(unified_state)); + } + *count = ARM_UNIFIED_THREAD_STATE_COUNT; + return (KERN_SUCCESS); +} + +kern_return_t +handle_set_arm32_thread_state( + const thread_state_t tstate, + mach_msg_type_number_t count, + arm_saved_state_t *saved_state) +{ + if (count != ARM_THREAD_STATE32_COUNT) + return (KERN_INVALID_ARGUMENT); + + (void)thread_state32_to_saved_state((const arm_thread_state32_t *)tstate, saved_state); + return KERN_SUCCESS; +} + +kern_return_t +handle_set_arm64_thread_state( + const thread_state_t tstate, + mach_msg_type_number_t count, + arm_saved_state_t *saved_state) +{ + if (count != ARM_THREAD_STATE64_COUNT) + return (KERN_INVALID_ARGUMENT); + + (void)thread_state64_to_saved_state((const arm_thread_state64_t *)tstate, saved_state); + return KERN_SUCCESS; +} + + +kern_return_t +handle_set_arm_thread_state( + const thread_state_t tstate, + mach_msg_type_number_t count, + arm_saved_state_t *saved_state) +{ + /* In an arm64 world, this flavor can be used to set the thread state of a + * 32-bit or 64-bit thread from a unified structure, but we need to support + * legacy clients who are only aware of 32-bit, so check the count to see + * what the client is expecting. + */ + if (count < ARM_UNIFIED_THREAD_STATE_COUNT) { + return handle_set_arm32_thread_state(tstate, count, saved_state); + } + + const arm_unified_thread_state_t *unified_state = (const arm_unified_thread_state_t *) tstate; +#if __arm64__ + if (is_thread_state64(unified_state)) { + (void)thread_state64_to_saved_state(const_thread_state64(unified_state), saved_state); + } else +#endif + { + (void)thread_state32_to_saved_state(const_thread_state32(unified_state), saved_state); + } + + return (KERN_SUCCESS); +} + +/* + * Routine: machine_thread_get_state + * + */ +kern_return_t +machine_thread_get_state( + thread_t thread, + thread_flavor_t flavor, + thread_state_t tstate, + mach_msg_type_number_t * count) +{ + switch (flavor) { + case THREAD_STATE_FLAVOR_LIST: + if (*count < 4) + return (KERN_INVALID_ARGUMENT); + + tstate[0] = ARM_THREAD_STATE; + tstate[1] = ARM_VFP_STATE; + tstate[2] = ARM_EXCEPTION_STATE; + tstate[3] = ARM_DEBUG_STATE; + *count = 4; + break; + + case THREAD_STATE_FLAVOR_LIST_NEW: + if (*count < 4) + return (KERN_INVALID_ARGUMENT); + + tstate[0] = ARM_THREAD_STATE; + tstate[1] = ARM_VFP_STATE; + tstate[2] = thread_is_64bit(thread) ? ARM_EXCEPTION_STATE64 : ARM_EXCEPTION_STATE; + tstate[3] = thread_is_64bit(thread) ? ARM_DEBUG_STATE64 : ARM_DEBUG_STATE32; + *count = 4; + break; + + case ARM_THREAD_STATE: + { + kern_return_t rn = handle_get_arm_thread_state(tstate, count, thread->machine.upcb); + if (rn) return rn; + break; + } + case ARM_THREAD_STATE32: + { + if (thread_is_64bit(thread)) + return KERN_INVALID_ARGUMENT; + + kern_return_t rn = handle_get_arm32_thread_state(tstate, count, thread->machine.upcb); + if (rn) return rn; + break; + } +#if __arm64__ + case ARM_THREAD_STATE64: + { + if (!thread_is_64bit(thread)) + return KERN_INVALID_ARGUMENT; + + kern_return_t rn = handle_get_arm64_thread_state(tstate, count, thread->machine.upcb); + if (rn) return rn; + break; + } +#endif + case ARM_EXCEPTION_STATE:{ + struct arm_exception_state *state; + struct arm_saved_state32 *saved_state; + + if (*count < ARM_EXCEPTION_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + if (thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (struct arm_exception_state *) tstate; + saved_state = saved_state32(thread->machine.upcb); + + state->exception = saved_state->exception; + state->fsr = saved_state->esr; + state->far = saved_state->far; + + *count = ARM_EXCEPTION_STATE_COUNT; + break; + } + case ARM_EXCEPTION_STATE64:{ + struct arm_exception_state64 *state; + struct arm_saved_state64 *saved_state; + + if (*count < ARM_EXCEPTION_STATE64_COUNT) + return (KERN_INVALID_ARGUMENT); + if (!thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (struct arm_exception_state64 *) tstate; + saved_state = saved_state64(thread->machine.upcb); + + state->exception = saved_state->exception; + state->far = saved_state->far; + state->esr = saved_state->esr; + + *count = ARM_EXCEPTION_STATE64_COUNT; + break; + } + case ARM_DEBUG_STATE:{ + arm_legacy_debug_state_t *state; + arm_debug_state32_t *thread_state; + + if (*count < ARM_LEGACY_DEBUG_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (arm_legacy_debug_state_t *) tstate; + thread_state = find_debug_state32(thread); + + if (thread_state == NULL) + bzero(state, sizeof(arm_legacy_debug_state_t)); + else + bcopy(thread_state, state, sizeof(arm_legacy_debug_state_t)); + + *count = ARM_LEGACY_DEBUG_STATE_COUNT; + break; + } + case ARM_DEBUG_STATE32:{ + arm_debug_state32_t *state; + arm_debug_state32_t *thread_state; + + if (*count < ARM_DEBUG_STATE32_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (arm_debug_state32_t *) tstate; + thread_state = find_debug_state32(thread); + + if (thread_state == NULL) + bzero(state, sizeof(arm_debug_state32_t)); + else + bcopy(thread_state, state, sizeof(arm_debug_state32_t)); + + *count = ARM_DEBUG_STATE32_COUNT; + break; + } + + case ARM_DEBUG_STATE64:{ + arm_debug_state64_t *state; + arm_debug_state64_t *thread_state; + + if (*count < ARM_DEBUG_STATE64_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (!thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (arm_debug_state64_t *) tstate; + thread_state = find_debug_state64(thread); + + if (thread_state == NULL) + bzero(state, sizeof(arm_debug_state64_t)); + else + bcopy(thread_state, state, sizeof(arm_debug_state64_t)); + + *count = ARM_DEBUG_STATE64_COUNT; + break; + } + + case ARM_VFP_STATE:{ + struct arm_vfp_state *state; + arm_neon_saved_state32_t *thread_state; + unsigned int max; + + if (*count < ARM_VFP_STATE_COUNT) { + if (*count < ARM_VFPV2_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + else + *count = ARM_VFPV2_STATE_COUNT; + } + + if (*count == ARM_VFPV2_STATE_COUNT) + max = 32; + else + max = 64; + + state = (struct arm_vfp_state *) tstate; + thread_state = neon_state32(thread->machine.uNeon); + /* ARM64 TODO: set fpsr and fpcr from state->fpscr */ + + bcopy(thread_state, state, (max + 1)*sizeof(uint32_t)); + *count = (max + 1); + break; + } + case ARM_NEON_STATE:{ + arm_neon_state_t *state; + arm_neon_saved_state32_t *thread_state; + + if (*count < ARM_NEON_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (arm_neon_state_t *)tstate; + thread_state = neon_state32(thread->machine.uNeon); + + assert(sizeof(*thread_state) == sizeof(*state)); + bcopy(thread_state, state, sizeof(arm_neon_state_t)); + + *count = ARM_NEON_STATE_COUNT; + break; + + } + + case ARM_NEON_STATE64:{ + arm_neon_state64_t *state; + arm_neon_saved_state64_t *thread_state; + + if (*count < ARM_NEON_STATE64_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (!thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (arm_neon_state64_t *)tstate; + thread_state = neon_state64(thread->machine.uNeon); + + /* For now, these are identical */ + assert(sizeof(*state) == sizeof(*thread_state)); + bcopy(thread_state, state, sizeof(arm_neon_state64_t)); + + *count = ARM_NEON_STATE64_COUNT; + break; + + } + + default: + return (KERN_INVALID_ARGUMENT); + } + return (KERN_SUCCESS); +} + + +/* + * Routine: machine_thread_get_kern_state + * + */ +kern_return_t +machine_thread_get_kern_state( + thread_t thread, + thread_flavor_t flavor, + thread_state_t tstate, + mach_msg_type_number_t * count) +{ + /* + * This works only for an interrupted kernel thread + */ + if (thread != current_thread() || getCpuDatap()->cpu_int_state == NULL) + return KERN_FAILURE; + + switch (flavor) { + case ARM_THREAD_STATE: + { + kern_return_t rn = handle_get_arm_thread_state(tstate, count, getCpuDatap()->cpu_int_state); + if (rn) return rn; + break; + } + case ARM_THREAD_STATE32: + { + kern_return_t rn = handle_get_arm32_thread_state(tstate, count, getCpuDatap()->cpu_int_state); + if (rn) return rn; + break; + } +#if __arm64__ + case ARM_THREAD_STATE64: + { + kern_return_t rn = handle_get_arm64_thread_state(tstate, count, getCpuDatap()->cpu_int_state); + if (rn) return rn; + break; + } +#endif + default: + return (KERN_INVALID_ARGUMENT); + } + return (KERN_SUCCESS); +} + +void +machine_thread_switch_addrmode(thread_t thread) +{ + if (task_has_64BitAddr(thread->task)) { + thread->machine.upcb->ash.flavor = ARM_SAVED_STATE64; + thread->machine.upcb->ash.count = ARM_SAVED_STATE64_COUNT; + thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE64; + thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE64_COUNT; + + /* + * Reinitialize the NEON state. + */ + bzero(&thread->machine.uNeon->uns, sizeof(thread->machine.uNeon->uns)); + thread->machine.uNeon->ns_64.fpcr = FPCR_DEFAULT; + } else { + thread->machine.upcb->ash.flavor = ARM_SAVED_STATE32; + thread->machine.upcb->ash.count = ARM_SAVED_STATE32_COUNT; + thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32; + thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT; + + /* + * Reinitialize the NEON state. + */ + bzero(&thread->machine.uNeon->uns, sizeof(thread->machine.uNeon->uns)); + thread->machine.uNeon->ns_32.fpcr = FPCR_DEFAULT_32; + } +} + +extern long long arm_debug_get(void); + +/* + * Routine: machine_thread_set_state + * + */ +kern_return_t +machine_thread_set_state( + thread_t thread, + thread_flavor_t flavor, + thread_state_t tstate, + mach_msg_type_number_t count) +{ + kern_return_t rn; + + switch (flavor) { + case ARM_THREAD_STATE: + rn = handle_set_arm_thread_state(tstate, count, thread->machine.upcb); + if (rn) return rn; + break; + + case ARM_THREAD_STATE32: + if (thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + rn = handle_set_arm32_thread_state(tstate, count, thread->machine.upcb); + if (rn) return rn; + break; + +#if __arm64__ + case ARM_THREAD_STATE64: + if (!thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + rn = handle_set_arm64_thread_state(tstate, count, thread->machine.upcb); + if (rn) return rn; + break; +#endif + case ARM_EXCEPTION_STATE:{ + + if (count != ARM_EXCEPTION_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + if (thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + break; + } + case ARM_EXCEPTION_STATE64:{ + + if (count != ARM_EXCEPTION_STATE64_COUNT) + return (KERN_INVALID_ARGUMENT); + if (!thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + break; + } + case ARM_DEBUG_STATE: + { + arm_legacy_debug_state_t *state; + boolean_t enabled = FALSE; + unsigned int i; + + if (count != ARM_LEGACY_DEBUG_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + if (thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (arm_legacy_debug_state_t *) tstate; + + for (i = 0; i < 16; i++) { + /* do not allow context IDs to be set */ + if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA) + || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED) + || ((state->wcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA) + || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) { + return KERN_PROTECTION_FAILURE; + } + if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) + || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) { + enabled = TRUE; + } + } + + + if (!enabled) { + arm_debug_state32_t *thread_state = find_debug_state32(thread); + if (thread_state != NULL) { + void *pTmp = thread->machine.DebugData; + thread->machine.DebugData = NULL; + zfree(ads_zone, pTmp); + } + } else { + arm_debug_state32_t *thread_state = find_debug_state32(thread); + if (thread_state == NULL) { + thread->machine.DebugData = zalloc(ads_zone); + bzero(thread->machine.DebugData, sizeof *(thread->machine.DebugData)); + thread->machine.DebugData->dsh.flavor = ARM_DEBUG_STATE32; + thread->machine.DebugData->dsh.count = ARM_DEBUG_STATE32_COUNT; + thread_state = find_debug_state32(thread); + } + assert(NULL != thread_state); + + for (i = 0; i < 16; i++) { + /* set appropriate privilege; mask out unknown bits */ + thread_state->bcr[i] = (state->bcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK + | ARM_DBGBCR_MATCH_MASK + | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK + | ARM_DBG_CR_ENABLE_MASK)) + | ARM_DBGBCR_TYPE_IVA + | ARM_DBG_CR_LINKED_UNLINKED + | ARM_DBG_CR_SECURITY_STATE_BOTH + | ARM_DBG_CR_MODE_CONTROL_USER; + thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK; + thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK + | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK + | ARM_DBGWCR_ACCESS_CONTROL_MASK + | ARM_DBG_CR_ENABLE_MASK)) + | ARM_DBG_CR_LINKED_UNLINKED + | ARM_DBG_CR_SECURITY_STATE_BOTH + | ARM_DBG_CR_MODE_CONTROL_USER; + thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK; + } + + thread_state->mdscr_el1 = 0ULL; // Legacy customers issuing ARM_DEBUG_STATE dont drive single stepping. + } + + if (thread == current_thread()) { + arm_debug_set32(thread->machine.DebugData); + } + + break; + } + case ARM_DEBUG_STATE32: + /* ARM64_TODO subtle bcr/wcr semantic differences e.g. wcr and ARM_DBGBCR_TYPE_IVA */ + { + arm_debug_state32_t *state; + boolean_t enabled = FALSE; + unsigned int i; + + if (count != ARM_DEBUG_STATE32_COUNT) + return (KERN_INVALID_ARGUMENT); + if (thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (arm_debug_state32_t *) tstate; + + if (state->mdscr_el1 & 0x1) + enabled = TRUE; + + for (i = 0; i < 16; i++) { + /* do not allow context IDs to be set */ + if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA) + || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED) + || ((state->wcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA) + || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) { + return KERN_PROTECTION_FAILURE; + } + if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) + || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) { + enabled = TRUE; + } + } + + if (!enabled) { + arm_debug_state32_t *thread_state = find_debug_state32(thread); + if (thread_state != NULL) { + void *pTmp = thread->machine.DebugData; + thread->machine.DebugData = NULL; + zfree(ads_zone, pTmp); + } + } else { + arm_debug_state32_t *thread_state = find_debug_state32(thread); + if (thread_state == NULL) { + thread->machine.DebugData = zalloc(ads_zone); + bzero(thread->machine.DebugData, sizeof *(thread->machine.DebugData)); + thread->machine.DebugData->dsh.flavor = ARM_DEBUG_STATE32; + thread->machine.DebugData->dsh.count = ARM_DEBUG_STATE32_COUNT; + thread_state = find_debug_state32(thread); + } + assert(NULL != thread_state); + + if (state->mdscr_el1 & 0x1) + thread_state->mdscr_el1 |= 0x1; + else + thread_state->mdscr_el1 &= ~0x1; + + for (i = 0; i < 16; i++) { + /* set appropriate privilege; mask out unknown bits */ + thread_state->bcr[i] = (state->bcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK + | ARM_DBGBCR_MATCH_MASK + | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK + | ARM_DBG_CR_ENABLE_MASK)) + | ARM_DBGBCR_TYPE_IVA + | ARM_DBG_CR_LINKED_UNLINKED + | ARM_DBG_CR_SECURITY_STATE_BOTH + | ARM_DBG_CR_MODE_CONTROL_USER; + thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK; + thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK + | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK + | ARM_DBGWCR_ACCESS_CONTROL_MASK + | ARM_DBG_CR_ENABLE_MASK)) + | ARM_DBG_CR_LINKED_UNLINKED + | ARM_DBG_CR_SECURITY_STATE_BOTH + | ARM_DBG_CR_MODE_CONTROL_USER; + thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK; + } + + } + + if (thread == current_thread()) { + arm_debug_set32(thread->machine.DebugData); + } + + break; + } + + case ARM_DEBUG_STATE64: + { + arm_debug_state64_t *state; + boolean_t enabled = FALSE; + unsigned int i; + + if (count != ARM_DEBUG_STATE64_COUNT) + return (KERN_INVALID_ARGUMENT); + if (!thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (arm_debug_state64_t *) tstate; + + if (state->mdscr_el1 & 0x1) + enabled = TRUE; + + for (i = 0; i < 16; i++) { + /* do not allow context IDs to be set */ + if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA) + || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED) + || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) { + return KERN_PROTECTION_FAILURE; + } + if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) + || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) { + enabled = TRUE; + } + } + + if (!enabled) { + arm_debug_state64_t *thread_state = find_debug_state64(thread); + if (thread_state != NULL) { + void *pTmp = thread->machine.DebugData; + thread->machine.DebugData = NULL; + zfree(ads_zone, pTmp); + } + } else { + arm_debug_state64_t *thread_state = find_debug_state64(thread); + if (thread_state == NULL) { + thread->machine.DebugData = zalloc(ads_zone); + bzero(thread->machine.DebugData, sizeof *(thread->machine.DebugData)); + thread->machine.DebugData->dsh.flavor = ARM_DEBUG_STATE64; + thread->machine.DebugData->dsh.count = ARM_DEBUG_STATE64_COUNT; + thread_state = find_debug_state64(thread); + } + assert(NULL != thread_state); + + if (state->mdscr_el1 & 0x1) + thread_state->mdscr_el1 |= 0x1; + else + thread_state->mdscr_el1 &= ~0x1; + + for (i = 0; i < 16; i++) { + /* set appropriate privilege; mask out unknown bits */ + thread_state->bcr[i] = (state->bcr[i] & (0 /* Was ARM_DBG_CR_ADDRESS_MASK_MASK deprecated in v8 */ + | 0 /* Was ARM_DBGBCR_MATCH_MASK, ignored in AArch64 state */ + | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK + | ARM_DBG_CR_ENABLE_MASK)) + | ARM_DBGBCR_TYPE_IVA + | ARM_DBG_CR_LINKED_UNLINKED + | ARM_DBG_CR_SECURITY_STATE_BOTH + | ARM_DBG_CR_MODE_CONTROL_USER; + thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK64; + thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK + | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK + | ARM_DBGWCR_ACCESS_CONTROL_MASK + | ARM_DBG_CR_ENABLE_MASK)) + | ARM_DBG_CR_LINKED_UNLINKED + | ARM_DBG_CR_SECURITY_STATE_BOTH + | ARM_DBG_CR_MODE_CONTROL_USER; + thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK64; + } + + } + + if (thread == current_thread()) { + arm_debug_set64(thread->machine.DebugData); + } + + break; + } + + case ARM_VFP_STATE:{ + struct arm_vfp_state *state; + arm_neon_saved_state32_t *thread_state; + unsigned int max; + + if (count != ARM_VFP_STATE_COUNT && count != ARM_VFPV2_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (count == ARM_VFPV2_STATE_COUNT) + max = 32; + else + max = 64; + + state = (struct arm_vfp_state *) tstate; + thread_state = neon_state32(thread->machine.uNeon); + /* ARM64 TODO: combine fpsr and fpcr into state->fpscr */ + + bcopy(state, thread_state, (max + 1)*sizeof(uint32_t)); + + thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32; + thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT; + break; + } + + case ARM_NEON_STATE:{ + arm_neon_state_t *state; + arm_neon_saved_state32_t *thread_state; + + if (count != ARM_NEON_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (arm_neon_state_t *)tstate; + thread_state = neon_state32(thread->machine.uNeon); + + assert(sizeof(*state) == sizeof(*thread_state)); + bcopy(state, thread_state, sizeof(arm_neon_state_t)); + + thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32; + thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT; + break; + + } + + case ARM_NEON_STATE64:{ + arm_neon_state64_t *state; + arm_neon_saved_state64_t *thread_state; + + if (count != ARM_NEON_STATE64_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (!thread_is_64bit(thread)) + return (KERN_INVALID_ARGUMENT); + + state = (arm_neon_state64_t *)tstate; + thread_state = neon_state64(thread->machine.uNeon); + + assert(sizeof(*state) == sizeof(*thread_state)); + bcopy(state, thread_state, sizeof(arm_neon_state64_t)); + + thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE64; + thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE64_COUNT; + break; + + } + + default: + return (KERN_INVALID_ARGUMENT); + } + return (KERN_SUCCESS); +} + +/* + * Routine: machine_thread_state_initialize + * + */ +kern_return_t +machine_thread_state_initialize( + thread_t thread) +{ + arm_context_t *context = thread->machine.contextData; + + /* + * Should always be set up later. For a kernel thread, we don't care + * about this state. For a user thread, we'll set the state up in + * setup_wqthread, bsdthread_create, load_main(), or load_unixthread(). + */ + + if (context != NULL) { + bzero(&context->ss.uss, sizeof(context->ss.uss)); + bzero(&context->ns.uns, sizeof(context->ns.uns)); + + if (context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64) { + context->ns.ns_64.fpcr = FPCR_DEFAULT; + } else { + context->ns.ns_32.fpcr = FPCR_DEFAULT_32; + } + } + + thread->machine.DebugData = NULL; + + return KERN_SUCCESS; +} + +/* + * Routine: machine_thread_dup + * + */ +kern_return_t +machine_thread_dup( + thread_t self, + thread_t target) +{ + struct arm_saved_state *self_saved_state; + struct arm_saved_state *target_saved_state; + + target->machine.cthread_self = self->machine.cthread_self; + target->machine.cthread_data = self->machine.cthread_data; + + self_saved_state = self->machine.upcb; + target_saved_state = target->machine.upcb; + bcopy(self_saved_state, target_saved_state, sizeof(struct arm_saved_state)); + + return (KERN_SUCCESS); +} + +/* + * Routine: get_user_regs + * + */ +struct arm_saved_state * +get_user_regs( + thread_t thread) +{ + return (thread->machine.upcb); +} + +arm_neon_saved_state_t * +get_user_neon_regs( + thread_t thread) +{ + return (thread->machine.uNeon); +} + +/* + * Routine: find_user_regs + * + */ +struct arm_saved_state * +find_user_regs( + thread_t thread) +{ + return (thread->machine.upcb); +} + +/* + * Routine: find_kern_regs + * + */ +struct arm_saved_state * +find_kern_regs( + thread_t thread) +{ + /* + * This works only for an interrupted kernel thread + */ + if (thread != current_thread() || getCpuDatap()->cpu_int_state == NULL) + return ((struct arm_saved_state *) NULL); + else + return (getCpuDatap()->cpu_int_state); + +} + +arm_debug_state32_t * +find_debug_state32( + thread_t thread) +{ + if (thread && thread->machine.DebugData) + return &(thread->machine.DebugData->uds.ds32); + else + return NULL; +} + +arm_debug_state64_t * +find_debug_state64( + thread_t thread) +{ + if (thread && thread->machine.DebugData) + return &(thread->machine.DebugData->uds.ds64); + else + return NULL; +} + +/* + * Routine: thread_userstack + * + */ +kern_return_t +thread_userstack( + thread_t thread, + int flavor, + thread_state_t tstate, + unsigned int count, + mach_vm_offset_t * user_stack, + int *customstack, + boolean_t is64bit +) +{ + register_t sp; + + switch (flavor) { + case ARM_THREAD_STATE: + if (count == ARM_UNIFIED_THREAD_STATE_COUNT) { +#if __arm64__ + if (thread_is_64bit(thread)) { + sp = ((arm_unified_thread_state_t *)tstate)->ts_64.sp; + } else +#endif + { + sp = ((arm_unified_thread_state_t *)tstate)->ts_32.sp; + } + + break; + } + + /* INTENTIONAL FALL THROUGH (see machine_thread_set_state) */ + case ARM_THREAD_STATE32: + if (count != ARM_THREAD_STATE32_COUNT) + return (KERN_INVALID_ARGUMENT); + if (is64bit) + return (KERN_INVALID_ARGUMENT); + + sp = ((arm_thread_state32_t *)tstate)->sp; + break; +#if __arm64__ + case ARM_THREAD_STATE64: + if (count != ARM_THREAD_STATE64_COUNT) + return (KERN_INVALID_ARGUMENT); + if (!is64bit) + return (KERN_INVALID_ARGUMENT); + + sp = ((arm_thread_state32_t *)tstate)->sp; + break; +#endif + default: + return (KERN_INVALID_ARGUMENT); + } + + if (sp) { + *user_stack = CAST_USER_ADDR_T(sp); + if (customstack) + *customstack = 1; + } else { + *user_stack = CAST_USER_ADDR_T(USRSTACK64); + if (customstack) + *customstack = 0; + } + + return (KERN_SUCCESS); +} + +/* + * thread_userstackdefault: + * + * Return the default stack location for the + * thread, if otherwise unknown. + */ +kern_return_t +thread_userstackdefault( + mach_vm_offset_t *default_user_stack, + boolean_t is64bit) +{ + if (is64bit) { + *default_user_stack = USRSTACK64; + } else { + *default_user_stack = USRSTACK; + } + + return (KERN_SUCCESS); +} + +/* + * Routine: thread_setuserstack + * + */ +void +thread_setuserstack(thread_t thread, mach_vm_address_t user_stack) +{ + struct arm_saved_state *sv; + + sv = get_user_regs(thread); + + set_saved_state_sp(sv, user_stack); + + return; +} + +/* + * Routine: thread_adjuserstack + * + */ +uint64_t +thread_adjuserstack(thread_t thread, int adjust) +{ + struct arm_saved_state *sv; + uint64_t sp; + + sv = get_user_regs(thread); + + sp = get_saved_state_sp(sv); + sp += adjust; + set_saved_state_sp(sv, sp);; + + return sp; +} + +/* + * Routine: thread_setentrypoint + * + */ +void +thread_setentrypoint(thread_t thread, mach_vm_offset_t entry) +{ + struct arm_saved_state *sv; + + sv = get_user_regs(thread); + + set_saved_state_pc(sv, entry); + + return; +} + +/* + * Routine: thread_entrypoint + * + */ +kern_return_t +thread_entrypoint( + __unused thread_t thread, + int flavor, + thread_state_t tstate, + unsigned int count __unused, + mach_vm_offset_t * entry_point +) +{ + switch (flavor) { + case ARM_THREAD_STATE: + { + struct arm_thread_state *state; + + state = (struct arm_thread_state *) tstate; + + /* + * If a valid entry point is specified, use it. + */ + if (state->pc) { + *entry_point = CAST_USER_ADDR_T(state->pc); + } else { + *entry_point = CAST_USER_ADDR_T(VM_MIN_ADDRESS); + } + } + break; + + case ARM_THREAD_STATE64: + { + struct arm_thread_state64 *state; + + state = (struct arm_thread_state64*) tstate; + + /* + * If a valid entry point is specified, use it. + */ + if (state->pc) { + *entry_point = CAST_USER_ADDR_T(state->pc); + } else { + *entry_point = CAST_USER_ADDR_T(VM_MIN_ADDRESS); + } + + break; + } + default: + return (KERN_INVALID_ARGUMENT); + } + + return (KERN_SUCCESS); +} + + +/* + * Routine: thread_set_child + * + */ +void +thread_set_child( + thread_t child, + int pid) +{ + struct arm_saved_state *child_state; + + child_state = get_user_regs(child); + + set_saved_state_reg(child_state, 0, pid); + set_saved_state_reg(child_state, 1, 1ULL); +} + + +/* + * Routine: thread_set_parent + * + */ +void +thread_set_parent( + thread_t parent, + int pid) +{ + struct arm_saved_state *parent_state; + + parent_state = get_user_regs(parent); + + set_saved_state_reg(parent_state, 0, pid); + set_saved_state_reg(parent_state, 1, 0); +} + + +struct arm_act_context { + struct arm_unified_thread_state ss; +#if __ARM_VFP__ + struct arm_neon_saved_state ns; +#endif +}; + +/* + * Routine: act_thread_csave + * + */ +void * +act_thread_csave(void) +{ + struct arm_act_context *ic; + kern_return_t kret; + unsigned int val; + thread_t thread = current_thread(); + + ic = (struct arm_act_context *) kalloc(sizeof(struct arm_act_context)); + if (ic == (struct arm_act_context *) NULL) + return ((void *) 0); + + val = ARM_UNIFIED_THREAD_STATE_COUNT; + kret = machine_thread_get_state(thread, ARM_THREAD_STATE, (thread_state_t)&ic->ss, &val); + if (kret != KERN_SUCCESS) { + kfree(ic, sizeof(struct arm_act_context)); + return ((void *) 0); + } + +#if __ARM_VFP__ + if (thread_is_64bit(thread)) { + val = ARM_NEON_STATE64_COUNT; + kret = machine_thread_get_state(thread, + ARM_NEON_STATE64, + (thread_state_t) & ic->ns, + &val); + } else { + val = ARM_NEON_STATE_COUNT; + kret = machine_thread_get_state(thread, + ARM_NEON_STATE, + (thread_state_t) & ic->ns, + &val); + } + if (kret != KERN_SUCCESS) { + kfree(ic, sizeof(struct arm_act_context)); + return ((void *) 0); + } +#endif + return (ic); +} + +/* + * Routine: act_thread_catt + * + */ +void +act_thread_catt(void *ctx) +{ + struct arm_act_context *ic; + kern_return_t kret; + thread_t thread = current_thread(); + + ic = (struct arm_act_context *) ctx; + if (ic == (struct arm_act_context *) NULL) + return; + + kret = machine_thread_set_state(thread, ARM_THREAD_STATE, (thread_state_t)&ic->ss, ARM_UNIFIED_THREAD_STATE_COUNT); + if (kret != KERN_SUCCESS) + goto out; + +#if __ARM_VFP__ + if (thread_is_64bit(thread)) { + kret = machine_thread_set_state(thread, + ARM_NEON_STATE64, + (thread_state_t) & ic->ns, + ARM_NEON_STATE64_COUNT); + } else { + kret = machine_thread_set_state(thread, + ARM_NEON_STATE, + (thread_state_t) & ic->ns, + ARM_NEON_STATE_COUNT); + } + if (kret != KERN_SUCCESS) + goto out; +#endif +out: + kfree(ic, sizeof(struct arm_act_context)); +} + +/* + * Routine: act_thread_catt + * + */ +void +act_thread_cfree(void *ctx) +{ + kfree(ctx, sizeof(struct arm_act_context)); +} + +kern_return_t +thread_set_wq_state32(thread_t thread, thread_state_t tstate) +{ + arm_thread_state_t *state; + struct arm_saved_state *saved_state; + struct arm_saved_state32 *saved_state_32; + thread_t curth = current_thread(); + spl_t s=0; + + assert(!thread_is_64bit(thread)); + + saved_state = thread->machine.upcb; + saved_state_32 = saved_state32(saved_state); + + state = (arm_thread_state_t *)tstate; + + if (curth != thread) { + s = splsched(); + thread_lock(thread); + } + + /* + * do not zero saved_state, it can be concurrently accessed + * and zero is not a valid state for some of the registers, + * like sp. + */ + thread_state32_to_saved_state(state, saved_state); + saved_state_32->cpsr = PSR64_USER32_DEFAULT; + + if (curth != thread) { + thread_unlock(thread); + splx(s); + } + + return KERN_SUCCESS; +} + +kern_return_t +thread_set_wq_state64(thread_t thread, thread_state_t tstate) +{ + arm_thread_state64_t *state; + struct arm_saved_state *saved_state; + struct arm_saved_state64 *saved_state_64; + thread_t curth = current_thread(); + spl_t s=0; + + assert(thread_is_64bit(thread)); + + saved_state = thread->machine.upcb; + saved_state_64 = saved_state64(saved_state); + state = (arm_thread_state64_t *)tstate; + + if (curth != thread) { + s = splsched(); + thread_lock(thread); + } + + /* + * do not zero saved_state, it can be concurrently accessed + * and zero is not a valid state for some of the registers, + * like sp. + */ + thread_state64_to_saved_state(state, saved_state); + saved_state_64->cpsr = PSR64_USER64_DEFAULT; + + if (curth != thread) { + thread_unlock(thread); + splx(s); + } + + return KERN_SUCCESS; +} diff --git a/osfmk/arm64/strncmp.s b/osfmk/arm64/strncmp.s new file mode 100644 index 000000000..eee2de722 --- /dev/null +++ b/osfmk/arm64/strncmp.s @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2012 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + * + * This file implements the following function for the arm64 architecture: + * + * int strncmp(const char *s1, const char *s2, size_t n); + * + * Returns 0 if the two strings are equal up to the first n bytes or to the + * end of the string, whichever comes first. Otherwise, returns the difference + * of the first mismatched characters interpreted as uint8_t. + */ + +.globl _strncmp + +/***************************************************************************** + * Macros * + *****************************************************************************/ + +.macro EstablishFrame + stp fp, lr, [sp, #-16]! + mov fp, sp +.endm + +.macro ClearFrameAndReturn + ldp fp, lr, [sp], #16 + ret +.endm + +#include "../mach/arm/vm_param.h" +#define kVectorSize 16 + +/***************************************************************************** + * Constants * + *****************************************************************************/ + +.text +.align 5 +L_mask: +.quad 0x0706050403020100, 0x0f0e0d0c0b0a0908 + +/***************************************************************************** + * Entrypoints * + *****************************************************************************/ + +_strncmp: + EstablishFrame + eor x3, x3, x3 + cbz x2, L_scalarDone +// Compare one byte at a time until s1 has vector alignment. +0: tst x0, #(kVectorSize-1) + b.eq L_s1aligned + ldrb w4, [x0],#1 // load byte from src1 + ldrb w5, [x1],#1 // load byte from src2 + subs x3, x4, x5 // if the are not equal + ccmp w4, #0, #4, eq // or we find an EOS + b.eq L_scalarDone // return the difference + subs x2, x2, #1 // decrement length + b.ne 0b // continue loop if non-zero + +// We found a mismatch or EOS before s1 became aligned. Simply return the +// difference between the last bytes that we loaded. +L_scalarDone: + mov x0, x3 + ClearFrameAndReturn + +L_s1aligned: +// If s2 is similarly aligned to s1, then we can use a naive vector comparison +// from this point on without worrying about spurious page faults; none of our +// loads will ever cross a page boundary, because they are all aligned. + tst x1, #(kVectorSize-1) + b.eq L_naiveVector + +/***************************************************************************** + * Careful chunk comparison * + *****************************************************************************/ + +// Otherwise, we need to be careful; although vector loads from s1 cannot +// cross a page boundary because they are aligned, s2 is not aligned. We +// compute the multiple of vector size that we can safely load before reaching +// a page boundary, and compare only that far before switching over to scalar +// comparisons to step across the page boundary. If this number happens to +// be zero, we jump directly to the scalar comparison. + neg x7, x1 + ands x7, x7, #(PAGE_MIN_SIZE-kVectorSize) + b.eq 2f + +.align 4 +// If n is less than the number of bytes before a page-crossing load, jump +// into the naive vector path instead, since we will not even reach a page +// crossing. Otherwise, decrement n by that number before we monkey with it, +// and set the decremented value aside. +0: cmp x2, x7 + b.ls L_naiveVector + sub x6, x2, x7 +// Use vector comparisons until a mismatch or EOS is encountered, or the next +// vector load from s2 would be page-crossing. +1: ldr q0, [x0],#(kVectorSize) + ldr q1, [x1],#(kVectorSize) + cmeq.16b v1, v0, v1 + and.16b v0, v0, v1 // contains zero byte iff mismatch or EOS + uminv.16b b1, v0 + fmov w3, s1 // zero only iff comparison is finished + cbz w3, L_vectorDone + subs x7, x7, #(kVectorSize) + b.ne 1b +// Restore the updated n to x2 + mov x2, x6 +// The next vector load will cross a page boundary. Instead, compare one byte +// at a time until s1 again has vector alignment, at which point we will have +// compared exactly 16 bytes. +2: ldrb w4, [x0],#1 // load byte from src1 + ldrb w5, [x1],#1 // load byte from src2 + subs x3, x4, x5 // if the are not equal + ccmp w4, #0, #4, eq // or we find an EOS + b.eq L_scalarDone // return the difference + subs x2, x2, #1 // decrement length + b.eq L_scalarDone // exit loop if zero. + tst x0, #(kVectorSize-1) + b.ne 2b +// Having compared one vector's worth of bytes using a scalar comparison, we +// know that we are safely across the page boundary. Initialize x7 and jump +// back into the vector comparison part of the loop. + mov x7, #(PAGE_MIN_SIZE-kVectorSize) + b 0b + +/***************************************************************************** + * Naive vector comparison * + *****************************************************************************/ + +.align 4 +L_naiveVector: + ldr q0, [x0],#(kVectorSize) + ldr q1, [x1],#(kVectorSize) + cmeq.16b v1, v0, v1 + and.16b v0, v0, v1 // contains zero byte iff mismatch or EOS + uminv.16b b1, v0 + fmov w3, s1 // zero only iff comparison is finished + cbz w3, L_vectorDone + subs x2, x2, #16 + b.hi L_naiveVector + +L_readNBytes: + eor x0, x0, x0 + ClearFrameAndReturn + +L_vectorDone: +// Load the bytes corresponding to the first mismatch or EOS and return +// their difference. + eor.16b v1, v1, v1 + cmhi.16b v0, v0, v1 // force non-zero lanes to 0xff + ldr q1, L_mask + orr.16b v0, v0, v1 // lane index in lanes containing mismatch or EOS + uminv.16b b1, v0 + fmov w3, s1 +// If the index of the mismatch or EOS is greater than or equal to n, it +// occurs after the first n bytes of the string, and doesn't count. + cmp x3, x2 + b.cs L_readNBytes + sub x3, x3, #(kVectorSize) + ldrb w4, [x0, x3] + ldrb w5, [x1, x3] + sub x0, x4, x5 + ClearFrameAndReturn diff --git a/osfmk/arm64/strnlen.s b/osfmk/arm64/strnlen.s new file mode 100644 index 000000000..3e0080669 --- /dev/null +++ b/osfmk/arm64/strnlen.s @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2012 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + * + * This file implements the following function for the arm64 architecture: + * + * size_t strnlen(const char *string, size_t maxlen); + * + * The strnlen function returns either strlen(string) or maxlen, whichever + * is amller, without reading beyond the first maxlen characters of string. + */ + +.globl _strlen +.globl _strnlen + +/***************************************************************************** + * Macros * + *****************************************************************************/ + +.macro EstablishFrame + stp fp, lr, [sp, #-16]! + mov fp, sp +.endm + +.macro ClearFrameAndReturn + ldp fp, lr, [sp], #16 + ret +.endm + +/***************************************************************************** + * Constants * + *****************************************************************************/ + +.text +.align 5 +L_masks: +.quad 0x0706050403020100, 0x0f0e0d0c0b0a0908 +.quad 0x0000000000000000, 0x0000000000000000 + +/***************************************************************************** + * strnlen entrypoint * + *****************************************************************************/ + +_strnlen: +// If n == 0, return NULL without loading any data from s. If n is so large +// that it exceeds the size of any buffer that can be allocted, jump into a +// simpler implementation that omits all length checks. This is both faster +// and lets us avoid some messy edgecases in the mainline. + tst x1, x1 + b.mi _strlen + b.eq L_maxlenIsZero + EstablishFrame +// Load the 16-byte aligned vector containing the start of the string. + and x2, x0, #-16 + ldr q0, [x2] +// Load a vector {0,1,2, ... ,15} for use in finding the index of the NUL +// byte once we identify one. We don't use this vector until the very end +// of the routine; it simply falls out naturally to load it now. + adr x3, L_masks + ldr q2, [x3],#16 +// The aligned vector that we loaded to q0 contains the start of the string, +// but if the string was not originally aligned, it also contains bytes +// which preceed the start of the string, and which may cause false positives +// when we search for the terminating NUL. We generate a mask to OR into the +// vector using an unaligned load to prevent this. The mask has non-zero +// values only in those bytes which correspond to bytes preceeding the start +// of the string in the aligned vector load. + and x4, x0, #0xf + sub x3, x3, x4 + ldr q1, [x3] + orr.16b v0, v0, v1 +// Adjust maxlen to account for bytes which preceed the start of the string, +// and jump into the main scanning loop. + add x1, x1, x4 + b 1f + +// Main loop. Identical to strlen, except that we also need to check that we +// don't read more than maxlen bytes. To that end, we decrement maxlen by 16 +// on each iteration, and exit the loop if the result is zero or negative. +.align 4 +0: ldr q0, [x2, #16]! +1: uminv.16b b1, v0 + fmov w3, s1 + cbz w3, L_foundNUL + subs x1, x1, #16 + b.hi 0b + +// We exhausted maxlen bytes without finding a terminating NUL character, so +// we need to return maxlen. + sub x0, x2, x0 + add x1, x1, #16 + add x0, x0, x1 + ClearFrameAndReturn + +L_maxlenIsZero: + mov x0, xzr + ret // No stack frame, so don't clear it. + +L_foundNUL: +// Compute the index of the NUL byte, and check if it occurs before maxlen +// bytes into the vector. If not, return maxlen. Otherwise, return the +// length of the string. + eor.16b v1, v1, v1 + cmhi.16b v0, v0, v1 + orr.16b v0, v0, v2 + uminv.16b b1, v0 + fmov w3, s1 // index of NUL byte in vector + sub x0, x2, x0 // index of vector in string + cmp x1, x3 // if NUL occurs before maxlen bytes + csel x1, x1, x3, cc // return strlen, else maxlen + add x0, x0, x1 + ClearFrameAndReturn + +/***************************************************************************** + * strlen entrypoint * + *****************************************************************************/ + +.align 4 +_strlen: + EstablishFrame +// Load the 16-byte aligned vector containing the start of the string. + and x1, x0, #-16 + ldr q0, [x1] +// Load a vector {0,1,2, ... ,15} for use in finding the index of the NUL +// byte once we identify one. We don't use this vector until the very end +// of the routine; it simply falls out naturally to load it now. + adr x3, L_masks + ldr q2, [x3],#16 +// The aligned vector that we loaded to q0 contains the start of the string, +// but if the string was not originally aligned, it also contains bytes +// which preceed the start of the string, and which may cause false positives +// when we search for the terminating NUL. We generate a mask to OR into the +// vector using an unaligned load to prevent this. The mask has non-zero +// values only in those bytes which correspond to bytes preceeding the start +// of the string in the aligned vector load. + and x2, x0, #0xf + sub x3, x3, x2 + ldr q1, [x3] + orr.16b v0, v0, v1 + b 1f + +// Main loop. On each iteration we do the following: +// +// q0 <-- next 16 aligned bytes of string +// b1 <-- unsigned minimum byte in q0 +// if (b1 != 0) continue +// +// Thus, we continue the loop until the 16 bytes we load contain a zero byte. +.align 4 +0: ldr q0, [x1, #16]! +1: uminv.16b b1, v0 + fmov w2, s1 // umov.b would be more natural, but requries 2 µops. + cbnz w2, 0b + +// A zero byte has been found. The following registers contain values that +// we need to compute the string's length: +// +// x0 pointer to start of string +// x1 pointer to vector containing terminating NUL byte +// v0 vector containing terminating NUL byte +// v2 {0, 1, 2, ... , 15} +// +// We compute the index of the terminating NUL byte in the string (which is +// precisely the length of the string) as follows: +// +// vec <-- mask(v0 != 0) | v2 +// index <-- x1 - x0 + unsignedMinimum(vec) + eor.16b v1, v1, v1 + cmhi.16b v0, v0, v1 + orr.16b v0, v0, v2 + uminv.16b b1, v0 + fmov w2, s1 + sub x0, x1, x0 + add x0, x0, x2 + ClearFrameAndReturn diff --git a/osfmk/atm/atm.c b/osfmk/atm/atm.c index 073aa37c2..009035687 100644 --- a/osfmk/atm/atm.c +++ b/osfmk/atm/atm.c @@ -68,11 +68,11 @@ queue_head_t atm_values_list; ipc_voucher_attr_control_t voucher_attr_control; /* communication channel from ATM to voucher system */ static zone_t atm_value_zone, atm_descriptors_zone, atm_link_objects_zone; -static aid_t get_aid(); -static mach_atm_subaid_t get_subaid(); +static aid_t get_aid(void); +static mach_atm_subaid_t get_subaid(void); static atm_value_t atm_value_alloc_init(aid_t); static void atm_value_dealloc(atm_value_t atm_value); -static void atm_hash_table_init(); +static void atm_hash_table_init(void); static kern_return_t atm_value_hash_table_insert(atm_value_t new_atm_value); static void atm_value_hash_table_delete(atm_value_t atm_value); static atm_value_t get_atm_value_from_aid(aid_t aid) __unused; diff --git a/osfmk/bank/bank.c b/osfmk/bank/bank.c index fa08f23ae..66a4798f5 100644 --- a/osfmk/bank/bank.c +++ b/osfmk/bank/bank.c @@ -39,6 +39,8 @@ #include #include #include +#include +#include #include #include #include @@ -63,7 +65,7 @@ queue_head_t bank_accounts_list; #endif static ledger_template_t bank_ledger_template = NULL; -struct _bank_ledger_indices bank_ledgers = { -1 }; +struct _bank_ledger_indices bank_ledgers = { -1, -1 }; static bank_task_t bank_task_alloc_init(task_t task); static bank_account_t bank_account_alloc_init(bank_task_t bank_holder, bank_task_t bank_merchant, @@ -791,6 +793,7 @@ bank_account_alloc_init( return BANK_ACCOUNT_NULL; ledger_entry_setactive(new_ledger, bank_ledgers.cpu_time); + ledger_entry_setactive(new_ledger, bank_ledgers.energy); new_bank_account = (bank_account_t) zalloc(bank_account_zone); if (new_bank_account == BANK_ACCOUNT_NULL) { ledger_dereference(new_ledger); @@ -930,6 +933,7 @@ bank_task_dealloc( lck_mtx_destroy(&bank_task->bt_acc_to_pay_lock, &bank_lock_grp); lck_mtx_destroy(&bank_task->bt_acc_to_charge_lock, &bank_lock_grp); + #if DEVELOPMENT || DEBUG lck_mtx_lock(&bank_tasks_list_lock); queue_remove(&bank_tasks_list, bank_task, bank_task_t, bt_global_elt); @@ -1020,25 +1024,28 @@ bank_rollup_chit_to_tasks( return; ret = ledger_get_entries(bill, bank_ledgers.cpu_time, &credit, &debit); - if (ret != KERN_SUCCESS) { - return; - } + if (ret == KERN_SUCCESS) { + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (BANK_CODE(BANK_ACCOUNT_INFO, (BANK_SETTLE_CPU_TIME))) | DBG_FUNC_NONE, + bank_merchant->bt_pid, bank_holder->bt_pid, credit, debit, 0); + ledger_credit(bank_holder->bt_creditcard, task_ledgers.cpu_time_billed_to_me, credit); + ledger_debit(bank_holder->bt_creditcard, task_ledgers.cpu_time_billed_to_me, debit); -#if DEVELOPMENT || DEBUG - if (debit != 0) { - panic("bank_rollup: debit: %lld non zero\n", debit); + ledger_credit(bank_merchant->bt_creditcard, task_ledgers.cpu_time_billed_to_others, credit); + ledger_debit(bank_merchant->bt_creditcard, task_ledgers.cpu_time_billed_to_others, debit); } -#endif - KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (BANK_CODE(BANK_ACCOUNT_INFO, (BANK_SETTLE_CPU_TIME))) | DBG_FUNC_NONE, + ret = ledger_get_entries(bill, bank_ledgers.energy, &credit, &debit); + if (ret == KERN_SUCCESS) { + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (BANK_CODE(BANK_ACCOUNT_INFO, (BANK_SETTLE_ENERGY))) | DBG_FUNC_NONE, bank_merchant->bt_pid, bank_holder->bt_pid, credit, debit, 0); -#if CONFIG_BANK - ledger_credit(bank_holder->bt_creditcard, task_ledgers.cpu_time_billed_to_me, credit); - ledger_debit(bank_holder->bt_creditcard, task_ledgers.cpu_time_billed_to_me, debit); - - ledger_credit(bank_merchant->bt_creditcard, task_ledgers.cpu_time_billed_to_others, credit); - ledger_debit(bank_merchant->bt_creditcard, task_ledgers.cpu_time_billed_to_others, debit); -#endif + ledger_credit(bank_holder->bt_creditcard, task_ledgers.energy_billed_to_me, credit); + ledger_debit(bank_holder->bt_creditcard, task_ledgers.energy_billed_to_me, debit); + + ledger_credit(bank_merchant->bt_creditcard, task_ledgers.energy_billed_to_others, credit); + ledger_debit(bank_merchant->bt_creditcard, task_ledgers.energy_billed_to_others, debit); + } } @@ -1091,23 +1098,30 @@ init_bank_ledgers(void) { if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) { panic("couldn't create cpu_time entry for bank ledger template"); } - bank_ledgers.cpu_time = idx; + + if ((idx = ledger_entry_add(t, "energy", "power", "nj")) < 0) { + panic("couldn't create energy entry for bank ledger template"); + } + bank_ledgers.energy = idx; + + ledger_template_complete(t); bank_ledger_template = t; } -/* Routine: bank_billed_time_safe +/* Routine: bank_billed_balance_safe * Purpose: Walk through all the bank accounts billed to me by other tasks and get the current billing balance. * Called from another task. It takes global bank task lock to make sure the bank context is not deallocated while accesing it. - * Returns: balance. + * Returns: cpu balance and energy balance in out paremeters. */ -uint64_t -bank_billed_time_safe(task_t task) +void +bank_billed_balance_safe(task_t task, uint64_t *cpu_time, uint64_t *energy) { bank_task_t bank_task = BANK_TASK_NULL; ledger_amount_t credit, debit; - uint64_t balance = 0; + uint64_t cpu_balance = 0; + uint64_t energy_balance = 0; kern_return_t kr; /* Task might be in exec, grab the global bank task lock before accessing bank context. */ @@ -1120,43 +1134,50 @@ bank_billed_time_safe(task_t task) global_bank_task_unlock(); if (bank_task) { - balance = bank_billed_time(bank_task); + bank_billed_balance(bank_task, &cpu_balance, &energy_balance); bank_task_dealloc(bank_task, 1); } else { kr = ledger_get_entries(task->ledger, task_ledgers.cpu_time_billed_to_me, &credit, &debit); if (kr == KERN_SUCCESS) { - balance = credit - debit; + cpu_balance = credit - debit; + } + kr = ledger_get_entries(task->ledger, task_ledgers.energy_billed_to_me, + &credit, &debit); + if (kr == KERN_SUCCESS) { + energy_balance = credit - debit; } } - return balance; + *cpu_time = cpu_balance; + *energy = energy_balance; + return; } /* * Routine: bank_billed_time * Purpose: Walk through the Accounts need to pay account list and get the current billing balance. - * Returns: balance. + * Returns: cpu balance and energy balance in out paremeters. */ -uint64_t -bank_billed_time(bank_task_t bank_task) +void +bank_billed_balance(bank_task_t bank_task, uint64_t *cpu_time, uint64_t *energy) { - int64_t balance = 0; -#ifdef CONFIG_BANK + int64_t cpu_balance = 0; + int64_t energy_balance = 0; bank_account_t bank_account; int64_t temp = 0; kern_return_t kr; -#endif if (bank_task == BANK_TASK_NULL) { - return balance; + *cpu_time = 0; + *energy = 0; + return; } -#ifdef CONFIG_BANK lck_mtx_lock(&bank_task->bt_acc_to_pay_lock); kr = ledger_get_balance(bank_task->bt_creditcard, task_ledgers.cpu_time_billed_to_me, &temp); if (kr == KERN_SUCCESS && temp >= 0) { - balance += temp; + cpu_balance += temp; } #if DEVELOPMENT || DEBUG else { @@ -1164,35 +1185,47 @@ bank_billed_time(bank_task_t bank_task) } #endif /* DEVELOPMENT || DEBUG */ + kr = ledger_get_balance(bank_task->bt_creditcard, task_ledgers.energy_billed_to_me, &temp); + if (kr == KERN_SUCCESS && temp >= 0) { + energy_balance += temp; + } + queue_iterate(&bank_task->bt_accounts_to_pay, bank_account, bank_account_t, ba_next_acc_to_pay) { temp = 0; kr = ledger_get_balance(bank_account->ba_bill, bank_ledgers.cpu_time, &temp); if (kr == KERN_SUCCESS && temp >= 0) { - balance += temp; + cpu_balance += temp; } #if DEVELOPMENT || DEBUG else { printf("bank_bill_time: ledger_get_balance failed or negative balance in ledger: %lld\n", temp); } #endif /* DEVELOPMENT || DEBUG */ + + kr = ledger_get_balance(bank_account->ba_bill, bank_ledgers.energy, &temp); + if (kr == KERN_SUCCESS && temp >= 0) { + energy_balance += temp; + } } lck_mtx_unlock(&bank_task->bt_acc_to_pay_lock); -#endif - return (uint64_t)balance; + *cpu_time = (uint64_t)cpu_balance; + *energy = (uint64_t)energy_balance; + return; } -/* Routine: bank_serviced_time_safe +/* Routine: bank_serviced_balance_safe * Purpose: Walk through the bank accounts billed to other tasks by me and get the current balance to be charged. * Called from another task. It takes global bank task lock to make sure the bank context is not deallocated while accesing it. - * Returns: balance. + * Returns: cpu balance and energy balance in out paremeters. */ -uint64_t -bank_serviced_time_safe(task_t task) +void +bank_serviced_balance_safe(task_t task, uint64_t *cpu_time, uint64_t *energy) { bank_task_t bank_task = BANK_TASK_NULL; ledger_amount_t credit, debit; - uint64_t balance = 0; + uint64_t cpu_balance = 0; + uint64_t energy_balance = 0; kern_return_t kr; /* Task might be in exec, grab the global bank task lock before accessing bank context. */ @@ -1205,43 +1238,51 @@ bank_serviced_time_safe(task_t task) global_bank_task_unlock(); if (bank_task) { - balance = bank_serviced_time(bank_task); + bank_serviced_balance(bank_task, &cpu_balance, &energy_balance); bank_task_dealloc(bank_task, 1); } else { kr = ledger_get_entries(task->ledger, task_ledgers.cpu_time_billed_to_others, &credit, &debit); if (kr == KERN_SUCCESS) { - balance = credit - debit; + cpu_balance = credit - debit; + } + + kr = ledger_get_entries(task->ledger, task_ledgers.energy_billed_to_others, + &credit, &debit); + if (kr == KERN_SUCCESS) { + energy_balance = credit - debit; } } - return balance; + *cpu_time = cpu_balance; + *energy = energy_balance; + return; } /* - * Routine: bank_serviced_time + * Routine: bank_serviced_balance * Purpose: Walk through the Account need to charge account list and get the current balance to be charged. - * Returns: balance. + * Returns: cpu balance and energy balance in out paremeters. */ -uint64_t -bank_serviced_time(bank_task_t bank_task) +void +bank_serviced_balance(bank_task_t bank_task, uint64_t *cpu_time, uint64_t *energy) { - int64_t balance = 0; -#ifdef CONFIG_BANK + int64_t cpu_balance = 0; + int64_t energy_balance = 0; bank_account_t bank_account; int64_t temp = 0; kern_return_t kr; -#endif if (bank_task == BANK_TASK_NULL) { - return balance; + *cpu_time = 0; + *energy = 0; + return; } -#ifdef CONFIG_BANK lck_mtx_lock(&bank_task->bt_acc_to_charge_lock); kr = ledger_get_balance(bank_task->bt_creditcard, task_ledgers.cpu_time_billed_to_others, &temp); if (kr == KERN_SUCCESS && temp >= 0) { - balance += temp; + cpu_balance += temp; } #if DEVELOPMENT || DEBUG else { @@ -1249,38 +1290,47 @@ bank_serviced_time(bank_task_t bank_task) } #endif /* DEVELOPMENT || DEBUG */ + kr = ledger_get_balance(bank_task->bt_creditcard, task_ledgers.energy_billed_to_others, &temp); + if (kr == KERN_SUCCESS && temp >= 0) { + energy_balance += temp; + } + queue_iterate(&bank_task->bt_accounts_to_charge, bank_account, bank_account_t, ba_next_acc_to_charge) { temp = 0; kr = ledger_get_balance(bank_account->ba_bill, bank_ledgers.cpu_time, &temp); if (kr == KERN_SUCCESS && temp >= 0) { - balance += temp; + cpu_balance += temp; } #if DEVELOPMENT || DEBUG else { printf("bank_serviced_time: ledger_get_balance failed or negative balance in ledger: %lld\n", temp); } #endif /* DEVELOPMENT || DEBUG */ + + kr = ledger_get_balance(bank_account->ba_bill, bank_ledgers.energy, &temp); + if (kr == KERN_SUCCESS && temp >= 0) { + energy_balance += temp; + } } lck_mtx_unlock(&bank_task->bt_acc_to_charge_lock); -#endif - return (uint64_t)balance; + *cpu_time = (uint64_t)cpu_balance; + *energy = (uint64_t)energy_balance; + return; } /* - * Routine: bank_get_voucher_ledger - * Purpose: Get the bankledger (chit) from the voucher. - * Returns: bank_ledger if bank_account attribute present in voucher. - * NULL on no attribute ot bank_task attribute. + * Routine: bank_get_voucher_bank_account + * Purpose: Get the bank account from the voucher. + * Returns: bank_account if bank_account attribute present in voucher. + * NULL on no attribute, no bank_element, or if holder and merchant bank accounts are the same. */ -ledger_t -bank_get_voucher_ledger(ipc_voucher_t voucher) +static bank_account_t +bank_get_voucher_bank_account(ipc_voucher_t voucher) { bank_element_t bank_element = BANK_ELEMENT_NULL; bank_account_t bank_account = BANK_ACCOUNT_NULL; mach_voucher_attr_value_handle_t vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED]; mach_voucher_attr_value_handle_array_size_t val_count; - ledger_t bankledger = NULL; - bank_task_t bank_merchant; kern_return_t kr; val_count = MACH_VOUCHER_ATTR_VALUE_MAX_NESTED; @@ -1289,53 +1339,85 @@ bank_get_voucher_ledger(ipc_voucher_t voucher) vals, &val_count); - if (kr != KERN_SUCCESS) - return NULL; - - if (val_count == 0) - return NULL; + if (kr != KERN_SUCCESS || val_count == 0) + return BANK_ACCOUNT_NULL; bank_element = HANDLE_TO_BANK_ELEMENT(vals[0]); if (bank_element == BANK_DEFAULT_VALUE) - return NULL; - - if (bank_element == BANK_DEFAULT_TASK_VALUE) { + return BANK_ACCOUNT_NULL; + if (bank_element == BANK_DEFAULT_TASK_VALUE) bank_element = CAST_TO_BANK_ELEMENT(get_bank_task_context(current_task(), FALSE)); - } if (bank_element->be_type == BANK_TASK) { - bankledger = NULL; + return BANK_ACCOUNT_NULL; } else if (bank_element->be_type == BANK_ACCOUNT) { bank_account = CAST_TO_BANK_ACCOUNT(bank_element); if (bank_account->ba_holder != bank_account->ba_merchant) { - /* Return the ledger, if the voucher is redeemed by currrent process. */ - bank_merchant = get_bank_task_context(current_task(), FALSE); - if (bank_account->ba_merchant == bank_merchant) { - bankledger = bank_account->ba_bill; - } + bank_task_t bank_merchant = get_bank_task_context(current_task(), FALSE); + if (bank_account->ba_merchant == bank_merchant) + return bank_account; + else + return BANK_ACCOUNT_NULL; + } else { + return BANK_ACCOUNT_NULL; } } else { - panic("Bogus bank type: %d passed in bank_get_voucher_ledger\n", bank_element->be_type); + panic("Bogus bank type: %d passed in bank_get_voucher_bank_account\n", bank_element->be_type); } + return BANK_ACCOUNT_NULL; +} + +/* + * Routine: bank_get_bank_account_ledger + * Purpose: Get the bankledger from the bank account + */ +static ledger_t +bank_get_bank_account_ledger(bank_account_t bank_account) +{ + ledger_t bankledger = NULL; + + if (bank_account != BANK_ACCOUNT_NULL) + bankledger = bank_account->ba_bill; return (bankledger); } + +/* + * Routine: bank_get_bank_ledger_and_thread_group + * Purpose: Get the bankledger (chit) and thread group from the voucher. + * Returns: bankledger and thread group if bank_account attribute present in voucher. + * + */ +kern_return_t +bank_get_bank_ledger_and_thread_group( + ipc_voucher_t voucher, + ledger_t *bankledger, + thread_group_t *banktg __unused) +{ + bank_account_t bank_account; + + bank_account = bank_get_voucher_bank_account(voucher); + *bankledger = bank_get_bank_account_ledger(bank_account); + return KERN_SUCCESS; +} + /* * Routine: bank_swap_thread_bank_ledger * Purpose: swap the bank ledger on the thread. - * Retunrs: None. + * Returns: None. * Note: Should be only called for current thread or thread which is not started. */ void bank_swap_thread_bank_ledger(thread_t thread __unused, ledger_t new_ledger __unused) { -#ifdef CONFIG_BANK spl_t s; processor_t processor; ledger_t old_ledger = thread->t_bankledger; int64_t ctime, effective_ledger_time_consumed = 0; - int64_t remainder = 0, consumed = 0; + int64_t remainder = 0, consumed = 0; + int64_t effective_energy_consumed = 0; + uint64_t thread_energy; if (old_ledger == NULL && new_ledger == NULL) return; @@ -1372,15 +1454,24 @@ bank_swap_thread_bank_ledger(thread_t thread __unused, ledger_t new_ledger __unu thread->t_deduct_bank_ledger_time = consumed; + thread_energy = ml_energy_stat(thread); + effective_energy_consumed = + thread_energy - thread->t_deduct_bank_ledger_energy; + assert(effective_energy_consumed >= 0); + thread->t_deduct_bank_ledger_energy = thread_energy; + thread->t_bankledger = new_ledger; thread_unlock(thread); splx(s); - if (old_ledger != NULL) + if (old_ledger != NULL) { ledger_credit(old_ledger, bank_ledgers.cpu_time, effective_ledger_time_consumed); -#endif + ledger_credit(old_ledger, + bank_ledgers.energy, + effective_energy_consumed); + } } diff --git a/osfmk/bank/bank_internal.h b/osfmk/bank/bank_internal.h index c733eeb88..eb8f5599c 100644 --- a/osfmk/bank/bank_internal.h +++ b/osfmk/bank/bank_internal.h @@ -35,6 +35,7 @@ #ifdef MACH_KERNEL_PRIVATE #include +#include #include #include #include @@ -162,6 +163,7 @@ typedef struct bank_account * bank_account_t; struct _bank_ledger_indices { int cpu_time; + int energy; }; extern struct _bank_ledger_indices bank_ledgers; @@ -169,11 +171,12 @@ extern struct _bank_ledger_indices bank_ledgers; extern void bank_init(void); extern void bank_task_destroy(task_t); extern void bank_task_initialize(task_t task); -extern uint64_t bank_billed_time_safe(task_t task); -extern uint64_t bank_billed_time(bank_task_t bank_task); -extern uint64_t bank_serviced_time_safe(task_t task); -extern uint64_t bank_serviced_time(bank_task_t bank_task); -extern ledger_t bank_get_voucher_ledger(ipc_voucher_t voucher); +extern void bank_billed_balance_safe(task_t task, uint64_t *cpu_time, uint64_t *energy); +extern void bank_billed_balance(bank_task_t bank_task, uint64_t *cpu_time, uint64_t *energy); +extern void bank_serviced_balance_safe(task_t task, uint64_t *cpu_time, uint64_t *energy); +extern void bank_serviced_balance(bank_task_t bank_task, uint64_t *cpu_time, uint64_t *energy); +extern kern_return_t bank_get_bank_ledger_and_thread_group(ipc_voucher_t voucher, + ledger_t *bankledger, thread_group_t *banktg); extern void bank_swap_thread_bank_ledger(thread_t thread, ledger_t ledger); #endif /* MACH_KERNEL_PRIVATE */ diff --git a/osfmk/chud/chud_thread.c b/osfmk/chud/chud_thread.c index 9074c6dd3..2b804af7a 100644 --- a/osfmk/chud/chud_thread.c +++ b/osfmk/chud/chud_thread.c @@ -54,6 +54,8 @@ // include the correct file to find real_ncpus #if defined(__i386__) || defined(__x86_64__) # include +#elif defined(__arm__) || defined (__arm64__) +# include #else // fall back on declaring it extern. The linker will sort us out. extern unsigned int real_ncpus; diff --git a/osfmk/chud/chud_xnu_glue.h b/osfmk/chud/chud_xnu_glue.h index b2ac2189c..f595ffef4 100644 --- a/osfmk/chud/chud_xnu_glue.h +++ b/osfmk/chud/chud_xnu_glue.h @@ -28,6 +28,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/chud_xnu_glue.h" +#elif defined (__arm__)|| defined (__arm64__) +#include "arm/chud_xnu_glue.h" #else #error architecture not supported #endif diff --git a/osfmk/chud/chud_xnu_private.h b/osfmk/chud/chud_xnu_private.h index 56b6eb22c..5a712aceb 100644 --- a/osfmk/chud/chud_xnu_private.h +++ b/osfmk/chud/chud_xnu_private.h @@ -35,6 +35,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "chud/i386/chud_xnu_private.h" +#elif defined (__arm__) || defined (__arm64__) +#include "chud/arm/chud_xnu_private.h" #else #error architecture not supported #endif diff --git a/osfmk/chud/i386/chud_osfmk_callback_i386.c b/osfmk/chud/i386/chud_osfmk_callback_i386.c index a6775c24c..f9b5243bc 100644 --- a/osfmk/chud/i386/chud_osfmk_callback_i386.c +++ b/osfmk/chud/i386/chud_osfmk_callback_i386.c @@ -268,6 +268,8 @@ chudxnu_private_chud_ast_callback(ast_t reasons, ast_t *myast) return retval; } +volatile perfASTCallback perfASTHook; + __private_extern__ kern_return_t chudxnu_perfmon_ast_callback_enter(chudxnu_perfmon_ast_callback_func_t func) { diff --git a/osfmk/conf/Makefile.arm b/osfmk/conf/Makefile.arm new file mode 100644 index 000000000..17d6b3a8b --- /dev/null +++ b/osfmk/conf/Makefile.arm @@ -0,0 +1,10 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# Files that must go in the __HIB segment: +HIB_FILES= + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/osfmk/conf/Makefile.arm64 b/osfmk/conf/Makefile.arm64 new file mode 100644 index 000000000..78235b8be --- /dev/null +++ b/osfmk/conf/Makefile.arm64 @@ -0,0 +1,13 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm64 +###################################################################### + +CWARNFLAGS = $(CWARNFLAGS_STD) -Wshorten-64-to-32 + +# Files that must go in the __HIB segment: +HIB_FILES= + +lz4.o_CFLAGS_ADD += -fbuiltin -O3 +###################################################################### +#END Machine dependent Makefile fragment for arm64 +###################################################################### diff --git a/osfmk/conf/Makefile.template b/osfmk/conf/Makefile.template index c025d381d..b1f41bbc2 100644 --- a/osfmk/conf/Makefile.template +++ b/osfmk/conf/Makefile.template @@ -17,6 +17,7 @@ include $(MakeInc_def) # XXX: CFLAGS # CFLAGS+= -include meta_features.h -DMACH_KERNEL_PRIVATE -DMACH_KERNEL +SFLAGS+= -include meta_features.h # Objects that don't want -Wcast-align warning (8474835) OBJS_NO_CAST_ALIGN = \ @@ -72,6 +73,10 @@ OBJS_NO_CAST_ALIGN = \ cchmac_init.o \ ccsha1.o \ ipc_object.o \ + ccmode_ctr_crypt.o \ + ccmode_factory_ctr_crypt.o \ + ccmode_ctr_init.o \ + ccmode_ctr_setctr.o \ ipc_kmsg.o \ ipc_right.o \ bsd_vm.o \ diff --git a/osfmk/conf/files b/osfmk/conf/files index e735d9e71..befcee59c 100644 --- a/osfmk/conf/files +++ b/osfmk/conf/files @@ -45,7 +45,6 @@ OPTIONS/mach_kprof optional mach_kprof OPTIONS/mach_ldebug optional mach_ldebug OPTIONS/mach_mp_debug optional mach_mp_debug OPTIONS/mach_pagemap optional mach_pagemap -OPTIONS/mach_rt optional mach_rt OPTIONS/mach_vm_debug optional mach_vm_debug OPTIONS/mach_page_hash_stats optional mach_page_hash_stats OPTIONS/mig_debug optional mig_debug @@ -88,6 +87,7 @@ osfmk/kdp/kdp.c optional config_kdp_interactive_debugging osfmk/kern/kern_stackshot.c standard osfmk/kdp/kdp_udp.c optional mach_kdp osfmk/kdp/kdp_core.c optional mach_kdp +osfmk/kdp/processor_core.c optional mach_kdp osfmk/kdp/kdp_serial.c optional config_serial_kdp osfmk/ipc/ipc_entry.c standard osfmk/ipc/ipc_hash.c standard @@ -169,15 +169,20 @@ osfmk/kern/task_swap.c standard osfmk/kern/thread.c standard osfmk/kern/thread_act.c standard osfmk/kern/thread_call.c standard +osfmk/kern/thread_group.c standard osfmk/kern/thread_policy.c standard osfmk/kern/timer.c standard osfmk/kern/timer_call.c standard osfmk/kern/waitq.c standard +osfmk/kern/work_interval.c standard osfmk/kern/xpr.c optional xpr_debug osfmk/kern/zalloc.c standard osfmk/kern/gzalloc.c optional config_gzalloc osfmk/kern/bsd_kern.c optional mach_bsd osfmk/kern/hibernate.c optional hibernation +osfmk/kern/memset_s.c standard +osfmk/kern/copyout_shim.c optional copyout_shim + ./mach/clock_server.c standard ./mach/clock_priv_server.c standard ./mach/clock_reply_user.c standard @@ -203,7 +208,7 @@ osfmk/kern/hibernate.c optional hibernation osfmk/corpses/corpse.c standard osfmk/kern/kern_cdata.c standard ./mach/telemetry_notification_user.c optional config_telemetry -osfmk/bank/bank.c optional config_bank +osfmk/bank/bank.c standard osfmk/atm/atm.c optional config_atm ./atm/atm_notification_user.c optional config_atm osfmk/voucher/ipc_pthread_priority.c standard @@ -288,6 +293,8 @@ osfmk/kperf/kdebug_trigger.c optional kperf osfmk/kern/kpc_thread.c optional kpc osfmk/kern/kpc_common.c optional kpc +osfmk/kern/kern_monotonic.c optional monotonic + osfmk/console/serial_general.c standard osfmk/console/serial_console.c optional serial_console osfmk/console/video_scroll.c optional video_console @@ -299,7 +306,7 @@ osfmk/kern/telemetry.c optional config_telemetry # Built-in corecrypto for early_random(): osfmk/corecrypto/cc/src/cc_clear.c standard osfmk/corecrypto/cc/src/cc_cmp_safe.c standard -osfmk/corecrypto/cc/src/cc_abort.c standard +osfmk/corecrypto/cc/src/cc_try_abort.c standard osfmk/corecrypto/ccdbrg/src/ccdrbg_nisthmac.c standard osfmk/corecrypto/ccdigest/src/ccdigest_init.c standard osfmk/corecrypto/ccdigest/src/ccdigest_update.c standard @@ -311,6 +318,18 @@ osfmk/corecrypto/ccsha1/src/ccdigest_final_64be.c standard osfmk/corecrypto/ccsha1/src/ccsha1_eay.c standard osfmk/corecrypto/ccsha1/src/ccsha1_initial_state.c standard +osfmk/corecrypto/ccsha2/src/ccsha256_di.c standard +osfmk/corecrypto/ccsha2/src/ccsha256_initial_state.c standard +osfmk/corecrypto/ccsha2/src/ccsha256_K.c standard +osfmk/corecrypto/ccsha2/src/ccsha256_ltc_compress.c standard +osfmk/corecrypto/ccsha2/src/ccsha256_ltc_di.c standard + +osfmk/corecrypto/ccaes/src/ccaes_ltc_ecb_encrypt_mode.c standard +osfmk/corecrypto/ccmode/src/ccmode_ctr_crypt.c standard +osfmk/corecrypto/ccmode/src/ccmode_ctr_init.c standard +osfmk/corecrypto/ccmode/src/ccmode_ctr_setctr.c standard +osfmk/corecrypto/ccmode/src/ccmode_factory_ctr_crypt.c standard + osfmk/prng/random.c standard osfmk/prng/prng_yarrow.c standard osfmk/prng/fips_sha1.c standard diff --git a/osfmk/conf/files.arm b/osfmk/conf/files.arm new file mode 100644 index 000000000..d79408fda --- /dev/null +++ b/osfmk/conf/files.arm @@ -0,0 +1,80 @@ +OPTIONS/fb optional fb + +OPTIONS/debug optional debug + +osfmk/vm/vm_apple_protect.c standard + +osfmk/arm/hi_res_clock_map.c optional hi_res_clock + +osfmk/arm/pmap.c standard + +osfmk/arm/bsd_arm.c optional mach_bsd +osfmk/arm/machdep_call.c optional mach_bsd + +osfmk/arm/caches.c standard +osfmk/arm/caches_asm.s standard +osfmk/arm/cpu.c standard +osfmk/arm/cpu_common.c standard +osfmk/arm/cpuid.c standard +osfmk/arm/data.s standard +osfmk/arm/dbgwrap.c standard +osfmk/arm/arm_timer.c standard +osfmk/arm/arm_init.c standard +osfmk/arm/arm_vm_init.c standard +osfmk/arm/io_map.c standard +osfmk/arm/loose_ends.c standard +osfmk/arm/locks_arm.c standard +osfmk/arm/locore.s standard +osfmk/arm/lowmem_vectors.c standard +osfmk/arm/start.s standard +osfmk/arm/cswitch.s standard +osfmk/arm/machine_cpuid.c standard +osfmk/arm/machine_routines.c standard +osfmk/arm/machine_routines_common.c standard +osfmk/arm/machine_routines_asm.s standard +osfmk/arm/machine_task.c standard +osfmk/arm/pal_routines.c standard +osfmk/arm/mcount.s optional profile +osfmk/arm/WKdmDecompress_new.s standard +osfmk/arm/WKdmCompress_new.s standard +osfmk/arm/WKdmData_new.s standard +osfmk/arm/lz4_decode_armv7NEON.s standard +osfmk/arm/lz4_encode_armv7.s standard +osfmk/arm/bcopy.s standard +osfmk/arm/bzero.s standard +osfmk/arm/strlen.s standard +osfmk/arm/strnlen.s standard +osfmk/arm/strncmp.s standard +osfmk/arm/strncpy.c standard +osfmk/arm/strlcpy.c standard + +osfmk/arm/model_dep.c standard +osfmk/arm/pcb.c standard +osfmk/arm/conf.c standard +osfmk/arm/rtclock.c standard +osfmk/arm/status.c standard +osfmk/arm/status_shared.c standard +osfmk/arm/trap.c standard + +osfmk/arm/commpage/commpage.c standard + +osfmk/kdp/ml/arm/kdp_machdep.c optional mach_kdp +osfmk/kdp/ml/arm/kdp_vm.c optional mach_kdp + + +# DUMMIES TO FORCE GENERATION OF .h FILES +osfmk/OPTIONS/ln optional ln +osfmk/OPTIONS/eisa optional eisa +osfmk/OPTIONS/himem optional himem +osfmk/OPTIONS/ec optional ec +osfmk/OPTIONS/hi_res_clock optional hi_res_clock + +# Kernel performance monitoring +osfmk/kperf/arm/kperf_mp.c optional kperf +osfmk/kperf/arm/kperf_meminfo.c optional kperf +osfmk/arm/kpc_arm.c optional kpc + +osfmk/arm/monotonic_arm.c optional monotonic + +# Support for early_random() +osfmk/corecrypto/ccn/src/arm/ccn_set.s standard diff --git a/osfmk/conf/files.arm64 b/osfmk/conf/files.arm64 new file mode 100644 index 000000000..e5234f0d9 --- /dev/null +++ b/osfmk/conf/files.arm64 @@ -0,0 +1,92 @@ +OPTIONS/fb optional fb + +OPTIONS/debug optional debug + +osfmk/vm/vm_apple_protect.c standard +osfmk/vm/vm_fourk_pager.c standard + +osfmk/arm64/hi_res_clock_map.c optional hi_res_clock + +osfmk/arm/pmap.c standard + +osfmk/arm64/bsd_arm64.c optional mach_bsd +osfmk/arm/machdep_call.c optional mach_bsd + +osfmk/arm64/WKdmDecompress_16k.s standard +osfmk/arm64/WKdmCompress_16k.s standard +osfmk/arm64/WKdmDecompress_4k.s standard +osfmk/arm64/WKdmCompress_4k.s standard +osfmk/arm64/WKdmData.s standard +osfmk/arm64/lz4_decode_arm64.s standard +osfmk/arm64/lz4_encode_arm64.s standard +osfmk/arm64/bcopy.s standard +osfmk/arm64/bzero.s standard +osfmk/arm/caches.c standard +osfmk/arm64/caches_asm.s standard +osfmk/arm64/copyio.c standard +osfmk/arm64/cpu.c standard +osfmk/arm/cpu_common.c standard +osfmk/arm/cpuid.c standard +osfmk/arm/data.s standard +osfmk/arm64/dbgwrap.c standard +osfmk/arm/arm_timer.c standard +osfmk/arm/arm_init.c standard +osfmk/arm64/arm_vm_init.c standard +osfmk/arm/io_map.c standard +osfmk/arm64/loose_ends.c standard +osfmk/arm/locks_arm.c standard +osfmk/arm64/locore.s standard +osfmk/arm64/lowmem_vectors.c standard +osfmk/arm64/sleh.c standard +osfmk/arm64/start.s standard +osfmk/arm64/pinst.s standard +osfmk/arm64/cswitch.s standard +osfmk/arm/machine_cpuid.c standard +osfmk/arm/machine_routines_common.c standard +osfmk/arm64/machine_routines.c standard +osfmk/arm64/machine_routines_asm.s standard +osfmk/arm64/machine_task.c standard +osfmk/arm/pal_routines.c standard +osfmk/arm64/mcount.s optional profile +osfmk/arm64/strnlen.s standard +osfmk/arm64/strncmp.s standard +osfmk/arm/strncpy.c standard +osfmk/arm/strlcpy.c standard + +osfmk/arm/model_dep.c standard +osfmk/arm64/pcb.c standard +osfmk/arm/conf.c standard +osfmk/arm/rtclock.c standard +osfmk/arm64/status.c standard +osfmk/arm/status_shared.c standard + +osfmk/arm/commpage/commpage.c standard + +osfmk/kdp/ml/arm/kdp_machdep.c optional mach_kdp +osfmk/kdp/ml/arm/kdp_vm.c optional mach_kdp + + +# DUMMIES TO FORCE GENERATION OF .h FILES +osfmk/OPTIONS/ln optional ln +osfmk/OPTIONS/eisa optional eisa +osfmk/OPTIONS/himem optional himem +osfmk/OPTIONS/ec optional ec +osfmk/OPTIONS/hi_res_clock optional hi_res_clock + +# Kernel performance monitoring +osfmk/kperf/arm/kperf_mp.c optional kperf +osfmk/kperf/arm/kperf_meminfo.c optional kperf +osfmk/arm64/kpc.c optional kpc + +osfmk/arm64/monotonic_arm64.c optional monotonic + +osfmk/arm64/platform_tests.c optional config_xnupost + +osfmk/arm64/alternate_debugger.c optional alternate_debugger +osfmk/arm64/alternate_debugger_asm.s optional alternate_debugger + +# Support for early_random() +osfmk/corecrypto/ccn/src/ccn_set.c standard + +osfmk/arm64/pgtrace.c standard +osfmk/arm64/pgtrace_decoder.c optional config_pgtrace_nonkext diff --git a/osfmk/conf/files.x86_64 b/osfmk/conf/files.x86_64 index 6bd6977a8..7fe01580c 100644 --- a/osfmk/conf/files.x86_64 +++ b/osfmk/conf/files.x86_64 @@ -120,6 +120,8 @@ osfmk/kperf/x86_64/kperf_mp.c optional kperf osfmk/kperf/x86_64/kperf_meminfo.c optional kperf osfmk/x86_64/kpc_x86.c optional kpc +osfmk/x86_64/monotonic_x86_64.c optional monotonic + osfmk/i386/startup64.c standard osfmk/x86_64/idt64.s standard diff --git a/osfmk/console/Makefile b/osfmk/console/Makefile index f8d5445eb..29b817b33 100644 --- a/osfmk/console/Makefile +++ b/osfmk/console/Makefile @@ -9,7 +9,8 @@ include $(MakeInc_def) DATAFILES = PRIVATE_DATAFILES = \ - video_console.h + video_console.h \ + serial_protos.h INSTALL_MI_LCL_LIST = ${PRIVATE_DATAFILES} diff --git a/osfmk/console/art/scalegear.c b/osfmk/console/art/scalegear.c index 91051837b..3dbc5ad0a 100644 --- a/osfmk/console/art/scalegear.c +++ b/osfmk/console/art/scalegear.c @@ -1,3 +1,30 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ // //cc scalegear.c -framework Accelerate -g -Wall */ diff --git a/osfmk/console/serial_console.c b/osfmk/console/serial_console.c index 358fbb8ed..8161ef280 100644 --- a/osfmk/console/serial_console.c +++ b/osfmk/console/serial_console.c @@ -46,6 +46,10 @@ #include #include +#if __arm__ || __arm64__ +#include +#include +#endif #ifndef MAX_CPU_SLOTS @@ -123,9 +127,34 @@ SECURITY_READ_ONLY_EARLY(uint32_t) nconsops = (sizeof cons_ops / sizeof cons_ops uint32_t cons_ops_index = VC_CONS_OPS; +#ifdef __arm__ +// NMI static variables +#define NMI_STRING_SIZE 32 +char nmi_string[NMI_STRING_SIZE] = "afDIGHr84A84jh19Kphgp428DNPdnapq"; +static int nmi_counter = 0; +#endif /* __arm__ */ static bool console_suspended = false; +/* Wrapper for ml_set_interrupts_enabled */ +static void +console_restore_interrupts_state(boolean_t state) +{ +#if INTERRUPT_MASKED_DEBUG + /* + * Serial console holds interrupts disabled for far too long + * and would trip the spin-debugger. If we are about to reenable + * interrupts then clear the timer and avoid panicking on the delay. + * Otherwise, let the code that printed with interrupt disabled + * take the panic when it reenables interrupts. + * Hopefully one day this is fixed so that this workaround is unnecessary. + */ + if (state == TRUE) + ml_spin_debug_clear_self(); +#endif /* INTERRUPT_MASKED_DEBUG */ + ml_set_interrupts_enabled(state); +} + static void console_ring_lock_init(void) { @@ -167,7 +196,7 @@ console_cpu_alloc(__unused boolean_t boot_processor) { console_buf_t * cbp; int i; - uint32_t * p; + uint32_t * p = NULL; console_init(); assert(console_ring.buffer != NULL); @@ -257,7 +286,7 @@ _cnputs(char * c, int size) */ hw_lock_data_t _shadow_lock; memcpy(&_shadow_lock, &cnputc_lock, sizeof(cnputc_lock)); - if (debug_mode) { + if (kernel_debugger_entry_count) { /* Since hw_lock_to takes a pre-emption count...*/ mp_enable_preemption(); hw_lock_init(&cnputc_lock); @@ -285,9 +314,21 @@ cnputc_unbuffered(char c) _cnputs(&c, 1); } + +void cnputcusr(char c) +{ + cnputsusr(&c, 1); +} + void -cnputcusr(char c) +cnputsusr(char *s, int size) { + + if (size > 1) { + console_write(s, size); + return; + } + boolean_t state; /* Spin (with pre-emption enabled) waiting for console_ring_try_empty() @@ -302,8 +343,9 @@ cnputcusr(char c) * interrupts off); we don't want to disable pre-emption indefinitely * here, and spinlocks and mutexes are inappropriate. */ - while (console_output != 0) - ; + while (console_output != 0) { + delay(1); + } /* * We disable interrupts to avoid issues caused by rendevous IPIs @@ -311,8 +353,8 @@ cnputcusr(char c) * core wants it. Stackshot is the prime example of this. */ state = ml_set_interrupts_enabled(FALSE); - _cnputs(&c, 1); - ml_set_interrupts_enabled(state); + _cnputs(s, 1); + console_restore_interrupts_state(state); } static void @@ -377,14 +419,14 @@ console_ring_try_empty(void) simple_unlock(&console_ring.read_lock); - ml_set_interrupts_enabled(state); + console_restore_interrupts_state(state); /* * In case we end up being the console drain thread * for far too long, break out. Except in panic/suspend cases * where we should clear out full buffer. */ - if (debug_mode == 0 && !console_suspended && (total_chars_out >= MAX_TOTAL_FLUSH_SIZE)) + if (!kernel_debugger_entry_count && !console_suspended && (total_chars_out >= MAX_TOTAL_FLUSH_SIZE)) break; } while (nchars_out > 0); @@ -420,7 +462,7 @@ console_write(char * str, int size) simple_lock_try_lock_loop(&console_ring.write_lock); while (chunk_size > console_ring_space()) { simple_unlock(&console_ring.write_lock); - ml_set_interrupts_enabled(state); + console_restore_interrupts_state(state); console_ring_try_empty(); @@ -434,7 +476,7 @@ console_write(char * str, int size) str = &str[i]; size -= chunk_size; simple_unlock(&console_ring.write_lock); - ml_set_interrupts_enabled(state); + console_restore_interrupts_state(state); } console_ring_try_empty(); @@ -491,7 +533,7 @@ restart: if (cpu_buffer_size(cbp) > console_ring_space()) { simple_unlock(&console_ring.write_lock); - ml_set_interrupts_enabled(state); + console_restore_interrupts_state(state); mp_enable_preemption(); console_ring_try_empty(); @@ -509,7 +551,7 @@ restart: needs_print = FALSE; if (c != '\n') { - ml_set_interrupts_enabled(state); + console_restore_interrupts_state(state); mp_enable_preemption(); return; } @@ -526,7 +568,7 @@ restart: if (cpu_buffer_size(cbp) > console_ring_space()) { simple_unlock(&console_ring.write_lock); - ml_set_interrupts_enabled(state); + console_restore_interrupts_state(state); mp_enable_preemption(); console_ring_try_empty(); @@ -539,7 +581,8 @@ restart: cbp->buf_ptr = cbp->buf_base; simple_unlock(&console_ring.write_lock); - ml_set_interrupts_enabled(state); + + console_restore_interrupts_state(state); mp_enable_preemption(); console_ring_try_empty(); @@ -555,6 +598,20 @@ _serial_getc(__unused int a, __unused int b, boolean_t wait, __unused boolean_t c = serial_getc(); } while (wait && c < 0); +#ifdef __arm__ + // Check for the NMI string + if (c == nmi_string[nmi_counter]) { + nmi_counter++; + if (nmi_counter == NMI_STRING_SIZE) { + // We've got the NMI string, now do an NMI + Debugger("Automatic NMI"); + nmi_counter = 0; + return '\n'; + } + } else if (c != -1) { + nmi_counter = 0; + } +#endif return c; } diff --git a/osfmk/console/serial_general.c b/osfmk/console/serial_general.c index c6d43f858..686564ed3 100644 --- a/osfmk/console/serial_general.c +++ b/osfmk/console/serial_general.c @@ -54,7 +54,7 @@ serial_keyboard_init(void) kern_return_t result; thread_t thread; - if(!(serialmode & 2)) /* Leave if we do not want a serial console */ + if(!(serialmode & SERIALMODE_INPUT)) /* Leave if we do not want a serial console */ return; kprintf("Serial keyboard started\n"); @@ -130,3 +130,58 @@ switch_to_old_console(int old_console) } else cons_ops_index = ops; } + +void +console_printbuf_state_init(struct console_printbuf_state * data, int write_on_newline, int can_block) +{ + if (data == NULL) + return; + bzero(data, sizeof(struct console_printbuf_state)); + if (write_on_newline) + data->flags |= CONS_PB_WRITE_NEWLINE; + if (can_block) + data->flags |= CONS_PB_CANBLOCK; +} + +void +console_printbuf_putc(int ch, void * arg) +{ + struct console_printbuf_state * info = (struct console_printbuf_state *)arg; + info->total += 1; + if (info->pos < (SERIAL_CONS_BUF_SIZE - 1)) { + info->str[info->pos] = ch; + info->pos += 1; + } else { + /* + * when len(line) > SERIAL_CONS_BUF_SIZE, we truncate the message + * if boot-arg 'drain_uart_sync=1' is set, then + * drain all the buffer right now and append new ch + */ + if (serialmode & SERIALMODE_SYNCDRAIN) { + info->str[info->pos] = '\0'; + console_write(info->str, info->pos); + info->pos = 0; + info->str[info->pos] = ch; + info->pos += 1; + } + } + + info->str[info->pos] = '\0'; + /* if newline, then try output to console */ + if (ch == '\n' && info->flags & CONS_PB_WRITE_NEWLINE) { + console_write(info->str, info->pos); + info->pos = 0; + info->str[info->pos] = '\0'; + } +} + +void +console_printbuf_clear(struct console_printbuf_state * info) { + if (info->pos != 0) { + console_write(info->str, info->pos); + } + info->pos = 0; + info->str[info->pos] = '\0'; + info->total = 0; +} + diff --git a/osfmk/console/serial_protos.h b/osfmk/console/serial_protos.h index cf372f624..722f528f8 100644 --- a/osfmk/console/serial_protos.h +++ b/osfmk/console/serial_protos.h @@ -35,14 +35,30 @@ #ifndef _CONSOLE_SERIAL_PROTOS_H_ #define _CONSOLE_SERIAL_PROTOS_H_ +#ifdef __cplusplus +extern "C" { +#endif + + void serial_keyboard_init(void); void serial_keyboard_start(void); void serial_keyboard_poll(void); extern uint32_t serialmode; + +#define SERIALMODE_OUTPUT 0x1 +#define SERIALMODE_INPUT 0x2 +#define SERIALMODE_SYNCDRAIN 0x4 + extern uint32_t cons_ops_index; extern const uint32_t nconsops; extern unsigned int disable_serial_output; +#if defined(__arm__) || defined(__arm64__) +/* ARM64_TODO */ +extern void *console_cpu_alloc(boolean_t); +extern void console_cpu_free(void *); +void console_init(void); +#endif int _serial_getc(int unit, int line, boolean_t wait, boolean_t raw); @@ -59,4 +75,30 @@ void switch_to_old_console(int old_console); #define SERIAL_CONS_OPS 0 #define VC_CONS_OPS 1 +#ifdef XNU_KERNEL_PRIVATE + +#define SERIAL_CONS_BUF_SIZE 256 +struct console_printbuf_state { + int pos; + int total; + int flags; +#define CONS_PB_WRITE_NEWLINE 0x1 +#define CONS_PB_CANBLOCK 0x2 + char str[SERIAL_CONS_BUF_SIZE]; +}; + +extern int console_printbuf_drain_initialized; +void console_printbuf_state_init(struct console_printbuf_state * data, int write_on_newline, int can_block); +void console_printbuf_putc(int ch, void *arg); +void console_printbuf_clear(struct console_printbuf_state * info); +int console_write_try(char * str, int size); + + +#endif /* XNU_KERNEL_PRIVATE */ + +#ifdef __cplusplus +} +#endif + + #endif /* _CONSOLE_SERIAL_PROTOS_H_ */ diff --git a/osfmk/console/video_console.c b/osfmk/console/video_console.c index 5295d3c03..6d54af493 100644 --- a/osfmk/console/video_console.c +++ b/osfmk/console/video_console.c @@ -106,7 +106,9 @@ #include #include "iso_font.c" +#if !CONFIG_EMBEDDED #include "progress_meter_data.c" +#endif #include "sys/msgbuf.h" @@ -243,7 +245,11 @@ enum { /* secs */ kProgressAcquireDelay = 0, +#if CONFIG_EMBEDDED + kProgressReacquireDelay = 5, +#else kProgressReacquireDelay = 5, +#endif }; static int8_t vc_rotate_matr[4][2][2] = { @@ -1287,7 +1293,7 @@ gc_update_color(int color, boolean_t fore) void vcputc(__unused int l, __unused int u, int c) { - if ( gc_initialized && ( gc_enabled || debug_mode ) ) + if ( gc_initialized && gc_enabled ) { spl_t s; @@ -1296,7 +1302,7 @@ vcputc(__unused int l, __unused int u, int c) x86_filter_TLB_coherency_interrupts(TRUE); #endif VCPUTC_LOCK_LOCK(); - if ( gc_enabled || debug_mode ) + if ( gc_enabled ) { gc_hide_cursor(gc_x, gc_y); gc_putchar(c); @@ -1353,6 +1359,7 @@ static int vc_rendered_char_size = 0; #define REN_MAX_DEPTH 32 static unsigned char vc_rendered_char[ISO_CHAR_HEIGHT * ((REN_MAX_DEPTH / 8) * ISO_CHAR_WIDTH)]; +#if !CONFIG_EMBEDDED static void internal_set_progressmeter(int new_value); static void @@ -1370,6 +1377,7 @@ enum kProgressMeterEnd = 512, }; +#endif /* !CONFIG_EMBEDDED */ static boolean_t vc_progress_white = #ifdef CONFIG_VC_PROGRESS_WHITE @@ -1865,6 +1873,7 @@ vc_progress_user_options vc_user_options; decl_simple_lock_data(,vc_progress_lock) +#if !CONFIG_EMBEDDED static int vc_progress_withmeter = 3; int vc_progressmeter_enable; static int vc_progressmeter_drawn; @@ -1879,6 +1888,7 @@ static void * vc_progressmeter_backbuffer; static boolean_t vc_progressmeter_hold; static uint32_t vc_progressmeter_diskspeed = 256; +#endif /* !CONFIG_EMBEDDED */ enum { kSave = 0x10, @@ -1923,7 +1933,9 @@ static void vc_blit_rect_30(int x, int y, int bx, unsigned int * backBuffer, unsigned int flags); static void vc_progress_task( void * arg0, void * arg ); +#if !CONFIG_EMBEDDED static void vc_progressmeter_task( void * arg0, void * arg ); +#endif /* !CONFIG_EMBEDDED */ static void vc_blit_rect(int x, int y, int bx, int width, int height, @@ -1999,6 +2011,20 @@ vc_blit_rect_8(int x, int y, __unused int bx, /* For ARM, 16-bit is 565 (RGB); it is 1555 (XRGB) on other platforms */ +#ifdef __arm__ +#define CLUT_MASK_R 0xf8 +#define CLUT_MASK_G 0xfc +#define CLUT_MASK_B 0xf8 +#define CLUT_SHIFT_R << 8 +#define CLUT_SHIFT_G << 3 +#define CLUT_SHIFT_B >> 3 +#define MASK_R 0xf800 +#define MASK_G 0x07e0 +#define MASK_B 0x001f +#define MASK_R_8 0x7f800 +#define MASK_G_8 0x01fe0 +#define MASK_B_8 0x000ff +#else #define CLUT_MASK_R 0xf8 #define CLUT_MASK_G 0xf8 #define CLUT_MASK_B 0xf8 @@ -2011,6 +2037,7 @@ vc_blit_rect_8(int x, int y, __unused int bx, #define MASK_R_8 0x3fc00 #define MASK_G_8 0x01fe0 #define MASK_B_8 0x000ff +#endif static void vc_blit_rect_16( int x, int y, int bx, int width, int height, @@ -2211,11 +2238,13 @@ static void vc_blit_rect_30(int x, int y, int bx, static void vc_clean_boot_graphics(void) { +#if !CONFIG_EMBEDDED // clean up possible FDE login graphics vc_progress_set(FALSE, 0); const unsigned char * color = (typeof(color))(uintptr_t)(vc_progress_white ? 0x00000000 : 0xBFBFBFBF); vc_blit_rect(0, 0, 0, vinfo.v_width, vinfo.v_height, 0, 0, color, NULL, 0); +#endif } /* @@ -2412,9 +2441,11 @@ vc_progress_initialize( vc_progress_element * desc, clock_interval_to_absolutetime_interval(vc_progress->time, 1000 * 1000, &abstime); vc_progress_interval = (uint32_t)abstime; +#if !CONFIG_EMBEDDED thread_call_setup(&vc_progressmeter_call, vc_progressmeter_task, NULL); clock_interval_to_absolutetime_interval(1000 / 8, 1000 * 1000, &abstime); vc_progressmeter_interval = (uint32_t)abstime; +#endif /* !CONFIG_EMBEDDED */ } @@ -2432,6 +2463,7 @@ vc_progress_set(boolean_t enable, uint32_t vc_delay) unsigned int pdata32; unsigned int * buf32; +#if !CONFIG_EMBEDDED if (kBootArgsFlagBlack & ((boot_args *) PE_state.bootArgs)->flags) return; @@ -2462,11 +2494,12 @@ vc_progress_set(boolean_t enable, uint32_t vc_delay) return; } +#endif /* !CONFIG_EMBEDDED */ if(!vc_progress) return; if( enable) { - saveLen = (vc_progress->width * vc_uiscale) * (vc_progress->height * vc_uiscale) * vinfo.v_depth / 8; + saveLen = (vc_progress->width * vc_uiscale) * (vc_progress->height * vc_uiscale) * ((vinfo.v_depth + 7) / 8); saveBuf = kalloc( saveLen ); switch( vinfo.v_depth) { @@ -2544,6 +2577,7 @@ vc_progress_set(boolean_t enable, uint32_t vc_delay) kfree( saveBuf, saveLen ); } +#if !CONFIG_EMBEDDED static uint32_t vc_progressmeter_range(uint32_t pos) { @@ -2589,6 +2623,7 @@ void vc_progress_setdiskspeed(uint32_t speed) vc_progressmeter_diskspeed = speed; } +#endif /* !CONFIG_EMBEDDED */ static void vc_progress_task(__unused void *arg0, __unused void *arg) @@ -2710,8 +2745,10 @@ gc_pause( boolean_t pause, boolean_t graphics_now ) if (vc_progress_enable) { +#if !CONFIG_EMBEDDED if (1 & vc_progress_withmeter) thread_call_enter_delayed(&vc_progressmeter_call, vc_progressmeter_deadline); else +#endif /* !CONFIG_EMBEDDED */ thread_call_enter_delayed(&vc_progress_call, vc_progress_deadline); } @@ -2722,6 +2759,20 @@ gc_pause( boolean_t pause, boolean_t graphics_now ) static void vc_initialize(__unused struct vc_info * vinfo_p) { +#ifdef __arm__ + unsigned long cnt, data16, data32; + + if (vinfo.v_depth == 16) { + for (cnt = 0; cnt < 8; cnt++) { + data32 = vc_colors[cnt][2]; + data16 = (data32 & 0x0000F8) << 8; + data16 |= (data32 & 0x00FC00) >> 5; + data16 |= (data32 & 0xF80000) >> 19; + data16 |= data16 << 16; + vc_colors[cnt][1] = data16; + } + } +#endif vinfo.v_rows = vinfo.v_height / ISO_CHAR_HEIGHT; vinfo.v_columns = vinfo.v_width / ISO_CHAR_WIDTH; @@ -2878,7 +2929,6 @@ initialize_screen(PE_Video * boot_vinfo, unsigned int op) break; case kPETextMode: - disable_debug_output = FALSE; gc_graphics_boot = FALSE; break; @@ -2918,7 +2968,6 @@ initialize_screen(PE_Video * boot_vinfo, unsigned int op) case kPETextScreen: if ( console_is_serial() ) break; - disable_debug_output = FALSE; if ( gc_acquired == FALSE ) { gc_desire_text = TRUE; @@ -2927,7 +2976,9 @@ initialize_screen(PE_Video * boot_vinfo, unsigned int op) if ( gc_graphics_boot == FALSE ) break; vc_progress_set( FALSE, 0 ); +#if !CONFIG_EMBEDDED vc_enable_progressmeter( FALSE ); +#endif gc_enable( TRUE ); break; @@ -2940,12 +2991,15 @@ initialize_screen(PE_Video * boot_vinfo, unsigned int op) vc_progress_set( FALSE, 0 ); vc_acquire_delay = kProgressReacquireDelay; vc_progress_white = TRUE; +#if !CONFIG_EMBEDDED vc_enable_progressmeter(FALSE); vc_progress_withmeter &= ~1; +#endif vc_clut8 = NULL; break; +#if !CONFIG_EMBEDDED case kPERefreshBootGraphics: { spl_t s; @@ -2970,6 +3024,7 @@ initialize_screen(PE_Video * boot_vinfo, unsigned int op) internal_enable_progressmeter(kProgressMeterOff); vc_progress_white = save; } +#endif } } @@ -2980,6 +3035,7 @@ vcattach(void) { vm_initialized = TRUE; +#if !CONFIG_EMBEDDED const boot_args * bootargs = (typeof(bootargs)) PE_state.bootArgs; vc_progress_white = (0 != ((kBootArgsFlagBlackBg | kBootArgsFlagLoginUI) @@ -2996,6 +3052,7 @@ vcattach(void) vc_progress_meter_start = 0; vc_progress_meter_end = kProgressMeterMax; } +#endif simple_lock_init(&vc_progress_lock, 0); if ( gc_graphics_boot == FALSE ) @@ -3025,6 +3082,7 @@ vcattach(void) } } +#if !CONFIG_EMBEDDED // redraw progress meter between pixels x1, x2, position at x3 static void @@ -3224,6 +3282,7 @@ vc_set_progressmeter(int new_value) splx(s); } +#endif /* !CONFIG_EMBEDDED */ diff --git a/osfmk/console/video_console.h b/osfmk/console/video_console.h index 468dc9e61..025c51817 100644 --- a/osfmk/console/video_console.h +++ b/osfmk/console/video_console.h @@ -133,6 +133,7 @@ int vc_display_lzss_icon(uint32_t dst_x, uint32_t dst_y, uint32_t compressed_size, const uint8_t *clut); +#if !CONFIG_EMBEDDED extern void vc_enable_progressmeter(int new_value); extern void vc_set_progressmeter(int new_value); @@ -140,6 +141,7 @@ extern int vc_progressmeter_enable; extern int vc_progressmeter_value; extern void vc_progress_setdiskspeed(uint32_t speed); +#endif /* !CONFIG_EMBEDDED */ #endif /* XNU_KERNEL_PRIVATE */ diff --git a/osfmk/console/video_scroll.c b/osfmk/console/video_scroll.c index 466430bd9..cdfef7d9f 100644 --- a/osfmk/console/video_scroll.c +++ b/osfmk/console/video_scroll.c @@ -29,8 +29,6 @@ #include #include -extern void bcopy(const void *, void *, size_t); - void video_scroll_up(void * start, void * end, void * dest) { diff --git a/osfmk/corecrypto/cc/src/cc_abort.c b/osfmk/corecrypto/cc/src/cc_abort.c deleted file mode 100644 index ac48bd9e3..000000000 --- a/osfmk/corecrypto/cc/src/cc_abort.c +++ /dev/null @@ -1,36 +0,0 @@ -/* - * cc_abort.c - * corecrypto - * - * Created on 7/16/2015 - * - * Copyright (c) 2014,2015 Apple Inc. All rights reserved. - * - */ - -#include - -//cc_abort() is implemented to comply with by FIPS 140-2, when DRBG produces -//two equal consecutive blocks. See radar 19129408 - -#if CC_KERNEL -#include -void cc_abort(const char * msg CC_UNUSED , ...) -{ - panic(msg); -} - -#elif CC_USE_SEPROM || CC_USE_S3 || CC_BASEBAND || CC_EFI || CC_IBOOT -void cc_abort(const char * msg CC_UNUSED, ...) -{ - //do nothing and return becasue we don't have panic() in those - //environments -} - -#else -#include -void cc_abort(const char * msg CC_UNUSED, ...) -{ - abort(); -} -#endif diff --git a/osfmk/corecrypto/cc/src/cc_clear.c b/osfmk/corecrypto/cc/src/cc_clear.c index a163e900d..5fff15bba 100644 --- a/osfmk/corecrypto/cc/src/cc_clear.c +++ b/osfmk/corecrypto/cc/src/cc_clear.c @@ -6,15 +6,41 @@ * * Copyright (c) 2014,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include +#include "corecrypto/fipspost_trace.h" //rdar://problem/26986552 #if ( CC_HAS_MEMSET_S == 1 ) && (defined( __STDC_WANT_LIB_EXT1__ ) && ( __STDC_WANT_LIB_EXT1__ == 1 ) ) void cc_clear(size_t len, void *dst) { + FIPSPOST_TRACE_EVENT; memset_s(dst,len,0,len); } #elif defined(_WIN32) && !defined(__clang__) //Clang with Microsoft CodeGen, doesn't support SecureZeroMemory @@ -26,6 +52,7 @@ static void cc_clear(size_t len, void *dst) #else void cc_clear(size_t len, void *dst) { + FIPSPOST_TRACE_EVENT; volatile char *vptr = (volatile char *)dst; while (len--) *vptr++ = '\0'; diff --git a/osfmk/corecrypto/cc/src/cc_cmp_safe.c b/osfmk/corecrypto/cc/src/cc_cmp_safe.c index b06c8a724..7a33dff61 100644 --- a/osfmk/corecrypto/cc/src/cc_cmp_safe.c +++ b/osfmk/corecrypto/cc/src/cc_cmp_safe.c @@ -6,6 +6,30 @@ * * Copyright (c) 2014,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include diff --git a/osfmk/corecrypto/cc/src/cc_try_abort.c b/osfmk/corecrypto/cc/src/cc_try_abort.c new file mode 100644 index 000000000..2a0437671 --- /dev/null +++ b/osfmk/corecrypto/cc/src/cc_try_abort.c @@ -0,0 +1,60 @@ +/* + * cc_try_abort.c + * corecrypto + * + * Created on 7/16/2015 + * + * Copyright (c) 2014,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include + +//cc_try_abort() is implemented to comply with by FIPS 140-2, when DRBG produces +//two equal consecutive blocks. See radar 19129408 + +#if CC_KERNEL +#include +void cc_try_abort(const char * msg CC_UNUSED , ...) +{ + panic(msg); +} + +#elif CC_USE_SEPROM || CC_USE_S3 || CC_BASEBAND || CC_EFI || CC_IBOOT || CC_RTKIT +void cc_try_abort(const char * msg CC_UNUSED, ...) +{ + //Do nothing and return because we don't have panic() in those + //environments. Make sure you return error, when using cc_try_abort() in above environments +} + +#else +#include +void cc_try_abort(const char * msg CC_UNUSED, ...) +{ + abort(); +} +#endif diff --git a/osfmk/corecrypto/ccaes/src/aes_tab.c b/osfmk/corecrypto/ccaes/src/aes_tab.c new file mode 100644 index 000000000..0fe7b19a6 --- /dev/null +++ b/osfmk/corecrypto/ccaes/src/aes_tab.c @@ -0,0 +1,1061 @@ +/* + * aes_tab.c + * corecrypto + * + * Created on 12/12/2010 + * + * Copyright (c) 2010,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtom.org + */ + +/* The precomputed tables for AES */ +/* +Te0[x] = S [x].[02, 01, 01, 03]; +Te1[x] = S [x].[03, 02, 01, 01]; +Te2[x] = S [x].[01, 03, 02, 01]; +Te3[x] = S [x].[01, 01, 03, 02]; +Te4[x] = S [x].[01, 01, 01, 01]; + +Td0[x] = Si[x].[0e, 09, 0d, 0b]; +Td1[x] = Si[x].[0b, 0e, 09, 0d]; +Td2[x] = Si[x].[0d, 0b, 0e, 09]; +Td3[x] = Si[x].[09, 0d, 0b, 0e]; +Td4[x] = Si[x].[01, 01, 01, 01]; +*/ + +#include + +/*! + @file aes_tab.c + AES tables +*/ +static const uint32_t TE0[256] = { + 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, + 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, + 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, + 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, + 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, + 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, + 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, + 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b, + 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, + 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, + 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, + 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, + 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, + 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, + 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, + 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, + 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, + 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, + 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, + 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497, + 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, + 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, + 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, + 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, + 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, + 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, + 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, + 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, + 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, + 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, + 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, + 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d, + 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, + 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, + 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, + 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, + 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, + 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, + 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, + 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, + 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, + 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, + 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, + 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b, + 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, + 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, + 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, + 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, + 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, + 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, + 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, + 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, + 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, + 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, + 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, + 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9, + 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, + 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, + 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, + 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, + 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, + 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, + 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, + 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, +}; + +#ifndef PELI_TAB +static const uint32_t Te4[256] = { + 0x63636363, 0x7c7c7c7c, 0x77777777, 0x7b7b7b7b, + 0xf2f2f2f2, 0x6b6b6b6b, 0x6f6f6f6f, 0xc5c5c5c5, + 0x30303030, 0x01010101, 0x67676767, 0x2b2b2b2b, + 0xfefefefe, 0xd7d7d7d7, 0xabababab, 0x76767676, + 0xcacacaca, 0x82828282, 0xc9c9c9c9, 0x7d7d7d7d, + 0xfafafafa, 0x59595959, 0x47474747, 0xf0f0f0f0, + 0xadadadad, 0xd4d4d4d4, 0xa2a2a2a2, 0xafafafaf, + 0x9c9c9c9c, 0xa4a4a4a4, 0x72727272, 0xc0c0c0c0, + 0xb7b7b7b7, 0xfdfdfdfd, 0x93939393, 0x26262626, + 0x36363636, 0x3f3f3f3f, 0xf7f7f7f7, 0xcccccccc, + 0x34343434, 0xa5a5a5a5, 0xe5e5e5e5, 0xf1f1f1f1, + 0x71717171, 0xd8d8d8d8, 0x31313131, 0x15151515, + 0x04040404, 0xc7c7c7c7, 0x23232323, 0xc3c3c3c3, + 0x18181818, 0x96969696, 0x05050505, 0x9a9a9a9a, + 0x07070707, 0x12121212, 0x80808080, 0xe2e2e2e2, + 0xebebebeb, 0x27272727, 0xb2b2b2b2, 0x75757575, + 0x09090909, 0x83838383, 0x2c2c2c2c, 0x1a1a1a1a, + 0x1b1b1b1b, 0x6e6e6e6e, 0x5a5a5a5a, 0xa0a0a0a0, + 0x52525252, 0x3b3b3b3b, 0xd6d6d6d6, 0xb3b3b3b3, + 0x29292929, 0xe3e3e3e3, 0x2f2f2f2f, 0x84848484, + 0x53535353, 0xd1d1d1d1, 0x00000000, 0xedededed, + 0x20202020, 0xfcfcfcfc, 0xb1b1b1b1, 0x5b5b5b5b, + 0x6a6a6a6a, 0xcbcbcbcb, 0xbebebebe, 0x39393939, + 0x4a4a4a4a, 0x4c4c4c4c, 0x58585858, 0xcfcfcfcf, + 0xd0d0d0d0, 0xefefefef, 0xaaaaaaaa, 0xfbfbfbfb, + 0x43434343, 0x4d4d4d4d, 0x33333333, 0x85858585, + 0x45454545, 0xf9f9f9f9, 0x02020202, 0x7f7f7f7f, + 0x50505050, 0x3c3c3c3c, 0x9f9f9f9f, 0xa8a8a8a8, + 0x51515151, 0xa3a3a3a3, 0x40404040, 0x8f8f8f8f, + 0x92929292, 0x9d9d9d9d, 0x38383838, 0xf5f5f5f5, + 0xbcbcbcbc, 0xb6b6b6b6, 0xdadadada, 0x21212121, + 0x10101010, 0xffffffff, 0xf3f3f3f3, 0xd2d2d2d2, + 0xcdcdcdcd, 0x0c0c0c0c, 0x13131313, 0xecececec, + 0x5f5f5f5f, 0x97979797, 0x44444444, 0x17171717, + 0xc4c4c4c4, 0xa7a7a7a7, 0x7e7e7e7e, 0x3d3d3d3d, + 0x64646464, 0x5d5d5d5d, 0x19191919, 0x73737373, + 0x60606060, 0x81818181, 0x4f4f4f4f, 0xdcdcdcdc, + 0x22222222, 0x2a2a2a2a, 0x90909090, 0x88888888, + 0x46464646, 0xeeeeeeee, 0xb8b8b8b8, 0x14141414, + 0xdededede, 0x5e5e5e5e, 0x0b0b0b0b, 0xdbdbdbdb, + 0xe0e0e0e0, 0x32323232, 0x3a3a3a3a, 0x0a0a0a0a, + 0x49494949, 0x06060606, 0x24242424, 0x5c5c5c5c, + 0xc2c2c2c2, 0xd3d3d3d3, 0xacacacac, 0x62626262, + 0x91919191, 0x95959595, 0xe4e4e4e4, 0x79797979, + 0xe7e7e7e7, 0xc8c8c8c8, 0x37373737, 0x6d6d6d6d, + 0x8d8d8d8d, 0xd5d5d5d5, 0x4e4e4e4e, 0xa9a9a9a9, + 0x6c6c6c6c, 0x56565656, 0xf4f4f4f4, 0xeaeaeaea, + 0x65656565, 0x7a7a7a7a, 0xaeaeaeae, 0x08080808, + 0xbabababa, 0x78787878, 0x25252525, 0x2e2e2e2e, + 0x1c1c1c1c, 0xa6a6a6a6, 0xb4b4b4b4, 0xc6c6c6c6, + 0xe8e8e8e8, 0xdddddddd, 0x74747474, 0x1f1f1f1f, + 0x4b4b4b4b, 0xbdbdbdbd, 0x8b8b8b8b, 0x8a8a8a8a, + 0x70707070, 0x3e3e3e3e, 0xb5b5b5b5, 0x66666666, + 0x48484848, 0x03030303, 0xf6f6f6f6, 0x0e0e0e0e, + 0x61616161, 0x35353535, 0x57575757, 0xb9b9b9b9, + 0x86868686, 0xc1c1c1c1, 0x1d1d1d1d, 0x9e9e9e9e, + 0xe1e1e1e1, 0xf8f8f8f8, 0x98989898, 0x11111111, + 0x69696969, 0xd9d9d9d9, 0x8e8e8e8e, 0x94949494, + 0x9b9b9b9b, 0x1e1e1e1e, 0x87878787, 0xe9e9e9e9, + 0xcececece, 0x55555555, 0x28282828, 0xdfdfdfdf, + 0x8c8c8c8c, 0xa1a1a1a1, 0x89898989, 0x0d0d0d0d, + 0xbfbfbfbf, 0xe6e6e6e6, 0x42424242, 0x68686868, + 0x41414141, 0x99999999, 0x2d2d2d2d, 0x0f0f0f0f, + 0xb0b0b0b0, 0x54545454, 0xbbbbbbbb, 0x16161616, +}; +#endif + +#ifndef ENCRYPT_ONLY + +static const uint32_t TD0[256] = { + 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, + 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, + 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, + 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f, + 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, + 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, + 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, + 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844, + 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, + 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, + 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, + 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94, + 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, + 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, + 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, + 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c, + 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, + 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, + 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, + 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051, + 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, + 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, + 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, + 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb, + 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, + 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, + 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, + 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a, + 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, + 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, + 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, + 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8, + 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, + 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, + 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, + 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120, + 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, + 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, + 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, + 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef, + 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, + 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, + 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, + 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5, + 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, + 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, + 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, + 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6, + 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, + 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, + 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, + 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f, + 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, + 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, + 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, + 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713, + 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, + 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, + 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, + 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86, + 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, + 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, + 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, + 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742, +}; + +static const uint32_t Td4[256] = { + 0x52525252, 0x09090909, 0x6a6a6a6a, 0xd5d5d5d5, + 0x30303030, 0x36363636, 0xa5a5a5a5, 0x38383838, + 0xbfbfbfbf, 0x40404040, 0xa3a3a3a3, 0x9e9e9e9e, + 0x81818181, 0xf3f3f3f3, 0xd7d7d7d7, 0xfbfbfbfb, + 0x7c7c7c7c, 0xe3e3e3e3, 0x39393939, 0x82828282, + 0x9b9b9b9b, 0x2f2f2f2f, 0xffffffff, 0x87878787, + 0x34343434, 0x8e8e8e8e, 0x43434343, 0x44444444, + 0xc4c4c4c4, 0xdededede, 0xe9e9e9e9, 0xcbcbcbcb, + 0x54545454, 0x7b7b7b7b, 0x94949494, 0x32323232, + 0xa6a6a6a6, 0xc2c2c2c2, 0x23232323, 0x3d3d3d3d, + 0xeeeeeeee, 0x4c4c4c4c, 0x95959595, 0x0b0b0b0b, + 0x42424242, 0xfafafafa, 0xc3c3c3c3, 0x4e4e4e4e, + 0x08080808, 0x2e2e2e2e, 0xa1a1a1a1, 0x66666666, + 0x28282828, 0xd9d9d9d9, 0x24242424, 0xb2b2b2b2, + 0x76767676, 0x5b5b5b5b, 0xa2a2a2a2, 0x49494949, + 0x6d6d6d6d, 0x8b8b8b8b, 0xd1d1d1d1, 0x25252525, + 0x72727272, 0xf8f8f8f8, 0xf6f6f6f6, 0x64646464, + 0x86868686, 0x68686868, 0x98989898, 0x16161616, + 0xd4d4d4d4, 0xa4a4a4a4, 0x5c5c5c5c, 0xcccccccc, + 0x5d5d5d5d, 0x65656565, 0xb6b6b6b6, 0x92929292, + 0x6c6c6c6c, 0x70707070, 0x48484848, 0x50505050, + 0xfdfdfdfd, 0xedededed, 0xb9b9b9b9, 0xdadadada, + 0x5e5e5e5e, 0x15151515, 0x46464646, 0x57575757, + 0xa7a7a7a7, 0x8d8d8d8d, 0x9d9d9d9d, 0x84848484, + 0x90909090, 0xd8d8d8d8, 0xabababab, 0x00000000, + 0x8c8c8c8c, 0xbcbcbcbc, 0xd3d3d3d3, 0x0a0a0a0a, + 0xf7f7f7f7, 0xe4e4e4e4, 0x58585858, 0x05050505, + 0xb8b8b8b8, 0xb3b3b3b3, 0x45454545, 0x06060606, + 0xd0d0d0d0, 0x2c2c2c2c, 0x1e1e1e1e, 0x8f8f8f8f, + 0xcacacaca, 0x3f3f3f3f, 0x0f0f0f0f, 0x02020202, + 0xc1c1c1c1, 0xafafafaf, 0xbdbdbdbd, 0x03030303, + 0x01010101, 0x13131313, 0x8a8a8a8a, 0x6b6b6b6b, + 0x3a3a3a3a, 0x91919191, 0x11111111, 0x41414141, + 0x4f4f4f4f, 0x67676767, 0xdcdcdcdc, 0xeaeaeaea, + 0x97979797, 0xf2f2f2f2, 0xcfcfcfcf, 0xcececece, + 0xf0f0f0f0, 0xb4b4b4b4, 0xe6e6e6e6, 0x73737373, + 0x96969696, 0xacacacac, 0x74747474, 0x22222222, + 0xe7e7e7e7, 0xadadadad, 0x35353535, 0x85858585, + 0xe2e2e2e2, 0xf9f9f9f9, 0x37373737, 0xe8e8e8e8, + 0x1c1c1c1c, 0x75757575, 0xdfdfdfdf, 0x6e6e6e6e, + 0x47474747, 0xf1f1f1f1, 0x1a1a1a1a, 0x71717171, + 0x1d1d1d1d, 0x29292929, 0xc5c5c5c5, 0x89898989, + 0x6f6f6f6f, 0xb7b7b7b7, 0x62626262, 0x0e0e0e0e, + 0xaaaaaaaa, 0x18181818, 0xbebebebe, 0x1b1b1b1b, + 0xfcfcfcfc, 0x56565656, 0x3e3e3e3e, 0x4b4b4b4b, + 0xc6c6c6c6, 0xd2d2d2d2, 0x79797979, 0x20202020, + 0x9a9a9a9a, 0xdbdbdbdb, 0xc0c0c0c0, 0xfefefefe, + 0x78787878, 0xcdcdcdcd, 0x5a5a5a5a, 0xf4f4f4f4, + 0x1f1f1f1f, 0xdddddddd, 0xa8a8a8a8, 0x33333333, + 0x88888888, 0x07070707, 0xc7c7c7c7, 0x31313131, + 0xb1b1b1b1, 0x12121212, 0x10101010, 0x59595959, + 0x27272727, 0x80808080, 0xecececec, 0x5f5f5f5f, + 0x60606060, 0x51515151, 0x7f7f7f7f, 0xa9a9a9a9, + 0x19191919, 0xb5b5b5b5, 0x4a4a4a4a, 0x0d0d0d0d, + 0x2d2d2d2d, 0xe5e5e5e5, 0x7a7a7a7a, 0x9f9f9f9f, + 0x93939393, 0xc9c9c9c9, 0x9c9c9c9c, 0xefefefef, + 0xa0a0a0a0, 0xe0e0e0e0, 0x3b3b3b3b, 0x4d4d4d4d, + 0xaeaeaeae, 0x2a2a2a2a, 0xf5f5f5f5, 0xb0b0b0b0, + 0xc8c8c8c8, 0xebebebeb, 0xbbbbbbbb, 0x3c3c3c3c, + 0x83838383, 0x53535353, 0x99999999, 0x61616161, + 0x17171717, 0x2b2b2b2b, 0x04040404, 0x7e7e7e7e, + 0xbabababa, 0x77777777, 0xd6d6d6d6, 0x26262626, + 0xe1e1e1e1, 0x69696969, 0x14141414, 0x63636363, + 0x55555555, 0x21212121, 0x0c0c0c0c, 0x7d7d7d7d, +}; + +#endif /* ENCRYPT_ONLY */ + +#ifdef LTC_SMALL_CODE + +#define Te0(x) TE0[x] +#define Te1(x) RORc(TE0[x], 8) +#define Te2(x) RORc(TE0[x], 16) +#define Te3(x) RORc(TE0[x], 24) + +#define Td0(x) TD0[x] +#define Td1(x) RORc(TD0[x], 8) +#define Td2(x) RORc(TD0[x], 16) +#define Td3(x) RORc(TD0[x], 24) + +#define Te4_0 0x000000FF & Te4 +#define Te4_1 0x0000FF00 & Te4 +#define Te4_2 0x00FF0000 & Te4 +#define Te4_3 0xFF000000 & Te4 + +#else + +#define Te0(x) TE0[x] +#define Te1(x) TE1[x] +#define Te2(x) TE2[x] +#define Te3(x) TE3[x] + +#define Td0(x) TD0[x] +#define Td1(x) TD1[x] +#define Td2(x) TD2[x] +#define Td3(x) TD3[x] + +static const uint32_t TE1[256] = { + 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, + 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, + 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, + 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, + 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, + 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, + 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, + 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, + 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, + 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, + 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, + 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, + 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, + 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, + 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, + 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, + 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, + 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, + 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, + 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, + 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, + 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, + 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, + 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, + 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, + 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, + 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, + 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, + 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, + 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, + 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, + 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, + 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, + 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, + 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, + 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, + 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, + 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, + 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, + 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, + 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, + 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, + 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, + 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, + 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, + 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, + 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, + 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, + 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, + 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, + 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, + 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, + 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, + 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, + 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, + 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, + 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, + 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, + 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, + 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, + 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, + 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, + 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, + 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, +}; +static const uint32_t TE2[256] = { + 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, + 0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5, + 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, + 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, + 0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d, + 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, + 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, + 0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0, + 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, + 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, + 0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1, + 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, + 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, + 0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a, + 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, + 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, + 0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a, + 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, + 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, + 0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384, + 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, + 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, + 0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239, + 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, + 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, + 0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185, + 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, + 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, + 0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f, + 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, + 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, + 0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2, + 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, + 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, + 0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d, + 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, + 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, + 0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88, + 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, + 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, + 0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a, + 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, + 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, + 0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279, + 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, + 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, + 0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea, + 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, + 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, + 0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6, + 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, + 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, + 0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66, + 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, + 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, + 0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e, + 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, + 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, + 0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9, + 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, + 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, + 0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068, + 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, + 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, +}; +static const uint32_t TE3[256] = { + + 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, + 0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491, + 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, + 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, + 0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa, + 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, + 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, + 0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b, + 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, + 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, + 0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9, + 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, + 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, + 0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f, + 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, + 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, + 0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34, + 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, + 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, + 0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713, + 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, + 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, + 0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72, + 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, + 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, + 0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411, + 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, + 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, + 0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05, + 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, + 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, + 0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf, + 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, + 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, + 0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a, + 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, + 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, + 0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b, + 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, + 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, + 0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14, + 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, + 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, + 0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2, + 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, + 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, + 0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf, + 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, + 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, + 0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197, + 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, + 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, + 0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc, + 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, + 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, + 0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927, + 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, + 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, + 0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9, + 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, + 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, + 0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0, + 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, + 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, +}; + +#ifndef PELI_TAB +static const uint32_t Te4_0[] = { +0x00000063, 0x0000007c, 0x00000077, 0x0000007b, 0x000000f2, 0x0000006b, 0x0000006f, 0x000000c5, +0x00000030, 0x00000001, 0x00000067, 0x0000002b, 0x000000fe, 0x000000d7, 0x000000ab, 0x00000076, +0x000000ca, 0x00000082, 0x000000c9, 0x0000007d, 0x000000fa, 0x00000059, 0x00000047, 0x000000f0, +0x000000ad, 0x000000d4, 0x000000a2, 0x000000af, 0x0000009c, 0x000000a4, 0x00000072, 0x000000c0, +0x000000b7, 0x000000fd, 0x00000093, 0x00000026, 0x00000036, 0x0000003f, 0x000000f7, 0x000000cc, +0x00000034, 0x000000a5, 0x000000e5, 0x000000f1, 0x00000071, 0x000000d8, 0x00000031, 0x00000015, +0x00000004, 0x000000c7, 0x00000023, 0x000000c3, 0x00000018, 0x00000096, 0x00000005, 0x0000009a, +0x00000007, 0x00000012, 0x00000080, 0x000000e2, 0x000000eb, 0x00000027, 0x000000b2, 0x00000075, +0x00000009, 0x00000083, 0x0000002c, 0x0000001a, 0x0000001b, 0x0000006e, 0x0000005a, 0x000000a0, +0x00000052, 0x0000003b, 0x000000d6, 0x000000b3, 0x00000029, 0x000000e3, 0x0000002f, 0x00000084, +0x00000053, 0x000000d1, 0x00000000, 0x000000ed, 0x00000020, 0x000000fc, 0x000000b1, 0x0000005b, +0x0000006a, 0x000000cb, 0x000000be, 0x00000039, 0x0000004a, 0x0000004c, 0x00000058, 0x000000cf, +0x000000d0, 0x000000ef, 0x000000aa, 0x000000fb, 0x00000043, 0x0000004d, 0x00000033, 0x00000085, +0x00000045, 0x000000f9, 0x00000002, 0x0000007f, 0x00000050, 0x0000003c, 0x0000009f, 0x000000a8, +0x00000051, 0x000000a3, 0x00000040, 0x0000008f, 0x00000092, 0x0000009d, 0x00000038, 0x000000f5, +0x000000bc, 0x000000b6, 0x000000da, 0x00000021, 0x00000010, 0x000000ff, 0x000000f3, 0x000000d2, +0x000000cd, 0x0000000c, 0x00000013, 0x000000ec, 0x0000005f, 0x00000097, 0x00000044, 0x00000017, +0x000000c4, 0x000000a7, 0x0000007e, 0x0000003d, 0x00000064, 0x0000005d, 0x00000019, 0x00000073, +0x00000060, 0x00000081, 0x0000004f, 0x000000dc, 0x00000022, 0x0000002a, 0x00000090, 0x00000088, +0x00000046, 0x000000ee, 0x000000b8, 0x00000014, 0x000000de, 0x0000005e, 0x0000000b, 0x000000db, +0x000000e0, 0x00000032, 0x0000003a, 0x0000000a, 0x00000049, 0x00000006, 0x00000024, 0x0000005c, +0x000000c2, 0x000000d3, 0x000000ac, 0x00000062, 0x00000091, 0x00000095, 0x000000e4, 0x00000079, +0x000000e7, 0x000000c8, 0x00000037, 0x0000006d, 0x0000008d, 0x000000d5, 0x0000004e, 0x000000a9, +0x0000006c, 0x00000056, 0x000000f4, 0x000000ea, 0x00000065, 0x0000007a, 0x000000ae, 0x00000008, +0x000000ba, 0x00000078, 0x00000025, 0x0000002e, 0x0000001c, 0x000000a6, 0x000000b4, 0x000000c6, +0x000000e8, 0x000000dd, 0x00000074, 0x0000001f, 0x0000004b, 0x000000bd, 0x0000008b, 0x0000008a, +0x00000070, 0x0000003e, 0x000000b5, 0x00000066, 0x00000048, 0x00000003, 0x000000f6, 0x0000000e, +0x00000061, 0x00000035, 0x00000057, 0x000000b9, 0x00000086, 0x000000c1, 0x0000001d, 0x0000009e, +0x000000e1, 0x000000f8, 0x00000098, 0x00000011, 0x00000069, 0x000000d9, 0x0000008e, 0x00000094, +0x0000009b, 0x0000001e, 0x00000087, 0x000000e9, 0x000000ce, 0x00000055, 0x00000028, 0x000000df, +0x0000008c, 0x000000a1, 0x00000089, 0x0000000d, 0x000000bf, 0x000000e6, 0x00000042, 0x00000068, +0x00000041, 0x00000099, 0x0000002d, 0x0000000f, 0x000000b0, 0x00000054, 0x000000bb, 0x00000016 +}; + +static const uint32_t Te4_1[] = { +0x00006300, 0x00007c00, 0x00007700, 0x00007b00, 0x0000f200, 0x00006b00, 0x00006f00, 0x0000c500, +0x00003000, 0x00000100, 0x00006700, 0x00002b00, 0x0000fe00, 0x0000d700, 0x0000ab00, 0x00007600, +0x0000ca00, 0x00008200, 0x0000c900, 0x00007d00, 0x0000fa00, 0x00005900, 0x00004700, 0x0000f000, +0x0000ad00, 0x0000d400, 0x0000a200, 0x0000af00, 0x00009c00, 0x0000a400, 0x00007200, 0x0000c000, +0x0000b700, 0x0000fd00, 0x00009300, 0x00002600, 0x00003600, 0x00003f00, 0x0000f700, 0x0000cc00, +0x00003400, 0x0000a500, 0x0000e500, 0x0000f100, 0x00007100, 0x0000d800, 0x00003100, 0x00001500, +0x00000400, 0x0000c700, 0x00002300, 0x0000c300, 0x00001800, 0x00009600, 0x00000500, 0x00009a00, +0x00000700, 0x00001200, 0x00008000, 0x0000e200, 0x0000eb00, 0x00002700, 0x0000b200, 0x00007500, +0x00000900, 0x00008300, 0x00002c00, 0x00001a00, 0x00001b00, 0x00006e00, 0x00005a00, 0x0000a000, +0x00005200, 0x00003b00, 0x0000d600, 0x0000b300, 0x00002900, 0x0000e300, 0x00002f00, 0x00008400, +0x00005300, 0x0000d100, 0x00000000, 0x0000ed00, 0x00002000, 0x0000fc00, 0x0000b100, 0x00005b00, +0x00006a00, 0x0000cb00, 0x0000be00, 0x00003900, 0x00004a00, 0x00004c00, 0x00005800, 0x0000cf00, +0x0000d000, 0x0000ef00, 0x0000aa00, 0x0000fb00, 0x00004300, 0x00004d00, 0x00003300, 0x00008500, +0x00004500, 0x0000f900, 0x00000200, 0x00007f00, 0x00005000, 0x00003c00, 0x00009f00, 0x0000a800, +0x00005100, 0x0000a300, 0x00004000, 0x00008f00, 0x00009200, 0x00009d00, 0x00003800, 0x0000f500, +0x0000bc00, 0x0000b600, 0x0000da00, 0x00002100, 0x00001000, 0x0000ff00, 0x0000f300, 0x0000d200, +0x0000cd00, 0x00000c00, 0x00001300, 0x0000ec00, 0x00005f00, 0x00009700, 0x00004400, 0x00001700, +0x0000c400, 0x0000a700, 0x00007e00, 0x00003d00, 0x00006400, 0x00005d00, 0x00001900, 0x00007300, +0x00006000, 0x00008100, 0x00004f00, 0x0000dc00, 0x00002200, 0x00002a00, 0x00009000, 0x00008800, +0x00004600, 0x0000ee00, 0x0000b800, 0x00001400, 0x0000de00, 0x00005e00, 0x00000b00, 0x0000db00, +0x0000e000, 0x00003200, 0x00003a00, 0x00000a00, 0x00004900, 0x00000600, 0x00002400, 0x00005c00, +0x0000c200, 0x0000d300, 0x0000ac00, 0x00006200, 0x00009100, 0x00009500, 0x0000e400, 0x00007900, +0x0000e700, 0x0000c800, 0x00003700, 0x00006d00, 0x00008d00, 0x0000d500, 0x00004e00, 0x0000a900, +0x00006c00, 0x00005600, 0x0000f400, 0x0000ea00, 0x00006500, 0x00007a00, 0x0000ae00, 0x00000800, +0x0000ba00, 0x00007800, 0x00002500, 0x00002e00, 0x00001c00, 0x0000a600, 0x0000b400, 0x0000c600, +0x0000e800, 0x0000dd00, 0x00007400, 0x00001f00, 0x00004b00, 0x0000bd00, 0x00008b00, 0x00008a00, +0x00007000, 0x00003e00, 0x0000b500, 0x00006600, 0x00004800, 0x00000300, 0x0000f600, 0x00000e00, +0x00006100, 0x00003500, 0x00005700, 0x0000b900, 0x00008600, 0x0000c100, 0x00001d00, 0x00009e00, +0x0000e100, 0x0000f800, 0x00009800, 0x00001100, 0x00006900, 0x0000d900, 0x00008e00, 0x00009400, +0x00009b00, 0x00001e00, 0x00008700, 0x0000e900, 0x0000ce00, 0x00005500, 0x00002800, 0x0000df00, +0x00008c00, 0x0000a100, 0x00008900, 0x00000d00, 0x0000bf00, 0x0000e600, 0x00004200, 0x00006800, +0x00004100, 0x00009900, 0x00002d00, 0x00000f00, 0x0000b000, 0x00005400, 0x0000bb00, 0x00001600 +}; + +static const uint32_t Te4_2[] = { +0x00630000, 0x007c0000, 0x00770000, 0x007b0000, 0x00f20000, 0x006b0000, 0x006f0000, 0x00c50000, +0x00300000, 0x00010000, 0x00670000, 0x002b0000, 0x00fe0000, 0x00d70000, 0x00ab0000, 0x00760000, +0x00ca0000, 0x00820000, 0x00c90000, 0x007d0000, 0x00fa0000, 0x00590000, 0x00470000, 0x00f00000, +0x00ad0000, 0x00d40000, 0x00a20000, 0x00af0000, 0x009c0000, 0x00a40000, 0x00720000, 0x00c00000, +0x00b70000, 0x00fd0000, 0x00930000, 0x00260000, 0x00360000, 0x003f0000, 0x00f70000, 0x00cc0000, +0x00340000, 0x00a50000, 0x00e50000, 0x00f10000, 0x00710000, 0x00d80000, 0x00310000, 0x00150000, +0x00040000, 0x00c70000, 0x00230000, 0x00c30000, 0x00180000, 0x00960000, 0x00050000, 0x009a0000, +0x00070000, 0x00120000, 0x00800000, 0x00e20000, 0x00eb0000, 0x00270000, 0x00b20000, 0x00750000, +0x00090000, 0x00830000, 0x002c0000, 0x001a0000, 0x001b0000, 0x006e0000, 0x005a0000, 0x00a00000, +0x00520000, 0x003b0000, 0x00d60000, 0x00b30000, 0x00290000, 0x00e30000, 0x002f0000, 0x00840000, +0x00530000, 0x00d10000, 0x00000000, 0x00ed0000, 0x00200000, 0x00fc0000, 0x00b10000, 0x005b0000, +0x006a0000, 0x00cb0000, 0x00be0000, 0x00390000, 0x004a0000, 0x004c0000, 0x00580000, 0x00cf0000, +0x00d00000, 0x00ef0000, 0x00aa0000, 0x00fb0000, 0x00430000, 0x004d0000, 0x00330000, 0x00850000, +0x00450000, 0x00f90000, 0x00020000, 0x007f0000, 0x00500000, 0x003c0000, 0x009f0000, 0x00a80000, +0x00510000, 0x00a30000, 0x00400000, 0x008f0000, 0x00920000, 0x009d0000, 0x00380000, 0x00f50000, +0x00bc0000, 0x00b60000, 0x00da0000, 0x00210000, 0x00100000, 0x00ff0000, 0x00f30000, 0x00d20000, +0x00cd0000, 0x000c0000, 0x00130000, 0x00ec0000, 0x005f0000, 0x00970000, 0x00440000, 0x00170000, +0x00c40000, 0x00a70000, 0x007e0000, 0x003d0000, 0x00640000, 0x005d0000, 0x00190000, 0x00730000, +0x00600000, 0x00810000, 0x004f0000, 0x00dc0000, 0x00220000, 0x002a0000, 0x00900000, 0x00880000, +0x00460000, 0x00ee0000, 0x00b80000, 0x00140000, 0x00de0000, 0x005e0000, 0x000b0000, 0x00db0000, +0x00e00000, 0x00320000, 0x003a0000, 0x000a0000, 0x00490000, 0x00060000, 0x00240000, 0x005c0000, +0x00c20000, 0x00d30000, 0x00ac0000, 0x00620000, 0x00910000, 0x00950000, 0x00e40000, 0x00790000, +0x00e70000, 0x00c80000, 0x00370000, 0x006d0000, 0x008d0000, 0x00d50000, 0x004e0000, 0x00a90000, +0x006c0000, 0x00560000, 0x00f40000, 0x00ea0000, 0x00650000, 0x007a0000, 0x00ae0000, 0x00080000, +0x00ba0000, 0x00780000, 0x00250000, 0x002e0000, 0x001c0000, 0x00a60000, 0x00b40000, 0x00c60000, +0x00e80000, 0x00dd0000, 0x00740000, 0x001f0000, 0x004b0000, 0x00bd0000, 0x008b0000, 0x008a0000, +0x00700000, 0x003e0000, 0x00b50000, 0x00660000, 0x00480000, 0x00030000, 0x00f60000, 0x000e0000, +0x00610000, 0x00350000, 0x00570000, 0x00b90000, 0x00860000, 0x00c10000, 0x001d0000, 0x009e0000, +0x00e10000, 0x00f80000, 0x00980000, 0x00110000, 0x00690000, 0x00d90000, 0x008e0000, 0x00940000, +0x009b0000, 0x001e0000, 0x00870000, 0x00e90000, 0x00ce0000, 0x00550000, 0x00280000, 0x00df0000, +0x008c0000, 0x00a10000, 0x00890000, 0x000d0000, 0x00bf0000, 0x00e60000, 0x00420000, 0x00680000, +0x00410000, 0x00990000, 0x002d0000, 0x000f0000, 0x00b00000, 0x00540000, 0x00bb0000, 0x00160000 +}; + +static const uint32_t Te4_3[] = { +0x63000000, 0x7c000000, 0x77000000, 0x7b000000, 0xf2000000, 0x6b000000, 0x6f000000, 0xc5000000, +0x30000000, 0x01000000, 0x67000000, 0x2b000000, 0xfe000000, 0xd7000000, 0xab000000, 0x76000000, +0xca000000, 0x82000000, 0xc9000000, 0x7d000000, 0xfa000000, 0x59000000, 0x47000000, 0xf0000000, +0xad000000, 0xd4000000, 0xa2000000, 0xaf000000, 0x9c000000, 0xa4000000, 0x72000000, 0xc0000000, +0xb7000000, 0xfd000000, 0x93000000, 0x26000000, 0x36000000, 0x3f000000, 0xf7000000, 0xcc000000, +0x34000000, 0xa5000000, 0xe5000000, 0xf1000000, 0x71000000, 0xd8000000, 0x31000000, 0x15000000, +0x04000000, 0xc7000000, 0x23000000, 0xc3000000, 0x18000000, 0x96000000, 0x05000000, 0x9a000000, +0x07000000, 0x12000000, 0x80000000, 0xe2000000, 0xeb000000, 0x27000000, 0xb2000000, 0x75000000, +0x09000000, 0x83000000, 0x2c000000, 0x1a000000, 0x1b000000, 0x6e000000, 0x5a000000, 0xa0000000, +0x52000000, 0x3b000000, 0xd6000000, 0xb3000000, 0x29000000, 0xe3000000, 0x2f000000, 0x84000000, +0x53000000, 0xd1000000, 0x00000000, 0xed000000, 0x20000000, 0xfc000000, 0xb1000000, 0x5b000000, +0x6a000000, 0xcb000000, 0xbe000000, 0x39000000, 0x4a000000, 0x4c000000, 0x58000000, 0xcf000000, +0xd0000000, 0xef000000, 0xaa000000, 0xfb000000, 0x43000000, 0x4d000000, 0x33000000, 0x85000000, +0x45000000, 0xf9000000, 0x02000000, 0x7f000000, 0x50000000, 0x3c000000, 0x9f000000, 0xa8000000, +0x51000000, 0xa3000000, 0x40000000, 0x8f000000, 0x92000000, 0x9d000000, 0x38000000, 0xf5000000, +0xbc000000, 0xb6000000, 0xda000000, 0x21000000, 0x10000000, 0xff000000, 0xf3000000, 0xd2000000, +0xcd000000, 0x0c000000, 0x13000000, 0xec000000, 0x5f000000, 0x97000000, 0x44000000, 0x17000000, +0xc4000000, 0xa7000000, 0x7e000000, 0x3d000000, 0x64000000, 0x5d000000, 0x19000000, 0x73000000, +0x60000000, 0x81000000, 0x4f000000, 0xdc000000, 0x22000000, 0x2a000000, 0x90000000, 0x88000000, +0x46000000, 0xee000000, 0xb8000000, 0x14000000, 0xde000000, 0x5e000000, 0x0b000000, 0xdb000000, +0xe0000000, 0x32000000, 0x3a000000, 0x0a000000, 0x49000000, 0x06000000, 0x24000000, 0x5c000000, +0xc2000000, 0xd3000000, 0xac000000, 0x62000000, 0x91000000, 0x95000000, 0xe4000000, 0x79000000, +0xe7000000, 0xc8000000, 0x37000000, 0x6d000000, 0x8d000000, 0xd5000000, 0x4e000000, 0xa9000000, +0x6c000000, 0x56000000, 0xf4000000, 0xea000000, 0x65000000, 0x7a000000, 0xae000000, 0x08000000, +0xba000000, 0x78000000, 0x25000000, 0x2e000000, 0x1c000000, 0xa6000000, 0xb4000000, 0xc6000000, +0xe8000000, 0xdd000000, 0x74000000, 0x1f000000, 0x4b000000, 0xbd000000, 0x8b000000, 0x8a000000, +0x70000000, 0x3e000000, 0xb5000000, 0x66000000, 0x48000000, 0x03000000, 0xf6000000, 0x0e000000, +0x61000000, 0x35000000, 0x57000000, 0xb9000000, 0x86000000, 0xc1000000, 0x1d000000, 0x9e000000, +0xe1000000, 0xf8000000, 0x98000000, 0x11000000, 0x69000000, 0xd9000000, 0x8e000000, 0x94000000, +0x9b000000, 0x1e000000, 0x87000000, 0xe9000000, 0xce000000, 0x55000000, 0x28000000, 0xdf000000, +0x8c000000, 0xa1000000, 0x89000000, 0x0d000000, 0xbf000000, 0xe6000000, 0x42000000, 0x68000000, +0x41000000, 0x99000000, 0x2d000000, 0x0f000000, 0xb0000000, 0x54000000, 0xbb000000, 0x16000000 +}; +#endif /* pelimac */ + +#ifndef ENCRYPT_ONLY + +static const uint32_t TD1[256] = { + 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, + 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, + 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, + 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, + 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, + 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, + 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, + 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, + 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, + 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, + 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, + 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, + 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, + 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, + 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, + 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, + 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, + 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, + 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, + 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, + 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, + 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, + 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, + 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, + 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, + 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, + 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, + 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, + 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, + 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, + 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, + 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, + 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, + 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, + 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, + 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, + 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, + 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, + 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, + 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, + 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, + 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, + 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, + 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, + 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, + 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, + 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, + 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, + 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, + 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, + 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, + 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, + 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, + 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, + 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, + 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, + 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, + 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, + 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, + 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, + 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, + 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, + 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, + 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857, +}; +static const uint32_t TD2[256] = { + 0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27, + 0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3, + 0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502, + 0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562, + 0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe, + 0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3, + 0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552, + 0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9, + 0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9, + 0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce, + 0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253, + 0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908, + 0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b, + 0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655, + 0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337, + 0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16, + 0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69, + 0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6, + 0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6, + 0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e, + 0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6, + 0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050, + 0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9, + 0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8, + 0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000, + 0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a, + 0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d, + 0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436, + 0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b, + 0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12, + 0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b, + 0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e, + 0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f, + 0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb, + 0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4, + 0xdccad731, 0x85104263, 0x22401397, 0x112084c6, + 0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729, + 0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1, + 0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9, + 0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233, + 0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4, + 0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad, + 0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e, + 0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3, + 0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25, + 0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b, + 0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f, + 0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15, + 0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0, + 0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2, + 0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7, + 0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791, + 0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496, + 0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665, + 0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b, + 0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6, + 0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13, + 0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47, + 0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7, + 0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844, + 0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3, + 0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d, + 0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456, + 0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8, +}; +static const uint32_t TD3[256] = { + 0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a, + 0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b, + 0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5, + 0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5, + 0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d, + 0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b, + 0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95, + 0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e, + 0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27, + 0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d, + 0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562, + 0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9, + 0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752, + 0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66, + 0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3, + 0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced, + 0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e, + 0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4, + 0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4, + 0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd, + 0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d, + 0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60, + 0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767, + 0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79, + 0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000, + 0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c, + 0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736, + 0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24, + 0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b, + 0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c, + 0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12, + 0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814, + 0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3, + 0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b, + 0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8, + 0x31dccad7, 0x63851042, 0x97224013, 0xc6112084, + 0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7, + 0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077, + 0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247, + 0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22, + 0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698, + 0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f, + 0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254, + 0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582, + 0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf, + 0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb, + 0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883, + 0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef, + 0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629, + 0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035, + 0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533, + 0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17, + 0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4, + 0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46, + 0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb, + 0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d, + 0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb, + 0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a, + 0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73, + 0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678, + 0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2, + 0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff, + 0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064, + 0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0, +}; + +static const uint32_t Tks0[] = { +0x00000000, 0x0e090d0b, 0x1c121a16, 0x121b171d, 0x3824342c, 0x362d3927, 0x24362e3a, 0x2a3f2331, +0x70486858, 0x7e416553, 0x6c5a724e, 0x62537f45, 0x486c5c74, 0x4665517f, 0x547e4662, 0x5a774b69, +0xe090d0b0, 0xee99ddbb, 0xfc82caa6, 0xf28bc7ad, 0xd8b4e49c, 0xd6bde997, 0xc4a6fe8a, 0xcaaff381, +0x90d8b8e8, 0x9ed1b5e3, 0x8ccaa2fe, 0x82c3aff5, 0xa8fc8cc4, 0xa6f581cf, 0xb4ee96d2, 0xbae79bd9, +0xdb3bbb7b, 0xd532b670, 0xc729a16d, 0xc920ac66, 0xe31f8f57, 0xed16825c, 0xff0d9541, 0xf104984a, +0xab73d323, 0xa57ade28, 0xb761c935, 0xb968c43e, 0x9357e70f, 0x9d5eea04, 0x8f45fd19, 0x814cf012, +0x3bab6bcb, 0x35a266c0, 0x27b971dd, 0x29b07cd6, 0x038f5fe7, 0x0d8652ec, 0x1f9d45f1, 0x119448fa, +0x4be30393, 0x45ea0e98, 0x57f11985, 0x59f8148e, 0x73c737bf, 0x7dce3ab4, 0x6fd52da9, 0x61dc20a2, +0xad766df6, 0xa37f60fd, 0xb16477e0, 0xbf6d7aeb, 0x955259da, 0x9b5b54d1, 0x894043cc, 0x87494ec7, +0xdd3e05ae, 0xd33708a5, 0xc12c1fb8, 0xcf2512b3, 0xe51a3182, 0xeb133c89, 0xf9082b94, 0xf701269f, +0x4de6bd46, 0x43efb04d, 0x51f4a750, 0x5ffdaa5b, 0x75c2896a, 0x7bcb8461, 0x69d0937c, 0x67d99e77, +0x3daed51e, 0x33a7d815, 0x21bccf08, 0x2fb5c203, 0x058ae132, 0x0b83ec39, 0x1998fb24, 0x1791f62f, +0x764dd68d, 0x7844db86, 0x6a5fcc9b, 0x6456c190, 0x4e69e2a1, 0x4060efaa, 0x527bf8b7, 0x5c72f5bc, +0x0605bed5, 0x080cb3de, 0x1a17a4c3, 0x141ea9c8, 0x3e218af9, 0x302887f2, 0x223390ef, 0x2c3a9de4, +0x96dd063d, 0x98d40b36, 0x8acf1c2b, 0x84c61120, 0xaef93211, 0xa0f03f1a, 0xb2eb2807, 0xbce2250c, +0xe6956e65, 0xe89c636e, 0xfa877473, 0xf48e7978, 0xdeb15a49, 0xd0b85742, 0xc2a3405f, 0xccaa4d54, +0x41ecdaf7, 0x4fe5d7fc, 0x5dfec0e1, 0x53f7cdea, 0x79c8eedb, 0x77c1e3d0, 0x65daf4cd, 0x6bd3f9c6, +0x31a4b2af, 0x3fadbfa4, 0x2db6a8b9, 0x23bfa5b2, 0x09808683, 0x07898b88, 0x15929c95, 0x1b9b919e, +0xa17c0a47, 0xaf75074c, 0xbd6e1051, 0xb3671d5a, 0x99583e6b, 0x97513360, 0x854a247d, 0x8b432976, +0xd134621f, 0xdf3d6f14, 0xcd267809, 0xc32f7502, 0xe9105633, 0xe7195b38, 0xf5024c25, 0xfb0b412e, +0x9ad7618c, 0x94de6c87, 0x86c57b9a, 0x88cc7691, 0xa2f355a0, 0xacfa58ab, 0xbee14fb6, 0xb0e842bd, +0xea9f09d4, 0xe49604df, 0xf68d13c2, 0xf8841ec9, 0xd2bb3df8, 0xdcb230f3, 0xcea927ee, 0xc0a02ae5, +0x7a47b13c, 0x744ebc37, 0x6655ab2a, 0x685ca621, 0x42638510, 0x4c6a881b, 0x5e719f06, 0x5078920d, +0x0a0fd964, 0x0406d46f, 0x161dc372, 0x1814ce79, 0x322bed48, 0x3c22e043, 0x2e39f75e, 0x2030fa55, +0xec9ab701, 0xe293ba0a, 0xf088ad17, 0xfe81a01c, 0xd4be832d, 0xdab78e26, 0xc8ac993b, 0xc6a59430, +0x9cd2df59, 0x92dbd252, 0x80c0c54f, 0x8ec9c844, 0xa4f6eb75, 0xaaffe67e, 0xb8e4f163, 0xb6edfc68, +0x0c0a67b1, 0x02036aba, 0x10187da7, 0x1e1170ac, 0x342e539d, 0x3a275e96, 0x283c498b, 0x26354480, +0x7c420fe9, 0x724b02e2, 0x605015ff, 0x6e5918f4, 0x44663bc5, 0x4a6f36ce, 0x587421d3, 0x567d2cd8, +0x37a10c7a, 0x39a80171, 0x2bb3166c, 0x25ba1b67, 0x0f853856, 0x018c355d, 0x13972240, 0x1d9e2f4b, +0x47e96422, 0x49e06929, 0x5bfb7e34, 0x55f2733f, 0x7fcd500e, 0x71c45d05, 0x63df4a18, 0x6dd64713, +0xd731dcca, 0xd938d1c1, 0xcb23c6dc, 0xc52acbd7, 0xef15e8e6, 0xe11ce5ed, 0xf307f2f0, 0xfd0efffb, +0xa779b492, 0xa970b999, 0xbb6bae84, 0xb562a38f, 0x9f5d80be, 0x91548db5, 0x834f9aa8, 0x8d4697a3 +}; + +static const uint32_t Tks1[] = { +0x00000000, 0x0b0e090d, 0x161c121a, 0x1d121b17, 0x2c382434, 0x27362d39, 0x3a24362e, 0x312a3f23, +0x58704868, 0x537e4165, 0x4e6c5a72, 0x4562537f, 0x74486c5c, 0x7f466551, 0x62547e46, 0x695a774b, +0xb0e090d0, 0xbbee99dd, 0xa6fc82ca, 0xadf28bc7, 0x9cd8b4e4, 0x97d6bde9, 0x8ac4a6fe, 0x81caaff3, +0xe890d8b8, 0xe39ed1b5, 0xfe8ccaa2, 0xf582c3af, 0xc4a8fc8c, 0xcfa6f581, 0xd2b4ee96, 0xd9bae79b, +0x7bdb3bbb, 0x70d532b6, 0x6dc729a1, 0x66c920ac, 0x57e31f8f, 0x5ced1682, 0x41ff0d95, 0x4af10498, +0x23ab73d3, 0x28a57ade, 0x35b761c9, 0x3eb968c4, 0x0f9357e7, 0x049d5eea, 0x198f45fd, 0x12814cf0, +0xcb3bab6b, 0xc035a266, 0xdd27b971, 0xd629b07c, 0xe7038f5f, 0xec0d8652, 0xf11f9d45, 0xfa119448, +0x934be303, 0x9845ea0e, 0x8557f119, 0x8e59f814, 0xbf73c737, 0xb47dce3a, 0xa96fd52d, 0xa261dc20, +0xf6ad766d, 0xfda37f60, 0xe0b16477, 0xebbf6d7a, 0xda955259, 0xd19b5b54, 0xcc894043, 0xc787494e, +0xaedd3e05, 0xa5d33708, 0xb8c12c1f, 0xb3cf2512, 0x82e51a31, 0x89eb133c, 0x94f9082b, 0x9ff70126, +0x464de6bd, 0x4d43efb0, 0x5051f4a7, 0x5b5ffdaa, 0x6a75c289, 0x617bcb84, 0x7c69d093, 0x7767d99e, +0x1e3daed5, 0x1533a7d8, 0x0821bccf, 0x032fb5c2, 0x32058ae1, 0x390b83ec, 0x241998fb, 0x2f1791f6, +0x8d764dd6, 0x867844db, 0x9b6a5fcc, 0x906456c1, 0xa14e69e2, 0xaa4060ef, 0xb7527bf8, 0xbc5c72f5, +0xd50605be, 0xde080cb3, 0xc31a17a4, 0xc8141ea9, 0xf93e218a, 0xf2302887, 0xef223390, 0xe42c3a9d, +0x3d96dd06, 0x3698d40b, 0x2b8acf1c, 0x2084c611, 0x11aef932, 0x1aa0f03f, 0x07b2eb28, 0x0cbce225, +0x65e6956e, 0x6ee89c63, 0x73fa8774, 0x78f48e79, 0x49deb15a, 0x42d0b857, 0x5fc2a340, 0x54ccaa4d, +0xf741ecda, 0xfc4fe5d7, 0xe15dfec0, 0xea53f7cd, 0xdb79c8ee, 0xd077c1e3, 0xcd65daf4, 0xc66bd3f9, +0xaf31a4b2, 0xa43fadbf, 0xb92db6a8, 0xb223bfa5, 0x83098086, 0x8807898b, 0x9515929c, 0x9e1b9b91, +0x47a17c0a, 0x4caf7507, 0x51bd6e10, 0x5ab3671d, 0x6b99583e, 0x60975133, 0x7d854a24, 0x768b4329, +0x1fd13462, 0x14df3d6f, 0x09cd2678, 0x02c32f75, 0x33e91056, 0x38e7195b, 0x25f5024c, 0x2efb0b41, +0x8c9ad761, 0x8794de6c, 0x9a86c57b, 0x9188cc76, 0xa0a2f355, 0xabacfa58, 0xb6bee14f, 0xbdb0e842, +0xd4ea9f09, 0xdfe49604, 0xc2f68d13, 0xc9f8841e, 0xf8d2bb3d, 0xf3dcb230, 0xeecea927, 0xe5c0a02a, +0x3c7a47b1, 0x37744ebc, 0x2a6655ab, 0x21685ca6, 0x10426385, 0x1b4c6a88, 0x065e719f, 0x0d507892, +0x640a0fd9, 0x6f0406d4, 0x72161dc3, 0x791814ce, 0x48322bed, 0x433c22e0, 0x5e2e39f7, 0x552030fa, +0x01ec9ab7, 0x0ae293ba, 0x17f088ad, 0x1cfe81a0, 0x2dd4be83, 0x26dab78e, 0x3bc8ac99, 0x30c6a594, +0x599cd2df, 0x5292dbd2, 0x4f80c0c5, 0x448ec9c8, 0x75a4f6eb, 0x7eaaffe6, 0x63b8e4f1, 0x68b6edfc, +0xb10c0a67, 0xba02036a, 0xa710187d, 0xac1e1170, 0x9d342e53, 0x963a275e, 0x8b283c49, 0x80263544, +0xe97c420f, 0xe2724b02, 0xff605015, 0xf46e5918, 0xc544663b, 0xce4a6f36, 0xd3587421, 0xd8567d2c, +0x7a37a10c, 0x7139a801, 0x6c2bb316, 0x6725ba1b, 0x560f8538, 0x5d018c35, 0x40139722, 0x4b1d9e2f, +0x2247e964, 0x2949e069, 0x345bfb7e, 0x3f55f273, 0x0e7fcd50, 0x0571c45d, 0x1863df4a, 0x136dd647, +0xcad731dc, 0xc1d938d1, 0xdccb23c6, 0xd7c52acb, 0xe6ef15e8, 0xede11ce5, 0xf0f307f2, 0xfbfd0eff, +0x92a779b4, 0x99a970b9, 0x84bb6bae, 0x8fb562a3, 0xbe9f5d80, 0xb591548d, 0xa8834f9a, 0xa38d4697 +}; + +static const uint32_t Tks2[] = { +0x00000000, 0x0d0b0e09, 0x1a161c12, 0x171d121b, 0x342c3824, 0x3927362d, 0x2e3a2436, 0x23312a3f, +0x68587048, 0x65537e41, 0x724e6c5a, 0x7f456253, 0x5c74486c, 0x517f4665, 0x4662547e, 0x4b695a77, +0xd0b0e090, 0xddbbee99, 0xcaa6fc82, 0xc7adf28b, 0xe49cd8b4, 0xe997d6bd, 0xfe8ac4a6, 0xf381caaf, +0xb8e890d8, 0xb5e39ed1, 0xa2fe8cca, 0xaff582c3, 0x8cc4a8fc, 0x81cfa6f5, 0x96d2b4ee, 0x9bd9bae7, +0xbb7bdb3b, 0xb670d532, 0xa16dc729, 0xac66c920, 0x8f57e31f, 0x825ced16, 0x9541ff0d, 0x984af104, +0xd323ab73, 0xde28a57a, 0xc935b761, 0xc43eb968, 0xe70f9357, 0xea049d5e, 0xfd198f45, 0xf012814c, +0x6bcb3bab, 0x66c035a2, 0x71dd27b9, 0x7cd629b0, 0x5fe7038f, 0x52ec0d86, 0x45f11f9d, 0x48fa1194, +0x03934be3, 0x0e9845ea, 0x198557f1, 0x148e59f8, 0x37bf73c7, 0x3ab47dce, 0x2da96fd5, 0x20a261dc, +0x6df6ad76, 0x60fda37f, 0x77e0b164, 0x7aebbf6d, 0x59da9552, 0x54d19b5b, 0x43cc8940, 0x4ec78749, +0x05aedd3e, 0x08a5d337, 0x1fb8c12c, 0x12b3cf25, 0x3182e51a, 0x3c89eb13, 0x2b94f908, 0x269ff701, +0xbd464de6, 0xb04d43ef, 0xa75051f4, 0xaa5b5ffd, 0x896a75c2, 0x84617bcb, 0x937c69d0, 0x9e7767d9, +0xd51e3dae, 0xd81533a7, 0xcf0821bc, 0xc2032fb5, 0xe132058a, 0xec390b83, 0xfb241998, 0xf62f1791, +0xd68d764d, 0xdb867844, 0xcc9b6a5f, 0xc1906456, 0xe2a14e69, 0xefaa4060, 0xf8b7527b, 0xf5bc5c72, +0xbed50605, 0xb3de080c, 0xa4c31a17, 0xa9c8141e, 0x8af93e21, 0x87f23028, 0x90ef2233, 0x9de42c3a, +0x063d96dd, 0x0b3698d4, 0x1c2b8acf, 0x112084c6, 0x3211aef9, 0x3f1aa0f0, 0x2807b2eb, 0x250cbce2, +0x6e65e695, 0x636ee89c, 0x7473fa87, 0x7978f48e, 0x5a49deb1, 0x5742d0b8, 0x405fc2a3, 0x4d54ccaa, +0xdaf741ec, 0xd7fc4fe5, 0xc0e15dfe, 0xcdea53f7, 0xeedb79c8, 0xe3d077c1, 0xf4cd65da, 0xf9c66bd3, +0xb2af31a4, 0xbfa43fad, 0xa8b92db6, 0xa5b223bf, 0x86830980, 0x8b880789, 0x9c951592, 0x919e1b9b, +0x0a47a17c, 0x074caf75, 0x1051bd6e, 0x1d5ab367, 0x3e6b9958, 0x33609751, 0x247d854a, 0x29768b43, +0x621fd134, 0x6f14df3d, 0x7809cd26, 0x7502c32f, 0x5633e910, 0x5b38e719, 0x4c25f502, 0x412efb0b, +0x618c9ad7, 0x6c8794de, 0x7b9a86c5, 0x769188cc, 0x55a0a2f3, 0x58abacfa, 0x4fb6bee1, 0x42bdb0e8, +0x09d4ea9f, 0x04dfe496, 0x13c2f68d, 0x1ec9f884, 0x3df8d2bb, 0x30f3dcb2, 0x27eecea9, 0x2ae5c0a0, +0xb13c7a47, 0xbc37744e, 0xab2a6655, 0xa621685c, 0x85104263, 0x881b4c6a, 0x9f065e71, 0x920d5078, +0xd9640a0f, 0xd46f0406, 0xc372161d, 0xce791814, 0xed48322b, 0xe0433c22, 0xf75e2e39, 0xfa552030, +0xb701ec9a, 0xba0ae293, 0xad17f088, 0xa01cfe81, 0x832dd4be, 0x8e26dab7, 0x993bc8ac, 0x9430c6a5, +0xdf599cd2, 0xd25292db, 0xc54f80c0, 0xc8448ec9, 0xeb75a4f6, 0xe67eaaff, 0xf163b8e4, 0xfc68b6ed, +0x67b10c0a, 0x6aba0203, 0x7da71018, 0x70ac1e11, 0x539d342e, 0x5e963a27, 0x498b283c, 0x44802635, +0x0fe97c42, 0x02e2724b, 0x15ff6050, 0x18f46e59, 0x3bc54466, 0x36ce4a6f, 0x21d35874, 0x2cd8567d, +0x0c7a37a1, 0x017139a8, 0x166c2bb3, 0x1b6725ba, 0x38560f85, 0x355d018c, 0x22401397, 0x2f4b1d9e, +0x642247e9, 0x692949e0, 0x7e345bfb, 0x733f55f2, 0x500e7fcd, 0x5d0571c4, 0x4a1863df, 0x47136dd6, +0xdccad731, 0xd1c1d938, 0xc6dccb23, 0xcbd7c52a, 0xe8e6ef15, 0xe5ede11c, 0xf2f0f307, 0xfffbfd0e, +0xb492a779, 0xb999a970, 0xae84bb6b, 0xa38fb562, 0x80be9f5d, 0x8db59154, 0x9aa8834f, 0x97a38d46 +}; + +static const uint32_t Tks3[] = { +0x00000000, 0x090d0b0e, 0x121a161c, 0x1b171d12, 0x24342c38, 0x2d392736, 0x362e3a24, 0x3f23312a, +0x48685870, 0x4165537e, 0x5a724e6c, 0x537f4562, 0x6c5c7448, 0x65517f46, 0x7e466254, 0x774b695a, +0x90d0b0e0, 0x99ddbbee, 0x82caa6fc, 0x8bc7adf2, 0xb4e49cd8, 0xbde997d6, 0xa6fe8ac4, 0xaff381ca, +0xd8b8e890, 0xd1b5e39e, 0xcaa2fe8c, 0xc3aff582, 0xfc8cc4a8, 0xf581cfa6, 0xee96d2b4, 0xe79bd9ba, +0x3bbb7bdb, 0x32b670d5, 0x29a16dc7, 0x20ac66c9, 0x1f8f57e3, 0x16825ced, 0x0d9541ff, 0x04984af1, +0x73d323ab, 0x7ade28a5, 0x61c935b7, 0x68c43eb9, 0x57e70f93, 0x5eea049d, 0x45fd198f, 0x4cf01281, +0xab6bcb3b, 0xa266c035, 0xb971dd27, 0xb07cd629, 0x8f5fe703, 0x8652ec0d, 0x9d45f11f, 0x9448fa11, +0xe303934b, 0xea0e9845, 0xf1198557, 0xf8148e59, 0xc737bf73, 0xce3ab47d, 0xd52da96f, 0xdc20a261, +0x766df6ad, 0x7f60fda3, 0x6477e0b1, 0x6d7aebbf, 0x5259da95, 0x5b54d19b, 0x4043cc89, 0x494ec787, +0x3e05aedd, 0x3708a5d3, 0x2c1fb8c1, 0x2512b3cf, 0x1a3182e5, 0x133c89eb, 0x082b94f9, 0x01269ff7, +0xe6bd464d, 0xefb04d43, 0xf4a75051, 0xfdaa5b5f, 0xc2896a75, 0xcb84617b, 0xd0937c69, 0xd99e7767, +0xaed51e3d, 0xa7d81533, 0xbccf0821, 0xb5c2032f, 0x8ae13205, 0x83ec390b, 0x98fb2419, 0x91f62f17, +0x4dd68d76, 0x44db8678, 0x5fcc9b6a, 0x56c19064, 0x69e2a14e, 0x60efaa40, 0x7bf8b752, 0x72f5bc5c, +0x05bed506, 0x0cb3de08, 0x17a4c31a, 0x1ea9c814, 0x218af93e, 0x2887f230, 0x3390ef22, 0x3a9de42c, +0xdd063d96, 0xd40b3698, 0xcf1c2b8a, 0xc6112084, 0xf93211ae, 0xf03f1aa0, 0xeb2807b2, 0xe2250cbc, +0x956e65e6, 0x9c636ee8, 0x877473fa, 0x8e7978f4, 0xb15a49de, 0xb85742d0, 0xa3405fc2, 0xaa4d54cc, +0xecdaf741, 0xe5d7fc4f, 0xfec0e15d, 0xf7cdea53, 0xc8eedb79, 0xc1e3d077, 0xdaf4cd65, 0xd3f9c66b, +0xa4b2af31, 0xadbfa43f, 0xb6a8b92d, 0xbfa5b223, 0x80868309, 0x898b8807, 0x929c9515, 0x9b919e1b, +0x7c0a47a1, 0x75074caf, 0x6e1051bd, 0x671d5ab3, 0x583e6b99, 0x51336097, 0x4a247d85, 0x4329768b, +0x34621fd1, 0x3d6f14df, 0x267809cd, 0x2f7502c3, 0x105633e9, 0x195b38e7, 0x024c25f5, 0x0b412efb, +0xd7618c9a, 0xde6c8794, 0xc57b9a86, 0xcc769188, 0xf355a0a2, 0xfa58abac, 0xe14fb6be, 0xe842bdb0, +0x9f09d4ea, 0x9604dfe4, 0x8d13c2f6, 0x841ec9f8, 0xbb3df8d2, 0xb230f3dc, 0xa927eece, 0xa02ae5c0, +0x47b13c7a, 0x4ebc3774, 0x55ab2a66, 0x5ca62168, 0x63851042, 0x6a881b4c, 0x719f065e, 0x78920d50, +0x0fd9640a, 0x06d46f04, 0x1dc37216, 0x14ce7918, 0x2bed4832, 0x22e0433c, 0x39f75e2e, 0x30fa5520, +0x9ab701ec, 0x93ba0ae2, 0x88ad17f0, 0x81a01cfe, 0xbe832dd4, 0xb78e26da, 0xac993bc8, 0xa59430c6, +0xd2df599c, 0xdbd25292, 0xc0c54f80, 0xc9c8448e, 0xf6eb75a4, 0xffe67eaa, 0xe4f163b8, 0xedfc68b6, +0x0a67b10c, 0x036aba02, 0x187da710, 0x1170ac1e, 0x2e539d34, 0x275e963a, 0x3c498b28, 0x35448026, +0x420fe97c, 0x4b02e272, 0x5015ff60, 0x5918f46e, 0x663bc544, 0x6f36ce4a, 0x7421d358, 0x7d2cd856, +0xa10c7a37, 0xa8017139, 0xb3166c2b, 0xba1b6725, 0x8538560f, 0x8c355d01, 0x97224013, 0x9e2f4b1d, +0xe9642247, 0xe0692949, 0xfb7e345b, 0xf2733f55, 0xcd500e7f, 0xc45d0571, 0xdf4a1863, 0xd647136d, +0x31dccad7, 0x38d1c1d9, 0x23c6dccb, 0x2acbd7c5, 0x15e8e6ef, 0x1ce5ede1, 0x07f2f0f3, 0x0efffbfd, +0x79b492a7, 0x70b999a9, 0x6bae84bb, 0x62a38fb5, 0x5d80be9f, 0x548db591, 0x4f9aa883, 0x4697a38d +}; + +#endif /* ENCRYPT_ONLY */ + +#endif /* SMALL CODE */ + +static const uint32_t rcon[] = { + 0x01000000, 0x02000000, 0x04000000, 0x08000000, + 0x10000000, 0x20000000, 0x40000000, 0x80000000, + 0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */ +}; diff --git a/osfmk/corecrypto/ccaes/src/ccaes_ltc_ecb_encrypt_mode.c b/osfmk/corecrypto/ccaes/src/ccaes_ltc_ecb_encrypt_mode.c new file mode 100644 index 000000000..0772f6861 --- /dev/null +++ b/osfmk/corecrypto/ccaes/src/ccaes_ltc_ecb_encrypt_mode.c @@ -0,0 +1,421 @@ +/* + * ccaes_ltc_ecb_encrypt_mode.c + * corecrypto + * + * Created on 12/12/2010 + * + * Copyright (c) 2010,2011,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + * Parts of this code adapted from LibTomCrypt + * + * LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtom.org + */ + + +#include +#include + +typedef struct ltc_rijndael_key { + uint32_t eK[60], dK[60]; + int Nr; +} ltc_rijndael_keysched; + +#include "aes_tab.c" + +static uint32_t setup_mix(uint32_t temp) +{ + return (Te4_3[cc_byte(temp, 2)]) ^ + (Te4_2[cc_byte(temp, 1)]) ^ + (Te4_1[cc_byte(temp, 0)]) ^ + (Te4_0[cc_byte(temp, 3)]); +} + +/*! + Initialize the AES (Rijndael) block cipher + @param key The symmetric key you wish to pass + @param keylen The key length in bytes + @param num_rounds The number of rounds desired (0 for default) + @param skey The key in as scheduled by this function. + @return CRYPT_OK if successful + */ +static int ccaes_ltc_init(const unsigned char *key, int keylen, int num_rounds, + ccecb_ctx *skey) +{ + int i, j; + uint32_t temp, *rk; +#ifndef ENCRYPT_ONLY + uint32_t *rrk; +#endif + ltc_rijndael_keysched *rijndael; + + rijndael = (ltc_rijndael_keysched *)skey; + + if (keylen != 16 && keylen != 24 && keylen != 32) { + return -1; //CRYPT_INVALID_KEYSIZE; + } + + if (num_rounds != 0 && num_rounds != (10 + ((keylen/8)-2)*2)) { + return -1; //CRYPT_INVALID_ROUNDS; + } + + rijndael->Nr = 10 + ((keylen/8)-2)*2; + + /* setup the forward key */ + i = 0; + rk = rijndael->eK; + CC_LOAD32_BE(rk[0], key ); + CC_LOAD32_BE(rk[1], key + 4); + CC_LOAD32_BE(rk[2], key + 8); + CC_LOAD32_BE(rk[3], key + 12); + if (keylen == 16) { + j = 44; + for (;;) { + temp = rk[3]; + rk[4] = rk[0] ^ setup_mix(temp) ^ rcon[i]; + rk[5] = rk[1] ^ rk[4]; + rk[6] = rk[2] ^ rk[5]; + rk[7] = rk[3] ^ rk[6]; + if (++i == 10) { + break; + } + rk += 4; + } + } else if (keylen == 24) { + j = 52; + CC_LOAD32_BE(rk[4], key + 16); + CC_LOAD32_BE(rk[5], key + 20); + for (;;) { +#ifdef _MSC_VER + temp = rijndael->eK[rk - rijndael->eK + 5]; +#else + temp = rk[5]; +#endif + rk[ 6] = rk[ 0] ^ setup_mix(temp) ^ rcon[i]; + rk[ 7] = rk[ 1] ^ rk[ 6]; + rk[ 8] = rk[ 2] ^ rk[ 7]; + rk[ 9] = rk[ 3] ^ rk[ 8]; + if (++i == 8) { + break; + } + rk[10] = rk[ 4] ^ rk[ 9]; + rk[11] = rk[ 5] ^ rk[10]; + rk += 6; + } + } else if (keylen == 32) { + j = 60; + CC_LOAD32_BE(rk[4], key + 16); + CC_LOAD32_BE(rk[5], key + 20); + CC_LOAD32_BE(rk[6], key + 24); + CC_LOAD32_BE(rk[7], key + 28); + for (;;) { +#ifdef _MSC_VER + temp = rijndael->eK[rk - rijndael->eK + 7]; +#else + temp = rk[7]; +#endif + rk[ 8] = rk[ 0] ^ setup_mix(temp) ^ rcon[i]; + rk[ 9] = rk[ 1] ^ rk[ 8]; + rk[10] = rk[ 2] ^ rk[ 9]; + rk[11] = rk[ 3] ^ rk[10]; + if (++i == 7) { + break; + } + temp = rk[11]; + rk[12] = rk[ 4] ^ setup_mix(CC_RORc(temp, 8)); + rk[13] = rk[ 5] ^ rk[12]; + rk[14] = rk[ 6] ^ rk[13]; + rk[15] = rk[ 7] ^ rk[14]; + rk += 8; + } + } else { + /* this can't happen */ + return -1; //CRYPT_ERROR; + } + +#ifndef ENCRYPT_ONLY + /* setup the inverse key now */ + rk = rijndael->dK; + rrk = rijndael->eK + j - 4; + + /* apply the inverse MixColumn transform to all round keys but the first and the last: */ + /* copy first */ + *rk++ = *rrk++; + *rk++ = *rrk++; + *rk++ = *rrk++; + *rk = *rrk; + rk -= 3; rrk -= 3; + + for (i = 1; i < rijndael->Nr; i++) { + rrk -= 4; + rk += 4; +#ifdef LTC_SMALL_CODE + temp = rrk[0]; + rk[0] = setup_mix2(temp); + temp = rrk[1]; + rk[1] = setup_mix2(temp); + temp = rrk[2]; + rk[2] = setup_mix2(temp); + temp = rrk[3]; + rk[3] = setup_mix2(temp); +#else + temp = rrk[0]; + rk[0] = + Tks0[cc_byte(temp, 3)] ^ + Tks1[cc_byte(temp, 2)] ^ + Tks2[cc_byte(temp, 1)] ^ + Tks3[cc_byte(temp, 0)]; + temp = rrk[1]; + rk[1] = + Tks0[cc_byte(temp, 3)] ^ + Tks1[cc_byte(temp, 2)] ^ + Tks2[cc_byte(temp, 1)] ^ + Tks3[cc_byte(temp, 0)]; + temp = rrk[2]; + rk[2] = + Tks0[cc_byte(temp, 3)] ^ + Tks1[cc_byte(temp, 2)] ^ + Tks2[cc_byte(temp, 1)] ^ + Tks3[cc_byte(temp, 0)]; + temp = rrk[3]; + rk[3] = + Tks0[cc_byte(temp, 3)] ^ + Tks1[cc_byte(temp, 2)] ^ + Tks2[cc_byte(temp, 1)] ^ + Tks3[cc_byte(temp, 0)]; +#endif + + } + + /* copy last */ + rrk -= 4; + rk += 4; + *rk++ = *rrk++; + *rk++ = *rrk++; + *rk++ = *rrk++; + *rk = *rrk; +#endif /* ENCRYPT_ONLY */ + + return 0; //CRYPT_OK; +} + +static int ccaes_ecb_encrypt_init(const struct ccmode_ecb *ecb CC_UNUSED, ccecb_ctx *key, + size_t rawkey_len, const void *rawkey) { + return ccaes_ltc_init(rawkey, (int)rawkey_len, 0, key); +} + +static void ccaes_ltc_ecb_encrypt(const ccecb_ctx *skey, const unsigned char *pt, + unsigned char *ct) +{ + uint32_t s0, s1, s2, s3, t0, t1, t2, t3; + const uint32_t *rk; + int Nr, r; + const ltc_rijndael_keysched *rijndael; + + rijndael = (const ltc_rijndael_keysched *)skey; + + Nr = rijndael->Nr; + rk = rijndael->eK; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + CC_LOAD32_BE(s0, pt ); s0 ^= rk[0]; + CC_LOAD32_BE(s1, pt + 4); s1 ^= rk[1]; + CC_LOAD32_BE(s2, pt + 8); s2 ^= rk[2]; + CC_LOAD32_BE(s3, pt + 12); s3 ^= rk[3]; + +#ifdef LTC_SMALL_CODE + + for (r = 0; ; r++) { + rk += 4; + t0 = + Te0(cc_byte(s0, 3)) ^ + Te1(cc_byte(s1, 2)) ^ + Te2(cc_byte(s2, 1)) ^ + Te3(cc_byte(s3, 0)) ^ + rk[0]; + t1 = + Te0(cc_byte(s1, 3)) ^ + Te1(cc_byte(s2, 2)) ^ + Te2(cc_byte(s3, 1)) ^ + Te3(cc_byte(s0, 0)) ^ + rk[1]; + t2 = + Te0(cc_byte(s2, 3)) ^ + Te1(cc_byte(s3, 2)) ^ + Te2(cc_byte(s0, 1)) ^ + Te3(cc_byte(s1, 0)) ^ + rk[2]; + t3 = + Te0(cc_byte(s3, 3)) ^ + Te1(cc_byte(s0, 2)) ^ + Te2(cc_byte(s1, 1)) ^ + Te3(cc_byte(s2, 0)) ^ + rk[3]; + if (r == Nr-2) { + break; + } + s0 = t0; s1 = t1; s2 = t2; s3 = t3; + } + rk += 4; + +#else + + /* + * Nr - 1 full rounds: + */ + r = Nr >> 1; + for (;;) { + t0 = + Te0(cc_byte(s0, 3)) ^ + Te1(cc_byte(s1, 2)) ^ + Te2(cc_byte(s2, 1)) ^ + Te3(cc_byte(s3, 0)) ^ + rk[4]; + t1 = + Te0(cc_byte(s1, 3)) ^ + Te1(cc_byte(s2, 2)) ^ + Te2(cc_byte(s3, 1)) ^ + Te3(cc_byte(s0, 0)) ^ + rk[5]; + t2 = + Te0(cc_byte(s2, 3)) ^ + Te1(cc_byte(s3, 2)) ^ + Te2(cc_byte(s0, 1)) ^ + Te3(cc_byte(s1, 0)) ^ + rk[6]; + t3 = + Te0(cc_byte(s3, 3)) ^ + Te1(cc_byte(s0, 2)) ^ + Te2(cc_byte(s1, 1)) ^ + Te3(cc_byte(s2, 0)) ^ + rk[7]; + + rk += 8; + if (--r == 0) { + break; + } + + s0 = + Te0(cc_byte(t0, 3)) ^ + Te1(cc_byte(t1, 2)) ^ + Te2(cc_byte(t2, 1)) ^ + Te3(cc_byte(t3, 0)) ^ + rk[0]; + s1 = + Te0(cc_byte(t1, 3)) ^ + Te1(cc_byte(t2, 2)) ^ + Te2(cc_byte(t3, 1)) ^ + Te3(cc_byte(t0, 0)) ^ + rk[1]; + s2 = + Te0(cc_byte(t2, 3)) ^ + Te1(cc_byte(t3, 2)) ^ + Te2(cc_byte(t0, 1)) ^ + Te3(cc_byte(t1, 0)) ^ + rk[2]; + s3 = + Te0(cc_byte(t3, 3)) ^ + Te1(cc_byte(t0, 2)) ^ + Te2(cc_byte(t1, 1)) ^ + Te3(cc_byte(t2, 0)) ^ + rk[3]; + } + +#endif + + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = + (Te4_3[cc_byte(t0, 3)]) ^ + (Te4_2[cc_byte(t1, 2)]) ^ + (Te4_1[cc_byte(t2, 1)]) ^ + (Te4_0[cc_byte(t3, 0)]) ^ + rk[0]; + CC_STORE32_BE(s0, ct); + s1 = + (Te4_3[cc_byte(t1, 3)]) ^ + (Te4_2[cc_byte(t2, 2)]) ^ + (Te4_1[cc_byte(t3, 1)]) ^ + (Te4_0[cc_byte(t0, 0)]) ^ + rk[1]; + CC_STORE32_BE(s1, ct+4); + s2 = + (Te4_3[cc_byte(t2, 3)]) ^ + (Te4_2[cc_byte(t3, 2)]) ^ + (Te4_1[cc_byte(t0, 1)]) ^ + (Te4_0[cc_byte(t1, 0)]) ^ + rk[2]; + CC_STORE32_BE(s2, ct+8); + s3 = + (Te4_3[cc_byte(t3, 3)]) ^ + (Te4_2[cc_byte(t0, 2)]) ^ + (Te4_1[cc_byte(t1, 1)]) ^ + (Te4_0[cc_byte(t2, 0)]) ^ + rk[3]; + CC_STORE32_BE(s3, ct+12); +} + +static int ccaes_ecb_encrypt(const ccecb_ctx *key, size_t nblocks, + const void *in, void *out) { + if (nblocks) { + const unsigned char *p = in; + unsigned char *c = out; + for (;;) { + ccaes_ltc_ecb_encrypt(key, p, c); + if (--nblocks) { + p += CCAES_BLOCK_SIZE; + c += CCAES_BLOCK_SIZE; + } else { + break; + } + } + } + + return 0; +} + +const struct ccmode_ecb ccaes_ltc_ecb_encrypt_mode = { + .size = sizeof(ltc_rijndael_keysched), + .block_size = CCAES_BLOCK_SIZE, + .init = ccaes_ecb_encrypt_init, + .ecb = ccaes_ecb_encrypt, +}; diff --git a/osfmk/corecrypto/ccaes/src/ccaes_private_types.h b/osfmk/corecrypto/ccaes/src/ccaes_private_types.h new file mode 100644 index 000000000..7a30fad3e --- /dev/null +++ b/osfmk/corecrypto/ccaes/src/ccaes_private_types.h @@ -0,0 +1,48 @@ +/* + * ccaes_private_types.h + * corecrypto + * + * Created on 02/15/2012 + * + * Copyright (c) 2012,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _CORECRYPTO_CCAES_PRIVATE_TYPES_H_ +#define _CORECRYPTO_CCAES_PRIVATE_TYPES_H_ + +#include +#include + +typedef struct ltc_rijndael_key +{ + uint32_t eK[60], dK[60]; + int Nr; +} ltc_rijndael_keysched; + + +#endif // _CORECRYPTO_CCAES_PRIVATE_TYPES_H_ diff --git a/osfmk/corecrypto/ccdbrg/src/ccdrbg_nisthmac.c b/osfmk/corecrypto/ccdbrg/src/ccdrbg_nisthmac.c index 2ccf60aed..47486191c 100644 --- a/osfmk/corecrypto/ccdbrg/src/ccdrbg_nisthmac.c +++ b/osfmk/corecrypto/ccdbrg/src/ccdrbg_nisthmac.c @@ -6,13 +6,36 @@ * * Copyright (c) 2014,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include #include #include #include -#include #include // Test vectors at: @@ -22,16 +45,16 @@ /* This HMAC DBRG is described in: - + SP 800-90 A Rev. 1 (2nd Draft) DRAFT Recommendation for Random Number Generation Using Deterministic Random Bit Generators April 2014 - - + + See in particular - 10.1.2 HMAC_DRBG (p 45) - B.2 HMAC_DRBGExample (p 83) - + We support maximum security strength of 256 bits Note that the example in B.2 is very limited, refer to §10.1.2 for more */ @@ -98,6 +121,8 @@ struct ccdrbg_nisthmac_state { #if DRBG_NISTHMAC_DEBUG +#include "cc_debug.h" + static void dumpState(const char *label, struct ccdrbg_nisthmac_state *state) { //cc_print(label, state->vsize, state->nextvptr); cc_print(label, state->vsize, state->vptr); @@ -110,7 +135,7 @@ static void done(struct ccdrbg_state *drbg); /* NIST SP 800-90A, Rev. 1 HMAC_DRBG April 2014, p 46 - + HMAC_DRBG_Update (provided_data, K, V): 1. provided_data: The data to be used. 2. K: The current value of Key. @@ -118,9 +143,9 @@ static void done(struct ccdrbg_state *drbg); Output: 1. K: The new value for Key. 2. V: The new value for V. - + HMAC_DRBG Update Process: - + 1. K = HMAC (K, V || 0x00 || provided_data). 2. V=HMAC(K,V). 3. If (provided_data = Null), then return K and V. @@ -145,13 +170,13 @@ static int hmac_dbrg_update(struct ccdrbg_state *drbg, int rc=CCDRBG_STATUS_ERROR; struct ccdrbg_nisthmac_state *state = (struct ccdrbg_nisthmac_state *)drbg; const struct ccdigest_info *di = state->custom->di; - + const unsigned char cZero = 0x00; const unsigned char cOne = 0x01; cchmac_ctx_decl(di->state_size, di->block_size, ctx); cchmac_init(di, ctx, state->keysize, state->key); - + // 1. K = HMAC (K, V || 0x00 || provided_data). cchmac_update(di, ctx, state->vsize, state->vptr); cchmac_update(di, ctx, 1, &cZero); @@ -190,7 +215,7 @@ static int hmac_dbrg_update(struct ccdrbg_state *drbg, //catastrophic error in SP 800-90A done(drbg); rc=CCDRBG_STATUS_ABORT; - cc_abort(NULL); + cc_try_abort(NULL); goto errOut; } rc=CCDRBG_STATUS_OK; @@ -207,23 +232,23 @@ static int validate_inputs(struct ccdrbg_nisthmac_state *state, int rc; const struct ccdrbg_nisthmac_custom *custom=state->custom; const struct ccdigest_info *di = custom->di; - + rc =CCDRBG_STATUS_ERROR; //buffer size checks cc_require (di->output_size<=sizeof(state->v)/2, end); //digest size too long cc_require (di->output_size<=sizeof(state->key), end); //digest size too long - + //NIST SP800 compliance checks //the following maximum checks are redundant if long is 32 bits. - + rc=CCDRBG_STATUS_PARAM_ERROR; cc_require (psLength <= CCDRBG_MAX_PSINPUT_SIZE, end); //personalization string too long cc_require (entropyLength <= CCDRBG_MAX_ENTROPY_SIZE, end); //supplied too much entropy cc_require (additionalInputLength <= CCDRBG_MAX_ADDITIONALINPUT_SIZE, end); //additional input too long cc_require (entropyLength >= MIN_REQ_ENTROPY(di), end); //supplied too litle entropy - + cc_require(di->output_size<=NH_MAX_OUTPUT_BLOCK_SIZE, end); //the requested security strength is not supported - + rc=CCDRBG_STATUS_OK; end: return rc; @@ -231,11 +256,11 @@ end: /* NIST SP 800-90A, Rev. 1 April 2014 B.2.2, p 84 - + HMAC_DRBG_Instantiate_algorithm (...): Input: bitstring (entropy_input, personalization_string). Output: bitstring (V, Key), integer reseed_counter. - + Process: 1. seed_material = entropy_input || personalization_string. 2. Set Key to outlen bits of zeros. @@ -255,21 +280,21 @@ static int hmac_dbrg_instantiate_algorithm(struct ccdrbg_state *drbg, { // TODO: The NIST code passes nonce (i.e. HMAC key) to generate, but cc interface isn't set up that way struct ccdrbg_nisthmac_state *state = (struct ccdrbg_nisthmac_state *)drbg; - + // 1. seed_material = entropy_input || nonce || personalization_string. - + // 2. Set Key to outlen bits of zeros. cc_zero(state->keysize, state->key); - + // 3. Set V to outlen/8 bytes of 0x01. CC_MEMSET(state->vptr, 0x01, state->vsize); - + // 4. (Key, V) = HMAC_DRBG_Update (seed_material, Key, V). hmac_dbrg_update(drbg, entropyLength, entropy, nonceLength, nonce, psLength, ps); - + // 5. reseed_counter = 1. state->reseed_counter = 1; - + return CCDRBG_STATUS_OK; } @@ -286,7 +311,7 @@ static int init(const struct ccdrbg_info *info, struct ccdrbg_state *drbg, struct ccdrbg_nisthmac_state *state=(struct ccdrbg_nisthmac_state *)drbg; state->bytesLeft = 0; state->custom = info->custom; //we only need to get the custom parameter from the info structure. - + int rc = validate_inputs(state , entropyLength, 0, psLength); if(rc!=CCDRBG_STATUS_OK){ //clear everything if cannot initialize. The idea is that if the caller doesn't check the output of init() and init() fails, @@ -303,7 +328,7 @@ static int init(const struct ccdrbg_info *info, struct ccdrbg_state *drbg, // 7. (V, Key, reseed_counter) = HMAC_DRBG_Instantiate_algorithm (entropy_input, personalization_string). hmac_dbrg_instantiate_algorithm(drbg, entropyLength, entropy, nonceLength, nonce, psLength, ps); - + #if DRBG_NISTHMAC_DEBUG dumpState("Init: ", state); #endif @@ -317,17 +342,17 @@ static int init(const struct ccdrbg_info *info, struct ccdrbg_state *drbg, The reseeding of an HMAC_DRBG instantiation requires a call to the Reseed_function specified in Section 9.2. Process step 6 of that function calls the reseed algorithm specified in this section. The values for min_length are provided in Table 2 of Section 10.1. - + The reseed algorithm: Let HMAC_DRBG_Update be the function specified in Section 10.1.2.2. The following process or its equivalent shall be used as the reseed algorithm for this DRBG mechanism (see step 6 of the reseed process in Section 9.2): - + HMAC_DRBG_Reseed_algorithm (working_state, entropy_input, additional_input): 1. working_state: The current values for V, Key and reseed_counter (see Section 10.1.2.1). 2. entropy_input: The string of bits obtained from the source of entropy input. 3. additional_input: The additional input string received from the consuming application. Note that the length of the additional_input string may be zero. - + Output: 1. new_working_state: The new values for V, Key and reseed_counter. HMAC_DRBG Reseed Process: 1. seed_material = entropy_input || additional_input. @@ -340,14 +365,14 @@ reseed(struct ccdrbg_state *drbg, size_t entropyLength, const void *entropy, size_t additionalLength, const void *additional) { - + struct ccdrbg_nisthmac_state *state = (struct ccdrbg_nisthmac_state *)drbg; int rc = validate_inputs(state, entropyLength, additionalLength, 0); if(rc!=CCDRBG_STATUS_OK) return rc; - + int rx = hmac_dbrg_update(drbg, entropyLength, entropy, additionalLength, additional, 0, NULL); state->reseed_counter = 1; - + #if DRBG_NISTHMAC_DEBUG dumpState("Reseed: ", state); #endif @@ -358,7 +383,7 @@ reseed(struct ccdrbg_state *drbg, HMAC_DRBG_Generate_algorithm: Input: bitstring (V, Key), integer (reseed_counter, requested_number_of_bits). Output: string status, bitstring (pseudorandom_bits, V, Key), integer reseed_counter. - + Process: 1. If (reseed_counter ≥ 10,000), then Return (“Reseed required”, Null, V, Key, reseed_counter). 2. temp = Null. @@ -375,17 +400,17 @@ static int validate_gen_params(uint64_t reseed_counter, size_t dataOutLength, s { int rc=CCDRBG_STATUS_PARAM_ERROR; - + // Zero byte in one request is a valid use-case (21208820) cc_require (dataOutLength <= CCDRBG_MAX_REQUEST_SIZE, end); //Requested too many bytes in one request cc_require (additionalLength<=CCDRBG_MAX_ADDITIONALINPUT_SIZE, end); //Additional input too long - + // 1. If (reseed_counter > 2^^48), then Return (“Reseed required”, Null, V, Key, reseed_counter). rc = CCDRBG_STATUS_NEED_RESEED; cc_require (reseed_counter <= CCDRBG_RESEED_INTERVAL, end); //Reseed required - + rc=CCDRBG_STATUS_OK; - + end: return rc; } @@ -396,14 +421,14 @@ static int generate(struct ccdrbg_state *drbg, size_t dataOutLength, void *dataO struct ccdrbg_nisthmac_state *state = (struct ccdrbg_nisthmac_state *)drbg; const struct ccdrbg_nisthmac_custom *custom = state->custom; const struct ccdigest_info *di = custom->di; - + int rc = validate_gen_params(state->reseed_counter, dataOutLength, additional==NULL?0:additionalLength); if(rc!=CCDRBG_STATUS_OK) return rc; - + // 2. If additional_input ≠ Null, then (Key, V) = HMAC_DRBG_Update (additional_input, Key, V). if (additional && additionalLength) hmac_dbrg_update(drbg, additionalLength, additional, 0, NULL, 0, NULL); - + // hmac_dbrg_generate_algorithm char *outPtr = (char *) dataOut; while (dataOutLength > 0) { @@ -419,7 +444,7 @@ static int generate(struct ccdrbg_state *drbg, size_t dataOutLength, void *dataO //catastrophic error in SP 800-90A done(drbg); rc=CCDRBG_STATUS_ABORT; - cc_abort(NULL); + cc_try_abort(NULL); goto errOut; } CC_SWAP(state->nextvptr, state->vptr); @@ -437,10 +462,10 @@ static int generate(struct ccdrbg_state *drbg, size_t dataOutLength, void *dataO // 6. (Key, V) = HMAC_DRBG_Update (additional_input, Key, V). hmac_dbrg_update(drbg, additionalLength, additional, 0, NULL, 0, NULL); - + // 7. reseed_counter = reseed_counter + 1. state->reseed_counter++; - + #if DRBG_NISTHMAC_DEBUG dumpState("generate end: ", state); cc_print("generate end nxt: ", state->vsize, state->nextvptr); @@ -475,4 +500,3 @@ void ccdrbg_factory_nisthmac(struct ccdrbg_info *info, const struct ccdrbg_nisth info->done = done; info->custom = custom; }; - diff --git a/osfmk/corecrypto/ccdigest/src/ccdigest_init.c b/osfmk/corecrypto/ccdigest/src/ccdigest_init.c index f5ccb3d71..0ddab7476 100644 --- a/osfmk/corecrypto/ccdigest/src/ccdigest_init.c +++ b/osfmk/corecrypto/ccdigest/src/ccdigest_init.c @@ -6,6 +6,30 @@ * * Copyright (c) 2010,2011,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include diff --git a/osfmk/corecrypto/ccdigest/src/ccdigest_update.c b/osfmk/corecrypto/ccdigest/src/ccdigest_update.c index 4df21c38a..089928f99 100644 --- a/osfmk/corecrypto/ccdigest/src/ccdigest_update.c +++ b/osfmk/corecrypto/ccdigest/src/ccdigest_update.c @@ -6,6 +6,30 @@ * * Copyright (c) 2010,2011,2014,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include diff --git a/osfmk/corecrypto/cchmac/src/cchmac.c b/osfmk/corecrypto/cchmac/src/cchmac.c index 61f859e6e..dbd6a3454 100644 --- a/osfmk/corecrypto/cchmac/src/cchmac.c +++ b/osfmk/corecrypto/cchmac/src/cchmac.c @@ -6,13 +6,41 @@ * * Copyright (c) 2010,2011,2012,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include +#include "corecrypto/fipspost_trace.h" + void cchmac(const struct ccdigest_info *di, size_t key_len, const void *key, size_t data_len, const void *data, unsigned char *mac) { + FIPSPOST_TRACE_EVENT; + cchmac_di_decl(di, hc); cchmac_init(di, hc, key_len, key); cchmac_update(di, hc, data_len, data); diff --git a/osfmk/corecrypto/cchmac/src/cchmac_final.c b/osfmk/corecrypto/cchmac/src/cchmac_final.c index 3c189a3fa..a7bfb84c0 100644 --- a/osfmk/corecrypto/cchmac/src/cchmac_final.c +++ b/osfmk/corecrypto/cchmac/src/cchmac_final.c @@ -6,6 +6,30 @@ * * Copyright (c) 2010,2011,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include diff --git a/osfmk/corecrypto/cchmac/src/cchmac_init.c b/osfmk/corecrypto/cchmac/src/cchmac_init.c index ffda5227e..e276fe1ba 100644 --- a/osfmk/corecrypto/cchmac/src/cchmac_init.c +++ b/osfmk/corecrypto/cchmac/src/cchmac_init.c @@ -6,6 +6,30 @@ * * Copyright (c) 2010,2011,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include diff --git a/osfmk/corecrypto/cchmac/src/cchmac_update.c b/osfmk/corecrypto/cchmac/src/cchmac_update.c index 3273d4385..a2c768896 100644 --- a/osfmk/corecrypto/cchmac/src/cchmac_update.c +++ b/osfmk/corecrypto/cchmac/src/cchmac_update.c @@ -6,6 +6,30 @@ * * Copyright (c) 2010,2011,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include diff --git a/osfmk/corecrypto/ccmode/src/ccmode_ctr_crypt.c b/osfmk/corecrypto/ccmode/src/ccmode_ctr_crypt.c new file mode 100644 index 000000000..3efce7dfd --- /dev/null +++ b/osfmk/corecrypto/ccmode/src/ccmode_ctr_crypt.c @@ -0,0 +1,72 @@ +/* + * ccmode_ctr_crypt.c + * corecrypto + * + * Created on 12/17/2010 + * + * Copyright (c) 2010,2011,2012,2014,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include "ccmode_internal.h" + +int ccmode_ctr_crypt(ccctr_ctx *key, + size_t nbytes, const void *in, void *out) { + const struct ccmode_ecb *ecb = CCMODE_CTR_KEY_ECB(key); + const ccecb_ctx *ecb_key = CCMODE_CTR_KEY_ECB_KEY(key); + uint8_t *ctr = (uint8_t *)CCMODE_CTR_KEY_CTR(key); + uint8_t *pad = (uint8_t *)CCMODE_CTR_KEY_PAD(key); + size_t pad_offset = CCMODE_CTR_KEY_PAD_OFFSET(key); + const uint8_t *in_bytes = in; + // Counter is 64bit wide for cipher with block size of 64bit or more + // This is to match the assembly + const size_t counter_size=(CC_MIN(ecb->block_size,(typeof(ecb->block_size))8)); + uint8_t *out_bytes = out; + size_t n; + + while (nbytes) { + if (pad_offset == ecb->block_size) { + ecb->ecb(ecb_key, 1, ctr, pad); + pad_offset = 0; + + /* increment the big endian counter */ + inc_uint(ctr + ecb->block_size - counter_size, counter_size); + + if (nbytes==0) break; + } + + n = CC_MIN(nbytes, ecb->block_size - pad_offset); + cc_xor(n, out_bytes, in_bytes, pad + pad_offset); + nbytes -= n; + in_bytes += n; + out_bytes += n; + pad_offset += n; + } + CCMODE_CTR_KEY_PAD_OFFSET(key) = pad_offset; + + return 0; +} diff --git a/osfmk/corecrypto/ccmode/src/ccmode_ctr_init.c b/osfmk/corecrypto/ccmode/src/ccmode_ctr_init.c new file mode 100644 index 000000000..00e3ca6c1 --- /dev/null +++ b/osfmk/corecrypto/ccmode/src/ccmode_ctr_init.c @@ -0,0 +1,49 @@ +/* + * ccmode_ctr_init.c + * corecrypto + * + * Created on 12/17/2010 + * + * Copyright (c) 2010,2011,2012,2014,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include "ccmode_internal.h" + +int ccmode_ctr_init(const struct ccmode_ctr *ctr, ccctr_ctx *key, + size_t rawkey_len, const void *rawkey, + const void *iv) { + int rc; + const struct ccmode_ecb *ecb = ctr->custom; + CCMODE_CTR_KEY_ECB(key) = ecb; + + rc = ecb->init(ecb, CCMODE_CTR_KEY_ECB_KEY(key), rawkey_len, rawkey); + + ccctr_setctr(ctr, key, iv); + + return rc; +} diff --git a/osfmk/corecrypto/ccmode/src/ccmode_ctr_setctr.c b/osfmk/corecrypto/ccmode/src/ccmode_ctr_setctr.c new file mode 100644 index 000000000..6b54e209f --- /dev/null +++ b/osfmk/corecrypto/ccmode/src/ccmode_ctr_setctr.c @@ -0,0 +1,43 @@ +/* + * ccmode_ctr_setctr.c + * corecrypto + * + * Created on 2/1/2017 + * + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include "ccmode_internal.h" + +int ccmode_ctr_setctr(CC_UNUSED const struct ccmode_ctr *mode, ccctr_ctx *ctx, const void *ctr) +{ + CCMODE_CTR_KEY_PAD_OFFSET(ctx) = CCMODE_CTR_KEY_ECB(ctx)->block_size; + CC_MEMCPY(CCMODE_CTR_KEY_CTR(ctx), ctr, CCMODE_CTR_KEY_ECB(ctx)->block_size); + + return 0; +} diff --git a/osfmk/corecrypto/ccmode/src/ccmode_factory_ctr_crypt.c b/osfmk/corecrypto/ccmode/src/ccmode_factory_ctr_crypt.c new file mode 100644 index 000000000..ddac576e3 --- /dev/null +++ b/osfmk/corecrypto/ccmode/src/ccmode_factory_ctr_crypt.c @@ -0,0 +1,41 @@ +/* + * ccmode_factory_ctr_crypt.c + * corecrypto + * + * Created on 05/19/2015 + * + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include "ccmode_internal.h" + +void ccmode_factory_ctr_crypt(struct ccmode_ctr *ctr, + const struct ccmode_ecb *ecb) { + struct ccmode_ctr ctr_crypt = CCMODE_FACTORY_CTR_CRYPT(ecb); + *ctr = ctr_crypt; +} diff --git a/osfmk/corecrypto/ccmode/src/ccmode_internal.h b/osfmk/corecrypto/ccmode/src/ccmode_internal.h new file mode 100644 index 000000000..0f7f0c617 --- /dev/null +++ b/osfmk/corecrypto/ccmode/src/ccmode_internal.h @@ -0,0 +1,297 @@ +/* + * ccmode_internal.h + * corecrypto + * + * Created on 12/12/2010 + * + * Copyright (c) 2010,2011,2012,2014,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _CORECRYPTO_CCMODE_INTERNAL_H_ +#define _CORECRYPTO_CCMODE_INTERNAL_H_ + +#include +#include +#include +#include + +#define CCMODE_INVALID_INPUT -1 +#define CCMODE_INVALID_CALL_SEQUENCE -2 +#define CCMODE_INTEGRITY_FAILURE -3 +#define CCMODE_NOT_SUPPORTED -4 +#define CCMODE_INTERNAL_ERROR -5 + +// VNG speed up for GCM's AES encrypton and finite fileld multiplication +#if \ +((CCAES_INTEL_ASM && defined(__x86_64__)) || (CCAES_ARM_ASM && defined(__ARM_NEON__))) +#define CCMODE_GCM_VNG_SPEEDUP 1 +#else +#define CCMODE_GCM_VNG_SPEEDUP 0 +#endif + + +#define CCMODE_GCM_USE_GF_LOOKUP_TABLES 1 + +/* Helper function used. TODO: Probably not specific to xts, since + gcm uses it too */ +void ccmode_xts_mult_alpha(cc_unit *tweak); + +/* Macros for accessing a CCMODE_CBC_KEY. + { + const struct ccmode_ecb *ecb + ccn_unit ecb_key[ecb->n] + } */ +#define _CCMODE_CBC_KEY(K) ((struct _ccmode_cbc_key *)(K)) +#define _CCMODE_CBC_KEY_CONST(K) ((const struct _ccmode_cbc_key *)(K)) +#define CCMODE_CBC_KEY_ECB(K) (_CCMODE_CBC_KEY(K)->ecb) +#define CCMODE_CBC_KEY_ECB_KEY(K) ((ccecb_ctx *)&_CCMODE_CBC_KEY(K)->u[0]) + +CC_CONST CC_INLINE +const struct ccmode_ecb * ccmode_cbc_key_ecb(const cccbc_ctx *K) { + return ((const struct _ccmode_cbc_key *)K)->ecb; +} + +CC_CONST CC_INLINE +const ccecb_ctx * ccmode_cbc_key_ecb_key(const cccbc_ctx *K) { + return (const ccecb_ctx *)&((const struct _ccmode_cbc_key *)K)->u[0]; +} + +/* Macros for accessing a CCMODE_CFB_KEY. +{ + const struct ccmode_ecb *ecb + cc_size pad_len; + ccn_unit pad[ecb->block_size / CCN_UNIT_SIZE]; + ccn_unit iv[ecb->block_size / CCN_UNIT_SIZE]; + ccn_unit ecb_key[ecb->n] +} */ +#define _CCMODE_CFB_KEY(K) ((struct _ccmode_cfb_key *)(K)) +#define CCMODE_CFB_KEY_ECB(K) (_CCMODE_CFB_KEY(K)->ecb) +#define CCMODE_CFB_KEY_PAD_LEN(K) (_CCMODE_CFB_KEY(K)->pad_len) +#define CCMODE_CFB_KEY_PAD(K) (&_CCMODE_CFB_KEY(K)->u[0]) +#define CCMODE_CFB_KEY_IV(K) (&_CCMODE_CFB_KEY(K)->u[ccn_nof_size(CCMODE_CFB_KEY_ECB(K)->block_size)]) +#define CCMODE_CFB_KEY_ECB_KEY(K) ((ccecb_ctx *)&_CCMODE_CFB_KEY(K)->u[2 * ccn_nof_size(CCMODE_CFB_KEY_ECB(K)->block_size)]) + +/* Macros for accessing a CCMODE_CFB8_KEY. +{ + const struct ccmode_ecb *ecb + ccn_unit pad[ecb->block_size / CCN_UNIT_SIZE]; + ccn_unit iv[ecb->block_size / CCN_UNIT_SIZE]; + ccn_unit ecb_key[ecb->n] +} */ +#define _CCMODE_CFB8_KEY(K) ((struct _ccmode_cfb8_key *)(K)) +#define CCMODE_CFB8_KEY_ECB(K) (_CCMODE_CFB8_KEY(K)->ecb) +#define CCMODE_CFB8_KEY_PAD(K) (&_CCMODE_CFB8_KEY(K)->u[0]) +#define CCMODE_CFB8_KEY_IV(K) (&_CCMODE_CFB8_KEY(K)->u[ccn_nof_size(CCMODE_CFB8_KEY_ECB(K)->block_size)]) +#define CCMODE_CFB8_KEY_ECB_KEY(K) ((ccecb_ctx *)&_CCMODE_CFB8_KEY(K)->u[2 * ccn_nof_size(CCMODE_CFB8_KEY_ECB(K)->block_size)]) + + +/* Macros for accessing a CCMODE_CTR_KEY. +{ + const struct ccmode_ecb *ecb + cc_size pad_offset; + ccn_unit pad[ecb->block_size / CCN_UNIT_SIZE]; + ccn_unit ctr[ecb->block_size / CCN_UNIT_SIZE]; + ccn_unit ecb_key[ecb->n] +} */ +#define _CCMODE_CTR_KEY(K) ((struct _ccmode_ctr_key *)(K)) +#define CCMODE_CTR_KEY_ECB(K) (_CCMODE_CTR_KEY(K)->ecb) +#define CCMODE_CTR_KEY_PAD_OFFSET(K) (_CCMODE_CTR_KEY(K)->pad_offset) +#define CCMODE_CTR_KEY_PAD(K) (&_CCMODE_CTR_KEY(K)->u[0]) +#define CCMODE_CTR_KEY_CTR(K) (&_CCMODE_CTR_KEY(K)->u[ccn_nof_size(CCMODE_CTR_KEY_ECB(K)->block_size)]) +#define CCMODE_CTR_KEY_ECB_KEY(K) ((ccecb_ctx *)&_CCMODE_CTR_KEY(K)->u[2 * ccn_nof_size(CCMODE_CTR_KEY_ECB(K)->block_size)]) + +CC_INLINE int ccctr_setctr(const struct ccmode_ctr *mode, ccctr_ctx *ctx, const void *ctr) +{ + return mode->setctr(mode, ctx, ctr); +} + +/* Macros for accessing a CCMODE_OFB_KEY. +{ + const struct ccmode_ecb *ecb + cc_size pad_len; + ccn_unit iv[ecb->block_size / CCN_UNIT_SIZE]; + ccn_unit ecb_key[ecb->n] +} */ +#define _CCMODE_OFB_KEY(K) ((struct _ccmode_ofb_key *)(K)) +#define CCMODE_OFB_KEY_ECB(K) (_CCMODE_OFB_KEY(K)->ecb) +#define CCMODE_OFB_KEY_PAD_LEN(K) (_CCMODE_OFB_KEY(K)->pad_len) +#define CCMODE_OFB_KEY_IV(K) (&_CCMODE_OFB_KEY(K)->u[0]) +#define CCMODE_OFB_KEY_ECB_KEY(K) ((ccecb_ctx *)&_CCMODE_OFB_KEY(K)->u[ccn_nof_size(CCMODE_OFB_KEY_ECB(K)->block_size)]) + + +/* Macros for accessing a CCMODE_XTS_KEY. +{ + const struct ccmode_ecb *ecb + const struct ccmode_ecb *ecb_encrypt + ccn_unit data_key[ecb->size] + ccn_unit tweak_key[ecb_encrypt->size] +} */ +#define _CCMODE_XTS_KEY(K) ((struct _ccmode_xts_key *)(K)) +#define CCMODE_XTS_KEY_ECB(K) (_CCMODE_XTS_KEY(K)->ecb) +#define CCMODE_XTS_KEY_ECB_ENCRYPT(K) (_CCMODE_XTS_KEY(K)->ecb_encrypt) +#define CCMODE_XTS_KEY_DATA_KEY(K) ((ccecb_ctx *)&_CCMODE_XTS_KEY(K)->u[0]) +#define CCMODE_XTS_KEY_TWEAK_KEY(K) ((ccecb_ctx *)&_CCMODE_XTS_KEY(K)->u[ccn_nof_size(CCMODE_XTS_KEY_ECB(K)->size)]) + +CC_CONST CC_INLINE +const struct ccmode_ecb * ccmode_xts_key_ecb(const ccxts_ctx *K) { + return ((const struct _ccmode_xts_key *)K)->ecb; +} + +CC_CONST CC_INLINE +const struct ccmode_ecb * ccmode_xts_key_ecb_encrypt(const ccxts_ctx *K) { + return ((const struct _ccmode_xts_key *)K)->ecb_encrypt; +} + +CC_CONST CC_INLINE +const ccecb_ctx * ccmode_xts_key_data_key(const ccxts_ctx *K) { + return (const ccecb_ctx *)&((const struct _ccmode_xts_key *)K)->u[0]; +} + +CC_CONST CC_INLINE +const ccecb_ctx * ccmode_xts_key_tweak_key(const ccxts_ctx *K) { + return (const ccecb_ctx *)&((const struct _ccmode_xts_key *)K)->u[ccn_nof_size(ccmode_xts_key_ecb(K)->size)]; +} + +/* Macros for accessing a CCMODE_XTS_TWEAK. +{ + size_t blocks_processed; + uint8_t value[16]; +} */ +#define _CCMODE_XTS_TWEAK(T) ((struct _ccmode_xts_tweak *)(T)) +#define CCMODE_XTS_TWEAK_BLOCK_PROCESSED(T)(_CCMODE_XTS_TWEAK(T)->blocks_processed) +#define CCMODE_XTS_TWEAK_VALUE(T) (_CCMODE_XTS_TWEAK(T)->u) + + +/* Macros for accessing a CCMODE_GCM_KEY. + Common to the generic (factory) and the VNG implementation +*/ + +#define _CCMODE_GCM_KEY(K) ((struct _ccmode_gcm_key *)(K)) +#define CCMODE_GCM_KEY_H(K) (_CCMODE_GCM_KEY(K)->H) +#define CCMODE_GCM_KEY_X(K) (_CCMODE_GCM_KEY(K)->X) +#define CCMODE_GCM_KEY_Y(K) (_CCMODE_GCM_KEY(K)->Y) +#define CCMODE_GCM_KEY_Y_0(K) (_CCMODE_GCM_KEY(K)->Y_0) +#define CCMODE_GCM_KEY_PAD_LEN(K) (_CCMODE_GCM_KEY(K)->buf_nbytes) +#define CCMODE_GCM_KEY_PAD(K) (_CCMODE_GCM_KEY(K)->buf) + +#define _CCMODE_GCM_ECB_MODE(K) ((struct _ccmode_gcm_key *)(K)) +#define CCMODE_GCM_KEY_ECB(K) (_CCMODE_GCM_ECB_MODE(K)->ecb) +#define CCMODE_GCM_KEY_ECB_KEY(K) ((ccecb_ctx *)_CCMODE_GCM_ECB_MODE(K)->ecb_key) // set in init function + +#define CCMODE_GCM_STATE_IV 1 +#define CCMODE_GCM_STATE_AAD 2 +#define CCMODE_GCM_STATE_TEXT 3 +#define CCMODE_GCM_STATE_FINAL 4 + +#define CCMODE_STATE_INIT 2 //first call to init +#define CCMODE_STATE_IV_START 3 //first call to set_iv + +// rdar://problem/23523093 +//this allows users to bypass set_iv(). +//this is a temporary setting mainly to allow Security framework to adapt +//ccgcm_set_iv_legacy() and check the tack on decyption without +//need to change the Security twice +//#define CCMODE_STATE_IV_CONT 2 //subsequent calls to set_iv +#define CCMODE_STATE_IV_CONT CCMODE_STATE_IV_START + +#define CCMODE_STATE_AAD 4 +#define CCMODE_STATE_TEXT 5 + +#define CCMODE_CCM_STATE_IV 1 + +void ccmode_gcm_gf_mult(const unsigned char *a, const unsigned char *b, + unsigned char *c); +void ccmode_gcm_mult_h(ccgcm_ctx *key, unsigned char *I); + +/* Macros for accessing a CCMODE_CCM_KEY. */ +#define _CCMODE_CCM_KEY(K) ((struct _ccmode_ccm_key *)(K)) +#define CCMODE_CCM_KEY_ECB(K) (_CCMODE_CCM_KEY(K)->ecb) +#define CCMODE_CCM_KEY_ECB_KEY(K) ((ccecb_ctx *)&_CCMODE_CCM_KEY(K)->u[0]) + +#define _CCMODE_CCM_NONCE(N) ((struct _ccmode_ccm_nonce *)(N)) +#define CCMODE_CCM_KEY_MAC(N) (_CCMODE_CCM_NONCE(N)->MAC) +#define CCMODE_CCM_KEY_A_I(N) (_CCMODE_CCM_NONCE(N)->A_i) +#define CCMODE_CCM_KEY_B_I(N) (_CCMODE_CCM_NONCE(N)->B_i) +#define CCMODE_CCM_KEY_PAD_LEN(N) (_CCMODE_CCM_NONCE(N)->buflen) +#define CCMODE_CCM_KEY_PAD(N) (_CCMODE_CCM_NONCE(N)->buf) +#define CCMODE_CCM_KEY_MAC_LEN(N) (_CCMODE_CCM_NONCE(N)->mac_size) +#define CCMODE_CCM_KEY_NONCE_LEN(N) (_CCMODE_CCM_NONCE(N)->nonce_size) +#define CCMODE_CCM_KEY_AUTH_LEN(N) (_CCMODE_CCM_NONCE(N)->b_i_len) + +/* Macros for accessing a CCMODE_OMAC_KEY. +{ + const struct ccmode_ecb *ecb + cc_size tweak_size; + ccn_unit ecb_key1[ecb->n] + ccn_unit ecb_key2[ecb->n] +} */ +#define _CCMODE_OMAC_KEY(K) ((struct _ccmode_omac_key *)(K)) +#define CCMODE_OMAC_KEY_ECB(K) (_CCMODE_OMAC_KEY(K)->ecb) +#define CCMODE_OMAC_KEY_TWEAK_LEN(K) (_CCMODE_OMAC_KEY(K)->tweak_len) +#define CCMODE_OMAC_KEY_ECB_KEY(K) ((ccecb_ctx *)&_CCMODE_OMAC_KEY(K)->u[0]) + +CC_INLINE void inc_uint(uint8_t *buf, size_t nbytes) +{ + size_t i; + for (i = 0; i < nbytes; i += 1) { + if (++buf[nbytes-1-i] & 255) { break; } + } +} + +CC_INLINE void ccmode_gcm_update_pad(ccgcm_ctx *key) +{ + inc_uint(CCMODE_GCM_KEY_Y(key) + 12, 4); + CCMODE_GCM_KEY_ECB(key)->ecb(CCMODE_GCM_KEY_ECB_KEY(key), 1, + CCMODE_GCM_KEY_Y(key), + CCMODE_GCM_KEY_PAD(key)); +} + +CC_INLINE void ccmode_gcm_aad_finalize(ccgcm_ctx *key) +{ + if (_CCMODE_GCM_KEY(key)->state == CCMODE_GCM_STATE_AAD) { + if (_CCMODE_GCM_KEY(key)->aad_nbytes % CCGCM_BLOCK_NBYTES > 0) { + ccmode_gcm_mult_h(key, CCMODE_GCM_KEY_X(key)); + } + _CCMODE_GCM_KEY(key)->state = CCMODE_GCM_STATE_TEXT; + } +} + +CC_INLINE void xor_128bits(unsigned char *r, const unsigned char *a, const unsigned char *b) +{ + cc_unit *r1 = (cc_unit *)r; + const cc_unit *a1 = (const cc_unit *)a; + const cc_unit *b1 = (const cc_unit *)b; + + for (int i=0; i<128/(CCN_UNIT_SIZE*8); i++) { + r1[i] = a1[i] ^ b1[i]; + } +} + + + +#endif /* _CORECRYPTO_CCMODE_INTERNAL_H_ */ diff --git a/osfmk/corecrypto/ccn/src/ccn_set.c b/osfmk/corecrypto/ccn/src/ccn_set.c index 26cdab52a..e288733f6 100644 --- a/osfmk/corecrypto/ccn/src/ccn_set.c +++ b/osfmk/corecrypto/ccn/src/ccn_set.c @@ -6,6 +6,30 @@ * * Copyright (c) 2012,2014,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include diff --git a/osfmk/corecrypto/ccsha1/src/ccdigest_final_64be.c b/osfmk/corecrypto/ccsha1/src/ccdigest_final_64be.c index 2102709e9..8bec7daf7 100644 --- a/osfmk/corecrypto/ccsha1/src/ccdigest_final_64be.c +++ b/osfmk/corecrypto/ccsha1/src/ccdigest_final_64be.c @@ -6,6 +6,30 @@ * * Copyright (c) 2010,2011,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include diff --git a/osfmk/corecrypto/ccsha1/src/ccsha1_eay.c b/osfmk/corecrypto/ccsha1/src/ccsha1_eay.c index 10c6210a0..3e945ad8c 100644 --- a/osfmk/corecrypto/ccsha1/src/ccsha1_eay.c +++ b/osfmk/corecrypto/ccsha1/src/ccsha1_eay.c @@ -53,6 +53,30 @@ * derivative of this code cannot be changed. i.e. this code cannot simply be * copied and put under another distribution licence * [including the GNU Public Licence.] + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ diff --git a/osfmk/corecrypto/ccsha1/src/ccsha1_initial_state.c b/osfmk/corecrypto/ccsha1/src/ccsha1_initial_state.c index fffabd452..63fb74fe1 100644 --- a/osfmk/corecrypto/ccsha1/src/ccsha1_initial_state.c +++ b/osfmk/corecrypto/ccsha1/src/ccsha1_initial_state.c @@ -6,6 +6,30 @@ * * Copyright (c) 2010,2015 Apple Inc. All rights reserved. * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include diff --git a/osfmk/corecrypto/ccsha2/src/ccsha256_K.c b/osfmk/corecrypto/ccsha2/src/ccsha256_K.c new file mode 100644 index 000000000..ed300d1b6 --- /dev/null +++ b/osfmk/corecrypto/ccsha2/src/ccsha256_K.c @@ -0,0 +1,53 @@ +/* + * ccsha256_K.c + * corecrypto + * + * Created on 12/14/2010 + * + * Copyright (c) 2010,2014,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include + +/* the K array */ +const uint32_t ccsha256_K[64] CC_ALIGNED(16) = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, + 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, + 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, + 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, + 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, + 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, + 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, + 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +}; diff --git a/osfmk/corecrypto/ccsha2/src/ccsha256_di.c b/osfmk/corecrypto/ccsha2/src/ccsha256_di.c new file mode 100644 index 000000000..c0b031a0d --- /dev/null +++ b/osfmk/corecrypto/ccsha2/src/ccsha256_di.c @@ -0,0 +1,59 @@ +/* + * ccsha256_di.c + * corecrypto + * + * Created on 09/18/2012 + * + * Copyright (c) 2012,2014,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include + +#include "corecrypto/fipspost_trace.h" + +const struct ccdigest_info *ccsha256_di(void) +{ + FIPSPOST_TRACE_EVENT; + +#if CCSHA2_VNG_INTEL +#if defined (__x86_64__) + return ( (CC_HAS_AVX2() ? &ccsha256_vng_intel_AVX2_di : + ( (CC_HAS_AVX1() ? &ccsha256_vng_intel_AVX1_di : + &ccsha256_vng_intel_SupplementalSSE3_di ) ) ) ) ; +#else + return &ccsha256_vng_intel_SupplementalSSE3_di; +#endif +#elif CCSHA2_VNG_ARMV7NEON + return &ccsha256_vng_armv7neon_di; +#elif CCSHA256_ARMV6M_ASM + return &ccsha256_v6m_di; +#else + return &ccsha256_ltc_di; +#endif +} diff --git a/osfmk/corecrypto/ccsha2/src/ccsha256_initial_state.c b/osfmk/corecrypto/ccsha2/src/ccsha256_initial_state.c new file mode 100644 index 000000000..591f1d999 --- /dev/null +++ b/osfmk/corecrypto/ccsha2/src/ccsha256_initial_state.c @@ -0,0 +1,46 @@ +/* + * ccsha256_initial_state.c + * corecrypto + * + * Created on 12/07/2010 + * + * Copyright (c) 2010,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include "ccsha2_internal.h" + +const uint32_t ccsha256_initial_state[8] = { + 0x6A09E667, + 0xBB67AE85, + 0x3C6EF372, + 0xA54FF53A, + 0x510E527F, + 0x9B05688C, + 0x1F83D9AB, + 0x5BE0CD19 +}; diff --git a/osfmk/corecrypto/ccsha2/src/ccsha256_ltc_compress.c b/osfmk/corecrypto/ccsha2/src/ccsha256_ltc_compress.c new file mode 100644 index 000000000..b9ff54b87 --- /dev/null +++ b/osfmk/corecrypto/ccsha2/src/ccsha256_ltc_compress.c @@ -0,0 +1,152 @@ +/* + * ccsha256_ltc_compress.c + * corecrypto + * + * Created on 12/03/2010 + * + * Copyright (c) 2010,2011,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + * Parts of this code adapted from LibTomCrypt + * + * LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtom.org + */ + +#include +#include +#include "ccsha2_internal.h" + +// Various logical functions +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) ror((x),(n)) +#define R(x, n) ((x)>>(n)) + +#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) + +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + +//It is beter if the following macros are defined as inline functions, +//but I found some compilers do not inline them. +#ifdef __CC_ARM + #define ror(val, shift) __ror(val,shift) +#else + #define ror(val, shift) ((val >> shift) | (val << (32 - shift))) +#endif + +#ifdef __CC_ARM + #define byte_swap32(x) __rev(x) +#elif defined(__clang__) && !defined(_MSC_VER) + #define byte_swap32(x) __builtin_bswap32(x); +#else + #define byte_swap32(x) ((ror(x, 8) & 0xff00ff00) | (ror(x, 24) & 0x00ff00ff)) +#endif + +#if CC_HANDLE_UNALIGNED_DATA + #define set_W(i) CC_LOAD32_BE(W[i], buf + (4*(i))) +#else + #define set_W(i) W[i] = byte_swap32(buf[i]) +#endif + +// the round function +#define RND(a,b,c,d,e,f,g,h,i) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + ccsha256_K[i] + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + +// compress 512-bits +void ccsha256_ltc_compress(ccdigest_state_t state, size_t nblocks, const void *in) +{ + uint32_t W[64], t0, t1; + uint32_t S0,S1,S2,S3,S4,S5,S6,S7; + int i; + uint32_t *s = ccdigest_u32(state); +#if CC_HANDLE_UNALIGNED_DATA + const unsigned char *buf = in; +#else + const uint32_t *buf = in; +#endif + + while(nblocks--) { + + // schedule W 0..15 + set_W(0); set_W(1); set_W(2); set_W(3); set_W(4); set_W(5); set_W(6); set_W(7); + set_W(8); set_W(9); set_W(10);set_W(11);set_W(12);set_W(13);set_W(14);set_W(15); + + // schedule W 16..63 + for (i = 16; i < 64; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; + } + + // copy state into S + S0= s[0]; + S1= s[1]; + S2= s[2]; + S3= s[3]; + S4= s[4]; + S5= s[5]; + S6= s[6]; + S7= s[7]; + + // Compress + for (i = 0; i < 64; i += 8) { + RND(S0,S1,S2,S3,S4,S5,S6,S7,i+0); + RND(S7,S0,S1,S2,S3,S4,S5,S6,i+1); + RND(S6,S7,S0,S1,S2,S3,S4,S5,i+2); + RND(S5,S6,S7,S0,S1,S2,S3,S4,i+3); + RND(S4,S5,S6,S7,S0,S1,S2,S3,i+4); + RND(S3,S4,S5,S6,S7,S0,S1,S2,i+5); + RND(S2,S3,S4,S5,S6,S7,S0,S1,i+6); + RND(S1,S2,S3,S4,S5,S6,S7,S0,i+7); + } + + // feedback + s[0] += S0; + s[1] += S1; + s[2] += S2; + s[3] += S3; + s[4] += S4; + s[5] += S5; + s[6] += S6; + s[7] += S7; + + buf+=CCSHA256_BLOCK_SIZE/sizeof(buf[0]); + } +} diff --git a/osfmk/corecrypto/ccsha2/src/ccsha256_ltc_di.c b/osfmk/corecrypto/ccsha2/src/ccsha256_ltc_di.c new file mode 100644 index 000000000..1e4109b60 --- /dev/null +++ b/osfmk/corecrypto/ccsha2/src/ccsha256_ltc_di.c @@ -0,0 +1,48 @@ +/* + * ccsha256_ltc_di.c + * corecrypto + * + * Created on 12/07/2010 + * + * Copyright (c) 2010,2012,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include "ccsha2_internal.h" + +const struct ccdigest_info ccsha256_ltc_di = { + .output_size = CCSHA256_OUTPUT_SIZE, + .state_size = CCSHA256_STATE_SIZE, + .block_size = CCSHA256_BLOCK_SIZE, + .oid_size = ccoid_sha256_len, + .oid = CC_DIGEST_OID_SHA256, + .initial_state = ccsha256_initial_state, + .compress = ccsha256_ltc_compress, + .final = ccdigest_final_64be, +}; diff --git a/osfmk/corecrypto/ccsha2/src/ccsha2_internal.h b/osfmk/corecrypto/ccsha2/src/ccsha2_internal.h new file mode 100644 index 000000000..14fd2d4fb --- /dev/null +++ b/osfmk/corecrypto/ccsha2/src/ccsha2_internal.h @@ -0,0 +1,63 @@ +/* + * ccsha2_internal.h + * corecrypto + * + * Created on 12/07/2010 + * + * Copyright (c) 2010,2011,2012,2014,2015 Apple Inc. All rights reserved. + * + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _CORECRYPTO_CCSHA2_INTERNAL_H_ +#define _CORECRYPTO_CCSHA2_INTERNAL_H_ + +#include + +void ccsha256_ltc_compress(ccdigest_state_t state, size_t nblocks, const void *buf); +void ccsha256_v6m_compress(ccdigest_state_t state, size_t nblocks, const void *buf); +void ccsha512_ltc_compress(ccdigest_state_t state, size_t nblocks, const void *in); + +#if CCSHA2_VNG_INTEL +#if defined __x86_64__ +void ccsha256_vng_intel_avx2_compress(ccdigest_state_t state, size_t nblocks, const void *in); +void ccsha256_vng_intel_avx1_compress(ccdigest_state_t state, size_t nblocks, const void *in); +void ccsha256_vng_intel_ssse3_compress(ccdigest_state_t state, size_t nblocks, const void *in); +void ccsha512_vng_intel_avx2_compress(ccdigest_state_t state, size_t nblocks, const void *in); +void ccsha512_vng_intel_avx1_compress(ccdigest_state_t state, size_t nblocks, const void *in); +void ccsha512_vng_intel_ssse3_compress(ccdigest_state_t state, size_t nblocks, const void *in); +#endif +void ccsha256_vng_intel_sse3_compress(ccdigest_state_t state, size_t nblocks, const void *in); +#endif + +void ccsha512_final(const struct ccdigest_info *di, ccdigest_ctx_t ctx, unsigned char *digest); + +extern const uint32_t ccsha224_initial_state[8]; +extern const uint32_t ccsha256_initial_state[8]; +extern const uint64_t ccsha384_initial_state[8]; +extern const uint64_t ccsha512_initial_state[8]; + +#endif /* _CORECRYPTO_CCSHA2_INTERNAL_H_ */ diff --git a/osfmk/corpses/corpse.c b/osfmk/corpses/corpse.c index 87914b459..b2078a91a 100644 --- a/osfmk/corpses/corpse.c +++ b/osfmk/corpses/corpse.c @@ -116,6 +116,7 @@ */ +#include #include #include #include @@ -128,6 +129,7 @@ #include #include #include +#include #if CONFIG_MACF #include @@ -138,26 +140,32 @@ */ #include -unsigned long total_corpses_count = 0; +union corpse_creation_gate { + struct { + uint16_t user_faults; + uint16_t corpses; + }; + uint32_t value; +}; + +static _Atomic uint32_t inflight_corpses; unsigned long total_corpses_created = 0; boolean_t corpse_enabled_config = TRUE; /* bootarg to turn on corpse forking for EXC_RESOURCE */ int exc_via_corpse_forking = 1; -/* bootarg to unify corpse blob allocation */ -int unify_corpse_blob_alloc = 1; - /* bootarg to generate corpse for fatal high memory watermark violation */ int corpse_for_fatal_memkill = 1; -kcdata_descriptor_t task_get_corpseinfo(task_t task); -kcdata_descriptor_t task_crashinfo_alloc_init(mach_vm_address_t crash_data_p, unsigned size, int get_corpseref, unsigned flags); -kern_return_t task_crashinfo_destroy(kcdata_descriptor_t data, int release_corpseref); -static kern_return_t task_crashinfo_get_ref(); -static kern_return_t task_crashinfo_release_ref(); +#ifdef __arm__ +static inline int IS_64BIT_PROCESS(__unused void *p) { return 0; } +#else extern int IS_64BIT_PROCESS(void *); -extern void gather_populate_corpse_crashinfo(void *p, void *crash_info_ptr, mach_exception_data_type_t code, mach_exception_data_type_t subcode, uint64_t *udata_buffer, int num_udata); +#endif /* __arm__ */ +extern void gather_populate_corpse_crashinfo(void *p, task_t task, + mach_exception_data_type_t code, mach_exception_data_type_t subcode, + uint64_t *udata_buffer, int num_udata, void *reason); extern void *proc_find(int pid); extern int proc_rele(void *p); @@ -165,7 +173,6 @@ extern int proc_rele(void *p); void corpses_init(){ char temp_buf[20]; int exc_corpse_forking; - int corpse_blob_alloc; int fatal_memkill; if (PE_parse_boot_argn("-no_corpses", temp_buf, sizeof(temp_buf))) { corpse_enabled_config = FALSE; @@ -173,9 +180,6 @@ void corpses_init(){ if (PE_parse_boot_argn("exc_via_corpse_forking", &exc_corpse_forking, sizeof(exc_corpse_forking))) { exc_via_corpse_forking = exc_corpse_forking; } - if (PE_parse_boot_argn("unify_corpse_blob_alloc", &corpse_blob_alloc, sizeof(corpse_blob_alloc))) { - unify_corpse_blob_alloc = corpse_blob_alloc; - } if (PE_parse_boot_argn("corpse_for_fatal_memkill", &fatal_memkill, sizeof(fatal_memkill))) { corpse_for_fatal_memkill = fatal_memkill; } @@ -190,57 +194,115 @@ boolean_t corpses_enabled() return corpse_enabled_config; } +unsigned long +total_corpses_count(void) +{ + union corpse_creation_gate gate; + + gate.value = atomic_load_explicit(&inflight_corpses, memory_order_relaxed); + return gate.corpses; +} + /* * Routine: task_crashinfo_get_ref() * Grab a slot at creating a corpse. * Returns: KERN_SUCCESS if the policy allows for creating a corpse. */ -kern_return_t task_crashinfo_get_ref() +static kern_return_t +task_crashinfo_get_ref(uint16_t kcd_u_flags) { - unsigned long counter = total_corpses_count; - counter = OSIncrementAtomic((SInt32 *)&total_corpses_count); - if (counter >= TOTAL_CORPSES_ALLOWED) { - OSDecrementAtomic((SInt32 *)&total_corpses_count); - return KERN_RESOURCE_SHORTAGE; + union corpse_creation_gate oldgate, newgate; + + assert(kcd_u_flags & CORPSE_CRASHINFO_HAS_REF); + + oldgate.value = atomic_load_explicit(&inflight_corpses, memory_order_relaxed); + for (;;) { + newgate = oldgate; + if (kcd_u_flags & CORPSE_CRASHINFO_USER_FAULT) { + if (newgate.user_faults++ >= TOTAL_USER_FAULTS_ALLOWED) { + return KERN_RESOURCE_SHORTAGE; + } + } + if (newgate.corpses++ >= TOTAL_CORPSES_ALLOWED) { + return KERN_RESOURCE_SHORTAGE; + } + + // this reloads the value in oldgate + if (atomic_compare_exchange_strong_explicit(&inflight_corpses, + &oldgate.value, newgate.value, memory_order_relaxed, + memory_order_relaxed)) { + return KERN_SUCCESS; + } } - OSIncrementAtomicLong((volatile long *)&total_corpses_created); - return KERN_SUCCESS; } /* * Routine: task_crashinfo_release_ref * release the slot for corpse being used. */ -kern_return_t task_crashinfo_release_ref() +static kern_return_t +task_crashinfo_release_ref(uint16_t kcd_u_flags) { - unsigned long __assert_only counter; - counter = OSDecrementAtomic((SInt32 *)&total_corpses_count); - assert(counter > 0); - return KERN_SUCCESS; + union corpse_creation_gate oldgate, newgate; + + assert(kcd_u_flags & CORPSE_CRASHINFO_HAS_REF); + + oldgate.value = atomic_load_explicit(&inflight_corpses, memory_order_relaxed); + for (;;) { + newgate = oldgate; + if (kcd_u_flags & CORPSE_CRASHINFO_USER_FAULT) { + if (newgate.user_faults-- == 0) { + panic("corpse in flight count over-release"); + } + } + if (newgate.corpses-- == 0) { + panic("corpse in flight count over-release"); + } + // this reloads the value in oldgate + if (atomic_compare_exchange_strong_explicit(&inflight_corpses, + &oldgate.value, newgate.value, memory_order_relaxed, + memory_order_relaxed)) { + return KERN_SUCCESS; + } + } } -kcdata_descriptor_t task_crashinfo_alloc_init(mach_vm_address_t crash_data_p, unsigned size, int get_corpseref, unsigned flags) +kcdata_descriptor_t +task_crashinfo_alloc_init(mach_vm_address_t crash_data_p, unsigned size, + uint32_t kc_u_flags, unsigned kc_flags) { - if(get_corpseref && KERN_SUCCESS != task_crashinfo_get_ref()) { - return NULL; + kcdata_descriptor_t kcdata; + + if (kc_u_flags & CORPSE_CRASHINFO_HAS_REF) { + if (KERN_SUCCESS != task_crashinfo_get_ref(kc_u_flags)) { + return NULL; + } } - return kcdata_memory_alloc_init(crash_data_p, TASK_CRASHINFO_BEGIN, size, flags); + kcdata = kcdata_memory_alloc_init(crash_data_p, TASK_CRASHINFO_BEGIN, size, + kc_flags); + if (kcdata) { + kcdata->kcd_user_flags = kc_u_flags; + } else if (kc_u_flags & CORPSE_CRASHINFO_HAS_REF) { + task_crashinfo_release_ref(kc_u_flags); + } + return kcdata; } /* * Free up the memory associated with task_crashinfo_data */ -kern_return_t task_crashinfo_destroy(kcdata_descriptor_t data, int release_corpseref) +kern_return_t +task_crashinfo_destroy(kcdata_descriptor_t data) { if (!data) { return KERN_INVALID_ARGUMENT; } - - if (release_corpseref) - task_crashinfo_release_ref(); + if (data->kcd_user_flags & CORPSE_CRASHINFO_HAS_REF) { + task_crashinfo_release_ref(data->kcd_user_flags); + } return kcdata_memory_destroy(data); } @@ -301,10 +363,15 @@ task_purge_all_corpses(void) /* Iterate through all the corpse tasks and clear all map entries */ queue_iterate(&corpse_tasks, task, task_t, corpse_tasks) { vm_map_remove(task->map, - task->map->min_offset, - task->map->max_offset, - /* no unnesting on final cleanup: */ - VM_MAP_REMOVE_NO_UNNESTING); + task->map->min_offset, + task->map->max_offset, + /* + * Final cleanup: + * + no unnesting + * + remove immutable mappings + */ + (VM_MAP_REMOVE_NO_UNNESTING | + VM_MAP_REMOVE_IMMUTABLE)); } lck_mtx_unlock(&tasks_corpse_lock); @@ -316,7 +383,7 @@ task_purge_all_corpses(void) * corpse_task - task port of the generated corpse * returns: KERN_SUCCESS on Success. * KERN_FAILURE on Failure. - * KERN_NO_SUPPORTED on corpse disabled. + * KERN_NOT_SUPPORTED on corpse disabled. * KERN_RESOURCE_SHORTAGE on memory alloc failure or reaching max corpse. */ kern_return_t @@ -342,7 +409,7 @@ task_generate_corpse( task_unlock(task); /* Generate a corpse for the given task, will return with a ref on corpse task */ - kr = task_generate_corpse_internal(task, &new_task, &thread, 0, 0); + kr = task_generate_corpse_internal(task, &new_task, &thread, 0, 0, 0, NULL); if (kr != KERN_SUCCESS) { return kr; } @@ -379,34 +446,42 @@ task_generate_corpse( /* * Routine: task_enqueue_exception_with_corpse * params: task - task to generate a corpse and enqueue it + * etype - EXC_RESOURCE or EXC_GUARD * code - exception code to be enqueued * codeCnt - code array count - code and subcode + * + * returns: KERN_SUCCESS on Success. + * KERN_FAILURE on Failure. + * KERN_INVALID_ARGUMENT on invalid arguments passed. + * KERN_NOT_SUPPORTED on corpse disabled. + * KERN_RESOURCE_SHORTAGE on memory alloc failure or reaching max corpse. */ -void +kern_return_t task_enqueue_exception_with_corpse( task_t task, + exception_type_t etype, mach_exception_data_t code, - mach_msg_type_number_t codeCnt) + mach_msg_type_number_t codeCnt, + void *reason) { task_t new_task = TASK_NULL; thread_t thread = THREAD_NULL; kern_return_t kr; if (codeCnt < 2) { - return; + return KERN_INVALID_ARGUMENT; } /* Generate a corpse for the given task, will return with a ref on corpse task */ - kr = task_generate_corpse_internal(task, &new_task, &thread, code[0], code[1]); - if (kr != KERN_SUCCESS) { - return; + kr = task_generate_corpse_internal(task, &new_task, &thread, + etype, code[0], code[1], reason); + if (kr == KERN_SUCCESS) { + assert(thread != THREAD_NULL); + assert(new_task != TASK_NULL); + assert(etype == EXC_RESOURCE || etype == EXC_GUARD); + thread_exception_enqueue(new_task, thread, etype); } - - assert(thread != THREAD_NULL); - assert(new_task != TASK_NULL); - thread_exception_enqueue(new_task, thread); - - return; + return kr; } /* @@ -414,11 +489,12 @@ task_enqueue_exception_with_corpse( * params: task - task to fork a corpse * corpse_task - task of the generated corpse * exc_thread - equivalent thread in corpse enqueuing exception + * etype - EXC_RESOURCE or EXC_GUARD or 0 * code - mach exception code to be passed in corpse blob - * subcode - mach excpetion subcode to be passed in corpse blob + * subcode - mach exception subcode to be passed in corpse blob * returns: KERN_SUCCESS on Success. * KERN_FAILURE on Failure. - * KERN_NO_SUPPORTED on corpse disabled. + * KERN_NOT_SUPPORTED on corpse disabled. * KERN_RESOURCE_SHORTAGE on memory alloc failure or reaching max corpse. */ kern_return_t @@ -426,8 +502,10 @@ task_generate_corpse_internal( task_t task, task_t *corpse_task, thread_t *exc_thread, + exception_type_t etype, mach_exception_data_type_t code, - mach_exception_data_type_t subcode) + mach_exception_data_type_t subcode, + void *reason) { task_t new_task = TASK_NULL; thread_t thread = THREAD_NULL; @@ -439,17 +517,24 @@ task_generate_corpse_internal( uint64_t *udata_buffer = NULL; int size = 0; int num_udata = 0; - boolean_t release_corpse_ref = FALSE; + uint16_t kc_u_flags = CORPSE_CRASHINFO_HAS_REF; +#if CONFIG_MACF + struct label *label = NULL; +#endif + if (!corpses_enabled()) { return KERN_NOT_SUPPORTED; } - kr = task_crashinfo_get_ref(); + if (etype == EXC_GUARD && EXC_GUARD_DECODE_GUARD_TYPE(code) == GUARD_TYPE_USER) { + kc_u_flags |= CORPSE_CRASHINFO_USER_FAULT; + } + + kr = task_crashinfo_get_ref(kc_u_flags); if (kr != KERN_SUCCESS) { return kr; } - release_corpse_ref = TRUE; /* Having a task reference does not guarantee a proc reference */ p = proc_find(task_pid(task)); @@ -461,6 +546,11 @@ task_generate_corpse_internal( is64bit = IS_64BIT_PROCESS(p); t_flags = TF_CORPSE_FORK | TF_PENDING_CORPSE | TF_CORPSE | (is64bit ? TF_64B_ADDR : TF_NONE); +#if CONFIG_MACF + /* Create the corpse label credentials from the process. */ + label = mac_exc_create_label_for_proc(p); +#endif + /* Create a task for corpse */ kr = task_create_internal(task, NULL, @@ -480,13 +570,19 @@ task_generate_corpse_internal( goto error_task_generate_corpse; } - kr = task_collect_crash_info(new_task, p, TRUE); + kr = task_collect_crash_info(new_task, +#if CONFIG_MACF + label, +#endif + TRUE); if (kr != KERN_SUCCESS) { goto error_task_generate_corpse; } - /* The corpse_info field in task in initialized, call to task_deallocate will drop corpse ref */ - release_corpse_ref = FALSE; + /* transfer our references to the corpse info */ + assert(new_task->corpse_info->kcd_user_flags == 0); + new_task->corpse_info->kcd_user_flags = kc_u_flags; + kc_u_flags = 0; kr = task_start_halt(new_task); if (kr != KERN_SUCCESS) { @@ -497,7 +593,8 @@ task_generate_corpse_internal( ipc_space_terminate(new_task->itk_space); /* Populate the corpse blob, use the proc struct of task instead of corpse task */ - gather_populate_corpse_crashinfo(p, task_get_corpseinfo(new_task), code, subcode, udata_buffer, num_udata); + gather_populate_corpse_crashinfo(p, new_task, + code, subcode, udata_buffer, num_udata, reason); /* Add it to global corpse task list */ task_add_to_corpse_task_list(new_task); @@ -506,6 +603,12 @@ task_generate_corpse_internal( *exc_thread = thread; error_task_generate_corpse: +#if CONFIG_MACF + if (label) { + mac_exc_free_label(label); + } +#endif + /* Release the proc reference */ if (p != NULL) { proc_rele(p); @@ -530,8 +633,8 @@ error_task_generate_corpse: task_terminate_internal(new_task); task_deallocate(new_task); } - if (release_corpse_ref) { - task_crashinfo_release_ref(); + if (kc_u_flags) { + task_crashinfo_release_ref(kc_u_flags); } } /* Free the udata buffer allocated in task_duplicate_map_and_threads */ @@ -594,23 +697,31 @@ task_map_corpse_info_64( kern_return_t kr; mach_vm_offset_t crash_data_ptr = 0; mach_vm_size_t size = CORPSEINFO_ALLOCATION_SIZE; + void *corpse_info_kernel = NULL; if (task == TASK_NULL || task_is_a_corpse_fork(task)) { return KERN_INVALID_ARGUMENT; } if (corpse_task == TASK_NULL || !task_is_a_corpse(corpse_task) || - corpse_task->corpse_info == NULL || corpse_task->corpse_info_kernel == NULL) { + kcdata_memory_get_begin_addr(corpse_task->corpse_info) == NULL) { return KERN_INVALID_ARGUMENT; } - kr = mach_vm_allocate(task->map, &crash_data_ptr, size, - (VM_MAKE_TAG(VM_MEMORY_CORPSEINFO) | VM_FLAGS_ANYWHERE)); + corpse_info_kernel = kcdata_memory_get_begin_addr(corpse_task->corpse_info); + kr = mach_vm_allocate_kernel(task->map, &crash_data_ptr, size, + VM_FLAGS_ANYWHERE, VM_MEMORY_CORPSEINFO); if (kr != KERN_SUCCESS) { return kr; } - copyout(corpse_task->corpse_info_kernel, crash_data_ptr, size); + copyout(corpse_info_kernel, crash_data_ptr, size); *kcd_addr_begin = crash_data_ptr; *kcd_size = size; return KERN_SUCCESS; } + +uint64_t +task_corpse_get_crashed_thread_id(task_t corpse_task) +{ + return corpse_task->crashed_thread_id; +} diff --git a/osfmk/corpses/task_corpse.h b/osfmk/corpses/task_corpse.h index fea95b8e9..d3d43107b 100644 --- a/osfmk/corpses/task_corpse.h +++ b/osfmk/corpses/task_corpse.h @@ -55,34 +55,39 @@ typedef struct kcdata_item *task_crashinfo_item_t; #define CORPSEINFO_ALLOCATION_SIZE (1024 * 16) #define TOTAL_CORPSES_ALLOWED 5 - +#define TOTAL_USER_FAULTS_ALLOWED 1 extern kern_return_t task_mark_corpse(task_t task); -extern kern_return_t task_deliver_crash_notification(task_t task, thread_t thread, mach_exception_data_type_t subcode); +extern kern_return_t task_deliver_crash_notification(task_t, thread_t, exception_type_t, mach_exception_subcode_t); + +/* In the corpseinfo kcd_user_flags */ +#define CORPSE_CRASHINFO_HAS_REF 0x1 +#define CORPSE_CRASHINFO_USER_FAULT 0x2 extern kcdata_descriptor_t task_get_corpseinfo(task_t task); -#define GET_CORPSE_REF TRUE -#define RELEASE_CORPSE_REF TRUE +extern unsigned long total_corposes_count(void) __attribute__((pure)); extern kcdata_descriptor_t task_crashinfo_alloc_init( mach_vm_address_t crash_data_p, - unsigned size, - int get_corpseref, unsigned flags); -extern kern_return_t task_crashinfo_destroy(kcdata_descriptor_t data, int release_corpseref); + unsigned size, uint32_t kc_u_flags, unsigned kc_flags); +extern kern_return_t task_crashinfo_destroy(kcdata_descriptor_t data); extern void corpses_init(void); +extern unsigned long total_corpses_count(void); extern boolean_t corpses_enabled(void); extern kern_return_t task_generate_corpse_internal( task_t task, task_t *corpse_task, thread_t *thread, + exception_type_t etype, mach_exception_data_type_t code, - mach_exception_data_type_t subcode); + mach_exception_data_type_t subcode, + void *reason); extern void task_clear_corpse(task_t task); @@ -93,16 +98,21 @@ extern kern_return_t task_duplicate_map_and_threads( thread_t *thread, uint64_t **udata_buffer, int *size, - int*num_udata); + int *num_udata); + +extern boolean_t task_allowed_vm_map_fork(task_t task __unused); -extern void task_enqueue_exception_with_corpse( +extern kern_return_t task_enqueue_exception_with_corpse( task_t task, + exception_type_t etype, mach_exception_data_t code, - mach_msg_type_number_t codeCnt); + mach_msg_type_number_t codeCnt, + void *reason); extern void task_add_to_corpse_task_list(task_t corpse_task); void task_remove_from_corpse_task_list(task_t corpse_task); void task_purge_all_corpses(void); +extern uint64_t task_corpse_get_crashed_thread_id(task_t corpse_task); #endif /* XNU_KERNEL_PRIVATE */ diff --git a/osfmk/device/device_port.h b/osfmk/device/device_port.h index 8948ed53f..a2445bab1 100644 --- a/osfmk/device/device_port.h +++ b/osfmk/device/device_port.h @@ -74,6 +74,7 @@ extern mach_port_t master_device_port; #define DEVICE_PAGER_COHERENT 0x2 #define DEVICE_PAGER_CACHE_INHIB 0x4 #define DEVICE_PAGER_WRITE_THROUGH 0x8 +#define DEVICE_PAGER_EARLY_ACK 0x20 #define DEVICE_PAGER_CONTIGUOUS 0x100 #define DEVICE_PAGER_NOPHYSCACHE 0x200 diff --git a/osfmk/device/iokit_rpc.c b/osfmk/device/iokit_rpc.c index 4e5da248b..d44eea33c 100644 --- a/osfmk/device/iokit_rpc.c +++ b/osfmk/device/iokit_rpc.c @@ -63,6 +63,9 @@ #if defined(__i386__) || defined(__x86_64__) #include #endif +#if defined(__arm__) || defined(__arm64__) +#include +#endif #include #define EXTERN @@ -511,9 +514,14 @@ kern_return_t IOMapPages(vm_map_t map, mach_vm_address_t va, mach_vm_address_t p case kIOMapCopybackCache: flags = VM_WIMG_COPYBACK; break; + case kIOMapCopybackInnerCache: flags = VM_WIMG_INNERWBACK; break; + + case kIOMapPostedWrite: + flags = VM_WIMG_POSTED; + break; } pmap_set_cache_attributes(pagenum, flags); @@ -522,9 +530,7 @@ kern_return_t IOMapPages(vm_map_t map, mach_vm_address_t va, mach_vm_address_t p // Set up a block mapped area - pmap_map_block(pmap, va, pagenum, (uint32_t) atop_64(round_page_64(length)), prot, 0, 0); - - return( KERN_SUCCESS ); + return pmap_map_block(pmap, va, pagenum, (uint32_t) atop_64(round_page_64(length)), prot, 0, 0); } kern_return_t IOUnmapPages(vm_map_t map, mach_vm_address_t va, mach_vm_size_t length) @@ -571,6 +577,14 @@ kern_return_t IOProtectCacheMode(vm_map_t __unused map, mach_vm_address_t __unus case kIOMapCopybackCache: flags = VM_WIMG_COPYBACK; break; + + case kIOMapCopybackInnerCache: + flags = VM_WIMG_INNERWBACK; + break; + + case kIOMapPostedWrite: + flags = VM_WIMG_POSTED; + break; } pmap_flush_context_init(&pmap_flush_context_storage); @@ -605,6 +619,8 @@ ppnum_t IOGetLastPageNumber(void) highest = lastPage; } return (highest); +#elif __arm__ || __arm64__ + return 0; #else #error unknown arch #endif diff --git a/osfmk/device/subrs.c b/osfmk/device/subrs.c index e36267988..711ded8cf 100644 --- a/osfmk/device/subrs.c +++ b/osfmk/device/subrs.c @@ -143,10 +143,18 @@ #undef strcmp #undef strncmp #undef strcpy -#undef strncpy #undef strlen #endif +/* to prevent recursion in the _chk functions */ +#undef strcat +#undef strncpy +#undef strncat +#undef memcpy +#undef memset +#undef memmove +#undef strlcpy +#undef strlcat /* * Abstract: * strcmp (s1, s2) compares the strings "s1" and "s2". @@ -187,6 +195,7 @@ strcmp( * comparison runs for at most "n" characters. */ +#if !defined __arm__ && !defined __arm64__ // ARM implementation in ../arm/strncmp.s // ARM64 implementation in ../arm64/strncmp.s int @@ -211,6 +220,7 @@ strncmp( return 0; } +#endif // #ifndef __arm__ // @@ -254,6 +264,39 @@ strncasecmp(const char *s1, const char *s2, size_t n) return (0); } +char * +strchr(const char *s, int c) +{ + if (!s) { + return NULL; + } + + do { + if (*s == c) { + return __CAST_AWAY_QUALIFIER(s, const, char *); + } + } while (*s++); + + return NULL; +} + +char * +strrchr(const char *s, int c) +{ + const char *found = NULL; + + if (!s) { + return NULL; + } + + do { + if (*s == c) { + found = s; + } + } while (*s++); + + return __CAST_AWAY_QUALIFIER(found, const, char *); +} /* * Abstract: @@ -263,6 +306,7 @@ strncasecmp(const char *s1, const char *s2, size_t n) * Deprecation Warning: * strcpy() is being deprecated. Please use strlcpy() instead. */ +#if !CONFIG_EMBEDDED char * strcpy( char *to, @@ -275,6 +319,7 @@ strcpy( return ret; } +#endif /* * Abstract: @@ -285,7 +330,9 @@ strcpy( * to the "to" string. */ +#if !defined __arm__ && !defined __arm64__ // ARM and ARM64 implementation in ../arm/strncpy.c +#undef strncpy char * strncpy( char *s1, @@ -303,6 +350,7 @@ strncpy( i++; return (os1); } +#endif // #ifndef __arm__ /* * atoi: @@ -380,8 +428,10 @@ atoi_term( * length of s or max; whichever is smaller */ +#if !defined __arm__ && !defined __arm64__ // ARM implementation in ../arm/strnlen.s // ARM64 implementation in ../arm64/strnlen.s +#undef strnlen size_t strnlen(const char *s, size_t max) { const char *es = s + max, *p = s; @@ -390,6 +440,7 @@ strnlen(const char *s, size_t max) { return p - s; } +#endif // #ifndef __arm__ /* * convert an integer to an ASCII string. @@ -432,6 +483,7 @@ itoa( * Deprecation Warning: * strcat() is being deprecated. Please use strlcat() instead. */ +#if !CONFIG_EMBEDDED char * strcat( char *dest, @@ -445,6 +497,7 @@ strcat( ; return (old); } +#endif /* * Appends src to string dst of size siz (unlike strncat, siz is the @@ -453,6 +506,7 @@ strcat( * Returns strlen(src) + MIN(siz, strlen(initial dst)). * If retval >= siz, truncation occurred. */ +#undef strlcat size_t strlcat(char *dst, const char *src, size_t siz) { @@ -487,7 +541,9 @@ strlcat(char *dst, const char *src, size_t siz) * Returns strlen(src); if retval >= siz, truncation occurred. */ +#if !defined __arm__ && !defined __arm64__ // ARM and ARM64 implementation in ../arm/strlcpy.c +#undef strlcpy size_t strlcpy(char *dst, const char *src, size_t siz) { @@ -513,6 +569,7 @@ strlcpy(char *dst, const char *src, size_t siz) return(s - src - 1); /* count does not include NUL */ } +#endif /* * STRDUP @@ -591,3 +648,98 @@ strnstr(char *s, const char *find, size_t slen) return (s); } +void * __memcpy_chk(void *dst, void const *src, size_t s, size_t chk_size); +void * __memmove_chk(void *dst, void const *src, size_t s, size_t chk_size); +void * __memset_chk(void *dst, int c, size_t s, size_t chk_size); +size_t __strlcpy_chk(char *dst, char const *src, size_t s, size_t chk_size); +size_t __strlcat_chk(char *dst, char const *src, size_t s, size_t chk_size); +char * __strncpy_chk (char *restrict dst, char *restrict src, size_t len, size_t chk_size); +char * __strncat_chk (char *restrict dst, const char *restrict src, size_t len, size_t chk_size); +char * __strcpy_chk(char *restrict dst, const char *restrict src, size_t chk_size); +char * __strcat_chk (char *restrict dst, const char *restrict src, size_t chk_size); + +void * +__memcpy_chk(void *dst, void const *src, size_t s, size_t chk_size) +{ + if (__improbable(chk_size < s)) + panic("__memcpy_chk object size check failed: dst %p, src %p, (%zu < %zu)", dst, src, chk_size, s); + return memcpy(dst, src, s); +} + +void * +__memmove_chk(void *dst, void const *src, size_t s, size_t chk_size) +{ + if (__improbable(chk_size < s)) + panic("__memmove_chk object size check failed: dst %p, src %p, (%zu < %zu)", dst, src, chk_size, s); + return memmove(dst, src, s); +} + +void * +__memset_chk(void *dst, int c, size_t s, size_t chk_size) +{ + if (__improbable(chk_size < s)) + panic("__memset_chk object size check failed: dst %p, c %c, (%zu < %zu)", dst, c, chk_size, s); + return memset(dst, c, s); +} + +size_t +__strlcat_chk(char *dst, char const *src, size_t s, size_t chk_size) +{ + if (__improbable(chk_size < s)) + panic("__strlcat_chk object size check failed: dst %p, src %p, (%zu < %zu)", dst, src, chk_size, s); + return strlcat(dst, src, s); +} + +size_t +__strlcpy_chk(char *dst, char const *src, size_t s, size_t chk_size) +{ + if (__improbable(chk_size < s)) + panic("__strlcpy_chk object size check failed: dst %p, src %p, (%zu < %zu)", dst, src, chk_size, s); + return strlcpy(dst, src, s); +} + +char * +__strncpy_chk (char *restrict dst, char *restrict src, + size_t len, size_t chk_size) +{ + if (__improbable(chk_size < len)) { + panic("__strncpy_chk object size check failed: dst %p, src %p, (%zu < %zu)", dst, src, chk_size, len); + } + return strncpy(dst, src, len); +} + +char * +__strncat_chk (char *restrict dst, const char *restrict src, + size_t len, size_t chk_size) +{ + size_t len1 = strlen(dst); + size_t len2 = strnlen(src, len); + if (__improbable (chk_size < len1 + len2 + 1)) { + panic("__strncat_chk object size check failed: dst %p, src %p, (%zu < %zu + %zu + 1)", dst, src, chk_size, len1, len2); + } + return strncat(dst, src, len); +} + +char * +__strcpy_chk (char *restrict dst, const char *restrict src, size_t chk_size) +{ + size_t len = strlen(src); + if (__improbable (chk_size < len + 1)) { + panic("__strcpy_chk object size check failed: dst %p, src %p, (%zu < %zu + 1)", dst, src, chk_size, len); + } + memcpy(dst, src, len+1); + return dst; +} + +char * +__strcat_chk (char *restrict dst, const char *restrict src, size_t chk_size) +{ + size_t len1 = strlen(dst); + size_t len2 = strlen(src); + size_t required_len = len1 + len2 + 1; + if (__improbable (chk_size < required_len)) { + panic("__strcat_chk object size check failed: dst %p, src %p, (%zu < %zu + %zu + 1)", dst, src, chk_size, len1, len2); + } + memcpy(dst + len1, src, len2 + 1); + return dst; +} diff --git a/osfmk/i386/AT386/conf.c b/osfmk/i386/AT386/conf.c index f90dac3ec..8fffcc657 100644 --- a/osfmk/i386/AT386/conf.c +++ b/osfmk/i386/AT386/conf.c @@ -62,6 +62,7 @@ #include #include +#include /* * Clock device subsystem configuration. The clock_list[] @@ -69,12 +70,12 @@ * the system. */ -extern struct clock_ops sysclk_ops, calend_ops; +extern const struct clock_ops sysclk_ops, calend_ops; /* * List of clock devices. */ -struct clock clock_list[] = { +SECURITY_READ_ONLY_LATE(struct clock) clock_list[] = { /* SYSTEM_CLOCK */ { &sysclk_ops, 0, 0 }, diff --git a/osfmk/i386/AT386/model_dep.c b/osfmk/i386/AT386/model_dep.c index ebb29b76b..99b974a13 100644 --- a/osfmk/i386/AT386/model_dep.c +++ b/osfmk/i386/AT386/model_dep.c @@ -67,6 +67,10 @@ */ +#define __APPLE_API_PRIVATE 1 +#define __APPLE_API_UNSTABLE 1 +#include + #include #include @@ -77,7 +81,6 @@ #include #include #include -#include #include #include #include @@ -126,7 +129,7 @@ #include #include -#if DEBUG +#if DEBUG || DEVELOPMENT #define DPRINTF(x...) kprintf(x) #else #define DPRINTF(x...) @@ -134,6 +137,7 @@ static void machine_conf(void); void panic_print_symbol_name(vm_address_t search); +void RecordPanicStackshot(void); extern const char version[]; extern char osversion[]; @@ -149,8 +153,6 @@ extern int proc_pid(void *p); #define FP_LR_OFFSET64 ((uint32_t)8) #define FP_MAX_NUM_TO_EVALUATE (50) -int db_run_mode; - volatile int pbtcpu = -1; hw_lock_data_t pbtlock; /* backtrace print lock */ uint32_t pbtcnt = 0; @@ -169,6 +171,22 @@ typedef struct _cframe_t { static unsigned panic_io_port; static unsigned commit_paniclog_to_nvram; +boolean_t coprocessor_paniclog_flush = FALSE; + +#if DEVELOPMENT || DEBUG +struct kcdata_descriptor kc_panic_data; +static boolean_t begun_panic_stackshot = FALSE; + +vm_offset_t panic_stackshot_buf = 0; +size_t panic_stackshot_len = 0; + +extern kern_return_t do_stackshot(void *); +extern void kdp_snapshot_preflight(int pid, void *tracebuf, + uint32_t tracebuf_size, uint32_t flags, + kcdata_descriptor_t data_p, + boolean_t enable_faulting); +extern int kdp_stack_snapshot_bytes_traced(void); +#endif SECURITY_READ_ONLY_LATE(unsigned int) debug_boot_arg; @@ -177,7 +195,7 @@ SECURITY_READ_ONLY_LATE(unsigned int) debug_boot_arg; */ void print_one_backtrace(pmap_t pmap, vm_offset_t topfp, const char *cur_marker, - boolean_t is_64_bit, boolean_t nvram_format) + boolean_t is_64_bit) { int i = 0; addr64_t lr; @@ -219,9 +237,9 @@ print_one_backtrace(pmap_t pmap, vm_offset_t topfp, const char *cur_marker, } } else { if (is_64_bit) { - kdb_printf("%s\t Could not read LR from frame at 0x%016llx\n", cur_marker, fp + FP_LR_OFFSET64); + paniclog_append_noflush("%s\t Could not read LR from frame at 0x%016llx\n", cur_marker, fp + FP_LR_OFFSET64); } else { - kdb_printf("%s\t Could not read LR from frame at 0x%08x\n", cur_marker, (uint32_t)(fp + FP_LR_OFFSET)); + paniclog_append_noflush("%s\t Could not read LR from frame at 0x%08x\n", cur_marker, (uint32_t)(fp + FP_LR_OFFSET)); } break; } @@ -237,25 +255,17 @@ print_one_backtrace(pmap_t pmap, vm_offset_t topfp, const char *cur_marker, } } else { if (is_64_bit) { - kdb_printf("%s\t Could not read FP from frame at 0x%016llx\n", cur_marker, fp); + paniclog_append_noflush("%s\t Could not read FP from frame at 0x%016llx\n", cur_marker, fp); } else { - kdb_printf("%s\t Could not read FP from frame at 0x%08x\n", cur_marker, (uint32_t)fp); + paniclog_append_noflush("%s\t Could not read FP from frame at 0x%08x\n", cur_marker, (uint32_t)fp); } break; } - if (nvram_format) { - if (is_64_bit) { - kdb_printf("%s\t0x%016llx\n", cur_marker, lr); - } else { - kdb_printf("%s\t0x%08x\n", cur_marker, (uint32_t)lr); - } - } else { - if (is_64_bit) { - kdb_printf("%s\t lr: 0x%016llx fp: 0x%016llx\n", cur_marker, lr, fp); - } else { - kdb_printf("%s\t lr: 0x%08x fp: 0x%08x\n", cur_marker, (uint32_t)lr, (uint32_t)fp); - } + if (is_64_bit) { + paniclog_append_noflush("%s\t0x%016llx\n", cur_marker, lr); + } else { + paniclog_append_noflush("%s\t0x%08x\n", cur_marker, (uint32_t)lr); } } while ((++i < FP_MAX_NUM_TO_EVALUATE) && (fp != topfp)); } @@ -274,13 +284,9 @@ machine_startup(void) #if DEVELOPMENT || DEBUG if (debug_boot_arg & DB_HALT) halt_in_debugger=1; #endif - if (debug_boot_arg & DB_PRT) disable_debug_output=FALSE; - if (debug_boot_arg & DB_SLOG) systemLogDiags=TRUE; - if (debug_boot_arg & DB_LOG_PI_SCRN) logPanicDataToScreen=TRUE; #if KDEBUG_MOJO_TRACE if (debug_boot_arg & DB_PRT_KDEBUG) { kdebug_serial = TRUE; - disable_debug_output = FALSE; } #endif } else { @@ -655,6 +661,17 @@ efi_init(void) return; } +/* Returns TRUE if a page belongs to the EFI Runtime Services (code or data) */ +boolean_t +efi_valid_page(ppnum_t ppn) +{ + boot_args *args = (boot_args *)PE_state.bootArgs; + ppnum_t pstart = args->efiRuntimeServicesPageStart; + ppnum_t pend = pstart + args->efiRuntimeServicesPageCount; + + return pstart <= ppn && ppn < pend; +} + /* Remap EFI runtime areas. */ void hibernate_newruntime_map(void * map, vm_size_t map_size, uint32_t system_table_offset) @@ -840,180 +857,156 @@ uint64_t panic_restart_timeout = ~(0ULL); #define PANIC_RESTART_TIMEOUT (3ULL * NSEC_PER_SEC) -static void -machine_halt_cpu(void) { - uint64_t deadline; - - panic_io_port_read(); - - /* Halt here forever if we're not rebooting */ - if (!PE_reboot_on_panic() && panic_restart_timeout == ~(0ULL)) { - pmCPUHalt(PM_HALT_DEBUG); +void +RecordPanicStackshot() +{ +#if DEVELOPMENT || DEBUG + int err = 0, bytes_traced = 0, bytes_used = 0; + /* Try to take a stackshot once at panic time */ + if (begun_panic_stackshot) { return; } + begun_panic_stackshot = TRUE; - if (PE_reboot_on_panic()) - deadline = mach_absolute_time() + PANIC_RESTART_TIMEOUT; - else - deadline = mach_absolute_time() + panic_restart_timeout; - - while (mach_absolute_time() < deadline) - cpu_pause(); + if (panic_stackshot_buf == 0) { + kdb_printf("No stackshot buffer allocated, skipping...\n"); + return; + } - kprintf("Invoking PE_halt_restart\n"); - /* Attempt restart via ACPI RESET_REG; at the time of this - * writing, this is routine is chained through AppleSMC-> - * AppleACPIPlatform - */ - if (PE_halt_restart) - (*PE_halt_restart)(kPERestartCPU); - pmCPUHalt(PM_HALT_DEBUG); -} + err = kcdata_memory_static_init(&kc_panic_data, (mach_vm_address_t)panic_stackshot_buf, KCDATA_BUFFER_BEGIN_STACKSHOT, + PANIC_STACKSHOT_BUFSIZE, KCFLAG_USE_MEMCOPY); + if (err != KERN_SUCCESS) { + kdb_printf("Failed to initialize kcdata buffer for panic stackshot, skipping ...\n"); + return; + } -void -DebuggerWithContext( - __unused unsigned int reason, - __unused void *ctx, - const char *message, - uint64_t debugger_options_mask) -{ - if (debugger_options_mask != DEBUGGER_OPTION_NONE) { - kprintf("debugger options (%llx) not supported for desktop.\n", debugger_options_mask); + kdp_snapshot_preflight(-1, (void *) panic_stackshot_buf, PANIC_STACKSHOT_BUFSIZE, (STACKSHOT_GET_GLOBAL_MEM_STATS | STACKSHOT_SAVE_LOADINFO | STACKSHOT_KCDATA_FORMAT | + STACKSHOT_ENABLE_BT_FAULTING | STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_FROM_PANIC | STACKSHOT_NO_IO_STATS + | STACKSHOT_THREAD_WAITINFO), &kc_panic_data, 0); + err = do_stackshot(NULL); + bytes_traced = (int) kdp_stack_snapshot_bytes_traced(); + if (bytes_traced > 0 && !err) { + panic_stackshot_len = bytes_traced; + kdb_printf("Panic stackshot succeeded, length: %u bytes\n", bytes_traced); + } else { + bytes_used = (int) kcdata_memory_get_used_bytes(&kc_panic_data); + if (bytes_used > 0) { + kdb_printf("Panic stackshot incomplete, consumed %u bytes\n", bytes_used); + } else { + kdb_printf("Panic stackshot incomplete, consumed %u bytes, error : %d \n", bytes_used, err); + } } - Debugger(message); +#endif /* DEVELOPMENT || DEBUG */ + return; } void -Debugger( - const char *message) +SavePanicInfo( + __unused const char *message, uint64_t panic_options) { - unsigned long pi_size = 0; void *stackptr; int cn = cpu_number(); - boolean_t old_doprnt_hide_pointers = doprnt_hide_pointers; - - hw_atomic_add(&debug_mode, 1); - if (!panic_is_inited) { - postcode(PANIC_HLT); - asm("hlt"); - } - - doprnt_hide_pointers = FALSE; - - printf("Debugger called: <%s>\n", message); - kprintf("Debugger called: <%s>\n", message); - /* - * Skip the graphical panic box if no panic string. - * This is the case if we're being called from - * host_reboot(,HOST_REBOOT_DEBUGGER) - * as a quiet way into the debugger. + * Issue an I/O port read if one has been requested - this is an event logic + * analyzers can use as a trigger point. */ + panic_io_port_read(); - if (panicstr) { - disable_preemption(); - -/* Issue an I/O port read if one has been requested - this is an event logic - * analyzers can use as a trigger point. - */ - panic_io_port_read(); + /* Obtain current frame pointer */ + __asm__ volatile("movq %%rbp, %0" : "=m" (stackptr)); - /* Obtain current frame pointer */ - __asm__ volatile("movq %%rbp, %0" : "=m" (stackptr)); + /* Print backtrace - callee is internally synchronized */ + if (panic_options & DEBUGGER_OPTION_INITPROC_PANIC) { + /* Special handling of launchd died panics */ + print_launchd_info(); + } else { + panic_i386_backtrace(stackptr, ((panic_double_fault_cpu == cn) ? 80: 48), NULL, FALSE, NULL); + } - /* Print backtrace - callee is internally synchronized */ - if (strncmp(panicstr, LAUNCHD_CRASHED_PREFIX, strlen(LAUNCHD_CRASHED_PREFIX)) == 0) { - /* Special handling of launchd died panics */ - print_launchd_info(); - } else { - panic_i386_backtrace(stackptr, ((panic_double_fault_cpu == cn) ? 80: 48), NULL, FALSE, NULL); - } + if (panic_options & DEBUGGER_OPTION_COPROC_INITIATED_PANIC) { + panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC; + } - /* everything should be printed now so copy to NVRAM - */ + /* Flush the paniclog */ + paniclog_flush(); - if( debug_buf_size > 0) { - /* Optionally sync the panic log, if any, to NVRAM - * This is the default. - */ - if (commit_paniclog_to_nvram) { - unsigned int bufpos; - uintptr_t cr0; - - debug_putc(0); - - /* Now call the compressor */ - /* XXX Consider using the WKdm compressor in the - * future, rather than just packing - would need to - * be co-ordinated with crashreporter, which decodes - * this post-restart. The compressor should be - * capable of in-place compression. - */ - bufpos = packA(debug_buf, - (unsigned int) (debug_buf_ptr - debug_buf), debug_buf_size); - /* If compression was successful, - * use the compressed length - */ - pi_size = bufpos ? bufpos : (unsigned) (debug_buf_ptr - debug_buf); - - /* Save panic log to non-volatile store - * Panic info handler must truncate data that is - * too long for this platform. - * This call must save data synchronously, - * since we can subsequently halt the system. - */ - - -/* The following sequence is a workaround for: - * SnowLeopard10A67: AppleEFINVRAM should not invoke - * any routines that use floating point (MMX in this case) when saving panic - * logs to nvram/flash. - */ - cr0 = get_cr0(); - clear_ts(); + /* Try to take a panic stackshot */ + RecordPanicStackshot(); +} - kprintf("Attempting to commit panic log to NVRAM\n"); - pi_size = PESavePanicInfo((unsigned char *)debug_buf, - (uint32_t)pi_size ); - set_cr0(cr0); +void +paniclog_flush() +{ + unsigned long pi_size = 0; - /* Uncompress in-place, to permit examination of - * the panic log by debuggers. - */ + assert(panic_info != NULL); + panic_info->mph_panic_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->mph_panic_log_offset; - if (bufpos) { - unpackA(debug_buf, bufpos); - } - } - } + /* + * If we've detected that we're on a co-processor system we flush the panic log via the kPEPanicSync + * panic callbacks, otherwise we flush via nvram (unless that has been disabled). + */ + if (coprocessor_paniclog_flush) { + /* Only need to calculate the CRC for co-processor platforms */ + panic_info->mph_crc = crc32(0L, &panic_info->mph_version, (debug_buf_size - offsetof(struct macos_panic_header, mph_version))); + + PESavePanicInfoAction(debug_buf, debug_buf_size); + } else if(commit_paniclog_to_nvram) { + assert(debug_buf_size != 0); + unsigned int bufpos; + uintptr_t cr0; + + debug_putc(0); + + + /* + * Now call the compressor + * XXX Consider using the WKdm compressor in the + * future, rather than just packing - would need to + * be co-ordinated with crashreporter, which decodes + * this post-restart. The compressor should be + * capable of in-place compression. + * + * Don't include the macOS panic header (for co-processor systems only) + */ + bufpos = packA(debug_buf_base, (unsigned int) (debug_buf_ptr - debug_buf_base), + debug_buf_size); + /* + * If compression was successful, use the compressed length + */ + pi_size = bufpos ? bufpos : (unsigned) (debug_buf_ptr - debug_buf_base); - if (!panicDebugging && !kdp_has_polled_corefile()) { - unsigned cnum; - /* Clear the MP rendezvous function lock, in the event - * that a panic occurred while in that codepath. - */ - mp_rendezvous_break_lock(); - - /* Non-maskably interrupt all other processors - * If a restart timeout is specified, this processor - * will attempt a restart. - */ - kprintf("Invoking machine_halt_cpu on CPU %d\n", cn); - for (cnum = 0; cnum < real_ncpus; cnum++) { - if (cnum != (unsigned) cn) { - cpu_NMI_interrupt(cnum); - } - } - machine_halt_cpu(); - /* NOT REACHED */ + /* + * The following sequence is a workaround for: + * SnowLeopard10A67: AppleEFINVRAM should not invoke + * any routines that use floating point (MMX in this case) when saving panic + * logs to nvram/flash. + */ + cr0 = get_cr0(); + clear_ts(); + + /* + * Save panic log to non-volatile store + * Panic info handler must truncate data that is + * too long for this platform. + * This call must save data synchronously, + * since we can subsequently halt the system. + */ + kprintf("Attempting to commit panic log to NVRAM\n"); + pi_size = PESavePanicInfo((unsigned char *)debug_buf_base, + (uint32_t)pi_size ); + set_cr0(cr0); + + /* + * Uncompress in-place, to permit examination of + * the panic log by debuggers. + */ + if (bufpos) { + unpackA(debug_buf_base, bufpos); } - } - - doprnt_hide_pointers = old_doprnt_hide_pointers; - __asm__("int3"); - hw_atomic_sub(&debug_mode, 1); + } } char * @@ -1089,9 +1082,9 @@ panic_print_macho_symbol_name(kernel_mach_header_t *mh, vm_address_t search, con if (bestsym != NULL) { if (diff != 0) { - kdb_printf("%s : %s + 0x%lx", module_name, bestsym, (unsigned long)diff); + paniclog_append_noflush("%s : %s + 0x%lx", module_name, bestsym, (unsigned long)diff); } else { - kdb_printf("%s : %s", module_name, bestsym); + paniclog_append_noflush("%s : %s", module_name, bestsym); } return 1; } @@ -1115,7 +1108,7 @@ panic_print_kmod_symbol_name(vm_address_t search) { kernel_mach_header_t *header = (kernel_mach_header_t *)(uintptr_t) summary->address; if (panic_print_macho_symbol_name(header, search, summary->name) == 0) { - kdb_printf("%s + %llu", summary->name, (unsigned long)search - summary->address); + paniclog_append_noflush("%s + %llu", summary->name, (unsigned long)search - summary->address); } break; } @@ -1173,12 +1166,12 @@ panic_i386_backtrace(void *_frame, int nframes, const char *msg, boolean_t regdu PE_parse_boot_argn("keepsyms", &keepsyms, sizeof (keepsyms)); if (msg != NULL) { - kdb_printf("%s", msg); + paniclog_append_noflush("%s", msg); } if ((regdump == TRUE) && (regs != NULL)) { x86_saved_state64_t *ss64p = saved_state64(regs); - kdb_printf( + paniclog_append_noflush( "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n" "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n" "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n" @@ -1193,7 +1186,7 @@ panic_i386_backtrace(void *_frame, int nframes, const char *msg, boolean_t regdu PC = ss64p->isf.rip; } - kdb_printf("Backtrace (CPU %d), " + paniclog_append_noflush("Backtrace (CPU %d), " #if PRINT_ARGS_FROM_STACK_FRAME "Frame : Return Address (4 potential args on stack)\n", cn); #else @@ -1207,23 +1200,23 @@ panic_i386_backtrace(void *_frame, int nframes, const char *msg, boolean_t regdu break; if (curframep & 0x3) { - kdb_printf("Unaligned frame\n"); + paniclog_append_noflush("Unaligned frame\n"); goto invalid; } if (!kvtophys(curframep) || !kvtophys(curframep + sizeof(cframe_t) - 1)) { - kdb_printf("No mapping exists for frame pointer\n"); + paniclog_append_noflush("No mapping exists for frame pointer\n"); goto invalid; } - kdb_printf("%p : 0x%lx ", frame, frame->caller); + paniclog_append_noflush("%p : 0x%lx ", frame, frame->caller); if (frame_index < DUMPFRAMES) raddrs[frame_index] = frame->caller; #if PRINT_ARGS_FROM_STACK_FRAME if (kvtophys((vm_offset_t)&(frame->args[3]))) - kdb_printf("(0x%x 0x%x 0x%x 0x%x) ", + paniclog_append_noflush("(0x%x 0x%x 0x%x 0x%x) ", frame->args[0], frame->args[1], frame->args[2], frame->args[3]); #endif @@ -1236,18 +1229,18 @@ panic_i386_backtrace(void *_frame, int nframes, const char *msg, boolean_t regdu if (keepsyms) panic_print_symbol_name((vm_address_t)frame->caller); - kdb_printf("\n"); + paniclog_append_noflush("\n"); frame = frame->prev; } if (frame_index >= nframes) - kdb_printf("\tBacktrace continues...\n"); + paniclog_append_noflush("\tBacktrace continues...\n"); goto out; invalid: - kdb_printf("Backtrace terminated-invalid frame pointer %p\n",frame); + paniclog_append_noflush("Backtrace terminated-invalid frame pointer %p\n",frame); out: /* Identify kernel modules in the backtrace and display their @@ -1309,7 +1302,7 @@ print_threads_registers(thread_t thread) x86_saved_state_t *savestate; savestate = get_user_regs(thread); - kdb_printf( + paniclog_append_noflush( "\nRAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n" "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n" "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n" @@ -1336,13 +1329,13 @@ print_tasks_user_threads(task_t task) for (j = 0, thread = (thread_t) queue_first(&task->threads); j < task->thread_count; ++j, thread = (thread_t) queue_next(&thread->task_threads)) { - kdb_printf("Thread %d: %p\n", j, thread); + paniclog_append_noflush("Thread %d: %p\n", j, thread); pmap = get_task_pmap(task); savestate = get_user_regs(thread); rbp = savestate->ss_64.rbp; - kdb_printf("\t0x%016llx\n", savestate->ss_64.isf.rip); - print_one_backtrace(pmap, (vm_offset_t)rbp, cur_marker, TRUE, TRUE); - kdb_printf("\n"); + paniclog_append_noflush("\t0x%016llx\n", savestate->ss_64.isf.rip); + print_one_backtrace(pmap, (vm_offset_t)rbp, cur_marker, TRUE); + paniclog_append_noflush("\n"); } } @@ -1357,7 +1350,7 @@ print_thread_num_that_crashed(task_t task) ++j, thread = (thread_t) queue_next(&thread->task_threads)) { if (c_thread == thread) { - kdb_printf("\nThread %d crashed\n", j); + paniclog_append_noflush("\nThread %d crashed\n", j); break; } } @@ -1402,7 +1395,7 @@ void print_uuid_info(task_t task) char *current_uuid_buffer = NULL; /* Copy in the UUID info array. It may be nonresident, in which case just fix up nloadinfos to 0 */ - kdb_printf("\nuuid info:\n"); + paniclog_append_noflush("\nuuid info:\n"); while (uuid_array_size) { if (uuid_array_size <= PANICLOG_UUID_BUF_SIZE) { uuid_copy_size = uuid_array_size; @@ -1413,7 +1406,7 @@ void print_uuid_info(task_t task) } if (have_pmap && !debug_copyin(task->map->pmap, uuid_info_addr, uuidbufptr, uuid_copy_size)) { - kdb_printf("Error!! Failed to copy UUID info for task %p pid %d\n", task, task_pid); + paniclog_append_noflush("Error!! Failed to copy UUID info for task %p pid %d\n", task, task_pid); uuid_image_count = 0; break; } @@ -1421,10 +1414,10 @@ void print_uuid_info(task_t task) if (uuid_image_count > 0) { current_uuid_buffer = uuidbufptr; for (k = 0; k < uuid_image_count; k++) { - kdb_printf(" %#llx", *(uint64_t *)current_uuid_buffer); + paniclog_append_noflush(" %#llx", *(uint64_t *)current_uuid_buffer); current_uuid_buffer += sizeof(uint64_t); uint8_t *uuid = (uint8_t *)current_uuid_buffer; - kdb_printf("\tuuid = <%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x>\n", + paniclog_append_noflush("\tuuid = <%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x>\n", uuid[0], uuid[1], uuid[2], uuid[3], uuid[4], uuid[5], uuid[6], uuid[7], uuid[8], uuid[9], uuid[10], uuid[11], uuid[12], uuid[13], uuid[14], uuid[15]); current_uuid_buffer += 16; diff --git a/osfmk/i386/Diagnostics.c b/osfmk/i386/Diagnostics.c index bd2ca2140..90eeb4192 100644 --- a/osfmk/i386/Diagnostics.c +++ b/osfmk/i386/Diagnostics.c @@ -75,6 +75,10 @@ #include #include +#if MONOTONIC +#include +#endif /* MONOTONIC */ + #define PERMIT_PERMCHECK (0) diagWork dgWork; @@ -230,6 +234,7 @@ diagCall64(x86_saved_state_t * state) */ switch (cpuid_cpufamily()) { case CPUFAMILY_INTEL_SKYLAKE: + case CPUFAMILY_INTEL_KABYLAKE: ia_perf_limits = MSR_IA32_IA_PERF_LIMIT_REASONS_SKL; break; default: @@ -276,14 +281,16 @@ diagCall64(x86_saved_state_t * state) cest.citime_total = cpu_data_ptr[i]->cpu_itime_total; cest.crtime_total = cpu_data_ptr[i]->cpu_rtime_total; - cest.cpu_idle_exits = cpu_data_ptr[i]->cpu_idle_exits; - cest.cpu_insns = cpu_data_ptr[i]->cpu_cur_insns; - cest.cpu_ucc = cpu_data_ptr[i]->cpu_cur_ucc; - cest.cpu_urc = cpu_data_ptr[i]->cpu_cur_urc; + cest.cpu_idle_exits = cpu_data_ptr[i]->cpu_idle_exits; +#if MONOTONIC + cest.cpu_insns = cpu_data_ptr[i]->cpu_monotonic.mtc_counts[MT_CORE_INSTRS]; + cest.cpu_ucc = cpu_data_ptr[i]->cpu_monotonic.mtc_counts[MT_CORE_CYCLES]; + cest.cpu_urc = cpu_data_ptr[i]->cpu_monotonic.mtc_counts[MT_CORE_REFCYCLES]; +#endif /* MONOTONIC */ #if DIAG_ALL_PMCS bcopy(&cpu_data_ptr[i]->cpu_gpmcs[0], &cest.gpmcs[0], sizeof(cest.gpmcs)); #endif /* DIAG_ALL_PMCS */ - (void) ml_set_interrupts_enabled(TRUE); + (void) ml_set_interrupts_enabled(TRUE); copyout(&cest, curpos, sizeof(cest)); curpos += sizeof(cest); @@ -366,9 +373,13 @@ void cpu_powerstats(__unused void *arg) { cdp->cpu_c7res = ((uint64_t)ch << 32) | cl; if (diag_pmc_enabled) { +#if MONOTONIC + mt_update_fixed_counts(); +#else /* MONOTONIC */ uint64_t insns = read_pmc(FIXED_PMC0); uint64_t ucc = read_pmc(FIXED_PMC1); uint64_t urc = read_pmc(FIXED_PMC2); +#endif /* !MONOTONIC */ #if DIAG_ALL_PMCS int i; @@ -376,13 +387,16 @@ void cpu_powerstats(__unused void *arg) { cdp->cpu_gpmcs[i] = read_pmc(i); } #endif /* DIAG_ALL_PMCS */ +#if !MONOTONIC cdp->cpu_cur_insns = insns; cdp->cpu_cur_ucc = ucc; cdp->cpu_cur_urc = urc; +#endif /* !MONOTONIC */ } } void cpu_pmc_control(void *enablep) { +#if !MONOTONIC boolean_t enable = *(boolean_t *)enablep; cpu_data_t *cdp = current_cpu_datap(); @@ -397,4 +411,7 @@ void cpu_pmc_control(void *enablep) { set_cr4((get_cr4() & ~CR4_PCE)); } cdp->cpu_fixed_pmcs_enabled = enable; +#else /* !MONOTONIC */ +#pragma unused(enablep) +#endif /* MONOTONIC */ } diff --git a/osfmk/i386/Makefile b/osfmk/i386/Makefile index b01634828..4df5e8510 100644 --- a/osfmk/i386/Makefile +++ b/osfmk/i386/Makefile @@ -17,6 +17,7 @@ EXPORT_ONLY_FILES = \ cpu_topology.h \ cpuid.h \ eflags.h \ + fpu.h \ io_map_entries.h \ lapic.h \ lock.h \ diff --git a/osfmk/i386/acpi.c b/osfmk/i386/acpi.c index 955301c6d..8d4ac8040 100644 --- a/osfmk/i386/acpi.c +++ b/osfmk/i386/acpi.c @@ -68,6 +68,10 @@ #include #include +#if MONOTONIC +#include +#endif /* MONOTONIC */ + #if CONFIG_SLEEP extern void acpi_sleep_cpu(acpi_sleep_callback, void * refcon); extern void acpi_wake_prot(void); @@ -169,6 +173,7 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) acpi_hibernate_callback_data_t data; #endif boolean_t did_hibernate; + cpu_data_t *cdp = current_cpu_datap(); unsigned int cpu; kern_return_t rc; unsigned int my_cpu; @@ -176,13 +181,13 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) uint64_t elapsed = 0; uint64_t elapsed_trace_start = 0; - kprintf("acpi_sleep_kernel hib=%d, cpu=%d\n", - current_cpu_datap()->cpu_hibernate, cpu_number()); + my_cpu = cpu_number(); + kprintf("acpi_sleep_kernel hib=%d, cpu=%d\n", cdp->cpu_hibernate, + my_cpu); - /* Get all CPUs to be in the "off" state */ - my_cpu = cpu_number(); + /* Get all CPUs to be in the "off" state */ for (cpu = 0; cpu < real_ncpus; cpu += 1) { - if (cpu == my_cpu) + if (cpu == my_cpu) continue; rc = pmCPUExitHaltToOff(cpu); if (rc != KERN_SUCCESS) @@ -198,6 +203,10 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) data.refcon = refcon; #endif +#if MONOTONIC + mt_cpu_down(cdp); +#endif /* MONOTONIC */ + /* Save power management timer state */ pmTimerSave(); @@ -211,7 +220,7 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) */ clear_ts(); - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_START, 0, 0, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_START); save_kdebug_enable = kdebug_enable; kdebug_enable = 0; @@ -264,8 +273,8 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) did_hibernate = FALSE; } - /* Re-enable mode (including 64-bit if applicable) */ - cpu_mode_init(current_cpu_datap()); + /* Re-enable fast syscall */ + cpu_syscall_init(current_cpu_datap()); #if CONFIG_MCA /* Re-enable machine check handling */ @@ -303,6 +312,14 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) if (lapic_probe()) lapic_configure(); +#if KASAN + /* + * The sleep implementation uses indirect noreturn calls, so we miss stack + * unpoisoning. Do it explicitly. + */ + __asan_handle_no_return(); +#endif + #if HIBERNATION hibernate_rebuild_vm_structs(); #endif @@ -329,18 +346,16 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) #if HIBERNATION if (did_hibernate) { - elapsed += mach_absolute_time() - start; - - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_START, elapsed, elapsed_trace_start, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_START); hibernate_machine_init(); - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_END, 0, 0, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_END); current_cpu_datap()->cpu_hibernate = 0; - - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_END, 0, 0, 0, 0, 0); - } else + } #endif /* HIBERNATION */ - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_END, 0, 0, 0, 0, 0); + + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_END, start, elapsed, + elapsed_trace_start, acpi_wake_abstime); /* Restore power management register state */ pmCPUMarkRunning(current_cpu_datap()); @@ -398,6 +413,8 @@ acpi_idle_kernel(acpi_sleep_callback func, void *refcon) * Call back to caller to indicate that interrupts will remain * disabled while we deep idle, wake and return. */ + IOCPURunPlatformQuiesceActions(); + func(refcon); acpi_idle_abstime = mach_absolute_time(); @@ -441,8 +458,11 @@ acpi_idle_kernel(acpi_sleep_callback func, void *refcon) /* Like S3 sleep, turn on tracing if trace_wake boot-arg is present */ if (kdebug_enable == 0) { - if (wake_nkdbufs) + if (wake_nkdbufs) { + __kdebug_only uint64_t start = mach_absolute_time(); kdebug_trace_start(wake_nkdbufs, NULL, TRUE); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 15), start); + } } IOCPURunPlatformActiveActions(); diff --git a/osfmk/i386/bsd_i386.c b/osfmk/i386/bsd_i386.c index 9b7094974..9dd47783c 100644 --- a/osfmk/i386/bsd_i386.c +++ b/osfmk/i386/bsd_i386.c @@ -26,7 +26,6 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifdef MACH_BSD -#include #include #include @@ -315,6 +314,12 @@ machdep_syscall(x86_saved_state_t *state) DEBUG_KPRINT_SYSCALL_MDEP("machdep_syscall: retval=%u\n", regs->eax); +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_get_kernel_state(current_thread())->allocation_name; + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + throttle_lowpri_io(1); thread_exception_return(); @@ -361,6 +366,12 @@ machdep_syscall64(x86_saved_state_t *state) DEBUG_KPRINT_SYSCALL_MDEP("machdep_syscall: retval=%llu\n", regs->rax); +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_get_kernel_state(current_thread())->allocation_name; + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + throttle_lowpri_io(1); thread_exception_return(); @@ -477,6 +488,12 @@ mach_call_munger(x86_saved_state_t *state) regs->eax = retval; +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_get_kernel_state(current_thread())->allocation_name; + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + throttle_lowpri_io(1); #if PROC_REF_DEBUG @@ -561,6 +578,12 @@ mach_call_munger64(x86_saved_state_t *state) MACHDBG_CODE(DBG_MACH_EXCP_SC,(call_number)) | DBG_FUNC_END, regs->rax, 0, 0, 0, 0); +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_get_kernel_state(current_thread())->allocation_name; + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + throttle_lowpri_io(1); #if PROC_REF_DEBUG diff --git a/osfmk/i386/bsd_i386_native.c b/osfmk/i386/bsd_i386_native.c index 8e49e214d..4ec100eff 100644 --- a/osfmk/i386/bsd_i386_native.c +++ b/osfmk/i386/bsd_i386_native.c @@ -25,7 +25,6 @@ * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#include #include #include diff --git a/osfmk/i386/commpage/commpage.c b/osfmk/i386/commpage/commpage.c index 6dae08567..81b962c1b 100644 --- a/osfmk/i386/commpage/commpage.c +++ b/osfmk/i386/commpage/commpage.c @@ -62,6 +62,7 @@ #include #include #include +#include #include @@ -100,6 +101,9 @@ static commpage_address_t commPageBaseOffset; // subtract from 32-bit runtime ad static commpage_time_data *time_data32 = NULL; static commpage_time_data *time_data64 = NULL; +static new_commpage_timeofday_data_t *gtod_time_data32 = NULL; +static new_commpage_timeofday_data_t *gtod_time_data64 = NULL; + decl_simple_lock_data(static,commpage_active_cpus_lock); @@ -126,11 +130,12 @@ commpage_allocate( if (submap == NULL) panic("commpage submap is null"); - if ((kr = vm_map(kernel_map, + if ((kr = vm_map_kernel(kernel_map, &kernel_addr, area_used, 0, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_OSFMK), + VM_FLAGS_ANYWHERE, + VM_KERN_MEMORY_OSFMK, NULL, 0, FALSE, @@ -139,10 +144,10 @@ commpage_allocate( VM_INHERIT_NONE))) panic("cannot allocate commpage %d", kr); - if ((kr = vm_map_wire(kernel_map, + if ((kr = vm_map_wire_kernel(kernel_map, kernel_addr, kernel_addr+area_used, - VM_PROT_DEFAULT|VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK), + VM_PROT_DEFAULT, VM_KERN_MEMORY_OSFMK, FALSE))) panic("cannot wire commpage: %d", kr); @@ -166,11 +171,12 @@ commpage_allocate( NULL ))) // parent_entry (what is this?) panic("cannot make entry for commpage %d", kr); - if ((kr = vm_map_64( submap, // target map (shared submap) + if ((kr = vm_map_64_kernel( submap, // target map (shared submap) &zero, // address (map into 1st page in submap) area_used, // size 0, // mask VM_FLAGS_FIXED, // flags (it must be 1st page in submap) + VM_KERN_MEMORY_NONE, handle, // port is the memory entry we just made 0, // offset (map 1st page in memory entry) FALSE, // copy @@ -306,10 +312,32 @@ commpage_init_cpu_capabilities( void ) setif(bits, kHasADX, cpuid_features() & CPUID_LEAF7_FEATURE_ADX); +#if 0 /* The kernel doesn't support MPX or SGX */ setif(bits, kHasMPX, cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_MPX); setif(bits, kHasSGX, cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_SGX); +#endif + +#if !defined(RC_HIDE_XNU_J137) + if (ml_fpu_avx512_enabled()) { + setif(bits, kHasAVX512F, cpuid_leaf7_features() & + CPUID_LEAF7_FEATURE_AVX512F); + setif(bits, kHasAVX512CD, cpuid_leaf7_features() & + CPUID_LEAF7_FEATURE_AVX512CD); + setif(bits, kHasAVX512DQ, cpuid_leaf7_features() & + CPUID_LEAF7_FEATURE_AVX512DQ); + setif(bits, kHasAVX512BW, cpuid_leaf7_features() & + CPUID_LEAF7_FEATURE_AVX512BW); + setif(bits, kHasAVX512VL, cpuid_leaf7_features() & + CPUID_LEAF7_FEATURE_AVX512VL); + setif(bits, kHasAVX512IFMA, cpuid_leaf7_features() & + CPUID_LEAF7_FEATURE_AVX512IFMA); + setif(bits, kHasAVX512VBMI, cpuid_leaf7_features() & + CPUID_LEAF7_FEATURE_AVX512VBMI); + } + +#endif /* not RC_HIDE_XNU_J137 */ uint64_t misc_enable = rdmsr64(MSR_IA32_MISC_ENABLE); setif(bits, kHasENFSTRG, (misc_enable & 1ULL) && (cpuid_leaf7_features() & @@ -403,6 +431,7 @@ commpage_populate_one( size_t area_used, // _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED commpage_address_t base_offset, // will become commPageBaseOffset commpage_time_data** time_data, // &time_data32 or &time_data64 + new_commpage_timeofday_data_t** gtod_time_data, // >od_time_data32 or >od_time_data64 const char* signature, // "commpage 32-bit" or "commpage 64-bit" vm_prot_t uperm) { @@ -419,6 +448,7 @@ commpage_populate_one( commPageBaseOffset = base_offset; *time_data = commpage_addr_of( _COMM_PAGE_TIME_DATA_START ); + *gtod_time_data = commpage_addr_of( _COMM_PAGE_NEWTIMEOFDAY_DATA ); /* Stuff in the constants. We move things into the comm page in strictly * ascending order, so we can check for overlap and panic if so. @@ -475,6 +505,7 @@ commpage_populate( void ) _COMM_PAGE32_AREA_USED, _COMM_PAGE32_BASE_ADDRESS, &time_data32, + >od_time_data32, "commpage 32-bit", VM_PROT_READ); #ifndef __LP64__ @@ -482,6 +513,7 @@ commpage_populate( void ) _COMM_PAGE32_AREA_USED/INTEL_PGBYTES); #endif time_data64 = time_data32; /* if no 64-bit commpage, point to 32-bit */ + gtod_time_data64 = gtod_time_data32; if (_cpu_capabilities & k64Bit) { commpage_populate_one( commpage64_map, @@ -489,6 +521,7 @@ commpage_populate( void ) _COMM_PAGE64_AREA_USED, _COMM_PAGE32_START_ADDRESS, /* commpage address are relative to 32-bit commpage placement */ &time_data64, + >od_time_data64, "commpage 64-bit", VM_PROT_READ); #ifndef __LP64__ @@ -611,51 +644,42 @@ commpage_set_nanotime( p64->nt_generation = next_gen; } - -/* Disable commpage gettimeofday(), forcing commpage to call through to the kernel. */ - -void -commpage_disable_timestamp( void ) -{ - time_data32->gtod_generation = 0; - time_data64->gtod_generation = 0; -} - - /* Update commpage gettimeofday() information. As with nanotime(), we interleave * updates to the 32- and 64-bit commpage, in order to keep time more nearly in sync * between the two environments. * * This routine must be serializeed by some external means, ie a lock. */ - - void - commpage_set_timestamp( - uint64_t abstime, - uint64_t secs ) + +void +commpage_set_timestamp( + uint64_t abstime, + uint64_t sec, + uint64_t frac, + uint64_t scale, + uint64_t tick_per_sec) { - commpage_time_data *p32 = time_data32; - commpage_time_data *p64 = time_data64; - static uint32_t generation = 0; - uint32_t next_gen; - - next_gen = ++generation; - if (next_gen == 0) - next_gen = ++generation; - - p32->gtod_generation = 0; /* mark invalid, so commpage won't try to use it */ - p64->gtod_generation = 0; - - p32->gtod_ns_base = abstime; - p64->gtod_ns_base = abstime; + new_commpage_timeofday_data_t *p32 = gtod_time_data32; + new_commpage_timeofday_data_t *p64 = gtod_time_data64; - p32->gtod_sec_base = secs; - p64->gtod_sec_base = secs; - - p32->gtod_generation = next_gen; /* mark data as valid */ - p64->gtod_generation = next_gen; -} + p32->TimeStamp_tick = 0x0ULL; + p64->TimeStamp_tick = 0x0ULL; + + p32->TimeStamp_sec = sec; + p64->TimeStamp_sec = sec; + + p32->TimeStamp_frac = frac; + p64->TimeStamp_frac = frac; + p32->Ticks_scale = scale; + p64->Ticks_scale = scale; + + p32->Ticks_per_sec = tick_per_sec; + p64->Ticks_per_sec = tick_per_sec; + + p32->TimeStamp_tick = abstime; + p64->TimeStamp_tick = abstime; +} /* Update _COMM_PAGE_MEMORY_PRESSURE. Called periodically from vm's compute_memory_pressure() */ @@ -804,25 +828,27 @@ commpage_update_mach_approximate_time(uint64_t abstime) cp = commPagePtr32; if ( cp ) { cp += (_COMM_PAGE_APPROX_TIME - _COMM_PAGE32_BASE_ADDRESS); - saved_data = *(uint64_t *)cp; + saved_data = atomic_load_explicit((_Atomic uint64_t *)(uintptr_t)cp, memory_order_relaxed); if (saved_data < abstime) { /* ignoring the success/fail return value assuming that * if the value has been updated since we last read it, * "someone" has a newer timestamp than us and ours is * now invalid. */ - OSCompareAndSwap64(saved_data, abstime, (uint64_t *)cp); + atomic_compare_exchange_strong_explicit((_Atomic uint64_t *)(uintptr_t)cp, + &saved_data, abstime, memory_order_relaxed, memory_order_relaxed); } } cp = commPagePtr64; if ( cp ) { cp += (_COMM_PAGE_APPROX_TIME - _COMM_PAGE32_START_ADDRESS); - saved_data = *(uint64_t *)cp; + saved_data = atomic_load_explicit((_Atomic uint64_t *)(uintptr_t)cp, memory_order_relaxed); if (saved_data < abstime) { /* ignoring the success/fail return value assuming that * if the value has been updated since we last read it, * "someone" has a newer timestamp than us and ours is * now invalid. */ - OSCompareAndSwap64(saved_data, abstime, (uint64_t *)cp); + atomic_compare_exchange_strong_explicit((_Atomic uint64_t *)(uintptr_t)cp, + &saved_data, abstime, memory_order_relaxed, memory_order_relaxed); } } #else diff --git a/osfmk/i386/commpage/commpage.h b/osfmk/i386/commpage/commpage.h index ac600307e..e4a2c7ac3 100644 --- a/osfmk/i386/commpage/commpage.h +++ b/osfmk/i386/commpage/commpage.h @@ -142,12 +142,11 @@ typedef volatile struct commpage_time_data { uint64_t gtod_sec_base; // _COMM_PAGE_GTOD_SEC_BASE } commpage_time_data; - extern char *commPagePtr32; // virt address of 32-bit commpage in kernel map extern char *commPagePtr64; // ...and of 64-bit commpage -extern void commpage_set_timestamp(uint64_t abstime, uint64_t secs); -extern void commpage_disable_timestamp( void ); +extern void commpage_set_timestamp(uint64_t abstime, uint64_t sec, uint64_t frac, uint64_t scale, uint64_t tick_per_sec); +#define commpage_disable_timestamp() commpage_set_timestamp( 0, 0, 0, 0, 0 ); extern void commpage_set_nanotime(uint64_t tsc_base, uint64_t ns_base, uint32_t scale, uint32_t shift); extern void commpage_set_memory_pressure(unsigned int pressure); extern void commpage_set_spin_count(unsigned int count); diff --git a/osfmk/i386/cpu.c b/osfmk/i386/cpu.c index 7de91627a..84cf06615 100644 --- a/osfmk/i386/cpu.c +++ b/osfmk/i386/cpu.c @@ -155,7 +155,7 @@ cpu_exit_wait( * a timeout if long-running interrupt were to occur here. */ intrs_enabled = ml_set_interrupts_enabled(FALSE); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); /* Set a generous timeout of several seconds (in TSC ticks) */ tsc_timeout = rdtsc64() + (10ULL * 1000 * 1000 * 1000); while ((cdp->lcpu.state != LCPU_HALT) @@ -167,7 +167,7 @@ cpu_exit_wait( if (rdtsc64() > tsc_timeout) panic("cpu_exit_wait(%d) timeout", cpu); ml_set_interrupts_enabled(FALSE); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); } simple_unlock(&x86_topo_lock); ml_set_interrupts_enabled(intrs_enabled); diff --git a/osfmk/i386/cpu_capabilities.h b/osfmk/i386/cpu_capabilities.h index 0f1b7dadd..dff0ae545 100644 --- a/osfmk/i386/cpu_capabilities.h +++ b/osfmk/i386/cpu_capabilities.h @@ -73,10 +73,20 @@ #define kHasADX 0x0000000400000000ULL #define kHasMPX 0x0000001000000000ULL #define kHasSGX 0x0000002000000000ULL +#if !defined(RC_HIDE_XNU_J137) +#define kHasAVX512F 0x0000004000000000ULL +#define kHasAVX512CD 0x0000008000000000ULL +#define kHasAVX512DQ 0x0000010000000000ULL +#define kHasAVX512BW 0x0000020000000000ULL +#define kHasAVX512IFMA 0x0000040000000000ULL +#define kHasAVX512VBMI 0x0000080000000000ULL +#define kHasAVX512VL 0x0000100000000000ULL +#endif /* not RC_HIDE_XNU_J137 */ #ifndef __ASSEMBLER__ #include +#include __BEGIN_DECLS extern uint64_t _get_cpu_capabilities( void ); @@ -208,6 +218,7 @@ int _NumCPUs( void ) /* Align following entries to next cache line */ #define _COMM_PAGE_CONT_TIMEBASE (_COMM_PAGE_START_ADDRESS+0x0C0) /* used by mach_continuous_time() */ #define _COMM_PAGE_BOOTTIME_USEC (_COMM_PAGE_START_ADDRESS+0x0C8) /* uint64_t boottime */ +#define _COMM_PAGE_NEWTIMEOFDAY_DATA (_COMM_PAGE_START_ADDRESS+0x0D0) /* used by gettimeofday(). Currently, sizeof(new_commpage_timeofday_data_t) = 40*/ #define _COMM_PAGE_END (_COMM_PAGE_START_ADDRESS+0xfff) /* end of common page */ diff --git a/osfmk/i386/cpu_data.h b/osfmk/i386/cpu_data.h index 057fc5b1c..27c493c24 100644 --- a/osfmk/i386/cpu_data.h +++ b/osfmk/i386/cpu_data.h @@ -47,11 +47,16 @@ #include #include #include +#include #if CONFIG_VMX #include #endif +#if MONOTONIC +#include +#endif /* MONOTONIC */ + #include /* @@ -72,13 +77,10 @@ typedef struct rtclock_timer { boolean_t has_expired; } rtclock_timer_t; - typedef struct { struct x86_64_tss *cdi_ktss; - struct __attribute__((packed)) { - uint16_t size; - void *ptr; - } cdi_gdt, cdi_idt; + x86_64_desc_register_t cdi_gdt; + x86_64_desc_register_t cdi_idt; struct fake_descriptor *cdi_ldt; vm_offset_t cdi_sstk; } cpu_desc_index_t; @@ -149,8 +151,11 @@ typedef struct cpu_data */ ast_t cpu_pending_ast; volatile int cpu_running; +#if !MONOTONIC boolean_t cpu_fixed_pmcs_enabled; +#endif /* !MONOTONIC */ rtclock_timer_t rtclock_timer; + uint64_t quantum_timer_deadline; volatile addr64_t cpu_active_cr3 __attribute((aligned(64))); union { volatile uint32_t cpu_tlb_invalid; @@ -197,6 +202,9 @@ typedef struct cpu_data uint64_t *cpu_kpc_shadow; uint64_t *cpu_kpc_reload; #endif +#if MONOTONIC + struct mt_cpu cpu_monotonic; +#endif /* MONOTONIC */ uint32_t cpu_pmap_pcid_enabled; pcid_t cpu_active_pcid; pcid_t cpu_last_pcid; @@ -220,11 +228,13 @@ typedef struct cpu_data uint64_t cpu_rtime_total; uint64_t cpu_ixtime; uint64_t cpu_idle_exits; - uint64_t cpu_rtimes[CPU_RTIME_BINS]; - uint64_t cpu_itimes[CPU_ITIME_BINS]; - uint64_t cpu_cur_insns; - uint64_t cpu_cur_ucc; - uint64_t cpu_cur_urc; + uint64_t cpu_rtimes[CPU_RTIME_BINS]; + uint64_t cpu_itimes[CPU_ITIME_BINS]; +#if !MONOTONIC + uint64_t cpu_cur_insns; + uint64_t cpu_cur_ucc; + uint64_t cpu_cur_urc; +#endif /* !MONOTONIC */ uint64_t cpu_gpmcs[4]; uint64_t cpu_max_observed_int_latency; int cpu_max_observed_int_latency_vector; diff --git a/osfmk/i386/cpu_threads.c b/osfmk/i386/cpu_threads.c index e58a9369e..5447514d8 100644 --- a/osfmk/i386/cpu_threads.c +++ b/osfmk/i386/cpu_threads.c @@ -35,6 +35,10 @@ #include #include +#if MONOTONIC +#include +#endif /* MONOTONIC */ + #define DIVISOR_GUARD(denom) \ if ((denom) == 0) { \ kprintf("%s: %d Zero divisor: " #denom, \ @@ -358,7 +362,7 @@ x86_core_alloc(int cpu) cpup = cpu_datap(cpu); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); if (free_cores != NULL) { core = free_cores; free_cores = core->next_in_die; @@ -385,7 +389,7 @@ x86_core_alloc(int cpu) static void x86_core_free(x86_core_t *core) { - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); core->next_in_die = free_cores; free_cores = core; simple_unlock(&x86_topo_lock); @@ -501,7 +505,7 @@ x86_die_alloc(int cpu) cpup = cpu_datap(cpu); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); if (free_dies != NULL) { die = free_dies; free_dies = die->next_in_pkg; @@ -528,7 +532,7 @@ x86_die_alloc(int cpu) static void x86_die_free(x86_die_t *die) { - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); die->next_in_pkg = free_dies; free_dies = die; atomic_decl((long *) &num_dies, 1); @@ -543,7 +547,7 @@ x86_package_alloc(int cpu) cpup = cpu_datap(cpu); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); if (free_pkgs != NULL) { pkg = free_pkgs; free_pkgs = pkg->next; @@ -570,7 +574,7 @@ x86_package_alloc(int cpu) static void x86_package_free(x86_pkg_t *pkg) { - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); pkg->next = free_pkgs; free_pkgs = pkg; atomic_decl((long *) &topoParms.nPackages, 1); @@ -617,7 +621,7 @@ x86_lcpu_add_caches(x86_lcpu_t *lcpu) */ list = x86_cache_list(); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); while (list != NULL) { /* @@ -734,7 +738,7 @@ x86_core_add_lcpu(x86_core_t *core, x86_lcpu_t *lcpu) assert(core != NULL); assert(lcpu != NULL); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); lcpu->next_in_core = core->lcpus; lcpu->core = core; @@ -812,7 +816,7 @@ cpu_thread_alloc(int cpu) /* * Only allow one to manipulate the topology at a time. */ - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); /* * Make sure all of the topology parameters have been initialized. @@ -846,7 +850,7 @@ cpu_thread_alloc(int cpu) */ simple_unlock(&x86_topo_lock); pkg = x86_package_alloc(cpu); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); if (x86_package_find(cpu) != NULL) { x86_package_free(pkg); continue; @@ -871,7 +875,7 @@ cpu_thread_alloc(int cpu) */ simple_unlock(&x86_topo_lock); die = x86_die_alloc(cpu); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); if (x86_die_find(cpu) != NULL) { x86_die_free(die); continue; @@ -895,7 +899,7 @@ cpu_thread_alloc(int cpu) */ simple_unlock(&x86_topo_lock); core = x86_core_alloc(cpu); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); if (x86_core_find(cpu) != NULL) { x86_core_free(core); continue; @@ -955,7 +959,7 @@ cpu_thread_init(void) * Do the CPU accounting. */ core = cpup->lcpu.core; - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); machine_info.logical_cpu += 1; if (core->active_lcpus == 0) machine_info.physical_cpu += 1; @@ -977,7 +981,7 @@ cpu_thread_halt(void) x86_core_t *core; cpu_data_t *cpup = current_cpu_datap(); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); machine_info.logical_cpu -= 1; core = cpup->lcpu.core; core->active_lcpus -= 1; diff --git a/osfmk/i386/cpu_threads.h b/osfmk/i386/cpu_threads.h index 31fe81779..ff028e02a 100644 --- a/osfmk/i386/cpu_threads.h +++ b/osfmk/i386/cpu_threads.h @@ -62,6 +62,8 @@ #define cpu_is_same_package(cpu1,cpu2) (cpu_to_package(cpu1) == cpu_to_package(cpu2)) #define cpus_share_cache(cpu1,cpu2,_cl) (cpu_to_lcpu(cpu1)->caches[_cl] == cpu_to_lcpu(cpu2)->caches[_cl]) +/* always take the x86_topo_lock with mp_safe_spin_lock */ +boolean_t mp_safe_spin_lock(usimple_lock_t lock); extern decl_simple_lock_data(, x86_topo_lock); extern void *cpu_thread_alloc(int); diff --git a/osfmk/i386/cpuid.c b/osfmk/i386/cpuid.c index db82b6eb3..0ebd786b5 100644 --- a/osfmk/i386/cpuid.c +++ b/osfmk/i386/cpuid.c @@ -361,6 +361,12 @@ cpuid_set_cache_info( i386_cpu_info_t * info_p ) */ if (type == L2U) info_p->cpuid_cache_L2_associativity = cache_associativity; + /* + * Adjust #sets to account for the N CBos + * This is because addresses are hashed across CBos + */ + if (type == L3U && info_p->core_count) + cache_sets = cache_sets / info_p->core_count; /* Compute the number of page colors for this cache, * which is: @@ -791,8 +797,15 @@ cpuid_set_cpufamily(i386_cpu_info_t *info_p) break; case CPUID_MODEL_SKYLAKE: case CPUID_MODEL_SKYLAKE_DT: +#if !defined(RC_HIDE_XNU_J137) + case CPUID_MODEL_SKYLAKE_W: +#endif cpufamily = CPUFAMILY_INTEL_SKYLAKE; break; + case CPUID_MODEL_KABYLAKE: + case CPUID_MODEL_KABYLAKE_DT: + cpufamily = CPUFAMILY_INTEL_KABYLAKE; + break; } break; } @@ -838,9 +851,10 @@ cpuid_set_info(void) } else { info_p->cpuid_cpu_subtype = CPU_SUBTYPE_X86_ARCH1; } + /* cpuid_set_cache_info must be invoked after set_generic_info */ - /* Must be invoked after set_generic_info */ - cpuid_set_cache_info(info_p); + if (info_p->cpuid_cpufamily == CPUFAMILY_INTEL_PENRYN) + cpuid_set_cache_info(info_p); /* * Find the number of enabled cores and threads @@ -859,6 +873,9 @@ cpuid_set_info(void) } default: { uint64_t msr = rdmsr64(MSR_CORE_THREAD_COUNT); + if (msr == 0) + /* Provide a non-zero default for some VMMs */ + msr = (1 << 16) + 1; info_p->core_count = bitfield32((uint32_t)msr, 31, 16); info_p->thread_count = bitfield32((uint32_t)msr, 15, 0); break; @@ -868,6 +885,10 @@ cpuid_set_info(void) info_p->core_count = info_p->cpuid_cores_per_package; info_p->thread_count = info_p->cpuid_logical_per_package; } + + if (info_p->cpuid_cpufamily != CPUFAMILY_INTEL_PENRYN) + cpuid_set_cache_info(info_p); + DBG("cpuid_set_info():\n"); DBG(" core_count : %d\n", info_p->core_count); DBG(" thread_count : %d\n", info_p->thread_count); @@ -970,6 +991,15 @@ leaf7_feature_map[] = { {CPUID_LEAF7_FEATURE_RDSEED, "RDSEED"}, {CPUID_LEAF7_FEATURE_ADX, "ADX"}, {CPUID_LEAF7_FEATURE_IPT, "IPT"}, +#if !defined(RC_HIDE_XNU_J137) + {CPUID_LEAF7_FEATURE_AVX512F, "AVX512F"}, + {CPUID_LEAF7_FEATURE_AVX512CD, "AVX512CD"}, + {CPUID_LEAF7_FEATURE_AVX512DQ, "AVX512DQ"}, + {CPUID_LEAF7_FEATURE_AVX512BW, "AVX512BW"}, + {CPUID_LEAF7_FEATURE_AVX512VL, "AVX512VL"}, + {CPUID_LEAF7_FEATURE_AVX512IFMA, "AVX512IFMA"}, + {CPUID_LEAF7_FEATURE_AVX512VBMI, "AVX512VBMI"}, +#endif /* not RC_HIDE_XNU_J137 */ {CPUID_LEAF7_FEATURE_SGX, "SGX"}, {CPUID_LEAF7_FEATURE_PQM, "PQM"}, {CPUID_LEAF7_FEATURE_FPU_CSDS, "FPU_CSDS"}, diff --git a/osfmk/i386/cpuid.h b/osfmk/i386/cpuid.h index 2c1f1803a..f8e8e24c2 100644 --- a/osfmk/i386/cpuid.h +++ b/osfmk/i386/cpuid.h @@ -144,8 +144,19 @@ #define CPUID_LEAF7_FEATURE_CLFSOPT _Bit(23) /* CLFSOPT */ #define CPUID_LEAF7_FEATURE_IPT _Bit(25) /* Intel Processor Trace */ #define CPUID_LEAF7_FEATURE_SHA _Bit(29) /* SHA instructions */ +#if !defined(RC_HIDE_XNU_J137) +#define CPUID_LEAF7_FEATURE_AVX512F _Bit(16) /* AVX512F instructions */ +#define CPUID_LEAF7_FEATURE_AVX512DQ _Bit(17) /* AVX512DQ instructions */ +#define CPUID_LEAF7_FEATURE_AVX512IFMA _Bit(21) /* AVX512IFMA instructions */ +#define CPUID_LEAF7_FEATURE_AVX512CD _Bit(28) /* AVX512CD instructions */ +#define CPUID_LEAF7_FEATURE_AVX512BW _Bit(30) /* AVX512BW instructions */ +#define CPUID_LEAF7_FEATURE_AVX512VL _Bit(31) /* AVX512VL instructions */ +#endif /* not RC_HIDE_XNU_J137 */ #define CPUID_LEAF7_FEATURE_PREFETCHWT1 _HBit(0)/* Prefetch Write/T1 hint */ +#if !defined(RC_HIDE_XNU_J137) +#define CPUID_LEAF7_FEATURE_AVX512VBMI _HBit(1)/* AVX512VBMI instructions */ +#endif /* not RC_HIDE_XNU_J137 */ /* * The CPUID_EXTFEATURE_XXX values define 64-bit values @@ -217,6 +228,13 @@ #define CPUID_MODEL_SKYLAKE_ULT 0x4E #define CPUID_MODEL_SKYLAKE_ULX 0x4E #define CPUID_MODEL_SKYLAKE_DT 0x5E +#if !defined(RC_HIDE_XNU_J137) +#define CPUID_MODEL_SKYLAKE_W 0x55 +#endif /* not RC_HIDE_XNU_J137 */ +#define CPUID_MODEL_KABYLAKE 0x8E +#define CPUID_MODEL_KABYLAKE_ULT 0x8E +#define CPUID_MODEL_KABYLAKE_ULX 0x8E +#define CPUID_MODEL_KABYLAKE_DT 0x9E #define CPUID_VMM_FAMILY_UNKNOWN 0x0 #define CPUID_VMM_FAMILY_VMWARE 0x1 diff --git a/osfmk/i386/fpu.c b/osfmk/i386/fpu.c index e9dce1877..41c2cad86 100644 --- a/osfmk/i386/fpu.c +++ b/osfmk/i386/fpu.c @@ -80,8 +80,8 @@ #include #include -int fp_kind = FP_NO; /* not inited */ -zone_t ifps_zone; /* zone for FPU save area */ +xstate_t fpu_capability = UNDEFINED; /* extended state capability */ +xstate_t fpu_default = UNDEFINED; /* default extended state */ #define ALIGNED(addr,size) (((uintptr_t)(addr)&((size)-1))==0) @@ -93,9 +93,10 @@ extern void fp_save( extern void fp_load( thread_t thr_act); -static void configure_mxcsr_capability_mask(struct x86_avx_thread_state *fps); +static void configure_mxcsr_capability_mask(x86_ext_thread_state_t *fps); +static xstate_t thread_xstate(thread_t); -struct x86_avx_thread_state initial_fp_state __attribute((aligned(64))); +x86_ext_thread_state_t initial_fp_state __attribute((aligned(64))); /* Global MXCSR capability bitmask */ @@ -122,32 +123,239 @@ static unsigned int mxcsr_capability_mask; #define fwait() \ __asm__("fwait"); -#define fxrstor(addr) __asm__ __volatile__("fxrstor %0" : : "m" (*(addr))) -#define fxsave(addr) __asm__ __volatile__("fxsave %0" : "=m" (*(addr))) +static inline void fxrstor(struct x86_fx_thread_state *a) { + __asm__ __volatile__("fxrstor %0" :: "m" (*a)); +} + +static inline void fxsave(struct x86_fx_thread_state *a) { + __asm__ __volatile__("fxsave %0" : "=m" (*a)); +} + +static inline void fxrstor64(struct x86_fx_thread_state *a) { + __asm__ __volatile__("fxrstor64 %0" :: "m" (*a)); +} + +static inline void fxsave64(struct x86_fx_thread_state *a) { + __asm__ __volatile__("fxsave64 %0" : "=m" (*a)); +} + +#if !defined(RC_HIDE_XNU_J137) +#define IS_VALID_XSTATE(x) ((x) == FP || (x) == AVX || (x) == AVX512) +#else +#define IS_VALID_XSTATE(x) ((x) == FP || (x) == AVX) +#endif + +zone_t ifps_zone[] = { + [FP] = NULL, + [AVX] = NULL, +#if !defined(RC_HIDE_XNU_J137) + [AVX512] = NULL +#endif +}; +static uint32_t fp_state_size[] = { + [FP] = sizeof(struct x86_fx_thread_state), + [AVX] = sizeof(struct x86_avx_thread_state), +#if !defined(RC_HIDE_XNU_J137) + [AVX512] = sizeof(struct x86_avx512_thread_state) +#endif +}; + +static const char *xstate_name[] = { + [UNDEFINED] = "UNDEFINED", + [FP] = "FP", + [AVX] = "AVX", +#if !defined(RC_HIDE_XNU_J137) + [AVX512] = "AVX512" +#endif +}; -static uint32_t fp_register_state_size = 0; -static uint32_t fpu_YMM_present = FALSE; +#if !defined(RC_HIDE_XNU_J137) +#define fpu_ZMM_capable (fpu_capability == AVX512) +#define fpu_YMM_capable (fpu_capability == AVX || fpu_capability == AVX512) +/* + * On-demand AVX512 support + * ------------------------ + * On machines with AVX512 support, by default, threads are created with + * AVX512 masked off in XCR0 and an AVX-sized savearea is used. However, AVX512 + * capabilities are advertised in the commpage and via sysctl. If a thread + * opts to use AVX512 instructions, the first will result in a #UD exception. + * Faulting AVX512 intructions are recognizable by their unique prefix. + * This exception results in the thread being promoted to use an AVX512-sized + * savearea and for the AVX512 bit masks being set in its XCR0. The faulting + * instruction is re-driven and the thread can proceed to perform AVX512 + * operations. + * + * In addition to AVX512 instructions causing promotion, the thread_set_state() + * primitive with an AVX512 state flavor result in promotion. + * + * AVX512 promotion of the first thread in a task causes the default xstate + * of the task to be promoted so that any subsequently created or subsequently + * DNA-faulted thread will have AVX512 xstate and it will not need to fault-in + * a promoted xstate. + * + * Two savearea zones are used: the default pool of AVX-sized (832 byte) areas + * and a second pool of larger AVX512-sized (2688 byte) areas. + * + * Note the initial state value is an AVX512 object but that the AVX initial + * value is a subset of it. + */ +#else +#define fpu_YMM_capable (fpu_capability == AVX) +#endif static uint32_t cpuid_reevaluated = 0; static void fpu_store_registers(void *, boolean_t); static void fpu_load_registers(void *); -extern void xsave64o(void); -extern void xrstor64o(void); - -#define XMASK ((uint32_t) (XFEM_X87 | XFEM_SSE | XFEM_YMM)) +#define FP_XMASK ((uint32_t) (XFEM_X87 | XFEM_SSE)) +#define AVX_XMASK ((uint32_t) (XFEM_X87 | XFEM_SSE | XFEM_YMM)) +#if !defined(RC_HIDE_XNU_J137) +#define AVX512_XMASK ((uint32_t) (XFEM_X87 | XFEM_SSE | XFEM_YMM | XFEM_ZMM)) +static const uint32_t xstate_xmask[] = { + [FP] = FP_XMASK, + [AVX] = AVX_XMASK, + [AVX512] = AVX512_XMASK +}; +#else +static const uint32_t xstate_xmask[] = { + [FP] = FP_XMASK, + [AVX] = AVX_XMASK, +}; +#endif static inline void xsetbv(uint32_t mask_hi, uint32_t mask_lo) { __asm__ __volatile__("xsetbv" :: "a"(mask_lo), "d"(mask_hi), "c" (XCR0)); } -static inline void xsave(struct x86_fx_thread_state *a) { - __asm__ __volatile__("xsave %0" :"=m" (*a) : "a"(XMASK), "d"(0)); +static inline void xsave(struct x86_fx_thread_state *a, uint32_t rfbm) { + __asm__ __volatile__("xsave %0" :"=m" (*a) : "a"(rfbm), "d"(0)); +} + +static inline void xsave64(struct x86_fx_thread_state *a, uint32_t rfbm) { + __asm__ __volatile__("xsave64 %0" :"=m" (*a) : "a"(rfbm), "d"(0)); +} + +static inline void xrstor(struct x86_fx_thread_state *a, uint32_t rfbm) { + __asm__ __volatile__("xrstor %0" :: "m" (*a), "a"(rfbm), "d"(0)); +} + +static inline void xrstor64(struct x86_fx_thread_state *a, uint32_t rfbm) { + __asm__ __volatile__("xrstor64 %0" :: "m" (*a), "a"(rfbm), "d"(0)); } -static inline void xrstor(struct x86_fx_thread_state *a) { - __asm__ __volatile__("xrstor %0" :: "m" (*a), "a"(XMASK), "d"(0)); +#if !defined(RC_HIDE_XNU_J137) +static inline void vzeroupper(void) { + __asm__ __volatile__("vzeroupper" ::); +} +#if DEVELOPMENT || DEBUG +static inline uint64_t xgetbv(uint32_t c) { + uint32_t mask_hi, mask_lo; + __asm__ __volatile__("xgetbv" : "=a"(mask_lo), "=d"(mask_hi) : "c" (c)); + return ((uint64_t) mask_hi<<32) + (uint64_t) mask_lo; } +#endif + +static boolean_t fpu_thread_promote_avx512(thread_t); /* Forward */ + +/* + * Define a wrapper for bcopy to defeat destination size checka. + * This is needed to treat repeated objects such as + * _STRUCT_XMM_REG fpu_ymmh0; + * ... + * _STRUCT_XMM_REG fpu_ymmh7; + * as an array and to copy like so: + * bcopy_nockch(src,&dst->fpu_ymmh0,8*sizeof(_STRUCT_XMM_REG)); + * without the compiler throwing a __builtin__memmove_chk error. + */ +static inline void bcopy_nochk(void *_src, void *_dst, size_t _len) { + bcopy(_src, _dst, _len); +} + +/* + * Furthermore, make compile-time asserts that no padding creeps into structures + * for which we're doing this. + */ +#define ASSERT_PACKED(t, m1, m2, n, mt) \ +extern char assert_packed_ ## t ## _ ## m1 ## _ ## m2 \ + [(offsetof(t,m2) - offsetof(t,m1) == (n - 1)*sizeof(mt)) ? 1 : -1] + +ASSERT_PACKED(x86_avx_state32_t, fpu_ymmh0, fpu_ymmh7, 8, _STRUCT_XMM_REG); + +ASSERT_PACKED(x86_avx_state64_t, fpu_ymmh0, fpu_ymmh15, 16, _STRUCT_XMM_REG); + +ASSERT_PACKED(x86_avx512_state32_t, fpu_k0, fpu_k7, 8, _STRUCT_OPMASK_REG); +ASSERT_PACKED(x86_avx512_state32_t, fpu_ymmh0, fpu_ymmh7, 8, _STRUCT_XMM_REG); +ASSERT_PACKED(x86_avx512_state32_t, fpu_zmmh0, fpu_zmmh7, 8, _STRUCT_YMM_REG); + +ASSERT_PACKED(x86_avx512_state64_t, fpu_k0, fpu_k7, 8, _STRUCT_OPMASK_REG); +ASSERT_PACKED(x86_avx512_state64_t, fpu_ymmh0, fpu_ymmh15, 16, _STRUCT_XMM_REG); +ASSERT_PACKED(x86_avx512_state64_t, fpu_zmmh0, fpu_zmmh15, 16, _STRUCT_YMM_REG); +ASSERT_PACKED(x86_avx512_state64_t, fpu_zmm16, fpu_zmm31, 16, _STRUCT_ZMM_REG); + +#if defined(DEBUG_AVX512) + +#define DBG(x...) kprintf("DBG: " x) + +typedef struct { uint8_t byte[8]; } opmask_t; +typedef struct { uint8_t byte[16]; } xmm_t; +typedef struct { uint8_t byte[32]; } ymm_t; +typedef struct { uint8_t byte[64]; } zmm_t; + +static void +DBG_AVX512_STATE(struct x86_avx512_thread_state *sp) +{ + int i, j; + xmm_t *xmm = (xmm_t *) &sp->fp.fx_XMM_reg; + xmm_t *ymmh = (xmm_t *) &sp->x_YMM_Hi128; + ymm_t *zmmh = (ymm_t *) &sp->x_ZMM_Hi256; + zmm_t *zmm = (zmm_t *) &sp->x_Hi16_ZMM; + opmask_t *k = (opmask_t *) &sp->x_Opmask; + + kprintf("x_YMM_Hi128: %lu\n", offsetof(struct x86_avx512_thread_state, x_YMM_Hi128)); + kprintf("x_Opmask: %lu\n", offsetof(struct x86_avx512_thread_state, x_Opmask)); + kprintf("x_ZMM_Hi256: %lu\n", offsetof(struct x86_avx512_thread_state, x_ZMM_Hi256)); + kprintf("x_Hi16_ZMM: %lu\n", offsetof(struct x86_avx512_thread_state, x_Hi16_ZMM)); + + kprintf("XCR0: 0x%016llx\n", xgetbv(XCR0)); + kprintf("XINUSE: 0x%016llx\n", xgetbv(1)); + + /* Print all ZMM registers */ + for (i = 0; i < 16; i++) { + kprintf("zmm%d:\t0x", i); + for (j = 0; j < 16; j++) + kprintf("%02x", xmm[i].byte[j]); + for (j = 0; j < 16; j++) + kprintf("%02x", ymmh[i].byte[j]); + for (j = 0; j < 32; j++) + kprintf("%02x", zmmh[i].byte[j]); + kprintf("\n"); + } + for (i = 0; i < 16; i++) { + kprintf("zmm%d:\t0x", 16+i); + for (j = 0; j < 64; j++) + kprintf("%02x", zmm[i].byte[j]); + kprintf("\n"); + } + for (i = 0; i < 8; i++) { + kprintf("k%d:\t0x", i); + for (j = 0; j < 8; j++) + kprintf("%02x", k[i].byte[j]); + kprintf("\n"); + } + + kprintf("xstate_bv: 0x%016llx\n", sp->_xh.xstate_bv); + kprintf("xcomp_bv: 0x%016llx\n", sp->_xh.xcomp_bv); +} +#else +#define DBG(x...) +static void +DBG_AVX512_STATE(__unused struct x86_avx512_thread_state *sp) +{ + return; +} +#endif /* DEBUG_AVX512 */ + +#endif #if DEBUG static inline unsigned short @@ -167,7 +375,7 @@ fnstsw(void) */ static void -configure_mxcsr_capability_mask(struct x86_avx_thread_state *fps) +configure_mxcsr_capability_mask(x86_ext_thread_state_t *fps) { /* XSAVE requires a 64 byte aligned store */ assert(ALIGNED(fps, 64)); @@ -177,29 +385,35 @@ configure_mxcsr_capability_mask(struct x86_avx_thread_state *fps) fpinit(); fpu_store_registers(fps, FALSE); - mxcsr_capability_mask = fps->fx_MXCSR_MASK; + mxcsr_capability_mask = fps->fx.fx_MXCSR_MASK; /* Set default mask value if necessary */ if (mxcsr_capability_mask == 0) mxcsr_capability_mask = 0xffbf; /* Clear vector register store */ - bzero(&fps->fx_XMM_reg[0][0], sizeof(fps->fx_XMM_reg)); - bzero(&fps->x_YMMH_reg[0][0], sizeof(fps->x_YMMH_reg)); + bzero(&fps->fx.fx_XMM_reg[0][0], sizeof(fps->fx.fx_XMM_reg)); + bzero(fps->avx.x_YMM_Hi128, sizeof(fps->avx.x_YMM_Hi128)); +#if !defined(RC_HIDE_XNU_J137) + if (fpu_ZMM_capable) { + bzero(fps->avx512.x_ZMM_Hi256, sizeof(fps->avx512.x_ZMM_Hi256)); + bzero(fps->avx512.x_Hi16_ZMM, sizeof(fps->avx512.x_Hi16_ZMM)); + bzero(fps->avx512.x_Opmask, sizeof(fps->avx512.x_Opmask)); + } +#endif - fps->fp_valid = TRUE; - fps->fp_save_layout = fpu_YMM_present ? XSAVE32: FXSAVE32; + fps->fx.fp_valid = TRUE; + fps->fx.fp_save_layout = fpu_YMM_capable ? XSAVE32: FXSAVE32; fpu_load_registers(fps); /* Poison values to trap unsafe usage */ - fps->fp_valid = 0xFFFFFFFF; - fps->fp_save_layout = FP_UNUSED; + fps->fx.fp_valid = 0xFFFFFFFF; + fps->fx.fp_save_layout = FP_UNUSED; /* Re-enable FPU/SSE DNA exceptions */ set_ts(); } - /* * Look for FPU and initialize it. * Called on each CPU. @@ -226,38 +440,73 @@ init_fpu(void) #endif /* Advertise SSE support */ if (cpuid_features() & CPUID_FEATURE_FXSR) { - fp_kind = FP_FXSR; set_cr4(get_cr4() | CR4_OSFXS); /* And allow SIMD exceptions if present */ if (cpuid_features() & CPUID_FEATURE_SSE) { set_cr4(get_cr4() | CR4_OSXMM); } - fp_register_state_size = sizeof(struct x86_fx_thread_state); - } else panic("fpu is not FP_FXSR"); + fpu_capability = fpu_default = FP; + +#if !defined(RC_HIDE_XNU_J137) + static boolean_t is_avx512_enabled = TRUE; + if (cpu_number() == master_cpu) { + if (cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_AVX512F) { + PE_parse_boot_argn("avx512", &is_avx512_enabled, sizeof(boolean_t)); + kprintf("AVX512 supported %s\n", + is_avx512_enabled ? "and enabled" : "but disabled"); + } + } +#endif + /* Configure the XSAVE context mechanism if the processor supports * AVX/YMM registers */ if (cpuid_features() & CPUID_FEATURE_XSAVE) { - cpuid_xsave_leaf_t *xsp = &cpuid_info()->cpuid_xsave_leaf[0]; - if (xsp->extended_state[0] & (uint32_t)XFEM_YMM) { - assert(xsp->extended_state[0] & (uint32_t) XFEM_SSE); + cpuid_xsave_leaf_t *xs0p = &cpuid_info()->cpuid_xsave_leaf[0]; +#if !defined(RC_HIDE_XNU_J137) + if (is_avx512_enabled && + (xs0p->extended_state[eax] & XFEM_ZMM) == XFEM_ZMM) { + assert(xs0p->extended_state[eax] & XFEM_SSE); + assert(xs0p->extended_state[eax] & XFEM_YMM); + fpu_capability = AVX512; + /* XSAVE container size for all features */ + set_cr4(get_cr4() | CR4_OSXSAVE); + xsetbv(0, AVX512_XMASK); + /* Re-evaluate CPUID, once, to reflect OSXSAVE */ + if (OSCompareAndSwap(0, 1, &cpuid_reevaluated)) + cpuid_set_info(); + /* Verify that now selected state can be accommodated */ + assert(xs0p->extended_state[ebx] == fp_state_size[AVX512]); + /* + * AVX set until AVX512 is used. + * See comment above about on-demand AVX512 support. + */ + xsetbv(0, AVX_XMASK); + fpu_default = AVX; + } else +#endif + if (xs0p->extended_state[eax] & XFEM_YMM) { + assert(xs0p->extended_state[eax] & XFEM_SSE); + fpu_capability = AVX; + fpu_default = AVX; /* XSAVE container size for all features */ - fp_register_state_size = sizeof(struct x86_avx_thread_state); - fpu_YMM_present = TRUE; set_cr4(get_cr4() | CR4_OSXSAVE); - xsetbv(0, XMASK); + xsetbv(0, AVX_XMASK); /* Re-evaluate CPUID, once, to reflect OSXSAVE */ if (OSCompareAndSwap(0, 1, &cpuid_reevaluated)) cpuid_set_info(); /* Verify that now selected state can be accommodated */ - assert(xsp->extended_state[1] == fp_register_state_size); + assert(xs0p->extended_state[ebx] == fp_state_size[AVX]); } } - else - fpu_YMM_present = FALSE; + + if (cpu_number() == master_cpu) + kprintf("fpu_state: %s, state_size: %d\n", + xstate_name[fpu_capability], + fp_state_size[fpu_capability]); fpinit(); @@ -268,27 +517,35 @@ init_fpu(void) } /* - * Allocate and initialize FP state for current thread. + * Allocate and initialize FP state for specified xstate. * Don't load state. */ static void * -fp_state_alloc(void) +fp_state_alloc(xstate_t xs) { - struct x86_fx_thread_state *ifps = zalloc(ifps_zone); + struct x86_fx_thread_state *ifps; + + assert(ifps_zone[xs] != NULL); + ifps = zalloc(ifps_zone[xs]); #if DEBUG if (!(ALIGNED(ifps,64))) { - panic("fp_state_alloc: %p, %u, %p, %u", ifps, (unsigned) ifps_zone->elem_size, (void *) ifps_zone->free_elements, (unsigned) ifps_zone->alloc_size); + panic("fp_state_alloc: %p, %u, %p, %u", + ifps, (unsigned) ifps_zone[xs]->elem_size, + (void *) ifps_zone[xs]->free_elements, + (unsigned) ifps_zone[xs]->alloc_size); } #endif - bzero(ifps, sizeof(*ifps)); + bzero(ifps, fp_state_size[xs]); + return ifps; } static inline void -fp_state_free(void *ifps) +fp_state_free(void *ifps, xstate_t xs) { - zfree(ifps_zone, ifps); + assert(ifps_zone[xs] != NULL); + zfree(ifps_zone[xs], ifps); } void clear_fpu(void) @@ -301,7 +558,10 @@ static void fpu_load_registers(void *fstate) { struct x86_fx_thread_state *ifps = fstate; fp_save_layout_t layout = ifps->fp_save_layout; - assert(layout == FXSAVE32 || layout == FXSAVE64 || layout == XSAVE32 || layout == XSAVE64); + assert(current_task() == NULL || \ + (thread_is_64bit(current_thread()) ? \ + (layout == FXSAVE64 || layout == XSAVE64) : \ + (layout == FXSAVE32 || layout == XSAVE32))); assert(ALIGNED(ifps, 64)); assert(ml_get_interrupts_enabled() == FALSE); @@ -310,34 +570,64 @@ static void fpu_load_registers(void *fstate) { struct x86_avx_thread_state *iavx = fstate; unsigned i; /* Verify reserved bits in the XSAVE header*/ - if (iavx->_xh.xsbv & ~7) - panic("iavx->_xh.xsbv: 0x%llx", iavx->_xh.xsbv); + if (iavx->_xh.xstate_bv & ~xstate_xmask[current_xstate()]) + panic("iavx->_xh.xstate_bv: 0x%llx", iavx->_xh.xstate_bv); for (i = 0; i < sizeof(iavx->_xh.xhrsvd); i++) if (iavx->_xh.xhrsvd[i]) panic("Reserved bit set"); } - if (fpu_YMM_present) { + if (fpu_YMM_capable) { if (layout != XSAVE32 && layout != XSAVE64) panic("Inappropriate layout: %u\n", layout); } #endif /* DEBUG */ - if ((layout == XSAVE64) || (layout == XSAVE32)) - xrstor(ifps); - else + switch (layout) { + case FXSAVE64: + fxrstor64(ifps); + break; + case FXSAVE32: fxrstor(ifps); + break; + case XSAVE64: + xrstor64(ifps, xstate_xmask[current_xstate()]); + break; + case XSAVE32: + xrstor(ifps, xstate_xmask[current_xstate()]); + break; + default: + panic("fpu_load_registers() bad layout: %d\n", layout); + } } static void fpu_store_registers(void *fstate, boolean_t is64) { struct x86_fx_thread_state *ifps = fstate; assert(ALIGNED(ifps, 64)); - if (fpu_YMM_present) { - xsave(ifps); - ifps->fp_save_layout = is64 ? XSAVE64 : XSAVE32; - } - else { - fxsave(ifps); - ifps->fp_save_layout = is64 ? FXSAVE64 : FXSAVE32; + xstate_t xs = current_xstate(); + switch (xs) { + case FP: + if (is64) { + fxsave64(fstate); + ifps->fp_save_layout = FXSAVE64; + } else { + fxsave(fstate); + ifps->fp_save_layout = FXSAVE32; + } + break; + case AVX: +#if !defined(RC_HIDE_XNU_J137) + case AVX512: +#endif + if (is64) { + xsave64(ifps, xstate_xmask[xs]); + ifps->fp_save_layout = XSAVE64; + } else { + xsave(ifps, xstate_xmask[xs]); + ifps->fp_save_layout = XSAVE32; + } + break; + default: + panic("fpu_store_registers() bad xstate: %d\n", xs); } } @@ -348,40 +638,56 @@ static void fpu_store_registers(void *fstate, boolean_t is64) { void fpu_module_init(void) { - if ((fp_register_state_size != sizeof(struct x86_fx_thread_state)) && - (fp_register_state_size != sizeof(struct x86_avx_thread_state))) - panic("fpu_module_init: incorrect savearea size %u\n", fp_register_state_size); + if (!IS_VALID_XSTATE(fpu_default)) + panic("fpu_module_init: invalid extended state %u\n", + fpu_default); - assert(fpu_YMM_present != 0xFFFFFFFF); - - /* We explicitly choose an allocation size of 64 + /* We explicitly choose an allocation size of 13 pages = 64 * 832 * to eliminate waste for the 832 byte sized * AVX XSAVE register save area. */ - ifps_zone = zinit(fp_register_state_size, - thread_max * fp_register_state_size, - 64 * fp_register_state_size, - "x86 fpsave state"); + ifps_zone[fpu_default] = zinit(fp_state_size[fpu_default], + thread_max * fp_state_size[fpu_default], + 64 * fp_state_size[fpu_default], + "x86 fpsave state"); /* To maintain the required alignment, disable * zone debugging for this zone as that appends * 16 bytes to each element. */ - zone_change(ifps_zone, Z_ALIGNMENT_REQUIRED, TRUE); + zone_change(ifps_zone[fpu_default], Z_ALIGNMENT_REQUIRED, TRUE); + +#if !defined(RC_HIDE_XNU_J137) + /* + * If AVX512 is supported, create a separate savearea zone. + * with allocation size: 19 pages = 32 * 2668 + */ + if (fpu_capability == AVX512) { + ifps_zone[AVX512] = zinit(fp_state_size[AVX512], + thread_max * fp_state_size[AVX512], + 32 * fp_state_size[AVX512], + "x86 avx512 save state"); + zone_change(ifps_zone[AVX512], Z_ALIGNMENT_REQUIRED, TRUE); + } +#endif + /* Determine MXCSR reserved bits and configure initial FPU state*/ configure_mxcsr_capability_mask(&initial_fp_state); } /* - * Save thread`s FPU context. + * Context switch fpu state. + * Always save old thread`s FPU context but don't load new .. allow that to fault-in. + * Switch to the new task's xstate. */ void -fpu_save_context(thread_t thread) +fpu_switch_context(thread_t old, thread_t new) { - struct x86_fx_thread_state *ifps; + struct x86_fx_thread_state *ifps; + boolean_t is_ts_cleared = FALSE; assert(ml_get_interrupts_enabled() == FALSE); - ifps = (thread)->machine.ifps; + ifps = (old)->machine.ifps; #if DEBUG if (ifps && ((ifps->fp_valid != FALSE) && (ifps->fp_valid != TRUE))) { panic("ifps->fp_valid: %u\n", ifps->fp_valid); @@ -394,10 +700,32 @@ fpu_save_context(thread_t thread) * (such as sendsig & sigreturn) manipulate TS directly. */ clear_ts(); + is_ts_cleared = TRUE; /* registers are in FPU - save to memory */ - fpu_store_registers(ifps, (thread_is_64bit(thread) && is_saved_state64(thread->machine.iss))); + fpu_store_registers(ifps, (thread_is_64bit(old) && is_saved_state64(old->machine.iss))); ifps->fp_valid = TRUE; } +#if !defined(RC_HIDE_XNU_J137) + xstate_t old_xstate = thread_xstate(old); + xstate_t new_xstate = new ? thread_xstate(new) : fpu_default; + if (old_xstate == AVX512 && ifps != 0) { + DBG_AVX512_STATE((struct x86_avx512_thread_state *) ifps); + /* + * Clear upper bits for potential power-saving + * but first ensure the TS bit is clear. + */ + if (!is_ts_cleared) + clear_ts(); + vzeroupper(); + } + if (new_xstate != old_xstate) { + DBG("fpu_switch_context(%p,%p) new xstate: %s\n", + old, new, xstate_name[new_xstate]); + xsetbv(0, xstate_xmask[new_xstate]); + } +#else +#pragma unused(new) +#endif set_ts(); } @@ -407,9 +735,12 @@ fpu_save_context(thread_t thread) * Called only when thread terminating - no locking necessary. */ void -fpu_free(void *fps) +fpu_free(thread_t thread, void *fps) { - fp_state_free(fps); + pcb_t pcb = THREAD_TO_PCB(thread); + + fp_state_free(fps, pcb->xstate); + pcb->xstate = UNDEFINED; } /* @@ -427,20 +758,26 @@ fpu_set_fxstate( thread_state_t tstate, thread_flavor_t f) { - struct x86_fx_thread_state *ifps; - struct x86_fx_thread_state *new_ifps; - x86_float_state64_t *state; - pcb_t pcb; - size_t state_size = sizeof(struct x86_fx_thread_state); - boolean_t old_valid, fresh_state = FALSE; + struct x86_fx_thread_state *ifps; + struct x86_fx_thread_state *new_ifps; + x86_float_state64_t *state; + pcb_t pcb; + boolean_t old_valid, fresh_state = FALSE; - if (fp_kind == FP_NO) + if (fpu_capability == UNDEFINED) return KERN_FAILURE; if ((f == x86_AVX_STATE32 || f == x86_AVX_STATE64) && - !ml_fpu_avx_enabled()) + fpu_capability < AVX) return KERN_FAILURE; +#if !defined(RC_HIDE_XNU_J137) + if ((f == x86_AVX512_STATE32 || f == x86_AVX512_STATE64) && + thread_xstate(thr_act) == AVX) + if (!fpu_thread_promote_avx512(thr_act)) + return KERN_FAILURE; +#endif + state = (x86_float_state64_t *)tstate; assert(thr_act != THREAD_NULL); @@ -459,7 +796,7 @@ fpu_set_fxstate( simple_unlock(&pcb->lock); if (ifps != 0) { - fp_state_free(ifps); + fp_state_free(ifps, thread_xstate(thr_act)); } } else { /* @@ -473,12 +810,13 @@ fpu_set_fxstate( if (ifps == 0) { if (new_ifps == 0) { simple_unlock(&pcb->lock); - new_ifps = fp_state_alloc(); + new_ifps = fp_state_alloc(thread_xstate(thr_act)); goto Retry; } ifps = new_ifps; new_ifps = 0; pcb->ifps = ifps; + pcb->xstate = thread_xstate(thr_act); fresh_state = TRUE; } @@ -500,32 +838,77 @@ fpu_set_fxstate( state->fpu_mxcsr &= mxcsr_capability_mask; - bcopy((char *)&state->fpu_fcw, (char *)ifps, state_size); + bcopy((char *)&state->fpu_fcw, (char *)ifps, fp_state_size[FP]); - if (fpu_YMM_present) { + switch (thread_xstate(thr_act)) { + case UNDEFINED: + panic("fpu_set_fxstate() UNDEFINED xstate"); + break; + case FP: + ifps->fp_save_layout = thread_is_64bit(thr_act) ? FXSAVE64 : FXSAVE32; + break; + case AVX: { struct x86_avx_thread_state *iavx = (void *) ifps; - uint32_t fpu_nyreg = 0; + x86_avx_state64_t *xs = (x86_avx_state64_t *) state; - if (f == x86_AVX_STATE32) - fpu_nyreg = 8; - else if (f == x86_AVX_STATE64) - fpu_nyreg = 16; + iavx->fp.fp_save_layout = thread_is_64bit(thr_act) ? XSAVE64 : XSAVE32; - if (fpu_nyreg) { - x86_avx_state64_t *ystate = (x86_avx_state64_t *) state; - bcopy(&ystate->__fpu_ymmh0, &iavx->x_YMMH_reg[0][0], fpu_nyreg * sizeof(_STRUCT_XMM_REG)); + /* Sanitize XSAVE header */ + bzero(&iavx->_xh.xhrsvd[0], sizeof(iavx->_xh.xhrsvd)); + iavx->_xh.xstate_bv = AVX_XMASK; + iavx->_xh.xcomp_bv = 0; + + if (f == x86_AVX_STATE32) { + bcopy_nochk(&xs->fpu_ymmh0, iavx->x_YMM_Hi128, 8 * sizeof(_STRUCT_XMM_REG)); + } else if (f == x86_AVX_STATE64) { + bcopy_nochk(&xs->fpu_ymmh0, iavx->x_YMM_Hi128, 16 * sizeof(_STRUCT_XMM_REG)); + } else { + iavx->_xh.xstate_bv = (XFEM_SSE | XFEM_X87); } + break; + } +#if !defined(RC_HIDE_XNU_J137) + case AVX512: { + struct x86_avx512_thread_state *iavx = (void *) ifps; + union { + thread_state_t ts; + x86_avx512_state32_t *s32; + x86_avx512_state64_t *s64; + } xs = { .ts = tstate }; + + iavx->fp.fp_save_layout = thread_is_64bit(thr_act) ? XSAVE64 : XSAVE32; - iavx->fp_save_layout = thread_is_64bit(thr_act) ? XSAVE64 : XSAVE32; /* Sanitize XSAVE header */ bzero(&iavx->_xh.xhrsvd[0], sizeof(iavx->_xh.xhrsvd)); - if (fpu_nyreg) - iavx->_xh.xsbv = (XFEM_YMM | XFEM_SSE | XFEM_X87); - else - iavx->_xh.xsbv = (XFEM_SSE | XFEM_X87); - } else { - ifps->fp_save_layout = thread_is_64bit(thr_act) ? FXSAVE64 : FXSAVE32; + iavx->_xh.xstate_bv = AVX512_XMASK; + iavx->_xh.xcomp_bv = 0; + + switch (f) { + case x86_AVX512_STATE32: + bcopy_nochk(&xs.s32->fpu_k0, iavx->x_Opmask, 8 * sizeof(_STRUCT_OPMASK_REG)); + bcopy_nochk(&xs.s32->fpu_zmmh0, iavx->x_ZMM_Hi256, 8 * sizeof(_STRUCT_YMM_REG)); + bcopy_nochk(&xs.s32->fpu_ymmh0, iavx->x_YMM_Hi128, 8 * sizeof(_STRUCT_XMM_REG)); + DBG_AVX512_STATE(iavx); + break; + case x86_AVX_STATE32: + bcopy_nochk(&xs.s32->fpu_ymmh0, iavx->x_YMM_Hi128, 8 * sizeof(_STRUCT_XMM_REG)); + break; + case x86_AVX512_STATE64: + bcopy_nochk(&xs.s64->fpu_k0, iavx->x_Opmask, 8 * sizeof(_STRUCT_OPMASK_REG)); + bcopy_nochk(&xs.s64->fpu_zmm16, iavx->x_Hi16_ZMM, 16 * sizeof(_STRUCT_ZMM_REG)); + bcopy_nochk(&xs.s64->fpu_zmmh0, iavx->x_ZMM_Hi256, 16 * sizeof(_STRUCT_YMM_REG)); + bcopy_nochk(&xs.s64->fpu_ymmh0, iavx->x_YMM_Hi128, 16 * sizeof(_STRUCT_XMM_REG)); + DBG_AVX512_STATE(iavx); + break; + case x86_AVX_STATE64: + bcopy_nochk(&xs.s64->fpu_ymmh0, iavx->x_YMM_Hi128, 16 * sizeof(_STRUCT_XMM_REG)); + break; + } + break; + } +#endif } + ifps->fp_valid = old_valid; if (old_valid == FALSE) { @@ -541,7 +924,7 @@ fpu_set_fxstate( simple_unlock(&pcb->lock); if (new_ifps != 0) - fp_state_free(new_ifps); + fp_state_free(new_ifps, thread_xstate(thr_act)); } return KERN_SUCCESS; } @@ -559,18 +942,23 @@ fpu_get_fxstate( thread_flavor_t f) { struct x86_fx_thread_state *ifps; - x86_float_state64_t *state; - kern_return_t ret = KERN_FAILURE; - pcb_t pcb; - size_t state_size = sizeof(struct x86_fx_thread_state); + x86_float_state64_t *state; + kern_return_t ret = KERN_FAILURE; + pcb_t pcb; - if (fp_kind == FP_NO) + if (fpu_capability == UNDEFINED) return KERN_FAILURE; if ((f == x86_AVX_STATE32 || f == x86_AVX_STATE64) && - !ml_fpu_avx_enabled()) + fpu_capability < AVX) return KERN_FAILURE; +#if !defined(RC_HIDE_XNU_J137) + if ((f == x86_AVX512_STATE32 || f == x86_AVX512_STATE64) && + thread_xstate(thr_act) != AVX512) + return KERN_FAILURE; +#endif + state = (x86_float_state64_t *)tstate; assert(thr_act != THREAD_NULL); @@ -585,7 +973,7 @@ fpu_get_fxstate( */ bcopy((char *)&initial_fp_state, (char *)&state->fpu_fcw, - state_size); + fp_state_size[FP]); simple_unlock(&pcb->lock); @@ -607,20 +995,55 @@ fpu_get_fxstate( (void)ml_set_interrupts_enabled(intr); } if (ifps->fp_valid) { - bcopy((char *)ifps, (char *)&state->fpu_fcw, state_size); - if (fpu_YMM_present) { + bcopy((char *)ifps, (char *)&state->fpu_fcw, fp_state_size[FP]); + switch (thread_xstate(thr_act)) { + case UNDEFINED: + panic("fpu_get_fxstate() UNDEFINED xstate"); + break; + case FP: + break; /* already done */ + case AVX: { struct x86_avx_thread_state *iavx = (void *) ifps; - uint32_t fpu_nyreg = 0; - - if (f == x86_AVX_STATE32) - fpu_nyreg = 8; - else if (f == x86_AVX_STATE64) - fpu_nyreg = 16; - - if (fpu_nyreg) { - x86_avx_state64_t *ystate = (x86_avx_state64_t *) state; - bcopy(&iavx->x_YMMH_reg[0][0], &ystate->__fpu_ymmh0, fpu_nyreg * sizeof(_STRUCT_XMM_REG)); + x86_avx_state64_t *xs = (x86_avx_state64_t *) state; + if (f == x86_AVX_STATE32) { + bcopy_nochk(iavx->x_YMM_Hi128, &xs->fpu_ymmh0, 8 * sizeof(_STRUCT_XMM_REG)); + } else if (f == x86_AVX_STATE64) { + bcopy_nochk(iavx->x_YMM_Hi128, &xs->fpu_ymmh0, 16 * sizeof(_STRUCT_XMM_REG)); + } + break; + } +#if !defined(RC_HIDE_XNU_J137) + case AVX512: { + struct x86_avx512_thread_state *iavx = (void *) ifps; + union { + thread_state_t ts; + x86_avx512_state32_t *s32; + x86_avx512_state64_t *s64; + } xs = { .ts = tstate }; + switch (f) { + case x86_AVX512_STATE32: + bcopy_nochk(iavx->x_Opmask, &xs.s32->fpu_k0, 8 * sizeof(_STRUCT_OPMASK_REG)); + bcopy_nochk(iavx->x_ZMM_Hi256, &xs.s32->fpu_zmmh0, 8 * sizeof(_STRUCT_YMM_REG)); + bcopy_nochk(iavx->x_YMM_Hi128, &xs.s32->fpu_ymmh0, 8 * sizeof(_STRUCT_XMM_REG)); + DBG_AVX512_STATE(iavx); + break; + case x86_AVX_STATE32: + bcopy_nochk(iavx->x_YMM_Hi128, &xs.s32->fpu_ymmh0, 8 * sizeof(_STRUCT_XMM_REG)); + break; + case x86_AVX512_STATE64: + bcopy_nochk(iavx->x_Opmask, &xs.s64->fpu_k0, 8 * sizeof(_STRUCT_OPMASK_REG)); + bcopy_nochk(iavx->x_Hi16_ZMM, &xs.s64->fpu_zmm16, 16 * sizeof(_STRUCT_ZMM_REG)); + bcopy_nochk(iavx->x_ZMM_Hi256, &xs.s64->fpu_zmmh0, 16 * sizeof(_STRUCT_YMM_REG)); + bcopy_nochk(iavx->x_YMM_Hi128, &xs.s64->fpu_ymmh0, 16 * sizeof(_STRUCT_XMM_REG)); + DBG_AVX512_STATE(iavx); + break; + case x86_AVX_STATE64: + bcopy_nochk(iavx->x_YMM_Hi128, &xs.s64->fpu_ymmh0, 16 * sizeof(_STRUCT_XMM_REG)); + break; } + break; + } +#endif } ret = KERN_SUCCESS; @@ -647,6 +1070,7 @@ fpu_dup_fxstate( struct x86_fx_thread_state *new_ifps = NULL; boolean_t intr; pcb_t ppcb; + xstate_t xstate = thread_xstate(parent); ppcb = THREAD_TO_PCB(parent); @@ -656,7 +1080,7 @@ fpu_dup_fxstate( if (child->machine.ifps) panic("fpu_dup_fxstate: child's ifps non-null"); - new_ifps = fp_state_alloc(); + new_ifps = fp_state_alloc(xstate); simple_lock(&ppcb->lock); @@ -677,10 +1101,10 @@ fpu_dup_fxstate( if (ifps->fp_valid) { child->machine.ifps = new_ifps; - assert((fp_register_state_size == sizeof(struct x86_fx_thread_state)) || - (fp_register_state_size == sizeof(struct x86_avx_thread_state))); + child->machine.xstate = xstate; bcopy((char *)(ppcb->ifps), - (char *)(child->machine.ifps), fp_register_state_size); + (char *)(child->machine.ifps), + fp_state_size[xstate]); /* Mark the new fp saved state as non-live. */ /* Temporarily disabled: radar 4647827 @@ -698,7 +1122,7 @@ fpu_dup_fxstate( simple_unlock(&ppcb->lock); if (new_ifps != NULL) - fp_state_free(new_ifps); + fp_state_free(new_ifps, xstate); } @@ -743,21 +1167,20 @@ fpnoextflt(void) thread_t thr_act; pcb_t pcb; struct x86_fx_thread_state *ifps = 0; + xstate_t xstate = current_xstate(); thr_act = current_thread(); pcb = THREAD_TO_PCB(thr_act); - assert(fp_register_state_size != 0); - if (pcb->ifps == 0 && !get_interrupt_level()) { - ifps = fp_state_alloc(); + ifps = fp_state_alloc(xstate); bcopy((char *)&initial_fp_state, (char *)ifps, - fp_register_state_size); + fp_state_size[xstate]); if (!thread_is_64bit(thr_act)) { - ifps->fp_save_layout = fpu_YMM_present ? XSAVE32 : FXSAVE32; + ifps->fp_save_layout = fpu_YMM_capable ? XSAVE32 : FXSAVE32; } else - ifps->fp_save_layout = fpu_YMM_present ? XSAVE64 : FXSAVE64; + ifps->fp_save_layout = fpu_YMM_capable ? XSAVE64 : FXSAVE64; ifps->fp_valid = TRUE; } intr = ml_set_interrupts_enabled(FALSE); @@ -780,6 +1203,7 @@ fpnoextflt(void) } else { if (pcb->ifps == 0) { pcb->ifps = ifps; + pcb->xstate = xstate; ifps = 0; } /* @@ -790,7 +1214,7 @@ fpnoextflt(void) (void)ml_set_interrupts_enabled(intr); if (ifps) - fp_state_free(ifps); + fp_state_free(ifps, xstate); } /* @@ -805,6 +1229,7 @@ fpextovrflt(void) pcb_t pcb; struct x86_fx_thread_state *ifps; boolean_t intr; + xstate_t xstate = current_xstate(); intr = ml_set_interrupts_enabled(FALSE); @@ -837,7 +1262,7 @@ fpextovrflt(void) (void)ml_set_interrupts_enabled(intr); if (ifps) - zfree(ifps_zone, ifps); + fp_state_free(ifps, xstate); /* * Raise exception. @@ -975,6 +1400,139 @@ fpSSEexterrflt(void) /*NOTREACHED*/ } + +#if !defined(RC_HIDE_XNU_J137) +/* + * If a thread is using an AVX-sized savearea: + * - allocate a new AVX512-sized area, + * - copy the 256-bit state into the 512-bit area, + * - deallocate the smaller area + */ +static void +fpu_savearea_promote_avx512(thread_t thread) +{ + struct x86_avx_thread_state *ifps; + struct x86_avx512_thread_state *ifps512; + pcb_t pcb = THREAD_TO_PCB(thread); + + DBG("fpu_upgrade_savearea(%p)\n", thread); + ifps512 = fp_state_alloc(AVX512); + simple_lock(&pcb->lock); + ifps = pcb->ifps; + if (ifps == NULL) { + /* nothing to be done */ + simple_unlock(&pcb->lock); + fp_state_free(ifps512, AVX512); + xsetbv(0, AVX512_XMASK); + DBG("fpu_upgrade_savearea() NULL ifps\n"); + return; + } + if (thread == current_thread()) { + boolean_t intr; + + intr = ml_set_interrupts_enabled(FALSE); + + clear_ts(); + fp_save(thread); + clear_fpu(); + + xsetbv(0, AVX512_XMASK); + + (void)ml_set_interrupts_enabled(intr); + } + assert(ifps->fp.fp_valid); + + /* Allocate an AVX512 savearea and copy AVX state into it */ + bcopy(ifps, ifps512, fp_state_size[AVX]); + pcb->ifps = ifps512; + pcb->xstate = AVX512; + fp_state_free(ifps, AVX); + + simple_unlock(&pcb->lock); +} + +/* + * Upgrade the calling thread to AVX512. + */ +boolean_t +fpu_thread_promote_avx512(thread_t thread) +{ + task_t task = current_task(); + + if (thread != current_thread()) + return FALSE; + if (!ml_fpu_avx512_enabled()) + return FALSE; + + fpu_savearea_promote_avx512(thread); + + /* Racy but the task's xstate is only a hint */ + task->xstate = AVX512; + + return TRUE; +} + + +/* + * Called from user_trap() when an invalid opcode fault is taken. + * If the user is attempting an AVX512 instruction on a machine + * that supports this, we switch the calling thread to use + * a larger savearea, set its XCR0 bit mask to enable AVX512 and + * return directly via thread_exception_return(). + * Otherwise simply return. + */ +void +fpUDflt(user_addr_t rip) +{ + uint8_t instruction_prefix; + boolean_t is_AVX512_instruction = FALSE; + + do { + if (copyin(rip, (char *) &instruction_prefix, 1)) + return; + DBG("fpUDflt(0x%016llx) prefix: 0x%x\n", + rip, instruction_prefix); + switch (instruction_prefix) { + case 0x2E: /* CS segment override */ + case 0x36: /* SS segment override */ + case 0x3E: /* DS segment override */ + case 0x26: /* ES segment override */ + case 0x64: /* FS segment override */ + case 0x65: /* GS segment override */ + case 0x67: /* address-size override */ + /* Skip optional prefixes */ + rip++; + break; + case 0x62: /* EVEX */ + case 0xC5: /* VEX 2-byte */ + case 0xC4: /* VEX 3-byte */ + is_AVX512_instruction = TRUE; + break; + default: + return; + } + } while (!is_AVX512_instruction); + + /* Here if we detect attempted execution of an AVX512 instruction */ + + /* + * Fail if this machine doesn't support AVX512 or + * the current thread is (strangely) already in AVX512 mode. + */ + if (fpu_capability != AVX512 || + current_xstate() == AVX512) + return; + + assert(xgetbv(XCR0) == AVX_XMASK); + + DBG("fpUDflt() switching xstate to AVX512\n"); + (void) fpu_thread_promote_avx512(current_thread()); + + thread_exception_return(); + /* NOT REACHED */ +} +#endif /* !defined(RC_HIDE_XNU_J137) */ + void fp_setvalid(boolean_t value) { thread_t thr_act = current_thread(); @@ -993,5 +1551,55 @@ fp_setvalid(boolean_t value) { boolean_t ml_fpu_avx_enabled(void) { - return (fpu_YMM_present == TRUE); + return (fpu_capability >= AVX); +} + +#if !defined(RC_HIDE_XNU_J137) +boolean_t +ml_fpu_avx512_enabled(void) { + return (fpu_capability == AVX512); +} +#endif + +static xstate_t +task_xstate(task_t task) +{ + if (task == TASK_NULL) + return fpu_default; + else + return task->xstate; +} + +static xstate_t +thread_xstate(thread_t thread) +{ + xstate_t xs = THREAD_TO_PCB(thread)->xstate; + if (xs == UNDEFINED) + return task_xstate(thread->task); + else + return xs; +} + +xstate_t +current_xstate(void) +{ + return thread_xstate(current_thread()); +} + +/* + * Called when exec'ing between bitnesses. + * If valid FPU state exists, adjust the layout. + */ +void +fpu_switch_addrmode(thread_t thread, boolean_t is_64bit) +{ + struct x86_fx_thread_state *ifps = thread->machine.ifps; + + if (ifps && ifps->fp_valid) { + if (thread_xstate(thread) == FP) { + ifps->fp_save_layout = is_64bit ? FXSAVE64 : FXSAVE32; + } else { + ifps->fp_save_layout = is_64bit ? XSAVE64 : XSAVE32; + } + } } diff --git a/osfmk/i386/fpu.h b/osfmk/i386/fpu.h index 12a5082b6..84a03460d 100644 --- a/osfmk/i386/fpu.h +++ b/osfmk/i386/fpu.h @@ -61,12 +61,10 @@ * Macro definitions for routines to manipulate the * floating-point processor. */ -#include #include #include #include #include -#include typedef enum { FXSAVE32 = 1, @@ -76,12 +74,20 @@ typedef enum { FP_UNUSED = 5 } fp_save_layout_t; -extern int fp_kind; +typedef enum { + UNDEFINED, + FP, + AVX, +#if !defined(RC_HIDE_XNU_J137) + AVX512 +#endif +} xstate_t; extern void init_fpu(void); extern void fpu_module_init(void); extern void fpu_free( - void * fps); + thread_t thr_act, + void *fps); extern kern_return_t fpu_set_fxstate( thread_t thr_act, thread_state_t state, @@ -100,7 +106,16 @@ extern void fpSSEexterrflt(void); extern void fpflush(thread_t); extern void fp_setvalid(boolean_t); -extern void clear_fpu(void); -extern void fpu_save_context(thread_t thread); +extern void clear_fpu(void); +extern void fpu_switch_context( + thread_t old, + thread_t new); +extern void fpu_switch_addrmode( + thread_t thread, + boolean_t is_64bit); + +extern xstate_t fpu_default; +extern xstate_t current_xstate(void); +extern void fpUDflt(user_addr_t rip); #endif /* _I386_FPU_H_ */ diff --git a/osfmk/i386/genassym.c b/osfmk/i386/genassym.c index 1a0c71cef..510bdf89f 100644 --- a/osfmk/i386/genassym.c +++ b/osfmk/i386/genassym.c @@ -54,6 +54,8 @@ * the rights to redistribute these changes. */ +#include + #include /* @@ -84,9 +86,6 @@ #include #include -#undef offsetof -#include - #if CONFIG_DTRACE #define NEED_DTRACE_DEFS #include <../bsd/sys/lockstat.h> @@ -193,19 +192,19 @@ main( DECLARE("MAP_PMAP", offsetof(struct _vm_map, pmap)); #define IEL_SIZE (sizeof(struct i386_exception_link *)) - DECLARE("IKS_SIZE", sizeof(struct x86_kernel_state)); + DECLARE("IKS_SIZE", sizeof(struct thread_kernel_state)); /* * KSS_* are offsets from the top of the kernel stack (cpu_kernel_stack) */ - DECLARE("KSS_RBX", offsetof(struct x86_kernel_state, k_rbx)); - DECLARE("KSS_RSP", offsetof(struct x86_kernel_state, k_rsp)); - DECLARE("KSS_RBP", offsetof(struct x86_kernel_state, k_rbp)); - DECLARE("KSS_R12", offsetof(struct x86_kernel_state, k_r12)); - DECLARE("KSS_R13", offsetof(struct x86_kernel_state, k_r13)); - DECLARE("KSS_R14", offsetof(struct x86_kernel_state, k_r14)); - DECLARE("KSS_R15", offsetof(struct x86_kernel_state, k_r15)); - DECLARE("KSS_RIP", offsetof(struct x86_kernel_state, k_rip)); + DECLARE("KSS_RBX", offsetof(struct thread_kernel_state, machine.k_rbx)); + DECLARE("KSS_RSP", offsetof(struct thread_kernel_state, machine.k_rsp)); + DECLARE("KSS_RBP", offsetof(struct thread_kernel_state, machine.k_rbp)); + DECLARE("KSS_R12", offsetof(struct thread_kernel_state, machine.k_r12)); + DECLARE("KSS_R13", offsetof(struct thread_kernel_state, machine.k_r13)); + DECLARE("KSS_R14", offsetof(struct thread_kernel_state, machine.k_r14)); + DECLARE("KSS_R15", offsetof(struct thread_kernel_state, machine.k_r15)); + DECLARE("KSS_RIP", offsetof(struct thread_kernel_state, machine.k_rip)); DECLARE("DS_DR0", offsetof(struct x86_debug_state32, dr0)); DECLARE("DS_DR1", offsetof(struct x86_debug_state32, dr1)); @@ -337,10 +336,8 @@ main( offsetof(cpu_data_t, cpu_kernel_stack)); DECLARE("CPU_INT_STACK_TOP", offsetof(cpu_data_t, cpu_int_stack_top)); -#if MACH_RT DECLARE("CPU_PREEMPTION_LEVEL", offsetof(cpu_data_t, cpu_preemption_level)); -#endif /* MACH_RT */ DECLARE("CPU_HIBERNATE", offsetof(cpu_data_t, cpu_hibernate)); DECLARE("CPU_INTERRUPT_LEVEL", diff --git a/osfmk/i386/hibernate_i386.c b/osfmk/i386/hibernate_i386.c index a16994234..21322b751 100644 --- a/osfmk/i386/hibernate_i386.c +++ b/osfmk/i386/hibernate_i386.c @@ -43,6 +43,7 @@ #include #include #include +#include extern ppnum_t max_ppnum; @@ -65,6 +66,7 @@ hibernate_page_list_allocate(boolean_t log) hibernate_bitmap_t dram_ranges[MAX_BANKS]; boot_args * args = (boot_args *) PE_state.bootArgs; uint32_t non_os_pagecount; + ppnum_t pnmax = max_ppnum; mptr = (EfiMemoryRange *)ml_static_ptovirt(args->MemoryMap); if (args->MemoryMapDescriptorSize == 0) @@ -72,6 +74,13 @@ hibernate_page_list_allocate(boolean_t log) msize = args->MemoryMapDescriptorSize; mcount = args->MemoryMapSize / msize; +#if KASAN + /* adjust max page number to include stolen memory */ + if (atop(shadow_ptop) > pnmax) { + pnmax = (ppnum_t)atop(shadow_ptop); + } +#endif + num_banks = 0; non_os_pagecount = 0; for (i = 0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) @@ -79,10 +88,20 @@ hibernate_page_list_allocate(boolean_t log) base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT); num = (ppnum_t) mptr->NumberOfPages; - if (base > max_ppnum) +#if KASAN + if (i == shadow_stolen_idx) { + /* + * Add all stolen pages to the bitmap. Later we will prune the unused + * pages. + */ + num += shadow_pages_total; + } +#endif + + if (base > pnmax) continue; - if ((base + num - 1) > max_ppnum) - num = max_ppnum - base + 1; + if ((base + num - 1) > pnmax) + num = pnmax - base + 1; if (!num) continue; @@ -225,14 +244,40 @@ hibernate_processor_setup(IOHibernateImageHeader * header) return (KERN_SUCCESS); } +static boolean_t hibernate_vm_locks_safe; + void hibernate_vm_lock(void) { - if (current_cpu_datap()->cpu_hibernate) hibernate_vm_lock_queues(); + if (current_cpu_datap()->cpu_hibernate) { + hibernate_vm_lock_queues(); + hibernate_vm_locks_safe = TRUE; + } } void hibernate_vm_unlock(void) { + assert(FALSE == ml_get_interrupts_enabled()); if (current_cpu_datap()->cpu_hibernate) hibernate_vm_unlock_queues(); + ml_set_is_quiescing(TRUE); +} + +// ACPI calls hibernate_vm_lock(), interrupt disable, hibernate_vm_unlock() on sleep, +// hibernate_vm_lock_end() and interrupt enable on wake. +// VM locks are safely single threaded between hibernate_vm_lock() and hibernate_vm_lock_end(). + +void +hibernate_vm_lock_end(void) +{ + assert(FALSE == ml_get_interrupts_enabled()); + hibernate_vm_locks_safe = FALSE; + ml_set_is_quiescing(FALSE); +} + +boolean_t +hibernate_vm_locks_are_safe(void) +{ + assert(FALSE == ml_get_interrupts_enabled()); + return (hibernate_vm_locks_safe); } diff --git a/osfmk/i386/hibernate_restore.c b/osfmk/i386/hibernate_restore.c index 278ff4e78..2e4c69e79 100644 --- a/osfmk/i386/hibernate_restore.c +++ b/osfmk/i386/hibernate_restore.c @@ -50,7 +50,7 @@ hibernate_restore_phys_page(uint64_t src, uint64_t dst, uint32_t len, uint32_t p d = (uint64_t *)pal_hib_map(DEST_COPY_AREA, dst); s = (uint64_t *) (uintptr_t)src; - memcpy(d, s, len); + __nosan_memcpy(d, s, len); return (uintptr_t)d; } diff --git a/osfmk/i386/i386_init.c b/osfmk/i386/i386_init.c index 2d8abbdb6..73a30a22b 100644 --- a/osfmk/i386/i386_init.c +++ b/osfmk/i386/i386_init.c @@ -102,6 +102,13 @@ #if DEBUG #include #endif + +#if MONOTONIC +#include +#endif /* MONOTONIC */ + +#include + #if DEBUG #define DBG(x...) kprintf(x) #else @@ -309,6 +316,47 @@ Idle_PTs_init(void) } +extern void vstart_trap_handler; + +#define BOOT_TRAP_VECTOR(t) \ + [t] = { \ + (uintptr_t) &vstart_trap_handler, \ + KERNEL64_CS, \ + 0, \ + ACC_P|ACC_PL_K|ACC_INTR_GATE, \ + 0 \ + }, + +/* Recursive macro to iterate 0..31 */ +#define L0(x,n) x(n) +#define L1(x,n) L0(x,n-1) L0(x,n) +#define L2(x,n) L1(x,n-2) L1(x,n) +#define L3(x,n) L2(x,n-4) L2(x,n) +#define L4(x,n) L3(x,n-8) L3(x,n) +#define L5(x,n) L4(x,n-16) L4(x,n) +#define FOR_0_TO_31(x) L5(x,31) + +/* + * Bootstrap IDT. Active only during early startup. + * Only the trap vectors are defined since interrupts are masked. + * All traps point to a common handler. + */ +struct fake_descriptor64 master_boot_idt64[IDTSZ] + __attribute__((section("__HIB,__desc"))) + __attribute__((aligned(PAGE_SIZE))) = { + FOR_0_TO_31(BOOT_TRAP_VECTOR) +}; + +static void +vstart_idt_init(void) +{ + x86_64_desc_register_t vstart_idt = { + sizeof(master_boot_idt64), + master_boot_idt64 }; + + fix_desc64(master_boot_idt64, 32); + lidt((void *)&vstart_idt); +} /* * vstart() is called in the natural mode (64bit for K64, 32 for K32) @@ -330,12 +378,18 @@ void vstart(vm_offset_t boot_args_start) { boolean_t is_boot_cpu = !(boot_args_start == 0); - int cpu; + int cpu = 0; uint32_t lphysfree; postcode(VSTART_ENTRY); if (is_boot_cpu) { + /* + * Set-up temporary trap handlers during page-table set-up. + */ + vstart_idt_init(); + postcode(VSTART_IDT_INIT); + /* * Get startup parameters. */ @@ -370,34 +424,49 @@ vstart(vm_offset_t boot_args_start) ml_static_ptovirt(boot_args_start); DBG("i386_init(0x%lx) kernelBootArgs=%p\n", (unsigned long)boot_args_start, kernelBootArgs); + +#if KASAN + kasan_reserve_memory(kernelBootArgs); +#endif + PE_init_platform(FALSE, kernelBootArgs); postcode(PE_INIT_PLATFORM_D); Idle_PTs_init(); postcode(VSTART_IDLE_PTS_INIT); +#if KASAN + /* Init kasan and map whatever was stolen from physfree */ + kasan_init(); + kasan_notify_stolen((uintptr_t)ml_static_ptovirt((vm_offset_t)physfree)); +#endif + +#if MONOTONIC + mt_init(); +#endif /* MONOTONIC */ + first_avail = (vm_offset_t)ID_MAP_VTOP(physfree); - cpu = 0; cpu_data_alloc(TRUE); + + cpu_desc_init(cpu_datap(0)); + postcode(VSTART_CPU_DESC_INIT); + cpu_desc_load(cpu_datap(0)); + + postcode(VSTART_CPU_MODE_INIT); + cpu_syscall_init(cpu_datap(0)); /* cpu_syscall_init() will be + * invoked on the APs + * via i386_init_slave() + */ } else { /* Switch to kernel's page tables (from the Boot PTs) */ set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4)); /* Find our logical cpu number */ cpu = lapic_to_cpu[(LAPIC_READ(ID)>>LAPIC_ID_SHIFT) & LAPIC_ID_MASK]; DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, rdmsr64(MSR_IA32_GS_BASE)); + cpu_desc_load(cpu_datap(cpu)); } - postcode(VSTART_CPU_DESC_INIT); - if(is_boot_cpu) - cpu_desc_init64(cpu_datap(cpu)); - cpu_desc_load64(cpu_datap(cpu)); - postcode(VSTART_CPU_MODE_INIT); - if (is_boot_cpu) - cpu_mode_init(current_cpu_datap()); /* cpu_mode_init() will be - * invoked on the APs - * via i386_init_slave() - */ postcode(VSTART_EXIT); x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init : (uintptr_t) i386_init_slave, @@ -455,13 +524,23 @@ i386_init(void) dgWork.dgFlags = 0; serialmode = 0; - if(PE_parse_boot_argn("serial", &serialmode, sizeof (serialmode))) { + if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) { /* We want a serial keyboard and/or console */ kprintf("Serial mode specified: %08X\n", serialmode); + int force_sync = serialmode & SERIALMODE_SYNCDRAIN; + if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) { + if (force_sync) { + serialmode |= SERIALMODE_SYNCDRAIN; + kprintf( + "WARNING: Forcing uart driver to output synchronously." + "printf()s/IOLogs will impact kernel performance.\n" + "You are advised to avoid using 'drain_uart_sync' boot-arg.\n"); + } + } } - if(serialmode & 1) { + if (serialmode & SERIALMODE_OUTPUT) { (void)switch_to_serial_console(); - disableConsoleOutput = FALSE; /* Allow printfs to happen */ + disableConsoleOutput = FALSE; /* Allow printfs to happen */ } /* setup console output */ @@ -542,7 +621,7 @@ do_init_slave(boolean_t fast_restart) assert(!ml_get_interrupts_enabled()); - cpu_mode_init(current_cpu_datap()); + cpu_syscall_init(current_cpu_datap()); pmap_cpu_init(); #if CONFIG_MCA diff --git a/osfmk/i386/i386_lock.s b/osfmk/i386/i386_lock.s index f54e040a1..d657afaee 100644 --- a/osfmk/i386/i386_lock.s +++ b/osfmk/i386/i386_lock.s @@ -35,7 +35,6 @@ * the terms and conditions for use and redistribution. */ -#include #include #include #include @@ -117,18 +116,37 @@ .text ; \ 1: +#define CHECK_MYLOCK(current, owner) \ + cmp current, owner ; \ + jne 1f ; \ + ALIGN_STACK() ; \ + LOAD_STRING_ARG0(2f) ; \ + CALL_PANIC() ; \ + hlt ; \ + .data ; \ +2: String "Attempt to recursively lock a non-recursive lock"; \ + .text ; \ +1: + +#else /* MACH_LDEBUG */ +#define CHECK_MUTEX_TYPE() +#define CHECK_MYLOCK(thd) +#endif /* MACH_LDEBUG */ + +#if DEVELOPMENT || DEBUG /* * If one or more simplelocks are currently held by a thread, * an attempt to acquire a mutex will cause this check to fail * (since a mutex lock may context switch, holding a simplelock * is not a good thing). */ -#if MACH_RT #define CHECK_PREEMPTION_LEVEL() \ - cmpl $0,%gs:CPU_HIBERNATE ; \ - jne 1f ; \ cmpl $0,%gs:CPU_PREEMPTION_LEVEL ; \ je 1f ; \ + cmpl $0,EXT(LckDisablePreemptCheck)(%rip) ; \ + jne 1f ; \ + cmpl $0,%gs:CPU_HIBERNATE ; \ + jne 1f ; \ ALIGN_STACK() ; \ movl %gs:CPU_PREEMPTION_LEVEL, %eax ; \ LOAD_ARG1(%eax) ; \ @@ -139,27 +157,9 @@ 2: String "preemption_level(%d) != 0!" ; \ .text ; \ 1: -#else /* MACH_RT */ -#define CHECK_PREEMPTION_LEVEL() -#endif /* MACH_RT */ - -#define CHECK_MYLOCK(current, owner) \ - cmp current, owner ; \ - jne 1f ; \ - ALIGN_STACK() ; \ - LOAD_STRING_ARG0(2f) ; \ - CALL_PANIC() ; \ - hlt ; \ - .data ; \ -2: String "Attempt to recursively lock a non-recursive lock"; \ - .text ; \ -1: - -#else /* MACH_LDEBUG */ -#define CHECK_MUTEX_TYPE() +#else /* DEVELOPMENT || DEBUG */ #define CHECK_PREEMPTION_LEVEL() -#define CHECK_MYLOCK(thd) -#endif /* MACH_LDEBUG */ +#endif /* DEVELOPMENT || DEBUG */ #define PREEMPTION_DISABLE \ incl %gs:CPU_PREEMPTION_LEVEL @@ -258,7 +258,7 @@ LEAF_ENTRY(hw_lock_byte_init) * void hw_lock_byte_lock(uint8_t *lock_byte) * * Acquire byte sized lock operand, spinning until it becomes available. - * MACH_RT: also return with preemption disabled. + * return with preemption disabled. */ LEAF_ENTRY(hw_lock_byte_lock) @@ -278,477 +278,14 @@ LEAF_ENTRY(hw_lock_byte_lock) /* * void hw_lock_byte_unlock(uint8_t *lock_byte) * - * Unconditionally release byte sized lock operand. - * MACH_RT: release preemption level. + * Unconditionally release byte sized lock operand, + * release preemption level. */ LEAF_ENTRY(hw_lock_byte_unlock) movb $0, (%rdi) /* Clear the lock byte */ PREEMPTION_ENABLE LEAF_RET - -/* - * Reader-writer lock fastpaths. These currently exist for the - * shared lock acquire, the exclusive lock acquire, the shared to - * exclusive upgrade and the release paths (where they reduce overhead - * considerably) -- these are by far the most frequently used routines - * - * The following should reflect the layout of the bitfield embedded within - * the lck_rw_t structure (see i386/locks.h). - */ -#define LCK_RW_INTERLOCK (0x1 << 16) - -#define LCK_RW_PRIV_EXCL (0x1 << 24) -#define LCK_RW_WANT_UPGRADE (0x2 << 24) -#define LCK_RW_WANT_WRITE (0x4 << 24) -#define LCK_R_WAITING (0x8 << 24) -#define LCK_W_WAITING (0x10 << 24) - -#define LCK_RW_SHARED_MASK (0xffff) - -/* - * For most routines, the lck_rw_t pointer is loaded into a - * register initially, and the flags bitfield loaded into another - * register and examined - */ - -#define RW_LOCK_SHARED_MASK (LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE) -/* - * void lck_rw_lock_shared(lck_rw_t *) - * - */ -Entry(lck_rw_lock_shared) - mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */ - incl TH_RWLOCK_COUNT(%rcx) /* Increment count before atomic CAS */ -1: - mov (%rdi), %eax /* Load state bitfield and interlock */ - testl $(RW_LOCK_SHARED_MASK), %eax /* Eligible for fastpath? */ - jne 3f - - movl %eax, %ecx /* original value in %eax for cmpxchgl */ - incl %ecx /* Increment reader refcount */ - lock - cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */ - jne 2f - -#if CONFIG_DTRACE - /* - * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_ACQUIRE - * Implemented by swapping between return and no-op instructions. - * See bsd/dev/dtrace/lockstat.c. - */ - LOCKSTAT_LABEL(_lck_rw_lock_shared_lockstat_patch_point) - ret - /* - Fall thru when patched, counting on lock pointer in %rdi - */ - LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, %rdi) -#endif - ret -2: - PAUSE - jmp 1b -3: - jmp EXT(lck_rw_lock_shared_gen) - - - -#define RW_TRY_LOCK_SHARED_MASK (LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE) -/* - * void lck_rw_try_lock_shared(lck_rw_t *) - * - */ -Entry(lck_rw_try_lock_shared) -1: - mov (%rdi), %eax /* Load state bitfield and interlock */ - testl $(LCK_RW_INTERLOCK), %eax - jne 2f - testl $(RW_TRY_LOCK_SHARED_MASK), %eax - jne 3f /* lock is busy */ - - movl %eax, %ecx /* original value in %eax for cmpxchgl */ - incl %ecx /* Increment reader refcount */ - lock - cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */ - jne 2f - - mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */ - incl TH_RWLOCK_COUNT(%rcx) /* Increment count on success. */ - /* There is a 3 instr window where preemption may not notice rwlock_count after cmpxchg */ - -#if CONFIG_DTRACE - movl $1, %eax - /* - * Dtrace lockstat event: LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE - * Implemented by swapping between return and no-op instructions. - * See bsd/dev/dtrace/lockstat.c. - */ - LOCKSTAT_LABEL(_lck_rw_try_lock_shared_lockstat_patch_point) - ret - /* Fall thru when patched, counting on lock pointer in %rdi */ - LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, %rdi) -#endif - movl $1, %eax /* return TRUE */ - ret -2: - PAUSE - jmp 1b -3: - xorl %eax, %eax - ret - - -#define RW_LOCK_EXCLUSIVE_HELD (LCK_RW_WANT_WRITE | LCK_RW_WANT_UPGRADE) -/* - * int lck_rw_grab_shared(lck_rw_t *) - * - */ -Entry(lck_rw_grab_shared) -1: - mov (%rdi), %eax /* Load state bitfield and interlock */ - testl $(LCK_RW_INTERLOCK), %eax - jne 5f - testl $(RW_LOCK_EXCLUSIVE_HELD), %eax - jne 3f -2: - movl %eax, %ecx /* original value in %eax for cmpxchgl */ - incl %ecx /* Increment reader refcount */ - lock - cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */ - jne 4f - - movl $1, %eax /* return success */ - ret -3: - testl $(LCK_RW_SHARED_MASK), %eax - je 4f - testl $(LCK_RW_PRIV_EXCL), %eax - je 2b -4: - xorl %eax, %eax /* return failure */ - ret -5: - PAUSE - jmp 1b - - - -#define RW_LOCK_EXCLUSIVE_MASK (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | \ - LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE) -/* - * void lck_rw_lock_exclusive(lck_rw_t*) - * - */ -Entry(lck_rw_lock_exclusive) - mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */ - incl TH_RWLOCK_COUNT(%rcx) /* Increment count before atomic CAS */ -1: - mov (%rdi), %eax /* Load state bitfield, interlock and shared count */ - testl $(RW_LOCK_EXCLUSIVE_MASK), %eax /* Eligible for fastpath? */ - jne 3f /* no, go slow */ - - movl %eax, %ecx /* original value in %eax for cmpxchgl */ - orl $(LCK_RW_WANT_WRITE), %ecx - lock - cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */ - jne 2f - -#if CONFIG_DTRACE - /* - * Dtrace lockstat event: LS_LCK_RW_LOCK_EXCL_ACQUIRE - * Implemented by swapping between return and no-op instructions. - * See bsd/dev/dtrace/lockstat.c. - */ - LOCKSTAT_LABEL(_lck_rw_lock_exclusive_lockstat_patch_point) - ret - /* Fall thru when patched, counting on lock pointer in %rdi */ - LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, %rdi) -#endif - ret -2: - PAUSE - jmp 1b -3: - jmp EXT(lck_rw_lock_exclusive_gen) - - - -#define RW_TRY_LOCK_EXCLUSIVE_MASK (LCK_RW_SHARED_MASK | LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE) -/* - * void lck_rw_try_lock_exclusive(lck_rw_t *) - * - * Tries to get a write lock. - * - * Returns FALSE if the lock is not held on return. - */ -Entry(lck_rw_try_lock_exclusive) -1: - mov (%rdi), %eax /* Load state bitfield, interlock and shared count */ - testl $(LCK_RW_INTERLOCK), %eax - jne 2f - testl $(RW_TRY_LOCK_EXCLUSIVE_MASK), %eax - jne 3f /* can't get it */ - - movl %eax, %ecx /* original value in %eax for cmpxchgl */ - orl $(LCK_RW_WANT_WRITE), %ecx - lock - cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */ - jne 2f - - mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */ - incl TH_RWLOCK_COUNT(%rcx) /* Increment count on success. */ - /* There is a 3 instr window where preemption may not notice rwlock_count after cmpxchg */ - -#if CONFIG_DTRACE - movl $1, %eax - /* - * Dtrace lockstat event: LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE - * Implemented by swapping between return and no-op instructions. - * See bsd/dev/dtrace/lockstat.c. - */ - LOCKSTAT_LABEL(_lck_rw_try_lock_exclusive_lockstat_patch_point) - ret - /* Fall thru when patched, counting on lock pointer in %rdi */ - LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, %rdi) -#endif - movl $1, %eax /* return TRUE */ - ret -2: - PAUSE - jmp 1b -3: - xorl %eax, %eax /* return FALSE */ - ret - - - -/* - * void lck_rw_lock_shared_to_exclusive(lck_rw_t*) - * - * fastpath can be taken if - * the current rw_shared_count == 1 - * AND the interlock is clear - * AND RW_WANT_UPGRADE is not set - * - * note that RW_WANT_WRITE could be set, but will not - * be indicative of an exclusive hold since we have - * a read count on the lock that we have not yet released - * we can blow by that state since the lck_rw_lock_exclusive - * function will block until rw_shared_count == 0 and - * RW_WANT_UPGRADE is clear... it does this check behind - * the interlock which we are also checking for - * - * to make the transition we must be able to atomically - * set RW_WANT_UPGRADE and get rid of the read count we hold - */ -Entry(lck_rw_lock_shared_to_exclusive) -1: - mov (%rdi), %eax /* Load state bitfield, interlock and shared count */ - testl $(LCK_RW_INTERLOCK), %eax - jne 7f - testl $(LCK_RW_WANT_UPGRADE), %eax - jne 2f - - movl %eax, %ecx /* original value in %eax for cmpxchgl */ - orl $(LCK_RW_WANT_UPGRADE), %ecx /* ask for WANT_UPGRADE */ - decl %ecx /* and shed our read count */ - lock - cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */ - jne 7f - /* we now own the WANT_UPGRADE */ - testl $(LCK_RW_SHARED_MASK), %ecx /* check to see if all of the readers are drained */ - jne 8f /* if not, we need to go wait */ - -#if CONFIG_DTRACE - movl $1, %eax - /* - * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE - * Implemented by swapping between return and no-op instructions. - * See bsd/dev/dtrace/lockstat.c. - */ - LOCKSTAT_LABEL(_lck_rw_lock_shared_to_exclusive_lockstat_patch_point) - ret - /* Fall thru when patched, counting on lock pointer in %rdi */ - LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, %rdi) -#endif - movl $1, %eax /* return success */ - ret - -2: /* someone else already holds WANT_UPGRADE */ - movl %eax, %ecx /* original value in %eax for cmpxchgl */ - decl %ecx /* shed our read count */ - testl $(LCK_RW_SHARED_MASK), %ecx - jne 3f /* we were the last reader */ - andl $(~LCK_W_WAITING), %ecx /* so clear the wait indicator */ -3: - lock - cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */ - jne 7f - - mov %eax, %esi /* put old flags as second arg */ - /* lock is alread in %rdi */ - call EXT(lck_rw_lock_shared_to_exclusive_failure) - ret /* and pass the failure return along */ -7: - PAUSE - jmp 1b -8: - jmp EXT(lck_rw_lock_shared_to_exclusive_success) - - - - .cstring -rwl_release_error_str: - .asciz "Releasing non-exclusive RW lock without a reader refcount!" - .text - -/* - * lck_rw_type_t lck_rw_done(lck_rw_t *) - * - */ -Entry(lck_rw_done) -1: - mov (%rdi), %eax /* Load state bitfield, interlock and reader count */ - testl $(LCK_RW_INTERLOCK), %eax - jne 7f /* wait for interlock to clear */ - - movl %eax, %ecx /* keep original value in %eax for cmpxchgl */ - testl $(LCK_RW_SHARED_MASK), %ecx /* if reader count == 0, must be exclusive lock */ - je 2f - decl %ecx /* Decrement reader count */ - testl $(LCK_RW_SHARED_MASK), %ecx /* if reader count has now gone to 0, check for waiters */ - je 4f - jmp 6f -2: - testl $(LCK_RW_WANT_UPGRADE), %ecx - je 3f - andl $(~LCK_RW_WANT_UPGRADE), %ecx - jmp 4f -3: - testl $(LCK_RW_WANT_WRITE), %ecx - je 8f /* lock is not 'owned', go panic */ - andl $(~LCK_RW_WANT_WRITE), %ecx -4: - /* - * test the original values to match what - * lck_rw_done_gen is going to do to determine - * which wakeups need to happen... - * - * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting) - */ - testl $(LCK_W_WAITING), %eax - je 5f - andl $(~LCK_W_WAITING), %ecx - - testl $(LCK_RW_PRIV_EXCL), %eax - jne 6f -5: - andl $(~LCK_R_WAITING), %ecx -6: - lock - cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */ - jne 7f - - mov %eax,%esi /* old flags in %rsi */ - /* lock is in %rdi already */ - call EXT(lck_rw_done_gen) - ret -7: - PAUSE - jmp 1b -8: - ALIGN_STACK() - LOAD_STRING_ARG0(rwl_release_error_str) - CALL_PANIC() - - - -/* - * lck_rw_type_t lck_rw_lock_exclusive_to_shared(lck_rw_t *) - * - */ -Entry(lck_rw_lock_exclusive_to_shared) -1: - mov (%rdi), %eax /* Load state bitfield, interlock and reader count */ - testl $(LCK_RW_INTERLOCK), %eax - jne 6f /* wait for interlock to clear */ - - movl %eax, %ecx /* keep original value in %eax for cmpxchgl */ - incl %ecx /* Increment reader count */ - - testl $(LCK_RW_WANT_UPGRADE), %ecx - je 2f - andl $(~LCK_RW_WANT_UPGRADE), %ecx - jmp 3f -2: - andl $(~LCK_RW_WANT_WRITE), %ecx -3: - /* - * test the original values to match what - * lck_rw_lock_exclusive_to_shared_gen is going to do to determine - * which wakeups need to happen... - * - * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting) - */ - testl $(LCK_W_WAITING), %eax - je 4f - testl $(LCK_RW_PRIV_EXCL), %eax - jne 5f -4: - andl $(~LCK_R_WAITING), %ecx -5: - lock - cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */ - jne 6f - - mov %eax,%esi - call EXT(lck_rw_lock_exclusive_to_shared_gen) - ret -6: - PAUSE - jmp 1b - - - -/* - * int lck_rw_grab_want(lck_rw_t *) - * - */ -Entry(lck_rw_grab_want) -1: - mov (%rdi), %eax /* Load state bitfield, interlock and reader count */ - testl $(LCK_RW_INTERLOCK), %eax - jne 3f /* wait for interlock to clear */ - testl $(LCK_RW_WANT_WRITE), %eax /* want_write has been grabbed by someone else */ - jne 2f /* go return failure */ - - movl %eax, %ecx /* original value in %eax for cmpxchgl */ - orl $(LCK_RW_WANT_WRITE), %ecx - lock - cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */ - jne 2f - /* we now own want_write */ - movl $1, %eax /* return success */ - ret -2: - xorl %eax, %eax /* return failure */ - ret -3: - PAUSE - jmp 1b - - -#define RW_LOCK_SHARED_OR_UPGRADE_MASK (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE) -/* - * int lck_rw_held_read_or_upgrade(lck_rw_t *) - * - */ -Entry(lck_rw_held_read_or_upgrade) - mov (%rdi), %eax - andl $(RW_LOCK_SHARED_OR_UPGRADE_MASK), %eax - ret - - /* * N.B.: On x86, statistics are currently recorded for all indirect mutexes. diff --git a/osfmk/i386/i386_timer.c b/osfmk/i386/i386_timer.c index ae6eb1029..f302314e9 100644 --- a/osfmk/i386/i386_timer.c +++ b/osfmk/i386/i386_timer.c @@ -129,8 +129,16 @@ timer_intr(int user_mode, DECR_PM_DEADLINE | DBG_FUNC_END, 0, 0, 0, 0, 0); timer_processed = TRUE; + abstime = mach_absolute_time(); /* Get the time again since we ran a bit */ } + uint64_t quantum_deadline = pp->quantum_timer_deadline; + /* is it the quantum timer expiration? */ + if ((quantum_deadline <= abstime) && (quantum_deadline > 0)) { + pp->quantum_timer_deadline = 0; + quantum_timer_expire(abstime); + } + /* schedule our next deadline */ x86_lcpu()->rtcDeadline = EndOfAllTime; timer_resync_deadlines(); @@ -160,6 +168,18 @@ void timer_set_deadline(uint64_t deadline) splx(s); } +void +quantum_timer_set_deadline(uint64_t deadline) +{ + cpu_data_t *pp; + /* We should've only come into this path with interrupts disabled */ + assert(ml_get_interrupts_enabled() == FALSE); + + pp = current_cpu_datap(); + pp->quantum_timer_deadline = deadline; + timer_resync_deadlines(); +} + /* * Re-evaluate the outstanding deadlines and select the most proximate. * @@ -170,6 +190,7 @@ timer_resync_deadlines(void) { uint64_t deadline = EndOfAllTime; uint64_t pmdeadline; + uint64_t quantum_deadline; rtclock_timer_t *mytimer; spl_t s = splclock(); cpu_data_t *pp; @@ -195,6 +216,13 @@ timer_resync_deadlines(void) if (0 < pmdeadline && pmdeadline < deadline) deadline = pmdeadline; + /* If we have the quantum timer setup, check that */ + quantum_deadline = pp->quantum_timer_deadline; + if ((quantum_deadline > 0) && + (quantum_deadline < deadline)) + deadline = quantum_deadline; + + /* * Go and set the "pop" event. */ @@ -202,11 +230,19 @@ timer_resync_deadlines(void) /* Record non-PM deadline for latency tool */ if (decr != 0 && deadline != pmdeadline) { + uint64_t queue_count = 0; + if (deadline != quantum_deadline) { + /* + * For non-quantum timer put the queue count + * in the tracepoint. + */ + queue_count = mytimer->queue.count; + } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, - DECR_SET_DEADLINE | DBG_FUNC_NONE, - decr, 2, - deadline, - mytimer->queue.count, 0); + DECR_SET_DEADLINE | DBG_FUNC_NONE, + decr, 2, + deadline, + queue_count, 0); } splx(s); } diff --git a/osfmk/i386/i386_vm_init.c b/osfmk/i386/i386_vm_init.c index 3ae00d697..e623d6004 100644 --- a/osfmk/i386/i386_vm_init.c +++ b/osfmk/i386/i386_vm_init.c @@ -165,12 +165,57 @@ uint64_t firmware_MMIO_bytes; */ extern void *last_kernel_symbol; -#if DEBUG -#define PRINT_PMAP_MEMORY_TABLE -#define DBG(x...) kprintf(x) +boolean_t memmap = FALSE; +#if DEBUG || DEVELOPMENT +static void +kprint_memmap(vm_offset_t maddr, unsigned int msize, unsigned int mcount) { + unsigned int i; + unsigned int j; + pmap_memory_region_t *p = pmap_memory_regions; + EfiMemoryRange *mptr; + addr64_t region_start, region_end; + addr64_t efi_start, efi_end; + + for (j = 0; j < pmap_memory_region_count; j++, p++) { + kprintf("pmap region %d type %d base 0x%llx alloc_up 0x%llx alloc_down 0x%llx top 0x%llx\n", + j, p->type, + (addr64_t) p->base << I386_PGSHIFT, + (addr64_t) p->alloc_up << I386_PGSHIFT, + (addr64_t) p->alloc_down << I386_PGSHIFT, + (addr64_t) p->end << I386_PGSHIFT); + region_start = (addr64_t) p->base << I386_PGSHIFT; + region_end = ((addr64_t) p->end << I386_PGSHIFT) - 1; + mptr = (EfiMemoryRange *) maddr; + for (i = 0; + i < mcount; + i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) { + if (mptr->Type != kEfiLoaderCode && + mptr->Type != kEfiLoaderData && + mptr->Type != kEfiBootServicesCode && + mptr->Type != kEfiBootServicesData && + mptr->Type != kEfiConventionalMemory) { + efi_start = (addr64_t)mptr->PhysicalStart; + efi_end = efi_start + ((vm_offset_t)mptr->NumberOfPages << I386_PGSHIFT) - 1; + if ((efi_start >= region_start && efi_start <= region_end) || + (efi_end >= region_start && efi_end <= region_end)) { + kprintf(" *** Overlapping region with EFI runtime region %d\n", i); + } + } + } + } +} +#define DPRINTF(x...) do { if (memmap) kprintf(x); } while (0) + #else -#define DBG(x...) + +static void +kprint_memmap(vm_offset_t maddr, unsigned int msize, unsigned int mcount) { +#pragma unused(maddr, msize, mcount) +} + +#define DPRINTF(x...) #endif /* DEBUG */ + /* * Basic VM initialization. */ @@ -184,6 +229,7 @@ i386_vm_init(uint64_t maxmem, EfiMemoryRange *mptr; unsigned int mcount; unsigned int msize; + vm_offset_t maddr; ppnum_t fap; unsigned int i; ppnum_t maxpg = 0; @@ -197,6 +243,8 @@ i386_vm_init(uint64_t maxmem, vm_offset_t base_address; vm_offset_t static_base_address; + PE_parse_boot_argn("memmap", &memmap, sizeof(memmap)); + /* * Establish the KASLR parameters. */ @@ -284,21 +332,21 @@ i386_vm_init(uint64_t maxmem, assert(((sconst|econst) & PAGE_MASK) == 0); - DBG("segTEXTB = %p\n", (void *) segTEXTB); - DBG("segDATAB = %p\n", (void *) segDATAB); - DBG("segLINKB = %p\n", (void *) segLINKB); - DBG("segHIBB = %p\n", (void *) segHIBB); - DBG("segPRELINKTEXTB = %p\n", (void *) segPRELINKTEXTB); - DBG("segPRELINKINFOB = %p\n", (void *) segPRELINKINFOB); - DBG("sHIB = %p\n", (void *) sHIB); - DBG("eHIB = %p\n", (void *) eHIB); - DBG("stext = %p\n", (void *) stext); - DBG("etext = %p\n", (void *) etext); - DBG("sdata = %p\n", (void *) sdata); - DBG("edata = %p\n", (void *) edata); - DBG("sconst = %p\n", (void *) sconst); - DBG("econst = %p\n", (void *) econst); - DBG("kernel_top = %p\n", (void *) &last_kernel_symbol); + DPRINTF("segTEXTB = %p\n", (void *) segTEXTB); + DPRINTF("segDATAB = %p\n", (void *) segDATAB); + DPRINTF("segLINKB = %p\n", (void *) segLINKB); + DPRINTF("segHIBB = %p\n", (void *) segHIBB); + DPRINTF("segPRELINKTEXTB = %p\n", (void *) segPRELINKTEXTB); + DPRINTF("segPRELINKINFOB = %p\n", (void *) segPRELINKINFOB); + DPRINTF("sHIB = %p\n", (void *) sHIB); + DPRINTF("eHIB = %p\n", (void *) eHIB); + DPRINTF("stext = %p\n", (void *) stext); + DPRINTF("etext = %p\n", (void *) etext); + DPRINTF("sdata = %p\n", (void *) sdata); + DPRINTF("edata = %p\n", (void *) edata); + DPRINTF("sconst = %p\n", (void *) sconst); + DPRINTF("econst = %p\n", (void *) econst); + DPRINTF("kernel_top = %p\n", (void *) &last_kernel_symbol); vm_kernel_base = sHIB; vm_kernel_top = (vm_offset_t) &last_kernel_symbol; @@ -309,9 +357,9 @@ i386_vm_init(uint64_t maxmem, vm_prelink_sinfo = segPRELINKINFOB; vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO; vm_slinkedit = segLINKB; - vm_elinkedit = segLINKB + segSizePRELINKTEXT; - vm_kernel_slid_base = vm_kext_base; - vm_kernel_slid_top = vm_elinkedit; + vm_elinkedit = segLINKB + segSizeLINK; + vm_kernel_slid_base = vm_kext_base + vm_kernel_slide; + vm_kernel_slid_top = vm_prelink_einfo; vm_set_page_size(); @@ -326,7 +374,8 @@ i386_vm_init(uint64_t maxmem, pmap_memory_region_count = pmap_memory_region_current = 0; fap = (ppnum_t) i386_btop(first_avail); - mptr = (EfiMemoryRange *)ml_static_ptovirt((vm_offset_t)args->MemoryMap); + maddr = ml_static_ptovirt((vm_offset_t)args->MemoryMap); + mptr = (EfiMemoryRange *)maddr; if (args->MemoryMapDescriptorSize == 0) panic("Invalid memory map descriptor size"); msize = args->MemoryMapDescriptorSize; @@ -436,7 +485,7 @@ i386_vm_init(uint64_t maxmem, break; } - DBG("EFI region %d: type %u/%d, base 0x%x, top 0x%x %s\n", + DPRINTF("EFI region %d: type %u/%d, base 0x%x, top 0x%x %s\n", i, mptr->Type, pmap_type, base, top, (mptr->Attribute&EFI_MEMORY_KERN_RESERVED)? "RESERVED" : (mptr->Attribute&EFI_MEMORY_RUNTIME)? "RUNTIME" : ""); @@ -570,39 +619,9 @@ i386_vm_init(uint64_t maxmem, } } -#ifdef PRINT_PMAP_MEMORY_TABLE - { - unsigned int j; - pmap_memory_region_t *p = pmap_memory_regions; - addr64_t region_start, region_end; - addr64_t efi_start, efi_end; - for (j=0;jtype, - (addr64_t) p->base << I386_PGSHIFT, - (addr64_t) p->alloc_up << I386_PGSHIFT, - (addr64_t) p->alloc_down << I386_PGSHIFT, - (addr64_t) p->end << I386_PGSHIFT); - region_start = (addr64_t) p->base << I386_PGSHIFT; - region_end = ((addr64_t) p->end << I386_PGSHIFT) - 1; - mptr = (EfiMemoryRange *) ml_static_ptovirt((vm_offset_t)args->MemoryMap); - for (i=0; iType != kEfiLoaderCode && - mptr->Type != kEfiLoaderData && - mptr->Type != kEfiBootServicesCode && - mptr->Type != kEfiBootServicesData && - mptr->Type != kEfiConventionalMemory) { - efi_start = (addr64_t)mptr->PhysicalStart; - efi_end = efi_start + ((vm_offset_t)mptr->NumberOfPages << I386_PGSHIFT) - 1; - if ((efi_start >= region_start && efi_start <= region_end) || - (efi_end >= region_start && efi_end <= region_end)) { - kprintf(" *** Overlapping region with EFI runtime region %d\n", i); - } - } - } - } + if (memmap) { + kprint_memmap(maddr, msize, mcount); } -#endif avail_start = first_avail; mem_actual = args->PhysicalMemorySize; diff --git a/osfmk/i386/io_map.c b/osfmk/i386/io_map.c index f68472955..012d4117f 100644 --- a/osfmk/i386/io_map.c +++ b/osfmk/i386/io_map.c @@ -61,6 +61,7 @@ #include #include #include +#include extern vm_offset_t virtual_avail; @@ -80,6 +81,9 @@ io_map(vm_map_offset_t phys_addr, vm_size_t size, unsigned int flags) start = virtual_avail; virtual_avail += round_page(size); +#if KASAN + kasan_notify_address(start, size); +#endif (void) pmap_map_bd(start, phys_addr, phys_addr + round_page(size), VM_PROT_READ|VM_PROT_WRITE, flags); diff --git a/osfmk/i386/lapic_native.c b/osfmk/i386/lapic_native.c index a0777959e..73e5e1c13 100644 --- a/osfmk/i386/lapic_native.c +++ b/osfmk/i386/lapic_native.c @@ -108,6 +108,7 @@ static void legacy_init(void) { int result; + kern_return_t kr; vm_map_entry_t entry; vm_map_offset_t lapic_vbase64; /* Establish a map to the local apic */ @@ -117,7 +118,10 @@ legacy_init(void) result = vm_map_find_space(kernel_map, &lapic_vbase64, round_page(LAPIC_SIZE), 0, - VM_MAKE_TAG(VM_KERN_MEMORY_IOKIT), &entry); + 0, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_IOKIT, + &entry); /* Convert 64-bit vm_map_offset_t to "pointer sized" vm_offset_t */ lapic_vbase = (vm_offset_t) lapic_vbase64; @@ -133,13 +137,15 @@ legacy_init(void) * MTRR physical range containing the local APIC's MMIO space as * UC and this will override the default PAT setting. */ - pmap_enter(pmap_kernel(), - lapic_vbase, - (ppnum_t) i386_btop(lapic_pbase), - VM_PROT_READ|VM_PROT_WRITE, - VM_PROT_NONE, - VM_WIMG_IO, - TRUE); + kr = pmap_enter(pmap_kernel(), + lapic_vbase, + (ppnum_t) i386_btop(lapic_pbase), + VM_PROT_READ|VM_PROT_WRITE, + VM_PROT_NONE, + VM_WIMG_IO, + TRUE); + + assert(kr == KERN_SUCCESS); } /* diff --git a/osfmk/i386/locks.h b/osfmk/i386/locks.h index 053de1577..3d337a1c8 100644 --- a/osfmk/i386/locks.h +++ b/osfmk/i386/locks.h @@ -37,6 +37,9 @@ #include extern unsigned int LcksOpts; +#if DEVELOPMENT || DEBUG +extern unsigned int LckDisablePreemptCheck; +#endif #define enaLkDeb 0x00000001 /* Request debug in default attribute */ #define enaLkStat 0x00000002 /* Request statistic in default attribute */ @@ -174,26 +177,52 @@ typedef struct __lck_mtx_ext_t__ lck_mtx_ext_t; #ifdef MACH_KERNEL_PRIVATE #pragma pack(1) /* Make sure the structure stays as we defined it */ -typedef struct _lck_rw_t_internal_ { - volatile uint16_t lck_rw_shared_count; /* No. of accepted readers */ - volatile uint8_t lck_rw_interlock; /* Interlock byte */ - volatile uint8_t - lck_rw_priv_excl:1, /* Writers prioritized if set */ - lck_rw_want_upgrade:1, /* Read-to-write upgrade waiting */ - lck_rw_want_write:1, /* Writer waiting or locked for write */ - lck_r_waiting:1, /* Reader is sleeping on lock */ - lck_w_waiting:1, /* Writer is sleeping on lock */ - lck_rw_can_sleep:1, /* Can attempts to lock go to sleep? */ - lck_rw_padb6:2; /* padding */ - - uint32_t lck_rw_tag; /* This can be obsoleted when stats - * are in - */ - uint32_t lck_rw_pad8; - uint32_t lck_rw_pad12; +typedef union _lck_rw_t_internal_ { + struct { + volatile uint16_t lck_rw_shared_count; /* No. of accepted readers */ + volatile uint8_t lck_rw_interlock; /* Interlock byte */ + volatile uint8_t + lck_rw_priv_excl:1, /* Writers prioritized if set */ + lck_rw_want_upgrade:1, /* Read-to-write upgrade waiting */ + lck_rw_want_write:1, /* Writer waiting or locked for write */ + lck_r_waiting:1, /* Reader is sleeping on lock */ + lck_w_waiting:1, /* Writer is sleeping on lock */ + lck_rw_can_sleep:1, /* Can attempts to lock go to sleep? */ + lck_rw_padb6:2; /* padding */ + uint32_t lck_rw_tag; /* This can be obsoleted when stats are in */ + thread_t lck_rw_owner; /* Unused */ + }; + struct { + uint32_t data; /* Single word for count, ilk, and bitfields */ + uint32_t lck_rw_pad4; + uint32_t lck_rw_pad8; + uint32_t lck_rw_pad12; + }; } lck_rw_t; #pragma pack() +#define LCK_RW_SHARED_SHIFT 0 +#define LCK_RW_INTERLOCK_BIT 16 +#define LCK_RW_PRIV_EXCL_BIT 24 +#define LCK_RW_WANT_UPGRADE_BIT 25 +#define LCK_RW_WANT_EXCL_BIT 26 +#define LCK_RW_R_WAITING_BIT 27 +#define LCK_RW_W_WAITING_BIT 28 +#define LCK_RW_CAN_SLEEP_BIT 29 + +#define LCK_RW_INTERLOCK (1 << LCK_RW_INTERLOCK_BIT) +#define LCK_RW_WANT_UPGRADE (1 << LCK_RW_WANT_UPGRADE_BIT) +#define LCK_RW_WANT_EXCL (1 << LCK_RW_WANT_EXCL_BIT) +#define LCK_RW_R_WAITING (1 << LCK_RW_R_WAITING_BIT) +#define LCK_RW_W_WAITING (1 << LCK_RW_W_WAITING_BIT) +#define LCK_RW_PRIV_EXCL (1 << LCK_RW_PRIV_EXCL_BIT) +#define LCK_RW_TAG_VALID (1 << LCK_RW_TAG_VALID_BIT) +#define LCK_RW_SHARED_MASK (0xffff << LCK_RW_SHARED_SHIFT) +#define LCK_RW_SHARED_READER (1 << LCK_RW_SHARED_SHIFT) + +#define LCK_RW_WANT_WRITE LCK_RW_WANT_EXCL + + #define LCK_RW_ATTR_DEBUG 0x1 #define LCK_RW_ATTR_DEBUGb 0 #define LCK_RW_ATTR_STAT 0x2 diff --git a/osfmk/i386/locks_i386.c b/osfmk/i386/locks_i386.c index 9b4639c24..2b2ff9695 100644 --- a/osfmk/i386/locks_i386.c +++ b/osfmk/i386/locks_i386.c @@ -76,6 +76,7 @@ #include #include /* machine_timeout_suspended() */ +#include #include #include @@ -90,6 +91,11 @@ #if CONFIG_DTRACE #define NEED_DTRACE_DEFS #include <../bsd/sys/lockstat.h> + +#define DTRACE_RW_SHARED 0x0 //reader +#define DTRACE_RW_EXCL 0x1 //writer +#define DTRACE_NO_FLAG 0x0 //not applicable + #endif #define LCK_RW_LCK_EXCLUSIVE_CODE 0x100 @@ -113,6 +119,10 @@ unsigned int LcksOpts=0; +#if DEVELOPMENT || DEBUG +unsigned int LckDisablePreemptCheck = 0; +#endif + /* Forwards */ #if USLOCK_DEBUG @@ -126,15 +136,6 @@ decl_simple_lock_data(extern , panic_lock) #endif /* USLOCK_DEBUG */ extern unsigned int not_in_kdp; -extern void kdp_lck_mtx_find_owner( - struct waitq * waitq, - event64_t event, - thread_waitinfo_t * waitinfo); - -extern void kdp_rwlck_find_owner( - struct waitq * waitq, - event64_t event, - thread_waitinfo_t * waitinfo); /* * We often want to know the addresses of the callers @@ -159,6 +160,63 @@ typedef void *pc_t; #endif /* lint */ #endif /* USLOCK_DEBUG */ +// Enforce program order of loads and stores. +#define ordered_load(target) _Generic( (target),\ + uint32_t* : __c11_atomic_load((_Atomic uint32_t* )(target), memory_order_relaxed), \ + uintptr_t*: __c11_atomic_load((_Atomic uintptr_t*)(target), memory_order_relaxed) ) +#define ordered_store(target, value) _Generic( (target),\ + uint32_t* : __c11_atomic_store((_Atomic uint32_t* )(target), (value), memory_order_relaxed), \ + uintptr_t*: __c11_atomic_store((_Atomic uintptr_t*)(target), (value), memory_order_relaxed) ) + +/* + * atomic exchange API is a low level abstraction of the operations + * to atomically read, modify, and write a pointer. This abstraction works + * for both Intel and ARMv8.1 compare and exchange atomic instructions as + * well as the ARM exclusive instructions. + * + * atomic_exchange_begin() - begin exchange and retrieve current value + * atomic_exchange_complete() - conclude an exchange + * atomic_exchange_abort() - cancel an exchange started with atomic_exchange_begin() + */ +static uint32_t +atomic_exchange_begin32(uint32_t *target, uint32_t *previous, enum memory_order ord) +{ + uint32_t val; + + (void)ord; // Memory order not used + val = __c11_atomic_load((_Atomic uint32_t *)target, memory_order_relaxed); + *previous = val; + return val; +} + +static boolean_t +atomic_exchange_complete32(uint32_t *target, uint32_t previous, uint32_t newval, enum memory_order ord) +{ + return __c11_atomic_compare_exchange_strong((_Atomic uint32_t *)target, &previous, newval, ord, memory_order_relaxed); +} + +static void +atomic_exchange_abort(void) { } + +static boolean_t +atomic_test_and_set32(uint32_t *target, uint32_t test_mask, uint32_t set_mask, enum memory_order ord, boolean_t wait) +{ + uint32_t value, prev; + + for ( ; ; ) { + value = atomic_exchange_begin32(target, &prev, ord); + if (value & test_mask) { + if (wait) + cpu_pause(); + else + atomic_exchange_abort(); + return FALSE; + } + value |= set_mask; + if (atomic_exchange_complete32(target, prev, value, ord)) + return TRUE; + } +} /* * Portable lock package implementation of usimple_locks. @@ -178,37 +236,20 @@ int usld_lock_common_checks(usimple_lock_t, char *); #endif /* USLOCK_DEBUG */ -extern int lck_rw_grab_want(lck_rw_t *lck); -extern int lck_rw_grab_shared(lck_rw_t *lck); -extern int lck_rw_held_read_or_upgrade(lck_rw_t *lck); - - /* * Forward definitions */ -void lck_rw_lock_shared_gen( - lck_rw_t *lck); - -void lck_rw_lock_exclusive_gen( - lck_rw_t *lck); - -boolean_t lck_rw_lock_shared_to_exclusive_success( - lck_rw_t *lck); - -boolean_t lck_rw_lock_shared_to_exclusive_failure( - lck_rw_t *lck, - int prior_lock_state); - -void lck_rw_lock_exclusive_to_shared_gen( - lck_rw_t *lck, - int prior_lock_state); - -lck_rw_type_t lck_rw_done_gen( - lck_rw_t *lck, - int prior_lock_state); - +static void lck_rw_lock_shared_gen(lck_rw_t *lck); +static void lck_rw_lock_exclusive_gen(lck_rw_t *lck); +static boolean_t lck_rw_lock_shared_to_exclusive_success(lck_rw_t *lck); +static boolean_t lck_rw_lock_shared_to_exclusive_failure(lck_rw_t *lck, uint32_t prior_lock_state); +static void lck_rw_lock_exclusive_to_shared_gen(lck_rw_t *lck, uint32_t prior_lock_state); +static lck_rw_type_t lck_rw_done_gen(lck_rw_t *lck, uint32_t prior_lock_state); void lck_rw_clear_promotions_x86(thread_t thread); +static boolean_t lck_rw_held_read_or_upgrade(lck_rw_t *lock); +static boolean_t lck_rw_grab_want(lck_rw_t *lock); +static boolean_t lck_rw_grab_shared(lck_rw_t *lock); /* * Routine: lck_spin_alloc_init @@ -374,19 +415,15 @@ volatile uint32_t spinlock_owner_cpu = ~0; volatile usimple_lock_t spinlock_timed_out; uint32_t spinlock_timeout_NMI(uintptr_t thread_addr) { - uint64_t deadline; uint32_t i; for (i = 0; i < real_ncpus; i++) { if ((uintptr_t)cpu_data_ptr[i]->cpu_active_thread == thread_addr) { spinlock_owner_cpu = i; - if ((uint32_t) cpu_number() == i) - break; - cpu_datap(i)->cpu_NMI_acknowledged = FALSE; - cpu_NMI_interrupt(i); - deadline = mach_absolute_time() + (LockTimeOut * 2); - while (mach_absolute_time() < deadline && cpu_datap(i)->cpu_NMI_acknowledged == FALSE) - cpu_pause(); + if ((uint32_t) cpu_number() != i) { + /* Cause NMI and panic on the owner's cpu */ + NMIPI_panic(cpu_to_cpumask(i), SPINLOCK_TIMEOUT); + } break; } } @@ -424,7 +461,8 @@ usimple_lock( uintptr_t lowner = (uintptr_t)l->interlock.lock_data; spinlock_timed_out = l; lock_cpu = spinlock_timeout_NMI(lowner); - panic("Spinlock acquisition timed out: lock=%p, lock owner thread=0x%lx, current_thread: %p, lock owner active on CPU 0x%x, current owner: 0x%lx", l, lowner, current_thread(), lock_cpu, (uintptr_t)l->interlock.lock_data); + panic("Spinlock acquisition timed out: lock=%p, lock owner thread=0x%lx, current_thread: %p, lock owner active on CPU 0x%x, current owner: 0x%lx, time: %llu", + l, lowner, current_thread(), lock_cpu, (uintptr_t)l->interlock.lock_data, mach_absolute_time()); } } #if DEVELOPMENT || DEBUG @@ -435,6 +473,9 @@ usimple_lock( #else simple_lock((simple_lock_t)l); #endif +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_ACQUIRE, l, 0); +#endif } @@ -862,18 +903,17 @@ lck_rw_destroy( * interrupt from exacerbating hold time. * Hence, local helper functions lck_interlock_lock()/lck_interlock_unlock(). */ -static boolean_t +static inline boolean_t lck_interlock_lock(lck_rw_t *lck) { boolean_t istate; istate = ml_set_interrupts_enabled(FALSE); hw_lock_byte_lock(&lck->lck_rw_interlock); - return istate; } -static void +static inline void lck_interlock_unlock(lck_rw_t *lck, boolean_t istate) { hw_lock_byte_unlock(&lck->lck_rw_interlock); @@ -894,6 +934,13 @@ lck_rw_lock_pause(boolean_t interrupts_enabled) cpu_pause(); } +static inline boolean_t +lck_rw_held_read_or_upgrade(lck_rw_t *lock) +{ + if (ordered_load(&lock->data) & (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE)) + return TRUE; + return FALSE; +} /* * compute the deadline to spin against when @@ -922,10 +969,64 @@ lck_rw_deadline_for_spin(lck_rw_t *lck) } +/* + * Spin while interlock is held. + */ + +static inline void +lck_rw_interlock_spin(lck_rw_t *lock) +{ + while (ordered_load(&lock->data) & LCK_RW_INTERLOCK) { + cpu_pause(); + } +} + +static boolean_t +lck_rw_grab_want(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->data, &prev, memory_order_relaxed); + if ((data & LCK_RW_INTERLOCK) == 0) + break; + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + } + if (data & LCK_RW_WANT_WRITE) { + atomic_exchange_abort(); + return FALSE; + } + data |= LCK_RW_WANT_WRITE; + return atomic_exchange_complete32(&lock->data, prev, data, memory_order_relaxed); +} + +static boolean_t +lck_rw_grab_shared(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->data, &prev, memory_order_acquire_smp); + if ((data & LCK_RW_INTERLOCK) == 0) + break; + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + } + if (data & (LCK_RW_WANT_WRITE | LCK_RW_WANT_UPGRADE)) { + if (((data & LCK_RW_SHARED_MASK) == 0) || (data & LCK_RW_PRIV_EXCL)) { + atomic_exchange_abort(); + return FALSE; + } + } + data += LCK_RW_SHARED_READER; + return atomic_exchange_complete32(&lock->data, prev, data, memory_order_acquire_smp); +} + /* * Routine: lck_rw_lock_exclusive */ -void +static void lck_rw_lock_exclusive_gen( lck_rw_t *lck) { @@ -1124,11 +1225,53 @@ lck_rw_lock_exclusive_gen( #endif } +/* + * Routine: lck_rw_done + */ + +lck_rw_type_t lck_rw_done(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->data, &prev, memory_order_release_smp); + if (data & LCK_RW_INTERLOCK) { /* wait for interlock to clear */ + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; + } + if (data & LCK_RW_SHARED_MASK) { + data -= LCK_RW_SHARED_READER; + if ((data & LCK_RW_SHARED_MASK) == 0) /* if reader count has now gone to 0, check for waiters */ + goto check_waiters; + } else { /* if reader count == 0, must be exclusive lock */ + if (data & LCK_RW_WANT_UPGRADE) { + data &= ~(LCK_RW_WANT_UPGRADE); + } else { + if (data & LCK_RW_WANT_WRITE) + data &= ~(LCK_RW_WANT_EXCL); + else /* lock is not 'owned', panic */ + panic("Releasing non-exclusive RW lock without a reader refcount!"); + } +check_waiters: + if (prev & LCK_RW_W_WAITING) { + data &= ~(LCK_RW_W_WAITING); + if ((prev & LCK_RW_PRIV_EXCL) == 0) + data &= ~(LCK_RW_R_WAITING); + } else + data &= ~(LCK_RW_R_WAITING); + } + if (atomic_exchange_complete32(&lock->data, prev, data, memory_order_release_smp)) + break; + cpu_pause(); + } + return lck_rw_done_gen(lock, prev); +} /* * Routine: lck_rw_done_gen * - * called from the assembly language wrapper... + * called from lck_rw_done() * prior_lock_state is the value in the 1st * word of the lock at the time of a successful * atomic compare and exchange with the new value... @@ -1142,10 +1285,10 @@ lck_rw_lock_exclusive_gen( * this by examining the state of the lock before * we changed it */ -lck_rw_type_t +static lck_rw_type_t lck_rw_done_gen( lck_rw_t *lck, - int prior_lock_state) + uint32_t prior_lock_state) { lck_rw_t *fake_lck; lck_rw_type_t lock_type; @@ -1258,6 +1401,32 @@ lck_rw_lock( panic("lck_rw_lock(): Invalid RW lock type: %x\n", lck_rw_type); } +/* + * Routine: lck_rw_lock_shared + */ +void +lck_rw_lock_shared(lck_rw_t *lock) +{ + uint32_t data, prev; + + current_thread()->rwlock_count++; + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->data, &prev, memory_order_acquire_smp); + if (data & (LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE | LCK_RW_INTERLOCK)) { + atomic_exchange_abort(); + lck_rw_lock_shared_gen(lock); + break; + } + data += LCK_RW_SHARED_READER; + if (atomic_exchange_complete32(&lock->data, prev, data, memory_order_acquire_smp)) + break; + cpu_pause(); + } +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, lock, DTRACE_RW_SHARED); +#endif /* CONFIG_DTRACE */ + return; +} /* * Routine: lck_rw_lock_shared_gen @@ -1266,7 +1435,7 @@ lck_rw_lock( * is held exclusively... this is where we spin/block * until we can acquire the lock in the shared mode */ -void +static void lck_rw_lock_shared_gen( lck_rw_t *lck) { @@ -1368,6 +1537,65 @@ lck_rw_lock_shared_gen( } +/* + * Routine: lck_rw_lock_exclusive + */ + +void +lck_rw_lock_exclusive(lck_rw_t *lock) +{ + current_thread()->rwlock_count++; + if (atomic_test_and_set32(&lock->data, + (LCK_RW_SHARED_MASK | LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE | LCK_RW_INTERLOCK), + LCK_RW_WANT_EXCL, memory_order_acquire_smp, FALSE)) { +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, lock, DTRACE_RW_EXCL); +#endif /* CONFIG_DTRACE */ + } else + lck_rw_lock_exclusive_gen(lock); +} + + +/* + * Routine: lck_rw_lock_shared_to_exclusive + */ + +boolean_t +lck_rw_lock_shared_to_exclusive(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; + } + if (data & LCK_RW_WANT_UPGRADE) { + data -= LCK_RW_SHARED_READER; + if ((data & LCK_RW_SHARED_MASK) == 0) /* we were the last reader */ + data &= ~(LCK_RW_W_WAITING); /* so clear the wait indicator */ + if (atomic_exchange_complete32(&lock->data, prev, data, memory_order_acquire_smp)) + return lck_rw_lock_shared_to_exclusive_failure(lock, prev); + } else { + data |= LCK_RW_WANT_UPGRADE; /* ask for WANT_UPGRADE */ + data -= LCK_RW_SHARED_READER; /* and shed our read count */ + if (atomic_exchange_complete32(&lock->data, prev, data, memory_order_acquire_smp)) + break; + } + cpu_pause(); + } + /* we now own the WANT_UPGRADE */ + if (data & LCK_RW_SHARED_MASK) /* check to see if all of the readers are drained */ + lck_rw_lock_shared_to_exclusive_success(lock); /* if not, we need to go wait */ +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, lock, 0); +#endif + return TRUE; +} + + /* * Routine: lck_rw_lock_shared_to_exclusive_failure * Function: @@ -1376,10 +1604,10 @@ lck_rw_lock_shared_gen( * if 'lck_rw_shared_count' == 0, its also already dropped 'lck_w_waiting' * all we need to do here is determine if a wakeup is needed */ -boolean_t +static boolean_t lck_rw_lock_shared_to_exclusive_failure( lck_rw_t *lck, - int prior_lock_state) + uint32_t prior_lock_state) { lck_rw_t *fake_lck; thread_t thread = current_thread(); @@ -1392,17 +1620,6 @@ lck_rw_lock_shared_to_exclusive_failure( panic("rw lock count underflow for thread %p", thread); } #endif - if ((rwlock_count == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { - /* sched_flags checked without lock, but will be rechecked while clearing */ - lck_rw_clear_promotion(thread); - } - - /* - * prior_lock state is a snapshot of the 1st word of the - * lock in question... we'll fake up a pointer to it - * and carefully not access anything beyond whats defined - * in the first word of a lck_rw_t - */ fake_lck = (lck_rw_t *)&prior_lock_state; if (fake_lck->lck_w_waiting && fake_lck->lck_rw_shared_count == 1) { @@ -1413,6 +1630,12 @@ lck_rw_lock_shared_to_exclusive_failure( */ thread_wakeup(RW_LOCK_WRITER_EVENT(lck)); } + + if ((rwlock_count == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { + /* sched_flags checked without lock, but will be rechecked while clearing */ + lck_rw_clear_promotion(thread); + } + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | DBG_FUNC_NONE, VM_KERNEL_UNSLIDE_OR_PERM(lck), lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, 0, 0); @@ -1428,7 +1651,7 @@ lck_rw_lock_shared_to_exclusive_failure( * we just need to wait for the rest of the readers to drain * and then we can return as the exclusive holder of this lock */ -boolean_t +static boolean_t lck_rw_lock_shared_to_exclusive_success( lck_rw_t *lck) { @@ -1529,29 +1752,52 @@ lck_rw_lock_shared_to_exclusive_success( return (TRUE); } +/* + * Routine: lck_rw_lock_exclusive_to_shared + */ + +void lck_rw_lock_exclusive_to_shared(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->data, &prev, memory_order_release_smp); + if (data & LCK_RW_INTERLOCK) { + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); /* wait for interlock to clear */ + continue; + } + data += LCK_RW_SHARED_READER; + if (data & LCK_RW_WANT_UPGRADE) + data &= ~(LCK_RW_WANT_UPGRADE); + else + data &= ~(LCK_RW_WANT_EXCL); + if (!((prev & LCK_RW_W_WAITING) && (prev & LCK_RW_PRIV_EXCL))) + data &= ~(LCK_RW_W_WAITING); + if (atomic_exchange_complete32(&lock->data, prev, data, memory_order_release_smp)) + break; + cpu_pause(); + } + return lck_rw_lock_exclusive_to_shared_gen(lock, prev); +} + /* - * Routine: lck_rw_lock_exclusive_to_shared + * Routine: lck_rw_lock_exclusive_to_shared_gen * Function: * assembly fast path has already dropped * our exclusive state and bumped lck_rw_shared_count * all we need to do here is determine if anyone * needs to be awakened. */ -void +static void lck_rw_lock_exclusive_to_shared_gen( lck_rw_t *lck, - int prior_lock_state) + uint32_t prior_lock_state) { __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lck); lck_rw_t *fake_lck; - /* - * prior_lock state is a snapshot of the 1st word of the - * lock in question... we'll fake up a pointer to it - * and carefully not access anything beyond whats defined - * in the first word of a lck_rw_t - */ fake_lck = (lck_rw_t *)&prior_lock_state; KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_TO_SH_CODE) | DBG_FUNC_START, @@ -1594,6 +1840,71 @@ lck_rw_try_lock( return(FALSE); } +/* + * Routine: lck_rw_try_lock_shared + */ + +boolean_t lck_rw_try_lock_shared(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; + } + if (data & (LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE)) { + atomic_exchange_abort(); + return FALSE; /* lock is busy */ + } + data += LCK_RW_SHARED_READER; /* Increment reader refcount */ + if (atomic_exchange_complete32(&lock->data, prev, data, memory_order_acquire_smp)) + break; + cpu_pause(); + } + current_thread()->rwlock_count++; + /* There is a 3 instr window where preemption may not notice rwlock_count after cmpxchg */ +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, lock, DTRACE_RW_SHARED); +#endif /* CONFIG_DTRACE */ + return TRUE; +} + + +/* + * Routine: lck_rw_try_lock_exclusive + */ + +boolean_t lck_rw_try_lock_exclusive(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; + } + if (data & (LCK_RW_SHARED_MASK | LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE)) { + atomic_exchange_abort(); + return FALSE; /* can't get it */ + } + data |= LCK_RW_WANT_EXCL; + if (atomic_exchange_complete32(&lock->data, prev, data, memory_order_acquire_smp)) + break; + cpu_pause(); + } + + current_thread()->rwlock_count++; +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, lock, DTRACE_RW_EXCL); +#endif /* CONFIG_DTRACE */ + return TRUE; +} + void lck_rw_assert( @@ -1648,6 +1959,20 @@ lck_rw_clear_promotions_x86(thread_t thread) #endif } +boolean_t +lck_rw_lock_yield_shared(lck_rw_t *lck, boolean_t force_yield) +{ + lck_rw_assert(lck, LCK_RW_ASSERT_SHARED); + + if (lck->lck_rw_want_write || lck->lck_rw_want_upgrade || force_yield) { + lck_rw_unlock_shared(lck); + mutex_pause(2); + lck_rw_lock_shared(lck); + return TRUE; + } + + return FALSE; +} /* * Routine: kdp_lck_rw_lock_is_acquired_exclusive diff --git a/osfmk/i386/machine_check.c b/osfmk/i386/machine_check.c index 862d280fc..ce8344659 100644 --- a/osfmk/i386/machine_check.c +++ b/osfmk/i386/machine_check.c @@ -242,14 +242,14 @@ mca_report_cpu_info(void) { i386_cpu_info_t *infop = cpuid_info(); - kdb_printf(" family: %d model: %d stepping: %d microcode: %d\n", + paniclog_append_noflush(" family: %d model: %d stepping: %d microcode: %d\n", infop->cpuid_family, infop->cpuid_model, infop->cpuid_stepping, infop->cpuid_microcode_version); - kdb_printf(" signature: 0x%x\n", + paniclog_append_noflush(" signature: 0x%x\n", infop->cpuid_signature); - kdb_printf(" %s\n", + paniclog_append_noflush(" %s\n", infop->cpuid_brand_string); } @@ -265,15 +265,15 @@ mca_dump_bank(mca_state_t *state, int i) if (!status.bits.val) return; - kdb_printf(" IA32_MC%d_STATUS(0x%x): 0x%016qx\n", + paniclog_append_noflush(" IA32_MC%d_STATUS(0x%x): 0x%016qx\n", i, IA32_MCi_STATUS(i), status.u64); if (status.bits.addrv) - kdb_printf(" IA32_MC%d_ADDR(0x%x): 0x%016qx\n", + paniclog_append_noflush(" IA32_MC%d_ADDR(0x%x): 0x%016qx\n", i, IA32_MCi_ADDR(i), bank->mca_mci_addr); if (status.bits.miscv) - kdb_printf(" IA32_MC%d_MISC(0x%x): 0x%016qx\n", + paniclog_append_noflush(" IA32_MC%d_MISC(0x%x): 0x%016qx\n", i, IA32_MCi_MISC(i), bank->mca_mci_misc); } @@ -332,11 +332,11 @@ mca_dump(void) /* * Report machine-check capabilities: */ - kdb_printf("Machine-check capabilities: 0x%016qx\n", ia32_mcg_cap.u64); + paniclog_append_noflush("Machine-check capabilities: 0x%016qx\n", ia32_mcg_cap.u64); mca_report_cpu_info(); - kdb_printf(" %d error-reporting banks\n", mca_error_bank_count); + paniclog_append_noflush(" %d error-reporting banks\n", mca_error_bank_count); /* * Dump all processor state: @@ -352,7 +352,7 @@ mca_dump(void) continue; } status = mcsp->mca_mcg_status; - kdb_printf("Processor %d: IA32_MCG_STATUS: 0x%016qx\n", + paniclog_append_noflush("Processor %d: IA32_MCG_STATUS: 0x%016qx\n", i, status.u64); mca_cpu_dump_error_banks(mcsp); } diff --git a/osfmk/i386/machine_routines.c b/osfmk/i386/machine_routines.c index 76652946c..cfe895601 100644 --- a/osfmk/i386/machine_routines.c +++ b/osfmk/i386/machine_routines.c @@ -65,6 +65,10 @@ #define DBG(x...) #endif +#if MONOTONIC +#include +#endif /* MONOTONIC */ + extern void wakeup(void *); static int max_cpus_initialized = 0; @@ -349,7 +353,7 @@ void ml_install_interrupt_handler( { boolean_t current_state; - current_state = ml_get_interrupts_enabled(); + current_state = ml_set_interrupts_enabled(FALSE); PE_install_interrupt_handler(nub, source, target, (IOInterruptHandler) handler, refCon); @@ -612,6 +616,24 @@ ml_get_max_cpus(void) (void) ml_set_interrupts_enabled(current_state); return(machine_info.max_cpus); } + +boolean_t +ml_wants_panic_trap_to_debugger(void) +{ + return FALSE; +} + +void +ml_panic_trap_to_debugger(__unused const char *panic_format_str, + __unused va_list *panic_args, + __unused unsigned int reason, + __unused void *ctx, + __unused uint64_t panic_options_mask, + __unused unsigned long panic_caller) +{ + return; +} + /* * Routine: ml_init_lock_timeout * Function: @@ -656,7 +678,7 @@ ml_init_lock_timeout(void) } #if DEVELOPMENT || DEBUG - reportphyreaddelayabs = LockTimeOut; + reportphyreaddelayabs = LockTimeOut >> 1; #endif if (PE_parse_boot_argn("phyreadmaxus", &slto, sizeof (slto))) { default_timeout_ns = slto * NSEC_PER_USEC; @@ -823,6 +845,31 @@ vm_offset_t ml_stack_remaining(void) } } +#if KASAN +vm_offset_t ml_stack_base(void); +vm_size_t ml_stack_size(void); + +vm_offset_t +ml_stack_base(void) +{ + if (ml_at_interrupt_context()) { + return current_cpu_datap()->cpu_int_stack_top - INTSTACK_SIZE; + } else { + return current_thread()->kernel_stack; + } +} + +vm_size_t +ml_stack_size(void) +{ + if (ml_at_interrupt_context()) { + return INTSTACK_SIZE; + } else { + return kernel_stack_size; + } +} +#endif + void kernel_preempt_check(void) { @@ -929,3 +976,22 @@ void _enable_preemption(void) { void plctrace_disable(void) { plctrace_enabled = 0; } + +static boolean_t ml_quiescing; + +void ml_set_is_quiescing(boolean_t quiescing) +{ + assert(FALSE == ml_get_interrupts_enabled()); + ml_quiescing = quiescing; +} + +boolean_t ml_is_quiescing(void) +{ + assert(FALSE == ml_get_interrupts_enabled()); + return (ml_quiescing); +} + +uint64_t ml_get_booter_memory_size(void) +{ + return (0); +} diff --git a/osfmk/i386/machine_routines.h b/osfmk/i386/machine_routines.h index 5e1a44656..da6db8347 100644 --- a/osfmk/i386/machine_routines.h +++ b/osfmk/i386/machine_routines.h @@ -40,6 +40,8 @@ #include #include +#include + __BEGIN_DECLS #ifdef XNU_KERNEL_PRIVATE @@ -80,7 +82,7 @@ void ml_install_interrupt_handler( void ml_entropy_collect(void); -uint64_t ml_get_timebase(); +uint64_t ml_get_timebase(void); void ml_init_lock_timeout(void); void ml_init_delay_spin_threshold(int); @@ -151,6 +153,16 @@ void plctrace_disable(void); /* Warm up a CPU to receive an interrupt */ kern_return_t ml_interrupt_prewarm(uint64_t deadline); +/* Check if the machine layer wants to intercept a panic call */ +boolean_t ml_wants_panic_trap_to_debugger(void); + +/* Machine layer routine for intercepting panics */ +void ml_panic_trap_to_debugger(const char *panic_format_str, + va_list *panic_args, + unsigned int reason, + void *ctx, + uint64_t panic_options_mask, + unsigned long panic_caller); #endif /* XNU_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE @@ -311,6 +323,12 @@ boolean_t ml_set_interrupts_enabled(boolean_t enable); /* Check if running at interrupt context */ boolean_t ml_at_interrupt_context(void); +#ifdef XNU_KERNEL_PRIVATE +extern boolean_t ml_is_quiescing(void); +extern void ml_set_is_quiescing(boolean_t); +extern uint64_t ml_get_booter_memory_size(void); +#endif + /* Zero bytes starting at a physical address */ void bzero_phys( addr64_t phys_address, @@ -329,6 +347,9 @@ pmap_verify_noncacheable(uintptr_t vaddr); #ifdef XNU_KERNEL_PRIVATE boolean_t ml_fpu_avx_enabled(void); +#if !defined(RC_HIDE_XNU_J137) +boolean_t ml_fpu_avx512_enabled(void); +#endif void interrupt_latency_tracker_setup(void); void interrupt_reset_latency_stats(void); diff --git a/osfmk/i386/machine_task.c b/osfmk/i386/machine_task.c index 956ad991b..f1cd81ce4 100644 --- a/osfmk/i386/machine_task.c +++ b/osfmk/i386/machine_task.c @@ -57,6 +57,7 @@ #include #include #include +#include #if HYPERVISOR #include @@ -282,3 +283,24 @@ machine_thread_inherit_taskwide( return KERN_SUCCESS; } + +void +machine_task_init(task_t new_task, + task_t parent_task, + boolean_t inherit_memory) +{ + new_task->uexc_range_start = 0; + new_task->uexc_range_size = 0; + new_task->uexc_handler = 0; + + new_task->i386_ldt = 0; + + if (parent_task != TASK_NULL) { + if (inherit_memory && parent_task->i386_ldt) + new_task->i386_ldt = user_ldt_copy(parent_task->i386_ldt); + new_task->xstate = parent_task->xstate; + } else { + assert(fpu_default != UNDEFINED); + new_task->xstate = fpu_default; + } +} diff --git a/osfmk/i386/misc_protos.h b/osfmk/i386/misc_protos.h index 6ad7834f5..3e57debe3 100644 --- a/osfmk/i386/misc_protos.h +++ b/osfmk/i386/misc_protos.h @@ -161,4 +161,6 @@ void cpu_pmc_control(void *); extern void pstate_trace(void); +extern void mp_interrupt_watchdog(void); + #endif /* _I386_MISC_PROTOS_H_ */ diff --git a/osfmk/i386/mp.c b/osfmk/i386/mp.c index 269ca1914..0c971b1e0 100644 --- a/osfmk/i386/mp.c +++ b/osfmk/i386/mp.c @@ -29,7 +29,6 @@ * @OSF_COPYRIGHT@ */ -#include #include #include #include @@ -85,6 +84,10 @@ #include +#if MONOTONIC +#include +#endif /* MONOTONIC */ + #if MP_DEBUG #define PAUSE delay(1000000) #define DBG(x...) kprintf(x) @@ -112,7 +115,6 @@ void i386_cpu_IPI(int cpu); static void mp_kdp_wait(boolean_t flush, boolean_t isNMI); #endif /* MACH_KDP */ -static boolean_t mp_safe_spin_lock(usimple_lock_t lock); #if MACH_KDP static boolean_t cpu_signal_pending(int cpu, mp_event_t event); #endif /* MACH_KDP */ @@ -122,7 +124,10 @@ boolean_t smp_initialized = FALSE; uint32_t TSC_sync_margin = 0xFFF; volatile boolean_t force_immediate_debugger_NMI = FALSE; volatile boolean_t pmap_tlb_flush_timeout = FALSE; -decl_simple_lock_data(,mp_kdp_lock); +#if DEBUG || DEVELOPMENT +boolean_t mp_interrupt_watchdog_enabled = TRUE; +uint32_t mp_interrupt_watchdog_events = 0; +#endif decl_simple_lock_data(,debugger_callback_lock); struct debugger_callback *debugger_callback = NULL; @@ -168,6 +173,8 @@ lck_mtx_ext_t mp_bc_lock_ext; static volatile int debugger_cpu = -1; volatile long NMIPI_acks = 0; volatile long NMI_count = 0; +static NMI_reason_t NMI_panic_reason = NONE; +static int vector_timed_out; extern void NMI_cpus(void); @@ -175,8 +182,6 @@ static void mp_cpus_call_init(void); static void mp_cpus_call_action(void); static void mp_call_PM(void); -static boolean_t mp_cpus_call_wait_timeout = FALSE; - char mp_slave_stack[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); // Temp stack for slave init /* PAL-related routines */ @@ -227,7 +232,6 @@ static void free_warm_timer_call(timer_call_t call); void smp_init(void) { - simple_lock_init(&mp_kdp_lock, 0); simple_lock_init(&mp_rv_lock, 0); simple_lock_init(&debugger_callback_lock, 0); lck_grp_attr_setdefault(&smp_lck_grp_attr); @@ -248,6 +252,15 @@ smp_init(void) mp_cpus_call_init(); mp_cpus_call_cpu_init(master_cpu); +#if DEBUG || DEVELOPMENT + if (PE_parse_boot_argn("interrupt_watchdog", + &mp_interrupt_watchdog_enabled, + sizeof(mp_interrupt_watchdog_enabled))) { + kprintf("Interrupt watchdog %sabled\n", + mp_interrupt_watchdog_enabled ? "en" : "dis"); + } +#endif + if (PE_parse_boot_argn("TSC_sync_margin", &TSC_sync_margin, sizeof(TSC_sync_margin))) { kprintf("TSC sync Margin 0x%x\n", TSC_sync_margin); @@ -444,7 +457,7 @@ intel_startCPU( * Initialize (or re-initialize) the descriptor tables for this cpu. * Propagate processor mode to slave. */ - cpu_desc_init64(cpu_datap(slot_num)); + cpu_desc_init(cpu_datap(slot_num)); /* Serialize use of the slave boot stack, etc. */ lck_mtx_lock(&mp_cpu_boot_lock); @@ -570,6 +583,8 @@ static int NMIInterruptHandler(x86_saved_state_t *regs) { void *stackptr; + char pstr[192]; + uint64_t now = mach_absolute_time(); if (panic_active() && !panicDebugging) { if (pmsafe_debug) @@ -586,29 +601,34 @@ NMIInterruptHandler(x86_saved_state_t *regs) if (cpu_number() == debugger_cpu) goto NMExit; - if (spinlock_timed_out) { - char pstr[192]; - snprintf(&pstr[0], sizeof(pstr), "Panic(CPU %d): NMIPI for spinlock acquisition timeout, spinlock: %p, spinlock owner: %p, current_thread: %p, spinlock_owner_cpu: 0x%x\n", cpu_number(), spinlock_timed_out, (void *) spinlock_timed_out->interlock.lock_data, current_thread(), spinlock_owner_cpu); - panic_i386_backtrace(stackptr, 64, &pstr[0], TRUE, regs); - } else if (mp_cpus_call_wait_timeout) { - char pstr[192]; - snprintf(&pstr[0], sizeof(pstr), "Panic(CPU %d): Unresponsive processor, this CPU timed-out during cross-call\n", cpu_number()); + if (NMI_panic_reason == SPINLOCK_TIMEOUT) { + snprintf(&pstr[0], sizeof(pstr), + "Panic(CPU %d, time %llu): NMIPI for spinlock acquisition timeout, spinlock: %p, spinlock owner: %p, current_thread: %p, spinlock_owner_cpu: 0x%x\n", + cpu_number(), now, spinlock_timed_out, (void *) spinlock_timed_out->interlock.lock_data, current_thread(), spinlock_owner_cpu); panic_i386_backtrace(stackptr, 64, &pstr[0], TRUE, regs); - } else if (pmap_tlb_flush_timeout == TRUE) { - char pstr[128]; - snprintf(&pstr[0], sizeof(pstr), "Panic(CPU %d): Unresponsive processor (this CPU did not acknowledge interrupts) TLB state:0x%x\n", cpu_number(), current_cpu_datap()->cpu_tlb_invalid); + } else if (NMI_panic_reason == TLB_FLUSH_TIMEOUT) { + snprintf(&pstr[0], sizeof(pstr), + "Panic(CPU %d, time %llu): NMIPI for unresponsive processor: TLB flush timeout, TLB state:0x%x\n", + cpu_number(), now, current_cpu_datap()->cpu_tlb_invalid); panic_i386_backtrace(stackptr, 48, &pstr[0], TRUE, regs); - } - + } else if (NMI_panic_reason == CROSSCALL_TIMEOUT) { + snprintf(&pstr[0], sizeof(pstr), + "Panic(CPU %d, time %llu): NMIPI for unresponsive processor: cross-call timeout\n", + cpu_number(), now); + panic_i386_backtrace(stackptr, 64, &pstr[0], TRUE, regs); + } else if (NMI_panic_reason == INTERRUPT_WATCHDOG) { + snprintf(&pstr[0], sizeof(pstr), + "Panic(CPU %d, time %llu): NMIPI for unresponsive processor: interrupt watchdog for vector 0x%x\n", + cpu_number(), now, vector_timed_out); + panic_i386_backtrace(stackptr, 64, &pstr[0], TRUE, regs); + } + #if MACH_KDP if (pmsafe_debug && !kdp_snapshot) pmSafeMode(¤t_cpu_datap()->lcpu, PM_SAFE_FL_SAFE); current_cpu_datap()->cpu_NMI_acknowledged = TRUE; i_bit_clear(MP_KDP, ¤t_cpu_datap()->cpu_signals); - if (pmap_tlb_flush_timeout || - spinlock_timed_out || - mp_cpus_call_wait_timeout || - panic_active()) { + if (panic_active() || NMI_panic_reason != NONE) { mp_kdp_wait(FALSE, TRUE); } else if (!mp_kdp_trap && !mp_kdp_is_NMI && @@ -791,7 +811,7 @@ mp_spin_timeout(uint64_t tsc_start) * are still serviced if interrupts are masked while we spin. * Returns current interrupt state. */ -static boolean_t +boolean_t mp_safe_spin_lock(usimple_lock_t lock) { if (ml_get_interrupts_enabled()) { @@ -807,11 +827,9 @@ mp_safe_spin_lock(usimple_lock_t lock) lock->interlock.lock_data; spinlock_timed_out = lock; lock_cpu = spinlock_timeout_NMI(lowner); - panic("mp_safe_spin_lock() timed out," - " lock: %p, owner thread: 0x%lx," - " current_thread: %p, owner on CPU 0x%x", - lock, lowner, - current_thread(), lock_cpu); + NMIPI_panic(cpu_to_cpumask(lock_cpu), SPINLOCK_TIMEOUT); + panic("mp_safe_spin_lock() timed out, lock: %p, owner thread: 0x%lx, current_thread: %p, owner on CPU 0x%x, time: %llu", + lock, lowner, current_thread(), lock_cpu, mach_absolute_time()); } } return FALSE; @@ -1008,18 +1026,33 @@ mp_call_head_lock(mp_call_queue_t *cqp) return intrs_enabled; } +/* + * Deliver an NMIPI to a set of processors to cause them to panic . + */ void -mp_cpus_NMIPI(cpumask_t cpu_mask) { +NMIPI_panic(cpumask_t cpu_mask, NMI_reason_t why) { unsigned int cpu, cpu_bit; uint64_t deadline; + NMI_panic_reason = why; + for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) { - if (cpu_mask & cpu_bit) - cpu_NMI_interrupt(cpu); + if ((cpu_mask & cpu_bit) == 0) + continue; + cpu_datap(cpu)->cpu_NMI_acknowledged = FALSE; + cpu_NMI_interrupt(cpu); + } + + /* Wait (only so long) for NMi'ed cpus to respond */ + deadline = mach_absolute_time() + LockTimeOut; + for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) { + if ((cpu_mask & cpu_bit) == 0) + continue; + while (!cpu_datap(cpu)->cpu_NMI_acknowledged && + mach_absolute_time() < deadline) { + cpu_pause(); + } } - deadline = mach_absolute_time() + (LockTimeOut); - while (mach_absolute_time() < deadline) - cpu_pause(); } #if MACH_ASSERT @@ -1200,9 +1233,8 @@ mp_cpus_call_wait(boolean_t intrs_enabled, if (mp_spin_timeout(tsc_spin_start)) { cpumask_t cpus_unresponsive; - mp_cpus_call_wait_timeout = TRUE; cpus_unresponsive = cpus_called & ~(*cpus_responded); - mp_cpus_NMIPI(cpus_unresponsive); + NMIPI_panic(cpus_unresponsive, CROSSCALL_TIMEOUT); panic("mp_cpus_call_wait() timeout, cpus: 0x%llx", cpus_unresponsive); } @@ -1452,7 +1484,7 @@ i386_activate_cpu(void) return; } - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); cdp->cpu_running = TRUE; started_cpu(); simple_unlock(&x86_topo_lock); @@ -1470,7 +1502,7 @@ i386_deactivate_cpu(void) TRACE_MP_CPU_DEACTIVATE | DBG_FUNC_START, 0, 0, 0, 0, 0); - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); cdp->cpu_running = FALSE; simple_unlock(&x86_topo_lock); @@ -1481,6 +1513,10 @@ i386_deactivate_cpu(void) timer_queue_shutdown(&cdp->rtclock_timer.queue); mp_cpus_call(cpu_to_cpumask(master_cpu), ASYNC, timer_queue_expire_local, NULL); +#if MONOTONIC + mt_cpu_down(cdp); +#endif /* MONOTONIC */ + /* * Open an interrupt window * and ensure any pending IPI or timer is serviced @@ -1514,7 +1550,7 @@ boolean_t mp_kdp_state; void -mp_kdp_enter(void) +mp_kdp_enter(boolean_t proceed_on_failure) { unsigned int cpu; unsigned int ncpus = 0; @@ -1523,11 +1559,6 @@ mp_kdp_enter(void) DBG("mp_kdp_enter()\n"); -#if DEBUG - if (!smp_initialized) - simple_lock_init(&mp_kdp_lock, 0); -#endif - /* * Here to enter the debugger. * In case of races, only one cpu is allowed to enter kdp after @@ -1542,26 +1573,44 @@ mp_kdp_enter(void) return; } - cpu_datap(my_cpu)->debugger_entry_time = mach_absolute_time(); - simple_lock(&mp_kdp_lock); - - if (pmsafe_debug && !kdp_snapshot) - pmSafeMode(¤t_cpu_datap()->lcpu, PM_SAFE_FL_SAFE); + uint64_t start_time = cpu_datap(my_cpu)->debugger_entry_time = mach_absolute_time(); + int locked = 0; + while (!locked || mp_kdp_trap) { + if (locked) { + simple_unlock(&x86_topo_lock); + } + if (proceed_on_failure) { + if (mach_absolute_time() - start_time > 500000000ll) { + kprintf("mp_kdp_enter() can't get x86_topo_lock! Debugging anyway! #YOLO\n"); + break; + } + locked = simple_lock_try(&x86_topo_lock); + if (!locked) { + cpu_pause(); + } + } else { + mp_safe_spin_lock(&x86_topo_lock); + locked = TRUE; + } - while (mp_kdp_trap) { - simple_unlock(&mp_kdp_lock); - DBG("mp_kdp_enter() race lost\n"); + if (locked && mp_kdp_trap) { + simple_unlock(&x86_topo_lock); + DBG("mp_kdp_enter() race lost\n"); #if MACH_KDP - mp_kdp_wait(TRUE, FALSE); + mp_kdp_wait(TRUE, FALSE); #endif - simple_lock(&mp_kdp_lock); + locked = FALSE; + } } + + if (pmsafe_debug && !kdp_snapshot) + pmSafeMode(¤t_cpu_datap()->lcpu, PM_SAFE_FL_SAFE); + debugger_cpu = my_cpu; ncpus = 1; - mp_kdp_ncpus = 1; /* self */ + atomic_incl((volatile long *)&mp_kdp_ncpus, 1); mp_kdp_trap = TRUE; debugger_entry_time = cpu_datap(my_cpu)->debugger_entry_time; - simple_unlock(&mp_kdp_lock); /* * Deliver a nudge to other cpus, counting how many @@ -1626,6 +1675,10 @@ mp_kdp_enter(void) cpu_NMI_interrupt(cpu); } + if (locked) { + simple_unlock(&x86_topo_lock); + } + DBG("mp_kdp_enter() %d processors done %s\n", (int)mp_kdp_ncpus, (mp_kdp_ncpus == ncpus) ? "OK" : "timed out"); @@ -1737,35 +1790,7 @@ mp_kdp_exit(void) DBG("mp_kdp_exit() done\n"); (void) ml_set_interrupts_enabled(mp_kdp_state); - postcode(0); -} - -#define TRAP_DEBUGGER __asm__ volatile("int3") - -kern_return_t -DebuggerWithCallback(kern_return_t (*callback) (void*), - void *callback_context, - boolean_t proceed_on_sync_failure) -{ - simple_lock(&debugger_callback_lock); - - struct debugger_callback callback_buf = { - .callback = callback, - .callback_context = callback_context, - .proceed_on_sync_failure = proceed_on_sync_failure, - .error = KERN_FAILURE - }; - - assert(debugger_callback == NULL); - debugger_callback = &callback_buf; - - TRAP_DEBUGGER; - - debugger_callback = NULL; - - simple_unlock(&debugger_callback_lock); - - return callback_buf.error; + postcode(MP_KDP_EXIT); } #endif /* MACH_KDP */ @@ -1941,14 +1966,95 @@ kernel_spin(uint64_t spin_ns) boolean_t istate; uint64_t spin_abs; uint64_t deadline; + cpu_data_t *cdp; kprintf("kernel_spin(%llu) spinning uninterruptibly\n", spin_ns); istate = ml_set_interrupts_enabled(FALSE); + cdp = current_cpu_datap(); nanoseconds_to_absolutetime(spin_ns, &spin_abs); + + /* Fake interrupt handler entry for testing mp_interrupt_watchdog() */ + cdp->cpu_int_event_time = mach_absolute_time(); + cdp->cpu_int_state = (void *) USER_STATE(current_thread()); + deadline = mach_absolute_time() + spin_ns; while (mach_absolute_time() < deadline) cpu_pause(); + + cdp->cpu_int_event_time = 0; + cdp->cpu_int_state = NULL; + ml_set_interrupts_enabled(istate); kprintf("kernel_spin() continuing\n"); } + +/* + * Called from the scheduler's maintenance thread, + * scan running processors for long-running ISRs and: + * - panic if longer than LockTimeOut, or + * - log if more than a quantum. + */ +void +mp_interrupt_watchdog(void) +{ + cpu_t cpu; + boolean_t intrs_enabled = FALSE; + uint16_t cpu_int_num; + uint64_t cpu_int_event_time; + uint64_t cpu_rip; + uint64_t cpu_int_duration; + uint64_t now; + x86_saved_state_t *cpu_int_state; + + if (__improbable(!mp_interrupt_watchdog_enabled)) + return; + + intrs_enabled = ml_set_interrupts_enabled(FALSE); + now = mach_absolute_time(); + /* + * While timeouts are not suspended, + * check all other processors for long outstanding interrupt handling. + */ + for (cpu = 0; + cpu < (cpu_t) real_ncpus && !machine_timeout_suspended(); + cpu++) { + if ((cpu == (cpu_t) cpu_number()) || + (!cpu_datap(cpu)->cpu_running)) + continue; + cpu_int_event_time = cpu_datap(cpu)->cpu_int_event_time; + if (cpu_int_event_time == 0) + continue; + if (__improbable(now < cpu_int_event_time)) + continue; /* skip due to inter-processor skew */ + cpu_int_state = cpu_datap(cpu)->cpu_int_state; + if (__improbable(cpu_int_state == NULL)) + /* The interrupt may have been dismissed */ + continue; + + /* Here with a cpu handling an interrupt */ + + cpu_int_duration = now - cpu_int_event_time; + if (__improbable(cpu_int_duration > LockTimeOut)) { + cpu_int_num = saved_state64(cpu_int_state)->isf.trapno; + cpu_rip = saved_state64(cpu_int_state)->isf.rip; + vector_timed_out = cpu_int_num; + NMIPI_panic(cpu_to_cpumask(cpu), INTERRUPT_WATCHDOG); + panic("Interrupt watchdog, " + "cpu: %d interrupt: 0x%x time: %llu..%llu state: %p RIP: 0x%llx", + cpu, cpu_int_num, cpu_int_event_time, now, cpu_int_state, cpu_rip); + /* NOT REACHED */ + } else if (__improbable(cpu_int_duration > (uint64_t) std_quantum)) { + mp_interrupt_watchdog_events++; + cpu_int_num = saved_state64(cpu_int_state)->isf.trapno; + cpu_rip = saved_state64(cpu_int_state)->isf.rip; + ml_set_interrupts_enabled(intrs_enabled); + printf("Interrupt watchdog, " + "cpu: %d interrupt: 0x%x time: %llu..%llu RIP: 0x%llx\n", + cpu, cpu_int_num, cpu_int_event_time, now, cpu_rip); + return; + } + } + + ml_set_interrupts_enabled(intrs_enabled); +} #endif diff --git a/osfmk/i386/mp.h b/osfmk/i386/mp.h index 892ae6337..6f46c5d4a 100644 --- a/osfmk/i386/mp.h +++ b/osfmk/i386/mp.h @@ -112,7 +112,7 @@ extern uint32_t spinlock_timeout_NMI(uintptr_t thread_addr); extern uint64_t LastDebuggerEntryAllowance; -extern void mp_kdp_enter(void); +extern void mp_kdp_enter(boolean_t proceed_on_failure); extern void mp_kdp_exit(void); extern boolean_t mp_recent_debugger_activity(void); @@ -187,7 +187,14 @@ extern cpu_t mp_cpus_call1( void *arg1, cpumask_t *cpus_calledp); -extern void mp_cpus_NMIPI(cpumask_t cpus); +typedef enum { + NONE = 0, + SPINLOCK_TIMEOUT, + TLB_FLUSH_TIMEOUT, + CROSSCALL_TIMEOUT, + INTERRUPT_WATCHDOG +} NMI_reason_t; +extern void NMIPI_panic(cpumask_t cpus, NMI_reason_t reason); /* Interrupt a set of cpus, forcing an exit out of non-root mode */ extern void mp_cpus_kick(cpumask_t cpus); diff --git a/osfmk/i386/mp_desc.c b/osfmk/i386/mp_desc.c index a886ae736..5f95beb4d 100644 --- a/osfmk/i386/mp_desc.c +++ b/osfmk/i386/mp_desc.c @@ -72,15 +72,19 @@ #include #include #include -#if defined(__i386__) || defined(__x86_64__) +#include #include -#endif /* i386 */ #if CONFIG_MCA #include #endif #include +#if MONOTONIC +#include +#endif /* MONOTONIC */ +#include + #define K_INTR_GATE (ACC_P|ACC_PL_K|ACC_INTR_GATE) #define U_INTR_GATE (ACC_P|ACC_PL_U|ACC_INTR_GATE) @@ -368,13 +372,14 @@ cpu_gdt_alias(vm_map_offset_t gdt, vm_map_offset_t alias) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NX); - - /* TLB flush unneccessry because target processor isn't running yet */ +#if KASAN + kasan_notify_address(alias, PAGE_SIZE); +#endif } void -cpu_desc_init64(cpu_data_t *cdp) +cpu_desc_init(cpu_data_t *cdp) { cpu_desc_index_t *cdi = &cdp->cpu_desc_index; @@ -412,7 +417,6 @@ cpu_desc_init64(cpu_data_t *cdp) master_ktss64.ist2 = (uintptr_t) low_eintstack; master_ktss64.ist1 = (uintptr_t) low_eintstack - sizeof(x86_64_intr_stack_frame_t); - } else if (cdi->cdi_ktss == NULL) { /* Skipping re-init on wake */ cpu_desc_table64_t *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep; @@ -467,17 +471,21 @@ cpu_desc_init64(cpu_data_t *cdp) /* Require that the top of the sysenter stack is 16-byte aligned */ if ((cdi->cdi_sstk % 16) != 0) - panic("cpu_desc_init64() sysenter stack not 16-byte aligned"); + panic("cpu_desc_init() sysenter stack not 16-byte aligned"); } void -cpu_desc_load64(cpu_data_t *cdp) +cpu_desc_load(cpu_data_t *cdp) { cpu_desc_index_t *cdi = &cdp->cpu_desc_index; + postcode(CPU_DESC_LOAD_ENTRY); + /* Stuff the kernel per-cpu data area address into the MSRs */ + postcode(CPU_DESC_LOAD_GS_BASE); wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp); + postcode(CPU_DESC_LOAD_KERNEL_GS_BASE); wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp); /* @@ -490,23 +498,34 @@ cpu_desc_load64(cpu_data_t *cdp) /* Load the GDT, LDT, IDT and TSS */ cdi->cdi_gdt.size = sizeof(struct real_descriptor)*GDTSZ - 1; cdi->cdi_idt.size = 0x1000 + cdp->cpu_number; + + postcode(CPU_DESC_LOAD_GDT); lgdt((uintptr_t *) &cdi->cdi_gdt); + postcode(CPU_DESC_LOAD_IDT); lidt((uintptr_t *) &cdi->cdi_idt); + postcode(CPU_DESC_LOAD_LDT); lldt(KERNEL_LDT); + postcode(CPU_DESC_LOAD_TSS); set_tr(KERNEL_TSS); #if GPROF // Hack to enable mcount to work on K64 __asm__ volatile("mov %0, %%gs" : : "rm" ((unsigned short)(KERNEL_DS))); #endif + postcode(CPU_DESC_LOAD_EXIT); } /* * Set MSRs for sysenter/sysexit and syscall/sysret for 64-bit. */ -static void -fast_syscall_init64(__unused cpu_data_t *cdp) +void +cpu_syscall_init(cpu_data_t *cdp) { +#if MONOTONIC + mt_cpu_up(cdp); +#else /* MONOTONIC */ +#pragma unused(cdp) +#endif /* !MONOTONIC */ wrmsr64(MSR_IA32_SYSENTER_CS, SYSENTER_CS); wrmsr64(MSR_IA32_SYSENTER_EIP, (uintptr_t) hi64_sysenter); wrmsr64(MSR_IA32_SYSENTER_ESP, current_sstk()); @@ -812,15 +831,6 @@ cpu_physwindow_init(int cpu) } #endif /* NCOPY_WINDOWS > 0 */ -/* - * Load the segment descriptor tables for the current processor. - */ -void -cpu_mode_init(cpu_data_t *cdp) -{ - fast_syscall_init64(cdp); -} - /* * Allocate a new interrupt stack for the boot processor from the * heap rather than continue to use the statically allocated space. diff --git a/osfmk/i386/mp_desc.h b/osfmk/i386/mp_desc.h index 1fc15f5f8..ebc75cfb0 100644 --- a/osfmk/i386/mp_desc.h +++ b/osfmk/i386/mp_desc.h @@ -101,12 +101,10 @@ typedef struct cpu_desc_table64 { #define ldt_desc_p(sel) \ (&((struct real_descriptor *)current_ldt())[sel_idx(sel)]) -extern void cpu_mode_init(cpu_data_t *cdp); +extern void cpu_syscall_init(cpu_data_t *cdp); extern void cpu_desc_init(cpu_data_t *cdp); -extern void cpu_desc_init64(cpu_data_t *cdp); extern void cpu_desc_load(cpu_data_t *cdp); -extern void cpu_desc_load64(cpu_data_t *cdp); extern boolean_t valid_user_data_selector(uint16_t selector); diff --git a/osfmk/i386/pcb.c b/osfmk/i386/pcb.c index ed6e82e53..5fb9112ea 100644 --- a/osfmk/i386/pcb.c +++ b/osfmk/i386/pcb.c @@ -54,7 +54,6 @@ * the rights to redistribute these changes. */ -#include #include #include @@ -118,6 +117,11 @@ unsigned int _MachineStateCount[] = { [x86_AVX_STATE32] = x86_AVX_STATE32_COUNT, [x86_AVX_STATE64] = x86_AVX_STATE64_COUNT, [x86_AVX_STATE] = x86_AVX_STATE_COUNT, +#if !defined(RC_HIDE_XNU_J137) + [x86_AVX512_STATE32] = x86_AVX512_STATE32_COUNT, + [x86_AVX512_STATE64] = x86_AVX512_STATE64_COUNT, + [x86_AVX512_STATE] = x86_AVX512_STATE_COUNT, +#endif /* not RC_HIDE_XNU_J137 */ }; zone_t iss_zone; /* zone for saved_state area */ @@ -127,7 +131,7 @@ zone_t ids_zone; /* zone for debug_state area */ extern void Thread_continue(void); extern void Load_context( - thread_t thread); + thread_t thread) __attribute__((noreturn)); static void get_exception_state32(thread_t thread, x86_exception_state32_t *es); @@ -410,16 +414,16 @@ machine_switch_context( thread_continue_t continuation, thread_t new) { -#if MACH_RT assert(current_cpu_datap()->cpu_active_stack == old->kernel_stack); -#endif +#if KPC kpc_off_cpu(old); +#endif /* KPC */ /* * Save FP registers if in use. */ - fpu_save_context(old); + fpu_switch_context(old, new); old->machine.specFlags &= ~OnProc; new->machine.specFlags |= OnProc; @@ -463,7 +467,7 @@ machine_processor_shutdown( #if CONFIG_VMX vmx_suspend(); #endif - fpu_save_context(thread); + fpu_switch_context(thread, NULL); pmap_switch_context(thread, processor->idle_thread, cpu_number()); return(Shutdown_context(thread, doshutdown, processor)); } @@ -869,8 +873,12 @@ machine_thread_set_state( } case x86_FLOAT_STATE32: + case x86_AVX_STATE32: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE32: +#endif /* not RC_HIDE_XNU_J137 */ { - if (count != x86_FLOAT_STATE32_COUNT) + if (count != _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); if (thread_is_64bit(thr_act)) @@ -880,11 +888,15 @@ machine_thread_set_state( } case x86_FLOAT_STATE64: + case x86_AVX_STATE64: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE64: +#endif /* not RC_HIDE_XNU_J137 */ { - if (count != x86_FLOAT_STATE64_COUNT) + if (count != _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); - if ( !thread_is_64bit(thr_act)) + if (!thread_is_64bit(thr_act)) return(KERN_INVALID_ARGUMENT); return fpu_set_fxstate(thr_act, tstate, flavor); @@ -909,49 +921,33 @@ machine_thread_set_state( return(KERN_INVALID_ARGUMENT); } - case x86_AVX_STATE32: - { - if (count != x86_AVX_STATE32_COUNT) - return(KERN_INVALID_ARGUMENT); - - if (thread_is_64bit(thr_act)) - return(KERN_INVALID_ARGUMENT); - - return fpu_set_fxstate(thr_act, tstate, flavor); - } - - case x86_AVX_STATE64: - { - if (count != x86_AVX_STATE64_COUNT) - return(KERN_INVALID_ARGUMENT); - - if (!thread_is_64bit(thr_act)) - return(KERN_INVALID_ARGUMENT); - - return fpu_set_fxstate(thr_act, tstate, flavor); - } - case x86_AVX_STATE: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE: +#endif { x86_avx_state_t *state; - if (count != x86_AVX_STATE_COUNT) + if (count != _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); state = (x86_avx_state_t *)tstate; - if (state->ash.flavor == x86_AVX_STATE64 && - state->ash.count == x86_FLOAT_STATE64_COUNT && + /* Flavors are defined to have sequential values: 32-bit, 64-bit, non-specific */ + /* 64-bit flavor? */ + if (state->ash.flavor == (flavor - 1) && + state->ash.count == _MachineStateCount[flavor - 1] && thread_is_64bit(thr_act)) { return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.as64, - x86_FLOAT_STATE64); + flavor - 1); } - if (state->ash.flavor == x86_FLOAT_STATE32 && - state->ash.count == x86_FLOAT_STATE32_COUNT && + /* 32-bit flavor? */ + if (state->ash.flavor == (flavor - 2) && + state->ash.count == _MachineStateCount[flavor - 2] && !thread_is_64bit(thr_act)) { return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.as32, - x86_FLOAT_STATE32); + flavor - 2); } return(KERN_INVALID_ARGUMENT); } @@ -1115,6 +1111,24 @@ machine_thread_get_state( break; } +#if !defined(RC_HIDE_XNU_J137) + case THREAD_STATE_FLAVOR_LIST_10_13: + { + if (*count < 6) + return (KERN_INVALID_ARGUMENT); + + tstate[0] = x86_THREAD_STATE; + tstate[1] = x86_FLOAT_STATE; + tstate[2] = x86_EXCEPTION_STATE; + tstate[3] = x86_DEBUG_STATE; + tstate[4] = x86_AVX_STATE; + tstate[5] = x86_AVX512_STATE; + + *count = 6; + break; + } + +#endif case x86_SAVED_STATE32: { x86_saved_state32_t *state; @@ -1224,58 +1238,64 @@ machine_thread_get_state( } case x86_AVX_STATE32: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE32: +#endif { - if (*count != x86_AVX_STATE32_COUNT) + if (*count != _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); if (thread_is_64bit(thr_act)) return(KERN_INVALID_ARGUMENT); - *count = x86_AVX_STATE32_COUNT; + *count = _MachineStateCount[flavor]; return fpu_get_fxstate(thr_act, tstate, flavor); } case x86_AVX_STATE64: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE64: +#endif { - if (*count != x86_AVX_STATE64_COUNT) + if (*count != _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); if ( !thread_is_64bit(thr_act)) return(KERN_INVALID_ARGUMENT); - *count = x86_AVX_STATE64_COUNT; + *count = _MachineStateCount[flavor]; return fpu_get_fxstate(thr_act, tstate, flavor); } case x86_AVX_STATE: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE: +#endif { x86_avx_state_t *state; - kern_return_t kret; + thread_state_t fstate; - if (*count < x86_AVX_STATE_COUNT) + if (*count < _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); + *count = _MachineStateCount[flavor]; state = (x86_avx_state_t *)tstate; - bzero((char *)state, sizeof(x86_avx_state_t)); + bzero((char *)state, *count * sizeof(int)); + if (thread_is_64bit(thr_act)) { - state->ash.flavor = x86_AVX_STATE64; - state->ash.count = x86_AVX_STATE64_COUNT; - kret = fpu_get_fxstate(thr_act, - (thread_state_t)&state->ufs.as64, - x86_AVX_STATE64); + flavor -= 1; /* 64-bit flavor */ + fstate = (thread_state_t) &state->ufs.as64; } else { - state->ash.flavor = x86_AVX_STATE32; - state->ash.count = x86_AVX_STATE32_COUNT; - kret = fpu_get_fxstate(thr_act, - (thread_state_t)&state->ufs.as32, - x86_AVX_STATE32); + flavor -= 2; /* 32-bit flavor */ + fstate = (thread_state_t) &state->ufs.as32; } - *count = x86_AVX_STATE_COUNT; + state->ash.flavor = flavor; + state->ash.count = _MachineStateCount[flavor]; - return(kret); + return fpu_get_fxstate(thr_act, fstate, flavor); } case x86_THREAD_STATE32: @@ -1642,6 +1662,9 @@ machine_thread_switch_addrmode(thread_t thread) */ machine_thread_create(thread, thread->task); + /* Adjust FPU state */ + fpu_switch_addrmode(thread, task_has_64BitAddr(thread->task)); + /* If we're switching ourselves, reset the pcb addresses etc. */ if (thread == current_thread()) { boolean_t istate = ml_set_interrupts_enabled(FALSE); @@ -1742,16 +1765,17 @@ machine_stack_attach( assert(stack); thread->kernel_stack = stack; + thread_initialize_kernel_state(thread); statep = STACK_IKS(stack); #if defined(__x86_64__) statep->k_rip = (unsigned long) Thread_continue; statep->k_rbx = (unsigned long) thread_continue; - statep->k_rsp = (unsigned long) (STACK_IKS(stack) - 1); + statep->k_rsp = (unsigned long) STACK_IKS(stack); #else statep->k_eip = (unsigned long) Thread_continue; statep->k_ebx = (unsigned long) thread_continue; - statep->k_esp = (unsigned long) (STACK_IKS(stack) - 1); + statep->k_esp = (unsigned long) STACK_IKS(stack); #endif return; @@ -1785,7 +1809,7 @@ machine_stack_handoff(thread_t old, */ new->kernel_stack = stack; - fpu_save_context(old); + fpu_switch_context(old, new); old->machine.specFlags &= ~OnProc; new->machine.specFlags |= OnProc; @@ -1798,6 +1822,7 @@ machine_stack_handoff(thread_t old, #endif machine_set_current_thread(new); + thread_initialize_kernel_state(new); return; } diff --git a/osfmk/i386/pcb_native.c b/osfmk/i386/pcb_native.c index b26756da1..c260ec7cd 100644 --- a/osfmk/i386/pcb_native.c +++ b/osfmk/i386/pcb_native.c @@ -54,7 +54,6 @@ * the rights to redistribute these changes. */ -#include #include #include @@ -429,7 +428,7 @@ machine_thread_destroy( #endif if (pcb->ifps != 0) - fpu_free(pcb->ifps); + fpu_free(thread, pcb->ifps); if (pcb->iss != 0) { zfree(iss_zone, pcb->iss); pcb->iss = 0; diff --git a/osfmk/i386/phys.c b/osfmk/i386/phys.c index 06ad22f10..49147fc2a 100644 --- a/osfmk/i386/phys.c +++ b/osfmk/i386/phys.c @@ -54,7 +54,6 @@ * the rights to redistribute these changes. */ -#include #include #include diff --git a/osfmk/i386/pmCPU.c b/osfmk/i386/pmCPU.c index 5791823d7..c9be4ab8b 100644 --- a/osfmk/i386/pmCPU.c +++ b/osfmk/i386/pmCPU.c @@ -364,7 +364,7 @@ static void pmLockCPUTopology(int lock) { if (lock) { - simple_lock(&x86_topo_lock); + mp_safe_spin_lock(&x86_topo_lock); } else { simple_unlock(&x86_topo_lock); } @@ -747,7 +747,7 @@ thread_tell_urgency(int urgency, uint64_t sched_latency, thread_t nthread) { - uint64_t urgency_notification_time_start, delta; + uint64_t urgency_notification_time_start = 0, delta; boolean_t urgency_assert = (urgency_notification_assert_abstime_threshold != 0); assert(get_preemption_level() > 0 || ml_get_interrupts_enabled() == FALSE); #if DEBUG @@ -788,6 +788,7 @@ void machine_thread_going_on_core(__unused thread_t new_thread, __unused int urgency, __unused uint64_t sched_latency, + __unused uint64_t same_pri_latency, __unused uint64_t dispatch_time) { } @@ -806,13 +807,23 @@ machine_max_runnable_latency(__unused uint64_t bg_max_latency, void machine_work_interval_notify(__unused thread_t thread, - __unused uint64_t work_interval_id, - __unused uint64_t start_abstime, - __unused uint64_t finish_abstime, - __unused uint64_t deadline_abstime, - __unused uint64_t next_start_abstime, - __unused uint16_t urgency, - __unused uint32_t flags) + __unused struct kern_work_interval_args* kwi_args) +{ +} + +void machine_switch_perfcontrol_context(__unused perfcontrol_event event, + __unused uint64_t timestamp, + __unused uint32_t flags, + __unused uint64_t new_thread_same_pri_latency, + __unused thread_t old, + __unused thread_t new) +{ +} + +void machine_switch_perfcontrol_state_update(__unused perfcontrol_event event, + __unused uint64_t timestamp, + __unused uint32_t flags, + __unused thread_t thread) { } diff --git a/osfmk/i386/pmap.h b/osfmk/i386/pmap.h index ccad03ea7..0c26eee60 100644 --- a/osfmk/i386/pmap.h +++ b/osfmk/i386/pmap.h @@ -222,6 +222,8 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) #define KERNEL_PML4_INDEX 511 #define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */ #define KERNEL_PHYSMAP_PML4_INDEX 509 /* virtual to physical map */ +#define KERNEL_KASAN_PML4_INDEX0 508 +#define KERNEL_KASAN_PML4_INDEX1 507 #define KERNEL_BASE (0ULL - NBPML4) #define KERNEL_BASEMENT (KERNEL_BASE - NBPML4) @@ -231,6 +233,7 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) /* ?? intel ?? */ #define VM_WIMG_IO (VM_MEM_COHERENT | \ VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED) +#define VM_WIMG_POSTED VM_WIMG_IO #define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED) /* write combining mode, aka store gather */ #define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT) @@ -595,7 +598,6 @@ extern unsigned pmap_memory_region_current; extern pmap_memory_region_t pmap_memory_regions[]; #include -#include static inline void set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) { @@ -713,15 +715,15 @@ extern void pmap_cpu_free( struct cpu_pmap *cp); #endif -extern void pmap_map_block( - pmap_t pmap, +extern kern_return_t pmap_map_block( + pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags); - + extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys); extern void flush_dcache(vm_offset_t addr, unsigned count, int phys); extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va); @@ -730,7 +732,7 @@ extern void pmap_cpu_init(void); extern void pmap_disable_NX(pmap_t pmap); extern void pt_fake_zone_init(int); -extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, +extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, uint64_t *, int *, int *, int *); extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2)); @@ -741,7 +743,7 @@ extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__pr #include - + #define PMAP_ACTIVATE_MAP(map, thread, my_cpu) { \ pmap_t tpmap; \ \ diff --git a/osfmk/i386/pmap_common.c b/osfmk/i386/pmap_common.c index a5cd8f269..d948f34a3 100644 --- a/osfmk/i386/pmap_common.c +++ b/osfmk/i386/pmap_common.c @@ -145,8 +145,17 @@ unsigned pmap_get_cache_attributes(ppnum_t pn, boolean_t is_ept) { boolean_t pmap_has_managed_page(ppnum_t first, ppnum_t last) { - ppnum_t pn; - boolean_t result; + ppnum_t pn, kdata_start, kdata_end; + boolean_t result; + boot_args * args; + + args = (boot_args *) PE_state.bootArgs; + + // Allow pages that the booter added to the end of the kernel. + // We may miss reporting some pages in this range that were freed + // with ml_static_free() + kdata_start = atop_32(args->kaddr); + kdata_end = atop_32(args->kaddr + args->ksize); assert(last_managed_page); assert(first <= last); @@ -157,6 +166,7 @@ pmap_has_managed_page(ppnum_t first, ppnum_t last) && (pn <= last_managed_page); pn++) { + if ((pn >= kdata_start) && (pn < kdata_end)) continue; result = (0 != (pmap_phys_attributes[pn] & PHYS_MANAGED)); } diff --git a/osfmk/i386/pmap_internal.h b/osfmk/i386/pmap_internal.h index b2ea44959..3523d8a14 100644 --- a/osfmk/i386/pmap_internal.h +++ b/osfmk/i386/pmap_internal.h @@ -63,16 +63,15 @@ #ifdef PMAP_TRACES extern boolean_t pmap_trace; -#define PMAP_TRACE(x,a,b,c,d,e) \ - if (pmap_trace) { \ - KERNEL_DEBUG_CONSTANT(x,a,b,c,d,e); \ +#define PMAP_TRACE(...) \ + if (pmap_trace) { \ + KDBG_RELEASE(__VA_ARGS__); \ } #else -#define PMAP_TRACE(x,a,b,c,d,e) KERNEL_DEBUG(x,a,b,c,d,e) +#define PMAP_TRACE(...) KDBG_DEBUG(__VA_ARGS__) #endif /* PMAP_TRACES */ -#define PMAP_TRACE_CONSTANT(x,a,b,c,d,e) \ - KERNEL_DEBUG_CONSTANT(x,a,b,c,d,e); \ +#define PMAP_TRACE_CONSTANT(...) KDBG_RELEASE(__VA_ARGS__) kern_return_t pmap_expand_pml4( pmap_t map, @@ -782,7 +781,7 @@ pmap_pv_remove_retry: if (pac == PMAP_ACTION_IGNORE) goto pmap_pv_remove_exit; else if (pac == PMAP_ACTION_ASSERT) - panic("Possible memory corruption: pmap_pv_remove(%p,0x%llx,0x%x, 0x%llx, %p, %p): null pv_list!", pmap, vaddr, ppn, *pte, ppnp, pte); + panic("Possible memory corruption: pmap_pv_remove(%p,0x%llx,0x%x, 0x%llx, %p, %p): null pv_list, priors: %d", pmap, vaddr, ppn, *pte, ppnp, pte, pmap_pagetable_corruption_incidents); else if (pac == PMAP_ACTION_RETRY_RELOCK) { LOCK_PVH(ppn_to_pai(*ppnp)); pmap_phys_attributes[ppn_to_pai(*ppnp)] |= (PHYS_MODIFIED | PHYS_REFERENCED); @@ -814,8 +813,8 @@ pmap_pv_remove_retry: pprevh = pvhash(pvhash_idx); if (PV_HASHED_ENTRY_NULL == *pprevh) { panic("Possible memory corruption: pmap_pv_remove(%p,0x%llx,0x%x): " - "empty hash, removing rooted", - pmap, vaddr, ppn); + "empty hash, removing rooted, priors: %d", + pmap, vaddr, ppn, pmap_pagetable_corruption_incidents); } pmap_pvh_unlink(pvh_e); UNLOCK_PV_HASH(pvhash_idx); @@ -837,8 +836,8 @@ pmap_pv_remove_retry: LOCK_PV_HASH(pvhash_idx); pprevh = pvhash(pvhash_idx); if (PV_HASHED_ENTRY_NULL == *pprevh) { - panic("Possible memory corruption: pmap_pv_remove(%p,0x%llx,0x%x, 0x%llx, %p): empty hash", - pmap, vaddr, ppn, *pte, pte); + panic("Possible memory corruption: pmap_pv_remove(%p,0x%llx,0x%x, 0x%llx, %p): empty hash, priors: %d", + pmap, vaddr, ppn, *pte, pte, pmap_pagetable_corruption_incidents); } pvh_e = *pprevh; pmap_pv_hashlist_walks++; @@ -857,7 +856,7 @@ pmap_pv_remove_retry: pmap_pagetable_corruption_action_t pac = pmap_classify_pagetable_corruption(pmap, vaddr, ppnp, pte, ROOT_PRESENT); if (pac == PMAP_ACTION_ASSERT) - panic("Possible memory corruption: pmap_pv_remove(%p, 0x%llx, 0x%x, 0x%llx, %p, %p): pv not on hash, head: %p, 0x%llx", pmap, vaddr, ppn, *pte, ppnp, pte, pv_h->pmap, PVE_VA(pv_h)); + panic("Possible memory corruption: pmap_pv_remove(%p, 0x%llx, 0x%x, 0x%llx, %p, %p): pv not on hash, head: %p, 0x%llx, priors: %d", pmap, vaddr, ppn, *pte, ppnp, pte, pv_h->pmap, PVE_VA(pv_h), pmap_pagetable_corruption_incidents); else { UNLOCK_PV_HASH(pvhash_idx); if (pac == PMAP_ACTION_RETRY_RELOCK) { diff --git a/osfmk/i386/pmap_x86_common.c b/osfmk/i386/pmap_x86_common.c index 1bfecd7cb..72fd27e76 100644 --- a/osfmk/i386/pmap_x86_common.c +++ b/osfmk/i386/pmap_x86_common.c @@ -112,8 +112,8 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t panic("pmap_nest: va_start(0x%llx) != nstart(0x%llx)\n", va_start, nstart); PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START, - (uintptr_t) grand, (uintptr_t) subord, - (uintptr_t) (va_start>>32), (uintptr_t) va_start, 0); + VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(subord), + VM_KERNEL_ADDRHIDE(va_start)); nvaddr = (vm_map_offset_t)nstart; num_pde = size >> PDESHIFT; @@ -201,7 +201,7 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t PMAP_UNLOCK(grand); - PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, 0, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, KERN_SUCCESS); return KERN_SUCCESS; } @@ -216,7 +216,6 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t */ kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) { - pd_entry_t *pde; unsigned int i; uint64_t num_pde; @@ -224,8 +223,7 @@ kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) { uint64_t npdpt = PMAP_INVALID_PDPTNUM; PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START, - (uintptr_t) grand, - (uintptr_t) (vaddr>>32), (uintptr_t) vaddr, 0, 0); + VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(vaddr)); if ((size & (pmap_nesting_size_min-1)) || (vaddr & (pmap_nesting_size_min-1))) { @@ -267,8 +265,8 @@ kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) { PMAP_UPDATE_TLBS(grand, va_start, va_end); PMAP_UNLOCK(grand); - - PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, 0, 0, 0, 0, 0); + + PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, KERN_SUCCESS); return KERN_SUCCESS; } @@ -452,7 +450,7 @@ void x86_filter_TLB_coherency_interrupts(boolean_t dofilter) { * insert this page into the given map NOW. */ -void +kern_return_t pmap_enter( pmap_t pmap, vm_map_offset_t vaddr, @@ -462,7 +460,7 @@ pmap_enter( unsigned int flags, boolean_t wired) { - (void) pmap_enter_options(pmap, vaddr, pn, prot, fault_type, flags, wired, PMAP_EXPAND_OPTIONS_NONE, NULL); + return pmap_enter_options(pmap, vaddr, pn, prot, fault_type, flags, wired, PMAP_EXPAND_OPTIONS_NONE, NULL); } @@ -495,10 +493,12 @@ pmap_enter_options( vm_object_t delpage_pm_obj = NULL; uint64_t delpage_pde_index = 0; pt_entry_t old_pte; - kern_return_t kr_expand; + kern_return_t kr; boolean_t is_ept; boolean_t is_altacct; + kr = KERN_FAILURE; + pmap_intr_assert(); if (pmap == PMAP_NULL) @@ -515,9 +515,8 @@ pmap_enter_options( return KERN_INVALID_ARGUMENT; PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START, - pmap, - (uint32_t) (vaddr >> 32), (uint32_t) vaddr, - pn, prot); + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(vaddr), pn, + prot); if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled) set_NX = FALSE; @@ -551,9 +550,9 @@ Retry: while ((pte = pmap64_pde(pmap, vaddr)) == PD_ENTRY_NULL) { /* need room for another pde entry */ PMAP_UNLOCK(pmap); - kr_expand = pmap_expand_pdpt(pmap, vaddr, options); - if (kr_expand != KERN_SUCCESS) - return kr_expand; + kr = pmap_expand_pdpt(pmap, vaddr, options); + if (kr != KERN_SUCCESS) + goto done; PMAP_LOCK(pmap); } } else { @@ -563,15 +562,16 @@ Retry: * going to grow pde level page(s) */ PMAP_UNLOCK(pmap); - kr_expand = pmap_expand(pmap, vaddr, options); - if (kr_expand != KERN_SUCCESS) - return kr_expand; + kr = pmap_expand(pmap, vaddr, options); + if (kr != KERN_SUCCESS) + goto done; PMAP_LOCK(pmap); } } if (options & PMAP_EXPAND_OPTIONS_NOENTER) { PMAP_UNLOCK(pmap); - return KERN_SUCCESS; + kr = KERN_SUCCESS; + goto done; } if (superpage && *pte && !(*pte & PTE_PS)) { @@ -709,7 +709,7 @@ Retry: /* Determine delta, PV locked */ need_tlbflush = ((old_attributes ^ template) != PTE_WIRED); - + if (need_tlbflush == TRUE && !(old_attributes & PTE_WRITE(is_ept))) { if ((old_attributes ^ template) == PTE_WRITE(is_ept)) need_tlbflush = FALSE; @@ -752,7 +752,7 @@ dont_update_pte: */ if (old_pa != (pmap_paddr_t) 0) { - boolean_t was_altacct; + boolean_t was_altacct = FALSE; /* * Don't do anything to pages outside valid memory here. @@ -934,7 +934,7 @@ dont_update_pte: } } } - + if (PV_HASHED_ENTRY_NULL == pvh_e) panic("Mapping alias chain exhaustion, possibly induced by numerous kernel virtual double mappings"); @@ -1115,8 +1115,10 @@ Done: PMAP_ZINFO_PFREE(pmap, PAGE_SIZE); } - PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, 0, 0, 0, 0, 0); - return KERN_SUCCESS; + kr = KERN_SUCCESS; +done: + PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, kr); + return kr; } /* @@ -1455,10 +1457,8 @@ pmap_remove_options( is_ept = is_ept_pmap(map); PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START, - map, - (uint32_t) (s64 >> 32), s64, - (uint32_t) (e64 >> 32), e64); - + VM_KERNEL_ADDRHIDE(map), VM_KERNEL_ADDRHIDE(s64), + VM_KERNEL_ADDRHIDE(e64)); PMAP_LOCK(map); @@ -1539,8 +1539,7 @@ pmap_remove_options( PMAP_UNLOCK(map); - PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END, - map, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END); } @@ -1593,8 +1592,8 @@ pmap_page_protect_options( */ return; } - PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, - pn, prot, 0, 0, 0); + + PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, pn, prot); /* * Determine the new protection. @@ -1867,8 +1866,7 @@ pmap_page_protect_options( done: UNLOCK_PVH(pai); - PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END, - 0, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END); } @@ -1884,7 +1882,7 @@ phys_attribute_clear( { pv_rooted_entry_t pv_h; pv_hashed_entry_t pv_e; - pt_entry_t *pte; + pt_entry_t *pte = NULL; int pai; pmap_t pmap; char attributes = 0; @@ -1919,8 +1917,7 @@ phys_attribute_clear( return; } - PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, - pn, bits, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, pn, bits); pv_h = pai_to_pvh(pai); @@ -2090,8 +2087,7 @@ phys_attribute_clear( UNLOCK_PVH(pai); - PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END, - 0, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END); } /* @@ -2305,9 +2301,8 @@ pmap_query_resident( is_ept = is_ept_pmap(pmap); PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START, - pmap, - (uint32_t) (s64 >> 32), s64, - (uint32_t) (e64 >> 32), e64); + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(s64), + VM_KERNEL_ADDRHIDE(e64)); resident_bytes = 0; compressed_bytes = 0; @@ -2353,7 +2348,7 @@ pmap_query_resident( PMAP_UNLOCK(pmap); PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END, - pmap, 0, 0, 0, 0); + resident_bytes); if (compressed_bytes_p) { *compressed_bytes_p = compressed_bytes; @@ -2426,16 +2421,23 @@ done: return KERN_SUCCESS; } -#if DEBUG || DEVELOPMENT -void -kernel_pmap_lock(void) +void pmap_set_jit_entitled(__unused pmap_t pmap) { - PMAP_LOCK(kernel_pmap); + /* The x86 pmap layer does not care if a map has a JIT entry. */ + return; } -void -kernel_pmap_unlock(void) +bool pmap_has_prot_policy(__unused vm_prot_t prot) { - PMAP_UNLOCK(kernel_pmap); + /* + * The x86 pmap layer does not apply any policy to any protection + * types. + */ + return FALSE; +} + +void pmap_release_pages_fast(void) +{ + return; } -#endif /* DEBUG || DEVELOPMENT */ + diff --git a/osfmk/i386/postcode.h b/osfmk/i386/postcode.h index d2a859354..82a4c0849 100644 --- a/osfmk/i386/postcode.h +++ b/osfmk/i386/postcode.h @@ -126,40 +126,41 @@ #define PSTART_BEFORE_PAGING 0xFE #define PSTART_VSTART 0xFD #define VSTART_ENTRY 0xFC -#define VSTART_IDLE_PTS_INIT 0xFB -#define VSTART_PHYSMAP_INIT 0xFA -#define VSTART_DESC_ALIAS_INIT 0xF9 -#define VSTART_SET_CR3 0xF8 -#define VSTART_CPU_DESC_INIT 0xF7 -#define VSTART_CPU_MODE_INIT 0xF6 -#define VSTART_EXIT 0xF5 -#define I386_INIT_ENTRY 0xF4 -#define CPU_INIT_D 0xF3 -#define PE_INIT_PLATFORM_D 0xF2 +#define VSTART_IDT_INIT 0xFB +#define VSTART_IDLE_PTS_INIT 0xFA +#define VSTART_PHYSMAP_INIT 0xF9 +#define VSTART_DESC_ALIAS_INIT 0xF8 +#define VSTART_SET_CR3 0xF7 +#define VSTART_CPU_DESC_INIT 0xF6 +#define VSTART_CPU_MODE_INIT 0xF5 +#define VSTART_EXIT 0xF4 +#define I386_INIT_ENTRY 0xF3 +#define CPU_INIT_D 0xF2 +#define PE_INIT_PLATFORM_D 0xF1 #define SLAVE_STARTPROG_ENTRY 0xEF #define SLAVE_PSTART 0xEE #define I386_INIT_SLAVE 0xED #define PANIC_DOUBLE_FAULT 0xDF /* Double Fault exception */ -#define PANIC_MACHINE_CHECK 0xDE /* Machine-Check */ -#define MP_KDP_ENTER 0xDB /* Machine in kdp DeBugger */ +#define PANIC_MACHINE_CHECK 0xDC /* Machine-Check */ +#define MP_KDP_ENTER 0xDB /* Debugger Begin */ +#define MP_KDP_EXIT 0xDE /* Debugger End */ #define PANIC_HLT 0xD1 /* Die an early death */ -#define NO_64BIT 0x64 /* No 64-bit support yet */ +#define BOOT_TRAP_HLT 0xD0 /* D'oh! even earlier */ #define ACPI_WAKE_START_ENTRY 0xCF #define ACPI_WAKE_PROT_ENTRY 0xCE #define ACPI_WAKE_PAGED_ENTRY 0xCD -#define CPU_IA32_ENABLE_ENTRY 0xBF -#define CPU_IA32_ENABLE_EXIT 0xBE -#define ML_LOAD_DESC64_ENTRY 0xBD -#define ML_LOAD_DESC64_GDT 0xBC -#define ML_LOAD_DESC64_IDT 0xBB -#define ML_LOAD_DESC64_LDT 0xBA -#define ML_LOAD_DESC64_EXIT 0xB9 -#define CPU_IA32_DISABLE_ENTRY 0xB8 -#define CPU_IA32_DISABLE_EXIT 0xB7 +#define CPU_DESC_LOAD_ENTRY 0xBF +#define CPU_DESC_LOAD_GS_BASE 0xBE +#define CPU_DESC_LOAD_KERNEL_GS_BASE 0xBD +#define CPU_DESC_LOAD_GDT 0xBC +#define CPU_DESC_LOAD_IDT 0xBB +#define CPU_DESC_LOAD_LDT 0xBA +#define CPU_DESC_LOAD_TSS 0xB9 +#define CPU_DESC_LOAD_EXIT 0xB7 #ifndef ASSEMBLER inline static void diff --git a/osfmk/i386/proc_reg.h b/osfmk/i386/proc_reg.h index e9455ab8c..9e1e744d1 100644 --- a/osfmk/i386/proc_reg.h +++ b/osfmk/i386/proc_reg.h @@ -173,16 +173,32 @@ #define XCR0_YMM (1ULL << 2) /* YMM state available */ #define XCR0_BNDREGS (1ULL << 3) /* MPX Bounds register state */ #define XCR0_BNDCSR (1ULL << 4) /* MPX Bounds configuration/state */ +#if !defined(RC_HIDE_XNU_J137) +#define XCR0_OPMASK (1ULL << 5) /* Opmask register state */ +#define XCR0_ZMM_HI256 (1ULL << 6) /* ZMM upper 256-bit state */ +#define XCR0_HI16_ZMM (1ULL << 7) /* ZMM16..ZMM31 512-bit state */ +#endif /* not RC_HIDE_XNU_J137 */ #define XFEM_X87 XCR0_X87 #define XFEM_SSE XCR0_SSE #define XFEM_YMM XCR0_YMM #define XFEM_BNDREGS XCR0_BNDREGS #define XFEM_BNDCSR XCR0_BNDCSR +#if !defined(XNU_HODE_J137) +#define XFEM_OPMASK XCR0_OPMASK +#define XFEM_ZMM_HI256 XCR0_ZMM_HI256 +#define XFEM_HI16_ZMM XCR0_HI16_ZMM +#define XFEM_ZMM (XFEM_ZMM_HI256 | XFEM_HI16_ZMM | XFEM_OPMASK) +#endif /* not XNU_HODE_J137 */ #define XCR0 (0) #define PMAP_PCID_PRESERVE (1ULL << 63) #define PMAP_PCID_MASK (0xFFF) +/* + * If thread groups are needed for x86, set this to 1 + */ +#define CONFIG_THREAD_GROUPS 0 + #ifndef ASSEMBLER #include @@ -358,6 +374,11 @@ static inline void swapgs(void) __asm__ volatile("swapgs"); } +static inline void hlt(void) +{ + __asm__ volatile("hlt"); +} + #ifdef MACH_KERNEL_PRIVATE static inline void flush_tlb_raw(void) @@ -416,6 +437,7 @@ extern void do_mfence(void); #define mfence() do_mfence() #endif +#ifdef __LP64__ static inline uint64_t rdpmc64(uint32_t pmc) { uint32_t lo=0, hi=0; @@ -451,7 +473,7 @@ static inline uint64_t rdtscp64(uint32_t *aux) : "ecx"); return ((hi) << 32) | (lo); } - +#endif /* __LP64__ */ /* * rdmsr_carefully() returns 0 when the MSR has been read successfully, diff --git a/osfmk/i386/rtclock.c b/osfmk/i386/rtclock.c index 6ed44cc73..c8abc4b1e 100644 --- a/osfmk/i386/rtclock.c +++ b/osfmk/i386/rtclock.c @@ -402,21 +402,9 @@ clock_get_system_nanotime( } void -clock_gettimeofday_set_commpage( - uint64_t abstime, - uint64_t epoch, - uint64_t offset, - clock_sec_t *secs, - clock_usec_t *microsecs) +clock_gettimeofday_set_commpage(uint64_t abstime, uint64_t sec, uint64_t frac, uint64_t scale, uint64_t tick_per_sec) { - uint64_t now = abstime + offset; - uint32_t remain; - - remain = _absolutetime_to_microtime(now, secs, microsecs); - - *secs += (clock_sec_t)epoch; - - commpage_set_timestamp(abstime - remain, *secs); + commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec); } void diff --git a/osfmk/i386/rtclock_asm.h b/osfmk/i386/rtclock_asm.h index 5ad7dde8a..dba69a6e7 100644 --- a/osfmk/i386/rtclock_asm.h +++ b/osfmk/i386/rtclock_asm.h @@ -128,7 +128,6 @@ */ #define TIME_INT_EXIT \ NANOTIME /* %rax := nanosecs */ ; \ - movq %rax,%gs:CPU_INT_EVENT_TIME /* save in cpu data */ ; \ movq %rax,%rsi /* save timestamp */ ; \ movq %gs:CPU_PROCESSOR,%rdx /* get processor */ ; \ movq KERNEL_TIMER(%rdx),%rcx /* get kernel timer */ ; \ @@ -142,7 +141,8 @@ TIMER_UPDATE(%rcx,%rax,0) /* update timer */ ; \ popq %rcx /* restore state */ ; \ movq %rcx,CURRENT_STATE(%rdx) /* set current state */ ; \ - movq %rsi,TIMER_TSTAMP(%rcx) /* set timestamp */ + movq %rsi,TIMER_TSTAMP(%rcx) /* set timestamp */ ; \ + movq $0,%gs:CPU_INT_EVENT_TIME /* clear interrupt entry time */ /* diff --git a/osfmk/i386/seg.h b/osfmk/i386/seg.h index 5d555bef9..75f5182e0 100644 --- a/osfmk/i386/seg.h +++ b/osfmk/i386/seg.h @@ -177,10 +177,14 @@ struct fake_descriptor64 { uint32_t reserved:32; /* reserved/zero */ }; +typedef struct __attribute__((packed)) { + uint16_t size; + void *ptr; +} x86_64_desc_register_t; + /* * Boot-time data for master (or only) CPU */ -extern struct fake_descriptor master_idt[IDTSZ]; extern struct real_descriptor master_gdt[GDTSZ]; extern struct real_descriptor master_ldt[LDTSZ]; extern struct i386_tss master_ktss; diff --git a/osfmk/i386/task.h b/osfmk/i386/task.h index b4035d57f..0ca7d549e 100644 --- a/osfmk/i386/task.h +++ b/osfmk/i386/task.h @@ -59,12 +59,14 @@ */ #include +#include -#define MACHINE_TASK \ - struct user_ldt * i386_ldt; \ - void* task_debug; \ - uint64_t uexc_range_start; \ - uint64_t uexc_range_size; \ - uint64_t uexc_handler; +#define MACHINE_TASK \ + struct user_ldt * i386_ldt; \ + void* task_debug; \ + uint64_t uexc_range_start; \ + uint64_t uexc_range_size; \ + uint64_t uexc_handler; \ + xstate_t xstate; diff --git a/osfmk/i386/thread.h b/osfmk/i386/thread.h index 1fea8c2d8..d9ec2568e 100644 --- a/osfmk/i386/thread.h +++ b/osfmk/i386/thread.h @@ -73,17 +73,19 @@ #include +#include #include #include #include #include #include +#include #include /* - * x86_kernel_state: + * machine_thread_kernel_state, x86_kernel_state: * * This structure corresponds to the state of kernel registers * as saved in a context-switch. It lives at the base of the stack. @@ -100,6 +102,11 @@ struct x86_kernel_state { uint64_t k_rip; }; +#ifdef MACH_KERNEL_PRIVATE +typedef struct x86_kernel_state machine_thread_kernel_state; +#include +#endif + /* * Maps state flavor to number of words in the state: */ @@ -116,8 +123,7 @@ struct machine_thread { void *ifps; void *ids; decl_simple_lock_data(,lock); /* protects ifps and ids */ - uint64_t iss_pte0; - uint64_t iss_pte1; + xstate_t xstate; #ifdef MACH_BSD uint64_t cthread_self; /* for use of cthread package */ @@ -174,15 +180,16 @@ extern void act_thread_cfree(void *ctx); /* * On the kernel stack is: * stack: ... - * struct x86_kernel_state + * struct thread_kernel_state * stack+kernel_stack_size */ + #define STACK_IKS(stack) \ - ((struct x86_kernel_state *)((stack) + kernel_stack_size) - 1) + (&(((struct thread_kernel_state *)((stack) + kernel_stack_size)) - 1)->machine) /* - * Return the current stack depth including x86_kernel_state + * Return the current stack depth including thread_kernel_state */ static inline vm_offset_t current_stack_depth(void) @@ -197,7 +204,7 @@ current_stack_depth(void) __asm__ volatile("mov %%esp, %0" : "=m" (stack_ptr)); #endif return (current_cpu_datap()->cpu_kernel_stack - + sizeof(struct x86_kernel_state) + + sizeof(struct thread_kernel_state) - stack_ptr); } diff --git a/osfmk/i386/trap.c b/osfmk/i386/trap.c index 7924f4f7d..ad07c8887 100644 --- a/osfmk/i386/trap.c +++ b/osfmk/i386/trap.c @@ -192,6 +192,13 @@ thread_syscall_return( ret); #endif } + +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_get_kernel_state(thr_act)->allocation_name; + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + throttle_lowpri_io(1); thread_exception_return(); @@ -349,39 +356,24 @@ interrupt(x86_saved_state_t *state) int ipl; int cnum = cpu_number(); cpu_data_t *cdp = cpu_data_ptr[cnum]; - int itype = 0; - - if (is_saved_state64(state) == TRUE) { - x86_saved_state64_t *state64; + int itype = DBG_INTR_TYPE_UNKNOWN; - state64 = saved_state64(state); - rip = state64->isf.rip; - rsp = state64->isf.rsp; - interrupt_num = state64->isf.trapno; -#ifdef __x86_64__ - if(state64->isf.cs & 0x03) -#endif - user_mode = TRUE; - } else { - x86_saved_state32_t *state32; - - state32 = saved_state32(state); - if (state32->cs & 0x03) - user_mode = TRUE; - rip = state32->eip; - rsp = state32->uesp; - interrupt_num = state32->trapno; - } + x86_saved_state64_t *state64 = saved_state64(state); + rip = state64->isf.rip; + rsp = state64->isf.rsp; + interrupt_num = state64->isf.trapno; + if(state64->isf.cs & 0x03) + user_mode = TRUE; if (cpu_data_ptr[cnum]->lcpu.package->num_idle == topoParms.nLThreadsPerPackage) cpu_data_ptr[cnum]->cpu_hwIntpexits[interrupt_num]++; if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_INTERPROCESSOR_INTERRUPT)) - itype = 1; + itype = DBG_INTR_TYPE_IPI; else if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_TIMER_INTERRUPT)) - itype = 2; + itype = DBG_INTR_TYPE_TIMER; else - itype = 3; + itype = DBG_INTR_TYPE_OTHER; KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START, @@ -454,7 +446,7 @@ interrupt(x86_saved_state_t *state) */ if (!user_mode) { uint64_t depth = cdp->cpu_kernel_stack - + sizeof(struct x86_kernel_state) + + sizeof(struct thread_kernel_state) + sizeof(struct i386_exception_link *) - rsp; if (__improbable(depth > kernel_stack_depth_max)) { @@ -564,7 +556,7 @@ kernel_trap( * as soon we possibly can to hold latency down */ if (__improbable(T_PREEMPT == type)) { - ast_taken(AST_PREEMPTION, FALSE); + ast_taken_kernel(); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE, @@ -723,7 +715,7 @@ kernel_trap( fault_result = result = vm_fault(map, vaddr, prot, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); if (result == KERN_SUCCESS) { @@ -1034,6 +1026,9 @@ user_trap( break; case T_INVALID_OPCODE: +#if !defined(RC_HIDE_XNU_J137) + fpUDflt(rip); /* May return from exception directly */ +#endif exc = EXC_BAD_INSTRUCTION; code = EXC_I386_INVOP; break; @@ -1094,7 +1089,7 @@ user_trap( prot |= VM_PROT_EXECUTE; kret = vm_fault(thread->map, vaddr, - prot, FALSE, + prot, FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE, NULL, 0); if (__probable((kret == KERN_SUCCESS) || (kret == KERN_ABORTED))) { @@ -1139,29 +1134,6 @@ user_trap( /* NOTREACHED */ } - -/* - * Handle AST traps for i386. - */ - -extern void log_thread_action (thread_t, char *); - -void -i386_astintr(int preemption) -{ - ast_t mask = AST_ALL; - spl_t s; - - if (preemption) - mask = AST_PREEMPTION; - - s = splsched(); - - ast_taken(mask, s); - - splx(s); -} - /* * Handle exceptions for i386. * @@ -1199,7 +1171,7 @@ i386_exception( void sync_iss_to_iks(x86_saved_state_t *saved_state) { - struct x86_kernel_state *iks; + struct x86_kernel_state *iks = NULL; vm_offset_t kstack; boolean_t record_active_regs = FALSE; @@ -1207,7 +1179,8 @@ sync_iss_to_iks(x86_saved_state_t *saved_state) if (saved_state && saved_state->flavor == THREAD_STATE_NONE) pal_get_kern_regs( saved_state ); - if ((kstack = current_thread()->kernel_stack) != 0) { + if (current_thread() != NULL && + (kstack = current_thread()->kernel_stack) != 0) { x86_saved_state64_t *regs = saved_state64(saved_state); iks = STACK_IKS(kstack); diff --git a/osfmk/i386/trap.h b/osfmk/i386/trap.h index e92e605d7..33676cf75 100644 --- a/osfmk/i386/trap.h +++ b/osfmk/i386/trap.h @@ -135,9 +135,6 @@ extern void interrupt(x86_saved_state_t *regs); extern void panic_double_fault64(x86_saved_state_t *regs); extern void panic_machine_check64(x86_saved_state_t *regs); -extern void i386_astintr(int preemption); - - typedef kern_return_t (*perfCallback)( int trapno, void *regs, @@ -151,7 +148,7 @@ extern volatile perfASTCallback perfASTHook; extern volatile perfCallback perfIntHook; extern void panic_i386_backtrace(void *, int, const char *, boolean_t, x86_saved_state_t *); -extern void print_one_backtrace(pmap_t pmap, vm_offset_t topfp, const char *cur_marker, boolean_t is_64_bit, boolean_t nvram_format); +extern void print_one_backtrace(pmap_t pmap, vm_offset_t topfp, const char *cur_marker, boolean_t is_64_bit); extern void print_thread_num_that_crashed(task_t task); extern void print_tasks_user_threads(task_t task); extern void print_threads_registers(thread_t thread); diff --git a/osfmk/i386/tsc.c b/osfmk/i386/tsc.c index e8de697d9..c776541db 100644 --- a/osfmk/i386/tsc.c +++ b/osfmk/i386/tsc.c @@ -165,6 +165,7 @@ tsc_init(void) } switch (cpuid_cpufamily()) { + case CPUFAMILY_INTEL_KABYLAKE: case CPUFAMILY_INTEL_SKYLAKE: { /* * SkyLake and later has an Always Running Timer (ART) providing diff --git a/osfmk/i386/ucode.c b/osfmk/i386/ucode.c index e83a1655d..5afd98a7b 100644 --- a/osfmk/i386/ucode.c +++ b/osfmk/i386/ucode.c @@ -1,3 +1,30 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ /* * ucode.c * @@ -135,7 +162,7 @@ ucode_update_wake() if (global_update) { kprintf("ucode: Re-applying update after wake (CPU #%d)\n", cpu_number()); update_microcode(); -#ifdef DEBUG +#if DEBUG } else { kprintf("ucode: No update to apply (CPU #%d)\n", cpu_number()); #endif diff --git a/osfmk/i386/ucode.h b/osfmk/i386/ucode.h index 55dc70645..e36380ba9 100644 --- a/osfmk/i386/ucode.h +++ b/osfmk/i386/ucode.h @@ -1,3 +1,30 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ /* * ucode.h * diff --git a/osfmk/ipc/flipc.c b/osfmk/ipc/flipc.c index 1c4cfc563..055a33550 100644 --- a/osfmk/ipc/flipc.c +++ b/osfmk/ipc/flipc.c @@ -154,7 +154,7 @@ flipc_port_destroy(ipc_port_t lport) int m = port_mq->data.port.msgcount; if (m > 0) { ipc_kmsg_t kmsg; -#ifdef DEBUG +#if DEBUG printf("flipc: destroying %p with %d undelivered msgs\n", lport, m); #endif diff --git a/osfmk/ipc/ipc_importance.c b/osfmk/ipc/ipc_importance.c index 749a9c278..f89005cf3 100644 --- a/osfmk/ipc/ipc_importance.c +++ b/osfmk/ipc/ipc_importance.c @@ -84,6 +84,8 @@ static lck_spin_t ipc_importance_lock_data; /* single lock for now */ lck_spin_try_lock(&ipc_importance_lock_data) #define ipc_importance_unlock() \ lck_spin_unlock(&ipc_importance_lock_data) +#define ipc_importance_assert_held() \ + lck_spin_assert(&ipc_importance_lock_data, LCK_ASSERT_OWNED) #define ipc_importance_sleep(elem) lck_spin_sleep(&ipc_importance_lock_data, \ LCK_SLEEP_DEFAULT, \ (event_t)(elem), \ @@ -352,7 +354,7 @@ ipc_importance_release_locked(ipc_importance_elem_t elem) { assert(0 < IIE_REFS(elem)); -#if DEVELOPMENT || DEBUG +#if IMPORTANCE_DEBUG ipc_importance_inherit_t temp_inherit; ipc_importance_task_t link_task; ipc_kmsg_t temp_kmsg; @@ -374,7 +376,7 @@ ipc_importance_release_locked(ipc_importance_elem_t elem) expected++; if (IIE_REFS(elem) < expected + 1) panic("ipc_importance_release_locked (%p)", elem); -#endif +#endif /* IMPORTANCE_DEBUG */ if (0 < ipc_importance_release_internal(elem)) { ipc_importance_unlock(); @@ -590,15 +592,18 @@ ipc_importance_task_check_transition( iit_update_type_t type, uint32_t delta) { - +#if IMPORTANCE_TRACE task_t target_task = task_imp->iit_task; +#endif boolean_t boost = (IIT_UPDATE_HOLD == type); boolean_t before_boosted, after_boosted; + ipc_importance_assert_held(); + if (!ipc_importance_task_is_any_receiver_type(task_imp)) return FALSE; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_START, @@ -611,7 +616,7 @@ ipc_importance_task_check_transition( /* Adjust the assertcnt appropriately */ if (boost) { task_imp->iit_assertcnt += delta; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE DTRACE_BOOST6(send_boost, task_t, target_task, int, target_pid, task_t, current_task(), int, proc_selfpid(), int, delta, int, task_imp->iit_assertcnt); #endif @@ -619,26 +624,17 @@ ipc_importance_task_check_transition( // assert(delta <= task_imp->iit_assertcnt); if (task_imp->iit_assertcnt < delta + IIT_EXTERN(task_imp)) { /* TODO: Turn this back into a panic */ - if (target_task != TASK_NULL) { - printf("Over-release of kernel-internal importance assertions for pid %d (%s), " - "dropping %d assertion(s) but task only has %d remaining (%d external).\n", - task_pid(target_task), - (target_task->bsd_info == NULL) ? "" : proc_name_address(target_task->bsd_info), - delta, - task_imp->iit_assertcnt, - IIT_EXTERN(task_imp)); - } task_imp->iit_assertcnt = IIT_EXTERN(task_imp); } else { task_imp->iit_assertcnt -= delta; } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE // This convers both legacy and voucher-based importance. DTRACE_BOOST4(drop_boost, task_t, target_task, int, target_pid, int, delta, int, task_imp->iit_assertcnt); #endif } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_END, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); #endif @@ -746,7 +742,7 @@ ipc_importance_task_propagate_helper( /* Adjust the task assertions and determine if an edge was crossed */ if (ipc_importance_task_check_transition(temp_task_imp, type, 1)) { - incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); + incr_ref_counter(temp_task_imp->iit_elem.iie_task_refs_added_transition); queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props); /* reference donated */ } else { @@ -811,7 +807,7 @@ ipc_importance_task_propagate_helper( assert(ipc_importance_task_is_any_receiver_type(temp_task_imp)); if (ipc_importance_task_check_transition(temp_task_imp, type, assertcnt)) { ipc_importance_task_reference(temp_task_imp); - incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); + incr_ref_counter(temp_task_imp->iit_elem.iie_task_refs_added_transition); queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props); } } @@ -1087,12 +1083,16 @@ ipc_importance_task_propagate_assertion_locked( queue_init(&updates); queue_init(&propagate); + ipc_importance_assert_held(); + /* * If we're going to update the policy for the provided task, * enqueue it on the propagate queue itself. Otherwise, only * enqueue downstream things. */ if (update_task_imp) { + ipc_importance_task_reference(task_imp); + incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); queue_enter(&propagate, task_imp, ipc_importance_task_t, iit_props); } else { ipc_importance_task_propagate_helper(task_imp, type, &propagate); @@ -1106,6 +1106,8 @@ ipc_importance_task_propagate_assertion_locked( boolean_t need_update; queue_remove_first(&propagate, temp_task_imp, ipc_importance_task_t, iit_props); + /* hold a reference on temp_task_imp */ + assert(IIT_NULL != temp_task_imp); /* only propagate for receivers not already marked as a donor */ @@ -1147,6 +1149,8 @@ ipc_importance_task_propagate_assertion_locked( assert(ipc_importance_task_is_marked_denap_receiver(temp_task_imp)); } } + + ipc_importance_task_release_internal(temp_task_imp); } /* apply updates to task (may drop importance lock) */ @@ -1333,7 +1337,7 @@ ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_im ipc_importance_lock(); target_task = task_imp->iit_task; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START, @@ -1359,7 +1363,7 @@ ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_im } ipc_importance_unlock(); -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); // This covers the legacy case where a task takes an extra boost. @@ -1407,7 +1411,7 @@ ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_im ipc_importance_lock(); target_task = task_imp->iit_task; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START, @@ -1452,7 +1456,7 @@ ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_im ret = KERN_SUCCESS; } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); #endif @@ -1485,7 +1489,7 @@ ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp, return KERN_FAILURE; } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_START, @@ -1499,12 +1503,12 @@ ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp, task_imp->iit_externcnt += count; ipc_importance_unlock(); -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_END, proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); // This is the legacy boosting path DTRACE_BOOST5(receive_boost, task_t, target_task, int, target_pid, int, sender_pid, int, count, int, IIT_LEGACY_EXTERN(task_imp)); -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ return(KERN_SUCCESS); } @@ -1549,7 +1553,7 @@ ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp) /* snapshot task live donor status - may change, but another call will accompany the change */ task_live_donor = target_task->effective_policy.tep_live_donor; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE int target_pid = task_pid(target_task); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, @@ -1576,7 +1580,7 @@ ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp) ipc_importance_task_propagate_assertion_locked(task_imp, type, FALSE); } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_END, target_pid, task_imp->iit_donor, task_live_donor, after_donor, 0); @@ -2210,6 +2214,9 @@ ipc_importance_check_circularity( boolean_t imp_lock_held = FALSE; int assertcnt = 0; ipc_port_t base; + sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0}; + sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0}; + boolean_t update_knote = FALSE; assert(port != IP_NULL); assert(dest != IP_NULL); @@ -2323,7 +2330,8 @@ ipc_importance_check_circularity( ip_lock(port); ipc_port_multiple_unlock(); - not_circular: +not_circular: + imq_lock(&base->ip_messages); /* port is in limbo */ @@ -2351,6 +2359,11 @@ ipc_importance_check_circularity( /* take the port out of limbo w.r.t. assertions */ port->ip_tempowner = 0; + /* Capture the sync qos count delta */ + for (int i = 0; i < THREAD_QOS_LAST; i++) { + sync_qos_delta_add[i] = port_sync_qos(port, i); + } + /* now unlock chain */ ip_unlock(port); @@ -2359,6 +2372,7 @@ ipc_importance_check_circularity( /* every port along chain track assertions behind it */ ipc_port_impcount_delta(dest, assertcnt, base); + update_knote = ipc_port_sync_qos_delta(dest, sync_qos_delta_add, sync_qos_delta_sub); if (dest == base) break; @@ -2411,6 +2425,10 @@ ipc_importance_check_circularity( } } + if (update_knote) { + KNOTE(&base->ip_messages.imq_klist, 0); + } + imq_unlock(&base->ip_messages); ip_unlock(base); /* @@ -2482,7 +2500,12 @@ ipc_importance_send( /* If forced sending a static boost, go update the port */ if ((option & MACH_SEND_IMPORTANCE) != 0) { - kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP; + /* acquire the importance lock while trying to hang on to port lock */ + if (!ipc_importance_lock_try()) { + port_lock_dropped = TRUE; + ip_unlock(port); + ipc_importance_lock(); + } goto portupdate; } @@ -2565,6 +2588,7 @@ ipc_importance_send( return port_lock_dropped; } +portupdate: /* Mark the fact that we are (currently) donating through this message */ kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP; @@ -2578,9 +2602,9 @@ ipc_importance_send( ip_lock(port); } - portupdate: - -#if IMPORTANCE_DEBUG + ipc_importance_assert_held(); + +#if IMPORTANCE_TRACE if (kdebug_enable) { mach_msg_max_trailer_t *dbgtrailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + round_msg(kmsg->ikm_header->msgh_size)); @@ -2589,7 +2613,7 @@ ipc_importance_send( KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_START, task_pid(task), sender_pid, imp_msgh_id, 0, 0); } -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ mach_port_delta_t delta = 1; boolean_t need_port_lock; @@ -2597,6 +2621,7 @@ ipc_importance_send( /* adjust port boost count (with importance and port locked) */ need_port_lock = ipc_port_importance_delta_internal(port, IPID_OPTION_NORMAL, &delta, &task_imp); + /* hold a reference on task_imp */ /* if we need to adjust a task importance as a result, apply that here */ if (IIT_NULL != task_imp && delta != 0) { @@ -2614,7 +2639,12 @@ ipc_importance_send( } } - ipc_importance_unlock(); + if (task_imp) { + ipc_importance_task_release_locked(task_imp); + /* importance unlocked */ + } else { + ipc_importance_unlock(); + } if (need_port_lock) { port_lock_dropped = TRUE; @@ -3194,7 +3224,7 @@ ipc_importance_receive( } } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE if (-1 < impresult) KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_DELV)) | DBG_FUNC_NONE, sender_pid, task_pid(task_self), @@ -3207,7 +3237,7 @@ ipc_importance_receive( */ DTRACE_BOOST5(receive_boost, task_t, task_self, int, task_pid(task_self), int, sender_pid, int, 1, int, task_self->task_imp_base->iit_assertcnt); } -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ } /* diff --git a/osfmk/ipc/ipc_init.c b/osfmk/ipc/ipc_init.c index d81098ce9..49221fa52 100644 --- a/osfmk/ipc/ipc_init.c +++ b/osfmk/ipc/ipc_init.c @@ -71,7 +71,6 @@ */ #include -#include #include #include @@ -238,13 +237,21 @@ ipc_init(void) vm_offset_t min; retval = kmem_suballoc(kernel_map, &min, ipc_kernel_map_size, - TRUE, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC), &ipc_kernel_map); + TRUE, + (VM_FLAGS_ANYWHERE), + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_IPC, + &ipc_kernel_map); if (retval != KERN_SUCCESS) panic("ipc_init: kmem_suballoc of ipc_kernel_map failed"); retval = kmem_suballoc(kernel_map, &min, ipc_kernel_copy_map_size, - TRUE, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC), &ipc_kernel_copy_map); + TRUE, + (VM_FLAGS_ANYWHERE), + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_IPC, + &ipc_kernel_copy_map); if (retval != KERN_SUCCESS) panic("ipc_init: kmem_suballoc of ipc_kernel_copy_map failed"); diff --git a/osfmk/ipc/ipc_kmsg.c b/osfmk/ipc/ipc_kmsg.c index ef7e61a4c..4ec900c7b 100644 --- a/osfmk/ipc/ipc_kmsg.c +++ b/osfmk/ipc/ipc_kmsg.c @@ -578,6 +578,7 @@ MACRO_END #define KMSG_TRACE_PORTS_SHIFT 0 #if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD) +#include extern boolean_t kdebug_debugid_enabled(uint32_t debugid); void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, @@ -852,6 +853,8 @@ mach_msg_return_t ipc_kmsg_copyin_body( ipc_space_t space, vm_map_t map); +extern int thread_qos_from_pthread_priority(unsigned long, unsigned long *); + /* * We keep a per-processor cache of kernel message buffers. * The cache saves the overhead/locking of using kalloc/kfree. @@ -1108,7 +1111,7 @@ ipc_kmsg_override_qos( cur->ikm_qos_override = override; if (cur == first) return TRUE; - cur = cur->ikm_next; + cur = cur->ikm_prev; } return FALSE; } @@ -1463,7 +1466,16 @@ ipc_kmsg_set_prealloc( assert(kmsg->ikm_prealloc == IP_NULL); kmsg->ikm_prealloc = IP_NULL; + /* take the mqueue lock since the sync qos is protected under it */ + imq_lock(&port->ip_messages); + + /* copy the sync qos values to kmsg */ + for (int i = 0; i < THREAD_QOS_LAST; i++) { + kmsg->sync_qos[i] = port_sync_qos(port, i); + } + kmsg->special_port_qos = port_special_qos(port); IP_SET_PREALLOC(port, kmsg); + imq_unlock(&port->ip_messages); } /* @@ -1481,7 +1493,18 @@ ipc_kmsg_clear_prealloc( assert(kmsg->ikm_prealloc == port); kmsg->ikm_prealloc = IP_NULL; + + /* take the mqueue lock since the sync qos is protected under it */ + imq_lock(&port->ip_messages); + IP_CLEAR_PREALLOC(port, kmsg); + + /* copy the sync qos values from kmsg to port */ + for (int i = 0; i < THREAD_QOS_LAST; i++) { + set_port_sync_qos(port, i, kmsg->sync_qos[i]); + } + set_port_special_qos(port, kmsg->special_port_qos); + imq_unlock(&port->ip_messages); } /* @@ -1546,6 +1569,14 @@ ipc_kmsg_get( if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied)) return MACH_SEND_INVALID_DATA; + /* + * If the message claims to be complex, it must at least + * have the length of a "base" message (header + dsc_count). + */ + if (len_copied < sizeof(mach_msg_legacy_base_t) && + (legacy_base.header.msgh_bits & MACH_MSGH_BITS_COMPLEX)) + return MACH_SEND_MSG_TOO_SMALL; + msg_addr += sizeof(legacy_base.header); #if defined(__LP64__) size += LEGACY_HEADER_SIZE_DELTA; @@ -1705,6 +1736,8 @@ ipc_kmsg_get_from_kernel( (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, size); + ikm_qos_init(kmsg); + kmsg->ikm_header->msgh_size = size; /* @@ -1768,10 +1801,10 @@ ipc_kmsg_send( #if IMPORTANCE_INHERITANCE boolean_t did_importance = FALSE; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE mach_msg_id_t imp_msgh_id = -1; int sender_pid = -1; -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ #endif /* IMPORTANCE_INHERITANCE */ /* don't allow the creation of a circular loop */ @@ -1899,10 +1932,10 @@ retry: default: break; } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END, task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0); -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ } #endif /* IMPORTANCE_INHERITANCE */ @@ -2069,13 +2102,16 @@ ipc_kmsg_put_to_kernel( unsigned long pthread_priority_canonicalize(unsigned long priority, boolean_t propagation); -static void +static kern_return_t ipc_kmsg_set_qos( ipc_kmsg_t kmsg, mach_msg_option_t options, mach_msg_priority_t override) { kern_return_t kr; + unsigned long flags = 0; + ipc_port_t special_reply_port = kmsg->ikm_header->msgh_local_port; + ipc_port_t dest_port = kmsg->ikm_header->msgh_remote_port; kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, &kmsg->ikm_qos); if (kr != KERN_SUCCESS) { @@ -2092,6 +2128,25 @@ ipc_kmsg_set_qos( if (canon > kmsg->ikm_qos) kmsg->ikm_qos_override = canon; } + + kr = KERN_SUCCESS; + if ((options & MACH_SEND_SYNC_OVERRIDE)) { + if (IP_VALID(special_reply_port) && + MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) { + /* + * Update the sync override count if the reply port is a special reply port, + * link the destination port to special reply port and update the qos count + * of destination port. + * + * Use the qos value passed by voucher and not the one passed by notify field. + */ + kr = ipc_port_link_special_reply_port_with_qos(special_reply_port, dest_port, + thread_qos_from_pthread_priority(kmsg->ikm_qos, &flags)); + } else { + kr = KERN_FAILURE; + } + } + return kr; } /* @@ -2524,13 +2579,13 @@ ipc_kmsg_copyin_header( voucher_type = MACH_MSG_TYPE_MOVE_SEND; } - /* capture the qos value(s) for the kmsg */ - ipc_kmsg_set_qos(kmsg, *optionp, override); - msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits); msg->msgh_remote_port = (ipc_port_t)dest_port; msg->msgh_local_port = (ipc_port_t)reply_port; + /* capture the qos value(s) for the kmsg */ + ipc_kmsg_set_qos(kmsg, *optionp, override); + if (release_port != IP_NULL) ip_release(release_port); @@ -3053,8 +3108,8 @@ ipc_kmsg_copyin_body( * space. */ if (space_needed) { - if (vm_allocate(ipc_kernel_copy_map, &paddr, space_needed, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)) != KERN_SUCCESS) { + if (vm_allocate_kernel(ipc_kernel_copy_map, &paddr, space_needed, + VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC) != KERN_SUCCESS) { mr = MACH_MSG_VM_KERNEL; goto clean_message; } @@ -4107,14 +4162,14 @@ ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, /* * Dynamically allocate the region */ - int anywhere = VM_FLAGS_ANYWHERE; - if (vm_kernel_map_is_kernel(map)) anywhere |= VM_MAKE_TAG(VM_KERN_MEMORY_IPC); - else anywhere |= VM_MAKE_TAG(VM_MEMORY_MACH_MSG); + vm_tag_t tag; + if (vm_kernel_map_is_kernel(map)) tag = VM_KERN_MEMORY_IPC; + else tag = VM_MEMORY_MACH_MSG; kern_return_t kr; - if ((kr = mach_vm_allocate(map, &rcv_addr, + if ((kr = mach_vm_allocate_kernel(map, &rcv_addr, (mach_vm_size_t)names_length, - anywhere)) != KERN_SUCCESS) { + VM_FLAGS_ANYWHERE, tag)) != KERN_SUCCESS) { ipc_kmsg_clean_body(kmsg, 1, (mach_msg_descriptor_t *)dsc); rcv_addr = 0; @@ -4422,6 +4477,9 @@ ipc_kmsg_copyout_pseudo( mach_port_name_t dest_name, reply_name; mach_msg_return_t mr; + /* Set ith_knote to ITH_KNOTE_PSEUDO */ + current_thread()->ith_knote = ITH_KNOTE_PSEUDO; + assert(IO_VALID(dest)); #if 0 @@ -4702,6 +4760,28 @@ ipc_kmsg_copyout_to_kernel_legacy( } #endif /* IKM_SUPPORT_LEGACY */ +#ifdef __arm64__ +/* + * Just sets those parts of the trailer that aren't set up at allocation time. + */ +static void +ipc_kmsg_munge_trailer(mach_msg_max_trailer_t *in, void *_out, boolean_t is64bit) +{ + if (is64bit) { + mach_msg_max_trailer64_t *out = (mach_msg_max_trailer64_t*)_out; + out->msgh_seqno = in->msgh_seqno; + out->msgh_context = in->msgh_context; + out->msgh_trailer_size = in->msgh_trailer_size; + out->msgh_ad = in->msgh_ad; + } else { + mach_msg_max_trailer32_t *out = (mach_msg_max_trailer32_t*)_out; + out->msgh_seqno = in->msgh_seqno; + out->msgh_context = (mach_port_context32_t)in->msgh_context; + out->msgh_trailer_size = in->msgh_trailer_size; + out->msgh_ad = in->msgh_ad; + } +} +#endif /* __arm64__ */ mach_msg_trailer_size_t ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, @@ -4711,10 +4791,25 @@ ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, { mach_msg_max_trailer_t *trailer; +#ifdef __arm64__ + mach_msg_max_trailer_t tmp_trailer; /* This accommodates U64, and we'll munge */ + void *real_trailer_out = (void*)(mach_msg_max_trailer_t *) + ((vm_offset_t)kmsg->ikm_header + + round_msg(kmsg->ikm_header->msgh_size)); + + /* + * Populate scratch with initial values set up at message allocation time. + * After, we reinterpret the space in the message as the right type + * of trailer for the address space in question. + */ + bcopy(real_trailer_out, &tmp_trailer, MAX_TRAILER_SIZE); + trailer = &tmp_trailer; +#else /* __arm64__ */ (void)thread; trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + round_msg(kmsg->ikm_header->msgh_size)); +#endif /* __arm64__ */ if (!(option & MACH_RCV_TRAILER_MASK)) { return trailer->msgh_trailer_size; @@ -4744,6 +4839,9 @@ ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, } done: +#ifdef __arm64__ + ipc_kmsg_munge_trailer(trailer, real_trailer_out, thread_is_64bit(thread)); +#endif /* __arm64__ */ return trailer->msgh_trailer_size; } diff --git a/osfmk/ipc/ipc_kmsg.h b/osfmk/ipc/ipc_kmsg.h index 6e11487c5..f7ff4059c 100644 --- a/osfmk/ipc/ipc_kmsg.h +++ b/osfmk/ipc/ipc_kmsg.h @@ -92,6 +92,9 @@ * * The ikm_header.msgh_remote_port field is the destination * of the message. + * + * sync_qos and special_port_qos stores the qos for prealloced + * port, this fields could be deleted once we remove ip_prealloc. */ struct ipc_kmsg { @@ -105,6 +108,8 @@ struct ipc_kmsg { mach_msg_priority_t ikm_qos_override; /* qos override on this kmsg */ struct ipc_importance_elem *ikm_importance; /* inherited from */ queue_chain_t ikm_inheritance; /* inherited from link */ + sync_qos_count_t sync_qos[THREAD_QOS_LAST]; /* sync qos counters for ikm_prealloc port */ + sync_qos_count_t special_port_qos; /* special port qos for ikm_prealloc port */ #if MACH_FLIPC struct mach_node *ikm_node; /* Originating node - needed for ack */ #endif @@ -164,12 +169,17 @@ MACRO_BEGIN \ (kmsg)->ikm_prealloc = IP_NULL; \ (kmsg)->ikm_voucher = IP_NULL; \ (kmsg)->ikm_importance = IIE_NULL; \ - (kmsg)->ikm_qos = MACH_MSG_PRIORITY_UNSPECIFIED; \ - (kmsg)->ikm_qos_override = MACH_MSG_PRIORITY_UNSPECIFIED; \ + ikm_qos_init(kmsg); \ ikm_flipc_init(kmsg); \ assert((kmsg)->ikm_prev = (kmsg)->ikm_next = IKM_BOGUS); \ MACRO_END +#define ikm_qos_init(kmsg) \ +MACRO_BEGIN \ + (kmsg)->ikm_qos = MACH_MSG_PRIORITY_UNSPECIFIED; \ + (kmsg)->ikm_qos_override = MACH_MSG_PRIORITY_UNSPECIFIED; \ +MACRO_END + #define ikm_check_init(kmsg, size) \ MACRO_BEGIN \ assert((kmsg)->ikm_size == (size)); \ diff --git a/osfmk/ipc/ipc_notify.c b/osfmk/ipc/ipc_notify.c index 498401cc0..d1f50d5c7 100644 --- a/osfmk/ipc/ipc_notify.c +++ b/osfmk/ipc/ipc_notify.c @@ -158,6 +158,8 @@ void ipc_notify_send_once( ipc_port_t port) { + ipc_port_unlink_special_reply_port(port, IPC_PORT_UNLINK_SR_NONE); + (void)mach_notify_send_once(port); /* send-once right consumed */ } diff --git a/osfmk/ipc/ipc_object.c b/osfmk/ipc/ipc_object.c index c308fa0fc..5ff293fca 100644 --- a/osfmk/ipc/ipc_object.c +++ b/osfmk/ipc/ipc_object.c @@ -70,8 +70,6 @@ * Functions to manipulate IPC objects. */ -#include - #include #include #include @@ -909,6 +907,8 @@ ipc_object_copyout_name( if (ipc_importance_task_is_any_receiver_type(task_imp)) { assertcnt = port->ip_impcount; ipc_importance_task_reference(task_imp); + } else { + task_imp = IIT_NULL; } } diff --git a/osfmk/ipc/ipc_object.h b/osfmk/ipc/ipc_object.h index aaddd33c2..6aaf285a6 100644 --- a/osfmk/ipc/ipc_object.h +++ b/osfmk/ipc/ipc_object.h @@ -72,8 +72,6 @@ #ifndef _IPC_IPC_OBJECT_H_ #define _IPC_IPC_OBJECT_H_ -#include - #include #include #include diff --git a/osfmk/ipc/ipc_port.c b/osfmk/ipc/ipc_port.c index b2f6d8641..871f98f49 100644 --- a/osfmk/ipc/ipc_port.c +++ b/osfmk/ipc/ipc_port.c @@ -91,6 +91,7 @@ #include #include #include +#include #include @@ -112,9 +113,6 @@ void ipc_port_callstack_init_debug( #endif /* MACH_ASSERT */ -void kdp_mqueue_send_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); -void kdp_mqueue_recv_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); - void ipc_port_release(ipc_port_t port) { @@ -639,7 +637,8 @@ ipc_port_init( port->ip_strict_guard = 0; port->ip_impcount = 0; - port->ip_reserved = 0; + port->ip_specialreply = 0; + port->ip_link_sync_qos = 0; ipc_mqueue_init(&port->ip_messages, FALSE /* !set */, NULL /* no reserved link */); @@ -873,6 +872,7 @@ ipc_port_destroy(ipc_port_t port) ipc_port_t pdrequest, nsrequest; ipc_mqueue_t mqueue; ipc_kmsg_t kmsg; + boolean_t special_reply = port->ip_specialreply; #if IMPORTANCE_INHERITANCE ipc_importance_task_t release_imp_task = IIT_NULL; @@ -923,6 +923,10 @@ ipc_port_destroy(ipc_port_t port) port->ip_destination = IP_NULL; ip_unlock(port); + if (special_reply) { + ipc_port_unlink_special_reply_port(port, + IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE); + } /* consumes our refs for port and pdrequest */ ipc_notify_port_destroyed(pdrequest, port); @@ -973,6 +977,12 @@ ipc_port_destroy(ipc_port_t port) ip_unlock(port); } + /* unlink the kmsg from special reply port */ + if (special_reply) { + ipc_port_unlink_special_reply_port(port, + IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE); + } + /* throw away no-senders request */ if (nsrequest != IP_NULL) ipc_notify_send_once(nsrequest); /* consumes ref */ @@ -1046,6 +1056,9 @@ ipc_port_check_circularity( return ipc_importance_check_circularity(port, dest); #else ipc_port_t base; + sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0}; + sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0}; + boolean_t update_knote = FALSE; assert(port != IP_NULL); assert(dest != IP_NULL); @@ -1127,7 +1140,8 @@ ipc_port_check_circularity( ip_lock(port); ipc_port_multiple_unlock(); - not_circular: +not_circular: + imq_lock(&base->ip_messages); /* port is in limbo */ @@ -1138,11 +1152,18 @@ ipc_port_check_circularity( ip_reference(dest); port->ip_destination = dest; + /* Capture the sync qos count delta */ + for (int i = 0; i < THREAD_QOS_LAST; i++) { + sync_qos_delta_add[i] = port_sync_qos(port, i); + } + /* now unlock chain */ ip_unlock(port); for (;;) { + /* every port along chain tracks override behind it */ + update_knote = ipc_port_sync_qos_delta(dest, sync_qos_delta_add, sync_qos_delta_sub); if (dest == base) break; @@ -1162,12 +1183,458 @@ ipc_port_check_circularity( (base->ip_receiver_name != MACH_PORT_NULL) || (base->ip_destination == IP_NULL)); + if (update_knote) { + KNOTE(&base->ip_messages.imq_klist, 0); + } + imq_unlock(&base->ip_messages); + ip_unlock(base); return FALSE; #endif /* !IMPORTANCE_INHERITANCE */ } +/* + * Routine: ipc_port_link_special_reply_port_with_qos + * Purpose: + * Link the special reply port with the destination port. + * Update the sync qos count of special reply port, + * destination port. + * + * Conditions: + * Nothing is locked. + */ +kern_return_t +ipc_port_link_special_reply_port_with_qos( + ipc_port_t special_reply_port, + ipc_port_t dest_port, + int qos) +{ + ipc_port_t next, base; + sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0}; + sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0}; + boolean_t update_knote = FALSE; + boolean_t multiple_lock = FALSE; + + ip_lock(dest_port); + + /* Check if dest is active */ + if (!ip_active(dest_port)) { + ip_unlock(dest_port); + return KERN_FAILURE; + } + + if ((dest_port->ip_receiver_name == MACH_PORT_NULL) && + (dest_port->ip_destination != IP_NULL)) { + /* dest_port is in transit; need to take the serialize lock */ + ip_unlock(dest_port); + goto take_multiple_lock; + } + + /* Check if the port is a special reply port */ + if (ip_lock_try(special_reply_port)) { + if (!special_reply_port->ip_specialreply || + !special_reply_port->ip_link_sync_qos || + (special_reply_port->ip_sync_qos_override_port != IP_NULL && + special_reply_port->ip_sync_qos_override_port != dest_port)) { + + boolean_t link_sync_qos = special_reply_port->ip_link_sync_qos; + ip_unlock(special_reply_port); + ip_unlock(dest_port); + /* return KERN_SUCCESS when link_sync_qos is not set */ + if (!link_sync_qos) { + return KERN_SUCCESS; + } + return KERN_FAILURE; + } else { + goto both_ports_locked; + } + } + + ip_unlock(dest_port); + +take_multiple_lock: + + ipc_port_multiple_lock(); /* massive serialization */ + multiple_lock = TRUE; + + ip_lock(special_reply_port); + + /* Check if the special reply port is marked regular */ + if (!special_reply_port->ip_specialreply || + !special_reply_port->ip_link_sync_qos || + (special_reply_port->ip_sync_qos_override_port != IP_NULL && + special_reply_port->ip_sync_qos_override_port != dest_port)) { + + boolean_t link_sync_qos = special_reply_port->ip_link_sync_qos; + ip_unlock(special_reply_port); + ipc_port_multiple_unlock(); + /* return KERN_SUCCESS when link_sync_qos is not set */ + if (!link_sync_qos) { + return KERN_SUCCESS; + } + return KERN_FAILURE; + } + + ip_lock(dest_port); + +both_ports_locked: + next = dest_port; + + /* Apply the qos to special reply port, capture the old qos */ + if (special_reply_port->ip_sync_qos_override_port != IP_NULL) { + /* Check if qos needs to be updated */ + if ((sync_qos_count_t)qos <= port_special_qos(special_reply_port)) { + imq_lock(&dest_port->ip_messages); + goto done_update; + } + sync_qos_delta_sub[port_special_qos(special_reply_port)]++; + } + + set_port_special_qos(special_reply_port, (sync_qos_count_t)qos); + sync_qos_delta_add[qos]++; + + /* Link the special reply port to dest port */ + if (special_reply_port->ip_sync_qos_override_port == IP_NULL) { + /* take a reference on dest_port */ + ip_reference(dest_port); + special_reply_port->ip_sync_qos_override_port = dest_port; + } + + /* Apply the sync qos delta to all in-transit ports */ + for (;;) { + boolean_t port_not_in_transit = FALSE; + if (!ip_active(next) || + (next->ip_receiver_name != MACH_PORT_NULL) || + (next->ip_destination == IP_NULL)) { + /* Get the mqueue lock for destination port to update knotes */ + imq_lock(&next->ip_messages); + port_not_in_transit = TRUE; + } + /* Apply the sync qos delta */ + update_knote = ipc_port_sync_qos_delta(next, sync_qos_delta_add, sync_qos_delta_sub); + + if (port_not_in_transit) + break; + + next = next->ip_destination; + ip_lock(next); + } +done_update: + + if (multiple_lock) { + ipc_port_multiple_unlock(); + } + + ip_unlock(special_reply_port); + base = next; + next = dest_port; + + while (next != base) { + ipc_port_t prev = next; + next = next->ip_destination; + + ip_unlock(prev); + } + + if (update_knote) { + KNOTE(&base->ip_messages.imq_klist, 0); + } + imq_unlock(&base->ip_messages); + ip_unlock(base); + return KERN_SUCCESS; +} + +/* + * Routine: ipc_port_unlink_special_reply_port_locked + * Purpose: + * If the special port is linked to a port, adjust it's sync qos override and unlink the port. + * Condition: + * Special reply port locked on entry. + * Special reply port unlocked on return. + * Returns: + * None. + */ +void +ipc_port_unlink_special_reply_port_locked( + ipc_port_t special_reply_port, + struct knote *kn, + uint8_t flags) +{ + ipc_port_t dest_port; + sync_qos_count_t sync_qos; + sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0}; + sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0}; + + /* Return if called from copy out in pseudo receive */ + if (kn == ITH_KNOTE_PSEUDO) { + ip_unlock(special_reply_port); + return; + } + + /* check if special port has a port linked to it */ + if (special_reply_port->ip_specialreply == 0 || + special_reply_port->ip_sync_qos_override_port == IP_NULL) { + set_port_special_qos(special_reply_port, 0); + if (flags & IPC_PORT_UNLINK_SR_CLEAR_SPECIAL_REPLY) { + special_reply_port->ip_specialreply = 0; + } + if (flags & IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE) { + special_reply_port->ip_link_sync_qos = 1; + } + ip_unlock(special_reply_port); + return; + } + + /* + * port->ip_sync_qos_override_port is not null and it is safe + * to access it since ip_specialreply is set. + */ + dest_port = special_reply_port->ip_sync_qos_override_port; + sync_qos_delta_sub[port_special_qos(special_reply_port)]++; + sync_qos = port_special_qos(special_reply_port); + + /* Clear qos delta for special reply port */ + set_port_special_qos(special_reply_port, 0); + special_reply_port->ip_sync_qos_override_port = IP_NULL; + if (flags & IPC_PORT_UNLINK_SR_CLEAR_SPECIAL_REPLY) { + special_reply_port->ip_specialreply = 0; + } + + if (flags & IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE) { + special_reply_port->ip_link_sync_qos = 1; + } else { + special_reply_port->ip_link_sync_qos = 0; + } + + ip_unlock(special_reply_port); + + /* Add the sync qos on knote */ + if (ITH_KNOTE_VALID(kn)) { + knote_adjust_sync_qos(kn, sync_qos, TRUE); + } + + /* Adjust the sync qos of destination */ + ipc_port_adjust_sync_qos(dest_port, sync_qos_delta_add, sync_qos_delta_sub); + ip_release(dest_port); +} + +/* + * Routine: ipc_port_unlink_special_reply_port + * Purpose: + * If the special port is linked to a port, adjust it's sync qos override and unlink the port. + * Condition: + * Nothing locked. + * Returns: + * None. + */ +void +ipc_port_unlink_special_reply_port( + ipc_port_t special_reply_port, + uint8_t flags) +{ + ip_lock(special_reply_port); + ipc_port_unlink_special_reply_port_locked(special_reply_port, NULL, flags); + /* special_reply_port unlocked */ +} + +/* + * Routine: ipc_port_sync_qos_delta + * Purpose: + * Adjust the sync qos count associated with a port. + * + * For now, be defensive during deductions to make sure the + * sync_qos count for the port doesn't underflow zero. + * Returns: + * TRUE: if max sync qos of the port changes. + * FALSE: otherwise. + * Conditions: + * The port is referenced and locked. + * The mqueue is locked if port is not in-transit. + */ +boolean_t +ipc_port_sync_qos_delta( + ipc_port_t port, + sync_qos_count_t *sync_qos_delta_add, + sync_qos_count_t *sync_qos_delta_sub) +{ + sync_qos_count_t max_sync_qos_index; + + if (!ip_active(port)) { + return FALSE; + } + + max_sync_qos_index = ipc_port_get_max_sync_qos_index(port); + + for (int i = 0; i < THREAD_QOS_LAST; i++) { + sync_qos_count_t port_sync_qos_count = port_sync_qos(port, i); + /* Do not let the sync qos underflow */ + if (sync_qos_delta_sub[i] > port_sync_qos_count) { + KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_UNDERFLOW), + i, VM_KERNEL_UNSLIDE_OR_PERM(port), + port_sync_qos_count, sync_qos_delta_sub[i]); + + set_port_sync_qos(port, i, 0); + } else if (sync_qos_delta_sub[i] != 0) { + KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_REMOVED), + i, VM_KERNEL_UNSLIDE_OR_PERM(port), + port_sync_qos_count, sync_qos_delta_sub[i]); + + set_port_sync_qos(port, i, (port_sync_qos_count - sync_qos_delta_sub[i])); + } + + port_sync_qos_count = port_sync_qos(port, i); + /* Do not let the sync qos overflow */ + if (UCHAR_MAX - sync_qos_delta_add[i] < port_sync_qos_count) { + KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_OVERFLOW), + i, VM_KERNEL_UNSLIDE_OR_PERM(port), + port_sync_qos_count, sync_qos_delta_add[i]); + + set_port_sync_qos(port, i, UCHAR_MAX); + } else if (sync_qos_delta_add[i] != 0) { + KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_APPLIED), + i, VM_KERNEL_UNSLIDE_OR_PERM(port), + port_sync_qos_count, sync_qos_delta_add[i]); + + set_port_sync_qos(port, i, (port_sync_qos_count + sync_qos_delta_add[i])); + } + } + return (ipc_port_get_max_sync_qos_index(port) != max_sync_qos_index); +} + +/* + * Routine: ipc_port_get_max_sync_qos_index + * Purpose: + * Return the max sync qos of the port. + * + * Conditions: + */ +sync_qos_count_t +ipc_port_get_max_sync_qos_index( + ipc_port_t port) +{ + int i; + for (i = THREAD_QOS_LAST - 1; i >= 0; i--) { + if (port_sync_qos(port, i) != 0) { + return i; + } + } + return THREAD_QOS_UNSPECIFIED; +} + +/* + * Routine: ipc_port_adjust_sync_qos + * Purpose: + * Adjust sync qos of the port and it's destination + * port if the port is in transit. + * Conditions: + * Nothing locked. + * Returns: + * None. + */ +void +ipc_port_adjust_sync_qos( + ipc_port_t port, + sync_qos_count_t *sync_qos_delta_add, + sync_qos_count_t *sync_qos_delta_sub) +{ + boolean_t update_knote; + boolean_t multiple_lock = FALSE; + ipc_port_t dest, base, next; + + ip_lock(port); + + /* Check if the port is in transit */ + if (!ip_active(port) || + (port->ip_receiver_name != MACH_PORT_NULL) || + (port->ip_destination == IP_NULL)) { + /* lock the mqueue since port is not in-transit */ + imq_lock(&port->ip_messages); + update_knote = ipc_port_sync_qos_delta(port, sync_qos_delta_add, sync_qos_delta_sub); + if (update_knote) { + KNOTE(&port->ip_messages.imq_klist, 0); + } + imq_unlock(&port->ip_messages); + ip_unlock(port); + return; + } + + dest = port->ip_destination; + assert(dest != IP_NULL); + + if (ip_lock_try(dest)) { + if (!ip_active(dest) || + (dest->ip_receiver_name != MACH_PORT_NULL) || + (dest->ip_destination == IP_NULL)) { + update_knote = ipc_port_sync_qos_delta(port, sync_qos_delta_add, sync_qos_delta_sub); + ip_unlock(port); + + /* lock the mqueue since dest is not in-transit */ + imq_lock(&dest->ip_messages); + update_knote = ipc_port_sync_qos_delta(dest, sync_qos_delta_add, sync_qos_delta_sub); + if (update_knote) { + KNOTE(&dest->ip_messages.imq_klist, 0); + } + imq_unlock(&dest->ip_messages); + ip_unlock(dest); + return; + } + + /* dest is in transit; need to take the serialize lock */ + ip_unlock(dest); + } + + ip_unlock(port); + + ipc_port_multiple_lock(); /* massive serialization */ + multiple_lock = TRUE; + + ip_lock(port); + next = port; + + /* Apply the sync qos delta to all in-transit ports */ + for (;;) { + boolean_t port_not_in_transit = FALSE; + + if (!ip_active(next) || + (next->ip_receiver_name != MACH_PORT_NULL) || + (next->ip_destination == IP_NULL)) { + /* Get the mqueue lock for destination port to update knotes */ + imq_lock(&next->ip_messages); + port_not_in_transit = TRUE; + } + + /* Apply the sync qos delta */ + update_knote = ipc_port_sync_qos_delta(next, sync_qos_delta_add, sync_qos_delta_sub); + + if (port_not_in_transit) + break; + + next = next->ip_destination; + ip_lock(next); + } + + if (multiple_lock) { + ipc_port_multiple_unlock(); + } + + base = next; + next = port; + + while (next != base) { + ipc_port_t prev = next; + next = next->ip_destination; + + ip_unlock(prev); + } + + if (update_knote) { + KNOTE(&base->ip_messages.imq_klist, 0); + } + imq_unlock(&base->ip_messages); + ip_unlock(base); +} + /* * Routine: ipc_port_impcount_delta * Purpose: @@ -1607,6 +2074,10 @@ ipc_port_release_send( ip_lock(port); assert(port->ip_srights > 0); + if (port->ip_srights == 0) { + panic("Over-release of port %p send right!", port); + } + port->ip_srights--; if (!ip_active(port)) { @@ -1694,9 +2165,14 @@ ipc_port_release_sonce( if (!IP_VALID(port)) return; + ipc_port_unlink_special_reply_port(port, IPC_PORT_UNLINK_SR_NONE); + ip_lock(port); assert(port->ip_sorights > 0); + if (port->ip_sorights == 0) { + panic("Over-release of port %p send-once right!", port); + } port->ip_sorights--; @@ -1871,7 +2347,7 @@ kdp_mqueue_send_find_owner(struct waitq * waitq, __assert_only event64_t event, if (ip_active(port)) { if (port->ip_tempowner) { - if (port->ip_imp_task != IIT_NULL) { + if (port->ip_imp_task != IIT_NULL && port->ip_imp_task->iit_task != NULL) { /* port is held by a tempowner */ waitinfo->owner = pid_from_task(port->ip_imp_task->iit_task); } else { diff --git a/osfmk/ipc/ipc_port.h b/osfmk/ipc/ipc_port.h index f7f2cede7..c2be8199b 100644 --- a/osfmk/ipc/ipc_port.h +++ b/osfmk/ipc/ipc_port.h @@ -74,7 +74,6 @@ #ifdef MACH_KERNEL_PRIVATE -#include #include #include @@ -129,12 +128,19 @@ struct ipc_port { union { ipc_kobject_t kobject; ipc_importance_task_t imp_task; + ipc_port_t sync_qos_override_port; } kdata; struct ipc_port *ip_nsrequest; struct ipc_port *ip_pdrequest; struct ipc_port_request *ip_requests; - struct ipc_kmsg *ip_premsg; + union { + struct ipc_kmsg *premsg; + struct { + sync_qos_count_t sync_qos[THREAD_QOS_LAST]; + sync_qos_count_t special_port_qos; + } qos_counter; + } kdata2; mach_vm_address_t ip_context; @@ -144,7 +150,8 @@ struct ipc_port { ip_tempowner:1, /* dont give donations to current receiver */ ip_guarded:1, /* port guarded (use context value as guard) */ ip_strict_guard:1, /* Strict guarding; Prevents user manipulation of context values directly */ - ip_reserved:2, + ip_specialreply:1, /* port is a special reply port */ + ip_link_sync_qos:1, /* link the special reply port to destination port */ ip_impcount:24; /* number of importance donations in nested queue */ mach_port_mscount_t ip_mscount; @@ -175,6 +182,32 @@ struct ipc_port { #define ip_kobject kdata.kobject #define ip_imp_task kdata.imp_task +#define ip_sync_qos_override_port kdata.sync_qos_override_port + +#define ip_premsg kdata2.premsg +#define ip_sync_qos kdata2.qos_counter.sync_qos +#define ip_special_port_qos kdata2.qos_counter.special_port_qos + +#define port_sync_qos(port, i) (IP_PREALLOC(port) ? (port)->ip_premsg->sync_qos[(i)] : (port)->ip_sync_qos[(i)]) +#define port_special_qos(port) (IP_PREALLOC(port) ? (port)->ip_premsg->special_port_qos : (port)->ip_special_port_qos) + +#define set_port_sync_qos(port, i, value) \ +MACRO_BEGIN \ +if (IP_PREALLOC(port)) { \ + (port)->ip_premsg->sync_qos[(i)] = (value); \ +} else { \ + (port)->ip_sync_qos[(i)] = (value); \ +} \ +MACRO_END + +#define set_port_special_qos(port, value) \ +MACRO_BEGIN \ +if (IP_PREALLOC(port)) { \ + (port)->ip_premsg->special_port_qos = (value); \ +} else { \ + (port)->ip_special_port_qos = (value); \ +} \ +MACRO_END #define IP_NULL IPC_PORT_NULL #define IP_DEAD IPC_PORT_DEAD @@ -441,6 +474,48 @@ enum { IPID_OPTION_SENDPOSSIBLE = 1, /* send-possible induced boost */ }; +/* link the destination port with special reply port */ +kern_return_t +ipc_port_link_special_reply_port_with_qos( + ipc_port_t special_reply_port, + ipc_port_t dest_port, + int qos); + +/* link the destination port with locked special reply port */ +void ipc_port_unlink_special_reply_port_locked( + ipc_port_t special_reply_port, + struct knote *kn, + uint8_t flags); + +/* Unlink the destination port from special reply port */ +void +ipc_port_unlink_special_reply_port( + ipc_port_t special_reply_port, + uint8_t flags); + +#define IPC_PORT_UNLINK_SR_NONE 0 +#define IPC_PORT_UNLINK_SR_CLEAR_SPECIAL_REPLY 0x1 +#define IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE 0x2 + +/* Get the max sync qos override index applied to the port */ +sync_qos_count_t +ipc_port_get_max_sync_qos_index( + ipc_port_t port); + +/* Apply qos delta to the port */ +boolean_t +ipc_port_sync_qos_delta( + ipc_port_t port, + sync_qos_count_t *sync_qos_delta_add, + sync_qos_count_t *sync_qos_delta_sub); + +/* Adjust the sync qos of the port and it's destination port */ +void +ipc_port_adjust_sync_qos( + ipc_port_t port, + sync_qos_count_t *sync_qos_delta_add, + sync_qos_count_t *sync_qos_delta_sub); + /* apply importance delta to port only */ extern mach_port_delta_t ipc_port_impcount_delta( diff --git a/osfmk/ipc/ipc_pset.c b/osfmk/ipc/ipc_pset.c index 4dc635a06..2fa71904d 100644 --- a/osfmk/ipc/ipc_pset.c +++ b/osfmk/ipc/ipc_pset.c @@ -76,6 +76,7 @@ #include #include +#include /* * Routine: ipc_pset_alloc @@ -349,24 +350,26 @@ ipc_pset_destroy( #include #include -static int filt_machportattach(struct knote *kn); +static int filt_machportattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_machportdetach(struct knote *kn); static int filt_machport(struct knote *kn, long hint); static int filt_machporttouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_machportprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); static unsigned filt_machportpeek(struct knote *kn); -struct filterops machport_filtops = { - .f_attach = filt_machportattach, - .f_detach = filt_machportdetach, - .f_event = filt_machport, - .f_touch = filt_machporttouch, - .f_process = filt_machportprocess, +SECURITY_READ_ONLY_EARLY(struct filterops) machport_filtops = { + .f_adjusts_qos = 1, + .f_attach = filt_machportattach, + .f_detach = filt_machportdetach, + .f_event = filt_machport, + .f_touch = filt_machporttouch, + .f_process = filt_machportprocess, .f_peek = filt_machportpeek, }; static int filt_machportattach( - struct knote *kn) + struct knote *kn, + __unused struct kevent_internal_s *kev) { mach_port_name_t name = (mach_port_name_t)kn->kn_kevent.ident; uint64_t wq_link_id = waitq_link_reserve(NULL); @@ -388,8 +391,10 @@ filt_machportattach( __IGNORE_WCASTALIGN(pset = (ipc_pset_t)entry->ie_object); mqueue = &pset->ips_messages; + ips_reference(pset); imq_lock(mqueue); + kn->kn_ptr.p_mqueue = mqueue; /* * Bind the portset wait queue directly to knote/kqueue. @@ -400,11 +405,15 @@ filt_machportattach( */ error = knote_link_waitq(kn, &mqueue->imq_wait_queue, &wq_link_id); if (!error) { - ips_reference(pset); - kn->kn_ptr.p_mqueue = mqueue; KNOTE_ATTACH(&mqueue->imq_klist, kn); + imq_unlock(mqueue); + + } + else { + kn->kn_ptr.p_mqueue = IMQ_NULL; + imq_unlock(mqueue); + ips_release(pset); } - imq_unlock(mqueue); is_read_unlock(space); @@ -432,8 +441,10 @@ filt_machportattach( kn->kn_ptr.p_mqueue = mqueue; KNOTE_ATTACH(&mqueue->imq_klist, kn); if ((first = ipc_kmsg_queue_first(&mqueue->imq_messages)) != IKM_NULL) { + int sync_qos_override_index = ipc_port_get_max_sync_qos_index(port); if (kn->kn_sfflags & MACH_RCV_MSG) - knote_adjust_qos(kn, first->ikm_qos, first->ikm_qos_override); + knote_adjust_qos(kn, first->ikm_qos, first->ikm_qos_override, + sync_qos_override_index); result = 1; } imq_unlock(mqueue); @@ -528,8 +539,12 @@ filt_machport( } else if (imq_is_valid(mqueue)) { assert(!imq_is_set(mqueue)); if ((first = ipc_kmsg_queue_first(&mqueue->imq_messages)) != IKM_NULL) { + ipc_port_t port = ip_from_mq(mqueue); + int sync_qos_override_index = ipc_port_get_max_sync_qos_index(port); + if (kn->kn_sfflags & MACH_RCV_MSG) - knote_adjust_qos(kn, first->ikm_qos, first->ikm_qos_override); + knote_adjust_qos(kn, first->ikm_qos, first->ikm_qos_override, + sync_qos_override_index); result = 1; } } @@ -564,13 +579,18 @@ filt_machporttouch( */ if (imq_is_valid(mqueue) && !imq_is_set(mqueue) && (first = ipc_kmsg_queue_first(&mqueue->imq_messages)) != IKM_NULL) { + ipc_port_t port = ip_from_mq(mqueue); + int sync_qos_override_index = ipc_port_get_max_sync_qos_index(port); + if (kn->kn_sfflags & MACH_RCV_MSG) - knote_adjust_qos(kn, first->ikm_qos, first->ikm_qos_override); + knote_adjust_qos(kn, first->ikm_qos, first->ikm_qos_override, + sync_qos_override_index); result = 1; } else if (kn->kn_sfflags & MACH_RCV_MSG) { knote_adjust_qos(kn, MACH_MSG_PRIORITY_UNSPECIFIED, - MACH_MSG_PRIORITY_UNSPECIFIED); + MACH_MSG_PRIORITY_UNSPECIFIED, + THREAD_QOS_UNSPECIFIED); } imq_unlock(mqueue); @@ -655,6 +675,7 @@ filt_machportprocess( self->ith_continuation = NULL; option |= MACH_RCV_TIMEOUT; // never wait self->ith_state = MACH_RCV_IN_PROGRESS; + self->ith_knote = kn; wresult = ipc_mqueue_receive_on_thread( mqueue, diff --git a/osfmk/ipc/ipc_right.c b/osfmk/ipc/ipc_right.c index 7a937f5be..04043b3ea 100644 --- a/osfmk/ipc/ipc_right.c +++ b/osfmk/ipc/ipc_right.c @@ -837,7 +837,7 @@ ipc_right_destroy( /* * Routine: ipc_right_dealloc * Purpose: - * Releases a send/send-once/dead-name user ref. + * Releases a send/send-once/dead-name/port_set user ref. * Like ipc_right_delta with a delta of -1, * but looks at the entry to determine the right. * Conditions: @@ -865,6 +865,26 @@ ipc_right_dealloc( assert(is_active(space)); switch (type) { + case MACH_PORT_TYPE_PORT_SET: { + ipc_pset_t pset; + + assert(IE_BITS_UREFS(bits) == 0); + assert(entry->ie_request == IE_REQ_NONE); + + pset = (ipc_pset_t) entry->ie_object; + assert(pset != IPS_NULL); + + entry->ie_object = IO_NULL; + ipc_entry_dealloc(space, name, entry); + + ips_lock(pset); + assert(ips_active(pset)); + is_write_unlock(space); + + ipc_pset_destroy(pset); /* consumes ref, unlocks */ + break; + } + case MACH_PORT_TYPE_DEAD_NAME: { dead_name: @@ -1412,8 +1432,12 @@ ipc_right_delta( break; } + case MACH_PORT_RIGHT_LABELH: + goto invalid_right; + default: - panic("ipc_right_delta: strange right"); + panic("ipc_right_delta: strange right %d for 0x%x (%p) in space:%p", + right, name, (void *)entry, (void *)space); } return KERN_SUCCESS; @@ -1721,6 +1745,9 @@ ipc_right_copyin_check( return FALSE; if (io_kotype(entry->ie_object) != IKOT_NONE) return FALSE; + port = (ipc_port_t) entry->ie_object; + if (port->ip_specialreply) + return FALSE; break; case MACH_MSG_TYPE_COPY_SEND: @@ -2517,8 +2544,13 @@ ipc_right_copyout( assert(IE_BITS_UREFS(bits) == 0); assert(port->ip_sorights > 0); - /* transfer send-once right and ref to entry */ - ip_unlock(port); + if (port->ip_specialreply) { + ipc_port_unlink_special_reply_port_locked(port, + current_thread()->ith_knote, IPC_PORT_UNLINK_SR_NONE); + /* port unlocked on return */ + } else { + ip_unlock(port); + } entry->ie_bits = bits | (MACH_PORT_TYPE_SEND_ONCE | 1); /* set urefs to 1 */ ipc_entry_modified(space, name, entry); @@ -2578,10 +2610,20 @@ ipc_right_copyout( case MACH_MSG_TYPE_PORT_RECEIVE: { ipc_port_t dest; + sync_qos_count_t max_sync_qos = THREAD_QOS_UNSPECIFIED; + sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0}; + sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0}; #if IMPORTANCE_INHERITANCE natural_t assertcnt = port->ip_impcount; #endif /* IMPORTANCE_INHERITANCE */ + /* Capture the sync qos count delta */ + for (int i = 0; i < THREAD_QOS_LAST; i++) { + sync_qos_delta_sub[i] = port_sync_qos(port, i); + if (sync_qos_delta_sub[i] != 0) { + max_sync_qos = i; + } + } assert(port->ip_mscount == 0); assert(port->ip_receiver_name == MACH_PORT_NULL); @@ -2614,6 +2656,11 @@ ipc_right_copyout( entry->ie_bits = bits | MACH_PORT_TYPE_RECEIVE; ipc_entry_modified(space, name, entry); + /* update the sync qos count on knote */ + if (ITH_KNOTE_VALID(current_thread()->ith_knote)) { + knote_adjust_sync_qos(current_thread()->ith_knote, max_sync_qos, TRUE); + } + if (dest != IP_NULL) { #if IMPORTANCE_INHERITANCE /* @@ -2626,6 +2673,8 @@ ipc_right_copyout( ipc_port_impcount_delta(dest, 0 - assertcnt, IP_NULL); ip_unlock(dest); #endif /* IMPORTANCE_INHERITANCE */ + /* Adjust the sync qos of destination */ + ipc_port_adjust_sync_qos(dest, sync_qos_delta_add, sync_qos_delta_sub); ip_release(dest); } break; diff --git a/osfmk/ipc/ipc_types.h b/osfmk/ipc/ipc_types.h index 26cd73edc..5523bcede 100644 --- a/osfmk/ipc/ipc_types.h +++ b/osfmk/ipc/ipc_types.h @@ -60,6 +60,7 @@ typedef struct ipc_table_size *ipc_table_size_t; typedef struct ipc_port_request *ipc_port_request_t; typedef struct ipc_pset *ipc_pset_t; typedef struct ipc_kmsg *ipc_kmsg_t; +typedef uint8_t sync_qos_count_t; #define IE_NULL ((ipc_entry_t) 0) diff --git a/osfmk/ipc/mach_kernelrpc.c b/osfmk/ipc/mach_kernelrpc.c index b24f4d5dd..b4ee58fec 100644 --- a/osfmk/ipc/mach_kernelrpc.c +++ b/osfmk/ipc/mach_kernelrpc.c @@ -51,7 +51,7 @@ _kernelrpc_mach_vm_allocate_trap(struct _kernelrpc_mach_vm_allocate_trap_args *a if (copyin(args->addr, (char *)&addr, sizeof (addr))) goto done; - rv = mach_vm_allocate(task->map, &addr, args->size, args->flags); + rv = mach_vm_allocate_external(task->map, &addr, args->size, args->flags); if (rv == KERN_SUCCESS) rv = copyout(&addr, args->addr, sizeof (addr)); @@ -109,7 +109,7 @@ _kernelrpc_mach_vm_map_trap(struct _kernelrpc_mach_vm_map_trap_args *args) if (copyin(args->addr, (char *)&addr, sizeof (addr))) goto done; - rv = mach_vm_map(task->map, &addr, args->size, args->mask, args->flags, + rv = mach_vm_map_external(task->map, &addr, args->size, args->mask, args->flags, IPC_PORT_NULL, 0, FALSE, args->cur_protection, VM_PROT_ALL, VM_INHERIT_DEFAULT); if (rv == KERN_SUCCESS) @@ -400,7 +400,7 @@ host_create_mach_voucher_trap(struct host_create_mach_voucher_args *args) if (args->recipes_size < MACH_VOUCHER_TRAP_STACK_LIMIT) { /* keep small recipes on the stack for speed */ uint8_t krecipes[args->recipes_size]; - if (copyin(args->recipes, (void *)krecipes, args->recipes_size)) { + if (copyin(CAST_USER_ADDR_T(args->recipes), (void *)krecipes, args->recipes_size)) { kr = KERN_MEMORY_ERROR; goto done; } @@ -412,7 +412,7 @@ host_create_mach_voucher_trap(struct host_create_mach_voucher_args *args) goto done; } - if (copyin(args->recipes, (void *)krecipes, args->recipes_size)) { + if (copyin(CAST_USER_ADDR_T(args->recipes), (void *)krecipes, args->recipes_size)) { kfree(krecipes, (vm_size_t)args->recipes_size); kr = KERN_MEMORY_ERROR; goto done; @@ -455,7 +455,7 @@ mach_voucher_extract_attr_recipe_trap(struct mach_voucher_extract_attr_recipe_ar if (sz < MACH_VOUCHER_TRAP_STACK_LIMIT) { /* keep small recipes on the stack for speed */ uint8_t krecipe[sz]; - if (copyin(args->recipe, (void *)krecipe, sz)) { + if (copyin(CAST_USER_ADDR_T(args->recipe), (void *)krecipe, sz)) { kr = KERN_MEMORY_ERROR; goto done; } @@ -464,7 +464,7 @@ mach_voucher_extract_attr_recipe_trap(struct mach_voucher_extract_attr_recipe_ar assert(sz <= max_sz); if (kr == KERN_SUCCESS && sz > 0) - kr = copyout(krecipe, (void *)args->recipe, sz); + kr = copyout(krecipe, CAST_USER_ADDR_T(args->recipe), sz); } else { uint8_t *krecipe = kalloc((vm_size_t)max_sz); if (!krecipe) { @@ -472,7 +472,7 @@ mach_voucher_extract_attr_recipe_trap(struct mach_voucher_extract_attr_recipe_ar goto done; } - if (copyin(args->recipe, (void *)krecipe, sz)) { + if (copyin(CAST_USER_ADDR_T(args->recipe), (void *)krecipe, sz)) { kfree(krecipe, (vm_size_t)max_sz); kr = KERN_MEMORY_ERROR; goto done; @@ -483,7 +483,7 @@ mach_voucher_extract_attr_recipe_trap(struct mach_voucher_extract_attr_recipe_ar assert(sz <= max_sz); if (kr == KERN_SUCCESS && sz > 0) - kr = copyout(krecipe, (void *)args->recipe, sz); + kr = copyout(krecipe, CAST_USER_ADDR_T(args->recipe), sz); kfree(krecipe, (vm_size_t)max_sz); } diff --git a/osfmk/ipc/mach_msg.c b/osfmk/ipc/mach_msg.c index 6a5241f71..128cd9605 100644 --- a/osfmk/ipc/mach_msg.c +++ b/osfmk/ipc/mach_msg.c @@ -91,6 +91,7 @@ #include #include #include +#include #include @@ -145,6 +146,14 @@ mach_msg_return_t msg_receive_error( ipc_space_t space, mach_msg_size_t *out_size); +static mach_msg_return_t +mach_msg_rcv_link_special_reply_port( + ipc_port_t special_reply_port, + mach_port_name_t dest_name_port); + +static void +mach_msg_rcv_unlink_special_reply_port(void); + security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE; audit_token_t KERNEL_AUDIT_TOKEN = KERNEL_AUDIT_TOKEN_VALUE; @@ -196,7 +205,9 @@ mach_msg_send( mach_msg_size_t msg_and_trailer_size; mach_msg_max_trailer_t *trailer; - if ((send_size < sizeof(mach_msg_header_t)) || (send_size & 3)) + if ((send_size & 3) || + send_size < sizeof(mach_msg_header_t) || + (send_size < sizeof(mach_msg_base_t) && (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX))) return MACH_SEND_MSG_TOO_SMALL; if (send_size > MACH_MSG_SIZE_MAX - MAX_TRAILER_SIZE) @@ -309,6 +320,8 @@ mach_msg_receive_results( mach_msg_trailer_size_t trailer_size; mach_msg_size_t size = 0; + /* unlink the special_reply_port before releasing reference to object */ + mach_msg_rcv_unlink_special_reply_port(); io_release(object); if (mr != MACH_MSG_SUCCESS) { @@ -475,6 +488,7 @@ mach_msg_receive( self->ith_msize = 0; self->ith_option = option; self->ith_continuation = continuation; + self->ith_knote = ITH_KNOTE_NULL; ipc_mqueue_receive(mqueue, option, rcv_size, rcv_timeout, THREAD_ABORTSAFE); if ((option & MACH_RCV_TIMEOUT) && rcv_timeout == 0) @@ -576,6 +590,18 @@ mach_msg_overwrite_trap( } /* hold ref for object */ + if ((option & MACH_RCV_SYNC_WAIT) && !(option & MACH_SEND_SYNC_OVERRIDE)) { + ipc_port_t special_reply_port; + __IGNORE_WCASTALIGN(special_reply_port = (ipc_port_t) object); + /* link the special reply port to the destination */ + mr = mach_msg_rcv_link_special_reply_port(special_reply_port, + (mach_port_name_t)override); + if (mr != MACH_MSG_SUCCESS) { + io_release(object); + return mr; + } + } + if (rcv_msg_addr != (mach_vm_address_t)0) self->ith_msg_addr = rcv_msg_addr; else @@ -586,6 +612,7 @@ mach_msg_overwrite_trap( self->ith_option = option; self->ith_receiver_name = MACH_PORT_NULL; self->ith_continuation = thread_syscall_return; + self->ith_knote = ITH_KNOTE_NULL; ipc_mqueue_receive(mqueue, option, rcv_size, msg_timeout, THREAD_ABORTSAFE); if ((option & MACH_RCV_TIMEOUT) && msg_timeout == 0) @@ -596,6 +623,82 @@ mach_msg_overwrite_trap( return MACH_MSG_SUCCESS; } +/* + * Routine: mach_msg_rcv_link_special_reply_port + * Purpose: + * Link the special reply port(rcv right) to the + * other end of the sync ipc channel. + * Conditions: + * Nothing locked. + * Returns: + * None. + */ +static mach_msg_return_t +mach_msg_rcv_link_special_reply_port( + ipc_port_t special_reply_port, + mach_port_name_t dest_name_port) +{ + ipc_port_t dest_port = IP_NULL; + kern_return_t kr; + int qos; + + if (current_thread()->ith_special_reply_port != special_reply_port) { + return MACH_RCV_INVALID_NOTIFY; + } + + /* Copyin the destination port */ + if (!MACH_PORT_VALID(dest_name_port)) { + return MACH_RCV_INVALID_NOTIFY; + } + + kr = ipc_object_copyin(current_space(), + dest_name_port, MACH_MSG_TYPE_COPY_SEND, + (ipc_object_t *) &dest_port); + + /* + * The receive right of dest port might have gone away, + * do not fail the receive in that case. + */ + if (kr == KERN_SUCCESS && IP_VALID(dest_port)) { + + /* Get the effective qos of the thread */ + qos = proc_get_effective_thread_policy(current_thread(), TASK_POLICY_QOS); + + ipc_port_link_special_reply_port_with_qos(special_reply_port, + dest_port, qos); + + /* release the send right */ + ipc_port_release_send(dest_port); + } + return MACH_MSG_SUCCESS; +} + +/* + * Routine: mach_msg_rcv_unlink_special_reply_port + * Purpose: + * Unlink the special reply port to the other end + * of the sync ipc channel. + * Condition: + * Nothing locked. + * Returns: + * None. + */ +static void +mach_msg_rcv_unlink_special_reply_port(void) +{ + thread_t self = current_thread(); + ipc_port_t special_reply_port = self->ith_special_reply_port; + mach_msg_option_t option = self->ith_option; + + if ((special_reply_port == IP_NULL) || + !(option & MACH_RCV_SYNC_WAIT)) { + return; + } + + ipc_port_unlink_special_reply_port(special_reply_port, + IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE); +} + /* * Routine: mach_msg_trap [mach trap] * Purpose: diff --git a/osfmk/ipc/mach_port.c b/osfmk/ipc/mach_port.c index 851a6a1ee..ced4e6384 100644 --- a/osfmk/ipc/mach_port.c +++ b/osfmk/ipc/mach_port.c @@ -71,7 +71,6 @@ */ #include -#include #include #include @@ -84,6 +83,7 @@ #include #include #include +#include #include #include #include @@ -288,11 +288,11 @@ mach_port_names( } size = size_needed; - kr = vm_allocate(ipc_kernel_map, &addr1, size, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)); + kr = vm_allocate_kernel(ipc_kernel_map, &addr1, size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC); if (kr != KERN_SUCCESS) return KERN_RESOURCE_SHORTAGE; - kr = vm_allocate(ipc_kernel_map, &addr2, size, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)); + kr = vm_allocate_kernel(ipc_kernel_map, &addr2, size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC); if (kr != KERN_SUCCESS) { kmem_free(ipc_kernel_map, addr1, size); return KERN_RESOURCE_SHORTAGE; @@ -300,13 +300,13 @@ mach_port_names( /* can't fault while we hold locks */ - kr = vm_map_wire( + kr = vm_map_wire_kernel( ipc_kernel_map, vm_map_trunc_page(addr1, VM_MAP_PAGE_MASK(ipc_kernel_map)), vm_map_round_page(addr1 + size, VM_MAP_PAGE_MASK(ipc_kernel_map)), - VM_PROT_READ|VM_PROT_WRITE|VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_IPC), + VM_PROT_READ|VM_PROT_WRITE, VM_KERN_MEMORY_IPC, FALSE); if (kr != KERN_SUCCESS) { kmem_free(ipc_kernel_map, addr1, size); @@ -314,13 +314,14 @@ mach_port_names( return KERN_RESOURCE_SHORTAGE; } - kr = vm_map_wire( + kr = vm_map_wire_kernel( ipc_kernel_map, vm_map_trunc_page(addr2, VM_MAP_PAGE_MASK(ipc_kernel_map)), vm_map_round_page(addr2 + size, VM_MAP_PAGE_MASK(ipc_kernel_map)), - VM_PROT_READ|VM_PROT_WRITE|VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_IPC), + VM_PROT_READ|VM_PROT_WRITE, + VM_KERN_MEMORY_IPC, FALSE); if (kr != KERN_SUCCESS) { kmem_free(ipc_kernel_map, addr1, size); @@ -774,8 +775,8 @@ mach_port_destroy( * Routine: mach_port_deallocate [kernel call] * Purpose: * Deallocates a user reference from a send right, - * send-once right, or a dead-name right. May - * deallocate the right, if this is the last uref, + * send-once right, dead-name right or a port_set right. + * May deallocate the right, if this is the last uref, * and destroy the name, if it doesn't denote * other rights. * Conditions: @@ -1253,14 +1254,14 @@ mach_port_get_set_status( ipc_object_t psobj; ipc_pset_t pset; - kr = vm_allocate(ipc_kernel_map, &addr, size, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)); + kr = vm_allocate_kernel(ipc_kernel_map, &addr, size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC); if (kr != KERN_SUCCESS) return KERN_RESOURCE_SHORTAGE; /* can't fault while we hold locks */ - kr = vm_map_wire(ipc_kernel_map, addr, addr + size, - VM_PROT_READ|VM_PROT_WRITE|VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_IPC), FALSE); + kr = vm_map_wire_kernel(ipc_kernel_map, addr, addr + size, + VM_PROT_READ|VM_PROT_WRITE, VM_KERN_MEMORY_IPC, FALSE); assert(kr == KERN_SUCCESS); kr = ipc_object_translate(space, name, MACH_PORT_RIGHT_PORT_SET, &psobj); @@ -1930,9 +1931,10 @@ mach_port_set_attributes( /* * don't allow temp-owner importance donation if user - * associated it with a kobject already (timer, host_notify target). + * associated it with a kobject already (timer, host_notify target), + * or is a special reply port. */ - if (is_ipc_kobject(ip_kotype(port))) { + if (is_ipc_kobject(ip_kotype(port)) || port->ip_specialreply) { ip_unlock(port); return KERN_INVALID_ARGUMENT; } @@ -1984,9 +1986,10 @@ mach_port_set_attributes( /* * don't allow importance donation if user associated - * it with a kobject already (timer, host_notify target). + * it with a kobject already (timer, host_notify target), + * or is a special reply port. */ - if (is_ipc_kobject(ip_kotype(port))) { + if (is_ipc_kobject(ip_kotype(port)) || port->ip_specialreply) { ip_unlock(port); return KERN_INVALID_ARGUMENT; } @@ -2228,36 +2231,13 @@ mach_port_guard_exception( uint64_t portguard, unsigned reason) { + mach_exception_code_t code = 0; + EXC_GUARD_ENCODE_TYPE(code, GUARD_TYPE_MACH_PORT); + EXC_GUARD_ENCODE_FLAVOR(code, reason); + EXC_GUARD_ENCODE_TARGET(code, name); + mach_exception_subcode_t subcode = (uint64_t)portguard; thread_t t = current_thread(); - uint64_t code, subcode; - - /* - * EXC_GUARD namespace for mach ports - * - * - * Mach Port guards use the exception codes like - * - * code: - * +----------------------------------------------------------------+ - * |[63:61] GUARD_TYPE_MACH_PORT | [60:32] flavor | [31:0] port name| - * +----------------------------------------------------------------+ - * - * subcode: - * +----------------------------------------------------------------+ - * | [63:0] guard value | - * +----------------------------------------------------------------+ - */ - - code = (((uint64_t)GUARD_TYPE_MACH_PORT) << 61) | - (((uint64_t)reason) << 32) | - ((uint64_t)name); - subcode = (uint64_t)(portguard); - - t->guard_exc_info.code = code; - t->guard_exc_info.subcode = subcode; - - /* Mark thread with AST_GUARD */ - thread_guard_violation(t, GUARD_TYPE_MACH_PORT); + thread_guard_violation(t, code, subcode); return KERN_FAILURE; } @@ -2273,14 +2253,16 @@ mach_port_guard_exception( */ void -mach_port_guard_ast(thread_t t) +mach_port_guard_ast(thread_t __unused t, + mach_exception_data_type_t code, mach_exception_data_type_t subcode) { + assert(t->task != kernel_task); + /* Raise an EXC_GUARD exception */ - task_exception_notify(EXC_GUARD, t->guard_exc_info.code, t->guard_exc_info.subcode); + task_exception_notify(EXC_GUARD, code, subcode); /* Terminate task which caused the exception */ task_bsdtask_kill(current_task()); - return; } /* diff --git a/osfmk/kdp/Makefile b/osfmk/kdp/Makefile index 7c1535007..c35f281c0 100644 --- a/osfmk/kdp/Makefile +++ b/osfmk/kdp/Makefile @@ -10,11 +10,13 @@ DATAFILES = \ kdp_callout.h \ kdp_en_debugger.h +PRIVATE_KERNELFILES = processor_core.h + EXPORT_MI_LIST = ${DATAFILES} kdp_dyld.h INSTALL_KF_MI_LIST = ${DATAFILES} -INSTALL_KF_MI_LCL_LIST = ${DATAFILES} +INSTALL_KF_MI_LCL_LIST = ${DATAFILES} ${PRIVATE_KERNELFILES} EXPORT_MI_DIR = kdp diff --git a/osfmk/kdp/kdp.c b/osfmk/kdp/kdp.c index ee2e9f8d8..6cb2ee5a8 100644 --- a/osfmk/kdp/kdp.c +++ b/osfmk/kdp/kdp.c @@ -119,7 +119,6 @@ static unsigned int breakpoints_initialized = 0; int reattach_wait = 0; int noresume_on_disconnect = 0; -extern unsigned int return_on_panic; kdp_error_t kdp_set_breakpoint_internal( @@ -276,7 +275,7 @@ kdp_disconnect( kdp.exception_seq = kdp.conn_seq = 0; kdp.session_key = 0; - if ((panicstr != NULL) && (return_on_panic == 0)) + if (debugger_panic_str != NULL) reattach_wait = 1; if (noresume_on_disconnect == 1) { diff --git a/osfmk/kdp/kdp_core.c b/osfmk/kdp/kdp_core.c index 02febf2f1..186e2e7f0 100644 --- a/osfmk/kdp/kdp_core.c +++ b/osfmk/kdp/kdp_core.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -41,20 +41,32 @@ #include #include #include +#include #include #include #include #include +#include -#if defined(__i386__) || defined(__x86_64__) +#if defined(__x86_64__) #include #include -#endif /* defined(__i386__) || defined(__x86_64__) */ +#include +#endif /* defined(__x86_64__) */ - -#if WITH_CONSISTENT_DBG +#if CONFIG_EMBEDDED +#include +#include #include -#endif /* WITH_CONSISTENT_DBG */ + +#if !defined(ROUNDUP) +#define ROUNDUP(a, b) (((a) + ((b) - 1)) & (~((b) - 1))) +#endif + +#if !defined(ROUNDDOWN) +#define ROUNDDOWN(a, b) ((a) & ~((b) - 1)) +#endif +#endif /* CONFIG_EMBEDDED */ typedef int (*pmap_traverse_callback)(vm_map_offset_t start, vm_map_offset_t end, @@ -66,15 +78,21 @@ extern int pmap_traverse_present_mappings(pmap_t pmap, pmap_traverse_callback callback, void *context); +static int kern_dump_save_summary(void *refcon, core_save_summary_cb callback, void *context); +static int kern_dump_save_seg_descriptions(void *refcon, core_save_segment_descriptions_cb callback, void *context); +static int kern_dump_save_thread_state(void *refcon, void *buf, core_save_thread_state_cb callback, void *context); +static int kern_dump_save_sw_vers(void *refcon, core_save_sw_vers_cb callback, void *context); +static int kern_dump_save_segment_data(void *refcon, core_save_segment_data_cb callback, void *context); static int kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start, vm_map_offset_t end, void *context); static int -kern_dump_pmap_traverse_send_seg_callback(vm_map_offset_t start, - vm_map_offset_t end, - void *context); +kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start, + vm_map_offset_t end, + void *context); + static int kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start, vm_map_offset_t end, @@ -87,7 +105,7 @@ typedef int (*kern_dump_output_proc)(unsigned int request, char *corename, struct kdp_core_out_vars { kern_dump_output_proc outproc; - z_output_func zoutput; + z_output_func zoutput; size_t zipped; uint64_t totalbytes; uint64_t lastpercent; @@ -98,22 +116,6 @@ struct kdp_core_out_vars Bytef * outbuf; }; -struct kern_dump_preflight_context -{ - uint32_t region_count; - uint64_t dumpable_bytes; -}; - -struct kern_dump_send_context -{ - struct kdp_core_out_vars * outvars; - uint64_t hoffset; - uint64_t foffset; - uint64_t header_size; - uint64_t dumpable_bytes; - uint32_t region_count; -}; - extern uint32_t kdp_crashdump_pkt_size; static vm_offset_t kdp_core_zmem; @@ -123,7 +125,7 @@ static z_stream kdp_core_zs; static uint64_t kdp_core_total_size; static uint64_t kdp_core_total_size_sent_uncomp; -#if WITH_CONSISTENT_DBG +#if CONFIG_EMBEDDED struct xnu_hw_shmem_dbg_command_info *hwsd_info = NULL; #define KDP_CORE_HW_SHMEM_DBG_NUM_BUFFERS 2 @@ -158,7 +160,11 @@ static uint64_t kdp_hw_shmem_dbg_contact_deadline = 0; static uint64_t kdp_hw_shmem_dbg_contact_deadline_interval = 0; #define KDP_HW_SHMEM_DBG_TIMEOUT_DEADLINE_SECS 30 -#endif /* WITH_CONSISTENT_DBG */ +#endif /* CONFIG_EMBEDDED */ + +static boolean_t kern_dump_successful = FALSE; + +struct mach_core_fileheader kdp_core_header = { }; /* * These variables will be modified by the BSD layer if the root device is @@ -167,14 +173,12 @@ static uint64_t kdp_hw_shmem_dbg_contact_deadline_interval = 0; uint64_t kdp_core_ramdisk_addr = 0; uint64_t kdp_core_ramdisk_size = 0; -#define DEBG kdb_printf - boolean_t kdp_has_polled_corefile(void) { return (NULL != gIOPolledCoreFileVars); } -#if WITH_CONSISTENT_DBG +#if CONFIG_EMBEDDED /* * Whenever we start a coredump, make sure the buffers * are all on the free queue and the state is as expected. @@ -234,11 +238,11 @@ kern_dump_hw_shmem_dbg_process_buffers() { FlushPoC_DcacheRegion((vm_offset_t) hwsd_info, sizeof(*hwsd_info)); if (hwsd_info->xhsdci_status == XHSDCI_COREDUMP_ERROR) { - kdb_printf("Detected remote error, terminating...\n"); + kern_coredump_log(NULL, "Detected remote error, terminating...\n"); return -1; } else if (hwsd_info->xhsdci_status == XHSDCI_COREDUMP_BUF_EMPTY) { if (hwsd_info->xhsdci_seq_no != (kdp_hw_shmem_dbg_seq_no + 1)) { - kdb_printf("Detected stale/invalid seq num. Expected: %d, received %d\n", + kern_coredump_log(NULL, "Detected stale/invalid seq num. Expected: %d, received %d\n", (kdp_hw_shmem_dbg_seq_no + 1), hwsd_info->xhsdci_seq_no); hwsd_info->xhsdci_status = XHSDCI_COREDUMP_ERROR; FlushPoC_DcacheRegion((vm_offset_t) hwsd_info, sizeof(*hwsd_info)); @@ -272,8 +276,8 @@ kern_dump_hw_shmem_dbg_process_buffers() return 0; } else if (mach_absolute_time() > kdp_hw_shmem_dbg_contact_deadline) { - kdb_printf("Kernel timed out waiting for hardware debugger to update handshake structure."); - kdb_printf(" No contact in %d seconds\n", KDP_HW_SHMEM_DBG_TIMEOUT_DEADLINE_SECS); + kern_coredump_log(NULL, "Kernel timed out waiting for hardware debugger to update handshake structure."); + kern_coredump_log(NULL, "No contact in %d seconds\n", KDP_HW_SHMEM_DBG_TIMEOUT_DEADLINE_SECS); hwsd_info->xhsdci_status = XHSDCI_COREDUMP_ERROR; FlushPoC_DcacheRegion((vm_offset_t) hwsd_info, sizeof(*hwsd_info)); @@ -346,14 +350,14 @@ kern_dump_hw_shmem_dbg_buffer_proc(unsigned int request, __unused char *corename * the sequence number to still match the last we saw. */ if (hwsd_info->xhsdci_seq_no < kdp_hw_shmem_dbg_seq_no) { - kdb_printf("EOF Flush: Detected stale/invalid seq num. Expected: %d, received %d\n", + kern_coredump_log(NULL, "EOF Flush: Detected stale/invalid seq num. Expected: %d, received %d\n", kdp_hw_shmem_dbg_seq_no, hwsd_info->xhsdci_seq_no); return -1; } kdp_hw_shmem_dbg_seq_no = hwsd_info->xhsdci_seq_no; - kdb_printf("Setting coredump status as done!\n"); + kern_coredump_log(NULL, "Setting coredump status as done!\n"); hwsd_info->xhsdci_seq_no = ++kdp_hw_shmem_dbg_seq_no; hwsd_info->xhsdci_status = XHSDCI_COREDUMP_STATUS_DONE; FlushPoC_DcacheRegion((vm_offset_t) hwsd_info, sizeof(*hwsd_info)); @@ -421,7 +425,7 @@ kern_dump_hw_shmem_dbg_buffer_proc(unsigned int request, __unused char *corename return ret; } -#endif /* WITH_CONSISTENT_DBG */ +#endif /* CONFIG_EMBEDDED */ static IOReturn kern_dump_disk_proc(unsigned int request, __unused char *corename, @@ -434,27 +438,57 @@ kern_dump_disk_proc(unsigned int request, __unused char *corename, { case KDP_WRQ: err = IOPolledFileSeek(gIOPolledCoreFileVars, 0); - if (kIOReturnSuccess != err) break; + if (kIOReturnSuccess != err) { + kern_coredump_log(NULL, "IOPolledFileSeek(gIOPolledCoreFileVars, 0) returned 0x%x\n", err); + break; + } err = IOPolledFilePollersOpen(gIOPolledCoreFileVars, kIOPolledBeforeSleepState, false); break; case KDP_SEEK: noffset = *((uint64_t *) data); err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL); - if (kIOReturnSuccess != err) break; + if (kIOReturnSuccess != err) { + kern_coredump_log(NULL, "IOPolledFileWrite (during seek) returned 0x%x\n", err); + break; + } err = IOPolledFileSeek(gIOPolledCoreFileVars, noffset); + if (kIOReturnSuccess != err) { + kern_coredump_log(NULL, "IOPolledFileSeek(0x%llx) returned 0x%x\n", noffset, err); + } break; case KDP_DATA: err = IOPolledFileWrite(gIOPolledCoreFileVars, data, length, NULL); - if (kIOReturnSuccess != err) break; + if (kIOReturnSuccess != err) { + kern_coredump_log(NULL, "IOPolledFileWrite(gIOPolledCoreFileVars, 0x%p, 0x%llx, NULL) returned 0x%x\n", + data, length, err); + break; + } + break; + +#if CONFIG_EMBEDDED + /* Only supported on embedded by the underlying polled mode driver */ + case KDP_FLUSH: + err = IOPolledFileFlush(gIOPolledCoreFileVars); + if (kIOReturnSuccess != err) { + kern_coredump_log(NULL, "IOPolledFileFlush() returned 0x%x\n", err); + break; + } break; +#endif case KDP_EOF: err = IOPolledFileWrite(gIOPolledCoreFileVars, 0, 0, NULL); - if (kIOReturnSuccess != err) break; + if (kIOReturnSuccess != err) { + kern_coredump_log(NULL, "IOPolledFileWrite (during EOF) returned 0x%x\n", err); + break; + } err = IOPolledFilePollersClose(gIOPolledCoreFileVars, kIOPolledBeforeSleepState); - if (kIOReturnSuccess != err) break; + if (kIOReturnSuccess != err) { + kern_coredump_log(NULL, "IOPolledFilePollersClose (during EOF) returned 0x%x\n", err); + break; + } break; } @@ -476,10 +510,11 @@ kdp_core_zoutput(z_streamp strm, Bytef *buf, unsigned len) { if ((ret = (*vars->outproc)(KDP_DATA, NULL, len, buf)) != kIOReturnSuccess) { - DEBG("KDP_DATA(0x%x)\n", ret); + kern_coredump_log(NULL, "(kdp_core_zoutput) outproc(KDP_DATA, NULL, 0x%x, 0x%p) returned 0x%x\n", + len, buf, ret); vars->error = ret; } - if (!buf && !len) DEBG("100.."); + if (!buf && !len) kern_coredump_log(NULL, "100.."); } return (len); } @@ -518,12 +553,13 @@ kdp_core_zoutputbuf(z_streamp strm, Bytef *inbuf, unsigned inlen) vars->outlen - vars->outremain, vars->outbuf)) != kIOReturnSuccess) { - DEBG("KDP_DATA(0x%x)\n", ret); + kern_coredump_log(NULL, "(kdp_core_zoutputbuf) outproc(KDP_DATA, NULL, 0x%x, 0x%p) returned 0x%x\n", + (vars->outlen - vars->outremain), vars->outbuf, ret); vars->error = ret; } if (flush) { - DEBG("100.."); + kern_coredump_log(NULL, "100.."); flush = false; } vars->outremain = vars->outlen; @@ -559,7 +595,7 @@ kdp_core_zinput(z_streamp strm, Bytef *buf, unsigned size) if ((percent - vars->lastpercent) >= 10) { vars->lastpercent = percent; - DEBG("%lld..\n", percent); + kern_coredump_log(NULL, "%lld..\n", percent); } } @@ -605,7 +641,7 @@ kdp_core_stream_output_chunk(struct kdp_core_out_vars * vars, unsigned length, v if (Z_STREAM_END == zr) break; if (zr != Z_OK) { - DEBG("ZERR %d\n", zr); + kern_coredump_log(NULL, "ZERR %d\n", zr); vars->error = zr; } } @@ -616,12 +652,13 @@ kdp_core_stream_output_chunk(struct kdp_core_out_vars * vars, unsigned length, v return (vars->error); } -static IOReturn -kdp_core_stream_output(struct kdp_core_out_vars * vars, uint64_t length, void * data) +kern_return_t +kdp_core_output(void *kdp_core_out_vars, uint64_t length, void * data) { IOReturn err; unsigned int chunk; enum { kMaxZLibChunk = 1024*1024*1024 }; + struct kdp_core_out_vars *vars = (struct kdp_core_out_vars *)kdp_core_out_vars; do { @@ -637,6 +674,10 @@ kdp_core_stream_output(struct kdp_core_out_vars * vars, uint64_t length, void * return (err); } +#if defined(__arm__) || defined(__arm64__) +extern pmap_paddr_t avail_start, avail_end; +extern struct vm_object pmap_object_store; +#endif extern vm_offset_t c_buffers; extern vm_size_t c_buffers_size; @@ -667,6 +708,24 @@ kernel_pmap_present_mapping(uint64_t vaddr, uint64_t * pvincr, uintptr_t * pvphy vincr = kdp_core_ramdisk_size; } else +#if defined(__arm64__) + if (vaddr == _COMM_PAGE64_BASE_ADDRESS) + { + /* not readable */ + ppn = 0; + vincr = _COMM_PAGE_AREA_LENGTH; + } + else +#endif /* defined(__arm64__) */ +#if defined(__arm__) || defined(__arm64__) + if (vaddr == phystokv(avail_start)) + { + /* physical memory map */ + ppn = 0; + vincr = (avail_end - avail_start); + } + else +#endif /* defined(__arm__) || defined(__arm64__) */ ppn = pmap_find_phys(kernel_pmap, vaddr); *pvincr = round_page_64(vincr); @@ -674,7 +733,11 @@ kernel_pmap_present_mapping(uint64_t vaddr, uint64_t * pvincr, uintptr_t * pvphy if (ppn && pvphysaddr) { uint64_t phys = ptoa_64(ppn); - if (physmap_enclosed(phys)) *pvphysaddr = PHYSMAP_PTOV(phys); +#if defined(__arm__) || defined(__arm64__) + if (isphysmem(phys)) *pvphysaddr = phystokv(phys); +#else + if (physmap_enclosed(phys)) *pvphysaddr = (uintptr_t)PHYSMAP_PTOV(phys); +#endif else ppn = 0; } @@ -690,15 +753,18 @@ pmap_traverse_present_mappings(pmap_t __unused pmap, { IOReturn ret; vm_map_offset_t vcurstart, vcur; - uint64_t vincr; + uint64_t vincr = 0; vm_map_offset_t debug_start; vm_map_offset_t debug_end; boolean_t lastvavalid; +#if defined(__arm__) || defined(__arm64__) + vm_page_t m = VM_PAGE_NULL; +#endif - debug_start = trunc_page((vm_map_offset_t) debug_buf_addr); - debug_end = round_page((vm_map_offset_t) (debug_buf_addr + debug_buf_size)); + debug_start = trunc_page((vm_map_offset_t) debug_buf_base); + debug_end = round_page((vm_map_offset_t) (debug_buf_base + debug_buf_size)); -#if defined(__i386__) || defined(__x86_64__) +#if defined(__x86_64__) assert(!is_ept_pmap(pmap)); #endif @@ -709,13 +775,53 @@ pmap_traverse_present_mappings(pmap_t __unused pmap, ret = KERN_SUCCESS; lastvavalid = FALSE; for (vcur = vcurstart = start; (ret == KERN_SUCCESS) && (vcur < end); ) { - ppnum_t ppn; + ppnum_t ppn = 0; + +#if defined(__arm__) || defined(__arm64__) + /* We're at the start of the physmap, so pull out the pagetable pages that + * are accessed through that region.*/ + if (vcur == phystokv(avail_start) && vm_object_lock_try_shared(&pmap_object_store)) + m = (vm_page_t)vm_page_queue_first(&pmap_object_store.memq); + if (m != VM_PAGE_NULL) + { + vm_map_offset_t vprev = vcur; + ppn = (ppnum_t)atop(avail_end); + while (!vm_page_queue_end(&pmap_object_store.memq, (vm_page_queue_entry_t)m)) + { + /* Ignore pages that come from the static region and have already been dumped.*/ + if (VM_PAGE_GET_PHYS_PAGE(m) >= atop(avail_start)) + { + ppn = VM_PAGE_GET_PHYS_PAGE(m); + break; + } + m = (vm_page_t)vm_page_queue_next(&m->listq); + } + vcur = phystokv(ptoa(ppn)); + if (vcur != vprev) + { + ret = callback(vcurstart, vprev, context); + lastvavalid = FALSE; + } + vincr = PAGE_SIZE_64; + if (ppn == atop(avail_end)) + { + vm_object_unlock(&pmap_object_store); + m = VM_PAGE_NULL; + } + else + m = (vm_page_t)vm_page_queue_next(&m->listq); + } + if (m == VM_PAGE_NULL) + ppn = kernel_pmap_present_mapping(vcur, &vincr, NULL); +#else /* defined(__arm__) || defined(__arm64__) */ ppn = kernel_pmap_present_mapping(vcur, &vincr, NULL); +#endif if (ppn != 0) { if (((vcur < debug_start) || (vcur >= debug_end)) - && !pmap_valid_page(ppn)) + && !(EFI_VALID_PAGE(ppn) || + pmap_valid_page(ppn))) { /* not something we want */ ppn = 0; @@ -735,7 +841,7 @@ pmap_traverse_present_mappings(pmap_t __unused pmap, lastvavalid = FALSE; } -#if defined(__i386__) || defined(__x86_64__) +#if defined(__x86_64__) /* Try to skip by 2MB if possible */ if (((vcur & PDMASK) == 0) && cpu_64bit) { pd_entry_t *pde; @@ -747,7 +853,7 @@ pmap_traverse_present_mappings(pmap_t __unused pmap, } } } -#endif /* defined(__i386__) || defined(__x86_64__) */ +#endif /* defined(__x86_64__) */ } vcur += vincr; } @@ -759,409 +865,522 @@ pmap_traverse_present_mappings(pmap_t __unused pmap, return (ret); } +struct kern_dump_preflight_context +{ + uint32_t region_count; + uint64_t dumpable_bytes; +}; + int kern_dump_pmap_traverse_preflight_callback(vm_map_offset_t start, vm_map_offset_t end, void *context) { - struct kern_dump_preflight_context *kdc = (struct kern_dump_preflight_context *)context; - IOReturn ret = KERN_SUCCESS; + struct kern_dump_preflight_context *kdc = (struct kern_dump_preflight_context *)context; + IOReturn ret = KERN_SUCCESS; - kdc->region_count++; - kdc->dumpable_bytes += (end - start); + kdc->region_count++; + kdc->dumpable_bytes += (end - start); - return (ret); + return (ret); } -int -kern_dump_pmap_traverse_send_seg_callback(vm_map_offset_t start, - vm_map_offset_t end, - void *context) -{ - struct kern_dump_send_context *kdc = (struct kern_dump_send_context *)context; - IOReturn ret = KERN_SUCCESS; - kernel_segment_command_t sc; - vm_size_t size = (vm_size_t)(end - start); - if (kdc->hoffset + sizeof(sc) > kdc->header_size) { - return (KERN_NO_SPACE); - } +struct kern_dump_send_seg_desc_context +{ + core_save_segment_descriptions_cb callback; + void *context; +}; - kdc->region_count++; - kdc->dumpable_bytes += (end - start); - - /* - * Fill in segment command structure. - */ - - sc.cmd = LC_SEGMENT_KERNEL; - sc.cmdsize = sizeof(kernel_segment_command_t); - sc.segname[0] = 0; - sc.vmaddr = (vm_address_t)start; - sc.vmsize = size; - sc.fileoff = (vm_address_t)kdc->foffset; - sc.filesize = size; - sc.maxprot = VM_PROT_READ; - sc.initprot = VM_PROT_READ; - sc.nsects = 0; - sc.flags = 0; - - if ((ret = kdp_core_stream_output(kdc->outvars, sizeof(kernel_segment_command_t), (caddr_t) &sc)) != kIOReturnSuccess) { - DEBG("kdp_core_stream_output(0x%x)\n", ret); - goto out; - } - - kdc->hoffset += sizeof(kernel_segment_command_t); - kdc->foffset += size; +int +kern_dump_pmap_traverse_send_segdesc_callback(vm_map_offset_t start, + vm_map_offset_t end, + void *context) +{ + struct kern_dump_send_seg_desc_context *kds_context = (struct kern_dump_send_seg_desc_context *)context; + uint64_t seg_start = (uint64_t) start; + uint64_t seg_end = (uint64_t) end; -out: - return (ret); + return kds_context->callback(seg_start, seg_end, kds_context->context); } +struct kern_dump_send_segdata_context +{ + core_save_segment_data_cb callback; + void *context; +}; int kern_dump_pmap_traverse_send_segdata_callback(vm_map_offset_t start, vm_map_offset_t end, void *context) { - struct kern_dump_send_context *kdc = (struct kern_dump_send_context *)context; - int ret = KERN_SUCCESS; - vm_size_t size = (vm_size_t)(end - start); - - kdc->region_count++; - kdc->dumpable_bytes += size; - if ((ret = kdp_core_stream_output(kdc->outvars, size, (caddr_t)(uintptr_t)start)) != kIOReturnSuccess) { - DEBG("kdp_core_stream_output(0x%x)\n", ret); - goto out; - } - kdc->foffset += size; + struct kern_dump_send_segdata_context *kds_context = (struct kern_dump_send_segdata_context *)context; -out: - return (ret); + return kds_context->callback((void *)start, (uint64_t)(end - start), kds_context->context); } static int -do_kern_dump(kern_dump_output_proc outproc, enum kern_dump_type kd_variant) +kern_dump_save_summary(__unused void *refcon, core_save_summary_cb callback, void *context) { - struct kern_dump_preflight_context kdc_preflight = { }; - struct kern_dump_send_context kdc_sendseg = { }; - struct kern_dump_send_context kdc_send = { }; - struct kdp_core_out_vars outvars = { }; - struct mach_core_fileheader hdr = { }; - struct ident_command ident = { }; - kernel_mach_header_t mh = { }; - - uint32_t segment_count = 0, tstate_count = 0; - size_t command_size = 0, header_size = 0, tstate_size = 0; - uint64_t hoffset = 0, foffset = 0; - int ret = 0; - char * log_start; - char * buf; - size_t log_size; - uint64_t new_logs = 0; - boolean_t opened; - - opened = false; - log_start = debug_buf_ptr; - log_size = debug_buf_ptr - debug_buf_addr; - assert (log_size <= debug_buf_size); - if (debug_buf_stackshot_start) - { - assert(debug_buf_stackshot_end >= debug_buf_stackshot_start); - log_size -= (debug_buf_stackshot_end - debug_buf_stackshot_start); + struct kern_dump_preflight_context kdc_preflight = { }; + uint64_t thread_state_size = 0, thread_count = 0; + kern_return_t ret; + + ret = pmap_traverse_present_mappings(kernel_pmap, + VM_MIN_KERNEL_AND_KEXT_ADDRESS, + VM_MAX_KERNEL_ADDRESS, + kern_dump_pmap_traverse_preflight_callback, + &kdc_preflight); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "save_summary: pmap traversal failed: %d\n", ret); + return ret; } - if (kd_variant == KERN_DUMP_DISK) - { - if ((ret = (*outproc)(KDP_WRQ, NULL, 0, &hoffset)) != kIOReturnSuccess) { - DEBG("KDP_WRQ(0x%x)\n", ret); - goto out; - } + kern_collectth_state_size(&thread_count, &thread_state_size); + + ret = callback(kdc_preflight.region_count, kdc_preflight.dumpable_bytes, + thread_count, thread_state_size, 0, context); + return ret; +} + +static int +kern_dump_save_seg_descriptions(__unused void *refcon, core_save_segment_descriptions_cb callback, void *context) +{ + kern_return_t ret; + struct kern_dump_send_seg_desc_context kds_context; + + kds_context.callback = callback; + kds_context.context = context; + + ret = pmap_traverse_present_mappings(kernel_pmap, + VM_MIN_KERNEL_AND_KEXT_ADDRESS, + VM_MAX_KERNEL_ADDRESS, + kern_dump_pmap_traverse_send_segdesc_callback, + &kds_context); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "save_seg_desc: pmap traversal failed: %d\n", ret); + return ret; } - opened = true; - // init gzip - bzero(&outvars, sizeof(outvars)); - bzero(&hdr, sizeof(hdr)); - outvars.outproc = outproc; + return KERN_SUCCESS; +} - /* - * Initialize zstream variables that point to input and output - * buffer info. - */ +static int +kern_dump_save_thread_state(__unused void *refcon, void *buf, core_save_thread_state_cb callback, void *context) +{ + kern_return_t ret; + uint64_t thread_state_size = 0, thread_count = 0; + + kern_collectth_state_size(&thread_count, &thread_state_size); + + if (thread_state_size > 0) { + void * iter = NULL; + do { + kern_collectth_state (current_thread(), buf, thread_state_size, &iter); + + ret = callback(buf, context); + if (ret != KERN_SUCCESS) { + return ret; + } + } while (iter); + } + + return KERN_SUCCESS; +} + +static int +kern_dump_save_sw_vers(__unused void *refcon, core_save_sw_vers_cb callback, void *context) +{ + return callback(&kdp_kernelversion_string, sizeof(kdp_kernelversion_string), context); +} + +static int +kern_dump_save_segment_data(__unused void *refcon, core_save_segment_data_cb callback, void *context) +{ + kern_return_t ret; + struct kern_dump_send_segdata_context kds_context; + + kds_context.callback = callback; + kds_context.context = context; + + ret = pmap_traverse_present_mappings(kernel_pmap, + VM_MIN_KERNEL_AND_KEXT_ADDRESS, + VM_MAX_KERNEL_ADDRESS, kern_dump_pmap_traverse_send_segdata_callback, &kds_context); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "save_seg_data: pmap traversal failed: %d\n", ret); + return ret; + } + + return KERN_SUCCESS; +} + +kern_return_t +kdp_reset_output_vars(void *kdp_core_out_vars, uint64_t totalbytes) +{ + struct kdp_core_out_vars *outvars = (struct kdp_core_out_vars *)kdp_core_out_vars; + + /* Re-initialize kdp_outvars */ + outvars->zipped = 0; + outvars->totalbytes = totalbytes; + outvars->lastpercent = 0; + outvars->error = kIOReturnSuccess; + outvars->outremain = 0; + outvars->outlen = 0; + outvars->writes = 0; + outvars->outbuf = NULL; + + if (outvars->outproc == &kdp_send_crashdump_data) { + /* KERN_DUMP_NET */ + outvars->outbuf = (Bytef *) (kdp_core_zmem + kdp_core_zoffset); + outvars->outremain = outvars->outlen = kdp_crashdump_pkt_size; + } + + kdp_core_total_size = totalbytes; + + /* Re-initialize zstream variables */ kdp_core_zs.avail_in = 0; kdp_core_zs.next_in = NULL; kdp_core_zs.avail_out = 0; kdp_core_zs.next_out = NULL; - kdp_core_zs.opaque = &outvars; - kdc_sendseg.outvars = &outvars; - kdc_send.outvars = &outvars; + kdp_core_zs.opaque = outvars; - enum { kHdrOffset = 4096, kMaxCoreLog = 16384 }; + deflateResetWithIO(&kdp_core_zs, kdp_core_zinput, outvars->zoutput); - if (kd_variant == KERN_DUMP_DISK) { - outvars.outbuf = NULL; - outvars.outlen = 0; - outvars.outremain = 0; - outvars.zoutput = kdp_core_zoutput; - // space for file header, panic log, core log - foffset = (kHdrOffset + log_size + kMaxCoreLog + 4095) & ~4095ULL; - hdr.log_offset = kHdrOffset; - hdr.gzip_offset = foffset; - if ((ret = (*outproc)(KDP_SEEK, NULL, sizeof(foffset), &foffset)) != kIOReturnSuccess) { - DEBG("KDP_SEEK(0x%x)\n", ret); - goto out; - } - } else if (kd_variant == KERN_DUMP_NET) { - outvars.outbuf = (Bytef *) (kdp_core_zmem + kdp_core_zoffset); - assert((kdp_core_zoffset + kdp_crashdump_pkt_size) <= kdp_core_zsize); - outvars.outlen = kdp_crashdump_pkt_size; - outvars.outremain = outvars.outlen; - outvars.zoutput = kdp_core_zoutputbuf; -#if WITH_CONSISTENT_DBG - } else { /* KERN_DUMP_HW_SHMEM_DBG */ - outvars.outbuf = NULL; - outvars.outlen = 0; - outvars.outremain = 0; - outvars.zoutput = kdp_core_zoutput; - kern_dump_hw_shmem_dbg_reset(); -#endif + return KERN_SUCCESS; +} + +static int +kern_dump_update_header(struct kdp_core_out_vars *outvars) +{ + uint64_t foffset; + int ret; + + /* Write the file header -- first seek to the beginning of the file */ + foffset = 0; + if ((ret = (outvars->outproc)(KDP_SEEK, NULL, sizeof(foffset), &foffset)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "(kern_dump_update_header) outproc(KDP_SEEK, NULL, %lu, 0x%p) foffset = 0x%llx returned 0x%x\n", + sizeof(foffset), &foffset, foffset, ret); + return ret; } - deflateResetWithIO(&kdp_core_zs, kdp_core_zinput, outvars.zoutput); + if ((ret = (outvars->outproc)(KDP_DATA, NULL, sizeof(kdp_core_header), &kdp_core_header)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "(kern_dump_update_header) outproc(KDP_DATA, NULL, %lu, 0x%p) returned 0x%x\n", + sizeof(kdp_core_header), &kdp_core_header, ret); + return ret; + } + if ((ret = (outvars->outproc)(KDP_DATA, NULL, 0, NULL)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "(kern_dump_update_header) outproc data flush returned 0x%x\n", ret); + return ret; + } - kdc_preflight.region_count = 0; - kdc_preflight.dumpable_bytes = 0; +#if CONFIG_EMBEDDED + if ((ret = (outvars->outproc)(KDP_FLUSH, NULL, 0, NULL)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "(kern_dump_update_header) outproc explicit flush returned 0x%x\n", ret); + return ret; + } +#endif - ret = pmap_traverse_present_mappings(kernel_pmap, - VM_MIN_KERNEL_AND_KEXT_ADDRESS, - VM_MAX_KERNEL_ADDRESS, - kern_dump_pmap_traverse_preflight_callback, - &kdc_preflight); - if (ret) - { - DEBG("pmap traversal failed: %d\n", ret); - return (ret); - } + return KERN_SUCCESS; +} - outvars.totalbytes = kdc_preflight.dumpable_bytes; - assert(outvars.totalbytes); - segment_count = kdc_preflight.region_count; +int +kern_dump_record_file(void *kdp_core_out_vars, const char *filename, uint64_t file_offset, uint64_t *out_file_length) +{ + int ret = 0; + struct kdp_core_out_vars *outvars = (struct kdp_core_out_vars *)kdp_core_out_vars; + + assert(kdp_core_header.num_files < KERN_COREDUMP_MAX_CORES); + assert(out_file_length != NULL); + *out_file_length = 0; + + kdp_core_header.files[kdp_core_header.num_files].gzip_offset = file_offset; + kdp_core_header.files[kdp_core_header.num_files].gzip_length = outvars->zipped; + strncpy((char *)&kdp_core_header.files[kdp_core_header.num_files].core_name, filename, + MACH_CORE_FILEHEADER_NAMELEN); + kdp_core_header.files[kdp_core_header.num_files].core_name[MACH_CORE_FILEHEADER_NAMELEN - 1] = '\0'; + kdp_core_header.num_files++; + kdp_core_header.signature = MACH_CORE_FILEHEADER_SIGNATURE; + + ret = kern_dump_update_header(outvars); + if (ret == KERN_SUCCESS) { + *out_file_length = outvars->zipped; + } - kdp_core_total_size = outvars.totalbytes; - kdp_core_total_size_sent_uncomp = 0; + return ret; +} - kern_collectth_state_size(&tstate_count, &tstate_size); +int +kern_dump_seek_to_next_file(void *kdp_core_out_vars, uint64_t next_file_offset) +{ + struct kdp_core_out_vars *outvars = (struct kdp_core_out_vars *)kdp_core_out_vars; + int ret; - command_size = segment_count * sizeof(kernel_segment_command_t) - + tstate_count * tstate_size - + sizeof(struct ident_command) + sizeof(kdp_kernelversion_string); + if ((ret = (outvars->outproc)(KDP_SEEK, NULL, sizeof(next_file_offset), &next_file_offset)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "(kern_dump_seek_to_next_file) outproc(KDP_SEEK, NULL, %lu, 0x%p) foffset = 0x%llx returned 0x%x\n", + sizeof(next_file_offset), &next_file_offset, next_file_offset, ret); + } - header_size = command_size + sizeof(kernel_mach_header_t); + return ret; +} - /* - * Set up Mach-O header for currently executing kernel. - */ +static int +do_kern_dump(kern_dump_output_proc outproc, enum kern_dump_type kd_variant) +{ + struct kdp_core_out_vars outvars = { }; - mh.magic = _mh_execute_header.magic; - mh.cputype = _mh_execute_header.cputype;; - mh.cpusubtype = _mh_execute_header.cpusubtype; - mh.filetype = MH_CORE; - mh.ncmds = segment_count + tstate_count + 1; - mh.sizeofcmds = (uint32_t)command_size; - mh.flags = 0; -#if defined(__LP64__) - mh.reserved = 0; -#endif + char *log_start = NULL, *buf = NULL; + size_t existing_log_size = 0, new_log_len = 0; + uint64_t foffset = 0; + int ret = 0; + boolean_t output_opened = FALSE, dump_succeeded = TRUE; - hoffset = 0; /* offset into header */ - foffset = (uint64_t) round_page(header_size); /* offset into file */ + /* + * Record the initial panic log buffer length so we can dump the coredump log + * and panic log to disk + */ + log_start = debug_buf_ptr; +#if CONFIG_EMBEDDED + assert(panic_info->eph_other_log_offset != 0); + assert(panic_info->eph_panic_log_len != 0); + /* Include any data from before the panic log as well */ + existing_log_size = (panic_info->eph_panic_log_offset - sizeof(struct embedded_panic_header)) + + panic_info->eph_panic_log_len + panic_info->eph_other_log_len; +#else /* CONFIG_EMBEDDED */ + existing_log_size = log_start - debug_buf_base; +#endif /* CONFIG_EMBEDDED */ + + assert (existing_log_size <= debug_buf_size); - /* Transmit the Mach-O MH_CORE header, and segment and thread commands - */ - if ((ret = kdp_core_stream_output(&outvars, sizeof(kernel_mach_header_t), (caddr_t) &mh) != kIOReturnSuccess)) - { - DEBG("KDP_DATA(0x%x)\n", ret); - goto out; - } + if (kd_variant == KERN_DUMP_DISK) { + /* Open the file for output */ + if ((ret = (*outproc)(KDP_WRQ, NULL, 0, NULL)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "outproc(KDP_WRQ, NULL, 0, NULL) returned 0x%x\n", ret); + dump_succeeded = FALSE; + goto exit; + } + } + output_opened = true; - hoffset += sizeof(kernel_mach_header_t); + /* Initialize gzip, output context */ + bzero(&outvars, sizeof(outvars)); + outvars.outproc = outproc; - DEBG("%s", (kd_variant == KERN_DUMP_DISK) ? "Writing local kernel core..." : - "Transmitting kernel state, please wait:\n"); + if (kd_variant == KERN_DUMP_DISK) { + outvars.zoutput = kdp_core_zoutput; + /* Space for file header, panic log, core log */ + foffset = (KERN_COREDUMP_HEADERSIZE + existing_log_size + KERN_COREDUMP_MAXDEBUGLOGSIZE + + KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN - 1) & ~(KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN - 1); + kdp_core_header.log_offset = KERN_COREDUMP_HEADERSIZE; - kdc_sendseg.region_count = 0; - kdc_sendseg.dumpable_bytes = 0; - kdc_sendseg.hoffset = hoffset; - kdc_sendseg.foffset = foffset; - kdc_sendseg.header_size = header_size; - - if ((ret = pmap_traverse_present_mappings(kernel_pmap, - VM_MIN_KERNEL_AND_KEXT_ADDRESS, - VM_MAX_KERNEL_ADDRESS, - kern_dump_pmap_traverse_send_seg_callback, - &kdc_sendseg)) != kIOReturnSuccess) - { - DEBG("pmap_traverse_present_mappings(0x%x)\n", ret); - goto out; - } + /* Seek the calculated offset (we'll scrollback later to flush the logs and header) */ + if ((ret = (*outproc)(KDP_SEEK, NULL, sizeof(foffset), &foffset)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "(do_kern_dump seek begin) outproc(KDP_SEEK, NULL, %lu, 0x%p) foffset = 0x%llx returned 0x%x\n", + sizeof(foffset), &foffset, foffset, ret); + dump_succeeded = FALSE; + goto exit; + } + } else if (kd_variant == KERN_DUMP_NET) { + assert((kdp_core_zoffset + kdp_crashdump_pkt_size) <= kdp_core_zsize); + outvars.zoutput = kdp_core_zoutputbuf; +#if CONFIG_EMBEDDED + } else { /* KERN_DUMP_HW_SHMEM_DBG */ + outvars.zoutput = kdp_core_zoutput; + kern_dump_hw_shmem_dbg_reset(); +#endif + } - hoffset = kdc_sendseg.hoffset; - /* - * Now send out the LC_THREAD load command, with the thread information - * for the current activation. - */ +#if defined(__arm__) || defined(__arm64__) + flush_mmu_tlb(); +#endif - if (tstate_size > 0) - { - void * iter; - char tstate[tstate_size]; - iter = NULL; - do { - /* - * Now send out the LC_THREAD load command, with the thread information - */ - kern_collectth_state (current_thread(), tstate, tstate_size, &iter); - - if ((ret = kdp_core_stream_output(&outvars, tstate_size, tstate)) != kIOReturnSuccess) { - DEBG("kdp_core_stream_output(0x%x)\n", ret); - goto out; - } + kern_coredump_log(NULL, "%s", (kd_variant == KERN_DUMP_DISK) ? "Writing local cores..." : + "Transmitting kernel state, please wait:\n"); + + if (kd_variant == KERN_DUMP_DISK) { + /* + * Dump co-processors as well, foffset will be overwritten with the + * offset of the next location in the file to be written to. + */ + if (kern_do_coredump(&outvars, FALSE, foffset, &foffset) != 0) { + dump_succeeded = FALSE; + } + } else { + /* Only the kernel */ + if (kern_do_coredump(&outvars, TRUE, foffset, &foffset) != 0) { + dump_succeeded = FALSE; + } } - while (iter); - } - ident.cmd = LC_IDENT; - ident.cmdsize = (uint32_t) (sizeof(struct ident_command) + sizeof(kdp_kernelversion_string)); - if ((ret = kdp_core_stream_output(&outvars, sizeof(ident), &ident)) != kIOReturnSuccess) { - DEBG("kdp_core_stream_output(0x%x)\n", ret); - goto out; - } - if ((ret = kdp_core_stream_output(&outvars, sizeof(kdp_kernelversion_string), &kdp_kernelversion_string[0])) != kIOReturnSuccess) { - DEBG("kdp_core_stream_output(0x%x)\n", ret); - goto out; - } + if (kd_variant == KERN_DUMP_DISK) { +#if defined(__x86_64__) && (DEVELOPMENT || DEBUG) + /* Write the macOS panic stackshot on its own to a separate 'corefile' */ + if (panic_stackshot_buf && panic_stackshot_len) { + uint64_t compressed_stackshot_len = 0; + + /* Seek to the offset of the next 'file' (foffset provided/updated from kern_do_coredump) */ + if ((ret = kern_dump_seek_to_next_file(&outvars, foffset)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "Failed to seek to stackshot file offset 0x%llx, kern_dump_seek_to_next_file returned 0x%x\n", foffset, ret); + dump_succeeded = FALSE; + } else if ((ret = kdp_reset_output_vars(&outvars, panic_stackshot_len)) != KERN_SUCCESS) { + kern_coredump_log(NULL, "Failed to reset outvars for stackshot with len 0x%zx, returned 0x%x\n", panic_stackshot_len, ret); + dump_succeeded = FALSE; + } else if ((ret = kdp_core_output(&outvars, panic_stackshot_len, (void *)panic_stackshot_buf)) != KERN_SUCCESS) { + kern_coredump_log(NULL, "Failed to write panic stackshot to file, kdp_coreoutput(outvars, %lu, 0x%p) returned 0x%x\n", + panic_stackshot_len, (void *) panic_stackshot_buf, ret); + dump_succeeded = FALSE; + } else if ((ret = kdp_core_output(&outvars, 0, NULL)) != KERN_SUCCESS) { + kern_coredump_log(NULL, "Failed to flush stackshot data : kdp_core_output(0x%p, 0, NULL) returned 0x%x\n", &outvars, ret); + dump_succeeded = FALSE; + } else if ((ret = kern_dump_record_file(&outvars, "panic_stackshot.kcdata", foffset, &compressed_stackshot_len)) != KERN_SUCCESS) { + kern_coredump_log(NULL, "Failed to record panic stackshot in corefile header, kern_dump_record_file returned 0x%x\n", ret); + dump_succeeded = FALSE; + } else { + kern_coredump_log(NULL, "Recorded panic stackshot in corefile at offset 0x%llx, compressed to %llu bytes\n", foffset, compressed_stackshot_len); + } + } +#endif /* defined(__x86_64__) && (DEVELOPMENT || DEBUG) */ - kdc_send.region_count = 0; - kdc_send.dumpable_bytes = 0; - foffset = (uint64_t) round_page(header_size); /* offset into file */ - kdc_send.foffset = foffset; - kdc_send.hoffset = 0; - foffset = round_page_64(header_size) - header_size; - if (foffset) - { - // zero fill to page align - if ((ret = kdp_core_stream_output(&outvars, foffset, NULL)) != kIOReturnSuccess) { - DEBG("kdp_core_stream_output(0x%x)\n", ret); - goto out; - } - } + /* Write the debug log -- first seek to the end of the corefile header */ + foffset = KERN_COREDUMP_HEADERSIZE; + if ((ret = (*outproc)(KDP_SEEK, NULL, sizeof(foffset), &foffset)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "(do_kern_dump seek logfile) outproc(KDP_SEEK, NULL, %lu, 0x%p) foffset = 0x%llx returned 0x%x\n", + sizeof(foffset), &foffset, foffset, ret); + dump_succeeded = FALSE; + goto exit; + } - ret = pmap_traverse_present_mappings(kernel_pmap, - VM_MIN_KERNEL_AND_KEXT_ADDRESS, - VM_MAX_KERNEL_ADDRESS, - kern_dump_pmap_traverse_send_segdata_callback, - &kdc_send); - if (ret) { - DEBG("pmap_traverse_present_mappings(0x%x)\n", ret); - goto out; - } + new_log_len = debug_buf_ptr - log_start; + if (new_log_len > KERN_COREDUMP_MAXDEBUGLOGSIZE) { + new_log_len = KERN_COREDUMP_MAXDEBUGLOGSIZE; + } - if ((ret = kdp_core_stream_output(&outvars, 0, NULL) != kIOReturnSuccess)) { - DEBG("kdp_core_stream_output(0x%x)\n", ret); - goto out; - } +#if CONFIG_EMBEDDED + /* This data is after the panic stackshot, we need to write it separately */ + existing_log_size -= panic_info->eph_other_log_len; +#endif -out: - if (kIOReturnSuccess == ret) DEBG("success\n"); - else outvars.zipped = 0; - - DEBG("Mach-o header: %lu\n", header_size); - DEBG("Region counts: [%u, %u, %u]\n", kdc_preflight.region_count, - kdc_sendseg.region_count, - kdc_send.region_count); - DEBG("Byte counts : [%llu, %llu, %llu, %lu, %lu]\n", kdc_preflight.dumpable_bytes, - kdc_sendseg.dumpable_bytes, - kdc_send.dumpable_bytes, - outvars.zipped, - (long) (debug_buf_ptr - debug_buf_addr)); - if ((kd_variant == KERN_DUMP_DISK) && opened) - { - // write debug log - foffset = kHdrOffset; - if ((ret = (*outproc)(KDP_SEEK, NULL, sizeof(foffset), &foffset)) != kIOReturnSuccess) { - DEBG("KDP_SEEK(0x%x)\n", ret); - goto exit; - } + /* + * Write out the paniclog (from the beginning of the debug + * buffer until the start of the stackshot) + */ + buf = debug_buf_base; + if ((ret = (*outproc)(KDP_DATA, NULL, existing_log_size, buf)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "(do_kern_dump paniclog) outproc(KDP_DATA, NULL, %lu, 0x%p) returned 0x%x\n", + existing_log_size, buf, ret); + dump_succeeded = FALSE; + goto exit; + } - new_logs = debug_buf_ptr - log_start; - if (new_logs > kMaxCoreLog) new_logs = kMaxCoreLog; - buf = debug_buf_addr; - if (debug_buf_stackshot_start) - { - if ((ret = (*outproc)(KDP_DATA, NULL, (debug_buf_stackshot_start - debug_buf_addr), debug_buf_addr)) != kIOReturnSuccess) - { - DEBG("KDP_DATA(0x%x)\n", ret); - goto exit; - } - buf = debug_buf_stackshot_end; - } - if ((ret = (*outproc)(KDP_DATA, NULL, (log_start + new_logs - buf), buf)) != kIOReturnSuccess) - { - DEBG("KDP_DATA(0x%x)\n", ret); - goto exit; - } - - // write header - - foffset = 0; - if ((ret = (*outproc)(KDP_SEEK, NULL, sizeof(foffset), &foffset)) != kIOReturnSuccess) { - DEBG("KDP_SEEK(0x%x)\n", ret); - goto exit; - } - - hdr.signature = MACH_CORE_FILEHEADER_SIGNATURE; - hdr.log_length = new_logs + log_size; - hdr.gzip_length = outvars.zipped; - - if ((ret = (*outproc)(KDP_DATA, NULL, sizeof(hdr), &hdr)) != kIOReturnSuccess) - { - DEBG("KDP_DATA(0x%x)\n", ret); - goto exit; +#if CONFIG_EMBEDDED + /* The next part of the log we're interested in is the beginning of the 'other' log */ + buf = (char *)(((char *)panic_info) + (uintptr_t) panic_info->eph_other_log_offset); + /* Include any data after the panic stackshot but before we started the coredump log (see above) */ + new_log_len += panic_info->eph_other_log_len; +#else /* CONFIG_EMBEDDED */ + buf += existing_log_size; +#endif /* CONFIG_EMBEDDED */ + + /* Write the coredump log */ + if ((ret = (*outproc)(KDP_DATA, NULL, new_log_len, buf)) != kIOReturnSuccess) { + kern_coredump_log(NULL, "(do_kern_dump coredump log) outproc(KDP_DATA, NULL, %lu, 0x%p) returned 0x%x\n", + new_log_len, buf, ret); + dump_succeeded = FALSE; + goto exit; + } + + kdp_core_header.log_length = existing_log_size + new_log_len; + kern_dump_update_header(&outvars); } - } exit: - /* close / last packet */ - if (opened && (ret = (*outproc)(KDP_EOF, NULL, 0, ((void *) 0))) != kIOReturnSuccess) - { - DEBG("KDP_EOF(0x%x)\n", ret); - } + /* close / last packet */ + if (output_opened && (ret = (*outproc)(KDP_EOF, NULL, 0, ((void *) 0))) != kIOReturnSuccess) { + kern_coredump_log(NULL, "(do_kern_dump close) outproc(KDP_EOF, NULL, 0, 0) returned 0x%x\n", ret); + dump_succeeded = FALSE; + } + return (dump_succeeded ? 0 : -1); +} - return (ret); +boolean_t +dumped_kernel_core() +{ + return kern_dump_successful; } int kern_dump(enum kern_dump_type kd_variant) { - static boolean_t dumped_local; + static boolean_t local_dump_in_progress = FALSE, dumped_local = FALSE; + int ret = -1; +#if KASAN + kasan_disable(); +#endif if (kd_variant == KERN_DUMP_DISK) { if (dumped_local) return (0); - dumped_local = TRUE; - return (do_kern_dump(&kern_dump_disk_proc, KERN_DUMP_DISK)); -#if WITH_CONSISTENT_DBG + if (local_dump_in_progress) return (-1); + local_dump_in_progress = TRUE; +#if CONFIG_EMBEDDED + hwsd_info->xhsdci_status = XHSDCI_STATUS_KERNEL_BUSY; +#endif + ret = do_kern_dump(&kern_dump_disk_proc, KERN_DUMP_DISK); + if (ret == 0) { + dumped_local = TRUE; + kern_dump_successful = TRUE; + local_dump_in_progress = FALSE; + } + + return ret; +#if CONFIG_EMBEDDED } else if (kd_variant == KERN_DUMP_HW_SHMEM_DBG) { - return (do_kern_dump(&kern_dump_hw_shmem_dbg_buffer_proc, KERN_DUMP_HW_SHMEM_DBG)); + ret = do_kern_dump(&kern_dump_hw_shmem_dbg_buffer_proc, KERN_DUMP_HW_SHMEM_DBG); + if (ret == 0) { + kern_dump_successful = TRUE; + } + return ret; #endif + } else { + ret = do_kern_dump(&kdp_send_crashdump_data, KERN_DUMP_NET); + if (ret == 0) { + kern_dump_successful = TRUE; + } + return ret; + } +} + +#if CONFIG_EMBEDDED +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wmissing-noreturn" +void +panic_spin_shmcon() +{ +#pragma clang diagnostic pop + kern_coredump_log(NULL, "\nPlease go to https://panic.apple.com to report this panic\n"); + kern_coredump_log(NULL, "Waiting for hardware shared memory debugger, handshake structure is at virt: %p, phys %p\n", + hwsd_info, (void *)kvtophys((vm_offset_t)hwsd_info)); + + assert(hwsd_info != NULL); + hwsd_info->xhsdci_status = XHSDCI_STATUS_KERNEL_READY; + hwsd_info->xhsdci_seq_no = 0; + FlushPoC_DcacheRegion((vm_offset_t) hwsd_info, sizeof(*hwsd_info)); + + for (;;) { + FlushPoC_DcacheRegion((vm_offset_t) hwsd_info, sizeof(*hwsd_info)); + if (hwsd_info->xhsdci_status == XHSDCI_COREDUMP_BEGIN) { + kern_dump(KERN_DUMP_HW_SHMEM_DBG); + } + + if ((hwsd_info->xhsdci_status == XHSDCI_COREDUMP_REMOTE_DONE) || + (hwsd_info->xhsdci_status == XHSDCI_COREDUMP_ERROR)) { + hwsd_info->xhsdci_status = XHSDCI_STATUS_KERNEL_READY; + hwsd_info->xhsdci_seq_no = 0; + FlushPoC_DcacheRegion((vm_offset_t) hwsd_info, sizeof(*hwsd_info)); + } } -#if CONFIG_KDP_INTERACTIVE_DEBUGGING - return (do_kern_dump(&kdp_send_crashdump_data, KERN_DUMP_NET)); -#else - return (-1); -#endif } +#endif /* CONFIG_EMBEDDED */ static void * kdp_core_zalloc(void * __unused ref, u_int items, u_int size) @@ -1179,8 +1398,13 @@ static void kdp_core_zfree(void * __unused ref, void * __unused ptr) {} +#if CONFIG_EMBEDDED +#define LEVEL Z_BEST_SPEED +#define NETBUF 0 +#else #define LEVEL Z_BEST_SPEED #define NETBUF 1440 +#endif void kdp_core_init(void) @@ -1188,11 +1412,13 @@ kdp_core_init(void) int wbits = 12; int memlevel = 3; kern_return_t kr; -#if WITH_CONSISTENT_DBG +#if CONFIG_EMBEDDED int i = 0; vm_offset_t kdp_core_hw_shmem_buf = 0; struct kdp_hw_shmem_dbg_buf_elm *cur_elm = NULL; + cache_info_t *cpuid_cache_info = NULL; #endif + kern_coredump_callback_config core_config = { }; if (kdp_core_zs.zalloc) return; kdp_core_zsize = round_page(NETBUF + zlib_deflate_memory_size(wbits, memlevel)); @@ -1211,7 +1437,20 @@ kdp_core_init(void) kdp_core_zoffset = 0; } -#if WITH_CONSISTENT_DBG + bzero(&kdp_core_header, sizeof(kdp_core_header)); + + core_config.kcc_coredump_init = NULL; /* TODO: consider doing mmu flush from an init function */ + core_config.kcc_coredump_get_summary = kern_dump_save_summary; + core_config.kcc_coredump_save_segment_descriptions = kern_dump_save_seg_descriptions; + core_config.kcc_coredump_save_thread_state = kern_dump_save_thread_state; + core_config.kcc_coredump_save_sw_vers = kern_dump_save_sw_vers; + core_config.kcc_coredump_save_segment_data = kern_dump_save_segment_data; + core_config.kcc_coredump_save_misc_data = NULL; + + kr = kern_register_xnu_coredump_helper(&core_config); + assert(KERN_SUCCESS == kr); + +#if CONFIG_EMBEDDED if (!PE_consistent_debug_enabled()) { return; } @@ -1238,10 +1477,16 @@ kdp_core_init(void) hwsd_info->xhsdci_coredump_total_size_sent_uncomp = 0; hwsd_info->xhsdci_page_size = PAGE_SIZE; + cpuid_cache_info = cache_info(); + assert(cpuid_cache_info != NULL); + kdp_core_hw_shmem_buf += sizeof(*hwsd_info); - kdp_hw_shmem_dbg_bufsize -= sizeof(*hwsd_info); - kdp_hw_shmem_dbg_bufsize = (kdp_hw_shmem_dbg_bufsize / KDP_CORE_HW_SHMEM_DBG_NUM_BUFFERS); - kdp_hw_shmem_dbg_bufsize -= (kdp_hw_shmem_dbg_bufsize % OPTIMAL_ASTRIS_READSIZE); + /* Leave the handshake structure on its own cache line so buffer writes don't cause flushes of old handshake data */ + kdp_core_hw_shmem_buf = ROUNDUP(kdp_core_hw_shmem_buf, (uint64_t) cpuid_cache_info->c_linesz); + kdp_hw_shmem_dbg_bufsize -= (uint32_t) (kdp_core_hw_shmem_buf - (vm_offset_t) hwsd_info); + kdp_hw_shmem_dbg_bufsize /= KDP_CORE_HW_SHMEM_DBG_NUM_BUFFERS; + /* The buffer size should be a cache-line length multiple */ + kdp_hw_shmem_dbg_bufsize -= (kdp_hw_shmem_dbg_bufsize % ROUNDDOWN(OPTIMAL_ASTRIS_READSIZE, cpuid_cache_info->c_linesz)); STAILQ_INIT(&free_hw_shmem_dbg_bufs); STAILQ_INIT(&hw_shmem_dbg_bufs_to_flush); @@ -1263,7 +1508,13 @@ kdp_core_init(void) PE_consistent_debug_register(kDbgIdAstrisConnection, kvtophys((vm_offset_t) hwsd_info), sizeof(pmap_paddr_t)); PE_consistent_debug_register(kDbgIdAstrisConnectionVers, CUR_XNU_HWSDCI_STRUCT_VERS, sizeof(uint32_t)); -#endif /* WITH_CONSISTENT_DBG */ +#endif /* CONFIG_EMBEDDED */ + +#if defined(__x86_64__) && (DEVELOPMENT || DEBUG) + /* Allocate space in the kernel map for the panic stackshot */ + kr = kmem_alloc(kernel_map, &panic_stackshot_buf, PANIC_STACKSHOT_BUFSIZE, VM_KERN_MEMORY_DIAG); + assert (KERN_SUCCESS == kr); +#endif /* defined(__x86_64__) && (DEVELOPMENT || DEBUG) */ } #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */ diff --git a/osfmk/kdp/kdp_core.h b/osfmk/kdp/kdp_core.h index 8c6135ccc..45cee67b1 100644 --- a/osfmk/kdp/kdp_core.h +++ b/osfmk/kdp/kdp_core.h @@ -36,7 +36,9 @@ #ifndef __KDP_CORE_H #define __KDP_CORE_H +#include #include +#include /* * Packet types. @@ -48,6 +50,7 @@ #define KDP_ERROR 5 /* error code */ #define KDP_SEEK 6 /* Seek to specified offset */ #define KDP_EOF 7 /* signal end of file */ +#define KDP_FLUSH 8 /* flush outstanding data */ #define KDP_FEATURE_MASK_STRING "features" enum {KDP_FEATURE_LARGE_CRASHDUMPS = 1, KDP_FEATURE_LARGE_PKT_SIZE = 2}; @@ -82,7 +85,7 @@ struct corehdr { #define CORE_REMOTE_PORT 1069 /* hardwired, we can't really query the services file */ -#if WITH_CONSISTENT_DBG +#if CONFIG_EMBEDDED /* * xnu shared memory hardware debugger support * @@ -117,9 +120,12 @@ struct xnu_hw_shmem_dbg_command_info { #define XHSDCI_COREDUMP_ERROR 7 /* indicates an error was encountered */ #define XHSDCI_COREDUMP_REMOTE_DONE 8 /* indicates that hardware debugger is done */ -#endif /* WITH_CONSISTENT_DBG */ +void panic_spin_shmcon(void); + +#endif /* CONFIG_EMBEDDED */ void kdp_panic_dump (void); +void begin_panic_transfer(void); void abort_panic_transfer (void); void kdp_set_dump_info(const uint32_t flags, const char *file, const char *destip, const char *routerip, const uint32_t port); @@ -128,12 +134,14 @@ void kdp_get_dump_info(kdp_dumpinfo_reply_t *rp); enum kern_dump_type { KERN_DUMP_DISK, /* local, on device core dump */ KERN_DUMP_NET, /* kdp network core dump */ -#if WITH_CONSISTENT_DBG +#if CONFIG_EMBEDDED KERN_DUMP_HW_SHMEM_DBG, /* coordinated hardware shared memory debugger core dump */ #endif }; -extern int kern_dump(enum kern_dump_type kd_variant); +int kern_dump(enum kern_dump_type kd_variant); + +boolean_t dumped_kernel_core(void); struct corehdr *create_panic_header(unsigned int request, const char *corename, unsigned length, unsigned block); @@ -143,9 +151,9 @@ int kdp_send_crashdump_pkt(unsigned int request, char *corename, int kdp_send_crashdump_data(unsigned int request, char *corename, uint64_t length, void * txstart); -void kern_collectth_state_size(uint32_t * tstate_count, size_t * tstate_size); +void kern_collectth_state_size(uint64_t * tstate_count, uint64_t * tstate_size); -void kern_collectth_state(thread_t thread, void *buffer, size_t size, void **iter); +void kern_collectth_state(thread_t thread, void *buffer, uint64_t size, void **iter); boolean_t kdp_has_polled_corefile(void); @@ -155,4 +163,22 @@ extern boolean_t kdp_corezip_disabled; #define KDP_CRASHDUMP_POLL_COUNT (2500) +#if PRIVATE +kern_return_t kdp_core_output(void *kdp_core_out_vars, uint64_t length, void * data); + +kern_return_t kdp_reset_output_vars(void *kdp_core_out_vars, uint64_t totalbytes); + +int kern_dump_record_file(void *kdp_core_out_vars, const char *filename, uint64_t file_offset, uint64_t *out_file_length); + +int kern_dump_seek_to_next_file(void *kdp_core_out_varss, uint64_t next_file_offset); + +extern boolean_t efi_valid_page(ppnum_t ppn); +#if defined(__x86_64__) +#define EFI_VALID_PAGE(x) efi_valid_page(x) +#elif defined(__arm__) || defined(__arm64__) +#define EFI_VALID_PAGE(x) (FALSE) +#endif /* defined (__x86_64__) */ + +#endif /* PRIVATE */ + #endif /* __KDP_CORE_H */ diff --git a/osfmk/kdp/kdp_dyld.h b/osfmk/kdp/kdp_dyld.h index 314d220b5..91110b6d7 100644 --- a/osfmk/kdp/kdp_dyld.h +++ b/osfmk/kdp/kdp_dyld.h @@ -31,6 +31,10 @@ * for each binary image not loaded from the shared cache during stackshots. */ +/* Some clients check the dyld version at runtime */ +#define DYLD_ALL_IMAGE_INFOS_ADDRESS_MINIMUM_VERSION 9 +#define DYLD_ALL_IMAGE_INFOS_TIMESTAMP_MINIMUM_VERSION 15 + /* Re-use dyld format for kext load addresses */ #if __LP64__ typedef struct user64_dyld_uuid_info kernel_uuid_info; @@ -83,10 +87,13 @@ struct user32_dyld_all_image_infos { user32_addr_t sharedCacheSlide; /* the following field is only in version 13 (Mac OS X 10.9, iOS 7.0) and later */ uint8_t sharedCacheUUID[16]; - /* the following field is only in version 14 (Mac OS X 10.9, iOS 7.0) and later */ - user32_addr_t reserved[16]; /* the following field is only in version 15 (Mac OS X 10.12, iOS 10.0) and later */ - uint64_t timestamp; + user32_addr_t sharedCacheBaseAddress; + uint64_t timestamp; + user32_addr_t reserved[14]; + /* the following fields are only in version 16 (macOS 10.13, iOS 12.0) and later */ + user32_addr_t compact_dyld_image_info_addr; + user32_size_t compact_dyld_image_info_size; }; struct user64_dyld_all_image_infos { @@ -118,8 +125,12 @@ struct user64_dyld_all_image_infos { user64_addr_t sharedCacheSlide; /* the following field is only in version 13 (Mac OS X 10.9, iOS 7.0) and later */ uint8_t sharedCacheUUID[16]; - /* the following field is only in version 14 (Mac OS X 10.9, iOS 7.0) and later */ - user64_addr_t reserved[16]; - /* the following field is only in version 15 (Mac OS X 10.12, iOS 10.0) and later */ - uint64_t timestamp; + /* the following field is only in version 15 (macOS 10.12, iOS 10.0) and later */ + user64_addr_t sharedCacheBaseAddress; + uint64_t timestamp; + user64_addr_t reserved[14]; + /* the following fields are only in version 16 (macOS 10.13, iOS 12.0) and later */ + user64_addr_t compact_dyld_image_info_addr; + user64_size_t compact_dyld_image_info_size; }; + diff --git a/osfmk/kdp/kdp_internal.h b/osfmk/kdp/kdp_internal.h index 8c0e39a80..9168ac7a8 100644 --- a/osfmk/kdp/kdp_internal.h +++ b/osfmk/kdp/kdp_internal.h @@ -31,6 +31,7 @@ */ #include +#include #include #include #include @@ -75,15 +76,6 @@ typedef boolean_t unsigned short * ); -struct debugger_callback { - kern_return_t (*callback) (void*); - void *callback_context; - boolean_t proceed_on_sync_failure; - kern_return_t error; -}; - -extern struct debugger_callback *debugger_callback; - extern boolean_t kdp_packet( diff --git a/osfmk/kdp/kdp_udp.c b/osfmk/kdp/kdp_udp.c index 2487c6071..5ab42e1af 100644 --- a/osfmk/kdp/kdp_udp.c +++ b/osfmk/kdp/kdp_udp.c @@ -44,7 +44,9 @@ #include #include +#if (MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING) #include +#endif #include #include #include @@ -76,7 +78,6 @@ extern unsigned int not_in_kdp; extern int kdp_snapshot; -extern void do_stackshot(void); #ifdef CONFIG_KDP_INTERACTIVE_DEBUGGING @@ -313,7 +314,6 @@ __private_extern__ volatile unsigned int flag_kdp_trigger_reboot = 0; extern unsigned int disableConsoleOutput; extern void kdp_call(void); -extern boolean_t kdp_call_kdb(void); void * kdp_get_interface(void); void kdp_set_gateway_mac(void *gatewaymac); @@ -334,11 +334,6 @@ char kdp_kernelversion_string[256]; static boolean_t gKDPDebug = FALSE; -#if WITH_CONSISTENT_DBG -#include -extern volatile struct xnu_hw_shmem_dbg_command_info *hwsd_info; -#endif - #define KDP_DEBUG(...) if (gKDPDebug) printf(__VA_ARGS__); #define SBLOCKSZ (2048) @@ -420,6 +415,14 @@ kdp_register_send_receive( PE_parse_boot_argn("debug", &debug, sizeof (debug)); +#if defined(__arm__) || defined(__arm64__) + { + uint32_t debug_flags; + + if (!PE_i_can_has_debugger(&debug_flags)) + debug = 0; + } +#endif if (!debug) return; @@ -1198,6 +1201,9 @@ kdp_connection_wait(void) printf("\nWaiting for remote debugger connection.\n"); kprintf("\nWaiting for remote debugger connection.\n"); +#ifdef ARM + printf("\nPlease go to https://panic.apple.com to report this panic\n"); +#endif if (reattach_wait == 0) { if((kdp_flag & KDP_GETC_ENA) && (0 != kdp_getc())) { @@ -1351,19 +1357,18 @@ kdp_debugger_loop( kdp_panic("kdp_raise_exception"); if (((kdp_flag & KDP_PANIC_DUMP_ENABLED) - || (kdp_flag & PANIC_LOG_DUMP) - || kdp_has_polled_corefile()) - && (panicstr != (char *) 0)) { + || (kdp_flag & PANIC_LOG_DUMP)) + && panic_active()) { kdp_panic_dump(); - if (kdp_flag & REBOOT_POST_CORE) + if (kdp_flag & REBOOT_POST_CORE && dumped_kernel_core()) kdp_machine_reboot(); } else { - if ((kdp_flag & PANIC_CORE_ON_NMI) && (panicstr == (char *) 0) + if ((kdp_flag & PANIC_CORE_ON_NMI) && panic_active() && !kdp.is_conn) { - disable_debug_output = disableConsoleOutput = FALSE; + disableConsoleOutput = FALSE; kdp_panic_dump(); - if (kdp_flag & REBOOT_POST_CORE) + if (kdp_flag & REBOOT_POST_CORE && dumped_kernel_core()) kdp_machine_reboot(); if (!(kdp_flag & DBG_POST_CORE)) @@ -1400,7 +1405,7 @@ again: if (1 == kdp_trigger_core_dump) { kdp_flag |= KDP_PANIC_DUMP_ENABLED; kdp_panic_dump(); - if (kdp_flag & REBOOT_POST_CORE) + if (kdp_flag & REBOOT_POST_CORE && dumped_kernel_core()) kdp_machine_reboot(); kdp_trigger_core_dump = 0; } @@ -1826,39 +1831,30 @@ kdp_set_dump_info(const uint32_t flags, const char *filename, TRUE : FALSE; reattach_wait = 1; - logPanicDataToScreen = 1; disableConsoleOutput = 0; - disable_debug_output = 0; kdp_trigger_core_dump = 1; } void kdp_get_dump_info(kdp_dumpinfo_reply_t *rp) { - if (rp->destip) { - if (panicd_specified) - strlcpy(rp->destip, panicd_ip_str, - sizeof(panicd_ip_str)); - else - rp->destip[0] = '\0'; - } - - if (rp->routerip) { - if (router_specified) - strlcpy(rp->routerip, router_ip_str, - sizeof(router_ip_str)); - else - rp->routerip[0] = '\0'; - } + if (panicd_specified) + strlcpy(rp->destip, panicd_ip_str, + sizeof(rp->destip)); + else + rp->destip[0] = '\0'; - if (rp->name) { - if (corename_specified) - strlcpy(rp->name, corename_str, - sizeof(corename_str)); - else - rp->name[0] = '\0'; + if (router_specified) + strlcpy(rp->routerip, router_ip_str, + sizeof(rp->routerip)); + else + rp->routerip[0] = '\0'; - } + if (corename_specified) + strlcpy(rp->name, corename_str, + sizeof(rp->name)); + else + rp->name[0] = '\0'; rp->port = panicd_port; @@ -1893,26 +1889,13 @@ kdp_panic_dump(void) printf("Entering system dump routine\n"); - /* try a local disk dump */ - if (kdp_has_polled_corefile()) { - flag_panic_dump_in_progress = TRUE; - kern_dump(KERN_DUMP_DISK); - abort_panic_transfer(); - } - - if (!strcmp("local", panicd_ip_str)) return; /* disk only request */ - if (!kdp_en_recv_pkt || !kdp_en_send_pkt) { - if (!kdp_has_polled_corefile()) { - kdb_printf("Error: No transport device registered for kernel crashdump\n"); - } + kdb_printf("Error: No transport device registered for kernel crashdump\n"); return; } if (!panicd_specified) { - if (!kdp_has_polled_corefile()) { - kdb_printf("A dump server was not specified in the boot-args, terminating kernel core dump.\n"); - } + kdb_printf("A dump server was not specified in the boot-args, terminating kernel core dump.\n"); goto panic_dump_exit; } @@ -1926,7 +1909,7 @@ kdp_panic_dump(void) if (!corename_specified) { coresuffix[0] = 0; /* Panic log bit takes precedence over core dump bit */ - if ((panicstr != (char *) 0) && (kdp_flag & PANIC_LOG_DUMP)) + if ((debugger_panic_str != (char *) 0) && (kdp_flag & PANIC_LOG_DUMP)) strlcpy(coreprefix, "paniclog", sizeof(coreprefix)); else if (kdp_flag & SYSTEM_LOG_DUMP) strlcpy(coreprefix, "systemlog", sizeof(coreprefix)); @@ -1994,11 +1977,11 @@ kdp_panic_dump(void) } /* Just the panic log requested */ - if ((panicstr != (char *) 0) && (kdp_flag & PANIC_LOG_DUMP)) { + if ((debugger_panic_str != (char *) 0) && (kdp_flag & PANIC_LOG_DUMP)) { kdb_printf_unbuffered("Transmitting panic log, please wait: "); kdp_send_crashdump_data(KDP_DATA, corename_str, - debug_buf_ptr - debug_buf_addr, - debug_buf_addr); + debug_buf_ptr - debug_buf_base, + debug_buf_base); kdp_send_crashdump_pkt (KDP_EOF, NULL, 0, ((void *) 0)); printf("Please file a bug report on this panic, if possible.\n"); goto panic_dump_exit; @@ -2038,6 +2021,12 @@ panic_dump_exit: return; } +void +begin_panic_transfer(void) +{ + flag_panic_dump_in_progress = TRUE; +} + void abort_panic_transfer(void) { @@ -2167,18 +2156,23 @@ kdp_init(void) boolean_t kdp_match_name_found = PE_parse_boot_argn("kdp_match_name", kdpname, sizeof(kdpname)); boolean_t kdp_not_serial = kdp_match_name_found ? (strncmp(kdpname, "serial", sizeof(kdpname))) : TRUE; +#if CONFIG_EMBEDDED + //respect any custom debugger boot-args + if(kdp_match_name_found && kdp_not_serial) + return; +#else /* CONFIG_EMBEDDED */ // serial must be explicitly requested if(!kdp_match_name_found || kdp_not_serial) return; +#endif /* CONFIG_EMBEDDED */ -#if WITH_CONSISTENT_DBG +#if CONFIG_EMBEDDED if (kdp_not_serial && PE_consistent_debug_enabled() && debug_boot_arg) { - current_debugger = HW_SHM_CUR_DB; return; } else { printf("Serial requested, consistent debug disabled or debug boot arg not present, configuring debugging over serial\n"); } -#endif /* WITH_CONSISTENT_DBG */ +#endif /* CONFIG_EMBEDDED */ kprintf("Initializing serial KDP\n"); @@ -2206,48 +2200,68 @@ kdp_init(void) } #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */ -#if !CONFIG_KDP_INTERACTIVE_DEBUGGING -__attribute__((noreturn)) -static void -panic_spin_forever() -{ - kdb_printf("\nPlease go to https://panic.apple.com to report this panic\n"); +#if !(MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING) +static struct kdp_ether_addr kdp_current_mac_address = {{0, 0, 0, 0, 0, 0}}; - for (;;) { } -} -#endif +/* XXX ugly forward declares to stop warnings */ +void *kdp_get_interface(void); +void kdp_set_ip_and_mac_addresses(struct kdp_in_addr *, struct kdp_ether_addr *); +void kdp_set_gateway_mac(void *); +void kdp_set_interface(void *); +void kdp_register_send_receive(void *, void *); +void kdp_unregister_send_receive(void *, void *); -#if WITH_CONSISTENT_DBG && CONFIG_KDP_INTERACTIVE_DEBUGGING -__attribute__((noreturn)) -static void -panic_spin_shmcon() +uint32_t kdp_stack_snapshot_bytes_traced(void); + +void +kdp_register_send_receive(__unused void *send, __unused void *receive) +{} + +void +kdp_unregister_send_receive(__unused void *send, __unused void *receive) +{} + +void * +kdp_get_interface( void) { - kdb_printf("\nPlease go to https://panic.apple.com to report this panic\n"); - kdb_printf("Waiting for hardware shared memory debugger, handshake structure is at virt: %p, phys %p\n", - hwsd_info, (void *)kvtophys((vm_offset_t)hwsd_info)); - - assert(hwsd_info != NULL); - hwsd_info->xhsdci_status = XHSDCI_STATUS_KERNEL_READY; - hwsd_info->xhsdci_seq_no = 0; - FlushPoC_DcacheRegion((vm_offset_t) hwsd_info, sizeof(*hwsd_info)); - - for (;;) { - FlushPoC_DcacheRegion((vm_offset_t) hwsd_info, sizeof(*hwsd_info)); - if (hwsd_info->xhsdci_status == XHSDCI_COREDUMP_BEGIN) { - kern_dump(KERN_DUMP_HW_SHMEM_DBG); - } + return(void *)0; +} - if ((hwsd_info->xhsdci_status == XHSDCI_COREDUMP_REMOTE_DONE) || - (hwsd_info->xhsdci_status == XHSDCI_COREDUMP_ERROR)) { - hwsd_info->xhsdci_status = XHSDCI_STATUS_KERNEL_READY; - hwsd_info->xhsdci_seq_no = 0; - FlushPoC_DcacheRegion((vm_offset_t) hwsd_info, sizeof(*hwsd_info)); - } - } +unsigned int +kdp_get_ip_address(void ) +{ return 0; } + +struct kdp_ether_addr +kdp_get_mac_addr(void) +{ + return kdp_current_mac_address; } -#endif /* WITH_CONSISTENT_DBG && CONFIG_KDP_INTERACTIVE_DEBUGGING */ + +void +kdp_set_ip_and_mac_addresses( + __unused struct kdp_in_addr *ipaddr, + __unused struct kdp_ether_addr *macaddr) +{} + +void +kdp_set_gateway_mac(__unused void *gatewaymac) +{} + +void +kdp_set_interface(__unused void *ifp) +{} + +void kdp_register_link(__unused kdp_link_t link, __unused kdp_mode_t mode) +{} + +void kdp_unregister_link(__unused kdp_link_t link, __unused kdp_mode_t mode) +{} + +#endif /* !(MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING) */ #if !CONFIG_KDP_INTERACTIVE_DEBUGGING +extern __attribute__((noreturn)) void panic_spin_forever(void); + __attribute__((noreturn)) void kdp_raise_exception( @@ -2266,50 +2280,16 @@ kdp_raise_exception( ) #endif { +#if CONFIG_EMBEDDED + assert(PE_i_can_has_debugger(NULL)); +#endif #if CONFIG_KDP_INTERACTIVE_DEBUGGING - unsigned int initial_not_in_kdp = not_in_kdp; - not_in_kdp = 0; - - disable_preemption(); - - if (current_debugger != KDP_CUR_DB) { - /* try a local disk dump */ - if (kdp_has_polled_corefile()) { -#if WITH_CONSISTENT_DBG - if (current_debugger == HW_SHM_CUR_DB) { - hwsd_info->xhsdci_status = XHSDCI_STATUS_KERNEL_BUSY; - } -#endif /* WITH_CONSISTENT_DBG */ - flag_panic_dump_in_progress = TRUE; - kern_dump(KERN_DUMP_DISK); - abort_panic_transfer(); - } -#if WITH_CONSISTENT_DBG - if (current_debugger == HW_SHM_CUR_DB) { - panic_spin_shmcon(); - } -#endif /* WITH_CONSISTENT_DBG */ - - - if (!panicDebugging) { - kdp_machine_reboot(); - } - } - kdp_debugger_loop(exception, code, subcode, saved_state); - not_in_kdp = initial_not_in_kdp; - enable_preemption(); #else /* CONFIG_KDP_INTERACTIVE_DEBUGGING */ assert(current_debugger != KDP_CUR_DB); - /* - * If kernel debugging is enabled via boot-args, but KDP debugging - * is not compiled into the kernel, spin here waiting for debugging - * via another method. Why here? Because we want to have watchdog - * disabled (via KDP callout) while sitting waiting to be debugged. - */ panic_spin_forever(); #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */ } diff --git a/osfmk/kdp/ml/arm/kdp_machdep.c b/osfmk/kdp/ml/arm/kdp_machdep.c new file mode 100644 index 000000000..1bf25ac74 --- /dev/null +++ b/osfmk/kdp/ml/arm/kdp_machdep.c @@ -0,0 +1,727 @@ +/* + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define KDP_TEST_HARNESS 0 +#if KDP_TEST_HARNESS +#define dprintf(x) kprintf x +#else +#define dprintf(x) do {} while (0) +#endif + +void halt_all_cpus(boolean_t); +void kdp_call(void); +int kdp_getc(void); +int machine_trace_thread(thread_t thread, + char * tracepos, + char * tracebound, + int nframes, + boolean_t user_p, + boolean_t trace_fp, + uint32_t * thread_trace_flags); +int machine_trace_thread64(thread_t thread, + char * tracepos, + char * tracebound, + int nframes, + boolean_t user_p, + boolean_t trace_fp, + uint32_t * thread_trace_flags); + +void kdp_trap(unsigned int, struct arm_saved_state * saved_state); + +extern vm_offset_t machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags); +extern void machine_trace_thread_clear_validation_cache(void); +extern vm_map_t kernel_map; + +#if CONFIG_KDP_INTERACTIVE_DEBUGGING +void +kdp_exception( + unsigned char * pkt, int * len, unsigned short * remote_port, unsigned int exception, unsigned int code, unsigned int subcode) +{ + struct { + kdp_exception_t pkt; + kdp_exc_info_t exc; + } aligned_pkt; + + kdp_exception_t * rq = (kdp_exception_t *)&aligned_pkt; + + bcopy((char *)pkt, (char *)rq, sizeof(*rq)); + rq->hdr.request = KDP_EXCEPTION; + rq->hdr.is_reply = 0; + rq->hdr.seq = kdp.exception_seq; + rq->hdr.key = 0; + rq->hdr.len = sizeof(*rq) + sizeof(kdp_exc_info_t); + + rq->n_exc_info = 1; + rq->exc_info[0].cpu = 0; + rq->exc_info[0].exception = exception; + rq->exc_info[0].code = code; + rq->exc_info[0].subcode = subcode; + + rq->hdr.len += rq->n_exc_info * sizeof(kdp_exc_info_t); + + bcopy((char *)rq, (char *)pkt, rq->hdr.len); + + kdp.exception_ack_needed = TRUE; + + *remote_port = kdp.exception_port; + *len = rq->hdr.len; +} + +boolean_t +kdp_exception_ack(unsigned char * pkt, int len) +{ + kdp_exception_ack_t aligned_pkt; + kdp_exception_ack_t * rq = (kdp_exception_ack_t *)&aligned_pkt; + + if ((unsigned)len < sizeof(*rq)) + return (FALSE); + + bcopy((char *)pkt, (char *)rq, sizeof(*rq)); + + if (!rq->hdr.is_reply || rq->hdr.request != KDP_EXCEPTION) + return (FALSE); + + dprintf(("kdp_exception_ack seq %x %x\n", rq->hdr.seq, kdp.exception_seq)); + + if (rq->hdr.seq == kdp.exception_seq) { + kdp.exception_ack_needed = FALSE; + kdp.exception_seq++; + } + return (TRUE); +} + +static void +kdp_getintegerstate(char * out_state) +{ +#if defined(__arm__) + struct arm_thread_state thread_state; + struct arm_saved_state *saved_state; + + saved_state = kdp.saved_state; + + bzero((char *) &thread_state, sizeof(struct arm_thread_state)); + + saved_state_to_thread_state32(saved_state, &thread_state); + + bcopy((char *) &thread_state, (char *) out_state, sizeof(struct arm_thread_state)); +#elif defined(__arm64__) + struct arm_thread_state64 thread_state64; + arm_saved_state_t *saved_state; + + saved_state = kdp.saved_state; + assert(is_saved_state64(saved_state)); + + bzero((char *) &thread_state64, sizeof(struct arm_thread_state64)); + + saved_state_to_thread_state64(saved_state, &thread_state64); + + bcopy((char *) &thread_state64, (char *) out_state, sizeof(struct arm_thread_state64)); +#else +#error Unknown architecture. +#endif +} + +kdp_error_t +kdp_machine_read_regs(__unused unsigned int cpu, unsigned int flavor, char * data, int * size) +{ + switch (flavor) { +#if defined(__arm__) + case ARM_THREAD_STATE: + dprintf(("kdp_readregs THREAD_STATE\n")); + kdp_getintegerstate(data); + *size = ARM_THREAD_STATE_COUNT * sizeof(int); + return KDPERR_NO_ERROR; +#elif defined(__arm64__) + case ARM_THREAD_STATE64: + dprintf(("kdp_readregs THREAD_STATE64\n")); + kdp_getintegerstate(data); + *size = ARM_THREAD_STATE64_COUNT * sizeof(int); + return KDPERR_NO_ERROR; +#endif + + case ARM_VFP_STATE: + dprintf(("kdp_readregs THREAD_FPSTATE\n")); + bzero((char *) data, sizeof(struct arm_vfp_state)); + *size = ARM_VFP_STATE_COUNT * sizeof(int); + return KDPERR_NO_ERROR; + + default: + dprintf(("kdp_readregs bad flavor %d\n")); + return KDPERR_BADFLAVOR; + } +} + +static void +kdp_setintegerstate(char * state_in) +{ +#if defined(__arm__) + struct arm_thread_state thread_state; + struct arm_saved_state *saved_state; + + bcopy((char *) state_in, (char *) &thread_state, sizeof(struct arm_thread_state)); + saved_state = kdp.saved_state; + + thread_state32_to_saved_state(&thread_state, saved_state); +#elif defined(__arm64__) + struct arm_thread_state64 thread_state64; + struct arm_saved_state *saved_state; + + bcopy((char *) state_in, (char *) &thread_state64, sizeof(struct arm_thread_state64)); + saved_state = kdp.saved_state; + assert(is_saved_state64(saved_state)); + + thread_state64_to_saved_state(&thread_state64, saved_state); +#else +#error Unknown architecture. +#endif +} + +kdp_error_t +kdp_machine_write_regs(__unused unsigned int cpu, unsigned int flavor, char * data, __unused int * size) +{ + switch (flavor) { +#if defined(__arm__) + case ARM_THREAD_STATE: + dprintf(("kdp_writeregs THREAD_STATE\n")); + kdp_setintegerstate(data); + return KDPERR_NO_ERROR; +#elif defined(__arm64__) + case ARM_THREAD_STATE64: + dprintf(("kdp_writeregs THREAD_STATE64\n")); + kdp_setintegerstate(data); + return KDPERR_NO_ERROR; +#endif + + case ARM_VFP_STATE: + dprintf(("kdp_writeregs THREAD_FPSTATE\n")); + return KDPERR_NO_ERROR; + + default: + dprintf(("kdp_writeregs bad flavor %d\n")); + return KDPERR_BADFLAVOR; + } +} + +void +kdp_machine_hostinfo(kdp_hostinfo_t * hostinfo) +{ + hostinfo->cpus_mask = 1; + hostinfo->cpu_type = slot_type(0); + hostinfo->cpu_subtype = slot_subtype(0); +} + +__attribute__((noreturn)) +void +kdp_panic(const char * msg) +{ + printf("kdp panic: %s\n", msg); + while (1) { + }; +} + +int +kdp_intr_disbl(void) +{ + return (splhigh()); +} + +void +kdp_intr_enbl(int s) +{ + splx(s); +} + +void +kdp_us_spin(int usec) +{ + delay(usec / 100); +} + +void +kdp_call(void) +{ + Debugger("inline call to debugger(machine_startup)"); +} + +int +kdp_getc(void) +{ + return (cnmaygetc()); +} + +void +kdp_machine_get_breakinsn(uint8_t * bytes, uint32_t * size) +{ + *(uint32_t *)bytes = GDB_TRAP_INSTR1; + *size = sizeof(uint32_t); +} + +void +kdp_sync_cache(void) +{ +} + +int +kdp_machine_ioport_read(kdp_readioport_req_t * rq, caddr_t data, uint16_t lcpu) +{ +#pragma unused(rq, data, lcpu) + return 0; +} + +int +kdp_machine_ioport_write(kdp_writeioport_req_t * rq, caddr_t data, uint16_t lcpu) +{ +#pragma unused(rq, data, lcpu) + return 0; +} + +int +kdp_machine_msr64_read(kdp_readmsr64_req_t *rq, caddr_t data, uint16_t lcpu) +{ +#pragma unused(rq, data, lcpu) + return 0; +} + +int +kdp_machine_msr64_write(kdp_writemsr64_req_t *rq, caddr_t data, uint16_t lcpu) +{ +#pragma unused(rq, data, lcpu) + return 0; +} +#endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */ + +void +kdp_trap(unsigned int exception, struct arm_saved_state * saved_state) +{ + handle_debugger_trap(exception, 0, 0, saved_state); + +#if defined(__arm__) + if (saved_state->cpsr & PSR_TF) { + unsigned short instr = *((unsigned short *)(saved_state->pc)); + if ((instr == (GDB_TRAP_INSTR1 & 0xFFFF)) || (instr == (GDB_TRAP_INSTR2 & 0xFFFF))) + saved_state->pc += 2; + } else { + unsigned int instr = *((unsigned int *)(saved_state->pc)); + if ((instr == GDB_TRAP_INSTR1) || (instr == GDB_TRAP_INSTR2)) + saved_state->pc += 4; + } + +#elif defined(__arm64__) + assert(is_saved_state64(saved_state)); + + uint32_t instr = *((uint32_t *)get_saved_state_pc(saved_state)); + + /* + * As long as we are using the arm32 trap encoding to handling + * traps to the debugger, we should identify both variants and + * increment for both of them. + */ + if ((instr == GDB_TRAP_INSTR1) || (instr == GDB_TRAP_INSTR2)) + set_saved_state_pc(saved_state, get_saved_state_pc(saved_state) + 4); +#else +#error Unknown architecture. +#endif +} + +#define ARM32_LR_OFFSET 4 +#define ARM64_LR_OFFSET 8 + +/* + * Since sizeof (struct thread_snapshot) % 4 == 2 + * make sure the compiler does not try to use word-aligned + * access to this data, which can result in alignment faults + * that can't be emulated in KDP context. + */ +typedef uint32_t uint32_align2_t __attribute__((aligned(2))); + +int +machine_trace_thread(thread_t thread, + char * tracepos, + char * tracebound, + int nframes, + boolean_t user_p, + boolean_t trace_fp, + uint32_t * thread_trace_flags) +{ + uint32_align2_t * tracebuf = (uint32_align2_t *)tracepos; + + vm_size_t framesize = (trace_fp ? 2 : 1) * sizeof(uint32_t); + + vm_offset_t stacklimit = 0; + vm_offset_t stacklimit_bottom = 0; + int framecount = 0; + uint32_t short_fp = 0; + vm_offset_t fp = 0; + vm_offset_t pc, sp; + vm_offset_t prevfp = 0; + uint32_t prevlr = 0; + struct arm_saved_state * state; + vm_offset_t kern_virt_addr = 0; + vm_map_t bt_vm_map = VM_MAP_NULL; + + nframes = (tracebound > tracepos) ? MIN(nframes, (int)((tracebound - tracepos) / framesize)) : 0; + if (!nframes) { + return (0); + } + framecount = 0; + + if (user_p) { + /* Examine the user savearea */ + state = get_user_regs(thread); + stacklimit = VM_MAX_ADDRESS; + stacklimit_bottom = VM_MIN_ADDRESS; + + /* Fake up a stack frame for the PC */ + *tracebuf++ = (uint32_t)get_saved_state_pc(state); + if (trace_fp) { + *tracebuf++ = (uint32_t)get_saved_state_sp(state); + } + framecount++; + bt_vm_map = thread->task->map; + } else { +#if defined(__arm64__) + panic("Attempted to trace kernel thread_t %p as a 32-bit context", thread); + return 0; +#elif defined(__arm__) + /* kstackptr may not always be there, so recompute it */ + state = &thread_get_kernel_state(thread)->machine; + + stacklimit = VM_MAX_KERNEL_ADDRESS; + stacklimit_bottom = VM_MIN_KERNEL_ADDRESS; + bt_vm_map = kernel_map; +#else +#error Unknown architecture. +#endif + } + + /* Get the frame pointer */ + fp = get_saved_state_fp(state); + + /* Fill in the current link register */ + prevlr = (uint32_t)get_saved_state_lr(state); + pc = get_saved_state_pc(state); + sp = get_saved_state_sp(state); + + if (!user_p && !prevlr && !fp && !sp && !pc) { + return 0; + } + + if (!user_p) { + /* This is safe since we will panic above on __arm64__ if !user_p */ + prevlr = (uint32_t)VM_KERNEL_UNSLIDE(prevlr); + } + + for (; framecount < nframes; framecount++) { + + *tracebuf++ = prevlr; + if (trace_fp) { + *tracebuf++ = (uint32_t)fp; + } + + /* Invalid frame */ + if (!fp) { + break; + } + /* Unaligned frame */ + if (fp & 0x0000003) { + break; + } + /* Frame is out of range, maybe a user FP while doing kernel BT */ + if (fp > stacklimit) { + break; + } + if (fp < stacklimit_bottom) { + break; + } + /* Stack grows downward */ + if (fp < prevfp) { + + boolean_t prev_in_interrupt_stack = FALSE; + + if (!user_p) { + /* + * As a special case, sometimes we are backtracing out of an interrupt + * handler, and the stack jumps downward because of the memory allocation + * pattern during early boot due to KASLR. + */ + int cpu; + int max_cpu = ml_get_max_cpu_number(); + + for (cpu=0; cpu <= max_cpu; cpu++) { + cpu_data_t *target_cpu_datap; + + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + if(target_cpu_datap == (cpu_data_t *)NULL) + continue; + + if ((prevfp >= (target_cpu_datap->intstack_top-INTSTACK_SIZE) && prevfp < target_cpu_datap->intstack_top) || + (prevfp >= (target_cpu_datap->fiqstack_top-PAGE_SIZE) && prevfp < target_cpu_datap->fiqstack_top)) { + prev_in_interrupt_stack = TRUE; + break; + } + } + } + + if (!prev_in_interrupt_stack) { + /* Corrupt frame pointer? */ + break; + } + } + /* Assume there's a saved link register, and read it */ + kern_virt_addr = machine_trace_thread_get_kva(fp + ARM32_LR_OFFSET, bt_vm_map, thread_trace_flags); + + if (!kern_virt_addr) { + if (thread_trace_flags) { + *thread_trace_flags |= kThreadTruncatedBT; + } + break; + } + + prevlr = *(uint32_t *)kern_virt_addr; + if (!user_p) { + /* This is safe since we will panic above on __arm64__ if !user_p */ + prevlr = (uint32_t)VM_KERNEL_UNSLIDE(prevlr); + } + + prevfp = fp; + + /* + * Next frame; read the fp value into short_fp first + * as it is 32-bit. + */ + kern_virt_addr = machine_trace_thread_get_kva(fp, bt_vm_map, thread_trace_flags); + + if (kern_virt_addr) { + short_fp = *(uint32_t *)kern_virt_addr; + fp = (vm_offset_t) short_fp; + } else { + fp = 0; + if (thread_trace_flags) { + *thread_trace_flags |= kThreadTruncatedBT; + } + } + + } + /* Reset the target pmap */ + machine_trace_thread_clear_validation_cache(); + return ((int)(((char *)tracebuf) - tracepos)); +} + +int +machine_trace_thread64(thread_t thread, + char * tracepos, + char * tracebound, + int nframes, + boolean_t user_p, + boolean_t trace_fp, + uint32_t * thread_trace_flags) +{ +#if defined(__arm__) +#pragma unused(thread, tracepos, tracebound, nframes, user_p, trace_fp, thread_trace_flags) + return 0; +#elif defined(__arm64__) + + uint64_t * tracebuf = (uint64_t *)tracepos; + vm_size_t framesize = (trace_fp ? 2 : 1) * sizeof(uint64_t); + + vm_offset_t stacklimit = 0; + vm_offset_t stacklimit_bottom = 0; + int framecount = 0; + vm_offset_t fp = 0; + vm_offset_t pc = 0; + vm_offset_t sp = 0; + vm_offset_t prevfp = 0; + uint64_t prevlr = 0; + struct arm_saved_state * state; + vm_offset_t kern_virt_addr = 0; + vm_map_t bt_vm_map = VM_MAP_NULL; + + nframes = (tracebound > tracepos) ? MIN(nframes, (int)((tracebound - tracepos) / framesize)) : 0; + if (!nframes) { + return (0); + } + framecount = 0; + + if (user_p) { + /* Examine the user savearea */ + state = thread->machine.upcb; + stacklimit = MACH_VM_MAX_ADDRESS; + stacklimit_bottom = MACH_VM_MIN_ADDRESS; + + /* Fake up a stack frame for the PC */ + *tracebuf++ = get_saved_state_pc(state); + if (trace_fp) { + *tracebuf++ = get_saved_state_sp(state); + } + framecount++; + bt_vm_map = thread->task->map; + } else { + /* kstackptr may not always be there, so recompute it */ + state = &thread_get_kernel_state(thread)->machine.ss; + stacklimit = VM_MAX_KERNEL_ADDRESS; + stacklimit_bottom = VM_MIN_KERNEL_ADDRESS; + bt_vm_map = kernel_map; + } + + /* Get the frame pointer */ + fp = get_saved_state_fp(state); + + /* Fill in the current link register */ + prevlr = get_saved_state_lr(state); + pc = get_saved_state_pc(state); + sp = get_saved_state_sp(state); + + if (!user_p && !prevlr && !fp && !sp && !pc) { + return 0; + } + + if (!user_p) { + prevlr = VM_KERNEL_UNSLIDE(prevlr); + } + + for (; framecount < nframes; framecount++) { + + *tracebuf++ = prevlr; + if (trace_fp) { + *tracebuf++ = fp; + } + + /* Invalid frame */ + if (!fp) { + break; + } + /* + * Unaligned frame; given that the stack register must always be + * 16-byte aligned, we are assured 8-byte alignment of the saved + * frame pointer and link register. + */ + if (fp & 0x0000007) { + break; + } + /* Frame is out of range, maybe a user FP while doing kernel BT */ + if (fp > stacklimit) { + break; + } + if (fp < stacklimit_bottom) { + break; + } + /* Stack grows downward */ + if (fp < prevfp) { + boolean_t switched_stacks = FALSE; + + if (!user_p) { + /* + * As a special case, sometimes we are backtracing out of an interrupt + * handler, and the stack jumps downward because of the memory allocation + * pattern during early boot due to KASLR. + */ + int cpu; + int max_cpu = ml_get_max_cpu_number(); + + for (cpu=0; cpu <= max_cpu; cpu++) { + cpu_data_t *target_cpu_datap; + + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + if(target_cpu_datap == (cpu_data_t *)NULL) + continue; + + if ((prevfp >= (target_cpu_datap->intstack_top-INTSTACK_SIZE) && prevfp < target_cpu_datap->intstack_top) || + (prevfp >= (target_cpu_datap->fiqstack_top-PAGE_SIZE) && prevfp < target_cpu_datap->fiqstack_top)) { + switched_stacks = TRUE; + break; + } + + } + + } + + if (!switched_stacks) { + /* Corrupt frame pointer? */ + break; + } + } + + /* Assume there's a saved link register, and read it */ + kern_virt_addr = machine_trace_thread_get_kva(fp + ARM64_LR_OFFSET, bt_vm_map, thread_trace_flags); + + if (!kern_virt_addr) { + if (thread_trace_flags) { + *thread_trace_flags |= kThreadTruncatedBT; + } + break; + } + + prevlr = *(uint64_t *)kern_virt_addr; + if (!user_p) { + prevlr = VM_KERNEL_UNSLIDE(prevlr); + } + + prevfp = fp; + /* Next frame */ + kern_virt_addr = machine_trace_thread_get_kva(fp, bt_vm_map, thread_trace_flags); + + if (kern_virt_addr) { + fp = *(uint64_t *)kern_virt_addr; + } else { + fp = 0; + if (thread_trace_flags) { + *thread_trace_flags |= kThreadTruncatedBT; + } + } + + } + /* Reset the target pmap */ + machine_trace_thread_clear_validation_cache(); + return ((int)(((char *)tracebuf) - tracepos)); +#else +#error Unknown architecture. +#endif +} + +void +kdp_ml_enter_debugger(void) +{ + __asm__ volatile(".long 0xe7ffdefe"); +} + diff --git a/osfmk/kdp/ml/arm/kdp_vm.c b/osfmk/kdp/ml/arm/kdp_vm.c new file mode 100644 index 000000000..82f5307c7 --- /dev/null +++ b/osfmk/kdp/ml/arm/kdp_vm.c @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +pmap_t kdp_pmap = 0; +boolean_t kdp_trans_off; +boolean_t kdp_read_io = 0; + +pmap_paddr_t kdp_vtophys(pmap_t pmap, vm_offset_t va); + +/* + * kdp_vtophys + */ +pmap_paddr_t +kdp_vtophys( + pmap_t pmap, + vm_offset_t va) +{ + pmap_paddr_t pa; + ppnum_t pp; + + /* Ensure that the provided va resides within the provided pmap range. */ + if(!pmap || ((pmap != kernel_pmap) && ((va < pmap->min) || (va >= pmap->max)))) + { +#ifdef KDP_VTOPHYS_DEBUG + printf("kdp_vtophys(%08x, %016lx) not in range %08x .. %08x\n", (unsigned int) pmap, + (unsigned long) va, + (unsigned int) (pmap ? pmap->min : 0), + (unsigned int) (pmap ? pmap->max : 0)); +#endif + return 0; /* Just return if no translation */ + } + + pp = pmap_find_phys(pmap, va); /* Get the page number */ + if (!pp) + return 0; /* Just return if no translation */ + + pa = ((pmap_paddr_t) pp << PAGE_SHIFT) | (va & PAGE_MASK); /* Insert page offset */ + return (pa); +} + + +/* + * kdp_machine_vm_read + * + * Verify that src is valid, and physically copy len bytes from src to + * dst, translating if necessary. If translation is enabled + * (kdp_trans_off is 0), a non-zero kdp_pmap specifies the pmap to use + * when translating src. + */ + +mach_vm_size_t +kdp_machine_vm_read( mach_vm_address_t src, caddr_t dst, mach_vm_size_t len) +{ + addr64_t cur_virt_src, cur_virt_dst; + addr64_t cur_phys_src, cur_phys_dst; + mach_vm_size_t resid, cnt; + pmap_t pmap; + +#ifdef KDP_VM_READ_DEBUG + kprintf("kdp_machine_vm_read1: src %x dst %x len %x - %08X %08X\n", src, dst, len, ((unsigned long *) src)[0], ((unsigned long *) src)[1]); +#endif + + cur_virt_src = (addr64_t) src; + cur_virt_dst = (addr64_t) dst; + + if (kdp_trans_off) { + kdp_readphysmem64_req_t rq; + mach_vm_size_t ret; + + rq.address = src; + rq.nbytes = (uint32_t)len; + ret = kdp_machine_phys_read(&rq, dst, 0 /* unused */); + return ret; + } else { + + resid = len; + + if (kdp_pmap) + pmap = kdp_pmap; /* If special pmap, use it */ + else + pmap = kernel_pmap; /* otherwise, use kernel's */ + + while (resid != 0) { + /* + * Always translate the destination using the + * kernel_pmap. + */ + if ((cur_phys_dst = kdp_vtophys(kernel_pmap, cur_virt_dst)) == 0) + goto exit; + + if ((cur_phys_src = kdp_vtophys(pmap, cur_virt_src)) == 0) + goto exit; + + /* Attempt to ensure that there are valid translations for src and dst. */ + if (!kdp_read_io && ((!pmap_valid_address(cur_phys_dst)) || (!pmap_valid_address(cur_phys_src)))) + goto exit; + + cnt = ARM_PGBYTES - (cur_virt_src & PAGE_MASK); /* Get length left on + * page */ + if (cnt > (ARM_PGBYTES - (cur_virt_dst & PAGE_MASK))) + cnt = ARM_PGBYTES - (cur_virt_dst & PAGE_MASK); + + if (cnt > resid) + cnt = resid; + +#ifdef KDP_VM_READ_DEBUG + kprintf("kdp_machine_vm_read2: pmap %08X, virt %016LLX, phys %016LLX\n", + pmap, cur_virt_src, cur_phys_src); +#endif + bcopy_phys(cur_phys_src, cur_phys_dst, cnt); + + cur_virt_src += cnt; + cur_virt_dst += cnt; + resid -= cnt; + } + } +exit: +#ifdef KDP_VM_READ_DEBUG + kprintf("kdp_machine_vm_read: ret %08X\n", len - resid); +#endif + return (len - resid); +} + +mach_vm_size_t +kdp_machine_phys_read(kdp_readphysmem64_req_t *rq, caddr_t dst, uint16_t lcpu __unused) +{ + mach_vm_address_t src = rq->address; + mach_vm_size_t len = rq->nbytes; + + addr64_t cur_virt_dst; + addr64_t cur_phys_src, cur_phys_dst; + mach_vm_size_t resid = len; + mach_vm_size_t cnt = 0, cnt_src, cnt_dst; + +#ifdef KDP_VM_READ_DEBUG + kprintf("kdp_phys_read src %x dst %p len %x\n", src, dst, len); +#endif + + cur_virt_dst = (addr64_t) dst; + cur_phys_src = (addr64_t) src; + + while (resid != 0) { + + if ((cur_phys_dst = kdp_vtophys(kernel_pmap, cur_virt_dst)) == 0) + goto exit; + + /* Get length left on page */ + + cnt_src = ARM_PGBYTES - (cur_phys_src & PAGE_MASK); + cnt_dst = ARM_PGBYTES - (cur_phys_dst & PAGE_MASK); + if (cnt_src > cnt_dst) + cnt = cnt_dst; + else + cnt = cnt_src; + if (cnt > resid) + cnt = resid; + + bcopy_phys(cur_phys_src, cur_phys_dst, cnt); /* Copy stuff over */ + cur_phys_src += cnt; + cur_virt_dst += cnt; + resid -= cnt; + } + +exit: + return (len - resid); +} + +/* + * kdp_vm_write + */ +mach_vm_size_t +kdp_machine_vm_write( caddr_t src, mach_vm_address_t dst, mach_vm_size_t len) +{ + addr64_t cur_virt_src, cur_virt_dst; + addr64_t cur_phys_src, cur_phys_dst; + mach_vm_size_t resid, cnt, cnt_src, cnt_dst; + +#ifdef KDP_VM_WRITE_DEBUG + printf("kdp_vm_write: src %x dst %x len %x - %08X %08X\n", src, dst, len, ((unsigned long *) src)[0], ((unsigned long *) src)[1]); +#endif + + cur_virt_src = (addr64_t) src; + cur_virt_dst = (addr64_t) dst; + + resid = len; + + while (resid != 0) { + if ((cur_phys_dst = kdp_vtophys(kernel_pmap, cur_virt_dst)) == 0) + goto exit; + + if ((cur_phys_src = kdp_vtophys(kernel_pmap, cur_virt_src)) == 0) + goto exit; + + /* Attempt to ensure that there are valid translations for src and dst. */ + /* No support for enabling writes for an invalid translation at the moment. */ + if ((!pmap_valid_address(cur_phys_dst)) || (!pmap_valid_address(cur_phys_src))) + goto exit; + + cnt_src = ((cur_phys_src + ARM_PGBYTES) & (-ARM_PGBYTES)) - cur_phys_src; + cnt_dst = ((cur_phys_dst + ARM_PGBYTES) & (-ARM_PGBYTES)) - cur_phys_dst; + + if (cnt_src > cnt_dst) + cnt = cnt_dst; + else + cnt = cnt_src; + if (cnt > resid) + cnt = resid; + +#ifdef KDP_VM_WRITE_DEBUG + printf("kdp_vm_write: cur_phys_src %x cur_phys_src %x len %x - %08X %08X\n", src, dst, cnt); +#endif + bcopy_phys(cur_phys_src, cur_phys_dst, cnt); /* Copy stuff over */ + flush_dcache64(cur_phys_dst, (unsigned int)cnt, TRUE); + invalidate_icache64(cur_phys_dst, (unsigned int)cnt, TRUE); + + cur_virt_src += cnt; + cur_virt_dst += cnt; + resid -= cnt; + } +exit: + return (len - resid); +} + +mach_vm_size_t +kdp_machine_phys_write(kdp_writephysmem64_req_t *rq __unused, caddr_t src __unused, + uint16_t lcpu __unused) +{ + return 0; /* unimplemented */ +} + +void +kern_collectth_state_size(uint64_t * tstate_count, uint64_t * tstate_size) +{ + uint64_t count = ml_get_max_cpu_number() + 1; + + *tstate_count = count; + *tstate_size = sizeof(struct thread_command) + + (sizeof(arm_state_hdr_t) +#if defined(__arm64__) + + ARM_THREAD_STATE64_COUNT * sizeof(uint32_t)); +#else + + ARM_THREAD_STATE32_COUNT * sizeof(uint32_t)); +#endif +} + +void +kern_collectth_state(thread_t thread __unused, void *buffer, uint64_t size, void ** iter) +{ + cpu_data_entry_t *cpuentryp = *iter; + if (cpuentryp == NULL) + cpuentryp = &CpuDataEntries[0]; + + if (cpuentryp == &CpuDataEntries[ml_get_max_cpu_number()]) + *iter = NULL; + else + *iter = cpuentryp + 1; + + struct cpu_data *cpudatap = cpuentryp->cpu_data_vaddr; + + struct thread_command *tc = (struct thread_command *)buffer; + arm_state_hdr_t *hdr = (arm_state_hdr_t *)(void *)(tc + 1); +#if defined(__arm64__) + hdr->flavor = ARM_THREAD_STATE64; + hdr->count = ARM_THREAD_STATE64_COUNT; + arm_thread_state64_t *state = (arm_thread_state64_t *)(void *)(hdr + 1); +#else + hdr->flavor = ARM_THREAD_STATE; + hdr->count = ARM_THREAD_STATE_COUNT; + arm_thread_state_t *state = (arm_thread_state_t *)(void *)(hdr + 1); +#endif + + tc->cmd = LC_THREAD; + tc->cmdsize = (uint32_t) size; + + if ((cpudatap != NULL) && (cpudatap->halt_status == CPU_HALTED_WITH_STATE)) { + *state = cpudatap->halt_state; + return; + } + + if ((cpudatap == NULL) || (cpudatap->cpu_processor == NULL) || (cpudatap->cpu_processor->active_thread == NULL)) { + bzero(state, hdr->count * sizeof(uint32_t)); + return; + } + + vm_offset_t kstackptr = (vm_offset_t) cpudatap->cpu_processor->active_thread->machine.kstackptr; + arm_saved_state_t *saved_state = (arm_saved_state_t *) kstackptr; + +#if defined(__arm64__) + + state->fp = saved_state->ss_64.fp; + state->lr = saved_state->ss_64.lr; + state->sp = saved_state->ss_64.sp; + state->pc = saved_state->ss_64.pc; + state->cpsr = saved_state->ss_64.cpsr; + bcopy(&saved_state->ss_64.x[0], &state->x[0], sizeof(state->x)); + +#else /* __arm64__ */ + + state->lr = saved_state->lr; + state->sp = saved_state->sp; + state->pc = saved_state->pc; + state->cpsr = saved_state->cpsr; + bcopy(&saved_state->r[0], &state->r[0], sizeof(state->r)); + +#endif /* !__arm64__ */ + + +} + + diff --git a/osfmk/kdp/ml/i386/kdp_x86_common.c b/osfmk/kdp/ml/i386/kdp_x86_common.c index b576ec666..a8b69d57b 100644 --- a/osfmk/kdp/ml/i386/kdp_x86_common.c +++ b/osfmk/kdp/ml/i386/kdp_x86_common.c @@ -408,10 +408,15 @@ kdp_machine_init(void) { return; vm_map_entry_t e; - kern_return_t kr = vm_map_find_space(kernel_map, - &debugger_window_kva, - PAGE_SIZE, 0, - VM_MAKE_TAG(VM_KERN_MEMORY_OSFMK), &e); + kern_return_t kr; + + kr = vm_map_find_space(kernel_map, + &debugger_window_kva, + PAGE_SIZE, 0, + 0, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_OSFMK, + &e); if (kr != KERN_SUCCESS) { panic("%s: vm_map_find_space failed with %d\n", __FUNCTION__, kr); diff --git a/osfmk/kdp/ml/x86_64/kdp_machdep.c b/osfmk/kdp/ml/x86_64/kdp_machdep.c index 0bea2c52b..0d716b5d1 100644 --- a/osfmk/kdp/ml/x86_64/kdp_machdep.c +++ b/osfmk/kdp/ml/x86_64/kdp_machdep.c @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include /* for PE_halt_restart */ @@ -64,14 +63,11 @@ extern vm_map_t kernel_map; void print_saved_state(void *); void kdp_call(void); int kdp_getc(void); -boolean_t kdp_call_kdb(void); void kdp_getstate(x86_thread_state64_t *); void kdp_setstate(x86_thread_state64_t *); void kdp_print_phys(int); unsigned machine_read64(addr64_t srcaddr, caddr_t dstaddr, uint32_t len); -static void kdp_callouts(kdp_event_t event); - void kdp_exception( unsigned char *pkt, @@ -285,18 +281,6 @@ kdp_panic( __asm__ volatile("hlt"); } - -void -kdp_machine_reboot(void) -{ - printf("Attempting system restart..."); - /* Call the platform specific restart*/ - if (PE_halt_restart) - (*PE_halt_restart)(kPERestartCPU); - /* If we do reach this, give up */ - halt_all_cpus(TRUE); -} - int kdp_intr_disbl(void) { @@ -383,7 +367,8 @@ kdp_i386_trap( vm_offset_t va ) { - unsigned int exception, subcode = 0, code; + unsigned int exception, code, subcode = 0; + boolean_t prev_interrupts_state; if (trapno != T_INT3 && trapno != T_DEBUG) { kprintf("Debugger: Unexpected kernel trap number: " @@ -391,10 +376,10 @@ kdp_i386_trap( trapno, saved_state->isf.rip, saved_state->cr2); if (!kdp.is_conn) return FALSE; - } + } - mp_kdp_enter(); - kdp_callouts(KDP_EVENT_ENTER); + prev_interrupts_state = ml_set_interrupts_enabled(FALSE); + disable_preemption(); if (saved_state->isf.rflags & EFL_TF) { enable_preemption_no_check(); @@ -468,14 +453,10 @@ kdp_i386_trap( saved_state = current_cpu_datap()->cpu_fatal_trap_state; } - if (debugger_callback) { - unsigned int initial_not_in_kdp = not_in_kdp; - not_in_kdp = 0; - debugger_callback->error = debugger_callback->callback(debugger_callback->callback_context); - not_in_kdp = initial_not_in_kdp; - } else { - kdp_raise_exception(exception, code, subcode, saved_state); - } + handle_debugger_trap(exception, code, subcode, saved_state); + + enable_preemption(); + ml_set_interrupts_enabled(prev_interrupts_state); /* If the instruction single step bit is set, disable kernel preemption */ @@ -483,19 +464,9 @@ kdp_i386_trap( disable_preemption(); } - kdp_callouts(KDP_EVENT_EXIT); - mp_kdp_exit(); - return TRUE; } -boolean_t -kdp_call_kdb( - void) -{ - return(FALSE); -} - void kdp_machine_get_breakinsn( uint8_t *bytes, @@ -693,53 +664,6 @@ machine_trace_thread64(thread_t thread, return (uint32_t) (((char *) tracebuf) - tracepos); } -static struct kdp_callout { - struct kdp_callout *callout_next; - kdp_callout_fn_t callout_fn; - void *callout_arg; -} *kdp_callout_list = NULL; - - -/* - * Called from kernel context to register a kdp event callout. - */ -void -kdp_register_callout( - kdp_callout_fn_t fn, - void *arg) -{ - struct kdp_callout *kcp; - struct kdp_callout *list_head; - - kcp = kalloc(sizeof(*kcp)); - if (kcp == NULL) - panic("kdp_register_callout() kalloc failed"); - - kcp->callout_fn = fn; - kcp->callout_arg = arg; - - /* Lock-less list insertion using compare and exchange. */ - do { - list_head = kdp_callout_list; - kcp->callout_next = list_head; - } while (!OSCompareAndSwapPtr(list_head, kcp, (void * volatile *)&kdp_callout_list)); -} - -/* - * Called at exception/panic time when extering or exiting kdp. - * We are single-threaded at this time and so we don't use locks. - */ -static void -kdp_callouts(kdp_event_t event) -{ - struct kdp_callout *kcp = kdp_callout_list; - - while (kcp) { - kcp->callout_fn(kcp->callout_arg, event); - kcp = kcp->callout_next; - } -} - void kdp_ml_enter_debugger(void) { diff --git a/osfmk/kdp/ml/x86_64/kdp_vm.c b/osfmk/kdp/ml/x86_64/kdp_vm.c index 89bc4778b..244ac7d07 100644 --- a/osfmk/kdp/ml/x86_64/kdp_vm.c +++ b/osfmk/kdp/ml/x86_64/kdp_vm.c @@ -44,10 +44,10 @@ static const x86_state_hdr_t thread_flavor_array [] = { }; void -kern_collectth_state_size(uint32_t * tstate_count, size_t * ptstate_size) +kern_collectth_state_size(uint64_t * tstate_count, uint64_t * ptstate_size) { unsigned int i; - size_t tstate_size = 0; + uint64_t tstate_size = 0; for (i = 0; i < sizeof(thread_flavor_array)/sizeof(thread_flavor_array[0]); i++) tstate_size += sizeof(x86_state_hdr_t) + @@ -58,11 +58,10 @@ kern_collectth_state_size(uint32_t * tstate_count, size_t * ptstate_size) } void -kern_collectth_state(thread_t thread, void *buffer, size_t size, void ** iter) +kern_collectth_state(thread_t thread, void *buffer, uint64_t size, void ** iter) { size_t hoffset; - size_t tstate_size; - uint32_t tstate_count; + uint64_t tstate_size, tstate_count; unsigned int i; struct thread_command *tc; diff --git a/osfmk/kdp/processor_core.c b/osfmk/kdp/processor_core.c new file mode 100644 index 000000000..e6ebc85a2 --- /dev/null +++ b/osfmk/kdp/processor_core.c @@ -0,0 +1,738 @@ +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_KDP_INTERACTIVE_DEBUGGING + +#define roundup(x, y) ((((x) % (y)) == 0) ? \ + (x) : ((x) + ((y) - ((x) % (y))))) + +/* + * The processor_core_context structure describes the current + * corefile that's being generated. It also includes a pointer + * to the core_outvars which is used by the KDP code for context + * about the specific output mechanism being used. + * + * We include *remaining variables to catch inconsistencies / bugs + * in the co-processor coredump callbacks. + */ +typedef struct { + struct kdp_core_out_vars * core_outvars; /* Output procedure info (see kdp_core.c) */ + kern_coredump_callback_config *core_config; /* Information about core currently being dumped */ + void *core_refcon; /* Reference constant associated with the coredump helper */ + boolean_t core_is64bit; /* Bitness of CPU */ + uint32_t core_mh_magic; /* Magic for mach header */ + cpu_type_t core_cpu_type; /* CPU type for mach header */ + cpu_subtype_t core_cpu_subtype; /* CPU subtype for mach header */ + uint64_t core_file_length; /* Overall corefile length including any zero padding */ + uint64_t core_file_length_compressed; /* File length after compression */ + uint64_t core_segment_count; /* Number of LC_SEGMENT*s in the core currently being dumped */ + uint64_t core_segments_remaining; /* Number of LC_SEGMENT*s that have not been added to the header */ + uint64_t core_segment_byte_total; /* Sum of all the data from the LC_SEGMENTS in the core */ + uint64_t core_segment_bytes_remaining; /* Quantity of data remaining from LC_SEGMENTs that have yet to be added */ + uint64_t core_thread_count; /* Number of LC_THREADs to be included */ + uint64_t core_threads_remaining; /* Number of LC_THREADs that have yet to be included */ + uint64_t core_thread_state_size; /* Size of each LC_THREAD */ + uint64_t core_misc_bytes_count; /* Quantity of LC_NOTE data to be included */ + uint64_t core_misc_bytes_remaining; /* Quantity of LC_NOTE data that has not yet been included */ + uint64_t core_cur_hoffset; /* Current offset in this core's header */ + uint64_t core_cur_foffset; /* Current offset in this core's overall file */ + uint64_t core_header_size; /* Size of this core's header */ + uint64_t core_total_bytes; /* Total amount of data to be included in this core (excluding zero fill) */ +} processor_core_context; + +/* + * The kern_coredump_core structure describes a core that has been + * registered for use by the coredump mechanism. + */ +struct kern_coredump_core { + struct kern_coredump_core *kcc_next; /* Next processor to dump */ + void *kcc_refcon; /* Reference constant to be passed to callbacks */ + char kcc_corename[MACH_CORE_FILEHEADER_NAMELEN]; /* Description of this processor */ + boolean_t kcc_is64bit; /* Processor bitness */ + uint32_t kcc_mh_magic; /* Magic for mach header */ + cpu_type_t kcc_cpu_type; /* CPU type for mach header */ + cpu_subtype_t kcc_cpu_subtype; /* CPU subtype for mach header */ + kern_coredump_callback_config kcc_cb; /* Registered processor callbacks for coredump */ +} * kern_coredump_core_list = NULL; + +uint32_t coredump_registered_count = 0; + +struct kern_coredump_core *kernel_helper = NULL; + +static struct kern_coredump_core * +kern_register_coredump_helper_internal(int kern_coredump_config_vers, kern_coredump_callback_config *kc_callbacks, + void *refcon, const char *core_description, boolean_t xnu_callback, boolean_t is64bit, + uint32_t mh_magic, cpu_type_t cpu_type, cpu_subtype_t cpu_subtype) +{ + struct kern_coredump_core *core_helper = NULL; + kern_coredump_callback_config *core_callbacks = NULL; + + if (kern_coredump_config_vers < KERN_COREDUMP_MIN_CONFIG_VERSION) + return NULL; + if (kc_callbacks == NULL) + return NULL;; + if (core_description == NULL) + return NULL; + + if (kc_callbacks->kcc_coredump_get_summary == NULL || + kc_callbacks->kcc_coredump_save_segment_descriptions == NULL || + kc_callbacks->kcc_coredump_save_segment_data == NULL || + kc_callbacks->kcc_coredump_save_thread_state == NULL || + kc_callbacks->kcc_coredump_save_sw_vers == NULL) + return NULL; + +#if !defined(__LP64__) + /* We don't support generating 64-bit cores on 32-bit platforms */ + if (is64bit) + return NULL; +#endif + + core_helper = kalloc(sizeof(*core_helper)); + core_helper->kcc_next = NULL; + core_helper->kcc_refcon = refcon; + if (xnu_callback) { + snprintf((char *)&core_helper->kcc_corename, MACH_CORE_FILEHEADER_NAMELEN, "%s", core_description); + } else { + /* Make sure there's room for the -coproc suffix (16 - NULL char - strlen(-coproc)) */ + snprintf((char *)&core_helper->kcc_corename, MACH_CORE_FILEHEADER_NAMELEN, "%.8s-coproc", core_description); + } + core_helper->kcc_is64bit = is64bit; + core_helper->kcc_mh_magic = mh_magic; + core_helper->kcc_cpu_type = cpu_type; + core_helper->kcc_cpu_subtype = cpu_subtype; + core_callbacks = &core_helper->kcc_cb; + + core_callbacks->kcc_coredump_init = kc_callbacks->kcc_coredump_init; + core_callbacks->kcc_coredump_get_summary = kc_callbacks->kcc_coredump_get_summary; + core_callbacks->kcc_coredump_save_segment_descriptions = kc_callbacks->kcc_coredump_save_segment_descriptions; + core_callbacks->kcc_coredump_save_segment_data = kc_callbacks->kcc_coredump_save_segment_data; + core_callbacks->kcc_coredump_save_thread_state = kc_callbacks->kcc_coredump_save_thread_state; + core_callbacks->kcc_coredump_save_misc_data = kc_callbacks->kcc_coredump_save_misc_data; + core_callbacks->kcc_coredump_save_sw_vers = kc_callbacks->kcc_coredump_save_sw_vers; + + if (xnu_callback) { + assert(kernel_helper == NULL); + kernel_helper = core_helper; + } else { + do { + core_helper->kcc_next = kern_coredump_core_list; + } while (!OSCompareAndSwapPtr(kern_coredump_core_list, core_helper, &kern_coredump_core_list)); + } + + OSAddAtomic(1, &coredump_registered_count); + kprintf("Registered coredump handler for %s\n", core_description); + + return core_helper; +} + +kern_return_t +kern_register_coredump_helper(int kern_coredump_config_vers, kern_coredump_callback_config *kc_callbacks, + void *refcon, const char *core_description, boolean_t is64bit, uint32_t mh_magic, + cpu_type_t cpu_type, cpu_subtype_t cpu_subtype) +{ + if (coredump_registered_count >= KERN_COREDUMP_MAX_CORES) + return KERN_RESOURCE_SHORTAGE; + + if (kern_register_coredump_helper_internal(kern_coredump_config_vers, kc_callbacks, refcon, core_description, FALSE, + is64bit, mh_magic, cpu_type, cpu_subtype) == NULL) + return KERN_INVALID_ARGUMENT; + + return KERN_SUCCESS; +} + +kern_return_t +kern_register_xnu_coredump_helper(kern_coredump_callback_config *kc_callbacks) +{ +#if defined(__LP64__) + boolean_t is64bit = TRUE; +#else + boolean_t is64bit = FALSE; +#endif + + if (kern_register_coredump_helper_internal(KERN_COREDUMP_CONFIG_VERSION, kc_callbacks, NULL, "kernel", TRUE, is64bit, + _mh_execute_header.magic, _mh_execute_header.cputype, _mh_execute_header.cpusubtype) == NULL) + return KERN_FAILURE; + + return KERN_SUCCESS; +} + +/* + * Save metadata about the core we're about to write, write out the mach header + */ +static int +coredump_save_summary(uint64_t core_segment_count, uint64_t core_byte_count, + uint64_t thread_count, uint64_t thread_state_size, + uint64_t misc_bytes_count, void *context) +{ + processor_core_context *core_context = (processor_core_context *)context; + uint32_t sizeofcmds = 0, numcmds = 0; + int ret = 0; + + if (!core_segment_count || !core_byte_count || !thread_count || !thread_state_size + || (thread_state_size > KERN_COREDUMP_THREADSIZE_MAX)) + return KERN_INVALID_ARGUMENT; + + /* Initialize core_context */ + core_context->core_segments_remaining = core_context->core_segment_count = core_segment_count; + core_context->core_segment_bytes_remaining = core_context->core_segment_byte_total = core_byte_count; + core_context->core_threads_remaining = core_context->core_thread_count = thread_count; + core_context->core_thread_state_size = thread_state_size; + core_context->core_misc_bytes_remaining = core_context->core_misc_bytes_count = misc_bytes_count; + + +#if defined(__LP64__) + if (core_context->core_is64bit) { + sizeofcmds = (uint32_t)(core_context->core_segment_count * sizeof(struct segment_command_64) + + (core_context->core_threads_remaining * core_context->core_thread_state_size) + + /* TODO: LC_NOTE */ 0 + sizeof(struct ident_command) + KERN_COREDUMP_VERSIONSTRINGMAXSIZE); + core_context->core_header_size = sizeofcmds + sizeof(struct mach_header_64); + } else +#endif /* defined(__LP64__) */ + { + sizeofcmds = (uint32_t)(core_context->core_segment_count * sizeof(struct segment_command) + + (core_context->core_threads_remaining * core_context->core_thread_state_size) + + /* TODO: LC_NOTE */ 0 + sizeof(struct ident_command) + KERN_COREDUMP_VERSIONSTRINGMAXSIZE); + core_context->core_header_size = sizeofcmds + sizeof(struct mach_header); + } + + core_context->core_total_bytes = core_context->core_header_size + core_context->core_segment_byte_total + /* TODO: LC_NOTE */ 0; + core_context->core_file_length = round_page(core_context->core_header_size) + core_context->core_segment_byte_total + /* TODO: LC_NOTE */ 0; + core_context->core_cur_foffset = round_page(core_context->core_header_size); + + numcmds = (uint32_t)(core_context->core_segment_count + core_context->core_thread_count + /* TODO: LC_NOTE */ 0 + + 1 /* ident command */); + + /* + * Reset the zstream and other output context before writing any data out. We do this here + * to update the total file length on the outvars before we start writing out. + */ + kdp_reset_output_vars(core_context->core_outvars, core_context->core_file_length); + + /* Construct core file header */ +#if defined(__LP64__) + if (core_context->core_is64bit) { + struct mach_header_64 core_header = { }; + + core_header.magic = core_context->core_mh_magic; + core_header.cputype = core_context->core_cpu_type; + core_header.cpusubtype = core_context->core_cpu_subtype; + core_header.filetype = MH_CORE; + core_header.ncmds = numcmds; + core_header.sizeofcmds = sizeofcmds; + core_header.flags = 0; + + /* Send the core_header to the output procedure */ + ret = kdp_core_output(core_context->core_outvars, sizeof(core_header), (caddr_t)&core_header); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "coredump_save_summary() : failed to write mach header : kdp_core_output(0x%p, %lu, 0x%p) returned error 0x%x\n", + core_context->core_outvars, sizeof(core_header), &core_header, ret); + return ret; + } + + core_context->core_cur_hoffset += sizeof(core_header); + } else +#endif /* defined(__LP64__) */ + { + struct mach_header core_header = { }; + + core_header.magic = core_context->core_mh_magic; + core_header.cputype = core_context->core_cpu_type; + core_header.cpusubtype = core_context->core_cpu_subtype; + core_header.filetype = MH_CORE; + core_header.ncmds = numcmds; + core_header.sizeofcmds = sizeofcmds; + core_header.flags = 0; + + /* Send the core_header to the output procedure */ + ret = kdp_core_output(core_context->core_outvars, sizeof(core_header), (caddr_t)&core_header); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "coredump_save_summary() : failed to write mach header : kdp_core_output(0x%p, %lu, 0x%p) returned error 0x%x\n", + core_context->core_outvars, sizeof(core_header), &core_header, ret); + return ret; + } + + core_context->core_cur_hoffset += sizeof(core_header); + } + + return KERN_SUCCESS; +} + +/* + * Construct a segment command for the specified segment. + */ +static int +coredump_save_segment_descriptions(uint64_t seg_start, uint64_t seg_end, + void *context) +{ + processor_core_context *core_context = (processor_core_context *)context; + int ret; + uint64_t size = seg_end - seg_start; + + if (seg_end <= seg_start) { + kern_coredump_log(context, "coredump_save_segment_descriptions(0x%llx, 0x%llx, 0x%p) : called with invalid addresses : start 0x%llx >= end 0x%llx\n", + seg_start, seg_end, context, seg_start, seg_end); + return KERN_INVALID_ARGUMENT; + } + + if (core_context->core_segments_remaining == 0) { + kern_coredump_log(context, "coredump_save_segment_descriptions(0x%llx, 0x%llx, 0x%p) : coredump_save_segment_descriptions() called too many times, %llu segment descriptions already recorded\n", + seg_start, seg_end, context, core_context->core_segment_count); + return KERN_INVALID_ARGUMENT; + } + + /* Construct segment command */ +#if defined(__LP64__) + if (core_context->core_is64bit) { + struct segment_command_64 seg_command = { }; + + if (core_context->core_cur_hoffset + sizeof(seg_command) > core_context->core_header_size) { + kern_coredump_log(context, "coredump_save_segment_descriptions(0x%llx, 0x%llx, 0x%p) : ran out of space to save commands with %llu of %llu remaining\n", + seg_start, seg_end, context, core_context->core_segments_remaining, core_context->core_segment_count); + return KERN_NO_SPACE; + } + + seg_command.cmd = LC_SEGMENT_64; + seg_command.cmdsize = sizeof(seg_command); + seg_command.segname[0] = 0; + seg_command.vmaddr = seg_start; + seg_command.vmsize = size; + seg_command.fileoff = core_context->core_cur_foffset; + seg_command.filesize = size; + seg_command.maxprot = VM_PROT_READ; + seg_command.initprot = VM_PROT_READ; + + /* Flush new command to output */ + ret = kdp_core_output(core_context->core_outvars, sizeof(seg_command), (caddr_t)&seg_command); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "coredump_save_segment_descriptions(0x%llx, 0x%llx, 0x%p) : failed to write segment %llu of %llu. kdp_core_output(0x%p, %lu, 0x%p) returned error %d\n", + seg_start, seg_end, context, core_context->core_segment_count - core_context->core_segments_remaining, + core_context->core_segment_count, core_context->core_outvars, sizeof(seg_command), &seg_command, ret); + return ret; + } + + core_context->core_cur_hoffset += sizeof(seg_command); + } else +#endif /* defined(__LP64__) */ + { + struct segment_command seg_command = { }; + + if (seg_start > UINT32_MAX || seg_end > UINT32_MAX) { + kern_coredump_log(context, "coredump_save_segment_descriptions(0x%llx, 0x%llx, 0x%p) : called with invalid addresses for 32-bit : start 0x%llx, end 0x%llx\n", + seg_start, seg_end, context, seg_start, seg_end); + return KERN_INVALID_ARGUMENT; + } + + if (core_context->core_cur_hoffset + sizeof(seg_command) > core_context->core_header_size) { + kern_coredump_log(context, "coredump_save_segment_descriptions(0x%llx, 0x%llx, 0x%p) : ran out of space to save commands with %llu of %llu remaining\n", + seg_start, seg_end, context, core_context->core_segments_remaining, core_context->core_segment_count); + return KERN_NO_SPACE; + } + + seg_command.cmd = LC_SEGMENT; + seg_command.cmdsize = sizeof(seg_command); + seg_command.segname[0] = 0; + seg_command.vmaddr = (uint32_t) seg_start; + seg_command.vmsize = (uint32_t) size; + seg_command.fileoff = (uint32_t) core_context->core_cur_foffset; + seg_command.filesize = (uint32_t) size; + seg_command.maxprot = VM_PROT_READ; + seg_command.initprot = VM_PROT_READ; + + /* Flush new command to output */ + ret = kdp_core_output(core_context->core_outvars, sizeof(seg_command), (caddr_t)&seg_command); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "coredump_save_segment_descriptions(0x%llx, 0x%llx, 0x%p) : failed to write segment %llu of %llu : kdp_core_output(0x%p, %lu, 0x%p) returned error 0x%x\n", + seg_start, seg_end, context, core_context->core_segment_count - core_context->core_segments_remaining, + core_context->core_segment_count, core_context->core_outvars, sizeof(seg_command), &seg_command, ret); + return ret; + } + + core_context->core_cur_hoffset += sizeof(seg_command); + } + + /* Update coredump context */ + core_context->core_segments_remaining--; + core_context->core_cur_foffset += size; + + return KERN_SUCCESS; +} + +/* + * Save thread state. + * + * Passed thread_state is expected to be a struct thread_command + */ +static int +coredump_save_thread_state(void *thread_state, void *context) +{ + processor_core_context *core_context = (processor_core_context *)context; + struct thread_command *tc = (struct thread_command *)thread_state; + int ret; + + if (tc->cmd != LC_THREAD) { + kern_coredump_log(context, "coredump_save_thread_state(0x%p, 0x%p) : found %d expected LC_THREAD (%d)\n", + thread_state, context, tc->cmd, LC_THREAD); + return KERN_INVALID_ARGUMENT; + } + + if (core_context->core_cur_hoffset + core_context->core_thread_state_size > core_context->core_header_size) { + kern_coredump_log(context, "coredump_save_thread_state(0x%p, 0x%p) : ran out of space to save threads with %llu of %llu remaining\n", + thread_state, context, core_context->core_threads_remaining, core_context->core_thread_count); + return KERN_NO_SPACE; + } + + ret = kdp_core_output(core_context->core_outvars, core_context->core_thread_state_size, (caddr_t)thread_state); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "coredump_save_thread_state(0x%p, 0x%p) : failed to write thread data : kdp_core_output(0x%p, %llu, 0x%p) returned 0x%x\n", + thread_state, context, core_context->core_outvars, core_context->core_thread_state_size, thread_state, ret); + return ret; + } + + core_context->core_threads_remaining--; + core_context->core_cur_hoffset += core_context->core_thread_state_size; + + return KERN_SUCCESS; +} + +static int +coredump_save_sw_vers(void *sw_vers, uint64_t length, void *context) +{ + processor_core_context *core_context = (processor_core_context *)context; + struct ident_command ident = { }; + int ret; + + if (length > KERN_COREDUMP_VERSIONSTRINGMAXSIZE || !length) { + kern_coredump_log(context, "coredump_save_sw_vers(0x%p, %llu, 0x%p) : called with invalid length %llu\n", + sw_vers, length, context, length); + return KERN_INVALID_ARGUMENT; + } + + if (core_context->core_cur_hoffset + sizeof(struct ident_command) + length > core_context->core_header_size) { + kern_coredump_log(context, "coredump_save_sw_vers(0x%p, %llu, 0x%p) : ran out of space to save data\n", + sw_vers, length, context); + return KERN_NO_SPACE; + } + + ident.cmd = LC_IDENT; + ident.cmdsize = (uint32_t)(sizeof(struct ident_command) + KERN_COREDUMP_VERSIONSTRINGMAXSIZE); + ret = kdp_core_output(core_context->core_outvars, sizeof(struct ident_command), (caddr_t)&ident); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "coredump_save_sw_vers(0x%p, %llu, 0x%p) : failed to write ident command : kdp_core_output(0x%p, %lu, 0x%p) returned 0x%x\n", + sw_vers, length, context, core_context->core_outvars, sizeof(struct ident_command), &ident, ret); + return ret; + } + + ret = kdp_core_output(core_context->core_outvars, length, (caddr_t)sw_vers); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "coredump_save_sw_vers(0x%p, %llu, 0x%p) : failed to write version string : kdp_core_output(0x%p, %llu, 0x%p) returned 0x%x\n", + sw_vers, length, context, core_context->core_outvars, length, sw_vers, ret); + return ret; + } + + if (length < KERN_COREDUMP_VERSIONSTRINGMAXSIZE) { + /* Zero fill to the full command size */ + ret = kdp_core_output(core_context->core_outvars, (KERN_COREDUMP_VERSIONSTRINGMAXSIZE - length), NULL); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "coredump_save_sw_vers(0x%p, %llu, 0x%p) : failed to write zero fill padding : kdp_core_output(0x%p, %llu, NULL) returned 0x%x\n", + sw_vers, length, context, core_context->core_outvars, (KERN_COREDUMP_VERSIONSTRINGMAXSIZE - length), ret); + return ret; + } + } + + core_context->core_cur_hoffset += sizeof(struct ident_command) + KERN_COREDUMP_VERSIONSTRINGMAXSIZE; + + return KERN_SUCCESS; +} + +static int +coredump_save_segment_data(void *seg_data, uint64_t length, void *context) +{ + int ret; + processor_core_context *core_context = (processor_core_context *)context; + + if (length > core_context->core_segment_bytes_remaining) { + kern_coredump_log(context, "coredump_save_segment_data(0x%p, %llu, 0x%p) : called with too much data, %llu written, %llu left\n", + seg_data, length, context, core_context->core_segment_byte_total - core_context->core_segment_bytes_remaining, + core_context->core_segment_bytes_remaining); + return KERN_INVALID_ARGUMENT; + } + + ret = kdp_core_output(core_context->core_outvars, length, (caddr_t)seg_data); + if (ret != KERN_SUCCESS) { + kern_coredump_log(context, "coredump_save_segment_data(0x%p, %llu, 0x%p) : failed to write data (%llu bytes remaining) :%d\n", + seg_data, length, context, core_context->core_segment_bytes_remaining, ret); + return ret; + } + + core_context->core_segment_bytes_remaining -= length; + core_context->core_cur_foffset += length; + + return KERN_SUCCESS; +} + +static kern_return_t +kern_coredump_routine(void *core_outvars, struct kern_coredump_core *current_core, uint64_t core_begin_offset, uint64_t *core_file_length, boolean_t *header_update_failed) +{ + kern_return_t ret; + processor_core_context context = { }; + *core_file_length = 0; + *header_update_failed = FALSE; + + /* Setup the coredump context */ + context.core_outvars = core_outvars; + context.core_config = ¤t_core->kcc_cb; + context.core_refcon = current_core->kcc_refcon; + context.core_is64bit = current_core->kcc_is64bit; + context.core_mh_magic = current_core->kcc_mh_magic; + context.core_cpu_type = current_core->kcc_cpu_type; + context.core_cpu_subtype = current_core->kcc_cpu_subtype; + + kern_coredump_log(&context, "\nBeginning coredump of %s\n", current_core->kcc_corename); + + if (current_core->kcc_cb.kcc_coredump_init != NULL) { + ret = current_core->kcc_cb.kcc_coredump_init(context.core_refcon, &context); + if (ret == KERN_NODE_DOWN) { + kern_coredump_log(&context, "coredump_init returned KERN_NODE_DOWN, skipping this core\n"); + return KERN_SUCCESS; + } else if (ret != KERN_SUCCESS) { + kern_coredump_log(&context, "(kern_coredump_routine) : coredump_init failed with %d\n", ret); + return ret; + } + } + + /* Populate the context with metadata about the corefile (cmd info, sizes etc) */ + ret = current_core->kcc_cb.kcc_coredump_get_summary(context.core_refcon, coredump_save_summary, &context); + if (ret != KERN_SUCCESS) { + kern_coredump_log(&context, "(kern_coredump_routine) : get_summary failed with %d\n", ret); + return ret; + } + + if (context.core_header_size == 0) { + kern_coredump_log(&context, "(kern_coredump_routine) : header size not populated after coredump_get_summary\n"); + return KERN_FAILURE; + } + + /* Save the segment descriptions for the segments to be included */ + ret = current_core->kcc_cb.kcc_coredump_save_segment_descriptions(context.core_refcon, coredump_save_segment_descriptions, + &context); + if (ret != KERN_SUCCESS) { + kern_coredump_log(&context, "(kern_coredump_routine) : save_segment_descriptions failed with %d\n", ret); + return ret; + } + + if (context.core_segments_remaining != 0) { + kern_coredump_log(&context, "(kern_coredump_routine) : save_segment_descriptions returned without all segment descriptions written, %llu of %llu remaining\n", + context.core_segments_remaining, context.core_segment_count); + return KERN_FAILURE; + } + + /* TODO: Add LC_NOTE command for miscellaneous data if requested */ + + /* + * Save the thread commands/state + * + * TODO: Should this buffer be allocated at boot rather than on the stack? + */ + if (context.core_thread_state_size) { + char threadstatebuf[context.core_thread_state_size]; + ret = current_core->kcc_cb.kcc_coredump_save_thread_state(context.core_refcon, &threadstatebuf, coredump_save_thread_state, + &context); + if (ret != KERN_SUCCESS) { + kern_coredump_log(&context, "(kern_coredump_routine) : save_thread_state failed with %d\n", ret); + return ret; + } + } + + if (context.core_threads_remaining != 0) { + kern_coredump_log(&context, "(kern_coredump_routine) : save_thread_state returned without all thread descriptions written, %llu of %llu remaining\n", + context.core_threads_remaining, context.core_thread_count); + return KERN_FAILURE; + } + + /* Save the sw version string */ + ret = current_core->kcc_cb.kcc_coredump_save_sw_vers(context.core_refcon, coredump_save_sw_vers, &context); + if (ret != KERN_SUCCESS) { + kern_coredump_log(&context, "(kern_coredump_routine) : save_sw_vers failed with %d\n", ret); + return ret; + } + + assert(context.core_cur_hoffset == context.core_header_size); + + /* Zero fill between the end of the header and the beginning of the segment data file offset */ + ret = kdp_core_output(context.core_outvars, (round_page(context.core_header_size) - context.core_header_size), NULL); + if (ret != KERN_SUCCESS) { + kern_coredump_log(&context, "(kern_coredump_routine) : failed to write zero fill padding (%llu bytes remaining) : kdp_core_output(0x%p, %llu, NULL) returned 0x%x\n", + context.core_segment_bytes_remaining, context.core_outvars, (round_page(context.core_header_size) - context.core_header_size), ret); + return ret; + } + + context.core_cur_foffset = round_page(context.core_header_size); + ret = current_core->kcc_cb.kcc_coredump_save_segment_data(context.core_refcon, coredump_save_segment_data, &context); + if (ret != KERN_SUCCESS) { + kern_coredump_log(&context, "coredump_save_segment_data failed with %d\n", ret); + return ret; + } + + if (context.core_segment_bytes_remaining != 0) { + kern_coredump_log(&context, "(kern_coredump_routine) : save_segment_data returned without all segment data written, %llu of %llu remaining\n", + context.core_segment_bytes_remaining, context.core_segment_byte_total); + return KERN_FAILURE; + } + + /* TODO: Save the miscellaneous data if requested */ + + /* Flush the last data out */ + ret = kdp_core_output(context.core_outvars, 0, NULL); + if (ret != KERN_SUCCESS) { + kern_coredump_log(&context, "(kern_coredump_routine) : failed to flush final core data : kdp_core_output(0x%p, 0, NULL) returned 0x%x\n", + context.core_outvars, ret); + return ret; + } + + kern_coredump_log(&context, "Done\nCoredump complete of %s, dumped %llu segments (%llu bytes), %llu threads (%llu bytes) overall uncompressed file length %llu bytes.", + current_core->kcc_corename, context.core_segment_count, context.core_segment_byte_total, context.core_thread_count, + (context.core_thread_count * context.core_thread_state_size), context.core_file_length); + + if (core_begin_offset) { + /* If we're writing to disk (we have a begin offset, we need to update the header */ + ret = kern_dump_record_file(context.core_outvars, current_core->kcc_corename, core_begin_offset, &context.core_file_length_compressed); + if (ret != KERN_SUCCESS) { + *header_update_failed = TRUE; + kern_coredump_log(&context, "\n(kern_coredump_routine) : kern_dump_record_file failed with %d\n", ret); + return ret; + } + } + + kern_coredump_log(&context, " Compressed file length is %llu bytes\n", context.core_file_length_compressed); + + *core_file_length = context.core_file_length_compressed; + + return KERN_SUCCESS; +} + +kern_return_t +kern_do_coredump(void *core_outvars, boolean_t kernel_only, uint64_t first_file_offset, uint64_t *last_file_offset) +{ + struct kern_coredump_core *current_core = NULL; + uint64_t prev_core_length = 0; + kern_return_t cur_ret = KERN_SUCCESS, ret = KERN_SUCCESS; + boolean_t header_update_failed = FALSE; + + assert(last_file_offset != NULL); + + *last_file_offset = first_file_offset; + cur_ret = kern_coredump_routine(core_outvars, kernel_helper, *last_file_offset, &prev_core_length, &header_update_failed); + if (cur_ret != KERN_SUCCESS) { + // As long as we didn't fail while updating the header for the raw file, we should be able to try + // to capture other corefiles. + if (header_update_failed) { + // The header may be in an inconsistent state, so bail now + return KERN_FAILURE; + } else { + prev_core_length = 0; + ret = KERN_FAILURE; + } + } + + *last_file_offset = roundup(((*last_file_offset) + prev_core_length), KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN); + prev_core_length = 0; + + if (kernel_only) { + return ret; + } + + current_core = kern_coredump_core_list; + while (current_core) { + /* Seek to the beginning of the next file */ + ret = kern_dump_seek_to_next_file(core_outvars, *last_file_offset); + if (ret != KERN_SUCCESS) { + kern_coredump_log(NULL, "Failed to seek to beginning of next core\n"); + return KERN_FAILURE; + } + + cur_ret = kern_coredump_routine(core_outvars, current_core, *last_file_offset , &prev_core_length, &header_update_failed); + if (cur_ret != KERN_SUCCESS) { + // As long as we didn't fail while updating the header for the raw file, we should be able to try + // to capture other corefiles. + if (header_update_failed) { + // The header may be in an inconsistent state, so bail now + return KERN_FAILURE; + } else { + // Try to capture other corefiles even if one failed, update the overall return + // status though + prev_core_length = 0; + ret = KERN_FAILURE; + } + } + + /* Calculate the offset of the beginning of the next core in the raw file */ + *last_file_offset = roundup(((*last_file_offset) + prev_core_length), KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN); + prev_core_length = 0; + current_core = current_core->kcc_next; + } + + return ret; +} +#else /* CONFIG_KDP_INTERACTIVE_DEBUGGING */ + +kern_return_t +kern_register_coredump_helper(int kern_coredump_config_vers, kern_coredump_callback_config *kc_callbacks, void* refcon, + const char *core_description, boolean_t is64bit, uint32_t mh_magic, + cpu_type_t cpu_type, cpu_subtype_t cpu_subtype) +{ +#pragma unused(kern_coredump_config_vers, kc_callbacks, refcon, core_description, is64bit, mh_magic, cpu_type, cpu_subtype) + return KERN_NOT_SUPPORTED; +} +#endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */ + +/* + * Must be callable with a NULL context + */ +void +kern_coredump_log(void *context, const char *string, ...) +{ +#pragma unused(context) + va_list coredump_log_args; + + va_start(coredump_log_args, string); + _doprnt(string, &coredump_log_args, consdebug_putc, 0); + va_end(coredump_log_args); + +#if CONFIG_EMBEDDED + paniclog_flush(); +#endif +} diff --git a/osfmk/kdp/processor_core.h b/osfmk/kdp/processor_core.h new file mode 100644 index 000000000..d0fd89a60 --- /dev/null +++ b/osfmk/kdp/processor_core.h @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _PROCESSOR_CORE_H_ +#define _PROCESSOR_CORE_H_ + +#include +#include +#include +#include +#include + +__BEGIN_DECLS + +/* + * Kernel support for generating corefiles on device. + * + * The kernel provides support for co-operatively generating core files + * for any co/processors that register a coredump handler callback. + * + * The kernel will use the provided callbacks to generate a compressed + * corefile in a file on disk. + * + * Corefiles consist of three main sections + * -- The headers that describe the corefile -- number of segments, etc + * -- The segment commands that describe the data in the corefile + * -- The segment data + * + * When a coredump handler is registered, a pointer to a kern_coredump_callback_config + * structure is provided with callbacks that will be called as part of generating the + * coredump. + * + * It's expected that each of these callbacks will return 0 on success (and non-zero on + * error). + */ + +void kern_coredump_log(void *context, const char *string, ...) __printflike(2,3); + +/* + * The core_save_summary callback is provided with the call to the kcc_coredump_get_summary + * routine that was registered. The caller should provide the following + * + * core_segment_count -- Number of segments (LC_SEGMENT_KERNEL) that will be recorded + * core_byte_count -- Overall length of all data to be included across all segments + * thread_count -- Number of threads that will be recorded with thread state (LC_THREAD) + * thread_state_size -- Size of a thread's saved state (should be the overall LC_THREAD command size) + * misc_bytes_count -- Length of misc data that will be included in the core + * mh_magic -- mh_magic to be included in corefile + * cpu_type -- CPU type + * cpu_subtype -- CPU subtype + * context -- Passed to kcc_coredump_get_summary_routine + */ +typedef kern_return_t (*core_save_summary_cb)(uint64_t core_segment_count, uint64_t core_byte_count, + uint64_t thread_count, uint64_t thread_state_size, + uint64_t misc_bytes_count, void *context); + +/* + * The core_save_segment_descriptions callback is provided with the call to the + * kcc_coredump_save_segment_descriptions routine that was registered. + * + * It's expected that the caller should iterate all of the segments they want to include in + * the corefile and call the callback with the following for each: + * + * Please note that seg_end is address of the byte immediately following the last byte in the segment. + * For example, if a segment spans addresses 0x1000 to 0x1FFF, seg_end would be 0x2000. + * + * seg_start -- Start of the segment in the core's address space + * seg_end -- End of the segment in the core's address space + * context -- Passed to kcc_coredump_save_segment_descriptions routine + */ +typedef kern_return_t (*core_save_segment_descriptions_cb)(uint64_t seg_start, uint64_t seg_end, + void *context); +/* + * The core_save_thread_state callback is provided with the call to the + * kcc_coredump_save_thread_state routine that was registered. + * + * The routine is provided a pointer to a buffer of thread_state_size (as specified + * previously) that can be used to populate thread state. + * + * It's expected that the caller should iterate all of the threads + * that they would like to include and call the callback with the following + * for each: + * + * thread_state -- A pointer to the buffer with an LC_THREAD command + * context -- Passed to kcc_coredump_save_thread_state routine + */ +typedef kern_return_t (*core_save_thread_state_cb)(void *thread_state, void *context); + +/* + * The core_save_sw_vers callback is provided with the call to the + * kcc_coredump_save_sw_vers routine that was registered. + * + * The caller should call the callback with the following: + * + * sw_vers -- A pointer the software version information + * length -- Length of the software version information to be copied (< KERN_COREDUMP_VERSIONSTRINGMAXSIZE) + * context -- Passed to kcc_coredump_save_sw_vers routine + */ +typedef kern_return_t (*core_save_sw_vers_cb)(void *sw_vers, uint64_t length, void *context); + +/* + * The core_save_segment_data callback is provided with the call to the + * kcc_coredump_save_segment_data routine that was registered. + * + * It's expected that the caller should iterate all of the segments they want to include in + * the corefile and call the callback with the following for each: + * + * seg_data -- A pointer to the segment data (mapped in the kernel's address space) + * length -- Length of the data to be copied from the segment + * context -- Passed to kcc_coredump_save_segment_data routine + */ +typedef kern_return_t (*core_save_segment_data_cb)(void *seg_data, uint64_t length, void *context); + +/* + * ---------------------- OPTIONAL ------------------------- + * The core_save_misc_data callback is provided with the call to the + * kcc_coredump_save_misc_data routine that was registered + * + * The caller should call the callback with the following: + * + * misc_data -- A pointer to the data to be copied + * length -- The length of the data to be copied + * context -- Passed to kcc_coredump_save_misc_data routine + */ +typedef kern_return_t (*core_save_misc_data_cb)(void *misc_data, uint64_t length, void *context); + +typedef struct { + kern_return_t (*kcc_coredump_init)(void *refcon, void *context); /* OPTIONAL -- return KERN_NODE_DOWN if the co-processor should be skipped */ + kern_return_t (*kcc_coredump_get_summary)(void *refcon, core_save_summary_cb callback, void *context); + kern_return_t (*kcc_coredump_save_segment_descriptions)(void *refcon, core_save_segment_descriptions_cb callback, void *context); + kern_return_t (*kcc_coredump_save_thread_state)(void *refcon, void *buf, core_save_thread_state_cb callback, void *context); + kern_return_t (*kcc_coredump_save_sw_vers)(void *refcon, core_save_sw_vers_cb callback, void *context); + kern_return_t (*kcc_coredump_save_segment_data)(void *refcon, core_save_segment_data_cb callback, void *context); + kern_return_t (*kcc_coredump_save_misc_data)(void *refcon, core_save_misc_data_cb callback, void *context); /* OPTIONAL */ + /* End of version 1 */ +} kern_coredump_callback_config; + +#define KERN_COREDUMP_MAX_CORES MACH_CORE_FILEHEADER_MAXFILES +#define KERN_COREDUMP_MIN_CONFIG_VERSION 1 +#define KERN_COREDUMP_CONFIG_VERSION 1 +#define KERN_COREDUMP_VERSIONSTRINGMAXSIZE 256 + +/* + * kern_register_coredump_helper is called to register a core with the kernel + * coredump infrastructure. In addition to the callback config and version of the config + * structure, a description of the core should be provided -- i.e.: AP + */ +kern_return_t kern_register_coredump_helper(int kern_coredump_config_vers, kern_coredump_callback_config *kc_callbacks, void *refcon, + const char *core_description, boolean_t is64bit, uint32_t mh_magic, cpu_type_t cpu_type, cpu_subtype_t cpu_subtype); + +#if PRIVATE + +kern_return_t kern_register_xnu_coredump_helper(kern_coredump_callback_config *kc_callbacks); + +kern_return_t kern_do_coredump(void *core_outvars, boolean_t kernel_only, uint64_t first_file_offset, uint64_t *last_file_offset); + +#define KERN_COREDUMP_HEADERSIZE 4096 +static_assert((sizeof(struct mach_core_fileheader) <= KERN_COREDUMP_HEADERSIZE), "struct mach_core_fileheader larger than KERN_COREDUMP_HEADERSIZE (space that will be allocated for it in the corefile)"); + +#define KERN_COREDUMP_MAXDEBUGLOGSIZE 16384 +#define KERN_COREDUMP_BEGIN_FILEBYTES_ALIGN 4096 +#define KERN_COREDUMP_THREADSIZE_MAX 1024 + +#endif /* PRIVATE */ + +__END_DECLS +#endif /* _PROCESSOR_CORE_H_ */ diff --git a/osfmk/kern/Makefile b/osfmk/kern/Makefile index 304d7cff9..e0480272f 100644 --- a/osfmk/kern/Makefile +++ b/osfmk/kern/Makefile @@ -7,14 +7,18 @@ include $(MakeInc_cmd) include $(MakeInc_def) DATAFILES = \ + exc_guard.h \ exc_resource.h \ kern_cdata.h \ kcdata.h PRIVATE_DATAFILES = \ + cs_blobs.h \ debug.h \ ecc.h \ - block_hint.h + block_hint.h \ + monotonic.h \ + arithmetic_128.h EXPORT_FILES = \ affinity.h \ @@ -55,13 +59,17 @@ EXPORT_FILES = \ telemetry.h \ thread.h \ thread_call.h \ + thread_group.h \ timer_call.h \ waitq.h \ + work_interval.h \ zalloc.h PRIVATE_EXPORT_FILES = \ build_config.h \ - mach_node_link.h + mach_node_link.h \ + copyout_shim.h + INSTALL_MI_LIST = ${DATAFILES} diff --git a/osfmk/kern/affinity.c b/osfmk/kern/affinity.c index 8e6b3ee5a..bcce9af9b 100644 --- a/osfmk/kern/affinity.c +++ b/osfmk/kern/affinity.c @@ -84,8 +84,13 @@ static affinity_set_t affinity_set_remove(affinity_set_t aset, thread_t thread); * has a single pset, and last-processor affinity is * more important than pset affinity. */ +#if CONFIG_EMBEDDED +boolean_t affinity_sets_enabled = FALSE; +int affinity_sets_mapping = 0; +#else /* !CONFIG_EMBEDDED */ boolean_t affinity_sets_enabled = TRUE; int affinity_sets_mapping = 1; +#endif /* !CONFIG_EMBEDDED */ boolean_t thread_affinity_is_supported(void) diff --git a/osfmk/kern/arithmetic_128.h b/osfmk/kern/arithmetic_128.h new file mode 100644 index 000000000..8cff16692 --- /dev/null +++ b/osfmk/kern/arithmetic_128.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 1999, 2003, 2006, 2007, 2010 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ +/* + * Code duplicated from Libc/gen/nanosleep.c + */ + +#ifndef _ARITHMETIC_128_H_ +#define _ARITHMETIC_128_H_ + +#include + +#if __LP64__ + +static __inline uint64_t +multi_overflow(uint64_t a, uint64_t b) +{ + __uint128_t prod; + prod = (__uint128_t)a * (__uint128_t)b; + return (uint64_t) (prod >> 64); +} + +#else + +typedef struct { + uint64_t high; + uint64_t low; +} uint128_data_t; + +/* 128-bit addition: acc += add */ +static __inline void +add128_128(uint128_data_t *acc, uint128_data_t *add) +{ + acc->high += add->high; + acc->low += add->low; + if(acc->low < add->low) + acc->high++; // carry +} + +/* 64x64 -> 128 bit multiplication */ +static __inline void +mul64x64(uint64_t x, uint64_t y, uint128_data_t *prod) +{ + uint128_data_t add; + /* + * Split the two 64-bit multiplicands into 32-bit parts: + * x => 2^32 * x1 + x2 + * y => 2^32 * y1 + y2 + */ + uint32_t x1 = (uint32_t)(x >> 32); + uint32_t x2 = (uint32_t)x; + uint32_t y1 = (uint32_t)(y >> 32); + uint32_t y2 = (uint32_t)y; + /* + * direct multiplication: + * x * y => 2^64 * (x1 * y1) + 2^32 (x1 * y2 + x2 * y1) + (x2 * y2) + * The first and last terms are direct assignmenet into the uint128_t + * structure. Then we add the middle two terms separately, to avoid + * 64-bit overflow. (We could use the Karatsuba algorithm to save + * one multiply, but it is harder to deal with 64-bit overflows.) + */ + prod->high = (uint64_t)x1 * (uint64_t)y1; + prod->low = (uint64_t)x2 * (uint64_t)y2; + add.low = (uint64_t)x1 * (uint64_t)y2; + add.high = (add.low >> 32); + add.low <<= 32; + add128_128(prod, &add); + add.low = (uint64_t)x2 * (uint64_t)y1; + add.high = (add.low >> 32); + add.low <<= 32; + add128_128(prod, &add); +} + +static __inline uint64_t +multi_overflow(uint64_t a, uint64_t b) +{ + uint128_data_t prod; + mul64x64(a, b, &prod); + return prod.high; +} + +#endif /* __LP64__ */ +#endif /* _ARITHMETIC_128_H_ */ diff --git a/osfmk/kern/assert.h b/osfmk/kern/assert.h index f884220e9..f35eadc43 100644 --- a/osfmk/kern/assert.h +++ b/osfmk/kern/assert.h @@ -77,12 +77,7 @@ extern void Assert( extern int kext_assertions_enable; -#ifdef CONFIG_NO_PANIC_STRINGS -#define Assert(file, line, ex) (Assert)("", line, "") -#define __Panic(fmt, args...) panic("", ##args) -#else /* CONFIG_NO_PANIC_STRINGS */ #define __Panic(fmt, args...) panic(fmt, ##args) -#endif /* CONFIG_NO_PANIC_STRINGS */ __END_DECLS diff --git a/osfmk/kern/ast.c b/osfmk/kern/ast.c index 62f060d65..8f282ce58 100644 --- a/osfmk/kern/ast.c +++ b/osfmk/kern/ast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2012 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -53,21 +53,9 @@ * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ -/* - */ - -/* - * - * This file contains routines to check whether an ast is needed. - * - * ast_check() - check whether ast is needed for interrupt or context - * switch. Usually called by clock interrupt handler. - * - */ #include #include -#include #include #include #include @@ -80,205 +68,287 @@ #endif #include #include +#include #include #include -#include // for CHUD AST hook -#include #include // for MACF AST hook +#include -volatile perfASTCallback perfASTHook; - +static void __attribute__((noinline, noreturn, disable_tail_calls)) +thread_preempted(__unused void* parameter, __unused wait_result_t result) +{ + /* + * We've been scheduled again after a userspace preemption, + * try again to return to userspace. + */ + thread_exception_return(); +} +/* + * AST_URGENT was detected while in kernel mode + * Called with interrupts disabled, returns the same way + * Must return to caller + */ void -ast_init(void) +ast_taken_kernel(void) { -} + assert(ml_get_interrupts_enabled() == FALSE); -#ifdef CONFIG_DTRACE -extern void dtrace_ast(void); -#endif + thread_t thread = current_thread(); + + /* Idle threads handle preemption themselves */ + if ((thread->state & TH_IDLE)) { + ast_off(AST_PREEMPTION); + return; + } + + /* + * It's possible for this to be called after AST_URGENT + * has already been handled, due to races in enable_preemption + */ + if (ast_peek(AST_URGENT) != AST_URGENT) + return; + + /* + * Don't preempt if the thread is already preparing to block. + * TODO: the thread can cheese this with clear_wait() + */ + if (waitq_wait_possible(thread) == FALSE) { + /* Consume AST_URGENT or the interrupt will call us again */ + ast_consume(AST_URGENT); + return; + } + + /* TODO: Should we csw_check again to notice if conditions have changed? */ + + ast_t urgent_reason = ast_consume(AST_PREEMPTION); + + assert(urgent_reason & AST_PREEMPT); + + counter(c_ast_taken_block++); + + thread_block_reason(THREAD_CONTINUE_NULL, NULL, urgent_reason); + + assert(ml_get_interrupts_enabled() == FALSE); +} /* - * Called at splsched. + * An AST flag was set while returning to user mode + * Called with interrupts disabled, returns with interrupts enabled + * May call continuation instead of returning */ void -ast_taken( - ast_t reasons, - boolean_t enable -) +ast_taken_user(void) { - boolean_t preempt_trap = (reasons == AST_PREEMPTION); - ast_t *myast = ast_pending(); - thread_t thread = current_thread(); - perfASTCallback perf_hook = perfASTHook; + assert(ml_get_interrupts_enabled() == FALSE); + + thread_t thread = current_thread(); + + /* We are about to return to userspace, there must not be a pending wait */ + assert(waitq_wait_possible(thread)); + assert((thread->state & TH_IDLE) == 0); + + /* TODO: Add more 'return to userspace' assertions here */ /* - * CHUD hook - all threads including idle processor threads + * If this thread was urgently preempted in userspace, + * take the preemption before processing the ASTs. + * The trap handler will call us again if we have more ASTs, so it's + * safe to block in a continuation here. */ - if (perf_hook) { - if (*myast & AST_CHUD_ALL) { - (*perf_hook)(reasons, myast); - - if (*myast == AST_NONE) - return; - } + if (ast_peek(AST_URGENT) == AST_URGENT) { + ast_t urgent_reason = ast_consume(AST_PREEMPTION); + + assert(urgent_reason & AST_PREEMPT); + + /* TODO: Should we csw_check again to notice if conditions have changed? */ + + thread_block_reason(thread_preempted, NULL, urgent_reason); + /* NOTREACHED */ } - else - *myast &= ~AST_CHUD_ALL; - reasons &= *myast; - *myast &= ~reasons; + /* + * AST_KEVENT does not send an IPI when setting the ast for a thread running in parallel + * on a different processor. Only the ast bit on the thread will be set. + * + * Force a propagate for concurrent updates without an IPI. + */ + ast_propagate(thread); /* - * Handle ASTs for all threads - * except idle processor threads. + * Consume all non-preemption processor ASTs matching reasons + * because we're handling them here. + * + * If one of the AST handlers blocks in a continuation, + * we'll reinstate the unserviced thread-level AST flags + * from the thread to the processor on context switch. + * If one of the AST handlers sets another AST, + * the trap handler will call ast_taken_user again. + * + * We expect the AST handlers not to thread_exception_return + * without an ast_propagate or context switch to reinstate + * the per-processor ASTs. + * + * TODO: Why are AST_DTRACE and AST_KPERF not per-thread ASTs? */ - if (!(thread->state & TH_IDLE)) { - /* - * Check for urgent preemption. - */ - if ( (reasons & AST_URGENT) && - waitq_wait_possible(thread) ) { - if (reasons & AST_PREEMPT) { - counter(c_ast_taken_block++); - thread_block_reason(THREAD_CONTINUE_NULL, NULL, - reasons & AST_PREEMPTION); - } - - reasons &= ~AST_PREEMPTION; - } + ast_t reasons = ast_consume(AST_PER_THREAD | AST_KPERF | AST_DTRACE); - /* - * The kernel preempt traps - * skip all other ASTs. - */ - if (!preempt_trap) { - ml_set_interrupts_enabled(enable); + ml_set_interrupts_enabled(TRUE); #if CONFIG_DTRACE - if (reasons & AST_DTRACE) { - dtrace_ast(); - } + if (reasons & AST_DTRACE) { + dtrace_ast(); + } #endif -#ifdef MACH_BSD - /* - * Handle BSD hook. - */ - if (reasons & AST_BSD) { - thread_ast_clear(thread, AST_BSD); - bsd_ast(thread); - } +#ifdef MACH_BSD + if (reasons & AST_BSD) { + thread_ast_clear(thread, AST_BSD); + bsd_ast(thread); + } #endif + #if CONFIG_MACF - /* - * Handle MACF hook. - */ - if (reasons & AST_MACF) { - thread_ast_clear(thread, AST_MACF); - mac_thread_userret(thread); - } + if (reasons & AST_MACF) { + thread_ast_clear(thread, AST_MACF); + mac_thread_userret(thread); + } #endif - /* - * Thread APC hook. - */ - if (reasons & AST_APC) { - thread_ast_clear(thread, AST_APC); - thread_apc_ast(thread); - } - - if (reasons & AST_GUARD) { - thread_ast_clear(thread, AST_GUARD); - guard_ast(thread); - } - - if (reasons & AST_LEDGER) { - thread_ast_clear(thread, AST_LEDGER); - ledger_ast(thread); - } - - /* - * Kernel Profiling Hook - */ - if (reasons & AST_KPERF) { - thread_ast_clear(thread, AST_KPERF); - kperf_kpc_thread_ast(thread); - } + + if (reasons & AST_APC) { + thread_ast_clear(thread, AST_APC); + thread_apc_ast(thread); + } + + if (reasons & AST_GUARD) { + thread_ast_clear(thread, AST_GUARD); + guard_ast(thread); + } + + if (reasons & AST_LEDGER) { + thread_ast_clear(thread, AST_LEDGER); + ledger_ast(thread); + } + + if (reasons & AST_KPERF) { + thread_ast_clear(thread, AST_KPERF); + kperf_kpc_thread_ast(thread); + } + + if (reasons & AST_KEVENT) { + thread_ast_clear(thread, AST_KEVENT); + uint16_t bits = atomic_exchange(&thread->kevent_ast_bits, 0); + if (bits) kevent_ast(thread, bits); + } #if CONFIG_TELEMETRY - if (reasons & AST_TELEMETRY_ALL) { - boolean_t interrupted_userspace = FALSE; - boolean_t io_telemetry = FALSE; - - assert((reasons & AST_TELEMETRY_ALL) != AST_TELEMETRY_ALL); /* only one is valid at a time */ - interrupted_userspace = (reasons & AST_TELEMETRY_USER) ? TRUE : FALSE; - io_telemetry = ((reasons & AST_TELEMETRY_IO) ? TRUE : FALSE); - thread_ast_clear(thread, AST_TELEMETRY_ALL); - telemetry_ast(thread, interrupted_userspace, io_telemetry); - } + if (reasons & AST_TELEMETRY_ALL) { + ast_t telemetry_reasons = reasons & AST_TELEMETRY_ALL; + thread_ast_clear(thread, AST_TELEMETRY_ALL); + telemetry_ast(thread, telemetry_reasons); + } #endif - ml_set_interrupts_enabled(FALSE); + spl_t s = splsched(); #if CONFIG_SCHED_SFI - if (reasons & AST_SFI) { - sfi_ast(thread); - } + /* + * SFI is currently a per-processor AST, not a per-thread AST + * TODO: SFI should be a per-thread AST + */ + if (ast_consume(AST_SFI) == AST_SFI) { + sfi_ast(thread); + } #endif - /* - * Check for preemption. Conditions may have changed from when the AST_PREEMPT was originally set. - */ - thread_lock(thread); - if (reasons & AST_PREEMPT) - reasons = csw_check(current_processor(), reasons & AST_QUANTUM); - thread_unlock(thread); + /* We are about to return to userspace, there must not be a pending wait */ + assert(waitq_wait_possible(thread)); + + /* + * We've handled all per-thread ASTs, time to handle non-urgent preemption. + * + * We delay reading the preemption bits until now in case the thread + * blocks while handling per-thread ASTs. + * + * If one of the AST handlers had managed to set a new AST bit, + * thread_exception_return will call ast_taken again. + */ + ast_t preemption_reasons = ast_consume(AST_PREEMPTION); + + if (preemption_reasons & AST_PREEMPT) { + /* Conditions may have changed from when the AST_PREEMPT was originally set, so re-check. */ + + thread_lock(thread); + preemption_reasons = csw_check(current_processor(), (preemption_reasons & AST_QUANTUM)); + thread_unlock(thread); - assert(waitq_wait_possible(thread)); +#if CONFIG_SCHED_SFI + /* csw_check might tell us that SFI is needed */ + if (preemption_reasons & AST_SFI) { + sfi_ast(thread); + } +#endif - if (reasons & AST_PREEMPT) { - counter(c_ast_taken_block++); - thread_block_reason((thread_continue_t)thread_exception_return, NULL, reasons & AST_PREEMPTION); - } + if (preemption_reasons & AST_PREEMPT) { + counter(c_ast_taken_block++); + /* switching to a continuation implicitly re-enables interrupts */ + thread_block_reason(thread_preempted, NULL, preemption_reasons); + /* NOTREACHED */ } } - ml_set_interrupts_enabled(enable); + splx(s); } /* - * Called at splsched. + * Handle preemption IPI or IPI in response to setting an AST flag + * Triggered by cause_ast_check + * Called at splsched */ void -ast_check( - processor_t processor) +ast_check(processor_t processor) { + if (processor->state != PROCESSOR_RUNNING && + processor->state != PROCESSOR_SHUTDOWN) + return; + thread_t thread = processor->active_thread; - if (processor->state == PROCESSOR_RUNNING || - processor->state == PROCESSOR_SHUTDOWN) { - ast_t preempt; + assert(thread == current_thread()); - /* - * Propagate thread ast to processor. - */ - pal_ast_check(thread); + thread_lock(thread); - ast_propagate(thread->ast); + /* + * Propagate thread ast to processor. + * (handles IPI in response to setting AST flag) + */ + ast_propagate(thread); + + boolean_t needs_callout = false; + processor->current_pri = thread->sched_pri; + processor->current_sfi_class = thread->sfi_class = sfi_thread_classify(thread); + processor->current_recommended_pset_type = recommended_pset_type(thread); + perfcontrol_class_t thread_class = thread_get_perfcontrol_class(thread); + if (thread_class != processor->current_perfctl_class) { + /* We updated the perfctl class of this thread from another core. + * Since we dont do CLPC callouts from another core, do a callout + * here to let CLPC know that the currently running thread has a new + * class. + */ + needs_callout = true; + } + processor->current_perfctl_class = thread_class; - /* - * Context switch check. - */ - thread_lock(thread); + ast_t preempt; - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class = sfi_thread_classify(thread); + if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE) + ast_on(preempt); - if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE) - ast_on(preempt); + thread_unlock(thread); - thread_unlock(thread); + if (needs_callout) { + machine_switch_perfcontrol_state_update(PERFCONTROL_ATTR_UPDATE, + mach_approximate_time(), 0, thread); } } @@ -306,6 +376,37 @@ ast_off(ast_t reasons) *pending_ast &= ~reasons; } +/* + * Consume the requested subset of the AST flags set on the processor + * Return the bits that were set + * Called at splsched + */ +ast_t +ast_consume(ast_t reasons) +{ + ast_t *pending_ast = ast_pending(); + + reasons &= *pending_ast; + *pending_ast &= ~reasons; + + return reasons; +} + +/* + * Read the requested subset of the AST flags set on the processor + * Return the bits that were set, don't modify the processor + * Called at splsched + */ +ast_t +ast_peek(ast_t reasons) +{ + ast_t *pending_ast = ast_pending(); + + reasons &= *pending_ast; + + return reasons; +} + /* * Re-set current processor's per-thread AST flags to those set on thread * Called at splsched @@ -318,9 +419,20 @@ ast_context(thread_t thread) *pending_ast = ((*pending_ast & ~AST_PER_THREAD) | thread->ast); } +/* + * Propagate ASTs set on a thread to the current processor + * Called at splsched + */ +void +ast_propagate(thread_t thread) +{ + ast_on(thread->ast); +} + void ast_dtrace_on(void) { ast_on(AST_DTRACE); } + diff --git a/osfmk/kern/ast.h b/osfmk/kern/ast.h index bd9dd0bb5..4710f64ed 100644 --- a/osfmk/kern/ast.h +++ b/osfmk/kern/ast.h @@ -69,25 +69,28 @@ #include /* - * A processor takes an AST when it is about to return from an - * interrupt context, and calls ast_taken. + * A processor detects an AST when it is about to return from an + * interrupt context, and calls ast_taken_kernel or ast_taken_user + * depending on whether it was returning from userspace or kernelspace. * * Machine-dependent code is responsible for maintaining - * a set of reasons for an AST, and passing this set to ast_taken. + * a set of reasons for an AST. */ -typedef uint32_t ast_t; +typedef uint32_t ast_t; /* * When returning from interrupt/trap context to kernel mode, - * the pending ASTs are masked with AST_URGENT to determine if - * ast_taken(AST_PREEMPTION) should be called, for instance to - * effect preemption of a kernel thread by a realtime thread. + * if AST_URGENT is set, then ast_taken_kernel is called, for + * instance to effect preemption of a kernel thread by a realtime + * thread. + * * This is also done when re-enabling preemption or re-enabling * interrupts, since an AST may have been set while preemption * was disabled, and it should take effect as soon as possible. * * When returning from interrupt/trap/syscall context to user - * mode, any and all ASTs that are pending should be handled. + * mode, any and all ASTs that are pending should be handled by + * calling ast_taken_user. * * If a thread context switches, only ASTs not in AST_PER_THREAD * remain active. The per-thread ASTs are stored in the thread_t @@ -102,6 +105,8 @@ typedef uint32_t ast_t; /* * Bits for reasons + * TODO: Split the context switch and return-to-user AST namespaces + * NOTE: Some of these are exported as the 'reason' code in scheduler tracepoints */ #define AST_PREEMPT 0x01 #define AST_QUANTUM 0x02 @@ -121,6 +126,8 @@ typedef uint32_t ast_t; #define AST_SFI 0x10000 /* Evaluate if SFI wait is needed before return to userspace */ #define AST_DTRACE 0x20000 #define AST_TELEMETRY_IO 0x40000 /* telemetry sample requested for I/O */ +#define AST_KEVENT 0x80000 +#define AST_REBALANCE 0x100000 /* thread context switched due to rebalancing */ #define AST_NONE 0x00 #define AST_ALL (~AST_NONE) @@ -132,15 +139,13 @@ typedef uint32_t ast_t; #define AST_TELEMETRY_ALL (AST_TELEMETRY_USER | AST_TELEMETRY_KERNEL | AST_TELEMETRY_IO) /* Per-thread ASTs follow the thread at context-switch time. */ -#define AST_PER_THREAD (AST_APC | AST_BSD | AST_MACF | AST_LEDGER | AST_GUARD | AST_TELEMETRY_ALL ) +#define AST_PER_THREAD (AST_APC | AST_BSD | AST_MACF | AST_LEDGER | AST_GUARD | AST_TELEMETRY_ALL | AST_KEVENT) -/* Initialize module */ -extern void ast_init(void); +/* Handle AST_URGENT detected while in the kernel */ +extern void ast_taken_kernel(void); -/* Handle ASTs */ -extern void ast_taken( - ast_t mask, - boolean_t enable); +/* Handle an AST flag set while returning to user mode (may continue via thread_exception_return) */ +extern void ast_taken_user(void); /* Check for pending ASTs */ extern void ast_check(processor_t processor); @@ -154,10 +159,17 @@ extern void ast_on(ast_t reasons); /* Clear AST flags on current processor */ extern void ast_off(ast_t reasons); +/* Consume specified AST flags from current processor */ +extern ast_t ast_consume(ast_t reasons); + +/* Read specified AST flags from current processor */ +extern ast_t ast_peek(ast_t reasons); + /* Re-set current processor's per-thread AST flags to those set on thread */ extern void ast_context(thread_t thread); -#define ast_propagate(reasons) ast_on(reasons) +/* Propagate ASTs set on a thread to the current processor */ +extern void ast_propagate(thread_t thread); /* * Set an AST on a thread with thread_ast_set. @@ -179,6 +191,10 @@ extern void bsd_ast(thread_t); #ifdef CONFIG_DTRACE extern void ast_dtrace_on(void); +extern void dtrace_ast(void); #endif /* CONFIG_DTRACE */ +extern void kevent_ast(thread_t thread, uint16_t bits); +extern void act_set_astkevent(thread_t thread, uint16_t bits); + #endif /* _KERN_AST_H_ */ diff --git a/osfmk/kern/backtrace.c b/osfmk/kern/backtrace.c index 19dbe70ff..b47ce7940 100644 --- a/osfmk/kern/backtrace.c +++ b/osfmk/kern/backtrace.c @@ -35,6 +35,11 @@ #include #include +#if defined(__arm__) || defined(__arm64__) +#include +#include +#endif + uint32_t __attribute__((noinline)) backtrace(uintptr_t *bt, uint32_t max_frames) @@ -56,9 +61,9 @@ backtrace_frame(uintptr_t *bt, uint32_t max_frames, void *start_frame) { thread_t thread = current_thread(); uintptr_t *fp; - uintptr_t *next_fp; uint32_t frame_index = 0; uintptr_t top, bottom; + bool in_valid_stack; assert(bt != NULL); assert(max_frames > 0); @@ -67,21 +72,27 @@ backtrace_frame(uintptr_t *bt, uint32_t max_frames, void *start_frame) bottom = thread->kernel_stack; top = bottom + kernel_stack_size; - if ((uintptr_t)fp >= top || (uintptr_t)fp < bottom) { +#define IN_STK_BOUNDS(__addr) \ + (((uintptr_t)(__addr) >= (uintptr_t)bottom) && \ + ((uintptr_t)(__addr) < (uintptr_t)top)) + + in_valid_stack = IN_STK_BOUNDS(fp); + + if (!in_valid_stack) { fp = NULL; } while (fp != NULL && frame_index < max_frames) { - next_fp = (uintptr_t *)*fp; + uintptr_t *next_fp = (uintptr_t *)*fp; /* * If the frame pointer is 0, backtracing has reached the top of * the stack and there is no return address. Some stacks might not * have set this up, so bounds check, as well. */ - if (next_fp == NULL || - (uintptr_t)next_fp >= top || - (uintptr_t)next_fp < bottom) + in_valid_stack = IN_STK_BOUNDS(next_fp); + + if (next_fp == NULL || !in_valid_stack) { break; } @@ -97,6 +108,7 @@ backtrace_frame(uintptr_t *bt, uint32_t max_frames, void *start_frame) } return frame_index; +#undef IN_STK_BOUNDS } #if defined(__x86_64__) @@ -135,6 +147,52 @@ interrupted_kernel_pc_fp(uintptr_t *pc, uintptr_t *fp) return KERN_SUCCESS; } +#elif defined(__arm64__) + +static kern_return_t +interrupted_kernel_pc_fp(uintptr_t *pc, uintptr_t *fp) +{ + struct arm_saved_state *state; + bool state_64; + + state = getCpuDatap()->cpu_int_state; + if (!state) { + return KERN_FAILURE; + } + state_64 = is_saved_state64(state); + + /* return early if interrupted a thread in user space */ + if (PSR64_IS_USER(get_saved_state_cpsr(state))) { + return KERN_FAILURE; + } + + *pc = get_saved_state_pc(state); + *fp = get_saved_state_fp(state); + return KERN_SUCCESS; +} + +#elif defined(__arm__) + +static kern_return_t +interrupted_kernel_pc_fp(uintptr_t *pc, uintptr_t *fp) +{ + struct arm_saved_state *state; + + state = getCpuDatap()->cpu_int_state; + if (!state) { + return KERN_FAILURE; + } + + /* return early if interrupted a thread in user space */ + if (PSR_IS_USER(get_saved_state_cpsr(state))) { + return KERN_FAILURE; + } + + *pc = get_saved_state_pc(state); + *fp = get_saved_state_fp(state); + return KERN_SUCCESS; +} + #else /* defined(__arm__) */ #error "interrupted_kernel_pc_fp: unsupported architecture" #endif /* !defined(__arm__) */ @@ -143,14 +201,14 @@ uint32_t backtrace_interrupted(uintptr_t *bt, uint32_t max_frames) { uintptr_t pc; - uintptr_t *fp; + uintptr_t fp; kern_return_t kr; assert(bt != NULL); assert(max_frames > 0); assert(ml_at_interrupt_context() == TRUE); - kr = interrupted_kernel_pc_fp(&pc, (uintptr_t)&fp); + kr = interrupted_kernel_pc_fp(&pc, &fp); if (kr != KERN_SUCCESS) { return 0; } @@ -160,7 +218,7 @@ backtrace_interrupted(uintptr_t *bt, uint32_t max_frames) return 1; } - return backtrace_frame(bt + 1, max_frames - 1, fp); + return backtrace_frame(bt + 1, max_frames - 1, (void *)fp); } int @@ -212,6 +270,34 @@ backtrace_thread_user(void *thread, uintptr_t *bt, uint32_t max_frames, fp = saved_state32(state)->ebp; } +#elif defined(__arm64__) + + /* ARM expects stack frames to be aligned to 16 bytes */ +#define INVALID_USER_FP(FP) ((FP) == 0 || ((FP) & 0x3UL) != 0UL) + + struct arm_saved_state *state = get_user_regs(thread); + if (!state) { + return EINVAL; + } + + user_64 = is_saved_state64(state); + pc = get_saved_state_pc(state); + fp = get_saved_state_fp(state); + +#elif defined(__arm__) + + /* ARM expects stack frames to be aligned to 16 bytes */ +#define INVALID_USER_FP(FP) ((FP) == 0 || ((FP) & 0x3UL) != 0UL) + + struct arm_saved_state *state = get_user_regs(thread); + if (!state) { + return EINVAL; + } + + user_64 = false; + pc = get_saved_state_pc(state); + fp = get_saved_state_fp(state); + #else /* defined(__arm__) */ #error "backtrace_thread_user: unsupported architecture" #endif /* !defined(__arm__) */ diff --git a/osfmk/kern/bits.h b/osfmk/kern/bits.h index 0305208a8..5c977497d 100644 --- a/osfmk/kern/bits.h +++ b/osfmk/kern/bits.h @@ -47,11 +47,35 @@ typedef unsigned int uint; #define bit_clear(x, b) ((x) &= ~BIT(b)) #define bit_test(x, b) ((bool)((x) & BIT(b))) +/* Non-atomically clear the bit and returns whether the bit value was changed */ +inline static bool +bit_clear_if_set(uint64_t bitmap, int bit) +{ + bool bit_is_set = bit_test(bitmap, bit); + bit_clear(bitmap, bit); + return bit_is_set; +} + +/* Non-atomically set the bit and returns whether the bit value was changed */ +inline static bool +bit_set_if_clear(uint64_t bitmap, int bit) +{ + bool bit_is_set = bit_test(bitmap, bit); + bit_set(bitmap, bit); + return !bit_is_set; +} + /* Returns the most significant '1' bit, or -1 if all zeros */ inline static int bit_first(uint64_t bitmap) { +#if defined(__arm64__) + int64_t result; + asm volatile("clz %0, %1" : "=r" (result) : "r" (bitmap)); + return 63 - (int)result; +#else return (bitmap == 0) ? -1 : 63 - __builtin_clzll(bitmap); +#endif } diff --git a/osfmk/kern/block_hint.h b/osfmk/kern/block_hint.h index c72ec7f62..c52b9488f 100644 --- a/osfmk/kern/block_hint.h +++ b/osfmk/kern/block_hint.h @@ -47,6 +47,26 @@ typedef enum thread_snapshot_wait_flags { kThreadWaitPThreadRWLockWrite = 0x0d, kThreadWaitPThreadCondVar = 0x0e, kThreadWaitParkedWorkQueue = 0x0f, + kThreadWaitWorkloopSyncWait = 0x10, } __attribute__((packed)) block_hint_t; +#ifdef XNU_KERNEL_PRIVATE + +struct waitq; +struct stackshot_thread_waitinfo; +typedef struct stackshot_thread_waitinfo thread_waitinfo_t; + +/* Used for stackshot_thread_waitinfo_unsafe */ +extern void kdp_lck_mtx_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); +extern void kdp_sema_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); +extern void kdp_mqueue_send_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); +extern void kdp_mqueue_recv_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); +extern void kdp_ulock_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); +extern void kdp_rwlck_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); +extern void kdp_pthread_find_owner(thread_t thread, thread_waitinfo_t *waitinfo); +extern void *kdp_pthread_get_thread_kwq(thread_t thread); +extern void kdp_workloop_sync_wait_find_owner(thread_t thread, event64_t event, thread_waitinfo_t *waitinfo); + +#endif /* XNU_KERNEL_PRIVATE */ + #endif /* !_KERN_BLOCK_HINT_H_ */ diff --git a/osfmk/kern/bsd_kern.c b/osfmk/kern/bsd_kern.c index e9e601433..7196cc5c6 100644 --- a/osfmk/kern/bsd_kern.c +++ b/osfmk/kern/bsd_kern.c @@ -45,6 +45,13 @@ #include #include +#if MONOTONIC +#include +#include +#endif /* MONOTONIC */ + +#include + #undef thread_should_halt /* BSD KERN COMPONENT INTERFACE */ @@ -60,11 +67,14 @@ kern_return_t get_signalact(task_t , thread_t *, int); int fill_task_rusage(task_t task, rusage_info_current *ri); int fill_task_io_rusage(task_t task, rusage_info_current *ri); int fill_task_qos_rusage(task_t task, rusage_info_current *ri); +void fill_task_monotonic_rusage(task_t task, rusage_info_current *ri); +uint64_t get_task_logical_writes(task_t task); void fill_task_billed_usage(task_t task, rusage_info_current *ri); void task_bsdtask_kill(task_t); extern uint64_t get_dispatchqueue_serialno_offset_from_proc(void *p); extern uint64_t proc_uniqueid(void *p); +extern int proc_pidversion(void *p); #if MACH_BSD extern void psignal(void *, int); @@ -326,7 +336,11 @@ swap_task_map(task_t task, thread_t thread, vm_map_t map) vm_commit_pagezero_status(map); if (doswitch) { +#if defined(__arm__) || defined(__arm64__) + PMAP_SWITCH_USER(thread, map, cpu_number()) +#else pmap_switch(map->pmap); +#endif } mp_enable_preemption(); task_unlock(task); @@ -403,11 +417,11 @@ uint64_t get_task_purgeable_size(task_t task) /* * */ -uint64_t get_task_phys_footprint(task_t task) -{ +uint64_t get_task_phys_footprint(task_t task) +{ kern_return_t ret; ledger_amount_t credit, debit; - + ret = ledger_get_entries(task->ledger, task_ledgers.phys_footprint, &credit, &debit); if (KERN_SUCCESS == ret) { return (credit - debit); @@ -419,12 +433,12 @@ uint64_t get_task_phys_footprint(task_t task) /* * */ -uint64_t get_task_phys_footprint_max(task_t task) -{ +uint64_t get_task_phys_footprint_recent_max(task_t task) +{ kern_return_t ret; ledger_amount_t max; - - ret = ledger_get_maximum(task->ledger, task_ledgers.phys_footprint, &max); + + ret = ledger_get_recent_max(task->ledger, task_ledgers.phys_footprint, &max); if (KERN_SUCCESS == ret) { return max; } @@ -432,6 +446,23 @@ uint64_t get_task_phys_footprint_max(task_t task) return 0; } +/* + * + */ +uint64_t get_task_phys_footprint_lifetime_max(task_t task) +{ + kern_return_t ret; + ledger_amount_t max; + + ret = ledger_get_lifetime_max(task->ledger, task_ledgers.phys_footprint, &max); + + if(KERN_SUCCESS == ret) { + return max; + } + + return 0; +} + /* * */ @@ -971,13 +1002,8 @@ fill_task_rusage(task_t task, rusage_info_current *ri) void fill_task_billed_usage(task_t task __unused, rusage_info_current *ri) { -#if CONFIG_BANK - ri->ri_billed_system_time = bank_billed_time_safe(task); - ri->ri_serviced_system_time = bank_serviced_time_safe(task); -#else - ri->ri_billed_system_time = 0; - ri->ri_serviced_system_time = 0; -#endif + bank_billed_balance_safe(task, &ri->ri_billed_system_time, &ri->ri_billed_energy); + bank_serviced_balance_safe(task, &ri->ri_serviced_system_time, &ri->ri_serviced_energy); } int @@ -1025,6 +1051,40 @@ fill_task_qos_rusage(task_t task, rusage_info_current *ri) return (0); } +void +fill_task_monotonic_rusage(task_t task, rusage_info_current *ri) +{ +#if MONOTONIC + if (!mt_core_supported) { + return; + } + + assert(task != TASK_NULL); + + uint64_t counts[MT_CORE_NFIXED] = {}; + mt_fixed_task_counts(task, counts); +#ifdef MT_CORE_INSTRS + ri->ri_instructions = counts[MT_CORE_INSTRS]; +#endif /* defined(MT_CORE_INSTRS) */ + ri->ri_cycles = counts[MT_CORE_CYCLES]; +#else /* MONOTONIC */ +#pragma unused(task, ri) +#endif /* !MONOTONIC */ +} + +uint64_t +get_task_logical_writes(task_t task) +{ + assert(task != TASK_NULL); + struct ledger_entry_info lei; + + task_lock(task); + ledger_get_entry_info(task->ledger, task_ledgers.logical_writes, &lei); + + task_unlock(task); + return lei.lei_balance; +} + uint64_t get_task_dispatchqueue_serialno_offset(task_t task) { @@ -1047,16 +1107,20 @@ get_task_uniqueid(task_t task) } } +int +get_task_version(task_t task) +{ + if (task->bsd_info) { + return proc_pidversion(task->bsd_info); + } else { + return INT_MAX; + } +} + #if CONFIG_MACF struct label * get_task_crash_label(task_t task) { return task->crash_label; } - -void -set_task_crash_label(task_t task, struct label *label) -{ - task->crash_label = label; -} #endif diff --git a/osfmk/kern/build_config.h b/osfmk/kern/build_config.h index 895437a83..7cec83982 100644 --- a/osfmk/kern/build_config.h +++ b/osfmk/kern/build_config.h @@ -31,7 +31,12 @@ #include #include +#include + +__BEGIN_DECLS bool kern_config_is_development(void) OS_CONST; +__END_DECLS + #endif /* _KERN_BUILD_CONFIG_H */ diff --git a/osfmk/kern/call_entry.h b/osfmk/kern/call_entry.h index 8b635f817..dede1bffb 100644 --- a/osfmk/kern/call_entry.h +++ b/osfmk/kern/call_entry.h @@ -35,7 +35,9 @@ #ifdef XNU_KERNEL_PRIVATE #include +#if !CONFIG_EMBEDDED #define TIMER_TRACE 1 +#endif typedef void *call_entry_param_t; typedef void (*call_entry_func_t)( @@ -78,9 +80,9 @@ call_entry_enqueue_tail( queue_t old_queue = entry->queue; if (old_queue != NULL) - (void)remque(qe(entry)); - - enqueue_tail(queue, qe(entry)); + re_queue_tail(queue, &entry->q_link); + else + enqueue_tail(queue, &entry->q_link); entry->queue = queue; diff --git a/osfmk/kern/clock.c b/osfmk/kern/clock.c index 6173d89b6..0a256f220 100644 --- a/osfmk/kern/clock.c +++ b/osfmk/kern/clock.c @@ -30,6 +30,37 @@ */ /* */ +/*- + * Copyright (c) 1982, 1986, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)time.h 8.5 (Berkeley) 5/4/95 + * $FreeBSD$ + */ #include @@ -44,16 +75,24 @@ #include #include +#include +#include #include #include #include +#include +#include uint32_t hz_tick_interval = 1; decl_simple_lock_data(,clock_lock) +lck_grp_attr_t * settime_lock_grp_attr; +lck_grp_t * settime_lock_grp; +lck_attr_t * settime_lock_attr; +lck_mtx_t settime_lock; #define clock_lock() \ simple_lock(&clock_lock) @@ -71,25 +110,166 @@ boolean_t kdp_clock_is_locked() } #endif +struct bintime { + time_t sec; + uint64_t frac; +}; + +static __inline void +bintime_addx(struct bintime *_bt, uint64_t _x) +{ + uint64_t _u; + + _u = _bt->frac; + _bt->frac += _x; + if (_u > _bt->frac) + _bt->sec++; +} + +static __inline void +bintime_subx(struct bintime *_bt, uint64_t _x) +{ + uint64_t _u; + + _u = _bt->frac; + _bt->frac -= _x; + if (_u < _bt->frac) + _bt->sec--; +} + +static __inline void +bintime_addns(struct bintime *bt, uint64_t ns) +{ + bt->sec += ns/ (uint64_t)NSEC_PER_SEC; + ns = ns % (uint64_t)NSEC_PER_SEC; + if (ns) { + /* 18446744073 = int(2^64 / NSEC_PER_SEC) */ + ns = ns * (uint64_t)18446744073LL; + bintime_addx(bt, ns); + } +} + +static __inline void +bintime_subns(struct bintime *bt, uint64_t ns) +{ + bt->sec -= ns/ (uint64_t)NSEC_PER_SEC; + ns = ns % (uint64_t)NSEC_PER_SEC; + if (ns) { + /* 18446744073 = int(2^64 / NSEC_PER_SEC) */ + ns = ns * (uint64_t)18446744073LL; + bintime_subx(bt, ns); + } +} + +static __inline void +bintime_addxns(struct bintime *bt, uint64_t a, int64_t xns) +{ + uint64_t uxns = (xns > 0)?(uint64_t )xns:(uint64_t)-xns; + uint64_t ns = multi_overflow(a, uxns); + if (xns > 0) { + if (ns) + bintime_addns(bt, ns); + ns = (a * uxns) / (uint64_t)NSEC_PER_SEC; + bintime_addx(bt, ns); + } + else{ + if (ns) + bintime_subns(bt, ns); + ns = (a * uxns) / (uint64_t)NSEC_PER_SEC; + bintime_subx(bt,ns); + } +} + + +static __inline void +bintime_add(struct bintime *_bt, const struct bintime *_bt2) +{ + uint64_t _u; + + _u = _bt->frac; + _bt->frac += _bt2->frac; + if (_u > _bt->frac) + _bt->sec++; + _bt->sec += _bt2->sec; +} + +static __inline void +bintime_sub(struct bintime *_bt, const struct bintime *_bt2) +{ + uint64_t _u; + + _u = _bt->frac; + _bt->frac -= _bt2->frac; + if (_u < _bt->frac) + _bt->sec--; + _bt->sec -= _bt2->sec; +} + +static __inline void +clock2bintime(const clock_sec_t *secs, const clock_usec_t *microsecs, struct bintime *_bt) +{ + + _bt->sec = *secs; + /* 18446744073709 = int(2^64 / 1000000) */ + _bt->frac = *microsecs * (uint64_t)18446744073709LL; +} + +static __inline void +bintime2usclock(const struct bintime *_bt, clock_sec_t *secs, clock_usec_t *microsecs) +{ + + *secs = _bt->sec; + *microsecs = ((uint64_t)USEC_PER_SEC * (uint32_t)(_bt->frac >> 32)) >> 32; +} + +static __inline void +bintime2nsclock(const struct bintime *_bt, clock_sec_t *secs, clock_usec_t *nanosecs) +{ + + *secs = _bt->sec; + *nanosecs = ((uint64_t)NSEC_PER_SEC * (uint32_t)(_bt->frac >> 32)) >> 32; +} + +static __inline void +bintime2absolutetime(const struct bintime *_bt, uint64_t *abs) +{ + uint64_t nsec; + nsec = (uint64_t) _bt->sec * (uint64_t)NSEC_PER_SEC + (((uint64_t)NSEC_PER_SEC * (uint32_t)(_bt->frac >> 32)) >> 32); + nanoseconds_to_absolutetime(nsec, abs); +} /* * Time of day (calendar) variables. * * Algorithm: * - * TOD <- (seconds + epoch, fraction) <- CONV(current absolute time + offset) + * TOD <- bintime + delta*scale * - * where CONV converts absolute time units into seconds and a fraction. + * where : + * bintime is a cumulative offset that includes bootime and scaled time elapsed betweed bootime and last scale update. + * delta is ticks elapsed since last scale update. + * scale is computed according to an adjustment provided by ntp_kern. */ static struct clock_calend { - uint64_t epoch; - uint64_t offset; - uint64_t epoch_absolute; - - int32_t adjdelta; /* Nanosecond time delta for this adjustment period */ - uint64_t adjstart; /* Absolute time value for start of this adjustment period */ - uint32_t adjoffset; /* Absolute time offset for this adjustment period as absolute value */ + uint64_t s_scale_ns; /* scale to apply for each second elapsed, it converts in ns */ + int64_t s_adj_nsx; /* additional adj to apply for each second elapsed, it is expressed in 64 bit frac of ns */ + uint64_t tick_scale_x; /* scale to apply for each tick elapsed, it converts in 64 bit frac of s */ + uint64_t offset_count; /* abs time from which apply current scales */ + struct bintime offset; /* cumulative offset expressed in (sec, 64 bits frac of a second) */ + struct bintime bintime; /* cumulative offset (it includes bootime) expressed in (sec, 64 bits frac of a second) */ + struct bintime boottime; /* boot time expressed in (sec, 64 bits frac of a second) */ + struct bintime basesleep; } clock_calend; +static uint64_t ticks_per_sec; /* ticks in a second (expressed in abs time) */ + +#if DEVELOPMENT || DEBUG +clock_sec_t last_utc_sec = 0; +clock_usec_t last_utc_usec = 0; +clock_sec_t max_utc_sec = 0; +clock_sec_t last_sys_sec = 0; +clock_usec_t last_sys_usec = 0; +#endif + #if CONFIG_DTRACE /* @@ -111,34 +291,10 @@ static void clock_track_calend_nowait(void); #endif -/* - * Calendar adjustment variables and values. - */ -#define calend_adjperiod (NSEC_PER_SEC / 100) /* adjustment period, ns */ -#define calend_adjskew (40 * NSEC_PER_USEC) /* "standard" skew, ns / period */ -#define calend_adjbig (NSEC_PER_SEC) /* use 10x skew above adjbig ns */ - -static int64_t calend_adjtotal; /* Nanosecond remaining total adjustment */ -static uint64_t calend_adjdeadline; /* Absolute time value for next adjustment period */ -static uint32_t calend_adjinterval; /* Absolute time interval of adjustment period */ - -static timer_call_data_t calend_adjcall; -static uint32_t calend_adjactive; - -static uint32_t calend_set_adjustment( - long *secs, - int *microsecs); - -static void calend_adjust_call(void); -static uint32_t calend_adjust(void); +void _clock_delay_until_deadline(uint64_t interval, uint64_t deadline); +void _clock_delay_until_deadline_with_leeway(uint64_t interval, uint64_t deadline, uint64_t leeway); -void _clock_delay_until_deadline(uint64_t interval, - uint64_t deadline); -void _clock_delay_until_deadline_with_leeway(uint64_t interval, - uint64_t deadline, - uint64_t leeway); - -/* Seconds boottime epoch */ +/* Boottime variables*/ static uint64_t clock_boottime; static uint32_t clock_boottime_usec; @@ -168,11 +324,19 @@ MACRO_END void clock_config(void) { + clock_lock_init(); - timer_call_setup(&calend_adjcall, (timer_call_func_t)calend_adjust_call, NULL); + settime_lock_grp_attr = lck_grp_attr_alloc_init(); + settime_lock_grp = lck_grp_alloc_init("settime grp", settime_lock_grp_attr); + settime_lock_attr = lck_attr_alloc_init(); + lck_mtx_init(&settime_lock, settime_lock_grp, settime_lock_attr); clock_oldconfig(); + + ntp_init(); + + nanoseconds_to_absolutetime((uint64_t)NSEC_PER_SEC, &ticks_per_sec); } /* @@ -199,9 +363,6 @@ clock_timebase_init(void) { uint64_t abstime; - nanoseconds_to_absolutetime(calend_adjperiod, &abstime); - calend_adjinterval = (uint32_t)abstime; - nanoseconds_to_absolutetime(NSEC_PER_SEC / 100, &abstime); hz_tick_interval = (uint32_t)abstime; @@ -239,49 +400,189 @@ mach_timebase_info_trap( */ void clock_get_calendar_microtime( - clock_sec_t *secs, + clock_sec_t *secs, clock_usec_t *microsecs) { clock_get_calendar_absolute_and_microtime(secs, microsecs, NULL); } +/* + * get_scale_factors_from_adj: + * + * computes scale factors from the value given in adjustment. + * + * Part of the code has been taken from tc_windup of FreeBSD + * written by Poul-Henning Kamp , Julien Ridoux and + * Konstantin Belousov. + * https://github.com/freebsd/freebsd/blob/master/sys/kern/kern_tc.c + */ +static void +get_scale_factors_from_adj(int64_t adjustment, uint64_t* tick_scale_x, uint64_t* s_scale_ns, int64_t* s_adj_nsx) +{ + uint64_t scale; + int64_t nano, frac; + + /*- + * Calculating the scaling factor. We want the number of 1/2^64 + * fractions of a second per period of the hardware counter, taking + * into account the th_adjustment factor which the NTP PLL/adjtime(2) + * processing provides us with. + * + * The th_adjustment is nanoseconds per second with 32 bit binary + * fraction and we want 64 bit binary fraction of second: + * + * x = a * 2^32 / 10^9 = a * 4.294967296 + * + * The range of th_adjustment is +/- 5000PPM so inside a 64bit int + * we can only multiply by about 850 without overflowing, that + * leaves no suitably precise fractions for multiply before divide. + * + * Divide before multiply with a fraction of 2199/512 results in a + * systematic undercompensation of 10PPM of th_adjustment. On a + * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. + * + * We happily sacrifice the lowest of the 64 bits of our result + * to the goddess of code clarity. + * + */ + scale = (uint64_t)1 << 63; + scale += (adjustment / 1024) * 2199; + scale /= ticks_per_sec; + *tick_scale_x = scale * 2; + + /* + * hi part of adj + * it contains ns (without fraction) to add to the next sec. + * Get ns scale factor for the next sec. + */ + nano = (adjustment > 0)? adjustment >> 32 : -((-adjustment) >> 32); + scale = (uint64_t) NSEC_PER_SEC; + scale += nano; + *s_scale_ns = scale; + + /* + * lo part of adj + * it contains 32 bit frac of ns to add to the next sec. + * Keep it as additional adjustment for the next sec. + */ + frac = (adjustment > 0)? ((uint32_t) adjustment) : -((uint32_t) (-adjustment)); + *s_adj_nsx = (frac>0)? frac << 32 : -( (-frac) << 32); + + return; +} + +/* + * scale_delta: + * + * returns a bintime struct representing delta scaled accordingly to the + * scale factors provided to this function. + */ +static struct bintime +scale_delta(uint64_t delta, uint64_t tick_scale_x, uint64_t s_scale_ns, int64_t s_adj_nsx) +{ + uint64_t sec, new_ns, over; + struct bintime bt; + + bt.sec = 0; + bt.frac = 0; + + /* + * If more than one second is elapsed, + * scale fully elapsed seconds using scale factors for seconds. + * s_scale_ns -> scales sec to ns. + * s_adj_nsx -> additional adj expressed in 64 bit frac of ns to apply to each sec. + */ + if (delta > ticks_per_sec) { + sec = (delta/ticks_per_sec); + new_ns = sec * s_scale_ns; + bintime_addns(&bt, new_ns); + if (s_adj_nsx) { + if (sec == 1) { + /* shortcut, no overflow can occur */ + if (s_adj_nsx > 0) + bintime_addx(&bt, (uint64_t)s_adj_nsx/ (uint64_t)NSEC_PER_SEC); + else + bintime_subx(&bt, (uint64_t)-s_adj_nsx/ (uint64_t)NSEC_PER_SEC); + } + else{ + /* + * s_adj_nsx is 64 bit frac of ns. + * sec*s_adj_nsx might overflow in int64_t. + * use bintime_addxns to not lose overflowed ns. + */ + bintime_addxns(&bt, sec, s_adj_nsx); + } + } + delta = (delta % ticks_per_sec); + } + + over = multi_overflow(tick_scale_x, delta); + if(over){ + bt.sec += over; + } + + /* + * scale elapsed ticks using the scale factor for ticks. + */ + bintime_addx(&bt, delta * tick_scale_x); + + return bt; +} + +/* + * get_scaled_time: + * + * returns the scaled time of the time elapsed from the last time + * scale factors were updated to now. + */ +static struct bintime +get_scaled_time(uint64_t now) +{ + uint64_t delta; + + /* + * Compute ticks elapsed since last scale update. + * This time will be scaled according to the value given by ntp kern. + */ + delta = now - clock_calend.offset_count; + + return scale_delta(delta, clock_calend.tick_scale_x, clock_calend.s_scale_ns, clock_calend.s_adj_nsx); +} + static void clock_get_calendar_absolute_and_microtime_locked( - clock_sec_t *secs, + clock_sec_t *secs, clock_usec_t *microsecs, uint64_t *abstime) { - uint64_t now = mach_absolute_time(); + uint64_t now; + struct bintime bt; + + now = mach_absolute_time(); if (abstime) *abstime = now; - if (clock_calend.adjdelta < 0) { - uint32_t t32; - - /* - * Since offset is decremented during a negative adjustment, - * ensure that time increases monotonically without going - * temporarily backwards. - * If the delta has not yet passed, now is set to the start - * of the current adjustment period; otherwise, we're between - * the expiry of the delta and the next call to calend_adjust(), - * and we offset accordingly. - */ - if (now > clock_calend.adjstart) { - t32 = (uint32_t)(now - clock_calend.adjstart); - - if (t32 > clock_calend.adjoffset) - now -= clock_calend.adjoffset; - else - now = clock_calend.adjstart; - } - } + bt = get_scaled_time(now); + bintime_add(&bt, &clock_calend.bintime); + bintime2usclock(&bt, secs, microsecs); +} - now += clock_calend.offset; +static void +clock_get_calendar_absolute_and_nanotime_locked( + clock_sec_t *secs, + clock_usec_t *nanosecs, + uint64_t *abstime) +{ + uint64_t now; + struct bintime bt; - absolutetime_to_microtime(now, secs, microsecs); + now = mach_absolute_time(); + if (abstime) + *abstime = now; - *secs += (clock_sec_t)clock_calend.epoch; + bt = get_scaled_time(now); + bintime_add(&bt, &clock_calend.bintime); + bintime2nsclock(&bt, secs, nanosecs); } /* @@ -294,7 +595,7 @@ clock_get_calendar_absolute_and_microtime_locked( */ void clock_get_calendar_absolute_and_microtime( - clock_sec_t *secs, + clock_sec_t *secs, clock_usec_t *microsecs, uint64_t *abstime) { @@ -321,7 +622,7 @@ clock_get_calendar_absolute_and_microtime( */ void clock_get_calendar_nanotime( - clock_sec_t *secs, + clock_sec_t *secs, clock_nsec_t *nanosecs) { spl_t s; @@ -329,9 +630,7 @@ clock_get_calendar_nanotime( s = splclock(); clock_lock(); - clock_get_calendar_absolute_and_microtime_locked(secs, nanosecs, NULL); - - *nanosecs *= NSEC_PER_USEC; + clock_get_calendar_absolute_and_nanotime_locked(secs, nanosecs, NULL); clock_unlock(); splx(s); @@ -350,7 +649,7 @@ clock_get_calendar_nanotime( */ void clock_gettimeofday( - clock_sec_t *secs, + clock_sec_t *secs, clock_usec_t *microsecs) { clock_gettimeofday_and_absolute_time(secs, microsecs, NULL); @@ -358,39 +657,23 @@ clock_gettimeofday( void clock_gettimeofday_and_absolute_time( - clock_sec_t *secs, + clock_sec_t *secs, clock_usec_t *microsecs, - uint64_t *mach_time) + uint64_t *mach_time) { uint64_t now; spl_t s; + struct bintime bt; s = splclock(); clock_lock(); now = mach_absolute_time(); + bt = get_scaled_time(now); + bintime_add(&bt, &clock_calend.bintime); + bintime2usclock(&bt, secs, microsecs); - if (clock_calend.adjdelta >= 0) { - clock_gettimeofday_set_commpage(now, clock_calend.epoch, clock_calend.offset, secs, microsecs); - } - else { - uint32_t t32; - - if (now > clock_calend.adjstart) { - t32 = (uint32_t)(now - clock_calend.adjstart); - - if (t32 > clock_calend.adjoffset) - now -= clock_calend.adjoffset; - else - now = clock_calend.adjstart; - } - - now += clock_calend.offset; - - absolutetime_to_microtime(now, secs, microsecs); - - *secs += (clock_sec_t)clock_calend.epoch; - } + clock_gettimeofday_set_commpage(now, bt.sec, bt.frac, clock_calend.tick_scale_x, ticks_per_sec); clock_unlock(); splx(s); @@ -414,21 +697,32 @@ clock_gettimeofday_and_absolute_time( */ void clock_set_calendar_microtime( - clock_sec_t secs, + clock_sec_t secs, clock_usec_t microsecs) { - clock_sec_t sys; - clock_usec_t microsys; - uint64_t absolutesys; - clock_sec_t newsecs; - clock_sec_t oldsecs; - clock_usec_t newmicrosecs; + uint64_t absolutesys; + clock_sec_t newsecs; + clock_sec_t oldsecs; + clock_usec_t newmicrosecs; clock_usec_t oldmicrosecs; - uint64_t commpage_value; - spl_t s; + uint64_t commpage_value; + spl_t s; + struct bintime bt; + clock_sec_t deltasecs; + clock_usec_t deltamicrosecs; + + newsecs = secs; + newmicrosecs = microsecs; - newsecs = secs; - newmicrosecs = microsecs; + /* + * settime_lock mtx is used to avoid that racing settimeofdays update the wall clock and + * the platform clock concurrently. + * + * clock_lock cannot be used for this race because it is acquired from interrupt context + * and it needs interrupts disabled while instead updating the platform clock needs to be + * called with interrupts enabled. + */ + lck_mtx_lock(&settime_lock); s = splclock(); clock_lock(); @@ -439,50 +733,50 @@ clock_set_calendar_microtime( * Adjust the boottime based on the delta. */ clock_get_calendar_absolute_and_microtime_locked(&oldsecs, &oldmicrosecs, &absolutesys); - if (oldsecs < secs || (oldsecs == secs && oldmicrosecs < microsecs)){ + + if (oldsecs < secs || (oldsecs == secs && oldmicrosecs < microsecs)) { // moving forwards - long deltasecs = secs, deltamicrosecs = microsecs; + deltasecs = secs; + deltamicrosecs = microsecs; + TIME_SUB(deltasecs, oldsecs, deltamicrosecs, oldmicrosecs, USEC_PER_SEC); TIME_ADD(clock_boottime, deltasecs, clock_boottime_usec, deltamicrosecs, USEC_PER_SEC); + + clock2bintime(&deltasecs, &deltamicrosecs, &bt); + bintime_add(&clock_calend.boottime, &bt); + bintime_add(&clock_calend.basesleep, &bt); + } else { // moving backwards - long deltasecs = oldsecs, deltamicrosecs = oldmicrosecs; + deltasecs = oldsecs; + deltamicrosecs = oldmicrosecs; + TIME_SUB(deltasecs, secs, deltamicrosecs, microsecs, USEC_PER_SEC); TIME_SUB(clock_boottime, deltasecs, clock_boottime_usec, deltamicrosecs, USEC_PER_SEC); - } - commpage_value = clock_boottime * USEC_PER_SEC + clock_boottime_usec; - /* - * Calculate the new calendar epoch based on - * the new value and the system clock. - */ - absolutetime_to_microtime(absolutesys, &sys, µsys); - TIME_SUB(secs, sys, microsecs, microsys, USEC_PER_SEC); + clock2bintime(&deltasecs, &deltamicrosecs, &bt); + bintime_sub(&clock_calend.boottime, &bt); + bintime_sub(&clock_calend.basesleep, &bt); + } - /* - * Set the new calendar epoch. - */ - clock_calend.epoch = secs; + clock_calend.bintime = clock_calend.boottime; + bintime_add(&clock_calend.bintime, &clock_calend.offset); - nanoseconds_to_absolutetime((uint64_t)microsecs * NSEC_PER_USEC, &clock_calend.offset); + clock2bintime((clock_sec_t *) &secs, (clock_usec_t *) µsecs, &bt); - clock_interval_to_absolutetime_interval((uint32_t) secs, NSEC_PER_SEC, &clock_calend.epoch_absolute); - clock_calend.epoch_absolute += clock_calend.offset; + clock_gettimeofday_set_commpage(absolutesys, bt.sec, bt.frac, clock_calend.tick_scale_x, ticks_per_sec); - /* - * Cancel any adjustment in progress. - */ - calend_adjtotal = clock_calend.adjdelta = 0; + commpage_value = clock_boottime * USEC_PER_SEC + clock_boottime_usec; clock_unlock(); + splx(s); /* * Set the new value for the platform clock. + * This call might block, so interrupts must be enabled. */ PESetUTCTimeOfDay(newsecs, newmicrosecs); - splx(s); - commpage_update_boottime(commpage_value); /* @@ -494,6 +788,84 @@ clock_set_calendar_microtime( #if CONFIG_DTRACE clock_track_calend_nowait(); #endif + + lck_mtx_unlock(&settime_lock); +} + +uint64_t mach_absolutetime_asleep = 0; +uint64_t mach_absolutetime_last_sleep = 0; + +void +clock_get_calendar_uptime(clock_sec_t *secs) +{ + uint64_t now; + spl_t s; + struct bintime bt; + + s = splclock(); + clock_lock(); + + now = mach_absolute_time(); + + bt = get_scaled_time(now); + bintime_add(&bt, &clock_calend.offset); + + *secs = bt.sec; + + clock_unlock(); + splx(s); +} + + +/* + * clock_update_calendar: + * + * called by ntp timer to update scale factors. + */ +void +clock_update_calendar(void) +{ + + uint64_t now, delta; + struct bintime bt; + spl_t s; + int64_t adjustment; + + s = splclock(); + clock_lock(); + + now = mach_absolute_time(); + + /* + * scale the time elapsed since the last update and + * add it to offset. + */ + bt = get_scaled_time(now); + bintime_add(&clock_calend.offset, &bt); + + /* + * update the base from which apply next scale factors. + */ + delta = now - clock_calend.offset_count; + clock_calend.offset_count += delta; + + clock_calend.bintime = clock_calend.offset; + bintime_add(&clock_calend.bintime, &clock_calend.boottime); + + /* + * recompute next adjustment. + */ + ntp_update_second(&adjustment, clock_calend.bintime.sec); + + /* + * recomputing scale factors. + */ + get_scale_factors_from_adj(adjustment, &clock_calend.tick_scale_x, &clock_calend.s_scale_ns, &clock_calend.s_adj_nsx); + + clock_gettimeofday_set_commpage(now, clock_calend.bintime.sec, clock_calend.bintime.frac, clock_calend.tick_scale_x, ticks_per_sec); + + clock_unlock(); + splx(s); } /* @@ -506,20 +878,17 @@ clock_set_calendar_microtime( * Also sends host notifications. */ -uint64_t mach_absolutetime_asleep; -uint64_t mach_absolutetime_last_sleep; - void clock_initialize_calendar(void) { - clock_sec_t sys; // sleepless time since boot in seconds - clock_sec_t secs; // Current UTC time - clock_sec_t utc_offset_secs; // Difference in current UTC time and sleepless time since boot + clock_sec_t sys; // sleepless time since boot in seconds + clock_sec_t secs; // Current UTC time + clock_sec_t utc_offset_secs; // Difference in current UTC time and sleepless time since boot clock_usec_t microsys; clock_usec_t microsecs; clock_usec_t utc_offset_microsecs; - uint64_t new_epoch; // utc_offset_secs in mach absolute time units - spl_t s; + spl_t s; + struct bintime bt; PEGetUTCTimeOfDay(&secs, µsecs); @@ -528,72 +897,64 @@ clock_initialize_calendar(void) commpage_disable_timestamp(); - if ((long)secs >= (long)clock_boottime) { - /* - * Initialize the boot time based on the platform clock. - */ - if (clock_boottime == 0){ - clock_boottime = secs; - clock_boottime_usec = microsecs; - commpage_update_boottime(clock_boottime * USEC_PER_SEC + clock_boottime_usec); - } - - /* - * Calculate the new calendar epoch based on - * the platform clock and the system clock. - */ - clock_get_system_microtime(&sys, µsys); - utc_offset_secs = secs; - utc_offset_microsecs = microsecs; + /* + * Calculate the new calendar epoch based on + * the platform clock and the system clock. + */ + clock_get_system_microtime(&sys, µsys); + utc_offset_secs = secs; + utc_offset_microsecs = microsecs; + +#if DEVELOPMENT || DEBUG + last_utc_sec = secs; + last_utc_usec = microsecs; + last_sys_sec = sys; + last_sys_usec = microsys; + if (secs > max_utc_sec) + max_utc_sec = secs; +#endif - // This macro mutates utc_offset_secs and micro_utc_offset - TIME_SUB(utc_offset_secs, sys, utc_offset_microsecs, microsys, USEC_PER_SEC); + /* + * We normally expect the UTC clock to be always-on and produce + * greater readings than the tick counter. There may be corner cases + * due to differing clock resolutions (UTC clock is likely lower) and + * and errors reading the UTC clock (some implementations return 0 + * on error) in which that doesn't hold true. Bring the UTC measurements + * in-line with the tick counter measurements as a best effort in that case. + */ + if ((sys > secs) || ((sys == secs) && (microsys > microsecs))) { + secs = utc_offset_secs = sys; + microsecs = utc_offset_microsecs = microsys; + } - /* - * Set the new calendar epoch. - */ + // This macro stores the subtraction result in utc_offset_secs and utc_offset_microsecs + TIME_SUB(utc_offset_secs, sys, utc_offset_microsecs, microsys, USEC_PER_SEC); - clock_calend.epoch = utc_offset_secs; + clock2bintime(&utc_offset_secs, &utc_offset_microsecs, &bt); - nanoseconds_to_absolutetime((uint64_t)utc_offset_microsecs * NSEC_PER_USEC, &clock_calend.offset); + /* + * Initialize the boot time based on the platform clock. + */ + clock_boottime = secs; + clock_boottime_usec = microsecs; + commpage_update_boottime(clock_boottime * USEC_PER_SEC + clock_boottime_usec); - clock_interval_to_absolutetime_interval((uint32_t) utc_offset_secs, NSEC_PER_SEC, &new_epoch); - new_epoch += clock_calend.offset; + nanoseconds_to_absolutetime((uint64_t)NSEC_PER_SEC, &ticks_per_sec); + clock_calend.boottime = bt; + clock_calend.bintime = bt; + clock_calend.offset.sec = 0; + clock_calend.offset.frac = 0; - if (clock_calend.epoch_absolute) - { - /* new_epoch is the difference between absolute_time and utc_time - * this value will remain constant until the system sleeps. - * Then, difference between values would go up by the time the system sleeps. - * epoch_absolute is the last difference between the two values - * so the difference in the differences would be the time of the last sleep - */ + clock_calend.tick_scale_x = (uint64_t)1 << 63; + clock_calend.tick_scale_x /= ticks_per_sec; + clock_calend.tick_scale_x *= 2; - if(new_epoch > clock_calend.epoch_absolute) { - mach_absolutetime_last_sleep = new_epoch - clock_calend.epoch_absolute; - } - else { - mach_absolutetime_last_sleep = 0; - } - mach_absolutetime_asleep += mach_absolutetime_last_sleep; - KERNEL_DEBUG_CONSTANT( - MACHDBG_CODE(DBG_MACH_CLOCK,MACH_EPOCH_CHANGE) | DBG_FUNC_NONE, - (uintptr_t) mach_absolutetime_last_sleep, - (uintptr_t) mach_absolutetime_asleep, - (uintptr_t) (mach_absolutetime_last_sleep >> 32), - (uintptr_t) (mach_absolutetime_asleep >> 32), - 0); - } - clock_calend.epoch_absolute = new_epoch; + clock_calend.s_scale_ns = NSEC_PER_SEC; + clock_calend.s_adj_nsx = 0; - /* - * Cancel any adjustment in progress. - */ - calend_adjtotal = clock_calend.adjdelta = 0; - } + clock_calend.basesleep = bt; commpage_update_mach_continuous_time(mach_absolutetime_asleep); - adjust_cont_time_thread_calls(); clock_unlock(); splx(s); @@ -608,37 +969,111 @@ clock_initialize_calendar(void) #endif } -/* - * clock_get_boottime_nanotime: - * - * Return the boottime, used by sysctl. - */ + void -clock_get_boottime_nanotime( - clock_sec_t *secs, - clock_nsec_t *nanosecs) +clock_wakeup_calendar(void) { - spl_t s; + clock_sec_t sys; // sleepless time since boot in seconds + clock_sec_t secs; // Current UTC time + clock_usec_t microsys; + clock_usec_t microsecs; + spl_t s; + struct bintime utc_offset_bt, last_sleep_bt; + + PEGetUTCTimeOfDay(&secs, µsecs); s = splclock(); clock_lock(); - *secs = (clock_sec_t)clock_boottime; - *nanosecs = (clock_nsec_t)clock_boottime_usec * NSEC_PER_USEC; + commpage_disable_timestamp(); + + /* + * Calculate the new calendar epoch based on + * the platform clock and the system clock. + */ + clock_get_system_microtime(&sys, µsys); + +#if DEVELOPMENT || DEBUG + last_utc_sec = secs; + last_utc_usec = microsecs; + last_sys_sec = sys; + last_sys_usec = microsys; + if (secs > max_utc_sec) + max_utc_sec = secs; +#endif + + /* + * We normally expect the UTC clock to be always-on and produce + * greater readings than the tick counter. There may be corner cases + * due to differing clock resolutions (UTC clock is likely lower) and + * errors reading the UTC clock (some implementations return 0 on error) + * in which that doesn't hold true. Bring the UTC measurements in-line + * with the tick counter measurements as a best effort in that case. + */ + if ((sys > secs) || ((sys == secs) && (microsys > microsecs))) { + secs = sys; + microsecs = microsys; + } + + // This macro stores the subtraction result in secs and microsecs + TIME_SUB(secs, sys, microsecs, microsys, USEC_PER_SEC); + clock2bintime(&secs, µsecs, &utc_offset_bt); + + /* + * Safety belt: the UTC clock will likely have a lower resolution than the tick counter. + * It's also possible that the device didn't fully transition to the powered-off state on + * the most recent sleep, so the tick counter may not have reset or may have only briefly + * tured off. In that case it's possible for the difference between the UTC clock and the + * tick counter to be less than the previously recorded value in clock.calend.basesleep. + * In that case simply record that we slept for 0 ticks. + */ + if ((utc_offset_bt.sec > clock_calend.basesleep.sec) || + ((utc_offset_bt.sec == clock_calend.basesleep.sec) && (utc_offset_bt.frac > clock_calend.basesleep.frac))) { + + last_sleep_bt = utc_offset_bt; + bintime_sub(&last_sleep_bt, &clock_calend.basesleep); + clock_calend.basesleep = utc_offset_bt; + + bintime2absolutetime(&last_sleep_bt, &mach_absolutetime_last_sleep); + mach_absolutetime_asleep += mach_absolutetime_last_sleep; + + bintime_add(&clock_calend.offset, &last_sleep_bt); + bintime_add(&clock_calend.bintime, &last_sleep_bt); + } else + mach_absolutetime_last_sleep = 0; + + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_CLOCK,MACH_EPOCH_CHANGE) | DBG_FUNC_NONE, + (uintptr_t) mach_absolutetime_last_sleep, + (uintptr_t) mach_absolutetime_asleep, + (uintptr_t) (mach_absolutetime_last_sleep >> 32), + (uintptr_t) (mach_absolutetime_asleep >> 32), + 0); + + commpage_update_mach_continuous_time(mach_absolutetime_asleep); + adjust_cont_time_thread_calls(); clock_unlock(); splx(s); + + host_notify_calendar_change(); + +#if CONFIG_DTRACE + clock_track_calend_nowait(); +#endif } + + /* * clock_get_boottime_nanotime: * * Return the boottime, used by sysctl. */ void -clock_get_boottime_microtime( +clock_get_boottime_nanotime( clock_sec_t *secs, - clock_usec_t *microsecs) + clock_nsec_t *nanosecs) { spl_t s; @@ -646,237 +1081,34 @@ clock_get_boottime_microtime( clock_lock(); *secs = (clock_sec_t)clock_boottime; - *microsecs = (clock_nsec_t)clock_boottime_usec; + *nanosecs = (clock_nsec_t)clock_boottime_usec * NSEC_PER_USEC; clock_unlock(); splx(s); } /* - * clock_adjtime: - * - * Interface to adjtime() syscall. + * clock_get_boottime_nanotime: * - * Calculates adjustment variables and - * initiates adjustment. + * Return the boottime, used by sysctl. */ void -clock_adjtime( - long *secs, - int *microsecs) -{ - uint32_t interval; - spl_t s; - - s = splclock(); - clock_lock(); - - interval = calend_set_adjustment(secs, microsecs); - if (interval != 0) { - calend_adjdeadline = mach_absolute_time() + interval; - if (!timer_call_enter(&calend_adjcall, calend_adjdeadline, TIMER_CALL_SYS_CRITICAL)) - calend_adjactive++; - } - else - if (timer_call_cancel(&calend_adjcall)) - calend_adjactive--; - - clock_unlock(); - splx(s); -} - -static uint32_t -calend_set_adjustment( - long *secs, - int *microsecs) -{ - uint64_t now, t64; - int64_t total, ototal; - uint32_t interval = 0; - - /* - * Compute the total adjustment time in nanoseconds. - */ - total = ((int64_t)*secs * (int64_t)NSEC_PER_SEC) + (*microsecs * (int64_t)NSEC_PER_USEC); - - /* - * Disable commpage gettimeofday(). - */ - commpage_disable_timestamp(); - - /* - * Get current absolute time. - */ - now = mach_absolute_time(); - - /* - * Save the old adjustment total for later return. - */ - ototal = calend_adjtotal; - - /* - * Is a new correction specified? - */ - if (total != 0) { - /* - * Set delta to the standard, small, adjustment skew. - */ - int32_t delta = calend_adjskew; - - if (total > 0) { - /* - * Positive adjustment. If greater than the preset 'big' - * threshold, slew at a faster rate, capping if necessary. - */ - if (total > (int64_t) calend_adjbig) - delta *= 10; - if (delta > total) - delta = (int32_t)total; - - /* - * Convert the delta back from ns to absolute time and store in adjoffset. - */ - nanoseconds_to_absolutetime((uint64_t)delta, &t64); - clock_calend.adjoffset = (uint32_t)t64; - } - else { - /* - * Negative adjustment; therefore, negate the delta. If - * greater than the preset 'big' threshold, slew at a faster - * rate, capping if necessary. - */ - if (total < (int64_t) -calend_adjbig) - delta *= 10; - delta = -delta; - if (delta < total) - delta = (int32_t)total; - - /* - * Save the current absolute time. Subsequent time operations occuring - * during this negative correction can make use of this value to ensure - * that time increases monotonically. - */ - clock_calend.adjstart = now; - - /* - * Convert the delta back from ns to absolute time and store in adjoffset. - */ - nanoseconds_to_absolutetime((uint64_t)-delta, &t64); - clock_calend.adjoffset = (uint32_t)t64; - } - - /* - * Store the total adjustment time in ns. - */ - calend_adjtotal = total; - - /* - * Store the delta for this adjustment period in ns. - */ - clock_calend.adjdelta = delta; - - /* - * Set the interval in absolute time for later return. - */ - interval = calend_adjinterval; - } - else { - /* - * No change; clear any prior adjustment. - */ - calend_adjtotal = clock_calend.adjdelta = 0; - } - - /* - * If an prior correction was in progress, return the - * remaining uncorrected time from it. - */ - if (ototal != 0) { - *secs = (long)(ototal / (long)NSEC_PER_SEC); - *microsecs = (int)((ototal % (int)NSEC_PER_SEC) / (int)NSEC_PER_USEC); - } - else - *secs = *microsecs = 0; - -#if CONFIG_DTRACE - clock_track_calend_nowait(); -#endif - - return (interval); -} - -static void -calend_adjust_call(void) +clock_get_boottime_microtime( + clock_sec_t *secs, + clock_usec_t *microsecs) { - uint32_t interval; - spl_t s; + spl_t s; s = splclock(); clock_lock(); - if (--calend_adjactive == 0) { - interval = calend_adjust(); - if (interval != 0) { - clock_deadline_for_periodic_event(interval, mach_absolute_time(), &calend_adjdeadline); - - if (!timer_call_enter(&calend_adjcall, calend_adjdeadline, TIMER_CALL_SYS_CRITICAL)) - calend_adjactive++; - } - } + *secs = (clock_sec_t)clock_boottime; + *microsecs = (clock_nsec_t)clock_boottime_usec; clock_unlock(); splx(s); } -static uint32_t -calend_adjust(void) -{ - uint64_t now, t64; - int32_t delta; - uint32_t interval = 0; - - commpage_disable_timestamp(); - - now = mach_absolute_time(); - - delta = clock_calend.adjdelta; - - if (delta > 0) { - clock_calend.offset += clock_calend.adjoffset; - - calend_adjtotal -= delta; - if (delta > calend_adjtotal) { - clock_calend.adjdelta = delta = (int32_t)calend_adjtotal; - - nanoseconds_to_absolutetime((uint64_t)delta, &t64); - clock_calend.adjoffset = (uint32_t)t64; - } - } - else - if (delta < 0) { - clock_calend.offset -= clock_calend.adjoffset; - - calend_adjtotal -= delta; - if (delta < calend_adjtotal) { - clock_calend.adjdelta = delta = (int32_t)calend_adjtotal; - - nanoseconds_to_absolutetime((uint64_t)-delta, &t64); - clock_calend.adjoffset = (uint32_t)t64; - } - - if (clock_calend.adjdelta != 0) - clock_calend.adjstart = now; - } - - if (clock_calend.adjdelta != 0) - interval = calend_adjinterval; - -#if CONFIG_DTRACE - clock_track_calend_nowait(); -#endif - - return (interval); -} /* * Wait / delay routines. @@ -1141,6 +1373,7 @@ clock_get_calendar_nanotime_nowait( int i = 0; uint64_t now; struct unlocked_clock_calend stable; + struct bintime bt; for (;;) { stable = flipflop[i]; /* take snapshot */ @@ -1161,31 +1394,17 @@ clock_get_calendar_nanotime_nowait( if (flipflop[i].gen == stable.gen) break; - /* Switch to the oher element of the flipflop, and try again. */ + /* Switch to the other element of the flipflop, and try again. */ i ^= 1; } now = mach_absolute_time(); - if (stable.calend.adjdelta < 0) { - uint32_t t32; - - if (now > stable.calend.adjstart) { - t32 = (uint32_t)(now - stable.calend.adjstart); - - if (t32 > stable.calend.adjoffset) - now -= stable.calend.adjoffset; - else - now = stable.calend.adjstart; - } - } - - now += stable.calend.offset; + bt = get_scaled_time(now); - absolutetime_to_microtime(now, secs, nanosecs); - *nanosecs *= NSEC_PER_USEC; + bintime_add(&bt, &clock_calend.bintime); - *secs += (clock_sec_t)stable.calend.epoch; + bintime2nsclock(&bt, secs, nanosecs); } static void diff --git a/osfmk/kern/clock.h b/osfmk/kern/clock.h index 7a7b44328..3671e5efe 100644 --- a/osfmk/kern/clock.h +++ b/osfmk/kern/clock.h @@ -45,6 +45,7 @@ #include + #ifdef __LP64__ typedef unsigned long clock_sec_t; @@ -78,7 +79,7 @@ struct clock_ops { clock_attr_t attr, mach_msg_type_number_t *count); }; -typedef struct clock_ops *clock_ops_t; +typedef const struct clock_ops *clock_ops_t; typedef struct clock_ops clock_ops_data_t; /* @@ -86,7 +87,7 @@ typedef struct clock_ops clock_ops_data_t; * dependent operations list and clock operation ports. */ struct clock { - clock_ops_t cl_ops; /* operations list */ + clock_ops_t cl_ops; /* operations list */ struct ipc_port *cl_service; /* service port */ struct ipc_port *cl_control; /* control port */ }; @@ -111,12 +112,12 @@ extern void clock_timebase_init(void); */ extern void clock_service_create(void); -extern void clock_gettimeofday_set_commpage( - uint64_t abstime, - uint64_t epoch, - uint64_t offset, - clock_sec_t *secs, - clock_usec_t *microsecs); +extern void clock_gettimeofday_set_commpage( + uint64_t abstime, + uint64_t sec, + uint64_t frac, + uint64_t scale, + uint64_t tick_per_sec); extern void machine_delay_until(uint64_t interval, uint64_t deadline); @@ -142,6 +143,12 @@ extern void clock_initialize_calendar(void); extern void clock_wakeup_calendar(void); +extern void clock_update_calendar(void); + +extern void clock_get_calendar_uptime(clock_sec_t *secs); + +extern void clock_gettimeofday_new(clock_sec_t *secs, + clock_usec_t *microsecs); extern void clock_gettimeofday( clock_sec_t *secs, clock_usec_t *microsecs); diff --git a/osfmk/kern/clock_oldops.c b/osfmk/kern/clock_oldops.c index a3debc0d9..28da75b50 100644 --- a/osfmk/kern/clock_oldops.c +++ b/osfmk/kern/clock_oldops.c @@ -59,6 +59,7 @@ #include #include +#include /* * Actual clock alarm structure. Used for user clock_sleep() and @@ -137,7 +138,7 @@ kern_return_t rtclock_getattr( clock_attr_t attr, mach_msg_type_number_t *count); -struct clock_ops sysclk_ops = { +SECURITY_READ_ONLY_EARLY(struct clock_ops) sysclk_ops = { NULL, rtclock_init, rtclock_gettime, rtclock_getattr, @@ -151,7 +152,7 @@ kern_return_t calend_getattr( clock_attr_t attr, mach_msg_type_number_t *count); -struct clock_ops calend_ops = { +SECURITY_READ_ONLY_EARLY(struct clock_ops) calend_ops = { NULL, NULL, calend_gettime, calend_getattr, diff --git a/osfmk/kern/coalition.c b/osfmk/kern/coalition.c index 02661f548..9db2789ef 100644 --- a/osfmk/kern/coalition.c +++ b/osfmk/kern/coalition.c @@ -36,6 +36,7 @@ #include #include /* for TASK_CHUNK */ #include +#include #include #include @@ -68,6 +69,7 @@ extern ledger_template_t task_ledger_template; #define COALITION_CHUNK TASK_CHUNK int unrestrict_coalition_syscalls; +int merge_adaptive_coalitions; lck_attr_t coalitions_lck_attr; lck_grp_t coalitions_lck_grp; @@ -175,6 +177,7 @@ struct i_resource_coalition { uint64_t logical_deferred_writes; uint64_t logical_invalidated_writes; uint64_t logical_metadata_writes; + uint64_t cpu_ptime; uint64_t task_count; /* tasks that have started in this coalition */ uint64_t dead_task_count; /* tasks that have exited in this coalition; @@ -209,6 +212,7 @@ struct i_jetsam_coalition { queue_head_t extensions; queue_head_t services; queue_head_t other; + thread_group_t thread_group; }; @@ -218,6 +222,7 @@ struct i_jetsam_coalition { struct coalition { uint64_t id; /* monotonically increasing */ uint32_t type; + uint32_t role; /* default task role (background, adaptive, interactive, etc) */ uint32_t ref_count; /* Number of references to the memory containing this struct */ uint32_t active_count; /* Number of members of (tasks in) the coalition, plus vouchers referring @@ -235,7 +240,8 @@ struct coalition { uint32_t terminated : 1; /* coalition became empty and spawns are now forbidden */ uint32_t reaped : 1; /* reaped, invisible to userspace, but waiting for ref_count to go to zero */ uint32_t notified : 1; /* no-more-processes notification was sent via special port */ -#if defined(DEVELOPMENT) || defined(DEBUG) + uint32_t efficient : 1; /* launchd has marked the coalition as efficient */ +#if DEVELOPMENT || DEBUG uint32_t should_notify : 1; /* should this coalition send notifications (default: yes) */ #endif @@ -287,6 +293,20 @@ s_coalition_types[COALITION_NUM_TYPES] = { #define coalition_lock(c) do{ lck_mtx_lock(&c->lock); }while(0) #define coalition_unlock(c) do{ lck_mtx_unlock(&c->lock); }while(0) +/* + * Define the coalition type to track focal tasks. + * On embedded, track them using jetsam coalitions since they have associated thread + * groups which reflect this property as a flag (and pass it down to CLPC). + * On non-embedded platforms, since not all coalitions have jetsam coalitions + * track focal counts on the resource coalition. + */ +#if CONFIG_EMBEDDED +#define COALITION_FOCAL_TASKS_ACCOUNTING COALITION_TYPE_JETSAM +#else /* CONFIG_EMBEDDED */ +#define COALITION_FOCAL_TASKS_ACCOUNTING COALITION_TYPE_RESOURCE +#endif /* CONFIG_EMBEDDED */ + + static void coalition_notify_user(uint64_t id, uint32_t flags) { @@ -394,11 +414,16 @@ i_coal_resource_remove_task(coalition_t coal, task_t task) ledger_rollup(cr->ledger, task->ledger); cr->bytesread += task->task_io_stats->disk_reads.size; cr->byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size; +#if !CONFIG_EMBEDDED cr->gpu_time += task_gpu_utilisation(task); +#else + cr->energy += task_energy(task); +#endif cr->logical_immediate_writes += task->task_immediate_writes; cr->logical_deferred_writes += task->task_deferred_writes; cr->logical_invalidated_writes += task->task_invalidated_writes; cr->logical_metadata_writes += task->task_metadata_writes; + cr->cpu_ptime += task_cpu_ptime(task); } /* remove the task from the coalition's list */ @@ -481,28 +506,13 @@ coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_us uint64_t logical_metadata_writes = coal->r.logical_metadata_writes; int64_t cpu_time_billed_to_me = 0; int64_t cpu_time_billed_to_others = 0; - - kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_me, (int64_t *)&cpu_time_billed_to_me); - if (kr != KERN_SUCCESS || cpu_time_billed_to_me < 0) { -#if DEVELOPMENT || DEBUG - printf("ledger_get_balance failed or ledger negative in coalition_resource_usage_internal: %lld\n", cpu_time_billed_to_me); -#endif /* DEVELOPMENT || DEBUG */ - cpu_time_billed_to_me = 0; - } - - kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_others, (int64_t *)&cpu_time_billed_to_others); - if (kr != KERN_SUCCESS || cpu_time_billed_to_others < 0) { -#if DEVELOPMENT || DEBUG - printf("ledger_get_balance failed or ledger negative in coalition_resource_usage_internal: %lld\n", cpu_time_billed_to_others); -#endif /* DEVELOPMENT || DEBUG */ - cpu_time_billed_to_others = 0; - } + int64_t energy_billed_to_me = 0; + int64_t energy_billed_to_others = 0; + uint64_t cpu_ptime = coal->r.cpu_ptime; /* * Add to that all the active tasks' ledgers. Tasks cannot deallocate * out from under us, since we hold the coalition lock. - * Do not use the on-behalf of cpu time from ledger for live tasks, since - * it will not have cpu time for active linkages between tasks. */ task_t task; qe_foreach_element(task, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) { @@ -517,13 +527,36 @@ coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_us ledger_rollup(sum_ledger, task->ledger); bytesread += task->task_io_stats->disk_reads.size; byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size; +#if !CONFIG_EMBEDDED gpu_time += task_gpu_utilisation(task); +#else + energy += task_energy(task); +#endif logical_immediate_writes += task->task_immediate_writes; logical_deferred_writes += task->task_deferred_writes; logical_invalidated_writes += task->task_invalidated_writes; logical_metadata_writes += task->task_metadata_writes; - cpu_time_billed_to_me += (int64_t)bank_billed_time_safe(task); - cpu_time_billed_to_others += (int64_t)bank_serviced_time_safe(task); + cpu_ptime += task_cpu_ptime(task); + } + + kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_me, (int64_t *)&cpu_time_billed_to_me); + if (kr != KERN_SUCCESS || cpu_time_billed_to_me < 0) { + cpu_time_billed_to_me = 0; + } + + kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_others, (int64_t *)&cpu_time_billed_to_others); + if (kr != KERN_SUCCESS || cpu_time_billed_to_others < 0) { + cpu_time_billed_to_others = 0; + } + + kr = ledger_get_balance(sum_ledger, task_ledgers.energy_billed_to_me, (int64_t *)&energy_billed_to_me); + if (kr != KERN_SUCCESS || energy_billed_to_me < 0) { + energy_billed_to_me = 0; + } + + kr = ledger_get_balance(sum_ledger, task_ledgers.energy_billed_to_others, (int64_t *)&energy_billed_to_others); + if (kr != KERN_SUCCESS || energy_billed_to_others < 0) { + energy_billed_to_others = 0; } /* collect information from the coalition itself */ @@ -544,6 +577,8 @@ coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_us cru_out->cpu_time = credit; cru_out->cpu_time_billed_to_me = (uint64_t)cpu_time_billed_to_me; cru_out->cpu_time_billed_to_others = (uint64_t)cpu_time_billed_to_others; + cru_out->energy_billed_to_me = (uint64_t)energy_billed_to_me; + cru_out->energy_billed_to_others = (uint64_t)energy_billed_to_others; kr = ledger_get_entries(sum_ledger, task_ledgers.interrupt_wakeups, &credit, &debit); @@ -567,6 +602,7 @@ coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_us cru_out->logical_deferred_writes = logical_deferred_writes; cru_out->logical_invalidated_writes = logical_invalidated_writes; cru_out->logical_metadata_writes = logical_metadata_writes; + cru_out->cpu_ptime = cpu_ptime; ledger_dereference(sum_ledger); sum_ledger = LEDGER_NULL; @@ -608,6 +644,7 @@ i_coal_jetsam_dealloc(__unused coalition_t coal) assert(queue_empty(&coal->j.services)); assert(queue_empty(&coal->j.other)); assert(coal->j.leader == TASK_NULL); + } static kern_return_t @@ -681,11 +718,6 @@ i_coal_jetsam_set_taskrole(coalition_t coal, task_t task, int role) /* set the coalition leader */ cj->leader = task; break; - case COALITION_TASKROLE_UNDEF: - coal_dbg("setting PID:%d as UNDEF in %lld", - task_pid(task), coal->id); - q = (queue_t)&cj->other; - break; case COALITION_TASKROLE_XPC: coal_dbg("setting PID:%d as XPC in %lld", task_pid(task), coal->id); @@ -696,6 +728,23 @@ i_coal_jetsam_set_taskrole(coalition_t coal, task_t task, int role) task_pid(task), coal->id); q = (queue_t)&cj->extensions; break; + case COALITION_TASKROLE_NONE: + /* + * Tasks with a role of "none" should fall through to an + * undefined role so long as the task is currently a member + * of the coalition. This scenario can happen if a task is + * killed (usually via jetsam) during exec. + */ + if (task->coalition[COALITION_TYPE_JETSAM] != coal) { + panic("%s: task %p attempting to set role %d " + "in coalition %p to which it does not belong!", __func__, task, role, coal); + } + /* fall through */ + case COALITION_TASKROLE_UNDEF: + coal_dbg("setting PID:%d as UNDEF in %lld", + task_pid(task), coal->id); + q = (queue_t)&cj->other; + break; default: panic("%s: invalid role(%d) for task", __func__, role); return KERN_INVALID_ARGUMENT; @@ -737,7 +786,7 @@ i_coal_jetsam_get_taskrole(coalition_t coal, task_t task) } /* task not in the coalition?! */ - return -1; + return COALITION_TASKROLE_NONE; } static void @@ -776,7 +825,7 @@ i_coal_jetsam_iterate_tasks(coalition_t coal, void *ctx, void (*callback)(coalit * Condition: coalitions_list_lock must be UNLOCKED. */ kern_return_t -coalition_create_internal(int type, boolean_t privileged, coalition_t *out) +coalition_create_internal(int type, int role, boolean_t privileged, coalition_t *out) { kern_return_t kr; struct coalition *new_coal; @@ -790,6 +839,7 @@ coalition_create_internal(int type, boolean_t privileged, coalition_t *out) bzero(new_coal, sizeof(*new_coal)); new_coal->type = type; + new_coal->role = role; /* initialize type-specific resources */ kr = coal_call(new_coal, init, privileged); @@ -802,7 +852,7 @@ coalition_create_internal(int type, boolean_t privileged, coalition_t *out) new_coal->ref_count = 2; new_coal->privileged = privileged ? TRUE : FALSE; -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG new_coal->should_notify = 1; #endif @@ -812,6 +862,9 @@ coalition_create_internal(int type, boolean_t privileged, coalition_t *out) new_coal->id = coalition_next_id++; coalition_count++; enqueue_tail(&coalitions_q, &new_coal->coalitions); + + KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_NEW), + new_coal->id, new_coal->type); lck_mtx_unlock(&coalitions_list_lock); coal_dbg("id:%llu, type:%s", new_coal->id, coal_type_str(new_coal->type)); @@ -851,6 +904,8 @@ coalition_release(coalition_t coal) assert(coal->reaped); assert(coal->focal_task_count == 0); assert(coal->nonfocal_task_count == 0); + KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_FREE), + coal->id, coal->type); coal_call(coal, dealloc); @@ -1016,7 +1071,7 @@ task_coalition_roles(task_t task, int roles[COALITION_NUM_TYPES]) get_taskrole, task); coalition_unlock(task->coalition[i]); } else { - roles[i] = -1; + roles[i] = COALITION_TASKROLE_NONE; } } } @@ -1028,6 +1083,24 @@ coalition_type(coalition_t coal) return coal->type; } +boolean_t +coalition_term_requested(coalition_t coal) +{ + return coal->termrequested; +} + +boolean_t +coalition_is_terminated(coalition_t coal) +{ + return coal->terminated; +} + +boolean_t +coalition_is_reaped(coalition_t coal) +{ + return coal->reaped; +} + boolean_t coalition_is_privileged(coalition_t coal) { @@ -1062,64 +1135,54 @@ void task_coalition_update_gpu_stats(task_t task, uint64_t gpu_ns_delta) coalition_unlock(coal); } -uint32_t task_coalition_adjust_focal_count(task_t task, int count) +boolean_t task_coalition_adjust_focal_count(task_t task, int count, uint32_t *new_count) { - coalition_t coal; - uint32_t ret; - - /* - * For now: only use the resource coalition. Perhaps in the - * future we may combine all coalition types, or even make - * a special coalition type just for this. - */ - coal = task->coalition[COALITION_TYPE_RESOURCE]; - assert(coal != COALITION_NULL); - - ret = hw_atomic_add(&coal->focal_task_count, count); + coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; + if (coal == COALITION_NULL) + return FALSE; - /* catch underflow */ - assert(ret != UINT32_MAX); - return ret; + *new_count = hw_atomic_add(&coal->focal_task_count, count); + assert(*new_count != UINT32_MAX); + return TRUE; } uint32_t task_coalition_focal_count(task_t task) { - coalition_t coal; - coal = task->coalition[COALITION_TYPE_RESOURCE]; - assert(coal != COALITION_NULL); + coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; + if (coal == COALITION_NULL) + return 0; return coal->focal_task_count; } -uint32_t task_coalition_adjust_nonfocal_count(task_t task, int count) +boolean_t task_coalition_adjust_nonfocal_count(task_t task, int count, uint32_t *new_count) { - coalition_t coal; - uint32_t ret; - - /* - * For now: only use the resource coalition. Perhaps in the - * future we may combine all coalition types, or even make - * a special coalition type just for this. - */ - coal = task->coalition[COALITION_TYPE_RESOURCE]; - assert(coal != COALITION_NULL); - - ret = hw_atomic_add(&coal->nonfocal_task_count, count); + coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; + if (coal == COALITION_NULL) + return FALSE; - /* catch underflow */ - assert(ret != UINT32_MAX); - return ret; + *new_count = hw_atomic_add(&coal->nonfocal_task_count, count); + assert(*new_count != UINT32_MAX); + return TRUE; } uint32_t task_coalition_nonfocal_count(task_t task) { - coalition_t coal; - coal = task->coalition[COALITION_TYPE_RESOURCE]; - assert(coal != COALITION_NULL); + coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; + if (coal == COALITION_NULL) + return 0; return coal->nonfocal_task_count; } +void coalition_set_efficient(coalition_t coal) +{ + coalition_lock(coal); + coal->efficient = TRUE; + coalition_unlock(coal); +} + + void coalition_for_each_task(coalition_t coal, void *ctx, void (*callback)(coalition_t, void *, task_t)) { @@ -1160,7 +1223,7 @@ coalition_remove_active(coalition_t coal) assert(!coal->notified); coal->notified = TRUE; -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG do_notify = coal->should_notify; #else do_notify = TRUE; @@ -1247,6 +1310,12 @@ out_unlock: uint32_t rc = coal->ref_count; uint32_t ct = coal->type; #endif + if (get_task_uniqueid(task) != UINT64_MAX) { + /* On 32-bit targets, uniqueid will get truncated to 32 bits */ + KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_ADOPT), + coal->id, get_task_uniqueid(task)); + } + coalition_unlock(coal); coal_dbg("task:%d, id:%llu type:%s ref_count:%u, kr=%d", @@ -1276,6 +1345,8 @@ coalition_remove_task_internal(task_t task, int type) int ac = coal->active_count; int ct = coal->type; #endif + KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_REMOVE), + coal->id, get_task_uniqueid(task)); coalition_unlock(coal); coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u, kr=%d", @@ -1425,7 +1496,7 @@ coalition_request_terminate_internal(coalition_t coal) assert(!coal->notified); coal->notified = TRUE; -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG do_notify = coal->should_notify; #else do_notify = TRUE; @@ -1490,7 +1561,7 @@ coalition_reap_internal(coalition_t coal) return KERN_SUCCESS; } -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG int coalition_should_notify(coalition_t coal) { int should; @@ -1533,6 +1604,11 @@ coalitions_init(void) unrestrict_coalition_syscalls = 0; } + if (!PE_parse_boot_argn("tg_adaptive", &merge_adaptive_coalitions, + sizeof (merge_adaptive_coalitions))) { + merge_adaptive_coalitions = 0; + } + lck_grp_attr_setdefault(&coalitions_lck_grp_attr); lck_grp_init(&coalitions_lck_grp, "coalition", &coalitions_lck_grp_attr); lck_attr_setdefault(&coalitions_lck_attr); @@ -1552,11 +1628,11 @@ coalitions_init(void) } if (!ctype->has_default) continue; - kr = coalition_create_internal(ctype->type, TRUE, &init_coalition[ctype->type]); + kr = coalition_create_internal(ctype->type, COALITION_ROLE_SYSTEM, TRUE, &init_coalition[ctype->type]); if (kr != KERN_SUCCESS) panic("%s: could not create init %s coalition: kr:%d", __func__, coal_type_str(i), kr); - kr = coalition_create_internal(ctype->type, FALSE, &corpse_coalition[ctype->type]); + kr = coalition_create_internal(ctype->type, COALITION_ROLE_SYSTEM, FALSE, &corpse_coalition[ctype->type]); if (kr != KERN_SUCCESS) panic("%s: could not create corpse %s coalition: kr:%d", __func__, coal_type_str(i), kr); @@ -1634,6 +1710,32 @@ boolean_t coalition_is_leader(task_t task, int coal_type, coalition_t *coal) return ret; } +kern_return_t coalition_iterate_stackshot(coalition_iterate_fn_t callout, void *arg, uint32_t coalition_type) +{ + coalition_t coal; + int i = 0; + + qe_foreach_element(coal, &coalitions_q, coalitions) { + if (coal == NULL || !ml_validate_nofault((vm_offset_t)coal, sizeof(struct coalition))) + return KERN_FAILURE; + + if (coalition_type == coal->type) + callout(arg, i++, coal); + } + + return KERN_SUCCESS; +} + +task_t kdp_coalition_get_leader(coalition_t coal) +{ + if (!coal) + return TASK_NULL; + + if (coal->type == COALITION_TYPE_JETSAM) { + return coal->j.leader; + } + return TASK_NULL; +} task_t coalition_get_leader(coalition_t coal) { diff --git a/osfmk/kern/coalition.h b/osfmk/kern/coalition.h index 87962bd8b..10cc5b742 100644 --- a/osfmk/kern/coalition.h +++ b/osfmk/kern/coalition.h @@ -32,6 +32,7 @@ /* only kernel-private interfaces */ #ifdef XNU_KERNEL_PRIVATE #include +#include #if CONFIG_COALITIONS @@ -63,14 +64,24 @@ void task_coalition_roles(task_t task, int roles[COALITION_NUM_TYPES]); int coalition_type(coalition_t coal); void task_coalition_update_gpu_stats(task_t task, uint64_t gpu_ns_delta); -uint32_t task_coalition_adjust_focal_count(task_t task, int count); +boolean_t task_coalition_adjust_focal_count(task_t task, int count, uint32_t *new_count); uint32_t task_coalition_focal_count(task_t task); -uint32_t task_coalition_adjust_nonfocal_count(task_t task, int count); +boolean_t task_coalition_adjust_nonfocal_count(task_t task, int count, uint32_t *new_count); uint32_t task_coalition_nonfocal_count(task_t task); +thread_group_t task_coalition_get_thread_group(task_t task); +void coalition_set_thread_group(coalition_t coal, thread_group_t tg); +thread_group_t kdp_coalition_get_thread_group(coalition_t coal); +thread_group_t coalition_get_thread_group(coalition_t coal); +void task_coalition_thread_group_focal_update(task_t task); void coalition_for_each_task(coalition_t coal, void *ctx, void (*callback)(coalition_t, void *, task_t)); +void coalition_set_efficient(coalition_t coal); + +typedef void (*coalition_iterate_fn_t)(void*, int, coalition_t); +kern_return_t coalition_iterate_stackshot(coalition_iterate_fn_t callout, void *arg, uint32_t coalition_type); + /* Returns with a reference, or COALITION_NULL. * There is no coalition with id 0. */ @@ -110,17 +121,22 @@ kern_return_t coalition_request_terminate_internal(coalition_t coal); * KERN_RESOURCE_SHORTAGE Unable to allocate kernel resources for a * new coalition. */ -kern_return_t coalition_create_internal(int type, boolean_t privileged, coalition_t *out); +kern_return_t coalition_create_internal(int type, int role, boolean_t privileged, coalition_t *out); +boolean_t coalition_term_requested(coalition_t coal); +boolean_t coalition_is_terminated(coalition_t coal); +boolean_t coalition_is_reaped(coalition_t coal); boolean_t coalition_is_privileged(coalition_t coal); boolean_t task_is_in_privileged_coalition(task_t task, int type); kern_return_t coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_usage *cru_out); +task_t kdp_coalition_get_leader(coalition_t coal); + /* * development/debug interfaces */ -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG int coalition_should_notify(coalition_t coal); void coalition_set_notify(coalition_t coal, int notify); #endif @@ -133,16 +149,18 @@ static inline void task_coalition_update_gpu_stats(__unused task_t task, return; } -static inline uint32_t task_coalition_adjust_focal_count(__unused task_t task, - __unused int count) +static inline boolean_t task_coalition_adjust_focal_count(__unused task_t task, + __unused int count, + __unused uint32_t *new_count) { - return 0; + return FALSE; } -static inline uint32_t task_coalition_adjust_nonfocal_count(__unused task_t task, - __unused int count) +static inline boolean_t task_coalition_adjust_nonfocal_count(__unused task_t task, + __unused int count, + __unused uint32_t *new_count) { - return 0; + return FALSE; } static inline uint32_t task_coalition_focal_count(__unused task_t task) diff --git a/osfmk/kern/copyout_shim.c b/osfmk/kern/copyout_shim.c new file mode 100644 index 000000000..ea553dc6a --- /dev/null +++ b/osfmk/kern/copyout_shim.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include +#include +#include +#include +#include + +#if (DEVELOPMENT || DEBUG) +#define UNUSED_IN_RELEASE(x) +#else +//supress compiler warnings about unused variables +#define UNUSED_IN_RELEASE(x) (void)(x) +#endif /* (DEVELOPMENT || DEBUG) */ + + +#if (DEVELOPMENT || DEBUG) +copyout_shim_fn_t copyout_shim_fn=NULL; +unsigned co_src_flags=0; +#endif + +kern_return_t register_copyout_shim(void (*fn)(const void *,user_addr_t,vm_size_t,unsigned co_src),unsigned types) +{ +#if (DEVELOPMENT || DEBUG) + int copyout_shim_enabled=0; + + if(!fn) + { + /* unregistration is always allowed */ + copyout_shim_fn=NULL; + return KERN_SUCCESS; + } + + if(copyout_shim_fn) + { + //need to unregister first before registering a new one. + return KERN_FAILURE; + } + + if(!PE_parse_boot_argn("enable_copyout_shim",©out_shim_enabled,sizeof(copyout_shim_enabled)) || !copyout_shim_enabled) + { + return KERN_FAILURE; + } + + + co_src_flags=types; + copyout_shim_fn=fn; + return KERN_SUCCESS; +#else + UNUSED_IN_RELEASE(fn); + UNUSED_IN_RELEASE(types); + return KERN_FAILURE; +#endif +} + +void *cos_kernel_unslide(const void *ptr) +{ +#if (DEVELOPMENT || DEBUG) + return (void *)(VM_KERNEL_UNSLIDE(ptr)); +#else + UNUSED_IN_RELEASE(ptr); + return NULL; +#endif +} + +void *cos_kernel_reslide(const void *ptr) +{ +#if (DEVELOPMENT || DEBUG) + return (void *)(VM_KERNEL_SLIDE(ptr)); +#else + UNUSED_IN_RELEASE(ptr); + return NULL; +#endif +} diff --git a/osfmk/kern/copyout_shim.h b/osfmk/kern/copyout_shim.h new file mode 100644 index 000000000..200cd4515 --- /dev/null +++ b/osfmk/kern/copyout_shim.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef __COPYOUT_SHIM_X86_64_H__ +#define __COPYOUT_SHIM_X86_64_H__ +#ifdef KERNEL_PRIVATE + +// Osfmk includes libsa/types.h which causes massive conflicts +// with sys/types.h +#if defined (__i386__) || defined(__x86_64__) +#include "i386/types.h" +#elif defined (__arm__) || defined (__arm64__) +//XXX when ready to turn on for arm: #include "arm/types.h" +#error ARM/ARM64 not supported +#else +#error architecture not supported +#endif + +#include + +#define CO_SRC_NORMAL 1 //copyout() called +#define CO_SRC_MSG (1<<1) //copyoutmsg() called +#define CO_SRC_PHYS (1<<2) //copyio(COPYOUTPHYS,...) called + +typedef void (*copyout_shim_fn_t)(const void *,user_addr_t,vm_size_t,unsigned co_src); + +#ifdef MACH_KERNEL_PRIVATE +#if(DEVELOPMENT || DEBUG) && (COPYOUT_SHIM > 0) + +extern copyout_shim_fn_t copyout_shim_fn; +extern unsigned co_src_flags; + +// void call_copyout_shim(const void *kernel_addr,user_addr_t user_addr,vm_size_t nbytes,int copy_type,int copyout_flavors); + +#define CALL_COPYOUT_SHIM_NRML(ka,ua,nb) \ + if(copyout_shim_fn && (co_src_flags & CO_SRC_NORMAL)) {copyout_shim_fn(ka,ua,nb,CO_SRC_NORMAL); } + +#define CALL_COPYOUT_SHIM_MSG(ka,ua,nb) \ + if(copyout_shim_fn && (co_src_flags & CO_SRC_MSG)){copyout_shim_fn(ka,ua,nb,CO_SRC_MSG); } + +#define CALL_COPYOUT_SHIM_PHYS(ka,ua,nb) \ + if(copyout_shim_fn && (co_src_flags & CO_SRC_PHYS)){copyout_shim_fn(ka,ua,nb,CO_SRC_PHYS); } + +#else + //Make these calls disappear if we're RELEASE or if COPYOUT_SHIM didn't get built +#define CALL_COPYOUT_SHIM_NRML(ka,ua,nb) +#define CALL_COPYOUT_SHIM_MSG(ka,ua,nb) +#define CALL_COPYOUT_SHIM_PHYS(ka,ua,nb) +#endif /* (DEVELOPMENT || DEBUG) && (COPYOUT_SHIM > 0) */ +#endif /* MACH_KERNEL_PRIVATE */ + + +kern_return_t +register_copyout_shim(copyout_shim_fn_t copyout_shim_fn,unsigned co_src_flags); + + +#define unregister_copyout_shim() register_copyout_shim(NULL,0) + +void * +cos_kernel_unslide(const void *); + +void * +cos_kernel_reslide(const void *); + +#endif /* KERNEL_PRIVATE */ +#endif /* __COPYOUT_SHIM_X86_64_H__ */ diff --git a/osfmk/kern/cpu_number.h b/osfmk/kern/cpu_number.h index 9a85e2514..2894c24a9 100644 --- a/osfmk/kern/cpu_number.h +++ b/osfmk/kern/cpu_number.h @@ -59,12 +59,8 @@ #ifndef _KERN_CPU_NUMBER_H_ #define _KERN_CPU_NUMBER_H_ -#ifdef MACH_KERNEL_PRIVATE - extern int master_cpu; -#endif /* MACH_KERNEL_PRIVATE */ - #include #endif /* _KERN_CPU_NUMBER_H_ */ diff --git a/osfmk/kern/cs_blobs.h b/osfmk/kern/cs_blobs.h new file mode 100644 index 000000000..e8007e9f2 --- /dev/null +++ b/osfmk/kern/cs_blobs.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _KERN_CODESIGN_H_ +#define _KERN_CODESIGN_H_ + +/* code signing attributes of a process */ +#define CS_VALID 0x0000001 /* dynamically valid */ +#define CS_ADHOC 0x0000002 /* ad hoc signed */ +#define CS_GET_TASK_ALLOW 0x0000004 /* has get-task-allow entitlement */ +#define CS_INSTALLER 0x0000008 /* has installer entitlement */ + +#define CS_HARD 0x0000100 /* don't load invalid pages */ +#define CS_KILL 0x0000200 /* kill process if it becomes invalid */ +#define CS_CHECK_EXPIRATION 0x0000400 /* force expiration checking */ +#define CS_RESTRICT 0x0000800 /* tell dyld to treat restricted */ +#define CS_ENFORCEMENT 0x0001000 /* require enforcement */ +#define CS_REQUIRE_LV 0x0002000 /* require library validation */ +#define CS_ENTITLEMENTS_VALIDATED 0x0004000 /* code signature permits restricted entitlements */ +#define CS_NVRAM_UNRESTRICTED 0x0008000 /* has com.apple.rootless.restricted-nvram-variables.heritable entitlement */ + +#define CS_ALLOWED_MACHO (CS_ADHOC | CS_HARD | CS_KILL | CS_CHECK_EXPIRATION | \ + CS_RESTRICT | CS_ENFORCEMENT | CS_REQUIRE_LV) + +#define CS_EXEC_SET_HARD 0x0100000 /* set CS_HARD on any exec'ed process */ +#define CS_EXEC_SET_KILL 0x0200000 /* set CS_KILL on any exec'ed process */ +#define CS_EXEC_SET_ENFORCEMENT 0x0400000 /* set CS_ENFORCEMENT on any exec'ed process */ +#define CS_EXEC_INHERIT_SIP 0x0800000 /* set CS_INSTALLER on any exec'ed process */ + +#define CS_KILLED 0x1000000 /* was killed by kernel for invalidity */ +#define CS_DYLD_PLATFORM 0x2000000 /* dyld used to load this is a platform binary */ +#define CS_PLATFORM_BINARY 0x4000000 /* this is a platform binary */ +#define CS_PLATFORM_PATH 0x8000000 /* platform binary by the fact of path (osx only) */ +#define CS_DEBUGGED 0x10000000 /* process is currently or has previously been debugged and allowed to run with invalid pages */ +#define CS_SIGNED 0x20000000 /* process has a signature (may have gone invalid) */ +#define CS_DEV_CODE 0x40000000 /* code is dev signed, cannot be loaded into prod signed code (will go away with rdar://problem/28322552) */ +#define CS_DATAVAULT_CONTROLLER 0x80000000 /* has Data Vault controller entitlement */ + +#define CS_ENTITLEMENT_FLAGS (CS_GET_TASK_ALLOW | CS_INSTALLER | CS_DATAVAULT_CONTROLLER | CS_NVRAM_UNRESTRICTED) + +/* executable segment flags */ + +#define CS_EXECSEG_MAIN_BINARY 0x1 /* executable segment denotes main binary */ +#define CS_EXECSEG_ALLOW_UNSIGNED 0x10 /* allow unsigned pages (for debugging) */ +#define CS_EXECSEG_DEBUGGER 0x20 /* main binary is debugger */ +#define CS_EXECSEG_JIT 0x40 /* JIT enabled */ +#define CS_EXECSEG_SKIP_LV 0x80 /* skip library validation */ +#define CS_EXECSEG_CAN_LOAD_CDHASH 0x100 /* can bless cdhash for execution */ +#define CS_EXECSEG_CAN_EXEC_CDHASH 0x200 /* can execute blessed cdhash */ + +/* + * Magic numbers used by Code Signing + */ +enum { + CSMAGIC_REQUIREMENT = 0xfade0c00, /* single Requirement blob */ + CSMAGIC_REQUIREMENTS = 0xfade0c01, /* Requirements vector (internal requirements) */ + CSMAGIC_CODEDIRECTORY = 0xfade0c02, /* CodeDirectory blob */ + CSMAGIC_EMBEDDED_SIGNATURE = 0xfade0cc0, /* embedded form of signature data */ + CSMAGIC_EMBEDDED_SIGNATURE_OLD = 0xfade0b02, /* XXX */ + CSMAGIC_EMBEDDED_ENTITLEMENTS = 0xfade7171, /* embedded entitlements */ + CSMAGIC_DETACHED_SIGNATURE = 0xfade0cc1, /* multi-arch collection of embedded signatures */ + CSMAGIC_BLOBWRAPPER = 0xfade0b01, /* CMS Signature, among other things */ + + CS_SUPPORTSSCATTER = 0x20100, + CS_SUPPORTSTEAMID = 0x20200, + CS_SUPPORTSCODELIMIT64 = 0x20300, + CS_SUPPORTSEXECSEG = 0x20400, + + CSSLOT_CODEDIRECTORY = 0, /* slot index for CodeDirectory */ + CSSLOT_INFOSLOT = 1, + CSSLOT_REQUIREMENTS = 2, + CSSLOT_RESOURCEDIR = 3, + CSSLOT_APPLICATION = 4, + CSSLOT_ENTITLEMENTS = 5, + + CSSLOT_ALTERNATE_CODEDIRECTORIES = 0x1000, /* first alternate CodeDirectory, if any */ + CSSLOT_ALTERNATE_CODEDIRECTORY_MAX = 5, /* max number of alternate CD slots */ + CSSLOT_ALTERNATE_CODEDIRECTORY_LIMIT = CSSLOT_ALTERNATE_CODEDIRECTORIES + CSSLOT_ALTERNATE_CODEDIRECTORY_MAX, /* one past the last */ + + CSSLOT_SIGNATURESLOT = 0x10000, /* CMS Signature */ + + CSTYPE_INDEX_REQUIREMENTS = 0x00000002, /* compat with amfi */ + CSTYPE_INDEX_ENTITLEMENTS = 0x00000005, /* compat with amfi */ + + CS_HASHTYPE_SHA1 = 1, + CS_HASHTYPE_SHA256 = 2, + CS_HASHTYPE_SHA256_TRUNCATED = 3, + CS_HASHTYPE_SHA384 = 4, + + CS_SHA1_LEN = 20, + CS_SHA256_LEN = 32, + CS_SHA256_TRUNCATED_LEN = 20, + + CS_CDHASH_LEN = 20, /* always - larger hashes are truncated */ + CS_HASH_MAX_SIZE = 48, /* max size of the hash we'll support */ + +/* + * Currently only to support Legacy VPN plugins, + * but intended to replace all the various platform code, dev code etc. bits. + */ + CS_SIGNER_TYPE_UNKNOWN = 0, + CS_SIGNER_TYPE_LEGACYVPN = 5, +}; + +#define KERNEL_HAVE_CS_CODEDIRECTORY 1 +#define KERNEL_CS_CODEDIRECTORY_HAVE_PLATFORM 1 + +/* + * C form of a CodeDirectory. + */ +typedef struct __CodeDirectory { + uint32_t magic; /* magic number (CSMAGIC_CODEDIRECTORY) */ + uint32_t length; /* total length of CodeDirectory blob */ + uint32_t version; /* compatibility version */ + uint32_t flags; /* setup and mode flags */ + uint32_t hashOffset; /* offset of hash slot element at index zero */ + uint32_t identOffset; /* offset of identifier string */ + uint32_t nSpecialSlots; /* number of special hash slots */ + uint32_t nCodeSlots; /* number of ordinary (code) hash slots */ + uint32_t codeLimit; /* limit to main image signature range */ + uint8_t hashSize; /* size of each hash in bytes */ + uint8_t hashType; /* type of hash (cdHashType* constants) */ + uint8_t platform; /* platform identifier; zero if not platform binary */ + uint8_t pageSize; /* log2(page size in bytes); 0 => infinite */ + uint32_t spare2; /* unused (must be zero) */ + + char end_earliest[0]; + + /* Version 0x20100 */ + uint32_t scatterOffset; /* offset of optional scatter vector */ + char end_withScatter[0]; + + /* Version 0x20200 */ + uint32_t teamOffset; /* offset of optional team identifier */ + char end_withTeam[0]; + + /* Version 0x20300 */ + uint32_t spare3; /* unused (must be zero) */ + uint64_t codeLimit64; /* limit to main image signature range, 64 bits */ + char end_withCodeLimit64[0]; + + /* Version 0x20400 */ + uint64_t execSegBase; /* offset of executable segment */ + uint64_t execSegLimit; /* limit of executable segment */ + uint64_t execSegFlags; /* executable segment flags */ + char end_withExecSeg[0]; + + /* followed by dynamic content as located by offset fields above */ +} CS_CodeDirectory +__attribute__ ((aligned(1))); + +/* + * Structure of an embedded-signature SuperBlob + */ + +typedef struct __BlobIndex { + uint32_t type; /* type of entry */ + uint32_t offset; /* offset of entry */ +} CS_BlobIndex +__attribute__ ((aligned(1))); + +typedef struct __SC_SuperBlob { + uint32_t magic; /* magic number */ + uint32_t length; /* total length of SuperBlob */ + uint32_t count; /* number of index entries following */ + CS_BlobIndex index[]; /* (count) entries */ + /* followed by Blobs in no particular order as indicated by offsets in index */ +} CS_SuperBlob +__attribute__ ((aligned(1))); + +#define KERNEL_HAVE_CS_GENERICBLOB 1 +typedef struct __SC_GenericBlob { + uint32_t magic; /* magic number */ + uint32_t length; /* total length of blob */ + char data[]; +} CS_GenericBlob +__attribute__ ((aligned(1))); + +typedef struct __SC_Scatter { + uint32_t count; // number of pages; zero for sentinel (only) + uint32_t base; // first page number + uint64_t targetOffset; // offset in target + uint64_t spare; // reserved +} SC_Scatter +__attribute__ ((aligned(1))); + + +#endif /* _KERN_CODESIGN_H */ diff --git a/osfmk/kern/debug.c b/osfmk/kern/debug.c index 5e74a85d1..a09bb897f 100644 --- a/osfmk/kern/debug.c +++ b/osfmk/kern/debug.c @@ -56,7 +56,10 @@ #include #include - +#include +#include +#include +#include #include #include #include @@ -72,9 +75,14 @@ #include #include #include +#include +#include +#include + #if !(MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING) #include #endif +#include #if defined(__i386__) || defined(__x86_64__) #include @@ -93,77 +101,130 @@ #include -#if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS) +#if CONFIG_EMBEDDED #include /* For gPanicBase */ +#include +#include +extern volatile struct xnu_hw_shmem_dbg_command_info *hwsd_info; #endif unsigned int halt_in_debugger = 0; -unsigned int switch_debugger = 0; unsigned int current_debugger = 0; unsigned int active_debugger = 0; -unsigned int debug_mode=0; -unsigned int disable_debug_output = TRUE; -unsigned int systemLogDiags = FALSE; unsigned int panicDebugging = FALSE; -unsigned int logPanicDataToScreen = FALSE; unsigned int kdebug_serial = FALSE; -boolean_t lock_panic_mode = FALSE; +unsigned int kernel_debugger_entry_count = 0; + + +#if defined(__arm__) +#define TRAP_DEBUGGER __asm__ volatile("trap") +#elif defined(__arm64__) +/* + * Magic number; this should be identical to the __arm__ encoding for trap. + */ +#define TRAP_DEBUGGER __asm__ volatile(".long 0xe7ffdeff") +#elif defined (__x86_64__) +#define TRAP_DEBUGGER __asm__("int3") +#else +#error No TRAP_DEBUGGER for this architecture +#endif + +#if defined(__i386__) || defined(__x86_64__) +#define panic_stop() pmCPUHalt(PM_HALT_PANIC) +#else +#define panic_stop() panic_spin_forever() +#endif + +#define CPUDEBUGGEROP PROCESSOR_DATA(current_processor(), debugger_state).db_current_op +#define CPUDEBUGGERMSG PROCESSOR_DATA(current_processor(), debugger_state).db_message +#define CPUPANICSTR PROCESSOR_DATA(current_processor(), debugger_state).db_panic_str +#define CPUPANICARGS PROCESSOR_DATA(current_processor(), debugger_state).db_panic_args +#define CPUPANICOPTS PROCESSOR_DATA(current_processor(), debugger_state).db_panic_options +#define CPUDEBUGGERSYNC PROCESSOR_DATA(current_processor(), debugger_state).db_proceed_on_sync_failure +#define CPUDEBUGGERCOUNT PROCESSOR_DATA(current_processor(), debugger_state).db_entry_count +#define CPUDEBUGGERRET PROCESSOR_DATA(current_processor(), debugger_state).db_op_return +#define CPUPANICCALLER PROCESSOR_DATA(current_processor(), debugger_state).db_panic_caller + +#if DEVELOPMENT || DEBUG +#define DEBUGGER_DEBUGGING_NESTED_PANIC_IF_REQUESTED(requested) \ +MACRO_BEGIN \ + if (requested) { \ + volatile int *badpointer = (int *)4; \ + *badpointer = 0; \ + } \ +MACRO_END +#endif /* DEVELOPMENT || DEBUG */ + +debugger_op debugger_current_op = DBOP_NONE; +const char *debugger_panic_str = NULL; +va_list *debugger_panic_args = NULL; +uint64_t debugger_panic_options = 0; +const char *debugger_message = NULL; +unsigned long debugger_panic_caller = 0; + +void panic_trap_to_debugger(const char *panic_format_str, va_list *panic_args, unsigned int reason, void *ctx, uint64_t panic_options_mask, unsigned long panic_caller); +static void kdp_machine_reboot_type(unsigned int type); +__attribute__((noreturn)) void panic_spin_forever(void); +extern kern_return_t do_stackshot(void); int mach_assert = 1; -const char *panicstr = (char *) 0; -decl_simple_lock_data(,panic_lock) -int paniccpu; -volatile int panicwait; -volatile unsigned int nestedpanic= 0; -unsigned int panic_is_inited = 0; -unsigned int return_on_panic = 0; -unsigned long panic_caller; +#define NESTEDDEBUGGERENTRYMAX 5 -#define DEBUG_BUF_SIZE (3 * PAGE_SIZE) +#if CONFIG_EMBEDDED +#define DEBUG_BUF_SIZE (4096) +#define KDBG_TRACE_PANIC_FILENAME "/var/log/panic.trace" +#else +/* + * DEBUG_BUF_SIZE can't grow without updates to SMC and iBoot to store larger panic logs on co-processor systems */ +#define DEBUG_BUF_SIZE ((3 * PAGE_SIZE) + offsetof(struct macos_panic_header, mph_data)) +#define KDBG_TRACE_PANIC_FILENAME "/var/tmp/panic.trace" +#endif -/* debug_buf is directly linked with iBoot panic region for ARM64 targets */ -#if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS) -char *debug_buf_addr = NULL; +/* debug_buf is directly linked with iBoot panic region for embedded targets */ +#if CONFIG_EMBEDDED +char *debug_buf_base = NULL; char *debug_buf_ptr = NULL; unsigned int debug_buf_size = 0; #else char debug_buf[DEBUG_BUF_SIZE]; -__used char *debug_buf_addr = debug_buf; -char *debug_buf_ptr = debug_buf; -unsigned int debug_buf_size = sizeof(debug_buf); +struct macos_panic_header *panic_info = (struct macos_panic_header *)debug_buf; +char *debug_buf_base = (debug_buf + offsetof(struct macos_panic_header, mph_data)); +char *debug_buf_ptr = (debug_buf + offsetof(struct macos_panic_header, mph_data)); + +/* + * We don't include the size of the panic header in the length of the data we actually write. + * On co-processor platforms, we lose sizeof(struct macos_panic_header) bytes from the end of + * the end of the log because we only support writing (3*PAGESIZE) bytes. + */ +const unsigned int debug_buf_size = (DEBUG_BUF_SIZE - offsetof(struct macos_panic_header, mph_data)); #endif -char *debug_buf_stackshot_start; -char *debug_buf_stackshot_end; +/* Debugger state */ +atomic_int debugger_cpu = ATOMIC_VAR_INIT(DEBUGGER_NO_CPU); +boolean_t debugger_allcpus_halted = FALSE; +boolean_t debugger_safe_to_return = TRUE; +unsigned int debugger_context = 0; static char model_name[64]; unsigned char *kernel_uuid; -/* uuid_string_t */ char kernel_uuid_string[37]; -char panic_disk_error_description[512]; -size_t panic_disk_error_description_size = sizeof(panic_disk_error_description); -static spl_t panic_prologue(const char *str); -static void panic_epilogue(spl_t s); +/* + * By default we treat Debugger() the same as calls to panic(), unless + * we have debug boot-args present and the DB_KERN_DUMP_ON_NMI *NOT* set. + * If DB_KERN_DUMP_ON_NMI is *NOT* set, return from Debugger() is supported. + * + * Return from Debugger() is currently only implemented on x86 + */ +static boolean_t debugger_is_panic = TRUE; -struct pasc { - unsigned a: 7; - unsigned b: 7; - unsigned c: 7; - unsigned d: 7; - unsigned e: 7; - unsigned f: 7; - unsigned g: 7; - unsigned h: 7; -} __attribute__((packed)); -typedef struct pasc pasc_t; +char kernel_uuid_string[37]; /* uuid_string_t */ +char panic_disk_error_description[512]; +size_t panic_disk_error_description_size = sizeof(panic_disk_error_description); -/* Prevent CPP from breaking the definition below */ -#ifdef CONFIG_NO_PANIC_STRINGS -#undef Assert -#endif +extern unsigned int write_trace_on_panic; int kext_assertions_enable = #if DEBUG || DEVELOPMENT @@ -172,54 +233,6 @@ int kext_assertions_enable = FALSE; #endif -void __attribute__((noinline)) -Assert( - const char *file, - int line, - const char *expression - ) -{ - int saved_return_on_panic; - - if (!mach_assert) { - kprintf("%s:%d non-fatal Assertion: %s", file, line, expression); - return; - } - - saved_return_on_panic = return_on_panic; - - /* - * If we don't have a debugger configured, returning from an - * assert is a bad, bad idea; there is no guarantee that we - * didn't simply assert before we were able to restart the - * platform. - */ - if (current_debugger != NO_CUR_DB) - return_on_panic = 1; - - panic_plain("%s:%d Assertion failed: %s", file, line, expression); - - return_on_panic = saved_return_on_panic; -} - -/* - * Carefully use the panic_lock. There's always a chance that - * somehow we'll call panic before getting to initialize the - * panic_lock -- in this case, we'll assume that the world is - * in uniprocessor mode and just avoid using the panic lock. - */ -#define PANIC_LOCK() \ -MACRO_BEGIN \ - if (panic_is_inited) \ - simple_lock(&panic_lock); \ -MACRO_END - -#define PANIC_UNLOCK() \ -MACRO_BEGIN \ - if (panic_is_inited) \ - simple_unlock(&panic_lock); \ -MACRO_END - void panic_init(void) { @@ -232,265 +245,725 @@ panic_init(void) uuid_unparse_upper(*(uuid_t *)uuid, kernel_uuid_string); } - simple_lock_init(&panic_lock, 0); - panic_is_inited = 1; - panic_caller = 0; - if (!PE_parse_boot_argn("assertions", &mach_assert, sizeof(mach_assert))) { mach_assert = 1; } + +#if !CONFIG_EMBEDDED + uint32_t debug_flags = 0; + + if (PE_i_can_has_debugger(&debug_flags) && !(debug_flags & DB_KERN_DUMP_ON_NMI)) { + debugger_is_panic = FALSE; + } +#endif + } void debug_log_init(void) { - if (debug_buf_size != 0) - return; -#if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS) +#if CONFIG_EMBEDDED if (!gPanicBase) { printf("debug_log_init: Error!! gPanicBase is still not initialized\n"); return; } - /* Shift debug buf start location and size by 8 bytes for magic header and crc value */ - debug_buf_addr = (char*)gPanicBase + 8; - debug_buf_ptr = debug_buf_addr; - debug_buf_size = gPanicSize - 8; + /* Shift debug buf start location and size by the length of the panic header */ + debug_buf_base = (char *)gPanicBase + sizeof(struct embedded_panic_header); + debug_buf_ptr = debug_buf_base; + debug_buf_size = gPanicSize - sizeof(struct embedded_panic_header); #else - debug_buf_addr = debug_buf; - debug_buf_ptr = debug_buf; - debug_buf_size = sizeof(debug_buf); + assert(debug_buf_base != NULL); + assert(debug_buf_ptr != NULL); + assert(debug_buf_size != 0); + bzero(debug_buf, sizeof(debug_buf)); #endif } -#if defined(__i386__) || defined(__x86_64__) -#define panic_stop() pmCPUHalt(PM_HALT_PANIC) -#define panic_safe() pmSafeMode(x86_lcpu(), PM_SAFE_FL_SAFE) -#define panic_normal() pmSafeMode(x86_lcpu(), PM_SAFE_FL_NORMAL) -#else -#define panic_stop() { while (1) ; } -#define panic_safe() -#define panic_normal() +static void +DebuggerLock() +{ + int my_cpu = cpu_number(); + int debugger_exp_cpu = DEBUGGER_NO_CPU; + assert(ml_get_interrupts_enabled() == FALSE); + + if (debugger_cpu == my_cpu) { + return; + } + + while(!atomic_compare_exchange_strong(&debugger_cpu, &debugger_exp_cpu, my_cpu)) { + debugger_exp_cpu = DEBUGGER_NO_CPU; + } + + return; +} + +static void +DebuggerUnlock() +{ + assert(debugger_cpu == cpu_number()); + + /* + * We don't do an atomic exchange here in case + * there's another CPU spinning to acquire the debugger_lock + * and we never get a chance to update it. We already have the + * lock so we can simply store DEBUGGER_NO_CPU and follow with + * a barrier. + */ + debugger_cpu = DEBUGGER_NO_CPU; + OSMemoryBarrier(); + + return; +} + +static kern_return_t +DebuggerHaltOtherCores(boolean_t proceed_on_failure) +{ +#if CONFIG_EMBEDDED + return DebuggerXCallEnter(proceed_on_failure); +#else /* CONFIG_EMBEDDED */ +#pragma unused(proceed_on_failure) + mp_kdp_enter(proceed_on_failure); + return KERN_SUCCESS; #endif +} + +static void +DebuggerResumeOtherCores() +{ +#if CONFIG_EMBEDDED + DebuggerXCallReturn(); +#else /* CONFIG_EMBEDDED */ + mp_kdp_exit(); +#endif +} + +static void +DebuggerSaveState(debugger_op db_op, const char *db_message, const char *db_panic_str, + va_list *db_panic_args, uint64_t db_panic_options, + boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller) +{ + CPUDEBUGGEROP = db_op; + + /* Preserve the original panic message */ + if (CPUDEBUGGERCOUNT == 1 || CPUPANICSTR == NULL) { + CPUDEBUGGERMSG = db_message; + CPUPANICSTR = db_panic_str; + CPUPANICARGS = db_panic_args; + CPUPANICCALLER = db_panic_caller; + } else if (CPUDEBUGGERCOUNT > 1 && db_panic_str != NULL) { + kprintf("Nested panic detected: %s", db_panic_str); + } + + CPUDEBUGGERSYNC = db_proceed_on_sync_failure; + CPUDEBUGGERRET = KERN_SUCCESS; + + /* Reset these on any nested panics */ + CPUPANICOPTS = db_panic_options; + + return; +} /* - * Prevent CPP from breaking the definition below, - * since all clients get a #define to prepend line numbers + * Save the requested debugger state/action into the current processor's processor_data + * and trap to the debugger. */ -#undef panic +kern_return_t +DebuggerTrapWithState(debugger_op db_op, const char *db_message, const char *db_panic_str, + va_list *db_panic_args, uint64_t db_panic_options, + boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller) +{ + kern_return_t ret; + + assert(ml_get_interrupts_enabled() == FALSE); + DebuggerSaveState(db_op, db_message, db_panic_str, + db_panic_args, db_panic_options, db_proceed_on_sync_failure, + db_panic_caller); + + TRAP_DEBUGGER; + + ret = CPUDEBUGGERRET; -void _consume_panic_args(int a __unused, ...) + DebuggerSaveState(DBOP_NONE, NULL, NULL, NULL, 0, FALSE, 0); + + return ret; +} + +void __attribute__((noinline)) +Assert( + const char *file, + int line, + const char *expression + ) { - panic("panic"); + if (!mach_assert) { + kprintf("%s:%d non-fatal Assertion: %s", file, line, expression); + return; + } + + panic_plain("%s:%d Assertion failed: %s", file, line, expression); } -extern unsigned int write_trace_on_panic; -static spl_t -panic_prologue(const char *str) +void +Debugger(const char *message) { - spl_t s; + DebuggerWithContext(0, NULL, message, DEBUGGER_OPTION_NONE); +} - if (write_trace_on_panic && kdebug_enable) { - if (get_preemption_level() == 0 && !ml_at_interrupt_context()) { - ml_set_interrupts_enabled(TRUE); - kdbg_dump_trace_to_file("/var/tmp/panic.trace"); - } - } +void +DebuggerWithContext(unsigned int reason, void *ctx, const char *message, + uint64_t debugger_options_mask) +{ + spl_t previous_interrupts_state; + boolean_t old_doprnt_hide_pointers = doprnt_hide_pointers; - s = splhigh(); + previous_interrupts_state = ml_set_interrupts_enabled(FALSE); disable_preemption(); - /* Locking code should relax some checks at panic time */ - lock_panic_mode = TRUE; -#if defined(__i386__) || defined(__x86_64__) - /* Attempt to display the unparsed panic string */ - const char *tstr = str; + CPUDEBUGGERCOUNT++; - kprintf("Panic initiated, string: "); - while (tstr && *tstr) - kprintf("%c", *tstr++); - kprintf("\n"); -#endif - - panic_safe(); + if (CPUDEBUGGERCOUNT > NESTEDDEBUGGERENTRYMAX) { + static boolean_t in_panic_kprintf = FALSE; - if( logPanicDataToScreen ) - disable_debug_output = FALSE; - - debug_mode = TRUE; + /* Notify any listeners that we've started a panic */ + PEHaltRestart(kPEPanicBegin); -restart: - PANIC_LOCK(); + if (!in_panic_kprintf) { + in_panic_kprintf = TRUE; + kprintf("Detected nested debugger entry count exceeding %d\n", + NESTEDDEBUGGERENTRYMAX); + in_panic_kprintf = FALSE; + } - if (panicstr) { - if (cpu_number() != paniccpu) { - PANIC_UNLOCK(); - /* - * Wait until message has been printed to identify correct - * cpu that made the first panic. - */ - while (panicwait) - continue; - goto restart; - } else { - nestedpanic +=1; - PANIC_UNLOCK(); - // Other cores will not be resumed on double panic - Debugger("double panic"); - // a printf statement here was removed to avoid a panic-loop caused - // by a panic from printf - panic_stop(); - /* NOTREACHED */ + if (!panicDebugging) { + kdp_machine_reboot_type(kPEPanicRestartCPU); } + + panic_spin_forever(); } - panicstr = str; - paniccpu = cpu_number(); - panicwait = 1; - PANIC_UNLOCK(); +#if DEVELOPMENT || DEBUG + DEBUGGER_DEBUGGING_NESTED_PANIC_IF_REQUESTED((debugger_options_mask & DEBUGGER_OPTION_RECURPANIC_ENTRY)); +#endif + + doprnt_hide_pointers = FALSE; + + if (ctx != NULL) { + DebuggerSaveState(DBOP_DEBUGGER, message, + NULL, NULL, debugger_options_mask, TRUE, 0); + handle_debugger_trap(reason, 0, 0, ctx); + DebuggerSaveState(DBOP_NONE, NULL, NULL, + NULL, 0, FALSE, 0); + } else { + DebuggerTrapWithState(DBOP_DEBUGGER, message, + NULL, NULL, debugger_options_mask, TRUE, 0); + } - // halt other cores now in anticipation of the debugger call - return(s); + CPUDEBUGGERCOUNT--; + doprnt_hide_pointers = old_doprnt_hide_pointers; + enable_preemption(); + ml_set_interrupts_enabled(previous_interrupts_state); +} + +static struct kdp_callout { + struct kdp_callout * callout_next; + kdp_callout_fn_t callout_fn; + boolean_t callout_in_progress; + void * callout_arg; +} * kdp_callout_list = NULL; + +/* + * Called from kernel context to register a kdp event callout. + */ +void +kdp_register_callout(kdp_callout_fn_t fn, void * arg) +{ + struct kdp_callout * kcp; + struct kdp_callout * list_head; + + kcp = kalloc(sizeof(*kcp)); + if (kcp == NULL) + panic("kdp_register_callout() kalloc failed"); + + kcp->callout_fn = fn; + kcp->callout_arg = arg; + kcp->callout_in_progress = FALSE; + + /* Lock-less list insertion using compare and exchange. */ + do { + list_head = kdp_callout_list; + kcp->callout_next = list_head; + } while (!OSCompareAndSwapPtr(list_head, kcp, &kdp_callout_list)); } -#if DEVELOPMENT || DEBUG -static void -panic_epilogue(spl_t s) -#else -#if !defined(__i386__) && !defined(__x86_64__) -__attribute__((noreturn)) -#endif static void -panic_epilogue(__unused spl_t s) -#endif +kdp_callouts(kdp_event_t event) { - /* - * Release panicstr so that we can handle normally other panics. - */ - PANIC_LOCK(); - panicstr = (char *)0; - PANIC_UNLOCK(); + struct kdp_callout *kcp = kdp_callout_list; -#if DEVELOPMENT || DEBUG - if (return_on_panic) { - // resume other cores as we are returning - panic_normal(); - enable_preemption(); - splx(s); - return; + while (kcp) { + if (!kcp->callout_in_progress) { + kcp->callout_in_progress = TRUE; + kcp->callout_fn(kcp->callout_arg, event); + kcp->callout_in_progress = FALSE; + } + kcp = kcp->callout_next; } -#endif - kdb_printf("panic: We are hanging here...\n"); - panic_stop(); - /* NOTREACHED */ } -#if !DEVELOPMENT && !DEBUG && !defined(__i386__) && !defined(__x86_64__) -__attribute__((noreturn)) -#endif +/* + * An overview of the xnu panic path: + * + * Several panic wrappers (panic(), panic_with_options(), etc.) all funnel into panic_trap_to_debugger(). + * panic_trap_to_debugger() sets the panic state in the current processor's processor_data_t prior + * to trapping into the debugger. Once we trap to the debugger, we end up in handle_debugger_trap() + * which tries to acquire the panic lock by atomically swapping the current CPU number into debugger_cpu. + * debugger_cpu acts as a synchronization point, from which the winning CPU can halt the other cores and + * continue to debugger_collect_diagnostics() where we write the paniclog, corefile (if appropriate) and proceed + * according to the device's boot-args. + */ +#undef panic void panic(const char *str, ...) { - va_list listp; - spl_t s; - boolean_t old_doprnt_hide_pointers = doprnt_hide_pointers; + va_list panic_str_args; + + va_start(panic_str_args, str); + panic_trap_to_debugger(str, &panic_str_args, 0, NULL, 0, (unsigned long)(char *)__builtin_return_address(0)); + va_end(panic_str_args); +} + +void +panic_with_options(unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...) +{ + va_list panic_str_args; + + va_start(panic_str_args, str); + panic_trap_to_debugger(str, &panic_str_args, reason, ctx, debugger_options_mask, (unsigned long)(char *)__builtin_return_address(0)); + va_end(panic_str_args); +} + +void +panic_context(unsigned int reason, void *ctx, const char *str, ...) +{ + va_list panic_str_args; + + va_start(panic_str_args, str); + panic_trap_to_debugger(str, &panic_str_args, reason, ctx, 0, (unsigned long)(char *)__builtin_return_address(0)); + va_end(panic_str_args); +} + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wmissing-noreturn" +void +panic_trap_to_debugger(const char *panic_format_str, va_list *panic_args, unsigned int reason, void + *ctx, uint64_t panic_options_mask, unsigned long panic_caller) +{ +#pragma clang diagnostic pop + + if (ml_wants_panic_trap_to_debugger()) { + ml_panic_trap_to_debugger(panic_format_str, panic_args, reason, ctx, panic_options_mask, panic_caller); + + /* + * This should not return, but we return here for the tail call + * as it simplifies the backtrace. + */ + return; + } + + CPUDEBUGGERCOUNT++; + + if (CPUDEBUGGERCOUNT > NESTEDDEBUGGERENTRYMAX) { + static boolean_t in_panic_kprintf = FALSE; + + /* Notify any listeners that we've started a panic */ + PEHaltRestart(kPEPanicBegin); + + if (!in_panic_kprintf) { + in_panic_kprintf = TRUE; + kprintf("Detected nested debugger entry count exceeding %d\n", + NESTEDDEBUGGERENTRYMAX); + in_panic_kprintf = FALSE; + } + + if (!panicDebugging) { + kdp_machine_reboot_type(kPEPanicRestartCPU); + } + + panic_spin_forever(); + } + +#if DEVELOPMENT || DEBUG + DEBUGGER_DEBUGGING_NESTED_PANIC_IF_REQUESTED((panic_options_mask & DEBUGGER_OPTION_RECURPANIC_ENTRY)); +#endif + +#if CONFIG_EMBEDDED + if (PE_arm_debug_panic_hook) + PE_arm_debug_panic_hook(panic_format_str); +#endif #if defined (__x86_64__) plctrace_disable(); #endif - /* panic_caller is initialized to 0. If set, don't change it */ - if ( ! panic_caller ) - panic_caller = (unsigned long)(char *)__builtin_return_address(0); + if (write_trace_on_panic && kdebug_enable) { + if (get_preemption_level() == 0 && !ml_at_interrupt_context()) { + ml_set_interrupts_enabled(TRUE); + KDBG(TRACE_PANIC); + kdbg_dump_trace_to_file(KDBG_TRACE_PANIC_FILENAME); + } + } + + ml_set_interrupts_enabled(FALSE); + disable_preemption(); - s = panic_prologue(str); +#if defined (__x86_64__) + pmSafeMode(x86_lcpu(), PM_SAFE_FL_SAFE); +#endif /* defined (__x86_64__) */ /* Never hide pointers from panic logs. */ doprnt_hide_pointers = FALSE; - kdb_printf("panic(cpu %d caller 0x%lx): ", (unsigned) paniccpu, panic_caller); - if (str) { - va_start(listp, str); - _doprnt(str, &listp, consdebug_putc, 0); - va_end(listp); + if (ctx != NULL) { + /* + * We called into panic from a trap, no need to trap again. Set the + * state on the current CPU and then jump to handle_debugger_trap. + */ + DebuggerSaveState(DBOP_PANIC, "panic", + panic_format_str, panic_args, + panic_options_mask, TRUE, panic_caller); + handle_debugger_trap(reason, 0, 0, ctx); } - kdb_printf("\n"); +#if defined(__arm64__) /* - * Release panicwait indicator so that other cpus may call Debugger(). + * Signal to fastsim that it should open debug ports (nop on hardware) */ - panicwait = 0; - Debugger("panic"); + __asm__ volatile("HINT 0x45"); +#endif /* defined(__arm64__) */ - doprnt_hide_pointers = old_doprnt_hide_pointers; + DebuggerTrapWithState(DBOP_PANIC, "panic", panic_format_str, + panic_args, panic_options_mask, TRUE, panic_caller); - panic_epilogue(s); + /* + * Not reached. + */ + panic_stop(); } -/* - * panic_with_options: wraps the panic call in a way that allows us to pass - * a bitmask of specific debugger options. - */ -#if !DEVELOPMENT && !DEBUG && !defined(__i386__) && !defined(__x86_64__) __attribute__((noreturn)) -#endif void -panic_with_options(unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...) +panic_spin_forever() { - va_list listp; - spl_t s; + paniclog_append_noflush("\nPlease go to https://panic.apple.com to report this panic\n"); + + for (;;) { } +} + +static void +kdp_machine_reboot_type(unsigned int type) +{ + printf("Attempting system restart..."); + PEHaltRestart(type); + halt_all_cpus(TRUE); +} + +void +kdp_machine_reboot(void) +{ + kdp_machine_reboot_type(kPEPanicRestartCPU); +} + +/* + * Gather and save diagnostic information about a panic (or Debugger call). + * + * On embedded, Debugger and Panic are treated very similarly -- WDT uses Debugger so we can + * theoretically return from it. On desktop, Debugger is treated as a conventional debugger -- i.e no + * paniclog is written and no core is written unless we request a core on NMI. + * + * This routine handles kicking off local coredumps, paniclogs, calling into the Debugger/KDP (if it's configured), + * and calling out to any other functions we have for collecting diagnostic info. + */ +static void +debugger_collect_diagnostics(unsigned int exception, unsigned int code, unsigned int subcode, void *state) +{ +#if DEVELOPMENT || DEBUG + DEBUGGER_DEBUGGING_NESTED_PANIC_IF_REQUESTED((debugger_panic_options & DEBUGGER_OPTION_RECURPANIC_PRELOG)); +#endif + +#if defined(__x86_64__) + kprintf("Debugger called: <%s>\n", debugger_message); +#endif + /* + * DB_HALT (halt_in_debugger) can be requested on startup, we shouldn't generate + * a coredump/paniclog for this type of debugger entry. If KDP isn't configured, + * we'll just spin in kdp_raise_exception. + */ + if (debugger_current_op == DBOP_DEBUGGER && halt_in_debugger) { + kdp_raise_exception(exception, code, subcode, state); + if (debugger_safe_to_return && !debugger_is_panic) { + return; + } + } + + if ((debugger_current_op == DBOP_PANIC) || + (debugger_current_op == DBOP_DEBUGGER && debugger_is_panic)) { + /* + * Attempt to notify listeners once and only once that we've started + * panicking. Only do this for Debugger() calls if we're treating + * Debugger() calls like panic(). + */ + PEHaltRestart(kPEPanicBegin); + + /* + * Set the begin pointer in the panic log structure. We key off of this + * static variable rather than contents from the panic header itself in case someone + * has stomped over the panic_info structure. Also initializes the header magic. + */ + static boolean_t began_writing_paniclog = FALSE; + if (!began_writing_paniclog) { + PE_init_panicheader(); + began_writing_paniclog = TRUE; + } else { + /* + * If we reached here, update the panic header to keep it as consistent + * as possible during a nested panic + */ + PE_update_panicheader_nestedpanic(); + } + } + + /* + * Write panic string if this was a panic. + * + * TODO: Consider moving to SavePanicInfo as this is part of the panic log. + */ + if (debugger_current_op == DBOP_PANIC) { + paniclog_append_noflush("panic(cpu %d caller 0x%lx): ", (unsigned) cpu_number(), debugger_panic_caller); + if (debugger_panic_str) { + _doprnt(debugger_panic_str, debugger_panic_args, consdebug_putc, 0); + } + paniclog_append_noflush("\n"); + } + + /* + * Debugger() is treated like panic() on embedded -- for example we use it for WDT + * panics (so we need to write a paniclog). On desktop Debugger() is used in the + * conventional sense. + */ +#if defined(__x86_64__) + if (debugger_current_op == DBOP_PANIC) +#endif + { + kdp_callouts(KDP_EVENT_PANICLOG); + /* + * Write paniclog and panic stackshot (if supported) + * TODO: Need to clear panic log when return from debugger + * hooked up for embedded + */ + SavePanicInfo(debugger_message, debugger_panic_options); - /* panic_caller is initialized to 0. If set, don't change it */ - if ( ! panic_caller ) - panic_caller = (unsigned long)(char *)__builtin_return_address(0); +#if DEVELOPMENT || DEBUG + DEBUGGER_DEBUGGING_NESTED_PANIC_IF_REQUESTED((debugger_panic_options & DEBUGGER_OPTION_RECURPANIC_POSTLOG)); +#endif - s = panic_prologue(str); - kdb_printf("panic(cpu %d caller 0x%lx): ", (unsigned) paniccpu, panic_caller); - if (str) { - va_start(listp, str); - _doprnt(str, &listp, consdebug_putc, 0); - va_end(listp); + /* DEBUGGER_OPTION_PANICLOGANDREBOOT is used for two finger resets on embedded so we get a paniclog */ + if (debugger_panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) + PEHaltRestart(kPEPanicRestartCPU); } - kdb_printf("\n"); + +#if CONFIG_KDP_INTERACTIVE_DEBUGGING + uint32_t debug_flags = 0; + PE_i_can_has_debugger(&debug_flags); /* - * Release panicwait indicator so that other cpus may call Debugger(). + * If reboot on panic is enabled and the caller of panic indicated that we should skip + * local coredumps, don't try to write these and instead go straight to reboot. This + * allows us to persist any data that's stored in the panic log. */ - panicwait = 0; - DebuggerWithContext(reason, ctx, "panic", debugger_options_mask); - panic_epilogue(s); + if ((debugger_panic_options & DEBUGGER_OPTION_SKIP_LOCAL_COREDUMP) && + (debug_flags & DB_REBOOT_POST_CORE)) { + kdp_machine_reboot_type(kPEPanicRestartCPU); + } + + /* + * Consider generating a local corefile if the infrastructure is configured + * and we haven't disabled on-device coredumps. + */ + if (kdp_has_polled_corefile() && !(debug_flags & DB_DISABLE_LOCAL_CORE)) { + int ret = -1; + +#if defined (__x86_64__) + /* On x86 we don't do a coredump on Debugger unless the DB_KERN_DUMP_ON_NMI boot-arg is specified. */ + if (debugger_current_op != DBOP_DEBUGGER || (debug_flags & DB_KERN_DUMP_ON_NMI)) +#endif + { + /* + * Doing an on-device coredump leaves the disk driver in a state + * that can not be resumed. + */ + debugger_safe_to_return = FALSE; + begin_panic_transfer(); + ret = kern_dump(KERN_DUMP_DISK); + abort_panic_transfer(); + +#if DEVELOPMENT || DEBUG + DEBUGGER_DEBUGGING_NESTED_PANIC_IF_REQUESTED((debugger_panic_options & DEBUGGER_OPTION_RECURPANIC_POSTCORE)); +#endif + } + + /* If we wrote a corefile and DB_REBOOT_POST_CORE is set, reboot */ + if (ret == 0 && (debug_flags & DB_REBOOT_POST_CORE)) + kdp_machine_reboot_type(kPEPanicRestartCPU); + } + + /* If KDP is configured, try to trap to the debugger */ + if (current_debugger != NO_CUR_DB) { + kdp_raise_exception(exception, code, subcode, state); + /* + * Only return if we entered via Debugger and it's safe to return + * (we halted the other cores successfully, this isn't a nested panic, etc) + */ + if (debugger_current_op == DBOP_DEBUGGER && + debugger_safe_to_return && + kernel_debugger_entry_count == 1 && + !debugger_is_panic) { + return; + } + } + +#if CONFIG_EMBEDDED + if (panicDebugging) { + /* If panic debugging is configured, spin for astris to connect */ + panic_spin_shmcon(); + } +#endif /* CONFIG_EMBEDDED */ +#endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */ + + if (!panicDebugging) { + kdp_machine_reboot_type(kPEPanicRestartCPU); + } + + panic_spin_forever(); } -#if !DEVELOPMENT && !DEBUG && !defined(__i386__) && !defined(__x86_64__) -__attribute__((noreturn)) +#if INTERRUPT_MASKED_DEBUG +uint64_t debugger_trap_timestamps[9]; +# define DEBUGGER_TRAP_TIMESTAMP(i) debugger_trap_timestamps[i] = mach_absolute_time(); +#else +# define DEBUGGER_TRAP_TIMESTAMP(i) #endif + void -panic_context(unsigned int reason, void *ctx, const char *str, ...) +handle_debugger_trap(unsigned int exception, unsigned int code, unsigned int subcode, void *state) { - va_list listp; - spl_t s; + unsigned int initial_not_in_kdp = not_in_kdp; + kern_return_t ret; + debugger_op db_prev_op = debugger_current_op; + + DEBUGGER_TRAP_TIMESTAMP(0); + + DebuggerLock(); + ret = DebuggerHaltOtherCores(CPUDEBUGGERSYNC); + DEBUGGER_TRAP_TIMESTAMP(1); - /* panic_caller is initialized to 0. If set, don't change it */ - if ( ! panic_caller ) - panic_caller = (unsigned long)(char *)__builtin_return_address(0); - - s = panic_prologue(str); - kdb_printf("panic(cpu %d caller 0x%lx): ", (unsigned) paniccpu, panic_caller); - if (str) { - va_start(listp, str); - _doprnt(str, &listp, consdebug_putc, 0); - va_end(listp); +#if INTERRUPT_MASKED_DEBUG + if (serialmode & SERIALMODE_OUTPUT) { + ml_spin_debug_reset(current_thread()); + } +#endif + if (ret != KERN_SUCCESS) { + CPUDEBUGGERRET = ret; + DebuggerUnlock(); + return; + } + + /* Update the global panic/debugger nested entry level */ + kernel_debugger_entry_count = CPUDEBUGGERCOUNT; + + /* + * TODO: Should we do anything special for nested panics here? i.e. if we've trapped more than twice + * should we call into the debugger if it's configured and then reboot if the panic log has been written? + */ + + if (CPUDEBUGGEROP == DBOP_NONE) { + /* If there was no debugger context setup, we trapped due to a software breakpoint */ + debugger_current_op = DBOP_BREAKPOINT; + } else { + /* Not safe to return from a nested panic/debugger call */ + if (debugger_current_op == DBOP_PANIC || + debugger_current_op == DBOP_DEBUGGER) { + debugger_safe_to_return = FALSE; + } + + debugger_current_op = CPUDEBUGGEROP; + + /* Only overwrite the panic message if there is none already - save the data from the first call */ + if (debugger_panic_str == NULL) { + debugger_panic_str = CPUPANICSTR; + debugger_panic_args = CPUPANICARGS; + debugger_message = CPUDEBUGGERMSG; + debugger_panic_caller = CPUPANICCALLER; + } + + debugger_panic_options = CPUPANICOPTS; } - kdb_printf("\n"); /* - * Release panicwait indicator so that other cpus may call Debugger(). + * Clear the op from the processor debugger context so we can handle + * breakpoints in the debugger */ - panicwait = 0; - DebuggerWithContext(reason, ctx, "panic", DEBUGGER_OPTION_NONE); - panic_epilogue(s); + CPUDEBUGGEROP = DBOP_NONE; + + DEBUGGER_TRAP_TIMESTAMP(2); + + kdp_callouts(KDP_EVENT_ENTER); + not_in_kdp = 0; + + DEBUGGER_TRAP_TIMESTAMP(3); + + if (debugger_current_op == DBOP_BREAKPOINT) { + kdp_raise_exception(exception, code, subcode, state); + } else if (debugger_current_op == DBOP_STACKSHOT) { + CPUDEBUGGERRET = do_stackshot(); +#if PGO + } else if (debugger_current_op == DBOP_RESET_PGO_COUNTERS) { + CPUDEBUGGERRET = do_pgo_reset_counters(); +#endif + } else { + debugger_collect_diagnostics(exception, code, subcode, state); + } + + DEBUGGER_TRAP_TIMESTAMP(4); + + not_in_kdp = initial_not_in_kdp; + kdp_callouts(KDP_EVENT_EXIT); + + DEBUGGER_TRAP_TIMESTAMP(5); + + if (debugger_current_op != DBOP_BREAKPOINT) { + debugger_panic_str = NULL; + debugger_panic_args = NULL; + debugger_panic_options = 0; + debugger_message = NULL; + } + + /* Restore the previous debugger state */ + debugger_current_op = db_prev_op; + + DEBUGGER_TRAP_TIMESTAMP(6); + + DebuggerResumeOtherCores(); + + DEBUGGER_TRAP_TIMESTAMP(7); + + DebuggerUnlock(); + + DEBUGGER_TRAP_TIMESTAMP(8); + + return; } __attribute__((noinline,not_tail_called)) @@ -520,8 +993,9 @@ void log(__unused int level, char *fmt, ...) } /* - * Skip appending log messages to the new logging infrastructure in contexts - * where safety is uncertain. These contexts include: + * Per , skip appending log messages to + * the new logging infrastructure in contexts where safety is + * uncertain. These contexts include: * - When we're in the debugger * - We're in a panic * - Interrupts are disabled @@ -529,28 +1003,50 @@ void log(__unused int level, char *fmt, ...) * In all the above cases, it is potentially unsafe to log messages. */ -boolean_t oslog_is_safe(void) { - return (debug_mode == 0 && +boolean_t +oslog_is_safe(void) { + return (kernel_debugger_entry_count == 0 && not_in_kdp == 1 && get_preemption_level() == 0 && ml_get_interrupts_enabled() == TRUE); } +boolean_t +debug_mode_active(void) +{ + return ((0 != kernel_debugger_entry_count != 0) || (0 == not_in_kdp)); +} + void debug_putc(char c) { if ((debug_buf_size != 0) && - ((debug_buf_ptr-debug_buf_addr) < (int)debug_buf_size)) { + ((debug_buf_ptr - debug_buf_base) < (int)debug_buf_size)) { *debug_buf_ptr=c; debug_buf_ptr++; } } -/* In-place packing routines -- inefficient, but they're called at most once. - * Assumes "buflen" is a multiple of 8. - */ +#if defined (__x86_64__) +struct pasc { + unsigned a: 7; + unsigned b: 7; + unsigned c: 7; + unsigned d: 7; + unsigned e: 7; + unsigned f: 7; + unsigned g: 7; + unsigned h: 7; +} __attribute__((packed)); -int packA(char *inbuf, uint32_t length, uint32_t buflen) +typedef struct pasc pasc_t; + +/* + * In-place packing routines -- inefficient, but they're called at most once. + * Assumes "buflen" is a multiple of 8. Used for compressing paniclogs on x86. + */ +int +packA(char *inbuf, uint32_t length, uint32_t buflen) { unsigned int i, j = 0; pasc_t pack; @@ -573,7 +1069,8 @@ int packA(char *inbuf, uint32_t length, uint32_t buflen) return j; } -void unpackA(char *inbuf, uint32_t length) +void +unpackA(char *inbuf, uint32_t length) { pasc_t packs; unsigned i = 0; @@ -592,6 +1089,7 @@ void unpackA(char *inbuf, uint32_t length) inbuf[i++] = packs.h; } } +#endif /* defined (__x86_64__) */ extern void *proc_name_address(void *p); @@ -606,18 +1104,21 @@ panic_display_process_name(void) { if(ml_nofault_copy((vm_offset_t)&ctask->bsd_info, (vm_offset_t)&cbsd_info, sizeof(cbsd_info)) == sizeof(cbsd_info)) if (cbsd_info && (ml_nofault_copy((vm_offset_t) proc_name_address(cbsd_info), (vm_offset_t) &proc_name, sizeof(proc_name)) > 0)) proc_name[sizeof(proc_name) - 1] = '\0'; - kdb_printf("\nBSD process name corresponding to current thread: %s\n", proc_name); + paniclog_append_noflush("\nBSD process name corresponding to current thread: %s\n", proc_name); } -unsigned panic_active(void) { - return ((panicstr != (char *) 0)); +unsigned +panic_active(void) { + return ((debugger_panic_str != (char *) 0)); } -void populate_model_name(char *model_string) { +void +populate_model_name(char *model_string) { strlcpy(model_name, model_string, sizeof(model_name)); } -void panic_display_model_name(void) { +void +panic_display_model_name(void) { char tmp_model_name[sizeof(model_name)]; if (ml_nofault_copy((vm_offset_t) &model_name, (vm_offset_t) &tmp_model_name, sizeof(model_name)) != sizeof(model_name)) @@ -626,44 +1127,49 @@ void panic_display_model_name(void) { tmp_model_name[sizeof(tmp_model_name) - 1] = '\0'; if (tmp_model_name[0] != 0) - kdb_printf("System model name: %s\n", tmp_model_name); + paniclog_append_noflush("System model name: %s\n", tmp_model_name); } -void panic_display_kernel_uuid(void) { +void +panic_display_kernel_uuid(void) { char tmp_kernel_uuid[sizeof(kernel_uuid_string)]; if (ml_nofault_copy((vm_offset_t) &kernel_uuid_string, (vm_offset_t) &tmp_kernel_uuid, sizeof(kernel_uuid_string)) != sizeof(kernel_uuid_string)) return; if (tmp_kernel_uuid[0] != '\0') - kdb_printf("Kernel UUID: %s\n", tmp_kernel_uuid); + paniclog_append_noflush("Kernel UUID: %s\n", tmp_kernel_uuid); } -void panic_display_kernel_aslr(void) { +void +panic_display_kernel_aslr(void) { if (vm_kernel_slide) { - kdb_printf("Kernel slide: 0x%016lx\n", (unsigned long) vm_kernel_slide); - kdb_printf("Kernel text base: %p\n", (void *) vm_kernel_stext); + paniclog_append_noflush("Kernel slide: 0x%016lx\n", (unsigned long) vm_kernel_slide); + paniclog_append_noflush("Kernel text base: %p\n", (void *) vm_kernel_stext); } } -void panic_display_hibb(void) { +void +panic_display_hibb(void) { #if defined(__i386__) || defined (__x86_64__) - kdb_printf("__HIB text base: %p\n", (void *) vm_hib_base); + paniclog_append_noflush("__HIB text base: %p\n", (void *) vm_hib_base); #endif } -static void panic_display_uptime(void) { +static void +panic_display_uptime(void) { uint64_t uptime; absolutetime_to_nanoseconds(mach_absolute_time(), &uptime); - kdb_printf("\nSystem uptime in nanoseconds: %llu\n", uptime); + paniclog_append_noflush("\nSystem uptime in nanoseconds: %llu\n", uptime); } -static void panic_display_disk_errors(void) { +static void +panic_display_disk_errors(void) { if (panic_disk_error_description[0]) { panic_disk_error_description[sizeof(panic_disk_error_description) - 1] = '\0'; - kdb_printf("Root disk errors: \"%s\"\n", panic_disk_error_description); + paniclog_append_noflush("Root disk errors: \"%s\"\n", panic_disk_error_description); } }; @@ -672,16 +1178,17 @@ extern char osversion[]; static volatile uint32_t config_displayed = 0; -__private_extern__ void panic_display_system_configuration(boolean_t launchd_exit) { +__private_extern__ void +panic_display_system_configuration(boolean_t launchd_exit) { if (!launchd_exit) panic_display_process_name(); if (OSCompareAndSwap(0, 1, &config_displayed)) { char buf[256]; if (!launchd_exit && strlcpy(buf, PE_boot_args(), sizeof(buf))) - kdb_printf("Boot args: %s\n", buf); - kdb_printf("\nMac OS version:\n%s\n", + paniclog_append_noflush("Boot args: %s\n", buf); + paniclog_append_noflush("\nMac OS version:\n%s\n", (osversion[0] != 0) ? osversion : "Not yet set"); - kdb_printf("\nKernel version:\n%s\n",version); + paniclog_append_noflush("\nKernel version:\n%s\n",version); panic_display_kernel_uuid(); if (!launchd_exit) { panic_display_kernel_aslr(); @@ -696,7 +1203,7 @@ __private_extern__ void panic_display_system_configuration(boolean_t launchd_exi #if CONFIG_ZLEAKS panic_display_ztrace(); #endif /* CONFIG_ZLEAKS */ - kext_dump_panic_lists(&kdb_log); + kext_dump_panic_lists(&paniclog_append_noflush); } } } @@ -704,44 +1211,45 @@ __private_extern__ void panic_display_system_configuration(boolean_t launchd_exi extern unsigned int stack_total; extern unsigned long long stack_allocs; -#if defined(__i386__) || defined (__x86_64__) +#if defined (__x86_64__) extern unsigned int inuse_ptepages_count; extern long long alloc_ptepages_count; #endif -extern boolean_t panic_include_zprint; -extern vm_offset_t panic_kext_memory_info; -extern vm_size_t panic_kext_memory_size; +extern boolean_t panic_include_zprint; +extern mach_memory_info_t *panic_kext_memory_info; +extern vm_size_t panic_kext_memory_size; -__private_extern__ void panic_display_zprint() +__private_extern__ void +panic_display_zprint() { if(panic_include_zprint == TRUE) { unsigned int i; struct zone zone_copy; - kdb_printf("%-20s %10s %10s\n", "Zone Name", "Cur Size", "Free Size"); + paniclog_append_noflush("%-20s %10s %10s\n", "Zone Name", "Cur Size", "Free Size"); for (i = 0; i < num_zones; i++) { if(ml_nofault_copy((vm_offset_t)(&zone_array[i]), (vm_offset_t)&zone_copy, sizeof(struct zone)) == sizeof(struct zone)) { if(zone_copy.cur_size > (1024*1024)) { - kdb_printf("%-20s %10lu %10lu\n",zone_copy.zone_name, (uintptr_t)zone_copy.cur_size,(uintptr_t)(zone_copy.countfree * zone_copy.elem_size)); + paniclog_append_noflush("%-20s %10lu %10lu\n",zone_copy.zone_name, (uintptr_t)zone_copy.cur_size,(uintptr_t)(zone_copy.countfree * zone_copy.elem_size)); } } } - kdb_printf("%-20s %10lu\n", "Kernel Stacks", (uintptr_t)(kernel_stack_size * stack_total)); + paniclog_append_noflush("%-20s %10lu\n", "Kernel Stacks", (uintptr_t)(kernel_stack_size * stack_total)); -#if defined(__i386__) || defined (__x86_64__) - kdb_printf("%-20s %10lu\n", "PageTables",(uintptr_t)(PAGE_SIZE * inuse_ptepages_count)); +#if defined (__x86_64__) + paniclog_append_noflush("%-20s %10lu\n", "PageTables",(uintptr_t)(PAGE_SIZE * inuse_ptepages_count)); #endif - kdb_printf("%-20s %10lu\n", "Kalloc.Large", (uintptr_t)kalloc_large_total); + paniclog_append_noflush("%-20s %10lu\n", "Kalloc.Large", (uintptr_t)kalloc_large_total); if (panic_kext_memory_info) { - mach_memory_info_t *mem_info = (mach_memory_info_t *)panic_kext_memory_info; - kdb_printf("\n%-5s %10s\n", "Kmod", "Size"); - for (i = 0; i < VM_KERN_MEMORY_COUNT + VM_KERN_COUNTER_COUNT; i++) { + mach_memory_info_t *mem_info = panic_kext_memory_info; + paniclog_append_noflush("\n%-5s %10s\n", "Kmod", "Size"); + for (i = 0; i < (panic_kext_memory_size / sizeof(mach_zone_info_t)); i++) { if (((mem_info[i].flags & VM_KERN_SITE_TYPE) == VM_KERN_SITE_KMOD) && (mem_info[i].size > (1024 * 1024))) { - kdb_printf("%-5lld %10lld\n", mem_info[i].site, mem_info[i].size); + paniclog_append_noflush("%-5lld %10lld\n", mem_info[i].site, mem_info[i].size); } } } @@ -749,12 +1257,13 @@ __private_extern__ void panic_display_zprint() } #if CONFIG_ECC_LOGGING -__private_extern__ void panic_display_ecc_errors() +__private_extern__ void +panic_display_ecc_errors() { uint32_t count = ecc_log_get_correction_count(); if (count > 0) { - kdb_printf("ECC Corrections:%u\n", count); + paniclog_append_noflush("ECC Corrections:%u\n", count); } } #endif /* CONFIG_ECC_LOGGING */ @@ -768,7 +1277,8 @@ void panic_print_symbol_name(vm_address_t search); * Prints the backtrace most suspected of being a leaker, if we paniced in the zone allocator. * top_ztrace and panic_include_ztrace comes from osfmk/kern/zalloc.c */ -__private_extern__ void panic_display_ztrace(void) +__private_extern__ void +panic_display_ztrace(void) { if(panic_include_ztrace == TRUE) { unsigned int i = 0; @@ -779,89 +1289,50 @@ __private_extern__ void panic_display_ztrace(void) /* Make sure not to trip another panic if there's something wrong with memory */ if(ml_nofault_copy((vm_offset_t)top_ztrace, (vm_offset_t)&top_ztrace_copy, sizeof(struct ztrace)) == sizeof(struct ztrace)) { - kdb_printf("\nBacktrace suspected of leaking: (outstanding bytes: %lu)\n", (uintptr_t)top_ztrace_copy.zt_size); + paniclog_append_noflush("\nBacktrace suspected of leaking: (outstanding bytes: %lu)\n", (uintptr_t)top_ztrace_copy.zt_size); /* Print the backtrace addresses */ for (i = 0; (i < top_ztrace_copy.zt_depth && i < MAX_ZTRACE_DEPTH) ; i++) { - kdb_printf("%p ", top_ztrace_copy.zt_stack[i]); + paniclog_append_noflush("%p ", top_ztrace_copy.zt_stack[i]); if (keepsyms) { panic_print_symbol_name((vm_address_t)top_ztrace_copy.zt_stack[i]); } - kdb_printf("\n"); + paniclog_append_noflush("\n"); } /* Print any kexts in that backtrace, along with their link addresses so we can properly blame them */ kmod_panic_dump((vm_offset_t *)&top_ztrace_copy.zt_stack[0], top_ztrace_copy.zt_depth); } else { - kdb_printf("\nCan't access top_ztrace...\n"); + paniclog_append_noflush("\nCan't access top_ztrace...\n"); } - kdb_printf("\n"); + paniclog_append_noflush("\n"); } } #endif /* CONFIG_ZLEAKS */ -#if ! (MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING) -static struct kdp_ether_addr kdp_current_mac_address = {{0, 0, 0, 0, 0, 0}}; - -/* XXX ugly forward declares to stop warnings */ -void *kdp_get_interface(void); -void kdp_set_ip_and_mac_addresses(struct kdp_in_addr *, struct kdp_ether_addr *); -void kdp_set_gateway_mac(void *); -void kdp_set_interface(void *); -void kdp_register_send_receive(void *, void *); -void kdp_unregister_send_receive(void *, void *); - -int kdp_stack_snapshot_geterror(void); -uint32_t kdp_stack_snapshot_bytes_traced(void); - -void * -kdp_get_interface( void) -{ - return(void *)0; -} - -unsigned int -kdp_get_ip_address(void ) -{ return 0; } - -struct kdp_ether_addr -kdp_get_mac_addr(void) -{ - return kdp_current_mac_address; -} - -void -kdp_set_ip_and_mac_addresses( - __unused struct kdp_in_addr *ipaddr, - __unused struct kdp_ether_addr *macaddr) -{} - -void -kdp_set_gateway_mac(__unused void *gatewaymac) -{} - -void -kdp_set_interface(__unused void *ifp) -{} - -void -kdp_register_send_receive(__unused void *send, __unused void *receive) -{} - -void -kdp_unregister_send_receive(__unused void *send, __unused void *receive) -{} - -void kdp_register_link(__unused kdp_link_t link, __unused kdp_mode_t mode) -{} - -void kdp_unregister_link(__unused kdp_link_t link, __unused kdp_mode_t mode) -{} - -#endif - #if !CONFIG_TELEMETRY -int telemetry_gather(user_addr_t buffer __unused, uint32_t *length __unused, boolean_t mark __unused) +int +telemetry_gather(user_addr_t buffer __unused, uint32_t *length __unused, boolean_t mark __unused) { return KERN_NOT_SUPPORTED; } #endif + +#include + +uint32_t kern_feature_overrides = 0; + +boolean_t kern_feature_override(uint32_t fmask) { + if (kern_feature_overrides == 0) { + uint32_t fdisables = 0; + /* Expected to be first invoked early, in a single-threaded + * environment + */ + if (PE_parse_boot_argn("validation_disables", &fdisables, sizeof(fdisables))) { + fdisables |= KF_INITIALIZED; + kern_feature_overrides = fdisables; + } else { + kern_feature_overrides |= KF_INITIALIZED; + } + } + return ((kern_feature_overrides & fmask) == fmask); +} diff --git a/osfmk/kern/debug.h b/osfmk/kern/debug.h index b05e7a90a..175a0ea61 100644 --- a/osfmk/kern/debug.h +++ b/osfmk/kern/debug.h @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -189,7 +190,6 @@ enum generic_snapshot_flags { kKernel64_p = 0x2 }; - #define VM_PRESSURE_TIME_WINDOW 5 /* seconds */ enum { @@ -238,120 +238,90 @@ enum { STACKSHOT_NO_IO_STATS = 0x800000, /* Report owners of and pointers to kernel objects that threads are blocked on */ STACKSHOT_THREAD_WAITINFO = 0x1000000, + STACKSHOT_THREAD_GROUP = 0x2000000, + STACKSHOT_SAVE_JETSAM_COALITIONS = 0x4000000, + STACKSHOT_INSTRS_CYCLES = 0x8000000, }; -#define STACKSHOT_THREAD_SNAPSHOT_MAGIC 0xfeedface -#define STACKSHOT_TASK_SNAPSHOT_MAGIC 0xdecafbad -#define STACKSHOT_MEM_AND_IO_SNAPSHOT_MAGIC 0xbfcabcde -#define STACKSHOT_MICRO_SNAPSHOT_MAGIC 0x31c54011 - -#endif /* __APPLE_API_UNSTABLE */ -#endif /* __APPLE_API_PRIVATE */ - -#ifdef KERNEL_PRIVATE - -extern unsigned int systemLogDiags; -#if (!defined(__arm64__) && !defined(NAND_PANIC_DEVICE)) || defined(LEGACY_PANIC_LOGS) -extern char debug_buf[]; -#endif -extern char *debug_buf_addr; -extern char *debug_buf_stackshot_start; -extern char *debug_buf_stackshot_end; - -extern unsigned int debug_boot_arg; -extern unsigned char *kernel_uuid; -extern char kernel_uuid_string[]; -extern char panic_disk_error_description[]; -extern size_t panic_disk_error_description_size; +#define STACKSHOT_THREAD_SNAPSHOT_MAGIC 0xfeedface +#define STACKSHOT_TASK_SNAPSHOT_MAGIC 0xdecafbad +#define STACKSHOT_MEM_AND_IO_SNAPSHOT_MAGIC 0xbfcabcde +#define STACKSHOT_MICRO_SNAPSHOT_MAGIC 0x31c54011 -#ifdef MACH_KERNEL_PRIVATE +#define KF_INITIALIZED (0x1) +#define KF_SERIAL_OVRD (0x2) +#define KF_PMAPV_OVRD (0x4) +#define KF_MATV_OVRD (0x8) +#define KF_STACKSHOT_OVRD (0x10) +#define KF_COMPRSV_OVRD (0x20) -extern boolean_t doprnt_hide_pointers; +boolean_t kern_feature_override(uint32_t fmask); -extern unsigned int halt_in_debugger; - -extern unsigned int switch_debugger; - -extern unsigned int current_debugger; -#define NO_CUR_DB 0x0 -#define KDP_CUR_DB 0x1 -//#define KDB_CUR_DB 0x2 -#define HW_SHM_CUR_DB 0x3 - -extern unsigned int active_debugger; -extern unsigned int debug_mode; -extern unsigned int disable_debug_output; - -extern unsigned int panicDebugging; -extern unsigned int logPanicDataToScreen; -extern unsigned int kdebug_serial; - -extern int db_run_mode; +/* + * Any updates to this header should be also updated in astris as it can not + * grab this header from the SDK. + * + * NOTE: DO NOT REMOVE OR CHANGE THE MEANING OF ANY FIELDS FROM THIS STRUCTURE. + * Any modifications should add new fields at the end, bump the version number + * and be done alongside astris and DumpPanic changes. + */ +struct embedded_panic_header { + uint32_t eph_magic; /* PANIC_MAGIC if valid */ + uint32_t eph_crc; /* CRC of everything following the ph_crc in the header and the contents */ + uint32_t eph_version; /* embedded_panic_header version */ + uint64_t eph_panic_flags; /* Flags indicating any state or relevant details */ + uint32_t eph_panic_log_offset; /* Offset of the beginning of the panic log from the beginning of the header */ + uint32_t eph_panic_log_len; /* length of the panic log */ + uint32_t eph_stackshot_offset; /* Offset of the beginning of the panic stackshot from the beginning of the header */ + uint32_t eph_stackshot_len; /* length of the panic stackshot (0 if not valid ) */ + uint32_t eph_other_log_offset; /* Offset of the other log (any logging subsequent to the stackshot) from the beginning of the header */ + uint32_t eph_other_log_len; /* length of the other log */ +} __attribute__((packed)); -/* modes the system may be running in */ +#define EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_COMPLETE 0x01 +#define EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_SUCCEEDED 0x02 +#define EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_DEBUGGERSYNC 0x04 +#define EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR 0x08 +#define EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_INCOMPLETE 0x10 +#define EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_NESTED 0x20 +#define EMBEDDED_PANIC_HEADER_FLAG_NESTED_PANIC 0x40 +#define EMBEDDED_PANIC_HEADER_FLAG_BUTTON_RESET_PANIC 0x80 +#define EMBEDDED_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC 0x100 + +#define EMBEDDED_PANIC_HEADER_CURRENT_VERSION 1 +#define EMBEDDED_PANIC_MAGIC 0x46554E4B /* FUNK */ + +struct macos_panic_header { + uint32_t mph_magic; /* PANIC_MAGIC if valid */ + uint32_t mph_crc; /* CRC of everything following mph_crc in the header and the contents */ + uint32_t mph_version; /* macos_panic_header version */ + uint32_t mph_padding; /* unused */ + uint64_t mph_panic_flags; /* Flags indicating any state or relevant details */ + uint32_t mph_panic_log_offset; /* Offset of the panic log from the beginning of the header */ + uint32_t mph_panic_log_len; /* length of the panic log */ + char mph_data[]; /* panic data -- DO NOT ACCESS THIS FIELD DIRECTLY. Use the offsets above relative to the beginning of the header */ +} __attribute__((packed)); -#define STEP_NONE 0 -#define STEP_ONCE 1 -#define STEP_RETURN 2 -#define STEP_CALLT 3 -#define STEP_CONTINUE 4 -#define STEP_INVISIBLE 5 -#define STEP_COUNT 6 -#define STEP_TRACE 7 /* Show all calls to functions and returns */ +#define MACOS_PANIC_HEADER_CURRENT_VERSION 1 +#define MACOS_PANIC_MAGIC 0x44454544 /* DEED */ -extern const char *panicstr; -extern volatile unsigned int nestedpanic; -extern int unsigned long panic_caller; +#define MACOS_PANIC_HEADER_FLAG_NESTED_PANIC 0x01 +#define MACOS_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC 0x02 -extern char *debug_buf_ptr; -extern unsigned int debug_buf_size; - -extern void debug_log_init(void); -extern void debug_putc(char); +#endif /* __APPLE_API_UNSTABLE */ +#endif /* __APPLE_API_PRIVATE */ -extern void panic_init(void); +#ifdef KERNEL -int packA(char *inbuf, uint32_t length, uint32_t buflen); -void unpackA(char *inbuf, uint32_t length); +__BEGIN_DECLS -void panic_display_system_configuration(boolean_t launchd_exit); -void panic_display_zprint(void); -void panic_display_kernel_aslr(void); -void panic_display_hibb(void); -void panic_display_model_name(void); -void panic_display_kernel_uuid(void); -#if CONFIG_ZLEAKS -void panic_display_ztrace(void); -#endif /* CONFIG_ZLEAKS */ -#if CONFIG_ECC_LOGGING -void panic_display_ecc_errors(void); -#endif /* CONFIG_ECC_LOGGING */ -#endif /* MACH_KERNEL_PRIVATE */ +extern void panic(const char *string, ...) __printflike(1,2); -#define DB_HALT 0x1 -#define DB_PRT 0x2 -#define DB_NMI 0x4 -#define DB_KPRT 0x8 -#define DB_KDB 0x10 -#define DB_SLOG 0x20 -#define DB_ARP 0x40 -#define DB_KDP_BP_DIS 0x80 -#define DB_LOG_PI_SCRN 0x100 -#define DB_KDP_GETC_ENA 0x200 +__END_DECLS -#define DB_KERN_DUMP_ON_PANIC 0x400 /* Trigger core dump on panic*/ -#define DB_KERN_DUMP_ON_NMI 0x800 /* Trigger core dump on NMI */ -#define DB_DBG_POST_CORE 0x1000 /*Wait in debugger after NMI core */ -#define DB_PANICLOG_DUMP 0x2000 /* Send paniclog on panic,not core*/ -#define DB_REBOOT_POST_CORE 0x4000 /* Attempt to reboot after - * post-panic crashdump/paniclog - * dump. - */ -#define DB_NMI_BTN_ENA 0x8000 /* Enable button to directly trigger NMI */ -#define DB_PRT_KDEBUG 0x10000 /* kprintf KDEBUG traces */ -#define DB_DISABLE_LOCAL_CORE 0x20000 /* ignore local kernel core dump support */ -#define DB_DISABLE_GZIP_CORE 0x40000 /* don't gzip kernel core dumps */ +#endif /* KERNEL */ +#ifdef KERNEL_PRIVATE #if DEBUG /* * For the DEBUG kernel, support the following: @@ -397,38 +367,73 @@ enum { #define DEBUG_KPRINT_SYSCALL_IPC(fmt, args...) \ DEBUG_KPRINT_SYSCALL_MASK(DEBUG_KPRINT_SYSCALL_IPC_MASK,fmt,args) -#endif /* KERNEL_PRIVATE */ - +/* Debug boot-args */ +#define DB_HALT 0x1 +//#define DB_PRT 0x2 -- obsolete +#define DB_NMI 0x4 +#define DB_KPRT 0x8 +#define DB_KDB 0x10 +#define DB_ARP 0x40 +#define DB_KDP_BP_DIS 0x80 +//#define DB_LOG_PI_SCRN 0x100 -- obsolete +#define DB_KDP_GETC_ENA 0x200 -#ifdef XNU_KERNEL_PRIVATE +#define DB_KERN_DUMP_ON_PANIC 0x400 /* Trigger core dump on panic*/ +#define DB_KERN_DUMP_ON_NMI 0x800 /* Trigger core dump on NMI */ +#define DB_DBG_POST_CORE 0x1000 /*Wait in debugger after NMI core */ +#define DB_PANICLOG_DUMP 0x2000 /* Send paniclog on panic,not core*/ +#define DB_REBOOT_POST_CORE 0x4000 /* Attempt to reboot after + * post-panic crashdump/paniclog + * dump. + */ +#define DB_NMI_BTN_ENA 0x8000 /* Enable button to directly trigger NMI */ +#define DB_PRT_KDEBUG 0x10000 /* kprintf KDEBUG traces */ +#define DB_DISABLE_LOCAL_CORE 0x20000 /* ignore local kernel core dump support */ +#define DB_DISABLE_GZIP_CORE 0x40000 /* don't gzip kernel core dumps */ /* - * @var not_in_kdp - * - * @abstract True if we're in normal kernel operation, False if we're in a - * single-core debugger context. + * Values for a 64-bit mask that's passed to the debugger. */ -extern unsigned int not_in_kdp; +#define DEBUGGER_OPTION_NONE 0x0ULL +#define DEBUGGER_OPTION_PANICLOGANDREBOOT 0x1ULL /* capture a panic log and then reboot immediately */ +#define DEBUGGER_OPTION_RECURPANIC_ENTRY 0x2ULL +#define DEBUGGER_OPTION_RECURPANIC_PRELOG 0x4ULL +#define DEBUGGER_OPTION_RECURPANIC_POSTLOG 0x8ULL +#define DEBUGGER_OPTION_RECURPANIC_POSTCORE 0x10ULL +#define DEBUGGER_OPTION_INITPROC_PANIC 0x20ULL +#define DEBUGGER_OPTION_COPROC_INITIATED_PANIC 0x40ULL /* panic initiated by a co-processor */ +#define DEBUGGER_OPTION_SKIP_LOCAL_COREDUMP 0x80ULL /* don't try to save local coredumps for this panic */ -/* - * @function DebuggerWithCallback - * - * @abstract Enter single-core debugger context and call a callback function. - * - * @param proceed_on_sync_failure If true, then go ahead and try to debug even - * if we can't synch with the other cores. This is inherently unsafe and should - * only be used if the kernel is going down in flames anyway. - * - * @result returns KERN_OPERATION_TIMED_OUT if synchronization times out and - * proceed_on_sync_failure is false. Otherwise return the return value of the - * callback. - */ -kern_return_t -DebuggerWithCallback(kern_return_t (*callback) (void*), - void *callback_context, - boolean_t proceed_on_sync_failure); +__BEGIN_DECLS + +#define panic_plain(ex, ...) (panic)(ex, ## __VA_ARGS__) + +#define __STRINGIFY(x) #x +#define LINE_NUMBER(x) __STRINGIFY(x) +#define PANIC_LOCATION __FILE__ ":" LINE_NUMBER(__LINE__) + +#if CONFIG_EMBEDDED +#define panic(ex, ...) (panic)(# ex, ## __VA_ARGS__) +#else +#define panic(ex, ...) (panic)(# ex "@" PANIC_LOCATION, ## __VA_ARGS__) +#endif + +void panic_context(unsigned int reason, void *ctx, const char *string, ...); +void panic_with_options(unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...); +void Debugger(const char * message); +void populate_model_name(char *); + +unsigned panic_active(void); + +__END_DECLS + +#endif /* KERNEL_PRIVATE */ + +#if XNU_KERNEL_PRIVATE boolean_t oslog_is_safe(void); +boolean_t debug_mode_active(void); +boolean_t stackshot_active(void); /* * @function stack_snapshot_from_kernel @@ -453,55 +458,111 @@ stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, } #endif +#if !CONFIG_EMBEDDED +extern char debug_buf[]; +extern boolean_t coprocessor_paniclog_flush; +#endif /* !CONFIG_EMBEDDED */ -#endif /* XNU_KERNEL_PRIVATE */ +extern char *debug_buf_base; -#ifdef KERNEL +extern char kernel_uuid_string[]; +extern char panic_disk_error_description[]; +extern size_t panic_disk_error_description_size; -__BEGIN_DECLS +extern unsigned char *kernel_uuid; +extern unsigned int debug_boot_arg; -extern void panic(const char *string, ...) __printflike(1,2); +#ifdef XNU_KERNEL_PRIVATE -#if KERNEL_PRIVATE -void _consume_panic_args(int, ...); -void panic_context(unsigned int reason, void *ctx, const char *string, ...); -void panic_with_options(unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...); +#ifdef __cplusplus +extern "C" { +#endif -/* launchd crashed prefix in message to signal special panic handling */ -#define LAUNCHD_CRASHED_PREFIX "initproc exited" +extern boolean_t doprnt_hide_pointers; -/* - * Values for a 64-bit mask that's passed to the debugger. - */ -#define DEBUGGER_OPTION_NONE 0x0ULL -#define DEBUGGER_OPTION_PANICLOGANDREBOOT 0x1ULL /* capture a panic log and then reboot immediately */ +#ifdef __cplusplus +} #endif -#ifdef CONFIG_NO_PANIC_STRINGS -#if KERNEL_PRIVATE -#define panic_plain(x, ...) _consume_panic_args( 0, ## __VA_ARGS__ ) -#define panic(x, ...) _consume_panic_args( 0, ## __VA_ARGS__ ) +extern unsigned int halt_in_debugger; /* pending halt in debugger after boot */ +extern unsigned int current_debugger; +#define NO_CUR_DB 0x0 +#define KDP_CUR_DB 0x1 + +extern unsigned int active_debugger; +extern unsigned int kernel_debugger_entry_count; + +extern unsigned int panicDebugging; +extern unsigned int kdebug_serial; + +extern const char *debugger_panic_str; + +extern char *debug_buf_ptr; + +#if CONFIG_EMBEDDED +extern unsigned int debug_buf_size; #else -#define panic_plain(...) (panic)((char *)0) -#define panic(...) (panic)((char *)0) +extern const unsigned int debug_buf_size; #endif -#else /* CONFIGS_NO_PANIC_STRINGS */ -#define panic_plain(ex, ...) \ - (panic)(ex, ## __VA_ARGS__) -#define __STRINGIFY(x) #x -#define LINE_NUMBER(x) __STRINGIFY(x) -#define PANIC_LOCATION __FILE__ ":" LINE_NUMBER(__LINE__) -#define panic(ex, ...) \ - (panic)(# ex "@" PANIC_LOCATION, ## __VA_ARGS__) -#endif /* CONFIGS_NO_PANIC_STRINGS */ -#ifdef KERNEL_PRIVATE -void populate_model_name(char *); -unsigned panic_active(void); -#endif +extern void debug_log_init(void); +extern void debug_putc(char); +extern void panic_init(void); + +#if defined (__x86_64__) +int packA(char *inbuf, uint32_t length, uint32_t buflen); +void unpackA(char *inbuf, uint32_t length); + +#if DEVELOPMENT || DEBUG +#define PANIC_STACKSHOT_BUFSIZE (1024 * 1024) + +extern uintptr_t panic_stackshot_buf; +extern size_t panic_stackshot_len; +#endif /* DEVELOPMENT || DEBUG */ +#endif /* defined (__x86_64__) */ + +void SavePanicInfo(const char *message, uint64_t panic_options); +void paniclog_flush(void); +void panic_display_system_configuration(boolean_t launchd_exit); +void panic_display_zprint(void); +void panic_display_kernel_aslr(void); +void panic_display_hibb(void); +void panic_display_model_name(void); +void panic_display_kernel_uuid(void); +#if CONFIG_ZLEAKS +void panic_display_ztrace(void); +#endif /* CONFIG_ZLEAKS */ +#if CONFIG_ECC_LOGGING +void panic_display_ecc_errors(void); +#endif /* CONFIG_ECC_LOGGING */ +#endif /* MACH_KERNEL_PRIVATE */ + +/* + * @var not_in_kdp + * + * @abstract True if we're in normal kernel operation, False if we're in a + * single-core debugger context. + */ +extern unsigned int not_in_kdp; + +#define DEBUGGER_NO_CPU -1 + +typedef enum { + DBOP_NONE, + DBOP_STACKSHOT, + DBOP_RESET_PGO_COUNTERS, + DBOP_PANIC, + DBOP_DEBUGGER, + DBOP_BREAKPOINT, +} debugger_op; + +kern_return_t DebuggerTrapWithState(debugger_op db_op, const char *db_message, const char *db_panic_str, va_list *db_panic_args, + uint64_t db_panic_options, boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller); +void handle_debugger_trap(unsigned int exception, unsigned int code, unsigned int subcode, void *state); + +void DebuggerWithContext(unsigned int reason, void *ctx, const char *message, uint64_t debugger_options_mask); -#if XNU_KERNEL_PRIVATE #if DEBUG || DEVELOPMENT /* leak pointer scan definitions */ @@ -518,20 +579,24 @@ enum typedef void (*leak_site_proc)(void * refCon, uint32_t siteCount, uint32_t zoneSize, uintptr_t * backtrace, uint32_t btCount); +#ifdef __cplusplus +extern "C" { +#endif + extern kern_return_t zone_leaks(const char * zoneName, uint32_t nameLen, leak_site_proc proc, void * refCon); extern void zone_leaks_scan(uintptr_t * instances, uint32_t count, uint32_t zoneSize, uint32_t * found); +#ifdef __cplusplus +} +#endif + extern boolean_t kdp_is_in_zone(void *addr, const char *zone_name); #endif /* DEBUG || DEVELOPMENT */ #endif /* XNU_KERNEL_PRIVATE */ -__END_DECLS - -#endif /* KERNEL */ - #endif /* _KERN_DEBUG_H_ */ diff --git a/osfmk/kern/exc_guard.h b/osfmk/kern/exc_guard.h new file mode 100644 index 000000000..8486ec569 --- /dev/null +++ b/osfmk/kern/exc_guard.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Mach Operating System + * Copyright (c) 1989 Carnegie-Mellon University + * Copyright (c) 1988 Carnegie-Mellon University + * Copyright (c) 1987 Carnegie-Mellon University + * All rights reserved. The CMU software License Agreement specifies + * the terms and conditions for use and redistribution. + */ + +/* + * EXC_GUARD related macros, namespace etc. + */ + +#ifndef _EXC_GUARD_H_ +#define _EXC_GUARD_H_ + +/* + * EXC_GUARD exception code namespace. + * + * code: + * +-------------------+----------------+--------------+ + * |[63:61] guard type | [60:32] flavor | [31:0] target| + * +-------------------+----------------+--------------+ + * + * subcode: + * +---------------------------------------------------+ + * |[63:0] guard identifier | + * +---------------------------------------------------+ + */ + +#define EXC_GUARD_DECODE_GUARD_TYPE(code) \ + (((code) >> 61) & 0x7ull) +#define EXC_GUARD_DECODE_GUARD_FLAVOR(code) \ + (((code) >> 32) & 0x1fffffff) +#define EXC_GUARD_DECODE_GUARD_TARGET(code) \ + ((uint32_t)(code)) + +/* EXC_GUARD types */ + +/* + * Mach port guards use the exception codes like this: + * + * code: + * +-----------------------------+----------------+-----------------+ + * |[63:61] GUARD_TYPE_MACH_PORT | [60:32] flavor | [31:0] port name| + * +-----------------------------+----------------+-----------------+ + * + * subcode: + * +----------------------------------------------------------------+ + * |[63:0] guard identifier | + * +----------------------------------------------------------------+ + */ + +#define GUARD_TYPE_MACH_PORT 0x1 /* guarded mach port */ + +/* + * File descriptor guards use the exception codes this: + * + * code: + * +-----------------------------+----------------+-----------------+ + * |[63:61] GUARD_TYPE_FD | [60:32] flavor | [31:0] fd | + * +-----------------------------+----------------+-----------------+ + * + * subcode: + * +----------------------------------------------------------------+ + * |[63:0] guard identifier | + * +----------------------------------------------------------------+ + */ + +#define GUARD_TYPE_FD 0x2 /* guarded file descriptor */ + +/* + * User generated guards use the exception codes this: + * + * code: + * +-----------------------------+----------------+-----------------+ + * |[63:61] GUARD_TYPE_USER | [60:32] unused | [31:0] namespc | + * +-----------------------------+----------------+-----------------+ + * + * subcode: + * +----------------------------------------------------------------+ + * |[63:0] reason_code | + * +----------------------------------------------------------------+ + */ + +#define GUARD_TYPE_USER 0x3 /* Userland assertions */ + +/* + * Vnode guards use the exception codes like this: + * + * code: + * +-----------------------------+----------------+-----------------+ + * |[63:61] GUARD_TYPE_VN | [60:32] flavor | [31:0] pid | + * +-----------------------------+----------------+-----------------+ + * + * subcode: + * +----------------------------------------------------------------+ + * |[63:0] guard identifier | + * +----------------------------------------------------------------+ + */ + +#define GUARD_TYPE_VN 0x4 /* guarded vnode */ + +#ifdef KERNEL + +#define EXC_GUARD_ENCODE_TYPE(code, type) \ + ((code) |= (((uint64_t)(type) & 0x7ull) << 61)) +#define EXC_GUARD_ENCODE_FLAVOR(code, flavor) \ + ((code) |= (((uint64_t)(flavor) & 0x1fffffffull) << 32)) +#define EXC_GUARD_ENCODE_TARGET(code, target) \ + ((code) |= (((uint64_t)(target) & 0xffffffffull))) + +#endif /* KERNEL */ + +#endif /* _EXC_GUARD_H_ */ diff --git a/osfmk/kern/exception.c b/osfmk/kern/exception.c index a93f38ca5..4042b91b3 100644 --- a/osfmk/kern/exception.c +++ b/osfmk/kern/exception.c @@ -139,15 +139,14 @@ exception_deliver( struct exception_action *excp, lck_mtx_t *mutex) { - ipc_port_t exc_port; + ipc_port_t exc_port = IPC_PORT_NULL; exception_data_type_t small_code[EXCEPTION_CODE_MAX]; int code64; int behavior; int flavor; kern_return_t kr; - int use_fast_retrieve = TRUE; task_t task; - ipc_port_t thread_port = NULL, task_port = NULL; + ipc_port_t thread_port = IPC_PORT_NULL, task_port = IPC_PORT_NULL; /* * Save work if we are terminating. @@ -214,21 +213,25 @@ exception_deliver( * to the next level. */ if (mac_exc_action_check_exception_send(task, excp) != 0) { - return KERN_FAILURE; + kr = KERN_FAILURE; + goto out_release_right; } #endif - if ((thread != current_thread() || exception == EXC_CORPSE_NOTIFY) - && behavior != EXCEPTION_STATE) { - use_fast_retrieve = FALSE; - - task_reference(task); - task_port = convert_task_to_port(task); - /* task ref consumed */ - thread_reference(thread); - thread_port = convert_thread_to_port(thread); - /* thread ref consumed */ + if (behavior != EXCEPTION_STATE) { + if (thread != current_thread() || exception == EXC_CORPSE_NOTIFY) { + task_reference(task); + task_port = convert_task_to_port(task); + /* task ref consumed */ + thread_reference(thread); + thread_port = convert_thread_to_port(thread); + /* thread ref consumed */ + } + else { + task_port = retrieve_task_self_fast(thread->task); + thread_port = retrieve_thread_self_fast(thread); + } } switch (behavior) { @@ -258,38 +261,38 @@ exception_deliver( state, state_cnt, state, &state_cnt); } - if (kr == MACH_MSG_SUCCESS && exception != EXC_CORPSE_NOTIFY) - kr = thread_setstatus(thread, flavor, - (thread_state_t)state, - state_cnt); + if (kr == KERN_SUCCESS) { + if (exception != EXC_CORPSE_NOTIFY) + kr = thread_setstatus(thread, flavor, + (thread_state_t)state, + state_cnt); + goto out_release_right; + } + } - return kr; + goto out_release_right; } case EXCEPTION_DEFAULT: c_thr_exc_raise++; if (code64) { kr = mach_exception_raise(exc_port, - use_fast_retrieve ? retrieve_thread_self_fast(thread) : - thread_port, - use_fast_retrieve ? retrieve_task_self_fast(thread->task) : - task_port, + thread_port, + task_port, exception, code, codeCnt); } else { kr = exception_raise(exc_port, - use_fast_retrieve ? retrieve_thread_self_fast(thread) : - thread_port, - use_fast_retrieve ? retrieve_task_self_fast(thread->task) : - task_port, + thread_port, + task_port, exception, small_code, codeCnt); } - return kr; + goto out_release_right; case EXCEPTION_STATE_IDENTITY: { mach_msg_type_number_t state_cnt; @@ -304,10 +307,8 @@ exception_deliver( if (code64) { kr = mach_exception_raise_state_identity( exc_port, - use_fast_retrieve ? retrieve_thread_self_fast(thread) : - thread_port, - use_fast_retrieve ? retrieve_task_self_fast(thread->task) : - task_port, + thread_port, + task_port, exception, code, codeCnt, @@ -316,10 +317,8 @@ exception_deliver( state, &state_cnt); } else { kr = exception_raise_state_identity(exc_port, - use_fast_retrieve ? retrieve_thread_self_fast(thread) : - thread_port, - use_fast_retrieve ? retrieve_task_self_fast(thread->task) : - task_port, + thread_port, + task_port, exception, small_code, codeCnt, @@ -327,19 +326,40 @@ exception_deliver( state, state_cnt, state, &state_cnt); } - if (kr == MACH_MSG_SUCCESS && exception != EXC_CORPSE_NOTIFY) - kr = thread_setstatus(thread, flavor, - (thread_state_t)state, - state_cnt); + + if (kr == KERN_SUCCESS) { + if (exception != EXC_CORPSE_NOTIFY) + kr = thread_setstatus(thread, flavor, + (thread_state_t)state, + state_cnt); + goto out_release_right; + } + } - return kr; + goto out_release_right; } default: panic ("bad exception behavior!"); return KERN_FAILURE; }/* switch */ + +out_release_right: + + if (task_port) { + ipc_port_release_send(task_port); + } + + if (thread_port) { + ipc_port_release_send(thread_port); + } + + if (exc_port) { + ipc_port_release_send(exc_port); + } + + return kr; } /* @@ -446,7 +466,7 @@ exception_triage_thread( */ host_priv = host_priv_self(); mutex = &host_priv->lock; - + if (KERN_SUCCESS == check_exc_receiver_dependency(exception, host_priv->exc_actions, mutex)) { kr = exception_deliver(thread, exception, code, codeCnt, host_priv->exc_actions, mutex); diff --git a/osfmk/kern/gzalloc.c b/osfmk/kern/gzalloc.c index 5062baa78..915b22782 100644 --- a/osfmk/kern/gzalloc.c +++ b/osfmk/kern/gzalloc.c @@ -59,6 +59,11 @@ * -nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel * enables the guard allocator for zones sized 8K-16K (if present) by * default, this option can disable that behaviour. + * gzname= target a zone by name. Can be coupled with size-based + * targeting. Naming conventions match those of the zlog boot-arg, i.e. + * "a period in the logname will match a space in the zone name" + * -gzalloc_no_dfree_check Eliminate double free checks + * gzalloc_zscale= specify size multiplier for the dedicated gzalloc submap */ #include @@ -99,7 +104,7 @@ uint32_t pdzalloc_count, pdzfree_count; #define GZDEADZONE ((zone_t) 0xDEAD201E) #define GZALLOC_SIGNATURE (0xABADCAFE) #define GZALLOC_RESERVE_SIZE_DEFAULT (2 * 1024 * 1024) -#define GZFC_DEFAULT_SIZE (1024) +#define GZFC_DEFAULT_SIZE (1536) char gzalloc_fill_pattern = 0x67; /* 'g' */ @@ -107,10 +112,11 @@ uint32_t gzalloc_min = ~0U; uint32_t gzalloc_max = 0; uint32_t gzalloc_size = 0; uint64_t gzalloc_allocated, gzalloc_freed, gzalloc_early_alloc, gzalloc_early_free, gzalloc_wasted; -boolean_t gzalloc_uf_mode = FALSE, gzalloc_consistency_checks = TRUE; +boolean_t gzalloc_uf_mode = FALSE, gzalloc_consistency_checks = TRUE, gzalloc_dfree_check = TRUE; vm_prot_t gzalloc_prot = VM_PROT_NONE; uint32_t gzalloc_guard = KMA_GUARD_LAST; uint32_t gzfc_size = GZFC_DEFAULT_SIZE; +uint32_t gzalloc_zonemap_scale = 6; vm_map_t gzalloc_map; vm_offset_t gzalloc_map_min, gzalloc_map_max; @@ -127,6 +133,9 @@ typedef struct gzalloc_header { extern zone_t vm_page_zone; +static zone_t gztrackzone = NULL; +static char gznamedzone[MAX_ZONE_NAME] = ""; + void gzalloc_reconfigure(__unused zone_t z) { /* Nothing for now */ } @@ -135,11 +144,22 @@ boolean_t gzalloc_enabled(void) { return gzalloc_mode; } +static inline boolean_t gzalloc_tracked(zone_t z) { + return (gzalloc_mode && + (((z->elem_size >= gzalloc_min) && (z->elem_size <= gzalloc_max)) || (z == gztrackzone)) && + (z->gzalloc_exempt == 0)); +} + void gzalloc_zone_init(zone_t z) { if (gzalloc_mode) { bzero(&z->gz, sizeof(z->gz)); - if (gzfc_size && (z->elem_size >= gzalloc_min) && (z->elem_size <= gzalloc_max) && (z->gzalloc_exempt == FALSE)) { + if (track_this_zone(z->zone_name, gznamedzone)) { + gztrackzone = z; + } + + if (gzfc_size && + gzalloc_tracked(z)) { vm_size_t gzfcsz = round_page(sizeof(*z->gz.gzfc) * gzfc_size); /* If the VM/kmem system aren't yet configured, carve @@ -165,6 +185,62 @@ void gzalloc_zone_init(zone_t z) { } } +/* Called by zdestroy() to dump the free cache elements so the zone count can drop to zero. */ +void gzalloc_empty_free_cache(zone_t zone) { + if (__improbable(gzalloc_tracked(zone))) { + kern_return_t kr; + int freed_elements = 0; + vm_offset_t free_addr = 0; + vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE); + vm_offset_t gzfcsz = round_page(sizeof(*zone->gz.gzfc) * gzfc_size); + vm_offset_t gzfc_copy; + + kr = kmem_alloc(kernel_map, &gzfc_copy, gzfcsz, VM_KERN_MEMORY_OSFMK); + if (kr != KERN_SUCCESS) { + panic("gzalloc_empty_free_cache: kmem_alloc: 0x%x", kr); + } + + /* Reset gzalloc_data. */ + lock_zone(zone); + memcpy((void *)gzfc_copy, (void *)zone->gz.gzfc, gzfcsz); + bzero((void *)zone->gz.gzfc, gzfcsz); + zone->gz.gzfc_index = 0; + unlock_zone(zone); + + /* Free up all the cached elements. */ + for (uint32_t index = 0; index < gzfc_size; index++) { + free_addr = ((vm_offset_t *)gzfc_copy)[index]; + if (free_addr && free_addr >= gzalloc_map_min && free_addr < gzalloc_map_max) { + kr = vm_map_remove( + gzalloc_map, + free_addr, + free_addr + rounded_size + (1 * PAGE_SIZE), + VM_MAP_REMOVE_KUNWIRE); + if (kr != KERN_SUCCESS) { + panic("gzalloc_empty_free_cache: vm_map_remove: %p, 0x%x", (void *)free_addr, kr); + } + OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed); + OSAddAtomic64(-((SInt32) (rounded_size - zone->elem_size)), &gzalloc_wasted); + + freed_elements++; + } + } + /* + * TODO: Consider freeing up zone->gz.gzfc as well if it didn't come from the gzalloc_reserve pool. + * For now we're reusing this buffer across zdestroy's. We would have to allocate it again on a + * subsequent zinit() as well. + */ + + /* Decrement zone counters. */ + lock_zone(zone); + zone->count -= freed_elements; + zone->cur_size -= (freed_elements * rounded_size); + unlock_zone(zone); + + kmem_free(kernel_map, gzfc_copy, gzfcsz); + } +} + void gzalloc_configure(void) { char temp_buf[16]; @@ -201,13 +277,24 @@ void gzalloc_configure(void) { gzalloc_guard = KMA_GUARD_FIRST; } + if (PE_parse_boot_argn("-gzalloc_no_dfree_check", temp_buf, sizeof(temp_buf))) { + gzalloc_dfree_check = FALSE; + } + + (void) PE_parse_boot_argn("gzalloc_zscale", &gzalloc_zonemap_scale, sizeof(gzalloc_zonemap_scale)); + if (PE_parse_boot_argn("-gzalloc_noconsistency", temp_buf, sizeof (temp_buf))) { gzalloc_consistency_checks = FALSE; } -#if DEBUG + + if (PE_parse_boot_argn("gzname", gznamedzone, sizeof(gznamedzone))) { + gzalloc_mode = TRUE; + } +#if DEBUG if (gzalloc_mode == FALSE) { - gzalloc_min = 8192; - gzalloc_max = 16384; + gzalloc_min = 1024; + gzalloc_max = 1024; + strlcpy(gznamedzone, "pmap", sizeof(gznamedzone)); gzalloc_prot = VM_PROT_READ; gzalloc_mode = TRUE; } @@ -225,13 +312,18 @@ void gzalloc_init(vm_size_t max_zonemap_size) { kern_return_t retval; if (gzalloc_mode) { - retval = kmem_suballoc(kernel_map, &gzalloc_map_min, (max_zonemap_size << 2), - FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT | VM_MAKE_TAG(VM_KERN_MEMORY_ZONE), - &gzalloc_map); + vm_map_kernel_flags_t vmk_flags; + + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_permanent = TRUE; + retval = kmem_suballoc(kernel_map, &gzalloc_map_min, (max_zonemap_size * gzalloc_zonemap_scale), + FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_ZONE, + &gzalloc_map); - if (retval != KERN_SUCCESS) - panic("zone_init: kmem_suballoc(gzalloc) failed"); - gzalloc_map_max = gzalloc_map_min + (max_zonemap_size << 2); + if (retval != KERN_SUCCESS) { + panic("zone_init: kmem_suballoc(gzalloc_map, 0x%lx, %u) failed", max_zonemap_size, gzalloc_zonemap_scale); + } + gzalloc_map_max = gzalloc_map_min + (max_zonemap_size * gzalloc_zonemap_scale); } } @@ -239,10 +331,7 @@ vm_offset_t gzalloc_alloc(zone_t zone, boolean_t canblock) { vm_offset_t addr = 0; - if (__improbable(gzalloc_mode && - (((zone->elem_size >= gzalloc_min) && - (zone->elem_size <= gzalloc_max))) && - (zone->gzalloc_exempt == 0))) { + if (__improbable(gzalloc_tracked(zone))) { if (get_preemption_level() != 0) { if (canblock == TRUE) { @@ -261,7 +350,7 @@ gzalloc_alloc(zone_t zone, boolean_t canblock) { /* Early allocations are supplied directly from the * reserve. */ - if (gzalloc_reserve_size < rounded_size) + if (gzalloc_reserve_size < (rounded_size + PAGE_SIZE)) panic("gzalloc reserve exhausted"); gzaddr = gzalloc_reserve; /* No guard page for these early allocations, just @@ -319,6 +408,7 @@ gzalloc_alloc(zone_t zone, boolean_t canblock) { } lock_zone(zone); + assert(zone->zone_valid); zone->count++; zone->sum_count++; zone->cur_size += rounded_size; @@ -334,10 +424,7 @@ boolean_t gzalloc_free(zone_t zone, void *addr) { boolean_t gzfreed = FALSE; kern_return_t kr; - if (__improbable(gzalloc_mode && - (((zone->elem_size >= gzalloc_min) && - (zone->elem_size <= gzalloc_max))) && - (zone->gzalloc_exempt == 0))) { + if (__improbable(gzalloc_tracked(zone))) { gzhdr_t *gzh; vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE); vm_offset_t residue = rounded_size - zone->elem_size; @@ -352,7 +439,24 @@ boolean_t gzalloc_free(zone_t zone, void *addr) { saddr = ((vm_offset_t)addr) - residue; } - assert((saddr & PAGE_MASK) == 0); + if ((saddr & PAGE_MASK) != 0) { + panic("gzalloc_free: invalid address supplied: %p (adjusted: 0x%lx) for zone with element sized 0x%lx\n", addr, saddr, zone->elem_size); + } + + if (gzfc_size) { + if (gzalloc_dfree_check) { + uint32_t gd; + + lock_zone(zone); + assert(zone->zone_valid); + for (gd = 0; gd < gzfc_size; gd++) { + if (zone->gz.gzfc[gd] == saddr) { + panic("gzalloc: double free detected, freed address: 0x%lx, current free cache index: %d, freed index: %d", saddr, zone->gz.gzfc_index, gd); + } + } + unlock_zone(zone); + } + } if (gzalloc_consistency_checks) { if (gzh->gzsig != GZALLOC_SIGNATURE) { @@ -365,6 +469,21 @@ boolean_t gzalloc_free(zone_t zone, void *addr) { if (gzh->gzsize != zone->elem_size) { panic("Mismatched zfree or under/overflow for zone %p, recorded size: 0x%x, element size: 0x%x, address: %p\n", zone, gzh->gzsize, (uint32_t) zone->elem_size, (void *)addr); } + + char *gzc, *checkstart, *checkend; + if (gzalloc_uf_mode) { + checkstart = (char *) ((uintptr_t) gzh + sizeof(gzh)); + checkend = (char *) ((((vm_offset_t)addr) & ~PAGE_MASK) + PAGE_SIZE); + } else { + checkstart = (char *) trunc_page_64(addr); + checkend = (char *)gzh; + } + + for (gzc = checkstart; gzc < checkend; gzc++) { + if (*gzc != gzalloc_fill_pattern) { + panic("GZALLOC: detected over/underflow, byte at %p, element %p, contents 0x%x from 0x%lx byte sized zone (%s) doesn't match fill pattern (%c)", gzc, addr, *gzc, zone->elem_size, zone->zone_name, gzalloc_fill_pattern); + } + } } if (!kmem_ready || gzh->gzone == GZDEADZONE) { @@ -398,6 +517,7 @@ boolean_t gzalloc_free(zone_t zone, void *addr) { } lock_zone(zone); + assert(zone->zone_valid); /* Insert newly freed element into the protected free element * cache, and rotate out the LRU element. @@ -418,6 +538,9 @@ boolean_t gzalloc_free(zone_t zone, void *addr) { unlock_zone(zone); if (free_addr) { + // TODO: consider using physical reads to check for + // corruption while on the protected freelist + // (i.e. physical corruption) kr = vm_map_remove( gzalloc_map, free_addr, @@ -425,7 +548,7 @@ boolean_t gzalloc_free(zone_t zone, void *addr) { VM_MAP_REMOVE_KUNWIRE); if (kr != KERN_SUCCESS) panic("gzfree: vm_map_remove: %p, 0x%x", (void *)free_addr, kr); - + // TODO: sysctl-ize for quick reference OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed); OSAddAtomic64(-((SInt32) (rounded_size - zone->elem_size)), &gzalloc_wasted); } @@ -437,7 +560,7 @@ boolean_t gzalloc_free(zone_t zone, void *addr) { boolean_t gzalloc_element_size(void *gzaddr, zone_t *z, vm_size_t *gzsz) { uintptr_t a = (uintptr_t)gzaddr; - if (__improbable(gzalloc_mode && (a >= gzalloc_map_min) && (a <= gzalloc_map_max))) { + if (__improbable(gzalloc_mode && (a >= gzalloc_map_min) && (a < gzalloc_map_max))) { gzhdr_t *gzh; /* Locate the gzalloc metadata adjoining the element */ @@ -466,8 +589,8 @@ boolean_t gzalloc_element_size(void *gzaddr, zone_t *z, vm_size_t *gzsz) { } *gzsz = gzh->gzone->elem_size; - if ((*gzsz < gzalloc_min) || (*gzsz > gzalloc_max)) { - panic("GZALLOC: invalid element size %lu\n", *gzsz); + if (__improbable((gzalloc_tracked(gzh->gzone)) == FALSE)) { + panic("GZALLOC: zone mismatch (%p)\n", gzh->gzone); } if (z) { diff --git a/osfmk/kern/host.c b/osfmk/kern/host.c index 2e5852334..e9cd7d872 100644 --- a/osfmk/kern/host.c +++ b/osfmk/kern/host.c @@ -294,9 +294,7 @@ host_info(host_t host, host_flavor_t flavor, host_info_t info, mach_msg_type_num #if CONFIG_COALITIONS debug_info->config_coalitions = 1; #endif -#if CONFIG_BANK debug_info->config_bank = 1; -#endif #if CONFIG_ATM debug_info->config_atm = 1; #endif @@ -385,7 +383,11 @@ host_statistics(host_t host, host_flavor_t flavor, host_info_t info, mach_msg_ty } } stat32->inactive_count = VM_STATISTICS_TRUNCATE_TO_32_BIT(vm_page_inactive_count); +#if CONFIG_EMBEDDED + stat32->wire_count = VM_STATISTICS_TRUNCATE_TO_32_BIT(vm_page_wire_count); +#else stat32->wire_count = VM_STATISTICS_TRUNCATE_TO_32_BIT(vm_page_wire_count + vm_page_throttled_count + vm_lopage_free_count); +#endif stat32->zero_fill_count = VM_STATISTICS_TRUNCATE_TO_32_BIT(host_vm_stat.zero_fill_count); stat32->reactivations = VM_STATISTICS_TRUNCATE_TO_32_BIT(host_vm_stat.reactivations); stat32->pageins = VM_STATISTICS_TRUNCATE_TO_32_BIT(host_vm_stat.pageins); @@ -490,17 +492,30 @@ host_statistics(host_t host, host_flavor_t flavor, host_info_t info, mach_msg_ty return (KERN_FAILURE); } - task_power_info_t tinfo = (task_power_info_t)info; + task_power_info_t tinfo1 = (task_power_info_t)info; + task_power_info_v2_t tinfo2 = (task_power_info_v2_t)info; - tinfo->task_interrupt_wakeups = dead_task_statistics.task_interrupt_wakeups; - tinfo->task_platform_idle_wakeups = dead_task_statistics.task_platform_idle_wakeups; + tinfo1->task_interrupt_wakeups = dead_task_statistics.task_interrupt_wakeups; + tinfo1->task_platform_idle_wakeups = dead_task_statistics.task_platform_idle_wakeups; - tinfo->task_timer_wakeups_bin_1 = dead_task_statistics.task_timer_wakeups_bin_1; + tinfo1->task_timer_wakeups_bin_1 = dead_task_statistics.task_timer_wakeups_bin_1; - tinfo->task_timer_wakeups_bin_2 = dead_task_statistics.task_timer_wakeups_bin_2; + tinfo1->task_timer_wakeups_bin_2 = dead_task_statistics.task_timer_wakeups_bin_2; - tinfo->total_user = dead_task_statistics.total_user_time; - tinfo->total_system = dead_task_statistics.total_system_time; + tinfo1->total_user = dead_task_statistics.total_user_time; + tinfo1->total_system = dead_task_statistics.total_system_time; + if (*count < TASK_POWER_INFO_V2_COUNT) { + *count = TASK_POWER_INFO_COUNT; + } + else if (*count >= TASK_POWER_INFO_V2_COUNT) { + tinfo2->gpu_energy.task_gpu_utilisation = dead_task_statistics.task_gpu_ns; +#if defined(__arm__) || defined(__arm64__) + tinfo2->task_energy = dead_task_statistics.task_energy; + tinfo2->task_ptime = dead_task_statistics.total_ptime; + tinfo2->task_pset_switches = dead_task_statistics.total_pset_switches; +#endif + *count = TASK_POWER_INFO_V2_COUNT; + } return (KERN_SUCCESS); } @@ -577,7 +592,11 @@ host_statistics64(host_t host, host_flavor_t flavor, host_info64_t info, mach_ms } } stat->inactive_count = vm_page_inactive_count; +#if CONFIG_EMBEDDED + stat->wire_count = vm_page_wire_count; +#else stat->wire_count = vm_page_wire_count + vm_page_throttled_count + vm_lopage_free_count; +#endif stat->zero_fill_count = host_vm_stat.zero_fill_count; stat->reactivations = host_vm_stat.reactivations; stat->pageins = host_vm_stat.pageins; @@ -703,7 +722,7 @@ get_sched_statistics(struct _processor_statistics_np * out, uint32_t * count) /* And include RT Queue information */ bzero(out, sizeof(*out)); out->ps_cpuid = (-1); - out->ps_runq_count_sum = rt_runq.runq_stats.count_sum; + out->ps_runq_count_sum = SCHED(rt_runq_count_sum)(); out++; *count += (uint32_t)sizeof(struct _processor_statistics_np); @@ -1009,7 +1028,21 @@ host_set_atm_diagnostic_flag(host_priv_t host_priv, uint32_t diagnostic_flag) kern_return_t host_set_multiuser_config_flags(host_priv_t host_priv, uint32_t multiuser_config) { +#if CONFIG_EMBEDDED + if (host_priv == HOST_PRIV_NULL) + return (KERN_INVALID_ARGUMENT); + + assert(host_priv == &realhost); + + /* + * Always enforce that the multiuser bit is set + * if a value is written to the commpage word. + */ + commpage_update_multiuser_config(multiuser_config | kIsMultiUserDevice); + return (KERN_SUCCESS); +#else (void)host_priv; (void)multiuser_config; return (KERN_NOT_SUPPORTED); +#endif } diff --git a/osfmk/kern/host.h b/osfmk/kern/host.h index 59458f00c..1b4dca21b 100644 --- a/osfmk/kern/host.h +++ b/osfmk/kern/host.h @@ -98,6 +98,10 @@ typedef struct { uint64_t task_platform_idle_wakeups; uint64_t task_timer_wakeups_bin_1; uint64_t task_timer_wakeups_bin_2; + uint64_t total_ptime; + uint64_t total_pset_switches; + uint64_t task_gpu_ns; + uint64_t task_energy; } expired_task_statistics_t; extern expired_task_statistics_t dead_task_statistics; diff --git a/osfmk/kern/ipc_host.c b/osfmk/kern/ipc_host.c index 8e61a5dfe..19a9c0a97 100644 --- a/osfmk/kern/ipc_host.c +++ b/osfmk/kern/ipc_host.c @@ -133,10 +133,10 @@ void ipc_host_init(void) for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; i++) { realhost.exc_actions[i].port = IP_NULL; - realhost.exc_actions[i].label = NULL; /* The mac framework is not yet initialized, so we defer * initializing the labels to later, when they are set * for the first time. */ + realhost.exc_actions[i].label = NULL; }/* for */ /* @@ -550,6 +550,11 @@ host_set_exception_ports( int i; ipc_port_t old_port[EXC_TYPES_COUNT]; +#if CONFIG_MACF + struct label *deferred_labels[EXC_TYPES_COUNT]; + struct label *new_label; +#endif + if (host_priv == HOST_PRIV_NULL) { return KERN_INVALID_ARGUMENT; } @@ -580,6 +585,16 @@ host_set_exception_ports( #if CONFIG_MACF if (mac_task_check_set_host_exception_ports(current_task(), exception_mask) != 0) return KERN_NO_ACCESS; + + new_label = mac_exc_create_label_for_current_proc(); + + for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; i++) { + if (host_priv->exc_actions[i].label == NULL) { + deferred_labels[i] = mac_exc_create_label(); + } else { + deferred_labels[i] = NULL; + } + } #endif assert(host_priv == &realhost); @@ -590,13 +605,14 @@ host_set_exception_ports( #if CONFIG_MACF if (host_priv->exc_actions[i].label == NULL) { // Lazy initialization (see ipc_port_init). - mac_exc_action_label_init(host_priv->exc_actions + i); + mac_exc_associate_action_label(&host_priv->exc_actions[i], deferred_labels[i]); + deferred_labels[i] = NULL; // Label is used, do not free. } #endif if ((exception_mask & (1 << i)) #if CONFIG_MACF - && mac_exc_action_label_update(current_task(), host_priv->exc_actions + i) == 0 + && mac_exc_update_action_label(&host_priv->exc_actions[i], new_label) == 0 #endif ) { old_port[i] = host_priv->exc_actions[i].port; @@ -614,9 +630,21 @@ host_set_exception_ports( * Consume send rights without any lock held. */ host_unlock(host_priv); - for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; i++) + +#if CONFIG_MACF + mac_exc_free_label(new_label); +#endif + + for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; i++) { if (IP_VALID(old_port[i])) ipc_port_release_send(old_port[i]); +#if CONFIG_MACF + if (deferred_labels[i] != NULL) { + /* Deferred label went unused: Another thread has completed the lazy initialization. */ + mac_exc_free_label(deferred_labels[i]); + } +#endif + } if (IP_VALID(new_port)) /* consume send right */ ipc_port_release_send(new_port); @@ -667,13 +695,6 @@ host_get_exception_ports( count = 0; for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; i++) { -#if CONFIG_MACF - if (host_priv->exc_actions[i].label == NULL) { - // Lazy initialization (see ipc_port_init). - mac_exc_action_label_init(host_priv->exc_actions + i); - } -#endif - if (exception_mask & (1 << i)) { for (j = 0; j < count; j++) { /* @@ -725,6 +746,11 @@ host_swap_exception_ports( count; ipc_port_t old_port[EXC_TYPES_COUNT]; +#if CONFIG_MACF + struct label *deferred_labels[EXC_TYPES_COUNT]; + struct label *new_label; +#endif + if (host_priv == HOST_PRIV_NULL) return KERN_INVALID_ARGUMENT; @@ -749,6 +775,16 @@ host_swap_exception_ports( #if CONFIG_MACF if (mac_task_check_set_host_exception_ports(current_task(), exception_mask) != 0) return KERN_NO_ACCESS; + + new_label = mac_exc_create_label_for_current_proc(); + + for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; i++) { + if (host_priv->exc_actions[i].label == NULL) { + deferred_labels[i] = mac_exc_create_label(); + } else { + deferred_labels[i] = NULL; + } + } #endif /* CONFIG_MACF */ host_lock(host_priv); @@ -758,13 +794,14 @@ host_swap_exception_ports( #if CONFIG_MACF if (host_priv->exc_actions[i].label == NULL) { // Lazy initialization (see ipc_port_init). - mac_exc_action_label_init(host_priv->exc_actions + i); + mac_exc_associate_action_label(&host_priv->exc_actions[i], deferred_labels[i]); + deferred_labels[i] = NULL; // Label is used, do not free. } #endif if ((exception_mask & (1 << i)) #if CONFIG_MACF - && mac_exc_action_label_update(current_task(), host_priv->exc_actions + i) == 0 + && mac_exc_update_action_label(&host_priv->exc_actions[i], new_label) == 0 #endif ) { for (j = 0; j < count; j++) { @@ -793,17 +830,27 @@ host_swap_exception_ports( ipc_port_copy_send(new_port); host_priv->exc_actions[i].behavior = new_behavior; host_priv->exc_actions[i].flavor = new_flavor; - } else + } else { old_port[i] = IP_NULL; + } }/* for */ host_unlock(host_priv); +#if CONFIG_MACF + mac_exc_free_label(new_label); +#endif + /* * Consume send rights without any lock held. */ while (--i >= FIRST_EXCEPTION) { if (IP_VALID(old_port[i])) ipc_port_release_send(old_port[i]); +#if CONFIG_MACF + if (deferred_labels[i] != NULL) { + mac_exc_free_label(deferred_labels[i]); // Label unused. + } +#endif } if (IP_VALID(new_port)) /* consume send right */ diff --git a/osfmk/kern/ipc_kobject.c b/osfmk/kern/ipc_kobject.c index 73e7a084a..1acc81c4a 100644 --- a/osfmk/kern/ipc_kobject.c +++ b/osfmk/kern/ipc_kobject.c @@ -72,8 +72,6 @@ #include #include -#include - #include #include #include @@ -131,6 +129,7 @@ #include #include #include +#include #include @@ -616,6 +615,12 @@ ipc_kobject_notify( trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)request_header + request_header->msgh_size); + + /* + * The kobject notification is privileged and can change the + * refcount on kernel-internal objects - make sure + * that the message wasn't faked! + */ if (0 != bcmp(&trailer->msgh_audit, &KERNEL_AUDIT_TOKEN, sizeof(trailer->msgh_audit))) { return FALSE; @@ -673,8 +678,13 @@ ipc_kobject_notify( case IKOT_FILEPORT: fileport_notify(request_header); return TRUE; + + case IKOT_WORK_INTERVAL: + work_interval_port_notify(request_header); + return TRUE; + } - break; + break; case MACH_NOTIFY_PORT_DELETED: case MACH_NOTIFY_PORT_DESTROYED: diff --git a/osfmk/kern/ipc_kobject.h b/osfmk/kern/ipc_kobject.h index daff8a51b..74c4768c0 100644 --- a/osfmk/kern/ipc_kobject.h +++ b/osfmk/kern/ipc_kobject.h @@ -126,11 +126,13 @@ typedef natural_t ipc_kobject_type_t; #define IKOT_TASK_RESUME 36 #define IKOT_VOUCHER 37 #define IKOT_VOUCHER_ATTR_CONTROL 38 +#define IKOT_WORK_INTERVAL 39 + /* * Add new entries here and adjust IKOT_UNKNOWN. * Please keep ipc/ipc_object.c:ikot_print_array up to date. */ -#define IKOT_UNKNOWN 39 /* magic catchall */ +#define IKOT_UNKNOWN 40 /* magic catchall */ #define IKOT_MAX_TYPE (IKOT_UNKNOWN+1) /* # of IKOT_ types */ diff --git a/osfmk/kern/ipc_mig.c b/osfmk/kern/ipc_mig.c index 3530f97ce..8114708a1 100644 --- a/osfmk/kern/ipc_mig.c +++ b/osfmk/kern/ipc_mig.c @@ -533,7 +533,9 @@ mach_msg_overwrite( mach_msg_size_t msg_and_trailer_size; mach_msg_max_trailer_t *max_trailer; - if ((send_size < sizeof(mach_msg_header_t)) || (send_size & 3)) + if ((send_size & 3) || + send_size < sizeof(mach_msg_header_t) || + (send_size < sizeof(mach_msg_body_t) && (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX))) return MACH_SEND_MSG_TOO_SMALL; if (send_size > MACH_MSG_SIZE_MAX - MAX_TRAILER_SIZE) @@ -771,7 +773,7 @@ mig_strncpy_zerofill( return retval; } -char * +void * mig_user_allocate( vm_size_t size) { diff --git a/osfmk/kern/ipc_tt.c b/osfmk/kern/ipc_tt.c index 3cfb2718b..920ac8fc5 100644 --- a/osfmk/kern/ipc_tt.c +++ b/osfmk/kern/ipc_tt.c @@ -98,10 +98,16 @@ #include +#if CONFIG_EMBEDDED && !SECURE_KERNEL +extern int cs_relax_platform_task_ports; +#endif + /* forward declarations */ task_t convert_port_to_locked_task(ipc_port_t port); task_inspect_t convert_port_to_locked_task_inspect(ipc_port_t port); - +static void ipc_port_bind_special_reply_port_locked(ipc_port_t port); +static kern_return_t ipc_port_unbind_special_reply_port(thread_t thread, boolean_t unbind_active_port); +kern_return_t task_conversion_eval(task_t caller, task_t victim); /* * Routine: ipc_task_init @@ -156,14 +162,17 @@ ipc_task_init( task->itk_debug_control = IP_NULL; task->itk_space = space; +#if CONFIG_MACF + for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; i++) { + mac_exc_associate_action_label(&task->exc_actions[i], mac_exc_create_label()); + } +#endif + if (parent == TASK_NULL) { ipc_port_t port; for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; i++) { task->exc_actions[i].port = IP_NULL; -#if CONFIG_MACF - mac_exc_action_label_init(task->exc_actions + i); -#endif }/* for */ kr = host_get_host_port(host_priv_self(), &port); @@ -199,7 +208,7 @@ ipc_task_init( task->exc_actions[i].privileged = parent->exc_actions[i].privileged; #if CONFIG_MACF - mac_exc_action_label_inherit(parent->exc_actions + i, task->exc_actions + i); + mac_exc_inherit_action_label(parent->exc_actions + i, task->exc_actions + i); #endif }/* for */ task->itk_host = @@ -336,7 +345,7 @@ ipc_task_terminate( ipc_port_release_send(task->exc_actions[i].port); } #if CONFIG_MACF - mac_exc_action_label_destroy(task->exc_actions + i); + mac_exc_free_action_label(task->exc_actions + i); #endif } @@ -392,6 +401,11 @@ ipc_task_reset( ipc_port_t old_exc_actions[EXC_TYPES_COUNT]; int i; +#if CONFIG_MACF + /* Fresh label to unset credentials in existing labels. */ + struct label *unset_label = mac_exc_create_label(); +#endif + new_kport = ipc_port_alloc_kernel(); if (new_kport == IP_NULL) panic("ipc_task_reset"); @@ -404,6 +418,9 @@ ipc_task_reset( /* the task is already terminated (can this happen?) */ itk_unlock(task); ipc_port_dealloc_kernel(new_kport); +#if CONFIG_MACF + mac_exc_free_label(unset_label); +#endif return; } @@ -428,7 +445,7 @@ ipc_task_reset( if (!task->exc_actions[i].privileged) { #if CONFIG_MACF - mac_exc_action_label_reset(task->exc_actions + i); + mac_exc_update_action_label(task->exc_actions + i, unset_label); #endif old_exc_actions[i] = task->exc_actions[i].port; task->exc_actions[i].port = IP_NULL; @@ -442,6 +459,10 @@ ipc_task_reset( itk_unlock(task); +#if CONFIG_MACF + mac_exc_free_label(unset_label); +#endif + /* release the naked send rights */ if (IP_VALID(old_sself)) @@ -477,6 +498,7 @@ ipc_thread_init( thread->ith_self = kport; thread->ith_sself = ipc_port_make_send(kport); + thread->ith_special_reply_port = NULL; thread->exc_actions = NULL; ipc_kobject_set(kport, (ipc_kobject_t)thread, IKOT_THREAD); @@ -501,7 +523,7 @@ ipc_thread_init_exc_actions( #if CONFIG_MACF for (size_t i = 0; i < EXC_TYPES_COUNT; ++i) { - mac_exc_action_label_init(thread->exc_actions + i); + mac_exc_associate_action_label(thread->exc_actions + i, mac_exc_create_label()); } #endif } @@ -513,7 +535,7 @@ ipc_thread_destroy_exc_actions( if (thread->exc_actions != NULL) { #if CONFIG_MACF for (size_t i = 0; i < EXC_TYPES_COUNT; ++i) { - mac_exc_action_label_destroy(thread->exc_actions + i); + mac_exc_free_action_label(thread->exc_actions + i); } #endif @@ -570,6 +592,11 @@ ipc_thread_terminate( assert(thread->ith_assertions == 0); #endif + /* unbind the thread special reply port */ + if (IP_VALID(thread->ith_special_reply_port)) { + ipc_port_unbind_special_reply_port(thread, TRUE); + } + assert(ipc_kmsg_queue_empty(&thread->ith_messages)); if (thread->ith_rpc_reply != IP_NULL) @@ -600,6 +627,10 @@ ipc_thread_reset( boolean_t has_old_exc_actions = FALSE; int i; +#if CONFIG_MACF + struct label *new_label = mac_exc_create_label(); +#endif + new_kport = ipc_port_alloc_kernel(); if (new_kport == IP_NULL) panic("ipc_task_reset"); @@ -612,6 +643,9 @@ ipc_thread_reset( /* the is already terminated (can this happen?) */ thread_mtx_unlock(thread); ipc_port_dealloc_kernel(new_kport); +#if CONFIG_MACF + mac_exc_free_label(new_label); +#endif return; } @@ -634,7 +668,7 @@ ipc_thread_reset( old_exc_actions[i] = IP_NULL; } else { #if CONFIG_MACF - mac_exc_action_label_reset(thread->exc_actions + i); + mac_exc_update_action_label(thread->exc_actions + i, new_label); #endif old_exc_actions[i] = thread->exc_actions[i].port; thread->exc_actions[i].port = IP_NULL; @@ -644,6 +678,10 @@ ipc_thread_reset( thread_mtx_unlock(thread); +#if CONFIG_MACF + mac_exc_free_label(new_label); +#endif + /* release the naked send rights */ if (IP_VALID(old_sself)) @@ -659,6 +697,11 @@ ipc_thread_reset( if (old_kport != IP_NULL) { ipc_port_dealloc_kernel(old_kport); } + + /* unbind the thread special reply port */ + if (IP_VALID(thread->ith_special_reply_port)) { + ipc_port_unbind_special_reply_port(thread, TRUE); + } } /* @@ -817,6 +860,101 @@ mach_reply_port( return name; } +/* + * Routine: thread_get_special_reply_port [mach trap] + * Purpose: + * Allocate a special reply port for the calling thread. + * Conditions: + * Nothing locked. + * Returns: + * MACH_PORT_NULL if there are any resource failures + * or other errors. + */ + +mach_port_name_t +thread_get_special_reply_port( + __unused struct thread_get_special_reply_port_args *args) +{ + ipc_port_t port; + mach_port_name_t name; + kern_return_t kr; + thread_t thread = current_thread(); + + /* unbind the thread special reply port */ + if (IP_VALID(thread->ith_special_reply_port)) { + kr = ipc_port_unbind_special_reply_port(thread, TRUE); + if (kr != KERN_SUCCESS) { + return MACH_PORT_NULL; + } + } + + kr = ipc_port_alloc(current_task()->itk_space, &name, &port); + if (kr == KERN_SUCCESS) { + ipc_port_bind_special_reply_port_locked(port); + ip_unlock(port); + } else { + name = MACH_PORT_NULL; + } + return name; +} + +/* + * Routine: ipc_port_bind_special_reply_port_locked + * Purpose: + * Bind the given port to current thread as a special reply port. + * Conditions: + * Port locked. + * Returns: + * None. + */ + +static void +ipc_port_bind_special_reply_port_locked( + ipc_port_t port) +{ + thread_t thread = current_thread(); + assert(thread->ith_special_reply_port == NULL); + + ip_reference(port); + thread->ith_special_reply_port = port; + port->ip_specialreply = 1; + port->ip_link_sync_qos = 1; +} + +/* + * Routine: ipc_port_unbind_special_reply_port + * Purpose: + * Unbind the thread's special reply port. + * If the special port is linked to a port, adjust it's sync qos delta`. + * Condition: + * Nothing locked. + * Returns: + * None. + */ +static kern_return_t +ipc_port_unbind_special_reply_port( + thread_t thread, + boolean_t unbind_active_port) +{ + ipc_port_t special_reply_port = thread->ith_special_reply_port; + + ip_lock(special_reply_port); + + /* Return error if port active and unbind_active_port set to FALSE */ + if (unbind_active_port == FALSE && ip_active(special_reply_port)) { + ip_unlock(special_reply_port); + return KERN_FAILURE; + } + + thread->ith_special_reply_port = NULL; + ipc_port_unlink_special_reply_port_locked(special_reply_port, NULL, + IPC_PORT_UNLINK_SR_CLEAR_SPECIAL_REPLY); + /* port unlocked */ + + ip_release(special_reply_port); + return KERN_SUCCESS; +} + /* * Routine: thread_get_special_port [kernel call] * Purpose: @@ -1208,6 +1346,50 @@ mach_ports_lookup( return KERN_SUCCESS; } +kern_return_t +task_conversion_eval(task_t caller, task_t victim) +{ + /* + * Tasks are allowed to resolve their own task ports, and the kernel is + * allowed to resolve anyone's task port. + */ + if (caller == kernel_task) { + return KERN_SUCCESS; + } + + if (caller == victim) { + return KERN_SUCCESS; + } + + /* + * Only the kernel can can resolve the kernel's task port. We've established + * by this point that the caller is not kernel_task. + */ + if (victim == kernel_task) { + return KERN_INVALID_SECURITY; + } + +#if CONFIG_EMBEDDED + /* + * On embedded platforms, only a platform binary can resolve the task port + * of another platform binary. + */ + if ((victim->t_flags & TF_PLATFORM) && !(caller->t_flags & TF_PLATFORM)) { +#if SECURE_KERNEL + return KERN_INVALID_SECURITY; +#else + if (cs_relax_platform_task_ports) { + return KERN_SUCCESS; + } else { + return KERN_INVALID_SECURITY; + } +#endif /* SECURE_KERNEL */ + } +#endif /* CONFIG_EMBEDDED */ + + return KERN_SUCCESS; +} + /* * Routine: convert_port_to_locked_task * Purpose: @@ -1220,9 +1402,10 @@ mach_ports_lookup( task_t convert_port_to_locked_task(ipc_port_t port) { - int try_failed_count = 0; + int try_failed_count = 0; while (IP_VALID(port)) { + task_t ct = current_task(); task_t task; ip_lock(port); @@ -1233,7 +1416,7 @@ convert_port_to_locked_task(ipc_port_t port) task = (task_t) port->ip_kobject; assert(task != TASK_NULL); - if (task == kernel_task && current_task() != kernel_task) { + if (task_conversion_eval(ct, task)) { ip_unlock(port); return TASK_NULL; } @@ -1333,10 +1516,11 @@ convert_port_to_task_with_exec_token( if ( ip_active(port) && ip_kotype(port) == IKOT_TASK ) { + task_t ct = current_task(); task = (task_t)port->ip_kobject; assert(task != TASK_NULL); - if (task == kernel_task && current_task() != kernel_task) { + if (task_conversion_eval(ct, task)) { ip_unlock(port); return TASK_NULL; } @@ -1952,6 +2136,10 @@ thread_set_exception_ports( boolean_t privileged = current_task()->sec_token.val[0] == 0; register int i; +#if CONFIG_MACF + struct label *new_label; +#endif + if (thread == THREAD_NULL) return (KERN_INVALID_ARGUMENT); @@ -1979,6 +2167,10 @@ thread_set_exception_ports( if (new_flavor != 0 && !VALID_THREAD_STATE_FLAVOR(new_flavor)) return (KERN_INVALID_ARGUMENT); +#if CONFIG_MACF + new_label = mac_exc_create_label_for_current_proc(); +#endif + thread_mtx_lock(thread); if (!thread->active) { @@ -1993,7 +2185,7 @@ thread_set_exception_ports( for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; ++i) { if ((exception_mask & (1 << i)) #if CONFIG_MACF - && mac_exc_action_label_update(current_task(), thread->exc_actions + i) == 0 + && mac_exc_update_action_label(&thread->exc_actions[i], new_label) == 0 #endif ) { old_port[i] = thread->exc_actions[i].port; @@ -2008,6 +2200,10 @@ thread_set_exception_ports( thread_mtx_unlock(thread); +#if CONFIG_MACF + mac_exc_free_label(new_label); +#endif + for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; ++i) if (IP_VALID(old_port[i])) ipc_port_release_send(old_port[i]); @@ -2030,6 +2226,10 @@ task_set_exception_ports( boolean_t privileged = current_task()->sec_token.val[0] == 0; register int i; +#if CONFIG_MACF + struct label *new_label; +#endif + if (task == TASK_NULL) return (KERN_INVALID_ARGUMENT); @@ -2057,6 +2257,10 @@ task_set_exception_ports( if (new_flavor != 0 && !VALID_THREAD_STATE_FLAVOR(new_flavor)) return (KERN_INVALID_ARGUMENT); +#if CONFIG_MACF + new_label = mac_exc_create_label_for_current_proc(); +#endif + itk_lock(task); if (task->itk_self == IP_NULL) { @@ -2068,7 +2272,7 @@ task_set_exception_ports( for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; ++i) { if ((exception_mask & (1 << i)) #if CONFIG_MACF - && mac_exc_action_label_update(current_task(), task->exc_actions + i) == 0 + && mac_exc_update_action_label(&task->exc_actions[i], new_label) == 0 #endif ) { old_port[i] = task->exc_actions[i].port; @@ -2084,6 +2288,10 @@ task_set_exception_ports( itk_unlock(task); +#if CONFIG_MACF + mac_exc_free_label(new_label); +#endif + for (i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT; ++i) if (IP_VALID(old_port[i])) ipc_port_release_send(old_port[i]); @@ -2138,6 +2346,10 @@ thread_swap_exception_ports( boolean_t privileged = current_task()->sec_token.val[0] == 0; unsigned int i, j, count; +#if CONFIG_MACF + struct label *new_label; +#endif + if (thread == THREAD_NULL) return (KERN_INVALID_ARGUMENT); @@ -2160,6 +2372,10 @@ thread_swap_exception_ports( if (new_flavor != 0 && !VALID_THREAD_STATE_FLAVOR(new_flavor)) return (KERN_INVALID_ARGUMENT); +#if CONFIG_MACF + new_label = mac_exc_create_label_for_current_proc(); +#endif + thread_mtx_lock(thread); if (!thread->active) { @@ -2176,7 +2392,7 @@ thread_swap_exception_ports( for (count = 0, i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT && count < *CountCnt; ++i) { if ((exception_mask & (1 << i)) #if CONFIG_MACF - && mac_exc_action_label_update(current_task(), thread->exc_actions + i) == 0 + && mac_exc_update_action_label(&thread->exc_actions[i], new_label) == 0 #endif ) { for (j = 0; j < count; ++j) { @@ -2213,6 +2429,10 @@ thread_swap_exception_ports( thread_mtx_unlock(thread); +#if CONFIG_MACF + mac_exc_free_label(new_label); +#endif + while (--i >= FIRST_EXCEPTION) { if (IP_VALID(old_port[i])) ipc_port_release_send(old_port[i]); @@ -2243,6 +2463,10 @@ task_swap_exception_ports( boolean_t privileged = current_task()->sec_token.val[0] == 0; unsigned int i, j, count; +#if CONFIG_MACF + struct label *new_label; +#endif + if (task == TASK_NULL) return (KERN_INVALID_ARGUMENT); @@ -2265,6 +2489,10 @@ task_swap_exception_ports( if (new_flavor != 0 && !VALID_THREAD_STATE_FLAVOR(new_flavor)) return (KERN_INVALID_ARGUMENT); +#if CONFIG_MACF + new_label = mac_exc_create_label_for_current_proc(); +#endif + itk_lock(task); if (task->itk_self == IP_NULL) { @@ -2277,7 +2505,7 @@ task_swap_exception_ports( for (count = 0, i = FIRST_EXCEPTION; i < EXC_TYPES_COUNT && count < *CountCnt; ++i) { if ((exception_mask & (1 << i)) #if CONFIG_MACF - && mac_exc_action_label_update(current_task(), task->exc_actions + i) == 0 + && mac_exc_update_action_label(&task->exc_actions[i], new_label) == 0 #endif ) { for (j = 0; j < count; j++) { @@ -2314,6 +2542,10 @@ task_swap_exception_ports( itk_unlock(task); +#if CONFIG_MACF + mac_exc_free_label(new_label); +#endif + while (--i >= FIRST_EXCEPTION) { if (IP_VALID(old_port[i])) ipc_port_release_send(old_port[i]); diff --git a/osfmk/kern/kalloc.c b/osfmk/kern/kalloc.c index ac6d89b50..97b04c739 100644 --- a/osfmk/kern/kalloc.c +++ b/osfmk/kern/kalloc.c @@ -79,6 +79,8 @@ #include #include +#include + #ifdef MACH_BSD zone_t kalloc_zone(vm_size_t); #endif @@ -300,6 +302,7 @@ kalloc_init( vm_offset_t min; vm_size_t size, kalloc_map_size; int i; + vm_map_kernel_flags_t vmk_flags; /* * Scale the kalloc_map_size to physical memory size: stay below @@ -313,8 +316,14 @@ kalloc_init( if (kalloc_map_size < KALLOC_MAP_SIZE_MIN) kalloc_map_size = KALLOC_MAP_SIZE_MIN; + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_permanent = TRUE; + retval = kmem_suballoc(kernel_map, &min, kalloc_map_size, - FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT | VM_MAKE_TAG(0), + FALSE, + (VM_FLAGS_ANYWHERE), + vmk_flags, + VM_KERN_MEMORY_KALLOC, &kalloc_map); if (retval != KERN_SUCCESS) @@ -348,6 +357,10 @@ kalloc_init( for (i = 0; i < (int)MAX_K_ZONE && (size = k_zone_size[i]) < kalloc_max; i++) { k_zone[i] = zinit(size, size, size, k_zone_name[i]); zone_change(k_zone[i], Z_CALLERACCT, FALSE); +#if VM_MAX_TAG_ZONES + if (zone_tagging_on) zone_change(k_zone[i], Z_TAGS_ENABLED, TRUE); +#endif + zone_change(k_zone[i], Z_KASAN_QUARANTINE, FALSE); } /* @@ -466,6 +479,18 @@ vm_map_lookup_kalloc_entry_locked( return (vm_entry->vme_end - vm_entry->vme_start); } +#if KASAN_KALLOC +/* + * KASAN kalloc stashes the original user-requested size away in the poisoned + * area. Return that directly. + */ +vm_size_t +kalloc_size(void *addr) +{ + (void)vm_map_lookup_kalloc_entry_locked; /* silence warning */ + return kasan_user_size((vm_offset_t)addr); +} +#else vm_size_t kalloc_size( void *addr) @@ -487,6 +512,7 @@ kalloc_size( vm_map_unlock_read(map); return size; } +#endif vm_size_t kalloc_bucket_size( @@ -513,6 +539,15 @@ kalloc_bucket_size( return vm_map_round_page(size, VM_MAP_PAGE_MASK(map)); } +#if KASAN_KALLOC +vm_size_t +kfree_addr(void *addr) +{ + vm_size_t origsz = kalloc_size(addr); + kfree(addr, origsz); + return origsz; +} +#else vm_size_t kfree_addr( void *addr) @@ -559,7 +594,8 @@ kfree_addr( KALLOC_ZINFO_SFREE(size); return size; } - +#endif + void * kalloc_canblock( vm_size_t * psize, @@ -568,9 +604,18 @@ kalloc_canblock( { zone_t z; vm_size_t size; + void *addr; + vm_tag_t tag; + tag = VM_KERN_MEMORY_KALLOC; size = *psize; +#if KASAN_KALLOC + /* expand the allocation to accomodate redzones */ + vm_size_t req_size = size; + size = kasan_alloc_resize(req_size); +#endif + if (size < MAX_SIZE_ZDLUT) z = get_zone_dlut(size); else if (size < kalloc_max_prerounded) @@ -582,20 +627,24 @@ kalloc_canblock( * krealloc can use kmem_realloc.) */ vm_map_t alloc_map; - void *addr; /* kmem_alloc could block so we return if noblock */ if (!canblock) { return(NULL); } +#if KASAN_KALLOC + /* large allocation - use guard pages instead of small redzones */ + size = round_page(req_size + 2 * PAGE_SIZE); + assert(size >= MAX_SIZE_ZDLUT && size >= kalloc_max_prerounded); +#endif + if (size >= kalloc_kernmap_size) alloc_map = kernel_map; else alloc_map = kalloc_map; - vm_tag_t tag; - tag = (site ? tag = vm_tag_alloc(site) : VM_KERN_MEMORY_KALLOC); + if (site) tag = vm_tag_alloc(site); if (kmem_alloc_flags(alloc_map, (vm_offset_t *)&addr, size, tag, KMA_ATOMIC) != KERN_SUCCESS) { if (alloc_map != kernel_map) { @@ -629,7 +678,12 @@ kalloc_canblock( KALLOC_ZINFO_SALLOC(size); } +#if KASAN_KALLOC + /* fixup the return address to skip the redzone */ + addr = (void *)kasan_alloc((vm_offset_t)addr, size, req_size, PAGE_SIZE); +#else *psize = round_page(size); +#endif return(addr); } #ifdef KALLOC_DEBUG @@ -637,9 +691,29 @@ kalloc_canblock( panic("%s: z %p (%s) but requested size %lu", __func__, z, z->zone_name, (unsigned long)size); #endif + assert(size <= z->elem_size); + +#if VM_MAX_TAG_ZONES + if (z->tags && site) + { + tag = vm_tag_alloc(site); + if (!canblock && !vm_allocation_zone_totals[tag]) tag = VM_KERN_MEMORY_KALLOC; + } +#endif + + addr = zalloc_canblock_tag(z, canblock, size, tag); + +#if KASAN_KALLOC + /* fixup the return address to skip the redzone */ + addr = (void *)kasan_alloc((vm_offset_t)addr, z->elem_size, req_size, KASAN_GUARD_SIZE); + + /* For KASan, the redzone lives in any additional space, so don't + * expand the allocation. */ +#else *psize = z->elem_size; - void *addr = zalloc_canblock(z, canblock); +#endif + return addr; } @@ -662,6 +736,20 @@ kfree( { zone_t z; +#if KASAN_KALLOC + /* + * Resize back to the real allocation size and hand off to the KASan + * quarantine. `data` may then point to a different allocation. + */ + vm_size_t user_size = size; + kasan_check_free((vm_address_t)data, size, KASAN_HEAP_KALLOC); + data = (void *)kasan_dealloc((vm_address_t)data, &size); + kasan_free(&data, &size, KASAN_HEAP_KALLOC, NULL, user_size, true); + if (!data) { + return; + } +#endif + if (size < MAX_SIZE_ZDLUT) z = get_zone_dlut(size); else if (size < kalloc_max_prerounded) @@ -732,35 +820,6 @@ kalloc_zone( } #endif -void -kalloc_fake_zone_init(int zone_index) -{ - kalloc_fake_zone_index = zone_index; -} - -void -kalloc_fake_zone_info(int *count, - vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size, - uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct) -{ - *count = kalloc_large_inuse; - *cur_size = kalloc_large_total; - *max_size = kalloc_large_max; - - if (kalloc_large_inuse) { - *elem_size = kalloc_large_total / kalloc_large_inuse; - *alloc_size = kalloc_large_total / kalloc_large_inuse; - } else { - *elem_size = 0; - *alloc_size = 0; - } - *sum_size = kalloc_large_sum; - *collectable = 0; - *exhaustable = 0; - *caller_acct = 0; -} - - void OSMalloc_init( void) diff --git a/osfmk/kern/kalloc.h b/osfmk/kern/kalloc.h index 5a12d09ee..3c6f5804b 100644 --- a/osfmk/kern/kalloc.h +++ b/osfmk/kern/kalloc.h @@ -86,64 +86,57 @@ extern vm_size_t kalloc_bucket_size( vm_size_t size); -#define kalloc(size) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ - vm_size_t tsize = (size); \ +#define kalloc(size) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ + vm_size_t tsize = (size); \ kalloc_canblock(&tsize, TRUE, &site); }) -#define kalloc_tag(size, tag) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))) \ - = { (tag), 0 }; \ - vm_size_t tsize = (size); \ +#define kalloc_tag(size, itag) \ + ({ VM_ALLOC_SITE_STATIC(0, (itag)); \ + vm_size_t tsize = (size); \ kalloc_canblock(&tsize, TRUE, &site); }) -#define kalloc_tag_bt(size, tag) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))) \ - = { (tag), VM_TAG_BT }; \ - vm_size_t tsize = (size); \ +#define kalloc_tag_bt(size, itag) \ + ({ VM_ALLOC_SITE_STATIC(VM_TAG_BT, (itag)); \ + vm_size_t tsize = (size); \ kalloc_canblock(&tsize, TRUE, &site); }) -#define kalloc_noblock(size) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ - vm_size_t tsize = (size); \ +#define kalloc_noblock(size) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ + vm_size_t tsize = (size); \ kalloc_canblock(&tsize, FALSE, &site); }) -#define kalloc_noblock_tag(size, tag) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))) \ - = { (tag), 0 }; \ - vm_size_t tsize = (size); \ +#define kalloc_noblock_tag(size, itag) \ + ({ VM_ALLOC_SITE_STATIC(0, (itag)); \ + vm_size_t tsize = (size); \ kalloc_canblock(&tsize, FALSE, &site); }) -#define kalloc_noblock_tag_bt(size, tag) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))) \ - = { (tag), VM_TAG_BT }; \ - vm_size_t tsize = (size); \ +#define kalloc_noblock_tag_bt(size, itag) \ + ({ VM_ALLOC_SITE_STATIC(VM_TAG_BT, (itag)); \ + vm_size_t tsize = (size); \ kalloc_canblock(&tsize, FALSE, &site); }) /* these versions update the size reference with the actual size allocated */ -#define kallocp(size) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ +#define kallocp(size) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ kalloc_canblock((size), TRUE, &site); }) -#define kallocp_tag(size, tag) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))) \ - = { (tag), 0 }; \ +#define kallocp_tag(size, itag) \ + ({ VM_ALLOC_SITE_STATIC(0, (itag)); \ kalloc_canblock((size), TRUE, &site); }) -#define kallocp_tag_bt(size, tag) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))) \ - = { (tag), VM_TAG_BT }; \ +#define kallocp_tag_bt(size, itag) \ + ({ VM_ALLOC_SITE_STATIC(VM_TAG_BT, (itag)); \ kalloc_canblock((size), TRUE, &site); }) -#define kallocp_noblock(size) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ +#define kallocp_noblock(size) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ kalloc_canblock((size), FALSE, &site); }) -#define kallocp_noblock_tag_bt(size, tag) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))) \ - = { (tag), VM_TAG_BT }; \ +#define kallocp_noblock_tag_bt(size, itag) \ + ({ VM_ALLOC_SITE_STATIC(VM_TAG_BT, (itag)); \ kalloc_canblock((size), FALSE, &site); }) @@ -153,9 +146,9 @@ extern void kfree(void *data, #else /* XNU_KERNEL_PRIVATE */ -extern void *kalloc(vm_size_t size); +extern void *kalloc(vm_size_t size) __attribute__((alloc_size(1))); -extern void *kalloc_noblock(vm_size_t size); +extern void *kalloc_noblock(vm_size_t size) __attribute__((alloc_size(1))); extern void kfree(void *data, vm_size_t size); diff --git a/osfmk/kern/kcdata.h b/osfmk/kern/kcdata.h index 9ed131e8c..741c7f864 100644 --- a/osfmk/kern/kcdata.h +++ b/osfmk/kern/kcdata.h @@ -469,6 +469,13 @@ struct kcdata_type_definition { #define STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS 0x91bu /* struct stackshot_fault_stats */ #define STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO 0x91cu /* kernelcache UUID -- same as KCDATA_TYPE_LIBRARY_LOADINFO64 */ #define STACKSHOT_KCTYPE_THREAD_WAITINFO 0x91du /* struct stackshot_thread_waitinfo */ +#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot */ +#define STACKSHOT_KCTYPE_THREAD_GROUP 0x91fu /* uint64_t */ +#define STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT 0x920u /* struct jetsam_coalition_snapshot */ +#define STACKSHOT_KCTYPE_JETSAM_COALITION 0x921u /* uint64_t */ +#define STACKSHOT_KCTYPE_INSTRS_CYCLES 0x923u /* struct instrs_cycles_snapshot */ + +#define STACKSHOT_KCTYPE_THREAD_POLICY_VERSION 0x922u /* THREAD_POLICY_INTERNAL_STRUCT_VERSION in uint32 */ struct stack_snapshot_frame32 { uint32_t lr; @@ -544,6 +551,7 @@ enum thread_snapshot_flags { kThreadTriedFaultBT = 0x400, /* We tried to fault in thread stack pages as part of BT */ kThreadOnCore = 0x800, /* Thread was on-core when we entered debugger context */ kThreadIdleWorker = 0x1000, /* Thread is an idle libpthread worker thread */ + kThreadMain = 0x2000, /* Thread is the main thread */ }; struct mem_and_io_snapshot { @@ -619,6 +627,57 @@ struct thread_snapshot_v3 { uint64_t ths_thread_t; } __attribute__((packed)); + +struct thread_snapshot_v4 { + uint64_t ths_thread_id; + uint64_t ths_wait_event; + uint64_t ths_continuation; + uint64_t ths_total_syscalls; + uint64_t ths_voucher_identifier; + uint64_t ths_dqserialnum; + uint64_t ths_user_time; + uint64_t ths_sys_time; + uint64_t ths_ss_flags; + uint64_t ths_last_run_time; + uint64_t ths_last_made_runnable_time; + uint32_t ths_state; + uint32_t ths_sched_flags; + int16_t ths_base_priority; + int16_t ths_sched_priority; + uint8_t ths_eqos; + uint8_t ths_rqos; + uint8_t ths_rqos_override; + uint8_t ths_io_tier; + uint64_t ths_thread_t; + uint64_t ths_requested_policy; + uint64_t ths_effective_policy; +} __attribute__((packed)); + + +struct thread_group_snapshot { + uint64_t tgs_id; + char tgs_name[16]; +} __attribute__((packed)); + +enum coalition_flags { + kCoalitionTermRequested = 0x1, + kCoalitionTerminated = 0x2, + kCoalitionReaped = 0x4, + kCoalitionPrivileged = 0x8, +}; + +struct jetsam_coalition_snapshot { + uint64_t jcs_id; + uint64_t jcs_flags; + uint64_t jcs_thread_group; + uint64_t jcs_leader_task_uniqueid; +} __attribute__((packed)); + +struct instrs_cycles_snapshot { + uint64_t ics_instructions; + uint64_t ics_cycles; +} __attribute__((packed)); + struct thread_delta_snapshot_v2 { uint64_t tds_thread_id; uint64_t tds_voucher_identifier; @@ -722,6 +781,8 @@ typedef struct stackshot_thread_waitinfo { #define STACKSHOT_WAITOWNER_PSET_LOCKED (UINT64_MAX - 3) #define STACKSHOT_WAITOWNER_INTRANSIT (UINT64_MAX - 4) #define STACKSHOT_WAITOWNER_MTXSPIN (UINT64_MAX - 5) +#define STACKSHOT_WAITOWNER_THREQUESTED (UINT64_MAX - 6) /* workloop waiting for a new worker thread */ +#define STACKSHOT_WAITOWNER_SUSPENDED (UINT64_MAX - 7) /* workloop is suspended */ /**************** definitions for crashinfo *********************/ @@ -776,6 +837,8 @@ typedef struct stackshot_thread_waitinfo { #define EXIT_REASON_USER_DESC 0x1002 /* string description of reason */ #define EXIT_REASON_USER_PAYLOAD 0x1003 /* user payload data */ #define EXIT_REASON_CODESIGNING_INFO 0x1004 +#define EXIT_REASON_WORKLOOP_ID 0x1005 +#define EXIT_REASON_DISPATCH_QUEUE_NO 0x1006 struct exit_reason_snapshot { uint32_t ers_namespace; diff --git a/osfmk/kern/kern_cdata.c b/osfmk/kern/kern_cdata.c index 46499f452..6070cdc26 100644 --- a/osfmk/kern/kern_cdata.c +++ b/osfmk/kern/kern_cdata.c @@ -100,6 +100,15 @@ kern_return_t kcdata_memory_static_init(kcdata_descriptor_t data, mach_vm_addres return kcdata_get_memory_addr(data, data_type, 0, &user_addr); } +void *kcdata_memory_get_begin_addr(kcdata_descriptor_t data) +{ + if (data == NULL) { + return NULL; + } + + return (void *)data->kcd_addr_begin; +} + uint64_t kcdata_memory_get_used_bytes(kcdata_descriptor_t kcd) { assert(kcd != NULL); @@ -343,7 +352,7 @@ kern_return_t kcdata_add_type_definition( uint32_t total_size = sizeof(struct kcdata_type_definition); bzero(&kc_type_definition, sizeof(kc_type_definition)); - if (strnlen(type_name, KCDATA_DESC_MAXLEN + 1) >= KCDATA_DESC_MAXLEN) + if (strlen(type_name) >= KCDATA_DESC_MAXLEN) return KERN_INVALID_ARGUMENT; strlcpy(&kc_type_definition.kct_name[0], type_name, KCDATA_DESC_MAXLEN); kc_type_definition.kct_num_elements = elements_count; @@ -380,7 +389,7 @@ struct _uint32_with_description_data { kern_return_t kcdata_add_uint64_with_description(kcdata_descriptor_t data_desc, uint64_t data, const char * description) { - if (strnlen(description, KCDATA_DESC_MAXLEN + 1) >= KCDATA_DESC_MAXLEN) + if (strlen(description) >= KCDATA_DESC_MAXLEN) return KERN_INVALID_ARGUMENT; kern_return_t kr = 0; @@ -411,7 +420,7 @@ kern_return_t kcdata_add_uint32_with_description( const char *description) { assert(strlen(description) < KCDATA_DESC_MAXLEN); - if (strnlen(description, KCDATA_DESC_MAXLEN + 1) >= KCDATA_DESC_MAXLEN) + if (strlen(description) >= KCDATA_DESC_MAXLEN) return KERN_INVALID_ARGUMENT; kern_return_t kr = 0; mach_vm_address_t user_addr; diff --git a/osfmk/kern/kern_cdata.h b/osfmk/kern/kern_cdata.h index fd7543342..ce49bf679 100644 --- a/osfmk/kern/kern_cdata.h +++ b/osfmk/kern/kern_cdata.h @@ -62,10 +62,11 @@ /* Structure to save information about corpse data */ struct kcdata_descriptor { uint32_t kcd_length; - uint32_t kcd_flags; + uint16_t kcd_flags; #define KCFLAG_USE_MEMCOPY 0x0 #define KCFLAG_USE_COPYOUT 0x1 #define KCFLAG_NO_AUTO_ENDBUFFER 0x2 + uint16_t kcd_user_flags; /* reserved for subsystems using kcdata */ mach_vm_address_t kcd_addr_begin; mach_vm_address_t kcd_addr_end; }; @@ -90,6 +91,7 @@ kern_return_t kcdata_add_uint32_with_description(kcdata_descriptor_t crashinfo, kern_return_t kcdata_undo_add_container_begin(kcdata_descriptor_t data); kern_return_t kcdata_write_buffer_end(kcdata_descriptor_t data); +void *kcdata_memory_get_begin_addr(kcdata_descriptor_t data); #else /* XNU_KERNEL_PRIVATE */ diff --git a/osfmk/kern/kern_monotonic.c b/osfmk/kern/kern_monotonic.c new file mode 100644 index 000000000..92bacff03 --- /dev/null +++ b/osfmk/kern/kern_monotonic.c @@ -0,0 +1,523 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +bool mt_debug = false; +_Atomic uint64_t mt_pmis = 0; +_Atomic uint64_t mt_retrograde = 0; + +#define MT_KDBG_INSTRS_CYCLES(CODE) \ + KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_INSTRS_CYCLES, CODE) + +#define MT_KDBG_IC_CPU_CSWITCH MT_KDBG_INSTRS_CYCLES(1) + +/* + * Updating the thread counters takes place in the context switch path, so it + * cannot introduce too much overhead. Thus, updating takes no locks, instead + * updating a generation count to an odd value to indicate that it's in the + * critical section and that readers should wait until the generation count + * returns to an even value. + * + * Reading the counters also needs to not see any "torn" states of the counters, + * where a few of the counters are from a previous state and the rest are from + * the current state. For this reason, the reader redrives the entire read + * operation if it sees mismatching generation counts at the beginning and end + * of reading. + */ + +#define MAXSPINS 100 +#define MAXRETRIES 10 + +int +mt_fixed_thread_counts(thread_t thread, uint64_t *counts_out) +{ + uint64_t start_gen, end_gen; + uint64_t spins = 0, retries = 0; + uint64_t counts[MT_CORE_NFIXED]; + + /* + * Try to read a thread's counter values by ensuring its gen count is + * even. If it's odd, it means that a thread is trying to update its + * counters. + * + * Spin until the gen count is even. + */ +spin: + start_gen = atomic_load_explicit(&thread->t_monotonic.mth_gen, + memory_order_acquire); +retry: + if (start_gen & 1) { + spins++; + if (spins > MAXSPINS) { + return EBUSY; + } + goto spin; + } + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + counts[i] = thread->t_monotonic.mth_counts[i]; + } + + /* + * After reading the counters, check the gen count again. If it is + * different from the value that we started with, the thread raced + * writing its counters with us reading them. We need to redrive the + * entire operation. + * + * Go back to check if the value we just read was even and try to read + * again. + */ + end_gen = atomic_load_explicit(&thread->t_monotonic.mth_gen, + memory_order_acquire); + if (end_gen != start_gen) { + retries++; + if (retries > MAXRETRIES) { + return EAGAIN; + } + start_gen = end_gen; + goto retry; + } + + /* + * Only after getting a consistent snapshot of the counters should we + * write them into the provided buffer. + */ + for (int i = 0; i < MT_CORE_NFIXED; i++) { + counts_out[i] = counts[i]; + } + return 0; +} + +static void mt_fixed_counts_internal(uint64_t *counts, uint64_t *counts_since); + +bool +mt_update_thread(thread_t thread) +{ + if (!mt_core_supported) { + return false; + } + + assert(ml_get_interrupts_enabled() == FALSE); + + uint64_t counts[MT_CORE_NFIXED], counts_since[MT_CORE_NFIXED]; + mt_fixed_counts_internal(counts, counts_since); + + /* + * Enter the update cycle by incrementing the gen count to be odd -- + * this tells any readers to spin on the gen count, waiting for it to go + * even. + */ + __assert_only uint64_t enter_gen = atomic_fetch_add_explicit( + &thread->t_monotonic.mth_gen, 1, memory_order_release); + /* + * Should not have pre-empted a modification to the counts. + */ + assert((enter_gen & 1) == 0); + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + thread->t_monotonic.mth_counts[i] += counts_since[i]; + } + + /* + * Exit the update by making the gen count even again. Readers check + * the gen count for equality, and will redrive the reads if the values + * before and after reading don't match. + */ + __assert_only uint64_t exit_gen = atomic_fetch_add_explicit( + &thread->t_monotonic.mth_gen, 1, memory_order_release); + /* + * Make sure no other writers came through behind us. + */ + assert(exit_gen == (enter_gen + 1)); + + return true; +} + +void +mt_sched_update(thread_t thread) +{ + bool updated = mt_update_thread(thread); + if (!updated) { + return; + } + + if (kdebug_debugid_explicitly_enabled(MT_KDBG_IC_CPU_CSWITCH)) { + struct mt_cpu *mtc = mt_cur_cpu(); + + KDBG_RELEASE(MT_KDBG_IC_CPU_CSWITCH, +#ifdef MT_CORE_INSTRS + mtc->mtc_counts[MT_CORE_INSTRS], +#else /* defined(MT_CORE_INSTRS) */ + 0, +#endif /* !defined(MT_CORE_INSTRS) */ + mtc->mtc_counts[MT_CORE_CYCLES]); + } +} + +int +mt_fixed_task_counts(task_t task, uint64_t *counts_out) +{ + assert(task != TASK_NULL); + assert(counts_out != NULL); + + uint64_t counts[MT_CORE_NFIXED]; + if (!mt_core_supported) { + for (int i = 0; i < MT_CORE_NFIXED; i++) { + counts[i] = 0; + } + return 0; + } + + task_lock(task); + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + counts[i] = task->task_monotonic.mtk_counts[i]; + } + + uint64_t thread_counts[MT_CORE_NFIXED] = {}; + thread_t thread = THREAD_NULL; + thread_t curthread = current_thread(); + bool needs_current = false; + int r = 0; + queue_iterate(&task->threads, thread, thread_t, task_threads) { + /* + * Get the current thread's counters after doing this + * processing, without holding the task lock. + */ + if (thread == curthread) { + needs_current = true; + continue; + } else { + r = mt_fixed_thread_counts(thread, thread_counts); + if (r) { + goto error; + } + } + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + counts[i] += thread_counts[i]; + } + } + + task_unlock(task); + + if (needs_current) { + mt_cur_thread_fixed_counts(thread_counts); + } + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + if (needs_current) { + counts[i] += thread_counts[i]; + } + counts_out[i] = counts[i]; + } + return 0; + +error: + task_unlock(task); + return r; +} + +uint64_t +mt_mtc_update_count(struct mt_cpu *mtc, unsigned int ctr) +{ + uint64_t snap = mt_core_snap(ctr); + if (snap < mtc->mtc_snaps[ctr]) { + if (mt_debug) { + kprintf("monotonic: cpu %d: thread %#llx: " + "retrograde counter %u value: %llu, last read = %llu\n", + cpu_number(), thread_tid(current_thread()), ctr, snap, + mtc->mtc_snaps[ctr]); + } + (void)atomic_fetch_add_explicit(&mt_retrograde, 1, + memory_order_relaxed); + mtc->mtc_snaps[ctr] = snap; + return 0; + } + + uint64_t count = snap - mtc->mtc_snaps[ctr]; + mtc->mtc_snaps[ctr] = snap; + + return count; +} + +uint64_t +mt_cpu_update_count(cpu_data_t *cpu, unsigned int ctr) +{ + return mt_mtc_update_count(&cpu->cpu_monotonic, ctr); +} + +static void +mt_fixed_counts_internal(uint64_t *counts, uint64_t *counts_since) +{ + assert(ml_get_interrupts_enabled() == FALSE); + + struct mt_cpu *mtc = mt_cur_cpu(); + assert(mtc != NULL); + + mt_mtc_update_fixed_counts(mtc, counts, counts_since); +} + +void +mt_mtc_update_fixed_counts(struct mt_cpu *mtc, uint64_t *counts, + uint64_t *counts_since) +{ + if (!mt_core_supported) { + return; + } + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + uint64_t last_delta; + uint64_t count; + + last_delta = mt_mtc_update_count(mtc, i); + count = mtc->mtc_counts[i] + last_delta; + + if (counts) { + counts[i] = count; + } + if (counts_since) { + assert(counts != NULL); + counts_since[i] = count - mtc->mtc_counts_last[i]; + mtc->mtc_counts_last[i] = count; + } + + mtc->mtc_counts[i] = count; + } +} + +void +mt_update_fixed_counts(void) +{ + assert(ml_get_interrupts_enabled() == FALSE); + +#if defined(__x86_64__) + __builtin_ia32_lfence(); +#elif defined(__arm__) || defined(__arm64__) + __builtin_arm_isb(ISB_SY); +#endif /* !defined(__x86_64__) && (defined(__arm__) || defined(__arm64__)) */ + + mt_fixed_counts_internal(NULL, NULL); +} + +void +mt_fixed_counts(uint64_t *counts) +{ +#if defined(__x86_64__) + __builtin_ia32_lfence(); +#elif defined(__arm__) || defined(__arm64__) + __builtin_arm_isb(ISB_SY); +#endif /* !defined(__x86_64__) && (defined(__arm__) || defined(__arm64__)) */ + + int intrs_en = ml_set_interrupts_enabled(FALSE); + mt_fixed_counts_internal(counts, NULL); + ml_set_interrupts_enabled(intrs_en); +} + +void +mt_cur_thread_fixed_counts(uint64_t *counts) +{ + if (!mt_core_supported) { + for (int i = 0; i < MT_CORE_NFIXED; i++) { + counts[i] = 0; + } + return; + } + + thread_t curthread = current_thread(); + int intrs_en = ml_set_interrupts_enabled(FALSE); + (void)mt_update_thread(curthread); + for (int i = 0; i < MT_CORE_NFIXED; i++) { + counts[i] = curthread->t_monotonic.mth_counts[i]; + } + ml_set_interrupts_enabled(intrs_en); +} + +void +mt_cur_task_fixed_counts(uint64_t *counts) +{ + task_t curtask = current_task(); + + mt_fixed_task_counts(curtask, counts); +} + +/* FIXME these should only update the counter that is being accessed */ + +uint64_t +mt_cur_thread_instrs(void) +{ +#ifdef MT_CORE_INSTRS + thread_t curthread = current_thread(); + boolean_t intrs_en; + uint64_t count; + + if (!mt_core_supported) { + return 0; + } + + intrs_en = ml_set_interrupts_enabled(FALSE); + (void)mt_update_thread(curthread); + count = curthread->t_monotonic.mth_counts[MT_CORE_INSTRS]; + ml_set_interrupts_enabled(intrs_en); + + return count; +#else /* defined(MT_CORE_INSTRS) */ + return 0; +#endif /* !defined(MT_CORE_INSTRS) */ +} + +uint64_t +mt_cur_thread_cycles(void) +{ + thread_t curthread = current_thread(); + boolean_t intrs_en; + uint64_t count; + + if (!mt_core_supported) { + return 0; + } + + intrs_en = ml_set_interrupts_enabled(FALSE); + (void)mt_update_thread(curthread); + count = curthread->t_monotonic.mth_counts[MT_CORE_CYCLES]; + ml_set_interrupts_enabled(intrs_en); + + return count; +} + +uint64_t +mt_cur_cpu_instrs(void) +{ +#ifdef MT_CORE_INSTRS + uint64_t counts[MT_CORE_NFIXED]; + + if (!mt_core_supported) { + return 0; + } + + mt_fixed_counts(counts); + return counts[MT_CORE_INSTRS]; +#else /* defined(MT_CORE_INSTRS) */ + return 0; +#endif /* !defined(MT_CORE_INSTRS) */ +} + +uint64_t +mt_cur_cpu_cycles(void) +{ + uint64_t counts[MT_CORE_NFIXED]; + + if (!mt_core_supported) { + return 0; + } + + mt_fixed_counts(counts); + return counts[MT_CORE_CYCLES]; +} + +void +mt_update_task(task_t task, thread_t thread) +{ + task_lock_assert_owned(task); + + if (!mt_core_supported) { + return; + } + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + task->task_monotonic.mtk_counts[i] += thread->t_monotonic.mth_counts[i]; + } +} + +void +mt_terminate_update(task_t task, thread_t thread) +{ + mt_update_task(task, thread); +} + +void +mt_perfcontrol(uint64_t *instrs, uint64_t *cycles) +{ + if (!mt_core_supported) { + *instrs = 0; + *cycles = 0; + return; + } + + struct mt_cpu *mtc = mt_cur_cpu(); + + /* + * The performance controller queries the hardware directly, so provide the + * last snapshot we took for the core. This is the value from when we + * updated the thread counts. + */ + +#ifdef MT_CORE_INSTRS + *instrs = mtc->mtc_snaps[MT_CORE_INSTRS]; +#else /* defined(MT_CORE_INSTRS) */ + *instrs = 0; +#endif /* !defined(MT_CORE_INSTRS) */ + + *cycles = mtc->mtc_snaps[MT_CORE_CYCLES]; +} + +void +mt_stackshot_thread(thread_t thread, uint64_t *instrs, uint64_t *cycles) +{ + assert(mt_core_supported); + +#ifdef MT_CORE_INSTRS + *instrs = thread->t_monotonic.mth_counts[MT_CORE_INSTRS]; +#else /* defined(MT_CORE_INSTRS) */ + *instrs = 0; +#endif /* !defined(MT_CORE_INSTRS) */ + + *cycles = thread->t_monotonic.mth_counts[MT_CORE_CYCLES]; +} + +void +mt_stackshot_task(task_t task, uint64_t *instrs, uint64_t *cycles) +{ + assert(mt_core_supported); + +#ifdef MT_CORE_INSTRS + *instrs = task->task_monotonic.mtk_counts[MT_CORE_INSTRS]; +#else /* defined(MT_CORE_INSTRS) */ + *instrs = 0; +#endif /* !defined(MT_CORE_INSTRS) */ + + *cycles = task->task_monotonic.mtk_counts[MT_CORE_CYCLES]; +} diff --git a/osfmk/kern/kern_stackshot.c b/osfmk/kern/kern_stackshot.c index 41bdf2962..6ac0d3665 100644 --- a/osfmk/kern/kern_stackshot.c +++ b/osfmk/kern/kern_stackshot.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2013 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -48,8 +48,10 @@ #include /* bcopy */ +#include #include #include +#include #include #include #include @@ -61,12 +63,21 @@ #include #include -#if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS) +#if CONFIG_EMBEDDED #include /* For gPanicBase/gPanicBase */ #endif +#if MONOTONIC +#include +#endif /* MONOTONIC */ + +#include + extern unsigned int not_in_kdp; +#if CONFIG_EMBEDDED +uuid_t kernelcache_uuid; +#endif /* indicate to the compiler that some accesses are unaligned */ typedef uint64_t unaligned_u64 __attribute__((aligned(1))); @@ -105,11 +116,10 @@ kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size uint32_t get_stackshot_estsize(uint32_t prev_size_hint); kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user); -kern_return_t do_stackshot(void *); -void kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags, kcdata_descriptor_t data_p, uint64_t since_timestamp); +kern_return_t do_stackshot(void *); +void kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags, kcdata_descriptor_t data_p, uint64_t since_timestamp); boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread); static int kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t *pBytesTraced); -kern_return_t kdp_stack_snapshot_geterror(void); uint32_t kdp_stack_snapshot_bytes_traced(void); static void kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap); static boolean_t kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_result); @@ -118,6 +128,12 @@ static uint64_t proc_was_throttled_from_task(task_t task); static void stackshot_thread_wait_owner_info(thread_t thread, thread_waitinfo_t * waitinfo); static int stackshot_thread_has_valid_waitinfo(thread_t thread); +#if CONFIG_COALITIONS +static void stackshot_coalition_jetsam_count(void *arg, int i, coalition_t coal); +static void stackshot_coalition_jetsam_snapshot(void *arg, int i, coalition_t coal); +#endif /* CONFIG_COALITIONS */ + + extern uint32_t workqueue_get_pwq_state_kdp(void *proc); extern int proc_pid(void *p); @@ -140,15 +156,6 @@ extern kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebu extern kern_return_t kern_stack_snapshot_with_reason(char* reason); extern kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user); -/* Used for stackshot_thread_waitinfo_unsafe */ -extern void kdp_lck_mtx_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); -extern void kdp_sema_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); -extern void kdp_mqueue_send_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); -extern void kdp_mqueue_recv_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); -extern void kdp_ulock_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); -extern void kdp_rwlck_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); -extern void kdp_pthread_find_owner(thread_t thread, thread_waitinfo_t *waitinfo); -extern void *kdp_pthread_get_thread_kwq(thread_t thread); /* * Validates that the given address is both a valid page and has @@ -261,12 +268,29 @@ static uint64_t safe_grab_timer_value(struct timer *t) #endif } +/* + * Called with interrupts disabled after stackshot context has been + * initialized. Updates stack_snapshot_ret. + */ +static kern_return_t +stackshot_trap() +{ + return DebuggerTrapWithState(DBOP_STACKSHOT, NULL, NULL, NULL, 0, FALSE, 0); +} + + kern_return_t stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, uint64_t delta_since_timestamp, unsigned *bytes_traced) { kern_return_t error = KERN_SUCCESS; boolean_t istate; +#if DEVELOPMENT || DEBUG + if (kern_feature_override(KF_STACKSHOT_OVRD) == TRUE) { + error = KERN_NOT_SUPPORTED; + goto out; + } +#endif if ((buf == NULL) || (size <= 0) || (bytes_traced == NULL)) { return KERN_INVALID_ARGUMENT; } @@ -285,8 +309,6 @@ stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, ui STACKSHOT_SUBSYS_LOCK(); } - istate = ml_set_interrupts_enabled(FALSE); - struct kcdata_descriptor kcdata; uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ? KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT; @@ -297,21 +319,23 @@ stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, ui goto out; } + istate = ml_set_interrupts_enabled(FALSE); + /* Preload trace parameters*/ kdp_snapshot_preflight(pid, buf, size, flags, &kcdata, delta_since_timestamp); - /* Trap to the debugger to obtain a coherent stack snapshot; this populates + /* + * Trap to the debugger to obtain a coherent stack snapshot; this populates * the trace buffer */ - stack_snapshot_ret = DebuggerWithCallback(do_stackshot, NULL, FALSE); + error = stackshot_trap(); ml_set_interrupts_enabled(istate); *bytes_traced = kdp_stack_snapshot_bytes_traced(); - error = kdp_stack_snapshot_geterror(); - out: + stackshot_kcdata_p = NULL; STACKSHOT_SUBSYS_UNLOCK(); return error; } @@ -425,8 +449,8 @@ stackshot_remap_buffer(void *stackshotbuf, uint32_t bytes_traced, uint64_t out_b mach_vm_offset_t stackshotbuf_user_addr = (mach_vm_offset_t)NULL; vm_prot_t cur_prot, max_prot; - error = mach_vm_remap(get_task_map(current_task()), &stackshotbuf_user_addr, bytes_traced, 0, - VM_FLAGS_ANYWHERE, kernel_map, (mach_vm_offset_t)stackshotbuf, FALSE, &cur_prot, &max_prot, VM_INHERIT_DEFAULT); + error = mach_vm_remap_kernel(get_task_map(current_task()), &stackshotbuf_user_addr, bytes_traced, 0, + VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_NONE, kernel_map, (mach_vm_offset_t)stackshotbuf, FALSE, &cur_prot, &max_prot, VM_INHERIT_DEFAULT); /* * If the call to mach_vm_remap fails, we return the appropriate converted error */ @@ -473,6 +497,14 @@ kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_confi if(stackshot_config == NULL) { return KERN_INVALID_ARGUMENT; } +#if DEVELOPMENT || DEBUG + /* TBD: ask stackshot clients to avoid issuing stackshots in this + * configuration in lieu of the kernel feature override. + */ + if (kern_feature_override(KF_STACKSHOT_OVRD) == TRUE) { + return KERN_NOT_SUPPORTED; + } +#endif switch (stackshot_config_version) { case STACKSHOT_CONFIG_TYPE: @@ -522,6 +554,14 @@ kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_confi return KERN_INVALID_ARGUMENT; } +#if MONOTONIC + if (!mt_core_supported) { + flags &= ~STACKSHOT_INSTRS_CYCLES; + } +#else /* MONOTONIC */ + flags &= ~STACKSHOT_INSTRS_CYCLES; +#endif /* !MONOTONIC */ + STACKSHOT_SUBSYS_LOCK(); if (flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER) { @@ -576,13 +616,6 @@ kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_confi goto error_exit; } - /* - * If someone has panicked, don't try and enter the debugger - */ - if (panic_active()) { - error = KERN_RESOURCE_SHORTAGE; - goto error_exit; - } uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ? KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT; kcdata_p = kcdata_memory_alloc_init((mach_vm_address_t)stackshotbuf, hdr_tag, stackshotbuf_size, @@ -601,10 +634,7 @@ kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_confi */ kdp_snapshot_preflight(pid, stackshotbuf, stackshotbuf_size, flags, kcdata_p, since_timestamp); - /* - * Trap to the debugger to obtain a stackshot (this will populate the buffer). - */ - stack_snapshot_ret = DebuggerWithCallback(do_stackshot, NULL, FALSE); + error = stackshot_trap(); ml_set_interrupts_enabled(prev_interrupt_state); @@ -615,7 +645,6 @@ kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_confi *stackshot_duration_outer = time_end - time_start; } - error = kdp_stack_snapshot_geterror(); if (error != KERN_SUCCESS) { if (kcdata_p != NULL) { kcdata_memory_destroy(kcdata_p); @@ -691,7 +720,9 @@ error_exit: return error; } -/* Cache stack snapshot parameters in preparation for a trace */ +/* + * Cache stack snapshot parameters in preparation for a trace. + */ void kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags, kcdata_descriptor_t data_p, uint64_t since_timestamp) @@ -708,15 +739,17 @@ kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_ panic_stackshot = ((flags & STACKSHOT_FROM_PANIC) != 0); - if (data_p != NULL) { - stackshot_kcdata_p = data_p; - } + assert(data_p != NULL); + assert(stackshot_kcdata_p == NULL); + stackshot_kcdata_p = data_p; + + stack_snapshot_bytes_traced = 0; } -kern_return_t -kdp_stack_snapshot_geterror(void) +boolean_t +stackshot_active() { - return stack_snapshot_ret; + return (stackshot_kcdata_p != NULL); } uint32_t @@ -793,16 +826,14 @@ kcdata_get_task_ss_flags(task_t task) } static kern_return_t -kcdata_record_shared_cache_info(kcdata_descriptor_t kcd, task_t task, struct dyld_uuid_info_64_v2 *sys_shared_cache_loadinfo, uint32_t trace_flags, unaligned_u64 *task_snap_ss_flags) +kcdata_record_shared_cache_info(kcdata_descriptor_t kcd, task_t task, struct dyld_uuid_info_64_v2 *sys_shared_cache_loadinfo, unaligned_u64 *task_snap_ss_flags) { kern_return_t error = KERN_SUCCESS; mach_vm_address_t out_addr = 0; - uint8_t shared_cache_identifier[16]; uint64_t shared_cache_slide = 0; uint64_t shared_cache_base_address = 0; int task_pid = pid_from_task(task); - boolean_t should_fault = (trace_flags & STACKSHOT_ENABLE_UUID_FAULTING); uint32_t kdp_fault_results = 0; assert(task_snap_ss_flags != NULL); @@ -812,36 +843,32 @@ kcdata_record_shared_cache_info(kcdata_descriptor_t kcd, task_t task, struct dyl shared_cache_base_address = sr->sr_base_address + sr->sr_first_mapping; } else { *task_snap_ss_flags |= kTaskSharedRegionInfoUnavailable; + goto error_exit; } - if (!shared_cache_base_address || - !kdp_copyin(task->map, shared_cache_base_address + offsetof(struct _dyld_cache_header, uuid), - shared_cache_identifier, sizeof(shared_cache_identifier), should_fault, &kdp_fault_results)) { + /* We haven't copied in the shared region UUID yet as part of setup */ + if (!shared_cache_base_address || !task->shared_region->sr_uuid_copied) { goto error_exit; } - if (task->shared_region) { - /* - * No refcounting here, but we are in debugger - * context, so that should be safe. - */ - shared_cache_slide = task->shared_region->sr_slide_info.slide; - } else { - shared_cache_slide = 0; - } + /* + * No refcounting here, but we are in debugger + * context, so that should be safe. + */ + shared_cache_slide = task->shared_region->sr_slide_info.slide; if (sys_shared_cache_loadinfo) { if (task_pid == 1) { /* save launchd's shared cache info as system level */ - stackshot_memcpy(sys_shared_cache_loadinfo->imageUUID, shared_cache_identifier, sizeof(sys_shared_cache_loadinfo->imageUUID)); + stackshot_memcpy(sys_shared_cache_loadinfo->imageUUID, &task->shared_region->sr_uuid, sizeof(task->shared_region->sr_uuid)); sys_shared_cache_loadinfo->imageLoadAddress = shared_cache_slide; sys_shared_cache_loadinfo->imageSlidBaseAddress = shared_cache_slide + task->shared_region->sr_base_address; goto error_exit; } else { if (shared_cache_slide == sys_shared_cache_loadinfo->imageLoadAddress && - 0 == memcmp(shared_cache_identifier, sys_shared_cache_loadinfo->imageUUID, - sizeof(sys_shared_cache_loadinfo->imageUUID))) { + 0 == memcmp(&task->shared_region->sr_uuid, sys_shared_cache_loadinfo->imageUUID, + sizeof(task->shared_region->sr_uuid))) { /* skip adding shared cache info. its same as system level one */ goto error_exit; } @@ -851,7 +878,7 @@ kcdata_record_shared_cache_info(kcdata_descriptor_t kcd, task_t task, struct dyl kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO, sizeof(struct dyld_uuid_info_64_v2), &out_addr)); struct dyld_uuid_info_64_v2 *shared_cache_data = (struct dyld_uuid_info_64_v2 *)out_addr; shared_cache_data->imageLoadAddress = shared_cache_slide; - stackshot_memcpy(shared_cache_data->imageUUID, shared_cache_identifier, sizeof(shared_cache_data->imageUUID)); + stackshot_memcpy(shared_cache_data->imageUUID, task->shared_region->sr_uuid, sizeof(task->shared_region->sr_uuid)); shared_cache_data->imageSlidBaseAddress = shared_cache_base_address; error_exit: @@ -900,7 +927,7 @@ kcdata_record_uuid_info(kcdata_descriptor_t kcd, task_t task, uint32_t trace_fla sizeof(struct user64_dyld_all_image_infos), should_fault, &kdp_fault_results)) { uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount; uuid_info_addr = task_image_infos.uuidArray; - if (task_image_infos.version >= 15) { + if (task_image_infos.version >= DYLD_ALL_IMAGE_INFOS_TIMESTAMP_MINIMUM_VERSION) { uuid_info_timestamp = task_image_infos.timestamp; } } @@ -910,7 +937,7 @@ kcdata_record_uuid_info(kcdata_descriptor_t kcd, task_t task, uint32_t trace_fla sizeof(struct user32_dyld_all_image_infos), should_fault, &kdp_fault_results)) { uuid_info_count = task_image_infos.uuidArrayCount; uuid_info_addr = task_image_infos.uuidArray; - if (task_image_infos.version >= 15) { + if (task_image_infos.version >= DYLD_ALL_IMAGE_INFOS_TIMESTAMP_MINIMUM_VERSION) { uuid_info_timestamp = task_image_infos.timestamp; } } @@ -959,6 +986,15 @@ kcdata_record_uuid_info(kcdata_descriptor_t kcd, task_t task, uint32_t trace_fla do { +#if CONFIG_EMBEDDED + if (!save_kextloadinfo_p) { + kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO, sizeof(struct dyld_uuid_info_64), &out_addr)); + struct dyld_uuid_info_64 *kc_uuid = (struct dyld_uuid_info_64 *)out_addr; + kc_uuid->imageLoadAddress = VM_MIN_KERNEL_AND_KEXT_ADDRESS; + stackshot_memcpy(&kc_uuid->imageUUID, &kernelcache_uuid, sizeof(uuid_t)); + break; + } +#endif /* CONFIG_EMBEDDED */ if (!kernel_uuid || !ml_validate_nofault((vm_offset_t)kernel_uuid, sizeof(uuid_t))) { /* Kernel UUID not found or inaccessible */ @@ -1042,6 +1078,9 @@ kcdata_record_task_snapshot(kcdata_descriptor_t kcd, task_t task, uint32_t trace { boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0); boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_TAILSPIN) && !(trace_flags & STACKSHOT_NO_IO_STATS); +#if MONOTONIC + boolean_t collect_instrs_cycles = ((trace_flags & STACKSHOT_INSTRS_CYCLES) != 0); +#endif /* MONOTONIC */ kern_return_t error = KERN_SUCCESS; mach_vm_address_t out_addr = 0; @@ -1066,7 +1105,11 @@ kcdata_record_task_snapshot(kcdata_descriptor_t kcd, task_t task, uint32_t trace proc_starttime_kdp(task->bsd_info, &proc_starttime_secs, NULL, NULL); cur_tsnap->ts_p_start_sec = proc_starttime_secs; +#if CONFIG_EMBEDDED + cur_tsnap->ts_task_size = have_pmap ? get_task_phys_footprint(task) : 0; +#else cur_tsnap->ts_task_size = have_pmap ? (pmap_resident_count(task->map->pmap) * PAGE_SIZE) : 0; +#endif cur_tsnap->ts_max_resident_size = get_task_resident_max(task); cur_tsnap->ts_suspend_count = task->suspend_count; cur_tsnap->ts_faults = task->faults; @@ -1079,8 +1122,16 @@ kcdata_record_task_snapshot(kcdata_descriptor_t kcd, task_t task, uint32_t trace cur_tsnap->ts_pid = task_pid; /* Add the BSD process identifiers */ - if (task_pid != -1 && task->bsd_info != NULL) + if (task_pid != -1 && task->bsd_info != NULL) { proc_name_kdp(task, cur_tsnap->ts_p_comm, sizeof(cur_tsnap->ts_p_comm)); +#if CONFIG_COALITIONS + if (trace_flags & STACKSHOT_SAVE_JETSAM_COALITIONS) { + uint64_t jetsam_coal_id = coalition_id(task->coalition[COALITION_TYPE_JETSAM]); + kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_JETSAM_COALITION, sizeof(jetsam_coal_id), &out_addr)); + stackshot_memcpy((void*)out_addr, &jetsam_coal_id, sizeof(jetsam_coal_id)); + } +#endif /* CONFIG_COALITIONS */ + } else { cur_tsnap->ts_p_comm[0] = '\0'; #if IMPORTANCE_INHERITANCE && (DEVELOPMENT || DEBUG) @@ -1088,13 +1139,25 @@ kcdata_record_task_snapshot(kcdata_descriptor_t kcd, task_t task, uint32_t trace stackshot_strlcpy(cur_tsnap->ts_p_comm, &task->task_imp_base->iit_procname[0], MIN((int)sizeof(task->task_imp_base->iit_procname), (int)sizeof(cur_tsnap->ts_p_comm))); } -#endif +#endif /* IMPORTANCE_INHERITANCE && (DEVELOPMENT || DEBUG) */ } if (collect_iostats) { kcd_exit_on_error(kcdata_record_task_iostats(kcd, task)); } +#if MONOTONIC + if (collect_instrs_cycles) { + uint64_t instrs = 0, cycles = 0; + mt_stackshot_task(task, &instrs, &cycles); + + kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_INSTRS_CYCLES, sizeof(struct instrs_cycles_snapshot), &out_addr)); + struct instrs_cycles_snapshot *instrs_cycles = (struct instrs_cycles_snapshot *)out_addr; + instrs_cycles->ics_instructions = instrs; + instrs_cycles->ics_cycles = cycles; + } +#endif /* MONOTONIC */ + error_exit: return error; } @@ -1120,7 +1183,11 @@ kcdata_record_task_delta_snapshot(kcdata_descriptor_t kcd, task_t task, boolean_ cur_tsnap->tds_user_time_in_terminated_threads = task->total_user_time; cur_tsnap->tds_system_time_in_terminated_threads = task->total_system_time; +#if CONFIG_EMBEDDED + cur_tsnap->tds_task_size = have_pmap ? get_task_phys_footprint(task) : 0; +#else cur_tsnap->tds_task_size = have_pmap ? (pmap_resident_count(task->map->pmap) * PAGE_SIZE) : 0; +#endif cur_tsnap->tds_max_resident_size = get_task_resident_max(task); cur_tsnap->tds_suspend_count = task->suspend_count; @@ -1179,18 +1246,21 @@ kcdata_record_thread_snapshot( boolean_t trace_fp_p = false; boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0); boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_TAILSPIN) && !(trace_flags & STACKSHOT_NO_IO_STATS); +#if MONOTONIC + boolean_t collect_instrs_cycles = ((trace_flags & STACKSHOT_INSTRS_CYCLES) != 0); +#endif /* MONOTONIC */ kern_return_t error = KERN_SUCCESS; mach_vm_address_t out_addr = 0; int saved_count = 0; - struct thread_snapshot_v3 * cur_thread_snap = NULL; + struct thread_snapshot_v4 * cur_thread_snap = NULL; char cur_thread_name[STACKSHOT_MAX_THREAD_NAME_SIZE]; uint64_t tval = 0; boolean_t task64 = task_has_64BitAddr(task); - kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_SNAPSHOT, sizeof(struct thread_snapshot_v3), &out_addr)); - cur_thread_snap = (struct thread_snapshot_v3 *)out_addr; + kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_SNAPSHOT, sizeof(struct thread_snapshot_v4), &out_addr)); + cur_thread_snap = (struct thread_snapshot_v4 *)out_addr; /* Populate the thread snapshot header */ cur_thread_snap->ths_thread_id = thread_tid(thread); @@ -1233,6 +1303,8 @@ kcdata_record_thread_snapshot( } cur_thread_snap->ths_ss_flags = 0; + if (thread->thread_tag & THREAD_TAG_MAINTHREAD) + cur_thread_snap->ths_ss_flags |= kThreadMain; if (thread->effective_policy.thep_darwinbg) cur_thread_snap->ths_ss_flags |= kThreadDarwinBG; if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO)) @@ -1265,7 +1337,12 @@ kcdata_record_thread_snapshot( cur_thread_snap->ths_rqos = thread->requested_policy.thrp_qos; cur_thread_snap->ths_rqos_override = thread->requested_policy.thrp_qos_override; cur_thread_snap->ths_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO); - cur_thread_snap->ths_thread_t = VM_KERNEL_ADDRPERM(thread); + cur_thread_snap->ths_thread_t = VM_KERNEL_UNSLIDE_OR_PERM(thread); + + static_assert(sizeof(thread->effective_policy) == sizeof(uint64_t)); + static_assert(sizeof(thread->requested_policy) == sizeof(uint64_t)); + cur_thread_snap->ths_requested_policy = *(unaligned_u64 *) &thread->requested_policy; + cur_thread_snap->ths_effective_policy = *(unaligned_u64 *) &thread->effective_policy; /* if there is thread name then add to buffer */ cur_thread_name[0] = '\0'; @@ -1348,10 +1425,23 @@ kcdata_record_thread_snapshot( } } + if (collect_iostats) { kcd_exit_on_error(kcdata_record_thread_iostats(kcd, thread)); } +#if MONOTONIC + if (collect_instrs_cycles) { + uint64_t instrs = 0, cycles = 0; + mt_stackshot_thread(thread, &instrs, &cycles); + + kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_INSTRS_CYCLES, sizeof(struct instrs_cycles_snapshot), &out_addr)); + struct instrs_cycles_snapshot *instrs_cycles = (struct instrs_cycles_snapshot *)out_addr; + instrs_cycles->ics_instructions = instrs; + instrs_cycles->ics_cycles = cycles; + } +#endif /* MONOTONIC */ + error_exit: return error; } @@ -1480,10 +1570,6 @@ kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTrac abs_time = mach_absolute_time(); -#if !(DEVELOPMENT || DEBUG) - trace_flags &= ~STACKSHOT_THREAD_WAITINFO; -#endif - /* process the flags */ boolean_t active_kthreads_only_p = ((trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0); boolean_t save_donating_pids_p = ((trace_flags & STACKSHOT_SAVE_IMP_DONATION_PIDS) != 0); @@ -1491,9 +1577,12 @@ kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTrac boolean_t minimize_nonrunnables = ((trace_flags & STACKSHOT_TAILSPIN) != 0); boolean_t use_fault_path = ((trace_flags & (STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_ENABLE_BT_FAULTING)) != 0); boolean_t save_owner_info = ((trace_flags & STACKSHOT_THREAD_WAITINFO) != 0); - stack_enable_faulting = (trace_flags & (STACKSHOT_ENABLE_BT_FAULTING)); +#if CONFIG_EMBEDDED + /* KEXTs can't be described by just a base address on embedded */ + trace_flags &= ~(STACKSHOT_SAVE_KEXT_LOADINFO); +#endif struct saved_uniqueids saved_uniqueids = {.count = 0}; @@ -1527,6 +1616,10 @@ kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTrac #endif if (!collect_delta_stackshot) { + tmp32 = THREAD_POLICY_INTERNAL_STRUCT_VERSION; + kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_THREAD_POLICY_VERSION, sizeof(uint32_t), &out_addr)); + stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32)); + tmp32 = PAGE_SIZE; kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_KERN_PAGE_SIZE, sizeof(uint32_t), &out_addr)); stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32)); @@ -1569,6 +1662,33 @@ kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTrac kdp_mem_and_io_snapshot((struct mem_and_io_snapshot *)out_addr); } +#if CONFIG_COALITIONS + int num_coalitions = 0; + struct jetsam_coalition_snapshot *coalitions = NULL; + /* Iterate over coalitions */ + if (trace_flags & STACKSHOT_SAVE_JETSAM_COALITIONS) { + if (coalition_iterate_stackshot(stackshot_coalition_jetsam_count, &num_coalitions, COALITION_TYPE_JETSAM) != KERN_SUCCESS) { + trace_flags &= ~(STACKSHOT_SAVE_JETSAM_COALITIONS); + } + } + if (trace_flags & STACKSHOT_SAVE_JETSAM_COALITIONS) { + if (num_coalitions > 0) { + kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT, sizeof(struct jetsam_coalition_snapshot), num_coalitions, &out_addr)); + coalitions = (struct jetsam_coalition_snapshot*)out_addr; + } + + if (coalition_iterate_stackshot(stackshot_coalition_jetsam_snapshot, coalitions, COALITION_TYPE_JETSAM) != KERN_SUCCESS) { + error = KERN_FAILURE; + goto error_exit; + } + + } +#else + trace_flags &= ~(STACKSHOT_SAVE_JETSAM_COALITIONS); +#endif /* CONFIG_COALITIONS */ + + trace_flags &= ~(STACKSHOT_THREAD_GROUP); + /* Iterate over tasks */ queue_head_t *task_list = &tasks; queue_iterate(task_list, task, task_t, tasks) { @@ -1577,6 +1697,7 @@ kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTrac int num_delta_thread_snapshots = 0; int num_nonrunnable_threads = 0; int num_waitinfo_threads = 0; + uint64_t task_start_abstime = 0; boolean_t task_delta_stackshot = FALSE; boolean_t task64 = FALSE, have_map = FALSE, have_pmap = FALSE; @@ -1803,7 +1924,7 @@ kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTrac * any thread from the task has run since the previous full stackshot */ - kcd_exit_on_error(kcdata_record_shared_cache_info(stackshot_kcdata_p, task, sys_shared_cache_loadinfo, trace_flags, task_snap_ss_flags)); + kcd_exit_on_error(kcdata_record_shared_cache_info(stackshot_kcdata_p, task, sys_shared_cache_loadinfo, task_snap_ss_flags)); kcd_exit_on_error(kcdata_record_uuid_info(stackshot_kcdata_p, task, trace_flags, have_pmap, task_snap_ss_flags)); } /* mark end of task snapshot data */ @@ -1834,6 +1955,7 @@ kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTrac #endif stackshot_memcpy((void *)abs_time_addr, &abs_time_end, sizeof(uint64_t)); + kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, trace_flags, "stackshot_out_flags")); kcd_exit_on_error(kcdata_write_buffer_end(stackshot_kcdata_p)); @@ -1842,6 +1964,16 @@ kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTrac *pBytesTraced = (uint32_t) kcdata_memory_get_used_bytes(stackshot_kcdata_p); error_exit: +#if INTERRUPT_MASKED_DEBUG + if (!panic_stackshot) { + /* + * Try to catch instances where stackshot takes too long BEFORE returning from + * the debugger + */ + ml_check_interrupts_disabled_duration(current_thread()); + } +#endif + stack_enable_faulting = FALSE; return error; @@ -1930,6 +2062,15 @@ kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap) void stackshot_memcpy(void *dst, const void *src, size_t len) { +#if CONFIG_EMBEDDED + if (panic_stackshot) { + uint8_t *dest_bytes = (uint8_t *)dst; + const uint8_t *src_bytes = (const uint8_t *)src; + for (size_t i = 0; i < len; i++) { + dest_bytes[i] = src_bytes[i]; + } + } else +#endif memcpy(dst, src, len); } @@ -2032,10 +2173,10 @@ kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_ size_t rem = size; char *kvaddr = dest; -#if (defined(__arm64__) || defined(NAND_PANIC_DEVICE)) && !defined(LEGACY_PANIC_LOGS) +#if CONFIG_EMBEDDED /* Identify if destination buffer is in panic storage area */ - if ((vm_offset_t)dest >= gPanicBase && (vm_offset_t)dest < gPanicBase + gPanicSize) { - if (((vm_offset_t)dest + size) >= (gPanicBase + gPanicSize)) { + if (panic_stackshot && ((vm_offset_t)dest >= gPanicBase) && ((vm_offset_t)dest < (gPanicBase + gPanicSize))) { + if (((vm_offset_t)dest + size) > (gPanicBase + gPanicSize)) { return FALSE; } } @@ -2050,6 +2191,15 @@ kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_ cur_size = MIN(cur_size, rem); if (phys_src && phys_dest) { +#if CONFIG_EMBEDDED + /* + * On embedded the panic buffer is mapped as device memory and doesn't allow + * unaligned accesses. To prevent these, we copy over bytes individually here. + */ + if (panic_stackshot) + stackshot_memcpy(kvaddr, (const void *)phystokv(phys_src), cur_size); + else +#endif /* CONFIG_EMBEDDED */ bcopy_phys(phys_src, phys_dest, cur_size); } else { break; @@ -2132,18 +2282,23 @@ machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t } #if __x86_64__ kern_virt_target_addr = (vm_offset_t) PHYSMAP_PTOV(cur_phys_addr); +#elif __arm__ || __arm64__ + kern_virt_target_addr = phystokv(cur_phys_addr); #else #error Oh come on... we should really unify the physical -> kernel virtual interface #endif prev_target_page = cur_target_page; prev_target_kva = (kern_virt_target_addr & ~PAGE_MASK); validate_next_addr = FALSE; - return kern_virt_target_addr; } else { /* We found a translation, so stash this page */ kern_virt_target_addr = prev_target_kva + (cur_target_addr & PAGE_MASK); - return kern_virt_target_addr; } + +#if KASAN + kasan_notify_address(kern_virt_target_addr, sizeof(uint64_t)); +#endif + return kern_virt_target_addr; } void @@ -2163,6 +2318,46 @@ stackshot_thread_is_idle_worker_unsafe(thread_t thread) (thread->block_hint == kThreadWaitParkedWorkQueue); } +#if CONFIG_COALITIONS +static void +stackshot_coalition_jetsam_count(void *arg, int i, coalition_t coal) +{ +#pragma unused(i, coal) + unsigned int *coalition_count = (unsigned int*)arg; + (*coalition_count)++; +} + +static void +stackshot_coalition_jetsam_snapshot(void *arg, int i, coalition_t coal) +{ + if (coalition_type(coal) != COALITION_TYPE_JETSAM) + return; + + struct jetsam_coalition_snapshot *coalitions = (struct jetsam_coalition_snapshot*)arg; + struct jetsam_coalition_snapshot *jcs = &coalitions[i]; + task_t leader = TASK_NULL; + jcs->jcs_id = coalition_id(coal); + jcs->jcs_flags = 0; + + if (coalition_term_requested(coal)) + jcs->jcs_flags |= kCoalitionTermRequested; + if (coalition_is_terminated(coal)) + jcs->jcs_flags |= kCoalitionTerminated; + if (coalition_is_reaped(coal)) + jcs->jcs_flags |= kCoalitionReaped; + if (coalition_is_privileged(coal)) + jcs->jcs_flags |= kCoalitionPrivileged; + + + leader = kdp_coalition_get_leader(coal); + if (leader) + jcs->jcs_leader_task_uniqueid = get_task_uniqueid(leader); + else + jcs->jcs_leader_task_uniqueid = 0; +} +#endif /* CONFIG_COALITIONS */ + + /* Determine if a thread has waitinfo that stackshot can provide */ static int stackshot_thread_has_valid_waitinfo(thread_t thread) @@ -2220,9 +2415,13 @@ stackshot_thread_wait_owner_info(thread_t thread, thread_waitinfo_t *waitinfo) case kThreadWaitPThreadCondVar: kdp_pthread_find_owner(thread, waitinfo); break; + case kThreadWaitWorkloopSyncWait: + kdp_workloop_sync_wait_find_owner(thread, thread->wait_event, waitinfo); + break; default: waitinfo->owner = 0; waitinfo->context = 0; break; } } + diff --git a/osfmk/kern/kern_types.h b/osfmk/kern/kern_types.h index c13795669..727712ec9 100644 --- a/osfmk/kern/kern_types.h +++ b/osfmk/kern/kern_types.h @@ -236,6 +236,66 @@ typedef struct _wait_queue_link *wait_queue_link_t; typedef struct perfcontrol_state *perfcontrol_state_t; #define PERFCONTROL_STATE_NULL ((perfcontrol_state_t)0) +/* + * Enum to define the event which caused the CLPC callout + */ +typedef enum perfcontrol_event { + /* + * Thread State Update Events + * Used to indicate events that update properties for + * a given thread. These events are passed as part of the + * sched_perfcontrol_state_update_t callout + */ + QUANTUM_EXPIRY = 1, + THREAD_GROUP_UPDATE = 2, + PERFCONTROL_ATTR_UPDATE = 3, + /* + * Context Switch Events + * Used to indicate events that switch from one thread + * to the other. These events are passed as part of the + * sched_perfcontrol_csw_t callout. + */ + CONTEXT_SWITCH = 10, + IDLE = 11 +} perfcontrol_event; + +/* + * Flags for the sched_perfcontrol_csw_t & sched_perfcontrol_state_update_t + * callouts. + * Currently defined flags are: + * PERFCONTROL_CALLOUT_WAKE_UNSAFE - Flag to indicate its unsafe to + * do a wakeup as part of this callout. If this is set, it + * indicates that the scheduler holds a spinlock which might be needed + * in the wakeup path. In that case CLPC should do a thread_call + * instead of a direct wakeup to run their workloop thread. + */ +#define PERFCONTROL_CALLOUT_WAKE_UNSAFE 0x1 + +/* + * Enum to define the perfcontrol class for thread. + * thread_get_perfcontrol_class() takes the thread's + * priority, QoS, urgency etc. into consideration and + * produces a value in this enum. + */ +typedef enum perfcontrol_class { + /* Idle thread */ + PERFCONTROL_CLASS_IDLE = 1, + /* Kernel thread */ + PERFCONTROL_CLASS_KERNEL = 2, + /* Realtime Thread */ + PERFCONTROL_CLASS_REALTIME = 3, + /* Background Thread */ + PERFCONTROL_CLASS_BACKGROUND = 4, + /* Utility Thread */ + PERFCONTROL_CLASS_UTILITY = 5, + /* Non-UI Thread (Default/Legacy) */ + PERFCONTROL_CLASS_NONUI = 6, + /* UI Thread (UI/IN) */ + PERFCONTROL_CLASS_UI = 7, + /* Above UI Thread */ + PERFCONTROL_CLASS_ABOVEUI = 8, +} perfcontrol_class_t; + #endif /* KERNEL_PRIVATE */ #endif /* _KERN_KERN_TYPES_H_ */ diff --git a/osfmk/kern/kext_alloc.c b/osfmk/kern/kext_alloc.c index 6e370a863..479d114e3 100644 --- a/osfmk/kern/kext_alloc.c +++ b/osfmk/kern/kext_alloc.c @@ -38,6 +38,7 @@ #include #include +#include #define KASLR_IOREG_DEBUG 0 @@ -103,7 +104,8 @@ kext_alloc_init(void) /* Allocate the sub block of the kernel map */ rval = kmem_suballoc(kernel_map, (vm_offset_t *) &kext_alloc_base, kext_alloc_size, /* pageable */ TRUE, - VM_FLAGS_FIXED|VM_FLAGS_OVERWRITE | VM_MAKE_TAG(VM_KERN_MEMORY_KEXT), + VM_FLAGS_FIXED|VM_FLAGS_OVERWRITE, + VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_KEXT, &g_kext_map); if (rval != KERN_SUCCESS) { panic("kext_alloc_init: kmem_suballoc failed 0x%x\n", rval); @@ -141,8 +143,6 @@ kext_alloc(vm_offset_t *_addr, vm_size_t size, boolean_t fixed) #endif int flags = (fixed) ? VM_FLAGS_FIXED : VM_FLAGS_ANYWHERE; - flags |= VM_MAKE_TAG(VM_KERN_MEMORY_KEXT); - #if CONFIG_KEXT_BASEMENT /* Allocate the kext virtual memory * 10608884 - use mach_vm_map since we want VM_FLAGS_ANYWHERE allocated past @@ -151,11 +151,12 @@ kext_alloc(vm_offset_t *_addr, vm_size_t size, boolean_t fixed) * fixed (post boot) kext allocations to start looking for free space * just past where prelinked kexts have loaded. */ - rval = mach_vm_map(g_kext_map, + rval = mach_vm_map_kernel(g_kext_map, &addr, size, 0, flags, + VM_KERN_MEMORY_KEXT, MACH_PORT_NULL, 0, TRUE, @@ -167,7 +168,7 @@ kext_alloc(vm_offset_t *_addr, vm_size_t size, boolean_t fixed) goto finish; } #else - rval = mach_vm_allocate(g_kext_map, &addr, size, flags); + rval = mach_vm_allocate_kernel(g_kext_map, &addr, size, flags, VM_KERN_MEMORY_KEXT); if (rval != KERN_SUCCESS) { printf("vm_allocate failed - %d\n", rval); goto finish; @@ -183,6 +184,9 @@ kext_alloc(vm_offset_t *_addr, vm_size_t size, boolean_t fixed) *_addr = (vm_offset_t)addr; rval = KERN_SUCCESS; +#if KASAN + kasan_notify_address(addr, size); +#endif finish: return rval; diff --git a/osfmk/kern/kpc.h b/osfmk/kern/kpc.h index 7c79c6953..aa2db20a1 100644 --- a/osfmk/kern/kpc.h +++ b/osfmk/kern/kpc.h @@ -213,6 +213,7 @@ extern boolean_t kpc_register_pm_handler(void (*handler)(boolean_t)); * * @param handler * Notification callback to use when PMCs are acquired/released by a task. + * Power management must acknowledge the change using kpc_pm_acknowledge. * * @param pmc_mask * Bitmask of the configurable PMCs used by the Power Manager. The number of bits @@ -237,6 +238,16 @@ extern boolean_t kpc_reserve_pm_counters(uint64_t pmc_mask, kpc_pm_handler_t han */ extern void kpc_release_pm_counters(void); +/* + * Acknowledge the callback that PMCs are available to power management. + * + * @param available_to_pm Whether the counters were made available to power + * management in the callback. Pass in whatever was passed into the handler + * function. After this point, power management is able to use POWER_CLASS + * counters. + */ +extern void kpc_pm_acknowledge(boolean_t available_to_pm); + /* * Is the PMU used by both the power manager and userspace? * diff --git a/osfmk/kern/kpc_common.c b/osfmk/kern/kpc_common.c index 38f06c48a..21fbbbf34 100644 --- a/osfmk/kern/kpc_common.c +++ b/osfmk/kern/kpc_common.c @@ -34,6 +34,9 @@ #include #include #include +#if defined(__arm64__) || defined(__arm__) +#include +#endif #include @@ -59,9 +62,12 @@ static lck_mtx_t kpc_config_lock; static boolean_t force_all_ctrs = FALSE; /* power manager */ -static kpc_pm_handler_t kpc_pm_handler; -static boolean_t kpc_pm_has_custom_config; -static uint64_t kpc_pm_pmc_mask; +static kpc_pm_handler_t kpc_pm_handler; +static boolean_t kpc_pm_has_custom_config; +static uint64_t kpc_pm_pmc_mask; +#if MACH_ASSERT +static bool kpc_calling_pm = false; +#endif /* MACH_ASSERT */ boolean_t kpc_context_switch_active = FALSE; @@ -177,18 +183,50 @@ kpc_force_all_ctrs(task_t task, int val) return 0; /* notify the power manager */ - if (kpc_pm_handler) + if (kpc_pm_handler) { +#if MACH_ASSERT + kpc_calling_pm = true; +#endif /* MACH_ASSERT */ kpc_pm_handler( new_state ? FALSE : TRUE ); +#if MACH_ASSERT + kpc_calling_pm = false; +#endif /* MACH_ASSERT */ + } - /* update the task bits */ - kpc_task_set_forced_all_ctrs(task, val); + /* + * This is a force -- ensure that counters are forced, even if power + * management fails to acknowledge it. + */ + if (force_all_ctrs != new_state) { + force_all_ctrs = new_state; + } - /* update the internal state */ - force_all_ctrs = val; + /* update the task bits */ + kpc_task_set_forced_all_ctrs(task, new_state); return 0; } +void +kpc_pm_acknowledge(boolean_t available_to_pm) +{ + /* + * Force-all-counters should still be true when the counters are being + * made available to power management and false when counters are going + * to be taken away. + */ + assert(force_all_ctrs == available_to_pm); + /* + * Make sure power management isn't playing games with us. + */ + assert(kpc_calling_pm == true); + + /* + * Counters being available means no one is forcing all counters. + */ + force_all_ctrs = available_to_pm ? FALSE : TRUE; +} + int kpc_get_force_all_ctrs(void) { diff --git a/osfmk/kern/kpc_thread.c b/osfmk/kern/kpc_thread.c index aa8f261d4..248ea3c56 100644 --- a/osfmk/kern/kpc_thread.c +++ b/osfmk/kern/kpc_thread.c @@ -41,6 +41,11 @@ #include #include +#if defined (__arm64__) +#include +#elif defined (__arm__) +#include +#endif /* global for whether to read PMCs on context switch */ int kpc_threads_counting = 0; diff --git a/osfmk/kern/ledger.c b/osfmk/kern/ledger.c index 6b2434e0d..4c27ea7fb 100644 --- a/osfmk/kern/ledger.c +++ b/osfmk/kern/ledger.c @@ -57,7 +57,7 @@ #define LF_REFILL_INPROGRESS 0x0800 /* the ledger is being refilled */ #define LF_CALLED_BACK 0x1000 /* callback was called for balance in deficit */ #define LF_WARNED 0x2000 /* callback was called for balance warning */ -#define LF_TRACKING_MAX 0x4000 /* track max balance over user-specfied time */ +#define LF_TRACKING_MAX 0x4000 /* track max balance. Exclusive w.r.t refill */ #define LF_PANIC_ON_NEGATIVE 0x8000 /* panic if it goes negative */ #define LF_TRACK_CREDIT_ONLY 0x10000 /* only update "credit" */ @@ -112,6 +112,7 @@ struct ledger_template { int lt_table_size; volatile uint32_t lt_inuse; lck_mtx_t lt_lock; + zone_t lt_zone; struct entry_template *lt_entries; }; @@ -130,9 +131,9 @@ struct ledger_template { } /* - * Use 2 "tocks" to track the rolling maximum balance of a ledger entry. + * Use NTOCKS "tocks" to track the rolling maximum balance of a ledger entry. */ -#define NTOCKS 2 +#define NTOCKS 1 /* * The explicit alignment is to ensure that atomic operations don't panic * on ARM. @@ -149,13 +150,16 @@ struct ledger_entry { * XXX - the following two fields can go away if we move all of * the refill logic into process policy */ - uint64_t le_refill_period; - uint64_t le_last_refill; + uint64_t le_refill_period; + uint64_t le_last_refill; } le_refill; - struct _le_peak { - uint32_t le_max; /* Lower 32-bits of observed max balance */ - uint32_t le_time; /* time when this peak was observed */ - } le_peaks[NTOCKS]; + struct _le_maxtracking { + struct _le_peak { + uint32_t le_max; /* Lower 32-bits of observed max balance */ + uint32_t le_time; /* time when this peak was observed */ + } le_peaks[NTOCKS]; + ledger_amount_t le_lifetime_max; /* greatest peak ever observed */ + } le_maxtracking; } _le; } __attribute__((aligned(8))); @@ -226,6 +230,7 @@ ledger_template_create(const char *name) template->lt_cnt = 0; template->lt_table_size = 1; template->lt_inuse = 0; + template->lt_zone = NULL; lck_mtx_init(&template->lt_lock, &ledger_lck_grp, LCK_ATTR_NULL); template->lt_entries = (struct entry_template *) @@ -262,7 +267,7 @@ ledger_entry_add(ledger_template_t template, const char *key, int idx; struct entry_template *et; - if ((key == NULL) || (strlen(key) >= LEDGER_NAME_MAX)) + if ((key == NULL) || (strlen(key) >= LEDGER_NAME_MAX) || (template->lt_zone != NULL)) return (-1); template_lock(template); @@ -347,6 +352,22 @@ ledger_key_lookup(ledger_template_t template, const char *key) return (idx); } +/* + * Complete the initialization of ledger template + * by initializing ledger zone. After initializing + * the ledger zone, adding an entry in the ledger + * template would fail. + */ +void +ledger_template_complete(ledger_template_t template) +{ + size_t ledger_size; + ledger_size = sizeof(struct ledger) + (template->lt_cnt * sizeof(struct ledger_entry)); + template->lt_zone = zinit(ledger_size, CONFIG_TASK_MAX * ledger_size, + ledger_size, + template->lt_name); +} + /* * Create a new ledger based on the specified template. As part of the * ledger creation we need to allocate space for a table of ledger entries. @@ -358,17 +379,16 @@ ledger_t ledger_instantiate(ledger_template_t template, int entry_type) { ledger_t ledger; - size_t cnt, sz; + size_t cnt; int i; template_lock(template); template->lt_refs++; cnt = template->lt_cnt; + assert(template->lt_zone); template_unlock(template); - sz = sizeof(*ledger) + (cnt * sizeof(struct ledger_entry)); - - ledger = (ledger_t)kalloc(sz); + ledger = (ledger_t)zalloc(template->lt_zone); if (ledger == NULL) { ledger_template_dereference(template); return LEDGER_NULL; @@ -457,8 +477,7 @@ ledger_dereference(ledger_t ledger) /* Just released the last reference. Free it. */ if (v == 1) { - kfree(ledger, - sizeof(*ledger) + ledger->l_size * sizeof(struct ledger_entry)); + zfree(ledger->l_template->lt_zone, ledger); } return (KERN_SUCCESS); @@ -685,7 +704,7 @@ ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry * if (le->le_flags & LF_TRACKING_MAX) { ledger_amount_t balance = le->le_credit - le->le_debit; uint32_t now = CURRENT_TOCKSTAMP(); - struct _le_peak *p = &le->_le.le_peaks[now % NTOCKS]; + struct _le_peak *p = &le->_le.le_maxtracking.le_peaks[now % NTOCKS]; if (!TOCKSTAMP_IS_STALE(now, p->le_time) || (balance > p->le_max)) { /* @@ -700,10 +719,17 @@ ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry * p->le_max = (uint32_t)balance; p->le_time = now; } + + struct _le_maxtracking *m = &le->_le.le_maxtracking; + if(balance > m->le_lifetime_max){ + m->le_lifetime_max = balance; + } } /* Check to see whether we're due a refill */ if (le->le_flags & LF_REFILL_SCHEDULED) { + assert(!(le->le_flags & LF_TRACKING_MAX)); + uint64_t now = mach_absolute_time(); if ((now - le->_le.le_refill.le_last_refill) > le->_le.le_refill.le_refill_period) ledger_refill(now, ledger, entry); @@ -948,7 +974,10 @@ ledger_set_limit(ledger_t ledger, int entry, ledger_amount_t limit, } le->le_limit = limit; - le->_le.le_refill.le_last_refill = 0; + if (le->le_flags & LF_REFILL_SCHEDULED) { + assert(!(le->le_flags & LF_TRACKING_MAX)); + le->_le.le_refill.le_last_refill = 0; + } flag_clear(&le->le_flags, LF_CALLED_BACK); flag_clear(&le->le_flags, LF_WARNED); ledger_limit_entry_wakeup(le); @@ -966,7 +995,7 @@ ledger_set_limit(ledger_t ledger, int entry, ledger_amount_t limit, } kern_return_t -ledger_get_maximum(ledger_t ledger, int entry, +ledger_get_recent_max(ledger_t ledger, int entry, ledger_amount_t *max_observed_balance) { struct ledger_entry *le; @@ -986,21 +1015,38 @@ ledger_get_maximum(ledger_t ledger, int entry, *max_observed_balance = le->le_credit - le->le_debit; for (i = 0; i < NTOCKS; i++) { - if (!TOCKSTAMP_IS_STALE(now, le->_le.le_peaks[i].le_time) && - (le->_le.le_peaks[i].le_max > *max_observed_balance)) { + if (!TOCKSTAMP_IS_STALE(now, le->_le.le_maxtracking.le_peaks[i].le_time) && + (le->_le.le_maxtracking.le_peaks[i].le_max > *max_observed_balance)) { /* * The peak for this time block isn't stale, and it * is greater than the current balance -- so use it. */ - *max_observed_balance = le->_le.le_peaks[i].le_max; + *max_observed_balance = le->_le.le_maxtracking.le_peaks[i].le_max; } } - + lprintf(("ledger_get_maximum: %lld\n", *max_observed_balance)); return (KERN_SUCCESS); } +kern_return_t +ledger_get_lifetime_max(ledger_t ledger, int entry, + ledger_amount_t *max_lifetime_balance) +{ + struct ledger_entry *le; + le = &ledger->l_entries[entry]; + + if (!ENTRY_VALID(ledger, entry) || !(le->le_flags & LF_TRACKING_MAX)) { + return (KERN_INVALID_VALUE); + } + + *max_lifetime_balance = le->_le.le_maxtracking.le_lifetime_max; + lprintf(("ledger_get_lifetime_max: %lld\n", *max_lifetime_balance)); + + return (KERN_SUCCESS); +} + /* * Enable tracking of periodic maximums for this ledger entry. */ @@ -1015,8 +1061,13 @@ ledger_track_maximum(ledger_template_t template, int entry, return (KERN_INVALID_VALUE); } + /* Refill is incompatible with max tracking. */ + if (template->lt_entries[entry].et_flags & LF_REFILL_SCHEDULED) { + return (KERN_INVALID_VALUE); + } + template->lt_entries[entry].et_flags |= LF_TRACKING_MAX; - template_unlock(template); + template_unlock(template); return (KERN_SUCCESS); } @@ -1410,6 +1461,8 @@ ledger_check_needblock(ledger_t l, uint64_t now) /* We're over the limit, so refill if we are eligible and past due. */ if (le->le_flags & LF_REFILL_SCHEDULED) { + assert(!(le->le_flags & LF_TRACKING_MAX)); + if ((le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period) > now) { ledger_refill(now, l, i); if (limit_exceeded(le) == FALSE) @@ -1451,6 +1504,8 @@ ledger_perform_blocking(ledger_t l) ((le->le_flags & LEDGER_ACTION_BLOCK) == 0)) continue; + assert(!(le->le_flags & LF_TRACKING_MAX)); + /* Prepare to sleep until the resource is refilled */ ret = assert_wait_deadline(le, TRUE, le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period); diff --git a/osfmk/kern/ledger.h b/osfmk/kern/ledger.h index 5d1c6818f..8e627bef4 100644 --- a/osfmk/kern/ledger.h +++ b/osfmk/kern/ledger.h @@ -110,20 +110,23 @@ extern int ledger_key_lookup(ledger_template_t template, const char *key); #define LEDGER_CREATE_ACTIVE_ENTRIES 0 #define LEDGER_CREATE_INACTIVE_ENTRIES 1 extern ledger_t ledger_instantiate(ledger_template_t template, int entry_type); +extern void ledger_template_complete(ledger_template_t template); extern kern_return_t ledger_disable_callback(ledger_t ledger, int entry); extern kern_return_t ledger_enable_callback(ledger_t ledger, int entry); extern kern_return_t ledger_get_limit(ledger_t ledger, int entry, ledger_amount_t *limit); extern kern_return_t ledger_set_limit(ledger_t ledger, int entry, ledger_amount_t limit, uint8_t warn_level_percentage); -extern kern_return_t ledger_get_maximum(ledger_t ledger, int entry, +extern kern_return_t ledger_get_recent_max(ledger_t ledger, int entry, ledger_amount_t *max_observed_balance); +extern kern_return_t ledger_get_lifetime_max(ledger_t ledger, int entry, + ledger_amount_t *max_lifetime_balance); extern kern_return_t ledger_get_actions(ledger_t ledger, int entry, int *actions); extern kern_return_t ledger_set_action(ledger_t ledger, int entry, int action); extern kern_return_t ledger_get_period(ledger_t ledger, int entry, - uint64_t *period); + uint64_t *period); extern kern_return_t ledger_set_period(ledger_t ledger, int entry, - uint64_t period); + uint64_t period); extern kern_return_t ledger_disable_refill(ledger_t l, int entry); extern kern_return_t ledger_entry_setactive(ledger_t ledger, int entry); extern void ledger_check_new_balance(ledger_t ledger, int entry); diff --git a/osfmk/kern/locks.c b/osfmk/kern/locks.c index 1fbb88e0f..25641b8be 100644 --- a/osfmk/kern/locks.c +++ b/osfmk/kern/locks.c @@ -124,6 +124,14 @@ lck_grp_attr_t LockDefaultGroupAttr; lck_grp_t LockCompatGroup; lck_attr_t LockDefaultLckAttr; +#if CONFIG_DTRACE && __SMP__ +#if defined (__x86_64__) +uint64_t dtrace_spin_threshold = 500; // 500ns +#elif defined(__arm__) || defined(__arm64__) +uint64_t dtrace_spin_threshold = LOCK_PANIC_TIMEOUT / 1000000; // 500ns +#endif +#endif + /* * Routine: lck_mod_init */ @@ -138,6 +146,12 @@ lck_mod_init( if (!PE_parse_boot_argn("lcks", &LcksOpts, sizeof (LcksOpts))) LcksOpts = 0; + +#if (DEVELOPMENT || DEBUG) && defined(__x86_64__) + if (!PE_parse_boot_argn("-disable_mtx_chk", &LckDisablePreemptCheck, sizeof (LckDisablePreemptCheck))) + LckDisablePreemptCheck = 0; +#endif /* (DEVELOPMENT || DEBUG) && defined(__x86_64__) */ + queue_init(&lck_grp_queue); /* @@ -162,7 +176,6 @@ lck_mod_init( lck_attr_setdefault(&LockDefaultLckAttr); lck_mtx_init_ext(&lck_grp_lock, &lck_grp_lock_ext, &LockCompatGroup, &LockDefaultLckAttr); - } /* @@ -391,7 +404,10 @@ void lck_attr_setdefault( lck_attr_t *attr) { -#if __i386__ || __x86_64__ +#if __arm__ || __arm64__ + /* : Using LCK_ATTR_DEBUG here causes panic at boot time for arm */ + attr->lck_attr_val = LCK_ATTR_NONE; +#elif __i386__ || __x86_64__ #if !DEBUG if (LcksOpts & enaLkDeb) attr->lck_attr_val = LCK_ATTR_DEBUG; @@ -463,8 +479,8 @@ hw_lock_init(hw_lock_t lock) * Routine: hw_lock_lock_contended * * Spin until lock is acquired or timeout expires. - * timeout is in mach_absolute_time ticks. - * MACH_RT: called with preemption disabled. + * timeout is in mach_absolute_time ticks. Called with + * preemption disabled. */ #if __SMP__ @@ -477,26 +493,35 @@ hw_lock_lock_contended(hw_lock_t lock, uintptr_t data, uint64_t timeout, boolean if (timeout == 0) timeout = LOCK_PANIC_TIMEOUT; - +#if CONFIG_DTRACE + uint64_t begin; + boolean_t dtrace_enabled = lockstat_probemap[LS_LCK_SPIN_LOCK_SPIN] != 0; + if (__improbable(dtrace_enabled)) + begin = mach_absolute_time(); +#endif for ( ; ; ) { for (i = 0; i < LOCK_SNOOP_SPINS; i++) { - boolean_t wait = FALSE; - cpu_pause(); #if (!__ARM_ENABLE_WFE_) || (LOCK_PRETEST) holder = ordered_load_hw(lock); if (holder != 0) continue; -#endif -#if __ARM_ENABLE_WFE_ - wait = TRUE; // Wait for event #endif if (atomic_compare_exchange(&lock->lock_data, 0, data, - memory_order_acquire_smp, wait)) + memory_order_acquire_smp, TRUE)) { +#if CONFIG_DTRACE + if (__improbable(dtrace_enabled)) { + uint64_t spintime = mach_absolute_time() - begin; + if (spintime > dtrace_spin_threshold) + LOCKSTAT_RECORD2(LS_LCK_SPIN_LOCK_SPIN, lock, spintime, dtrace_spin_threshold); + } +#endif return 1; + } } - if (end == 0) + if (end == 0) { end = ml_get_timebase() + timeout; + } else if (ml_get_timebase() >= end) break; } @@ -513,8 +538,8 @@ hw_lock_lock_contended(hw_lock_t lock, uintptr_t data, uint64_t timeout, boolean /* * Routine: hw_lock_lock * - * Acquire lock, spinning until it becomes available. - * MACH_RT: also return with preemption disabled. + * Acquire lock, spinning until it becomes available, + * return with preemption disabled. */ void hw_lock_lock(hw_lock_t lock) @@ -526,22 +551,28 @@ hw_lock_lock(hw_lock_t lock) disable_preemption_for_thread(thread); state = LCK_MTX_THREAD_TO_STATE(thread) | PLATFORM_LCK_ILOCK; #if __SMP__ + #if LOCK_PRETEST if (ordered_load_hw(lock)) goto contended; #endif // LOCK_PRETEST if (atomic_compare_exchange(&lock->lock_data, 0, state, - memory_order_acquire_smp, TRUE)) - return; + memory_order_acquire_smp, TRUE)) { + goto end; + } #if LOCK_PRETEST contended: #endif // LOCK_PRETEST hw_lock_lock_contended(lock, state, 0, TRUE); +end: #else // __SMP__ if (lock->lock_data) panic("Spinlock held %p", lock); lock->lock_data = state; #endif // __SMP__ +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_ACQUIRE, lock, 0); +#endif return; } @@ -549,43 +580,53 @@ contended: * Routine: hw_lock_to * * Acquire lock, spinning until it becomes available or timeout. - * timeout is in mach_absolute_time ticks. - * MACH_RT: also return with preemption disabled. + * Timeout is in mach_absolute_time ticks, return with + * preemption disabled. */ unsigned int hw_lock_to(hw_lock_t lock, uint64_t timeout) { thread_t thread; uintptr_t state; + unsigned int success = 0; thread = current_thread(); disable_preemption_for_thread(thread); state = LCK_MTX_THREAD_TO_STATE(thread) | PLATFORM_LCK_ILOCK; #if __SMP__ + #if LOCK_PRETEST if (ordered_load_hw(lock)) goto contended; #endif // LOCK_PRETEST if (atomic_compare_exchange(&lock->lock_data, 0, state, - memory_order_acquire_smp, TRUE)) - return 1; + memory_order_acquire_smp, TRUE)) { + success = 1; + goto end; + } #if LOCK_PRETEST contended: #endif // LOCK_PRETEST - return hw_lock_lock_contended(lock, state, timeout, FALSE); + success = hw_lock_lock_contended(lock, state, timeout, FALSE); +end: #else // __SMP__ (void)timeout; if (ordered_load_hw(lock) == 0) { ordered_store_hw(lock, state); - return 1; + success = 1; } - return 0; #endif // __SMP__ +#if CONFIG_DTRACE + if (success) + LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_ACQUIRE, lock, 0); +#endif + return success; } /* * Routine: hw_lock_try - * MACH_RT: returns with preemption disabled on success. + * + * returns with preemption disabled on success. */ unsigned int hw_lock_try(hw_lock_t lock) @@ -628,25 +669,34 @@ failed: if (!success) enable_preemption(); #endif // LOCK_TRY_DISABLE_INT +#if CONFIG_DTRACE + if (success) + LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_ACQUIRE, lock, 0); +#endif return success; } /* * Routine: hw_lock_unlock * - * Unconditionally release lock. - * MACH_RT: release preemption level. + * Unconditionally release lock, release preemption level. */ void hw_lock_unlock(hw_lock_t lock) { __c11_atomic_store((_Atomic uintptr_t *)&lock->lock_data, 0, memory_order_release_smp); +#if __arm__ || __arm64__ + // ARM tests are only for open-source exclusion + set_event(); +#endif // __arm__ || __arm64__ +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_SPIN_UNLOCK_RELEASE, lock, 0); +#endif /* CONFIG_DTRACE */ enable_preemption(); } /* - * RoutineL hw_lock_held - * MACH_RT: doesn't change preemption state. + * Routine hw_lock_held, doesn't change preemption state. * N.B. Racy, of course. */ unsigned int @@ -785,6 +835,8 @@ lck_mtx_sleep( if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { if ((lck_sleep_action & LCK_SLEEP_SPIN)) lck_mtx_lock_spin(lck); + else if ((lck_sleep_action & LCK_SLEEP_SPIN_ALWAYS)) + lck_mtx_lock_spin_always(lck); else lck_mtx_lock(lck); } @@ -939,7 +991,7 @@ lck_mtx_lock_wait ( KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0); #if CONFIG_DTRACE /* - * Record the Dtrace lockstat probe for blocking, block time + * Record the DTrace lockstat probe for blocking, block time * measured from when we were entered. */ if (sleep_start) { diff --git a/osfmk/kern/locks.h b/osfmk/kern/locks.h index 5a29c2e14..99017d2cc 100644 --- a/osfmk/kern/locks.h +++ b/osfmk/kern/locks.h @@ -59,8 +59,9 @@ typedef unsigned int lck_sleep_action_t; #define LCK_SLEEP_EXCLUSIVE 0x04 /* Reclaim the lock in exclusive mode (RW only) */ #define LCK_SLEEP_SPIN 0x08 /* Reclaim the lock in spin mode (mutex only) */ #define LCK_SLEEP_PROMOTED_PRI 0x10 /* Sleep at a promoted priority */ +#define LCK_SLEEP_SPIN_ALWAYS 0x20 /* Reclaim the lock in spin-always mode (mutex only) */ -#define LCK_SLEEP_MASK 0x1f /* Valid actions */ +#define LCK_SLEEP_MASK 0x3f /* Valid actions */ #ifdef MACH_KERNEL_PRIVATE @@ -358,14 +359,22 @@ extern void lck_mtx_assert( #if MACH_ASSERT #define LCK_MTX_ASSERT(lck,type) lck_mtx_assert((lck),(type)) +#define LCK_SPIN_ASSERT(lck,type) lck_spin_assert((lck),(type)) +#define LCK_RW_ASSERT(lck,type) lck_rw_assert((lck),(type)) #else /* MACH_ASSERT */ #define LCK_MTX_ASSERT(lck,type) +#define LCK_SPIN_ASSERT(lck,type) +#define LCK_RW_ASSERT(lck,type) #endif /* MACH_ASSERT */ #if DEBUG #define LCK_MTX_ASSERT_DEBUG(lck,type) lck_mtx_assert((lck),(type)) +#define LCK_SPIN_ASSERT_DEBUG(lck,type) lck_spin_assert((lck),(type)) +#define LCK_RW_ASSERT_DEBUG(lck,type) lck_rw_assert((lck),(type)) #else /* DEBUG */ #define LCK_MTX_ASSERT_DEBUG(lck,type) +#define LCK_SPIN_ASSERT_DEBUG(lck,type) +#define LCK_RW_ASSERT_DEBUG(lck,type) #endif /* DEBUG */ __END_DECLS @@ -437,6 +446,10 @@ extern void lck_rw_lock_shared( extern void lck_rw_unlock_shared( lck_rw_t *lck); +extern boolean_t lck_rw_lock_yield_shared( + lck_rw_t *lck, + boolean_t force_yield); + extern void lck_rw_lock_exclusive( lck_rw_t *lck); diff --git a/osfmk/kern/ltable.c b/osfmk/kern/ltable.c index 4aedca0d4..073f27c9a 100644 --- a/osfmk/kern/ltable.c +++ b/osfmk/kern/ltable.c @@ -50,7 +50,7 @@ static lck_grp_t g_lt_lck_grp; /* default VA space for link tables (zone allocated) */ #define DEFAULT_MAX_TABLE_SIZE P2ROUNDUP(8 * 1024 * 1024, PAGE_SIZE) -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG /* global for lldb macros */ uint64_t g_lt_idx_max = LT_IDX_MAX; #endif @@ -61,7 +61,7 @@ uint64_t g_lt_idx_max = LT_IDX_MAX; /* cast through 'void *' to avoid compiler alignment warning messages */ \ ((struct lt_elem *)((void *)((uintptr_t)(slab) + ((ofst) & (slab_msk))))) -#if defined(CONFIG_LTABLE_STATS) +#if CONFIG_LTABLE_STATS /* version that makes no assumption on waste within a slab */ static inline struct lt_elem * lt_elem_idx(struct link_table *table, uint32_t idx) @@ -87,7 +87,7 @@ lt_elem_idx(struct link_table *table, uint32_t idx) assert(slab->lt_id.idx <= idx && (slab->lt_id.idx + table->slab_elem) > idx); return lt_elem_ofst_slab(slab, table->slab_msk, ofst); } -#endif /* !CONFIG_LTABLE_STATS */ +#endif /* CONFIG_LTABLE_STATS */ static int __assert_only lt_elem_in_range(struct lt_elem *elem, struct link_table *table) diff --git a/osfmk/kern/machine.c b/osfmk/kern/machine.c index 30d213539..cc0290ee1 100644 --- a/osfmk/kern/machine.c +++ b/osfmk/kern/machine.c @@ -128,6 +128,8 @@ processor_up( ++pset->online_processor_count; enqueue_tail(&pset->active_queue, (queue_entry_t)processor); processor->state = PROCESSOR_RUNNING; + pset->active_processor_count++; + sched_update_pset_load_average(pset); (void)hw_atomic_add(&processor_avail_count, 1); commpage_update_active_cpus(); pset_unlock(pset); @@ -228,11 +230,13 @@ processor_shutdown( return (KERN_SUCCESS); } - if (processor->state == PROCESSOR_IDLE) + if (processor->state == PROCESSOR_IDLE) { remqueue((queue_entry_t)processor); - else - if (processor->state == PROCESSOR_RUNNING) + } else if (processor->state == PROCESSOR_RUNNING) { remqueue((queue_entry_t)processor); + pset->active_processor_count--; + sched_update_pset_load_average(pset); + } processor->state = PROCESSOR_SHUTDOWN; @@ -287,6 +291,7 @@ processor_doshutdown( commpage_update_active_cpus(); SCHED(processor_queue_shutdown)(processor); /* pset lock dropped */ + SCHED(rt_queue_shutdown)(processor); /* * Continue processor shutdown in shutdown context. @@ -327,10 +332,8 @@ processor_offline( thread_t new_thread = processor->idle_thread; processor->active_thread = new_thread; - processor->current_pri = IDLEPRI; - processor->current_thmode = TH_MODE_NONE; + processor_state_update_idle(processor); processor->starting_pri = IDLEPRI; - processor->current_sfi_class = SFI_CLASS_KERNEL; processor->deadline = UINT64_MAX; new_thread->last_processor = processor; @@ -342,7 +345,6 @@ processor_offline( /* Update processor->thread_timer and ->kernel_timer to point to the new thread */ thread_timer_event(ctime, &new_thread->system_timer); PROCESSOR_DATA(processor, kernel_timer) = &new_thread->system_timer; - timer_stop(PROCESSOR_DATA(processor, current_state), ctime); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, @@ -431,7 +433,7 @@ unsigned long long ml_io_read(uintptr_t vaddr, int size) { (void)ml_set_interrupts_enabled(istate); if (__improbable((eabs - sabs) > reportphyreaddelayabs)) { - if (phyreadpanic) { + if (phyreadpanic && (machine_timeout_suspended() == FALSE)) { panic("Read from IO virtual addr 0x%lx took %llu ns, result: 0x%llx (start: %llu, end: %llu), ceiling: %llu", vaddr, (eabs - sabs), result, sabs, eabs, reportphyreaddelayabs); } #if CONFIG_DTRACE diff --git a/osfmk/kern/machine.h b/osfmk/kern/machine.h index 37d69bf88..1f5c7816a 100644 --- a/osfmk/kern/machine.h +++ b/osfmk/kern/machine.h @@ -35,6 +35,10 @@ #include #include #include +#include + +#include + /* * Machine support declarations. @@ -131,6 +135,7 @@ extern void consider_machine_collect(void); extern void machine_thread_going_on_core(thread_t new_thread, int urgency, uint64_t sched_latency, + uint64_t same_pri_latency, uint64_t dispatch_time); extern void machine_thread_going_off_core(thread_t old_thread, boolean_t thread_terminating, uint64_t last_dispatch); @@ -139,13 +144,21 @@ extern void machine_max_runnable_latency(uint64_t bg_max_latency, uint64_t default_max_latency, uint64_t realtime_max_latency); -extern void machine_work_interval_notify(thread_t thread, - uint64_t work_id, - uint64_t start_abstime, - uint64_t finish_abstime, - uint64_t deadline_abstime, - uint64_t next_start_abstime, - uint16_t urgency, - uint32_t flags); +extern void machine_work_interval_notify(thread_t thread, struct kern_work_interval_args* kwi_args); + +extern void machine_perfcontrol_deadline_passed(uint64_t deadline); + +extern void machine_switch_perfcontrol_context(perfcontrol_event event, + uint64_t timestamp, + uint32_t flags, + uint64_t new_thread_same_pri_latency, + thread_t old, + thread_t new); + +extern void machine_switch_perfcontrol_state_update(perfcontrol_event event, + uint64_t timestamp, + uint32_t flags, + thread_t thread); + #endif /* _KERN_MACHINE_H_ */ diff --git a/osfmk/kern/memset_s.c b/osfmk/kern/memset_s.c new file mode 100644 index 000000000..f13d0f6cd --- /dev/null +++ b/osfmk/kern/memset_s.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include + +extern void *secure_memset(void *, int, size_t); + +/* + * The memset_s function copies the value c into the first n bytes + * pointed by s. No more than smax bytes will be copied. + * + * In contrast to the memset function, calls to memset_s will never + * be ''optimised away'' by a compiler, ensuring the memory copy + * even if s is not accessed anymore after this call. + */ +int +memset_s(void *s, size_t smax, int c, size_t n) +{ + int err = 0; + + if (s == NULL) return EINVAL; + if (smax > RSIZE_MAX) return E2BIG; + if (n > smax) { + n = smax; + err = EOVERFLOW; + } + + /* + * secure_memset is defined in assembly, we therefore + * expect that the compiler will not inline the call. + */ + secure_memset(s, c, n); + + return err; +} + diff --git a/osfmk/kern/misc_protos.h b/osfmk/kern/misc_protos.h index 355316d22..63eb58c09 100644 --- a/osfmk/kern/misc_protos.h +++ b/osfmk/kern/misc_protos.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2011 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -69,6 +69,14 @@ extern int ffsbit( int *bitmap); extern int ffs( unsigned int mask); +extern int ffsll( + unsigned long long mask); + +/* Find the last set bit in a bit array */ +extern int fls( + unsigned int mask); +extern int flsll( + unsigned long long mask); /* * Test if indicated bit is set in bit string. @@ -147,6 +155,8 @@ int _consume_printf_args(int, ...); #endif #endif +extern int paniclog_append_noflush(const char *format, ...) __printflike(1,2); + extern int kdb_printf(const char *format, ...) __printflike(1,2); extern int kdb_log(const char *format, ...) __printflike(1,2); @@ -188,6 +198,8 @@ extern void safe_gets( extern void cnputcusr(char); +extern void cnputsusr(char *, int); + extern void conslog_putc(char); extern void cons_putc_locked(char); @@ -221,26 +233,6 @@ extern int _longjmp( extern void bootstrap_create(void); -/* - * Halt other cores before invoking debugger - * Halting other cores as early as possible helps preserve - * the current system state for debugging - */ -extern void DebuggerHaltOtherCores(void); - -/* Resume other cores */ -extern void DebuggerResumeOtherCores(void); - -extern void Debugger( - const char * message); - -extern void DebuggerWithContext( - unsigned int reason, - void *ctx, - const char *message, - uint64_t debugger_options_mask); - - #if DIPC extern boolean_t no_bootstrap_task(void); extern ipc_port_t get_root_master_device_port(void); diff --git a/osfmk/kern/monotonic.h b/osfmk/kern/monotonic.h new file mode 100644 index 000000000..a082a3535 --- /dev/null +++ b/osfmk/kern/monotonic.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef KERN_MONOTONIC_H +#define KERN_MONOTONIC_H + +#include +#include + +extern bool mt_debug; +extern _Atomic uint64_t mt_pmis; +extern _Atomic uint64_t mt_retrograde; + +void mt_fixed_counts(uint64_t *counts); +void mt_cur_thread_fixed_counts(uint64_t *counts); +void mt_cur_task_fixed_counts(uint64_t *counts); +uint64_t mt_cur_cpu_instrs(void); +uint64_t mt_cur_cpu_cycles(void); +uint64_t mt_cur_thread_instrs(void); +uint64_t mt_cur_thread_cycles(void); + +#if MACH_KERNEL_PRIVATE + +#include +#include +#include + +#if defined(__arm__) || defined(__arm64__) +#include +#elif defined(__x86_64__) +#include +#else /* !defined(__arm__) && !defined(__arm64__) && !defined(__x86_64__) */ +#error unsupported architecture +#endif /* !defined(__arm__) && !defined(__arm64__) && !defined(__x86_64__) */ + +void mt_init(void); +void mt_update_fixed_counts(void); +void mt_update_task(task_t task, thread_t thread); +bool mt_update_thread(thread_t thread); +int mt_fixed_thread_counts(thread_t thread, uint64_t *counts_out); +int mt_fixed_task_counts(task_t task, uint64_t *counts_out); + +/* + * Called when a thread is switching off-core or expires its quantum. + */ +void mt_sched_update(thread_t thread); + +/* + * Called when a thread is terminating to save its counters into the task. The + * task lock must be held and the thread should be removed from the task's + * thread list in that same critical section. + */ +void mt_terminate_update(task_t task, thread_t thread); + +/* + * Called when a core receives a PMI. + */ +void mt_cpu_pmi(cpu_data_t *cpu, uint64_t pmsr); +uint64_t mt_cpu_update_count(cpu_data_t *cpu, unsigned int ctr); + +/* + * Called when a core is idling and exiting from idle. + */ +void mt_cpu_idle(cpu_data_t *cpu); +void mt_cpu_run(cpu_data_t *cpu); + +/* + * Called when a core is shutting down or powering up. + */ +void mt_cpu_down(cpu_data_t *cpu); +void mt_cpu_up(cpu_data_t *cpu); + +/* + * Called while single-threaded when the system is going to sleep and waking up. + */ +void mt_sleep(void); +void mt_wake(void); + +/* + * Private API for the performance controller callout. + */ +void mt_perfcontrol(uint64_t *instrs, uint64_t *cycles); + +/* + * Private API for stackshot. + */ +void mt_stackshot_thread(thread_t thread, uint64_t *instrs, uint64_t *cycles); +void mt_stackshot_task(task_t task, uint64_t *instrs, uint64_t *cycles); + +#endif /* MACH_KERNEL_PRIVATE */ + +#endif /* !defined(KERN_MONOTONIC_H) */ diff --git a/osfmk/kern/policy_internal.h b/osfmk/kern/policy_internal.h index 5daa832d7..91e4c0d12 100644 --- a/osfmk/kern/policy_internal.h +++ b/osfmk/kern/policy_internal.h @@ -108,6 +108,7 @@ extern kern_return_t task_importance(task_t task, integer_t importance); #define TASK_POLICY_QOS_AND_RELPRIO 0x38 /* QoS as value1, relative priority as value2 */ #define TASK_POLICY_QOS_PROMOTE 0x3C #define TASK_POLICY_QOS_IPC_OVERRIDE 0x3D +#define TASK_POLICY_QOS_SYNC_IPC_OVERRIDE 0x3E #define TASK_POLICY_MAX 0x3F @@ -151,8 +152,13 @@ extern void proc_inherit_task_role(task_t new_task, task_t old_task); #define THROTTLE_LEVEL_COMPRESSOR_TIER1 THROTTLE_LEVEL_TIER1 #define THROTTLE_LEVEL_COMPRESSOR_TIER2 THROTTLE_LEVEL_TIER2 +#if CONFIG_EMBEDDED +#define THROTTLE_LEVEL_PAGEOUT_THROTTLED THROTTLE_LEVEL_TIER3 +#define THROTTLE_LEVEL_PAGEOUT_UNTHROTTLED THROTTLE_LEVEL_TIER1 +#else #define THROTTLE_LEVEL_PAGEOUT_THROTTLED THROTTLE_LEVEL_TIER2 #define THROTTLE_LEVEL_PAGEOUT_UNTHROTTLED THROTTLE_LEVEL_TIER1 +#endif #if CONFIG_IOSCHED #define IOSCHED_METADATA_TIER THROTTLE_LEVEL_TIER1 @@ -190,6 +196,10 @@ extern void proc_thread_qos_deallocate(thread_t thread); extern int task_clear_cpuusage(task_t task, int cpumon_entitled); +#if CONFIG_EMBEDDED +/* Taskwatch related external BSD interface */ +extern int proc_lf_pidbind(task_t curtask, uint64_t tid, task_t target_task, int bind); +#endif /* CONFIG_EMBEDDED */ /* Importance inheritance functions not under IMPORTANCE_INHERITANCE */ extern void task_importance_mark_donor(task_t task, boolean_t donating); @@ -239,13 +249,6 @@ extern void task_clear_has_been_notified(task_t task, int pressurelevel); extern void task_clear_used_for_purging(task_t task); extern int task_importance_estimate(task_t task); -/* - * Allocate/assign a single work interval ID for a thread, - * and support deallocating it. - */ -extern kern_return_t thread_policy_create_work_interval(thread_t thread, uint64_t *work_interval_id); -extern kern_return_t thread_policy_destroy_work_interval(thread_t thread, uint64_t work_interval_id); - extern kern_return_t thread_policy_set_internal(thread_t thread, thread_policy_flavor_t flavor, thread_policy_t policy_info, mach_msg_type_number_t count); @@ -260,10 +263,18 @@ extern void thread_user_promotion_add(thread_t thread, thread_t promoter, struct extern void thread_user_promotion_update(thread_t thread, thread_t promoter, struct promote_token* promote_token); extern void thread_user_promotion_drop(thread_t thread); +/* for thread exec promotion */ +#define EXEC_BOOST_PRIORITY 31 + +extern void thread_set_exec_promotion(thread_t thread); +extern void thread_clear_exec_promotion(thread_t thread); + /* for IPC override management */ extern void thread_add_ipc_override(thread_t thread, uint32_t qos_override); extern void thread_update_ipc_override(thread_t thread, uint32_t qos_override); extern void thread_drop_ipc_override(thread_t thread); +extern void thread_add_sync_ipc_override(thread_t thread); +extern void thread_drop_sync_ipc_override(thread_t thread); extern uint32_t thread_get_ipc_override(thread_t thread); /* @@ -287,7 +298,8 @@ typedef struct task_pend_token { tpt_update_coal_sfi :1, tpt_update_throttle :1, tpt_update_thread_sfi :1, - tpt_force_recompute_pri :1; + tpt_force_recompute_pri :1, + tpt_update_tg_ui_flag :1; } *task_pend_token_t; extern void task_policy_update_complete_unlocked(task_t task, task_pend_token_t pend_token); @@ -325,6 +337,14 @@ extern void thread_policy_create(thread_t thread); extern boolean_t task_is_daemon(task_t task); extern boolean_t task_is_app(task_t task); +#if CONFIG_EMBEDDED +/* Taskwatch related external interface */ +extern void thead_remove_taskwatch(thread_t thread); +extern void task_removewatchers(task_t task); +extern void task_watch_init(void); + +typedef struct task_watcher task_watch_t; +#endif /* CONFIG_EMBEDDED */ #if IMPORTANCE_INHERITANCE extern boolean_t task_is_marked_importance_donor(task_t task); diff --git a/osfmk/kern/printf.c b/osfmk/kern/printf.c index c93df6826..54a3220d3 100644 --- a/osfmk/kern/printf.c +++ b/osfmk/kern/printf.c @@ -141,7 +141,7 @@ * + print '+' if positive * blank print ' ' if positive * - * z signed hexadecimal + * z set length equal to size_t * r signed, 'radix' * n unsigned, 'radix' * @@ -159,6 +159,7 @@ #include #include #include +#include #include #include #include @@ -170,6 +171,14 @@ #include #include +#ifdef __x86_64__ +#include +#endif /* __x86_64__ */ + +#if __arm__ || __arm64__ +#include +#endif + #define isdigit(d) ((d) >= '0' && (d) <= '9') #define Ctod(c) ((c) - '0') @@ -331,7 +340,14 @@ __doprnt( } else if (c == 'q' || c == 'L') { long_long = 1; c = *++fmt; - } + } + + if (c == 'z' || c == 'Z') { + c = *++fmt; + if (sizeof(size_t) == sizeof(unsigned long)){ + long_long = 1; + } + } truncate = FALSE; capitals=0; /* Assume lower case printing */ @@ -518,17 +534,7 @@ __doprnt( base = 16; capitals=16; /* Print in upper case */ goto print_unsigned; - - case 'z': - truncate = _doprnt_truncates; - base = 16; - goto print_signed; - case 'Z': - base = 16; - capitals=16; /* Print in upper case */ - goto print_signed; - case 'r': truncate = _doprnt_truncates; case 'R': @@ -794,11 +800,11 @@ void conslog_putc( char c) { - if ((debug_mode && !disable_debug_output) || !disableConsoleOutput) + if (!disableConsoleOutput) cnputc(c); #ifdef MACH_BSD - if (debug_mode == 0) + if (!kernel_debugger_entry_count) log_putc(c); #endif } @@ -807,26 +813,37 @@ void cons_putc_locked( char c) { - if ((debug_mode && !disable_debug_output) || !disableConsoleOutput) + if (!disableConsoleOutput) cnputc(c); } static int vprintf_internal(const char *fmt, va_list ap_in, void *caller) { + cpu_data_t * cpu_data_p; if (fmt) { + struct console_printbuf_state info_data; + cpu_data_p = current_cpu_datap(); + va_list ap; va_copy(ap, ap_in); - - disable_preemption(); - _doprnt_log(fmt, &ap, cons_putc_locked, 16); - enable_preemption(); + /* + * for early boot printf()s console may not be setup, + * fallback to good old cnputc + */ + if (cpu_data_p->cpu_console_buf != NULL) { + console_printbuf_state_init(&info_data, TRUE, TRUE); + __doprnt(fmt, ap, console_printbuf_putc, &info_data, 16, TRUE); + console_printbuf_clear(&info_data); + } else { + disable_preemption(); + _doprnt_log(fmt, &ap, cons_putc_locked, 16); + enable_preemption(); + } va_end(ap); - if (debug_mode == 0) { - os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, fmt, ap_in, caller); - } + os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, fmt, ap_in, caller); } return 0; } @@ -855,26 +872,24 @@ vprintf(const char *fmt, va_list ap) void consdebug_putc(char c) { - if ((debug_mode && !disable_debug_output) || !disableConsoleOutput) + if (!disableConsoleOutput) cnputc(c); debug_putc(c); - if (!console_is_serial()) - if (!disable_serial_output) - PE_kputc(c); + if (!console_is_serial() && !disable_serial_output) + PE_kputc(c); } void consdebug_putc_unbuffered(char c) { - if ((debug_mode && !disable_debug_output) || !disableConsoleOutput) + if (!disableConsoleOutput) cnputc_unbuffered(c); debug_putc(c); - if (!console_is_serial()) - if (!disable_serial_output) + if (!console_is_serial() && !disable_serial_output) PE_kputc(c); } @@ -884,6 +899,24 @@ consdebug_log(char c) debug_putc(c); } +/* + * Append contents to the paniclog buffer but don't flush + * it. This is mainly used for writing the actual paniclog + * contents since flushing once for every line written + * would be prohibitively expensive for the paniclog + */ +int +paniclog_append_noflush(const char *fmt, ...) +{ + va_list listp; + + va_start(listp, fmt); + _doprnt_log(fmt, &listp, consdebug_putc, 16); + va_end(listp); + + return 0; +} + int kdb_printf(const char *fmt, ...) { @@ -892,6 +925,11 @@ kdb_printf(const char *fmt, ...) va_start(listp, fmt); _doprnt_log(fmt, &listp, consdebug_putc, 16); va_end(listp); + +#if CONFIG_EMBEDDED + paniclog_flush(); +#endif + return 0; } @@ -903,6 +941,11 @@ kdb_log(const char *fmt, ...) va_start(listp, fmt); _doprnt(fmt, &listp, consdebug_log, 16); va_end(listp); + +#if CONFIG_EMBEDDED + paniclog_flush(); +#endif + return 0; } @@ -914,9 +957,15 @@ kdb_printf_unbuffered(const char *fmt, ...) va_start(listp, fmt); _doprnt(fmt, &listp, consdebug_putc_unbuffered, 16); va_end(listp); + +#if CONFIG_EMBEDDED + paniclog_flush(); +#endif + return 0; } +#if !CONFIG_EMBEDDED static void copybyte(int c, void *arg) @@ -949,3 +998,4 @@ sprintf(char *buf, const char *fmt, ...) *copybyte_str = '\0'; return (int)strlen(buf); } +#endif /* !CONFIG_EMBEDDED */ diff --git a/osfmk/kern/priority.c b/osfmk/kern/priority.c index dcf7715bd..d61fb3584 100644 --- a/osfmk/kern/priority.c +++ b/osfmk/kern/priority.c @@ -81,6 +81,10 @@ #include /* for commpage_update_mach_approximate_time */ #endif +#if MONOTONIC +#include +#endif /* MONOTONIC */ + static void sched_update_thread_bucket(thread_t thread); /* @@ -122,13 +126,11 @@ thread_quantum_expire( */ ledger_credit(thread->t_ledger, task_ledgers.cpu_time, thread->quantum_remaining); ledger_credit(thread->t_threadledger, thread_ledgers.cpu_time, thread->quantum_remaining); -#ifdef CONFIG_BANK if (thread->t_bankledger) { ledger_credit(thread->t_bankledger, bank_ledgers.cpu_time, (thread->quantum_remaining - thread->t_deduct_bank_ledger_time)); } thread->t_deduct_bank_ledger_time = 0; -#endif ctime = mach_absolute_time(); @@ -136,6 +138,10 @@ thread_quantum_expire( commpage_update_mach_approximate_time(ctime); #endif +#if MONOTONIC + mt_sched_update(thread); +#endif /* MONOTONIC */ + thread_lock(thread); /* @@ -176,12 +182,7 @@ thread_quantum_expire( if (thread->sched_mode != TH_MODE_REALTIME) SCHED(quantum_expire)(thread); - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - - /* Tell platform layer that we are still running this thread */ - urgency = thread_get_urgency(thread, &ignore1, &ignore2); - machine_thread_going_on_core(thread, urgency, 0, 0); + processor_state_update_from_thread(processor, thread); /* * This quantum is up, give this thread another. @@ -206,23 +207,49 @@ thread_quantum_expire( PROCESSOR_DATA(processor, thread_timer)); } + processor->quantum_end = ctime + thread->quantum_remaining; /* - * Context switch check. + * Context switch check + * + * non-urgent flags don't affect kernel threads, so upgrade to urgent + * to ensure that rebalancing and non-recommendation kick in quickly. */ - if ((preempt = csw_check(processor, AST_QUANTUM)) != AST_NONE) + + ast_t check_reason = AST_QUANTUM; + if (thread->task == kernel_task) + check_reason |= AST_URGENT; + + if ((preempt = csw_check(processor, check_reason)) != AST_NONE) ast_on(preempt); + /* + * AST_KEVENT does not send an IPI when setting the AST, + * to avoid waiting for the next context switch to propagate the AST, + * the AST is propagated here at quantum expiration. + */ + ast_propagate(thread); + thread_unlock(thread); - timer_call_enter1(&processor->quantum_timer, thread, - processor->quantum_end, TIMER_CALL_SYS_CRITICAL | TIMER_CALL_LOCAL); + timer_call_quantum_timer_enter(&processor->quantum_timer, thread, + processor->quantum_end, ctime); + + /* Tell platform layer that we are still running this thread */ + urgency = thread_get_urgency(thread, &ignore1, &ignore2); + machine_thread_going_on_core(thread, urgency, 0, 0, ctime); + machine_switch_perfcontrol_state_update(QUANTUM_EXPIRY, ctime, + 0, thread); #if defined(CONFIG_SCHED_TIMESHARE_CORE) sched_timeshare_consider_maintenance(ctime); #endif /* CONFIG_SCHED_TIMESHARE_CORE */ +#if __arm__ || __arm64__ + if (thread->sched_mode == TH_MODE_REALTIME) + sched_consider_recommended_cores(ctime, thread); +#endif /* __arm__ || __arm64__ */ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_QUANTUM_EXPIRED) | DBG_FUNC_END, preempt, 0, 0, 0, 0); } @@ -241,14 +268,37 @@ void sched_set_thread_base_priority(thread_t thread, int priority) { assert(priority >= MINPRI); + uint64_t ctime = 0; if (thread->sched_mode == TH_MODE_REALTIME) assert(priority <= BASEPRI_RTQUEUES); else assert(priority < BASEPRI_RTQUEUES); + int old_base_pri = thread->base_pri; thread->base_pri = priority; + if ((thread->state & TH_RUN) == TH_RUN) { + assert(thread->last_made_runnable_time != THREAD_NOT_RUNNABLE); + ctime = mach_approximate_time(); + thread->last_basepri_change_time = ctime; + } else { + assert(thread->last_basepri_change_time == THREAD_NOT_RUNNABLE); + assert(thread->last_made_runnable_time == THREAD_NOT_RUNNABLE); + } + + /* + * Currently the perfcontrol_attr depends on the base pri of the + * thread. Therefore, we use this function as the hook for the + * perfcontrol callout. + */ + if (thread == current_thread() && old_base_pri != priority) { + if (!ctime) { + ctime = mach_approximate_time(); + } + machine_switch_perfcontrol_state_update(PERFCONTROL_ATTR_UPDATE, + ctime, PERFCONTROL_CALLOUT_WAKE_UNSAFE, thread); + } sched_update_thread_bucket(thread); thread_recompute_sched_pri(thread, FALSE); @@ -377,6 +427,39 @@ static struct shift_data sched_decay_shifts[SCHED_DECAY_TICKS] = { */ extern int sched_pri_decay_band_limit; +#ifdef CONFIG_EMBEDDED + +int +sched_compute_timeshare_priority(thread_t thread) +{ + int decay_amount = (thread->sched_usage >> thread->pri_shift); + int decay_limit = sched_pri_decay_band_limit; + + if (thread->base_pri > BASEPRI_FOREGROUND) { + decay_limit += (thread->base_pri - BASEPRI_FOREGROUND); + } + + if (decay_amount > decay_limit) { + decay_amount = decay_limit; + } + + /* start with base priority */ + int priority = thread->base_pri - decay_amount; + + if (priority < MAXPRI_THROTTLE) { + if (thread->task->max_priority > MAXPRI_THROTTLE) { + priority = MAXPRI_THROTTLE; + } else if (priority < MINPRI_USER) { + priority = MINPRI_USER; + } + } else if (priority > MAXPRI_KERNEL) { + priority = MAXPRI_KERNEL; + } + + return priority; +} + +#else /* CONFIG_EMBEDDED */ int sched_compute_timeshare_priority(thread_t thread) @@ -392,6 +475,7 @@ sched_compute_timeshare_priority(thread_t thread) return priority; } +#endif /* CONFIG_EMBEDDED */ /* * can_update_priority diff --git a/osfmk/kern/processor.c b/osfmk/kern/processor.c index b0a13fb08..5aad73e37 100644 --- a/osfmk/kern/processor.c +++ b/osfmk/kern/processor.c @@ -80,6 +80,7 @@ #include + /* * Exported interface */ @@ -111,6 +112,10 @@ processor_t master_processor; int master_cpu = 0; boolean_t sched_stats_active = FALSE; +processor_t processor_array[MAX_SCHED_CPUS] = { 0 }; + + + void processor_bootstrap(void) { @@ -145,16 +150,14 @@ processor_init( spl_t s; if (processor != master_processor) { - /* Scheduler state deferred until sched_init() */ + /* Scheduler state for master_processor initialized in sched_init() */ SCHED(processor_init)(processor); } processor->state = PROCESSOR_OFF_LINE; processor->active_thread = processor->next_thread = processor->idle_thread = THREAD_NULL; processor->processor_set = pset; - processor->current_pri = MINPRI; - processor->current_thmode = TH_MODE_NONE; - processor->current_sfi_class = SFI_CLASS_KERNEL; + processor_state_update_idle(processor); processor->starting_pri = MINPRI; processor->cpu_id = cpu_id; timer_call_setup(&processor->quantum_timer, thread_quantum_expire, processor); @@ -171,6 +174,7 @@ processor_init( s = splsched(); pset_lock(pset); + bit_set(pset->cpu_bitmask, cpu_id); if (pset->cpu_set_count++ == 0) pset->cpu_set_low = pset->cpu_set_hi = cpu_id; else { @@ -187,6 +191,8 @@ processor_init( processor_list_tail->processor_list = processor; processor_list_tail = processor; processor_count++; + assert(cpu_id < MAX_SCHED_CPUS); + processor_array[cpu_id] = processor; simple_unlock(&processor_list_lock); } @@ -220,6 +226,34 @@ processor_pset( return (processor->processor_set); } +void +processor_state_update_idle(processor_t processor) +{ + processor->current_pri = IDLEPRI; + processor->current_sfi_class = SFI_CLASS_KERNEL; + processor->current_recommended_pset_type = PSET_SMP; + processor->current_perfctl_class = PERFCONTROL_CLASS_IDLE; +} + +void +processor_state_update_from_thread(processor_t processor, thread_t thread) +{ + processor->current_pri = thread->sched_pri; + processor->current_sfi_class = thread->sfi_class; + processor->current_recommended_pset_type = recommended_pset_type(thread); + processor->current_perfctl_class = thread_get_perfcontrol_class(thread); +} + +void +processor_state_update_explicit(processor_t processor, int pri, sfi_class_id_t sfi_class, + pset_cluster_type_t pset_type, perfcontrol_class_t perfctl_class) +{ + processor->current_pri = pri; + processor->current_sfi_class = sfi_class; + processor->current_recommended_pset_type = pset_type; + processor->current_perfctl_class = perfctl_class; +} + pset_node_t pset_node_root(void) { @@ -253,6 +287,33 @@ pset_create( return (pset); } +/* + * Find processor set in specified node with specified cluster_id. + * Returns default_pset if not found. + */ +processor_set_t +pset_find( + uint32_t cluster_id, + processor_set_t default_pset) +{ + simple_lock(&pset_node_lock); + pset_node_t node = &pset_node0; + processor_set_t pset = NULL; + + do { + pset = node->psets; + while (pset != NULL) { + if (pset->pset_cluster_id == cluster_id) + break; + pset = pset->pset_list; + } + } while ((node = node->node_list) != NULL); + simple_unlock(&pset_node_lock); + if (pset == NULL) + return default_pset; + return (pset); +} + /* * Initialize the given processor_set structure. */ @@ -262,26 +323,34 @@ pset_init( pset_node_t node) { if (pset != &pset0) { - /* Scheduler state deferred until sched_init() */ + /* Scheduler state for pset0 initialized in sched_init() */ SCHED(pset_init)(pset); + SCHED(rt_init)(pset); } queue_init(&pset->active_queue); queue_init(&pset->idle_queue); queue_init(&pset->idle_secondary_queue); + queue_init(&pset->unused_queue); pset->online_processor_count = 0; + pset->active_processor_count = 0; + pset->load_average = 0; pset->cpu_set_low = pset->cpu_set_hi = 0; pset->cpu_set_count = 0; + pset->cpu_bitmask = 0; pset->recommended_bitmask = ~0ULL; pset->pending_AST_cpu_mask = 0; #if defined(CONFIG_SCHED_DEFERRED_AST) pset->pending_deferred_AST_cpu_mask = 0; #endif + pset->pending_spill_cpu_mask = 0; pset_lock_init(pset); pset->pset_self = IP_NULL; pset->pset_name_self = IP_NULL; pset->pset_list = PROCESSOR_SET_NULL; pset->node = node; + pset->pset_cluster_type = PSET_SMP; + pset->pset_cluster_id = 0; } kern_return_t @@ -1071,6 +1140,15 @@ processor_set_threads( { return KERN_FAILURE; } +#elif defined(CONFIG_EMBEDDED) +kern_return_t +processor_set_threads( + __unused processor_set_t pset, + __unused thread_array_t *thread_list, + __unused mach_msg_type_number_t *count) +{ + return KERN_NOT_SUPPORTED; +} #else kern_return_t processor_set_threads( @@ -1127,3 +1205,10 @@ __unused processor_set_t pset) { return; } + +pset_cluster_type_t +recommended_pset_type(thread_t thread) +{ + (void)thread; + return PSET_SMP; +} diff --git a/osfmk/kern/processor.h b/osfmk/kern/processor.h index abf663053..09caf6a7f 100644 --- a/osfmk/kern/processor.h +++ b/osfmk/kern/processor.h @@ -82,15 +82,23 @@ #include #include +typedef enum { + PSET_SMP, +} pset_cluster_type_t; + struct processor_set { queue_head_t active_queue; /* active processors */ queue_head_t idle_queue; /* idle processors */ - queue_head_t idle_secondary_queue; /* idle secondary processors */ + queue_head_t idle_secondary_queue; /* idle secondary processors */ + queue_head_t unused_queue; /* processors not recommended by CLPC */ int online_processor_count; + int active_processor_count; + int load_average; int cpu_set_low, cpu_set_hi; int cpu_set_count; + uint64_t cpu_bitmask; uint64_t recommended_bitmask; #if __SMP__ @@ -100,6 +108,7 @@ struct processor_set { #if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ) struct run_queue pset_runq; /* runq for this processor set */ #endif + struct rt_queue rt_runq; /* realtime runq for this processor set */ #if defined(CONFIG_SCHED_TRADITIONAL) int pset_runq_bound_count; @@ -110,7 +119,7 @@ struct processor_set { uint64_t pending_AST_cpu_mask; #if defined(CONFIG_SCHED_DEFERRED_AST) /* - * A seperate mask, for ASTs that we may be able to cancel. This is dependent on + * A separate mask, for ASTs that we may be able to cancel. This is dependent on * some level of support for requesting an AST on a processor, and then quashing * that request later. * @@ -122,12 +131,15 @@ struct processor_set { */ uint64_t pending_deferred_AST_cpu_mask; #endif + uint64_t pending_spill_cpu_mask; struct ipc_port * pset_self; /* port for operations */ struct ipc_port * pset_name_self; /* port for information */ processor_set_t pset_list; /* chain of associated psets */ - pset_node_t node; + pset_node_t node; + uint32_t pset_cluster_id; + pset_cluster_type_t pset_cluster_type; }; extern struct processor_set pset0; @@ -161,11 +173,12 @@ struct processor { processor_set_t processor_set; /* assigned set */ - int current_pri; /* priority of current thread */ - sched_mode_t current_thmode; /* sched mode of current thread */ + int current_pri; /* priority of current thread */ sfi_class_id_t current_sfi_class; /* SFI class of current thread */ + perfcontrol_class_t current_perfctl_class; /* Perfcontrol class for current thread */ int starting_pri; /* priority of current thread as it was when scheduled */ - int cpu_id; /* platform numeric id */ + pset_cluster_type_t current_recommended_pset_type; /* Cluster type recommended for current thread */ + int cpu_id; /* platform numeric id */ timer_call_data_t quantum_timer; /* timer for quantum expiration */ uint64_t quantum_end; /* time when current quantum ends */ @@ -196,9 +209,11 @@ struct processor { }; extern processor_t processor_list; -extern unsigned int processor_count; decl_simple_lock_data(extern,processor_list_lock) +#define MAX_SCHED_CPUS 64 /* Maximum number of CPUs supported by the scheduler. bits.h:bitmap_*() macros need to be used to support greater than 64 */ +extern processor_t processor_array[MAX_SCHED_CPUS]; /* array indexed by cpuid */ + extern uint32_t processor_avail_count; extern processor_t master_processor; @@ -268,10 +283,18 @@ extern processor_t current_processor(void); #define pset_lock(p) simple_lock(&(p)->sched_lock) #define pset_unlock(p) simple_unlock(&(p)->sched_lock) #define pset_lock_init(p) simple_lock_init(&(p)->sched_lock, 0) + +#define rt_lock_lock(p) simple_lock(&SCHED(rt_runq)(p)->rt_lock) +#define rt_lock_unlock(p) simple_unlock(&SCHED(rt_runq)(p)->rt_lock) +#define rt_lock_init(p) simple_lock_init(&SCHED(rt_runq)(p)->rt_lock, 0) #else #define pset_lock(p) do { (void)p; } while(0) #define pset_unlock(p) do { (void)p; } while(0) #define pset_lock_init(p) do { (void)p; } while(0) + +#define rt_lock_lock(p) do { (void)p; } while(0) +#define rt_lock_unlock(p) do { (void)p; } while(0) +#define rt_lock_init(p) do { (void)p; } while(0) #endif extern void processor_bootstrap(void); @@ -303,6 +326,10 @@ extern void pset_init( processor_set_t pset, pset_node_t node); +extern processor_set_t pset_find( + uint32_t cluster_id, + processor_set_t default_pset); + extern kern_return_t processor_info_count( processor_flavor_t flavor, mach_msg_type_number_t *count); @@ -328,6 +355,20 @@ extern kern_return_t processor_set_things( mach_msg_type_number_t *count, int type); +extern pset_cluster_type_t recommended_pset_type(thread_t thread); + +inline static bool +pset_is_recommended(processor_set_t pset) +{ + return ((pset->recommended_bitmask & pset->cpu_bitmask) != 0); +} + +extern void processor_state_update_idle(processor_t processor); +extern void processor_state_update_from_thread(processor_t processor, thread_t thread); +extern void processor_state_update_explicit(processor_t processor, int pri, + sfi_class_id_t sfi_class, pset_cluster_type_t pset_type, + perfcontrol_class_t perfctl_class); + #else /* MACH_KERNEL_PRIVATE */ __BEGIN_DECLS @@ -344,6 +385,7 @@ __END_DECLS #ifdef KERNEL_PRIVATE __BEGIN_DECLS +extern unsigned int processor_count; extern processor_t cpu_to_processor(int cpu); __END_DECLS diff --git a/osfmk/kern/processor_data.c b/osfmk/kern/processor_data.c index 9f81a2d18..a62dbdb14 100644 --- a/osfmk/kern/processor_data.c +++ b/osfmk/kern/processor_data.c @@ -33,6 +33,7 @@ #include #include +#include void processor_data_init( @@ -43,4 +44,6 @@ processor_data_init( timer_init(&PROCESSOR_DATA(processor, idle_state)); timer_init(&PROCESSOR_DATA(processor, system_state)); timer_init(&PROCESSOR_DATA(processor, user_state)); + + PROCESSOR_DATA(processor, debugger_state).db_current_op = DBOP_NONE; } diff --git a/osfmk/kern/processor_data.h b/osfmk/kern/processor_data.h index 799a31dc7..3f563c08f 100644 --- a/osfmk/kern/processor_data.h +++ b/osfmk/kern/processor_data.h @@ -40,6 +40,7 @@ #include #include +#include struct processor_sched_statistics { uint32_t csw_count; @@ -97,6 +98,17 @@ struct processor_data { uint64_t wakeups_issued_total; /* Count of thread wakeups issued * by this processor */ + struct debugger_state { + debugger_op db_current_op; + const char *db_message; + const char *db_panic_str; + va_list *db_panic_args; + uint64_t db_panic_options; + boolean_t db_proceed_on_sync_failure; + uint32_t db_entry_count; /* incremented whenever we panic or call Debugger (current CPU panic level) */ + kern_return_t db_op_return; + unsigned long db_panic_caller; + } debugger_state; }; typedef struct processor_data processor_data_t; diff --git a/osfmk/kern/queue.h b/osfmk/kern/queue.h index dc99d000f..1cdcd4f1b 100644 --- a/osfmk/kern/queue.h +++ b/osfmk/kern/queue.h @@ -197,7 +197,18 @@ struct queue_entry { struct queue_entry *next; /* next element */ struct queue_entry *prev; /* previous element */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Since this type is so often cast to various 64-bit aligned types + * aligning it to 64-bits will avoid -wcast-align without needing + * to disable it entirely. The impact on memory footprint should be + * negligible. + */ +} __attribute__ ((aligned (8))); +#else }; +#endif typedef struct queue_entry *queue_t; typedef struct queue_entry queue_head_t; diff --git a/osfmk/kern/sched.h b/osfmk/kern/sched.h index f1225c9d2..d8fe8ee36 100644 --- a/osfmk/kern/sched.h +++ b/osfmk/kern/sched.h @@ -69,6 +69,7 @@ #include #include +#include #include #include #include @@ -144,7 +145,10 @@ #define BASEPRI_REALTIME (MAXPRI - (NRQS / 4) + 1) /* 96 */ #define MAXPRI_KERNEL (BASEPRI_REALTIME - 1) /* 95 */ -#define BASEPRI_PREEMPT (MAXPRI_KERNEL - 2) /* 93 */ +#define BASEPRI_PREEMPT_HIGH (BASEPRI_PREEMPT + 1) /* 93 */ +#define BASEPRI_PREEMPT (MAXPRI_KERNEL - 3) /* 92 */ +#define BASEPRI_VM (BASEPRI_PREEMPT - 1) /* 91 */ + #define BASEPRI_KERNEL (MINPRI_KERNEL + 1) /* 81 */ #define MINPRI_KERNEL (MAXPRI_KERNEL - (NRQS / 8) + 1) /* 80 */ @@ -223,11 +227,14 @@ rq_bitmap_clear(bitmap_t *map, u_int n) #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) || defined(CONFIG_SCHED_PROTO) */ struct rt_queue { - int count; /* # of threads total */ + _Atomic int count; /* # of threads total */ queue_head_t queue; /* all runnable RT threads */ - +#if __SMP__ + decl_simple_lock_data(,rt_lock) +#endif struct runq_stats runq_stats; }; +typedef struct rt_queue *rt_queue_t; #if defined(CONFIG_SCHED_GRRR_CORE) @@ -275,7 +282,9 @@ struct grrr_run_queue { #endif /* defined(CONFIG_SCHED_GRRR_CORE) */ -extern struct rt_queue rt_runq; +extern int rt_runq_count(processor_set_t); +extern void rt_runq_count_incr(processor_set_t); +extern void rt_runq_count_decr(processor_set_t); #if defined(CONFIG_SCHED_MULTIQ) sched_group_t sched_group_create(void); diff --git a/osfmk/kern/sched_dualq.c b/osfmk/kern/sched_dualq.c index 48ff5a038..f465d803e 100644 --- a/osfmk/kern/sched_dualq.c +++ b/osfmk/kern/sched_dualq.c @@ -126,6 +126,20 @@ const struct sched_dispatch_table sched_dualq_dispatch = { .direct_dispatch_to_idle_processors = FALSE, .multiple_psets_enabled = TRUE, .sched_groups_enabled = FALSE, + .avoid_processor_enabled = FALSE, + .thread_avoid_processor = NULL, + .processor_balance = sched_SMT_balance, + + .rt_runq = sched_rtglobal_runq, + .rt_init = sched_rtglobal_init, + .rt_queue_shutdown = sched_rtglobal_queue_shutdown, + .rt_runq_scan = sched_rtglobal_runq_scan, + .rt_runq_count_sum = sched_rtglobal_runq_count_sum, + + .qos_max_parallelism = sched_qos_max_parallelism, + .check_spill = sched_check_spill, + .ipi_policy = sched_ipi_policy, + .thread_should_yield = sched_thread_should_yield, }; __attribute__((always_inline)) @@ -271,9 +285,6 @@ sched_dualq_processor_queue_has_priority(processor_t processor, run_queue_t main_runq = dualq_main_runq(processor); run_queue_t bound_runq = dualq_bound_runq(processor); - if (main_runq->count == 0 && bound_runq->count == 0) - return FALSE; - int qpri = MAX(main_runq->highq, bound_runq->highq); if (gte) diff --git a/osfmk/kern/sched_grrr.c b/osfmk/kern/sched_grrr.c index 10441edd3..8cc140183 100644 --- a/osfmk/kern/sched_grrr.c +++ b/osfmk/kern/sched_grrr.c @@ -222,6 +222,20 @@ const struct sched_dispatch_table sched_grrr_dispatch = { .direct_dispatch_to_idle_processors = TRUE, .multiple_psets_enabled = TRUE, .sched_groups_enabled = FALSE, + .avoid_processor_enabled = FALSE, + .thread_avoid_processor = NULL, + .processor_balance = sched_SMT_balance, + + .rt_runq = sched_rtglobal_runq, + .rt_init = sched_rtglobal_init, + .rt_queue_shutdown = sched_rtglobal_queue_shutdown, + .rt_runq_scan = sched_rtglobal_runq_scan, + .rt_runq_count_sum = sched_rtglobal_runq_count_sum, + + .qos_max_parallelism = sched_qos_max_parallelism, + .check_spill = sched_check_spill, + .ipi_policy = sched_ipi_policy, + .thread_should_yield = sched_thread_should_yield, }; extern int max_unsafe_quanta; diff --git a/osfmk/kern/sched_multiq.c b/osfmk/kern/sched_multiq.c index d37de1f3c..24df80d88 100644 --- a/osfmk/kern/sched_multiq.c +++ b/osfmk/kern/sched_multiq.c @@ -319,6 +319,20 @@ const struct sched_dispatch_table sched_multiq_dispatch = { .direct_dispatch_to_idle_processors = FALSE, .multiple_psets_enabled = FALSE, .sched_groups_enabled = TRUE, + .avoid_processor_enabled = FALSE, + .thread_avoid_processor = NULL, + .processor_balance = sched_SMT_balance, + + .rt_runq = sched_rtglobal_runq, + .rt_init = sched_rtglobal_init, + .rt_queue_shutdown = sched_rtglobal_queue_shutdown, + .rt_runq_scan = sched_rtglobal_runq_scan, + .rt_runq_count_sum = sched_rtglobal_runq_count_sum, + + .qos_max_parallelism = sched_qos_max_parallelism, + .check_spill = sched_check_spill, + .ipi_policy = sched_ipi_policy, + .thread_should_yield = sched_thread_should_yield, }; @@ -1218,9 +1232,6 @@ sched_multiq_processor_queue_has_priority( run_queue_t main_runq = multiq_main_entryq(processor); run_queue_t bound_runq = multiq_bound_runq(processor); - if (main_runq->count == 0 && bound_runq->count == 0) - return FALSE; - int qpri = MAX(main_runq->highq, bound_runq->highq); if (gte) @@ -1457,5 +1468,3 @@ sched_multiq_thread_update_scan(sched_update_scan_context_t scan_context) } while (restart_needed); } - - diff --git a/osfmk/kern/sched_prim.c b/osfmk/kern/sched_prim.c index f2052bb3f..bd590a042 100644 --- a/osfmk/kern/sched_prim.c +++ b/osfmk/kern/sched_prim.c @@ -76,6 +76,7 @@ #include #include #include +#include #ifdef CONFIG_MACH_APPROXIMATE_TIME #include @@ -92,6 +93,9 @@ #include #include #include +#if MONOTONIC +#include +#endif /* MONOTONIC */ #include #include #include @@ -108,30 +112,34 @@ #include #include #include +#include #include +#include +#include #include #include #include - +#include #include +#include +#include -struct rt_queue rt_runq; +int rt_runq_count(processor_set_t pset) +{ + return atomic_load_explicit(&SCHED(rt_runq)(pset)->count, memory_order_relaxed); +} -uintptr_t sched_thread_on_rt_queue = (uintptr_t)0xDEAFBEE0; +void rt_runq_count_incr(processor_set_t pset) +{ + atomic_fetch_add_explicit(&SCHED(rt_runq)(pset)->count, 1, memory_order_relaxed); +} -/* Lock RT runq, must be done with interrupts disabled (under splsched()) */ -#if __SMP__ -decl_simple_lock_data(static,rt_lock); -#define rt_lock_init() simple_lock_init(&rt_lock, 0) -#define rt_lock_lock() simple_lock(&rt_lock) -#define rt_lock_unlock() simple_unlock(&rt_lock) -#else -#define rt_lock_init() do { } while(0) -#define rt_lock_lock() do { } while(0) -#define rt_lock_unlock() do { } while(0) -#endif +void rt_runq_count_decr(processor_set_t pset) +{ + atomic_fetch_sub_explicit(&SCHED(rt_runq)(pset)->count, 1, memory_order_relaxed); +} #define DEFAULT_PREEMPTION_RATE 100 /* (1/s) */ int default_preemption_rate = DEFAULT_PREEMPTION_RATE; @@ -198,6 +206,16 @@ uint64_t timer_deadline_tracking_bin_2; thread_t sched_maintenance_thread; +#if __arm__ || __arm64__ +/* interrupts disabled lock to guard recommended cores state */ +decl_simple_lock_data(static,sched_recommended_cores_lock); +static void sched_recommended_cores_maintenance(void); +static void sched_update_recommended_cores(uint32_t recommended_cores); + +uint64_t perfcontrol_failsafe_starvation_threshold; +extern char *proc_name_address(struct proc *p); + +#endif /* __arm__ || __arm64__ */ uint64_t sched_one_second_interval; @@ -210,11 +228,6 @@ static void preempt_pri_init(void); #endif /* CONFIG_SCHED_TIMESHARE_CORE */ -static thread_t thread_select( - thread_t thread, - processor_t processor, - ast_t reason); - #if CONFIG_SCHED_IDLE_IN_PLACE static thread_t thread_select_idle( thread_t thread, @@ -235,9 +248,6 @@ static void processor_setrun( thread_t thread, integer_t options); -static void -sched_realtime_init(void); - static void sched_realtime_timebase_init(void); @@ -285,38 +295,25 @@ const struct sched_dispatch_table *sched_current_dispatch = NULL; */ char sched_string[SCHED_STRING_MAX_LENGTH]; -uint32_t sched_debug_flags; +uint32_t sched_debug_flags = SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS; /* Global flag which indicates whether Background Stepper Context is enabled */ static int cpu_throttle_enabled = 1; -void -sched_init(void) +#if DEBUG + +/* Since using the indirect function dispatch table has a negative impact on + * context switch performance, only allow DEBUG kernels to use that mechanism. + */ +static void +sched_init_override(void) { char sched_arg[SCHED_STRING_MAX_LENGTH] = { '\0' }; /* Check for runtime selection of the scheduler algorithm */ if (!PE_parse_boot_argn("sched", sched_arg, sizeof (sched_arg))) { - /* If no boot-args override, look in device tree */ - if (!PE_get_default("kern.sched", sched_arg, - SCHED_STRING_MAX_LENGTH)) { - sched_arg[0] = '\0'; - } + sched_arg[0] = '\0'; } - - - if (!PE_parse_boot_argn("sched_pri_decay_limit", &sched_pri_decay_band_limit, sizeof(sched_pri_decay_band_limit))) { - /* No boot-args, check in device tree */ - if (!PE_get_default("kern.sched_pri_decay_limit", - &sched_pri_decay_band_limit, - sizeof(sched_pri_decay_band_limit))) { - /* Allow decay all the way to normal limits */ - sched_pri_decay_band_limit = DEFAULT_DECAY_BAND_LIMIT; - } - } - - kprintf("Setting scheduler priority decay band limit %d\n", sched_pri_decay_band_limit); - if (strlen(sched_arg) > 0) { if (0) { /* Allow pattern below */ @@ -326,14 +323,6 @@ sched_init(void) } else if (0 == strcmp(sched_arg, sched_traditional_with_pset_runqueue_dispatch.sched_name)) { sched_current_dispatch = &sched_traditional_with_pset_runqueue_dispatch; #endif -#if defined(CONFIG_SCHED_PROTO) - } else if (0 == strcmp(sched_arg, sched_proto_dispatch.sched_name)) { - sched_current_dispatch = &sched_proto_dispatch; -#endif -#if defined(CONFIG_SCHED_GRRR) - } else if (0 == strcmp(sched_arg, sched_grrr_dispatch.sched_name)) { - sched_current_dispatch = &sched_grrr_dispatch; -#endif #if defined(CONFIG_SCHED_MULTIQ) } else if (0 == strcmp(sched_arg, sched_multiq_dispatch.sched_name)) { sched_current_dispatch = &sched_multiq_dispatch; @@ -355,25 +344,43 @@ sched_init(void) sched_current_dispatch = &sched_multiq_dispatch; #elif defined(CONFIG_SCHED_TRADITIONAL) sched_current_dispatch = &sched_traditional_with_pset_runqueue_dispatch; -#elif defined(CONFIG_SCHED_PROTO) - sched_current_dispatch = &sched_proto_dispatch; -#elif defined(CONFIG_SCHED_GRRR) - sched_current_dispatch = &sched_grrr_dispatch; #else #error No default scheduler implementation #endif kprintf("Scheduler: Default of %s\n", SCHED(sched_name)); } +} - strlcpy(sched_string, SCHED(sched_name), sizeof(sched_string)); +#endif /* DEBUG */ +void +sched_init(void) +{ +#if DEBUG + sched_init_override(); +#else /* DEBUG */ + kprintf("Scheduler: Default of %s\n", SCHED(sched_name)); +#endif /* DEBUG */ + + if (!PE_parse_boot_argn("sched_pri_decay_limit", &sched_pri_decay_band_limit, sizeof(sched_pri_decay_band_limit))) { + /* No boot-args, check in device tree */ + if (!PE_get_default("kern.sched_pri_decay_limit", + &sched_pri_decay_band_limit, + sizeof(sched_pri_decay_band_limit))) { + /* Allow decay all the way to normal limits */ + sched_pri_decay_band_limit = DEFAULT_DECAY_BAND_LIMIT; + } + } + + kprintf("Setting scheduler priority decay band limit %d\n", sched_pri_decay_band_limit); + if (PE_parse_boot_argn("sched_debug", &sched_debug_flags, sizeof(sched_debug_flags))) { kprintf("Scheduler: Debug flags 0x%08x\n", sched_debug_flags); } - + strlcpy(sched_string, SCHED(sched_name), sizeof(sched_string)); + SCHED(init)(); - sched_realtime_init(); - ast_init(); + SCHED(rt_init)(&pset0); sched_timer_deadline_tracking_init(); SCHED(pset_init)(&pset0); @@ -467,17 +474,48 @@ sched_timeshare_timebase_init(void) default_timeshare_computation = std_quantum / 2; default_timeshare_constraint = std_quantum; +#if __arm__ || __arm64__ + perfcontrol_failsafe_starvation_threshold = (2 * sched_tick_interval); +#endif /* __arm__ || __arm64__ */ } #endif /* CONFIG_SCHED_TIMESHARE_CORE */ -static void -sched_realtime_init(void) +void +pset_rt_init(processor_set_t pset) +{ + rt_lock_init(pset); + + pset->rt_runq.count = 0; + queue_init(&pset->rt_runq.queue); + memset(&pset->rt_runq.runq_stats, 0, sizeof pset->rt_runq.runq_stats); +} + +rt_queue_t +sched_rtglobal_runq(processor_set_t pset) { - rt_lock_init(); + (void)pset; + + return &pset0.rt_runq; +} + +void +sched_rtglobal_init(processor_set_t pset) +{ + if (pset == &pset0) { + return pset_rt_init(pset); + } + + /* Only pset0 rt_runq is used, so make it easy to detect + * buggy accesses to others. + */ + memset(&pset->rt_runq, 0xfd, sizeof pset->rt_runq); +} - rt_runq.count = 0; - queue_init(&rt_runq.queue); +void +sched_rtglobal_queue_shutdown(processor_t processor) +{ + (void)processor; } static void @@ -498,6 +536,23 @@ sched_realtime_timebase_init(void) } +void +sched_check_spill(processor_set_t pset, thread_t thread) +{ + (void)pset; + (void)thread; + + return; +} + +bool +sched_thread_should_yield(processor_t processor, thread_t thread) +{ + (void)thread; + + return (!SCHED(processor_queue_empty)(processor) || rt_runq_count(processor->processor_set) > 0); +} + #if defined(CONFIG_SCHED_TIMESHARE_CORE) /* @@ -630,7 +685,7 @@ thread_unblock( if (!(thread->state & TH_RUN)) { thread->state |= TH_RUN; - thread->last_made_runnable_time = mach_approximate_time(); + thread->last_made_runnable_time = thread->last_basepri_change_time = mach_approximate_time(); ready_for_runq = TRUE; @@ -857,28 +912,6 @@ thread_interrupt_level( return result; } -/* - * Check to see if an assert wait is possible, without actually doing one. - * This is used by debug code in locks and elsewhere to verify that it is - * always OK to block when trying to take a blocking lock (since waiting - * for the actual assert_wait to catch the case may make it hard to detect - * this case. - */ -boolean_t -assert_wait_possible(void) -{ - - thread_t thread; - -#if DEBUG - if(debug_mode) return TRUE; /* Always succeed in debug mode */ -#endif - - thread = current_thread(); - - return (thread == NULL || waitq_wait_possible(thread)); -} - /* * assert_wait: * @@ -1407,6 +1440,9 @@ thread_wakeup_thread( if (__improbable(event == NO_EVENT)) panic("%s() called with NO_EVENT", __func__); + if (__improbable(thread == THREAD_NULL)) + panic("%s() called with THREAD_NULL", __func__); + struct waitq *wq = global_eventq(event); return waitq_wakeup64_thread(wq, CAST_EVENT64_T(event), thread, THREAD_AWAKENED); @@ -1679,7 +1715,7 @@ int sched_smt_balance = 1; #if __SMP__ /* Invoked with pset locked, returns with pset unlocked */ -static void +void sched_SMT_balance(processor_t cprocessor, processor_set_t cpset) { processor_t ast_processor = NULL; @@ -1702,15 +1738,19 @@ sched_SMT_balance(processor_t cprocessor, processor_set_t cpset) { processor_t sprocessor; + sched_ipi_type_t ipi_type = SCHED_IPI_NONE; qe_foreach_element(sprocessor, &cpset->active_queue, processor_queue) { if ((sprocessor->state == PROCESSOR_RUNNING) && (sprocessor->processor_primary != sprocessor) && (sprocessor->processor_primary->state == PROCESSOR_RUNNING) && - (sprocessor->current_pri < BASEPRI_RTQUEUES) && - ((cpset->pending_AST_cpu_mask & (1ULL << sprocessor->cpu_id)) == 0)) { - assert(sprocessor != cprocessor); - ast_processor = sprocessor; - break; + (sprocessor->current_pri < BASEPRI_RTQUEUES)) { + + ipi_type = sched_ipi_action(sprocessor, NULL, false, SCHED_IPI_EVENT_SMT_REBAL); + if (ipi_type != SCHED_IPI_NONE) { + assert(sprocessor != cprocessor); + ast_processor = sprocessor; + break; + } } } @@ -1719,9 +1759,16 @@ smt_balance_exit: if (ast_processor) { KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_SMT_BALANCE), ast_processor->cpu_id, ast_processor->state, ast_processor->processor_primary->state, 0, 0); - cause_ast_check(ast_processor); + sched_ipi_perform(ast_processor, ipi_type); } } +#else +/* Invoked with pset locked, returns with pset unlocked */ +void +sched_SMT_balance(__unused processor_t cprocessor, processor_set_t cpset) +{ + pset_unlock(cpset); +} #endif /* __SMP__ */ /* @@ -1732,10 +1779,9 @@ smt_balance_exit: * May select the current thread, which must be locked. */ static thread_t -thread_select( - thread_t thread, - processor_t processor, - ast_t reason) +thread_select(thread_t thread, + processor_t processor, + ast_t *reason) { processor_set_t pset = processor->processor_set; thread_t new_thread = THREAD_NULL; @@ -1750,9 +1796,7 @@ thread_select( if (SCHED(can_update_priority)(thread)) SCHED(update_priority)(thread); - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class; + processor_state_update_from_thread(processor, thread); pset_lock(pset); @@ -1775,13 +1819,12 @@ thread_select( * An exception is that bound threads are dispatched to a processor without going through * choose_processor(), so in those cases we should continue trying to dequeue work. */ - if (!SCHED(processor_bound_count)(processor) && !queue_empty(&pset->idle_queue) && !rt_runq.count) { + if (!SCHED(processor_bound_count)(processor) && + !queue_empty(&pset->idle_queue) && !rt_runq_count(pset)) { goto idle; } } - rt_lock_lock(); - /* * Test to see if the current thread should continue * to run on this processor. Must not be attempting to wait, and not @@ -1793,78 +1836,111 @@ thread_select( * This code is very insanely tricky. */ - if (((thread->state & (TH_TERMINATE|TH_IDLE|TH_WAIT|TH_RUN|TH_SUSP)) == TH_RUN) && - (thread->sched_pri >= BASEPRI_RTQUEUES || processor->processor_primary == processor) && - (thread->bound_processor == PROCESSOR_NULL || thread->bound_processor == processor) && - (thread->affinity_set == AFFINITY_SET_NULL || thread->affinity_set->aset_pset == pset)) { + /* i.e. not waiting, not TH_SUSP'ed */ + boolean_t still_running = ((thread->state & (TH_TERMINATE|TH_IDLE|TH_WAIT|TH_RUN|TH_SUSP)) == TH_RUN); + + /* + * Threads running on SMT processors are forced to context switch. Don't rebalance realtime threads. + * TODO: This should check if it's worth it to rebalance, i.e. 'are there any idle primary processors' + */ + boolean_t needs_smt_rebalance = (thread->sched_pri < BASEPRI_RTQUEUES && processor->processor_primary != processor); + + boolean_t affinity_mismatch = (thread->affinity_set != AFFINITY_SET_NULL && thread->affinity_set->aset_pset != pset); + + boolean_t bound_elsewhere = (thread->bound_processor != PROCESSOR_NULL && thread->bound_processor != processor); + + boolean_t avoid_processor = (SCHED(avoid_processor_enabled) && SCHED(thread_avoid_processor)(processor, thread)); + + if (still_running && !needs_smt_rebalance && !affinity_mismatch && !bound_elsewhere && !avoid_processor) { /* + * This thread is eligible to keep running on this processor. + * * RT threads with un-expired quantum stay on processor, * unless there's a valid RT thread with an earlier deadline. */ if (thread->sched_pri >= BASEPRI_RTQUEUES && processor->first_timeslice) { - if (rt_runq.count > 0) { - thread_t next_rt = qe_queue_first(&rt_runq.queue, struct thread, runq_links); - - assert(next_rt->runq == THREAD_ON_RT_RUNQ); + if (rt_runq_count(pset) > 0) { - if (next_rt->realtime.deadline < processor->deadline && - (next_rt->bound_processor == PROCESSOR_NULL || - next_rt->bound_processor == processor)) { - /* The next RT thread is better, so pick it off the runqueue. */ - goto pick_new_rt_thread; + rt_lock_lock(pset); + + if (rt_runq_count(pset) > 0) { + + thread_t next_rt = qe_queue_first(&SCHED(rt_runq)(pset)->queue, struct thread, runq_links); + + if (next_rt->realtime.deadline < processor->deadline && + (next_rt->bound_processor == PROCESSOR_NULL || + next_rt->bound_processor == processor)) { + /* The next RT thread is better, so pick it off the runqueue. */ + goto pick_new_rt_thread; + } } + + rt_lock_unlock(pset); } /* This is still the best RT thread to run. */ processor->deadline = thread->realtime.deadline; - rt_lock_unlock(); + sched_update_pset_load_average(pset); pset_unlock(pset); return (thread); } - if ((rt_runq.count == 0) && + if ((rt_runq_count(pset) == 0) && SCHED(processor_queue_has_priority)(processor, thread->sched_pri, TRUE) == FALSE) { /* This thread is still the highest priority runnable (non-idle) thread */ processor->deadline = UINT64_MAX; - rt_lock_unlock(); + sched_update_pset_load_average(pset); pset_unlock(pset); return (thread); } + } else { + /* + * This processor must context switch. + * If it's due to a rebalance, we should aggressively find this thread a new home. + */ + if (needs_smt_rebalance || affinity_mismatch || bound_elsewhere || avoid_processor) + *reason |= AST_REBALANCE; } /* OK, so we're not going to run the current thread. Look at the RT queue. */ - if (rt_runq.count > 0) { - thread_t next_rt = qe_queue_first(&rt_runq.queue, struct thread, runq_links); + if (rt_runq_count(pset) > 0) { - assert(next_rt->runq == THREAD_ON_RT_RUNQ); + rt_lock_lock(pset); + + if (rt_runq_count(pset) > 0) { + thread_t next_rt = qe_queue_first(&SCHED(rt_runq)(pset)->queue, struct thread, runq_links); - if (__probable((next_rt->bound_processor == PROCESSOR_NULL || + if (__probable((next_rt->bound_processor == PROCESSOR_NULL || (next_rt->bound_processor == processor)))) { pick_new_rt_thread: - new_thread = qe_dequeue_head(&rt_runq.queue, struct thread, runq_links); + new_thread = qe_dequeue_head(&SCHED(rt_runq)(pset)->queue, struct thread, runq_links); - new_thread->runq = PROCESSOR_NULL; - SCHED_STATS_RUNQ_CHANGE(&rt_runq.runq_stats, rt_runq.count); - rt_runq.count--; + new_thread->runq = PROCESSOR_NULL; + SCHED_STATS_RUNQ_CHANGE(&SCHED(rt_runq)(pset)->runq_stats, rt_runq_count(pset)); + rt_runq_count_decr(pset); - processor->deadline = new_thread->realtime.deadline; + processor->deadline = new_thread->realtime.deadline; - rt_lock_unlock(); - pset_unlock(pset); + rt_lock_unlock(pset); + sched_update_pset_load_average(pset); + pset_unlock(pset); - return (new_thread); + return (new_thread); + } } + + rt_lock_unlock(pset); } processor->deadline = UINT64_MAX; - rt_lock_unlock(); /* No RT threads, so let's look at the regular threads. */ - if ((new_thread = SCHED(choose_thread)(processor, MINPRI, reason)) != THREAD_NULL) { + if ((new_thread = SCHED(choose_thread)(processor, MINPRI, *reason)) != THREAD_NULL) { + sched_update_pset_load_average(pset); pset_unlock(pset); return (new_thread); } @@ -1884,7 +1960,7 @@ pick_new_rt_thread: * If other threads have appeared, shortcut * around again. */ - if (!SCHED(processor_queue_empty)(processor) || rt_runq.count > 0) + if (!SCHED(processor_queue_empty)(processor) || rt_runq_count(pset) > 0) continue; pset_lock(pset); @@ -1899,16 +1975,21 @@ pick_new_rt_thread: if (processor->state == PROCESSOR_RUNNING) { processor->state = PROCESSOR_IDLE; - if (processor->processor_primary == processor) { + if (!processor->is_recommended) { + re_queue_head(&pset->unused_queue, &processor->processor_queue); + } else if (processor->processor_primary == processor) { re_queue_head(&pset->idle_queue, &processor->processor_queue); } else { re_queue_head(&pset->idle_secondary_queue, &processor->processor_queue); } + + pset->active_processor_count--; + sched_update_pset_load_average(pset); } #if __SMP__ /* Invoked with pset locked, returns with pset unlocked */ - sched_SMT_balance(processor, pset); + SCHED(processor_balance)(processor, pset); #else pset_unlock(pset); #endif @@ -1966,9 +2047,7 @@ thread_select_idle( sched_run_decr(thread); thread->state |= TH_IDLE; - processor->current_pri = IDLEPRI; - processor->current_thmode = TH_MODE_NONE; - processor->current_sfi_class = SFI_CLASS_KERNEL; + processor_state_update_idle(procssor); /* Reload precise timing global policy to thread-local policy */ thread->precise_user_kernel_time = use_precise_user_kernel_time(thread); @@ -1988,10 +2067,11 @@ thread_select_idle( thread_timer_event(processor->last_dispatch, &processor->idle_thread->system_timer); PROCESSOR_DATA(processor, kernel_timer) = &processor->idle_thread->system_timer; + /* * Cancel the quantum timer while idling. */ - timer_call_cancel(&processor->quantum_timer); + timer_call_quantum_timer_cancel(&processor->quantum_timer); processor->first_timeslice = FALSE; (*thread->sched_call)(SCHED_CALL_BLOCK, thread); @@ -2019,10 +2099,10 @@ thread_select_idle( processor->last_dispatch = mach_absolute_time(); thread_timer_event(processor->last_dispatch, &thread->system_timer); PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer; - thread_quantum_init(thread); processor->quantum_end = processor->last_dispatch + thread->quantum_remaining; - timer_call_enter1(&processor->quantum_timer, thread, processor->quantum_end, TIMER_CALL_SYS_CRITICAL | TIMER_CALL_LOCAL); + timer_call_quantum_timer_enter(&processor->quantum_timer, + thread, processor->quantum_end, processor->last_dispatch); processor->first_timeslice = TRUE; thread->computation_epoch = processor->last_dispatch; @@ -2078,9 +2158,14 @@ thread_invoke( #endif #if defined(CONFIG_SCHED_TIMESHARE_CORE) - sched_timeshare_consider_maintenance(ctime); + if ((thread->state & TH_IDLE) == 0) + sched_timeshare_consider_maintenance(ctime); #endif +#if MONOTONIC + mt_sched_update(self); +#endif /* MONOTONIC */ + assert_thread_magic(self); assert(self == current_thread()); assert(self->runq == PROCESSOR_NULL); @@ -2098,11 +2183,25 @@ thread_invoke( /* Update SFI class based on other factors */ thread->sfi_class = sfi_thread_classify(thread); + + /* Update the same_pri_latency for the thread (used by perfcontrol callouts) */ + thread->same_pri_latency = ctime - thread->last_basepri_change_time; + /* + * In case a base_pri update happened between the timestamp and + * taking the thread lock + */ + if (ctime <= thread->last_basepri_change_time) + thread->same_pri_latency = ctime - thread->last_made_runnable_time; /* Allow realtime threads to hang onto a stack. */ if ((self->sched_mode == TH_MODE_REALTIME) && !self->reserved_stack) self->reserved_stack = self->kernel_stack; + /* Prepare for spin debugging */ +#if INTERRUPT_MASKED_DEBUG + ml_spin_debug_clear(thread); +#endif + if (continuation != NULL) { if (!thread->kernel_stack) { /* @@ -2121,9 +2220,8 @@ thread_invoke( processor = current_processor(); processor->active_thread = thread; - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class; + processor_state_update_from_thread(processor, thread); + if (thread->last_processor != processor && thread->last_processor != NULL) { if (thread->last_processor->processor_set != processor->processor_set) thread->ps_switch++; @@ -2177,6 +2275,11 @@ thread_invoke( kperf_on_cpu(thread, continuation, NULL); #endif /* KPERF */ +#if KASAN + kasan_unpoison_fakestack(self); + kasan_unpoison_stack(thread->kernel_stack, kernel_stack_size); +#endif + thread_dispatch(self, thread); thread->continuation = thread->parameter = NULL; @@ -2204,6 +2307,11 @@ thread_invoke( MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED) | DBG_FUNC_NONE, self->reason, (uintptr_t)thread_tid(thread), self->sched_pri, thread->sched_pri, 0); +#if KASAN + kasan_unpoison_fakestack(self); + kasan_unpoison_stack(self->kernel_stack, kernel_stack_size); +#endif + self->continuation = self->parameter = NULL; (void) spllo(); @@ -2241,9 +2349,8 @@ need_stack: */ processor = current_processor(); processor->active_thread = thread; - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class; + processor_state_update_from_thread(processor, thread); + if (thread->last_processor != processor && thread->last_processor != NULL) { if (thread->last_processor->processor_set != processor->processor_set) thread->ps_switch++; @@ -2394,8 +2501,8 @@ pset_cancel_deferred_dispatch( * interrupt while IDLE. */ if ((active_processor->state == PROCESSOR_DISPATCHING) && - (pset->pending_deferred_AST_cpu_mask & (1ULL << active_processor->cpu_id)) && - (!(pset->pending_AST_cpu_mask & (1ULL << active_processor->cpu_id))) && + (bit_test(pset->pending_deferred_AST_cpu_mask, active_processor->cpu_id)) && + (!bit_test(pset->pending_AST_cpu_mask, active_processor->cpu_id)) && (active_processor != processor)) { /* * Squash all of the processor state back to some @@ -2407,16 +2514,22 @@ pset_cancel_deferred_dispatch( * The tail? At the (relative) old position in the * queue? Or something else entirely? */ - re_queue_head(&pset->idle_queue, &active_processor->processor_queue); + if (!active_processor->is_recommended) { + re_queue_head(&pset->unused_queue, &active_processor->processor_queue); + } else if (active_processor->processor_primary == active_processor) { + re_queue_head(&pset->idle_queue, &active_processor->processor_queue); + } else { + re_queue_head(&pset->idle_secondary_queue, &active_processor->processor_queue); + } - assert(active_processor->next_thread == THREAD_NULL); + pset->active_processor_count--; + sched_update_pset_load_average(pset); - active_processor->current_pri = IDLEPRI; - active_processor->current_thmode = TH_MODE_FIXED; - active_processor->current_sfi_class = SFI_CLASS_KERNEL; + assert(active_processor->next_thread == THREAD_NULL); + processor_state_update_idle(active_processor); active_processor->deadline = UINT64_MAX; active_processor->state = PROCESSOR_IDLE; - pset->pending_deferred_AST_cpu_mask &= ~(1U << active_processor->cpu_id); + bit_clear(pset->pending_deferred_AST_cpu_mask, active_processor->cpu_id); machine_signal_idle_cancel(active_processor); } @@ -2429,6 +2542,19 @@ pset_cancel_deferred_dispatch( /* We don't support deferred ASTs; everything is candycanes and sunshine. */ #endif +static void +thread_csw_callout( + thread_t old, + thread_t new, + uint64_t timestamp) +{ + perfcontrol_event event = (new->state & TH_IDLE) ? IDLE : CONTEXT_SWITCH; + uint64_t same_pri_latency = (new->state & TH_IDLE) ? 0 : new->same_pri_latency; + machine_switch_perfcontrol_context(event, timestamp, 0, + same_pri_latency, old, new); +} + + /* * thread_dispatch: * @@ -2454,6 +2580,20 @@ thread_dispatch( assert(thread != self); if (thread != THREAD_NULL) { + /* + * Do the perfcontrol callout for context switch. + * The reason we do this here is: + * - thread_dispatch() is called from various places that are not + * the direct context switch path for eg. processor shutdown etc. + * So adding the callout here covers all those cases. + * - We want this callout as early as possible to be close + * to the timestamp taken in thread_invoke() + * - We want to avoid holding the thread lock while doing the + * callout + * - We do not want to callout if "thread" is NULL. + */ + thread_csw_callout(thread, self, processor->last_dispatch); + /* * If blocked at a continuation, discard * the stack. @@ -2485,7 +2625,6 @@ thread_dispatch( task_ledgers.cpu_time, consumed); ledger_credit(thread->t_threadledger, thread_ledgers.cpu_time, consumed); -#ifdef CONFIG_BANK if (thread->t_bankledger) { ledger_credit(thread->t_bankledger, bank_ledgers.cpu_time, @@ -2493,7 +2632,6 @@ thread_dispatch( } thread->t_deduct_bank_ledger_time =0; -#endif } wake_lock(thread); @@ -2575,16 +2713,34 @@ thread_dispatch( /* * Still runnable. */ - thread->last_made_runnable_time = mach_approximate_time(); + thread->last_made_runnable_time = thread->last_basepri_change_time = processor->last_dispatch; machine_thread_going_off_core(thread, FALSE, processor->last_dispatch); - if (thread->reason & AST_QUANTUM) - thread_setrun(thread, SCHED_TAILQ); - else if (thread->reason & AST_PREEMPT) - thread_setrun(thread, SCHED_HEADQ); + ast_t reason = thread->reason; + sched_options_t options = SCHED_NONE; + + if (reason & AST_REBALANCE) { + options |= SCHED_REBALANCE; + if (reason & AST_QUANTUM) { + /* Having gone to the trouble of forcing this thread off a less preferred core, + * we should force the preferable core to reschedule immediatey to give this + * thread a chance to run instead of just sitting on the run queue where + * it may just be stolen back by the idle core we just forced it off. + * But only do this at the end of a quantum to prevent cascading effects. + */ + options |= SCHED_PREEMPT; + } + } + + if (reason & AST_QUANTUM) + options |= SCHED_TAILQ; + else if (reason & AST_PREEMPT) + options |= SCHED_HEADQ; else - thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ); + options |= (SCHED_PREEMPT | SCHED_TAILQ); + + thread_setrun(thread, options); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, MACHDBG_CODE(DBG_MACH_SCHED,MACH_DISPATCH) | DBG_FUNC_NONE, @@ -2618,7 +2774,7 @@ thread_dispatch( } thread->state &= ~TH_RUN; - thread->last_made_runnable_time = ~0ULL; + thread->last_made_runnable_time = thread->last_basepri_change_time = THREAD_NOT_RUNNABLE; thread->chosen_processor = PROCESSOR_NULL; new_run_count = sched_run_decr(thread); @@ -2657,12 +2813,14 @@ thread_dispatch( } } + int urgency = THREAD_URGENCY_NONE; + uint64_t latency = 0; + /* Update (new) current thread and reprogram quantum timer */ thread_lock(self); + if (!(self->state & TH_IDLE)) { uint64_t arg1, arg2; - int urgency; - uint64_t latency; #if CONFIG_SCHED_SFI ast_t new_ast; @@ -2674,15 +2832,19 @@ thread_dispatch( } #endif - assertf(processor->last_dispatch >= self->last_made_runnable_time, "Non-monotonic time? dispatch at 0x%llx, runnable at 0x%llx", processor->last_dispatch, self->last_made_runnable_time); + assertf(processor->last_dispatch >= self->last_made_runnable_time, + "Non-monotonic time? dispatch at 0x%llx, runnable at 0x%llx", + processor->last_dispatch, self->last_made_runnable_time); + + assert(self->last_made_runnable_time <= self->last_basepri_change_time); + latency = processor->last_dispatch - self->last_made_runnable_time; + assert(latency >= self->same_pri_latency); urgency = thread_get_urgency(self, &arg1, &arg2); thread_tell_urgency(urgency, arg1, arg2, latency, self); - machine_thread_going_on_core(self, urgency, latency, processor->last_dispatch); - /* * Get a new quantum if none remaining. */ @@ -2694,15 +2856,15 @@ thread_dispatch( * Set up quantum timer and timeslice. */ processor->quantum_end = processor->last_dispatch + self->quantum_remaining; - timer_call_enter1(&processor->quantum_timer, self, processor->quantum_end, TIMER_CALL_SYS_CRITICAL | TIMER_CALL_LOCAL); + timer_call_quantum_timer_enter(&processor->quantum_timer, self, + processor->quantum_end, processor->last_dispatch); processor->first_timeslice = TRUE; } else { - timer_call_cancel(&processor->quantum_timer); + timer_call_quantum_timer_cancel(&processor->quantum_timer); processor->first_timeslice = FALSE; thread_tell_urgency(THREAD_URGENCY_NONE, 0, 0, 0, self); - machine_thread_going_on_core(self, THREAD_URGENCY_NONE, 0, processor->last_dispatch); } assert(self->block_hint == kThreadWaitNone); @@ -2712,6 +2874,9 @@ thread_dispatch( thread_unlock(self); + machine_thread_going_on_core(self, urgency, latency, self->same_pri_latency, + processor->last_dispatch); + #if defined(CONFIG_SCHED_DEFERRED_AST) /* * TODO: Can we state that redispatching our old thread is also @@ -2781,7 +2946,7 @@ thread_block_reason( do { thread_lock(self); - new_thread = thread_select(self, processor, reason); + new_thread = thread_select(self, processor, &reason); thread_unlock(self); } while (!thread_invoke(self, new_thread, reason)); @@ -2827,18 +2992,20 @@ thread_run( void *parameter, thread_t new_thread) { - ast_t handoff = AST_HANDOFF; + ast_t reason = AST_HANDOFF; self->continuation = continuation; self->parameter = parameter; - while (!thread_invoke(self, new_thread, handoff)) { - processor_t processor = current_processor(); + while (!thread_invoke(self, new_thread, reason)) { + /* the handoff failed, so we have to fall back to the normal block path */ + processor_t processor = current_processor(); + + reason = AST_NONE; thread_lock(self); - new_thread = thread_select(self, processor, AST_NONE); + new_thread = thread_select(self, processor, &reason); thread_unlock(self); - handoff = AST_NONE; } return (self->wait_result); @@ -2871,6 +3038,11 @@ thread_continue( self->continuation = self->parameter = NULL; +#if INTERRUPT_MASKED_DEBUG + /* Reset interrupt-masked spin debugging timeout */ + ml_spin_debug_clear(self); +#endif + if (thread != THREAD_NULL) (void)spllo(); @@ -3029,24 +3201,31 @@ run_queue_remove( /* Assumes RT lock is not held, and acquires splsched/rt_lock itself */ void -rt_runq_scan(sched_update_scan_context_t scan_context) +sched_rtglobal_runq_scan(sched_update_scan_context_t scan_context) { spl_t s; thread_t thread; + processor_set_t pset = &pset0; + s = splsched(); - rt_lock_lock(); + rt_lock_lock(pset); - qe_foreach_element_safe(thread, &rt_runq.queue, runq_links) { + qe_foreach_element_safe(thread, &pset->rt_runq.queue, runq_links) { if (thread->last_made_runnable_time < scan_context->earliest_rt_make_runnable_time) { scan_context->earliest_rt_make_runnable_time = thread->last_made_runnable_time; } } - rt_lock_unlock(); + rt_lock_unlock(pset); splx(s); } +int64_t +sched_rtglobal_runq_count_sum(void) +{ + return pset0.rt_runq.runq_stats.count_sum; +} /* * realtime_queue_insert: @@ -3054,13 +3233,13 @@ rt_runq_scan(sched_update_scan_context_t scan_context) * Enqueue a thread for realtime execution. */ static boolean_t -realtime_queue_insert(thread_t thread) +realtime_queue_insert(processor_t processor, processor_set_t pset, thread_t thread) { - queue_t queue = &rt_runq.queue; + queue_t queue = &SCHED(rt_runq)(pset)->queue; uint64_t deadline = thread->realtime.deadline; boolean_t preempt = FALSE; - rt_lock_lock(); + rt_lock_lock(pset); if (queue_empty(queue)) { enqueue_tail(queue, &thread->runq_links); @@ -3084,11 +3263,11 @@ realtime_queue_insert(thread_t thread) } } - thread->runq = THREAD_ON_RT_RUNQ; - SCHED_STATS_RUNQ_CHANGE(&rt_runq.runq_stats, rt_runq.count); - rt_runq.count++; + thread->runq = processor; + SCHED_STATS_RUNQ_CHANGE(&SCHED(rt_runq)(pset)->runq_stats, rt_runq_count(pset)); + rt_runq_count_incr(pset); - rt_lock_unlock(); + rt_lock_unlock(pset); return (preempt); } @@ -3109,7 +3288,7 @@ realtime_setrun( processor_set_t pset = processor->processor_set; ast_t preempt; - boolean_t do_signal_idle = FALSE, do_cause_ast = FALSE; + sched_ipi_type_t ipi_type = SCHED_IPI_NONE; thread->chosen_processor = processor; @@ -3123,25 +3302,17 @@ realtime_setrun( && processor->state == PROCESSOR_IDLE) { re_queue_tail(&pset->active_queue, &processor->processor_queue); + pset->active_processor_count++; + sched_update_pset_load_average(pset); + processor->next_thread = thread; - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class; + processor_state_update_from_thread(processor, thread); processor->deadline = thread->realtime.deadline; processor->state = PROCESSOR_DISPATCHING; - if (processor != current_processor()) { - if (!(pset->pending_AST_cpu_mask & (1ULL << processor->cpu_id))) { - /* cleared on exit from main processor_idle() loop */ - pset->pending_AST_cpu_mask |= (1ULL << processor->cpu_id); - do_signal_idle = TRUE; - } - } + ipi_type = sched_ipi_action(processor, thread, true, SCHED_IPI_EVENT_BOUND_THR); pset_unlock(pset); - - if (do_signal_idle) { - machine_signal_idle(processor); - } + sched_ipi_perform(processor, ipi_type); return; } @@ -3152,43 +3323,35 @@ realtime_setrun( else preempt = AST_NONE; - realtime_queue_insert(thread); + realtime_queue_insert(processor, pset, thread); + ipi_type = SCHED_IPI_NONE; if (preempt != AST_NONE) { if (processor->state == PROCESSOR_IDLE) { re_queue_tail(&pset->active_queue, &processor->processor_queue); + + pset->active_processor_count++; + sched_update_pset_load_average(pset); processor->next_thread = THREAD_NULL; - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class; + processor_state_update_from_thread(processor, thread); processor->deadline = thread->realtime.deadline; processor->state = PROCESSOR_DISPATCHING; if (processor == current_processor()) { ast_on(preempt); } else { - if (!(pset->pending_AST_cpu_mask & (1ULL << processor->cpu_id))) { - /* cleared on exit from main processor_idle() loop */ - pset->pending_AST_cpu_mask |= (1ULL << processor->cpu_id); - do_signal_idle = TRUE; - } + ipi_type = sched_ipi_action(processor, thread, true, SCHED_IPI_EVENT_PREEMPT); } } else if (processor->state == PROCESSOR_DISPATCHING) { if ((processor->next_thread == THREAD_NULL) && ((processor->current_pri < thread->sched_pri) || (processor->deadline > thread->realtime.deadline))) { - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class; + processor_state_update_from_thread(processor, thread); processor->deadline = thread->realtime.deadline; } } else { if (processor == current_processor()) { ast_on(preempt); } else { - if (!(pset->pending_AST_cpu_mask & (1ULL << processor->cpu_id))) { - /* cleared after IPI causes csw_check() to be called */ - pset->pending_AST_cpu_mask |= (1ULL << processor->cpu_id); - do_cause_ast = TRUE; - } + ipi_type = sched_ipi_action(processor, thread, false, SCHED_IPI_EVENT_PREEMPT); } } } else { @@ -3196,14 +3359,116 @@ realtime_setrun( } pset_unlock(pset); + sched_ipi_perform(processor, ipi_type); +} - if (do_signal_idle) { - machine_signal_idle(processor); - } else if (do_cause_ast) { - cause_ast_check(processor); - } + +sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset, processor_t dst, + __unused sched_ipi_event_t event) +{ +#if defined(CONFIG_SCHED_DEFERRED_AST) + if (!bit_test(pset->pending_deferred_AST_cpu_mask, dst->cpu_id)) { + return SCHED_IPI_DEFERRED; + } +#else /* CONFIG_SCHED_DEFERRED_AST */ + panic("Request for deferred IPI on an unsupported platform; pset: %p CPU: %d", pset, dst->cpu_id); +#endif /* CONFIG_SCHED_DEFERRED_AST */ + return SCHED_IPI_NONE; +} + +sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event) +{ + sched_ipi_type_t ipi_type = SCHED_IPI_NONE; + assert(dst != NULL); + + processor_set_t pset = dst->processor_set; + if (current_processor() == dst) { + return SCHED_IPI_NONE; + } + + if (bit_test(pset->pending_AST_cpu_mask, dst->cpu_id)) { + return SCHED_IPI_NONE; + } + + ipi_type = SCHED(ipi_policy)(dst, thread, dst_idle, event); + switch(ipi_type) { + case SCHED_IPI_NONE: + return SCHED_IPI_NONE; +#if defined(CONFIG_SCHED_DEFERRED_AST) + case SCHED_IPI_DEFERRED: + bit_set(pset->pending_deferred_AST_cpu_mask, dst->cpu_id); + break; +#endif /* CONFIG_SCHED_DEFERRED_AST */ + default: + bit_set(pset->pending_AST_cpu_mask, dst->cpu_id); + break; + } + return ipi_type; +} + +sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event) +{ + sched_ipi_type_t ipi_type = SCHED_IPI_NONE; + boolean_t deferred_ipi_supported = false; + processor_set_t pset = dst->processor_set; + +#if defined(CONFIG_SCHED_DEFERRED_AST) + deferred_ipi_supported = true; +#endif /* CONFIG_SCHED_DEFERRED_AST */ + + switch(event) { + case SCHED_IPI_EVENT_SPILL: + case SCHED_IPI_EVENT_SMT_REBAL: + case SCHED_IPI_EVENT_REBALANCE: + case SCHED_IPI_EVENT_BOUND_THR: + /* + * The spill, SMT rebalance, rebalance and the bound thread + * scenarios use immediate IPIs always. + */ + ipi_type = dst_idle ? SCHED_IPI_IDLE : SCHED_IPI_IMMEDIATE; + break; + case SCHED_IPI_EVENT_PREEMPT: + /* In the preemption case, use immediate IPIs for RT threads */ + if (thread && (thread->sched_pri >= BASEPRI_RTQUEUES)) { + ipi_type = dst_idle ? SCHED_IPI_IDLE : SCHED_IPI_IMMEDIATE; + break; + } + + /* + * For Non-RT threads preemption, + * If the core is active, use immediate IPIs. + * If the core is idle, use deferred IPIs if supported; otherwise immediate IPI. + */ + if (deferred_ipi_supported && dst_idle) { + return sched_ipi_deferred_policy(pset, dst, event); + } + ipi_type = dst_idle ? SCHED_IPI_IDLE : SCHED_IPI_IMMEDIATE; + break; + default: + panic("Unrecognized scheduler IPI event type %d", event); + } + assert(ipi_type != SCHED_IPI_NONE); + return ipi_type; } +void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi) +{ + switch (ipi) { + case SCHED_IPI_NONE: + break; + case SCHED_IPI_IDLE: + machine_signal_idle(dst); + break; + case SCHED_IPI_IMMEDIATE: + cause_ast_check(dst); + break; + case SCHED_IPI_DEFERRED: + machine_signal_idle_deferred(dst); + break; + default: + panic("Unrecognized scheduler IPI type: %d", ipi); + } +} #if defined(CONFIG_SCHED_TIMESHARE_CORE) @@ -3233,9 +3498,8 @@ processor_setrun( processor_set_t pset = processor->processor_set; ast_t preempt; enum { eExitIdle, eInterruptRunning, eDoNothing } ipi_action = eDoNothing; - enum { eNoSignal, eDoSignal, eDoDeferredSignal } do_signal_idle = eNoSignal; - boolean_t do_cause_ast = FALSE; + sched_ipi_type_t ipi_type = SCHED_IPI_NONE; thread->chosen_processor = processor; @@ -3248,25 +3512,17 @@ processor_setrun( re_queue_tail(&pset->active_queue, &processor->processor_queue); + pset->active_processor_count++; + sched_update_pset_load_average(pset); + processor->next_thread = thread; - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class; + processor_state_update_from_thread(processor, thread); processor->deadline = UINT64_MAX; processor->state = PROCESSOR_DISPATCHING; - if (!(pset->pending_AST_cpu_mask & (1ULL << processor->cpu_id))) { - /* cleared on exit from main processor_idle() loop */ - pset->pending_AST_cpu_mask |= (1ULL << processor->cpu_id); - do_signal_idle = eDoSignal; - } - + ipi_type = sched_ipi_action(processor, thread, true, SCHED_IPI_EVENT_BOUND_THR); pset_unlock(pset); - - if (do_signal_idle == eDoSignal) { - machine_signal_idle(processor); - } - + sched_ipi_perform(processor, ipi_type); return; } @@ -3290,24 +3546,20 @@ processor_setrun( preempt = (options & SCHED_PREEMPT)? AST_PREEMPT: AST_NONE; SCHED(processor_enqueue)(processor, thread, options); + sched_update_pset_load_average(pset); if (preempt != AST_NONE) { if (processor->state == PROCESSOR_IDLE) { re_queue_tail(&pset->active_queue, &processor->processor_queue); - + pset->active_processor_count++; processor->next_thread = THREAD_NULL; - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class; + processor_state_update_from_thread(processor, thread); processor->deadline = UINT64_MAX; processor->state = PROCESSOR_DISPATCHING; - ipi_action = eExitIdle; } else if ( processor->state == PROCESSOR_DISPATCHING) { if ((processor->next_thread == THREAD_NULL) && (processor->current_pri < thread->sched_pri)) { - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class; + processor_state_update_from_thread(processor, thread); processor->deadline = UINT64_MAX; } } else if ( (processor->state == PROCESSOR_RUNNING || @@ -3326,10 +3578,11 @@ processor_setrun( } else if (processor->state == PROCESSOR_IDLE) { re_queue_tail(&pset->active_queue, &processor->processor_queue); + pset->active_processor_count++; + // sched_update_pset_load_average(pset); + processor->next_thread = THREAD_NULL; - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class; + processor_state_update_from_thread(processor, thread); processor->deadline = UINT64_MAX; processor->state = PROCESSOR_DISPATCHING; @@ -3337,67 +3590,17 @@ processor_setrun( } } - switch (ipi_action) { - case eDoNothing: - break; - case eExitIdle: - if (processor == current_processor()) { - if (csw_check_locked(processor, pset, AST_NONE) != AST_NONE) - ast_on(preempt); - } else { -#if defined(CONFIG_SCHED_DEFERRED_AST) - if (!(pset->pending_deferred_AST_cpu_mask & (1ULL << processor->cpu_id)) && - !(pset->pending_AST_cpu_mask & (1ULL << processor->cpu_id))) { - /* cleared on exit from main processor_idle() loop */ - pset->pending_deferred_AST_cpu_mask |= (1ULL << processor->cpu_id); - do_signal_idle = eDoDeferredSignal; - } -#else - if (!(pset->pending_AST_cpu_mask & (1ULL << processor->cpu_id))) { - /* cleared on exit from main processor_idle() loop */ - pset->pending_AST_cpu_mask |= (1ULL << processor->cpu_id); - do_signal_idle = eDoSignal; - } -#endif - } - break; - case eInterruptRunning: - if (processor == current_processor()) { - if (csw_check_locked(processor, pset, AST_NONE) != AST_NONE) - ast_on(preempt); - } else { - if (!(pset->pending_AST_cpu_mask & (1ULL << processor->cpu_id))) { - /* cleared after IPI causes csw_check() to be called */ - pset->pending_AST_cpu_mask |= (1ULL << processor->cpu_id); - do_cause_ast = TRUE; - } - } - break; + if (ipi_action != eDoNothing) { + if (processor == current_processor()) { + if (csw_check_locked(processor, pset, AST_NONE) != AST_NONE) + ast_on(preempt); + } else { + sched_ipi_event_t event = (options & SCHED_REBALANCE) ? SCHED_IPI_EVENT_REBALANCE : SCHED_IPI_EVENT_PREEMPT; + ipi_type = sched_ipi_action(processor, thread, (ipi_action == eExitIdle), event); + } } - pset_unlock(pset); - - if (do_signal_idle == eDoSignal) { - machine_signal_idle(processor); - } -#if defined(CONFIG_SCHED_DEFERRED_AST) - else if (do_signal_idle == eDoDeferredSignal) { - /* - * TODO: The ability to cancel this signal could make - * sending it outside of the pset lock an issue. Do - * we need to address this? Or would the only fallout - * be that the core takes a signal? As long as we do - * not run the risk of having a core marked as signal - * outstanding, with no real signal outstanding, the - * only result should be that we fail to cancel some - * signals. - */ - machine_signal_idle_deferred(processor); - } -#endif - else if (do_cause_ast) { - cause_ast_check(processor); - } + sched_ipi_perform(processor, ipi_type); } /* @@ -3729,6 +3932,10 @@ choose_processor( } while (processor == PROCESSOR_NULL); + if (processor->state == PROCESSOR_RUNNING) { + re_queue_tail(&cset->active_queue, &processor->processor_queue); + } + return (processor); } @@ -3775,6 +3982,7 @@ thread_setrun( pset_lock(pset); processor = SCHED(choose_processor)(pset, PROCESSOR_NULL, thread); + pset = processor->processor_set; SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHOOSE_PROCESSOR)|DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), (uintptr_t)-1, processor->cpu_id, processor->state, 0); @@ -3786,6 +3994,7 @@ thread_setrun( pset = processor->processor_set; pset_lock(pset); processor = SCHED(choose_processor)(pset, processor, thread); + pset = processor->processor_set; SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHOOSE_PROCESSOR)|DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), thread->last_processor->cpu_id, processor->cpu_id, processor->state, 0); @@ -3806,7 +4015,8 @@ thread_setrun( pset_lock(pset); processor = SCHED(choose_processor)(pset, PROCESSOR_NULL, thread); - task->pset_hint = processor->processor_set; + pset = processor->processor_set; + task->pset_hint = pset; SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHOOSE_PROCESSOR)|DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), (uintptr_t)-1, processor->cpu_id, processor->state, 0); @@ -3836,10 +4046,15 @@ thread_setrun( * Dispatch the thread on the chosen processor. * TODO: This should be based on sched_mode, not sched_pri */ - if (thread->sched_pri >= BASEPRI_RTQUEUES) + if (thread->sched_pri >= BASEPRI_RTQUEUES) { realtime_setrun(processor, thread); - else + } else { processor_setrun(processor, thread, options); + /* pset is now unlocked */ + if (thread->bound_processor == PROCESSOR_NULL) { + SCHED(check_spill)(pset, thread); + } + } } processor_set_t @@ -3871,7 +4086,7 @@ csw_check( pset_lock(pset); /* If we were sent a remote AST and interrupted a running processor, acknowledge it here with pset lock held */ - pset->pending_AST_cpu_mask &= ~(1ULL << processor->cpu_id); + bit_clear(pset->pending_AST_cpu_mask, processor->cpu_id); result = csw_check_locked(processor, pset, check_reason); @@ -3887,18 +4102,18 @@ csw_check( ast_t csw_check_locked( processor_t processor, - processor_set_t pset __unused, + processor_set_t pset, ast_t check_reason) { ast_t result; thread_t thread = processor->active_thread; if (processor->first_timeslice) { - if (rt_runq.count > 0) + if (rt_runq_count(pset) > 0) return (check_reason | AST_PREEMPT | AST_URGENT); } else { - if (rt_runq.count > 0) { + if (rt_runq_count(pset) > 0) { if (BASEPRI_RTQUEUES > processor->current_pri) return (check_reason | AST_PREEMPT | AST_URGENT); else @@ -3917,8 +4132,18 @@ csw_check_locked( * (non-urgently) get to a point and then block, and which point thread_select() should * try to idle the processor and re-dispatch the thread to a recommended processor. */ - if (!processor->is_recommended) + if (!processor->is_recommended) { return (check_reason | AST_PREEMPT); + } + + /* + * Same for avoid-processor + * + * TODO: Should these set AST_REBALANCE? + */ + if (SCHED(avoid_processor_enabled) && SCHED(thread_avoid_processor)(processor, thread)) { + return (check_reason | AST_PREEMPT); + } /* * Even though we could continue executing on this processor, a @@ -3967,7 +4192,7 @@ csw_check_locked( void set_sched_pri( thread_t thread, - int priority) + int new_priority) { thread_t cthread = current_thread(); boolean_t is_current_thread = (thread == cthread) ? TRUE : FALSE; @@ -3975,8 +4200,10 @@ set_sched_pri( uint64_t urgency_param1, urgency_param2; boolean_t removed_from_runq = FALSE; + int old_priority = thread->sched_pri; + /* If we're already at this priority, no need to mess with the runqueue */ - if (priority == thread->sched_pri) + if (new_priority == old_priority) return; if (is_current_thread) { @@ -3986,7 +4213,7 @@ set_sched_pri( removed_from_runq = thread_run_queue_remove(thread); } - thread->sched_pri = priority; + thread->sched_pri = new_priority; KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHANGE_PRIORITY), (uintptr_t)thread_tid(thread), @@ -4003,28 +4230,35 @@ set_sched_pri( * those are lazily handled. QoS classes have distinct priority bands, and QoS * inheritance is expected to involve priority changes. */ + uint64_t ctime = mach_approximate_time(); if (nurgency != curgency) { thread_tell_urgency(nurgency, urgency_param1, urgency_param2, 0, thread); - machine_thread_going_on_core(thread, nurgency, 0, 0); } + machine_thread_going_on_core(thread, nurgency, 0, 0, ctime); } - /* TODO: Should this be TAILQ if it went down, HEADQ if it went up? */ if (removed_from_runq) thread_run_queue_reinsert(thread, SCHED_PREEMPT | SCHED_TAILQ); else if (thread->state & TH_RUN) { processor_t processor = thread->last_processor; if (is_current_thread) { - ast_t preempt; + processor_state_update_from_thread(processor, thread); - processor->current_pri = priority; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = thread->sfi_class = sfi_thread_classify(thread); - if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE) - ast_on(preempt); - } else if (processor != PROCESSOR_NULL && processor->active_thread == thread) + /* + * When dropping in priority, check if the thread no longer belongs on core. + * If a thread raises its own priority, don't aggressively rebalance it. + * + */ + if (new_priority < old_priority) { + ast_t preempt; + + if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE) + ast_on(preempt); + } + } else if (processor != PROCESSOR_NULL && processor->active_thread == thread) { cause_ast_check(processor); + } } } @@ -4117,7 +4351,9 @@ thread_run_queue_remove( return SCHED(processor_queue_remove)(processor, thread); } - rt_lock_lock(); + processor_set_t pset = processor->processor_set; + + rt_lock_lock(pset); if (thread->runq != PROCESSOR_NULL) { /* @@ -4125,18 +4361,16 @@ thread_run_queue_remove( * that run queue. */ - assert(thread->runq == THREAD_ON_RT_RUNQ); - remqueue(&thread->runq_links); - SCHED_STATS_RUNQ_CHANGE(&rt_runq.runq_stats, rt_runq.count); - rt_runq.count--; + SCHED_STATS_RUNQ_CHANGE(&SCHED(rt_runq)(pset)->runq_stats, rt_runq_count(pset)); + rt_runq_count_decr(pset); thread->runq = PROCESSOR_NULL; removed = TRUE; } - rt_lock_unlock(); + rt_lock_unlock(pset); return (removed); } @@ -4152,10 +4386,9 @@ void thread_run_queue_reinsert(thread_t thread, integer_t options) { assert(thread->runq == PROCESSOR_NULL); + assert(thread->state & (TH_RUN)); - assert(thread->state & (TH_RUN)); - thread_setrun(thread, options); - + thread_setrun(thread, options); } void @@ -4200,6 +4433,29 @@ thread_get_urgency(thread_t thread, uint64_t *arg1, uint64_t *arg2) } } +perfcontrol_class_t +thread_get_perfcontrol_class(thread_t thread) +{ + /* Special case handling */ + if (thread->state & TH_IDLE) + return PERFCONTROL_CLASS_IDLE; + if (thread->task == kernel_task) + return PERFCONTROL_CLASS_KERNEL; + if (thread->sched_mode == TH_MODE_REALTIME) + return PERFCONTROL_CLASS_REALTIME; + + /* perfcontrol_class based on base_pri */ + if (thread->base_pri <= MAXPRI_THROTTLE) + return PERFCONTROL_CLASS_BACKGROUND; + else if (thread->base_pri <= BASEPRI_UTILITY) + return PERFCONTROL_CLASS_UTILITY; + else if (thread->base_pri <= BASEPRI_DEFAULT) + return PERFCONTROL_CLASS_NONUI; + else if (thread->base_pri <= BASEPRI_FOREGROUND) + return PERFCONTROL_CLASS_UI; + else + return PERFCONTROL_CLASS_ABOVEUI; +} /* * This is the processor idle loop, which just looks for other threads @@ -4236,12 +4492,24 @@ processor_idle( PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, idle_state); while (1) { - if (processor->state != PROCESSOR_IDLE) /* unsafe, but worst case we loop around once */ + /* + * Ensure that updates to my processor and pset state, + * made by the IPI source processor before sending the IPI, + * are visible on this processor now (even though we don't + * take the pset lock yet). + */ + atomic_thread_fence(memory_order_acquire); + + if (processor->state != PROCESSOR_IDLE) + break; + if (bit_test(pset->pending_AST_cpu_mask, processor->cpu_id)) break; - if (pset->pending_AST_cpu_mask & (1ULL << processor->cpu_id)) +#if defined(CONFIG_SCHED_DEFERRED_AST) + if (bit_test(pset->pending_deferred_AST_cpu_mask, processor->cpu_id)) break; +#endif if (processor->is_recommended) { - if (rt_runq.count) + if (rt_runq_count(pset)) break; } else { if (SCHED(processor_bound_count)(processor)) @@ -4257,7 +4525,7 @@ processor_idle( #endif IDLE_KERNEL_DEBUG_CONSTANT( - MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), rt_runq.count, SCHED(processor_runq_count)(processor), -1, 0); + MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), rt_runq_count(pset), SCHED(processor_runq_count)(processor), -1, 0); machine_track_platform_idle(TRUE); @@ -4268,7 +4536,7 @@ processor_idle( (void)splsched(); IDLE_KERNEL_DEBUG_CONSTANT( - MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), rt_runq.count, SCHED(processor_runq_count)(processor), -2, 0); + MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), rt_runq_count(pset), SCHED(processor_runq_count)(processor), -2, 0); if (!SCHED(processor_queue_empty)(processor)) { /* Secondary SMT processors respond to directed wakeups @@ -4286,9 +4554,9 @@ processor_idle( pset_lock(pset); /* If we were sent a remote AST and came out of idle, acknowledge it here with pset lock held */ - pset->pending_AST_cpu_mask &= ~(1ULL << processor->cpu_id); + bit_clear(pset->pending_AST_cpu_mask, processor->cpu_id); #if defined(CONFIG_SCHED_DEFERRED_AST) - pset->pending_deferred_AST_cpu_mask &= ~(1ULL << processor->cpu_id); + bit_clear(pset->pending_deferred_AST_cpu_mask, processor->cpu_id); #endif state = processor->state; @@ -4301,17 +4569,15 @@ processor_idle( processor->state = PROCESSOR_RUNNING; if ((new_thread != THREAD_NULL) && (SCHED(processor_queue_has_priority)(processor, new_thread->sched_pri, FALSE) || - (rt_runq.count > 0)) ) { + (rt_runq_count(pset) > 0)) ) { /* Something higher priority has popped up on the runqueue - redispatch this thread elsewhere */ - processor->current_pri = IDLEPRI; - processor->current_thmode = TH_MODE_FIXED; - processor->current_sfi_class = SFI_CLASS_KERNEL; + processor_state_update_idle(processor); processor->deadline = UINT64_MAX; pset_unlock(pset); thread_lock(new_thread); - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REDISPATCH), (uintptr_t)thread_tid(new_thread), new_thread->sched_pri, rt_runq.count, 0, 0); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REDISPATCH), (uintptr_t)thread_tid(new_thread), new_thread->sched_pri, rt_runq_count(pset), 0, 0); thread_setrun(new_thread, SCHED_HEADQ); thread_unlock(new_thread); @@ -4322,6 +4588,8 @@ processor_idle( return (THREAD_NULL); } + sched_update_pset_load_average(pset); + pset_unlock(pset); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, @@ -4333,10 +4601,11 @@ processor_idle( } else if (state == PROCESSOR_IDLE) { re_queue_tail(&pset->active_queue, &processor->processor_queue); + pset->active_processor_count++; + sched_update_pset_load_average(pset); + processor->state = PROCESSOR_RUNNING; - processor->current_pri = IDLEPRI; - processor->current_thmode = TH_MODE_FIXED; - processor->current_sfi_class = SFI_CLASS_KERNEL; + processor_state_update_idle(processor); processor->deadline = UINT64_MAX; } else if (state == PROCESSOR_SHUTDOWN) { @@ -4346,9 +4615,7 @@ processor_idle( */ if ((new_thread = processor->next_thread) != THREAD_NULL) { processor->next_thread = THREAD_NULL; - processor->current_pri = IDLEPRI; - processor->current_thmode = TH_MODE_FIXED; - processor->current_sfi_class = SFI_CLASS_KERNEL; + processor_state_update_idle(processor); processor->deadline = UINT64_MAX; pset_unlock(pset); @@ -4402,11 +4669,15 @@ idle_thread_create( kern_return_t result; thread_t thread; spl_t s; + char name[MAXTHREADNAMESIZE]; result = kernel_thread_create((thread_continue_t)idle_thread, NULL, MAXPRI_KERNEL, &thread); if (result != KERN_SUCCESS) return (result); + snprintf(name, sizeof(name), "idle #%d", processor->cpu_id); + thread_set_thread_name(thread, name); + s = splsched(); thread_lock(thread); thread->bound_processor = processor; @@ -4437,6 +4708,9 @@ sched_startup(void) simple_lock_init(&sched_vm_group_list_lock, 0); +#if __arm__ || __arm64__ + simple_lock_init(&sched_recommended_cores_lock, 0); +#endif /* __arm__ || __arm64__ */ result = kernel_thread_start_priority((thread_continue_t)sched_init_thread, (void *)SCHED(maintenance_continuation), MAXPRI_KERNEL, &thread); @@ -4458,12 +4732,19 @@ sched_startup(void) thread_block(THREAD_CONTINUE_NULL); } +#if __arm64__ +static _Atomic uint64_t sched_perfcontrol_callback_deadline; +#endif /* __arm64__ */ + + #if defined(CONFIG_SCHED_TIMESHARE_CORE) static volatile uint64_t sched_maintenance_deadline; static uint64_t sched_tick_last_abstime; static uint64_t sched_tick_delta; uint64_t sched_tick_max_delta; + + /* * sched_init_thread: * @@ -4518,6 +4799,8 @@ sched_timeshare_maintenance_continue(void) sched_tick += sched_tick_delta; + update_vm_info(); + /* * Compute various averages. */ @@ -4530,7 +4813,7 @@ sched_timeshare_maintenance_continue(void) */ SCHED(thread_update_scan)(&scan_context); - rt_runq_scan(&scan_context); + SCHED(rt_runq_scan)(&scan_context); uint64_t ctime = mach_absolute_time(); @@ -4550,6 +4833,19 @@ sched_timeshare_maintenance_continue(void) */ sched_vm_group_maintenance(); +#if __arm__ || __arm64__ + /* Check to see if the recommended cores failsafe is active */ + sched_recommended_cores_maintenance(); +#endif /* __arm__ || __arm64__ */ + + +#if DEBUG || DEVELOPMENT +#if __x86_64__ +#include + /* Check for long-duration interrupts */ + mp_interrupt_watchdog(); +#endif /* __x86_64__ */ +#endif /* DEBUG || DEVELOPMENT */ KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_MAINTENANCE) | DBG_FUNC_END, sched_pri_shifts[TH_BUCKET_SHARE_FG], sched_pri_shifts[TH_BUCKET_SHARE_BG], @@ -4588,6 +4884,19 @@ sched_timeshare_consider_maintenance(uint64_t ctime) { sched_maintenance_wakeups++; } } + +#if __arm64__ + uint64_t perf_deadline = __c11_atomic_load(&sched_perfcontrol_callback_deadline, memory_order_relaxed); + + if (__improbable(perf_deadline && ctime >= perf_deadline)) { + /* CAS in 0, if success, make callback. Otherwise let the next context switch check again. */ + if (__c11_atomic_compare_exchange_strong(&sched_perfcontrol_callback_deadline, &perf_deadline, 0, + memory_order_relaxed, memory_order_relaxed)) { + machine_perfcontrol_deadline_passed(perf_deadline); + } + } +#endif /* __arm64__ */ + } #endif /* CONFIG_SCHED_TIMESHARE_CORE */ @@ -4841,36 +5150,353 @@ sched_timer_deadline_tracking_init(void) { nanoseconds_to_absolutetime(TIMER_DEADLINE_TRACKING_BIN_2_DEFAULT, &timer_deadline_tracking_bin_2); } +#if __arm__ || __arm64__ -kern_return_t -sched_work_interval_notify(thread_t thread, uint64_t work_interval_id, uint64_t start, uint64_t finish, uint64_t deadline, uint64_t next_start, uint32_t flags) +uint32_t perfcontrol_requested_recommended_cores = ALL_CORES_RECOMMENDED; +uint32_t perfcontrol_requested_recommended_core_count = MAX_CPUS; +boolean_t perfcontrol_failsafe_active = FALSE; + +uint64_t perfcontrol_failsafe_maintenance_runnable_time; +uint64_t perfcontrol_failsafe_activation_time; +uint64_t perfcontrol_failsafe_deactivation_time; + +/* data covering who likely caused it and how long they ran */ +#define FAILSAFE_NAME_LEN 33 /* (2*MAXCOMLEN)+1 from size of p_name */ +char perfcontrol_failsafe_name[FAILSAFE_NAME_LEN]; +int perfcontrol_failsafe_pid; +uint64_t perfcontrol_failsafe_tid; +uint64_t perfcontrol_failsafe_thread_timer_at_start; +uint64_t perfcontrol_failsafe_thread_timer_last_seen; +uint32_t perfcontrol_failsafe_recommended_at_trigger; + +/* + * Perf controller calls here to update the recommended core bitmask. + * If the failsafe is active, we don't immediately apply the new value. + * Instead, we store the new request and use it after the failsafe deactivates. + * + * If the failsafe is not active, immediately apply the update. + * + * No scheduler locks are held, no other locks are held that scheduler might depend on, + * interrupts are enabled + * + * currently prototype is in osfmk/arm/machine_routines.h + */ +void +sched_perfcontrol_update_recommended_cores(uint32_t recommended_cores) { - int urgency; - uint64_t urgency_param1, urgency_param2; - spl_t s; + assert(preemption_enabled()); + + spl_t s = splsched(); + simple_lock(&sched_recommended_cores_lock); - if (work_interval_id == 0) { - return (KERN_INVALID_ARGUMENT); + perfcontrol_requested_recommended_cores = recommended_cores; + perfcontrol_requested_recommended_core_count = __builtin_popcountll(recommended_cores); + + if (perfcontrol_failsafe_active == FALSE) + sched_update_recommended_cores(perfcontrol_requested_recommended_cores); + else + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_SCHED,MACH_REC_CORES_FAILSAFE) | DBG_FUNC_NONE, + perfcontrol_requested_recommended_cores, + sched_maintenance_thread->last_made_runnable_time, 0, 0, 0); + + simple_unlock(&sched_recommended_cores_lock); + splx(s); +} + +/* + * Consider whether we need to activate the recommended cores failsafe + * + * Called from quantum timer interrupt context of a realtime thread + * No scheduler locks are held, interrupts are disabled + */ +void +sched_consider_recommended_cores(uint64_t ctime, thread_t cur_thread) +{ + /* + * Check if a realtime thread is starving the system + * and bringing up non-recommended cores would help + * + * TODO: Is this the correct check for recommended == possible cores? + * TODO: Validate the checks without the relevant lock are OK. + */ + + if (__improbable(perfcontrol_failsafe_active == TRUE)) { + /* keep track of how long the responsible thread runs */ + + simple_lock(&sched_recommended_cores_lock); + + if (perfcontrol_failsafe_active == TRUE && + cur_thread->thread_id == perfcontrol_failsafe_tid) { + perfcontrol_failsafe_thread_timer_last_seen = timer_grab(&cur_thread->user_timer) + + timer_grab(&cur_thread->system_timer); + } + + simple_unlock(&sched_recommended_cores_lock); + + /* we're already trying to solve the problem, so bail */ + return; } - assert(thread == current_thread()); + /* The failsafe won't help if there are no more processors to enable */ + if (__probable(perfcontrol_requested_recommended_core_count >= processor_count)) + return; - thread_mtx_lock(thread); - if (thread->work_interval_id != work_interval_id) { - thread_mtx_unlock(thread); - return (KERN_INVALID_ARGUMENT); + uint64_t too_long_ago = ctime - perfcontrol_failsafe_starvation_threshold; + + /* Use the maintenance thread as our canary in the coal mine */ + thread_t m_thread = sched_maintenance_thread; + + /* If it doesn't look bad, nothing to see here */ + if (__probable(m_thread->last_made_runnable_time >= too_long_ago)) + return; + + /* It looks bad, take the lock to be sure */ + thread_lock(m_thread); + + if (m_thread->runq == PROCESSOR_NULL || + (m_thread->state & (TH_RUN|TH_WAIT)) != TH_RUN || + m_thread->last_made_runnable_time >= too_long_ago) { + /* + * Maintenance thread is either on cpu or blocked, and + * therefore wouldn't benefit from more cores + */ + thread_unlock(m_thread); + return; } - thread_mtx_unlock(thread); - s = splsched(); - thread_lock(thread); - urgency = thread_get_urgency(thread, &urgency_param1, &urgency_param2); - thread_unlock(thread); + uint64_t maintenance_runnable_time = m_thread->last_made_runnable_time; + + thread_unlock(m_thread); + + /* + * There are cores disabled at perfcontrol's recommendation, but the + * system is so overloaded that the maintenance thread can't run. + * That likely means that perfcontrol can't run either, so it can't fix + * the recommendation. We have to kick in a failsafe to keep from starving. + * + * When the maintenance thread has been starved for too long, + * ignore the recommendation from perfcontrol and light up all the cores. + * + * TODO: Consider weird states like boot, sleep, or debugger + */ + + simple_lock(&sched_recommended_cores_lock); + + if (perfcontrol_failsafe_active == TRUE) { + simple_unlock(&sched_recommended_cores_lock); + return; + } + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_SCHED,MACH_REC_CORES_FAILSAFE) | DBG_FUNC_START, + perfcontrol_requested_recommended_cores, maintenance_runnable_time, 0, 0, 0); + + perfcontrol_failsafe_active = TRUE; + perfcontrol_failsafe_activation_time = mach_absolute_time(); + perfcontrol_failsafe_maintenance_runnable_time = maintenance_runnable_time; + perfcontrol_failsafe_recommended_at_trigger = perfcontrol_requested_recommended_cores; + + /* Capture some data about who screwed up (assuming that the thread on core is at fault) */ + task_t task = cur_thread->task; + perfcontrol_failsafe_pid = task_pid(task); + strlcpy(perfcontrol_failsafe_name, proc_name_address(task->bsd_info), sizeof(perfcontrol_failsafe_name)); + + perfcontrol_failsafe_tid = cur_thread->thread_id; + + /* Blame the thread for time it has run recently */ + uint64_t recent_computation = (ctime - cur_thread->computation_epoch) + cur_thread->computation_metered; + + uint64_t last_seen = timer_grab(&cur_thread->user_timer) + timer_grab(&cur_thread->system_timer); + + /* Compute the start time of the bad behavior in terms of the thread's on core time */ + perfcontrol_failsafe_thread_timer_at_start = last_seen - recent_computation; + perfcontrol_failsafe_thread_timer_last_seen = last_seen; + + /* Ignore the previously recommended core configuration */ + sched_update_recommended_cores(ALL_CORES_RECOMMENDED); + + simple_unlock(&sched_recommended_cores_lock); +} + +/* + * Now that our bacon has been saved by the failsafe, consider whether to turn it off + * + * Runs in the context of the maintenance thread, no locks held + */ +static void +sched_recommended_cores_maintenance(void) +{ + /* Common case - no failsafe, nothing to be done here */ + if (__probable(perfcontrol_failsafe_active == FALSE)) + return; + + uint64_t ctime = mach_absolute_time(); + + boolean_t print_diagnostic = FALSE; + char p_name[FAILSAFE_NAME_LEN] = ""; + + spl_t s = splsched(); + simple_lock(&sched_recommended_cores_lock); + + /* Check again, under the lock, to avoid races */ + if (perfcontrol_failsafe_active == FALSE) + goto out; + + /* + * Ensure that the other cores get another few ticks to run some threads + * If we don't have this hysteresis, the maintenance thread is the first + * to run, and then it immediately kills the other cores + */ + if ((ctime - perfcontrol_failsafe_activation_time) < perfcontrol_failsafe_starvation_threshold) + goto out; + + /* Capture some diagnostic state under the lock so we can print it out later */ + + int pid = perfcontrol_failsafe_pid; + uint64_t tid = perfcontrol_failsafe_tid; + + uint64_t thread_usage = perfcontrol_failsafe_thread_timer_last_seen - + perfcontrol_failsafe_thread_timer_at_start; + uint32_t rec_cores_before = perfcontrol_failsafe_recommended_at_trigger; + uint32_t rec_cores_after = perfcontrol_requested_recommended_cores; + uint64_t failsafe_duration = ctime - perfcontrol_failsafe_activation_time; + strlcpy(p_name, perfcontrol_failsafe_name, sizeof(p_name)); + + print_diagnostic = TRUE; + + /* Deactivate the failsafe and reinstate the requested recommendation settings */ + + perfcontrol_failsafe_deactivation_time = ctime; + perfcontrol_failsafe_active = FALSE; + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_SCHED,MACH_REC_CORES_FAILSAFE) | DBG_FUNC_END, + perfcontrol_requested_recommended_cores, failsafe_duration, 0, 0, 0); + + sched_update_recommended_cores(perfcontrol_requested_recommended_cores); + +out: + simple_unlock(&sched_recommended_cores_lock); splx(s); - machine_work_interval_notify(thread, work_interval_id, start, finish, deadline, next_start, urgency, flags); - return (KERN_SUCCESS); + if (print_diagnostic) { + uint64_t failsafe_duration_ms = 0, thread_usage_ms = 0; + + absolutetime_to_nanoseconds(failsafe_duration, &failsafe_duration_ms); + failsafe_duration_ms = failsafe_duration_ms / NSEC_PER_MSEC; + + absolutetime_to_nanoseconds(thread_usage, &thread_usage_ms); + thread_usage_ms = thread_usage_ms / NSEC_PER_MSEC; + + printf("recommended core failsafe kicked in for %lld ms " + "likely due to %s[%d] thread 0x%llx spending " + "%lld ms on cpu at realtime priority - " + "new recommendation: 0x%x -> 0x%x\n", + failsafe_duration_ms, p_name, pid, tid, thread_usage_ms, + rec_cores_before, rec_cores_after); + } +} + +/* + * Apply a new recommended cores mask to the processors it affects + * Runs after considering failsafes and such + * + * Iterate over processors and update their ->is_recommended field. + * If a processor is running, we let it drain out at its next + * quantum expiration or blocking point. If a processor is idle, there + * may be more work for it to do, so IPI it. + * + * interrupts disabled, sched_recommended_cores_lock is held + */ +static void +sched_update_recommended_cores(uint32_t recommended_cores) +{ + processor_set_t pset, nset; + processor_t processor; + uint64_t needs_exit_idle_mask = 0x0; + + processor = processor_list; + pset = processor->processor_set; + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED_UPDATE_REC_CORES) | DBG_FUNC_START, + recommended_cores, perfcontrol_failsafe_active, 0, 0, 0); + + if (__builtin_popcount(recommended_cores) == 0) { + recommended_cores |= 0x1U; /* add boot processor or we hang */ + } + + /* First set recommended cores */ + pset_lock(pset); + do { + + nset = processor->processor_set; + if (nset != pset) { + pset_unlock(pset); + pset = nset; + pset_lock(pset); + } + + pset->recommended_bitmask = recommended_cores; + + if (recommended_cores & (1ULL << processor->cpu_id)) { + processor->is_recommended = TRUE; + + if (processor->state == PROCESSOR_IDLE) { + if (processor->processor_primary == processor) { + re_queue_head(&pset->idle_queue, &processor->processor_queue); + } else { + re_queue_head(&pset->idle_secondary_queue, &processor->processor_queue); + } + if (processor != current_processor()) { + needs_exit_idle_mask |= (1ULL << processor->cpu_id); + } + } + } + } while ((processor = processor->processor_list) != NULL); + pset_unlock(pset); + + /* Now shutdown not recommended cores */ + processor = processor_list; + pset = processor->processor_set; + + pset_lock(pset); + do { + + nset = processor->processor_set; + if (nset != pset) { + pset_unlock(pset); + pset = nset; + pset_lock(pset); + } + + if (!(recommended_cores & (1ULL << processor->cpu_id))) { + processor->is_recommended = FALSE; + if (processor->state == PROCESSOR_IDLE) { + re_queue_head(&pset->unused_queue, &processor->processor_queue); + } + SCHED(processor_queue_shutdown)(processor); + /* pset unlocked */ + + SCHED(rt_queue_shutdown)(processor); + + pset_lock(pset); + } + } while ((processor = processor->processor_list) != NULL); + pset_unlock(pset); + + /* Issue all pending IPIs now that the pset lock has been dropped */ + for (int cpuid = lsb_first(needs_exit_idle_mask); cpuid >= 0; cpuid = lsb_next(needs_exit_idle_mask, cpuid)) { + processor = processor_array[cpuid]; + machine_signal_idle(processor); + } + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED_UPDATE_REC_CORES) | DBG_FUNC_END, + needs_exit_idle_mask, 0, 0, 0, 0); } +#endif /* __arm__ || __arm64__ */ void thread_set_options(uint32_t thopt) { spl_t x; @@ -4888,3 +5514,89 @@ void thread_set_options(uint32_t thopt) { void thread_set_pending_block_hint(thread_t thread, block_hint_t block_hint) { thread->pending_block_hint = block_hint; } + +uint32_t qos_max_parallelism(int qos, uint64_t options) +{ + return SCHED(qos_max_parallelism)(qos, options); +} + +uint32_t sched_qos_max_parallelism(__unused int qos, uint64_t options) +{ + host_basic_info_data_t hinfo; + mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT; + /* Query the machine layer for core information */ + __assert_only kern_return_t kret = host_info(host_self(), HOST_BASIC_INFO, + (host_info_t)&hinfo, &count); + assert(kret == KERN_SUCCESS); + + /* We would not want multiple realtime threads running on the + * same physical core; even for SMT capable machines. + */ + if (options & QOS_PARALLELISM_REALTIME) { + return hinfo.physical_cpu; + } + + if (options & QOS_PARALLELISM_COUNT_LOGICAL) { + return hinfo.logical_cpu; + } else { + return hinfo.physical_cpu; + } +} + +#if __arm64__ + +/* + * Set up or replace old timer with new timer + * + * Returns true if canceled old timer, false if it did not + */ +boolean_t +sched_perfcontrol_update_callback_deadline(uint64_t new_deadline) +{ + /* + * Exchange deadline for new deadline, if old deadline was nonzero, + * then I cancelled the callback, otherwise I didn't + */ + + uint64_t old_deadline = __c11_atomic_load(&sched_perfcontrol_callback_deadline, + memory_order_relaxed); + + + while (!__c11_atomic_compare_exchange_weak(&sched_perfcontrol_callback_deadline, + &old_deadline, new_deadline, + memory_order_relaxed, memory_order_relaxed)); + + + /* now old_deadline contains previous value, which might not be the same if it raced */ + + return (old_deadline != 0) ? TRUE : FALSE; +} + +#endif /* __arm64__ */ + +int +sched_get_pset_load_average(processor_set_t pset) +{ + return pset->load_average >> (PSET_LOAD_NUMERATOR_SHIFT - PSET_LOAD_FRACTIONAL_SHIFT); +} + +void +sched_update_pset_load_average(processor_set_t pset) +{ +#if DEBUG + queue_entry_t iter; + int count = 0; + qe_foreach(iter, &pset->active_queue) { + count++; + } + assertf(count == pset->active_processor_count, "count %d pset->active_processor_count %d\n", count, pset->active_processor_count); +#endif + + int load = ((pset->active_processor_count + pset->pset_runq.count + rt_runq_count(pset)) << PSET_LOAD_NUMERATOR_SHIFT); + int new_load_average = (pset->load_average + load) >> 1; + + pset->load_average = new_load_average; + +#if (DEVELOPMENT || DEBUG) +#endif +} diff --git a/osfmk/kern/sched_prim.h b/osfmk/kern/sched_prim.h index ca3227ec1..c0014e86b 100644 --- a/osfmk/kern/sched_prim.h +++ b/osfmk/kern/sched_prim.h @@ -86,6 +86,20 @@ extern void sched_startup(void); extern void sched_timebase_init(void); +extern void pset_rt_init(processor_set_t pset); + +extern void sched_rtglobal_init(processor_set_t pset); + +extern rt_queue_t sched_rtglobal_runq(processor_set_t pset); + +extern void sched_rtglobal_queue_shutdown(processor_t processor); + +extern int64_t sched_rtglobal_runq_count_sum(void); + +extern void sched_check_spill(processor_set_t pset, thread_t thread); + +extern bool sched_thread_should_yield(processor_t processor, thread_t thread); + /* Force a preemption point for a thread and wait for it to stop running */ extern boolean_t thread_stop( thread_t thread, @@ -199,12 +213,13 @@ extern void thread_setrun( thread_t thread, integer_t options); -#define SCHED_TAILQ 1 -#define SCHED_HEADQ 2 -#define SCHED_PREEMPT 4 - -extern uintptr_t sched_thread_on_rt_queue; -#define THREAD_ON_RT_RUNQ ((processor_t)(uintptr_t)&sched_thread_on_rt_queue) +typedef enum { + SCHED_NONE = 0x0, + SCHED_TAILQ = 0x1, + SCHED_HEADQ = 0x2, + SCHED_PREEMPT = 0x4, + SCHED_REBALANCE = 0x8, +} sched_options_t; extern processor_set_t task_choose_pset( task_t task); @@ -219,6 +234,9 @@ extern processor_t choose_processor( processor_t processor, thread_t thread); +extern void sched_SMT_balance( + processor_t processor, + processor_set_t pset); extern void thread_quantum_init( thread_t thread); @@ -247,6 +265,50 @@ struct sched_update_scan_context }; typedef struct sched_update_scan_context *sched_update_scan_context_t; +extern void sched_rtglobal_runq_scan(sched_update_scan_context_t scan_context); + +/* + * Enum to define various events which need IPIs. The IPI policy + * engine decides what kind of IPI to use based on destination + * processor state, thread and one of the following scheduling events. + */ +typedef enum { + SCHED_IPI_EVENT_BOUND_THR = 0x1, + SCHED_IPI_EVENT_PREEMPT = 0x2, + SCHED_IPI_EVENT_SMT_REBAL = 0x3, + SCHED_IPI_EVENT_SPILL = 0x4, + SCHED_IPI_EVENT_REBALANCE = 0x5, +} sched_ipi_event_t; + + +/* Enum to define various IPI types used by the scheduler */ +typedef enum { + SCHED_IPI_NONE = 0x0, + SCHED_IPI_IMMEDIATE = 0x1, + SCHED_IPI_IDLE = 0x2, + SCHED_IPI_DEFERRED = 0x3, +} sched_ipi_type_t; + +/* The IPI policy engine behaves in the following manner: + * - All scheduler events which need an IPI invoke sched_ipi_action() with + * the appropriate destination processor, thread and event. + * - sched_ipi_action() performs basic checks, invokes the scheduler specific + * ipi_policy routine and sets pending_AST bits based on the result. + * - Once the pset lock is dropped, the scheduler invokes sched_ipi_perform() + * routine which actually sends the appropriate IPI to the destination core. + */ +extern sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread, + boolean_t dst_idle, sched_ipi_event_t event); +extern void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi); + +/* sched_ipi_policy() is the global default IPI policy for all schedulers */ +extern sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread, + boolean_t dst_idle, sched_ipi_event_t event); + +/* sched_ipi_deferred_policy() is the global default deferred IPI policy for all schedulers */ +extern sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset, + processor_t dst, sched_ipi_event_t event); + #if defined(CONFIG_SCHED_TIMESHARE_CORE) extern boolean_t thread_update_add_thread(thread_t thread); @@ -264,8 +326,6 @@ extern int sched_compute_timeshare_priority(thread_t thread); #endif /* CONFIG_SCHED_TIMESHARE_CORE */ -extern void rt_runq_scan(sched_update_scan_context_t scan_context); - /* Remove thread from its run queue */ extern boolean_t thread_run_queue_remove(thread_t thread); thread_t thread_run_queue_remove_for_handoff(thread_t thread); @@ -306,6 +366,7 @@ extern void sched_stats_handle_runq_change( int old_count); +#if DEBUG #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) \ do { \ @@ -324,6 +385,13 @@ do { \ } \ } while (0) +#else /* DEBUG */ + +#define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) do { }while(0) +#define SCHED_STATS_RUNQ_CHANGE(stats, old_count) do { }while(0) + +#endif /* DEBUG */ + extern uint32_t sched_debug_flags; #define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS 0x00000001 #define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS 0x00000002 @@ -363,14 +431,25 @@ extern void thread_tell_urgency( extern void active_rt_threads( boolean_t active); +/* Returns the perfcontrol attribute for the thread */ +extern perfcontrol_class_t thread_get_perfcontrol_class( + thread_t thread); + +#define PSET_LOAD_NUMERATOR_SHIFT 16 +#define PSET_LOAD_FRACTIONAL_SHIFT 4 + +extern int sched_get_pset_load_average(processor_set_t pset); +extern void sched_update_pset_load_average(processor_set_t pset); + +/* Generic routine for Non-AMP schedulers to calculate parallelism */ +extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options); + #endif /* MACH_KERNEL_PRIVATE */ __BEGIN_DECLS #ifdef XNU_KERNEL_PRIVATE -extern boolean_t assert_wait_possible(void); - /* Toggles a global override to turn off CPU Throttling */ #define CPU_THROTTLE_DISABLE 0 #define CPU_THROTTLE_ENABLE 1 @@ -397,8 +476,6 @@ extern void thread_exception_return(void) __dead2; /* String declaring the name of the current scheduler */ extern char sched_string[SCHED_STRING_MAX_LENGTH]; -extern kern_return_t sched_work_interval_notify(thread_t thread, uint64_t work_interval_id, uint64_t start, uint64_t finish, uint64_t deadline, uint64_t next_start, uint32_t flags); - extern thread_t port_name_to_thread_for_ulock(mach_port_name_t thread_name); /* Attempt to context switch to a specific runnable thread */ @@ -417,6 +494,11 @@ extern thread_t thread_wakeup_identify(event_t event, int priority); extern void thread_set_pending_block_hint( thread_t thread, block_hint_t block_hint); + +#define QOS_PARALLELISM_COUNT_LOGICAL 0x1 +#define QOS_PARALLELISM_REALTIME 0x2 +extern uint32_t qos_max_parallelism(int qos, uint64_t options); + #endif /* KERNEL_PRIVATE */ /* Context switch */ @@ -492,7 +574,18 @@ extern boolean_t preemption_enabled(void); #error Enable at least one scheduler algorithm in osfmk/conf/MASTER.XXX #endif +#if DEBUG #define SCHED(f) (sched_current_dispatch->f) +#else /* DEBUG */ + +/* + * For DEV & REL kernels, use a static dispatch table instead of + * using the indirect function table. + */ +extern const struct sched_dispatch_table sched_multiq_dispatch; +#define SCHED(f) (sched_multiq_dispatch.f) + +#endif /* DEBUG */ struct sched_dispatch_table { const char *sched_name; @@ -629,6 +722,29 @@ struct sched_dispatch_table { boolean_t multiple_psets_enabled; /* Supports scheduler groups */ boolean_t sched_groups_enabled; + + /* Supports avoid-processor */ + boolean_t avoid_processor_enabled; + + /* Returns true if this processor should avoid running this thread. */ + bool (*thread_avoid_processor)(processor_t processor, thread_t thread); + + /* + * Invoked when a processor is about to choose the idle thread + * Used to send IPIs to a processor which would be preferred to be idle instead. + * Called with pset lock held, returns pset lock unlocked. + */ + void (*processor_balance)(processor_t processor, processor_set_t pset); + rt_queue_t (*rt_runq)(processor_set_t pset); + void (*rt_init)(processor_set_t pset); + void (*rt_queue_shutdown)(processor_t processor); + void (*rt_runq_scan)(sched_update_scan_context_t scan_context); + int64_t (*rt_runq_count_sum)(void); + + uint32_t (*qos_max_parallelism)(int qos, uint64_t options); + void (*check_spill)(processor_set_t pset, thread_t thread); + sched_ipi_type_t (*ipi_policy)(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event); + bool (*thread_should_yield)(processor_t processor, thread_t thread); }; #if defined(CONFIG_SCHED_TRADITIONAL) diff --git a/osfmk/kern/sched_proto.c b/osfmk/kern/sched_proto.c index e0d3c14ff..1df45e7ab 100644 --- a/osfmk/kern/sched_proto.c +++ b/osfmk/kern/sched_proto.c @@ -185,6 +185,20 @@ const struct sched_dispatch_table sched_proto_dispatch = { .direct_dispatch_to_idle_processors = TRUE, .multiple_psets_enabled = TRUE, .sched_groups_enabled = FALSE, + .avoid_processor_enabled = FALSE, + .thread_avoid_processor = NULL, + .processor_balance = sched_SMT_balance, + + .rt_runq = sched_rtglobal_runq, + .rt_init = sched_rtglobal_init, + .rt_queue_shutdown = sched_rtglobal_queue_shutdown, + .rt_runq_scan = sched_rtglobal_runq_scan, + .rt_runq_count_sum = sched_rtglobal_runq_count_sum, + + .qos_max_parallelism = sched_qos_max_parallelism, + .check_spill = sched_check_spill, + .ipi_policy = sched_ipi_policy, + .thread_should_yield = sched_thread_should_yield, }; static struct run_queue *global_runq; @@ -473,12 +487,10 @@ sched_proto_processor_queue_has_priority(processor_t processor __unused, simple_lock(&global_runq_lock); - if (global_runq->count == 0) - result = FALSE; - else if (gte) + if (gte) result = global_runq->highq >= priority; else - result = global_runq->highq >= priority; + result = global_runq->highq > priority; simple_unlock(&global_runq_lock); diff --git a/osfmk/kern/sched_traditional.c b/osfmk/kern/sched_traditional.c index 80f950feb..7bc3d4393 100644 --- a/osfmk/kern/sched_traditional.c +++ b/osfmk/kern/sched_traditional.c @@ -160,6 +160,20 @@ const struct sched_dispatch_table sched_traditional_dispatch = { .direct_dispatch_to_idle_processors = TRUE, .multiple_psets_enabled = TRUE, .sched_groups_enabled = FALSE, + .avoid_processor_enabled = FALSE, + .thread_avoid_processor = NULL, + .processor_balance = sched_SMT_balance, + + .rt_runq = sched_rtglobal_runq, + .rt_init = sched_rtglobal_init, + .rt_queue_shutdown = sched_rtglobal_queue_shutdown, + .rt_runq_scan = sched_rtglobal_runq_scan, + .rt_runq_count_sum = sched_rtglobal_runq_count_sum, + + .qos_max_parallelism = sched_qos_max_parallelism, + .check_spill = sched_check_spill, + .ipi_policy = sched_ipi_policy, + .thread_should_yield = sched_thread_should_yield, }; const struct sched_dispatch_table sched_traditional_with_pset_runqueue_dispatch = { @@ -194,6 +208,20 @@ const struct sched_dispatch_table sched_traditional_with_pset_runqueue_dispatch .direct_dispatch_to_idle_processors = FALSE, .multiple_psets_enabled = TRUE, .sched_groups_enabled = FALSE, + .avoid_processor_enabled = FALSE, + .thread_avoid_processor = NULL, + .processor_balance = sched_SMT_balance, + + .rt_runq = sched_rtglobal_runq, + .rt_init = sched_rtglobal_init, + .rt_queue_shutdown = sched_rtglobal_queue_shutdown, + .rt_runq_scan = sched_rtglobal_runq_scan, + .rt_runq_count_sum = sched_rtglobal_runq_count_sum, + + .qos_max_parallelism = sched_qos_max_parallelism, + .check_spill = sched_check_spill, + .ipi_policy = sched_ipi_policy, + .thread_should_yield = sched_thread_should_yield, }; static void @@ -431,11 +459,7 @@ sched_traditional_processor_queue_has_priority(processor_t processor, int priority, boolean_t gte) { - run_queue_t runq = runq_for_processor(processor); - - if (runq->count == 0) - return FALSE; - else if (gte) + if (gte) return runq_for_processor(processor)->highq >= priority; else return runq_for_processor(processor)->highq > priority; diff --git a/osfmk/kern/sfi.c b/osfmk/kern/sfi.c index 8a67ec034..80fa2c105 100644 --- a/osfmk/kern/sfi.c +++ b/osfmk/kern/sfi.c @@ -380,15 +380,14 @@ static void sfi_timer_global_off( pset_unlock(pset); - processor = processor_list; - do { - if (needs_cause_ast_mask & (1U << processor->cpu_id)) { - if (processor == current_processor()) - ast_on(AST_SFI); - else - cause_ast_check(processor); + for (int cpuid = lsb_first(needs_cause_ast_mask); cpuid >= 0; cpuid = lsb_next(needs_cause_ast_mask, cpuid)) { + processor = processor_array[cpuid]; + if (processor == current_processor()) { + ast_on(AST_SFI); + } else { + cause_ast_check(processor); } - } while ((processor = processor->processor_list) != NULL); + } /* Re-arm timer if still enabled */ simple_lock(&sfi_lock); @@ -915,7 +914,9 @@ static inline void _sfi_wait_cleanup(sched_call_t callback) { simple_unlock(&sfi_lock); splx(s); assert((SFI_CLASS_UNSPECIFIED < current_sfi_wait_class) && (current_sfi_wait_class < MAX_SFI_CLASS_ID)); +#if !CONFIG_EMBEDDED ledger_credit(self->task->ledger, task_ledgers.sfi_wait_times[current_sfi_wait_class], sfi_wait_time); +#endif /* !CONFIG_EMBEDDED */ } /* diff --git a/osfmk/kern/stack.c b/osfmk/kern/stack.c index 0cb793286..18db3f24b 100644 --- a/osfmk/kern/stack.c +++ b/osfmk/kern/stack.c @@ -45,6 +45,7 @@ #include #include +#include /* * We allocate stacks from generic kernel VM. @@ -80,25 +81,6 @@ vm_offset_t kernel_stack_size; vm_offset_t kernel_stack_mask; vm_offset_t kernel_stack_depth_max; -static inline void -STACK_ZINFO_PALLOC(thread_t thread) -{ - ledger_credit(thread->t_ledger, task_ledgers.tkm_private, kernel_stack_size); -} - -static inline void -STACK_ZINFO_PFREE(thread_t thread) -{ - ledger_debit(thread->t_ledger, task_ledgers.tkm_private, kernel_stack_size); -} - -static inline void -STACK_ZINFO_HANDOFF(thread_t from, thread_t to) -{ - ledger_debit(from->t_ledger, task_ledgers.tkm_private, kernel_stack_size); - ledger_credit(to->t_ledger, task_ledgers.tkm_private, kernel_stack_size); -} - /* * The next field is at the base of the stack, * so the low end is left unsullied. @@ -160,9 +142,10 @@ stack_init(void) static vm_offset_t stack_alloc_internal(void) { - vm_offset_t stack; + vm_offset_t stack = 0; spl_t s; - int guard_flags; + int flags = 0; + kern_return_t kr = KERN_SUCCESS; s = splsched(); stack_lock(); @@ -189,14 +172,15 @@ stack_alloc_internal(void) * for these. */ - guard_flags = KMA_GUARD_FIRST | KMA_GUARD_LAST; - if (kernel_memory_allocate(kernel_map, &stack, + flags = KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT; + kr = kernel_memory_allocate(kernel_map, &stack, kernel_stack_size + (2*PAGE_SIZE), stack_addr_mask, - KMA_KSTACK | KMA_KOBJECT | guard_flags, - VM_KERN_MEMORY_STACK) - != KERN_SUCCESS) - panic("stack_alloc: kernel_memory_allocate"); + flags, + VM_KERN_MEMORY_STACK); + if (kr != KERN_SUCCESS) { + panic("stack_alloc: kernel_memory_allocate(size:0x%llx, mask: 0x%llx, flags: 0x%x) failed with %d\n", (uint64_t)(kernel_stack_size + (2*PAGE_SIZE)), (uint64_t)stack_addr_mask, flags, kr); + } /* * The stack address that comes back is the address of the lower @@ -215,7 +199,6 @@ stack_alloc( assert(thread->kernel_stack == 0); machine_stack_attach(thread, stack_alloc_internal()); - STACK_ZINFO_PALLOC(thread); } void @@ -223,7 +206,6 @@ stack_handoff(thread_t from, thread_t to) { assert(from == current_thread()); machine_stack_handoff(from, to); - STACK_ZINFO_HANDOFF(from, to); } /* @@ -237,9 +219,13 @@ stack_free( { vm_offset_t stack = machine_stack_detach(thread); +#if KASAN + kasan_unpoison_stack(stack, kernel_stack_size); + kasan_unpoison_fakestack(thread); +#endif + assert(stack); if (stack != thread->reserved_stack) { - STACK_ZINFO_PFREE(thread); stack_free_stack(stack); } } @@ -249,8 +235,10 @@ stack_free_reserved( thread_t thread) { if (thread->reserved_stack != thread->kernel_stack) { +#if KASAN + kasan_unpoison_stack(thread->reserved_stack, kernel_stack_size); +#endif stack_free_stack(thread->reserved_stack); - STACK_ZINFO_PFREE(thread); } } @@ -300,7 +288,6 @@ stack_alloc_try( cache = &PROCESSOR_DATA(current_processor(), stack_cache); stack = cache->free; if (stack != 0) { - STACK_ZINFO_PALLOC(thread); cache->free = stack_next(stack); cache->count--; } @@ -309,7 +296,6 @@ stack_alloc_try( stack_lock(); stack = stack_free_list; if (stack != 0) { - STACK_ZINFO_PALLOC(thread); stack_free_list = stack_next(stack); stack_free_count--; stack_free_delta--; diff --git a/osfmk/kern/startup.c b/osfmk/kern/startup.c index 34ebd7481..c76524bac 100644 --- a/osfmk/kern/startup.c +++ b/osfmk/kern/startup.c @@ -132,9 +132,7 @@ #include #endif -#if CONFIG_BANK #include -#endif #if ALTERNATE_DEBUGGER #include @@ -146,6 +144,9 @@ #if CONFIG_MACF #include +#if CONFIG_VNGUARD +extern void vnguard_policy_init(void); +#endif #endif #if KPC @@ -156,6 +157,11 @@ #include #endif +#include + +#if defined(__arm__) || defined(__arm64__) +#include // for arm_vm_prot_finalize +#endif #include static void kernel_bootstrap_thread(void); @@ -197,7 +203,7 @@ unsigned int wake_nkdbufs = 0; unsigned int write_trace_on_panic = 0; static char trace_typefilter[64] = { 0 }; boolean_t trace_serial = FALSE; -boolean_t oslog_early_boot_complete = FALSE; +boolean_t early_boot_complete = FALSE; /* mach leak logging */ int log_leaks = 0; @@ -277,6 +283,11 @@ kernel_bootstrap(void) oslog_init(); +#if KASAN + kernel_bootstrap_log("kasan_late_init"); + kasan_late_init(); +#endif + #if CONFIG_TELEMETRY kernel_bootstrap_log("telemetry_init"); telemetry_init(); @@ -339,6 +350,7 @@ kernel_bootstrap(void) /* * Initialize the IPC, task, and thread subsystems. */ + #if CONFIG_COALITIONS kernel_bootstrap_log("coalitions_init"); coalitions_init(); @@ -358,11 +370,9 @@ kernel_bootstrap(void) kernel_bootstrap_log("mach_init_activity_id"); mach_init_activity_id(); -#if CONFIG_BANK /* Initialize the BANK Manager. */ kernel_bootstrap_log("bank_init"); bank_init(); -#endif kernel_bootstrap_log("ipc_pthread_priority_init"); ipc_pthread_priority_init(); @@ -370,8 +380,6 @@ kernel_bootstrap(void) /* initialize the corpse config based on boot-args */ corpses_init(); - vm_user_init(); - /* * Create a kernel thread to execute the kernel bootstrap. */ @@ -394,6 +402,8 @@ int kth_started = 0; vm_offset_t vm_kernel_addrperm; vm_offset_t buf_kernel_addrperm; vm_offset_t vm_kernel_addrperm_ext; +uint64_t vm_kernel_addrhash_salt; +uint64_t vm_kernel_addrhash_salt_ext; /* * Now running in a thread. Kick off other services, @@ -442,6 +452,11 @@ kernel_bootstrap_thread(void) kernel_bootstrap_thread_log("thread_bind"); thread_bind(processor); +#if __arm64__ + if (IORamDiskBSDRoot()) { + cpm_preallocate_early(); + } +#endif /* __arm64__ */ /* * Initialize ipc thread call support. @@ -511,9 +526,7 @@ kernel_bootstrap_thread(void) kernel_bootstrap_thread_log("ktrace_init"); ktrace_init(); - if (new_nkdbufs > 0 || kdebug_serial || log_leaks) { - kdebug_boot_trace(new_nkdbufs, trace_typefilter); - } + kdebug_init(new_nkdbufs, trace_typefilter); kernel_bootstrap_log("prng_init"); prng_cpu_init(master_cpu); @@ -523,6 +536,10 @@ kernel_bootstrap_thread(void) bsd_early_init(); #endif +#if defined(__arm64__) + ml_lockdown_init(); +#endif + #ifdef IOKIT kernel_bootstrap_log("PE_init_iokit"); PE_init_iokit(); @@ -530,10 +547,16 @@ kernel_bootstrap_thread(void) assert(ml_get_interrupts_enabled() == FALSE); - // Set this flag to indicate that it is now okay to start testing - // for interrupts / preemeption disabled while logging - oslog_early_boot_complete = TRUE; + /* + * Past this point, kernel subsystems that expect to operate with + * interrupts or preemption enabled may begin enforcement. + */ + early_boot_complete = TRUE; +#if INTERRUPT_MASKED_DEBUG + // Reset interrupts masked timeout before we enable interrupts + ml_spin_debug_clear_self(); +#endif (void) spllo(); /* Allow interruptions */ #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 @@ -557,8 +580,20 @@ kernel_bootstrap_thread(void) #if CONFIG_MACF kernel_bootstrap_log("mac_policy_initmach"); mac_policy_initmach(); +#if CONFIG_VNGUARD + vnguard_policy_init(); +#endif #endif +#if defined(__arm__) || defined(__arm64__) +#if CONFIG_KERNEL_INTEGRITY + machine_lockdown_preflight(); +#endif + /* + * Finalize protections on statically mapped pages now that comm page mapping is established. + */ + arm_vm_prot_finalize(PE_state.bootArgs); +#endif #if CONFIG_SCHED_SFI kernel_bootstrap_log("sfi_init"); @@ -580,6 +615,8 @@ kernel_bootstrap_thread(void) buf_kernel_addrperm |= 1; read_random(&vm_kernel_addrperm_ext, sizeof(vm_kernel_addrperm_ext)); vm_kernel_addrperm_ext |= 1; + read_random(&vm_kernel_addrhash_salt, sizeof(vm_kernel_addrhash_salt)); + read_random(&vm_kernel_addrhash_salt_ext, sizeof(vm_kernel_addrhash_salt_ext)); vm_set_restrictions(); @@ -598,6 +635,11 @@ kernel_bootstrap_thread(void) */ OSKextRemoveKextBootstrap(); + /* + * Get rid of pages used for early boot tracing. + */ + kdebug_free_early_buf(); + serial_keyboard_init(); /* Start serial keyboard if wanted */ vm_page_init_local_q(); @@ -671,7 +713,7 @@ processor_start_thread(void *machine_param) * * Start the first thread on a processor. */ -static void +static void __attribute__((noreturn)) load_context( thread_t thread) { @@ -709,11 +751,9 @@ load_context( sched_run_incr(thread); processor->active_thread = thread; - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; - processor->current_sfi_class = SFI_CLASS_KERNEL; + processor_state_update_explicit(processor, thread->sched_pri, + SFI_CLASS_KERNEL, PSET_SMP, thread_get_perfcontrol_class(thread)); processor->starting_pri = thread->sched_pri; - processor->deadline = UINT64_MAX; thread->last_processor = processor; @@ -724,6 +764,7 @@ load_context( timer_start(&PROCESSOR_DATA(processor, system_state), processor->last_dispatch); PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, system_state); + PMAP_ACTIVATE_USER(thread, processor->cpu_id); load_context_kprintf("machine_load_context\n"); @@ -745,8 +786,15 @@ scale_setup() if (scale > 16) scale = 16; task_max_base = 2500; - } else if ((uint64_t)sane_size >= (uint64_t)(3 * 1024 * 1024 *1024ULL)) - scale = 2; + /* Raise limits for machines with >= 3GB */ + } else if ((uint64_t)sane_size >= (uint64_t)(3 * 1024 * 1024 *1024ULL)) { + if ((uint64_t)sane_size < (uint64_t)(8 * 1024 * 1024 *1024ULL)) { + scale = 2; + } else { + /* limit to 64GB */ + scale = MIN(16, (int)((uint64_t)sane_size / (uint64_t)(4 * 1024 * 1024 *1024ULL))); + } + } task_max = MAX(task_max, task_max_base * scale); diff --git a/osfmk/kern/sync_sema.c b/osfmk/kern/sync_sema.c index 7f0fca493..0a6f7b33d 100644 --- a/osfmk/kern/sync_sema.c +++ b/osfmk/kern/sync_sema.c @@ -119,12 +119,6 @@ semaphore_wait_internal( int option, void (*caller_cont)(kern_return_t)); -void -kdp_sema_find_owner( - struct waitq * waitq, - event64_t event, - thread_waitinfo_t * waitinfo); - static __inline__ uint64_t semaphore_deadline( unsigned int sec, diff --git a/osfmk/kern/syscall_subr.c b/osfmk/kern/syscall_subr.c index a0c1d38e3..e944c79d9 100644 --- a/osfmk/kern/syscall_subr.c +++ b/osfmk/kern/syscall_subr.c @@ -109,11 +109,11 @@ static void swtch_continue(void) { processor_t myprocessor; - boolean_t result; + boolean_t result; - disable_preemption(); + disable_preemption(); myprocessor = current_processor(); - result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; + result = SCHED(thread_should_yield)(myprocessor, current_thread()); enable_preemption(); thread_syscall_return(result); @@ -129,7 +129,7 @@ swtch( disable_preemption(); myprocessor = current_processor(); - if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { + if (!SCHED(thread_should_yield)(myprocessor, current_thread())) { mp_enable_preemption(); return (FALSE); @@ -142,7 +142,7 @@ swtch( disable_preemption(); myprocessor = current_processor(); - result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; + result = SCHED(thread_should_yield)(myprocessor, current_thread()); enable_preemption(); return (result); @@ -152,13 +152,13 @@ static void swtch_pri_continue(void) { processor_t myprocessor; - boolean_t result; + boolean_t result; thread_depress_abort_internal(current_thread()); - disable_preemption(); + disable_preemption(); myprocessor = current_processor(); - result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; + result = SCHED(thread_should_yield)(myprocessor, current_thread()); mp_enable_preemption(); thread_syscall_return(result); @@ -174,7 +174,7 @@ __unused struct swtch_pri_args *args) disable_preemption(); myprocessor = current_processor(); - if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { + if (!SCHED(thread_should_yield)(myprocessor, current_thread())) { mp_enable_preemption(); return (FALSE); @@ -191,7 +191,7 @@ __unused struct swtch_pri_args *args) disable_preemption(); myprocessor = current_processor(); - result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; + result = SCHED(thread_should_yield)(myprocessor, current_thread()); enable_preemption(); return (result); @@ -496,6 +496,7 @@ thread_depress_abstime( 0); myprocessor->current_pri = self->sched_pri; + myprocessor->current_perfctl_class = thread_get_perfcontrol_class(self); self->sched_flags |= TH_SFLAG_DEPRESS; if (interval != 0) { @@ -599,6 +600,7 @@ thread_poll_yield( 0); myprocessor->current_pri = self->sched_pri; + myprocessor->current_perfctl_class = thread_get_perfcontrol_class(self); } self->computation_epoch = abstime; self->computation_metered = 0; @@ -626,7 +628,7 @@ thread_yield_internal( disable_preemption(); myprocessor = current_processor(); - if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { + if (!SCHED(thread_should_yield)(myprocessor, current_thread())) { mp_enable_preemption(); return; diff --git a/osfmk/kern/syscall_sw.c b/osfmk/kern/syscall_sw.c index e9529cfb1..5c4bd06f3 100644 --- a/osfmk/kern/syscall_sw.c +++ b/osfmk/kern/syscall_sw.c @@ -152,7 +152,7 @@ const mach_trap_t mach_trap_table[MACH_TRAP_TABLE_COUNT] = { /* 47 */ MACH_TRAP(kern_invalid, 0, 0, NULL), /* 48 */ MACH_TRAP(macx_swapon, 4, 5, munge_lwww), /* 49 */ MACH_TRAP(macx_swapoff, 2, 3, munge_lw), -/* 50 */ MACH_TRAP(kern_invalid, 0, 0, NULL), +/* 50 */ MACH_TRAP(thread_get_special_reply_port, 0, 0, NULL), /* 51 */ MACH_TRAP(macx_triggers, 4, 4, munge_wwww), /* 52 */ MACH_TRAP(macx_backing_store_suspend, 1, 1, munge_w), /* 53 */ MACH_TRAP(macx_backing_store_recovery, 1, 1, munge_w), @@ -287,7 +287,7 @@ const char * mach_syscall_name_table[MACH_TRAP_TABLE_COUNT] = { /* 47 */ "kern_invalid", /* 48 */ "macx_swapon", /* 49 */ "macx_swapoff", -/* 50 */ "kern_invalid", +/* 50 */ "thread_get_special_reply_port", /* 51 */ "macx_triggers", /* 52 */ "macx_backing_store_suspend", /* 53 */ "macx_backing_store_recovery", diff --git a/osfmk/kern/syscall_sw.h b/osfmk/kern/syscall_sw.h index 5137a1741..2816a65fc 100644 --- a/osfmk/kern/syscall_sw.h +++ b/osfmk/kern/syscall_sw.h @@ -68,6 +68,8 @@ */ #if CONFIG_REQUIRES_U32_MUNGING typedef void mach_munge_t(void *); +#elif __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +typedef int mach_munge_t(const void *, void *); #endif typedef struct { diff --git a/osfmk/kern/task.c b/osfmk/kern/task.c index 133f6b718..c2a29291e 100644 --- a/osfmk/kern/task.c +++ b/osfmk/kern/task.c @@ -94,6 +94,7 @@ #include #include #include +#include #include #include @@ -130,6 +131,13 @@ #include #endif +#if MONOTONIC +#include +#include +#endif /* MONOTONIC */ + +#include + #include #include #include /* for kernel_map, ipc_kernel_map */ @@ -175,7 +183,6 @@ lck_grp_t task_lck_grp; lck_grp_attr_t task_lck_grp_attr; extern int exc_via_corpse_forking; -extern int unify_corpse_blob_alloc; extern int corpse_for_fatal_memkill; /* Flag set by core audio when audio is playing. Used to stifle EXC_RESOURCE generation when active. */ @@ -192,10 +199,11 @@ ledger_template_t task_ledger_template = NULL; struct _task_ledger_indices task_ledgers __attribute__((used)) = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +#if !CONFIG_EMBEDDED { 0 /* initialized at runtime */}, -#ifdef CONFIG_BANK +#endif /* !CONFIG_EMBEDDED */ + -1, -1, -1, -1, -#endif -1, -1, }; @@ -214,7 +222,6 @@ void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_PROCESS_IS_CAUSING_TOO kern_return_t task_suspend_internal(task_t); kern_return_t task_resume_internal(task_t); static kern_return_t task_start_halt_locked(task_t task, boolean_t should_mark_corpse); -int proc_list_uptrs(void *p, uint64_t *udata_buffer, int size); extern kern_return_t iokit_task_terminate(task_t task); @@ -275,11 +282,13 @@ extern int proc_pid(struct proc *p); extern int proc_selfpid(void); extern char *proc_name_address(struct proc *p); extern uint64_t get_dispatchqueue_offset_from_proc(void *); +extern int kevent_proc_copy_uptrs(void *proc, uint64_t *buf, int bufsize); #if CONFIG_MEMORYSTATUS extern void proc_memstat_terminated(struct proc* p, boolean_t set); extern void memorystatus_on_ledger_footprint_exceeded(int warning, boolean_t memlimit_is_active, boolean_t memlimit_is_fatal); extern void memorystatus_log_exception(const int max_footprint_mb, boolean_t memlimit_is_active, boolean_t memlimit_is_fatal); +extern boolean_t memorystatus_allowed_vm_map_fork(__unused task_t task); #endif /* CONFIG_MEMORYSTATUS */ #endif /* MACH_BSD */ @@ -336,6 +345,8 @@ task_set_64bit( machine_thread_switch_addrmode(thread); thread_mtx_unlock(thread); +#if defined(__arm64__) + /* specifically, if running on H9 */ if (thread == current_thread()) { uint64_t arg1, arg2; int urgency; @@ -348,13 +359,16 @@ task_set_64bit( * * This is needed for bring-up, a different callback should be used * in the future. + * + * TODO: Remove this callout when we no longer support 32-bit code on H9 */ thread_lock(thread); urgency = thread_get_urgency(thread, &arg1, &arg2); - machine_thread_going_on_core(thread, urgency, 0, 0); + machine_thread_going_on_core(thread, urgency, 0, 0, mach_approximate_time()); thread_unlock(thread); splx(spl); } +#endif /* defined(__arm64__) */ } #endif /* defined(__x86_64__) || defined(__arm64__) */ @@ -362,14 +376,30 @@ out: task_unlock(task); } +void +task_set_platform_binary( + task_t task, + boolean_t is_platform) +{ + task_lock(task); + if (is_platform) { + task->t_flags |= TF_PLATFORM; + } else { + task->t_flags &= ~(TF_PLATFORM); + } + task_unlock(task); +} void -task_set_dyld_info(task_t task, mach_vm_address_t addr, mach_vm_size_t size) +task_set_dyld_info( + task_t task, + mach_vm_address_t addr, + mach_vm_size_t size) { task_lock(task); task->all_image_info_addr = addr; - task->all_image_info_size = size; - task_unlock(task); + task->all_image_info_size = size; + task_unlock(task); } void @@ -387,12 +417,9 @@ task_atm_reset(__unused task_t task) { void task_bank_reset(__unused task_t task) { -#if CONFIG_BANK if (task->bank_context != NULL) { bank_task_destroy(task); } -#endif - } /* @@ -403,13 +430,10 @@ task_bank_reset(__unused task_t task) { void task_bank_init(__unused task_t task) { -#if CONFIG_BANK if (task->bank_context != NULL) { panic("Task bank init called with non null bank context for task: %p and bank_context: %p", task, task->bank_context); } bank_task_initialize(task); -#endif - } void @@ -494,6 +518,12 @@ task_is_active(task_t task) return task->active; } +boolean_t +task_is_halting(task_t task) +{ + return task->halting; +} + #if TASK_REFERENCE_LEAK_DEBUG #include @@ -550,6 +580,9 @@ task_init(void) zone_change(task_zone, Z_NOENCRYPT, TRUE); +#if CONFIG_EMBEDDED + task_watch_init(); +#endif /* CONFIG_EMBEDDED */ /* * Configure per-task memory limit. @@ -691,6 +724,7 @@ task_init(void) #endif panic("task_init\n"); + vm_map_deallocate(kernel_task->map); kernel_task->map = kernel_map; lck_spin_init(&dead_task_statistics_lock, &task_lck_grp, &task_lck_attr); @@ -860,12 +894,12 @@ init_task_ledgers(void) assert(task_ledgers.sfi_wait_times[MAX_SFI_CLASS_ID -1] != -1); #endif /* CONFIG_SCHED_SFI */ -#ifdef CONFIG_BANK task_ledgers.cpu_time_billed_to_me = ledger_entry_add(t, "cpu_time_billed_to_me", "sched", "ns"); task_ledgers.cpu_time_billed_to_others = ledger_entry_add(t, "cpu_time_billed_to_others", "sched", "ns"); -#endif task_ledgers.physical_writes = ledger_entry_add(t, "physical_writes", "res", "bytes"); task_ledgers.logical_writes = ledger_entry_add(t, "logical_writes", "res", "bytes"); + task_ledgers.energy_billed_to_me = ledger_entry_add(t, "energy_billed_to_me", "power", "nj"); + task_ledgers.energy_billed_to_others = ledger_entry_add(t, "energy_billed_to_others", "power", "nj"); if ((task_ledgers.cpu_time < 0) || (task_ledgers.tkm_private < 0) || @@ -885,11 +919,11 @@ init_task_ledgers(void) (task_ledgers.purgeable_nonvolatile_compressed < 0) || (task_ledgers.platform_idle_wakeups < 0) || (task_ledgers.interrupt_wakeups < 0) || -#ifdef CONFIG_BANK (task_ledgers.cpu_time_billed_to_me < 0) || (task_ledgers.cpu_time_billed_to_others < 0) || -#endif (task_ledgers.physical_writes < 0) || - (task_ledgers.logical_writes < 0) + (task_ledgers.logical_writes < 0) || + (task_ledgers.energy_billed_to_me < 0) || + (task_ledgers.energy_billed_to_others < 0) ) { panic("couldn't create entries for task ledger template"); } @@ -930,7 +964,9 @@ init_task_ledgers(void) ledger_set_callback(t, task_ledgers.interrupt_wakeups, task_wakeups_rate_exceeded, NULL, NULL); ledger_set_callback(t, task_ledgers.physical_writes, task_io_rate_exceeded, (void *)FLAVOR_IO_PHYSICAL_WRITES, NULL); - ledger_set_callback(t, task_ledgers.logical_writes, task_io_rate_exceeded, (void *)FLAVOR_IO_LOGICAL_WRITES, NULL); + ledger_set_callback(t, task_ledgers.logical_writes, task_io_rate_exceeded, (void *)FLAVOR_IO_LOGICAL_WRITES, NULL); + + ledger_template_complete(t); task_ledger_template = t; } @@ -964,6 +1000,7 @@ task_create_internal( return(KERN_RESOURCE_SHORTAGE); } + new_task->ledger = ledger; #if defined(CONFIG_SCHED_MULTIQ) @@ -996,15 +1033,13 @@ task_create_internal( new_task->t_flags = t_flags; new_task->t_procflags = t_procflags; new_task->importance = 0; - new_task->corpse_info_kernel = NULL; + new_task->crashed_thread_id = 0; new_task->exec_token = 0; #if CONFIG_ATM new_task->atm_context = NULL; #endif -#if CONFIG_BANK new_task->bank_context = NULL; -#endif #ifdef MACH_BSD new_task->bsd_info = NULL; @@ -1032,9 +1067,7 @@ task_create_internal( task_io_monitor_ctl(new_task, &flags); #endif /* CONFIG_IO_ACCOUNTING */ -#if defined(__i386__) || defined(__x86_64__) - new_task->i386_ldt = 0; -#endif + machine_task_init(new_task, parent_task, inherit_memory); new_task->task_debug = NULL; @@ -1071,16 +1104,18 @@ task_create_internal( new_task->hv_task_target = NULL; #endif /* HYPERVISOR */ +#if CONFIG_EMBEDDED + queue_init(&new_task->task_watchers); + new_task->num_taskwatchers = 0; + new_task->watchapplying = 0; +#endif /* CONFIG_EMBEDDED */ new_task->mem_notify_reserved = 0; + new_task->memlimit_attrs_reserved = 0; #if IMPORTANCE_INHERITANCE new_task->task_imp_base = NULL; #endif /* IMPORTANCE_INHERITANCE */ -#if defined(__x86_64__) - new_task->uexc_range_start = new_task->uexc_range_size = new_task->uexc_handler = 0; -#endif - new_task->requested_policy = default_task_requested_policy; new_task->effective_policy = default_task_effective_policy; @@ -1097,10 +1132,6 @@ task_create_internal( new_task->all_image_info_addr = parent_task->all_image_info_addr; new_task->all_image_info_size = parent_task->all_image_info_size; -#if defined(__i386__) || defined(__x86_64__) - if (inherit_memory && parent_task->i386_ldt) - new_task->i386_ldt = user_ldt_copy(parent_task->i386_ldt); -#endif if (inherit_memory && parent_task->affinity_space) task_affinity_create(parent_task, new_task); @@ -1115,6 +1146,12 @@ task_create_internal( assert(IIT_NULL != new_task_imp); ipc_importance_task_mark_donor(new_task_imp, TRUE); } +#if CONFIG_EMBEDDED + /* Embedded only wants to inherit for exec copy task */ + if ((t_procflags & TPF_EXEC_COPY) == 0) { + inherit_receive = FALSE; + } +#endif /* CONFIG_EMBEDDED */ if (inherit_receive) { if (task_is_marked_importance_receiver(parent_task)) { @@ -1182,6 +1219,7 @@ task_create_internal( /* Initialize to zero for standard fork/spawn case */ new_task->total_user_time = 0; new_task->total_system_time = 0; + new_task->total_ptime = 0; new_task->faults = 0; new_task->pageins = 0; new_task->cow_faults = 0; @@ -1209,6 +1247,9 @@ task_create_internal( new_task->task_invalidated_writes = 0; new_task->task_metadata_writes = 0; new_task->task_energy = 0; +#if MONOTONIC + memset(&new_task->task_monotonic, 0, sizeof(new_task->task_monotonic)); +#endif /* MONOTONIC */ } @@ -1228,6 +1269,15 @@ task_create_internal( /* TODO: assert that new_task will be PID 1 (launchd) */ coalitions_adopt_init_task(new_task); } + /* + * on exec, we need to transfer the coalition roles from the + * parent task to the exec copy task. + */ + if (parent_task && (t_procflags & TPF_EXEC_COPY)) { + int coal_roles[COALITION_NUM_TYPES]; + task_coalition_roles(parent_task, coal_roles); + (void)coalitions_set_roles(new_task->coalition, new_task, coal_roles); + } } else { coalitions_adopt_corpse_task(new_task); } @@ -1285,6 +1335,7 @@ task_rollup_accounting_info(task_t to_task, task_t from_task) to_task->total_user_time = from_task->total_user_time; to_task->total_system_time = from_task->total_system_time; + to_task->total_ptime = from_task->total_ptime; to_task->faults = from_task->faults; to_task->pageins = from_task->pageins; to_task->cow_faults = from_task->cow_faults; @@ -1321,12 +1372,12 @@ task_rollup_accounting_info(task_t to_task, task_t from_task) ledger_rollup_entry(to_task->ledger, from_task->ledger, task_ledgers.sfi_wait_times[class_id]); } #endif -#if CONFIG_BANK ledger_rollup_entry(to_task->ledger, from_task->ledger, task_ledgers.cpu_time_billed_to_me); ledger_rollup_entry(to_task->ledger, from_task->ledger, task_ledgers.cpu_time_billed_to_others); -#endif ledger_rollup_entry(to_task->ledger, from_task->ledger, task_ledgers.physical_writes); ledger_rollup_entry(to_task->ledger, from_task->ledger, task_ledgers.logical_writes); + ledger_rollup_entry(to_task->ledger, from_task->ledger, task_ledgers.energy_billed_to_me); + ledger_rollup_entry(to_task->ledger, from_task->ledger, task_ledgers.energy_billed_to_others); } int task_dropped_imp_count = 0; @@ -1351,6 +1402,8 @@ task_deallocate( #if IMPORTANCE_INHERITANCE if (refs > 1) return; + + atomic_load_explicit(&task->ref_count, memory_order_acquire); if (refs == 1) { /* @@ -1367,6 +1420,9 @@ task_deallocate( #else if (refs > 0) return; + + atomic_load_explicit(&task->ref_count, memory_order_acquire); + #endif /* IMPORTANCE_INHERITANCE */ lck_mtx_lock(&tasks_threads_lock); @@ -1444,6 +1500,10 @@ task_deallocate( dead_task_statistics.task_timer_wakeups_bin_1 += task->task_timer_wakeups_bin_1; dead_task_statistics.task_timer_wakeups_bin_2 += task->task_timer_wakeups_bin_2; + dead_task_statistics.total_ptime += task->total_ptime; + dead_task_statistics.total_pset_switches += task->ps_switch; + dead_task_statistics.task_gpu_ns += task->task_gpu_ns; + dead_task_statistics.task_energy += task->task_energy; lck_spin_unlock(&dead_task_statistics_lock); lck_mtx_destroy(&task->lock, &task_lck_grp); @@ -1473,17 +1533,19 @@ task_deallocate( #if MACH_BSD /* clean up collected information since last reference to task is gone */ if (task->corpse_info) { - task_crashinfo_destroy(task->corpse_info, RELEASE_CORPSE_REF); + void *corpse_info_kernel = kcdata_memory_get_begin_addr(task->corpse_info); + task_crashinfo_destroy(task->corpse_info); task->corpse_info = NULL; + if (corpse_info_kernel) { + kfree(corpse_info_kernel, CORPSEINFO_ALLOCATION_SIZE); + } } #endif - if (task->corpse_info_kernel) { - kfree(task->corpse_info_kernel, CORPSEINFO_ALLOCATION_SIZE); - } #if CONFIG_MACF if (task->crash_label) { - mac_exc_action_label_task_destroy(task); + mac_exc_free_label(task->crash_label); + task->crash_label = NULL; } #endif @@ -1533,7 +1595,12 @@ task_suspension_token_deallocate( * collect crash info from bsd and mach based data */ kern_return_t -task_collect_crash_info(task_t task, struct proc *proc, int is_corpse_fork) +task_collect_crash_info( + task_t task, +#ifdef CONFIG_MACF + struct label *crash_label, +#endif + int is_corpse_fork) { kern_return_t kr = KERN_SUCCESS; @@ -1543,60 +1610,57 @@ task_collect_crash_info(task_t task, struct proc *proc, int is_corpse_fork) mach_vm_offset_t crash_data_ptr = 0; void *crash_data_kernel = NULL; void *crash_data_kernel_release = NULL; - int corpse_blob_kernel_alloc = (is_corpse_fork || unify_corpse_blob_alloc); +#if CONFIG_MACF + struct label *label, *free_label; +#endif if (!corpses_enabled()) { return KERN_NOT_SUPPORTED; } +#if CONFIG_MACF + free_label = label = mac_exc_create_label(); +#endif + task_lock(task); assert(is_corpse_fork || task->bsd_info != NULL); if (task->corpse_info == NULL && (is_corpse_fork || task->bsd_info != NULL)) { #if CONFIG_MACF - /* Update the corpse label, used by the exception delivery mac hook */ - mac_exc_action_label_task_update(task, proc); + /* Set the crash label, used by the exception delivery mac hook */ + free_label = task->crash_label; // Most likely NULL. + task->crash_label = label; + mac_exc_update_task_crash_label(task, crash_label); #endif task_unlock(task); - if (!corpse_blob_kernel_alloc) { - /* map crash data memory in task's vm map */ - kr = mach_vm_allocate(task->map, &crash_data_ptr, size, (VM_MAKE_TAG(VM_MEMORY_CORPSEINFO) | VM_FLAGS_ANYWHERE)); - } else { - crash_data_kernel = (void *) kalloc(CORPSEINFO_ALLOCATION_SIZE); - if (crash_data_kernel == 0) - kr = KERN_RESOURCE_SHORTAGE; - bzero(crash_data_kernel, CORPSEINFO_ALLOCATION_SIZE); - crash_data_ptr = (mach_vm_offset_t) crash_data_kernel; - } - if (kr != KERN_SUCCESS) + crash_data_kernel = (void *) kalloc(CORPSEINFO_ALLOCATION_SIZE); + if (crash_data_kernel == NULL) { + kr = KERN_RESOURCE_SHORTAGE; goto out_no_lock; + } + bzero(crash_data_kernel, CORPSEINFO_ALLOCATION_SIZE); + crash_data_ptr = (mach_vm_offset_t) crash_data_kernel; /* Do not get a corpse ref for corpse fork */ - crash_data = task_crashinfo_alloc_init((mach_vm_address_t)crash_data_ptr, size, is_corpse_fork ? !GET_CORPSE_REF : GET_CORPSE_REF, corpse_blob_kernel_alloc ? KCFLAG_USE_MEMCOPY: KCFLAG_USE_COPYOUT); + crash_data = task_crashinfo_alloc_init((mach_vm_address_t)crash_data_ptr, size, + is_corpse_fork ? 0 : CORPSE_CRASHINFO_HAS_REF, + KCFLAG_USE_MEMCOPY); if (crash_data) { task_lock(task); crash_data_release = task->corpse_info; - crash_data_kernel_release = task->corpse_info_kernel; + crash_data_kernel_release = kcdata_memory_get_begin_addr(crash_data_release); task->corpse_info = crash_data; - task->corpse_info_kernel = crash_data_kernel; task_unlock(task); kr = KERN_SUCCESS; } else { - /* if failed to create corpse info, free the mapping */ - if (!corpse_blob_kernel_alloc) { - if (KERN_SUCCESS != mach_vm_deallocate(task->map, crash_data_ptr, size)) { - printf("mach_vm_deallocate failed to clear corpse_data for pid %d.\n", task_pid(task)); - } - } else { - kfree(crash_data_kernel, CORPSEINFO_ALLOCATION_SIZE); - } + kfree(crash_data_kernel, CORPSEINFO_ALLOCATION_SIZE); kr = KERN_FAILURE; } if (crash_data_release != NULL) { - task_crashinfo_destroy(crash_data_release, is_corpse_fork ? !RELEASE_CORPSE_REF : RELEASE_CORPSE_REF); + task_crashinfo_destroy(crash_data_release); } if (crash_data_kernel_release != NULL) { kfree(crash_data_kernel_release, CORPSEINFO_ALLOCATION_SIZE); @@ -1606,6 +1670,11 @@ task_collect_crash_info(task_t task, struct proc *proc, int is_corpse_fork) } out_no_lock: +#if CONFIG_MACF + if (free_label != NULL) { + mac_exc_free_label(free_label); + } +#endif return kr; } @@ -1615,7 +1684,11 @@ out_no_lock: * Makes outcall to registered host port for a corpse. */ kern_return_t -task_deliver_crash_notification(task_t task, thread_t thread, mach_exception_data_type_t subcode) +task_deliver_crash_notification( + task_t task, + thread_t thread, + exception_type_t etype, + mach_exception_subcode_t subcode) { kcdata_descriptor_t crash_info = task->corpse_info; thread_t th_iter = NULL; @@ -1629,10 +1702,10 @@ task_deliver_crash_notification(task_t task, thread_t thread, mach_exception_dat task_lock(task); if (task_is_a_corpse_fork(task)) { - /* Populate code with EXC_RESOURCE for corpse fork */ - code[0] = EXC_RESOURCE; + /* Populate code with EXC_{RESOURCE,GUARD} for corpse fork */ + code[0] = etype; code[1] = subcode; - } else if (unify_corpse_blob_alloc) { + } else { /* Populate code with EXC_CRASH for corpses */ code[0] = EXC_CRASH; code[1] = 0; @@ -1640,11 +1713,8 @@ task_deliver_crash_notification(task_t task, thread_t thread, mach_exception_dat if (corpse_for_fatal_memkill) { code[1] = subcode; } - } else { - /* Populate code with address and length for EXC_CRASH */ - code[0] = crash_info->kcd_addr_begin; - code[1] = crash_info->kcd_length; } + queue_iterate(&task->threads, th_iter, thread_t, task_threads) { if (th_iter->corpse_dup == FALSE) { @@ -1729,14 +1799,25 @@ task_mark_corpse(task_t task) thread_t self_thread; (void) self_thread; wait_interrupt_t wsave; +#if CONFIG_MACF + struct label *crash_label = NULL; +#endif assert(task != kernel_task); assert(task == current_task()); assert(!task_is_a_corpse(task)); - kr = task_collect_crash_info(task, (struct proc*)task->bsd_info, FALSE); +#if CONFIG_MACF + crash_label = mac_exc_create_label_for_proc((struct proc*)task->bsd_info); +#endif + + kr = task_collect_crash_info(task, +#if CONFIG_MACF + crash_label, +#endif + FALSE); if (kr != KERN_SUCCESS) { - return kr; + goto out; } self_thread = current_thread(); @@ -1746,6 +1827,7 @@ task_mark_corpse(task_t task) task_set_corpse_pending_report(task); task_set_corpse(task); + task->crashed_thread_id = thread_tid(self_thread); kr = task_start_halt_locked(task, TRUE); assert(kr == KERN_SUCCESS); @@ -1767,6 +1849,11 @@ task_mark_corpse(task_t task) (void) thread_interrupt_level(wsave); assert(task->halting == TRUE); + +out: +#if CONFIG_MACF + mac_exc_free_label(crash_label); +#endif return kr; } @@ -1895,22 +1982,26 @@ task_duplicate_map_and_threads( return KERN_FAILURE; } - /* Setup new task's vmmap, switch from parent task's map to it COW map */ - oldmap = new_task->map; - new_task->map = vm_map_fork(new_task->ledger, - task->map, - (VM_MAP_FORK_SHARE_IF_INHERIT_NONE | - VM_MAP_FORK_PRESERVE_PURGEABLE)); - vm_map_deallocate(oldmap); - - /* Get all the udata pointers from kqueue */ - est_knotes = proc_list_uptrs(p, NULL, 0); - if (est_knotes > 0) { - buf_size = (est_knotes + 32) * sizeof(uint64_t); - buffer = (uint64_t *) kalloc(buf_size); - num_knotes = proc_list_uptrs(p, buffer, buf_size); - if (num_knotes > est_knotes + 32) { - num_knotes = est_knotes + 32; + /* Check with VM if vm_map_fork is allowed for this task */ + if (task_allowed_vm_map_fork(task)) { + + /* Setup new task's vmmap, switch from parent task's map to it COW map */ + oldmap = new_task->map; + new_task->map = vm_map_fork(new_task->ledger, + task->map, + (VM_MAP_FORK_SHARE_IF_INHERIT_NONE | + VM_MAP_FORK_PRESERVE_PURGEABLE)); + vm_map_deallocate(oldmap); + + /* Get all the udata pointers from kqueue */ + est_knotes = kevent_proc_copy_uptrs(p, NULL, 0); + if (est_knotes > 0) { + buf_size = (est_knotes + 32) * sizeof(uint64_t); + buffer = (uint64_t *) kalloc(buf_size); + num_knotes = kevent_proc_copy_uptrs(p, buffer, buf_size); + if (num_knotes > est_knotes + 32) { + num_knotes = est_knotes + 32; + } } } @@ -1953,6 +2044,7 @@ task_duplicate_map_and_threads( /* Equivalent of current thread in corpse */ if (thread_array[i] == self) { thread_return = new_thread; + new_task->crashed_thread_id = thread_tid(new_thread); } else { /* drop the extra ref returned by thread_create_with_continuation */ thread_deallocate(new_thread); @@ -1995,6 +2087,16 @@ task_duplicate_map_and_threads( return kr; } +/* + * Place holder function to be filled by VM to return + * TRUE if vm_map_fork is allowed on the given task. + */ +boolean_t +task_allowed_vm_map_fork(task_t task __unused) +{ + return memorystatus_allowed_vm_map_fork(task); +} + #if CONFIG_SECLUDED_MEMORY extern void task_set_can_use_secluded_mem_locked( task_t task, @@ -2118,6 +2220,13 @@ task_terminate_internal( // PR-17045188: Revisit implementation // task_partial_reap(task, pid); +#if CONFIG_EMBEDDED + /* + * remove all task watchers + */ + task_removewatchers(task); + +#endif /* CONFIG_EMBEDDED */ /* * Destroy all synchronizers owned by the task. @@ -2161,8 +2270,13 @@ task_terminate_internal( vm_map_remove(task->map, task->map->min_offset, task->map->max_offset, - /* no unnesting on final cleanup: */ - VM_MAP_REMOVE_NO_UNNESTING); + /* + * Final cleanup: + * + no unnesting + * + remove immutable mappings + */ + (VM_MAP_REMOVE_NO_UNNESTING | + VM_MAP_REMOVE_IMMUTABLE)); /* release our shared region */ vm_shared_region_set(task, NULL); @@ -2363,8 +2477,13 @@ task_complete_halt(task_t task) */ vm_map_remove(task->map, task->map->min_offset, task->map->max_offset, - /* no unnesting on final cleanup: */ - VM_MAP_REMOVE_NO_UNNESTING); + /* + * Final cleanup: + * + no unnesting + * + remove immutable mappings + */ + (VM_MAP_REMOVE_NO_UNNESTING | + VM_MAP_REMOVE_IMMUTABLE)); /* * Kick out any IOKitUser handles to the task. At best they're stale, @@ -3221,7 +3340,7 @@ task_disconnect_page_mappings(task_t task) * Conditions: * The caller holds a reference to the task */ -extern void vm_wake_compactor_swapper(); +extern void vm_wake_compactor_swapper(void); extern queue_head_t c_swapout_list_head; kern_return_t @@ -3445,6 +3564,9 @@ task_info( case TASK_BASIC_INFO_32: case TASK_BASIC2_INFO_32: +#if defined(__arm__) || defined(__arm64__) + case TASK_BASIC_INFO_64: +#endif { task_basic_info_32_t basic_info; vm_map_t map; @@ -3489,6 +3611,46 @@ task_info( break; } +#if defined(__arm__) || defined(__arm64__) + case TASK_BASIC_INFO_64_2: + { + task_basic_info_64_2_t basic_info; + vm_map_t map; + clock_sec_t secs; + clock_usec_t usecs; + + if (*task_info_count < TASK_BASIC_INFO_64_2_COUNT) { + error = KERN_INVALID_ARGUMENT; + break; + } + + basic_info = (task_basic_info_64_2_t)task_info_out; + + map = (task == kernel_task)? kernel_map: task->map; + basic_info->virtual_size = map->size; + basic_info->resident_size = + (mach_vm_size_t)(pmap_resident_count(map->pmap)) + * PAGE_SIZE_64; + + basic_info->policy = ((task != kernel_task)? + POLICY_TIMESHARE: POLICY_RR); + basic_info->suspend_count = task->user_stop_count; + + absolutetime_to_microtime(task->total_user_time, &secs, &usecs); + basic_info->user_time.seconds = + (typeof(basic_info->user_time.seconds))secs; + basic_info->user_time.microseconds = usecs; + + absolutetime_to_microtime(task->total_system_time, &secs, &usecs); + basic_info->system_time.seconds = + (typeof(basic_info->system_time.seconds))secs; + basic_info->system_time.microseconds = usecs; + + *task_info_count = TASK_BASIC_INFO_64_2_COUNT; + break; + } + +#else /* defined(__arm__) || defined(__arm64__) */ case TASK_BASIC_INFO_64: { task_basic_info_64_t basic_info; @@ -3526,6 +3688,7 @@ task_info( *task_info_count = TASK_BASIC_INFO_64_COUNT; break; } +#endif /* defined(__arm__) || defined(__arm64__) */ case MACH_TASK_BASIC_INFO: { @@ -3939,14 +4102,12 @@ task_info( case TASK_POWER_INFO_V2: { - if (*task_info_count < TASK_POWER_INFO_V2_COUNT) { + if (*task_info_count < TASK_POWER_INFO_V2_COUNT_OLD) { error = KERN_INVALID_ARGUMENT; break; } task_power_info_v2_t tpiv2 = (task_power_info_v2_t) task_info_out; - - uint64_t *task_energy = NULL; - task_power_info_locked(task, &tpiv2->cpu_energy, &tpiv2->gpu_energy, task_energy); + task_power_info_locked(task, &tpiv2->cpu_energy, &tpiv2->gpu_energy, tpiv2); break; } @@ -4190,6 +4351,41 @@ task_info( return (error); } +/* + * task_info_from_user + * + * When calling task_info from user space, + * this function will be executed as mig server side + * instead of calling directly into task_info. + * This gives the possibility to perform more security + * checks on task_port. + * + * In the case of TASK_DYLD_INFO, we require the more + * privileged task_port not the less-privileged task_name_port. + * + */ +kern_return_t +task_info_from_user( + mach_port_t task_port, + task_flavor_t flavor, + task_info_t task_info_out, + mach_msg_type_number_t *task_info_count) +{ + task_t task; + kern_return_t ret; + + if (flavor == TASK_DYLD_INFO) + task = convert_port_to_task(task_port); + else + task = convert_port_to_task_name(task_port); + + ret = task_info(task, flavor, task_info_out, task_info_count); + + task_deallocate(task); + + return ret; +} + /* * task_power_info * @@ -4201,7 +4397,7 @@ task_power_info_locked( task_t task, task_power_info_t info, gpu_energy_data_t ginfo, - uint64_t *task_energy) + task_power_info_v2_t infov2) { thread_t thread; ledger_amount_t tmp; @@ -4219,14 +4415,21 @@ task_power_info_locked( info->total_user = task->total_user_time; info->total_system = task->total_system_time; - if (task_energy) { - *task_energy = task->task_energy; +#if CONFIG_EMBEDDED + if (infov2) { + infov2->task_energy = task->task_energy; } +#endif if (ginfo) { ginfo->task_gpu_utilisation = task->task_gpu_ns; } + if (infov2) { + infov2->task_ptime = task->total_ptime; + infov2->task_pset_switches = task->ps_switch; + } + queue_iterate(&task->threads, thread, thread_t, task_threads) { uint64_t tval; spl_t x; @@ -4240,13 +4443,21 @@ task_power_info_locked( info->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1; info->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2; - if (task_energy) { - *task_energy += ml_energy_stat(thread); +#if CONFIG_EMBEDDED + if (infov2) { + infov2->task_energy += ml_energy_stat(thread); } +#endif tval = timer_grab(&thread->user_timer); info->total_user += tval; + if (infov2) { + tval = timer_grab(&thread->ptime); + infov2->task_ptime += tval; + infov2->task_pset_switches += thread->ps_switch; + } + tval = timer_grab(&thread->system_timer); if (thread->precise_user_kernel_time) { info->total_system += tval; @@ -4274,6 +4485,7 @@ task_gpu_utilisation( task_t task) { uint64_t gpu_time = 0; +#if !CONFIG_EMBEDDED thread_t thread; task_lock(task); @@ -4289,6 +4501,10 @@ task_gpu_utilisation( } task_unlock(task); +#else /* CONFIG_EMBEDDED */ + /* silence compiler warning */ + (void)task; +#endif /* !CONFIG_EMBEDDED */ return gpu_time; } @@ -4321,6 +4537,15 @@ task_energy( return energy; } + +uint64_t +task_cpu_ptime( + __unused task_t task) +{ + return 0; +} + + kern_return_t task_purgable_info( task_t task, @@ -4636,6 +4861,45 @@ task_get_state( return ret; } + +static kern_return_t __attribute__((noinline,not_tail_called)) +PROC_VIOLATED_GUARD__SEND_EXC_GUARD_AND_SUSPEND( + mach_exception_code_t code, + mach_exception_subcode_t subcode, + void *reason) +{ +#ifdef MACH_BSD + if (1 == proc_selfpid()) + return KERN_NOT_SUPPORTED; // initproc is immune +#endif + mach_exception_data_type_t codes[EXCEPTION_CODE_MAX] = { + [0] = code, + [1] = subcode, + }; + task_t task = current_task(); + kern_return_t kr; + + /* (See jetsam-related comments below) */ + + proc_memstat_terminated(task->bsd_info, TRUE); + kr = task_enqueue_exception_with_corpse(task, EXC_GUARD, codes, 2, reason); + proc_memstat_terminated(task->bsd_info, FALSE); + return kr; +} + +extern kern_return_t +task_violated_guard(mach_exception_code_t, mach_exception_subcode_t, void *); + +kern_return_t +task_violated_guard( + mach_exception_code_t code, + mach_exception_subcode_t subcode, + void *reason) +{ + return PROC_VIOLATED_GUARD__SEND_EXC_GUARD_AND_SUSPEND(code, subcode, reason); +} + + #if CONFIG_MEMORYSTATUS boolean_t @@ -4643,7 +4907,11 @@ task_get_memlimit_is_active(task_t task) { assert (task != NULL); - return (task->memlimit_is_active ? TRUE : FALSE); + if (task->memlimit_is_active == 1) { + return(TRUE); + } else { + return (FALSE); + } } void @@ -4651,7 +4919,11 @@ task_set_memlimit_is_active(task_t task, boolean_t memlimit_is_active) { assert (task != NULL); - memlimit_is_active ? (task->memlimit_is_active = 1) : (task->memlimit_is_active = 0); + if (memlimit_is_active) { + task->memlimit_is_active = 1; + } else { + task->memlimit_is_active = 0; + } } boolean_t @@ -4659,7 +4931,11 @@ task_get_memlimit_is_fatal(task_t task) { assert(task != NULL); - return (task->memlimit_is_fatal ? TRUE : FALSE); + if (task->memlimit_is_fatal == 1) { + return(TRUE); + } else { + return(FALSE); + } } void @@ -4667,7 +4943,11 @@ task_set_memlimit_is_fatal(task_t task, boolean_t memlimit_is_fatal) { assert (task != NULL); - memlimit_is_fatal ? (task->memlimit_is_fatal = 1) : (task->memlimit_is_fatal = 0); + if (memlimit_is_fatal) { + task->memlimit_is_fatal = 1; + } else { + task->memlimit_is_fatal = 0; + } } boolean_t @@ -4773,9 +5053,6 @@ PROC_CROSSED_HIGH_WATERMARK__SEND_EXC_RESOURCE_AND_SUSPEND(int max_footprint_mb, */ proc_memstat_terminated(current_task()->bsd_info, TRUE); - printf("process %s[%d] crossed memory high watermark (%d MB); sending " - "EXC_RESOURCE.\n", procname, pid, max_footprint_mb); - code[0] = code[1] = 0; EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_MEMORY); EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_HIGH_WATERMARK); @@ -4794,7 +5071,8 @@ PROC_CROSSED_HIGH_WATERMARK__SEND_EXC_RESOURCE_AND_SUSPEND(int max_footprint_mb, task_resume_internal(task); } } else { - task_enqueue_exception_with_corpse(task, code, EXCEPTION_CODE_MAX); + task_enqueue_exception_with_corpse(task, EXC_RESOURCE, + code, EXCEPTION_CODE_MAX, NULL); } /* @@ -4919,14 +5197,15 @@ task_set_phys_footprint_limit_internal( ledger_amount_t old; ledger_get_limit(task->ledger, task_ledgers.phys_footprint, &old); + + /* + * Check that limit >> 20 will not give an "unexpected" 32-bit + * result. There are, however, implicit assumptions that -1 mb limit + * equates to LEDGER_LIMIT_INFINITY. + */ + assert(((old & 0xFFF0000000000000LL) == 0) || (old == LEDGER_LIMIT_INFINITY)); if (old_limit_mb) { - /* - * Check that limit >> 20 will not give an "unexpected" 32-bit - * result. There are, however, implicit assumptions that -1 mb limit - * equates to LEDGER_LIMIT_INFINITY. - */ - assert(((old & 0xFFF0000000000000LL) == 0) || (old == LEDGER_LIMIT_INFINITY)); *old_limit_mb = (int)(old >> 20); } @@ -4938,8 +5217,10 @@ task_set_phys_footprint_limit_internal( max_task_footprint ? max_task_footprint : LEDGER_LIMIT_INFINITY, max_task_footprint ? max_task_footprint_warning_level : 0); + task_lock(task); task_set_memlimit_is_active(task, memlimit_is_active); task_set_memlimit_is_fatal(task, memlimit_is_fatal); + task_unlock(task); return (KERN_SUCCESS); } @@ -4950,6 +5231,16 @@ task_set_phys_footprint_limit_internal( task_lock(task); + if ((memlimit_is_active == task_get_memlimit_is_active(task)) && + (memlimit_is_fatal == task_get_memlimit_is_fatal(task)) && + (((ledger_amount_t)new_limit_mb << 20) == old)) { + /* + * memlimit state is not changing + */ + task_unlock(task); + return(KERN_SUCCESS); + } + task_set_memlimit_is_active(task, memlimit_is_active); task_set_memlimit_is_fatal(task, memlimit_is_fatal); @@ -5253,7 +5544,7 @@ SENDING_NOTIFICATION__THIS_PROCESS_IS_CAUSING_TOO_MANY_WAKEUPS(void) fatal = task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_FATAL_WAKEUPSMON; trace_resource_violation(RMON_CPUWAKES_VIOLATED, &lei); - printf("process %s[%d] caught waking the CPU %llu times " + os_log(OS_LOG_DEFAULT, "process %s[%d] caught waking the CPU %llu times " "over ~%llu seconds, averaging %llu wakes / second and " "violating a %slimit of %llu wakes over %llu seconds.\n", procname, pid, @@ -5276,12 +5567,12 @@ SENDING_NOTIFICATION__THIS_PROCESS_IS_CAUSING_TOO_MANY_WAKEUPS(void) return; } if (audio_active) { - printf("process %s[%d] caught causing excessive wakeups. EXC_RESOURCE " + os_log(OS_LOG_DEFAULT, "process %s[%d] caught causing excessive wakeups. EXC_RESOURCE " "supressed due to audio playback\n", procname, pid); return; } if (lei.lei_last_refill == 0) { - printf("process %s[%d] caught causing excessive wakeups. EXC_RESOURCE " + os_log(OS_LOG_DEFAULT, "process %s[%d] caught causing excessive wakeups. EXC_RESOURCE " "supressed due to lei.lei_last_refill = 0 \n", procname, pid); } @@ -5439,7 +5730,7 @@ void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_PROCESS_IS_CAUSING_TOO if (flavor == FLAVOR_IO_LOGICAL_WRITES) { trace_resource_violation(RMON_LOGWRITES_VIOLATED, &lei); } - printf("process [%d] caught causing excessive I/O (flavor: %d). Task I/O: %lld MB. [Limit : %lld MB per %lld secs]\n", + os_log(OS_LOG_DEFAULT, "process [%d] caught causing excessive I/O (flavor: %d). Task I/O: %lld MB. [Limit : %lld MB per %lld secs]\n", pid, flavor, (lei.lei_balance / (1024 * 1024)), (lei.lei_limit / (1024 * 1024)), (lei.lei_refill_period / NSEC_PER_SEC)); kr = send_resource_violation(send_disk_writes_violation, task, &lei, kRNFlagsNone); @@ -5567,6 +5858,10 @@ kdebug_trace_dyld(task_t task, uint32_t base_code, vm_map_offset_t map_data; vm_offset_t data; + if (!infos_copy) { + return KERN_INVALID_ADDRESS; + } + if (!kdebug_enable || !kdebug_debugid_enabled(KDBG_EVENTID(DBG_DYLD, DBG_DYLD_UUID, 0))) { @@ -5574,8 +5869,6 @@ kdebug_trace_dyld(task_t task, uint32_t base_code, return KERN_SUCCESS; } - assert(infos_copy != NULL); - if (task == NULL || task != current_task()) { return KERN_INVALID_TASK; } @@ -5650,6 +5943,57 @@ task_register_dyld_get_process_state(__unused task_t task, return KERN_NOT_SUPPORTED; } +kern_return_t +task_inspect(task_inspect_t task_insp, task_inspect_flavor_t flavor, + task_inspect_info_t info_out, mach_msg_type_number_t *size_in_out) +{ +#if MONOTONIC + task_t task = (task_t)task_insp; + kern_return_t kr = KERN_SUCCESS; + mach_msg_type_number_t size; + + if (task == TASK_NULL) { + return KERN_INVALID_ARGUMENT; + } + + size = *size_in_out; + + switch (flavor) { + case TASK_INSPECT_BASIC_COUNTS: { + struct task_inspect_basic_counts *bc; + uint64_t task_counts[MT_CORE_NFIXED]; + + if (size < TASK_INSPECT_BASIC_COUNTS_COUNT) { + kr = KERN_INVALID_ARGUMENT; + break; + } + + mt_fixed_task_counts(task, task_counts); + bc = (struct task_inspect_basic_counts *)info_out; +#ifdef MT_CORE_INSTRS + bc->instructions = task_counts[MT_CORE_INSTRS]; +#else /* defined(MT_CORE_INSTRS) */ + bc->instructions = 0; +#endif /* !defined(MT_CORE_INSTRS) */ + bc->cycles = task_counts[MT_CORE_CYCLES]; + size = TASK_INSPECT_BASIC_COUNTS_COUNT; + break; + } + default: + kr = KERN_INVALID_ARGUMENT; + break; + } + + if (kr == KERN_SUCCESS) { + *size_in_out = size; + } + return kr; +#else /* MONOTONIC */ +#pragma unused(task_insp, flavor, info_out, size_in_out) + return KERN_NOT_SUPPORTED; +#endif /* !MONOTONIC */ +} + #if CONFIG_SECLUDED_MEMORY int num_tasks_can_use_secluded_mem = 0; @@ -5734,3 +6078,9 @@ task_io_user_clients(task_t task) { return (&task->io_user_clients); } + +void +task_copy_fields_for_exec(task_t dst_task, task_t src_task) +{ + dst_task->vtimers = src_task->vtimers; +} diff --git a/osfmk/kern/task.h b/osfmk/kern/task.h index f1f821261..847120f95 100644 --- a/osfmk/kern/task.h +++ b/osfmk/kern/task.h @@ -113,6 +113,10 @@ #include #include +#if MONOTONIC +#include +#endif /* MONOTONIC */ + #include #include #include @@ -122,6 +126,7 @@ #include #include +#include #ifdef CONFIG_ATM #include @@ -137,14 +142,12 @@ struct _cpu_time_qos_stats { uint64_t cpu_time_qos_user_interactive; }; -#ifdef CONFIG_BANK #include -#endif struct task { /* Synchronization/destruction information */ decl_lck_mtx_data(,lock) /* Task's lock */ - uint32_t ref_count; /* Number of references to me */ + _Atomic uint32_t ref_count; /* Number of references to me */ boolean_t active; /* Task has not been terminated */ boolean_t halting; /* Task is being halted */ @@ -183,6 +186,7 @@ struct task { /* Statistics */ uint64_t total_user_time; /* terminated threads only */ uint64_t total_system_time; + uint64_t total_ptime; /* Virtual timers */ uint32_t vtimers; @@ -232,7 +236,7 @@ struct task { void *bsd_info; #endif kcdata_descriptor_t corpse_info; - void * corpse_info_kernel; + uint64_t crashed_thread_id; queue_chain_t corpse_tasks; #ifdef CONFIG_MACF struct label * crash_label; @@ -251,7 +255,7 @@ struct task { #define TF_CORPSE_FORK 0x00000080 /* task is a forked corpse */ #define TF_LRETURNWAIT 0x00000100 /* task is waiting for fork/posix_spawn/exec to complete */ #define TF_LRETURNWAITER 0x00000200 /* task is waiting for TF_LRETURNWAIT to get cleared */ - +#define TF_PLATFORM 0x00000400 /* task is a platform binary */ #define task_has_64BitAddr(task) \ (((task)->t_flags & TF_64B_ADDR) != 0) @@ -315,13 +319,16 @@ struct task { uint64_t rusage_cpu_perthr_interval; /* Per-thread CPU limit interval */ uint64_t rusage_cpu_deadline; thread_call_t rusage_cpu_callt; +#if CONFIG_EMBEDDED + queue_head_t task_watchers; /* app state watcher threads */ + int num_taskwatchers; + int watchapplying; +#endif /* CONFIG_EMBEDDED */ #if CONFIG_ATM struct atm_task_descriptor *atm_context; /* pointer to per task atm descriptor */ #endif -#if CONFIG_BANK struct bank_task *bank_context; /* pointer to per task bank structure */ -#endif #if IMPORTANCE_INHERITANCE struct ipc_importance_task *task_imp_base; /* Base of IPC importance chain */ @@ -369,6 +376,11 @@ struct task { uint64_t task_gpu_ns; uint64_t task_energy; +#if MONOTONIC + /* Read and written under task_lock */ + struct mt_task task_monotonic; +#endif /* MONOTONIC */ + /* # of purgeable volatile VM objects owned by this task: */ int task_volatile_objects; /* # of purgeable but not volatile VM objects owned by this task: */ @@ -407,7 +419,7 @@ struct task { }; #define task_lock(task) lck_mtx_lock(&(task)->lock) -#define task_lock_assert_owned(task) lck_mtx_assert(&(task)->lock, LCK_MTX_ASSERT_OWNED) +#define task_lock_assert_owned(task) LCK_MTX_ASSERT(&(task)->lock, LCK_MTX_ASSERT_OWNED) #define task_lock_try(task) lck_mtx_try_lock(&(task)->lock) #define task_unlock(task) lck_mtx_unlock(&(task)->lock) @@ -423,10 +435,10 @@ extern void task_reference_internal(task_t task); extern uint32_t task_deallocate_internal(task_t task); #else #define task_reference_internal(task) \ - (void)hw_atomic_add(&(task)->ref_count, 1) + (void)atomic_fetch_add_explicit(&(task)->ref_count, 1, memory_order_relaxed) #define task_deallocate_internal(task) \ - hw_atomic_sub(&(task)->ref_count, 1) + (atomic_fetch_sub_explicit(&task->ref_count, 1, memory_order_release) - 1) #endif #define task_reference(task) \ @@ -557,12 +569,17 @@ extern kern_return_t task_create_internal( uint32_t procflags, task_t *child_task); /* OUT */ +extern kern_return_t task_info( + task_t task, + task_flavor_t flavor, + task_info_t task_info_out, + mach_msg_type_number_t *task_info_count); extern void task_power_info_locked( task_t task, task_power_info_t info, - gpu_energy_data_t gpu_energy, - uint64_t *task_power); + gpu_energy_data_t gpu_energy, + task_power_info_v2_t infov2); extern uint64_t task_gpu_utilisation( task_t task); @@ -570,6 +587,9 @@ extern uint64_t task_gpu_utilisation( extern uint64_t task_energy( task_t task); +extern uint64_t task_cpu_ptime( + task_t task); + extern void task_vtimer_set( task_t task, integer_t which); @@ -591,6 +611,10 @@ extern void task_set_64bit( task_t task, boolean_t is64bit); +extern void task_set_platform_binary( + task_t task, + boolean_t is_platform); + extern void task_backing_store_privileged( task_t task); @@ -604,7 +628,14 @@ extern int get_task_numacts( task_t task); extern int get_task_numactivethreads(task_t task); -extern kern_return_t task_collect_crash_info(task_t task, struct proc *p, int is_corpse_fork); + +struct label; +extern kern_return_t task_collect_crash_info( + task_t task, +#if CONFIG_MACF + struct label *crash_label, +#endif + int is_corpse_fork); void task_port_notify(mach_msg_header_t *msg); void task_wait_till_threads_terminate_locked(task_t task); @@ -617,13 +648,15 @@ extern uint64_t get_task_resident_size(task_t); extern uint64_t get_task_compressed(task_t); extern uint64_t get_task_resident_max(task_t); extern uint64_t get_task_phys_footprint(task_t); -extern uint64_t get_task_phys_footprint_max(task_t); +extern uint64_t get_task_phys_footprint_recent_max(task_t); +extern uint64_t get_task_phys_footprint_lifetime_max(task_t); extern uint64_t get_task_phys_footprint_limit(task_t); extern uint64_t get_task_purgeable_size(task_t); extern uint64_t get_task_cpu_time(task_t); extern uint64_t get_task_dispatchqueue_offset(task_t); extern uint64_t get_task_dispatchqueue_serialno_offset(task_t); -extern uint64_t get_task_uniqueid(task_t); +extern uint64_t get_task_uniqueid(task_t task); +extern int get_task_version(task_t task); extern uint64_t get_task_internal(task_t); extern uint64_t get_task_internal_compressed(task_t); @@ -688,12 +721,12 @@ struct _task_ledger_indices { #if CONFIG_SCHED_SFI int sfi_wait_times[MAX_SFI_CLASS_ID]; #endif /* CONFIG_SCHED_SFI */ -#ifdef CONFIG_BANK int cpu_time_billed_to_me; int cpu_time_billed_to_others; -#endif int physical_writes; int logical_writes; + int energy_billed_to_me; + int energy_billed_to_others; }; extern struct _task_ledger_indices task_ledgers; @@ -710,6 +743,7 @@ extern void task_clear_exec_copy_flag(task_t task); extern boolean_t task_is_exec_copy(task_t); extern boolean_t task_did_exec(task_t task); extern boolean_t task_is_active(task_t task); +extern boolean_t task_is_halting(task_t task); extern void task_clear_return_wait(task_t task); extern void task_wait_to_return(void); extern event_t task_get_return_wait_event(task_t task); @@ -727,6 +761,8 @@ extern boolean_t task_is_gpu_denied(task_t task); extern queue_head_t * task_io_user_clients(task_t task); +extern void task_copy_fields_for_exec(task_t dst_task, task_t src_task); + #endif /* XNU_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE @@ -773,7 +809,6 @@ extern boolean_t task_could_use_secluded_mem(task_t task); #if CONFIG_MACF extern struct label *get_task_crash_label(task_t task); -extern void set_task_crash_label(task_t task, struct label *label); #endif /* CONFIG_MACF */ #endif /* KERNEL_PRIVATE */ diff --git a/osfmk/kern/task_policy.c b/osfmk/kern/task_policy.c index 2ac70855d..b13186523 100644 --- a/osfmk/kern/task_policy.c +++ b/osfmk/kern/task_policy.c @@ -48,12 +48,16 @@ #if CONFIG_TELEMETRY #include #endif +#if CONFIG_EMBEDDED +#include +#include +#endif /* CONFIG_EMBEDDED */ #if IMPORTANCE_INHERITANCE #include -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE #include -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ #endif /* IMPORTANCE_INHERITACE */ #include @@ -141,9 +145,7 @@ static void task_policy_update_internal_locked(task_t task, boolean_t in_create, static void proc_set_task_policy2(task_t task, int category, int flavor, int value1, int value2); static void proc_get_task_policy2(task_t task, int category, int flavor, int *value1, int *value2); -#if CONFIG_SCHED_SFI -static boolean_t task_policy_update_coalition_focal_tasks(task_t task, int prev_role, int next_role); -#endif +static boolean_t task_policy_update_coalition_focal_tasks(task_t task, int prev_role, int next_role, task_pend_token_t pend_token); static uint64_t task_requested_bitfield(task_t task); static uint64_t task_effective_bitfield(task_t task); @@ -176,6 +178,32 @@ extern int proc_pidpathinfo_internal(proc_t p, uint64_t arg, #endif /* MACH_BSD */ +#if CONFIG_EMBEDDED +/* TODO: make CONFIG_TASKWATCH */ +/* Taskwatch related helper functions */ +static void set_thread_appbg(thread_t thread, int setbg,int importance); +static void add_taskwatch_locked(task_t task, task_watch_t * twp); +static void remove_taskwatch_locked(task_t task, task_watch_t * twp); +static void task_watch_lock(void); +static void task_watch_unlock(void); +static void apply_appstate_watchers(task_t task); + +typedef struct task_watcher { + queue_chain_t tw_links; /* queueing of threads */ + task_t tw_task; /* task that is being watched */ + thread_t tw_thread; /* thread that is watching the watch_task */ + int tw_state; /* the current app state of the thread */ + int tw_importance; /* importance prior to backgrounding */ +} task_watch_t; + +typedef struct thread_watchlist { + thread_t thread; /* thread being worked on for taskwatch action */ + int importance; /* importance to be restored if thread is being made active */ +} thread_watchlist_t; + +#endif /* CONFIG_EMBEDDED */ + +extern int memorystatus_update_priority_for_appnap(proc_t p, boolean_t is_appnap); /* Importance Inheritance related helper functions */ @@ -198,11 +226,11 @@ static void task_set_boost_locked(task_t task, boolean_t boost_active); #endif /* IMPORTANCE_INHERITANCE */ -#if IMPORTANCE_DEBUG -#define __impdebug_only -#else -#define __impdebug_only __unused -#endif +#if IMPORTANCE_TRACE +#define __imptrace_only +#else /* IMPORTANCE_TRACE */ +#define __imptrace_only __unused +#endif /* !IMPORTANCE_TRACE */ #if IMPORTANCE_INHERITANCE #define __imp_only @@ -220,7 +248,11 @@ int proc_tal_disk_tier = THROTTLE_LEVEL_TIER1; int proc_graphics_timer_qos = (LATENCY_QOS_TIER_0 & 0xFF); +#if CONFIG_EMBEDDED +const int proc_default_bg_iotier = THROTTLE_LEVEL_TIER3; +#else const int proc_default_bg_iotier = THROTTLE_LEVEL_TIER2; +#endif /* Latency/throughput QoS fields remain zeroed, i.e. TIER_UNSPECIFIED at creation */ const struct task_requested_policy default_task_requested_policy = { @@ -314,6 +346,11 @@ task_policy_set( if (count < TASK_CATEGORY_POLICY_COUNT) return (KERN_INVALID_ARGUMENT); +#if CONFIG_EMBEDDED + /* On embedded, you can't modify your own role. */ + if (current_task() == task) + return (KERN_INVALID_ARGUMENT); +#endif switch(info->role) { case TASK_FOREGROUND_APPLICATION: @@ -400,6 +437,14 @@ task_policy_set( case TASK_SUPPRESSION_POLICY: { +#if CONFIG_EMBEDDED + /* + * Suppression policy is not enabled for embedded + * because apps aren't marked as denap receivers + */ + result = KERN_INVALID_ARGUMENT; + break; +#else /* CONFIG_EMBEDDED */ task_suppression_policy_t info = (task_suppression_policy_t)policy_info; @@ -450,6 +495,7 @@ task_policy_set( break; +#endif /* CONFIG_EMBEDDED */ } default: @@ -895,9 +941,14 @@ task_policy_update_internal_locked(task_t task, boolean_t in_create, task_pend_t next.tep_io_passive = 1; /* Calculate suppression-active flag */ + boolean_t memorystatus_appnap_transition = FALSE; + if (requested.trp_sup_active && requested.trp_boosted == 0) next.tep_sup_active = 1; + if (task->effective_policy.tep_sup_active != next.tep_sup_active) + memorystatus_appnap_transition = TRUE; + /* Calculate timer QOS */ int latency_qos = requested.trp_base_latency_qos; @@ -1015,6 +1066,10 @@ task_policy_update_internal_locked(task_t task, boolean_t in_create, task_pend_t if (prev.tep_latency_qos > next.tep_latency_qos) pend_token->tpt_update_timers = 1; +#if CONFIG_EMBEDDED + if (prev.tep_watchers_bg != next.tep_watchers_bg) + pend_token->tpt_update_watchers = 1; +#endif /* CONFIG_EMBEDDED */ if (prev.tep_live_donor != next.tep_live_donor) pend_token->tpt_update_live_donor = 1; @@ -1053,15 +1108,11 @@ task_policy_update_internal_locked(task_t task, boolean_t in_create, task_pend_t prev.tep_sfi_managed != next.tep_sfi_managed ) update_sfi = TRUE; -#if CONFIG_SCHED_SFI /* Reflect task role transitions into the coalition role counters */ if (prev.tep_role != next.tep_role) { - if (task_policy_update_coalition_focal_tasks(task, prev.tep_role, next.tep_role)) { + if (task_policy_update_coalition_focal_tasks(task, prev.tep_role, next.tep_role, pend_token)) update_sfi = TRUE; - pend_token->tpt_update_coal_sfi = 1; - } } -#endif /* !CONFIG_SCHED_SFI */ boolean_t update_priority = FALSE; @@ -1139,41 +1190,63 @@ task_policy_update_internal_locked(task_t task, boolean_t in_create, task_pend_t thread_policy_update_complete_unlocked(thread, &thread_pend_token); } } + + /* + * Use the app-nap transitions to influence the + * transition of the process within the jetsam band. + * On macOS only. + */ + if (memorystatus_appnap_transition == TRUE) { + if (task->effective_policy.tep_sup_active == 1) { + memorystatus_update_priority_for_appnap(((proc_t) task->bsd_info), TRUE); + } else { + memorystatus_update_priority_for_appnap(((proc_t) task->bsd_info), FALSE); + } + } } -#if CONFIG_SCHED_SFI /* * Yet another layering violation. We reach out and bang on the coalition directly. */ static boolean_t -task_policy_update_coalition_focal_tasks(task_t task, - int prev_role, - int next_role) +task_policy_update_coalition_focal_tasks(task_t task, + int prev_role, + int next_role, + task_pend_token_t pend_token) { boolean_t sfi_transition = FALSE; - + uint32_t new_count = 0; + /* task moving into/out-of the foreground */ if (prev_role != TASK_FOREGROUND_APPLICATION && next_role == TASK_FOREGROUND_APPLICATION) { - if (task_coalition_adjust_focal_count(task, 1) == 1) + if (task_coalition_adjust_focal_count(task, 1, &new_count) && (new_count == 1)) { sfi_transition = TRUE; + pend_token->tpt_update_tg_ui_flag = TRUE; + } } else if (prev_role == TASK_FOREGROUND_APPLICATION && next_role != TASK_FOREGROUND_APPLICATION) { - if (task_coalition_adjust_focal_count(task, -1) == 0) + if (task_coalition_adjust_focal_count(task, -1, &new_count) && (new_count == 0)) { sfi_transition = TRUE; + pend_token->tpt_update_tg_ui_flag = TRUE; + } } /* task moving into/out-of background */ if (prev_role != TASK_BACKGROUND_APPLICATION && next_role == TASK_BACKGROUND_APPLICATION) { - if (task_coalition_adjust_nonfocal_count(task, 1) == 1) + if (task_coalition_adjust_nonfocal_count(task, 1, &new_count) && (new_count == 1)) sfi_transition = TRUE; } else if (prev_role == TASK_BACKGROUND_APPLICATION && next_role != TASK_BACKGROUND_APPLICATION) { - if (task_coalition_adjust_nonfocal_count(task, -1) == 0) + if (task_coalition_adjust_nonfocal_count(task, -1, &new_count) && (new_count == 0)) sfi_transition = TRUE; } + if (sfi_transition) + pend_token->tpt_update_coal_sfi = 1; return sfi_transition; } +#if CONFIG_SCHED_SFI + /* coalition object is locked */ static void task_sfi_reevaluate_cb(coalition_t coal, void *ctx, task_t task) @@ -1212,6 +1285,10 @@ task_policy_update_complete_unlocked(task_t task, task_pend_token_t pend_token) if (pend_token->tpt_update_timers) ml_timer_evaluate(); +#if CONFIG_EMBEDDED + if (pend_token->tpt_update_watchers) + apply_appstate_watchers(task); +#endif /* CONFIG_EMBEDDED */ if (pend_token->tpt_update_live_donor) task_importance_update_live_donor(task); @@ -1222,6 +1299,7 @@ task_policy_update_complete_unlocked(task_t task, task_pend_token_t pend_token) coalition_for_each_task(task->coalition[COALITION_TYPE_RESOURCE], (void *)task, task_sfi_reevaluate_cb); #endif /* CONFIG_SCHED_SFI */ + } /* @@ -1739,8 +1817,12 @@ proc_set_task_spawnpolicy(task_t task, int apptype, int qos_clamp, int role, task_importance_mark_donor(task, FALSE); task_importance_mark_live_donor(task, TRUE); task_importance_mark_receiver(task, FALSE); +#if CONFIG_EMBEDDED + task_importance_mark_denap_receiver(task, FALSE); +#else /* Apps are de-nap recievers on desktop for suppression behaviors */ task_importance_mark_denap_receiver(task, TRUE); +#endif /* CONFIG_EMBEDDED */ break; case TASK_APPTYPE_DAEMON_INTERACTIVE: @@ -1812,6 +1894,12 @@ proc_set_task_spawnpolicy(task_t task, int apptype, int qos_clamp, int role, task->requested_policy.trp_apptype = apptype; } +#if CONFIG_EMBEDDED + /* Remove this after launchd starts setting it properly */ + if (apptype == TASK_APPTYPE_APP_DEFAULT && role == TASK_UNSPECIFIED) { + task->requested_policy.trp_role = TASK_FOREGROUND_APPLICATION; + } else +#endif if (role != TASK_UNSPECIFIED) { task->requested_policy.trp_role = role; } @@ -1951,6 +2039,13 @@ proc_get_darwinbgstate(task_t task, uint32_t * flagsp) if (task->requested_policy.trp_int_darwinbg) *flagsp |= PROC_FLAG_DARWINBG; +#if CONFIG_EMBEDDED + if (task->requested_policy.trp_apptype == TASK_APPTYPE_DAEMON_BACKGROUND) + *flagsp |= PROC_FLAG_IOS_APPLEDAEMON; + + if (task->requested_policy.trp_apptype == TASK_APPTYPE_DAEMON_ADAPTIVE) + *flagsp |= PROC_FLAG_IOS_IMPPROMOTION; +#endif /* CONFIG_EMBEDDED */ if (task->requested_policy.trp_apptype == TASK_APPTYPE_APP_DEFAULT || task->requested_policy.trp_apptype == TASK_APPTYPE_APP_TAL) @@ -2713,6 +2808,292 @@ task_action_cpuusage(thread_call_param_t param0, __unused thread_call_param_t pa * Routines for taskwatch and pidbind */ +#if CONFIG_EMBEDDED + +lck_mtx_t task_watch_mtx; + +void +task_watch_init(void) +{ + lck_mtx_init(&task_watch_mtx, &task_lck_grp, &task_lck_attr); +} + +static void +task_watch_lock(void) +{ + lck_mtx_lock(&task_watch_mtx); +} + +static void +task_watch_unlock(void) +{ + lck_mtx_unlock(&task_watch_mtx); +} + +static void +add_taskwatch_locked(task_t task, task_watch_t * twp) +{ + queue_enter(&task->task_watchers, twp, task_watch_t *, tw_links); + task->num_taskwatchers++; + +} + +static void +remove_taskwatch_locked(task_t task, task_watch_t * twp) +{ + queue_remove(&task->task_watchers, twp, task_watch_t *, tw_links); + task->num_taskwatchers--; +} + + +int +proc_lf_pidbind(task_t curtask, uint64_t tid, task_t target_task, int bind) +{ + thread_t target_thread = NULL; + int ret = 0, setbg = 0; + task_watch_t *twp = NULL; + task_t task = TASK_NULL; + + target_thread = task_findtid(curtask, tid); + if (target_thread == NULL) + return ESRCH; + /* holds thread reference */ + + if (bind != 0) { + /* task is still active ? */ + task_lock(target_task); + if (target_task->active == 0) { + task_unlock(target_task); + ret = ESRCH; + goto out; + } + task_unlock(target_task); + + twp = (task_watch_t *)kalloc(sizeof(task_watch_t)); + if (twp == NULL) { + task_watch_unlock(); + ret = ENOMEM; + goto out; + } + + bzero(twp, sizeof(task_watch_t)); + + task_watch_lock(); + + if (target_thread->taskwatch != NULL){ + /* already bound to another task */ + task_watch_unlock(); + + kfree(twp, sizeof(task_watch_t)); + ret = EBUSY; + goto out; + } + + task_reference(target_task); + + setbg = proc_get_effective_task_policy(target_task, TASK_POLICY_WATCHERS_BG); + + twp->tw_task = target_task; /* holds the task reference */ + twp->tw_thread = target_thread; /* holds the thread reference */ + twp->tw_state = setbg; + twp->tw_importance = target_thread->importance; + + add_taskwatch_locked(target_task, twp); + + target_thread->taskwatch = twp; + + task_watch_unlock(); + + if (setbg) + set_thread_appbg(target_thread, setbg, INT_MIN); + + /* retain the thread reference as it is in twp */ + target_thread = NULL; + } else { + /* unbind */ + task_watch_lock(); + if ((twp = target_thread->taskwatch) != NULL) { + task = twp->tw_task; + target_thread->taskwatch = NULL; + remove_taskwatch_locked(task, twp); + + task_watch_unlock(); + + task_deallocate(task); /* drop task ref in twp */ + set_thread_appbg(target_thread, 0, twp->tw_importance); + thread_deallocate(target_thread); /* drop thread ref in twp */ + kfree(twp, sizeof(task_watch_t)); + } else { + task_watch_unlock(); + ret = 0; /* return success if it not alredy bound */ + goto out; + } + } +out: + thread_deallocate(target_thread); /* drop thread ref acquired in this routine */ + return(ret); +} + +static void +set_thread_appbg(thread_t thread, int setbg, __unused int importance) +{ + int enable = (setbg ? TASK_POLICY_ENABLE : TASK_POLICY_DISABLE); + + proc_set_thread_policy(thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_PIDBIND_BG, enable); +} + +static void +apply_appstate_watchers(task_t task) +{ + int numwatchers = 0, i, j, setbg; + thread_watchlist_t * threadlist; + task_watch_t * twp; + +retry: + /* if no watchers on the list return */ + if ((numwatchers = task->num_taskwatchers) == 0) + return; + + threadlist = (thread_watchlist_t *)kalloc(numwatchers*sizeof(thread_watchlist_t)); + if (threadlist == NULL) + return; + + bzero(threadlist, numwatchers*sizeof(thread_watchlist_t)); + + task_watch_lock(); + /*serialize application of app state changes */ + + if (task->watchapplying != 0) { + lck_mtx_sleep(&task_watch_mtx, LCK_SLEEP_DEFAULT, &task->watchapplying, THREAD_UNINT); + task_watch_unlock(); + kfree(threadlist, numwatchers*sizeof(thread_watchlist_t)); + goto retry; + } + + if (numwatchers != task->num_taskwatchers) { + task_watch_unlock(); + kfree(threadlist, numwatchers*sizeof(thread_watchlist_t)); + goto retry; + } + + setbg = proc_get_effective_task_policy(task, TASK_POLICY_WATCHERS_BG); + + task->watchapplying = 1; + i = 0; + queue_iterate(&task->task_watchers, twp, task_watch_t *, tw_links) { + + threadlist[i].thread = twp->tw_thread; + thread_reference(threadlist[i].thread); + if (setbg != 0) { + twp->tw_importance = twp->tw_thread->importance; + threadlist[i].importance = INT_MIN; + } else + threadlist[i].importance = twp->tw_importance; + i++; + if (i > numwatchers) + break; + } + + task_watch_unlock(); + + for (j = 0; j< i; j++) { + set_thread_appbg(threadlist[j].thread, setbg, threadlist[j].importance); + thread_deallocate(threadlist[j].thread); + } + kfree(threadlist, numwatchers*sizeof(thread_watchlist_t)); + + + task_watch_lock(); + task->watchapplying = 0; + thread_wakeup_one(&task->watchapplying); + task_watch_unlock(); +} + +void +thead_remove_taskwatch(thread_t thread) +{ + task_watch_t * twp; + int importance = 0; + + task_watch_lock(); + if ((twp = thread->taskwatch) != NULL) { + thread->taskwatch = NULL; + remove_taskwatch_locked(twp->tw_task, twp); + } + task_watch_unlock(); + if (twp != NULL) { + thread_deallocate(twp->tw_thread); + task_deallocate(twp->tw_task); + importance = twp->tw_importance; + kfree(twp, sizeof(task_watch_t)); + /* remove the thread and networkbg */ + set_thread_appbg(thread, 0, importance); + } +} + +void +task_removewatchers(task_t task) +{ + int numwatchers = 0, i, j; + task_watch_t ** twplist = NULL; + task_watch_t * twp = NULL; + +retry: + if ((numwatchers = task->num_taskwatchers) == 0) + return; + + twplist = (task_watch_t **)kalloc(numwatchers*sizeof(task_watch_t *)); + if (twplist == NULL) + return; + + bzero(twplist, numwatchers*sizeof(task_watch_t *)); + + task_watch_lock(); + if (task->num_taskwatchers == 0) { + task_watch_unlock(); + goto out; + } + + if (numwatchers != task->num_taskwatchers) { + task_watch_unlock(); + kfree(twplist, numwatchers*sizeof(task_watch_t *)); + numwatchers = 0; + goto retry; + } + + i = 0; + while((twp = (task_watch_t *)dequeue_head(&task->task_watchers)) != NULL) + { + twplist[i] = twp; + task->num_taskwatchers--; + + /* + * Since the linkage is removed and thead state cleanup is already set up, + * remove the refernce from the thread. + */ + twp->tw_thread->taskwatch = NULL; /* removed linkage, clear thread holding ref */ + i++; + if ((task->num_taskwatchers == 0) || (i > numwatchers)) + break; + } + + task_watch_unlock(); + + for (j = 0; j< i; j++) { + + twp = twplist[j]; + /* remove thread and network bg */ + set_thread_appbg(twp->tw_thread, 0, twp->tw_importance); + thread_deallocate(twp->tw_thread); + task_deallocate(twp->tw_task); + kfree(twp, sizeof(task_watch_t)); + } + +out: + kfree(twplist, numwatchers*sizeof(task_watch_t *)); + +} +#endif /* CONFIG_EMBEDDED */ /* * Routines for importance donation/inheritance/boosting @@ -2812,14 +3193,14 @@ task_importance_reset(__imp_only task_t task) static void task_set_boost_locked(task_t task, boolean_t boost_active) { -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_BOOST, (boost_active ? IMP_BOOSTED : IMP_UNBOOSTED)) | DBG_FUNC_START), proc_selfpid(), task_pid(task), trequested_0(task), trequested_1(task), 0); -#endif +#endif /* IMPORTANCE_TRACE */ task->requested_policy.trp_boosted = boost_active; -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE if (boost_active == TRUE){ DTRACE_BOOST2(boost, task_t, task, int, task_pid(task)); } else { @@ -2828,7 +3209,7 @@ task_set_boost_locked(task_t task, boolean_t boost_active) KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_BOOST, (boost_active ? IMP_BOOSTED : IMP_UNBOOSTED)) | DBG_FUNC_END), proc_selfpid(), task_pid(task), trequested_0(task), trequested_1(task), 0); -#endif +#endif /* IMPORTANCE_TRACE */ } /* @@ -3030,8 +3411,8 @@ task_add_importance_watchport(task_t task, mach_port_t port, int *boostp) { int boost = 0; - __impdebug_only int released_pid = 0; - __impdebug_only int pid = task_pid(task); + __imptrace_only int released_pid = 0; + __imptrace_only int pid = task_pid(task); ipc_importance_task_t release_imp_task = IIT_NULL; @@ -3080,10 +3461,10 @@ task_add_importance_watchport(task_t task, mach_port_t port, int *boostp) // released_pid = task_pid(release_imp_task); /* TODO: Need ref-safe way to get pid */ ipc_importance_task_release(release_imp_task); } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_WATCHPORT, 0)) | DBG_FUNC_NONE, proc_selfpid(), pid, boost, released_pid, 0); -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ } *boostp = boost; diff --git a/osfmk/kern/telemetry.c b/osfmk/kern/telemetry.c index 2e292d1dd..68f2f7f9a 100644 --- a/osfmk/kern/telemetry.c +++ b/osfmk/kern/telemetry.c @@ -161,7 +161,11 @@ void telemetry_init(void) */ if (!PE_parse_boot_argn("telemetry_sample_all_tasks", &telemetry_sample_all_tasks, sizeof(telemetry_sample_all_tasks))) { +#if CONFIG_EMBEDDED && !(DEVELOPMENT || DEBUG) + telemetry_sample_all_tasks = FALSE; +#else telemetry_sample_all_tasks = TRUE; +#endif /* CONFIG_EMBEDDED && !(DEVELOPMENT || DEBUG) */ } @@ -302,7 +306,7 @@ void telemetry_mark_curthread(boolean_t interrupted_userspace) telemetry_needs_record = FALSE; thread_ast_set(thread, ast_bits); - ast_propagate(thread->ast); + ast_propagate(thread); } void compute_telemetry(void *arg __unused) @@ -334,12 +338,17 @@ telemetry_notify_user(void) ipc_port_release_send(user_port); } -void telemetry_ast(thread_t thread, boolean_t interrupted_userspace, boolean_t io_telemetry) +void telemetry_ast(thread_t thread, ast_t reasons) { + assert((reasons & AST_TELEMETRY_ALL) != AST_TELEMETRY_ALL); /* only one is valid at a time */ + + boolean_t io_telemetry = (reasons & AST_TELEMETRY_IO) ? TRUE : FALSE; + boolean_t interrupted_userspace = (reasons & AST_TELEMETRY_USER) ? TRUE : FALSE; + uint8_t microsnapshot_flags = kInterruptRecord; - if (io_telemetry == TRUE) { + + if (io_telemetry == TRUE) microsnapshot_flags = kIORecord; - } if (interrupted_userspace) microsnapshot_flags |= kUserMode; diff --git a/osfmk/kern/telemetry.h b/osfmk/kern/telemetry.h index 4cbd028ba..b5e023401 100644 --- a/osfmk/kern/telemetry.h +++ b/osfmk/kern/telemetry.h @@ -42,7 +42,7 @@ extern void telemetry_init(void); extern void compute_telemetry(void *); -extern void telemetry_ast(thread_t, boolean_t interrupted_userspace, boolean_t io_telemetry); +extern void telemetry_ast(thread_t thread, uint32_t reasons); extern int telemetry_gather(user_addr_t buffer, uint32_t *length, boolean_t mark); diff --git a/osfmk/kern/thread.c b/osfmk/kern/thread.c index 8e32c589c..0057c988f 100644 --- a/osfmk/kern/thread.c +++ b/osfmk/kern/thread.c @@ -112,10 +112,13 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include @@ -124,6 +127,11 @@ #include #endif +#if MONOTONIC +#include +#include +#endif /* MONOTONIC */ + #include #include #include @@ -134,6 +142,9 @@ #include #include #include +#include + +#include /* * Exported interfaces @@ -164,9 +175,10 @@ decl_simple_lock_data(static,thread_exception_lock) static queue_head_t thread_exception_queue; struct thread_exception_elt { - queue_chain_t elt; - task_t exception_task; - thread_t exception_thread; + queue_chain_t elt; + exception_type_t exception_type; + task_t exception_task; + thread_t exception_thread; }; static struct thread thread_template, init_thread; @@ -179,6 +191,7 @@ static void sched_call_null( extern void proc_exit(void *); extern mach_exception_data_type_t proc_encode_exit_exception_code(void *); extern uint64_t get_dispatchqueue_offset_from_proc(void *); +extern uint64_t get_return_to_kernel_offset_from_proc(void *p); extern int proc_selfpid(void); extern char * proc_name_address(void *p); #endif /* MACH_BSD */ @@ -271,7 +284,9 @@ thread_bootstrap(void) thread_template.quantum_remaining = 0; thread_template.last_run_time = 0; - thread_template.last_made_runnable_time = 0; + thread_template.last_made_runnable_time = THREAD_NOT_RUNNABLE; + thread_template.last_basepri_change_time = THREAD_NOT_RUNNABLE; + thread_template.same_pri_latency = 0; thread_template.computation_metered = 0; thread_template.computation_epoch = 0; @@ -284,6 +299,11 @@ thread_bootstrap(void) #endif thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0; +#if MONOTONIC + memset(&thread_template.t_monotonic, 0, + sizeof(thread_template.t_monotonic)); +#endif /* MONOTONIC */ + thread_template.bound_processor = PROCESSOR_NULL; thread_template.last_processor = PROCESSOR_NULL; @@ -291,6 +311,7 @@ thread_bootstrap(void) timer_init(&thread_template.user_timer); timer_init(&thread_template.system_timer); + timer_init(&thread_template.ptime); thread_template.user_timer_save = 0; thread_template.system_timer_save = 0; thread_template.vtimer_user_save = 0; @@ -347,18 +368,20 @@ thread_bootstrap(void) thread_template.t_ledger = LEDGER_NULL; thread_template.t_threadledger = LEDGER_NULL; -#ifdef CONFIG_BANK thread_template.t_bankledger = LEDGER_NULL; thread_template.t_deduct_bank_ledger_time = 0; -#endif thread_template.requested_policy = (struct thread_requested_policy) {}; thread_template.effective_policy = (struct thread_effective_policy) {}; bzero(&thread_template.overrides, sizeof(thread_template.overrides)); + thread_template.sync_ipc_overrides = 0; thread_template.iotier_override = THROTTLE_LEVEL_NONE; thread_template.thread_io_stats = NULL; +#if CONFIG_EMBEDDED + thread_template.taskwatch = NULL; +#endif /* CONFIG_EMBEDDED */ thread_template.thread_callout_interrupt_wakeups = thread_template.thread_callout_platform_idle_wakeups = 0; thread_template.thread_timer_wakeups_bin_1 = thread_template.thread_timer_wakeups_bin_2 = 0; @@ -369,7 +392,7 @@ thread_bootstrap(void) thread_template.ith_voucher_name = MACH_PORT_NULL; thread_template.ith_voucher = IPC_VOUCHER_NULL; - thread_template.work_interval_id = 0; + thread_template.th_work_interval = NULL; init_thread = thread_template; machine_set_current_thread(&init_thread); @@ -432,8 +455,14 @@ thread_corpse_continue(void) thread_t thread = current_thread(); thread_terminate_internal(thread); - ml_set_interrupts_enabled(FALSE); - ast_taken(AST_APC, TRUE); + + /* + * Handle the thread termination directly + * here instead of returning to userspace. + */ + assert(thread->active == FALSE); + thread_ast_clear(thread, AST_APC); + thread_apc_ast(thread); panic("thread_corpse_continue"); /*NOTREACHED*/ @@ -499,6 +528,11 @@ thread_terminate_self(void) thread_unlock(thread); splx(s); +#if CONFIG_EMBEDDED + thead_remove_taskwatch(thread); +#endif /* CONFIG_EMBEDDED */ + + work_interval_thread_terminate(thread); thread_mtx_lock(thread); @@ -506,20 +540,31 @@ thread_terminate_self(void) thread_mtx_unlock(thread); + bank_swap_thread_bank_ledger(thread, NULL); + task = thread->task; uthread_cleanup(task, thread->uthread, task->bsd_info); - threadcnt = hw_atomic_sub(&task->active_thread_count, 1); if (task->bsd_info && !task_is_exec_copy(task)) { /* trace out pid before we sign off */ - long dbg_arg1 = 0; - - kdbg_trace_data(thread->task->bsd_info, &dbg_arg1); + long dbg_arg1 = 0; + long dbg_arg2 = 0; + + kdbg_trace_data(thread->task->bsd_info, &dbg_arg1, &dbg_arg2); KERNEL_DEBUG_CONSTANT(TRACE_DATA_THREAD_TERMINATE_PID | DBG_FUNC_NONE, dbg_arg1, 0, 0, 0, 0); } + /* + * After this subtraction, this thread should never access + * task->bsd_info unless it got 0 back from the hw_atomic_sub. It + * could be racing with other threads to be the last thread in the + * process, and the last thread in the process will tear down the proc + * structure and zero-out task->bsd_info. + */ + threadcnt = hw_atomic_sub(&task->active_thread_count, 1); + /* * If we are the last thread to terminate and the task is * associated with a BSD process, perform BSD process exit. @@ -545,7 +590,7 @@ thread_terminate_self(void) * last thread for this task. */ if (task->corpse_info) { - task_deliver_crash_notification(task, current_thread(), subcode); + task_deliver_crash_notification(task, current_thread(), EXC_RESOURCE, subcode); } } @@ -613,7 +658,7 @@ thread_deallocate_safe(thread_t thread) { assert_thread_magic(thread); - uint32_t old_refcount = hw_atomic_sub(&(thread)->ref_count, 1) + 1; + uint32_t old_refcount = atomic_fetch_sub_explicit(&thread->ref_count, 1, memory_order_release); if (__improbable(old_refcount <= 1)) panic("bad thread refcount: %d", old_refcount); @@ -624,17 +669,24 @@ thread_deallocate( thread_t thread) { task_t task; + __assert_only uint32_t th_ref_count; if (thread == THREAD_NULL) return; assert_thread_magic(thread); - assert(thread->ref_count > 0); - if (__probable(hw_atomic_sub(&(thread)->ref_count, 1) > 0)) - return; + if (__probable(atomic_fetch_sub_explicit(&thread->ref_count, 1, + memory_order_release) - 1 > 0)) { + return; + } + + th_ref_count = atomic_load_explicit(&thread->ref_count, memory_order_acquire); + assert(th_ref_count == 0); + + assert(thread_owned_workloops_count(thread) == 0); - if(!(thread->state & TH_TERMINATE2)) + if (!(thread->state & TH_TERMINATE2)) panic("thread_deallocate: thread not properly terminated\n"); assert(thread->runq == PROCESSOR_NULL); @@ -687,6 +739,29 @@ thread_deallocate( zfree(thread_zone, thread); } +void +thread_starts_owning_workloop(thread_t thread) +{ + atomic_fetch_add_explicit(&thread->kqwl_owning_count, 1, + memory_order_relaxed); +} + +void +thread_ends_owning_workloop(thread_t thread) +{ + __assert_only uint32_t count; + count = atomic_fetch_sub_explicit(&thread->kqwl_owning_count, 1, + memory_order_relaxed); + assert(count > 0); +} + +uint32_t +thread_owned_workloops_count(thread_t thread) +{ + return atomic_load_explicit(&thread->kqwl_owning_count, + memory_order_relaxed); +} + /* * thread_inspect_deallocate: * @@ -702,7 +777,7 @@ thread_inspect_deallocate( /* * thread_exception_daemon: * - * Deliver EXC_RESOURCE exception + * Deliver EXC_{RESOURCE,GUARD} exception */ static void thread_exception_daemon(void) @@ -710,16 +785,18 @@ thread_exception_daemon(void) struct thread_exception_elt *elt; task_t task; thread_t thread; + exception_type_t etype; simple_lock(&thread_exception_lock); while ((elt = (struct thread_exception_elt *)dequeue_head(&thread_exception_queue)) != NULL) { simple_unlock(&thread_exception_lock); + etype = elt->exception_type; task = elt->exception_task; thread = elt->exception_thread; assert_thread_magic(thread); - kfree(elt, sizeof(struct thread_exception_elt)); + kfree(elt, sizeof (*elt)); /* wait for all the threads in the task to terminate */ task_lock(task); @@ -731,8 +808,8 @@ thread_exception_daemon(void) /* Consumes the thread ref returned by task_generate_corpse_internal */ thread_deallocate(thread); - /* Deliver the EXC_RESOURCE notification, also clears the corpse. */ - task_deliver_crash_notification(task, thread, 0); + /* Deliver the notification, also clears the corpse. */ + task_deliver_crash_notification(task, thread, etype, 0); simple_lock(&thread_exception_lock); } @@ -746,16 +823,17 @@ thread_exception_daemon(void) /* * thread_exception_enqueue: * - * Enqueue a corpse port to be delivered an EXC_RESOURCE. + * Enqueue a corpse port to be delivered an EXC_{RESOURCE,GUARD}. */ void thread_exception_enqueue( task_t task, - thread_t thread) + thread_t thread, + exception_type_t etype) { - struct thread_exception_elt *elt = (struct thread_exception_elt*) kalloc( - sizeof(struct thread_exception_elt)); - + assert(EXC_RESOURCE == etype || EXC_GUARD == etype); + struct thread_exception_elt *elt = kalloc(sizeof (*elt)); + elt->exception_type = etype; elt->exception_task = task; elt->exception_thread = thread; @@ -784,12 +862,16 @@ thread_copy_resource_info( dst_thread->precise_user_kernel_time = src_thread->precise_user_kernel_time; dst_thread->user_timer = src_thread->user_timer; dst_thread->user_timer_save = src_thread->user_timer_save; + dst_thread->system_timer = src_thread->system_timer; dst_thread->system_timer_save = src_thread->system_timer_save; + dst_thread->vtimer_user_save = src_thread->vtimer_user_save; + dst_thread->vtimer_prof_save = src_thread->vtimer_prof_save; + dst_thread->vtimer_rlim_save = src_thread->vtimer_rlim_save; + dst_thread->vtimer_qos_save = src_thread->vtimer_qos_save; dst_thread->syscalls_unix = src_thread->syscalls_unix; dst_thread->syscalls_mach = src_thread->syscalls_mach; ledger_rollup(dst_thread->t_threadledger, src_thread->t_threadledger); *dst_thread->thread_io_stats = *src_thread->thread_io_stats; - } /* @@ -829,6 +911,7 @@ thread_terminate_daemon(void) task_lock(task); task->total_user_time += timer_grab(&thread->user_timer); + task->total_ptime += timer_grab(&thread->ptime); if (thread->precise_user_kernel_time) { task->total_system_time += timer_grab(&thread->system_timer); } else { @@ -847,6 +930,10 @@ thread_terminate_daemon(void) task->task_gpu_ns += ml_gpu_stat(thread); task->task_energy += ml_energy_stat(thread); +#if MONOTONIC + mt_terminate_update(task, thread); +#endif /* MONOTONIC */ + thread_update_qos_cpu_time(thread); queue_remove(&task->threads, thread, thread_t, task_threads); @@ -1016,7 +1103,7 @@ thread_daemon_init(void) simple_lock_init(&thread_stack_lock, 0); queue_init(&thread_stack_queue); - result = kernel_thread_start_priority((thread_continue_t)thread_stack_daemon, NULL, BASEPRI_PREEMPT, &thread); + result = kernel_thread_start_priority((thread_continue_t)thread_stack_daemon, NULL, BASEPRI_PREEMPT_HIGH, &thread); if (result != KERN_SUCCESS) panic("thread_daemon_init: thread_stack_daemon"); @@ -1112,6 +1199,11 @@ thread_create_internal( new_thread->thread_io_stats = (io_stat_info_t)kalloc(sizeof(struct io_stat_info)); assert(new_thread->thread_io_stats != NULL); bzero(new_thread->thread_io_stats, sizeof(struct io_stat_info)); + new_thread->sync_ipc_overrides = 0; + +#if KASAN + kasan_init_thread(&new_thread->kasan_data); +#endif #if CONFIG_IOSCHED /* Clear out the I/O Scheduling info for AppleFSCompression */ @@ -1171,10 +1263,9 @@ thread_create_internal( ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time); } -#ifdef CONFIG_BANK new_thread->t_bankledger = LEDGER_NULL; new_thread->t_deduct_bank_ledger_time = 0; -#endif + new_thread->t_deduct_bank_ledger_energy = 0; new_thread->t_ledger = new_thread->task->ledger; if (new_thread->t_ledger) @@ -1204,6 +1295,11 @@ thread_create_internal( new_priority = (priority < 0)? parent_task->priority: priority; if (new_priority > new_thread->max_priority) new_priority = new_thread->max_priority; +#if CONFIG_EMBEDDED + if (new_priority < MAXPRI_THROTTLE) { + new_priority = MAXPRI_THROTTLE; + } +#endif /* CONFIG_EMBEDDED */ new_thread->importance = new_priority - new_thread->task_priority; @@ -1214,6 +1310,10 @@ thread_create_internal( new_thread->pri_shift = sched_pri_shifts[new_thread->th_sched_bucket]; #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */ +#if CONFIG_EMBEDDED + if (parent_task->max_priority <= MAXPRI_THROTTLE) + sched_thread_mode_demote(new_thread, TH_SFLAG_THROTTLED); +#endif /* CONFIG_EMBEDDED */ thread_policy_create(new_thread); @@ -1227,6 +1327,7 @@ thread_create_internal( /* Protected by the tasks_threads_lock */ new_thread->thread_id = ++thread_unique_id; + queue_enter(&threads, new_thread, thread_t, threads); threads_count++; @@ -1243,7 +1344,7 @@ thread_create_internal( { long dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4; - kdbg_trace_data(parent_task->bsd_info, &dbg_arg2); + kdbg_trace_data(parent_task->bsd_info, &dbg_arg2, &dbg_arg4); /* * Starting with 26604425, exec'ing creates a new task/thread. @@ -1260,9 +1361,10 @@ thread_create_internal( */ dbg_arg3 = (task_is_exec_copy(parent_task)) ? TRUE : 0; + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, TRACE_DATA_NEWTHREAD | DBG_FUNC_NONE, - (vm_address_t)(uintptr_t)thread_tid(new_thread), dbg_arg2, dbg_arg3, 0, 0); + (vm_address_t)(uintptr_t)thread_tid(new_thread), dbg_arg2, dbg_arg3, dbg_arg4, 0); kdbg_trace_string(parent_task->bsd_info, &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4); @@ -1532,7 +1634,7 @@ kernel_thread_create( thread_t thread; task_t task = kernel_task; - result = thread_create_internal(task, priority, continuation, TH_OPTION_NONE, &thread); + result = thread_create_internal(task, priority, continuation, TH_OPTION_NOCRED | TH_OPTION_NONE, &thread); if (result != KERN_SUCCESS) return (result); @@ -1541,6 +1643,9 @@ kernel_thread_create( stack_alloc(thread); assert(thread->kernel_stack != 0); +#if CONFIG_EMBEDDED + if (priority > BASEPRI_KERNEL) +#endif thread->reserved_stack = thread->kernel_stack; thread->parameter = parameter; @@ -2039,13 +2144,16 @@ clear_thread_rwlock_boost(void) } } + /* * XXX assuming current thread only, for now... */ void -thread_guard_violation(thread_t thread, unsigned type) +thread_guard_violation(thread_t thread, + mach_exception_data_type_t code, mach_exception_data_type_t subcode) { assert(thread == current_thread()); + assert(thread->task != kernel_task); spl_t s = splsched(); /* @@ -2053,9 +2161,11 @@ thread_guard_violation(thread_t thread, unsigned type) * to store all info required to handle the AST when * returning to userspace */ - thread->guard_exc_info.type = type; + assert(EXC_GUARD_DECODE_GUARD_TYPE(code)); + thread->guard_exc_info.code = code; + thread->guard_exc_info.subcode = subcode; thread_ast_set(thread, AST_GUARD); - ast_propagate(thread->ast); + ast_propagate(thread); splx(s); } @@ -2070,12 +2180,27 @@ thread_guard_violation(thread_t thread, unsigned type) * info and raises the exception. */ void -guard_ast(thread_t thread) -{ - if (thread->guard_exc_info.type == GUARD_TYPE_MACH_PORT) - mach_port_guard_ast(thread); - else - fd_guard_ast(thread); +guard_ast(thread_t t) +{ + const mach_exception_data_type_t + code = t->guard_exc_info.code, + subcode = t->guard_exc_info.subcode; + + switch (EXC_GUARD_DECODE_GUARD_TYPE(code)) { + case GUARD_TYPE_MACH_PORT: + mach_port_guard_ast(t, code, subcode); + break; + case GUARD_TYPE_FD: + fd_guard_ast(t, code, subcode); + break; +#if CONFIG_VNGUARD + case GUARD_TYPE_VN: + vn_guard_ast(t, code, subcode); + break; +#endif + default: + panic("guard_exc_info %llx %llx", code, subcode); + } } static void @@ -2310,7 +2435,8 @@ init_thread_ledgers(void) { } thread_ledgers.cpu_time = idx; - + + ledger_template_complete(t); thread_ledger_template = t; } @@ -2558,6 +2684,37 @@ thread_dispatchqaddr( return dispatchqueue_addr; } +uint64_t +thread_rettokern_addr( + thread_t thread) +{ + uint64_t rettokern_addr; + uint64_t rettokern_offset; + uint64_t thread_handle; + + if (thread == THREAD_NULL) + return 0; + + thread_handle = thread->machine.cthread_self; + if (thread_handle == 0) + return 0; + + if (thread->task->bsd_info) { + rettokern_offset = get_return_to_kernel_offset_from_proc(thread->task->bsd_info); + + /* Return 0 if return to kernel offset is not initialized. */ + if (rettokern_offset == 0) { + rettokern_addr = 0; + } else { + rettokern_addr = thread_handle + rettokern_offset; + } + } else { + rettokern_addr = 0; + } + + return rettokern_addr; +} + /* * Export routines to other components for things that are done as macros * within the osfmk component. @@ -2598,9 +2755,8 @@ thread_set_voucher_name(mach_port_name_t voucher_name) thread_t thread = current_thread(); ipc_voucher_t new_voucher = IPC_VOUCHER_NULL; ipc_voucher_t voucher; -#ifdef CONFIG_BANK ledger_t bankledger = NULL; -#endif + thread_group_t banktg = NULL; if (MACH_PORT_DEAD == voucher_name) return KERN_INVALID_RIGHT; @@ -2613,19 +2769,16 @@ thread_set_voucher_name(mach_port_name_t voucher_name) if (IPC_VOUCHER_NULL == new_voucher) return KERN_INVALID_ARGUMENT; } -#ifdef CONFIG_BANK - bankledger = bank_get_voucher_ledger(new_voucher); -#endif + bank_get_bank_ledger_and_thread_group(new_voucher, &bankledger, &banktg); thread_mtx_lock(thread); voucher = thread->ith_voucher; thread->ith_voucher_name = voucher_name; thread->ith_voucher = new_voucher; -#ifdef CONFIG_BANK - bank_swap_thread_bank_ledger(thread, bankledger); -#endif thread_mtx_unlock(thread); + bank_swap_thread_bank_ledger(thread, bankledger); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), @@ -2727,30 +2880,26 @@ thread_set_mach_voucher( ipc_voucher_t voucher) { ipc_voucher_t old_voucher; -#ifdef CONFIG_BANK ledger_t bankledger = NULL; -#endif + thread_group_t banktg = NULL; if (THREAD_NULL == thread) return KERN_INVALID_ARGUMENT; - if (thread != current_thread() || thread->started) + if (thread != current_thread() && thread->started) return KERN_INVALID_ARGUMENT; - ipc_voucher_reference(voucher); -#ifdef CONFIG_BANK - bankledger = bank_get_voucher_ledger(voucher); -#endif + bank_get_bank_ledger_and_thread_group(voucher, &bankledger, &banktg); + thread_mtx_lock(thread); old_voucher = thread->ith_voucher; thread->ith_voucher = voucher; thread->ith_voucher_name = MACH_PORT_NULL; -#ifdef CONFIG_BANK - bank_swap_thread_bank_ledger(thread, bankledger); -#endif thread_mtx_unlock(thread); + bank_swap_thread_bank_ledger(thread, bankledger); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), @@ -2781,19 +2930,16 @@ thread_swap_mach_voucher( { mach_port_name_t old_voucher_name; ipc_voucher_t old_voucher; -#ifdef CONFIG_BANK ledger_t bankledger = NULL; -#endif + thread_group_t banktg = NULL; if (THREAD_NULL == thread) return KERN_INVALID_TASK; - if (thread != current_thread() || thread->started) + if (thread != current_thread() && thread->started) return KERN_INVALID_ARGUMENT; -#ifdef CONFIG_BANK - bankledger = bank_get_voucher_ledger(new_voucher); -#endif + bank_get_bank_ledger_and_thread_group(new_voucher, &bankledger, &banktg); thread_mtx_lock(thread); @@ -2823,10 +2969,8 @@ thread_swap_mach_voucher( ipc_voucher_reference(new_voucher); thread->ith_voucher = new_voucher; thread->ith_voucher_name = MACH_PORT_NULL; -#ifdef CONFIG_BANK - bank_swap_thread_bank_ledger(thread, bankledger); -#endif thread_mtx_unlock(thread); + bank_swap_thread_bank_ledger(thread, bankledger); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE, @@ -2871,6 +3015,7 @@ thread_get_current_voucher_origin_pid( return kr; } + boolean_t thread_has_thread_name(thread_t th) { @@ -2904,6 +3049,20 @@ void thread_enable_send_importance(thread_t thread, boolean_t enable) thread->options &= ~TH_OPT_SEND_IMPORTANCE; } +/* + * thread_set_allocation_name - . + */ + +kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name) +{ + kern_allocation_name_t ret; + thread_kernel_state_t kstate = thread_get_kernel_state(current_thread()); + ret = kstate->allocation_name; + // fifo + if (!new_name || !kstate->allocation_name) kstate->allocation_name = new_name; + return ret; +} + #if CONFIG_DTRACE uint32_t dtrace_get_thread_predcache(thread_t thread) { @@ -2954,6 +3113,14 @@ vm_offset_t dtrace_get_kernel_stack(thread_t thread) return 0; } +#if KASAN +struct kasan_thread_data * +kasan_get_thread_data(thread_t thread) +{ + return &thread->kasan_data; +} +#endif + int64_t dtrace_calc_thread_recent_vtime(thread_t thread) { if (thread != THREAD_NULL) { diff --git a/osfmk/kern/thread.h b/osfmk/kern/thread.h index 69e0ee2e9..02bdc0607 100644 --- a/osfmk/kern/thread.h +++ b/osfmk/kern/thread.h @@ -94,6 +94,7 @@ #include #include +#include #include @@ -115,6 +116,7 @@ #include #include #include +#include #include #include #include @@ -123,12 +125,22 @@ #include #include +#include #include #include #include +#if MONOTONIC +#include +#include +#endif /* MONOTONIC */ + +#if CONFIG_EMBEDDED +/* Taskwatch related. TODO: find this a better home */ +typedef struct task_watcher task_watch_t; +#endif /* CONFIG_EMBEDDED */ struct thread { @@ -177,6 +189,7 @@ struct thread { #define TH_OPT_SCHED_VM_GROUP 0x0200 /* Thread belongs to special scheduler VM group */ #define TH_OPT_HONOR_QLIMIT 0x0400 /* Thread will honor qlimit while sending mach_msg, regardless of MACH_SEND_ALWAYS */ #define TH_OPT_SEND_IMPORTANCE 0x0800 /* Thread will allow importance donation from kernel rpc */ +#define TH_OPT_ZONE_GC 0x1000 /* zone_gc() called on this thread */ boolean_t wake_active; /* wake event on stop */ int at_safe_point; /* thread_abort_safely allowed */ @@ -192,6 +205,10 @@ struct thread { vm_offset_t kernel_stack; /* current kernel stack */ vm_offset_t reserved_stack; /* reserved kernel stack */ +#if KASAN + struct kasan_thread_data kasan_data; +#endif + /* Thread state: */ int state; /* @@ -235,7 +252,10 @@ struct thread { #define TH_SFLAG_RW_PROMOTED 0x0400 /* sched pri has been promoted due to blocking with RW lock held */ /* unused TH_SFLAG_THROTTLE_DEMOTED 0x0800 */ #define TH_SFLAG_WAITQ_PROMOTED 0x1000 /* sched pri promoted from waitq wakeup (generally for IPC receive) */ -#define TH_SFLAG_PROMOTED_MASK (TH_SFLAG_PROMOTED | TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED) + + +#define TH_SFLAG_EXEC_PROMOTED 0x8000 /* sched pri has been promoted since thread is in an exec */ +#define TH_SFLAG_PROMOTED_MASK (TH_SFLAG_PROMOTED | TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED | TH_SFLAG_EXEC_PROMOTED) #define TH_SFLAG_RW_PROMOTED_BIT (10) /* 0x400 */ @@ -252,7 +272,7 @@ struct thread { int16_t promotions; /* level of promotion */ int16_t pending_promoter_index; - uint32_t ref_count; /* number of references to me */ + _Atomic uint32_t ref_count; /* number of references to me */ void *pending_promoter[2]; uint32_t rwlock_count; /* Number of lck_rw_t locks held by thread */ @@ -274,6 +294,10 @@ struct thread { uint64_t last_run_time; /* time when thread was switched away from */ uint64_t last_made_runnable_time; /* time when thread was unblocked or preempted */ + uint64_t last_basepri_change_time; /* time when thread was last changed in basepri while runnable */ + uint64_t same_pri_latency; +#define THREAD_NOT_RUNNABLE (~0ULL) + #if defined(CONFIG_SCHED_MULTIQ) sched_group_t sched_group; @@ -322,6 +346,8 @@ struct thread { uint64_t vtimer_rlim_save; uint64_t vtimer_qos_save; + timer_data_t ptime; /* time executing in P mode */ + #if CONFIG_SCHED_SFI /* Timing for wait state */ uint64_t wait_sfi_begin_time; /* start time for thread waiting in SFI */ @@ -340,7 +366,7 @@ struct thread { affinity_set_t affinity_set; queue_chain_t affinity_threads; - /* Various bits of stashed state */ + /* Various bits of state to stash across a continuation, exclusive to the current thread block point */ union { struct { mach_msg_return_t state; /* receive state */ @@ -351,6 +377,7 @@ struct thread { mach_msg_size_t msize; /* actual size for recvd msg */ mach_msg_option_t option; /* options for receive */ mach_port_name_t receiver_name; /* the receive port name */ + struct knote *knote; /* knote fired for rcv */ union { struct ipc_kmsg *kmsg; /* received message */ struct ipc_mqueue *peekq; /* mqueue to peek at */ @@ -372,15 +399,22 @@ struct thread { int option; /* switch option */ boolean_t reenable_workq_callback; /* on entry, callbacks were suspended */ } swtch; - int misc; /* catch-all for other state */ } saved; - /* Structure to save information about guard exception */ - struct { - unsigned type; /* EXC_GUARD reason/type */ - mach_exception_data_type_t code; /* Exception code */ - mach_exception_data_type_t subcode; /* Exception sub-code */ - } guard_exc_info; + /* Only user threads can cause guard exceptions, only kernel threads can be thread call threads */ + union { + /* Group and call this thread is working on behalf of */ + struct { + struct thread_call_group * thc_group; + struct thread_call * thc_call; /* debug only, may be deallocated */ + } thc_state; + + /* Structure to save information about guard exception */ + struct { + mach_exception_code_t code; + mach_exception_subcode_t subcode; + } guard_exc_info; + }; /* Kernel holds on this thread */ int16_t suspend_count; @@ -426,6 +460,7 @@ struct thread { /* Ports associated with this thread */ struct ipc_port *ith_self; /* not a right, doesn't hold ref */ struct ipc_port *ith_sself; /* a send right */ + struct ipc_port *ith_special_reply_port; /* ref to special reply port */ struct exception_action *exc_actions; #ifdef MACH_BSD @@ -484,10 +519,16 @@ struct thread { uint32_t syscalls_mach; ledger_t t_ledger; ledger_t t_threadledger; /* per thread ledger */ -#ifdef CONFIG_BANK - ledger_t t_bankledger; /* ledger to charge someone */ - uint64_t t_deduct_bank_ledger_time; /* cpu time to be deducted from bank ledger */ -#endif + ledger_t t_bankledger; /* ledger to charge someone */ + uint64_t t_deduct_bank_ledger_time; /* cpu time to be deducted from bank ledger */ + uint64_t t_deduct_bank_ledger_energy; /* energy to be deducted from bank ledger */ + +#if MONOTONIC + struct mt_thread t_monotonic; +#endif /* MONOTONIC */ + + /*** Machine-dependent state ***/ + struct machine_thread machine; /* policy is protected by the thread mutex */ struct thread_requested_policy requested_policy; @@ -502,9 +543,12 @@ struct thread { user_addr_t override_resource; } *overrides; + _Atomic uint32_t kqwl_owning_count; uint32_t ipc_overrides; + uint32_t sync_ipc_overrides; uint32_t user_promotions; uint16_t user_promotion_basepri; + _Atomic uint16_t kevent_ast_bits; block_hint_t pending_block_hint; block_hint_t block_hint; /* What type of primitive last caused us to block. */ @@ -512,6 +556,9 @@ struct thread { int iotier_override; /* atomic operations to set, cleared on ret to user */ io_stat_info_t thread_io_stats; /* per-thread I/O statistics */ +#if CONFIG_EMBEDDED + task_watch_t * taskwatch; /* task watch */ +#endif /* CONFIG_EMBEDDED */ uint32_t thread_callout_interrupt_wakeups; uint32_t thread_callout_platform_idle_wakeups; @@ -529,11 +576,8 @@ struct thread { void *decmp_upl; #endif /* CONFIG_IOSCHED */ - /* work interval ID (if any) associated with the thread. Uses thread mutex */ - uint64_t work_interval_id; - - /*** Machine-dependent state ***/ - struct machine_thread machine; + /* work interval (if any) associated with the thread. Uses thread mutex */ + struct work_interval *th_work_interval; #if SCHED_TRACE_THREAD_WAKEUPS uintptr_t thread_wakeup_bt[64]; @@ -550,6 +594,7 @@ struct thread { #define ith_continuation saved.receive.continuation #define ith_kmsg saved.receive.kmsg #define ith_peekq saved.receive.peekq +#define ith_knote saved.receive.knote #define ith_qos saved.receive.received_qos.qos #define ith_qos_override saved.receive.received_qos.oqos #define ith_seqno saved.receive.seqno @@ -560,6 +605,10 @@ struct thread { #define sth_result saved.sema.result #define sth_continuation saved.sema.continuation +#define ITH_KNOTE_NULL ((void *)NULL) +#define ITH_KNOTE_PSEUDO ((void *)0xdeadbeef) +#define ITH_KNOTE_VALID(kn) ((kn) != ITH_KNOTE_NULL && (kn) != ITH_KNOTE_PSEUDO) + #if MACH_ASSERT #define assert_thread_magic(thread) assertf((thread)->thread_magic == THREAD_MAGIC, \ "bad thread magic 0x%llx for thread %p, expected 0x%llx", \ @@ -575,7 +624,7 @@ extern void thread_init(void); extern void thread_daemon_init(void); #define thread_reference_internal(thread) \ - (void)hw_atomic_add(&(thread)->ref_count, 1) + (void)atomic_fetch_add_explicit(&(thread)->ref_count, 1, memory_order_relaxed) #define thread_reference(thread) \ MACRO_BEGIN \ @@ -610,7 +659,8 @@ extern void thread_terminate_enqueue( extern void thread_exception_enqueue( task_t task, - thread_t thread); + thread_t thread, + exception_type_t etype); extern void thread_copy_resource_info( thread_t dst_thread, @@ -710,7 +760,8 @@ extern thread_t machine_switch_context( thread_t new_thread); extern void machine_load_context( - thread_t thread); + thread_t thread) __attribute__((noreturn)); + extern kern_return_t machine_thread_state_initialize( thread_t thread); @@ -783,6 +834,7 @@ static inline uint16_t thread_get_tag_internal(thread_t thread) { extern void thread_set_options(uint32_t thopt); + #else /* MACH_KERNEL_PRIVATE */ __BEGIN_DECLS @@ -803,7 +855,20 @@ __END_DECLS __BEGIN_DECLS -extern uint64_t thread_dispatchqaddr( +extern void thread_starts_owning_workloop( + thread_t thread); + +extern void thread_ends_owning_workloop( + thread_t thread); + +extern uint32_t thread_owned_workloops_count( + thread_t thread); + + +extern uint64_t thread_dispatchqaddr( + thread_t thread); + +extern uint64_t thread_rettokern_addr( thread_t thread); __END_DECLS @@ -813,7 +878,7 @@ __END_DECLS #ifdef KERNEL __BEGIN_DECLS -extern uint64_t thread_tid(thread_t thread); +extern uint64_t thread_tid(thread_t thread); __END_DECLS @@ -955,9 +1020,12 @@ extern kern_return_t thread_dup(thread_t); extern kern_return_t thread_dup2(thread_t, thread_t); +#if !defined(_SCHED_CALL_T_DEFINED) +#define _SCHED_CALL_T_DEFINED typedef void (*sched_call_t)( int type, thread_t thread); +#endif #define SCHED_CALL_BLOCK 0x1 #define SCHED_CALL_UNBLOCK 0x2 @@ -1041,11 +1109,18 @@ extern vm_offset_t kernel_stack_mask; extern vm_offset_t kernel_stack_size; extern vm_offset_t kernel_stack_depth_max; -void guard_ast(thread_t thread); -extern void fd_guard_ast(thread_t thread); -extern void mach_port_guard_ast(thread_t thread); -extern void thread_guard_violation(thread_t thread, unsigned type); -extern void thread_update_io_stats(thread_t thread, int size, int io_flags); +extern void guard_ast(thread_t); +extern void fd_guard_ast(thread_t, + mach_exception_code_t, mach_exception_subcode_t); +#if CONFIG_VNGUARD +extern void vn_guard_ast(thread_t, + mach_exception_code_t, mach_exception_subcode_t); +#endif +extern void mach_port_guard_ast(thread_t, + mach_exception_code_t, mach_exception_subcode_t); +extern void thread_guard_violation(thread_t, + mach_exception_code_t, mach_exception_subcode_t); +extern void thread_update_io_stats(thread_t, int size, int io_flags); extern kern_return_t thread_set_voucher_name(mach_port_name_t name); extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid); @@ -1110,6 +1185,7 @@ extern kern_return_t kernel_thread_start( thread_continue_t continuation, void *parameter, thread_t *new_thread); + #ifdef KERNEL_PRIVATE void thread_set_eager_preempt(thread_t thread); void thread_clear_eager_preempt(thread_t thread); @@ -1117,6 +1193,7 @@ extern ipc_port_t convert_thread_to_port(thread_t); extern ipc_port_t convert_thread_inspect_to_port(thread_inspect_t); extern boolean_t is_vm_privileged(void); extern boolean_t set_vm_privilege(boolean_t); +extern kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name); #endif /* KERNEL_PRIVATE */ __END_DECLS diff --git a/osfmk/kern/thread_act.c b/osfmk/kern/thread_act.c index 1047d59af..6f16bfd1c 100644 --- a/osfmk/kern/thread_act.c +++ b/osfmk/kern/thread_act.c @@ -74,6 +74,8 @@ #include #include +#include + #include static void act_abort(thread_t thread); @@ -191,26 +193,27 @@ kern_return_t thread_terminate( thread_t thread) { - kern_return_t result; - if (thread == THREAD_NULL) return (KERN_INVALID_ARGUMENT); - if ( thread->task == kernel_task && - thread != current_thread() ) + /* Kernel threads can't be terminated without their own cooperation */ + if (thread->task == kernel_task && thread != current_thread()) return (KERN_FAILURE); - result = thread_terminate_internal(thread); + kern_return_t result = thread_terminate_internal(thread); /* - * If a kernel thread is terminating itself, force an AST here. - * Kernel threads don't normally pass through the AST checking - * code - and all threads finish their own termination in mach_apc_ast. + * If a kernel thread is terminating itself, force handle the APC_AST here. + * Kernel threads don't pass through the return-to-user AST checking code, + * but all threads must finish their own termination in thread_apc_ast. */ if (thread->task == kernel_task) { - ml_set_interrupts_enabled(FALSE); - ast_taken(AST_APC, TRUE); + assert(thread->active == FALSE); + thread_ast_clear(thread, AST_APC); + thread_apc_ast(thread); + panic("thread_terminate"); + /* NOTREACHED */ } return (result); @@ -836,7 +839,7 @@ thread_set_apc_ast_locked(thread_t thread) thread_ast_set(thread, AST_APC); if (thread == current_thread()) { - ast_propagate(thread->ast); + ast_propagate(thread); } else { processor_t processor = thread->last_processor; @@ -884,6 +887,7 @@ thread_suspended(__unused void *parameter, wait_result_t result) if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) { thread->sched_pri = DEPRESSPRI; thread->last_processor->current_pri = thread->sched_pri; + thread->last_processor->current_perfctl_class = thread_get_perfcontrol_class(thread); KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHANGE_PRIORITY), (uintptr_t)thread_tid(thread), @@ -1002,7 +1006,7 @@ act_set_ast( if (thread == current_thread()) { thread_ast_set(thread, ast); - ast_propagate(thread->ast); + ast_propagate(thread); } else { processor_t processor; @@ -1026,6 +1030,27 @@ act_set_astbsd( act_set_ast( thread, AST_BSD ); } +void +act_set_astkevent(thread_t thread, uint16_t bits) +{ + spl_t s = splsched(); + + /* + * Do not send an IPI if the thread is running on + * another processor, wait for the next quantum + * expiration to load the AST. + */ + + atomic_fetch_or(&thread->kevent_ast_bits, bits); + thread_ast_set(thread, AST_KEVENT); + + if (thread == current_thread()) { + ast_propagate(thread); + } + + splx(s); +} + void act_set_kperf( thread_t thread) diff --git a/osfmk/kern/thread_call.c b/osfmk/kern/thread_call.c index 53f406b1d..e403f17dc 100644 --- a/osfmk/kern/thread_call.c +++ b/osfmk/kern/thread_call.c @@ -37,6 +37,7 @@ #include #include #include +#include #include @@ -56,52 +57,96 @@ static zone_t thread_call_zone; static struct waitq daemon_waitq; -struct thread_call_group { +typedef enum { + TCF_ABSOLUTE = 0, + TCF_CONTINUOUS = 1, + TCF_COUNT = 2, +} thread_call_flavor_t; + +typedef enum { + TCG_NONE = 0x0, + TCG_PARALLEL = 0x1, + TCG_DEALLOC_ACTIVE = 0x2, +} thread_call_group_flags_t; + +static struct thread_call_group { + const char * tcg_name; + queue_head_t pending_queue; uint32_t pending_count; - queue_head_t delayed_queue; - uint32_t delayed_count; + queue_head_t delayed_queues[TCF_COUNT]; + timer_call_data_t delayed_timers[TCF_COUNT]; - timer_call_data_t delayed_timer; timer_call_data_t dealloc_timer; struct waitq idle_waitq; - uint32_t idle_count, active_count; + uint32_t idle_count, active_count, blocked_count; - integer_t pri; + uint32_t tcg_thread_pri; uint32_t target_thread_count; uint64_t idle_timestamp; - uint32_t flags; - sched_call_t sched_call; + thread_call_group_flags_t flags; + +} thread_call_groups[THREAD_CALL_INDEX_MAX] = { + [THREAD_CALL_INDEX_HIGH] = { + .tcg_name = "high", + .tcg_thread_pri = BASEPRI_PREEMPT_HIGH, + .target_thread_count = 4, + .flags = TCG_NONE, + }, + [THREAD_CALL_INDEX_KERNEL] = { + .tcg_name = "kernel", + .tcg_thread_pri = BASEPRI_KERNEL, + .target_thread_count = 1, + .flags = TCG_PARALLEL, + }, + [THREAD_CALL_INDEX_USER] = { + .tcg_name = "user", + .tcg_thread_pri = BASEPRI_DEFAULT, + .target_thread_count = 1, + .flags = TCG_PARALLEL, + }, + [THREAD_CALL_INDEX_LOW] = { + .tcg_name = "low", + .tcg_thread_pri = MAXPRI_THROTTLE, + .target_thread_count = 1, + .flags = TCG_PARALLEL, + }, + [THREAD_CALL_INDEX_KERNEL_HIGH] = { + .tcg_name = "kernel-high", + .tcg_thread_pri = BASEPRI_PREEMPT, + .target_thread_count = 2, + .flags = TCG_NONE, + }, + [THREAD_CALL_INDEX_QOS_UI] = { + .tcg_name = "qos-ui", + .tcg_thread_pri = BASEPRI_FOREGROUND, + .target_thread_count = 1, + .flags = TCG_NONE, + }, + [THREAD_CALL_INDEX_QOS_IN] = { + .tcg_name = "qos-in", + .tcg_thread_pri = BASEPRI_USER_INITIATED, + .target_thread_count = 1, + .flags = TCG_NONE, + }, + [THREAD_CALL_INDEX_QOS_UT] = { + .tcg_name = "qos-ut", + .tcg_thread_pri = BASEPRI_UTILITY, + .target_thread_count = 1, + .flags = TCG_NONE, + }, }; typedef struct thread_call_group *thread_call_group_t; -#define TCG_PARALLEL 0x01 -#define TCG_DEALLOC_ACTIVE 0x02 -#define TCG_CONTINUOUS 0x04 - -#define THREAD_CALL_PRIO_COUNT 4 -#define THREAD_CALL_ABSTIME_COUNT 4 -#define THREAD_CALL_CONTTIME_COUNT 4 -#define THREAD_CALL_GROUP_COUNT (THREAD_CALL_CONTTIME_COUNT + THREAD_CALL_ABSTIME_COUNT) -#define THREAD_CALL_THREAD_MIN 4 #define INTERNAL_CALL_COUNT 768 -#define THREAD_CALL_DEALLOC_INTERVAL_NS (5 * 1000 * 1000) /* 5 ms */ +#define THREAD_CALL_DEALLOC_INTERVAL_NS (5 * NSEC_PER_MSEC) /* 5 ms */ #define THREAD_CALL_ADD_RATIO 4 #define THREAD_CALL_MACH_FACTOR_CAP 3 - -#define IS_CONT_GROUP(group) \ - (((group)->flags & TCG_CONTINUOUS) ? TRUE : FALSE) - -// groups [0..4]: thread calls in mach_absolute_time -// groups [4..8]: thread calls in mach_continuous_time -static struct thread_call_group thread_call_groups[THREAD_CALL_GROUP_COUNT]; - -static struct thread_call_group *abstime_thread_call_groups; -static struct thread_call_group *conttime_thread_call_groups; +#define THREAD_CALL_GROUP_MAX_THREADS 500 static boolean_t thread_call_daemon_awake; static thread_call_data_t internal_call_storage[INTERNAL_CALL_COUNT]; @@ -112,36 +157,29 @@ static uint64_t thread_call_dealloc_interval_abs; static __inline__ thread_call_t _internal_call_allocate(thread_call_func_t func, thread_call_param_t param0); static __inline__ void _internal_call_release(thread_call_t call); static __inline__ boolean_t _pending_call_enqueue(thread_call_t call, thread_call_group_t group); -static __inline__ boolean_t _delayed_call_enqueue(thread_call_t call, thread_call_group_t group, uint64_t deadline); +static boolean_t _delayed_call_enqueue(thread_call_t call, thread_call_group_t group, + uint64_t deadline, thread_call_flavor_t flavor); static __inline__ boolean_t _call_dequeue(thread_call_t call, thread_call_group_t group); static __inline__ void thread_call_wake(thread_call_group_t group); -static __inline__ void _set_delayed_call_timer(thread_call_t call, thread_call_group_t group); -static boolean_t _remove_from_pending_queue(thread_call_func_t func, thread_call_param_t param0, boolean_t remove_all); -static boolean_t _remove_from_delayed_queue(thread_call_func_t func, thread_call_param_t param0, boolean_t remove_all); static void thread_call_daemon(void *arg); static void thread_call_thread(thread_call_group_t group, wait_result_t wres); -extern void thread_call_delayed_timer(timer_call_param_t p0, timer_call_param_t p1); static void thread_call_dealloc_timer(timer_call_param_t p0, timer_call_param_t p1); -static void thread_call_group_setup(thread_call_group_t group, thread_call_priority_t pri, uint32_t target_thread_count, boolean_t parallel, boolean_t continuous); +static void thread_call_group_setup(thread_call_group_t group); static void sched_call_thread(int type, thread_t thread); static void thread_call_start_deallocate_timer(thread_call_group_t group); -static void thread_call_wait_locked(thread_call_t call); +static void thread_call_wait_locked(thread_call_t call, spl_t s); +static boolean_t thread_call_wait_once_locked(thread_call_t call, spl_t s); + static boolean_t thread_call_enter_delayed_internal(thread_call_t call, thread_call_func_t alt_func, thread_call_param_t alt_param0, thread_call_param_t param1, uint64_t deadline, uint64_t leeway, unsigned int flags); -#define qe(x) ((queue_entry_t)(x)) -#define TC(x) ((thread_call_t)(x)) - +/* non-static so dtrace can find it rdar://problem/31156135&31379348 */ +extern void thread_call_delayed_timer(timer_call_param_t p0, timer_call_param_t p1); -lck_grp_t thread_call_queues_lck_grp; lck_grp_t thread_call_lck_grp; -lck_attr_t thread_call_lck_attr; -lck_grp_attr_t thread_call_lck_grp_attr; - -lck_mtx_t thread_call_lock_data; - +lck_mtx_t thread_call_lock_data; #define thread_call_lock_spin() \ lck_mtx_lock_spin_always(&thread_call_lock_data) @@ -149,27 +187,26 @@ lck_mtx_t thread_call_lock_data; #define thread_call_unlock() \ lck_mtx_unlock_always(&thread_call_lock_data) +#define tc_deadline tc_call.deadline + extern boolean_t mach_timer_coalescing_enabled; static inline spl_t disable_ints_and_lock(void) { - spl_t s; - - s = splsched(); + spl_t s = splsched(); thread_call_lock_spin(); return s; } -static inline void +static inline void enable_ints_and_unlock(spl_t s) { thread_call_unlock(); splx(s); } - static inline boolean_t group_isparallel(thread_call_group_t group) { @@ -177,11 +214,15 @@ group_isparallel(thread_call_group_t group) } static boolean_t -thread_call_group_should_add_thread(thread_call_group_t group) +thread_call_group_should_add_thread(thread_call_group_t group) { - uint32_t thread_count; + if ((group->active_count + group->blocked_count + group->idle_count) >= THREAD_CALL_GROUP_MAX_THREADS) { + panic("thread_call group '%s' reached max thread cap (%d): active: %d, blocked: %d, idle: %d", + group->tcg_name, THREAD_CALL_GROUP_MAX_THREADS, + group->active_count, group->blocked_count, group->idle_count); + } - if (!group_isparallel(group)) { + if (group_isparallel(group) == FALSE) { if (group->pending_count > 0 && group->active_count == 0) { return TRUE; } @@ -194,7 +235,7 @@ thread_call_group_should_add_thread(thread_call_group_t group) return FALSE; } - thread_count = group->active_count; + uint32_t thread_count = group->active_count; /* * Add a thread if either there are no threads, @@ -202,7 +243,7 @@ thread_call_group_should_add_thread(thread_call_group_t group) * threads, or the amount of work is large relative * to the number of threads. In the last case, pay attention * to the total load on the system, and back off if - * it's high. + * it's high. */ if ((thread_count == 0) || (thread_count < group->target_thread_count) || @@ -211,81 +252,42 @@ thread_call_group_should_add_thread(thread_call_group_t group) return TRUE; } } - - return FALSE; -} -static inline integer_t -thread_call_priority_to_sched_pri(thread_call_priority_t pri) -{ - switch (pri) { - case THREAD_CALL_PRIORITY_HIGH: - return BASEPRI_PREEMPT; - case THREAD_CALL_PRIORITY_KERNEL: - return BASEPRI_KERNEL; - case THREAD_CALL_PRIORITY_USER: - return BASEPRI_DEFAULT; - case THREAD_CALL_PRIORITY_LOW: - return MAXPRI_THROTTLE; - default: - panic("Invalid priority."); - } - - return 0; + return FALSE; } /* Lock held */ static inline thread_call_group_t -thread_call_get_group( - thread_call_t call) +thread_call_get_group(thread_call_t call) { - thread_call_priority_t pri = call->tc_pri; - - assert(pri == THREAD_CALL_PRIORITY_LOW || - pri == THREAD_CALL_PRIORITY_USER || - pri == THREAD_CALL_PRIORITY_KERNEL || - pri == THREAD_CALL_PRIORITY_HIGH); + thread_call_index_t index = call->tc_index; - thread_call_group_t group; + assert(index >= 0 && index < THREAD_CALL_INDEX_MAX); - if(call->tc_flags & THREAD_CALL_CONTINUOUS) { - group = &conttime_thread_call_groups[pri]; - } else { - group = &abstime_thread_call_groups[pri]; - } + return &thread_call_groups[index]; +} - assert(IS_CONT_GROUP(group) == ((call->tc_flags & THREAD_CALL_CONTINUOUS) ? TRUE : FALSE)); - return group; +/* Lock held */ +static inline thread_call_flavor_t +thread_call_get_flavor(thread_call_t call) +{ + return (call->tc_flags & THREAD_CALL_CONTINUOUS) ? TCF_CONTINUOUS : TCF_ABSOLUTE; } static void -thread_call_group_setup( - thread_call_group_t group, - thread_call_priority_t pri, - uint32_t target_thread_count, - boolean_t parallel, - boolean_t continuous) +thread_call_group_setup(thread_call_group_t group) { queue_init(&group->pending_queue); - queue_init(&group->delayed_queue); + queue_init(&group->delayed_queues[TCF_ABSOLUTE]); + queue_init(&group->delayed_queues[TCF_CONTINUOUS]); - timer_call_setup(&group->delayed_timer, thread_call_delayed_timer, group); + /* TODO: Consolidate to one hard timer for each group */ + timer_call_setup(&group->delayed_timers[TCF_ABSOLUTE], thread_call_delayed_timer, group); + timer_call_setup(&group->delayed_timers[TCF_CONTINUOUS], thread_call_delayed_timer, group); timer_call_setup(&group->dealloc_timer, thread_call_dealloc_timer, group); - waitq_init(&group->idle_waitq, SYNC_POLICY_FIFO|SYNC_POLICY_DISABLE_IRQ); - - group->target_thread_count = target_thread_count; - group->pri = thread_call_priority_to_sched_pri(pri); - - group->sched_call = sched_call_thread; - if (parallel) { - group->flags |= TCG_PARALLEL; - group->sched_call = NULL; - } - - if(continuous) { - group->flags |= TCG_CONTINUOUS; - } + /* Reverse the wait order so we re-use the most recently parked thread from the pool */ + waitq_init(&group->idle_waitq, SYNC_POLICY_REVERSED|SYNC_POLICY_DISABLE_IRQ); } /* @@ -299,20 +301,29 @@ thread_call_thread_create( thread_t thread; kern_return_t result; - result = kernel_thread_start_priority((thread_continue_t)thread_call_thread, group, group->pri, &thread); + int thread_pri = group->tcg_thread_pri; + + result = kernel_thread_start_priority((thread_continue_t)thread_call_thread, + group, thread_pri, &thread); if (result != KERN_SUCCESS) { return result; } - if (group->pri < BASEPRI_PREEMPT) { + if (thread_pri <= BASEPRI_KERNEL) { /* - * New style doesn't get to run to completion in - * kernel if there are higher priority threads - * available. + * THREAD_CALL_PRIORITY_KERNEL and lower don't get to run to completion + * in kernel if there are higher priority threads available. */ thread_set_eager_preempt(thread); } + char name[MAXTHREADNAMESIZE] = ""; + + int group_thread_count = group->idle_count + group->active_count + group->blocked_count; + + snprintf(name, sizeof(name), "thread call %s #%d", group->tcg_name, group_thread_count); + thread_set_thread_name(thread, name); + thread_deallocate(thread); return KERN_SUCCESS; } @@ -326,46 +337,29 @@ thread_call_thread_create( void thread_call_initialize(void) { - thread_call_t call; - kern_return_t result; - thread_t thread; - int i; - spl_t s; - - i = sizeof (thread_call_data_t); - thread_call_zone = zinit(i, 4096 * i, 16 * i, "thread_call"); + int tc_size = sizeof (thread_call_data_t); + thread_call_zone = zinit(tc_size, 4096 * tc_size, 16 * tc_size, "thread_call"); zone_change(thread_call_zone, Z_CALLERACCT, FALSE); zone_change(thread_call_zone, Z_NOENCRYPT, TRUE); - abstime_thread_call_groups = &thread_call_groups[0]; - conttime_thread_call_groups = &thread_call_groups[THREAD_CALL_ABSTIME_COUNT]; + lck_grp_init(&thread_call_lck_grp, "thread_call", LCK_GRP_ATTR_NULL); + lck_mtx_init(&thread_call_lock_data, &thread_call_lck_grp, LCK_ATTR_NULL); - lck_attr_setdefault(&thread_call_lck_attr); - lck_grp_attr_setdefault(&thread_call_lck_grp_attr); - lck_grp_init(&thread_call_queues_lck_grp, "thread_call_queues", &thread_call_lck_grp_attr); - lck_grp_init(&thread_call_lck_grp, "thread_call", &thread_call_lck_grp_attr); - lck_mtx_init(&thread_call_lock_data, &thread_call_lck_grp, &thread_call_lck_attr); nanotime_to_absolutetime(0, THREAD_CALL_DEALLOC_INTERVAL_NS, &thread_call_dealloc_interval_abs); waitq_init(&daemon_waitq, SYNC_POLICY_DISABLE_IRQ | SYNC_POLICY_FIFO); - thread_call_group_setup(&abstime_thread_call_groups[THREAD_CALL_PRIORITY_LOW], THREAD_CALL_PRIORITY_LOW, 0, TRUE, FALSE); - thread_call_group_setup(&abstime_thread_call_groups[THREAD_CALL_PRIORITY_USER], THREAD_CALL_PRIORITY_USER, 0, TRUE, FALSE); - thread_call_group_setup(&abstime_thread_call_groups[THREAD_CALL_PRIORITY_KERNEL], THREAD_CALL_PRIORITY_KERNEL, 1, TRUE, FALSE); - thread_call_group_setup(&abstime_thread_call_groups[THREAD_CALL_PRIORITY_HIGH], THREAD_CALL_PRIORITY_HIGH, THREAD_CALL_THREAD_MIN, FALSE, FALSE); - thread_call_group_setup(&conttime_thread_call_groups[THREAD_CALL_PRIORITY_LOW], THREAD_CALL_PRIORITY_LOW, 0, TRUE, TRUE); - thread_call_group_setup(&conttime_thread_call_groups[THREAD_CALL_PRIORITY_USER], THREAD_CALL_PRIORITY_USER, 0, TRUE, TRUE); - thread_call_group_setup(&conttime_thread_call_groups[THREAD_CALL_PRIORITY_KERNEL], THREAD_CALL_PRIORITY_KERNEL, 0, TRUE, TRUE); - thread_call_group_setup(&conttime_thread_call_groups[THREAD_CALL_PRIORITY_HIGH], THREAD_CALL_PRIORITY_HIGH, 1, FALSE, TRUE); + for (uint32_t i = 0; i < THREAD_CALL_INDEX_MAX; i++) + thread_call_group_setup(&thread_call_groups[i]); - s = disable_ints_and_lock(); + spl_t s = disable_ints_and_lock(); queue_init(&thread_call_internal_queue); for ( - call = internal_call_storage; + thread_call_t call = internal_call_storage; call < &internal_call_storage[INTERNAL_CALL_COUNT]; call++) { - enqueue_tail(&thread_call_internal_queue, qe(call)); + enqueue_tail(&thread_call_internal_queue, &call->tc_call.q_link); thread_call_internal_queue_count++; } @@ -373,7 +367,11 @@ thread_call_initialize(void) enable_ints_and_unlock(s); - result = kernel_thread_start_priority((thread_continue_t)thread_call_daemon, NULL, BASEPRI_PREEMPT + 1, &thread); + thread_t thread; + kern_return_t result; + + result = kernel_thread_start_priority((thread_continue_t)thread_call_daemon, + NULL, BASEPRI_PREEMPT_HIGH + 1, &thread); if (result != KERN_SUCCESS) panic("thread_call_initialize"); @@ -388,7 +386,11 @@ thread_call_setup( { bzero(call, sizeof(*call)); call_entry_setup((call_entry_t)call, func, param0); - call->tc_pri = THREAD_CALL_PRIORITY_HIGH; /* Default priority */ + + /* Thread calls default to the HIGH group unless otherwise specified */ + call->tc_index = THREAD_CALL_INDEX_HIGH; + + /* THREAD_CALL_ALLOC not set, memory owned by caller */ } /* @@ -405,8 +407,9 @@ _internal_call_allocate(thread_call_func_t func, thread_call_param_t param0) if (queue_empty(&thread_call_internal_queue)) panic("_internal_call_allocate"); - - call = TC(dequeue_head(&thread_call_internal_queue)); + + call = qe_dequeue_head(&thread_call_internal_queue, struct thread_call, tc_call.q_link); + thread_call_internal_queue_count--; thread_call_setup(call, func, param0); @@ -427,13 +430,12 @@ _internal_call_allocate(thread_call_func_t func, thread_call_param_t param0) * Called with thread_call_lock held. */ static __inline__ void -_internal_call_release( - thread_call_t call) +_internal_call_release(thread_call_t call) { - if ( call >= internal_call_storage && - call < &internal_call_storage[INTERNAL_CALL_COUNT] ) { + if (call >= internal_call_storage && + call < &internal_call_storage[INTERNAL_CALL_COUNT]) { assert((call->tc_flags & THREAD_CALL_ALLOC) == 0); - enqueue_head(&thread_call_internal_queue, qe(call)); + enqueue_head(&thread_call_internal_queue, &call->tc_call.q_link); thread_call_internal_queue_count++; } } @@ -450,18 +452,29 @@ _internal_call_release( * Called with thread_call_lock held. */ static __inline__ boolean_t -_pending_call_enqueue( - thread_call_t call, - thread_call_group_t group) +_pending_call_enqueue(thread_call_t call, + thread_call_group_t group) { - queue_head_t *old_queue; + if ((THREAD_CALL_ONCE | THREAD_CALL_RUNNING) + == (call->tc_flags & (THREAD_CALL_ONCE | THREAD_CALL_RUNNING))) { + call->tc_deadline = 0; + + uint32_t flags = call->tc_flags; + call->tc_flags |= THREAD_CALL_RESCHEDULE; + + if ((flags & THREAD_CALL_RESCHEDULE) != 0) + return (TRUE); + else + return (FALSE); + } - old_queue = call_entry_enqueue_tail(CE(call), &group->pending_queue); + queue_head_t *old_queue = call_entry_enqueue_tail(CE(call), &group->pending_queue); if (old_queue == NULL) { call->tc_submit_count++; } else if (old_queue != &group->pending_queue && - old_queue != &group->delayed_queue){ + old_queue != &group->delayed_queues[TCF_ABSOLUTE] && + old_queue != &group->delayed_queues[TCF_CONTINUOUS]) { panic("tried to move a thread call (%p) between groups (old_queue: %p)", call, old_queue); } @@ -484,21 +497,37 @@ _pending_call_enqueue( * * Called with thread_call_lock held. */ -static __inline__ boolean_t +static boolean_t _delayed_call_enqueue( - thread_call_t call, - thread_call_group_t group, - uint64_t deadline) + thread_call_t call, + thread_call_group_t group, + uint64_t deadline, + thread_call_flavor_t flavor) { - queue_head_t *old_queue; + if ((THREAD_CALL_ONCE | THREAD_CALL_RUNNING) + == (call->tc_flags & (THREAD_CALL_ONCE | THREAD_CALL_RUNNING))) { + call->tc_deadline = deadline; - old_queue = call_entry_enqueue_deadline(CE(call), &group->delayed_queue, deadline); + uint32_t flags = call->tc_flags; + call->tc_flags |= THREAD_CALL_RESCHEDULE; + + if ((flags & THREAD_CALL_RESCHEDULE) != 0) + return (TRUE); + else + return (FALSE); + } + + queue_head_t *old_queue = call_entry_enqueue_deadline(CE(call), + &group->delayed_queues[flavor], + deadline); if (old_queue == &group->pending_queue) { group->pending_count--; } else if (old_queue == NULL) { call->tc_submit_count++; - } else if (old_queue == &group->delayed_queue) { + } else if (old_queue == &group->delayed_queues[TCF_ABSOLUTE] || + old_queue == &group->delayed_queues[TCF_CONTINUOUS]) { + /* TODO: if it's in the other delayed queue, that might not be OK */ // we did nothing, and that's fine } else { panic("tried to move a thread call (%p) between groups (old_queue: %p)", call, old_queue); @@ -526,6 +555,10 @@ _call_dequeue( old_queue = call_entry_dequeue(CE(call)); if (old_queue != NULL) { + assert(old_queue == &group->pending_queue || + old_queue == &group->delayed_queues[TCF_ABSOLUTE] || + old_queue == &group->delayed_queues[TCF_CONTINUOUS]); + call->tc_finish_count++; if (old_queue == &group->pending_queue) group->pending_count--; @@ -535,121 +568,95 @@ _call_dequeue( } /* - * _set_delayed_call_timer: + * _arm_delayed_call_timer: * - * Reset the timer so that it - * next expires when the entry is due. + * Check if the timer needs to be armed for this flavor, + * and if so, arm it. * - * Called with thread_call_lock held. + * If call is non-NULL, only re-arm the timer if the specified call + * is the first in the queue. + * + * Returns true if the timer was armed/re-armed, false if it was left unset + * Caller should cancel the timer if need be. + * + * Called with thread_call_lock held. */ -static __inline__ void -_set_delayed_call_timer( - thread_call_t call, - thread_call_group_t group) +static bool +_arm_delayed_call_timer(thread_call_t new_call, + thread_call_group_t group, + thread_call_flavor_t flavor) { - uint64_t leeway, fire_at; + /* No calls implies no timer needed */ + if (queue_empty(&group->delayed_queues[flavor])) + return false; + + thread_call_t call = qe_queue_first(&group->delayed_queues[flavor], struct thread_call, tc_call.q_link); + + /* We only need to change the hard timer if this new call is the first in the list */ + if (new_call != NULL && new_call != call) + return false; assert((call->tc_soft_deadline != 0) && ((call->tc_soft_deadline <= call->tc_call.deadline))); - assert(IS_CONT_GROUP(group) == ((call->tc_flags & THREAD_CALL_CONTINUOUS) ? TRUE : FALSE)); - fire_at = call->tc_soft_deadline; + uint64_t fire_at = call->tc_soft_deadline; - if (IS_CONT_GROUP(group)) { + if (flavor == TCF_CONTINUOUS) { + assert((call->tc_flags & THREAD_CALL_CONTINUOUS) == THREAD_CALL_CONTINUOUS); fire_at = continuoustime_to_absolutetime(fire_at); + } else { + assert((call->tc_flags & THREAD_CALL_CONTINUOUS) == 0); } - leeway = call->tc_call.deadline - call->tc_soft_deadline; - timer_call_enter_with_leeway(&group->delayed_timer, NULL, + /* + * Note: This picks the soonest-deadline call's leeway as the hard timer's leeway, + * which does not take into account later-deadline timers with a larger leeway. + * This is a valid coalescing behavior, but masks a possible window to + * fire a timer instead of going idle. + */ + uint64_t leeway = call->tc_call.deadline - call->tc_soft_deadline; + + timer_call_enter_with_leeway(&group->delayed_timers[flavor], (timer_call_param_t)flavor, fire_at, leeway, TIMER_CALL_SYS_CRITICAL|TIMER_CALL_LEEWAY, ((call->tc_flags & THREAD_CALL_RATELIMITED) == THREAD_CALL_RATELIMITED)); -} - -/* - * _remove_from_pending_queue: - * - * Remove the first (or all) matching - * entries from the pending queue. - * - * Returns TRUE if any matching entries - * were found. - * - * Called with thread_call_lock held. - */ -static boolean_t -_remove_from_pending_queue( - thread_call_func_t func, - thread_call_param_t param0, - boolean_t remove_all) -{ - boolean_t call_removed = FALSE; - thread_call_t call; - thread_call_group_t group = &abstime_thread_call_groups[THREAD_CALL_PRIORITY_HIGH]; - - call = TC(queue_first(&group->pending_queue)); - - while (!queue_end(&group->pending_queue, qe(call))) { - if (call->tc_call.func == func && - call->tc_call.param0 == param0) { - thread_call_t next = TC(queue_next(qe(call))); - _call_dequeue(call, group); - - _internal_call_release(call); - - call_removed = TRUE; - if (!remove_all) - break; - - call = next; - } - else - call = TC(queue_next(qe(call))); - } - - return (call_removed); + return true; } /* - * _remove_from_delayed_queue: + * _cancel_func_from_queue: * * Remove the first (or all) matching - * entries from the delayed queue. + * entries from the specified queue. * - * Returns TRUE if any matching entries + * Returns TRUE if any matching entries * were found. * * Called with thread_call_lock held. */ static boolean_t -_remove_from_delayed_queue( - thread_call_func_t func, - thread_call_param_t param0, - boolean_t remove_all) +_cancel_func_from_queue(thread_call_func_t func, + thread_call_param_t param0, + thread_call_group_t group, + boolean_t remove_all, + queue_head_t *queue) { - boolean_t call_removed = FALSE; - thread_call_t call; - thread_call_group_t group = &abstime_thread_call_groups[THREAD_CALL_PRIORITY_HIGH]; - - call = TC(queue_first(&group->delayed_queue)); - - while (!queue_end(&group->delayed_queue, qe(call))) { - if (call->tc_call.func == func && - call->tc_call.param0 == param0) { - thread_call_t next = TC(queue_next(qe(call))); + boolean_t call_removed = FALSE; + thread_call_t call; - _call_dequeue(call, group); + qe_foreach_element_safe(call, queue, tc_call.q_link) { + if (call->tc_call.func != func || + call->tc_call.param0 != param0) { + continue; + } - _internal_call_release(call); + _call_dequeue(call, group); - call_removed = TRUE; - if (!remove_all) - break; + _internal_call_release(call); - call = next; - } - else - call = TC(queue_next(qe(call))); + call_removed = TRUE; + if (!remove_all) + break; } return (call_removed); @@ -699,6 +706,9 @@ thread_call_func_delayed_with_leeway( * in that order. * * Returns TRUE if any calls were cancelled. + * + * This iterates all of the pending or delayed thread calls in the group, + * which is really inefficient. Switch to an allocated thread call instead. */ boolean_t thread_call_func_cancel( @@ -707,31 +717,36 @@ thread_call_func_cancel( boolean_t cancel_all) { boolean_t result; - spl_t s; assert(func != NULL); - s = splsched(); - thread_call_lock_spin(); + spl_t s = disable_ints_and_lock(); - if (cancel_all) - result = _remove_from_pending_queue(func, param, cancel_all) | - _remove_from_delayed_queue(func, param, cancel_all); - else - result = _remove_from_pending_queue(func, param, cancel_all) || - _remove_from_delayed_queue(func, param, cancel_all); + /* Function-only thread calls are only kept in the default HIGH group */ + thread_call_group_t group = &thread_call_groups[THREAD_CALL_INDEX_HIGH]; - thread_call_unlock(); - splx(s); + if (cancel_all) { + /* exhaustively search every queue, and return true if any search found something */ + result = _cancel_func_from_queue(func, param, group, cancel_all, &group->pending_queue) | + _cancel_func_from_queue(func, param, group, cancel_all, &group->delayed_queues[TCF_ABSOLUTE]) | + _cancel_func_from_queue(func, param, group, cancel_all, &group->delayed_queues[TCF_CONTINUOUS]); + } else { + /* early-exit as soon as we find something, don't search other queues */ + result = _cancel_func_from_queue(func, param, group, cancel_all, &group->pending_queue) || + _cancel_func_from_queue(func, param, group, cancel_all, &group->delayed_queues[TCF_ABSOLUTE]) || + _cancel_func_from_queue(func, param, group, cancel_all, &group->delayed_queues[TCF_CONTINUOUS]); + } + + enable_ints_and_unlock(s); return (result); } /* - * Allocate a thread call with a given priority. Importances - * other than THREAD_CALL_PRIORITY_HIGH will be run in threads - * with eager preemption enabled (i.e. may be aggressively preempted - * by higher-priority threads which are not in the normal "urgent" bands). + * Allocate a thread call with a given priority. Importances other than + * THREAD_CALL_PRIORITY_HIGH or THREAD_CALL_PRIORITY_KERNEL_HIGH will be run in threads + * with eager preemption enabled (i.e. may be aggressively preempted by higher-priority + * threads which are not in the normal "urgent" bands). */ thread_call_t thread_call_allocate_with_priority( @@ -739,18 +754,91 @@ thread_call_allocate_with_priority( thread_call_param_t param0, thread_call_priority_t pri) { - thread_call_t call; + return thread_call_allocate_with_options(func, param0, pri, 0); +} - if (pri > THREAD_CALL_PRIORITY_LOW) { - panic("Invalid pri: %d\n", pri); +thread_call_t +thread_call_allocate_with_options( + thread_call_func_t func, + thread_call_param_t param0, + thread_call_priority_t pri, + thread_call_options_t options) +{ + thread_call_t call = thread_call_allocate(func, param0); + + switch (pri) { + case THREAD_CALL_PRIORITY_HIGH: + call->tc_index = THREAD_CALL_INDEX_HIGH; + break; + case THREAD_CALL_PRIORITY_KERNEL: + call->tc_index = THREAD_CALL_INDEX_KERNEL; + break; + case THREAD_CALL_PRIORITY_USER: + call->tc_index = THREAD_CALL_INDEX_USER; + break; + case THREAD_CALL_PRIORITY_LOW: + call->tc_index = THREAD_CALL_INDEX_LOW; + break; + case THREAD_CALL_PRIORITY_KERNEL_HIGH: + call->tc_index = THREAD_CALL_INDEX_KERNEL_HIGH; + break; + default: + panic("Invalid thread call pri value: %d", pri); + break; } - call = thread_call_allocate(func, param0); - call->tc_pri = pri; + if (options & THREAD_CALL_OPTIONS_ONCE) { + call->tc_flags |= THREAD_CALL_ONCE; + } + if (options & THREAD_CALL_OPTIONS_SIGNAL) { + call->tc_flags |= THREAD_CALL_SIGNAL | THREAD_CALL_ONCE; + } return call; } +thread_call_t +thread_call_allocate_with_qos(thread_call_func_t func, + thread_call_param_t param0, + int qos_tier, + thread_call_options_t options) +{ + thread_call_t call = thread_call_allocate(func, param0); + + switch (qos_tier) { + case THREAD_QOS_UNSPECIFIED: + call->tc_index = THREAD_CALL_INDEX_HIGH; + break; + case THREAD_QOS_LEGACY: + call->tc_index = THREAD_CALL_INDEX_USER; + break; + case THREAD_QOS_MAINTENANCE: + case THREAD_QOS_BACKGROUND: + call->tc_index = THREAD_CALL_INDEX_LOW; + break; + case THREAD_QOS_UTILITY: + call->tc_index = THREAD_CALL_INDEX_QOS_UT; + break; + case THREAD_QOS_USER_INITIATED: + call->tc_index = THREAD_CALL_INDEX_QOS_IN; + break; + case THREAD_QOS_USER_INTERACTIVE: + call->tc_index = THREAD_CALL_INDEX_QOS_UI; + break; + default: + panic("Invalid thread call qos value: %d", qos_tier); + break; + } + + if (options & THREAD_CALL_OPTIONS_ONCE) + call->tc_flags |= THREAD_CALL_ONCE; + + /* does not support THREAD_CALL_OPTIONS_SIGNAL */ + + return call; +} + + /* * thread_call_allocate: * @@ -776,33 +864,40 @@ thread_call_allocate( * Release a callout. If the callout is currently * executing, it will be freed when all invocations * finish. + * + * If the callout is currently armed to fire again, then + * freeing is not allowed and returns FALSE. The + * client must have canceled the pending invocation before freeing. */ boolean_t thread_call_free( thread_call_t call) { - spl_t s; - int32_t refs; + spl_t s = disable_ints_and_lock(); - s = splsched(); - thread_call_lock_spin(); - - if (call->tc_call.queue != NULL) { + if (call->tc_call.queue != NULL || + ((call->tc_flags & THREAD_CALL_RESCHEDULE) != 0)) { thread_call_unlock(); splx(s); return (FALSE); } - refs = --call->tc_refs; + int32_t refs = --call->tc_refs; if (refs < 0) { panic("Refcount negative: %d\n", refs); - } + } - thread_call_unlock(); - splx(s); + if ((THREAD_CALL_SIGNAL | THREAD_CALL_RUNNING) + == ((THREAD_CALL_SIGNAL | THREAD_CALL_RUNNING) & call->tc_flags)) { + thread_call_wait_once_locked(call, s); + /* thread call lock has been unlocked */ + } else { + enable_ints_and_unlock(s); + } if (refs == 0) { + assert(call->tc_finish_count == call->tc_submit_count); zfree(thread_call_zone, call); } @@ -831,14 +926,14 @@ thread_call_enter1( { boolean_t result = TRUE; thread_call_group_t group; - spl_t s; assert(call->tc_call.func != NULL); + assert((call->tc_flags & THREAD_CALL_SIGNAL) == 0); + group = thread_call_get_group(call); - s = splsched(); - thread_call_lock_spin(); + spl_t s = disable_ints_and_lock(); if (call->tc_call.queue != &group->pending_queue) { result = _pending_call_enqueue(call, group); @@ -846,8 +941,7 @@ thread_call_enter1( call->tc_call.param1 = param1; - thread_call_unlock(); - splx(s); + enable_ints_and_unlock(s); return (result); } @@ -920,38 +1014,40 @@ thread_call_enter_delayed_internal( { boolean_t result = TRUE; thread_call_group_t group; - spl_t s; - uint64_t abstime, conttime, sdeadline, slop; + uint64_t now, sdeadline, slop; uint32_t urgency; - const boolean_t is_cont_time = (flags & THREAD_CALL_CONTINUOUS) ? TRUE : FALSE; + + thread_call_flavor_t flavor = (flags & THREAD_CALL_CONTINUOUS) ? TCF_CONTINUOUS : TCF_ABSOLUTE; /* direct mapping between thread_call, timer_call, and timeout_urgency values */ urgency = (flags & TIMEOUT_URGENCY_MASK); - s = splsched(); - thread_call_lock_spin(); + spl_t s = disable_ints_and_lock(); if (call == NULL) { /* allocate a structure out of internal storage, as a convenience for BSD callers */ call = _internal_call_allocate(alt_func, alt_param0); } - if (is_cont_time) { + assert(call->tc_call.func != NULL); + group = thread_call_get_group(call); + + /* TODO: assert that call is not enqueued before flipping the flag */ + if (flavor == TCF_CONTINUOUS) { + now = mach_continuous_time(); call->tc_flags |= THREAD_CALL_CONTINUOUS; + } else { + now = mach_absolute_time(); + call->tc_flags &= ~THREAD_CALL_CONTINUOUS; } - assert(call->tc_call.func != NULL); - group = thread_call_get_group(call); - abstime = mach_absolute_time(); - conttime = absolutetime_to_continuoustime(abstime); - call->tc_flags |= THREAD_CALL_DELAYED; call->tc_soft_deadline = sdeadline = deadline; boolean_t ratelimited = FALSE; - slop = timer_call_slop(deadline, is_cont_time ? conttime : abstime, urgency, current_thread(), &ratelimited); - + slop = timer_call_slop(deadline, now, urgency, current_thread(), &ratelimited); + if ((flags & THREAD_CALL_DELAY_LEEWAY) != 0 && leeway > slop) slop = leeway; @@ -966,75 +1062,84 @@ thread_call_enter_delayed_internal( call->tc_flags &= ~TIMER_CALL_RATELIMITED; } - call->tc_call.param1 = param1; - if(is_cont_time) { - call->ttd = (sdeadline > conttime) ? (sdeadline - conttime) : 0; - } - else { - call->ttd = (sdeadline > abstime) ? (sdeadline - abstime) : 0; - } + call->tc_ttd = (sdeadline > now) ? (sdeadline - now) : 0; - result = _delayed_call_enqueue(call, group, deadline); + result = _delayed_call_enqueue(call, group, deadline, flavor); - if (queue_first(&group->delayed_queue) == qe(call)) { - _set_delayed_call_timer(call, group); - } + _arm_delayed_call_timer(call, group, flavor); #if CONFIG_DTRACE - DTRACE_TMR5(thread_callout__create, thread_call_func_t, call->tc_call.func, uint64_t, (deadline - sdeadline), uint64_t, (call->ttd >> 32), (unsigned) (call->ttd & 0xFFFFFFFF), call); + DTRACE_TMR5(thread_callout__create, thread_call_func_t, call->tc_call.func, + uint64_t, (deadline - sdeadline), uint64_t, (call->tc_ttd >> 32), + (unsigned) (call->tc_ttd & 0xFFFFFFFF), call); #endif - thread_call_unlock(); - splx(s); + enable_ints_and_unlock(s); return (result); } /* - * thread_call_cancel: - * - * Dequeue a callout entry. - * - * Returns TRUE if the call was - * on a queue. + * Remove a callout entry from the queue + * Called with thread_call_lock held */ -boolean_t -thread_call_cancel( - thread_call_t call) +static boolean_t +thread_call_cancel_locked(thread_call_t call) { - boolean_t result, do_cancel_callout = FALSE; - thread_call_group_t group; - spl_t s; + boolean_t canceled = (0 != (THREAD_CALL_RESCHEDULE & call->tc_flags)); + call->tc_flags &= ~THREAD_CALL_RESCHEDULE; - group = thread_call_get_group(call); + if (canceled) { + /* if reschedule was set, it must not have been queued */ + assert(call->tc_call.queue == NULL); + } else { + boolean_t do_cancel_callout = FALSE; - s = splsched(); - thread_call_lock_spin(); + thread_call_flavor_t flavor = thread_call_get_flavor(call); + thread_call_group_t group = thread_call_get_group(call); - if ((call->tc_call.deadline != 0) && - (queue_first(&group->delayed_queue) == qe(call))) { - assert (call->tc_call.queue == &group->delayed_queue); - do_cancel_callout = TRUE; - } + if ((call->tc_call.deadline != 0) && + (call == qe_queue_first(&group->delayed_queues[flavor], struct thread_call, tc_call.q_link))) { + assert(call->tc_call.queue == &group->delayed_queues[flavor]); + do_cancel_callout = TRUE; + } - result = _call_dequeue(call, group); + canceled = _call_dequeue(call, group); - if (do_cancel_callout) { - timer_call_cancel(&group->delayed_timer); - if (!queue_empty(&group->delayed_queue)) { - _set_delayed_call_timer(TC(queue_first(&group->delayed_queue)), group); + if (do_cancel_callout) { + if (_arm_delayed_call_timer(NULL, group, flavor) == false) + timer_call_cancel(&group->delayed_timers[flavor]); } } - thread_call_unlock(); - splx(s); #if CONFIG_DTRACE - DTRACE_TMR4(thread_callout__cancel, thread_call_func_t, call->tc_call.func, 0, (call->ttd >> 32), (unsigned) (call->ttd & 0xFFFFFFFF)); + DTRACE_TMR4(thread_callout__cancel, thread_call_func_t, call->tc_call.func, + 0, (call->tc_ttd >> 32), (unsigned) (call->tc_ttd & 0xFFFFFFFF)); #endif - return (result); + return canceled; +} + +/* + * thread_call_cancel: + * + * Dequeue a callout entry. + * + * Returns TRUE if the call was + * on a queue. + */ +boolean_t +thread_call_cancel(thread_call_t call) +{ + spl_t s = disable_ints_and_lock(); + + boolean_t result = thread_call_cancel_locked(call); + + enable_ints_and_unlock(s); + + return result; } /* @@ -1046,30 +1151,47 @@ thread_call_cancel( * to the call to thread_call_cancel_wait will have finished. */ boolean_t -thread_call_cancel_wait( - thread_call_t call) +thread_call_cancel_wait(thread_call_t call) { - boolean_t result; - thread_call_group_t group; + if ((call->tc_flags & THREAD_CALL_ALLOC) == 0) + panic("thread_call_cancel_wait: can't wait on thread call whose storage I don't own"); - if ((call->tc_flags & THREAD_CALL_ALLOC) == 0) { - panic("%s: Can't wait on thread call whose storage I don't own.", __FUNCTION__); - } + if (!ml_get_interrupts_enabled()) + panic("unsafe thread_call_cancel_wait"); - group = thread_call_get_group(call); + if (current_thread()->thc_state.thc_call == call) + panic("thread_call_cancel_wait: deadlock waiting on self from inside call: %p to function %p", + call, call->tc_call.func); - (void) splsched(); - thread_call_lock_spin(); + spl_t s = disable_ints_and_lock(); - result = _call_dequeue(call, group); - if (result == FALSE) { - thread_call_wait_locked(call); - } + boolean_t canceled = thread_call_cancel_locked(call); - thread_call_unlock(); - (void) spllo(); + if ((call->tc_flags & THREAD_CALL_ONCE) == THREAD_CALL_ONCE) { + /* + * A cancel-wait on a 'once' call will both cancel + * the pending call and wait for the in-flight call + */ - return result; + thread_call_wait_once_locked(call, s); + /* thread call lock unlocked */ + } else { + /* + * A cancel-wait on a normal call will only wait for the in-flight calls + * if it did not cancel the pending call. + * + * TODO: This seems less than useful - shouldn't it do the wait as well? + */ + + if (canceled == FALSE) { + thread_call_wait_locked(call, s); + /* thread call lock unlocked */ + } else { + enable_ints_and_unlock(s); + } + } + + return canceled; } @@ -1116,29 +1238,33 @@ thread_call_wake( /* * sched_call_thread: * - * Call out invoked by the scheduler. Used only for high-priority - * thread call group. + * Call out invoked by the scheduler. */ static void sched_call_thread( int type, - __unused thread_t thread) + thread_t thread) { thread_call_group_t group; - group = &thread_call_groups[THREAD_CALL_PRIORITY_HIGH]; /* XXX */ + group = thread->thc_state.thc_group; + assert((group - &thread_call_groups[0]) < THREAD_CALL_INDEX_MAX); thread_call_lock_spin(); switch (type) { case SCHED_CALL_BLOCK: + assert(group->active_count); --group->active_count; + group->blocked_count++; if (group->pending_count > 0) thread_call_wake(group); break; case SCHED_CALL_UNBLOCK: + assert(group->blocked_count); + --group->blocked_count; group->active_count++; break; } @@ -1152,17 +1278,57 @@ sched_call_thread( * anyone who might be waiting on this work item and frees it * if the client has so requested. */ -static void -thread_call_finish(thread_call_t call, spl_t *s) +static boolean_t +thread_call_finish(thread_call_t call, thread_call_group_t group, spl_t *s) { + uint64_t time; + uint32_t flags; + boolean_t signal; boolean_t dowake = FALSE; + boolean_t repend = FALSE; call->tc_finish_count++; - call->tc_refs--; + flags = call->tc_flags; + signal = ((THREAD_CALL_SIGNAL & flags) != 0); + + if (!signal) { + /* The thread call thread owns a ref until the call is finished */ + if (call->tc_refs <= 0) + panic("thread_call_finish: detected over-released thread call: %p", call); + call->tc_refs--; + } + + call->tc_flags &= ~(THREAD_CALL_RESCHEDULE | THREAD_CALL_RUNNING | THREAD_CALL_WAIT); + + if ((call->tc_refs != 0) && ((flags & THREAD_CALL_RESCHEDULE) != 0)) { + assert(flags & THREAD_CALL_ONCE); + thread_call_flavor_t flavor = thread_call_get_flavor(call); + + if (THREAD_CALL_DELAYED & flags) { + time = mach_absolute_time(); + if (flavor == TCF_CONTINUOUS) { + time = absolutetime_to_continuoustime(time); + } + if (call->tc_soft_deadline <= time) { + call->tc_flags &= ~(THREAD_CALL_DELAYED | TIMER_CALL_RATELIMITED); + call->tc_deadline = 0; + } + } + if (call->tc_deadline) { + _delayed_call_enqueue(call, group, call->tc_deadline, flavor); + if (!signal) { + _arm_delayed_call_timer(call, group, flavor); + } + } else if (signal) { + call->tc_submit_count++; + repend = TRUE; + } else { + _pending_call_enqueue(call, group); + } + } - if ((call->tc_flags & THREAD_CALL_WAIT) != 0) { + if ((flags & THREAD_CALL_WAIT) != 0) { dowake = TRUE; - call->tc_flags &= ~THREAD_CALL_WAIT; /* * Dropping lock here because the sched call for the @@ -1172,13 +1338,16 @@ thread_call_finish(thread_call_t call, spl_t *s) thread_call_unlock(); thread_wakeup((event_t)call); thread_call_lock_spin(); + /* THREAD_CALL_SIGNAL call may have been freed */ } - if (call->tc_refs == 0) { + if (!signal && (call->tc_refs == 0)) { if (dowake) { panic("Someone waiting on a thread call that is scheduled for free: %p\n", call->tc_call.func); } + assert(call->tc_finish_count == call->tc_submit_count); + enable_ints_and_unlock(*s); zfree(thread_call_zone, call); @@ -1186,6 +1355,48 @@ thread_call_finish(thread_call_t call, spl_t *s) *s = disable_ints_and_lock(); } + return (repend); +} + +/* + * thread_call_invoke + * + * Invoke the function provided for this thread call + * + * Note that the thread call object can be deallocated by the function if we do not control its storage. + */ +static void __attribute__((noinline)) +thread_call_invoke(thread_call_func_t func, thread_call_param_t param0, thread_call_param_t param1, thread_call_t call) +{ + current_thread()->thc_state.thc_call = call; + +#if DEVELOPMENT || DEBUG + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED,MACH_CALLOUT) | DBG_FUNC_START, + VM_KERNEL_UNSLIDE(func), VM_KERNEL_ADDRHIDE(param0), VM_KERNEL_ADDRHIDE(param1), 0, 0); +#endif /* DEVELOPMENT || DEBUG */ + +#if CONFIG_DTRACE + uint64_t tc_ttd = call->tc_ttd; + boolean_t is_delayed = call->tc_flags & THREAD_CALL_DELAYED; + DTRACE_TMR6(thread_callout__start, thread_call_func_t, func, int, 0, int, (tc_ttd >> 32), + (unsigned) (tc_ttd & 0xFFFFFFFF), is_delayed, call); +#endif + + (*func)(param0, param1); + +#if CONFIG_DTRACE + DTRACE_TMR6(thread_callout__end, thread_call_func_t, func, int, 0, int, (tc_ttd >> 32), + (unsigned) (tc_ttd & 0xFFFFFFFF), is_delayed, call); +#endif + +#if DEVELOPMENT || DEBUG + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED,MACH_CALLOUT) | DBG_FUNC_END, + VM_KERNEL_UNSLIDE(func), 0, 0, 0, 0); +#endif /* DEVELOPMENT || DEBUG */ + + current_thread()->thc_state.thc_call = NULL; } /* @@ -1198,7 +1409,6 @@ thread_call_thread( { thread_t self = current_thread(); boolean_t canwait; - spl_t s; if ((thread_get_tag_internal(self) & THREAD_TAG_CALLOUT) == 0) (void)thread_set_tag_internal(self, THREAD_TAG_CALLOUT); @@ -1214,16 +1424,17 @@ thread_call_thread( panic("thread_terminate() returned?"); } - s = disable_ints_and_lock(); + spl_t s = disable_ints_and_lock(); - thread_sched_call(self, group->sched_call); + self->thc_state.thc_group = group; + thread_sched_call(self, sched_call_thread); while (group->pending_count > 0) { thread_call_t call; thread_call_func_t func; thread_call_param_t param0, param1; - call = TC(dequeue_head(&group->pending_queue)); + call = qe_dequeue_head(&group->pending_queue, struct thread_call, tc_call.q_link); assert(call != NULL); group->pending_count--; @@ -1241,27 +1452,14 @@ thread_call_thread( */ if ((call->tc_flags & THREAD_CALL_ALLOC) != 0) { canwait = TRUE; + call->tc_flags |= THREAD_CALL_RUNNING; call->tc_refs++; /* Delay free until we're done */ } else canwait = FALSE; enable_ints_and_unlock(s); -#if DEVELOPMENT || DEBUG - KERNEL_DEBUG_CONSTANT( - MACHDBG_CODE(DBG_MACH_SCHED,MACH_CALLOUT) | DBG_FUNC_NONE, - VM_KERNEL_UNSLIDE(func), VM_KERNEL_UNSLIDE_OR_PERM(param0), VM_KERNEL_UNSLIDE_OR_PERM(param1), 0, 0); -#endif /* DEVELOPMENT || DEBUG */ - -#if CONFIG_DTRACE - DTRACE_TMR6(thread_callout__start, thread_call_func_t, func, int, 0, int, (call->ttd >> 32), (unsigned) (call->ttd & 0xFFFFFFFF), (call->tc_flags & THREAD_CALL_DELAYED), call); -#endif - - (*func)(param0, param1); - -#if CONFIG_DTRACE - DTRACE_TMR6(thread_callout__end, thread_call_func_t, func, int, 0, int, (call->ttd >> 32), (unsigned) (call->ttd & 0xFFFFFFFF), (call->tc_flags & THREAD_CALL_DELAYED), call); -#endif + thread_call_invoke(func, param0, param1, call); if (get_preemption_level() != 0) { int pl = get_preemption_level(); @@ -1270,10 +1468,10 @@ thread_call_thread( } s = disable_ints_and_lock(); - + if (canwait) { /* Frees if so desired */ - thread_call_finish(call, &s); + thread_call_finish(call, group, &s); } } @@ -1346,22 +1544,17 @@ thread_call_thread( static void thread_call_daemon_continue(__unused void *arg) { - int i; - kern_return_t kr; - thread_call_group_t group; - spl_t s; - - s = disable_ints_and_lock(); + spl_t s = disable_ints_and_lock(); /* Starting at zero happens to be high-priority first. */ - for (i = 0; i < THREAD_CALL_GROUP_COUNT; i++) { - group = &thread_call_groups[i]; + for (int i = 0; i < THREAD_CALL_INDEX_MAX; i++) { + thread_call_group_t group = &thread_call_groups[i]; while (thread_call_group_should_add_thread(group)) { group->active_count++; enable_ints_and_unlock(s); - kr = thread_call_thread_create(group); + kern_return_t kr = thread_call_thread_create(group); if (kr != KERN_SUCCESS) { /* * On failure, just pause for a moment and give up. @@ -1395,6 +1588,8 @@ thread_call_daemon( self->options |= TH_OPT_VMPRIV; vm_page_free_reserve(2); /* XXX */ + thread_set_thread_name(self, "thread_call_daemon"); + thread_call_daemon_continue(NULL); /* NOTREACHED */ } @@ -1422,104 +1617,135 @@ thread_call_start_deallocate_timer( } } +/* non-static so dtrace can find it rdar://problem/31156135&31379348 */ void -thread_call_delayed_timer( - timer_call_param_t p0, - __unused timer_call_param_t p1 -) +thread_call_delayed_timer(timer_call_param_t p0, timer_call_param_t p1) { - thread_call_t call; - thread_call_group_t group = p0; - uint64_t timestamp; - - thread_call_lock_spin(); + thread_call_group_t group = (thread_call_group_t) p0; + thread_call_flavor_t flavor = (thread_call_flavor_t) p1; - const boolean_t is_cont_time = IS_CONT_GROUP(group) ? TRUE : FALSE; - - if (is_cont_time) { - timestamp = mach_continuous_time(); - } - else { - timestamp = mach_absolute_time(); - } + thread_call_t call; + uint64_t now; + boolean_t restart; + boolean_t repend; - call = TC(queue_first(&group->delayed_queue)); + thread_call_lock_spin(); - while (!queue_end(&group->delayed_queue, qe(call))) { - assert((!is_cont_time) || (call->tc_flags & THREAD_CALL_CONTINUOUS)); + if (flavor == TCF_CONTINUOUS) + now = mach_continuous_time(); + else if (flavor == TCF_ABSOLUTE) + now = mach_absolute_time(); + else + panic("invalid timer flavor: %d", flavor); + + do { + restart = FALSE; + qe_foreach_element_safe(call, &group->delayed_queues[flavor], tc_call.q_link) { + if (flavor == TCF_CONTINUOUS) + assert((call->tc_flags & THREAD_CALL_CONTINUOUS) == THREAD_CALL_CONTINUOUS); + else + assert((call->tc_flags & THREAD_CALL_CONTINUOUS) == 0); + + /* + * if we hit a call that isn't yet ready to expire, + * then we're done for now + * TODO: The next timer in the list could have a larger leeway + * and therefore be ready to expire. + * Sort by deadline then by soft deadline to avoid this + */ + if (call->tc_soft_deadline > now) + break; - if (call->tc_soft_deadline <= timestamp) { + /* + * If we hit a rate-limited timer, don't eagerly wake it up. + * Wait until it reaches the end of the leeway window. + * + * TODO: What if the next timer is not rate-limited? + * Have a separate rate-limited queue to avoid this + */ if ((call->tc_flags & THREAD_CALL_RATELIMITED) && - (CE(call)->deadline > timestamp) && + (call->tc_call.deadline > now) && (ml_timer_forced_evaluation() == FALSE)) { break; } - _pending_call_enqueue(call, group); - } /* TODO, identify differentially coalesced timers */ - else - break; - call = TC(queue_first(&group->delayed_queue)); - } + if (THREAD_CALL_SIGNAL & call->tc_flags) { + __assert_only queue_head_t *old_queue; + old_queue = call_entry_dequeue(&call->tc_call); + assert(old_queue == &group->delayed_queues[flavor]); - if (!queue_end(&group->delayed_queue, qe(call))) { - _set_delayed_call_timer(call, group); - } + do { + thread_call_func_t func = call->tc_call.func; + thread_call_param_t param0 = call->tc_call.param0; + thread_call_param_t param1 = call->tc_call.param1; + + call->tc_flags |= THREAD_CALL_RUNNING; + thread_call_unlock(); + thread_call_invoke(func, param0, param1, call); + thread_call_lock_spin(); + + repend = thread_call_finish(call, group, NULL); + } while (repend); + + /* call may have been freed */ + restart = TRUE; + break; + } else { + _pending_call_enqueue(call, group); + } + } + } while (restart); + + _arm_delayed_call_timer(call, group, flavor); thread_call_unlock(); } static void -thread_call_delayed_timer_rescan(thread_call_group_t group) +thread_call_delayed_timer_rescan(thread_call_group_t group, + thread_call_flavor_t flavor) { - thread_call_t call; - uint64_t timestamp; - boolean_t istate; + thread_call_t call; + uint64_t now; - istate = ml_set_interrupts_enabled(FALSE); - thread_call_lock_spin(); + spl_t s = disable_ints_and_lock(); assert(ml_timer_forced_evaluation() == TRUE); - if (IS_CONT_GROUP(group)) { - timestamp = mach_continuous_time(); + if (flavor == TCF_CONTINUOUS) { + now = mach_continuous_time(); } else { - timestamp = mach_absolute_time(); + now = mach_absolute_time(); } - call = TC(queue_first(&group->delayed_queue)); - - while (!queue_end(&group->delayed_queue, qe(call))) { - if (call->tc_soft_deadline <= timestamp) { + qe_foreach_element_safe(call, &group->delayed_queues[flavor], tc_call.q_link) { + if (call->tc_soft_deadline <= now) { _pending_call_enqueue(call, group); - call = TC(queue_first(&group->delayed_queue)); - } - else { + } else { uint64_t skew = call->tc_call.deadline - call->tc_soft_deadline; assert (call->tc_call.deadline >= call->tc_soft_deadline); - /* On a latency quality-of-service level change, + /* + * On a latency quality-of-service level change, * re-sort potentially rate-limited callout. The platform * layer determines which timers require this. */ if (timer_resort_threshold(skew)) { _call_dequeue(call, group); - _delayed_call_enqueue(call, group, call->tc_soft_deadline); + _delayed_call_enqueue(call, group, call->tc_soft_deadline, flavor); } - call = TC(queue_next(qe(call))); } } - if (!queue_empty(&group->delayed_queue)) - _set_delayed_call_timer(TC(queue_first(&group->delayed_queue)), group); - thread_call_unlock(); - ml_set_interrupts_enabled(istate); + _arm_delayed_call_timer(NULL, group, flavor); + + enable_ints_and_unlock(s); } void thread_call_delayed_timer_rescan_all(void) { - int i; - for(i = 0; i < THREAD_CALL_GROUP_COUNT; i++) { - thread_call_delayed_timer_rescan(&thread_call_groups[i]); + for (int i = 0; i < THREAD_CALL_INDEX_MAX; i++) { + thread_call_delayed_timer_rescan(&thread_call_groups[i], TCF_ABSOLUTE); + thread_call_delayed_timer_rescan(&thread_call_groups[i], TCF_CONTINUOUS); } } @@ -1575,14 +1801,91 @@ thread_call_dealloc_timer( thread_call_unlock(); } +/* + * Wait for the invocation of the thread call to complete + * We know there's only one in flight because of the 'once' flag. + * + * If a subsequent invocation comes in before we wake up, that's OK + * + * TODO: Here is where we will add priority inheritance to the thread executing + * the thread call in case it's lower priority than the current thread + * Priority inheritance for thread_call_wait_once + * + * Takes the thread call lock locked, returns unlocked + * This lets us avoid a spurious take/drop after waking up from thread_block + */ +static boolean_t +thread_call_wait_once_locked(thread_call_t call, spl_t s) +{ + assert(call->tc_flags & THREAD_CALL_ALLOC); + assert(call->tc_flags & THREAD_CALL_ONCE); + + if ((call->tc_flags & THREAD_CALL_RUNNING) == 0) { + enable_ints_and_unlock(s); + return FALSE; + } + + /* call is running, so we have to wait for it */ + call->tc_flags |= THREAD_CALL_WAIT; + + wait_result_t res = assert_wait(call, THREAD_UNINT); + if (res != THREAD_WAITING) + panic("Unable to assert wait: %d", res); + + enable_ints_and_unlock(s); + + res = thread_block(THREAD_CONTINUE_NULL); + if (res != THREAD_AWAKENED) + panic("Awoken with %d?", res); + + /* returns unlocked */ + return TRUE; +} + +/* + * Wait for an in-flight invocation to complete + * Does NOT try to cancel, so the client doesn't need to hold their + * lock while calling this function. + * + * Returns whether or not it had to wait. + * + * Only works for THREAD_CALL_ONCE calls. + */ +boolean_t +thread_call_wait_once(thread_call_t call) +{ + if ((call->tc_flags & THREAD_CALL_ALLOC) == 0) + panic("thread_call_wait_once: can't wait on thread call whose storage I don't own"); + + if ((call->tc_flags & THREAD_CALL_ONCE) == 0) + panic("thread_call_wait_once: can't wait_once on a non-once call"); + + if (!ml_get_interrupts_enabled()) + panic("unsafe thread_call_wait_once"); + + if (current_thread()->thc_state.thc_call == call) + panic("thread_call_wait_once: deadlock waiting on self from inside call: %p to function %p", + call, call->tc_call.func); + + spl_t s = disable_ints_and_lock(); + + boolean_t waited = thread_call_wait_once_locked(call, s); + /* thread call lock unlocked */ + + return waited; +} + + /* * Wait for all requested invocations of a thread call prior to now - * to finish. Can only be invoked on thread calls whose storage we manage. + * to finish. Can only be invoked on thread calls whose storage we manage. * Just waits for the finish count to catch up to the submit count we find * at the beginning of our wait. + * + * Called with thread_call_lock held. Returns with lock released. */ static void -thread_call_wait_locked(thread_call_t call) +thread_call_wait_locked(thread_call_t call, spl_t s) { uint64_t submit_count; wait_result_t res; @@ -1595,21 +1898,19 @@ thread_call_wait_locked(thread_call_t call) call->tc_flags |= THREAD_CALL_WAIT; res = assert_wait(call, THREAD_UNINT); - if (res != THREAD_WAITING) { - panic("Unable to assert wait?"); - } + if (res != THREAD_WAITING) + panic("Unable to assert wait: %d", res); - thread_call_unlock(); - (void) spllo(); + enable_ints_and_unlock(s); - res = thread_block(NULL); - if (res != THREAD_AWAKENED) { + res = thread_block(THREAD_CONTINUE_NULL); + if (res != THREAD_AWAKENED) panic("Awoken with %d?", res); - } - - (void) splsched(); - thread_call_lock_spin(); + + s = disable_ints_and_lock(); } + + enable_ints_and_unlock(s); } /* @@ -1620,9 +1921,8 @@ boolean_t thread_call_isactive(thread_call_t call) { boolean_t active; - spl_t s; - s = disable_ints_and_lock(); + spl_t s = disable_ints_and_lock(); active = (call->tc_submit_count > call->tc_finish_count); enable_ints_and_unlock(s); @@ -1636,21 +1936,16 @@ thread_call_isactive(thread_call_t call) void adjust_cont_time_thread_calls(void) { - thread_call_group_t group; + spl_t s = disable_ints_and_lock(); - spl_t s; - int i; - s = disable_ints_and_lock(); - - for (i = 0; i < THREAD_CALL_CONTTIME_COUNT; i++) { - // only the continuous thread call groups - group = &conttime_thread_call_groups[i]; - assert(IS_CONT_GROUP(group)); + for (int i = 0; i < THREAD_CALL_INDEX_MAX; i++) { + thread_call_group_t group = &thread_call_groups[i]; - if (!queue_empty(&group->delayed_queue)) { - _set_delayed_call_timer(TC(queue_first(&group->delayed_queue)), group); - } - } + /* only the continuous timers need to be re-armed */ + + _arm_delayed_call_timer(NULL, group, TCF_CONTINUOUS); + } enable_ints_and_unlock(s); } + diff --git a/osfmk/kern/thread_call.h b/osfmk/kern/thread_call.h index 5b486cbbf..7b326053e 100644 --- a/osfmk/kern/thread_call.h +++ b/osfmk/kern/thread_call.h @@ -62,14 +62,27 @@ typedef void (*thread_call_func_t)( threads. @constant THREAD_CALL_PRIORITY_USER Importance similar to that of normal user threads. @constant THREAD_CALL_PRIORITY_LOW Very low importance. + @constant THREAD_CALL_PRIORITY_KERNEL_HIGH Importance higher than most kernel + threads. */ typedef enum { - THREAD_CALL_PRIORITY_HIGH = 0, - THREAD_CALL_PRIORITY_KERNEL = 1, - THREAD_CALL_PRIORITY_USER = 2, - THREAD_CALL_PRIORITY_LOW = 3 + THREAD_CALL_PRIORITY_HIGH = 0, + THREAD_CALL_PRIORITY_KERNEL = 1, + THREAD_CALL_PRIORITY_USER = 2, + THREAD_CALL_PRIORITY_LOW = 3, + THREAD_CALL_PRIORITY_KERNEL_HIGH = 4 } thread_call_priority_t; +enum { + /* if call is re-submitted while the call is executing on a call thread, then delay the re-enqueue until it returns */ + THREAD_CALL_OPTIONS_ONCE = 0x00000001, +#ifdef XNU_KERNEL_PRIVATE + /* execute call from the timer interrupt instead of from the thread call thread, private interface for IOTES workloop signaling */ + THREAD_CALL_OPTIONS_SIGNAL = 0x00000002, +#endif /* XNU_KERNEL_PRIVATE */ +}; +typedef uint32_t thread_call_options_t; + __BEGIN_DECLS /*! @@ -255,6 +268,52 @@ extern thread_call_t thread_call_allocate_with_priority( thread_call_param_t param0, thread_call_priority_t pri); + /*! + @function thread_call_allocate_with_options + @abstract Allocate a thread call to execute with a specified priority. + @discussion Identical to thread_call_allocate, except that priority + and options are specified by caller. + @param func Callback to invoke when thread call is scheduled. + @param param0 First argument to pass to callback. + @param pri Priority of item. + @param options Options for item. + @result Thread call which can be passed to thread_call_enter variants. + */ +extern thread_call_t thread_call_allocate_with_options( + thread_call_func_t func, + thread_call_param_t param0, + thread_call_priority_t pri, + thread_call_options_t options); + +#ifdef KERNEL_PRIVATE + /*! + @function thread_call_allocate_with_qos + @abstract Allocate a thread call to execute with a specified QoS. + @discussion Identical to thread_call_allocate_with_options, except it uses the QoS namespace. + Private interface for pthread kext. + @param func Callback to invoke when thread call is scheduled. + @param param0 First argument to pass to callback. + @param qos_tier QoS tier to execute callback at (as in THREAD_QOS_POLICY) + @param options flags from thread_call_options_t to influence the thread call behavior + @result Thread call which can be passed to thread_call_enter variants. + */ +extern thread_call_t +thread_call_allocate_with_qos(thread_call_func_t func, + thread_call_param_t param0, + int qos_tier, + thread_call_options_t options); + +/*! + @function thread_call_wait_once + @abstract Wait for a THREAD_CALL_OPTIONS_ONCE call to finish executing if it is executing + @discussion Only works on THREAD_CALL_OPTIONS_ONCE calls + @param call The thread call to wait for + @result True if it waited, false if it did not wait + */ +extern boolean_t +thread_call_wait_once(thread_call_t call); +#endif /* KERNEL_PRIVATE */ + /*! @function thread_call_free @abstract Release a thread call. @@ -285,21 +344,38 @@ __END_DECLS #include +typedef enum { + THREAD_CALL_INDEX_HIGH = 0, + THREAD_CALL_INDEX_KERNEL = 1, + THREAD_CALL_INDEX_USER = 2, + THREAD_CALL_INDEX_LOW = 3, + THREAD_CALL_INDEX_KERNEL_HIGH = 4, + THREAD_CALL_INDEX_QOS_UI = 5, + THREAD_CALL_INDEX_QOS_IN = 6, + THREAD_CALL_INDEX_QOS_UT = 7, + THREAD_CALL_INDEX_MAX = 8, /* count of thread call indexes */ +} thread_call_index_t; + struct thread_call { - struct call_entry tc_call; /* Must be first */ + struct call_entry tc_call; /* Must be first for queue macros */ uint64_t tc_submit_count; uint64_t tc_finish_count; - uint64_t ttd; /* Time to deadline at creation */ + uint64_t tc_ttd; /* Time to deadline at creation */ uint64_t tc_soft_deadline; - thread_call_priority_t tc_pri; + thread_call_index_t tc_index; uint32_t tc_flags; int32_t tc_refs; }; -#define THREAD_CALL_ALLOC 0x01 -#define THREAD_CALL_WAIT 0x02 -#define THREAD_CALL_DELAYED 0x04 -#define THREAD_CALL_RATELIMITED TIMEOUT_URGENCY_RATELIMITED +#define THREAD_CALL_ALLOC 0x01 /* memory owned by thread_call.c */ +#define THREAD_CALL_WAIT 0x02 /* thread waiting for call to finish running */ +#define THREAD_CALL_DELAYED 0x04 /* deadline based */ +#define THREAD_CALL_RUNNING 0x08 /* currently executing on a thread */ +#define THREAD_CALL_SIGNAL 0x10 /* call from timer interrupt instead of thread */ +#define THREAD_CALL_ONCE 0x20 /* pend the enqueue if re-armed while running */ +#define THREAD_CALL_RESCHEDULE 0x40 /* enqueue is pending due to re-arm while running */ +#define THREAD_CALL_RATELIMITED TIMEOUT_URGENCY_RATELIMITED /* 0x80 */ +/* THREAD_CALL_CONTINUOUS 0x100 */ typedef struct thread_call thread_call_data_t; @@ -336,6 +412,12 @@ extern void thread_call_func_delayed_with_leeway( uint64_t leeway, uint32_t flags); +/* + * This iterates all of the pending or delayed thread calls in the group, + * which is really inefficient. + * + * This is deprecated, switch to an allocated thread call instead. + */ extern boolean_t thread_call_func_cancel( thread_call_func_t func, thread_call_param_t param, diff --git a/osfmk/kern/thread_group.c b/osfmk/kern/thread_group.c new file mode 100644 index 000000000..87d740bc4 --- /dev/null +++ b/osfmk/kern/thread_group.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#if CONFIG_EMBEDDED +void +sched_perfcontrol_thread_group_recommend(void *machine_data __unused, cluster_type_t new_recommendation __unused) +{ +} +#endif /* CONFIG_EMBEDDED */ + diff --git a/osfmk/kern/thread_group.h b/osfmk/kern/thread_group.h new file mode 100644 index 000000000..6e7991507 --- /dev/null +++ b/osfmk/kern/thread_group.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + * Thread group support routines. + */ +#ifndef _KERN_THREAD_GROUP_H_ +#define _KERN_THREAD_GROUP_H_ + +struct thread_group; +typedef struct thread_group *thread_group_t; + +#include /* for proc_reg.h / CONFIG_THREAD_GROUPS */ + + + +#endif // _KERN_THREAD_GROUP_H_ diff --git a/osfmk/kern/thread_kernel_state.h b/osfmk/kern/thread_kernel_state.h new file mode 100644 index 000000000..521835ddc --- /dev/null +++ b/osfmk/kern/thread_kernel_state.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _KERN_THREAD_KERNEL_STATE_H_ + +#include + +struct thread_kernel_state { + machine_thread_kernel_state machine; /* must be first */ + kern_allocation_name_t allocation_name; +} __attribute__((aligned(16))); + +typedef struct thread_kernel_state * thread_kernel_state_t; + +#define thread_get_kernel_state(thread) ((thread_kernel_state_t) \ + ((thread)->kernel_stack + kernel_stack_size - sizeof(struct thread_kernel_state))) + +#define thread_initialize_kernel_state(thread) \ + thread_get_kernel_state((thread))->allocation_name = NULL; + +#endif /* _KERN_THREAD_KERNEL_STATE_H_ */ diff --git a/osfmk/kern/thread_policy.c b/osfmk/kern/thread_policy.c index f30639e47..add093661 100644 --- a/osfmk/kern/thread_policy.c +++ b/osfmk/kern/thread_policy.c @@ -83,6 +83,15 @@ const qos_policy_params_t thread_qos_policy_params = { * This table defines the highest IO priority that a thread marked with this * QoS class can have. */ +#if CONFIG_EMBEDDED + .qos_iotier[THREAD_QOS_UNSPECIFIED] = THROTTLE_LEVEL_TIER0, + .qos_iotier[THREAD_QOS_USER_INTERACTIVE] = THROTTLE_LEVEL_TIER0, + .qos_iotier[THREAD_QOS_USER_INITIATED] = THROTTLE_LEVEL_TIER0, + .qos_iotier[THREAD_QOS_LEGACY] = THROTTLE_LEVEL_TIER0, + .qos_iotier[THREAD_QOS_UTILITY] = THROTTLE_LEVEL_TIER0, + .qos_iotier[THREAD_QOS_BACKGROUND] = THROTTLE_LEVEL_TIER3, + .qos_iotier[THREAD_QOS_MAINTENANCE] = THROTTLE_LEVEL_TIER3, +#else .qos_iotier[THREAD_QOS_UNSPECIFIED] = THROTTLE_LEVEL_TIER0, .qos_iotier[THREAD_QOS_USER_INTERACTIVE] = THROTTLE_LEVEL_TIER0, .qos_iotier[THREAD_QOS_USER_INITIATED] = THROTTLE_LEVEL_TIER0, @@ -90,6 +99,7 @@ const qos_policy_params_t thread_qos_policy_params = { .qos_iotier[THREAD_QOS_UTILITY] = THROTTLE_LEVEL_TIER1, .qos_iotier[THREAD_QOS_BACKGROUND] = THROTTLE_LEVEL_TIER2, /* possibly overridden by bg_iotier */ .qos_iotier[THREAD_QOS_MAINTENANCE] = THROTTLE_LEVEL_TIER3, +#endif /* * This table defines the highest QoS level that @@ -433,6 +443,37 @@ thread_policy_set_internal( return thread_affinity_set(thread, info->affinity_tag); } +#if CONFIG_EMBEDDED + case THREAD_BACKGROUND_POLICY: + { + thread_background_policy_t info; + + if (count < THREAD_BACKGROUND_POLICY_COUNT) { + result = KERN_INVALID_ARGUMENT; + break; + } + + if (thread->task != current_task()) { + result = KERN_PROTECTION_FAILURE; + break; + } + + info = (thread_background_policy_t) policy_info; + + int enable; + + if (info->priority == THREAD_BACKGROUND_POLICY_DARWIN_BG) + enable = TASK_POLICY_ENABLE; + else + enable = TASK_POLICY_DISABLE; + + int category = (current_thread() == thread) ? TASK_POLICY_INTERNAL : TASK_POLICY_EXTERNAL; + + proc_set_thread_policy_locked(thread, category, TASK_POLICY_DARWIN_BG, enable, 0, &pend_token); + + break; + } +#endif /* CONFIG_EMBEDDED */ case THREAD_THROUGHPUT_QOS_POLICY: { @@ -854,6 +895,11 @@ thread_recompute_priority( priority = BASEPRI_DEFAULT; } +#if CONFIG_EMBEDDED + /* No one can have a base priority less than MAXPRI_THROTTLE */ + if (priority < MAXPRI_THROTTLE) + priority = MAXPRI_THROTTLE; +#endif /* CONFIG_EMBEDDED */ sched_set_thread_base_priority(thread, priority); } @@ -882,6 +928,23 @@ thread_policy_update_tasklocked( thread->task_priority = priority; thread->max_priority = max_priority; +#if CONFIG_EMBEDDED + /* + * When backgrounding a thread, iOS has the semantic that + * realtime and fixed priority threads should be demoted + * to timeshare background threads. + * + * On OSX, realtime and fixed priority threads don't lose their mode. + * + * TODO: Do this inside the thread policy update routine in order to avoid double + * remove/reinsert for a runnable thread + */ + if ((max_priority <= MAXPRI_THROTTLE) && (old_max_priority > MAXPRI_THROTTLE)) { + sched_thread_mode_demote(thread, TH_SFLAG_THROTTLED); + } else if ((max_priority > MAXPRI_THROTTLE) && (old_max_priority <= MAXPRI_THROTTLE)) { + sched_thread_mode_undemote(thread, TH_SFLAG_THROTTLED); + } +#endif /* CONFIG_EMBEDDED */ thread_policy_update_spinlocked(thread, TRUE, pend_token); @@ -1195,45 +1258,6 @@ thread_policy_get( return (result); } -static volatile uint64_t unique_work_interval_id = 1; /* Start at 1, 0 is not a valid work interval ID */ - -kern_return_t -thread_policy_create_work_interval( - thread_t thread, - uint64_t *work_interval_id) -{ - thread_mtx_lock(thread); - if (thread->work_interval_id) { - /* already assigned a work interval ID */ - thread_mtx_unlock(thread); - return (KERN_INVALID_VALUE); - } - - thread->work_interval_id = OSIncrementAtomic64((volatile int64_t *)&unique_work_interval_id); - *work_interval_id = thread->work_interval_id; - - thread_mtx_unlock(thread); - return KERN_SUCCESS; -} - -kern_return_t -thread_policy_destroy_work_interval( - thread_t thread, - uint64_t work_interval_id) -{ - thread_mtx_lock(thread); - if (work_interval_id == 0 || thread->work_interval_id == 0 || thread->work_interval_id != work_interval_id) { - /* work ID isn't valid or doesn't match previously assigned work interval ID */ - thread_mtx_unlock(thread); - return (KERN_INVALID_ARGUMENT); - } - - thread->work_interval_id = 0; - - thread_mtx_unlock(thread); - return KERN_SUCCESS; -} - void thread_policy_create(thread_t thread) { @@ -1339,6 +1363,10 @@ thread_policy_update_internal_spinlocked(thread_t thread, boolean_t recompute_pr next.thep_qos = MIN(task_effective.tep_qos_ceiling, next.thep_qos); } + /* Apply the sync ipc qos override */ + if (requested.thrp_qos_sync_ipc_override != THREAD_QOS_UNSPECIFIED) + next.thep_qos = MAX(requested.thrp_qos_sync_ipc_override, next.thep_qos); + /* * The QoS relative priority is only applicable when the original programmer's * intended (requested) QoS is in effect. When the QoS is clamped (e.g. @@ -1771,6 +1799,11 @@ thread_set_requested_policy_spinlocked(thread_t thread, requested.thrp_qos_ipc_override = value; break; + case TASK_POLICY_QOS_SYNC_IPC_OVERRIDE: + assert(category == TASK_POLICY_ATTRIBUTE); + requested.thrp_qos_sync_ipc_override = value; + break; + case TASK_POLICY_TERMINATED: assert(category == TASK_POLICY_ATTRIBUTE); requested.thrp_terminated = value; @@ -2489,13 +2522,19 @@ proc_thread_qos_remove_override_internal(thread_t thread, prev_qos_override = thread_get_requested_policy_spinlocked(thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_QOS_OVERRIDE, NULL); if (squash) { + int prev_ipc_override; + int prev_override; + /* * Remove the specified overrides, and set the current override as the new base QoS. * Return the new QoS value. */ + prev_ipc_override = thread_get_requested_policy_spinlocked(thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_QOS_IPC_OVERRIDE, NULL); + prev_override = MAX(prev_qos_override, prev_ipc_override); + prev_qos = thread_get_requested_policy_spinlocked(thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_QOS, NULL); - new_qos = MAX(prev_qos, prev_qos_override); + new_qos = MAX(prev_qos, prev_override); if (new_qos != prev_qos) proc_set_thread_policy_spinlocked(thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_QOS, new_qos, 0, &pend_token); } @@ -2621,6 +2660,8 @@ void proc_thread_qos_deallocate(thread_t thread) /* This thread must have no more IPC overrides. */ assert(thread->ipc_overrides == 0); assert(thread->requested_policy.thrp_qos_ipc_override == THREAD_QOS_UNSPECIFIED); + assert(thread->sync_ipc_overrides == 0); + assert(thread->requested_policy.thrp_qos_sync_ipc_override == THREAD_QOS_UNSPECIFIED); /* * Clear out any lingering override objects. @@ -2849,12 +2890,15 @@ thread_ipc_override(thread_t thread, boolean_t is_new_override) { struct task_pend_token pend_token = {}; + boolean_t needs_update; spl_t s = splsched(); thread_lock(thread); uint32_t old_override = thread->requested_policy.thrp_qos_ipc_override; + assert(qos_override > THREAD_QOS_UNSPECIFIED); + assert(qos_override < THREAD_QOS_LAST); if (is_new_override) { if (thread->ipc_overrides++ == 0) { /* This add is the first override for this thread */ @@ -2869,13 +2913,22 @@ thread_ipc_override(thread_t thread, assert(old_override > THREAD_QOS_UNSPECIFIED); } - uint32_t new_override = MAX(old_override, qos_override); - - proc_set_thread_policy_spinlocked(thread, TASK_POLICY_ATTRIBUTE, - TASK_POLICY_QOS_IPC_OVERRIDE, - new_override, 0, &pend_token); + /* + * We can't allow lowering if there are several IPC overrides because + * the caller can't possibly know the whole truth + */ + if (thread->ipc_overrides == 1) { + needs_update = qos_override != old_override; + } else { + needs_update = qos_override > old_override; + } - assert(pend_token.tpt_update_sockets == 0); + if (needs_update) { + proc_set_thread_policy_spinlocked(thread, TASK_POLICY_ATTRIBUTE, + TASK_POLICY_QOS_IPC_OVERRIDE, + qos_override, 0, &pend_token); + assert(pend_token.tpt_update_sockets == 0); + } thread_unlock(thread); splx(s); @@ -2932,6 +2985,76 @@ thread_drop_ipc_override(thread_t thread) thread_policy_update_complete_unlocked(thread, &pend_token); } +void +thread_add_sync_ipc_override(thread_t thread) +{ + struct task_pend_token pend_token = {}; + + spl_t s = splsched(); + thread_lock(thread); + + uint32_t old_override __unused = thread->requested_policy.thrp_qos_sync_ipc_override; + + if (thread->sync_ipc_overrides++ == 0) { + /* This add is the first override for this thread */ + assert(old_override == THREAD_QOS_UNSPECIFIED); + } else { + /* There are already other overrides in effect for this thread */ + assert(old_override == THREAD_QOS_USER_INTERACTIVE); + thread_unlock(thread); + splx(s); + return; + } + + uint32_t new_override = THREAD_QOS_USER_INTERACTIVE; + + proc_set_thread_policy_spinlocked(thread, TASK_POLICY_ATTRIBUTE, + TASK_POLICY_QOS_SYNC_IPC_OVERRIDE, + new_override, 0, &pend_token); + + assert(pend_token.tpt_update_sockets == 0); + + thread_unlock(thread); + splx(s); + + /* + * this is only safe after rethrottle_thread supports + * being called from spinlock context + */ + thread_policy_update_complete_unlocked(thread, &pend_token); +} + +void +thread_drop_sync_ipc_override(thread_t thread) +{ + struct task_pend_token pend_token = {}; + + spl_t s = splsched(); + thread_lock(thread); + + assert(thread->sync_ipc_overrides > 0); + + if (--thread->sync_ipc_overrides == 0) { + /* + * There are no more overrides for this thread, so we should + * clear out the saturated override value + */ + + proc_set_thread_policy_spinlocked(thread, TASK_POLICY_ATTRIBUTE, + TASK_POLICY_QOS_SYNC_IPC_OVERRIDE, THREAD_QOS_UNSPECIFIED, + 0, &pend_token); + } + + thread_unlock(thread); + splx(s); + + /* + * this is only safe after rethrottle_thread supports + * being called from spinlock context + */ + thread_policy_update_complete_unlocked(thread, &pend_token); +} + /* Get current IPC override, may be called from spinlock context */ uint32_t thread_get_ipc_override(thread_t thread) @@ -2939,3 +3062,72 @@ thread_get_ipc_override(thread_t thread) return proc_get_thread_policy_locked(thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_QOS_IPC_OVERRIDE, NULL); } +/* + * This function will promote the thread priority + * since exec could block other threads calling + * proc_find on the proc. This boost must be removed + * via call to thread_clear_exec_promotion. + */ +void +thread_set_exec_promotion(thread_t thread) +{ + spl_t s; + + s = splsched(); + thread_lock(thread); + + assert((thread->sched_flags & TH_SFLAG_EXEC_PROMOTED) == 0); + + if (thread->sched_pri < EXEC_BOOST_PRIORITY || + !(thread->sched_flags & TH_SFLAG_EXEC_PROMOTED)) { + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_EXEC_PROMOTE) | DBG_FUNC_NONE, + (uintptr_t)thread_tid(thread), + thread->sched_pri, thread->base_pri, + EXEC_BOOST_PRIORITY, 0); + thread->sched_flags |= TH_SFLAG_EXEC_PROMOTED; + if (thread->sched_pri < EXEC_BOOST_PRIORITY) + set_sched_pri(thread, EXEC_BOOST_PRIORITY); + } + + thread_unlock(thread); + splx(s); +} + +/* + * This function will clear the exec thread + * promotion set on the thread by thread_set_exec_promotion. + */ +void +thread_clear_exec_promotion(thread_t thread) +{ + spl_t s; + + s = splsched(); + thread_lock(thread); + assert(thread->sched_flags & TH_SFLAG_EXEC_PROMOTED); + + if (thread->sched_flags & TH_SFLAG_EXEC_PROMOTED) { + thread->sched_flags &= ~TH_SFLAG_EXEC_PROMOTED; + + if (thread->sched_flags & TH_SFLAG_PROMOTED_MASK) { + /* it still has other promotions (mutex/rw_lock) */ + } else if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) { + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_EXEC_DEMOTE) | DBG_FUNC_NONE, + (uintptr_t)thread_tid(thread), + thread->sched_pri, + thread->base_pri, + DEPRESSPRI, 0); + set_sched_pri(thread, DEPRESSPRI); + } else { + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_EXEC_DEMOTE) | DBG_FUNC_NONE, + (uintptr_t)thread_tid(thread), + thread->sched_pri, + thread->base_pri, + thread->base_pri, 0); + thread_recompute_sched_pri(thread, FALSE); + } + } + + thread_unlock(thread); + splx(s); +} diff --git a/osfmk/kern/timer.c b/osfmk/kern/timer.c index 2ddcbd9b6..f101eb171 100644 --- a/osfmk/kern/timer.c +++ b/osfmk/kern/timer.c @@ -64,7 +64,11 @@ #include #include +#if CONFIG_EMBEDDED +int precise_user_kernel_time = 0; +#else int precise_user_kernel_time = 1; +#endif /* * timer_init initializes a timer. diff --git a/osfmk/kern/timer_call.c b/osfmk/kern/timer_call.c index 047d6951e..93c347ddd 100644 --- a/osfmk/kern/timer_call.c +++ b/osfmk/kern/timer_call.c @@ -97,13 +97,26 @@ lck_grp_attr_t timer_longterm_lck_grp_attr; * is responsible for moving each timer to its local timer queue * if and when that timer becomes due within the threshold. */ + +/* Sentinel for "no time set": */ #define TIMER_LONGTERM_NONE EndOfAllTime +/* The default threadhold is the delta above which a timer is "long-term" */ #if defined(__x86_64__) -#define TIMER_LONGTERM_THRESHOLD (1ULL * NSEC_PER_SEC) +#define TIMER_LONGTERM_THRESHOLD (1ULL * NSEC_PER_SEC) /* 1 sec */ #else -#define TIMER_LONGTERM_THRESHOLD TIMER_LONGTERM_NONE +#define TIMER_LONGTERM_THRESHOLD TIMER_LONGTERM_NONE /* disabled */ #endif +/* + * The scan limit throttles processing of the longterm queue. + * If the scan time exceeds this limit, we terminate, unlock + * and repeat after this same interval. This prevents unbounded holding of + * timer queue locks with interrupts masked. + */ +#define TIMER_LONGTERM_SCAN_LIMIT (1ULL * NSEC_PER_MSEC) /* 1 msec */ +/* Sentinel for "scan limit exceeded": */ +#define TIMER_LONGTERM_SCAN_AGAIN 0 + typedef struct { uint64_t interval; /* longterm timer interval */ uint64_t margin; /* fudge factor (10% of interval */ @@ -127,9 +140,13 @@ typedef struct { uint64_t escalates; /* num timers becoming shortterm */ uint64_t scan_time; /* last time the list was scanned */ threshold_t threshold; /* longterm timer threshold */ + uint64_t scan_limit; /* maximum scan time */ + uint64_t scan_pauses; /* num scans exceeding time limit */ } timer_longterm_t; -timer_longterm_t timer_longterm; +timer_longterm_t timer_longterm = { + .scan_limit = TIMER_LONGTERM_SCAN_LIMIT, + }; static mpqueue_head_t *timer_longterm_queue = NULL; @@ -503,6 +520,20 @@ timer_call_dequeue_unlocked( return (old_queue); } +static uint64_t +past_deadline_timer_handle(uint64_t deadline, uint64_t ctime) +{ + uint64_t delta = (ctime - deadline); + + past_deadline_timers++; + past_deadline_deltas += delta; + if (delta > past_deadline_longest) + past_deadline_longest = deadline; + if (delta < past_deadline_shortest) + past_deadline_shortest = delta; + + return (ctime + past_deadline_timer_adjustment); +} /* * Timer call entry locking model @@ -573,7 +604,7 @@ timer_call_enter_internal( TIMER_KDEBUG_TRACE(KDEBUG_TRACE, DECR_TIMER_ENTER | DBG_FUNC_START, VM_KERNEL_UNSLIDE_OR_PERM(call), - VM_KERNEL_UNSLIDE_OR_PERM(param1), deadline, flags, 0); + VM_KERNEL_ADDRHIDE(param1), deadline, flags, 0); urgency = (flags & TIMER_CALL_URGENCY_MASK); @@ -590,16 +621,7 @@ timer_call_enter_internal( } if (__improbable(deadline < ctime)) { - uint64_t delta = (ctime - deadline); - - past_deadline_timers++; - past_deadline_deltas += delta; - if (delta > past_deadline_longest) - past_deadline_longest = deadline; - if (delta < past_deadline_shortest) - past_deadline_shortest = delta; - - deadline = ctime + past_deadline_timer_adjustment; + deadline = past_deadline_timer_handle(deadline, ctime); sdeadline = deadline; } @@ -679,6 +701,81 @@ timer_call_enter_with_leeway( return timer_call_enter_internal(call, param1, deadline, leeway, flags, ratelimited); } +boolean_t +timer_call_quantum_timer_enter( + timer_call_t call, + timer_call_param_t param1, + uint64_t deadline, + uint64_t ctime) +{ + assert(call->call_entry.func != NULL); + assert(ml_get_interrupts_enabled() == FALSE); + + uint32_t flags = TIMER_CALL_SYS_CRITICAL | TIMER_CALL_LOCAL; + + TIMER_KDEBUG_TRACE(KDEBUG_TRACE, DECR_TIMER_ENTER | DBG_FUNC_START, + VM_KERNEL_UNSLIDE_OR_PERM(call), + VM_KERNEL_ADDRHIDE(param1), deadline, + flags, 0); + + if (__improbable(deadline < ctime)) { + deadline = past_deadline_timer_handle(deadline, ctime); + } + + uint64_t ttd = deadline - ctime; +#if CONFIG_DTRACE + DTRACE_TMR7(callout__create, timer_call_func_t, TCE(call)->func, + timer_call_param_t, TCE(call)->param0, uint32_t, flags, 0, + (ttd >> 32), (unsigned) (ttd & 0xFFFFFFFF), call); +#endif + + quantum_timer_set_deadline(deadline); + TCE(call)->deadline = deadline; + TCE(call)->param1 = param1; + call->ttd = ttd; + call->flags = flags; + +#if TIMER_TRACE + TCE(call)->entry_time = ctime; +#endif + + TIMER_KDEBUG_TRACE(KDEBUG_TRACE, DECR_TIMER_ENTER | DBG_FUNC_END, + VM_KERNEL_UNSLIDE_OR_PERM(call), + 1, deadline, 0, 0); + + return true; +} + + +boolean_t +timer_call_quantum_timer_cancel( + timer_call_t call) +{ + assert(ml_get_interrupts_enabled() == FALSE); + + TIMER_KDEBUG_TRACE(KDEBUG_TRACE, + DECR_TIMER_CANCEL | DBG_FUNC_START, + VM_KERNEL_UNSLIDE_OR_PERM(call), TCE(call)->deadline, + 0, call->flags, 0); + + TCE(call)->deadline = 0; + quantum_timer_set_deadline(0); + + TIMER_KDEBUG_TRACE(KDEBUG_TRACE, + DECR_TIMER_CANCEL | DBG_FUNC_END, + VM_KERNEL_UNSLIDE_OR_PERM(call), 0, + TCE(call)->deadline - mach_absolute_time(), + TCE(call)->deadline - TCE(call)->entry_time, 0); + +#if CONFIG_DTRACE + DTRACE_TMR6(callout__cancel, timer_call_func_t, TCE(call)->func, + timer_call_param_t, TCE(call)->param0, uint32_t, call->flags, 0, + (call->ttd >> 32), (unsigned) (call->ttd & 0xFFFFFFFF)); +#endif + + return true; +} + boolean_t timer_call_cancel( timer_call_t call) @@ -782,12 +879,7 @@ timer_queue_shutdown( timer_queue_shutdown_discarded++; } - /* The only lingering LOCAL timer should be this thread's - * quantum expiration timer. - */ - assert((call_local == FALSE) || - (TCE(call)->func == thread_quantum_expire)); - + assert(call_local == FALSE); simple_unlock(&call->lock); } @@ -795,6 +887,51 @@ timer_queue_shutdown( splx(s); } + +void +quantum_timer_expire( + uint64_t deadline) +{ + processor_t processor = current_processor(); + timer_call_t call = TIMER_CALL(&(processor->quantum_timer)); + + if (__improbable(TCE(call)->deadline > deadline)) + panic("CPU quantum timer deadlin out of sync with timer call deadline"); + + TIMER_KDEBUG_TRACE(KDEBUG_TRACE, + DECR_TIMER_EXPIRE | DBG_FUNC_NONE, + VM_KERNEL_UNSLIDE_OR_PERM(call), + TCE(call)->deadline, + TCE(call)->deadline, + TCE(call)->entry_time, 0); + + timer_call_func_t func = TCE(call)->func; + timer_call_param_t param0 = TCE(call)->param0; + timer_call_param_t param1 = TCE(call)->param1; + + TIMER_KDEBUG_TRACE(KDEBUG_TRACE, + DECR_TIMER_CALLOUT | DBG_FUNC_START, + VM_KERNEL_UNSLIDE_OR_PERM(call), VM_KERNEL_UNSLIDE(func), + VM_KERNEL_ADDRHIDE(param0), + VM_KERNEL_ADDRHIDE(param1), + 0); + +#if CONFIG_DTRACE + DTRACE_TMR7(callout__start, timer_call_func_t, func, + timer_call_param_t, param0, unsigned, call->flags, + 0, (call->ttd >> 32), + (unsigned) (call->ttd & 0xFFFFFFFF), call); +#endif + (*func)(param0, param1); + + TIMER_KDEBUG_TRACE(KDEBUG_TRACE, + DECR_TIMER_CALLOUT | DBG_FUNC_END, + VM_KERNEL_UNSLIDE_OR_PERM(call), VM_KERNEL_UNSLIDE(func), + VM_KERNEL_ADDRHIDE(param0), + VM_KERNEL_ADDRHIDE(param1), + 0); +} + static uint32_t timer_queue_expire_lock_skips; uint64_t timer_queue_expire_with_options( @@ -857,8 +994,8 @@ timer_queue_expire_with_options( TIMER_KDEBUG_TRACE(KDEBUG_TRACE, DECR_TIMER_CALLOUT | DBG_FUNC_START, VM_KERNEL_UNSLIDE_OR_PERM(call), VM_KERNEL_UNSLIDE(func), - VM_KERNEL_UNSLIDE_OR_PERM(param0), - VM_KERNEL_UNSLIDE_OR_PERM(param1), + VM_KERNEL_ADDRHIDE(param0), + VM_KERNEL_ADDRHIDE(param1), 0); #if CONFIG_DTRACE @@ -882,8 +1019,8 @@ timer_queue_expire_with_options( TIMER_KDEBUG_TRACE(KDEBUG_TRACE, DECR_TIMER_CALLOUT | DBG_FUNC_END, VM_KERNEL_UNSLIDE_OR_PERM(call), VM_KERNEL_UNSLIDE(func), - VM_KERNEL_UNSLIDE_OR_PERM(param0), - VM_KERNEL_UNSLIDE_OR_PERM(param1), + VM_KERNEL_ADDRHIDE(param0), + VM_KERNEL_ADDRHIDE(param1), 0); call = NULL; timer_queue_lock_spin(queue); @@ -1050,7 +1187,7 @@ timer_queue_trace_cpu(int ncpu) { timer_call_nosync_cpu( ncpu, - (void(*)())timer_queue_trace, + (void(*)(void *))timer_queue_trace, (void*) timer_queue_cpu(ncpu)); } @@ -1193,24 +1330,26 @@ timer_longterm_enqueue_unlocked(timer_call_t call, * - enter on the local queue (unless being deleted), * - otherwise: * - if sooner, deadline becomes the next threshold deadline. + * The total scan time is limited to TIMER_LONGTERM_SCAN_LIMIT. Should this be + * exceeded, we abort and reschedule again so that we don't shut others from + * the timer queues. Longterm timers firing late is not critical. */ void timer_longterm_scan(timer_longterm_t *tlp, - uint64_t now) + uint64_t time_start) { queue_entry_t qe; timer_call_t call; uint64_t threshold; uint64_t deadline; + uint64_t time_limit = time_start + tlp->scan_limit; mpqueue_head_t *timer_master_queue; assert(!ml_get_interrupts_enabled()); assert(cpu_number() == master_cpu); if (tlp->threshold.interval != TIMER_LONGTERM_NONE) - threshold = now + tlp->threshold.interval; - else - threshold = TIMER_LONGTERM_NONE; + threshold = time_start + tlp->threshold.interval; tlp->threshold.deadline = TIMER_LONGTERM_NONE; tlp->threshold.call = NULL; @@ -1245,12 +1384,12 @@ timer_longterm_scan(timer_longterm_t *tlp, * to the local (boot) processor's queue. */ #ifdef TIMER_ASSERT - if (deadline < now) + if (deadline < time_start) TIMER_KDEBUG_TRACE(KDEBUG_TRACE, DECR_TIMER_OVERDUE | DBG_FUNC_NONE, VM_KERNEL_UNSLIDE_OR_PERM(call), deadline, - now, + time_start, threshold, 0); #endif @@ -1277,6 +1416,15 @@ timer_longterm_scan(timer_longterm_t *tlp, } } simple_unlock(&call->lock); + + /* Abort scan if we're taking too long. */ + if (mach_absolute_time() > time_limit) { + tlp->threshold.deadline = TIMER_LONGTERM_SCAN_AGAIN; + tlp->scan_pauses++; + DBG("timer_longterm_scan() paused %llu, qlen: %llu\n", + time_limit, tlp->queue.count); + break; + } } timer_queue_unlock(timer_master_queue); @@ -1337,10 +1485,16 @@ timer_longterm_update_locked(timer_longterm_t *tlp) tlp->threshold.deadline_set = tlp->threshold.deadline; /* The next deadline timer to be set is adjusted */ - if (tlp->threshold.deadline != TIMER_LONGTERM_NONE) { + if (tlp->threshold.deadline != TIMER_LONGTERM_NONE && + tlp->threshold.deadline != TIMER_LONGTERM_SCAN_AGAIN) { tlp->threshold.deadline_set -= tlp->threshold.margin; tlp->threshold.deadline_set -= tlp->threshold.latency; } + + /* Throttle next scan time */ + uint64_t scan_clamp = mach_absolute_time() + tlp->scan_limit; + if (tlp->threshold.deadline_set < scan_clamp) + tlp->threshold.deadline_set = scan_clamp; TIMER_KDEBUG_TRACE(KDEBUG_TRACE, DECR_TIMER_UPDATE | DBG_FUNC_END, @@ -1423,7 +1577,7 @@ timer_longterm_init(void) enum { THRESHOLD, QCOUNT, ENQUEUES, DEQUEUES, ESCALATES, SCANS, PREEMPTS, - LATENCY, LATENCY_MIN, LATENCY_MAX + LATENCY, LATENCY_MIN, LATENCY_MAX, SCAN_LIMIT, PAUSES }; uint64_t timer_sysctl_get(int oid) @@ -1452,6 +1606,10 @@ timer_sysctl_get(int oid) return tlp->threshold.latency_min; case LATENCY_MAX: return tlp->threshold.latency_max; + case SCAN_LIMIT: + return tlp->scan_limit; + case PAUSES: + return tlp->scan_pauses; default: return 0; } @@ -1563,6 +1721,7 @@ timer_sysctl_set_threshold(uint64_t value) tlp->enqueues = 0; tlp->dequeues = 0; tlp->escalates = 0; + tlp->scan_pauses = 0; tlp->threshold.scans = 0; tlp->threshold.preempts = 0; tlp->threshold.latency = 0; @@ -1583,6 +1742,9 @@ timer_sysctl_set(int oid, uint64_t value) (void (*)(void *)) timer_sysctl_set_threshold, (void *) value); return KERN_SUCCESS; + case SCAN_LIMIT: + timer_longterm.scan_limit = value; + return KERN_SUCCESS; default: return KERN_INVALID_ARGUMENT; } diff --git a/osfmk/kern/timer_call.h b/osfmk/kern/timer_call.h index 03c062ed6..6581d90c2 100644 --- a/osfmk/kern/timer_call.h +++ b/osfmk/kern/timer_call.h @@ -124,9 +124,18 @@ extern boolean_t timer_call_enter_with_leeway( uint32_t flags, boolean_t ratelimited); +extern boolean_t timer_call_quantum_timer_enter( + timer_call_t call, + timer_call_param_t param1, + uint64_t deadline, + uint64_t ctime); + extern boolean_t timer_call_cancel( timer_call_t call); +extern boolean_t timer_call_quantum_timer_cancel( + timer_call_t call); + extern void timer_call_init(void); extern void timer_call_setup( diff --git a/osfmk/kern/timer_queue.h b/osfmk/kern/timer_queue.h index 879a311fd..99284d42e 100644 --- a/osfmk/kern/timer_queue.h +++ b/osfmk/kern/timer_queue.h @@ -149,6 +149,9 @@ extern uint64_t timer_queue_expire_with_options( uint64_t, boolean_t); +extern void quantum_timer_expire( + uint64_t deadline); + /* Shutdown a timer queue and reassign existing activities */ extern void timer_queue_shutdown( mpqueue_head_t *queue); @@ -174,6 +177,8 @@ extern void timer_resync_deadlines(void); extern void timer_set_deadline(uint64_t deadline); +extern void quantum_timer_set_deadline(uint64_t deadline); + /* Migrate the local timer queue of a given cpu to the master cpu */ extern uint32_t timer_queue_migrate_cpu(int target_cpu); diff --git a/osfmk/kern/waitq.c b/osfmk/kern/waitq.c index 884f3f647..380b3f1f9 100644 --- a/osfmk/kern/waitq.c +++ b/osfmk/kern/waitq.c @@ -53,6 +53,14 @@ * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ + +/* + * un-comment the following lines to debug the link/prepost tables + * NOTE: this expands each element by ~40 bytes + */ +//#define KEEP_WAITQ_LINK_STATS +//#define KEEP_WAITQ_PREPOST_STATS + #include #include #include @@ -72,11 +80,11 @@ #include -#if defined(CONFIG_WAITQ_LINK_STATS) || defined(CONFIG_WAITQ_PREPOST_STATS) -# if !defined(CONFIG_LTABLE_STATS) +#if defined(KEEP_WAITQ_LINK_STATS) || defined(KEEP_WAITQ_PREPOST_STATS) +# if !CONFIG_LTABLE_STATS # error "You must configure LTABLE_STATS to use WAITQ_[LINK|PREPOST]_STATS" # endif -# if !defined(CONFIG_WAITQ_STATS) +# if !CONFIG_WAITQ_STATS # error "You must configure WAITQ_STATS to use WAITQ_[LINK|PREPOST]_STATS" # endif #endif @@ -101,14 +109,6 @@ #define wqerr(fmt,...) \ printf("WQ[%s] ERROR: " fmt "\n", __func__, ## __VA_ARGS__) - -/* - * un-comment the following lines to debug the link/prepost tables - * NOTE: this expands each element by ~40 bytes - */ -//#define CONFIG_WAITQ_LINK_STATS -//#define CONFIG_WAITQ_PREPOST_STATS - /* * file-static functions / data */ @@ -128,13 +128,25 @@ static zone_t waitq_set_zone; #define ROUNDDOWN(x,y) (((x)/(y))*(y)) -#if defined(CONFIG_LTABLE_STATS) || defined(CONFIG_WAITQ_STATS) +#if CONFIG_LTABLE_STATS || CONFIG_WAITQ_STATS static __inline__ void waitq_grab_backtrace(uintptr_t bt[NWAITQ_BTFRAMES], int skip); #endif +#if __arm64__ + +#define waitq_lock_to(wq,to) \ + (hw_lock_bit_to(&(wq)->waitq_interlock, LCK_ILOCK, (uint32_t)to)) + +#define waitq_lock_unlock(wq) \ + (hw_unlock_bit(&(wq)->waitq_interlock, LCK_ILOCK)) + +#define waitq_lock_init(wq) \ + (wq->waitq_interlock = 0) + +#else #define waitq_lock_to(wq,to) \ - (hw_lock_to(&(wq)->waitq_interlock, to)) + (hw_lock_to(&(wq)->waitq_interlock, (uint32_t)to)) #define waitq_lock_unlock(wq) \ (hw_lock_unlock(&(wq)->waitq_interlock)) @@ -142,6 +154,7 @@ static __inline__ void waitq_grab_backtrace(uintptr_t bt[NWAITQ_BTFRAMES], int s #define waitq_lock_init(wq) \ (hw_lock_init(&(wq)->waitq_interlock)) +#endif /* __arm64__ */ /* * Prepost callback function for specially marked waitq sets @@ -184,7 +197,7 @@ struct waitq_link { uint64_t right_setid; } wql_link; }; -#ifdef CONFIG_WAITQ_LINK_STATS +#ifdef KEEP_WAITQ_LINK_STATS thread_t sl_alloc_th; task_t sl_alloc_task; uintptr_t sl_alloc_bt[NWAITQ_BTFRAMES]; @@ -196,7 +209,7 @@ struct waitq_link { uint64_t sl_free_ts; #endif }; -#if !defined(CONFIG_WAITQ_LINK_STATS) +#if !defined(KEEP_WAITQ_LINK_STATS) static_assert((sizeof(struct waitq_link) & (sizeof(struct waitq_link) - 1)) == 0, "waitq_link struct must be a power of two!"); #endif @@ -237,7 +250,7 @@ static void wql_poison(struct link_table *table, struct lt_elem *elem) default: break; } -#ifdef CONFIG_WAITQ_LINK_STATS +#ifdef KEEP_WAITQ_LINK_STATS memset(link->sl_alloc_bt, 0, sizeof(link->sl_alloc_bt)); link->sl_alloc_ts = 0; memset(link->sl_mkvalid_bt, 0, sizeof(link->sl_mkvalid_bt)); @@ -250,7 +263,7 @@ static void wql_poison(struct link_table *table, struct lt_elem *elem) #endif } -#ifdef CONFIG_WAITQ_LINK_STATS +#ifdef KEEP_WAITQ_LINK_STATS static __inline__ void wql_do_alloc_stats(struct lt_elem *elem) { if (elem) { @@ -297,7 +310,7 @@ static __inline__ void wql_do_mkvalid_stats(struct lt_elem *elem) #define wql_do_alloc_stats(e) #define wql_do_invalidate_stats(e) #define wql_do_mkvalid_stats(e) -#endif /* CONFIG_WAITQ_LINK_STATS */ +#endif /* KEEP_WAITQ_LINK_STATS */ static void wql_init(void) { @@ -347,7 +360,7 @@ static struct waitq_link *wql_alloc_link(int type) static void wql_realloc_link(struct waitq_link *link, int type) { ltable_realloc_elem(&g_wqlinktable, &link->wqte, type); -#ifdef CONFIG_WAITQ_LINK_STATS +#ifdef KEEP_WAITQ_LINK_STATS memset(link->sl_alloc_bt, 0, sizeof(link->sl_alloc_bt)); link->sl_alloc_ts = 0; wql_do_alloc_stats(&link->wqte); @@ -585,13 +598,13 @@ struct wq_prepost { uint64_t wqp_wq_id; } wqp_post; }; -#ifdef CONFIG_WAITQ_PREPOST_STATS +#ifdef KEEP_WAITQ_PREPOST_STATS thread_t wqp_alloc_th; task_t wqp_alloc_task; uintptr_t wqp_alloc_bt[NWAITQ_BTFRAMES]; #endif }; -#if !defined(CONFIG_WAITQ_PREPOST_STATS) +#if !defined(KEEP_WAITQ_PREPOST_STATS) static_assert((sizeof(struct wq_prepost) & (sizeof(struct wq_prepost) - 1)) == 0, "wq_prepost struct must be a power of two!"); #endif @@ -630,7 +643,7 @@ static void wqp_poison(struct link_table *table, struct lt_elem *elem) } } -#ifdef CONFIG_WAITQ_PREPOST_STATS +#ifdef KEEP_WAITQ_PREPOST_STATS static __inline__ void wqp_do_alloc_stats(struct lt_elem *elem) { if (!elem) @@ -653,7 +666,7 @@ static __inline__ void wqp_do_alloc_stats(struct lt_elem *elem) } #else #define wqp_do_alloc_stats(e) -#endif /* CONFIG_WAITQ_LINK_STATS */ +#endif /* KEEP_WAITQ_LINK_STATS */ static void wqp_init(void) { @@ -1520,7 +1533,7 @@ static void wq_prepost_do_post_locked(struct waitq_set *wqset, * Stats collection / reporting * * ---------------------------------------------------------------------- */ -#if defined(CONFIG_LTABLE_STATS) && defined(CONFIG_WAITQ_STATS) +#if CONFIG_LTABLE_STATS && CONFIG_WAITQ_STATS static void wq_table_stats(struct link_table *table, struct wq_table_stats *stats) { stats->version = WAITQ_STATS_VERSION; @@ -1590,7 +1603,7 @@ struct waitq *global_waitq(int index) } -#if defined(CONFIG_LTABLE_STATS) || defined(CONFIG_WAITQ_STATS) +#if CONFIG_LTABLE_STATS || CONFIG_WAITQ_STATS /* this global is for lldb */ const uint32_t g_nwaitq_btframes = NWAITQ_BTFRAMES; @@ -2149,64 +2162,6 @@ static __inline__ int waitq_select_n_locked(struct waitq *waitq, return nthreads; } - -/** - * callback function that uses thread parameters to determine wakeup eligibility - * - * Conditions: - * 'waitq' is locked - * 'thread' is not locked - */ -static thread_t waitq_select_one_cb(void *ctx, struct waitq *waitq, - int is_global, thread_t thread) -{ - int fifo_q, realtime; - boolean_t thread_imp_donor = FALSE; - - (void)ctx; - (void)waitq; - (void)is_global; - realtime = 0; - - fifo_q = 1; /* default to FIFO for all queues for now */ -#if IMPORTANCE_INHERITANCE - if (is_global) - fifo_q = 0; /* 'thread_imp_donor' takes the place of FIFO checking */ -#endif - - if (thread->sched_pri >= BASEPRI_REALTIME) - realtime = 1; - -#if IMPORTANCE_INHERITANCE - /* - * Checking imp donor bit does not need thread lock or - * or task lock since we have the wait queue lock and - * thread can not be removed from it without acquiring - * wait queue lock. The imp donor bit may change - * once we read its value, but it is ok to wake - * a thread while someone drops importance assertion - * on the that thread. - */ - thread_imp_donor = task_is_importance_donor(thread->task); -#endif /* IMPORTANCE_INHERITANCE */ - - if (fifo_q || thread_imp_donor == TRUE - || realtime || (thread->options & TH_OPT_VMPRIV)) { - /* - * If this thread's task is an importance donor, - * or it's a realtime thread, or it's a VM privileged - * thread, OR the queue is marked as FIFO: - * select the thread - */ - return thread; - } - - /* by default, _don't_ select the thread */ - return THREAD_NULL; -} - - - /** * select from a waitq a single thread waiting for a given event * @@ -2228,7 +2183,7 @@ static thread_t waitq_select_one_locked(struct waitq *waitq, event64_t event, queue_init(&threadq); - nthreads = waitq_select_n_locked(waitq, event, waitq_select_one_cb, NULL, + nthreads = waitq_select_n_locked(waitq, event, NULL, NULL, reserved_preposts, &threadq, 1, spl); /* if we selected a thread, return it (still locked) */ @@ -3308,7 +3263,7 @@ kern_return_t waitq_set_free(struct waitq_set *wqset) return KERN_SUCCESS; } -#if defined(DEVLEOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG #if CONFIG_WAITQ_DEBUG /** * return the set ID of 'wqset' @@ -4191,6 +4146,43 @@ void waitq_unlink_by_prepost_id(uint64_t wqp_id, struct waitq_set *wqset) } +/** + * reference and lock a waitq by its prepost ID + * + * Conditions: + * wqp_id may be valid or invalid + * + * Returns: + * a locked waitq if wqp_id was valid + * NULL on failure + */ +struct waitq *waitq_lock_by_prepost_id(uint64_t wqp_id) +{ + struct waitq *wq = NULL; + struct wq_prepost *wqp; + + disable_preemption(); + wqp = wq_prepost_get(wqp_id); + if (wqp) { + wq = wqp->wqp_wq.wqp_wq_ptr; + + assert(!waitq_irq_safe(wq)); + + waitq_lock(wq); + wq_prepost_put(wqp); + + if (!waitq_valid(wq)) { + /* someone already tore down this waitq! */ + waitq_unlock(wq); + enable_preemption(); + return NULL; + } + } + enable_preemption(); + return wq; +} + + /** * unlink 'waitq' from all sets to which it belongs * diff --git a/osfmk/kern/waitq.h b/osfmk/kern/waitq.h index c15f6e908..e5874895e 100644 --- a/osfmk/kern/waitq.h +++ b/osfmk/kern/waitq.h @@ -79,7 +79,17 @@ jenkins_hash(char *key, size_t length) } /* Opaque sizes and alignment used for struct verification */ -#if __x86_64__ +#if __arm__ || __arm64__ + #define WQ_OPAQUE_ALIGN __BIGGEST_ALIGNMENT__ + #define WQS_OPAQUE_ALIGN __BIGGEST_ALIGNMENT__ + #if __arm__ + #define WQ_OPAQUE_SIZE 32 + #define WQS_OPAQUE_SIZE 48 + #else + #define WQ_OPAQUE_SIZE 40 + #define WQS_OPAQUE_SIZE 56 + #endif +#elif __x86_64__ #define WQ_OPAQUE_ALIGN 8 #define WQS_OPAQUE_ALIGN 8 #define WQ_OPAQUE_SIZE 48 @@ -154,7 +164,11 @@ struct waitq { waitq_isvalid:1, /* waitq structure is valid */ waitq_eventmask:_EVENT_MASK_BITS; /* the wait queue set (set-of-sets) to which this queue belongs */ +#if __arm64__ + hw_lock_bit_t waitq_interlock; /* interlock */ +#else hw_lock_data_t waitq_interlock; /* interlock */ +#endif /* __arm64__ */ uint64_t waitq_set_id; uint64_t waitq_prepost_id; @@ -205,6 +219,15 @@ extern void waitq_invalidate_locked(struct waitq *wq); #define waitq_empty(wq) \ (queue_empty(&(wq)->waitq_queue)) +#if __arm64__ + +#define waitq_held(wq) \ + (hw_lock_bit_held(&(wq)->waitq_interlock, LCK_ILOCK)) + +#define waitq_lock_try(wq) \ + (hw_lock_bit_try(&(wq)->waitq_interlock, LCK_ILOCK)) + +#else #define waitq_held(wq) \ (hw_lock_held(&(wq)->waitq_interlock)) @@ -212,12 +235,12 @@ extern void waitq_invalidate_locked(struct waitq *wq); #define waitq_lock_try(wq) \ (hw_lock_try(&(wq)->waitq_interlock)) +#endif /* __arm64__ */ #define waitq_wait_possible(thread) \ ((thread)->waitq == NULL) extern void waitq_lock(struct waitq *wq); -extern void waitq_unlock(struct waitq *wq); #define waitq_set_lock(wqs) waitq_lock(&(wqs)->wqset_q) #define waitq_set_unlock(wqs) waitq_unlock(&(wqs)->wqset_q) @@ -376,7 +399,7 @@ extern void waitq_set_deinit(struct waitq_set *wqset); extern kern_return_t waitq_set_free(struct waitq_set *wqset); -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG #if CONFIG_WAITQ_DEBUG extern uint64_t wqset_id(struct waitq_set *wqset); @@ -422,6 +445,7 @@ extern void waitq_set_clear_preposts(struct waitq_set *wqset); */ extern uint64_t waitq_get_prepost_id(struct waitq *waitq); extern void waitq_unlink_by_prepost_id(uint64_t wqp_id, struct waitq_set *wqset); +extern struct waitq *waitq_lock_by_prepost_id(uint64_t wqp_id); /* * waitq attributes @@ -507,6 +531,9 @@ waitq_wakeup64_identify(struct waitq *waitq, wait_result_t result, int priority); +/* take the waitq lock */ +extern void waitq_unlock(struct waitq *wq); + #endif /* XNU_KERNEL_PRIVATE */ __END_DECLS diff --git a/osfmk/kern/work_interval.c b/osfmk/kern/work_interval.c new file mode 100644 index 000000000..2f4cd62a1 --- /dev/null +++ b/osfmk/kern/work_interval.c @@ -0,0 +1,459 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +/* + * Work Interval structs + * + * This struct represents a thread group and/or work interval context + * in a mechanism that is represented with a kobject. + * + * Every thread that has joined a WI has a +1 ref, and the port + * has a +1 ref as well. + * + * TODO: groups need to have a 'is for WI' flag + * and they need a flag to create that says 'for WI' + * This would allow CLPC to avoid allocating WI support + * data unless it is needed + * + * TODO: Enforce not having more than one non-group joinable work + * interval per thread group. + * CLPC only wants to see one WI-notify callout per group. + */ + +struct work_interval { + uint64_t wi_id; + _Atomic uint32_t wi_ref_count; + uint32_t wi_create_flags; + + /* for debugging purposes only, does not hold a ref on port */ + ipc_port_t wi_port; + + /* + * holds uniqueid and version of creating process, + * used to permission-gate notify + * TODO: you'd think there would be a better way to do this + */ + uint64_t wi_creator_uniqueid; + uint32_t wi_creator_pid; + int wi_creator_pidversion; + +}; + +static inline void +wi_retain(struct work_interval *work_interval) +{ + uint32_t old_count; + old_count = atomic_fetch_add_explicit(&work_interval->wi_ref_count, + 1, memory_order_relaxed); + assert(old_count > 0); +} + +static inline void +wi_release(struct work_interval *work_interval) +{ + uint32_t old_count; + old_count = atomic_fetch_sub_explicit(&work_interval->wi_ref_count, + 1, memory_order_relaxed); + assert(old_count > 0); + + if (old_count == 1) { + + + kfree(work_interval, sizeof(struct work_interval)); + } +} + +/* + * work_interval_port_alloc + * + * Description: Obtain a send right for the given work interval struct. + * + * Parameters: work_interval - A work_interval struct + * Consumes a +1 ref count on work_interval, now owned by the port. + * + * Returns: Port of type IKOT_WORK_INTERVAL with work_interval set as its kobject. + * Returned with a +1 send right and no-senders notification armed. + * Work interval struct reference is held by the port. + */ +static ipc_port_t +work_interval_port_alloc(struct work_interval *work_interval) +{ + ipc_port_t work_interval_port = ipc_port_alloc_kernel(); + + if (work_interval_port == IP_NULL) + panic("failed to allocate work interval port"); + + assert(work_interval->wi_port == IP_NULL); + + ip_lock(work_interval_port); + ipc_kobject_set_atomically(work_interval_port, (ipc_kobject_t)work_interval, + IKOT_WORK_INTERVAL); + + ipc_port_t notify_port = ipc_port_make_sonce_locked(work_interval_port); + ipc_port_t old_notify_port = IP_NULL; + ipc_port_nsrequest(work_interval_port, 1, notify_port, &old_notify_port); + /* port unlocked */ + + assert(old_notify_port == IP_NULL); + + /* This is the only make-send that will happen on this port */ + ipc_port_t send_port = ipc_port_make_send(work_interval_port); + assert(IP_VALID(send_port)); + + work_interval->wi_port = work_interval_port; + + return send_port; +} + +/* + * work_interval_port_convert + * + * Called with port locked, returns reference to work interval + * if indeed the port is a work interval kobject port + */ +static struct work_interval * +work_interval_port_convert_locked(ipc_port_t port) +{ + struct work_interval *work_interval = NULL; + + if (!IP_VALID(port)) + return NULL; + + if (!ip_active(port)) + return NULL; + + if (IKOT_WORK_INTERVAL != ip_kotype(port)) + return NULL; + + work_interval = (struct work_interval *)port->ip_kobject; + + wi_retain(work_interval); + + return work_interval; +} + +/* + * port_name_to_work_interval + * + * Description: Obtain a reference to the work_interval associated with a given port. + * + * Parameters: name A Mach port name to translate. + * + * Returns: NULL The given Mach port did not reference a work_interval. + * !NULL The work_interval that is associated with the Mach port. + */ +static kern_return_t +port_name_to_work_interval(mach_port_name_t name, + struct work_interval **work_interval) +{ + if (!MACH_PORT_VALID(name)) + return KERN_INVALID_NAME; + + ipc_port_t port = IPC_PORT_NULL; + kern_return_t kr = KERN_SUCCESS; + + kr = ipc_port_translate_send(current_space(), name, &port); + if (kr != KERN_SUCCESS) + return kr; + /* port is locked */ + + assert(IP_VALID(port)); + + struct work_interval *converted_work_interval; + + converted_work_interval = work_interval_port_convert_locked(port); + + /* the port is valid, but doesn't denote a work_interval */ + if (converted_work_interval == NULL) + kr = KERN_INVALID_CAPABILITY; + + ip_unlock(port); + + if (kr == KERN_SUCCESS) + *work_interval = converted_work_interval; + + return kr; + +} + + +/* + * work_interval_port_notify + * + * Description: Handle a no-senders notification for a work interval port. + * Destroys the port and releases its reference on the work interval. + * + * Parameters: msg A Mach no-senders notification message. + * + * Note: This assumes that there is only one create-right-from-work-interval point, + * if the ability to extract another send right after creation is added, + * this will have to change to handle make-send counts correctly. + */ +void +work_interval_port_notify(mach_msg_header_t *msg) +{ + mach_no_senders_notification_t *notification = (void *)msg; + ipc_port_t port = notification->not_header.msgh_remote_port; + struct work_interval *work_interval = NULL; + + if (!IP_VALID(port)) + panic("work_interval_port_notify(): invalid port"); + + ip_lock(port); + + if (!ip_active(port)) + panic("work_interval_port_notify(): inactive port %p", port); + + if (ip_kotype(port) != IKOT_WORK_INTERVAL) + panic("work_interval_port_notify(): not the right kobject: %p, %d\n", + port, ip_kotype(port)); + + if (port->ip_mscount != notification->not_count) + panic("work_interval_port_notify(): unexpected make-send count: %p, %d, %d", + port, port->ip_mscount, notification->not_count); + + if (port->ip_srights != 0) + panic("work_interval_port_notify(): unexpected send right count: %p, %d", + port, port->ip_srights); + + work_interval = (struct work_interval *)port->ip_kobject; + + if (work_interval == NULL) + panic("work_interval_port_notify(): missing kobject: %p", port); + + ipc_kobject_set_atomically(port, IKO_NULL, IKOT_NONE); + + work_interval->wi_port = MACH_PORT_NULL; + + ip_unlock(port); + + ipc_port_dealloc_kernel(port); + wi_release(work_interval); +} + +/* + * Change thread's bound work interval to the passed-in work interval + * Consumes +1 ref on work_interval + * + * May also pass NULL to un-set work_interval on the thread + * + * Will deallocate any old work interval on the thread + */ +static void +thread_set_work_interval(thread_t thread, + struct work_interval *work_interval) +{ + assert(thread == current_thread()); + + struct work_interval *old_th_wi = thread->th_work_interval; + + /* transfer +1 ref to thread */ + thread->th_work_interval = work_interval; + + + if (old_th_wi != NULL) + wi_release(old_th_wi); +} + +void +work_interval_thread_terminate(thread_t thread) +{ + if (thread->th_work_interval != NULL) + thread_set_work_interval(thread, NULL); +} + + + +kern_return_t +kern_work_interval_notify(thread_t thread, struct kern_work_interval_args* kwi_args) +{ + assert(thread == current_thread()); + assert(kwi_args->work_interval_id != 0); + + struct work_interval *work_interval = thread->th_work_interval; + + if (work_interval == NULL || + work_interval->wi_id != kwi_args->work_interval_id) { + /* This thread must have adopted the work interval to be able to notify */ + return (KERN_INVALID_ARGUMENT); + } + + task_t notifying_task = current_task(); + + if (work_interval->wi_creator_uniqueid != get_task_uniqueid(notifying_task) || + work_interval->wi_creator_pidversion != get_task_version(notifying_task)) { + /* Only the creating task can do a notify */ + return (KERN_INVALID_ARGUMENT); + } + + spl_t s = splsched(); + + + uint64_t urgency_param1, urgency_param2; + kwi_args->urgency = thread_get_urgency(thread, &urgency_param1, &urgency_param2); + + splx(s); + + /* called without interrupts disabled */ + machine_work_interval_notify(thread, kwi_args); + + return (KERN_SUCCESS); +} + +/* Start at 1, 0 is not a valid work interval ID */ +static _Atomic uint64_t unique_work_interval_id = 1; + +kern_return_t +kern_work_interval_create(thread_t thread, + struct kern_work_interval_create_args *create_params) +{ + assert(thread == current_thread()); + + if (thread->th_work_interval != NULL) { + /* already assigned a work interval */ + return (KERN_FAILURE); + } + + struct work_interval *work_interval = kalloc(sizeof(*work_interval)); + + if (work_interval == NULL) + panic("failed to allocate work_interval"); + + bzero(work_interval, sizeof(*work_interval)); + + uint64_t old_value = atomic_fetch_add_explicit(&unique_work_interval_id, 1, + memory_order_relaxed); + + uint64_t work_interval_id = old_value + 1; + + uint32_t create_flags = create_params->wica_create_flags; + + task_t creating_task = current_task(); + + *work_interval = (struct work_interval) { + .wi_id = work_interval_id, + .wi_ref_count = 1, + .wi_create_flags = create_flags, + .wi_creator_pid = pid_from_task(creating_task), + .wi_creator_uniqueid = get_task_uniqueid(creating_task), + .wi_creator_pidversion = get_task_version(creating_task), + }; + + + if (create_flags & WORK_INTERVAL_FLAG_JOINABLE) { + /* work_interval has a +1 ref, moves to the port */ + ipc_port_t port = work_interval_port_alloc(work_interval); + mach_port_name_t name = MACH_PORT_NULL; + + name = ipc_port_copyout_send(port, current_space()); + + if (!MACH_PORT_VALID(name)) { + /* + * copyout failed (port is already deallocated) + * Because of the port-destroyed magic, + * the work interval is already deallocated too. + */ + return KERN_RESOURCE_SHORTAGE; + } + + create_params->wica_port = name; + } else { + /* work_interval has a +1 ref, moves to the thread */ + thread_set_work_interval(thread, work_interval); + create_params->wica_port = MACH_PORT_NULL; + } + + create_params->wica_id = work_interval_id; + + return KERN_SUCCESS; +} + +kern_return_t +kern_work_interval_destroy(thread_t thread, + uint64_t work_interval_id) +{ + if (work_interval_id == 0) + return KERN_INVALID_ARGUMENT; + + if (thread->th_work_interval == NULL || + thread->th_work_interval->wi_id != work_interval_id) { + /* work ID isn't valid or doesn't match joined work interval ID */ + return (KERN_INVALID_ARGUMENT); + } + + thread_set_work_interval(thread, NULL); + + return KERN_SUCCESS; +} + +kern_return_t +kern_work_interval_join(thread_t thread, + mach_port_name_t port_name) +{ + struct work_interval *work_interval = NULL; + kern_return_t kr; + + if (port_name == MACH_PORT_NULL) { + /* 'Un-join' the current work interval */ + thread_set_work_interval(thread, NULL); + return KERN_SUCCESS; + } + + kr = port_name_to_work_interval(port_name, &work_interval); + if (kr != KERN_SUCCESS) + return kr; + /* work_interval has a +1 ref */ + + assert(work_interval != NULL); + + thread_set_work_interval(thread, work_interval); + + /* ref was consumed by passing it to the thread */ + + return KERN_SUCCESS; +} + + + diff --git a/osfmk/kern/work_interval.h b/osfmk/kern/work_interval.h new file mode 100644 index 000000000..31f009bfa --- /dev/null +++ b/osfmk/kern/work_interval.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _KERN_WORK_INTERVAL_H_ +#define _KERN_WORK_INTERVAL_H_ + +#include + +#include +#include + +#include + +__BEGIN_DECLS + +struct work_interval; + +struct kern_work_interval_args { + uint64_t work_interval_id; + uint64_t start; + uint64_t finish; + uint64_t deadline; + uint64_t next_start; + uint32_t notify_flags; + uint32_t create_flags; + uint16_t urgency; +}; + +struct kern_work_interval_create_args { + uint64_t wica_id; /* out param */ + uint32_t wica_port; /* out param */ + uint32_t wica_create_flags; +}; + +/* + * Allocate/assign a single work interval ID for a thread, + * and support deallocating it. + */ +extern kern_return_t +kern_work_interval_create(thread_t thread, struct kern_work_interval_create_args *create_params); +extern kern_return_t +kern_work_interval_destroy(thread_t thread, uint64_t work_interval_id); +extern kern_return_t +kern_work_interval_join(thread_t thread, mach_port_name_t port_name); + + +extern kern_return_t +kern_work_interval_notify(thread_t thread, struct kern_work_interval_args* kwi_args); + +#ifdef MACH_KERNEL_PRIVATE + +extern void work_interval_port_notify(mach_msg_header_t *msg); + + +extern void work_interval_thread_terminate(thread_t thread); + +#endif /* MACH_KERNEL_PRIVATE */ + +__END_DECLS + +#endif /* !defined(_KERN_WORK_INTERVAL_H_) */ diff --git a/osfmk/kern/zalloc.c b/osfmk/kern/zalloc.c index be40d8260..699dc3c74 100644 --- a/osfmk/kern/zalloc.c +++ b/osfmk/kern/zalloc.c @@ -73,6 +73,7 @@ #include #include +#include #include #include #include @@ -100,6 +101,8 @@ #include #include +#include + /* * ZONE_ALIAS_ADDR (deprecated) */ @@ -208,6 +211,9 @@ vm_size_t zp_tiny_zone_limit = 0; uintptr_t zp_poisoned_cookie = 0; uintptr_t zp_nopoison_cookie = 0; +#if VM_MAX_TAG_ZONES +boolean_t zone_tagging_on; +#endif /* VM_MAX_TAG_ZONES */ /* * initialize zone poisoning @@ -315,7 +321,7 @@ zp_init(void) /* * These macros are used to keep track of the number * of pages being used by the zone currently. The - * z->page_count is protected by the zone lock. + * z->page_count is not protected by the zone lock. */ #define ZONE_PAGE_COUNT_INCR(z, count) \ { \ @@ -356,15 +362,28 @@ struct zone_free_element { }; /* - * Protects num_zones and zone_array + * Protects zone_array, num_zones, num_zones_in_use, and zone_empty_bitmap */ decl_simple_lock_data(, all_zones_lock) +unsigned int num_zones_in_use; unsigned int num_zones; -#define MAX_ZONES 256 +#define MAX_ZONES 288 struct zone zone_array[MAX_ZONES]; -#define MULTIPAGE_METADATA_MAGIC (0xff) +/* Used to keep track of empty slots in the zone_array */ +bitmap_t zone_empty_bitmap[BITMAP_LEN(MAX_ZONES)]; + +#if DEBUG || DEVELOPMENT +/* + * Used for sysctl kern.run_zone_test which is not thread-safe. Ensure only one thread goes through at a time. + * Or we can end up with multiple test zones (if a second zinit() comes through before zdestroy()), which could lead us to + * run out of zones. + */ +decl_simple_lock_data(, zone_test_lock) +static boolean_t zone_test_running = FALSE; +static zone_t test_zone_ptr = NULL; +#endif /* DEBUG || DEVELOPMENT */ #define PAGE_METADATA_GET_ZINDEX(page_meta) \ (page_meta->zindex) @@ -397,12 +416,10 @@ struct zone_page_metadata { /* * For the first page in the allocation chunk, this represents the total number of free elements in * the chunk. - * For all other pages, it represents the number of free elements on that page (used - * for garbage collection of zones with large multipage allocation size) */ uint16_t free_count; - uint8_t zindex; /* Zone index within the zone_array */ - uint8_t page_count; /* Count of pages within the allocation chunk */ + unsigned zindex : ZINDEX_BITS; /* Zone index within the zone_array */ + unsigned page_count : PAGECOUNT_BITS; /* Count of pages within the allocation chunk */ }; /* Macro to get page index (within zone_map) of page containing element */ @@ -428,6 +445,9 @@ struct zone_page_metadata { /* Magic value to indicate empty element free list */ #define PAGE_METADATA_EMPTY_FREELIST ((uint32_t)(~0)) +boolean_t is_zone_map_nearing_exhaustion(void); +extern void vm_pageout_garbage_collect(int collect); + static inline void * page_metadata_get_freelist(struct zone_page_metadata *page_meta) { @@ -498,11 +518,14 @@ zone_populate_metadata_page(struct zone_page_metadata *page_meta) /* All updates to the zone_metadata_region are done under the zone_metadata_region_lck */ lck_mtx_lock(&zone_metadata_region_lck); if (0 == pmap_find_phys(kernel_pmap, (vm_map_address_t)page_metadata_begin)) { - kernel_memory_populate(zone_map, + kern_return_t __unused ret = kernel_memory_populate(zone_map, page_metadata_begin, PAGE_SIZE, KMA_KOBJECT, VM_KERN_MEMORY_OSFMK); + + /* should not fail with the given arguments */ + assert(ret == KERN_SUCCESS); } lck_mtx_unlock(&zone_metadata_region_lck); } @@ -535,7 +558,7 @@ get_zone_page_metadata(struct zone_free_element *element, boolean_t init) page_meta = (struct zone_page_metadata *)(trunc_page((vm_offset_t)element)); } if (init) - bzero((char *)page_meta, sizeof(struct zone_page_metadata)); + __nosan_bzero((char *)page_meta, sizeof(struct zone_page_metadata)); return ((PAGE_METADATA_GET_ZINDEX(page_meta) != MULTIPAGE_METADATA_MAGIC) ? page_meta : page_metadata_get_realmeta(page_meta)); } @@ -549,6 +572,424 @@ get_zone_page(struct zone_page_metadata *page_meta) return (vm_offset_t)(trunc_page(page_meta)); } +/* + * ZTAGS + */ + +#if VM_MAX_TAG_ZONES + +// for zones with tagging enabled: + +// calculate a pointer to the tag base entry, +// holding either a uint32_t the first tag offset for a page in the zone map, +// or two uint16_t tags if the page can only hold one or two elements + +#define ZTAGBASE(zone, element) \ + (&((uint32_t *)zone_tagbase_min)[atop((element) - zone_map_min_address)]) + +// pointer to the tag for an element +#define ZTAG(zone, element) \ + ({ \ + vm_tag_t * result; \ + if ((zone)->tags_inline) { \ + result = (vm_tag_t *) ZTAGBASE((zone), (element)); \ + if ((page_mask & element) >= (zone)->elem_size) result++; \ + } else { \ + result = &((vm_tag_t *)zone_tags_min)[ZTAGBASE((zone), (element))[0] + ((element) & page_mask) / (zone)->elem_size]; \ + } \ + result; \ + }) + + +static vm_offset_t zone_tagbase_min; +static vm_offset_t zone_tagbase_max; +static vm_offset_t zone_tagbase_map_size; +static vm_map_t zone_tagbase_map; + +static vm_offset_t zone_tags_min; +static vm_offset_t zone_tags_max; +static vm_offset_t zone_tags_map_size; +static vm_map_t zone_tags_map; + +// simple heap allocator for allocating the tags for new memory + +decl_lck_mtx_data(,ztLock) /* heap lock */ +enum +{ + ztFreeIndexCount = 8, + ztFreeIndexMax = (ztFreeIndexCount - 1), + ztTagsPerBlock = 4 +}; + +struct ztBlock +{ +#if __LITTLE_ENDIAN__ + uint64_t free:1, + next:21, + prev:21, + size:21; +#else +// ztBlock needs free bit least significant +#error !__LITTLE_ENDIAN__ +#endif +}; +typedef struct ztBlock ztBlock; + +static ztBlock * ztBlocks; +static uint32_t ztBlocksCount; +static uint32_t ztBlocksFree; + +static uint32_t +ztLog2up(uint32_t size) +{ + if (1 == size) size = 0; + else size = 32 - __builtin_clz(size - 1); + return (size); +} + +static uint32_t +ztLog2down(uint32_t size) +{ + size = 31 - __builtin_clz(size); + return (size); +} + +static void +ztFault(vm_map_t map, const void * address, size_t size, uint32_t flags) +{ + vm_map_offset_t addr = (vm_map_offset_t) address; + vm_map_offset_t page, end; + + page = trunc_page(addr); + end = round_page(addr + size); + + for (; page < end; page += page_size) + { + if (!pmap_find_phys(kernel_pmap, page)) + { + kern_return_t __unused + ret = kernel_memory_populate(map, page, PAGE_SIZE, + KMA_KOBJECT | flags, VM_KERN_MEMORY_DIAG); + assert(ret == KERN_SUCCESS); + } + } +} + +static boolean_t +ztPresent(const void * address, size_t size) +{ + vm_map_offset_t addr = (vm_map_offset_t) address; + vm_map_offset_t page, end; + boolean_t result; + + page = trunc_page(addr); + end = round_page(addr + size); + for (result = TRUE; (page < end); page += page_size) + { + result = pmap_find_phys(kernel_pmap, page); + if (!result) break; + } + return (result); +} + + +void __unused +ztDump(boolean_t sanity); +void __unused +ztDump(boolean_t sanity) +{ + uint32_t q, cq, p; + + for (q = 0; q <= ztFreeIndexMax; q++) + { + p = q; + do + { + if (sanity) + { + cq = ztLog2down(ztBlocks[p].size); + if (cq > ztFreeIndexMax) cq = ztFreeIndexMax; + if (!ztBlocks[p].free + || ((p != q) && (q != cq)) + || (ztBlocks[ztBlocks[p].next].prev != p) + || (ztBlocks[ztBlocks[p].prev].next != p)) + { + kprintf("zterror at %d", p); + ztDump(FALSE); + kprintf("zterror at %d", p); + assert(FALSE); + } + continue; + } + kprintf("zt[%03d]%c %d, %d, %d\n", + p, ztBlocks[p].free ? 'F' : 'A', + ztBlocks[p].next, ztBlocks[p].prev, + ztBlocks[p].size); + p = ztBlocks[p].next; + if (p == q) break; + } + while (p != q); + if (!sanity) printf("\n"); + } + if (!sanity) printf("-----------------------\n"); +} + + + +#define ZTBDEQ(idx) \ + ztBlocks[ztBlocks[(idx)].prev].next = ztBlocks[(idx)].next; \ + ztBlocks[ztBlocks[(idx)].next].prev = ztBlocks[(idx)].prev; + +static void +ztFree(zone_t zone __unused, uint32_t index, uint32_t count) +{ + uint32_t q, w, p, size, merge; + + assert(count); + ztBlocksFree += count; + + // merge with preceding + merge = (index + count); + if ((merge < ztBlocksCount) + && ztPresent(&ztBlocks[merge], sizeof(ztBlocks[merge])) + && ztBlocks[merge].free) + { + ZTBDEQ(merge); + count += ztBlocks[merge].size; + } + + // merge with following + merge = (index - 1); + if ((merge > ztFreeIndexMax) + && ztPresent(&ztBlocks[merge], sizeof(ztBlocks[merge])) + && ztBlocks[merge].free) + { + size = ztBlocks[merge].size; + count += size; + index -= size; + ZTBDEQ(index); + } + + q = ztLog2down(count); + if (q > ztFreeIndexMax) q = ztFreeIndexMax; + w = q; + // queue in order of size + while (TRUE) + { + p = ztBlocks[w].next; + if (p == q) break; + if (ztBlocks[p].size >= count) break; + w = p; + } + ztBlocks[p].prev = index; + ztBlocks[w].next = index; + + // fault in first + ztFault(zone_tags_map, &ztBlocks[index], sizeof(ztBlocks[index]), 0); + + // mark first & last with free flag and size + ztBlocks[index].free = TRUE; + ztBlocks[index].size = count; + ztBlocks[index].prev = w; + ztBlocks[index].next = p; + if (count > 1) + { + index += (count - 1); + // fault in last + ztFault(zone_tags_map, &ztBlocks[index], sizeof(ztBlocks[index]), 0); + ztBlocks[index].free = TRUE; + ztBlocks[index].size = count; + } +} + +static uint32_t +ztAlloc(zone_t zone, uint32_t count) +{ + uint32_t q, w, p, leftover; + + assert(count); + + q = ztLog2up(count); + if (q > ztFreeIndexMax) q = ztFreeIndexMax; + do + { + w = q; + while (TRUE) + { + p = ztBlocks[w].next; + if (p == q) break; + if (ztBlocks[p].size >= count) + { + // dequeue, mark both ends allocated + ztBlocks[w].next = ztBlocks[p].next; + ztBlocks[ztBlocks[p].next].prev = w; + ztBlocks[p].free = FALSE; + ztBlocksFree -= ztBlocks[p].size; + if (ztBlocks[p].size > 1) ztBlocks[p + ztBlocks[p].size - 1].free = FALSE; + + // fault all the allocation + ztFault(zone_tags_map, &ztBlocks[p], count * sizeof(ztBlocks[p]), 0); + // mark last as allocated + if (count > 1) ztBlocks[p + count - 1].free = FALSE; + // free remainder + leftover = ztBlocks[p].size - count; + if (leftover) ztFree(zone, p + ztBlocks[p].size - leftover, leftover); + + return (p); + } + w = p; + } + q++; + } + while (q <= ztFreeIndexMax); + + return (-1U); +} + +static void +ztInit(vm_size_t max_zonemap_size, lck_grp_t * group) +{ + kern_return_t ret; + vm_map_kernel_flags_t vmk_flags; + uint32_t idx; + + lck_mtx_init(&ztLock, group, LCK_ATTR_NULL); + + // allocate submaps VM_KERN_MEMORY_DIAG + + zone_tagbase_map_size = atop(max_zonemap_size) * sizeof(uint32_t); + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_permanent = TRUE; + ret = kmem_suballoc(kernel_map, &zone_tagbase_min, zone_tagbase_map_size, + FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_DIAG, + &zone_tagbase_map); + + if (ret != KERN_SUCCESS) panic("zone_init: kmem_suballoc failed"); + zone_tagbase_max = zone_tagbase_min + round_page(zone_tagbase_map_size); + + zone_tags_map_size = 2048*1024 * sizeof(vm_tag_t); + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_permanent = TRUE; + ret = kmem_suballoc(kernel_map, &zone_tags_min, zone_tags_map_size, + FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_DIAG, + &zone_tags_map); + + if (ret != KERN_SUCCESS) panic("zone_init: kmem_suballoc failed"); + zone_tags_max = zone_tags_min + round_page(zone_tags_map_size); + + ztBlocks = (ztBlock *) zone_tags_min; + ztBlocksCount = (uint32_t)(zone_tags_map_size / sizeof(ztBlock)); + + // initialize the qheads + lck_mtx_lock(&ztLock); + + ztFault(zone_tags_map, &ztBlocks[0], sizeof(ztBlocks[0]), 0); + for (idx = 0; idx < ztFreeIndexCount; idx++) + { + ztBlocks[idx].free = TRUE; + ztBlocks[idx].next = idx; + ztBlocks[idx].prev = idx; + ztBlocks[idx].size = 0; + } + // free remaining space + ztFree(NULL, ztFreeIndexCount, ztBlocksCount - ztFreeIndexCount); + + lck_mtx_unlock(&ztLock); +} + +static void +ztMemoryAdd(zone_t zone, vm_offset_t mem, vm_size_t size) +{ + uint32_t * tagbase; + uint32_t count, block, blocks, idx; + size_t pages; + + pages = atop(size); + tagbase = ZTAGBASE(zone, mem); + + lck_mtx_lock(&ztLock); + + // fault tagbase + ztFault(zone_tagbase_map, tagbase, pages * sizeof(uint32_t), 0); + + if (!zone->tags_inline) + { + // allocate tags + count = (uint32_t)(size / zone->elem_size); + blocks = ((count + ztTagsPerBlock - 1) / ztTagsPerBlock); + block = ztAlloc(zone, blocks); + if (-1U == block) ztDump(false); + assert(-1U != block); + } + + lck_mtx_unlock(&ztLock); + + if (!zone->tags_inline) + { + // set tag base for each page + block *= ztTagsPerBlock; + for (idx = 0; idx < pages; idx++) + { + tagbase[idx] = block + (uint32_t)((ptoa(idx) + (zone->elem_size - 1)) / zone->elem_size); + } + } +} + +static void +ztMemoryRemove(zone_t zone, vm_offset_t mem, vm_size_t size) +{ + uint32_t * tagbase; + uint32_t count, block, blocks, idx; + size_t pages; + + // set tag base for each page + pages = atop(size); + tagbase = ZTAGBASE(zone, mem); + block = tagbase[0]; + for (idx = 0; idx < pages; idx++) + { + tagbase[idx] = 0xFFFFFFFF; + } + + lck_mtx_lock(&ztLock); + if (!zone->tags_inline) + { + count = (uint32_t)(size / zone->elem_size); + blocks = ((count + ztTagsPerBlock - 1) / ztTagsPerBlock); + assert(block != 0xFFFFFFFF); + block /= ztTagsPerBlock; + ztFree(NULL /* zone is unlocked */, block, blocks); + } + + lck_mtx_unlock(&ztLock); +} + +uint32_t +zone_index_from_tag_index(uint32_t tag_zone_index, vm_size_t * elem_size) +{ + zone_t z; + uint32_t idx; + + simple_lock(&all_zones_lock); + + for (idx = 0; idx < num_zones; idx++) + { + z = &(zone_array[idx]); + if (!z->tags) continue; + if (tag_zone_index != z->tag_zone_index) continue; + *elem_size = z->elem_size; + break; + } + + simple_unlock(&all_zones_lock); + + if (idx == num_zones) idx = -1U; + + return (idx); +} + +#endif /* VM_MAX_TAG_ZONES */ + /* Routine to get the size of a zone allocated address. If the address doesnt belong to the * zone_map, returns 0. */ @@ -575,6 +1016,35 @@ zone_element_size(void *addr, zone_t *z) } } +#if DEBUG || DEVELOPMENT + +vm_size_t +zone_element_info(void *addr, vm_tag_t * ptag) +{ + vm_size_t size = 0; + vm_tag_t tag = VM_KERN_MEMORY_NONE; + struct zone * src_zone; + + if (from_zone_map(addr, sizeof(void *))) { + struct zone_page_metadata *page_meta = get_zone_page_metadata((struct zone_free_element *)addr, FALSE); + src_zone = PAGE_METADATA_GET_ZONE(page_meta); +#if VM_MAX_TAG_ZONES + if (__improbable(src_zone->tags)) { + tag = (ZTAG(src_zone, (vm_offset_t) addr)[0] >> 1); + } +#endif /* VM_MAX_TAG_ZONES */ + size = src_zone->elem_size; + } else { +#if CONFIG_GZALLOC + gzalloc_element_size(addr, NULL, &size); +#endif /* CONFIG_GZALLOC */ + } + *ptag = tag; + return size; +} + +#endif /* DEBUG || DEVELOPMENT */ + /* * Zone checking helper function. * A pointer that satisfies these conditions is OK to be a freelist next pointer @@ -693,7 +1163,7 @@ backup_ptr_mismatch_panic(zone_t zone, /* The backup is definitely the corrupted one */ if (sane_primary && !sane_backup) zone_element_was_modified_panic(zone, element, backup, - (primary ^ (element_was_poisoned ? zp_poisoned_cookie : zp_nopoison_cookie)), + (likely_primary ^ (element_was_poisoned ? zp_poisoned_cookie : zp_nopoison_cookie)), zone->elem_size - sizeof(vm_offset_t)); /* @@ -703,10 +1173,10 @@ backup_ptr_mismatch_panic(zone_t zone, * primary pointer has been overwritten with a sane but incorrect address. */ if (sane_primary && sane_backup) - zone_element_was_modified_panic(zone, element, primary, likely_backup, 0); + zone_element_was_modified_panic(zone, element, primary, (likely_backup ^ zp_nopoison_cookie), 0); /* Neither are sane, so just guess. */ - zone_element_was_modified_panic(zone, element, primary, likely_backup, 0); + zone_element_was_modified_panic(zone, element, primary, (likely_backup ^ zp_nopoison_cookie), 0); } /* @@ -772,6 +1242,10 @@ free_to_zone(zone_t zone, } zone->count--; zone->countfree++; + +#if KASAN_ZALLOC + kasan_poison_range(element, zone->elem_size, ASAN_HEAP_FREED); +#endif } @@ -782,6 +1256,7 @@ free_to_zone(zone_t zone, */ static inline vm_offset_t try_alloc_from_zone(zone_t zone, + vm_tag_t tag __unused, boolean_t* check_poison) { vm_offset_t element; @@ -875,6 +1350,18 @@ try_alloc_from_zone(zone_t zone, zone->count++; zone->sum_count++; +#if VM_MAX_TAG_ZONES + if (__improbable(zone->tags)) { + // set the tag with b0 clear so the block remains inuse + ZTAG(zone, element)[0] = (tag << 1); + } +#endif /* VM_MAX_TAG_ZONES */ + + +#if KASAN_ZALLOC + kasan_poison_range(element, zone->elem_size, ASAN_VALID); +#endif + return element; } @@ -887,8 +1374,6 @@ try_alloc_from_zone(zone_t zone, */ #define ZINFO_SLOTS MAX_ZONES /* for now */ -void zone_display_zprint(void); - zone_t zone_find_largest(void); /* @@ -917,7 +1402,7 @@ static thread_call_data_t call_async_alloc; #define zone_wakeup(zone) thread_wakeup((event_t)(zone)) #define zone_sleep(zone) \ - (void) lck_mtx_sleep(&(zone)->lock, LCK_SLEEP_SPIN, (event_t)(zone), THREAD_UNINT); + (void) lck_mtx_sleep(&(zone)->lock, LCK_SLEEP_SPIN_ALWAYS, (event_t)(zone), THREAD_UNINT); /* * The zone_locks_grp allows for collecting lock statistics. @@ -950,7 +1435,7 @@ lck_mtx_ext_t zone_gc_lck_ext; boolean_t zone_gc_allowed = TRUE; boolean_t panic_include_zprint = FALSE; -vm_offset_t panic_kext_memory_info = 0; +mach_memory_info_t *panic_kext_memory_info = NULL; vm_size_t panic_kext_memory_size = 0; #define ZALLOC_DEBUG_ZONEGC 0x00000001 @@ -990,13 +1475,11 @@ uint32_t zalloc_debug = 0; static boolean_t log_records_init = FALSE; static int log_records; /* size of the log, expressed in number of records */ -#define MAX_NUM_ZONES_ALLOWED_LOGGING 5 /* Maximum 5 zones can be logged at once */ +#define MAX_NUM_ZONES_ALLOWED_LOGGING 10 /* Maximum 10 zones can be logged at once */ static int max_num_zones_to_log = MAX_NUM_ZONES_ALLOWED_LOGGING; static int num_zones_logged = 0; -#define MAX_ZONE_NAME 32 /* max length of a zone name we can take from the boot-args */ - static char zone_name_to_log[MAX_ZONE_NAME] = ""; /* the zone name we're logging, if any */ /* Log allocations and frees to help debug a zone element corruption */ @@ -1044,8 +1527,8 @@ boolean_t leak_scan_debug_flag = FALSE; /* enabled by "-zl" boot-ar * match a space in the zone name. */ -static int -log_this_zone(const char *zonename, const char *logname) +int +track_this_zone(const char *zonename, const char *logname) { int len; const char *zc = zonename; @@ -1181,6 +1664,15 @@ zleak_init(vm_size_t max_zonemap_size) zleak_global_tracking_threshold = max_zonemap_size / 2; zleak_per_zone_tracking_threshold = zleak_global_tracking_threshold / 8; +#if CONFIG_EMBEDDED + if (PE_parse_boot_argn("-zleakon", scratch_buf, sizeof(scratch_buf))) { + zleak_enable_flag = TRUE; + printf("zone leak detection enabled\n"); + } else { + zleak_enable_flag = FALSE; + printf("zone leak detection disabled\n"); + } +#else /* CONFIG_EMBEDDED */ /* -zleakoff (flag to disable zone leak monitor) */ if (PE_parse_boot_argn("-zleakoff", scratch_buf, sizeof(scratch_buf))) { zleak_enable_flag = FALSE; @@ -1189,6 +1681,7 @@ zleak_init(vm_size_t max_zonemap_size) zleak_enable_flag = TRUE; printf("zone leak detection enabled\n"); } +#endif /* CONFIG_EMBEDDED */ /* zfactor=XXXX (override how often to sample the zone allocator) */ if (PE_parse_boot_argn("zfactor", &zleak_sample_factor, sizeof(zleak_sample_factor))) { @@ -1549,11 +2042,36 @@ hashaddr(uintptr_t pt, uint32_t max_size) #define ZONE_MAX_ALLOC_SIZE (32 * 1024) #define ZONE_ALLOC_FRAG_PERCENT(alloc_size, ele_size) (((alloc_size % ele_size) * 100) / alloc_size) +/* Used to manage copying in of new zone names */ +static vm_offset_t zone_names_start; +static vm_offset_t zone_names_next; + +static vm_size_t +compute_element_size(vm_size_t requested_size) +{ + vm_size_t element_size = requested_size; + + /* Zone elements must fit both a next pointer and a backup pointer */ + vm_size_t minimum_element_size = sizeof(vm_offset_t) * 2; + if (element_size < minimum_element_size) + element_size = minimum_element_size; + + /* + * Round element size to a multiple of sizeof(pointer) + * This also enforces that allocations will be aligned on pointer boundaries + */ + element_size = ((element_size-1) + sizeof(vm_offset_t)) - + ((element_size-1) % sizeof(vm_offset_t)); + + return element_size; +} + /* * zinit initializes a new zone. The zone data structures themselves * are stored in a zone, which is initially a static structure that * is initialized by zone_init. */ + zone_t zinit( vm_size_t size, /* the size of an element */ @@ -1561,40 +2079,100 @@ zinit( vm_size_t alloc, /* allocation size */ const char *name) /* a name for the zone */ { - zone_t z; + zone_t z; + + size = compute_element_size(size); simple_lock(&all_zones_lock); + assert(num_zones < MAX_ZONES); + assert(num_zones_in_use <= num_zones); + + /* If possible, find a previously zdestroy'ed zone in the zone_array that we can reuse instead of initializing a new zone. */ + for (int index = bitmap_first(zone_empty_bitmap, MAX_ZONES); + index >= 0 && index < (int)num_zones; + index = bitmap_next(zone_empty_bitmap, index)) { + z = &(zone_array[index]); + + /* + * If the zone name and the element size are the same, we can just reuse the old zone struct. + * Otherwise hand out a new zone from the zone_array. + */ + if (!strcmp(z->zone_name, name)) { + vm_size_t old_size = z->elem_size; +#if KASAN_ZALLOC + old_size -= z->kasan_redzone * 2; +#endif + if (old_size == size) { + /* Clear the empty bit for this zone, increment num_zones_in_use, and mark the zone as valid again. */ + bitmap_clear(zone_empty_bitmap, index); + num_zones_in_use++; + z->zone_valid = TRUE; + + /* All other state is already set up since the zone was previously in use. Return early. */ + simple_unlock(&all_zones_lock); + return (z); + } + } + } + + /* If we're here, it means we didn't find a zone above that we could simply reuse. Set up a new zone. */ + + /* Clear the empty bit for the new zone */ + bitmap_clear(zone_empty_bitmap, num_zones); + z = &(zone_array[num_zones]); z->index = num_zones; - num_zones++; - simple_unlock(&all_zones_lock); - /* Zone elements must fit both a next pointer and a backup pointer */ - vm_size_t minimum_element_size = sizeof(vm_offset_t) * 2; - if (size < minimum_element_size) - size = minimum_element_size; + num_zones++; + num_zones_in_use++; /* - * Round element size to a multiple of sizeof(pointer) - * This also enforces that allocations will be aligned on pointer boundaries + * Initialize the zone lock here before dropping the all_zones_lock. Otherwise we could race with + * zalloc_async() and try to grab the zone lock before it has been initialized, causing a panic. */ - size = ((size-1) + sizeof(vm_offset_t)) - - ((size-1) % sizeof(vm_offset_t)); + lock_zone_init(z); + + simple_unlock(&all_zones_lock); - if (alloc == 0) - alloc = PAGE_SIZE; +#if KASAN_ZALLOC + /* Expand the zone allocation size to include the redzones. For page-multiple + * zones add a full guard page because they likely require alignment. kalloc + * and fakestack handles its own KASan state, so ignore those zones. */ + /* XXX: remove this when zinit_with_options() is a thing */ + const char *kalloc_name = "kalloc."; + const char *fakestack_name = "fakestack."; + if (strncmp(name, kalloc_name, strlen(kalloc_name)) == 0) { + z->kasan_redzone = 0; + } else if (strncmp(name, fakestack_name, strlen(fakestack_name)) == 0) { + z->kasan_redzone = 0; + } else { + if ((size % PAGE_SIZE) != 0) { + z->kasan_redzone = KASAN_GUARD_SIZE; + } else { + z->kasan_redzone = PAGE_SIZE; + } + max = (max / size) * (size + z->kasan_redzone * 2); + size += z->kasan_redzone * 2; + } +#endif - alloc = round_page(alloc); - max = round_page(max); + max = round_page(max); vm_size_t best_alloc = PAGE_SIZE; - vm_size_t alloc_size; - for (alloc_size = (2 * PAGE_SIZE); alloc_size <= ZONE_MAX_ALLOC_SIZE; alloc_size += PAGE_SIZE) { - if (ZONE_ALLOC_FRAG_PERCENT(alloc_size, size) < ZONE_ALLOC_FRAG_PERCENT(best_alloc, size)) { - best_alloc = alloc_size; + + if ((size % PAGE_SIZE) == 0) { + /* zero fragmentation by definition */ + best_alloc = size; + } else { + vm_size_t alloc_size; + for (alloc_size = (2 * PAGE_SIZE); alloc_size <= ZONE_MAX_ALLOC_SIZE; alloc_size += PAGE_SIZE) { + if (ZONE_ALLOC_FRAG_PERCENT(alloc_size, size) < ZONE_ALLOC_FRAG_PERCENT(best_alloc, size)) { + best_alloc = alloc_size; + } } } + alloc = best_alloc; if (max && (max < alloc)) max = alloc; @@ -1609,7 +2187,6 @@ zinit( z->max_size = max; z->elem_size = size; z->alloc_size = alloc; - z->zone_name = name; z->count = 0; z->countfree = 0; z->count_all_free_pages = 0; @@ -1632,13 +2209,41 @@ zinit( z->prio_refill_watermark = 0; z->zone_replenish_thread = NULL; z->zp_count = 0; + z->kasan_quarantine = TRUE; + z->zone_valid = TRUE; #if CONFIG_ZLEAKS z->zleak_capture = 0; z->zleak_on = FALSE; #endif /* CONFIG_ZLEAKS */ - lock_zone_init(z); + /* + * If the VM is ready to handle kmem_alloc requests, copy the zone name passed in. + * + * Else simply maintain a pointer to the name string. The only zones we'll actually have + * to do this for would be the VM-related zones that are created very early on before any + * kexts can be loaded (unloaded). So we should be fine with just a pointer in this case. + */ + if (kmem_alloc_ready) { + size_t len = MIN(strlen(name)+1, MACH_ZONE_NAME_MAX_LEN); + + if (zone_names_start == 0 || ((zone_names_next - zone_names_start) + len) > PAGE_SIZE) { + printf("zalloc: allocating memory for zone names buffer\n"); + kern_return_t retval = kmem_alloc_kobject(kernel_map, &zone_names_start, + PAGE_SIZE, VM_KERN_MEMORY_OSFMK); + if (retval != KERN_SUCCESS) { + panic("zalloc: zone_names memory allocation failed"); + } + bzero((char *)zone_names_start, PAGE_SIZE); + zone_names_next = zone_names_start; + } + + strlcpy((char *)zone_names_next, name, len); + z->zone_name = (char *)zone_names_next; + zone_names_next += len; + } else { + z->zone_name = name; + } /* * Check for and set up zone leak detection if requested via boot-args. We recognized two @@ -1662,11 +2267,13 @@ zinit( snprintf(zlog_name, MAX_ZONE_NAME, "zlog%d", i); if (PE_parse_boot_argn(zlog_name, zone_name_to_log, sizeof(zone_name_to_log)) == TRUE) { - if (log_this_zone(z->zone_name, zone_name_to_log)) { - z->zone_logging = TRUE; - zone_logging_enabled = TRUE; - num_zones_logged++; - break; + if (track_this_zone(z->zone_name, zone_name_to_log)) { + if (z->zone_valid) { + z->zone_logging = TRUE; + zone_logging_enabled = TRUE; + num_zones_logged++; + break; + } } } i++; @@ -1679,10 +2286,12 @@ zinit( * boot-args. */ if (PE_parse_boot_argn("zlog", zone_name_to_log, sizeof(zone_name_to_log)) == TRUE) { - if (log_this_zone(z->zone_name, zone_name_to_log)) { + if (track_this_zone(z->zone_name, zone_name_to_log)) { + if (z->zone_valid) { z->zone_logging = TRUE; zone_logging_enabled = TRUE; num_zones_logged++; + } } } } @@ -1724,6 +2333,10 @@ zinit( curr_zone = &(zone_array[zone_idx]); + if (!curr_zone->zone_valid) { + continue; + } + /* * We work with the zone unlocked here because we could end up needing the zone lock to * enable logging for this zone e.g. need a VM object to allocate memory to enable logging for the @@ -1751,6 +2364,7 @@ zinit( #if CONFIG_GZALLOC gzalloc_zone_init(z); #endif + return(z); } unsigned zone_replenish_loops, zone_replenish_wakeups, zone_replenish_wakeups_initiated, zone_replenish_throttle_count; @@ -1769,6 +2383,7 @@ zone_replenish_thread(zone_t z) for (;;) { lock_zone(z); + assert(z->zone_valid); z->zone_replenishing = TRUE; assert(z->prio_refill_watermark != 0); while ((free_size = (z->cur_size - (z->count * z->elem_size))) < (z->prio_refill_watermark * z->elem_size)) { @@ -1789,6 +2404,11 @@ zone_replenish_thread(zone_t z) if (z->noencrypt) zflags |= KMA_NOENCRYPT; + /* Trigger jetsams via the vm_pageout_garbage_collect thread if we're running out of zone memory */ + if (is_zone_map_nearing_exhaustion()) { + thread_wakeup((event_t) &vm_pageout_garbage_collect); + } + kr = kernel_memory_allocate(zone_map, &space, alloc_size, 0, zflags, VM_KERN_MEMORY_ZONE); if (kr == KERN_SUCCESS) { @@ -1806,6 +2426,7 @@ zone_replenish_thread(zone_t z) } lock_zone(z); + assert(z->zone_valid); zone_replenish_loops++; } @@ -1837,18 +2458,83 @@ zone_prio_refill_configure(zone_t z, vm_size_t low_water_mark) { thread_deallocate(z->zone_replenish_thread); } -/* Initialize the metadata for an allocation chunk */ -static inline void -zcram_metadata_init(vm_offset_t newmem, vm_size_t size, struct zone_page_metadata *chunk_metadata) +void +zdestroy(zone_t z) { - struct zone_page_metadata *page_metadata; + unsigned int zindex; - /* The first page is the real metadata for this allocation chunk. We mark the others as fake metadata */ - size -= PAGE_SIZE; - newmem += PAGE_SIZE; + assert(z != NULL); - for (; size > 0; newmem += PAGE_SIZE, size -= PAGE_SIZE) { - page_metadata = get_zone_page_metadata((struct zone_free_element *)newmem, TRUE); + lock_zone(z); + assert(z->zone_valid); + + /* Assert that the zone does not have any allocations in flight */ + assert(z->doing_alloc_without_vm_priv == FALSE); + assert(z->doing_alloc_with_vm_priv == FALSE); + assert(z->async_pending == FALSE); + assert(z->waiting == FALSE); + assert(z->async_prio_refill == FALSE); + +#if !KASAN_ZALLOC + /* + * Unset the valid bit. We'll hit an assert failure on further operations on this zone, until zinit() is called again. + * Leave the zone valid for KASan as we will see zfree's on quarantined free elements even after the zone is destroyed. + */ + z->zone_valid = FALSE; +#endif + unlock_zone(z); + + /* Dump all the free elements */ + drop_free_elements(z); + +#if CONFIG_GZALLOC + /* If the zone is gzalloc managed dump all the elements in the free cache */ + gzalloc_empty_free_cache(z); +#endif + + lock_zone(z); + +#if !KASAN_ZALLOC + /* Assert that all counts are zero */ + assert(z->count == 0); + assert(z->countfree == 0); + assert(z->cur_size == 0); + assert(z->page_count == 0); + assert(z->count_all_free_pages == 0); + + /* Assert that all queues except the foreign queue are empty. The zone allocator doesn't know how to free up foreign memory. */ + assert(queue_empty(&z->pages.all_used)); + assert(queue_empty(&z->pages.intermediate)); + assert(queue_empty(&z->pages.all_free)); +#endif + + zindex = z->index; + + unlock_zone(z); + + simple_lock(&all_zones_lock); + + assert(!bitmap_test(zone_empty_bitmap, zindex)); + /* Mark the zone as empty in the bitmap */ + bitmap_set(zone_empty_bitmap, zindex); + num_zones_in_use--; + assert(num_zones_in_use > 0); + + simple_unlock(&all_zones_lock); +} + +/* Initialize the metadata for an allocation chunk */ +static inline void +zcram_metadata_init(vm_offset_t newmem, vm_size_t size, struct zone_page_metadata *chunk_metadata) +{ + struct zone_page_metadata *page_metadata; + + /* The first page is the real metadata for this allocation chunk. We mark the others as fake metadata */ + size -= PAGE_SIZE; + newmem += PAGE_SIZE; + + for (; size > 0; newmem += PAGE_SIZE, size -= PAGE_SIZE) { + page_metadata = get_zone_page_metadata((struct zone_free_element *)newmem, TRUE); assert(page_metadata != chunk_metadata); PAGE_METADATA_SET_ZINDEX(page_metadata, MULTIPAGE_METADATA_MAGIC); page_metadata_set_realmeta(page_metadata, chunk_metadata); @@ -1913,13 +2599,14 @@ random_free_to_zone( vm_size_t elem_size; int index; + assert(element_count <= ZONE_CHUNK_MAXELEMENTS); elem_size = zone->elem_size; last_element_offset = first_element_offset + ((element_count * elem_size) - elem_size); for (index = 0; index < element_count; index++) { assert(first_element_offset <= last_element_offset); if ( #if DEBUG || DEVELOPMENT - leak_scan_debug_flag || + leak_scan_debug_flag || __improbable(zone->tags) || #endif /* DEBUG || DEVELOPMENT */ random_bool_gen(entropy_buffer, index, MAX_ENTROPY_PER_ZCRAM)) { element_addr = newmem + first_element_offset; @@ -1957,7 +2644,7 @@ zcram( elem_size = zone->elem_size; - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_ZALLOC, ZALLOC_ZCRAM) | DBG_FUNC_START, VM_KERNEL_ADDRPERM(zone), size, 0, 0, 0); + KDBG(MACHDBG_CODE(DBG_MACH_ZALLOC, ZALLOC_ZCRAM) | DBG_FUNC_START, zone->index, size); if (from_zone_map(newmem, size)) from_zm = TRUE; @@ -1995,11 +2682,20 @@ zcram( page_metadata_set_freelist(chunk_metadata, 0); PAGE_METADATA_SET_ZINDEX(chunk_metadata, zone->index); chunk_metadata->free_count = 0; - chunk_metadata->page_count = (size / PAGE_SIZE); + assert((size / PAGE_SIZE) <= ZONE_CHUNK_MAXPAGES); + chunk_metadata->page_count = (unsigned)(size / PAGE_SIZE); zcram_metadata_init(newmem, size, chunk_metadata); +#if VM_MAX_TAG_ZONES + if (__improbable(zone->tags)) { + assert(from_zm); + ztMemoryAdd(zone, newmem, size); + } +#endif /* VM_MAX_TAG_ZONES */ + lock_zone(zone); + assert(zone->zone_valid); enqueue_tail(&zone->pages.all_used, &(chunk_metadata->pages)); if (!from_zm) { @@ -2024,42 +2720,48 @@ zcram( } unlock_zone(zone); - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_ZALLOC, ZALLOC_ZCRAM) | DBG_FUNC_END, VM_KERNEL_ADDRPERM(zone), 0, 0, 0, 0); + KDBG(MACHDBG_CODE(DBG_MACH_ZALLOC, ZALLOC_ZCRAM) | DBG_FUNC_END, zone->index); } /* * Fill a zone with enough memory to contain at least nelem elements. - * Memory is obtained with kmem_alloc_kobject from the kernel_map. * Return the number of elements actually put into the zone, which may * be more than the caller asked for since the memory allocation is - * rounded up to a full page. + * rounded up to the next zone allocation size. */ int zfill( zone_t zone, int nelem) { - kern_return_t kr; - vm_size_t size; + kern_return_t kr; vm_offset_t memory; - int nalloc; - assert(nelem > 0); - if (nelem <= 0) - return 0; - size = nelem * zone->elem_size; - size = round_page(size); - kr = kmem_alloc_kobject(kernel_map, &memory, size, VM_KERN_MEMORY_ZONE); - if (kr != KERN_SUCCESS) + vm_size_t alloc_size = zone->alloc_size; + vm_size_t elem_per_alloc = alloc_size / zone->elem_size; + vm_size_t nalloc = (nelem + elem_per_alloc - 1) / elem_per_alloc; + + /* Don't mix-and-match zfill with foreign memory */ + assert(!zone->allows_foreign); + + /* Trigger jetsams via the vm_pageout_garbage_collect thread if we're running out of zone memory */ + if (is_zone_map_nearing_exhaustion()) { + thread_wakeup((event_t) &vm_pageout_garbage_collect); + } + + kr = kernel_memory_allocate(zone_map, &memory, nalloc * alloc_size, 0, KMA_KOBJECT, VM_KERN_MEMORY_ZONE); + if (kr != KERN_SUCCESS) { + printf("%s: kernel_memory_allocate() of %lu bytes failed\n", + __func__, (unsigned long)(nalloc * alloc_size)); return 0; + } - zone_change(zone, Z_FOREIGN, TRUE); - zcram(zone, memory, size); - nalloc = (int)(size / zone->elem_size); - assert(nalloc >= nelem); + for (vm_size_t i = 0; i < nalloc; i++) { + zcram(zone, memory + i * alloc_size, alloc_size); + } - return nalloc; + return (int)(nalloc * elem_per_alloc); } /* @@ -2091,6 +2793,12 @@ zone_bootstrap(void) } #if DEBUG || DEVELOPMENT +#if VM_MAX_TAG_ZONES + /* enable tags for zones that ask for */ + if (PE_parse_boot_argn("-zt", temp_buf, sizeof(temp_buf))) { + zone_tagging_on = TRUE; + } +#endif /* VM_MAX_TAG_ZONES */ /* disable element location randomization in a page */ if (PE_parse_boot_argn("-zl", temp_buf, sizeof(temp_buf))) { leak_scan_debug_flag = TRUE; @@ -2099,7 +2807,16 @@ zone_bootstrap(void) simple_lock_init(&all_zones_lock, 0); + num_zones_in_use = 0; num_zones = 0; + /* Mark all zones as empty */ + bitmap_full(zone_empty_bitmap, BITMAP_LEN(MAX_ZONES)); + zone_names_next = zone_names_start = 0; + +#if DEBUG || DEVELOPMENT + simple_lock_init(&zone_test_lock, 0); +#endif /* DEBUG || DEVELOPMENT */ + thread_call_setup(&call_async_alloc, zalloc_async, NULL); /* initializing global lock group for zones */ @@ -2110,6 +2827,101 @@ zone_bootstrap(void) lck_mtx_init_ext(&zone_metadata_region_lck, &zone_metadata_region_lck_ext, &zone_locks_grp, &zone_metadata_lock_attr); } +/* + * We're being very conservative here and picking a value of 95%. We might need to lower this if + * we find that we're not catching the problem and are still hitting zone map exhaustion panics. + */ +#define ZONE_MAP_JETSAM_LIMIT_DEFAULT 95 + +/* + * Trigger zone-map-exhaustion jetsams if the zone map is X% full, where X=zone_map_jetsam_limit. + * Can be set via boot-arg "zone_map_jetsam_limit". Set to 95% by default. + */ +unsigned int zone_map_jetsam_limit = ZONE_MAP_JETSAM_LIMIT_DEFAULT; + +/* + * Returns pid of the task with the largest number of VM map entries. + */ +extern pid_t find_largest_process_vm_map_entries(void); + +/* + * Callout to jetsam. If pid is -1, we wake up the memorystatus thread to do asynchronous kills. + * For any other pid we try to kill that process synchronously. + */ +boolean_t memorystatus_kill_on_zone_map_exhaustion(pid_t pid); + +void get_zone_map_size(uint64_t *current_size, uint64_t *capacity) +{ + *current_size = zone_map->size; + *capacity = vm_map_max(zone_map) - vm_map_min(zone_map); +} + +void get_largest_zone_info(char *zone_name, size_t zone_name_len, uint64_t *zone_size) +{ + zone_t largest_zone = zone_find_largest(); + strlcpy(zone_name, largest_zone->zone_name, zone_name_len); + *zone_size = largest_zone->cur_size; +} + +boolean_t is_zone_map_nearing_exhaustion(void) +{ + uint64_t size = zone_map->size; + uint64_t capacity = vm_map_max(zone_map) - vm_map_min(zone_map); + if (size > ((capacity * zone_map_jetsam_limit) / 100)) { + return TRUE; + } + return FALSE; +} + +extern zone_t vm_map_entry_zone; +extern zone_t vm_object_zone; + +#define VMENTRY_TO_VMOBJECT_COMPARISON_RATIO 98 + +/* + * Tries to kill a single process if it can attribute one to the largest zone. If not, wakes up the memorystatus thread + * to walk through the jetsam priority bands and kill processes. + */ +static void kill_process_in_largest_zone(void) +{ + pid_t pid = -1; + zone_t largest_zone = zone_find_largest(); + + printf("zone_map_exhaustion: Zone map size %lld, capacity %lld [jetsam limit %d%%]\n", (uint64_t)zone_map->size, + (uint64_t)(vm_map_max(zone_map) - vm_map_min(zone_map)), zone_map_jetsam_limit); + printf("zone_map_exhaustion: Largest zone %s, size %lu\n", largest_zone->zone_name, (uintptr_t)largest_zone->cur_size); + + /* + * We want to make sure we don't call this function from userspace. Or we could end up trying to synchronously kill the process + * whose context we're in, causing the system to hang. + */ + assert(current_task() == kernel_task); + + /* + * If vm_object_zone is the largest, check to see if the number of elements in vm_map_entry_zone is comparable. If so, consider + * vm_map_entry_zone as the largest. This lets us target a specific process to jetsam to quickly recover from the zone map bloat. + */ + if (largest_zone == vm_object_zone) { + int vm_object_zone_count = vm_object_zone->count; + int vm_map_entry_zone_count = vm_map_entry_zone->count; + /* Is the VM map entries zone count >= 98% of the VM objects zone count? */ + if (vm_map_entry_zone_count >= ((vm_object_zone_count * VMENTRY_TO_VMOBJECT_COMPARISON_RATIO) / 100)) { + largest_zone = vm_map_entry_zone; + printf("zone_map_exhaustion: Picking VM map entries as the zone to target, size %lu\n", (uintptr_t)largest_zone->cur_size); + } + } + + /* TODO: Extend this to check for the largest process in other zones as well. */ + if (largest_zone == vm_map_entry_zone) { + pid = find_largest_process_vm_map_entries(); + } else { + printf("zone_map_exhaustion: Nothing to do for the largest zone [%s]. Waking up memorystatus thread.\n", largest_zone->zone_name); + } + if (!memorystatus_kill_on_zone_map_exhaustion(pid)) { + printf("zone_map_exhaustion: Call to memorystatus failed, victim pid: %d\n", pid); + } +} + /* Global initialization of Zone Allocator. * Runs after zone_bootstrap. */ @@ -2122,9 +2934,16 @@ zone_init( vm_offset_t zone_max; vm_offset_t zone_metadata_space; unsigned int zone_pages; + vm_map_kernel_flags_t vmk_flags; + +#if VM_MAX_TAG_ZONES + if (zone_tagging_on) ztInit(max_zonemap_size, &zone_locks_grp); +#endif + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_permanent = TRUE; retval = kmem_suballoc(kernel_map, &zone_min, max_zonemap_size, - FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT | VM_MAKE_TAG(VM_KERN_MEMORY_ZONE), + FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_ZONE, &zone_map); if (retval != KERN_SUCCESS) @@ -2171,6 +2990,15 @@ zone_init( */ zleak_init(max_zonemap_size); #endif /* CONFIG_ZLEAKS */ + +#if VM_MAX_TAG_ZONES + if (zone_tagging_on) vm_allocation_zones_init(); +#endif + + int jetsam_limit_temp = 0; + if (PE_parse_boot_argn("zone_map_jetsam_limit", &jetsam_limit_temp, sizeof (jetsam_limit_temp)) && + jetsam_limit_temp > 0 && jetsam_limit_temp <= 100) + zone_map_jetsam_limit = jetsam_limit_temp; } extern volatile SInt32 kfree_nop_count; @@ -2178,6 +3006,8 @@ extern volatile SInt32 kfree_nop_count; #pragma mark - #pragma mark zalloc_canblock +extern boolean_t early_boot_complete; + /* * zalloc returns an element from the specified zone. */ @@ -2185,16 +3015,19 @@ static void * zalloc_internal( zone_t zone, boolean_t canblock, - boolean_t nopagewait) + boolean_t nopagewait, + vm_size_t +#if !VM_MAX_TAG_ZONES + __unused +#endif + reqsize, + vm_tag_t tag) { vm_offset_t addr = 0; kern_return_t retval; uintptr_t zbt[MAX_ZTRACE_DEPTH]; /* used in zone leak logging and zone leak detection */ int numsaved = 0; boolean_t zone_replenish_wakeup = FALSE, zone_alloc_throttle = FALSE; -#if CONFIG_GZALLOC - boolean_t did_gzalloc = FALSE; -#endif thread_t thr = current_thread(); boolean_t check_poison = FALSE; boolean_t set_doing_alloc_with_vm_priv = FALSE; @@ -2203,13 +3036,27 @@ zalloc_internal( uint32_t zleak_tracedepth = 0; /* log this allocation if nonzero */ #endif /* CONFIG_ZLEAKS */ +#if KASAN + /* + * KASan uses zalloc() for fakestack, which can be called anywhere. However, + * we make sure these calls can never block. + */ + boolean_t irq_safe = FALSE; + const char *fakestack_name = "fakestack."; + if (strncmp(zone->zone_name, fakestack_name, strlen(fakestack_name)) == 0) { + irq_safe = TRUE; + } +#elif MACH_ASSERT + /* In every other case, zalloc() from interrupt context is unsafe. */ + const boolean_t irq_safe = FALSE; +#endif + assert(zone != ZONE_NULL); + assert(irq_safe || ml_get_interrupts_enabled() || ml_is_quiescing() || debug_mode_active() || !early_boot_complete); #if CONFIG_GZALLOC addr = gzalloc_alloc(zone, canblock); - did_gzalloc = (addr != 0); #endif - /* * If zone logging is turned on and this is the zone we're tracking, grab a backtrace. */ @@ -2230,21 +3077,33 @@ zalloc_internal( } #endif /* CONFIG_ZLEAKS */ +#if VM_MAX_TAG_ZONES + if (__improbable(zone->tags)) vm_tag_will_update_zone(tag, zone->tag_zone_index); +#endif /* VM_MAX_TAG_ZONES */ + lock_zone(zone); + assert(zone->zone_valid); if (zone->async_prio_refill && zone->zone_replenish_thread) { - do { - vm_size_t zfreec = (zone->cur_size - (zone->count * zone->elem_size)); - vm_size_t zrefillwm = zone->prio_refill_watermark * zone->elem_size; - zone_replenish_wakeup = (zfreec < zrefillwm); - zone_alloc_throttle = (zfreec < (zrefillwm / 2)) && ((thr->options & TH_OPT_VMPRIV) == 0); + vm_size_t zfreec = (zone->cur_size - (zone->count * zone->elem_size)); + vm_size_t zrefillwm = zone->prio_refill_watermark * zone->elem_size; + zone_replenish_wakeup = (zfreec < zrefillwm); + zone_alloc_throttle = (((zfreec < (zrefillwm / 2)) && ((thr->options & TH_OPT_VMPRIV) == 0)) || (zfreec == 0)); + do { if (zone_replenish_wakeup) { zone_replenish_wakeups_initiated++; /* Signal the potentially waiting * refill thread. */ thread_wakeup(&zone->zone_replenish_thread); + + /* We don't want to wait around for zone_replenish_thread to bump up the free count + * if we're in zone_gc(). This keeps us from deadlocking with zone_replenish_thread. + */ + if (thr->options & TH_OPT_ZONE_GC) + break; + unlock_zone(zone); /* Scheduling latencies etc. may prevent * the refill thread from keeping up @@ -2258,13 +3117,27 @@ zalloc_internal( thread_block(THREAD_CONTINUE_NULL); } lock_zone(zone); + assert(zone->zone_valid); } + + zfreec = (zone->cur_size - (zone->count * zone->elem_size)); + zrefillwm = zone->prio_refill_watermark * zone->elem_size; + zone_replenish_wakeup = (zfreec < zrefillwm); + zone_alloc_throttle = (((zfreec < (zrefillwm / 2)) && ((thr->options & TH_OPT_VMPRIV) == 0)) || (zfreec == 0)); + } while (zone_alloc_throttle == TRUE); } if (__probable(addr == 0)) - addr = try_alloc_from_zone(zone, &check_poison); + addr = try_alloc_from_zone(zone, tag, &check_poison); + /* If we're here because of zone_gc(), we didn't wait for zone_replenish_thread to finish. + * So we need to ensure that we did successfully grab an element. And we only need to assert + * this for zones that have a replenish thread configured (in this case, the Reserved VM map + * entries zone). + */ + if (thr->options & TH_OPT_ZONE_GC && zone->async_prio_refill) + assert(addr != 0); while ((addr == 0) && canblock) { /* @@ -2350,6 +3223,11 @@ zalloc_internal( if (zone->noencrypt) zflags |= KMA_NOENCRYPT; + /* Trigger jetsams via the vm_pageout_garbage_collect thread if we're running out of zone memory */ + if (is_zone_map_nearing_exhaustion()) { + thread_wakeup((event_t) &vm_pageout_garbage_collect); + } + retval = kernel_memory_allocate(zone_map, &space, alloc_size, 0, zflags, VM_KERN_MEMORY_ZONE); if (retval == KERN_SUCCESS) { #if CONFIG_ZLEAKS @@ -2376,11 +3254,6 @@ zalloc_internal( } else if (retval != KERN_RESOURCE_SHORTAGE) { retry++; - if (retry == 2) { - zone_gc(); - printf("zalloc did gc\n"); - zone_display_zprint(); - } if (retry == 3) { panic_include_zprint = TRUE; #if CONFIG_ZLEAKS @@ -2402,6 +3275,7 @@ zalloc_internal( } } lock_zone(zone); + assert(zone->zone_valid); if (set_doing_alloc_with_vm_priv == TRUE) zone->doing_alloc_with_vm_priv = FALSE; @@ -2414,7 +3288,7 @@ zalloc_internal( } clear_thread_rwlock_boost(); - addr = try_alloc_from_zone(zone, &check_poison); + addr = try_alloc_from_zone(zone, tag, &check_poison); if (addr == 0 && retval == KERN_RESOURCE_SHORTAGE) { if (nopagewait == TRUE) @@ -2423,10 +3297,11 @@ zalloc_internal( VM_PAGE_WAIT(); lock_zone(zone); + assert(zone->zone_valid); } } if (addr == 0) - addr = try_alloc_from_zone(zone, &check_poison); + addr = try_alloc_from_zone(zone, tag, &check_poison); } #if CONFIG_ZLEAKS @@ -2448,13 +3323,21 @@ zalloc_internal( unlock_zone(zone); thread_call_enter(&call_async_alloc); lock_zone(zone); - addr = try_alloc_from_zone(zone, &check_poison); + assert(zone->zone_valid); + addr = try_alloc_from_zone(zone, tag, &check_poison); } - vm_offset_t inner_size = zone->elem_size; +#if VM_MAX_TAG_ZONES + if (__improbable(zone->tags) && addr) { + if (reqsize) reqsize = zone->elem_size - reqsize; + vm_tag_update_zone_size(tag, zone->tag_zone_index, zone->elem_size, reqsize); + } +#endif /* VM_MAX_TAG_ZONES */ unlock_zone(zone); + vm_offset_t inner_size = zone->elem_size; + if (__improbable(DO_LOGGING(zone) && addr)) { btlog_add_entry(zone->zlog_btlog, (void *)addr, ZOP_ALLOC, (void **)zbt, numsaved); } @@ -2497,32 +3380,46 @@ zalloc_internal( } TRACE_MACHLEAKS(ZALLOC_CODE, ZALLOC_CODE_2, zone->elem_size, addr); + +#if KASAN_ZALLOC + /* Fixup the return address to skip the redzone */ + if (zone->kasan_redzone) { + addr = kasan_alloc(addr, zone->elem_size, + zone->elem_size - 2 * zone->kasan_redzone, zone->kasan_redzone); + } +#endif + return((void *)addr); } - void * zalloc(zone_t zone) { - return (zalloc_internal(zone, TRUE, FALSE)); + return (zalloc_internal(zone, TRUE, FALSE, 0, VM_KERN_MEMORY_NONE)); } void * zalloc_noblock(zone_t zone) { - return (zalloc_internal(zone, FALSE, FALSE)); + return (zalloc_internal(zone, FALSE, FALSE, 0, VM_KERN_MEMORY_NONE)); } void * zalloc_nopagewait(zone_t zone) { - return (zalloc_internal(zone, TRUE, TRUE)); + return (zalloc_internal(zone, TRUE, TRUE, 0, VM_KERN_MEMORY_NONE)); +} + +void * +zalloc_canblock_tag(zone_t zone, boolean_t canblock, vm_size_t reqsize, vm_tag_t tag) +{ + return (zalloc_internal(zone, canblock, FALSE, reqsize, tag)); } void * zalloc_canblock(zone_t zone, boolean_t canblock) { - return (zalloc_internal(zone, canblock, FALSE)); + return (zalloc_internal(zone, canblock, FALSE, 0, VM_KERN_MEMORY_NONE)); } @@ -2541,15 +3438,21 @@ zalloc_async( simple_unlock(&all_zones_lock); for (i = 0; i < max_zones; i++) { current_z = &(zone_array[i]); + + if (current_z->no_callout == TRUE) { + /* async_pending will never be set */ + continue; + } + lock_zone(current_z); - if (current_z->async_pending == TRUE) { + if (current_z->zone_valid && current_z->async_pending == TRUE) { current_z->async_pending = FALSE; pending = TRUE; } unlock_zone(current_z); if (pending == TRUE) { - elt = zalloc_canblock(current_z, TRUE); + elt = zalloc_canblock_tag(current_z, TRUE, 0, VM_KERN_MEMORY_OSFMK); zfree(current_z, elt); pending = FALSE; } @@ -2564,7 +3467,7 @@ void * zget( zone_t zone) { - return zalloc_internal(zone, FALSE, TRUE); + return zalloc_internal(zone, FALSE, TRUE, 0, VM_KERN_MEMORY_NONE); } /* Keep this FALSE by default. Large memory machine run orders of magnitude @@ -2620,9 +3523,33 @@ zfree( int numsaved = 0; boolean_t gzfreed = FALSE; boolean_t poison = FALSE; +#if VM_MAX_TAG_ZONES + vm_tag_t tag; +#endif /* VM_MAX_TAG_ZONES */ assert(zone != ZONE_NULL); +#if KASAN_ZALLOC + /* + * Resize back to the real allocation size and hand off to the KASan + * quarantine. `addr` may then point to a different allocation. + */ + vm_size_t usersz = zone->elem_size - 2 * zone->kasan_redzone; + vm_size_t sz = usersz; + if (addr && zone->kasan_redzone) { + kasan_check_free((vm_address_t)addr, usersz, KASAN_HEAP_ZALLOC); + addr = (void *)kasan_dealloc((vm_address_t)addr, &sz); + assert(sz == zone->elem_size); + } + if (addr && zone->kasan_quarantine) { + kasan_free(&addr, &sz, KASAN_HEAP_ZALLOC, &zone, usersz, true); + if (!addr) { + return; + } + } + elem = (vm_offset_t)addr; +#endif + /* * If zone logging is turned on and this is the zone we're tracking, grab a backtrace. */ @@ -2708,13 +3635,22 @@ zfree( } lock_zone(zone); + assert(zone->zone_valid); if (zone_check) { zone_check_freelist(zone, elem); } - if (__probable(!gzfreed)) + if (__probable(!gzfreed)) { +#if VM_MAX_TAG_ZONES + if (__improbable(zone->tags)) { + tag = (ZTAG(zone, elem)[0] >> 1); + // set the tag with b0 clear so the block remains inuse + ZTAG(zone, elem)[0] = 0xFFFE; + } +#endif /* VM_MAX_TAG_ZONES */ free_to_zone(zone, elem, poison); + } #if MACH_ASSERT if (zone->count < 0) @@ -2732,10 +3668,15 @@ zfree( } #endif /* CONFIG_ZLEAKS */ +#if VM_MAX_TAG_ZONES + if (__improbable(zone->tags) && __probable(!gzfreed)) { + vm_tag_update_zone_size(tag, zone->tag_zone_index, -((int64_t)zone->elem_size), 0); + } +#endif /* VM_MAX_TAG_ZONES */ + unlock_zone(zone); } - /* Change a zone's flags. * This routine must be called immediately after zinit. */ @@ -2770,6 +3711,16 @@ zone_change( case Z_NOCALLOUT: zone->no_callout = value; break; + case Z_TAGS_ENABLED: +#if VM_MAX_TAG_ZONES + { + static int tag_zone_index; + zone->tags = TRUE; + zone->tags_inline = (((page_size + zone->elem_size - 1) / zone->elem_size) <= (sizeof(uint32_t) / sizeof(uint16_t))); + zone->tag_zone_index = OSAddAtomic(1, &tag_zone_index); + } +#endif /* VM_MAX_TAG_ZONES */ + break; case Z_GZALLOC_EXEMPT: zone->gzalloc_exempt = value; #if CONFIG_GZALLOC @@ -2778,10 +3729,21 @@ zone_change( break; case Z_ALIGNMENT_REQUIRED: zone->alignment_required = value; +#if KASAN_ZALLOC + if (zone->kasan_redzone == KASAN_GUARD_SIZE) { + /* Don't disturb alignment with the redzone for zones with + * specific alignment requirements. */ + zone->elem_size -= zone->kasan_redzone * 2; + zone->kasan_redzone = 0; + } +#endif #if CONFIG_GZALLOC gzalloc_reconfigure(zone); #endif break; + case Z_KASAN_QUARANTINE: + zone->kasan_quarantine = value; + break; default: panic("Zone_change: Wrong Item Type!"); /* break; */ @@ -2809,26 +3771,100 @@ zone_free_count(zone_t zone) return(free_count); } +/* Drops the elements in the free queue of a zone. Called by zone_gc() on each zone, and when a zone is zdestroy'ed. */ +void +drop_free_elements(zone_t z) +{ + vm_size_t elt_size, size_freed; + int total_freed_pages = 0; + uint64_t old_all_free_count; + struct zone_page_metadata *page_meta; + queue_head_t page_meta_head; + + lock_zone(z); + if (queue_empty(&z->pages.all_free)) { + unlock_zone(z); + return; + } + + /* + * Snatch all of the free elements away from the zone. + */ + elt_size = z->elem_size; + old_all_free_count = z->count_all_free_pages; + queue_new_head(&z->pages.all_free, &page_meta_head, struct zone_page_metadata *, pages); + queue_init(&z->pages.all_free); + z->count_all_free_pages = 0; + unlock_zone(z); + + /* Iterate through all elements to find out size and count of elements we snatched */ + size_freed = 0; + queue_iterate(&page_meta_head, page_meta, struct zone_page_metadata *, pages) { + assert(from_zone_map((vm_address_t)page_meta, sizeof(*page_meta))); /* foreign elements should be in any_free_foreign */ + size_freed += elt_size * page_meta->free_count; + } + + /* Update the zone size and free element count */ + lock_zone(z); + z->cur_size -= size_freed; + z->countfree -= size_freed/elt_size; + unlock_zone(z); + + while ((page_meta = (struct zone_page_metadata *)dequeue_head(&page_meta_head)) != NULL) { + vm_address_t free_page_address; + /* Free the pages for metadata and account for them */ + free_page_address = get_zone_page(page_meta); + ZONE_PAGE_COUNT_DECR(z, page_meta->page_count); + total_freed_pages += page_meta->page_count; + old_all_free_count -= page_meta->page_count; +#if KASAN_ZALLOC + kasan_poison_range(free_page_address, page_meta->page_count * PAGE_SIZE, ASAN_VALID); +#endif +#if VM_MAX_TAG_ZONES + if (z->tags) ztMemoryRemove(z, free_page_address, (page_meta->page_count * PAGE_SIZE)); +#endif /* VM_MAX_TAG_ZONES */ + kmem_free(zone_map, free_page_address, (page_meta->page_count * PAGE_SIZE)); + if (current_thread()->options & TH_OPT_ZONE_GC) { + thread_yield_to_preemption(); + } + } + + /* We freed all the pages from the all_free list for this zone */ + assert(old_all_free_count == 0); + + if (zalloc_debug & ZALLOC_DEBUG_ZONEGC) + kprintf("zone_gc() of zone %s freed %lu elements, %d pages\n", z->zone_name, (unsigned long)size_freed/elt_size, total_freed_pages); +} + /* Zone garbage collection * * zone_gc will walk through all the free elements in all the * zones that are marked collectable looking for reclaimable * pages. zone_gc is called by consider_zone_gc when the system * begins to run out of memory. + * + * We should ensure that zone_gc never blocks. */ -extern zone_t vm_map_entry_reserved_zone; -uint64_t zone_gc_bailed = 0; - void -zone_gc(void) +zone_gc(boolean_t consider_jetsams) { unsigned int max_zones; zone_t z; unsigned int i; - zone_t zres = vm_map_entry_reserved_zone; + + if (consider_jetsams) { + kill_process_in_largest_zone(); + /* + * If we do end up jetsamming something, we need to do a zone_gc so that + * we can reclaim free zone elements and update the zone map size. + * Fall through. + */ + } lck_mtx_lock(&zone_gc_lock); + current_thread()->options |= TH_OPT_ZONE_GC; + simple_lock(&all_zones_lock); max_zones = num_zones; simple_unlock(&all_zones_lock); @@ -2838,102 +3874,21 @@ zone_gc(void) for (i = 0; i < max_zones; i++) { z = &(zone_array[i]); - vm_size_t elt_size, size_freed; - int total_freed_pages = 0; - struct zone_page_metadata *page_meta; - queue_head_t page_meta_head; - assert(z != ZONE_NULL); - if (!z->collectable) - continue; - - if (queue_empty(&z->pages.all_free)) { + if (!z->collectable) { continue; } - /* - * Since kmem_free() might use VM entries from the reserved VM entries zone, we should bail from zone_gc() if we - * are below the critical threshold for that zone. Otherwise, there could be a deadlock between the zone_gc - * thread and the zone_replenish thread for the VM entries zone on the zone_map lock. - */ - if (zres->zone_replenishing) { - zone_gc_bailed++; - break; - } - - lock_zone(z); - elt_size = z->elem_size; - if (queue_empty(&z->pages.all_free)) { - unlock_zone(z); continue; } - - /* - * Snatch all of the free elements away from the zone. - */ - uint64_t old_all_free_count = z->count_all_free_pages; - queue_new_head(&z->pages.all_free, &page_meta_head, struct zone_page_metadata *, pages); - queue_init(&z->pages.all_free); - z->count_all_free_pages = 0; - unlock_zone(z); - - /* Iterate through all elements to find out size and count of elements we snatched */ - size_freed = 0; - queue_iterate(&page_meta_head, page_meta, struct zone_page_metadata *, pages) { - assert(from_zone_map((vm_address_t)page_meta, sizeof(*page_meta))); /* foreign elements should be in any_free_foreign */ - size_freed += elt_size * page_meta->free_count; - } - - /* Update the zone size and free element count */ - lock_zone(z); - z->cur_size -= size_freed; - z->countfree -= size_freed/elt_size; - unlock_zone(z); - - while ((page_meta = (struct zone_page_metadata *)dequeue_head(&page_meta_head)) != NULL) { - vm_address_t free_page_address; - if (zres->zone_replenishing) - break; - /* Free the pages for metadata and account for them */ - free_page_address = get_zone_page(page_meta); - ZONE_PAGE_COUNT_DECR(z, page_meta->page_count); - total_freed_pages += page_meta->page_count; - old_all_free_count -= page_meta->page_count; - size_freed -= (elt_size * page_meta->free_count); - kmem_free(zone_map, free_page_address, (page_meta->page_count * PAGE_SIZE)); - thread_yield_to_preemption(); - } - if (page_meta != NULL) { - /* - * We bailed because the VM entry reserved zone is replenishing. Put the remaining - * metadata objects back on the all_free list and bail. - */ - queue_entry_t qe; - enqueue_head(&page_meta_head, &(page_meta->pages)); - zone_gc_bailed++; - - lock_zone(z); - qe_foreach_safe(qe, &page_meta_head) { - re_queue_tail(&z->pages.all_free, qe); - } - z->count_all_free_pages += (int)old_all_free_count; - z->cur_size += size_freed; - z->countfree += size_freed/elt_size; - unlock_zone(z); - if (zalloc_debug & ZALLOC_DEBUG_ZONEGC) - kprintf("zone_gc() bailed due to VM entry zone replenishing (zone_gc_bailed: %lld)\n", zone_gc_bailed); - break; - } - /* We freed all the pages from the all_free list for this zone */ - assert(old_all_free_count == 0); - - if (zalloc_debug & ZALLOC_DEBUG_ZONEGC) - kprintf("zone_gc() of zone %s freed %lu elements, %d pages\n", z->zone_name, (unsigned long)size_freed/elt_size, total_freed_pages); + drop_free_elements(z); } + current_thread()->options &= ~TH_OPT_ZONE_GC; + lck_mtx_unlock(&zone_gc_lock); } @@ -2947,7 +3902,7 @@ extern unsigned int kmapoff_pgcnt; */ void -consider_zone_gc(void) +consider_zone_gc(boolean_t consider_jetsams) { if (kmapoff_kaddr != 0) { /* @@ -2960,7 +3915,7 @@ consider_zone_gc(void) } if (zone_gc_allowed) - zone_gc(); + zone_gc(consider_jetsams); } kern_return_t @@ -2986,17 +3941,6 @@ mach_zone_info( } -kern_return_t -host_zone_info( - host_priv_t host, - zone_name_array_t *namesp, - mach_msg_type_number_t *namesCntp, - zone_info_array_t *infop, - mach_msg_type_number_t *infoCntp) -{ - return (mach_memory_info(host, (mach_zone_name_array_t *)namesp, namesCntp, (mach_zone_info_array_t *)infop, infoCntp, NULL, NULL)); -} - kern_return_t mach_memory_info( host_priv_t host, @@ -3019,9 +3963,9 @@ mach_memory_info( vm_offset_t memory_info_addr; vm_size_t memory_info_size; vm_size_t memory_info_vmsize; - unsigned int num_sites; + unsigned int num_info; - unsigned int max_zones, i; + unsigned int max_zones, used_zones, i; zone_t z; mach_zone_name_t *zn; mach_zone_info_t *zi; @@ -3067,17 +4011,23 @@ mach_memory_info( zn = &names[0]; zi = &info[0]; + used_zones = max_zones; for (i = 0; i < max_zones; i++) { struct zone zcopy; z = &(zone_array[i]); assert(z != ZONE_NULL); lock_zone(z); + if (!z->zone_valid) { + unlock_zone(z); + used_zones--; + continue; + } zcopy = *z; unlock_zone(z); /* assuming here the name data is static */ - (void) strncpy(zn->mzn_name, zcopy.zone_name, + (void) __nosan_strncpy(zn->mzn_name, zcopy.zone_name, sizeof zn->mzn_name); zn->mzn_name[sizeof zn->mzn_name - 1] = '\0'; @@ -3094,7 +4044,7 @@ mach_memory_info( zi++; } - used = max_zones * sizeof *names; + used = used_zones * sizeof *names; if (used != names_size) bzero((char *) (names_addr + used), names_size - used); @@ -3103,9 +4053,9 @@ mach_memory_info( assert(kr == KERN_SUCCESS); *namesp = (mach_zone_name_t *) copy; - *namesCntp = max_zones; + *namesCntp = used_zones; - used = max_zones * sizeof *info; + used = used_zones * sizeof *info; if (used != info_size) bzero((char *) (info_addr + used), info_size - used); @@ -3115,15 +4065,15 @@ mach_memory_info( assert(kr == KERN_SUCCESS); *infop = (mach_zone_info_t *) copy; - *infoCntp = max_zones; + *infoCntp = used_zones; - num_sites = 0; + num_info = 0; memory_info_addr = 0; if (memoryInfop && memoryInfoCntp) { - num_sites = VM_KERN_MEMORY_COUNT + VM_KERN_COUNTER_COUNT; - memory_info_size = num_sites * sizeof(*info); + num_info = vm_page_diagnose_estimate(); + memory_info_size = num_info * sizeof(*memory_info); memory_info_vmsize = round_page(memory_info_size); kr = kmem_alloc_pageable(ipc_kernel_map, &memory_info_addr, memory_info_vmsize, VM_KERN_MEMORY_IPC); @@ -3135,12 +4085,12 @@ mach_memory_info( return kr; } - kr = vm_map_wire(ipc_kernel_map, memory_info_addr, memory_info_addr + memory_info_vmsize, - VM_PROT_READ|VM_PROT_WRITE|VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_IPC), FALSE); + kr = vm_map_wire_kernel(ipc_kernel_map, memory_info_addr, memory_info_addr + memory_info_vmsize, + VM_PROT_READ|VM_PROT_WRITE, VM_KERN_MEMORY_IPC, FALSE); assert(kr == KERN_SUCCESS); memory_info = (mach_memory_info_t *) memory_info_addr; - vm_page_diagnose(memory_info, num_sites, zones_collectable_bytes); + vm_page_diagnose(memory_info, num_info, zones_collectable_bytes); kr = vm_map_unwire(ipc_kernel_map, memory_info_addr, memory_info_addr + memory_info_vmsize, FALSE); assert(kr == KERN_SUCCESS); @@ -3150,22 +4100,101 @@ mach_memory_info( assert(kr == KERN_SUCCESS); *memoryInfop = (mach_memory_info_t *) copy; - *memoryInfoCntp = num_sites; + *memoryInfoCntp = num_info; } return KERN_SUCCESS; } +uint64_t +get_zones_collectable_bytes(void) +{ + zone_t z; + unsigned int i, max_zones; + uint64_t zones_collectable_bytes = 0; + + simple_lock(&all_zones_lock); + max_zones = (unsigned int)(num_zones); + simple_unlock(&all_zones_lock); + + for (i = 0; i < max_zones; i++) { + z = &(zone_array[i]); + assert(z != ZONE_NULL); + + lock_zone(z); + zones_collectable_bytes += ((uint64_t)z->count_all_free_pages * PAGE_SIZE); + unlock_zone(z); + } + + return zones_collectable_bytes; +} + +#if DEBUG || DEVELOPMENT + +kern_return_t +mach_memory_info_check(void) +{ + mach_memory_info_t * memory_info; + mach_memory_info_t * info; + zone_t zone; + unsigned int idx, num_info, max_zones; + vm_offset_t memory_info_addr; + kern_return_t kr; + size_t memory_info_size, memory_info_vmsize; + uint64_t top_wired, zonestotal, total; + + num_info = vm_page_diagnose_estimate(); + memory_info_size = num_info * sizeof(*memory_info); + memory_info_vmsize = round_page(memory_info_size); + kr = kmem_alloc(kernel_map, &memory_info_addr, memory_info_vmsize, VM_KERN_MEMORY_DIAG); + assert (kr == KERN_SUCCESS); + + memory_info = (mach_memory_info_t *) memory_info_addr; + vm_page_diagnose(memory_info, num_info, 0); + + simple_lock(&all_zones_lock); + max_zones = num_zones; + simple_unlock(&all_zones_lock); + + top_wired = total = zonestotal = 0; + for (idx = 0; idx < max_zones; idx++) + { + zone = &(zone_array[idx]); + assert(zone != ZONE_NULL); + lock_zone(zone); + zonestotal += ptoa_64(zone->page_count); + unlock_zone(zone); + } + for (idx = 0; idx < num_info; idx++) + { + info = &memory_info[idx]; + if (!info->size) continue; + if (VM_KERN_COUNT_WIRED == info->site) top_wired = info->size; + if (VM_KERN_SITE_HIDE & info->flags) continue; + if (!(VM_KERN_SITE_WIRED & info->flags)) continue; + total += info->size; + } + total += zonestotal; + + printf("vm_page_diagnose_check %qd of %qd, zones %qd, short 0x%qx\n", total, top_wired, zonestotal, top_wired - total); + + kmem_free(kernel_map, memory_info_addr, memory_info_vmsize); + + return (kr); +} + +#endif /* DEBUG || DEVELOPMENT */ + kern_return_t mach_zone_force_gc( host_t host) { - if (host == HOST_NULL) return KERN_INVALID_HOST; - consider_zone_gc(); - +#if DEBUG || DEVELOPMENT + consider_zone_gc(FALSE); +#endif /* DEBUG || DEVELOPMENT */ return (KERN_SUCCESS); } @@ -3177,26 +4206,6 @@ extern unsigned int inuse_ptepages_count; extern long long alloc_ptepages_count; #endif -void zone_display_zprint() -{ - unsigned int i; - zone_t the_zone; - - for (i = 0; i < num_zones; i++) { - the_zone = &(zone_array[i]); - if(the_zone->cur_size > (1024*1024)) { - printf("%.20s:\t%lu\n",the_zone->zone_name,(uintptr_t)the_zone->cur_size); - } - } - printf("Kernel Stacks:\t%lu\n",(uintptr_t)(kernel_stack_size * stack_total)); - -#if defined(__i386__) || defined (__x86_64__) - printf("PageTables:\t%lu\n",(uintptr_t)(PAGE_SIZE * inuse_ptepages_count)); -#endif - - printf("Kalloc.Large:\t%lu\n",(uintptr_t)kalloc_large_total); -} - zone_t zone_find_largest(void) { @@ -3293,13 +4302,18 @@ zone_leaks(const char * zoneName, uint32_t nameLen, leak_site_proc proc, void * uint32_t btidx, btcount, nobtcount, btfound; uint32_t elemSize; uint64_t maxElems; - kern_return_t kr; + unsigned int max_zones; + kern_return_t kr; - for (idx = 0; idx < num_zones; idx++) + simple_lock(&all_zones_lock); + max_zones = num_zones; + simple_unlock(&all_zones_lock); + + for (idx = 0; idx < max_zones; idx++) { if (!strncmp(zoneName, zone_array[idx].zone_name, nameLen)) break; } - if (idx >= num_zones) return (KERN_INVALID_NAME); + if (idx >= max_zones) return (KERN_INVALID_NAME); zone = &zone_array[idx]; elemSize = (uint32_t) zone->elem_size; @@ -3369,41 +4383,78 @@ zone_leaks(const char * zoneName, uint32_t nameLen, leak_site_proc proc, void * return (KERN_SUCCESS); } -void -kern_wired_diagnose(void) +boolean_t +kdp_is_in_zone(void *addr, const char *zone_name) { - unsigned int count = VM_KERN_MEMORY_COUNT + VM_KERN_COUNTER_COUNT; - mach_memory_info_t info[count]; - unsigned int idx; - uint64_t total_zone, total_wired, top_wired, osfmk_wired; + zone_t z; + return (zone_element_size(addr, &z) && !strcmp(z->zone_name, zone_name)); +} - if (KERN_SUCCESS != vm_page_diagnose(info, count, 0)) return; +boolean_t +run_zone_test(void) +{ + int i = 0, max_iter = 5; + void * test_ptr; + zone_t test_zone; - total_zone = total_wired = top_wired = osfmk_wired = 0; - for (idx = 0; idx < num_zones; idx++) - { - total_zone += ptoa_64(zone_array[idx].page_count); - } - total_wired = total_zone; + simple_lock(&zone_test_lock); + if (!zone_test_running) { + zone_test_running = TRUE; + } else { + simple_unlock(&zone_test_lock); + printf("run_zone_test: Test already running.\n"); + return FALSE; + } + simple_unlock(&zone_test_lock); - for (idx = 0; idx < count; idx++) - { - if (VM_KERN_COUNT_WIRED == info[idx].site) top_wired = info[idx].size; - if (VM_KERN_MEMORY_OSFMK == info[idx].site) osfmk_wired = info[idx].size; - if (VM_KERN_SITE_HIDE & info[idx].flags) continue; - if (!(VM_KERN_SITE_WIRED & info[idx].flags)) continue; - total_wired += info[idx].size; - } + printf("run_zone_test: Testing zinit(), zalloc(), zfree() and zdestroy() on zone \"test_zone_sysctl\"\n"); - printf("top 0x%qx, total 0x%qx, zone 0x%qx, osfmk 0x%qx\n", - top_wired, total_wired, total_zone, osfmk_wired); -} + /* zinit() and zdestroy() a zone with the same name a bunch of times, verify that we get back the same zone each time */ + do { + test_zone = zinit(sizeof(uint64_t), 100 * sizeof(uint64_t), sizeof(uint64_t), "test_zone_sysctl"); + if (test_zone == NULL) { + printf("run_zone_test: zinit() failed\n"); + return FALSE; + } -boolean_t -kdp_is_in_zone(void *addr, const char *zone_name) -{ - zone_t z; - return (zone_element_size(addr, &z) && !strcmp(z->zone_name, zone_name)); +#if KASAN_ZALLOC + if (test_zone_ptr == NULL && zone_free_count(test_zone) != 0) { +#else + if (zone_free_count(test_zone) != 0) { +#endif + printf("run_zone_test: free count is not zero\n"); + return FALSE; + } + + if (test_zone_ptr == NULL) { + /* Stash the zone pointer returned on the fist zinit */ + printf("run_zone_test: zone created for the first time\n"); + test_zone_ptr = test_zone; + } else if (test_zone != test_zone_ptr) { + printf("run_zone_test: old zone pointer and new zone pointer don't match\n"); + return FALSE; + } + + test_ptr = zalloc(test_zone); + if (test_ptr == NULL) { + printf("run_zone_test: zalloc() failed\n"); + return FALSE; + } + zfree(test_zone, test_ptr); + + zdestroy(test_zone); + i++; + + printf("run_zone_test: Iteration %d successful\n", i); + } while (i < max_iter); + + printf("run_zone_test: Test passed\n"); + + simple_lock(&zone_test_lock); + zone_test_running = FALSE; + simple_unlock(&zone_test_lock); + + return TRUE; } #endif /* DEBUG || DEVELOPMENT */ diff --git a/osfmk/kern/zalloc.h b/osfmk/kern/zalloc.h index 787bfc815..6a585b83f 100644 --- a/osfmk/kern/zalloc.h +++ b/osfmk/kern/zalloc.h @@ -79,6 +79,11 @@ #include #include +#if KASAN +#include +#include +#endif + #if CONFIG_GZALLOC typedef struct gzalloc_data { uint32_t gzfc_index; @@ -135,7 +140,12 @@ struct zone { /* boolean_t */ alignment_required :1, /* boolean_t */ zone_logging :1, /* Enable zone logging for this zone. */ /* boolean_t */ zone_replenishing :1, - /* future */ _reserved :15; + /* boolean_t */ kasan_quarantine :1, + /* boolean_t */ tags :1, + /* boolean_t */ tags_inline :1, + /* future */ tag_zone_index :6, + /* boolean_t */ zone_valid :1, + /* future */ _reserved :5; int index; /* index into zone_info arrays for this zone */ const char *zone_name; /* a name for the zone */ @@ -150,6 +160,10 @@ struct zone { gzalloc_data_t gz; #endif /* CONFIG_GZALLOC */ +#if KASAN_ZALLOC + vm_size_t kasan_redzone; +#endif + btlog_t *zlog_btlog; /* zone logging structure to hold stacks and element references to those stacks. */ }; @@ -163,8 +177,25 @@ typedef struct zinfo_usage_store_t { uint64_t free __attribute__((aligned(8))); /* free counter */ } zinfo_usage_store_t; -extern void zone_gc(void); -extern void consider_zone_gc(void); +/* + * For sysctl kern.zones_collectable_bytes used by memory_maintenance to check if a + * userspace reboot is needed. The only other way to query for this information + * is via mach_memory_info() which is unavailable on release kernels. + */ +extern uint64_t get_zones_collectable_bytes(void); + +/* + * zone_gc also checks if the zone_map is getting close to full and triggers jetsams if needed, provided + * consider_jetsams is set to TRUE. To avoid deadlocks, we only pass a value of TRUE from within the + * vm_pageout_garbage_collect thread. + */ +extern void zone_gc(boolean_t consider_jetsams); +extern void consider_zone_gc(boolean_t consider_jetsams); +extern void drop_free_elements(zone_t z); + +/* Debug logging for zone-map-exhaustion jetsams. */ +extern void get_zone_map_size(uint64_t *current_size, uint64_t *capacity); +extern void get_largest_zone_info(char *zone_name, size_t zone_name_len, uint64_t *zone_size); /* Bootstrap zone module (create zone zone) */ extern void zone_bootstrap(void); @@ -202,6 +233,21 @@ extern void zone_debug_disable( extern unsigned int num_zones; extern struct zone zone_array[]; +/* zindex and page_count must pack into 16 bits + * update tools/lldbmacros/memory.py:GetRealMetadata + * when these values change */ + +#define ZINDEX_BITS (10U) +#define PAGECOUNT_BITS (16U - ZINDEX_BITS) +#define MULTIPAGE_METADATA_MAGIC ((1UL << ZINDEX_BITS) - 1) +#define ZONE_CHUNK_MAXPAGES ((1UL << PAGECOUNT_BITS) - 1) + +/* + * The max # of elements in a chunk should fit into zone_page_metadata.free_count (uint16_t). + * Update this if the type of free_count changes. + */ +#define ZONE_CHUNK_MAXELEMENTS (UINT16_MAX) + #endif /* MACH_KERNEL_PRIVATE */ __BEGIN_DECLS @@ -217,8 +263,10 @@ __BEGIN_DECLS #define Z_NOCALLOUT 7 /* Don't asynchronously replenish the zone via callouts */ #define Z_ALIGNMENT_REQUIRED 8 #define Z_GZALLOC_EXEMPT 9 /* Not tracked in guard allocation mode */ - - +#define Z_KASAN_QUARANTINE 10 /* Allow zone elements to be quarantined on free */ +#ifdef XNU_KERNEL_PRIVATE +#define Z_TAGS_ENABLED 11 /* Store tags */ +#endif /* XNU_KERNEL_PRIVATE */ #ifdef XNU_KERNEL_PRIVATE @@ -230,15 +278,18 @@ extern vm_offset_t zone_map_max_address; extern void * zalloc_nopagewait( zone_t zone); -/* Non-blocking version of zalloc */ -extern void * zalloc_noblock( - zone_t zone); - /* selective version of zalloc */ extern void * zalloc_canblock( zone_t zone, boolean_t canblock); +/* selective version of zalloc */ +extern void * zalloc_canblock_tag( + zone_t zone, + boolean_t canblock, + vm_size_t reqsize, + vm_tag_t tag); + /* Get from zone free list */ extern void * zget( zone_t zone); @@ -306,6 +357,17 @@ extern int get_zleak_state(void); #endif /* CONFIG_ZLEAKS */ +#ifndef VM_MAX_TAG_ZONES +#error MAX_TAG_ZONES +#endif + +#if VM_MAX_TAG_ZONES + +extern boolean_t zone_tagging_on; +extern uint32_t zone_index_from_tag_index(uint32_t tag_zone_index, vm_size_t * elem_size); + +#endif /* VM_MAX_TAG_ZONES */ + /* These functions used for leak detection both in zalloc.c and mbuf.c */ extern uintptr_t hash_mix(uintptr_t); extern uint32_t hashbacktrace(uintptr_t *, uint32_t, uint32_t); @@ -313,7 +375,7 @@ extern uint32_t hashaddr(uintptr_t, uint32_t); #define lock_zone(zone) \ MACRO_BEGIN \ - lck_mtx_lock_spin(&(zone)->lock); \ + lck_mtx_lock_spin_always(&(zone)->lock); \ MACRO_END #define unlock_zone(zone) \ @@ -326,6 +388,7 @@ void gzalloc_init(vm_size_t); void gzalloc_zone_init(zone_t); void gzalloc_configure(void); void gzalloc_reconfigure(zone_t); +void gzalloc_empty_free_cache(zone_t); boolean_t gzalloc_enabled(void); vm_offset_t gzalloc_alloc(zone_t, boolean_t); @@ -337,12 +400,26 @@ boolean_t gzalloc_element_size(void *, zone_t *, vm_size_t *); void zlog_btlog_lock(__unused void *); void zlog_btlog_unlock(__unused void *); +#ifdef MACH_KERNEL_PRIVATE +#define MAX_ZONE_NAME 32 /* max length of a zone name we can take from the boot-args */ +int track_this_zone(const char *zonename, const char *logname); +#endif + +#if DEBUG || DEVELOPMENT +extern boolean_t run_zone_test(void); +extern vm_size_t zone_element_info(void *addr, vm_tag_t * ptag); +#endif /* DEBUG || DEVELOPMENT */ + #endif /* XNU_KERNEL_PRIVATE */ /* Allocate from zone */ extern void * zalloc( zone_t zone); +/* Non-blocking version of zalloc */ +extern void * zalloc_noblock( + zone_t zone); + /* Free zone element */ extern void zfree( zone_t zone, @@ -361,6 +438,9 @@ extern void zone_change( unsigned int item, boolean_t value); +/* Destroy the zone */ +extern void zdestroy( + zone_t zone); __END_DECLS diff --git a/osfmk/kperf/action.c b/osfmk/kperf/action.c index 34098fa93..ab33ded44 100644 --- a/osfmk/kperf/action.c +++ b/osfmk/kperf/action.c @@ -38,6 +38,8 @@ #include #include #include +#include +#include #include #include @@ -71,6 +73,29 @@ static struct action *actionv = NULL; /* should emit tracepoint on context switch */ int kperf_kdebug_cswitch = 0; +bool +kperf_sample_has_non_system(unsigned actionid) +{ + if (actionid > actionc) { + return false; + } + + if (actionv[actionid - 1].sample & ~SAMPLER_SYS_MEM) { + return true; + } else { + return false; + } +} + +static void +kperf_system_memory_log(void) +{ + BUF_DATA(PERF_MI_SYS_DATA, (uintptr_t)vm_page_free_count, + (uintptr_t)vm_page_wire_count, (uintptr_t)vm_page_external_count, + (uintptr_t)(vm_page_active_count + vm_page_inactive_count + + vm_page_speculative_count)); +} + static kern_return_t kperf_sample_internal(struct kperf_sample *sbuf, struct kperf_context *context, @@ -79,6 +104,8 @@ kperf_sample_internal(struct kperf_sample *sbuf, { int pended_ucallstack = 0; int pended_th_dispatch = 0; + bool on_idle_thread = false; + uint32_t userdata = actionid; /* not much point continuing here, but what to do ? return * Shutdown? cut a tracepoint and continue? @@ -92,6 +119,10 @@ kperf_sample_internal(struct kperf_sample *sbuf, sample_what &= ~(SAMPLER_KSTACK | SAMPLER_USTACK); } + if (sample_flags & SAMPLE_FLAG_ONLY_SYSTEM) { + sample_what &= SAMPLER_SYS_MEM; + } + context->cur_thread->kperf_pet_gen = kperf_pet_gen; boolean_t is_kernel = (context->cur_pid == 0); @@ -121,7 +152,8 @@ kperf_sample_internal(struct kperf_sample *sbuf, /* See if we should drop idle thread samples */ if (!(sample_flags & SAMPLE_FLAG_IDLE_THREADS)) { if (sbuf->th_info.kpthi_runmode & 0x40) { - return SAMPLE_CONTINUE; + on_idle_thread = true; + goto log_sample; } } } @@ -177,12 +209,10 @@ kperf_sample_internal(struct kperf_sample *sbuf, kperf_kpc_cpu_sample(&(sbuf->kpcdata), sample_what); } +log_sample: /* lookup the user tag, if any */ - uint32_t userdata; if (actionid && (actionid <= actionc)) { userdata = actionv[actionid - 1].userdata; - } else { - userdata = actionid; } /* avoid logging if this sample only pended samples */ @@ -200,6 +230,15 @@ kperf_sample_internal(struct kperf_sample *sbuf, BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, sample_what, actionid, userdata, sample_flags); + if (sample_flags & SAMPLE_FLAG_SYSTEM) { + if (sample_what & SAMPLER_SYS_MEM) { + kperf_system_memory_log(); + } + } + if (on_idle_thread) { + goto log_sample_end; + } + if (sample_what & SAMPLER_TH_INFO) { kperf_thread_info_log(&sbuf->th_info); } @@ -212,6 +251,9 @@ kperf_sample_internal(struct kperf_sample *sbuf, if (sample_what & SAMPLER_KSTACK) { kperf_kcallstack_log(&sbuf->kcallstack); } + if (sample_what & SAMPLER_TH_INSCYC) { + kperf_thread_inscyc_log(context); + } if (sample_what & SAMPLER_TK_SNAPSHOT) { kperf_task_snapshot_log(&(sbuf->tk_snapshot)); } @@ -248,7 +290,8 @@ kperf_sample_internal(struct kperf_sample *sbuf, kperf_kpc_cpu_log(&(sbuf->kpcdata)); } - BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what); +log_sample_end: + BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what, on_idle_thread ? 1 : 0); /* intrs back on */ ml_set_interrupts_enabled(enabled); diff --git a/osfmk/kperf/action.h b/osfmk/kperf/action.h index 1233da6d0..f4e2e72bd 100644 --- a/osfmk/kperf/action.h +++ b/osfmk/kperf/action.h @@ -30,6 +30,7 @@ #define KPERF_ACTION_H #include +#include /* fwd decl */ struct kperf_sample; @@ -47,10 +48,12 @@ struct kperf_context; #define SAMPLER_TH_SCHEDULING (1U << 8) #define SAMPLER_TH_DISPATCH (1U << 9) #define SAMPLER_TK_SNAPSHOT (1U << 10) +#define SAMPLER_SYS_MEM (1U << 11) +#define SAMPLER_TH_INSCYC (1U << 12) /* flags for sample calls */ -/* pend certain samplers until AST boundary, instead of sampling them */ +/* pend samplers requiring copyin until AST boundary */ #define SAMPLE_FLAG_PEND_USER (1U << 0) /* sample idle threads */ #define SAMPLE_FLAG_IDLE_THREADS (1U << 1) @@ -60,6 +63,10 @@ struct kperf_context; #define SAMPLE_FLAG_CONTINUATION (1U << 3) /* sample is occurring outside of interrupt context */ #define SAMPLE_FLAG_NON_INTERRUPT (1U << 4) +/* sample should include system samplers */ +#define SAMPLE_FLAG_SYSTEM (1U << 5) +/* sample should not include non-system samplers */ +#define SAMPLE_FLAG_ONLY_SYSTEM (1U << 6) /* Take a sample into "sbuf" using current thread "cur_thread" */ kern_return_t kperf_sample(struct kperf_sample *sbuf, @@ -67,6 +74,9 @@ kern_return_t kperf_sample(struct kperf_sample *sbuf, unsigned actionid, unsigned sample_flags); +/* Whether the action provided samples non-system values. */ +bool kperf_sample_has_non_system(unsigned actionid); + /* return codes from taking a sample * either keep trigger, or something went wrong (or we're shutting down) * so turn off. diff --git a/bsd/kern/spl.c b/osfmk/kperf/arm/kperf_meminfo.c similarity index 59% rename from bsd/kern/spl.c rename to osfmk/kperf/arm/kperf_meminfo.c index 1078cdc8a..e9d6b1049 100644 --- a/bsd/kern/spl.c +++ b/osfmk/kperf/arm/kperf_meminfo.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2015 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -25,112 +25,30 @@ * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#include -#include -#include - -unsigned -sploff( - void) -{ - return(0); -} - -unsigned -splhigh( - void) -{ - return(0); -} - -unsigned -splsched( - void) -{ - return(0); -} - -unsigned -splclock ( - void) -{ - return(0); -} - -unsigned -splpower ( - void) -{ - return(0); -} - -unsigned -splvm( - void) -{ - return(0); -} - -unsigned -splbio ( - void) -{ - return(0); -} -unsigned -splimp( - void) -{ - return(0); -} - -unsigned -spltty(void) -{ - return(0); -} - -unsigned -splnet( - void) -{ - return(0); -} - -unsigned -splsoftclock(void) -{ - return(0); -} - -void -spllo(void) -{ - return; -} +#include +#include /* task_ledgers */ +#include +#include -void -spl0(void) -{ - return; -} +#include -void -spln(__unused unsigned t) +kern_return_t +kperf_get_phys_footprint(task_t task, uint64_t *phys_footprint_out) { - return; -} + kern_return_t kr; + ledger_amount_t credit, debit; + uint64_t phys_footprint; -void -splx(__unused unsigned l) -{ - return; -} + kr = ledger_get_entries(task->ledger, task_ledgers.phys_footprint, + &credit, &debit); + if (kr == KERN_SUCCESS) { + phys_footprint = credit - debit; + } else { + return kr; + } -void -splon(__unused unsigned l) -{ - return; + *phys_footprint_out = phys_footprint; + return KERN_SUCCESS; } diff --git a/osfmk/kperf/arm/kperf_mp.c b/osfmk/kperf/arm/kperf_mp.c new file mode 100644 index 000000000..ee298723e --- /dev/null +++ b/osfmk/kperf/arm/kperf_mp.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2011-2016 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* xcall */ +#include + +void +kperf_signal_handler(unsigned int cpu_number) +{ + uint64_t cpu_mask = UINT64_C(1) << cpu_number; + + /* find all the timers that caused a signal */ + for(int i = 0; i < (int)kperf_timerc; i++) { + uint64_t pending_cpus; + struct kperf_timer *timer = &kperf_timerv[i]; + + pending_cpus = atomic_fetch_and_explicit(&timer->pending_cpus, + ~cpu_mask, memory_order_relaxed); + if (pending_cpus & cpu_mask) { + kperf_ipi_handler(timer); + } + } +} + +bool +kperf_mp_broadcast_other_running(struct kperf_timer *timer) +{ + int current_cpu = cpu_number(); + bool system_only_self = true; + int n_cpus = machine_info.logical_cpu_max; + + /* signal all running processors */ + for (int i = 0; i < n_cpus; i++) { + uint64_t i_bit = UINT64_C(1) << i; + processor_t processor = cpu_to_processor(i); + + /* do not IPI processors that are not scheduling threads */ + if (processor == PROCESSOR_NULL || + processor->state != PROCESSOR_RUNNING || + processor->active_thread == THREAD_NULL) + { + continue; + } + + if (i == current_cpu) { + system_only_self = false; + continue; + } + + /* nor processors that have not responded to the last broadcast */ + uint64_t already_pending = atomic_fetch_or_explicit( + &timer->pending_cpus, i_bit, memory_order_relaxed); + if (already_pending & i_bit) + { +#if DEVELOPMENT || DEBUG + atomic_fetch_add_explicit(&kperf_pending_ipis, 1, + memory_order_relaxed); +#endif /* DEVELOPMENT || DEBUG */ + continue; + } + + cpu_signal(cpu_datap(i), SIGPkptimer, NULL, NULL); + } + + return system_only_self; +} diff --git a/osfmk/kperf/buffer.h b/osfmk/kperf/buffer.h index 31aee62ce..8dbe06fbc 100644 --- a/osfmk/kperf/buffer.h +++ b/osfmk/kperf/buffer.h @@ -56,24 +56,28 @@ #define PERF_GEN_CODE(code) PERF_CODE(PERF_GENERIC, code) #define PERF_GEN_EVENT PERF_GEN_CODE(0) -#define PERF_TI_CODE(code) PERF_CODE(PERF_THREADINFO, code) -#define PERF_TI_SAMPLE PERF_TI_CODE(0) -#define PERF_TI_DATA PERF_TI_CODE(1) -#define PERF_TI_XSAMPLE PERF_TI_CODE(2) -#define PERF_TI_XPEND PERF_TI_CODE(3) -#define PERF_TI_XDATA PERF_TI_CODE(4) -#define PERF_TI_CSWITCH PERF_TI_CODE(5) -#define PERF_TI_SCHEDSAMPLE PERF_TI_CODE(6) -#define PERF_TI_SCHEDDATA PERF_TI_CODE(7) -#define PERF_TI_SNAPSAMPLE PERF_TI_CODE(8) -#define PERF_TI_SNAPDATA PERF_TI_CODE(9) -#define PERF_TI_DISPSAMPLE PERF_TI_CODE(10) -#define PERF_TI_DISPDATA PERF_TI_CODE(11) -#define PERF_TI_DISPPEND PERF_TI_CODE(12) -#define PERF_TI_SNAPDATA_32 PERF_TI_CODE(13) -#define PERF_TI_DISPDATA_32 PERF_TI_CODE(14) -#define PERF_TI_SCHEDDATA1_32 PERF_TI_CODE(15) -#define PERF_TI_SCHEDDATA2_32 PERF_TI_CODE(16) +#define PERF_TI_CODE(code) PERF_CODE(PERF_THREADINFO, code) +#define PERF_TI_SAMPLE PERF_TI_CODE(0) +#define PERF_TI_DATA PERF_TI_CODE(1) +#define PERF_TI_XSAMPLE PERF_TI_CODE(2) +#define PERF_TI_XPEND PERF_TI_CODE(3) +#define PERF_TI_XDATA PERF_TI_CODE(4) +#define PERF_TI_CSWITCH PERF_TI_CODE(5) +#define PERF_TI_SCHEDSAMPLE PERF_TI_CODE(6) +#define PERF_TI_SCHEDDATA PERF_TI_CODE(7) +#define PERF_TI_SNAPSAMPLE PERF_TI_CODE(8) +#define PERF_TI_SNAPDATA PERF_TI_CODE(9) +#define PERF_TI_DISPSAMPLE PERF_TI_CODE(10) +#define PERF_TI_DISPDATA PERF_TI_CODE(11) +#define PERF_TI_DISPPEND PERF_TI_CODE(12) +#define PERF_TI_SNAPDATA_32 PERF_TI_CODE(13) +#define PERF_TI_DISPDATA_32 PERF_TI_CODE(14) +#define PERF_TI_SCHEDDATA1_32 PERF_TI_CODE(15) +#define PERF_TI_SCHEDDATA2_32 PERF_TI_CODE(16) +#define PERF_TI_INSCYCDATA PERF_TI_CODE(17) +#define PERF_TI_INSCYCDATA_32 PERF_TI_CODE(18) +#define PERF_TI_SCHEDDATA_2 PERF_TI_CODE(19) +#define PERF_TI_SCHEDDATA2_32_2 PERF_TI_CODE(20) #define PERF_CS_CODE(code) PERF_CODE(PERF_CALLSTACK, code) #define PERF_CS_KSAMPLE PERF_CS_CODE(0) @@ -91,6 +95,8 @@ #define PERF_TM_FIRE PERF_TM_CODE(0) #define PERF_TM_SCHED PERF_TM_CODE(1) #define PERF_TM_HNDLR PERF_TM_CODE(2) +#define PERF_TM_PENDING PERF_TM_CODE(3) +#define PERF_TM_SKIPPED PERF_TM_CODE(4) #define PERF_PET_CODE(code) PERF_CODE(PERF_PET, code) #define PERF_PET_THREAD PERF_PET_CODE(0) @@ -134,6 +140,7 @@ #define PERF_MI_CODE(code) PERF_CODE(PERF_MEMINFO, code) #define PERF_MI_SAMPLE PERF_MI_CODE(0) #define PERF_MI_DATA PERF_MI_CODE(1) +#define PERF_MI_SYS_DATA PERF_MI_CODE(2) /* error sub-codes for trace data */ enum diff --git a/osfmk/kperf/callstack.c b/osfmk/kperf/callstack.c index b45c3f0e7..7c8f0a7e7 100644 --- a/osfmk/kperf/callstack.c +++ b/osfmk/kperf/callstack.c @@ -39,6 +39,10 @@ #include #include +#if defined(__arm__) || defined(__arm64__) +#include +#include +#endif static void callstack_fixup_user(struct callstack *cs, thread_t thread) @@ -71,6 +75,20 @@ callstack_fixup_user(struct callstack *cs, thread_t thread) &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t)); } +#elif defined(__arm64__) || defined(__arm__) + + struct arm_saved_state *state = get_user_regs(thread); + if (!state) { + goto out; + } + + /* encode thumb mode into low bit of PC */ + if (get_saved_state_cpsr(state) & PSR_TF) { + cs->frames[0] |= 1ULL; + } + + fixup_val = get_saved_state_lr(state); + #else #error "callstack_fixup_user: unsupported architecture" #endif @@ -125,6 +143,44 @@ interrupted_kernel_sp_value(uintptr_t *sp_val) return KERN_SUCCESS; } +#elif defined(__arm64__) + +__attribute__((used)) +static kern_return_t +interrupted_kernel_lr(uintptr_t *lr) +{ + struct arm_saved_state *state; + + state = getCpuDatap()->cpu_int_state; + + /* return early if interrupted a thread in user space */ + if (PSR64_IS_USER(get_saved_state_cpsr(state))) { + return KERN_FAILURE; + } + + *lr = get_saved_state_lr(state); + return KERN_SUCCESS; +} + +#elif defined(__arm__) + +__attribute__((used)) +static kern_return_t +interrupted_kernel_lr(uintptr_t *lr) +{ + struct arm_saved_state *state; + + state = getCpuDatap()->cpu_int_state; + + /* return early if interrupted a thread in user space */ + if (PSR_IS_USER(get_saved_state_cpsr(state))) { + return KERN_FAILURE; + } + + *lr = get_saved_state_lr(state); + return KERN_SUCCESS; +} + #else /* defined(__arm__) */ #error "interrupted_kernel_{sp,lr}: unsupported architecture" #endif /* !defined(__arm__) */ @@ -142,11 +198,13 @@ callstack_fixup_interrupted(struct callstack *cs) #if DEVELOPMENT || DEBUG #if defined(__x86_64__) (void)interrupted_kernel_sp_value(&fixup_val); +#elif defined(__arm64__) || defined(__arm__) + (void)interrupted_kernel_lr(&fixup_val); #endif /* defined(__x86_64__) */ #endif /* DEVELOPMENT || DEBUG */ - cs->frames[cs->nframes++] = fixup_val ? - VM_KERNEL_UNSLIDE_OR_PERM(fixup_val) : 0; + assert(cs->flags & CALLSTACK_KERNEL); + cs->frames[cs->nframes++] = fixup_val; } void @@ -293,10 +351,14 @@ kperf_ucallstack_sample(struct callstack *cs, struct kperf_context *context) } static inline uintptr_t -scrub_kernel_frame(uintptr_t *bt, int n_frames, int frame) +scrub_word(uintptr_t *bt, int n_frames, int frame, bool kern) { if (frame < n_frames) { - return VM_KERNEL_UNSLIDE(bt[frame]); + if (kern) { + return VM_KERNEL_UNSLIDE(bt[frame]); + } else { + return bt[frame]; + } } else { return 0; } @@ -321,29 +383,34 @@ callstack_log(struct callstack *cs, uint32_t hcode, uint32_t dcode) BUF_DATA(hcode, cs->flags, cs->nframes); /* how many batches of 4 */ - unsigned int n = cs->nframes / 4; - unsigned int ovf = cs->nframes % 4; + unsigned int nframes = cs->nframes; + unsigned int n = nframes / 4; + unsigned int ovf = nframes % 4; if (ovf != 0) { n++; } + bool kern = cs->flags & CALLSTACK_KERNEL; + if (cs->flags & CALLSTACK_KERNEL_WORDS) { + uintptr_t *frames = (uintptr_t *)cs->frames; for (unsigned int i = 0; i < n; i++) { unsigned int j = i * 4; BUF_DATA(dcode, - scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 0), - scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 1), - scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 2), - scrub_kernel_frame((uintptr_t *)cs->frames, cs->nframes, j + 3)); + scrub_word(frames, nframes, j + 0, kern), + scrub_word(frames, nframes, j + 1, kern), + scrub_word(frames, nframes, j + 2, kern), + scrub_word(frames, nframes, j + 3, kern)); } } else { for (unsigned int i = 0; i < n; i++) { + uint64_t *frames = cs->frames; unsigned int j = i * 4; BUF_DATA(dcode, - scrub_frame(cs->frames, cs->nframes, j + 0), - scrub_frame(cs->frames, cs->nframes, j + 1), - scrub_frame(cs->frames, cs->nframes, j + 2), - scrub_frame(cs->frames, cs->nframes, j + 3)); + scrub_frame(frames, nframes, j + 0), + scrub_frame(frames, nframes, j + 1), + scrub_frame(frames, nframes, j + 2), + scrub_frame(frames, nframes, j + 3)); } } diff --git a/osfmk/kperf/kperf.c b/osfmk/kperf/kperf.c index 45d441e5f..76d75d78b 100644 --- a/osfmk/kperf/kperf.c +++ b/osfmk/kperf/kperf.c @@ -41,10 +41,13 @@ #include #include +/* from libkern/libkern.h */ +extern uint64_t strtouq(const char *, char **, int); + lck_grp_t kperf_lck_grp; -/* thread on CPUs before starting the PET thread */ -thread_t *kperf_thread_on_cpus = NULL; +/* IDs of threads on CPUs before starting the PET thread */ +uint64_t *kperf_tid_on_cpus = NULL; /* one wired sample buffer per CPU */ static struct kperf_sample *intr_samplev; @@ -76,8 +79,6 @@ kperf_init(void) { static lck_grp_attr_t lck_grp_attr; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); - unsigned ncpus = 0; int err; @@ -91,13 +92,13 @@ kperf_init(void) ncpus = machine_info.logical_cpu_max; /* create buffers to remember which threads don't need to be sampled by PET */ - kperf_thread_on_cpus = kalloc_tag(ncpus * sizeof(*kperf_thread_on_cpus), + kperf_tid_on_cpus = kalloc_tag(ncpus * sizeof(*kperf_tid_on_cpus), VM_KERN_MEMORY_DIAG); - if (kperf_thread_on_cpus == NULL) { + if (kperf_tid_on_cpus == NULL) { err = ENOMEM; goto error; } - bzero(kperf_thread_on_cpus, ncpus * sizeof(*kperf_thread_on_cpus)); + bzero(kperf_tid_on_cpus, ncpus * sizeof(*kperf_tid_on_cpus)); /* create the interrupt buffers */ intr_samplec = ncpus; @@ -124,9 +125,9 @@ error: intr_samplec = 0; } - if (kperf_thread_on_cpus) { - kfree(kperf_thread_on_cpus, ncpus * sizeof(*kperf_thread_on_cpus)); - kperf_thread_on_cpus = NULL; + if (kperf_tid_on_cpus) { + kfree(kperf_tid_on_cpus, ncpus * sizeof(*kperf_tid_on_cpus)); + kperf_tid_on_cpus = NULL; } return err; @@ -135,8 +136,6 @@ error: void kperf_reset(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); - /* turn off sampling first */ (void)kperf_sampling_disable(); @@ -150,6 +149,70 @@ kperf_reset(void) kperf_action_reset(); } +void +kperf_kernel_configure(const char *config) +{ + int pairs = 0; + char *end; + bool pet = false; + + assert(config != NULL); + + ktrace_start_single_threaded(); + + ktrace_kernel_configure(KTRACE_KPERF); + + if (config[0] == 'p') { + pet = true; + config++; + } + + do { + uint32_t action_samplers; + uint64_t timer_period; + + pairs += 1; + kperf_action_set_count(pairs); + kperf_timer_set_count(pairs); + + action_samplers = (uint32_t)strtouq(config, &end, 0); + if (config == end) { + kprintf("kperf: unable to parse '%s' as action sampler\n", config); + goto out; + } + config = end; + + kperf_action_set_samplers(pairs, action_samplers); + + if (config[0] == '\0') { + kprintf("kperf: missing timer period in config\n"); + goto out; + } + config++; + + timer_period = strtouq(config, &end, 0); + if (config == end) { + kprintf("kperf: unable to parse '%s' as timer period\n", config); + goto out; + } + config = end; + + kperf_timer_set_period(pairs - 1, timer_period); + kperf_timer_set_action(pairs - 1, pairs); + + if (pet) { + kperf_timer_set_petid(pairs - 1); + kperf_set_lightweight_pet(1); + pet = false; + } + } while (*(config++) == ','); + + kperf_sampling_enable(); + +out: + ktrace_end_single_threaded(); +} + void kperf_on_cpu_internal(thread_t thread, thread_continue_t continuation, uintptr_t *starting_fp) diff --git a/osfmk/kperf/kperf.h b/osfmk/kperf/kperf.h index ec0ab45db..040d032c1 100644 --- a/osfmk/kperf/kperf.h +++ b/osfmk/kperf/kperf.h @@ -124,6 +124,11 @@ kperf_kdebug_callback(uint32_t debugid, uintptr_t *starting_fp) */ extern void kperf_reset(void); +/* + * Configure kperf from the kernel (e.g. during boot). + */ +void kperf_kernel_configure(const char *config); + /* get and set whether we're recording stacks on interesting kdebug events */ extern int kperf_kdbg_get_stacks(void); extern int kperf_kdbg_set_stacks(int); diff --git a/osfmk/kperf/kperf_arch.h b/osfmk/kperf/kperf_arch.h index 44719601e..c83992191 100644 --- a/osfmk/kperf/kperf_arch.h +++ b/osfmk/kperf/kperf_arch.h @@ -29,9 +29,9 @@ #define KPERF_ARCH_H struct kperf_timer; -void kperf_mp_broadcast_running(struct kperf_timer *trigger); +bool kperf_mp_broadcast_other_running(struct kperf_timer *trigger); -void kperf_signal_handler(void); +void kperf_signal_handler(unsigned int cpu_number); kern_return_t kperf_get_phys_footprint(task_t, uint64_t *); #endif /* KPERF_ARCH_H */ diff --git a/osfmk/kperf/kperf_timer.c b/osfmk/kperf/kperf_timer.c index cfa429f2f..a6cf3c2e3 100644 --- a/osfmk/kperf/kperf_timer.c +++ b/osfmk/kperf/kperf_timer.c @@ -31,6 +31,7 @@ #include #include /* current_thread() */ #include +#include #include #include #include @@ -58,17 +59,6 @@ static unsigned int pet_timer_id = 999; /* maximum number of timers we can construct */ #define TIMER_MAX (16) -#if defined(__x86_64__) - -#define MIN_PERIOD_NS (20 * NSEC_PER_USEC) -#define MIN_PERIOD_BG_NS (10 * NSEC_PER_MSEC) -#define MIN_PERIOD_PET_NS (2 * NSEC_PER_MSEC) -#define MIN_PERIOD_PET_BG_NS (10 * NSEC_PER_MSEC) - -#else /* defined(__x86_64__) */ -#error "unsupported architecture" -#endif /* defined(__x86_64__) */ - static uint64_t min_period_abstime; static uint64_t min_period_bg_abstime; static uint64_t min_period_pet_abstime; @@ -111,11 +101,11 @@ kperf_timer_schedule(struct kperf_timer *timer, uint64_t now) timer_call_enter(&timer->tcall, deadline, TIMER_CALL_SYS_CRITICAL); } -void -kperf_ipi_handler(void *param) +static void +kperf_sample_cpu(struct kperf_timer *timer, bool system_sample, + bool only_system) { struct kperf_context ctx; - struct kperf_timer *timer = param; assert(timer != NULL); @@ -135,7 +125,7 @@ kperf_ipi_handler(void *param) ctx.trigger_id = (unsigned int)(timer - kperf_timerv); if (ctx.trigger_id == pet_timer_id && ncpu < machine_info.logical_cpu_max) { - kperf_thread_on_cpus[ncpu] = ctx.cur_thread; + kperf_tid_on_cpus[ncpu] = thread_tid(ctx.cur_thread); } /* make sure sampling is on */ @@ -149,14 +139,21 @@ kperf_ipi_handler(void *param) } /* call the action -- kernel-only from interrupt, pend user */ - int r = kperf_sample(intbuf, &ctx, timer->actionid, SAMPLE_FLAG_PEND_USER); + int r = kperf_sample(intbuf, &ctx, timer->actionid, + SAMPLE_FLAG_PEND_USER | (system_sample ? SAMPLE_FLAG_SYSTEM : 0) | + (only_system ? SAMPLE_FLAG_ONLY_SYSTEM : 0)); /* end tracepoint is informational */ BUF_INFO(PERF_TM_HNDLR | DBG_FUNC_END, r); -#if defined(__x86_64__) - (void)atomic_bit_clear(&(timer->pending_cpus), ncpu, __ATOMIC_RELAXED); -#endif /* defined(__x86_64__) */ + (void)atomic_fetch_and_explicit(&timer->pending_cpus, + ~(UINT64_C(1) << ncpu), memory_order_relaxed); +} + +void +kperf_ipi_handler(void *param) +{ + kperf_sample_cpu((struct kperf_timer *)param, false, false); } static void @@ -165,6 +162,11 @@ kperf_timer_handler(void *param0, __unused void *param1) struct kperf_timer *timer = param0; unsigned int ntimer = (unsigned int)(timer - kperf_timerv); unsigned int ncpus = machine_info.logical_cpu_max; + bool system_only_self = true; + + if (timer->actionid == 0) { + return; + } timer->active = 1; @@ -180,11 +182,20 @@ kperf_timer_handler(void *param0, __unused void *param1) kperf_pet_fire_before(); /* clean-up the thread-on-CPUs cache */ - bzero(kperf_thread_on_cpus, ncpus * sizeof(*kperf_thread_on_cpus)); + bzero(kperf_tid_on_cpus, ncpus * sizeof(*kperf_tid_on_cpus)); } - /* ping all CPUs */ - kperf_mp_broadcast_running(timer); + /* + * IPI other cores only if the action has non-system samplers. + */ + if (kperf_sample_has_non_system(timer->actionid)) { + /* + * If the core that's handling the timer is not scheduling + * threads, only run system samplers. + */ + system_only_self = kperf_mp_broadcast_other_running(timer); + } + kperf_sample_cpu(timer, true, system_only_self); /* release the pet thread? */ if (ntimer == pet_timer_id) { @@ -255,7 +266,7 @@ kperf_timer_pet_rearm(uint64_t elapsed_ticks) BUF_INFO(PERF_PET_SCHED, timer->period, period, elapsed_ticks, deadline); /* re-schedule the timer, making sure we don't apply slop */ - timer_call_enter(&(timer->tcall), deadline, TIMER_CALL_SYS_CRITICAL); + timer_call_enter(&timer->tcall, deadline, TIMER_CALL_SYS_CRITICAL); return; } @@ -291,7 +302,7 @@ kperf_timer_stop(void) /* wait for the timer to stop */ while (kperf_timerv[i].active); - timer_call_cancel(&(kperf_timerv[i].tcall)); + timer_call_cancel(&kperf_timerv[i].tcall); } /* wait for PET to stop, too */ @@ -399,9 +410,7 @@ kperf_timer_reset(void) for (unsigned int i = 0; i < kperf_timerc; i++) { kperf_timerv[i].period = 0; kperf_timerv[i].actionid = 0; -#if defined(__x86_64__) kperf_timerv[i].pending_cpus = 0; -#endif /* defined(__x86_64__) */ } } @@ -412,10 +421,10 @@ kperf_timer_set_count(unsigned int count) unsigned int old_count; if (min_period_abstime == 0) { - nanoseconds_to_absolutetime(MIN_PERIOD_NS, &min_period_abstime); - nanoseconds_to_absolutetime(MIN_PERIOD_BG_NS, &min_period_bg_abstime); - nanoseconds_to_absolutetime(MIN_PERIOD_PET_NS, &min_period_pet_abstime); - nanoseconds_to_absolutetime(MIN_PERIOD_PET_BG_NS, + nanoseconds_to_absolutetime(KP_MIN_PERIOD_NS, &min_period_abstime); + nanoseconds_to_absolutetime(KP_MIN_PERIOD_BG_NS, &min_period_bg_abstime); + nanoseconds_to_absolutetime(KP_MIN_PERIOD_PET_NS, &min_period_pet_abstime); + nanoseconds_to_absolutetime(KP_MIN_PERIOD_PET_BG_NS, &min_period_pet_bg_abstime); assert(min_period_abstime > 0); } @@ -471,7 +480,7 @@ kperf_timer_set_count(unsigned int count) /* (re-)setup the timer call info for all entries */ for (unsigned int i = 0; i < count; i++) { - timer_call_setup(&(new_timerv[i].tcall), kperf_timer_handler, &(new_timerv[i])); + timer_call_setup(&new_timerv[i].tcall, kperf_timer_handler, &new_timerv[i]); } kperf_timerv = new_timerv; diff --git a/osfmk/kperf/kperf_timer.h b/osfmk/kperf/kperf_timer.h index 4229b4d9f..946cc21c0 100644 --- a/osfmk/kperf/kperf_timer.h +++ b/osfmk/kperf/kperf_timer.h @@ -59,6 +59,31 @@ void kperf_ipi_handler(void *param); #define TIMER_REPROGRAM (0) #define TIMER_STOP (1) +#if defined(__x86_64__) + +#define KP_MIN_PERIOD_NS (20 * NSEC_PER_USEC) +#define KP_MIN_PERIOD_BG_NS (10 * NSEC_PER_MSEC) +#define KP_MIN_PERIOD_PET_NS (2 * NSEC_PER_MSEC) +#define KP_MIN_PERIOD_PET_BG_NS (10 * NSEC_PER_MSEC) + +#elif defined(__arm64__) + +#define KP_MIN_PERIOD_NS (50 * NSEC_PER_USEC) +#define KP_MIN_PERIOD_BG_NS (20 * NSEC_PER_MSEC) +#define KP_MIN_PERIOD_PET_NS (2 * NSEC_PER_MSEC) +#define KP_MIN_PERIOD_PET_BG_NS (50 * NSEC_PER_MSEC) + +#elif defined(__arm__) + +#define KP_MIN_PERIOD_NS (100 * NSEC_PER_USEC) +#define KP_MIN_PERIOD_BG_NS (50 * NSEC_PER_MSEC) +#define KP_MIN_PERIOD_PET_NS (2 * NSEC_PER_MSEC) +#define KP_MIN_PERIOD_PET_BG_NS (100 * NSEC_PER_MSEC) + +#else /* defined(__x86_64__) */ +#error "unsupported architecture" +#endif /* defined(__x86_64__) */ + /* getters and setters on timers */ unsigned kperf_timer_get_count(void); int kperf_timer_set_count(unsigned int count); diff --git a/osfmk/kperf/kperfbsd.c b/osfmk/kperf/kperfbsd.c index 1b3ab5f5d..b89125126 100644 --- a/osfmk/kperf/kperfbsd.c +++ b/osfmk/kperf/kperfbsd.c @@ -145,9 +145,14 @@ kperf_sysctl_get_set_unsigned_uint32(struct sysctl_req *req, assert(get != NULL); assert(set != NULL); - int error; - uint64_t inputs[2]; - if ((error = SYSCTL_IN(req, inputs, sizeof(inputs)))) { + int error = 0; + uint64_t inputs[2] = {}; + + if (req->newptr == USER_ADDR_NULL) { + return EFAULT; + } + + if ((error = copyin(req->newptr, inputs, sizeof(inputs)))) { return error; } @@ -161,17 +166,10 @@ kperf_sysctl_get_set_unsigned_uint32(struct sysctl_req *req, } inputs[1] = value_out; - } else { - if ((error = set(action_id, new_value))) { - return error; - } - } - if (req->oldptr != USER_ADDR_NULL) { - error = SYSCTL_OUT(req, inputs, sizeof(inputs)); - return error; + return copyout(inputs, req->oldptr, sizeof(inputs)); } else { - return 0; + return set(action_id, new_value); } } @@ -183,11 +181,16 @@ kperf_sysctl_get_set_unsigned_uint32(struct sysctl_req *req, static int sysctl_timer_period(struct sysctl_req *req) { + int error; + uint64_t inputs[2] = {}; + assert(req != NULL); - int error; - uint64_t inputs[2]; - if ((error = SYSCTL_IN(req, inputs, sizeof(inputs)))) { + if (req->newptr == USER_ADDR_NULL) { + return EFAULT; + } + + if ((error = copyin(req->newptr, inputs, sizeof(inputs)))) { return error; } @@ -201,23 +204,26 @@ sysctl_timer_period(struct sysctl_req *req) } inputs[1] = period_out; + + return copyout(inputs, req->oldptr, sizeof(inputs)); } else { - if ((error = kperf_timer_set_period(timer, new_period))) { - return error; - } + return kperf_timer_set_period(timer, new_period); } - - return SYSCTL_OUT(req, inputs, sizeof(inputs)); } static int -sysctl_action_filter(struct sysctl_req *req, boolean_t is_task_t) +sysctl_action_filter(struct sysctl_req *req, bool is_task_t) { + int error = 0; + uint64_t inputs[2] = {}; + assert(req != NULL); - int error; - uint64_t inputs[2]; - if ((error = SYSCTL_IN(req, inputs, sizeof(inputs)))) { + if (req->newptr == USER_ADDR_NULL) { + return EFAULT; + } + + if ((error = copyin(req->newptr, inputs, sizeof(inputs)))) { return error; } @@ -231,16 +237,13 @@ sysctl_action_filter(struct sysctl_req *req, boolean_t is_task_t) } inputs[1] = filter_out; + return copyout(inputs, req->oldptr, sizeof(inputs)); } else { int pid = is_task_t ? kperf_port_to_pid((mach_port_name_t)new_filter) - : new_filter; + : new_filter; - if ((error = kperf_action_set_filter(actionid, pid))) { - return error; - } + return kperf_action_set_filter(actionid, pid); } - - return SYSCTL_OUT(req, inputs, sizeof(inputs)); } static int @@ -411,16 +414,16 @@ kperf_sysctl SYSCTL_HANDLER_ARGS int ret; uintptr_t type = (uintptr_t)arg1; - lck_mtx_lock(ktrace_lock); + ktrace_lock(); if (req->oldptr == USER_ADDR_NULL && req->newptr != USER_ADDR_NULL) { if ((ret = ktrace_configure(KTRACE_KPERF))) { - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); return ret; } } else { if ((ret = ktrace_read_check())) { - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); return ret; } } @@ -455,10 +458,10 @@ kperf_sysctl SYSCTL_HANDLER_ARGS ret = sysctl_kdbg_cswitch(req); break; case REQ_ACTION_FILTER_BY_TASK: - ret = sysctl_action_filter(req, TRUE); + ret = sysctl_action_filter(req, true); break; case REQ_ACTION_FILTER_BY_PID: - ret = sysctl_action_filter(req, FALSE); + ret = sysctl_action_filter(req, false); break; case REQ_KDEBUG_ACTION: ret = sysctl_kdebug_action(req); @@ -489,7 +492,7 @@ kperf_sysctl SYSCTL_HANDLER_ARGS break; } - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); return ret; } @@ -500,7 +503,7 @@ kperf_sysctl_bless_handler SYSCTL_HANDLER_ARGS #pragma unused(oidp, arg2) int ret; - lck_mtx_lock(ktrace_lock); + ktrace_lock(); /* if setting a new "blessed pid" (ktrace owning pid) */ if (req->newptr != USER_ADDR_NULL) { @@ -515,13 +518,13 @@ kperf_sysctl_bless_handler SYSCTL_HANDLER_ARGS kauth_cred_issuser(kauth_cred_get()))) { if ((ret = ktrace_configure(KTRACE_KPERF))) { - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); return ret; } } } else { if ((ret = ktrace_read_check())) { - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); return ret; } } @@ -533,7 +536,7 @@ kperf_sysctl_bless_handler SYSCTL_HANDLER_ARGS ret = ENOENT; } - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); return ret; } @@ -671,6 +674,64 @@ SYSCTL_PROC(_kperf, OID_AUTO, lightweight_pet, sizeof(int), kperf_sysctl, "I", "Status of lightweight PET mode"); +/* limits */ + +SYSCTL_NODE(_kperf, OID_AUTO, limits, CTLFLAG_RW | CTLFLAG_LOCKED, 0, + "limits"); + +#define REQ_LIM_PERIOD_NS (1) +#define REQ_LIM_BG_PERIOD_NS (2) +#define REQ_LIM_PET_PERIOD_NS (3) +#define REQ_LIM_BG_PET_PERIOD_NS (4) + +static int +kperf_sysctl_limits SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg2) + int type = (int)arg1; + uint64_t limit = 0; + + switch (type) { + case REQ_LIM_PERIOD_NS: + limit = KP_MIN_PERIOD_NS; + break; + + case REQ_LIM_BG_PERIOD_NS: + limit = KP_MIN_PERIOD_BG_NS; + break; + + case REQ_LIM_PET_PERIOD_NS: + limit = KP_MIN_PERIOD_PET_NS; + break; + + case REQ_LIM_BG_PET_PERIOD_NS: + limit = KP_MIN_PERIOD_PET_BG_NS; + break; + + default: + return ENOENT; + } + + return sysctl_io_number(req, limit, sizeof(limit), &limit, NULL); +} + +SYSCTL_PROC(_kperf_limits, OID_AUTO, timer_min_period_ns, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_LOCKED, + (void *)REQ_LIM_PERIOD_NS, sizeof(uint64_t), kperf_sysctl_limits, + "Q", "Minimum timer period in nanoseconds"); +SYSCTL_PROC(_kperf_limits, OID_AUTO, timer_min_bg_period_ns, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_LOCKED, + (void *)REQ_LIM_BG_PERIOD_NS, sizeof(uint64_t), kperf_sysctl_limits, + "Q", "Minimum background timer period in nanoseconds"); +SYSCTL_PROC(_kperf_limits, OID_AUTO, timer_min_pet_period_ns, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_LOCKED, + (void *)REQ_LIM_PET_PERIOD_NS, sizeof(uint64_t), kperf_sysctl_limits, + "Q", "Minimum PET timer period in nanoseconds"); +SYSCTL_PROC(_kperf_limits, OID_AUTO, timer_min_bg_pet_period_ns, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_LOCKED, + (void *)REQ_LIM_BG_PET_PERIOD_NS, sizeof(uint64_t), kperf_sysctl_limits, + "Q", "Minimum background PET timer period in nanoseconds"); + /* debug */ SYSCTL_INT(_kperf, OID_AUTO, debug_level, CTLFLAG_RW | CTLFLAG_LOCKED, &kperf_debug_level, 0, "debug level"); diff --git a/osfmk/kperf/pet.c b/osfmk/kperf/pet.c index 52d2909ce..5af12821c 100644 --- a/osfmk/kperf/pet.c +++ b/osfmk/kperf/pet.c @@ -457,8 +457,7 @@ pet_sample_task(task_t task, uint32_t idle_rate) /* do not sample the thread if it was on a CPU during the IPI. */ for (cpu = 0; cpu < machine_info.logical_cpu_max; cpu++) { - thread_t candidate = kperf_thread_on_cpus[cpu]; - if (candidate && (thread_tid(candidate) == thread_tid(thread))) { + if (kperf_tid_on_cpus[cpu] == thread_tid(thread)) { break; } } diff --git a/osfmk/kperf/sample.h b/osfmk/kperf/sample.h index 42637fa31..35175294f 100644 --- a/osfmk/kperf/sample.h +++ b/osfmk/kperf/sample.h @@ -53,6 +53,6 @@ struct kperf_sample { }; /* cache of threads on each CPU during a timer fire */ -extern thread_t *kperf_thread_on_cpus; +extern uint64_t *kperf_tid_on_cpus; #endif /* !defined(KPERF_SAMPLE_H) */ diff --git a/osfmk/kperf/thread_samplers.c b/osfmk/kperf/thread_samplers.c index 2442a40ed..e62f8934d 100644 --- a/osfmk/kperf/thread_samplers.c +++ b/osfmk/kperf/thread_samplers.c @@ -40,6 +40,11 @@ #include #include +#if MONOTONIC +#include +#include +#endif /* MONOTONIC */ + extern boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread); /* @@ -86,10 +91,12 @@ kperf_thread_info_runmode_legacy(thread_t thread) kperf_state |= KPERF_TI_IDLE; } +#if !TARGET_OS_EMBEDDED /* on desktop, if state is blank, leave not idle set */ if (kperf_state == 0) { return (TH_IDLE << 16); } +#endif /* !TARGET_OS_EMBEDDED */ /* high two bytes are inverted mask, low two bytes are normal */ return (((~kperf_state & 0xffff) << 16) | (kperf_state & 0xffff)); @@ -149,6 +156,9 @@ kperf_thread_scheduling_sample(struct kperf_thread_scheduling *thsc, thsc->kpthsc_effective_qos = thread->effective_policy.thep_qos; thsc->kpthsc_requested_qos = thread->requested_policy.thrp_qos; thsc->kpthsc_requested_qos_override = thread->requested_policy.thrp_qos_override; + thsc->kpthsc_requested_qos_promote = thread->requested_policy.thrp_qos_promote; + thsc->kpthsc_requested_qos_ipc_override = thread->requested_policy.thrp_qos_ipc_override; + thsc->kpthsc_requested_qos_sync_ipc_override = thread->requested_policy.thrp_qos_sync_ipc_override; thsc->kpthsc_effective_latency_qos = thread->effective_policy.thep_latency_qos; BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_END); @@ -160,27 +170,36 @@ kperf_thread_scheduling_log(struct kperf_thread_scheduling *thsc) { assert(thsc != NULL); #if defined(__LP64__) - BUF_DATA(PERF_TI_SCHEDDATA, thsc->kpthsc_user_time, - thsc->kpthsc_system_time, - (((uint64_t)thsc->kpthsc_base_priority) << 48) - | ((uint64_t)thsc->kpthsc_sched_priority << 32) - | ((uint64_t)(thsc->kpthsc_state & 0xff) << 24) - | (thsc->kpthsc_effective_qos << 6) - | (thsc->kpthsc_requested_qos << 3) - | thsc->kpthsc_requested_qos_override, - ((uint64_t)thsc->kpthsc_effective_latency_qos << 61)); + BUF_DATA(PERF_TI_SCHEDDATA_2, thsc->kpthsc_user_time, + thsc->kpthsc_system_time, + (((uint64_t)thsc->kpthsc_base_priority) << 48) + | ((uint64_t)thsc->kpthsc_sched_priority << 32) + | ((uint64_t)(thsc->kpthsc_state & 0xff) << 24) + | (thsc->kpthsc_effective_qos << 6) + | (thsc->kpthsc_requested_qos << 3) + | thsc->kpthsc_requested_qos_override, + ((uint64_t)thsc->kpthsc_effective_latency_qos << 61) + | ((uint64_t)thsc->kpthsc_requested_qos_promote << 58) + | ((uint64_t)thsc->kpthsc_requested_qos_ipc_override << 55) + | ((uint64_t)thsc->kpthsc_requested_qos_sync_ipc_override << 52) + ); #else BUF_DATA(PERF_TI_SCHEDDATA1_32, UPPER_32(thsc->kpthsc_user_time), - LOWER_32(thsc->kpthsc_user_time), - UPPER_32(thsc->kpthsc_system_time), - LOWER_32(thsc->kpthsc_system_time)); - BUF_DATA(PERF_TI_SCHEDDATA2_32, (((uint32_t)thsc->kpthsc_base_priority) << 16) - | thsc->kpthsc_sched_priority, - ((thsc->kpthsc_state & 0xff) << 24) - | (thsc->kpthsc_effective_qos << 6) - | (thsc->kpthsc_requested_qos << 3) - | thsc->kpthsc_requested_qos_override, - (uint32_t)thsc->kpthsc_effective_latency_qos << 29); + LOWER_32(thsc->kpthsc_user_time), + UPPER_32(thsc->kpthsc_system_time), + LOWER_32(thsc->kpthsc_system_time) + ); + BUF_DATA(PERF_TI_SCHEDDATA2_32_2, (((uint32_t)thsc->kpthsc_base_priority) << 16) + | thsc->kpthsc_sched_priority, + ((thsc->kpthsc_state & 0xff) << 24) + | (thsc->kpthsc_effective_qos << 6) + | (thsc->kpthsc_requested_qos << 3) + | thsc->kpthsc_requested_qos_override, + ((uint32_t)thsc->kpthsc_effective_latency_qos << 29) + | ((uint32_t)thsc->kpthsc_requested_qos_promote << 26) + | ((uint32_t)thsc->kpthsc_requested_qos_ipc_override << 23) + | ((uint32_t)thsc->kpthsc_requested_qos_sync_ipc_override << 20) + ); #endif /* defined(__LP64__) */ } @@ -315,3 +334,37 @@ kperf_thread_dispatch_log(struct kperf_thread_dispatch *thdi) LOWER_32(thdi->kpthdi_dq_serialno)); #endif /* defined(__LP64__) */ } + +/* + * A bit different from other samplers -- since logging disables interrupts, + * it's a fine place to sample the thread counters. + */ +void +kperf_thread_inscyc_log(struct kperf_context *context) +{ +#if MONOTONIC + thread_t cur_thread = current_thread(); + + if (context->cur_thread != cur_thread) { + /* can't safely access another thread's counters */ + return; + } + + uint64_t counts[MT_CORE_NFIXED]; + + int ret = mt_fixed_thread_counts(cur_thread, counts); + if (ret) { + return; + } + +#if defined(__LP64__) + BUF_DATA(PERF_TI_INSCYCDATA, counts[MT_CORE_INSTRS], counts[MT_CORE_CYCLES]); +#else /* defined(__LP64__) */ + /* 32-bit platforms don't count instructions */ + BUF_DATA(PERF_TI_INSCYCDATA_32, 0, 0, UPPER_32(counts[MT_CORE_CYCLES]), + LOWER_32(counts[MT_CORE_CYCLES])); +#endif /* !defined(__LP64__) */ +#else /* MONOTONIC */ +#pragma unused(context) +#endif /* !MONOTONIC */ +} diff --git a/osfmk/kperf/thread_samplers.h b/osfmk/kperf/thread_samplers.h index 38195a629..e5a9eaeff 100644 --- a/osfmk/kperf/thread_samplers.h +++ b/osfmk/kperf/thread_samplers.h @@ -50,10 +50,13 @@ struct kperf_thread_scheduling { unsigned int kpthsc_state; uint16_t kpthsc_base_priority; uint16_t kpthsc_sched_priority; - unsigned int kpthsc_effective_qos : 3; - unsigned int kpthsc_requested_qos : 3; - unsigned int kpthsc_requested_qos_override : 3; - unsigned int kpthsc_effective_latency_qos : 3; + unsigned int kpthsc_effective_qos :3, + kpthsc_requested_qos :3, + kpthsc_requested_qos_override :3, + kpthsc_requested_qos_promote :3, + kpthsc_requested_qos_ipc_override :3, + kpthsc_requested_qos_sync_ipc_override :3, + kpthsc_effective_latency_qos :3; }; void kperf_thread_scheduling_sample(struct kperf_thread_scheduling *, @@ -82,4 +85,6 @@ void kperf_thread_dispatch_sample(struct kperf_thread_dispatch *, int kperf_thread_dispatch_pend(struct kperf_context *); void kperf_thread_dispatch_log(struct kperf_thread_dispatch *); +void kperf_thread_inscyc_log(struct kperf_context *); + #endif /* !defined(KPERF_THREAD_SAMPLERS_H) */ diff --git a/osfmk/kperf/x86_64/kperf_mp.c b/osfmk/kperf/x86_64/kperf_mp.c index d9a292aa9..25c510ed5 100644 --- a/osfmk/kperf/x86_64/kperf_mp.c +++ b/osfmk/kperf/x86_64/kperf_mp.c @@ -26,43 +26,65 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +#include #include #include -#include - +#include #include #include #include +#include -void -kperf_mp_broadcast_running(struct kperf_timer *trigger) +bool +kperf_mp_broadcast_other_running(struct kperf_timer *trigger) { + int current_cpu = cpu_number(); int ncpus = machine_info.logical_cpu_max; + bool system_only_self = true; cpumask_t cpu_mask = 0; - assert(ncpus < 64); for (int i = 0; i < ncpus; i++) { - /* do not IPI processors that are not scheduling threads */ + uint64_t i_bit = UINT64_C(1) << i; processor_t processor = cpu_to_processor(i); + + /* do not IPI processors that are not scheduling threads */ if (processor == PROCESSOR_NULL || - processor->state != PROCESSOR_RUNNING || - processor->active_thread == THREAD_NULL) + processor->state != PROCESSOR_RUNNING || + processor->active_thread == THREAD_NULL) { +#if DEVELOPMENT || DEBUG + BUF_VERB(PERF_TM_SKIPPED, i, + processor != PROCESSOR_NULL ? processor->state : 0, + processor != PROCESSOR_NULL ? processor->active_thread : 0); +#endif /* DEVELOPMENT || DEBUG */ + continue; + } + + /* don't run the handler on the current processor */ + if (i == current_cpu) { + system_only_self = false; continue; } /* nor processors that have not responded to the last IPI */ - bool already_pending = atomic_bit_set(&(trigger->pending_cpus), i, - __ATOMIC_RELAXED); - if (already_pending) { + uint64_t already_pending = atomic_fetch_or_explicit( + &trigger->pending_cpus, i_bit, + memory_order_relaxed); + if (already_pending & i_bit) { #if DEVELOPMENT || DEBUG - __c11_atomic_fetch_add(&kperf_pending_ipis, 1, __ATOMIC_RELAXED); -#endif + BUF_VERB(PERF_TM_PENDING, i_bit, already_pending); + atomic_fetch_add_explicit(&kperf_pending_ipis, 1, + memory_order_relaxed); +#endif /* DEVELOPMENT || DEBUG */ continue; } cpu_mask |= cpu_to_cpumask(i); } - mp_cpus_call(cpu_mask, NOSYNC, kperf_ipi_handler, trigger); + if (cpu_mask != 0) { + mp_cpus_call(cpu_mask, NOSYNC, kperf_ipi_handler, trigger); + } + + return system_only_self; } diff --git a/osfmk/libsa/arm/types.h b/osfmk/libsa/arm/types.h new file mode 100644 index 000000000..9a6836ca3 --- /dev/null +++ b/osfmk/libsa/arm/types.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * HISTORY + * + * Revision 1.2 1998/09/30 21:21:00 wsanchez + * Merged in IntelMerge1 (mburg: Intel support) + * + * Revision 1.1.2.1 1998/09/30 18:19:50 mburg + * Changes for Intel port + * + * Revision 1.1.1.1 1998/03/07 02:25:36 wsanchez + * Import of OSF Mach kernel (~mburg) + * + * Revision 1.1.2.1 1996/09/17 16:56:28 bruel + * created from standalone mach servers + * [1996/09/17 16:18:08 bruel] + * + * Revision 1.1.7.1 1996/04/11 13:46:28 barbou + * Self-Contained Mach Distribution: + * created. + * [95/12/28 barbou] + * [96/03/28 barbou] + * + * $EndLog$ + */ + +#ifndef _MACH_MACHINE_TYPES_H_ +#define _MACH_MACHINE_TYPES_H_ 1 + +typedef long dev_t; /* device number (major+minor) */ + +typedef signed char bit8_t; /* signed 8-bit quantity */ +typedef unsigned char u_bit8_t; /* unsigned 8-bit quantity */ + +typedef short bit16_t; /* signed 16-bit quantity */ +typedef unsigned short u_bit16_t; /* unsigned 16-bit quantity */ + +typedef int bit32_t; /* signed 32-bit quantity */ +typedef unsigned int u_bit32_t; /* unsigned 32-bit quantity */ + +/* Only 32 bits of the "bit64_t" are significant on this 32-bit machine */ +typedef struct { int __val[2]; } bit64_t; /* signed 64-bit quantity */ +typedef struct { unsigned int __val[2]; } u_bit64_t;/* unsigned 64-bit quantity */ +#define _SIG64_BITS __val[0] /* bits of interest (32) */ + +#endif /* _MACH_MACHINE_TYPES_H_ */ diff --git a/osfmk/libsa/machine/types.h b/osfmk/libsa/machine/types.h index 0a6f4bb69..08753f955 100644 --- a/osfmk/libsa/machine/types.h +++ b/osfmk/libsa/machine/types.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "libsa/i386/types.h" +#elif defined (__arm__)|| defined (__arm64__) +#include "libsa/arm/types.h" #else #error architecture not supported #endif diff --git a/osfmk/libsa/string.h b/osfmk/libsa/string.h index 731cd7653..894027e28 100644 --- a/osfmk/libsa/string.h +++ b/osfmk/libsa/string.h @@ -60,6 +60,7 @@ extern void *memcpy(void *, const void *, size_t); extern int memcmp(const void *, const void *, size_t); extern void *memmove(void *, const void *, size_t); extern void *memset(void *, int, size_t); +extern int memset_s(void *, size_t, int, size_t); extern size_t strlen(const char *); extern size_t strnlen(const char *, size_t); @@ -83,6 +84,9 @@ extern int strcasecmp(const char *s1, const char *s2); extern int strncasecmp(const char *s1, const char *s2, size_t n); extern char *strnstr(char *s, const char *find, size_t slen); extern char *strchr(const char *s, int c); +#ifdef XNU_KERNEL_PRIVATE +extern char *strrchr(const char *s, int c); +#endif extern char *STRDUP(const char *, int); extern int strprefix(const char *s1, const char *s2); @@ -90,6 +94,52 @@ extern int bcmp(const void *, const void *, size_t); extern void bcopy(const void *, void *, size_t); extern void bzero(void *, size_t); +#ifdef PRIVATE +#include +#endif + +#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_10_13 +/* older deployment target */ +#elif defined(KASAN) || (defined (_FORTIFY_SOURCE) && _FORTIFY_SOURCE == 0) +/* FORTIFY_SOURCE disabled */ +#else /* _chk macros */ +#if __has_builtin(__builtin___memcpy_chk) +#define memcpy(dest, src, len) __builtin___memcpy_chk(dest, src, len, __builtin_object_size(dest, 0)) +#endif + +#if __has_builtin(__builtin___memmove_chk) +#define memmove(dest, src, len) __builtin___memmove_chk(dest, src, len, __builtin_object_size(dest, 0)) +#endif + +#if __has_builtin(__builtin___strncpy_chk) +#define strncpy(dest, src, len) __builtin___strncpy_chk(dest, src, len, __builtin_object_size(dest, 1)) +#endif + +#if __has_builtin(__builtin___strncat_chk) +#define strncat(dest, src, len) __builtin___strncat_chk(dest, src, len, __builtin_object_size(dest, 1)) +#endif + +#if __has_builtin(__builtin___strlcat_chk) +#define strlcat(dest, src, len) __builtin___strlcat_chk(dest, src, len, __builtin_object_size(dest, 1)) +#endif + +#if __has_builtin(__builtin___strlcpy_chk) +#define strlcpy(dest, src, len) __builtin___strlcpy_chk(dest, src, len, __builtin_object_size(dest, 1)) +#endif + +#if __has_builtin(__builtin___strcpy_chk) +#define strcpy(dest, src, len) __builtin___strcpy_chk(dest, src, __builtin_object_size(dest, 1)) +#endif + +#if __has_builtin(__builtin___strcat_chk) +#define strcat(dest, src) __builtin___strcat_chk(dest, src, __builtin_object_size(dest, 1)) +#endif + +#if __has_builtin(__builtin___memmove_chk) +#define bcopy(src, dest, len) __builtin___memmove_chk(dest, src, len, __builtin_object_size(dest, 0)) +#endif + +#endif /* _chk macros */ #ifdef __cplusplus } #endif diff --git a/osfmk/mach/Makefile b/osfmk/mach/Makefile index 079e45c41..815aa22d9 100644 --- a/osfmk/mach/Makefile +++ b/osfmk/mach/Makefile @@ -137,6 +137,7 @@ DATAFILES = \ sync_policy.h \ syscall_sw.h \ task_info.h \ + task_inspect.h \ task_policy.h \ task_special_ports.h \ thread_info.h \ @@ -182,6 +183,7 @@ PRIVATE_DATAFILES = \ syscall_sw.h \ sysdiagnose_notification.defs \ task_info.h \ + task_inspect.h \ task_policy.h \ thread_policy.h \ thread_switch.h \ diff --git a/osfmk/mach/arm/Makefile b/osfmk/mach/arm/Makefile new file mode 100644 index 000000000..2edb04c4e --- /dev/null +++ b/osfmk/mach/arm/Makefile @@ -0,0 +1,32 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +VPATH := $(VPATH):$(SOURCE)/../../arm + +DATAFILES = \ + boolean.h exception.h kern_return.h ndr_def.h \ + processor_info.h rpc.h thread_state.h thread_status.h \ + vm_param.h vm_types.h \ + syscall_sw.h _structs.h sdt_isa.h + +INSTALL_MD_LIST = ${DATAFILES} + +INSTALL_MD_GEN_LIST = \ + asm.h + +INSTALL_MD_DIR = mach/arm + +EXPORT_MD_LIST = ${DATAFILES} + +EXPORT_MD_GEN_LIST = \ + asm.h + +EXPORT_MD_DIR = mach/arm + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/osfmk/mach/arm/_structs.h b/osfmk/mach/arm/_structs.h new file mode 100644 index 000000000..b7db3d2b6 --- /dev/null +++ b/osfmk/mach/arm/_structs.h @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2004-2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +#ifndef _MACH_ARM__STRUCTS_H_ +#define _MACH_ARM__STRUCTS_H_ + +#include /* __DARWIN_UNIX03 */ +#include /* __uint32_t */ + +#if __DARWIN_UNIX03 +#define _STRUCT_ARM_EXCEPTION_STATE struct __darwin_arm_exception_state +_STRUCT_ARM_EXCEPTION_STATE +{ + __uint32_t __exception; /* number of arm exception taken */ + __uint32_t __fsr; /* Fault status */ + __uint32_t __far; /* Virtual Fault Address */ +}; +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_ARM_EXCEPTION_STATE struct arm_exception_state +_STRUCT_ARM_EXCEPTION_STATE +{ + __uint32_t exception; /* number of arm exception taken */ + __uint32_t fsr; /* Fault status */ + __uint32_t far; /* Virtual Fault Address */ +}; +#endif /* __DARWIN_UNIX03 */ + +#if __DARWIN_UNIX03 +#define _STRUCT_ARM_EXCEPTION_STATE64 struct __darwin_arm_exception_state64 +_STRUCT_ARM_EXCEPTION_STATE64 +{ + __uint64_t __far; /* Virtual Fault Address */ + __uint32_t __esr; /* Exception syndrome */ + __uint32_t __exception; /* number of arm exception taken */ +}; +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_ARM_EXCEPTION_STATE64 struct arm_exception_state64 +_STRUCT_ARM_EXCEPTION_STATE64 +{ + __uint64_t far; /* Virtual Fault Address */ + __uint32_t esr; /* Exception syndrome */ + __uint32_t exception; /* number of arm exception taken */ +}; +#endif /* __DARWIN_UNIX03 */ + +#if __DARWIN_UNIX03 +#define _STRUCT_ARM_THREAD_STATE struct __darwin_arm_thread_state +_STRUCT_ARM_THREAD_STATE +{ + __uint32_t __r[13]; /* General purpose register r0-r12 */ + __uint32_t __sp; /* Stack pointer r13 */ + __uint32_t __lr; /* Link register r14 */ + __uint32_t __pc; /* Program counter r15 */ + __uint32_t __cpsr; /* Current program status register */ +}; +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_ARM_THREAD_STATE struct arm_thread_state +_STRUCT_ARM_THREAD_STATE +{ + __uint32_t r[13]; /* General purpose register r0-r12 */ + __uint32_t sp; /* Stack pointer r13 */ + __uint32_t lr; /* Link register r14 */ + __uint32_t pc; /* Program counter r15 */ + __uint32_t cpsr; /* Current program status register */ +}; +#endif /* __DARWIN_UNIX03 */ + +#if __DARWIN_UNIX03 +#define _STRUCT_ARM_THREAD_STATE64 struct __darwin_arm_thread_state64 +_STRUCT_ARM_THREAD_STATE64 +{ + __uint64_t __x[29]; /* General purpose registers x0-x28 */ + __uint64_t __fp; /* Frame pointer x29 */ + __uint64_t __lr; /* Link register x30 */ + __uint64_t __sp; /* Stack pointer x31 */ + __uint64_t __pc; /* Program counter */ + __uint32_t __cpsr; /* Current program status register */ + __uint32_t __pad; /* Same size for 32-bit or 64-bit clients */ +}; +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_ARM_THREAD_STATE64 struct arm_thread_state64 +_STRUCT_ARM_THREAD_STATE64 +{ + __uint64_t x[29]; /* General purpose registers x0-x28 */ + __uint64_t fp; /* Frame pointer x29 */ + __uint64_t lr; /* Link register x30 */ + __uint64_t sp; /* Stack pointer x31 */ + __uint64_t pc; /* Program counter */ + __uint32_t cpsr; /* Current program status register */ + __uint32_t __pad; /* Same size for 32-bit or 64-bit clients */ +}; +#endif /* __DARWIN_UNIX03 */ + +#if __DARWIN_UNIX03 +#define _STRUCT_ARM_VFP_STATE struct __darwin_arm_vfp_state +_STRUCT_ARM_VFP_STATE +{ + __uint32_t __r[64]; + __uint32_t __fpscr; + +}; +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_ARM_VFP_STATE struct arm_vfp_state +_STRUCT_ARM_VFP_STATE +{ + __uint32_t r[64]; + __uint32_t fpscr; +}; +#endif /* __DARWIN_UNIX03 */ + +#if __DARWIN_UNIX03 +#define _STRUCT_ARM_NEON_STATE64 struct __darwin_arm_neon_state64 +#define _STRUCT_ARM_NEON_STATE struct __darwin_arm_neon_state + +#if defined(__arm64__) +_STRUCT_ARM_NEON_STATE64 +{ + __uint128_t __v[32]; + __uint32_t __fpsr; + __uint32_t __fpcr; +}; + +_STRUCT_ARM_NEON_STATE +{ + __uint128_t __v[16]; + __uint32_t __fpsr; + __uint32_t __fpcr; +}; + +#elif defined(__arm__) +/* + * No 128-bit intrinsic for ARM; leave it opaque for now. + */ +_STRUCT_ARM_NEON_STATE64 +{ + char opaque[(32 * 16) + (2 * sizeof(__uint32_t))]; +} __attribute__((aligned(16))); + +_STRUCT_ARM_NEON_STATE +{ + char opaque[(16 * 16) + (2 * sizeof(__uint32_t))]; +} __attribute__((aligned(16))); + +#else +#error Unknown architecture. +#endif + +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_ARM_NEON_STATE64 struct arm_neon_state64 +#define _STRUCT_ARM_NEON_STATE struct arm_neon_state + +#if defined(__arm64__) +_STRUCT_ARM_NEON_STATE64 +{ + __uint128_t q[32]; + uint32_t fpsr; + uint32_t fpcr; + +}; +_STRUCT_ARM_NEON_STATE +{ + __uint128_t q[16]; + uint32_t fpsr; + uint32_t fpcr; + +}; +#elif defined(__arm__) +/* + * No 128-bit intrinsic for ARM; leave it opaque for now. + */ +_STRUCT_ARM_NEON_STATE64 +{ + char opaque[(32 * 16) + (2 * sizeof(__uint32_t))]; +} __attribute__((aligned(16))); + +_STRUCT_ARM_NEON_STATE +{ + char opaque[(16 * 16) + (2 * sizeof(__uint32_t))]; +} __attribute__((aligned(16))); + +#else +#error Unknown architecture. +#endif + +#endif /* __DARWIN_UNIX03 */ + +/* + * Debug State + */ +#if defined(__arm__) +/* Old-fashioned debug state is only for ARM */ + +#if __DARWIN_UNIX03 +#define _STRUCT_ARM_DEBUG_STATE struct __darwin_arm_debug_state +_STRUCT_ARM_DEBUG_STATE +{ + __uint32_t __bvr[16]; + __uint32_t __bcr[16]; + __uint32_t __wvr[16]; + __uint32_t __wcr[16]; +}; +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_ARM_DEBUG_STATE struct arm_debug_state +_STRUCT_ARM_DEBUG_STATE +{ + __uint32_t bvr[16]; + __uint32_t bcr[16]; + __uint32_t wvr[16]; + __uint32_t wcr[16]; +}; +#endif /* __DARWIN_UNIX03 */ + +#elif defined(__arm64__) + +/* ARM's arm_debug_state is ARM64's arm_legacy_debug_state */ + +#if __DARWIN_UNIX03 +#define _STRUCT_ARM_LEGACY_DEBUG_STATE struct arm_legacy_debug_state +_STRUCT_ARM_LEGACY_DEBUG_STATE +{ + __uint32_t __bvr[16]; + __uint32_t __bcr[16]; + __uint32_t __wvr[16]; + __uint32_t __wcr[16]; +}; +#else /* __DARWIN_UNIX03 */ +#define _STRUCT_ARM_LEGACY_DEBUG_STATE struct arm_legacy_debug_state +_STRUCT_ARM_LEGACY_DEBUG_STATE +{ + __uint32_t bvr[16]; + __uint32_t bcr[16]; + __uint32_t wvr[16]; + __uint32_t wcr[16]; +}; +#endif /* __DARWIN_UNIX03 */ +#else +#error unknown architecture +#endif + +#if __DARWIN_UNIX03 +#define _STRUCT_ARM_DEBUG_STATE32 struct __darwin_arm_debug_state32 +_STRUCT_ARM_DEBUG_STATE32 +{ + __uint32_t __bvr[16]; + __uint32_t __bcr[16]; + __uint32_t __wvr[16]; + __uint32_t __wcr[16]; + __uint64_t __mdscr_el1; /* Bit 0 is SS (Hardware Single Step) */ +}; + +#define _STRUCT_ARM_DEBUG_STATE64 struct __darwin_arm_debug_state64 +_STRUCT_ARM_DEBUG_STATE64 +{ + __uint64_t __bvr[16]; + __uint64_t __bcr[16]; + __uint64_t __wvr[16]; + __uint64_t __wcr[16]; + __uint64_t __mdscr_el1; /* Bit 0 is SS (Hardware Single Step) */ +}; +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_ARM_DEBUG_STATE32 struct arm_debug_state32 +_STRUCT_ARM_DEBUG_STATE32 +{ + __uint32_t bvr[16]; + __uint32_t bcr[16]; + __uint32_t wvr[16]; + __uint32_t wcr[16]; + __uint64_t mdscr_el1; /* Bit 0 is SS (Hardware Single Step) */ +}; + +#define _STRUCT_ARM_DEBUG_STATE64 struct arm_debug_state64 +_STRUCT_ARM_DEBUG_STATE64 +{ + __uint64_t bvr[16]; + __uint64_t bcr[16]; + __uint64_t wvr[16]; + __uint64_t wcr[16]; + __uint64_t mdscr_el1; /* Bit 0 is SS (Hardware Single Step) */ +}; +#endif /* __DARWIN_UNIX03 */ + +#if __DARWIN_UNIX03 +#define _STRUCT_ARM_CPMU_STATE64 struct __darwin_arm_cpmu_state64 +_STRUCT_ARM_CPMU_STATE64 +{ + __uint64_t __ctrs[16]; +}; +#else /* __DARWIN_UNIX03 */ +#define _STRUCT_ARM_CPMU_STATE64 struct arm_cpmu_state64 +_STRUCT_ARM_CPMU_STATE64 +{ + __uint64_t ctrs[16]; +}; +#endif /* !__DARWIN_UNIX03 */ + +#endif /* _MACH_ARM__STRUCTS_H_ */ diff --git a/osfmk/mach/arm/boolean.h b/osfmk/mach/arm/boolean.h new file mode 100644 index 000000000..dd0afc6e3 --- /dev/null +++ b/osfmk/mach/arm/boolean.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +/* + * File: boolean.h + * + * Boolean type, for ARM. + */ + +#ifndef _MACH_ARM_BOOLEAN_H_ +#define _MACH_ARM_BOOLEAN_H_ + +typedef int boolean_t; + +#endif /* _MACH_ARM_BOOLEAN_H_ */ diff --git a/osfmk/mach/arm/exception.h b/osfmk/mach/arm/exception.h new file mode 100644 index 000000000..857c1fa0a --- /dev/null +++ b/osfmk/mach/arm/exception.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _MACH_ARM_EXCEPTION_H_ +#define _MACH_ARM_EXCEPTION_H_ + +#define EXC_TYPES_COUNT 14 /* incl. illegal exception 0 */ + +#define EXC_MASK_MACHINE 0 + +#define EXCEPTION_CODE_MAX 2 /* code and subcode */ + +/* + * Trap numbers as defined by the hardware exception vectors. + */ + +/* + * EXC_BAD_INSTRUCTION + */ + +#define EXC_ARM_UNDEFINED 1 /* Undefined */ + + +/* + * EXC_BAD_ACCESS + * Note: do not conflict with kern_return_t values returned by vm_fault + */ + +#define EXC_ARM_DA_ALIGN 0x101 /* Alignment Fault */ +#define EXC_ARM_DA_DEBUG 0x102 /* Debug (watch/break) Fault */ +#define EXC_ARM_SP_ALIGN 0x103 /* SP Alignment Fault */ +#define EXC_ARM_SWP 0x104 /* SWP instruction */ + +/* + * EXC_BREAKPOINT + */ + +#define EXC_ARM_BREAKPOINT 1 /* breakpoint trap */ + + +#endif /* _MACH_ARM_EXCEPTION_H_ */ diff --git a/osfmk/mach/arm/kern_return.h b/osfmk/mach/arm/kern_return.h new file mode 100644 index 000000000..eb36fc2b2 --- /dev/null +++ b/osfmk/mach/arm/kern_return.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +/* + * File: kern_return.h + * Author: Avadis Tevanian, Jr., Michael Wayne Young + * Date: 1985 + * + * Machine-dependent kernel return definitions. + */ + +#ifndef _MACH_ARM_KERN_RETURN_H_ +#define _MACH_ARM_KERN_RETURN_H_ + +#ifndef ASSEMBLER +typedef int kern_return_t; +#endif /* ASSEMBLER */ + +#endif /* _MACH_ARM_KERN_RETURN_H_ */ diff --git a/osfmk/mach/arm/ndr_def.h b/osfmk/mach/arm/ndr_def.h new file mode 100644 index 000000000..49ea78052 --- /dev/null +++ b/osfmk/mach/arm/ndr_def.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* NDR record for Intel x86s */ + +#include + +NDR_record_t NDR_record = { + 0, /* mig_reserved */ + 0, /* mig_reserved */ + 0, /* mig_reserved */ + NDR_PROTOCOL_2_0, + NDR_INT_LITTLE_ENDIAN, + NDR_CHAR_ASCII, + NDR_FLOAT_IEEE, + 0, +}; diff --git a/osfmk/mach/arm/processor_info.h b/osfmk/mach/arm/processor_info.h new file mode 100644 index 000000000..67f9df9b8 --- /dev/null +++ b/osfmk/mach/arm/processor_info.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * FILE_ID: processor_info.h + */ + +#ifndef _MACH_ARM_PROCESSOR_INFO_H_ +#define _MACH_ARM_PROCESSOR_INFO_H_ + +#define PROCESSOR_CPU_STAT 0x10000003 /* Low level cpu statistics */ + +#include /* uint32_t */ + +struct processor_cpu_stat { + uint32_t irq_ex_cnt; + uint32_t ipi_cnt; + uint32_t timer_cnt; + uint32_t undef_ex_cnt; + uint32_t unaligned_cnt; + uint32_t vfp_cnt; + uint32_t vfp_shortv_cnt; + uint32_t data_ex_cnt; + uint32_t instr_ex_cnt; +}; + +typedef struct processor_cpu_stat processor_cpu_stat_data_t; +typedef struct processor_cpu_stat *processor_cpu_stat_t; +#define PROCESSOR_CPU_STAT_COUNT ((mach_msg_type_number_t) \ + (sizeof(processor_cpu_stat_data_t)/sizeof(natural_t))) + + +#endif /* _MACH_ARM_PROCESSOR_INFO_H_ */ diff --git a/osfmk/mach/arm/rpc.h b/osfmk/mach/arm/rpc.h new file mode 100644 index 000000000..9aebaead5 --- /dev/null +++ b/osfmk/mach/arm/rpc.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#ifndef _MACH_ARM_RPC_H_ +#define _MACH_ARM_RPC_H_ + +#endif /* _MACH_ARM_RPC_H_ */ diff --git a/osfmk/mach/arm/sdt_isa.h b/osfmk/mach/arm/sdt_isa.h new file mode 100644 index 000000000..318134d1f --- /dev/null +++ b/osfmk/mach/arm/sdt_isa.h @@ -0,0 +1,440 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _MACH_ARM_SDT_ISA_H +#define _MACH_ARM_SDT_ISA_H + +/* #pragma ident "@(#)sdt.h 1.7 05/06/08 SMI" */ + +/* + * Only define when testing. This makes the calls into actual calls to + * test functions. + */ +/* #define DTRACE_CALL_TEST */ + +#define DTRACE_STRINGIFY(s) #s +#define DTRACE_TOSTRING(s) DTRACE_STRINGIFY(s) + +#if defined(KERNEL) +/* + * For the kernel, set an explicit global label so the symbol can be located + */ +#ifdef __arm__ +#define DTRACE_LAB(p, n) \ + "__dtrace_probe$" DTRACE_TOSTRING(%=__LINE__) DTRACE_STRINGIFY(_##p##___##n) + +#define DTRACE_LABEL(p, n) \ + ".pushsection __DATA, __data\n\t" \ + ".globl " DTRACE_LAB(p, n) "\n\t" \ + DTRACE_LAB(p, n) ":" ".long 1f""\n\t" \ + ".popsection" "\n\t" \ + "1:" +#else /* __arm64__ */ +#define DTRACE_LAB(p, n) \ + "__dtrace_probe$" DTRACE_TOSTRING(%=__LINE__) DTRACE_STRINGIFY(_##p##___##n) + +#define DTRACE_LABEL(p, n) \ + ".pushsection __DATA, __data\n\t" \ + ".globl " DTRACE_LAB(p, n) "\n\t" \ + DTRACE_LAB(p, n) ":" ".quad 1f""\n\t" \ + ".popsection" "\n\t" \ + "1:" +#endif +#else /* !KERNEL */ +#define DTRACE_LABEL(p, n) \ + "__dtrace_probe$" DTRACE_TOSTRING(%=__LINE__) DTRACE_STRINGIFY(_##p##___##n) ":" "\n\t" +#endif /* !KERNEL */ + +#ifdef DTRACE_CALL_TEST + +#define DTRACE_CALL(p,n) \ + DTRACE_LABEL(p,n) \ + DTRACE_CALL_INSN(p,n) + +#else /* !DTRACE_CALL_TEST */ + +#define DTRACE_CALL(p,n) \ + DTRACE_LABEL(p,n) \ + DTRACE_NOPS + +#endif /* !DTRACE_CALL_TEST */ + +#if defined(__arm__) + +#define DTRACE_NOPS \ + "nop" "\n\t" + +#define DTRACE_CALL_INSN(p,n) \ + "blx _dtracetest" DTRACE_STRINGIFY(_##p##_##n) "\n\t" + +#ifdef __thumb__ +#define DTRACE_ALLOC_STACK(n) \ + "sub sp, #" #n "\n\t" +#define DTRACE_DEALLOC_STACK(n) \ + "add sp, #" #n "\n\t" +#else +#define DTRACE_ALLOC_STACK(n) \ + "sub sp, sp, #" #n "\n\t" +#define DTRACE_DEALLOC_STACK(n) \ + "add sp, sp, #" #n "\n\t" +#endif + +#define ARG1_EXTENT 1 +#define ARGS2_EXTENT 2 +#define ARGS3_EXTENT 3 +#define ARGS4_EXTENT 4 +#define ARGS5_EXTENT 5 +#define ARGS6_EXTENT 6 +#define ARGS7_EXTENT 7 +#define ARGS8_EXTENT 8 +#define ARGS9_EXTENT 9 +#define ARGS10_EXTENT 10 + +#define DTRACE_CALL0ARGS(provider, name) \ + asm volatile ( \ + DTRACE_CALL(provider, name) \ + "# eat trailing nl+tab from DTRACE_CALL" \ + : \ + : \ + ); + +#define DTRACE_CALL1ARG(provider, name) \ + asm volatile ("ldr r0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "l" (__dtrace_args) \ + : "memory", "r0" \ + ); + +#define DTRACE_CALL2ARGS(provider, name) \ + asm volatile ("ldr r1, [%0, #4]" "\n\t" \ + "ldr r0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "l" (__dtrace_args) \ + : "memory", "r0", "r1" \ + ); + +#define DTRACE_CALL3ARGS(provider, name) \ + asm volatile ("ldr r2, [%0, #8]" "\n\t" \ + "ldr r1, [%0, #4]" "\n\t" \ + "ldr r0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "l" (__dtrace_args) \ + : "memory", "r0", "r1", "r2" \ + ); + +#define DTRACE_CALL4ARGS(provider, name) \ + asm volatile ("ldr r3, [%0, #12]" "\n\t" \ + "ldr r2, [%0, #8]" "\n\t" \ + "ldr r1, [%0, #4]" "\n\t" \ + "ldr r0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "l" (__dtrace_args) \ + : "memory", "r0", "r1", "r2", "r3" \ + ); + +/* + * One of our ARM32 ABIs (armv7k) mandates that the stack be aligned to 16 bytes. + * We currently apply this constraint to all ARM32 DTRACE_CALL macros; hence the + * macros below will overallocate for some ABIs. + */ +#define DTRACE_CALL5ARGS(provider, name) \ + asm volatile ( \ + DTRACE_ALLOC_STACK(16) \ + "ldr r0, [%0, #16]" "\n\t" \ + "str r0, [sp]" "\n\t" \ + "ldr r3, [%0, #12]" "\n\t" \ + "ldr r2, [%0, #8]" "\n\t" \ + "ldr r1, [%0, #4]" "\n\t" \ + "ldr r0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + DTRACE_DEALLOC_STACK(16) \ + : \ + : "l" (__dtrace_args) \ + : "memory", "r0", "r1", "r2", "r3" \ + ); + +#define DTRACE_CALL6ARGS(provider, name) \ + asm volatile ( \ + DTRACE_ALLOC_STACK(16) \ + "ldr r1, [%0, #20]" "\n\t" \ + "ldr r0, [%0, #16]" "\n\t" \ + "str r1, [sp, #4]" "\n\t" \ + "str r0, [sp]" "\n\t" \ + "ldr r3, [%0, #12]" "\n\t" \ + "ldr r2, [%0, #8]" "\n\t" \ + "ldr r1, [%0, #4]" "\n\t" \ + "ldr r0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + DTRACE_DEALLOC_STACK(16) \ + : \ + : "l" (__dtrace_args) \ + : "memory", "r0", "r1", "r2", "r3" \ + ); + +#define DTRACE_CALL7ARGS(provider, name) \ + asm volatile ( \ + DTRACE_ALLOC_STACK(16) \ + "ldr r2, [%0, #24]" "\n\t" \ + "ldr r1, [%0, #20]" "\n\t" \ + "ldr r0, [%0, #16]" "\n\t" \ + "str r2, [sp, #8]" "\n\t" \ + "str r1, [sp, #4]" "\n\t" \ + "str r0, [sp]" "\n\t" \ + "ldr r3, [%0, #12]" "\n\t" \ + "ldr r2, [%0, #8]" "\n\t" \ + "ldr r1, [%0, #4]" "\n\t" \ + "ldr r0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + DTRACE_DEALLOC_STACK(16) \ + : \ + : "l" (__dtrace_args) \ + : "memory", "r0", "r1", "r2", "r3" \ + ); + +#define DTRACE_CALL8ARGS(provider, name) \ + asm volatile ( \ + DTRACE_ALLOC_STACK(16) \ + "ldr r3, [%0, #28]" "\n\t" \ + "ldr r2, [%0, #24]" "\n\t" \ + "ldr r1, [%0, #20]" "\n\t" \ + "ldr r0, [%0, #16]" "\n\t" \ + "str r3, [sp, #12]" "\n\t" \ + "str r2, [sp, #8]" "\n\t" \ + "str r1, [sp, #4]" "\n\t" \ + "str r0, [sp]" "\n\t" \ + "ldr r3, [%0, #12]" "\n\t" \ + "ldr r2, [%0, #8]" "\n\t" \ + "ldr r1, [%0, #4]" "\n\t" \ + "ldr r0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + DTRACE_DEALLOC_STACK(16) \ + : \ + : "l" (__dtrace_args) \ + : "memory", "r0", "r1", "r2", "r3" \ + ); + +#define DTRACE_CALL9ARGS(provider, name) \ + asm volatile ( \ + DTRACE_ALLOC_STACK(32) \ + "ldr r0, [%0, #32]" "\n\t" \ + "str r0, [sp, #16]" "\n\t" \ + "ldr r3, [%0, #28]" "\n\t" \ + "ldr r2, [%0, #24]" "\n\t" \ + "ldr r1, [%0, #20]" "\n\t" \ + "ldr r0, [%0, #16]" "\n\t" \ + "str r3, [sp, #12]" "\n\t" \ + "str r2, [sp, #8]" "\n\t" \ + "str r1, [sp, #4]" "\n\t" \ + "str r0, [sp]" "\n\t" \ + "ldr r3, [%0, #12]" "\n\t" \ + "ldr r2, [%0, #8]" "\n\t" \ + "ldr r1, [%0, #4]" "\n\t" \ + "ldr r0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + DTRACE_DEALLOC_STACK(32) \ + : \ + : "l" (__dtrace_args) \ + : "memory", "r0", "r1", "r2", "r3" \ + ); + +#define DTRACE_CALL10ARGS(provider, name) \ + asm volatile ( \ + DTRACE_ALLOC_STACK(32) \ + "ldr r1, [%0, #36]" "\n\t" \ + "ldr r0, [%0, #32]" "\n\t" \ + "str r1, [sp, #20]" "\n\t" \ + "str r0, [sp, #16]" "\n\t" \ + "ldr r3, [%0, #28]" "\n\t" \ + "ldr r2, [%0, #24]" "\n\t" \ + "ldr r1, [%0, #20]" "\n\t" \ + "ldr r0, [%0, #16]" "\n\t" \ + "str r3, [sp, #12]" "\n\t" \ + "str r2, [sp, #8]" "\n\t" \ + "str r1, [sp, #4]" "\n\t" \ + "str r0, [sp]" "\n\t" \ + "ldr r3, [%0, #12]" "\n\t" \ + "ldr r2, [%0, #8]" "\n\t" \ + "ldr r1, [%0, #4]" "\n\t" \ + "ldr r0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + DTRACE_DEALLOC_STACK(32) \ + : \ + : "l" (__dtrace_args) \ + : "memory", "r0", "r1", "r2", "r3" \ + ); + +#elif defined(__arm64__) + +#define DTRACE_NOPS \ + "nop" "\n\t" + + +#define DTRACE_CALL_INSN(p,n) \ + "bl _dtracetest" DTRACE_STRINGIFY(_##p##_##n) "\n\t" + +#define DTRACE_ALLOC_STACK(n) \ + "sub sp, sp, #" #n "\n\t" +#define DTRACE_DEALLOC_STACK(n) \ + "add sp, sp, #" #n "\n\t" + +#define ARG1_EXTENT 1 +#define ARGS2_EXTENT 2 +#define ARGS3_EXTENT 3 +#define ARGS4_EXTENT 4 +#define ARGS5_EXTENT 5 +#define ARGS6_EXTENT 6 +#define ARGS7_EXTENT 7 +#define ARGS8_EXTENT 8 +#define ARGS9_EXTENT 9 +#define ARGS10_EXTENT 10 + +#define DTRACE_CALL0ARGS(provider, name) \ + asm volatile ( \ + DTRACE_CALL(provider, name) \ + "# eat trailing nl+tab from DTRACE_CALL" \ + : \ + : \ + ); + +#define DTRACE_CALL1ARG(provider, name) \ + asm volatile ("ldr x0, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "r" (__dtrace_args) \ + : "memory", "x0" \ + ); + +#define DTRACE_CALL2ARGS(provider, name) \ + asm volatile ("ldp x0, x1, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "r" (__dtrace_args) \ + : "memory", "x0", "x1" \ + ); + +#define DTRACE_CALL3ARGS(provider, name) \ + asm volatile ("ldr x2, [%0, #16]" "\n\t" \ + "ldp x0, x1, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "r" (__dtrace_args) \ + : "memory", "x0", "x1", "x2" \ + ); + +#define DTRACE_CALL4ARGS(provider, name) \ + asm volatile ("ldp x2, x3, [%0, #16]" "\n\t" \ + "ldp x0, x1, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "r" (__dtrace_args) \ + : "memory", "x0", "x1", "x2", "x3" \ + ); + +#define DTRACE_CALL5ARGS(provider, name) \ + asm volatile ("ldr x4, [%0, #32]" "\n\t" \ + "ldp x2, x3, [%0, #16]" "\n\t" \ + "ldp x0, x1, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "r" (__dtrace_args) \ + : "memory", "x0", "x1", "x2", "x3", "x4" \ + ); + +#define DTRACE_CALL6ARGS(provider, name) \ + asm volatile ("ldp x4, x5, [%0, #32]" "\n\t" \ + "ldp x2, x3, [%0, #16]" "\n\t" \ + "ldp x0, x1, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "r" (__dtrace_args) \ + : "memory", "x0", "x1", "x2", "x3", "x4", "x5" \ + ); + +#define DTRACE_CALL7ARGS(provider, name) \ + asm volatile ("ldr x6, [%0, #48]" "\n\t" \ + "ldp x4, x5, [%0, #32]" "\n\t" \ + "ldp x2, x3, [%0, #16]" "\n\t" \ + "ldp x0, x1, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "r" (__dtrace_args) \ + : "memory", "x0", "x1", "x2", "x3", "x4", "x5", "x6" \ + ); + +#define DTRACE_CALL8ARGS(provider, name) \ + asm volatile ("ldp x6, x7, [%0, #48]" "\n\t" \ + "ldp x4, x5, [%0, #32]" "\n\t" \ + "ldp x2, x3, [%0, #16]" "\n\t" \ + "ldp x0, x1, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + : \ + : "r" (__dtrace_args) \ + : "memory", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7" \ + ); + +/* Keep stack 16 byte aligned per ABI requirements */ +#define DTRACE_CALL9ARGS(provider, name) \ + asm volatile ( \ + DTRACE_ALLOC_STACK(16) \ + "ldr x0, [%0, #64]" "\n\t" \ + "str x0, [sp]" "\n\t" \ + "ldp x6, x7, [%0, #48]" "\n\t" \ + "ldp x4, x5, [%0, #32]" "\n\t" \ + "ldp x2, x3, [%0, #16]" "\n\t" \ + "ldp x0, x1, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + DTRACE_DEALLOC_STACK(16) \ + : \ + : "r" (__dtrace_args) \ + : "memory", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7" \ + ); + +#define DTRACE_CALL10ARGS(provider, name) \ + asm volatile ( \ + DTRACE_ALLOC_STACK(16) \ + "ldp x0, x1, [%0, #64]" "\n\t" \ + "stp x0, x1, [sp]" "\n\t" \ + "ldp x6, x7, [%0, #48]" "\n\t" \ + "ldp x4, x5, [%0, #32]" "\n\t" \ + "ldp x2, x3, [%0, #16]" "\n\t" \ + "ldp x0, x1, [%0]" "\n\t" \ + DTRACE_CALL(provider, name) \ + DTRACE_DEALLOC_STACK(16) \ + : \ + : "r" (__dtrace_args) \ + : "memory", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7" \ + ); + +#endif /* __arm__ */ + +#endif /* _MACH_ARM_SDT_ISA_H */ diff --git a/osfmk/mach/arm/syscall_sw.h b/osfmk/mach/arm/syscall_sw.h new file mode 100644 index 000000000..af11c2668 --- /dev/null +++ b/osfmk/mach/arm/syscall_sw.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * FILE_ID: syscall_sw.h + */ + +#ifndef _MACH_ARM_SYSCALL_SW_H_ +#define _MACH_ARM_SYSCALL_SW_H_ + +#if defined(__arm__) + +#include +#include + +/* 0 to 4 args are already loaded in r0-r3 */ +#define _kernel_trap_0to4(trap_name, trap_number) \ + mov r12, # ## trap_number /* load syscall number */ ; \ + swi #SWI_SYSCALL ; \ + bx lr /* return */ ; + +#define _kernel_trap_5(trap_name, trap_number) \ + mov ip, sp /* save pointer to args */ ; \ + stmfd sp!, { r4-r5 } /* save r4-r5, keep stack 64-bit aligned */; \ + ldr r4, [ ip ] /* load arg 5 */ ; \ + mov r12, # ## trap_number /* load syscall number */ ; \ + swi #SWI_SYSCALL ; \ + ldmfd sp!, { r4-r5 } /* restore r4-r5 */ ;\ + bx lr /* return */ ; + +#define _kernel_trap_6to9(trap_name, trap_number, save_regs, arg_regs) \ + mov ip, sp /* save pointer to args */ ; \ + stmfd sp!, { save_regs } /* callee saved regs */; \ + ldmia ip, { arg_regs } /* load arg registers (above r0-r3) */ ;\ + mov r12, # ## trap_number /* load syscall number */ ; \ + swi #SWI_SYSCALL ; \ + ldmfd sp!, { save_regs } /* restore callee saved regs */ ;\ + bx lr /* return */ ; + +#define COMMA , + +/* For the armv7k ABI, the alignment requirements may add padding. So we + * let the kernel figure it out and push extra on the stack to avoid un-needed + * copy-ins. We are relying on arguments that aren't in registers starting + * 32 bytes from sp. */ +#if __BIGGEST_ALIGNMENT__ > 4 + +#define _kernel_trap_0(trap_name, trap_number) _kernel_trap_0to4(trap_name, trap_number) +#define _kernel_trap_1(trap_name, trap_number) _kernel_trap_0to4(trap_name, trap_number) +#define _kernel_trap_2(trap_name, trap_number) _kernel_trap_0to4(trap_name, trap_number) +#define _kernel_trap_3(trap_name, trap_number) _kernel_trap_0to4(trap_name, trap_number) +#define _kernel_trap_4(trap_name, trap_number) _kernel_trap_6to9(trap_name, trap_number, r4-r5, r4-r5) +#undef _kernel_trap_5 +#define _kernel_trap_5(trap_name, trap_number) _kernel_trap_6to9(trap_name, trap_number, r4-r5, r4-r5) +#define _kernel_trap_6(trap_name, trap_number) _kernel_trap_6to9(trap_name, trap_number, r4-r6 COMMA r8, r4-r6 COMMA r8) +#define _kernel_trap_7(trap_name, trap_number) _kernel_trap_6to9(trap_name, trap_number, r4-r6 COMMA r8, r4-r6 COMMA r8) +#define _kernel_trap_8(trap_name, trap_number) _kernel_trap_6to9(trap_name, trap_number, r4-r6 COMMA r8, r4-r6 COMMA r8) +#define _kernel_trap_9(trap_name, trap_number) _kernel_trap_6to9(trap_name, trap_number, r4-r6 COMMA r8, r4-r6 COMMA r8) +#else // !(__BIGGEST_ALIGNMENT__ > 4) + +#define _kernel_trap_0(trap_name, trap_number) _kernel_trap_0to4(trap_name, trap_number) +#define _kernel_trap_1(trap_name, trap_number) _kernel_trap_0to4(trap_name, trap_number) +#define _kernel_trap_2(trap_name, trap_number) _kernel_trap_0to4(trap_name, trap_number) +#define _kernel_trap_3(trap_name, trap_number) _kernel_trap_0to4(trap_name, trap_number) +#define _kernel_trap_4(trap_name, trap_number) _kernel_trap_0to4(trap_name, trap_number) +/* _kernel_trap_5 defined above */ +#define _kernel_trap_6(trap_name, trap_number) _kernel_trap_6to9(trap_name, trap_number, r4-r5, r4-r5) +/* need to save r8 not just for alignment but because mach_msg_trap overwrites the eighth argument */ +#define _kernel_trap_7(trap_name, trap_number) _kernel_trap_6to9(trap_name, trap_number, r4-r6 COMMA r8, r4-r6) +#define _kernel_trap_8(trap_name, trap_number) _kernel_trap_6to9(trap_name, trap_number, r4-r6 COMMA r8, r4-r6 COMMA r8) +/* there is only one nine-argument trap (mach_msg_overwrite_trap) and it doesn't use the ninth argument */ +#define _kernel_trap_9(trap_name, trap_number) _kernel_trap_8(trap_name, trap_number) + +#endif // __BIGGEST_ALIGNMENT__ > 4 + + +/* select the appropriate trap macro based off the number of args */ +#define kernel_trap(trap_name, trap_number, num_args) \ + LEAF(_##trap_name, 0) \ + _kernel_trap_##num_args(trap_name, trap_number) + +#elif defined(__arm64__) + +#include + +#define kernel_trap(trap_name, trap_number, num_args) \ +.globl _##trap_name %% \ +.text %% \ +.align 2 %% \ +_##trap_name: %% \ + mov x16, #(trap_number) %% \ + svc #SWI_SYSCALL %% \ + ret + +#else +#error Unsupported architecture +#endif + +#endif /* _MACH_ARM_SYSCALL_SW_H_ */ diff --git a/bsd/machine/spl.h b/osfmk/mach/arm/thread_state.h similarity index 58% rename from bsd/machine/spl.h rename to osfmk/mach/arm/thread_state.h index ea806c0ce..02d787317 100644 --- a/bsd/machine/spl.h +++ b/osfmk/mach/arm/thread_state.h @@ -2,7 +2,7 @@ * Copyright (c) 2000-2007 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,46 +22,23 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#ifndef _BSD_MACHINE_SPL_H_ -#define _BSD_MACHINE_SPL_H_ - -#ifdef KERNEL -#ifndef __ASSEMBLER__ - -#if !defined(__LP64__) || defined(XNU_KERNEL_PRIVATE) /* - * Machine-dependent SPL definitions. - * + * @OSF_COPYRIGHT@ */ -typedef unsigned spl_t; - -extern unsigned int sploff(void); -extern unsigned int splhigh(void); -extern unsigned int splsched(void); -extern unsigned int splclock(void); -extern unsigned int splpower(void); -extern unsigned int splvm(void); -extern unsigned int splbio(void); -extern unsigned int splimp(void); -extern unsigned int spltty(void); -extern unsigned int splnet(void); -extern unsigned int splsoftclock(void); - -extern void spllo(void); -extern void spl0(void); -extern void splon(unsigned int level); -extern void splx(unsigned int level); -extern void spln(unsigned int level); -#define splstatclock() splhigh() - -#endif /* !__LP64__ || XNU_KERNEL_PRIVATE */ -#endif /* __ASSEMBLER__ */ +#ifndef _MACH_ARM_THREAD_STATE_H_ +#define _MACH_ARM_THREAD_STATE_H_ -#endif /* KERNEL */ +/* Size of maximum exported thread state in words */ +#define ARM_THREAD_STATE_MAX (144) /* Size of biggest state possible */ +#if defined (__arm__) || defined(__arm64__) +#define THREAD_STATE_MAX ARM_THREAD_STATE_MAX +#else +#error Unsupported arch +#endif -#endif /* _BSD_MACHINE_SPL_H_ */ +#endif /* _MACH_ARM_THREAD_STATE_H_ */ diff --git a/osfmk/mach/arm/thread_status.h b/osfmk/mach/arm/thread_status.h new file mode 100644 index 000000000..7f4ac7d04 --- /dev/null +++ b/osfmk/mach/arm/thread_status.h @@ -0,0 +1,708 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * FILE_ID: thread_status.h + */ + + +#ifndef _ARM_THREAD_STATUS_H_ +#define _ARM_THREAD_STATUS_H_ + +#include +#include +#include + +/* + * Support for determining the state of a thread + */ + + +/* + * Flavors + */ + +#define ARM_THREAD_STATE 1 +#define ARM_UNIFIED_THREAD_STATE ARM_THREAD_STATE +#define ARM_VFP_STATE 2 +#define ARM_EXCEPTION_STATE 3 +#define ARM_DEBUG_STATE 4 /* pre-armv8 */ +#define THREAD_STATE_NONE 5 +#define ARM_THREAD_STATE64 6 +#define ARM_EXCEPTION_STATE64 7 +// ARM_THREAD_STATE_LAST (legacy) 8 +#define ARM_THREAD_STATE32 9 + +/* API */ +#define ARM_DEBUG_STATE32 14 +#define ARM_DEBUG_STATE64 15 +#define ARM_NEON_STATE 16 +#define ARM_NEON_STATE64 17 +#define ARM_CPMU_STATE64 18 + +#ifdef XNU_KERNEL_PRIVATE +/* For kernel use */ +#define ARM_SAVED_STATE32 20 +#define ARM_SAVED_STATE64 21 +#define ARM_NEON_SAVED_STATE32 22 +#define ARM_NEON_SAVED_STATE64 23 +#endif /* XNU_KERNEL_PRIVATE */ + +#define VALID_THREAD_STATE_FLAVOR(x)\ +((x == ARM_THREAD_STATE) || \ + (x == ARM_VFP_STATE) || \ + (x == ARM_EXCEPTION_STATE) || \ + (x == ARM_DEBUG_STATE) || \ + (x == THREAD_STATE_NONE) || \ + (x == ARM_THREAD_STATE32) || \ + (x == ARM_THREAD_STATE64) || \ + (x == ARM_EXCEPTION_STATE64) || \ + (x == ARM_NEON_STATE) || \ + (x == ARM_NEON_STATE64) || \ + (x == ARM_DEBUG_STATE32) || \ + (x == ARM_DEBUG_STATE64)) + +struct arm_state_hdr { + uint32_t flavor; + uint32_t count; +}; +typedef struct arm_state_hdr arm_state_hdr_t; + +typedef _STRUCT_ARM_THREAD_STATE arm_thread_state_t; +typedef _STRUCT_ARM_THREAD_STATE arm_thread_state32_t; +typedef _STRUCT_ARM_THREAD_STATE64 arm_thread_state64_t; + +struct arm_unified_thread_state { + arm_state_hdr_t ash; + union { + arm_thread_state32_t ts_32; + arm_thread_state64_t ts_64; + } uts; +}; +#define ts_32 uts.ts_32 +#define ts_64 uts.ts_64 +typedef struct arm_unified_thread_state arm_unified_thread_state_t; + +#define ARM_THREAD_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_thread_state_t)/sizeof(uint32_t))) +#define ARM_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_thread_state32_t)/sizeof(uint32_t))) +#define ARM_THREAD_STATE64_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_thread_state64_t)/sizeof(uint32_t))) +#define ARM_UNIFIED_THREAD_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_unified_thread_state_t)/sizeof(uint32_t))) + + +typedef _STRUCT_ARM_VFP_STATE arm_vfp_state_t; +typedef _STRUCT_ARM_NEON_STATE arm_neon_state_t; +typedef _STRUCT_ARM_NEON_STATE arm_neon_state32_t; +typedef _STRUCT_ARM_NEON_STATE64 arm_neon_state64_t; + +typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state_t; +typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state32_t; +typedef _STRUCT_ARM_EXCEPTION_STATE64 arm_exception_state64_t; + +typedef _STRUCT_ARM_DEBUG_STATE32 arm_debug_state32_t; +typedef _STRUCT_ARM_DEBUG_STATE64 arm_debug_state64_t; + +#if defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) +/* See below for ARM64 kernel structure definition for arm_debug_state. */ +#else +/* + * Otherwise not ARM64 kernel and we must preserve legacy ARM definitions of + * arm_debug_state for binary compatability of userland consumers of this file. + */ +#if defined(__arm__) +typedef _STRUCT_ARM_DEBUG_STATE arm_debug_state_t; +#elif defined(__arm64__) +typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_debug_state_t; +#else +#error Undefined architecture +#endif +#endif + +#define ARM_VFP_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_vfp_state_t)/sizeof(uint32_t))) + +#define ARM_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_exception_state_t)/sizeof(uint32_t))) + +#define ARM_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_exception_state64_t)/sizeof(uint32_t))) + +#define ARM_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_debug_state_t)/sizeof(uint32_t))) + +#define ARM_DEBUG_STATE32_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_debug_state32_t)/sizeof(uint32_t))) + +#define ARM_DEBUG_STATE64_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_debug_state64_t)/sizeof(uint32_t))) + +#define ARM_NEON_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_neon_state_t)/sizeof(uint32_t))) + +#define ARM_NEON_STATE64_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_neon_state64_t)/sizeof(uint32_t))) + +#define MACHINE_THREAD_STATE ARM_THREAD_STATE +#define MACHINE_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT + +/* + * Largest state on this machine: + */ +#define THREAD_MACHINE_STATE_MAX THREAD_STATE_MAX + +#ifdef XNU_KERNEL_PRIVATE + +static inline boolean_t +is_thread_state32(const arm_unified_thread_state_t *its) +{ + return (its->ash.flavor == ARM_THREAD_STATE32); +} + +static inline boolean_t +is_thread_state64(const arm_unified_thread_state_t *its) +{ + return (its->ash.flavor == ARM_THREAD_STATE64); +} + +static inline arm_thread_state32_t* +thread_state32(arm_unified_thread_state_t *its) +{ + return &its->ts_32; +} + +static inline arm_thread_state64_t* +thread_state64(arm_unified_thread_state_t *its) +{ + return &its->ts_64; +} + +static inline const arm_thread_state32_t* +const_thread_state32(const arm_unified_thread_state_t *its) +{ + return &its->ts_32; +} + +static inline const arm_thread_state64_t* +const_thread_state64(const arm_unified_thread_state_t *its) +{ + return &its->ts_64; +} + +#if defined(__arm__) +#include + +#define ARM_SAVED_STATE THREAD_STATE_NONE + 1 + +struct arm_saved_state { + uint32_t r[13]; /* General purpose register r0-r12 */ + uint32_t sp; /* Stack pointer r13 */ + uint32_t lr; /* Link register r14 */ + uint32_t pc; /* Program counter r15 */ + uint32_t cpsr; /* Current program status register */ + uint32_t fsr; /* Fault status */ + uint32_t far; /* Virtual Fault Address */ + uint32_t exception; /* exception number */ +}; +typedef struct arm_saved_state arm_saved_state_t; + +/* + * Just for coexistence with AArch64 code. + */ +typedef struct arm_saved_state arm_saved_state32_t; + +static inline arm_saved_state32_t* +saved_state32(arm_saved_state_t *iss) +{ + return iss; +} + +static inline boolean_t +is_saved_state32(const arm_saved_state_t *iss __unused) +{ + return TRUE; +} + + +struct arm_saved_state_tagged { + uint32_t tag; + struct arm_saved_state state; +}; +typedef struct arm_saved_state_tagged arm_saved_state_tagged_t; + +#define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_saved_state_t)/sizeof(unsigned int))) + + +static inline register_t +get_saved_state_pc(const arm_saved_state_t *iss) +{ + return iss->pc; +} + +static inline void +set_saved_state_pc(arm_saved_state_t *iss, register_t pc) +{ + iss->pc = pc; +} + +static inline register_t +get_saved_state_sp(const arm_saved_state_t *iss) +{ + return iss->sp; +} + +static inline void +set_saved_state_sp(arm_saved_state_t *iss, register_t sp) +{ + iss->sp = sp; +} + +static inline register_t +get_saved_state_fp(const arm_saved_state_t *iss) +{ + return iss->r[7]; +} + +static inline void +set_saved_state_fp(arm_saved_state_t *iss, register_t fp) +{ + iss->r[7] = fp; +} + +static inline register_t +get_saved_state_lr(const arm_saved_state_t *iss) +{ + return iss->lr; +} + +static inline void +set_saved_state_lr(arm_saved_state_t *iss, register_t lr) +{ + iss->lr = lr; +} + +static inline register_t +get_saved_state_cpsr(const arm_saved_state_t *iss) +{ + return iss->cpsr; +} + +static inline void +set_saved_state_cpsr(arm_saved_state_t *iss, register_t cpsr) +{ + iss->cpsr = cpsr; +} + +static inline register_t +get_saved_state_reg(const arm_saved_state_t *iss, unsigned regno) +{ + return iss->r[regno]; +} + +static inline void +set_saved_state_reg(arm_saved_state_t *iss, unsigned regno, register_t val) +{ + iss->r[regno] = val; +} + +#elif defined(__arm64__) + +#include +#include +#define CAST_ASSERT_SAFE(type, val) (assert((val) == ((type)(val))), (type)(val)) + +/* + * GPR context + */ + +struct arm_saved_state32 { + uint32_t r[13]; /* General purpose register r0-r12 */ + uint32_t sp; /* Stack pointer r13 */ + uint32_t lr; /* Link register r14 */ + uint32_t pc; /* Program counter r15 */ + uint32_t cpsr; /* Current program status register */ + uint32_t far; /* Virtual fault address */ + uint32_t esr; /* Exception syndrome register */ + uint32_t exception; /* Exception number */ +}; +typedef struct arm_saved_state32 arm_saved_state32_t; + +struct arm_saved_state32_tagged { + uint32_t tag; + struct arm_saved_state32 state; +}; +typedef struct arm_saved_state32_tagged arm_saved_state32_tagged_t; + +#define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_saved_state32_t)/sizeof(unsigned int))) + +struct arm_saved_state64 { + uint64_t x[29]; /* General purpose registers x0-x28 */ + uint64_t fp; /* Frame pointer x29 */ + uint64_t lr; /* Link register x30 */ + uint64_t sp; /* Stack pointer x31 */ + uint64_t pc; /* Program counter */ + uint32_t cpsr; /* Current program status register */ + uint32_t reserved; /* Reserved padding */ + uint64_t far; /* Virtual fault address */ + uint32_t esr; /* Exception syndrome register */ + uint32_t exception; /* Exception number */ +}; +typedef struct arm_saved_state64 arm_saved_state64_t; + +#define ARM_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_saved_state64_t)/sizeof(unsigned int))) + +struct arm_saved_state64_tagged { + uint32_t tag; + struct arm_saved_state64 state; +}; +typedef struct arm_saved_state64_tagged arm_saved_state64_tagged_t; + +struct arm_saved_state { + arm_state_hdr_t ash; + union { + struct arm_saved_state32 ss_32; + struct arm_saved_state64 ss_64; + } uss; +} __attribute__((aligned(16))); +#define ss_32 uss.ss_32 +#define ss_64 uss.ss_64 + +typedef struct arm_saved_state arm_saved_state_t; + + +static inline boolean_t +is_saved_state32(const arm_saved_state_t *iss) +{ + return (iss->ash.flavor == ARM_SAVED_STATE32); +} + +static inline boolean_t +is_saved_state64(const arm_saved_state_t *iss) +{ + return (iss->ash.flavor == ARM_SAVED_STATE64); +} + +static inline arm_saved_state32_t* +saved_state32(arm_saved_state_t *iss) +{ + return &iss->ss_32; +} + +static inline const arm_saved_state32_t* +const_saved_state32(const arm_saved_state_t *iss) +{ + return &iss->ss_32; +} + +static inline arm_saved_state64_t* +saved_state64(arm_saved_state_t *iss) +{ + return &iss->ss_64; +} + +static inline const arm_saved_state64_t* +const_saved_state64(const arm_saved_state_t *iss) +{ + return &iss->ss_64; +} + +static inline register_t +get_saved_state_pc(const arm_saved_state_t *iss) +{ + return (is_saved_state32(iss) ? const_saved_state32(iss)->pc : const_saved_state64(iss)->pc); +} + +static inline void +set_saved_state_pc(arm_saved_state_t *iss, register_t pc) +{ + if (is_saved_state32(iss)) { + saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc); + } else { + saved_state64(iss)->pc = pc; + } +} + +static inline register_t +get_saved_state_sp(const arm_saved_state_t *iss) +{ + return (is_saved_state32(iss) ? const_saved_state32(iss)->sp : const_saved_state64(iss)->sp); +} + +static inline void +set_saved_state_sp(arm_saved_state_t *iss, register_t sp) +{ + if (is_saved_state32(iss)) { + saved_state32(iss)->sp = CAST_ASSERT_SAFE(uint32_t, sp); + } else { + saved_state64(iss)->sp = sp; + } +} + +static inline register_t +get_saved_state_lr(const arm_saved_state_t *iss) +{ + return (is_saved_state32(iss) ? const_saved_state32(iss)->lr : const_saved_state64(iss)->lr); +} + +static inline void +set_saved_state_lr(arm_saved_state_t *iss, register_t lr) +{ + if (is_saved_state32(iss)) { + saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr); + } else { + saved_state64(iss)->lr = lr; + } +} + +static inline register_t +get_saved_state_fp(const arm_saved_state_t *iss) +{ + return (is_saved_state32(iss) ? const_saved_state32(iss)->r[7] : const_saved_state64(iss)->fp); +} + +static inline void +set_saved_state_fp(arm_saved_state_t *iss, register_t fp) +{ + if (is_saved_state32(iss)) { + saved_state32(iss)->r[7] = CAST_ASSERT_SAFE(uint32_t, fp); + } else { + saved_state64(iss)->fp = fp; + } +} + +static inline int +check_saved_state_reglimit(const arm_saved_state_t *iss, unsigned reg) +{ + return (is_saved_state32(iss) ? (reg < ARM_SAVED_STATE32_COUNT) : (reg < ARM_SAVED_STATE64_COUNT)); +} + +static inline register_t +get_saved_state_reg(const arm_saved_state_t *iss, unsigned reg) +{ + if (!check_saved_state_reglimit(iss, reg)) return 0; + + return (is_saved_state32(iss) ? (const_saved_state32(iss)->r[reg]) : (const_saved_state64(iss)->x[reg])); +} + +static inline void +set_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value) +{ + if (!check_saved_state_reglimit(iss, reg)) return; + + if (is_saved_state32(iss)) { + saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value); + } else { + saved_state64(iss)->x[reg] = value; + } +} + +static inline uint32_t +get_saved_state_cpsr(const arm_saved_state_t *iss) +{ + return (is_saved_state32(iss) ? const_saved_state32(iss)->cpsr : const_saved_state64(iss)->cpsr); +} + +static inline void +set_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr) +{ + if (is_saved_state32(iss)) { + saved_state32(iss)->cpsr = cpsr; + } else { + saved_state64(iss)->cpsr = cpsr; + } +} + +static inline register_t +get_saved_state_far(const arm_saved_state_t *iss) +{ + return (is_saved_state32(iss) ? const_saved_state32(iss)->far : const_saved_state64(iss)->far); +} + +static inline void +set_saved_state_far(arm_saved_state_t *iss, register_t far) +{ + if (is_saved_state32(iss)) { + saved_state32(iss)->far = CAST_ASSERT_SAFE(uint32_t, far); + } else { + saved_state64(iss)->far = far; + } +} + +static inline uint32_t +get_saved_state_esr(const arm_saved_state_t *iss) +{ + return (is_saved_state32(iss) ? const_saved_state32(iss)->esr : const_saved_state64(iss)->esr); +} + +static inline void +set_saved_state_esr(arm_saved_state_t *iss, uint32_t esr) +{ + if (is_saved_state32(iss)) { + saved_state32(iss)->esr = esr; + } else { + saved_state64(iss)->esr = esr; + } +} + +static inline uint32_t +get_saved_state_exc(const arm_saved_state_t *iss) +{ + return (is_saved_state32(iss) ? const_saved_state32(iss)->exception : const_saved_state64(iss)->exception); +} + +static inline void +set_saved_state_exc(arm_saved_state_t *iss, uint32_t exc) +{ + if (is_saved_state32(iss)) { + saved_state32(iss)->exception = exc; + } else { + saved_state64(iss)->exception = exc; + } +} + +extern void panic_unimplemented(void); + +static inline int +get_saved_state_svc_number(const arm_saved_state_t *iss) +{ + return (is_saved_state32(iss) ? (int)const_saved_state32(iss)->r[12] : (int)const_saved_state64(iss)->x[ARM64_SYSCALL_CODE_REG_NUM]); /* Only first word counts here */ +} + +typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_legacy_debug_state_t; + +struct arm_debug_aggregate_state { + arm_state_hdr_t dsh; + union { + arm_debug_state32_t ds32; + arm_debug_state64_t ds64; + } uds; +} __attribute__((aligned(16))); + +typedef struct arm_debug_aggregate_state arm_debug_state_t; + +#define ARM_LEGACY_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_legacy_debug_state_t)/sizeof(uint32_t))) + +/* + * NEON context + */ +typedef __uint128_t uint128_t; +typedef uint64_t uint64x2_t __attribute__((ext_vector_type(2))); +typedef uint32_t uint32x4_t __attribute__((ext_vector_type(4))); + +struct arm_neon_saved_state32 { + union { + uint128_t q[16]; + uint64_t d[32]; + uint32_t s[32]; + } v; + uint32_t fpsr; + uint32_t fpcr; +}; +typedef struct arm_neon_saved_state32 arm_neon_saved_state32_t; + +#define ARM_NEON_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_neon_saved_state32_t)/sizeof(unsigned int))) + +struct arm_neon_saved_state64 { + union { + uint128_t q[32]; + uint64x2_t d[32]; + uint32x4_t s[32]; + } v; + uint32_t fpsr; + uint32_t fpcr; +}; +typedef struct arm_neon_saved_state64 arm_neon_saved_state64_t; + +#define ARM_NEON_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_neon_saved_state64_t)/sizeof(unsigned int))) + +struct arm_neon_saved_state { + arm_state_hdr_t nsh; + union { + struct arm_neon_saved_state32 ns_32; + struct arm_neon_saved_state64 ns_64; + } uns; +}; +typedef struct arm_neon_saved_state arm_neon_saved_state_t; +#define ns_32 uns.ns_32 +#define ns_64 uns.ns_64 + +static inline boolean_t +is_neon_saved_state32(const arm_neon_saved_state_t *state) +{ + return (state->nsh.flavor == ARM_NEON_SAVED_STATE32); +} + +static inline boolean_t +is_neon_saved_state64(const arm_neon_saved_state_t *state) +{ + return (state->nsh.flavor == ARM_NEON_SAVED_STATE64); +} + +static inline arm_neon_saved_state32_t * +neon_state32(arm_neon_saved_state_t *state) +{ + return &state->ns_32; +} + +static inline arm_neon_saved_state64_t * +neon_state64(arm_neon_saved_state_t *state) +{ + return &state->ns_64; +} + + +/* + * Aggregated context + */ + +struct arm_context { + struct arm_saved_state ss; + struct arm_neon_saved_state ns; +}; +typedef struct arm_context arm_context_t; + +extern void saved_state_to_thread_state64(const arm_saved_state_t*, arm_thread_state64_t*); +extern void thread_state64_to_saved_state(const arm_thread_state64_t*, arm_saved_state_t*); + +#else +#error Unknown arch +#endif + +extern void saved_state_to_thread_state32(const arm_saved_state_t*, arm_thread_state32_t*); +extern void thread_state32_to_saved_state(const arm_thread_state32_t*, arm_saved_state_t*); + +#endif /* XNU_KERNEL_PRIVATE */ + +#endif /* _ARM_THREAD_STATUS_H_ */ diff --git a/osfmk/mach/arm/vm_param.h b/osfmk/mach/arm/vm_param.h new file mode 100644 index 000000000..715ee1d31 --- /dev/null +++ b/osfmk/mach/arm/vm_param.h @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * FILE_ID: vm_param.h + */ + +/* + * ARM machine dependent virtual memory parameters. + */ + +#ifndef _MACH_ARM_VM_PARAM_H_ +#define _MACH_ARM_VM_PARAM_H_ + +#if defined(KERNEL_PRIVATE) && __ARM_16K_PG__ +#include +#endif + +#if !defined (KERNEL) && !defined (__ASSEMBLER__) +#include +#endif + +#define BYTE_SIZE 8 /* byte size in bits */ + +#if defined (KERNEL) + +#ifndef __ASSEMBLER__ + +#ifdef __arm__ +#define PAGE_SHIFT_CONST 12 +#elif defined(__arm64__) +extern unsigned PAGE_SHIFT_CONST; +#else +#error Unsupported arch +#endif + +#if defined(KERNEL_PRIVATE) && __ARM_16K_PG__ +#define PAGE_SHIFT ARM_PGSHIFT +#else +#define PAGE_SHIFT PAGE_SHIFT_CONST +#endif +#define PAGE_SIZE (1 << PAGE_SHIFT) +#define PAGE_MASK (PAGE_SIZE-1) + +#define VM_PAGE_SIZE PAGE_SIZE + +#define machine_ptob(x) ((x) << PAGE_SHIFT) + +/* + * Defined for the purpose of testing the pmap advertised page + * size; this does not necessarily match the hardware page size. + */ +#define TEST_PAGE_SIZE_16K ((PAGE_SHIFT_CONST == 14)) +#define TEST_PAGE_SIZE_4K ((PAGE_SHIFT_CONST == 12)) + +#endif /* !__ASSEMBLER__ */ + +#else + +#define PAGE_SHIFT vm_page_shift +#define PAGE_SIZE vm_page_size +#define PAGE_MASK vm_page_mask + +#define VM_PAGE_SIZE vm_page_size + +#define machine_ptob(x) ((x) << PAGE_SHIFT) + +#endif + +#define PAGE_MAX_SHIFT 14 +#define PAGE_MAX_SIZE (1 << PAGE_MAX_SHIFT) +#define PAGE_MAX_MASK (PAGE_MAX_SIZE-1) + +#define PAGE_MIN_SHIFT 12 +#define PAGE_MIN_SIZE (1 << PAGE_MIN_SHIFT) +#define PAGE_MIN_MASK (PAGE_MIN_SIZE-1) + +#ifndef __ASSEMBLER__ + +#ifdef MACH_KERNEL_PRIVATE + +#define VM32_SUPPORT 1 +#define VM32_MIN_ADDRESS ((vm32_offset_t) 0) +#define VM32_MAX_ADDRESS ((vm32_offset_t) (VM_MAX_PAGE_ADDRESS & 0xFFFFFFFF)) +#define VM_MAX_PAGE_ADDRESS VM_MAX_ADDRESS /* ARM64_TODO: ?? */ + +/* + * kalloc() parameters: + * + * Historically kalloc's underlying zones were power-of-2 sizes, with a + * KALLOC_MINSIZE of 16 bytes. Thus the allocator ensured that + * (sizeof == alignof) >= 16 for all kalloc allocations. + * + * Today kalloc may use zones with intermediate (small) sizes, constrained by + * KALLOC_MINSIZE and a minimum alignment, expressed by KALLOC_LOG2_MINALIGN. + * + * Note that most dynamically allocated data structures contain more than + * one int/long/pointer member, so KALLOC_MINSIZE should probably start at 8. + */ + +#if defined (__arm__) + +#define KALLOC_MINSIZE 8 /* minimum allocation size */ +#define KALLOC_LOG2_MINALIGN 3 /* log2 minimum alignment */ + +#elif defined(__arm64__) + +#define KALLOC_MINSIZE 16 /* minimum allocation size */ +#define KALLOC_LOG2_MINALIGN 4 /* log2 minimum alignment */ + +#else +#error Unsupported arch +#endif + +#endif + +#if defined (__arm__) + +#define VM_MIN_ADDRESS ((vm_address_t) 0x00000000) +#define VM_MAX_ADDRESS ((vm_address_t) 0x80000000) + +/* system-wide values */ +#define MACH_VM_MIN_ADDRESS ((mach_vm_offset_t) 0) +#define MACH_VM_MAX_ADDRESS ((mach_vm_offset_t) VM_MAX_ADDRESS) + +#elif defined (__arm64__) + +#define VM_MIN_ADDRESS ((vm_address_t) 0x0000000000000000ULL) +#define VM_MAX_ADDRESS ((vm_address_t) 0x0000000080000000ULL) + +/* system-wide values */ +#define MACH_VM_MIN_ADDRESS ((mach_vm_offset_t) 0x0ULL) +#define MACH_VM_MAX_ADDRESS ((mach_vm_offset_t) 0x0000001000000000ULL) + +#else +#error architecture not supported +#endif + +#define VM_MAP_MIN_ADDRESS VM_MIN_ADDRESS +#define VM_MAP_MAX_ADDRESS VM_MAX_ADDRESS + +#ifdef KERNEL + +#if defined (__arm__) +#define VM_MIN_KERNEL_ADDRESS ((vm_address_t) 0x80000000) +#define VM_MAX_KERNEL_ADDRESS ((vm_address_t) 0xFFFEFFFF) +#define VM_HIGH_KERNEL_WINDOW ((vm_address_t) 0xFFFE0000) +#elif defined (__arm64__) +/* + * The minimum and maximum kernel address; some configurations may + * constrain the address space further. + */ +#define VM_MIN_KERNEL_ADDRESS ((vm_address_t) 0xffffffe000000000ULL) +#define VM_MAX_KERNEL_ADDRESS ((vm_address_t) 0xfffffff3ffffffffULL) +#else +#error architecture not supported +#endif + +#define VM_MIN_KERNEL_AND_KEXT_ADDRESS \ + VM_MIN_KERNEL_ADDRESS + +#define VM_KERNEL_ADDRESS(va) ((((vm_address_t)(va))>=VM_MIN_KERNEL_ADDRESS) && \ + (((vm_address_t)(va))<=VM_MAX_KERNEL_ADDRESS)) + +#ifdef MACH_KERNEL_PRIVATE +/* + * Physical memory is mapped linearly at an offset virtual memory. + */ +extern unsigned long gVirtBase, gPhysBase, gPhysSize; + +#define isphysmem(a) (((vm_address_t)(a) - gPhysBase) < gPhysSize) +#define phystokv(a) ((vm_address_t)(a) - gPhysBase + gVirtBase) + +#if KASAN +/* Increase the stack sizes to account for the redzones that get added to every + * stack object. */ +# define KERNEL_STACK_SIZE (4*4*4096) +# define INTSTACK_SIZE (4*4*4096) +#else +# define KERNEL_STACK_SIZE (4*4096) +# define INTSTACK_SIZE (4*4096) +#endif + +#if defined (__arm__) +#define HIGH_EXC_VECTORS ((vm_address_t) 0xFFFF0000) +#endif + +#endif /* MACH_KERNEL_PRIVATE */ +#endif /* KERNEL */ + +#endif /* !__ASSEMBLER__ */ + +#define SWI_SYSCALL 0x80 + +#endif /* _MACH_ARM_VM_PARAM_H_ */ diff --git a/osfmk/mach/arm/vm_types.h b/osfmk/mach/arm/vm_types.h new file mode 100644 index 000000000..d15ce8c1b --- /dev/null +++ b/osfmk/mach/arm/vm_types.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +/* + * File: vm_types.h + * Author: Avadis Tevanian, Jr. + * Date: 1985 + * + * Header file for VM data types. ARM version. + */ + +#ifndef _MACH_ARM_VM_TYPES_H_ +#define _MACH_ARM_VM_TYPES_H_ + +#ifndef ASSEMBLER + +#include +#include +#include + +/* + * natural_t and integer_t are Mach's legacy types for machine- + * independent integer types (unsigned, and signed, respectively). + * Their original purpose was to define other types in a machine/ + * compiler independent way. + * + * They also had an implicit "same size as pointer" characteristic + * to them (i.e. Mach's traditional types are very ILP32 or ILP64 + * centric). We will likely support x86 ABIs that do not follow + * either ofthese models (specifically LP64). Therefore, we had to + * make a choice between making these types scale with pointers or stay + * tied to integers. Because their use is predominantly tied to + * to the size of an integer, we are keeping that association and + * breaking free from pointer size guarantees. + * + * New use of these types is discouraged. + */ +typedef __darwin_natural_t natural_t; +typedef int integer_t; + +/* + * A vm_offset_t is a type-neutral pointer, + * e.g. an offset into a virtual memory space. + */ +#ifdef __LP64__ +typedef uintptr_t vm_offset_t; +typedef uintptr_t vm_size_t; + +typedef uint64_t mach_vm_address_t; +typedef uint64_t mach_vm_offset_t; +typedef uint64_t mach_vm_size_t; + +typedef uint64_t vm_map_offset_t; +typedef uint64_t vm_map_address_t; +typedef uint64_t vm_map_size_t; +#else +typedef natural_t vm_offset_t; +/* + * A vm_size_t is the proper type for e.g. + * expressing the difference between two + * vm_offset_t entities. + */ +typedef natural_t vm_size_t; + +/* + * This new type is independent of a particular vm map's + * implementation size - and represents appropriate types + * for all possible maps. This is used for interfaces + * where the size of the map is not known - or we don't + * want to have to distinguish. + */ +#if defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && (__IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_5_0) +typedef uint32_t mach_vm_address_t; +typedef uint32_t mach_vm_offset_t; +typedef uint32_t mach_vm_size_t; +#else +typedef uint64_t mach_vm_address_t; +typedef uint64_t mach_vm_offset_t; +typedef uint64_t mach_vm_size_t; +#endif + +typedef uint32_t vm_map_offset_t; +typedef uint32_t vm_map_address_t; +typedef uint32_t vm_map_size_t; +#endif /* __LP64__ */ + + +typedef uint32_t vm32_offset_t; +typedef uint32_t vm32_address_t; +typedef uint32_t vm32_size_t; + +typedef vm_offset_t mach_port_context_t; + +#ifdef MACH_KERNEL_PRIVATE +typedef vm32_offset_t mach_port_context32_t; +typedef mach_vm_offset_t mach_port_context64_t; +#endif + +#endif /* ASSEMBLER */ + +/* + * If composing messages by hand (please do not) + */ +#define MACH_MSG_TYPE_INTEGER_T MACH_MSG_TYPE_INTEGER_32 + +#endif /* _MACH_ARM_VM_TYPES_H_ */ diff --git a/osfmk/mach/arm64/Makefile b/osfmk/mach/arm64/Makefile new file mode 100644 index 000000000..ff5153443 --- /dev/null +++ b/osfmk/mach/arm64/Makefile @@ -0,0 +1,28 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +VPATH := $(VPATH):$(SOURCE)/../../arm64 + +DATAFILES = + +INSTALL_MD_LIST = ${DATAFILES} + +INSTALL_MD_GEN_LIST = \ + asm.h + +INSTALL_MD_DIR = mach/arm64 + +EXPORT_MD_LIST = ${DATAFILES} + +EXPORT_MD_GEN_LIST = \ + asm.h + +EXPORT_MD_DIR = mach/arm64 + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/osfmk/mach/coalition.h b/osfmk/mach/coalition.h index e548852dd..40824547e 100644 --- a/osfmk/mach/coalition.h +++ b/osfmk/mach/coalition.h @@ -31,7 +31,7 @@ /* code shared by userspace and xnu */ -#define COALITION_CREATE_FLAGS_MASK ((uint32_t)0xF1) +#define COALITION_CREATE_FLAGS_MASK ((uint32_t)0xFF1) #define COALITION_CREATE_FLAGS_PRIVILEGED ((uint32_t)0x01) #define COALITION_CREATE_FLAGS_TYPE_MASK ((uint32_t)0xF0) @@ -47,6 +47,28 @@ & COALITION_CREATE_FLAGS_TYPE_MASK); \ } while (0) +#define COALITION_CREATE_FLAGS_ROLE_MASK ((uint32_t)0xF00) +#define COALITION_CREATE_FLAGS_ROLE_SHIFT (8) + +#define COALITION_CREATE_FLAGS_GET_ROLE(flags) \ + (((flags) & COALITION_CREATE_FLAGS_ROLE_MASK) >> COALITION_CREATE_FLAGS_ROLE_SHIFT) + +#define COALITION_CREATE_FLAGS_SET_ROLE(flags, role) \ + do { \ + flags &= ~COALITION_CREATE_FLAGS_ROLE_MASK; \ + flags |= (((role) << COALITION_CREATE_FLAGS_ROLE_SHIFT) \ + & COALITION_CREATE_FLAGS_ROLE_MASK); \ + } while (0) + +/* + * Default scheduling policy of the lead/parent task in a coalition + */ +#define COALITION_ROLE_UNDEF (0) +#define COALITION_ROLE_SYSTEM (1) +#define COALITION_ROLE_BACKGROUND (2) +#define COALITION_ROLE_ADAPTIVE (3) +#define COALITION_ROLE_INTERACTIVE (4) +#define COALITION_NUM_ROLES (5) #define COALITION_TYPE_RESOURCE (0) #define COALITION_TYPE_JETSAM (1) @@ -54,6 +76,7 @@ #define COALITION_NUM_TYPES (COALITION_TYPE_MAX + 1) +#define COALITION_TASKROLE_NONE (-1) /* task plays no role in the given coalition */ #define COALITION_TASKROLE_UNDEF (0) #define COALITION_TASKROLE_LEADER (1) #define COALITION_TASKROLE_XPC (2) @@ -76,6 +99,26 @@ #define COALITION_NUM_SORT (6) +/* Coalition Efficiency Interface Support */ + +/* Flags for coalition efficiency */ +#define COALITION_FLAGS_EFFICIENT (0x1) + +/* + * Mapping of launchd plist values to coalition efficiency flags. + * Launchd uses this mapping to pass the correct flags to + * coalition_info_set_efficiency(cid, flags); + * + * Current supported values mapping: + * { "Efficient" : COALITION_FLAGS_EFFICIENT, } + */ +static const char *coalition_efficiency_names[] = { + "Efficient", +}; +static const uint64_t coalition_efficiency_flags[] = { + COALITION_FLAGS_EFFICIENT, +}; + struct coalition_resource_usage { uint64_t tasks_started; uint64_t tasks_exited; @@ -93,6 +136,9 @@ struct coalition_resource_usage { uint64_t logical_deferred_writes; uint64_t logical_invalidated_writes; uint64_t logical_metadata_writes; + uint64_t energy_billed_to_me; + uint64_t energy_billed_to_others; + uint64_t cpu_ptime; }; #ifdef PRIVATE @@ -105,6 +151,10 @@ struct coalition_resource_usage { /* coalition_info flavors */ #define COALITION_INFO_RESOURCE_USAGE 1 +#define COALITION_INFO_SET_NAME 2 +#define COALITION_INFO_SET_EFFICIENCY 3 + +#define COALITION_EFFICIENCY_VALID_FLAGS (COALITION_FLAGS_EFFICIENT) /* structure returned from libproc coalition listing interface */ struct procinfo_coalinfo { diff --git a/osfmk/mach/error.h b/osfmk/mach/error.h index a32eaee53..c0ceeebea 100644 --- a/osfmk/mach/error.h +++ b/osfmk/mach/error.h @@ -80,7 +80,7 @@ #define ERR_ROUTINE_NIL (mach_error_fn_t)0 -#define err_system(x) (((x)&0x3f)<<26) +#define err_system(x) ((signed)((((unsigned)(x))&0x3f)<<26)) #define err_sub(x) (((x)&0xfff)<<14) #define err_get_system(err) (((err)>>26)&0x3f) diff --git a/osfmk/mach/exc.defs b/osfmk/mach/exc.defs index de6aee7b9..c412d3192 100644 --- a/osfmk/mach/exc.defs +++ b/osfmk/mach/exc.defs @@ -75,25 +75,15 @@ type exception_data_t = array[*:2] of integer_t; type exception_type_t = int; routine exception_raise( -#if KERNEL_USER - exception_port : mach_port_move_send_t; - thread : mach_port_move_send_t; - task : mach_port_move_send_t; -#else /* KERNEL_USER */ exception_port : mach_port_t; thread : mach_port_t; task : mach_port_t; -#endif /* KERNEL_USER */ exception : exception_type_t; code : exception_data_t ); routine exception_raise_state( -#if KERNEL_USER - exception_port : mach_port_move_send_t; -#else /* KERNEL_USER */ exception_port : mach_port_t; -#endif /* KERNEL_USER */ exception : exception_type_t; code : exception_data_t, const; inout flavor : int; @@ -101,15 +91,9 @@ routine exception_raise_state( out new_state : thread_state_t); routine exception_raise_state_identity( -#if KERNEL_USER - exception_port : mach_port_move_send_t; - thread : mach_port_move_send_t; - task : mach_port_move_send_t; -#else /* KERNEL_USER */ exception_port : mach_port_t; thread : mach_port_t; task : mach_port_t; -#endif /* KERNEL_USER */ exception : exception_type_t; code : exception_data_t; inout flavor : int; diff --git a/osfmk/mach/exception_types.h b/osfmk/mach/exception_types.h index 0b56db3a0..6fc534c5b 100644 --- a/osfmk/mach/exception_types.h +++ b/osfmk/mach/exception_types.h @@ -92,7 +92,7 @@ #define EXC_MACH_SYSCALL 8 /* Mach system calls. */ #define EXC_RPC_ALERT 9 /* RPC alert */ - + #define EXC_CRASH 10 /* Abnormal process exit */ #define EXC_RESOURCE 11 /* Hit resource consumption limit */ @@ -100,7 +100,9 @@ #define EXC_GUARD 12 /* Violated guarded resource protections */ -#define EXC_CORPSE_NOTIFY 13 /* Abnormal process exited to corpse state */ +#define EXC_CORPSE_NOTIFY 13 /* Abnormal process exited to corpse state */ + +#define EXC_CORPSE_VARIANT_BIT 0x100 /* bit set for EXC_*_CORPSE variants of EXC_* */ /* diff --git a/osfmk/mach/host_priv.defs b/osfmk/mach/host_priv.defs index ac4997b22..83f601b5a 100644 --- a/osfmk/mach/host_priv.defs +++ b/osfmk/mach/host_priv.defs @@ -72,6 +72,13 @@ subsystem userprefix r_; #endif /* KERNEL_USER */ +#define CONCAT(a,b) a ## b +#if KERNEL_SERVER +#define KERNEL_SERVER_SUFFIX(NAME) CONCAT(NAME, _external) +#else +#define KERNEL_SERVER_SUFFIX(NAME) NAME +#endif + #include #include #include @@ -274,7 +281,7 @@ skip; /* old host_load_symbol_table */ * * [ To unwire the pages, specify VM_PROT_NONE. ] */ -routine mach_vm_wire( +routine KERNEL_SERVER_SUFFIX(mach_vm_wire)( host_priv : host_priv_t; task : vm_map_t; address : mach_vm_address_t; @@ -304,26 +311,9 @@ routine host_processor_set_priv( /* The following routines are going away in a future release */ /* use the appropriate variant of host_set_special_port instead */ /************************************************************************/ - -/* - * Set the dynamic_pager control port. Other entities - * can request a send right to this port to talk with - * the dynamic_pager utility, setting behavioral parameters - * within the dynamic pager and getting low/high backing store - * resource notifications. - */ -routine set_dp_control_port( - host : host_priv_t; - in control_port : mach_port_t); -/* - * Get the dynamic_pager control port. This port - * allows the holder to talk directly with the dynamic - * pager utility. - */ -routine get_dp_control_port( - host : host_priv_t; - out contorl_port :mach_port_t); +skip;/* old set_dp_control_port */ +skip;/* old get_dp_control_port */ /* * Set the UserNotification daemon access port for this host. diff --git a/osfmk/mach/host_special_ports.h b/osfmk/mach/host_special_ports.h index 52a4a7449..73efdc4f4 100644 --- a/osfmk/mach/host_special_ports.h +++ b/osfmk/mach/host_special_ports.h @@ -100,8 +100,9 @@ #define HOST_CONTAINERD_PORT (18 + HOST_MAX_SPECIAL_KERNEL_PORT) #define HOST_NODE_PORT (19 + HOST_MAX_SPECIAL_KERNEL_PORT) #define HOST_RESOURCE_NOTIFY_PORT (20 + HOST_MAX_SPECIAL_KERNEL_PORT) +#define HOST_CLOSURED_PORT (21 + HOST_MAX_SPECIAL_KERNEL_PORT) -#define HOST_MAX_SPECIAL_PORT HOST_RESOURCE_NOTIFY_PORT +#define HOST_MAX_SPECIAL_PORT HOST_CLOSURED_PORT /* MAX = last since rdar://19421223 */ /* @@ -236,6 +237,12 @@ #define host_set_node_port(host, port) \ (host_set_special_port((host), HOST_NODE_PORT, (port))) +#define host_get_closured_port(host, port) \ + (host_get_special_port((host), \ + HOST_LOCAL_NODE, HOST_CLOSURED_PORT, (port))) +#define host_set_closured_port(host, port) \ + (host_set_special_port((host), HOST_CLOSURED_PORT, (port))) + /* HOST_RESOURCE_NOTIFY_PORT doesn't #defines these conveniences. All lookups go through send_resource_violation() */ diff --git a/osfmk/mach/i386/_structs.h b/osfmk/mach/i386/_structs.h index 9dd3f4416..0f687777c 100644 --- a/osfmk/mach/i386/_structs.h +++ b/osfmk/mach/i386/_structs.h @@ -32,6 +32,9 @@ #ifndef _MACH_I386__STRUCTS_H_ #define _MACH_I386__STRUCTS_H_ +#include /* __DARWIN_UNIX03 */ +#include /* __uint8_t */ + /* * i386 is the structure that is exported to user threads for * use in status/mutate calls. This structure should never change. @@ -219,6 +222,54 @@ _STRUCT_XMM_REG }; #endif /* !__DARWIN_UNIX03 */ +#if !defined(RC_HIDE_XNU_J137) +/* defn of 256 bit YMM regs */ + +#if __DARWIN_UNIX03 +#define _STRUCT_YMM_REG struct __darwin_ymm_reg +_STRUCT_YMM_REG +{ + char __ymm_reg[32]; +}; +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_YMM_REG struct ymm_reg +_STRUCT_YMM_REG +{ + char ymm_reg[32]; +}; +#endif /* !__DARWIN_UNIX03 */ + +/* defn of 512 bit ZMM regs */ + +#if __DARWIN_UNIX03 +#define _STRUCT_ZMM_REG struct __darwin_zmm_reg +_STRUCT_ZMM_REG +{ + char __zmm_reg[64]; +}; +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_ZMM_REG struct zmm_reg +_STRUCT_ZMM_REG +{ + char zmm_reg[64]; +}; +#endif /* !__DARWIN_UNIX03 */ + +#if __DARWIN_UNIX03 +#define _STRUCT_OPMASK_REG struct __darwin_opmask_reg +_STRUCT_OPMASK_REG +{ + char __opmask_reg[8]; +}; +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_OPMASK_REG struct opmask_reg +_STRUCT_OPMASK_REG +{ + char opmask_reg[8]; +}; +#endif /* !__DARWIN_UNIX03 */ +#endif /* not RC_HIDE_XNU_J137 */ + /* * Floating point state. */ @@ -311,6 +362,70 @@ _STRUCT_X86_AVX_STATE32 _STRUCT_XMM_REG __fpu_ymmh7; /* YMMH 7 */ }; +#if !defined(RC_HIDE_XNU_J137) +#define _STRUCT_X86_AVX512_STATE32 struct __darwin_i386_avx512_state +_STRUCT_X86_AVX512_STATE32 +{ + int __fpu_reserved[2]; + _STRUCT_FP_CONTROL __fpu_fcw; /* x87 FPU control word */ + _STRUCT_FP_STATUS __fpu_fsw; /* x87 FPU status word */ + __uint8_t __fpu_ftw; /* x87 FPU tag word */ + __uint8_t __fpu_rsrv1; /* reserved */ + __uint16_t __fpu_fop; /* x87 FPU Opcode */ + __uint32_t __fpu_ip; /* x87 FPU Instruction Pointer offset */ + __uint16_t __fpu_cs; /* x87 FPU Instruction Pointer Selector */ + __uint16_t __fpu_rsrv2; /* reserved */ + __uint32_t __fpu_dp; /* x87 FPU Instruction Operand(Data) Pointer offset */ + __uint16_t __fpu_ds; /* x87 FPU Instruction Operand(Data) Pointer Selector */ + __uint16_t __fpu_rsrv3; /* reserved */ + __uint32_t __fpu_mxcsr; /* MXCSR Register state */ + __uint32_t __fpu_mxcsrmask; /* MXCSR mask */ + _STRUCT_MMST_REG __fpu_stmm0; /* ST0/MM0 */ + _STRUCT_MMST_REG __fpu_stmm1; /* ST1/MM1 */ + _STRUCT_MMST_REG __fpu_stmm2; /* ST2/MM2 */ + _STRUCT_MMST_REG __fpu_stmm3; /* ST3/MM3 */ + _STRUCT_MMST_REG __fpu_stmm4; /* ST4/MM4 */ + _STRUCT_MMST_REG __fpu_stmm5; /* ST5/MM5 */ + _STRUCT_MMST_REG __fpu_stmm6; /* ST6/MM6 */ + _STRUCT_MMST_REG __fpu_stmm7; /* ST7/MM7 */ + _STRUCT_XMM_REG __fpu_xmm0; /* XMM 0 */ + _STRUCT_XMM_REG __fpu_xmm1; /* XMM 1 */ + _STRUCT_XMM_REG __fpu_xmm2; /* XMM 2 */ + _STRUCT_XMM_REG __fpu_xmm3; /* XMM 3 */ + _STRUCT_XMM_REG __fpu_xmm4; /* XMM 4 */ + _STRUCT_XMM_REG __fpu_xmm5; /* XMM 5 */ + _STRUCT_XMM_REG __fpu_xmm6; /* XMM 6 */ + _STRUCT_XMM_REG __fpu_xmm7; /* XMM 7 */ + char __fpu_rsrv4[14*16]; /* reserved */ + int __fpu_reserved1; + char __avx_reserved1[64]; + _STRUCT_XMM_REG __fpu_ymmh0; /* YMMH 0 */ + _STRUCT_XMM_REG __fpu_ymmh1; /* YMMH 1 */ + _STRUCT_XMM_REG __fpu_ymmh2; /* YMMH 2 */ + _STRUCT_XMM_REG __fpu_ymmh3; /* YMMH 3 */ + _STRUCT_XMM_REG __fpu_ymmh4; /* YMMH 4 */ + _STRUCT_XMM_REG __fpu_ymmh5; /* YMMH 5 */ + _STRUCT_XMM_REG __fpu_ymmh6; /* YMMH 6 */ + _STRUCT_XMM_REG __fpu_ymmh7; /* YMMH 7 */ + _STRUCT_OPMASK_REG __fpu_k0; /* K0 */ + _STRUCT_OPMASK_REG __fpu_k1; /* K1 */ + _STRUCT_OPMASK_REG __fpu_k2; /* K2 */ + _STRUCT_OPMASK_REG __fpu_k3; /* K3 */ + _STRUCT_OPMASK_REG __fpu_k4; /* K4 */ + _STRUCT_OPMASK_REG __fpu_k5; /* K5 */ + _STRUCT_OPMASK_REG __fpu_k6; /* K6 */ + _STRUCT_OPMASK_REG __fpu_k7; /* K7 */ + _STRUCT_YMM_REG __fpu_zmmh0; /* ZMMH 0 */ + _STRUCT_YMM_REG __fpu_zmmh1; /* ZMMH 1 */ + _STRUCT_YMM_REG __fpu_zmmh2; /* ZMMH 2 */ + _STRUCT_YMM_REG __fpu_zmmh3; /* ZMMH 3 */ + _STRUCT_YMM_REG __fpu_zmmh4; /* ZMMH 4 */ + _STRUCT_YMM_REG __fpu_zmmh5; /* ZMMH 5 */ + _STRUCT_YMM_REG __fpu_zmmh6; /* ZMMH 6 */ + _STRUCT_YMM_REG __fpu_zmmh7; /* ZMMH 7 */ +}; +#endif /* not RC_HIDE_XNU_J137 */ + #else /* !__DARWIN_UNIX03 */ #define _STRUCT_X86_FLOAT_STATE32 struct i386_float_state _STRUCT_X86_FLOAT_STATE32 @@ -384,16 +499,80 @@ _STRUCT_X86_AVX_STATE32 _STRUCT_XMM_REG fpu_xmm7; /* XMM 7 */ char fpu_rsrv4[14*16]; /* reserved */ int fpu_reserved1; - char __avx_reserved1[64]; - _STRUCT_XMM_REG __fpu_ymmh0; /* YMMH 0 */ - _STRUCT_XMM_REG __fpu_ymmh1; /* YMMH 1 */ - _STRUCT_XMM_REG __fpu_ymmh2; /* YMMH 2 */ - _STRUCT_XMM_REG __fpu_ymmh3; /* YMMH 3 */ - _STRUCT_XMM_REG __fpu_ymmh4; /* YMMH 4 */ - _STRUCT_XMM_REG __fpu_ymmh5; /* YMMH 5 */ - _STRUCT_XMM_REG __fpu_ymmh6; /* YMMH 6 */ - _STRUCT_XMM_REG __fpu_ymmh7; /* YMMH 7 */ + char avx_reserved1[64]; + _STRUCT_XMM_REG fpu_ymmh0; /* YMMH 0 */ + _STRUCT_XMM_REG fpu_ymmh1; /* YMMH 1 */ + _STRUCT_XMM_REG fpu_ymmh2; /* YMMH 2 */ + _STRUCT_XMM_REG fpu_ymmh3; /* YMMH 3 */ + _STRUCT_XMM_REG fpu_ymmh4; /* YMMH 4 */ + _STRUCT_XMM_REG fpu_ymmh5; /* YMMH 5 */ + _STRUCT_XMM_REG fpu_ymmh6; /* YMMH 6 */ + _STRUCT_XMM_REG fpu_ymmh7; /* YMMH 7 */ +}; + +#if !defined(RC_HIDE_XNU_J137) +#define _STRUCT_X86_AVX512_STATE32 struct i386_avx512_state +_STRUCT_X86_AVX512_STATE32 +{ + int fpu_reserved[2]; + _STRUCT_FP_CONTROL fpu_fcw; /* x87 FPU control word */ + _STRUCT_FP_STATUS fpu_fsw; /* x87 FPU status word */ + __uint8_t fpu_ftw; /* x87 FPU tag word */ + __uint8_t fpu_rsrv1; /* reserved */ + __uint16_t fpu_fop; /* x87 FPU Opcode */ + __uint32_t fpu_ip; /* x87 FPU Instruction Pointer offset */ + __uint16_t fpu_cs; /* x87 FPU Instruction Pointer Selector */ + __uint16_t fpu_rsrv2; /* reserved */ + __uint32_t fpu_dp; /* x87 FPU Instruction Operand(Data) Pointer offset */ + __uint16_t fpu_ds; /* x87 FPU Instruction Operand(Data) Pointer Selector */ + __uint16_t fpu_rsrv3; /* reserved */ + __uint32_t fpu_mxcsr; /* MXCSR Register state */ + __uint32_t fpu_mxcsrmask; /* MXCSR mask */ + _STRUCT_MMST_REG fpu_stmm0; /* ST0/MM0 */ + _STRUCT_MMST_REG fpu_stmm1; /* ST1/MM1 */ + _STRUCT_MMST_REG fpu_stmm2; /* ST2/MM2 */ + _STRUCT_MMST_REG fpu_stmm3; /* ST3/MM3 */ + _STRUCT_MMST_REG fpu_stmm4; /* ST4/MM4 */ + _STRUCT_MMST_REG fpu_stmm5; /* ST5/MM5 */ + _STRUCT_MMST_REG fpu_stmm6; /* ST6/MM6 */ + _STRUCT_MMST_REG fpu_stmm7; /* ST7/MM7 */ + _STRUCT_XMM_REG fpu_xmm0; /* XMM 0 */ + _STRUCT_XMM_REG fpu_xmm1; /* XMM 1 */ + _STRUCT_XMM_REG fpu_xmm2; /* XMM 2 */ + _STRUCT_XMM_REG fpu_xmm3; /* XMM 3 */ + _STRUCT_XMM_REG fpu_xmm4; /* XMM 4 */ + _STRUCT_XMM_REG fpu_xmm5; /* XMM 5 */ + _STRUCT_XMM_REG fpu_xmm6; /* XMM 6 */ + _STRUCT_XMM_REG fpu_xmm7; /* XMM 7 */ + char fpu_rsrv4[14*16]; /* reserved */ + int fpu_reserved1; + char avx_reserved1[64]; + _STRUCT_XMM_REG fpu_ymmh0; /* YMMH 0 */ + _STRUCT_XMM_REG fpu_ymmh1; /* YMMH 1 */ + _STRUCT_XMM_REG fpu_ymmh2; /* YMMH 2 */ + _STRUCT_XMM_REG fpu_ymmh3; /* YMMH 3 */ + _STRUCT_XMM_REG fpu_ymmh4; /* YMMH 4 */ + _STRUCT_XMM_REG fpu_ymmh5; /* YMMH 5 */ + _STRUCT_XMM_REG fpu_ymmh6; /* YMMH 6 */ + _STRUCT_XMM_REG fpu_ymmh7; /* YMMH 7 */ + _STRUCT_OPMASK_REG fpu_k0; /* K0 */ + _STRUCT_OPMASK_REG fpu_k1; /* K1 */ + _STRUCT_OPMASK_REG fpu_k2; /* K2 */ + _STRUCT_OPMASK_REG fpu_k3; /* K3 */ + _STRUCT_OPMASK_REG fpu_k4; /* K4 */ + _STRUCT_OPMASK_REG fpu_k5; /* K5 */ + _STRUCT_OPMASK_REG fpu_k6; /* K6 */ + _STRUCT_OPMASK_REG fpu_k7; /* K7 */ + _STRUCT_YMM_REG fpu_zmmh0; /* ZMMH 0 */ + _STRUCT_YMM_REG fpu_zmmh1; /* ZMMH 1 */ + _STRUCT_YMM_REG fpu_zmmh2; /* ZMMH 2 */ + _STRUCT_YMM_REG fpu_zmmh3; /* ZMMH 3 */ + _STRUCT_YMM_REG fpu_zmmh4; /* ZMMH 4 */ + _STRUCT_YMM_REG fpu_zmmh5; /* ZMMH 5 */ + _STRUCT_YMM_REG fpu_zmmh6; /* ZMMH 6 */ + _STRUCT_YMM_REG fpu_zmmh7; /* ZMMH 7 */ }; +#endif /* not RC_HIDE_XNU_J137 */ #endif /* !__DARWIN_UNIX03 */ @@ -624,6 +803,116 @@ _STRUCT_X86_AVX_STATE64 _STRUCT_XMM_REG __fpu_ymmh15; /* YMMH 15 */ }; +#if !defined(RC_HIDE_XNU_J137) +#define _STRUCT_X86_AVX512_STATE64 struct __darwin_x86_avx512_state64 +_STRUCT_X86_AVX512_STATE64 +{ + int __fpu_reserved[2]; + _STRUCT_FP_CONTROL __fpu_fcw; /* x87 FPU control word */ + _STRUCT_FP_STATUS __fpu_fsw; /* x87 FPU status word */ + __uint8_t __fpu_ftw; /* x87 FPU tag word */ + __uint8_t __fpu_rsrv1; /* reserved */ + __uint16_t __fpu_fop; /* x87 FPU Opcode */ + + /* x87 FPU Instruction Pointer */ + __uint32_t __fpu_ip; /* offset */ + __uint16_t __fpu_cs; /* Selector */ + + __uint16_t __fpu_rsrv2; /* reserved */ + + /* x87 FPU Instruction Operand(Data) Pointer */ + __uint32_t __fpu_dp; /* offset */ + __uint16_t __fpu_ds; /* Selector */ + + __uint16_t __fpu_rsrv3; /* reserved */ + __uint32_t __fpu_mxcsr; /* MXCSR Register state */ + __uint32_t __fpu_mxcsrmask; /* MXCSR mask */ + _STRUCT_MMST_REG __fpu_stmm0; /* ST0/MM0 */ + _STRUCT_MMST_REG __fpu_stmm1; /* ST1/MM1 */ + _STRUCT_MMST_REG __fpu_stmm2; /* ST2/MM2 */ + _STRUCT_MMST_REG __fpu_stmm3; /* ST3/MM3 */ + _STRUCT_MMST_REG __fpu_stmm4; /* ST4/MM4 */ + _STRUCT_MMST_REG __fpu_stmm5; /* ST5/MM5 */ + _STRUCT_MMST_REG __fpu_stmm6; /* ST6/MM6 */ + _STRUCT_MMST_REG __fpu_stmm7; /* ST7/MM7 */ + _STRUCT_XMM_REG __fpu_xmm0; /* XMM 0 */ + _STRUCT_XMM_REG __fpu_xmm1; /* XMM 1 */ + _STRUCT_XMM_REG __fpu_xmm2; /* XMM 2 */ + _STRUCT_XMM_REG __fpu_xmm3; /* XMM 3 */ + _STRUCT_XMM_REG __fpu_xmm4; /* XMM 4 */ + _STRUCT_XMM_REG __fpu_xmm5; /* XMM 5 */ + _STRUCT_XMM_REG __fpu_xmm6; /* XMM 6 */ + _STRUCT_XMM_REG __fpu_xmm7; /* XMM 7 */ + _STRUCT_XMM_REG __fpu_xmm8; /* XMM 8 */ + _STRUCT_XMM_REG __fpu_xmm9; /* XMM 9 */ + _STRUCT_XMM_REG __fpu_xmm10; /* XMM 10 */ + _STRUCT_XMM_REG __fpu_xmm11; /* XMM 11 */ + _STRUCT_XMM_REG __fpu_xmm12; /* XMM 12 */ + _STRUCT_XMM_REG __fpu_xmm13; /* XMM 13 */ + _STRUCT_XMM_REG __fpu_xmm14; /* XMM 14 */ + _STRUCT_XMM_REG __fpu_xmm15; /* XMM 15 */ + char __fpu_rsrv4[6*16]; /* reserved */ + int __fpu_reserved1; + char __avx_reserved1[64]; + _STRUCT_XMM_REG __fpu_ymmh0; /* YMMH 0 */ + _STRUCT_XMM_REG __fpu_ymmh1; /* YMMH 1 */ + _STRUCT_XMM_REG __fpu_ymmh2; /* YMMH 2 */ + _STRUCT_XMM_REG __fpu_ymmh3; /* YMMH 3 */ + _STRUCT_XMM_REG __fpu_ymmh4; /* YMMH 4 */ + _STRUCT_XMM_REG __fpu_ymmh5; /* YMMH 5 */ + _STRUCT_XMM_REG __fpu_ymmh6; /* YMMH 6 */ + _STRUCT_XMM_REG __fpu_ymmh7; /* YMMH 7 */ + _STRUCT_XMM_REG __fpu_ymmh8; /* YMMH 8 */ + _STRUCT_XMM_REG __fpu_ymmh9; /* YMMH 9 */ + _STRUCT_XMM_REG __fpu_ymmh10; /* YMMH 10 */ + _STRUCT_XMM_REG __fpu_ymmh11; /* YMMH 11 */ + _STRUCT_XMM_REG __fpu_ymmh12; /* YMMH 12 */ + _STRUCT_XMM_REG __fpu_ymmh13; /* YMMH 13 */ + _STRUCT_XMM_REG __fpu_ymmh14; /* YMMH 14 */ + _STRUCT_XMM_REG __fpu_ymmh15; /* YMMH 15 */ + _STRUCT_OPMASK_REG __fpu_k0; /* K0 */ + _STRUCT_OPMASK_REG __fpu_k1; /* K1 */ + _STRUCT_OPMASK_REG __fpu_k2; /* K2 */ + _STRUCT_OPMASK_REG __fpu_k3; /* K3 */ + _STRUCT_OPMASK_REG __fpu_k4; /* K4 */ + _STRUCT_OPMASK_REG __fpu_k5; /* K5 */ + _STRUCT_OPMASK_REG __fpu_k6; /* K6 */ + _STRUCT_OPMASK_REG __fpu_k7; /* K7 */ + _STRUCT_YMM_REG __fpu_zmmh0; /* ZMMH 0 */ + _STRUCT_YMM_REG __fpu_zmmh1; /* ZMMH 1 */ + _STRUCT_YMM_REG __fpu_zmmh2; /* ZMMH 2 */ + _STRUCT_YMM_REG __fpu_zmmh3; /* ZMMH 3 */ + _STRUCT_YMM_REG __fpu_zmmh4; /* ZMMH 4 */ + _STRUCT_YMM_REG __fpu_zmmh5; /* ZMMH 5 */ + _STRUCT_YMM_REG __fpu_zmmh6; /* ZMMH 6 */ + _STRUCT_YMM_REG __fpu_zmmh7; /* ZMMH 7 */ + _STRUCT_YMM_REG __fpu_zmmh8; /* ZMMH 8 */ + _STRUCT_YMM_REG __fpu_zmmh9; /* ZMMH 9 */ + _STRUCT_YMM_REG __fpu_zmmh10; /* ZMMH 10 */ + _STRUCT_YMM_REG __fpu_zmmh11; /* ZMMH 11 */ + _STRUCT_YMM_REG __fpu_zmmh12; /* ZMMH 12 */ + _STRUCT_YMM_REG __fpu_zmmh13; /* ZMMH 13 */ + _STRUCT_YMM_REG __fpu_zmmh14; /* ZMMH 14 */ + _STRUCT_YMM_REG __fpu_zmmh15; /* ZMMH 15 */ + _STRUCT_ZMM_REG __fpu_zmm16; /* ZMM 16 */ + _STRUCT_ZMM_REG __fpu_zmm17; /* ZMM 17 */ + _STRUCT_ZMM_REG __fpu_zmm18; /* ZMM 18 */ + _STRUCT_ZMM_REG __fpu_zmm19; /* ZMM 19 */ + _STRUCT_ZMM_REG __fpu_zmm20; /* ZMM 20 */ + _STRUCT_ZMM_REG __fpu_zmm21; /* ZMM 21 */ + _STRUCT_ZMM_REG __fpu_zmm22; /* ZMM 22 */ + _STRUCT_ZMM_REG __fpu_zmm23; /* ZMM 23 */ + _STRUCT_ZMM_REG __fpu_zmm24; /* ZMM 24 */ + _STRUCT_ZMM_REG __fpu_zmm25; /* ZMM 25 */ + _STRUCT_ZMM_REG __fpu_zmm26; /* ZMM 26 */ + _STRUCT_ZMM_REG __fpu_zmm27; /* ZMM 27 */ + _STRUCT_ZMM_REG __fpu_zmm28; /* ZMM 28 */ + _STRUCT_ZMM_REG __fpu_zmm29; /* ZMM 29 */ + _STRUCT_ZMM_REG __fpu_zmm30; /* ZMM 30 */ + _STRUCT_ZMM_REG __fpu_zmm31; /* ZMM 31 */ +}; +#endif /* not RC_HIDE_XNU_J137 */ + #else /* !__DARWIN_UNIX03 */ #define _STRUCT_X86_FLOAT_STATE64 struct x86_float_state64 _STRUCT_X86_FLOAT_STATE64 @@ -725,24 +1014,134 @@ _STRUCT_X86_AVX_STATE64 _STRUCT_XMM_REG fpu_xmm15; /* XMM 15 */ char fpu_rsrv4[6*16]; /* reserved */ int fpu_reserved1; - char __avx_reserved1[64]; - _STRUCT_XMM_REG __fpu_ymmh0; /* YMMH 0 */ - _STRUCT_XMM_REG __fpu_ymmh1; /* YMMH 1 */ - _STRUCT_XMM_REG __fpu_ymmh2; /* YMMH 2 */ - _STRUCT_XMM_REG __fpu_ymmh3; /* YMMH 3 */ - _STRUCT_XMM_REG __fpu_ymmh4; /* YMMH 4 */ - _STRUCT_XMM_REG __fpu_ymmh5; /* YMMH 5 */ - _STRUCT_XMM_REG __fpu_ymmh6; /* YMMH 6 */ - _STRUCT_XMM_REG __fpu_ymmh7; /* YMMH 7 */ - _STRUCT_XMM_REG __fpu_ymmh8; /* YMMH 8 */ - _STRUCT_XMM_REG __fpu_ymmh9; /* YMMH 9 */ - _STRUCT_XMM_REG __fpu_ymmh10; /* YMMH 10 */ - _STRUCT_XMM_REG __fpu_ymmh11; /* YMMH 11 */ - _STRUCT_XMM_REG __fpu_ymmh12; /* YMMH 12 */ - _STRUCT_XMM_REG __fpu_ymmh13; /* YMMH 13 */ - _STRUCT_XMM_REG __fpu_ymmh14; /* YMMH 14 */ - _STRUCT_XMM_REG __fpu_ymmh15; /* YMMH 15 */ + char avx_reserved1[64]; + _STRUCT_XMM_REG fpu_ymmh0; /* YMMH 0 */ + _STRUCT_XMM_REG fpu_ymmh1; /* YMMH 1 */ + _STRUCT_XMM_REG fpu_ymmh2; /* YMMH 2 */ + _STRUCT_XMM_REG fpu_ymmh3; /* YMMH 3 */ + _STRUCT_XMM_REG fpu_ymmh4; /* YMMH 4 */ + _STRUCT_XMM_REG fpu_ymmh5; /* YMMH 5 */ + _STRUCT_XMM_REG fpu_ymmh6; /* YMMH 6 */ + _STRUCT_XMM_REG fpu_ymmh7; /* YMMH 7 */ + _STRUCT_XMM_REG fpu_ymmh8; /* YMMH 8 */ + _STRUCT_XMM_REG fpu_ymmh9; /* YMMH 9 */ + _STRUCT_XMM_REG fpu_ymmh10; /* YMMH 10 */ + _STRUCT_XMM_REG fpu_ymmh11; /* YMMH 11 */ + _STRUCT_XMM_REG fpu_ymmh12; /* YMMH 12 */ + _STRUCT_XMM_REG fpu_ymmh13; /* YMMH 13 */ + _STRUCT_XMM_REG fpu_ymmh14; /* YMMH 14 */ + _STRUCT_XMM_REG fpu_ymmh15; /* YMMH 15 */ +}; + +#if !defined(RC_HIDE_XNU_J137) +#define _STRUCT_X86_AVX512_STATE64 struct x86_avx512_state64 +_STRUCT_X86_AVX512_STATE64 +{ + int fpu_reserved[2]; + _STRUCT_FP_CONTROL fpu_fcw; /* x87 FPU control word */ + _STRUCT_FP_STATUS fpu_fsw; /* x87 FPU status word */ + __uint8_t fpu_ftw; /* x87 FPU tag word */ + __uint8_t fpu_rsrv1; /* reserved */ + __uint16_t fpu_fop; /* x87 FPU Opcode */ + + /* x87 FPU Instruction Pointer */ + __uint32_t fpu_ip; /* offset */ + __uint16_t fpu_cs; /* Selector */ + + __uint16_t fpu_rsrv2; /* reserved */ + + /* x87 FPU Instruction Operand(Data) Pointer */ + __uint32_t fpu_dp; /* offset */ + __uint16_t fpu_ds; /* Selector */ + + __uint16_t fpu_rsrv3; /* reserved */ + __uint32_t fpu_mxcsr; /* MXCSR Register state */ + __uint32_t fpu_mxcsrmask; /* MXCSR mask */ + _STRUCT_MMST_REG fpu_stmm0; /* ST0/MM0 */ + _STRUCT_MMST_REG fpu_stmm1; /* ST1/MM1 */ + _STRUCT_MMST_REG fpu_stmm2; /* ST2/MM2 */ + _STRUCT_MMST_REG fpu_stmm3; /* ST3/MM3 */ + _STRUCT_MMST_REG fpu_stmm4; /* ST4/MM4 */ + _STRUCT_MMST_REG fpu_stmm5; /* ST5/MM5 */ + _STRUCT_MMST_REG fpu_stmm6; /* ST6/MM6 */ + _STRUCT_MMST_REG fpu_stmm7; /* ST7/MM7 */ + _STRUCT_XMM_REG fpu_xmm0; /* XMM 0 */ + _STRUCT_XMM_REG fpu_xmm1; /* XMM 1 */ + _STRUCT_XMM_REG fpu_xmm2; /* XMM 2 */ + _STRUCT_XMM_REG fpu_xmm3; /* XMM 3 */ + _STRUCT_XMM_REG fpu_xmm4; /* XMM 4 */ + _STRUCT_XMM_REG fpu_xmm5; /* XMM 5 */ + _STRUCT_XMM_REG fpu_xmm6; /* XMM 6 */ + _STRUCT_XMM_REG fpu_xmm7; /* XMM 7 */ + _STRUCT_XMM_REG fpu_xmm8; /* XMM 8 */ + _STRUCT_XMM_REG fpu_xmm9; /* XMM 9 */ + _STRUCT_XMM_REG fpu_xmm10; /* XMM 10 */ + _STRUCT_XMM_REG fpu_xmm11; /* XMM 11 */ + _STRUCT_XMM_REG fpu_xmm12; /* XMM 12 */ + _STRUCT_XMM_REG fpu_xmm13; /* XMM 13 */ + _STRUCT_XMM_REG fpu_xmm14; /* XMM 14 */ + _STRUCT_XMM_REG fpu_xmm15; /* XMM 15 */ + char fpu_rsrv4[6*16]; /* reserved */ + int fpu_reserved1; + char avx_reserved1[64]; + _STRUCT_XMM_REG fpu_ymmh0; /* YMMH 0 */ + _STRUCT_XMM_REG fpu_ymmh1; /* YMMH 1 */ + _STRUCT_XMM_REG fpu_ymmh2; /* YMMH 2 */ + _STRUCT_XMM_REG fpu_ymmh3; /* YMMH 3 */ + _STRUCT_XMM_REG fpu_ymmh4; /* YMMH 4 */ + _STRUCT_XMM_REG fpu_ymmh5; /* YMMH 5 */ + _STRUCT_XMM_REG fpu_ymmh6; /* YMMH 6 */ + _STRUCT_XMM_REG fpu_ymmh7; /* YMMH 7 */ + _STRUCT_XMM_REG fpu_ymmh8; /* YMMH 8 */ + _STRUCT_XMM_REG fpu_ymmh9; /* YMMH 9 */ + _STRUCT_XMM_REG fpu_ymmh10; /* YMMH 10 */ + _STRUCT_XMM_REG fpu_ymmh11; /* YMMH 11 */ + _STRUCT_XMM_REG fpu_ymmh12; /* YMMH 12 */ + _STRUCT_XMM_REG fpu_ymmh13; /* YMMH 13 */ + _STRUCT_XMM_REG fpu_ymmh14; /* YMMH 14 */ + _STRUCT_XMM_REG fpu_ymmh15; /* YMMH 15 */ + _STRUCT_OPMASK_REG fpu_k0; /* K0 */ + _STRUCT_OPMASK_REG fpu_k1; /* K1 */ + _STRUCT_OPMASK_REG fpu_k2; /* K2 */ + _STRUCT_OPMASK_REG fpu_k3; /* K3 */ + _STRUCT_OPMASK_REG fpu_k4; /* K4 */ + _STRUCT_OPMASK_REG fpu_k5; /* K5 */ + _STRUCT_OPMASK_REG fpu_k6; /* K6 */ + _STRUCT_OPMASK_REG fpu_k7; /* K7 */ + _STRUCT_YMM_REG fpu_zmmh0; /* ZMMH 0 */ + _STRUCT_YMM_REG fpu_zmmh1; /* ZMMH 1 */ + _STRUCT_YMM_REG fpu_zmmh2; /* ZMMH 2 */ + _STRUCT_YMM_REG fpu_zmmh3; /* ZMMH 3 */ + _STRUCT_YMM_REG fpu_zmmh4; /* ZMMH 4 */ + _STRUCT_YMM_REG fpu_zmmh5; /* ZMMH 5 */ + _STRUCT_YMM_REG fpu_zmmh6; /* ZMMH 6 */ + _STRUCT_YMM_REG fpu_zmmh7; /* ZMMH 7 */ + _STRUCT_YMM_REG fpu_zmmh8; /* ZMMH 8 */ + _STRUCT_YMM_REG fpu_zmmh9; /* ZMMH 9 */ + _STRUCT_YMM_REG fpu_zmmh10; /* ZMMH 10 */ + _STRUCT_YMM_REG fpu_zmmh11; /* ZMMH 11 */ + _STRUCT_YMM_REG fpu_zmmh12; /* ZMMH 12 */ + _STRUCT_YMM_REG fpu_zmmh13; /* ZMMH 13 */ + _STRUCT_YMM_REG fpu_zmmh14; /* ZMMH 14 */ + _STRUCT_YMM_REG fpu_zmmh15; /* ZMMH 15 */ + _STRUCT_ZMM_REG fpu_zmm16; /* ZMM 16 */ + _STRUCT_ZMM_REG fpu_zmm17; /* ZMM 17 */ + _STRUCT_ZMM_REG fpu_zmm18; /* ZMM 18 */ + _STRUCT_ZMM_REG fpu_zmm19; /* ZMM 19 */ + _STRUCT_ZMM_REG fpu_zmm20; /* ZMM 20 */ + _STRUCT_ZMM_REG fpu_zmm21; /* ZMM 21 */ + _STRUCT_ZMM_REG fpu_zmm22; /* ZMM 22 */ + _STRUCT_ZMM_REG fpu_zmm23; /* ZMM 23 */ + _STRUCT_ZMM_REG fpu_zmm24; /* ZMM 24 */ + _STRUCT_ZMM_REG fpu_zmm25; /* ZMM 25 */ + _STRUCT_ZMM_REG fpu_zmm26; /* ZMM 26 */ + _STRUCT_ZMM_REG fpu_zmm27; /* ZMM 27 */ + _STRUCT_ZMM_REG fpu_zmm28; /* ZMM 28 */ + _STRUCT_ZMM_REG fpu_zmm29; /* ZMM 29 */ + _STRUCT_ZMM_REG fpu_zmm30; /* ZMM 30 */ + _STRUCT_ZMM_REG fpu_zmm31; /* ZMM 31 */ }; +#endif /* not RC_HIDE_XNU_J137 */ #endif /* !__DARWIN_UNIX03 */ @@ -794,4 +1193,18 @@ _STRUCT_X86_DEBUG_STATE64 }; #endif /* !__DARWIN_UNIX03 */ +#if __DARWIN_UNIX03 +#define _STRUCT_X86_CPMU_STATE64 struct __darwin_x86_cpmu_state64 +_STRUCT_X86_CPMU_STATE64 +{ + __uint64_t __ctrs[16]; +}; +#else /* __DARWIN_UNIX03 */ +#define _STRUCT_X86_CPMU_STATE64 struct x86_cpmu_state64 +_STRUCT_X86_CPMU_STATE64 +{ + __uint64_t ctrs[16]; +}; +#endif /* !__DARWIN_UNIX03 */ + #endif /* _MACH_I386__STRUCTS_H_ */ diff --git a/osfmk/mach/i386/fp_reg.h b/osfmk/mach/i386/fp_reg.h index 92d504f3f..718298b59 100644 --- a/osfmk/mach/i386/fp_reg.h +++ b/osfmk/mach/i386/fp_reg.h @@ -68,12 +68,20 @@ struct x86_fx_thread_state { unsigned char fx_tag; /* register tags */ unsigned char fx_bbz1; /* better be zero when calling fxrtstor */ unsigned short fx_opcode; - unsigned int fx_eip; /* eip instruction */ - unsigned short fx_cs; /* cs instruction */ - unsigned short fx_bbz2; /* better be zero when calling fxrtstor */ - unsigned int fx_dp; /* data address */ - unsigned short fx_ds; /* data segment */ - unsigned short fx_bbz3; /* better be zero when calling fxrtstor */ + union { + struct { /* 32-bit layout: */ + unsigned int fx_eip; /* eip instruction */ + unsigned short fx_cs; /* cs instruction */ + unsigned short fx_bbz2; /* better be zero when calling fxrtstor */ + unsigned int fx_dp; /* data address */ + unsigned short fx_ds; /* data segment */ + unsigned short fx_bbz3; /* better be zero when calling fxrtstor */ + }; + struct { /* 64-bit layout: */ + uint64_t fx_rip; /* instruction pointer */ + uint64_t fx_rdp; /* data pointer */ + }; + }; unsigned int fx_MXCSR; unsigned int fx_MXCSR_MASK; unsigned short fx_reg_word[8][8]; /* STx/MMx registers */ @@ -87,36 +95,50 @@ struct x86_fx_thread_state { unsigned char fx_pad[8]; }__attribute__ ((packed)); +struct xsave_header { + uint64_t xstate_bv; + uint64_t xcomp_bv; + uint8_t xhrsvd[48]; +}; + +typedef struct { uint64_t lo64, hi64; }__attribute__ ((packed)) reg128_t; +typedef struct { reg128_t lo128, hi128; }__attribute__ ((packed)) reg256_t; +typedef struct { reg256_t lo256, hi256; }__attribute__ ((packed)) reg512_t; + struct x86_avx_thread_state { - unsigned short fx_control; /* control */ - unsigned short fx_status; /* status */ - unsigned char fx_tag; /* register tags */ - unsigned char fx_bbz1; /* reserved zero */ - unsigned short fx_opcode; - unsigned int fx_eip; /* eip instruction */ - unsigned short fx_cs; /* cs instruction */ - unsigned short fx_bbz2; /* reserved zero */ - unsigned int fx_dp; /* data address */ - unsigned short fx_ds; /* data segment */ - unsigned short fx_bbz3; /* reserved zero */ - unsigned int fx_MXCSR; - unsigned int fx_MXCSR_MASK; - unsigned short fx_reg_word[8][8]; /* STx/MMx registers */ - unsigned short fx_XMM_reg[8][16]; /* XMM0-XMM15 on 64 bit processors */ - /* XMM0-XMM7 on 32 bit processors... unused storage reserved */ - unsigned char fx_reserved[16*5]; /* reserved */ - unsigned int fp_valid; - unsigned int fp_save_layout; - unsigned char fx_pad[8]; + struct x86_fx_thread_state fp; + struct xsave_header _xh; /* Offset 512, xsave header */ + reg128_t x_YMM_Hi128[16]; /* Offset 576, high YMMs `*/ + /* Offset 832, end */ +}__attribute__ ((packed)); - struct xsave_header { /* Offset 512, xsave header */ - uint64_t xsbv; - char xhrsvd[56]; - }_xh; +struct x86_avx512_thread_state { + struct x86_fx_thread_state fp; + struct xsave_header _xh; /* Offset 512, xsave header */ + reg128_t x_YMM_Hi128[16]; /* Offset 576, high YMMs */ - unsigned int x_YMMH_reg[4][16]; /* Offset 576, high YMMs*/ + uint64_t x_pad[16]; /* Offset 832, unused AMD LWP */ + uint64_t x_BNDREGS[8]; /* Offset 960, unused MPX */ + uint64_t x_BNDCTL[8]; /* Offset 1024, unused MPX */ + + uint64_t x_Opmask[8]; /* Offset 1088, K0-K7 */ + reg256_t x_ZMM_Hi256[16]; /* Offset 1152, ZMM0..15[511:256] */ + reg512_t x_Hi16_ZMM[16]; /* Offset 1664, ZMM16..31[511:0] */ + /* Offset 2688, end */ }__attribute__ ((packed)); +typedef union { + struct x86_fx_thread_state fx; + struct x86_avx_thread_state avx; +#if !defined(RC_HIDE_XNU_J137) + struct x86_avx512_thread_state avx512; +#endif +} x86_ext_thread_state_t; + +#define EVEX_PREFIX 0x62 /* AVX512's EVEX vector operation prefix */ +#define VEX2_PREFIX 0xC5 /* VEX 2-byte prefix for Opmask instructions */ +#define VEX3_PREFIX 0xC4 /* VEX 3-byte prefix for Opmask instructions */ + #endif /* MACH_KERNEL_PRIVATE */ /* * Control register diff --git a/osfmk/mach/i386/thread_state.h b/osfmk/mach/i386/thread_state.h index 2d542d3dd..9ed704007 100644 --- a/osfmk/mach/i386/thread_state.h +++ b/osfmk/mach/i386/thread_state.h @@ -33,7 +33,11 @@ #define _MACH_I386_THREAD_STATE_H_ /* Size of maximum exported thread state in words */ +#if !defined(RC_HIDE_XNU_J137) +#define I386_THREAD_STATE_MAX (614) /* Size of biggest state possible */ +#else #define I386_THREAD_STATE_MAX (224) /* Size of biggest state possible */ +#endif /* !defined(RC_HIDE_XNU_J137) */ #if defined (__i386__) || defined(__x86_64__) #define THREAD_STATE_MAX I386_THREAD_STATE_MAX diff --git a/osfmk/mach/i386/thread_status.h b/osfmk/mach/i386/thread_status.h index 03cc0f91b..fea611b4d 100644 --- a/osfmk/mach/i386/thread_status.h +++ b/osfmk/mach/i386/thread_status.h @@ -67,12 +67,16 @@ #ifndef _MACH_I386_THREAD_STATUS_H_ #define _MACH_I386_THREAD_STATUS_H_ -#include +#include #include #include #include #include +#ifdef KERNEL_PRIVATE +#include +#endif + /* * the i386_xxxx form is kept for legacy purposes since these types * are externally known... eventually they should be deprecated. @@ -111,9 +115,15 @@ #define x86_DEBUG_STATE 12 #define THREAD_STATE_NONE 13 /* 14 and 15 are used for the internal x86_SAVED_STATE flavours */ +/* Arrange for flavors to take sequential values, 32-bit, 64-bit, non-specific */ #define x86_AVX_STATE32 16 -#define x86_AVX_STATE64 17 -#define x86_AVX_STATE 18 +#define x86_AVX_STATE64 (x86_AVX_STATE32 + 1) +#define x86_AVX_STATE (x86_AVX_STATE32 + 2) +#if !defined(RC_HIDE_XNU_J137) +#define x86_AVX512_STATE32 19 +#define x86_AVX512_STATE64 (x86_AVX512_STATE32 + 1) +#define x86_AVX512_STATE (x86_AVX512_STATE32 + 2) +#endif /* not RC_HIDE_XNU_J137 */ /* @@ -128,6 +138,28 @@ * platform. The macro must be manually updated to include all of the valid * exception flavors as defined above. */ +#if !defined(RC_HIDE_XNU_J137) +#define VALID_THREAD_STATE_FLAVOR(x) \ + ((x == x86_THREAD_STATE32) || \ + (x == x86_FLOAT_STATE32) || \ + (x == x86_EXCEPTION_STATE32) || \ + (x == x86_DEBUG_STATE32) || \ + (x == x86_THREAD_STATE64) || \ + (x == x86_FLOAT_STATE64) || \ + (x == x86_EXCEPTION_STATE64) || \ + (x == x86_DEBUG_STATE64) || \ + (x == x86_THREAD_STATE) || \ + (x == x86_FLOAT_STATE) || \ + (x == x86_EXCEPTION_STATE) || \ + (x == x86_DEBUG_STATE) || \ + (x == x86_AVX_STATE32) || \ + (x == x86_AVX_STATE64) || \ + (x == x86_AVX_STATE) || \ + (x == x86_AVX512_STATE32) || \ + (x == x86_AVX512_STATE64) || \ + (x == x86_AVX512_STATE) || \ + (x == THREAD_STATE_NONE)) +#else #define VALID_THREAD_STATE_FLAVOR(x) \ ((x == x86_THREAD_STATE32) || \ (x == x86_FLOAT_STATE32) || \ @@ -145,10 +177,11 @@ (x == x86_AVX_STATE64) || \ (x == x86_AVX_STATE) || \ (x == THREAD_STATE_NONE)) +#endif /* not RC_HIDE_XNU_J137 */ struct x86_state_hdr { - int flavor; - int count; + uint32_t flavor; + uint32_t count; }; typedef struct x86_state_hdr x86_state_hdr_t; @@ -187,6 +220,12 @@ typedef _STRUCT_X86_AVX_STATE32 x86_avx_state32_t; #define x86_AVX_STATE32_COUNT ((mach_msg_type_number_t) \ (sizeof(x86_avx_state32_t)/sizeof(unsigned int))) +#if !defined(RC_HIDE_XNU_J137) +typedef _STRUCT_X86_AVX512_STATE32 x86_avx512_state32_t; +#define x86_AVX512_STATE32_COUNT ((mach_msg_type_number_t) \ + (sizeof(x86_avx512_state32_t)/sizeof(unsigned int))) +#endif /* not RC_HIDE_XNU_J137 */ + /* * to be deprecated in the future */ @@ -218,6 +257,12 @@ typedef _STRUCT_X86_AVX_STATE64 x86_avx_state64_t; #define x86_AVX_STATE64_COUNT ((mach_msg_type_number_t) \ (sizeof(x86_avx_state64_t)/sizeof(unsigned int))) +#if !defined(RC_HIDE_XNU_J137) +typedef _STRUCT_X86_AVX512_STATE64 x86_avx512_state64_t; +#define x86_AVX512_STATE64_COUNT ((mach_msg_type_number_t) \ + (sizeof(x86_avx512_state64_t)/sizeof(unsigned int))) +#endif /* not RC_HIDE_XNU_J137 */ + typedef _STRUCT_X86_EXCEPTION_STATE64 x86_exception_state64_t; #define x86_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \ ( sizeof (x86_exception_state64_t) / sizeof (int) )) @@ -273,6 +318,16 @@ struct x86_avx_state { } ufs; }; +#if !defined(RC_HIDE_XNU_J137) +struct x86_avx512_state { + x86_state_hdr_t ash; + union { + x86_avx512_state32_t as32; + x86_avx512_state64_t as64; + } ufs; +}; +#endif /* not RC_HIDE_XNU_J137 */ + typedef struct x86_thread_state x86_thread_state_t; #define x86_THREAD_STATE_COUNT ((mach_msg_type_number_t) \ ( sizeof (x86_thread_state_t) / sizeof (int) )) @@ -293,6 +348,12 @@ typedef struct x86_avx_state x86_avx_state_t; #define x86_AVX_STATE_COUNT ((mach_msg_type_number_t) \ (sizeof(x86_avx_state_t)/sizeof(unsigned int))) +#if !defined(RC_HIDE_XNU_J137) +typedef struct x86_avx512_state x86_avx512_state_t; +#define x86_AVX512_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof(x86_avx512_state_t)/sizeof(unsigned int))) +#endif /* not RC_HIDE_XNU_J137 */ + /* * Machine-independent way for servers and Mach's exception mechanism to * choose the most efficient state flavor for exception RPC's: diff --git a/osfmk/mach/i386/vm_param.h b/osfmk/mach/i386/vm_param.h index 965e596ae..040472dfe 100644 --- a/osfmk/mach/i386/vm_param.h +++ b/osfmk/mach/i386/vm_param.h @@ -137,7 +137,11 @@ /* * default top of user stack... it grows down from here */ -#define VM_USRSTACK64 ((user_addr_t) 0x00007FFF5FC00000ULL) +#define VM_USRSTACK64 ((user_addr_t) 0x00007FFEEFC00000ULL) + +/* + * XXX TODO: Obsolete? + */ #define VM_DYLD64 ((user_addr_t) 0x00007FFF5FC00000ULL) #define VM_LIB64_SHR_DATA ((user_addr_t) 0x00007FFF60000000ULL) #define VM_LIB64_SHR_TEXT ((user_addr_t) 0x00007FFF80000000ULL) @@ -199,14 +203,22 @@ #define KEXT_ALLOC_BASE(x) ((x) - KEXT_ALLOC_MAX_OFFSET) #define KEXT_ALLOC_SIZE(x) (KEXT_ALLOC_MAX_OFFSET - (x)) - -#define KERNEL_STACK_SIZE (I386_PGBYTES*4) +#define VM_KERNEL_ADDRESS(va) ((((vm_address_t)(va))>=VM_MIN_KERNEL_AND_KEXT_ADDRESS) && \ + (((vm_address_t)(va))<=VM_MAX_KERNEL_ADDRESS)) #define VM_MAP_MIN_ADDRESS MACH_VM_MIN_ADDRESS #define VM_MAP_MAX_ADDRESS MACH_VM_MAX_ADDRESS /* FIXME - always leave like this? */ -#define INTSTACK_SIZE (I386_PGBYTES*4) +#if KASAN +/* Increase the stack sizes to account for the redzones that get added to every + * stack object. */ +# define INTSTACK_SIZE (I386_PGBYTES*4*4) +# define KERNEL_STACK_SIZE (I386_PGBYTES*4*4) +#else +# define INTSTACK_SIZE (I386_PGBYTES*4) +# define KERNEL_STACK_SIZE (I386_PGBYTES*4) +#endif #ifdef MACH_KERNEL_PRIVATE diff --git a/osfmk/mach/mach_exc.defs b/osfmk/mach/mach_exc.defs index 00fae2853..a2a7669da 100644 --- a/osfmk/mach/mach_exc.defs +++ b/osfmk/mach/mach_exc.defs @@ -75,25 +75,15 @@ type mach_exception_data_t = array[*:2] of int64_t; type exception_type_t = int; routine mach_exception_raise( -#if KERNEL_USER - exception_port : mach_port_move_send_t; - thread : mach_port_move_send_t; - task : mach_port_move_send_t; -#else /* KERNEL_USER */ exception_port : mach_port_t; thread : mach_port_t; task : mach_port_t; -#endif /* KERNEL_USER */ exception : exception_type_t; code : mach_exception_data_t ); routine mach_exception_raise_state( -#if KERNEL_USER - exception_port : mach_port_move_send_t; -#else /* KERNEL_USER */ exception_port : mach_port_t; -#endif /* KERNEL_USER */ exception : exception_type_t; code : mach_exception_data_t, const; inout flavor : int; @@ -101,15 +91,9 @@ routine mach_exception_raise_state( out new_state : thread_state_t); routine mach_exception_raise_state_identity( -#if KERNEL_USER - exception_port : mach_port_move_send_t; - thread : mach_port_move_send_t; - task : mach_port_move_send_t; -#else /* KERNEL_USER */ exception_port : mach_port_t; thread : mach_port_t; task : mach_port_t; -#endif /* KERNEL_USER */ exception : exception_type_t; code : mach_exception_data_t; inout flavor : int; diff --git a/osfmk/mach/mach_host.defs b/osfmk/mach/mach_host.defs index 04c44fde2..637109763 100644 --- a/osfmk/mach/mach_host.defs +++ b/osfmk/mach/mach_host.defs @@ -161,18 +161,8 @@ routine kmod_get_info( host : host_t; out modules : kmod_args_t); -/* - * Returns information about the memory allocation zones. - * Supported in all kernels.. - * - * DEPRECATED! Use mach_zone_info() instead. - */ -routine host_zone_info( - host : host_priv_t; - out names : zone_name_array_t, - Dealloc; - out info : zone_info_array_t, - Dealloc); + +skip; /* was host_zone_info */ /* * Returns information about the global VP table. @@ -250,7 +240,7 @@ routine host_statistics64( /* * Returns information about the memory allocation zones. * Data returned is compatible with various caller and kernel - * address space sizes (unlike host_zone_info()). + * address space sizes. */ routine mach_zone_info( host : host_priv_t; diff --git a/osfmk/mach/mach_traps.h b/osfmk/mach/mach_traps.h index 66b5d4c8e..462d7972f 100644 --- a/osfmk/mach/mach_traps.h +++ b/osfmk/mach/mach_traps.h @@ -87,6 +87,8 @@ __BEGIN_DECLS extern mach_port_name_t mach_reply_port(void); +extern mach_port_name_t thread_get_special_reply_port(void); + extern mach_port_name_t thread_self_trap(void); extern mach_port_name_t host_self_trap(void); @@ -344,8 +346,12 @@ extern kern_return_t pid_for_task( #else #define PAD_(t) (sizeof(uint32_t) <= sizeof(t) \ ? 0 : sizeof(uint32_t) - sizeof(t)) +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +#define PAD_ARG_8 +#else #define PAD_ARG_8 char arg8_pad_[sizeof(uint32_t)]; #endif +#endif #if BYTE_ORDER == LITTLE_ENDIAN #define PADL_(t) 0 @@ -377,6 +383,12 @@ struct mach_reply_port_args { extern mach_port_name_t mach_reply_port( struct mach_reply_port_args *args); +struct thread_get_special_reply_port_args { + int32_t dummy; +}; +extern mach_port_name_t thread_get_special_reply_port( + struct thread_get_special_reply_port_args *args); + struct thread_self_trap_args { int32_t dummy; }; diff --git a/osfmk/mach/mach_types.defs b/osfmk/mach/mach_types.defs index 465fe8f0e..dc6b2e47d 100644 --- a/osfmk/mach/mach_types.defs +++ b/osfmk/mach/mach_types.defs @@ -269,6 +269,9 @@ type task_purgable_info_t = struct[68] of integer_t; type task_policy_flavor_t = natural_t; type task_policy_t = array[*:16] of integer_t; +type task_inspect_flavor_t = natural_t; +type task_inspect_info_t = array[*:4] of integer_t; + type mem_entry_name_port_t = mach_port_t #if KERNEL_SERVER intran: mem_entry_name_port_t null_conversion(mach_port_t) @@ -584,9 +587,6 @@ type mach_voucher_attr_value_handle_t = uint64_t; type mach_voucher_attr_value_handle_array_t = array[*:4] of mach_voucher_attr_value_handle_t; type mach_voucher_attr_value_reference_t = uint32_t; -type task_inspect_flavor_t = natural_t; -type task_inspect_data_t = array[] of char; - /* kernel module loader */ type kmod_t = int; type kmod_control_flavor_t = int; diff --git a/osfmk/mach/mach_types.h b/osfmk/mach/mach_types.h index d7b745d27..eab35b14c 100644 --- a/osfmk/mach/mach_types.h +++ b/osfmk/mach/mach_types.h @@ -89,6 +89,7 @@ #include #include #include +#include #include #include #include diff --git a/osfmk/mach/mach_vm.defs b/osfmk/mach/mach_vm.defs index e0c7828e6..88097761f 100644 --- a/osfmk/mach/mach_vm.defs +++ b/osfmk/mach/mach_vm.defs @@ -76,12 +76,19 @@ subsystem #include #include +#define CONCAT(a,b) a ## b #if !KERNEL && !LIBSYSCALL_INTERFACE -#define PREFIX(NAME) _kernelrpc_ ## NAME +#define PREFIX(NAME) CONCAT(_kernelrpc_, NAME) #else #define PREFIX(NAME) NAME #endif +#if KERNEL_SERVER +#define KERNEL_SERVER_SUFFIX(NAME) CONCAT(NAME, _external) +#else +#define KERNEL_SERVER_SUFFIX(NAME) NAME +#endif + /* * Allocate zero-filled memory in the address space * of the target task, either at the specified address, @@ -90,7 +97,7 @@ subsystem * allocation actually took place is returned. */ #if !defined(_MACH_VM_PUBLISH_AS_LOCAL_) -routine PREFIX(mach_vm_allocate) ( +routine PREFIX(KERNEL_SERVER_SUFFIX(mach_vm_allocate)) ( target : vm_task_entry_t; inout address : mach_vm_address_t; size : mach_vm_size_t; @@ -101,7 +108,7 @@ routine PREFIX(mach_vm_allocate) ( #if !KERNEL && !LIBSYSCALL_INTERFACE skip; #else -routine PREFIX(vm_allocate) ( +routine PREFIX(KERNEL_SERVER_SUFFIX(vm_allocate)) ( target : vm_task_entry_t; inout address : mach_vm_address_t; size : mach_vm_size_t; @@ -333,12 +340,12 @@ routine vm_behavior_set( * for further consistency.] */ #if !defined(_MACH_VM_PUBLISH_AS_LOCAL_) -routine PREFIX(mach_vm_map) ( +routine PREFIX(KERNEL_SERVER_SUFFIX(mach_vm_map)) ( #else #if defined(__arm__) && !LIBSYSCALL_INTERFACE routine _vm_map_arm( #else -routine PREFIX(vm_map) ( +routine PREFIX(KERNEL_SERVER_SUFFIX(vm_map)) ( #endif #endif target_task : vm_task_entry_t; @@ -373,9 +380,9 @@ routine vm_machine_attribute( * Map portion of a task's address space. */ #if !defined(_MACH_VM_PUBLISH_AS_LOCAL_) -routine PREFIX(mach_vm_remap) ( +routine PREFIX(KERNEL_SERVER_SUFFIX(mach_vm_remap)) ( #else -routine PREFIX(vm_remap) ( +routine PREFIX(KERNEL_SERVER_SUFFIX(vm_remap)) ( #endif target_task : vm_map_t; inout target_address : mach_vm_address_t; @@ -492,6 +499,17 @@ routine mach_vm_page_info( skip; #endif +#if !defined(_MACH_VM_PUBLISH_AS_LOCAL_) +routine mach_vm_page_range_query( + target_map : vm_map_t; + address : mach_vm_offset_t; + size : mach_vm_size_t; + dispositions : mach_vm_address_t; + inout dispositions_count : mach_vm_size_t); +#else +skip; +#endif + /****************************** Legacy section ***************************/ /* The following definitions are exist to provide compatibility with */ /* the legacy APIs. They are no different. We just need to produce */ diff --git a/osfmk/mach/machine.h b/osfmk/mach/machine.h index c057fb799..87c1ba64f 100644 --- a/osfmk/mach/machine.h +++ b/osfmk/mach/machine.h @@ -1,5 +1,6 @@ /* - * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2007-2016 Apple, Inc. All rights reserved. + * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -393,6 +394,7 @@ __END_DECLS #define CPUFAMILY_INTEL_HASWELL 0x10b282dc #define CPUFAMILY_INTEL_BROADWELL 0x582ed09c #define CPUFAMILY_INTEL_SKYLAKE 0x37fc219f +#define CPUFAMILY_INTEL_KABYLAKE 0x0f817246 #define CPUFAMILY_ARM_9 0xe73283ae #define CPUFAMILY_ARM_11 0x8ff620d8 #define CPUFAMILY_ARM_XSCALE 0x53b005f5 diff --git a/osfmk/mach/machine/Makefile b/osfmk/mach/machine/Makefile index 5034c34d0..5c868ea6d 100644 --- a/osfmk/mach/machine/Makefile +++ b/osfmk/mach/machine/Makefile @@ -10,7 +10,7 @@ DATAFILES = \ asm.h boolean.h exception.h kern_return.h ndr_def.h rpc.h \ processor_info.h thread_state.h thread_status.h \ vm_param.h vm_types.h machine_types.defs \ - syscall_sw.h sdt.h sdt_isa.h + syscall_sw.h sdt.h sdt_isa.h _structs.h PRIVATE_DATAFILES = \ syscall_sw.h diff --git a/osfmk/mach/machine/_structs.h b/osfmk/mach/machine/_structs.h new file mode 100644 index 000000000..89aa41f91 --- /dev/null +++ b/osfmk/mach/machine/_structs.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _MACH_MACHINE__STRUCTS_H_ +#define _MACH_MACHINE__STRUCTS_H_ + +#if defined (__i386__) || defined(__x86_64__) +#include "mach/i386/_structs.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/_structs.h" +#else +#error architecture not supported +#endif + +#endif /* _MACH_MACHINE__STRUCTS_H_ */ diff --git a/osfmk/mach/machine/asm.h b/osfmk/mach/machine/asm.h index 1cdbb8109..44d69261f 100644 --- a/osfmk/mach/machine/asm.h +++ b/osfmk/mach/machine/asm.h @@ -31,6 +31,10 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/asm.h" +#elif defined (__arm__) +#include "mach/arm/asm.h" +#elif defined(__arm64__) +#include "mach/arm64/asm.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/boolean.h b/osfmk/mach/machine/boolean.h index 521033b72..7df6d4f68 100644 --- a/osfmk/mach/machine/boolean.h +++ b/osfmk/mach/machine/boolean.h @@ -31,6 +31,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/boolean.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/boolean.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/exception.h b/osfmk/mach/machine/exception.h index 5fce0e919..c0e76acfe 100644 --- a/osfmk/mach/machine/exception.h +++ b/osfmk/mach/machine/exception.h @@ -31,6 +31,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/exception.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/exception.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/kern_return.h b/osfmk/mach/machine/kern_return.h index e2b5bc677..a948cf4b4 100644 --- a/osfmk/mach/machine/kern_return.h +++ b/osfmk/mach/machine/kern_return.h @@ -31,6 +31,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/kern_return.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/kern_return.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/ndr_def.h b/osfmk/mach/machine/ndr_def.h index 2d3451472..15c17d31a 100644 --- a/osfmk/mach/machine/ndr_def.h +++ b/osfmk/mach/machine/ndr_def.h @@ -31,6 +31,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/ndr_def.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/ndr_def.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/processor_info.h b/osfmk/mach/machine/processor_info.h index c7ddb5b01..237fad686 100644 --- a/osfmk/mach/machine/processor_info.h +++ b/osfmk/mach/machine/processor_info.h @@ -31,6 +31,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/processor_info.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/processor_info.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/rpc.h b/osfmk/mach/machine/rpc.h index 3e543a88f..fc4ccbad9 100644 --- a/osfmk/mach/machine/rpc.h +++ b/osfmk/mach/machine/rpc.h @@ -31,6 +31,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/rpc.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/rpc.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/sdt_isa.h b/osfmk/mach/machine/sdt_isa.h index edd26dcc2..7145e43d1 100644 --- a/osfmk/mach/machine/sdt_isa.h +++ b/osfmk/mach/machine/sdt_isa.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include +#elif defined (__arm__) || defined (__arm64__) +#include #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/syscall_sw.h b/osfmk/mach/machine/syscall_sw.h index 902b6815e..bc364108a 100644 --- a/osfmk/mach/machine/syscall_sw.h +++ b/osfmk/mach/machine/syscall_sw.h @@ -33,6 +33,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/syscall_sw.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/syscall_sw.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/thread_state.h b/osfmk/mach/machine/thread_state.h index 061477698..aca71548f 100644 --- a/osfmk/mach/machine/thread_state.h +++ b/osfmk/mach/machine/thread_state.h @@ -31,6 +31,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/thread_state.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/thread_state.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/thread_status.h b/osfmk/mach/machine/thread_status.h index 74cda9596..9ce08e045 100644 --- a/osfmk/mach/machine/thread_status.h +++ b/osfmk/mach/machine/thread_status.h @@ -31,6 +31,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/thread_status.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/thread_status.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/vm_param.h b/osfmk/mach/machine/vm_param.h index 5898fdba1..30f94df85 100644 --- a/osfmk/mach/machine/vm_param.h +++ b/osfmk/mach/machine/vm_param.h @@ -31,6 +31,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/vm_param.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/vm_param.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/machine/vm_types.h b/osfmk/mach/machine/vm_types.h index 2b7526570..68be87969 100644 --- a/osfmk/mach/machine/vm_types.h +++ b/osfmk/mach/machine/vm_types.h @@ -31,6 +31,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "mach/i386/vm_types.h" +#elif defined (__arm__) || defined (__arm64__) +#include "mach/arm/vm_types.h" #else #error architecture not supported #endif diff --git a/osfmk/mach/memory_object_control.defs b/osfmk/mach/memory_object_control.defs index e13d85b0e..0ba43bee3 100644 --- a/osfmk/mach/memory_object_control.defs +++ b/osfmk/mach/memory_object_control.defs @@ -147,7 +147,8 @@ routine memory_object_upl_request( in size : upl_size_t; out upl : upl_t; out page_list : upl_page_info_array_t, CountInOut; - in cntrl_flags : integer_t); + in cntrl_flags : integer_t; + in tag : integer_t); routine memory_object_super_upl_request( memory_control : memory_object_control_t; @@ -156,7 +157,8 @@ routine memory_object_super_upl_request( in super_size : upl_size_t; out upl : upl_t; out page_list : upl_page_info_array_t, CountInOut; - in cntrl_flags : integer_t); + in cntrl_flags : integer_t; + in tag : integer_t); routine memory_object_cluster_size( diff --git a/osfmk/mach/memory_object_types.h b/osfmk/mach/memory_object_types.h index 433dda23d..399657d05 100644 --- a/osfmk/mach/memory_object_types.h +++ b/osfmk/mach/memory_object_types.h @@ -95,22 +95,27 @@ typedef unsigned long long vm_object_id_t; #ifdef KERNEL_PRIVATE +/* IMPORTANT: this type must match "ipc_object_bits_t" from ipc/ipc_port.h */ +typedef natural_t mo_ipc_object_bits_t; + struct memory_object_pager_ops; /* forward declaration */ +/* + * "memory_object" and "memory_object_control" types used to be Mach ports + * in user space and can be passed as such to some kernel APIs. + * Their first field must match the "io_bits" field of a + * "struct ipc_object" to identify them as a "IKOT_MEMORY_OBJECT" and + * "IKOT_MEM_OBJ_CONTROL" respectively. + */ typedef struct memory_object { - unsigned int _pad1; /* struct ipc_object_header */ -#ifdef __LP64__ - unsigned int _pad2; /* pad to natural boundary */ -#endif + mo_ipc_object_bits_t mo_ikot; /* DO NOT CHANGE */ const struct memory_object_pager_ops *mo_pager_ops; + struct memory_object_control *mo_control; } *memory_object_t; typedef struct memory_object_control { - unsigned int moc_ikot; /* struct ipc_object_header */ -#ifdef __LP64__ - unsigned int _pad; /* pad to natural boundary */ -#endif - struct vm_object *moc_object; + mo_ipc_object_bits_t moc_ikot; /* DO NOT CHANGE */ + struct vm_object *moc_object; } *memory_object_control_t; typedef const struct memory_object_pager_ops { @@ -371,6 +376,7 @@ typedef struct memory_object_attr_info memory_object_attr_info_data_t; #define MAP_MEM_WCOMB 4 /* Write combining mode */ /* aka store gather */ #define MAP_MEM_INNERWBACK 5 +#define MAP_MEM_POSTED 6 #define GET_MAP_MEM(flags) \ ((((unsigned int)(flags)) >> 24) & 0xFF) @@ -379,7 +385,8 @@ typedef struct memory_object_attr_info memory_object_attr_info_data_t; ((flags) = ((((unsigned int)(caching)) << 24) \ & 0xFF000000) | ((flags) & 0xFFFFFF)); -/* leave room for vm_prot bits */ +/* leave room for vm_prot bits (0xFF ?) */ +#define MAP_MEM_PURGABLE_KERNEL_ONLY 0x004000 /* volatility controlled by kernel */ #define MAP_MEM_GRAB_SECLUDED 0x008000 /* can grab secluded pages */ #define MAP_MEM_ONLY 0x010000 /* change processor caching */ #define MAP_MEM_NAMED_CREATE 0x020000 /* create extant object */ @@ -390,6 +397,19 @@ typedef struct memory_object_attr_info memory_object_attr_info_data_t; #define MAP_MEM_VM_SHARE 0x400000 /* extract a VM range for remap */ #define MAP_MEM_4K_DATA_ADDR 0x800000 /* preserve 4K aligned address of data */ +#define MAP_MEM_FLAGS_MASK 0x00FFFF00 +#define MAP_MEM_FLAGS_USER ( \ + MAP_MEM_PURGABLE_KERNEL_ONLY | \ + MAP_MEM_GRAB_SECLUDED | \ + MAP_MEM_ONLY | \ + MAP_MEM_NAMED_CREATE | \ + MAP_MEM_PURGABLE | \ + MAP_MEM_NAMED_REUSE | \ + MAP_MEM_USE_DATA_ADDR | \ + MAP_MEM_VM_COPY | \ + MAP_MEM_VM_SHARE | \ + MAP_MEM_4K_DATA_ADDR) + #ifdef KERNEL /* @@ -404,8 +424,10 @@ typedef struct memory_object_attr_info memory_object_attr_info_data_t; #define MAX_UPL_TRANSFER_BYTES (1024 * 1024) #define MAX_UPL_SIZE_BYTES (1024 * 1024 * 64) +#ifndef CONFIG_EMBEDDED #define MAX_UPL_SIZE (MAX_UPL_SIZE_BYTES / PAGE_SIZE) #define MAX_UPL_TRANSFER (MAX_UPL_TRANSFER_BYTES / PAGE_SIZE) +#endif struct upl_page_info { @@ -486,11 +508,6 @@ typedef uint64_t upl_control_flags_t; #define UPL_NOZEROFILLIO 0x40000000ULL /* allow non zerofill pages present */ #define UPL_REQUEST_FORCE_COHERENCY 0x80000000ULL -#define UPL_MEMORY_TAG_MASK 0xFF00000000ULL -#define UPL_MEMORY_TAG_SHIFT 32 -#define UPL_MEMORY_TAG(x) (((x) >> UPL_MEMORY_TAG_SHIFT) & 0xFF) -#define UPL_MEMORY_TAG_MAKE(x) (((upl_control_flags_t)((x) & 0xFF)) << UPL_MEMORY_TAG_SHIFT) - /* UPL flags known by this kernel */ #define UPL_VALID_FLAGS 0xFFFFFFFFFFULL diff --git a/osfmk/mach/message.h b/osfmk/mach/message.h index 74fe65b24..13481e1bd 100644 --- a/osfmk/mach/message.h +++ b/osfmk/mach/message.h @@ -496,6 +496,27 @@ typedef struct mach_port_context_t msgh_context; } mach_msg_context_trailer_t; +#if defined(MACH_KERNEL_PRIVATE) && defined(__arm64__) +typedef struct +{ + mach_msg_trailer_type_t msgh_trailer_type; + mach_msg_trailer_size_t msgh_trailer_size; + mach_port_seqno_t msgh_seqno; + security_token_t msgh_sender; + audit_token_t msgh_audit; + mach_port_context32_t msgh_context; +} mach_msg_context_trailer32_t; + +typedef struct +{ + mach_msg_trailer_type_t msgh_trailer_type; + mach_msg_trailer_size_t msgh_trailer_size; + mach_port_seqno_t msgh_seqno; + security_token_t msgh_sender; + audit_token_t msgh_audit; + mach_port_context64_t msgh_context; +} mach_msg_context_trailer64_t; +#endif typedef struct @@ -520,6 +541,32 @@ typedef struct msg_labels_t msgh_labels; } mach_msg_mac_trailer_t; +#if defined(MACH_KERNEL_PRIVATE) && defined(__arm64__) +typedef struct +{ + mach_msg_trailer_type_t msgh_trailer_type; + mach_msg_trailer_size_t msgh_trailer_size; + mach_port_seqno_t msgh_seqno; + security_token_t msgh_sender; + audit_token_t msgh_audit; + mach_port_context32_t msgh_context; + int msgh_ad; + msg_labels_t msgh_labels; +} mach_msg_mac_trailer32_t; + +typedef struct +{ + mach_msg_trailer_type_t msgh_trailer_type; + mach_msg_trailer_size_t msgh_trailer_size; + mach_port_seqno_t msgh_seqno; + security_token_t msgh_sender; + audit_token_t msgh_audit; + mach_port_context64_t msgh_context; + int msgh_ad; + msg_labels_t msgh_labels; +} mach_msg_mac_trailer64_t; + +#endif #define MACH_MSG_TRAILER_MINIMUM_SIZE sizeof(mach_msg_trailer_t) @@ -532,6 +579,10 @@ typedef struct * another module may exceed the local modules notion of * MAX_TRAILER_SIZE. */ +#if defined(MACH_KERNEL_PRIVATE) && defined(__arm64__) +typedef mach_msg_mac_trailer64_t mach_msg_max_trailer64_t; +typedef mach_msg_mac_trailer32_t mach_msg_max_trailer32_t; +#endif typedef mach_msg_mac_trailer_t mach_msg_max_trailer_t; #define MAX_TRAILER_SIZE ((mach_msg_size_t)sizeof(mach_msg_max_trailer_t)) @@ -669,6 +720,7 @@ typedef integer_t mach_msg_option_t; #define MACH_SEND_NOIMPORTANCE 0x00040000 /* msg won't carry importance */ #define MACH_SEND_NODENAP MACH_SEND_NOIMPORTANCE #define MACH_SEND_IMPORTANCE 0x00080000 /* msg carries importance - kernel only */ +#define MACH_SEND_SYNC_OVERRIDE 0x00100000 /* msg should do sync ipc override */ #define MACH_RCV_TIMEOUT 0x00000100 /* timeout value applies to receive */ @@ -676,6 +728,7 @@ typedef integer_t mach_msg_option_t; #define MACH_RCV_INTERRUPT 0x00000400 /* don't restart interrupted receive */ #define MACH_RCV_VOUCHER 0x00000800 /* willing to receive voucher port */ #define MACH_RCV_OVERWRITE 0x00001000 /* scatter receive (deprecated) */ +#define MACH_RCV_SYNC_WAIT 0x00004000 /* sync waiter waiting for rcv */ #ifdef XNU_KERNEL_PRIVATE @@ -718,11 +771,13 @@ typedef integer_t mach_msg_option_t; /* The options that the kernel honors when passed from user space */ #define MACH_SEND_USER (MACH_SEND_MSG | MACH_SEND_TIMEOUT | \ MACH_SEND_NOTIFY | MACH_SEND_OVERRIDE | \ - MACH_SEND_TRAILER | MACH_SEND_NOIMPORTANCE ) + MACH_SEND_TRAILER | MACH_SEND_NOIMPORTANCE | \ + MACH_SEND_SYNC_OVERRIDE) #define MACH_RCV_USER (MACH_RCV_MSG | MACH_RCV_TIMEOUT | \ MACH_RCV_LARGE | MACH_RCV_LARGE_IDENTITY | \ - MACH_RCV_VOUCHER | MACH_RCV_TRAILER_MASK) + MACH_RCV_VOUCHER | MACH_RCV_TRAILER_MASK | \ + MACH_RCV_SYNC_WAIT) #define MACH_MSG_OPTION_USER (MACH_SEND_USER | MACH_RCV_USER) @@ -770,7 +825,25 @@ typedef integer_t mach_msg_option_t; #ifdef XNU_KERNEL_PRIVATE +#if defined(__arm64__) +#define REQUESTED_TRAILER_SIZE(is64, y) \ + ((mach_msg_trailer_size_t) \ + ((GET_RCV_ELEMENTS(y) == MACH_RCV_TRAILER_NULL) ? \ + sizeof(mach_msg_trailer_t) : \ + ((GET_RCV_ELEMENTS(y) == MACH_RCV_TRAILER_SEQNO) ? \ + sizeof(mach_msg_seqno_trailer_t) : \ + ((GET_RCV_ELEMENTS(y) == MACH_RCV_TRAILER_SENDER) ? \ + sizeof(mach_msg_security_trailer_t) : \ + ((GET_RCV_ELEMENTS(y) == MACH_RCV_TRAILER_AUDIT) ? \ + sizeof(mach_msg_audit_trailer_t) : \ + ((GET_RCV_ELEMENTS(y) == MACH_RCV_TRAILER_CTX) ? \ + ((is64) ? sizeof(mach_msg_context_trailer64_t) : sizeof(mach_msg_context_trailer32_t)) : \ + ((GET_RCV_ELEMENTS(y) == MACH_RCV_TRAILER_AV) ? \ + ((is64) ? sizeof(mach_msg_mac_trailer64_t) : sizeof(mach_msg_mac_trailer32_t)) : \ + sizeof(mach_msg_max_trailer_t)))))))) +#else #define REQUESTED_TRAILER_SIZE(is64, y) REQUESTED_TRAILER_SIZE_NATIVE(y) +#endif #else /* XNU_KERNEL_PRIVATE */ #define REQUESTED_TRAILER_SIZE(y) REQUESTED_TRAILER_SIZE_NATIVE(y) diff --git a/osfmk/mach/mig.h b/osfmk/mach/mig.h index f6bf29a70..44a208dd6 100644 --- a/osfmk/mach/mig.h +++ b/osfmk/mach/mig.h @@ -278,7 +278,7 @@ extern int mig_strncpy_zerofill(char *dest, const char *src, int len); #ifdef KERNEL_PRIVATE /* Allocate memory for out-of-stack mig structures */ -extern char *mig_user_allocate(vm_size_t size); +extern void *mig_user_allocate(vm_size_t size); /* Deallocate memory used for out-of-stack mig structures */ extern void mig_user_deallocate(char *data, vm_size_t size); diff --git a/osfmk/mach/mig_strncpy_zerofill_support.h b/osfmk/mach/mig_strncpy_zerofill_support.h index 47cdc6159..92d0ff8e4 100644 --- a/osfmk/mach/mig_strncpy_zerofill_support.h +++ b/osfmk/mach/mig_strncpy_zerofill_support.h @@ -1,3 +1,30 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ //This dummy header file is created for mig to check when to call mig_strncpy_zerofill. //Mig checks if this file is available to include and knows that Libsyscall has the new mig_strncpy_zerofill symbols to link to. //Do not delete this file, mig will stop calling mig_strncpy_zerofill. diff --git a/osfmk/mach/mig_voucher_support.h b/osfmk/mach/mig_voucher_support.h index 064755ab6..27e6b601c 100644 --- a/osfmk/mach/mig_voucher_support.h +++ b/osfmk/mach/mig_voucher_support.h @@ -1,3 +1,30 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ //This dummy header file is created for mig to check when to include voucher code. //Mig checks if this file is available to include and knows that Libsyscall has the new voucher symbols to link to. //Do not delete this file, mig will stop including vouchers code. diff --git a/osfmk/mach/port.h b/osfmk/mach/port.h index 49c248d6a..30b55a1b0 100644 --- a/osfmk/mach/port.h +++ b/osfmk/mach/port.h @@ -343,8 +343,8 @@ typedef integer_t *mach_port_info_t; /* varying array of natural_t */ /* Flavors for mach_port_get/set_attributes() */ typedef int mach_port_flavor_t; -#define MACH_PORT_LIMITS_INFO 1 /* uses mach_port_status_t */ -#define MACH_PORT_RECEIVE_STATUS 2 /* uses mach_port_limits_t */ +#define MACH_PORT_LIMITS_INFO 1 /* uses mach_port_limits_t */ +#define MACH_PORT_RECEIVE_STATUS 2 /* uses mach_port_status_t */ #define MACH_PORT_DNREQUESTS_SIZE 3 /* info is int */ #define MACH_PORT_TEMPOWNER 4 /* indicates receive right will be reassigned to another task */ #define MACH_PORT_IMPORTANCE_RECEIVER 5 /* indicates recieve right accepts priority donation */ diff --git a/osfmk/mach/shared_memory_server.h b/osfmk/mach/shared_memory_server.h index ea7b9127f..bd792319c 100644 --- a/osfmk/mach/shared_memory_server.h +++ b/osfmk/mach/shared_memory_server.h @@ -55,12 +55,21 @@ #define VM_PROT_COW 0x8 /* must not interfere with normal prot assignments */ #define VM_PROT_ZF 0x10 /* must not interfere with normal prot assignments */ +#ifdef __arm__ +#define GLOBAL_SHARED_TEXT_SEGMENT 0x30000000U +#define GLOBAL_SHARED_DATA_SEGMENT 0x38000000U +#define GLOBAL_SHARED_SEGMENT_MASK 0xF8000000U + +#define SHARED_TEXT_REGION_SIZE 0x08000000 +#define SHARED_DATA_REGION_SIZE 0x08000000 +#else #define GLOBAL_SHARED_TEXT_SEGMENT 0x90000000U #define GLOBAL_SHARED_DATA_SEGMENT 0xA0000000U #define GLOBAL_SHARED_SEGMENT_MASK 0xF0000000U #define SHARED_TEXT_REGION_SIZE 0x10000000 #define SHARED_DATA_REGION_SIZE 0x10000000 +#endif #if !defined(__LP64__) @@ -74,8 +83,13 @@ * i.e. if the size is 0x10000000 the object can be mapped at * 0x20000000, or 0x30000000, but not 0x1000000 */ +#ifdef __arm__ +#define SHARED_TEXT_REGION_MASK 0x07FFFFFF +#define SHARED_DATA_REGION_MASK 0x07FFFFFF +#else #define SHARED_TEXT_REGION_MASK 0x0FFFFFFF #define SHARED_DATA_REGION_MASK 0x0FFFFFFF +#endif /* flags field aliases for copyin_shared_file and load_shared_file */ diff --git a/osfmk/mach/shared_region.h b/osfmk/mach/shared_region.h index e460db001..19351b07f 100644 --- a/osfmk/mach/shared_region.h +++ b/osfmk/mach/shared_region.h @@ -42,10 +42,10 @@ #define SHARED_REGION_NESTING_MIN_I386 0x00200000ULL #define SHARED_REGION_NESTING_MAX_I386 0xFFE00000ULL -#define SHARED_REGION_BASE_X86_64 0x00007FFF70000000ULL -#define SHARED_REGION_SIZE_X86_64 0x000000008FE00000ULL -#define SHARED_REGION_NESTING_BASE_X86_64 0x00007FFF70000000ULL -#define SHARED_REGION_NESTING_SIZE_X86_64 0x000000008FE00000ULL +#define SHARED_REGION_BASE_X86_64 0x00007FFF00000000ULL +#define SHARED_REGION_SIZE_X86_64 0x00000000FFE00000ULL +#define SHARED_REGION_NESTING_BASE_X86_64 0x00007FFF00000000ULL +#define SHARED_REGION_NESTING_SIZE_X86_64 0x00000000FFE00000ULL #define SHARED_REGION_NESTING_MIN_X86_64 0x0000000000200000ULL #define SHARED_REGION_NESTING_MAX_X86_64 0xFFFFFFFFFFE00000ULL @@ -94,6 +94,20 @@ #define SHARED_REGION_NESTING_SIZE SHARED_REGION_NESTING_SIZE_X86_64 #define SHARED_REGION_NESTING_MIN SHARED_REGION_NESTING_MIN_X86_64 #define SHARED_REGION_NESTING_MAX SHARED_REGION_NESTING_MAX_X86_64 +#elif defined(__arm__) +#define SHARED_REGION_BASE SHARED_REGION_BASE_ARM +#define SHARED_REGION_SIZE SHARED_REGION_SIZE_ARM +#define SHARED_REGION_NESTING_BASE SHARED_REGION_NESTING_BASE_ARM +#define SHARED_REGION_NESTING_SIZE SHARED_REGION_NESTING_SIZE_ARM +#define SHARED_REGION_NESTING_MIN SHARED_REGION_NESTING_MIN_ARM +#define SHARED_REGION_NESTING_MAX SHARED_REGION_NESTING_MAX_ARM +#elif defined(__arm64__) +#define SHARED_REGION_BASE SHARED_REGION_BASE_ARM64 +#define SHARED_REGION_SIZE SHARED_REGION_SIZE_ARM64 +#define SHARED_REGION_NESTING_BASE SHARED_REGION_NESTING_BASE_ARM64 +#define SHARED_REGION_NESTING_SIZE SHARED_REGION_NESTING_SIZE_ARM64 +#define SHARED_REGION_NESTING_MIN SHARED_REGION_NESTING_MIN_ARM64 +#define SHARED_REGION_NESTING_MAX SHARED_REGION_NESTING_MAX_ARM64 #endif #ifdef KERNEL_PRIVATE diff --git a/osfmk/mach/syscall_sw.h b/osfmk/mach/syscall_sw.h index 97edad81c..381bfc510 100644 --- a/osfmk/mach/syscall_sw.h +++ b/osfmk/mach/syscall_sw.h @@ -129,6 +129,7 @@ kernel_trap(macx_swapoff,-49, 2) kernel_trap(macx_swapon,-48, 5) kernel_trap(macx_swapoff,-49, 3) #endif /* __LP64__ */ +kernel_trap(thread_get_special_reply_port,-50,0) kernel_trap(macx_triggers,-51, 4) kernel_trap(macx_backing_store_suspend,-52, 1) kernel_trap(macx_backing_store_recovery,-53, 1) diff --git a/osfmk/mach/task.defs b/osfmk/mach/task.defs index a60622e5b..5ac64e7d5 100644 --- a/osfmk/mach/task.defs +++ b/osfmk/mach/task.defs @@ -118,11 +118,18 @@ routine mach_ports_lookup( /* * Returns information about the target task. */ +#ifdef KERNEL_SERVER +routine task_info_from_user( + target_task : mach_port_t; + flavor : task_flavor_t; + out task_info_out : task_info_t, CountInOut); +#else routine task_info( target_task : task_name_t; flavor : task_flavor_t; out task_info_out : task_info_t, CountInOut); +#endif /* * Set task information. */ @@ -492,5 +499,10 @@ routine task_map_corpse_info_64( out kcd_addr_begin :mach_vm_address_t; out kcd_size :mach_vm_size_t); +routine task_inspect( + task : task_inspect_t; + flavor : task_inspect_flavor_t; + out info_out : task_inspect_info_t, CountInOut); + /* vim: set ft=c : */ diff --git a/osfmk/mach/task_info.h b/osfmk/mach/task_info.h index ebb290871..b06cf4170 100644 --- a/osfmk/mach/task_info.h +++ b/osfmk/mach/task_info.h @@ -112,8 +112,19 @@ typedef struct task_basic_info_32 *task_basic_info_32_t; /* Don't use this, use MACH_TASK_BASIC_INFO instead */ struct task_basic_info_64 { integer_t suspend_count; /* suspend count for task */ +#if defined(__arm__) || defined(__arm64__) +#if defined(KERNEL) + /* Compatibility with old 32-bit mach_vm_size_t */ + natural_t virtual_size; /* virtual memory size (bytes) */ + natural_t resident_size; /* resident memory size (bytes) */ +#else mach_vm_size_t virtual_size; /* virtual memory size (bytes) */ mach_vm_size_t resident_size; /* resident memory size (bytes) */ +#endif +#else /* defined(__arm__) || defined(__arm64__) */ + mach_vm_size_t virtual_size; /* virtual memory size (bytes) */ + mach_vm_size_t resident_size; /* resident memory size (bytes) */ +#endif /* defined(__arm__) || defined(__arm64__) */ time_value_t user_time; /* total user run time for terminated threads */ time_value_t system_time; /* total system run time for @@ -123,9 +134,39 @@ struct task_basic_info_64 { typedef struct task_basic_info_64 task_basic_info_64_data_t; typedef struct task_basic_info_64 *task_basic_info_64_t; +#if defined(__arm__) || defined(__arm64__) + #if defined(KERNEL) + /* + * Backwards-compatibility for old mach_vm*_t types. + * The kernel knows about old and new, and if you are compiled + * to run on an earlier iOS version, you interact with the old + * (narrow) version. If you are compiled for a newer OS + * version, however, you are mapped to the wide version. + */ + + #define TASK_BASIC_INFO_64 5 + #define TASK_BASIC_INFO_64_COUNT \ + (sizeof(task_basic_info_64_data_t) / sizeof(natural_t)) + + #elif defined(__arm__) && defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && (__IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_5_0) + /* + * Note: arm64 can't use the old flavor. If you somehow manage to, + * you can cope with the nonsense data yourself. + */ + #define TASK_BASIC_INFO_64 5 + #define TASK_BASIC_INFO_64_COUNT \ + (sizeof(task_basic_info_64_data_t) / sizeof(natural_t)) + + #else + + #define TASK_BASIC_INFO_64 TASK_BASIC_INFO_64_2 + #define TASK_BASIC_INFO_64_COUNT TASK_BASIC_INFO_64_2_COUNT + #endif +#else /* defined(__arm__) || defined(__arm64__) */ #define TASK_BASIC_INFO_64 5 /* 64-bit capable basic info */ #define TASK_BASIC_INFO_64_COUNT \ (sizeof(task_basic_info_64_data_t) / sizeof(natural_t)) +#endif /* localized structure - cannot be safely passed between tasks of differing sizes */ @@ -249,6 +290,27 @@ typedef struct task_dyld_info *task_dyld_info_t; #define TASK_DYLD_ALL_IMAGE_INFO_32 0 /* format value */ #define TASK_DYLD_ALL_IMAGE_INFO_64 1 /* format value */ +#if defined(__arm__) || defined(__arm64__) + +/* Don't use this, use MACH_TASK_BASIC_INFO instead */ +/* Compatibility for old 32-bit mach_vm_*_t */ +#define TASK_BASIC_INFO_64_2 18 /* 64-bit capable basic info */ + +struct task_basic_info_64_2 { + integer_t suspend_count; /* suspend count for task */ + mach_vm_size_t virtual_size; /* virtual memory size (bytes) */ + mach_vm_size_t resident_size; /* resident memory size (bytes) */ + time_value_t user_time; /* total user run time for + terminated threads */ + time_value_t system_time; /* total system run time for + terminated threads */ + policy_t policy; /* default policy for new threads */ +}; +typedef struct task_basic_info_64_2 task_basic_info_64_2_data_t; +typedef struct task_basic_info_64_2 *task_basic_info_64_2_t; +#define TASK_BASIC_INFO_64_2_COUNT \ + (sizeof(task_basic_info_64_2_data_t) / sizeof(natural_t)) +#endif #define TASK_EXTMOD_INFO 19 @@ -377,13 +439,19 @@ typedef gpu_energy_data *gpu_energy_data_t; struct task_power_info_v2 { task_power_info_data_t cpu_energy; gpu_energy_data gpu_energy; +#if defined(__arm__) || defined(__arm64__) + uint64_t task_energy; +#endif + uint64_t task_ptime; + uint64_t task_pset_switches; }; typedef struct task_power_info_v2 task_power_info_v2_data_t; typedef struct task_power_info_v2 *task_power_info_v2_t; -#define TASK_POWER_INFO_V2_COUNT ((mach_msg_type_number_t) \ - (sizeof (task_power_info_v2_data_t) / sizeof (natural_t))) - +#define TASK_POWER_INFO_V2_COUNT_OLD \ + ((mach_msg_type_number_t) (sizeof (task_power_info_v2_data_t) - sizeof(uint64_t)*2) / sizeof (natural_t)) +#define TASK_POWER_INFO_V2_COUNT \ + ((mach_msg_type_number_t) (sizeof (task_power_info_v2_data_t) / sizeof (natural_t))) #define TASK_VM_INFO_PURGEABLE_ACCOUNT 27 /* Used for xnu purgeable vm unit tests */ diff --git a/osfmk/mach/task_inspect.h b/osfmk/mach/task_inspect.h new file mode 100644 index 000000000..b13310f75 --- /dev/null +++ b/osfmk/mach/task_inspect.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef MACH_TASK_INSPECT_H +#define MACH_TASK_INSPECT_H + +/* + * XXX These interfaces are still in development -- they are subject to change + * without notice. + */ + +typedef natural_t task_inspect_flavor_t; + +enum task_inspect_flavor { + TASK_INSPECT_BASIC_COUNTS = 1, +}; + +struct task_inspect_basic_counts { + uint64_t instructions; + uint64_t cycles; +}; +#define TASK_INSPECT_BASIC_COUNTS_COUNT \ + (sizeof(struct task_inspect_basic_counts) / sizeof(natural_t)) +typedef struct task_inspect_basic_counts task_inspect_basic_counts_data_t; +typedef struct task_inspect_basic_counts *task_inspect_basic_counts_t; + +typedef integer_t *task_inspect_info_t; + +#endif /* !defined(MACH_TASK_INSPECT_H) */ diff --git a/osfmk/mach/thread_act.defs b/osfmk/mach/thread_act.defs index 1197cb716..716026ac5 100644 --- a/osfmk/mach/thread_act.defs +++ b/osfmk/mach/thread_act.defs @@ -325,7 +325,6 @@ routine thread_get_assignment( thread : thread_act_t; out assigned_set : processor_set_name_t); - /* * OBSOLETE interface. */ diff --git a/osfmk/mach/thread_policy.h b/osfmk/mach/thread_policy.h index 0badb50ac..915425333 100644 --- a/osfmk/mach/thread_policy.h +++ b/osfmk/mach/thread_policy.h @@ -404,8 +404,9 @@ struct thread_requested_policy { thrp_qos_promote :3, /* thread qos class from promotion */ thrp_qos_ipc_override :3, /* thread qos class from ipc override */ thrp_terminated :1, /* heading for termination */ + thrp_qos_sync_ipc_override:3, /* thread qos class from sync ipc override */ - thrp_reserved :32; + thrp_reserved :29; }; struct thread_effective_policy { diff --git a/osfmk/mach/thread_status.h b/osfmk/mach/thread_status.h index cef4380bf..dd799ec0e 100644 --- a/osfmk/mach/thread_status.h +++ b/osfmk/mach/thread_status.h @@ -88,6 +88,7 @@ typedef natural_t thread_state_data_t[THREAD_STATE_MAX]; #define THREAD_STATE_FLAVOR_LIST 0 /* List of valid flavors */ #define THREAD_STATE_FLAVOR_LIST_NEW 128 #define THREAD_STATE_FLAVOR_LIST_10_9 129 +#define THREAD_STATE_FLAVOR_LIST_10_13 130 typedef int thread_state_flavor_t; typedef thread_state_flavor_t *thread_state_flavor_array_t; diff --git a/osfmk/mach/vm_map.defs b/osfmk/mach/vm_map.defs index 54f85c7e5..c6bda2cbb 100644 --- a/osfmk/mach/vm_map.defs +++ b/osfmk/mach/vm_map.defs @@ -71,12 +71,19 @@ subsystem #include #include +#define CONCAT(a,b) a ## b #if !KERNEL && !LIBSYSCALL_INTERFACE -#define PREFIX(NAME) _kernelrpc_ ## NAME +#define PREFIX(NAME) CONCAT(_kernelrpc_, NAME) #else #define PREFIX(NAME) NAME #endif +#if KERNEL_SERVER +#define KERNEL_SERVER_SUFFIX(NAME) CONCAT(NAME, _external) +#else +#define KERNEL_SERVER_SUFFIX(NAME) NAME +#endif + /* * Returns information about the contents of the virtual * address space of the target task at the specified @@ -109,7 +116,7 @@ routine vm_region( #if !KERNEL && !LIBSYSCALL_INTERFACE skip; #else -routine PREFIX(vm_allocate)( +routine PREFIX(KERNEL_SERVER_SUFFIX(vm_allocate))( target_task : vm_task_entry_t; inout address : vm_address_t; size : vm_size_t; @@ -283,7 +290,7 @@ routine vm_behavior_set( * The user-defined memory manager for this object is responsible * for further consistency.] */ -routine PREFIX(vm_map) ( +routine PREFIX(KERNEL_SERVER_SUFFIX(vm_map)) ( target_task : vm_task_entry_t; inout address : vm_address_t; size : vm_size_t; @@ -311,7 +318,7 @@ routine vm_machine_attribute( /* * Map portion of a task's address space. */ -routine PREFIX(vm_remap) ( +routine PREFIX(KERNEL_SERVER_SUFFIX(vm_remap)) ( target_task : vm_map_t; inout target_address : vm_address_t; size : vm_size_t; @@ -442,7 +449,7 @@ routine mach_make_memory_entry_64( -routine vm_map_64( +routine KERNEL_SERVER_SUFFIX(vm_map_64)( target_task : vm_task_entry_t; inout address : vm_address_t; size : vm_size_t; diff --git a/osfmk/mach/vm_param.h b/osfmk/mach/vm_param.h index b76a10b21..96bd1f445 100644 --- a/osfmk/mach/vm_param.h +++ b/osfmk/mach/vm_param.h @@ -248,6 +248,8 @@ extern uint64_t max_mem; /* 64-bit size of memory - limited by maxmem */ #ifdef XNU_KERNEL_PRIVATE +#include + extern uint64_t mem_actual; /* 64-bit size of memory - not limited by maxmem */ extern uint64_t sane_size; /* Memory size to use for defaults calculations */ extern addr64_t vm_last_addr; /* Highest kernel virtual address known to the VM system */ @@ -280,50 +282,105 @@ extern vm_offset_t vm_hib_base; * (e.g. stackshot, proc_info syscall, etc.). It is important to understand * the goal of each macro and choose the right one depending on what you are * trying to do. Misuse of these macros can result in critical data leaks - * which in turn lead to all sorts of system vulnerabilities. - * - * Note that in general the ideal goal is to protect addresses from userspace - * in a way that is reversible assuming you know the permutation and/or slide. + * which in turn lead to all sorts of system vulnerabilities. It is invalid to + * call these macros on a non-kernel address (NULL is allowed). * - * The macros are as follows: - * * VM_KERNEL_UNSLIDE: * Use this macro when you are exposing an address to userspace which is - * a "static" kernel or kext address (i.e. coming from text or data - * sections). These are the addresses which get "slid" via ASLR on kernel - * or kext load, and it's precisely the slide value we are trying to + * *guaranteed* to be a "static" kernel or kext address (i.e. coming from text + * or data sections). These are the addresses which get "slid" via ASLR on + * kernel or kext load, and it's precisely the slide value we are trying to * protect from userspace. * - * VM_KERNEL_ADDRPERM: - * Use this macro when you are exposing an address to userspace which is - * coming from the kernel's "heap". Since these adresses are not "loaded" - * from anywhere, there is no slide applied and we instead apply the - * permutation value to obscure the address. + * VM_KERNEL_ADDRHIDE: + * Use when exposing an address for internal purposes: debugging, tracing, + * etc. The address will be unslid if necessary. Other addresses will be + * hidden on customer builds, and unmodified on internal builds. * - * VM_KERNEL_UNSLIDE_OR_ADDRPERM: - * Use this macro when you are exposing an address to userspace that could - * come from either kernel text/data *or* the heap. This is a rare case, - * but one that does come up and must be handled correctly. If the argument - * is known to be lower than any potential heap address, no transformation - * is applied, to avoid revealing the operation on a constant. + * VM_KERNEL_ADDRHASH: + * Use this macro when exposing a kernel address to userspace on customer + * builds. The address can be from the static kernel or kext regions, or the + * kernel heap. The address will be unslid or hashed as appropriate. + * + * + * ** SECURITY WARNING: The following macros can leak kernel secrets. + * Use *only* in performance *critical* code. + * + * VM_KERNEL_ADDRPERM: + * VM_KERNEL_UNSLIDE_OR_PERM: + * Use these macros when exposing a kernel address to userspace on customer + * builds. The address can be from the static kernel or kext regions, or the + * kernel heap. The address will be unslid or permuted as appropriate. * * Nesting of these macros should be considered invalid. */ -#define VM_KERNEL_UNSLIDE(_v) \ - ((VM_KERNEL_IS_SLID(_v)) ? \ - (vm_offset_t)(_v) - vm_kernel_slide : \ - (vm_offset_t)(_v)) - -#define VM_KERNEL_ADDRPERM(_v) \ - (((vm_offset_t)(_v) == 0) ? \ - (vm_offset_t)(0) : \ - (vm_offset_t)(_v) + vm_kernel_addrperm) - -#define VM_KERNEL_UNSLIDE_OR_PERM(_v) \ - ((VM_KERNEL_IS_SLID(_v)) ? \ - (vm_offset_t)(_v) - vm_kernel_slide : \ - ((vm_offset_t)(_v) >= VM_MIN_KERNEL_AND_KEXT_ADDRESS ? VM_KERNEL_ADDRPERM(_v) : (vm_offset_t)(_v))) - + +__BEGIN_DECLS +extern vm_offset_t vm_kernel_addrhash(vm_offset_t addr); +__END_DECLS + +#define __DO_UNSLIDE(_v) ((vm_offset_t)(_v) - vm_kernel_slide) + +#if DEBUG || DEVELOPMENT +# define VM_KERNEL_ADDRHIDE(_v) (VM_KERNEL_IS_SLID(_v) ? __DO_UNSLIDE(_v) : (vm_address_t)(_v)) +#else +# define VM_KERNEL_ADDRHIDE(_v) (VM_KERNEL_IS_SLID(_v) ? __DO_UNSLIDE(_v) : (vm_address_t)0) +#endif + +#define VM_KERNEL_ADDRHASH(_v) vm_kernel_addrhash((vm_offset_t)(_v)) + +#define VM_KERNEL_UNSLIDE_OR_PERM(_v) ({ \ + VM_KERNEL_IS_SLID(_v) ? __DO_UNSLIDE(_v) : \ + VM_KERNEL_ADDRESS(_v) ? ((vm_offset_t)(_v) + vm_kernel_addrperm) : \ + (vm_offset_t)(_v); \ + }) + +#define VM_KERNEL_UNSLIDE(_v) ({ \ + VM_KERNEL_IS_SLID(_v) ? __DO_UNSLIDE(_v) : (vm_offset_t)0; \ + }) + +#define VM_KERNEL_ADDRPERM(_v) VM_KERNEL_UNSLIDE_OR_PERM(_v) + +#undef mach_vm_round_page +#undef round_page +#undef round_page_32 +#undef round_page_64 + +static inline mach_vm_offset_t +mach_vm_round_page(mach_vm_offset_t x) +{ + if (round_page_overflow(x, &x)) { + panic("overflow detected"); + } + return x; +} + +static inline vm_offset_t +round_page(vm_offset_t x) +{ + if (round_page_overflow(x, &x)) { + panic("overflow detected"); + } + return x; +} + +static inline mach_vm_offset_t +round_page_64(mach_vm_offset_t x) +{ + if (round_page_overflow(x, &x)) { + panic("overflow detected"); + } + return x; +} + +static inline uint32_t +round_page_32(uint32_t x) +{ + if (round_page_overflow(x, &x)) { + panic("overflow detected"); + } + return x; +} #endif /* XNU_KERNEL_PRIVATE */ diff --git a/osfmk/mach/vm_prot.h b/osfmk/mach/vm_prot.h index 0d4d5bf3e..6998a31f0 100644 --- a/osfmk/mach/vm_prot.h +++ b/osfmk/mach/vm_prot.h @@ -157,10 +157,4 @@ typedef int vm_prot_t; #define VM_PROT_STRIP_READ ((vm_prot_t) 0x80) #define VM_PROT_EXECUTE_ONLY (VM_PROT_EXECUTE|VM_PROT_STRIP_READ) - -#define VM_PROT_MEMORY_TAG_MASK 0xFF000000 -#define VM_PROT_MEMORY_TAG_SHIFT 24 -#define VM_PROT_MEMORY_TAG(x) (((x) >> VM_PROT_MEMORY_TAG_SHIFT) & 0xFF) -#define VM_PROT_MEMORY_TAG_MAKE(x) (((x) & 0xFF) << VM_PROT_MEMORY_TAG_SHIFT) - #endif /* _MACH_VM_PROT_H_ */ diff --git a/osfmk/mach/vm_purgable.h b/osfmk/mach/vm_purgable.h index 2fefb86e6..4ae0892b8 100644 --- a/osfmk/mach/vm_purgable.h +++ b/osfmk/mach/vm_purgable.h @@ -58,6 +58,29 @@ typedef int vm_purgable_t; #define VM_PURGABLE_SET_STATE ((vm_purgable_t) 0) /* set state of purgeable object */ #define VM_PURGABLE_GET_STATE ((vm_purgable_t) 1) /* get state of purgeable object */ #define VM_PURGABLE_PURGE_ALL ((vm_purgable_t) 2) /* purge all volatile objects now */ +#define VM_PURGABLE_SET_STATE_FROM_KERNEL ((vm_purgable_t) 3) /* set state from kernel */ + +/* + * Purgeable state: + * + * 31 15 14 13 12 11 10 8 7 6 5 4 3 2 1 0 + * +-----+--+-----+--+----+-+-+---+---+---+ + * | |NA|DEBUG| | GRP| |B|ORD| |STA| + * +-----+--+-----+--+----+-+-+---+---+---+ + * " ": unused (i.e. reserved) + * STA: purgeable state + * see: VM_PURGABLE_NONVOLATILE=0 to VM_PURGABLE_DENY=3 + * ORD: order + * see:VM_VOLATILE_ORDER_* + * B: behavior + * see: VM_PURGABLE_BEHAVIOR_* + * GRP: group + * see: VM_VOLATILE_GROUP_* + * DEBUG: debug + * see: VM_PURGABLE_DEBUG_* + * NA: no aging + * see: VM_PURGABLE_NO_AGING* + */ #define VM_PURGABLE_NO_AGING_SHIFT 16 #define VM_PURGABLE_NO_AGING_MASK (0x1 << VM_PURGABLE_NO_AGING_SHIFT) diff --git a/osfmk/mach/vm_statistics.h b/osfmk/mach/vm_statistics.h index 51c99d7cc..f6ffb8b0a 100644 --- a/osfmk/mach/vm_statistics.h +++ b/osfmk/mach/vm_statistics.h @@ -290,49 +290,31 @@ typedef struct pmap_statistics *pmap_statistics_t; * queue instead of the active queue. In other words, they are not * cached so that they will be stolen first if memory runs low. */ + #define VM_FLAGS_FIXED 0x0000 #define VM_FLAGS_ANYWHERE 0x0001 #define VM_FLAGS_PURGABLE 0x0002 -#ifdef KERNEL_PRIVATE -#endif /* KERNEL_PRIVATE */ #define VM_FLAGS_RANDOM_ADDR 0x0008 #define VM_FLAGS_NO_CACHE 0x0010 #define VM_FLAGS_RESILIENT_CODESIGN 0x0020 #define VM_FLAGS_RESILIENT_MEDIA 0x0040 -#ifdef KERNEL_PRIVATE -#define VM_FLAGS_ATOMIC_ENTRY 0x0080 -#define VM_FLAGS_PERMANENT 0x0100 /* mapping can NEVER be unmapped */ -#define VM_FLAGS_GUARD_AFTER 0x0200 /* guard page after the mapping */ -#define VM_FLAGS_GUARD_BEFORE 0x0400 /* guard page before the mapping */ -#define VM_FLAGS_SUBMAP 0x0800 /* mapping a VM submap */ -#define VM_FLAGS_ALREADY 0x1000 /* OK if same mapping already exists */ -#define VM_FLAGS_BEYOND_MAX 0x2000 /* map beyond the map's max offset */ -#endif /* KERNEL_PRIVATE */ #define VM_FLAGS_OVERWRITE 0x4000 /* delete any existing mappings first */ -#ifdef KERNEL_PRIVATE -#define VM_FLAGS_NO_PMAP_CHECK 0x8000 /* do not check that pmap is empty */ -#endif /* KERNEL_PRIVATE */ /* * VM_FLAGS_SUPERPAGE_MASK * 3 bits that specify whether large pages should be used instead of * base pages (!=0), as well as the requested page size. */ #define VM_FLAGS_SUPERPAGE_MASK 0x70000 /* bits 0x10000, 0x20000, 0x40000 */ -#ifdef KERNEL_PRIVATE -#define VM_FLAGS_MAP_JIT 0x80000 /* Used to mark an entry as describing a JIT region */ -#endif /* KERNEL_PRIVATE */ #define VM_FLAGS_RETURN_DATA_ADDR 0x100000 /* Return address of target data, rather than base of page */ -#ifdef KERNEL_PRIVATE -#define VM_FLAGS_IOKIT_ACCT 0x200000 /* IOKit accounting */ -#define VM_FLAGS_KEEP_MAP_LOCKED 0x400000 /* Keep the map locked when returning from vm_map_enter() */ -#endif /* KERNEL_PRIVATE */ #define VM_FLAGS_RETURN_4K_DATA_ADDR 0x800000 /* Return 4K aligned address of target data */ #define VM_FLAGS_ALIAS_MASK 0xFF000000 #define VM_GET_FLAGS_ALIAS(flags, alias) \ (alias) = ((flags) & VM_FLAGS_ALIAS_MASK) >> 24 +#if !XNU_KERNEL_PRIVATE #define VM_SET_FLAGS_ALIAS(flags, alias) \ (flags) = (((flags) & ~VM_FLAGS_ALIAS_MASK) | \ (((alias) & ~VM_FLAGS_ALIAS_MASK) << 24)) +#endif /* !XNU_KERNEL_PRIVATE */ /* These are the flags that we accept from user-space */ #define VM_FLAGS_USER_ALLOCATE (VM_FLAGS_FIXED | \ @@ -363,6 +345,45 @@ typedef struct pmap_statistics *pmap_statistics_t; #define VM_FLAGS_SUPERPAGE_SIZE_2MB (SUPERPAGE_SIZE_2MB< /* natural_t */ + /* * Remember to update the mig type definitions * in mach_debug_types.defs when adding/removing fields. diff --git a/osfmk/mach_debug/mach_debug_types.defs b/osfmk/mach_debug/mach_debug_types.defs index 51e62a584..b4faebbbf 100644 --- a/osfmk/mach_debug/mach_debug_types.defs +++ b/osfmk/mach_debug/mach_debug_types.defs @@ -105,7 +105,7 @@ type symtab_name_t = c_string[*:32]; type lockgroup_info_t = struct[33] of uint64_t; type lockgroup_info_array_t = array[] of lockgroup_info_t; -type mach_memory_info_t = struct[8] of uint64_t; +type mach_memory_info_t = struct[22] of uint64_t; type mach_memory_info_array_t = array[] of mach_memory_info_t; import ; diff --git a/osfmk/mach_debug/mach_debug_types.h b/osfmk/mach_debug/mach_debug_types.h index ff9aeac3c..54e81d7e7 100644 --- a/osfmk/mach_debug/mach_debug_types.h +++ b/osfmk/mach_debug/mach_debug_types.h @@ -69,16 +69,27 @@ #include #include +#define MACH_CORE_FILEHEADER_SIGNATURE 0x0063614d20646152ULL +#define MACH_CORE_FILEHEADER_MAXFILES 16 +#define MACH_CORE_FILEHEADER_NAMELEN 16 + typedef char symtab_name_t[32]; +struct mach_core_details +{ + uint64_t gzip_offset; + uint64_t gzip_length; + char core_name[MACH_CORE_FILEHEADER_NAMELEN]; +}; + struct mach_core_fileheader { uint64_t signature; uint64_t log_offset; uint64_t log_length; - uint64_t gzip_offset; - uint64_t gzip_length; + uint64_t num_files; + struct mach_core_details files[MACH_CORE_FILEHEADER_MAXFILES]; }; -#define MACH_CORE_FILEHEADER_SIGNATURE 0x0063614d20646153ULL + #endif /* _MACH_DEBUG_MACH_DEBUG_TYPES_H_ */ diff --git a/osfmk/mach_debug/zone_info.h b/osfmk/mach_debug/zone_info.h index 937b594ab..c54a4c5b3 100644 --- a/osfmk/mach_debug/zone_info.h +++ b/osfmk/mach_debug/zone_info.h @@ -109,7 +109,7 @@ typedef struct mach_zone_info_data { uint64_t mzi_count; /* count of elements in use */ uint64_t mzi_cur_size; /* current memory utilization */ uint64_t mzi_max_size; /* how large can this zone grow */ - uint64_t mzi_elem_size; /* size of an element */ + uint64_t mzi_elem_size; /* size of an element */ uint64_t mzi_alloc_size; /* size used for more memory */ uint64_t mzi_sum_size; /* sum of all allocs (life of zone) */ uint64_t mzi_exhaustible; /* merely return if empty? */ @@ -122,7 +122,7 @@ typedef struct task_zone_info_data { uint64_t tzi_count; /* count of elements in use */ uint64_t tzi_cur_size; /* current memory utilization */ uint64_t tzi_max_size; /* how large can this zone grow */ - uint64_t tzi_elem_size; /* size of an element */ + uint64_t tzi_elem_size; /* size of an element */ uint64_t tzi_alloc_size; /* size used for more memory */ uint64_t tzi_sum_size; /* sum of all allocs (life of zone) */ uint64_t tzi_exhaustible; /* merely return if empty? */ @@ -134,14 +134,22 @@ typedef struct task_zone_info_data { typedef task_zone_info_t *task_zone_info_array_t; +#define MACH_MEMORY_INFO_NAME_MAX_LEN 80 + typedef struct mach_memory_info { - uint64_t flags; - uint64_t site; - uint64_t size; - uint64_t free; - uint64_t largest; + uint64_t flags; + uint64_t site; + uint64_t size; + uint64_t free; + uint64_t largest; uint64_t collectable_bytes; - uint64_t _resv[2]; + uint64_t mapped; + uint64_t peak; + uint16_t tag; + uint16_t zone; + uint16_t _resvA[2]; + uint64_t _resv[3]; + char name[MACH_MEMORY_INFO_NAME_MAX_LEN]; } mach_memory_info_t; typedef mach_memory_info_t *mach_memory_info_array_t; diff --git a/osfmk/machine/Makefile b/osfmk/machine/Makefile index 669d20515..abe9992d9 100644 --- a/osfmk/machine/Makefile +++ b/osfmk/machine/Makefile @@ -19,12 +19,18 @@ KERNELFILES = \ machine_cpuid.h \ machine_routines.h \ machine_kpc.h \ + monotonic.h \ pal_routines.h \ pal_hibernate.h \ simple_lock.h +EXPORT_FILES = + + INSTALL_MI_LCL_LIST = ${PRIVATE_DATAFILES} +INSTALL_KF_MD_LCL_LIST = ${EXPORT_FILES} + INSTALL_MI_DIR = machine EXPORT_MI_LIST = ${KERNELFILES} diff --git a/osfmk/machine/asm.h b/osfmk/machine/asm.h index c43a64523..db79977c7 100644 --- a/osfmk/machine/asm.h +++ b/osfmk/machine/asm.h @@ -30,6 +30,10 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/asm.h" +#elif defined (__arm__) +#include "arm/asm.h" +#elif defined (__arm64__) +#include "arm64/asm.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/atomic.h b/osfmk/machine/atomic.h index db0417c2e..b200f9363 100644 --- a/osfmk/machine/atomic.h +++ b/osfmk/machine/atomic.h @@ -33,6 +33,8 @@ #if defined (__x86_64__) #include "i386/atomic.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/atomic.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/commpage.h b/osfmk/machine/commpage.h index 2a525b0f2..93eb341d5 100644 --- a/osfmk/machine/commpage.h +++ b/osfmk/machine/commpage.h @@ -31,11 +31,14 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/commpage/commpage.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/commpage/commpage.h" #else #error architecture not supported #endif #ifndef __ASSEMBLER__ +#include extern void commpage_populate( void ); /* called once during startup */ extern void commpage_text_populate( void ); diff --git a/osfmk/machine/config.h b/osfmk/machine/config.h new file mode 100644 index 000000000..8bf35954b --- /dev/null +++ b/osfmk/machine/config.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _MACHINE_CONFIG_H +#define _MACHINE_CONFIG_H + +#if defined (__i386__) || defined (__x86_64__) +#elif defined (__arm__) +#include +#elif defined (__arm64__) +#include +#else +#error architecture not supported +#endif + +#endif /* _MACHINE_CONFIG_H */ diff --git a/osfmk/machine/cpu_affinity.h b/osfmk/machine/cpu_affinity.h index 5b3e47ac0..b8915c4e4 100644 --- a/osfmk/machine/cpu_affinity.h +++ b/osfmk/machine/cpu_affinity.h @@ -32,6 +32,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/cpu_affinity.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/cpu_affinity.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/cpu_capabilities.h b/osfmk/machine/cpu_capabilities.h index a722dc93f..2a9615cef 100644 --- a/osfmk/machine/cpu_capabilities.h +++ b/osfmk/machine/cpu_capabilities.h @@ -33,6 +33,8 @@ #ifdef KERNEL_PRIVATE #if defined (__i386__) || defined (__x86_64__) #include "i386/cpu_capabilities.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/cpu_capabilities.h" #else #error architecture not supported #endif @@ -40,6 +42,8 @@ #else /* !KERNEL_PRIVATE -- System Framework header */ #if defined (__i386__) || defined(__x86_64__) #include +#elif defined (__arm__) || defined(__arm64__) +#include #else #error architecture not supported #endif diff --git a/osfmk/machine/cpu_data.h b/osfmk/machine/cpu_data.h index 347235ec9..8e44b2e71 100644 --- a/osfmk/machine/cpu_data.h +++ b/osfmk/machine/cpu_data.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/cpu_data.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/cpu_data.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/cpu_number.h b/osfmk/machine/cpu_number.h index 45c4b2b4d..a26e933ba 100644 --- a/osfmk/machine/cpu_number.h +++ b/osfmk/machine/cpu_number.h @@ -32,6 +32,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/cpu_number.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/cpu_number.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/endian.h b/osfmk/machine/endian.h index 5078c0fd7..d7ee76198 100644 --- a/osfmk/machine/endian.h +++ b/osfmk/machine/endian.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/endian.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/endian.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/io_map_entries.h b/osfmk/machine/io_map_entries.h index 49306bc16..2de96f22b 100644 --- a/osfmk/machine/io_map_entries.h +++ b/osfmk/machine/io_map_entries.h @@ -32,6 +32,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/io_map_entries.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/io_map_entries.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/lock.h b/osfmk/machine/lock.h index a870743a5..6c4181b0a 100644 --- a/osfmk/machine/lock.h +++ b/osfmk/machine/lock.h @@ -32,6 +32,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/lock.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/lock.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/locks.h b/osfmk/machine/locks.h index ad7dcdcbe..5f198abc0 100644 --- a/osfmk/machine/locks.h +++ b/osfmk/machine/locks.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/locks.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/locks.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/lowglobals.h b/osfmk/machine/lowglobals.h index 319b758b4..08ebee88d 100644 --- a/osfmk/machine/lowglobals.h +++ b/osfmk/machine/lowglobals.h @@ -30,6 +30,10 @@ #if defined (__x86_64__) #include "x86_64/lowglobals.h" +#elif defined (__arm__) +#include "arm/lowglobals.h" +#elif defined (__arm64__) +#include "arm64/lowglobals.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/machine_cpu.h b/osfmk/machine/machine_cpu.h index 734cf8f30..b383bc710 100644 --- a/osfmk/machine/machine_cpu.h +++ b/osfmk/machine/machine_cpu.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/machine_cpu.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/machine_cpu.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/machine_cpuid.h b/osfmk/machine/machine_cpuid.h index ebe9396f6..dd441dcbe 100644 --- a/osfmk/machine/machine_cpuid.h +++ b/osfmk/machine/machine_cpuid.h @@ -30,7 +30,13 @@ #ifndef _MACHINE_CPUID_H #define _MACHINE_CPUID_H +#if defined (__arm__) +#include +#elif defined (__arm64__) +#include +#else #error architecture not supported +#endif #endif /* _MACHINE_CPUID_H */ diff --git a/osfmk/machine/machine_kpc.h b/osfmk/machine/machine_kpc.h index de9593c3f..3aaece0c0 100644 --- a/osfmk/machine/machine_kpc.h +++ b/osfmk/machine/machine_kpc.h @@ -30,6 +30,10 @@ #if defined (__x86_64__) #include "x86_64/machine_kpc.h" +#elif defined (__arm64__) +#include "arm64/machine_kpc.h" +#elif defined (__arm__) +#include "arm/machine_kpc.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/machine_routines.h b/osfmk/machine/machine_routines.h index 361dee046..3fd9a0a79 100644 --- a/osfmk/machine/machine_routines.h +++ b/osfmk/machine/machine_routines.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/machine_routines.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/machine_routines.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/machine_rpc.h b/osfmk/machine/machine_rpc.h index c158a0c19..1b306f3d3 100644 --- a/osfmk/machine/machine_rpc.h +++ b/osfmk/machine/machine_rpc.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/machine_rpc.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/machine_rpc.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/machlimits.h b/osfmk/machine/machlimits.h index fee4ddf0c..70bb0e797 100644 --- a/osfmk/machine/machlimits.h +++ b/osfmk/machine/machlimits.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/machlimits.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/machlimits.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/machparam.h b/osfmk/machine/machparam.h index 9657b8cce..a62b1965a 100644 --- a/osfmk/machine/machparam.h +++ b/osfmk/machine/machparam.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/machparam.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/machparam.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/monotonic.h b/osfmk/machine/monotonic.h new file mode 100644 index 000000000..b3e75c8d3 --- /dev/null +++ b/osfmk/machine/monotonic.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef MACHINE_MONOTONIC_H +#define MACHINE_MONOTONIC_H + +#if defined(__x86_64__) +#include +#elif defined(__arm64__) +#include +#elif defined(__arm__) +#include +#else +#error unsupported architecture +#endif + +#include +#include +#include + +extern bool mt_core_supported; + +struct mt_cpu { + uint64_t mtc_snaps[MT_CORE_NFIXED]; + uint64_t mtc_counts[MT_CORE_NFIXED]; + uint64_t mtc_counts_last[MT_CORE_NFIXED]; +}; + +struct mt_thread { + _Atomic uint64_t mth_gen; + uint64_t mth_counts[MT_CORE_NFIXED]; +}; + +struct mt_task { + uint64_t mtk_counts[MT_CORE_NFIXED]; +}; + +struct mt_cpu *mt_cur_cpu(void); +void mt_mtc_update_fixed_counts(struct mt_cpu *mtc, uint64_t *counts, + uint64_t *counts_since); +uint64_t mt_mtc_update_count(struct mt_cpu *mtc, unsigned int ctr); +uint64_t mt_core_snap(unsigned int ctr); +void mt_core_set_snap(unsigned int ctr, uint64_t snap); +void mt_mtc_set_snap(struct mt_cpu *mtc, unsigned int ctr, uint64_t snap); + +#endif /* !defined(MACHINE_MONOTONIC_H) */ diff --git a/osfmk/machine/pal_hibernate.h b/osfmk/machine/pal_hibernate.h index 238896dc5..3c8d6a1c0 100644 --- a/osfmk/machine/pal_hibernate.h +++ b/osfmk/machine/pal_hibernate.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/pal_hibernate.h" +#elif defined (__arm__) +//#include "arm/pal_hibernate.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/pal_routines.h b/osfmk/machine/pal_routines.h index 755b532e9..9cc6a139d 100644 --- a/osfmk/machine/pal_routines.h +++ b/osfmk/machine/pal_routines.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/pal_routines.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/pal_routines.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/pmap.h b/osfmk/machine/pmap.h index 78bef764e..475700c55 100644 --- a/osfmk/machine/pmap.h +++ b/osfmk/machine/pmap.h @@ -30,6 +30,8 @@ #if defined (__x86_64__) || defined (__i386__) #include "i386/pmap.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/pmap.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/sched_param.h b/osfmk/machine/sched_param.h index 2c5dc0d0d..3ce7907ec 100644 --- a/osfmk/machine/sched_param.h +++ b/osfmk/machine/sched_param.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/sched_param.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/sched_param.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/setjmp.h b/osfmk/machine/setjmp.h index 142e4f677..51a877635 100644 --- a/osfmk/machine/setjmp.h +++ b/osfmk/machine/setjmp.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/setjmp.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/setjmp.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/simple_lock.h b/osfmk/machine/simple_lock.h index 30e2b44c5..0613ecf65 100644 --- a/osfmk/machine/simple_lock.h +++ b/osfmk/machine/simple_lock.h @@ -32,6 +32,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/simple_lock.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/simple_lock.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/smp.h b/osfmk/machine/smp.h index b49db7ad8..108c153ad 100644 --- a/osfmk/machine/smp.h +++ b/osfmk/machine/smp.h @@ -31,6 +31,8 @@ #if defined (__x86_64__) #include "i386/smp.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/smp.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/task.h b/osfmk/machine/task.h index 3e9fc821a..4c0a3688a 100644 --- a/osfmk/machine/task.h +++ b/osfmk/machine/task.h @@ -30,8 +30,12 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/task.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/task.h" #else #error architecture not supported #endif +extern void machine_task_init(task_t new_task, task_t parent_task, boolean_t memory_inherit); + #endif /* _MACHINE_TASK_H */ diff --git a/osfmk/machine/thread.h b/osfmk/machine/thread.h index 840d103b7..0b91ffe04 100644 --- a/osfmk/machine/thread.h +++ b/osfmk/machine/thread.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/thread.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/thread.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/trap.h b/osfmk/machine/trap.h index 5fb2aa18d..b08f4c64f 100644 --- a/osfmk/machine/trap.h +++ b/osfmk/machine/trap.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/trap.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/trap.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/vm_tuning.h b/osfmk/machine/vm_tuning.h index 324d9d25a..84bb4efe5 100644 --- a/osfmk/machine/vm_tuning.h +++ b/osfmk/machine/vm_tuning.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/vm_tuning.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/vm_tuning.h" #else #error architecture not supported #endif diff --git a/osfmk/machine/xpr.h b/osfmk/machine/xpr.h index ad747c34f..9ed04ff5f 100644 --- a/osfmk/machine/xpr.h +++ b/osfmk/machine/xpr.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "i386/xpr.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/xpr.h" #else #error architecture not supported #endif diff --git a/osfmk/prng/random.c b/osfmk/prng/random.c index b927d7f80..cb3555d39 100644 --- a/osfmk/prng/random.c +++ b/osfmk/prng/random.c @@ -48,76 +48,229 @@ #include #include #include +#include +#include #include #include #include -static lck_grp_t *gPRNGGrp; -static lck_attr_t *gPRNGAttr; -static lck_grp_attr_t *gPRNGGrpAttr; -static lck_mtx_t *gPRNGMutex = NULL; +#if defined(__x86_64__) +#include + +static int rdseed_step(uint64_t *seed) +{ + uint8_t ok; + + asm volatile ("rdseed %0; setc %1" : "=r" (*seed), "=qm" (ok)); + + return (int) ok; +} + +static int rdseed_retry(uint64_t *seed, size_t nretries) +{ + size_t i; + + for (i = 0; i < nretries; i += 1) { + if (rdseed_step(seed)) { + return 1; + } else { + asm volatile ("pause"); + } + } + + return 0; +} + +static size_t rdseed_seed(void *buf, size_t nwords) +{ + uint64_t *buf_words; + size_t i; + + if (nwords > 8) { + nwords = 8; + } + + buf_words = buf; + for (i = 0; i < nwords; i += 1) { + if (!rdseed_retry(buf_words + i, 10)) { + return i; + } + } + + return nwords; +} + +static int rdrand_step(uint64_t *rand) +{ + uint8_t ok; + + asm volatile ("rdrand %0; setc %1" : "=r" (*rand), "=qm" (ok)); + + return (int) ok; +} + +static int rdrand_retry(uint64_t *rand, size_t nretries) +{ + size_t i; + + for (i = 0; i < nretries; i += 1) { + if (rdrand_step(rand)) { + return 1; + } + } + + return 0; +} + +static size_t rdrand_seed(void *buf, size_t nwords) +{ + size_t i; + uint64_t w; + uint8_t hash[CCSHA256_OUTPUT_SIZE]; + const struct ccdigest_info *di = &ccsha256_ltc_di; + + ccdigest_di_decl(di, ctx); + ccdigest_init(di, ctx); + + for (i = 0; i < 1023; i += 1) { + if (!rdrand_retry(&w, 10)) { + nwords = 0; + goto out; + } + ccdigest_update(di, ctx, sizeof w, &w); + } + + ccdigest_final(di, ctx, hash); + + if (nwords > 2) { + nwords = 2; + } + + memcpy(buf, hash, nwords * sizeof (uint64_t)); + +out: + ccdigest_di_clear(di, ctx); + bzero(hash, sizeof hash); + bzero(&w, sizeof w); + + return nwords; +} + +static void intel_entropysource(void *buf, size_t *nbytes) +{ + size_t nwords; + + /* only handle complete words */ + assert(*nbytes % sizeof (uint64_t) == 0); + + nwords = (*nbytes) / sizeof (uint64_t); + if (cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_RDSEED) { + nwords = rdseed_seed(buf, nwords); + *nbytes = nwords * sizeof (uint64_t); + } else if (cpuid_features() & CPUID_FEATURE_RDRAND) { + nwords = rdrand_seed(buf, nwords); + *nbytes = nwords * sizeof (uint64_t); + } else { + *nbytes = 0; + } +} + +#endif + +typedef void (*entropysource)(void *buf, size_t *nbytes); + +static const entropysource entropysources[] = { + entropy_buffer_read, +#if defined(__x86_64__) + intel_entropysource, +#endif +}; + +static const size_t nsources = sizeof entropysources / sizeof entropysources[0]; + +static size_t entropy_readall(void *buf, size_t nbytes_persource) +{ + uint8_t *buf_bytes = buf; + size_t i; + size_t nbytes_total = 0; + + for (i = 0; i < nsources; i += 1) { + size_t nbytes = nbytes_persource; + entropysources[i](buf_bytes, &nbytes); + bzero(buf_bytes + nbytes, nbytes_persource - nbytes); + nbytes_total += nbytes; + buf_bytes += nbytes_persource; + } + + return nbytes_total; +} + +static struct { + lck_grp_t *group; + lck_attr_t *attrs; + lck_grp_attr_t *group_attrs; + lck_mtx_t *mutex; +} lock; typedef struct prngContext { - struct ccdrbg_info *infop; - struct ccdrbg_state *statep; - uint64_t bytes_generated; - uint64_t bytes_reseeded; + struct ccdrbg_info *infop; + struct ccdrbg_state *statep; + uint64_t bytes_generated; + uint64_t bytes_reseeded; } *prngContextp; ccdrbg_factory_t prng_ccdrbg_factory = NULL; -entropy_data_t EntropyData = { .index_ptr = EntropyData.buffer }; - -boolean_t erandom_seed_set = FALSE; -char erandom_seed[EARLY_RANDOM_SEED_SIZE]; -typedef struct ccdrbg_state ccdrbg_state_t; -uint8_t master_erandom_state[EARLY_RANDOM_STATE_STATIC_SIZE]; -ccdrbg_state_t *erandom_state[MAX_CPUS]; -struct ccdrbg_info erandom_info; -decl_simple_lock_data(,entropy_lock); +entropy_data_t EntropyData = { + .index_ptr = EntropyData.buffer +}; -struct ccdrbg_nisthmac_custom erandom_custom = { - .di = &ccsha1_eay_di, - .strictFIPS = 0, +static struct { + uint8_t seed[nsources][EARLY_RANDOM_SEED_SIZE]; + size_t seedset; + uint8_t master_drbg_state[EARLY_RANDOM_STATE_STATIC_SIZE]; + struct ccdrbg_state *drbg_states[MAX_CPUS]; + struct ccdrbg_info drbg_info; + const struct ccdrbg_nisthmac_custom drbg_custom; +} erandom = { + .drbg_custom = { + .di = &ccsha1_eay_di, + .strictFIPS = 0, + } }; -static void read_erandom(void *buffer, u_int numBytes); /* Forward */ +static void read_erandom(void *buf, uint32_t nbytes); void -entropy_buffer_read(char *buffer, - unsigned int *count) +entropy_buffer_read(void *buffer, size_t *count) { - boolean_t current_state; - unsigned int i, j; + boolean_t current_state; + unsigned int i, j; - if (!erandom_seed_set) { + if (!erandom.seedset) { panic("early_random was never invoked"); } - if ((*count) > (ENTROPY_BUFFER_SIZE * sizeof(unsigned int))) - *count = ENTROPY_BUFFER_SIZE * sizeof(unsigned int); + if (*count > ENTROPY_BUFFER_BYTE_SIZE) { + *count = ENTROPY_BUFFER_BYTE_SIZE; + } current_state = ml_set_interrupts_enabled(FALSE); -#if defined (__x86_64__) - simple_lock(&entropy_lock); -#endif - memcpy((char *) buffer, (char *) EntropyData.buffer, *count); + memcpy(buffer, EntropyData.buffer, *count); + /* Consider removing this mixing step rdar://problem/31668239 */ for (i = 0, j = (ENTROPY_BUFFER_SIZE - 1); i < ENTROPY_BUFFER_SIZE; j = i, i++) EntropyData.buffer[i] = EntropyData.buffer[i] ^ EntropyData.buffer[j]; -#if defined (__x86_64__) - simple_unlock(&entropy_lock); -#endif (void) ml_set_interrupts_enabled(current_state); #if DEVELOPMENT || DEBUG - uint32_t *word = (uint32_t *) (void *) buffer; + uint32_t *word = buffer; /* Good for both 32-bit and 64-bit kernels. */ for (i = 0; i < ENTROPY_BUFFER_SIZE; i += 4) - /* + /* * We use "EARLY" here so that we can grab early entropy on * ARM, where tracing is not started until after PRNG is * initialized. @@ -136,7 +289,7 @@ entropy_buffer_read(char *buffer, * This provides cryptographically secure randomness. * Each processor has its own generator instance. * It is seeded (lazily) with entropy provided by the Booter. -* + * * For the algorithm switched from LCG to * NIST HMAC DBRG as follows: * - When first called (on OSX this is very early while page tables are being @@ -147,7 +300,7 @@ entropy_buffer_read(char *buffer, * The initial entropy is 16 bytes of boot entropy. * The nonce is the first 8 bytes of entropy xor'ed with a timestamp * from ml_get_timebase(). - * The personalization data provided is null. + * The personalization data provided is null. * - The first 64-bit random value is returned on the boot processor from * an invocation of the ccdbrg_generate method. * - Non-boot processor's DRBG state structures are allocated dynamically @@ -157,7 +310,7 @@ entropy_buffer_read(char *buffer, * an 8-byte random value. read_erandom() ensures that pre-emption is * disabled and selects the DBRG state from the current processor. * The ccdbrg_generate method is called for the required random output. - * If this method returns CCDRBG_STATUS_NEED_RESEED, the erandom_seed buffer + * If this method returns CCDRBG_STATUS_NEED_RESEED, the erandom.seed buffer * is re-filled with kernel-harvested entropy and the ccdbrg_reseed method is * called with this new entropy. The kernel panics if a reseed fails. */ @@ -169,11 +322,10 @@ early_random(void) uint64_t nonce; int rc; int ps; - ccdrbg_state_t *state; + struct ccdrbg_state *state; - if (!erandom_seed_set) { - simple_lock_init(&entropy_lock,0); - erandom_seed_set = TRUE; + if (!erandom.seedset) { + erandom.seedset = 1; cnt = PE_get_random_seed((unsigned char *) EntropyData.buffer, sizeof(EntropyData.buffer)); @@ -186,34 +338,23 @@ early_random(void) sizeof(EntropyData.buffer), cnt); } - /* - * Use some of the supplied entropy as a basis for early_random; - * reuse is ugly, but simplifies things. Ideally, we would guard - * early random values well enough that it isn't safe to attack - * them, but this cannot be guaranteed; thus, initial entropy - * can be considered 8 bytes weaker for a given boot if any - * early random values are conclusively determined. - * - * early_random_seed could be larger than EntopyData.buffer... - * but it won't be. - */ - bcopy(EntropyData.buffer, &erandom_seed, sizeof(erandom_seed)); + entropy_readall(&erandom.seed, EARLY_RANDOM_SEED_SIZE); /* Init DRBG for NIST HMAC */ - ccdrbg_factory_nisthmac(&erandom_info, &erandom_custom); - assert(erandom_info.size <= sizeof(master_erandom_state)); - state = (ccdrbg_state_t *) master_erandom_state; - erandom_state[0] = state; + ccdrbg_factory_nisthmac(&erandom.drbg_info, &erandom.drbg_custom); + assert(erandom.drbg_info.size <= sizeof(erandom.master_drbg_state)); + state = (struct ccdrbg_state *) erandom.master_drbg_state; + erandom.drbg_states[master_cpu] = state; /* * Init our DBRG from the boot entropy and a timestamp as nonce * and the cpu number as personalization. */ - assert(sizeof(erandom_seed) > sizeof(nonce)); + assert(sizeof(erandom.seed) > sizeof(nonce)); nonce = ml_get_timebase(); ps = 0; /* boot cpu */ - rc = ccdrbg_init(&erandom_info, state, - sizeof(erandom_seed), erandom_seed, + rc = ccdrbg_init(&erandom.drbg_info, state, + sizeof(erandom.seed), erandom.seed, sizeof(nonce), &nonce, sizeof(ps), &ps); cc_clear(sizeof(nonce), &nonce); @@ -221,9 +362,9 @@ early_random(void) panic("ccdrbg_init() returned %d", rc); /* Generate output */ - rc = ccdrbg_generate(&erandom_info, state, - sizeof(result), &result, - 0, NULL); + rc = ccdrbg_generate(&erandom.drbg_info, state, + sizeof(result), &result, + 0, NULL); if (rc != CCDRBG_STATUS_OK) panic("ccdrbg_generate() returned %d", rc); @@ -240,29 +381,28 @@ read_erandom(void *buffer, u_int numBytes) { int cpu; int rc; - uint32_t cnt; - ccdrbg_state_t *state; + size_t nbytes; + struct ccdrbg_state *state; mp_disable_preemption(); cpu = cpu_number(); - state = erandom_state[cpu]; + state = erandom.drbg_states[cpu]; assert(state); - while (TRUE) { + for (;;) { /* Generate output */ - rc = ccdrbg_generate(&erandom_info, state, - numBytes, buffer, - 0, NULL); + rc = ccdrbg_generate(&erandom.drbg_info, state, + numBytes, buffer, + 0, NULL); if (rc == CCDRBG_STATUS_OK) break; if (rc == CCDRBG_STATUS_NEED_RESEED) { /* It's time to reseed. Get more entropy */ - cnt = sizeof(erandom_seed); - entropy_buffer_read(erandom_seed, &cnt); - assert(cnt == sizeof(erandom_seed)); - rc = ccdrbg_reseed(&erandom_info, state, - sizeof(erandom_seed), erandom_seed, + nbytes = entropy_readall(erandom.seed, EARLY_RANDOM_SEED_SIZE); + assert(nbytes >= EARLY_RANDOM_SEED_SIZE); + rc = ccdrbg_reseed(&erandom.drbg_info, state, + sizeof(erandom.seed), erandom.seed, 0, NULL); - cc_clear(sizeof(erandom_seed), erandom_seed); + cc_clear(sizeof(erandom.seed), erandom.seed); if (rc == CCDRBG_STATUS_OK) continue; panic("read_erandom reseed error %d\n", rc); @@ -275,8 +415,8 @@ read_erandom(void *buffer, u_int numBytes) void read_frandom(void *buffer, u_int numBytes) { - char *cp = (char *) buffer; - int nbytes; + uint8_t *buffer_bytes = buffer; + int nbytes; /* * Split up into requests for blocks smaller than @@ -285,8 +425,8 @@ read_frandom(void *buffer, u_int numBytes) */ while (numBytes) { nbytes = MIN(numBytes, PAGE_SIZE); - read_erandom(cp, nbytes); - cp += nbytes; + read_erandom(buffer_bytes, nbytes); + buffer_bytes += nbytes; numBytes -= nbytes; } } @@ -307,28 +447,28 @@ prng_cpu_init(int cpu) { uint64_t nonce; int rc; - ccdrbg_state_t *state; + struct ccdrbg_state *state; prngContextp pp; /* * Allocate state and initialize DBRG state for early_random() * for this processor, if necessary. */ - if (erandom_state[cpu] == NULL) { + if (erandom.drbg_states[cpu] == NULL) { - state = kalloc(erandom_info.size); + state = kalloc(erandom.drbg_info.size); if (state == NULL) { panic("prng_init kalloc failed\n"); } - erandom_state[cpu] = state; + erandom.drbg_states[cpu] = state; /* * Init our DBRG from boot entropy, nonce as timestamp * and use the cpu number as the personalization parameter. */ nonce = ml_get_timebase(); - rc = ccdrbg_init(&erandom_info, state, - sizeof(erandom_seed), erandom_seed, + rc = ccdrbg_init(&erandom.drbg_info, state, + sizeof(erandom.seed), erandom.seed, sizeof(nonce), &nonce, sizeof(cpu), &cpu); cc_clear(sizeof(nonce), &nonce); @@ -342,13 +482,13 @@ prng_cpu_init(int cpu) return; } - assert(gPRNGMutex == NULL); /* Once only, please */ + assert(lock.mutex == NULL); /* Once only, please */ /* make a mutex to control access */ - gPRNGGrpAttr = lck_grp_attr_alloc_init(); - gPRNGGrp = lck_grp_alloc_init("random", gPRNGGrpAttr); - gPRNGAttr = lck_attr_alloc_init(); - gPRNGMutex = lck_mtx_alloc_init(gPRNGGrp, gPRNGAttr); + lock.group_attrs = lck_grp_attr_alloc_init(); + lock.group = lck_grp_alloc_init("random", lock.group_attrs); + lock.attrs = lck_attr_alloc_init(); + lock.mutex = lck_mtx_alloc_init(lock.group, lock.attrs); pp = kalloc(sizeof(*pp)); if (pp == NULL) @@ -363,10 +503,13 @@ prng_cpu_init(int cpu) master_prng_context() = pp; } -static ccdrbg_info_t * +static struct ccdrbg_info * prng_infop(prngContextp pp) { - lck_mtx_assert(gPRNGMutex, LCK_MTX_ASSERT_OWNED); + uint8_t buf[nsources][ENTROPY_BUFFER_BYTE_SIZE]; + size_t nbytes; + + lck_mtx_assert(lock.mutex, LCK_MTX_ASSERT_OWNED); /* Usual case: the info is all set */ if (pp->infop) @@ -379,18 +522,18 @@ prng_infop(prngContextp pp) while (prng_ccdrbg_factory == NULL ) { wait_result_t wait_result; assert_wait_timeout((event_t) &prng_ccdrbg_factory, TRUE, - 10, NSEC_PER_USEC); - lck_mtx_unlock(gPRNGMutex); + 10, NSEC_PER_USEC); + lck_mtx_unlock(lock.mutex); wait_result = thread_block(THREAD_CONTINUE_NULL); if (wait_result == THREAD_TIMED_OUT) panic("prng_ccdrbg_factory registration timeout"); - lck_mtx_lock(gPRNGMutex); + lck_mtx_lock(lock.mutex); } /* Check we didn't lose the set-up race */ if (pp->infop) return pp->infop; - pp->infop = (ccdrbg_info_t *) kalloc(sizeof(ccdrbg_info_t)); + pp->infop = (struct ccdrbg_info *) kalloc(sizeof(struct ccdrbg_info)); if (pp->infop == NULL) panic("Unable to allocate prng info"); @@ -400,32 +543,29 @@ prng_infop(prngContextp pp) if (pp->statep == NULL) panic("Unable to allocate prng state"); - char rdBuffer[ENTROPY_BUFFER_BYTE_SIZE]; - unsigned int bytesToInput = sizeof(rdBuffer); - - entropy_buffer_read(rdBuffer, &bytesToInput); + nbytes = entropy_readall(buf, ENTROPY_BUFFER_BYTE_SIZE); (void) ccdrbg_init(pp->infop, pp->statep, - bytesToInput, rdBuffer, + nbytes, buf, 0, NULL, 0, NULL); - cc_clear(sizeof(rdBuffer), rdBuffer); + cc_clear(sizeof (buf), buf); return pp->infop; } static void Reseed(prngContextp pp) { - char rdBuffer[ENTROPY_BUFFER_BYTE_SIZE]; - unsigned int bytesToInput = sizeof(rdBuffer); - - entropy_buffer_read(rdBuffer, &bytesToInput); + uint8_t buf[nsources][ENTROPY_BUFFER_BYTE_SIZE]; + size_t nbytes; + + nbytes = entropy_readall(buf, ENTROPY_BUFFER_BYTE_SIZE); PRNG_CCDRBG((void) ccdrbg_reseed(pp->infop, pp->statep, - bytesToInput, rdBuffer, + nbytes, buf, 0, NULL)); - cc_clear(sizeof(rdBuffer), rdBuffer); + cc_clear(sizeof (buf), buf); pp->bytes_reseeded = pp->bytes_generated; } @@ -434,11 +574,11 @@ Reseed(prngContextp pp) void read_random(void* buffer, u_int numbytes) { - prngContextp pp; - ccdrbg_info_t *infop; - int ccdrbg_err; + prngContextp pp; + struct ccdrbg_info *infop; + int ccdrbg_err; - lck_mtx_lock(gPRNGMutex); + lck_mtx_lock(lock.mutex); pp = current_prng_context(); infop = prng_infop(pp); @@ -446,11 +586,11 @@ read_random(void* buffer, u_int numbytes) /* * Call DRBG, reseeding and retrying if requested. */ - while (TRUE) { + for (;;) { PRNG_CCDRBG( ccdrbg_err = ccdrbg_generate(infop, pp->statep, - numbytes, buffer, - 0, NULL)); + numbytes, buffer, + 0, NULL)); if (ccdrbg_err == CCDRBG_STATUS_OK) break; if (ccdrbg_err == CCDRBG_STATUS_NEED_RESEED) { @@ -461,7 +601,7 @@ read_random(void* buffer, u_int numbytes) } pp->bytes_generated += numbytes; - lck_mtx_unlock(gPRNGMutex); + lck_mtx_unlock(lock.mutex); } int @@ -471,7 +611,7 @@ write_random(void* buffer, u_int numbytes) int retval = 0; prngContextp pp; - lck_mtx_lock(gPRNGMutex); + lck_mtx_lock(lock.mutex); pp = current_prng_context(); @@ -479,10 +619,10 @@ write_random(void* buffer, u_int numbytes) bytesToInput, rdBuffer, 0, NULL) != 0) retval = EIO; - lck_mtx_unlock(gPRNGMutex); + lck_mtx_unlock(lock.mutex); return retval; #else -#pragma unused(buffer, numbytes) - return 0; +#pragma unused(buffer, numbytes) + return 0; #endif } diff --git a/osfmk/prng/random.h b/osfmk/prng/random.h index 2f721e288..aa269df35 100644 --- a/osfmk/prng/random.h +++ b/osfmk/prng/random.h @@ -33,13 +33,17 @@ __BEGIN_DECLS #ifdef XNU_KERNEL_PRIVATE -#define ENTROPY_BUFFER_BYTE_SIZE 64 +#define ENTROPY_BUFFER_BYTE_SIZE 64 -#define ENTROPY_BUFFER_SIZE ENTROPY_BUFFER_BYTE_SIZE/sizeof(uint32_t) +#define ENTROPY_BUFFER_SIZE (ENTROPY_BUFFER_BYTE_SIZE / sizeof(uint32_t)) typedef struct entropy_data { - uint32_t *index_ptr; - uint32_t buffer[ENTROPY_BUFFER_SIZE]; + /* + * TODO: Should index_ptr be volatile? Are we exposed to any races that + * we care about if it is not? + */ + uint32_t *index_ptr; + uint32_t buffer[ENTROPY_BUFFER_SIZE]; } entropy_data_t; extern entropy_data_t EntropyData; @@ -49,12 +53,16 @@ extern entropy_data_t EntropyData; /* * Early_random implementation params: */ -#define EARLY_RANDOM_SEED_SIZE (16) -#define EARLY_RANDOM_STATE_STATIC_SIZE (264) +#define EARLY_RANDOM_SEED_SIZE (16) +#define EARLY_RANDOM_STATE_STATIC_SIZE (264) #if defined (__x86_64__) #define current_prng_context() (current_cpu_datap()->cpu_prng) #define master_prng_context() (cpu_datap(master_cpu)->cpu_prng) +#elif defined (__arm__) || defined(__arm64__) +#include // For MAX_CPUS +#define current_prng_context() (getCpuDatap()->cpu_prng) +#define master_prng_context() (cpu_datap(master_cpu)->cpu_prng) #else #error architecture unknown #endif @@ -62,16 +70,15 @@ extern entropy_data_t EntropyData; #include #include -typedef struct ccdrbg_info ccdrbg_info_t; -typedef void (*ccdrbg_factory_t)(ccdrbg_info_t *info, const void *custom); +typedef void (*ccdrbg_factory_t)(struct ccdrbg_info *info, const void *custom); -extern void ccdrbg_factory_yarrow(ccdrbg_info_t *info, const void *custom); +extern void ccdrbg_factory_yarrow(struct ccdrbg_info *info, const void *custom); -void prng_factory_register(ccdrbg_factory_t factory); -void prng_cpu_init(int cpu); +void prng_factory_register(ccdrbg_factory_t factory); +void prng_cpu_init(int cpu); -void entropy_buffer_read(char *buffer, unsigned int *count); -void entropy_boot_trace(void); +void entropy_buffer_read(void *buffer, size_t *count); +void entropy_boot_trace(void); /* * Wrapper for requesting a CCDRBG operation. diff --git a/osfmk/profiling/machine/profile-md.h b/osfmk/profiling/machine/profile-md.h index 2861ee13f..0488b28ff 100644 --- a/osfmk/profiling/machine/profile-md.h +++ b/osfmk/profiling/machine/profile-md.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined (__x86_64__) #include "profiling/i386/profile-md.h" +#elif defined (__arm__) || defined (__arm64__) +#include "profiling/arm/profile-md.h" #else #error architecture not supported #endif diff --git a/osfmk/vm/WKdm_new.h b/osfmk/vm/WKdm_new.h index 3713b2d8b..ee9e884ee 100644 --- a/osfmk/vm/WKdm_new.h +++ b/osfmk/vm/WKdm_new.h @@ -73,6 +73,30 @@ extern "C" { typedef unsigned int WK_word; +#if defined(__arm64__) + +void +WKdm_decompress_4k (const WK_word* src_buf, + WK_word* dest_buf, + WK_word* scratch, + unsigned int bytes); +int +WKdm_compress_4k (const WK_word* src_buf, + WK_word* dest_buf, + WK_word* scratch, + unsigned int limit); + +void +WKdm_decompress_16k (WK_word* src_buf, + WK_word* dest_buf, + WK_word* scratch, + unsigned int bytes); +int +WKdm_compress_16k (WK_word* src_buf, + WK_word* dest_buf, + WK_word* scratch, + unsigned int limit); +#else void WKdm_decompress_new (WK_word* src_buf, @@ -84,6 +108,7 @@ WKdm_compress_new (const WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int limit); +#endif #ifdef __cplusplus } /* extern "C" */ diff --git a/osfmk/vm/bsd_vm.c b/osfmk/vm/bsd_vm.c index 78e39a753..b9d138421 100644 --- a/osfmk/vm/bsd_vm.c +++ b/osfmk/vm/bsd_vm.c @@ -111,17 +111,15 @@ const struct memory_object_pager_ops vnode_pager_ops = { }; typedef struct vnode_pager { - struct ipc_object_header pager_header; /* fake ip_kotype() */ - memory_object_pager_ops_t pager_ops; /* == &vnode_pager_ops */ + /* mandatory generic header */ + struct memory_object vn_pgr_hdr; + + /* pager-specific */ unsigned int ref_count; /* reference count */ - memory_object_control_t control_handle; /* mem object control handle */ struct vnode *vnode_handle; /* vnode handle */ } *vnode_pager_t; -#define pager_ikot pager_header.io_bits - - kern_return_t vnode_pager_cluster_read( /* forward */ vnode_pager_t, @@ -257,13 +255,6 @@ memory_object_control_uiomove( if (dst_page->laundry) vm_pageout_steal_laundry(dst_page, FALSE); - /* - * this routine is only called when copying - * to/from real files... no need to consider - * encrypted swap pages - */ - assert(!dst_page->encrypted); - if (mark_dirty) { if (dst_page->dirty == FALSE) dirty_count++; @@ -375,6 +366,9 @@ vnode_pager_bootstrap(void) apple_protect_pager_bootstrap(); #endif /* CONFIG_CODE_DECRYPTION */ swapfile_pager_bootstrap(); +#if __arm64__ + fourk_pager_bootstrap(); +#endif /* __arm64__ */ return; } @@ -419,7 +413,7 @@ vnode_pager_init(memory_object_t mem_obj, memory_object_control_reference(control); - vnode_object->control_handle = control; + vnode_object->vn_pgr_hdr.mo_control = control; attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ @@ -633,7 +627,9 @@ vnode_pager_data_request( size = MAX_UPL_TRANSFER_BYTES; base_offset = offset; - if (memory_object_cluster_size(vnode_object->control_handle, &base_offset, &size, &io_streaming, fault_info) != KERN_SUCCESS) + if (memory_object_cluster_size(vnode_object->vn_pgr_hdr.mo_control, + &base_offset, &size, &io_streaming, + fault_info) != KERN_SUCCESS) size = PAGE_SIZE; assert(offset >= base_offset && @@ -699,20 +695,13 @@ vnode_pager_terminate( */ kern_return_t vnode_pager_synchronize( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length, __unused vm_sync_t sync_flags) { - vnode_pager_t vnode_object; - - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %p\n", mem_obj)); - - vnode_object = vnode_pager_lookup(mem_obj); - - memory_object_synchronize_completed(vnode_object->control_handle, offset, length); - - return (KERN_SUCCESS); + panic("vnode_pager_synchronize: memory_object_synchronize no longer supported\n"); + return (KERN_FAILURE); } /* @@ -882,9 +871,9 @@ vnode_pager_cluster_read( UPL_SET_INTERNAL); count = 0; assert((upl_size_t) cnt == cnt); - kr = memory_object_upl_request(vnode_object->control_handle, + kr = memory_object_upl_request(vnode_object->vn_pgr_hdr.mo_control, base_offset, (upl_size_t) cnt, - &upl, NULL, &count, uplflags); + &upl, NULL, &count, uplflags, VM_KERN_MEMORY_NONE); if (kr == KERN_SUCCESS) { upl_abort(upl, 0); upl_deallocate(upl); @@ -905,18 +894,6 @@ vnode_pager_cluster_read( } - -/* - * - */ -void -vnode_pager_release_from_cache( - int *cnt) -{ - memory_object_free_from_cache( - &realhost, &vnode_pager_ops, cnt); -} - /* * */ @@ -937,10 +914,11 @@ vnode_object_create( * we reserve the first word in the object for a fake ip_kotype * setting - that will tell vm_map to use it as a memory object. */ - vnode_object->pager_ops = &vnode_pager_ops; - vnode_object->pager_ikot = IKOT_MEMORY_OBJECT; + vnode_object->vn_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; + vnode_object->vn_pgr_hdr.mo_pager_ops = &vnode_pager_ops; + vnode_object->vn_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; + vnode_object->ref_count = 1; - vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL; vnode_object->vnode_handle = vp; return(vnode_object); @@ -956,7 +934,7 @@ vnode_pager_lookup( vnode_pager_t vnode_object; vnode_object = (vnode_pager_t)name; - assert(vnode_object->pager_ops == &vnode_pager_ops); + assert(vnode_object->vn_pgr_hdr.mo_pager_ops == &vnode_pager_ops); return (vnode_object); } @@ -967,7 +945,7 @@ vnode_pager_lookup_vnode( { vnode_pager_t vnode_object; vnode_object = (vnode_pager_t)name; - if(vnode_object->pager_ops == &vnode_pager_ops) + if(vnode_object->vn_pgr_hdr.mo_pager_ops == &vnode_pager_ops) return (vnode_object->vnode_handle); else return NULL; diff --git a/osfmk/vm/device_vm.c b/osfmk/vm/device_vm.c index abe786665..c4f953e1c 100644 --- a/osfmk/vm/device_vm.c +++ b/osfmk/vm/device_vm.c @@ -48,6 +48,7 @@ #include #include #include +#include /* Device VM COMPONENT INTERFACES */ @@ -85,17 +86,30 @@ typedef uintptr_t device_port_t; * The start of "struct device_pager" MUST match a "struct memory_object". */ typedef struct device_pager { - struct ipc_object_header pager_header; /* fake ip_kotype() */ - memory_object_pager_ops_t pager_ops; /* == &device_pager_ops */ + /* mandatory generic header */ + struct memory_object dev_pgr_hdr; + + /* pager-specific data */ + lck_mtx_t lock; unsigned int ref_count; /* reference count */ - memory_object_control_t control_handle; /* mem object's cntrl handle */ device_port_t device_handle; /* device_handle */ vm_size_t size; int flags; + boolean_t is_mapped; } *device_pager_t; -#define pager_ikot pager_header.io_bits +lck_grp_t device_pager_lck_grp; +lck_grp_attr_t device_pager_lck_grp_attr; +lck_attr_t device_pager_lck_attr; +#define device_pager_lock_init(pager) \ + lck_mtx_init(&(pager)->lock, \ + &device_pager_lck_grp, \ + &device_pager_lck_attr) +#define device_pager_lock_destroy(pager) \ + lck_mtx_destroy(&(pager)->lock, &device_pager_lck_grp) +#define device_pager_lock(pager) lck_mtx_lock(&(pager)->lock) +#define device_pager_unlock(pager) lck_mtx_unlock(&(pager)->lock) device_pager_t device_pager_lookup( /* forward */ @@ -128,6 +142,11 @@ device_pager_bootstrap(void) device_pager_zone = zinit(size, (vm_size_t) MAX_DNODE*size, PAGE_SIZE, "device node pager structures"); zone_change(device_pager_zone, Z_CALLERACCT, FALSE); + + lck_grp_attr_setdefault(&device_pager_lck_grp_attr); + lck_grp_init(&device_pager_lck_grp, "device_pager", &device_pager_lck_grp_attr); + lck_attr_setdefault(&device_pager_lck_attr); + return; } @@ -142,6 +161,8 @@ device_pager_setup( int flags) { device_pager_t device_object; + memory_object_control_t control; + vm_object_t object; device_object = device_object_create(); if (device_object == DEVICE_PAGER_NULL) @@ -151,7 +172,20 @@ device_pager_setup( device_object->size = size; device_object->flags = flags; - return((memory_object_t)device_object); + memory_object_create_named((memory_object_t) device_object, + size, + &control); + object = memory_object_control_to_vm_object(control); + + assert(object != VM_OBJECT_NULL); + vm_object_lock(object); + object->true_share = TRUE; + if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) { + object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; + } + vm_object_unlock(object); + + return (memory_object_t)device_object; } /* @@ -174,7 +208,7 @@ device_pager_populate_object( return KERN_FAILURE; vm_object = (vm_object_t)memory_object_control_to_vm_object( - device_object->control_handle); + device_object->dev_pgr_hdr.mo_control); if(vm_object == NULL) return KERN_FAILURE; @@ -190,7 +224,8 @@ device_pager_populate_object( (vm_object_offset_t)offset, (upl_size_t) size, &upl, NULL, &null_size, - (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE)); + (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE), + VM_KERN_MEMORY_NONE); if(kr != KERN_SUCCESS) panic("device_pager_populate_object: list_req failed"); @@ -207,13 +242,14 @@ device_pager_populate_object( */ device_pager_t device_pager_lookup( - memory_object_t name) + memory_object_t mem_obj) { device_pager_t device_object; - device_object = (device_pager_t)name; - assert(device_object->pager_ops == &device_pager_ops); - return (device_object); + assert(mem_obj->mo_pager_ops == &device_pager_ops); + device_object = (device_pager_t)mem_obj; + assert(device_object->ref_count > 0); + return device_object; } /* @@ -238,7 +274,7 @@ device_pager_init( device_object = device_pager_lookup(mem_obj); memory_object_control_reference(control); - device_object->control_handle = control; + device_object->dev_pgr_hdr.mo_control = control; /* The following settings should be done through an expanded change */ @@ -337,6 +373,10 @@ device_pager_reference( device_object = device_pager_lookup(mem_obj); new_ref_count = hw_atomic_add(&device_object->ref_count, 1); assert(new_ref_count > 1); + DTRACE_VM2(device_pager_reference, + device_pager_t, device_object, + unsigned int, device_object->ref_count); + } /* @@ -348,27 +388,43 @@ device_pager_deallocate( { device_pager_t device_object; memory_object_control_t device_control; + unsigned int ref_count; device_object = device_pager_lookup(mem_obj); + assert(device_object->ref_count > 0); + + DTRACE_VM2(device_pager_deallocate, + device_pager_t, device_object, + unsigned int, device_object->ref_count); + + ref_count = hw_atomic_sub(&device_object->ref_count, 1); + + if (ref_count == 1) { + /* + * The last reference is our "named" reference. + * Close the device and "destroy" the VM object. + */ + + DTRACE_VM2(device_pager_destroy, + device_pager_t, device_object, + unsigned int, device_object->ref_count); - if (hw_atomic_sub(&device_object->ref_count, 1) == 0) { + assert(device_object->is_mapped == FALSE); if (device_object->device_handle != (device_port_t) NULL) { device_close(device_object->device_handle); device_object->device_handle = (device_port_t) NULL; } - device_control = device_object->control_handle; - if (device_control != MEMORY_OBJECT_CONTROL_NULL) { - /* - * The VM object should already have been disconnected - * from the pager at this point. - * We still have to release the "memory object control" - * handle. - */ - assert(device_control->moc_object == VM_OBJECT_NULL); - memory_object_control_deallocate(device_control); - device_object->control_handle = - MEMORY_OBJECT_CONTROL_NULL; - } + device_control = device_object->dev_pgr_hdr.mo_control; + memory_object_destroy(device_control, 0); + } else if (ref_count == 0) { + /* + * No more references: free the pager. + */ + DTRACE_VM2(device_pager_free, + device_pager_t, device_object, + unsigned int, device_object->ref_count); + + device_pager_lock_destroy(device_object); zfree(device_pager_zone, device_object); } @@ -409,19 +465,13 @@ device_pager_terminate( */ kern_return_t device_pager_synchronize( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length, __unused vm_sync_t sync_flags) { - device_pager_t device_object; - - device_object = device_pager_lookup(mem_obj); - - memory_object_synchronize_completed( - device_object->control_handle, offset, length); - - return KERN_SUCCESS; + panic("device_pager_synchronize: memory_object_synchronize no longer supported\n"); + return KERN_FAILURE; } /* @@ -429,16 +479,52 @@ device_pager_synchronize( */ kern_return_t device_pager_map( - __unused memory_object_t mem_obj, + memory_object_t mem_obj, __unused vm_prot_t prot) { + device_pager_t device_object; + + device_object = device_pager_lookup(mem_obj); + + device_pager_lock(device_object); + assert(device_object->ref_count > 0); + if (device_object->is_mapped == FALSE) { + /* + * First mapping of this pager: take an extra reference + * that will remain until all the mappings of this pager + * are removed. + */ + device_object->is_mapped = TRUE; + device_pager_reference(mem_obj); + } + device_pager_unlock(device_object); + return KERN_SUCCESS; } kern_return_t device_pager_last_unmap( - __unused memory_object_t mem_obj) + memory_object_t mem_obj) { + device_pager_t device_object; + boolean_t drop_ref; + + device_object = device_pager_lookup(mem_obj); + + device_pager_lock(device_object); + assert(device_object->ref_count > 0); + if (device_object->is_mapped) { + device_object->is_mapped = FALSE; + drop_ref = TRUE; + } else { + drop_ref = FALSE; + } + device_pager_unlock(device_object); + + if (drop_ref) { + device_pager_deallocate(mem_obj); + } + return KERN_SUCCESS; } @@ -455,11 +541,20 @@ device_object_create(void) device_object = (struct device_pager *) zalloc(device_pager_zone); if (device_object == DEVICE_PAGER_NULL) return(DEVICE_PAGER_NULL); - device_object->pager_ops = &device_pager_ops; - device_object->pager_ikot = IKOT_MEMORY_OBJECT; + + bzero(device_object, sizeof (*device_object)); + + device_object->dev_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; + device_object->dev_pgr_hdr.mo_pager_ops = &device_pager_ops; + device_object->dev_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; + + device_pager_lock_init(device_object); device_object->ref_count = 1; - device_object->control_handle = MEMORY_OBJECT_CONTROL_NULL; + device_object->is_mapped = FALSE; + DTRACE_VM2(device_pager_create, + device_pager_t, device_object, + unsigned int, device_object->ref_count); return(device_object); } diff --git a/osfmk/vm/lz4.c b/osfmk/vm/lz4.c index 7cb4e365f..3c1d5be0a 100644 --- a/osfmk/vm/lz4.c +++ b/osfmk/vm/lz4.c @@ -32,6 +32,7 @@ // early abort detection (Derek Kumar) #include "lz4.h" +#define memcpy __builtin_memcpy size_t lz4raw_decode_buffer(uint8_t * __restrict dst_buffer, size_t dst_size, const uint8_t * __restrict src_buffer, size_t src_size, diff --git a/osfmk/vm/lz4.h b/osfmk/vm/lz4.h index ccade9c11..c6af5edfd 100644 --- a/osfmk/vm/lz4.h +++ b/osfmk/vm/lz4.h @@ -117,3 +117,5 @@ UTIL_FUNCTION void store64(void * ptr,vector_uchar64 data) { *(packed_uchar64 *) UTIL_FUNCTION void copy8(void * dst,const void * src) { store8(dst,load8(src)); } UTIL_FUNCTION void copy16(void * dst,const void * src) { *(packed_uchar16 *)dst = *(const packed_uchar16 *)src; } UTIL_FUNCTION void copy32(void * dst,const void * src) { *(packed_uchar32 *)dst = *(const packed_uchar32 *)src; } + +#undef memcpy diff --git a/osfmk/vm/lz4_assembly_select.h b/osfmk/vm/lz4_assembly_select.h index 7971711f4..c320cad03 100644 --- a/osfmk/vm/lz4_assembly_select.h +++ b/osfmk/vm/lz4_assembly_select.h @@ -32,7 +32,10 @@ // Rule: one define for each assembly source file // To enable assembly -#if defined __ARM_NEON__ +#if defined __arm64__ +#define LZ4_ENABLE_ASSEMBLY_ENCODE_ARM64 1 +#define LZ4_ENABLE_ASSEMBLY_DECODE_ARM64 1 +#elif defined __ARM_NEON__ #define LZ4_ENABLE_ASSEMBLY_ENCODE_ARMV7 1 #define LZ4_ENABLE_ASSEMBLY_DECODE_ARMV7 1 #elif defined __x86_64__ diff --git a/osfmk/vm/memory_object.c b/osfmk/vm/memory_object.c index d4bf4dcd8..d37eb4224 100644 --- a/osfmk/vm/memory_object.c +++ b/osfmk/vm/memory_object.c @@ -1045,79 +1045,17 @@ BYPASS_COW_COPYIN: } -/* - * Routine: memory_object_synchronize_completed [user interface] - * - * Tell kernel that previously synchronized data - * (memory_object_synchronize) has been queue or placed on the - * backing storage. - * - * Note: there may be multiple synchronize requests for a given - * memory object outstanding but they will not overlap. - */ - -kern_return_t -memory_object_synchronize_completed( - memory_object_control_t control, - memory_object_offset_t offset, - memory_object_size_t length) -{ - vm_object_t object; - msync_req_t msr; - - object = memory_object_control_to_vm_object(control); - - XPR(XPR_MEMORY_OBJECT, - "m_o_sync_completed, object 0x%X, offset 0x%X length 0x%X\n", - object, offset, length, 0, 0); - - /* - * Look for bogus arguments - */ - - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); - - vm_object_lock(object); - -/* - * search for sync request structure - */ - queue_iterate(&object->msr_q, msr, msync_req_t, msr_q) { - if (msr->offset == offset && msr->length == length) { - queue_remove(&object->msr_q, msr, msync_req_t, msr_q); - break; - } - }/* queue_iterate */ - - if (queue_end(&object->msr_q, (queue_entry_t)msr)) { - vm_object_unlock(object); - return KERN_INVALID_ARGUMENT; - } - - msr_lock(msr); - vm_object_unlock(object); - msr->flag = VM_MSYNC_DONE; - msr_unlock(msr); - thread_wakeup((event_t) msr); - - return KERN_SUCCESS; -}/* memory_object_synchronize_completed */ - static kern_return_t vm_object_set_attributes_common( vm_object_t object, boolean_t may_cache, - memory_object_copy_strategy_t copy_strategy, - boolean_t temporary, - __unused boolean_t silent_overwrite, - boolean_t advisory_pageout) + memory_object_copy_strategy_t copy_strategy) { boolean_t object_became_ready; XPR(XPR_MEMORY_OBJECT, "m_o_set_attr_com, object 0x%X flg %x strat %d\n", - object, (may_cache&1)|((temporary&1)<1), copy_strategy, 0, 0); + object, (may_cache&1), copy_strategy, 0, 0); if (object == VM_OBJECT_NULL) return(KERN_INVALID_ARGUMENT); @@ -1136,8 +1074,6 @@ vm_object_set_attributes_common( if (may_cache) may_cache = TRUE; - if (temporary) - temporary = TRUE; vm_object_lock(object); @@ -1148,9 +1084,6 @@ vm_object_set_attributes_common( object_became_ready = !object->pager_ready; object->copy_strategy = copy_strategy; object->can_persist = may_cache; - object->temporary = temporary; -// object->silent_overwrite = silent_overwrite; - object->advisory_pageout = advisory_pageout; /* * Wake up anyone waiting for the ready attribute @@ -1167,6 +1100,18 @@ vm_object_set_attributes_common( return(KERN_SUCCESS); } + +kern_return_t +memory_object_synchronize_completed( + __unused memory_object_control_t control, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length) +{ + panic("memory_object_synchronize_completed no longer supported\n"); + return(KERN_FAILURE); +} + + /* * Set the memory object attribute as provided. * @@ -1184,12 +1129,9 @@ memory_object_change_attributes( { vm_object_t object; kern_return_t result = KERN_SUCCESS; - boolean_t temporary; boolean_t may_cache; boolean_t invalidate; memory_object_copy_strategy_t copy_strategy; - boolean_t silent_overwrite; - boolean_t advisory_pageout; object = memory_object_control_to_vm_object(control); if (object == VM_OBJECT_NULL) @@ -1197,12 +1139,8 @@ memory_object_change_attributes( vm_object_lock(object); - temporary = object->temporary; may_cache = object->can_persist; copy_strategy = object->copy_strategy; -// silent_overwrite = object->silent_overwrite; - silent_overwrite = FALSE; - advisory_pageout = object->advisory_pageout; #if notyet invalidate = object->invalidate; #endif @@ -1220,7 +1158,6 @@ memory_object_change_attributes( behave = (old_memory_object_behave_info_t) attributes; - temporary = behave->temporary; invalidate = behave->invalidate; copy_strategy = behave->copy_strategy; @@ -1238,11 +1175,8 @@ memory_object_change_attributes( behave = (memory_object_behave_info_t) attributes; - temporary = behave->temporary; invalidate = behave->invalidate; copy_strategy = behave->copy_strategy; - silent_overwrite = behave->silent_overwrite; - advisory_pageout = behave->advisory_pageout; break; } @@ -1292,7 +1226,6 @@ memory_object_change_attributes( copy_strategy = attr->copy_strategy; may_cache = attr->may_cache_object; - temporary = attr->temporary; break; } @@ -1307,9 +1240,6 @@ memory_object_change_attributes( if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) { copy_strategy = MEMORY_OBJECT_COPY_DELAY; - temporary = TRUE; - } else { - temporary = FALSE; } /* @@ -1318,10 +1248,7 @@ memory_object_change_attributes( */ return (vm_object_set_attributes_common(object, may_cache, - copy_strategy, - temporary, - silent_overwrite, - advisory_pageout)); + copy_strategy)); } kern_return_t @@ -1352,7 +1279,7 @@ memory_object_get_attributes( behave = (old_memory_object_behave_info_t) attributes; behave->copy_strategy = object->copy_strategy; - behave->temporary = object->temporary; + behave->temporary = FALSE; #if notyet /* remove when vm_msync complies and clean in place fini */ behave->invalidate = object->invalidate; #else @@ -1374,14 +1301,13 @@ memory_object_get_attributes( behave = (memory_object_behave_info_t) attributes; behave->copy_strategy = object->copy_strategy; - behave->temporary = object->temporary; + behave->temporary = FALSE; #if notyet /* remove when vm_msync complies and clean in place fini */ behave->invalidate = object->invalidate; #else behave->invalidate = FALSE; #endif - behave->advisory_pageout = object->advisory_pageout; -// behave->silent_overwrite = object->silent_overwrite; + behave->advisory_pageout = FALSE; behave->silent_overwrite = FALSE; *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT; break; @@ -1434,7 +1360,7 @@ memory_object_get_attributes( attr->copy_strategy = object->copy_strategy; attr->cluster_size = PAGE_SIZE; attr->may_cache_object = object->can_persist; - attr->temporary = object->temporary; + attr->temporary = FALSE; *count = MEMORY_OBJECT_ATTR_INFO_COUNT; break; @@ -1459,7 +1385,8 @@ memory_object_iopl_request( upl_t *upl_ptr, upl_page_info_array_t user_page_list, unsigned int *page_list_count, - upl_control_flags_t *flags) + upl_control_flags_t *flags, + vm_tag_t tag) { vm_object_t object; kern_return_t ret; @@ -1512,45 +1439,9 @@ memory_object_iopl_request( named_entry_lock(named_entry); - if (named_entry->is_pager) { - object = vm_object_enter(named_entry->backing.pager, - named_entry->offset + named_entry->size, - named_entry->internal, - FALSE, - FALSE); - if (object == VM_OBJECT_NULL) { - named_entry_unlock(named_entry); - return(KERN_INVALID_OBJECT); - } - - /* JMM - drop reference on pager here? */ - - /* create an extra reference for the named entry */ - vm_object_lock(object); - vm_object_reference_locked(object); - named_entry->backing.object = object; - named_entry->is_pager = FALSE; - named_entry_unlock(named_entry); - - /* wait for object to be ready */ - while (!object->pager_ready) { - vm_object_wait(object, - VM_OBJECT_EVENT_PAGER_READY, - THREAD_UNINT); - vm_object_lock(object); - } - vm_object_unlock(object); - } else { - /* This is the case where we are going to map */ - /* an already mapped object. If the object is */ - /* not ready it is internal. An external */ - /* object cannot be mapped until it is ready */ - /* we can therefore avoid the ready check */ - /* in this case. */ - object = named_entry->backing.object; - vm_object_reference(object); - named_entry_unlock(named_entry); - } + object = named_entry->backing.object; + vm_object_reference(object); + named_entry_unlock(named_entry); } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) { memory_object_control_t control; control = (memory_object_control_t) port; @@ -1582,7 +1473,8 @@ memory_object_iopl_request( upl_ptr, user_page_list, page_list_count, - caller_flags); + caller_flags, + tag); vm_object_deallocate(object); return ret; } @@ -1604,7 +1496,8 @@ memory_object_upl_request( upl_t *upl_ptr, upl_page_info_array_t user_page_list, unsigned int *page_list_count, - int cntrl_flags) + int cntrl_flags, + int tag) { vm_object_t object; @@ -1618,7 +1511,8 @@ memory_object_upl_request( upl_ptr, user_page_list, page_list_count, - (upl_control_flags_t)(unsigned int) cntrl_flags); + (upl_control_flags_t)(unsigned int) cntrl_flags, + tag); } /* @@ -1641,7 +1535,8 @@ memory_object_super_upl_request( upl_t *upl, upl_page_info_t *user_page_list, unsigned int *page_list_count, - int cntrl_flags) + int cntrl_flags, + int tag) { vm_object_t object; @@ -1656,7 +1551,8 @@ memory_object_super_upl_request( upl, user_page_list, page_list_count, - (upl_control_flags_t)(unsigned int) cntrl_flags); + (upl_control_flags_t)(unsigned int) cntrl_flags, + tag); } kern_return_t @@ -1680,12 +1576,6 @@ memory_object_cluster_size(memory_object_control_t control, memory_object_offset } -int vm_stat_discard_cleared_reply = 0; -int vm_stat_discard_cleared_unset = 0; -int vm_stat_discard_cleared_too_late = 0; - - - /* * Routine: host_default_memory_manager [interface] * Purpose: @@ -2102,6 +1992,22 @@ memory_object_control_to_vm_object( return (control->moc_object); } +__private_extern__ vm_object_t +memory_object_to_vm_object( + memory_object_t mem_obj) +{ + memory_object_control_t mo_control; + + if (mem_obj == MEMORY_OBJECT_NULL) { + return VM_OBJECT_NULL; + } + mo_control = mem_obj->mo_control; + if (mo_control == NULL) { + return VM_OBJECT_NULL; + } + return memory_object_control_to_vm_object(mo_control); +} + memory_object_control_t convert_port_to_mo_control( __unused mach_port_t port) @@ -2296,6 +2202,8 @@ kern_return_t memory_object_synchronize vm_sync_t sync_flags ) { + panic("memory_object_syncrhonize no longer supported\n"); + return (memory_object->mo_pager_ops->memory_object_synchronize)( memory_object, offset, diff --git a/osfmk/vm/memory_object.h b/osfmk/vm/memory_object.h index 2b26870ea..d14b5e3c9 100644 --- a/osfmk/vm/memory_object.h +++ b/osfmk/vm/memory_object.h @@ -91,6 +91,9 @@ void memory_object_control_collapse( __private_extern__ vm_object_t memory_object_control_to_vm_object( memory_object_control_t control); +__private_extern__ +vm_object_t memory_object_to_vm_object( + memory_object_t mem_obj); extern mach_port_t convert_mo_control_to_port( @@ -118,11 +121,6 @@ extern ipc_port_t convert_upl_to_port( upl_t ); __private_extern__ void upl_no_senders(ipc_port_t, mach_port_mscount_t); -extern kern_return_t memory_object_free_from_cache( - host_t host, - memory_object_pager_ops_t pager_ops, - int *count); - extern kern_return_t memory_object_pages_resident( memory_object_control_t control, boolean_t * has_pages_resident); diff --git a/osfmk/vm/pmap.h b/osfmk/vm/pmap.h index d907093ac..332c6e909 100644 --- a/osfmk/vm/pmap.h +++ b/osfmk/vm/pmap.h @@ -212,7 +212,7 @@ extern void pmap_set_process(pmap_t pmap, char *procname); #endif /* MACH_ASSERT */ -extern void pmap_enter( /* Enter a mapping */ +extern kern_return_t pmap_enter( /* Enter a mapping */ pmap_t pmap, vm_map_offset_t v, ppnum_t pn, @@ -313,6 +313,16 @@ extern unsigned int (pmap_cache_attributes)( extern void pmap_set_cache_attributes( ppnum_t, unsigned int); +#if defined(__arm__) || defined(__arm64__) +/* ARM64_TODO */ +extern boolean_t pmap_batch_set_cache_attributes( + ppnum_t, + unsigned int, + unsigned int, + unsigned int, + boolean_t, + unsigned int*); +#endif extern void pmap_sync_page_data_phys(ppnum_t pa); extern void pmap_sync_page_attributes_phys(ppnum_t pa); @@ -408,7 +418,8 @@ extern kern_return_t (pmap_attribute)( /* Get/Set special memory /* * Macro to be used in place of pmap_enter() */ -#define PMAP_ENTER(pmap, virtual_address, page, protection, fault_type, flags, wired) \ +#define PMAP_ENTER(pmap, virtual_address, page, protection, fault_type, \ + flags, wired, result) \ MACRO_BEGIN \ pmap_t __pmap = (pmap); \ vm_page_t __page = (page); \ @@ -423,15 +434,15 @@ extern kern_return_t (pmap_attribute)( /* Get/Set special memory if (__page->reusable || __obj->all_reusable) { \ __options |= PMAP_OPTIONS_REUSABLE; \ } \ - (void) pmap_enter_options(__pmap, \ - (virtual_address), \ - VM_PAGE_GET_PHYS_PAGE(__page), \ - (protection), \ - (fault_type), \ - (flags), \ - (wired), \ - __options, \ - NULL); \ + result = pmap_enter_options(__pmap, \ + (virtual_address), \ + VM_PAGE_GET_PHYS_PAGE(__page), \ + (protection), \ + (fault_type), \ + (flags), \ + (wired), \ + __options, \ + NULL); \ MACRO_END #endif /* !PMAP_ENTER */ @@ -475,6 +486,48 @@ extern kern_return_t (pmap_attribute)( /* Get/Set special memory #endif /* PMAP_SET_CACHE_ATTR */ #ifndef PMAP_BATCH_SET_CACHE_ATTR +#if defined(__arm__) || defined(__arm64__) +#define PMAP_BATCH_SET_CACHE_ATTR(object, user_page_list, \ + cache_attr, num_pages, batch_pmap_op) \ + MACRO_BEGIN \ + if ((batch_pmap_op)) { \ + unsigned int __page_idx=0; \ + unsigned int res=0; \ + boolean_t batch=TRUE; \ + while (__page_idx < (num_pages)) { \ + if (!pmap_batch_set_cache_attributes( \ + user_page_list[__page_idx].phys_addr, \ + (cache_attr), \ + (num_pages), \ + (__page_idx), \ + FALSE, \ + (&res))) { \ + batch = FALSE; \ + break; \ + } \ + __page_idx++; \ + } \ + __page_idx=0; \ + res=0; \ + while (__page_idx < (num_pages)) { \ + if (batch) \ + (void)pmap_batch_set_cache_attributes( \ + user_page_list[__page_idx].phys_addr, \ + (cache_attr), \ + (num_pages), \ + (__page_idx), \ + TRUE, \ + (&res)); \ + else \ + pmap_set_cache_attributes( \ + user_page_list[__page_idx].phys_addr, \ + (cache_attr)); \ + __page_idx++; \ + } \ + (object)->set_cache_attr = TRUE; \ + } \ + MACRO_END +#else #define PMAP_BATCH_SET_CACHE_ATTR(object, user_page_list, \ cache_attr, num_pages, batch_pmap_op) \ MACRO_BEGIN \ @@ -489,13 +542,11 @@ extern kern_return_t (pmap_attribute)( /* Get/Set special memory (object)->set_cache_attr = TRUE; \ } \ MACRO_END +#endif #endif /* PMAP_BATCH_SET_CACHE_ATTR */ #define PMAP_ENTER_CHECK(pmap, page) \ { \ - if ((pmap) != kernel_pmap) { \ - ASSERT_PAGE_DECRYPTED(page); \ - } \ if ((page)->error) { \ panic("VM page %p should not have an error\n", \ (page)); \ @@ -632,6 +683,9 @@ extern pmap_t kernel_pmap; /* The kernel's map */ #define PMAP_OPTIONS_CLEAR_REUSABLE 0x400 /* page no longer "reusable" */ #define PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED 0x800 /* credit the compressor * iff page was modified */ +#define PMAP_OPTIONS_PROTECT_IMMEDIATE 0x1000 /* allow protections to be + * be upgraded */ + #if !defined(__LP64__) extern vm_offset_t pmap_extract(pmap_t pmap, @@ -668,6 +722,21 @@ mach_vm_size_t pmap_query_resident(pmap_t pmap, vm_map_offset_t e, mach_vm_size_t *compressed_bytes_p); +/* Inform the pmap layer that there is a JIT entry in this map. */ +extern void pmap_set_jit_entitled(pmap_t pmap); + +/* + * Indicates if any special policy is applied to this protection by the pmap + * layer. + */ +bool pmap_has_prot_policy(vm_prot_t prot); + +/* + * Causes the pmap to return any available pages that it can return cheaply to + * the VM. + */ +void pmap_release_pages_fast(void); + #define PMAP_QUERY_PAGE_PRESENT 0x01 #define PMAP_QUERY_PAGE_REUSABLE 0x02 #define PMAP_QUERY_PAGE_INTERNAL 0x04 diff --git a/osfmk/vm/vm32_user.c b/osfmk/vm/vm32_user.c index 73e0b2bea..f68d9c5aa 100644 --- a/osfmk/vm/vm32_user.c +++ b/osfmk/vm/vm32_user.c @@ -114,7 +114,7 @@ vm32_allocate( kern_return_t result; maddr = *addr; - result = mach_vm_allocate(map, &maddr, size, flags); + result = mach_vm_allocate_external(map, &maddr, size, flags); *addr = CAST_DOWN_EXPLICIT(vm32_offset_t, maddr); return result; @@ -265,7 +265,7 @@ vm32_map_64( kern_return_t result; maddress = *address; - result = mach_vm_map(target_map, &maddress, size, mask, + result = mach_vm_map_external(target_map, &maddress, size, mask, flags, port, offset, copy, cur_protection, max_protection, inheritance); *address = CAST_DOWN_EXPLICIT(vm32_offset_t, maddress); @@ -310,7 +310,7 @@ vm32_remap( kern_return_t result; maddress = *address; - result = mach_vm_remap(target_map, &maddress, size, mask, + result = mach_vm_remap_external(target_map, &maddress, size, mask, anywhere, src_map, memory_address, copy, cur_protection, max_protection, inheritance); *address = CAST_DOWN_EXPLICIT(vm32_offset_t, maddress); diff --git a/osfmk/vm/vm_apple_protect.c b/osfmk/vm/vm_apple_protect.c index 50be9b657..707c3e695 100644 --- a/osfmk/vm/vm_apple_protect.c +++ b/osfmk/vm/vm_apple_protect.c @@ -142,13 +142,14 @@ const struct memory_object_pager_ops apple_protect_pager_ops = { * the "apple protect" EMM. */ typedef struct apple_protect_pager { - struct ipc_object_header pager_header; /* fake ip_kotype() */ - memory_object_pager_ops_t pager_ops; /* == &apple_protect_pager_ops */ + /* mandatory generic header */ + struct memory_object ap_pgr_hdr; + + /* pager-specific data */ queue_chain_t pager_queue; /* next & prev pagers */ unsigned int ref_count; /* reference count */ boolean_t is_ready; /* is this pager ready ? */ boolean_t is_mapped; /* is this mem_obj mapped ? */ - memory_object_control_t pager_control; /* mem object control handle */ vm_object_t backing_object; /* VM obj w/ encrypted data */ vm_object_offset_t backing_offset; vm_object_offset_t crypto_backing_offset; /* for key... */ @@ -157,7 +158,6 @@ typedef struct apple_protect_pager { struct pager_crypt_info *crypt_info; } *apple_protect_pager_t; #define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL) -#define pager_ikot pager_header.io_bits /* * List of memory objects managed by this EMM. @@ -259,7 +259,7 @@ apple_protect_pager_init( memory_object_control_reference(control); - pager->pager_control = control; + pager->ap_pgr_hdr.mo_control = control; attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ @@ -389,7 +389,7 @@ apple_protect_pager_data_request( /* * Gather in a UPL all the VM pages requested by VM. */ - mo_control = pager->pager_control; + mo_control = pager->ap_pgr_hdr.mo_control; upl_size = length; upl_flags = @@ -401,7 +401,7 @@ apple_protect_pager_data_request( pl_count = 0; kr = memory_object_upl_request(mo_control, offset, upl_size, - &upl, NULL, NULL, upl_flags); + &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY); if (kr != KERN_SUCCESS) { retval = kr; goto done; @@ -428,6 +428,7 @@ apple_protect_pager_data_request( 2 * PAGE_SIZE_64, 0, 0, + VM_MAP_KERNEL_FLAGS_NONE, &map_entry); if (kr != KERN_SUCCESS) { vm_object_deallocate(kernel_object); @@ -541,14 +542,20 @@ apple_protect_pager_data_request( src_vaddr = (vm_map_offset_t) PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) << PAGE_SHIFT); +#elif __arm__ || __arm64__ + src_vaddr = (vm_map_offset_t) + phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) + << PAGE_SHIFT); #else - pmap_enter(kernel_pmap, - src_vaddr, - VM_PAGE_GET_PHYS_PAGE(src_page), - VM_PROT_READ, - VM_PROT_NONE, - 0, - TRUE); + kr = pmap_enter(kernel_pmap, + src_vaddr, + VM_PAGE_GET_PHYS_PAGE(src_page), + VM_PROT_READ, + VM_PROT_NONE, + 0, + TRUE); + + assert(kr == KERN_SUCCESS); #endif /* * Establish an explicit pmap mapping of the destination @@ -562,14 +569,19 @@ apple_protect_pager_data_request( #if __x86_64__ dst_vaddr = (vm_map_offset_t) PHYSMAP_PTOV((pmap_paddr_t)dst_pnum << PAGE_SHIFT); +#elif __arm__ || __arm64__ + dst_vaddr = (vm_map_offset_t) + phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT); #else - pmap_enter(kernel_pmap, - dst_vaddr, - dst_pnum, - VM_PROT_READ | VM_PROT_WRITE, - VM_PROT_NONE, - 0, - TRUE); + kr = pmap_enter(kernel_pmap, + dst_vaddr, + dst_pnum, + VM_PROT_READ | VM_PROT_WRITE, + VM_PROT_NONE, + 0, + TRUE); + + assert(kr == KERN_SUCCESS); #endif src_page_object = VM_PAGE_OBJECT(src_page); @@ -911,7 +923,7 @@ apple_protect_pager_terminate_internal( pager->crypt_info = NULL; /* trigger the destruction of the memory object */ - memory_object_destroy(pager->pager_control, 0); + memory_object_destroy(pager->ap_pgr_hdr.mo_control, 0); } /* @@ -963,9 +975,9 @@ apple_protect_pager_deallocate_internal( * pager structure. */ lck_mtx_unlock(&apple_protect_pager_lock); - if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) { - memory_object_control_deallocate(pager->pager_control); - pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; + if (pager->ap_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { + memory_object_control_deallocate(pager->ap_pgr_hdr.mo_control); + pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; } kfree(pager, sizeof (*pager)); pager = APPLE_PROTECT_PAGER_NULL; @@ -1017,21 +1029,13 @@ apple_protect_pager_terminate( */ kern_return_t apple_protect_pager_synchronize( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length, __unused vm_sync_t sync_flags) { - apple_protect_pager_t pager; - - PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %p\n", mem_obj)); - - pager = apple_protect_pager_lookup(mem_obj); - - memory_object_synchronize_completed(pager->pager_control, - offset, length); - - return KERN_SUCCESS; + panic("apple_protect_pager_synchronize: memory_object_synchronize no longer supported\n"); + return KERN_FAILURE; } /* @@ -1120,8 +1124,8 @@ apple_protect_pager_lookup( { apple_protect_pager_t pager; + assert(mem_obj->mo_pager_ops == &apple_protect_pager_ops); pager = (apple_protect_pager_t) mem_obj; - assert(pager->pager_ops == &apple_protect_pager_ops); assert(pager->ref_count > 0); return pager; } @@ -1152,13 +1156,14 @@ apple_protect_pager_create( * we reserve the first word in the object for a fake ip_kotype * setting - that will tell vm_map to use it as a memory object. */ - pager->pager_ops = &apple_protect_pager_ops; - pager->pager_ikot = IKOT_MEMORY_OBJECT; + pager->ap_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; + pager->ap_pgr_hdr.mo_pager_ops = &apple_protect_pager_ops; + pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; + pager->is_ready = FALSE;/* not ready until it has a "name" */ pager->ref_count = 1; /* existence reference (for the cache) */ pager->ref_count++; /* for the caller */ pager->is_mapped = FALSE; - pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; pager->backing_object = backing_object; pager->backing_offset = backing_offset; pager->crypto_backing_offset = crypto_backing_offset; diff --git a/osfmk/vm/vm_compressor.c b/osfmk/vm/vm_compressor.c index b62fd6840..919dbe8b2 100644 --- a/osfmk/vm/vm_compressor.c +++ b/osfmk/vm/vm_compressor.c @@ -37,14 +37,57 @@ #include #include #include +#include #include /* for host_info() */ #include #include +#include +#include +#if !CONFIG_EMBEDDED #include +#endif #include +#if POPCOUNT_THE_COMPRESSED_DATA +boolean_t popcount_c_segs = TRUE; + +static inline uint32_t vmc_pop(uintptr_t ins, int sz) { + uint32_t rv = 0; + + if (__probable(popcount_c_segs == FALSE)) { + return 0xDEAD707C; + } + + while (sz >= 16) { + uint32_t rv1, rv2; + uint64_t *ins64 = (uint64_t *) ins; + uint64_t *ins642 = (uint64_t *) (ins + 8); + rv1 = __builtin_popcountll(*ins64); + rv2 = __builtin_popcountll(*ins642); + rv += rv1 + rv2; + sz -= 16; + ins += 16; + } + + while (sz >= 4) { + uint32_t *ins32 = (uint32_t *) ins; + rv += __builtin_popcount(*ins32); + sz -= 4; + ins += 4; + } + + while (sz > 0) { + char *ins8 = (char *)ins; + rv += __builtin_popcount(*ins8); + sz--; + ins++; + } + return rv; +} +#endif + /* * vm_compressor_mode has a heirarchy of control to set its value. * boot-args are checked first, then device-tree, and finally @@ -52,9 +95,28 @@ * the boot-arg & device-tree code. */ +#if CONFIG_EMBEDDED + +#if CONFIG_FREEZE +int vm_compressor_mode = VM_PAGER_FREEZER_DEFAULT; + +void *freezer_chead; /* The chead used to track c_segs allocated for the exclusive use of holding just one task's compressed memory.*/ +char *freezer_compressor_scratch_buf = NULL; + +#define VM_MAX_FREEZER_CSEG_SWAP_COUNT 64 /* The maximum number of c_segs holding just one task's compressed memory that can be swapped out to disk.*/ +extern int c_freezer_swapout_count; /* This count keeps track of the # of c_segs holding just one task's compressed memory on the swapout queue. This count is used during each freeze i.e. on a per-task basis.*/ + +#else /* CONFIG_FREEZE */ +int vm_compressor_mode = VM_PAGER_NOT_CONFIGURED; +#endif /* CONFIG_FREEZE */ + +int vm_scale = 1; + +#else /* CONFIG_EMBEDDED */ int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP; int vm_scale = 16; +#endif /* CONFIG_EMBEDDED */ int vm_compressor_is_active = 0; int vm_compression_limit = 0; @@ -64,6 +126,15 @@ extern void vm_pageout_io_throttle(void); #if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA extern unsigned int hash_string(char *cp, int len); +static unsigned int vmc_hash(char *, int); +boolean_t checksum_c_segs = TRUE; + +unsigned int vmc_hash(char *cp, int len) { + if (__probable(checksum_c_segs == FALSE)) { + return 0xDEAD7A37; + } + return hash_string(cp, len); +} #endif #define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size) @@ -91,15 +162,6 @@ struct c_sv_hash_entry { #define C_SV_CSEG_ID ((1 << 22) - 1) -struct c_slot_mapping { - uint32_t s_cseg:22, /* segment number + 1 */ - s_cindx:10; /* index in the segment */ -}; -#define C_SLOT_MAX_INDEX (1 << 10) - -typedef struct c_slot_mapping *c_slot_mapping_t; - - union c_segu { c_segment_t c_seg; uintptr_t c_segno; @@ -224,6 +286,8 @@ uint32_t vm_ripe_target_age = (60 * 60 * 48); uint32_t swapout_target_age = 0; uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE]; uint32_t overage_decompressions_during_sample_period = 0; +uint32_t vm_compressor_pages_grabbed = 0; + void do_fastwake_warmup(queue_head_t *, boolean_t); boolean_t fastwake_warmup = FALSE; @@ -242,16 +306,15 @@ int64_t compressor_bytes_used __attribute__((aligned(8))) = 0; struct c_sv_hash_entry c_segment_sv_hash_table[C_SV_HASH_SIZE] __attribute__ ((aligned (8))); - static boolean_t compressor_needs_to_swap(void); static void vm_compressor_swap_trigger_thread(void); static void vm_compressor_do_delayed_compactions(boolean_t); static void vm_compressor_compact_and_swap(boolean_t); static void vm_compressor_age_swapped_in_segments(boolean_t); +#if !CONFIG_EMBEDDED static void vm_compressor_take_paging_space_action(void); - -boolean_t vm_compressor_low_on_space(void); +#endif void compute_swapout_target_age(void); @@ -312,6 +375,17 @@ vm_compressor_low_on_space(void) return (FALSE); } + + +boolean_t +vm_compressor_out_of_space(void) +{ + if ((c_segment_pages_compressed >= c_segment_pages_compressed_limit) || + (c_segment_count >= c_segments_limit)) + return (TRUE); + + return (FALSE); +} int @@ -334,6 +408,7 @@ boolean_t kill_on_no_paging_space = FALSE; /* On compressor/swap exhaustion, kil * its chosen process policy. Controlled by a boot-arg of the same name. */ #endif /* DEVELOPMENT || DEBUG */ +#if !CONFIG_EMBEDDED static uint32_t no_paging_space_action_in_progress = 0; extern void memorystatus_send_low_swap_note(void); @@ -364,6 +439,7 @@ vm_compressor_take_paging_space_action(void) } } } +#endif /* !CONFIG_EMBEDDED */ void @@ -405,12 +481,33 @@ static inline void cslot_copy(c_slot_t cdst, c_slot_t csrc) { #endif #if CHECKSUM_THE_COMPRESSED_DATA cdst->c_hash_compressed_data = csrc->c_hash_compressed_data; +#endif +#if POPCOUNT_THE_COMPRESSED_DATA + cdst->c_pop_cdata = csrc->c_pop_cdata; #endif cdst->c_size = csrc->c_size; cdst->c_packed_ptr = csrc->c_packed_ptr; +#if defined(__arm__) || defined(__arm64__) + cdst->c_codec = csrc->c_codec; +#endif } vm_map_t compressor_map; +uint64_t compressor_pool_max_size; +uint64_t compressor_pool_size; +uint32_t compressor_pool_multiplier; + +#if DEVELOPMENT || DEBUG +/* + * Compressor segments are write-protected in development/debug + * kernels to help debug memory corruption. + * In cases where performance is a concern, this can be disabled + * via the boot-arg "-disable_cseg_write_protection". + */ +boolean_t write_protect_c_segs = TRUE; +int vm_compressor_test_seg_wp; +uint32_t vm_ktrace_enabled; +#endif /* DEVELOPMENT || DEBUG */ void vm_compressor_init(void) @@ -420,9 +517,11 @@ vm_compressor_init(void) c_slot_t cs = &cs_dummy; int c_segment_min_size; int c_segment_padded_size; + int attempts = 1; kern_return_t retval = KERN_SUCCESS; vm_offset_t start_addr = 0; vm_size_t c_segments_arr_size = 0, compressor_submap_size = 0; + vm_map_kernel_flags_t vmk_flags; #if RECORD_THE_COMPRESSED_DATA vm_size_t c_compressed_record_sbuf_size = 0; #endif /* RECORD_THE_COMPRESSED_DATA */ @@ -432,6 +531,24 @@ vm_compressor_init(void) if (PE_parse_boot_argn("-kill_on_no_paging_space", bootarg_name, sizeof (bootarg_name))) { kill_on_no_paging_space = TRUE; } + if (PE_parse_boot_argn("-disable_cseg_write_protection", bootarg_name, sizeof (bootarg_name))) { + write_protect_c_segs = FALSE; + } + int vmcval = 1; + PE_parse_boot_argn("vm_compressor_validation", &vmcval, sizeof(vmcval)); + + if (kern_feature_override(KF_COMPRSV_OVRD)) { + vmcval = 0; + } + if (vmcval == 0) { +#if POPCOUNT_THE_COMPRESSED_DATA + popcount_c_segs = FALSE; +#endif +#if CHECKSUM_THE_DATA || CHECKSUM_THE_COMPRESSED_DATA + checksum_c_segs = FALSE; +#endif + write_protect_c_segs = FALSE; + } #endif /* DEVELOPMENT || DEBUG */ /* @@ -453,6 +570,12 @@ vm_compressor_init(void) PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit)); +#ifdef CONFIG_EMBEDDED + vm_compressor_minorcompact_threshold_divisor = 20; + vm_compressor_majorcompact_threshold_divisor = 30; + vm_compressor_unthrottle_threshold_divisor = 40; + vm_compressor_catchup_threshold_divisor = 60; +#else if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) { vm_compressor_minorcompact_threshold_divisor = 11; vm_compressor_majorcompact_threshold_divisor = 13; @@ -464,6 +587,7 @@ vm_compressor_init(void) vm_compressor_unthrottle_threshold_divisor = 35; vm_compressor_catchup_threshold_divisor = 50; } +#endif /* * vm_page_init_lck_grp is now responsible for calling vm_compressor_init_locks * c_master_lock needs to be available early so that "vm_page_find_contiguous" can @@ -482,43 +606,67 @@ vm_compressor_init(void) queue_init(&c_swappedout_list_head); queue_init(&c_swappedout_sparse_list_head); - c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot)); - - for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1); - - compressor_segment_zone = zinit(c_segment_padded_size, 128000 * c_segment_padded_size, PAGE_SIZE, "compressor_segment"); - zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE); - zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE); - - c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot); - c_free_segno_head = -1; c_segments_available = 0; - if (vm_compression_limit == 0) { - c_segment_pages_compressed_limit = (uint32_t)((max_mem / PAGE_SIZE)) * vm_scale; + if (vm_compression_limit) + compressor_pool_size = (uint64_t)vm_compression_limit * PAGE_SIZE_64; -#define OLD_SWAP_LIMIT (1024 * 1024 * 16) -#define MAX_SWAP_LIMIT (1024 * 1024 * 128) - - if (c_segment_pages_compressed_limit > (OLD_SWAP_LIMIT)) - c_segment_pages_compressed_limit = OLD_SWAP_LIMIT; + compressor_pool_max_size = C_SEG_MAX_LIMIT; + compressor_pool_max_size *= C_SEG_BUFSIZE; - if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE_64)) - c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE_64); - } else { - if (vm_compression_limit < MAX_SWAP_LIMIT) - c_segment_pages_compressed_limit = vm_compression_limit; +#if defined(__x86_64__) + + if (vm_compression_limit == 0) { + + if (max_mem <= (4ULL * 1024ULL * 1024ULL * 1024ULL)) + compressor_pool_size = 16ULL * max_mem; + else if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL)) + compressor_pool_size = 8ULL * max_mem; + else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL)) + compressor_pool_size = 4ULL * max_mem; else - c_segment_pages_compressed_limit = MAX_SWAP_LIMIT; + compressor_pool_size = 2ULL * max_mem; } - if ((c_segments_limit = c_segment_pages_compressed_limit / (C_SEG_BUFSIZE / PAGE_SIZE)) > C_SEG_MAX_LIMIT) - c_segments_limit = C_SEG_MAX_LIMIT; + if (max_mem <= (8ULL * 1024ULL * 1024ULL * 1024ULL)) + compressor_pool_multiplier = 1; + else if (max_mem <= (32ULL * 1024ULL * 1024ULL * 1024ULL)) + compressor_pool_multiplier = 2; + else + compressor_pool_multiplier = 4; - c_segment_pages_compressed_nearing_limit = (c_segment_pages_compressed_limit * 98) / 100; - c_segments_nearing_limit = (c_segments_limit * 98) / 100; +#elif defined(__arm__) - c_segments_busy = FALSE; +#define VM_RESERVE_SIZE (1024 * 1024 * 256) +#define MAX_COMPRESSOR_POOL_SIZE (1024 * 1024 * 450) + + if (compressor_pool_max_size > MAX_COMPRESSOR_POOL_SIZE) + compressor_pool_max_size = MAX_COMPRESSOR_POOL_SIZE; + + if (vm_compression_limit == 0) + compressor_pool_size = ((kernel_map->max_offset - kernel_map->min_offset) - kernel_map->size) - VM_RESERVE_SIZE; + compressor_pool_multiplier = 1; +#else + if (compressor_pool_max_size > max_mem) + compressor_pool_max_size = max_mem; + + if (vm_compression_limit == 0) + compressor_pool_size = max_mem; + compressor_pool_multiplier = 1; +#endif + if (compressor_pool_size > compressor_pool_max_size) + compressor_pool_size = compressor_pool_max_size; + +try_again: + c_segments_limit = (uint32_t)(compressor_pool_size / (vm_size_t)(C_SEG_ALLOCSIZE)); + c_segments_nearing_limit = (uint32_t)(((uint64_t)c_segments_limit * 98ULL) / 100ULL); + + c_segment_pages_compressed_limit = (c_segments_limit * (C_SEG_BUFSIZE / PAGE_SIZE) * compressor_pool_multiplier); + + if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE)) + c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE); + + c_segment_pages_compressed_nearing_limit = (uint32_t)(((uint64_t)c_segment_pages_compressed_limit * 98ULL) / 100ULL); /* * Submap needs space for: @@ -526,7 +674,7 @@ vm_compressor_init(void) * - c_buffers * - swap reclaimations -- C_SEG_BUFSIZE */ - c_segments_arr_size = vm_map_round_page((sizeof(union c_segu) * c_segments_limit),VM_MAP_PAGE_MASK(kernel_map)); + c_segments_arr_size = vm_map_round_page((sizeof(union c_segu) * c_segments_limit), VM_MAP_PAGE_MASK(kernel_map)); c_buffers_size = vm_map_round_page(((vm_size_t)C_SEG_ALLOCSIZE * (vm_size_t)c_segments_limit), VM_MAP_PAGE_MASK(kernel_map)); compressor_submap_size = c_segments_arr_size + c_buffers_size + C_SEG_BUFSIZE; @@ -536,18 +684,39 @@ vm_compressor_init(void) compressor_submap_size += c_compressed_record_sbuf_size; #endif /* RECORD_THE_COMPRESSED_DATA */ + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_permanent = TRUE; retval = kmem_suballoc(kernel_map, &start_addr, compressor_submap_size, - FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT | VM_MAKE_TAG(0), + FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_COMPRESSOR, &compressor_map); - if (retval != KERN_SUCCESS) - panic("vm_compressor_init: kmem_suballoc failed"); + if (retval != KERN_SUCCESS) { + if (++attempts > 3) + panic("vm_compressor_init: kmem_suballoc failed - 0x%llx", (uint64_t)compressor_submap_size); + compressor_pool_size = compressor_pool_size / 2; + + kprintf("retrying creation of the compressor submap at 0x%llx bytes\n", compressor_pool_size); + goto try_again; + } if (kernel_memory_allocate(compressor_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) panic("vm_compressor_init: kernel_memory_allocate failed - c_segments\n"); if (kernel_memory_allocate(compressor_map, &c_buffers, c_buffers_size, 0, KMA_COMPRESSOR | KMA_VAONLY | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR) != KERN_SUCCESS) panic("vm_compressor_init: kernel_memory_allocate failed - c_buffers\n"); + + c_segment_min_size = sizeof(struct c_segment) + (C_SEG_SLOT_VAR_ARRAY_MIN_LEN * sizeof(struct c_slot)); + + for (c_segment_padded_size = 128; c_segment_padded_size < c_segment_min_size; c_segment_padded_size = c_segment_padded_size << 1); + + compressor_segment_zone = zinit(c_segment_padded_size, c_segments_limit * c_segment_padded_size, PAGE_SIZE, "compressor_segment"); + zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE); + zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE); + + c_seg_fixed_array_len = (c_segment_padded_size - sizeof(struct c_segment)) / sizeof(struct c_slot); + + c_segments_busy = FALSE; + c_segments_next_page = (caddr_t)c_segments; vm_compressor_algorithm_init(); @@ -579,7 +748,7 @@ vm_compressor_init(void) #endif if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL, - BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) { + BASEPRI_VM, &thread) != KERN_SUCCESS) { panic("vm_compressor_swap_trigger_thread: create failed"); } thread_deallocate(thread); @@ -610,7 +779,6 @@ c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact) { int c_indx; int32_t bytes_used; - int32_t bytes_unused; uint32_t c_rounded_size; uint32_t c_size; c_slot_t cs; @@ -626,7 +794,6 @@ c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact) panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size); } bytes_used = 0; - bytes_unused = 0; for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) { @@ -639,8 +806,11 @@ c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact) bytes_used += c_rounded_size; #if CHECKSUM_THE_COMPRESSED_DATA - if (c_size && cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size)) - panic("compressed data doesn't match original"); + unsigned csvhash; + if (c_size && cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) { + addr64_t csvphys = kvtophys((vm_offset_t)&c_seg->c_store.c_buffer[cs->c_offset]); + panic("Compressed data doesn't match original %p phys: 0x%llx %d %p %d %d 0x%x 0x%x", c_seg, csvphys, cs->c_offset, cs, c_indx, c_size, cs->c_hash_compressed_data, csvhash); + } #endif } @@ -793,7 +963,7 @@ c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, bo * in the next major compaction sweep... if we don't do this * we will eventually run into the c_segments_limit */ - if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT(c_seg)) { + if (c_seg->c_state == C_ON_MAJORCOMPACT_Q && C_SEG_SHOULD_MAJORCOMPACT_NOW(c_seg)) { c_seg_switch_state(c_seg, C_ON_AGE_Q, FALSE); } @@ -1053,6 +1223,7 @@ c_seg_free_locked(c_segment_t c_seg) uint64_t c_swap_handle = 0; assert(c_seg->c_busy); + assert(c_seg->c_slots_used == 0); assert(!c_seg->c_on_minorcompact_q); assert(!c_seg->c_busy_swapping); @@ -1088,7 +1259,10 @@ c_seg_free_locked(c_segment_t c_seg) vm_swap_free(c_swap_handle); } lck_mtx_lock_spin_always(&c_seg->c_lock); - + /* + * c_seg must remain busy until + * after the call to vm_swap_free + */ C_SEG_WAKEUP_DONE(c_seg); lck_mtx_unlock_always(&c_seg->c_lock); @@ -1116,8 +1290,9 @@ c_seg_free_locked(c_segment_t c_seg) zfree(compressor_segment_zone, c_seg); } - +#if DEVELOPMENT || DEBUG int c_seg_trim_page_count = 0; +#endif void c_seg_trim_tail(c_segment_t c_seg) @@ -1146,13 +1321,16 @@ c_seg_trim_tail(c_segment_t c_seg) c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size); c_seg->c_nextoffset = c_offset; - c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1); + c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & + ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1); if (c_seg->c_firstemptyslot > c_seg->c_nextslot) c_seg->c_firstemptyslot = c_seg->c_nextslot; - +#if DEVELOPMENT || DEBUG c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) - - round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE); + round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / + PAGE_SIZE); +#endif } break; } @@ -1188,7 +1366,9 @@ c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy) if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE) goto done; - + +/* TODO: assert first emptyslot's c_size is actually 0 */ + #if DEVELOPMENT || DEBUG C_SEG_MAKE_WRITEABLE(c_seg); #endif @@ -1212,7 +1392,7 @@ c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy) continue; c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK; - +/* N.B.: This memcpy may be an overlapping copy */ memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_rounded_size); cslot_copy(c_dst, c_src); @@ -1420,6 +1600,7 @@ c_seg_major_compact( if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot) c_seg_dst->c_firstemptyslot++; + c_seg_dst->c_slots_used++; c_seg_dst->c_nextslot++; c_seg_dst->c_bytes_used += c_rounded_size; c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size); @@ -1430,6 +1611,9 @@ c_seg_major_compact( c_seg_src->c_bytes_unused += c_rounded_size; c_seg_src->c_firstemptyslot = 0; + assert(c_seg_src->c_slots_used); + c_seg_src->c_slots_used--; + if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX_INDEX) { /* dest segment is now full */ keep_compacting = FALSE; @@ -1684,12 +1868,12 @@ compressor_needs_to_swap(void) should_swap = TRUE; #if CONFIG_JETSAM - if (should_swap || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) { + if (should_swap || vm_compressor_low_on_space() == TRUE) { if (vm_compressor_thrashing_detected == FALSE) { vm_compressor_thrashing_detected = TRUE; - if (swapout_target_age || c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) { + if (swapout_target_age || vm_compressor_low_on_space() == TRUE) { memorystatus_kill_on_VM_thrashing(TRUE /* async */); compressor_thrashing_induced_jetsam++; } else { @@ -1960,7 +2144,9 @@ vm_compressor_do_delayed_compactions(boolean_t flush_all) boolean_t needs_to_swap = FALSE; +#if !CONFIG_EMBEDDED LCK_MTX_ASSERT(c_list_lock, LCK_MTX_ASSERT_OWNED); +#endif /* !CONFIG_EMBEDDED */ while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) { @@ -2124,7 +2310,7 @@ vm_compressor_swap_trigger_thread(void) if (vm_restricted_to_single_processor == TRUE) thread_vm_bind_group_add(); - + thread_set_thread_name(current_thread(), "VM_cswap_trigger"); compaction_swapper_init_now = 0; } lck_mtx_lock_spin_always(c_list_lock); @@ -2440,8 +2626,10 @@ vm_compressor_compact_and_swap(boolean_t flush_all) needs_to_swap = compressor_needs_to_swap(); +#if !CONFIG_EMBEDDED if (needs_to_swap == TRUE && vm_swap_low_on_space()) vm_compressor_take_paging_space_action(); +#endif /* !CONFIG_EMBEDDED */ lck_mtx_lock_spin_always(c_list_lock); @@ -2611,8 +2799,10 @@ c_seg_allocate(c_segment_t *current_chead) int min_needed; int size_to_populate; +#if !CONFIG_EMBEDDED if (vm_compressor_low_on_space()) vm_compressor_take_paging_space_action(); +#endif /* !CONFIG_EMBEDDED */ if ( (c_seg = *current_chead) == NULL ) { uint32_t c_segno; @@ -2713,6 +2903,7 @@ c_seg_allocate(c_segment_t *current_chead) if (size_to_populate > C_SEG_MAX_POPULATE_SIZE) size_to_populate = C_SEG_MAX_POPULATE_SIZE; + vm_compressor_pages_grabbed += size_to_populate / PAGE_SIZE; kernel_memory_populate(compressor_map, (vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset], @@ -2744,8 +2935,17 @@ c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead) unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset)); +#ifndef _OPEN_SOURCE + /* TODO: The HW codec can generate, lazily, a '2nd page not mapped' + * exception. So on such a platform, or platforms where we're confident + * the codec does not require a buffer page to absorb trailing writes, + * we can create an unmapped hole at the tail of the segment, rather + * than a populated mapping. This will also guarantee that the codec + * does not overwrite valid data past the edge of the segment and + * thus eliminate the depopulation overhead. + */ +#endif if (unused_bytes) { - offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))); /* @@ -2909,13 +3109,10 @@ c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_ if (vm_swap_get(c_seg, f_offset, io_size) != KERN_SUCCESS) { PAGE_REPLACEMENT_DISALLOWED(TRUE); - c_seg->c_store.c_swap_handle = f_offset; - kernel_memory_depopulate(compressor_map, addr, io_size, KMA_COMPRESSOR); c_seg_swapin_requeue(c_seg, FALSE, TRUE, age_on_swapin_q); } else { - c_seg->c_store.c_buffer = (int32_t*) addr; #if ENCRYPTED_SWAP vm_swap_decrypt(c_seg); #endif /* ENCRYPTED_SWAP */ @@ -2924,7 +3121,7 @@ c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_ if (c_seg->cseg_swap_size != io_size) panic("swapin size doesn't match swapout size"); - if (c_seg->cseg_hash != hash_string((char*) c_seg->c_store.c_buffer, (int)io_size)) { + if (c_seg->cseg_hash != vmc_hash((char*) c_seg->c_store.c_buffer, (int)io_size)) { panic("c_seg_swapin - Swap hash mismatch\n"); } #endif /* CHECKSUM_THE_SWAP */ @@ -2938,6 +3135,9 @@ c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction, boolean_t age_ if (force_minor_compaction == TRUE) { if (c_seg_minor_compaction_and_unlock(c_seg, FALSE)) { /* + * c_seg was completely empty so it was freed, + * so be careful not to reference it again + * * Drop the rwlock_count so that the thread priority * is returned back to where it is supposed to be. */ @@ -3055,8 +3255,9 @@ c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0); retry: - if ((c_seg = c_seg_allocate(current_chead)) == NULL) + if ((c_seg = c_seg_allocate(current_chead)) == NULL) { return (1); + } /* * returns with c_seg lock held * and PAGE_REPLACEMENT_DISALLOWED(TRUE)... @@ -3078,30 +3279,71 @@ retry: max_csize = PAGE_SIZE; #if CHECKSUM_THE_DATA - cs->c_hash_data = hash_string(src, PAGE_SIZE); + cs->c_hash_data = vmc_hash(src, PAGE_SIZE); #endif + boolean_t incomp_copy = FALSE; + int max_csize_adj = (max_csize - 4); if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) { +#if defined(__arm__) || defined(__arm64__) + uint16_t ccodec = CINVALID; + + if (max_csize >= C_SEG_OFFSET_ALIGNMENT_BOUNDARY) { + c_size = metacompressor((const uint8_t *) src, + (uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset], + max_csize_adj, &ccodec, + scratch_buf, &incomp_copy); +#if C_SEG_OFFSET_ALIGNMENT_BOUNDARY > 4 + if (c_size > max_csize_adj) { + c_size = -1; + } +#endif + } else { + c_size = -1; + } + assert(ccodec == CCWK || ccodec == CCLZ4); + cs->c_codec = ccodec; +#endif } else { +#if defined(__arm__) || defined(__arm64__) + cs->c_codec = CCWK; +#endif +#if defined(__arm64__) + __unreachable_ok_push + if (PAGE_SIZE == 4096) + c_size = WKdm_compress_4k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset], + (WK_word *)(uintptr_t)scratch_buf, max_csize_adj); + else { + c_size = WKdm_compress_16k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset], + (WK_word *)(uintptr_t)scratch_buf, max_csize_adj); + } + __unreachable_ok_pop +#else c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset], - (WK_word *)(uintptr_t)scratch_buf, max_csize - 4); + (WK_word *)(uintptr_t)scratch_buf, max_csize_adj); +#endif } - assert(c_size <= (max_csize - 4) && c_size >= -1); + assertf(((c_size <= max_csize_adj) && (c_size >= -1)), + "c_size invalid (%d, %d), cur compressions: %d", c_size, max_csize_adj, c_segment_pages_compressed); if (c_size == -1) { - if (max_csize < PAGE_SIZE) { c_current_seg_filled(c_seg, current_chead); assert(*current_chead == NULL); lck_mtx_unlock_always(&c_seg->c_lock); - + /* TODO: it may be worth requiring codecs to distinguish + * between incompressible inputs and failures due to + * budget exhaustion. + */ PAGE_REPLACEMENT_DISALLOWED(FALSE); goto retry; } c_size = PAGE_SIZE; - memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size); + if (incomp_copy == FALSE) { + memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size); + } OSAddAtomic(1, &c_segment_noncompressible_pages); @@ -3133,15 +3375,18 @@ retry: #if RECORD_THE_COMPRESSED_DATA c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size); #endif - #if CHECKSUM_THE_COMPRESSED_DATA - cs->c_hash_compressed_data = hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size); + cs->c_hash_compressed_data = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size); +#endif +#if POPCOUNT_THE_COMPRESSED_DATA + cs->c_pop_cdata = vmc_pop((uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset], c_size); #endif c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK; PACK_C_SIZE(cs, c_size); c_seg->c_bytes_used += c_rounded_size; c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size); + c_seg->c_slots_used++; slot_ptr->s_cindx = c_seg->c_nextslot++; /* would mean "empty slot", so use csegno+1 */ @@ -3191,6 +3436,7 @@ static inline void sv_decompress(int32_t *ddst, int32_t pattern) { * is currently not a constant. */ + __unreachable_ok_push if (PAGE_SIZE == 4096) { for (i = 0; i < (4096U / sizeof(int32_t)); i += 4) { *ddst++ = pattern; @@ -3207,6 +3453,7 @@ static inline void sv_decompress(int32_t *ddst, int32_t pattern) { *ddst++ = pattern; } } + __unreachable_ok_pop #endif } @@ -3229,7 +3476,7 @@ c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int panic("C_KDP passed to decompress page from outside of debugger context"); } - assert((flags & C_KEEP) == C_KEEP); + assert((flags & C_KEEP) == C_KEEP); assert((flags & C_DONT_BLOCK) == C_DONT_BLOCK); if ((flags & (C_DONT_BLOCK | C_KEEP)) != (C_DONT_BLOCK | C_KEEP)) { @@ -3259,7 +3506,7 @@ ReTry: * to disk... in this state we allow freeing of compressed * pages and must honor the C_DONT_BLOCK case */ - if (dst && decompressions_blocked == TRUE) { + if (__improbable(dst && decompressions_blocked == TRUE)) { if (flags & C_DONT_BLOCK) { if (__probable(!kdp_mode)) { @@ -3360,13 +3607,25 @@ bypass_busy_check: } if (c_seg->c_state == C_ON_BAD_Q) { assert(c_seg->c_store.c_buffer == NULL); + *zeroslot = 0; retval = -1; - goto c_seg_invalid_data; + goto done; + } + +#if POPCOUNT_THE_COMPRESSED_DATA + unsigned csvpop; + uintptr_t csvaddr = (uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset]; + if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) { + panic("Compressed data popcount doesn't match original, bit distance: %d %p (phys: %p) %p %p 0x%llx 0x%x 0x%x 0x%x", (csvpop - cs->c_pop_cdata), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, cs->c_offset, c_size, csvpop, cs->c_pop_cdata); } +#endif + #if CHECKSUM_THE_COMPRESSED_DATA - if (cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size)) - panic("compressed data doesn't match original hash: 0x%x, seg: %p, offset: %d, c_size: %d", cs->c_hash_compressed_data, c_seg, cs->c_offset, c_size); + unsigned csvhash; + if (cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) { + panic("Compressed data doesn't match original %p %p %u %u %u", c_seg, cs, c_size, cs->c_hash_compressed_data, csvhash); + } #endif if (c_rounded_size == PAGE_SIZE) { /* @@ -3406,16 +3665,38 @@ bypass_busy_check: } if (vm_compressor_algorithm() != VM_COMPRESSOR_DEFAULT_CODEC) { +#if defined(__arm__) || defined(__arm64__) + uint16_t c_codec = cs->c_codec; + metadecompressor((const uint8_t *) &c_seg->c_store.c_buffer[cs->c_offset], + (uint8_t *)dst, c_size, c_codec, (void *)scratch_buf); +#endif } else { +#if defined(__arm64__) + __unreachable_ok_push + if (PAGE_SIZE == 4096) + WKdm_decompress_4k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset], + (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size); + else { + WKdm_decompress_16k((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset], + (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size); + } + __unreachable_ok_pop +#else WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset], (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size); +#endif } } #if CHECKSUM_THE_DATA - if (cs->c_hash_data != hash_string(dst, PAGE_SIZE)) - panic("decompressed data doesn't match original cs: %p, hash: %d, offset: %d, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size); - + if (cs->c_hash_data != vmc_hash(dst, PAGE_SIZE)) { +#if defined(__arm__) || defined(__arm64__) + int32_t *dinput = &c_seg->c_store.c_buffer[cs->c_offset]; + panic("decompressed data doesn't match original cs: %p, hash: 0x%x, offset: %d, c_size: %d, c_rounded_size: %d, codec: %d, header: 0x%x 0x%x 0x%x", cs, cs->c_hash_data, cs->c_offset, c_size, c_rounded_size, cs->c_codec, *dinput, *(dinput + 1), *(dinput + 2)); +#else + panic("decompressed data doesn't match original cs: %p, hash: %d, offset: 0x%x, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size); +#endif + } #endif if (c_seg->c_swappedin_ts == 0 && !kdp_mode) { @@ -3430,8 +3711,6 @@ bypass_busy_check: OSAddAtomic(1, &sample_period_decompression_count); } } -c_seg_invalid_data: - if (flags & C_KEEP) { *zeroslot = 0; goto done; @@ -3440,6 +3719,10 @@ c_seg_invalid_data: c_seg->c_bytes_unused += c_rounded_size; c_seg->c_bytes_used -= c_rounded_size; + + assert(c_seg->c_slots_used); + c_seg->c_slots_used--; + PACK_C_SIZE(cs, 0); if (c_indx < c_seg->c_firstemptyslot) @@ -3499,7 +3782,7 @@ c_seg_invalid_data: assert(c_seg->c_state != C_ON_BAD_Q); - if (C_SEG_SHOULD_MINORCOMPACT(c_seg)) { + if (C_SEG_SHOULD_MINORCOMPACT_NOW(c_seg)) { c_seg_try_minor_compaction_and_unlock(c_seg); need_unlock = FALSE; } @@ -3527,6 +3810,10 @@ done: if (consider_defragmenting == TRUE) vm_swap_consider_defragmenting(); +#if CONFIG_EMBEDDED + if ((c_minor_count && COMPRESSOR_NEEDS_TO_MINOR_COMPACT()) || vm_compressor_needs_to_major_compact()) + vm_wake_compactor_swapper(); +#endif return (retval); } @@ -3542,6 +3829,8 @@ vm_compressor_get(ppnum_t pn, int *slot, int flags) #if __x86_64__ dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT); +#elif __arm__ || __arm64__ + dst = (char *) phystokv((pmap_paddr_t)pn << PAGE_SHIFT); #else #error "unsupported architecture" #endif @@ -3644,9 +3933,12 @@ vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_bu #if __x86_64__ src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT); +#elif __arm__ || __arm64__ + src = (char *) phystokv((pmap_paddr_t)pn << PAGE_SHIFT); #else #error "unsupported architecture" #endif + retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf); return (retval); @@ -3877,7 +4169,7 @@ Relookup_src: c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot); memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size); - +//is platform alignment actually necessary since wkdm aligns its output? c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK; cslot_copy(c_dst, c_src); @@ -3886,6 +4178,7 @@ Relookup_src: if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot) c_seg_dst->c_firstemptyslot++; + c_seg_dst->c_slots_used++; c_seg_dst->c_nextslot++; c_seg_dst->c_bytes_used += c_rounded_size; c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size); @@ -3895,6 +4188,9 @@ Relookup_src: c_seg_src->c_bytes_used -= c_rounded_size; c_seg_src->c_bytes_unused += c_rounded_size; + + assert(c_seg_src->c_slots_used); + c_seg_src->c_slots_used--; if (c_indx < c_seg_src->c_firstemptyslot) { c_seg_src->c_firstemptyslot = c_indx; diff --git a/osfmk/vm/vm_compressor.h b/osfmk/vm/vm_compressor.h index 6c877bbe6..ad2f586fe 100644 --- a/osfmk/vm/vm_compressor.h +++ b/osfmk/vm/vm_compressor.h @@ -39,17 +39,36 @@ #include +#if defined(__arm64__) +#include +#endif + #define C_SEG_OFFSET_BITS 16 #define C_SEG_BUFSIZE (1024 * 256) #define C_SEG_MAX_PAGES (C_SEG_BUFSIZE / PAGE_SIZE) +#if CONFIG_EMBEDDED +#define C_SEG_OFF_LIMIT (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 512))) +#define C_SEG_ALLOCSIZE (C_SEG_BUFSIZE + PAGE_SIZE) +#else #define C_SEG_OFF_LIMIT (C_SEG_BYTES_TO_OFFSET((C_SEG_BUFSIZE - 128))) #define C_SEG_ALLOCSIZE (C_SEG_BUFSIZE) +#endif #define C_SEG_MAX_POPULATE_SIZE (4 * PAGE_SIZE) +#if defined(__arm64__) + +#if DEVELOPMENT || DEBUG + + +#endif + +#endif + #if DEBUG || COMPRESSOR_INTEGRITY_CHECKS #define ENABLE_SWAP_CHECKS 1 #define ENABLE_COMPRESSOR_CHECKS 1 +#define POPCOUNT_THE_COMPRESSED_DATA (1) #else #define ENABLE_SWAP_CHECKS 0 #define ENABLE_COMPRESSOR_CHECKS 0 @@ -64,15 +83,27 @@ struct c_slot { uint64_t c_offset:C_SEG_OFFSET_BITS, +#if defined(__arm64__) + c_size:14, + c_codec:1, + c_packed_ptr:33; +#elif defined(__arm__) + c_size:12, + c_codec:1, + c_packed_ptr:35; +#else c_size:12, c_packed_ptr:36; +#endif #if CHECKSUM_THE_DATA unsigned int c_hash_data; #endif #if CHECKSUM_THE_COMPRESSED_DATA unsigned int c_hash_compressed_data; #endif - +#if POPCOUNT_THE_COMPRESSED_DATA + unsigned int c_pop_cdata; +#endif }; #define C_IS_EMPTY 0 @@ -92,27 +123,29 @@ struct c_segment { queue_chain_t c_age_list; queue_chain_t c_list; - uint64_t c_generation_id; - int32_t c_bytes_used; - int32_t c_bytes_unused; - -#define C_SEG_MAX_LIMIT (1 << 19) /* this needs to track the size of c_mysegno */ - uint32_t c_mysegno:19, +#define C_SEG_MAX_LIMIT (1 << 20) /* this needs to track the size of c_mysegno */ + uint32_t c_mysegno:20, c_busy:1, c_busy_swapping:1, - c_wanted:1, + c_wanted:1, c_on_minorcompact_q:1, /* can also be on the age_q, the majorcompact_q or the swappedin_q */ c_state:4, /* what state is the segment in which dictates which q to find it on */ c_overage_swap:1, - c_reserved:4; + c_reserved:3; + + uint32_t c_creation_ts; + uint64_t c_generation_id; + + int32_t c_bytes_used; + int32_t c_bytes_unused; + uint32_t c_slots_used; uint16_t c_firstemptyslot; uint16_t c_nextslot; uint32_t c_nextoffset; uint32_t c_populated_offset; - uint32_t c_creation_ts; uint32_t c_swappedin_ts; union { @@ -139,6 +172,16 @@ struct c_segment { struct c_slot c_slot_fixed_array[0]; }; + +struct c_slot_mapping { + uint32_t s_cseg:22, /* segment number + 1 */ + s_cindx:10; /* index in the segment */ +}; +#define C_SLOT_MAX_INDEX (1 << 10) + +typedef struct c_slot_mapping *c_slot_mapping_t; + + #define C_SEG_SLOT_VAR_ARRAY_MIN_LEN C_SEG_MAX_PAGES extern int c_seg_fixed_array_len; @@ -151,13 +194,30 @@ extern vm_offset_t c_buffers; #define C_SEG_BYTES_TO_OFFSET(bytes) ((bytes) / (int) sizeof(int32_t)) #define C_SEG_UNUSED_BYTES(cseg) (cseg->c_bytes_unused + (C_SEG_OFFSET_TO_BYTES(cseg->c_populated_offset - cseg->c_nextoffset))) +//todo opensource -#define C_SEG_OFFSET_ALIGNMENT_MASK 0x3 +#ifndef __PLATFORM_WKDM_ALIGNMENT_MASK__ +#define C_SEG_OFFSET_ALIGNMENT_MASK 0x3ULL +#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY 0x4 +#else +#define C_SEG_OFFSET_ALIGNMENT_MASK __PLATFORM_WKDM_ALIGNMENT_MASK__ +#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY __PLATFORM_WKDM_ALIGNMENT_BOUNDARY__ +#endif + +#define C_SEG_SHOULD_MINORCOMPACT_NOW(cseg) ((C_SEG_UNUSED_BYTES(cseg) >= (C_SEG_BUFSIZE / 4)) ? 1 : 0) -#define C_SEG_ONDISK_IS_SPARSE(cseg) ((cseg->c_bytes_used < (C_SEG_BUFSIZE / 2)) ? 1 : 0) -#define C_SEG_SHOULD_MINORCOMPACT(cseg) ((C_SEG_UNUSED_BYTES(cseg) >= (C_SEG_BUFSIZE / 3)) ? 1 : 0) -#define C_SEG_SHOULD_MAJORCOMPACT(cseg) (((cseg->c_bytes_unused + (C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))) >= (C_SEG_BUFSIZE / 8)) ? 1 : 0) +/* + * the decsion to force a c_seg to be major compacted is based on 2 criteria + * 1) is the c_seg buffer almost empty (i.e. we have a chance to merge it with another c_seg) + * 2) are there at least a minimum number of slots unoccupied so that we have a chance + * of combining this c_seg with another one. + */ +#define C_SEG_SHOULD_MAJORCOMPACT_NOW(cseg) \ + ((((cseg->c_bytes_unused + (C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))) >= (C_SEG_BUFSIZE / 8)) && \ + ((C_SLOT_MAX_INDEX - cseg->c_slots_used) > (C_SEG_BUFSIZE / PAGE_SIZE))) \ + ? 1 : 0) +#define C_SEG_ONDISK_IS_SPARSE(cseg) ((cseg->c_bytes_used < cseg->c_bytes_unused) ? 1 : 0) #define C_SEG_IS_ONDISK(cseg) ((cseg->c_state == C_ON_SWAPPEDOUT_Q || cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q)) @@ -182,25 +242,37 @@ extern vm_offset_t c_buffers; MACRO_END -#if DEVELOPMENT || DEBUG extern vm_map_t compressor_map; +#if DEVELOPMENT || DEBUG +extern boolean_t write_protect_c_segs; +extern int vm_compressor_test_seg_wp; + #define C_SEG_MAKE_WRITEABLE(cseg) \ MACRO_BEGIN \ - vm_map_protect(compressor_map, \ - (vm_map_offset_t)cseg->c_store.c_buffer, \ - (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\ - VM_PROT_READ | VM_PROT_WRITE, \ - 0); \ + if (write_protect_c_segs) { \ + vm_map_protect(compressor_map, \ + (vm_map_offset_t)cseg->c_store.c_buffer, \ + (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\ + VM_PROT_READ | VM_PROT_WRITE, \ + 0); \ + } \ MACRO_END #define C_SEG_WRITE_PROTECT(cseg) \ MACRO_BEGIN \ - vm_map_protect(compressor_map, \ - (vm_map_offset_t)cseg->c_store.c_buffer, \ - (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\ - VM_PROT_READ, \ - 0); \ + if (write_protect_c_segs) { \ + vm_map_protect(compressor_map, \ + (vm_map_offset_t)cseg->c_store.c_buffer, \ + (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(C_SEG_ALLOCSIZE)],\ + VM_PROT_READ, \ + 0); \ + } \ + if (vm_compressor_test_seg_wp) { \ + volatile uint32_t vmtstmp = *(volatile uint32_t *)cseg->c_store.c_buffer; \ + *(volatile uint32_t *)cseg->c_store.c_buffer = 0xDEADABCD; \ + (void) vmtstmp; \ + } \ MACRO_END #endif @@ -299,7 +371,13 @@ extern uint64_t vm_compressor_compute_elapsed_msecs(clock_sec_t, clock_nsec_t, c #define VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 1)) #define VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 1)) +#ifdef CONFIG_EMBEDDED +#define AVAILABLE_NON_COMPRESSED_MIN 20000 +#define COMPRESSOR_NEEDS_TO_SWAP() (((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) || \ + (AVAILABLE_NON_COMPRESSED_MEMORY < AVAILABLE_NON_COMPRESSED_MIN)) ? 1 : 0) +#else #define COMPRESSOR_NEEDS_TO_SWAP() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0) +#endif #define VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE() \ (vm_compressor_mode == VM_PAGER_COMPRESSOR_WITH_SWAP && \ @@ -309,7 +387,11 @@ extern uint64_t vm_compressor_compute_elapsed_msecs(clock_sec_t, clock_nsec_t, c #define COMPRESSOR_NEEDS_TO_MINOR_COMPACT() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0) +#ifdef CONFIG_EMBEDDED +#define COMPRESSOR_FREE_RESERVED_LIMIT 28 +#else #define COMPRESSOR_FREE_RESERVED_LIMIT 128 +#endif uint32_t vm_compressor_get_encode_scratch_size(void); uint32_t vm_compressor_get_decode_scratch_size(void); @@ -322,3 +404,14 @@ extern void c_compressed_record_write(char *, int); #endif extern lck_mtx_t *c_list_lock; + +#if DEVELOPMENT || DEBUG +extern uint32_t vm_ktrace_enabled; + +#define VMKDBG(x, ...) \ +MACRO_BEGIN \ +if (vm_ktrace_enabled) { \ + KDBG(x, ## __VA_ARGS__);\ +} \ +MACRO_END +#endif diff --git a/osfmk/vm/vm_compressor_algorithms.c b/osfmk/vm/vm_compressor_algorithms.c index e7135a7fc..3af35c8ec 100644 --- a/osfmk/vm/vm_compressor_algorithms.c +++ b/osfmk/vm/vm_compressor_algorithms.c @@ -35,6 +35,10 @@ #include #define MZV_MAGIC (17185) +#if defined(__arm64__) +#include +#endif + #define LZ4_SCRATCH_ALIGN (64) #define WKC_SCRATCH_ALIGN (64) @@ -100,14 +104,24 @@ enum compressor_preselect_t { vm_compressor_mode_t vm_compressor_current_codec = VM_COMPRESSOR_DEFAULT_CODEC; +boolean_t vm_compressor_force_sw_wkdm = FALSE; + boolean_t verbose = FALSE; -#if DEVELOPMENT || DEBUG -#define VERBOSE(x...) \ +#define VMDBGSTAT (DEBUG) +#if VMDBGSTATS +#define VM_COMPRESSOR_STAT_DBG(x...) \ do { \ - if (verbose) \ - printf(x); \ + (x); \ } while(0) +#else +#define VM_COMPRESSOR_STAT_DBG(x...) \ + do { \ + } while (0) +#endif + +#define VMCSTATS (DEVELOPMENT || DEBUG) +#if VMCSTATS #define VM_COMPRESSOR_STAT(x...) \ do { \ (x); \ @@ -118,9 +132,6 @@ boolean_t verbose = FALSE; (x); \ } while(0) #else -#define VERBOSE(x...) \ - do { \ - }while (0) #define VM_COMPRESSOR_STAT(x...) \ do { \ }while (0) @@ -199,28 +210,76 @@ static inline void compressor_selector_update(int lz4sz, int didwk, int wksz) { } } + +static inline void WKdm_hv(uint32_t *wkbuf) { +#if DEVELOPMENT || DEBUG + uint32_t *inw = (uint32_t *) wkbuf; + if (*inw != MZV_MAGIC) { + if ((*inw | *(inw + 1) | *(inw + 2)) & 0xFFFF0000) { + panic("WKdm(%p): invalid header 0x%x 0x%x 0x%x\n", wkbuf, *inw, *(inw +1), *(inw+2)); + } + } +#else /* DEVELOPMENT || DEBUG */ + (void) wkbuf; +#endif +} + //todo fix clang diagnostic #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wincompatible-pointer-types" +#if defined(__arm64__) +#endif + static inline void WKdmD(WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes) { -#if DEVELOPMENT || DEBUG - uint32_t *inw = (uint32_t *) src_buf; - if (*inw != MZV_MAGIC) { - if ((*inw | *(inw+1) | *(inw+2)) & 0xFFFF0000) { - panic("WKdmDecompress: invalid header 0x%x 0x%x 0x%x\n", *inw, *(inw +1), *(inw+2)); - } +#if defined(__arm64__) +#endif + WKdm_hv(src_buf); +#if defined(__arm64__) + if (PAGE_SIZE == 4096) { + WKdm_decompress_4k(src_buf, dest_buf, scratch, bytes); + } else { + __unused uint64_t wdsstart; + + VM_COMPRESSOR_STAT_DBG(wdsstart = mach_absolute_time()); + WKdm_decompress_16k(src_buf, dest_buf, scratch, bytes); + + VM_COMPRESSOR_STAT_DBG(compressor_stats.wks_dabstime += mach_absolute_time() - wdsstart); + VM_COMPRESSOR_STAT(compressor_stats.wks_decompressions++); } -#endif /* DEVELOPMENT || DEBUG */ +#else /* !defined arm64 */ WKdm_decompress_new(src_buf, dest_buf, scratch, bytes); +#endif } +#if DEVELOPMENT || DEBUG +int precompy, wkswhw; +#endif -static inline int WKdmC(WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int limit) { - return WKdm_compress_new(src_buf, dest_buf, scratch, limit); +static inline int WKdmC(WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, boolean_t *incomp_copy, unsigned int limit) { + (void)incomp_copy; + int wkcval; +#if defined(__arm64__) + if (PAGE_SIZE == 4096) { + wkcval = WKdm_compress_4k(src_buf, dest_buf, scratch, limit); + } else { + __unused uint64_t wcswstart; + + VM_COMPRESSOR_STAT_DBG(wcswstart = mach_absolute_time()); + + int wkswsz = WKdm_compress_16k(src_buf, dest_buf, scratch, limit); + + VM_COMPRESSOR_STAT_DBG(compressor_stats.wks_cabstime += mach_absolute_time() - wcswstart); + VM_COMPRESSOR_STAT(compressor_stats.wks_compressions++); + wkcval = wkswsz; + } +#else + wkcval = WKdm_compress_new(src_buf, dest_buf, scratch, limit); +#endif + return wkcval; } -int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t *codec, void *cscratchin) { +int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t *codec, void *cscratchin, boolean_t *incomp_copy) { int sz = -1; int dowk = FALSE, dolz4 = FALSE, skiplz4 = FALSE; int insize = PAGE_SIZE; @@ -246,10 +305,9 @@ int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t if (dowk) { *codec = CCWK; - sz = WKdmC(in, cdst, &cscratch->wkscratch[0], outbufsz); VM_COMPRESSOR_STAT(compressor_stats.wk_compressions++); + sz = WKdmC(in, cdst, &cscratch->wkscratch[0], incomp_copy, outbufsz); - VERBOSE("WKDm Compress: %d\n", sz); if (sz == -1) { VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=PAGE_SIZE); VM_COMPRESSOR_STAT(compressor_stats.wk_compression_failures++); @@ -260,7 +318,7 @@ int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t goto cexit; } else if (sz == 0) { VM_COMPRESSOR_STAT(compressor_stats.wk_sv_compressions++); - VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=8); + VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=4); } else { VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_total+=sz); } @@ -270,7 +328,9 @@ lz4eval: if (((sz == -1) || (sz >= vmctune.lz4_threshold)) && (skiplz4 == FALSE)) { dolz4 = TRUE; } else { - __unused int wkc = (sz == -1) ? PAGE_SIZE : sz; +#if DEVELOPMENT || DEBUG + int wkc = (sz == -1) ? PAGE_SIZE : sz; +#endif VM_COMPRESSOR_STAT(compressor_stats.wk_compressions_exclusive++); VM_COMPRESSOR_STAT(compressor_stats.wk_compressed_bytes_exclusive+=wkc); goto cexit; @@ -288,7 +348,6 @@ lz4compress: sz = (int) lz4raw_encode_buffer(cdst, outbufsz, in, insize, &cscratch->lz4state[0]); - VERBOSE("LZ4 Compress: %d\n", sz); compressor_selector_update(sz, dowk, wksz); if (sz == 0) { sz = -1; @@ -308,12 +367,16 @@ void metadecompressor(const uint8_t *source, uint8_t *dest, uint32_t csize, uint rval = (int)lz4raw_decode_buffer(dest, PAGE_SIZE, source, csize, &compressor_dscratch->lz4decodestate[0]); VM_DECOMPRESSOR_STAT(compressor_stats.lz4_decompressions+=1); VM_DECOMPRESSOR_STAT(compressor_stats.lz4_decompressed_bytes+=csize); - - assertf(rval == PAGE_SIZE, "LZ4 decode: size != pgsize %d", rval); - +#if DEVELOPMENT || DEBUG + uint32_t *d32 = dest; +#endif + assertf(rval == PAGE_SIZE, "LZ4 decode: size != pgsize %d, header: 0x%x, 0x%x, 0x%x", + rval, *d32, *(d32+1), *(d32+2)); } else { assert(ccodec == CCWK); + WKdmD(source, dest, &compressor_dscratch->wkdecompscratch[0], csize); + VM_DECOMPRESSOR_STAT(compressor_stats.wk_decompressions+=1); VM_DECOMPRESSOR_STAT(compressor_stats.wk_decompressed_bytes+=csize); } @@ -344,18 +407,27 @@ int vm_compressor_algorithm(void) { void vm_compressor_algorithm_init(void) { vm_compressor_mode_t new_codec = VM_COMPRESSOR_DEFAULT_CODEC; +#if defined(__arm64__) + new_codec = CMODE_HYB; + + if (PAGE_SIZE == 16384) { + vmctune.lz4_threshold = 12288; + } +#endif PE_parse_boot_argn("vm_compressor_codec", &new_codec, sizeof(new_codec)); assertf(((new_codec == VM_COMPRESSOR_DEFAULT_CODEC) || (new_codec == CMODE_WK) || (new_codec == CMODE_LZ4) || (new_codec = CMODE_HYB)), "Invalid VM compression codec: %u", new_codec); - - if (PE_parse_boot_argn("-vm_compressor_wk", &new_codec, sizeof(new_codec))) { +#if defined(__arm__)||defined(__arm64__) + uint32_t tmpc; + if (PE_parse_boot_argn("-vm_compressor_wk", &tmpc, sizeof(tmpc))) { new_codec = VM_COMPRESSOR_DEFAULT_CODEC; - } else if (PE_parse_boot_argn("-vm_compressor_hybrid", &new_codec, sizeof(new_codec))) { + } else if (PE_parse_boot_argn("-vm_compressor_hybrid", &tmpc, sizeof(tmpc))) { new_codec = CMODE_HYB; } + vm_compressor_current_codec = new_codec; +#endif /* arm/arm64 */ } -//TODO check open-sourceability of lz4 diff --git a/osfmk/vm/vm_compressor_algorithms.h b/osfmk/vm/vm_compressor_algorithms.h index dce1ea150..46d022f5e 100644 --- a/osfmk/vm/vm_compressor_algorithms.h +++ b/osfmk/vm/vm_compressor_algorithms.h @@ -41,6 +41,7 @@ typedef struct { uint64_t lz4_post_wk_compressions; uint64_t wk_compressions; + uint64_t wk_cabstime; uint64_t wk_sv_compressions; uint64_t wk_mzv_compressions; uint64_t wk_compression_failures; @@ -48,11 +49,27 @@ typedef struct { uint64_t wk_compressions_exclusive; uint64_t wk_compressed_bytes_exclusive; + uint64_t wkh_compressions; + uint64_t wkh_cabstime; + uint64_t wks_compressions; + uint64_t wks_cabstime; + uint64_t wks_compressed_bytes; + uint64_t wks_compression_failures; + uint64_t wks_sv_compressions; + uint64_t lz4_decompressions; uint64_t lz4_decompressed_bytes; uint64_t uc_decompressions; uint64_t wk_decompressions; + uint64_t wk_dabstime; + + uint64_t wkh_decompressions; + uint64_t wkh_dabstime; + + uint64_t wks_decompressions; + uint64_t wks_dabstime; + uint64_t wk_decompressed_bytes; uint64_t wk_sv_decompressions; } compressor_stats_t; @@ -73,7 +90,7 @@ typedef struct { extern compressor_tuneables_t vmctune; -int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t *codec, void *cscratch); +int metacompressor(const uint8_t *in, uint8_t *cdst, int32_t outbufsz, uint16_t *codec, void *cscratch, boolean_t *); void metadecompressor(const uint8_t *source, uint8_t *dest, uint32_t csize, uint16_t ccodec, void *compressor_dscratch); typedef enum { diff --git a/osfmk/vm/vm_compressor_backing_store.c b/osfmk/vm/vm_compressor_backing_store.c index 7015971be..4a2eb0fea 100644 --- a/osfmk/vm/vm_compressor_backing_store.c +++ b/osfmk/vm/vm_compressor_backing_store.c @@ -57,6 +57,8 @@ boolean_t vm_swappin_enabled = FALSE; unsigned int vm_swapfile_total_segs_alloced = 0; unsigned int vm_swapfile_total_segs_used = 0; +char swapfilename[MAX_SWAPFILENAME_LEN + 1] = SWAP_FILE_NAME; + extern vm_map_t compressor_map; @@ -102,12 +104,28 @@ static void vm_swap_free_now(struct swapfile *swf, uint64_t f_offset); static void vm_swapout_thread(void); static void vm_swapfile_create_thread(void); static void vm_swapfile_gc_thread(void); -static void vm_swap_defragment(); +static void vm_swap_defragment(void); static void vm_swap_handle_delayed_trims(boolean_t); -static void vm_swap_do_delayed_trim(); +static void vm_swap_do_delayed_trim(struct swapfile *); static void vm_swap_wait_on_trim_handling_in_progress(void); +#if CONFIG_EMBEDDED +/* + * Only 1 swap file currently allowed. + */ +#define VM_MAX_SWAP_FILE_NUM 1 +#define VM_SWAPFILE_DELAYED_TRIM_MAX 4 + +#define VM_SWAP_SHOULD_DEFRAGMENT() (c_swappedout_sparse_count > (vm_swapfile_total_segs_used / 16) ? 1 : 0) +#define VM_SWAP_SHOULD_RECLAIM() FALSE +#define VM_SWAP_SHOULD_ABORT_RECLAIM() FALSE +#define VM_SWAP_SHOULD_PIN(_size) FALSE +#define VM_SWAP_SHOULD_CREATE(cur_ts) ((vm_num_swap_files < VM_MAX_SWAP_FILE_NUM) && ((vm_swapfile_total_segs_alloced - vm_swapfile_total_segs_used) < (unsigned int)VM_SWAPFILE_HIWATER_SEGS) && \ + ((cur_ts - vm_swapfile_last_failed_to_create_ts) > VM_SWAPFILE_DELAYED_CREATE) ? 1 : 0) +#define VM_SWAP_SHOULD_TRIM(swf) ((swf->swp_delayed_trim_count >= VM_SWAPFILE_DELAYED_TRIM_MAX) ? 1 : 0) + +#else /* CONFIG_EMBEDDED */ #define VM_MAX_SWAP_FILE_NUM 100 #define VM_SWAPFILE_DELAYED_TRIM_MAX 128 @@ -120,6 +138,7 @@ static void vm_swap_wait_on_trim_handling_in_progress(void); ((cur_ts - vm_swapfile_last_failed_to_create_ts) > VM_SWAPFILE_DELAYED_CREATE) ? 1 : 0) #define VM_SWAP_SHOULD_TRIM(swf) ((swf->swp_delayed_trim_count >= VM_SWAPFILE_DELAYED_TRIM_MAX) ? 1 : 0) +#endif /* CONFIG_EMBEDDED */ #define VM_SWAPFILE_DELAYED_CREATE 15 @@ -139,15 +158,6 @@ void c_compressed_record_init(void); void c_compressed_record_write(char *, int); #endif -#if ENCRYPTED_SWAP -extern boolean_t swap_crypt_ctx_initialized; -extern void swap_crypt_ctx_initialize(void); -extern const unsigned char swap_crypt_null_iv[AES_BLOCK_SIZE]; -extern aes_ctx swap_crypt_ctx; -extern unsigned long vm_page_encrypt_counter; -extern unsigned long vm_page_decrypt_counter; -#endif /* ENCRYPTED_SWAP */ - extern void vm_pageout_io_throttle(void); static struct swapfile *vm_swapfile_for_handle(uint64_t); @@ -185,146 +195,116 @@ vm_swapfile_for_handle(uint64_t f_offset) return swf; } -void -vm_compressor_swap_init() -{ - thread_t thread = NULL; - - lck_grp_attr_setdefault(&vm_swap_data_lock_grp_attr); - lck_grp_init(&vm_swap_data_lock_grp, - "vm_swap_data", - &vm_swap_data_lock_grp_attr); - lck_attr_setdefault(&vm_swap_data_lock_attr); - lck_mtx_init_ext(&vm_swap_data_lock, - &vm_swap_data_lock_ext, - &vm_swap_data_lock_grp, - &vm_swap_data_lock_attr); - - queue_init(&swf_global_queue); - - - if (kernel_thread_start_priority((thread_continue_t)vm_swapout_thread, NULL, - BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) { - panic("vm_swapout_thread: create failed"); - } - vm_swapout_thread_id = thread->thread_id; - - thread_deallocate(thread); - - if (kernel_thread_start_priority((thread_continue_t)vm_swapfile_create_thread, NULL, - BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) { - panic("vm_swapfile_create_thread: create failed"); - } - - thread_deallocate(thread); - - if (kernel_thread_start_priority((thread_continue_t)vm_swapfile_gc_thread, NULL, - BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) { - panic("vm_swapfile_gc_thread: create failed"); - } - thread_deallocate(thread); - - proc_set_thread_policy_with_tid(kernel_task, thread->thread_id, - TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2); - proc_set_thread_policy_with_tid(kernel_task, thread->thread_id, - TASK_POLICY_INTERNAL, TASK_POLICY_PASSIVE_IO, TASK_POLICY_ENABLE); - #if ENCRYPTED_SWAP - if (swap_crypt_ctx_initialized == FALSE) { - swap_crypt_ctx_initialize(); - } -#endif /* ENCRYPTED_SWAP */ - - memset(swapfilename, 0, MAX_SWAPFILENAME_LEN + 1); - printf("VM Swap Subsystem is ON\n"); -} +#include +extern u_int32_t random(void); /* from */ +#define SWAP_CRYPT_AES_KEY_SIZE 128 /* XXX 192 and 256 don't work ! */ -#if RECORD_THE_COMPRESSED_DATA +boolean_t swap_crypt_ctx_initialized; +void swap_crypt_ctx_initialize(void); -void -c_compressed_record_init() -{ - if (c_compressed_record_init_done == FALSE) { - vm_swapfile_open("/tmp/compressed_data", &c_compressed_record_vp); - c_compressed_record_init_done = TRUE; - } -} +aes_ctx swap_crypt_ctx; +const unsigned char swap_crypt_null_iv[AES_BLOCK_SIZE] = {0xa, }; +uint32_t swap_crypt_key[8]; /* big enough for a 256 key */ -void -c_compressed_record_write(char *buf, int size) -{ - if (c_compressed_record_write_error == 0) { - c_compressed_record_write_error = vm_record_file_write(c_compressed_record_vp, c_compressed_record_file_offset, buf, size); - c_compressed_record_file_offset += size; - } -} -#endif +unsigned long vm_page_encrypt_counter; +unsigned long vm_page_decrypt_counter; -int compaction_swapper_inited = 0; +#if DEBUG +boolean_t swap_crypt_ctx_tested = FALSE; +unsigned char swap_crypt_test_page_ref[4096] __attribute__((aligned(4096))); +unsigned char swap_crypt_test_page_encrypt[4096] __attribute__((aligned(4096))); +unsigned char swap_crypt_test_page_decrypt[4096] __attribute__((aligned(4096))); +#endif /* DEBUG */ +/* + * Initialize the encryption context: key and key size. + */ +void swap_crypt_ctx_initialize(void); /* forward */ void -vm_compaction_swapper_do_init(void) +swap_crypt_ctx_initialize(void) { - struct vnode *vp; - char *pathname; - int namelen; - - if (compaction_swapper_inited) - return; + unsigned int i; - if (vm_compressor_mode != VM_PAGER_COMPRESSOR_WITH_SWAP) { - compaction_swapper_inited = 1; - return; + /* + * No need for locking to protect swap_crypt_ctx_initialized + * because the first use of encryption will come from the + * pageout thread (we won't pagein before there's been a pageout) + * and there's only one pageout thread. + */ + if (swap_crypt_ctx_initialized == FALSE) { + for (i = 0; + i < (sizeof (swap_crypt_key) / + sizeof (swap_crypt_key[0])); + i++) { + swap_crypt_key[i] = random(); + } + aes_encrypt_key((const unsigned char *) swap_crypt_key, + SWAP_CRYPT_AES_KEY_SIZE, + &swap_crypt_ctx.encrypt); + aes_decrypt_key((const unsigned char *) swap_crypt_key, + SWAP_CRYPT_AES_KEY_SIZE, + &swap_crypt_ctx.decrypt); + swap_crypt_ctx_initialized = TRUE; } - lck_mtx_lock(&vm_swap_data_lock); - if ( !compaction_swapper_inited) { - - if (strlen(swapfilename) == 0) { - /* - * If no swapfile name has been set, we'll - * use the default name. - * - * Also, this function is only called from the vm_pageout_scan thread - * via vm_consider_waking_compactor_swapper, - * so we don't need to worry about a race in checking/setting the name here. - */ - strlcpy(swapfilename, SWAP_FILE_NAME, MAX_SWAPFILENAME_LEN); +#if DEBUG + /* + * Validate the encryption algorithms. + */ + if (swap_crypt_ctx_tested == FALSE) { + /* initialize */ + for (i = 0; i < 4096; i++) { + swap_crypt_test_page_ref[i] = (char) i; } - namelen = (int)strlen(swapfilename) + SWAPFILENAME_INDEX_LEN + 1; - pathname = (char*)kalloc(namelen); - memset(pathname, 0, namelen); - snprintf(pathname, namelen, "%s%d", swapfilename, 0); - - vm_swapfile_open(pathname, &vp); - - if (vp) { - - if (vnode_pager_isSSD(vp) == FALSE) { - vm_compressor_minorcompact_threshold_divisor = 18; - vm_compressor_majorcompact_threshold_divisor = 22; - vm_compressor_unthrottle_threshold_divisor = 32; + /* encrypt */ + aes_encrypt_cbc(swap_crypt_test_page_ref, + swap_crypt_null_iv, + PAGE_SIZE / AES_BLOCK_SIZE, + swap_crypt_test_page_encrypt, + &swap_crypt_ctx.encrypt); + /* decrypt */ + aes_decrypt_cbc(swap_crypt_test_page_encrypt, + swap_crypt_null_iv, + PAGE_SIZE / AES_BLOCK_SIZE, + swap_crypt_test_page_decrypt, + &swap_crypt_ctx.decrypt); + /* compare result with original */ + for (i = 0; i < 4096; i ++) { + if (swap_crypt_test_page_decrypt[i] != + swap_crypt_test_page_ref[i]) { + panic("encryption test failed"); } - vnode_setswapmount(vp); - vm_swappin_avail = vnode_getswappin_avail(vp); + } - if (vm_swappin_avail) - vm_swappin_enabled = TRUE; - vm_swapfile_close((uint64_t)pathname, vp); + /* encrypt again */ + aes_encrypt_cbc(swap_crypt_test_page_decrypt, + swap_crypt_null_iv, + PAGE_SIZE / AES_BLOCK_SIZE, + swap_crypt_test_page_decrypt, + &swap_crypt_ctx.encrypt); + /* decrypt in place */ + aes_decrypt_cbc(swap_crypt_test_page_decrypt, + swap_crypt_null_iv, + PAGE_SIZE / AES_BLOCK_SIZE, + swap_crypt_test_page_decrypt, + &swap_crypt_ctx.decrypt); + for (i = 0; i < 4096; i ++) { + if (swap_crypt_test_page_decrypt[i] != + swap_crypt_test_page_ref[i]) { + panic("in place encryption test failed"); + } } - kfree(pathname, namelen); - compaction_swapper_inited = 1; + swap_crypt_ctx_tested = TRUE; } - lck_mtx_unlock(&vm_swap_data_lock); +#endif /* DEBUG */ } - -#if ENCRYPTED_SWAP void vm_swap_encrypt(c_segment_t c_seg) { @@ -426,6 +406,143 @@ vm_swap_decrypt(c_segment_t c_seg) #endif /* ENCRYPTED_SWAP */ +void +vm_compressor_swap_init() +{ + thread_t thread = NULL; + + lck_grp_attr_setdefault(&vm_swap_data_lock_grp_attr); + lck_grp_init(&vm_swap_data_lock_grp, + "vm_swap_data", + &vm_swap_data_lock_grp_attr); + lck_attr_setdefault(&vm_swap_data_lock_attr); + lck_mtx_init_ext(&vm_swap_data_lock, + &vm_swap_data_lock_ext, + &vm_swap_data_lock_grp, + &vm_swap_data_lock_attr); + + queue_init(&swf_global_queue); + + + if (kernel_thread_start_priority((thread_continue_t)vm_swapout_thread, NULL, + BASEPRI_VM, &thread) != KERN_SUCCESS) { + panic("vm_swapout_thread: create failed"); + } + vm_swapout_thread_id = thread->thread_id; + + thread_deallocate(thread); + + if (kernel_thread_start_priority((thread_continue_t)vm_swapfile_create_thread, NULL, + BASEPRI_VM, &thread) != KERN_SUCCESS) { + panic("vm_swapfile_create_thread: create failed"); + } + + thread_deallocate(thread); + + if (kernel_thread_start_priority((thread_continue_t)vm_swapfile_gc_thread, NULL, + BASEPRI_VM, &thread) != KERN_SUCCESS) { + panic("vm_swapfile_gc_thread: create failed"); + } + thread_deallocate(thread); + + proc_set_thread_policy_with_tid(kernel_task, thread->thread_id, + TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2); + proc_set_thread_policy_with_tid(kernel_task, thread->thread_id, + TASK_POLICY_INTERNAL, TASK_POLICY_PASSIVE_IO, TASK_POLICY_ENABLE); + +#if ENCRYPTED_SWAP + if (swap_crypt_ctx_initialized == FALSE) { + swap_crypt_ctx_initialize(); + } +#endif /* ENCRYPTED_SWAP */ + +#if CONFIG_EMBEDDED + /* + * dummy value until the swap file gets created + * when we drive the first c_segment_t to the + * swapout queue... at that time we will + * know the true size we have to work with + */ + c_overage_swapped_limit = 16; +#endif + printf("VM Swap Subsystem is ON\n"); +} + + +#if RECORD_THE_COMPRESSED_DATA + +void +c_compressed_record_init() +{ + if (c_compressed_record_init_done == FALSE) { + vm_swapfile_open("/tmp/compressed_data", &c_compressed_record_vp); + c_compressed_record_init_done = TRUE; + } +} + +void +c_compressed_record_write(char *buf, int size) +{ + if (c_compressed_record_write_error == 0) { + c_compressed_record_write_error = vm_record_file_write(c_compressed_record_vp, c_compressed_record_file_offset, buf, size); + c_compressed_record_file_offset += size; + } +} +#endif + + +int compaction_swapper_inited = 0; + +void +vm_compaction_swapper_do_init(void) +{ + struct vnode *vp; + char *pathname; + int namelen; + + if (compaction_swapper_inited) + return; + + if (vm_compressor_mode != VM_PAGER_COMPRESSOR_WITH_SWAP) { + compaction_swapper_inited = 1; + return; + } + lck_mtx_lock(&vm_swap_data_lock); + + if ( !compaction_swapper_inited) { + + namelen = (int)strlen(swapfilename) + SWAPFILENAME_INDEX_LEN + 1; + pathname = (char*)kalloc(namelen); + memset(pathname, 0, namelen); + snprintf(pathname, namelen, "%s%d", swapfilename, 0); + + vm_swapfile_open(pathname, &vp); + + if (vp) { + + if (vnode_pager_isSSD(vp) == FALSE) { + vm_compressor_minorcompact_threshold_divisor = 18; + vm_compressor_majorcompact_threshold_divisor = 22; + vm_compressor_unthrottle_threshold_divisor = 32; + } +#if !CONFIG_EMBEDDED + vnode_setswapmount(vp); + vm_swappin_avail = vnode_getswappin_avail(vp); + + if (vm_swappin_avail) + vm_swappin_enabled = TRUE; +#endif + vm_swapfile_close((uint64_t)pathname, vp); + } + kfree(pathname, namelen); + + compaction_swapper_inited = 1; + } + lck_mtx_unlock(&vm_swap_data_lock); +} + + + void vm_swap_consider_defragmenting() { @@ -945,18 +1062,6 @@ vm_swap_create_file() if (swap_file_reuse == FALSE) { - if (strlen(swapfilename) == 0) { - /* - * If no swapfile name has been set, we'll - * use the default name. - * - * Also, this function is only called from the swapfile management thread. - * So we don't need to worry about a race in checking/setting the name here. - */ - - strlcpy(swapfilename, SWAP_FILE_NAME, MAX_SWAPFILENAME_LEN); - } - namelen = (int)strlen(swapfilename) + SWAPFILENAME_INDEX_LEN + 1; swf = (struct swapfile*) kalloc(sizeof *swf); @@ -1039,6 +1144,15 @@ vm_swap_create_file() lck_mtx_unlock(&vm_swap_data_lock); thread_wakeup((event_t) &vm_num_swap_files); +#if CONFIG_EMBEDDED + if (vm_num_swap_files == 1) { + + c_overage_swapped_limit = (uint32_t)size / C_SEG_BUFSIZE; + + if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) + c_overage_swapped_limit /= 2; + } +#endif break; } else { @@ -1085,7 +1199,7 @@ vm_swap_get(c_segment_t c_seg, uint64_t f_offset, uint64_t size) C_SEG_MAKE_WRITEABLE(c_seg); #endif file_offset = (f_offset & SWAP_SLOT_MASK); - retval = vm_swapfile_io(swf->swp_vp, file_offset, c_seg->c_store.c_buffer, (int)(size / PAGE_SIZE_64), SWAP_READ); + retval = vm_swapfile_io(swf->swp_vp, file_offset, (uint64_t)c_seg->c_store.c_buffer, (int)(size / PAGE_SIZE_64), SWAP_READ); #if DEVELOPMENT || DEBUG C_SEG_WRITE_PROTECT(c_seg); diff --git a/osfmk/vm/vm_compressor_backing_store.h b/osfmk/vm/vm_compressor_backing_store.h index ea3075050..9dda1ab75 100644 --- a/osfmk/vm/vm_compressor_backing_store.h +++ b/osfmk/vm/vm_compressor_backing_store.h @@ -38,11 +38,19 @@ #include #include +#if CONFIG_EMBEDDED + +#define MIN_SWAP_FILE_SIZE (64 * 1024 * 1024) + +#define MAX_SWAP_FILE_SIZE (128 * 1024 * 1024) + +#else /* CONFIG_EMBEDDED */ #define MIN_SWAP_FILE_SIZE (256 * 1024 * 1024) #define MAX_SWAP_FILE_SIZE (1 * 1024 * 1024 * 1024) +#endif /* CONFIG_EMBEDDED */ #define COMPRESSED_SWAP_CHUNK_SIZE (C_SEG_BUFSIZE) @@ -52,11 +60,9 @@ #define SWAPFILE_RECLAIM_MINIMUM_SEGS ((13 * (MAX_SWAP_FILE_SIZE / COMPRESSED_SWAP_CHUNK_SIZE)) / 10) -#define SWAP_FILE_NAME "/var/vm/swapfile" +#define SWAP_FILE_NAME "/private/var/vm/swapfile" #define SWAPFILENAME_LEN (int)(strlen(SWAP_FILE_NAME)) -char swapfilename[MAX_SWAPFILENAME_LEN + 1]; - #define SWAP_SLOT_MASK 0x1FFFFFFFF #define SWAP_DEVICE_SHIFT 33 diff --git a/osfmk/vm/vm_compressor_pager.c b/osfmk/vm/vm_compressor_pager.c index 73e4dc5cc..c5fe750fc 100644 --- a/osfmk/vm/vm_compressor_pager.c +++ b/osfmk/vm/vm_compressor_pager.c @@ -149,11 +149,11 @@ struct { typedef int compressor_slot_t; typedef struct compressor_pager { - struct ipc_object_header cpgr_pager_header; /* fake ip_kotype */ - memory_object_pager_ops_t cpgr_pager_ops; /* == &compressor_pager_ops */ - memory_object_control_t cpgr_control; - lck_mtx_t cpgr_lock; + /* mandatory generic header */ + struct memory_object cpgr_hdr; + /* pager-specific data */ + lck_mtx_t cpgr_lock; unsigned int cpgr_references; unsigned int cpgr_num_slots; unsigned int cpgr_num_slots_occupied; @@ -218,9 +218,9 @@ compressor_memory_object_init( compressor_pager_lookup(mem_obj, pager); compressor_pager_lock(pager); - if (pager->cpgr_control != MEMORY_OBJECT_CONTROL_NULL) + if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) panic("compressor_memory_object_init: bad request"); - pager->cpgr_control = control; + pager->cpgr_hdr.mo_control = control; compressor_pager_unlock(pager); @@ -229,18 +229,13 @@ compressor_memory_object_init( kern_return_t compressor_memory_object_synchronize( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length, __unused vm_sync_t flags) { - compressor_pager_t pager; - - compressor_pager_lookup(mem_obj, pager); - - memory_object_synchronize_completed(pager->cpgr_control, offset, length); - - return KERN_SUCCESS; + panic("compressor_memory_object_synchronize: memory_object_synchronize no longer supported\n"); + return KERN_FAILURE; } kern_return_t @@ -290,8 +285,8 @@ compressor_memory_object_terminate( * to prepare for a new init. */ - control = pager->cpgr_control; - pager->cpgr_control = MEMORY_OBJECT_CONTROL_NULL; + control = pager->cpgr_hdr.mo_control; + pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; compressor_pager_unlock(pager); @@ -346,7 +341,7 @@ compressor_memory_object_deallocate( * We shouldn't get a deallocation call * when the kernel has the object cached. */ - if (pager->cpgr_control != MEMORY_OBJECT_CONTROL_NULL) + if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) panic("compressor_memory_object_deallocate(): bad request"); /* @@ -442,7 +437,7 @@ compressor_memory_object_data_request( /* find the compressor slot for that page */ compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p); - if (offset / PAGE_SIZE > pager->cpgr_num_slots) { + if (offset / PAGE_SIZE >= pager->cpgr_num_slots) { /* out of range */ kr = KERN_FAILURE; } else if (slot_p == NULL || *slot_p == 0) { @@ -549,7 +544,6 @@ compressor_memory_object_create( } compressor_pager_lock_init(pager); - pager->cpgr_control = MEMORY_OBJECT_CONTROL_NULL; pager->cpgr_references = 1; pager->cpgr_num_slots = (uint32_t)(new_size/PAGE_SIZE); pager->cpgr_num_slots_occupied = 0; @@ -570,9 +564,9 @@ compressor_memory_object_create( * Set up associations between this memory object * and this compressor_pager structure */ - - pager->cpgr_pager_ops = &compressor_pager_ops; - pager->cpgr_pager_header.io_bits = IKOT_MEMORY_OBJECT; + pager->cpgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; + pager->cpgr_hdr.mo_pager_ops = &compressor_pager_ops; + pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; *new_mem_obj = (memory_object_t) pager; return KERN_SUCCESS; @@ -633,7 +627,7 @@ compressor_pager_slot_lookup( *slot_pp = NULL; return; } - if (page_num > pager->cpgr_num_slots) { + if (page_num >= pager->cpgr_num_slots) { /* out of range */ *slot_pp = NULL; return; @@ -776,7 +770,7 @@ vm_compressor_pager_get( /* find the compressor slot for that page */ compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p); - if (offset / PAGE_SIZE > pager->cpgr_num_slots) { + if (offset / PAGE_SIZE >= pager->cpgr_num_slots) { /* out of range */ kr = KERN_MEMORY_FAILURE; } else if (slot_p == NULL || *slot_p == 0) { @@ -878,7 +872,7 @@ vm_compressor_pager_state_get( /* find the compressor slot for that page */ compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p); - if (offset / PAGE_SIZE > pager->cpgr_num_slots) { + if (offset / PAGE_SIZE >= pager->cpgr_num_slots) { /* out of range */ return VM_EXTERNAL_STATE_ABSENT; } else if (slot_p == NULL || *slot_p == 0) { @@ -967,7 +961,7 @@ vm_compressor_pager_transfer( /* find the compressor slot for the destination */ assert((uint32_t) dst_offset == dst_offset); compressor_pager_lookup(dst_mem_obj, dst_pager); - assert(dst_offset / PAGE_SIZE <= dst_pager->cpgr_num_slots); + assert(dst_offset / PAGE_SIZE < dst_pager->cpgr_num_slots); compressor_pager_slot_lookup(dst_pager, TRUE, (uint32_t) dst_offset, &dst_slot_p); assert(dst_slot_p != NULL); @@ -976,7 +970,7 @@ vm_compressor_pager_transfer( /* find the compressor slot for the source */ assert((uint32_t) src_offset == src_offset); compressor_pager_lookup(src_mem_obj, src_pager); - assert(src_offset / PAGE_SIZE <= src_pager->cpgr_num_slots); + assert(src_offset / PAGE_SIZE < src_pager->cpgr_num_slots); compressor_pager_slot_lookup(src_pager, FALSE, (uint32_t) src_offset, &src_slot_p); assert(src_slot_p != NULL); @@ -1007,7 +1001,7 @@ vm_compressor_pager_next_compressed( /* overflow */ return (memory_object_offset_t) -1; } - if (page_num > pager->cpgr_num_slots) { + if (page_num >= pager->cpgr_num_slots) { /* out of range */ return (memory_object_offset_t) -1; } @@ -1056,7 +1050,7 @@ vm_compressor_pager_next_compressed( next_slot = ((chunk_idx * COMPRESSOR_SLOTS_PER_CHUNK) + slot_idx); - if (next_slot > pager->cpgr_num_slots) { + if (next_slot >= pager->cpgr_num_slots) { /* went beyond end of object */ return (memory_object_offset_t) -1; } diff --git a/osfmk/vm/vm_debug.c b/osfmk/vm/vm_debug.c index e29eed60f..12826e385 100644 --- a/osfmk/vm/vm_debug.c +++ b/osfmk/vm/vm_debug.c @@ -237,7 +237,7 @@ vm32_region_info( vio->vio_internal = cobject->internal; vio->vio_temporary = - cobject->temporary; + FALSE; vio->vio_alive = cobject->alive; vio->vio_purgable = @@ -270,17 +270,18 @@ vm32_region_info( size = vm_map_round_page(2 * used * sizeof(vm_info_object_t), VM_MAP_PAGE_MASK(ipc_kernel_map)); - kr = vm_allocate(ipc_kernel_map, &addr, size, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)); + kr = vm_allocate_kernel(ipc_kernel_map, &addr, size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC); if (kr != KERN_SUCCESS) return KERN_RESOURCE_SHORTAGE; - kr = vm_map_wire( + kr = vm_map_wire_kernel( ipc_kernel_map, vm_map_trunc_page(addr, VM_MAP_PAGE_MASK(ipc_kernel_map)), vm_map_round_page(addr + size, VM_MAP_PAGE_MASK(ipc_kernel_map)), VM_PROT_READ|VM_PROT_WRITE, + VM_KERN_MEMORY_IPC, FALSE); assert(kr == KERN_SUCCESS); } @@ -451,7 +452,7 @@ vm32_region_info_64( vio->vio_internal = cobject->internal; vio->vio_temporary = - cobject->temporary; + FALSE; vio->vio_alive = cobject->alive; vio->vio_purgable = @@ -484,17 +485,18 @@ vm32_region_info_64( size = vm_map_round_page(2 * used * sizeof(vm_info_object_t), VM_MAP_PAGE_MASK(ipc_kernel_map)); - kr = vm_allocate(ipc_kernel_map, &addr, size, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)); + kr = vm_allocate_kernel(ipc_kernel_map, &addr, size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC); if (kr != KERN_SUCCESS) return KERN_RESOURCE_SHORTAGE; - kr = vm_map_wire( + kr = vm_map_wire_kernel( ipc_kernel_map, vm_map_trunc_page(addr, VM_MAP_PAGE_MASK(ipc_kernel_map)), vm_map_round_page(addr + size, VM_MAP_PAGE_MASK(ipc_kernel_map)), VM_PROT_READ|VM_PROT_WRITE, + VM_KERN_MEMORY_IPC, FALSE); assert(kr == KERN_SUCCESS); } @@ -562,7 +564,7 @@ vm32_mapped_pages_info( VM_MAP_PAGE_MASK(ipc_kernel_map)); for (;;) { - (void) vm_allocate(ipc_kernel_map, &addr, size, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)); + (void) vm_allocate_kernel(ipc_kernel_map, &addr, size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC); (void) vm_map_unwire( ipc_kernel_map, vm_map_trunc_page(addr, @@ -602,13 +604,14 @@ vm32_mapped_pages_info( size_used = (actual * sizeof(vm_offset_t)); vmsize_used = vm_map_round_page(size_used, VM_MAP_PAGE_MASK(ipc_kernel_map)); - (void) vm_map_wire( + (void) vm_map_wire_kernel( ipc_kernel_map, vm_map_trunc_page(addr, VM_MAP_PAGE_MASK(ipc_kernel_map)), vm_map_round_page(addr + size, VM_MAP_PAGE_MASK(ipc_kernel_map)), VM_PROT_READ|VM_PROT_WRITE, + VM_KERN_MEMORY_IPC, FALSE); (void) vm_map_copyin(ipc_kernel_map, (vm_map_address_t)addr, @@ -675,8 +678,8 @@ host_virtual_physical_table_info( size = vm_map_round_page(actual * sizeof *info, VM_MAP_PAGE_MASK(ipc_kernel_map)); - kr = vm_allocate(ipc_kernel_map, &addr, size, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)); + kr = vm_allocate_kernel(ipc_kernel_map, &addr, size, + VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC); if (kr != KERN_SUCCESS) return KERN_RESOURCE_SHORTAGE; diff --git a/osfmk/vm/vm_fault.c b/osfmk/vm/vm_fault.c index ae31a4038..256c70dfe 100644 --- a/osfmk/vm/vm_fault.c +++ b/osfmk/vm/vm_fault.c @@ -2,7 +2,7 @@ * Copyright (c) 2000-2009 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,34 +22,34 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ -/* +/* * Mach Operating System * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ @@ -108,7 +108,7 @@ #include #include -#include /* for struct timespec */ +#include #define VM_FAULT_CLASSIFY 0 @@ -117,11 +117,11 @@ unsigned int vm_object_pagein_throttle = 16; /* - * We apply a hard throttle to the demand zero rate of tasks that we believe are running out of control which + * We apply a hard throttle to the demand zero rate of tasks that we believe are running out of control which * kicks in when swap space runs out. 64-bit programs have massive address spaces and can leak enormous amounts * of memory if they're buggy and can run the system completely out of swap space. If this happens, we * impose a hard throttle on them to prevent them from taking the last bit of memory left. This helps - * keep the UI active so that the user has a chance to kill the offending task before the system + * keep the UI active so that the user has a chance to kill the offending task before the system * completely hangs. * * The hard throttle is only applied when the system is nearly completely out of swap space and is only applied @@ -157,6 +157,7 @@ static kern_return_t vm_fault_wire_fast( vm_map_t map, vm_map_offset_t va, vm_prot_t prot, + vm_tag_t wire_tag, vm_map_entry_t entry, pmap_t pmap, vm_map_offset_t pmap_addr, @@ -167,6 +168,7 @@ static kern_return_t vm_fault_internal( vm_map_offset_t vaddr, vm_prot_t caller_prot, boolean_t change_wiring, + vm_tag_t wire_tag, int interruptible, pmap_t pmap, vm_map_offset_t pmap_addr, @@ -229,7 +231,7 @@ vm_fault_init(void) if (PE_parse_boot_argn("vm_compressor", &vm_compressor_temp, sizeof (vm_compressor_temp))) { for ( i = 0; i < VM_PAGER_MAX_MODES; i++) { - if (vm_compressor_temp > 0 && + if (vm_compressor_temp > 0 && ((vm_compressor_temp & ( 1 << i)) == vm_compressor_temp)) { need_default_val = FALSE; vm_compressor_mode = vm_compressor_temp; @@ -238,25 +240,13 @@ vm_fault_init(void) } if (need_default_val) printf("Ignoring \"vm_compressor\" boot arg %d\n", vm_compressor_temp); - } + } if (need_default_val) { /* If no boot arg or incorrect boot arg, try device tree. */ PE_get_default("kern.vm_compressor", &vm_compressor_mode, sizeof(vm_compressor_mode)); } PE_parse_boot_argn("vm_compressor_threads", &vm_compressor_thread_count, sizeof (vm_compressor_thread_count)); - if (PE_parse_boot_argn("vm_compressor_immediate", &vm_compressor_temp, sizeof (vm_compressor_temp))) - vm_compressor_immediate_preferred_override = TRUE; - else { - if (PE_get_default("kern.vm_compressor_immediate", &vm_compressor_temp, sizeof(vm_compressor_temp))) - vm_compressor_immediate_preferred_override = TRUE; - } - if (vm_compressor_immediate_preferred_override == TRUE) { - if (vm_compressor_temp) - vm_compressor_immediate_preferred = TRUE; - else - vm_compressor_immediate_preferred = FALSE; - } printf("\"vm_compressor_mode\" is %d\n", vm_compressor_mode); } @@ -314,8 +304,8 @@ struct { boolean_t vm_page_deactivate_behind = TRUE; -/* - * default sizes given VM_BEHAVIOR_DEFAULT reference behavior +/* + * default sizes given VM_BEHAVIOR_DEFAULT reference behavior */ #define VM_DEFAULT_DEACTIVATE_BEHIND_WINDOW 128 #define VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER 16 /* don't make this too big... */ @@ -591,15 +581,15 @@ vm_page_throttled(boolean_t page_kept) clock_sec_t elapsed_sec; clock_sec_t tv_sec; clock_usec_t tv_usec; - + thread_t thread = current_thread(); - + if (thread->options & TH_OPT_VMPRIV) return (0); if (thread->t_page_creation_throttled) { thread->t_page_creation_throttled = 0; - + if (page_kept == FALSE) goto no_throttle; } @@ -613,7 +603,7 @@ vm_page_throttled(boolean_t page_kept) if ((vm_page_free_count < vm_page_throttle_limit || (VM_CONFIG_COMPRESSOR_IS_PRESENT && SWAPPER_NEEDS_TO_UNTHROTTLE())) && thread->t_page_creation_count > (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS * VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC)) { - + if (vm_page_free_wanted == 0 && vm_page_free_wanted_privileged == 0) { #if (DEVELOPMENT || DEBUG) OSAddAtomic64(1, &vm_page_creation_throttle_avoided); @@ -634,7 +624,7 @@ vm_page_throttled(boolean_t page_kept) * over a long period of time a chance to get out of * the throttled state... we reset the counter and timestamp * so that if it stays under the rate limit for the next second - * it will be back in our good graces... if it exceeds it, it + * it will be back in our good graces... if it exceeds it, it * will remain in the throttled state */ thread->t_page_creation_time = tv_sec; @@ -772,7 +762,7 @@ vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill) * execution. i.e. it is the responsibility * of higher layers to call for an instruction * sync after changing the contents and before - * sending a program into this area. We + * sending a program into this area. We * choose this approach for performance */ m->pmapped = TRUE; @@ -832,7 +822,7 @@ vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill) * The required permissions for the page is given * in "fault_type". Desired permissions are included * in "protection". - * fault_info is passed along to determine pagein cluster + * fault_info is passed along to determine pagein cluster * limits... it contains the expected reference pattern, * cluster size if available, etc... * @@ -864,7 +854,7 @@ vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill) * The "result_page" is also left busy. It is not removed * from the pageout queues. * Special Case: - * A return value of VM_FAULT_SUCCESS_NO_PAGE means that the + * A return value of VM_FAULT_SUCCESS_NO_PAGE means that the * fault succeeded but there's no VM page (i.e. the VM object * does not actually hold VM pages, but device memory or * large pages). The object is still locked and we still hold a @@ -892,7 +882,7 @@ vm_fault_page( /* More arguments: */ kern_return_t *error_code, /* code if page is in error */ boolean_t no_zero_fill, /* don't zero fill absent pages */ - boolean_t data_supply, /* treat as data_supply if + boolean_t data_supply, /* treat as data_supply if * it is a write fault and a full * page is provided */ vm_object_fault_info_t fault_info) @@ -924,10 +914,10 @@ vm_fault_page( int grab_options; /* - * MUST_ASK_PAGER() evaluates to TRUE if the page specified by object/offset is + * MUST_ASK_PAGER() evaluates to TRUE if the page specified by object/offset is * marked as paged out in the compressor pager or the pager doesn't exist. - * Note also that if the pager for an internal object - * has not been created, the pager is not invoked regardless of the value + * Note also that if the pager for an internal object + * has not been created, the pager is not invoked regardless of the value * of MUST_ASK_PAGER(). * * PAGED_OUT() evaluates to TRUE if the page specified by the object/offset @@ -965,7 +955,7 @@ vm_fault_page( interruptible = fault_info->interruptible; interruptible_state = thread_interrupt_level(interruptible); - + /* * INVARIANTS (through entire routine): * @@ -1112,7 +1102,7 @@ vm_fault_page( if (m->laundry) { m->free_when_done = FALSE; - if (!m->cleaning) + if (!m->cleaning) vm_pageout_steal_laundry(m, FALSE); } if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) { @@ -1204,7 +1194,7 @@ vm_fault_page( /* * check for any conditions that prevent * us from creating a new zero-fill page - * vm_fault_check will do all of the + * vm_fault_check will do all of the * fault cleanup in the case of an error condition * including resetting the thread_interrupt_level */ @@ -1232,7 +1222,7 @@ vm_fault_page( vm_object_unlock(object); /* - * grab the original page we + * grab the original page we * 'soldered' in place and * retake lock on 'first_object' */ @@ -1290,7 +1280,7 @@ vm_fault_page( vm_object_unlock(object); object = next_object; vm_object_paging_begin(object); - + /* * reset to default type of fault */ @@ -1325,7 +1315,7 @@ vm_fault_page( vm_object_reference_locked(object); vm_fault_cleanup(object, first_m); - + counter(c_vm_fault_page_block_backoff_kernel++); vm_object_lock(object); assert(object->ref_count > 0); @@ -1373,29 +1363,6 @@ vm_fault_page( } assert(object == VM_PAGE_OBJECT(m)); - if (m->encrypted) { - /* - * ENCRYPTED SWAP: - * the user needs access to a page that we - * encrypted before paging it out. - * Decrypt the page now. - * Keep it busy to prevent anyone from - * accessing it during the decryption. - */ - m->busy = TRUE; - vm_page_decrypt(m, 0); - assert(m->busy); - PAGE_WAKEUP_DONE(m); - - /* - * Retry from the top, in case - * something changed while we were - * decrypting. - */ - continue; - } - ASSERT_PAGE_DECRYPTED(m); - if (object->code_signed) { /* * CODE SIGNING: @@ -1427,7 +1394,7 @@ vm_fault_page( m->busy = TRUE; break; } - + /* * we get here when there is no page present in the object at @@ -1436,6 +1403,7 @@ vm_fault_page( * this object can provide the data or we're the top object... * object is locked; m == NULL */ + if (must_be_resident) { if (fault_type == VM_PROT_NONE && object == kernel_object) { @@ -1454,10 +1422,14 @@ vm_fault_page( goto dont_look_for_page; } + + /* Don't expect to fault pages into the kernel object. */ + assert(object != kernel_object); + data_supply = FALSE; look_for_page = (object->pager_created && (MUST_ASK_PAGER(object, offset, external_state) == TRUE) && !data_supply); - + #if TRACEFAULTPAGE dbgTrace(0xBEEF000C, (unsigned int) look_for_page, (unsigned int) object); /* (TEST/DEBUG) */ #endif @@ -1597,7 +1569,7 @@ vm_fault_page( } } assert(m->busy); - + m->absent = TRUE; pager = object->pager; @@ -1697,7 +1669,7 @@ vm_fault_page( goto data_requested; } my_fault_type = DBG_PAGEIN_FAULT; - + if (m != VM_PAGE_NULL) { VM_PAGE_FREE(m); m = VM_PAGE_NULL; @@ -1709,7 +1681,7 @@ vm_fault_page( /* * It's possible someone called vm_object_destroy while we weren't - * holding the object lock. If that has happened, then bail out + * holding the object lock. If that has happened, then bail out * here. */ @@ -1726,6 +1698,10 @@ vm_fault_page( * so we can release the object lock. */ + if (object->object_slid == TRUE) { + set_thread_rwlock_boost(); + } + vm_object_unlock(object); /* @@ -1769,7 +1745,7 @@ vm_fault_page( */ assert(first_m != VM_PAGE_NULL); assert(VM_PAGE_OBJECT(first_m) == first_object); - + vm_object_lock(first_object); VM_PAGE_FREE(first_m); vm_object_paging_end(first_object); @@ -1810,6 +1786,10 @@ vm_fault_page( #endif vm_object_lock(object); + if (object->object_slid == TRUE) { + clear_thread_rwlock_boost(); + } + data_requested: if (rc != KERN_SUCCESS) { @@ -1846,7 +1826,7 @@ vm_fault_page( if (m == VM_PAGE_NULL && object->phys_contiguous) { /* * No page here means that the object we - * initially looked up was "physically + * initially looked up was "physically * contiguous" (i.e. device memory). However, * with Virtual VRAM, the object might not * be backed by that device memory anymore, @@ -1876,7 +1856,7 @@ vm_fault_page( } dont_look_for_page: /* - * We get here if the object has no pager, or an existence map + * We get here if the object has no pager, or an existence map * exists and indicates the page isn't present on the pager * or we're unwiring a page. If a pager exists, but there * is no existence map, then the m->absent case above handles @@ -1919,7 +1899,7 @@ dont_look_for_page: /* * check for any conditions that prevent * us from creating a new zero-fill page - * vm_fault_check will do all of the + * vm_fault_check will do all of the * fault cleanup in the case of an error condition * including resetting the thread_interrupt_level */ @@ -1995,13 +1975,6 @@ dont_look_for_page: !first_m->active && !first_m->inactive && !first_m->secluded)); #endif /* EXTRA_ASSERTIONS */ - /* - * ENCRYPTED SWAP: - * If we found a page, we must have decrypted it before we - * get here... - */ - ASSERT_PAGE_DECRYPTED(m); - XPR(XPR_VM_FAULT, "vm_f_page: FOUND obj 0x%X, off 0x%X, page 0x%X, 1_obj 0x%X, 1_m 0x%X\n", object, offset, m, @@ -2136,7 +2109,7 @@ dont_look_for_page: */ VM_PAGE_FREE(first_m); first_m = VM_PAGE_NULL; - + /* * and replace it with the * page we just copied into @@ -2151,8 +2124,8 @@ dont_look_for_page: * way, let's try to collapse the top object. * But we have to play ugly games with * paging_in_progress to do that... - */ - vm_object_paging_end(object); + */ + vm_object_paging_end(object); vm_object_collapse(object, offset, TRUE); vm_object_paging_begin(object); @@ -2249,12 +2222,7 @@ dont_look_for_page: copy_object->ref_count--; assert(copy_object->ref_count > 0); copy_m = vm_page_lookup(copy_object, copy_offset); - /* - * ENCRYPTED SWAP: - * it's OK if the "copy_m" page is encrypted, - * because we're not moving it nor handling its - * contents. - */ + if (copy_m != VM_PAGE_NULL && copy_m->busy) { PAGE_ASSERT_WAIT(copy_m, interruptible); @@ -2329,7 +2297,7 @@ dont_look_for_page: * Must copy page into copy-object. */ vm_page_copy(m, copy_m); - + /* * If the old page was in use by any users * of the copy-object, it must be removed @@ -2361,7 +2329,7 @@ dont_look_for_page: PAGE_WAKEUP_DONE(copy_m); } else { - + assert(copy_m->busy == TRUE); assert(!m->cleaning); @@ -2432,7 +2400,7 @@ dont_look_for_page: copy_object->ref_count--; assert(copy_object->ref_count > 0); - VM_OBJ_RES_DECR(copy_object); + VM_OBJ_RES_DECR(copy_object); vm_object_unlock(copy_object); break; @@ -2536,6 +2504,7 @@ vm_fault_enter(vm_page_t m, vm_prot_t caller_prot, boolean_t wired, boolean_t change_wiring, + vm_tag_t wire_tag, boolean_t no_cache, boolean_t cs_bypass, __unused int user_tag, @@ -2550,11 +2519,18 @@ vm_fault_enter(vm_page_t m, int cs_enforcement_enabled; vm_prot_t fault_type; vm_object_t object; - + fault_type = change_wiring ? VM_PROT_NONE : caller_prot; object = VM_PAGE_OBJECT(m); vm_object_lock_assert_held(object); + +#if KASAN + if (pmap == kernel_pmap) { + kasan_notify_address(vaddr, PAGE_SIZE); + } +#endif + LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED); if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) { @@ -2578,7 +2554,13 @@ vm_fault_enter(vm_page_t m, * soft-fault again if we need write * access later... */ - prot &= ~VM_PROT_WRITE; + + /* This had better not be a JIT page. */ + if (!pmap_has_prot_policy(prot)) { + prot &= ~VM_PROT_WRITE; + } else { + assert(cs_bypass); + } } if (m->pmapped == FALSE) { @@ -2594,7 +2576,7 @@ vm_fault_enter(vm_page_t m, *type_of_fault = DBG_PAGEIND_FAULT; else *type_of_fault = DBG_PAGEINV_FAULT; - + VM_PAGE_COUNT_AS_PAGEIN(m); } VM_PAGE_CONSUME_CLUSTERED(m); @@ -2617,7 +2599,7 @@ vm_fault_enter(vm_page_t m, vm_cs_revalidates++; } - /* VM map is locked, so 1 ref will remain on VM object - + /* VM map is locked, so 1 ref will remain on VM object - * so no harm if vm_page_validate_cs drops the object lock */ vm_page_validate_cs(m); } @@ -2628,21 +2610,21 @@ vm_fault_enter(vm_page_t m, map_is_switched = ((pmap != vm_map_pmap(current_task()->map)) && (pmap == vm_map_pmap(current_thread()->map))); map_is_switch_protected = current_thread()->map->switch_protect; - + /* If the map is switched, and is switch-protected, we must protect - * some pages from being write-faulted: immutable pages because by + * some pages from being write-faulted: immutable pages because by * definition they may not be written, and executable pages because that * would provide a way to inject unsigned code. * If the page is immutable, we can simply return. However, we can't * immediately determine whether a page is executable anywhere. But, * we can disconnect it everywhere and remove the executable protection - * from the current map. We do that below right before we do the + * from the current map. We do that below right before we do the * PMAP_ENTER. */ cs_enforcement_enabled = cs_enforcement(NULL); - if(cs_enforcement_enabled && map_is_switched && - map_is_switch_protected && page_immutable(m, prot) && + if(cs_enforcement_enabled && map_is_switched && + map_is_switch_protected && page_immutable(m, prot) && (prot & VM_PROT_WRITE)) { return KERN_CODESIGN_ERROR; @@ -2666,20 +2648,24 @@ vm_fault_enter(vm_page_t m, * fault, so we should not raise a code-signing violation * (and possibly kill the process) below. * Instead, let's just remove the "execute" access request. - * + * * This can happen on devices with a 4K page size if a 16K * page contains a mix of signed&executable and * unsigned&non-executable 4K pages, making the whole 16K * mapping "executable". */ - prot &= ~VM_PROT_EXECUTE; + if (!pmap_has_prot_policy(prot)) { + prot &= ~VM_PROT_EXECUTE; + } else { + assert(cs_bypass); + } } /* A page could be tainted, or pose a risk of being tainted later. * Check whether the receiving process wants it, and make it feel * the consequences (that hapens in cs_invalid_page()). - * For CS Enforcement, two other conditions will - * cause that page to be tainted as well: + * For CS Enforcement, two other conditions will + * cause that page to be tainted as well: * - pmapping an unsigned page executable - this means unsigned code; * - writeable mapping of a validated page - the content of that page * can be changed without the kernel noticing, therefore unsigned @@ -2692,20 +2678,20 @@ vm_fault_enter(vm_page_t m, (!m->cs_validated && (prot & VM_PROT_EXECUTE)) || /* The page should be immutable, but is in danger of being modified * This is the case where we want policy from the code directory - - * is the page immutable or not? For now we have to assume that + * is the page immutable or not? For now we have to assume that * code pages will be immutable, data pages not. - * We'll assume a page is a code page if it has a code directory + * We'll assume a page is a code page if it has a code directory * and we fault for execution. * That is good enough since if we faulted the code page for * writing in another map before, it is wpmapped; if we fault - * it for writing in this map later it will also be faulted for executing + * it for writing in this map later it will also be faulted for executing * at the same time; and if we fault for writing in another map * later, we will disconnect it from this pmap so we'll notice * the change. */ (page_immutable(m, prot) && ((prot & VM_PROT_WRITE) || m->wpmapped)) )) - )) + )) { /* We will have a tainted page. Have to handle the special case * of a switched map now. If the map is not switched, standard @@ -2715,13 +2701,13 @@ vm_fault_enter(vm_page_t m, * it will not be executing from the map. So we don't call * cs_invalid_page() in that case. */ boolean_t reject_page, cs_killed; - if(map_is_switched) { + if(map_is_switched) { assert(pmap==vm_map_pmap(current_thread()->map)); assert(!(prot & VM_PROT_WRITE) || (map_is_switch_protected == FALSE)); reject_page = FALSE; } else { if (cs_debug > 5) - printf("vm_fault: signed: %s validate: %s tainted: %s wpmapped: %s slid: %s prot: 0x%x\n", + printf("vm_fault: signed: %s validate: %s tainted: %s wpmapped: %s slid: %s prot: 0x%x\n", object->code_signed ? "yes" : "no", m->cs_validated ? "yes" : "no", m->cs_tainted ? "yes" : "no", @@ -2730,7 +2716,7 @@ vm_fault_enter(vm_page_t m, (int)prot); reject_page = cs_invalid_page((addr64_t) vaddr, &cs_killed); } - + if (reject_page) { /* reject the invalid page: abort the page fault */ int pid; @@ -2982,7 +2968,7 @@ vm_fault_enter(vm_page_t m, } #endif } - + } else { /* proceed with the valid page */ kr = KERN_SUCCESS; @@ -3025,7 +3011,7 @@ MACRO_END if (wired) { if (kr == KERN_SUCCESS) { - vm_page_wire(m, VM_PROT_MEMORY_TAG(caller_prot), TRUE); + vm_page_wire(m, wire_tag, TRUE); } } else { vm_page_unwire(m, TRUE); @@ -3050,7 +3036,7 @@ MACRO_END ((m->vm_page_q_state != VM_PAGE_ON_THROTTLED_Q) && no_cache)) && !VM_PAGE_WIRED(m)) { - if (vm_page_local_q && + if (vm_page_local_q && (*type_of_fault == DBG_COW_FAULT || *type_of_fault == DBG_ZERO_FILL_FAULT) ) { struct vpl *lq; @@ -3087,7 +3073,7 @@ MACRO_END m->vm_page_q_state = VM_PAGE_ON_ACTIVE_LOCAL_Q; m->local_id = lid; lq->vpl_count++; - + if (object->internal) lq->vpl_internal_count++; else @@ -3143,7 +3129,7 @@ MACRO_END * that they can be readily * recycled if free memory runs * low. Otherwise the page is - * activated as normal. + * activated as normal. */ if (no_cache && @@ -3151,7 +3137,7 @@ MACRO_END m->no_cache)) { m->no_cache = TRUE; - if (m->vm_page_q_state != VM_PAGE_ON_SPECULATIVE_Q) + if (m->vm_page_q_state != VM_PAGE_ON_SPECULATIVE_Q) vm_page_speculate(m, FALSE); } else if ( !VM_PAGE_ACTIVE_OR_INACTIVE(m)) { @@ -3176,7 +3162,7 @@ MACRO_END if (kr == KERN_SUCCESS) { /* * NOTE: we may only hold the vm_object lock SHARED - * at this point, so we need the phys_page lock to + * at this point, so we need the phys_page lock to * properly serialize updating the pmapped and * xpmapped bits */ @@ -3191,7 +3177,7 @@ MACRO_END * just below */ m->pmapped = TRUE; - + if (!m->xpmapped) { m->xpmapped = TRUE; @@ -3201,6 +3187,9 @@ MACRO_END if (!object->internal) OSAddAtomic(1, &vm_page_xpmapped_external_count); +#if defined(__arm__) || defined(__arm64__) + pmap_sync_page_data_phys(phys_page); +#else if (object->internal && object->pager != NULL) { /* @@ -3217,6 +3206,7 @@ MACRO_END assert(VM_CONFIG_COMPRESSOR_IS_PRESENT); pmap_sync_page_data_phys(phys_page); } +#endif } else pmap_unlock_phys_page(phys_page); } else { @@ -3261,21 +3251,22 @@ MACRO_END } if (must_disconnect) { /* - * We can only get here + * We can only get here * because of the CSE logic */ assert(cs_enforcement_enabled); pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); - /* + /* * If we are faulting for a write, we can clear * the execute bit - that will ensure the page is * checked again before being executable, which * protects against a map switch. * This only happens the first time the page - * gets tainted, so we won't get stuck here + * gets tainted, so we won't get stuck here * to make an already writeable page executable. */ if (!cs_bypass){ + assert(!pmap_has_prot_policy(prot)); prot &= ~VM_PROT_EXECUTE; } } @@ -3289,6 +3280,19 @@ MACRO_END wired, pmap_options | PMAP_OPTIONS_NOWAIT, pe_result); +#if __x86_64__ + if (pe_result == KERN_INVALID_ARGUMENT && + pmap == PMAP_NULL && + wired) { + /* + * Wiring a page in a pmap-less VM map: + * VMware's "vmmon" kernel extension does this + * to grab pages. + * Let it proceed even though the PMAP_ENTER() failed. + */ + pe_result = KERN_SUCCESS; + } +#endif /* __x86_64__ */ if(pe_result == KERN_RESOURCE_SHORTAGE) { @@ -3298,10 +3302,10 @@ MACRO_END * on the top-object in this chain... we can't just drop * the lock on the object we're inserting the page into * and recall the PMAP_ENTER since we can still cause - * a deadlock if one of the critical paths tries to + * a deadlock if one of the critical paths tries to * acquire the lock on the top-object and we're blocked * in PMAP_ENTER waiting for memory... our only recourse - * is to deal with it at a higher level where we can + * is to deal with it at a higher level where we can * drop both locks. */ *need_retry = TRUE; @@ -3310,7 +3314,7 @@ MACRO_END } /* The nonblocking version of pmap_enter did not succeed. * and we don't need to drop other locks and retry - * at the level above us, so + * at the level above us, so * use the blocking version instead. Requires marking * the page busy and unlocking the object */ boolean_t was_busy = m->busy; @@ -3319,16 +3323,16 @@ MACRO_END m->busy = TRUE; vm_object_unlock(object); - + PMAP_ENTER_OPTIONS(pmap, vaddr, m, prot, fault_type, 0, wired, pmap_options, pe_result); - + assert(VM_PAGE_OBJECT(m) == object); /* Take the object lock again. */ vm_object_lock(object); - + /* If the page was busy, someone else will wake it up. * Otherwise, we have to do it now. */ assert(m->busy); @@ -3337,6 +3341,8 @@ MACRO_END } vm_pmap_enter_blocked++; } + + kr = pe_result; } after_the_pmap_enter: @@ -3348,13 +3354,14 @@ vm_pre_fault(vm_map_offset_t vaddr) { if (pmap_find_phys(current_map()->pmap, vaddr) == 0) { - vm_fault(current_map(), /* map */ - vaddr, /* vaddr */ - VM_PROT_READ, /* fault_type */ - FALSE, /* change_wiring */ - THREAD_UNINT, /* interruptible */ - NULL, /* caller_pmap */ - 0 /* caller_pmap_addr */); + vm_fault(current_map(), /* map */ + vaddr, /* vaddr */ + VM_PROT_READ, /* fault_type */ + FALSE, /* change_wiring */ + VM_KERN_MEMORY_NONE, /* tag - not wiring */ + THREAD_UNINT, /* interruptible */ + NULL, /* caller_pmap */ + 0 /* caller_pmap_addr */); } } @@ -3381,7 +3388,7 @@ unsigned long vm_fault_collapse_skipped = 0; kern_return_t -vm_fault( +vm_fault_external( vm_map_t map, vm_map_offset_t vaddr, vm_prot_t fault_type, @@ -3390,11 +3397,26 @@ vm_fault( pmap_t caller_pmap, vm_map_offset_t caller_pmap_addr) { - return vm_fault_internal(map, vaddr, fault_type, change_wiring, + return vm_fault_internal(map, vaddr, fault_type, change_wiring, vm_tag_bt(), interruptible, caller_pmap, caller_pmap_addr, NULL); } +kern_return_t +vm_fault( + vm_map_t map, + vm_map_offset_t vaddr, + vm_prot_t fault_type, + boolean_t change_wiring, + vm_tag_t wire_tag, /* if wiring must pass tag != VM_KERN_MEMORY_NONE */ + int interruptible, + pmap_t caller_pmap, + vm_map_offset_t caller_pmap_addr) +{ + return vm_fault_internal(map, vaddr, fault_type, change_wiring, wire_tag, + interruptible, caller_pmap, caller_pmap_addr, + NULL); +} kern_return_t vm_fault_internal( @@ -3402,6 +3424,7 @@ vm_fault_internal( vm_map_offset_t vaddr, vm_prot_t caller_prot, boolean_t change_wiring, + vm_tag_t wire_tag, /* if wiring must pass tag != VM_KERN_MEMORY_NONE */ int interruptible, pmap_t caller_pmap, vm_map_offset_t caller_pmap_addr, @@ -3429,6 +3452,7 @@ vm_fault_internal( boolean_t interruptible_state; vm_map_t real_map = map; vm_map_t original_map = map; + boolean_t object_locks_dropped = FALSE; vm_prot_t fault_type; vm_prot_t original_fault_type; struct vm_object_fault_info fault_info; @@ -3452,13 +3476,13 @@ vm_fault_internal( vaddr = vm_map_trunc_page(vaddr, PAGE_MASK); if (map == kernel_map) { - trace_vaddr = VM_KERNEL_UNSLIDE_OR_PERM(vaddr); - trace_real_vaddr = VM_KERNEL_UNSLIDE_OR_PERM(trace_real_vaddr); + trace_vaddr = VM_KERNEL_ADDRHIDE(vaddr); + trace_real_vaddr = VM_KERNEL_ADDRHIDE(trace_real_vaddr); } else { trace_vaddr = vaddr; } - KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_START, ((uint64_t)trace_vaddr >> 32), trace_vaddr, @@ -3467,7 +3491,7 @@ vm_fault_internal( 0); if (get_preemption_level() != 0) { - KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END, ((uint64_t)trace_vaddr >> 32), trace_vaddr, @@ -3477,7 +3501,7 @@ vm_fault_internal( return (KERN_FAILURE); } - + interruptible_state = thread_interrupt_level(interruptible); fault_type = (change_wiring ? VM_PROT_NONE : caller_prot); @@ -3493,6 +3517,14 @@ vm_fault_internal( cur_object_lock_type = OBJECT_LOCK_SHARED; + if ((map == kernel_map) && (caller_prot & VM_PROT_WRITE)) { + if (compressor_map) { + if ((vaddr >= vm_map_min(compressor_map)) && (vaddr < vm_map_max(compressor_map))) { + panic("Write fault on compressor map, va: %p type: %u bounds: %p->%p", (void *) vaddr, caller_prot, (void *) vm_map_min(compressor_map), (void *) vm_map_max(compressor_map)); + + } + } + } RetryFault: /* * assume we will hit a page in the cache @@ -3515,7 +3547,6 @@ RetryFault: &fault_info, &real_map); - if (kr != KERN_SUCCESS) { vm_map_unlock_read(map); goto done; @@ -3589,6 +3620,24 @@ RetryFault: * */ +#if defined(__arm64__) + /* + * Fail if reading an execute-only page in a + * pmap that enforces execute-only protection. + */ + if (fault_type == VM_PROT_READ && + (prot & VM_PROT_EXECUTE) && + !(prot & VM_PROT_READ) && + pmap_enforces_execute_only(pmap)) { + vm_object_unlock(object); + vm_map_unlock_read(map); + if (real_map != map) { + vm_map_unlock(real_map); + } + kr = KERN_PROTECTION_FAILURE; + goto done; + } +#endif /* * If this page is to be inserted in a copy delay object @@ -3782,82 +3831,7 @@ reclaimed_from_pageout: kr = KERN_MEMORY_ERROR; goto done; } - - if (m->encrypted) { - /* - * ENCRYPTED SWAP: - * We've soft-faulted (because it's not in the page - * table) on an encrypted page. - * Keep the page "busy" so that no one messes with - * it during the decryption. - * Release the extra locks we're holding, keep only - * the page's VM object lock. - * - * in order to set 'busy' on 'm', we must - * have object that 'm' belongs to locked exclusively - */ - if (object != cur_object) { - vm_object_unlock(object); - - if (cur_object_lock_type == OBJECT_LOCK_SHARED) { - - cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE; - - if (vm_object_lock_upgrade(cur_object) == FALSE) { - /* - * couldn't upgrade so go do a full retry - * immediately since we've already dropped - * the top object lock associated with this page - * and the current one got dropped due to the - * failed upgrade... the state is no longer valid - */ - vm_map_unlock_read(map); - if (real_map != map) - vm_map_unlock(real_map); - - goto RetryFault; - } - } - } else if (object_lock_type == OBJECT_LOCK_SHARED) { - - object_lock_type = OBJECT_LOCK_EXCLUSIVE; - - if (vm_object_lock_upgrade(object) == FALSE) { - /* - * couldn't upgrade, so explictly take the lock - * exclusively and go relookup the page since we - * will have dropped the object lock and - * a different thread could have inserted - * a page at this offset - * no need for a full retry since we're - * at the top level of the object chain - */ - vm_object_lock(object); - - continue; - } - } - m->busy = TRUE; - - vm_map_unlock_read(map); - if (real_map != map) - vm_map_unlock(real_map); - - vm_page_decrypt(m, 0); - - assert(m->busy); - PAGE_WAKEUP_DONE(m); - - vm_object_unlock(cur_object); - /* - * Retry from the top, in case anything - * changed while we were decrypting... - */ - goto RetryFault; - } - ASSERT_PAGE_DECRYPTED(m); - - if(vm_page_is_slideable(m)) { + if (vm_page_is_slideable(m)) { /* * We might need to slide this page, and so, * we want to hold the VM object exclusively. @@ -3940,8 +3914,20 @@ upgrade_for_validation: } if ((fault_type & VM_PROT_WRITE) == 0) { - - prot &= ~VM_PROT_WRITE; + if (!pmap_has_prot_policy(prot)) { + prot &= ~VM_PROT_WRITE; + } else { + /* + * For a protection that the pmap cares + * about, we must hand over the full + * set of protections (so that the pmap + * layer can apply any desired policy). + * This means that cs_bypass must be + * set, as this can force us to pass + * RWX. + */ + assert(fault_info.cs_bypass); + } if (object != cur_object) { /* @@ -3994,6 +3980,7 @@ FastPmapEnter: caller_prot, wired, change_wiring, + wire_tag, fault_info.no_cache, fault_info.cs_bypass, fault_info.user_tag, @@ -4008,6 +3995,7 @@ FastPmapEnter: caller_prot, wired, change_wiring, + wire_tag, fault_info.no_cache, fault_info.cs_bypass, fault_info.user_tag, @@ -4057,7 +4045,7 @@ FastPmapEnter: if (need_collapse == TRUE) vm_object_collapse(object, offset, TRUE); - + if (need_retry == FALSE && (type_of_fault == DBG_PAGEIND_FAULT || type_of_fault == DBG_PAGEINV_FAULT || type_of_fault == DBG_CACHE_HIT_FAULT)) { /* @@ -4093,7 +4081,7 @@ FastPmapEnter: (void)pmap_enter_options( pmap, vaddr, 0, 0, 0, 0, 0, PMAP_OPTIONS_NOENTER, NULL); - + need_retry = FALSE; goto RetryFault; } @@ -4118,14 +4106,14 @@ FastPmapEnter: */ break; } - + /* * This is now a shadow based copy on write * fault -- it requires a copy up the shadow * chain. */ assert(m_object == VM_PAGE_OBJECT(m)); - + if ((cur_object_lock_type == OBJECT_LOCK_SHARED) && VM_FAULT_NEED_CS_VALIDATION(NULL, m, m_object)) { goto upgrade_for_validation; @@ -4137,7 +4125,7 @@ FastPmapEnter: * need to remember current page, as it's the * source of the copy. * - * at this point we hold locks on both + * at this point we hold locks on both * object and cur_object... no need to take * paging refs or mark pages BUSY since * we don't drop either object lock until @@ -4170,7 +4158,7 @@ FastPmapEnter: */ if (object->ref_count > 1 && cur_m->pmapped) pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(cur_m)); - + if (cur_m->clustered) { VM_PAGE_COUNT_AS_PAGEIN(cur_m); VM_PAGE_CONSUME_CLUSTERED(cur_m); @@ -4311,7 +4299,7 @@ FastPmapEnter: * at the top level of the object chain */ vm_object_lock(object); - + continue; } } @@ -4475,7 +4463,7 @@ FastPmapEnter: if (vm_backing_store_low) { /* * we are protecting the system from - * backing store exhaustion... + * backing store exhaustion... * must take the slow path if we're * not privileged */ @@ -4519,7 +4507,7 @@ FastPmapEnter: /* * Now zero fill page... - * the page is probably going to + * the page is probably going to * be written soon, so don't bother * to clear the modified bit * @@ -4583,8 +4571,8 @@ handle_copy_delay: vm_map_unlock(real_map); if (__improbable(object == compressor_object || - object == kernel_object || - object == vm_submap_object)) { + object == kernel_object || + object == vm_submap_object)) { /* * These objects are explicitly managed and populated by the * kernel. The virtual ranges backed by these objects should @@ -4637,7 +4625,7 @@ handle_copy_delay: * * the object is returned locked with a paging reference * - * if top_page != NULL, then it's BUSY and the + * if top_page != NULL, then it's BUSY and the * object it belongs to has a paging reference * but is returned unlocked */ @@ -4653,7 +4641,7 @@ handle_copy_delay: */ switch (kr) { case VM_FAULT_MEMORY_SHORTAGE: - if (vm_page_wait((change_wiring) ? + if (vm_page_wait((change_wiring) ? THREAD_UNINT : THREAD_ABORTSAFE)) goto RetryFault; @@ -4701,22 +4689,47 @@ handle_copy_delay: } \ MACRO_END + + object_locks_dropped = FALSE; /* * We must verify that the maps have not changed - * since our last lookup. + * since our last lookup. vm_map_verify() needs the + * map lock (shared) but we are holding object locks. + * So we do a try_lock() first and, if that fails, we + * drop the object locks and go in for the map lock again. */ - if (m != VM_PAGE_NULL) { - old_copy_object = m_object->copy; - vm_object_unlock(m_object); - } else { - old_copy_object = VM_OBJECT_NULL; - vm_object_unlock(object); + if (!vm_map_try_lock_read(original_map)) { + + if (m != VM_PAGE_NULL) { + old_copy_object = m_object->copy; + vm_object_unlock(m_object); + } else { + old_copy_object = VM_OBJECT_NULL; + vm_object_unlock(object); + } + + object_locks_dropped = TRUE; + + vm_map_lock_read(original_map); } - /* - * no object locks are held at this point - */ if ((map != original_map) || !vm_map_verify(map, &version)) { + + if (object_locks_dropped == FALSE) { + if (m != VM_PAGE_NULL) { + old_copy_object = m_object->copy; + vm_object_unlock(m_object); + } else { + old_copy_object = VM_OBJECT_NULL; + vm_object_unlock(object); + } + + object_locks_dropped = TRUE; + } + + /* + * no object locks are held at this point + */ vm_object_t retry_object; vm_object_offset_t retry_offset; vm_prot_t retry_prot; @@ -4731,7 +4744,6 @@ handle_copy_delay: * take another fault. */ map = original_map; - vm_map_lock_read(map); kr = vm_map_lookup_locked(&map, vaddr, fault_type & ~VM_PROT_WRITE, @@ -4813,20 +4825,31 @@ handle_copy_delay: * Check whether the protection has changed or the object * has been copied while we left the map unlocked. */ - prot &= retry_prot; + if (pmap_has_prot_policy(retry_prot)) { + /* If the pmap layer cares, pass the full set. */ + prot = retry_prot; + } else { + prot &= retry_prot; + } } - if (m != VM_PAGE_NULL) { - vm_object_lock(m_object); - if (m_object->copy != old_copy_object) { - /* - * The copy object changed while the top-level object - * was unlocked, so take away write permission. - */ - prot &= ~VM_PROT_WRITE; - } - } else - vm_object_lock(object); + if (object_locks_dropped == TRUE) { + if (m != VM_PAGE_NULL) { + vm_object_lock(m_object); + + if (m_object->copy != old_copy_object) { + /* + * The copy object changed while the top-level object + * was unlocked, so take away write permission. + */ + assert(!pmap_has_prot_policy(prot)); + prot &= ~VM_PROT_WRITE; + } + } else + vm_object_lock(object); + + object_locks_dropped = FALSE; + } /* * If we want to wire down this page, but no longer have @@ -4834,7 +4857,7 @@ handle_copy_delay: */ if (wired && (fault_type != (prot | VM_PROT_WRITE))) { - vm_map_verify_done(map, &version); + vm_map_unlock_read(map); if (real_map != map) vm_map_unlock(real_map); @@ -4867,6 +4890,7 @@ handle_copy_delay: caller_prot, wired, change_wiring, + wire_tag, fault_info.no_cache, fault_info.cs_bypass, fault_info.user_tag, @@ -4881,6 +4905,7 @@ handle_copy_delay: caller_prot, wired, change_wiring, + wire_tag, fault_info.no_cache, fault_info.cs_bypass, fault_info.user_tag, @@ -4908,7 +4933,7 @@ handle_copy_delay: #endif if (kr != KERN_SUCCESS) { /* abort this page fault */ - vm_map_verify_done(map, &version); + vm_map_unlock_read(map); if (real_map != map) vm_map_unlock(real_map); PAGE_WAKEUP_DONE(m); @@ -4930,9 +4955,9 @@ handle_copy_delay: vm_map_offset_t laddr; vm_map_offset_t ldelta, hdelta; - /* + /* * do a pmap block mapping from the physical address - * in the object + * in the object */ #ifdef ppc @@ -4944,7 +4969,7 @@ handle_copy_delay: if ((fault_type & VM_PROT_EXECUTE) && (!pmap_eligible_for_execute((ppnum_t)(object->vo_shadow_offset >> 12)))) { - vm_map_verify_done(map, &version); + vm_map_unlock_read(map); if (real_map != map) vm_map_unlock(real_map); @@ -4977,8 +5002,8 @@ handle_copy_delay: if (hdelta > (entry->vme_end - laddr)) hdelta = entry->vme_end - laddr; if (entry->is_sub_map) { - - laddr = ((laddr - entry->vme_start) + + laddr = ((laddr - entry->vme_start) + VME_OFFSET(entry)); vm_map_lock_read(VME_SUBMAP(entry)); @@ -4989,13 +5014,13 @@ handle_copy_delay: real_map = VME_SUBMAP(entry); } map = VME_SUBMAP(entry); - + } else { break; } } - if (vm_map_lookup_entry(map, laddr, &entry) && + if (vm_map_lookup_entry(map, laddr, &entry) && (VME_OBJECT(entry) != NULL) && (VME_OBJECT(entry) == object)) { int superpage; @@ -5025,31 +5050,48 @@ handle_copy_delay: * Set up a block mapped area */ assert((uint32_t)((ldelta + hdelta) >> PAGE_SHIFT) == ((ldelta + hdelta) >> PAGE_SHIFT)); - pmap_map_block(caller_pmap, - (addr64_t)(caller_pmap_addr - ldelta), - (ppnum_t)((((vm_map_offset_t) (VME_OBJECT(entry)->vo_shadow_offset)) + - VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT), - (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot, - (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0); - } else { + kr = pmap_map_block(caller_pmap, + (addr64_t)(caller_pmap_addr - ldelta), + (ppnum_t)((((vm_map_offset_t) (VME_OBJECT(entry)->vo_shadow_offset)) + + VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT), + (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot, + (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0); + + if (kr != KERN_SUCCESS) { + goto cleanup; + } + } else { /* * Set up a block mapped area */ assert((uint32_t)((ldelta + hdelta) >> PAGE_SHIFT) == ((ldelta + hdelta) >> PAGE_SHIFT)); - pmap_map_block(real_map->pmap, - (addr64_t)(vaddr - ldelta), - (ppnum_t)((((vm_map_offset_t)(VME_OBJECT(entry)->vo_shadow_offset)) + - VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT), - (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot, - (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0); + kr = pmap_map_block(real_map->pmap, + (addr64_t)(vaddr - ldelta), + (ppnum_t)((((vm_map_offset_t)(VME_OBJECT(entry)->vo_shadow_offset)) + + VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT), + (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot, + (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0); + + if (kr != KERN_SUCCESS) { + goto cleanup; + } } } } + /* + * Success + */ + kr = KERN_SUCCESS; + + /* + * TODO: could most of the done cases just use cleanup? + */ +cleanup: /* * Unlock everything, and return */ - vm_map_verify_done(map, &version); + vm_map_unlock_read(map); if (real_map != map) vm_map_unlock(real_map); @@ -5066,7 +5108,6 @@ handle_copy_delay: #undef RELEASE_PAGE - kr = KERN_SUCCESS; done: thread_interrupt_level(interruptible_state); @@ -5092,7 +5133,7 @@ done: } } } - KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END, ((uint64_t)trace_vaddr >> 32), trace_vaddr, @@ -5113,6 +5154,7 @@ vm_fault_wire( vm_map_t map, vm_map_entry_t entry, vm_prot_t prot, + vm_tag_t wire_tag, pmap_t pmap, vm_map_offset_t pmap_addr, ppnum_t *physpage_p) @@ -5123,8 +5165,8 @@ vm_fault_wire( assert(entry->in_transition); - if ((VME_OBJECT(entry) != NULL) && - !entry->is_sub_map && + if ((VME_OBJECT(entry) != NULL) && + !entry->is_sub_map && VME_OBJECT(entry)->phys_contiguous) { return KERN_SUCCESS; } @@ -5135,7 +5177,7 @@ vm_fault_wire( * page tables and such can be locked down as well. */ - pmap_pageable(pmap, pmap_addr, + pmap_pageable(pmap, pmap_addr, pmap_addr + (end_addr - entry->vme_start), FALSE); /* @@ -5144,14 +5186,14 @@ vm_fault_wire( */ for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) { - rc = vm_fault_wire_fast(map, va, prot, entry, pmap, + rc = vm_fault_wire_fast(map, va, prot, wire_tag, entry, pmap, pmap_addr + (va - entry->vme_start), physpage_p); if (rc != KERN_SUCCESS) { - rc = vm_fault_internal(map, va, prot, TRUE, + rc = vm_fault_internal(map, va, prot, TRUE, wire_tag, ((pmap == kernel_pmap) ? THREAD_UNINT - : THREAD_ABORTSAFE), + : THREAD_ABORTSAFE), pmap, (pmap_addr + (va - entry->vme_start)), @@ -5164,7 +5206,7 @@ vm_fault_wire( /* unwire wired pages */ tmp_entry.vme_end = va; - vm_fault_unwire(map, + vm_fault_unwire(map, &tmp_entry, FALSE, pmap, pmap_addr); return rc; @@ -5190,6 +5232,7 @@ vm_fault_unwire( vm_map_offset_t end_addr = entry->vme_end; vm_object_t object; struct vm_object_fault_info fault_info; + unsigned int unwired_pages; object = (entry->is_sub_map) ? VM_OBJECT_NULL : VME_OBJECT(entry); @@ -5219,6 +5262,8 @@ vm_fault_unwire( fault_info.mark_zf_absent = FALSE; fault_info.batch_pmap_op = FALSE; + unwired_pages = 0; + /* * Since the pages are wired down, we must be able to * get their mappings from the physical map system. @@ -5228,11 +5273,11 @@ vm_fault_unwire( if (object == VM_OBJECT_NULL) { if (pmap) { - pmap_change_wiring(pmap, + pmap_change_wiring(pmap, pmap_addr + (va - entry->vme_start), FALSE); } - (void) vm_fault(map, va, VM_PROT_NONE, - TRUE, THREAD_UNINT, pmap, pmap_addr); + (void) vm_fault(map, va, VM_PROT_NONE, + TRUE, VM_KERN_MEMORY_NONE, THREAD_UNINT, pmap, pmap_addr); } else { vm_prot_t prot; vm_page_t result_page; @@ -5265,7 +5310,7 @@ vm_fault_unwire( FALSE, /* page not looked up */ &prot, &result_page, &top_page, (int *)0, - NULL, map->no_zero_fill, + NULL, map->no_zero_fill, FALSE, &fault_info); } while (result == VM_FAULT_RETRY); @@ -5275,7 +5320,7 @@ vm_fault_unwire( * move on to the next one in case the remaining pages are mapped from * different objects. During a forced unmount, the object is terminated * so the alive flag will be false if this happens. A forced unmount will - * will occur when an external disk is unplugged before the user does an + * will occur when an external disk is unplugged before the user does an * eject, so we don't want to panic in that situation. */ @@ -5303,10 +5348,13 @@ vm_fault_unwire( assert(VM_PAGE_GET_PHYS_PAGE(result_page) != vm_page_fictitious_addr); pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(result_page)); + if (VM_PAGE_WIRED(result_page)) { + unwired_pages++; + } VM_PAGE_FREE(result_page); } else { if ((pmap) && (VM_PAGE_GET_PHYS_PAGE(result_page) != vm_page_guard_addr)) - pmap_change_wiring(pmap, + pmap_change_wiring(pmap, pmap_addr + (va - entry->vme_start), FALSE); @@ -5314,6 +5362,7 @@ vm_fault_unwire( vm_page_lockspin_queues(); vm_page_unwire(result_page, TRUE); vm_page_unlock_queues(); + unwired_pages++; } if(entry->zero_wired_pages) { pmap_zero_page(VM_PAGE_GET_PHYS_PAGE(result_page)); @@ -5332,9 +5381,12 @@ vm_fault_unwire( * such may be unwired themselves. */ - pmap_pageable(pmap, pmap_addr, + pmap_pageable(pmap, pmap_addr, pmap_addr + (end_addr - entry->vme_start), TRUE); + if (kernel_object == object) { + vm_tag_update_size(fault_info.user_tag, -ptoa_64(unwired_pages)); + } } /* @@ -5361,7 +5413,8 @@ static kern_return_t vm_fault_wire_fast( __unused vm_map_t map, vm_map_offset_t va, - vm_prot_t caller_prot, + __unused vm_prot_t caller_prot, + vm_tag_t wire_tag, vm_map_entry_t entry, pmap_t pmap, vm_map_offset_t pmap_addr, @@ -5456,17 +5509,13 @@ vm_fault_wire_fast( /* * Look for page in top-level object. If it's not there or * there's something going on, give up. - * ENCRYPTED SWAP: use the slow fault path, since we'll need to - * decrypt the page before wiring it down. */ m = vm_page_lookup(object, offset); - if ((m == VM_PAGE_NULL) || (m->busy) || (m->encrypted) || + if ((m == VM_PAGE_NULL) || (m->busy) || (m->unusual && ( m->error || m->restart || m->absent))) { GIVE_UP; } - ASSERT_PAGE_DECRYPTED(m); - if (m->fictitious && VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) { /* @@ -5479,11 +5528,11 @@ vm_fault_wire_fast( /* * Wire the page down now. All bail outs beyond this - * point must unwire the page. + * point must unwire the page. */ vm_page_lockspin_queues(); - vm_page_wire(m, VM_PROT_MEMORY_TAG(caller_prot), TRUE); + vm_page_wire(m, wire_tag, TRUE); vm_page_unlock_queues(); /* @@ -5510,10 +5559,11 @@ vm_fault_wire_fast( pmap_addr, prot, prot, - TRUE, - FALSE, - FALSE, - FALSE, + TRUE, /* wired */ + FALSE, /* change_wiring */ + wire_tag, + FALSE, /* no_cache */ + FALSE, /* cs_bypass */ VME_ALIAS(entry), ((entry->iokit_acct || (!entry->is_sub_map && !entry->use_pmap)) @@ -5589,7 +5639,7 @@ vm_fault_copy_dst_cleanup( vm_page_lockspin_queues(); vm_page_unwire(page, TRUE); vm_page_unlock_queues(); - vm_object_paging_end(object); + vm_object_paging_end(object); vm_object_unlock(object); } } @@ -5633,7 +5683,7 @@ vm_fault_copy( int interruptible) { vm_page_t result_page; - + vm_page_t src_page; vm_page_t src_top_page; vm_prot_t src_prot; @@ -5763,7 +5813,7 @@ vm_fault_copy( * same, the call to vm_fault_page() for the * destination page will deadlock. To prevent this we * wire the page so we can drop busy without having - * the page daemon steal the page. We clean up the + * the page daemon steal the page. We clean up the * top page but keep the paging reference on the object * holding the dest page so it doesn't go away. */ @@ -5814,11 +5864,11 @@ vm_fault_copy( 0,0,0,0,0); result_page = VM_PAGE_NULL; result = vm_fault_page( - src_object, + src_object, vm_object_trunc_page(src_offset), VM_PROT_READ, FALSE, FALSE, /* page not looked up */ - &src_prot, + &src_prot, &result_page, &src_top_page, (int *)0, &error, FALSE, FALSE, &fault_info_src); @@ -5860,7 +5910,10 @@ vm_fault_copy( vm_object_unlock(result_page_object); } + vm_map_lock_read(dst_map); + if (!vm_map_verify(dst_map, dst_version)) { + vm_map_unlock_read(dst_map); if (result_page != VM_PAGE_NULL && src_page != dst_page) vm_fault_copy_cleanup(result_page, src_top_page); vm_fault_copy_dst_cleanup(dst_page); @@ -5872,7 +5925,7 @@ vm_fault_copy( if (dst_object->copy != old_copy_object) { vm_object_unlock(dst_object); - vm_map_verify_done(dst_map, dst_version); + vm_map_unlock_read(dst_map); if (result_page != VM_PAGE_NULL && src_page != dst_page) vm_fault_copy_cleanup(result_page, src_top_page); vm_fault_copy_dst_cleanup(dst_page); @@ -5949,7 +6002,7 @@ vm_fault_copy( * Unlock everything, and return */ - vm_map_verify_done(dst_map, dst_version); + vm_map_unlock_read(dst_map); if (result_page != VM_PAGE_NULL && src_page != dst_page) vm_fault_copy_cleanup(result_page, src_top_page); @@ -5963,7 +6016,7 @@ vm_fault_copy( RETURN(KERN_SUCCESS); #undef RETURN - /*NOTREACHED*/ + /*NOTREACHED*/ } #if VM_FAULT_CLASSIFY @@ -5996,7 +6049,7 @@ vm_fault_classify(vm_object_t object, while (TRUE) { m = vm_page_lookup(object, offset); - if (m != VM_PAGE_NULL) { + if (m != VM_PAGE_NULL) { if (m->busy || m->error || m->restart || m->absent) { type = VM_FAULT_TYPE_OTHER; break; @@ -6004,7 +6057,7 @@ vm_fault_classify(vm_object_t object, if (((fault_type & VM_PROT_WRITE) == 0) || ((level == 0) && object->copy == VM_OBJECT_NULL)) { type = VM_FAULT_TYPE_MAP_IN; - break; + break; } type = VM_FAULT_TYPE_COPY; break; @@ -6126,16 +6179,6 @@ kdp_lightweight_fault(vm_map_t map, vm_offset_t cur_target_addr) return 0; } - assert(!m->encrypted); - if (m->encrypted) { - return 0; - } - - assert(!m->encrypted_cleaning); - if (m->encrypted_cleaning) { - return 0; - } - assert(m->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR); if (m->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) { return 0; @@ -6301,7 +6344,7 @@ vm_page_validate_cs( } assert(!page->slid); -#if CHECK_CS_VALIDATION_BITMAP +#if CHECK_CS_VALIDATION_BITMAP if ( vnode_pager_cs_check_validation_bitmap( object->pager, trunc_page(page->offset + object->paging_offset), CS_BITMAP_CHECK ) == KERN_SUCCESS) { page->cs_validated = TRUE; page->cs_tainted = FALSE; @@ -6319,7 +6362,7 @@ vm_page_validate_cs( /* keep page busy while we map (and unlock) the VM object */ page->busy = TRUE; } - + /* * Take a paging reference on the VM object * to protect it from collapse or bypass, @@ -6347,7 +6390,7 @@ vm_page_validate_cs( /* validate the mapped page */ vm_page_validate_cs_mapped(page, (const void *) kaddr); -#if CHECK_CS_VALIDATION_BITMAP +#if CHECK_CS_VALIDATION_BITMAP if ( page->cs_validated == TRUE && page->cs_tainted == FALSE ) { vnode_pager_cs_check_validation_bitmap( object->pager, trunc_page( offset + object->paging_offset), CS_BITMAP_SET ); } diff --git a/osfmk/vm/vm_fault.h b/osfmk/vm/vm_fault.h index d5e8ed44a..666b7ef52 100644 --- a/osfmk/vm/vm_fault.h +++ b/osfmk/vm/vm_fault.h @@ -91,6 +91,9 @@ extern kern_return_t vm_fault( vm_map_offset_t vaddr, vm_prot_t fault_type, boolean_t change_wiring, +#if XNU_KERNEL_PRIVATE + vm_tag_t wire_tag, /* if wiring must pass tag != VM_KERN_MEMORY_NONE */ +#endif int interruptible, pmap_t pmap, vm_map_offset_t pmap_addr); @@ -105,6 +108,16 @@ extern void vm_pre_fault(vm_map_offset_t); extern void vm_fault_init(void); +/* exported kext version */ +extern kern_return_t vm_fault_external( + vm_map_t map, + vm_map_offset_t vaddr, + vm_prot_t fault_type, + boolean_t change_wiring, + int interruptible, + pmap_t caller_pmap, + vm_map_offset_t caller_pmap_addr); + /* * Page fault handling based on vm_object only. */ @@ -138,6 +151,7 @@ extern kern_return_t vm_fault_wire( vm_map_t map, vm_map_entry_t entry, vm_prot_t prot, + vm_tag_t wire_tag, pmap_t pmap, vm_map_offset_t pmap_addr, ppnum_t *physpage_p); @@ -167,6 +181,7 @@ extern kern_return_t vm_fault_enter( vm_prot_t fault_type, boolean_t wired, boolean_t change_wiring, + vm_tag_t wire_tag, /* if wiring must pass tag != VM_KERN_MEMORY_NONE */ boolean_t no_cache, boolean_t cs_bypass, int user_tag, diff --git a/osfmk/vm/vm_fourk_pager.c b/osfmk/vm/vm_fourk_pager.c index 785bbf5f8..407cbb916 100644 --- a/osfmk/vm/vm_fourk_pager.c +++ b/osfmk/vm/vm_fourk_pager.c @@ -143,9 +143,10 @@ typedef struct fourk_pager_backing { vm_object_offset_t backing_offset; } *fourk_pager_backing_t; typedef struct fourk_pager { - struct ipc_object_header pager_header; /* fake ip_kotype() */ - memory_object_pager_ops_t pager_ops; /* == &fourk_pager_ops */ - memory_object_control_t pager_control; /* mem object control handle */ + /* mandatory generic header */ + struct memory_object fourk_pgr_hdr; + + /* pager-specific data */ queue_chain_t pager_queue; /* next & prev pagers */ unsigned int ref_count; /* reference count */ int is_ready; /* is this pager ready ? */ @@ -154,7 +155,6 @@ typedef struct fourk_pager { 4K-chunk */ } *fourk_pager_t; #define FOURK_PAGER_NULL ((fourk_pager_t) NULL) -#define pager_ikot pager_header.io_bits /* * List of memory objects managed by this EMM. @@ -249,7 +249,7 @@ fourk_pager_init( memory_object_control_reference(control); - pager->pager_control = control; + pager->fourk_pgr_hdr.mo_control = control; attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ @@ -395,7 +395,7 @@ fourk_pager_terminate_internal( } /* trigger the destruction of the memory object */ - memory_object_destroy(pager->pager_control, 0); + memory_object_destroy(pager->fourk_pgr_hdr.mo_control, 0); } /* @@ -447,9 +447,9 @@ fourk_pager_deallocate_internal( * pager structure. */ lck_mtx_unlock(&fourk_pager_lock); - if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) { - memory_object_control_deallocate(pager->pager_control); - pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; + if (pager->fourk_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { + memory_object_control_deallocate(pager->fourk_pgr_hdr.mo_control); + pager->fourk_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; } kfree(pager, sizeof (*pager)); pager = FOURK_PAGER_NULL; @@ -501,21 +501,13 @@ fourk_pager_terminate( */ kern_return_t fourk_pager_synchronize( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length, __unused vm_sync_t sync_flags) { - fourk_pager_t pager; - - PAGER_DEBUG(PAGER_ALL, ("fourk_pager_synchronize: %p\n", mem_obj)); - - pager = fourk_pager_lookup(mem_obj); - - memory_object_synchronize_completed(pager->pager_control, - offset, length); - - return KERN_SUCCESS; + panic("fourk_pager_synchronize: memory_object_synchronize no longer supported\n"); + return (KERN_FAILURE); } /* @@ -604,8 +596,8 @@ fourk_pager_lookup( { fourk_pager_t pager; + assert(mem_obj->mo_pager_ops == &fourk_pager_ops); pager = (fourk_pager_t) mem_obj; - assert(pager->pager_ops == &fourk_pager_ops); assert(pager->ref_count > 0); return pager; } @@ -701,8 +693,8 @@ fourk_pager_to_vm_object( } assert(pager->ref_count > 0); - assert(pager->pager_control != MEMORY_OBJECT_CONTROL_NULL); - object = memory_object_control_to_vm_object(pager->pager_control); + assert(pager->fourk_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL); + object = memory_object_control_to_vm_object(pager->fourk_pgr_hdr.mo_control); assert(object != VM_OBJECT_NULL); return object; } @@ -734,9 +726,10 @@ fourk_pager_create(void) * we reserve the first word in the object for a fake ip_kotype * setting - that will tell vm_map to use it as a memory object. */ - pager->pager_ops = &fourk_pager_ops; - pager->pager_ikot = IKOT_MEMORY_OBJECT; - pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; + pager->fourk_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; + pager->fourk_pgr_hdr.mo_pager_ops = &fourk_pager_ops; + pager->fourk_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; + pager->ref_count = 2; /* existence + setup reference */ pager->is_ready = FALSE;/* not ready until it has a "name" */ pager->is_mapped = FALSE; @@ -821,7 +814,7 @@ fourk_pager_data_request( /* * Gather in a UPL all the VM pages requested by VM. */ - mo_control = pager->pager_control; + mo_control = pager->fourk_pgr_hdr.mo_control; upl_size = length; upl_flags = @@ -833,7 +826,7 @@ fourk_pager_data_request( pl_count = 0; kr = memory_object_upl_request(mo_control, offset, upl_size, - &upl, NULL, NULL, upl_flags); + &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_NONE); if (kr != KERN_SUCCESS) { retval = kr; goto done; @@ -857,6 +850,7 @@ fourk_pager_data_request( 2 * PAGE_SIZE_64, 0, 0, + VM_MAP_KERNEL_FLAGS_NONE, &map_entry); if (kr != KERN_SUCCESS) { vm_object_deallocate(kernel_object); @@ -899,14 +893,19 @@ fourk_pager_data_request( #if __x86_64__ dst_vaddr = (vm_map_offset_t) PHYSMAP_PTOV((pmap_paddr_t)dst_pnum << PAGE_SHIFT); +#elif __arm__ || __arm64__ + dst_vaddr = (vm_map_offset_t) + phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT); #else - pmap_enter(kernel_pmap, - dst_vaddr, - dst_pnum, - VM_PROT_READ | VM_PROT_WRITE, - VM_PROT_NONE, - 0, - TRUE); + kr = pmap_enter(kernel_pmap, + dst_vaddr, + dst_pnum, + VM_PROT_READ | VM_PROT_WRITE, + VM_PROT_NONE, + 0, + TRUE); + + assert(kr == KERN_SUCCESS); #endif /* retrieve appropriate data for each 4K-page in this page */ @@ -1087,18 +1086,24 @@ fourk_pager_data_request( src_vaddr = (vm_map_offset_t) PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) << PAGE_SHIFT); +#elif __arm__ || __arm64__ + src_vaddr = (vm_map_offset_t) + phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) + << PAGE_SHIFT); #else /* * Establish an explicit mapping of the source * physical page. */ - pmap_enter(kernel_pmap, - src_vaddr, - VM_PAGE_GET_PHYS_PAGE(src_page), - VM_PROT_READ, - VM_PROT_NONE, - 0, - TRUE); + kr = pmap_enter(kernel_pmap, + src_vaddr, + VM_PAGE_GET_PHYS_PAGE(src_page), + VM_PROT_READ, + VM_PROT_NONE, + 0, + TRUE); + + assert(kr == KERN_SUCCESS); #endif /* @@ -1327,7 +1332,7 @@ fourk_pager_populate( } assert(pager->ref_count > 0); - assert(pager->pager_control != MEMORY_OBJECT_CONTROL_NULL); + assert(pager->fourk_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL); if (index < 0 || index > FOURK_PAGER_SLOTS) { return KERN_INVALID_ARGUMENT; diff --git a/osfmk/vm/vm_init.c b/osfmk/vm/vm_init.c index bfbb2e54e..e4944b459 100644 --- a/osfmk/vm/vm_init.c +++ b/osfmk/vm/vm_init.c @@ -97,6 +97,9 @@ boolean_t zlog_ready = FALSE; vm_offset_t kmapoff_kaddr; unsigned int kmapoff_pgcnt; +#if CONFIG_EMBEDDED +extern int log_executable_mem_entry; +#endif /* CONFIG_EMBEDDED */ static inline void vm_mem_bootstrap_log(const char *message) @@ -153,10 +156,15 @@ vm_mem_bootstrap(void) kmapoff_pgcnt = early_random() & 0x1ff; /* 9 bits */ if (kmapoff_pgcnt > 0 && - vm_allocate(kernel_map, &kmapoff_kaddr, - kmapoff_pgcnt * PAGE_SIZE_64, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_OSFMK)) != KERN_SUCCESS) + vm_allocate_kernel(kernel_map, &kmapoff_kaddr, + kmapoff_pgcnt * PAGE_SIZE_64, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_OSFMK) != KERN_SUCCESS) panic("cannot vm_allocate %u kernel_map pages", kmapoff_pgcnt); +#if CONFIG_EMBEDDED + PE_parse_boot_argn("log_executable_mem_entry", + &log_executable_mem_entry, + sizeof (log_executable_mem_entry)); +#endif /* CONFIG_EMBEDDED */ vm_mem_bootstrap_log("pmap_init"); pmap_init(); @@ -181,6 +189,25 @@ vm_mem_bootstrap(void) zsize = ZONE_MAP_MAX; /* Clamp to 1.5GB max for K32 */ #endif /* !__LP64__ */ +#if CONFIG_EMBEDDED +#if defined(__LP64__) + { + mach_vm_size_t max_zsize; + + /* + * because of the limited kernel virtual space for embedded systems, + * we need to clamp the size of the zone map being created... replicate + * the above calculation for a 1Gbyte, LP64 system and use that as the + * maximum size for the zone map + */ + max_zsize = (1024ULL * 1024ULL * 1024ULL) >> 2ULL; + max_zsize += max_zsize >> 1; + + if (zsize > max_zsize) + zsize = max_zsize; + } +#endif +#endif vm_mem_bootstrap_log("kext_alloc_init"); kext_alloc_init(); diff --git a/osfmk/vm/vm_init.h b/osfmk/vm/vm_init.h index 5901f9e2e..8e23b580b 100644 --- a/osfmk/vm/vm_init.h +++ b/osfmk/vm/vm_init.h @@ -35,6 +35,5 @@ extern void vm_mem_bootstrap(void); extern void vm_mem_init(void); extern void vm_map_steal_memory(void);; -extern void vm_user_init(void); #endif /* VM_INIT_H */ diff --git a/osfmk/vm/vm_kern.c b/osfmk/vm/vm_kern.c index 8d37f4cf0..a0301085e 100644 --- a/osfmk/vm/vm_kern.c +++ b/osfmk/vm/vm_kern.c @@ -71,6 +71,7 @@ #include #include #include +#include #include #include #include @@ -78,8 +79,11 @@ #include #include +#include #include +#include + /* * Variables exported by this module. */ @@ -117,6 +121,8 @@ kmem_alloc_contig( vm_page_t m, pages; kern_return_t kr; + assert(VM_KERN_MEMORY_NONE != tag); + if (map == VM_MAP_NULL || (flags & ~(KMA_KOBJECT | KMA_LOMEM | KMA_NOPAGEWAIT))) return KERN_INVALID_ARGUMENT; @@ -142,7 +148,8 @@ kmem_alloc_contig( object = vm_object_allocate(map_size); } - kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, &entry); + kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, + VM_MAP_KERNEL_FLAGS_NONE, tag, &entry); if (KERN_SUCCESS != kr) { vm_object_deallocate(object); return kr; @@ -155,7 +162,6 @@ kmem_alloc_contig( } VME_OBJECT_SET(entry, object); VME_OFFSET_SET(entry, offset); - VME_ALIAS_SET(entry, tag); /* Take an extra object ref in case the map entry gets deleted */ vm_object_reference(object); @@ -185,12 +191,12 @@ kmem_alloc_contig( } vm_object_unlock(object); - kr = vm_map_wire(map, + kr = vm_map_wire_kernel(map, vm_map_trunc_page(map_addr, VM_MAP_PAGE_MASK(map)), vm_map_round_page(map_addr + map_size, VM_MAP_PAGE_MASK(map)), - VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(tag), + VM_PROT_DEFAULT, tag, FALSE); if (kr != KERN_SUCCESS) { @@ -210,11 +216,13 @@ kmem_alloc_contig( } vm_object_deallocate(object); - if (object == kernel_object) + if (object == kernel_object) { vm_map_simplify(map, map_addr); - + vm_tag_update_size(tag, map_size); + } *addrp = (vm_offset_t) map_addr; assert((vm_map_offset_t) *addrp == map_addr); + return KERN_SUCCESS; } @@ -259,6 +267,7 @@ kernel_memory_allocate( int wired_page_count = 0; int i; int vm_alloc_flags; + vm_map_kernel_flags_t vmk_flags; vm_prot_t kma_prot; if (! vm_kernel_ready) { @@ -269,7 +278,8 @@ kernel_memory_allocate( VM_MAP_PAGE_MASK(map)); map_mask = (vm_map_offset_t) mask; - vm_alloc_flags = VM_MAKE_TAG(tag); + vm_alloc_flags = 0; //VM_MAKE_TAG(tag); + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; /* Check for zero allocation size (either directly or via overflow) */ if (map_size == 0) { @@ -307,7 +317,7 @@ kernel_memory_allocate( fill_size = map_size; if (flags & KMA_GUARD_FIRST) { - vm_alloc_flags |= VM_FLAGS_GUARD_BEFORE; + vmk_flags.vmkf_guard_before = TRUE; fill_start += PAGE_SIZE_64; fill_size -= PAGE_SIZE_64; if (map_size < fill_start + fill_size) { @@ -318,7 +328,7 @@ kernel_memory_allocate( guard_page_count++; } if (flags & KMA_GUARD_LAST) { - vm_alloc_flags |= VM_FLAGS_GUARD_AFTER; + vmk_flags.vmkf_guard_after = TRUE; fill_size -= PAGE_SIZE_64; if (map_size <= fill_start + fill_size) { /* no space for a guard page */ @@ -375,6 +385,7 @@ kernel_memory_allocate( } VM_PAGE_WAIT(); } + if (KMA_ZERO & flags) vm_page_zero_fill(mem); mem->snext = wired_page_list; wired_page_list = mem; } @@ -394,12 +405,12 @@ kernel_memory_allocate( object = vm_object_allocate(map_size); } - if (flags & KMA_ATOMIC) - vm_alloc_flags |= VM_FLAGS_ATOMIC_ENTRY; - + if (flags & KMA_ATOMIC) + vmk_flags.vmkf_atomic_entry = TRUE; + kr = vm_map_find_space(map, &map_addr, fill_size, map_mask, - vm_alloc_flags, &entry); + vm_alloc_flags, vmk_flags, tag, &entry); if (KERN_SUCCESS != kr) { vm_object_deallocate(object); goto out; @@ -443,6 +454,13 @@ kernel_memory_allocate( kma_prot = VM_PROT_READ | VM_PROT_WRITE; +#if KASAN + if (!(flags & KMA_VAONLY)) { + /* for VAONLY mappings we notify in populate only */ + kasan_notify_address(map_addr, size); + } +#endif + if (flags & KMA_VAONLY) { pg_offset = fill_start + fill_size; } else { @@ -478,16 +496,21 @@ kernel_memory_allocate( vm_object_unlock(object); PMAP_ENTER(kernel_pmap, map_addr + pg_offset, mem, - kma_prot, VM_PROT_NONE, ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE); + kma_prot, VM_PROT_NONE, ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE, + pe_result); vm_object_lock(object); } + + assert(pe_result == KERN_SUCCESS); + if (flags & KMA_NOENCRYPT) { bzero(CAST_DOWN(void *, (map_addr + pg_offset)), PAGE_SIZE); pmap_set_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)); } } + if (kernel_object == object) vm_tag_update_size(tag, fill_size); } if ((fill_start + fill_size) < map_size) { if (guard_page_list == NULL) @@ -569,6 +592,7 @@ kernel_memory_populate( VM_PAGE_WAIT(); } + if (KMA_ZERO & flags) vm_page_zero_fill(mem); mem->snext = page_list; page_list = mem; @@ -605,6 +629,13 @@ kernel_memory_populate( } vm_object_unlock(object); +#if KASAN + if (map == compressor_map) { + kasan_notify_address_nopoison(addr, size); + } else { + kasan_notify_address(addr, size); + } +#endif return KERN_SUCCESS; } @@ -629,6 +660,7 @@ kernel_memory_populate( } VM_PAGE_WAIT(); } + if (KMA_ZERO & flags) vm_page_zero_fill(mem); mem->snext = page_list; page_list = mem; } @@ -687,10 +719,14 @@ kernel_memory_populate( PMAP_ENTER(kernel_pmap, addr + pg_offset, mem, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, - ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE); + ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE, + pe_result); vm_object_lock(object); } + + assert(pe_result == KERN_SUCCESS); + if (flags & KMA_NOENCRYPT) { bzero(CAST_DOWN(void *, (addr + pg_offset)), PAGE_SIZE); pmap_set_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)); @@ -700,8 +736,17 @@ kernel_memory_populate( vm_page_wire_count += page_count; vm_page_unlock_queues(); + if (kernel_object == object) vm_tag_update_size(tag, size); + vm_object_unlock(object); +#if KASAN + if (map == compressor_map) { + kasan_notify_address_nopoison(addr, size); + } else { + kasan_notify_address(addr, size); + } +#endif return KERN_SUCCESS; out: @@ -734,7 +779,6 @@ kernel_memory_depopulate( } else if (flags & KMA_KOBJECT) { offset = addr; object = kernel_object; - vm_object_lock(object); } else { offset = 0; @@ -899,7 +943,10 @@ kmem_realloc( */ kr = vm_map_find_space(map, &newmapaddr, newmapsize, - (vm_map_offset_t) 0, 0, &newentry); + (vm_map_offset_t) 0, 0, + VM_MAP_KERNEL_FLAGS_NONE, + tag, + &newentry); if (kr != KERN_SUCCESS) { vm_object_lock(object); for(offset = oldmapsize; @@ -915,7 +962,6 @@ kmem_realloc( } VME_OBJECT_SET(newentry, object); VME_OFFSET_SET(newentry, 0); - VME_ALIAS_SET(newentry, tag); assert(newentry->wired_count == 0); @@ -924,8 +970,8 @@ kmem_realloc( vm_object_reference(object); vm_map_unlock(map); - kr = vm_map_wire(map, newmapaddr, newmapaddr + newmapsize, - VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(tag), FALSE); + kr = vm_map_wire_kernel(map, newmapaddr, newmapaddr + newmapsize, + VM_PROT_DEFAULT, tag, FALSE); if (KERN_SUCCESS != kr) { vm_map_remove(map, newmapaddr, newmapaddr + newmapsize, 0); vm_object_lock(object); @@ -941,6 +987,8 @@ kmem_realloc( } vm_object_deallocate(object); + if (kernel_object == object) vm_tag_update_size(tag, newmapsize); + *newaddrp = CAST_DOWN(vm_offset_t, newmapaddr); return KERN_SUCCESS; } @@ -1030,13 +1078,18 @@ kmem_alloc_pageable( kr = vm_map_enter(map, &map_addr, map_size, (vm_map_offset_t) 0, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(tag), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + tag, VM_OBJECT_NULL, (vm_object_offset_t) 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) return kr; +#if KASAN + kasan_notify_address(map_addr, map_size); +#endif *addrp = CAST_DOWN(vm_offset_t, map_addr); return KERN_SUCCESS; } @@ -1136,6 +1189,8 @@ kmem_suballoc( vm_size_t size, boolean_t pageable, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, vm_map_t *new_map) { vm_map_t map; @@ -1159,7 +1214,7 @@ kmem_suballoc( VM_MAP_PAGE_MASK(parent))); kr = vm_map_enter(parent, &map_addr, map_size, - (vm_map_offset_t) 0, flags, + (vm_map_offset_t) 0, flags, vmk_flags, tag, vm_submap_object, (vm_object_offset_t) 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) { @@ -1202,12 +1257,54 @@ kmem_init( { vm_map_offset_t map_start; vm_map_offset_t map_end; + vm_map_kernel_flags_t vmk_flags; + + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_permanent = TRUE; + vmk_flags.vmkf_no_pmap_check = TRUE; map_start = vm_map_trunc_page(start, VM_MAP_PAGE_MASK(kernel_map)); map_end = vm_map_round_page(end, VM_MAP_PAGE_MASK(kernel_map)); +#if defined(__arm__) || defined(__arm64__) + kernel_map = vm_map_create(pmap_kernel(),VM_MIN_KERNEL_AND_KEXT_ADDRESS, + VM_MAX_KERNEL_ADDRESS, FALSE); + /* + * Reserve virtual memory allocated up to this time. + */ + { + unsigned int region_select = 0; + vm_map_offset_t region_start; + vm_map_size_t region_size; + vm_map_offset_t map_addr; + kern_return_t kr; + + while (pmap_virtual_region(region_select, ®ion_start, ®ion_size)) { + + map_addr = region_start; + kr = vm_map_enter(kernel_map, &map_addr, + vm_map_round_page(region_size, + VM_MAP_PAGE_MASK(kernel_map)), + (vm_map_offset_t) 0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + VM_OBJECT_NULL, + (vm_object_offset_t) 0, FALSE, VM_PROT_NONE, VM_PROT_NONE, + VM_INHERIT_DEFAULT); + + if (kr != KERN_SUCCESS) { + panic("kmem_init(0x%llx,0x%llx): vm_map_enter(0x%llx,0x%llx) error 0x%x\n", + (uint64_t) start, (uint64_t) end, (uint64_t) region_start, + (uint64_t) region_size, kr); + } + + region_select++; + } + } +#else kernel_map = vm_map_create(pmap_kernel(),VM_MIN_KERNEL_AND_KEXT_ADDRESS, map_end, FALSE); /* @@ -1217,16 +1314,21 @@ kmem_init( vm_map_offset_t map_addr; kern_return_t kr; + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_no_pmap_check = TRUE; + map_addr = VM_MIN_KERNEL_AND_KEXT_ADDRESS; kr = vm_map_enter(kernel_map, - &map_addr, - (vm_map_size_t)(map_start - VM_MIN_KERNEL_AND_KEXT_ADDRESS), - (vm_map_offset_t) 0, - VM_FLAGS_FIXED | VM_FLAGS_NO_PMAP_CHECK, - VM_OBJECT_NULL, - (vm_object_offset_t) 0, FALSE, - VM_PROT_NONE, VM_PROT_NONE, - VM_INHERIT_DEFAULT); + &map_addr, + (vm_map_size_t)(map_start - VM_MIN_KERNEL_AND_KEXT_ADDRESS), + (vm_map_offset_t) 0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + VM_OBJECT_NULL, + (vm_object_offset_t) 0, FALSE, + VM_PROT_NONE, VM_PROT_NONE, + VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) { panic("kmem_init(0x%llx,0x%llx): vm_map_enter(0x%llx,0x%llx) error 0x%x\n", @@ -1236,6 +1338,7 @@ kmem_init( kr); } } +#endif /* * Set the default global user wire limit which limits the amount of @@ -1322,120 +1425,6 @@ copyoutmap( return KERN_SUCCESS; } - -kern_return_t -vm_conflict_check( - vm_map_t map, - vm_map_offset_t off, - vm_map_size_t len, - memory_object_t pager, - vm_object_offset_t file_off) -{ - vm_map_entry_t entry; - vm_object_t obj; - vm_object_offset_t obj_off; - vm_map_t base_map; - vm_map_offset_t base_offset; - vm_map_offset_t original_offset; - kern_return_t kr; - vm_map_size_t local_len; - - base_map = map; - base_offset = off; - original_offset = off; - kr = KERN_SUCCESS; - vm_map_lock(map); - while(vm_map_lookup_entry(map, off, &entry)) { - local_len = len; - - if (VME_OBJECT(entry) == VM_OBJECT_NULL) { - vm_map_unlock(map); - return KERN_SUCCESS; - } - if (entry->is_sub_map) { - vm_map_t old_map; - - old_map = map; - vm_map_lock(VME_SUBMAP(entry)); - map = VME_SUBMAP(entry); - off = VME_OFFSET(entry) + (off - entry->vme_start); - vm_map_unlock(old_map); - continue; - } - obj = VME_OBJECT(entry); - obj_off = (off - entry->vme_start) + VME_OFFSET(entry); - while(obj->shadow) { - obj_off += obj->vo_shadow_offset; - obj = obj->shadow; - } - if((obj->pager_created) && (obj->pager == pager)) { - if(((obj->paging_offset) + obj_off) == file_off) { - if(off != base_offset) { - vm_map_unlock(map); - return KERN_FAILURE; - } - kr = KERN_ALREADY_WAITING; - } else { - vm_object_offset_t obj_off_aligned; - vm_object_offset_t file_off_aligned; - - obj_off_aligned = obj_off & ~PAGE_MASK; - file_off_aligned = file_off & ~PAGE_MASK; - - if (file_off_aligned == (obj->paging_offset + obj_off_aligned)) { - /* - * the target map and the file offset start in the same page - * but are not identical... - */ - vm_map_unlock(map); - return KERN_FAILURE; - } - if ((file_off < (obj->paging_offset + obj_off_aligned)) && - ((file_off + len) > (obj->paging_offset + obj_off_aligned))) { - /* - * some portion of the tail of the I/O will fall - * within the encompass of the target map - */ - vm_map_unlock(map); - return KERN_FAILURE; - } - if ((file_off_aligned > (obj->paging_offset + obj_off)) && - (file_off_aligned < (obj->paging_offset + obj_off) + len)) { - /* - * the beginning page of the file offset falls within - * the target map's encompass - */ - vm_map_unlock(map); - return KERN_FAILURE; - } - } - } else if(kr != KERN_SUCCESS) { - vm_map_unlock(map); - return KERN_FAILURE; - } - - if(len <= ((entry->vme_end - entry->vme_start) - - (off - entry->vme_start))) { - vm_map_unlock(map); - return kr; - } else { - len -= (entry->vme_end - entry->vme_start) - - (off - entry->vme_start); - } - base_offset = base_offset + (local_len - len); - file_off = file_off + (local_len - len); - off = base_offset; - if(map != base_map) { - vm_map_unlock(map); - vm_map_lock(base_map); - map = base_map; - } - } - - vm_map_unlock(map); - return kr; -} - /* * * The following two functions are to be used when exposing kernel @@ -1447,43 +1436,84 @@ vm_conflict_check( * NOTE: USE THE MACRO VERSIONS OF THESE FUNCTIONS (in vm_param.h) FROM WITHIN THE KERNEL */ -/* - * vm_kernel_addrperm_external: - * - * Used when exposing an address to userspace which is in the kernel's - * "heap". These addresses are not loaded from anywhere and are resultingly - * unslid. We apply a permutation value to obscure the address. - */ -void -vm_kernel_addrperm_external( +static void +vm_kernel_addrhash_internal( vm_offset_t addr, - vm_offset_t *perm_addr) + vm_offset_t *hash_addr, + uint64_t salt) { + assert(salt != 0); + if (addr == 0) { - *perm_addr = 0; + *hash_addr = 0; return; } - *perm_addr = (addr + vm_kernel_addrperm_ext); - return; + if (VM_KERNEL_IS_SLID(addr)) { + *hash_addr = VM_KERNEL_UNSLIDE(addr); + return; + } + + vm_offset_t sha_digest[SHA256_DIGEST_LENGTH/sizeof(vm_offset_t)]; + SHA256_CTX sha_ctx; + + SHA256_Init(&sha_ctx); + SHA256_Update(&sha_ctx, &salt, sizeof(salt)); + SHA256_Update(&sha_ctx, &addr, sizeof(addr)); + SHA256_Final(sha_digest, &sha_ctx); + + *hash_addr = sha_digest[0]; +} + +void +vm_kernel_addrhash_external( + vm_offset_t addr, + vm_offset_t *hash_addr) +{ + return vm_kernel_addrhash_internal(addr, hash_addr, vm_kernel_addrhash_salt_ext); +} + +vm_offset_t +vm_kernel_addrhash(vm_offset_t addr) +{ + vm_offset_t hash_addr; + vm_kernel_addrhash_internal(addr, &hash_addr, vm_kernel_addrhash_salt); + return hash_addr; +} + +void +vm_kernel_addrhide( + vm_offset_t addr, + vm_offset_t *hide_addr) +{ + *hide_addr = VM_KERNEL_ADDRHIDE(addr); } /* + * vm_kernel_addrperm_external: * vm_kernel_unslide_or_perm_external: * - * Use this macro when exposing an address to userspace that could come from + * Use these macros when exposing an address to userspace that could come from * either kernel text/data *or* the heap. */ void -vm_kernel_unslide_or_perm_external( +vm_kernel_addrperm_external( vm_offset_t addr, - vm_offset_t *up_addr) + vm_offset_t *perm_addr) { if (VM_KERNEL_IS_SLID(addr)) { - *up_addr = addr - vm_kernel_slide; - return; + *perm_addr = VM_KERNEL_UNSLIDE(addr); + } else if (VM_KERNEL_ADDRESS(addr)) { + *perm_addr = addr + vm_kernel_addrperm_ext; + } else { + *perm_addr = addr; } +} +void +vm_kernel_unslide_or_perm_external( + vm_offset_t addr, + vm_offset_t *up_addr) +{ vm_kernel_addrperm_external(addr, up_addr); - return; } diff --git a/osfmk/vm/vm_kern.h b/osfmk/vm/vm_kern.h index 69afb3548..8cab89ce4 100644 --- a/osfmk/vm/vm_kern.h +++ b/osfmk/vm/vm_kern.h @@ -97,6 +97,7 @@ extern kern_return_t kernel_memory_allocate( #define KMA_VAONLY 0x200 #define KMA_COMPRESSOR 0x400 /* Pages belonging to the compressor are not on the paging queues, nor are they counted as wired. */ #define KMA_ATOMIC 0x800 +#define KMA_ZERO 0x1000 extern kern_return_t kmem_alloc( vm_map_t map, @@ -152,6 +153,8 @@ extern kern_return_t kmem_suballoc( vm_size_t size, boolean_t pageable, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, vm_map_t *new_map); extern kern_return_t kmem_alloc_kobject( @@ -180,55 +183,47 @@ extern kern_return_t memory_object_iopl_request( upl_t *upl_ptr, upl_page_info_array_t user_page_list, unsigned int *page_list_count, - upl_control_flags_t *flags); + upl_control_flags_t *flags, + vm_tag_t tag); struct mach_memory_info; -extern kern_return_t vm_page_diagnose(struct mach_memory_info * sites, - unsigned int num_sites, uint64_t zones_collectable_bytes); +extern kern_return_t vm_page_diagnose(struct mach_memory_info * info, + unsigned int num_info, uint64_t zones_collectable_bytes); + +extern uint32_t vm_page_diagnose_estimate(void); #if DEBUG || DEVELOPMENT -extern void kern_wired_diagnose(void); + +extern kern_return_t mach_memory_info_check(void); + +extern kern_return_t vm_kern_allocation_info(uintptr_t addr, vm_size_t * size, vm_tag_t * tag, vm_size_t * zone_size); + #endif /* DEBUG || DEVELOPMENT */ extern vm_tag_t vm_tag_bt(void); extern vm_tag_t vm_tag_alloc(vm_allocation_site_t * site); -extern void vm_tag_alloc_locked(vm_allocation_site_t * site); +extern void vm_tag_alloc_locked(vm_allocation_site_t * site, vm_allocation_site_t ** releasesiteP); -extern vm_tag_t vm_tag_bt_debug(void); +extern void vm_tag_update_size(vm_tag_t tag, int64_t size); -extern uint32_t vm_tag_get_kext(vm_tag_t tag, char * name, vm_size_t namelen); +#if VM_MAX_TAG_ZONES +extern void vm_allocation_zones_init(void); +extern void vm_tag_will_update_zone(vm_tag_t tag, uint32_t zidx); +extern void vm_tag_update_zone_size(vm_tag_t tag, uint32_t zidx, int64_t delta, int64_t dwaste); -#if DEBUG || DEVELOPMENT - -struct vm_tag_set_entry -{ - vm_tag_t tag; - uint32_t count; -}; - -struct vm_tag_set -{ - lck_spin_t lock; - struct vm_tag_set_entry entries[0]; -}; +extern vm_allocation_zone_total_t ** vm_allocation_zone_totals; -typedef struct vm_tag_set * vm_tag_set_t; +#endif /* VM_MAX_TAG_ZONES */ -extern void vm_tag_set_init(vm_tag_set_t, uint32_t count); -extern kern_return_t vm_tag_set_enter(vm_tag_set_t set, uint32_t count, vm_tag_t tag); -extern kern_return_t vm_tag_set_remove(vm_tag_set_t set, uint32_t count, vm_tag_t tag, vm_tag_t * new_tag); +extern vm_tag_t vm_tag_bt_debug(void); -#endif /* DEBUG || DEVELOPMENT */ +extern uint32_t vm_tag_get_kext(vm_tag_t tag, char * name, vm_size_t namelen); extern boolean_t vm_kernel_map_is_kernel(vm_map_t map); extern ppnum_t kernel_pmap_present_mapping(uint64_t vaddr, uint64_t * pvincr, uintptr_t * pvphysaddr); -#if DEBUG || DEVELOPMENT -extern void kernel_pmap_lock(void); -extern void kernel_pmap_unlock(void); -#endif /* DEBUG || DEVELOPMENT */ #else /* XNU_KERNEL_PRIVATE */ @@ -255,6 +250,23 @@ extern void kmem_free( #endif /* !XNU_KERNEL_PRIVATE */ +#ifdef XNU_KERNEL_PRIVATE +typedef struct vm_allocation_site kern_allocation_name; +typedef kern_allocation_name * kern_allocation_name_t; +#else /* XNU_KERNEL_PRIVATE */ +struct kern_allocation_name; +typedef struct kern_allocation_name * kern_allocation_name_t; +#endif /* !XNU_KERNEL_PRIVATE */ + +extern kern_allocation_name_t kern_allocation_name_allocate(const char * name, uint32_t suballocs); +extern void kern_allocation_name_release(kern_allocation_name_t allocation); +extern const char * kern_allocation_get_name(kern_allocation_name_t allocation); +#ifdef XNU_KERNEL_PRIVATE +extern void kern_allocation_update_size(kern_allocation_name_t allocation, int64_t delta); +extern void kern_allocation_update_subtotal(kern_allocation_name_t allocation, uint32_t subtag, int64_t delta); +extern vm_tag_t kern_allocation_name_get_vm_tag(kern_allocation_name_t allocation); +#endif /* XNU_KERNEL_PRIVATE */ + #ifdef MACH_KERNEL_PRIVATE extern void kmem_init( @@ -274,13 +286,6 @@ extern kern_return_t copyoutmap( vm_map_offset_t toaddr, vm_size_t length); -extern kern_return_t vm_conflict_check( - vm_map_t map, - vm_map_offset_t off, - vm_map_size_t len, - memory_object_t pager, - vm_object_offset_t file_off); - extern kern_return_t kmem_alloc_external( vm_map_t map, vm_offset_t *addrp, @@ -298,6 +303,120 @@ extern kern_return_t kmem_alloc_pageable_external( #endif /* MACH_KERNEL_PRIVATE */ +#ifdef XNU_KERNEL_PRIVATE + +extern kern_return_t mach_vm_allocate_kernel( + vm_map_t map, + mach_vm_offset_t *addr, + mach_vm_size_t size, + int flags, + vm_tag_t tag); + +extern kern_return_t vm_allocate_kernel( + vm_map_t map, + vm_offset_t *addr, + vm_size_t size, + int flags, + vm_tag_t tag); + + +extern kern_return_t mach_vm_map_kernel( + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t initial_size, + mach_vm_offset_t mask, + int flags, + vm_tag_t tag, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance); + + +extern kern_return_t vm_map_kernel( + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + vm_tag_t tag, + ipc_port_t port, + vm_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance); + +extern kern_return_t mach_vm_remap_kernel( + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t size, + mach_vm_offset_t mask, + int flags, + vm_tag_t tag, + vm_map_t src_map, + mach_vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance); + +extern kern_return_t vm_remap_kernel( + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + vm_tag_t tag, + vm_map_t src_map, + vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance); + +extern kern_return_t vm_map_64_kernel( + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + vm_tag_t tag, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance); + +extern kern_return_t mach_vm_wire_kernel( + host_priv_t host_priv, + vm_map_t map, + mach_vm_offset_t start, + mach_vm_size_t size, + vm_prot_t access, + vm_tag_t tag); + +extern kern_return_t vm_map_wire_kernel( + vm_map_t map, + vm_map_offset_t start, + vm_map_offset_t end, + vm_prot_t caller_prot, + vm_tag_t tag, + boolean_t user_wire); + +extern kern_return_t vm_map_wire_and_extract_kernel( + vm_map_t map, + vm_map_offset_t start, + vm_prot_t caller_prot, + vm_tag_t tag, + boolean_t user_wire, + ppnum_t *physpage_p); + +#endif /* XNU_KERNEL_PRIVATE */ + extern vm_map_t kernel_map; extern vm_map_t kernel_pageable_map; extern vm_map_t ipc_kernel_map; @@ -306,6 +425,14 @@ extern vm_map_t ipc_kernel_map; #ifdef KERNEL +__BEGIN_DECLS +extern vm_offset_t vm_kernel_addrhash(vm_offset_t addr); +__END_DECLS + +extern void vm_kernel_addrhide( + vm_offset_t addr, + vm_offset_t *hide_addr); + extern vm_offset_t vm_kernel_addrperm_ext; extern void vm_kernel_addrperm_external( @@ -315,5 +442,17 @@ extern void vm_kernel_addrperm_external( extern void vm_kernel_unslide_or_perm_external( vm_offset_t addr, vm_offset_t *up_addr); + +#if MACH_KERNEL_PRIVATE +extern uint64_t vm_kernel_addrhash_salt; +extern uint64_t vm_kernel_addrhash_salt_ext; + +extern void vm_kernel_addrhash_external( + vm_offset_t addr, + vm_offset_t *perm_addr); +#endif /* MACH_KERNEL_PRIVATE */ + +extern void vm_init_before_launchd(void); + #endif /* KERNEL */ #endif /* _VM_VM_KERN_H_ */ diff --git a/osfmk/vm/vm_map.c b/osfmk/vm/vm_map.c index 69150e2a7..dbdf22e41 100644 --- a/osfmk/vm/vm_map.c +++ b/osfmk/vm/vm_map.c @@ -2,7 +2,7 @@ * Copyright (c) 2000-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,34 +22,34 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ -/* +/* * Mach Operating System * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ @@ -110,6 +110,12 @@ #include #include +#include + +#if __arm64__ +extern int fourk_binary_compatibility_unsafe; +extern int fourk_binary_compatibility_allow_wx; +#endif /* __arm64__ */ extern int proc_selfpid(void); extern char *proc_name_address(void *p); @@ -120,6 +126,9 @@ int vm_map_debug_apple_protect = 0; int vm_map_debug_fourk = 0; #endif /* VM_MAP_DEBUG_FOURK */ +int vm_map_executable_immutable = 0; +int vm_map_executable_immutable_no_log = 0; + extern u_int32_t random(void); /* from */ /* Internal prototypes */ @@ -230,8 +239,9 @@ static kern_return_t vm_map_wire_nested( vm_map_offset_t start, vm_map_offset_t end, vm_prot_t caller_prot, + vm_tag_t tag, boolean_t user_wire, - pmap_t map_pmap, + pmap_t map_pmap, vm_map_offset_t pmap_addr, ppnum_t *physpage_p); @@ -274,6 +284,8 @@ static kern_return_t vm_map_remap_range_allocate( vm_map_size_t size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, vm_map_entry_t *map_entry); static void vm_map_region_look_for_page( @@ -318,6 +330,8 @@ static kern_return_t vm_map_pageout( vm_map_offset_t end); #endif /* MACH_ASSERT */ +pid_t find_largest_process_vm_map_entries(void); + /* * Macros to copy a vm_map_entry. We must be careful to correctly * manage the wired page count. vm_map_entry_copy() creates a new @@ -355,14 +369,14 @@ MACRO_END /* * Decide if we want to allow processes to execute from their data or stack areas. - * override_nx() returns true if we do. Data/stack execution can be enabled independently + * override_nx() returns true if we do. Data/stack execution can be enabled independently * for 32 and 64 bit processes. Set the VM_ABI_32 or VM_ABI_64 flags in allow_data_exec * or allow_stack_exec to enable data execution for that type of data area for that particular * ABI (or both by or'ing the flags together). These are initialized in the architecture - * specific pmap files since the default behavior varies according to architecture. The - * main reason it varies is because of the need to provide binary compatibility with old - * applications that were written before these restrictions came into being. In the old - * days, an app could execute anything it could read, but this has slowly been tightened + * specific pmap files since the default behavior varies according to architecture. The + * main reason it varies is because of the need to provide binary compatibility with old + * applications that were written before these restrictions came into being. In the old + * days, an app could execute anything it could read, but this has slowly been tightened * up over time. The default behavior is: * * 32-bit PPC apps may execute from both stack and data areas @@ -370,7 +384,7 @@ MACRO_END * 64-bit PPC/Intel apps may not execute from either data or stack * * An application on any architecture may override these defaults by explicitly - * adding PROT_EXEC permission to the page in question with the mprotect(2) + * adding PROT_EXEC permission to the page in question with the mprotect(2) * system call. This code here just determines what happens when an app tries to * execute from a page that lacks execute permission. * @@ -402,7 +416,7 @@ override_nx(vm_map_t map, uint32_t user_tag) /* map unused on arm */ current_abi = VM_ABI_32; /* - * Determine if we should allow the execution based on whether it's a + * Determine if we should allow the execution based on whether it's a * stack or data area and the current architecture. */ @@ -430,7 +444,7 @@ override_nx(vm_map_t map, uint32_t user_tag) /* map unused on arm */ * the asymmetric (delayed) strategy is used for shared temporary * objects instead of the symmetric (shadow) strategy. All maps * are now "top level" maps (either task map, kernel map or submap - * of the kernel map). + * of the kernel map). * * Since portions of maps are specified by start/end addreses, * which may not align with existing map entries, all @@ -456,12 +470,11 @@ override_nx(vm_map_t map, uint32_t user_tag) /* map unused on arm */ * vm_object_copy_strategically() in vm_object.c. */ -static zone_t vm_map_zone; /* zone for vm_map structures */ -static zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */ -zone_t vm_map_entry_reserved_zone; /* zone with reserve for non-blocking - * allocations */ -static zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */ -zone_t vm_map_holes_zone; /* zone for vm map holes (vm_map_links) structures */ +static zone_t vm_map_zone; /* zone for vm_map structures */ +zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */ +static zone_t vm_map_entry_reserved_zone; /* zone with reserve for non-blocking allocations */ +static zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */ +zone_t vm_map_holes_zone; /* zone for vm map holes (vm_map_links) structures */ /* @@ -479,7 +492,11 @@ static vm_size_t kentry_data_size; static void *map_holes_data; static vm_size_t map_holes_data_size; +#if CONFIG_EMBEDDED +#define NO_COALESCE_LIMIT 0 +#else #define NO_COALESCE_LIMIT ((1024 * 128) - 1) +#endif /* Skip acquiring locks if we're in the midst of a kernel core dump */ unsigned int not_in_kdp = 1; @@ -529,7 +546,7 @@ done: #if CONFIG_CODE_DECRYPTION /* * vm_map_apple_protected: - * This remaps the requested part of the object with an object backed by + * This remaps the requested part of the object with an object backed by * the decrypting pager. * crypt_info contains entry points and session data for the crypt module. * The crypt_info block will be copied by vm_map_apple_protected. The data structures @@ -553,6 +570,10 @@ vm_map_apple_protected( vm_map_offset_t start_aligned, end_aligned; vm_object_offset_t crypto_start, crypto_end; int vm_flags; + vm_map_kernel_flags_t vmk_flags; + + vm_flags = 0; + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; map_locked = FALSE; unprotected_mem_obj = MEMORY_OBJECT_NULL; @@ -562,8 +583,24 @@ vm_map_apple_protected( start_aligned = vm_map_trunc_page(start_aligned, VM_MAP_PAGE_MASK(map)); end_aligned = vm_map_round_page(end_aligned, VM_MAP_PAGE_MASK(map)); +#if __arm64__ + /* + * "start" and "end" might be 4K-aligned but not 16K-aligned, + * so we might have to loop and establish up to 3 mappings: + * + * + the first 16K-page, which might overlap with the previous + * 4K-aligned mapping, + * + the center, + * + the last 16K-page, which might overlap with the next + * 4K-aligned mapping. + * Each of these mapping might be backed by a vnode pager (if + * properly page-aligned) or a "fourk_pager", itself backed by a + * vnode pager (if 4K-aligned but not page-aligned). + */ +#else /* __arm64__ */ assert(start_aligned == start); assert(end_aligned == end); +#endif /* __arm64__ */ map_addr = start_aligned; for (map_addr = start_aligned; @@ -655,6 +692,20 @@ vm_map_apple_protected( } vm_flags = VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE; + /* can overwrite an immutable mapping */ + vmk_flags.vmkf_overwrite_immutable = TRUE; +#if __arm64__ + if (tmp_entry.used_for_jit && + (VM_MAP_PAGE_SHIFT(map) != FOURK_PAGE_SHIFT || + PAGE_SHIFT != FOURK_PAGE_SHIFT) && + fourk_binary_compatibility_unsafe && + fourk_binary_compatibility_allow_wx) { + printf("** FOURK_COMPAT [%d]: " + "allowing write+execute at 0x%llx\n", + proc_selfpid(), tmp_entry.vme_start); + vmk_flags.vmkf_map_jit = TRUE; + } +#endif /* __arm64__ */ /* map this memory object in place of the current one */ map_addr = tmp_entry.vme_start; @@ -664,14 +715,21 @@ vm_map_apple_protected( tmp_entry.vme_start), (mach_vm_offset_t) 0, vm_flags, + vmk_flags, + VM_KERN_MEMORY_NONE, (ipc_port_t) unprotected_mem_obj, 0, TRUE, tmp_entry.protection, tmp_entry.max_protection, tmp_entry.inheritance); - assert(kr == KERN_SUCCESS); - assert(map_addr == tmp_entry.vme_start); + assertf(kr == KERN_SUCCESS, + "kr = 0x%x\n", kr); + assertf(map_addr == tmp_entry.vme_start, + "map_addr=0x%llx vme_start=0x%llx tmp_entry=%p\n", + (uint64_t)map_addr, + (uint64_t) tmp_entry.vme_start, + &tmp_entry); #if VM_MAP_DEBUG_APPLE_PROTECT if (vm_map_debug_apple_protect) { @@ -691,7 +749,7 @@ vm_map_apple_protected( crypto_end); } #endif /* VM_MAP_DEBUG_APPLE_PROTECT */ - + /* * Release the reference obtained by * apple_protect_pager_setup(). @@ -770,6 +828,8 @@ vm_map_init( kentry_data_size * 64, kentry_data_size, "Reserved VM map entries"); zone_change(vm_map_entry_reserved_zone, Z_NOENCRYPT, TRUE); + /* Don't quarantine because we always need elements available */ + zone_change(vm_map_entry_reserved_zone, Z_KASAN_QUARANTINE, FALSE); vm_map_copy_zone = zinit((vm_map_size_t) sizeof(struct vm_map_copy), 16*1024, PAGE_SIZE, "VM map copies"); @@ -785,6 +845,7 @@ vm_map_init( */ zone_change(vm_map_zone, Z_COLLECT, FALSE); zone_change(vm_map_zone, Z_FOREIGN, TRUE); + zone_change(vm_map_zone, Z_GZALLOC_EXEMPT, TRUE); zone_change(vm_map_entry_reserved_zone, Z_COLLECT, FALSE); zone_change(vm_map_entry_reserved_zone, Z_EXPAND, FALSE); @@ -801,17 +862,29 @@ vm_map_init( zone_change(vm_map_holes_zone, Z_CALLERACCT, TRUE); zone_change(vm_map_holes_zone, Z_GZALLOC_EXEMPT, TRUE); - /* + /* * Add the stolen memory to zones, adjust zone size and stolen counts. + * zcram only up to the maximum number of pages for each zone chunk. */ zcram(vm_map_zone, (vm_offset_t)map_data, map_data_size); - zcram(vm_map_entry_reserved_zone, (vm_offset_t)kentry_data, kentry_data_size); - zcram(vm_map_holes_zone, (vm_offset_t)map_holes_data, map_holes_data_size); + + const vm_size_t stride = ZONE_CHUNK_MAXPAGES * PAGE_SIZE; + for (vm_offset_t off = 0; off < kentry_data_size; off += stride) { + zcram(vm_map_entry_reserved_zone, + (vm_offset_t)kentry_data + off, + MIN(kentry_data_size - off, stride)); + } + for (vm_offset_t off = 0; off < map_holes_data_size; off += stride) { + zcram(vm_map_holes_zone, + (vm_offset_t)map_holes_data + off, + MIN(map_holes_data_size - off, stride)); + } + VM_PAGE_MOVE_STOLEN(atop_64(map_data_size) + atop_64(kentry_data_size) + atop_64(map_holes_data_size)); lck_grp_attr_setdefault(&vm_map_lck_grp_attr); lck_grp_init(&vm_map_lck_grp, "vm_map", &vm_map_lck_grp_attr); - lck_attr_setdefault(&vm_map_lck_attr); + lck_attr_setdefault(&vm_map_lck_attr); lck_attr_setdefault(&vm_map_lck_rw_attr); lck_attr_cleardebug(&vm_map_lck_rw_attr); @@ -826,6 +899,12 @@ vm_map_init( &vm_map_debug_fourk, sizeof(vm_map_debug_fourk)); #endif /* VM_MAP_DEBUG_FOURK */ + PE_parse_boot_argn("vm_map_executable_immutable", + &vm_map_executable_immutable, + sizeof(vm_map_executable_immutable)); + PE_parse_boot_argn("vm_map_executable_immutable_no_log", + &vm_map_executable_immutable_no_log, + sizeof(vm_map_executable_immutable_no_log)); } void @@ -864,10 +943,17 @@ vm_map_steal_memory( map_holes_data = pmap_steal_memory(map_holes_data_size); } +boolean_t vm_map_supports_hole_optimization = FALSE; + void vm_kernel_reserved_entry_init(void) { zone_prio_refill_configure(vm_map_entry_reserved_zone, (6*PAGE_SIZE)/sizeof(struct vm_map_entry)); + + /* + * Once we have our replenish thread set up, we can start using the vm_map_holes zone. + */ zone_prio_refill_configure(vm_map_holes_zone, (6*PAGE_SIZE)/sizeof(struct vm_map_links)); + vm_map_supports_hole_optimization = TRUE; } void @@ -915,8 +1001,6 @@ vm_kernel_map_is_kernel(vm_map_t map) { * the given lower and upper address bounds. */ -boolean_t vm_map_supports_hole_optimization = TRUE; - vm_map_t vm_map_create( pmap_t pmap, @@ -938,12 +1022,15 @@ vm_map_create( result->hdr.entries_pageable = pageable; vm_map_store_init( &(result->hdr) ); - + result->hdr.page_shift = PAGE_SHIFT; result->size = 0; result->user_wire_limit = MACH_VM_MAX_ADDRESS; /* default limit is unlimited */ result->user_wire_size = 0; +#if __x86_64__ + result->vmmap_high_start = 0; +#endif /* __x86_64__ */ result->ref_count = 1; #if TASK_SWAPPER result->res_count = 1; @@ -966,11 +1053,15 @@ vm_map_create( result->color_rr = (color_seed++) & vm_color_mask; result->jit_entry_exists = FALSE; - if (vm_map_supports_hole_optimization && pmap != kernel_pmap) { + if (vm_map_supports_hole_optimization) { hole_entry = zalloc(vm_map_holes_zone); hole_entry->start = min; +#if defined(__arm__) || defined(__arm64__) + hole_entry->end = result->max_offset; +#else hole_entry->end = (max > (vm_map_offset_t)MACH_VM_MAX_ADDRESS) ? max : (vm_map_offset_t)MACH_VM_MAX_ADDRESS; +#endif result->holes_list = result->hole_hint = hole_entry; hole_entry->prev = hole_entry->next = (vm_map_entry_t) hole_entry; result->holelistenabled = TRUE; @@ -982,7 +1073,7 @@ vm_map_create( vm_map_lock_init(result); lck_mtx_init_ext(&result->s_lock, &result->s_lock_ext, &vm_map_lck_grp, &vm_map_lck_attr); - + return(result); } @@ -1081,8 +1172,8 @@ first_free_is_valid( { if (!first_free_check) return TRUE; - - return( first_free_is_valid_store( map )); + + return( first_free_is_valid_store( map )); } #endif /* MACH_ASSERT */ @@ -1171,24 +1262,28 @@ void vm_map_destroy( vm_map_t map, int flags) -{ +{ vm_map_lock(map); /* final cleanup: no need to unnest shared region */ flags |= VM_MAP_REMOVE_NO_UNNESTING; + /* final cleanup: ok to remove immutable mappings */ + flags |= VM_MAP_REMOVE_IMMUTABLE; /* clean up regular map entries */ (void) vm_map_delete(map, map->min_offset, map->max_offset, flags, VM_MAP_NULL); /* clean up leftover special mappings (commpage, etc...) */ +#if !defined(__arm__) && !defined(__arm64__) (void) vm_map_delete(map, 0x0, 0xFFFFFFFFFFFFF000ULL, flags, VM_MAP_NULL); +#endif /* !__arm__ && !__arm64__ */ vm_map_disable_hole_optimization(map); vm_map_unlock(map); assert(map->hdr.nentries == 0); - + if(map->pmap) pmap_destroy(map->pmap); @@ -1217,16 +1312,48 @@ vm_map_destroy( zfree(vm_map_zone, map); } +/* + * Returns pid of the task with the largest number of VM map entries. + * Used in the zone-map-exhaustion jetsam path. + */ +pid_t +find_largest_process_vm_map_entries(void) +{ + pid_t victim_pid = -1; + int max_vm_map_entries = 0; + task_t task = TASK_NULL; + queue_head_t *task_list = &tasks; + + lck_mtx_lock(&tasks_threads_lock); + queue_iterate(task_list, task, task_t, tasks) { + if (task == kernel_task || !task->active) + continue; + + vm_map_t task_map = task->map; + if (task_map != VM_MAP_NULL) { + int task_vm_map_entries = task_map->hdr.nentries; + if (task_vm_map_entries > max_vm_map_entries) { + max_vm_map_entries = task_vm_map_entries; + victim_pid = pid_from_task(task); + } + } + } + lck_mtx_unlock(&tasks_threads_lock); + + printf("zone_map_exhaustion: victim pid %d, vm region count: %d\n", victim_pid, max_vm_map_entries); + return victim_pid; +} + #if TASK_SWAPPER /* * vm_map_swapin/vm_map_swapout * - * Swap a map in and out, either referencing or releasing its resources. + * Swap a map in and out, either referencing or releasing its resources. * These functions are internal use only; however, they must be exported * because they may be called from macros, which are exported. * - * In the case of swapout, there could be races on the residence count, - * so if the residence count is up, we return, assuming that a + * In the case of swapout, there could be races on the residence count, + * so if the residence count is up, we return, assuming that a * vm_map_deallocate() call in the near future will bring us back. * * Locking: @@ -1242,7 +1369,7 @@ vm_map_destroy( * 2) A vm_map_reference() call at this time is illegal, and will * cause a panic. vm_map_reference() is only allowed on resident * maps, since it refuses to block. - * 3) A vm_map_swapin() call during a swapin will block, and + * 3) A vm_map_swapin() call during a swapin will block, and * proceeed when the first swapin is done, turning into a nop. * This is the reason the res_count is not incremented until * after the swapin is complete. @@ -1250,9 +1377,9 @@ vm_map_destroy( * the map lock is taken, during which a swapin may get the lock * before a swapout about to happen. If this happens, the swapin * will detect the state and increment the reference count, causing - * the swapout to be a nop, thereby delaying it until a later - * vm_map_deallocate. If the swapout gets the lock first, then - * the swapin will simply block until the swapout is done, and + * the swapout to be a nop, thereby delaying it until a later + * vm_map_deallocate. If the swapout gets the lock first, then + * the swapin will simply block until the swapout is done, and * then proceed. * * Because vm_map_swapin() is potentially an expensive operation, it @@ -1279,7 +1406,7 @@ void vm_map_swapin (vm_map_t map) * First deal with various races. */ if (map->sw_state == MAP_SW_IN) - /* + /* * we raced with swapout and won. Returning will incr. * the res_count, turning the swapout into a nop. */ @@ -1301,7 +1428,7 @@ void vm_map_swapin (vm_map_t map) assert(map->sw_state == MAP_SW_OUT); /* - * We now operate upon each map entry. If the entry is a sub- + * We now operate upon each map entry. If the entry is a sub- * or share-map, we call vm_map_res_reference upon it. * If the entry is an object, we call vm_object_res_reference * (this may iterate through the shadow chain). @@ -1338,7 +1465,7 @@ void vm_map_swapin (vm_map_t map) void vm_map_swapout(vm_map_t map) { vm_map_entry_t entry; - + /* * Map is locked * First deal with various races. @@ -1362,7 +1489,7 @@ void vm_map_swapout(vm_map_t map) return; /* - * We now operate upon each map entry. If the entry is a sub- + * We now operate upon each map entry. If the entry is a sub- * or share-map, we call vm_map_res_deallocate upon it. * If the entry is an object, we call vm_object_res_deallocate * (this may iterate through the shadow chain). @@ -1383,9 +1510,9 @@ void vm_map_swapout(vm_map_t map) vm_object_t object = VME_OBJECT(entry); vm_object_lock(object); /* - * This call may take a long time, - * since it could actively push - * out pages (if we implement it + * This call may take a long time, + * since it could actively push + * out pages (if we implement it * that way). */ vm_object_res_deallocate(object); @@ -1403,8 +1530,8 @@ void vm_map_swapout(vm_map_t map) /* * vm_map_lookup_entry: [ internal use only ] * - * Calls into the vm map store layer to find the map - * entry containing (or immediately preceding) the + * Calls into the vm map store layer to find the map + * entry containing (or immediately preceding) the * specified address in the given map; the entry is returned * in the "entry" parameter. The boolean * result indicates whether the address is @@ -1438,7 +1565,9 @@ vm_map_find_space( vm_map_offset_t *address, /* OUT */ vm_map_size_t size, vm_map_offset_t mask, - int flags, + int flags __unused, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, vm_map_entry_t *o_entry) /* OUT */ { vm_map_entry_t entry, new_entry; @@ -1451,7 +1580,7 @@ vm_map_find_space( return KERN_INVALID_ARGUMENT; } - if (flags & VM_FLAGS_GUARD_AFTER) { + if (vmk_flags.vmkf_guard_after) { /* account for the back guard page in the size */ size += VM_MAP_PAGE_SIZE(map); } @@ -1505,12 +1634,12 @@ vm_map_find_space( * wrap around the address. */ - if (flags & VM_FLAGS_GUARD_BEFORE) { + if (vmk_flags.vmkf_guard_before) { /* reserve space for the front guard page */ start += VM_MAP_PAGE_SIZE(map); } end = ((start + mask) & ~mask); - + if (end < start) { vm_map_entry_dispose(map, new_entry); vm_map_unlock(map); @@ -1584,7 +1713,7 @@ vm_map_find_space( * the map should be locked. */ - if (flags & VM_FLAGS_GUARD_BEFORE) { + if (vmk_flags.vmkf_guard_before) { /* go back for the front guard page */ start -= VM_MAP_PAGE_SIZE(map); } @@ -1631,14 +1760,12 @@ vm_map_find_space( new_entry->iokit_acct = FALSE; new_entry->vme_resilient_codesign = FALSE; new_entry->vme_resilient_media = FALSE; - if (flags & VM_FLAGS_ATOMIC_ENTRY) + if (vmk_flags.vmkf_atomic_entry) new_entry->vme_atomic = TRUE; else new_entry->vme_atomic = FALSE; - int alias; - VM_GET_FLAGS_ALIAS(flags, alias); - VME_ALIAS_SET(new_entry, alias); + VME_ALIAS_SET(new_entry, tag); /* * Insert the new entry into the list @@ -1669,7 +1796,7 @@ int vm_map_pmap_enter_enable = FALSE; * As soon as a page not found in the object the scan ends. * * Returns: - * Nothing. + * Nothing. * * In/out conditions: * The source map should not be locked on entry. @@ -1706,13 +1833,8 @@ vm_map_pmap_enter( vm_object_lock(object); m = vm_page_lookup(object, offset); - /* - * ENCRYPTED SWAP: - * The user should never see encrypted data, so do not - * enter an encrypted page in the page table. - */ - if (m == VM_PAGE_NULL || m->busy || m->encrypted || - m->fictitious || + + if (m == VM_PAGE_NULL || m->busy || m->fictitious || (m->unusual && ( m->error || m->restart || m->absent))) { vm_object_unlock(object); return; @@ -1725,11 +1847,15 @@ vm_map_pmap_enter( } type_of_fault = DBG_CACHE_HIT_FAULT; kr = vm_fault_enter(m, map->pmap, addr, protection, protection, - VM_PAGE_WIRED(m), FALSE, FALSE, FALSE, - 0, /* XXX need user tag / alias? */ - 0, /* alternate accounting? */ - NULL, - &type_of_fault); + VM_PAGE_WIRED(m), + FALSE, /* change_wiring */ + VM_KERN_MEMORY_NONE, /* tag - not wiring */ + FALSE, /* no_cache */ + FALSE, /* cs_bypass */ + 0, /* XXX need user tag / alias? */ + 0, /* pmap_options */ + NULL, /* need_retry */ + &type_of_fault); vm_object_unlock(object); @@ -1846,6 +1972,8 @@ vm_map_enter( vm_map_size_t size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t alias, vm_object_t object, vm_object_offset_t offset, boolean_t needs_copy, @@ -1864,25 +1992,27 @@ vm_map_enter( boolean_t map_locked = FALSE; boolean_t pmap_empty = TRUE; boolean_t new_mapping_established = FALSE; - boolean_t keep_map_locked = ((flags & VM_FLAGS_KEEP_MAP_LOCKED) != 0); + boolean_t keep_map_locked = vmk_flags.vmkf_keep_map_locked; boolean_t anywhere = ((flags & VM_FLAGS_ANYWHERE) != 0); boolean_t purgable = ((flags & VM_FLAGS_PURGABLE) != 0); boolean_t overwrite = ((flags & VM_FLAGS_OVERWRITE) != 0); boolean_t no_cache = ((flags & VM_FLAGS_NO_CACHE) != 0); - boolean_t is_submap = ((flags & VM_FLAGS_SUBMAP) != 0); - boolean_t permanent = ((flags & VM_FLAGS_PERMANENT) != 0); - boolean_t entry_for_jit = ((flags & VM_FLAGS_MAP_JIT) != 0); - boolean_t iokit_acct = ((flags & VM_FLAGS_IOKIT_ACCT) != 0); + boolean_t is_submap = vmk_flags.vmkf_submap; + boolean_t permanent = vmk_flags.vmkf_permanent; + boolean_t entry_for_jit = vmk_flags.vmkf_map_jit; + boolean_t iokit_acct = vmk_flags.vmkf_iokit_acct; boolean_t resilient_codesign = ((flags & VM_FLAGS_RESILIENT_CODESIGN) != 0); boolean_t resilient_media = ((flags & VM_FLAGS_RESILIENT_MEDIA) != 0); boolean_t random_address = ((flags & VM_FLAGS_RANDOM_ADDR) != 0); unsigned int superpage_size = ((flags & VM_FLAGS_SUPERPAGE_MASK) >> VM_FLAGS_SUPERPAGE_SHIFT); - vm_tag_t alias, user_alias; + vm_tag_t user_alias; vm_map_offset_t effective_min_offset, effective_max_offset; kern_return_t kr; boolean_t clear_map_aligned = FALSE; vm_map_entry_t hole_entry; + assertf(vmk_flags.__vmkf_unused == 0, "vmk_flags unused=0x%x\n", vmk_flags.__vmkf_unused); + if (superpage_size) { switch (superpage_size) { /* @@ -1909,6 +2039,16 @@ vm_map_enter( } +#if CONFIG_EMBEDDED + if (cur_protection & VM_PROT_WRITE){ + if ((cur_protection & VM_PROT_EXECUTE) && !entry_for_jit){ + printf("EMBEDDED: %s: curprot cannot be write+execute. " + "turning off execute\n", + __FUNCTION__); + cur_protection &= ~VM_PROT_EXECUTE; + } + } +#endif /* CONFIG_EMBEDDED */ if (resilient_codesign || resilient_media) { if ((cur_protection & (VM_PROT_WRITE | VM_PROT_EXECUTE)) || @@ -1927,7 +2067,7 @@ vm_map_enter( return KERN_INVALID_ARGUMENT; } } - if (flags & VM_FLAGS_ALREADY) { + if (vmk_flags.vmkf_already) { /* * VM_FLAGS_ALREADY says that it's OK if the same mapping * is already present. For it to be meaningul, the requested @@ -1943,13 +2083,15 @@ vm_map_enter( effective_min_offset = map->min_offset; - if (flags & VM_FLAGS_BEYOND_MAX) { + if (vmk_flags.vmkf_beyond_max) { /* * Allow an insertion beyond the map's max offset. */ +#if !defined(__arm__) && !defined(__arm64__) if (vm_map_is_64bit(map)) effective_max_offset = 0xFFFFFFFFFFFFF000ULL; else +#endif /* __arm__ */ effective_max_offset = 0x00000000FFFFF000ULL; } else { effective_max_offset = map->max_offset; @@ -1961,7 +2103,6 @@ vm_map_enter( return KERN_INVALID_ARGUMENT; } - VM_GET_FLAGS_ALIAS(flags, alias); if (map->pmap == kernel_pmap) { user_alias = VM_KERN_MEMORY_NONE; } else { @@ -1984,7 +2125,7 @@ vm_map_enter( */ clear_map_aligned = TRUE; } - if (!anywhere && + if (!anywhere && !VM_MAP_PAGE_ALIGNED(*address, VM_MAP_PAGE_MASK(map))) { /* * We've been asked to map at a fixed address and that @@ -2034,7 +2175,7 @@ StartAgain: ; if (anywhere) { vm_map_lock(map); map_locked = TRUE; - + if (entry_for_jit) { if (map->jit_entry_exists) { result = KERN_INVALID_ARGUMENT; @@ -2053,6 +2194,13 @@ StartAgain: ; } start = *address; } +#if __x86_64__ + else if ((start == 0 || start == vm_map_min(map)) && + !map->disable_vmentry_reuse && + map->vmmap_high_start != 0) { + start = map->vmmap_high_start; + } +#endif /* __x86_64__ */ /* @@ -2272,14 +2420,20 @@ StartAgain: ; } if (overwrite && zap_old_map != VM_MAP_NULL) { + int remove_flags; /* * Fixed mapping and "overwrite" flag: attempt to * remove all existing mappings in the specified * address range, saving them in our "zap_old_map". */ + remove_flags = VM_MAP_REMOVE_SAVE_ENTRIES; + remove_flags |= VM_MAP_REMOVE_NO_MAP_ALIGN; + if (vmk_flags.vmkf_overwrite_immutable) { + /* we can overwrite immutable mappings */ + remove_flags |= VM_MAP_REMOVE_IMMUTABLE; + } (void) vm_map_delete(map, start, end, - (VM_MAP_REMOVE_SAVE_ENTRIES | - VM_MAP_REMOVE_NO_MAP_ALIGN), + remove_flags, zap_old_map); } @@ -2288,7 +2442,7 @@ StartAgain: ; */ if (vm_map_lookup_entry(map, start, &entry)) { - if (! (flags & VM_FLAGS_ALREADY)) { + if (! (vmk_flags.vmkf_already)) { RETURN(KERN_NO_SPACE); } /* @@ -2299,7 +2453,7 @@ StartAgain: ; if (entry->vme_start < start) { tmp_start -= start - entry->vme_start; tmp_offset -= start - entry->vme_start; - + } for (; entry->vme_start < end; entry = entry->vme_next) { @@ -2438,6 +2592,9 @@ StartAgain: ; (VME_ALIAS(entry) == alias)) && (entry->no_cache == no_cache) && (entry->permanent == permanent) && + /* no coalescing for immutable executable mappings */ + !((entry->protection & VM_PROT_EXECUTE) && + entry->permanent) && (!entry->superpage_size && !superpage_size) && /* * No coalescing if not map-aligned, to avoid propagating @@ -2510,32 +2667,28 @@ StartAgain: ; if (object == VM_OBJECT_NULL && size > (vm_map_size_t)ANON_CHUNK_SIZE && max_protection != VM_PROT_NONE && - superpage_size == 0) + superpage_size == 0) tmp_end = tmp_start + (vm_map_size_t)ANON_CHUNK_SIZE; else tmp_end = tmp2_end; do { - new_entry = vm_map_entry_insert(map, entry, tmp_start, tmp_end, - object, offset, needs_copy, - FALSE, FALSE, - cur_protection, max_protection, - VM_BEHAVIOR_DEFAULT, - (entry_for_jit)? VM_INHERIT_NONE: inheritance, - 0, no_cache, - permanent, - superpage_size, - clear_map_aligned, - is_submap); + new_entry = vm_map_entry_insert( + map, entry, tmp_start, tmp_end, + object, offset, needs_copy, + FALSE, FALSE, + cur_protection, max_protection, + VM_BEHAVIOR_DEFAULT, + (entry_for_jit)? VM_INHERIT_NONE: inheritance, + 0, + no_cache, + permanent, + superpage_size, + clear_map_aligned, + is_submap, + entry_for_jit, + alias); assert((object != kernel_object) || (VM_KERN_MEMORY_NONE != alias)); - VME_ALIAS_SET(new_entry, alias); - - if (entry_for_jit){ - if (!(map->jit_entry_exists)){ - new_entry->used_for_jit = TRUE; - map->jit_entry_exists = TRUE; - } - } if (resilient_codesign && ! ((cur_protection | max_protection) & @@ -2605,6 +2758,11 @@ StartAgain: ; if (submap->pmap == NULL) { /* let's proceed without nesting... */ } +#if defined(__arm__) || defined(__arm64__) + else { + pmap_set_nested(submap->pmap); + } +#endif } if (use_pmap && submap->pmap != NULL) { kr = pmap_nest(map->pmap, @@ -2632,6 +2790,7 @@ StartAgain: ; if (superpage_size) { vm_page_t pages, m; vm_object_t sp_object; + vm_object_offset_t sp_offset; VME_OFFSET_SET(entry, 0); @@ -2654,18 +2813,20 @@ StartAgain: ; /* enter the base pages into the object */ vm_object_lock(sp_object); - for (offset = 0; offset < SUPERPAGE_SIZE; offset += PAGE_SIZE) { + for (sp_offset = 0; + sp_offset < SUPERPAGE_SIZE; + sp_offset += PAGE_SIZE) { m = pages; pmap_zero_page(VM_PAGE_GET_PHYS_PAGE(m)); pages = NEXT_PAGE(m); *(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL; - vm_page_insert_wired(m, sp_object, offset, VM_KERN_MEMORY_OSFMK); + vm_page_insert_wired(m, sp_object, sp_offset, VM_KERN_MEMORY_OSFMK); } vm_object_unlock(sp_object); } - } while (tmp_end != tmp2_end && + } while (tmp_end != tmp2_end && (tmp_start = tmp_end) && - (tmp_end = (tmp2_end - tmp_end > (vm_map_size_t)ANON_CHUNK_SIZE) ? + (tmp_end = (tmp2_end - tmp_end > (vm_map_size_t)ANON_CHUNK_SIZE) ? tmp_end + (vm_map_size_t)ANON_CHUNK_SIZE : tmp2_end)); } @@ -2680,7 +2841,7 @@ BailOut: #if DEBUG if (pmap_empty && - !(flags & VM_FLAGS_NO_PMAP_CHECK)) { + !(vmk_flags.vmkf_no_pmap_check)) { assert(vm_map_pmap_is_empty(map, *address, *address+size)); @@ -2746,8 +2907,8 @@ BailOut: if ((map->wiring_required)||(superpage_size)) { assert(!keep_map_locked); pmap_empty = FALSE; /* pmap won't be empty */ - kr = vm_map_wire(map, start, end, - new_entry->protection | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK), + kr = vm_map_wire_kernel(map, start, end, + new_entry->protection, VM_KERN_MEMORY_MLOCK, TRUE); result = kr; } @@ -2861,118 +3022,741 @@ BailOut: #undef RETURN } - -/* - * Counters for the prefault optimization. - */ -int64_t vm_prefault_nb_pages = 0; -int64_t vm_prefault_nb_bailout = 0; - -static kern_return_t -vm_map_enter_mem_object_helper( - vm_map_t target_map, - vm_map_offset_t *address, - vm_map_size_t initial_size, +#if __arm64__ +extern const struct memory_object_pager_ops fourk_pager_ops; +kern_return_t +vm_map_enter_fourk( + vm_map_t map, + vm_map_offset_t *address, /* IN/OUT */ + vm_map_size_t size, vm_map_offset_t mask, int flags, - ipc_port_t port, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t alias, + vm_object_t object, vm_object_offset_t offset, - boolean_t copy, + boolean_t needs_copy, vm_prot_t cur_protection, vm_prot_t max_protection, - vm_inherit_t inheritance, - upl_page_list_ptr_t page_list, - unsigned int page_list_count) + vm_inherit_t inheritance) { - vm_map_address_t map_addr; - vm_map_size_t map_size; - vm_object_t object; - vm_object_size_t size; - kern_return_t result; - boolean_t mask_cur_protection, mask_max_protection; - boolean_t try_prefault = (page_list_count != 0); - vm_map_offset_t offset_in_mapping = 0; + vm_map_entry_t entry, new_entry; + vm_map_offset_t start, fourk_start; + vm_map_offset_t end, fourk_end; + vm_map_size_t fourk_size; + kern_return_t result = KERN_SUCCESS; + vm_map_t zap_old_map = VM_MAP_NULL; + vm_map_t zap_new_map = VM_MAP_NULL; + boolean_t map_locked = FALSE; + boolean_t pmap_empty = TRUE; + boolean_t new_mapping_established = FALSE; + boolean_t keep_map_locked = vmk_flags.vmkf_keep_map_locked; + boolean_t anywhere = ((flags & VM_FLAGS_ANYWHERE) != 0); + boolean_t purgable = ((flags & VM_FLAGS_PURGABLE) != 0); + boolean_t overwrite = ((flags & VM_FLAGS_OVERWRITE) != 0); + boolean_t no_cache = ((flags & VM_FLAGS_NO_CACHE) != 0); + boolean_t is_submap = vmk_flags.vmkf_submap; + boolean_t permanent = vmk_flags.vmkf_permanent; + boolean_t entry_for_jit = vmk_flags.vmkf_map_jit; +// boolean_t iokit_acct = vmk_flags.vmkf_iokit_acct; + unsigned int superpage_size = ((flags & VM_FLAGS_SUPERPAGE_MASK) >> VM_FLAGS_SUPERPAGE_SHIFT); + vm_map_offset_t effective_min_offset, effective_max_offset; + kern_return_t kr; + boolean_t clear_map_aligned = FALSE; + memory_object_t fourk_mem_obj; + vm_object_t fourk_object; + vm_map_offset_t fourk_pager_offset; + int fourk_pager_index_start, fourk_pager_index_num; + int cur_idx; + boolean_t fourk_copy; + vm_object_t copy_object; + vm_object_offset_t copy_offset; + + fourk_mem_obj = MEMORY_OBJECT_NULL; + fourk_object = VM_OBJECT_NULL; - mask_cur_protection = cur_protection & VM_PROT_IS_MASK; - mask_max_protection = max_protection & VM_PROT_IS_MASK; - cur_protection &= ~VM_PROT_IS_MASK; - max_protection &= ~VM_PROT_IS_MASK; + if (superpage_size) { + return KERN_NOT_SUPPORTED; + } - /* - * Check arguments for validity - */ - if ((target_map == VM_MAP_NULL) || - (cur_protection & ~VM_PROT_ALL) || - (max_protection & ~VM_PROT_ALL) || - (inheritance > VM_INHERIT_LAST_VALID) || - (try_prefault && (copy || !page_list)) || - initial_size == 0) { +#if CONFIG_EMBEDDED + if (cur_protection & VM_PROT_WRITE) { + if ((cur_protection & VM_PROT_EXECUTE) && + !entry_for_jit) { + printf("EMBEDDED: %s: curprot cannot be write+execute. " + "turning off execute\n", + __FUNCTION__); + cur_protection &= ~VM_PROT_EXECUTE; + } + } +#endif /* CONFIG_EMBEDDED */ + + if (is_submap) { + return KERN_NOT_SUPPORTED; + } + if (vmk_flags.vmkf_already) { + return KERN_NOT_SUPPORTED; + } + if (purgable || entry_for_jit) { + return KERN_NOT_SUPPORTED; + } + + effective_min_offset = map->min_offset; + + if (vmk_flags.vmkf_beyond_max) { + return KERN_NOT_SUPPORTED; + } else { + effective_max_offset = map->max_offset; + } + + if (size == 0 || + (offset & FOURK_PAGE_MASK) != 0) { + *address = 0; return KERN_INVALID_ARGUMENT; } - - { - map_addr = vm_map_trunc_page(*address, - VM_MAP_PAGE_MASK(target_map)); - map_size = vm_map_round_page(initial_size, - VM_MAP_PAGE_MASK(target_map)); + +#define RETURN(value) { result = value; goto BailOut; } + + assert(VM_MAP_PAGE_ALIGNED(*address, FOURK_PAGE_MASK)); + assert(VM_MAP_PAGE_ALIGNED(size, FOURK_PAGE_MASK)); + + if (!anywhere && overwrite) { + return KERN_NOT_SUPPORTED; + } + if (!anywhere && overwrite) { + /* + * Create a temporary VM map to hold the old mappings in the + * affected area while we create the new one. + * This avoids releasing the VM map lock in + * vm_map_entry_delete() and allows atomicity + * when we want to replace some mappings with a new one. + * It also allows us to restore the old VM mappings if the + * new mapping fails. + */ + zap_old_map = vm_map_create(PMAP_NULL, + *address, + *address + size, + map->hdr.entries_pageable); + vm_map_set_page_shift(zap_old_map, VM_MAP_PAGE_SHIFT(map)); + vm_map_disable_hole_optimization(zap_old_map); } - size = vm_object_round_page(initial_size); - /* - * Find the vm object (if any) corresponding to this port. - */ - if (!IP_VALID(port)) { - object = VM_OBJECT_NULL; - offset = 0; - copy = FALSE; - } else if (ip_kotype(port) == IKOT_NAMED_ENTRY) { - vm_named_entry_t named_entry; + fourk_start = *address; + fourk_size = size; + fourk_end = fourk_start + fourk_size; - named_entry = (vm_named_entry_t) port->ip_kobject; + start = vm_map_trunc_page(*address, VM_MAP_PAGE_MASK(map)); + end = vm_map_round_page(fourk_end, VM_MAP_PAGE_MASK(map)); + size = end - start; - if (flags & (VM_FLAGS_RETURN_DATA_ADDR | - VM_FLAGS_RETURN_4K_DATA_ADDR)) { - offset += named_entry->data_offset; - } - - /* a few checks to make sure user is obeying rules */ - if (size == 0) { - if (offset >= named_entry->size) - return KERN_INVALID_RIGHT; - size = named_entry->size - offset; - } - if (mask_max_protection) { - max_protection &= named_entry->protection; - } - if (mask_cur_protection) { - cur_protection &= named_entry->protection; + if (anywhere) { + return KERN_NOT_SUPPORTED; + } else { + /* + * Verify that: + * the address doesn't itself violate + * the mask requirement. + */ + + vm_map_lock(map); + map_locked = TRUE; + if ((start & mask) != 0) { + RETURN(KERN_NO_SPACE); } - if ((named_entry->protection & max_protection) != - max_protection) - return KERN_INVALID_RIGHT; - if ((named_entry->protection & cur_protection) != - cur_protection) - return KERN_INVALID_RIGHT; - if (offset + size < offset) { - /* overflow */ - return KERN_INVALID_ARGUMENT; + + /* + * ... the address is within bounds + */ + + end = start + size; + + if ((start < effective_min_offset) || + (end > effective_max_offset) || + (start >= end)) { + RETURN(KERN_INVALID_ADDRESS); } - if (named_entry->size < (offset + initial_size)) { - return KERN_INVALID_ARGUMENT; + + if (overwrite && zap_old_map != VM_MAP_NULL) { + /* + * Fixed mapping and "overwrite" flag: attempt to + * remove all existing mappings in the specified + * address range, saving them in our "zap_old_map". + */ + (void) vm_map_delete(map, start, end, + (VM_MAP_REMOVE_SAVE_ENTRIES | + VM_MAP_REMOVE_NO_MAP_ALIGN), + zap_old_map); } - if (named_entry->is_copy) { - /* for a vm_map_copy, we can only map it whole */ - if ((size != named_entry->size) && - (vm_map_round_page(size, - VM_MAP_PAGE_MASK(target_map)) == - named_entry->size)) { - /* XXX FBDP use the rounded size... */ + /* + * ... the starting address isn't allocated + */ + if (vm_map_lookup_entry(map, start, &entry)) { + vm_object_t cur_object, shadow_object; + + /* + * We might already some 4K mappings + * in a 16K page here. + */ + + if (entry->vme_end - entry->vme_start + != SIXTEENK_PAGE_SIZE) { + RETURN(KERN_NO_SPACE); + } + if (entry->is_sub_map) { + RETURN(KERN_NO_SPACE); + } + if (VME_OBJECT(entry) == VM_OBJECT_NULL) { + RETURN(KERN_NO_SPACE); + } + + /* go all the way down the shadow chain */ + cur_object = VME_OBJECT(entry); + vm_object_lock(cur_object); + while (cur_object->shadow != VM_OBJECT_NULL) { + shadow_object = cur_object->shadow; + vm_object_lock(shadow_object); + vm_object_unlock(cur_object); + cur_object = shadow_object; + shadow_object = VM_OBJECT_NULL; + } + if (cur_object->internal || + cur_object->pager == NULL) { + vm_object_unlock(cur_object); + RETURN(KERN_NO_SPACE); + } + if (cur_object->pager->mo_pager_ops + != &fourk_pager_ops) { + vm_object_unlock(cur_object); + RETURN(KERN_NO_SPACE); + } + fourk_object = cur_object; + fourk_mem_obj = fourk_object->pager; + + /* keep the "4K" object alive */ + vm_object_reference_locked(fourk_object); + vm_object_unlock(fourk_object); + + /* merge permissions */ + entry->protection |= cur_protection; + entry->max_protection |= max_protection; + if ((entry->protection & (VM_PROT_WRITE | + VM_PROT_EXECUTE)) == + (VM_PROT_WRITE | VM_PROT_EXECUTE) && + fourk_binary_compatibility_unsafe && + fourk_binary_compatibility_allow_wx) { + /* write+execute: need to be "jit" */ + entry->used_for_jit = TRUE; + } + + goto map_in_fourk_pager; + } + + /* + * ... the next region doesn't overlap the + * end point. + */ + + if ((entry->vme_next != vm_map_to_entry(map)) && + (entry->vme_next->vme_start < end)) { + RETURN(KERN_NO_SPACE); + } + } + + /* + * At this point, + * "start" and "end" should define the endpoints of the + * available new range, and + * "entry" should refer to the region before the new + * range, and + * + * the map should be locked. + */ + + /* create a new "4K" pager */ + fourk_mem_obj = fourk_pager_create(); + fourk_object = fourk_pager_to_vm_object(fourk_mem_obj); + assert(fourk_object); + + /* keep the "4" object alive */ + vm_object_reference(fourk_object); + + /* create a "copy" object, to map the "4K" object copy-on-write */ + fourk_copy = TRUE; + result = vm_object_copy_strategically(fourk_object, + 0, + end - start, + ©_object, + ©_offset, + &fourk_copy); + assert(result == KERN_SUCCESS); + assert(copy_object != VM_OBJECT_NULL); + assert(copy_offset == 0); + + /* take a reference on the copy object, for this mapping */ + vm_object_reference(copy_object); + + /* map the "4K" pager's copy object */ + new_entry = + vm_map_entry_insert(map, entry, + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(end, + VM_MAP_PAGE_MASK(map)), + copy_object, + 0, /* offset */ + FALSE, /* needs_copy */ + FALSE, FALSE, + cur_protection, max_protection, + VM_BEHAVIOR_DEFAULT, + ((entry_for_jit) + ? VM_INHERIT_NONE + : inheritance), + 0, + no_cache, + permanent, + superpage_size, + clear_map_aligned, + is_submap, + FALSE, /* jit */ + alias); + entry = new_entry; + +#if VM_MAP_DEBUG_FOURK + if (vm_map_debug_fourk) { + printf("FOURK_PAGER: map %p [0x%llx:0x%llx] new pager %p\n", + map, + (uint64_t) entry->vme_start, + (uint64_t) entry->vme_end, + fourk_mem_obj); + } +#endif /* VM_MAP_DEBUG_FOURK */ + + new_mapping_established = TRUE; + +map_in_fourk_pager: + /* "map" the original "object" where it belongs in the "4K" pager */ + fourk_pager_offset = (fourk_start & SIXTEENK_PAGE_MASK); + fourk_pager_index_start = (int) (fourk_pager_offset / FOURK_PAGE_SIZE); + if (fourk_size > SIXTEENK_PAGE_SIZE) { + fourk_pager_index_num = 4; + } else { + fourk_pager_index_num = (int) (fourk_size / FOURK_PAGE_SIZE); + } + if (fourk_pager_index_start + fourk_pager_index_num > 4) { + fourk_pager_index_num = 4 - fourk_pager_index_start; + } + for (cur_idx = 0; + cur_idx < fourk_pager_index_num; + cur_idx++) { + vm_object_t old_object; + vm_object_offset_t old_offset; + + kr = fourk_pager_populate(fourk_mem_obj, + TRUE, /* overwrite */ + fourk_pager_index_start + cur_idx, + object, + (object + ? (offset + + (cur_idx * FOURK_PAGE_SIZE)) + : 0), + &old_object, + &old_offset); +#if VM_MAP_DEBUG_FOURK + if (vm_map_debug_fourk) { + if (old_object == (vm_object_t) -1 && + old_offset == (vm_object_offset_t) -1) { + printf("FOURK_PAGER: map %p [0x%llx:0x%llx] " + "pager [%p:0x%llx] " + "populate[%d] " + "[object:%p,offset:0x%llx]\n", + map, + (uint64_t) entry->vme_start, + (uint64_t) entry->vme_end, + fourk_mem_obj, + VME_OFFSET(entry), + fourk_pager_index_start + cur_idx, + object, + (object + ? (offset + (cur_idx * FOURK_PAGE_SIZE)) + : 0)); + } else { + printf("FOURK_PAGER: map %p [0x%llx:0x%llx] " + "pager [%p:0x%llx] " + "populate[%d] [object:%p,offset:0x%llx] " + "old [%p:0x%llx]\n", + map, + (uint64_t) entry->vme_start, + (uint64_t) entry->vme_end, + fourk_mem_obj, + VME_OFFSET(entry), + fourk_pager_index_start + cur_idx, + object, + (object + ? (offset + (cur_idx * FOURK_PAGE_SIZE)) + : 0), + old_object, + old_offset); + } + } +#endif /* VM_MAP_DEBUG_FOURK */ + + assert(kr == KERN_SUCCESS); + if (object != old_object && + object != VM_OBJECT_NULL && + object != (vm_object_t) -1) { + vm_object_reference(object); + } + if (object != old_object && + old_object != VM_OBJECT_NULL && + old_object != (vm_object_t) -1) { + vm_object_deallocate(old_object); + } + } + +BailOut: + assert(map_locked == TRUE); + + if (fourk_object != VM_OBJECT_NULL) { + vm_object_deallocate(fourk_object); + fourk_object = VM_OBJECT_NULL; + fourk_mem_obj = MEMORY_OBJECT_NULL; + } + + if (result == KERN_SUCCESS) { + vm_prot_t pager_prot; + memory_object_t pager; + +#if DEBUG + if (pmap_empty && + !(vmk_flags.vmkf_no_pmap_check)) { + assert(vm_map_pmap_is_empty(map, + *address, + *address+size)); + } +#endif /* DEBUG */ + + /* + * For "named" VM objects, let the pager know that the + * memory object is being mapped. Some pagers need to keep + * track of this, to know when they can reclaim the memory + * object, for example. + * VM calls memory_object_map() for each mapping (specifying + * the protection of each mapping) and calls + * memory_object_last_unmap() when all the mappings are gone. + */ + pager_prot = max_protection; + if (needs_copy) { + /* + * Copy-On-Write mapping: won't modify + * the memory object. + */ + pager_prot &= ~VM_PROT_WRITE; + } + if (!is_submap && + object != VM_OBJECT_NULL && + object->named && + object->pager != MEMORY_OBJECT_NULL) { + vm_object_lock(object); + pager = object->pager; + if (object->named && + pager != MEMORY_OBJECT_NULL) { + assert(object->pager_ready); + vm_object_mapping_wait(object, THREAD_UNINT); + vm_object_mapping_begin(object); + vm_object_unlock(object); + + kr = memory_object_map(pager, pager_prot); + assert(kr == KERN_SUCCESS); + + vm_object_lock(object); + vm_object_mapping_end(object); + } + vm_object_unlock(object); + } + if (!is_submap && + fourk_object != VM_OBJECT_NULL && + fourk_object->named && + fourk_object->pager != MEMORY_OBJECT_NULL) { + vm_object_lock(fourk_object); + pager = fourk_object->pager; + if (fourk_object->named && + pager != MEMORY_OBJECT_NULL) { + assert(fourk_object->pager_ready); + vm_object_mapping_wait(fourk_object, + THREAD_UNINT); + vm_object_mapping_begin(fourk_object); + vm_object_unlock(fourk_object); + + kr = memory_object_map(pager, VM_PROT_READ); + assert(kr == KERN_SUCCESS); + + vm_object_lock(fourk_object); + vm_object_mapping_end(fourk_object); + } + vm_object_unlock(fourk_object); + } + } + + assert(map_locked == TRUE); + + if (!keep_map_locked) { + vm_map_unlock(map); + map_locked = FALSE; + } + + /* + * We can't hold the map lock if we enter this block. + */ + + if (result == KERN_SUCCESS) { + + /* Wire down the new entry if the user + * requested all new map entries be wired. + */ + if ((map->wiring_required)||(superpage_size)) { + assert(!keep_map_locked); + pmap_empty = FALSE; /* pmap won't be empty */ + kr = vm_map_wire_kernel(map, start, end, + new_entry->protection, VM_KERN_MEMORY_MLOCK, + TRUE); + result = kr; + } + + } + + if (result != KERN_SUCCESS) { + if (new_mapping_established) { + /* + * We have to get rid of the new mappings since we + * won't make them available to the user. + * Try and do that atomically, to minimize the risk + * that someone else create new mappings that range. + */ + zap_new_map = vm_map_create(PMAP_NULL, + *address, + *address + size, + map->hdr.entries_pageable); + vm_map_set_page_shift(zap_new_map, + VM_MAP_PAGE_SHIFT(map)); + vm_map_disable_hole_optimization(zap_new_map); + + if (!map_locked) { + vm_map_lock(map); + map_locked = TRUE; + } + (void) vm_map_delete(map, *address, *address+size, + (VM_MAP_REMOVE_SAVE_ENTRIES | + VM_MAP_REMOVE_NO_MAP_ALIGN), + zap_new_map); + } + if (zap_old_map != VM_MAP_NULL && + zap_old_map->hdr.nentries != 0) { + vm_map_entry_t entry1, entry2; + + /* + * The new mapping failed. Attempt to restore + * the old mappings, saved in the "zap_old_map". + */ + if (!map_locked) { + vm_map_lock(map); + map_locked = TRUE; + } + + /* first check if the coast is still clear */ + start = vm_map_first_entry(zap_old_map)->vme_start; + end = vm_map_last_entry(zap_old_map)->vme_end; + if (vm_map_lookup_entry(map, start, &entry1) || + vm_map_lookup_entry(map, end, &entry2) || + entry1 != entry2) { + /* + * Part of that range has already been + * re-mapped: we can't restore the old + * mappings... + */ + vm_map_enter_restore_failures++; + } else { + /* + * Transfer the saved map entries from + * "zap_old_map" to the original "map", + * inserting them all after "entry1". + */ + for (entry2 = vm_map_first_entry(zap_old_map); + entry2 != vm_map_to_entry(zap_old_map); + entry2 = vm_map_first_entry(zap_old_map)) { + vm_map_size_t entry_size; + + entry_size = (entry2->vme_end - + entry2->vme_start); + vm_map_store_entry_unlink(zap_old_map, + entry2); + zap_old_map->size -= entry_size; + vm_map_store_entry_link(map, entry1, entry2); + map->size += entry_size; + entry1 = entry2; + } + if (map->wiring_required) { + /* + * XXX TODO: we should rewire the + * old pages here... + */ + } + vm_map_enter_restore_successes++; + } + } + } + + /* + * The caller is responsible for releasing the lock if it requested to + * keep the map locked. + */ + if (map_locked && !keep_map_locked) { + vm_map_unlock(map); + } + + /* + * Get rid of the "zap_maps" and all the map entries that + * they may still contain. + */ + if (zap_old_map != VM_MAP_NULL) { + vm_map_destroy(zap_old_map, VM_MAP_REMOVE_NO_PMAP_CLEANUP); + zap_old_map = VM_MAP_NULL; + } + if (zap_new_map != VM_MAP_NULL) { + vm_map_destroy(zap_new_map, VM_MAP_REMOVE_NO_PMAP_CLEANUP); + zap_new_map = VM_MAP_NULL; + } + + return result; + +#undef RETURN +} +#endif /* __arm64__ */ + +/* + * Counters for the prefault optimization. + */ +int64_t vm_prefault_nb_pages = 0; +int64_t vm_prefault_nb_bailout = 0; + +static kern_return_t +vm_map_enter_mem_object_helper( + vm_map_t target_map, + vm_map_offset_t *address, + vm_map_size_t initial_size, + vm_map_offset_t mask, + int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance, + upl_page_list_ptr_t page_list, + unsigned int page_list_count) +{ + vm_map_address_t map_addr; + vm_map_size_t map_size; + vm_object_t object; + vm_object_size_t size; + kern_return_t result; + boolean_t mask_cur_protection, mask_max_protection; + boolean_t kernel_prefault, try_prefault = (page_list_count != 0); + vm_map_offset_t offset_in_mapping = 0; +#if __arm64__ + boolean_t fourk = vmk_flags.vmkf_fourk; +#endif /* __arm64__ */ + + assertf(vmk_flags.__vmkf_unused == 0, "vmk_flags unused=0x%x\n", vmk_flags.__vmkf_unused); + + mask_cur_protection = cur_protection & VM_PROT_IS_MASK; + mask_max_protection = max_protection & VM_PROT_IS_MASK; + cur_protection &= ~VM_PROT_IS_MASK; + max_protection &= ~VM_PROT_IS_MASK; + + /* + * Check arguments for validity + */ + if ((target_map == VM_MAP_NULL) || + (cur_protection & ~VM_PROT_ALL) || + (max_protection & ~VM_PROT_ALL) || + (inheritance > VM_INHERIT_LAST_VALID) || + (try_prefault && (copy || !page_list)) || + initial_size == 0) { + return KERN_INVALID_ARGUMENT; + } + +#if __arm64__ + if (fourk) { + map_addr = vm_map_trunc_page(*address, FOURK_PAGE_MASK); + map_size = vm_map_round_page(initial_size, FOURK_PAGE_MASK); + } else +#endif /* __arm64__ */ + { + map_addr = vm_map_trunc_page(*address, + VM_MAP_PAGE_MASK(target_map)); + map_size = vm_map_round_page(initial_size, + VM_MAP_PAGE_MASK(target_map)); + } + size = vm_object_round_page(initial_size); + + /* + * Find the vm object (if any) corresponding to this port. + */ + if (!IP_VALID(port)) { + object = VM_OBJECT_NULL; + offset = 0; + copy = FALSE; + } else if (ip_kotype(port) == IKOT_NAMED_ENTRY) { + vm_named_entry_t named_entry; + + named_entry = (vm_named_entry_t) port->ip_kobject; + + if (flags & (VM_FLAGS_RETURN_DATA_ADDR | + VM_FLAGS_RETURN_4K_DATA_ADDR)) { + offset += named_entry->data_offset; + } + + /* a few checks to make sure user is obeying rules */ + if (size == 0) { + if (offset >= named_entry->size) + return KERN_INVALID_RIGHT; + size = named_entry->size - offset; + } + if (mask_max_protection) { + max_protection &= named_entry->protection; + } + if (mask_cur_protection) { + cur_protection &= named_entry->protection; + } + if ((named_entry->protection & max_protection) != + max_protection) + return KERN_INVALID_RIGHT; + if ((named_entry->protection & cur_protection) != + cur_protection) + return KERN_INVALID_RIGHT; + if (offset + size < offset) { + /* overflow */ + return KERN_INVALID_ARGUMENT; + } + if (named_entry->size < (offset + initial_size)) { + return KERN_INVALID_ARGUMENT; + } + + if (named_entry->is_copy) { + /* for a vm_map_copy, we can only map it whole */ + if ((size != named_entry->size) && + (vm_map_round_page(size, + VM_MAP_PAGE_MASK(target_map)) == + named_entry->size)) { + /* XXX FBDP use the rounded size... */ size = vm_map_round_page( size, VM_MAP_PAGE_MASK(target_map)); } - + if (!(flags & VM_FLAGS_ANYWHERE) && (offset != 0 || size != named_entry->size)) { @@ -2989,7 +3773,7 @@ vm_map_enter_mem_object_helper( /* the callers parameter offset is defined to be the */ /* offset from beginning of named entry offset in object */ offset = offset + named_entry->offset; - + if (! VM_MAP_PAGE_ALIGNED(size, VM_MAP_PAGE_MASK(target_map))) { /* @@ -3015,11 +3799,15 @@ vm_map_enter_mem_object_helper( vm_map_unlock(submap); named_entry_unlock(named_entry); + vmk_flags.vmkf_submap = TRUE; + result = vm_map_enter(target_map, &map_addr, map_size, mask, - flags | VM_FLAGS_SUBMAP, + flags, + vmk_flags, + tag, (vm_object_t) submap, offset, copy, @@ -3043,7 +3831,7 @@ vm_map_enter_mem_object_helper( * This submap is being mapped in a map * that uses a different pmap. * Set its "mapped_in_other_pmaps" flag - * to indicate that we now need to + * to indicate that we now need to * remove mappings from all pmaps rather * than just the submap's pmap. */ @@ -3055,88 +3843,6 @@ vm_map_enter_mem_object_helper( } return result; - } else if (named_entry->is_pager) { - unsigned int access; - vm_prot_t protections; - unsigned int wimg_mode; - - protections = named_entry->protection & VM_PROT_ALL; - access = GET_MAP_MEM(named_entry->protection); - - if (flags & (VM_FLAGS_RETURN_DATA_ADDR| - VM_FLAGS_RETURN_4K_DATA_ADDR)) { - panic("VM_FLAGS_RETURN_DATA_ADDR not expected for submap."); - } - - object = vm_object_enter(named_entry->backing.pager, - named_entry->size, - named_entry->internal, - FALSE, - FALSE); - if (object == VM_OBJECT_NULL) { - named_entry_unlock(named_entry); - return KERN_INVALID_OBJECT; - } - - /* JMM - drop reference on pager here */ - - /* create an extra ref for the named entry */ - vm_object_lock(object); - vm_object_reference_locked(object); - named_entry->backing.object = object; - named_entry->is_pager = FALSE; - named_entry_unlock(named_entry); - - wimg_mode = object->wimg_bits; - - if (access == MAP_MEM_IO) { - wimg_mode = VM_WIMG_IO; - } else if (access == MAP_MEM_COPYBACK) { - wimg_mode = VM_WIMG_USE_DEFAULT; - } else if (access == MAP_MEM_INNERWBACK) { - wimg_mode = VM_WIMG_INNERWBACK; - } else if (access == MAP_MEM_WTHRU) { - wimg_mode = VM_WIMG_WTHRU; - } else if (access == MAP_MEM_WCOMB) { - wimg_mode = VM_WIMG_WCOMB; - } - - /* wait for object (if any) to be ready */ - if (!named_entry->internal) { - while (!object->pager_ready) { - vm_object_wait( - object, - VM_OBJECT_EVENT_PAGER_READY, - THREAD_UNINT); - vm_object_lock(object); - } - } - - if (object->wimg_bits != wimg_mode) - vm_object_change_wimg_mode(object, wimg_mode); - -#if VM_OBJECT_TRACKING_OP_TRUESHARE - if (!object->true_share && - vm_object_tracking_inited) { - void *bt[VM_OBJECT_TRACKING_BTDEPTH]; - int num = 0; - - num = OSBacktrace(bt, - VM_OBJECT_TRACKING_BTDEPTH); - btlog_add_entry(vm_object_tracking_btlog, - object, - VM_OBJECT_TRACKING_OP_TRUESHARE, - bt, - num); - } -#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */ - - object->true_share = TRUE; - - if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) - object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; - vm_object_unlock(object); - } else if (named_entry->is_copy) { kern_return_t kr; vm_map_copy_t copy_map; @@ -3146,7 +3852,6 @@ vm_map_enter_mem_object_helper( if (flags & ~(VM_FLAGS_FIXED | VM_FLAGS_ANYWHERE | VM_FLAGS_OVERWRITE | - VM_FLAGS_IOKIT_ACCT | VM_FLAGS_RETURN_4K_DATA_ADDR | VM_FLAGS_RETURN_DATA_ADDR | VM_FLAGS_ALIAS_MASK)) { @@ -3183,10 +3888,10 @@ vm_map_enter_mem_object_helper( mask, flags & (VM_FLAGS_ANYWHERE | VM_FLAGS_OVERWRITE | - VM_FLAGS_IOKIT_ACCT | VM_FLAGS_RETURN_4K_DATA_ADDR | - VM_FLAGS_RETURN_DATA_ADDR | - VM_FLAGS_ALIAS_MASK), + VM_FLAGS_RETURN_DATA_ADDR), + vmk_flags, + tag, VM_OBJECT_NULL, 0, FALSE, /* copy */ @@ -3203,13 +3908,17 @@ vm_map_enter_mem_object_helper( for (copy_entry = vm_map_copy_first_entry(copy_map); copy_entry != vm_map_copy_to_entry(copy_map); copy_entry = copy_entry->vme_next) { - int remap_flags = 0; + int remap_flags; + vm_map_kernel_flags_t vmk_remap_flags; vm_map_t copy_submap; vm_object_t copy_object; vm_map_size_t copy_size; vm_object_offset_t copy_offset; int copy_vm_alias; + remap_flags = 0; + vmk_remap_flags = VM_MAP_KERNEL_FLAGS_NONE; + copy_object = VME_OBJECT(copy_entry); copy_offset = VME_OFFSET(copy_entry); copy_size = (copy_entry->vme_end - @@ -3236,7 +3945,7 @@ vm_map_enter_mem_object_helper( /* take a reference on the object */ if (copy_entry->is_sub_map) { - remap_flags |= VM_FLAGS_SUBMAP; + vmk_remap_flags.vmkf_submap = TRUE; copy_submap = VME_SUBMAP(copy_entry); vm_map_lock(copy_submap); vm_map_reference(copy_submap); @@ -3302,7 +4011,6 @@ vm_map_enter_mem_object_helper( remap_flags |= VM_FLAGS_FIXED; remap_flags |= VM_FLAGS_OVERWRITE; remap_flags &= ~VM_FLAGS_ANYWHERE; - remap_flags |= VM_MAKE_TAG(copy_vm_alias); if (!copy && !copy_entry->is_sub_map) { /* * copy-on-write should have been @@ -3316,6 +4024,8 @@ vm_map_enter_mem_object_helper( copy_size, (vm_map_offset_t) 0, remap_flags, + vmk_remap_flags, + copy_vm_alias, copy_object, copy_offset, copy, @@ -3335,7 +4045,7 @@ vm_map_enter_mem_object_helper( /* next mapping */ copy_addr += copy_size; } - + if (kr == KERN_SUCCESS) { if (flags & (VM_FLAGS_RETURN_DATA_ADDR | VM_FLAGS_RETURN_4K_DATA_ADDR)) { @@ -3380,14 +4090,17 @@ vm_map_enter_mem_object_helper( } return kr; - + } else { - /* This is the case where we are going to map */ - /* an already mapped object. If the object is */ - /* not ready it is internal. An external */ - /* object cannot be mapped until it is ready */ - /* we can therefore avoid the ready check */ - /* in this case. */ + unsigned int access; + vm_prot_t protections; + unsigned int wimg_mode; + + /* we are mapping a VM object */ + + protections = named_entry->protection & VM_PROT_ALL; + access = GET_MAP_MEM(named_entry->protection); + if (flags & (VM_FLAGS_RETURN_DATA_ADDR | VM_FLAGS_RETURN_4K_DATA_ADDR)) { offset_in_mapping = offset - vm_object_trunc_page(offset); @@ -3395,12 +4108,21 @@ vm_map_enter_mem_object_helper( offset_in_mapping &= ~((signed)(0xFFF)); offset = vm_object_trunc_page(offset); map_size = vm_object_round_page(offset + offset_in_mapping + initial_size) - offset; - } + } object = named_entry->backing.object; assert(object != VM_OBJECT_NULL); + vm_object_lock(object); named_entry_unlock(named_entry); - vm_object_reference(object); + + vm_object_reference_locked(object); + + wimg_mode = object->wimg_bits; + vm_prot_to_wimg(access, &wimg_mode); + if (object->wimg_bits != wimg_mode) + vm_object_change_wimg_mode(object, wimg_mode); + + vm_object_unlock(object); } } else if (ip_kotype(port) == IKOT_MEMORY_OBJECT) { /* @@ -3416,10 +4138,10 @@ vm_map_enter_mem_object_helper( panic("VM_FLAGS_RETURN_DATA_ADDR not expected for raw memory object."); } - object = vm_object_enter((memory_object_t)port, - size, FALSE, FALSE, FALSE); + object = memory_object_to_vm_object((memory_object_t)port); if (object == VM_OBJECT_NULL) return KERN_INVALID_OBJECT; + vm_object_reference(object); /* wait for object (if any) to be ready */ if (object != VM_OBJECT_NULL) { @@ -3542,17 +4264,37 @@ vm_map_enter_mem_object_helper( } /* - * If users want to try to prefault pages, the mapping and prefault + * If non-kernel users want to try to prefault pages, the mapping and prefault * needs to be atomic. */ - if (try_prefault) - flags |= VM_FLAGS_KEEP_MAP_LOCKED; - + kernel_prefault = (try_prefault && vm_kernel_map_is_kernel(target_map)); + vmk_flags.vmkf_keep_map_locked = (try_prefault && !kernel_prefault); + +#if __arm64__ + if (fourk) { + /* map this object in a "4K" pager */ + result = vm_map_enter_fourk(target_map, + &map_addr, + map_size, + (vm_map_offset_t) mask, + flags, + vmk_flags, + tag, + object, + offset, + copy, + cur_protection, + max_protection, + inheritance); + } else +#endif /* __arm64__ */ { result = vm_map_enter(target_map, &map_addr, map_size, (vm_map_offset_t)mask, flags, + vmk_flags, + tag, object, offset, copy, cur_protection, max_protection, @@ -3570,13 +4312,19 @@ vm_map_enter_mem_object_helper( unsigned int i = 0; int pmap_options; - pmap_options = PMAP_OPTIONS_NOWAIT; + pmap_options = kernel_prefault ? 0 : PMAP_OPTIONS_NOWAIT; if (object->internal) { pmap_options |= PMAP_OPTIONS_INTERNAL; } for (i = 0; i < page_list_count; ++i) { - if (UPL_VALID_PAGE(page_list, i)) { + if (!UPL_VALID_PAGE(page_list, i)) { + if (kernel_prefault) { + assertf(FALSE, "kernel_prefault && !UPL_VALID_PAGE"); + result = KERN_MEMORY_ERROR; + break; + } + } else { /* * If this function call failed, we should stop * trying to optimize, other calls are likely @@ -3592,6 +4340,9 @@ vm_map_enter_mem_object_helper( 0, TRUE, pmap_options, NULL); if (kr != KERN_SUCCESS) { OSIncrementAtomic64(&vm_prefault_nb_bailout); + if (kernel_prefault) { + result = kr; + } break; } OSIncrementAtomic64(&vm_prefault_nb_pages); @@ -3600,7 +4351,9 @@ vm_map_enter_mem_object_helper( /* Next virtual address */ va += PAGE_SIZE; } - vm_map_unlock(target_map); + if (vmk_flags.vmkf_keep_map_locked) { + vm_map_unlock(target_map); + } } if (flags & (VM_FLAGS_RETURN_DATA_ADDR | @@ -3619,6 +4372,8 @@ vm_map_enter_mem_object( vm_map_size_t initial_size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, ipc_port_t port, vm_object_offset_t offset, boolean_t copy, @@ -3626,9 +4381,31 @@ vm_map_enter_mem_object( vm_prot_t max_protection, vm_inherit_t inheritance) { - return vm_map_enter_mem_object_helper(target_map, address, initial_size, mask, flags, - port, offset, copy, cur_protection, max_protection, - inheritance, NULL, 0); + kern_return_t ret; + + ret = vm_map_enter_mem_object_helper(target_map, + address, + initial_size, + mask, + flags, + vmk_flags, + tag, + port, + offset, + copy, + cur_protection, + max_protection, + inheritance, + NULL, + 0); + +#if KASAN + if (ret == KERN_SUCCESS && address && target_map->pmap == kernel_pmap) { + kasan_notify_address(*address, initial_size); + } +#endif + + return ret; } kern_return_t @@ -3638,6 +4415,8 @@ vm_map_enter_mem_object_prefault( vm_map_size_t initial_size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, ipc_port_t port, vm_object_offset_t offset, vm_prot_t cur_protection, @@ -3645,9 +4424,31 @@ vm_map_enter_mem_object_prefault( upl_page_list_ptr_t page_list, unsigned int page_list_count) { - return vm_map_enter_mem_object_helper(target_map, address, initial_size, mask, flags, - port, offset, FALSE, cur_protection, max_protection, - VM_INHERIT_DEFAULT, page_list, page_list_count); + kern_return_t ret; + + ret = vm_map_enter_mem_object_helper(target_map, + address, + initial_size, + mask, + flags, + vmk_flags, + tag, + port, + offset, + FALSE, + cur_protection, + max_protection, + VM_INHERIT_DEFAULT, + page_list, + page_list_count); + +#if KASAN + if (ret == KERN_SUCCESS && address && target_map->pmap == kernel_pmap) { + kasan_notify_address(*address, initial_size); + } +#endif + + return ret; } @@ -3658,6 +4459,8 @@ vm_map_enter_mem_object_control( vm_map_size_t initial_size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, memory_object_control_t control, vm_object_offset_t offset, boolean_t copy, @@ -3673,6 +4476,9 @@ vm_map_enter_mem_object_control( memory_object_t pager; vm_prot_t pager_prot; kern_return_t kr; +#if __arm64__ + boolean_t fourk = vmk_flags.vmkf_fourk; +#endif /* __arm64__ */ /* * Check arguments for validity @@ -3685,6 +4491,14 @@ vm_map_enter_mem_object_control( return KERN_INVALID_ARGUMENT; } +#if __arm64__ + if (fourk) { + map_addr = vm_map_trunc_page(*address, + FOURK_PAGE_MASK); + map_size = vm_map_round_page(initial_size, + FOURK_PAGE_MASK); + } else +#endif /* __arm64__ */ { map_addr = vm_map_trunc_page(*address, VM_MAP_PAGE_MASK(target_map)); @@ -3789,11 +4603,28 @@ vm_map_enter_mem_object_control( offset = new_offset; } +#if __arm64__ + if (fourk) { + result = vm_map_enter_fourk(target_map, + &map_addr, + map_size, + (vm_map_offset_t)mask, + flags, + vmk_flags, + tag, + object, offset, + copy, + cur_protection, max_protection, + inheritance); + } else +#endif /* __arm64__ */ { result = vm_map_enter(target_map, &map_addr, map_size, (vm_map_offset_t)mask, flags, + vmk_flags, + tag, object, offset, copy, cur_protection, max_protection, @@ -3891,13 +4722,6 @@ vm_map_enter_cpm( assert(!m->pageout); assert(!m->tabled); assert(VM_PAGE_WIRED(m)); - /* - * ENCRYPTED SWAP: - * "m" is not supposed to be pageable, so it - * should not be encrypted. It wouldn't be safe - * to enter it in a new VM object while encrypted. - */ - ASSERT_PAGE_DECRYPTED(m); assert(m->busy); assert(VM_PAGE_GET_PHYS_PAGE(m)>=(avail_start>>PAGE_SHIFT) && VM_PAGE_GET_PHYS_PAGE(m)<=(avail_end>>PAGE_SHIFT)); @@ -3930,6 +4754,7 @@ vm_map_enter_cpm( size, (vm_map_offset_t)0, flags, + VM_MAP_KERNEL_FLAGS_NONE, cpm_obj, (vm_object_offset_t)0, FALSE, @@ -3980,8 +4805,15 @@ vm_map_enter_cpm( type_of_fault = DBG_ZERO_FILL_FAULT; vm_fault_enter(m, pmap, va, VM_PROT_ALL, VM_PROT_WRITE, - VM_PAGE_WIRED(m), FALSE, FALSE, FALSE, 0, NULL, - &type_of_fault); + VM_PAGE_WIRED(m), + FALSE, /* change_wiring */ + VM_KERN_MEMORY_NONE, /* tag - not wiring */ + FALSE, /* no_cache */ + FALSE, /* cs_bypass */ + 0, /* user_tag */ + 0, /* pmap_options */ + NULL, /* need_retry */ + &type_of_fault); vm_object_unlock(cpm_obj); } @@ -4176,7 +5008,7 @@ vm_map_clip_start( } if (entry->vme_atomic) { panic("Attempting to clip an atomic VM entry! (map: %p, entry: %p)\n", map, entry); - } + } _vm_map_clip_start(&map->hdr, entry, startaddr); if (map->holelistenabled) { vm_map_store_update_first_free(map, NULL, FALSE); @@ -4359,7 +5191,7 @@ _vm_map_clip_end( /* * vm_map_range_check: [ internal use only ] - * + * * Check that the region defined by the specified start and * end addresses are wholly contained within a single map * entry or set of adjacent map entries of the spacified map, @@ -4395,7 +5227,7 @@ vm_map_range_check( return (FALSE); /* - * Optimize for the case that the region is contained + * Optimize for the case that the region is contained * in a single map entry. */ if (entry != (vm_map_entry_t *) NULL) @@ -4488,7 +5320,7 @@ vm_map_submap( * This submap is being mapped in a map * that uses a different pmap. * Set its "mapped_in_other_pmaps" flag - * to indicate that we now need to + * to indicate that we now need to * remove mappings from all pmaps rather * than just the submap's pmap. */ @@ -4506,9 +5338,12 @@ vm_map_submap( vm_map_unlock(map); return(KERN_NO_SPACE); } +#if defined(__arm__) || defined(__arm64__) + pmap_set_nested(submap->pmap); +#endif } result = pmap_nest(map->pmap, - (VME_SUBMAP(entry))->pmap, + (VME_SUBMAP(entry))->pmap, (addr64_t)start, (addr64_t)start, (uint64_t)(end - start)); @@ -4526,6 +5361,11 @@ vm_map_submap( return(result); } +#if CONFIG_EMBEDDED && (DEVELOPMENT || DEBUG) +#include +extern int proc_selfcsflags(void); +extern int panic_on_unsigned_execute; +#endif /* CONFIG_EMBEDDED && (DEVELOPMENT || DEBUG) */ /* * vm_map_protect: @@ -4547,6 +5387,7 @@ vm_map_protect( vm_map_offset_t prev; vm_map_entry_t entry; vm_prot_t new_max; + int pmap_options = 0; XPR(XPR_VM_MAP, "vm_map_protect, 0x%X start 0x%X end 0x%X, new 0x%X %d", @@ -4613,12 +5454,41 @@ vm_map_protect( return(KERN_PROTECTION_FAILURE); } } - + +#if CONFIG_EMBEDDED + if (new_prot & VM_PROT_WRITE) { + if ((new_prot & VM_PROT_EXECUTE) && !(current->used_for_jit)) { + printf("EMBEDDED: %s can't have both write and exec at the same time\n", __FUNCTION__); + new_prot &= ~VM_PROT_EXECUTE; + } + } +#endif prev = current->vme_end; current = current->vme_next; } +#if __arm64__ + if (end > prev && + end == vm_map_round_page(prev, VM_MAP_PAGE_MASK(map))) { + vm_map_entry_t prev_entry; + + prev_entry = current->vme_prev; + if (prev_entry != vm_map_to_entry(map) && + !prev_entry->map_aligned && + (vm_map_round_page(prev_entry->vme_end, + VM_MAP_PAGE_MASK(map)) + == end)) { + /* + * The last entry in our range is not "map-aligned" + * but it would have reached all the way to "end" + * if it had been map-aligned, so this is not really + * a hole in the range and we can proceed. + */ + prev = end; + } + } +#endif /* __arm64__ */ if (end > prev) { vm_map_unlock(map); @@ -4660,7 +5530,7 @@ vm_map_protect( if (current->is_sub_map == FALSE && VME_OBJECT(current) == VM_OBJECT_NULL) { - VME_OBJECT_SET(current, + VME_OBJECT_SET(current, vm_object_allocate( (vm_map_size_t) (current->vme_end - @@ -4675,7 +5545,7 @@ vm_map_protect( if (set_max) current->protection = - (current->max_protection = + (current->max_protection = new_prot & ~VM_PROT_COPY) & old_prot; else @@ -4683,11 +5553,11 @@ vm_map_protect( /* * Update physical map if necessary. - * If the request is to turn off write protection, - * we won't do it for real (in pmap). This is because - * it would cause copy-on-write to fail. We've already - * set, the new protection in the map, so if a - * write-protect fault occurred, it will be fixed up + * If the request is to turn off write protection, + * we won't do it for real (in pmap). This is because + * it would cause copy-on-write to fail. We've already + * set, the new protection in the map, so if a + * write-protect fault occurred, it will be fixed up * properly, COW or not. */ if (current->protection != old_prot) { @@ -4708,17 +5578,63 @@ vm_map_protect( if (override_nx(map, VME_ALIAS(current)) && prot) prot |= VM_PROT_EXECUTE; +#if CONFIG_EMBEDDED && (DEVELOPMENT || DEBUG) + if (!(old_prot & VM_PROT_EXECUTE) && + (prot & VM_PROT_EXECUTE) && + (proc_selfcsflags() & CS_KILL) && + panic_on_unsigned_execute) { + panic("vm_map_protect(%p,0x%llx,0x%llx) old=0x%x new=0x%x - code-signing bypass?\n", map, (uint64_t)current->vme_start, (uint64_t)current->vme_end, old_prot, prot); + } +#endif /* CONFIG_EMBEDDED && (DEVELOPMENT || DEBUG) */ + + if (pmap_has_prot_policy(prot)) { + if (current->wired_count) { + panic("vm_map_protect(%p,0x%llx,0x%llx) new=0x%x wired=%x\n", + map, (uint64_t)current->vme_start, (uint64_t)current->vme_end, prot, current->wired_count); + } + + /* If the pmap layer cares about this + * protection type, force a fault for + * each page so that vm_fault will + * repopulate the page with the full + * set of protections. + */ + /* + * TODO: We don't seem to need this, + * but this is due to an internal + * implementation detail of + * pmap_protect. Do we want to rely + * on this? + */ + prot = VM_PROT_NONE; + } if (current->is_sub_map && current->use_pmap) { - pmap_protect(VME_SUBMAP(current)->pmap, + pmap_protect(VME_SUBMAP(current)->pmap, current->vme_start, current->vme_end, prot); } else { - pmap_protect(map->pmap, - current->vme_start, - current->vme_end, - prot); + if (prot & VM_PROT_WRITE) { + if (VME_OBJECT(current) == compressor_object) { + /* + * For write requests on the + * compressor, we wil ask the + * pmap layer to prevent us from + * taking a write fault when we + * attempt to access the mapping + * next. + */ + pmap_options |= PMAP_OPTIONS_PROTECT_IMMEDIATE; + } + } + + pmap_protect_options(map->pmap, + current->vme_start, + current->vme_end, + prot, + pmap_options, + NULL); } } current = current->vme_next; @@ -4808,9 +5724,9 @@ vm_map_inherit( static kern_return_t add_wire_counts( vm_map_t map, - vm_map_entry_t entry, + vm_map_entry_t entry, boolean_t user_wire) -{ +{ vm_map_size_t size; if (user_wire) { @@ -4823,7 +5739,7 @@ add_wire_counts( if (entry->user_wired_count == 0) { size = entry->vme_end - entry->vme_start; - + /* * Since this is the first time the user is wiring this map entry, check to see if we're * exceeding the user wire limits. There is a per map limit which is the smaller of either @@ -4876,9 +5792,9 @@ add_wire_counts( static void subtract_wire_counts( vm_map_t map, - vm_map_entry_t entry, + vm_map_entry_t entry, boolean_t user_wire) -{ +{ if (user_wire) { @@ -4912,6 +5828,9 @@ subtract_wire_counts( } } +#if CONFIG_EMBEDDED +int cs_executable_wire = 0; +#endif /* CONFIG_EMBEDDED */ /* * vm_map_wire: @@ -4932,8 +5851,9 @@ vm_map_wire_nested( vm_map_offset_t start, vm_map_offset_t end, vm_prot_t caller_prot, + vm_tag_t tag, boolean_t user_wire, - pmap_t map_pmap, + pmap_t map_pmap, vm_map_offset_t pmap_addr, ppnum_t *physpage_p) { @@ -5044,7 +5964,7 @@ vm_map_wire_nested( /* * User wiring is interruptible */ - wait_result = vm_map_entry_wait(map, + wait_result = vm_map_entry_wait(map, (user_wire) ? THREAD_ABORTSAFE : THREAD_UNINT); if (user_wire && wait_result == THREAD_INTERRUPTED) { @@ -5079,7 +5999,7 @@ vm_map_wire_nested( entry = first_entry; continue; } - + if (entry->is_sub_map) { vm_map_offset_t sub_start; vm_map_offset_t sub_end; @@ -5104,7 +6024,7 @@ vm_map_wire_nested( sub_start = VME_OFFSET(entry); sub_end = entry->vme_end; sub_end += VME_OFFSET(entry) - entry->vme_start; - + local_end = entry->vme_end; if(map_pmap == NULL) { vm_object_t object; @@ -5151,7 +6071,7 @@ vm_map_wire_nested( lookup_map = map; vm_map_lock_write_to_read(map); if(vm_map_lookup_locked( - &lookup_map, local_start, + &lookup_map, local_start, access_type | VM_PROT_COPY, OBJECT_LOCK_EXCLUSIVE, &version, &object, @@ -5172,7 +6092,7 @@ vm_map_wire_nested( vm_map_lock(map); /* we unlocked, so must re-lookup */ - if (!vm_map_lookup_entry(map, + if (!vm_map_lookup_entry(map, local_start, &local_entry)) { rc = KERN_FAILURE; @@ -5208,9 +6128,9 @@ vm_map_wire_nested( entry->in_transition = TRUE; vm_map_unlock(map); - rc = vm_map_wire_nested(VME_SUBMAP(entry), + rc = vm_map_wire_nested(VME_SUBMAP(entry), sub_start, sub_end, - caller_prot, + caller_prot, tag, user_wire, pmap, pmap_addr, NULL); vm_map_lock(map); @@ -5346,6 +6266,33 @@ vm_map_wire_nested( * Unwired entry or wire request transmitted via submap */ +#if CONFIG_EMBEDDED + /* + * Wiring would copy the pages to the shadow object. + * The shadow object would not be code-signed so + * attempting to execute code from these copied pages + * would trigger a code-signing violation. + */ + if (entry->protection & VM_PROT_EXECUTE) { +#if MACH_ASSERT + printf("pid %d[%s] wiring executable range from " + "0x%llx to 0x%llx: rejected to preserve " + "code-signing\n", + proc_selfpid(), + (current_task()->bsd_info + ? proc_name_address(current_task()->bsd_info) + : "?"), + (uint64_t) entry->vme_start, + (uint64_t) entry->vme_end); +#endif /* MACH_ASSERT */ + DTRACE_VM2(cs_executable_wire, + uint64_t, (uint64_t)entry->vme_start, + uint64_t, (uint64_t)entry->vme_end); + cs_executable_wire++; + rc = KERN_PROTECTION_FAILURE; + goto done; + } +#endif /* CONFIG_EMBEDDED */ /* @@ -5445,12 +6392,12 @@ vm_map_wire_nested( interruptible_state = THREAD_UNINT; if(map_pmap) - rc = vm_fault_wire(map, - &tmp_entry, caller_prot, map_pmap, pmap_addr, + rc = vm_fault_wire(map, + &tmp_entry, caller_prot, tag, map_pmap, pmap_addr, physpage_p); else - rc = vm_fault_wire(map, - &tmp_entry, caller_prot, map->pmap, + rc = vm_fault_wire(map, + &tmp_entry, caller_prot, tag, map->pmap, tmp_entry.vme_start, physpage_p); @@ -5541,24 +6488,23 @@ vm_map_wire_external( { kern_return_t kret; - caller_prot &= ~VM_PROT_MEMORY_TAG_MASK; - caller_prot |= VM_PROT_MEMORY_TAG_MAKE(vm_tag_bt()); - kret = vm_map_wire_nested(map, start, end, caller_prot, + kret = vm_map_wire_nested(map, start, end, caller_prot, vm_tag_bt(), user_wire, (pmap_t)NULL, 0, NULL); return kret; } kern_return_t -vm_map_wire( +vm_map_wire_kernel( vm_map_t map, vm_map_offset_t start, vm_map_offset_t end, vm_prot_t caller_prot, + vm_tag_t tag, boolean_t user_wire) { kern_return_t kret; - kret = vm_map_wire_nested(map, start, end, caller_prot, + kret = vm_map_wire_nested(map, start, end, caller_prot, tag, user_wire, (pmap_t)NULL, 0, NULL); return kret; } @@ -5573,12 +6519,11 @@ vm_map_wire_and_extract_external( { kern_return_t kret; - caller_prot &= ~VM_PROT_MEMORY_TAG_MASK; - caller_prot |= VM_PROT_MEMORY_TAG_MAKE(vm_tag_bt()); kret = vm_map_wire_nested(map, start, start+VM_MAP_PAGE_SIZE(map), - caller_prot, + caller_prot, + vm_tag_bt(), user_wire, (pmap_t)NULL, 0, @@ -5591,10 +6536,11 @@ vm_map_wire_and_extract_external( } kern_return_t -vm_map_wire_and_extract( +vm_map_wire_and_extract_kernel( vm_map_t map, vm_map_offset_t start, vm_prot_t caller_prot, + vm_tag_t tag, boolean_t user_wire, ppnum_t *physpage_p) { @@ -5603,7 +6549,8 @@ vm_map_wire_and_extract( kret = vm_map_wire_nested(map, start, start+VM_MAP_PAGE_SIZE(map), - caller_prot, + caller_prot, + tag, user_wire, (pmap_t)NULL, 0, @@ -5706,7 +6653,7 @@ vm_map_unwire_nested( * going on. * We should probably just wait and retry, * but then we have to be careful that this - * entry could get "simplified" after + * entry could get "simplified" after * "in_transition" gets unset and before * we re-lookup the entry, so we would * have to re-clip the entry to avoid @@ -5732,7 +6679,7 @@ vm_map_unwire_nested( vm_map_offset_t sub_end; vm_map_offset_t local_end; pmap_t pmap; - + vm_map_clip_start(map, entry, start); vm_map_clip_end(map, entry, end); @@ -5761,9 +6708,9 @@ vm_map_unwire_nested( * Holes: Next entry should be contiguous unless * this is the end of the region. */ - if (((entry->vme_end < end) && + if (((entry->vme_end < end) && ((entry->vme_next == vm_map_to_entry(map)) || - (entry->vme_next->vme_start + (entry->vme_next->vme_start > entry->vme_end)))) { if (!user_wire) panic("vm_map_unwire: non-contiguous region"); @@ -5788,16 +6735,16 @@ vm_map_unwire_nested( * guarantees existance of the entry. */ vm_map_unlock(map); - vm_map_unwire_nested(VME_SUBMAP(entry), + vm_map_unwire_nested(VME_SUBMAP(entry), sub_start, sub_end, user_wire, pmap, pmap_addr); vm_map_lock(map); if (last_timestamp+1 != map->timestamp) { /* - * Find the entry again. It could have been + * Find the entry again. It could have been * clipped or deleted after we unlocked the map. */ - if (!vm_map_lookup_entry(map, + if (!vm_map_lookup_entry(map, tmp_entry.vme_start, &first_entry)) { if (!user_wire) @@ -5810,7 +6757,7 @@ vm_map_unwire_nested( /* * clear transition bit for all constituent entries - * that were in the original entry (saved in + * that were in the original entry (saved in * tmp_entry). Also check for waiters. */ while ((entry != vm_map_to_entry(map)) && @@ -5833,10 +6780,10 @@ vm_map_unwire_nested( if (last_timestamp+1 != map->timestamp) { /* - * Find the entry again. It could have been + * Find the entry again. It could have been * clipped or deleted after we unlocked the map. */ - if (!vm_map_lookup_entry(map, + if (!vm_map_lookup_entry(map, tmp_entry.vme_start, &first_entry)) { if (!user_wire) @@ -5858,7 +6805,7 @@ vm_map_unwire_nested( entry = entry->vme_next; continue; } - + assert(entry->wired_count > 0 && (!user_wire || entry->user_wired_count > 0)); @@ -5870,7 +6817,7 @@ vm_map_unwire_nested( * Holes: Next entry should be contiguous unless * this is the end of the region. */ - if (((entry->vme_end < end) && + if (((entry->vme_end < end) && ((entry->vme_next == vm_map_to_entry(map)) || (entry->vme_next->vme_start > entry->vme_end)))) { @@ -5900,11 +6847,11 @@ vm_map_unwire_nested( */ vm_map_unlock(map); if(map_pmap) { - vm_fault_unwire(map, + vm_fault_unwire(map, &tmp_entry, FALSE, map_pmap, pmap_addr); } else { - vm_fault_unwire(map, - &tmp_entry, FALSE, map->pmap, + vm_fault_unwire(map, + &tmp_entry, FALSE, map->pmap, tmp_entry.vme_start); } vm_map_lock(map); @@ -5968,7 +6915,7 @@ vm_map_unwire( vm_map_offset_t end, boolean_t user_wire) { - return vm_map_unwire_nested(map, start, end, + return vm_map_unwire_nested(map, start, end, user_wire, (pmap_t)NULL, 0); } @@ -5977,7 +6924,7 @@ vm_map_unwire( * vm_map_entry_delete: [ internal use only ] * * Deallocate the given entry from the target map. - */ + */ static void vm_map_entry_delete( vm_map_t map, @@ -6042,11 +6989,11 @@ vm_map_submap_pmap_clean( vm_map_lock_read(sub_map); if(vm_map_lookup_entry(sub_map, offset, &entry)) { - + remove_size = (entry->vme_end - entry->vme_start); if(offset > entry->vme_start) remove_size -= offset - entry->vme_start; - + if(submap_end < entry->vme_end) { remove_size -= @@ -6074,8 +7021,8 @@ vm_map_submap_pmap_clean( VM_PROT_NONE, PMAP_OPTIONS_REMOVE); } else { - pmap_remove(map->pmap, - (addr64_t)start, + pmap_remove(map->pmap, + (addr64_t)start, (addr64_t)(start + remove_size)); } } @@ -6083,9 +7030,9 @@ vm_map_submap_pmap_clean( entry = entry->vme_next; - while((entry != vm_map_to_entry(sub_map)) + while((entry != vm_map_to_entry(sub_map)) && (entry->vme_start < submap_end)) { - remove_size = (entry->vme_end - entry->vme_start); + remove_size = (entry->vme_end - entry->vme_start); if(submap_end < entry->vme_end) { remove_size -= entry->vme_end - submap_end; } @@ -6108,10 +7055,10 @@ vm_map_submap_pmap_clean( VM_PROT_NONE, PMAP_OPTIONS_REMOVE); } else { - pmap_remove(map->pmap, - (addr64_t)((start + entry->vme_start) + pmap_remove(map->pmap, + (addr64_t)((start + entry->vme_start) - offset), - (addr64_t)(((start + entry->vme_start) + (addr64_t)(((start + entry->vme_start) - offset) + remove_size)); } } @@ -6148,7 +7095,7 @@ vm_map_delete( unsigned int last_timestamp = ~0; /* unlikely value */ int interruptible; - interruptible = (flags & VM_MAP_REMOVE_INTERRUPTIBLE) ? + interruptible = (flags & VM_MAP_REMOVE_INTERRUPTIBLE) ? THREAD_ABORTSAFE : THREAD_UNINT; /* @@ -6317,9 +7264,48 @@ vm_map_delete( } if (entry->permanent) { - panic("attempt to remove permanent VM map entry " - "%p [0x%llx:0x%llx]\n", - entry, (uint64_t) s, (uint64_t) end); + if (map->pmap == kernel_pmap) { + panic("%s(%p,0x%llx,0x%llx): " + "attempt to remove permanent " + "VM map entry " + "%p [0x%llx:0x%llx]\n", + __FUNCTION__, + map, + (uint64_t) start, + (uint64_t) end, + entry, + (uint64_t) entry->vme_start, + (uint64_t) entry->vme_end); + } else if (flags & VM_MAP_REMOVE_IMMUTABLE) { +// printf("FBDP %d[%s] removing permanent entry %p [0x%llx:0x%llx] prot 0x%x/0x%x\n", proc_selfpid(), (current_task()->bsd_info ? proc_name_address(current_task()->bsd_info) : "?"), entry, (uint64_t)entry->vme_start, (uint64_t)entry->vme_end, entry->protection, entry->max_protection); + entry->permanent = FALSE; + } else { + if (!vm_map_executable_immutable_no_log) { + printf("%d[%s] %s(0x%llx,0x%llx): " + "permanent entry [0x%llx:0x%llx] " + "prot 0x%x/0x%x\n", + proc_selfpid(), + (current_task()->bsd_info + ? proc_name_address(current_task()->bsd_info) + : "?"), + __FUNCTION__, + (uint64_t) start, + (uint64_t) end, + (uint64_t)entry->vme_start, + (uint64_t)entry->vme_end, + entry->protection, + entry->max_protection); + } + /* + * dtrace -n 'vm_map_delete_permanent { print("start=0x%llx end=0x%llx prot=0x%x/0x%x\n", arg0, arg1, arg2, arg3); stack(); ustack(); }' + */ + DTRACE_VM5(vm_map_delete_permanent, + vm_map_offset_t, entry->vme_start, + vm_map_offset_t, entry->vme_end, + vm_prot_t, entry->protection, + vm_prot_t, entry->max_protection, + int, VME_ALIAS(entry)); + } } @@ -6382,7 +7368,7 @@ vm_map_delete( if (flags & VM_MAP_REMOVE_KUNWIRE) { entry->wired_count--; } - + /* * Remove all user wirings for proper accounting */ @@ -6411,9 +7397,9 @@ vm_map_delete( if (interruptible && wait_result == THREAD_INTERRUPTED) { /* - * We do not clear the - * needs_wakeup flag, since we - * cannot tell if we were the + * We do not clear the + * needs_wakeup flag, since we + * cannot tell if we were the * only one. */ return KERN_ABORTED; @@ -6424,7 +7410,7 @@ vm_map_delete( * it may not exist anymore. Look it * up again. */ - if (!vm_map_lookup_entry(map, s, + if (!vm_map_lookup_entry(map, s, &first_entry)) { assert(map != kernel_map); /* @@ -6462,7 +7448,7 @@ vm_map_delete( vm_map_offset_t sub_start, sub_end; pmap_t pmap; vm_map_offset_t pmap_addr; - + sub_map = VME_SUBMAP(&tmp_entry); sub_start = VME_OFFSET(&tmp_entry); @@ -6503,7 +7489,7 @@ vm_map_delete( * been clipped after we unlocked the map. */ if (!vm_map_lookup_entry(map, s, &first_entry)){ - assert((map != kernel_map) && + assert((map != kernel_map) && (!entry->is_sub_map)); first_entry = first_entry->vme_next; s = first_entry->vme_start; @@ -6672,8 +7658,15 @@ vm_map_delete( s = next->vme_start; last_timestamp = map->timestamp; - if ((flags & VM_MAP_REMOVE_SAVE_ENTRIES) && - zap_map != VM_MAP_NULL) { + if (entry->permanent) { + /* + * A permanent entry can not be removed, so leave it + * in place but remove all access permissions. + */ + entry->protection = VM_PROT_NONE; + entry->max_protection = VM_PROT_NONE; + } else if ((flags & VM_MAP_REMOVE_SAVE_ENTRIES) && + zap_map != VM_MAP_NULL) { vm_map_size_t entry_size; /* * The caller wants to save the affected VM map entries @@ -6717,9 +7710,9 @@ vm_map_delete( } else { SAVE_HINT_MAP_WRITE(map, entry->vme_prev); } - /* - * others can not only allocate behind us, we can - * also see coalesce while we don't have the map lock + /* + * others can not only allocate behind us, we can + * also see coalesce while we don't have the map lock */ if(entry == vm_map_to_entry(map)) { break; @@ -6963,13 +7956,13 @@ start_pass_1: if(entry->vme_end < dst_end) sub_end = entry->vme_end; - else + else sub_end = dst_end; sub_end -= entry->vme_start; sub_end += VME_OFFSET(entry); local_end = entry->vme_end; vm_map_unlock(dst_map); - + result = vm_map_overwrite_submap_recurse( VME_SUBMAP(entry), sub_start, @@ -6980,7 +7973,7 @@ start_pass_1: if (dst_end <= entry->vme_end) return KERN_SUCCESS; vm_map_lock(dst_map); - if(!vm_map_lookup_entry(dst_map, local_end, + if(!vm_map_lookup_entry(dst_map, local_end, &tmp_entry)) { vm_map_unlock(dst_map); return(KERN_INVALID_ADDRESS); @@ -7131,7 +8124,7 @@ vm_map_copy_overwrite_nested( if (copy->type == VM_MAP_COPY_KERNEL_BUFFER) { return(vm_map_copyout_kernel_buffer( - dst_map, &dst_addr, + dst_map, &dst_addr, copy, copy->size, TRUE, discard_on_success)); } @@ -7179,7 +8172,7 @@ vm_map_copy_overwrite_nested( vm_map_unlock(dst_map); return(KERN_INVALID_ADDRESS); } - + start_pass_1: if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) { vm_map_unlock(dst_map); @@ -7219,12 +8212,12 @@ start_pass_1: if(entry->vme_end < dst_end) sub_end = entry->vme_end; - else + else sub_end = dst_end; sub_end -= entry->vme_start; sub_end += VME_OFFSET(entry); vm_map_unlock(dst_map); - + kr = vm_map_overwrite_submap_recurse( VME_SUBMAP(entry), sub_start, @@ -7236,7 +8229,7 @@ start_pass_1: if (dst_end <= entry->vme_end) goto start_overwrite; - if(!vm_map_lookup_entry(dst_map, local_end, + if(!vm_map_lookup_entry(dst_map, local_end, &entry)) { vm_map_unlock(dst_map); return(KERN_INVALID_ADDRESS); @@ -7325,7 +8318,7 @@ start_overwrite: } else { copy_size = copy->size; } - + base_addr = dst_addr; while(TRUE) { /* deconstruct the copy object and do in parts */ @@ -7336,7 +8329,7 @@ start_overwrite: int nentries; int remaining_entries = 0; vm_map_offset_t new_offset = 0; - + for (entry = tmp_entry; copy_size == 0;) { vm_map_entry_t next; @@ -7357,7 +8350,7 @@ start_overwrite: entry->needs_wakeup = TRUE; vm_map_entry_wait(dst_map, THREAD_UNINT); - if(!vm_map_lookup_entry(dst_map, base_addr, + if(!vm_map_lookup_entry(dst_map, base_addr, &tmp_entry)) { vm_map_unlock(dst_map); return(KERN_INVALID_ADDRESS); @@ -7366,7 +8359,7 @@ start_overwrite: entry = tmp_entry; continue; } - if(entry->is_sub_map) { + if (entry->is_sub_map) { vm_map_offset_t sub_start; vm_map_offset_t sub_end; vm_map_offset_t local_end; @@ -7377,11 +8370,11 @@ start_overwrite: /* anonymous entry */ if(entry->vme_end < dst_end) sub_end = entry->vme_end; - else + else sub_end = dst_end; if(entry->vme_start < base_addr) sub_start = base_addr; - else + else sub_start = entry->vme_start; vm_map_clip_end( dst_map, entry, sub_end); @@ -7391,24 +8384,16 @@ start_overwrite: entry->is_sub_map = FALSE; vm_map_deallocate( VME_SUBMAP(entry)); - VME_SUBMAP_SET(entry, NULL); + VME_OBJECT_SET(entry, NULL); + VME_OFFSET_SET(entry, 0); entry->is_shared = FALSE; entry->needs_copy = FALSE; - VME_OFFSET_SET(entry, 0); - /* - * XXX FBDP - * We should propagate the protections - * of the submap entry here instead - * of forcing them to VM_PROT_ALL... - * Or better yet, we should inherit - * the protection of the copy_entry. - */ - entry->protection = VM_PROT_ALL; + entry->protection = VM_PROT_DEFAULT; entry->max_protection = VM_PROT_ALL; entry->wired_count = 0; entry->user_wired_count = 0; - if(entry->inheritance - == VM_INHERIT_SHARE) + if(entry->inheritance + == VM_INHERIT_SHARE) entry->inheritance = VM_INHERIT_COPY; continue; } @@ -7416,7 +8401,7 @@ start_overwrite: /* entries to send */ if(base_addr < entry->vme_start) { /* stuff to send */ - copy_size = + copy_size = entry->vme_start - base_addr; break; } @@ -7424,7 +8409,7 @@ start_overwrite: if(entry->vme_end < dst_end) sub_end = entry->vme_end; - else + else sub_end = dst_end; sub_end -= entry->vme_start; sub_end += VME_OFFSET(entry); @@ -7440,31 +8425,31 @@ start_overwrite: nentries = 1; new_offset = copy->offset; copy_entry = vm_map_copy_first_entry(copy); - while(copy_entry != + while(copy_entry != vm_map_copy_to_entry(copy)){ - entry_size = copy_entry->vme_end - + entry_size = copy_entry->vme_end - copy_entry->vme_start; if((local_size < copy_size) && - ((local_size + entry_size) + ((local_size + entry_size) >= copy_size)) { - vm_map_copy_clip_end(copy, - copy_entry, + vm_map_copy_clip_end(copy, + copy_entry, copy_entry->vme_start + (copy_size - local_size)); - entry_size = copy_entry->vme_end - + entry_size = copy_entry->vme_end - copy_entry->vme_start; local_size += entry_size; new_offset += entry_size; } if(local_size >= copy_size) { next_copy = copy_entry->vme_next; - copy_entry->vme_next = + copy_entry->vme_next = vm_map_copy_to_entry(copy); - previous_prev = + previous_prev = copy->cpy_hdr.links.prev; copy->cpy_hdr.links.prev = copy_entry; copy->size = copy_size; - remaining_entries = + remaining_entries = copy->cpy_hdr.nentries; remaining_entries -= nentries; copy->cpy_hdr.nentries = nentries; @@ -7477,13 +8462,13 @@ start_overwrite: copy_entry = copy_entry->vme_next; } } - + if((entry->use_pmap) && (pmap == NULL)) { kr = vm_map_copy_overwrite_nested( VME_SUBMAP(entry), sub_start, copy, - interruptible, + interruptible, VME_SUBMAP(entry)->pmap, TRUE); } else if (pmap != NULL) { @@ -7504,11 +8489,11 @@ start_overwrite: } if(kr != KERN_SUCCESS) { if(next_copy != NULL) { - copy->cpy_hdr.nentries += + copy->cpy_hdr.nentries += remaining_entries; - copy->cpy_hdr.links.prev->vme_next = + copy->cpy_hdr.links.prev->vme_next = next_copy; - copy->cpy_hdr.links.prev + copy->cpy_hdr.links.prev = previous_prev; copy->size = total_size; } @@ -7519,7 +8504,7 @@ start_overwrite: } /* otherwise copy no longer exists, it was */ /* destroyed after successful copy_overwrite */ - copy = (vm_map_copy_t) + copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); copy->c_u.hdr.rb_head_store.rbh_root = (void*)(int)SKIP_RB_TREE; vm_map_copy_first_entry(copy) = @@ -7542,20 +8527,20 @@ start_overwrite: copy->cpy_hdr.links.next = next_copy; copy->cpy_hdr.links.prev = previous_prev; copy->size = total_size; - next_copy->vme_prev = + next_copy->vme_prev = vm_map_copy_to_entry(copy); next_copy = NULL; } base_addr = local_end; vm_map_lock(dst_map); - if(!vm_map_lookup_entry(dst_map, + if(!vm_map_lookup_entry(dst_map, local_end, &tmp_entry)) { vm_map_unlock(dst_map); return(KERN_INVALID_ADDRESS); } entry = tmp_entry; continue; - } + } if (dst_end <= entry->vme_end) { copy_size = dst_end - base_addr; break; @@ -7581,28 +8566,28 @@ start_overwrite: new_offset = copy->offset; copy_entry = vm_map_copy_first_entry(copy); while(copy_entry != vm_map_copy_to_entry(copy)) { - entry_size = copy_entry->vme_end - + entry_size = copy_entry->vme_end - copy_entry->vme_start; if((local_size < copy_size) && - ((local_size + entry_size) + ((local_size + entry_size) >= copy_size)) { - vm_map_copy_clip_end(copy, copy_entry, + vm_map_copy_clip_end(copy, copy_entry, copy_entry->vme_start + (copy_size - local_size)); - entry_size = copy_entry->vme_end - + entry_size = copy_entry->vme_end - copy_entry->vme_start; local_size += entry_size; new_offset += entry_size; } if(local_size >= copy_size) { next_copy = copy_entry->vme_next; - copy_entry->vme_next = + copy_entry->vme_next = vm_map_copy_to_entry(copy); - previous_prev = + previous_prev = copy->cpy_hdr.links.prev; copy->cpy_hdr.links.prev = copy_entry; copy->size = copy_size; - remaining_entries = + remaining_entries = copy->cpy_hdr.nentries; remaining_entries -= nentries; copy->cpy_hdr.nentries = nentries; @@ -7624,15 +8609,15 @@ start_overwrite: else local_pmap = dst_map->pmap; - if ((kr = vm_map_copy_overwrite_aligned( + if ((kr = vm_map_copy_overwrite_aligned( dst_map, tmp_entry, copy, base_addr, local_pmap)) != KERN_SUCCESS) { if(next_copy != NULL) { - copy->cpy_hdr.nentries += + copy->cpy_hdr.nentries += remaining_entries; - copy->cpy_hdr.links.prev->vme_next = + copy->cpy_hdr.links.prev->vme_next = next_copy; - copy->cpy_hdr.links.prev = + copy->cpy_hdr.links.prev = previous_prev; copy->size += copy_size; } @@ -7661,9 +8646,9 @@ start_overwrite: if(next_copy != NULL) { copy->cpy_hdr.nentries += remaining_entries; - copy->cpy_hdr.links.prev->vme_next = + copy->cpy_hdr.links.prev->vme_next = next_copy; - copy->cpy_hdr.links.prev = + copy->cpy_hdr.links.prev = previous_prev; copy->size += copy_size; } @@ -7685,7 +8670,7 @@ start_overwrite: } vm_map_lock(dst_map); while(TRUE) { - if (!vm_map_lookup_entry(dst_map, + if (!vm_map_lookup_entry(dst_map, base_addr, &tmp_entry)) { vm_map_unlock(dst_map); return(KERN_INVALID_ADDRESS); @@ -7726,6 +8711,7 @@ vm_map_copy_overwrite( vm_map_offset_t head_addr, tail_addr; vm_map_entry_t entry; kern_return_t kr; + vm_map_offset_t effective_page_mask, effective_page_size; head_size = 0; tail_size = 0; @@ -7750,15 +8736,20 @@ vm_map_copy_overwrite( TRUE); } - if (copy->size < 3 * PAGE_SIZE) { + effective_page_mask = MAX(VM_MAP_PAGE_MASK(dst_map), PAGE_MASK); + effective_page_mask = MAX(VM_MAP_COPY_PAGE_MASK(copy), + effective_page_mask); + effective_page_size = effective_page_mask + 1; + + if (copy->size < 3 * effective_page_size) { /* * Too small to bother with optimizing... */ goto blunt_copy; } - if ((dst_addr & VM_MAP_PAGE_MASK(dst_map)) != - (copy->offset & VM_MAP_PAGE_MASK(dst_map))) { + if ((dst_addr & effective_page_mask) != + (copy->offset & effective_page_mask)) { /* * Incompatible mis-alignment of source and destination... */ @@ -7770,21 +8761,26 @@ vm_map_copy_overwrite( * Let's try and do a small unaligned copy first (if needed) * and then an aligned copy for the rest. */ - if (!page_aligned(dst_addr)) { + if (!vm_map_page_aligned(dst_addr, effective_page_mask)) { head_addr = dst_addr; - head_size = (VM_MAP_PAGE_SIZE(dst_map) - - (copy->offset & VM_MAP_PAGE_MASK(dst_map))); + head_size = (effective_page_size - + (copy->offset & effective_page_mask)); + head_size = MIN(head_size, copy->size); } - if (!page_aligned(copy->offset + copy->size)) { + if (!vm_map_page_aligned(copy->offset + copy->size, + effective_page_mask)) { /* * Mis-alignment at the end. * Do an aligned copy up to the last page and * then an unaligned copy for the remaining bytes. */ tail_size = ((copy->offset + copy->size) & - VM_MAP_PAGE_MASK(dst_map)); + effective_page_mask); + tail_size = MIN(tail_size, copy->size); tail_addr = dst_addr + copy->size - tail_size; + assert(tail_addr >= head_addr + head_size); } + assert(head_size + tail_size <= copy->size); if (head_size + tail_size == copy->size) { /* @@ -7820,7 +8816,7 @@ vm_map_copy_overwrite( * Unaligned copy of the first "head_size" bytes, to reach * a page boundary. */ - + /* * Extract "head_copy" out of "copy". */ @@ -7836,13 +8832,16 @@ vm_map_copy_overwrite( copy->cpy_hdr.entries_pageable; vm_map_store_init(&head_copy->cpy_hdr); + entry = vm_map_copy_first_entry(copy); + if (entry->vme_end < copy->offset + head_size) { + head_size = entry->vme_end - copy->offset; + } + head_copy->offset = copy->offset; head_copy->size = head_size; - copy->offset += head_size; copy->size -= head_size; - entry = vm_map_copy_first_entry(copy); vm_map_copy_clip_end(copy, entry, copy->offset); vm_map_copy_entry_unlink(copy, entry); vm_map_copy_entry_link(head_copy, @@ -8004,7 +9003,7 @@ vm_map_copy_overwrite_unaligned( amount_left; kern_return_t kr = KERN_SUCCESS; - + copy_entry = vm_map_copy_first_entry(copy); vm_map_lock_write_to_read(dst_map); @@ -8014,7 +9013,7 @@ vm_map_copy_overwrite_unaligned( /* * unaligned so we never clipped this entry, we need the offset into * the vm_object not just the data. - */ + */ while (amount_left > 0) { if (entry == vm_map_to_entry(dst_map)) { @@ -8221,12 +9220,12 @@ vm_map_copy_overwrite_aligned( vm_map_size_t copy_size; vm_map_size_t size; vm_map_entry_t entry; - + while ((copy_entry = vm_map_copy_first_entry(copy)) != vm_map_copy_to_entry(copy)) { copy_size = (copy_entry->vme_end - copy_entry->vme_start); - + entry = tmp_entry; if (entry->is_sub_map) { /* unnested when clipped earlier */ @@ -8296,8 +9295,8 @@ vm_map_copy_overwrite_aligned( */ object = VME_OBJECT(entry); - if ((!entry->is_shared && - ((object == VM_OBJECT_NULL) || + if ((!entry->is_shared && + ((object == VM_OBJECT_NULL) || (object->internal && !object->true_share))) || entry->needs_copy) { vm_object_t old_object = VME_OBJECT(entry); @@ -8321,6 +9320,7 @@ vm_map_copy_overwrite_aligned( continue; } +#if !CONFIG_EMBEDDED #define __TRADEOFF1_OBJ_SIZE (64 * 1024 * 1024) /* 64 MB */ #define __TRADEOFF1_COPY_SIZE (128 * 1024) /* 128 KB */ if (VME_OBJECT(copy_entry) != VM_OBJECT_NULL && @@ -8338,6 +9338,7 @@ vm_map_copy_overwrite_aligned( vm_map_copy_overwrite_aligned_src_large++; goto slow_copy; } +#endif /* !CONFIG_EMBEDDED */ if ((dst_map->pmap != kernel_pmap) && (VME_ALIAS(entry) >= VM_MEMORY_MALLOC) && @@ -8353,8 +9354,10 @@ vm_map_copy_overwrite_aligned( vm_object_lock_shared(new_object); } while (new_object != VM_OBJECT_NULL && +#if !CONFIG_EMBEDDED !new_object->true_share && new_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC && +#endif /* !CONFIG_EMBEDDED */ new_object->internal) { new_shadow = new_object->shadow; if (new_shadow == VM_OBJECT_NULL) { @@ -8379,6 +9382,7 @@ vm_map_copy_overwrite_aligned( vm_object_unlock(new_object); goto slow_copy; } +#if !CONFIG_EMBEDDED if (new_object->true_share || new_object->copy_strategy != MEMORY_OBJECT_COPY_SYMMETRIC) { /* @@ -8391,6 +9395,7 @@ vm_map_copy_overwrite_aligned( vm_object_unlock(new_object); goto slow_copy; } +#endif /* !CONFIG_EMBEDDED */ vm_object_unlock(new_object); } /* @@ -8405,7 +9410,7 @@ vm_map_copy_overwrite_aligned( if(entry->is_sub_map) { if(entry->use_pmap) { #ifndef NO_NESTED_PMAP - pmap_unnest(dst_map->pmap, + pmap_unnest(dst_map->pmap, (addr64_t)entry->vme_start, entry->vme_end - entry->vme_start); #endif /* NO_NESTED_PMAP */ @@ -8420,7 +9425,7 @@ vm_map_copy_overwrite_aligned( } } else { vm_map_submap_pmap_clean( - dst_map, entry->vme_start, + dst_map, entry->vme_start, entry->vme_end, VME_SUBMAP(entry), VME_OFFSET(entry)); @@ -8431,7 +9436,7 @@ vm_map_copy_overwrite_aligned( vm_object_pmap_protect_options( VME_OBJECT(entry), VME_OFFSET(entry), - entry->vme_end + entry->vme_end - entry->vme_start, PMAP_NULL, entry->vme_start, @@ -8439,8 +9444,8 @@ vm_map_copy_overwrite_aligned( PMAP_OPTIONS_REMOVE); } else { pmap_remove_options( - dst_map->pmap, - (addr64_t)(entry->vme_start), + dst_map->pmap, + (addr64_t)(entry->vme_start), (addr64_t)(entry->vme_end), PMAP_OPTIONS_REMOVE); } @@ -8455,7 +9460,7 @@ vm_map_copy_overwrite_aligned( entry->wired_count = 0; entry->user_wired_count = 0; offset = VME_OFFSET(copy_entry); - VME_OFFSET_SET(entry, offset); + VME_OFFSET_SET(entry, offset); vm_map_copy_entry_unlink(copy, copy_entry); vm_map_copy_entry_dispose(copy, copy_entry); @@ -8521,7 +9526,7 @@ vm_map_copy_overwrite_aligned( VME_OBJECT_SET(entry, dst_object); VME_OFFSET_SET(entry, dst_offset); assert(entry->use_pmap); - + } vm_object_reference(dst_object); @@ -8658,7 +9663,7 @@ vm_map_copyin_kernel_buffer( (void) vm_map_remove( src_map, vm_map_trunc_page(src_addr, - VM_MAP_PAGE_MASK(src_map)), + VM_MAP_PAGE_MASK(src_map)), vm_map_round_page(src_addr + len, VM_MAP_PAGE_MASK(src_map)), (VM_MAP_REMOVE_INTERRUPTIBLE | @@ -8708,27 +9713,34 @@ vm_map_copyout_kernel_buffer( * Allocate space in the target map for the data */ *addr = 0; - kr = vm_map_enter(map, - addr, + kr = vm_map_enter(map, + addr, vm_map_round_page(copy_size, VM_MAP_PAGE_MASK(map)), - (vm_map_offset_t) 0, + (vm_map_offset_t) 0, VM_FLAGS_ANYWHERE, - VM_OBJECT_NULL, - (vm_object_offset_t) 0, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, + VM_OBJECT_NULL, + (vm_object_offset_t) 0, FALSE, - VM_PROT_DEFAULT, + VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) return kr; +#if KASAN + if (map->pmap == kernel_pmap) { + kasan_notify_address(*addr, copy->size); + } +#endif } /* * Copyout the data from the kernel buffer to the target map. - */ + */ if (thread->map == map) { - + /* * If the target map is the current map, just do * the copy. @@ -8754,7 +9766,7 @@ vm_map_copyout_kernel_buffer( vm_map_copyout_kernel_buffer_failures++; kr = KERN_INVALID_ADDRESS; } - + (void) vm_map_switch(oldmap); vm_map_deallocate(map); } @@ -8785,10 +9797,10 @@ vm_map_copyout_kernel_buffer( return kr; } - + /* * Macro: vm_map_copy_insert - * + * * Description: * Link a copy chain ("copy") into a map at the * specified location (after "where"). @@ -8986,6 +9998,8 @@ vm_map_copyout_internal( *dst_addr = 0; kr = vm_map_enter(dst_map, dst_addr, size, (vm_map_offset_t) 0, VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, object, offset, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); @@ -9256,16 +10270,6 @@ StartAgain: ; m->absent) panic("vm_map_copyout: wiring %p", m); - /* - * ENCRYPTED SWAP: - * The page is assumed to be wired here, so it - * shouldn't be encrypted. Otherwise, we - * couldn't enter it in the page table, since - * we don't want the user to see the encrypted - * data. - */ - ASSERT_PAGE_DECRYPTED(m); - prot = entry->protection; if (override_nx(dst_map, VME_ALIAS(entry)) && @@ -9275,14 +10279,19 @@ StartAgain: ; type_of_fault = DBG_CACHE_HIT_FAULT; vm_fault_enter(m, dst_map->pmap, va, prot, prot, - VM_PAGE_WIRED(m), FALSE, FALSE, - FALSE, VME_ALIAS(entry), - ((entry->iokit_acct || - (!entry->is_sub_map && - !entry->use_pmap)) - ? PMAP_OPTIONS_ALT_ACCT - : 0), - NULL, &type_of_fault); + VM_PAGE_WIRED(m), + FALSE, /* change_wiring */ + VM_KERN_MEMORY_NONE, /* tag - not wiring */ + FALSE, /* no_cache */ + FALSE, /* cs_bypass */ + VME_ALIAS(entry), + ((entry->iokit_acct || + (!entry->is_sub_map && + !entry->use_pmap)) + ? PMAP_OPTIONS_ALT_ACCT + : 0), /* pmap_options */ + NULL, /* need_retry */ + &type_of_fault); vm_object_unlock(object); @@ -9300,6 +10309,10 @@ after_adjustments: *dst_addr = start + (copy->offset - vm_copy_start); +#if KASAN + kasan_notify_address(*dst_addr, size); +#endif + /* * Update the hints and the map size */ @@ -9438,11 +10451,13 @@ vm_map_copyin_internal( boolean_t src_destroy; boolean_t use_maxprot; boolean_t preserve_purgeable; + boolean_t entry_was_shared; + vm_map_entry_t saved_src_entry; if (flags & ~VM_MAP_COPYIN_ALL_FLAGS) { return KERN_INVALID_ARGUMENT; } - + src_destroy = (flags & VM_MAP_COPYIN_SRC_DESTROY) ? TRUE : FALSE; use_maxprot = (flags & VM_MAP_COPYIN_USE_MAXPROT) ? TRUE : FALSE; preserve_purgeable = @@ -9501,7 +10516,7 @@ vm_map_copyin_internal( /* * Allocate a header element for the list. * - * Use the start and end in the header to + * Use the start and end in the header to * remember the endpoints prior to rounding. */ @@ -9529,7 +10544,7 @@ vm_map_copyin_internal( copy->offset = src_addr; copy->size = len; - + new_entry = vm_map_copy_entry_create(copy, !copy->cpy_hdr.entries_pageable); #define RETURN(x) \ @@ -9626,7 +10641,7 @@ vm_map_copyin_internal( if(submap_len > (src_end-src_start)) submap_len = src_end-src_start; ptr->base_len = submap_len; - + src_start -= tmp_entry->vme_start; src_start += VME_OFFSET(tmp_entry); src_end = src_start + submap_len; @@ -9646,7 +10661,7 @@ vm_map_copyin_internal( } /* we are now in the lowest level submap... */ - if ((VME_OBJECT(tmp_entry) != VM_OBJECT_NULL) && + if ((VME_OBJECT(tmp_entry) != VM_OBJECT_NULL) && (VME_OBJECT(tmp_entry)->phys_contiguous)) { /* This is not, supported for now.In future */ /* we will need to detect the phys_contig */ @@ -9658,7 +10673,7 @@ vm_map_copyin_internal( RETURN(KERN_PROTECTION_FAILURE); } /* - * Create a new address map entry to hold the result. + * Create a new address map entry to hold the result. * Fill in the fields from the appropriate source entries. * We must unlock the source map to do this if we need * to allocate a map entry. @@ -9767,7 +10782,7 @@ vm_map_copyin_internal( src_object, src_offset, src_size, - (src_entry->is_shared ? + (src_entry->is_shared ? PMAP_NULL : src_map->pmap), src_entry->vme_start, @@ -9786,6 +10801,8 @@ vm_map_copyin_internal( goto CopySuccessful; } + entry_was_shared = tmp_entry->is_shared; + /* * Take an object reference, so that we may * release the map lock(s). @@ -9801,6 +10818,9 @@ vm_map_copyin_internal( version.main_timestamp = src_map->timestamp; vm_map_unlock(src_map); /* Increments timestamp once! */ + saved_src_entry = src_entry; + tmp_entry = VM_MAP_ENTRY_NULL; + src_entry = VM_MAP_ENTRY_NULL; /* * Perform the copy @@ -9820,13 +10840,13 @@ vm_map_copyin_internal( } else if (src_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC && - (tmp_entry->is_shared || map_share)) { + (entry_was_shared || map_share)) { vm_object_t new_object; vm_object_lock_shared(src_object); new_object = vm_object_copy_delayed( src_object, - src_offset, + src_offset, src_size, TRUE); if (new_object == VM_OBJECT_NULL) @@ -9880,7 +10900,7 @@ vm_map_copyin_internal( state = src_object->purgable; vm_object_purgable_control( new_object, - VM_PURGABLE_SET_STATE, + VM_PURGABLE_SET_STATE_FROM_KERNEL, &state); } vm_object_unlock(new_object); @@ -9906,8 +10926,11 @@ vm_map_copyin_internal( vm_map_lock(src_map); - if ((version.main_timestamp + 1) == src_map->timestamp) + if ((version.main_timestamp + 1) == src_map->timestamp) { + /* src_map hasn't changed: src_entry is still valid */ + src_entry = saved_src_entry; goto VerificationSuccessful; + } /* * Simple version comparison failed. @@ -9959,7 +10982,7 @@ vm_map_copyin_internal( * This entry might have been extended * (vm_map_entry_simplify() or coalesce) * or been replaced with an entry that ends farther - * from "src_start" than before. + * from "src_start" than before. * * We've called vm_object_copy_*() only on * the previous range, so we can't @@ -10008,7 +11031,7 @@ vm_map_copyin_internal( vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry); - + /* * Determine whether the entire region * has been copied. @@ -10087,7 +11110,7 @@ vm_map_copyin_internal( /* * If the source should be destroyed, do it now, since the - * copy was successful. + * copy was successful. */ if (src_destroy) { (void) vm_map_delete( @@ -10104,16 +11127,17 @@ vm_map_copyin_internal( vm_map_simplify_range( src_map, vm_map_trunc_page(src_addr, - VM_MAP_PAGE_MASK(src_map)), + VM_MAP_PAGE_MASK(src_map)), vm_map_round_page(src_end, VM_MAP_PAGE_MASK(src_map))); } vm_map_unlock(src_map); + tmp_entry = VM_MAP_ENTRY_NULL; if (VM_MAP_PAGE_SHIFT(src_map) != PAGE_SHIFT) { vm_map_offset_t original_start, original_offset, original_end; - + assert(VM_MAP_COPY_PAGE_MASK(copy) == PAGE_MASK); /* adjust alignment of first copy_entry's "vme_start" */ @@ -10230,7 +11254,7 @@ vm_map_copyin_internal( */ tmp_entry->map_aligned = FALSE; - tmp_entry->vme_end = copy_addr + + tmp_entry->vme_end = copy_addr + (tmp_entry->vme_end - tmp_entry->vme_start); tmp_entry->vme_start = copy_addr; assert(tmp_entry->vme_start < tmp_entry->vme_end); @@ -10304,7 +11328,7 @@ vm_map_copy_extract( /* * Allocate a header element for the list. * - * Use the start and end in the header to + * Use the start and end in the header to * remember the endpoints prior to rounding. */ @@ -10390,7 +11414,7 @@ vm_map_fork_share( * or someone else already has one, then * make a new shadow and share it. */ - + object = VME_OBJECT(old_entry); if (old_entry->is_sub_map) { assert(old_entry->wired_count == 0); @@ -10398,8 +11422,8 @@ vm_map_fork_share( if(old_entry->use_pmap) { kern_return_t result; - result = pmap_nest(new_map->pmap, - (VME_SUBMAP(old_entry))->pmap, + result = pmap_nest(new_map->pmap, + (VME_SUBMAP(old_entry))->pmap, (addr64_t)old_entry->vme_start, (addr64_t)old_entry->vme_start, (uint64_t)(old_entry->vme_end - old_entry->vme_start)); @@ -10413,16 +11437,15 @@ vm_map_fork_share( VME_OFFSET_SET(old_entry, 0); VME_OBJECT_SET(old_entry, object); old_entry->use_pmap = TRUE; - assert(!old_entry->needs_copy); } else if (object->copy_strategy != MEMORY_OBJECT_COPY_SYMMETRIC) { - + /* * We are already using an asymmetric * copy, and therefore we already have * the right object. */ - + assert(! old_entry->needs_copy); } else if (old_entry->needs_copy || /* case 1 */ @@ -10432,7 +11455,7 @@ vm_map_fork_share( (object->vo_size > (vm_map_size_t)(old_entry->vme_end - old_entry->vme_start)))) { - + /* * We need to create a shadow. * There are three cases here. @@ -10512,7 +11535,7 @@ vm_map_fork_share( VME_OBJECT_SHADOW(old_entry, (vm_map_size_t) (old_entry->vme_end - old_entry->vme_start)); - + /* * If we're making a shadow for other than * copy on write reasons, then we have @@ -10523,11 +11546,16 @@ vm_map_fork_share( (old_entry->protection & VM_PROT_WRITE)) { vm_prot_t prot; + assert(!pmap_has_prot_policy(old_entry->protection)); + prot = old_entry->protection & ~VM_PROT_WRITE; + assert(!pmap_has_prot_policy(prot)); + if (override_nx(old_map, VME_ALIAS(old_entry)) && prot) prot |= VM_PROT_EXECUTE; + if (old_map->mapped_in_other_pmaps) { vm_object_pmap_protect( VME_OBJECT(old_entry), @@ -10544,12 +11572,12 @@ vm_map_fork_share( prot); } } - + old_entry->needs_copy = FALSE; object = VME_OBJECT(old_entry); } - + /* * If object was using a symmetric copy strategy, * change its copy strategy to the default @@ -10558,7 +11586,7 @@ vm_map_fork_share( * norma case. Bump the reference count for the * new entry. */ - + if(old_entry->is_sub_map) { vm_map_lock(VME_SUBMAP(old_entry)); vm_map_reference(VME_SUBMAP(old_entry)); @@ -10571,12 +11599,12 @@ vm_map_fork_share( } vm_object_unlock(object); } - + /* * Clone the entry, using object ref from above. * Mark both entries as shared. */ - + new_entry = vm_map_entry_create(new_map, FALSE); /* Never the kernel * map or descendants */ vm_map_entry_copy(new_entry, old_entry); @@ -10593,19 +11621,19 @@ vm_map_fork_share( new_entry->protection &= ~VM_PROT_WRITE; new_entry->max_protection &= ~VM_PROT_WRITE; } - + /* * Insert the entry into the new map -- we * know we're inserting at the end of the new * map. */ - + vm_map_store_entry_link(new_map, vm_map_last_entry(new_map), new_entry); - + /* * Update the physical map */ - + if (old_entry->is_sub_map) { /* Bill Angell pmap support goes here */ } else { @@ -10658,21 +11686,21 @@ vm_map_fork_copy( * that INVALID_ADDRESS and * PROTECTION_FAILURE are handled above. */ - + return FALSE; } - + /* * Insert the copy into the new map */ - + vm_map_copy_insert(new_map, last, copy); - + /* * Pick up the traversal at the end of * the copied region. */ - + vm_map_lock(old_map); start += entry_size; if (! vm_map_lookup_entry(old_map, start, &last)) { @@ -10727,6 +11755,10 @@ vm_map_fork( pmap_is64bit = #if defined(__i386__) || defined(__x86_64__) old_map->pmap->pm_task_map != TASK_MAP_32BIT; +#elif defined(__arm64__) + old_map->pmap->max == MACH_VM_MAX_ADDRESS; +#elif defined(__arm__) + FALSE; #else #error Unknown architecture. #endif @@ -10740,6 +11772,7 @@ vm_map_fork( old_map->min_offset, old_map->max_offset, old_map->hdr.entries_pageable); + vm_map_lock(new_map); vm_commit_pagezero_status(new_map); /* inherit the parent map's page size */ vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(old_map)); @@ -10807,22 +11840,26 @@ vm_map_fork( /* * Handle copy-on-write obligations */ - + if (src_needs_copy && !old_entry->needs_copy) { vm_prot_t prot; + assert(!pmap_has_prot_policy(old_entry->protection)); + prot = old_entry->protection & ~VM_PROT_WRITE; if (override_nx(old_map, VME_ALIAS(old_entry)) && prot) prot |= VM_PROT_EXECUTE; + assert(!pmap_has_prot_policy(prot)); + vm_object_pmap_protect( VME_OBJECT(old_entry), VME_OFFSET(old_entry), (old_entry->vme_end - old_entry->vme_start), - ((old_entry->is_shared + ((old_entry->is_shared || old_map->mapped_in_other_pmaps) ? PMAP_NULL : old_map->pmap), @@ -10833,12 +11870,12 @@ vm_map_fork( old_entry->needs_copy = TRUE; } new_entry->needs_copy = new_entry_needs_copy; - + /* * Insert the entry at the end * of the map. */ - + vm_map_store_entry_link(new_map, vm_map_last_entry(new_map), new_entry); new_size += entry_size; @@ -10861,8 +11898,12 @@ vm_map_fork( old_entry = old_entry->vme_next; } +#if defined(__arm64__) + pmap_insert_sharedpage(new_map->pmap); +#endif new_map->size = new_size; + vm_map_unlock(new_map); vm_map_unlock(old_map); vm_map_deallocate(old_map); @@ -10985,7 +12026,7 @@ RetryLookup: if (!vm_map_lookup_entry(map, vaddr, &tmp_entry)) { if((cow_sub_map_parent) && (cow_sub_map_parent != map)) vm_map_unlock(cow_sub_map_parent); - if((*real_map != map) + if((*real_map != map) && (*real_map != cow_sub_map_parent)) vm_map_unlock(*real_map); return KERN_INVALID_ADDRESS; @@ -11007,8 +12048,10 @@ submap_recurse: if (entry->is_sub_map) { vm_map_offset_t local_vaddr; vm_map_offset_t end_delta; - vm_map_offset_t start_delta; + vm_map_offset_t start_delta; vm_map_entry_t submap_entry; + vm_prot_t subentry_protection; + vm_prot_t subentry_max_protection; boolean_t mapped_needs_copy=FALSE; local_vaddr = vaddr; @@ -11017,7 +12060,7 @@ submap_recurse: ! ((fault_type & VM_PROT_WRITE) || force_copy))) { /* if real_map equals map we unlock below */ - if ((*real_map != map) && + if ((*real_map != map) && (*real_map != cow_sub_map_parent)) vm_map_unlock(*real_map); *real_map = VME_SUBMAP(entry); @@ -11051,7 +12094,7 @@ submap_recurse: } } else { vm_map_lock_read(VME_SUBMAP(entry)); - *var_map = VME_SUBMAP(entry); + *var_map = VME_SUBMAP(entry); /* leave map locked if it is a target */ /* cow sub_map above otherwise, just */ /* follow the maps down to the object */ @@ -11071,7 +12114,7 @@ submap_recurse: if((cow_sub_map_parent) && (cow_sub_map_parent != map)){ vm_map_unlock(cow_sub_map_parent); } - if((*real_map != map) + if((*real_map != map) && (*real_map != cow_sub_map_parent)) { vm_map_unlock(*real_map); } @@ -11092,12 +12135,12 @@ submap_recurse: start_delta = submap_entry->vme_start > VME_OFFSET(entry) ? submap_entry->vme_start - VME_OFFSET(entry) : 0; - end_delta = + end_delta = (VME_OFFSET(entry) + start_delta + (old_end - old_start)) <= submap_entry->vme_end ? - 0 : (VME_OFFSET(entry) + + 0 : (VME_OFFSET(entry) + (old_end - old_start)) - - submap_entry->vme_end; + - submap_entry->vme_end; old_start += start_delta; old_end -= end_delta; @@ -11136,9 +12179,9 @@ submap_recurse: VME_OBJECT_SET(submap_entry, sub_object); VME_OFFSET_SET(submap_entry, 0); } - local_start = local_vaddr - + local_start = local_vaddr - (cow_parent_vaddr - old_start); - local_end = local_vaddr + + local_end = local_vaddr + (old_end - cow_parent_vaddr); vm_map_clip_start(map, submap_entry, local_start); vm_map_clip_end(map, submap_entry, local_end); @@ -11165,7 +12208,7 @@ submap_recurse: ©_object); copied_slowly = TRUE; } else { - + /* set up shadow object */ copy_object = sub_object; vm_object_lock(sub_object); @@ -11176,7 +12219,10 @@ submap_recurse: assert(submap_entry->wired_count == 0); submap_entry->needs_copy = TRUE; - prot = submap_entry->protection & ~VM_PROT_WRITE; + prot = submap_entry->protection; + assert(!pmap_has_prot_policy(prot)); + prot = prot & ~VM_PROT_WRITE; + assert(!pmap_has_prot_policy(prot)); if (override_nx(old_map, VME_ALIAS(submap_entry)) @@ -11186,15 +12232,15 @@ submap_recurse: vm_object_pmap_protect( sub_object, VME_OFFSET(submap_entry), - submap_entry->vme_end - + submap_entry->vme_end - submap_entry->vme_start, - (submap_entry->is_shared + (submap_entry->is_shared || map->mapped_in_other_pmaps) ? PMAP_NULL : map->pmap, submap_entry->vme_start, prot); } - + /* * Adjust the fault offset to the submap entry. */ @@ -11209,7 +12255,11 @@ submap_recurse: /* the sub_map, substituting the */ /* new copy object, */ + subentry_protection = submap_entry->protection; + subentry_max_protection = submap_entry->max_protection; vm_map_unlock(map); + submap_entry = NULL; /* not valid after map unlock */ + local_start = old_start; local_end = old_end; map = cow_sub_map_parent; @@ -11217,14 +12267,14 @@ submap_recurse: vaddr = cow_parent_vaddr; cow_sub_map_parent = NULL; - if(!vm_map_lookup_entry(map, + if(!vm_map_lookup_entry(map, vaddr, &entry)) { vm_object_deallocate( copy_object); vm_map_lock_write_to_read(map); return KERN_INVALID_ADDRESS; } - + /* clip out the portion of space */ /* mapped by the sub map which */ /* corresponds to the underlying */ @@ -11268,8 +12318,17 @@ submap_recurse: VME_OBJECT_SET(entry, copy_object); /* propagate the submap entry's protections */ - entry->protection |= submap_entry->protection; - entry->max_protection |= submap_entry->max_protection; + entry->protection |= subentry_protection; + entry->max_protection |= subentry_max_protection; + +#if CONFIG_EMBEDDED + if (entry->protection & VM_PROT_WRITE) { + if ((entry->protection & VM_PROT_EXECUTE) && !(entry->used_for_jit)) { + printf("EMBEDDED: %s can't have both write and exec at the same time\n", __FUNCTION__); + entry->protection &= ~VM_PROT_EXECUTE; + } + } +#endif if(copied_slowly) { VME_OFFSET_SET(entry, local_start - old_start); @@ -11279,12 +12338,12 @@ submap_recurse: VME_OFFSET_SET(entry, copy_offset); assert(entry->wired_count == 0); entry->needs_copy = TRUE; - if(entry->inheritance == VM_INHERIT_SHARE) + if(entry->inheritance == VM_INHERIT_SHARE) entry->inheritance = VM_INHERIT_COPY; if (map != old_map) entry->is_shared = TRUE; } - if(entry->inheritance == VM_INHERIT_SHARE) + if(entry->inheritance == VM_INHERIT_SHARE) entry->inheritance = VM_INHERIT_COPY; vm_map_lock_write_to_read(map); @@ -11298,7 +12357,7 @@ submap_recurse: vaddr = local_vaddr; } } - + /* * Check whether this task is allowed to have * this page. @@ -11320,6 +12379,10 @@ submap_recurse: } } if (((fault_type & prot) != fault_type) +#if __arm64__ + /* prefetch abort in execute-only page */ + && !(prot == VM_PROT_EXECUTE && fault_type == (VM_PROT_READ | VM_PROT_EXECUTE)) +#endif ) { protection_failure: if (*real_map != map) { @@ -11453,7 +12516,7 @@ submap_recurse: vm_object_lock(*object); else vm_object_lock_shared(*object); - + /* * Save the version number */ @@ -11468,8 +12531,9 @@ submap_recurse: * vm_map_verify: * * Verifies that the map in question has not changed - * since the given version. If successful, the map - * will not change until vm_map_verify_done() is called. + * since the given version. The map has to be locked + * ("shared" mode is fine) before calling this function + * and it will be returned locked too. */ boolean_t vm_map_verify( @@ -11478,25 +12542,12 @@ vm_map_verify( { boolean_t result; - vm_map_lock_read(map); + vm_map_lock_assert_held(map); result = (map->timestamp == version->main_timestamp); - if (!result) - vm_map_unlock_read(map); - return(result); } -/* - * vm_map_verify_done: - * - * Releases locks acquired by a vm_map_verify. - * - * This is now a macro in vm/vm_map.h. It does a - * vm_map_unlock_read on the map. - */ - - /* * TEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARY * Goes away after regular vm_region_recurse function migrates to @@ -11529,13 +12580,13 @@ vm_map_region_recurse_64( * "curr_entry" is the VM map entry preceding or including the * address we're looking for. * "curr_map" is the map or sub-map containing "curr_entry". - * "curr_address" is the equivalent of the top map's "user_address" + * "curr_address" is the equivalent of the top map's "user_address" * in the current map. * "curr_offset" is the cumulated offset of "curr_map" in the * target task's address space. * "curr_depth" is the depth of "curr_map" in the chain of * sub-maps. - * + * * "curr_max_below" and "curr_max_above" limit the range (around * "curr_address") we should take into account in the current (sub)map. * They limit the range to what's visible through the map entries @@ -11572,7 +12623,7 @@ vm_map_region_recurse_64( return KERN_INVALID_ARGUMENT; } - + if (*count < VM_REGION_SUBMAP_SHORT_INFO_COUNT_64) { /* * "info" structure is not big enough and @@ -11580,9 +12631,9 @@ vm_map_region_recurse_64( */ return KERN_INVALID_ARGUMENT; } - + original_count = *count; - + if (original_count < VM_REGION_SUBMAP_INFO_V0_COUNT_64) { *count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64; look_for_pages = FALSE; @@ -11592,15 +12643,15 @@ vm_map_region_recurse_64( look_for_pages = TRUE; *count = VM_REGION_SUBMAP_INFO_V0_COUNT_64; short_info = NULL; - + if (original_count >= VM_REGION_SUBMAP_INFO_V1_COUNT_64) { *count = VM_REGION_SUBMAP_INFO_V1_COUNT_64; } } - + user_address = *address; user_max_depth = *nesting_depth; - + if (not_in_kdp) { vm_map_lock_read(map); } @@ -11876,7 +12927,7 @@ recurse_again: if (look_for_pages) { submap_info->user_tag = VME_ALIAS(curr_entry); - submap_info->offset = VME_OFFSET(curr_entry); + submap_info->offset = VME_OFFSET(curr_entry); submap_info->protection = curr_entry->protection; submap_info->inheritance = curr_entry->inheritance; submap_info->max_protection = curr_entry->max_protection; @@ -11886,7 +12937,7 @@ recurse_again: submap_info->object_id = INFO_MAKE_OBJECT_ID(VME_OBJECT(curr_entry)); } else { short_info->user_tag = VME_ALIAS(curr_entry); - short_info->offset = VME_OFFSET(curr_entry); + short_info->offset = VME_OFFSET(curr_entry); short_info->protection = curr_entry->protection; short_info->inheritance = curr_entry->inheritance; short_info->max_protection = curr_entry->max_protection; @@ -11947,7 +12998,7 @@ recurse_again: submap_info->shadow_depth = extended.shadow_depth; submap_info->share_mode = extended.share_mode; submap_info->ref_count = extended.ref_count; - + if (original_count >= VM_REGION_SUBMAP_INFO_V1_COUNT_64) { submap_info->pages_reusable = extended.pages_reusable; } @@ -11991,7 +13042,7 @@ vm_map_region( vm_map_entry_t entry; vm_map_offset_t start; - if (map == VM_MAP_NULL) + if (map == VM_MAP_NULL) return(KERN_INVALID_ARGUMENT); switch (flavor) { @@ -12144,7 +13195,7 @@ vm_map_region( return(KERN_SUCCESS); } case VM_REGION_TOP_INFO: - { + { vm_region_top_info_t top; if (*count < VM_REGION_TOP_INFO_COUNT) @@ -12228,7 +13279,7 @@ vm_map_region_top_walk( OBJ_RESIDENT_COUNT(obj, entry_size); top->ref_count = ref_count; top->share_mode = SM_COW; - + while ((tmp_obj = obj->shadow)) { vm_object_lock(tmp_obj); vm_object_unlock(obj); @@ -12410,7 +13461,7 @@ vm_map_region_walk( } } extended->ref_count = ref_count - extended->shadow_depth; - + for (i = 0; i < extended->shadow_depth; i++) { if ((tmp_obj = obj->shadow) == 0) break; @@ -12469,7 +13520,7 @@ vm_map_region_look_for_page( shadow = object->shadow; caller_object = object; - + while (TRUE) { if ( !(object->pager_trusted) && !(object->internal)) @@ -12521,7 +13572,7 @@ vm_map_region_look_for_page( if (ref_count > max_refcnt) max_refcnt = ref_count; - + if(object != caller_object) vm_object_unlock(object); @@ -12709,7 +13760,7 @@ vm_map_simplify_range( entry = entry->vme_next; } } - + while (entry != vm_map_to_entry(map) && entry->vme_start <= end) { /* try and coalesce "entry" with its previous entry */ @@ -12744,7 +13795,7 @@ vm_map_machine_attribute( kern_return_t ret; vm_map_size_t sync_size; vm_map_entry_t entry; - + if (start < vm_map_min(map) || end > vm_map_max(map)) return KERN_INVALID_ADDRESS; @@ -12752,8 +13803,8 @@ vm_map_machine_attribute( sync_size = end - start; vm_map_lock(map); - - if (attribute != MATTR_CACHE) { + + if (attribute != MATTR_CACHE) { /* If we don't have to find physical addresses, we */ /* don't have to do an explicit traversal here. */ ret = pmap_attribute(map->pmap, start, end-start, @@ -12778,11 +13829,11 @@ vm_map_machine_attribute( vm_map_offset_t sub_start; vm_map_offset_t sub_end; - sub_start = (start - entry->vme_start) + sub_start = (start - entry->vme_start) + VME_OFFSET(entry); sub_end = sub_start + sub_size; vm_map_machine_attribute( - VME_SUBMAP(entry), + VME_SUBMAP(entry), sub_start, sub_end, attribute, value); @@ -12810,12 +13861,12 @@ vm_map_machine_attribute( object, offset); if (m && !m->fictitious) { - ret = + ret = pmap_attribute_cache_sync( - VM_PAGE_GET_PHYS_PAGE(m), - PAGE_SIZE, + VM_PAGE_GET_PHYS_PAGE(m), + PAGE_SIZE, attribute, value); - + } else if (object->shadow) { offset = offset + object->vo_shadow_offset; last_object = object; @@ -12843,7 +13894,7 @@ vm_map_machine_attribute( vm_map_unlock(map); return KERN_FAILURE; } - + } vm_map_unlock(map); @@ -12856,10 +13907,10 @@ vm_map_machine_attribute( * * Sets the paging reference behavior of the specified address * range in the target map. Paging reference behavior affects - * how pagein operations resulting from faults on the map will be + * how pagein operations resulting from faults on the map will be * clustered. */ -kern_return_t +kern_return_t vm_map_behavior_set( vm_map_t map, vm_map_offset_t start, @@ -12893,10 +13944,10 @@ vm_map_behavior_set( case VM_BEHAVIOR_RSEQNTL: case VM_BEHAVIOR_ZERO_WIRED_PAGES: vm_map_lock(map); - + /* * The entire address range must be valid for the map. - * Note that vm_map_range_check() does a + * Note that vm_map_range_check() does a * vm_map_lookup_entry() internally and returns the * entry containing the start of the address range if * the entire range is valid. @@ -12909,13 +13960,13 @@ vm_map_behavior_set( vm_map_unlock(map); return(KERN_INVALID_ADDRESS); } - + while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) { vm_map_clip_end(map, entry, end); if (entry->is_sub_map) { assert(!entry->use_pmap); } - + if( new_behavior == VM_BEHAVIOR_ZERO_WIRED_PAGES ) { entry->zero_wired_pages = TRUE; } else { @@ -12923,13 +13974,13 @@ vm_map_behavior_set( } entry = entry->vme_next; } - + vm_map_unlock(map); break; /* * The rest of these are different from the above in that they cause - * an immediate action to take place as opposed to setting a behavior that + * an immediate action to take place as opposed to setting a behavior that * affects future actions. */ @@ -13025,7 +14076,7 @@ vm_map_willneed( * Examine each vm_map_entry_t in the range. */ for (; entry != vm_map_to_entry(map) && start < end; ) { - + /* * The first time through, the start address could be anywhere * within the vm_map_entry we found. So adjust the offset to @@ -13051,7 +14102,7 @@ vm_map_willneed( len = (vm_size_t) (0 - PAGE_SIZE); } fault_info.cluster_size = (vm_size_t) len; - fault_info.lo_offset = offset; + fault_info.lo_offset = offset; fault_info.hi_offset = offset + len; fault_info.user_tag = VME_ALIAS(entry); fault_info.pmap_options = 0; @@ -13207,7 +14258,7 @@ vm_map_entry_is_reusable( * shared. * We check for this later when processing the actual * VM pages, so the contents will be safe if shared. - * + * * But we can still mark this memory region as "reusable" to * acknowledge that the caller did let us know that the memory * could be re-used and should not be penalized for holding @@ -13227,8 +14278,8 @@ vm_map_entry_is_reusable( return TRUE; } return FALSE; - - + + } static kern_return_t @@ -13310,7 +14361,7 @@ vm_map_reuse_pages( VME_ALIAS_SET(entry, VM_MEMORY_MALLOC_LARGE_REUSED); } } - + vm_map_unlock_read(map); vm_page_stats_reusable.reuse_pages_success++; return KERN_SUCCESS; @@ -13442,7 +14493,7 @@ vm_map_reusable_pages( VME_ALIAS_SET(entry, VM_MEMORY_MALLOC_LARGE_REUSABLE); } } - + vm_map_unlock_read(map); vm_page_stats_reusable.reusable_pages_success++; return KERN_SUCCESS; @@ -13491,7 +14542,7 @@ vm_map_can_reuse( return KERN_INVALID_ADDRESS; } } - + vm_map_unlock_read(map); vm_page_stats_reusable.can_reuse_success++; return KERN_SUCCESS; @@ -13543,7 +14594,7 @@ vm_map_pageout( submap = VME_SUBMAP(entry); submap_start = VME_OFFSET(entry); - submap_end = submap_start + (entry->vme_end - + submap_end = submap_start + (entry->vme_end - entry->vme_start); vm_map_lock_read(submap); @@ -13582,7 +14633,7 @@ vm_map_pageout( vm_object_pageout(object); } - + vm_map_unlock_read(map); return KERN_SUCCESS; } @@ -13614,7 +14665,9 @@ vm_map_entry_insert( boolean_t permanent, unsigned int superpage_size, boolean_t clear_map_aligned, - boolean_t is_submap) + boolean_t is_submap, + boolean_t used_for_jit, + int alias) { vm_map_entry_t new_entry; @@ -13671,7 +14724,7 @@ vm_map_entry_insert( */ new_entry->use_pmap = TRUE; } - VME_ALIAS_SET(new_entry, 0); + VME_ALIAS_SET(new_entry, alias); new_entry->zero_wired_pages = FALSE; new_entry->no_cache = no_cache; new_entry->permanent = permanent; @@ -13679,7 +14732,17 @@ vm_map_entry_insert( new_entry->superpage_size = TRUE; else new_entry->superpage_size = FALSE; - new_entry->used_for_jit = FALSE; + if (used_for_jit){ + if (!(map->jit_entry_exists)){ + new_entry->used_for_jit = TRUE; + map->jit_entry_exists = TRUE; + + /* Tell the pmap that it supports JIT. */ + pmap_set_jit_entitled(map->pmap); + } + } else { + new_entry->used_for_jit = FALSE; + } new_entry->iokit_acct = FALSE; new_entry->vme_resilient_codesign = FALSE; new_entry->vme_resilient_media = FALSE; @@ -13728,7 +14791,7 @@ vm_map_remap_extract( vm_map_offset_t map_address; vm_map_offset_t src_start; /* start of entry to map */ vm_map_offset_t src_end; /* end of region to be mapped */ - vm_object_t object; + vm_object_t object; vm_map_version_t version; boolean_t src_needs_copy; boolean_t new_entry_needs_copy; @@ -13765,7 +14828,7 @@ vm_map_remap_extract( mapped_size = 0; result = KERN_SUCCESS; - /* + /* * The specified source virtual space might correspond to * multiple map entries, need to loop on them. */ @@ -13775,7 +14838,7 @@ vm_map_remap_extract( /* * Find the beginning of the region. - */ + */ if (! vm_map_lookup_entry(map, src_start, &src_entry)) { result = KERN_INVALID_ADDRESS; break; @@ -13841,6 +14904,8 @@ vm_map_remap_extract( (src_entry->protection & VM_PROT_WRITE)) { vm_prot_t prot; + assert(!pmap_has_prot_policy(src_entry->protection)); + prot = src_entry->protection & ~VM_PROT_WRITE; if (override_nx(map, @@ -13848,6 +14913,8 @@ vm_map_remap_extract( && prot) prot |= VM_PROT_EXECUTE; + assert(!pmap_has_prot_policy(prot)); + if(map->mapped_in_other_pmaps) { vm_object_pmap_protect( VME_OBJECT(src_entry), @@ -13871,9 +14938,9 @@ vm_map_remap_extract( vm_object_lock(object); vm_object_reference_locked(object); /* object ref. for new entry */ - if (object->copy_strategy == + if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) { - object->copy_strategy = + object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; } vm_object_unlock(object); @@ -13896,7 +14963,7 @@ vm_map_remap_extract( assert(new_entry->vme_start < new_entry->vme_end); new_entry->inheritance = inheritance; VME_OFFSET_SET(new_entry, offset); - + /* * The new region has to be copied now if required. */ @@ -13912,7 +14979,7 @@ vm_map_remap_extract( } src_entry->is_shared = TRUE; new_entry->is_shared = TRUE; - if (!(new_entry->is_sub_map)) + if (!(new_entry->is_sub_map)) new_entry->needs_copy = FALSE; } else if (src_entry->is_sub_map) { @@ -13937,6 +15004,8 @@ vm_map_remap_extract( if (src_needs_copy && !src_entry->needs_copy) { vm_prot_t prot; + assert(!pmap_has_prot_policy(src_entry->protection)); + prot = src_entry->protection & ~VM_PROT_WRITE; if (override_nx(map, @@ -13944,10 +15013,12 @@ vm_map_remap_extract( && prot) prot |= VM_PROT_EXECUTE; + assert(!pmap_has_prot_policy(prot)); + vm_object_pmap_protect(object, offset, entry_size, - ((src_entry->is_shared + ((src_entry->is_shared || map->mapped_in_other_pmaps) ? PMAP_NULL : map->pmap), src_entry->vme_start, @@ -13982,7 +15053,8 @@ vm_map_remap_extract( result = vm_object_copy_slowly( object, offset, - entry_size, + (new_entry->vme_end - + new_entry->vme_start), THREAD_UNINT, &VME_OBJECT(new_entry)); @@ -13995,7 +15067,8 @@ vm_map_remap_extract( result = vm_object_copy_strategically( object, offset, - entry_size, + (new_entry->vme_end - + new_entry->vme_start), &VME_OBJECT(new_entry), &new_offset, &new_entry_needs_copy); @@ -14097,6 +15170,8 @@ vm_map_remap( vm_map_size_t size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, vm_map_t src_map, vm_map_offset_t memory_address, boolean_t copy, @@ -14125,24 +15200,24 @@ vm_map_remap( return KERN_INVALID_ARGUMENT; } - /* - * If the user is requesting that we return the address of the - * first byte of the data (rather than the base of the page), - * then we use different rounding semantics: specifically, + /* + * If the user is requesting that we return the address of the + * first byte of the data (rather than the base of the page), + * then we use different rounding semantics: specifically, * we assume that (memory_address, size) describes a region * all of whose pages we must cover, rather than a base to be truncated * down and a size to be added to that base. So we figure out * the highest page that the requested region includes and make * sure that the size will cover it. - * + * * The key example we're worried about it is of the form: * * memory_address = 0x1ff0, size = 0x20 - * - * With the old semantics, we round down the memory_address to 0x1000 + * + * With the old semantics, we round down the memory_address to 0x1000 * and round up the size to 0x1000, resulting in our covering *only* * page 0x1000. With the new semantics, we'd realize that the region covers - * 0x1ff0-0x2010, and compute a size of 0x2000. Thus, we cover both page + * 0x1ff0-0x2010, and compute a size of 0x2000. Thus, we cover both page * 0x1000 and page 0x2000 in the region we remap. */ if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) { @@ -14150,7 +15225,10 @@ vm_map_remap( size = vm_map_round_page(memory_address + size - vm_map_trunc_page(memory_address, PAGE_MASK), PAGE_MASK); } else { size = vm_map_round_page(size, PAGE_MASK); - } + } + if (size == 0) { + return KERN_INVALID_ARGUMENT; + } result = vm_map_remap_extract(src_map, memory_address, size, copy, &map_header, @@ -14172,7 +15250,8 @@ vm_map_remap( VM_MAP_PAGE_MASK(target_map)); vm_map_lock(target_map); result = vm_map_remap_range_allocate(target_map, address, size, - mask, flags, &insp_entry); + mask, flags, vmk_flags, tag, + &insp_entry); for (entry = map_header.links.next; entry != (struct vm_map_entry *)&map_header.links; @@ -14217,16 +15296,17 @@ vm_map_remap( if (result == KERN_SUCCESS) { target_map->size += size; SAVE_HINT_MAP_WRITE(target_map, insp_entry); + } vm_map_unlock(target_map); if (result == KERN_SUCCESS && target_map->wiring_required) - result = vm_map_wire(target_map, *address, - *address + size, *cur_protection | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK), + result = vm_map_wire_kernel(target_map, *address, + *address + size, *cur_protection, VM_KERN_MEMORY_MLOCK, TRUE); - /* - * If requested, return the address of the data pointed to by the + /* + * If requested, return the address of the data pointed to by the * request, rather than the base of the resulting page. */ if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) { @@ -14254,6 +15334,8 @@ vm_map_remap_range_allocate( vm_map_size_t size, vm_map_offset_t mask, int flags, + __unused vm_map_kernel_flags_t vmk_flags, + __unused vm_tag_t tag, vm_map_entry_t *map_entry) /* OUT */ { vm_map_entry_t entry; @@ -14288,7 +15370,7 @@ StartAgain: ; start = map->min_offset; if (start > map->max_offset) return(KERN_NO_SPACE); - + /* * Look for the first possible address; * if there's already something at this @@ -14347,7 +15429,7 @@ StartAgain: ; start = vm_map_round_page(start, VM_MAP_PAGE_MASK(map)); } - + /* * In any case, the "entry" always precedes * the proposed new region throughout the @@ -14382,7 +15464,7 @@ StartAgain: ; goto StartAgain; } } - + return(KERN_NO_SPACE); } @@ -14438,7 +15520,7 @@ StartAgain: ; } else { vm_map_entry_t temp_entry; - + /* * Verify that: * the address doesn't itself violate @@ -14712,7 +15794,8 @@ vm_map_purgable_control( if (control != VM_PURGABLE_SET_STATE && control != VM_PURGABLE_GET_STATE && - control != VM_PURGABLE_PURGE_ALL) + control != VM_PURGABLE_PURGE_ALL && + control != VM_PURGABLE_SET_STATE_FROM_KERNEL) return(KERN_INVALID_ARGUMENT); if (control == VM_PURGABLE_PURGE_ALL) { @@ -14720,7 +15803,8 @@ vm_map_purgable_control( return KERN_SUCCESS; } - if (control == VM_PURGABLE_SET_STATE && + if ((control == VM_PURGABLE_SET_STATE || + control == VM_PURGABLE_SET_STATE_FROM_KERNEL) && (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) || ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK))) return(KERN_INVALID_ARGUMENT); @@ -14753,11 +15837,11 @@ vm_map_purgable_control( vm_map_unlock_read(map); return KERN_INVALID_ARGUMENT; } - + vm_object_lock(object); #if 00 - if (VME_OFFSET(entry) != 0 || + if (VME_OFFSET(entry) != 0 || entry->vme_end - entry->vme_start != object->vo_size) { /* * Can only apply purgable controls to the whole (existing) @@ -14818,25 +15902,42 @@ vm_map_page_query_internal( return kr; } - + kern_return_t vm_map_page_info( vm_map_t map, - vm_map_offset_t offset, + vm_map_offset_t offset, + vm_page_info_flavor_t flavor, + vm_page_info_t info, + mach_msg_type_number_t *count) +{ + return (vm_map_page_range_info_internal(map, + offset, /* start of range */ + (offset + 1), /* this will get rounded in the call to the page boundary */ + flavor, + info, + count)); +} + +kern_return_t +vm_map_page_range_info_internal( + vm_map_t map, + vm_map_offset_t start_offset, + vm_map_offset_t end_offset, vm_page_info_flavor_t flavor, vm_page_info_t info, mach_msg_type_number_t *count) { - vm_map_entry_t map_entry; - vm_object_t object; - vm_page_t m; + vm_map_entry_t map_entry = VM_MAP_ENTRY_NULL; + vm_object_t object = VM_OBJECT_NULL, curr_object = VM_OBJECT_NULL; + vm_page_t m = VM_PAGE_NULL; kern_return_t retval = KERN_SUCCESS; - boolean_t top_object; - int disposition; - int ref_count; - vm_page_info_basic_t basic_info; - int depth; - vm_map_offset_t offset_in_page; + int disposition = 0; + int ref_count = 0; + int depth = 0, info_idx = 0; + vm_page_info_basic_t basic_info = 0; + vm_map_offset_t offset_in_page = 0, offset_in_object = 0, curr_offset_in_object = 0; + vm_map_offset_t start = 0, end = 0, curr_s_offset = 0, curr_e_offset = 0; switch (flavor) { case VM_PAGE_INFO_BASIC: @@ -14856,157 +15957,312 @@ vm_map_page_info( disposition = 0; ref_count = 0; - top_object = TRUE; depth = 0; - + info_idx = 0; /* Tracks the next index within the info structure to be filled.*/ retval = KERN_SUCCESS; - offset_in_page = offset & PAGE_MASK; - offset = vm_map_trunc_page(offset, PAGE_MASK); + + offset_in_page = start_offset & PAGE_MASK; + start = vm_map_trunc_page(start_offset, PAGE_MASK); + end = vm_map_round_page(end_offset, PAGE_MASK); + + assert ((end - start) <= MAX_PAGE_RANGE_QUERY); vm_map_lock_read(map); - /* - * First, find the map entry covering "offset", going down - * submaps if necessary. - */ - for (;;) { - if (!vm_map_lookup_entry(map, offset, &map_entry)) { - vm_map_unlock_read(map); - return KERN_INVALID_ADDRESS; + for (curr_s_offset = start; curr_s_offset < end;) { + /* + * New lookup needs reset of these variables. + */ + curr_object = object = VM_OBJECT_NULL; + offset_in_object = 0; + ref_count = 0; + depth = 0; + + /* + * First, find the map entry covering "curr_s_offset", going down + * submaps if necessary. + */ + if (!vm_map_lookup_entry(map, curr_s_offset, &map_entry)) { + /* no entry -> no object -> no page */ + + if (curr_s_offset < vm_map_min(map)) { + /* + * Illegal address that falls below map min. + */ + curr_e_offset = MIN(end, vm_map_min(map)); + + } else if (curr_s_offset >= vm_map_max(map)) { + /* + * Illegal address that falls on/after map max. + */ + curr_e_offset = end; + + } else if (map_entry == vm_map_to_entry(map)) { + /* + * Hit a hole. + */ + if (map_entry->vme_next == vm_map_to_entry(map)) { + /* + * Empty map. + */ + curr_e_offset = MIN(map->max_offset, end); + } else { + /* + * Hole at start of the map. + */ + curr_e_offset = MIN(map_entry->vme_next->vme_start, end); + } + } else { + if (map_entry->vme_next == vm_map_to_entry(map)) { + /* + * Hole at the end of the map. + */ + curr_e_offset = MIN(map->max_offset, end); + } else { + curr_e_offset = MIN(map_entry->vme_next->vme_start, end); + } + } + + assert(curr_e_offset >= curr_s_offset); + + uint64_t num_pages = (curr_e_offset - curr_s_offset) >> PAGE_SHIFT; + + void *info_ptr = (void*) (((uintptr_t) info) + (info_idx * sizeof(struct vm_page_info_basic))); + + bzero(info_ptr, num_pages * sizeof(struct vm_page_info_basic)); + + curr_s_offset = curr_e_offset; + + info_idx += num_pages; + + continue; } + /* compute offset from this map entry's start */ - offset -= map_entry->vme_start; + offset_in_object = curr_s_offset - map_entry->vme_start; + /* compute offset into this map entry's object (or submap) */ - offset += VME_OFFSET(map_entry); + offset_in_object += VME_OFFSET(map_entry); if (map_entry->is_sub_map) { - vm_map_t sub_map; + vm_map_t sub_map = VM_MAP_NULL; + vm_page_info_t submap_info = 0; + vm_map_offset_t submap_s_offset = 0, submap_e_offset = 0, range_len = 0; + + range_len = MIN(map_entry->vme_end, end) - curr_s_offset; + + submap_s_offset = offset_in_object; + submap_e_offset = submap_s_offset + range_len; sub_map = VME_SUBMAP(map_entry); - vm_map_lock_read(sub_map); + + vm_map_reference(sub_map); vm_map_unlock_read(map); - map = sub_map; + submap_info = (vm_page_info_t) (((uintptr_t) info) + (info_idx * sizeof(struct vm_page_info_basic))); + + retval = vm_map_page_range_info_internal(sub_map, + submap_s_offset, + submap_e_offset, + VM_PAGE_INFO_BASIC, + (vm_page_info_t) submap_info, + count); + + assert(retval == KERN_SUCCESS); + + vm_map_lock_read(map); + vm_map_deallocate(sub_map); + + /* Move the "info" index by the number of pages we inspected.*/ + info_idx += range_len >> PAGE_SHIFT; + + /* Move our current offset by the size of the range we inspected.*/ + curr_s_offset += range_len; - ref_count = MAX(ref_count, map->ref_count); continue; } - break; - } - object = VME_OBJECT(map_entry); - if (object == VM_OBJECT_NULL) { - /* no object -> no page */ + object = VME_OBJECT(map_entry); + if (object == VM_OBJECT_NULL) { + + /* + * We don't have an object here and, hence, + * no pages to inspect. We'll fill up the + * info structure appropriately. + */ + + curr_e_offset = MIN(map_entry->vme_end, end); + + uint64_t num_pages = (curr_e_offset - curr_s_offset) >> PAGE_SHIFT; + + void *info_ptr = (void*) (((uintptr_t) info) + (info_idx * sizeof(struct vm_page_info_basic))); + + bzero(info_ptr, num_pages * sizeof(struct vm_page_info_basic)); + + curr_s_offset = curr_e_offset; + + info_idx += num_pages; + + continue; + } + + vm_object_reference(object); + /* + * Shared mode -- so we can allow other readers + * to grab the lock too. + */ + vm_object_lock_shared(object); + + curr_e_offset = MIN(map_entry->vme_end, end); + vm_map_unlock_read(map); - goto done; - } - vm_object_lock(object); - vm_map_unlock_read(map); + map_entry = NULL; /* map is unlocked, the entry is no longer valid. */ - /* - * Go down the VM object shadow chain until we find the page - * we're looking for. - */ - for (;;) { - ref_count = MAX(ref_count, object->ref_count); + curr_object = object; - m = vm_page_lookup(object, offset); + for (; curr_s_offset < curr_e_offset;) { - if (m != VM_PAGE_NULL) { - disposition |= VM_PAGE_QUERY_PAGE_PRESENT; - break; - } else { - if (object->internal && - object->alive && - !object->terminating && - object->pager_ready) { - - if (VM_COMPRESSOR_PAGER_STATE_GET(object, offset) - == VM_EXTERNAL_STATE_EXISTS) { - /* the pager has that page */ - disposition |= VM_PAGE_QUERY_PAGE_PAGED_OUT; + if (object == curr_object) { + ref_count = curr_object->ref_count - 1; /* account for our object reference above. */ + } else { + ref_count = curr_object->ref_count; + } + + curr_offset_in_object = offset_in_object; + + for (;;) { + m = vm_page_lookup(curr_object, curr_offset_in_object); + + if (m != VM_PAGE_NULL) { + + disposition |= VM_PAGE_QUERY_PAGE_PRESENT; break; + + } else { + if (curr_object->internal && + curr_object->alive && + !curr_object->terminating && + curr_object->pager_ready) { + + if (VM_COMPRESSOR_PAGER_STATE_GET(curr_object, curr_offset_in_object) + == VM_EXTERNAL_STATE_EXISTS) { + /* the pager has that page */ + disposition |= VM_PAGE_QUERY_PAGE_PAGED_OUT; + break; + } + } + + /* + * Go down the VM object shadow chain until we find the page + * we're looking for. + */ + + if (curr_object->shadow != VM_OBJECT_NULL) { + vm_object_t shadow = VM_OBJECT_NULL; + + curr_offset_in_object += curr_object->vo_shadow_offset; + shadow = curr_object->shadow; + + vm_object_lock_shared(shadow); + vm_object_unlock(curr_object); + + curr_object = shadow; + depth++; + continue; + } else { + + break; + } } } - if (object->shadow != VM_OBJECT_NULL) { - vm_object_t shadow; + /* The ref_count is not strictly accurate, it measures the number */ + /* of entities holding a ref on the object, they may not be mapping */ + /* the object or may not be mapping the section holding the */ + /* target page but its still a ball park number and though an over- */ + /* count, it picks up the copy-on-write cases */ - offset += object->vo_shadow_offset; - shadow = object->shadow; - - vm_object_lock(shadow); - vm_object_unlock(object); + /* We could also get a picture of page sharing from pmap_attributes */ + /* but this would under count as only faulted-in mappings would */ + /* show up. */ - object = shadow; - top_object = FALSE; - depth++; - } else { -// if (!object->internal) -// break; -// retval = KERN_FAILURE; -// goto done_with_object; - break; + if ((curr_object == object) && curr_object->shadow) + disposition |= VM_PAGE_QUERY_PAGE_COPIED; + + if (! curr_object->internal) + disposition |= VM_PAGE_QUERY_PAGE_EXTERNAL; + + if (m != VM_PAGE_NULL) { + + if (m->fictitious) { + + disposition |= VM_PAGE_QUERY_PAGE_FICTITIOUS; + + } else { + if (m->dirty || pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m))) + disposition |= VM_PAGE_QUERY_PAGE_DIRTY; + + if (m->reference || pmap_is_referenced(VM_PAGE_GET_PHYS_PAGE(m))) + disposition |= VM_PAGE_QUERY_PAGE_REF; + + if (m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) + disposition |= VM_PAGE_QUERY_PAGE_SPECULATIVE; + + if (m->cs_validated) + disposition |= VM_PAGE_QUERY_PAGE_CS_VALIDATED; + if (m->cs_tainted) + disposition |= VM_PAGE_QUERY_PAGE_CS_TAINTED; + if (m->cs_nx) + disposition |= VM_PAGE_QUERY_PAGE_CS_NX; + } } - } - } - /* The ref_count is not strictly accurate, it measures the number */ - /* of entities holding a ref on the object, they may not be mapping */ - /* the object or may not be mapping the section holding the */ - /* target page but its still a ball park number and though an over- */ - /* count, it picks up the copy-on-write cases */ - /* We could also get a picture of page sharing from pmap_attributes */ - /* but this would under count as only faulted-in mappings would */ - /* show up. */ + switch (flavor) { + case VM_PAGE_INFO_BASIC: + basic_info = (vm_page_info_basic_t) (((uintptr_t) info) + (info_idx * sizeof(struct vm_page_info_basic))); + basic_info->disposition = disposition; + basic_info->ref_count = ref_count; + basic_info->object_id = (vm_object_id_t) (uintptr_t) + VM_KERNEL_ADDRPERM(curr_object); + basic_info->offset = + (memory_object_offset_t) curr_offset_in_object + offset_in_page; + basic_info->depth = depth; + + info_idx++; + break; + } - if (top_object == TRUE && object->shadow) - disposition |= VM_PAGE_QUERY_PAGE_COPIED; + disposition = 0; + offset_in_page = 0; // This doesn't really make sense for any offset other than the starting offset. - if (! object->internal) - disposition |= VM_PAGE_QUERY_PAGE_EXTERNAL; + /* + * Move to next offset in the range and in our object. + */ + curr_s_offset += PAGE_SIZE; + offset_in_object += PAGE_SIZE; + curr_offset_in_object = offset_in_object; - if (m == VM_PAGE_NULL) - goto done_with_object; + if (curr_object != object) { - if (m->fictitious) { - disposition |= VM_PAGE_QUERY_PAGE_FICTITIOUS; - goto done_with_object; - } - if (m->dirty || pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m))) - disposition |= VM_PAGE_QUERY_PAGE_DIRTY; + vm_object_unlock(curr_object); - if (m->reference || pmap_is_referenced(VM_PAGE_GET_PHYS_PAGE(m))) - disposition |= VM_PAGE_QUERY_PAGE_REF; + curr_object = object; - if (m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) - disposition |= VM_PAGE_QUERY_PAGE_SPECULATIVE; + vm_object_lock_shared(curr_object); + } else { - if (m->cs_validated) - disposition |= VM_PAGE_QUERY_PAGE_CS_VALIDATED; - if (m->cs_tainted) - disposition |= VM_PAGE_QUERY_PAGE_CS_TAINTED; - if (m->cs_nx) - disposition |= VM_PAGE_QUERY_PAGE_CS_NX; + vm_object_lock_yield_shared(curr_object); + } + } -done_with_object: - vm_object_unlock(object); -done: + vm_object_unlock(curr_object); + vm_object_deallocate(curr_object); - switch (flavor) { - case VM_PAGE_INFO_BASIC: - basic_info = (vm_page_info_basic_t) info; - basic_info->disposition = disposition; - basic_info->ref_count = ref_count; - basic_info->object_id = (vm_object_id_t) (uintptr_t) - VM_KERNEL_ADDRPERM(object); - basic_info->offset = - (memory_object_offset_t) offset + offset_in_page; - basic_info->depth = depth; - break; + vm_map_lock_read(map); } + vm_map_unlock_read(map); return retval; } @@ -15055,17 +16311,13 @@ vm_map_msync( vm_map_size_t size, vm_sync_t sync_flags) { - msync_req_t msr; - msync_req_t new_msr; - queue_chain_t req_q; /* queue of requests for this msync */ vm_map_entry_t entry; vm_map_size_t amount_left; vm_object_offset_t offset; boolean_t do_sync_req; boolean_t had_hole = FALSE; - memory_object_t pager; vm_map_offset_t pmap_offset; - + if ((sync_flags & VM_SYNC_ASYNCHRONOUS) && (sync_flags & VM_SYNC_SYNCHRONOUS)) return(KERN_INVALID_ARGUMENT); @@ -15086,7 +16338,6 @@ vm_map_msync( if (size == 0) return(KERN_SUCCESS); - queue_init(&req_q); amount_left = size; while (amount_left > 0) { @@ -15250,112 +16501,22 @@ vm_map_msync( ((sync_flags & VM_SYNC_SYNCHRONOUS) || (sync_flags & VM_SYNC_ASYNCHRONOUS)), sync_flags & VM_SYNC_SYNCHRONOUS); - /* - * only send a m_o_s if we returned pages or if the entry - * is writable (ie dirty pages may have already been sent back) - */ - if (!do_sync_req) { - if ((sync_flags & VM_SYNC_INVALIDATE) && object->resident_page_count == 0) { - /* - * clear out the clustering and read-ahead hints - */ - vm_object_lock(object); - - object->pages_created = 0; - object->pages_used = 0; - object->sequential = 0; - object->last_alloc = 0; - - vm_object_unlock(object); - } - vm_object_deallocate(object); - continue; - } - msync_req_alloc(new_msr); - - vm_object_lock(object); - offset += object->paging_offset; - - new_msr->offset = offset; - new_msr->length = flush_size; - new_msr->object = object; - new_msr->flag = VM_MSYNC_SYNCHRONIZING; - re_iterate: - /* - * We can't sync this object if there isn't a pager. The - * pager can disappear anytime we're not holding the object - * lock. So this has to be checked anytime we goto re_iterate. - */ + if ((sync_flags & VM_SYNC_INVALIDATE) && object->resident_page_count == 0) { + /* + * clear out the clustering and read-ahead hints + */ + vm_object_lock(object); - pager = object->pager; + object->pages_created = 0; + object->pages_used = 0; + object->sequential = 0; + object->last_alloc = 0; - if (pager == MEMORY_OBJECT_NULL) { vm_object_unlock(object); - vm_object_deallocate(object); - msync_req_free(new_msr); - new_msr = NULL; - continue; } - - queue_iterate(&object->msr_q, msr, msync_req_t, msr_q) { - /* - * need to check for overlapping entry, if found, wait - * on overlapping msr to be done, then reiterate - */ - msr_lock(msr); - if (msr->flag == VM_MSYNC_SYNCHRONIZING && - ((offset >= msr->offset && - offset < (msr->offset + msr->length)) || - (msr->offset >= offset && - msr->offset < (offset + flush_size)))) - { - assert_wait((event_t) msr,THREAD_INTERRUPTIBLE); - msr_unlock(msr); - vm_object_unlock(object); - thread_block(THREAD_CONTINUE_NULL); - vm_object_lock(object); - goto re_iterate; - } - msr_unlock(msr); - }/* queue_iterate */ - - queue_enter(&object->msr_q, new_msr, msync_req_t, msr_q); - - vm_object_paging_begin(object); - vm_object_unlock(object); - - queue_enter(&req_q, new_msr, msync_req_t, req_q); - - (void) memory_object_synchronize( - pager, - offset, - flush_size, - sync_flags & ~VM_SYNC_CONTIGUOUS); - - vm_object_lock(object); - vm_object_paging_end(object); - vm_object_unlock(object); - }/* while */ - - /* - * wait for memory_object_sychronize_completed messages from pager(s) - */ - - while (!queue_empty(&req_q)) { - msr = (msync_req_t)queue_first(&req_q); - msr_lock(msr); - while(msr->flag != VM_MSYNC_DONE) { - assert_wait((event_t) msr, THREAD_INTERRUPTIBLE); - msr_unlock(msr); - thread_block(THREAD_CONTINUE_NULL); - msr_lock(msr); - }/* while */ - queue_remove(&req_q, msr, msync_req_t, req_q); - msr_unlock(msr); - vm_object_deallocate(msr->object); - msync_req_free(msr); - }/* queue_iterate */ + vm_object_deallocate(object); + } /* while */ /* for proper msync() behaviour */ if (had_hole == TRUE && (sync_flags & VM_SYNC_CONTIGUOUS)) @@ -15387,7 +16548,7 @@ convert_port_entry_to_map( if(IP_VALID(port) && (ip_kotype(port) == IKOT_NAMED_ENTRY)) { while(TRUE) { ip_lock(port); - if(ip_active(port) && (ip_kotype(port) + if(ip_active(port) && (ip_kotype(port) == IKOT_NAMED_ENTRY)) { named_entry = (vm_named_entry_t)port->ip_kobject; @@ -15402,7 +16563,7 @@ convert_port_entry_to_map( lck_mtx_unlock(&(named_entry)->Lock); ip_unlock(port); if ((named_entry->is_sub_map) && - (named_entry->protection + (named_entry->protection & VM_PROT_WRITE)) { map = named_entry->backing.map; } else { @@ -15413,7 +16574,7 @@ convert_port_entry_to_map( mach_destroy_memory_entry(port); break; } - else + else return VM_MAP_NULL; } } @@ -15428,7 +16589,7 @@ convert_port_entry_to_map( * Purpose: * Convert from a port specifying a named entry to an * object. Doesn't consume the port ref; produces a map ref, - * which may be null. + * which may be null. * Conditions: * Nothing locked. */ @@ -15459,7 +16620,6 @@ convert_port_entry_to_object( lck_mtx_unlock(&(named_entry)->Lock); ip_unlock(port); if (!(named_entry->is_sub_map) && - !(named_entry->is_pager) && !(named_entry->is_copy) && (named_entry->protection & VM_PROT_WRITE)) { object = named_entry->backing.object; @@ -15537,7 +16697,7 @@ vm_map_deallocate( #if TASK_SWAPPER /* * The map residence count isn't decremented here because - * the vm_map_delete below will traverse the entire map, + * the vm_map_delete below will traverse the entire map, * deleting entries, and the residence counts on objects * and sharing maps will go away then. */ @@ -15573,14 +16733,22 @@ vm_map_disallow_data_exec(vm_map_t map) void vm_map_set_32bit(vm_map_t map) { +#if defined(__arm__) || defined(__arm64__) + map->max_offset = pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_DEVICE); +#else map->max_offset = (vm_map_offset_t)VM_MAX_ADDRESS; +#endif } void vm_map_set_64bit(vm_map_t map) { +#if defined(__arm__) || defined(__arm64__) + map->max_offset = pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_DEVICE); +#else map->max_offset = (vm_map_offset_t)MACH_VM_MAX_ADDRESS; +#endif } /* @@ -15589,27 +16757,92 @@ vm_map_set_64bit(vm_map_t map) void vm_map_set_jumbo(vm_map_t map) { +#if defined (__arm64__) + vm_map_offset_t old_max_offset = map->max_offset; + map->max_offset = pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_JUMBO); + if (map->holes_list->prev->vme_end == pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_DEVICE)) { + /* + * There is already a hole at the end of the map; simply make it bigger. + */ + map->holes_list->prev->vme_end = map->max_offset; + } else { + /* + * There is no hole at the end, so we need to create a new hole + * for the new empty space we're creating. + */ + struct vm_map_links *new_hole = zalloc(vm_map_holes_zone); + new_hole->start = old_max_offset; + new_hole->end = map->max_offset; + new_hole->prev = map->holes_list->prev; + new_hole->next = (struct vm_map_entry *)map->holes_list; + map->holes_list->prev->links.next = (struct vm_map_entry *)new_hole; + map->holes_list->prev = (struct vm_map_entry *)new_hole; + } +#else /* arm64 */ (void) map; +#endif } vm_map_offset_t vm_compute_max_offset(boolean_t is64) { +#if defined(__arm__) || defined(__arm64__) + return (pmap_max_offset(is64, ARM_PMAP_MAX_OFFSET_DEVICE)); +#else return (is64 ? (vm_map_offset_t)MACH_VM_MAX_ADDRESS : (vm_map_offset_t)VM_MAX_ADDRESS); +#endif +} + +void +vm_map_get_max_aslr_slide_section( + vm_map_t map __unused, + int64_t *max_sections, + int64_t *section_size) +{ +#if defined(__arm64__) + *max_sections = 3; + *section_size = ARM_TT_TWIG_SIZE; +#else + *max_sections = 1; + *section_size = 0; +#endif } uint64_t -vm_map_get_max_aslr_slide_pages(vm_map_t map) +vm_map_get_max_aslr_slide_pages(vm_map_t map) { +#if defined(__arm64__) + /* Limit arm64 slide to 16MB to conserve contiguous VA space in the more + * limited embedded address space; this is also meant to minimize pmap + * memory usage on 16KB page systems. + */ + return (1 << (24 - VM_MAP_PAGE_SHIFT(map))); +#else + return (1 << (vm_map_is_64bit(map) ? 16 : 8)); +#endif +} + +uint64_t +vm_map_get_max_loader_aslr_slide_pages(vm_map_t map) +{ +#if defined(__arm64__) + /* We limit the loader slide to 4MB, in order to ensure at least 8 bits + * of independent entropy on 16KB page systems. + */ + return (1 << (22 - VM_MAP_PAGE_SHIFT(map))); +#else return (1 << (vm_map_is_64bit(map) ? 16 : 8)); +#endif } +#ifndef __arm__ boolean_t vm_map_is_64bit( vm_map_t map) { return map->max_offset > ((vm_map_offset_t)VM_MAX_ADDRESS); } +#endif boolean_t vm_map_has_hard_pagezero( @@ -15643,7 +16876,7 @@ vm_map_raise_max_offset( ret = KERN_INVALID_ADDRESS; if (new_max_offset >= map->max_offset) { - if (!vm_map_is_64bit(map)) { + if (!vm_map_is_64bit(map)) { if (new_max_offset <= (vm_map_offset_t)VM_MAX_ADDRESS) { map->max_offset = new_max_offset; ret = KERN_SUCCESS; @@ -15729,8 +16962,8 @@ vm_map_set_user_wire_limit(vm_map_t map, } -void vm_map_switch_protect(vm_map_t map, - boolean_t val) +void vm_map_switch_protect(vm_map_t map, + boolean_t val) { vm_map_lock(map); map->switch_protect=val; @@ -15748,7 +16981,7 @@ vm_map_iokit_mapped_region(vm_map_t map, vm_size_t bytes) pmap_t pmap = vm_map_pmap(map); ledger_credit(pmap->ledger, task_ledgers.iokit_mapped, bytes); - ledger_credit(pmap->ledger, task_ledgers.phys_footprint, bytes); + ledger_credit(pmap->ledger, task_ledgers.phys_footprint, bytes); } void @@ -15757,28 +16990,28 @@ vm_map_iokit_unmapped_region(vm_map_t map, vm_size_t bytes) pmap_t pmap = vm_map_pmap(map); ledger_debit(pmap->ledger, task_ledgers.iokit_mapped, bytes); - ledger_debit(pmap->ledger, task_ledgers.phys_footprint, bytes); + ledger_debit(pmap->ledger, task_ledgers.phys_footprint, bytes); } /* Add (generate) code signature for memory range */ #if CONFIG_DYNAMIC_CODE_SIGNING -kern_return_t vm_map_sign(vm_map_t map, - vm_map_offset_t start, +kern_return_t vm_map_sign(vm_map_t map, + vm_map_offset_t start, vm_map_offset_t end) { vm_map_entry_t entry; vm_page_t m; vm_object_t object; - + /* * Vet all the input parameters and current type and state of the * underlaying object. Return with an error if anything is amiss. */ if (map == VM_MAP_NULL) return(KERN_INVALID_ARGUMENT); - + vm_map_lock_read(map); - + if (!vm_map_lookup_entry(map, start, &entry) || entry->is_sub_map) { /* * Must pass a valid non-submap address. @@ -15786,7 +17019,7 @@ kern_return_t vm_map_sign(vm_map_t map, vm_map_unlock_read(map); return(KERN_INVALID_ADDRESS); } - + if((entry->vme_start > start) || (entry->vme_end < end)) { /* * Map entry doesn't cover the requested range. Not handling @@ -15795,7 +17028,7 @@ kern_return_t vm_map_sign(vm_map_t map, vm_map_unlock_read(map); return(KERN_INVALID_ARGUMENT); } - + object = VME_OBJECT(entry); if (object == VM_OBJECT_NULL) { /* @@ -15804,57 +17037,57 @@ kern_return_t vm_map_sign(vm_map_t map, vm_map_unlock_read(map); return KERN_INVALID_ARGUMENT; } - + vm_object_lock(object); vm_map_unlock_read(map); - + while(start < end) { uint32_t refmod; - + m = vm_page_lookup(object, start - entry->vme_start + VME_OFFSET(entry)); if (m==VM_PAGE_NULL) { - /* shoud we try to fault a page here? we can probably + /* shoud we try to fault a page here? we can probably * demand it exists and is locked for this request */ vm_object_unlock(object); return KERN_FAILURE; } /* deal with special page status */ - if (m->busy || + if (m->busy || (m->unusual && (m->error || m->restart || m->private || m->absent))) { vm_object_unlock(object); return KERN_FAILURE; } - + /* Page is OK... now "validate" it */ - /* This is the place where we'll call out to create a code + /* This is the place where we'll call out to create a code * directory, later */ m->cs_validated = TRUE; /* The page is now "clean" for codesigning purposes. That means - * we don't consider it as modified (wpmapped) anymore. But + * we don't consider it as modified (wpmapped) anymore. But * we'll disconnect the page so we note any future modification * attempts. */ m->wpmapped = FALSE; refmod = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); - - /* Pull the dirty status from the pmap, since we cleared the + + /* Pull the dirty status from the pmap, since we cleared the * wpmapped bit */ if ((refmod & VM_MEM_MODIFIED) && !m->dirty) { SET_PAGE_DIRTY(m, FALSE); } - + /* On to the next page */ start += PAGE_SIZE; } vm_object_unlock(object); - + return KERN_SUCCESS; } #endif kern_return_t vm_map_partial_reap(vm_map_t map, unsigned int *reclaimed_resident, unsigned int *reclaimed_compressed) -{ +{ vm_map_entry_t entry = VM_MAP_ENTRY_NULL; vm_map_entry_t next_entry; kern_return_t kr = KERN_SUCCESS; @@ -15875,7 +17108,7 @@ kern_return_t vm_map_partial_reap(vm_map_t map, unsigned int *reclaimed_resident return KERN_RESOURCE_SHORTAGE; } - vm_map_set_page_shift(zap_map, + vm_map_set_page_shift(zap_map, VM_MAP_PAGE_SHIFT(map)); vm_map_disable_hole_optimization(zap_map); @@ -15883,7 +17116,7 @@ kern_return_t vm_map_partial_reap(vm_map_t map, unsigned int *reclaimed_resident entry != vm_map_to_entry(map); entry = next_entry) { next_entry = entry->vme_next; - + if (VME_OBJECT(entry) && !entry->is_sub_map && (VME_OBJECT(entry)->internal == TRUE) && @@ -15892,9 +17125,9 @@ kern_return_t vm_map_partial_reap(vm_map_t map, unsigned int *reclaimed_resident *reclaimed_resident += VME_OBJECT(entry)->resident_page_count; *reclaimed_compressed += vm_compressor_pager_get_count(VME_OBJECT(entry)->pager); - (void)vm_map_delete(map, - entry->vme_start, - entry->vme_end, + (void)vm_map_delete(map, + entry->vme_start, + entry->vme_end, VM_MAP_REMOVE_SAVE_ENTRIES, zap_map); } @@ -15986,7 +17219,7 @@ kern_return_t vm_map_freeze( unsigned int *dirty_count, __unused unsigned int dirty_budget, boolean_t *has_shared) -{ +{ vm_map_entry_t entry2 = VM_MAP_ENTRY_NULL; kern_return_t kr = KERN_SUCCESS; @@ -16004,7 +17237,7 @@ kern_return_t vm_map_freeze( if (vm_compressor_low_on_space() || vm_swap_low_on_space()) { kr = KERN_NO_SPACE; - goto done; + goto done; } c_freezer_compression_count = 0; @@ -16013,7 +17246,7 @@ kern_return_t vm_map_freeze( for (entry2 = vm_map_first_entry(map); entry2 != vm_map_to_entry(map); entry2 = entry2->vme_next) { - + vm_object_t src_object = VME_OBJECT(entry2); if (src_object && @@ -16022,7 +17255,7 @@ kern_return_t vm_map_freeze( /* If eligible, scan the entry, moving eligible pages over to our parent object */ if (src_object->internal == TRUE) { - + if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) { /* * Pages belonging to this object could be swapped to disk. @@ -16037,14 +17270,14 @@ kern_return_t vm_map_freeze( if (vm_compressor_low_on_space() || vm_swap_low_on_space()) { kr = KERN_NO_SPACE; - break; + break; } } } } done: vm_map_unlock(map); - + vm_object_compressed_freezer_done(); if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) { @@ -16150,7 +17383,7 @@ vm_map_entry_should_cow_for_true_share( return TRUE; } -vm_map_offset_t +vm_map_offset_t vm_map_round_page_mask( vm_map_offset_t offset, vm_map_offset_t mask) @@ -16158,7 +17391,7 @@ vm_map_round_page_mask( return VM_MAP_ROUND_PAGE(offset, mask); } -vm_map_offset_t +vm_map_offset_t vm_map_trunc_page_mask( vm_map_offset_t offset, vm_map_offset_t mask) @@ -16355,7 +17588,7 @@ int vm_map_shadow_max( shadows_max = 0; vm_map_lock_read(map); - + for (entry = vm_map_first_entry(map); entry != vm_map_to_entry(map); entry = entry->vme_next) { @@ -16389,3 +17622,13 @@ int vm_map_shadow_max( void vm_commit_pagezero_status(vm_map_t lmap) { pmap_advise_pagezero_range(lmap->pmap, lmap->min_offset); } + +#if __x86_64__ +void +vm_map_set_high_start( + vm_map_t map, + vm_map_offset_t high_start) +{ + map->vmmap_high_start = high_start; +} +#endif /* __x86_64__ */ diff --git a/osfmk/vm/vm_map.h b/osfmk/vm/vm_map.h index 52869a1b4..f8e1aa623 100644 --- a/osfmk/vm/vm_map.h +++ b/osfmk/vm/vm_map.h @@ -170,7 +170,6 @@ struct vm_named_entry { decl_lck_mtx_data(, Lock) /* Synchronization */ union { vm_object_t object; /* object I point to */ - memory_object_t pager; /* amo pager port */ vm_map_t map; /* map backing submap */ vm_map_copy_t copy; /* a VM map copy */ } backing; @@ -182,7 +181,6 @@ struct vm_named_entry { unsigned int /* Is backing.xxx : */ /* boolean_t */ internal:1, /* ... an internal object */ /* boolean_t */ is_sub_map:1, /* ... a submap? */ - /* boolean_t */ is_pager:1, /* ... a pager port */ /* boolean_t */ is_copy:1; /* ... a VM map copy */ }; @@ -385,6 +383,9 @@ struct _vm_map { vm_map_size_t size; /* virtual size */ vm_map_size_t user_wire_limit;/* rlimit on user locked memory */ vm_map_size_t user_wire_size; /* current size of user locked memory in this map */ +#if __x86_64__ + vm_map_offset_t vmmap_high_start; +#endif /* __x86_64__ */ union { /* @@ -562,6 +563,9 @@ struct vm_map_copy { */ #define vm_map_lock_read_to_write(map) (lck_rw_lock_shared_to_exclusive(&(map)->lock) != TRUE) +#define vm_map_try_lock(map) lck_rw_try_lock_exclusive(&(map)->lock) +#define vm_map_try_lock_read(map) lck_rw_try_lock_shared(&(map)->lock) + #if MACH_ASSERT || DEBUG #define vm_map_lock_assert_held(map) \ lck_rw_assert(&(map)->lock, LCK_RW_ASSERT_HELD) @@ -595,6 +599,8 @@ extern kern_return_t vm_map_find_space( vm_map_size_t size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, vm_map_entry_t *o_entry); /* OUT */ extern void vm_map_clip_start( @@ -662,7 +668,9 @@ extern vm_map_entry_t vm_map_entry_insert( boolean_t permanent, unsigned int superpage_size, boolean_t clear_map_aligned, - boolean_t is_submap); + boolean_t is_submap, + boolean_t used_for_jit, + int alias); /* @@ -679,11 +687,6 @@ extern vm_map_entry_t vm_map_entry_insert( /* Physical map associated * with this address map */ -#define vm_map_verify_done(map, version) vm_map_unlock_read(map) - /* Operation that required - * a verified lookup is - * now complete */ - /* * Macros/functions for map residence counts and swapin/out of vm maps */ @@ -848,6 +851,8 @@ extern kern_return_t vm_map_enter( vm_map_size_t size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, vm_object_t object, vm_object_offset_t offset, boolean_t needs_copy, @@ -855,6 +860,22 @@ extern kern_return_t vm_map_enter( vm_prot_t max_protection, vm_inherit_t inheritance); +#if __arm64__ +extern kern_return_t vm_map_enter_fourk( + vm_map_t map, + vm_map_offset_t *address, + vm_map_size_t size, + vm_map_offset_t mask, + int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, + vm_object_t object, + vm_object_offset_t offset, + boolean_t needs_copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance); +#endif /* __arm64__ */ /* XXX should go away - replaced with regular enter of contig object */ extern kern_return_t vm_map_enter_cpm( @@ -869,6 +890,8 @@ extern kern_return_t vm_map_remap( vm_map_size_t size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, vm_map_t src_map, vm_map_offset_t memory_address, boolean_t copy, @@ -928,12 +951,6 @@ extern kern_return_t vm_map_behavior_set( vm_map_offset_t end, vm_behavior_t new_behavior); -extern kern_return_t vm_map_purgable_control( - vm_map_t map, - vm_map_offset_t address, - vm_purgable_t control, - int *state); - extern kern_return_t vm_map_region( vm_map_t map, vm_map_offset_t *address, @@ -1001,23 +1018,6 @@ extern kern_return_t vm_map_set_cache_attr( extern int override_nx(vm_map_t map, uint32_t user_tag); - -/* kext exported versions */ - -extern kern_return_t vm_map_wire_external( - vm_map_t map, - vm_map_offset_t start, - vm_map_offset_t end, - vm_prot_t caller_prot, - boolean_t user_wire); - -extern kern_return_t vm_map_wire_and_extract_external( - vm_map_t map, - vm_map_offset_t start, - vm_prot_t caller_prot, - boolean_t user_wire, - ppnum_t *physpage_p); - #endif /* MACH_KERNEL_PRIVATE */ __BEGIN_DECLS @@ -1059,6 +1059,43 @@ extern boolean_t vm_map_check_protection( vm_prot_t protection); /* wire down a region */ + +#ifdef XNU_KERNEL_PRIVATE + +extern kern_return_t vm_map_wire_kernel( + vm_map_t map, + vm_map_offset_t start, + vm_map_offset_t end, + vm_prot_t access_type, + vm_tag_t tag, + boolean_t user_wire); + +extern kern_return_t vm_map_wire_and_extract_kernel( + vm_map_t map, + vm_map_offset_t start, + vm_prot_t access_type, + vm_tag_t tag, + boolean_t user_wire, + ppnum_t *physpage_p); + +/* kext exported versions */ + +extern kern_return_t vm_map_wire_external( + vm_map_t map, + vm_map_offset_t start, + vm_map_offset_t end, + vm_prot_t access_type, + boolean_t user_wire); + +extern kern_return_t vm_map_wire_and_extract_external( + vm_map_t map, + vm_map_offset_t start, + vm_prot_t access_type, + boolean_t user_wire, + ppnum_t *physpage_p); + +#else /* XNU_KERNEL_PRIVATE */ + extern kern_return_t vm_map_wire( vm_map_t map, vm_map_offset_t start, @@ -1073,6 +1110,8 @@ extern kern_return_t vm_map_wire_and_extract( boolean_t user_wire, ppnum_t *physpage_p); +#endif /* !XNU_KERNEL_PRIVATE */ + /* unwire a region */ extern kern_return_t vm_map_unwire( vm_map_t map, @@ -1080,6 +1119,8 @@ extern kern_return_t vm_map_unwire( vm_map_offset_t end, boolean_t user_wire); +#ifdef XNU_KERNEL_PRIVATE + /* Enter a mapping of a memory object */ extern kern_return_t vm_map_enter_mem_object( vm_map_t map, @@ -1087,6 +1128,8 @@ extern kern_return_t vm_map_enter_mem_object( vm_map_size_t size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, ipc_port_t port, vm_object_offset_t offset, boolean_t needs_copy, @@ -1101,6 +1144,8 @@ extern kern_return_t vm_map_enter_mem_object_prefault( vm_map_size_t size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, ipc_port_t port, vm_object_offset_t offset, vm_prot_t cur_protection, @@ -1115,6 +1160,8 @@ extern kern_return_t vm_map_enter_mem_object_control( vm_map_size_t size, vm_map_offset_t mask, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, memory_object_control_t control, vm_object_offset_t offset, boolean_t needs_copy, @@ -1122,6 +1169,8 @@ extern kern_return_t vm_map_enter_mem_object_control( vm_prot_t max_protection, vm_inherit_t inheritance); +#endif /* !XNU_KERNEL_PRIVATE */ + /* Deallocate a region */ extern kern_return_t vm_map_remove( vm_map_t map, @@ -1232,8 +1281,12 @@ extern boolean_t vm_map_has_hard_pagezero( vm_map_offset_t pagezero_size); extern void vm_commit_pagezero_status(vm_map_t tmap); +#ifdef __arm__ +static inline boolean_t vm_map_is_64bit(__unused vm_map_t map) { return 0; } +#else extern boolean_t vm_map_is_64bit( vm_map_t map); +#endif extern kern_return_t vm_map_raise_max_offset( @@ -1243,19 +1296,32 @@ extern kern_return_t vm_map_raise_max_offset( extern kern_return_t vm_map_raise_min_offset( vm_map_t map, vm_map_offset_t new_min_offset); +#if __x86_64__ +extern void vm_map_set_high_start( + vm_map_t map, + vm_map_offset_t high_start); +#endif /* __x86_64__ */ extern vm_map_offset_t vm_compute_max_offset( boolean_t is64); +extern void vm_map_get_max_aslr_slide_section( + vm_map_t map, + int64_t *max_sections, + int64_t *section_size); + extern uint64_t vm_map_get_max_aslr_slide_pages( vm_map_t map); - + +extern uint64_t vm_map_get_max_loader_aslr_slide_pages( + vm_map_t map); + extern void vm_map_set_user_wire_limit( vm_map_t map, vm_size_t limit); extern void vm_map_switch_protect( - vm_map_t map, + vm_map_t map, boolean_t val); extern void vm_map_iokit_mapped_region( @@ -1297,6 +1363,13 @@ extern kern_return_t vm_map_page_info( vm_page_info_flavor_t flavor, vm_page_info_t info, mach_msg_type_number_t *count); +extern kern_return_t vm_map_page_range_info_internal( + vm_map_t map, + vm_map_offset_t start_offset, + vm_map_offset_t end_offset, + vm_page_info_flavor_t flavor, + vm_page_info_t info, + mach_msg_type_number_t *count); #endif /* XNU_KERNEL_PRIVATE */ @@ -1335,6 +1408,21 @@ extern kern_return_t vm_map_page_info( #define VM_MAP_PAGE_MASK(map) (VM_MAP_PAGE_SIZE((map)) - 1) #define VM_MAP_PAGE_ALIGNED(x,pgmask) (((x) & (pgmask)) == 0) +static inline void vm_prot_to_wimg(unsigned int prot, unsigned int *wimg) +{ + switch (prot) { + case MAP_MEM_NOOP: break; + case MAP_MEM_IO: *wimg = VM_WIMG_IO; break; + case MAP_MEM_COPYBACK: *wimg = VM_WIMG_USE_DEFAULT; break; + case MAP_MEM_INNERWBACK: *wimg = VM_WIMG_INNERWBACK; break; + case MAP_MEM_POSTED: *wimg = VM_WIMG_POSTED; break; + case MAP_MEM_WTHRU: *wimg = VM_WIMG_WTHRU; break; + case MAP_MEM_WCOMB: *wimg = VM_WIMG_WCOMB; break; + default: + panic("Unrecognized mapping type %u\n", prot); + } +} + #endif /* MACH_KERNEL_PRIVATE */ #ifdef XNU_KERNEL_PRIVATE @@ -1355,9 +1443,12 @@ extern kern_return_t vm_map_set_page_shift(vm_map_t map, int pageshift); #define VM_MAP_REMOVE_NO_PMAP_CLEANUP 0x10 #define VM_MAP_REMOVE_NO_MAP_ALIGN 0x20 #define VM_MAP_REMOVE_NO_UNNESTING 0x40 +#define VM_MAP_REMOVE_IMMUTABLE 0x80 /* Support for UPLs from vm_maps */ +#ifdef XNU_KERNEL_PRIVATE + extern kern_return_t vm_map_get_upl( vm_map_t target_map, vm_map_offset_t map_offset, @@ -1366,8 +1457,11 @@ extern kern_return_t vm_map_get_upl( upl_page_info_array_t page_info, unsigned int *page_infoCnt, upl_control_flags_t *flags, + vm_tag_t tag, int force_data_sync); +#endif /* XNU_KERNEL_PRIVATE */ + extern void vm_map_sizes(vm_map_t map, vm_map_size_t * psize, diff --git a/osfmk/vm/vm_map_store_rb.c b/osfmk/vm/vm_map_store_rb.c index 130e4c8d3..70fb9be4c 100644 --- a/osfmk/vm/vm_map_store_rb.c +++ b/osfmk/vm/vm_map_store_rb.c @@ -553,6 +553,14 @@ update_holes_on_entry_creation(vm_map_t map, vm_map_entry_t new_entry) copy_hole_info(hole_entry, &old_hole_entry); #endif /* DEBUG */ + /* + * This check makes sense only for regular maps, not copy maps. + * With a regular map, the VM entry is first linked and then + * the hole is deleted. So the check below, which makes sure that + * the map's bounds are being respected, is valid. + * But for copy maps, the hole is deleted before the VM entry is + * linked (vm_map_store_copy_insert) and so this check is invalid. + * if (hole_entry == (vm_map_entry_t) map->holes_list) { if (hole_entry->vme_next == (vm_map_entry_t) map->holes_list) { @@ -561,6 +569,7 @@ update_holes_on_entry_creation(vm_map_t map, vm_map_entry_t new_entry) assert(next_hole_entry->vme_end >= map->max_offset); } } + */ vm_map_delete_hole(map, hole_entry); diff --git a/osfmk/vm/vm_object.c b/osfmk/vm/vm_object.c index de9542c3a..a7820e3b0 100644 --- a/osfmk/vm/vm_object.c +++ b/osfmk/vm/vm_object.c @@ -210,9 +210,6 @@ vm_object_tracking_init(void) static kern_return_t vm_object_terminate( vm_object_t object); -extern void vm_object_remove( - vm_object_t object); - static kern_return_t vm_object_copy_call( vm_object_t src_object, vm_object_offset_t src_offset, @@ -228,10 +225,9 @@ static void vm_object_do_bypass( vm_object_t backing_object); static void vm_object_release_pager( - memory_object_t pager, - boolean_t hashed); + memory_object_t pager); -static zone_t vm_object_zone; /* vm backing store zone */ +zone_t vm_object_zone; /* vm backing store zone */ /* * All wired-down kernel memory belongs to a single virtual @@ -265,51 +261,6 @@ unsigned int vm_page_purged_wired = 0; unsigned int vm_page_purged_busy = 0; unsigned int vm_page_purged_others = 0; -#if VM_OBJECT_CACHE -/* - * Virtual memory objects that are not referenced by - * any address maps, but that are allowed to persist - * (an attribute specified by the associated memory manager), - * are kept in a queue (vm_object_cached_list). - * - * When an object from this queue is referenced again, - * for example to make another address space mapping, - * it must be removed from the queue. That is, the - * queue contains *only* objects with zero references. - * - * The kernel may choose to terminate objects from this - * queue in order to reclaim storage. The current policy - * is to permit a fixed maximum number of unreferenced - * objects (vm_object_cached_max). - * - * A spin lock (accessed by routines - * vm_object_cache_{lock,lock_try,unlock}) governs the - * object cache. It must be held when objects are - * added to or removed from the cache (in vm_object_terminate). - * The routines that acquire a reference to a virtual - * memory object based on one of the memory object ports - * must also lock the cache. - * - * Ideally, the object cache should be more isolated - * from the reference mechanism, so that the lock need - * not be held to make simple references. - */ -static vm_object_t vm_object_cache_trim( - boolean_t called_from_vm_object_deallocate); - -static void vm_object_deactivate_all_pages( - vm_object_t object); - -static int vm_object_cached_high; /* highest # cached objects */ -static int vm_object_cached_max = 512; /* may be patched*/ - -#define vm_object_cache_lock() \ - lck_mtx_lock(&vm_object_cached_lock_data) -#define vm_object_cache_lock_try() \ - lck_mtx_try_lock(&vm_object_cached_lock_data) - -#endif /* VM_OBJECT_CACHE */ - static queue_head_t vm_object_cached_list; static uint32_t vm_object_cache_pages_freed = 0; static uint32_t vm_object_cache_pages_moved = 0; @@ -333,36 +284,6 @@ static uint32_t vm_object_page_grab_reactivations = 0; static void vm_object_cache_remove_locked(vm_object_t); -#define VM_OBJECT_HASH_COUNT 1024 -#define VM_OBJECT_HASH_LOCK_COUNT 512 - -static lck_mtx_t vm_object_hashed_lock_data[VM_OBJECT_HASH_LOCK_COUNT]; -static lck_mtx_ext_t vm_object_hashed_lock_data_ext[VM_OBJECT_HASH_LOCK_COUNT]; - -static queue_head_t vm_object_hashtable[VM_OBJECT_HASH_COUNT]; -static struct zone *vm_object_hash_zone; - -struct vm_object_hash_entry { - queue_chain_t hash_link; /* hash chain link */ - memory_object_t pager; /* pager we represent */ - vm_object_t object; /* corresponding object */ - boolean_t waiting; /* someone waiting for - * termination */ -}; - -typedef struct vm_object_hash_entry *vm_object_hash_entry_t; -#define VM_OBJECT_HASH_ENTRY_NULL ((vm_object_hash_entry_t) 0) - -#define VM_OBJECT_HASH_SHIFT 5 -#define vm_object_hash(pager) \ - ((int)((((uintptr_t)pager) >> VM_OBJECT_HASH_SHIFT) % VM_OBJECT_HASH_COUNT)) - -#define vm_object_lock_hash(pager) \ - ((int)((((uintptr_t)pager) >> VM_OBJECT_HASH_SHIFT) % VM_OBJECT_HASH_LOCK_COUNT)) - -void vm_object_hash_entry_free( - vm_object_hash_entry_t entry); - static void vm_object_reap(vm_object_t object); static void vm_object_reap_async(vm_object_t object); static void vm_object_reaper_thread(void); @@ -416,109 +337,6 @@ void vm_decmp_upl_reprioritize(upl_t, int); #endif -static lck_mtx_t * -vm_object_hash_lock_spin( - memory_object_t pager) -{ - int index; - - index = vm_object_lock_hash(pager); - - lck_mtx_lock_spin(&vm_object_hashed_lock_data[index]); - - return (&vm_object_hashed_lock_data[index]); -} - -static void -vm_object_hash_unlock(lck_mtx_t *lck) -{ - lck_mtx_unlock(lck); -} - - -/* - * vm_object_hash_lookup looks up a pager in the hashtable - * and returns the corresponding entry, with optional removal. - */ -static vm_object_hash_entry_t -vm_object_hash_lookup( - memory_object_t pager, - boolean_t remove_entry) -{ - queue_t bucket; - vm_object_hash_entry_t entry; - - bucket = &vm_object_hashtable[vm_object_hash(pager)]; - - entry = (vm_object_hash_entry_t)queue_first(bucket); - while (!queue_end(bucket, (queue_entry_t)entry)) { - if (entry->pager == pager) { - if (remove_entry) { - queue_remove(bucket, entry, - vm_object_hash_entry_t, hash_link); - } - return(entry); - } - entry = (vm_object_hash_entry_t)queue_next(&entry->hash_link); - } - return(VM_OBJECT_HASH_ENTRY_NULL); -} - -/* - * vm_object_hash_enter enters the specified - * pager / cache object association in the hashtable. - */ - -static void -vm_object_hash_insert( - vm_object_hash_entry_t entry, - vm_object_t object) -{ - queue_t bucket; - - assert(vm_object_hash_lookup(entry->pager, FALSE) == NULL); - - bucket = &vm_object_hashtable[vm_object_hash(entry->pager)]; - - queue_enter(bucket, entry, vm_object_hash_entry_t, hash_link); - - if (object->hashed) { - /* - * "hashed" was pre-set on this (new) object to avoid - * locking issues in vm_object_enter() (can't attempt to - * grab the object lock while holding the hash lock as - * a spinlock), so no need to set it here (and no need to - * hold the object's lock). - */ - } else { - vm_object_lock_assert_exclusive(object); - object->hashed = TRUE; - } - - entry->object = object; -} - -static vm_object_hash_entry_t -vm_object_hash_entry_alloc( - memory_object_t pager) -{ - vm_object_hash_entry_t entry; - - entry = (vm_object_hash_entry_t)zalloc(vm_object_hash_zone); - entry->pager = pager; - entry->object = VM_OBJECT_NULL; - entry->waiting = FALSE; - - return(entry); -} - -void -vm_object_hash_entry_free( - vm_object_hash_entry_t entry) -{ - zfree(vm_object_hash_zone, entry); -} - /* * vm_object_allocate: * @@ -536,7 +354,6 @@ _vm_object_allocate( *object = vm_object_template; vm_page_queue_init(&object->memq); - queue_init(&object->msr_q); #if UPL_DEBUG || CONFIG_IOSCHED queue_init(&object->uplq); #endif @@ -590,9 +407,10 @@ lck_attr_t compressor_object_lck_attr; __private_extern__ void vm_object_bootstrap(void) { - int i; vm_size_t vm_object_size; + assert(sizeof (mo_ipc_object_bits_t) == sizeof (ipc_object_bits_t)); + vm_object_size = (sizeof(struct vm_object) + (VM_PACKED_POINTER_ALIGNMENT-1)) & ~(VM_PACKED_POINTER_ALIGNMENT - 1); vm_object_zone = zinit(vm_object_size, @@ -601,6 +419,7 @@ vm_object_bootstrap(void) "vm objects"); zone_change(vm_object_zone, Z_CALLERACCT, FALSE); /* don't charge caller */ zone_change(vm_object_zone, Z_NOENCRYPT, TRUE); + zone_change(vm_object_zone, Z_ALIGNMENT_REQUIRED, TRUE); vm_object_init_lck_grp(); @@ -613,28 +432,11 @@ vm_object_bootstrap(void) queue_init(&vm_object_reaper_queue); - for (i = 0; i < VM_OBJECT_HASH_LOCK_COUNT; i++) { - lck_mtx_init_ext(&vm_object_hashed_lock_data[i], - &vm_object_hashed_lock_data_ext[i], - &vm_object_lck_grp, - &vm_object_lck_attr); - } lck_mtx_init_ext(&vm_object_reaper_lock_data, &vm_object_reaper_lock_data_ext, &vm_object_lck_grp, &vm_object_lck_attr); - vm_object_hash_zone = - zinit((vm_size_t) sizeof (struct vm_object_hash_entry), - round_page(512*1024), - round_page(12*1024), - "vm object hash entries"); - zone_change(vm_object_hash_zone, Z_CALLERACCT, FALSE); - zone_change(vm_object_hash_zone, Z_NOENCRYPT, TRUE); - - for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) - queue_init(&vm_object_hashtable[i]); - /* * Fill in a template object, for quick initialization @@ -642,7 +444,6 @@ vm_object_bootstrap(void) /* memq; Lock; init after allocation */ - vm_object_template.memq.prev = 0; vm_object_template.memq.next = 0; #if 0 @@ -665,7 +466,8 @@ vm_object_bootstrap(void) vm_object_template.res_count = 1; #endif /* TASK_SWAPPER */ vm_object_template.resident_page_count = 0; - vm_object_template.wired_page_count = 0; + // static vm_object_template is zeroed + // vm_object_template.wired_page_count = 0; vm_object_template.reusable_page_count = 0; vm_object_template.copy = VM_OBJECT_NULL; vm_object_template.shadow = VM_OBJECT_NULL; @@ -688,14 +490,13 @@ vm_object_bootstrap(void) vm_object_template.pager_trusted = FALSE; vm_object_template.can_persist = FALSE; vm_object_template.internal = TRUE; - vm_object_template.temporary = TRUE; vm_object_template.private = FALSE; vm_object_template.pageout = FALSE; vm_object_template.alive = TRUE; vm_object_template.purgable = VM_PURGABLE_DENY; vm_object_template.purgeable_when_ripe = FALSE; + vm_object_template.purgeable_only_by_kernel = FALSE; vm_object_template.shadowed = FALSE; - vm_object_template.advisory_pageout = FALSE; vm_object_template.true_share = FALSE; vm_object_template.terminating = FALSE; vm_object_template.named = FALSE; @@ -706,8 +507,6 @@ vm_object_bootstrap(void) vm_object_template.cached_list.prev = NULL; vm_object_template.cached_list.next = NULL; - vm_object_template.msr_q.prev = NULL; - vm_object_template.msr_q.next = NULL; vm_object_template.last_alloc = (vm_object_offset_t) 0; vm_object_template.sequential = (vm_object_offset_t) 0; @@ -718,16 +517,12 @@ vm_object_bootstrap(void) vm_object_template.phantom_object_id = 0; #endif vm_object_template.cow_hint = ~(vm_offset_t)0; -#if MACH_ASSERT - vm_object_template.paging_object = VM_OBJECT_NULL; -#endif /* MACH_ASSERT */ /* cache bitfields */ vm_object_template.wimg_bits = VM_WIMG_USE_DEFAULT; vm_object_template.set_cache_attr = FALSE; vm_object_template.object_slid = FALSE; vm_object_template.code_signed = FALSE; - vm_object_template.hashed = FALSE; vm_object_template.transposed = FALSE; vm_object_template.mapping_in_progress = FALSE; vm_object_template.phantom_isssd = FALSE; @@ -790,6 +585,7 @@ vm_object_bootstrap(void) compressor_object); kernel_object->copy_strategy = MEMORY_OBJECT_COPY_NONE; compressor_object->copy_strategy = MEMORY_OBJECT_COPY_NONE; + kernel_object->no_tag_update = TRUE; /* * Initialize the "submap object". Make it as large as the @@ -843,7 +639,7 @@ vm_object_reaper_init(void) kr = kernel_thread_start_priority( (thread_continue_t) vm_object_reaper_thread, NULL, - BASEPRI_PREEMPT - 1, + BASEPRI_VM, &thread); if (kr != KERN_SUCCESS) { panic("failed to launch vm_object_reaper_thread kr=0x%x", kr); @@ -876,12 +672,6 @@ vm_object_init_lck_grp(void) lck_attr_cleardebug(&compressor_object_lck_attr); } -#if VM_OBJECT_CACHE -#define MIGHT_NOT_CACHE_SHADOWS 1 -#if MIGHT_NOT_CACHE_SHADOWS -static int cache_shadows = TRUE; -#endif /* MIGHT_NOT_CACHE_SHADOWS */ -#endif /* * vm_object_deallocate: @@ -902,10 +692,6 @@ __private_extern__ void vm_object_deallocate( vm_object_t object) { -#if VM_OBJECT_CACHE - boolean_t retry_cache_trim = FALSE; - uint32_t try_failed_count = 0; -#endif vm_object_t shadow = VM_OBJECT_NULL; // if(object)dbgLog(object, object->ref_count, object->can_persist, 3); /* (TEST/DEBUG) */ @@ -1076,13 +862,6 @@ vm_object_deallocate( vm_object_collapse(object, 0, FALSE); } vm_object_unlock(object); -#if VM_OBJECT_CACHE - if (retry_cache_trim && - ((object = vm_object_cache_trim(TRUE)) != - VM_OBJECT_NULL)) { - continue; - } -#endif return; } @@ -1102,160 +881,32 @@ vm_object_deallocate( continue; } -#if VM_OBJECT_CACHE + XPR(XPR_VM_OBJECT, + "vm_o_deallocate: 0x%X res %d paging_ops %d thread 0x%p ref %d\n", + object, object->resident_page_count, + object->paging_in_progress, + (void *)current_thread(),object->ref_count); + + VM_OBJ_RES_DECR(object); /* XXX ? */ /* - * If this object can persist, then enter it in - * the cache. Otherwise, terminate it. - * - * NOTE: Only permanent objects are cached, and - * permanent objects cannot have shadows. This - * affects the residence counting logic in a minor - * way (can do it in-line, mostly). + * Terminate this object. If it had a shadow, + * then deallocate it; otherwise, if we need + * to retry a cache trim, do so now; otherwise, + * we are done. "pageout" objects have a shadow, + * but maintain a "paging reference" rather than + * a normal reference. */ + shadow = object->pageout?VM_OBJECT_NULL:object->shadow; - if ((object->can_persist) && (object->alive)) { - /* - * Now it is safe to decrement reference count, - * and to return if reference count is > 0. - */ - - vm_object_lock_assert_exclusive(object); - if (--object->ref_count > 0) { - vm_object_res_deallocate(object); - vm_object_unlock(object); - - if (retry_cache_trim && - ((object = vm_object_cache_trim(TRUE)) != - VM_OBJECT_NULL)) { - continue; - } - return; - } - -#if MIGHT_NOT_CACHE_SHADOWS - /* - * Remove shadow now if we don't - * want to cache shadows. - */ - if (! cache_shadows) { - shadow = object->shadow; - object->shadow = VM_OBJECT_NULL; - } -#endif /* MIGHT_NOT_CACHE_SHADOWS */ - - /* - * Enter the object onto the queue of - * cached objects, and deactivate - * all of its pages. - */ - assert(object->shadow == VM_OBJECT_NULL); - VM_OBJ_RES_DECR(object); - XPR(XPR_VM_OBJECT, - "vm_o_deallocate: adding %x to cache, queue = (%x, %x)\n", - object, - vm_object_cached_list.next, - vm_object_cached_list.prev,0,0); - - - vm_object_unlock(object); - - try_failed_count = 0; - for (;;) { - vm_object_cache_lock(); - - /* - * if we try to take a regular lock here - * we risk deadlocking against someone - * holding a lock on this object while - * trying to vm_object_deallocate a different - * object - */ - if (vm_object_lock_try(object)) - break; - vm_object_cache_unlock(); - try_failed_count++; - - mutex_pause(try_failed_count); /* wait a bit */ - } - vm_object_cached_count++; - if (vm_object_cached_count > vm_object_cached_high) - vm_object_cached_high = vm_object_cached_count; - queue_enter(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cache_unlock(); - - vm_object_deactivate_all_pages(object); - vm_object_unlock(object); - -#if MIGHT_NOT_CACHE_SHADOWS - /* - * If we have a shadow that we need - * to deallocate, do so now, remembering - * to trim the cache later. - */ - if (! cache_shadows && shadow != VM_OBJECT_NULL) { - object = shadow; - retry_cache_trim = TRUE; - continue; - } -#endif /* MIGHT_NOT_CACHE_SHADOWS */ - - /* - * Trim the cache. If the cache trim - * returns with a shadow for us to deallocate, - * then remember to retry the cache trim - * when we are done deallocating the shadow. - * Otherwise, we are done. - */ - - object = vm_object_cache_trim(TRUE); - if (object == VM_OBJECT_NULL) { - return; - } - retry_cache_trim = TRUE; - } else -#endif /* VM_OBJECT_CACHE */ - { - /* - * This object is not cachable; terminate it. - */ - XPR(XPR_VM_OBJECT, - "vm_o_deallocate: !cacheable 0x%X res %d paging_ops %d thread 0x%p ref %d\n", - object, object->resident_page_count, - object->paging_in_progress, - (void *)current_thread(),object->ref_count); - - VM_OBJ_RES_DECR(object); /* XXX ? */ - /* - * Terminate this object. If it had a shadow, - * then deallocate it; otherwise, if we need - * to retry a cache trim, do so now; otherwise, - * we are done. "pageout" objects have a shadow, - * but maintain a "paging reference" rather than - * a normal reference. - */ - shadow = object->pageout?VM_OBJECT_NULL:object->shadow; - - if (vm_object_terminate(object) != KERN_SUCCESS) { - return; - } - if (shadow != VM_OBJECT_NULL) { - object = shadow; - continue; - } -#if VM_OBJECT_CACHE - if (retry_cache_trim && - ((object = vm_object_cache_trim(TRUE)) != - VM_OBJECT_NULL)) { - continue; - } -#endif - return; + if (vm_object_terminate(object) != KERN_SUCCESS) { + return; + } + if (shadow != VM_OBJECT_NULL) { + object = shadow; + continue; } + return; } -#if VM_OBJECT_CACHE - assert(! retry_cache_trim); -#endif } @@ -1630,99 +1281,6 @@ vm_object_cache_evict( return (ep_freed); } - -#if VM_OBJECT_CACHE -/* - * Check to see whether we really need to trim - * down the cache. If so, remove an object from - * the cache, terminate it, and repeat. - * - * Called with, and returns with, cache lock unlocked. - */ -vm_object_t -vm_object_cache_trim( - boolean_t called_from_vm_object_deallocate) -{ - vm_object_t object = VM_OBJECT_NULL; - vm_object_t shadow; - - for (;;) { - - /* - * If we no longer need to trim the cache, - * then we are done. - */ - if (vm_object_cached_count <= vm_object_cached_max) - return VM_OBJECT_NULL; - - vm_object_cache_lock(); - if (vm_object_cached_count <= vm_object_cached_max) { - vm_object_cache_unlock(); - return VM_OBJECT_NULL; - } - - /* - * We must trim down the cache, so remove - * the first object in the cache. - */ - XPR(XPR_VM_OBJECT, - "vm_object_cache_trim: removing from front of cache (%x, %x)\n", - vm_object_cached_list.next, - vm_object_cached_list.prev, 0, 0, 0); - - object = (vm_object_t) queue_first(&vm_object_cached_list); - if(object == (vm_object_t) &vm_object_cached_list) { - /* something's wrong with the calling parameter or */ - /* the value of vm_object_cached_count, just fix */ - /* and return */ - if(vm_object_cached_max < 0) - vm_object_cached_max = 0; - vm_object_cached_count = 0; - vm_object_cache_unlock(); - return VM_OBJECT_NULL; - } - vm_object_lock(object); - queue_remove(&vm_object_cached_list, object, vm_object_t, - cached_list); - vm_object_cached_count--; - - vm_object_cache_unlock(); - /* - * Since this object is in the cache, we know - * that it is initialized and has no references. - * Take a reference to avoid recursive deallocations. - */ - - assert(object->pager_initialized); - assert(object->ref_count == 0); - vm_object_lock_assert_exclusive(object); - object->ref_count++; - - /* - * Terminate the object. - * If the object had a shadow, we let vm_object_deallocate - * deallocate it. "pageout" objects have a shadow, but - * maintain a "paging reference" rather than a normal - * reference. - * (We are careful here to limit recursion.) - */ - shadow = object->pageout?VM_OBJECT_NULL:object->shadow; - - if(vm_object_terminate(object) != KERN_SUCCESS) - continue; - - if (shadow != VM_OBJECT_NULL) { - if (called_from_vm_object_deallocate) { - return shadow; - } else { - vm_object_deallocate(shadow); - } - } - } -} -#endif - - /* * Routine: vm_object_terminate * Purpose: @@ -1751,7 +1309,7 @@ vm_object_terminate( vm_object_lock_assert_exclusive(object); - if (!object->pageout && (!object->temporary || object->can_persist) && + if (!object->pageout && (!object->internal && object->can_persist) && (object->pager != NULL || object->shadow_severed)) { /* * Clear pager_trusted bit so that the pages get yanked @@ -1796,13 +1354,6 @@ vm_object_terminate( if ( !object->internal && (object->objq.next || object->objq.prev)) vm_object_cache_remove(object); - if (object->hashed) { - lck_mtx_t *lck; - - lck = vm_object_hash_lock_spin(object->pager); - vm_object_remove(object); - vm_object_hash_unlock(lck); - } /* * Detach the object from its shadow if we are the shadow's * copy. The reference we hold on the shadow must be dropped @@ -1986,18 +1537,21 @@ vm_object_reap( assert(object->objq.prev == NULL); } - /* - * Clean or free the pages, as appropriate. - * It is possible for us to find busy/absent pages, - * if some faults on this object were aborted. - */ if (object->pageout) { + /* + * free all remaining pages tabled on + * this object + * clean up it's shadow + */ assert(object->shadow != VM_OBJECT_NULL); vm_pageout_object_terminate(object); - } else if (((object->temporary && !object->can_persist) || (pager == MEMORY_OBJECT_NULL))) { - + } else if (object->resident_page_count) { + /* + * free all remaining pages tabled on + * this object + */ vm_object_reap_pages(object, REAP_REAP); } assert(vm_page_queue_empty(&object->memq)); @@ -2012,7 +1566,7 @@ vm_object_reap( */ if (pager != MEMORY_OBJECT_NULL) { vm_object_unlock(object); - vm_object_release_pager(pager, object->hashed); + vm_object_release_pager(pager); vm_object_lock(object); } @@ -2245,7 +1799,7 @@ restart_after_sleep: * flush page... page will be freed * upon completion of I/O */ - (void)vm_pageout_cluster(p, FALSE, FALSE); + vm_pageout_cluster(p); } vm_page_unlock_queues(); /* @@ -2368,49 +1922,14 @@ vm_object_reaper_thread(void) /*NOTREACHED*/ } -/* - * Routine: vm_object_pager_wakeup - * Purpose: Wake up anyone waiting for termination of a pager. - */ - -static void -vm_object_pager_wakeup( - memory_object_t pager) -{ - vm_object_hash_entry_t entry; - boolean_t waiting = FALSE; - lck_mtx_t *lck; - - /* - * If anyone was waiting for the memory_object_terminate - * to be queued, wake them up now. - */ - lck = vm_object_hash_lock_spin(pager); - entry = vm_object_hash_lookup(pager, TRUE); - if (entry != VM_OBJECT_HASH_ENTRY_NULL) - waiting = entry->waiting; - vm_object_hash_unlock(lck); - - if (entry != VM_OBJECT_HASH_ENTRY_NULL) { - if (waiting) - thread_wakeup((event_t) pager); - vm_object_hash_entry_free(entry); - } -} - /* * Routine: vm_object_release_pager * Purpose: Terminate the pager and, upon completion, * release our last reference to it. - * just like memory_object_terminate, except - * that we wake up anyone blocked in vm_object_enter - * waiting for termination message to be queued - * before calling memory_object_init. */ static void vm_object_release_pager( - memory_object_t pager, - boolean_t hashed) + memory_object_t pager) { /* @@ -2419,13 +1938,6 @@ vm_object_release_pager( (void) memory_object_terminate(pager); - if (hashed == TRUE) { - /* - * Wakeup anyone waiting for this terminate - * and remove the entry from the hash - */ - vm_object_pager_wakeup(pager); - } /* * Release reference to pager. */ @@ -2463,15 +1975,6 @@ vm_object_destroy( object->named = FALSE; object->alive = FALSE; - if (object->hashed) { - lck_mtx_t *lck; - /* - * Rip out the pager from the vm_object now... - */ - lck = vm_object_hash_lock_spin(object->pager); - vm_object_remove(object); - vm_object_hash_unlock(lck); - } old_pager = object->pager; object->pager = MEMORY_OBJECT_NULL; if (old_pager != MEMORY_OBJECT_NULL) @@ -2489,7 +1992,7 @@ vm_object_destroy( * Terminate the object now. */ if (old_pager != MEMORY_OBJECT_NULL) { - vm_object_release_pager(old_pager, object->hashed); + vm_object_release_pager(old_pager); /* * JMM - Release the caller's reference. This assumes the @@ -2504,71 +2007,6 @@ vm_object_destroy( return(KERN_SUCCESS); } - -#if VM_OBJECT_CACHE - -#define VM_OBJ_DEACT_ALL_STATS DEBUG -#if VM_OBJ_DEACT_ALL_STATS -uint32_t vm_object_deactivate_all_pages_batches = 0; -uint32_t vm_object_deactivate_all_pages_pages = 0; -#endif /* VM_OBJ_DEACT_ALL_STATS */ -/* - * vm_object_deactivate_all_pages - * - * Deactivate all pages in the specified object. (Keep its pages - * in memory even though it is no longer referenced.) - * - * The object must be locked. - */ -static void -vm_object_deactivate_all_pages( - vm_object_t object) -{ - vm_page_t p; - int loop_count; -#if VM_OBJ_DEACT_ALL_STATS - int pages_count; -#endif /* VM_OBJ_DEACT_ALL_STATS */ -#define V_O_D_A_P_MAX_BATCH 256 - - loop_count = BATCH_LIMIT(V_O_D_A_P_MAX_BATCH); -#if VM_OBJ_DEACT_ALL_STATS - pages_count = 0; -#endif /* VM_OBJ_DEACT_ALL_STATS */ - vm_page_lock_queues(); - vm_page_queue_iterate(&object->memq, p, vm_page_t, listq) { - if (--loop_count == 0) { -#if VM_OBJ_DEACT_ALL_STATS - hw_atomic_add(&vm_object_deactivate_all_pages_batches, - 1); - hw_atomic_add(&vm_object_deactivate_all_pages_pages, - pages_count); - pages_count = 0; -#endif /* VM_OBJ_DEACT_ALL_STATS */ - lck_mtx_yield(&vm_page_queue_lock); - loop_count = BATCH_LIMIT(V_O_D_A_P_MAX_BATCH); - } - if (!p->busy && (p->vm_page_q_state != VM_PAGE_ON_THROTTLED_Q)) { -#if VM_OBJ_DEACT_ALL_STATS - pages_count++; -#endif /* VM_OBJ_DEACT_ALL_STATS */ - vm_page_deactivate(p); - } - } -#if VM_OBJ_DEACT_ALL_STATS - if (pages_count) { - hw_atomic_add(&vm_object_deactivate_all_pages_batches, 1); - hw_atomic_add(&vm_object_deactivate_all_pages_pages, - pages_count); - pages_count = 0; - } -#endif /* VM_OBJ_DEACT_ALL_STATS */ - vm_page_unlock_queues(); -} -#endif /* VM_OBJECT_CACHE */ - - - /* * The "chunk" macros are used by routines below when looking for pages to deactivate. These * exist because of the need to handle shadow chains. When deactivating pages, we only @@ -2743,7 +2181,8 @@ deactivate_pages_in_object( MARK_PAGE_HANDLED(*chunk_state, p); - if (( !VM_PAGE_WIRED(m)) && (!m->private) && (!m->gobbled) && (!m->busy) && (!m->laundry)) { + if (( !VM_PAGE_WIRED(m)) && (!m->private) && (!m->gobbled) && (!m->busy) && + (!m->laundry) && (!m->cleaning) && !(m->free_when_done)) { int clear_refmod; int pmap_options; @@ -3152,7 +2591,7 @@ vm_object_reuse_pages( * remove access to all pages in shadowed objects. * * The object must *not* be locked. The object must - * be temporary/internal. + * be internal. * * If pmap is not NULL, this routine assumes that * the only mappings for the pages are in that @@ -4431,10 +3870,6 @@ vm_object_shadow( * [Furthermore, each routine must cope with the simultaneous * or previous operations of the others.] * - * In addition to the lock on the object, the vm_object_hash_lock - * governs the associations. References gained through the - * association require use of the hash lock. - * * Because the pager field may be cleared spontaneously, it * cannot be used to determine whether a memory object has * ever been associated with a particular vm_object. [This @@ -4453,251 +3888,85 @@ vm_object_shadow( /* - * Routine: vm_object_enter + * Routine: vm_object_memory_object_associate * Purpose: - * Find a VM object corresponding to the given - * pager; if no such object exists, create one, - * and initialize the pager. + * Associate a VM object to the given pager. + * If a VM object is not provided, create one. + * Initialize the pager. */ vm_object_t -vm_object_enter( +vm_object_memory_object_associate( memory_object_t pager, + vm_object_t object, vm_object_size_t size, - boolean_t internal, - boolean_t init, boolean_t named) { - vm_object_t object; - vm_object_t new_object; - boolean_t must_init; - vm_object_hash_entry_t entry, new_entry; - uint32_t try_failed_count = 0; - lck_mtx_t *lck; + memory_object_control_t control; - if (pager == MEMORY_OBJECT_NULL) - return(vm_object_allocate(size)); + assert(pager != MEMORY_OBJECT_NULL); - new_object = VM_OBJECT_NULL; - new_entry = VM_OBJECT_HASH_ENTRY_NULL; - must_init = init; + if (object != VM_OBJECT_NULL) { + assert(object->internal); + assert(object->pager_created); + assert(!object->pager_initialized); + assert(!object->pager_ready); + } else { + object = vm_object_allocate(size); + assert(object != VM_OBJECT_NULL); + object->internal = FALSE; + object->pager_trusted = FALSE; + /* copy strategy invalid until set by memory manager */ + object->copy_strategy = MEMORY_OBJECT_COPY_INVALID; + } /* - * Look for an object associated with this port. + * Allocate request port. */ -Retry: - lck = vm_object_hash_lock_spin(pager); - do { - entry = vm_object_hash_lookup(pager, FALSE); - - if (entry == VM_OBJECT_HASH_ENTRY_NULL) { - if (new_object == VM_OBJECT_NULL) { - /* - * We must unlock to create a new object; - * if we do so, we must try the lookup again. - */ - vm_object_hash_unlock(lck); - assert(new_entry == VM_OBJECT_HASH_ENTRY_NULL); - new_entry = vm_object_hash_entry_alloc(pager); - new_object = vm_object_allocate(size); - /* - * Set new_object->hashed now, while noone - * knows about this object yet and we - * don't need to lock it. Once it's in - * the hash table, we would have to lock - * the object to set its "hashed" bit and - * we can't lock the object while holding - * the hash lock as a spinlock... - */ - new_object->hashed = TRUE; - lck = vm_object_hash_lock_spin(pager); - } else { - /* - * Lookup failed twice, and we have something - * to insert; set the object. - */ - /* - * We can't lock the object here since we're - * holding the hash lock as a spin lock. - * We've already pre-set "new_object->hashed" - * when we created "new_object" above, so we - * won't need to modify the object in - * vm_object_hash_insert(). - */ - assert(new_object->hashed); - vm_object_hash_insert(new_entry, new_object); - entry = new_entry; - new_entry = VM_OBJECT_HASH_ENTRY_NULL; - new_object = VM_OBJECT_NULL; - must_init = TRUE; - } - } else if (entry->object == VM_OBJECT_NULL) { - /* - * If a previous object is being terminated, - * we must wait for the termination message - * to be queued (and lookup the entry again). - */ - entry->waiting = TRUE; - entry = VM_OBJECT_HASH_ENTRY_NULL; - assert_wait((event_t) pager, THREAD_UNINT); - vm_object_hash_unlock(lck); - - thread_block(THREAD_CONTINUE_NULL); - lck = vm_object_hash_lock_spin(pager); - } - } while (entry == VM_OBJECT_HASH_ENTRY_NULL); - - object = entry->object; - assert(object != VM_OBJECT_NULL); - - if (!must_init) { - if ( !vm_object_lock_try(object)) { - - vm_object_hash_unlock(lck); - - try_failed_count++; - mutex_pause(try_failed_count); /* wait a bit */ - goto Retry; - } - assert(!internal || object->internal); -#if VM_OBJECT_CACHE - if (object->ref_count == 0) { - if ( !vm_object_cache_lock_try()) { - - vm_object_hash_unlock(lck); - vm_object_unlock(object); - - try_failed_count++; - mutex_pause(try_failed_count); /* wait a bit */ - goto Retry; - } - XPR(XPR_VM_OBJECT_CACHE, - "vm_object_enter: removing %x from cache, head (%x, %x)\n", - object, - vm_object_cached_list.next, - vm_object_cached_list.prev, 0,0); - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - - vm_object_cache_unlock(); - } -#endif - if (named) { - assert(!object->named); - object->named = TRUE; - } - vm_object_lock_assert_exclusive(object); - object->ref_count++; - vm_object_res_reference(object); - - vm_object_hash_unlock(lck); - vm_object_unlock(object); - - VM_STAT_INCR(hits); - } else - vm_object_hash_unlock(lck); - assert(object->ref_count > 0); + control = memory_object_control_allocate(object); + assert (control != MEMORY_OBJECT_CONTROL_NULL); - VM_STAT_INCR(lookups); + vm_object_lock(object); - XPR(XPR_VM_OBJECT, - "vm_o_enter: pager 0x%x obj 0x%x must_init %d\n", - pager, object, must_init, 0, 0); + assert(!object->pager_ready); + assert(!object->pager_initialized); + assert(object->pager == NULL); + assert(object->pager_control == NULL); /* - * If we raced to create a vm_object but lost, let's - * throw away ours. + * Copy the reference we were given. */ - if (new_object != VM_OBJECT_NULL) { - /* - * Undo the pre-setting of "new_object->hashed" before - * deallocating "new_object", since we did not insert it - * into the hash table after all. - */ - assert(new_object->hashed); - new_object->hashed = FALSE; - vm_object_deallocate(new_object); - } - - if (new_entry != VM_OBJECT_HASH_ENTRY_NULL) - vm_object_hash_entry_free(new_entry); - - if (must_init) { - memory_object_control_t control; - - /* - * Allocate request port. - */ - - control = memory_object_control_allocate(object); - assert (control != MEMORY_OBJECT_CONTROL_NULL); - - vm_object_lock(object); - assert(object != kernel_object); - - /* - * Copy the reference we were given. - */ - - memory_object_reference(pager); - object->pager_created = TRUE; - object->pager = pager; - object->internal = internal; - object->pager_trusted = internal; - if (!internal) { - /* copy strategy invalid until set by memory manager */ - object->copy_strategy = MEMORY_OBJECT_COPY_INVALID; - } - object->pager_control = control; - object->pager_ready = FALSE; - - vm_object_unlock(object); - - /* - * Let the pager know we're using it. - */ - - (void) memory_object_init(pager, - object->pager_control, - PAGE_SIZE); - - vm_object_lock(object); - if (named) - object->named = TRUE; - if (internal) { - vm_object_lock_assert_exclusive(object); - object->pager_ready = TRUE; - vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY); - } + memory_object_reference(pager); + object->pager_created = TRUE; + object->pager = pager; + object->pager_control = control; + object->pager_ready = FALSE; - object->pager_initialized = TRUE; - vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED); - } else { - vm_object_lock(object); - } + vm_object_unlock(object); /* - * [At this point, the object must be locked] + * Let the pager know we're using it. */ - /* - * Wait for the work above to be done by the first - * thread to map this object. - */ + (void) memory_object_init(pager, + object->pager_control, + PAGE_SIZE); - while (!object->pager_initialized) { - vm_object_sleep(object, - VM_OBJECT_EVENT_INITIALIZED, - THREAD_UNINT); + vm_object_lock(object); + if (named) + object->named = TRUE; + if (object->internal) { + object->pager_ready = TRUE; + vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY); } + + object->pager_initialized = TRUE; + vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED); + vm_object_unlock(object); - XPR(XPR_VM_OBJECT, - "vm_object_enter: vm_object %x, memory_object %x, internal %d\n", - object, object->pager, internal, 0,0); - return(object); + return object; } /* @@ -4719,8 +3988,6 @@ vm_object_compressor_pager_create( vm_object_t object) { memory_object_t pager; - vm_object_hash_entry_t entry; - lck_mtx_t *lck; vm_object_t pager_object = VM_OBJECT_NULL; assert(object != kernel_object); @@ -4775,8 +4042,6 @@ vm_object_compressor_pager_create( * user will ever map this object. */ { - assert(object->temporary); - /* create our new memory object */ assert((uint32_t) (object->vo_size/PAGE_SIZE) == (object->vo_size/PAGE_SIZE)); @@ -4790,22 +4055,16 @@ vm_object_compressor_pager_create( } } - entry = vm_object_hash_entry_alloc(pager); - - vm_object_lock(object); - lck = vm_object_hash_lock_spin(pager); - vm_object_hash_insert(entry, object); - vm_object_hash_unlock(lck); - vm_object_unlock(object); - /* * A reference was returned by * memory_object_create(), and it is - * copied by vm_object_enter(). + * copied by vm_object_memory_object_associate(). */ - pager_object = vm_object_enter(pager, object->vo_size, TRUE, TRUE, FALSE); - + pager_object = vm_object_memory_object_associate(pager, + object, + object->vo_size, + FALSE); if (pager_object != object) { panic("vm_object_compressor_pager_create: mismatch (pager: %p, pager_object: %p, orig_object: %p, orig_object size: 0x%llx)\n", pager, pager_object, object, (uint64_t) object->vo_size); } @@ -4823,30 +4082,6 @@ vm_object_compressor_pager_create( vm_object_paging_end(object); } -/* - * Routine: vm_object_remove - * Purpose: - * Eliminate the pager/object association - * for this pager. - * Conditions: - * The object cache must be locked. - */ -__private_extern__ void -vm_object_remove( - vm_object_t object) -{ - memory_object_t pager; - - if ((pager = object->pager) != MEMORY_OBJECT_NULL) { - vm_object_hash_entry_t entry; - - entry = vm_object_hash_lookup(pager, FALSE); - if (entry != VM_OBJECT_HASH_ENTRY_NULL) - entry->object = VM_OBJECT_NULL; - } - -} - /* * Global variables for vm_object_collapse(): * @@ -4859,8 +4094,6 @@ static long object_bypasses = 0; static boolean_t vm_object_collapse_allowed = TRUE; static boolean_t vm_object_bypass_allowed = TRUE; -unsigned long vm_object_collapse_encrypted = 0; - void vm_object_do_collapse_compressor(vm_object_t object, vm_object_t backing_object); void @@ -4993,18 +4226,6 @@ vm_object_do_collapse( if (p->offset < backing_offset || new_offset >= size) { VM_PAGE_FREE(p); } else { - /* - * ENCRYPTED SWAP: - * The encryption key includes the "pager" and the - * "paging_offset". These will not change during the - * object collapse, so we can just move an encrypted - * page from one object to the other in this case. - * We can't decrypt the page here, since we can't drop - * the object lock. - */ - if (p->encrypted) { - vm_object_collapse_encrypted++; - } pp = vm_page_lookup(object, new_offset); if (pp == VM_PAGE_NULL) { @@ -5024,8 +4245,7 @@ vm_object_do_collapse( * Move the backing object's page * up. */ - vm_page_rename(p, object, new_offset, - TRUE); + vm_page_rename(p, object, new_offset); } } else { assert(! pp->absent); @@ -5048,7 +4268,6 @@ vm_object_do_collapse( vm_object_do_collapse_compressor(object, backing_object); } else if (backing_object->pager != MEMORY_OBJECT_NULL) { - vm_object_hash_entry_t entry; assert((!object->pager_created && (object->pager == MEMORY_OBJECT_NULL)) || @@ -5068,17 +4287,6 @@ vm_object_do_collapse( assert(object->pager == NULL); object->pager = backing_object->pager; - if (backing_object->hashed) { - lck_mtx_t *lck; - - lck = vm_object_hash_lock_spin(backing_object->pager); - entry = vm_object_hash_lookup(object->pager, FALSE); - assert(entry != VM_OBJECT_HASH_ENTRY_NULL); - entry->object = object; - vm_object_hash_unlock(lck); - - object->hashed = TRUE; - } object->pager_created = backing_object->pager_created; object->pager_control = backing_object->pager_control; object->pager_ready = backing_object->pager_ready; @@ -5966,12 +5174,6 @@ vm_object_populate_with_private( } VM_PAGE_SET_PHYS_PAGE(m, base_page); } - if (m->encrypted) { - /* - * we should never see this on a ficticious or private page - */ - panic("vm_object_populate_with_private - %p encrypted", m); - } } else { while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL) @@ -6010,90 +5212,6 @@ vm_object_populate_with_private( return KERN_SUCCESS; } -/* - * memory_object_free_from_cache: - * - * Walk the vm_object cache list, removing and freeing vm_objects - * which are backed by the pager identified by the caller, (pager_ops). - * Remove up to "count" objects, if there are that may available - * in the cache. - * - * Walk the list at most once, return the number of vm_objects - * actually freed. - */ - -__private_extern__ kern_return_t -memory_object_free_from_cache( - __unused host_t host, - __unused memory_object_pager_ops_t pager_ops, - int *count) -{ -#if VM_OBJECT_CACHE - int object_released = 0; - - vm_object_t object = VM_OBJECT_NULL; - vm_object_t shadow; - -/* - if(host == HOST_NULL) - return(KERN_INVALID_ARGUMENT); -*/ - - try_again: - vm_object_cache_lock(); - - queue_iterate(&vm_object_cached_list, object, - vm_object_t, cached_list) { - if (object->pager && - (pager_ops == object->pager->mo_pager_ops)) { - vm_object_lock(object); - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - - vm_object_cache_unlock(); - /* - * Since this object is in the cache, we know - * that it is initialized and has only a pager's - * (implicit) reference. Take a reference to avoid - * recursive deallocations. - */ - - assert(object->pager_initialized); - assert(object->ref_count == 0); - vm_object_lock_assert_exclusive(object); - object->ref_count++; - - /* - * Terminate the object. - * If the object had a shadow, we let - * vm_object_deallocate deallocate it. - * "pageout" objects have a shadow, but - * maintain a "paging reference" rather - * than a normal reference. - * (We are careful here to limit recursion.) - */ - shadow = object->pageout?VM_OBJECT_NULL:object->shadow; - - if ((vm_object_terminate(object) == KERN_SUCCESS) - && (shadow != VM_OBJECT_NULL)) { - vm_object_deallocate(shadow); - } - - if(object_released++ == *count) - return KERN_SUCCESS; - goto try_again; - } - } - vm_object_cache_unlock(); - *count = object_released; -#else - *count = 0; -#endif - return KERN_SUCCESS; -} - - kern_return_t memory_object_create_named( @@ -6102,24 +5220,17 @@ memory_object_create_named( memory_object_control_t *control) { vm_object_t object; - vm_object_hash_entry_t entry; - lck_mtx_t *lck; *control = MEMORY_OBJECT_CONTROL_NULL; if (pager == MEMORY_OBJECT_NULL) return KERN_INVALID_ARGUMENT; - lck = vm_object_hash_lock_spin(pager); - entry = vm_object_hash_lookup(pager, FALSE); - - if ((entry != VM_OBJECT_HASH_ENTRY_NULL) && - (entry->object != VM_OBJECT_NULL)) { - if (entry->object->named == TRUE) - panic("memory_object_create_named: caller already holds the right"); } - vm_object_hash_unlock(lck); - - if ((object = vm_object_enter(pager, size, FALSE, FALSE, TRUE)) == VM_OBJECT_NULL) { - return(KERN_INVALID_OBJECT); + object = vm_object_memory_object_associate(pager, + VM_OBJECT_NULL, + size, + TRUE); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_OBJECT; } /* wait for object (if any) to be ready */ @@ -6180,24 +5291,6 @@ restart: vm_object_unlock(object); return KERN_SUCCESS; } -#if VM_OBJECT_CACHE - if ((object->ref_count == 0) && (!object->terminating)) { - if (!vm_object_cache_lock_try()) { - vm_object_unlock(object); - goto restart; - } - queue_remove(&vm_object_cached_list, object, - vm_object_t, cached_list); - vm_object_cached_count--; - XPR(XPR_VM_OBJECT_CACHE, - "memory_object_recover_named: removing %X, head (%X, %X)\n", - object, - vm_object_cached_list.next, - vm_object_cached_list.prev, 0,0); - - vm_object_cache_unlock(); - } -#endif object->named = TRUE; vm_object_lock_assert_exclusive(object); object->ref_count++; @@ -6604,6 +5697,16 @@ vm_object_purgable_control( return KERN_SUCCESS; } + if (control == VM_PURGABLE_SET_STATE && + object->purgeable_only_by_kernel) { + return KERN_PROTECTION_FAILURE; + } + + if (control != VM_PURGABLE_SET_STATE && + control != VM_PURGABLE_SET_STATE_FROM_KERNEL) { + return KERN_INVALID_ARGUMENT; + } + if ((*state) & VM_PURGABLE_DEBUG_EMPTY) { object->volatile_empty = TRUE; } @@ -6625,6 +5728,11 @@ vm_object_purgable_control( switch (new_state) { case VM_PURGABLE_DENY: + /* + * Attempting to convert purgeable memory to non-purgeable: + * not allowed. + */ + return KERN_INVALID_ARGUMENT; case VM_PURGABLE_NONVOLATILE: object->purgable = new_state; @@ -7079,31 +6187,6 @@ vm_object_reference( vm_object_unlock(object); } -#ifdef MACH_BSD -/* - * Scale the vm_object_cache - * This is required to make sure that the vm_object_cache is big - * enough to effectively cache the mapped file. - * This is really important with UBC as all the regular file vnodes - * have memory object associated with them. Havving this cache too - * small results in rapid reclaim of vnodes and hurts performance a LOT! - * - * This is also needed as number of vnodes can be dynamically scaled. - */ -kern_return_t -adjust_vm_object_cache( - __unused vm_size_t oval, - __unused vm_size_t nval) -{ -#if VM_OBJECT_CACHE - vm_object_cached_max = nval; - vm_object_cache_trim(FALSE); -#endif - return (KERN_SUCCESS); -} -#endif /* MACH_BSD */ - - /* * vm_object_transpose * @@ -7126,8 +6209,6 @@ vm_object_transpose( boolean_t object1_locked, object2_locked; vm_page_t page; vm_object_offset_t page_offset; - lck_mtx_t *hash_lck; - vm_object_hash_entry_t hash_entry; tmp_object = VM_OBJECT_NULL; object1_locked = FALSE; object2_locked = FALSE; @@ -7233,7 +6314,7 @@ vm_object_transpose( */ while (!vm_page_queue_empty(&object2->memq)) { page = (vm_page_t) vm_page_queue_first(&object2->memq); - vm_page_rename(page, object1, page->offset, FALSE); + vm_page_rename(page, object1, page->offset); } assert(vm_page_queue_empty(&object2->memq)); } else if (object2->phys_contiguous || vm_page_queue_empty(&object2->memq)) { @@ -7244,7 +6325,7 @@ vm_object_transpose( */ while (!vm_page_queue_empty(&object1->memq)) { page = (vm_page_t) vm_page_queue_first(&object1->memq); - vm_page_rename(page, object2, page->offset, FALSE); + vm_page_rename(page, object2, page->offset); } assert(vm_page_queue_empty(&object1->memq)); } else { @@ -7260,7 +6341,7 @@ vm_object_transpose( /* transfer object2's pages to object1 */ while (!vm_page_queue_empty(&object2->memq)) { page = (vm_page_t) vm_page_queue_first(&object2->memq); - vm_page_rename(page, object1, page->offset, FALSE); + vm_page_rename(page, object1, page->offset); } assert(vm_page_queue_empty(&object2->memq)); /* transfer tmp_object's pages to object2 */ @@ -7285,6 +6366,8 @@ MACRO_END assert(object1->vo_size == object2->vo_size); /* "memq_hint" was updated above when transposing pages */ /* "ref_count" refers to the object not its contents */ + assert(object1->ref_count >= 1); + assert(object2->ref_count >= 1); #if TASK_SWAPPER /* "res_count" refers to the object not its contents */ #endif @@ -7323,7 +6406,6 @@ MACRO_END __TRANSPOSE_FIELD(pager_trusted); __TRANSPOSE_FIELD(can_persist); __TRANSPOSE_FIELD(internal); - __TRANSPOSE_FIELD(temporary); __TRANSPOSE_FIELD(private); __TRANSPOSE_FIELD(pageout); /* "alive" should be set */ @@ -7334,11 +6416,22 @@ MACRO_END assert(object2->purgable == VM_PURGABLE_DENY); /* "shadowed" refers to the the object not its contents */ __TRANSPOSE_FIELD(purgeable_when_ripe); - __TRANSPOSE_FIELD(advisory_pageout); __TRANSPOSE_FIELD(true_share); /* "terminating" should not be set */ assert(!object1->terminating); assert(!object2->terminating); + /* transfer "named" reference if needed */ + if (object1->named && !object2->named) { + assert(object1->ref_count >= 2); + assert(object2->ref_count >= 1); + object1->ref_count--; + object2->ref_count++; + } else if (!object1->named && object2->named) { + assert(object1->ref_count >= 1); + assert(object2->ref_count >= 2); + object1->ref_count++; + object2->ref_count--; + } __TRANSPOSE_FIELD(named); /* "shadow_severed" refers to the object not its contents */ __TRANSPOSE_FIELD(phys_contiguous); @@ -7349,36 +6442,15 @@ MACRO_END /* "cached_list.prev" should be NULL */ assert(object1->cached_list.prev == NULL); assert(object2->cached_list.prev == NULL); - /* "msr_q" is linked to the object not its contents */ - assert(queue_empty(&object1->msr_q)); - assert(queue_empty(&object2->msr_q)); __TRANSPOSE_FIELD(last_alloc); __TRANSPOSE_FIELD(sequential); __TRANSPOSE_FIELD(pages_created); __TRANSPOSE_FIELD(pages_used); __TRANSPOSE_FIELD(scan_collisions); __TRANSPOSE_FIELD(cow_hint); -#if MACH_ASSERT - __TRANSPOSE_FIELD(paging_object); -#endif __TRANSPOSE_FIELD(wimg_bits); __TRANSPOSE_FIELD(set_cache_attr); __TRANSPOSE_FIELD(code_signed); - if (object1->hashed) { - hash_lck = vm_object_hash_lock_spin(object2->pager); - hash_entry = vm_object_hash_lookup(object2->pager, FALSE); - assert(hash_entry != VM_OBJECT_HASH_ENTRY_NULL); - hash_entry->object = object2; - vm_object_hash_unlock(hash_lck); - } - if (object2->hashed) { - hash_lck = vm_object_hash_lock_spin(object1->pager); - hash_entry = vm_object_hash_lookup(object1->pager, FALSE); - assert(hash_entry != VM_OBJECT_HASH_ENTRY_NULL); - hash_entry->object = object1; - vm_object_hash_unlock(hash_lck); - } - __TRANSPOSE_FIELD(hashed); object1->transposed = TRUE; object2->transposed = TRUE; __TRANSPOSE_FIELD(mapping_in_progress); @@ -7445,7 +6517,6 @@ done: * */ extern int speculative_reads_disabled; -extern int ignore_is_ssd; /* * Try to always keep these values an even multiple of PAGE_SIZE. We use these values @@ -7454,7 +6525,11 @@ extern int ignore_is_ssd; * that could give us non-page-size aligned values if we start out with values that * are odd multiples of PAGE_SIZE. */ +#if CONFIG_EMBEDDED + unsigned int preheat_max_bytes = (1024 * 512); +#else /* CONFIG_EMBEDDED */ unsigned int preheat_max_bytes = MAX_UPL_TRANSFER_BYTES; +#endif /* CONFIG_EMBEDDED */ unsigned int preheat_min_bytes = (1024 * 32); @@ -7512,12 +6587,12 @@ vm_object_cluster_size(vm_object_t object, vm_object_offset_t *start, if (object->pager == MEMORY_OBJECT_NULL) goto out; /* pager is gone for this object, nothing more to do */ - if (!ignore_is_ssd) - vnode_pager_get_isSSD(object->pager, &isSSD); + vnode_pager_get_isSSD(object->pager, &isSSD); min_ph_size = round_page(preheat_min_bytes); max_ph_size = round_page(preheat_max_bytes); +#if !CONFIG_EMBEDDED if (isSSD) { min_ph_size /= 2; max_ph_size /= 8; @@ -7530,6 +6605,7 @@ vm_object_cluster_size(vm_object_t object, vm_object_offset_t *start, max_ph_size = trunc_page(max_ph_size); } } +#endif /* !CONFIG_EMBEDDED */ if (min_ph_size < PAGE_SIZE) min_ph_size = PAGE_SIZE; @@ -7884,47 +6960,12 @@ vm_object_page_op( PAGE_WAKEUP(dst_page); } } - - if (dst_page->encrypted) { - /* - * ENCRYPTED SWAP: - * We need to decrypt this encrypted page before the - * caller can access its contents. - * But if the caller really wants to access the page's - * contents, they have to keep the page "busy". - * Otherwise, the page could get recycled or re-encrypted - * at any time. - */ - if ((ops & UPL_POP_SET) && (ops & UPL_POP_BUSY) && - dst_page->busy) { - /* - * The page is stable enough to be accessed by - * the caller, so make sure its contents are - * not encrypted. - */ - vm_page_decrypt(dst_page, 0); - } else { - /* - * The page is not busy, so don't bother - * decrypting it, since anything could - * happen to it between now and when the - * caller wants to access it. - * We should not give the caller access - * to this page. - */ - assert(!phys_entry); - } - } - if (phys_entry) { /* * The physical page number will remain valid * only if the page is kept busy. - * ENCRYPTED SWAP: make sure we don't let the - * caller access an encrypted page. */ assert(dst_page->busy); - assert(!dst_page->encrypted); *phys_entry = VM_PAGE_GET_PHYS_PAGE(dst_page); } @@ -8140,6 +7181,20 @@ vm_object_lock_shared(vm_object_t object) lck_rw_lock_shared(&object->Lock); } +boolean_t +vm_object_lock_yield_shared(vm_object_t object) +{ + boolean_t retval = FALSE, force_yield = FALSE;; + + vm_object_lock_assert_shared(object); + + force_yield = vm_object_lock_avoid(object); + + retval = lck_rw_lock_yield_shared(&object->Lock, force_yield); + + return (retval); +} + boolean_t vm_object_lock_try_shared(vm_object_t object) { @@ -8456,7 +7511,6 @@ vm_object_pageout( { vm_page_t p, next; struct vm_pageout_queue *iq; - boolean_t need_unlock = TRUE; if (!VM_CONFIG_COMPRESSOR_IS_PRESENT) return; @@ -8500,7 +7554,6 @@ ReScan: assert(p->vm_page_q_state != VM_PAGE_ON_FREE_Q); if ((p->vm_page_q_state == VM_PAGE_ON_THROTTLED_Q) || - p->encrypted_cleaning || p->cleaning || p->laundry || p->busy || @@ -8517,7 +7570,6 @@ ReScan: /* Throw to the pageout queue */ vm_page_lockspin_queues(); - need_unlock = TRUE; if (vm_compressor_low_on_space()) { vm_page_unlock_queues(); @@ -8578,16 +7630,12 @@ ReScan: VM_PAGE_FREE(p); continue; } - vm_page_queues_remove(p, TRUE); - if (vm_pageout_cluster(p, FALSE, TRUE)) - need_unlock = FALSE; - - if (need_unlock == TRUE) - vm_page_unlock_queues(); + vm_pageout_cluster(p); + + vm_page_unlock_queues(); } - vm_object_unlock(object); } diff --git a/osfmk/vm/vm_object.h b/osfmk/vm/vm_object.h index daef84711..1ef57792d 100644 --- a/osfmk/vm/vm_object.h +++ b/osfmk/vm/vm_object.h @@ -171,7 +171,8 @@ struct vm_object { int ref_count; /* Number of references */ unsigned int resident_page_count; /* number of resident pages */ - unsigned int wired_page_count; /* number of wired pages */ + const unsigned int wired_page_count; /* number of wired pages + use VM_OBJECT_WIRED_PAGE_UPDATE macros to update */ unsigned int reusable_page_count; struct vm_object *copy; /* Object that should receive @@ -255,15 +256,6 @@ struct vm_object { * therefore, managed by the * default memory manger) */ - /* boolean_t */ temporary:1, /* Permanent objects may be - * changed externally by the - * memory manager, and changes - * made in memory must be - * reflected back to the memory - * manager. Temporary objects - * lack both of these - * characteristics. - */ /* boolean_t */ private:1, /* magic device_pager object, * holds private pages only */ /* boolean_t */ pageout:1, /* pageout object. contains @@ -274,16 +266,11 @@ struct vm_object { /* boolean_t */ purgable:2, /* Purgable state. See * VM_PURGABLE_* */ + /* boolean_t */ purgeable_only_by_kernel:1, /* boolean_t */ purgeable_when_ripe:1, /* Purgeable when a token * becomes ripe. */ /* boolean_t */ shadowed:1, /* Shadow may exist */ - /* boolean_t */ advisory_pageout:1, - /* Instead of sending page - * via OOL, just notify - * pager that the kernel - * wants to discard it, page - * remains in object */ /* boolean_t */ true_share:1, /* This object is mapped * in more than one place @@ -323,22 +310,19 @@ struct vm_object { * memory rules w.r.t pmap * access bits. */ - /* boolean_t */ nophyscache:1; + /* boolean_t */ nophyscache:1, /* When mapped at the * pmap level, don't allow * primary caching. (for * I/O) */ + /* boolean_t */ _object5_unused_bits:1; queue_chain_t cached_list; /* Attachment point for the * list of objects cached as a * result of their can_persist * value */ - - queue_head_t msr_q; /* memory object synchronise - request queue */ - /* * the following fields are not protected by any locks * they are updated via atomic compare and swap @@ -350,19 +334,12 @@ struct vm_object { uint32_t pages_used; vm_offset_t cow_hint; /* last page present in */ /* shadow but not in object */ -#if MACH_ASSERT - struct vm_object *paging_object; /* object which pages to be - * swapped out are temporary - * put in current object - */ -#endif /* hold object lock when altering */ unsigned int wimg_bits:8, /* cache WIMG bits */ code_signed:1, /* pages are signed and should be validated; the signatures are stored with the pager */ - hashed:1, /* object/pager entered in hash */ transposed:1, /* object was transposed with another */ mapping_in_progress:1, /* pager being mapped/unmapped */ phantom_isssd:1, @@ -375,6 +352,7 @@ struct vm_object { purgeable_queue_type:2, purgeable_queue_group:3, io_tracking:1, + no_tag_update:1, /* */ #if CONFIG_SECLUDED_MEMORY eligible_for_secluded:1, can_grab_secluded:1, @@ -434,45 +412,21 @@ unsigned int vm_object_absent_max; /* maximum number of absent pages # define VM_MSYNC_SYNCHRONIZING 1 # define VM_MSYNC_DONE 2 -struct msync_req { - queue_chain_t msr_q; /* object request queue */ - queue_chain_t req_q; /* vm_msync request queue */ - unsigned int flag; - vm_object_offset_t offset; - vm_object_size_t length; - vm_object_t object; /* back pointer */ - decl_lck_mtx_data(, msync_req_lock) /* Lock for this structure */ -}; - -typedef struct msync_req *msync_req_t; -#define MSYNC_REQ_NULL ((msync_req_t) 0) - extern lck_grp_t vm_map_lck_grp; extern lck_attr_t vm_map_lck_attr; -/* - * Macros to allocate and free msync_reqs - */ -#define msync_req_alloc(msr) \ - MACRO_BEGIN \ - (msr) = (msync_req_t)kalloc(sizeof(struct msync_req)); \ - lck_mtx_init(&(msr)->msync_req_lock, &vm_map_lck_grp, &vm_map_lck_attr); \ - msr->flag = VM_MSYNC_INITIALIZED; \ - MACRO_END - -#define msync_req_free(msr) \ - MACRO_BEGIN \ - lck_mtx_destroy(&(msr)->msync_req_lock, &vm_map_lck_grp); \ - kfree((msr), sizeof(struct msync_req)); \ - MACRO_END - -#define msr_lock(msr) lck_mtx_lock(&(msr)->msync_req_lock) -#define msr_unlock(msr) lck_mtx_unlock(&(msr)->msync_req_lock) +#ifndef VM_TAG_ACTIVE_UPDATE +#error VM_TAG_ACTIVE_UPDATE +#endif -#define VM_OBJECT_WIRED(object) \ +#define VM_OBJECT_WIRED(object, tag) \ MACRO_BEGIN \ - if ((object)->purgable == VM_PURGABLE_DENY) \ + assert(VM_KERN_MEMORY_NONE != (tag)); \ + assert(VM_KERN_MEMORY_NONE == (object)->wire_tag); \ + (object)->wire_tag = (tag); \ + if (!VM_TAG_ACTIVE_UPDATE \ + && ((object)->purgable == VM_PURGABLE_DENY)) \ { \ lck_spin_lock(&vm_objects_wired_lock); \ assert(!(object)->objq.next); \ @@ -482,17 +436,62 @@ extern lck_attr_t vm_map_lck_attr; } \ MACRO_END -#define VM_OBJECT_UNWIRED(object) \ - MACRO_BEGIN \ - (object)->wire_tag = VM_KERN_MEMORY_NONE; \ - if (((object)->purgable == VM_PURGABLE_DENY) && (object)->objq.next) \ - { \ - lck_spin_lock(&vm_objects_wired_lock); \ - queue_remove(&vm_objects_wired, (object), vm_object_t, objq); \ - lck_spin_unlock(&vm_objects_wired_lock); \ - } \ +#define VM_OBJECT_UNWIRED(object) \ + MACRO_BEGIN \ + if (!VM_TAG_ACTIVE_UPDATE \ + && ((object)->purgable == VM_PURGABLE_DENY) && (object)->objq.next) \ + { \ + lck_spin_lock(&vm_objects_wired_lock); \ + queue_remove(&vm_objects_wired, (object), vm_object_t, objq); \ + lck_spin_unlock(&vm_objects_wired_lock); \ + } \ + if (VM_KERN_MEMORY_NONE != (object)->wire_tag) { \ + vm_tag_update_size((object)->wire_tag, -ptoa_64((object)->wired_page_count)); \ + (object)->wire_tag = VM_KERN_MEMORY_NONE; \ + } \ + MACRO_END + +// These two macros start & end a C block +#define VM_OBJECT_WIRED_PAGE_UPDATE_START(object) \ + MACRO_BEGIN \ + { \ + int64_t __wireddelta = 0; vm_tag_t __waswired = (object)->wire_tag; + +#define VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag) \ + if (__wireddelta) { \ + boolean_t __overflow __assert_only = \ + os_add_overflow((object)->wired_page_count, __wireddelta, \ + (unsigned int *)(uintptr_t)&(object)->wired_page_count); \ + assert(!__overflow); \ + if (!(object)->pageout && !(object)->no_tag_update) { \ + if (__wireddelta > 0) { \ + assert (VM_KERN_MEMORY_NONE != (tag)); \ + if (VM_KERN_MEMORY_NONE == __waswired) { \ + VM_OBJECT_WIRED((object), (tag)); \ + } \ + vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta)); \ + } else if (VM_KERN_MEMORY_NONE != __waswired) { \ + assert (VM_KERN_MEMORY_NONE != (object)->wire_tag); \ + vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta)); \ + if (!(object)->wired_page_count) { \ + VM_OBJECT_UNWIRED((object)); \ + } \ + } \ + } \ + } \ + } \ MACRO_END +#define VM_OBJECT_WIRED_PAGE_COUNT(object, delta) \ + __wireddelta += delta; \ + +#define VM_OBJECT_WIRED_PAGE_ADD(object, m) \ + if (!m->private && !m->fictitious) __wireddelta++; + +#define VM_OBJECT_WIRED_PAGE_REMOVE(object, m) \ + if (!m->private && !m->fictitious) __wireddelta--; + + #define OBJECT_LOCK_SHARED 0 #define OBJECT_LOCK_EXCLUSIVE 1 @@ -510,6 +509,7 @@ extern boolean_t vm_object_lock_try(vm_object_t); extern boolean_t _vm_object_lock_try(vm_object_t); extern boolean_t vm_object_lock_avoid(vm_object_t); extern void vm_object_lock_shared(vm_object_t); +extern boolean_t vm_object_lock_yield_shared(vm_object_t); extern boolean_t vm_object_lock_try_shared(vm_object_t); extern void vm_object_unlock(vm_object_t); extern boolean_t vm_object_lock_upgrade(vm_object_t); @@ -760,7 +760,8 @@ __private_extern__ kern_return_t vm_object_upl_request( upl_t *upl, upl_page_info_t *page_info, unsigned int *count, - upl_control_flags_t flags); + upl_control_flags_t flags, + vm_tag_t tag); __private_extern__ kern_return_t vm_object_transpose( vm_object_t object1, @@ -795,11 +796,10 @@ __private_extern__ kern_return_t vm_object_lock_request( -__private_extern__ vm_object_t vm_object_enter( +__private_extern__ vm_object_t vm_object_memory_object_associate( memory_object_t pager, + vm_object_t object, vm_object_size_t size, - boolean_t internal, - boolean_t init, boolean_t check_named); diff --git a/osfmk/vm/vm_page.h b/osfmk/vm/vm_page.h index b34e386bb..559af3f0b 100644 --- a/osfmk/vm/vm_page.h +++ b/osfmk/vm/vm_page.h @@ -280,8 +280,6 @@ struct vm_page { start again at top of chain */ unusual:1, /* Page is absent, error, restart or page locked */ - encrypted:1, /* encrypted for secure swap (O) */ - encrypted_cleaning:1, /* encrypting page */ cs_validated:1, /* code-signing: page was checked */ cs_tainted:1, /* code-signing: page is tainted */ cs_nx:1, /* code-signing: page is nx */ @@ -289,10 +287,12 @@ struct vm_page { lopage:1, slid:1, written_by_kernel:1, /* page was written by kernel (i.e. decompressed) */ - __unused_object_bits:5; /* 5 bits available here */ + __unused_object_bits:7; /* 7 bits available here */ +#if !defined(__arm__) && !defined(__arm64__) ppnum_t phys_page; /* Physical address of page, passed * to pmap_enter (read-only) */ +#endif }; @@ -302,6 +302,36 @@ extern vm_page_t vm_page_array_beginning_addr; extern vm_page_t vm_page_array_ending_addr; +#if defined(__arm__) || defined(__arm64__) + +extern unsigned int vm_first_phys_ppnum; + +struct vm_page_with_ppnum { + struct vm_page vm_page_wo_ppnum; + + ppnum_t phys_page; +}; +typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t; + + +static inline ppnum_t VM_PAGE_GET_PHYS_PAGE(vm_page_t m) +{ + if (m >= vm_page_array_beginning_addr && m < vm_page_array_ending_addr) + return ((ppnum_t)((uintptr_t)(m - vm_page_array_beginning_addr) + vm_first_phys_ppnum)); + else + return (((vm_page_with_ppnum_t)m)->phys_page); +} + +#define VM_PAGE_SET_PHYS_PAGE(m, ppnum) \ + MACRO_BEGIN \ + if ((m) < vm_page_array_beginning_addr || (m) >= vm_page_array_ending_addr) \ + ((vm_page_with_ppnum_t)(m))->phys_page = ppnum; \ + assert(ppnum == VM_PAGE_GET_PHYS_PAGE(m)); \ + MACRO_END + +#define VM_PAGE_GET_COLOR(m) (VM_PAGE_GET_PHYS_PAGE(m) & vm_color_mask) + +#else /* defined(__arm__) || defined(__arm64__) */ struct vm_page_with_ppnum { @@ -316,21 +346,10 @@ typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t; (page)->phys_page = ppnum; \ MACRO_END +#define VM_PAGE_GET_CLUMP(m) ((VM_PAGE_GET_PHYS_PAGE(m)) >> vm_clump_shift) +#define VM_PAGE_GET_COLOR(m) ((VM_PAGE_GET_CLUMP(m)) & vm_color_mask) - - -#define DEBUG_ENCRYPTED_SWAP 1 -#if DEBUG_ENCRYPTED_SWAP -#define ASSERT_PAGE_DECRYPTED(page) \ - MACRO_BEGIN \ - if ((page)->encrypted) { \ - panic("VM page %p should not be encrypted here\n", \ - (page)); \ - } \ - MACRO_END -#else /* DEBUG_ENCRYPTED_SWAP */ -#define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted) -#endif /* DEBUG_ENCRYPTED_SWAP */ +#endif /* defined(__arm__) || defined(__arm64__) */ @@ -473,6 +492,122 @@ MACRO_BEGIN \ MACRO_END +/* + * These are helper macros for vm_page_queue_enter_clump to assist + * with conditional compilation (release / debug / development) + */ +#if DEVELOPMENT || DEBUG + +#define __DEBUG_CHECK_BUDDIES(__check, __prev, __p, field) \ +MACRO_BEGIN \ + if(__check) { /* if first forward buddy.. */ \ + if(__prev) { /* ..and if a backward buddy was found, verify link consistency */ \ + assert(__p == (vm_page_t) VM_PAGE_UNPACK_PTR(__prev->next)); \ + assert(__prev == (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__p->field.prev)); \ + } \ + __check=0; \ + } \ +MACRO_END + +#define __DEBUG_VERIFY_LINKS(__i, __first, __n_free, __last_next) \ +MACRO_BEGIN \ + vm_page_queue_entry_t __tmp; \ + for(__i=0, __tmp=__first; __i<__n_free; __i++) \ + __tmp=(vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__tmp->next); \ + assert(__tmp == __last_next); \ +MACRO_END + +#define __DEBUG_STAT_INCREMENT_INRANGE vm_clump_inrange++ +#define __DEBUG_STAT_INCREMENT_INSERTS vm_clump_inserts++ +#define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free) vm_clump_promotes+=__n_free + +#else + +#define __DEBUG_CHECK_BUDDIES(__check, __prev, __p, field) __check=1 +#define __DEBUG_VERIFY_LINKS(__i, __first, __n_free, __last_next) +#define __DEBUG_STAT_INCREMENT_INRANGE +#define __DEBUG_STAT_INCREMENT_INSERTS +#define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free) + +#endif /* if DEVELOPMENT || DEBUG */ + +/* + * Macro: vm_page_queue_enter_clump + * Function: + * Insert a new element into the free queue and clump pages within the same 16K boundary together + * + * Header: + * void vm_page_queue_enter_clump(q, elt, type, field) + * queue_t q; + * elt; + * is what's in our queue + * is the chain field in (*) + * Note: + * This should only be used with Method 2 queue iteration (element chains) + */ +#if defined(__x86_64__) +#define vm_page_queue_enter_clump(head, elt, type, field) \ +MACRO_BEGIN \ + ppnum_t __clump_num; \ + unsigned int __i, __n, __n_free=1, __check=1; \ + vm_page_queue_entry_t __prev=0, __next, __last, __last_next, __first, __first_prev, __head_next; \ + vm_page_t __p; \ + \ + /* if elt is part of vm_pages[] */ \ + if((elt) >= vm_page_array_beginning_addr && (elt) < vm_page_array_boundary) { \ + __first = __last = (vm_page_queue_entry_t) (elt); \ + __clump_num = VM_PAGE_GET_CLUMP(elt); \ + __n = VM_PAGE_GET_PHYS_PAGE(elt) & vm_clump_mask; \ + /* scan backward looking for a buddy page */ \ + for(__i=0, __p=(elt)-1; __i<__n && __p>=vm_page_array_beginning_addr; __i++, __p--) { \ + if(__p->vm_page_q_state == VM_PAGE_ON_FREE_Q && __clump_num == VM_PAGE_GET_CLUMP(__p)) { \ + if(__prev == 0) __prev = (vm_page_queue_entry_t) __p; \ + __first = (vm_page_queue_entry_t) __p; \ + __n_free++; \ + } \ + } \ + /* scan forward looking for a buddy page */ \ + for(__i=__n+1, __p=(elt)+1; __ivm_page_q_state == VM_PAGE_ON_FREE_Q && __clump_num == VM_PAGE_GET_CLUMP(__p)) { \ + __DEBUG_CHECK_BUDDIES(__check, __prev, __p, field); \ + if(__prev == 0) __prev = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__p->field.prev); \ + __last = (vm_page_queue_entry_t) __p; \ + __n_free++; \ + } \ + } \ + __DEBUG_STAT_INCREMENT_INRANGE; \ + } \ + /* if elt is not part of vm_pages or if 1st page in clump, insert at tail */ \ + if(__prev == 0) __prev = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR((head)->prev); \ + \ + /* insert the element */ \ + __next = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__prev->next); \ + (elt)->field.next = __prev->next; \ + (elt)->field.prev = __next->prev; \ + __prev->next = __next->prev = VM_PAGE_PACK_PTR(elt); \ + __DEBUG_STAT_INCREMENT_INSERTS; \ + \ + /* check if clump needs to be promoted to head */ \ + if(__n_free >= vm_clump_promote_threshold && __n_free > 1) { \ + __first_prev = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__first->prev); \ + if(__first_prev != (head)) { /* if not at head already */ \ + __last_next = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__last->next); \ + /* verify that the links within the clump are consistent */ \ + __DEBUG_VERIFY_LINKS(__i, __first, __n_free, __last_next); \ + /* promote clump to head */ \ + __first_prev->next = __last->next; \ + __last_next->prev = __first->prev; \ + __first->prev = VM_PAGE_PACK_PTR(head); \ + __last->next = (head)->next; \ + __head_next = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR((head)->next); \ + __head_next->prev = VM_PAGE_PACK_PTR(__last); \ + (head)->next = VM_PAGE_PACK_PTR(__first); \ + __DEBUG_STAT_INCREMENT_PROMOTES(__n_free); \ + } \ + } \ +MACRO_END +#endif + /* * Macro: vm_page_queue_enter_first * Function: @@ -564,6 +699,44 @@ MACRO_BEGIN \ MACRO_END +/* + * Macro: vm_page_queue_remove_first_with_clump + * Function: + * Remove and return the entry at the head of the free queue + * end is set to 1 to indicate that we just returned the last page in a clump + * + * Header: + * vm_page_queue_remove_first_with_clump(head, entry, type, field, end) + * entry is returned by reference + * end is returned by reference + * Note: + * This should only be used with Method 2 queue iteration (element chains) + */ +#if defined(__x86_64__) +#define vm_page_queue_remove_first_with_clump(head, entry, type, field, end) \ +MACRO_BEGIN \ + vm_page_queue_entry_t __next; \ + \ + (entry) = (type)(void *) VM_PAGE_UNPACK_PTR(((head)->next)); \ + __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((entry)->field.next)); \ + \ + (end)=0; \ + if ((head) == __next) { \ + (head)->prev = VM_PAGE_PACK_PTR(head); \ + (end)=1; \ + } \ + else { \ + ((type)(void *)(__next))->field.prev = VM_PAGE_PACK_PTR(head); \ + if(VM_PAGE_GET_CLUMP(entry) != VM_PAGE_GET_CLUMP(((type)(void *)(__next)))) (end)=1; \ + } \ + (head)->next = VM_PAGE_PACK_PTR(__next); \ + \ + (entry)->field.next = 0; \ + (entry)->field.prev = 0; \ + \ +MACRO_END +#endif + /* * Macro: vm_page_queue_end * Function: @@ -910,8 +1083,6 @@ lck_spin_t vm_objects_wired_lock; #define VM_PAGE_BG_DISABLED 0 #define VM_PAGE_BG_LEVEL_1 1 -#define VM_PAGE_BG_LEVEL_2 2 -#define VM_PAGE_BG_LEVEL_3 3 extern vm_page_queue_head_t vm_page_queue_background; @@ -920,8 +1091,6 @@ uint64_t vm_page_background_promoted_count; extern uint32_t vm_page_background_count; extern -uint32_t vm_page_background_limit; -extern uint32_t vm_page_background_target; extern uint32_t vm_page_background_internal_count; @@ -1021,10 +1190,10 @@ extern unsigned int vm_page_free_wanted_secluded; /* how many threads are waiting for secluded memory */ #endif /* CONFIG_SECLUDED_MEMORY */ -extern ppnum_t vm_page_fictitious_addr; +extern const ppnum_t vm_page_fictitious_addr; /* (fake) phys_addr of fictitious pages */ -extern ppnum_t vm_page_guard_addr; +extern const ppnum_t vm_page_guard_addr; /* (fake) phys_addr of guard pages */ @@ -1145,8 +1314,7 @@ extern void vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t n extern void vm_page_rename( vm_page_t page, vm_object_t new_object, - vm_object_offset_t new_offset, - boolean_t encrypted_ok); + vm_object_offset_t new_offset); extern void vm_page_insert( vm_page_t page, @@ -1253,9 +1421,15 @@ extern void memorystatus_pages_update(unsigned int pages_avail); #else /* CONFIG_JETSAM */ +#if CONFIG_EMBEDDED + +#define VM_CHECK_MEMORYSTATUS do {} while(0) + +#else /* CONFIG_EMBEDDED */ #define VM_CHECK_MEMORYSTATUS vm_pressure_response() +#endif /* CONFIG_EMBEDDED */ #endif /* CONFIG_JETSAM */ @@ -1264,11 +1438,22 @@ extern void memorystatus_pages_update(unsigned int pages_avail); * protected by the object lock. */ +#if CONFIG_EMBEDDED +#define SET_PAGE_DIRTY(m, set_pmap_modified) \ + MACRO_BEGIN \ + vm_page_t __page__ = (m); \ + if (__page__->dirty == FALSE && (set_pmap_modified)) { \ + pmap_set_modify(VM_PAGE_GET_PHYS_PAGE(__page__)); \ + } \ + __page__->dirty = TRUE; \ + MACRO_END +#else /* CONFIG_EMBEDDED */ #define SET_PAGE_DIRTY(m, set_pmap_modified) \ MACRO_BEGIN \ vm_page_t __page__ = (m); \ __page__->dirty = TRUE; \ MACRO_END +#endif /* CONFIG_EMBEDDED */ #define PAGE_ASSERT_WAIT(m, interruptible) \ (((m)->wanted = TRUE), \ diff --git a/osfmk/vm/vm_pageout.c b/osfmk/vm/vm_pageout.c index b94d1962d..b24e87a9a 100644 --- a/osfmk/vm/vm_pageout.c +++ b/osfmk/vm/vm_pageout.c @@ -90,6 +90,7 @@ #include #include #include +#include #include #include @@ -107,14 +108,11 @@ #include #include +#include + #if CONFIG_PHANTOM_CACHE #include #endif -/* - * ENCRYPTED SWAP: - */ -#include -extern u_int32_t random(void); /* from */ extern int cs_debug; @@ -125,9 +123,16 @@ extern int cs_debug; extern void m_drain(void); #if VM_PRESSURE_EVENTS +#if CONFIG_JETSAM extern unsigned int memorystatus_available_pages; extern unsigned int memorystatus_available_pages_pressure; extern unsigned int memorystatus_available_pages_critical; +#else /* CONFIG_JETSAM */ +extern uint64_t memorystatus_available_pages; +extern uint64_t memorystatus_available_pages_pressure; +extern uint64_t memorystatus_available_pages_critical; +#endif /* CONFIG_JETSAM */ + extern unsigned int memorystatus_frozen_count; extern unsigned int memorystatus_suspended_count; @@ -150,8 +155,12 @@ boolean_t vm_pressure_changed = FALSE; #endif #ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE /* maximum iterations of the inactive queue w/o stealing/cleaning a page */ +#ifdef CONFIG_EMBEDDED +#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 1024 +#else #define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096 #endif +#endif #ifndef VM_PAGEOUT_DEADLOCK_RELIEF #define VM_PAGEOUT_DEADLOCK_RELIEF 100 /* number of pages to move to break deadlock */ @@ -201,11 +210,6 @@ unsigned int vm_page_speculative_percentage = 5; #endif /* VM_PAGE_SPECULATIVE_TARGET */ -#ifndef VM_PAGE_INACTIVE_HEALTHY_LIMIT -#define VM_PAGE_INACTIVE_HEALTHY_LIMIT(total) ((total) * 1 / 200) -#endif /* VM_PAGE_INACTIVE_HEALTHY_LIMIT */ - - /* * To obtain a reasonable LRU approximation, the inactive queue * needs to be large enough to give pages on it a chance to be @@ -219,7 +223,11 @@ unsigned int vm_page_speculative_percentage = 5; */ #ifndef VM_PAGE_INACTIVE_TARGET +#ifdef CONFIG_EMBEDDED +#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 3) +#else #define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 2) +#endif #endif /* VM_PAGE_INACTIVE_TARGET */ /* @@ -228,7 +236,11 @@ unsigned int vm_page_speculative_percentage = 5; */ #ifndef VM_PAGE_FREE_TARGET +#ifdef CONFIG_EMBEDDED +#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 100) +#else #define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80) +#endif #endif /* VM_PAGE_FREE_TARGET */ @@ -238,12 +250,22 @@ unsigned int vm_page_speculative_percentage = 5; */ #ifndef VM_PAGE_FREE_MIN +#ifdef CONFIG_EMBEDDED +#define VM_PAGE_FREE_MIN(free) (10 + (free) / 200) +#else #define VM_PAGE_FREE_MIN(free) (10 + (free) / 100) +#endif #endif /* VM_PAGE_FREE_MIN */ +#ifdef CONFIG_EMBEDDED +#define VM_PAGE_FREE_RESERVED_LIMIT 100 +#define VM_PAGE_FREE_MIN_LIMIT 1500 +#define VM_PAGE_FREE_TARGET_LIMIT 2000 +#else #define VM_PAGE_FREE_RESERVED_LIMIT 1700 #define VM_PAGE_FREE_MIN_LIMIT 3500 #define VM_PAGE_FREE_TARGET_LIMIT 4000 +#endif /* * When vm_page_free_count falls below vm_page_free_reserved, @@ -268,7 +290,11 @@ unsigned int vm_page_speculative_percentage = 5; */ #define VM_PAGE_REACTIVATE_LIMIT_MAX 20000 #ifndef VM_PAGE_REACTIVATE_LIMIT +#ifdef CONFIG_EMBEDDED +#define VM_PAGE_REACTIVATE_LIMIT(avail) (VM_PAGE_INACTIVE_TARGET(avail) / 2) +#else #define VM_PAGE_REACTIVATE_LIMIT(avail) (MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX)) +#endif #endif /* VM_PAGE_REACTIVATE_LIMIT */ #define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 1000 @@ -293,13 +319,9 @@ struct cq { char *scratch_buf; int id; }; -#define MAX_COMPRESSOR_THREAD_COUNT 8 struct cq ciq[MAX_COMPRESSOR_THREAD_COUNT]; -void *vm_pageout_immediate_chead; -char *vm_pageout_immediate_scratch_buf; - #if VM_PRESSURE_EVENTS void vm_pressure_thread(void); @@ -310,20 +332,20 @@ boolean_t VM_PRESSURE_WARNING_TO_CRITICAL(void); boolean_t VM_PRESSURE_WARNING_TO_NORMAL(void); boolean_t VM_PRESSURE_CRITICAL_TO_WARNING(void); #endif -static void vm_pageout_garbage_collect(int); +void vm_pageout_garbage_collect(int); static void vm_pageout_iothread_external(void); static void vm_pageout_iothread_internal(struct cq *cq); -static void vm_pageout_adjust_io_throttles(struct vm_pageout_queue *, struct vm_pageout_queue *, boolean_t); +static void vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *, boolean_t); extern void vm_pageout_continue(void); extern void vm_pageout_scan(void); +void vm_tests(void); /* forward */ -static void vm_pageout_immediate(vm_page_t, boolean_t); -boolean_t vm_compressor_immediate_preferred = FALSE; -boolean_t vm_compressor_immediate_preferred_override = FALSE; boolean_t vm_restricted_to_single_processor = FALSE; +#if !CONFIG_EMBEDDED static boolean_t vm_pageout_waiter = FALSE; static boolean_t vm_pageout_running = FALSE; +#endif /* !CONFIG_EMBEDDED */ static thread_t vm_pageout_external_iothread = THREAD_NULL; @@ -365,12 +387,16 @@ unsigned int vm_pageout_inactive_used = 0; /* debugging */ unsigned int vm_pageout_cache_evicted = 0; /* debugging */ unsigned int vm_pageout_inactive_clean = 0; /* debugging */ unsigned int vm_pageout_speculative_clean = 0; /* debugging */ +unsigned int vm_pageout_speculative_dirty = 0; /* debugging */ unsigned int vm_pageout_freed_from_cleaned = 0; unsigned int vm_pageout_freed_from_speculative = 0; unsigned int vm_pageout_freed_from_inactive_clean = 0; +unsigned int vm_pageout_freed_after_compression = 0; + +extern uint32_t vm_compressor_pages_grabbed; +extern uint32_t c_segment_pages_compressed; -unsigned int vm_pageout_enqueued_cleaned_from_inactive_clean = 0; unsigned int vm_pageout_enqueued_cleaned_from_inactive_dirty = 0; unsigned int vm_pageout_cleaned_reclaimed = 0; /* debugging; how many cleaned pages are reclaimed by the pageout scan */ @@ -393,7 +419,6 @@ unsigned int vm_stat_discard_sent = 0; /* debugging */ unsigned int vm_stat_discard_failure = 0; /* debugging */ unsigned int vm_stat_discard_throttle = 0; /* debugging */ unsigned int vm_pageout_reactivation_limit_exceeded = 0; /* debugging */ -unsigned int vm_pageout_catch_ups = 0; /* debugging */ unsigned int vm_pageout_inactive_force_reclaim = 0; /* debugging */ unsigned int vm_pageout_scan_reclaimed_throttled = 0; @@ -413,6 +438,7 @@ unsigned int vm_pageout_scan_yield_unthrottled = 0; /* debugging */ unsigned int vm_page_speculative_count_drifts = 0; unsigned int vm_page_speculative_count_drift_max = 0; +uint32_t vm_compressor_failed; /* * Backing store throttle when BS is exhausted @@ -426,17 +452,6 @@ unsigned int vm_page_steal_pageout_page = 0; struct vm_config vm_config; -/* - * ENCRYPTED SWAP: - * counters and statistics... - */ -unsigned long vm_page_decrypt_counter = 0; -unsigned long vm_page_decrypt_for_upl_counter = 0; -unsigned long vm_page_encrypt_counter = 0; -unsigned long vm_page_encrypt_abort_counter = 0; -unsigned long vm_page_encrypt_already_encrypted_counter = 0; -boolean_t vm_pages_encrypted = FALSE; /* are there encrypted pages ? */ - struct vm_pageout_queue vm_pageout_queue_internal __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); struct vm_pageout_queue vm_pageout_queue_external __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); @@ -468,28 +483,6 @@ uint32_t vm_grab_anon_nops = 0; #endif -/* - * Routine: vm_backing_store_disable - * Purpose: - * Suspend non-privileged threads wishing to extend - * backing store when we are low on backing store - * (Synchronized by caller) - */ -void -vm_backing_store_disable( - boolean_t disable) -{ - if(disable) { - vm_backing_store_low = 1; - } else { - if(vm_backing_store_low) { - vm_backing_store_low = 0; - thread_wakeup((event_t) &vm_backing_store_low); - } - } -} - - #if MACH_CLUSTER_STATS unsigned long vm_pageout_cluster_dirtied = 0; unsigned long vm_pageout_cluster_cleaned = 0; @@ -504,6 +497,11 @@ unsigned long vm_pageout_target_page_freed = 0; #define CLUSTER_STAT(clause) #endif /* MACH_CLUSTER_STATS */ + +#if DEVELOPMENT || DEBUG +vmct_stats_t vmct_stats; +#endif + /* * Routine: vm_pageout_object_terminate * Purpose: @@ -575,7 +573,6 @@ vm_pageout_object_terminate( assert(m->vm_page_q_state == VM_PAGE_IS_WIRED); assert(m->wire_count == 1); m->cleaning = FALSE; - m->encrypted_cleaning = FALSE; m->free_when_done = FALSE; #if MACH_CLUSTER_STATS if (m->wanted) vm_pageout_target_collisions++; @@ -668,10 +665,6 @@ vm_pageout_object_terminate( m->dirty = FALSE; #endif } - if (m->encrypted_cleaning == TRUE) { - m->encrypted_cleaning = FALSE; - m->busy = FALSE; - } m->cleaning = FALSE; /* @@ -873,9 +866,16 @@ struct { * * The page must not be on any pageout queue. */ +int32_t vmct_active = 0; +typedef enum vmct_state_t { + VMCT_IDLE, + VMCT_AWAKENED, + VMCT_ACTIVE, +} vmct_state_t; +vmct_state_t vmct_state[MAX_COMPRESSOR_THREAD_COUNT]; -int -vm_pageout_cluster(vm_page_t m, boolean_t immediate_ok, boolean_t keep_object_locked) +void +vm_pageout_cluster(vm_page_t m) { vm_object_t object = VM_PAGE_OBJECT(m); struct vm_pageout_queue *q; @@ -906,17 +906,6 @@ vm_pageout_cluster(vm_page_t m, boolean_t immediate_ok, boolean_t keep_object_lo m->busy = TRUE; - if (vm_compressor_immediate_preferred == TRUE && immediate_ok == TRUE) { - panic("immediate compressor mode no longer supported\n"); - - if (keep_object_locked == FALSE) - vm_object_unlock(object); - vm_page_unlock_queues(); - - vm_pageout_immediate(m, keep_object_locked); - - return (1); - } q = &vm_pageout_queue_internal; } else q = &vm_pageout_queue_external; @@ -929,14 +918,12 @@ vm_pageout_cluster(vm_page_t m, boolean_t immediate_ok, boolean_t keep_object_lo m->vm_page_q_state = VM_PAGE_ON_PAGEOUT_Q; vm_page_queue_enter(&q->pgo_pending, m, vm_page_t, pageq); - + if (q->pgo_idle == TRUE) { q->pgo_idle = FALSE; thread_wakeup((event_t) &q->pgo_pending); } VM_PAGE_CHECK(m); - - return (0); } @@ -1040,8 +1027,15 @@ vm_pageout_throttle_up_batch( #define VM_PAGEOUT_STAT_SIZE 31 struct vm_pageout_stat { unsigned int considered; - unsigned int reclaimed; -} vm_pageout_stats[VM_PAGEOUT_STAT_SIZE] = {{0,0}, }; + unsigned int reclaimed_clean; + unsigned int pages_compressed; + unsigned int pages_grabbed_by_compressor; + unsigned int cleaned_dirty_external; + unsigned int throttled_internal_q; + unsigned int throttled_external_q; + unsigned int failed_compressions; +} vm_pageout_stats[VM_PAGEOUT_STAT_SIZE] = {{0,0,0,0,0,0,0,0}, }; + unsigned int vm_pageout_stat_now = 0; unsigned int vm_memory_pressure = 0; @@ -1072,14 +1066,21 @@ compute_memory_pressure( #endif /* VM_PAGE_BUCKETS_CHECK */ vm_memory_pressure = - vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].reclaimed; + vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].reclaimed_clean; commpage_set_memory_pressure( vm_memory_pressure ); /* move "now" forward */ vm_pageout_next = VM_PAGEOUT_STAT_AFTER(vm_pageout_stat_now); vm_pageout_stats[vm_pageout_next].considered = 0; - vm_pageout_stats[vm_pageout_next].reclaimed = 0; + vm_pageout_stats[vm_pageout_next].reclaimed_clean = 0; + vm_pageout_stats[vm_pageout_next].throttled_internal_q = 0; + vm_pageout_stats[vm_pageout_next].throttled_external_q = 0; + vm_pageout_stats[vm_pageout_next].cleaned_dirty_external = 0; + vm_pageout_stats[vm_pageout_next].pages_compressed = 0; + vm_pageout_stats[vm_pageout_next].pages_grabbed_by_compressor = 0; + vm_pageout_stats[vm_pageout_next].failed_compressions = 0; + vm_pageout_stat_now = vm_pageout_next; } @@ -1175,7 +1176,7 @@ mach_vm_pressure_monitor( nsecs_monitored-- != 0; vm_pageout_then = VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) { - pages_reclaimed += vm_pageout_stats[vm_pageout_then].reclaimed; + pages_reclaimed += vm_pageout_stats[vm_pageout_then].reclaimed_clean; } *pages_reclaimed_p = pages_reclaimed; @@ -1285,7 +1286,7 @@ vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount) l_object = m_object; } - if ( !m_object->alive || m->encrypted_cleaning || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) { + if ( !m_object->alive || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) { /* * put it back on the head of its queue */ @@ -1451,7 +1452,7 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount) } l_object = m_object; } - if ( !m_object->alive || m->encrypted_cleaning || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) { + if ( !m_object->alive || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) { /* * page is not to be cleaned * put it back on the head of its queue @@ -1524,7 +1525,7 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount) LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); - vm_pageout_cluster(m, FALSE, FALSE); + vm_pageout_cluster(m); goto next_pg; @@ -1593,11 +1594,14 @@ struct flow_control { }; #if CONFIG_BACKGROUND_QUEUE +uint64_t vm_pageout_skipped_bq_internal = 0; uint64_t vm_pageout_considered_bq_internal = 0; uint64_t vm_pageout_considered_bq_external = 0; uint64_t vm_pageout_rejected_bq_internal = 0; uint64_t vm_pageout_rejected_bq_external = 0; #endif + +uint32_t vm_pageout_no_victim = 0; uint32_t vm_pageout_considered_page = 0; uint32_t vm_page_filecache_min = 0; @@ -1605,11 +1609,209 @@ uint32_t vm_page_filecache_min = 0; #if CONFIG_SECLUDED_MEMORY extern vm_page_t vm_page_grab_secluded(void); -uint64_t vm_pageout_freed_from_secluded = 0; -uint64_t vm_pageout_secluded_reactivated = 0; /* debugging; how many secluded pages are found to be referenced on pageout (and are therefore reactivated) */ uint64_t vm_pageout_secluded_burst_count = 0; #endif /* CONFIG_SECLUDED_MEMORY */ + +static void vm_pageout_delayed_unlock(int *, int *, vm_page_t *); +static void vm_pageout_prepare_to_block(vm_object_t *, int *, vm_page_t *, int *, int); + +#define VM_PAGEOUT_PB_NO_ACTION 0 +#define VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER 1 +#define VM_PAGEOUT_PB_THREAD_YIELD 2 + + +static void +vm_pageout_delayed_unlock(int *delayed_unlock, int *local_freed, vm_page_t *local_freeq) +{ + if (*local_freeq) { + vm_page_unlock_queues(); + + VM_DEBUG_EVENT( + vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, + vm_page_free_count, *local_freed, 0, 1); + + vm_page_free_list(*local_freeq, TRUE); + + VM_DEBUG_EVENT(vm_pageout_freelist,VM_PAGEOUT_FREELIST, DBG_FUNC_END, + vm_page_free_count, 0, 0, 1); + + *local_freeq = NULL; + *local_freed = 0; + + vm_page_lock_queues(); + } else { + lck_mtx_yield(&vm_page_queue_lock); + } + *delayed_unlock = 1; +} + + +static void +vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock, + vm_page_t *local_freeq, int *local_freed, int action) +{ + vm_page_unlock_queues(); + + if (*object != NULL) { + vm_object_unlock(*object); + *object = NULL; + } + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + + if (*local_freeq) { + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, + vm_page_free_count, *local_freed, 0, 2); + + vm_page_free_list(*local_freeq, TRUE); + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, + vm_page_free_count, 0, 0, 2); + + *local_freeq = NULL; + *local_freed = 0; + } + *delayed_unlock = 1; + + switch (action) { + + case VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER: + vm_consider_waking_compactor_swapper(); + break; + case VM_PAGEOUT_PB_THREAD_YIELD: + thread_yield_internal(1); + break; + case VM_PAGEOUT_PB_NO_ACTION: + default: + break; + } + vm_page_lock_queues(); +} + + +int last_vm_pageout_freed_from_inactive_clean = 0; +int last_vm_pageout_freed_from_cleaned = 0; +int last_vm_pageout_freed_from_speculative = 0; +int last_vm_pageout_freed_after_compression = 0; +int last_vm_pageout_enqueued_cleaned_from_inactive_dirty = 0; +int last_vm_pageout_inactive_force_reclaim = 0; +int last_vm_pageout_scan_inactive_throttled_external = 0; +int last_vm_pageout_scan_inactive_throttled_internal = 0; +int last_vm_pageout_reactivation_limit_exceeded = 0; +int last_vm_pageout_considered_page = 0; +int last_vm_compressor_pages_grabbed = 0; +int last_vm_compressor_failed = 0; + +void update_vm_info(void) +{ + int tmp1, tmp2, tmp3; + + if (!kdebug_enable) + return; + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO1)) | DBG_FUNC_NONE, + vm_page_active_count, + vm_page_speculative_count, + vm_page_inactive_count, + vm_page_anonymous_count, + 0); + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO2)) | DBG_FUNC_NONE, + vm_page_free_count, + vm_page_wire_count, + VM_PAGE_COMPRESSOR_COUNT, + 0, 0); + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO3)) | DBG_FUNC_NONE, + c_segment_pages_compressed, + vm_page_internal_count, + vm_page_external_count, + vm_page_xpmapped_external_count, + 0); + + + if ((vm_pageout_considered_page - last_vm_pageout_considered_page) == 0 && + (vm_pageout_enqueued_cleaned_from_inactive_dirty - last_vm_pageout_enqueued_cleaned_from_inactive_dirty == 0) && + (vm_pageout_freed_after_compression - last_vm_pageout_freed_after_compression == 0)) + return; + + + tmp1 = vm_pageout_considered_page; + tmp2 = vm_pageout_freed_from_speculative; + tmp3 = vm_pageout_freed_from_inactive_clean; + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO4)) | DBG_FUNC_NONE, + tmp1 - last_vm_pageout_considered_page, + tmp2 - last_vm_pageout_freed_from_speculative, + tmp3 - last_vm_pageout_freed_from_inactive_clean, + 0, 0); + + last_vm_pageout_considered_page = tmp1; + last_vm_pageout_freed_from_speculative = tmp2; + last_vm_pageout_freed_from_inactive_clean = tmp3; + + + tmp1 = vm_pageout_scan_inactive_throttled_external; + tmp2 = vm_pageout_enqueued_cleaned_from_inactive_dirty; + tmp3 = vm_pageout_freed_from_cleaned; + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO5)) | DBG_FUNC_NONE, + tmp1 - last_vm_pageout_scan_inactive_throttled_external, + tmp2 - last_vm_pageout_enqueued_cleaned_from_inactive_dirty, + tmp3 - last_vm_pageout_freed_from_cleaned, + 0, 0); + + vm_pageout_stats[vm_pageout_stat_now].throttled_external_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_external); + vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external += (tmp2 - last_vm_pageout_enqueued_cleaned_from_inactive_dirty); + + last_vm_pageout_scan_inactive_throttled_external = tmp1; + last_vm_pageout_enqueued_cleaned_from_inactive_dirty = tmp2; + last_vm_pageout_freed_from_cleaned = tmp3; + + + tmp1 = vm_pageout_scan_inactive_throttled_internal; + tmp2 = vm_pageout_freed_after_compression; + tmp3 = vm_compressor_pages_grabbed; + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO6)) | DBG_FUNC_NONE, + tmp1 - last_vm_pageout_scan_inactive_throttled_internal, + tmp2 - last_vm_pageout_freed_after_compression, + tmp3 - last_vm_compressor_pages_grabbed, + 0, 0); + + vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_internal); + vm_pageout_stats[vm_pageout_stat_now].pages_compressed += (tmp2 - last_vm_pageout_freed_after_compression); + vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor += (tmp3 - last_vm_compressor_pages_grabbed); + + last_vm_pageout_scan_inactive_throttled_internal = tmp1; + last_vm_pageout_freed_after_compression = tmp2; + last_vm_compressor_pages_grabbed = tmp3; + + + if ((vm_pageout_reactivation_limit_exceeded - last_vm_pageout_reactivation_limit_exceeded) == 0 && + (vm_pageout_inactive_force_reclaim - last_vm_pageout_inactive_force_reclaim) == 0 && + (vm_compressor_failed - last_vm_compressor_failed) == 0) + return; + + tmp1 = vm_pageout_reactivation_limit_exceeded; + tmp2 = vm_pageout_inactive_force_reclaim; + tmp3 = vm_compressor_failed; + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO7)) | DBG_FUNC_NONE, + tmp1 - last_vm_pageout_reactivation_limit_exceeded, + tmp2 - last_vm_pageout_inactive_force_reclaim, + tmp3 - last_vm_compressor_failed, + 0, 0); + + vm_pageout_stats[vm_pageout_stat_now].failed_compressions += (tmp3 - last_vm_compressor_failed); + + last_vm_pageout_reactivation_limit_exceeded = tmp1; + last_vm_pageout_inactive_force_reclaim = tmp2; + last_vm_compressor_failed = tmp3; +} + + /* * vm_pageout_scan does the dirty work for the pageout daemon. * It returns with both vm_page_queue_free_lock and vm_page_queue_lock @@ -1637,21 +1839,17 @@ vm_pageout_scan(void) boolean_t try_failed; mach_timespec_t ts; unsigned int msecs = 0; - vm_object_t object; - vm_object_t last_object_tried; - uint32_t catch_up_count = 0; + vm_object_t object = NULL; uint32_t inactive_reclaim_run; boolean_t exceeded_burst_throttle; boolean_t grab_anonymous = FALSE; boolean_t force_anonymous = FALSE; + boolean_t force_speculative_aging = FALSE; int anons_grabbed = 0; int page_prev_q_state = 0; #if CONFIG_BACKGROUND_QUEUE - boolean_t ignore_reference = FALSE; + boolean_t page_from_bg_q = FALSE; #endif -#if CONFIG_SECLUDED_MEMORY - boolean_t ignore_reference_secluded; -#endif /* CONFIG_SECLUDED_MEMORY */ int cache_evict_throttle = 0; uint32_t vm_pageout_inactive_external_forced_reactivate_limit = 0; int force_purge = 0; @@ -1675,9 +1873,11 @@ vm_pageout_scan(void) XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0); - + /* Ask the pmap layer to return any pages it no longer needs. */ + pmap_release_pages_fast(); + vm_page_lock_queues(); - delayed_unlock = 1; /* must be nonzero if Qs are locked, 0 if unlocked */ + delayed_unlock = 1; /* * Calculate the max number of referenced pages on the inactive @@ -1710,8 +1910,8 @@ vm_pageout_scan(void) Restart: - - assert(delayed_unlock!=0); + assert(object == NULL); + assert(delayed_unlock != 0); /* * Recalculate vm_page_inactivate_target. @@ -1737,172 +1937,34 @@ Restart: vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count + vm_page_inactive_count); - object = NULL; - last_object_tried = NULL; try_failed = FALSE; - - if ((vm_page_inactive_count + vm_page_speculative_count) < VM_PAGE_INACTIVE_HEALTHY_LIMIT(vm_page_active_count)) - catch_up_count = vm_page_inactive_count + vm_page_speculative_count; - else - catch_up_count = 0; - + for (;;) { vm_page_t m; DTRACE_VM2(rev, int, 1, (uint64_t *), NULL); -#if CONFIG_SECLUDED_MEMORY - if (vm_page_secluded_count > vm_page_secluded_target && - object != NULL) { - vm_object_unlock(object); - object = NULL; - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - } + if (vm_upl_wait_for_pages < 0) + vm_upl_wait_for_pages = 0; + + delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages; + + if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX) + delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX; +#if CONFIG_SECLUDED_MEMORY /* * Deal with secluded_q overflow. */ - if (vm_page_secluded_count > vm_page_secluded_target && - secluded_aging_policy == SECLUDED_AGING_FIFO) { - unsigned int secluded_overflow; - vm_page_t secluded_page; - - /* - * SECLUDED_AGING_FIFO: - * No aging, just reclaim the excess pages - * at the tail of the secluded queue. - * We're reclaiming pages and we're not hogging - * any global lock, so no need for throttling. - */ - - secluded_overflow = (vm_page_secluded_count - - vm_page_secluded_target); - /* transfer to free queue */ - vm_page_unlock_queues(); - while (secluded_overflow--) { - secluded_page = vm_page_grab_secluded(); - if (secluded_page == VM_PAGE_NULL) { - break; - } - assert(secluded_page->busy); - assert(secluded_page->pageq.next == 0 && - secluded_page->pageq.prev == 0); - - secluded_page->snext = local_freeq; - local_freeq = secluded_page; - local_freed++; - secluded_page = VM_PAGE_NULL; - } - } else if (vm_page_secluded_count > vm_page_secluded_target && - secluded_aging_policy == SECLUDED_AGING_ALONG_ACTIVE) { + if (vm_page_secluded_count > vm_page_secluded_target) { unsigned int secluded_overflow; vm_page_t secluded_page; - /* - * SECLUDED_AGING_ALONG_ACTIVE: - * There might be free pages at the tail of the - * secluded queue: - * just move them to the free queue (in batches). - * There can also be an excessive number of "inuse" - * pages: - * we age them by resetting their "referenced" bit and - * moving them to the inactive queue. Their trip - * through the secluded queue was equivalent to a trip - * through the active queue. - * - * We're holding the page queue lock, so we need - * to throttle and give someone else a chance to - * grab that lock if needed. - * - * We're also limiting the number of secluded "inuse" - * pages that get moved to the inactive queue, using - * the same "active_bust_count" method we use when - * balancing the active and inactive queues, because - * there can be a large number - * of extra "inuse" pages and handling them gets in the - * way of actually reclaiming memory. - */ - - active_burst_count = MIN(vm_pageout_burst_active_throttle, - vm_page_secluded_count_inuse); - delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT; - delayed_unlock = 1; - secluded_overflow = (vm_page_secluded_count - - vm_page_secluded_target); - while (secluded_overflow-- > 0 && - vm_page_secluded_count > vm_page_secluded_target) { - assert((vm_page_secluded_count_free + - vm_page_secluded_count_inuse) == - vm_page_secluded_count); - secluded_page = vm_page_queue_first(&vm_page_queue_secluded); - assert(secluded_page->vm_page_q_state == - VM_PAGE_ON_SECLUDED_Q); - vm_page_queues_remove(secluded_page, FALSE); - assert(!secluded_page->fictitious); - assert(!VM_PAGE_WIRED(secluded_page)); - if (secluded_page->vm_page_object == 0) { - /* transfer to free queue */ - assert(secluded_page->busy); - secluded_page->snext = local_freeq; - local_freeq = secluded_page; - local_freed++; - } else { - /* transfer to head of inactive queue */ - pmap_clear_refmod_options( - VM_PAGE_GET_PHYS_PAGE(secluded_page), - VM_MEM_REFERENCED, - PMAP_OPTIONS_NOFLUSH, - (void *)NULL); - vm_page_enqueue_inactive(secluded_page, - FALSE); - if (active_burst_count-- == 0) { - vm_pageout_secluded_burst_count++; - break; - } - } - secluded_page = VM_PAGE_NULL; - if (delayed_unlock++ > delayed_unlock_limit) { - if (local_freeq) { - vm_page_unlock_queues(); - VM_DEBUG_EVENT( - vm_pageout_freelist, - VM_PAGEOUT_FREELIST, - DBG_FUNC_START, - vm_page_free_count, - local_freed, - delayed_unlock_limit, - 1); - vm_page_free_list(local_freeq, - TRUE); - VM_DEBUG_EVENT( - vm_pageout_freelist, - VM_PAGEOUT_FREELIST, - DBG_FUNC_END, - vm_page_free_count, - 0, 0, 1); - local_freeq = NULL; - local_freed = 0; - vm_page_lock_queues(); - } else { - lck_mtx_yield(&vm_page_queue_lock); - } - delayed_unlock = 1; - } + if (object != NULL) { + vm_object_unlock(object); + object = NULL; + vm_pageout_scan_wants_object = VM_OBJECT_NULL; } - delayed_unlock = 1; - } else if (vm_page_secluded_count > vm_page_secluded_target && - secluded_aging_policy == SECLUDED_AGING_AFTER_INACTIVE) { - /* - * SECLUDED_AGING_AFTER_INACTIVE: - * No balancing needed at this point: when we get to - * the "choose a victim" part below, we'll consider the - * extra secluded pages before any inactive page. - */ - } else if (vm_page_secluded_count > vm_page_secluded_target && - secluded_aging_policy == SECLUDED_AGING_BEFORE_ACTIVE) { - unsigned int secluded_overflow; - vm_page_t secluded_page; - /* * SECLUDED_AGING_BEFORE_ACTIVE: * Excess secluded pages go to the active queue and @@ -1910,8 +1972,6 @@ Restart: */ active_burst_count = MIN(vm_pageout_burst_active_throttle, vm_page_secluded_count_inuse); - delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT; - delayed_unlock = 1; secluded_overflow = (vm_page_secluded_count - vm_page_secluded_target); while (secluded_overflow-- > 0 && @@ -1919,7 +1979,7 @@ Restart: assert((vm_page_secluded_count_free + vm_page_secluded_count_inuse) == vm_page_secluded_count); - secluded_page = vm_page_queue_first(&vm_page_queue_secluded); + secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded); assert(secluded_page->vm_page_q_state == VM_PAGE_ON_SECLUDED_Q); vm_page_queues_remove(secluded_page, FALSE); @@ -1933,81 +1993,26 @@ Restart: local_freed++; } else { /* transfer to head of active queue */ - vm_page_enqueue_active(secluded_page, - FALSE); + vm_page_enqueue_active(secluded_page, FALSE); if (active_burst_count-- == 0) { vm_pageout_secluded_burst_count++; break; } } secluded_page = VM_PAGE_NULL; + if (delayed_unlock++ > delayed_unlock_limit) { - if (local_freeq) { - vm_page_unlock_queues(); - VM_DEBUG_EVENT( - vm_pageout_freelist, - VM_PAGEOUT_FREELIST, - DBG_FUNC_START, - vm_page_free_count, - local_freed, - delayed_unlock_limit, - 1); - vm_page_free_list(local_freeq, - TRUE); - VM_DEBUG_EVENT( - vm_pageout_freelist, - VM_PAGEOUT_FREELIST, - DBG_FUNC_END, - vm_page_free_count, - 0, 0, 1); - local_freeq = NULL; - local_freed = 0; - vm_page_lock_queues(); - } else { - lck_mtx_yield(&vm_page_queue_lock); - } - delayed_unlock = 1; + vm_pageout_delayed_unlock(&delayed_unlock, &local_freed, &local_freeq); } } - delayed_unlock = 1; - } else if (vm_page_secluded_count > vm_page_secluded_target) { - panic("unsupported secluded_aging_policy %d\n", - secluded_aging_policy); - } - if (local_freeq) { - vm_page_unlock_queues(); - VM_DEBUG_EVENT(vm_pageout_freelist, - VM_PAGEOUT_FREELIST, - DBG_FUNC_START, - vm_page_free_count, - local_freed, - 0, - 0); - vm_page_free_list(local_freeq, TRUE); - VM_DEBUG_EVENT(vm_pageout_freelist, - VM_PAGEOUT_FREELIST, - DBG_FUNC_END, - vm_page_free_count, 0, 0, 0); - local_freeq = NULL; - local_freed = 0; - vm_page_lock_queues(); } #endif /* CONFIG_SECLUDED_MEMORY */ assert(delayed_unlock); - if (vm_upl_wait_for_pages < 0) - vm_upl_wait_for_pages = 0; - - delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages; - - if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX) - delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX; - /* * Move pages from active to inactive if we're below the target */ - /* if we are trying to make clean, we need to make sure we actually have inactive - mj */ if ((vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target) goto done_moving_active_pages; @@ -2065,32 +2070,7 @@ Restart: vm_page_deactivate_internal(m, FALSE); if (delayed_unlock++ > delayed_unlock_limit) { - - if (local_freeq) { - vm_page_unlock_queues(); - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, - vm_page_free_count, local_freed, delayed_unlock_limit, 1); - - vm_page_free_list(local_freeq, TRUE); - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, - vm_page_free_count, 0, 0, 1); - - local_freeq = NULL; - local_freed = 0; - vm_page_lock_queues(); - } else { - lck_mtx_yield(&vm_page_queue_lock); - } - - delayed_unlock = 1; - - /* - * continue the while loop processing - * the active queue... need to hold - * the page queues lock - */ + vm_pageout_delayed_unlock(&delayed_unlock, &local_freed, &local_freeq); } } @@ -2099,45 +2079,17 @@ Restart: memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_END); /********************************************************************** - * above this point we're playing with the active queue + * above this point we're playing with the active and secluded queues * below this point we're playing with the throttling mechanisms * and the inactive queue **********************************************************************/ done_moving_active_pages: -#if CONFIG_BACKGROUND_QUEUE - if ((vm_page_free_count + local_freed >= vm_page_free_target) && - ((vm_page_background_mode < VM_PAGE_BG_LEVEL_2) || (vm_page_background_count <= vm_page_background_target))) -#else if (vm_page_free_count + local_freed >= vm_page_free_target) -#endif { - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - } - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - - vm_page_unlock_queues(); - - if (local_freeq) { - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, - vm_page_free_count, local_freed, delayed_unlock_limit, 2); - - vm_page_free_list(local_freeq, TRUE); - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, - vm_page_free_count, local_freed, 0, 2); - - local_freeq = NULL; - local_freed = 0; - } - vm_consider_waking_compactor_swapper(); - - vm_page_lock_queues(); - + vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, + VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER); /* * make sure the pageout I/O threads are running * throttled in case there are still requests @@ -2146,7 +2098,7 @@ done_moving_active_pages: * fashion... so let's avoid interfering with foreground * activity */ - vm_pageout_adjust_io_throttles(iq, eq, TRUE); + vm_pageout_adjust_eq_iothrottle(eq, TRUE); /* * recalculate vm_page_inactivate_target @@ -2154,6 +2106,7 @@ done_moving_active_pages: vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count + vm_page_inactive_count + vm_page_speculative_count); +#ifndef CONFIG_EMBEDDED if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) && !vm_page_queue_empty(&vm_page_queue_active)) { /* @@ -2162,6 +2115,7 @@ done_moving_active_pages: */ continue; } +#endif lck_mtx_lock(&vm_page_queue_free_lock); if ((vm_page_free_count >= vm_page_free_target) && @@ -2279,7 +2233,7 @@ return_from_scan: continue; } - if (vm_page_speculative_count > vm_page_speculative_target) + if (vm_page_speculative_count > vm_page_speculative_target || force_speculative_aging == TRUE) can_steal = TRUE; else { if (!delay_speculative_age) { @@ -2310,6 +2264,8 @@ return_from_scan: if (can_steal == TRUE) vm_page_speculate_ageit(aq); } + force_speculative_aging = FALSE; + #if CONFIG_BACKGROUND_QUEUE if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0 && ((vm_page_background_mode == VM_PAGE_BG_DISABLED) || (vm_page_background_count <= vm_page_background_target))) @@ -2340,7 +2296,7 @@ return_from_scan: */ continue; } else - cache_evict_throttle = 100; + cache_evict_throttle = 1000; } if (cache_evict_throttle) cache_evict_throttle--; @@ -2362,10 +2318,14 @@ return_from_scan: else vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 7); #else - /* - * don't let the filecache_min fall below 33% of available memory... - */ - vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 3); + if (vm_compressor_out_of_space()) + vm_page_filecache_min = 0; + else { + /* + * don't let the filecache_min fall below 33% of available memory... + */ + vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 3); + } #endif if (vm_page_free_count < (vm_page_free_reserved / 4)) vm_page_filecache_min = 0; @@ -2411,31 +2371,8 @@ return_from_scan: case FCS_IDLE: if ((vm_page_free_count + local_freed) < vm_page_free_target) { - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - } - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - - vm_page_unlock_queues(); - - if (local_freeq) { - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, - vm_page_free_count, local_freed, delayed_unlock_limit, 3); - - vm_page_free_list(local_freeq, TRUE); - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, - vm_page_free_count, local_freed, 0, 3); - - local_freeq = NULL; - local_freed = 0; - } - thread_yield_internal(1); - - vm_page_lock_queues(); - + vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, + VM_PAGEOUT_PB_THREAD_YIELD); if (!VM_PAGE_Q_THROTTLED(iq)) { vm_pageout_scan_yield_unthrottled++; continue; @@ -2506,30 +2443,8 @@ reset_deadlock_timer: } vm_pageout_scan_delay: - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - } - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - - vm_page_unlock_queues(); - - if (local_freeq) { - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, - vm_page_free_count, local_freed, delayed_unlock_limit, 3); - - vm_page_free_list(local_freeq, TRUE); - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, - vm_page_free_count, local_freed, 0, 3); - - local_freeq = NULL; - local_freed = 0; - } - vm_consider_waking_compactor_swapper(); - - vm_page_lock_queues(); + vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, + VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER); if (flow_control.state == FCS_DELAYED && !VM_PAGE_Q_THROTTLED(iq)) { @@ -2556,14 +2471,14 @@ vm_pageout_scan_delay: * activity * * we don't want to hold vm_page_queue_free_lock when - * calling vm_pageout_adjust_io_throttles (since it + * calling vm_pageout_adjust_eq_iothrottle (since it * may cause other locks to be taken), we do the intitial * check outside of the lock. Once we take the lock, * we recheck the condition since it may have changed. * if it has, no problem, we will make the threads * non-throttled before actually blocking */ - vm_pageout_adjust_io_throttles(iq, eq, TRUE); + vm_pageout_adjust_eq_iothrottle(eq, TRUE); } lck_mtx_lock(&vm_page_queue_free_lock); @@ -2587,7 +2502,7 @@ vm_pageout_scan_delay: * running unthrottled since the sum of free + * clean pages is still under our free target */ - vm_pageout_adjust_io_throttles(iq, eq, FALSE); + vm_pageout_adjust_eq_iothrottle(eq, FALSE); } if (vm_page_cleaned_count > 0 && exceeded_burst_throttle == FALSE) { /* @@ -2626,7 +2541,6 @@ vm_pageout_scan_delay: memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END); vm_page_lock_queues(); - delayed_unlock = 1; iq->pgo_throttled = FALSE; @@ -2655,7 +2569,7 @@ consider_inactive: uint32_t inactive_external_count; #if CONFIG_BACKGROUND_QUEUE - ignore_reference = FALSE; + page_from_bg_q = FALSE; #endif /* CONFIG_BACKGROUND_QUEUE */ m = NULL; @@ -2666,42 +2580,33 @@ consider_inactive: assert(vm_page_queue_empty(&vm_page_queue_throttled)); } - -#if CONFIG_SECLUDED_MEMORY - if ((secluded_aging_policy == - SECLUDED_AGING_AFTER_INACTIVE) && - vm_page_secluded_count > vm_page_secluded_target) { - /* - * SECLUDED_AGING_AFTER_INACTIVE: - * Secluded pages have already been aged - * through the active and inactive queues, and - * we now have too many of them, so let's - * balance that queue by considering reclaiming - * the oldest page in the secluded queue. - */ - assert(!vm_page_queue_empty(&vm_page_queue_secluded)); - m = (vm_page_t) vm_page_queue_first(&vm_page_queue_secluded); - if (m->vm_page_object == 0) { - /* - * It's already a free page: - * just move it to a free queue. - */ - vm_page_queues_remove(m, TRUE); - assert(m->busy); - assert(m->pageq.next == 0); - assert(m->pageq.prev == 0); - m->snext = local_freeq; - local_freeq = m; - local_freed++; - goto done_with_inactivepage; - } - /* - * Not a free page: we've found our next - * "victim". - */ + /* + * Try for a clean-queue inactive page. + * These are pages that vm_pageout_scan tried to steal earlier, but + * were dirty and had to be cleaned. Pick them up now that they are clean. + */ + if (!vm_page_queue_empty(&vm_page_queue_cleaned)) { + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned); + + assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q); + break; } -#endif /* CONFIG_SECLUDED_MEMORY */ + + /* + * The next most eligible pages are ones we paged in speculatively, + * but which have not yet been touched and have been aged out. + */ + if (!vm_page_queue_empty(&sq->age_q)) { + m = (vm_page_t) vm_page_queue_first(&sq->age_q); + + assert(m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q); + + if (!m->dirty || force_anonymous == FALSE) + break; + else + m = NULL; + } #if CONFIG_BACKGROUND_QUEUE if (vm_page_background_mode != VM_PAGE_BG_DISABLED && (vm_page_background_count > vm_page_background_target)) { @@ -2723,40 +2628,26 @@ consider_inactive: * page. */ } else if (force_anonymous == FALSE || bg_m_object->internal) { - ignore_reference = TRUE; + + if (bg_m_object->internal && + ((vm_compressor_out_of_space() == TRUE) || + (vm_page_free_count < (vm_page_free_reserved / 4)))) { + + vm_pageout_skipped_bq_internal++; + } else { + page_from_bg_q = TRUE; - if (bg_m_object->internal) - vm_pageout_considered_bq_internal++; - else - vm_pageout_considered_bq_external++; + if (bg_m_object->internal) + vm_pageout_considered_bq_internal++; + else + vm_pageout_considered_bq_external++; - break; + break; + } } } #endif - /* - * The most eligible pages are ones we paged in speculatively, - * but which have not yet been touched. - */ - if (!vm_page_queue_empty(&sq->age_q) && force_anonymous == FALSE) { - m = (vm_page_t) vm_page_queue_first(&sq->age_q); - - assert(m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q); - - break; - } - /* - * Try a clean-queue inactive page. - */ - if (!vm_page_queue_empty(&vm_page_queue_cleaned)) { - m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned); - - assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q); - - break; - } - grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min); inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count; @@ -2775,7 +2666,7 @@ consider_inactive: * external page queues, as those are expected to be * much smaller relative to the global pools. */ - if (grab_anonymous) { + if (grab_anonymous == TRUE && !VM_PAGE_Q_THROTTLED(eq)) { if (vm_page_pageable_external_count > vm_page_filecache_min) { if ((vm_page_pageable_external_count * @@ -2791,7 +2682,6 @@ consider_inactive: #if DEVELOPMENT || DEBUG if (grab_anonymous) { vm_grab_anon_nops++; - } #endif } @@ -2828,41 +2718,25 @@ consider_inactive: /* * if we've gotten here, we have no victim page. - * if making clean, free the local freed list and return. - * if making free, check to see if we've finished balancing the queues - * yet, if we haven't just continue, else panic + * check to see if we've not finished balancing the queues + * or we have a page on the aged speculative queue that we + * skipped due to force_anonymous == TRUE.. or we have + * speculative pages that we can prematurely age... if + * one of these cases we'll keep going, else panic */ - vm_page_unlock_queues(); - - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - } - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - - if (local_freeq) { - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, - vm_page_free_count, local_freed, delayed_unlock_limit, 5); - - vm_page_free_list(local_freeq, TRUE); - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, - vm_page_free_count, local_freed, 0, 5); - - local_freeq = NULL; - local_freed = 0; - } - vm_page_lock_queues(); - delayed_unlock = 1; - force_anonymous = FALSE; + vm_pageout_no_victim++; if ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) - goto Restart; + goto done_with_inactivepage; if (!vm_page_queue_empty(&sq->age_q)) - goto Restart; + goto done_with_inactivepage; + if (vm_page_speculative_count) { + force_speculative_aging = TRUE; + goto done_with_inactivepage; + } panic("vm_pageout: no victim"); /* NOTREACHED */ @@ -2885,10 +2759,8 @@ consider_inactive: assert(m_object != kernel_object); assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr); - - if (page_prev_q_state != VM_PAGE_ON_SPECULATIVE_Q && - page_prev_q_state != VM_PAGE_ON_SECLUDED_Q) - vm_pageout_stats[vm_pageout_stat_now].considered++; + vm_pageout_stats[vm_pageout_stat_now].considered++; + vm_pageout_considered_page++; DTRACE_VM2(scan, int, 1, (uint64_t *), NULL); @@ -2974,32 +2846,7 @@ consider_inactive: assert(m_object == object); assert(VM_PAGE_OBJECT(m) == m_object); - if (catch_up_count) - catch_up_count--; - if (m->busy) { - if (m->encrypted_cleaning) { - /* - * ENCRYPTED SWAP: - * if this page has already been picked up as - * part of a page-out cluster, it will be busy - * because it is being encrypted (see - * vm_object_upl_request()). But we still - * want to demote it from "clean-in-place" - * (aka "adjacent") to "clean-and-free" (aka - * "target"), so let's ignore its "busy" bit - * here and proceed to check for "cleaning" a - * little bit below... - * - * CAUTION CAUTION: - * A "busy" page should still be left alone for - * most purposes, so we have to be very careful - * not to process that page too much. - */ - assert(m->cleaning); - goto consider_inactive_page; - } - /* * Somebody is already playing with this page. * Put it back on the appropriate queue @@ -3013,10 +2860,9 @@ requeue_page: if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) vm_page_enqueue_inactive(m, FALSE); else - vm_page_activate(m); - + vm_page_activate(m); #if CONFIG_BACKGROUND_QUEUE - if (ignore_reference == TRUE) { + if (page_from_bg_q == TRUE) { if (m_object->internal) vm_pageout_rejected_bq_internal++; else @@ -3074,10 +2920,6 @@ reclaim_page: local_freeq = m; local_freed++; -#if CONFIG_SECLUDED_MEMORY - if (page_prev_q_state == VM_PAGE_ON_SECLUDED_Q) - vm_pageout_freed_from_secluded++; -#endif /* CONFIG_SECLUDED_MEMORY */ if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) vm_pageout_freed_from_speculative++; else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) @@ -3085,9 +2927,7 @@ reclaim_page: else vm_pageout_freed_from_inactive_clean++; - if (page_prev_q_state != VM_PAGE_ON_SPECULATIVE_Q && - page_prev_q_state != VM_PAGE_ON_SECLUDED_Q) - vm_pageout_stats[vm_pageout_stat_now].reclaimed++; + vm_pageout_stats[vm_pageout_stat_now].reclaimed_clean++; inactive_burst_count = 0; goto done_with_inactivepage; @@ -3141,29 +2981,6 @@ reclaim_page: goto reactivate_page; } } - -consider_inactive_page: - if (m->busy) { - /* - * CAUTION CAUTION: - * A "busy" page should always be left alone, except... - */ - if (m->cleaning && m->encrypted_cleaning) { - /* - * ENCRYPTED_SWAP: - * We could get here with a "busy" page - * if it's being encrypted during a - * "clean-in-place" operation. We'll deal - * with it right away by testing if it has been - * referenced and either reactivating it or - * promoting it from "clean-in-place" to - * "clean-and-free". - */ - } else { - panic("\"busy\" page considered for pageout\n"); - } - } - /* * If it's being used, reactivate. * (Fictitious pages are either busy or absent.) @@ -3189,10 +3006,6 @@ consider_inactive_page: * We can leave the page mapped, and upl_commit_range * will put it on the clean queue. * - * note: if m->encrypted_cleaning == TRUE, then - * m->cleaning == TRUE - * and we'll handle it here - * * if (m->free_when_done && !m->cleaning) * an msync INVALIDATE is in progress... * this page has been marked for destruction @@ -3230,32 +3043,10 @@ consider_inactive_page: VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, m_object); } -#if CONFIG_SECLUDED_MEMORY - if (secluded_for_filecache && - vm_page_secluded_target > 0 && - m_object->eligible_for_secluded && - secluded_aging_policy == SECLUDED_AGING_FIFO) { - /* - * SECLUDED_AGING_FIFO: - * This victim page is eligible for the secluded pool - * and we're not aging secluded pages, so let's not - * reactivate it if it's been re-referenced. - * Later on, we'll move it to the secluded queue - * instead of freeing it. - */ - ignore_reference_secluded = TRUE; - } else { - ignore_reference_secluded = FALSE; - } -#endif /* CONFIG_SECLUDED_MEMORY */ - if (!m->no_cache && #if CONFIG_BACKGROUND_QUEUE - ignore_reference == FALSE && + page_from_bg_q == FALSE && #endif -#if CONFIG_SECLUDED_MEMORY - ignore_reference_secluded == FALSE && -#endif /* CONFIG_SECLUDED_MEMORY */ (m->reference || (m->xpmapped && !object->internal && (vm_page_xpmapped_external_count < (vm_page_external_count / 4))))) { /* @@ -3268,8 +3059,6 @@ consider_inactive_page: */ if (++reactivated_this_call >= reactivate_limit) { vm_pageout_reactivation_limit_exceeded++; - } else if (catch_up_count) { - vm_pageout_catch_ups++; } else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) { vm_pageout_inactive_force_reclaim++; } else { @@ -3277,7 +3066,6 @@ consider_inactive_page: if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) vm_pageout_cleaned_reference_reactivated++; - reactivate_page: if ( !object->internal && object->pager != MEMORY_OBJECT_NULL && vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) { @@ -3297,7 +3085,7 @@ must_activate_page: inactive_burst_count = 0; } #if CONFIG_BACKGROUND_QUEUE - if (ignore_reference == TRUE) { + if (page_from_bg_q == TRUE) { if (m_object->internal) vm_pageout_rejected_bq_internal++; else @@ -3306,11 +3094,6 @@ must_activate_page: #endif if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) vm_pageout_cleaned_reactivated++; -#if CONFIG_SECLUDED_MEMORY - if (page_prev_q_state == VM_PAGE_ON_SECLUDED_Q) - vm_pageout_secluded_reactivated++; -#endif /* CONFIG_SECLUDED_MEMORY */ - vm_pageout_inactive_used++; goto done_with_inactivepage; @@ -3401,7 +3184,7 @@ throttle_inactive: * the deadlock will break, the external pageout queue will empty and it will no longer * be throttled * - * if we have jestam configured, keep a count of the pages reactivated this way so + * if we have jetsam configured, keep a count of the pages reactivated this way so * that we can try to find clean pages in the active/inactive queues before * deciding to jetsam a process */ @@ -3414,7 +3197,7 @@ throttle_inactive: vm_page_active_count++; vm_page_pageable_external_count++; - vm_pageout_adjust_io_throttles(iq, eq, FALSE); + vm_pageout_adjust_eq_iothrottle(eq, FALSE); #if CONFIG_MEMORYSTATUS && CONFIG_JETSAM vm_pageout_inactive_external_forced_reactivate_limit--; @@ -3450,7 +3233,6 @@ throttle_inactive: goto done_with_inactivepage; } else { vm_pageout_scan_inactive_throttled_internal++; - goto must_activate_page; } } @@ -3537,37 +3319,6 @@ throttle_inactive: vm_pageout_inactive_clean++; } - -#if CONFIG_SECLUDED_MEMORY - if (secluded_for_filecache && - vm_page_secluded_target > 0 && - !m->fictitious && - m_object->eligible_for_secluded && - num_tasks_can_use_secluded_mem == 0 && - (secluded_aging_policy == SECLUDED_AGING_FIFO || - ((secluded_aging_policy == - SECLUDED_AGING_AFTER_INACTIVE) && - (page_prev_q_state != VM_PAGE_ON_SECLUDED_Q)))) { - assert(page_prev_q_state != VM_PAGE_ON_SECLUDED_Q); - assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); - LCK_MTX_ASSERT(&vm_page_queue_lock, - LCK_MTX_ASSERT_OWNED); - vm_page_queue_enter(&vm_page_queue_secluded, - m, - vm_page_t, - pageq); - m->vm_page_q_state = VM_PAGE_ON_SECLUDED_Q; - vm_object_unlock(m_object); - object = VM_OBJECT_NULL; - vm_page_secluded_count++; - vm_page_secluded_count_inuse++; - assert(!m_object->internal); -// vm_page_pageable_external_count++; - m = VM_PAGE_NULL; - goto done_with_inactivepage; - } -#endif /* CONFIG_SECLUDED_MEMORY */ - /* * OK, at this point we have found a page we are going to free. */ @@ -3611,8 +3362,11 @@ throttle_inactive: #endif /* CONFIG_JETSAM */ #endif /* VM_PRESSURE_EVENTS */ - if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) + if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) + vm_pageout_speculative_dirty++; + else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) vm_pageout_inactive_anonymous++; + if (object->internal) vm_pageout_inactive_dirty_internal++; else @@ -3624,44 +3378,17 @@ throttle_inactive: * anyway, so we may as well put it on the clean queue first and take it from there later * if necessary. that way, we'll ensure we don't free up too much. -mj */ - vm_pageout_cluster(m, FALSE, FALSE); + vm_pageout_cluster(m); done_with_inactivepage: if (delayed_unlock++ > delayed_unlock_limit || try_failed == TRUE) { - boolean_t need_delay = TRUE; - - if (object != NULL) { - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - vm_object_unlock(object); - object = NULL; - } - vm_page_unlock_queues(); - - if (local_freeq) { - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, - vm_page_free_count, local_freed, delayed_unlock_limit, 4); - - vm_page_free_list(local_freeq, TRUE); - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, - vm_page_free_count, local_freed, 0, 4); - - local_freeq = NULL; - local_freed = 0; - need_delay = FALSE; - } - vm_consider_waking_compactor_swapper(); - vm_page_lock_queues(); - - if (need_delay == TRUE) + vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, + VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER); + if (try_failed == TRUE) lck_mtx_yield(&vm_page_queue_lock); - - delayed_unlock = 1; } - vm_pageout_considered_page++; /* * back to top of pageout scan loop @@ -3721,9 +3448,11 @@ vm_pageout_continue(void) DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL); vm_pageout_scan_event_counter++; +#if !CONFIG_EMBEDDED lck_mtx_lock(&vm_page_queue_free_lock); vm_pageout_running = TRUE; lck_mtx_unlock(&vm_page_queue_free_lock); +#endif /* CONFIG_EMBEDDED */ vm_pageout_scan(); /* @@ -3734,11 +3463,13 @@ vm_pageout_continue(void) assert(vm_page_free_wanted_privileged == 0); assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT); +#if !CONFIG_EMBEDDED vm_pageout_running = FALSE; if (vm_pageout_waiter) { vm_pageout_waiter = FALSE; thread_wakeup((event_t)&vm_pageout_waiter); } +#endif /* !CONFIG_EMBEDDED */ lck_mtx_unlock(&vm_page_queue_free_lock); vm_page_unlock_queues(); @@ -3748,6 +3479,7 @@ vm_pageout_continue(void) /*NOTREACHED*/ } +#if !CONFIG_EMBEDDED kern_return_t vm_pageout_wait(uint64_t deadline) { @@ -3766,6 +3498,7 @@ vm_pageout_wait(uint64_t deadline) return (kr); } +#endif /* !CONFIG_EMBEDDED */ static void @@ -3776,7 +3509,10 @@ vm_pageout_iothread_external_continue(struct vm_pageout_queue *q) vm_object_offset_t offset; memory_object_t pager; - + /* On systems without a compressor, the external IO thread clears its + * VM privileged bit to accommodate large allocations (e.g. bulk UPL + * creation) + */ if (vm_pageout_internal_iothread != THREAD_NULL) current_thread()->options &= ~TH_OPT_VMPRIV; @@ -3913,15 +3649,19 @@ vm_pageout_iothread_external_continue(struct vm_pageout_queue *q) } -uint32_t vm_compressor_failed; - #define MAX_FREE_BATCH 32 uint32_t vm_compressor_time_thread; /* Set via sysctl to record time accrued by * this thread. */ -uint64_t vm_compressor_thread_runtime; -static void + +#if DEVELOPMENT || DEBUG +uint64_t compressor_epoch_start, compressor_epoch_stop, compressor_threads_runtime; +#endif + +void +vm_pageout_iothread_internal_continue(struct cq *); +void vm_pageout_iothread_internal_continue(struct cq *cq) { struct vm_pageout_queue *q; @@ -3932,8 +3672,10 @@ vm_pageout_iothread_internal_continue(struct cq *cq) vm_page_t local_freeq = NULL; int local_freed = 0; int local_batch_size; - - + int ncomps = 0; +#if DEVELOPMENT || DEBUG + boolean_t marked_active = FALSE; +#endif KERNEL_DEBUG(0xe040000c | DBG_FUNC_END, 0, 0, 0, 0, 0); q = cq->q; @@ -3950,9 +3692,18 @@ vm_pageout_iothread_internal_continue(struct cq *cq) local_q = NULL; KERNEL_DEBUG(0xe0400014 | DBG_FUNC_START, 0, 0, 0, 0, 0); - - vm_page_lock_queues(); + vm_page_lock_queues(); +#if DEVELOPMENT || DEBUG + if (marked_active == FALSE) { + vmct_active++; + vmct_state[cq->id] = VMCT_ACTIVE; + marked_active = TRUE; + if (vmct_active == 1) { + compressor_epoch_start = mach_absolute_time(); + } + } +#endif KERNEL_DEBUG(0xe0400014 | DBG_FUNC_END, 0, 0, 0, 0, 0); KERNEL_DEBUG(0xe0400018 | DBG_FUNC_START, q->pgo_laundry, 0, 0, 0, 0); @@ -3985,8 +3736,9 @@ vm_pageout_iothread_internal_continue(struct cq *cq) vm_page_unlock_queues(); #if !RECORD_THE_COMPRESSED_DATA - if (pages_left_on_q >= local_batch_size && cq->id < (vm_compressor_thread_count - 1)) + if (pages_left_on_q >= local_batch_size && cq->id < (vm_compressor_thread_count - 1)) { thread_wakeup((event_t) ((uintptr_t)&q->pgo_pending + cq->id + 1)); + } #endif KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, q->pgo_laundry, 0, 0, 0, 0); @@ -3999,12 +3751,13 @@ vm_pageout_iothread_internal_continue(struct cq *cq) m->snext = NULL; if (vm_pageout_compress_page(&cq->current_chead, cq->scratch_buf, m, FALSE) == KERN_SUCCESS) { - + ncomps++; m->snext = local_freeq; local_freeq = m; local_freed++; if (local_freed >= MAX_FREE_BATCH) { + vm_pageout_freed_after_compression += local_freed; vm_page_free_list(local_freeq, TRUE); local_freeq = NULL; @@ -4017,8 +3770,9 @@ vm_pageout_iothread_internal_continue(struct cq *cq) int need_wakeup = 0; if (local_freeq) { - vm_page_free_list(local_freeq, TRUE); + vm_pageout_freed_after_compression += local_freed; + vm_page_free_list(local_freeq, TRUE); local_freeq = NULL; local_freed = 0; @@ -4046,8 +3800,9 @@ vm_pageout_iothread_internal_continue(struct cq *cq) #endif } if (local_freeq) { + vm_pageout_freed_after_compression += local_freed; + vm_page_free_list(local_freeq, TRUE); - local_freeq = NULL; local_freed = 0; } @@ -4066,11 +3821,36 @@ vm_pageout_iothread_internal_continue(struct cq *cq) q->pgo_idle = TRUE; assert_wait((event_t) ((uintptr_t)&q->pgo_pending + cq->id), THREAD_UNINT); - vm_page_unlock_queues(); +#if DEVELOPMENT || DEBUG + if (marked_active == TRUE) { + vmct_active--; + vmct_state[cq->id] = VMCT_IDLE; + + if (vmct_active == 0) { + compressor_epoch_stop = mach_absolute_time(); + assert(compressor_epoch_stop > compressor_epoch_start); + /* This interval includes intervals where one or more + * compressor threads were pre-empted + */ + vmct_stats.vmct_cthreads_total += compressor_epoch_stop - compressor_epoch_start; + } + } +#endif + vm_page_unlock_queues(); +#if DEVELOPMENT || DEBUG if (__improbable(vm_compressor_time_thread)) { - vm_compressor_thread_runtime = thread_get_runtime_self(); + vmct_stats.vmct_runtimes[cq->id] = thread_get_runtime_self(); + vmct_stats.vmct_pages[cq->id] += ncomps; + vmct_stats.vmct_iterations[cq->id]++; + if (ncomps > vmct_stats.vmct_maxpages[cq->id]) { + vmct_stats.vmct_maxpages[cq->id] = ncomps; + } + if (ncomps < vmct_stats.vmct_minpages[cq->id]) { + vmct_stats.vmct_minpages[cq->id] = ncomps; + } } +#endif KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, 0, 0, 0, 0, 0); @@ -4079,20 +3859,6 @@ vm_pageout_iothread_internal_continue(struct cq *cq) } - -static void -vm_pageout_immediate(vm_page_t m, boolean_t object_locked_by_caller) -{ - assert(vm_pageout_immediate_scratch_buf); - - if (vm_pageout_compress_page(&vm_pageout_immediate_chead, vm_pageout_immediate_scratch_buf, m, object_locked_by_caller) == KERN_SUCCESS) { - - vm_page_free_prepare_object(m, TRUE); - vm_page_release(m, TRUE); - } -} - - kern_return_t vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, boolean_t object_locked_by_caller) { @@ -4222,20 +3988,15 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b static void -vm_pageout_adjust_io_throttles(struct vm_pageout_queue *iq, struct vm_pageout_queue *eq, boolean_t req_lowpriority) +vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpriority) { uint32_t policy; - boolean_t set_iq = FALSE; - boolean_t set_eq = FALSE; if (hibernate_cleaning_in_progress == TRUE) req_lowpriority = FALSE; - if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority) - set_eq = TRUE; - - if (set_iq == TRUE || set_eq == TRUE) { - + if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority) { + vm_page_unlock_queues(); if (req_lowpriority == TRUE) { @@ -4245,18 +4006,11 @@ vm_pageout_adjust_io_throttles(struct vm_pageout_queue *iq, struct vm_pageout_qu policy = THROTTLE_LEVEL_PAGEOUT_UNTHROTTLED; DTRACE_VM(laundryunthrottle); } - if (set_iq == TRUE) { - proc_set_thread_policy_with_tid(kernel_task, iq->pgo_tid, - TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy); + proc_set_thread_policy_with_tid(kernel_task, eq->pgo_tid, + TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy); - iq->pgo_lowpriority = req_lowpriority; - } - if (set_eq == TRUE) { - proc_set_thread_policy_with_tid(kernel_task, eq->pgo_tid, - TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy); + eq->pgo_lowpriority = req_lowpriority; - eq->pgo_lowpriority = req_lowpriority; - } vm_page_lock_queues(); } } @@ -4306,6 +4060,11 @@ vm_pageout_iothread_internal(struct cq *cq) if (vm_restricted_to_single_processor == TRUE) thread_vm_bind_group_add(); + + thread_set_thread_name(current_thread(), "VM_compressor"); +#if DEVELOPMENT || DEBUG + vmct_stats.vmct_minpages[cq->id] = INT32_MAX; +#endif vm_pageout_iothread_internal_continue(cq); /*NOTREACHED*/ @@ -4341,9 +4100,16 @@ vm_pressure_response(void) if (vm_pressure_events_enabled == FALSE) return; +#if CONFIG_EMBEDDED + + available_memory = (uint64_t) memorystatus_available_pages; + +#else /* CONFIG_EMBEDDED */ available_memory = (uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY; + memorystatus_available_pages = (uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY; +#endif /* CONFIG_EMBEDDED */ total_pages = (unsigned int) atop_64(max_mem); #if CONFIG_SECLUDED_MEMORY @@ -4414,7 +4180,11 @@ vm_pressure_response(void) kern_return_t mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused unsigned int *pressure_level) { -#if !VM_PRESSURE_EVENTS +#if CONFIG_EMBEDDED + + return KERN_FAILURE; + +#elif !VM_PRESSURE_EVENTS return KERN_FAILURE; @@ -4493,37 +4263,82 @@ compute_pageout_gc_throttle(__unused void *arg) } } +/* + * vm_pageout_garbage_collect can also be called when the zone allocator needs + * to call zone_gc on a different thread in order to trigger zone-map-exhaustion + * jetsams. We need to check if the zone map size is above its jetsam limit to + * decide if this was indeed the case. + * + * We need to do this on a different thread because of the following reasons: + * + * 1. In the case of synchronous jetsams, the leaking process can try to jetsam + * itself causing the system to hang. We perform synchronous jetsams if we're + * leaking in the VM map entries zone, so the leaking process could be doing a + * zalloc for a VM map entry while holding its vm_map lock, when it decides to + * jetsam itself. We also need the vm_map lock on the process termination path, + * which would now lead the dying process to deadlock against itself. + * + * 2. The jetsam path might need to allocate zone memory itself. We could try + * using the non-blocking variant of zalloc for this path, but we can still + * end up trying to do a kernel_memory_allocate when the zone_map is almost + * full. + */ + +extern boolean_t is_zone_map_nearing_exhaustion(void); -static void +void vm_pageout_garbage_collect(int collect) { - if (collect) { - boolean_t buf_large_zfree = FALSE; - boolean_t first_try = TRUE; + if (is_zone_map_nearing_exhaustion()) { + /* + * Woken up by the zone allocator for zone-map-exhaustion jetsams. + * + * Bail out after calling zone_gc (which triggers the + * zone-map-exhaustion jetsams). If we fall through, the subsequent + * operations that clear out a bunch of caches might allocate zone + * memory themselves (for eg. vm_map operations would need VM map + * entries). Since the zone map is almost full at this point, we + * could end up with a panic. We just need to quickly jetsam a + * process and exit here. + * + * It could so happen that we were woken up to relieve memory + * pressure and the zone map also happened to be near its limit at + * the time, in which case we'll skip out early. But that should be + * ok; if memory pressure persists, the thread will simply be woken + * up again. + */ + consider_zone_gc(TRUE); - stack_collect(); + } else { + /* Woken up by vm_pageout_scan or compute_pageout_gc_throttle. */ + boolean_t buf_large_zfree = FALSE; + boolean_t first_try = TRUE; - consider_machine_collect(); - m_drain(); + stack_collect(); - do { - if (consider_buffer_cache_collect != NULL) { - buf_large_zfree = (*consider_buffer_cache_collect)(0); - } - if (first_try == TRUE || buf_large_zfree == TRUE) { - /* - * consider_zone_gc should be last, because the other operations - * might return memory to zones. - */ - consider_zone_gc(); - } - first_try = FALSE; + consider_machine_collect(); + m_drain(); + + do { + if (consider_buffer_cache_collect != NULL) { + buf_large_zfree = (*consider_buffer_cache_collect)(0); + } + if (first_try == TRUE || buf_large_zfree == TRUE) { + /* + * consider_zone_gc should be last, because the other operations + * might return memory to zones. + */ + consider_zone_gc(FALSE); + } + first_try = FALSE; - } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target); + } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target); - consider_machine_adjust(); + consider_machine_adjust(); + } } + assert_wait((event_t) &vm_pageout_garbage_collect, THREAD_UNINT); thread_block_parameter((thread_continue_t) vm_pageout_garbage_collect, (void *)1); @@ -4538,26 +4353,6 @@ extern vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end; #endif /* VM_PAGE_BUCKETS_CHECK */ -#define FBDP_TEST_COLLAPSE_COMPRESSOR 0 -#define FBDP_TEST_WIRE_AND_EXTRACT 0 -#define FBDP_TEST_PAGE_WIRE_OVERFLOW 0 -#define FBDP_TEST_KERNEL_OBJECT_FAULT 0 - -#if FBDP_TEST_KERNEL_OBJECT_FAULT -#endif /* FBDP_TEST_KERNEL_OBJECT_FAULT */ - -#if FBDP_TEST_COLLAPSE_COMPRESSOR -extern boolean_t vm_object_collapse_compressor_allowed; -#include -#endif /* FBDP_TEST_COLLAPSE_COMPRESSOR */ - -#if FBDP_TEST_WIRE_AND_EXTRACT -extern ledger_template_t task_ledger_template; -#include -extern ppnum_t vm_map_get_phys_page(vm_map_t map, - vm_offset_t offset); -#endif /* FBDP_TEST_WIRE_AND_EXTRACT */ - void vm_set_restrictions() @@ -4591,7 +4386,6 @@ vm_set_restrictions() } } - void vm_pageout(void) { @@ -4607,7 +4401,7 @@ vm_pageout(void) thread_lock(self); self->options |= TH_OPT_VMPRIV; - sched_set_thread_base_priority(self, BASEPRI_PREEMPT - 1); + sched_set_thread_base_priority(self, BASEPRI_VM); thread_unlock(self); if (!self->reserved_stack) @@ -4618,6 +4412,8 @@ vm_pageout(void) splx(s); + thread_set_thread_name(current_thread(), "VM_pageout_scan"); + /* * Initialize some paging parameters. */ @@ -4698,7 +4494,7 @@ vm_pageout(void) /* external pageout and garbage collection threads started here */ result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL, - BASEPRI_PREEMPT - 1, + BASEPRI_VM, &vm_pageout_external_iothread); if (result != KERN_SUCCESS) panic("vm_pageout_iothread_external: create failed"); @@ -4796,280 +4592,7 @@ vm_pageout(void) vm_object_tracking_init(); #endif /* VM_OBJECT_TRACKING */ - -#if FBDP_TEST_COLLAPSE_COMPRESSOR - vm_object_size_t backing_size, top_size; - vm_object_t backing_object, top_object; - vm_map_offset_t backing_offset, top_offset; - unsigned char *backing_address, *top_address; - kern_return_t kr; - - printf("FBDP_TEST_COLLAPSE_COMPRESSOR:\n"); - - /* create backing object */ - backing_size = 15 * PAGE_SIZE; - backing_object = vm_object_allocate(backing_size); - assert(backing_object != VM_OBJECT_NULL); - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n", - backing_object); - /* map backing object */ - backing_offset = 0; - kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0, - VM_FLAGS_ANYWHERE, backing_object, 0, FALSE, - VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT); - assert(kr == KERN_SUCCESS); - backing_address = (unsigned char *) backing_offset; - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " - "mapped backing object %p at 0x%llx\n", - backing_object, (uint64_t) backing_offset); - /* populate with pages to be compressed in backing object */ - backing_address[0x1*PAGE_SIZE] = 0xB1; - backing_address[0x4*PAGE_SIZE] = 0xB4; - backing_address[0x7*PAGE_SIZE] = 0xB7; - backing_address[0xa*PAGE_SIZE] = 0xBA; - backing_address[0xd*PAGE_SIZE] = 0xBD; - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " - "populated pages to be compressed in " - "backing_object %p\n", backing_object); - /* compress backing object */ - vm_object_pageout(backing_object); - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n", - backing_object); - /* wait for all the pages to be gone */ - while (*(volatile int *)&backing_object->resident_page_count != 0) - IODelay(10); - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n", - backing_object); - /* populate with pages to be resident in backing object */ - backing_address[0x0*PAGE_SIZE] = 0xB0; - backing_address[0x3*PAGE_SIZE] = 0xB3; - backing_address[0x6*PAGE_SIZE] = 0xB6; - backing_address[0x9*PAGE_SIZE] = 0xB9; - backing_address[0xc*PAGE_SIZE] = 0xBC; - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " - "populated pages to be resident in " - "backing_object %p\n", backing_object); - /* leave the other pages absent */ - /* mess with the paging_offset of the backing_object */ - assert(backing_object->paging_offset == 0); - backing_object->paging_offset = 0x3000; - - /* create top object */ - top_size = 9 * PAGE_SIZE; - top_object = vm_object_allocate(top_size); - assert(top_object != VM_OBJECT_NULL); - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: created top object %p\n", - top_object); - /* map top object */ - top_offset = 0; - kr = vm_map_enter(kernel_map, &top_offset, top_size, 0, - VM_FLAGS_ANYWHERE, top_object, 0, FALSE, - VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT); - assert(kr == KERN_SUCCESS); - top_address = (unsigned char *) top_offset; - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " - "mapped top object %p at 0x%llx\n", - top_object, (uint64_t) top_offset); - /* populate with pages to be compressed in top object */ - top_address[0x3*PAGE_SIZE] = 0xA3; - top_address[0x4*PAGE_SIZE] = 0xA4; - top_address[0x5*PAGE_SIZE] = 0xA5; - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " - "populated pages to be compressed in " - "top_object %p\n", top_object); - /* compress top object */ - vm_object_pageout(top_object); - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n", - top_object); - /* wait for all the pages to be gone */ - while (top_object->resident_page_count != 0); - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n", - top_object); - /* populate with pages to be resident in top object */ - top_address[0x0*PAGE_SIZE] = 0xA0; - top_address[0x1*PAGE_SIZE] = 0xA1; - top_address[0x2*PAGE_SIZE] = 0xA2; - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " - "populated pages to be resident in " - "top_object %p\n", top_object); - /* leave the other pages absent */ - - /* link the 2 objects */ - vm_object_reference(backing_object); - top_object->shadow = backing_object; - top_object->vo_shadow_offset = 0x3000; - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n", - top_object, backing_object); - - /* unmap backing object */ - vm_map_remove(kernel_map, - backing_offset, - backing_offset + backing_size, - 0); - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " - "unmapped backing_object %p [0x%llx:0x%llx]\n", - backing_object, - (uint64_t) backing_offset, - (uint64_t) (backing_offset + backing_size)); - - /* collapse */ - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object); - vm_object_lock(top_object); - vm_object_collapse(top_object, 0, FALSE); - vm_object_unlock(top_object); - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: collapsed %p\n", top_object); - - /* did it work? */ - if (top_object->shadow != VM_OBJECT_NULL) { - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: not collapsed\n"); - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n"); - if (vm_object_collapse_compressor_allowed) { - panic("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n"); - } - } else { - /* check the contents of the mapping */ - unsigned char expect[9] = - { 0xA0, 0xA1, 0xA2, /* resident in top */ - 0xA3, 0xA4, 0xA5, /* compressed in top */ - 0xB9, /* resident in backing + shadow_offset */ - 0xBD, /* compressed in backing + shadow_offset + paging_offset */ - 0x00 }; /* absent in both */ - unsigned char actual[9]; - unsigned int i, errors; - - errors = 0; - for (i = 0; i < sizeof (actual); i++) { - actual[i] = (unsigned char) top_address[i*PAGE_SIZE]; - if (actual[i] != expect[i]) { - errors++; - } - } - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " - "actual [%x %x %x %x %x %x %x %x %x] " - "expect [%x %x %x %x %x %x %x %x %x] " - "%d errors\n", - actual[0], actual[1], actual[2], actual[3], - actual[4], actual[5], actual[6], actual[7], - actual[8], - expect[0], expect[1], expect[2], expect[3], - expect[4], expect[5], expect[6], expect[7], - expect[8], - errors); - if (errors) { - panic("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n"); - } else { - printf("FBDP_TEST_COLLAPSE_COMPRESSOR: PASS\n"); - } - } -#endif /* FBDP_TEST_COLLAPSE_COMPRESSOR */ - -#if FBDP_TEST_WIRE_AND_EXTRACT - ledger_t ledger; - vm_map_t user_map, wire_map; - mach_vm_address_t user_addr, wire_addr; - mach_vm_size_t user_size, wire_size; - mach_vm_offset_t cur_offset; - vm_prot_t cur_prot, max_prot; - ppnum_t user_ppnum, wire_ppnum; - kern_return_t kr; - - ledger = ledger_instantiate(task_ledger_template, - LEDGER_CREATE_ACTIVE_ENTRIES); - user_map = vm_map_create(pmap_create(ledger, 0, PMAP_CREATE_64BIT), - 0x100000000ULL, - 0x200000000ULL, - TRUE); - wire_map = vm_map_create(NULL, - 0x100000000ULL, - 0x200000000ULL, - TRUE); - user_addr = 0; - user_size = 0x10000; - kr = mach_vm_allocate(user_map, - &user_addr, - user_size, - VM_FLAGS_ANYWHERE); - assert(kr == KERN_SUCCESS); - wire_addr = 0; - wire_size = user_size; - kr = mach_vm_remap(wire_map, - &wire_addr, - wire_size, - 0, - VM_FLAGS_ANYWHERE, - user_map, - user_addr, - FALSE, - &cur_prot, - &max_prot, - VM_INHERIT_NONE); - assert(kr == KERN_SUCCESS); - for (cur_offset = 0; - cur_offset < wire_size; - cur_offset += PAGE_SIZE) { - kr = vm_map_wire_and_extract(wire_map, - wire_addr + cur_offset, - VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK), - TRUE, - &wire_ppnum); - assert(kr == KERN_SUCCESS); - user_ppnum = vm_map_get_phys_page(user_map, - user_addr + cur_offset); - printf("FBDP_TEST_WIRE_AND_EXTRACT: kr=0x%x " - "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n", - kr, - user_map, user_addr + cur_offset, user_ppnum, - wire_map, wire_addr + cur_offset, wire_ppnum); - if (kr != KERN_SUCCESS || - wire_ppnum == 0 || - wire_ppnum != user_ppnum) { - panic("FBDP_TEST_WIRE_AND_EXTRACT: FAIL\n"); - } - } - cur_offset -= PAGE_SIZE; - kr = vm_map_wire_and_extract(wire_map, - wire_addr + cur_offset, - VM_PROT_DEFAULT, - TRUE, - &wire_ppnum); - assert(kr == KERN_SUCCESS); - printf("FBDP_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x " - "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n", - kr, - user_map, user_addr + cur_offset, user_ppnum, - wire_map, wire_addr + cur_offset, wire_ppnum); - if (kr != KERN_SUCCESS || - wire_ppnum == 0 || - wire_ppnum != user_ppnum) { - panic("FBDP_TEST_WIRE_AND_EXTRACT: FAIL\n"); - } - - printf("FBDP_TEST_WIRE_AND_EXTRACT: PASS\n"); -#endif /* FBDP_TEST_WIRE_AND_EXTRACT */ - -#if FBDP_TEST_PAGE_WIRE_OVERFLOW - vm_object_t fbdp_object; - vm_page_t fbdp_page; - - printf("FBDP_TEST_PAGE_WIRE_OVERFLOW: starting...\n"); - - fbdp_object = vm_object_allocate(PAGE_SIZE); - vm_object_lock(fbdp_object); - fbdp_page = vm_page_alloc(fbdp_object, 0x0); - vm_page_lock_queues(); - do { - vm_page_wire(fbdp_page, 1, FALSE); - } while (fbdp_page->wire_count != 0); - vm_page_unlock_queues(); - vm_object_unlock(fbdp_object); - panic("FBDP(%p,%p): wire_count overflow not detected\n", - fbdp_object, fbdp_page); -#endif /* FBDP_TEST_PAGE_WIRE_OVERFLOW */ - -#if FBDP_TEST_KERNEL_OBJECT_FAULT - { - } -#endif /* FBDP_TEST_KERNEL_OBJECT_FAULT */ + vm_tests(); vm_pageout_continue(); @@ -5099,7 +4622,11 @@ vm_pageout(void) +#if CONFIG_EMBEDDED +int vm_compressor_thread_count = 1; +#else int vm_compressor_thread_count = 2; +#endif kern_return_t vm_pageout_internal_start(void) @@ -5116,6 +4643,7 @@ vm_pageout_internal_start(void) assert(hinfo.max_cpus > 0); + PE_parse_boot_argn("vmcomp_threads", &vm_compressor_thread_count, sizeof(vm_compressor_thread_count)); if (vm_compressor_thread_count >= hinfo.max_cpus) vm_compressor_thread_count = hinfo.max_cpus - 1; if (vm_compressor_thread_count <= 0) @@ -5123,22 +4651,17 @@ vm_pageout_internal_start(void) else if (vm_compressor_thread_count > MAX_COMPRESSOR_THREAD_COUNT) vm_compressor_thread_count = MAX_COMPRESSOR_THREAD_COUNT; - if (vm_compressor_immediate_preferred == TRUE) { - vm_pageout_immediate_chead = NULL; - vm_pageout_immediate_scratch_buf = kalloc(vm_compressor_get_encode_scratch_size()); - - vm_compressor_thread_count = 1; - } - vm_pageout_queue_internal.pgo_maxlaundry = (vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX; + PE_parse_boot_argn("vmpgoi_maxlaundry", &vm_pageout_queue_internal.pgo_maxlaundry, sizeof(vm_pageout_queue_internal.pgo_maxlaundry)); + for (i = 0; i < vm_compressor_thread_count; i++) { ciq[i].id = i; ciq[i].q = &vm_pageout_queue_internal; ciq[i].current_chead = NULL; ciq[i].scratch_buf = kalloc(COMPRESSOR_SCRATCH_BUF_SIZE); - result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, (void *)&ciq[i], BASEPRI_PREEMPT - 1, &vm_pageout_internal_iothread); + result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, (void *)&ciq[i], BASEPRI_VM, &vm_pageout_internal_iothread); if (result == KERN_SUCCESS) thread_deallocate(vm_pageout_internal_iothread); @@ -5458,7 +4981,8 @@ vm_object_upl_request( upl_t *upl_ptr, upl_page_info_array_t user_page_list, unsigned int *page_list_count, - upl_control_flags_t cntrl_flags) + upl_control_flags_t cntrl_flags, + vm_tag_t tag) { vm_page_t dst_page = VM_PAGE_NULL; vm_object_offset_t dst_offset; @@ -5569,16 +5093,6 @@ vm_object_upl_request( upl->flags |= UPL_SHADOWED; } - /* - * ENCRYPTED SWAP: - * Just mark the UPL as "encrypted" here. - * We'll actually encrypt the pages later, - * in upl_encrypt(), when the caller has - * selected which pages need to go to swap. - */ - if (cntrl_flags & UPL_ENCRYPT) - upl->flags |= UPL_ENCRYPTED; - if (cntrl_flags & UPL_FOR_PAGEOUT) upl->flags |= UPL_PAGEOUT; @@ -5648,7 +5162,11 @@ vm_object_upl_request( if ((cntrl_flags & UPL_WILL_MODIFY) && must_throttle_writes() == TRUE) { boolean_t isSSD = FALSE; +#if CONFIG_EMBEDDED + isSSD = TRUE; +#else vnode_pager_get_isSSD(object->pager, &isSSD); +#endif vm_object_unlock(object); OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages); @@ -5778,32 +5296,6 @@ check_busy: continue; } - /* - * ENCRYPTED SWAP: - * The caller is gathering this page and might - * access its contents later on. Decrypt the - * page before adding it to the UPL, so that - * the caller never sees encrypted data. - */ - if (! (cntrl_flags & UPL_ENCRYPT) && dst_page->encrypted) { - int was_busy; - - /* - * save the current state of busy - * mark page as busy while decrypt - * is in progress since it will drop - * the object lock... - */ - was_busy = dst_page->busy; - dst_page->busy = TRUE; - - vm_page_decrypt(dst_page, 0); - vm_page_decrypt_for_upl_counter++; - /* - * restore to original busy state - */ - dst_page->busy = was_busy; - } if (dst_page->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) { vm_page_lockspin_queues(); @@ -5882,24 +5374,6 @@ check_busy: if (!dirty) dst_page->precious = TRUE; - if ( (cntrl_flags & UPL_ENCRYPT) ) { - /* - * ENCRYPTED SWAP: - * We want to deny access to the target page - * because its contents are about to be - * encrypted and the user would be very - * confused to see encrypted data instead - * of their data. - * We also set "encrypted_cleaning" to allow - * vm_pageout_scan() to demote that page - * from "adjacent/clean-in-place" to - * "target/clean-and-free" if it bumps into - * this page during its scanning while we're - * still processing this cluster. - */ - dst_page->busy = TRUE; - dst_page->encrypted_cleaning = TRUE; - } if ( !(cntrl_flags & UPL_CLEAN_IN_PLACE) ) { if ( !VM_PAGE_WIRED(dst_page)) dst_page->free_when_done = TRUE; @@ -6072,22 +5546,6 @@ check_busy: } phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); - /* - * ENCRYPTED SWAP: - */ - if (cntrl_flags & UPL_ENCRYPT) { - /* - * The page is going to be encrypted when we - * get it from the pager, so mark it so. - */ - dst_page->encrypted = TRUE; - } else { - /* - * Otherwise, the page will not contain - * encrypted data. - */ - dst_page->encrypted = FALSE; - } dst_page->overwriting = TRUE; if (dst_page->pmapped) { @@ -6234,7 +5692,7 @@ try_next_page: VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count); if (dw_count >= dw_limit) { - vm_page_do_delayed_work(object, UPL_MEMORY_TAG(cntrl_flags), &dw_array[0], dw_count); + vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count); dwp = &dw_array[0]; dw_count = 0; @@ -6245,7 +5703,7 @@ try_next_page: xfer_size -= PAGE_SIZE; } if (dw_count) - vm_page_do_delayed_work(object, UPL_MEMORY_TAG(cntrl_flags), &dw_array[0], dw_count); + vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count); if (alias_page != NULL) { VM_PAGE_FREE(alias_page); @@ -6285,7 +5743,8 @@ vm_object_super_upl_request( upl_t *upl, upl_page_info_t *user_page_list, unsigned int *page_list_count, - upl_control_flags_t cntrl_flags) + upl_control_flags_t cntrl_flags, + vm_tag_t tag) { if (object->paging_offset > offset || ((cntrl_flags & UPL_VECTOR)==UPL_VECTOR)) return KERN_FAILURE; @@ -6325,9 +5784,14 @@ vm_object_super_upl_request( offset = base_offset; size = super_size; } - return vm_object_upl_request(object, offset, size, upl, user_page_list, page_list_count, cntrl_flags); + return vm_object_upl_request(object, offset, size, upl, user_page_list, page_list_count, cntrl_flags, tag); } +#if CONFIG_EMBEDDED +int cs_executable_create_upl = 0; +extern int proc_selfpid(void); +extern char *proc_name_address(void *p); +#endif /* CONFIG_EMBEDDED */ kern_return_t vm_map_create_upl( @@ -6337,7 +5801,8 @@ vm_map_create_upl( upl_t *upl, upl_page_info_array_t page_list, unsigned int *count, - upl_control_flags_t *flags) + upl_control_flags_t *flags, + vm_tag_t tag) { vm_map_entry_t entry; upl_control_flags_t caller_flags; @@ -6422,6 +5887,67 @@ REDISCOVER_ENTRY: return KERN_PROTECTION_FAILURE; } +#if CONFIG_EMBEDDED + if (map->pmap != kernel_pmap && + (caller_flags & UPL_COPYOUT_FROM) && + (entry->protection & VM_PROT_EXECUTE) && + !(entry->protection & VM_PROT_WRITE)) { + vm_offset_t kaddr; + vm_size_t ksize; + + /* + * We're about to create a read-only UPL backed by + * memory from an executable mapping. + * Wiring the pages would result in the pages being copied + * (due to the "MAP_PRIVATE" mapping) and no longer + * code-signed, so no longer eligible for execution. + * Instead, let's copy the data into a kernel buffer and + * create the UPL from this kernel buffer. + * The kernel buffer is then freed, leaving the UPL holding + * the last reference on the VM object, so the memory will + * be released when the UPL is committed. + */ + + vm_map_unlock_read(map); + /* allocate kernel buffer */ + ksize = round_page(*upl_size); + kaddr = 0; + ret = kmem_alloc_pageable(kernel_map, + &kaddr, + ksize, + tag); + if (ret == KERN_SUCCESS) { + /* copyin the user data */ + assert(page_aligned(offset)); + ret = copyinmap(map, offset, (void *)kaddr, *upl_size); + } + if (ret == KERN_SUCCESS) { + if (ksize > *upl_size) { + /* zero out the extra space in kernel buffer */ + memset((void *)(kaddr + *upl_size), + 0, + ksize - *upl_size); + } + /* create the UPL from the kernel buffer */ + ret = vm_map_create_upl(kernel_map, kaddr, upl_size, + upl, page_list, count, flags, tag); + } + if (kaddr != 0) { + /* free the kernel buffer */ + kmem_free(kernel_map, kaddr, ksize); + kaddr = 0; + ksize = 0; + } +#if DEVELOPMENT || DEBUG + DTRACE_VM4(create_upl_from_executable, + vm_map_t, map, + vm_map_address_t, offset, + upl_size_t, *upl_size, + kern_return_t, ret); +#endif /* DEVELOPMENT || DEBUG */ + return ret; + } +#endif /* CONFIG_EMBEDDED */ local_object = VME_OBJECT(entry); assert(local_object != VM_OBJECT_NULL); @@ -6571,7 +6097,7 @@ REDISCOVER_ENTRY: ret = vm_map_create_upl(submap, local_offset + (offset - local_start), - upl_size, upl, page_list, count, flags); + upl_size, upl, page_list, count, flags, tag); vm_map_deallocate(submap); return ret; @@ -6636,6 +6162,31 @@ REDISCOVER_ENTRY: local_offset = VME_OFFSET(entry); local_start = entry->vme_start; +#if CONFIG_EMBEDDED + /* + * Wiring will copy the pages to the shadow object. + * The shadow object will not be code-signed so + * attempting to execute code from these copied pages + * would trigger a code-signing violation. + */ + if (entry->protection & VM_PROT_EXECUTE) { +#if MACH_ASSERT + printf("pid %d[%s] create_upl out of executable range from " + "0x%llx to 0x%llx: side effects may include " + "code-signing violations later on\n", + proc_selfpid(), + (current_task()->bsd_info + ? proc_name_address(current_task()->bsd_info) + : "?"), + (uint64_t) entry->vme_start, + (uint64_t) entry->vme_end); +#endif /* MACH_ASSERT */ + DTRACE_VM2(cs_executable_create_upl, + uint64_t, (uint64_t)entry->vme_start, + uint64_t, (uint64_t)entry->vme_end); + cs_executable_create_upl++; + } +#endif /* CONFIG_EMBEDDED */ vm_object_lock(local_object); @@ -6685,7 +6236,8 @@ REDISCOVER_ENTRY: upl, page_list, count, - caller_flags); + caller_flags, + tag); vm_object_deallocate(local_object); return ret; @@ -6741,7 +6293,9 @@ vm_map_enter_upl( } } - kr = kmem_suballoc(map, &vector_upl_dst_addr, vector_upl->size, FALSE, VM_FLAGS_ANYWHERE, &vector_upl_submap); + kr = kmem_suballoc(map, &vector_upl_dst_addr, vector_upl->size, FALSE, + VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE, + &vector_upl_submap); if( kr != KERN_SUCCESS ) panic("Vector UPL submap allocation failed\n"); map = vector_upl_submap; @@ -6844,22 +6398,6 @@ process_upl_to_enter: vm_page_wire(alias_page, VM_KERN_MEMORY_NONE, TRUE); vm_page_unlock_queues(); - /* - * ENCRYPTED SWAP: - * The virtual page ("m") has to be wired in some way - * here or its backing physical page could - * be recycled at any time. - * Assuming this is enforced by the caller, we can't - * get an encrypted page here. Since the encryption - * key depends on the VM page's "pager" object and - * the "paging_offset", we couldn't handle 2 pageable - * VM pages (with different pagers and paging_offsets) - * sharing the same physical page: we could end up - * encrypting with one key (via one VM page) and - * decrypting with another key (via the alias VM page). - */ - ASSERT_PAGE_DECRYPTED(m); - vm_page_insert_wired(alias_page, upl->map_object, new_offset, VM_KERN_MEMORY_NONE); assert(!alias_page->wanted); @@ -6887,7 +6425,7 @@ process_upl_to_enter: * NEED A UPL_MAP ALIAS */ kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_OSFMK), + VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK, upl->map_object, offset, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); @@ -6899,7 +6437,7 @@ process_upl_to_enter: } else { kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0, - VM_FLAGS_FIXED | VM_MAKE_TAG(VM_KERN_MEMORY_OSFMK), + VM_FLAGS_FIXED, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK, upl->map_object, offset, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if(kr) @@ -6919,7 +6457,12 @@ process_upl_to_enter: /* m->wpmapped = TRUE; */ assert(map->pmap == kernel_pmap); - PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE); + PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE, kr); + + assert(kr == KERN_SUCCESS); +#if KASAN + kasan_notify_address(addr, PAGE_SIZE_64); +#endif } offset += PAGE_SIZE_64; } @@ -7169,6 +6712,8 @@ process_upl_to_commit: else vm_object_lock(shadow_object); + VM_OBJECT_WIRED_PAGE_UPDATE_START(shadow_object); + if (upl->flags & UPL_ACCESS_BLOCKED) { assert(shadow_object->blocked_access); shadow_object->blocked_access = FALSE; @@ -7470,11 +7015,6 @@ process_upl_to_commit: } m->overwriting = FALSE; } - if (m->encrypted_cleaning == TRUE) { - m->encrypted_cleaning = FALSE; - - dwp->dw_mask |= DW_clear_busy | DW_PAGE_WAKEUP; - } m->cleaning = FALSE; if (m->free_when_done) { @@ -7693,11 +7233,7 @@ commit_next_page: } vm_page_unlock_queues(); - shadow_object->wired_page_count -= unwired_count; - - if (!shadow_object->wired_page_count) { - VM_OBJECT_UNWIRED(shadow_object); - } + VM_OBJECT_WIRED_PAGE_COUNT(shadow_object, -unwired_count); } } occupied = 1; @@ -7754,6 +7290,7 @@ commit_next_page: */ } } + VM_OBJECT_WIRED_PAGE_UPDATE_END(shadow_object, shadow_object->wire_tag); vm_object_unlock(shadow_object); if (object != shadow_object) vm_object_unlock(object); @@ -7986,18 +7523,7 @@ process_upl_to_abort: */ must_free = TRUE; } - - /* - * ENCRYPTED SWAP: - * If the page was already encrypted, - * we don't really need to decrypt it - * now. It will get decrypted later, - * on demand, as soon as someone needs - * to access its contents. - */ - m->cleaning = FALSE; - m->encrypted_cleaning = FALSE; if (m->overwriting && !m->busy) { /* @@ -8053,11 +7579,6 @@ process_upl_to_abort: } m->overwriting = FALSE; } - if (m->encrypted_cleaning == TRUE) { - m->encrypted_cleaning = FALSE; - - dwp->dw_mask |= DW_clear_busy; - } m->free_when_done = FALSE; m->cleaning = FALSE; @@ -8220,7 +7741,8 @@ upl_commit( void iopl_valid_data( - upl_t upl) + upl_t upl, + vm_tag_t tag) { vm_object_t object; vm_offset_t offset; @@ -8248,6 +7770,7 @@ iopl_valid_data( size = upl->size; vm_object_lock(object); + VM_OBJECT_WIRED_PAGE_UPDATE_START(object); if (object->vo_size == size && object->resident_page_count == (size / PAGE_SIZE)) nxt_page = (vm_page_t)vm_page_queue_first(&object->memq); @@ -8295,10 +7818,7 @@ iopl_valid_data( } if (wired_count) { - if (!object->wired_page_count) { - VM_OBJECT_WIRED(object); - } - object->wired_page_count += wired_count; + VM_OBJECT_WIRED_PAGE_COUNT(object, wired_count); assert(object->resident_page_count >= object->wired_page_count); /* no need to adjust purgeable accounting for this object: */ @@ -8309,37 +7829,10 @@ iopl_valid_data( vm_page_wire_count += wired_count; vm_page_unlock_queues(); } + VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag); vm_object_unlock(object); } -vm_tag_t -iopl_set_tag( - upl_t upl, - vm_tag_t tag) -{ - vm_object_t object; - vm_tag_t prior_tag; - - if (upl == NULL) - panic("%s: NULL upl", __FUNCTION__); - if (vector_upl_is_valid(upl)) - panic("%s: vector upl", __FUNCTION__); - if (kernel_object == upl->map_object) - return (tag); - if ((upl->flags & (UPL_DEVICE_MEMORY|UPL_SHADOWED|UPL_ACCESS_BLOCKED|UPL_IO_WIRE|UPL_INTERNAL)) != UPL_IO_WIRE) - return (tag); - - object = upl->map_object; - vm_object_lock(object); - - prior_tag = object->wire_tag; - object->wire_tag = tag; - if (VM_KERN_MEMORY_NONE == prior_tag) prior_tag = tag; - vm_object_unlock(object); - - return (prior_tag); -} - void vm_object_set_pmap_cache_attr( @@ -8358,17 +7851,16 @@ vm_object_set_pmap_cache_attr( } -boolean_t vm_object_iopl_wire_full(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t); -kern_return_t vm_object_iopl_wire_empty(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_object_offset_t *, int); +boolean_t vm_object_iopl_wire_full(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t); +kern_return_t vm_object_iopl_wire_empty(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t, vm_object_offset_t *, int); boolean_t vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list, - wpl_array_t lite_list, upl_control_flags_t cntrl_flags) + wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag) { vm_page_t dst_page; - vm_tag_t tag; unsigned int entry; int page_count; int delayed_unlock = 0; @@ -8382,7 +7874,6 @@ vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t us assert(object->copy == NULL); assert(object->shadow == NULL); - tag = UPL_MEMORY_TAG(cntrl_flags); page_count = object->resident_page_count; dst_page = (vm_page_t)vm_page_queue_first(&object->memq); @@ -8396,7 +7887,6 @@ vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t us dst_page->error || dst_page->cleaning || dst_page->restart || - dst_page->encrypted || dst_page->laundry) { retval = FALSE; goto done; @@ -8454,10 +7944,9 @@ done: kern_return_t vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list, - wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_object_offset_t *dst_offset, int page_count) + wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag, vm_object_offset_t *dst_offset, int page_count) { vm_page_t dst_page; - vm_tag_t tag; boolean_t no_zero_fill = FALSE; int interruptible; int pages_wired = 0; @@ -8483,8 +7972,6 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) no_zero_fill = TRUE; - tag = UPL_MEMORY_TAG(cntrl_flags); - grab_options = 0; #if CONFIG_SECLUDED_MEMORY if (object->can_grab_secluded) { @@ -8606,7 +8093,8 @@ vm_object_iopl_request( upl_t *upl_ptr, upl_page_info_array_t user_page_list, unsigned int *page_list_count, - upl_control_flags_t cntrl_flags) + upl_control_flags_t cntrl_flags, + vm_tag_t tag) { vm_page_t dst_page; vm_object_offset_t dst_offset; @@ -8657,16 +8145,6 @@ vm_object_iopl_request( return KERN_INVALID_ADDRESS; } } - - if (cntrl_flags & UPL_ENCRYPT) { - /* - * ENCRYPTED SWAP: - * The paging path doesn't use this interface, - * so we don't support the UPL_ENCRYPT flag - * here. We won't encrypt the pages. - */ - assert(! (cntrl_flags & UPL_ENCRYPT)); - } if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) no_zero_fill = TRUE; @@ -8880,7 +8358,7 @@ vm_object_iopl_request( if (fast_path_full_req) { - if (vm_object_iopl_wire_full(object, upl, user_page_list, lite_list, cntrl_flags) == TRUE) + if (vm_object_iopl_wire_full(object, upl, user_page_list, lite_list, cntrl_flags, tag) == TRUE) goto finish; /* * we couldn't complete the processing of this request on the fast path @@ -8893,7 +8371,7 @@ vm_object_iopl_request( ret = KERN_MEMORY_ERROR; goto return_err; } - ret = vm_object_iopl_wire_empty(object, upl, user_page_list, lite_list, cntrl_flags, &dst_offset, size_in_pages); + ret = vm_object_iopl_wire_empty(object, upl, user_page_list, lite_list, cntrl_flags, tag, &dst_offset, size_in_pages); if (ret) { free_wired_pages = TRUE; @@ -8936,14 +8414,8 @@ vm_object_iopl_request( } dst_page = vm_page_lookup(object, dst_offset); - /* - * ENCRYPTED SWAP: - * If the page is encrypted, we need to decrypt it, - * so force a soft page fault. - */ if (dst_page == VM_PAGE_NULL || dst_page->busy || - dst_page->encrypted || dst_page->error || dst_page->restart || dst_page->absent || @@ -9237,7 +8709,7 @@ skip_page: VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count); if (dw_count >= dw_limit) { - vm_page_do_delayed_work(object, UPL_MEMORY_TAG(cntrl_flags), &dw_array[0], dw_count); + vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count); dwp = &dw_array[0]; dw_count = 0; @@ -9247,7 +8719,7 @@ skip_page: assert(entry == size_in_pages); if (dw_count) - vm_page_do_delayed_work(object, UPL_MEMORY_TAG(cntrl_flags), &dw_array[0], dw_count); + vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count); finish: if (user_page_list && set_cache_attr_needed == TRUE) vm_object_set_pmap_cache_attr(object, user_page_list, size_in_pages, TRUE); @@ -9455,47 +8927,13 @@ upl_range_needed( /* - * ENCRYPTED SWAP: - * - * Rationale: the user might have some encrypted data on disk (via - * FileVault or any other mechanism). That data is then decrypted in - * memory, which is safe as long as the machine is secure. But that - * decrypted data in memory could be paged out to disk by the default - * pager. The data would then be stored on disk in clear (not encrypted) - * and it could be accessed by anyone who gets physical access to the - * disk (if the laptop or the disk gets stolen for example). This weakens - * the security offered by FileVault. - * - * Solution: the default pager will optionally request that all the - * pages it gathers for pageout be encrypted, via the UPL interfaces, - * before it sends this UPL to disk via the vnode_pageout() path. - * - * Notes: - * - * To avoid disrupting the VM LRU algorithms, we want to keep the - * clean-in-place mechanisms, which allow us to send some extra pages to - * swap (clustering) without actually removing them from the user's - * address space. We don't want the user to unknowingly access encrypted - * data, so we have to actually remove the encrypted pages from the page - * table. When the user accesses the data, the hardware will fail to - * locate the virtual page in its page table and will trigger a page - * fault. We can then decrypt the page and enter it in the page table - * again. Whenever we allow the user to access the contents of a page, - * we have to make sure it's not encrypted. - * - * - */ -/* - * ENCRYPTED SWAP: * Reserve of virtual addresses in the kernel address space. * We need to map the physical pages in the kernel, so that we - * can call the encryption/decryption routines with a kernel + * can call the code-signing or slide routines with a kernel * virtual address. We keep this pool of pre-allocated kernel * virtual addresses so that we don't have to scan the kernel's - * virtaul address space each time we need to encrypt or decrypt + * virtaul address space each time we need to work with * a physical page. - * It would be nice to be able to encrypt and decrypt in physical - * mode but that might not always be more efficient... */ decl_simple_lock_data(,vm_paging_lock) #define VM_PAGING_NUM_PAGES 64 @@ -9529,6 +8967,8 @@ vm_paging_map_init(void) VM_PAGING_NUM_PAGES * PAGE_SIZE, 0, 0, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, &map_entry); if (kr != KERN_SUCCESS) { panic("vm_paging_map_init: kernel_map full\n"); @@ -9546,7 +8986,6 @@ vm_paging_map_init(void) } /* - * ENCRYPTED SWAP: * vm_paging_map_object: * Maps part of a VM object's pages in the kernel * virtual address space, using the pre-allocated @@ -9585,6 +9024,11 @@ vm_paging_map_object( PAGE_SHIFT); *need_unmap = FALSE; return KERN_SUCCESS; +#elif __arm__ || __arm64__ + *address = (vm_map_offset_t) + phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << PAGE_SHIFT); + *need_unmap = FALSE; + return KERN_SUCCESS; #else #warn "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..." #endif @@ -9664,12 +9108,18 @@ vm_paging_map_object( protection, VM_PROT_NONE, 0, - TRUE); + TRUE, + kr); + assert(kr == KERN_SUCCESS); vm_paging_objects_mapped++; vm_paging_pages_mapped++; *address = page_map_offset; *need_unmap = TRUE; +#if KASAN + kasan_notify_address(page_map_offset, PAGE_SIZE); +#endif + /* all done and mapped, ready to use ! */ return KERN_SUCCESS; } @@ -9707,6 +9157,8 @@ vm_paging_map_object( map_size, 0, VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, object, object_offset, FALSE, @@ -9761,7 +9213,12 @@ vm_paging_map_object( protection, VM_PROT_NONE, 0, - TRUE); + TRUE, + kr); + assert(kr == KERN_SUCCESS); +#if KASAN + kasan_notify_address(*address + page_map_offset, PAGE_SIZE); +#endif } vm_paging_objects_mapped_slow++; @@ -9773,7 +9230,6 @@ vm_paging_map_object( } /* - * ENCRYPTED SWAP: * vm_paging_unmap_object: * Unmaps part of a VM object's pages from the kernel * virtual address space. @@ -9829,655 +9285,99 @@ vm_paging_unmap_object( } } -#if ENCRYPTED_SWAP -/* - * Encryption data. - * "iv" is the "initial vector". Ideally, we want to - * have a different one for each page we encrypt, so that - * crackers can't find encryption patterns too easily. - */ -#define SWAP_CRYPT_AES_KEY_SIZE 128 /* XXX 192 and 256 don't work ! */ -boolean_t swap_crypt_ctx_initialized = FALSE; -uint32_t swap_crypt_key[8]; /* big enough for a 256 key */ -aes_ctx swap_crypt_ctx; -const unsigned char swap_crypt_null_iv[AES_BLOCK_SIZE] = {0xa, }; - -#if DEBUG -boolean_t swap_crypt_ctx_tested = FALSE; -unsigned char swap_crypt_test_page_ref[4096] __attribute__((aligned(4096))); -unsigned char swap_crypt_test_page_encrypt[4096] __attribute__((aligned(4096))); -unsigned char swap_crypt_test_page_decrypt[4096] __attribute__((aligned(4096))); -#endif /* DEBUG */ /* - * Initialize the encryption context: key and key size. + * page->object must be locked */ -void swap_crypt_ctx_initialize(void); /* forward */ void -swap_crypt_ctx_initialize(void) +vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked) { - unsigned int i; + if (!queues_locked) { + vm_page_lockspin_queues(); + } + page->free_when_done = FALSE; /* - * No need for locking to protect swap_crypt_ctx_initialized - * because the first use of encryption will come from the - * pageout thread (we won't pagein before there's been a pageout) - * and there's only one pageout thread. - */ - if (swap_crypt_ctx_initialized == FALSE) { - for (i = 0; - i < (sizeof (swap_crypt_key) / - sizeof (swap_crypt_key[0])); - i++) { - swap_crypt_key[i] = random(); - } - aes_encrypt_key((const unsigned char *) swap_crypt_key, - SWAP_CRYPT_AES_KEY_SIZE, - &swap_crypt_ctx.encrypt); - aes_decrypt_key((const unsigned char *) swap_crypt_key, - SWAP_CRYPT_AES_KEY_SIZE, - &swap_crypt_ctx.decrypt); - swap_crypt_ctx_initialized = TRUE; - } - -#if DEBUG - /* - * Validate the encryption algorithms. + * need to drop the laundry count... + * we may also need to remove it + * from the I/O paging queue... + * vm_pageout_throttle_up handles both cases + * + * the laundry and pageout_queue flags are cleared... */ - if (swap_crypt_ctx_tested == FALSE) { - /* initialize */ - for (i = 0; i < 4096; i++) { - swap_crypt_test_page_ref[i] = (char) i; - } - /* encrypt */ - aes_encrypt_cbc(swap_crypt_test_page_ref, - swap_crypt_null_iv, - PAGE_SIZE / AES_BLOCK_SIZE, - swap_crypt_test_page_encrypt, - &swap_crypt_ctx.encrypt); - /* decrypt */ - aes_decrypt_cbc(swap_crypt_test_page_encrypt, - swap_crypt_null_iv, - PAGE_SIZE / AES_BLOCK_SIZE, - swap_crypt_test_page_decrypt, - &swap_crypt_ctx.decrypt); - /* compare result with original */ - for (i = 0; i < 4096; i ++) { - if (swap_crypt_test_page_decrypt[i] != - swap_crypt_test_page_ref[i]) { - panic("encryption test failed"); - } - } - - /* encrypt again */ - aes_encrypt_cbc(swap_crypt_test_page_decrypt, - swap_crypt_null_iv, - PAGE_SIZE / AES_BLOCK_SIZE, - swap_crypt_test_page_decrypt, - &swap_crypt_ctx.encrypt); - /* decrypt in place */ - aes_decrypt_cbc(swap_crypt_test_page_decrypt, - swap_crypt_null_iv, - PAGE_SIZE / AES_BLOCK_SIZE, - swap_crypt_test_page_decrypt, - &swap_crypt_ctx.decrypt); - for (i = 0; i < 4096; i ++) { - if (swap_crypt_test_page_decrypt[i] != - swap_crypt_test_page_ref[i]) { - panic("in place encryption test failed"); - } - } - - swap_crypt_ctx_tested = TRUE; - } -#endif /* DEBUG */ -} + vm_pageout_throttle_up(page); -/* - * ENCRYPTED SWAP: - * vm_page_encrypt: - * Encrypt the given page, for secure paging. - * The page might already be mapped at kernel virtual - * address "kernel_mapping_offset". Otherwise, we need - * to map it. - * - * Context: - * The page's object is locked, but this lock will be released - * and re-acquired. - * The page is busy and not accessible by users (not entered in any pmap). - */ -void -vm_page_encrypt( - vm_page_t page, - vm_map_offset_t kernel_mapping_offset) -{ - kern_return_t kr; - vm_map_size_t kernel_mapping_size; - boolean_t kernel_mapping_needs_unmap; - vm_offset_t kernel_vaddr; - vm_object_t page_object; - union { - unsigned char aes_iv[AES_BLOCK_SIZE]; - struct { - memory_object_t pager_object; - vm_object_offset_t paging_offset; - } vm; - } encrypt_iv; + vm_page_steal_pageout_page++; - if (! vm_pages_encrypted) { - vm_pages_encrypted = TRUE; + if (!queues_locked) { + vm_page_unlock_queues(); } +} + +upl_t +vector_upl_create(vm_offset_t upl_offset) +{ + int vector_upl_size = sizeof(struct _vector_upl); + int i=0; + upl_t upl; + vector_upl_t vector_upl = (vector_upl_t)kalloc(vector_upl_size); - assert(page->busy); + upl = upl_create(0,UPL_VECTOR,0); + upl->vector_upl = vector_upl; + upl->offset = upl_offset; + vector_upl->size = 0; + vector_upl->offset = upl_offset; + vector_upl->invalid_upls=0; + vector_upl->num_upls=0; + vector_upl->pagelist = NULL; - if (page->encrypted) { - /* - * Already encrypted: no need to do it again. - */ - vm_page_encrypt_already_encrypted_counter++; - return; + for(i=0; i < MAX_VECTOR_UPL_ELEMENTS ; i++) { + vector_upl->upl_iostates[i].size = 0; + vector_upl->upl_iostates[i].offset = 0; + } - assert(page->dirty || page->precious); - - ASSERT_PAGE_DECRYPTED(page); - - page_object = VM_PAGE_OBJECT(page); - - /* - * Take a paging-in-progress reference to keep the object - * alive even if we have to unlock it (in vm_paging_map_object() - * for example)... - */ - vm_object_paging_begin(page_object); + return upl; +} - if (kernel_mapping_offset == 0) { - /* - * The page hasn't already been mapped in kernel space - * by the caller. Map it now, so that we can access - * its contents and encrypt them. - */ - kernel_mapping_size = PAGE_SIZE; - kernel_mapping_needs_unmap = FALSE; - kr = vm_paging_map_object(page, - page_object, - page->offset, - VM_PROT_READ | VM_PROT_WRITE, - FALSE, - &kernel_mapping_size, - &kernel_mapping_offset, - &kernel_mapping_needs_unmap); - if (kr != KERN_SUCCESS) { - panic("vm_page_encrypt: " - "could not map page in kernel: 0x%x\n", - kr); +void +vector_upl_deallocate(upl_t upl) +{ + if(upl) { + vector_upl_t vector_upl = upl->vector_upl; + if(vector_upl) { + if(vector_upl->invalid_upls != vector_upl->num_upls) + panic("Deallocating non-empty Vectored UPL\n"); + kfree(vector_upl->pagelist,(sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE))); + vector_upl->invalid_upls=0; + vector_upl->num_upls = 0; + vector_upl->pagelist = NULL; + vector_upl->size = 0; + vector_upl->offset = 0; + kfree(vector_upl, sizeof(struct _vector_upl)); + vector_upl = (vector_upl_t)0xfeedfeed; } - } else { - kernel_mapping_size = 0; - kernel_mapping_needs_unmap = FALSE; - } - kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset); - - if (swap_crypt_ctx_initialized == FALSE) { - swap_crypt_ctx_initialize(); + else + panic("vector_upl_deallocate was passed a non-vectored upl\n"); } - assert(swap_crypt_ctx_initialized); - - /* - * Prepare an "initial vector" for the encryption. - * We use the "pager" and the "paging_offset" for that - * page to obfuscate the encrypted data a bit more and - * prevent crackers from finding patterns that they could - * use to break the key. - */ - bzero(&encrypt_iv.aes_iv[0], sizeof (encrypt_iv.aes_iv)); - encrypt_iv.vm.pager_object = page_object->pager; - encrypt_iv.vm.paging_offset = - page_object->paging_offset + page->offset; - - /* encrypt the "initial vector" */ - aes_encrypt_cbc((const unsigned char *) &encrypt_iv.aes_iv[0], - swap_crypt_null_iv, - 1, - &encrypt_iv.aes_iv[0], - &swap_crypt_ctx.encrypt); - - /* - * Encrypt the page. - */ - aes_encrypt_cbc((const unsigned char *) kernel_vaddr, - &encrypt_iv.aes_iv[0], - PAGE_SIZE / AES_BLOCK_SIZE, - (unsigned char *) kernel_vaddr, - &swap_crypt_ctx.encrypt); - - vm_page_encrypt_counter++; + else + panic("vector_upl_deallocate was passed a NULL upl\n"); +} - /* - * Unmap the page from the kernel's address space, - * if we had to map it ourselves. Otherwise, let - * the caller undo the mapping if needed. - */ - if (kernel_mapping_needs_unmap) { - vm_paging_unmap_object(page_object, - kernel_mapping_offset, - kernel_mapping_offset + kernel_mapping_size); +boolean_t +vector_upl_is_valid(upl_t upl) +{ + if(upl && ((upl->flags & UPL_VECTOR)==UPL_VECTOR)) { + vector_upl_t vector_upl = upl->vector_upl; + if(vector_upl == NULL || vector_upl == (vector_upl_t)0xfeedfeed || vector_upl == (vector_upl_t)0xfeedbeef) + return FALSE; + else + return TRUE; } - - /* - * Clear the "reference" and "modified" bits. - * This should clean up any impact the encryption had - * on them. - * The page was kept busy and disconnected from all pmaps, - * so it can't have been referenced or modified from user - * space. - * The software bits will be reset later after the I/O - * has completed (in upl_commit_range()). - */ - pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(page), VM_MEM_REFERENCED | VM_MEM_MODIFIED); - - page->encrypted = TRUE; - - vm_object_paging_end(page_object); + return FALSE; } -/* - * ENCRYPTED SWAP: - * vm_page_decrypt: - * Decrypt the given page. - * The page might already be mapped at kernel virtual - * address "kernel_mapping_offset". Otherwise, we need - * to map it. - * - * Context: - * The page's VM object is locked but will be unlocked and relocked. - * The page is busy and not accessible by users (not entered in any pmap). - */ -void -vm_page_decrypt( - vm_page_t page, - vm_map_offset_t kernel_mapping_offset) -{ - kern_return_t kr; - vm_map_size_t kernel_mapping_size; - vm_offset_t kernel_vaddr; - boolean_t kernel_mapping_needs_unmap; - vm_object_t page_object; - union { - unsigned char aes_iv[AES_BLOCK_SIZE]; - struct { - memory_object_t pager_object; - vm_object_offset_t paging_offset; - } vm; - } decrypt_iv; - boolean_t was_dirty; - - assert(page->busy); - assert(page->encrypted); - - page_object = VM_PAGE_OBJECT(page); - was_dirty = page->dirty; - - /* - * Take a paging-in-progress reference to keep the object - * alive even if we have to unlock it (in vm_paging_map_object() - * for example)... - */ - vm_object_paging_begin(page_object); - - if (kernel_mapping_offset == 0) { - /* - * The page hasn't already been mapped in kernel space - * by the caller. Map it now, so that we can access - * its contents and decrypt them. - */ - kernel_mapping_size = PAGE_SIZE; - kernel_mapping_needs_unmap = FALSE; - kr = vm_paging_map_object(page, - page_object, - page->offset, - VM_PROT_READ | VM_PROT_WRITE, - FALSE, - &kernel_mapping_size, - &kernel_mapping_offset, - &kernel_mapping_needs_unmap); - if (kr != KERN_SUCCESS) { - panic("vm_page_decrypt: " - "could not map page in kernel: 0x%x\n", - kr); - } - } else { - kernel_mapping_size = 0; - kernel_mapping_needs_unmap = FALSE; - } - kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset); - - assert(swap_crypt_ctx_initialized); - - /* - * Prepare an "initial vector" for the decryption. - * It has to be the same as the "initial vector" we - * used to encrypt that page. - */ - bzero(&decrypt_iv.aes_iv[0], sizeof (decrypt_iv.aes_iv)); - decrypt_iv.vm.pager_object = page_object->pager; - decrypt_iv.vm.paging_offset = - page_object->paging_offset + page->offset; - - /* encrypt the "initial vector" */ - aes_encrypt_cbc((const unsigned char *) &decrypt_iv.aes_iv[0], - swap_crypt_null_iv, - 1, - &decrypt_iv.aes_iv[0], - &swap_crypt_ctx.encrypt); - - /* - * Decrypt the page. - */ - aes_decrypt_cbc((const unsigned char *) kernel_vaddr, - &decrypt_iv.aes_iv[0], - PAGE_SIZE / AES_BLOCK_SIZE, - (unsigned char *) kernel_vaddr, - &swap_crypt_ctx.decrypt); - vm_page_decrypt_counter++; - - /* - * Unmap the page from the kernel's address space, - * if we had to map it ourselves. Otherwise, let - * the caller undo the mapping if needed. - */ - if (kernel_mapping_needs_unmap) { - vm_paging_unmap_object(page_object, - kernel_vaddr, - kernel_vaddr + PAGE_SIZE); - } - - if (was_dirty) { - /* - * The pager did not specify that the page would be - * clean when it got paged in, so let's not clean it here - * either. - */ - } else { - /* - * After decryption, the page is actually still clean. - * It was encrypted as part of paging, which "cleans" - * the "dirty" pages. - * Noone could access it after it was encrypted - * and the decryption doesn't count. - */ - page->dirty = FALSE; - assert (page->cs_validated == FALSE); - pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(page), VM_MEM_MODIFIED | VM_MEM_REFERENCED); - } - page->encrypted = FALSE; - - /* - * We've just modified the page's contents via the data cache and part - * of the new contents might still be in the cache and not yet in RAM. - * Since the page is now available and might get gathered in a UPL to - * be part of a DMA transfer from a driver that expects the memory to - * be coherent at this point, we have to flush the data cache. - */ - pmap_sync_page_attributes_phys(VM_PAGE_GET_PHYS_PAGE(page)); - /* - * Since the page is not mapped yet, some code might assume that it - * doesn't need to invalidate the instruction cache when writing to - * that page. That code relies on "pmapped" being FALSE, so that the - * caches get synchronized when the page is first mapped. - */ - assert(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(page))); - page->pmapped = FALSE; - page->wpmapped = FALSE; - - vm_object_paging_end(page_object); -} - -#if DEVELOPMENT || DEBUG -unsigned long upl_encrypt_upls = 0; -unsigned long upl_encrypt_pages = 0; -#endif - -/* - * ENCRYPTED SWAP: - * - * upl_encrypt: - * Encrypts all the pages in the UPL, within the specified range. - * - */ -void -upl_encrypt( - upl_t upl, - upl_offset_t crypt_offset, - upl_size_t crypt_size) -{ - upl_size_t upl_size, subupl_size=crypt_size; - upl_offset_t offset_in_upl, subupl_offset=crypt_offset; - vm_object_t upl_object; - vm_object_offset_t upl_offset; - vm_page_t page; - vm_object_t shadow_object; - vm_object_offset_t shadow_offset; - vm_object_offset_t paging_offset; - vm_object_offset_t base_offset; - int isVectorUPL = 0; - upl_t vector_upl = NULL; - - if((isVectorUPL = vector_upl_is_valid(upl))) - vector_upl = upl; - -process_upl_to_encrypt: - if(isVectorUPL) { - crypt_size = subupl_size; - crypt_offset = subupl_offset; - upl = vector_upl_subupl_byoffset(vector_upl, &crypt_offset, &crypt_size); - if(upl == NULL) - panic("upl_encrypt: Accessing a sub-upl that doesn't exist\n"); - subupl_size -= crypt_size; - subupl_offset += crypt_size; - } - -#if DEVELOPMENT || DEBUG - upl_encrypt_upls++; - upl_encrypt_pages += crypt_size / PAGE_SIZE; -#endif - upl_object = upl->map_object; - upl_offset = upl->offset; - upl_size = upl->size; - - vm_object_lock(upl_object); - - /* - * Find the VM object that contains the actual pages. - */ - if (upl_object->pageout) { - shadow_object = upl_object->shadow; - /* - * The offset in the shadow object is actually also - * accounted for in upl->offset. It possibly shouldn't be - * this way, but for now don't account for it twice. - */ - shadow_offset = 0; - assert(upl_object->paging_offset == 0); /* XXX ? */ - vm_object_lock(shadow_object); - } else { - shadow_object = upl_object; - shadow_offset = 0; - } - - paging_offset = shadow_object->paging_offset; - vm_object_paging_begin(shadow_object); - - if (shadow_object != upl_object) - vm_object_unlock(upl_object); - - - base_offset = shadow_offset; - base_offset += upl_offset; - base_offset += crypt_offset; - base_offset -= paging_offset; - - assert(crypt_offset + crypt_size <= upl_size); - - for (offset_in_upl = 0; - offset_in_upl < crypt_size; - offset_in_upl += PAGE_SIZE) { - page = vm_page_lookup(shadow_object, - base_offset + offset_in_upl); - if (page == VM_PAGE_NULL) { - panic("upl_encrypt: " - "no page for (obj=%p,off=0x%llx+0x%x)!\n", - shadow_object, - base_offset, - offset_in_upl); - } - /* - * Disconnect the page from all pmaps, so that nobody can - * access it while it's encrypted. After that point, all - * accesses to this page will cause a page fault and block - * while the page is busy being encrypted. After the - * encryption completes, any access will cause a - * page fault and the page gets decrypted at that time. - */ - pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(page)); - vm_page_encrypt(page, 0); - - if (vm_object_lock_avoid(shadow_object)) { - /* - * Give vm_pageout_scan() a chance to convert more - * pages from "clean-in-place" to "clean-and-free", - * if it's interested in the same pages we selected - * in this cluster. - */ - vm_object_unlock(shadow_object); - mutex_pause(2); - vm_object_lock(shadow_object); - } - } - - vm_object_paging_end(shadow_object); - vm_object_unlock(shadow_object); - - if(isVectorUPL && subupl_size) - goto process_upl_to_encrypt; -} - -#else /* ENCRYPTED_SWAP */ -void -upl_encrypt( - __unused upl_t upl, - __unused upl_offset_t crypt_offset, - __unused upl_size_t crypt_size) -{ -} - -void -vm_page_encrypt( - __unused vm_page_t page, - __unused vm_map_offset_t kernel_mapping_offset) -{ -} - -void -vm_page_decrypt( - __unused vm_page_t page, - __unused vm_map_offset_t kernel_mapping_offset) -{ -} - -#endif /* ENCRYPTED_SWAP */ - -/* - * page->object must be locked - */ -void -vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked) -{ - if (!queues_locked) { - vm_page_lockspin_queues(); - } - - page->free_when_done = FALSE; - /* - * need to drop the laundry count... - * we may also need to remove it - * from the I/O paging queue... - * vm_pageout_throttle_up handles both cases - * - * the laundry and pageout_queue flags are cleared... - */ - vm_pageout_throttle_up(page); - - vm_page_steal_pageout_page++; - - if (!queues_locked) { - vm_page_unlock_queues(); - } -} - -upl_t -vector_upl_create(vm_offset_t upl_offset) -{ - int vector_upl_size = sizeof(struct _vector_upl); - int i=0; - upl_t upl; - vector_upl_t vector_upl = (vector_upl_t)kalloc(vector_upl_size); - - upl = upl_create(0,UPL_VECTOR,0); - upl->vector_upl = vector_upl; - upl->offset = upl_offset; - vector_upl->size = 0; - vector_upl->offset = upl_offset; - vector_upl->invalid_upls=0; - vector_upl->num_upls=0; - vector_upl->pagelist = NULL; - - for(i=0; i < MAX_VECTOR_UPL_ELEMENTS ; i++) { - vector_upl->upl_iostates[i].size = 0; - vector_upl->upl_iostates[i].offset = 0; - - } - return upl; -} - -void -vector_upl_deallocate(upl_t upl) -{ - if(upl) { - vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl) { - if(vector_upl->invalid_upls != vector_upl->num_upls) - panic("Deallocating non-empty Vectored UPL\n"); - kfree(vector_upl->pagelist,(sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE))); - vector_upl->invalid_upls=0; - vector_upl->num_upls = 0; - vector_upl->pagelist = NULL; - vector_upl->size = 0; - vector_upl->offset = 0; - kfree(vector_upl, sizeof(struct _vector_upl)); - vector_upl = (vector_upl_t)0xfeedfeed; - } - else - panic("vector_upl_deallocate was passed a non-vectored upl\n"); - } - else - panic("vector_upl_deallocate was passed a NULL upl\n"); -} - -boolean_t -vector_upl_is_valid(upl_t upl) -{ - if(upl && ((upl->flags & UPL_VECTOR)==UPL_VECTOR)) { - vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl == NULL || vector_upl == (vector_upl_t)0xfeedfeed || vector_upl == (vector_upl_t)0xfeedbeef) - return FALSE; - else - return TRUE; - } - return FALSE; -} - -boolean_t -vector_upl_set_subupl(upl_t upl,upl_t subupl, uint32_t io_size) +boolean_t +vector_upl_set_subupl(upl_t upl,upl_t subupl, uint32_t io_size) { if(vector_upl_is_valid(upl)) { vector_upl_t vector_upl = upl->vector_upl; @@ -11064,6 +9964,17 @@ vm_countdirtypages(void) } #endif /* MACH_BSD */ + +#if CONFIG_IOSCHED +int upl_get_cached_tier(upl_t upl) +{ + assert(upl); + if (upl->flags & UPL_TRACKED_BY_OBJECT) + return (upl->upl_priority); + return (-1); +} +#endif /* CONFIG_IOSCHED */ + ppnum_t upl_get_highest_page( upl_t upl) { @@ -11163,7 +10074,7 @@ VM_PRESSURE_WARNING_TO_NORMAL(void) { if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) { /* Available pages above our threshold */ - unsigned int target_threshold = memorystatus_available_pages_pressure + ((15 * memorystatus_available_pages_pressure) / 100); + unsigned int target_threshold = (unsigned int) (memorystatus_available_pages_pressure + ((15 * memorystatus_available_pages_pressure) / 100)); if (memorystatus_available_pages > target_threshold) { return TRUE; } @@ -11179,7 +10090,7 @@ VM_PRESSURE_CRITICAL_TO_WARNING(void) { if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) { /* Available pages above our threshold */ - unsigned int target_threshold = memorystatus_available_pages_critical + ((15 * memorystatus_available_pages_critical) / 100); + unsigned int target_threshold = (unsigned int)(memorystatus_available_pages_critical + ((15 * memorystatus_available_pages_critical) / 100)); if (memorystatus_available_pages > target_threshold) { return TRUE; } @@ -11190,3 +10101,428 @@ VM_PRESSURE_CRITICAL_TO_WARNING(void) { } #endif /* VM_PRESSURE_EVENTS */ + + +#define VM_TEST_COLLAPSE_COMPRESSOR 0 +#define VM_TEST_WIRE_AND_EXTRACT 0 +#define VM_TEST_PAGE_WIRE_OVERFLOW_PANIC 0 +#if __arm64__ +#define VM_TEST_KERNEL_OBJECT_FAULT 0 +#endif /* __arm64__ */ +#define VM_TEST_DEVICE_PAGER_TRANSPOSE (DEVELOPMENT || DEBUG) + +#if VM_TEST_COLLAPSE_COMPRESSOR +extern boolean_t vm_object_collapse_compressor_allowed; +#include +static void +vm_test_collapse_compressor(void) +{ + vm_object_size_t backing_size, top_size; + vm_object_t backing_object, top_object; + vm_map_offset_t backing_offset, top_offset; + unsigned char *backing_address, *top_address; + kern_return_t kr; + + printf("VM_TEST_COLLAPSE_COMPRESSOR:\n"); + + /* create backing object */ + backing_size = 15 * PAGE_SIZE; + backing_object = vm_object_allocate(backing_size); + assert(backing_object != VM_OBJECT_NULL); + printf("VM_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n", + backing_object); + /* map backing object */ + backing_offset = 0; + kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0, + VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, + backing_object, 0, FALSE, + VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT); + assert(kr == KERN_SUCCESS); + backing_address = (unsigned char *) backing_offset; + printf("VM_TEST_COLLAPSE_COMPRESSOR: " + "mapped backing object %p at 0x%llx\n", + backing_object, (uint64_t) backing_offset); + /* populate with pages to be compressed in backing object */ + backing_address[0x1*PAGE_SIZE] = 0xB1; + backing_address[0x4*PAGE_SIZE] = 0xB4; + backing_address[0x7*PAGE_SIZE] = 0xB7; + backing_address[0xa*PAGE_SIZE] = 0xBA; + backing_address[0xd*PAGE_SIZE] = 0xBD; + printf("VM_TEST_COLLAPSE_COMPRESSOR: " + "populated pages to be compressed in " + "backing_object %p\n", backing_object); + /* compress backing object */ + vm_object_pageout(backing_object); + printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n", + backing_object); + /* wait for all the pages to be gone */ + while (*(volatile int *)&backing_object->resident_page_count != 0) + IODelay(10); + printf("VM_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n", + backing_object); + /* populate with pages to be resident in backing object */ + backing_address[0x0*PAGE_SIZE] = 0xB0; + backing_address[0x3*PAGE_SIZE] = 0xB3; + backing_address[0x6*PAGE_SIZE] = 0xB6; + backing_address[0x9*PAGE_SIZE] = 0xB9; + backing_address[0xc*PAGE_SIZE] = 0xBC; + printf("VM_TEST_COLLAPSE_COMPRESSOR: " + "populated pages to be resident in " + "backing_object %p\n", backing_object); + /* leave the other pages absent */ + /* mess with the paging_offset of the backing_object */ + assert(backing_object->paging_offset == 0); + backing_object->paging_offset = 0x3000; + + /* create top object */ + top_size = 9 * PAGE_SIZE; + top_object = vm_object_allocate(top_size); + assert(top_object != VM_OBJECT_NULL); + printf("VM_TEST_COLLAPSE_COMPRESSOR: created top object %p\n", + top_object); + /* map top object */ + top_offset = 0; + kr = vm_map_enter(kernel_map, &top_offset, top_size, 0, + VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, + top_object, 0, FALSE, + VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT); + assert(kr == KERN_SUCCESS); + top_address = (unsigned char *) top_offset; + printf("VM_TEST_COLLAPSE_COMPRESSOR: " + "mapped top object %p at 0x%llx\n", + top_object, (uint64_t) top_offset); + /* populate with pages to be compressed in top object */ + top_address[0x3*PAGE_SIZE] = 0xA3; + top_address[0x4*PAGE_SIZE] = 0xA4; + top_address[0x5*PAGE_SIZE] = 0xA5; + printf("VM_TEST_COLLAPSE_COMPRESSOR: " + "populated pages to be compressed in " + "top_object %p\n", top_object); + /* compress top object */ + vm_object_pageout(top_object); + printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n", + top_object); + /* wait for all the pages to be gone */ + while (top_object->resident_page_count != 0) + IODelay(10); + printf("VM_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n", + top_object); + /* populate with pages to be resident in top object */ + top_address[0x0*PAGE_SIZE] = 0xA0; + top_address[0x1*PAGE_SIZE] = 0xA1; + top_address[0x2*PAGE_SIZE] = 0xA2; + printf("VM_TEST_COLLAPSE_COMPRESSOR: " + "populated pages to be resident in " + "top_object %p\n", top_object); + /* leave the other pages absent */ + + /* link the 2 objects */ + vm_object_reference(backing_object); + top_object->shadow = backing_object; + top_object->vo_shadow_offset = 0x3000; + printf("VM_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n", + top_object, backing_object); + + /* unmap backing object */ + vm_map_remove(kernel_map, + backing_offset, + backing_offset + backing_size, + 0); + printf("VM_TEST_COLLAPSE_COMPRESSOR: " + "unmapped backing_object %p [0x%llx:0x%llx]\n", + backing_object, + (uint64_t) backing_offset, + (uint64_t) (backing_offset + backing_size)); + + /* collapse */ + printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object); + vm_object_lock(top_object); + vm_object_collapse(top_object, 0, FALSE); + vm_object_unlock(top_object); + printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsed %p\n", top_object); + + /* did it work? */ + if (top_object->shadow != VM_OBJECT_NULL) { + printf("VM_TEST_COLLAPSE_COMPRESSOR: not collapsed\n"); + printf("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n"); + if (vm_object_collapse_compressor_allowed) { + panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n"); + } + } else { + /* check the contents of the mapping */ + unsigned char expect[9] = + { 0xA0, 0xA1, 0xA2, /* resident in top */ + 0xA3, 0xA4, 0xA5, /* compressed in top */ + 0xB9, /* resident in backing + shadow_offset */ + 0xBD, /* compressed in backing + shadow_offset + paging_offset */ + 0x00 }; /* absent in both */ + unsigned char actual[9]; + unsigned int i, errors; + + errors = 0; + for (i = 0; i < sizeof (actual); i++) { + actual[i] = (unsigned char) top_address[i*PAGE_SIZE]; + if (actual[i] != expect[i]) { + errors++; + } + } + printf("VM_TEST_COLLAPSE_COMPRESSOR: " + "actual [%x %x %x %x %x %x %x %x %x] " + "expect [%x %x %x %x %x %x %x %x %x] " + "%d errors\n", + actual[0], actual[1], actual[2], actual[3], + actual[4], actual[5], actual[6], actual[7], + actual[8], + expect[0], expect[1], expect[2], expect[3], + expect[4], expect[5], expect[6], expect[7], + expect[8], + errors); + if (errors) { + panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n"); + } else { + printf("VM_TEST_COLLAPSE_COMPRESSOR: PASS\n"); + } + } +} +#else /* VM_TEST_COLLAPSE_COMPRESSOR */ +#define vm_test_collapse_compressor() +#endif /* VM_TEST_COLLAPSE_COMPRESSOR */ + +#if VM_TEST_WIRE_AND_EXTRACT +extern ledger_template_t task_ledger_template; +#include +extern ppnum_t vm_map_get_phys_page(vm_map_t map, + vm_offset_t offset); +static void +vm_test_wire_and_extract(void) +{ + ledger_t ledger; + vm_map_t user_map, wire_map; + mach_vm_address_t user_addr, wire_addr; + mach_vm_size_t user_size, wire_size; + mach_vm_offset_t cur_offset; + vm_prot_t cur_prot, max_prot; + ppnum_t user_ppnum, wire_ppnum; + kern_return_t kr; + + ledger = ledger_instantiate(task_ledger_template, + LEDGER_CREATE_ACTIVE_ENTRIES); + user_map = vm_map_create(pmap_create(ledger, 0, PMAP_CREATE_64BIT), + 0x100000000ULL, + 0x200000000ULL, + TRUE); + wire_map = vm_map_create(NULL, + 0x100000000ULL, + 0x200000000ULL, + TRUE); + user_addr = 0; + user_size = 0x10000; + kr = mach_vm_allocate(user_map, + &user_addr, + user_size, + VM_FLAGS_ANYWHERE); + assert(kr == KERN_SUCCESS); + wire_addr = 0; + wire_size = user_size; + kr = mach_vm_remap(wire_map, + &wire_addr, + wire_size, + 0, + VM_FLAGS_ANYWHERE, + user_map, + user_addr, + FALSE, + &cur_prot, + &max_prot, + VM_INHERIT_NONE); + assert(kr == KERN_SUCCESS); + for (cur_offset = 0; + cur_offset < wire_size; + cur_offset += PAGE_SIZE) { + kr = vm_map_wire_and_extract(wire_map, + wire_addr + cur_offset, + VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK), + TRUE, + &wire_ppnum); + assert(kr == KERN_SUCCESS); + user_ppnum = vm_map_get_phys_page(user_map, + user_addr + cur_offset); + printf("VM_TEST_WIRE_AND_EXTRACT: kr=0x%x " + "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n", + kr, + user_map, user_addr + cur_offset, user_ppnum, + wire_map, wire_addr + cur_offset, wire_ppnum); + if (kr != KERN_SUCCESS || + wire_ppnum == 0 || + wire_ppnum != user_ppnum) { + panic("VM_TEST_WIRE_AND_EXTRACT: FAIL\n"); + } + } + cur_offset -= PAGE_SIZE; + kr = vm_map_wire_and_extract(wire_map, + wire_addr + cur_offset, + VM_PROT_DEFAULT, + TRUE, + &wire_ppnum); + assert(kr == KERN_SUCCESS); + printf("VM_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x " + "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n", + kr, + user_map, user_addr + cur_offset, user_ppnum, + wire_map, wire_addr + cur_offset, wire_ppnum); + if (kr != KERN_SUCCESS || + wire_ppnum == 0 || + wire_ppnum != user_ppnum) { + panic("VM_TEST_WIRE_AND_EXTRACT: FAIL\n"); + } + + printf("VM_TEST_WIRE_AND_EXTRACT: PASS\n"); +} +#else /* VM_TEST_WIRE_AND_EXTRACT */ +#define vm_test_wire_and_extract() +#endif /* VM_TEST_WIRE_AND_EXTRACT */ + +#if VM_TEST_PAGE_WIRE_OVERFLOW_PANIC +static void +vm_test_page_wire_overflow_panic(void) +{ + vm_object_t object; + vm_page_t page; + + printf("VM_TEST_PAGE_WIRE_OVERFLOW_PANIC: starting...\n"); + + object = vm_object_allocate(PAGE_SIZE); + vm_object_lock(object); + page = vm_page_alloc(object, 0x0); + vm_page_lock_queues(); + do { + vm_page_wire(page, 1, FALSE); + } while (page->wire_count != 0); + vm_page_unlock_queues(); + vm_object_unlock(object); + panic("FBDP(%p,%p): wire_count overflow not detected\n", + object, page); +} +#else /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */ +#define vm_test_page_wire_overflow_panic() +#endif /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */ + +#if __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT +extern int copyinframe(vm_address_t fp, char *frame, boolean_t is64bit); +static void +vm_test_kernel_object_fault(void) +{ + kern_return_t kr; + vm_offset_t stack; + uintptr_t frameb[2]; + int ret; + + kr = kernel_memory_allocate(kernel_map, &stack, + kernel_stack_size + (2*PAGE_SIZE), + 0, + (KMA_KSTACK | KMA_KOBJECT | + KMA_GUARD_FIRST | KMA_GUARD_LAST), + VM_KERN_MEMORY_STACK); + if (kr != KERN_SUCCESS) { + panic("VM_TEST_KERNEL_OBJECT_FAULT: kernel_memory_allocate kr 0x%x\n", kr); + } + ret = copyinframe((uintptr_t)stack, (char *)frameb, TRUE); + if (ret != 0) { + printf("VM_TEST_KERNEL_OBJECT_FAULT: PASS\n"); + } else { + printf("VM_TEST_KERNEL_OBJECT_FAULT: FAIL\n"); + } + vm_map_remove(kernel_map, + stack, + stack + kernel_stack_size + (2*PAGE_SIZE), + VM_MAP_REMOVE_KUNWIRE); + stack = 0; +} +#else /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */ +#define vm_test_kernel_object_fault() +#endif /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */ + +#if VM_TEST_DEVICE_PAGER_TRANSPOSE +static void +vm_test_device_pager_transpose(void) +{ + memory_object_t device_pager; + vm_object_t anon_object, device_object; + vm_size_t size; + vm_map_offset_t anon_mapping, device_mapping; + kern_return_t kr; + + size = 3 * PAGE_SIZE; + anon_object = vm_object_allocate(size); + assert(anon_object != VM_OBJECT_NULL); + device_pager = device_pager_setup(NULL, 0, size, 0); + assert(device_pager != NULL); + device_object = memory_object_to_vm_object(device_pager); + assert(device_object != VM_OBJECT_NULL); + anon_mapping = 0; + kr = vm_map_enter(kernel_map, &anon_mapping, size, 0, + VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE, + anon_object, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, + VM_INHERIT_DEFAULT); + assert(kr == KERN_SUCCESS); + device_mapping = 0; + kr = vm_map_enter_mem_object(kernel_map, &device_mapping, size, 0, + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, + (void *)device_pager, 0, FALSE, + VM_PROT_DEFAULT, VM_PROT_ALL, + VM_INHERIT_DEFAULT); + assert(kr == KERN_SUCCESS); + memory_object_deallocate(device_pager); + + vm_object_lock(anon_object); + vm_object_activity_begin(anon_object); + anon_object->blocked_access = TRUE; + vm_object_unlock(anon_object); + vm_object_lock(device_object); + vm_object_activity_begin(device_object); + device_object->blocked_access = TRUE; + vm_object_unlock(device_object); + + assert(anon_object->ref_count == 1); + assert(!anon_object->named); + assert(device_object->ref_count == 2); + assert(device_object->named); + + kr = vm_object_transpose(device_object, anon_object, size); + assert(kr == KERN_SUCCESS); + + vm_object_lock(anon_object); + vm_object_activity_end(anon_object); + anon_object->blocked_access = FALSE; + vm_object_unlock(anon_object); + vm_object_lock(device_object); + vm_object_activity_end(device_object); + device_object->blocked_access = FALSE; + vm_object_unlock(device_object); + + assert(anon_object->ref_count == 2); + assert(anon_object->named); + kr = vm_deallocate(kernel_map, anon_mapping, size); + assert(kr == KERN_SUCCESS); + assert(device_object->ref_count == 1); + assert(!device_object->named); + kr = vm_deallocate(kernel_map, device_mapping, size); + assert(kr == KERN_SUCCESS); + + printf("VM_TEST_DEVICE_PAGER_TRANSPOSE: PASS\n"); +} +#else /* VM_TEST_DEVICE_PAGER_TRANSPOSE */ +#define vm_test_device_pager_transpose() +#endif /* VM_TEST_DEVICE_PAGER_TRANSPOSE */ + +void +vm_tests(void) +{ + vm_test_collapse_compressor(); + vm_test_wire_and_extract(); + vm_test_page_wire_overflow_panic(); + vm_test_kernel_object_fault(); + vm_test_device_pager_transpose(); +} diff --git a/osfmk/vm/vm_pageout.h b/osfmk/vm/vm_pageout.h index c829afa08..a39763477 100644 --- a/osfmk/vm/vm_pageout.h +++ b/osfmk/vm/vm_pageout.h @@ -130,6 +130,13 @@ extern int vm_debug_events; #define VM_PAGEOUT_CACHE_EVICT 0x108 #define VM_PAGEOUT_THREAD_BLOCK 0x109 #define VM_PAGEOUT_JETSAM 0x10A +#define VM_INFO1 0x10B +#define VM_INFO2 0x10C +#define VM_INFO3 0x10D +#define VM_INFO4 0x10E +#define VM_INFO5 0x10F +#define VM_INFO6 0x110 +#define VM_INFO7 0x111 #define VM_UPL_PAGE_WAIT 0x120 #define VM_IOPL_PAGE_WAIT 0x121 @@ -138,6 +145,7 @@ extern int vm_debug_events; #if CONFIG_IOSCHED #define VM_PAGE_SLEEP 0x123 #define VM_PAGE_EXPEDITE 0x124 +#define VM_PAGE_EXPEDITE_NO_MEMORY 0x125 #endif #define VM_PRESSURE_EVENT 0x130 @@ -160,14 +168,12 @@ extern int vm_debug_events; extern void memoryshot(unsigned int event, unsigned int control); -extern kern_return_t vm_map_create_upl( - vm_map_t map, - vm_map_address_t offset, - upl_size_t *upl_size, - upl_t *upl, - upl_page_info_array_t page_list, - unsigned int *count, - upl_control_flags_t *flags); +extern void update_vm_info(void); + +#if CONFIG_IOSCHED +extern int upl_get_cached_tier( + upl_t upl); +#endif extern ppnum_t upl_get_highest_page( upl_t upl); @@ -178,14 +184,21 @@ extern upl_size_t upl_get_size( extern upl_t upl_associated_upl(upl_t upl); extern void upl_set_associated_upl(upl_t upl, upl_t associated_upl); -extern void iopl_valid_data( - upl_t upl_ptr); - #ifdef XNU_KERNEL_PRIVATE -extern vm_tag_t iopl_set_tag( +extern kern_return_t vm_map_create_upl( + vm_map_t map, + vm_map_address_t offset, + upl_size_t *upl_size, + upl_t *upl, + upl_page_info_array_t page_list, + unsigned int *count, + upl_control_flags_t *flags, + vm_tag_t tag); + +extern void iopl_valid_data( upl_t upl_ptr, - vm_tag_t tag); + vm_tag_t tag); #endif /* XNU_KERNEL_PRIVATE */ @@ -211,7 +224,9 @@ extern vm_page_t vm_page_get_next(vm_page_t page); extern kern_return_t mach_vm_pressure_level_monitor(boolean_t wait_for_pressure, unsigned int *pressure_level); +#if !CONFIG_EMBEDDED extern kern_return_t vm_pageout_wait(uint64_t deadline); +#endif #ifdef MACH_KERNEL_PRIVATE @@ -257,10 +272,8 @@ extern kern_return_t vm_pageout_internal_start(void); extern void vm_pageout_object_terminate( vm_object_t object); -extern int vm_pageout_cluster( - vm_page_t m, - boolean_t immediate_ok, - boolean_t keep_object_locked); +extern void vm_pageout_cluster( + vm_page_t m); extern void vm_pageout_initialize_page( vm_page_t m); @@ -357,7 +370,6 @@ struct upl { #define UPL_LITE 0x100 #define UPL_IO_WIRE 0x200 #define UPL_ACCESS_BLOCKED 0x400 -#define UPL_ENCRYPTED 0x800 #define UPL_SHADOWED 0x1000 #define UPL_KERNEL_OBJECT 0x2000 #define UPL_VECTOR 0x4000 @@ -401,7 +413,8 @@ extern kern_return_t vm_object_iopl_request( upl_t *upl_ptr, upl_page_info_array_t user_page_list, unsigned int *page_list_count, - upl_control_flags_t cntrl_flags); + upl_control_flags_t cntrl_flags, + vm_tag_t tag); extern kern_return_t vm_object_super_upl_request( vm_object_t object, @@ -411,7 +424,8 @@ extern kern_return_t vm_object_super_upl_request( upl_t *upl, upl_page_info_t *user_page_list, unsigned int *page_list_count, - upl_control_flags_t cntrl_flags); + upl_control_flags_t cntrl_flags, + vm_tag_t tag); /* should be just a regular vm_map_enter() */ extern kern_return_t vm_map_enter_upl( @@ -432,20 +446,6 @@ extern void vm_page_free_reserve(int pages); extern void vm_pageout_throttle_down(vm_page_t page); extern void vm_pageout_throttle_up(vm_page_t page); -/* - * ENCRYPTED SWAP: - */ -extern void upl_encrypt( - upl_t upl, - upl_offset_t crypt_offset, - upl_size_t crypt_size); -extern void vm_page_encrypt( - vm_page_t page, - vm_map_offset_t kernel_map_offset); -extern boolean_t vm_pages_encrypted; /* are there encrypted pages ? */ -extern void vm_page_decrypt( - vm_page_t page, - vm_map_offset_t kernel_map_offset); extern kern_return_t vm_paging_map_object( vm_page_t page, vm_object_t object, @@ -538,8 +538,6 @@ extern void vm_set_restrictions(void); extern int vm_compressor_mode; extern int vm_compressor_thread_count; extern boolean_t vm_restricted_to_single_processor; -extern boolean_t vm_compressor_immediate_preferred; -extern boolean_t vm_compressor_immediate_preferred_override; extern kern_return_t vm_pageout_compress_page(void **, char *, vm_page_t, boolean_t); extern void vm_pageout_anonymous_pages(void); extern void vm_pageout_disconnect_all_pages(void); @@ -575,4 +573,18 @@ extern struct vm_config vm_config; #endif /* KERNEL_PRIVATE */ +#ifdef XNU_KERNEL_PRIVATE +#define MAX_COMPRESSOR_THREAD_COUNT 8 + +#if DEVELOPMENT || DEBUG +typedef struct vmct_stats_s { + uint64_t vmct_runtimes[MAX_COMPRESSOR_THREAD_COUNT]; + uint64_t vmct_pages[MAX_COMPRESSOR_THREAD_COUNT]; + uint64_t vmct_iterations[MAX_COMPRESSOR_THREAD_COUNT]; + uint64_t vmct_cthreads_total; + int32_t vmct_minpages[MAX_COMPRESSOR_THREAD_COUNT]; + int32_t vmct_maxpages[MAX_COMPRESSOR_THREAD_COUNT]; +} vmct_stats_t; +#endif +#endif #endif /* _VM_VM_PAGEOUT_H_ */ diff --git a/osfmk/vm/vm_phantom_cache.c b/osfmk/vm/vm_phantom_cache.c index 4d8043702..a075f53fa 100644 --- a/osfmk/vm/vm_phantom_cache.c +++ b/osfmk/vm/vm_phantom_cache.c @@ -36,13 +36,21 @@ uint32_t phantom_cache_eval_period_in_msecs = 250; uint32_t phantom_cache_thrashing_threshold_ssd = 1000; +#if CONFIG_EMBEDDED +uint32_t phantom_cache_thrashing_threshold = 500; +#else uint32_t phantom_cache_thrashing_threshold = 100; +#endif /* * Number of consecutive thrashing periods required before * vm_phantom_cache_check_pressure() returns true. */ +#if CONFIG_EMBEDDED +unsigned phantom_cache_contiguous_periods = 4; +#else unsigned phantom_cache_contiguous_periods = 2; +#endif clock_sec_t pc_start_of_eval_period_sec = 0; clock_nsec_t pc_start_of_eval_period_nsec = 0; @@ -103,7 +111,11 @@ vm_phantom_cache_init() if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) return; +#if CONFIG_EMBEDDED + num_entries = (uint32_t)(((max_mem / PAGE_SIZE) / 10) / VM_GHOST_PAGES_PER_ENTRY); +#else num_entries = (uint32_t)(((max_mem / PAGE_SIZE) / 4) / VM_GHOST_PAGES_PER_ENTRY); +#endif vm_phantom_cache_num_entries = 1; while (vm_phantom_cache_num_entries < num_entries) diff --git a/osfmk/vm/vm_protos.h b/osfmk/vm/vm_protos.h index 8c12eb600..de4756f67 100644 --- a/osfmk/vm/vm_protos.h +++ b/osfmk/vm/vm_protos.h @@ -75,6 +75,8 @@ extern task_t port_name_to_task( mach_port_name_t name); extern task_t port_name_to_task_inspect( mach_port_name_t name); +extern void ipc_port_release_send( + ipc_port_t port); #endif /* _IPC_IPC_PORT_H_ */ extern ipc_space_t get_task_ipcspace( @@ -101,6 +103,12 @@ extern int get_map_nentries(vm_map_t); extern vm_map_offset_t vm_map_page_mask(vm_map_t); +extern kern_return_t vm_map_purgable_control( + vm_map_t map, + vm_map_offset_t address, + vm_purgable_t control, + int *state); + #if CONFIG_COREDUMP extern boolean_t coredumpok(vm_map_t map, vm_offset_t va); #endif @@ -170,6 +178,29 @@ extern memory_object_control_t swapfile_pager_control(memory_object_t mem_obj); #define SIXTEENK_PAGE_SHIFT 14 #endif /* __arm64__ || ((__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS)) */ +#if __arm64__ +#define FOURK_PAGE_SIZE 0x1000 +#define FOURK_PAGE_MASK 0xFFF +#define FOURK_PAGE_SHIFT 12 + +extern unsigned int page_shift_user32; + +#define VM_MAP_DEBUG_FOURK MACH_ASSERT +#if VM_MAP_DEBUG_FOURK +extern int vm_map_debug_fourk; +#endif /* VM_MAP_DEBUG_FOURK */ +extern void fourk_pager_bootstrap(void); +extern memory_object_t fourk_pager_create(void); +extern vm_object_t fourk_pager_to_vm_object(memory_object_t mem_obj); +extern kern_return_t fourk_pager_populate( + memory_object_t mem_obj, + boolean_t overwrite, + int index, + vm_object_t new_backing_object, + vm_object_offset_t new_backing_offset, + vm_object_t *old_backing_object, + vm_object_offset_t *old_backing_offset); +#endif /* __arm64__ */ /* * bsd @@ -327,8 +358,6 @@ extern kern_return_t vnode_pager_terminate( memory_object_t); extern void vnode_pager_vrele( struct vnode *vp); -extern void vnode_pager_release_from_cache( - int *); extern struct vnode *vnode_pager_lookup_vnode( memory_object_t); @@ -428,6 +457,11 @@ extern boolean_t cs_validate_range(struct vnode *vp, vm_size_t size, unsigned *result); +extern kern_return_t memory_entry_purgeable_control_internal( + ipc_port_t entry_port, + vm_purgable_t control, + int *state); + extern kern_return_t mach_memory_entry_purgable_control( ipc_port_t entry_port, vm_purgable_t control, @@ -480,6 +514,7 @@ extern kern_return_t compressor_memory_object_create( memory_object_t *); extern boolean_t vm_compressor_low_on_space(void); +extern boolean_t vm_compressor_out_of_space(void); extern int vm_swap_low_on_space(void); void do_fastwake_warmup_all(void); #if CONFIG_JETSAM @@ -540,46 +575,14 @@ extern int secluded_for_filecache; extern int secluded_for_fbdp; #endif -/* - * "secluded_aging_policy" controls the aging of secluded pages: - * - * SECLUDED_AGING_FIFO - * When a page eligible for the secluded queue is activated or - * deactivated, it is inserted in the secluded queue. - * When it get pushed out of the secluded queue, it gets freed. - * - * SECLUDED_AGING_ALONG_ACTIVE - * When a page eligible for the secluded queue is activated, it is - * inserted in the secluded queue. - * When it gets pushed out of the secluded queue, its "referenced" bit - * is reset and it is inserted in the inactive queue. - * - * SECLUDED_AGING_AFTER_INACTIVE - * A page eligible for the secluded queue first makes its way through the - * active and inactive queues. - * When it is pushed out of the inactive queue without being re-activated, - * it is inserted in the secluded queue instead of being reclaimed. - * When it is pushed out of the secluded queue, it is either freed if it - * hasn't been re-referenced, or re-activated if it has been re-referenced. - * - * SECLUDED_AGING_BEFORE_ACTIVE - * A page eligible for the secluded queue will first make its way through - * the secluded queue. When it gets pushed out of the secluded queue (by - * new secluded pages), it goes back to the normal aging path, through the - * active queue and then the inactive queue. - */ -extern int secluded_aging_policy; -#define SECLUDED_AGING_FIFO 0 -#define SECLUDED_AGING_ALONG_ACTIVE 1 -#define SECLUDED_AGING_AFTER_INACTIVE 2 -#define SECLUDED_AGING_BEFORE_ACTIVE 3 - extern void memory_object_mark_eligible_for_secluded( memory_object_control_t control, boolean_t eligible_for_secluded); #endif /* CONFIG_SECLUDED_MEMORY */ +#define MAX_PAGE_RANGE_QUERY (1ULL * 1024 * 1024 * 1024) /* 1 GB */ + #ifdef __cplusplus } #endif diff --git a/osfmk/vm/vm_purgeable.c b/osfmk/vm/vm_purgeable.c index d15e25f27..a58b75001 100644 --- a/osfmk/vm/vm_purgeable.c +++ b/osfmk/vm/vm_purgeable.c @@ -712,7 +712,13 @@ vm_purgeable_object_find_and_lock( owner = object->vo_purgeable_owner; if (owner) { +#if CONFIG_EMBEDDED +#if CONFIG_JETSAM + object_task_importance = proc_get_memstat_priority((struct proc *)get_bsdtask_info(owner), TRUE); +#endif /* CONFIG_JETSAM */ +#else /* CONFIG_EMBEDDED */ object_task_importance = task_importance_estimate(owner); +#endif /* CONFIG_EMBEDDED */ } if (object_task_importance < best_object_task_importance) { diff --git a/osfmk/vm/vm_resident.c b/osfmk/vm/vm_resident.c index ed19cd337..feeaddd1e 100644 --- a/osfmk/vm/vm_resident.c +++ b/osfmk/vm/vm_resident.c @@ -90,6 +90,7 @@ #include #include #include +#include #include #include @@ -184,10 +185,26 @@ unsigned int vm_page_hash_shift; /* Shift for hash function */ uint32_t vm_page_bucket_hash; /* Basic bucket hash */ unsigned int vm_page_bucket_lock_count = 0; /* How big is array of locks? */ +#ifndef VM_TAG_ACTIVE_UPDATE +#error VM_TAG_ACTIVE_UPDATE +#endif +#ifndef VM_MAX_TAG_ZONES +#error VM_MAX_TAG_ZONES +#endif + +boolean_t vm_tag_active_update = VM_TAG_ACTIVE_UPDATE; lck_spin_t *vm_page_bucket_locks; lck_spin_t vm_objects_wired_lock; lck_spin_t vm_allocation_sites_lock; +vm_allocation_site_t vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC + 1]; +vm_allocation_site_t * vm_allocation_sites[VM_MAX_TAG_VALUE]; +#if VM_MAX_TAG_ZONES +vm_allocation_zone_total_t ** vm_allocation_zone_totals; +#endif /* VM_MAX_TAG_ZONES */ + +vm_tag_t vm_allocation_tag_highest; + #if VM_PAGE_BUCKETS_CHECK boolean_t vm_page_buckets_check_ready = FALSE; #if VM_PAGE_FAKE_BUCKETS @@ -244,9 +261,15 @@ hash_debug(void) * module must use the PAGE_SIZE, PAGE_MASK and PAGE_SHIFT * constants. */ +#if defined(__arm__) || defined(__arm64__) +vm_size_t page_size; +vm_size_t page_mask; +int page_shift; +#else vm_size_t page_size = PAGE_SIZE; vm_size_t page_mask = PAGE_MASK; int page_shift = PAGE_SHIFT; +#endif /* * Resident page structures are initialized from @@ -261,6 +284,7 @@ struct vm_page vm_page_template; vm_page_t vm_pages = VM_PAGE_NULL; vm_page_t vm_page_array_beginning_addr; vm_page_t vm_page_array_ending_addr; +vm_page_t vm_page_array_boundary; unsigned int vm_pages_count = 0; ppnum_t vm_page_lowest = 0; @@ -321,7 +345,7 @@ struct vplq *vm_page_local_q = NULL; * For debugging, this should be a strange value * that the pmap module can recognize in assertions. */ -ppnum_t vm_page_fictitious_addr = (ppnum_t) -1; +const ppnum_t vm_page_fictitious_addr = (ppnum_t) -1; /* * Guard pages are not accessible so they don't @@ -331,7 +355,7 @@ ppnum_t vm_page_fictitious_addr = (ppnum_t) -1; * we don't use a real physical page with that * physical address. */ -ppnum_t vm_page_guard_addr = (ppnum_t) -2; +const ppnum_t vm_page_guard_addr = (ppnum_t) -2; /* * Resident page structures are also chained on @@ -355,7 +379,6 @@ queue_head_t vm_objects_wired; #if CONFIG_BACKGROUND_QUEUE vm_page_queue_head_t vm_page_queue_background __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); -uint32_t vm_page_background_limit; uint32_t vm_page_background_target; uint32_t vm_page_background_count; uint64_t vm_page_background_promoted_count; @@ -379,6 +402,7 @@ unsigned int vm_page_throttled_count; unsigned int vm_page_speculative_count; unsigned int vm_page_wire_count; +unsigned int vm_page_wire_count_on_boot = 0; unsigned int vm_page_stolen_count; unsigned int vm_page_wire_count_initial; unsigned int vm_page_pages_initial; @@ -467,8 +491,50 @@ vm_set_page_size(void) break; } -#define COLOR_GROUPS_TO_STEAL 4 +#if defined (__x86_64__) + +#define MAX_CLUMP_SIZE 16 +#define DEFAULT_CLUMP_SIZE 4 + +unsigned int vm_clump_size, vm_clump_mask, vm_clump_shift, vm_clump_promote_threshold; + +#if DEVELOPMENT || DEBUG +unsigned long vm_clump_stats[MAX_CLUMP_SIZE+1]; +unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes; + +static inline void vm_clump_update_stats(unsigned int c) { + assert(c<=vm_clump_size); + if(c>0 && c<=vm_clump_size) vm_clump_stats[c]+=c; + vm_clump_allocs+=c; +} +#endif /* if DEVELOPMENT || DEBUG */ + +/* Called once to setup the VM clump knobs */ +static void +vm_page_setup_clump( void ) +{ + unsigned int override, n; + + vm_clump_size = DEFAULT_CLUMP_SIZE; + if ( PE_parse_boot_argn("clump_size", &override, sizeof (override)) ) vm_clump_size = override; + + if(vm_clump_size > MAX_CLUMP_SIZE) panic("vm_page_setup_clump:: clump_size is too large!"); + if(vm_clump_size < 1) panic("vm_page_setup_clump:: clump_size must be >= 1"); + if((vm_clump_size & (vm_clump_size-1)) != 0) panic("vm_page_setup_clump:: clump_size must be a power of 2"); + + vm_clump_promote_threshold = vm_clump_size; + vm_clump_mask = vm_clump_size - 1; + for(vm_clump_shift=0, n=vm_clump_size; n>1; n>>=1, vm_clump_shift++); + +#if DEVELOPMENT || DEBUG + bzero(vm_clump_stats, sizeof(vm_clump_stats)); + vm_clump_allocs = vm_clump_inserts = vm_clump_inrange = vm_clump_promotes = 0; +#endif /* if DEVELOPMENT || DEBUG */ +} +#endif /* #if defined (__x86_64__) */ + +#define COLOR_GROUPS_TO_STEAL 4 /* Called once during statup, once the cache geometry is known. */ @@ -476,7 +542,11 @@ static void vm_page_set_colors( void ) { unsigned int n, override; - + +#if defined (__x86_64__) + /* adjust #colors because we need to color outside the clump boundary */ + vm_cache_geometry_colors >>= vm_clump_shift; +#endif if ( PE_parse_boot_argn("colors", &override, sizeof (override)) ) /* colors specified as a boot-arg? */ n = override; else if ( vm_cache_geometry_colors ) /* do we know what the cache geometry is? */ @@ -490,12 +560,18 @@ vm_page_set_colors( void ) /* the count must be a power of 2 */ if ( ( n & (n - 1)) != 0 ) - panic("vm_page_set_colors"); + n = DEFAULT_COLORS; /* use default if all else fails */ vm_colors = n; vm_color_mask = n - 1; vm_free_magazine_refill_limit = vm_colors * COLOR_GROUPS_TO_STEAL; + +#if defined (__x86_64__) + /* adjust for reduction in colors due to clumping and multiple cores */ + if (real_ncpus) + vm_free_magazine_refill_limit *= (vm_clump_size * real_ncpus); +#endif } @@ -541,7 +617,14 @@ vm_page_init_local_q() * no point in this for a uni-processor system */ if (num_cpus >= 2) { +#if KASAN + /* KASAN breaks the expectation of a size-aligned object by adding a + * rezone, so explicitly align. */ + t_local_q = (struct vplq *)kalloc(num_cpus * sizeof(struct vplq) + VM_PACKED_POINTER_ALIGNMENT); + t_local_q = (void *)(((uintptr_t)t_local_q + (VM_PACKED_POINTER_ALIGNMENT-1)) & ~(VM_PACKED_POINTER_ALIGNMENT-1)); +#else t_local_q = (struct vplq *)kalloc(num_cpus * sizeof(struct vplq)); +#endif for (i = 0; i < num_cpus; i++) { struct vpl *lq; @@ -559,6 +642,17 @@ vm_page_init_local_q() } } +/* + * vm_init_before_launchd + * + * This should be called right before launchd is loaded. + */ +void +vm_init_before_launchd() +{ + vm_page_wire_count_on_boot = vm_page_wire_count; +} + /* * vm_page_bootstrap: @@ -612,7 +706,9 @@ vm_page_bootstrap( m->private = FALSE; m->__unused_pageq_bits = 0; +#if !defined(__arm__) && !defined(__arm64__) VM_PAGE_SET_PHYS_PAGE(m, 0); /* reset later */ +#endif m->busy = TRUE; m->wanted = FALSE; m->tabled = FALSE; @@ -630,8 +726,6 @@ vm_page_bootstrap( m->overwriting = FALSE; m->restart = FALSE; m->unusual = FALSE; - m->encrypted = FALSE; - m->encrypted_cleaning = FALSE; m->cs_validated = FALSE; m->cs_tainted = FALSE; m->cs_nx = FALSE; @@ -701,7 +795,6 @@ vm_page_bootstrap( if (vm_page_background_target > VM_PAGE_BACKGROUND_TARGET_MAX) vm_page_background_target = VM_PAGE_BACKGROUND_TARGET_MAX; - vm_page_background_limit = vm_page_background_target + 256; vm_page_background_mode = VM_PAGE_BG_LEVEL_1; vm_page_background_exclude_external = 0; @@ -709,13 +802,9 @@ vm_page_bootstrap( PE_parse_boot_argn("vm_page_bg_mode", &vm_page_background_mode, sizeof(vm_page_background_mode)); PE_parse_boot_argn("vm_page_bg_exclude_external", &vm_page_background_exclude_external, sizeof(vm_page_background_exclude_external)); PE_parse_boot_argn("vm_page_bg_target", &vm_page_background_target, sizeof(vm_page_background_target)); - PE_parse_boot_argn("vm_page_bg_limit", &vm_page_background_limit, sizeof(vm_page_background_limit)); - if (vm_page_background_mode > VM_PAGE_BG_LEVEL_3) + if (vm_page_background_mode > VM_PAGE_BG_LEVEL_1) vm_page_background_mode = VM_PAGE_BG_LEVEL_1; - - if (vm_page_background_limit <= vm_page_background_target) - vm_page_background_limit = vm_page_background_target + 256; #endif vm_page_free_wanted = 0; vm_page_free_wanted_privileged = 0; @@ -723,6 +812,11 @@ vm_page_bootstrap( vm_page_free_wanted_secluded = 0; #endif /* CONFIG_SECLUDED_MEMORY */ +#if defined (__x86_64__) + /* this must be called before vm_page_set_colors() */ + vm_page_setup_clump(); +#endif + vm_page_set_colors(); bzero(vm_page_inactive_states, sizeof(vm_page_inactive_states)); @@ -760,6 +854,15 @@ vm_page_bootstrap( vm_page_active_or_inactive_states[VM_PAGE_ON_SECLUDED_Q] = 1; #endif /* CONFIG_SECLUDED_MEMORY */ + for (i = 0; i < VM_KERN_MEMORY_FIRST_DYNAMIC; i++) + { + vm_allocation_sites_static[i].refcount = 2; + vm_allocation_sites_static[i].tag = i; + vm_allocation_sites[i] = &vm_allocation_sites_static[i]; + } + vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC].refcount = 2; + vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC].tag = VM_KERN_MEMORY_ANY; + vm_allocation_sites[VM_KERN_MEMORY_ANY] = &vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC]; /* * Steal memory for the map and zone subsystems. @@ -916,8 +1019,9 @@ void * pmap_steal_memory( vm_size_t size) { + kern_return_t kr; vm_offset_t addr, vaddr; - ppnum_t phys_page; + ppnum_t phys_page; /* * We round the size to a round multiple. @@ -967,12 +1071,22 @@ pmap_steal_memory( * but some pmap modules barf if they are. */ #if defined(__LP64__) +#ifdef __arm64__ + /* ARM64_TODO: verify that we really don't need this */ +#else pmap_pre_expand(kernel_pmap, vaddr); #endif +#endif + + kr = pmap_enter(kernel_pmap, vaddr, phys_page, + VM_PROT_READ|VM_PROT_WRITE, VM_PROT_NONE, + VM_WIMG_USE_DEFAULT, FALSE); + + if (kr != KERN_SUCCESS) { + panic("pmap_steal_memory() pmap_enter failed, vaddr=%#lx, phys_page=%u", + (unsigned long)vaddr, phys_page); + } - pmap_enter(kernel_pmap, vaddr, phys_page, - VM_PROT_READ|VM_PROT_WRITE, VM_PROT_NONE, - VM_WIMG_USE_DEFAULT, FALSE); /* * Account for newly stolen memory */ @@ -980,6 +1094,9 @@ pmap_steal_memory( vm_page_stolen_count++; } +#if KASAN + kasan_notify_address(round_page(addr), size); +#endif return (void *) addr; } @@ -992,10 +1109,13 @@ int secluded_for_filecache = 2; /* filecache can use seclude memory */ #if 11 int secluded_for_fbdp = 0; #endif -int secluded_aging_policy = SECLUDED_AGING_BEFORE_ACTIVE; #endif /* CONFIG_SECLUDED_MEMORY */ +#if defined(__arm__) || defined(__arm64__) +extern void patch_low_glo_vm_page_info(void *, void *, uint32_t); +unsigned int vm_first_phys_ppnum = 0; +#endif void vm_page_release_startup(vm_page_t mem); @@ -1037,10 +1157,16 @@ pmap_startup( vm_page_array_beginning_addr = &vm_pages[0]; vm_page_array_ending_addr = &vm_pages[npages]; - for (i = 0, pages_initialized = 0; i < npages; i++) { if (!pmap_next_page(&phys_page)) break; +#if defined(__arm__) || defined(__arm64__) + if (pages_initialized == 0) { + vm_first_phys_ppnum = phys_page; + patch_low_glo_vm_page_info((void *)vm_page_array_beginning_addr, (void *)vm_page_array_ending_addr, vm_first_phys_ppnum); + } + assert((i + vm_first_phys_ppnum) == phys_page); +#endif if (pages_initialized == 0 || phys_page < vm_page_lowest) vm_page_lowest = phys_page; @@ -1049,6 +1175,7 @@ pmap_startup( pages_initialized++; } vm_pages_count = pages_initialized; + vm_page_array_boundary = &vm_pages[pages_initialized]; #if defined(__LP64__) @@ -1108,9 +1235,6 @@ pmap_startup( &secluded_for_fbdp, sizeof (secluded_for_fbdp)); #endif - PE_parse_boot_argn("secluded_aging_policy", - &secluded_aging_policy, - sizeof (secluded_aging_policy)); #endif /* CONFIG_SECLUDED_MEMORY */ // -debug code remove @@ -1234,6 +1358,7 @@ vm_page_module_init(void) zone_change(vm_page_zone, Z_EXHAUST, TRUE); zone_change(vm_page_zone, Z_FOREIGN, TRUE); zone_change(vm_page_zone, Z_GZALLOC_EXEMPT, TRUE); + zone_change(vm_page_zone, Z_ALIGNMENT_REQUIRED, TRUE); } /* @@ -1346,6 +1471,7 @@ vm_page_insert_internal( LCK_MTX_ASSERT(&vm_page_queue_lock, queues_lock_held ? LCK_MTX_ASSERT_OWNED : LCK_MTX_ASSERT_NOTOWNED); + if (queues_lock_held == FALSE) assert(!VM_PAGE_PAGEABLE(mem)); @@ -1356,7 +1482,11 @@ vm_page_insert_internal( "already in (obj=%p,off=0x%llx)", mem, object, offset, VM_PAGE_OBJECT(mem), mem->offset); #endif - assert(!object->internal || offset < object->vo_size); + if (object->internal && (offset >= object->vo_size)) { + panic("vm_page_insert_internal: (page=%p,obj=%p,off=0x%llx,size=0x%llx) inserted at offset past object bounds", + mem, object, offset, object->vo_size); + } + assert(vm_page_lookup(object, offset) == VM_PAGE_NULL); /* @@ -1416,17 +1546,9 @@ vm_page_insert_internal( object->resident_page_count++; if (VM_PAGE_WIRED(mem)) { assert(mem->wire_count > 0); - - if (!mem->private && !mem->fictitious) - { - if (!object->wired_page_count) - { - assert(VM_KERN_MEMORY_NONE != tag); - object->wire_tag = tag; - VM_OBJECT_WIRED(object); - } - } - object->wired_page_count++; + VM_OBJECT_WIRED_PAGE_UPDATE_START(object); + VM_OBJECT_WIRED_PAGE_ADD(object, mem); + VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag); } assert(object->resident_page_count >= object->wired_page_count); @@ -1738,11 +1860,9 @@ vm_page_remove( if (VM_PAGE_WIRED(mem)) { assert(mem->wire_count > 0); - assert(m_object->wired_page_count > 0); - m_object->wired_page_count--; - if (!m_object->wired_page_count) { - VM_OBJECT_UNWIRED(m_object); - } + VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object); + VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem); + VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag); } assert(m_object->resident_page_count >= m_object->wired_page_count); @@ -1793,6 +1913,7 @@ vm_page_remove( OSAddAtomic(-1, &vm_page_purgeable_count); } } + if (m_object->set_cache_attr == TRUE) pmap_set_cache_attributes(VM_PAGE_GET_PHYS_PAGE(mem), 0); @@ -2030,8 +2151,7 @@ void vm_page_rename( vm_page_t mem, vm_object_t new_object, - vm_object_offset_t new_offset, - boolean_t encrypted_ok) + vm_object_offset_t new_offset) { boolean_t internal_to_external, external_to_internal; vm_tag_t tag; @@ -2042,23 +2162,6 @@ vm_page_rename( assert(m_object != new_object); assert(m_object); - /* - * ENCRYPTED SWAP: - * The encryption key is based on the page's memory object - * (aka "pager") and paging offset. Moving the page to - * another VM object changes its "pager" and "paging_offset" - * so it has to be decrypted first, or we would lose the key. - * - * One exception is VM object collapsing, where we transfer pages - * from one backing object to its parent object. This operation also - * transfers the paging information, so the info - * should remain consistent. The caller (vm_object_do_collapse()) - * sets "encrypted_ok" in this case. - */ - if (!encrypted_ok && mem->encrypted) { - panic("vm_page_rename: page %p is encrypted\n", mem); - } - XPR(XPR_VM_PAGE, "vm_page_rename, new object 0x%X, offset 0x%X page 0x%X\n", new_object, new_offset, @@ -2189,10 +2292,16 @@ vm_page_grab_fictitious(void) return vm_page_grab_fictitious_common(vm_page_fictitious_addr); } +int vm_guard_count; + + vm_page_t vm_page_grab_guard(void) { - return vm_page_grab_fictitious_common(vm_page_guard_addr); + vm_page_t page; + page = vm_page_grab_fictitious_common(vm_page_guard_addr); + if (page) OSAddAtomic(1, &vm_guard_count); + return page; } @@ -2210,6 +2319,9 @@ vm_page_release_fictitious( assert(VM_PAGE_GET_PHYS_PAGE(m) == vm_page_fictitious_addr || VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr); + +if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) OSAddAtomic(-1, &vm_guard_count); + c_vm_page_release_fictitious++; zfree(vm_page_zone, m); @@ -2273,7 +2385,7 @@ void vm_page_more_fictitious(void) } retval = kernel_memory_allocate(zone_map, - &addr, PAGE_SIZE, VM_PROT_ALL, + &addr, PAGE_SIZE, 0, KMA_KOBJECT|KMA_NOPAGEWAIT, VM_KERN_MEMORY_ZONE); if (retval != KERN_SUCCESS) { /* @@ -2543,6 +2655,10 @@ vm_page_grab(void) return vm_page_grab_options(0); } +#if HIBERNATION +boolean_t hibernate_rebuild_needed = FALSE; +#endif /* HIBERNATION */ + vm_page_t vm_page_grab_options( int grab_options) @@ -2555,6 +2671,11 @@ vm_page_grab_options( return_page_from_cpu_list: assert(mem->vm_page_q_state == VM_PAGE_ON_FREE_LOCAL_Q); +#if HIBERNATION + if (hibernate_rebuild_needed) { + panic("%s:%d should not modify cpu->free_pages while hibernating", __FUNCTION__, __LINE__); + } +#endif /* HIBERNATION */ PROCESSOR_DATA(current_processor(), page_grab_count) += 1; PROCESSOR_DATA(current_processor(), free_pages) = mem->snext; @@ -2568,7 +2689,6 @@ return_page_from_cpu_list: assert(!mem->laundry); assert(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(mem))); assert(mem->busy); - assert(!mem->encrypted); assert(!mem->pmapped); assert(!mem->wpmapped); assert(!pmap_is_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem))); @@ -2636,6 +2756,7 @@ return_page_from_cpu_list: vm_page_t tail; unsigned int pages_to_steal; unsigned int color; + unsigned int clump_end, sub_count; while ( vm_page_free_count == 0 ) { @@ -2673,21 +2794,47 @@ return_page_from_cpu_list: head = tail = NULL; vm_page_free_count -= pages_to_steal; + clump_end = sub_count = 0; while (pages_to_steal--) { while (vm_page_queue_empty(&vm_page_queue_free[color].qhead)) color = (color + 1) & vm_color_mask; - +#if defined(__x86_64__) + vm_page_queue_remove_first_with_clump(&vm_page_queue_free[color].qhead, + mem, + vm_page_t, + pageq, + clump_end); +#else vm_page_queue_remove_first(&vm_page_queue_free[color].qhead, - mem, - vm_page_t, - pageq); + mem, + vm_page_t, + pageq); +#endif + assert(mem->vm_page_q_state == VM_PAGE_ON_FREE_Q); VM_PAGE_ZERO_PAGEQ_ENTRY(mem); - + +#if defined(__arm__) || defined(__arm64__) color = (color + 1) & vm_color_mask; +#else + +#if DEVELOPMENT || DEBUG + + sub_count++; + if (clump_end) { + vm_clump_update_stats(sub_count); + sub_count = 0; + color = (color + 1) & vm_color_mask; + } +#else + if (clump_end) color = (color + 1) & vm_color_mask; + +#endif /* if DEVELOPMENT || DEBUG */ + +#endif /* if defined(__arm__) || defined(__arm64__) */ if (head == NULL) head = mem; @@ -2704,13 +2851,20 @@ return_page_from_cpu_list: assert(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(mem))); assert(mem->busy); - assert(!mem->encrypted); assert(!mem->pmapped); assert(!mem->wpmapped); assert(!pmap_is_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem))); } +#if defined (__x86_64__) && (DEVELOPMENT || DEBUG) + vm_clump_update_stats(sub_count); +#endif lck_mtx_unlock(&vm_page_queue_free_lock); +#if HIBERNATION + if (hibernate_rebuild_needed) { + panic("%s:%d should not modify cpu->free_pages while hibernating", __FUNCTION__, __LINE__); + } +#endif /* HIBERNATION */ PROCESSOR_DATA(current_processor(), free_pages) = head->snext; PROCESSOR_DATA(current_processor(), start_color) = color; @@ -2740,10 +2894,6 @@ return_page_from_cpu_list: ((vm_page_free_count < vm_page_free_target) && ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_min))) thread_wakeup((event_t) &vm_page_free_wanted); -#if CONFIG_BACKGROUND_QUEUE - if (vm_page_background_mode == VM_PAGE_BG_LEVEL_3 && (vm_page_background_count > vm_page_background_limit)) - thread_wakeup((event_t) &vm_page_free_wanted); -#endif VM_CHECK_MEMORYSTATUS; @@ -2801,7 +2951,7 @@ vm_page_grab_secluded(void) } assert(!vm_page_queue_empty(&vm_page_queue_secluded)); LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); - mem = vm_page_queue_first(&vm_page_queue_secluded); + mem = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded); assert(mem->vm_page_q_state == VM_PAGE_ON_SECLUDED_Q); vm_page_queues_remove(mem, TRUE); @@ -3000,11 +3150,18 @@ vm_page_release( mem->lopage = FALSE; mem->vm_page_q_state = VM_PAGE_ON_FREE_Q; - color = VM_PAGE_GET_PHYS_PAGE(mem) & vm_color_mask; - vm_page_queue_enter_first(&vm_page_queue_free[color].qhead, + color = VM_PAGE_GET_COLOR(mem); +#if defined(__x86_64__) + vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead, mem, vm_page_t, pageq); +#else + vm_page_queue_enter(&vm_page_queue_free[color].qhead, + mem, + vm_page_t, + pageq); +#endif vm_page_free_count++; /* * Check if we should wake up someone waiting for page. @@ -3086,9 +3243,16 @@ vm_page_release_startup( mem->lopage = FALSE; mem->vm_page_q_state = VM_PAGE_ON_FREE_Q; vm_page_free_count++; - queue_free = &vm_page_queue_free[VM_PAGE_GET_PHYS_PAGE(mem) & vm_color_mask].qhead; + queue_free = &vm_page_queue_free[VM_PAGE_GET_COLOR(mem)].qhead; } - vm_page_queue_enter_first(queue_free, mem, vm_page_t, pageq); + if (mem->vm_page_q_state == VM_PAGE_ON_FREE_Q) { +#if defined(__x86_64__) + vm_page_queue_enter_clump(queue_free, mem, vm_page_t, pageq); +#else + vm_page_queue_enter(queue_free, mem, vm_page_t, pageq); +#endif + } else + vm_page_queue_enter_first(queue_free, mem, vm_page_t, pageq); } /* @@ -3294,11 +3458,10 @@ vm_page_free_prepare_queues( assert(mem->wire_count > 0); if (m_object) { - assert(m_object->wired_page_count > 0); - m_object->wired_page_count--; - if (!m_object->wired_page_count) { - VM_OBJECT_UNWIRED(m_object); - } + + VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object); + VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem); + VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag); assert(m_object->resident_page_count >= m_object->wired_page_count); @@ -3455,7 +3618,8 @@ vm_page_free_list( */ while (mem && pg_count < 64) { - assert(mem->vm_page_q_state == VM_PAGE_NOT_ON_Q); + assert((mem->vm_page_q_state == VM_PAGE_NOT_ON_Q) || + (mem->vm_page_q_state == VM_PAGE_IS_WIRED)); #if CONFIG_BACKGROUND_QUEUE assert(mem->vm_page_backgroundq.next == 0 && mem->vm_page_backgroundq.prev == 0 && @@ -3529,11 +3693,18 @@ vm_page_free_list( mem->lopage = FALSE; mem->vm_page_q_state = VM_PAGE_ON_FREE_Q; - color = VM_PAGE_GET_PHYS_PAGE(mem) & vm_color_mask; - vm_page_queue_enter_first(&vm_page_queue_free[color].qhead, + color = VM_PAGE_GET_COLOR(mem); +#if defined(__x86_64__) + vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead, mem, vm_page_t, pageq); +#else + vm_page_queue_enter(&vm_page_queue_free[color].qhead, + mem, + vm_page_t, + pageq); +#endif mem = nxt; } vm_page_free_count += pg_count; @@ -3680,16 +3851,9 @@ vm_page_wire( if (m_object) { - if (!mem->private && !mem->fictitious) - { - if (!m_object->wired_page_count) - { - assert(VM_KERN_MEMORY_NONE != tag); - m_object->wire_tag = tag; - VM_OBJECT_WIRED(m_object); - } - } - m_object->wired_page_count++; + VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object); + VM_OBJECT_WIRED_PAGE_ADD(m_object, mem); + VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, tag); assert(m_object->resident_page_count >= m_object->wired_page_count); @@ -3746,15 +3910,6 @@ vm_page_wire( if (check_memorystatus == TRUE) { VM_CHECK_MEMORYSTATUS; } - /* - * ENCRYPTED SWAP: - * The page could be encrypted, but - * We don't have to decrypt it here - * because we don't guarantee that the - * data is actually valid at this point. - * The page will get decrypted in - * vm_fault_wire() if needed. - */ } assert(!mem->gobbled); assert(mem->vm_page_q_state == VM_PAGE_IS_WIRED); @@ -3792,16 +3947,16 @@ vm_page_unwire( vm_object_lock_assert_exclusive(m_object); LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); if (--mem->wire_count == 0) { + mem->vm_page_q_state = VM_PAGE_NOT_ON_Q; + VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object); + VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem); + VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag); if (!mem->private && !mem->fictitious) { vm_page_wire_count--; } - assert(m_object->wired_page_count > 0); - m_object->wired_page_count--; - if (!m_object->wired_page_count) { - VM_OBJECT_UNWIRED(m_object); - } + assert(m_object->resident_page_count >= m_object->wired_page_count); if (m_object->purgable == VM_PURGABLE_VOLATILE) { @@ -3958,7 +4113,10 @@ void vm_page_enqueue_cleaned(vm_page_t m) assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr); LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); assert( !(m->absent && !m->unusual)); - assert( !VM_PAGE_WIRED(m)); + + if (VM_PAGE_WIRED(m)) { + return; + } if (m->gobbled) { if (!m->private && !m->fictitious) @@ -4070,12 +4228,7 @@ vm_page_activate( if (secluded_for_filecache && vm_page_secluded_target != 0 && num_tasks_can_use_secluded_mem == 0 && - m_object->eligible_for_secluded && - ((secluded_aging_policy == SECLUDED_AGING_FIFO) || - (secluded_aging_policy == - SECLUDED_AGING_ALONG_ACTIVE) || - (secluded_aging_policy == - SECLUDED_AGING_BEFORE_ACTIVE))) { + m_object->eligible_for_secluded) { vm_page_queue_enter(&vm_page_queue_secluded, m, vm_page_t, pageq); m->vm_page_q_state = VM_PAGE_ON_SECLUDED_Q; @@ -4542,10 +4695,6 @@ vm_page_part_copy( * vm_page_copy: * * Copy one page to another - * - * ENCRYPTED SWAP: - * The source page should not be encrypted. The caller should - * make sure the page is decrypted first, if necessary. */ int vm_page_copy_cs_validations = 0; @@ -4575,17 +4724,6 @@ vm_page_copy( #endif vm_object_lock_assert_held(src_m_object); - /* - * ENCRYPTED SWAP: - * The source page should not be encrypted at this point. - * The destination page will therefore not contain encrypted - * data after the copy. - */ - if (src_m->encrypted) { - panic("vm_page_copy: source page %p is encrypted\n", src_m); - } - dest_m->encrypted = FALSE; - if (src_m_object != VM_OBJECT_NULL && src_m_object->code_signed) { /* @@ -4666,12 +4804,10 @@ _vm_page_print( (p->cleaning ? "" : "!"), (p->precious ? "" : "!"), (p->clustered ? "" : "!")); - printf(" %soverwriting, %srestart, %sunusual, %sencrypted, %sencrypted_cleaning\n", + printf(" %soverwriting, %srestart, %sunusual\n", (p->overwriting ? "" : "!"), (p->restart ? "" : "!"), - (p->unusual ? "" : "!"), - (p->encrypted ? "" : "!"), - (p->encrypted_cleaning ? "" : "!")); + (p->unusual ? "" : "!")); printf(" %scs_validated, %scs_tainted, %scs_nx, %sno_cache\n", (p->cs_validated ? "" : "!"), (p->cs_tainted ? "" : "!"), @@ -4753,9 +4889,9 @@ vm_page_verify_free_list( panic("vm_page_verify_free_list(color=%u, npages=%u): page %p not busy\n", color, npages, m); if (color != (unsigned int) -1) { - if ((VM_PAGE_GET_PHYS_PAGE(m) & vm_color_mask) != color) + if (VM_PAGE_GET_COLOR(m) != color) panic("vm_page_verify_free_list(color=%u, npages=%u): page %p wrong color %u instead of %u\n", - color, npages, m, VM_PAGE_GET_PHYS_PAGE(m) & vm_color_mask, color); + color, npages, m, VM_PAGE_GET_COLOR(m), color); if (m->vm_page_q_state != VM_PAGE_ON_FREE_Q) panic("vm_page_verify_free_list(color=%u, npages=%u): page %p - expecting q_state == VM_PAGE_ON_FREE_Q, found %d\n", color, npages, m, m->vm_page_q_state); @@ -4846,6 +4982,42 @@ vm_page_verify_free_lists( void ) +#if __arm64__ +/* + * 1 or more clients (currently only SEP) ask for a large contiguous chunk of memory + * after the system has 'aged'. To ensure that other allocation requests don't mess + * with the chances of that request being satisfied, we pre-allocate a single contiguous + * 10MB buffer and hand it out to the first request of >= 4MB. + */ + +kern_return_t cpm_preallocate_early(void); + +vm_page_t cpm_preallocated_pages_list = NULL; +boolean_t preallocated_buffer_available = FALSE; + +#define PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT ((10 * 1024 * 1024) / PAGE_SIZE_64) /* 10 MB */ +#define MIN_CONTIG_PAGES_REQUEST_FOR_PREALLOCATED_BUFFER ((4 * 1024 *1024) / PAGE_SIZE_64) /* 4 MB */ + +kern_return_t +cpm_preallocate_early(void) +{ + + kern_return_t kr = KERN_SUCCESS; + vm_map_size_t prealloc_size = (PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT * PAGE_SIZE_64); + + printf("cpm_preallocate_early called to preallocate contiguous buffer of %llu pages\n", PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT); + + kr = cpm_allocate(CAST_DOWN(vm_size_t, prealloc_size), &cpm_preallocated_pages_list, 0, 0, TRUE, 0); + + if (kr != KERN_SUCCESS) { + printf("cpm_allocate for preallocated contig buffer failed with %d.\n", kr); + } else { + preallocated_buffer_available = TRUE; + } + + return kr; +} +#endif /* __arm64__ */ extern boolean_t (* volatile consider_buffer_cache_collect)(int); @@ -4925,17 +5097,18 @@ vm_page_find_contiguous( int flags) { vm_page_t m = NULL; - ppnum_t prevcontaddr; - ppnum_t start_pnum; - unsigned int npages, considered, scanned; - unsigned int page_idx, start_idx, last_idx, orig_last_idx; + ppnum_t prevcontaddr = 0; + ppnum_t start_pnum = 0; + unsigned int npages = 0, considered = 0, scanned = 0; + unsigned int page_idx = 0, start_idx = 0, last_idx = 0, orig_last_idx = 0; unsigned int idx_last_contig_page_found = 0; - int free_considered, free_available; - int substitute_needed; + int free_considered = 0, free_available = 0; + int substitute_needed = 0; boolean_t wrapped, zone_gc_called = FALSE; + kern_return_t kr; #if DEBUG - clock_sec_t tv_start_sec, tv_end_sec; - clock_usec_t tv_start_usec, tv_end_usec; + clock_sec_t tv_start_sec = 0, tv_end_sec = 0; + clock_usec_t tv_start_usec = 0, tv_end_usec = 0; #endif int yielded = 0; @@ -4959,6 +5132,57 @@ full_scan_again: vm_page_lock_queues(); +#if __arm64__ + if (preallocated_buffer_available) { + + if ((contig_pages >= MIN_CONTIG_PAGES_REQUEST_FOR_PREALLOCATED_BUFFER) && (contig_pages <= PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT)) { + + m = cpm_preallocated_pages_list; + + start_idx = (unsigned int) (m - &vm_pages[0]); + + if (wire == FALSE) { + + last_idx = start_idx; + + for(npages = 0; npages < contig_pages; npages++, last_idx++) { + + assert(vm_pages[last_idx].gobbled == FALSE); + + vm_pages[last_idx].gobbled = TRUE; + vm_page_gobble_count++; + + assert(1 == vm_pages[last_idx].wire_count); + /* + * Gobbled pages are counted as wired pages. So no need to drop + * the global wired page count. Just the page's wire count is fine. + */ + vm_pages[last_idx].wire_count--; + vm_pages[last_idx].vm_page_q_state = VM_PAGE_NOT_ON_Q; + } + + } + + last_idx = start_idx + contig_pages - 1; + + vm_pages[last_idx].snext = NULL; + + printf("Using preallocated buffer: Requested size (pages):%d... index range: %d-%d...freeing %llu pages\n", contig_pages, start_idx, last_idx, PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT - contig_pages); + + last_idx += 1; + for(npages = contig_pages; npages < PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT; npages++, last_idx++) { + + VM_PAGE_ZERO_PAGEQ_ENTRY(&vm_pages[last_idx]); + vm_page_free(&vm_pages[last_idx]); + } + + cpm_preallocated_pages_list = NULL; + preallocated_buffer_available = FALSE; + + goto done_scanning; + } + } +#endif /* __arm64__ */ lck_mtx_lock(&vm_page_queue_free_lock); @@ -5009,7 +5233,7 @@ retry: RESET_STATE_OF_RUN(); } else if (VM_PAGE_WIRED(m) || m->gobbled || - m->encrypted_cleaning || m->laundry || m->wanted || + m->laundry || m->wanted || m->cleaning || m->overwriting || m->free_when_done) { /* * page is in a transient state @@ -5192,7 +5416,7 @@ did_consider: if (m1->vm_page_q_state == VM_PAGE_ON_FREE_Q) { unsigned int color; - color = VM_PAGE_GET_PHYS_PAGE(m1) & vm_color_mask; + color = VM_PAGE_GET_COLOR(m1); #if MACH_ASSERT vm_page_verify_free_list(&vm_page_queue_free[color].qhead, color, m1, TRUE); #endif @@ -5272,7 +5496,7 @@ did_consider: } if (locked_object == VM_OBJECT_NULL || (VM_PAGE_WIRED(m1) || m1->gobbled || - m1->encrypted_cleaning || m1->laundry || m1->wanted || + m1->laundry || m1->wanted || m1->cleaning || m1->overwriting || m1->free_when_done || m1->busy) || (m1->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q)) { @@ -5356,8 +5580,6 @@ did_consider: assert(!m1->overwriting); m2->restart = m1->restart; m2->unusual = m1->unusual; - m2->encrypted = m1->encrypted; - assert(!m1->encrypted_cleaning); m2->cs_validated = m1->cs_validated; m2->cs_tainted = m1->cs_tainted; m2->cs_nx = m1->cs_nx; @@ -5421,7 +5643,9 @@ did_consider: m2->wpmapped = TRUE; PMAP_ENTER(kernel_pmap, m2->offset, m2, - VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, TRUE); + VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, TRUE, kr); + + assert(kr == KERN_SUCCESS); compressed_pages++; @@ -5576,7 +5800,7 @@ done_scanning: (void)(*consider_buffer_cache_collect)(1); } - consider_zone_gc(); + consider_zone_gc(FALSE); zone_gc_called = TRUE; @@ -5973,9 +6197,9 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount) struct vm_pageout_queue *eq; struct vm_pageout_queue *tq; + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 4) | DBG_FUNC_START, + VM_KERNEL_UNSLIDE_OR_PERM(q), qcount); - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 4) | DBG_FUNC_START, q, qcount, 0, 0, 0); - iq = &vm_pageout_queue_internal; eq = &vm_pageout_queue_external; @@ -6034,7 +6258,7 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount) l_object = m_object; } } - if ( !m_object->alive || m->encrypted_cleaning || m->cleaning || m->laundry || m->busy || m->absent || m->error) { + if ( !m_object->alive || m->cleaning || m->laundry || m->busy || m->absent || m->error) { /* * page is not to be cleaned * put it back on the head of its queue @@ -6146,7 +6370,7 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount) if (m_object->internal == TRUE) pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m), PMAP_OPTIONS_COMPRESSOR, NULL); - (void)vm_pageout_cluster(m, FALSE, FALSE); + vm_pageout_cluster(m); hibernate_stats.hibernate_found_dirty++; @@ -6280,7 +6504,7 @@ hibernate_flush_memory() orig_wire_count = vm_page_wire_count; (void)(*consider_buffer_cache_collect)(1); - consider_zone_gc(); + consider_zone_gc(FALSE); HIBLOG("hibernate_flush_memory: buffer_cache_gc freed up %d wired pages\n", orig_wire_count - vm_page_wire_count); @@ -6600,6 +6824,7 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list, #if MACH_ASSERT || DEBUG if (!preflight) { + assert(hibernate_vm_locks_are_safe()); vm_page_lock_queues(); if (vm_page_local_q) { for (i = 0; i < vm_page_local_q_count; i++) { @@ -6863,7 +7088,7 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list, } if (preflight == FALSE && discard_all == TRUE) { - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_START, 0, 0, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_START); HIBLOG("hibernate_teardown started\n"); count_discard_vm_struct_pages = hibernate_teardown_vm_structs(page_list, page_list_wired); @@ -6874,7 +7099,7 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list, hibernate_stats.cd_vm_struct_pages_unneeded = count_discard_vm_struct_pages; - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_END, 0, 0, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_END); } if (!preflight) { @@ -7070,7 +7295,6 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list) } boolean_t hibernate_paddr_map_inited = FALSE; -boolean_t hibernate_rebuild_needed = FALSE; unsigned int hibernate_teardown_last_valid_compact_indx = -1; vm_page_t hibernate_rebuild_hash_list = NULL; @@ -7213,11 +7437,18 @@ hibernate_free_range(int sindx, int eindx) mem->lopage = FALSE; mem->vm_page_q_state = VM_PAGE_ON_FREE_Q; - color = VM_PAGE_GET_PHYS_PAGE(mem) & vm_color_mask; - vm_page_queue_enter_first(&vm_page_queue_free[color].qhead, + color = VM_PAGE_GET_COLOR(mem); +#if defined(__x86_64__) + vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead, mem, vm_page_t, pageq); +#else + vm_page_queue_enter(&vm_page_queue_free[color].qhead, + mem, + vm_page_t, + pageq); +#endif vm_page_free_count++; sindx++; @@ -7230,7 +7461,7 @@ extern void hibernate_rebuild_pmap_structs(void); void hibernate_rebuild_vm_structs(void) { - int cindx, sindx, eindx; + int i, cindx, sindx, eindx; vm_page_t mem, tmem, mem_next; AbsoluteTime startTime, endTime; uint64_t nsec; @@ -7238,7 +7469,7 @@ hibernate_rebuild_vm_structs(void) if (hibernate_rebuild_needed == FALSE) return; - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_START, 0, 0, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_START); HIBLOG("hibernate_rebuild started\n"); clock_get_uptime(&startTime); @@ -7248,9 +7479,18 @@ hibernate_rebuild_vm_structs(void) bzero(&vm_page_buckets[0], vm_page_bucket_count * sizeof(vm_page_bucket_t)); eindx = vm_pages_count; + /* + * Mark all the vm_pages[] that have not been initialized yet as being + * transient. This is needed to ensure that buddy page search is corrrect. + * Without this random data in these vm_pages[] can trip the buddy search + */ + for (i = hibernate_teardown_last_valid_compact_indx+1; i < eindx; ++i) + vm_pages[i].vm_page_q_state = VM_PAGE_NOT_ON_Q; + for (cindx = hibernate_teardown_last_valid_compact_indx; cindx >= 0; cindx--) { mem = &vm_pages[cindx]; + assert(mem->vm_page_q_state != VM_PAGE_ON_FREE_Q); /* * hibernate_teardown_vm_structs leaves the location where * this vm_page_t must be located in "next". @@ -7305,7 +7545,7 @@ hibernate_rebuild_vm_structs(void) hibernate_rebuild_needed = FALSE; - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_END, 0, 0, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_END); } @@ -7329,6 +7569,8 @@ hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_l if (hibernate_should_abort()) return (0); + hibernate_rebuild_needed = TRUE; + HIBLOG("hibernate_teardown: wired_pages %d, free_pages %d, active_pages %d, inactive_pages %d, speculative_pages %d, cleaned_pages %d, compressor_pages %d\n", vm_page_wire_count, vm_page_free_count, vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count, vm_page_cleaned_count, compressor_object->resident_page_count); @@ -7365,7 +7607,7 @@ hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_l assert(mem->busy); assert(!mem->lopage); - color = VM_PAGE_GET_PHYS_PAGE(mem) & vm_color_mask; + color = VM_PAGE_GET_COLOR(mem); vm_page_queue_remove(&vm_page_queue_free[color].qhead, mem, @@ -7415,8 +7657,6 @@ hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_l } HIBLOG("hibernate_teardown: mark_as_unneeded_pages %d, %d, %d\n", unneeded_vm_page_bucket_pages, unneeded_vm_pages_pages, unneeded_pmap_pages); - hibernate_rebuild_needed = TRUE; - return (mark_as_unneeded_pages); } @@ -7805,23 +8045,6 @@ vm_page_enqueue_inactive(vm_page_t mem, boolean_t first) assert(mem->vm_page_q_state == VM_PAGE_NOT_ON_Q); vm_page_check_pageable_safe(mem); -#if CONFIG_SECLUDED_MEMORY - if (secluded_for_filecache && - vm_page_secluded_target != 0 && - num_tasks_can_use_secluded_mem == 0 && - m_object->eligible_for_secluded && - secluded_aging_policy == SECLUDED_AGING_FIFO) { - mem->vm_page_q_state = VM_PAGE_ON_SECLUDED_Q; - vm_page_queue_enter(&vm_page_queue_secluded, mem, - vm_page_t, pageq); - vm_page_secluded_count++; - vm_page_secluded_count_inuse++; - assert(!m_object->internal); -// vm_page_pageable_external_count++; - return; - } -#endif /* CONFIG_SECLUDED_MEMORY */ - if (m_object->internal) { mem->vm_page_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q; @@ -7916,8 +8139,14 @@ vm_page_check_pageable_safe(vm_page_t page) #include -vm_allocation_site_t * -vm_allocation_sites[VM_KERN_MEMORY_COUNT]; +#define KA_SIZE(namelen, subtotalscount) \ + (sizeof(struct vm_allocation_site) + (namelen) + 1 + ((subtotalscount) * sizeof(struct vm_allocation_total))) + +#define KA_NAME(alloc) \ + ((char *)(&(alloc)->subtotals[(alloc->subtotalscount)])) + +#define KA_NAME_LEN(alloc) \ + (VM_TAG_NAME_LEN_MAX & (alloc->flags >> VM_TAG_NAME_LEN_SHIFT)) vm_tag_t vm_tag_bt(void) @@ -7928,10 +8157,17 @@ vm_tag_bt(void) uintptr_t kstackb, kstackt; const vm_allocation_site_t * site; thread_t cthread; + kern_allocation_name_t name; cthread = current_thread(); if (__improbable(cthread == NULL)) return VM_KERN_MEMORY_OSFMK; + if ((name = thread_get_kernel_state(cthread)->allocation_name)) + { + if (!name->tag) vm_tag_alloc(name); + return name->tag; + } + kstackb = cthread->kernel_stack; kstackt = kstackb + kernel_stack_size; @@ -7949,53 +8185,74 @@ vm_tag_bt(void) /* Pull return address from one spot above the frame pointer */ retaddr = *(frameptr + 1); + if ((retaddr < vm_kernel_stext) || (retaddr > vm_kernel_top)) { - site = OSKextGetAllocationSiteForCaller(retaddr); - break; + site = OSKextGetAllocationSiteForCaller(retaddr); + break; } - frameptr = frameptr_next; } + return (site ? site->tag : VM_KERN_MEMORY_NONE); } -static uint64_t free_tag_bits[256/64]; +static uint64_t free_tag_bits[VM_MAX_TAG_VALUE/64]; void -vm_tag_alloc_locked(vm_allocation_site_t * site) +vm_tag_alloc_locked(vm_allocation_site_t * site, vm_allocation_site_t ** releasesiteP) { vm_tag_t tag; uint64_t avail; - uint64_t idx; + uint32_t idx; + vm_allocation_site_t * prev; if (site->tag) return; idx = 0; while (TRUE) { - avail = free_tag_bits[idx]; - if (avail) - { - tag = __builtin_clzll(avail); - avail &= ~(1ULL << (63 - tag)); - free_tag_bits[idx] = avail; - tag += (idx << 6); - break; - } - idx++; - if (idx >= (sizeof(free_tag_bits) / sizeof(free_tag_bits[0]))) - { - tag = VM_KERN_MEMORY_ANY; - break; - } + avail = free_tag_bits[idx]; + if (avail) + { + tag = __builtin_clzll(avail); + avail &= ~(1ULL << (63 - tag)); + free_tag_bits[idx] = avail; + tag += (idx << 6); + break; + } + idx++; + if (idx >= ARRAY_COUNT(free_tag_bits)) + { + for (idx = 0; idx < ARRAY_COUNT(vm_allocation_sites); idx++) + { + prev = vm_allocation_sites[idx]; + if (!prev) continue; + if (!KA_NAME_LEN(prev)) continue; + if (!prev->tag) continue; + if (prev->total) continue; + if (1 != prev->refcount) continue; + + assert(idx == prev->tag); + tag = idx; + prev->tag = VM_KERN_MEMORY_NONE; + *releasesiteP = prev; + break; + } + if (idx >= ARRAY_COUNT(vm_allocation_sites)) + { + tag = VM_KERN_MEMORY_ANY; + } + break; + } } site->tag = tag; - if (VM_KERN_MEMORY_ANY != tag) - { - assert(!vm_allocation_sites[tag]); - vm_allocation_sites[tag] = site; - } + + OSAddAtomic16(1, &site->refcount); + + if (VM_KERN_MEMORY_ANY != tag) vm_allocation_sites[tag] = site; + + if (tag > vm_allocation_tag_highest) vm_allocation_tag_highest = tag; } static void @@ -8023,60 +8280,298 @@ vm_tag_init(void) { vm_tag_free_locked(tag); } + + for (tag = VM_KERN_MEMORY_ANY + 1; tag < VM_MAX_TAG_VALUE; tag++) + { + vm_tag_free_locked(tag); + } } vm_tag_t vm_tag_alloc(vm_allocation_site_t * site) { vm_tag_t tag; + vm_allocation_site_t * releasesite; if (VM_TAG_BT & site->flags) { - tag = vm_tag_bt(); - if (VM_KERN_MEMORY_NONE != tag) return (tag); + tag = vm_tag_bt(); + if (VM_KERN_MEMORY_NONE != tag) return (tag); } if (!site->tag) { - lck_spin_lock(&vm_allocation_sites_lock); - vm_tag_alloc_locked(site); - lck_spin_unlock(&vm_allocation_sites_lock); + releasesite = NULL; + lck_spin_lock(&vm_allocation_sites_lock); + vm_tag_alloc_locked(site, &releasesite); + lck_spin_unlock(&vm_allocation_sites_lock); + if (releasesite) kern_allocation_name_release(releasesite); } return (site->tag); } +void +vm_tag_update_size(vm_tag_t tag, int64_t delta) +{ + vm_allocation_site_t * allocation; + uint64_t prior; + + assert(VM_KERN_MEMORY_NONE != tag); + assert(tag < VM_MAX_TAG_VALUE); + + allocation = vm_allocation_sites[tag]; + assert(allocation); + + if (delta < 0) { + assertf(allocation->total >= ((uint64_t)-delta), "tag %d, site %p", tag, allocation); + } + prior = OSAddAtomic64(delta, &allocation->total); + +#if DEBUG || DEVELOPMENT + + uint64_t new, peak; + new = prior + delta; + do + { + peak = allocation->peak; + if (new <= peak) break; + } + while (!OSCompareAndSwap64(peak, new, &allocation->peak)); + +#endif /* DEBUG || DEVELOPMENT */ + + if (tag < VM_KERN_MEMORY_FIRST_DYNAMIC) return; + + if (!prior && !allocation->tag) vm_tag_alloc(allocation); +} + +void +kern_allocation_update_size(kern_allocation_name_t allocation, int64_t delta) +{ + uint64_t prior; + + if (delta < 0) { + assertf(allocation->total >= ((uint64_t)-delta), "name %p", allocation); + } + prior = OSAddAtomic64(delta, &allocation->total); + +#if DEBUG || DEVELOPMENT + + uint64_t new, peak; + new = prior + delta; + do + { + peak = allocation->peak; + if (new <= peak) break; + } + while (!OSCompareAndSwap64(peak, new, &allocation->peak)); + +#endif /* DEBUG || DEVELOPMENT */ + + if (!prior && !allocation->tag) vm_tag_alloc(allocation); +} + +#if VM_MAX_TAG_ZONES + +void +vm_allocation_zones_init(void) +{ + kern_return_t ret; + vm_offset_t addr; + vm_size_t size; + + size = VM_MAX_TAG_VALUE * sizeof(vm_allocation_zone_total_t **) + + 2 * VM_MAX_TAG_ZONES * sizeof(vm_allocation_zone_total_t); + + ret = kernel_memory_allocate(kernel_map, + &addr, round_page(size), 0, + KMA_ZERO, VM_KERN_MEMORY_DIAG); + assert(KERN_SUCCESS == ret); + + vm_allocation_zone_totals = (vm_allocation_zone_total_t **) addr; + addr += VM_MAX_TAG_VALUE * sizeof(vm_allocation_zone_total_t **); + + // prepopulate VM_KERN_MEMORY_DIAG & VM_KERN_MEMORY_KALLOC so allocations + // in vm_tag_update_zone_size() won't recurse + vm_allocation_zone_totals[VM_KERN_MEMORY_DIAG] = (vm_allocation_zone_total_t *) addr; + addr += VM_MAX_TAG_ZONES * sizeof(vm_allocation_zone_total_t); + vm_allocation_zone_totals[VM_KERN_MEMORY_KALLOC] = (vm_allocation_zone_total_t *) addr; +} + +void +vm_tag_will_update_zone(vm_tag_t tag, uint32_t zidx) +{ + vm_allocation_zone_total_t * zone; + + assert(VM_KERN_MEMORY_NONE != tag); + assert(tag < VM_MAX_TAG_VALUE); + + if (zidx >= VM_MAX_TAG_ZONES) return; + + zone = vm_allocation_zone_totals[tag]; + if (!zone) + { + zone = kalloc_tag(VM_MAX_TAG_ZONES * sizeof(*zone), VM_KERN_MEMORY_DIAG); + if (!zone) return; + bzero(zone, VM_MAX_TAG_ZONES * sizeof(*zone)); + if (!OSCompareAndSwapPtr(NULL, zone, &vm_allocation_zone_totals[tag])) + { + kfree(zone, VM_MAX_TAG_ZONES * sizeof(*zone)); + } + } +} + +void +vm_tag_update_zone_size(vm_tag_t tag, uint32_t zidx, int64_t delta, int64_t dwaste) +{ + vm_allocation_zone_total_t * zone; + uint32_t new; + + assert(VM_KERN_MEMORY_NONE != tag); + assert(tag < VM_MAX_TAG_VALUE); + + if (zidx >= VM_MAX_TAG_ZONES) return; + + zone = vm_allocation_zone_totals[tag]; + assert(zone); + zone += zidx; + + /* the zone is locked */ + if (delta < 0) + { + assertf(zone->total >= ((uint64_t)-delta), "zidx %d, tag %d, %p", zidx, tag, zone); + zone->total += delta; + } + else + { + zone->total += delta; + if (zone->total > zone->peak) zone->peak = zone->total; + if (dwaste) + { + new = zone->waste; + if (zone->wastediv < 65536) zone->wastediv++; + else new -= (new >> 16); + __assert_only bool ov = os_add_overflow(new, dwaste, &new); + assert(!ov); + zone->waste = new; + } + } +} + +#endif /* VM_MAX_TAG_ZONES */ + +void +kern_allocation_update_subtotal(kern_allocation_name_t allocation, uint32_t subtag, int64_t delta) +{ + kern_allocation_name_t other; + struct vm_allocation_total * total; + uint32_t subidx; + + subidx = 0; + assert(VM_KERN_MEMORY_NONE != subtag); + for (; subidx < allocation->subtotalscount; subidx++) + { + if (VM_KERN_MEMORY_NONE == allocation->subtotals[subidx].tag) + { + allocation->subtotals[subidx].tag = subtag; + break; + } + if (subtag == allocation->subtotals[subidx].tag) break; + } + assert(subidx < allocation->subtotalscount); + if (subidx >= allocation->subtotalscount) return; + + total = &allocation->subtotals[subidx]; + other = vm_allocation_sites[subtag]; + assert(other); + + if (delta < 0) + { + assertf(total->total >= ((uint64_t)-delta), "name %p", allocation); + OSAddAtomic64(delta, &total->total); + assertf(other->mapped >= ((uint64_t)-delta), "other %p", other); + OSAddAtomic64(delta, &other->mapped); + } + else + { + OSAddAtomic64(delta, &other->mapped); + OSAddAtomic64(delta, &total->total); + } +} + +const char * +kern_allocation_get_name(kern_allocation_name_t allocation) +{ + return (KA_NAME(allocation)); +} + +kern_allocation_name_t +kern_allocation_name_allocate(const char * name, uint32_t subtotalscount) +{ + uint32_t namelen; + + namelen = (uint32_t) strnlen(name, MACH_MEMORY_INFO_NAME_MAX_LEN - 1); + + kern_allocation_name_t allocation; + allocation = kalloc(KA_SIZE(namelen, subtotalscount)); + bzero(allocation, KA_SIZE(namelen, subtotalscount)); + + allocation->refcount = 1; + allocation->subtotalscount = subtotalscount; + allocation->flags = (namelen << VM_TAG_NAME_LEN_SHIFT); + strlcpy(KA_NAME(allocation), name, namelen + 1); + + return (allocation); +} + +void +kern_allocation_name_release(kern_allocation_name_t allocation) +{ + assert(allocation->refcount > 0); + if (1 == OSAddAtomic16(-1, &allocation->refcount)) + { + kfree(allocation, KA_SIZE(KA_NAME_LEN(allocation), allocation->subtotalscount)); + } +} + +vm_tag_t +kern_allocation_name_get_vm_tag(kern_allocation_name_t allocation) +{ + return (vm_tag_alloc(allocation)); +} + static void -vm_page_count_object(mach_memory_info_t * sites, unsigned int __unused num_sites, vm_object_t object) +vm_page_count_object(mach_memory_info_t * info, unsigned int __unused num_info, vm_object_t object) { if (!object->wired_page_count) return; if (object != kernel_object) { - assert(object->wire_tag < num_sites); - sites[object->wire_tag].size += ptoa_64(object->wired_page_count); + assert(object->wire_tag < num_info); + info[object->wire_tag].size += ptoa_64(object->wired_page_count); } } -typedef void (*vm_page_iterate_proc)(mach_memory_info_t * sites, - unsigned int num_sites, vm_object_t object); +typedef void (*vm_page_iterate_proc)(mach_memory_info_t * info, + unsigned int num_info, vm_object_t object); static void -vm_page_iterate_purgeable_objects(mach_memory_info_t * sites, unsigned int num_sites, +vm_page_iterate_purgeable_objects(mach_memory_info_t * info, unsigned int num_info, vm_page_iterate_proc proc, purgeable_q_t queue, int group) { vm_object_t object; for (object = (vm_object_t) queue_first(&queue->objq[group]); - !queue_end(&queue->objq[group], (queue_entry_t) object); - object = (vm_object_t) queue_next(&object->objq)) + !queue_end(&queue->objq[group], (queue_entry_t) object); + object = (vm_object_t) queue_next(&object->objq)) { - proc(sites, num_sites, object); + proc(info, num_info, object); } } static void -vm_page_iterate_objects(mach_memory_info_t * sites, unsigned int num_sites, +vm_page_iterate_objects(mach_memory_info_t * info, unsigned int num_info, vm_page_iterate_proc proc) { purgeable_q_t volatile_q; @@ -8090,130 +8585,230 @@ vm_page_iterate_objects(mach_memory_info_t * sites, unsigned int num_sites, vm_object_t, objq) { - proc(sites, num_sites, object); + proc(info, num_info, object); } lck_spin_unlock(&vm_objects_wired_lock); lck_mtx_lock(&vm_purgeable_queue_lock); nonvolatile_q = &purgeable_nonvolatile_queue; for (object = (vm_object_t) queue_first(nonvolatile_q); - !queue_end(nonvolatile_q, (queue_entry_t) object); - object = (vm_object_t) queue_next(&object->objq)) + !queue_end(nonvolatile_q, (queue_entry_t) object); + object = (vm_object_t) queue_next(&object->objq)) { - proc(sites, num_sites, object); + proc(info, num_info, object); } volatile_q = &purgeable_queues[PURGEABLE_Q_TYPE_OBSOLETE]; - vm_page_iterate_purgeable_objects(sites, num_sites, proc, volatile_q, 0); + vm_page_iterate_purgeable_objects(info, num_info, proc, volatile_q, 0); volatile_q = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO]; for (group = 0; group < NUM_VOLATILE_GROUPS; group++) { - vm_page_iterate_purgeable_objects(sites, num_sites, proc, volatile_q, group); + vm_page_iterate_purgeable_objects(info, num_info, proc, volatile_q, group); } volatile_q = &purgeable_queues[PURGEABLE_Q_TYPE_LIFO]; for (group = 0; group < NUM_VOLATILE_GROUPS; group++) { - vm_page_iterate_purgeable_objects(sites, num_sites, proc, volatile_q, group); + vm_page_iterate_purgeable_objects(info, num_info, proc, volatile_q, group); } lck_mtx_unlock(&vm_purgeable_queue_lock); } static uint64_t -process_account(mach_memory_info_t * sites, unsigned int __unused num_sites, uint64_t zones_collectable_bytes) +process_account(mach_memory_info_t * info, unsigned int num_info, uint64_t zones_collectable_bytes, boolean_t iterated) { - uint64_t found; - unsigned int idx; + size_t namelen; + unsigned int idx, count, nextinfo; vm_allocation_site_t * site; + lck_spin_lock(&vm_allocation_sites_lock); - assert(num_sites >= VM_KERN_MEMORY_COUNT); - found = 0; - for (idx = 0; idx < VM_KERN_MEMORY_COUNT; idx++) + for (idx = 0; idx <= vm_allocation_tag_highest; idx++) { - found += sites[idx].size; - if (idx < VM_KERN_MEMORY_FIRST_DYNAMIC) - { - sites[idx].site = idx; - sites[idx].flags |= VM_KERN_SITE_TAG; - if (VM_KERN_MEMORY_ZONE == idx) - { - sites[idx].flags |= VM_KERN_SITE_HIDE; - sites[idx].collectable_bytes = zones_collectable_bytes; - } else sites[idx].flags |= VM_KERN_SITE_WIRED; - continue; - } - lck_spin_lock(&vm_allocation_sites_lock); - if ((site = vm_allocation_sites[idx])) - { - if (sites[idx].size) - { - sites[idx].flags |= VM_KERN_SITE_WIRED; - if (VM_TAG_KMOD == (VM_KERN_SITE_TYPE & site->flags)) + site = vm_allocation_sites[idx]; + if (!site) continue; + info[idx].mapped = site->mapped; + info[idx].tag = site->tag; + if (!iterated) + { + info[idx].size = site->total; +#if DEBUG || DEVELOPMENT + info[idx].peak = site->peak; +#endif /* DEBUG || DEVELOPMENT */ + } + else + { + if (!site->subtotalscount && (site->total != info[idx].size)) + { + printf("tag mismatch[%d] 0x%qx, iter 0x%qx\n", idx, site->total, info[idx].size); + info[idx].size = site->total; + } + } + } + + nextinfo = (vm_allocation_tag_highest + 1); + count = nextinfo; + if (count >= num_info) count = num_info; + + for (idx = 0; idx < count; idx++) + { + site = vm_allocation_sites[idx]; + if (!site) continue; + info[idx].flags |= VM_KERN_SITE_WIRED; + if (idx < VM_KERN_MEMORY_FIRST_DYNAMIC) + { + info[idx].site = idx; + info[idx].flags |= VM_KERN_SITE_TAG; + if (VM_KERN_MEMORY_ZONE == idx) + { + info[idx].flags |= VM_KERN_SITE_HIDE; + info[idx].flags &= ~VM_KERN_SITE_WIRED; + info[idx].collectable_bytes = zones_collectable_bytes; + } + } + else if ((namelen = (VM_TAG_NAME_LEN_MAX & (site->flags >> VM_TAG_NAME_LEN_SHIFT)))) + { + info[idx].site = 0; + info[idx].flags |= VM_KERN_SITE_NAMED; + if (namelen > sizeof(info[idx].name)) namelen = sizeof(info[idx].name); + strncpy(&info[idx].name[0], KA_NAME(site), namelen); + } + else if (VM_TAG_KMOD & site->flags) { - sites[idx].site = OSKextGetKmodIDForSite(site, NULL, 0); - sites[idx].flags |= VM_KERN_SITE_KMOD; + info[idx].site = OSKextGetKmodIDForSite(site, NULL, 0); + info[idx].flags |= VM_KERN_SITE_KMOD; } else { - sites[idx].site = VM_KERNEL_UNSLIDE(site); - sites[idx].flags |= VM_KERN_SITE_KERNEL; + info[idx].site = VM_KERNEL_UNSLIDE(site); + info[idx].flags |= VM_KERN_SITE_KERNEL; } - site = NULL; - } - else - { -#if 1 - site = NULL; -#else - /* this code would free a site with no allocations but can race a new - * allocation being made */ - vm_tag_free_locked(site->tag); - site->tag = VM_KERN_MEMORY_NONE; - vm_allocation_sites[idx] = NULL; - if (!(VM_TAG_UNLOAD & site->flags)) site = NULL; -#endif - } +#if VM_MAX_TAG_ZONES + vm_allocation_zone_total_t * zone; + unsigned int zidx; + vm_size_t elem_size; + + if (vm_allocation_zone_totals + && (zone = vm_allocation_zone_totals[idx]) + && (nextinfo < num_info)) + { + for (zidx = 0; zidx < VM_MAX_TAG_ZONES; zidx++) + { + if (!zone[zidx].peak) continue; + info[nextinfo] = info[idx]; + info[nextinfo].zone = zone_index_from_tag_index(zidx, &elem_size); + info[nextinfo].flags &= ~VM_KERN_SITE_WIRED; + info[nextinfo].flags |= VM_KERN_SITE_ZONE; + info[nextinfo].size = zone[zidx].total; + info[nextinfo].peak = zone[zidx].peak; + info[nextinfo].mapped = 0; + if (zone[zidx].wastediv) + { + info[nextinfo].collectable_bytes = ((zone[zidx].waste * zone[zidx].total / elem_size) / zone[zidx].wastediv); + } + nextinfo++; + } + } +#endif /* VM_MAX_TAG_ZONES */ + if (site->subtotalscount) + { + uint64_t mapped, mapcost, take; + uint32_t sub; + vm_tag_t alloctag; + + info[idx].size = site->total; + mapped = info[idx].size; + info[idx].mapped = mapped; + mapcost = 0; + for (sub = 0; sub < site->subtotalscount; sub++) + { + alloctag = site->subtotals[sub].tag; + assert(alloctag < num_info); + if (info[alloctag].name[0]) continue; + take = info[alloctag].mapped; + if (take > info[alloctag].size) take = info[alloctag].size; + if (take > mapped) take = mapped; + info[alloctag].mapped -= take; + info[alloctag].size -= take; + mapped -= take; + mapcost += take; + } + info[idx].size = mapcost; + } } lck_spin_unlock(&vm_allocation_sites_lock); - if (site) OSKextFreeSite(site); + + return (0); +} + +uint32_t +vm_page_diagnose_estimate(void) +{ + vm_allocation_site_t * site; + uint32_t count; + uint32_t idx; + + lck_spin_lock(&vm_allocation_sites_lock); + for (count = idx = 0; idx < VM_MAX_TAG_VALUE; idx++) + { + site = vm_allocation_sites[idx]; + if (!site) continue; + count++; +#if VM_MAX_TAG_ZONES + if (vm_allocation_zone_totals) + { + vm_allocation_zone_total_t * zone; + zone = vm_allocation_zone_totals[idx]; + if (!zone) continue; + for (uint32_t zidx = 0; zidx < VM_MAX_TAG_ZONES; zidx++) if (zone[zidx].peak) count++; + } +#endif } + lck_spin_unlock(&vm_allocation_sites_lock); - return (found); + /* some slop for new tags created */ + count += 8; + count += VM_KERN_COUNTER_COUNT; + + return (count); } + kern_return_t -vm_page_diagnose(mach_memory_info_t * sites, unsigned int num_sites, uint64_t zones_collectable_bytes) +vm_page_diagnose(mach_memory_info_t * info, unsigned int num_info, uint64_t zones_collectable_bytes) { - enum { kMaxKernelDepth = 1 }; - vm_map_t maps [kMaxKernelDepth]; - vm_map_entry_t entries[kMaxKernelDepth]; - vm_map_t map; - vm_map_entry_t entry; - vm_object_offset_t offset; - vm_page_t page; - int stackIdx, count; uint64_t wired_size; uint64_t wired_managed_size; uint64_t wired_reserved_size; + uint64_t booter_size; + boolean_t iterate; mach_memory_info_t * counts; - bzero(sites, num_sites * sizeof(mach_memory_info_t)); + bzero(info, num_info * sizeof(mach_memory_info_t)); if (!vm_page_wire_count_initial) return (KERN_ABORTED); - vm_page_iterate_objects(sites, num_sites, &vm_page_count_object); - +#if CONFIG_EMBEDDED + wired_size = ptoa_64(vm_page_wire_count); + wired_reserved_size = ptoa_64(vm_page_wire_count_initial - vm_page_stolen_count); +#else wired_size = ptoa_64(vm_page_wire_count + vm_lopage_free_count + vm_page_throttled_count); wired_reserved_size = ptoa_64(vm_page_wire_count_initial - vm_page_stolen_count + vm_page_throttled_count); +#endif wired_managed_size = ptoa_64(vm_page_wire_count - vm_page_wire_count_initial); - assert(num_sites >= (VM_KERN_MEMORY_COUNT + VM_KERN_COUNTER_COUNT)); - counts = &sites[VM_KERN_MEMORY_COUNT]; + booter_size = ml_get_booter_memory_size(); + wired_size += booter_size; + + assert(num_info >= VM_KERN_COUNTER_COUNT); + num_info -= VM_KERN_COUNTER_COUNT; + counts = &info[num_info]; -#define SET_COUNT(xcount, xsize, xflags) \ - counts[xcount].site = (xcount); \ - counts[xcount].size = (xsize); \ +#define SET_COUNT(xcount, xsize, xflags) \ + counts[xcount].tag = VM_MAX_TAG_VALUE + xcount; \ + counts[xcount].site = (xcount); \ + counts[xcount].size = (xsize); \ + counts[xcount].mapped = (xsize); \ counts[xcount].flags = VM_KERN_SITE_COUNTER | xflags; SET_COUNT(VM_KERN_COUNT_MANAGED, ptoa_64(vm_page_pages), 0); @@ -8222,10 +8817,13 @@ vm_page_diagnose(mach_memory_info_t * sites, unsigned int num_sites, uint64_t zo SET_COUNT(VM_KERN_COUNT_RESERVED, wired_reserved_size, VM_KERN_SITE_WIRED); SET_COUNT(VM_KERN_COUNT_STOLEN, ptoa_64(vm_page_stolen_count), VM_KERN_SITE_WIRED); SET_COUNT(VM_KERN_COUNT_LOPAGE, ptoa_64(vm_lopage_free_count), VM_KERN_SITE_WIRED); + SET_COUNT(VM_KERN_COUNT_WIRED_BOOT, ptoa_64(vm_page_wire_count_on_boot), 0); + SET_COUNT(VM_KERN_COUNT_BOOT_STOLEN, booter_size, VM_KERN_SITE_WIRED); -#define SET_MAP(xcount, xsize, xfree, xlargest) \ +#define SET_MAP(xcount, xsize, xfree, xlargest) \ counts[xcount].site = (xcount); \ counts[xcount].size = (xsize); \ + counts[xcount].mapped = (xsize); \ counts[xcount].free = (xfree); \ counts[xcount].largest = (xlargest); \ counts[xcount].flags = VM_KERN_SITE_COUNTER; @@ -8241,166 +8839,134 @@ vm_page_diagnose(mach_memory_info_t * sites, unsigned int num_sites, uint64_t zo vm_map_sizes(kalloc_map, &map_size, &map_free, &map_largest); SET_MAP(VM_KERN_COUNT_MAP_KALLOC, map_size, map_free, map_largest); - map = kernel_map; - stackIdx = 0; - while (map) - { - vm_map_lock(map); - for (entry = map->hdr.links.next; map; entry = entry->links.next) + iterate = !VM_TAG_ACTIVE_UPDATE; + if (iterate) { - if (entry->is_sub_map) - { - assert(stackIdx < kMaxKernelDepth); - maps[stackIdx] = map; - entries[stackIdx] = entry; - stackIdx++; - map = VME_SUBMAP(entry); - entry = NULL; - break; - } - if (VME_OBJECT(entry) == kernel_object) + enum { kMaxKernelDepth = 1 }; + vm_map_t maps [kMaxKernelDepth]; + vm_map_entry_t entries[kMaxKernelDepth]; + vm_map_t map; + vm_map_entry_t entry; + vm_object_offset_t offset; + vm_page_t page; + int stackIdx, count; + + vm_page_iterate_objects(info, num_info, &vm_page_count_object); + + map = kernel_map; + stackIdx = 0; + while (map) { - count = 0; - vm_object_lock(VME_OBJECT(entry)); - for (offset = entry->links.start; offset < entry->links.end; offset += page_size) - { - page = vm_page_lookup(VME_OBJECT(entry), offset); - if (page && VM_PAGE_WIRED(page)) count++; - } - vm_object_unlock(VME_OBJECT(entry)); + vm_map_lock(map); + for (entry = map->hdr.links.next; map; entry = entry->links.next) + { + if (entry->is_sub_map) + { + assert(stackIdx < kMaxKernelDepth); + maps[stackIdx] = map; + entries[stackIdx] = entry; + stackIdx++; + map = VME_SUBMAP(entry); + entry = NULL; + break; + } + if (VME_OBJECT(entry) == kernel_object) + { + count = 0; + vm_object_lock(VME_OBJECT(entry)); + for (offset = entry->links.start; offset < entry->links.end; offset += page_size) + { + page = vm_page_lookup(VME_OBJECT(entry), offset); + if (page && VM_PAGE_WIRED(page)) count++; + } + vm_object_unlock(VME_OBJECT(entry)); - if (count) - { - assert(VME_ALIAS(entry) < num_sites); - sites[VME_ALIAS(entry)].size += ptoa_64(count); - } - } - while (map && (entry == vm_map_last_entry(map))) - { - vm_map_unlock(map); - if (!stackIdx) map = NULL; - else - { - --stackIdx; - map = maps[stackIdx]; - entry = entries[stackIdx]; - } + if (count) + { + assert(VME_ALIAS(entry) != VM_KERN_MEMORY_NONE); + assert(VME_ALIAS(entry) < num_info); + info[VME_ALIAS(entry)].size += ptoa_64(count); + } + } + while (map && (entry == vm_map_last_entry(map))) + { + vm_map_unlock(map); + if (!stackIdx) map = NULL; + else + { + --stackIdx; + map = maps[stackIdx]; + entry = entries[stackIdx]; + } + } + } } - } } - process_account(sites, num_sites, zones_collectable_bytes); + process_account(info, num_info, zones_collectable_bytes, iterate); return (KERN_SUCCESS); } -uint32_t -vm_tag_get_kext(vm_tag_t tag, char * name, vm_size_t namelen) -{ - vm_allocation_site_t * site; - uint32_t kmodId; - - kmodId = 0; - lck_spin_lock(&vm_allocation_sites_lock); - if ((site = vm_allocation_sites[tag])) - { - if (VM_TAG_KMOD == (VM_KERN_SITE_TYPE & site->flags)) - { - kmodId = OSKextGetKmodIDForSite(site, name, namelen); - } - } - lck_spin_unlock(&vm_allocation_sites_lock); - - return (kmodId); -} - #if DEBUG || DEVELOPMENT -#define vm_tag_set_lock(set) lck_spin_lock(&set->lock) -#define vm_tag_set_unlock(set) lck_spin_unlock(&set->lock) - -void -vm_tag_set_init(vm_tag_set_t set, uint32_t count) -{ - lck_spin_init(&set->lock, &vm_page_lck_grp_bucket, &vm_page_lck_attr); - bzero(&set->entries, count * sizeof(struct vm_tag_set_entry)); -} - kern_return_t -vm_tag_set_enter(vm_tag_set_t set, uint32_t count, vm_tag_t tag) +vm_kern_allocation_info(uintptr_t addr, vm_size_t * size, vm_tag_t * tag, vm_size_t * zone_size) { - kern_return_t kr; - uint32_t idx, free; - - vm_tag_set_lock(set); + kern_return_t ret; + vm_size_t zsize; + vm_map_t map; + vm_map_entry_t entry; - assert(tag != VM_KERN_MEMORY_NONE); - - kr = KERN_NO_SPACE; - free = -1U; - for (idx = 0; idx < count; idx++) + zsize = zone_element_info((void *) addr, tag); + if (zsize) { - if (tag == set->entries[idx].tag) - { - set->entries[idx].count++; - kr = KERN_SUCCESS; - break; - } - if ((free == -1U) && !set->entries[idx].count) free = idx; + *zone_size = *size = zsize; + return (KERN_SUCCESS); } - if ((KERN_SUCCESS != kr) && (free != -1U)) - { - set->entries[free].tag = tag; - set->entries[free].count = 1; - kr = KERN_SUCCESS; - } - - vm_tag_set_unlock(set); + *zone_size = 0; + ret = KERN_INVALID_ADDRESS; + for (map = kernel_map; map; ) + { + vm_map_lock(map); + if (!vm_map_lookup_entry(map, addr, &entry)) break; + if (entry->is_sub_map) + { + if (map != kernel_map) break; + map = VME_SUBMAP(entry); + continue; + } + if (entry->vme_start != addr) break; + *tag = VME_ALIAS(entry); + *size = (entry->vme_end - addr); + ret = KERN_SUCCESS; + break; + } + if (map != kernel_map) vm_map_unlock(map); + vm_map_unlock(kernel_map); - return (kr); + return (ret); } -kern_return_t -vm_tag_set_remove(vm_tag_set_t set, uint32_t count, vm_tag_t tag, vm_tag_t * new_tagp) -{ - kern_return_t kr; - uint32_t idx; - vm_tag_t new_tag; +#endif /* DEBUG || DEVELOPMENT */ - assert(tag != VM_KERN_MEMORY_NONE); - new_tag = VM_KERN_MEMORY_NONE; - vm_tag_set_lock(set); +uint32_t +vm_tag_get_kext(vm_tag_t tag, char * name, vm_size_t namelen) +{ + vm_allocation_site_t * site; + uint32_t kmodId; - kr = KERN_NOT_IN_SET; - for (idx = 0; idx < count; idx++) + kmodId = 0; + lck_spin_lock(&vm_allocation_sites_lock); + if ((site = vm_allocation_sites[tag])) { - if ((tag != VM_KERN_MEMORY_NONE) - && (tag == set->entries[idx].tag) - && set->entries[idx].count) - { - set->entries[idx].count--; - kr = KERN_SUCCESS; - if (set->entries[idx].count) - { - new_tag = tag; - break; - } - if (!new_tagp) break; - tag = VM_KERN_MEMORY_NONE; - } - - if (set->entries[idx].count && (VM_KERN_MEMORY_NONE == new_tag)) + if (VM_TAG_KMOD & site->flags) { - new_tag = set->entries[idx].tag; - if (VM_KERN_MEMORY_NONE == tag) break; + kmodId = OSKextGetKmodIDForSite(site, name, namelen); } } + lck_spin_unlock(&vm_allocation_sites_lock); - vm_tag_set_unlock(set); - if (new_tagp) *new_tagp = new_tag; - - return (kr); + return (kmodId); } - -#endif /* DEBUG || DEVELOPMENT */ diff --git a/osfmk/vm/vm_shared_region.c b/osfmk/vm/vm_shared_region.c index e984f2b00..f7018253b 100644 --- a/osfmk/vm/vm_shared_region.c +++ b/osfmk/vm/vm_shared_region.c @@ -124,12 +124,14 @@ int shared_region_persistence = 0; /* no by default */ /* delay before reclaiming an unused shared region */ int shared_region_destroy_delay = 120; /* in seconds */ +#ifndef CONFIG_EMBEDDED /* * Only one cache gets to slide on Desktop, since we can't * tear down slide info properly today and the desktop actually * produces lots of shared caches. */ boolean_t shared_region_completed_slide = FALSE; +#endif /* this lock protects all the shared region data structures */ lck_grp_t *vm_shared_region_lck_grp; @@ -642,6 +644,14 @@ vm_shared_region_create( /* figure out the correct settings for the desired environment */ if (is_64bit) { switch (cputype) { +#if defined(__arm64__) + case CPU_TYPE_ARM64: + base_address = SHARED_REGION_BASE_ARM64; + size = SHARED_REGION_SIZE_ARM64; + pmap_nesting_start = SHARED_REGION_NESTING_BASE_ARM64; + pmap_nesting_size = SHARED_REGION_NESTING_SIZE_ARM64; + break; +#elif !defined(__arm__) case CPU_TYPE_I386: base_address = SHARED_REGION_BASE_X86_64; size = SHARED_REGION_SIZE_X86_64; @@ -654,6 +664,7 @@ vm_shared_region_create( pmap_nesting_start = SHARED_REGION_NESTING_BASE_PPC64; pmap_nesting_size = SHARED_REGION_NESTING_SIZE_PPC64; break; +#endif default: SHARED_REGION_TRACE_ERROR( ("shared_region: create: unknown cpu type %d\n", @@ -664,6 +675,15 @@ vm_shared_region_create( } } else { switch (cputype) { +#if defined(__arm__) || defined(__arm64__) + case CPU_TYPE_ARM: + case CPU_TYPE_ARM64: + base_address = SHARED_REGION_BASE_ARM; + size = SHARED_REGION_SIZE_ARM; + pmap_nesting_start = SHARED_REGION_NESTING_BASE_ARM; + pmap_nesting_size = SHARED_REGION_NESTING_SIZE_ARM; + break; +#else case CPU_TYPE_I386: base_address = SHARED_REGION_BASE_I386; size = SHARED_REGION_SIZE_I386; @@ -676,6 +696,7 @@ vm_shared_region_create( pmap_nesting_start = SHARED_REGION_NESTING_BASE_PPC; pmap_nesting_size = SHARED_REGION_NESTING_SIZE_PPC; break; +#endif default: SHARED_REGION_TRACE_ERROR( ("shared_region: create: unknown cpu type %d\n", @@ -698,10 +719,35 @@ vm_shared_region_create( goto done; } +#if defined(__arm__) || defined(__arm64__) + { + struct pmap *pmap_nested; + + pmap_nested = pmap_create(NULL, 0, is_64bit); + if (pmap_nested != PMAP_NULL) { + pmap_set_nested(pmap_nested); + sub_map = vm_map_create(pmap_nested, 0, size, TRUE); +#if defined(__arm64__) + if (is_64bit || + page_shift_user32 == SIXTEENK_PAGE_SHIFT) { + /* enforce 16KB alignment of VM map entries */ + vm_map_set_page_shift(sub_map, + SIXTEENK_PAGE_SHIFT); + } +#elif (__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS) + /* enforce 16KB alignment for watch targets with new ABI */ + vm_map_set_page_shift(sub_map, SIXTEENK_PAGE_SHIFT); +#endif /* __arm64__ */ + } else { + sub_map = VM_MAP_NULL; + } + } +#else /* create a VM sub map and its pmap */ sub_map = vm_map_create(pmap_create(NULL, 0, is_64bit), 0, size, TRUE); +#endif if (sub_map == VM_MAP_NULL) { ipc_port_release_send(mem_entry_port); kfree(shared_region, sizeof (*shared_region)); @@ -753,6 +799,9 @@ vm_shared_region_create( si->slide_info_size = 0; si->slide_info_entry = NULL; + /* Initialize UUID */ + memset(&shared_region->sr_uuid, '\0', sizeof(shared_region->sr_uuid)); + shared_region->sr_uuid_copied = FALSE; done: if (shared_region) { SHARED_REGION_TRACE_INFO( @@ -803,7 +852,6 @@ vm_shared_region_destroy( mem_entry = (vm_named_entry_t) shared_region->sr_mem_entry->ip_kobject; assert(mem_entry->is_sub_map); assert(!mem_entry->internal); - assert(!mem_entry->is_pager); assert(!mem_entry->is_copy); map = mem_entry->backing.map; @@ -1044,8 +1092,19 @@ vm_shared_region_map_file( struct shared_file_mapping_np *mapping_to_slide = NULL; mach_vm_offset_t first_mapping = (mach_vm_offset_t) -1; vm_map_offset_t lowest_unnestable_addr = 0; + vm_map_kernel_flags_t vmk_flags; +#if __arm64__ + if ((shared_region->sr_64bit || + page_shift_user32 == SIXTEENK_PAGE_SHIFT) && + ((slide & SIXTEENK_PAGE_MASK) != 0)) { + printf("FOURK_COMPAT: %s: rejecting mis-aligned slide 0x%x\n", + __FUNCTION__, slide); + kr = KERN_INVALID_ARGUMENT; + goto done; + } +#endif /* __arm64__ */ kr = KERN_SUCCESS; @@ -1096,6 +1155,8 @@ vm_shared_region_map_file( /* get the VM object associated with the file to be mapped */ file_object = memory_object_control_to_vm_object(file_control); + assert(file_object); + /* establish the mappings */ for (i = 0; i < mappings_count; i++) { SHARED_REGION_TRACE_INFO( @@ -1143,6 +1204,9 @@ vm_shared_region_map_file( target_address = mappings[i].sfm_address - sr_base_address; + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_already = TRUE; + /* establish that mapping, OK if it's "already" there */ if (map_port == MACH_PORT_NULL) { /* @@ -1163,7 +1227,9 @@ vm_shared_region_map_file( vm_map_round_page(mappings[i].sfm_size, VM_MAP_PAGE_MASK(sr_map)), 0, - VM_FLAGS_FIXED | VM_FLAGS_ALREADY, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, object, 0, TRUE, @@ -1179,7 +1245,9 @@ vm_shared_region_map_file( vm_map_round_page(mappings[i].sfm_size, VM_MAP_PAGE_MASK(sr_map)), 0, - VM_FLAGS_FIXED | VM_FLAGS_ALREADY, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, map_port, mappings[i].sfm_file_offset, TRUE, @@ -1312,6 +1380,30 @@ vm_shared_region_map_file( shared_region->sr_first_mapping == (mach_vm_offset_t) -1) { shared_region->sr_first_mapping = first_mapping; } + + + /* copy in the shared region UUID to the shared region structure */ + if (kr == KERN_SUCCESS && !shared_region->sr_uuid_copied) { + int error = copyin((shared_region->sr_base_address + shared_region->sr_first_mapping + + offsetof(struct _dyld_cache_header, uuid)), + (char *)&shared_region->sr_uuid, + sizeof(shared_region->sr_uuid)); + if (error == 0) { + shared_region->sr_uuid_copied = TRUE; + } else { +#if DEVELOPMENT || DEBUG + panic("shared_region: copyin_UUID(sr_base_addr:0x%016llx sr_first_mapping:0x%016llx " + "offset:0x%016llx size:0x%016llx) failed with %d\n", + (long long)shared_region->sr_base_address, + (long long)shared_region->sr_first_mapping, + (long long)offsetof(struct _dyld_cache_header, uuid), + (long long)sizeof(shared_region->sr_uuid), + error); +#endif /* DEVELOPMENT || DEBUG */ + shared_region->sr_uuid_copied = FALSE; + } + } + /* we're done working on that shared region */ shared_region->sr_mapping_in_progress = FALSE; thread_wakeup((event_t) &shared_region->sr_mapping_in_progress); @@ -1409,6 +1501,8 @@ vm_shared_region_enter( mapping_size, 0, VM_FLAGS_FIXED, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, sr_handle, sr_offset, TRUE, @@ -1459,7 +1553,9 @@ vm_shared_region_enter( &target_address, mapping_size, 0, - (VM_FLAGS_FIXED | VM_MAKE_TAG(VM_MEMORY_SHARED_PMAP)), + VM_FLAGS_FIXED, + VM_MAP_KERNEL_FLAGS_NONE, + VM_MEMORY_SHARED_PMAP, sr_handle, sr_offset, TRUE, @@ -1498,6 +1594,8 @@ vm_shared_region_enter( mapping_size, 0, VM_FLAGS_FIXED, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, sr_handle, sr_offset, TRUE, @@ -2146,6 +2244,7 @@ vm_commpage_text_init(void) commpage_text64_location = (user64_addr_t) (_COMM_PAGE64_TEXT_START + offset); commpage_text_populate(); +#elif defined(__arm64__) || defined(__arm__) #else #error Unknown architecture. #endif /* __i386__ || __x86_64__ */ @@ -2201,10 +2300,23 @@ vm_commpage_enter( task_t task, boolean_t is64bit) { +#if defined(__arm__) +#pragma unused(is64bit) + (void)task; + (void)map; + return KERN_SUCCESS; +#elif defined(__arm64__) +#pragma unused(is64bit) + (void)task; + (void)map; + pmap_insert_sharedpage(vm_map_pmap(map)); + return KERN_SUCCESS; +#else ipc_port_t commpage_handle, commpage_text_handle; vm_map_offset_t commpage_address, objc_address, commpage_text_address; vm_map_size_t commpage_size, objc_size, commpage_text_size; int vm_flags; + vm_map_kernel_flags_t vmk_flags; kern_return_t kr; SHARED_REGION_TRACE_DEBUG( @@ -2214,7 +2326,9 @@ vm_commpage_enter( commpage_text_size = _COMM_PAGE_TEXT_AREA_LENGTH; /* the comm page is likely to be beyond the actual end of the VM map */ - vm_flags = VM_FLAGS_FIXED | VM_FLAGS_BEYOND_MAX; + vm_flags = VM_FLAGS_FIXED; + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_beyond_max = TRUE; /* select the appropriate comm page for this task */ assert(! (is64bit ^ vm_map_is_64bit(map))); @@ -2237,10 +2351,11 @@ vm_commpage_enter( commpage_text_address = (vm_map_offset_t) commpage_text32_location; } + vm_tag_t tag = VM_KERN_MEMORY_NONE; if ((commpage_address & (pmap_nesting_size_min - 1)) == 0 && (commpage_size & (pmap_nesting_size_min - 1)) == 0) { /* the commpage is properly aligned or sized for pmap-nesting */ - vm_flags |= VM_MAKE_TAG(VM_MEMORY_SHARED_PMAP); + tag = VM_MEMORY_SHARED_PMAP; } /* map the comm page in the task's address space */ assert(commpage_handle != IPC_PORT_NULL); @@ -2250,6 +2365,8 @@ vm_commpage_enter( commpage_size, 0, vm_flags, + vmk_flags, + tag, commpage_handle, 0, FALSE, @@ -2274,6 +2391,8 @@ vm_commpage_enter( commpage_text_size, 0, vm_flags, + vmk_flags, + tag, commpage_text_handle, 0, FALSE, @@ -2300,7 +2419,9 @@ vm_commpage_enter( &objc_address, objc_size, 0, - VM_FLAGS_FIXED | VM_FLAGS_BEYOND_MAX, + VM_FLAGS_FIXED, + vmk_flags, + tag, IPC_PORT_NULL, 0, FALSE, @@ -2322,6 +2443,7 @@ vm_commpage_enter( (void *)VM_KERNEL_ADDRPERM(map), (void *)VM_KERNEL_ADDRPERM(task), kr)); return kr; +#endif } int @@ -2357,7 +2479,9 @@ vm_shared_region_slide(uint32_t slide, vm_shared_region_sleep(&sr->sr_slide_in_progress, THREAD_UNINT); } if (sr->sr_slid +#ifndef CONFIG_EMBEDDED || shared_region_completed_slide +#endif ) { vm_shared_region_unlock(); @@ -2421,7 +2545,9 @@ done: * Therefore, take a dangling reference to prevent teardown. */ sr->sr_ref_count++; +#ifndef CONFIG_EMBEDDED shared_region_completed_slide = TRUE; +#endif } vm_shared_region_unlock(); diff --git a/osfmk/vm/vm_shared_region.h b/osfmk/vm/vm_shared_region.h index 87097ee39..34becaefb 100644 --- a/osfmk/vm/vm_shared_region.h +++ b/osfmk/vm/vm_shared_region.h @@ -172,6 +172,8 @@ struct vm_shared_region { mach_vm_size_t sr_pmap_nesting_size; thread_call_t sr_timer_call; struct vm_shared_region_slide_info sr_slide_info; + uuid_t sr_uuid; + boolean_t sr_uuid_copied; }; extern kern_return_t vm_shared_region_slide_page(vm_shared_region_slide_info_t si, diff --git a/osfmk/vm/vm_swapfile_pager.c b/osfmk/vm/vm_swapfile_pager.c index e50177fdb..f24307d74 100644 --- a/osfmk/vm/vm_swapfile_pager.c +++ b/osfmk/vm/vm_swapfile_pager.c @@ -64,12 +64,7 @@ * the disk blocks they were allocated. The "super-user" could see the * contents of free blocks anyway, so this is not a new security issue but * it may be perceive as one. - * * ENCRYPTED SWAP: - * When swap is encrypted, one does not expect to find any clear contents - * in the swap files. Since unused blocks are not scrubbed, they could still - * contain clear contents. If these contents are visible through a mapping - * of the swap file, it makes it look like swap is not really encrypted. - * + * * We can't legitimately prevent a user process with appropriate privileges * from mapping a swap file, but we can prevent it from accessing its actual * contents. @@ -139,17 +134,17 @@ const struct memory_object_pager_ops swapfile_pager_ops = { * the "swapfile" EMM. */ typedef struct swapfile_pager { - struct ipc_object_header pager_header; /* fake ip_kotype() */ - memory_object_pager_ops_t pager_ops; /* == &swapfile_pager_ops */ + /* mandatory generic header */ + struct memory_object swp_pgr_hdr; + + /* pager-specific data */ queue_chain_t pager_queue; /* next & prev pagers */ unsigned int ref_count; /* reference count */ boolean_t is_ready; /* is this pager ready ? */ boolean_t is_mapped; /* is this pager mapped ? */ - memory_object_control_t pager_control; /* mem object control handle */ struct vnode *swapfile_vnode;/* the swapfile's vnode */ } *swapfile_pager_t; #define SWAPFILE_PAGER_NULL ((swapfile_pager_t) NULL) -#define pager_ikot pager_header.io_bits /* * List of memory objects managed by this EMM. @@ -235,7 +230,7 @@ swapfile_pager_init( memory_object_control_reference(control); - pager->pager_control = control; + pager->swp_pgr_hdr.mo_control = control; attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; attributes.cluster_size = (1 << (PAGE_SHIFT)); @@ -343,7 +338,7 @@ swapfile_pager_data_request( /* * Gather in a UPL all the VM pages requested by VM. */ - mo_control = pager->pager_control; + mo_control = pager->swp_pgr_hdr.mo_control; upl_size = length; upl_flags = @@ -355,7 +350,7 @@ swapfile_pager_data_request( pl_count = 0; kr = memory_object_upl_request(mo_control, offset, upl_size, - &upl, NULL, NULL, upl_flags); + &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_OSFMK); if (kr != KERN_SUCCESS) { retval = kr; goto done; @@ -374,6 +369,8 @@ swapfile_pager_data_request( PAGE_SIZE_64, 0, 0, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, &map_entry); if (kr != KERN_SUCCESS) { vm_object_deallocate(kernel_object); @@ -408,13 +405,19 @@ swapfile_pager_data_request( dst_pnum = (ppnum_t) upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); assert(dst_pnum != 0); - pmap_enter(kernel_pmap, - kernel_mapping, - dst_pnum, - VM_PROT_READ | VM_PROT_WRITE, - VM_PROT_NONE, - 0, - TRUE); + retval = pmap_enter(kernel_pmap, + kernel_mapping, + dst_pnum, + VM_PROT_READ | VM_PROT_WRITE, + VM_PROT_NONE, + 0, + TRUE); + + assert(retval == KERN_SUCCESS); + + if (retval != KERN_SUCCESS) { + goto done; + } memset(dst_ptr, '\0', PAGE_SIZE); /* add an end-of-line to keep line counters happy */ @@ -542,7 +545,7 @@ swapfile_pager_terminate_internal( } /* trigger the destruction of the memory object */ - memory_object_destroy(pager->pager_control, 0); + memory_object_destroy(pager->swp_pgr_hdr.mo_control, 0); } /* @@ -582,9 +585,9 @@ swapfile_pager_deallocate_internal( * pager structure. */ lck_mtx_unlock(&swapfile_pager_lock); - if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) { - memory_object_control_deallocate(pager->pager_control); - pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; + if (pager->swp_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { + memory_object_control_deallocate(pager->swp_pgr_hdr.mo_control); + pager->swp_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; } kfree(pager, sizeof (*pager)); pager = SWAPFILE_PAGER_NULL; @@ -633,21 +636,13 @@ swapfile_pager_terminate( */ kern_return_t swapfile_pager_synchronize( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, + __unused memory_object_t mem_obbj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length, __unused vm_sync_t sync_flags) { - swapfile_pager_t pager; - - PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_synchronize: %p\n", mem_obj)); - - pager = swapfile_pager_lookup(mem_obj); - - memory_object_synchronize_completed(pager->pager_control, - offset, length); - - return KERN_SUCCESS; + panic("swapfile_pager_synchronize: memory_object_synchronize no longer supported\n"); + return (KERN_FAILURE); } /* @@ -728,8 +723,8 @@ swapfile_pager_lookup( { swapfile_pager_t pager; + assert(mem_obj->mo_pager_ops == &swapfile_pager_ops); __IGNORE_WCASTALIGN(pager = (swapfile_pager_t) mem_obj); - assert(pager->pager_ops == &swapfile_pager_ops); assert(pager->ref_count > 0); return pager; } @@ -754,12 +749,13 @@ swapfile_pager_create( * we reserve the second word in the object for a fake ip_kotype * setting - that will tell vm_map to use it as a memory object. */ - pager->pager_ops = &swapfile_pager_ops; - pager->pager_ikot = IKOT_MEMORY_OBJECT; + pager->swp_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; + pager->swp_pgr_hdr.mo_pager_ops = &swapfile_pager_ops; + pager->swp_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; + pager->is_ready = FALSE;/* not ready until it has a "name" */ pager->ref_count = 1; /* setup reference */ pager->is_mapped = FALSE; - pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; pager->swapfile_vnode = vp; lck_mtx_lock(&swapfile_pager_lock); @@ -872,7 +868,10 @@ swapfile_pager_control( { swapfile_pager_t pager; + if (mem_obj == MEMORY_OBJECT_NULL || + mem_obj->mo_pager_ops != &swapfile_pager_ops) { + return MEMORY_OBJECT_CONTROL_NULL; + } pager = swapfile_pager_lookup(mem_obj); - - return pager->pager_control; + return pager->swp_pgr_hdr.mo_control; } diff --git a/osfmk/vm/vm_user.c b/osfmk/vm/vm_user.c index 886dbb6ff..491d27fe9 100644 --- a/osfmk/vm/vm_user.c +++ b/osfmk/vm/vm_user.c @@ -118,26 +118,38 @@ #include #include +#include + vm_size_t upl_offset_to_pagelist = 0; #if VM_CPM #include #endif /* VM_CPM */ -lck_grp_t dynamic_pager_control_port_lock_group; -decl_lck_mtx_data(, dynamic_pager_control_port_lock); -ipc_port_t dynamic_pager_control_port=NULL; - /* * mach_vm_allocate allocates "zero fill" memory in the specfied * map. */ kern_return_t -mach_vm_allocate( +mach_vm_allocate_external( vm_map_t map, mach_vm_offset_t *addr, mach_vm_size_t size, int flags) +{ + vm_tag_t tag; + + VM_GET_FLAGS_ALIAS(flags, tag); + return (mach_vm_allocate_kernel(map, addr, size, flags, tag)); +} + +kern_return_t +mach_vm_allocate_kernel( + vm_map_t map, + mach_vm_offset_t *addr, + mach_vm_size_t size, + int flags, + vm_tag_t tag) { vm_map_offset_t map_addr; vm_map_size_t map_size; @@ -185,6 +197,8 @@ mach_vm_allocate( map_size, (vm_map_offset_t)0, flags, + VM_MAP_KERNEL_FLAGS_NONE, + tag, VM_OBJECT_NULL, (vm_object_offset_t)0, FALSE, @@ -202,11 +216,25 @@ mach_vm_allocate( * map (which is limited to the same size as the kernel). */ kern_return_t -vm_allocate( +vm_allocate_external( vm_map_t map, vm_offset_t *addr, vm_size_t size, int flags) +{ + vm_tag_t tag; + + VM_GET_FLAGS_ALIAS(flags, tag); + return (vm_allocate_kernel(map, addr, size, flags, tag)); +} + +kern_return_t +vm_allocate_kernel( + vm_map_t map, + vm_offset_t *addr, + vm_size_t size, + int flags, + vm_tag_t tag) { vm_map_offset_t map_addr; vm_map_size_t map_size; @@ -254,6 +282,8 @@ vm_allocate( map_size, (vm_map_offset_t)0, flags, + VM_MAP_KERNEL_FLAGS_NONE, + tag, VM_OBJECT_NULL, (vm_object_offset_t)0, FALSE, @@ -261,6 +291,12 @@ vm_allocate( VM_PROT_ALL, VM_INHERIT_DEFAULT); +#if KASAN + if (result == KERN_SUCCESS && map->pmap == kernel_pmap) { + kasan_notify_address(map_addr, map_size); + } +#endif + *addr = CAST_DOWN(vm_offset_t, map_addr); return(result); } @@ -899,7 +935,7 @@ vm_copy( * */ kern_return_t -mach_vm_map( +mach_vm_map_external( vm_map_t target_map, mach_vm_offset_t *address, mach_vm_size_t initial_size, @@ -911,6 +947,28 @@ mach_vm_map( vm_prot_t cur_protection, vm_prot_t max_protection, vm_inherit_t inheritance) +{ + vm_tag_t tag; + + VM_GET_FLAGS_ALIAS(flags, tag); + return (mach_vm_map_kernel(target_map, address, initial_size, mask, flags, tag, port, + offset, copy, cur_protection, max_protection, inheritance)); +} + +kern_return_t +mach_vm_map_kernel( + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t initial_size, + mach_vm_offset_t mask, + int flags, + vm_tag_t tag, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) { kern_return_t kr; vm_map_offset_t vmmaddr; @@ -922,16 +980,24 @@ mach_vm_map( return KERN_INVALID_ARGUMENT; kr = vm_map_enter_mem_object(target_map, - &vmmaddr, - initial_size, - mask, - flags, - port, - offset, - copy, - cur_protection, - max_protection, - inheritance); + &vmmaddr, + initial_size, + mask, + flags, + VM_MAP_KERNEL_FLAGS_NONE, + tag, + port, + offset, + copy, + cur_protection, + max_protection, + inheritance); + +#if KASAN + if (kr == KERN_SUCCESS && target_map->pmap == kernel_pmap) { + kasan_notify_address(vmmaddr, initial_size); + } +#endif *address = vmmaddr; return kr; @@ -940,7 +1006,7 @@ mach_vm_map( /* legacy interface */ kern_return_t -vm_map_64( +vm_map_64_external( vm_map_t target_map, vm_offset_t *address, vm_size_t size, @@ -952,6 +1018,28 @@ vm_map_64( vm_prot_t cur_protection, vm_prot_t max_protection, vm_inherit_t inheritance) +{ + vm_tag_t tag; + + VM_GET_FLAGS_ALIAS(flags, tag); + return (vm_map_64_kernel(target_map, address, size, mask, flags, tag, port, offset, + copy, cur_protection, max_protection, inheritance)); +} + +kern_return_t +vm_map_64_kernel( + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + vm_tag_t tag, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) { mach_vm_address_t map_addr; mach_vm_size_t map_size; @@ -962,7 +1050,7 @@ vm_map_64( map_size = (mach_vm_size_t)size; map_mask = (mach_vm_offset_t)mask; - kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags, + kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask, flags, tag, port, offset, copy, cur_protection, max_protection, inheritance); *address = CAST_DOWN(vm_offset_t, map_addr); @@ -971,12 +1059,33 @@ vm_map_64( /* temporary, until world build */ kern_return_t -vm_map( +vm_map_external( + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + ipc_port_t port, + vm_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) +{ + vm_tag_t tag; + + VM_GET_FLAGS_ALIAS(flags, tag); + return (vm_map_kernel(target_map, address, size, mask, flags, tag, port, offset, copy, cur_protection, max_protection, inheritance)); +} + +kern_return_t +vm_map_kernel( vm_map_t target_map, vm_offset_t *address, vm_size_t size, vm_offset_t mask, int flags, + vm_tag_t tag, ipc_port_t port, vm_offset_t offset, boolean_t copy, @@ -995,7 +1104,7 @@ vm_map( map_mask = (mach_vm_offset_t)mask; obj_offset = (vm_object_offset_t)offset; - kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags, + kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask, flags, tag, port, obj_offset, copy, cur_protection, max_protection, inheritance); *address = CAST_DOWN(vm_offset_t, map_addr); @@ -1009,14 +1118,35 @@ vm_map( * over top of itself (with altered permissions and/or * as an in-place copy of itself). */ +kern_return_t +mach_vm_remap_external( + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t size, + mach_vm_offset_t mask, + int flags, + vm_map_t src_map, + mach_vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance) +{ + vm_tag_t tag; + VM_GET_FLAGS_ALIAS(flags, tag); + + return (mach_vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map, memory_address, + copy, cur_protection, max_protection, inheritance)); +} kern_return_t -mach_vm_remap( +mach_vm_remap_kernel( vm_map_t target_map, mach_vm_offset_t *address, mach_vm_size_t size, mach_vm_offset_t mask, int flags, + vm_tag_t tag, vm_map_t src_map, mach_vm_offset_t memory_address, boolean_t copy, @@ -1041,6 +1171,8 @@ mach_vm_remap( size, mask, flags, + VM_MAP_KERNEL_FLAGS_NONE, + tag, src_map, memory_address, copy, @@ -1063,7 +1195,7 @@ mach_vm_remap( * kernel context). */ kern_return_t -vm_remap( +vm_remap_external( vm_map_t target_map, vm_offset_t *address, vm_size_t size, @@ -1075,6 +1207,28 @@ vm_remap( vm_prot_t *cur_protection, vm_prot_t *max_protection, vm_inherit_t inheritance) +{ + vm_tag_t tag; + VM_GET_FLAGS_ALIAS(flags, tag); + + return (vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map, + memory_address, copy, cur_protection, max_protection, inheritance)); +} + +kern_return_t +vm_remap_kernel( + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + vm_tag_t tag, + vm_map_t src_map, + vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance) { vm_map_offset_t map_addr; kern_return_t kr; @@ -1093,6 +1247,8 @@ vm_remap( size, mask, flags, + VM_MAP_KERNEL_FLAGS_NONE, + tag, src_map, memory_address, copy, @@ -1117,12 +1273,24 @@ vm_remap( * [ To unwire the pages, specify VM_PROT_NONE. ] */ kern_return_t -mach_vm_wire( +mach_vm_wire_external( host_priv_t host_priv, vm_map_t map, mach_vm_offset_t start, mach_vm_size_t size, vm_prot_t access) +{ + return (mach_vm_wire_kernel(host_priv, map, start, size, access, VM_KERN_MEMORY_MLOCK)); +} + +kern_return_t +mach_vm_wire_kernel( + host_priv_t host_priv, + vm_map_t map, + mach_vm_offset_t start, + mach_vm_size_t size, + vm_prot_t access, + vm_tag_t tag) { kern_return_t rc; @@ -1138,12 +1306,12 @@ mach_vm_wire( return KERN_INVALID_ARGUMENT; if (access != VM_PROT_NONE) { - rc = vm_map_wire(map, + rc = vm_map_wire_kernel(map, vm_map_trunc_page(start, VM_MAP_PAGE_MASK(map)), vm_map_round_page(start+size, VM_MAP_PAGE_MASK(map)), - access | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK), + access, tag, TRUE); } else { rc = vm_map_unwire(map, @@ -1188,12 +1356,12 @@ vm_wire( if (size == 0) { rc = KERN_SUCCESS; } else if (access != VM_PROT_NONE) { - rc = vm_map_wire(map, + rc = vm_map_wire_kernel(map, vm_map_trunc_page(start, VM_MAP_PAGE_MASK(map)), vm_map_round_page(start+size, VM_MAP_PAGE_MASK(map)), - access | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK), + access, VM_KERN_MEMORY_OSFMK, TRUE); } else { rc = vm_map_unwire(map, @@ -1681,6 +1849,11 @@ mach_vm_purgable_control( if (VM_MAP_NULL == map) return KERN_INVALID_ARGUMENT; + if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) { + /* not allowed from user-space */ + return KERN_INVALID_ARGUMENT; + } + return vm_map_purgable_control(map, vm_map_trunc_page(address, PAGE_MASK), control, @@ -1697,6 +1870,11 @@ vm_purgable_control( if (VM_MAP_NULL == map) return KERN_INVALID_ARGUMENT; + if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) { + /* not allowed from user-space */ + return KERN_INVALID_ARGUMENT; + } + return vm_map_purgable_control(map, vm_map_trunc_page(address, PAGE_MASK), control, @@ -1785,6 +1963,128 @@ vm_map_page_query( disposition, ref_count); } +kern_return_t +mach_vm_page_range_query( + vm_map_t map, + mach_vm_offset_t address, + mach_vm_size_t size, + mach_vm_address_t dispositions_addr, + mach_vm_size_t *dispositions_count) +{ + kern_return_t kr = KERN_SUCCESS; + int num_pages = 0, i = 0; + mach_vm_size_t curr_sz = 0, copy_sz = 0; + mach_vm_size_t disp_buf_req_size = 0, disp_buf_total_size = 0; + mach_msg_type_number_t count = 0; + + void *info = NULL; + void *local_disp = NULL;; + vm_map_size_t info_size = 0, local_disp_size = 0; + mach_vm_offset_t start = 0, end = 0; + + if (map == VM_MAP_NULL || dispositions_count == NULL) { + return KERN_INVALID_ARGUMENT; + } + + disp_buf_req_size = ( *dispositions_count * sizeof(int)); + start = mach_vm_trunc_page(address); + end = mach_vm_round_page(address + size); + + if (end < start) { + return KERN_INVALID_ARGUMENT; + } + + if (disp_buf_req_size == 0 || (end == start)) { + return KERN_SUCCESS; + } + + /* + * For large requests, we will go through them + * MAX_PAGE_RANGE_QUERY chunk at a time. + */ + + curr_sz = MIN(end - start, MAX_PAGE_RANGE_QUERY); + num_pages = (int) (curr_sz >> PAGE_SHIFT); + + info_size = num_pages * sizeof(vm_page_info_basic_data_t); + info = kalloc(info_size); + + if (info == NULL) { + return KERN_RESOURCE_SHORTAGE; + } + + local_disp_size = num_pages * sizeof(int); + local_disp = kalloc(local_disp_size); + + if (local_disp == NULL) { + + kfree(info, info_size); + info = NULL; + return KERN_RESOURCE_SHORTAGE; + } + + while (size) { + + count = VM_PAGE_INFO_BASIC_COUNT; + kr = vm_map_page_range_info_internal( + map, + start, + mach_vm_round_page(start + curr_sz), + VM_PAGE_INFO_BASIC, + (vm_page_info_t) info, + &count); + + assert(kr == KERN_SUCCESS); + + for (i = 0; i < num_pages; i++) { + + ((int*)local_disp)[i] = ((vm_page_info_basic_t)info)[i].disposition; + } + + copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int)/* an int per page */); + kr = copyout(local_disp, (mach_vm_address_t)dispositions_addr, copy_sz); + + start += curr_sz; + disp_buf_req_size -= copy_sz; + disp_buf_total_size += copy_sz; + + if (kr != 0) { + break; + } + + if ((disp_buf_req_size == 0) || (curr_sz >= size)) { + + /* + * We might have inspected the full range OR + * more than it esp. if the user passed in + * non-page aligned start/size and/or if we + * descended into a submap. We are done here. + */ + + size = 0; + + } else { + + dispositions_addr += copy_sz; + + size -= curr_sz; + + curr_sz = MIN(mach_vm_round_page(size), MAX_PAGE_RANGE_QUERY); + num_pages = (int)(curr_sz >> PAGE_SHIFT); + } + } + + *dispositions_count = disp_buf_total_size / sizeof(int); + + kfree(local_disp, local_disp_size); + local_disp = NULL; + + kfree(info, info_size); + info = NULL; + + return kr; +} + kern_return_t mach_vm_page_info( vm_map_t map, @@ -1843,6 +2143,7 @@ vm_map_get_upl( upl_page_info_array_t page_list, unsigned int *count, upl_control_flags_t *flags, + vm_tag_t tag, int force_data_sync) { upl_control_flags_t map_flags; @@ -1861,12 +2162,19 @@ vm_map_get_upl( upl, page_list, count, - &map_flags); + &map_flags, + tag); *flags = (map_flags & ~UPL_FORCE_DATA_SYNC); return kr; } +#if CONFIG_EMBEDDED +extern int proc_selfpid(void); +extern char *proc_name_address(void *p); +int cs_executable_mem_entry = 0; +int log_executable_mem_entry = 0; +#endif /* CONFIG_EMBEDDED */ /* * mach_make_memory_entry_64 @@ -1876,7 +2184,6 @@ vm_map_get_upl( * somewhere else. Rather than doing it all at once (and * without needing access to the other whole map). */ - kern_return_t mach_make_memory_entry_64( vm_map_t target_map, @@ -1927,16 +2234,7 @@ mach_make_memory_entry_64( boolean_t use_data_addr; boolean_t use_4K_compat; - if (((permission & 0x00FF0000) & - ~(MAP_MEM_ONLY | - MAP_MEM_NAMED_CREATE | - MAP_MEM_GRAB_SECLUDED | /* XXX FBDP TODO: restrict usage? */ - MAP_MEM_PURGABLE | - MAP_MEM_NAMED_REUSE | - MAP_MEM_USE_DATA_ADDR | - MAP_MEM_VM_COPY | - MAP_MEM_4K_DATA_ADDR | - MAP_MEM_VM_SHARE))) { + if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_USER) { /* * Unknown flag: reject for forward compatibility. */ @@ -1976,8 +2274,7 @@ mach_make_memory_entry_64( return KERN_INVALID_ARGUMENT; } - parent_is_object = !(parent_entry->is_sub_map || - parent_entry->is_pager); + parent_is_object = !parent_entry->is_sub_map; object = parent_entry->backing.object; if(parent_is_object && object != VM_OBJECT_NULL) wimg_mode = object->wimg_bits; @@ -1987,22 +2284,9 @@ mach_make_memory_entry_64( !(parent_entry->protection & VM_PROT_WRITE)) { return KERN_INVALID_RIGHT; } - if(access == MAP_MEM_IO) { - SET_MAP_MEM(access, parent_entry->protection); - wimg_mode = VM_WIMG_IO; - } else if (access == MAP_MEM_COPYBACK) { - SET_MAP_MEM(access, parent_entry->protection); - wimg_mode = VM_WIMG_USE_DEFAULT; - } else if (access == MAP_MEM_INNERWBACK) { - SET_MAP_MEM(access, parent_entry->protection); - wimg_mode = VM_WIMG_INNERWBACK; - } else if (access == MAP_MEM_WTHRU) { - SET_MAP_MEM(access, parent_entry->protection); - wimg_mode = VM_WIMG_WTHRU; - } else if (access == MAP_MEM_WCOMB) { - SET_MAP_MEM(access, parent_entry->protection); - wimg_mode = VM_WIMG_WCOMB; - } + vm_prot_to_wimg(access, &wimg_mode); + if (access != MAP_MEM_NOOP) + SET_MAP_MEM(access, parent_entry->protection); if (parent_is_object && object && (access != MAP_MEM_NOOP) && (!(object->nophyscache))) { @@ -2053,12 +2337,20 @@ mach_make_memory_entry_64( goto make_mem_done; } object->purgable = VM_PURGABLE_NONVOLATILE; + if (permission & MAP_MEM_PURGABLE_KERNEL_ONLY) { + object->purgeable_only_by_kernel = TRUE; + } assert(object->vo_purgeable_owner == NULL); assert(object->resident_page_count == 0); assert(object->wired_page_count == 0); vm_object_lock(object); - vm_purgeable_nonvolatile_enqueue(object, - current_task()); + if (object->purgeable_only_by_kernel) { + vm_purgeable_nonvolatile_enqueue(object, + kernel_task); + } else { + vm_purgeable_nonvolatile_enqueue(object, + current_task()); + } vm_object_unlock(object); } @@ -2087,20 +2379,11 @@ mach_make_memory_entry_64( */ wimg_mode = object->wimg_bits; - if (access == MAP_MEM_IO) { - wimg_mode = VM_WIMG_IO; - } else if (access == MAP_MEM_COPYBACK) { - wimg_mode = VM_WIMG_USE_DEFAULT; - } else if (access == MAP_MEM_INNERWBACK) { - wimg_mode = VM_WIMG_INNERWBACK; - } else if (access == MAP_MEM_WTHRU) { - wimg_mode = VM_WIMG_WTHRU; - } else if (access == MAP_MEM_WCOMB) { - wimg_mode = VM_WIMG_WCOMB; - } - if (access != MAP_MEM_NOOP) { - object->wimg_bits = wimg_mode; - } + vm_prot_to_wimg(access, &wimg_mode); + if (access != MAP_MEM_NOOP) { + object->wimg_bits = wimg_mode; + } + /* the object has no pages, so no WIMG bits to update here */ /* @@ -2118,7 +2401,6 @@ mach_make_memory_entry_64( user_entry->backing.object = object; user_entry->internal = TRUE; user_entry->is_sub_map = FALSE; - user_entry->is_pager = FALSE; user_entry->offset = 0; user_entry->data_offset = 0; user_entry->protection = protections; @@ -2169,7 +2451,6 @@ mach_make_memory_entry_64( user_entry->backing.copy = copy; user_entry->internal = FALSE; user_entry->is_sub_map = FALSE; - user_entry->is_pager = FALSE; user_entry->is_copy = TRUE; user_entry->offset = 0; user_entry->protection = protections; @@ -2242,7 +2523,6 @@ mach_make_memory_entry_64( user_entry->backing.copy = copy; user_entry->internal = FALSE; user_entry->is_sub_map = FALSE; - user_entry->is_pager = FALSE; user_entry->is_copy = TRUE; user_entry->offset = 0; user_entry->protection = protections; @@ -2301,6 +2581,50 @@ redo_lookup: */ protections &= prot; } +#if CONFIG_EMBEDDED + /* + * Wiring would copy the pages to a shadow object. + * The shadow object would not be code-signed so + * attempting to execute code from these copied pages + * would trigger a code-signing violation. + */ + if (prot & VM_PROT_EXECUTE) { + if (log_executable_mem_entry) { + void *bsd_info; + bsd_info = current_task()->bsd_info; + printf("pid %d[%s] making memory entry out of " + "executable range from 0x%llx to 0x%llx:" + "might cause code-signing issues " + "later\n", + proc_selfpid(), + (bsd_info != NULL + ? proc_name_address(bsd_info) + : "?"), + (uint64_t) map_start, + (uint64_t) map_end); + } + DTRACE_VM2(cs_executable_mem_entry, + uint64_t, (uint64_t)map_start, + uint64_t, (uint64_t)map_end); + cs_executable_mem_entry++; + +#if 11 + /* + * We don't know how the memory entry will be used. + * It might never get wired and might not cause any + * trouble, so let's not reject this request... + */ +#else /* 11 */ + kr = KERN_PROTECTION_FAILURE; + vm_object_unlock(object); + vm_map_unlock_read(target_map); + if(real_map != target_map) + vm_map_unlock_read(real_map); + goto make_mem_done; +#endif /* 11 */ + + } +#endif /* CONFIG_EMBEDDED */ if (((prot & protections) != protections) || (object == kernel_object)) { @@ -2625,19 +2949,8 @@ redo_lookup: /* against delayed copy, etc. is mostly defensive. */ wimg_mode = object->wimg_bits; - if(!(object->nophyscache)) { - if(access == MAP_MEM_IO) { - wimg_mode = VM_WIMG_IO; - } else if (access == MAP_MEM_COPYBACK) { - wimg_mode = VM_WIMG_USE_DEFAULT; - } else if (access == MAP_MEM_INNERWBACK) { - wimg_mode = VM_WIMG_INNERWBACK; - } else if (access == MAP_MEM_WTHRU) { - wimg_mode = VM_WIMG_WTHRU; - } else if (access == MAP_MEM_WCOMB) { - wimg_mode = VM_WIMG_WCOMB; - } - } + if(!(object->nophyscache)) + vm_prot_to_wimg(access, &wimg_mode); #if VM_OBJECT_TRACKING_OP_TRUESHARE if (!object->true_share && @@ -2693,7 +3006,6 @@ redo_lookup: parent_entry->backing.object == object && parent_entry->internal == object->internal && parent_entry->is_sub_map == FALSE && - parent_entry->is_pager == FALSE && parent_entry->offset == obj_off && parent_entry->protection == protections && parent_entry->size == map_size && @@ -2735,7 +3047,6 @@ redo_lookup: user_entry->backing.object = object; user_entry->internal = object->internal; user_entry->is_sub_map = FALSE; - user_entry->is_pager = FALSE; user_entry->offset = obj_off; user_entry->data_offset = offset_in_page; user_entry->protection = protections; @@ -2762,8 +3073,8 @@ redo_lookup: * submaps and pagers should only be accessible from within * the kernel, which shouldn't use the data address flag, so can fail here. */ - if (parent_entry->is_pager || parent_entry->is_sub_map) { - panic("Shouldn't be using data address with a parent entry that is a submap or pager."); + if (parent_entry->is_sub_map) { + panic("Shouldn't be using data address with a parent entry that is a submap."); } /* * Account for offset to data in parent entry and @@ -2814,7 +3125,6 @@ redo_lookup: user_entry->offset = parent_entry->offset + map_start; user_entry->data_offset = offset_in_page; user_entry->is_sub_map = parent_entry->is_sub_map; - user_entry->is_pager = parent_entry->is_pager; user_entry->is_copy = parent_entry->is_copy; user_entry->internal = parent_entry->internal; user_entry->protection = protections; @@ -2828,10 +3138,6 @@ redo_lookup: vm_map_lock(user_entry->backing.map); user_entry->backing.map->ref_count++; vm_map_unlock(user_entry->backing.map); - } - else if (parent_entry->is_pager) { - user_entry->backing.pager = parent_entry->backing.pager; - /* JMM - don't we need a reference here? */ } else { object = parent_entry->backing.object; assert(object != VM_OBJECT_NULL); @@ -2980,9 +3286,8 @@ mach_memory_entry_allocate( ipc_port_nsrequest(user_handle, 1, user_handle, &previous); /* nsrequest unlocks user_handle */ - user_entry->backing.pager = NULL; + user_entry->backing.object = NULL; user_entry->is_sub_map = FALSE; - user_entry->is_pager = FALSE; user_entry->is_copy = FALSE; user_entry->internal = FALSE; user_entry->size = 0; @@ -3005,8 +3310,6 @@ mach_memory_entry_allocate( * * Create a named entry backed by the provided pager. * - * JMM - we need to hold a reference on the pager - - * and release it when the named entry is destroyed. */ kern_return_t mach_memory_object_memory_entry_64( @@ -3020,29 +3323,44 @@ mach_memory_object_memory_entry_64( unsigned int access; vm_named_entry_t user_entry; ipc_port_t user_handle; + vm_object_t object; if (host == HOST_NULL) return(KERN_INVALID_HOST); + if (pager == MEMORY_OBJECT_NULL && internal) { + object = vm_object_allocate(size); + } else { + object = memory_object_to_vm_object(pager); + if (object != VM_OBJECT_NULL) { + vm_object_reference(object); + } + } + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } + if (mach_memory_entry_allocate(&user_entry, &user_handle) != KERN_SUCCESS) { + vm_object_deallocate(object); return KERN_FAILURE; } - user_entry->backing.pager = pager; user_entry->size = size; user_entry->offset = 0; user_entry->protection = permission & VM_PROT_ALL; access = GET_MAP_MEM(permission); SET_MAP_MEM(access, user_entry->protection); - user_entry->internal = internal; user_entry->is_sub_map = FALSE; - user_entry->is_pager = TRUE; assert(user_entry->ref_count == 1); + user_entry->backing.object = object; + user_entry->internal = object->internal; + assert(object->internal == internal); + *entry_handle = user_handle; return KERN_SUCCESS; -} +} kern_return_t mach_memory_object_memory_entry( @@ -3063,6 +3381,20 @@ mach_memory_entry_purgable_control( ipc_port_t entry_port, vm_purgable_t control, int *state) +{ + if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) { + /* not allowed from user-space */ + return KERN_INVALID_ARGUMENT; + } + + return memory_entry_purgeable_control_internal(entry_port, control, state); +} + +kern_return_t +memory_entry_purgeable_control_internal( + ipc_port_t entry_port, + vm_purgable_t control, + int *state) { kern_return_t kr; vm_named_entry_t mem_entry; @@ -3073,10 +3405,12 @@ mach_memory_entry_purgable_control( return KERN_INVALID_ARGUMENT; } if (control != VM_PURGABLE_SET_STATE && - control != VM_PURGABLE_GET_STATE) + control != VM_PURGABLE_GET_STATE && + control != VM_PURGABLE_SET_STATE_FROM_KERNEL) return(KERN_INVALID_ARGUMENT); - if (control == VM_PURGABLE_SET_STATE && + if ((control == VM_PURGABLE_SET_STATE || + control == VM_PURGABLE_SET_STATE_FROM_KERNEL) && (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) || ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK))) return(KERN_INVALID_ARGUMENT); @@ -3086,7 +3420,6 @@ mach_memory_entry_purgable_control( named_entry_lock(mem_entry); if (mem_entry->is_sub_map || - mem_entry->is_pager || mem_entry->is_copy) { named_entry_unlock(mem_entry); return KERN_INVALID_ARGUMENT; @@ -3138,7 +3471,6 @@ mach_memory_entry_get_page_counts( named_entry_lock(mem_entry); if (mem_entry->is_sub_map || - mem_entry->is_pager || mem_entry->is_copy) { named_entry_unlock(mem_entry); return KERN_INVALID_ARGUMENT; @@ -3207,8 +3539,6 @@ mach_destroy_memory_entry( if(named_entry->ref_count == 0) { if (named_entry->is_sub_map) { vm_map_deallocate(named_entry->backing.map); - } else if (named_entry->is_pager) { - /* JMM - need to drop reference on pager in that case */ } else if (named_entry->is_copy) { vm_map_copy_discard(named_entry->backing.copy); } else { @@ -3250,7 +3580,6 @@ mach_memory_entry_page_op( named_entry_lock(mem_entry); if (mem_entry->is_sub_map || - mem_entry->is_pager || mem_entry->is_copy) { named_entry_unlock(mem_entry); return KERN_INVALID_ARGUMENT; @@ -3304,7 +3633,6 @@ mach_memory_entry_range_op( named_entry_lock(mem_entry); if (mem_entry->is_sub_map || - mem_entry->is_pager || mem_entry->is_copy) { named_entry_unlock(mem_entry); return KERN_INVALID_ARGUMENT; @@ -3330,49 +3658,6 @@ mach_memory_entry_range_op( return kr; } -static void dp_control_port_init(void) -{ - lck_grp_init(&dynamic_pager_control_port_lock_group,"dp_control_port", LCK_GRP_ATTR_NULL); - lck_mtx_init(&dynamic_pager_control_port_lock, &dynamic_pager_control_port_lock_group, LCK_ATTR_NULL); -} - -kern_return_t -set_dp_control_port( - host_priv_t host_priv, - ipc_port_t control_port) -{ - ipc_port_t old_port; - - if (host_priv == HOST_PRIV_NULL) - return (KERN_INVALID_HOST); - - lck_mtx_lock(&dynamic_pager_control_port_lock); - old_port = dynamic_pager_control_port; - dynamic_pager_control_port = control_port; - lck_mtx_unlock(&dynamic_pager_control_port_lock); - - if (IP_VALID(old_port)) - ipc_port_release_send(old_port); - - return KERN_SUCCESS; -} - -kern_return_t -get_dp_control_port( - host_priv_t host_priv, - ipc_port_t *control_port) -{ - if (host_priv == HOST_PRIV_NULL) - return (KERN_INVALID_HOST); - - lck_mtx_lock(&dynamic_pager_control_port_lock); - *control_port = ipc_port_copy_send(dynamic_pager_control_port); - lck_mtx_unlock(&dynamic_pager_control_port_lock); - - return KERN_SUCCESS; - -} - /* ******* Temporary Internal calls to UPL for BSD ***** */ extern int kernel_upl_map( @@ -3586,7 +3871,8 @@ vm_map_get_phys_page( /* need to call vm_fault */ vm_map_unlock(map); vm_fault(map, map_offset, VM_PROT_NONE, - FALSE, THREAD_UNINT, NULL, 0); + FALSE /* change_wiring */, VM_KERN_MEMORY_NONE, + THREAD_UNINT, NULL, 0); vm_map_lock(map); continue; } @@ -3629,12 +3915,6 @@ vm_map_get_phys_page( return phys_page; } -void -vm_user_init(void) -{ - dp_control_port_init(); -} - #if 0 kern_return_t kernel_object_iopl_request( /* forward */ vm_named_entry_t named_entry, @@ -3703,48 +3983,15 @@ kernel_object_iopl_request( named_entry_lock(named_entry); - if (named_entry->is_pager) { - object = vm_object_enter(named_entry->backing.pager, - named_entry->offset + named_entry->size, - named_entry->internal, - FALSE, - FALSE); - if (object == VM_OBJECT_NULL) { - named_entry_unlock(named_entry); - return(KERN_INVALID_OBJECT); - } - - /* JMM - drop reference on the pager here? */ - - /* create an extra reference for the object */ - vm_object_lock(object); - vm_object_reference_locked(object); - named_entry->backing.object = object; - named_entry->is_pager = FALSE; - named_entry_unlock(named_entry); - - /* wait for object (if any) to be ready */ - if (!named_entry->internal) { - while (!object->pager_ready) { - vm_object_wait(object, - VM_OBJECT_EVENT_PAGER_READY, - THREAD_UNINT); - vm_object_lock(object); - } - } - vm_object_unlock(object); - - } else { - /* This is the case where we are going to operate */ - /* an an already known object. If the object is */ - /* not ready it is internal. An external */ - /* object cannot be mapped until it is ready */ - /* we can therefore avoid the ready check */ - /* in this case. */ - object = named_entry->backing.object; - vm_object_reference(object); - named_entry_unlock(named_entry); - } + /* This is the case where we are going to operate */ + /* on an already known object. If the object is */ + /* not ready it is internal. An external */ + /* object cannot be mapped until it is ready */ + /* we can therefore avoid the ready check */ + /* in this case. */ + object = named_entry->backing.object; + vm_object_reference(object); + named_entry_unlock(named_entry); if (!object->private) { if (*upl_size > MAX_UPL_TRANSFER_BYTES) @@ -3769,3 +4016,110 @@ kernel_object_iopl_request( return ret; } #endif + +/* + * These symbols are looked up at runtime by vmware, VirtualBox, + * despite not being exported in the symbol sets. + */ + +#if defined(__x86_64__) + +kern_return_t +mach_vm_map( + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t initial_size, + mach_vm_offset_t mask, + int flags, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance); + +kern_return_t +mach_vm_remap( + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t size, + mach_vm_offset_t mask, + int flags, + vm_map_t src_map, + mach_vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance); + +kern_return_t +mach_vm_map( + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t initial_size, + mach_vm_offset_t mask, + int flags, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) +{ + return (mach_vm_map_external(target_map, address, initial_size, mask, flags, port, + offset, copy, cur_protection, max_protection, inheritance)); +} + +kern_return_t +mach_vm_remap( + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t size, + mach_vm_offset_t mask, + int flags, + vm_map_t src_map, + mach_vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance) +{ + return (mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address, + copy, cur_protection, max_protection, inheritance)); +} + +kern_return_t +vm_map( + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + ipc_port_t port, + vm_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance); + +kern_return_t +vm_map( + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + ipc_port_t port, + vm_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) +{ + vm_tag_t tag; + + VM_GET_FLAGS_ALIAS(flags, tag); + return (vm_map_kernel(target_map, address, size, mask, flags, tag, port, offset, copy, cur_protection, max_protection, inheritance)); +} + +#endif /* __x86_64__ */ diff --git a/osfmk/x86_64/Makefile b/osfmk/x86_64/Makefile index d83d1669e..91dc2251e 100644 --- a/osfmk/x86_64/Makefile +++ b/osfmk/x86_64/Makefile @@ -6,9 +6,19 @@ export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir include $(MakeInc_cmd) include $(MakeInc_def) +EXPORT_FILES = + + +EXPORT_ONLY_FILES = \ + machine_kpc.h \ + monotonic.h + +INSTALL_KF_MD_LCL_LIST = ${EXPORT_FILES} + +INSTALL_MD_DIR = x86_64 + +EXPORT_MD_LIST = ${EXPORT_ONLY_FILES} ${EXPORT_FILES} -EXPORT_ONLY_FILES = machine_kpc.h -EXPORT_MD_LIST = ${EXPORT_ONLY_FILES} EXPORT_MD_DIR = x86_64 include $(MakeInc_rule) diff --git a/osfmk/x86_64/bzero.s b/osfmk/x86_64/bzero.s index be490d18c..ccaf05f1c 100644 --- a/osfmk/x86_64/bzero.s +++ b/osfmk/x86_64/bzero.s @@ -58,6 +58,13 @@ #include +/* + * void *secure_memset(void * addr, int pattern, size_t length) + * + * It is important that this function remains defined in assembly to avoid + * compiler optimizations. + */ +ENTRY(secure_memset) /* * void *memset(void * addr, int pattern, size_t length) */ diff --git a/osfmk/x86_64/copyio.c b/osfmk/x86_64/copyio.c index 5ce1c3b73..c86d1136c 100644 --- a/osfmk/x86_64/copyio.c +++ b/osfmk/x86_64/copyio.c @@ -38,9 +38,14 @@ #include #include #include +#include #include +#include + + + static int copyio(int, user_addr_t, char *, vm_size_t, vm_size_t *, int); static int copyio_phys(addr64_t, addr64_t, vm_size_t, int); @@ -52,7 +57,7 @@ static int copyio_phys(addr64_t, addr64_t, vm_size_t, int); * user and wired kernel memory in a single invocation on this * platform. */ -#define COPYSIZELIMIT_PANIC (64*MB) +const int copysize_limit_panic = (64 * MB); /* * The copy engine has the following characteristics @@ -163,7 +168,9 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, #endif boolean_t nopagezero = thread->map->pmap->pagezero_accessible; - assert(nbytes < COPYSIZELIMIT_PANIC); + if (__improbable(nbytes > copysize_limit_panic)) + panic("%s(%p, %p, %lu) - transfer too large", __func__, + (void *)user_addr, (void *)kernel_addr, nbytes); COPYIO_TRACE(debug_type | DBG_FUNC_START, user_addr, kernel_addr, nbytes, use_kernel_map, 0); @@ -185,6 +192,14 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, goto out; } +#if KASAN + if (copy_type == COPYIN || copy_type == COPYINSTR || copy_type == COPYINWORD) { + __asan_storeN((uptr)kernel_addr, nbytes); + } else if (copy_type == COPYOUT) { + __asan_loadN((uptr)kernel_addr, nbytes); + } +#endif + /* * If the no_shared_cr3 boot-arg is set (true), the kernel runs on * its own pmap and cr3 rather than the user's -- so that wild accesses @@ -344,6 +359,7 @@ copyio_phys(addr64_t source, addr64_t sink, vm_size_t csize, int which) paddr = (char *)source; vaddr = (user_addr_t)sink; ctype = COPYOUTPHYS; + CALL_COPYOUT_SHIM_PHYS((void *)PHYSMAP_PTOV(source),sink,csize) } return copyio(ctype, vaddr, paddr, csize, NULL, which & cppvKmap); } @@ -389,12 +405,14 @@ copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_ int copyoutmsg(const char *kernel_addr, user_addr_t user_addr, mach_msg_size_t nbytes) { + CALL_COPYOUT_SHIM_MSG(kernel_addr,user_addr,(vm_size_t)nbytes) return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0); } int copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes) { + CALL_COPYOUT_SHIM_NRML(kernel_addr,user_addr,nbytes) return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0); } diff --git a/osfmk/x86_64/idt64.s b/osfmk/x86_64/idt64.s index 78b07486b..7c42b1dee 100644 --- a/osfmk/x86_64/idt64.s +++ b/osfmk/x86_64/idt64.s @@ -330,6 +330,8 @@ L_common_dispatch: mov %rcx, %cr3 4: mov %gs:CPU_ACTIVE_THREAD, %rcx /* Get the active thread */ + testq %rcx, %rcx + je 5f movl $-1, TH_IOTIER_OVERRIDE(%rcx) /* Reset IO tier override to -1 before handling trap */ cmpq $0, TH_PCB_IDS(%rcx) /* Is there a debug register state? */ je 5f @@ -1087,11 +1089,9 @@ L_return_from_trap_with_ast: movl %eax, R64_RBX(%r15) /* let the PFZ know we've pended an AST */ jmp EXT(return_to_user) 2: - sti /* interrupts always enabled on return to user mode */ - xor %edi, %edi /* zero %rdi */ xorq %rbp, %rbp /* clear framepointer */ - CCALL(i386_astintr) /* take the AST */ + CCALL(ast_taken_user) /* handle all ASTs (enables interrupts, may return via continuation) */ cli mov %rsp, %r15 /* AST changes stack, saved state */ @@ -1131,7 +1131,7 @@ trap_from_kernel: testq %rcx,%rcx /* are we on the kernel stack? */ jne ret_to_kernel /* no, skip it */ - CCALL1(i386_astintr, $1) /* take the AST */ + CCALL(ast_taken_kernel) /* take the AST */ mov %rsp, %r15 /* AST changes stack, saved state */ jmp ret_to_kernel @@ -1242,7 +1242,7 @@ LEXT(return_to_iret) /* (label for kdb_kintr and hardclock) */ * to do as much as the case where the interrupt came from user * space. */ - CCALL1(i386_astintr, $1) + CCALL(ast_taken_kernel) mov %rsp, %r15 /* AST changes stack, saved state */ jmp ret_to_kernel diff --git a/osfmk/x86_64/kpc_x86.c b/osfmk/x86_64/kpc_x86.c index 82fd80e54..4da5d9809 100644 --- a/osfmk/x86_64/kpc_x86.c +++ b/osfmk/x86_64/kpc_x86.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include @@ -602,8 +601,6 @@ kpc_set_running_arch(struct kpc_running_remote *mp_config) { assert(mp_config); - lapic_set_pmi_func((i386_intr_func_t)kpc_pmi_handler); - /* dispatch to all CPUs */ mp_cpus_call(CPUMASK_ALL, ASYNC, kpc_set_running_mp_call, mp_config); diff --git a/osfmk/x86_64/locore.s b/osfmk/x86_64/locore.s index 88c3372aa..0638e5162 100644 --- a/osfmk/x86_64/locore.s +++ b/osfmk/x86_64/locore.s @@ -53,8 +53,8 @@ * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ + #include -#include #include #include @@ -62,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -352,3 +353,12 @@ L_copyin_word_fail: RECOVERY_SECTION RECOVER_TABLE_END + +/* + * Vector here on any exception at startup prior to switching to + * the kernel's idle page-tables and installing the kernel master IDT. + */ +Entry(vstart_trap_handler) + POSTCODE(BOOT_TRAP_HLT) + hlt + diff --git a/osfmk/x86_64/loose_ends.c b/osfmk/x86_64/loose_ends.c index 0736226fb..07c07327d 100644 --- a/osfmk/x86_64/loose_ends.c +++ b/osfmk/x86_64/loose_ends.c @@ -95,6 +95,9 @@ #endif +/* prevent infinite recursion when memmove calls bcopy; in string.h, bcopy is defined to call memmove */ +#undef bcopy + /* XXX - should be gone from here */ extern void invalidate_icache64(addr64_t addr, unsigned cnt, int phys); extern void flush_dcache64(addr64_t addr, unsigned count, int phys); @@ -167,6 +170,40 @@ ffs(unsigned int mask) return 1 + __builtin_ctz(mask); } +int +ffsll(unsigned long long mask) +{ + if (mask == 0) + return 0; + + /* + * NOTE: cannot use __builtin_ffsll because it generates a call to + * 'ffsll' + */ + return 1 + __builtin_ctzll(mask); +} + +/* + * Find last bit set in bit string. + */ +int +fls(unsigned int mask) +{ + if (mask == 0) + return 0; + + return (sizeof (mask) << 3) - __builtin_clz(mask); +} + +int +flsll(unsigned long long mask) +{ + if (mask == 0) + return 0; + + return (sizeof (mask) << 3) - __builtin_clzll(mask); +} + void bzero_phys_nc( addr64_t src64, @@ -253,6 +290,7 @@ ovbcopy( uint64_t reportphyreaddelayabs; uint32_t reportphyreadosbt; + #if DEVELOPMENT || DEBUG uint32_t phyreadpanic = 1; #else @@ -264,8 +302,8 @@ ml_phys_read_data(pmap_paddr_t paddr, int size) { uint64_t result = 0; unsigned char s1; unsigned short s2; - boolean_t istate, timeread = FALSE; - uint64_t sabs, eabs; + boolean_t istate = TRUE, timeread = FALSE; + uint64_t sabs = 0, eabs; if (__improbable(!physmap_enclosed(paddr))) panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr); @@ -301,7 +339,7 @@ ml_phys_read_data(pmap_paddr_t paddr, int size) { (void)ml_set_interrupts_enabled(istate); if (__improbable((eabs - sabs) > reportphyreaddelayabs)) { - if (phyreadpanic) { + if (phyreadpanic && (machine_timeout_suspended() == FALSE)) { panic_io_port_read(); panic("Read from physical addr 0x%llx took %llu ns, result: 0x%llx (start: %llu, end: %llu), ceiling: %llu", paddr, (eabs - sabs), result, sabs, eabs, reportphyreaddelayabs); } @@ -498,6 +536,7 @@ ml_probe_read_64(addr64_t paddr64, unsigned int *val) } +#undef bcmp int bcmp( const void *pa, const void *pb, @@ -517,6 +556,7 @@ int bcmp( return (int)len; } +#undef memcmp int memcmp(const void *s1, const void *s2, size_t n) { @@ -531,6 +571,7 @@ memcmp(const void *s1, const void *s2, size_t n) return (0); } +#undef memmove void * memmove(void *dst, const void *src, size_t ulen) { @@ -544,6 +585,7 @@ memmove(void *dst, const void *src, size_t ulen) * the terminating null character. */ +#undef strlen size_t strlen( const char *string) diff --git a/osfmk/x86_64/lz4_decode_x86_64.s b/osfmk/x86_64/lz4_decode_x86_64.s index ae0c69324..953aaff9a 100644 --- a/osfmk/x86_64/lz4_decode_x86_64.s +++ b/osfmk/x86_64/lz4_decode_x86_64.s @@ -1,3 +1,30 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ #include #if LZ4_ENABLE_ASSEMBLY_DECODE_X86_64 diff --git a/osfmk/x86_64/monotonic.h b/osfmk/x86_64/monotonic.h new file mode 100644 index 000000000..cf0d9cc5a --- /dev/null +++ b/osfmk/x86_64/monotonic.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef X86_64_MONOTONIC_H +#define X86_64_MONOTONIC_H + +#include + +#define MT_NDEVS 1 + +#define MT_CORE_NFIXED 3 + +#define MT_CORE_INSTRS 0 +#define MT_CORE_CYCLES 1 +#define MT_CORE_REFCYCLES 2 +#define MT_CORE_MAXVAL ((UINT64_C(1) << 48) - 1) + +#endif /* !defined(X86_64_MONOTONIC_H) */ diff --git a/osfmk/x86_64/monotonic_x86_64.c b/osfmk/x86_64/monotonic_x86_64.c new file mode 100644 index 000000000..12c5c4b76 --- /dev/null +++ b/osfmk/x86_64/monotonic_x86_64.c @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include /* static_assert, assert */ +#include +#include +#include +#include + +/* + * Sanity check the compiler. + */ + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif /* !defined(__has_builtin) */ +#if !__has_builtin(__builtin_ia32_rdpmc) +#error requires __builtin_ia32_rdpmc builtin +#endif /* !__has_builtin(__builtin_ia32_rdpmc) */ + +#pragma mark core counters + +bool mt_core_supported = false; + +/* + * PMC[0-2]_{RD,WR} allow reading and writing the fixed PMCs. + * + * There are separate defines for access type because the read side goes through + * the rdpmc instruction, which has a different counter encoding than the msr + * path. + */ +#define PMC_FIXED_RD(CTR) ((UINT64_C(1) << 30) | (CTR)) +#define PMC_FIXED_WR(CTR) (MSR_IA32_PERF_FIXED_CTR0 + (CTR)) +#define PMC0_RD PMC_FIXED_RD(0) +#define PMC0_WR PMC_FIXED_WR(0) +#define PMC1_RD PMC_FIXED_RD(1) +#define PMC1_WR PMC_FIXED_WR(1) +#define PMC2_RD PMC_FIXED_RD(2) +#define PMC2_WR PMC_FIXED_WR(2) + +struct mt_cpu * +mt_cur_cpu(void) +{ + return ¤t_cpu_datap()->cpu_monotonic; +} + +uint64_t +mt_core_snap(unsigned int ctr) +{ + if (!mt_core_supported) { + return 0; + } + + switch (ctr) { + case 0: + return __builtin_ia32_rdpmc(PMC0_RD); + case 1: + return __builtin_ia32_rdpmc(PMC1_RD); + case 2: + return __builtin_ia32_rdpmc(PMC2_RD); + default: + panic("monotonic: invalid core counter read: %u", ctr); + __builtin_trap(); + } +} + +void +mt_core_set_snap(unsigned int ctr, uint64_t count) +{ + if (!mt_core_supported) { + return; + } + + switch (ctr) { + case 0: + wrmsr64(PMC0_WR, count); + break; + case 1: + wrmsr64(PMC1_WR, count); + break; + case 2: + wrmsr64(PMC2_WR, count); + break; + default: + panic("monotonic: invalid core counter write: %u", ctr); + __builtin_trap(); + } +} + +/* + * FIXED_CTR_CTRL controls which rings fixed counters are enabled in and if they + * deliver PMIs. + * + * Each fixed counters has 4 bits: [0:1] controls which ring it's enabled in, + * [2] counts all hardware threads in each logical core (we don't want this), + * and [3] enables PMIs on overflow. + */ + +#define FIXED_CTR_CTRL 0x38d + +/* + * Fixed counters are enabled in all rings, so hard-code this register state to + * enable in all rings and deliver PMIs. + */ +#define FIXED_CTR_CTRL_INIT (0x888 | 0x333) + +/* + * GLOBAL_CTRL controls which counters are enabled -- the high 32-bits control + * the fixed counters and the lower half is for the configurable counters. + */ + +#define GLOBAL_CTRL 0x38f + +/* + * Fixed counters are always enabled -- and there are three of them. + */ +#define GLOBAL_CTRL_FIXED_EN (((UINT64_C(1) << 3) - 1) << 32) + +/* + * GLOBAL_STATUS reports the state of counters, like those that have overflowed. + */ +#define GLOBAL_STATUS 0x38e + +#define CTR_MAX ((UINT64_C(1) << 48) - 1) +#define CTR_FIX_POS(CTR) ((UINT64_C(1) << (CTR)) << 32) + +#define GLOBAL_OVF 0x390 + +static void +core_down(cpu_data_t *cpu) +{ + if (!mt_core_supported) { + return; + } + + assert(ml_get_interrupts_enabled() == FALSE); + + wrmsr64(GLOBAL_CTRL, 0); + mt_mtc_update_fixed_counts(&cpu->cpu_monotonic, NULL, NULL); +} + +static void +core_up(cpu_data_t *cpu) +{ + struct mt_cpu *mtc; + + if (!mt_core_supported) { + return; + } + + assert(ml_get_interrupts_enabled() == FALSE); + + mtc = &cpu->cpu_monotonic; + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + mt_core_set_snap(i, mtc->mtc_snaps[i]); + } + wrmsr64(FIXED_CTR_CTRL, FIXED_CTR_CTRL_INIT); + wrmsr64(GLOBAL_CTRL, GLOBAL_CTRL_FIXED_EN); +} + +void +mt_cpu_down(cpu_data_t *cpu) +{ + core_down(cpu); +} + +void +mt_cpu_up(cpu_data_t *cpu) +{ + boolean_t intrs_en; + intrs_en = ml_set_interrupts_enabled(FALSE); + core_up(cpu); + ml_set_interrupts_enabled(intrs_en); +} + +static int +mt_pmi_x86_64(x86_saved_state_t *state) +{ + uint64_t status; + struct mt_cpu *mtc; + bool fixed_ovf = false; + + assert(ml_get_interrupts_enabled() == FALSE); + mtc = mt_cur_cpu(); + status = rdmsr64(GLOBAL_STATUS); + + (void)atomic_fetch_add_explicit(&mt_pmis, 1, memory_order_relaxed); + + for (int i = 0; i < MT_CORE_NFIXED; i++) { + if (status & CTR_FIX_POS(i)) { + fixed_ovf = true; + uint64_t prior; + + prior = CTR_MAX - mtc->mtc_snaps[i]; + assert(prior <= CTR_MAX); + prior += 1; /* wrapped */ + + mtc->mtc_counts[i] += prior; + mtc->mtc_snaps[i] = 0; + mt_mtc_update_count(mtc, i); + } + } + + /* if any of the configurable counters overflowed, tell kpc */ + if (status & ((UINT64_C(1) << 4) - 1)) { + extern void kpc_pmi_handler(x86_saved_state_t *state); + kpc_pmi_handler(state); + } + return 0; +} + +void +mt_init(void) +{ + uint32_t cpuinfo[4]; + + do_cpuid(0xA, cpuinfo); + + if ((cpuinfo[0] & 0xff) >= 2) { + lapic_set_pmi_func((i386_intr_func_t)mt_pmi_x86_64); + mt_core_supported = true; + } +} + +static int +core_init(void) +{ + return ENOTSUP; +} + +#pragma mark common hooks + +const struct monotonic_dev monotonic_devs[] = { + [0] = { + .mtd_name = "monotonic/core", + .mtd_init = core_init + } +}; + +static_assert( + (sizeof(monotonic_devs) / sizeof(monotonic_devs[0])) == MT_NDEVS, + "MT_NDEVS macro should be same as the length of monotonic_devs"); diff --git a/osfmk/x86_64/pmap.c b/osfmk/x86_64/pmap.c index cb6438168..9b2a9675e 100644 --- a/osfmk/x86_64/pmap.c +++ b/osfmk/x86_64/pmap.c @@ -138,6 +138,7 @@ #endif #include +#include #include #include @@ -282,12 +283,21 @@ pmap_map( vm_prot_t prot, unsigned int flags) { + kern_return_t kr; int ps; ps = PAGE_SIZE; while (start_addr < end_addr) { - pmap_enter(kernel_pmap, (vm_map_offset_t)virt, - (ppnum_t) i386_btop(start_addr), prot, VM_PROT_NONE, flags, TRUE); + kr = pmap_enter(kernel_pmap, (vm_map_offset_t)virt, + (ppnum_t) i386_btop(start_addr), prot, VM_PROT_NONE, flags, TRUE); + + if (kr != KERN_SUCCESS) { + panic("%s: failed pmap_enter, " + "virt=%p, start_addr=%p, end_addr=%p, prot=%#x, flags=%#x", + __FUNCTION__, + (void *)virt, (void *)start_addr, (void *)end_addr, prot, flags); + } + virt += ps; start_addr += ps; } @@ -354,10 +364,12 @@ pmap_cpu_init(void) } } +#if !MONOTONIC if (cdp->cpu_fixed_pmcs_enabled) { boolean_t enable = TRUE; cpu_pmc_control(&enable); } +#endif /* !MONOTONIC */ } static uint32_t pmap_scale_shift(void) { @@ -489,6 +501,7 @@ pmap_bootstrap( printf("Stack canary: 0x%lx\n", __stack_chk_guard[0]); printf("early_random(): 0x%qx\n", early_random()); #endif +#if DEVELOPMENT || DEBUG boolean_t ptmp; /* Check if the user has requested disabling stack or heap no-execute * enforcement. These are "const" variables; that qualifier is cast away @@ -505,6 +518,7 @@ pmap_bootstrap( boolean_t *pdknhp = (boolean_t *) &pmap_disable_kstack_nx; *pdknhp = TRUE; } +#endif /* DEVELOPMENT || DEBUG */ boot_args *args = (boot_args *)PE_state.bootArgs; if (args->efiMode == kBootArgsEfiMode32) { @@ -521,17 +535,17 @@ pmap_bootstrap( * in the DEBUG kernel) to force the kernel to switch to its own map * (and cr3) when control is in kernelspace. The kernel's map does not * include (i.e. share) userspace so wild references will cause - * a panic. Only copyin and copyout are exempt from this. + * a panic. Only copyin and copyout are exempt from this. */ (void) PE_parse_boot_argn("-no_shared_cr3", &no_shared_cr3, sizeof (no_shared_cr3)); if (no_shared_cr3) kprintf("Kernel not sharing user map\n"); - + #ifdef PMAP_TRACES if (PE_parse_boot_argn("-pmap_trace", &pmap_trace, sizeof (pmap_trace))) { kprintf("Kernel traces for pmap operations enabled\n"); - } + } #endif /* PMAP_TRACES */ #if MACH_ASSERT @@ -799,6 +813,7 @@ pmap_init(void) pv_hashed_list_zone = zinit(s, 10000*s /* Expandable zone */, 4096 * 3 /* LCM x86_64*/, "pv_list"); zone_change(pv_hashed_list_zone, Z_NOENCRYPT, TRUE); + zone_change(pv_hashed_list_zone, Z_GZALLOC_EXEMPT, TRUE); /* create pv entries for kernel pages mapped by low level startup code. these have to exist so we can pmap_remove() @@ -1284,8 +1299,7 @@ pmap_create_options( pml4_entry_t *pml4; pml4_entry_t *kpml4; - PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, - (uint32_t) (sz>>32), (uint32_t) sz, flags, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, sz, flags); size = (vm_size_t) sz; @@ -1367,6 +1381,11 @@ pmap_create_options( pml4[KERNEL_PML4_INDEX] = kpml4[KERNEL_PML4_INDEX]; pml4[KERNEL_KEXTS_INDEX] = kpml4[KERNEL_KEXTS_INDEX]; pml4[KERNEL_PHYSMAP_PML4_INDEX] = kpml4[KERNEL_PHYSMAP_PML4_INDEX]; + +#if KASAN + pml4[KERNEL_KASAN_PML4_INDEX0] = kpml4[KERNEL_KASAN_PML4_INDEX0]; + pml4[KERNEL_KASAN_PML4_INDEX1] = kpml4[KERNEL_KASAN_PML4_INDEX1]; +#endif } #if MACH_ASSERT @@ -1374,8 +1393,8 @@ pmap_create_options( strlcpy(p->pmap_procname, "", sizeof (p->pmap_procname)); #endif /* MACH_ASSERT */ - PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, - p, flags, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_END, + VM_KERNEL_ADDRHIDE(p)); return(p); } @@ -1502,7 +1521,7 @@ pmap_destroy(pmap_t p) return; PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START, - p, 0, 0, 0, 0); + VM_KERNEL_ADDRHIDe(p)); PMAP_LOCK(p); @@ -1525,8 +1544,7 @@ pmap_destroy(pmap_t p) PMAP_UNLOCK(p); if (c != 0) { - PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END, - p, 1, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END); pmap_assert(p == kernel_pmap); return; /* still in use */ } @@ -1555,8 +1573,7 @@ pmap_destroy(pmap_t p) ledger_dereference(p->ledger); zfree(pmap_zone, p); - PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END, - 0, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END); } /* @@ -1630,10 +1647,10 @@ pmap_protect_options( pmap_remove_options(map, sva, eva, options); return; } + PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START, - map, - (uint32_t) (sva >> 32), (uint32_t) sva, - (uint32_t) (eva >> 32), (uint32_t) eva); + VM_KERNEL_ADDRHIDE(map), VM_KERNEL_ADDRHIDE(sva), + VM_KERNEL_ADDRHIDE(eva)); if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !map->nx_enabled) set_NX = FALSE; @@ -1701,15 +1718,14 @@ pmap_protect_options( } PMAP_UNLOCK(map); - PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END, - 0, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END); } /* Map a (possibly) autogenned block */ -void +kern_return_t pmap_map_block( - pmap_t pmap, + pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, @@ -1717,19 +1733,38 @@ pmap_map_block( int attr, __unused unsigned int flags) { + kern_return_t kr; + addr64_t original_va = va; uint32_t page; int cur_page_size; if (attr & VM_MEM_SUPERPAGE) cur_page_size = SUPERPAGE_SIZE; - else + else cur_page_size = PAGE_SIZE; for (page = 0; page < size; page+=cur_page_size/PAGE_SIZE) { - pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE); + kr = pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE); + + if (kr != KERN_SUCCESS) { + /* + * This will panic for now, as it is unclear that + * removing the mappings is correct. + */ + panic("%s: failed pmap_enter, " + "pmap=%p, va=%#llx, pa=%u, size=%u, prot=%#x, flags=%#x", + __FUNCTION__, + pmap, va, pa, size, prot, flags); + + pmap_remove(pmap, original_va, va - original_va); + return kr; + } + va += cur_page_size; pa+=cur_page_size/PAGE_SIZE; } + + return KERN_SUCCESS; } kern_return_t @@ -2439,7 +2474,7 @@ pmap_flush( cpus_to_signal = pfc->pfc_cpus; PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_DELAYED_TLBS) | DBG_FUNC_START, - NULL, cpus_to_signal, 0, 0, 0); + NULL, cpus_to_signal); for (cpu = 0, cpu_bit = 1; cpu < real_ncpus && cpus_to_signal; cpu++, cpu_bit <<= 1) { @@ -2480,7 +2515,7 @@ pmap_flush( deadline = mach_absolute_time() + (TLBTimeOut ? TLBTimeOut : LockTimeOut); boolean_t is_timeout_traced = FALSE; - + /* * Wait for those other cpus to acknowledge */ @@ -2508,23 +2543,23 @@ pmap_flush( if (TLBTimeOut == 0) { if (is_timeout_traced) continue; + PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_TLBS_TO), - NULL, cpus_to_signal, cpus_to_respond, 0, 0); + NULL, cpus_to_signal, cpus_to_respond); + is_timeout_traced = TRUE; continue; } - pmap_tlb_flush_timeout = TRUE; orig_acks = NMIPI_acks; - mp_cpus_NMIPI(cpus_to_respond); - - panic("TLB invalidation IPI timeout: " - "CPU(s) failed to respond to interrupts, unresponsive CPU bitmap: 0x%llx, NMIPI acks: orig: 0x%lx, now: 0x%lx", - cpus_to_respond, orig_acks, NMIPI_acks); + NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT); + panic("TLB invalidation IPI timeout, unresponsive CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu", + cpus_to_respond, orig_acks, NMIPI_acks, deadline); } } } + PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_DELAYED_TLBS) | DBG_FUNC_END, - NULL, cpus_signaled, flush_self, 0, 0); + NULL, cpus_signaled, flush_self); mp_enable_preemption(); } @@ -2587,7 +2622,8 @@ pmap_flush_tlbs(pmap_t pmap, vm_map_offset_t startv, vm_map_offset_t endv, int o } PMAP_TRACE_CONSTANT(event_code | DBG_FUNC_START, - VM_KERNEL_UNSLIDE_OR_PERM(pmap), options, event_startv, event_endv, 0); + VM_KERNEL_UNSLIDE_OR_PERM(pmap), options, + event_startv, event_endv); if (is_ept) { mp_cpus_call(CPUMASK_ALL, ASYNC, invept, (void*)pmap->pm_eptp); @@ -2709,19 +2745,20 @@ pmap_flush_tlbs(pmap_t pmap, vm_map_offset_t startv, vm_map_offset_t endv, int o /* cut tracepoint but don't panic */ if (is_timeout_traced) continue; - PMAP_TRACE_CONSTANT( - PMAP_CODE(PMAP__FLUSH_TLBS_TO), - VM_KERNEL_UNSLIDE_OR_PERM(pmap), cpus_to_signal, cpus_to_respond, 0, 0); + + PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_TLBS_TO), + VM_KERNEL_UNSLIDE_OR_PERM(pmap), + cpus_to_signal, + cpus_to_respond); + is_timeout_traced = TRUE; continue; } - pmap_tlb_flush_timeout = TRUE; orig_acks = NMIPI_acks; - mp_cpus_NMIPI(cpus_to_respond); - panic("TLB invalidation IPI timeout: " - "CPU(s) failed to respond to interrupts, unresponsive CPU bitmap: 0x%llx, NMIPI acks: orig: 0x%lx, now: 0x%lx", - cpus_to_respond, orig_acks, NMIPI_acks); + NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT); + panic("TLB invalidation IPI timeout, unresponsive CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu", + cpus_to_respond, orig_acks, NMIPI_acks, deadline); } } } @@ -2732,7 +2769,8 @@ pmap_flush_tlbs(pmap_t pmap, vm_map_offset_t startv, vm_map_offset_t endv, int o out: PMAP_TRACE_CONSTANT(event_code | DBG_FUNC_END, - VM_KERNEL_UNSLIDE_OR_PERM(pmap), cpus_to_signal, event_startv, event_endv, 0); + VM_KERNEL_UNSLIDE_OR_PERM(pmap), cpus_to_signal, + event_startv, event_endv); } @@ -2763,14 +2801,12 @@ process_pmap_updates(void) void pmap_update_interrupt(void) { - PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_START, - 0, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_START); if (current_cpu_datap()->cpu_tlb_invalid) process_pmap_updates(); - PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_END, - 0, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_END); } #include /* mach_vm_region_recurse() */ @@ -3251,3 +3287,11 @@ void pmap_verify_noncacheable(uintptr_t vaddr) { return; panic("pmap_verify_noncacheable: IO read from a cacheable address? address: 0x%lx, PTE: %p, *PTE: 0x%llx", vaddr, ptep, *ptep); } + +#if KASAN +void kasan_map_low_fixed_regions(void) { + kasan_map_shadow(MASTER_GDT_ALIAS, PAGE_SIZE, false); + kasan_map_shadow(MASTER_IDT_ALIAS, PAGE_SIZE, false); + kasan_map_shadow(LOWGLOBAL_ALIAS, PAGE_SIZE, false); +} +#endif diff --git a/pexpert/arm/pe_bootargs.c b/pexpert/arm/pe_bootargs.c new file mode 100644 index 000000000..c6b87b081 --- /dev/null +++ b/pexpert/arm/pe_bootargs.c @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +#include +#include + +char * +PE_boot_args( + void) +{ + return (char *)((boot_args *)PE_state.bootArgs)->CommandLine; +} diff --git a/pexpert/arm/pe_consistent_debug.c b/pexpert/arm/pe_consistent_debug.c new file mode 100644 index 000000000..103a1bb90 --- /dev/null +++ b/pexpert/arm/pe_consistent_debug.c @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2011-2013 Apple Inc. All rights reserved. + * + * This document is the property of Apple Inc. + * It is considered confidential and proprietary. + * + * This document may not be reproduced or transmitted in any form, + * in whole or in part, without the express written permission of + * Apple Inc. + */ + +#include +#include +#include +#include +#include + +static dbg_registry_t * consistent_debug_registry = NULL; + +static dbg_record_header_t* consistent_debug_allocate_entry(void) { + unsigned int i; + + if (!consistent_debug_registry) + return NULL; + for (i = 0; i < consistent_debug_registry->top_level_header.num_records; i++) { + dbg_record_header_t *record = &consistent_debug_registry->records[i]; + if (OSCompareAndSwap64(kDbgIdUnusedEntry, kDbgIdReservedEntry, &record->record_id)) { + // Reserved an entry at position i. + return (dbg_record_header_t*)record; + } + } + return NULL; +} + +int PE_consistent_debug_inherit(void) +{ + DTEntry entryP; + uintptr_t *prop_data; + uintptr_t root_pointer = 0; + uint32_t size; + + if ((DTLookupEntry(NULL, "/chosen", &entryP) == kSuccess)) + if (DTGetProperty(entryP, "consistent-debug-root", (void **)&prop_data, &size) == kSuccess) + root_pointer = prop_data[0]; + if (root_pointer == 0) + return -1; + consistent_debug_registry = (dbg_registry_t *)ml_map_high_window(root_pointer, sizeof(dbg_registry_t)); + return 0; +} + +int PE_consistent_debug_register(uint64_t record_id, uint64_t physaddr, uint64_t length) +{ + dbg_record_header_t *allocated_header = consistent_debug_allocate_entry(); + if (allocated_header == NULL) + return -1; + allocated_header->length = length; + allocated_header->physaddr = physaddr; + // Make sure the hdr/length are visible before the record_id. + __asm__ volatile("dmb ish" : : : "memory"); + allocated_header->record_id = record_id; + return 0; +} + +int PE_consistent_debug_enabled(void) +{ + return (consistent_debug_registry != NULL); +} + diff --git a/pexpert/arm/pe_identify_machine.c b/pexpert/arm/pe_identify_machine.c new file mode 100644 index 000000000..63eb8929f --- /dev/null +++ b/pexpert/arm/pe_identify_machine.c @@ -0,0 +1,670 @@ +/* + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. + * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + */ +#include +#include +#include +#include + +#if defined(__arm__) +#include +#elif defined(__arm64__) +#include +#endif + +#include +#if DEVELOPMENT || DEBUG +#include +#include +#endif +/* Local declarations */ +void pe_identify_machine(boot_args * bootArgs); + +/* External declarations */ +extern void clean_mmu_dcache(void); + +static char *gPESoCDeviceType; +static char gPESoCDeviceTypeBuffer[SOC_DEVICE_TYPE_BUFFER_SIZE]; +static vm_offset_t gPESoCBasePhys; + +static uint32_t gTCFG0Value; + +static uint32_t pe_arm_init_timer(void *args); + +#if DEVELOPMENT || DEBUG +decl_simple_lock_data(, panic_trace_lock;) +#endif +/* + * pe_identify_machine: + * + * Sets up platform parameters. Returns: nothing + */ +void +pe_identify_machine(boot_args * bootArgs) +{ + OpaqueDTEntryIterator iter; + DTEntry cpus, cpu; + uint32_t mclk = 0, hclk = 0, pclk = 0, tclk = 0, use_dt = 0; + unsigned long *value; + unsigned int size; + int err; + + (void)bootArgs; + + if (pe_arm_get_soc_base_phys() == 0) + return; + + /* Clear the gPEClockFrequencyInfo struct */ + bzero((void *)&gPEClockFrequencyInfo, sizeof(clock_frequency_info_t)); + + if (!strcmp(gPESoCDeviceType, "s3c2410-io")) { + mclk = 192 << 23; + hclk = mclk / 2; + pclk = hclk / 2; + tclk = (1 << (23 + 2)) / 10; + tclk = pclk / tclk; + + gTCFG0Value = tclk - 1; + + tclk = pclk / (4 * tclk); /* Calculate the "actual" + * Timer0 frequency in fixed + * point. */ + + mclk = (mclk >> 17) * (125 * 125); + hclk = (hclk >> 17) * (125 * 125); + pclk = (pclk >> 17) * (125 * 125); + tclk = (((((tclk * 125) + 2) >> 2) * 125) + (1 << 14)) >> 15; + + } else if (!strcmp(gPESoCDeviceType, "integratorcp-io")) { + mclk = 200000000; + hclk = mclk / 2; + pclk = hclk / 2; + tclk = 100000; + } else if (!strcmp(gPESoCDeviceType, "olocreek-io")) { + mclk = 1000000000; + hclk = mclk / 8; + pclk = hclk / 2; + tclk = pclk; + } else if (!strcmp(gPESoCDeviceType, "omap3430sdp-io")) { + mclk = 332000000; + hclk = 19200000; + pclk = hclk; + tclk = pclk; + } else if (!strcmp(gPESoCDeviceType, "s5i3000-io")) { + mclk = 400000000; + hclk = mclk / 4; + pclk = hclk / 2; + tclk = 100000; /* timer is at 100khz */ + + } else + use_dt = 1; + + if (use_dt) { + /* Start with default values. */ + gPEClockFrequencyInfo.timebase_frequency_hz = 24000000; + gPEClockFrequencyInfo.bus_clock_rate_hz = 100000000; + gPEClockFrequencyInfo.cpu_clock_rate_hz = 400000000; + + err = DTLookupEntry(NULL, "/cpus", &cpus); + assert(err == kSuccess); + + err = DTInitEntryIterator(cpus, &iter); + assert(err == kSuccess); + + while (kSuccess == DTIterateEntries(&iter, &cpu)) { + if ((kSuccess != DTGetProperty(cpu, "state", (void **)&value, &size)) || + (strncmp((char*)value, "running", size) != 0)) + continue; + + /* Find the time base frequency first. */ + if (DTGetProperty(cpu, "timebase-frequency", (void **)&value, &size) == kSuccess) { + /* + * timebase_frequency_hz is only 32 bits, and + * the device tree should never provide 64 + * bits so this if should never be taken. + */ + if (size == 8) + gPEClockFrequencyInfo.timebase_frequency_hz = *(unsigned long long *)value; + else + gPEClockFrequencyInfo.timebase_frequency_hz = *value; + } + gPEClockFrequencyInfo.dec_clock_rate_hz = gPEClockFrequencyInfo.timebase_frequency_hz; + + /* Find the bus frequency next. */ + if (DTGetProperty(cpu, "bus-frequency", (void **)&value, &size) == kSuccess) { + if (size == 8) + gPEClockFrequencyInfo.bus_frequency_hz = *(unsigned long long *)value; + else + gPEClockFrequencyInfo.bus_frequency_hz = *value; + } + gPEClockFrequencyInfo.bus_frequency_min_hz = gPEClockFrequencyInfo.bus_frequency_hz; + gPEClockFrequencyInfo.bus_frequency_max_hz = gPEClockFrequencyInfo.bus_frequency_hz; + + if (gPEClockFrequencyInfo.bus_frequency_hz < 0x100000000ULL) + gPEClockFrequencyInfo.bus_clock_rate_hz = gPEClockFrequencyInfo.bus_frequency_hz; + else + gPEClockFrequencyInfo.bus_clock_rate_hz = 0xFFFFFFFF; + + /* Find the memory frequency next. */ + if (DTGetProperty(cpu, "memory-frequency", (void **)&value, &size) == kSuccess) { + if (size == 8) + gPEClockFrequencyInfo.mem_frequency_hz = *(unsigned long long *)value; + else + gPEClockFrequencyInfo.mem_frequency_hz = *value; + } + gPEClockFrequencyInfo.mem_frequency_min_hz = gPEClockFrequencyInfo.mem_frequency_hz; + gPEClockFrequencyInfo.mem_frequency_max_hz = gPEClockFrequencyInfo.mem_frequency_hz; + + /* Find the peripheral frequency next. */ + if (DTGetProperty(cpu, "peripheral-frequency", (void **)&value, &size) == kSuccess) { + if (size == 8) + gPEClockFrequencyInfo.prf_frequency_hz = *(unsigned long long *)value; + else + gPEClockFrequencyInfo.prf_frequency_hz = *value; + } + gPEClockFrequencyInfo.prf_frequency_min_hz = gPEClockFrequencyInfo.prf_frequency_hz; + gPEClockFrequencyInfo.prf_frequency_max_hz = gPEClockFrequencyInfo.prf_frequency_hz; + + /* Find the fixed frequency next. */ + if (DTGetProperty(cpu, "fixed-frequency", (void **)&value, &size) == kSuccess) { + if (size == 8) + gPEClockFrequencyInfo.fix_frequency_hz = *(unsigned long long *)value; + else + gPEClockFrequencyInfo.fix_frequency_hz = *value; + } + /* Find the cpu frequency last. */ + if (DTGetProperty(cpu, "clock-frequency", (void **)&value, &size) == kSuccess) { + if (size == 8) + gPEClockFrequencyInfo.cpu_frequency_hz = *(unsigned long long *)value; + else + gPEClockFrequencyInfo.cpu_frequency_hz = *value; + } + gPEClockFrequencyInfo.cpu_frequency_min_hz = gPEClockFrequencyInfo.cpu_frequency_hz; + gPEClockFrequencyInfo.cpu_frequency_max_hz = gPEClockFrequencyInfo.cpu_frequency_hz; + + if (gPEClockFrequencyInfo.cpu_frequency_hz < 0x100000000ULL) + gPEClockFrequencyInfo.cpu_clock_rate_hz = gPEClockFrequencyInfo.cpu_frequency_hz; + else + gPEClockFrequencyInfo.cpu_clock_rate_hz = 0xFFFFFFFF; + } + } else { + /* Use the canned values. */ + gPEClockFrequencyInfo.timebase_frequency_hz = tclk; + gPEClockFrequencyInfo.fix_frequency_hz = tclk; + gPEClockFrequencyInfo.bus_frequency_hz = hclk; + gPEClockFrequencyInfo.cpu_frequency_hz = mclk; + gPEClockFrequencyInfo.prf_frequency_hz = pclk; + + gPEClockFrequencyInfo.bus_frequency_min_hz = gPEClockFrequencyInfo.bus_frequency_hz; + gPEClockFrequencyInfo.bus_frequency_max_hz = gPEClockFrequencyInfo.bus_frequency_hz; + gPEClockFrequencyInfo.cpu_frequency_min_hz = gPEClockFrequencyInfo.cpu_frequency_hz; + gPEClockFrequencyInfo.cpu_frequency_max_hz = gPEClockFrequencyInfo.cpu_frequency_hz; + gPEClockFrequencyInfo.prf_frequency_min_hz = gPEClockFrequencyInfo.prf_frequency_hz; + gPEClockFrequencyInfo.prf_frequency_max_hz = gPEClockFrequencyInfo.prf_frequency_hz; + + gPEClockFrequencyInfo.dec_clock_rate_hz = gPEClockFrequencyInfo.timebase_frequency_hz; + gPEClockFrequencyInfo.bus_clock_rate_hz = gPEClockFrequencyInfo.bus_frequency_hz; + gPEClockFrequencyInfo.cpu_clock_rate_hz = gPEClockFrequencyInfo.cpu_frequency_hz; + } + + /* Set the num / den pairs form the hz values. */ + gPEClockFrequencyInfo.bus_clock_rate_num = gPEClockFrequencyInfo.bus_clock_rate_hz; + gPEClockFrequencyInfo.bus_clock_rate_den = 1; + + gPEClockFrequencyInfo.bus_to_cpu_rate_num = + (2 * gPEClockFrequencyInfo.cpu_clock_rate_hz) / gPEClockFrequencyInfo.bus_clock_rate_hz; + gPEClockFrequencyInfo.bus_to_cpu_rate_den = 2; + + gPEClockFrequencyInfo.bus_to_dec_rate_num = 1; + gPEClockFrequencyInfo.bus_to_dec_rate_den = + gPEClockFrequencyInfo.bus_clock_rate_hz / gPEClockFrequencyInfo.dec_clock_rate_hz; +} + +vm_offset_t +pe_arm_get_soc_base_phys(void) +{ + DTEntry entryP; + uintptr_t *ranges_prop; + uint32_t prop_size; + char *tmpStr; + + if (DTFindEntry("name", "arm-io", &entryP) == kSuccess) { + if (gPESoCDeviceType == 0) { + DTGetProperty(entryP, "device_type", (void **)&tmpStr, &prop_size); + strlcpy(gPESoCDeviceTypeBuffer, tmpStr, SOC_DEVICE_TYPE_BUFFER_SIZE); + gPESoCDeviceType = gPESoCDeviceTypeBuffer; + + DTGetProperty(entryP, "ranges", (void **)&ranges_prop, &prop_size); + gPESoCBasePhys = *(ranges_prop + 1); + } + return gPESoCBasePhys; + } + return 0; +} + +uint32_t +pe_arm_get_soc_revision(void) +{ + DTEntry entryP; + uint32_t *value; + uint32_t size; + + if ((DTFindEntry("name", "arm-io", &entryP) == kSuccess) + && (DTGetProperty(entryP, "chip-revision", (void **)&value, &size) == kSuccess)) { + if (size == 8) + return((uint32_t)*(unsigned long long *)value); + else + return(*value); + } + return 0; +} + + +extern void fleh_fiq_generic(void); + +#if defined(ARM_BOARD_CLASS_S5L8960X) +static struct tbd_ops s5l8960x_funcs = {NULL, NULL, NULL}; +#endif /* defined(ARM_BOARD_CLASS_S5L8960X) */ + +#if defined(ARM_BOARD_CLASS_T7000) +static struct tbd_ops t7000_funcs = {NULL, NULL, NULL}; +#endif /* defined(ARM_BOARD_CLASS_T7000) */ + +#if defined(ARM_BOARD_CLASS_S7002) +extern void fleh_fiq_s7002(void); +extern uint32_t s7002_get_decrementer(void); +extern void s7002_set_decrementer(uint32_t); +static struct tbd_ops s7002_funcs = {&fleh_fiq_s7002, &s7002_get_decrementer, &s7002_set_decrementer}; +#endif /* defined(ARM_BOARD_CLASS_S7002) */ + +#if defined(ARM_BOARD_CLASS_S8000) +static struct tbd_ops s8000_funcs = {NULL, NULL, NULL}; +#endif /* defined(ARM_BOARD_CLASS_T7000) */ + +#if defined(ARM_BOARD_CLASS_T8002) +extern void fleh_fiq_t8002(void); +extern uint32_t t8002_get_decrementer(void); +extern void t8002_set_decrementer(uint32_t); +static struct tbd_ops t8002_funcs = {&fleh_fiq_t8002, &t8002_get_decrementer, &t8002_set_decrementer}; +#endif /* defined(ARM_BOARD_CLASS_T8002) */ + +#if defined(ARM_BOARD_CLASS_T8010) +static struct tbd_ops t8010_funcs = {NULL, NULL, NULL}; +#endif /* defined(ARM_BOARD_CLASS_T8010) */ + +#if defined(ARM_BOARD_CLASS_T8011) +static struct tbd_ops t8011_funcs = {NULL, NULL, NULL}; +#endif /* defined(ARM_BOARD_CLASS_T8011) */ + + + + + + +vm_offset_t gPicBase; +vm_offset_t gTimerBase; +vm_offset_t gSocPhys; + +#if DEVELOPMENT || DEBUG +// This block contains the panic trace implementation + +// These variables are local to this file, and contain the panic trace configuration information +typedef enum +{ + panic_trace_disabled = 0, + panic_trace_unused, + panic_trace_enabled, + panic_trace_alt_enabled, +} panic_trace_t; +static panic_trace_t bootarg_panic_trace; + +// The command buffer contains the converted commands from the device tree for commanding cpu_halt, enable_trace, etc. +#define DEBUG_COMMAND_BUFFER_SIZE 100 +typedef struct command_buffer_element{ + uintptr_t address; + uint16_t destination_cpu_selector; + uintptr_t value; +} command_buffer_element_t; +static command_buffer_element_t debug_command_buffer[DEBUG_COMMAND_BUFFER_SIZE]; // statically allocate to prevent needing alloc at runtime +static uint32_t next_command_bufffer_entry = 0; // index of next unused slot in debug_command_buffer + +#define CPU_SELECTOR_SHIFT ((sizeof(int)-2)*8) +#define CPU_SELECTOR_MASK (0xFFFF << CPU_SELECTOR_SHIFT) +#define REGISTER_OFFSET_MASK (~CPU_SELECTOR_MASK) +#define REGISTER_OFFSET(register_prop) (register_prop & REGISTER_OFFSET_MASK) +#define CPU_SELECTOR(register_offset) (register_offset >> CPU_SELECTOR_SHIFT) // Upper 16bits holds the cpu selector +#define MAX_WINDOW_SIZE 0xFFFF +#define PE_ISSPACE(c) (c == ' ' || c == '\t' || c == '\n' || c == '\12') +/* +0x0000 - all cpus +0x0001 - cpu 0 +0x0002 - cpu 1 +0x0004 - cpu 2 +0x0003 - cpu 0 and 1 +since it's 16bits, we can have up to 16 cpus +*/ +#define ALL_CPUS 0x0000 +#define IS_CPU_SELECTED(cpu_number, cpu_selector) (cpu_selector == ALL_CPUS || (cpu_selector & (1< DEBUG_COMMAND_BUFFER_SIZE-1) { + panic("pe_init_debug_command: property %s is %u bytes, command buffer only has %lu bytes remaining\n", + entry_name, prop_size, ((DEBUG_COMMAND_BUFFER_SIZE-1) - next_command_bufffer_entry) * sizeof(uintptr_t) ); + } + + // Hold the pointer in a temp variable and later assign it to command buffer, in case we panic while half-initialized + command_starting_index = next_command_bufffer_entry; + + // convert to real virt addresses and stuff commands into debug_command_buffer + for( ; prop_size ; reg_prop += 2, prop_size -= 2*sizeof(uintptr_t) ) { + if (*reg_prop == RESET_VIRTUAL_ADDRESS_WINDOW) { + debug_reg_window = 0; // Create a new window + } + else if (debug_reg_window==0) { + // create a window from virtual address to the specified physical address + reg_window_size = ((uint32_t)*(reg_prop + 1)); + if (reg_window_size > MAX_WINDOW_SIZE) { + panic("pe_init_debug_command: Command page size is %0x, exceeds the Maximum allowed page size 0f 0%x\n", reg_window_size, MAX_WINDOW_SIZE ); + } + debug_reg_window = ml_io_map(gSocPhys + *reg_prop, reg_window_size); + // for debug -- kprintf("pe_init_debug_command: %s registers @ 0x%08lX for 0x%08lX\n", entry_name, debug_reg_window, *(reg_prop + 1) ); + } else { + if ((REGISTER_OFFSET(*reg_prop)+ sizeof(uintptr_t)) >= reg_window_size) { + panic("pe_init_debug_command: Command Offset is %lx, exceeds allocated size of %x\n", REGISTER_OFFSET(*reg_prop),reg_window_size ); + } + debug_command_buffer[next_command_bufffer_entry].address = debug_reg_window + REGISTER_OFFSET(*reg_prop); + debug_command_buffer[next_command_bufffer_entry].destination_cpu_selector = CPU_SELECTOR(*reg_prop); + debug_command_buffer[next_command_bufffer_entry++].value = *(reg_prop+1); + } + } + + // null terminate the address field of the command to end it + debug_command_buffer[next_command_bufffer_entry++].address = 0; + + // save pointer into table for this command + *command_buffer = &debug_command_buffer[command_starting_index]; +} + +static void +pe_run_debug_command(command_buffer_element_t *command_buffer) +{ + // When both the CPUs panic, one will get stuck on the lock and the other CPU will be halted when the first executes the debug command + simple_lock(&panic_trace_lock); + running_debug_command_on_cpu_number = cpu_number(); + + while( command_buffer && command_buffer->address ) { + if (IS_CPU_SELECTED(running_debug_command_on_cpu_number, command_buffer->destination_cpu_selector)) { + *((volatile uintptr_t*)(command_buffer->address)) = command_buffer->value; // register = value; + } + command_buffer++; + } + + running_debug_command_on_cpu_number = -1; + simple_unlock(&panic_trace_lock); +} + + +void +PE_arm_debug_enable_trace(void) +{ + switch (bootarg_panic_trace) { + case panic_trace_enabled: + pe_run_debug_command(enable_trace); + break; + + case panic_trace_alt_enabled: + pe_run_debug_command(enable_alt_trace); + break; + + default: + break; + } +} + +static void +PEARMDebugPanicHook(const char *str) +{ + (void)str; // not used + + // if panic trace is enabled + if (bootarg_panic_trace != 0) { + if (running_debug_command_on_cpu_number == cpu_number()) { + // This is going to end badly if we don't trap, since we'd be panic-ing during our own code + kprintf("## Panic Trace code caused the panic ##\n"); + return; // allow the normal panic operation to occur. + } + + // Stop tracing to freze the buffer and return to normal panic processing. + pe_run_debug_command(trace_halt); + } +} + +void (*PE_arm_debug_panic_hook)(const char *str) = PEARMDebugPanicHook; + +#else + +void (*PE_arm_debug_panic_hook)(const char *str) = NULL; + +#endif // DEVELOPMENT || DEBUG + +void +pe_arm_init_debug(void *args) +{ + DTEntry entryP; + uintptr_t *reg_prop; + uint32_t prop_size; + + if (gSocPhys == 0 ) { + kprintf("pe_arm_init_debug: failed to initialize gSocPhys == 0\n"); + return; + } + + if ( DTFindEntry("device_type", "cpu-debug-interface", &entryP) == kSuccess ) { + if (args != NULL) { + if (DTGetProperty(entryP, "reg", (void **)®_prop, &prop_size) == kSuccess) { + ml_init_arm_debug_interface(args, ml_io_map(gSocPhys + *reg_prop, *(reg_prop + 1))); + } +#if DEVELOPMENT || DEBUG + // When args != NULL, this means we're being called from arm_init on the boot CPU. + // This controls one-time initialization of the Panic Trace infrastructure + + simple_lock_init(&panic_trace_lock, 0); //assuming single threaded mode + + // Panic_halt is deprecated. Please use panic_trace istead. + unsigned int temp_bootarg_panic_trace; + if (PE_parse_boot_argn("panic_trace", &temp_bootarg_panic_trace, sizeof(temp_bootarg_panic_trace)) || + PE_parse_boot_argn("panic_halt", &temp_bootarg_panic_trace, sizeof(temp_bootarg_panic_trace))) { + + kprintf("pe_arm_init_debug: panic_trace=%d\n", temp_bootarg_panic_trace); + + // Prepare debug command buffers. + pe_init_debug_command(entryP, &cpu_halt, "cpu_halt"); + pe_init_debug_command(entryP, &enable_trace, "enable_trace"); + pe_init_debug_command(entryP, &enable_alt_trace, "enable_alt_trace"); + pe_init_debug_command(entryP, &trace_halt, "trace_halt"); + + // now that init's are done, enable the panic halt capture (allows pe_init_debug_command to panic normally if necessary) + bootarg_panic_trace = temp_bootarg_panic_trace; + + // start tracing now if enabled + PE_arm_debug_enable_trace(); + } +#endif + } + } else { + kprintf("pe_arm_init_debug: failed to find cpu-debug-interface\n"); + } +} + +static uint32_t +pe_arm_map_interrupt_controller(void) +{ + DTEntry entryP; + uintptr_t *reg_prop; + uint32_t prop_size; + vm_offset_t soc_phys = 0; + + gSocPhys = pe_arm_get_soc_base_phys(); + + soc_phys = gSocPhys; + kprintf("pe_arm_map_interrupt_controller: soc_phys: 0x%lx\n", (unsigned long)soc_phys); + if (soc_phys == 0) + return 0; + + if (DTFindEntry("interrupt-controller", "master", &entryP) == kSuccess) { + kprintf("pe_arm_map_interrupt_controller: found interrupt-controller\n"); + DTGetProperty(entryP, "reg", (void **)®_prop, &prop_size); + gPicBase = ml_io_map(soc_phys + *reg_prop, *(reg_prop + 1)); + kprintf("pe_arm_map_interrupt_controller: gPicBase: 0x%lx\n", (unsigned long)gPicBase); + } + if (gPicBase == 0) { + kprintf("pe_arm_map_interrupt_controller: failed to find the interrupt-controller.\n"); + return 0; + } + + if (DTFindEntry("device_type", "timer", &entryP) == kSuccess) { + kprintf("pe_arm_map_interrupt_controller: found timer\n"); + DTGetProperty(entryP, "reg", (void **)®_prop, &prop_size); + gTimerBase = ml_io_map(soc_phys + *reg_prop, *(reg_prop + 1)); + kprintf("pe_arm_map_interrupt_controller: gTimerBase: 0x%lx\n", (unsigned long)gTimerBase); + } + if (gTimerBase == 0) { + kprintf("pe_arm_map_interrupt_controller: failed to find the timer.\n"); + return 0; + } + + return 1; +} + +uint32_t +pe_arm_init_interrupts(void *args) +{ + kprintf("pe_arm_init_interrupts: args: %p\n", args); + + /* Set up mappings for interrupt controller and possibly timers (if they haven't been set up already) */ + if (args != NULL) { + if (!pe_arm_map_interrupt_controller()) { + return 0; + } + } + + return pe_arm_init_timer(args); +} + +static uint32_t +pe_arm_init_timer(void *args) +{ + vm_offset_t pic_base = 0; + vm_offset_t timer_base = 0; + vm_offset_t soc_phys; + vm_offset_t eoi_addr = 0; + uint32_t eoi_value = 0; + struct tbd_ops generic_funcs = {&fleh_fiq_generic, NULL, NULL}; + tbd_ops_t tbd_funcs = &generic_funcs; + + /* The SoC headers expect to use pic_base, timer_base, etc... */ + pic_base = gPicBase; + timer_base = gTimerBase; + soc_phys = gSocPhys; + +#if defined(ARM_BOARD_CLASS_S5L8960X) + if (!strcmp(gPESoCDeviceType, "s5l8960x-io")) { + + tbd_funcs = &s5l8960x_funcs; + } else +#endif +#if defined(ARM_BOARD_CLASS_T7000) + if (!strcmp(gPESoCDeviceType, "t7000-io") || + !strcmp(gPESoCDeviceType, "t7001-io")) { + tbd_funcs = &t7000_funcs; + } else +#endif +#if defined(ARM_BOARD_CLASS_S7002) + if (!strcmp(gPESoCDeviceType, "s7002-io")) { + +#ifdef ARM_BOARD_WFE_TIMEOUT_NS + // Enable the WFE Timer + rPMGR_EVENT_TMR_PERIOD = ((uint64_t)(ARM_BOARD_WFE_TIMEOUT_NS) * gPEClockFrequencyInfo.timebase_frequency_hz) / NSEC_PER_SEC; + rPMGR_EVENT_TMR = rPMGR_EVENT_TMR_PERIOD; + rPMGR_EVENT_TMR_CTL = PMGR_EVENT_TMR_CTL_EN; +#endif /* ARM_BOARD_WFE_TIMEOUT_NS */ + + rPMGR_INTERVAL_TMR = 0x7FFFFFFF; + rPMGR_INTERVAL_TMR_CTL = PMGR_INTERVAL_TMR_CTL_EN | PMGR_INTERVAL_TMR_CTL_CLR_INT; + + eoi_addr = timer_base; + eoi_value = PMGR_INTERVAL_TMR_CTL_EN | PMGR_INTERVAL_TMR_CTL_CLR_INT; + tbd_funcs = &s7002_funcs; + } else +#endif +#if defined(ARM_BOARD_CLASS_S8000) + if (!strcmp(gPESoCDeviceType, "s8000-io") || + !strcmp(gPESoCDeviceType, "s8001-io")) { + tbd_funcs = &s8000_funcs; + } else +#endif +#if defined(ARM_BOARD_CLASS_T8002) + if (!strcmp(gPESoCDeviceType, "t8002-io") || + !strcmp(gPESoCDeviceType, "t8004-io")) { + + /* Enable the Decrementer */ + aic_write32(kAICTmrCnt, 0x7FFFFFFF); + aic_write32(kAICTmrCfg, kAICTmrCfgEn); + aic_write32(kAICTmrIntStat, kAICTmrIntStatPct); +#ifdef ARM_BOARD_WFE_TIMEOUT_NS + // Enable the WFE Timer + rPMGR_EVENT_TMR_PERIOD = ((uint64_t)(ARM_BOARD_WFE_TIMEOUT_NS) * gPEClockFrequencyInfo.timebase_frequency_hz) / NSEC_PER_SEC; + rPMGR_EVENT_TMR = rPMGR_EVENT_TMR_PERIOD; + rPMGR_EVENT_TMR_CTL = PMGR_EVENT_TMR_CTL_EN; +#endif /* ARM_BOARD_WFE_TIMEOUT_NS */ + + eoi_addr = pic_base; + eoi_value = kAICTmrIntStatPct; + tbd_funcs = &t8002_funcs; + } else +#endif +#if defined(ARM_BOARD_CLASS_T8010) + if (!strcmp(gPESoCDeviceType, "t8010-io")) { + tbd_funcs = &t8010_funcs; + } else +#endif +#if defined(ARM_BOARD_CLASS_T8011) + if (!strcmp(gPESoCDeviceType, "t8011-io")) { + tbd_funcs = &t8011_funcs; + } else +#endif + return 0; + + if (args != NULL) + ml_init_timebase(args, tbd_funcs, eoi_addr, eoi_value); + + return 1; +} + diff --git a/pexpert/arm/pe_init.c b/pexpert/arm/pe_init.c new file mode 100644 index 000000000..8accd1743 --- /dev/null +++ b/pexpert/arm/pe_init.c @@ -0,0 +1,661 @@ +/* + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. + * + * arm platform expert initialization. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* extern references */ +extern void pe_identify_machine(boot_args *bootArgs); + +/* static references */ +static void pe_prepare_images(void); + +/* private globals */ +PE_state_t PE_state; +#define FW_VERS_LEN 128 +char firmware_version[FW_VERS_LEN]; + +/* + * This variable is only modified once, when the BSP starts executing. We put it in __TEXT + * as page protections on kernel text early in startup are read-write. The kernel is + * locked down later in start-up, said mappings become RO and thus this + * variable becomes immutable. + * + * See osfmk/arm/arm_vm_init.c for more information. + */ +SECURITY_READ_ONLY_SPECIAL_SECTION(volatile uint32_t, "__TEXT,__const") debug_enabled = FALSE; + +uint8_t gPlatformECID[8]; +uint32_t gPlatformMemoryID; +static boolean_t vc_progress_initialized = FALSE; +uint64_t last_hwaccess_thread = 0; +char gTargetTypeBuffer[8]; +char gModelTypeBuffer[32]; + +/* Clock Frequency Info */ +clock_frequency_info_t gPEClockFrequencyInfo; + +vm_offset_t gPanicBase; +unsigned int gPanicSize; +struct embedded_panic_header *panic_info = NULL; + +/* Maximum size of panic log excluding headers, in bytes */ +static unsigned int panic_text_len; + +/* Whether a console is standing by for panic logging */ +static boolean_t panic_console_available = FALSE; + +extern uint32_t crc32(uint32_t crc, const void *buf, size_t size); + +static void +check_for_panic_log(void) +{ + DTEntry entry, chosen; + unsigned int size; + uintptr_t *reg_prop; + uint32_t *panic_region_length; + + /* + * Find the vram node in the device tree + */ + if (kSuccess != DTLookupEntry(0, "pram", &entry)) + return; + + if (kSuccess != DTGetProperty(entry, "reg", (void **)®_prop, &size)) + return; + + if (kSuccess != DTLookupEntry(0, "/chosen", &chosen)) + return; + + if (kSuccess != DTGetProperty(chosen, "embedded-panic-log-size", (void **) &panic_region_length, &size)) + return; + + /* + * Map the first page of VRAM into the kernel for use in case of + * panic + */ + /* Note: map as normal memory. */ + gPanicBase = ml_io_map_wcomb(reg_prop[0], panic_region_length[0]); + + /* Deduct the size of the panic header from the panic region size */ + panic_text_len = panic_region_length[0] - sizeof(struct embedded_panic_header); + gPanicSize = panic_region_length[0]; + panic_info = (struct embedded_panic_header *)gPanicBase; + + /* Check if a shared memory console is running in the panic buffer */ + if (panic_info->eph_magic == 'SHMC') { + panic_console_available = TRUE; + return; + } + + /* Check if there's a boot profile in the panic buffer */ + if (panic_info->eph_magic == 'BTRC') { + return; + } + + /* + * Check to see if a panic (FUNK) is in VRAM from the last time + */ + if (panic_info->eph_magic == EMBEDDED_PANIC_MAGIC) { + printf("iBoot didn't extract panic log from previous session crash, this is bad\n"); + } + + /* Clear panic region */ + bzero((void *)gPanicBase, gPanicSize); +} + +int +PE_initialize_console(PE_Video * info, int op) +{ + static int last_console = -1; + + if (info && (info != &PE_state.video)) info->v_scale = PE_state.video.v_scale; + + switch (op) { + + case kPEDisableScreen: + initialize_screen(info, op); + last_console = switch_to_serial_console(); + kprintf("kPEDisableScreen %d\n", last_console); + break; + + case kPEEnableScreen: + initialize_screen(info, op); + if (info) + PE_state.video = *info; + kprintf("kPEEnableScreen %d\n", last_console); + if (last_console != -1) + switch_to_old_console(last_console); + break; + + case kPEReleaseScreen: + /* + * we don't show the progress indicator on boot, but want to + * show it afterwards. + */ + if (!vc_progress_initialized) { + default_progress.dx = 0; + default_progress.dy = 0; + vc_progress_initialize(&default_progress, + default_progress_data1x, + default_progress_data2x, + default_progress_data3x, + (unsigned char *) appleClut8); + vc_progress_initialized = TRUE; + } + initialize_screen(info, op); + break; + + default: + initialize_screen(info, op); + break; + } + + return 0; +} + +void +PE_init_iokit(void) +{ + DTEntry entry; + unsigned int size, scale; + unsigned long display_size; + void **map; + unsigned int show_progress; + int *delta, image_size, flip; + uint32_t start_time_value = 0; + uint32_t debug_wait_start_value = 0; + uint32_t load_kernel_start_value = 0; + uint32_t populate_registry_time_value = 0; + + PE_init_kprintf(TRUE); + PE_init_printf(TRUE); + + printf("iBoot version: %s\n", firmware_version); + + if (kSuccess == DTLookupEntry(0, "/chosen/memory-map", &entry)) { + + boot_progress_element *bootPict; + + if (kSuccess == DTGetProperty(entry, "BootCLUT", (void **) &map, &size)) + bcopy(map[0], appleClut8, sizeof(appleClut8)); + + if (kSuccess == DTGetProperty(entry, "Pict-FailedBoot", (void **) &map, &size)) { + + bootPict = (boot_progress_element *) map[0]; + default_noroot.width = bootPict->width; + default_noroot.height = bootPict->height; + default_noroot.dx = 0; + default_noroot.dy = bootPict->yOffset; + default_noroot_data = &bootPict->data[0]; + } + } + + pe_prepare_images(); + + scale = PE_state.video.v_scale; + flip = 1; + + if (PE_parse_boot_argn("-progress", &show_progress, sizeof (show_progress)) && show_progress) { + /* Rotation: 0:normal, 1:right 90, 2:left 180, 3:left 90 */ + switch (PE_state.video.v_rotate) { + case 2: + flip = -1; + /* fall through */ + case 0: + display_size = PE_state.video.v_height; + image_size = default_progress.height; + delta = &default_progress.dy; + break; + case 1: + flip = -1; + /* fall through */ + case 3: + default: + display_size = PE_state.video.v_width; + image_size = default_progress.width; + delta = &default_progress.dx; + } + assert(*delta >= 0); + while (((unsigned)(*delta + image_size)) >= (display_size / 2)) { + *delta -= 50 * scale; + assert(*delta >= 0); + } + *delta *= flip; + + /* Check for DT-defined progress y delta */ + PE_get_default("progress-dy", &default_progress.dy, sizeof(default_progress.dy)); + + vc_progress_initialize(&default_progress, + default_progress_data1x, + default_progress_data2x, + default_progress_data3x, + (unsigned char *) appleClut8); + vc_progress_initialized = TRUE; + } + + if (kdebug_enable && kdebug_debugid_enabled(IOKDBG_CODE(DBG_BOOTER, 0))) { + /* Trace iBoot-provided timing information. */ + if (kSuccess == DTLookupEntry(0, "/chosen/iBoot", &entry)) { + uint32_t * value_ptr; + + if (kSuccess == DTGetProperty(entry, "start-time", (void **)&value_ptr, &size)) { + if (size == sizeof(start_time_value)) + start_time_value = *value_ptr; + } + + if (kSuccess == DTGetProperty(entry, "debug-wait-start", (void **)&value_ptr, &size)) { + if (size == sizeof(debug_wait_start_value)) + debug_wait_start_value = *value_ptr; + } + + if (kSuccess == DTGetProperty(entry, "load-kernel-start", (void **)&value_ptr, &size)) { + if (size == sizeof(load_kernel_start_value)) + load_kernel_start_value = *value_ptr; + } + + if (kSuccess == DTGetProperty(entry, "populate-registry-time", (void **)&value_ptr, &size)) { + if (size == sizeof(populate_registry_time_value)) + populate_registry_time_value = *value_ptr; + } + } + + KDBG_RELEASE(IOKDBG_CODE(DBG_BOOTER, 0), start_time_value, debug_wait_start_value, load_kernel_start_value, populate_registry_time_value); + } + + StartIOKit(PE_state.deviceTreeHead, PE_state.bootArgs, (void *) 0, (void *) 0); +} + +void +PE_init_platform(boolean_t vm_initialized, void *args) +{ + DTEntry entry; + unsigned int size; + void **prop; + boot_args *boot_args_ptr = (boot_args *) args; + + if (PE_state.initialized == FALSE) { + PE_state.initialized = TRUE; + PE_state.bootArgs = boot_args_ptr; + PE_state.deviceTreeHead = boot_args_ptr->deviceTreeP; + PE_state.video.v_baseAddr = boot_args_ptr->Video.v_baseAddr; + PE_state.video.v_rowBytes = boot_args_ptr->Video.v_rowBytes; + PE_state.video.v_width = boot_args_ptr->Video.v_width; + PE_state.video.v_height = boot_args_ptr->Video.v_height; + PE_state.video.v_depth = (boot_args_ptr->Video.v_depth >> kBootVideoDepthDepthShift) & kBootVideoDepthMask; + PE_state.video.v_rotate = (boot_args_ptr->Video.v_depth >> kBootVideoDepthRotateShift) & kBootVideoDepthMask; + PE_state.video.v_scale = ((boot_args_ptr->Video.v_depth >> kBootVideoDepthScaleShift) & kBootVideoDepthMask) + 1; + PE_state.video.v_display = boot_args_ptr->Video.v_display; + strlcpy(PE_state.video.v_pixelFormat, "BBBBBBBBGGGGGGGGRRRRRRRR", sizeof(PE_state.video.v_pixelFormat)); + } + if (!vm_initialized) { + /* + * Setup the Device Tree routines + * so the console can be found and the right I/O space + * can be used.. + */ + DTInit(PE_state.deviceTreeHead); + pe_identify_machine(boot_args_ptr); + } else { + pe_arm_init_interrupts(args); + pe_arm_init_debug(args); + } + + if (!vm_initialized) { + if (kSuccess == (DTFindEntry("name", "device-tree", &entry))) { + if (kSuccess == DTGetProperty(entry, "target-type", + (void **)&prop, &size)) { + if (size > sizeof(gTargetTypeBuffer)) + size = sizeof(gTargetTypeBuffer); + bcopy(prop,gTargetTypeBuffer,size); + gTargetTypeBuffer[size-1]='\0'; + } + } + if (kSuccess == (DTFindEntry("name", "device-tree", &entry))) { + if (kSuccess == DTGetProperty(entry, "model", + (void **)&prop, &size)) { + if (size > sizeof(gModelTypeBuffer)) + size = sizeof(gModelTypeBuffer); + bcopy(prop,gModelTypeBuffer,size); + gModelTypeBuffer[size-1]='\0'; + } + } + if (kSuccess == DTLookupEntry(NULL, "/chosen", &entry)) { + if (kSuccess == DTGetProperty(entry, "debug-enabled", + (void **) &prop, &size)) { + /* + * We purposefully modify a constified variable as + * it will get locked down by a trusted monitor or + * via page table mappings. We don't want people easily + * modifying this variable... + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" + boolean_t *modify_debug_enabled = (boolean_t *) &debug_enabled; + if (size > sizeof(uint32_t)) + size = sizeof(uint32_t); + bcopy(prop, modify_debug_enabled, size); +#pragma clang diagnostic pop + } + if (kSuccess == DTGetProperty(entry, "firmware-version", + (void **) &prop, &size)) { + if (size > sizeof(firmware_version)) + size = sizeof(firmware_version); + bcopy(prop, firmware_version, size); + firmware_version[size - 1] = '\0'; + } + if (kSuccess == DTGetProperty(entry, "unique-chip-id", + (void **) &prop, &size)) { + if (size > sizeof(gPlatformECID)) + size = sizeof(gPlatformECID); + bcopy(prop,gPlatformECID,size); + } + if (kSuccess == DTGetProperty(entry, "dram-vendor-id", + (void **) &prop, &size)) { + if (size > sizeof(gPlatformMemoryID)) + size = sizeof(gPlatformMemoryID); + bcopy(prop,&gPlatformMemoryID,size); + } + } + pe_init_debug(); + } +} + +void +PE_create_console(void) +{ + /* + * Check the head of VRAM for a panic log saved on last panic. + * Do this before the VRAM is trashed. + */ + check_for_panic_log(); + + if (PE_state.video.v_display) + PE_initialize_console(&PE_state.video, kPEGraphicsMode); + else + PE_initialize_console(&PE_state.video, kPETextMode); +} + +int +PE_current_console(PE_Video * info) +{ + *info = PE_state.video; + return (0); +} + +void +PE_display_icon(__unused unsigned int flags, __unused const char *name) +{ + if (default_noroot_data) + vc_display_icon(&default_noroot, default_noroot_data); +} + +extern boolean_t +PE_get_hotkey(__unused unsigned char key) +{ + return (FALSE); +} + +static timebase_callback_func gTimebaseCallback; + +void +PE_register_timebase_callback(timebase_callback_func callback) +{ + gTimebaseCallback = callback; + + PE_call_timebase_callback(); +} + +void +PE_call_timebase_callback(void) +{ + struct timebase_freq_t timebase_freq; + + timebase_freq.timebase_num = gPEClockFrequencyInfo.timebase_frequency_hz; + timebase_freq.timebase_den = 1; + + if (gTimebaseCallback) + gTimebaseCallback(&timebase_freq); +} + +/* + * The default PE_poll_input handler. + */ +static int +PE_stub_poll_input(__unused unsigned int options, char *c) +{ + *c = uart_getc(); + return 0; /* 0 for success, 1 for unsupported */ +} + +/* + * Called by the kernel debugger to poll for keyboard input. + * Keyboard drivers may replace the default stub function + * with their polled-mode input function. + */ +int (*PE_poll_input) (unsigned int options, char *c) = PE_stub_poll_input; + +/* + * This routine will return 1 if you are running on a device with a variant + * of iBoot that allows debugging. This is typically not the case on production + * fused parts (even when running development variants of iBoot). + * + * The routine takes an optional argument of the flags passed to debug="" so + * kexts don't have to parse the boot arg themselves. + */ +uint32_t +PE_i_can_has_debugger(uint32_t *debug_flags) +{ + if (debug_flags) { + if (debug_enabled) + *debug_flags = debug_boot_arg; + else + *debug_flags = 0; + } + return (debug_enabled); +} + +void +PE_save_buffer_to_vram(unsigned char *buf, unsigned int *size) +{ + if (!panic_info || !size) { + return; + } + + if (!buf) { + *size = panic_text_len; + return; + } + + if (*size == 0) { + return; + } + + *size = *size > panic_text_len ? panic_text_len : *size; + if (panic_info->eph_magic != EMBEDDED_PANIC_MAGIC) + printf("Error!! Current Magic 0x%X, expected value 0x%x", panic_info->eph_magic, EMBEDDED_PANIC_MAGIC); + + /* CRC everything after the CRC itself - starting with the panic header version */ + panic_info->eph_crc = crc32(0L, &panic_info->eph_version, (panic_text_len + + sizeof(struct embedded_panic_header) - offsetof(struct embedded_panic_header, eph_version))); +} + +uint32_t +PE_get_offset_into_panic_region(char *location) +{ + assert(panic_info != NULL); + assert(location > (char *) panic_info); + assert((unsigned int)(location - (char *) panic_info) < panic_text_len); + + return (uint32_t) (location - gPanicBase); +} + +void +PE_init_panicheader() +{ + if (!panic_info) + return; + + bzero(panic_info, sizeof(struct embedded_panic_header)); + + /* + * The panic log begins immediately after the panic header -- debugger synchronization and other functions + * may log into this region before we've become the exclusive panicking CPU and initialize the header here. + */ + panic_info->eph_panic_log_offset = PE_get_offset_into_panic_region(debug_buf_base); + + panic_info->eph_magic = EMBEDDED_PANIC_MAGIC; + panic_info->eph_version = EMBEDDED_PANIC_HEADER_CURRENT_VERSION; + + return; +} + +/* + * Tries to update the panic header to keep it consistent on nested panics. + * + * NOTE: The purpose of this function is NOT to detect/correct corruption in the panic region, + * it is to update the panic header to make it consistent when we nest panics. + */ +void +PE_update_panicheader_nestedpanic() +{ + if (!panic_info) + return; + + /* + * If the panic log offset is not set, re-init the panic header + */ + if (panic_info->eph_panic_log_offset == 0) { + PE_init_panicheader(); + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_NESTED_PANIC; + return; + } + + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_NESTED_PANIC; + + /* + * If the panic log length is not set, set the end to + * the current location of the debug_buf_ptr to close it. + */ + if (panic_info->eph_panic_log_len == 0) { + panic_info->eph_panic_log_len = PE_get_offset_into_panic_region(debug_buf_ptr); + + /* If this assert fires, it's indicative of corruption in the panic region */ + assert(panic_info->eph_other_log_offset == panic_info->eph_other_log_len == 0); + } + + /* If this assert fires, it's likely indicative of corruption in the panic region */ + assert(((panic_info->eph_stackshot_offset == 0) && (panic_info->eph_stackshot_len == 0)) || + ((panic_info->eph_stackshot_offset != 0) && (panic_info->eph_stackshot_len != 0))); + + /* + * If we haven't set up the other log yet, set the beginning of the other log + * to the current location of the debug_buf_ptr + */ + if (panic_info->eph_other_log_offset == 0) { + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + + /* If this assert fires, it's indicative of corruption in the panic region */ + assert(panic_info->eph_other_log_len == 0); + } + + return; +} + +boolean_t +PE_reboot_on_panic(void) +{ + uint32_t debug_flags; + + if (PE_i_can_has_debugger(&debug_flags) + && (debug_flags & DB_NMI)) { + /* kernel debugging is active */ + return FALSE; + } else { + return TRUE; + } +} + +void +PE_sync_panic_buffers(void) +{ + /* + * rdar://problem/26453070: + * The iBoot panic region is write-combined on arm64. We must flush dirty lines + * from L1/L2 as late as possible before reset, with no further reads of the panic + * region between the flush and the reset. Some targets have an additional memcache (L3), + * and a read may bring dirty lines out of L3 and back into L1/L2, causing the lines to + * be discarded on reset. If we can make sure the lines are flushed to L3/DRAM, + * the platform reset handler will flush any L3. + */ + if (gPanicBase) + CleanPoC_DcacheRegion_Force(gPanicBase, gPanicSize); +} + +static void +pe_prepare_images(void) +{ + if ((1 & PE_state.video.v_rotate) != 0) { + // Only square square images with radial symmetry are supported + // No need to actually rotate the data + + // Swap the dx and dy offsets + uint32_t tmp = default_progress.dx; + default_progress.dx = default_progress.dy; + default_progress.dy = tmp; + } +#if 0 + uint32_t cnt, cnt2, cnt3, cnt4; + uint32_t tmp, width, height; + uint8_t data, *new_data; + const uint8_t *old_data; + + width = default_progress.width; + height = default_progress.height * default_progress.count; + + // Scale images if the UI is being scaled + if (PE_state.video.v_scale > 1) { + new_data = kalloc(width * height * scale * scale); + if (new_data != 0) { + old_data = default_progress_data; + default_progress_data = new_data; + for (cnt = 0; cnt < height; cnt++) { + for (cnt2 = 0; cnt2 < width; cnt2++) { + data = *(old_data++); + for (cnt3 = 0; cnt3 < scale; cnt3++) { + for (cnt4 = 0; cnt4 < scale; cnt4++) { + new_data[width * scale * cnt3 + cnt4] = data; + } + } + new_data += scale; + } + new_data += width * scale * (scale - 1); + } + default_progress.width *= scale; + default_progress.height *= scale; + default_progress.dx *= scale; + default_progress.dy *= scale; + } + } +#endif +} + +void +PE_mark_hwaccess(uint64_t thread) +{ + last_hwaccess_thread = thread; + asm volatile("dmb ish"); +} diff --git a/pexpert/arm/pe_kprintf.c b/pexpert/arm/pe_kprintf.c new file mode 100644 index 000000000..c0ec13792 --- /dev/null +++ b/pexpert/arm/pe_kprintf.c @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * file: pe_kprintf.c + * arm platform expert debugging output initialization. + */ +#include +#include +#include +#include +#include +#include +#include + +/* Globals */ +void (*PE_kputc) (char c) = 0; + +SECURITY_READ_ONLY_LATE(unsigned int) disable_serial_output = TRUE; + +decl_simple_lock_data(static, kprintf_lock) + +void +PE_init_kprintf(boolean_t vm_initialized) +{ + unsigned int boot_arg; + + if (PE_state.initialized == FALSE) + panic("Platform Expert not initialized"); + + if (!vm_initialized) { + simple_lock_init(&kprintf_lock, 0); + + if (PE_parse_boot_argn("debug", &boot_arg, sizeof (boot_arg))) + if (boot_arg & DB_KPRT) + disable_serial_output = FALSE; + + if (serial_init()) + PE_kputc = serial_putc; + else + PE_kputc = cnputc; + } +} + +#ifdef MP_DEBUG +static void +_kprintf(const char *format,...) +{ + va_list listp; + + va_start(listp, format); + _doprnt_log(format, &listp, PE_kputc, 16); + va_end(listp); +} +#define MP_DEBUG_KPRINTF(x...) _kprintf(x) +#else /* MP_DEBUG */ +#define MP_DEBUG_KPRINTF(x...) +#endif /* MP_DEBUG */ + +#if CONFIG_NO_KPRINTF_STRINGS +/* Prevent CPP from breaking the definition below */ +#undef kprintf +#endif + +static int cpu_last_locked = 0; + +__attribute__((noinline,not_tail_called)) +void kprintf(const char *fmt,...) +{ + va_list listp; + va_list listp2; + boolean_t state; + void *caller = __builtin_return_address(0); + + if (!disable_serial_output) { + + /* + * Spin to get kprintf lock but re-enable interrupts while failing. + * This allows interrupts to be handled while waiting but + * interrupts are disabled once we have the lock. + */ + state = ml_set_interrupts_enabled(FALSE); + while (!simple_lock_try(&kprintf_lock)) { + ml_set_interrupts_enabled(state); + ml_set_interrupts_enabled(FALSE); + } + + if (cpu_number() != cpu_last_locked) { + MP_DEBUG_KPRINTF("[cpu%d...]\n", cpu_number()); + cpu_last_locked = cpu_number(); + } + + va_start(listp, fmt); + va_copy(listp2, listp); + _doprnt_log(fmt, &listp, PE_kputc, 16); + va_end(listp); + + simple_unlock(&kprintf_lock); + +#if INTERRUPT_MASKED_DEBUG + /* + * kprintf holds interrupts disabled for far too long + * and would trip the spin-debugger. If we are about to reenable + * interrupts then clear the timer and avoid panicking on the delay. + * Otherwise, let the code that printed with interrupt disabled + * take the panic when it reenables interrupts. + * Hopefully one day this is fixed so that this workaround is unnecessary. + */ + if (state == TRUE) + ml_spin_debug_clear_self(); +#endif + ml_set_interrupts_enabled(state); + + // If interrupts are enabled + if (ml_get_interrupts_enabled()) { + os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, fmt, listp2, caller); + } + va_end(listp2); + } + else { + // If interrupts are enabled + if (ml_get_interrupts_enabled()) { + va_start(listp, fmt); + os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, fmt, listp, caller); + va_end(listp); + } + } +} + +void +serial_putc(char c) +{ + uart_putc(c); +} + +int +serial_getc(void) +{ + return uart_getc(); +} + diff --git a/pexpert/arm/pe_serial.c b/pexpert/arm/pe_serial.c new file mode 100644 index 000000000..ff4a301e2 --- /dev/null +++ b/pexpert/arm/pe_serial.c @@ -0,0 +1,831 @@ +/* + * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + */ + +/* + * file: pe_serial.c Polled-mode UART0 driver for S3c2410 and PL011. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#if defined __arm__ +#include +#include +#include +#include +#include +#elif defined __arm64__ +#include +#include +#include +#endif + +struct pe_serial_functions { + void (*uart_init) (void); + void (*uart_set_baud_rate) (int unit, uint32_t baud_rate); + int (*tr0) (void); + void (*td0) (int c); + int (*rr0) (void); + int (*rd0) (void); +}; + +static struct pe_serial_functions *gPESF; + +static int uart_initted = 0; /* 1 if init'ed */ + +static vm_offset_t uart_base; + + +/*****************************************************************************/ + +#ifdef S3CUART + +static int32_t dt_pclk = -1; +static int32_t dt_sampling = -1; +static int32_t dt_ubrdiv = -1; + +static void +ln2410_uart_init(void) +{ + uint32_t ucon0 = 0x405; /* NCLK, No interrupts, No DMA - just polled */ + + rULCON0 = 0x03; /* 81N, not IR */ + + // Override with pclk dt entry + if (dt_pclk != -1) + ucon0 = ucon0 & ~0x400; + + rUCON0 = ucon0; + rUMCON0 = 0x00; /* Clear Flow Control */ + + gPESF->uart_set_baud_rate(0, 115200); + + rUFCON0 = 0x03; /* Clear & Enable FIFOs */ + rUMCON0 = 0x01; /* Assert RTS on UART0 */ +} + +static void +ln2410_uart_set_baud_rate(__unused int unit, uint32_t baud_rate) +{ + uint32_t div = 0; + uint32_t uart_clock = 0; + uint32_t sample_rate = 16; + + if (baud_rate < 300) + baud_rate = 9600; + + if (rUCON0 & 0x400) + // NCLK + uart_clock = (uint32_t)gPEClockFrequencyInfo.fix_frequency_hz; + else + // PCLK + uart_clock = (uint32_t)gPEClockFrequencyInfo.prf_frequency_hz; + + if (dt_sampling != -1) { + // Use the sampling rate specified in the Device Tree + sample_rate = dt_sampling & 0xf; + } + + if (dt_ubrdiv != -1) { + // Use the ubrdiv specified in the Device Tree + div = dt_ubrdiv & 0xffff; + } else { + // Calculate ubrdiv. UBRDIV = (SourceClock / (BPS * Sample Rate)) - 1 + div = uart_clock / (baud_rate * sample_rate); + + uint32_t actual_baud = uart_clock / ((div + 0) * sample_rate); + uint32_t baud_low = uart_clock / ((div + 1) * sample_rate); + + // Adjust div to get the closest target baudrate + if ((baud_rate - baud_low) > (actual_baud - baud_rate)) + div--; + } + + // Sample Rate [19:16], UBRDIV [15:0] + rUBRDIV0 = ((16 - sample_rate) << 16) | div; +} + +static int +ln2410_tr0(void) +{ + return rUTRSTAT0 & 0x04; +} +static void +ln2410_td0(int c) +{ + rUTXH0 = (unsigned)(c & 0xff); +} +static int +ln2410_rr0(void) +{ + return rUTRSTAT0 & 0x01; +} +static int +ln2410_rd0(void) +{ + return (int)rURXH0; +} + +static struct pe_serial_functions ln2410_serial_functions = { + ln2410_uart_init, ln2410_uart_set_baud_rate, +ln2410_tr0, ln2410_td0, ln2410_rr0, ln2410_rd0}; + +#endif /* S3CUART */ + +/*****************************************************************************/ + + +static unsigned int +read_dtr(void) +{ +#ifdef __arm__ + unsigned int c; + __asm__ volatile( + "mrc p14, 0, %0, c0, c5\n" +: "=r"(c)); + return c; +#else + /* ARM64_TODO */ + panic_unimplemented(); + return 0; +#endif +} +static void +write_dtr(unsigned int c) +{ +#ifdef __arm__ + __asm__ volatile( + "mcr p14, 0, %0, c0, c5\n" + : + :"r"(c)); +#else + /* ARM64_TODO */ + (void)c; + panic_unimplemented(); +#endif +} + +static int +dcc_tr0(void) +{ +#ifdef __arm__ + return !(arm_debug_read_dscr() & ARM_DBGDSCR_TXFULL); +#else + /* ARM64_TODO */ + panic_unimplemented(); + return 0; +#endif +} + +static void +dcc_td0(int c) +{ + write_dtr(c); +} + +static int +dcc_rr0(void) +{ +#ifdef __arm__ + return arm_debug_read_dscr() & ARM_DBGDSCR_RXFULL; +#else + /* ARM64_TODO */ + panic_unimplemented(); + return 0; +#endif +} + +static int +dcc_rd0(void) +{ + return read_dtr(); +} + +static struct pe_serial_functions dcc_serial_functions = { + NULL, NULL, +dcc_tr0, dcc_td0, dcc_rr0, dcc_rd0}; + +/*****************************************************************************/ + +#ifdef SHMCON + +#define CPU_CACHELINE_SIZE (1 << MMU_CLINE) + +#ifndef SHMCON_NAME +#define SHMCON_NAME "AP-xnu" +#endif + +#define SHMCON_MAGIC 'SHMC' +#define SHMCON_VERSION 2 +#define CBUF_IN 0 +#define CBUF_OUT 1 +#define INBUF_SIZE (panic_size / 16) +#define FULL_ALIGNMENT (64) + +#define FLAG_CACHELINE_32 1 +#define FLAG_CACHELINE_64 2 + +/* Defines to clarify the master/slave fields' use as circular buffer pointers */ +#define head_in sidx[CBUF_IN] +#define tail_in midx[CBUF_IN] +#define head_out midx[CBUF_OUT] +#define tail_out sidx[CBUF_OUT] + +/* TODO: get from device tree/target */ +#define NUM_CHILDREN 5 + +#define WRAP_INCR(len, x) do{ (x)++; if((x) >= (len)) (x) = 0; } while(0) +#define ROUNDUP(a, b) (((a) + ((b) - 1)) & (~((b) - 1))) + +#define MAX(a,b) ((a) > (b) ? (a) : (b)) +#define MIN(a,b) ((a) < (b) ? (a) : (b)) + +#define shmcon_barrier() do {__asm__ volatile("dmb ish" : : : "memory");} while(0) + +struct shm_buffer_info { + uint64_t base; + uint32_t unused; + uint32_t magic; +}; + +struct shmcon_header { + uint32_t magic; + uint8_t version; + uint8_t children; /* number of child entries in child_ent */ + uint16_t flags; + uint64_t buf_paddr[2]; /* Physical address for buffers (in, out) */ + uint32_t buf_len[2]; + uint8_t name[8]; + + /* Slave-modified data - invalidate before read */ + uint32_t sidx[2] __attribute__((aligned (FULL_ALIGNMENT))); /* In head, out tail */ + + /* Master-modified data - clean after write */ + uint32_t midx[2] __attribute__((aligned (FULL_ALIGNMENT))); /* In tail, out head */ + + uint64_t child[0]; /* Physical address of child header pointers */ +}; + +static volatile struct shmcon_header *shmcon = NULL; +static volatile uint8_t *shmbuf[2]; +#ifdef SHMCON_THROTTLED +static uint64_t grace = 0; +static uint64_t full_timeout = 0; +#endif + +static void shmcon_set_baud_rate(__unused int unit, __unused uint32_t baud_rate) +{ + return; +} + +static int shmcon_tr0(void) +{ +#ifdef SHMCON_THROTTLED + uint32_t head = shmcon->head_out; + uint32_t tail = shmcon->tail_out; + uint32_t len = shmcon->buf_len[CBUF_OUT]; + + WRAP_INCR(len, head); + if (head != tail) { + full_timeout = 0; + return 1; + } + + /* Full. Is this buffer being serviced? */ + if (full_timeout == 0) { + full_timeout = mach_absolute_time() + grace; + return 0; + } + if (full_timeout > mach_absolute_time()) + return 0; + + /* Timeout - slave not really there or not keeping up */ + tail += (len / 4); + if (tail >= len) + tail -= len; + shmcon_barrier(); + shmcon->tail_out = tail; + full_timeout = 0; +#endif + return 1; +} + +static void shmcon_td0(int c) +{ + uint32_t head = shmcon->head_out; + uint32_t len = shmcon->buf_len[CBUF_OUT]; + + shmbuf[CBUF_OUT][head] = (uint8_t)c; + WRAP_INCR(len, head); + shmcon_barrier(); + shmcon->head_out = head; +} + +static int shmcon_rr0(void) +{ + if (shmcon->tail_in == shmcon->head_in) + return 0; + return 1; +} + +static int shmcon_rd0(void) +{ + int c; + uint32_t tail = shmcon->tail_in; + uint32_t len = shmcon->buf_len[CBUF_IN]; + + c = shmbuf[CBUF_IN][tail]; + WRAP_INCR(len, tail); + shmcon_barrier(); + shmcon->tail_in = tail; + return c; +} + +static void shmcon_init(void) +{ + DTEntry entry; + uintptr_t *reg_prop; + volatile struct shm_buffer_info *end; + size_t i, header_size; + unsigned int size; + vm_offset_t pa_panic_base, panic_size, va_buffer_base, va_buffer_end; + + if (kSuccess != DTLookupEntry(0, "pram", &entry)) + return; + + if (kSuccess != DTGetProperty(entry, "reg", (void **)®_prop, &size)) + return; + + pa_panic_base = reg_prop[0]; + panic_size = reg_prop[1]; + + shmcon = (struct shmcon_header *)ml_map_high_window(pa_panic_base, panic_size); + header_size = sizeof(*shmcon) + (NUM_CHILDREN * sizeof(shmcon->child[0])); + va_buffer_base = ROUNDUP((uintptr_t)(shmcon) + header_size, CPU_CACHELINE_SIZE); + va_buffer_end = (uintptr_t)shmcon + panic_size - (sizeof(*end)); + + if ((shmcon->magic == SHMCON_MAGIC) && (shmcon->version == SHMCON_VERSION)) { + vm_offset_t pa_buffer_base, pa_buffer_end; + + pa_buffer_base = ml_vtophys(va_buffer_base); + pa_buffer_end = ml_vtophys(va_buffer_end); + + /* Resume previous console session */ + for (i = 0; i < 2; i++) { + vm_offset_t pa_buf; + uint32_t len; + + pa_buf = (uintptr_t)shmcon->buf_paddr[i]; + len = shmcon->buf_len[i]; + /* Validate buffers */ + if ((pa_buf < pa_buffer_base) || + (pa_buf >= pa_buffer_end) || + ((pa_buf + len) > pa_buffer_end) || + (shmcon->midx[i] >= len) || /* Index out of bounds */ + (shmcon->sidx[i] >= len) || + (pa_buf != ROUNDUP(pa_buf, CPU_CACHELINE_SIZE)) || /* Unaligned pa_buffer */ + (len < 1024) || + (len > (pa_buffer_end - pa_buffer_base)) || + (shmcon->children != NUM_CHILDREN)) + goto validation_failure; + /* Compute the VA offset of the buffer */ + shmbuf[i] = (uint8_t *)(uintptr_t)shmcon + ((uintptr_t)pa_buf - (uintptr_t)pa_panic_base); + } + /* Check that buffers don't overlap */ + if ((uintptr_t)shmbuf[0] < (uintptr_t)shmbuf[1]) { + if ((uintptr_t)(shmbuf[0] + shmcon->buf_len[0]) > (uintptr_t)shmbuf[1]) + goto validation_failure; + } else { + if ((uintptr_t)(shmbuf[1] + shmcon->buf_len[1]) > (uintptr_t)shmbuf[0]) + goto validation_failure; + } + shmcon->tail_in = shmcon->head_in; /* Clear input buffer */ + shmcon_barrier(); + } else { +validation_failure: + shmcon->magic = 0; + shmcon_barrier(); + shmcon->buf_len[CBUF_IN] = (uint32_t)INBUF_SIZE; + shmbuf[CBUF_IN] = (uint8_t *)va_buffer_base; + shmbuf[CBUF_OUT] = (uint8_t *)ROUNDUP(va_buffer_base + INBUF_SIZE, CPU_CACHELINE_SIZE); + for (i = 0; i < 2; i++) { + shmcon->midx[i] = 0; + shmcon->sidx[i] = 0; + shmcon->buf_paddr[i] = (uintptr_t)ml_vtophys((vm_offset_t)shmbuf[i]); + } + shmcon->buf_len[CBUF_OUT] = (uint32_t)(va_buffer_end - (uintptr_t)shmbuf[CBUF_OUT]); + shmcon->version = SHMCON_VERSION; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" + memset((void *)shmcon->name, ' ', sizeof(shmcon->name)); + memcpy((void *)shmcon->name, SHMCON_NAME, MIN(sizeof(shmcon->name), strlen(SHMCON_NAME))); +#pragma clang diagnostic pop + for (i = 0; i < NUM_CHILDREN; i++) + shmcon->child[0] = 0; + shmcon_barrier(); + shmcon->magic = SHMCON_MAGIC; + } + end = (volatile struct shm_buffer_info *)va_buffer_end; + end->base = pa_panic_base; + end->unused = 0; + shmcon_barrier(); + end->magic = SHMCON_MAGIC; +#ifdef SHMCON_THROTTLED + grace = gPEClockFrequencyInfo.timebase_frequency_hz; +#endif + + PE_consistent_debug_register(kDbgIdConsoleHeaderAP, pa_panic_base, panic_size); +} + +static struct pe_serial_functions shmcon_serial_functions = +{ + .uart_init = shmcon_init, + .uart_set_baud_rate = shmcon_set_baud_rate, + .tr0 = shmcon_tr0, + .td0 = shmcon_td0, + .rr0 = shmcon_rr0, + .rd0 = shmcon_rd0 +}; + +int pe_shmcon_set_child(uint64_t paddr, uint32_t entry) +{ + if (shmcon == NULL) + return -1; + + if (shmcon->children >= entry) + return -1; + + shmcon->child[entry] = paddr; + return 0; +} + +#endif /* SHMCON */ + +/*****************************************************************************/ + +#ifdef DOCKFIFO_UART + + +// Allow a 30ms stall of wall clock time before DockFIFO starts dropping characters +#define DOCKFIFO_WR_MAX_STALL_US (30*1000) + +static uint64_t prev_dockfifo_drained_time; // Last time we've seen the DockFIFO drained by an external agent +static uint64_t prev_dockfifo_spaces; // Previous w_stat level of the DockFIFO. +static uint32_t dockfifo_capacity; +static uint64_t dockfifo_stall_grace; + + +//======================= +// Local funtions +//======================= + +static int dockfifo_drain_on_stall() +{ + // Called when DockFIFO runs out of spaces. + // Check if the DockFIFO reader has stalled. If so, empty the DockFIFO ourselves. + // Return number of bytes drained. + + if (mach_absolute_time() - prev_dockfifo_drained_time >= dockfifo_stall_grace) { + // It's been more than DOCKFIFO_WR_MAX_STALL_US and nobody read from the FIFO + // Drop a character. + (void)rDOCKFIFO_R_DATA(DOCKFIFO_UART_READ, 1); + prev_dockfifo_spaces++; + return 1; + } + return 0; +} + + +static int dockfifo_uart_tr0(void) +{ + uint32_t spaces = rDOCKFIFO_W_STAT(DOCKFIFO_UART_WRITE) & 0xffff; + if (spaces >= dockfifo_capacity || spaces > prev_dockfifo_spaces) { + // More spaces showed up. That can only mean someone read the FIFO. + // Note that if the DockFIFO is empty we cannot tell if someone is listening, + // we can only give them the benefit of the doubt. + + prev_dockfifo_drained_time = mach_absolute_time(); + } + prev_dockfifo_spaces = spaces; + + return spaces || dockfifo_drain_on_stall(); + +} + +static void dockfifo_uart_td0(int c) +{ + rDOCKFIFO_W_DATA(DOCKFIFO_UART_WRITE, 1) = (unsigned)(c & 0xff); + prev_dockfifo_spaces--; // After writing a byte we have one fewer space than previously expected. + +} + +static int dockfifo_uart_rr0(void) +{ + return rDOCKFIFO_R_DATA(DOCKFIFO_UART_READ, 0) & 0x7f; +} + +static int dockfifo_uart_rd0(void) +{ + return (int)((rDOCKFIFO_R_DATA(DOCKFIFO_UART_READ, 1) >> 8) & 0xff); +} + +static void dockfifo_uart_init(void) +{ + nanoseconds_to_absolutetime(DOCKFIFO_WR_MAX_STALL_US * 1000, &dockfifo_stall_grace); + + // Disable autodraining of the FIFO. We now purely manage it in software. + rDOCKFIFO_DRAIN(DOCKFIFO_UART_WRITE) = 0; + + // Empty the DockFIFO by draining it until OCCUPANCY is 0, then measure its capacity + while (rDOCKFIFO_R_DATA(DOCKFIFO_UART_WRITE, 3) & 0x7F); + dockfifo_capacity = rDOCKFIFO_W_STAT(DOCKFIFO_UART_WRITE) & 0xffff; +} + +static struct pe_serial_functions dockfifo_uart_serial_functions = +{ + .uart_init = dockfifo_uart_init, + .uart_set_baud_rate = NULL, + .tr0 = dockfifo_uart_tr0, + .td0 = dockfifo_uart_td0, + .rr0 = dockfifo_uart_rr0, + .rd0 = dockfifo_uart_rd0 +}; + +#endif /* DOCKFIFO_UART */ + +/*****************************************************************************/ + +#ifdef DOCKCHANNEL_UART +#define DOCKCHANNEL_WR_MAX_STALL_US (30*1000) + +static vm_offset_t dock_agent_base; +static uint32_t max_dockchannel_drain_period; +static bool use_sw_drain; +static uint64_t prev_dockchannel_drained_time; // Last time we've seen the DockChannel drained by an external agent +static uint64_t prev_dockchannel_spaces; // Previous w_stat level of the DockChannel. +static uint64_t dockchannel_stall_grace; + +//======================= +// Local funtions +//======================= + +static int dockchannel_drain_on_stall() +{ + // Called when DockChannel runs out of spaces. + // Check if the DockChannel reader has stalled. If so, empty the DockChannel ourselves. + // Return number of bytes drained. + + if ((mach_absolute_time() - prev_dockchannel_drained_time) >= dockchannel_stall_grace) { + // It's been more than DOCKCHANEL_WR_MAX_STALL_US and nobody read from the FIFO + // Drop a character. + (void)rDOCKCHANNELS_DEV_RDATA1(DOCKCHANNEL_UART_CHANNEL); + prev_dockchannel_spaces++; + return 1; + } + return 0; +} + +static int dockchannel_uart_tr0(void) +{ + if (use_sw_drain) { + uint32_t spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & 0x1ff; + if (spaces > prev_dockchannel_spaces) { + // More spaces showed up. That can only mean someone read the FIFO. + // Note that if the DockFIFO is empty we cannot tell if someone is listening, + // we can only give them the benefit of the doubt. + prev_dockchannel_drained_time = mach_absolute_time(); + } + prev_dockchannel_spaces = spaces; + + return spaces || dockchannel_drain_on_stall(); + } else { + // Returns spaces in dockchannel fifo + return (rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & 0x1ff); + } +} + +static void dockchannel_uart_td0(int c) +{ + rDOCKCHANNELS_DEV_WDATA1(DOCKCHANNEL_UART_CHANNEL) = (unsigned)(c & 0xff); + if (use_sw_drain) { + prev_dockchannel_spaces--; // After writing a byte we have one fewer space than previously expected. + } +} + +static int dockchannel_uart_rr0(void) +{ + return rDOCKCHANNELS_DEV_RDATA0(DOCKCHANNEL_UART_CHANNEL) & 0x7f; +} + +static int dockchannel_uart_rd0(void) +{ + return (int)((rDOCKCHANNELS_DEV_RDATA1(DOCKCHANNEL_UART_CHANNEL)>> 8) & 0xff); +} + +static void dockchannel_uart_init(void) +{ + if (use_sw_drain) { + nanoseconds_to_absolutetime(DOCKCHANNEL_WR_MAX_STALL_US * NSEC_PER_USEC, &dockchannel_stall_grace); + } + + // Clear all interrupt enable and status bits + rDOCKCHANNELS_AGENT_AP_INTR_CTRL &= ~(0x3); + rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x3; + rDOCKCHANNELS_AGENT_AP_ERR_INTR_CTRL &= ~(0x3); + rDOCKCHANNELS_AGENT_AP_ERR_INTR_STATUS |= 0x3; + + // Setup DRAIN timer + rDOCKCHANNELS_DEV_DRAIN_CFG(DOCKCHANNEL_UART_CHANNEL) = max_dockchannel_drain_period; + + // Drain timer doesnt get loaded with value from drain period register if fifo + // is already full. Drop a character from the fifo. + // Refer https://seg-docs.ecs.apple.com/projects/cayman//release/specs/Apple/DockChannels/DockChannels_Specification.pdf + // Chapter 8 for more details. + rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL); +} + +static struct pe_serial_functions dockchannel_uart_serial_functions = +{ + .uart_init = dockchannel_uart_init, + .uart_set_baud_rate = NULL, + .tr0 = dockchannel_uart_tr0, + .td0 = dockchannel_uart_td0, + .rr0 = dockchannel_uart_rr0, + .rd0 = dockchannel_uart_rd0 +}; + +#endif /* DOCKCHANNEL_UART */ + +/*****************************************************************************/ + +int +serial_init(void) +{ + DTEntry entryP = NULL; + uint32_t prop_size, dccmode; + vm_offset_t soc_base; + uintptr_t *reg_prop; + uint32_t *prop_value = NULL; + char *serial_compat = 0; +#ifdef SHMCON + uint32_t jconmode; +#endif +#ifdef DOCKFIFO_UART + uint32_t no_dockfifo_uart; +#endif +#ifdef DOCKCHANNEL_UART + uint32_t no_dockchannel_uart; +#endif + + if (uart_initted) { + gPESF->uart_init(); + kprintf("reinit serial\n"); + return 1; + } + dccmode = 0; + if (PE_parse_boot_argn("dcc", &dccmode, sizeof (dccmode))) { + gPESF = &dcc_serial_functions; + uart_initted = 1; + return 1; + } +#ifdef SHMCON + jconmode = 0; + if (PE_parse_boot_argn("jcon", &jconmode, sizeof jconmode)) { + gPESF = &shmcon_serial_functions; + gPESF->uart_init(); + uart_initted = 1; + return 1; + } +#endif /* SHMCON */ + + soc_base = pe_arm_get_soc_base_phys(); + + if (soc_base == 0) + return 0; + +#ifdef DOCKFIFO_UART + no_dockfifo_uart = 0; + PE_parse_boot_argn("no-dockfifo-uart", &no_dockfifo_uart, sizeof(no_dockfifo_uart)); + if (no_dockfifo_uart == 0) { + if (DTFindEntry("name", "dockfifo-uart", &entryP) == kSuccess) { + DTGetProperty(entryP, "reg", (void **)®_prop, &prop_size); + uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1)); + } + else { + return 0; + } + gPESF = &dockfifo_uart_serial_functions; + gPESF->uart_init(); + uart_initted = 1; + return 1; + } +#endif /* DOCKFIFO_UART */ + +#ifdef DOCKCHANNEL_UART + no_dockchannel_uart = 0; + // Keep the old name for boot-arg + PE_parse_boot_argn("no-dockfifo-uart", &no_dockchannel_uart, sizeof(no_dockchannel_uart)); + if (no_dockchannel_uart == 0) { + if (DTFindEntry("name", "dockchannel-uart", &entryP) == kSuccess) { + DTGetProperty(entryP, "reg", (void **)®_prop, &prop_size); + // Should be two reg entries + if (prop_size/sizeof(uintptr_t) != 4) + panic("Malformed dockchannel-uart property"); + uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1)); + dock_agent_base = ml_io_map(soc_base + *(reg_prop + 2), *(reg_prop + 3)); + gPESF = &dockchannel_uart_serial_functions; + DTGetProperty(entryP, "max-aop-clk", (void **)&prop_value, &prop_size); + max_dockchannel_drain_period = (uint32_t)((prop_value)? (*prop_value * 0.03) : DOCKCHANNEL_DRAIN_PERIOD); + DTGetProperty(entryP, "enable-sw-drain", (void **)&prop_value, &prop_size); + use_sw_drain = (prop_value)? *prop_value : 0; + gPESF->uart_init(); + uart_initted = 1; + return 1; + } + // If no dockchannel-uart is found in the device tree, fall back + // to looking for the traditional UART serial console. + } +#endif /* DOCKCHANNEL_UART */ + + /* + * The boot serial port should have a property named "boot-console". + * If we don't find it there, look for "uart0" and "uart1". + */ + + if (DTFindEntry("boot-console", NULL, &entryP) == kSuccess) { + DTGetProperty(entryP, "reg", (void **)®_prop, &prop_size); + uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1)); + if (serial_compat == 0) + DTGetProperty(entryP, "compatible", (void **)&serial_compat, &prop_size); + } else if (DTFindEntry("name", "uart0", &entryP) == kSuccess) { + DTGetProperty(entryP, "reg", (void **)®_prop, &prop_size); + uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1)); + if (serial_compat == 0) + DTGetProperty(entryP, "compatible", (void **)&serial_compat, &prop_size); + } else if (DTFindEntry("name", "uart1", &entryP) == kSuccess) { + DTGetProperty(entryP, "reg", (void **)®_prop, &prop_size); + uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1)); + if (serial_compat == 0) + DTGetProperty(entryP, "compatible", (void **)&serial_compat, &prop_size); + } +#ifdef S3CUART + if (NULL != entryP) { + DTGetProperty(entryP, "pclk", (void **)&prop_value, &prop_size); + if (prop_value) dt_pclk = *prop_value; + + prop_value = NULL; + DTGetProperty(entryP, "sampling", (void **)&prop_value, &prop_size); + if (prop_value) dt_sampling = *prop_value; + + prop_value = NULL; + DTGetProperty(entryP, "ubrdiv", (void **)&prop_value, &prop_size); + if (prop_value) dt_ubrdiv = *prop_value; + } + if (!strcmp(serial_compat, "uart,16550")) + gPESF = &ln2410_serial_functions; + else if (!strcmp(serial_compat, "uart-16550")) + gPESF = &ln2410_serial_functions; + else if (!strcmp(serial_compat, "uart,s5i3000")) + gPESF = &ln2410_serial_functions; + else if (!strcmp(serial_compat, "uart-1,samsung")) + gPESF = &ln2410_serial_functions; +#elif defined (ARM_BOARD_CONFIG_MV88F6710) + if (!strcmp(serial_compat, "uart16x50,mmio")) + gPESF = &uart16x50_serial_functions; +#endif + else + return 0; + + gPESF->uart_init(); + + uart_initted = 1; + + return 1; +} + +void +uart_putc(char c) +{ + if (uart_initted) { + while (!gPESF->tr0()); /* Wait until THR is empty. */ + gPESF->td0(c); + } +} + +int +uart_getc(void) +{ /* returns -1 if no data available */ + if (uart_initted) { + if (!gPESF->rr0()) + return -1; /* Receive data read */ + return gPESF->rd0(); + } + return -1; +} diff --git a/pexpert/conf/Makefile.arm b/pexpert/conf/Makefile.arm new file mode 100644 index 000000000..606a53c3b --- /dev/null +++ b/pexpert/conf/Makefile.arm @@ -0,0 +1,7 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/pexpert/conf/Makefile.arm64 b/pexpert/conf/Makefile.arm64 new file mode 100644 index 000000000..b946c893d --- /dev/null +++ b/pexpert/conf/Makefile.arm64 @@ -0,0 +1,7 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm64 +###################################################################### + +###################################################################### +#END Machine dependent Makefile fragment for arm64 +###################################################################### diff --git a/pexpert/conf/Makefile.template b/pexpert/conf/Makefile.template index cd16445fa..b9962d602 100644 --- a/pexpert/conf/Makefile.template +++ b/pexpert/conf/Makefile.template @@ -17,6 +17,7 @@ include $(MakeInc_def) # CFLAGS # CFLAGS+= -include meta_features.h -DPEXPERT_KERNEL_PRIVATE +SFLAGS+= -include meta_features.h # Objects that don't want -Wcast-align warning (8474835) pe_identify_machine.o_CWARNFLAGS_ADD = -Wno-cast-align diff --git a/pexpert/conf/files.arm b/pexpert/conf/files.arm new file mode 100644 index 000000000..b6b86ef4a --- /dev/null +++ b/pexpert/conf/files.arm @@ -0,0 +1,8 @@ +OPTIONS/gprof optional gprof + +pexpert/arm/pe_bootargs.c standard +pexpert/arm/pe_identify_machine.c standard +pexpert/arm/pe_init.c standard +pexpert/arm/pe_kprintf.c standard +pexpert/arm/pe_serial.c standard +pexpert/arm/pe_consistent_debug.c standard diff --git a/pexpert/conf/files.arm64 b/pexpert/conf/files.arm64 new file mode 100644 index 000000000..aada62e1e --- /dev/null +++ b/pexpert/conf/files.arm64 @@ -0,0 +1,8 @@ +OPTIONS/gprof optional gprof + +pexpert/arm/pe_bootargs.c standard +pexpert/arm/pe_consistent_debug.c standard +pexpert/arm/pe_identify_machine.c standard +pexpert/arm/pe_init.c standard +pexpert/arm/pe_kprintf.c standard +pexpert/arm/pe_serial.c standard diff --git a/pexpert/gen/bootargs.c b/pexpert/gen/bootargs.c index 5a010bc37..754513a5c 100644 --- a/pexpert/gen/bootargs.c +++ b/pexpert/gen/bootargs.c @@ -32,7 +32,9 @@ typedef boolean_t (*argsep_func_t) (char c); static boolean_t isargsep( char c); static boolean_t israngesep( char c); +#ifndef CONFIG_EMBEDDED static int argstrcpy(char *from, char *to); +#endif static int argstrcpy2(char *from,char *to, unsigned maxlen); static int argnumcpy(long long val, void *to, unsigned maxlen); static int getval(char *s, long long *val, argsep_func_t issep, boolean_t skip_equal_sign); @@ -57,6 +59,10 @@ PE_parse_boot_arg( { int max_len = -1; +#if CONFIG_EMBEDDED + /* Limit arg size to 4 byte when no size is given */ + max_len = 4; +#endif return PE_parse_boot_argn(arg_string, arg_ptr, max_len); } @@ -79,6 +85,9 @@ PE_parse_boot_argn_internal( args = PE_boot_args(); if (*args == '\0') return FALSE; +#ifdef CONFIG_EMBEDDED + if (max_len == -1) return FALSE; +#endif arg_found = FALSE; @@ -153,10 +162,12 @@ PE_parse_boot_argn_internal( else if (max_len == 0) { arg_found = TRUE; } +#if !CONFIG_EMBEDDED else if (max_len == -1) { /* unreachable on embedded */ argstrcpy(++cp, (char *)arg_ptr); arg_found = TRUE; } +#endif break; } goto gotit; @@ -208,6 +219,7 @@ israngesep(char c) return (FALSE); } +#if !CONFIG_EMBEDDED static int argstrcpy( char *from, @@ -222,6 +234,7 @@ argstrcpy( *to = 0; return(i); } +#endif static int argstrcpy2( diff --git a/pexpert/gen/device_tree.c b/pexpert/gen/device_tree.c index d78bed0bf..651ccd77b 100644 --- a/pexpert/gen/device_tree.c +++ b/pexpert/gen/device_tree.c @@ -35,50 +35,29 @@ #include #include +#include #include #include +#include #include -#ifndef NULL -#define NULL ((void *) 0) -#endif - -#define round_long(x) (((x) + 3UL) & ~(3UL)) -#define next_prop(x) ((DeviceTreeNodeProperty *) (((uintptr_t)x) + sizeof(DeviceTreeNodeProperty) + round_long(x->length))) - -/* Entry*/ -typedef DeviceTreeNode *RealDTEntry; - -typedef struct DTSavedScope { - struct DTSavedScope * nextScope; - RealDTEntry scope; - RealDTEntry entry; - unsigned long index; -} *DTSavedScopePtr; - -/* Entry Iterator*/ -typedef struct OpaqueDTEntryIterator { - RealDTEntry outerScope; - RealDTEntry currentScope; - RealDTEntry currentEntry; - DTSavedScopePtr savedScope; - unsigned long currentIndex; -} *RealDTEntryIterator; - -/* Property Iterator*/ -typedef struct OpaqueDTPropertyIterator { - RealDTEntry entry; - DeviceTreeNodeProperty *currentProperty; - unsigned long currentIndex; -} *RealDTPropertyIterator; - static int DTInitialized; static RealDTEntry DTRootNode; /* * Support Routines */ +static inline DeviceTreeNodeProperty* +next_prop(DeviceTreeNodeProperty* prop) +{ + uintptr_t next_addr; + if (os_add3_overflow((uintptr_t)prop, prop->length, sizeof(DeviceTreeNodeProperty) + 3, &next_addr)) + panic("Device tree property overflow: prop %p, length 0x%x\n", prop, prop->length); + next_addr &= ~(3ULL); + return (DeviceTreeNodeProperty*)next_addr; +} + static RealDTEntry skipProperties(RealDTEntry entry) { @@ -280,15 +259,12 @@ DTLookupEntry(const DTEntry searchPoint, const char *pathName, DTEntry *foundEnt } int -DTCreateEntryIterator(const DTEntry startEntry, DTEntryIterator *iterator) +DTInitEntryIterator(const DTEntry startEntry, DTEntryIterator iter) { - RealDTEntryIterator iter; - if (!DTInitialized) { return kError; } - iter = (RealDTEntryIterator) kalloc(sizeof(struct OpaqueDTEntryIterator)); if (startEntry != NULL) { iter->outerScope = (RealDTEntry) startEntry; iter->currentScope = (RealDTEntry) startEntry; @@ -300,28 +276,12 @@ DTCreateEntryIterator(const DTEntry startEntry, DTEntryIterator *iterator) iter->savedScope = NULL; iter->currentIndex = 0; - *iterator = iter; - return kSuccess; -} - -int -DTDisposeEntryIterator(DTEntryIterator iterator) -{ - RealDTEntryIterator iter = iterator; - DTSavedScopePtr scope; - - while ((scope = iter->savedScope) != NULL) { - iter->savedScope = scope->nextScope; - kfree(scope, sizeof(struct DTSavedScope)); - } - kfree(iterator, sizeof(struct OpaqueDTEntryIterator)); return kSuccess; } int -DTEnterEntry(DTEntryIterator iterator, DTEntry childEntry) +DTEnterEntry(DTEntryIterator iter, DTEntry childEntry) { - RealDTEntryIterator iter = iterator; DTSavedScopePtr newScope; if (childEntry == NULL) { @@ -342,9 +302,8 @@ DTEnterEntry(DTEntryIterator iterator, DTEntry childEntry) } int -DTExitEntry(DTEntryIterator iterator, DTEntry *currentPosition) +DTExitEntry(DTEntryIterator iter, DTEntry *currentPosition) { - RealDTEntryIterator iter = iterator; DTSavedScopePtr newScope; newScope = iter->savedScope; @@ -363,10 +322,8 @@ DTExitEntry(DTEntryIterator iterator, DTEntry *currentPosition) } int -DTIterateEntries(DTEntryIterator iterator, DTEntry *nextEntry) +DTIterateEntries(DTEntryIterator iter, DTEntry *nextEntry) { - RealDTEntryIterator iter = iterator; - if (iter->currentIndex >= iter->currentScope->nChildren) { *nextEntry = NULL; return kIterationDone; @@ -383,9 +340,8 @@ DTIterateEntries(DTEntryIterator iterator, DTEntry *nextEntry) } int -DTRestartEntryIteration(DTEntryIterator iterator) +DTRestartEntryIteration(DTEntryIterator iter) { - RealDTEntryIterator iter = iterator; #if 0 // This commented out code allows a second argument (outer) // which (if true) causes restarting at the outer scope @@ -429,31 +385,18 @@ DTGetProperty(const DTEntry entry, const char *propertyName, void **propertyValu } int -DTCreatePropertyIterator(const DTEntry entry, DTPropertyIterator *iterator) +DTInitPropertyIterator(const DTEntry entry, DTPropertyIterator iter) { - RealDTPropertyIterator iter; - iter = (RealDTPropertyIterator) kalloc(sizeof(struct OpaqueDTPropertyIterator)); iter->entry = entry; iter->currentProperty = NULL; iter->currentIndex = 0; - - *iterator = iter; return kSuccess; } int -DTDisposePropertyIterator(DTPropertyIterator iterator) +DTIterateProperties(DTPropertyIterator iter, char **foundProperty) { - kfree(iterator, sizeof(struct OpaqueDTPropertyIterator)); - return kSuccess; -} - -int -DTIterateProperties(DTPropertyIterator iterator, char **foundProperty) -{ - RealDTPropertyIterator iter = iterator; - if (iter->currentIndex >= iter->entry->nProperties) { *foundProperty = NULL; return kIterationDone; @@ -470,10 +413,8 @@ DTIterateProperties(DTPropertyIterator iterator, char **foundProperty) } int -DTRestartPropertyIteration(DTPropertyIterator iterator) +DTRestartPropertyIteration(DTPropertyIterator iter) { - RealDTPropertyIterator iter = iterator; - iter->currentProperty = NULL; iter->currentIndex = 0; return kSuccess; diff --git a/pexpert/gen/pe_gen.c b/pexpert/gen/pe_gen.c index 60664a6a0..5e130b161 100644 --- a/pexpert/gen/pe_gen.c +++ b/pexpert/gen/pe_gen.c @@ -34,10 +34,17 @@ #include #include +#if CONFIG_EMBEDDED +#include +#endif static int DEBUGFlag; +#if CONFIG_EMBEDDED +SECURITY_READ_ONLY_LATE(static uint32_t) gPEKernelConfigurationBitmask; +#else static uint32_t gPEKernelConfigurationBitmask; +#endif int32_t gPESerialBaud = -1; diff --git a/pexpert/i386/pe_init.c b/pexpert/i386/pe_init.c index f1e24b31e..2a82b7f36 100644 --- a/pexpert/i386/pe_init.c +++ b/pexpert/i386/pe_init.c @@ -345,8 +345,7 @@ uint32_t PE_i_can_has_debugger(uint32_t *debug_flags) { #if CONFIG_CSR - if (csr_check(CSR_ALLOW_KERNEL_DEBUGGER) != 0 && - csr_check(CSR_ALLOW_APPLE_INTERNAL) != 0) { + if (csr_check(CSR_ALLOW_KERNEL_DEBUGGER) != 0) { if (debug_flags) *debug_flags = 0; return FALSE; @@ -357,3 +356,47 @@ PE_i_can_has_debugger(uint32_t *debug_flags) } return TRUE; } + +uint32_t +PE_get_offset_into_panic_region(char *location) +{ + assert(panic_info != NULL); + assert(location > (char *) panic_info); + + return (uint32_t) (location - debug_buf); +} + +void +PE_init_panicheader() +{ + bzero(panic_info, offsetof(struct macos_panic_header, mph_data)); + panic_info->mph_panic_log_offset = PE_get_offset_into_panic_region(debug_buf_base); + + panic_info->mph_magic = MACOS_PANIC_MAGIC; + panic_info->mph_version = MACOS_PANIC_HEADER_CURRENT_VERSION; + + return; +} + +/* + * Tries to update the panic header to keep it consistent on nested panics. + * + * NOTE: The purpose of this function is NOT to detect/correct corruption in the panic region, + * it is to update the panic header to make it consistent when we nest panics. + */ +void +PE_update_panicheader_nestedpanic() +{ + /* If the panic log offset is not set, re-init the panic header */ + if (panic_info->mph_panic_log_offset == 0) { + PE_init_panicheader(); + panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_NESTED_PANIC; + return; + } + + panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_NESTED_PANIC; + + /* macOS panic logs include nested panic data, so don't touch the panic log length here */ + + return; +} diff --git a/pexpert/pexpert/arm/AIC.h b/pexpert/pexpert/arm/AIC.h new file mode 100644 index 000000000..df60b1017 --- /dev/null +++ b/pexpert/pexpert/arm/AIC.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2009 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_AIC_H +#define _PEXPERT_ARM_AIC_H + +#ifndef ASSEMBLER + +static inline unsigned long _aic_read32(unsigned long addr) +{ + unsigned long data; + data = *(volatile unsigned *)addr; + return data; +} + +static inline void _aic_write32(unsigned long addr, unsigned long data) +{ + *(volatile unsigned *)(addr) = data; +} + +#define aic_read32(offset, data) (_aic_read32(pic_base + (offset))) +#define aic_write32(offset, data) (_aic_write32(pic_base + (offset), (data))) + +#endif + +// AIC +#define kAICAicRev (0x0000) +#define kAICAicCap0 (0x0004) +#define kAICAicCap0Int(n) ((n) & 0x3FF) +#define kAICAicCap0Proc(n) ((((n) >> 16) & 0x1F) + 1) +#define kAICAicCap1 (0x0008) +#define kAICAicRst (0x000C) +#define kAICGlbCfg (0x0010) +#define kAICMainTimLo (0x0020) +#define kAICMainTimHi (0x0028) +#define kAICIPINormalDbg (0x0030) +#define kAICIPISelfDbg (0x0034) + +#define kAICWhoAmI (0x2000) +#define kAICIack (0x2004) +#define kAICIackVecType(n) (((n) >> 16) & 0x7) +#define kAICIackVecTypeSpurious (0) +#define kAICIackVecTypeExtInt (1) +#define kAICIackVecTypeIPI (4) +#define kAICIackVecTypeTimer (7) +#define kAICIackVecExtInt(n) ((n) & 0x3FF) +#define kAICIackVecIPIType(n) ((n) & 0x003) +#define kAICIackVecIPITypeNormal (1) +#define kAICIackVecIPITypeSelf (2) +#define kAICIPISet (0x2008) +#define kAICIPIClr (0x200C) +#define kAICIPIClrSelf (0x80000000) +#define kAICTmrCfg (0x2010) +#define kAICTmrCfgEn (1) +#define kAICTmrCfgFslPTI (0 << 4) +#define kAICTmrCfgFslSGTI (1 << 4) +#define kAICTmrCfgFslETI (2 << 4) +#define kAICTmrCnt (0x2014) +#define kAICTmrIntStat (0x2018) +#define kAICTmrIntStatPct (1) +#define kAICTmrStateSet (0x201C) +#define kAICTmrStateClr (0x2020) +#define kAICBankedCoreRegs (0x2000) +#define kAICBankedCoreTmrCnt (0x14) +#define kAICBankedCoreTmrIntStat (0x18) + +#define kAICTgtDst(n) (0x3000 + (n) * 4) +#define kAICSwGenSet(n) (0x4000 + (n) * 4) +#define kAICSwGenClr(n) (0x4080 + (n) * 4) +#define kAICIntMaskSet(n) (0x4100 + (n) * 4) +#define kAICIntMaskClr(n) (0x4180 + (n) * 4) +#define kAICHwIntMon(n) (0x4200 + (n) * 4) + +#define kAICAliasWhoAmI(n) (0x5000 + (n) * 0x80 + 0x00) +#define kAICAliasIack(n) (0x5000 + (n) * 0x80 + 0x04) +#define kAICAliasIPISet(n) (0x5000 + (n) * 0x80 + 0x08) +#define kAICAliasIPIClr(n) (0x5000 + (n) * 0x80 + 0x0C) +#define kAICAliasTmrCfg(n) (0x5000 + (n) * 0x80 + 0x10) +#define kAICAliasTmrCnt(n) (0x5000 + (n) * 0x80 + 0x14) +#define kAICAliasTmrIntStat(n) (0x5000 + (n) * 0x80 + 0x18) +#define kAICAliasTmrStateSet(n) (0x5000 + (n) * 0x80 + 0x1C) +#define kAICAliasTmrStateClr(n) (0x5000 + (n) * 0x80 + 0x20) + +#define kAICExtIntShift (5) +#define kAICExtIntMask (0x1F) + +#endif /* ! _PEXPERT_ARM_AIC_H */ diff --git a/pexpert/pexpert/arm/Makefile b/pexpert/pexpert/arm/Makefile new file mode 100644 index 000000000..d5b46d1a9 --- /dev/null +++ b/pexpert/pexpert/arm/Makefile @@ -0,0 +1,29 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +DATAFILES = \ + AIC.h \ + board_config.h \ + boot.h \ + consistent_debug.h \ + PL192_VIC.h \ + protos.h \ + S3cUART.h \ + S7002.h \ + T8002.h + +INSTALL_MD_LIST = ${DATAFILES} + +INSTALL_MD_DIR = pexpert/arm + +EXPORT_MD_LIST = ${DATAFILES} + +EXPORT_MD_DIR = pexpert/arm + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/pexpert/pexpert/arm/PL192_VIC.h b/pexpert/pexpert/arm/PL192_VIC.h new file mode 100644 index 000000000..091ab72a6 --- /dev/null +++ b/pexpert/pexpert/arm/PL192_VIC.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2005-2007 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_PL192_VIC_H +#define _PEXPERT_ARM_PL192_VIC_H + +#define ARM_CELL_PL192_VIC + +// VIC +#define rVICIRQSTATUS (*(volatile unsigned *)(pic_base + 0x000)) // VIC IRQ Status Register +#define rVICFIQSTATUS (*(volatile unsigned *)(pic_base + 0x004)) // VIC FIQ Status Register +#define rVICRAWINTR (*(volatile unsigned *)(pic_base + 0x008)) // VIC Raw Interrupt Status Register +#define rVICINTSELECT (*(volatile unsigned *)(pic_base + 0x00C)) // VIC Interrupt Select Register +#define rVICINTENABLE (*(volatile unsigned *)(pic_base + 0x010)) // VIC Interrupt Enable Register +#define rVICINTENCLEAR (*(volatile unsigned *)(pic_base + 0x014)) // VIC Interrupt Enable Clear Register +#define rVICSOFTINT (*(volatile unsigned *)(pic_base + 0x018)) // VIC Soft Interrupt Register +#define rVICSOFTINTCLEAR (*(volatile unsigned *)(pic_base + 0x01C)) // VIC Soft Interrupt Clear Register +#define rVICPROTECTION (*(volatile unsigned *)(pic_base + 0x020)) // VIC Protection Register +#define rVICSWPRIORITYMASK (*(volatile unsigned *)(pic_base + 0x024)) // VIC Software Priority Mask Register +#define rVICPRIORITYDAISY (*(volatile unsigned *)(pic_base + 0x028)) // VIC Priority Daisy Chain Register +#define rVICVECTOR(x) (*(volatile unsigned *)(pic_base + 0x100 + 4 * (x))) // VIC Vector Registers +#define rVICVECTPRIORITY(x) (*(volatile unsigned *)(pic_base + 0x200 + 4 * (x))) // VIC Vector Priority Registers +#define rVICPERIPHID0 (*(volatile unsigned *)(pic_base + 0xFE0)) // VIC Peripheral ID 0 Register +#define rVICPERIPHID1 (*(volatile unsigned *)(pic_base + 0xFE4)) // VIC Peripheral ID 1 Register +#define rVICPERIPHID2 (*(volatile unsigned *)(pic_base + 0xFE8)) // VIC Peripheral ID 2 Register +#define rVICPERIPHID3 (*(volatile unsigned *)(pic_base + 0xFEC)) // VIC Peripheral ID 3 Register +#define rVICPCELLID0 (*(volatile unsigned *)(pic_base + 0xFF0)) // VIC PrimeCell ID 0 Register +#define rVICPCELLID1 (*(volatile unsigned *)(pic_base + 0xFF4)) // VIC PrimeCell ID 1 Register +#define rVICPCELLID2 (*(volatile unsigned *)(pic_base + 0xFF8)) // VIC PrimeCell ID 2 Register +#define rVICPCELLID3 (*(volatile unsigned *)(pic_base + 0xFFC)) // VIC PrimeCell ID 3 Register + +#endif /* ! _PEXPERT_ARM_PL192_VIC_H */ diff --git a/pexpert/pexpert/arm/S3cUART.h b/pexpert/pexpert/arm/S3cUART.h new file mode 100644 index 000000000..a8410b685 --- /dev/null +++ b/pexpert/pexpert/arm/S3cUART.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +#ifndef _PEXPERT_ARM_S3CUART_H +#define _PEXPERT_ARM_S3CUART_H + +#define S3CUART + + +// UART +#define rULCON0 (*(volatile unsigned *)(uart_base + 0x00)) //UART 0 Line control +#define rUCON0 (*(volatile unsigned *)(uart_base + 0x04)) //UART 0 Control +#define rUFCON0 (*(volatile unsigned *)(uart_base + 0x08)) //UART 0 FIFO control +#define rUMCON0 (*(volatile unsigned *)(uart_base + 0x0c)) //UART 0 Modem control +#define rUTRSTAT0 (*(volatile unsigned *)(uart_base + 0x10)) //UART 0 Tx/Rx status +#define rUERSTAT0 (*(volatile unsigned *)(uart_base + 0x14)) //UART 0 Rx error status +#define rUFSTAT0 (*(volatile unsigned *)(uart_base + 0x18)) //UART 0 FIFO status +#define rUMSTAT0 (*(volatile unsigned *)(uart_base + 0x1c)) //UART 0 Modem status +#define rUTXH0 (*(volatile unsigned *)(uart_base + 0x20)) //UART 0 Transmission Hold +#define rURXH0 (*(volatile unsigned *)(uart_base + 0x24)) //UART 0 Receive buffer +#define rUBRDIV0 (*(volatile unsigned *)(uart_base + 0x28)) //UART 0 Baud rate divisor +#define rUDIVSLOT0 (*(volatile unsigned *)(uart_base + 0x2C)) //UART 0 Baud rate divisor + +#define rULCON1 (*(volatile unsigned *)(uart1_base + 0x00)) //UART 1 Line control +#define rUCON1 (*(volatile unsigned *)(uart1_base + 0x04)) //UART 1 Control +#define rUFCON1 (*(volatile unsigned *)(uart1_base + 0x08)) //UART 1 FIFO control +#define rUMCON1 (*(volatile unsigned *)(uart1_base + 0x0c)) //UART 1 Modem control +#define rUTRSTAT1 (*(volatile unsigned *)(uart1_base + 0x10)) //UART 1 Tx/Rx status +#define rUERSTAT1 (*(volatile unsigned *)(uart1_base + 0x14)) //UART 1 Rx error status +#define rUFSTAT1 (*(volatile unsigned *)(uart1_base + 0x18)) //UART 1 FIFO status +#define rUMSTAT1 (*(volatile unsigned *)(uart1_base + 0x1c)) //UART 1 Modem status +#define rUTXH1 (*(volatile unsigned *)(uart1_base + 0x20)) //UART 1 Transmission Hold +#define rURXH1 (*(volatile unsigned *)(uart1_base + 0x24)) //UART 1 Receive buffer +#define rUBRDIV1 (*(volatile unsigned *)(uart1_base + 0x28)) //UART 1 Baud rate divisor + +#define rULCON2 (*(volatile unsigned *)(uart2_base + 0x00)) //UART 2 Line control +#define rUCON2 (*(volatile unsigned *)(uart2_base + 0x04)) //UART 2 Control +#define rUFCON2 (*(volatile unsigned *)(uart2_base + 0x08)) //UART 2 FIFO control +#define rUMCON2 (*(volatile unsigned *)(uart2_base + 0x0c)) //UART 2 Modem control +#define rUTRSTAT2 (*(volatile unsigned *)(uart2_base + 0x10)) //UART 2 Tx/Rx status +#define rUERSTAT2 (*(volatile unsigned *)(uart2_base + 0x14)) //UART 2 Rx error status +#define rUFSTAT2 (*(volatile unsigned *)(uart2_base + 0x18)) //UART 2 FIFO status +#define rUMSTAT2 (*(volatile unsigned *)(uart2_base + 0x1c)) //UART 2 Modem status +#define rUTXH2 (*(volatile unsigned *)(uart2_base + 0x20)) //UART 2 Transmission Hold +#define rURXH2 (*(volatile unsigned *)(uart2_base + 0x24)) //UART 2 Receive buffer +#define rUBRDIV2 (*(volatile unsigned *)(uart2_base + 0x28)) //UART 2 Baud rate divisor + +#if 0 +#define WrUTXH0(ch) (*(volatile unsigned char *)0x50000020)=(unsigned char)(ch) +#define RdURXH0() (*(volatile unsigned char *)0x50000024) +#define WrUTXH1(ch) (*(volatile unsigned char *)0x50004020)=(unsigned char)(ch) +#define RdURXH1() (*(volatile unsigned char *)0x50004024) +#define WrUTXH2(ch) (*(volatile unsigned char *)0x50008020)=(unsigned char)(ch) +#define RdURXH2() (*(volatile unsigned char *)0x50008024) + +#define UTXH0 (0x50000020) //Byte_access address by DMA +#define URXH0 (0x50000024) +#define UTXH1 (0x50004020) +#define URXH1 (0x50004024) +#define UTXH2 (0x50008020) +#define URXH2 (0x50008024) +#endif + +#endif /* _PEXPERT_ARM_S3CUART_H */ diff --git a/pexpert/pexpert/arm/S7002.h b/pexpert/pexpert/arm/S7002.h new file mode 100644 index 000000000..9774e01b8 --- /dev/null +++ b/pexpert/pexpert/arm/S7002.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2013 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_S7002_H +#define _PEXPERT_ARM_S7002_H + +#ifndef ASSEMBLER + +#include + +#define rPMGR_EVENT_TMR (*(volatile unsigned *) (timer_base + 0x00100)) +#define rPMGR_EVENT_TMR_PERIOD (*(volatile unsigned *) (timer_base + 0x00104)) +#define rPMGR_EVENT_TMR_CTL (*(volatile unsigned *) (timer_base + 0x00108)) +#define rPMGR_INTERVAL_TMR (*(volatile unsigned *) (timer_base + 0x00200)) +#define rPMGR_INTERVAL_TMR_CTL (*(volatile unsigned *) (timer_base + 0x00204)) + +#define PMGR_EVENT_TMR_CTL_EN (1 << 0) +#define PMGR_INTERVAL_TMR_CTL_EN (1 << 0) +#define PMGR_INTERVAL_TMR_CTL_CLR_INT (1 << 8) + +#define DOCKFIFO_UART (1) +#define DOCKFIFO_UART_WRITE (0) +#define DOCKFIFO_UART_READ (1) +#define DOCKFIFO_W_SPACING (0x1000) +#define DOCKFIFO_SPACING (0x3000) + +#define rDOCKFIFO_R_DATA(_f, _n) (*(volatile uint32_t *)(uart_base + ((_f) * DOCKFIFO_SPACING) + ((_n) * 4))) +#define rDOCKFIFO_R_STAT(_f) (*(volatile uint32_t *)(uart_base + ((_f) * DOCKFIFO_SPACING) + 0x14)) +#define rDOCKFIFO_W_DATA(_f, _n) (*(volatile uint32_t *)(uart_base + ((_f) * DOCKFIFO_SPACING) + DOCKFIFO_W_SPACING + ((_n) * 4))) +#define rDOCKFIFO_W_STAT(_f) (*(volatile uint32_t *)(uart_base + ((_f) * DOCKFIFO_SPACING) + DOCKFIFO_W_SPACING + 0x14)) +#define rDOCKFIFO_CNFG(_f) (*(volatile uint32_t *)(uart_base + ((_f) * DOCKFIFO_SPACING) + 0x2000)) +#define rDOCKFIFO_DRAIN(_f) (*(volatile uint32_t *)(uart_base + ((_f) * DOCKFIFO_SPACING) + 0x2004)) +#define rDOCKFIFO_INTMASK(_f) (*(volatile uint32_t *)(uart_base + ((_f) * DOCKFIFO_SPACING) + 0x2008)) + +#endif + +#define PMGR_INTERVAL_TMR_OFFSET (0x200) +#define PMGR_INTERVAL_TMR_CTL_OFFSET (0x204) + +#endif /* ! _PEXPERT_ARM_S7002_H */ diff --git a/pexpert/pexpert/arm/T8002.h b/pexpert/pexpert/arm/T8002.h new file mode 100644 index 000000000..19fb3aeff --- /dev/null +++ b/pexpert/pexpert/arm/T8002.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2014-2015 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_T8002_H +#define _PEXPERT_ARM_T8002_H + +#include + +#ifndef ASSEMBLER + +#include + +#define rPMGR_EVENT_TMR (*(volatile uint32_t *) (timer_base + 0x00000)) +#define rPMGR_EVENT_TMR_PERIOD (*(volatile uint32_t *) (timer_base + 0x00004)) +#define rPMGR_EVENT_TMR_CTL (*(volatile uint32_t *) (timer_base + 0x00008)) + +#define PMGR_EVENT_TMR_CTL_EN (1 << 0) + +#define DOCKCHANNEL_UART (1) +#define DOCKCHANNEL_STRIDE (0x10000) + +// Channel index +#define DOCKCHANNEL_UART_CHANNEL (0) + +// AOP_CLOCK frequency * 30 ms +#define DOCKCHANNEL_DRAIN_PERIOD (96000000 * 0.03) + +#define rDOCKCHANNELS_AGENT_AP_INTR_CTRL (*(volatile uint32_t *) (dock_agent_base + 0x00)) +#define rDOCKCHANNELS_AGENT_AP_INTR_STATUS (*(volatile uint32_t *) (dock_agent_base + 0x04)) +#define rDOCKCHANNELS_AGENT_AP_ERR_INTR_CTRL (*(volatile uint32_t *) (dock_agent_base + 0x08)) +#define rDOCKCHANNELS_AGENT_AP_ERR_INTR_STATUS (*(volatile uint32_t *) (dock_agent_base + 0x0c)) + +#define rDOCKCHANNELS_DEV_DRAIN_CFG(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0x0008)) + +#define rDOCKCHANNELS_DEV_WDATA1(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0x4004)) +#define rDOCKCHANNELS_DEV_WSTAT(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0x4014)) +#define rDOCKCHANNELS_DEV_RDATA0(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0x4018)) +#define rDOCKCHANNELS_DEV_RDATA1(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0x401c)) + +#define rDOCKCHANNELS_DOCK_RDATA1(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0xc01c)) +#define rDOCKCHANNELS_DOCK_RDATA3(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0xc024)) +#endif + +#endif /* ! _PEXPERT_ARM_T8002_H */ diff --git a/pexpert/pexpert/arm/board_config.h b/pexpert/pexpert/arm/board_config.h new file mode 100644 index 000000000..a6d930e41 --- /dev/null +++ b/pexpert/pexpert/arm/board_config.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2007-2014 Apple Inc. All rights reserved. + * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved. + */ +#ifndef _PEXPERT_ARM_BOARD_CONFIG_H +#define _PEXPERT_ARM_BOARD_CONFIG_H + +#ifdef ARM_BOARD_CONFIG_S7002 +#define ARMA7 +#define __XNU_UP__ +#include + +#define ARM_BOARD_WFE_TIMEOUT_NS 1000 +#define __ARM_L2CACHE_SIZE_LOG__ 18 + +#define ARM_BOARD_CLASS_S7002 +#endif /* ARM_BOARD_CONFIG_S7002 */ + +#ifdef ARM_BOARD_CONFIG_T8002 +#define ARMA7 +#include + +#define ARM_BOARD_WFE_TIMEOUT_NS 1000 +#define __ARM_L2CACHE_SIZE_LOG__ 19 + +#define ARM_BOARD_CLASS_T8002 +#endif /* ARM_BOARD_CONFIG_T8002 */ + +#ifdef ARM_BOARD_CONFIG_T8004 +#define ARMA7 +#include + +#define ARM_BOARD_WFE_TIMEOUT_NS 1000 +#define __ARM_L2CACHE_SIZE_LOG__ 20 + +#define ARM_BOARD_CLASS_T8002 +#endif /* ARM_BOARD_CONFIG_T8004 */ + + +#endif /* ! _PEXPERT_ARM_BOARD_CONFIG_H */ diff --git a/pexpert/pexpert/arm/boot.h b/pexpert/pexpert/arm/boot.h new file mode 100644 index 000000000..dc0605fc8 --- /dev/null +++ b/pexpert/pexpert/arm/boot.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2007-2009 Apple Inc. All rights reserved. + * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + */ +/* + * @OSF_COPYRIGHT@ + */ + +#ifndef _PEXPERT_ARM_BOOT_H_ +#define _PEXPERT_ARM_BOOT_H_ + +#include + +#define BOOT_LINE_LENGTH 256 + +/* + * Video information.. + */ + +struct Boot_Video { + unsigned long v_baseAddr; /* Base address of video memory */ + unsigned long v_display; /* Display Code (if Applicable */ + unsigned long v_rowBytes; /* Number of bytes per pixel row */ + unsigned long v_width; /* Width */ + unsigned long v_height; /* Height */ + unsigned long v_depth; /* Pixel Depth and other parameters */ +}; + +#define kBootVideoDepthMask (0xFF) +#define kBootVideoDepthDepthShift (0) +#define kBootVideoDepthRotateShift (8) +#define kBootVideoDepthScaleShift (16) + +#define kBootFlagsDarkBoot (1 << 0) + +typedef struct Boot_Video Boot_Video; + +/* Boot argument structure - passed into Mach kernel at boot time. + */ +#define kBootArgsRevision 1 +#define kBootArgsRevision2 2 /* added boot_args->bootFlags */ +#define kBootArgsVersion1 1 +#define kBootArgsVersion2 2 + +typedef struct boot_args { + uint16_t Revision; /* Revision of boot_args structure */ + uint16_t Version; /* Version of boot_args structure */ + uint32_t virtBase; /* Virtual base of memory */ + uint32_t physBase; /* Physical base of memory */ + uint32_t memSize; /* Size of memory */ + uint32_t topOfKernelData; /* Highest physical address used in kernel data area */ + Boot_Video Video; /* Video Information */ + uint32_t machineType; /* Machine Type */ + void *deviceTreeP; /* Base of flattened device tree */ + uint32_t deviceTreeLength; /* Length of flattened tree */ + char CommandLine[BOOT_LINE_LENGTH]; /* Passed in command line */ + uint32_t bootFlags; /* Additional flags specified by the bootloader */ + uint32_t memSizeActual; /* Actual size of memory */ +} boot_args; + +#define SOC_DEVICE_TYPE_BUFFER_SIZE 32 + +#define PC_TRACE_BUF_SIZE 1024 + +#define CDBG_MEM ((sizeof(dbg_registry_t) + PAGE_SIZE - 1) & ~PAGE_MASK) + +#endif /* _PEXPERT_ARM_BOOT_H_ */ diff --git a/pexpert/pexpert/arm/consistent_debug.h b/pexpert/pexpert/arm/consistent_debug.h new file mode 100644 index 000000000..97cf27cf0 --- /dev/null +++ b/pexpert/pexpert/arm/consistent_debug.h @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2011-2012 Apple Inc. All rights reserved. + * + * This document is the property of Apple Inc. + * It is considered confidential and proprietary. + * + * This document may not be reproduced or transmitted in any form, + * in whole or in part, without the express written permission of + * Apple Inc. + */ +#ifndef PE_CONSISTENT_DEBUG_H +#define PE_CONSISTENT_DEBUG_H + +#include + +#define DEBUG_RECORD_ID_LONG(a, b,c ,d, e, f, g, h) \ + ( ((uint64_t)( (((h) << 24) & 0xFF000000) | \ + (((g) << 16) & 0x00FF0000) | \ + (((f) << 8) & 0x0000FF00) | \ + ((e) & 0x000000FF) ) << 32) | \ + (uint64_t)( (((d) << 24) & 0xFF000000) | \ + (((c) << 16) & 0x00FF0000) | \ + (((b) << 8) & 0x0000FF00) | \ + ((a) & 0x000000FF) ) ) +#define DEBUG_RECORD_ID_SHORT(a,b,c,d) DEBUG_RECORD_ID_LONG(a,b,c,d,0,0,0,0) + +/* + * Shared Memory Console Descriptors: + * Record ID: One per SHMConsole + */ + +typedef enum { + DBG_PROCESSOR_AP = 1, + DBG_COPROCESSOR_ANS, + DBG_COPROCESSOR_SEP, + DBG_COPROCESSOR_SIO, + DBG_COPROCESSOR_ISP, + DBG_COPROCESSOR_OSCAR, + DBG_NUM_PROCESSORS +} dbg_processor_t; + +#define DbgIdConsoleHeaderForIOP(which_dbg_processor, which_num) (DEBUG_RECORD_ID_LONG('C','O','N',0,0,0,which_dbg_processor,which_num)) + +#define kDbgIdConsoleHeaderAP DbgIdConsoleHeaderForIOP(DBG_PROCESSOR_AP, 0) +#define kDbgIdConsoleHeaderANS DbgIdConsoleHeaderForIOP(DBG_COPROCESSOR_ANS, 0) +#define kDbgIdConsoleHeaderSIO DbgIdConsoleHeaderForIOP(DBG_COPROCESSOR_SIO, 0) +#define kDbgIdConsoleHeaderSEP DbgIdConsoleHeaderForIOP(DBG_COPROCESSOR_SEP, 0) +#define kDbgIdConsoleHeaderISP DbgIdConsoleHeaderForIOP(DBG_COPROCESSOR_ISP, 0) +#define kDbgIdConsoleHeaderOscar DbgIdConsoleHeaderForIOP(DBG_COPROCESSOR_OSCAR, 0) + +#define kDbgIdAstrisConnection DEBUG_RECORD_ID_LONG('A','S','T','R','C','N','X','N') +#define kDbgIdAstrisConnectionVers DEBUG_RECORD_ID_LONG('A','S','T','R','C','V','E','R') + +#define kDbgIdUnusedEntry 0x0ULL +#define kDbgIdReservedEntry DEBUG_RECORD_ID_LONG('R','E','S','E','R','V','E', 'D') +#define kDbgIdFreeReqEntry DEBUG_RECORD_ID_LONG('F','R','E','E','-','R','E','Q') +#define kDbgIdFreeAckEntry DEBUG_RECORD_ID_LONG('F','R','E','E','-','A','C','K') + +#define DEBUG_REGISTRY_MAX_RECORDS 512 + +typedef struct { + uint64_t record_id; // = kDbgIdTopLevelHeader + uint32_t num_records; // = DEBUG_REGISTRY_MAX_RECORDS + uint32_t record_size_bytes; // = sizeof(dbg_record_header_t) +} dbg_top_level_header_t; + +typedef struct { + uint64_t record_id; // 32-bit unique ID identifying the record + uint64_t length; // Length of the payload + uint64_t physaddr; // System physical address of entry +} dbg_record_header_t; + +typedef struct { + uint64_t timestamp; + uint32_t cp_state; // One of the cp_state_t enumerations + uint32_t cp_state_arg; // IOP-defined supplemental value +} dbg_cpr_state_entry_t; + +#define CPR_MAX_STATE_ENTRIES 16 // Arbitrary value + +// This second-level struct should be what the Debug Registry record (e.g. kDbgIdCPRHeaderANS) points to. +typedef struct { + uint32_t rdptr; + uint32_t wrptr; + uint32_t num_cp_state_entries; + uint32_t checksum; + dbg_cpr_state_entry_t cp_state_entries[CPR_MAX_STATE_ENTRIES]; +} dbg_cpr_t; + +typedef struct { + dbg_top_level_header_t top_level_header; + dbg_record_header_t records[DEBUG_REGISTRY_MAX_RECORDS]; + + // Stuff the AP's Progress Report buffer at the end of this + // structure. It's currently the only processor that doesn't + // have some easier form of persistent memory that survives the + // iBoot->iOS handoff (e.g. ANS has its private heap) + dbg_cpr_t ap_cpr_region; +} dbg_registry_t; + +/* + * Inherit the consistent debug structure from bootloader + */ +int PE_consistent_debug_inherit(void); + +/* + * Register a region in the consistent debug structure + */ +int PE_consistent_debug_register(uint64_t record_id, uint64_t physaddr, uint64_t length); + +/* + * Returns whether consistent debug is enabled on the current device. + */ +int PE_consistent_debug_enabled(void); + +#endif // PE_CONSISTENT_DEBUG_H + diff --git a/pexpert/pexpert/arm/protos.h b/pexpert/pexpert/arm/protos.h new file mode 100644 index 000000000..26b7aece6 --- /dev/null +++ b/pexpert/pexpert/arm/protos.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + */ +#ifndef _PEXPERT_ARM_PROTOS_H +#define _PEXPERT_ARM_PROTOS_H + +#if defined __arm64__ +#define SHMCON 1 +#endif + +extern vm_offset_t pe_arm_get_soc_base_phys(void); +extern uint32_t pe_arm_get_soc_revision(void); +extern uint32_t pe_arm_init_interrupts(void *args); +extern void pe_arm_init_debug(void *args); + + +#ifdef PEXPERT_KERNEL_PRIVATE +extern void cnputc(char); +#endif +int serial_init(void); +int serial_getc(void); +void serial_putc(char); +void uart_putc(char); +int uart_getc(void); + +int switch_to_serial_console(void); +void switch_to_old_console(int); + +__BEGIN_DECLS +int pe_shmcon_set_child(uint64_t paddr, uint32_t entry); +__END_DECLS + +#endif /* _PEXPERT_ARM_PROTOS_H */ diff --git a/pexpert/pexpert/arm64/AIC.h b/pexpert/pexpert/arm64/AIC.h new file mode 100644 index 000000000..db5faabe3 --- /dev/null +++ b/pexpert/pexpert/arm64/AIC.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_AIC_H +#define _PEXPERT_ARM_AIC_H + +#ifndef ASSEMBLER + +#include + +static inline uint32_t _aic_read32(uintptr_t addr) +{ + return (*(volatile uint32_t *)addr); +} + +static inline void _aic_write32(uintptr_t addr, uint32_t data) +{ + *(volatile uint32_t *)(addr) = data; +} + +#define aic_read32(offset, data) (_aic_read32(pic_base + (offset))) +#define aic_write32(offset, data) (_aic_write32(pic_base + (offset), (data))) + +#endif + +// AIC timebase registers (timer base address in DT node is setup as AIC_BASE + 0x1000) +#define kAICMainTimLo (0x20) +#define kAICMainTimHi (0x28) + +#endif /* ! _PEXPERT_ARM_AIC_H */ + diff --git a/pexpert/pexpert/arm64/AMCC.h b/pexpert/pexpert/arm64/AMCC.h new file mode 100644 index 000000000..2e7f3d8be --- /dev/null +++ b/pexpert/pexpert/arm64/AMCC.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_AMCC_H +#define _PEXPERT_ARM_AMCC_H + +/* + * AMCC registers for KTRR/RoRegion related lockdown in early kernel bootstrap. + * amcc_base must be retrieved from device tree before using. + */ + +//#if defined(KERNEL_INTEGRITY_KTRR) +#define rMCCGEN (*(volatile uint32_t *) (amcc_base + 0x780)) +#define rRORGNBASEADDR (*(volatile uint32_t *) (amcc_base + 0x7e4)) +#define rRORGNENDADDR (*(volatile uint32_t *) (amcc_base + 0x7e8)) +#define rRORGNLOCK (*(volatile uint32_t *) (amcc_base + 0x7ec)) +//#endif + + +#endif /* _PEXPERT_ARM_AMCC_H */ diff --git a/pexpert/pexpert/arm64/Makefile b/pexpert/pexpert/arm64/Makefile new file mode 100644 index 000000000..6bdb8fc40 --- /dev/null +++ b/pexpert/pexpert/arm64/Makefile @@ -0,0 +1,34 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +DATAFILES = \ + AIC.h \ + arm64_common.h \ + board_config.h \ + boot.h \ + S3c2410x.h \ + S5L8960X.h \ + T7000.h \ + S8000.h \ + T8010.h \ + cyclone.h \ + typhoon.h \ + twister.h \ + hurricane.h + + +INSTALL_MD_LIST = ${DATAFILES} + +INSTALL_MD_DIR = pexpert/arm64 + +EXPORT_MD_LIST = ${DATAFILES} + +EXPORT_MD_DIR = pexpert/arm64 + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/pexpert/pexpert/arm64/S3c2410x.h b/pexpert/pexpert/arm64/S3c2410x.h new file mode 100644 index 000000000..a56cb781d --- /dev/null +++ b/pexpert/pexpert/arm64/S3c2410x.h @@ -0,0 +1,544 @@ +/* + * Copyright (c) 2005-2007 Apple Inc. All rights reserved. + */ + +//============================================================================= +// File Name : 2410addr.h +// Function : S3C2410 Define Address Register +// Program : Shin, On Pil (SOP) +// Date : May 06, 2002 +// Version : 0.0 +// History +// 0.0 : Programming start (February 15,2002) -> SOP +// INTERRUPT rPRIORITY 0x4a00000a -> 0x4a00000c (May 02, 2002 SOP) +// RTC BCD DAY and DATE Register Name Correction (May 06, 2002 SOP) +//============================================================================= + +#ifndef __2410ADDR_H__ +#define __2410ADDR_H__ + +#define ARM_BOARD_CONFIG_S3C2410 + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#if 0 +#define _ISR_STARTADDRESS 0x30000000 + +// Memory control +#define rBWSCON (*(volatile unsigned *)0x48000000) //Bus width & wait status +#define rBANKCON0 (*(volatile unsigned *)0x48000004) //Boot ROM control +#define rBANKCON1 (*(volatile unsigned *)0x48000008) //BANK1 control +#define rBANKCON2 (*(volatile unsigned *)0x4800000c) //BANK2 cControl +#define rBANKCON3 (*(volatile unsigned *)0x48000010) //BANK3 control +#define rBANKCON4 (*(volatile unsigned *)0x48000014) //BANK4 control +#define rBANKCON5 (*(volatile unsigned *)0x48000018) //BANK5 control +#define rBANKCON6 (*(volatile unsigned *)0x4800001c) //BANK6 control +#define rBANKCON7 (*(volatile unsigned *)0x48000020) //BANK7 control +#define rREFRESH (*(volatile unsigned *)0x48000024) //DRAM/SDRAM refresh +#define rBANKSIZE (*(volatile unsigned *)0x48000028) //Flexible Bank Size +#define rMRSRB6 (*(volatile unsigned *)0x4800002c) //Mode register set for SDRAM +#define rMRSRB7 (*(volatile unsigned *)0x48000030) //Mode register set for SDRAM + + +// USB Host +#endif + +// INTERRUPT +#define rSRCPND (*(volatile unsigned *)(pic_base + 0x00)) //Interrupt request status +#define rINTMOD (*(volatile unsigned *)(pic_base + 0x04)) //Interrupt mode control +#define rINTMSK (*(volatile unsigned *)(pic_base + 0x08)) //Interrupt mask control +#define rPRIORITY (*(volatile unsigned *)(pic_base + 0x0c)) //IRQ priority control +#define rINTPND (*(volatile unsigned *)(pic_base + 0x10)) //Interrupt request status +#define rINTOFFSET (*(volatile unsigned *)(pic_base + 0x14)) //Interruot request source offset +#define rSUBSRCPND (*(volatile unsigned *)(pic_base + 0x18)) //Sub source pending +#define rINTSUBMSK (*(volatile unsigned *)(pic_base + 0x1c)) //Interrupt sub mask + +#if 0 +// DMA +#define rDISRC0 (*(volatile unsigned *)0x4b000000) //DMA 0 Initial source +#define rDISRCC0 (*(volatile unsigned *)0x4b000004) //DMA 0 Initial source control +#define rDIDST0 (*(volatile unsigned *)0x4b000008) //DMA 0 Initial Destination +#define rDIDSTC0 (*(volatile unsigned *)0x4b00000c) //DMA 0 Initial Destination control +#define rDCON0 (*(volatile unsigned *)0x4b000010) //DMA 0 Control +#define rDSTAT0 (*(volatile unsigned *)0x4b000014) //DMA 0 Status +#define rDCSRC0 (*(volatile unsigned *)0x4b000018) //DMA 0 Current source +#define rDCDST0 (*(volatile unsigned *)0x4b00001c) //DMA 0 Current destination +#define rDMASKTRIG0 (*(volatile unsigned *)0x4b000020) //DMA 0 Mask trigger + +#define rDISRC1 (*(volatile unsigned *)0x4b000040) //DMA 1 Initial source +#define rDISRCC1 (*(volatile unsigned *)0x4b000044) //DMA 1 Initial source control +#define rDIDST1 (*(volatile unsigned *)0x4b000048) //DMA 1 Initial Destination +#define rDIDSTC1 (*(volatile unsigned *)0x4b00004c) //DMA 1 Initial Destination control +#define rDCON1 (*(volatile unsigned *)0x4b000050) //DMA 1 Control +#define rDSTAT1 (*(volatile unsigned *)0x4b000054) //DMA 1 Status +#define rDCSRC1 (*(volatile unsigned *)0x4b000058) //DMA 1 Current source +#define rDCDST1 (*(volatile unsigned *)0x4b00005c) //DMA 1 Current destination +#define rDMASKTRIG1 (*(volatile unsigned *)0x4b000060) //DMA 1 Mask trigger + +#define rDISRC2 (*(volatile unsigned *)0x4b000080) //DMA 2 Initial source +#define rDISRCC2 (*(volatile unsigned *)0x4b000084) //DMA 2 Initial source control +#define rDIDST2 (*(volatile unsigned *)0x4b000088) //DMA 2 Initial Destination +#define rDIDSTC2 (*(volatile unsigned *)0x4b00008c) //DMA 2 Initial Destination control +#define rDCON2 (*(volatile unsigned *)0x4b000090) //DMA 2 Control +#define rDSTAT2 (*(volatile unsigned *)0x4b000094) //DMA 2 Status +#define rDCSRC2 (*(volatile unsigned *)0x4b000098) //DMA 2 Current source +#define rDCDST2 (*(volatile unsigned *)0x4b00009c) //DMA 2 Current destination +#define rDMASKTRIG2 (*(volatile unsigned *)0x4b0000a0) //DMA 2 Mask trigger + +#define rDISRC3 (*(volatile unsigned *)0x4b0000c0) //DMA 3 Initial source +#define rDISRCC3 (*(volatile unsigned *)0x4b0000c4) //DMA 3 Initial source control +#define rDIDST3 (*(volatile unsigned *)0x4b0000c8) //DMA 3 Initial Destination +#define rDIDSTC3 (*(volatile unsigned *)0x4b0000cc) //DMA 3 Initial Destination control +#define rDCON3 (*(volatile unsigned *)0x4b0000d0) //DMA 3 Control +#define rDSTAT3 (*(volatile unsigned *)0x4b0000d4) //DMA 3 Status +#define rDCSRC3 (*(volatile unsigned *)0x4b0000d8) //DMA 3 Current source +#define rDCDST3 (*(volatile unsigned *)0x4b0000dc) //DMA 3 Current destination +#define rDMASKTRIG3 (*(volatile unsigned *)0x4b0000e0) //DMA 3 Mask trigger + + +// CLOCK & POWER MANAGEMENT +#define rLOCKTIME (*(volatile unsigned *)0x4c000000) //PLL lock time counter +#define rMPLLCON (*(volatile unsigned *)0x4c000004) //MPLL Control +#define rUPLLCON (*(volatile unsigned *)0x4c000008) //UPLL Control +#define rCLKCON (*(volatile unsigned *)0x4c00000c) //Clock generator control +#define rCLKSLOW (*(volatile unsigned *)0x4c000010) //Slow clock control +#define rCLKDIVN (*(volatile unsigned *)0x4c000014) //Clock divider control + + +// LCD CONTROLLER +#define rLCDCON1 (*(volatile unsigned *)0x4d000000) //LCD control 1 +#define rLCDCON2 (*(volatile unsigned *)0x4d000004) //LCD control 2 +#define rLCDCON3 (*(volatile unsigned *)0x4d000008) //LCD control 3 +#define rLCDCON4 (*(volatile unsigned *)0x4d00000c) //LCD control 4 +#define rLCDCON5 (*(volatile unsigned *)0x4d000010) //LCD control 5 +#define rLCDSADDR1 (*(volatile unsigned *)0x4d000014) //STN/TFT Frame buffer start address 1 +#define rLCDSADDR2 (*(volatile unsigned *)0x4d000018) //STN/TFT Frame buffer start address 2 +#define rLCDSADDR3 (*(volatile unsigned *)0x4d00001c) //STN/TFT Virtual screen address set +#define rREDLUT (*(volatile unsigned *)0x4d000020) //STN Red lookup table +#define rGREENLUT (*(volatile unsigned *)0x4d000024) //STN Green lookup table +#define rBLUELUT (*(volatile unsigned *)0x4d000028) //STN Blue lookup table +#define rDITHMODE (*(volatile unsigned *)0x4d00004c) //STN Dithering mode +#define rTPAL (*(volatile unsigned *)0x4d000050) //TFT Temporary palette +#define rLCDINTPND (*(volatile unsigned *)0x4d000054) //LCD Interrupt pending +#define rLCDSRCPND (*(volatile unsigned *)0x4d000058) //LCD Interrupt source +#define rLCDINTMSK (*(volatile unsigned *)0x4d00005c) //LCD Interrupt mask +#define rLPCSEL (*(volatile unsigned *)0x4d000060) //LPC3600 Control +#define PALETTE 0x4d000400 //Palette start address + + +// NAND flash +#define rNFCONF (*(volatile unsigned *)0x4e000000) //NAND Flash configuration +#define rNFCMD (*(volatile U8 *)0x4e000004) //NADD Flash command +#define rNFADDR (*(volatile U8 *)0x4e000008) //NAND Flash address +#define rNFDATA (*(volatile U8 *)0x4e00000c) //NAND Flash data +#define rNFSTAT (*(volatile unsigned *)0x4e000010) //NAND Flash operation status +#define rNFECC (*(volatile unsigned *)0x4e000014) //NAND Flash ECC +#define rNFECC0 (*(volatile U8 *)0x4e000014) +#define rNFECC1 (*(volatile U8 *)0x4e000015) +#define rNFECC2 (*(volatile U8 *)0x4e000016) +#endif + +// PWM TIMER +#define rTCFG0 (*(volatile unsigned *)(timer_base + 0x00)) //Timer 0 configuration +#define rTCFG1 (*(volatile unsigned *)(timer_base + 0x04)) //Timer 1 configuration +#define rTCON (*(volatile unsigned *)(timer_base + 0x08)) //Timer control +#define rTCNTB0 (*(volatile unsigned *)(timer_base + 0x0c)) //Timer count buffer 0 +#define rTCMPB0 (*(volatile unsigned *)(timer_base + 0x10)) //Timer compare buffer 0 +#define rTCNTO0 (*(volatile unsigned *)(timer_base + 0x14)) //Timer count observation 0 +#define rTCNTB1 (*(volatile unsigned *)(timer_base + 0x18)) //Timer count buffer 1 +#define rTCMPB1 (*(volatile unsigned *)(timer_base + 0x1c)) //Timer compare buffer 1 +#define rTCNTO1 (*(volatile unsigned *)(timer_base + 0x20)) //Timer count observation 1 +#define rTCNTB2 (*(volatile unsigned *)(timer_base + 0x24)) //Timer count buffer 2 +#define rTCMPB2 (*(volatile unsigned *)(timer_base + 0x28)) //Timer compare buffer 2 +#define rTCNTO2 (*(volatile unsigned *)(timer_base + 0x2c)) //Timer count observation 2 +#define rTCNTB3 (*(volatile unsigned *)(timer_base + 0x30)) //Timer count buffer 3 +#define rTCMPB3 (*(volatile unsigned *)(timer_base + 0x34)) //Timer compare buffer 3 +#define rTCNTO3 (*(volatile unsigned *)(timer_base + 0x38)) //Timer count observation 3 +#define rTCNTB4 (*(volatile unsigned *)(timer_base + 0x3c)) //Timer count buffer 4 +#define rTCNTO4 (*(volatile unsigned *)(timer_base + 0x40)) //Timer count observation 4 +#define rTCNTCLRINT0 (*(volatile unsigned *)(timer_base + 0x44)) //Timer0 Interrupt Clear Register +#define rTCNTCLRINT1 (*(volatile unsigned *)(timer_base + 0x48)) //Timer0 Interrupt Clear Register +#define rTCNTCLRINT2 (*(volatile unsigned *)(timer_base + 0x4C)) //Timer0 Interrupt Clear Register +#define rTCNTCLRINT3 (*(volatile unsigned *)(timer_base + 0x54)) //Timer0 Interrupt Clear Register +#define rTCNTCLRINT4 (*(volatile unsigned *)(timer_base + 0x54)) //Timer0 Interrupt Clear Register + + +#if 0 +// USB DEVICE +#ifdef __BIG_ENDIAN + +#define rFUNC_ADDR_REG (*(volatile unsigned char *)0x52000143) //Function address +#define rPWR_REG (*(volatile unsigned char *)0x52000147) //Power management +#define rEP_INT_REG (*(volatile unsigned char *)0x5200014b) //EP Interrupt pending and clear +#define rUSB_INT_REG (*(volatile unsigned char *)0x5200015b) //USB Interrupt pending and clear +#define rEP_INT_EN_REG (*(volatile unsigned char *)0x5200015f) //Interrupt enable +#define rUSB_INT_EN_REG (*(volatile unsigned char *)0x5200016f) +#define rFRAME_NUM1_REG (*(volatile unsigned char *)0x52000173) //Frame number lower byte +#define rFRAME_NUM2_REG (*(volatile unsigned char *)0x52000177) //Frame number higher byte +#define rINDEX_REG (*(volatile unsigned char *)0x5200017b) //Register index +#define rMAXP_REG (*(volatile unsigned char *)0x52000183) //Endpoint max packet +#define rEP0_CSR (*(volatile unsigned char *)0x52000187) //Endpoint 0 status +#define rIN_CSR1_REG (*(volatile unsigned char *)0x52000187) //In endpoint control status +#define rIN_CSR2_REG (*(volatile unsigned char *)0x5200018b) +#define rOUT_CSR1_REG (*(volatile unsigned char *)0x52000193) //Out endpoint control status +#define rOUT_CSR2_REG (*(volatile unsigned char *)0x52000197) +#define rOUT_FIFO_CNT1_REG (*(volatile unsigned char *)0x5200019b) //Endpoint out write count +#define rOUT_FIFO_CNT2_REG (*(volatile unsigned char *)0x5200019f) +#define rEP0_FIFO (*(volatile unsigned char *)0x520001c3) //Endpoint 0 FIFO +#define rEP1_FIFO (*(volatile unsigned char *)0x520001c7) //Endpoint 1 FIFO +#define rEP2_FIFO (*(volatile unsigned char *)0x520001cb) //Endpoint 2 FIFO +#define rEP3_FIFO (*(volatile unsigned char *)0x520001cf) //Endpoint 3 FIFO +#define rEP4_FIFO (*(volatile unsigned char *)0x520001d3) //Endpoint 4 FIFO +#define rEP1_DMA_CON (*(volatile unsigned char *)0x52000203) //EP1 DMA interface control +#define rEP1_DMA_UNIT (*(volatile unsigned char *)0x52000207) //EP1 DMA Tx unit counter +#define rEP1_DMA_FIFO (*(volatile unsigned char *)0x5200020b) //EP1 DMA Tx FIFO counter +#define rEP1_DMA_TTC_L (*(volatile unsigned char *)0x5200020f) //EP1 DMA total Tx counter +#define rEP1_DMA_TTC_M (*(volatile unsigned char *)0x52000213) +#define rEP1_DMA_TTC_H (*(volatile unsigned char *)0x52000217) +#define rEP2_DMA_CON (*(volatile unsigned char *)0x5200021b) //EP2 DMA interface control +#define rEP2_DMA_UNIT (*(volatile unsigned char *)0x5200021f) //EP2 DMA Tx unit counter +#define rEP2_DMA_FIFO (*(volatile unsigned char *)0x52000223) //EP2 DMA Tx FIFO counter +#define rEP2_DMA_TTC_L (*(volatile unsigned char *)0x52000227) //EP2 DMA total Tx counter +#define rEP2_DMA_TTC_M (*(volatile unsigned char *)0x5200022b) +#define rEP2_DMA_TTC_H (*(volatile unsigned char *)0x5200022f) +#define rEP3_DMA_CON (*(volatile unsigned char *)0x52000243) //EP3 DMA interface control +#define rEP3_DMA_UNIT (*(volatile unsigned char *)0x52000247) //EP3 DMA Tx unit counter +#define rEP3_DMA_FIFO (*(volatile unsigned char *)0x5200024b) //EP3 DMA Tx FIFO counter +#define rEP3_DMA_TTC_L (*(volatile unsigned char *)0x5200024f) //EP3 DMA total Tx counter +#define rEP3_DMA_TTC_M (*(volatile unsigned char *)0x52000253) +#define rEP3_DMA_TTC_H (*(volatile unsigned char *)0x52000257) +#define rEP4_DMA_CON (*(volatile unsigned char *)0x5200025b) //EP4 DMA interface control +#define rEP4_DMA_UNIT (*(volatile unsigned char *)0x5200025f) //EP4 DMA Tx unit counter +#define rEP4_DMA_FIFO (*(volatile unsigned char *)0x52000263) //EP4 DMA Tx FIFO counter +#define rEP4_DMA_TTC_L (*(volatile unsigned char *)0x52000267) //EP4 DMA total Tx counter +#define rEP4_DMA_TTC_M (*(volatile unsigned char *)0x5200026b) +#define rEP4_DMA_TTC_H (*(volatile unsigned char *)0x5200026f) + +#else // Little Endian +#define rFUNC_ADDR_REG (*(volatile unsigned char *)0x52000140) //Function address +#define rPWR_REG (*(volatile unsigned char *)0x52000144) //Power management +#define rEP_INT_REG (*(volatile unsigned char *)0x52000148) //EP Interrupt pending and clear +#define rUSB_INT_REG (*(volatile unsigned char *)0x52000158) //USB Interrupt pending and clear +#define rEP_INT_EN_REG (*(volatile unsigned char *)0x5200015c) //Interrupt enable +#define rUSB_INT_EN_REG (*(volatile unsigned char *)0x5200016c) +#define rFRAME_NUM1_REG (*(volatile unsigned char *)0x52000170) //Frame number lower byte +#define rFRAME_NUM2_REG (*(volatile unsigned char *)0x52000174) //Frame number higher byte +#define rINDEX_REG (*(volatile unsigned char *)0x52000178) //Register index +#define rMAXP_REG (*(volatile unsigned char *)0x52000180) //Endpoint max packet +#define rEP0_CSR (*(volatile unsigned char *)0x52000184) //Endpoint 0 status +#define rIN_CSR1_REG (*(volatile unsigned char *)0x52000184) //In endpoint control status +#define rIN_CSR2_REG (*(volatile unsigned char *)0x52000188) +#define rOUT_CSR1_REG (*(volatile unsigned char *)0x52000190) //Out endpoint control status +#define rOUT_CSR2_REG (*(volatile unsigned char *)0x52000194) +#define rOUT_FIFO_CNT1_REG (*(volatile unsigned char *)0x52000198) //Endpoint out write count +#define rOUT_FIFO_CNT2_REG (*(volatile unsigned char *)0x5200019c) +#define rEP0_FIFO (*(volatile unsigned char *)0x520001c0) //Endpoint 0 FIFO +#define rEP1_FIFO (*(volatile unsigned char *)0x520001c4) //Endpoint 1 FIFO +#define rEP2_FIFO (*(volatile unsigned char *)0x520001c8) //Endpoint 2 FIFO +#define rEP3_FIFO (*(volatile unsigned char *)0x520001cc) //Endpoint 3 FIFO +#define rEP4_FIFO (*(volatile unsigned char *)0x520001d0) //Endpoint 4 FIFO +#define rEP1_DMA_CON (*(volatile unsigned char *)0x52000200) //EP1 DMA interface control +#define rEP1_DMA_UNIT (*(volatile unsigned char *)0x52000204) //EP1 DMA Tx unit counter +#define rEP1_DMA_FIFO (*(volatile unsigned char *)0x52000208) //EP1 DMA Tx FIFO counter +#define rEP1_DMA_TTC_L (*(volatile unsigned char *)0x5200020c) //EP1 DMA total Tx counter +#define rEP1_DMA_TTC_M (*(volatile unsigned char *)0x52000210) +#define rEP1_DMA_TTC_H (*(volatile unsigned char *)0x52000214) +#define rEP2_DMA_CON (*(volatile unsigned char *)0x52000218) //EP2 DMA interface control +#define rEP2_DMA_UNIT (*(volatile unsigned char *)0x5200021c) //EP2 DMA Tx unit counter +#define rEP2_DMA_FIFO (*(volatile unsigned char *)0x52000220) //EP2 DMA Tx FIFO counter +#define rEP2_DMA_TTC_L (*(volatile unsigned char *)0x52000224) //EP2 DMA total Tx counter +#define rEP2_DMA_TTC_M (*(volatile unsigned char *)0x52000228) +#define rEP2_DMA_TTC_H (*(volatile unsigned char *)0x5200022c) +#define rEP3_DMA_CON (*(volatile unsigned char *)0x52000240) //EP3 DMA interface control +#define rEP3_DMA_UNIT (*(volatile unsigned char *)0x52000244) //EP3 DMA Tx unit counter +#define rEP3_DMA_FIFO (*(volatile unsigned char *)0x52000248) //EP3 DMA Tx FIFO counter +#define rEP3_DMA_TTC_L (*(volatile unsigned char *)0x5200024c) //EP3 DMA total Tx counter +#define rEP3_DMA_TTC_M (*(volatile unsigned char *)0x52000250) +#define rEP3_DMA_TTC_H (*(volatile unsigned char *)0x52000254) +#define rEP4_DMA_CON (*(volatile unsigned char *)0x52000258) //EP4 DMA interface control +#define rEP4_DMA_UNIT (*(volatile unsigned char *)0x5200025c) //EP4 DMA Tx unit counter +#define rEP4_DMA_FIFO (*(volatile unsigned char *)0x52000260) //EP4 DMA Tx FIFO counter +#define rEP4_DMA_TTC_L (*(volatile unsigned char *)0x52000264) //EP4 DMA total Tx counter +#define rEP4_DMA_TTC_M (*(volatile unsigned char *)0x52000268) +#define rEP4_DMA_TTC_H (*(volatile unsigned char *)0x5200026c) +#endif // __BIG_ENDIAN + + +// WATCH DOG TIMER +#define rWTCON (*(volatile unsigned *)0x53000000) //Watch-dog timer mode +#define rWTDAT (*(volatile unsigned *)0x53000004) //Watch-dog timer data +#define rWTCNT (*(volatile unsigned *)0x53000008) //Eatch-dog timer count + + +// IIC +#define rIICCON (*(volatile unsigned *)0x54000000) //IIC control +#define rIICSTAT (*(volatile unsigned *)0x54000004) //IIC status +#define rIICADD (*(volatile unsigned *)0x54000008) //IIC address +#define rIICDS (*(volatile unsigned *)0x5400000c) //IIC data shift + + +// IIS +#define rIISCON (*(volatile unsigned *)0x55000000) //IIS Control +#define rIISMOD (*(volatile unsigned *)0x55000004) //IIS Mode +#define rIISPSR (*(volatile unsigned *)0x55000008) //IIS Prescaler +#define rIISFCON (*(volatile unsigned *)0x5500000c) //IIS FIFO control + +#ifdef __BIG_ENDIAN +#define IISFIFO ((volatile unsigned short *)0x55000012) //IIS FIFO entry + +#else //Little Endian +#define IISFIFO ((volatile unsigned short *)0x55000010) //IIS FIFO entry + +#endif + + +// I/O PORT +#define rGPACON (*(volatile unsigned *)0x56000000) //Port A control +#define rGPADAT (*(volatile unsigned *)0x56000004) //Port A data + +#define rGPBCON (*(volatile unsigned *)0x56000010) //Port B control +#define rGPBDAT (*(volatile unsigned *)0x56000014) //Port B data +#define rGPBUP (*(volatile unsigned *)0x56000018) //Pull-up control B + +#define rGPCCON (*(volatile unsigned *)0x56000020) //Port C control +#define rGPCDAT (*(volatile unsigned *)0x56000024) //Port C data +#define rGPCUP (*(volatile unsigned *)0x56000028) //Pull-up control C + +#define rGPDCON (*(volatile unsigned *)0x56000030) //Port D control +#define rGPDDAT (*(volatile unsigned *)0x56000034) //Port D data +#define rGPDUP (*(volatile unsigned *)0x56000038) //Pull-up control D + +#define rGPECON (*(volatile unsigned *)0x56000040) //Port E control +#define rGPEDAT (*(volatile unsigned *)0x56000044) //Port E data +#define rGPEUP (*(volatile unsigned *)0x56000048) //Pull-up control E + +#define rGPFCON (*(volatile unsigned *)0x56000050) //Port F control +#define rGPFDAT (*(volatile unsigned *)0x56000054) //Port F data +#define rGPFUP (*(volatile unsigned *)0x56000058) //Pull-up control F + +#define rGPGCON (*(volatile unsigned *)0x56000060) //Port G control +#define rGPGDAT (*(volatile unsigned *)0x56000064) //Port G data +#define rGPGUP (*(volatile unsigned *)0x56000068) //Pull-up control G + +#define rGPHCON (*(volatile unsigned *)0x56000070) //Port H control +#define rGPHDAT (*(volatile unsigned *)0x56000074) //Port H data +#define rGPHUP (*(volatile unsigned *)0x56000078) //Pull-up control H + +#define rMISCCR (*(volatile unsigned *)0x56000080) //Miscellaneous control +#define rDCLKCON (*(volatile unsigned *)0x56000084) //DCLK0/1 control +#define rEXTINT0 (*(volatile unsigned *)0x56000088) //External interrupt control register 0 +#define rEXTINT1 (*(volatile unsigned *)0x5600008c) //External interrupt control register 1 +#define rEXTINT2 (*(volatile unsigned *)0x56000090) //External interrupt control register 2 +#define rEINTFLT0 (*(volatile unsigned *)0x56000094) //Reserved +#define rEINTFLT1 (*(volatile unsigned *)0x56000098) //Reserved +#define rEINTFLT2 (*(volatile unsigned *)0x5600009c) //External interrupt filter control register 2 +#define rEINTFLT3 (*(volatile unsigned *)0x560000a0) //External interrupt filter control register 3 +#define rEINTMASK (*(volatile unsigned *)0x560000a4) //External interrupt mask +#define rEINTPEND (*(volatile unsigned *)0x560000a8) //External interrupt pending +#define rGSTATUS0 (*(volatile unsigned *)0x560000ac) //External pin status +#define rGSTATUS1 (*(volatile unsigned *)0x560000b0) //Chip ID(0x32410000) +#define rGSTATUS2 (*(volatile unsigned *)0x560000b4) //Reset type +#define rGSTATUS3 (*(volatile unsigned *)0x560000b8) //Saved data0(32-bit) before entering POWER_OFF mode +#define rGSTATUS4 (*(volatile unsigned *)0x560000bc) //Saved data0(32-bit) before entering POWER_OFF mode + + +// RTC +#ifdef __BIG_ENDIAN +#define rRTCCON (*(volatile unsigned char *)0x57000043) //RTC control +#define rTICNT (*(volatile unsigned char *)0x57000047) //Tick time count +#define rRTCALM (*(volatile unsigned char *)0x57000053) //RTC alarm control +#define rALMSEC (*(volatile unsigned char *)0x57000057) //Alarm second +#define rALMMIN (*(volatile unsigned char *)0x5700005b) //Alarm minute +#define rALMHOUR (*(volatile unsigned char *)0x5700005f) //Alarm Hour +#define rALMDATE (*(volatile unsigned char *)0x57000063) //Alarm day <-- May 06, 2002 SOP +#define rALMMON (*(volatile unsigned char *)0x57000067) //Alarm month +#define rALMYEAR (*(volatile unsigned char *)0x5700006b) //Alarm year +#define rRTCRST (*(volatile unsigned char *)0x5700006f) //RTC round reset +#define rBCDSEC (*(volatile unsigned char *)0x57000073) //BCD second +#define rBCDMIN (*(volatile unsigned char *)0x57000077) //BCD minute +#define rBCDHOUR (*(volatile unsigned char *)0x5700007b) //BCD hour +#define rBCDDATE (*(volatile unsigned char *)0x5700007f) //BCD day <-- May 06, 2002 SOP +#define rBCDDAY (*(volatile unsigned char *)0x57000083) //BCD date <-- May 06, 2002 SOP +#define rBCDMON (*(volatile unsigned char *)0x57000087) //BCD month +#define rBCDYEAR (*(volatile unsigned char *)0x5700008b) //BCD year + +#else //Little Endian +#define rRTCCON (*(volatile unsigned char *)0x57000040) //RTC control +#define rTICNT (*(volatile unsigned char *)0x57000044) //Tick time count +#define rRTCALM (*(volatile unsigned char *)0x57000050) //RTC alarm control +#define rALMSEC (*(volatile unsigned char *)0x57000054) //Alarm second +#define rALMMIN (*(volatile unsigned char *)0x57000058) //Alarm minute +#define rALMHOUR (*(volatile unsigned char *)0x5700005c) //Alarm Hour +#define rALMDATE (*(volatile unsigned char *)0x57000060) //Alarm day <-- May 06, 2002 SOP +#define rALMMON (*(volatile unsigned char *)0x57000064) //Alarm month +#define rALMYEAR (*(volatile unsigned char *)0x57000068) //Alarm year +#define rRTCRST (*(volatile unsigned char *)0x5700006c) //RTC round reset +#define rBCDSEC (*(volatile unsigned char *)0x57000070) //BCD second +#define rBCDMIN (*(volatile unsigned char *)0x57000074) //BCD minute +#define rBCDHOUR (*(volatile unsigned char *)0x57000078) //BCD hour +#define rBCDDATE (*(volatile unsigned char *)0x5700007c) //BCD day <-- May 06, 2002 SOP +#define rBCDDAY (*(volatile unsigned char *)0x57000080) //BCD date <-- May 06, 2002 SOP +#define rBCDMON (*(volatile unsigned char *)0x57000084) //BCD month +#define rBCDYEAR (*(volatile unsigned char *)0x57000088) //BCD year +#endif //RTC + + +// ADC +#define rADCCON (*(volatile unsigned *)0x58000000) //ADC control +#define rADCTSC (*(volatile unsigned *)0x58000004) //ADC touch screen control +#define rADCDLY (*(volatile unsigned *)0x58000008) //ADC start or Interval Delay +#define rADCDAT0 (*(volatile unsigned *)0x5800000c) //ADC conversion data 0 +#define rADCDAT1 (*(volatile unsigned *)0x58000010) //ADC conversion data 1 + +// SPI +#define rSPCON0 (*(volatile unsigned *)0x59000000) //SPI0 control +#define rSPSTA0 (*(volatile unsigned *)0x59000004) //SPI0 status +#define rSPPIN0 (*(volatile unsigned *)0x59000008) //SPI0 pin control +#define rSPPRE0 (*(volatile unsigned *)0x5900000c) //SPI0 baud rate prescaler +#define rSPTDAT0 (*(volatile unsigned *)0x59000010) //SPI0 Tx data +#define rSPRDAT0 (*(volatile unsigned *)0x59000014) //SPI0 Rx data + +#define rSPCON1 (*(volatile unsigned *)0x59000020) //SPI1 control +#define rSPSTA1 (*(volatile unsigned *)0x59000024) //SPI1 status +#define rSPPIN1 (*(volatile unsigned *)0x59000028) //SPI1 pin control +#define rSPPRE1 (*(volatile unsigned *)0x5900002c) //SPI1 baud rate prescaler +#define rSPTDAT1 (*(volatile unsigned *)0x59000030) //SPI1 Tx data +#define rSPRDAT1 (*(volatile unsigned *)0x59000034) //SPI1 Rx data + + +// SD Interface +#define rSDICON (*(volatile unsigned *)0x5a000000) //SDI control +#define rSDIPRE (*(volatile unsigned *)0x5a000004) //SDI baud rate prescaler +#define rSDICARG (*(volatile unsigned *)0x5a000008) //SDI command argument +#define rSDICCON (*(volatile unsigned *)0x5a00000c) //SDI command control +#define rSDICSTA (*(volatile unsigned *)0x5a000010) //SDI command status +#define rSDIRSP0 (*(volatile unsigned *)0x5a000014) //SDI response 0 +#define rSDIRSP1 (*(volatile unsigned *)0x5a000018) //SDI response 1 +#define rSDIRSP2 (*(volatile unsigned *)0x5a00001c) //SDI response 2 +#define rSDIRSP3 (*(volatile unsigned *)0x5a000020) //SDI response 3 +#define rSDIDTIMER (*(volatile unsigned *)0x5a000024) //SDI data/busy timer +#define rSDIBSIZE (*(volatile unsigned *)0x5a000028) //SDI block size +#define rSDIDCON (*(volatile unsigned *)0x5a00002c) //SDI data control +#define rSDIDCNT (*(volatile unsigned *)0x5a000030) //SDI data remain counter +#define rSDIDSTA (*(volatile unsigned *)0x5a000034) //SDI data status +#define rSDIFSTA (*(volatile unsigned *)0x5a000038) //SDI FIFO status +#define rSDIIMSK (*(volatile unsigned *)0x5a000040) //SDI interrupt mask + +#ifdef __BIG_ENDIAN +#define rSDIDAT (*(volatile unsigned *)0x5a00003f) //SDI data +#define SDIDAT 0x5a00003f +#else // Little Endian +#define rSDIDAT (*(volatile unsigned *)0x5a00003c) //SDI data +#define SDIDAT 0x5a00003c +#endif //SD Interface + + +// ISR +#define pISR_RESET (*(unsigned *)(_ISR_STARTADDRESS+0x0)) +#define pISR_UNDEF (*(unsigned *)(_ISR_STARTADDRESS+0x4)) +#define pISR_SWI (*(unsigned *)(_ISR_STARTADDRESS+0x8)) +#define pISR_PABORT (*(unsigned *)(_ISR_STARTADDRESS+0xc)) +#define pISR_DABORT (*(unsigned *)(_ISR_STARTADDRESS+0x10)) +#define pISR_RESERVED (*(unsigned *)(_ISR_STARTADDRESS+0x14)) +#define pISR_IRQ (*(unsigned *)(_ISR_STARTADDRESS+0x18)) +#define pISR_FIQ (*(unsigned *)(_ISR_STARTADDRESS+0x1c)) + +#define pISR_EINT0 (*(unsigned *)(_ISR_STARTADDRESS+0x20)) +#define pISR_EINT1 (*(unsigned *)(_ISR_STARTADDRESS+0x24)) +#define pISR_EINT2 (*(unsigned *)(_ISR_STARTADDRESS+0x28)) +#define pISR_EINT3 (*(unsigned *)(_ISR_STARTADDRESS+0x2c)) +#define pISR_EINT4_7 (*(unsigned *)(_ISR_STARTADDRESS+0x30)) +#define pISR_EINT8_23 (*(unsigned *)(_ISR_STARTADDRESS+0x34)) +#define pISR_NOTUSED6 (*(unsigned *)(_ISR_STARTADDRESS+0x38)) +#define pISR_BAT_FLT (*(unsigned *)(_ISR_STARTADDRESS+0x3c)) +#define pISR_TICK (*(unsigned *)(_ISR_STARTADDRESS+0x40)) +#define pISR_WDT (*(unsigned *)(_ISR_STARTADDRESS+0x44)) +#define pISR_TIMER0 (*(unsigned *)(_ISR_STARTADDRESS+0x48)) +#define pISR_TIMER1 (*(unsigned *)(_ISR_STARTADDRESS+0x4c)) +#define pISR_TIMER2 (*(unsigned *)(_ISR_STARTADDRESS+0x50)) +#define pISR_TIMER3 (*(unsigned *)(_ISR_STARTADDRESS+0x54)) +#define pISR_TIMER4 (*(unsigned *)(_ISR_STARTADDRESS+0x58)) +#define pISR_UART2 (*(unsigned *)(_ISR_STARTADDRESS+0x5c)) +#define pISR_LCD (*(unsigned *)(_ISR_STARTADDRESS+0x60)) +#define pISR_DMA0 (*(unsigned *)(_ISR_STARTADDRESS+0x64)) +#define pISR_DMA1 (*(unsigned *)(_ISR_STARTADDRESS+0x68)) +#define pISR_DMA2 (*(unsigned *)(_ISR_STARTADDRESS+0x6c)) +#define pISR_DMA3 (*(unsigned *)(_ISR_STARTADDRESS+0x70)) +#define pISR_SDI (*(unsigned *)(_ISR_STARTADDRESS+0x74)) +#define pISR_SPI0 (*(unsigned *)(_ISR_STARTADDRESS+0x78)) +#define pISR_UART1 (*(unsigned *)(_ISR_STARTADDRESS+0x7c)) +#define pISR_NOTUSED24 (*(unsigned *)(_ISR_STARTADDRESS+0x80)) +#define pISR_USBD (*(unsigned *)(_ISR_STARTADDRESS+0x84)) +#define pISR_USBH (*(unsigned *)(_ISR_STARTADDRESS+0x88)) +#define pISR_IIC (*(unsigned *)(_ISR_STARTADDRESS+0x8c)) +#define pISR_UART0 (*(unsigned *)(_ISR_STARTADDRESS+0x90)) +#define pISR_SPI1 (*(unsigned *)(_ISR_STARTADDRESS+0x94)) +#define pISR_RTC (*(unsigned *)(_ISR_STARTADDRESS+0x98)) +#define pISR_ADC (*(unsigned *)(_ISR_STARTADDRESS+0x9c)) + + +// PENDING BIT +#define BIT_EINT0 (0x1) +#define BIT_EINT1 (0x1<<1) +#define BIT_EINT2 (0x1<<2) +#define BIT_EINT3 (0x1<<3) +#define BIT_EINT4_7 (0x1<<4) +#define BIT_EINT8_23 (0x1<<5) +#define BIT_NOTUSED6 (0x1<<6) +#define BIT_BAT_FLT (0x1<<7) +#define BIT_TICK (0x1<<8) +#define BIT_WDT (0x1<<9) +#define BIT_TIMER0 (0x1<<10) +#define BIT_TIMER1 (0x1<<11) +#define BIT_TIMER2 (0x1<<12) +#define BIT_TIMER3 (0x1<<13) +#define BIT_TIMER4 (0x1<<14) +#define BIT_UART2 (0x1<<15) +#define BIT_LCD (0x1<<16) +#define BIT_DMA0 (0x1<<17) +#define BIT_DMA1 (0x1<<18) +#define BIT_DMA2 (0x1<<19) +#define BIT_DMA3 (0x1<<20) +#define BIT_SDI (0x1<<21) +#define BIT_SPI0 (0x1<<22) +#define BIT_UART1 (0x1<<23) +#define BIT_NOTUSED24 (0x1<<24) +#define BIT_USBD (0x1<<25) +#define BIT_USBH (0x1<<26) +#define BIT_IIC (0x1<<27) +#define BIT_UART0 (0x1<<28) +#define BIT_SPI1 (0x1<<29) +#define BIT_RTC (0x1<<30) +#define BIT_ADC (0x1<<31) +#define BIT_ALLMSK (0xffffffff) + +#define BIT_SUB_ALLMSK (0x7ff) +#define BIT_SUB_ADC (0x1<<10) +#define BIT_SUB_TC (0x1<<9) +#define BIT_SUB_ERR2 (0x1<<8) +#define BIT_SUB_TXD2 (0x1<<7) +#define BIT_SUB_RXD2 (0x1<<6) +#define BIT_SUB_ERR1 (0x1<<5) +#define BIT_SUB_TXD1 (0x1<<4) +#define BIT_SUB_RXD1 (0x1<<3) +#define BIT_SUB_ERR0 (0x1<<2) +#define BIT_SUB_TXD0 (0x1<<1) +#define BIT_SUB_RXD0 (0x1<<0) + +#define ClearPending(bit) {\ + rSRCPND = bit;\ + rINTPND = bit;\ + rINTPND;\ + } +//Wait until rINTPND is changed for the case that the ISR is very short. + +#endif + +#ifdef __cplusplus +} +#endif +#endif //__2410ADDR_H___ diff --git a/pexpert/pexpert/arm64/S5L8960X.h b/pexpert/pexpert/arm64/S5L8960X.h new file mode 100644 index 000000000..782481ff2 --- /dev/null +++ b/pexpert/pexpert/arm64/S5L8960X.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2011 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_S5L8960X_H +#define _PEXPERT_ARM_S5L8960X_H + +#include +#include + +#define WITH_CLASSIC_S2R 1 + +#ifndef ASSEMBLER + +#include + +#endif + +#endif /* ! _PEXPERT_ARM_S5L8960X_H */ diff --git a/pexpert/pexpert/arm64/S8000.h b/pexpert/pexpert/arm64/S8000.h new file mode 100644 index 000000000..1879560eb --- /dev/null +++ b/pexpert/pexpert/arm64/S8000.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2014 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_S8000_H +#define _PEXPERT_ARM_S8000_H + +#include +#include + +#ifndef ASSEMBLER + +#include + +#endif + +#endif /* ! _PEXPERT_ARM_S8000_H */ diff --git a/pexpert/pexpert/arm64/T7000.h b/pexpert/pexpert/arm64/T7000.h new file mode 100644 index 000000000..9c755083d --- /dev/null +++ b/pexpert/pexpert/arm64/T7000.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_T7000_H +#define _PEXPERT_ARM_T7000_H + +#include +#include + +#define WITH_CLASSIC_S2R 1 + +#ifndef ASSEMBLER + +#include + +#endif + +#endif /* ! _PEXPERT_ARM_T7000_H */ diff --git a/pexpert/pexpert/arm64/T8010.h b/pexpert/pexpert/arm64/T8010.h new file mode 100644 index 000000000..0210e7bd6 --- /dev/null +++ b/pexpert/pexpert/arm64/T8010.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2014-2015 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_T8010_H +#define _PEXPERT_ARM_T8010_H + +#include +#include + +#ifndef ASSEMBLER + +#include +#include + +#define DOCKCHANNEL_UART (1) +#define DOCKCHANNEL_STRIDE (0x10000) + +// Channel index +#define DOCKCHANNEL_UART_CHANNEL (0) + +// AOP_CLOCK frequency * 30 ms +#define DOCKCHANNEL_DRAIN_PERIOD (192000000 * 0.03) + +#define rDOCKCHANNELS_AGENT_AP_INTR_CTRL (*(volatile uint32_t *) (dock_agent_base + 0x00)) +#define rDOCKCHANNELS_AGENT_AP_INTR_STATUS (*(volatile uint32_t *) (dock_agent_base + 0x04)) +#define rDOCKCHANNELS_AGENT_AP_ERR_INTR_CTRL (*(volatile uint32_t *) (dock_agent_base + 0x08)) +#define rDOCKCHANNELS_AGENT_AP_ERR_INTR_STATUS (*(volatile uint32_t *) (dock_agent_base + 0x0c)) + +#define rDOCKCHANNELS_DEV_DRAIN_CFG(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0x0008)) + +#define rDOCKCHANNELS_DEV_WDATA1(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0x4004)) +#define rDOCKCHANNELS_DEV_WSTAT(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0x4014)) +#define rDOCKCHANNELS_DEV_RDATA0(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0x4018)) +#define rDOCKCHANNELS_DEV_RDATA1(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0x401c)) + +#define rDOCKCHANNELS_DOCK_RDATA1(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0xc01c)) +#define rDOCKCHANNELS_DOCK_RDATA3(_ch) (*(volatile uint32_t *) (uart_base + ((_ch) * DOCKCHANNEL_STRIDE) + 0xc024)) + +#endif + +#endif /* ! _PEXPERT_ARM_T8010_H */ diff --git a/pexpert/pexpert/arm64/arm64_common.h b/pexpert/pexpert/arm64/arm64_common.h new file mode 100644 index 000000000..51ff2c59a --- /dev/null +++ b/pexpert/pexpert/arm64/arm64_common.h @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM64_COMMON_H +#define _PEXPERT_ARM64_COMMON_H + +#ifdef APPLE_ARM64_ARCH_FAMILY + +#define ARM64_REG_HID0 S3_0_c15_c0_0 +#define ARM64_REG_HID0_LoopBuffDisb (1<<20) +#define ARM64_REG_HID0_ICPrefLimitOneBrn (1<<25) +#define ARM64_REG_HID0_PMULLFuseDisable (1ULL<<33) +#define ARM64_REG_HID0_ICPrefDepth_bshift 60 +#define ARM64_REG_HID0_ICPrefDepth_bmsk (7ULL < +#define __ARM_L2CACHE_SIZE_LOG__ 20 +#define ARM_BOARD_WFE_TIMEOUT_NS 1000 +#define ARM_BOARD_CLASS_S5L8960X +#define KERNEL_INTEGRITY_WT 1 +#endif /* ARM64_BOARD_CONFIG_S5L8960X */ + +#ifdef ARM64_BOARD_CONFIG_T7000 +#define APPLE_ARM64_ARCH_FAMILY 1 +#define APPLETYPHOON +#define ARM_ARCH_TIMER +#include +#define __ARM_L2CACHE_SIZE_LOG__ 20 +#define ARM_BOARD_WFE_TIMEOUT_NS 1000 +#define ARM_BOARD_CLASS_T7000 +#define PEXPERT_3X_IMAGES 1 +#define KERNEL_INTEGRITY_WT 1 +#endif /* ARM64_BOARD_CONFIG_T7000 */ + +#ifdef ARM64_BOARD_CONFIG_T7001 +#define APPLE_ARM64_ARCH_FAMILY 1 +#define APPLETYPHOON +#define ARM_ARCH_TIMER +#include +#define __ARM_L2CACHE_SIZE_LOG__ 21 +#define ARM_BOARD_WFE_TIMEOUT_NS 1000 +#define ARM_BOARD_CLASS_T7000 +#define PEXPERT_3X_IMAGES 1 +#define KERNEL_INTEGRITY_WT 1 +#define CPU_COUNT 3 +#endif /* ARM64_BOARD_CONFIG_T7001 */ + +#ifdef ARM64_BOARD_CONFIG_S8000 +/* + * The L2 size for twister is in fact 3MB, not 4MB; we round up due + * to the code being architected for power of 2 cache sizes, and rely + * on the expected behavior that out of bounds operations will be + * ignored. + */ +#define APPLE_ARM64_ARCH_FAMILY 1 +#define APPLETWISTER +#define ARM_ARCH_TIMER +#include +#define __ARM_L2CACHE_SIZE_LOG__ 22 +#define ARM_BOARD_WFE_TIMEOUT_NS 1000 +#define ARM_BOARD_CLASS_S8000 +#define KERNEL_INTEGRITY_WT 1 +#endif /* ARM64_BOARD_CONFIG_S8000 */ + +#ifdef ARM64_BOARD_CONFIG_S8001 +/* + * The L2 size for twister is in fact 3MB, not 4MB; we round up due + * to the code being architected for power of 2 cache sizes, and rely + * on the expect behavior that out of bounds operations will be + * ignored. + */ +#define APPLE_ARM64_ARCH_FAMILY 1 +#define APPLETWISTER +#define ARM_ARCH_TIMER +#include +#define __ARM_L2CACHE_SIZE_LOG__ 22 +#define ARM_BOARD_WFE_TIMEOUT_NS 1000 +#define ARM_BOARD_CLASS_S8000 +#define KERNEL_INTEGRITY_WT 1 +#endif /* ARM64_BOARD_CONFIG_S8001 */ + +#ifdef ARM64_BOARD_CONFIG_T8010 +/* + * The L2 size for hurricane/zephyr is in fact 3MB, not 4MB; we round up due + * to the code being architected for power of 2 cache sizes, and rely + * on the expect behavior that out of bounds operations will be + * ignored. + */ +#define APPLE_ARM64_ARCH_FAMILY 1 +#define APPLEHURRICANE +#define ARM_ARCH_TIMER +#include +#define __ARM_L2CACHE_SIZE_LOG__ 22 +#define ARM_BOARD_WFE_TIMEOUT_NS 1000 +#define ARM_BOARD_CLASS_T8010 +#endif /* ARM64_BOARD_CONFIG_T8010 */ + +#ifdef ARM64_BOARD_CONFIG_T8011 +#define APPLE_ARM64_ARCH_FAMILY 1 +#define APPLEHURRICANE +#define ARM_ARCH_TIMER +#include +#define __ARM_L2CACHE_SIZE_LOG__ 23 +#define ARM_BOARD_WFE_TIMEOUT_NS 1000 +#define ARM_BOARD_CLASS_T8011 +#define CPU_COUNT 3 +#endif /* ARM64_BOARD_CONFIG_T8011 */ + + + + + + +#endif /* ! _PEXPERT_ARM_BOARD_CONFIG_H */ diff --git a/pexpert/pexpert/arm64/boot.h b/pexpert/pexpert/arm64/boot.h new file mode 100644 index 000000000..c665c4721 --- /dev/null +++ b/pexpert/pexpert/arm64/boot.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2007-2009 Apple Inc. All rights reserved. + * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + */ +/* + * @OSF_COPYRIGHT@ + */ + +#ifndef _PEXPERT_ARM64_BOOT_H_ +#define _PEXPERT_ARM64_BOOT_H_ + +#include +#include +#include + +#define BOOT_LINE_LENGTH 256 + +/* + * Video information.. + */ + +struct Boot_Video { + unsigned long v_baseAddr; /* Base address of video memory */ + unsigned long v_display; /* Display Code (if Applicable */ + unsigned long v_rowBytes; /* Number of bytes per pixel row */ + unsigned long v_width; /* Width */ + unsigned long v_height; /* Height */ + unsigned long v_depth; /* Pixel Depth and other parameters */ +}; + +#define kBootVideoDepthMask (0xFF) +#define kBootVideoDepthDepthShift (0) +#define kBootVideoDepthRotateShift (8) +#define kBootVideoDepthScaleShift (16) + +#define kBootFlagsDarkBoot (1 << 0) + +typedef struct Boot_Video Boot_Video; + +/* Boot argument structure - passed into Mach kernel at boot time. + */ +#define kBootArgsRevision 1 +#define kBootArgsRevision2 2 /* added boot_args.bootFlags */ +#define kBootArgsVersion1 1 +#define kBootArgsVersion2 2 + +typedef struct boot_args { + uint16_t Revision; /* Revision of boot_args structure */ + uint16_t Version; /* Version of boot_args structure */ + uint64_t virtBase; /* Virtual base of memory */ + uint64_t physBase; /* Physical base of memory */ + uint64_t memSize; /* Size of memory */ + uint64_t topOfKernelData; /* Highest physical address used in kernel data area */ + Boot_Video Video; /* Video Information */ + uint32_t machineType; /* Machine Type */ + void *deviceTreeP; /* Base of flattened device tree */ + uint32_t deviceTreeLength; /* Length of flattened tree */ + char CommandLine[BOOT_LINE_LENGTH]; /* Passed in command line */ + uint64_t bootFlags; /* Additional flags specified by the bootloader */ + uint64_t memSizeActual; /* Actual size of memory */ +} boot_args; + +#define SOC_DEVICE_TYPE_BUFFER_SIZE 32 + +#define PC_TRACE_BUF_SIZE 1024 + +#if SHMCON +#define SHMCON_MEM 0x4000 +#else +#define SHMCON_MEM 0 +#endif + +#define CDBG_MEM ((sizeof(dbg_registry_t) + PAGE_SIZE - 1) & ~PAGE_MASK) + +#define PE_EARLY_BOOT_VA (SHMCON_MEM + CDBG_MEM) + +#endif /* _PEXPERT_ARM64_BOOT_H_ */ + diff --git a/pexpert/pexpert/arm64/cyclone.h b/pexpert/pexpert/arm64/cyclone.h new file mode 100644 index 000000000..4d39b88cf --- /dev/null +++ b/pexpert/pexpert/arm64/cyclone.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_CYCLONE_H +#define _PEXPERT_ARM_CYCLONE_H + +#ifdef APPLECYCLONE +#include "arm64_common.h" + +#define MONITOR 1 /* Use EL3 monitor */ +#define NO_ECORE 1 +#define HAS_32BIT_DBGWRAP 1 + +/* + * Determined by experiment (not described in manual): + * A0 is variant 0, B0 is variant 1. See arm64/proc_reg.h + * for how these values are constructed from the MIDR. + */ +#define CYCLONE_CPU_VERSION_A0 0x00 +#define CYCLONE_CPU_VERSION_B0 0x10 + +#endif + +#endif /* ! _PEXPERT_ARM_CYCLONE_H */ diff --git a/pexpert/pexpert/arm64/hurricane.h b/pexpert/pexpert/arm64/hurricane.h new file mode 100644 index 000000000..84a8d1bf9 --- /dev/null +++ b/pexpert/pexpert/arm64/hurricane.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2014 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_HURRICANE_H +#define _PEXPERT_ARM_HURRICANE_H + +#define NO_MONITOR 1 /* No EL3 for this CPU -- ever */ +#define HAS_MIGSTS 1 /* Has MIGSTS register, and supports migration between p-core and e-core */ +#define HAS_KTRR 1 /* Has KTRR registers */ + +#ifdef APPLEHURRICANE +#include "arm64_common.h" +#endif + +/* + * A0 is variant 0, B0 is variant 1. See arm64/proc_reg.h + * for how these values are constructed from the MIDR. + */ +#define HURRICANE_CPU_VERSION_A0 0x00 +#define HURRICANE_CPU_VERSION_B0 0x10 + +// Hurricane and Zephyr require workaround for radar 20619637 +#define SINGLE_STEP_RETIRE_ERRATA 1 + +#endif /* ! _PEXPERT_ARM_HURRICANE_H */ diff --git a/pexpert/pexpert/arm64/twister.h b/pexpert/pexpert/arm64/twister.h new file mode 100644 index 000000000..759d1ba14 --- /dev/null +++ b/pexpert/pexpert/arm64/twister.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2014 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_TWISTER_H +#define _PEXPERT_ARM_TWISTER_H + +#define MONITOR 1 /* Use EL3 monitor */ +#define NO_ECORE 1 +#define HAS_32BIT_DBGWRAP 1 + +#ifdef APPLETWISTER +#include "arm64_common.h" +#endif + +#endif /* ! _PEXPERT_ARM_TWISTER_H */ diff --git a/pexpert/pexpert/arm64/typhoon.h b/pexpert/pexpert/arm64/typhoon.h new file mode 100644 index 000000000..366fe7232 --- /dev/null +++ b/pexpert/pexpert/arm64/typhoon.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + */ + +#ifndef _PEXPERT_ARM_TYPHOON_H +#define _PEXPERT_ARM_TYPHOON_H + +#define MONITOR 1 /* Use EL3 monitor */ +#define NO_ECORE 1 +#define HAS_32BIT_DBGWRAP 1 + +#ifdef APPLETYPHOON +#include "arm64_common.h" +#endif + +#endif /* ! _PEXPERT_ARM_TYPHOON_H */ diff --git a/pexpert/pexpert/device_tree.h b/pexpert/pexpert/device_tree.h index d6fd3d9f8..8b22b9645 100644 --- a/pexpert/pexpert/device_tree.h +++ b/pexpert/pexpert/device_tree.h @@ -63,26 +63,7 @@ enum { /* length of DTEntryNameBuf = kDTMaxEntryNameLength +1*/ typedef char DTEntryNameBuf[kDTMaxEntryNameLength+1]; - -/* Entry*/ -typedef struct OpaqueDTEntry* DTEntry; - -/* Entry Iterator*/ -typedef struct OpaqueDTEntryIterator* DTEntryIterator; - -/* Property Iterator*/ -typedef struct OpaqueDTPropertyIterator* DTPropertyIterator; - - -/* status values*/ -enum { - kError = -1, - kIterationDone = 0, - kSuccess = 1 -}; - /* - Structures for a Flattened Device Tree */ @@ -102,6 +83,48 @@ typedef struct OpaqueDTEntry { // DeviceTreeNode children[]; // array size == nChildren } DeviceTreeNode; +typedef DeviceTreeNode *RealDTEntry; + +typedef struct DTSavedScope { + struct DTSavedScope * nextScope; + RealDTEntry scope; + RealDTEntry entry; + unsigned long index; +} *DTSavedScopePtr; + +/* Entry Iterator*/ +typedef struct OpaqueDTEntryIterator { + RealDTEntry outerScope; + RealDTEntry currentScope; + RealDTEntry currentEntry; + DTSavedScopePtr savedScope; + unsigned long currentIndex; +} OpaqueDTEntryIterator, *DTEntryIterator; + +/* Property Iterator*/ +typedef struct OpaqueDTPropertyIterator { + RealDTEntry entry; + DeviceTreeNodeProperty *currentProperty; + unsigned long currentIndex; +} OpaqueDTPropertyIterator, *DTPropertyIterator; + +/* Entry*/ +typedef struct OpaqueDTEntry* DTEntry; + +/* Entry Iterator*/ +typedef struct OpaqueDTEntryIterator* DTEntryIterator; + +/* Property Iterator*/ +typedef struct OpaqueDTPropertyIterator* DTPropertyIterator; + + +/* status values*/ +enum { + kError = -1, + kIterationDone = 0, + kSuccess = 1 +}; + #ifndef __MWERKS__ /* @@ -158,16 +181,13 @@ extern int DTLookupEntry(const DTEntry searchPoint, const char *pathName, DTEntr currently in. And third is a "currentPosition" which is the last entry returned during an iteration. - Create Entry Iterator - Create the iterator structure. The outermostScope and currentScope of the iterator + Initialize Entry Iterator + Fill out the iterator structure. The outermostScope and currentScope of the iterator are set to "startEntry". If "startEntry" = NULL, the outermostScope and currentScope are set to the root entry. The currentPosition for the iterator is set to "nil". */ -extern int DTCreateEntryIterator(const DTEntry startEntry, DTEntryIterator *iterator); - -/* Dispose Entry Iterator*/ -extern int DTDisposeEntryIterator(DTEntryIterator iterator); +extern int DTInitEntryIterator(const DTEntry startEntry, DTEntryIterator iter); /* Enter Child Entry @@ -223,15 +243,10 @@ extern int DTGetProperty(const DTEntry entry, const char *propertyName, void **p ------------------------------------------------------------------------------- */ /* - Create Property Iterator - Create the property iterator structure. The target entry is defined by entry. + Initialize Property Iterator + Fill out the property iterator structure. The target entry is defined by entry. */ - -extern int DTCreatePropertyIterator(const DTEntry entry, - DTPropertyIterator *iterator); - -/* Dispose Property Iterator*/ -extern int DTDisposePropertyIterator(DTPropertyIterator iterator); +extern int DTInitPropertyIterator(const DTEntry entry, DTPropertyIterator iter); /* Iterate Properites diff --git a/pexpert/pexpert/i386/boot.h b/pexpert/pexpert/i386/boot.h index 656ee5fe6..55a2cab46 100644 --- a/pexpert/pexpert/i386/boot.h +++ b/pexpert/pexpert/i386/boot.h @@ -190,7 +190,11 @@ typedef struct boot_args { uint16_t bootProgressMeterStart; uint16_t bootProgressMeterEnd; Boot_Video Video; /* Video Information */ - uint32_t __reserved4[712]; + + uint32_t apfsDataStart; /* Physical address of apfs volume key structure */ + uint32_t apfsDataSize; + + uint32_t __reserved4[710]; } boot_args; diff --git a/pexpert/pexpert/machine/boot.h b/pexpert/pexpert/machine/boot.h index 26ba42c37..1afdf9167 100644 --- a/pexpert/pexpert/machine/boot.h +++ b/pexpert/pexpert/machine/boot.h @@ -30,6 +30,10 @@ #if defined (__i386__) || defined(__x86_64__) #include "pexpert/i386/boot.h" +#elif defined (__arm64__) +#include "pexpert/arm64/boot.h" +#elif defined (__arm__) +#include "pexpert/arm/boot.h" #else #error architecture not supported #endif diff --git a/pexpert/pexpert/machine/protos.h b/pexpert/pexpert/machine/protos.h index 5d71753d9..1ec64a18f 100644 --- a/pexpert/pexpert/machine/protos.h +++ b/pexpert/pexpert/machine/protos.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "pexpert/i386/protos.h" +#elif defined (__arm__) || defined (__arm64__) +#include "pexpert/arm/protos.h" #else #error architecture not supported #endif diff --git a/pexpert/pexpert/pexpert.h b/pexpert/pexpert/pexpert.h index e9e6c463d..33b07cda2 100644 --- a/pexpert/pexpert/pexpert.h +++ b/pexpert/pexpert/pexpert.h @@ -51,6 +51,24 @@ typedef void *cpu_id_t; typedef void *cpu_id_t; #endif +#if XNU_KERNEL_PRIVATE +#if CONFIG_EMBEDDED +extern struct embedded_panic_header *panic_info; +extern vm_offset_t gPanicBase; +extern unsigned int gPanicSize; + +/* + * If invoked with NULL first argument, return the max buffer size that can + * be saved in the second argument + */ +void PE_save_buffer_to_vram( + unsigned char *, + unsigned int *); + +#else /* CONFIG_EMBEDDED */ +extern struct macos_panic_header *panic_info; +#endif /* CONFIG_EMBEDDED */ +#endif /* XNU_KERNEL_PRIVATE */ void PE_enter_debugger( const char *cause); @@ -71,6 +89,21 @@ uint32_t PE_get_random_seed( uint32_t PE_i_can_has_debugger( uint32_t *); +#if defined(__arm__) || defined(__arm64__) +void PE_mark_hwaccess(uint64_t thread); +#endif /* defined(__arm__) || defined(__arm64__) */ + +/* Return the offset of the specified address into the panic region */ +uint32_t PE_get_offset_into_panic_region( + char *location); + +/* Zeroes the panic header, sets the panic magic and initializes the header to be used */ +void PE_init_panicheader( + void); + +/* Updates the panic header during a nested panic */ +void PE_update_panicheader_nestedpanic( + void); #if KERNEL_PRIVATE @@ -345,6 +378,16 @@ extern void pe_init_debug(void); extern boolean_t PE_imgsrc_mount_supported(void); +#if defined(__arm__) || defined(__arm64__) +typedef void (*perfmon_interrupt_handler_func)(cpu_id_t source); +extern kern_return_t PE_cpu_perfmon_interrupt_install_handler(perfmon_interrupt_handler_func handler); +extern void PE_cpu_perfmon_interrupt_enable(cpu_id_t target, boolean_t enable); + +extern void (*PE_arm_debug_panic_hook)(const char *str); +#if DEVELOPMENT || DEBUG +extern void PE_arm_debug_enable_trace(void); +#endif +#endif #if KERNEL_PRIVATE boolean_t PE_reboot_on_panic(void); diff --git a/san/Kasan.exports b/san/Kasan.exports new file mode 100644 index 000000000..fbb1d3e43 --- /dev/null +++ b/san/Kasan.exports @@ -0,0 +1,2 @@ +# The KASan kext is expected to exist for both KASan and non-KASan configurations, +# so we generate an empty export list for non-KASan. diff --git a/san/Kasan_kasan.exports b/san/Kasan_kasan.exports new file mode 100644 index 000000000..122be1c84 --- /dev/null +++ b/san/Kasan_kasan.exports @@ -0,0 +1,112 @@ +___kasan_runtests +___asan_before_dynamic_init +___asan_after_dynamic_init +___asan_option_detect_stack_use_after_return +___asan_shadow_memory_dynamic_address +___asan_set_shadow_00 +___asan_set_shadow_f1 +___asan_set_shadow_f2 +___asan_set_shadow_f3 +___asan_set_shadow_f5 +___asan_set_shadow_f8 +___asan_report_load1 +___asan_report_load2 +___asan_report_load4 +___asan_report_load8 +___asan_report_load16 +___asan_report_load_n +___asan_report_store1 +___asan_report_store2 +___asan_report_store4 +___asan_report_store8 +___asan_report_store16 +___asan_report_store_n +___asan_handle_no_return +___asan_stack_malloc_0 +___asan_stack_malloc_1 +___asan_stack_malloc_2 +___asan_stack_malloc_3 +___asan_stack_malloc_4 +___asan_stack_malloc_5 +___asan_stack_malloc_6 +___asan_stack_malloc_7 +___asan_stack_malloc_8 +___asan_stack_malloc_9 +___asan_stack_malloc_10 +___asan_stack_free_0 +___asan_stack_free_1 +___asan_stack_free_2 +___asan_stack_free_3 +___asan_stack_free_4 +___asan_stack_free_5 +___asan_stack_free_6 +___asan_stack_free_7 +___asan_stack_free_8 +___asan_stack_free_9 +___asan_stack_free_10 +___asan_load1 +___asan_load2 +___asan_load4 +___asan_load8 +___asan_load16 +___asan_loadN +___asan_store1 +___asan_store2 +___asan_store4 +___asan_store8 +___asan_store16 +___asan_storeN +___asan_exp_load1 +___asan_exp_load2 +___asan_exp_load4 +___asan_exp_load8 +___asan_exp_load16 +___asan_exp_loadN +___asan_exp_store1 +___asan_exp_store2 +___asan_exp_store4 +___asan_exp_store8 +___asan_exp_store16 +___asan_exp_storeN +___asan_report_exp_load1 +___asan_report_exp_load2 +___asan_report_exp_load4 +___asan_report_exp_load8 +___asan_report_exp_load16 +___asan_report_exp_load_n +___asan_report_exp_store1 +___asan_report_exp_store2 +___asan_report_exp_store4 +___asan_report_exp_store8 +___asan_report_exp_store16 +___asan_report_exp_store_n +___asan_load_cxx_array_cookie +___asan_poison_cxx_array_cookie +___asan_poison_stack_memory +___asan_unpoison_stack_memory +___sanitizer_annotate_contiguous_container +___sanitizer_ptr_sub +___sanitizer_ptr_cmp +___asan_alloca_poison +___asan_allocas_unpoison +___asan_register_globals +___asan_unregister_globals +___asan_register_image_globals +___asan_unregister_image_globals +___asan_version_mismatch_check_v8 +___asan_version_mismatch_check_apple_802 +___asan_version_mismatch_check_apple_900 +___asan_init +___asan_memcpy +___asan_memmove +___asan_memset +___asan_bcopy +___asan_bzero +___asan_bcmp +___asan_memcmp +___asan_strlcpy +___asan_strncpy +___asan_strlcat +___asan_strncat +___asan_strlen +___asan_strnlen diff --git a/san/Makefile b/san/Makefile new file mode 100644 index 000000000..a2f25e8bc --- /dev/null +++ b/san/Makefile @@ -0,0 +1,93 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +DATAFILES = \ + +PRIVATE_DATAFILES = \ + +KERNELFILES = \ + +PRIVATE_KERNELFILES = \ + memintrinsics.h + +# Available only in xnu proper +PRIVATE_XNUFILES = \ + kasan.h + +INSTALL_MI_LIST = ${DATAFILES} +INSTALL_MI_LCL_LIST = ${PRIVATE_DATAFILES} +INSTALL_KF_MI_LIST = ${KERNELFILES} +INSTALL_KF_MI_LCL_LIST = ${KERNELFILES} ${PRIVATE_KERNELFILES} + +EXPORT_MI_LIST = ${PRIVATE_XNUFILES} ${KERNELFILES} ${PRIVATE_KERNELFILES} + +INSTALL_MI_DIR = san +EXPORT_MI_DIR = san +COMP_SUBDIRS = conf + +$(OBJROOT)/san/kasan-blacklist-%: $(SOURCE)/kasan-blacklist $(SOURCE)/kasan-blacklist-% + @echo "$(ColorH)GENERATING$(Color0) $(ColorLF)$(notdir $@)$(Color0)" + $(_v)sed -e 's,^src:\./,src:'"$(SRCROOT)/," $^ > $@ + +do_build_setup:: $(OBJROOT)/san/kasan-blacklist-x86_64 + + +# +# Kasan System.kext plugin +# + +DSTROOT_KEXT_PATH = $(addprefix $(DSTROOT)$(INSTALL_EXTENSIONS_DIR),/System.kext/PlugIns/Kasan.kext) +SYMROOT_KEXT_PATH = $(addprefix $(SYMROOT),/System.kext/PlugIns/Kasan.kext) + +ifneq ($(INSTALL_KASAN_ONLY),1) +DSTROOT_KEXT = $(DSTROOT_KEXT_PATH)/Kasan +SYMROOT_KEXT = $(SYMROOT_KEXT_PATH)/Kasan +endif + +ifeq ($(KASAN),1) +DSTROOT_KEXT += $(DSTROOT_KEXT_PATH)/Kasan_kasan +SYMROOT_KEXT += $(SYMROOT_KEXT_PATH)/Kasan_kasan +endif + +# Our external dependency on allsymbols is fine because this runs in a later phase (config_install vs. config_all) +$(OBJPATH)/%.symbolset: $(SOURCE)/%.exports + @echo "$(ColorH)SYMBOLSET$(Color0) $(ColorF)$*$(Color0) \"($(ColorLF)$(CURRENT_ARCH_CONFIG_LC)$(Color0))\"" + $(_v)$(KEXT_CREATE_SYMBOL_SET) \ + $(ARCH_FLAGS_$(CURRENT_ARCH_CONFIG)) \ + -import $(OBJPATH)/allsymbols \ + -export $< \ + -output $@ $(_vstdout) + +$(DSTROOT_KEXT): $(DSTROOT_KEXT_PATH)/% : $(OBJPATH)/%.symbolset + $(_v)$(MKDIR) $(dir $@) + @echo "$(ColorF)INSTALL$(Color0) $(ColorF)$(notdir $@)$(Color0) \"($(ColorLF)$(CURRENT_ARCH_CONFIG_LC)$(Color0))\"" + $(_v)$(INSTALL) $(EXEC_INSTALL_FLAGS) $< $@ + +$(SYMROOT_KEXT): $(SYMROOT_KEXT_PATH)/% : $(DSTROOT_KEXT_PATH)/% + $(_v)$(MKDIR) $(dir $@) + @echo "$(ColorF)INSTALL$(Color0) $(ColorF)$(notdir $@)$(Color0) \"($(ColorLF)$(CURRENT_ARCH_CONFIG_LC)$(Color0))\"" + $(_v)$(INSTALL) $(EXEC_INSTALL_FLAGS) $< $@ + +do_config_install:: $(DSTROOT_KEXT) $(SYMROOT_KEXT) + + +# Install helper scripts + +ifeq ($(KASAN),1) +KASAN_HELPER_SCRIPTS += $(DSTROOT)/$(DEVELOPER_EXTRAS_DIR)/kasan_install +endif + +$(KASAN_HELPER_SCRIPTS): $(DSTROOT)/$(DEVELOPER_EXTRAS_DIR)/% : $(SOURCE)/tools/% + $(_v)$(MKDIR) $(dir $@) + @echo "$(ColorH)INSTALL$(Color0) $(ColorF)$(@F)$(Color0)" + $(_v)$(INSTALL) $(EXEC_INSTALL_FLAGS) $< $@ + +do_config_install:: $(KASAN_HELPER_SCRIPTS) + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/san/conf/Makefile b/san/conf/Makefile new file mode 100644 index 000000000..7bd79d9ae --- /dev/null +++ b/san/conf/Makefile @@ -0,0 +1,43 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +# Special handling for x86_64h which shares a MASTER config file with x86_64: +ifeq ($(CURRENT_ARCH_CONFIG_LC),x86_64h) +DOCONF_ARCH_CONFIG_LC = x86_64 +else +DOCONF_ARCH_CONFIG_LC = $(CURRENT_ARCH_CONFIG_LC) +endif + +MASTERCONFDIR = $(SRCROOT)/config +DOCONFDEPS = $(addprefix $(MASTERCONFDIR)/, MASTER MASTER.$(DOCONF_ARCH_CONFIG_LC)) \ + $(addprefix $(SOURCE)/, Makefile.template Makefile.$(DOCONF_ARCH_CONFIG_LC) files files.$(DOCONF_ARCH_CONFIG_LC)) + +ifneq (,$(wildcard $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC))) +DOCONFDEPS += $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC) +endif + +$(TARGET)/$(CURRENT_KERNEL_CONFIG)/Makefile: $(SRCROOT)/SETUP/config/doconf $(OBJROOT)/SETUP/config $(DOCONFDEPS) + $(_v)$(MKDIR) $(TARGET)/$(CURRENT_KERNEL_CONFIG) + $(_v)$(SRCROOT)/SETUP/config/doconf -c -cpu $(DOCONF_ARCH_CONFIG_LC) -soc $(CURRENT_MACHINE_CONFIG_LC) -d $(TARGET)/$(CURRENT_KERNEL_CONFIG) -s $(SOURCE) -m $(MASTERCONFDIR) $(CURRENT_KERNEL_CONFIG); + +do_all: $(TARGET)/$(CURRENT_KERNEL_CONFIG)/Makefile + $(_v)${MAKE} \ + -C $(TARGET)/$(CURRENT_KERNEL_CONFIG) \ + -f $(TARGET)/$(CURRENT_KERNEL_CONFIG)/Makefile \ + CURRENT_KERNEL_CONFIG=${CURRENT_KERNEL_CONFIG} \ + CURRENT_ARCH_CONFIG=${CURRENT_ARCH_CONFIG} \ + CURRENT_MACHINE_CONFIG=${CURRENT_MACHINE_CONFIG} \ + SOURCE=$(subst conf/,,$(SOURCE)) \ + TARGET=${TARGET} \ + OBJPATH=${OBJPATH} \ + build_all; + +do_build_all:: do_all + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/san/conf/Makefile.template b/san/conf/Makefile.template new file mode 100644 index 000000000..03e60fa81 --- /dev/null +++ b/san/conf/Makefile.template @@ -0,0 +1,91 @@ +# +# Mach Operating System +# Copyright (c) 1986 Carnegie-Mellon University +# All rights reserved. The CMU software License Agreement specifies +# the terms and conditions for use and redistribution. +# + +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +CFLAGS+= + +# +# Directories for mig generated files +# +COMP_SUBDIRS = + +# +# Make sure we don't remove this by accident if interrupted at the wrong +# time. +# +.PRECIOUS: Makefile + +# +# Theses macros are filled in by the config program depending on the +# current configuration. The MACHDEP macro is replaced by the +# contents of the machine dependent makefile template and the others +# are replaced by the corresponding symbol definitions for the +# configuration. +# + +%OBJS + +%CFILES + +%CXXFILES + +%SFILES + +%MACHDEP + +# Rebuild if per-file overrides change +${OBJS}: $(firstword $(MAKEFILE_LIST)) + +ifneq ($(KASAN),1) +# nothing to build for non-KASAN +OBJS = +COBJS = +SOBJS = +endif + +# Rebuild if global compile flags change +$(COBJS): .CFLAGS +.CFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(KCC) $(CFLAGS) $(INCFLAGS) +$(CXXOBJS): .CXXFLAGS +.CXXFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CXXFLAGS) $(INCFLAGS) +$(SOBJS): .SFLAGS +.SFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(S_KCC) $(SFLAGS) $(INCFLAGS) + +# rebuild file list if kasan-ness changes +.KASANFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(KASAN) + +$(COMPONENT).filelist: $(OBJS) .KASANFLAGS + @echo "$(ColorL)LDFILELIST$(Color0) $(ColorLF)$(COMPONENT)$(Color0)" + $(_v)for obj in ${OBJS}; do \ + echo $(TARGET)/$(CURRENT_KERNEL_CONFIG)/$${obj}; \ + done > $(COMPONENT).filelist + +$(TARGET)/$(CURRENT_KERNEL_CONFIG)/kasan_blacklist_dynamic.h: $(SRCROOT)/$(COMPONENT)/kasan-blacklist-dynamic + @echo "$(ColorH)GENERATING$(Color0) $(ColorLF)$(notdir $@)$(Color0)" + @$(SRCROOT)/$(COMPONENT)/tools/generate_dynamic_blacklist.py "$<" > "$@" + +$(SRCROOT)/$(COMPONENT)/kasan_dynamic_blacklist.c: $(TARGET)/$(CURRENT_KERNEL_CONFIG)/kasan_blacklist_dynamic.h + +do_all: $(COMPONENT).filelist + +do_build_all:: do_all + +%RULES + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/san/conf/Makefile.x86_64 b/san/conf/Makefile.x86_64 new file mode 100644 index 000000000..7b0de925d --- /dev/null +++ b/san/conf/Makefile.x86_64 @@ -0,0 +1,7 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for x86_64 +###################################################################### + +###################################################################### +#END Machine dependent Makefile fragment for x86_64 +###################################################################### diff --git a/san/conf/files b/san/conf/files new file mode 100644 index 000000000..30036fb3e --- /dev/null +++ b/san/conf/files @@ -0,0 +1,5 @@ +san/kasan.c standard +san/kasan-fakestack.c standard +san/kasan-test.c standard +san/kasan-memintrinsics.c standard +san/kasan_dynamic_blacklist.c standard diff --git a/san/conf/files.x86_64 b/san/conf/files.x86_64 new file mode 100644 index 000000000..bd884e798 --- /dev/null +++ b/san/conf/files.x86_64 @@ -0,0 +1,5 @@ +# options + +# KASAN +san/kasan-x86_64.c standard +san/kasan-test-x86_64.s standard diff --git a/san/kasan-arm64.c b/san/kasan-arm64.c new file mode 100644 index 000000000..49137763c --- /dev/null +++ b/san/kasan-arm64.c @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +extern uint64_t *cpu_tte; +extern unsigned long gVirtBase, gPhysBase; +#define phystokv(a) ((vm_address_t)(a) - gPhysBase + gVirtBase) + +vm_offset_t physmap_vbase; +vm_offset_t physmap_vtop; + +vm_offset_t shadow_pbase; +vm_offset_t shadow_ptop; +static vm_offset_t shadow_pnext; + +static vm_offset_t zero_page_phys; +static vm_offset_t bootstrap_pgtable_phys; + +extern vm_offset_t intstack, intstack_top; +extern vm_offset_t excepstack, excepstack_top; + +void kasan_bootstrap(boot_args *, vm_offset_t pgtable); +void flush_mmu_tlb(void); + +#ifndef __ARM_16K_PG__ +#error "Unsupported HW config: Assuming 16K pages" +#endif + +#define KASAN_SHIFT_ARM64 0xdffffff800000000ULL /* Defined in makedefs/MakeInc.def */ +#define KASAN_SHADOW_MIN 0xfffffff400000000ULL +#define KASAN_SHADOW_MAX 0xfffffff680000000ULL + +_Static_assert(KASAN_SHIFT == KASAN_SHIFT_ARM64, "KASan inconsistent shadow shift"); +_Static_assert(VM_MAX_KERNEL_ADDRESS < KASAN_SHADOW_MIN, "KASan shadow overlaps with kernel VM"); +_Static_assert((VM_MIN_KERNEL_ADDRESS >> 3) + KASAN_SHIFT_ARM64 >= KASAN_SHADOW_MIN, "KASan shadow does not cover kernel VM"); +_Static_assert((VM_MAX_KERNEL_ADDRESS >> 3) + KASAN_SHIFT_ARM64 < KASAN_SHADOW_MAX, "KASan shadow does not cover kernel VM"); + +static uintptr_t +alloc_page(void) +{ + if (shadow_pnext + ARM_PGBYTES >= shadow_ptop) { + panic("KASAN: OOM"); + } + + uintptr_t mem = shadow_pnext; + shadow_pnext += ARM_PGBYTES; + shadow_pages_used++; + + return mem; +} + +static uintptr_t +alloc_zero_page(void) +{ + uintptr_t mem = alloc_page(); + __nosan_bzero((void *)phystokv(mem), ARM_PGBYTES); + return mem; +} + +static void +kasan_map_shadow_internal(vm_offset_t address, vm_size_t size, bool is_zero, bool back_page) +{ + size = vm_map_round_page(size, ARM_PGMASK); + vm_size_t j; + uint64_t *pte; + + /* XXX: this could be more efficient by walking through the shadow pages + * instead of the source pages */ + + for (j = 0; j < size; j += ARM_PGBYTES) { + vm_offset_t virt_shadow_target = (vm_offset_t)SHADOW_FOR_ADDRESS(address + j); + + assert(virt_shadow_target >= KASAN_SHADOW_MIN); + assert(virt_shadow_target < KASAN_SHADOW_MAX); + + uint64_t *base = cpu_tte; + +#if !__ARM64_TWO_LEVEL_PMAP__ + /* lookup L1 entry */ + pte = base + ((virt_shadow_target & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + if (*pte & ARM_TTE_VALID) { + assert((*pte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE); + } else { + /* create new L1 table */ + *pte = ((uint64_t)alloc_zero_page() & ARM_TTE_TABLE_MASK) | ARM_TTE_VALID | ARM_TTE_TYPE_TABLE; + } + base = (uint64_t *)phystokv(*pte & ARM_TTE_TABLE_MASK); +#endif + + /* lookup L2 entry */ + pte = base + ((virt_shadow_target & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); + if (*pte & ARM_TTE_VALID) { + assert((*pte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE); + } else { + /* create new L3 table */ + *pte = ((uint64_t)alloc_zero_page() & ARM_TTE_TABLE_MASK) | ARM_TTE_VALID | ARM_TTE_TYPE_TABLE; + } + base = (uint64_t *)phystokv(*pte & ARM_TTE_TABLE_MASK); + + if (!back_page) { + continue; + } + + /* lookup L3 entry */ + pte = base + ((virt_shadow_target & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT); + if ((*pte & ARM_PTE_TYPE_VALID) && + ((((*pte) & ARM_PTE_APMASK) != ARM_PTE_AP(AP_RONA)) || is_zero)) { + /* nothing to do - page already mapped and we are not + * upgrading */ + } else { + /* create new L3 entry */ + uint64_t newpte; + if (is_zero) { + /* map the zero page RO */ + newpte = (uint64_t)zero_page_phys | ARM_PTE_AP(AP_RONA); + } else { + /* map a fresh page RW */ + newpte = (uint64_t)alloc_zero_page() | ARM_PTE_AP(AP_RWNA); + } + newpte |= ARM_PTE_TYPE_VALID + | ARM_PTE_AF + | ARM_PTE_SH(SH_OUTER_MEMORY) + | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) + | ARM_PTE_NX + | ARM_PTE_PNX; + *pte = newpte; + } + } + + flush_mmu_tlb(); +} + +void +kasan_map_shadow(vm_offset_t address, vm_size_t size, bool is_zero) +{ + kasan_map_shadow_internal(address, size, is_zero, true); +} + +/* + * TODO: mappings here can be reclaimed after kasan_init() + */ +static void +kasan_map_shadow_early(vm_offset_t address, vm_size_t size, bool is_zero) +{ + size = vm_map_round_page(size, ARM_PGMASK); + vm_size_t j; + uint64_t *pte; + + for (j = 0; j < size; j += ARM_PGBYTES) { + vm_offset_t virt_shadow_target = (vm_offset_t)SHADOW_FOR_ADDRESS(address + j); + + assert(virt_shadow_target >= KASAN_SHADOW_MIN); + assert(virt_shadow_target < KASAN_SHADOW_MAX); + + uint64_t *base = (uint64_t *)bootstrap_pgtable_phys; + +#if !__ARM64_TWO_LEVEL_PMAP__ + /* lookup L1 entry */ + pte = base + ((virt_shadow_target & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + if (*pte & ARM_TTE_VALID) { + assert((*pte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE); + } else { + /* create new L1 table */ + vm_address_t pg = alloc_page(); + __nosan_bzero((void *)pg, ARM_PGBYTES); + *pte = ((uint64_t)pg & ARM_TTE_TABLE_MASK) | ARM_TTE_VALID | ARM_TTE_TYPE_TABLE; + } + base = (uint64_t *)(*pte & ARM_TTE_TABLE_MASK); +#endif + + /* lookup L2 entry */ + pte = base + ((virt_shadow_target & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); + if (*pte & ARM_TTE_VALID) { + assert((*pte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE); + } else { + /* create new L3 table */ + vm_address_t pg = alloc_page(); + __nosan_bzero((void *)pg, ARM_PGBYTES); + *pte = ((uint64_t)pg & ARM_TTE_TABLE_MASK) | ARM_TTE_VALID | ARM_TTE_TYPE_TABLE; + } + base = (uint64_t *)(*pte & ARM_TTE_TABLE_MASK); + + /* lookup L3 entry */ + pte = base + ((virt_shadow_target & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT); + + if ((*pte & (ARM_PTE_TYPE|ARM_PTE_APMASK)) == (ARM_PTE_TYPE_VALID|ARM_PTE_AP(AP_RWNA))) { + /* L3 entry valid and mapped RW - do nothing */ + } else { + /* Not mapped, or mapped RO - create new L3 entry or upgrade to RW */ + + uint64_t newpte; + if (is_zero) { + /* map the zero page RO */ + newpte = (uint64_t)zero_page_phys | ARM_PTE_AP(AP_RONA); + } else { + /* map a fresh page RW */ + vm_address_t pg = alloc_page(); + __nosan_bzero((void *)pg, ARM_PGBYTES); + newpte = pg | ARM_PTE_AP(AP_RWNA); + } + + /* add the default attributes */ + newpte |= ARM_PTE_TYPE_VALID + | ARM_PTE_AF + | ARM_PTE_SH(SH_OUTER_MEMORY) + | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) + | ARM_PTE_NX + | ARM_PTE_PNX; + + *pte = newpte; + } + } + + flush_mmu_tlb(); +} + +void +kasan_arch_init(void) +{ + assert(KASAN_SHADOW_MIN >= VM_MAX_KERNEL_ADDRESS); + + /* Map the physical aperture */ + kasan_map_shadow(kernel_vtop, physmap_vtop - kernel_vtop, true); + +#if defined(KERNEL_INTEGRITY_KTRR) + /* Pre-allocate all the L3 page table pages to avoid triggering KTRR */ + kasan_map_shadow_internal(VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS + 1, false, false); +#endif +} + +/* + * Steal memory for the shadow, and shadow map the bootstrap page tables so we can + * run until kasan_init(). Called while running with identity (V=P) map active. + */ +void +kasan_bootstrap(boot_args *args, vm_offset_t pgtable) +{ + uintptr_t tosteal; + + vm_address_t pbase = args->physBase; + vm_address_t ptop = args->topOfKernelData; + vm_offset_t extra = (vm_offset_t)&_mh_execute_header - pbase; + + kernel_vbase = args->virtBase; + kernel_vtop = args->virtBase + ptop - pbase; + + /* Steal ~15% of physical memory */ + tosteal = vm_map_trunc_page(args->memSize / 6, ARM_PGMASK); + args->memSize -= tosteal; + + /* Initialize the page allocator */ + shadow_pbase = vm_map_round_page(pbase + args->memSize, ARM_PGMASK); + shadow_ptop = shadow_pbase + tosteal; + shadow_pnext = shadow_pbase; + shadow_pages_total = (long)((shadow_ptop - shadow_pbase) / ARM_PGBYTES); + + /* Set aside a page of zeros we can use for dummy shadow mappings */ + zero_page_phys = alloc_page(); + __nosan_bzero((void *)zero_page_phys, ARM_PGBYTES); + + /* Shadow the KVA bootstrap mapping: start of kernel Mach-O to end of physical */ + bootstrap_pgtable_phys = pgtable; + kasan_map_shadow_early(kernel_vbase + extra, args->memSize - extra, true); + + /* Shadow the early stacks */ + vm_offset_t p2v = args->virtBase - args->physBase; + + vm_offset_t intstack_virt = (vm_offset_t)&intstack + p2v; + vm_offset_t excepstack_virt = (vm_offset_t)&excepstack + p2v; + vm_offset_t intstack_size = (vm_offset_t)&intstack_top - (vm_offset_t)&intstack; + vm_offset_t excepstack_size = (vm_offset_t)&excepstack_top - (vm_offset_t)&excepstack; + + kasan_map_shadow_early(intstack_virt, intstack_size, false); + kasan_map_shadow_early(excepstack_virt, excepstack_size, false); +} diff --git a/san/kasan-blacklist b/san/kasan-blacklist new file mode 100644 index 000000000..5832fc0e1 --- /dev/null +++ b/san/kasan-blacklist @@ -0,0 +1,24 @@ +# This file declares the list of source files that should be exempt from +# AddressSanitizer instrumentation. Usually, this is because a file is used by +# the AddressSanitizer runtime itself, or because the code executes before +# the runtime has been initialized. + +# Exclude linker sets +type:struct linker_set_entry +type:linker_set_entry + +# Exclude KASAN itself +src:./san/kasan.c +src:./san/kasan-fakestack.c +src:./san/kasan-x86_64.c +src:./san/kasan-memintrinsics.c +src:./san/kasan_dynamic_blacklist.c + +# Try really hard to avoid panicing while debugging +src:./osfmk/kdp/* +src:./osfmk/kern/debug.c + +# Exclude KASAN dependencies +# XXX: could this be relaxed since fakestack is reentrant? +src:./osfmk/kern/zalloc.c + diff --git a/san/kasan-blacklist-arm64 b/san/kasan-blacklist-arm64 new file mode 100644 index 000000000..d91dac4a8 --- /dev/null +++ b/san/kasan-blacklist-arm64 @@ -0,0 +1,10 @@ +# ARM64 specific blacklist + +# Exclude KASan runtime +src:./san/kasan-arm64.c + +# Uses a local to work out if we're on the interrupt stack, but ends up with a +# fakestack allocation +fun:ml_at_interrupt_context + + diff --git a/san/kasan-blacklist-x86_64 b/san/kasan-blacklist-x86_64 new file mode 100644 index 000000000..189a3076d --- /dev/null +++ b/san/kasan-blacklist-x86_64 @@ -0,0 +1,71 @@ +# x86_64 specific blacklist + +# Early boot AUTOGEN +src:./bsd/kern/kdebug.c +src:./bsd/kern/kern_csr.c +src:./osfmk/corecrypto/cc/src/cc_clear.c +src:./osfmk/corecrypto/ccdbrg/src/ccdrbg_nisthmac.c +src:./osfmk/device/subrs.c +src:./osfmk/i386/Diagnostics.c +src:./osfmk/i386/acpi.c +src:./osfmk/i386/cpu.c +src:./osfmk/i386/i386_init.c +src:./osfmk/i386/locks_i386.c +src:./osfmk/i386/machine_routines.c +src:./osfmk/i386/mp.c +src:./osfmk/i386/mtrr.c +src:./osfmk/i386/pal_routines.c +src:./osfmk/i386/panic_hooks.c +src:./osfmk/i386/rtclock.c +src:./osfmk/i386/vmx/vmx_cpu.c +src:./osfmk/kern/locks.c +src:./osfmk/prng/random.c +src:./osfmk/x86_64/loose_ends.c +src:./osfmk/x86_64/xcpm/xcpm_dvfs.c +src:./osfmk/x86_64/xcpm/xcpm_idle.c +src:./osfmk/x86_64/xcpm/xcpm_ioctl.c +src:./pexpert/gen/bootargs.c +src:./pexpert/gen/device_tree.c +src:./pexpert/gen/pe_gen.c +src:./pexpert/i386/pe_bootargs.c +src:./pexpert/i386/pe_identify_machine.c +src:./pexpert/i386/pe_init.c +src:./pexpert/i386/pe_serial.c + +# Nothing below is needed before kasan init, so most of it should go away. +src:./osfmk/corecrypto/*.c +src:./pexpert/* +src:./osfmk/console/video_console.c +src:./bsd/dev/unix_startup.c +src:./bsd/kern/subr_xxx.c +src:./iokit/Kernel/IOHibernateRestoreKernel.c +src:./iokit/Kernel/IOStringFuncs.c +src:./osfmk/kdp/kdp_udp.c +src:./osfmk/kern/gzalloc.c +src:./osfmk/kern/hibernate.c +src:./osfmk/kern/startup.c +src:./osfmk/kern/timer_call.c +src:./osfmk/kern/printf.c +src:./osfmk/kdp/ml/i386/kdp_x86_common.c +src:./osfmk/kdp/ml/x86_64/kdp_machdep.c +src:./osfmk/x86_64/boot_pt.c +src:./osfmk/x86_64/loose_ends.c +src:./osfmk/x86_64/pmap.c +src:./osfmk/x86_64/pmap_pcid.c +src:./osfmk/i386/AT386/model_dep.c +src:./osfmk/i386/cpuid.c +src:./osfmk/i386/gdt.c +src:./osfmk/i386/hibernate_i386.c +src:./osfmk/i386/hibernate_restore.c +src:./osfmk/i386/i386_vm_init.c +src:./osfmk/i386/machine_check.c +src:./osfmk/i386/machine_routines.c +src:./osfmk/i386/mp_desc.c +src:./osfmk/i386/pmap_common.c +src:./osfmk/i386/pmap_x86_common.c +src:./osfmk/i386/pmCPU.c +src:./osfmk/i386/startup64.c +src:./osfmk/i386/lapic_native.c +src:./osfmk/i386/fpu.c +src:./osfmk/vm/vm_compressor.c + diff --git a/san/kasan-fakestack.c b/san/kasan-fakestack.c new file mode 100644 index 000000000..12869e512 --- /dev/null +++ b/san/kasan-fakestack.c @@ -0,0 +1,339 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +int __asan_option_detect_stack_use_after_return = 0; + +#define FAKESTACK_HEADER_SZ 64 +#define FAKESTACK_NUM_SZCLASS 7 + +#define FAKESTACK_FREED 0 /* forced by clang */ +#define FAKESTACK_ALLOCATED 1 + +#if FAKESTACK + +struct fakestack_header { + LIST_ENTRY(fakestack_header) list; + void *site; /* allocation site */ + struct { + uint8_t flag; + vm_size_t realsz : 52; + vm_size_t sz_class : 4; + }; + uint64_t __pad0; +}; +_Static_assert(sizeof(struct fakestack_header) <= FAKESTACK_HEADER_SZ, "fakestack_header size mismatch"); + +static zone_t fakestack_zones[FAKESTACK_NUM_SZCLASS]; +static char fakestack_names[FAKESTACK_NUM_SZCLASS][16]; +static const unsigned long fakestack_min = 1 << 6; +static const unsigned long __unused fakestack_max = 1 << 16; + +/* + * Mark the current thread as being in a fakestack operation, to avoid reentrancy + * issues. If set, disable fakestack allocation. + */ +static boolean_t +thread_enter_fakestack(void) +{ + thread_t thread = current_thread(); + if (thread) { + return OSIncrementAtomic(&kasan_get_thread_data(current_thread())->in_fakestack); + } else { + return 0; + } +} + +static boolean_t +thread_exit_fakestack(void) +{ + thread_t thread = current_thread(); + if (thread) { + return OSDecrementAtomic(&kasan_get_thread_data(current_thread())->in_fakestack); + } else { + return 0; + } +} + +static bool +ptr_is_on_stack(uptr ptr) +{ + vm_offset_t base = dtrace_get_kernel_stack(current_thread()); + + if (ptr >= base && ptr < (base + kernel_stack_size)) { + return true; + } else { + return false; + } +} + +/* free all unused fakestack entries */ +static void NOINLINE +kasan_fakestack_gc(thread_t thread) +{ + struct fakestack_header *cur, *tmp; + LIST_HEAD(, fakestack_header) tofree = LIST_HEAD_INITIALIZER(tofree); + + /* move all the freed elements off the main list */ + struct fakestack_header_list *head = &kasan_get_thread_data(thread)->fakestack_head; + LIST_FOREACH_SAFE(cur, head, list, tmp) { + if (cur->flag == FAKESTACK_FREED) { + LIST_REMOVE(cur, list); + LIST_INSERT_HEAD(&tofree, cur, list); + } + } + + /* ... then actually free them */ + LIST_FOREACH_SAFE(cur, &tofree, list, tmp) { + zone_t zone = fakestack_zones[cur->sz_class]; + size_t sz = (fakestack_min << cur->sz_class) + FAKESTACK_HEADER_SZ; + LIST_REMOVE(cur, list); + + void *ptr = (void *)cur; + kasan_free_internal(&ptr, &sz, KASAN_HEAP_FAKESTACK, &zone, cur->realsz, 1, FAKESTACK_QUARANTINE); + if (ptr) { + zfree(zone, ptr); + } + } +} + +static uint8_t ** +fakestack_flag_ptr(vm_offset_t ptr, vm_size_t sz) +{ + uint8_t **x = (uint8_t **)ptr; + size_t idx = sz / 8; + return &x[idx - 1]; +} + +static uptr ALWAYS_INLINE +kasan_fakestack_alloc(int sz_class, size_t realsz) +{ + if (!__asan_option_detect_stack_use_after_return) { + return 0; + } + + if (sz_class >= FAKESTACK_NUM_SZCLASS) { + return 0; + } + + boolean_t flags; + uptr ret = 0; + size_t sz = fakestack_min << sz_class; + assert(realsz <= sz); + assert(sz <= fakestack_max); + zone_t zone = fakestack_zones[sz_class]; + + if (thread_enter_fakestack()) { + return 0; + } + + kasan_lock(&flags); + kasan_fakestack_gc(current_thread()); /* XXX: optimal? */ + + ret = (uptr)zget(zone); + + thread_exit_fakestack(); + + if (ret) { + size_t leftrz = 32 + FAKESTACK_HEADER_SZ; + size_t validsz = realsz - 32 - 16; /* remove redzones */ + size_t rightrz = sz - validsz - 32; /* 16 bytes, plus whatever is left over */ + struct fakestack_header *hdr = (struct fakestack_header *)ret; + + kasan_poison(ret, validsz, leftrz, rightrz, ASAN_STACK_RZ); + + hdr->site = __builtin_return_address(0); + hdr->realsz = realsz; + hdr->sz_class = sz_class; + hdr->flag = FAKESTACK_ALLOCATED; + ret += FAKESTACK_HEADER_SZ; + + *fakestack_flag_ptr(ret, sz) = &hdr->flag; /* back ptr to the slot */ + struct fakestack_header_list *head = &kasan_get_thread_data(current_thread())->fakestack_head; + LIST_INSERT_HEAD(head, hdr, list); + } + + kasan_unlock(flags); + return ret; +} + +static void NOINLINE +kasan_fakestack_free(int sz_class, uptr dst, size_t realsz) +{ + if (ptr_is_on_stack(dst)) { + return; + } + + assert(realsz <= (fakestack_min << sz_class)); + assert(__asan_option_detect_stack_use_after_return); + + vm_size_t sz = fakestack_min << sz_class; + zone_t zone = fakestack_zones[sz_class]; + assert(zone); + + /* TODO: check the magic? */ + + dst -= FAKESTACK_HEADER_SZ; + sz += FAKESTACK_HEADER_SZ; + + struct fakestack_header *hdr = (struct fakestack_header *)dst; + assert(hdr->sz_class == sz_class); + + boolean_t flags; + kasan_lock(&flags); + + LIST_REMOVE(hdr, list); + + kasan_free_internal((void **)&dst, &sz, KASAN_HEAP_FAKESTACK, &zone, realsz, 1, FAKESTACK_QUARANTINE); + if (dst) { + zfree(zone, (void *)dst); + } + + kasan_unlock(flags); +} + +void NOINLINE +kasan_unpoison_fakestack(thread_t thread) +{ + if (!__asan_option_detect_stack_use_after_return) { + return; + } + + boolean_t flags; + kasan_lock(&flags); + + thread_enter_fakestack(); + + struct fakestack_header_list *head = &kasan_get_thread_data(thread)->fakestack_head; + struct fakestack_header *cur; + LIST_FOREACH(cur, head, list) { + if (cur->flag == FAKESTACK_ALLOCATED) { + cur->flag = FAKESTACK_FREED; + } + } + + kasan_fakestack_gc(thread); + thread_exit_fakestack(); + kasan_unlock(flags); +} + +void NOINLINE +kasan_init_fakestack(void) +{ + /* allocate the fakestack zones */ + for (int i = 0; i < FAKESTACK_NUM_SZCLASS; i++) { + zone_t z; + unsigned long sz = (fakestack_min << i) + FAKESTACK_HEADER_SZ; + size_t maxsz = 256UL * 1024; + + if (i <= 3) { + /* size classes 0..3 are much more common */ + maxsz *= 4; + } + + snprintf(fakestack_names[i], 16, "fakestack.%d", i); + z = zinit(sz, maxsz, sz, fakestack_names[i]); + assert(z); + zone_change(z, Z_NOCALLOUT, TRUE); + zone_change(z, Z_EXHAUST, TRUE); + zone_change(z, Z_EXPAND, FALSE); + zone_change(z, Z_COLLECT, FALSE); + zone_change(z, Z_KASAN_QUARANTINE, FALSE); + zfill(z, maxsz / sz); + fakestack_zones[i] = z; + } + + /* globally enable */ + __asan_option_detect_stack_use_after_return = 1; +} + +#else /* FAKESTACK */ + +void +kasan_init_fakestack(void) +{ + assert(__asan_option_detect_stack_use_after_return == 0); +} + +void +kasan_unpoison_fakestack(thread_t __unused thread) +{ + assert(__asan_option_detect_stack_use_after_return == 0); +} + +static uptr +kasan_fakestack_alloc(int __unused sz_class, size_t __unused realsz) +{ + assert(__asan_option_detect_stack_use_after_return == 0); + return 0; +} + +static void +kasan_fakestack_free(int __unused sz_class, uptr __unused dst, size_t __unused realsz) +{ + assert(__asan_option_detect_stack_use_after_return == 0); + panic("fakestack_free called on non-FAKESTACK config\n"); +} + +#endif + +void kasan_init_thread(struct kasan_thread_data *td) +{ + td->in_fakestack = 0; + LIST_INIT(&td->fakestack_head); +} + +#define FAKESTACK_DECLARE(szclass) \ + uptr __asan_stack_malloc_##szclass(size_t sz) { return kasan_fakestack_alloc(szclass, sz); } \ + void __asan_stack_free_##szclass(uptr dst, size_t sz) { kasan_fakestack_free(szclass, dst, sz); } + +FAKESTACK_DECLARE(0) +FAKESTACK_DECLARE(1) +FAKESTACK_DECLARE(2) +FAKESTACK_DECLARE(3) +FAKESTACK_DECLARE(4) +FAKESTACK_DECLARE(5) +FAKESTACK_DECLARE(6) +FAKESTACK_DECLARE(7) +FAKESTACK_DECLARE(8) +FAKESTACK_DECLARE(9) +FAKESTACK_DECLARE(10) diff --git a/san/kasan-memintrinsics.c b/san/kasan-memintrinsics.c new file mode 100644 index 000000000..c95207cf2 --- /dev/null +++ b/san/kasan-memintrinsics.c @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include + +#include +#include +#include + +#include +#include + +#if MEMINTRINSICS +static bool check_intrinsics = true; +#else +static bool check_intrinsics = false; +#endif + +void +__asan_bcopy(const void *src, void *dst, size_t sz) +{ + if (check_intrinsics) { + kasan_check_range(src, sz, TYPE_MEMLD); + kasan_check_range(dst, sz, TYPE_MEMSTR); + } + __nosan_bcopy(src, dst, sz); +} + +void * +__asan_memmove(void *src, const void *dst, size_t sz) +{ + if (check_intrinsics) { + kasan_check_range(src, sz, TYPE_MEMLD); + kasan_check_range(dst, sz, TYPE_MEMSTR); + } + return __nosan_memmove(src, dst, sz); +} + +void * +__asan_memcpy(void *dst, const void *src, size_t sz) +{ + if (check_intrinsics) { + kasan_check_range(src, sz, TYPE_MEMLD); + kasan_check_range(dst, sz, TYPE_MEMSTR); + } + return __nosan_memcpy(dst, src, sz); +} + +void * +__asan_memset(void *dst, int c, size_t sz) +{ + if (check_intrinsics) { + kasan_check_range(dst, sz, TYPE_MEMSTR); + } + return __nosan_memset(dst, c, sz); +} + +void +__asan_bzero(void *dst, size_t sz) +{ + if (check_intrinsics) { + kasan_check_range(dst, sz, TYPE_MEMSTR); + } + __nosan_bzero(dst, sz); +} + +int +__asan_bcmp(const void *a, const void *b, size_t len) +{ + if (check_intrinsics) { + kasan_check_range(a, len, TYPE_MEMLD); + kasan_check_range(b, len, TYPE_MEMLD); + } + return __nosan_bcmp(a, b, len); +} + +int +__asan_memcmp(const void *a, const void *b, size_t n) +{ + if (check_intrinsics) { + kasan_check_range(a, n, TYPE_MEMLD); + kasan_check_range(b, n, TYPE_MEMLD); + } + return __nosan_memcmp(a, b, n); +} + +size_t +__asan_strlcpy(char *dst, const char *src, size_t sz) +{ + if (check_intrinsics) { + kasan_check_range(dst, sz, TYPE_STRINGSTR); + } + return __nosan_strlcpy(dst, src, sz); +} + +size_t +__asan_strlcat(char *dst, const char *src, size_t sz) +{ + if (check_intrinsics) { + kasan_check_range(dst, sz, TYPE_STRINGSTR); + } + return __nosan_strlcat(dst, src, sz); +} + +char * +__asan_strncpy(char *dst, const char *src, size_t sz) +{ + if (check_intrinsics) { + kasan_check_range(dst, sz, TYPE_STRINGSTR); + } + return __nosan_strncpy(dst, src, sz); +} + +char * +__asan_strncat(char *dst, const char *src, size_t sz) +{ + if (check_intrinsics) { + kasan_check_range(dst, strlen(dst) + sz + 1, TYPE_STRINGSTR); + } + return __nosan_strncat(dst, src, sz); +} + +size_t +__asan_strnlen(const char *src, size_t sz) +{ + if (check_intrinsics) { + kasan_check_range(src, sz, TYPE_STRINGLD); + } + + return __nosan_strnlen(src, sz); +} + +size_t +__asan_strlen(const char *src) +{ + size_t sz = __nosan_strlen(src); + if (check_intrinsics) { + kasan_check_range(src, sz + 1, TYPE_STRINGLD); + } + return sz; +} diff --git a/san/kasan-test-arm64.s b/san/kasan-test-arm64.s new file mode 100644 index 000000000..737dcafd4 --- /dev/null +++ b/san/kasan-test-arm64.s @@ -0,0 +1,58 @@ +#include +#include +#include + +#define JMP_r19_20 #0x00 +#define JMP_r21_22 #0x10 +#define JMP_r23_24 #0x20 +#define JMP_r25_26 #0x30 +#define JMP_r27_28 #0x40 +#define JMP_r29_lr #0x50 +#define JMP_fp_sp #0x60 + +#define JMP_d8_d9 #0x70 +#define JMP_d10_d11 #0x80 +#define JMP_d12_d13 #0x90 +#define JMP_d14_d15 #0xA0 + +.text + +.align 5 +.globl EXT(_setjmp) +LEXT(_setjmp) + add x1, sp, #0 /* can't STP from sp */ + stp x19, x20, [x0, JMP_r19_20] + stp x21, x22, [x0, JMP_r21_22] + stp x23, x24, [x0, JMP_r23_24] + stp x25, x26, [x0, JMP_r25_26] + stp x27, x28, [x0, JMP_r27_28] + stp x29, lr, [x0, JMP_r29_lr] + stp fp, x1, [x0, JMP_fp_sp] + stp d8, d9, [x0, JMP_d8_d9] + stp d10, d11, [x0, JMP_d10_d11] + stp d12, d13, [x0, JMP_d12_d13] + stp d14, d15, [x0, JMP_d14_d15] + mov x0, #0 + ret + +.align 5 +.globl EXT(_longjmp) +LEXT(_longjmp) + ldp x19, x20, [x0, JMP_r19_20] + ldp x21, x22, [x0, JMP_r21_22] + ldp x23, x24, [x0, JMP_r23_24] + ldp x25, x26, [x0, JMP_r25_26] + ldp x27, x28, [x0, JMP_r27_28] + ldp x29, lr, [x0, JMP_r29_lr] + ldp fp, x2, [x0, JMP_fp_sp] + ldp d8, d9, [x0, JMP_d8_d9] + ldp d10, d11, [x0, JMP_d10_d11] + ldp d12, d13, [x0, JMP_d12_d13] + ldp d14, d15, [x0, JMP_d14_d15] + add sp, x2, #0 + mov x0, x1 + cmp x0, #0 /* longjmp returns 1 if val is 0 */ + b.ne 1f + add x0, x0, #1 +1: ret + diff --git a/san/kasan-test-x86_64.s b/san/kasan-test-x86_64.s new file mode 100644 index 000000000..84bc90bbd --- /dev/null +++ b/san/kasan-test-x86_64.s @@ -0,0 +1,117 @@ +/* + * Copyright (c) 1999-2016 Apple Computer, Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ +/* + * Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved + * + * HISTORY + * 20-Apr-92 Bruce Martin (bmartin@next.com) + * Created from M68K sources. + */ + +/* + * C library -- _setjmp, _longjmp + * + * _longjmp(a,v) + * will generate a "return(v)" from + * the last call to + * _setjmp(a) + * by restoring registers from the stack, + * The previous signal state is NOT restored. + * + */ + +#include + +#define JB_RBX 0 +#define JB_RBP 8 +#define JB_RSP 16 +#define JB_R12 24 +#define JB_R13 32 +#define JB_R14 40 +#define JB_R15 48 +#define JB_RIP 56 +#define JB_RFLAGS 64 +#define JB_MXCSR 72 +#define JB_FPCONTROL 76 +#define JB_MASK 80 + +LEAF(__setjmp, 0) + // %rdi is a jmp_buf (struct sigcontext *) + + // now build sigcontext + movq %rbx, JB_RBX(%rdi) + movq %rbp, JB_RBP(%rdi) + movq %r12, JB_R12(%rdi) + movq %r13, JB_R13(%rdi) + movq %r14, JB_R14(%rdi) + movq %r15, JB_R15(%rdi) + + // RIP is set to the frame return address value + movq (%rsp), %rax + movq %rax, JB_RIP(%rdi) + // RSP is set to the frame return address plus 8 + leaq 8(%rsp), %rax + movq %rax, JB_RSP(%rdi) + + // save fp control word + fnstcw JB_FPCONTROL(%rdi) + + // save MXCSR + stmxcsr JB_MXCSR(%rdi) + + // return 0 + xorl %eax, %eax + ret + + +LEAF(__longjmp, 0) + fninit // Clear all FP exceptions + // %rdi is a jmp_buf (struct sigcontext *) + // %esi is the return value + movl %esi, %eax + testl %esi, %esi + jnz 1f + incl %eax + + // general registers +1: + movq JB_RBX(%rdi), %rbx + movq JB_RBP(%rdi), %rbp + movq JB_RSP(%rdi), %rsp + movq JB_R12(%rdi), %r12 + movq JB_R13(%rdi), %r13 + movq JB_R14(%rdi), %r14 + movq JB_R15(%rdi), %r15 + + // restore FP control word + fldcw JB_FPCONTROL(%rdi) + + // restore MXCSR + ldmxcsr JB_MXCSR(%rdi) + + + // Make sure DF is reset + cld + + jmp *JB_RIP(%rdi) + diff --git a/san/kasan-test.c b/san/kasan-test.c new file mode 100644 index 000000000..820af1b7f --- /dev/null +++ b/san/kasan-test.c @@ -0,0 +1,624 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STATIC_ARRAY_SZ 66 +#define STACK_ARRAY_SZ 9 +#define BUFSZ 34 +#define LBUFSZ 255 + +enum { + TEST_PASS, + TEST_FAIL_NOFAULT, + TEST_FAIL_BADFAULT, + TEST_SETUP_FAIL = 1, + TEST_INVALID, + TEST_UNKNOWN +}; + +unsigned long static_array[STATIC_ARRAY_SZ]; + +static jmp_buf jbuf; +static volatile int in_test = 0; + +struct kasan_test { + int (* func)(struct kasan_test *); + void (* cleanup)(struct kasan_test *); + const char *name; + int result; + void *data; + size_t datasz; +}; + +#define TEST_BARRIER() do { __asm__ __volatile__ ("" ::: "memory"); } while(0) +#define TEST_START(t) do { t->result = 1; TEST_BARRIER(); } while (0) +#define TEST_FAULT(t) do { TEST_BARRIER(); t->result = 0; TEST_BARRIER(); } while (0) +#define TEST_NOFAULT(t) do { TEST_BARRIER(); t->result = 1; TEST_BARRIER(); } while (0) +#define TEST_DONE(t,res) do { t->result = (res); kasan_handle_test(); } while (0) +#define DECLARE_TEST(f,s) { .func = f, .name = s } +#define DECLARE_TEST3(f,c,s) { .func = f, .cleanup = c, .name = s } + +static void heap_cleanup(struct kasan_test *t) +{ + if (t->data) { + kfree(t->data, t->datasz); + t->data = NULL; + } +} + +static int test_global_overflow(struct kasan_test __unused *t) +{ + int i; + /* rookie error */ + for (i = 0; i <= STATIC_ARRAY_SZ; i++) { + static_array[i] = i; + } + return 0; +} + +static int test_heap_underflow(struct kasan_test __unused *t) +{ + uint8_t *x = kalloc(BUFSZ); + if (!x) { + return 1; + } + t->datasz = BUFSZ; + t->data = x; + x[-1] = 0x12; + return 0; +} + +static int test_heap_overflow(struct kasan_test __unused *t) +{ + uint8_t *x = kalloc(BUFSZ); + if (!x) { + return 1; + } + t->datasz = BUFSZ; + t->data = x; + x[BUFSZ] = 0x11; + return 0; +} + +static int test_heap_uaf(struct kasan_test __unused *t) +{ + uint8_t *x = kalloc(LBUFSZ); + if (!x) { + return 1; + } + kfree(x, LBUFSZ); + x[0] = 0x10; + return 0; +} + +static int test_heap_inval_free(struct kasan_test __unused *t) +{ + int x; + kfree(&x, BUFSZ); + return 0; +} + +static int test_heap_double_free(struct kasan_test *t) +{ + TEST_START(t); + + uint8_t *x = kalloc(BUFSZ); + if (!x) { + return 1; + } + kfree(x, BUFSZ); + + TEST_FAULT(t); + kfree(x, BUFSZ); + + return 0; +} + +static int test_heap_small_free(struct kasan_test *t) +{ + TEST_START(t); + + uint8_t *x = kalloc(BUFSZ); + if (!x) { + return 1; + } + t->datasz = BUFSZ; + t->data = x; + + TEST_FAULT(t); + kfree(x, BUFSZ-2); + t->data = NULL; + t->datasz = 0; + + return 0; +} + +static int test_stack_overflow(struct kasan_test *t) +{ + TEST_START(t); + + int i; + volatile uint8_t a[STACK_ARRAY_SZ]; + + for (i = 0; i < STACK_ARRAY_SZ; i++) { + a[i] = i; + } + + TEST_FAULT(t); + a[i] = i; /* rookie error */ + TEST_NOFAULT(t); + + TEST_BARRIER(); + + return !(a[0] == 0); +} + +static int test_stack_underflow(struct kasan_test *t) +{ + TEST_START(t); + + long idx; + uint8_t a[STACK_ARRAY_SZ]; + + __nosan_memset(a, 0, STACK_ARRAY_SZ); + + /* generate a negative index without the compiler noticing */ +#if __x86_64__ + __asm__ __volatile__("movq $-1, %0" : "=r"(idx) :: "memory"); +#else + __asm__ __volatile__("mov %0, #-1" : "=r"(idx) :: "memory"); +#endif + + TEST_FAULT(t); + a[idx] = 0xbd; + TEST_NOFAULT(t); + + TEST_BARRIER(); + return (a[0] == 0); +} + +static int test_memcpy(struct kasan_test *t) +{ + TEST_START(t); + uint8_t a1[STACK_ARRAY_SZ]; + uint8_t a2[STACK_ARRAY_SZ]; + + /* should work */ + memcpy(a1, a2, STACK_ARRAY_SZ); + + TEST_BARRIER(); + + /* should fail */ + TEST_FAULT(t); + memcpy(a2, a1, STACK_ARRAY_SZ+1); + TEST_NOFAULT(t); + + return 0; +} + +static int test_memmove(struct kasan_test *t) +{ + TEST_START(t); + uint8_t a1[STACK_ARRAY_SZ]; + uint8_t a2[STACK_ARRAY_SZ]; + + /* should work */ + memmove(a1, a2, STACK_ARRAY_SZ); + + TEST_BARRIER(); + + /* should fail */ + TEST_FAULT(t); + memmove(a2, a1, STACK_ARRAY_SZ+1); + TEST_NOFAULT(t); + + return 0; +} + +static int test_bcopy(struct kasan_test *t) +{ + TEST_START(t); + uint8_t a1[STACK_ARRAY_SZ]; + uint8_t a2[STACK_ARRAY_SZ]; + + /* should work */ + bcopy(a1, a2, STACK_ARRAY_SZ); + + TEST_BARRIER(); + + /* should fail */ + TEST_FAULT(t); + bcopy(a2, a1, STACK_ARRAY_SZ+1); + TEST_NOFAULT(t); + + return 0; +} + +static int test_memset(struct kasan_test *t) +{ + TEST_START(t); + uint8_t a1[STACK_ARRAY_SZ]; + + /* should work */ + memset(a1, 'e', STACK_ARRAY_SZ); + + TEST_BARRIER(); + + /* should fail */ + TEST_FAULT(t); + memset(a1, 'f', STACK_ARRAY_SZ+1); + TEST_NOFAULT(t); + + return 0; +} + +static int test_memcmp(struct kasan_test *t) +{ + TEST_START(t); + uint8_t *a1; + uint8_t *a2; + + a1 = kalloc(STACK_ARRAY_SZ); + if (!a1) + return 1; + a2 = kalloc(STACK_ARRAY_SZ+1); + if (!a2) + return 1; + + /* should work */ + memcmp(a1, a2, STACK_ARRAY_SZ); + memcmp(a1, a2+1, STACK_ARRAY_SZ); + + TEST_BARRIER(); + + /* should fail */ + TEST_FAULT(t); + memcmp(a1, a2, STACK_ARRAY_SZ+1); + TEST_NOFAULT(t); + + return 0; +} + +static int test_bcmp(struct kasan_test *t) +{ + TEST_START(t); + uint8_t *a1; + uint8_t *a2; + + a1 = kalloc(STACK_ARRAY_SZ); + if (!a1) + return 1; + a2 = kalloc(STACK_ARRAY_SZ+1); + if (!a2) + return 1; + + /* should work */ + bcmp(a1, a2, STACK_ARRAY_SZ); + bcmp(a1, a2+1, STACK_ARRAY_SZ); + + TEST_BARRIER(); + + /* should fail */ + TEST_FAULT(t); + bcmp(a1, a2, STACK_ARRAY_SZ+1); + TEST_NOFAULT(t); + + return 0; +} + +static int test_bzero(struct kasan_test *t) +{ + TEST_START(t); + uint8_t a1[STACK_ARRAY_SZ]; + + /* should work */ + bzero(a1, STACK_ARRAY_SZ); + + TEST_BARRIER(); + + /* should fail */ + TEST_FAULT(t); + bzero(a1, STACK_ARRAY_SZ+1); + TEST_NOFAULT(t); + + return 0; +} + +static int test_strlcpy(struct kasan_test *t) +{ + TEST_START(t); + char a1[8]; + + /* should not fault */ + strlcpy(a1, "small", 8); + strlcpy(a1, "looooonnnnggg", 8); + + TEST_FAULT(t); + strlcpy(a1, "looooooooonnnnggg", 9); + TEST_NOFAULT(t); + + return 0; +} + +static int test_strncpy(struct kasan_test *t) +{ + TEST_START(t); + char a1[9]; + + /* should not fault */ + strncpy(a1, "small", 9); + strncpy(a1, "looooonnnnggg", 9); + + TEST_FAULT(t); + strncpy(a1, "looooonnnnggg", 10); + TEST_NOFAULT(t); + + return a1[0] != 'l'; +} + +static int test_strlcat(struct kasan_test *t) +{ + TEST_START(t); + char a1[9] = {}; + + /* should not fault */ + strlcat(a1, "abcd", 9); + strlcat(a1, "efgh", 9); + strlcat(a1, "ijkl", 9); + a1[0] = '\0'; + strlcat(a1, "looooonnnnggg", 9); + + a1[0] = '\0'; + TEST_FAULT(t); + strlcat(a1, "looooonnnnggg", 10); + TEST_NOFAULT(t); + + return a1[0] != 'l'; +} + +static int test_strncat(struct kasan_test *t) +{ + TEST_START(t); + char a1[9] = {}; + + /* should not fault */ + strncat(a1, "abcd", 4); + strncat(a1, "efgh", 4); + + TEST_FAULT(t); + strncat(a1, "i", 1); + TEST_NOFAULT(t); + + return a1[0] != 'a'; +} + +/* we ignore the top *two* frames in backtrace - so add an extra one */ +static int NOINLINE test_blacklist_helper(void) +{ + return kasan_is_blacklisted(TYPE_TEST); +} + +static int NOINLINE test_blacklist(struct kasan_test *t) +{ + TEST_START(t); + int res = (int)!test_blacklist_helper(); + TEST_DONE(t, res); + return 0; +} + +static int NOINLINE test_blacklist_str(struct kasan_test *t) +{ + TEST_START(t); + char a1[8]; + + strlcpy(a1, "looooooooonnnnggg", 9); + + TEST_DONE(t, 0); /* success */ + return 0; +} + +#if 0 +static int test_strnlen(struct kasan_test *t) +{ + TEST_START(t); + const char *a1 = "abcdef"; + + /* should not fault */ + if (strnlen(a1, 6) != 6) + return 1; + if (strnlen(a1, 7) != 6) + return 1; + + TEST_FAULT(t); + if (strnlen(a1, 8) != 6) + return 1; + TEST_NOFAULT(t); + + return a1[0] != 'a'; +} +#endif + +int *uaf_ptr; +static int * NOINLINE +stack_uaf_helper(void) +{ + int x; + uaf_ptr = &x; + return uaf_ptr; +} + +static int test_stack_uaf(struct kasan_test __unused *t) +{ + int *x = stack_uaf_helper(); + *x = 0xb4d; + TEST_BARRIER(); + return !(*x == 0xb4d); +} + +static struct kasan_test xnu_tests[] = { + DECLARE_TEST(NULL, NULL), + DECLARE_TEST(test_global_overflow, "Global overflow"), + DECLARE_TEST3(test_heap_underflow, heap_cleanup, "Heap underflow"), + DECLARE_TEST3(test_heap_overflow, heap_cleanup, "Heap overflow"), + DECLARE_TEST(test_heap_uaf, "Heap use-after-free"), + DECLARE_TEST(test_heap_inval_free, "Heap invalid free"), + DECLARE_TEST(test_heap_double_free,"Heap double free"), + DECLARE_TEST3(test_heap_small_free, heap_cleanup, "Heap small free"), + DECLARE_TEST(test_stack_overflow, "Stack overflow"), + DECLARE_TEST(test_stack_underflow, "Stack underflow"), + DECLARE_TEST(test_stack_uaf, "Stack use-after-return"), + DECLARE_TEST(test_memcpy, "memcpy"), + DECLARE_TEST(test_memmove, "memmmove"), + DECLARE_TEST(test_bcopy, "bcopy"), + DECLARE_TEST(test_memset, "memset"), + DECLARE_TEST(test_memcmp, "memcmp"), + DECLARE_TEST(test_bcmp, "bcmp"), + DECLARE_TEST(test_bzero, "bzero"), + DECLARE_TEST(test_strlcpy, "strlcpy"), + DECLARE_TEST(test_strlcat, "strlcat"), + DECLARE_TEST(test_strncpy, "strncpy"), + DECLARE_TEST(test_strncat, "strncat"), + DECLARE_TEST(test_blacklist, "blacklist"), + DECLARE_TEST(test_blacklist_str, "blacklist_str"), + // DECLARE_TEST(test_strnlen, "strnlen"), +}; +static int num_xnutests = sizeof(xnu_tests)/sizeof(xnu_tests[0]); + +static int +kasan_run_test(struct kasan_test *test_list, int testno, int fail) +{ + int status = TEST_UNKNOWN; + struct kasan_test *t = &test_list[testno]; + + if (testno < 0 || testno >= num_xnutests || !t->func) { + printf("KASan: test.%02d INVALID\n", testno); + return TEST_INVALID; + } + + // printf("KASan: test.%02d RUNNING (%s)\n", testno, t->name); + + if (!fail) { + in_test = 1; + } + + if (_setjmp(jbuf) == 0) { + t->result = 0; + int ret = t->func(t); + if (ret) { + printf("KASan: test.%02d SETUP FAIL (%s)\n", testno, t->name); + status = ret; + } else { + /* did not fault when it should have */ + printf("KASan: test.%02d FAIL (%s)\n", testno, t->name); + status = TEST_FAIL_NOFAULT; + } + } else { + /* Triggering a KASan violation will return here by longjmp, bypassing + * stack unpoisoning, so do it here explicitly. We just hope that + * fakestack free will happen later... */ + kasan_unpoison_curstack(); + + if (t->result) { + /* faulted, but at the wrong place */ + printf("KASan: test.%02d FAIL %d (%s)\n", testno, t->result, t->name); + status = TEST_FAIL_BADFAULT; + } else { + printf("KASan: test.%02d PASS (%s)\n", testno, t->name); + status = TEST_PASS; + } + } + in_test = 0; + if (t->cleanup) { + t->cleanup(t); + } + + return status; +} + +void +kasan_test(int testno, int fail) +{ + int i = 1; + int pass = 0, total = 0; + int ret; + + if (testno == -1) { + /* shorthand for all tests */ + testno = (1U << (num_xnutests-1)) - 1; + } + + while (testno) { + if (testno & 0x1) { + ret = kasan_run_test(xnu_tests, i, fail); + if (ret == TEST_PASS) { + pass++; + } + if (ret != TEST_INVALID) { + total++; + } + } + + i++; + testno >>= 1; + } + printf("KASan: TEST SUMMARY %d/%d passed\n", pass, total); +} + +void +kasan_handle_test(void) +{ + if (in_test) { + _longjmp(jbuf, 1); + /* NOTREACHED */ + } +} + +void +__kasan_runtests(struct kasan_test *kext_tests, int numtests) +{ + int i; + for (i = 0; i < numtests; i++) { + kasan_run_test(kext_tests, i, 0); + } +} diff --git a/san/kasan-x86_64.c b/san/kasan-x86_64.c new file mode 100644 index 000000000..72339d8cd --- /dev/null +++ b/san/kasan-x86_64.c @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern uint64_t *IdlePML4; +extern uintptr_t physmap_base; +extern uintptr_t physmap_max; +#define phys2virt(x) ((uintptr_t)(x) + physmap_base) + +#define INTEL_PTE_VALID 0x00000001ULL +#define INTEL_PTE_WRITE 0x00000002ULL +#define INTEL_PTE_RW 0x00000002ULL +#define INTEL_PTE_USER 0x00000004ULL +#define INTEL_PTE_WTHRU 0x00000008ULL +#define INTEL_PTE_NCACHE 0x00000010ULL +#define INTEL_PTE_REF 0x00000020ULL +#define INTEL_PTE_MOD 0x00000040ULL +#define INTEL_PTE_PS 0x00000080ULL +#define INTEL_PTE_PTA 0x00000080ULL +#define INTEL_PTE_GLOBAL 0x00000100ULL +#define INTEL_PTE_WIRED 0x00000200ULL +#define INTEL_PDPTE_NESTED 0x00000400ULL +#define INTEL_PTE_PFN PG_FRAME +#define INTEL_PTE_NX (1ULL << 63) +#define INTEL_PTE_INVALID 0 + +vm_offset_t shadow_pbase; +vm_offset_t shadow_ptop; +vm_offset_t shadow_pnext; +unsigned shadow_stolen_idx; + +static vm_offset_t zero_superpage_phys; + +typedef struct { + unsigned int pml4 : 9; + unsigned int pdpt : 9; + unsigned int pd : 9; + unsigned int pt : 9; + unsigned int offset : 12; +} split_addr_t; + +static split_addr_t +split_address(vm_offset_t address) +{ + split_addr_t addr; + + addr.pml4 = (address >> 39) & 0x1ff; + addr.pdpt = (address >> 30) & 0x1ff; + addr.pd = (address >> 21) & 0x1ff; + addr.pt = (address >> 12) & 0x1ff; + // addr.offset = address & PAGE_MASK; + + return addr; +} + +static uintptr_t +alloc_page(void) +{ + if (shadow_pnext + I386_PGBYTES >= shadow_ptop) { + panic("KASAN: OOM"); + } + + uintptr_t mem = shadow_pnext; + shadow_pnext += I386_PGBYTES; + shadow_pages_used++; + + return mem; +} + +#define ROUND_SUPERPAGE(x) ((((uintptr_t)(x)) + I386_LPGBYTES - 1) & ~(I386_LPGMASK)) + +static uintptr_t +alloc_superpage(void) +{ + uintptr_t mem; + shadow_pnext = ROUND_SUPERPAGE(shadow_pnext); + assert((shadow_pnext & I386_LPGMASK) == 0); + mem = shadow_pnext; + shadow_pnext += I386_LPGBYTES; + shadow_pages_used += I386_LPGBYTES / I386_PGBYTES; + /* XXX: not accounting for superpage rounding */ + return mem; +} + +static uintptr_t +alloc_page_zero(void) +{ + uintptr_t mem = alloc_page(); + bzero_phys(mem, I386_PGBYTES); + return mem; +} + +static void +kasan_map_shadow_superpage_zero(vm_offset_t address, vm_size_t size) +{ + address = vm_map_trunc_page(address, I386_LPGMASK); + size = vm_map_round_page(size, I386_LPGMASK); + + vm_size_t j; + for (j = 0; j < size; j += I386_LPGBYTES * 8) { + + vm_offset_t virt_shadow_target = (vm_offset_t)SHADOW_FOR_ADDRESS(address + j); + + split_addr_t addr = split_address(virt_shadow_target); + assert(addr.pml4 == 507 || addr.pml4 == 508); + + uint64_t *L3; + uint64_t *L2; + uint64_t *L1; + + L3 = (uint64_t *)(IdlePML4[addr.pml4] & ~PAGE_MASK); + if (L3 == NULL) { + uintptr_t pmem = alloc_page_zero(); + L3 = (uint64_t *)phys2virt(pmem); + IdlePML4[addr.pml4] = pmem + | INTEL_PTE_VALID + | INTEL_PTE_WRITE; + } else { + L3 = (uint64_t *)phys2virt(L3); + } + + L2 = (uint64_t *)(L3[addr.pdpt] & ~PAGE_MASK); + if (L2 == NULL) { + uintptr_t pmem = alloc_page_zero(); + L2 = (uint64_t *)phys2virt(pmem); + L3[addr.pdpt] = pmem + | INTEL_PTE_VALID + | INTEL_PTE_WRITE; + } else { + L2 = (uint64_t *)phys2virt(L2); + } + + L1 = (uint64_t *)(L2[addr.pd] & ~PAGE_MASK); + if (L1 == NULL) { + L2[addr.pd] = (uint64_t)zero_superpage_phys + | INTEL_PTE_VALID + | INTEL_PTE_PS + | INTEL_PTE_NX; + } else { + panic("Unexpected shadow mapping, addr = %lx, sz = %lu\n", + address, size); + } + + /* adding a new entry, this is not strictly required */ + invlpg(virt_shadow_target); + } +} + +void +kasan_map_shadow(vm_offset_t address, vm_size_t size, bool is_zero) +{ + size = vm_map_round_page(size, PAGE_MASK); + vm_size_t j; + + for (j = 0; j < size; j += I386_PGBYTES) { + + vm_offset_t virt_shadow_target = (vm_offset_t)SHADOW_FOR_ADDRESS(address + j); + + split_addr_t addr = split_address(virt_shadow_target); + assert(addr.pml4 == 507 || addr.pml4 == 508); + + uint64_t *L3; + uint64_t *L2; + uint64_t *L1; + uint64_t *pte; + + L3 = (uint64_t *)(IdlePML4[addr.pml4] & ~PAGE_MASK); + if (L3 == NULL) { + uintptr_t pmem = alloc_page_zero(); + L3 = (uint64_t *)phys2virt(pmem); + IdlePML4[addr.pml4] = pmem + | INTEL_PTE_VALID + | INTEL_PTE_WRITE; + } else { + L3 = (uint64_t *)phys2virt(L3); + } + + L2 = (uint64_t *)(L3[addr.pdpt] & ~PAGE_MASK); + if (L2 == NULL) { + uintptr_t pmem = alloc_page_zero(); + L2 = (uint64_t *)phys2virt(pmem); + L3[addr.pdpt] = pmem + | INTEL_PTE_VALID + | INTEL_PTE_WRITE; + } else { + L2 = (uint64_t *)phys2virt(L2); + } + + uint64_t pde = L2[addr.pd]; + if ((pde & (INTEL_PTE_VALID|INTEL_PTE_PS)) == (INTEL_PTE_VALID|INTEL_PTE_PS)) { + /* Already mapped as a superpage */ + continue; + } + + L1 = (uint64_t *)(pde & ~PAGE_MASK); + if (L1 == NULL) { + uintptr_t pmem = alloc_page_zero(); + L1 = (uint64_t *)phys2virt(pmem); + L2[addr.pd] = pmem + | INTEL_PTE_VALID + | INTEL_PTE_WRITE; + } else { + L1 = (uint64_t *)phys2virt(L1); + } + + pte = (uint64_t *)(L1[addr.pt] & ~PAGE_MASK); + if (pte == NULL) { + uint64_t newpte; + if (is_zero) { + newpte = (uint64_t)zero_superpage_phys; + } else { + newpte = (vm_offset_t)alloc_page_zero() + | INTEL_PTE_WRITE; + } + L1[addr.pt] = newpte + | INTEL_PTE_VALID + | INTEL_PTE_NX; + } + + /* adding a new entry, this is not strictly required */ + invlpg(virt_shadow_target); + } +} + +void +kasan_arch_init(void) +{ + __nosan_bzero((void *)phys2virt(zero_superpage_phys), I386_LPGBYTES); + + /* Map the physical aperture */ + kasan_map_shadow_superpage_zero(physmap_base, physmap_max - physmap_base); + /* Establish shadow mappings for the x86 descriptor tables and + * "low global" page; these are specially alias-mapped at fixed VAs + * early in boot + */ + kasan_map_low_fixed_regions(); +} + +/* + * Steal some memory from EFI for the shadow map. + */ +void +kasan_reserve_memory(void *_args) +{ + boot_args *args = (boot_args *)_args; + vm_address_t pbase = args->kaddr; + vm_address_t ptop = args->kaddr + args->ksize; + + kernel_vbase = ml_static_ptovirt(pbase); + kernel_vtop = ml_static_ptovirt(ptop); + + EfiMemoryRange *mptr, *mptr_tmp; + unsigned int mcount; + unsigned int msize; + unsigned int i; + unsigned long total_pages; + unsigned long to_steal; + + mptr = (EfiMemoryRange *)ml_static_ptovirt((vm_offset_t)args->MemoryMap); + msize = args->MemoryMapDescriptorSize; + mcount = args->MemoryMapSize / msize; + + /* sum total physical memory */ + total_pages = 0; + for (i = 0, mptr_tmp = mptr; i < mcount; i++, mptr_tmp = (EfiMemoryRange *)(((vm_offset_t)mptr_tmp) + msize)) { + total_pages += mptr_tmp->NumberOfPages; + } + + /* steal 25% of physical memory */ + to_steal = total_pages / 4; + + /* Search for a range large enough to steal from */ + for (i = 0, mptr_tmp = mptr; i < mcount; i++, mptr_tmp = (EfiMemoryRange *)(((vm_offset_t)mptr_tmp) + msize)) { + ppnum_t base, top; + base = (ppnum_t)(mptr_tmp->PhysicalStart >> I386_PGSHIFT); + top = (ppnum_t)((mptr_tmp->PhysicalStart >> I386_PGSHIFT) + mptr_tmp->NumberOfPages - 1); + + if ((mptr_tmp->Type == kEfiConventionalMemory) && (mptr_tmp->NumberOfPages > to_steal)) { + /* Found a region with sufficient space - steal from the end */ + mptr_tmp->NumberOfPages -= to_steal; + + shadow_pbase = mptr_tmp->PhysicalStart + (mptr_tmp->NumberOfPages << I386_PGSHIFT); + shadow_ptop = shadow_pbase + (to_steal << I386_PGSHIFT); + shadow_pnext = shadow_pbase; + shadow_pages_total = to_steal; + shadow_stolen_idx = i; + + /* Set aside a page of zeros we can use for dummy shadow mappings */ + zero_superpage_phys = alloc_superpage(); + + return; + } + } + + panic("KASAN: could not reserve memory"); +} + diff --git a/san/kasan.c b/san/kasan.c new file mode 100644 index 000000000..960dc25b4 --- /dev/null +++ b/san/kasan.c @@ -0,0 +1,1241 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#if !KASAN_DEBUG +# undef NOINLINE +# define NOINLINE +#endif + +const uintptr_t __asan_shadow_memory_dynamic_address = KASAN_SHIFT; + +static long kexts_loaded; + +long shadow_pages_total; +long shadow_pages_used; + +vm_offset_t kernel_vbase; +vm_offset_t kernel_vtop; + +static bool kasan_initialized; +static int kasan_enabled; +static int quarantine_enabled = 1; + +static void kasan_crash_report(uptr p, uptr width, unsigned access_type); +extern vm_offset_t ml_stack_base(void); +extern vm_size_t ml_stack_size(void); + +#define ABI_UNSUPPORTED do { panic("KASan: unsupported ABI: %s\n", __func__); } while (0) + +#define BACKTRACE_MAXFRAMES 16 + +decl_simple_lock_data(, kasan_vm_lock); + +_Atomic int unsafe_count = 0; + +void +kasan_unsafe_start(void) +{ + if (__c11_atomic_fetch_add(&unsafe_count, 1, memory_order_relaxed) == 128) { + panic("kasan_unsafe_start overflow"); + } +} + +void +kasan_unsafe_end(void) +{ + if (__c11_atomic_fetch_sub(&unsafe_count, 1, memory_order_relaxed) == 0) { + panic("kasan_unsafe_end underflow"); + } +} + +static bool +kasan_in_unsafe(void) +{ + return atomic_load_explicit(&unsafe_count, memory_order_relaxed) != 0; +} + +/* + * kasan is called from the interrupt path, so we need to disable interrupts to + * ensure atomicity manipulating the global objects + */ +void +kasan_lock(boolean_t *b) +{ + *b = ml_set_interrupts_enabled(false); + simple_lock(&kasan_vm_lock); +} + +void +kasan_unlock(boolean_t b) +{ + simple_unlock(&kasan_vm_lock); + ml_set_interrupts_enabled(b); +} + +/* + * poison redzones in the shadow map + */ +void NOINLINE +kasan_poison(vm_offset_t base, vm_size_t size, vm_size_t leftrz, vm_size_t rightrz, uint8_t flags) +{ + uint8_t *shadow = SHADOW_FOR_ADDRESS(base); + uint8_t partial = size & 0x07; + vm_size_t total = leftrz + size + rightrz; + vm_size_t i = 0; + + /* base must be 8-byte aligned */ + /* any left redzone must be a multiple of 8 */ + /* total region must cover 8-byte multiple */ + assert((base & 0x07) == 0); + assert((leftrz & 0x07) == 0); + assert((total & 0x07) == 0); + + if (!kasan_enabled || !kasan_initialized) { + return; + } + + leftrz /= 8; + size /= 8; + total /= 8; + + uint8_t l_flags = flags; + uint8_t r_flags = flags; + + if (flags == ASAN_STACK_RZ) { + l_flags = ASAN_STACK_LEFT_RZ; + r_flags = ASAN_STACK_RIGHT_RZ; + } else if (flags == ASAN_HEAP_RZ) { + l_flags = ASAN_HEAP_LEFT_RZ; + r_flags = ASAN_HEAP_RIGHT_RZ; + } + + /* + * poison the redzones and unpoison the valid bytes + */ + for (; i < leftrz; i++) { + shadow[i] = l_flags; + } + for (; i < leftrz + size; i++) { + shadow[i] = ASAN_VALID; /* not strictly necessary */ + } + if (partial && (i < total)) { + shadow[i] = partial; + i++; + } + for (; i < total; i++) { + shadow[i] = r_flags; + } + + asm volatile("" ::: "memory"); /* compiler barrier XXX: is this needed? */ +} + +void +kasan_poison_range(vm_offset_t base, vm_size_t size, uint8_t flags) +{ + /* base must be 8-byte aligned */ + /* total region must cover 8-byte multiple */ + assert((base & 0x07) == 0); + assert((size & 0x07) == 0); + kasan_poison(base, 0, 0, size, flags); +} + +void NOINLINE +kasan_unpoison(void *base, vm_size_t size) +{ + kasan_poison((vm_offset_t)base, size, 0, 0, 0); +} + +void NOINLINE +kasan_unpoison_stack(vm_offset_t base, vm_size_t size) +{ + assert(base); + assert(size); + kasan_unpoison((void *)base, size); +} + +/* + * write junk into the redzones +*/ +static void NOINLINE +kasan_rz_clobber(vm_offset_t base, vm_size_t size, vm_size_t leftrz, vm_size_t rightrz) +{ +#if KASAN_DEBUG + vm_size_t i; + const uint8_t deadbeef[] = { 0xde, 0xad, 0xbe, 0xef }; + const uint8_t c0ffee[] = { 0xc0, 0xff, 0xee, 0xc0 }; + uint8_t *buf = (uint8_t *)base; + + /* base must be 8-byte aligned */ + /* any left redzone must be a multiple of 8 */ + /* total region must cover 8-byte multiple */ + assert((base & 0x07) == 0); + assert((leftrz & 0x07) == 0); + assert(((size + leftrz + rightrz) & 0x07) == 0); + + for (i = 0; i < leftrz; i++) { + buf[i] = deadbeef[i % 4]; + } + + for (i = 0; i < rightrz; i++) { + buf[i + size + leftrz] = c0ffee[i % 4]; + } +#else + (void)base; + (void)size; + (void)leftrz; + (void)rightrz; +#endif +} + +void NOINLINE +kasan_check_range(const void *x, size_t sz, unsigned access_type) +{ + vm_offset_t invalid; + + if (kasan_in_unsafe()) { + return; + } + + if (kasan_range_poisoned((vm_offset_t)x, sz, &invalid)) { + if (kasan_is_blacklisted(access_type)) { + return; + } + kasan_crash_report(invalid, sz, access_type); + /* NOTREACHED */ + } +} + +/* + * Check that [base, base+sz) has shadow value `shadow' + * If not, report a KASan-violation on `addr' + */ +static void +kasan_assert_shadow(vm_address_t base, vm_size_t sz, vm_address_t addr, uint8_t shadow) +{ + sz -= 8 - (base % 8); + base += 8 - (base % 8); + + vm_address_t end = base + sz; + + while (base < end) { + uint8_t *sh = SHADOW_FOR_ADDRESS(base); + if (*sh != shadow) { + __asan_report_load1(addr); + } + base += 8; + } +} + +/* + * + * KASAN violation reporting + * + */ + +static const char * +access_type_str(unsigned type) +{ + if (type & TYPE_LOAD_ALL) { + return "load"; + } else if (type & TYPE_STORE_ALL) { + return "store"; + } else if (type & TYPE_FREE) { + return "free"; + } else { + return "access"; + } +} + +static const char *shadow_strings[] = { + [ASAN_VALID] = "VALID", + [ASAN_PARTIAL1] = "PARTIAL1", + [ASAN_PARTIAL2] = "PARTIAL2", + [ASAN_PARTIAL3] = "PARTIAL3", + [ASAN_PARTIAL4] = "PARTIAL4", + [ASAN_PARTIAL5] = "PARTIAL5", + [ASAN_PARTIAL6] = "PARTIAL6", + [ASAN_PARTIAL7] = "PARTIAL7", + [ASAN_STACK_RZ] = "", + [ASAN_STACK_LEFT_RZ] = "STACK_LEFT_RZ", + [ASAN_STACK_MID_RZ] = "STACK_MID_RZ", + [ASAN_STACK_RIGHT_RZ] = "STACK_RIGHT_RZ", + [ASAN_STACK_FREED] = "STACK_FREED", + [ASAN_GLOBAL_RZ] = "GLOBAL_RZ", + [ASAN_HEAP_RZ] = "", + [ASAN_HEAP_LEFT_RZ] = "HEAP_LEFT_RZ", + [ASAN_HEAP_RIGHT_RZ] = "HEAP_RIGHT_RZ", + [ASAN_HEAP_FREED] = "HEAP_FREED", + [0xff] = "", +}; + +#define CRASH_CONTEXT_BEFORE 5 +#define CRASH_CONTEXT_AFTER 5 + +static size_t +kasan_shadow_crashlog(uptr p, char *buf, size_t len) +{ + int i,j; + size_t l = 0; + int before = CRASH_CONTEXT_BEFORE; + int after = CRASH_CONTEXT_AFTER; + + uptr shadow = (uptr)SHADOW_FOR_ADDRESS(p); + uptr shadow_p = shadow; + + /* rewind to start of context block */ + shadow &= ~((uptr)0xf); + shadow -= 16 * before; + + for (i = 0; i < 1 + before + after; i++, shadow += 16) { + if (vm_map_round_page(shadow, PAGE_MASK) != vm_map_round_page(shadow_p, PAGE_MASK)) { + /* don't cross a page boundary, in case the shadow is unmapped */ + /* XXX: ideally we check instead of ignore */ + continue; + } + + l += snprintf(buf+l, len-l, " %#16lx: ", shadow); + + for (j = 0; j < 16; j++) { + uint8_t *x = (uint8_t *)(shadow + j); + l += snprintf(buf+l, len-l, "%02x ", (unsigned)*x); + } + l += snprintf(buf+l, len-l, "\n"); + } + + l += snprintf(buf+l, len-l, "\n"); + return l; +} + +static void NOINLINE +kasan_crash_report(uptr p, uptr width, unsigned access_type) +{ + const size_t len = 4096; + static char buf[len]; + size_t l = 0; + + uint8_t *shadow_ptr = SHADOW_FOR_ADDRESS(p); + uint8_t shadow_type = *shadow_ptr; + const char *shadow_str = shadow_strings[shadow_type]; + + kasan_handle_test(); + + buf[0] = '\0'; + l += snprintf(buf+l, len-l, + "KASan: invalid %lu-byte %s @ %#lx [%s]\n" + "Shadow %#02x @ %#lx\n\n", + width, access_type_str(access_type), p, shadow_str, + (unsigned)shadow_type, (unsigned long)shadow_ptr); + + l += kasan_shadow_crashlog(p, buf+l, len-l); + + panic("%s", buf); +} + +#define REPORT_DECLARE(n) \ + void __asan_report_load##n(uptr p) { kasan_crash_report(p, n, TYPE_LOAD); } \ + void __asan_report_store##n(uptr p) { kasan_crash_report(p, n, TYPE_STORE); } \ + void __asan_report_exp_load##n(uptr, int32_t); \ + void __asan_report_exp_store##n(uptr, int32_t); \ + void __asan_report_exp_load##n(uptr __unused p, int32_t __unused e) { ABI_UNSUPPORTED; } \ + void __asan_report_exp_store##n(uptr __unused p, int32_t __unused e) { ABI_UNSUPPORTED; } + +REPORT_DECLARE(1) +REPORT_DECLARE(2) +REPORT_DECLARE(4) +REPORT_DECLARE(8) +REPORT_DECLARE(16) + +void __asan_report_load_n(uptr p, unsigned long sz) { kasan_crash_report(p, sz, TYPE_LOAD); } +void __asan_report_store_n(uptr p, unsigned long sz) { kasan_crash_report(p, sz, TYPE_STORE); } + +/* unpoison the current stack */ +/* XXX: as an optimization, we could unpoison only up to the current stack depth */ +void NOINLINE +kasan_unpoison_curstack(void) +{ + kasan_unpoison_stack(ml_stack_base(), ml_stack_size()); +} + +void NOINLINE +__asan_handle_no_return(void) +{ + kasan_unpoison_curstack(); + kasan_unpoison_fakestack(current_thread()); +} + +bool NOINLINE +kasan_range_poisoned(vm_offset_t base, vm_size_t size, vm_offset_t *first_invalid) +{ + uint8_t *shadow; + vm_size_t i; + + if (!kasan_initialized || !kasan_enabled) { + return false; + } + + size += base & 0x07; + base &= ~(vm_offset_t)0x07; + + shadow = SHADOW_FOR_ADDRESS(base); + vm_size_t limit = (size + 7) / 8; + + /* XXX: to make debugging easier, catch unmapped shadow here */ + + for (i = 0; i < limit; i++, size -= 8) { + assert(size > 0); + uint8_t s = shadow[i]; + if (s == 0 || (size < 8 && s >= size && s <= 7)) { + /* valid */ + } else { + goto fail; + } + } + + return false; + + fail: + if (first_invalid) { + /* XXX: calculate the exact first byte that failed */ + *first_invalid = base + i*8; + } + return true; +} + +static void NOINLINE +kasan_init_globals(vm_offset_t base, vm_size_t size) +{ + struct asan_global *glob = (struct asan_global *)base; + struct asan_global *glob_end = (struct asan_global *)(base + size); + for (; glob < glob_end; glob++) { + /* handle one global */ + kasan_poison(glob->addr, glob->size, 0, glob->size_with_redzone - glob->size, ASAN_GLOBAL_RZ); + } +} + +void NOINLINE +kasan_load_kext(vm_offset_t base, vm_size_t __unused size, const void *bundleid) +{ + unsigned long sectsz; + void *sect; + + /* find the kasan globals segment/section */ + sect = getsectdatafromheader((void *)base, KASAN_GLOBAL_SEGNAME, KASAN_GLOBAL_SECTNAME, §sz); + if (sect) { + kasan_init_globals((vm_address_t)sect, (vm_size_t)sectsz); + kexts_loaded++; + } + +#if KASAN_DYNAMIC_BLACKLIST + kasan_dybl_load_kext(base, bundleid); +#endif +} + +void NOINLINE +kasan_unload_kext(vm_offset_t base, vm_size_t size) +{ + unsigned long sectsz; + void *sect; + + /* find the kasan globals segment/section */ + sect = getsectdatafromheader((void *)base, KASAN_GLOBAL_SEGNAME, KASAN_GLOBAL_SECTNAME, §sz); + if (sect) { + kasan_unpoison((void *)base, size); + kexts_loaded--; + } + +#if KASAN_DYNAMIC_BLACKLIST + kasan_dybl_unload_kext(base); +#endif +} + +void NOINLINE +kasan_disable(void) +{ + __asan_option_detect_stack_use_after_return = 0; + kasan_enabled = 0; +} + +static void NOINLINE +kasan_init_xnu_globals(void) +{ + const char *seg = KASAN_GLOBAL_SEGNAME; + const char *sect = KASAN_GLOBAL_SECTNAME; + unsigned long _size; + vm_offset_t globals; + vm_size_t size; + kernel_mach_header_t *header = (kernel_mach_header_t *)&_mh_execute_header; + + if (!header) { + printf("KASAN: failed to find kernel mach header\n"); + printf("KASAN: redzones for globals not poisoned\n"); + return; + } + + globals = (vm_offset_t)getsectdatafromheader(header, seg, sect, &_size); + if (!globals) { + printf("KASAN: failed to find segment %s section %s\n", seg, sect); + printf("KASAN: redzones for globals not poisoned\n"); + return; + } + size = (vm_size_t)_size; + + printf("KASAN: found (%s,%s) at %#lx + %lu\n", seg, sect, globals, size); + printf("KASAN: poisoning redzone for %lu globals\n", size / sizeof(struct asan_global)); + + kasan_init_globals(globals, size); +} + +void NOINLINE +kasan_late_init(void) +{ + kasan_init_fakestack(); + kasan_init_xnu_globals(); + +#if KASAN_DYNAMIC_BLACKLIST + kasan_init_dybl(); +#endif +} + +void NOINLINE +kasan_notify_stolen(vm_offset_t top) +{ + kasan_map_shadow(kernel_vtop, top - kernel_vtop, false); +} + +static void NOINLINE +kasan_debug_touch_mappings(vm_offset_t base, vm_size_t sz) +{ +#if KASAN_DEBUG + vm_size_t i; + uint8_t tmp1, tmp2; + + /* Hit every byte in the shadow map. Don't write due to the zero mappings. */ + for (i = 0; i < sz; i += sizeof(uint64_t)) { + vm_offset_t addr = base + i; + uint8_t *x = SHADOW_FOR_ADDRESS(addr); + tmp1 = *x; + asm volatile("" ::: "memory"); + tmp2 = *x; + asm volatile("" ::: "memory"); + assert(tmp1 == tmp2); + } +#else + (void)base; + (void)sz; +#endif +} + +void NOINLINE +kasan_init(void) +{ + simple_lock_init(&kasan_vm_lock, 0); + + /* Map all of the kernel text and data */ + kasan_map_shadow(kernel_vbase, kernel_vtop - kernel_vbase, false); + + kasan_arch_init(); + + kasan_initialized = 1; + kasan_enabled = 1; +} + +static void NOINLINE +kasan_notify_address_internal(vm_offset_t address, vm_size_t size, bool is_zero) +{ + assert(address < VM_MAX_KERNEL_ADDRESS); + + if (!kasan_initialized || !kasan_enabled) { + return; + } + + if (address < VM_MIN_KERNEL_AND_KEXT_ADDRESS) { + /* only map kernel addresses */ + return; + } + + if (!size) { + /* nothing to map */ + return; + } + + boolean_t flags; + kasan_lock(&flags); + kasan_map_shadow(address, size, is_zero); + kasan_unlock(flags); + kasan_debug_touch_mappings(address, size); +} + +void +kasan_notify_address(vm_offset_t address, vm_size_t size) +{ + kasan_notify_address_internal(address, size, false); +} + +/* + * Allocate read-only, all-zeros shadow for memory that can never be poisoned + */ +void +kasan_notify_address_nopoison(vm_offset_t address, vm_size_t size) +{ + kasan_notify_address_internal(address, size, true); +} + +/* + * + * allocator hooks + * + */ + +struct kasan_alloc_header { + uint32_t magic; + uint32_t alloc_size; + uint32_t user_size; + struct { + uint32_t left_rz : 28; + uint32_t frames : 4; + }; +}; +_Static_assert(sizeof(struct kasan_alloc_header) <= KASAN_GUARD_SIZE, "kasan alloc header exceeds guard size"); + +struct kasan_alloc_footer { + uint32_t backtrace[0]; +}; +_Static_assert(sizeof(struct kasan_alloc_footer) <= KASAN_GUARD_SIZE, "kasan alloc footer exceeds guard size"); + +#define MAGIC_XOR ((uint32_t)0xA110C8ED) +static uint32_t +magic_for_addr(vm_offset_t addr) +{ + return (uint32_t)addr ^ MAGIC_XOR; +} + +static struct kasan_alloc_header * +header_for_user_addr(vm_offset_t addr) +{ + return (void *)(addr - sizeof(struct kasan_alloc_header)); +} + +static struct kasan_alloc_footer * +footer_for_user_addr(vm_offset_t addr, vm_size_t *size) +{ + struct kasan_alloc_header *h = header_for_user_addr(addr); + vm_size_t rightrz = h->alloc_size - h->user_size - h->left_rz; + *size = rightrz; + return (void *)(addr + h->user_size); +} + +/* + * size: user-requested allocation size + * ret: minimum size for the real allocation + */ +vm_size_t +kasan_alloc_resize(vm_size_t size) +{ + vm_size_t tmp; + if (os_add_overflow(size, 4 * PAGE_SIZE, &tmp)) { + panic("allocation size overflow (%lu)", size); + } + + /* add left and right redzones */ + size += KASAN_GUARD_PAD; + + /* ensure the final allocation is an 8-byte multiple */ + size += 8 - (size % 8); + + return size; +} + +extern vm_offset_t vm_kernel_slid_base; + +static vm_size_t +kasan_alloc_bt(uint32_t *ptr, vm_size_t sz, vm_size_t skip) +{ + uintptr_t buf[BACKTRACE_MAXFRAMES]; + uintptr_t *bt = buf; + + sz /= sizeof(uint32_t); + vm_size_t frames = sz; + + if (frames > 0) { + frames = min(frames + skip, BACKTRACE_MAXFRAMES); + frames = backtrace(bt, frames); + + while (frames > sz && skip > 0) { + bt++; + frames--; + skip--; + } + + /* only store the offset from kernel base, and cram that into 32 + * bits */ + for (vm_size_t i = 0; i < frames; i++) { + ptr[i] = (uint32_t)(bt[i] - vm_kernel_slid_base); + } + } + return frames; +} + +/* + * addr: base address of full allocation (including redzones) + * size: total size of allocation (include redzones) + * req: user-requested allocation size + * lrz: size of the left redzone in bytes + * ret: address of usable allocation + */ +vm_address_t +kasan_alloc(vm_offset_t addr, vm_size_t size, vm_size_t req, vm_size_t leftrz) +{ + if (!addr) { + return 0; + } + assert(size > 0); + assert((addr % 8) == 0); + assert((size % 8) == 0); + + vm_size_t rightrz = size - req - leftrz; + + kasan_poison(addr, req, leftrz, rightrz, ASAN_HEAP_RZ); + kasan_rz_clobber(addr, req, leftrz, rightrz); + + addr += leftrz; + + /* stash the allocation sizes in the left redzone */ + struct kasan_alloc_header *h = header_for_user_addr(addr); + h->magic = magic_for_addr(addr); + h->left_rz = leftrz; + h->alloc_size = size; + h->user_size = req; + + /* ... and a backtrace in the right redzone */ + vm_size_t fsize; + struct kasan_alloc_footer *f = footer_for_user_addr(addr, &fsize); + h->frames = kasan_alloc_bt(f->backtrace, fsize, 2); + + return addr; +} + +/* + * addr: user pointer + * size: returns full original allocation size + * ret: original allocation ptr + */ +vm_address_t +kasan_dealloc(vm_offset_t addr, vm_size_t *size) +{ + assert(size && addr); + struct kasan_alloc_header *h = header_for_user_addr(addr); + if (h->magic != magic_for_addr(addr)) { + /* no point blacklisting here - this is fatal */ + kasan_crash_report(addr, *size, TYPE_FREE); + } + *size = h->alloc_size; + return addr - h->left_rz; +} + +/* + * return the original user-requested allocation size + * addr: user alloc pointer + */ +vm_size_t +kasan_user_size(vm_offset_t addr) +{ + struct kasan_alloc_header *h = header_for_user_addr(addr); + assert(h->magic == magic_for_addr(addr)); + return h->user_size; +} + +/* + * Verify that `addr' (user pointer) is a valid allocation of `type' + */ +void +kasan_check_free(vm_offset_t addr, vm_size_t size, unsigned heap_type) +{ + struct kasan_alloc_header *h = header_for_user_addr(addr); + + /* map heap type to an internal access type */ + unsigned type; + if (heap_type == KASAN_HEAP_KALLOC) { + type = TYPE_KFREE; + } else if (heap_type == KASAN_HEAP_ZALLOC) { + type = TYPE_ZFREE; + } else if (heap_type == KASAN_HEAP_FAKESTACK) { + type = TYPE_FSFREE; + } + + /* check the magic matches */ + if (h->magic != magic_for_addr(addr)) { + if (kasan_is_blacklisted(type)) { + return; + } + kasan_crash_report(addr, size, type); + } + + /* check the freed size matches what we recorded at alloc time */ + if (h->user_size != size) { + if (kasan_is_blacklisted(type)) { + return; + } + kasan_crash_report(addr, size, type); + } + + vm_size_t rightrz_sz = h->alloc_size - h->left_rz - h->user_size; + + /* Check that the redzones are valid */ + kasan_assert_shadow(addr - h->left_rz, h->left_rz, addr, ASAN_HEAP_LEFT_RZ); + kasan_assert_shadow(addr + h->user_size, rightrz_sz, addr, ASAN_HEAP_RIGHT_RZ); + + /* Check the allocated range is not poisoned */ + kasan_check_range((void *)addr, size, type); +} + +/* + * + * Quarantine + * + */ + +struct freelist_entry { + uint32_t magic; + uint32_t checksum; + STAILQ_ENTRY(freelist_entry) list; + union { + struct { + vm_size_t size : 28; + vm_size_t user_size : 28; + vm_size_t frames : 4; /* number of frames in backtrace */ + vm_size_t __unused : 4; + }; + uint64_t bits; + }; + zone_t zone; + uint32_t backtrace[]; +}; +_Static_assert(sizeof(struct freelist_entry) <= KASAN_GUARD_PAD, "kasan freelist header exceeds padded size"); + +#define FREELIST_MAGIC_XOR ((uint32_t)0xF23333D) +static uint32_t +freelist_magic(vm_offset_t addr) +{ + return (uint32_t)addr ^ FREELIST_MAGIC_XOR; +} + +struct quarantine { + STAILQ_HEAD(freelist_head, freelist_entry) freelist; + unsigned long entries; + unsigned long max_entries; + vm_size_t size; + vm_size_t max_size; +}; + +struct quarantine quarantines[] = { + { STAILQ_HEAD_INITIALIZER((quarantines[KASAN_HEAP_ZALLOC].freelist)), 0, QUARANTINE_ENTRIES, 0, QUARANTINE_MAXSIZE }, + { STAILQ_HEAD_INITIALIZER((quarantines[KASAN_HEAP_KALLOC].freelist)), 0, QUARANTINE_ENTRIES, 0, QUARANTINE_MAXSIZE }, + { STAILQ_HEAD_INITIALIZER((quarantines[KASAN_HEAP_FAKESTACK].freelist)), 0, QUARANTINE_ENTRIES, 0, QUARANTINE_MAXSIZE } +}; + +/* + * addr, sizep: pointer/size of full allocation including redzone + */ +void NOINLINE +kasan_free_internal(void **addrp, vm_size_t *sizep, int type, + zone_t *zone, vm_size_t user_size, int locked, + bool doquarantine) +{ + vm_size_t size = *sizep; + vm_offset_t addr = *(vm_offset_t *)addrp; + + assert(type >= 0 && type < KASAN_HEAP_TYPES); + if (type == KASAN_HEAP_KALLOC) { + /* zero-size kalloc allocations are allowed */ + assert(!zone); + } else if (type == KASAN_HEAP_ZALLOC) { + assert(zone && user_size); + } else if (type == KASAN_HEAP_FAKESTACK) { + assert(zone && user_size); + } + + /* clobber the entire freed region */ + kasan_rz_clobber(addr, 0, size, 0); + + if (!doquarantine || !quarantine_enabled) { + goto free_current; + } + + /* poison the entire freed region */ + uint8_t flags = (type == KASAN_HEAP_FAKESTACK) ? ASAN_STACK_FREED : ASAN_HEAP_FREED; + kasan_poison(addr, 0, size, 0, flags); + + struct freelist_entry *fle, *tofree = NULL; + struct quarantine *q = &quarantines[type]; + assert(size >= sizeof(struct freelist_entry)); + + /* create a new freelist entry */ + fle = (struct freelist_entry *)addr; + fle->magic = freelist_magic((vm_offset_t)fle); + fle->size = size; + fle->user_size = user_size; + fle->frames = 0; + fle->zone = ZONE_NULL; + if (zone) { + fle->zone = *zone; + } + if (type != KASAN_HEAP_FAKESTACK) { + fle->frames = kasan_alloc_bt(fle->backtrace, fle->size - sizeof(struct freelist_entry), 3); + } + + boolean_t flg; + if (!locked) { + kasan_lock(&flg); + } + + if (q->size + size > q->max_size) { + /* + * Adding this entry would put us over the max quarantine size. Free the + * larger of the current object and the quarantine head object. + */ + tofree = STAILQ_FIRST(&q->freelist); + if (fle->size > tofree->size) { + goto free_current_locked; + } + } + + STAILQ_INSERT_TAIL(&q->freelist, fle, list); + q->entries++; + q->size += size; + + /* free the oldest entry, if necessary */ + if (tofree || q->entries > q->max_entries) { + tofree = STAILQ_FIRST(&q->freelist); + STAILQ_REMOVE_HEAD(&q->freelist, list); + + assert(q->entries > 0 && q->size >= tofree->size); + q->entries--; + q->size -= tofree->size; + + if (type != KASAN_HEAP_KALLOC) { + assert((vm_offset_t)zone >= VM_MIN_KERNEL_AND_KEXT_ADDRESS && + (vm_offset_t)zone <= VM_MAX_KERNEL_ADDRESS); + *zone = tofree->zone; + } + + size = tofree->size; + addr = (vm_offset_t)tofree; + if (tofree->magic != freelist_magic(addr)) { + kasan_crash_report(addr, size, TYPE_FREE); + } + + /* clobber the quarantine header */ + kasan_rz_clobber(addr, 0, sizeof(struct freelist_entry), 0); + + } else { + /* quarantine is not full - don't really free anything */ + addr = 0; + } + + free_current_locked: + if (!locked) { + kasan_unlock(flg); + } + + free_current: + *addrp = (void *)addr; + if (addr) { + kasan_unpoison((void *)addr, size); + *sizep = size; + } +} + +void NOINLINE +kasan_free(void **addrp, vm_size_t *sizep, int type, zone_t *zone, + vm_size_t user_size, bool quarantine) +{ + kasan_free_internal(addrp, sizep, type, zone, user_size, 0, quarantine); +} + +uptr +__asan_load_cxx_array_cookie(uptr *p) +{ + uint8_t *shadow = SHADOW_FOR_ADDRESS((uptr)p); + if (*shadow == ASAN_ARRAY_COOKIE) { + return *p; + } else if (*shadow == ASAN_HEAP_FREED) { + return 0; + } else { + return *p; + } +} + +void +__asan_poison_cxx_array_cookie(uptr p) +{ + uint8_t *shadow = SHADOW_FOR_ADDRESS(p); + *shadow = ASAN_ARRAY_COOKIE; +} + +#define ACCESS_CHECK_DECLARE(type, sz, access_type) \ + void __asan_##type##sz(uptr addr) { \ + kasan_check_range((const void *)addr, sz, access_type); \ + } \ + void __asan_exp_##type##sz(uptr, int32_t); \ + void __asan_exp_##type##sz(uptr __unused addr, int32_t __unused e) { ABI_UNSUPPORTED; } + +ACCESS_CHECK_DECLARE(load, 1, TYPE_LOAD); +ACCESS_CHECK_DECLARE(load, 2, TYPE_LOAD); +ACCESS_CHECK_DECLARE(load, 4, TYPE_LOAD); +ACCESS_CHECK_DECLARE(load, 8, TYPE_LOAD); +ACCESS_CHECK_DECLARE(load, 16, TYPE_LOAD); +ACCESS_CHECK_DECLARE(store, 1, TYPE_STORE); +ACCESS_CHECK_DECLARE(store, 2, TYPE_STORE); +ACCESS_CHECK_DECLARE(store, 4, TYPE_STORE); +ACCESS_CHECK_DECLARE(store, 8, TYPE_STORE); +ACCESS_CHECK_DECLARE(store, 16, TYPE_STORE); + +void +__asan_loadN(uptr addr, size_t sz) +{ + kasan_check_range((const void *)addr, sz, TYPE_LOAD); +} + +void +__asan_storeN(uptr addr, size_t sz) +{ + kasan_check_range((const void *)addr, sz, TYPE_STORE); +} + +void __asan_exp_loadN(uptr, size_t, int32_t); +void __asan_exp_storeN(uptr, size_t, int32_t); +void __asan_exp_loadN(uptr __unused addr, size_t __unused sz, int32_t __unused e) { ABI_UNSUPPORTED; } +void __asan_exp_storeN(uptr __unused addr, size_t __unused sz, int32_t __unused e) { ABI_UNSUPPORTED; } + +void __asan_report_exp_load_n(uptr, unsigned long, int32_t); +void __asan_report_exp_store_n(uptr, unsigned long, int32_t); +void __asan_report_exp_load_n(uptr __unused p, unsigned long __unused sz, int32_t __unused e) { ABI_UNSUPPORTED; } +void __asan_report_exp_store_n(uptr __unused p, unsigned long __unused sz, int32_t __unused e) { ABI_UNSUPPORTED; } + +static void +kasan_set_shadow(uptr addr, size_t sz, uint8_t val) +{ + __nosan_memset((void *)addr, val, sz); +} + +#define SET_SHADOW_DECLARE(val) \ + void __asan_set_shadow_##val(uptr addr, size_t sz) { \ + kasan_set_shadow(addr, sz, 0x##val); \ + } + +SET_SHADOW_DECLARE(00) +SET_SHADOW_DECLARE(f1) +SET_SHADOW_DECLARE(f2) +SET_SHADOW_DECLARE(f3) +SET_SHADOW_DECLARE(f5) +SET_SHADOW_DECLARE(f8) + +/* + * XXX: implement these + */ + +void __asan_alloca_poison(uptr addr, uptr size) +{ + (void)addr; + (void)size; +} + +void __asan_allocas_unpoison(uptr top, uptr bottom) +{ + (void)top; + (void)bottom; +} + +void +__sanitizer_ptr_sub(uptr a, uptr b) +{ + (void)a; + (void)b; +} + +void +__sanitizer_ptr_cmp(uptr a, uptr b) +{ + (void)a; + (void)b; +} + +void +__asan_poison_stack_memory(uptr addr, size_t size) +{ + (void)addr; + (void)size; +} + +void +__asan_unpoison_stack_memory(uptr addr, size_t size) +{ + (void)addr; + (void)size; +} + +void +__sanitizer_annotate_contiguous_container(const void *beg, + const void *end, + const void *old_mid, + const void *new_mid) +{ + (void)beg; + (void)end; + (void)old_mid; + (void)new_mid; +} + +/* + */ + +void +__asan_init(void) +{ +} + +#define VERSION_DECLARE(v) \ + void __asan_version_mismatch_check_##v(void); \ + void __asan_version_mismatch_check_##v(void) {} + +VERSION_DECLARE(v8) +VERSION_DECLARE(apple_802) +VERSION_DECLARE(apple_900) + +void +__asan_register_globals(uptr __unused a, uptr __unused b) +{ + ABI_UNSUPPORTED; +} + +void +__asan_unregister_globals(uptr __unused a, uptr __unused b) +{ + ABI_UNSUPPORTED; +} + +void +__asan_register_image_globals(uptr __unused ptr) +{ +} + +void +__asan_unregister_image_globals(uptr __unused ptr) +{ +} + +void +__asan_init_v5(void) +{ +} + +void +__asan_before_dynamic_init(uptr __unused arg) +{ +} + +void +__asan_after_dynamic_init(void) +{ +} + + +/* + * + * SYSCTL + * + */ + +static int +sysctl_kasan_test(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req) +{ + int mask = 0; + int ch; + int err; + err = sysctl_io_number(req, 0, sizeof(int), &mask, &ch); + + if (!err && mask) { + kasan_test(mask, arg2); + } + + return err; +} + +SYSCTL_DECL(kasan); +SYSCTL_NODE(_kern, OID_AUTO, kasan, CTLFLAG_RW | CTLFLAG_LOCKED, 0, ""); + +SYSCTL_COMPAT_INT(_kern_kasan, OID_AUTO, available, CTLFLAG_RD, NULL, KASAN, ""); +SYSCTL_INT(_kern_kasan, OID_AUTO, enabled, CTLFLAG_RD, &kasan_enabled, 0, ""); +SYSCTL_INT(_kern_kasan, OID_AUTO, quarantine, CTLFLAG_RW, &quarantine_enabled, 0, ""); +SYSCTL_LONG(_kern_kasan, OID_AUTO, memused, CTLFLAG_RD, &shadow_pages_used, ""); +SYSCTL_LONG(_kern_kasan, OID_AUTO, memtotal, CTLFLAG_RD, &shadow_pages_total, ""); +SYSCTL_LONG(_kern_kasan, OID_AUTO, kexts, CTLFLAG_RD, &kexts_loaded, ""); + +SYSCTL_COMPAT_INT(_kern_kasan, OID_AUTO, debug, CTLFLAG_RD, NULL, KASAN_DEBUG, ""); +SYSCTL_COMPAT_INT(_kern_kasan, OID_AUTO, zalloc, CTLFLAG_RD, NULL, KASAN_ZALLOC, ""); +SYSCTL_COMPAT_INT(_kern_kasan, OID_AUTO, kalloc, CTLFLAG_RD, NULL, KASAN_KALLOC, ""); +SYSCTL_COMPAT_INT(_kern_kasan, OID_AUTO, fakestack, CTLFLAG_RD, NULL, FAKESTACK, ""); +SYSCTL_COMPAT_INT(_kern_kasan, OID_AUTO, dynamicbl, CTLFLAG_RD, NULL, KASAN_DYNAMIC_BLACKLIST, ""); +SYSCTL_COMPAT_INT(_kern_kasan, OID_AUTO, memintrinsics, CTLFLAG_RD, NULL, MEMINTRINSICS, ""); + +SYSCTL_PROC(_kern_kasan, OID_AUTO, test, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + 0, 0, sysctl_kasan_test, "I", ""); + +SYSCTL_PROC(_kern_kasan, OID_AUTO, fail, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + 0, 1, sysctl_kasan_test, "I", ""); diff --git a/san/kasan.h b/san/kasan.h new file mode 100644 index 000000000..a4d985d56 --- /dev/null +++ b/san/kasan.h @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _KASAN_H_ +#define _KASAN_H_ + +#if KERNEL_PRIVATE + +#if KASAN && !__has_feature(address_sanitizer) +# error "KASAN selected, but not enabled in compiler" +#endif + +#if !KASAN && __has_feature(address_sanitizer) +# error "ASAN enabled in compiler, but kernel is not configured for KASAN" +#endif + +#define KASAN_GLOBAL_SEGNAME "__DATA" +#define KASAN_GLOBAL_SECTNAME "__asan_globals" + +typedef uintptr_t uptr; + +#if KASAN + +#define KASAN_KALLOC 1 +#define KASAN_ZALLOC 1 +#define KASAN_DYNAMIC_BLACKLIST 1 + +#define KASAN_GUARD_SIZE (16) +#define KASAN_GUARD_PAD (KASAN_GUARD_SIZE * 2) + +#define KASAN_HEAP_ZALLOC 0 +#define KASAN_HEAP_KALLOC 1 +#define KASAN_HEAP_FAKESTACK 2 +#define KASAN_HEAP_TYPES 3 + +/* shadow map byte values */ +#define ASAN_VALID 0x00 +#define ASAN_PARTIAL1 0x01 +#define ASAN_PARTIAL2 0x02 +#define ASAN_PARTIAL3 0x03 +#define ASAN_PARTIAL4 0x04 +#define ASAN_PARTIAL5 0x05 +#define ASAN_PARTIAL6 0x06 +#define ASAN_PARTIAL7 0x07 +#define ASAN_ARRAY_COOKIE 0xac +#define ASAN_STACK_RZ 0xf0 +#define ASAN_STACK_LEFT_RZ 0xf1 +#define ASAN_STACK_MID_RZ 0xf2 +#define ASAN_STACK_RIGHT_RZ 0xf3 +#define ASAN_STACK_FREED 0xf5 +#define ASAN_GLOBAL_RZ 0xf9 +#define ASAN_HEAP_RZ 0xe9 +#define ASAN_HEAP_LEFT_RZ 0xfa +#define ASAN_HEAP_RIGHT_RZ 0xfb +#define ASAN_HEAP_FREED 0xfd + +/* + * KASAN internal interface + */ + +__BEGIN_DECLS +void kasan_map_shadow(vm_offset_t address, vm_size_t size, bool is_zero); +void kasan_disable(void); +void kasan_reserve_memory(void *); +void kasan_late_init(void); +void kasan_init(void); +void kasan_notify_stolen(vm_offset_t top); + +void kasan_load_kext(vm_offset_t base, vm_size_t size, const void *bundleid); +void kasan_unload_kext(vm_offset_t base, vm_size_t size); + +void kasan_poison_range(vm_offset_t base, vm_size_t sz, uint8_t flags); +void kasan_notify_address(vm_offset_t address, vm_size_t size); +void kasan_notify_address_nopoison(vm_offset_t address, vm_size_t size); +void kasan_unpoison_stack(vm_offset_t stack, vm_size_t size); +void kasan_unpoison_fakestack(thread_t thread); + +struct kasan_test; +void __kasan_runtests(struct kasan_test *, int numtests); + +#if XNU_KERNEL_PRIVATE +extern long shadow_pages_total; + +#if __arm64__ +void kasan_notify_address_zero(vm_offset_t, vm_size_t); +#elif __x86_64__ +extern void kasan_map_low_fixed_regions(void); +extern unsigned shadow_stolen_idx; +extern vm_offset_t shadow_pnext, shadow_ptop; +#endif +#endif +/* + * Allocator hooks + */ + +vm_size_t kasan_alloc_resize(vm_size_t size); +vm_size_t kasan_user_size(vm_offset_t addr); + +vm_address_t kasan_alloc(vm_offset_t addr, vm_size_t size, vm_size_t req, vm_size_t leftrz); +vm_address_t kasan_dealloc(vm_offset_t addr, vm_size_t *size); + +void kasan_check_free(vm_offset_t addr, vm_size_t size, unsigned type); +void kasan_free(void **addr, vm_size_t *size, int type, zone_t *zone, vm_size_t user_size, bool doquarantine); + +__END_DECLS + +/* thread interface */ +struct kasan_thread_data { + int in_fakestack; + LIST_HEAD(fakestack_header_list, fakestack_header) fakestack_head; +}; +struct kasan_thread_data *kasan_get_thread_data(thread_t); +void kasan_init_thread(struct kasan_thread_data *); + +#endif /* KASAN */ + +#if __has_feature(address_sanitizer) +# define NOKASAN __attribute__ ((no_sanitize_address)) +#else +# define NOKASAN +#endif + +/* + * Delimit areas of code that may do kasan-unsafe operations + */ +__BEGIN_DECLS +#if KASAN +void kasan_unsafe_start(void); +void kasan_unsafe_end(void); +#else +static inline void kasan_unsafe_start(void) {} +static inline void kasan_unsafe_end(void) {} +#endif +__END_DECLS + +/* + * ASAN callbacks - inserted by the compiler + */ + +extern int __asan_option_detect_stack_use_after_return; +extern const uintptr_t __asan_shadow_memory_dynamic_address; + +__BEGIN_DECLS +void __asan_report_load1(uptr p); +void __asan_report_load2(uptr p); +void __asan_report_load4(uptr p); +void __asan_report_load8(uptr p); +void __asan_report_load16(uptr p); +void __asan_report_store1(uptr p); +void __asan_report_store2(uptr p); +void __asan_report_store4(uptr p); +void __asan_report_store8(uptr p); +void __asan_report_store16(uptr p); +void __asan_report_load_n(uptr p, unsigned long size); +void __asan_report_store_n(uptr p, unsigned long size); +void __asan_handle_no_return(void); +uptr __asan_stack_malloc_0(size_t); +uptr __asan_stack_malloc_1(size_t); +uptr __asan_stack_malloc_2(size_t); +uptr __asan_stack_malloc_3(size_t); +uptr __asan_stack_malloc_4(size_t); +uptr __asan_stack_malloc_5(size_t); +uptr __asan_stack_malloc_6(size_t); +uptr __asan_stack_malloc_7(size_t); +uptr __asan_stack_malloc_8(size_t); +uptr __asan_stack_malloc_9(size_t); +uptr __asan_stack_malloc_10(size_t); +void __asan_stack_free_0(uptr, size_t); +void __asan_stack_free_1(uptr, size_t); +void __asan_stack_free_2(uptr, size_t); +void __asan_stack_free_3(uptr, size_t); +void __asan_stack_free_4(uptr, size_t); +void __asan_stack_free_5(uptr, size_t); +void __asan_stack_free_6(uptr, size_t); +void __asan_stack_free_7(uptr, size_t); +void __asan_stack_free_8(uptr, size_t); +void __asan_stack_free_9(uptr, size_t); +void __asan_stack_free_10(uptr, size_t); +void __asan_poison_cxx_array_cookie(uptr); +uptr __asan_load_cxx_array_cookie(uptr *); +void __asan_poison_stack_memory(uptr addr, size_t size); +void __asan_unpoison_stack_memory(uptr addr, size_t size); +void __asan_alloca_poison(uptr addr, uptr size); +void __asan_allocas_unpoison(uptr top, uptr bottom); +void __asan_load1(uptr); +void __asan_load2(uptr); +void __asan_load4(uptr); +void __asan_load8(uptr); +void __asan_load16(uptr); +void __asan_loadN(uptr, size_t); +void __asan_store1(uptr); +void __asan_store2(uptr); +void __asan_store4(uptr); +void __asan_store8(uptr); +void __asan_store16(uptr); +void __asan_storeN(uptr, size_t); +void __sanitizer_ptr_sub(uptr a, uptr b); +void __sanitizer_ptr_cmp(uptr a, uptr b); +void __sanitizer_annotate_contiguous_container(const void *beg, const void *end, const void *old_mid, const void *new_mid); + +void __asan_set_shadow_00(uptr, size_t); +void __asan_set_shadow_f1(uptr, size_t); +void __asan_set_shadow_f2(uptr, size_t); +void __asan_set_shadow_f3(uptr, size_t); +void __asan_set_shadow_f5(uptr, size_t); +void __asan_set_shadow_f8(uptr, size_t); + +void __asan_init_v5(void); +void __asan_before_dynamic_init(uptr); +void __asan_after_dynamic_init(void); +void __asan_unregister_globals(uptr a, uptr b); +void __asan_register_globals(uptr a, uptr b); +void __asan_init(void); +void __asan_unregister_image_globals(uptr); +void __asan_register_image_globals(uptr); +__END_DECLS + +#endif /* KERNEL_PRIVATE */ +#endif /* _KASAN_H_ */ diff --git a/san/kasan_dynamic_blacklist.c b/san/kasan_dynamic_blacklist.c new file mode 100644 index 000000000..97fc3a561 --- /dev/null +++ b/san/kasan_dynamic_blacklist.c @@ -0,0 +1,473 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kasan.h" +#include "kasan_internal.h" + +#if KASAN_DYNAMIC_BLACKLIST + +#define MAX_FRAMES 8 +#define HASH_NBUCKETS 128U +#define HASH_MASK (HASH_NBUCKETS-1) +#define HASH_CACHE_NENTRIES 128 + +struct blacklist_entry { + const char *kext_name; + const char *func_name; + const unsigned type_mask; + + /* internal */ + uint64_t count; +}; + +#include "kasan_blacklist_dynamic.h" +static const size_t blacklist_entries = sizeof(blacklist)/sizeof(blacklist[0]); + +decl_simple_lock_data(static, _dybl_lock); + +static void +dybl_lock(boolean_t *b) +{ + *b = ml_set_interrupts_enabled(false); + simple_lock(&_dybl_lock); +} + +static void +dybl_unlock(boolean_t b) +{ + simple_unlock(&_dybl_lock); + ml_set_interrupts_enabled(b); +} + + +/* + * blacklist call site hash table + */ + +struct blacklist_hash_entry { + SLIST_ENTRY(blacklist_hash_entry) chain; // next element in chain + struct blacklist_entry *ble; // blacklist entry that this caller is an instance of + uintptr_t addr; // callsite address + uint64_t count; // hit count +}; + +struct hash_chain_head { + SLIST_HEAD(, blacklist_hash_entry); +}; + +unsigned cache_next_entry = 0; +struct blacklist_hash_entry blhe_cache[HASH_CACHE_NENTRIES]; +struct hash_chain_head hash_buckets[HASH_NBUCKETS]; + +static struct blacklist_hash_entry * +alloc_hash_entry(void) +{ + unsigned idx = cache_next_entry++; + if (idx >= HASH_CACHE_NENTRIES) { + cache_next_entry = HASH_CACHE_NENTRIES; // avoid overflow + return NULL; + } + return &blhe_cache[idx]; +} + +static unsigned +hash_addr(uintptr_t addr) +{ + addr ^= (addr >> 7); /* mix in some of the bits likely to select the kext */ + return (unsigned)addr & HASH_MASK; +} + +static struct blacklist_hash_entry * +blacklist_hash_lookup(uintptr_t addr) +{ + unsigned idx = hash_addr(addr); + struct blacklist_hash_entry *blhe; + + SLIST_FOREACH(blhe, &hash_buckets[idx], chain) { + if (blhe->addr == addr) { + return blhe; + } + } + + return NULL; +} + +static struct blacklist_hash_entry * +blacklist_hash_add(uintptr_t addr, struct blacklist_entry *ble) +{ + unsigned idx = hash_addr(addr); + + struct blacklist_hash_entry *blhe = alloc_hash_entry(); + if (!blhe) { + return NULL; + } + + blhe->ble = ble; + blhe->addr = addr; + blhe->count = 1; + + SLIST_INSERT_HEAD(&hash_buckets[idx], blhe, chain); + + return blhe; +} + +static void +hash_drop(void) +{ + if (cache_next_entry > 0) { + bzero(&hash_buckets, sizeof(hash_buckets)); + bzero(&blhe_cache, sizeof(struct blacklist_hash_entry) * cache_next_entry); + cache_next_entry = 0; + } +} + +/* + * kext range lookup tree + */ + +struct range_tree_entry { + RB_ENTRY(range_tree_entry) tree; + + uintptr_t base; + + struct { + uint64_t size : 63; + uint64_t accessed : 1; // blacklist entry exists in this range + }; + + /* kext name */ + const char *bundleid; + + /* mach header for corresponding kext */ + kernel_mach_header_t *mh; +}; + +static int NOINLINE +range_tree_cmp(const struct range_tree_entry *e1, const struct range_tree_entry *e2) +{ + if (e1->size == 0 || e2->size == 0) { + /* lookup */ + if (e1->base + e1->size < e2->base) { + return -1; + } else if (e1->base > e2->base + e2->size) { + return 1; + } else { + return 0; + } + } else { + /* compare */ + if (e1->base + e1->size <= e2->base) { + return -1; + } else if (e1->base >= e2->base + e2->size) { + return 1; + } else { + panic("bad compare\n"); + return 0; + } + } +} + +RB_HEAD(range_tree, range_tree_entry) range_tree_root; +RB_PROTOTYPE(range_tree, range_tree_entry, tree, range_tree_cmp); +RB_GENERATE(range_tree, range_tree_entry, tree, range_tree_cmp); + +/* for each executable section, insert a range tree entry */ +void +kasan_dybl_load_kext(uintptr_t addr, const char *kextname) +{ + int i; + + struct load_command *cmd = NULL; + kernel_mach_header_t *mh = (void *)addr; + + cmd = (struct load_command *)&mh[1]; + + for (i = 0; i < (int)mh->ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_KERNEL) { + kernel_segment_command_t *seg = (void *)cmd; + bool is_exec = seg->initprot & VM_PROT_EXECUTE; + +#if CONFIG_EMBEDDED + if (is_exec && strcmp("__TEXT_EXEC", seg->segname) != 0) { + is_exec = false; + } +#endif + + if (is_exec) { + struct range_tree_entry *e = kalloc(sizeof(struct range_tree_entry)); + bzero(e, sizeof(*e)); + + e->base = seg->vmaddr; + e->size = seg->vmsize; + e->bundleid = kextname; + e->mh = mh; + + boolean_t flag; + dybl_lock(&flag); + RB_INSERT(range_tree, &range_tree_root, e); + dybl_unlock(flag); + } + } + + cmd = (void *)((uintptr_t)cmd + cmd->cmdsize); + } +} + +void +kasan_dybl_unload_kext(uintptr_t addr) +{ + int i; + + struct load_command *cmd = NULL; + kernel_mach_header_t *mh = (void *)addr; + + cmd = (struct load_command *)&mh[1]; + + for (i = 0; i < (int)mh->ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_KERNEL) { + kernel_segment_command_t *seg = (void *)cmd; + bool is_exec = seg->initprot & VM_PROT_EXECUTE; + +#if CONFIG_EMBEDDED + if (is_exec && strcmp("__TEXT_EXEC", seg->segname) != 0) { + is_exec = false; + } +#endif + + if (is_exec) { + struct range_tree_entry key = { .base = seg->vmaddr, .size = 0 }; + struct range_tree_entry *e; + boolean_t flag; + dybl_lock(&flag); + e = RB_FIND(range_tree, &range_tree_root, &key); + if (e) { + RB_REMOVE(range_tree, &range_tree_root, e); + if (e->accessed) { + /* there was a blacklist entry in this range */ + hash_drop(); + } + } + dybl_unlock(flag); + + if (e) { + kfree(e, sizeof(*e)); + } + } + } + + cmd = (void *)((uintptr_t)cmd + cmd->cmdsize); + } +} + +/* + * return the closest function name at or before addr + */ +static const NOINLINE char * +addr_to_func(uintptr_t addr, const kernel_mach_header_t *mh) +{ + int i; + uintptr_t cur_addr = 0; + + const struct load_command *cmd = NULL; + const struct symtab_command *st = NULL; + const kernel_segment_command_t *le = NULL; + const char *strings; + const kernel_nlist_t *syms; + const char *cur_name = NULL; + + cmd = (const struct load_command *)&mh[1]; + + /* + * find the symtab command and linkedit segment + */ + for (i = 0; i < (int)mh->ncmds; i++) { + if (cmd->cmd == LC_SYMTAB) { + st = (const struct symtab_command *)cmd; + } else if (cmd->cmd == LC_SEGMENT_KERNEL) { + const kernel_segment_command_t *seg = (const void *)cmd; + if (!strcmp(seg->segname, SEG_LINKEDIT)) { + le = (const void *)cmd; + } + } + cmd = (const void *)((uintptr_t)cmd + cmd->cmdsize); + } + + /* locate the symbols and strings in the symtab */ + strings = (const void *)((le->vmaddr - le->fileoff) + st->stroff); + syms = (const void *)((le->vmaddr - le->fileoff) + st->symoff); + + /* + * iterate the symbols, looking for the closest one to `addr' + */ + for (i = 0; i < (int)st->nsyms; i++) { + + uint8_t n_type = syms[i].n_type; + const char *name = strings + syms[i].n_un.n_strx; + + if (n_type & N_STAB) { + /* ignore debug entries */ + continue; + } + + n_type &= N_TYPE; + if (syms[i].n_un.n_strx == 0 || !(n_type == N_SECT || n_type == N_ABS)) { + /* only use named and defined symbols */ + continue; + } + +#if 0 + if (mh != &_mh_execute_header) { + printf("sym '%s' 0x%x 0x%lx\n", name, (unsigned)syms[i].n_type, (unsigned long)syms[i].n_value); + } +#endif + + if (*name == '_') { + name += 1; + } + + /* this symbol is closer than the one we had */ + if (syms[i].n_value <= addr && syms[i].n_value > cur_addr) { + cur_name = name; + cur_addr = syms[i].n_value; + } + } + + /* best guess for name of function at addr */ + return cur_name; +} + +bool NOINLINE +kasan_is_blacklisted(unsigned mask) +{ + uint32_t nframes = 0; + uintptr_t frames[MAX_FRAMES]; + uintptr_t *bt = frames; + nframes = backtrace(bt, MAX_FRAMES); + boolean_t flag; + + if (nframes >= 2) { + /* ignore self and direct caller */ + nframes -= 2; + bt += 2; + } + + struct blacklist_hash_entry *blhe = NULL; + + dybl_lock(&flag); + + /* First check if any frame hits in the hash */ + for (uint32_t i = 0; i < nframes; i++) { + blhe = blacklist_hash_lookup(bt[i]); + if (blhe) { + if ((blhe->ble->type_mask & mask) != mask) { + /* wrong type */ + continue; + } + + /* hit */ + blhe->count++; + blhe->ble->count++; + // printf("KASan: blacklist cache hit (%s:%s [0x%lx] 0x%x)\n", + // ble->kext_name ?: "" , ble->func_name ?: "", VM_KERNEL_UNSLIDE(bt[i]), mask); + dybl_unlock(flag); + return true; + } + } + + /* no hits - slowpath */ + for (uint32_t i = 0; i < nframes; i++) { + + const char *kextname = NULL; + const char *funcname = NULL; + + struct range_tree_entry key = { .base = bt[i], .size = 0 }; + struct range_tree_entry *e = RB_FIND(range_tree, &range_tree_root, &key); + + if (!e) { + /* no match at this address - kinda weird? */ + continue; + } + + /* get the function and bundle name for the current frame */ + funcname = addr_to_func(bt[i], e->mh); + if (e->bundleid) { + kextname = strrchr(e->bundleid, '.'); + if (kextname) { + kextname++; + } else { + kextname = e->bundleid; + } + } + + // printf("%s: a = 0x%016lx,0x%016lx f = %s, k = %s\n", __func__, bt[i], VM_KERNEL_UNSLIDE(bt[i]), funcname, kextname); + + /* check if kextname or funcname are in the blacklist */ + for (size_t j = 0; j < blacklist_entries; j++) { + struct blacklist_entry *ble = &blacklist[j]; + uint64_t count; + + if ((ble->type_mask & mask) != mask) { + /* wrong type */ + continue; + } + + if (ble->kext_name && kextname && strncmp(kextname, ble->kext_name, KMOD_MAX_NAME) != 0) { + /* wrong kext name */ + continue; + } + + if (ble->func_name && funcname && strncmp(funcname, ble->func_name, 128) != 0) { + /* wrong func name */ + continue; + } + + /* found a matching function or kext */ + blhe = blacklist_hash_add(bt[i], ble); + count = ble->count++; + e->accessed = 1; + + dybl_unlock(flag); + + if (count == 0) { + printf("KASan: ignoring blacklisted violation (%s:%s [0x%lx] %d 0x%x)\n", + kextname, funcname, VM_KERNEL_UNSLIDE(bt[i]), i, mask); + } + + return true; + } + } + + dybl_unlock(flag); + return false; +} + +void +kasan_init_dybl(void) +{ + simple_lock_init(&_dybl_lock, 0); + + /* add the fake kernel kext */ + kasan_dybl_load_kext((uintptr_t)&_mh_execute_header, "__kernel__"); +} + +#else /* KASAN_DYNAMIC_BLACKLIST */ + +bool +kasan_is_blacklisted(unsigned __unused mask) +{ + return false; +} +#endif diff --git a/san/kasan_internal.h b/san/kasan_internal.h new file mode 100644 index 000000000..36ae0b234 --- /dev/null +++ b/san/kasan_internal.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _KASAN_INTERNAL_H_ +#define _KASAN_INTERNAL_H_ + +#include +#include +#include + +typedef uintptr_t uptr; + +/* + * KASAN features and config + */ +#define KASAN_DEBUG 1 +#define FAKESTACK 1 +#define MEMINTRINSICS 1 +/* KASAN_KALLOC defined in kasan.h */ +/* KASAN_ZALLOC defined in kasan.h */ +#define FAKESTACK_QUARANTINE (1 && FAKESTACK) + +#define QUARANTINE_ENTRIES 5000 +#define QUARANTINE_MAXSIZE (10UL * 1024 * 1024) + +#ifndef KASAN +# error KASAN undefined +#endif + +#ifndef KASAN_SHIFT +# error KASAN_SHIFT undefined +#endif + +#define ADDRESS_FOR_SHADOW(x) (((x) - KASAN_SHIFT) << 3) +#define SHADOW_FOR_ADDRESS(x) (uint8_t *)(((x) >> 3) + KASAN_SHIFT) + +#define NOINLINE __attribute__ ((noinline)) +#define ALWAYS_INLINE inline __attribute__((always_inline)) + +#define CLANG_MIN_VERSION(x) (defined(__apple_build_version__) && (__apple_build_version__ >= (x))) + +#define BIT(x) (1U << (x)) + +enum kasan_access_type { + /* exactly one of these bits must be set */ + TYPE_LOAD = BIT(0), + TYPE_STORE = BIT(1), + TYPE_KFREE = BIT(2), + TYPE_ZFREE = BIT(3), + TYPE_FSFREE = BIT(4), /* fakestack free */ + TYPE_MEMLD = BIT(5), /* memory intrinsic - load */ + TYPE_MEMSTR = BIT(6), /* memory intrinsic - store */ + TYPE_STRINGLD = BIT(7), /* string intrinsic - load */ + TYPE_STRINGSTR = BIT(8), /* string intrinsic - store */ + TYPE_TEST = BIT(15), + + /* masks */ + TYPE_LDSTR = TYPE_LOAD|TYPE_STORE, /* regular loads and stores */ + TYPE_FREE = TYPE_KFREE|TYPE_ZFREE|TYPE_FSFREE, + TYPE_MEM = TYPE_MEMLD|TYPE_MEMSTR, + TYPE_STRING = TYPE_STRINGLD|TYPE_STRINGSTR, + TYPE_LOAD_ALL = TYPE_LOAD|TYPE_MEMLD|TYPE_STRINGLD, + TYPE_STORE_ALL = TYPE_STORE|TYPE_MEMSTR|TYPE_STRINGSTR, + TYPE_ALL = ~0U +}; + +bool kasan_range_poisoned(vm_offset_t base, vm_size_t size, vm_offset_t *first_invalid); +void kasan_check_range(const void *x, size_t sz, unsigned access_type); +void kasan_test(int testno, int fail); +void kasan_handle_test(void); +void kasan_unpoison_curstack(void); +void kasan_free_internal(void **addrp, vm_size_t *sizep, int type, zone_t *, vm_size_t user_size, int locked, bool doquarantine); +void kasan_poison(vm_offset_t base, vm_size_t size, vm_size_t leftrz, vm_size_t rightrz, uint8_t flags); +void kasan_unpoison(void *base, vm_size_t size); +void kasan_lock(boolean_t *b); +void kasan_unlock(boolean_t b); +void kasan_init_fakestack(void); + +/* dynamic blacklist */ +void kasan_init_dybl(void); +bool kasan_is_blacklisted(unsigned type); +void kasan_dybl_load_kext(uintptr_t addr, const char *kextname); +void kasan_dybl_unload_kext(uintptr_t addr); + +/* arch-specific interface */ +void kasan_arch_init(void); + +extern vm_address_t kernel_vbase; +extern vm_address_t kernel_vtop; + +extern long shadow_pages_used; + +/* Describes the source location where a global is defined. */ +struct asan_global_source_location { + const char *filename; + int line_no; + int column_no; +}; + +/* Describes an instrumented global variable. */ +struct asan_global { + uptr addr; + uptr size; + uptr size_with_redzone; + const char *name; + const char *module; + uptr has_dynamic_init; + struct asan_global_source_location *location; +#if CLANG_MIN_VERSION(8020000) + uptr odr_indicator; +#endif +}; + +#if defined(__x86_64__) +# define _JBLEN ((9 * 2) + 3 + 16) +#endif + + +typedef int jmp_buf[_JBLEN]; +void _longjmp(jmp_buf env, int val); +int _setjmp(jmp_buf env); + +#endif /* _KASAN_INTERNAL_H_ */ diff --git a/san/memintrinsics.h b/san/memintrinsics.h new file mode 100644 index 000000000..5c7a75a23 --- /dev/null +++ b/san/memintrinsics.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _SAN_MEMINTRINSICS_H_ +#define _SAN_MEMINTRINSICS_H_ + +/* + * Non-sanitized versions of memory intrinsics + */ +static inline void *__nosan_memcpy(void *dst, const void *src, size_t sz) { return memcpy(dst, src, sz); } +static inline void *__nosan_memset(void *src, int c, size_t sz) { return memset(src, c, sz); } +static inline void *__nosan_memmove(void *src, const void *dst, size_t sz) { return memmove(src, dst, sz); } +static inline int __nosan_bcmp(const void *a, const void *b, size_t sz) { return bcmp(a, b, sz); } +static inline void __nosan_bcopy(const void *src, void *dst, size_t sz) { return bcopy(src, dst, sz); } +static inline int __nosan_memcmp(const void *a, const void *b, size_t sz) { return memcmp(a, b, sz); } +static inline void __nosan_bzero(void *dst, size_t sz) { return bzero(dst, sz); } + +static inline size_t __nosan_strlcpy(char *dst, const char *src, size_t sz) { return strlcpy(dst, src, sz); } +static inline char *__nosan_strncpy(char *dst, const char *src, size_t sz) { return strncpy(dst, src, sz); } +static inline size_t __nosan_strlcat(char *dst, const char *src, size_t sz) { return strlcat(dst, src, sz); } +static inline char *__nosan_strncat(char *dst, const char *src, size_t sz) { return strncat(dst, src, sz); } +static inline size_t __nosan_strnlen(const char *src, size_t sz) { return strnlen(src, sz); } +static inline size_t __nosan_strlen(const char *src) { return strlen(src); } + +#if KASAN +void *__asan_memcpy(void *src, const void *dst, size_t sz); +void *__asan_memset(void *src, int c, size_t sz); +void *__asan_memmove(void *src, const void *dst, size_t sz); +void __asan_bcopy(const void *src, void *dst, size_t sz); +void __asan_bzero(void *dst, size_t sz); +int __asan_bcmp(const void *a, const void *b, size_t sz); +int __asan_memcmp(const void *a, const void *b, size_t sz); + +size_t __asan_strlcpy(char *dst, const char *src, size_t sz); +char *__asan_strncpy(char *dst, const char *src, size_t sz); +size_t __asan_strlcat(char *dst, const char *src, size_t sz); +char *__asan_strncat(char *dst, const char *src, size_t sz); +size_t __asan_strnlen(const char *src, size_t sz); +size_t __asan_strlen(const char *src); + +#define memcpy __asan_memcpy +#define memmove __asan_memmove +#define memset __asan_memset +#define bcopy __asan_bcopy +#define bzero __asan_bzero +#define bcmp __asan_bcmp +#define memcmp __asan_memcmp + +#define strlcpy __asan_strlcpy +#define strncpy __asan_strncpy +#define strlcat __asan_strlcat +#define strncat __asan_strncat +// #define strnlen __asan_strnlen +// #define strlen __asan_strlen + +#endif + +#endif /* _SAN_MEMINTRINSICS_H_ */ diff --git a/san/tools/generate_dynamic_blacklist.py b/san/tools/generate_dynamic_blacklist.py new file mode 100755 index 000000000..3f4c06bed --- /dev/null +++ b/san/tools/generate_dynamic_blacklist.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +import sys +import re + +def type_map(x): + return "TYPE_" + x.upper() + +re_comments=re.compile(r'#.*$') + +bl = file(sys.argv[1]) + +print r'struct blacklist_entry blacklist[] = {' + +for line in bl.readlines(): + line = re_comments.sub("", line).strip() + + if not line: + continue + + fields = line.split(":") + if len(fields) != 3: + continue + + (kext, func, ty) = fields + + if kext == "": + kext = "NULL"; + else: + kext = '"' + kext + '"' + + if func == "": + func = "NULL"; + else: + func = '"' + func + '"' + + if ty == "": + ty = "all"; + + print """ {{ + .kext_name = {}, + .func_name = {}, + .type_mask = {}, + }},""".format(kext, func, type_map(ty)) + +print r'};' diff --git a/san/tools/kasan_install b/san/tools/kasan_install new file mode 100755 index 000000000..5052dc945 --- /dev/null +++ b/san/tools/kasan_install @@ -0,0 +1,159 @@ +#!/bin/bash + +# +# kasan_install: set up a system to run the KASan kernel. Run with "--uninstall" +# to reverse the setup. +# +# Installs a symlink to the kernel kasan in /System/Library/Kernels/kernel.kasan +# and adds kcsuffix=kasan to boot-args. +# + + +kernel_name=kernel.kasan +kernel_src=/AppleInternal/CoreOS/xnu_kasan/${kernel_name} +SLK=/System/Library/Kernels/ +kernel_dst=${SLK}${kernel_name} + +if [[ `whoami` != root ]] ; then + echo "Re-running with sudo" + sudo "$0" "$@" + exit $? +fi + +sip_enabled() { + csrutil status |grep -q enabled +} + +prompt() { + echo -n "$@ [y/N] " + read ans + case "$ans" in + [yY]*) return 0 ;; + *) return 1 ;; + esac +} + +kasan_install() { + + dosymlink=0 + dobootargs=0 + + if [[ ! -f $kernel_src ]] ; then + echo "No KASan kernel found at $kernel_src" + exit 1 + fi + + echo -n "Installing KASan kernel... " + + if [[ -L $kernel_dst && $kernel_dst -ef $kernel_src ]] ; then + echo "already installed." + elif [[ -f $kernel_dst ]] ; then + prompt "file exists. Overwrite?" && { + echo -n "Overwriting KASan kernel... " + dosymlink=1 + } + else + dosymlink=1 + fi + + # Use a temporary directory with a symlink to kernel.kasan. We can ditto + # from there into /S/L/K, even with SIP enabled. + [[ $dosymlink -eq 1 ]] && { + tmp=$(mktemp -d) || exit $? + ln -s "$kernel_src" "$tmp" || exit $? + ditto "$tmp" "$SLK" || exit $? + rm -r "$tmp" + echo "done." + } + + + echo -n "Checking KASan boot args... " + + bootargs=$(nvram boot-args | cut -f2) + cursuffix=$(echo $bootargs | sed -n 's/.*kcsuffix=\([^ ]\)/\1/p') + + if [[ "$cursuffix" == kasan ]] ; then + echo "already set." + elif [[ -n "$cursuffix" ]] ; then + prompt "custom kcsuffix ($cursuffix) is set. Overwrite?" && { + bootargs=$(echo "$bootargs" | sed 's/[ ]*kcsuffix=[^ ]*//') + dobootargs=1 + } + else + prompt "not set. Modify?" && { + dobootargs=1 + } + fi + + [[ $dobootargs -eq 1 ]] && { + echo -n "Adding boot arg kcsuffix=kasan... " + newlen=$(echo -n "$bootargs kcsuffix=kasan" |wc -c) + if [[ $newlen -ge 512 ]] ; then + echo "boot-args too long. Bailing." + exit 3 + fi + + nvram boot-args="$bootargs kcsuffix=kasan" || exit $? + echo "done." + } + + [[ $dosymlink -eq 1 ]] && { + echo -n "Triggering kernel cache rebuild... " + touch /System/Library/Extensions || exit $? + echo "done." + } + +} + + +kasan_uninstall() { + + echo -n "Removing kasan kernel... " + + dorm=0 + + if [[ -L $kernel_dst && $kernel_dst -ef $kernel_src ]] ; then + dorm=1 + elif [[ -f $kernel_dst ]] ; then + prompt "unexpected file. Remove anyway?" && { + dorm=1 + } + else + echo "not installed." + fi + + [[ $dorm -eq 1 ]] && { + if rm "$kernel_dst" ; then + echo "done." + else + if sip_enabled ; then + echo "failed due to SIP - this is normal." + fi + fi + } + + + echo -n "Removing boot args... " + + bootargs=$(nvram boot-args | cut -f2) + cursuffix=$(echo $bootargs | sed -n 's/.*kcsuffix=\([^ ]\)/\1/p') + + if [[ $cursuffix == "kasan" ]] ; then + prompt "remove kcsuffix=kasan?" && { + echo -n "Removing kcsuffix... " + bootargs=$(echo "$bootargs" | sed 's/[ ]*kcsuffix=[^ ]*//') + nvram boot-args="$bootargs" + echo "done." + } + else + echo "not set." + fi + +} + +case "$1" in + *uninstall|*del*|*remove|*rm) + kasan_uninstall ;; + *) + kasan_install ;; +esac diff --git a/security/Makefile b/security/Makefile index ca89616de..1917d6e86 100644 --- a/security/Makefile +++ b/security/Makefile @@ -6,12 +6,28 @@ export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir include $(MakeInc_cmd) include $(MakeInc_def) -DATAFILES = \ - mac.h \ - mac_policy.h - -PRIVATE_DATAFILES = \ +INCDIR=/usr/local/include + +# Installs header file for user level - +# $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders +# $(DSTROOT)/usr/(local/)include/ +DATAFILES= \ + mac.h + +# Installs header file for kernel extensions - +# $(DSTROOT)/System/Library/Frameworks/Kernel.framework/Headers +# $(DSTROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders +KERNELFILES= \ + +# Installs header file for Apple internal use in user level - +# $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders +PRIVATE_DATAFILES = ${DATAFILES} + +# Installs header file for Apple internal use for kernel extensions - +# $(DSTROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders +PRIVATE_KERNELFILES = \ _label.h \ + mac.h \ mac_alloc.h \ mac_data.h \ mac_framework.h \ @@ -19,20 +35,22 @@ PRIVATE_DATAFILES = \ mac_mach_internal.h \ mac_policy.h -# Installed in /usr/include/security/ INSTALL_MI_LIST = ${DATAFILES} +# /System/Library/Frameworks/System.framework/PrivateHeaders +INSTALL_MI_LCL_LIST = ${PRIVATE_DATAFILES} + INSTALL_MI_DIR = security -EXPORT_MI_LIST = $(sort ${DATAFILES} ${PRIVATE_DATAFILES}) +EXPORT_MI_LIST = ${PRIVATE_KERNELFILES} -EXPORT_MI_DIR = security +EXPORT_MI_DIR = ${INSTALL_MI_DIR} -# /System/Library/Frameworks/System.framework/PrivateHeaders -INSTALL_MI_LCL_LIST = ${PRIVATE_DATAFILES} +# /System/Library/Frameworks/Kernel.framework/Headers +INSTALL_KF_MI_LIST = $(sort ${KERNELFILES}) # /System/Library/Frameworks/Kernel.framework/PrivateHeaders -INSTALL_KF_MI_LCL_LIST = $(sort ${DATAFILES} ${PRIVATE_DATAFILES}) +INSTALL_KF_MI_LCL_LIST = $(sort ${KERNELFILES} ${PRIVATE_KERNELFILES}) COMP_SUBDIRS = conf diff --git a/security/_label.h b/security/_label.h index 509d3a07d..cb4d9e8a6 100644 --- a/security/_label.h +++ b/security/_label.h @@ -68,7 +68,15 @@ * XXXMAC: This shouldn't be exported to userland, but is because of ucred.h * and various other messes. */ +#if CONFIG_EMBEDDED +#if CONFIG_VNGUARD +#define MAC_MAX_SLOTS 4 +#else +#define MAC_MAX_SLOTS 3 +#endif +#else #define MAC_MAX_SLOTS 7 +#endif #define MAC_FLAG_INITIALIZED 0x0000001 /* Is initialized for use. */ diff --git a/security/conf/Makefile.arm b/security/conf/Makefile.arm new file mode 100644 index 000000000..cf0b8b6db --- /dev/null +++ b/security/conf/Makefile.arm @@ -0,0 +1,7 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/security/conf/Makefile.arm64 b/security/conf/Makefile.arm64 new file mode 100644 index 000000000..cf0b8b6db --- /dev/null +++ b/security/conf/Makefile.arm64 @@ -0,0 +1,7 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/security/conf/Makefile.template b/security/conf/Makefile.template index f857074e3..8330c0a5f 100644 --- a/security/conf/Makefile.template +++ b/security/conf/Makefile.template @@ -17,6 +17,7 @@ include $(MakeInc_def) # XXX: CFLAGS # CFLAGS+= -include meta_features.h -DBSD_KERNEL_PRIVATE +SFLAGS+= -include meta_features.h # Objects that don't want -Wcast-align warning (8474835) OBJS_NO_CAST_ALIGN = \ diff --git a/security/conf/files b/security/conf/files index a2cd80b6f..bcd319fde 100644 --- a/security/conf/files +++ b/security/conf/files @@ -18,6 +18,7 @@ security/mac_label.c optional config_macf security/mac_process.c optional config_macf security/mac_vfs.c optional config_macf security/mac_vfs_subr.c optional config_macf +security/mac_skywalk.c optional config_macf skywalk security/mac_system.c optional config_macf security/mac_sysv_sem.c optional config_macf security/mac_sysv_shm.c optional config_macf diff --git a/security/conf/files.arm b/security/conf/files.arm new file mode 100644 index 000000000..e69de29bb diff --git a/security/conf/files.arm64 b/security/conf/files.arm64 new file mode 100644 index 000000000..e69de29bb diff --git a/security/mac.h b/security/mac.h index 9563a7796..0e58baf99 100644 --- a/security/mac.h +++ b/security/mac.h @@ -126,27 +126,6 @@ struct user64_mac { }; #endif /* KERNEL */ -/* - * Flags to control which MAC subsystems are enforced - * on a per-process/thread/credential basis. - */ -#define MAC_SYSTEM_ENFORCE 0x0001 /* system management */ -#define MAC_PROC_ENFORCE 0x0002 /* process management */ -#define MAC_MACH_ENFORCE 0x0004 /* mach interfaces */ -#define MAC_VM_ENFORCE 0x0008 /* VM interfaces */ -#define MAC_FILE_ENFORCE 0x0010 /* file operations */ -#define MAC_SOCKET_ENFORCE 0x0020 /* socket operations */ -#define MAC_PIPE_ENFORCE 0x0040 /* pipes */ -#define MAC_VNODE_ENFORCE 0x0080 /* vnode operations */ -#define MAC_NET_ENFORCE 0x0100 /* network management */ -#define MAC_MBUF_ENFORCE 0x0200 /* network traffic */ -#define MAC_POSIXSEM_ENFORCE 0x0400 /* posix semaphores */ -#define MAC_POSIXSHM_ENFORCE 0x0800 /* posix shared memory */ -#define MAC_SYSVMSG_ENFORCE 0x1000 /* SysV message queues */ -#define MAC_SYSVSEM_ENFORCE 0x2000 /* SysV semaphores */ -#define MAC_SYSVSHM_ENFORCE 0x4000 /* SysV shared memory */ -#define MAC_ALL_ENFORCE 0x7fff /* enforce everything */ - /* * Device types for mac_iokit_check_device() */ diff --git a/security/mac_alloc.c b/security/mac_alloc.c index cdc8f8a1f..7c19ae14a 100644 --- a/security/mac_alloc.c +++ b/security/mac_alloc.c @@ -122,8 +122,8 @@ int mac_wire(void *start, void *end) { - return (vm_map_wire(kalloc_map, CAST_USER_ADDR_T(start), - CAST_USER_ADDR_T(end), VM_PROT_READ|VM_PROT_WRITE, FALSE)); + return (vm_map_wire_kernel(kalloc_map, CAST_USER_ADDR_T(start), + CAST_USER_ADDR_T(end), VM_PROT_READ|VM_PROT_WRITE, VM_KERN_MEMORY_SECURITY, FALSE)); } int diff --git a/security/mac_audit.c b/security/mac_audit.c index 2454b57aa..5459cf54a 100644 --- a/security/mac_audit.c +++ b/security/mac_audit.c @@ -117,12 +117,12 @@ mac_proc_check_getauid(struct proc *curp) int error; #if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_proc_enforce) - return 0; + /* 21167099 - only check if we allow write */ + if (!mac_proc_enforce) + return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); @@ -139,12 +139,12 @@ mac_proc_check_setauid(struct proc *curp, uid_t auid) int error; #if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_proc_enforce) - return 0; + /* 21167099 - only check if we allow write */ + if (!mac_proc_enforce) + return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) - return 0; + if (!mac_proc_check_enforce(curp)) + return 0; cred = kauth_cred_proc_ref(curp); MAC_CHECK(proc_check_setauid, cred, auid); @@ -160,12 +160,12 @@ mac_proc_check_getaudit(struct proc *curp) int error; #if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_proc_enforce) - return 0; + /* 21167099 - only check if we allow write */ + if (!mac_proc_enforce) + return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) - return 0; + if (!mac_proc_check_enforce(curp)) + return 0; cred = kauth_cred_proc_ref(curp); MAC_CHECK(proc_check_getaudit, cred); @@ -181,12 +181,12 @@ mac_proc_check_setaudit(struct proc *curp, struct auditinfo_addr *ai) int error; #if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_proc_enforce) - return 0; + /* 21167099 - only check if we allow write */ + if (!mac_proc_enforce) + return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) - return 0; + if (!mac_proc_check_enforce(curp)) + return 0; cred = kauth_cred_proc_ref(curp); MAC_CHECK(proc_check_setaudit, cred, ai); diff --git a/security/mac_base.c b/security/mac_base.c index b3cf964b9..ec5955df8 100644 --- a/security/mac_base.c +++ b/security/mac_base.c @@ -105,6 +105,9 @@ #include #endif +#if CONFIG_EMBEDDED +#include +#endif /* * define MB_DEBUG to display run-time debugging information @@ -182,17 +185,6 @@ unsigned int mac_label_vnodes = 0; SYSCTL_UINT(_security_mac, OID_AUTO, labelvnodes, SECURITY_MAC_CTLFLAGS, &mac_label_vnodes, 0, "Label all vnodes"); - -unsigned int mac_mmap_revocation = 0; -SYSCTL_UINT(_security_mac, OID_AUTO, mmap_revocation, SECURITY_MAC_CTLFLAGS, - &mac_mmap_revocation, 0, "Revoke mmap access to files on subject " - "relabel"); - -unsigned int mac_mmap_revocation_via_cow = 0; -SYSCTL_UINT(_security_mac, OID_AUTO, mmap_revocation_via_cow, SECURITY_MAC_CTLFLAGS, - &mac_mmap_revocation_via_cow, 0, "Revoke mmap access to files via " - "copy-on-write semantics, or by removing all write access"); - unsigned int mac_device_enforce = 1; SYSCTL_UINT(_security_mac, OID_AUTO, device_enforce, SECURITY_MAC_CTLFLAGS, &mac_device_enforce, 0, "Enforce MAC policy on device operations"); @@ -276,7 +268,12 @@ static lck_mtx_t *mac_policy_mtx; static int mac_policy_busy; +#if CONFIG_EMBEDDED +SECURITY_READ_ONLY_LATE(mac_policy_list_t) mac_policy_list; +SECURITY_READ_ONLY_LATE(static struct mac_policy_list_element) mac_policy_static_entries[MAC_POLICY_LIST_CHUNKSIZE]; +#else mac_policy_list_t mac_policy_list; +#endif /* * mac_label_element_list holds the master list of label namespaces for @@ -363,7 +360,11 @@ mac_policy_init(void) mac_policy_list.freehint = 0; mac_policy_list.chunks = 1; +#if CONFIG_EMBEDDED + mac_policy_list.entries = mac_policy_static_entries; +#else mac_policy_list.entries = kalloc(sizeof(struct mac_policy_list_element) * MAC_POLICY_LIST_CHUNKSIZE); +#endif bzero(mac_policy_list.entries, sizeof(struct mac_policy_list_element) * MAC_POLICY_LIST_CHUNKSIZE); @@ -640,7 +641,9 @@ int mac_policy_register(struct mac_policy_conf *mpc, mac_policy_handle_t *handlep, void *xd) { +#if !CONFIG_EMBEDDED struct mac_policy_list_element *tmac_policy_list_element; +#endif int error, slot, static_entry = 0; u_int i; @@ -672,6 +675,7 @@ mac_policy_register(struct mac_policy_conf *mpc, mac_policy_handle_t *handlep, } if (mac_policy_list.numloaded >= mac_policy_list.max) { +#if !CONFIG_EMBEDDED /* allocate new policy list array, zero new chunk */ tmac_policy_list_element = kalloc((sizeof(struct mac_policy_list_element) * @@ -695,6 +699,10 @@ mac_policy_register(struct mac_policy_conf *mpc, mac_policy_handle_t *handlep, /* Update maximums, etc */ mac_policy_list.max += MAC_POLICY_LIST_CHUNKSIZE; mac_policy_list.chunks++; +#else + printf("out of space in mac_policy_list.\n"); + return (ENOMEM); +#endif /* CONFIG_EMBEDDED */ } /* Check for policy with same name already loaded */ @@ -970,8 +978,8 @@ element_loop: mpc = mac_policy_list.entries[mll->mll_handle].mpc; if (mpc == NULL) continue; - mpo_externalize = *(typeof(mpo_externalize) *) - ((char *)mpc->mpc_ops + mpo_externalize_off); + mpo_externalize = *(const typeof(mpo_externalize) *) + ((const char *)mpc->mpc_ops + mpo_externalize_off); if (mpo_externalize == NULL) continue; error = sbuf_printf(sb, "%s/", name); @@ -1099,8 +1107,8 @@ element_loop: mpc = mac_policy_list.entries[mll->mll_handle].mpc; if (mpc == NULL) continue; - mpo_internalize = *(typeof(mpo_internalize) *) - ((char *)mpc->mpc_ops + mpo_internalize_off); + mpo_internalize = *(const typeof(mpo_internalize) *) + ((const char *)mpc->mpc_ops + mpo_internalize_off); if (mpo_internalize == NULL) continue; error = mpo_internalize(label, element_name, @@ -1993,12 +2001,6 @@ void mac_label_set(struct label *l __unused, int slot __unused, intptr_t v __unu return; } -void mac_proc_set_enforce(proc_t p, int enforce_flags); -void mac_proc_set_enforce(proc_t p __unused, int enforce_flags __unused) -{ - return; -} - int mac_iokit_check_hid_control(kauth_cred_t cred __unused); int mac_iokit_check_hid_control(kauth_cred_t cred __unused) { diff --git a/security/mac_framework.h b/security/mac_framework.h index ef711a32b..b5560a320 100644 --- a/security/mac_framework.h +++ b/security/mac_framework.h @@ -341,7 +341,6 @@ void mac_posixshm_label_init(struct pshminfo *pshm); int mac_priv_check(kauth_cred_t cred, int priv); int mac_priv_grant(kauth_cred_t cred, int priv); int mac_proc_check_debug(proc_t proc1, proc_t proc2); -int mac_proc_check_cpumon(proc_t curp); int mac_proc_check_proc_info(proc_t curp, proc_t target, int callnum, int flavor); int mac_proc_check_get_cs_info(proc_t curp, proc_t target, unsigned int op); int mac_proc_check_set_cs_info(proc_t curp, proc_t target, unsigned int op); @@ -369,7 +368,7 @@ int mac_proc_check_setlcid(proc_t proc1, proc_t proc2, int mac_proc_check_signal(proc_t proc1, proc_t proc2, int signum); int mac_proc_check_wait(proc_t proc1, proc_t proc2); -void mac_proc_set_enforce(proc_t p, int enforce_flags); +void mac_proc_notify_exit(proc_t proc); int mac_setsockopt_label(kauth_cred_t cred, struct socket *so, struct mac *extmac); int mac_socket_check_accept(kauth_cred_t cred, struct socket *so); @@ -381,6 +380,8 @@ int mac_socket_check_connect(kauth_cred_t cred, struct socket *so, int mac_socket_check_create(kauth_cred_t cred, int domain, int type, int protocol); int mac_socket_check_deliver(struct socket *so, struct mbuf *m); +int mac_socket_check_ioctl(kauth_cred_t cred, struct socket *so, + unsigned int cmd); int mac_socket_check_kqfilter(kauth_cred_t cred, struct knote *kn, struct socket *so); int mac_socket_check_listen(kauth_cred_t cred, struct socket *so); @@ -502,6 +503,8 @@ int mac_vnode_check_link(vfs_context_t ctx, struct vnode *dvp, int mac_vnode_check_listextattr(vfs_context_t ctx, struct vnode *vp); int mac_vnode_check_lookup(vfs_context_t ctx, struct vnode *dvp, struct componentname *cnp); +int mac_vnode_check_lookup_preflight(vfs_context_t ctx, struct vnode *dvp, + const char *path, size_t pathlen); int mac_vnode_check_open(vfs_context_t ctx, struct vnode *vp, int acc_mode); int mac_vnode_check_read(vfs_context_t ctx, @@ -532,7 +535,7 @@ int mac_vnode_check_setutimes(vfs_context_t ctx, struct vnode *vp, struct timespec atime, struct timespec mtime); int mac_vnode_check_signature(struct vnode *vp, struct cs_blob *cs_blob, struct image_params *imgp, - unsigned int *cs_flags, + unsigned int *cs_flags, unsigned int *signer_type, int flags); int mac_vnode_check_stat(vfs_context_t ctx, kauth_cred_t file_cred, struct vnode *vp); @@ -591,6 +594,8 @@ void mac_pty_notify_close(proc_t p, struct tty *tp, dev_t dev, struct label *lab int mac_kext_check_load(kauth_cred_t cred, const char *identifier); int mac_kext_check_unload(kauth_cred_t cred, const char *identifier); int mac_kext_check_query(kauth_cred_t cred); +int mac_skywalk_flow_check_connect(proc_t p, void *flow, const struct sockaddr *addr, int type, int protocol); +int mac_skywalk_flow_check_listen(proc_t p, void *flow, const struct sockaddr *addr, int type, int protocol); void psem_label_associate(struct fileproc *fp, struct vnode *vp, struct vfs_context *ctx); void pshm_label_associate(struct fileproc *fp, struct vnode *vp, struct vfs_context *ctx); diff --git a/security/mac_internal.h b/security/mac_internal.h index 4ea01c77a..0f034d23f 100644 --- a/security/mac_internal.h +++ b/security/mac_internal.h @@ -170,51 +170,31 @@ extern unsigned int mac_label_mbufs; extern unsigned int mac_label_vnodes; -static int mac_proc_check_enforce(proc_t p, int enforce_flags); +static bool mac_proc_check_enforce(proc_t p); -static __inline__ int mac_proc_check_enforce(proc_t p, int enforce_flags) +static __inline__ bool mac_proc_check_enforce(proc_t p) { #if CONFIG_MACF -#if SECURITY_MAC_CHECK_ENFORCE // 21167099 - only check if we allow write - return ((p->p_mac_enforce & enforce_flags) != 0); + // Don't apply policies to the kernel itself. + return (p != kernproc); #else -#pragma unused(p,enforce_flags) - return 1; -#endif // SECURITY_MAC_CHECK_ENFORCE -#else -#pragma unused(p,enforce_flags) - return 0; +#pragma unused(p) + return false; #endif // CONFIG_MACF } -static int mac_context_check_enforce(vfs_context_t ctx, int enforce_flags); -static void mac_context_set_enforce(vfs_context_t ctx, int enforce_flags); - -static __inline__ int mac_context_check_enforce(vfs_context_t ctx, int enforce_flags) -{ - proc_t proc = vfs_context_proc(ctx); - - if (proc == NULL) - return 0; - - return (mac_proc_check_enforce(proc, enforce_flags)); -} +static bool mac_cred_check_enforce(kauth_cred_t cred); -static __inline__ void mac_context_set_enforce(vfs_context_t ctx, int enforce_flags) +static __inline__ bool mac_cred_check_enforce(kauth_cred_t cred) { #if CONFIG_MACF - proc_t proc = vfs_context_proc(ctx); - - if (proc == NULL) - return; - - mac_proc_set_enforce(proc, enforce_flags); + return (cred != proc_ucred(kernproc)); #else -#pragma unused(ctx,enforce_flags) -#endif +#pragma unused(p) + return false; +#endif // CONFIG_MACF } - /* * MAC Framework infrastructure functions. */ diff --git a/security/mac_kext.c b/security/mac_kext.c index 404749060..f84a6cc99 100644 --- a/security/mac_kext.c +++ b/security/mac_kext.c @@ -1,3 +1,31 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + #include #include #include diff --git a/security/mac_mach.c b/security/mac_mach.c index 222b77f2a..5669e0daf 100644 --- a/security/mac_mach.c +++ b/security/mac_mach.c @@ -149,81 +149,105 @@ mac_thread_userret(struct thread *td) MAC_PERFORM(thread_userret, td); } -static struct label * -mac_exc_action_label_alloc(void) +/**** Exception Policy + * + * Note that the functions below do not fully follow the usual convention for mac policy functions + * in the kernel. Besides avoiding confusion in how the mac function names are mixed with the actual + * policy function names, we diverge because the exception policy is somewhat special: + * It is used in places where allocation and association must be separate, and its labels do not + * only belong to one type of object as usual, but to two (on exception actions and on tasks as + * crash labels). + */ + +// Label allocation and deallocation, may sleep. + +struct label * +mac_exc_create_label(void) { struct label *label = mac_labelzone_alloc(MAC_WAITOK); + if (label == NULL) { + return NULL; + } + + // Policy initialization of the label, typically performs allocations as well. + // (Unless the policy's full data really fits into a pointer size.) MAC_PERFORM(exc_action_label_init, label); + return label; } -static void -mac_exc_action_label_free(struct label *label) +void +mac_exc_free_label(struct label *label) { MAC_PERFORM(exc_action_label_destroy, label); mac_labelzone_free(label); } -void -mac_exc_action_label_init(struct exception_action *action) -{ - action->label = mac_exc_action_label_alloc(); - MAC_PERFORM(exc_action_label_associate, action, action->label); -} +// Action label initialization and teardown, may sleep. void -mac_exc_action_label_inherit(struct exception_action *parent, struct exception_action *child) +mac_exc_associate_action_label(struct exception_action *action, struct label *label) { - mac_exc_action_label_init(child); - MAC_PERFORM(exc_action_label_copy, parent->label, child->label); + action->label = label; + MAC_PERFORM(exc_action_label_associate, action, action->label); } void -mac_exc_action_label_destroy(struct exception_action *action) +mac_exc_free_action_label(struct exception_action *action) { - struct label *label = action->label; + mac_exc_free_label(action->label); action->label = NULL; - mac_exc_action_label_free(label); } -int mac_exc_action_label_update(struct task *task, struct exception_action *action) { - if (task == kernel_task) { - // The kernel may set exception ports without any check. - return 0; - } +// Action label update and inheritance, may NOT sleep and must be quick. - struct proc *p = mac_task_get_proc(task); - if (p == NULL) - return ESRCH; +int +mac_exc_update_action_label(struct exception_action *action, + struct label *newlabel) { + int error; + + MAC_CHECK(exc_action_label_update, action, action->label, newlabel); + + return (error); +} - MAC_PERFORM(exc_action_label_update, p, action->label); - proc_rele(p); - return 0; +int +mac_exc_inherit_action_label(struct exception_action *parent, + struct exception_action *child) { + return mac_exc_update_action_label(child, parent->label); } -void mac_exc_action_label_reset(struct exception_action *action) { - struct label *old_label = action->label; - mac_exc_action_label_init(action); - mac_exc_action_label_free(old_label); +int mac_exc_update_task_crash_label(struct task *task, struct label *label) { + int error; + + assert(task != kernel_task); + + struct label *crash_label = get_task_crash_label(task); + + MAC_CHECK(exc_action_label_update, NULL, crash_label, label); + + return (error); } -void mac_exc_action_label_task_update(struct task *task, struct proc *proc) { - if (get_task_crash_label(task) != NULL) { - MAC_MACH_UNEXPECTED("task already has a crash_label attached to it"); - return; - } +// Process label creation, may sleep. - struct label *label = mac_exc_action_label_alloc(); - MAC_PERFORM(exc_action_label_update, proc, label); - set_task_crash_label(task, label); +struct label * +mac_exc_create_label_for_proc(struct proc *proc) +{ + struct label *label = mac_exc_create_label(); + MAC_PERFORM(exc_action_label_populate, label, proc); + return label; } -void mac_exc_action_label_task_destroy(struct task *task) { - mac_exc_action_label_free(get_task_crash_label(task)); - set_task_crash_label(task, NULL); +struct label * +mac_exc_create_label_for_current_proc(void) +{ + return mac_exc_create_label_for_proc(current_proc()); } +// Exception handler policy checking, may sleep. + int mac_exc_action_check_exception_send(struct task *victim_task, struct exception_action *action) { @@ -235,22 +259,21 @@ mac_exc_action_check_exception_send(struct task *victim_task, struct exception_a if (p != NULL) { // Create a label from the still existing bsd process... - label = bsd_label = mac_exc_action_label_alloc(); - MAC_PERFORM(exc_action_label_update, p, bsd_label); + label = bsd_label = mac_exc_create_label_for_proc(p); } else { // ... otherwise use the crash label on the task. label = get_task_crash_label(victim_task); } if (label == NULL) { - MAC_MACH_UNEXPECTED("mac_exc_action_check_exception_send: no exc_action label for proc %p", p); + MAC_MACH_UNEXPECTED("mac_exc_action_check_exception_send: no exc_action label for process"); return EPERM; } MAC_CHECK(exc_action_check_exception_send, label, action, action->label); if (bsd_label != NULL) { - mac_exc_action_label_free(bsd_label); + mac_exc_free_label(bsd_label); } return (error); diff --git a/security/mac_mach_internal.h b/security/mac_mach_internal.h index 406216029..4849bfabd 100644 --- a/security/mac_mach_internal.h +++ b/security/mac_mach_internal.h @@ -61,7 +61,10 @@ /* mac_do_machexc() flags */ #define MAC_DOEXCF_TRACED 0x01 /* Only do mach exeception if being ptrace()'ed */ +struct exception_action; struct uthread; +struct task; + int mac_do_machexc(int64_t code, int64_t subcode, uint32_t flags __unused); int mac_schedule_userret(void); @@ -84,16 +87,21 @@ void act_set_astmacf(struct thread *); void mac_thread_userret(struct thread *); /* exception actions */ -void mac_exc_action_label_init(struct exception_action *action); -void mac_exc_action_label_inherit(struct exception_action *parent, struct exception_action *child); -void mac_exc_action_label_destroy(struct exception_action *action); -int mac_exc_action_label_update(struct task *task, struct exception_action *action); -void mac_exc_action_label_reset(struct exception_action *action); +struct label *mac_exc_create_label(void); +void mac_exc_free_label(struct label *label); + +void mac_exc_associate_action_label(struct exception_action *action, struct label *label); +void mac_exc_free_action_label(struct exception_action *action); -void mac_exc_action_label_task_update(struct task *task, struct proc *proc); -void mac_exc_action_label_task_destroy(struct task *task); +int mac_exc_update_action_label(struct exception_action *action, struct label *newlabel); +int mac_exc_inherit_action_label(struct exception_action *parent, struct exception_action *child); +int mac_exc_update_task_crash_label(struct task *task, struct label *newlabel); int mac_exc_action_check_exception_send(struct task *victim_task, struct exception_action *action); + +struct label *mac_exc_create_label_for_proc(struct proc *proc); +struct label *mac_exc_create_label_for_current_proc(void); + #endif /* MAC */ #endif /* !_SECURITY_MAC_MACH_INTERNAL_H_ */ diff --git a/security/mac_policy.h b/security/mac_policy.h index dad8d9017..eedfeab13 100644 --- a/security/mac_policy.h +++ b/security/mac_policy.h @@ -698,7 +698,13 @@ typedef void mpo_devfs_label_update_t( @param exclabel Policy label for exception action Determine whether the the exception message caused by the victim - process can be sent to the exception action. + process can be sent to the exception action. The policy may compare + credentials in the crashlabel, which are derived from the process at + the time the exception occurs, with the credentials in the exclabel, + which was set at the time the exception port was set, to determine + its decision. Note that any process from which the policy derived + any credentials may not exist anymore at the time of this policy + operation. Sleeping is permitted. @return Return 0 if the message can be sent, otherwise an appropriate value for errno should be returned. @@ -709,7 +715,7 @@ typedef int mpo_exc_action_check_exception_send_t( struct label *exclabel ); /** - @brief Create an exception action label + @brief Associate an exception action label @param action Exception action to label @param exclabel Policy label to be filled in for exception action @@ -719,49 +725,60 @@ typedef void mpo_exc_action_label_associate_t( struct exception_action *action, struct label *exclabel ); -/** - @brief Copy an exception action label - @param src Source exception action label - @param dest Destination exception action label - - Copy the label information from src to dest. - Exception actions are often inherited, e.g. from parent to child. - In that case, the labels are copied instead of created fresh. -*/ -typedef void mpo_exc_action_label_copy_t( - struct label *src, - struct label *dest -); /** @brief Destroy exception action label @param label The label to be destroyed - Destroy the label on an exception action. In this entry point, a - policy module should free any internal storage associated with - label so that it may be destroyed. + Destroy the label on an exception action. Since the object is going + out of scope, policy modules should free any internal storage + associated with the label so that it may be destroyed. Sleeping is + permitted. */ typedef void mpo_exc_action_label_destroy_t( struct label *label ); +/** + @brief Populate an exception action label with process credentials + @param label The label to be populated + @param proc Process to derive credentials from + + Populate a label with credentials derived from a process. At + exception delivery time, the policy should compare credentials of the + process that set an exception ports with the credentials of the + process or corpse that experienced the exception. Note that the + process that set the port may not exist at that time anymore, so + labels should carry copies of live credentials if necessary. +*/ +typedef void mpo_exc_action_label_populate_t( + struct label *label, + struct proc *proc +); /** @brief Initialize exception action label @param label New label to initialize - Initialize a label for an exception action. + Initialize a label for an exception action. Usually performs + policy specific allocations. Sleeping is permitted. */ typedef int mpo_exc_action_label_init_t( struct label *label ); /** @brief Update the label on an exception action - @param p Process to update the label from - @param exclabel Policy label to be updated for exception action + @param action Exception action that the label belongs to (may be + NULL if none) + @param label Policy label to update + @param newlabel New label for update - Update the credentials of an exception action with the given task. + Update the credentials of an exception action from the given + label. The policy should copy over any credentials (process and + otherwise) from the new label into the label to update. Must not + sleep, must be quick and can be called with locks held. */ -typedef void mpo_exc_action_label_update_t( - struct proc *p, - struct label *exclabel +typedef int mpo_exc_action_label_update_t( + struct exception_action *action, + struct label *label, + struct label *newlabel ); /** @brief Access control for changing the offset of a file descriptor @@ -2859,19 +2876,6 @@ typedef int mpo_proc_check_ledger_t( struct proc *target, int op ); -/** - @brief Access control check for escaping default CPU usage monitor parameters. - @param cred Subject credential - - Determine if a credential has permission to program CPU usage monitor parameters - that are less restrictive than the global system-wide defaults. - - @return Return 0 if access is granted, otherwise an appropriate value for - errno should be returned. -*/ -typedef int mpo_proc_check_cpumon_t( - kauth_cred_t cred -); /** @brief Access control check for retrieving process information. @param cred Subject credential @@ -3089,6 +3093,19 @@ typedef int mpo_proc_check_wait_t( kauth_cred_t cred, struct proc *proc ); +/** + @brief Inform MAC policies that a process has exited. + @param proc Object process + + Called after all of the process's threads have terminated and + it has been removed from the process list. KPI that identifies + the process by pid will fail to find the process; KPI that + identifies the process by the object process pointer functions + normally. proc_exiting() returns true for the object process. +*/ +typedef void mpo_proc_notify_exit_t( + struct proc *proc +); /** @brief Destroy process label @param label The label to be destroyed @@ -3115,6 +3132,50 @@ typedef void mpo_proc_label_destroy_t( typedef void mpo_proc_label_init_t( struct label *label ); +/** + @brief Access control check for skywalk flow connect + @param cred Subject credential + @param flow Flow object + @param addr Remote address for flow to send data to + @param type Flow type (e.g. SOCK_STREAM or SOCK_DGRAM) + @param protocol Network protocol (e.g. IPPROTO_TCP) + + Determine whether the subject identified by the credential can + create a flow for sending data to the remote host specified by + addr. + + @return Return 0 if access if granted, otherwise an appropriate + value for errno should be returned. +*/ +typedef int mpo_skywalk_flow_check_connect_t( + kauth_cred_t cred, + void *flow, + const struct sockaddr *addr, + int type, + int protocol +); +/** + @brief Access control check for skywalk flow listen + @param cred Subject credential + @param flow Flow object + @param addr Local address for flow to listen on + @param type Flow type (e.g. SOCK_STREAM or SOCK_DGRAM) + @param protocol Network protocol (e.g. IPPROTO_TCP) + + Determine whether the subject identified by the credential can + create a flow for receiving data on the local address specified + by addr. + + @return Return 0 if access if granted, otherwise an appropriate + value for errno should be returned. +*/ +typedef int mpo_skywalk_flow_check_listen_t( + kauth_cred_t cred, + void *flow, + const struct sockaddr *addr, + int type, + int protocol +); /** @brief Access control check for socket accept @param cred Subject credential @@ -3253,6 +3314,30 @@ typedef int mpo_socket_check_deliver_t( struct mbuf *m, struct label *m_label ); +/** + @brief Access control check for socket ioctl. + @param cred Subject credential + @param so Object socket + @param cmd The ioctl command; see ioctl(2) + @param socklabel Policy label for socket + + Determine whether the subject identified by the credential can perform + the ioctl operation indicated by cmd on the given socket. + + @warning Since ioctl data is opaque from the standpoint of the MAC + framework, and since ioctls can affect many aspects of system + operation, policies must exercise extreme care when implementing + access control checks. + + @return Return 0 if access is granted, otherwise an appropriate value for + errno should be returned. +*/ +typedef int mpo_socket_check_ioctl_t( + kauth_cred_t cred, + socket_t so, + unsigned int cmd, + struct label *socklabel +); /** @brief Access control check for socket kqfilter @param cred Subject credential @@ -4862,6 +4947,32 @@ typedef int mpo_vnode_check_listextattr_t( struct vnode *vp, struct label *vlabel ); +/** + @brief Access control check for lookup + @param cred Subject credential + @param dvp Directory vnode + @param dlabel Policy label for dvp + @param path Path being looked up + @param pathlen Length of path in bytes + + Determine whether the subject identified by the credential can perform + a lookup of the passed path relative to the passed directory vnode. + + @return Return 0 if access is granted, otherwise an appropriate value for + errno should be returned. Suggested failure: EACCES for label mismatch or + EPERM for lack of privilege. + + @note The path may contain untrusted input. If approved, lookup proceeds + on the path; if a component is found to be a symlink then this hook is + called again with the updated path. +*/ +typedef int mpo_vnode_check_lookup_preflight_t( + kauth_cred_t cred, + struct vnode *dvp, + struct label *dlabel, + const char *path, + size_t pathlen +); /** @brief Access control check for lookup @param cred Subject credential @@ -5280,6 +5391,7 @@ typedef int mpo_vnode_check_setutimes_t( @param label label associated with the vnode @param cs_blob the code signature to check @param cs_flags update code signing flags if needed + @param signer_type output parameter for the code signature's signer type @param flags operational flag to mpo_vnode_check_signature @param fatal_failure_desc description of fatal failure @param fatal_failure_desc_len failure description len, failure is fatal if non-0 @@ -5292,6 +5404,7 @@ typedef int mpo_vnode_check_signature_t( struct label *label, struct cs_blob *cs_blob, unsigned int *cs_flags, + unsigned int *signer_type, int flags, char **fatal_failure_desc, size_t *fatal_failure_desc_len ); @@ -6188,7 +6301,7 @@ typedef void mpo_reserved_hook_t(void); * Please note that this should be kept in sync with the check assumptions * policy in bsd/kern/policy_check.c (policy_ops struct). */ -#define MAC_POLICY_OPS_VERSION 47 /* inc when new reserved slots are taken */ +#define MAC_POLICY_OPS_VERSION 52 /* inc when new reserved slots are taken */ struct mac_policy_ops { mpo_audit_check_postselect_t *mpo_audit_check_postselect; mpo_audit_check_preselect_t *mpo_audit_check_preselect; @@ -6335,7 +6448,7 @@ struct mac_policy_ops { mpo_proc_check_set_host_exception_port_t *mpo_proc_check_set_host_exception_port; mpo_exc_action_check_exception_send_t *mpo_exc_action_check_exception_send; mpo_exc_action_label_associate_t *mpo_exc_action_label_associate; - mpo_exc_action_label_copy_t *mpo_exc_action_label_copy; + mpo_exc_action_label_populate_t *mpo_exc_action_label_populate; mpo_exc_action_label_destroy_t *mpo_exc_action_label_destroy; mpo_exc_action_label_init_t *mpo_exc_action_label_init; mpo_exc_action_label_update_t *mpo_exc_action_label_update; @@ -6344,8 +6457,8 @@ struct mac_policy_ops { mpo_reserved_hook_t *mpo_reserved2; mpo_reserved_hook_t *mpo_reserved3; mpo_reserved_hook_t *mpo_reserved4; - mpo_reserved_hook_t *mpo_reserved5; - mpo_reserved_hook_t *mpo_reserved6; + mpo_skywalk_flow_check_connect_t *mpo_skywalk_flow_check_connect; + mpo_skywalk_flow_check_listen_t *mpo_skywalk_flow_check_listen; mpo_posixsem_check_create_t *mpo_posixsem_check_create; mpo_posixsem_check_open_t *mpo_posixsem_check_open; @@ -6423,7 +6536,7 @@ struct mac_policy_ops { mpo_system_check_settime_t *mpo_system_check_settime; mpo_system_check_swapoff_t *mpo_system_check_swapoff; mpo_system_check_swapon_t *mpo_system_check_swapon; - mpo_reserved_hook_t *mpo_reserved7; + mpo_socket_check_ioctl_t *mpo_socket_check_ioctl; mpo_sysvmsg_label_associate_t *mpo_sysvmsg_label_associate; mpo_sysvmsg_label_destroy_t *mpo_sysvmsg_label_destroy; @@ -6456,7 +6569,7 @@ struct mac_policy_ops { mpo_sysvshm_label_init_t *mpo_sysvshm_label_init; mpo_sysvshm_label_recycle_t *mpo_sysvshm_label_recycle; - mpo_reserved_hook_t *mpo_reserved8; + mpo_proc_notify_exit_t *mpo_proc_notify_exit; mpo_mount_check_snapshot_revert_t *mpo_mount_check_snapshot_revert; mpo_vnode_check_getattr_t *mpo_vnode_check_getattr; mpo_mount_check_snapshot_create_t *mpo_mount_check_snapshot_create; @@ -6553,7 +6666,7 @@ struct mac_policy_ops { mpo_system_check_kas_info_t *mpo_system_check_kas_info; - mpo_proc_check_cpumon_t *mpo_proc_check_cpumon; + mpo_vnode_check_lookup_preflight_t *mpo_vnode_check_lookup_preflight; mpo_vnode_notify_open_t *mpo_vnode_notify_open; @@ -6610,7 +6723,7 @@ struct mac_policy_conf { const char *mpc_fullname; /** full name */ char const * const *mpc_labelnames; /** managed label namespaces */ unsigned int mpc_labelname_count; /** number of managed label namespaces */ - struct mac_policy_ops *mpc_ops; /** operation vector */ + const struct mac_policy_ops *mpc_ops; /** operation vector */ int mpc_loadtime_flags; /** load time flags */ int *mpc_field_off; /** label slot */ int mpc_runtime_flags; /** run time flags */ diff --git a/security/mac_priv.c b/security/mac_priv.c index 7d72ce88d..59f14de12 100644 --- a/security/mac_priv.c +++ b/security/mac_priv.c @@ -86,6 +86,9 @@ mac_priv_check(kauth_cred_t cred, int priv) { int error; + if (!mac_cred_check_enforce(cred)) + return 0; + MAC_CHECK(priv_check, cred, priv); return (error); @@ -100,6 +103,9 @@ mac_priv_grant(kauth_cred_t cred, int priv) { int error; + if (!mac_cred_check_enforce(cred)) + return 0; + MAC_GRANT(priv_grant, cred, priv); return (error); diff --git a/security/mac_process.c b/security/mac_process.c index 193507f5d..f3ea32890 100644 --- a/security/mac_process.c +++ b/security/mac_process.c @@ -301,14 +301,6 @@ mac_cred_check_visible(kauth_cred_t u1, kauth_cred_t u2) return (error); } -/* - * called with process locked. - */ -void mac_proc_set_enforce(proc_t p, int enforce_flags) -{ - p->p_mac_enforce |= enforce_flags; -} - int mac_proc_check_debug(proc_t curp, struct proc *proc) { @@ -320,7 +312,7 @@ mac_proc_check_debug(proc_t curp, struct proc *proc) if (!mac_proc_enforce) return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); @@ -341,7 +333,7 @@ mac_proc_check_fork(proc_t curp) if (!mac_proc_enforce) return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); @@ -408,7 +400,7 @@ mac_proc_check_map_anon(proc_t proc, user_addr_t u_addr, if (!mac_vm_enforce) return 0; #endif - if (!mac_proc_check_enforce(proc, MAC_VM_ENFORCE)) + if (!mac_proc_check_enforce(proc)) return (0); cred = kauth_cred_proc_ref(proc); @@ -430,7 +422,7 @@ mac_proc_check_mprotect(proc_t proc, if (!mac_vm_enforce) return 0; #endif - if (!mac_proc_check_enforce(proc, MAC_VM_ENFORCE)) + if (!mac_proc_check_enforce(proc)) return (0); cred = kauth_cred_proc_ref(proc); @@ -467,7 +459,7 @@ mac_proc_check_sched(proc_t curp, struct proc *proc) if (!mac_proc_enforce) return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); @@ -488,7 +480,7 @@ mac_proc_check_signal(proc_t curp, struct proc *proc, int signum) if (!mac_proc_enforce) return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); @@ -509,7 +501,7 @@ mac_proc_check_wait(proc_t curp, struct proc *proc) if (!mac_proc_enforce) return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); @@ -519,6 +511,12 @@ mac_proc_check_wait(proc_t curp, struct proc *proc) return (error); } +void +mac_proc_notify_exit(struct proc *proc) +{ + MAC_PERFORM(proc_notify_exit, proc); +} + int mac_proc_check_suspend_resume(proc_t curp, int sr) { @@ -530,7 +528,7 @@ mac_proc_check_suspend_resume(proc_t curp, int sr) if (!mac_proc_enforce) return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); @@ -551,7 +549,7 @@ mac_proc_check_ledger(proc_t curp, proc_t proc, int ledger_op) if (!mac_proc_enforce) return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); @@ -561,27 +559,6 @@ mac_proc_check_ledger(proc_t curp, proc_t proc, int ledger_op) return (error); } -int -mac_proc_check_cpumon(proc_t curp) -{ - kauth_cred_t cred; - int error = 0; - -#if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_proc_enforce) - return 0; -#endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) - return 0; - - cred = kauth_cred_proc_ref(curp); - MAC_CHECK(proc_check_cpumon, cred); - kauth_cred_unref(&cred); - - return (error); -} - int mac_proc_check_proc_info(proc_t curp, proc_t target, int callnum, int flavor) { @@ -593,7 +570,7 @@ mac_proc_check_proc_info(proc_t curp, proc_t target, int callnum, int flavor) if (!mac_proc_enforce) return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); @@ -614,7 +591,7 @@ mac_proc_check_get_cs_info(proc_t curp, proc_t target, unsigned int op) if (!mac_proc_enforce) return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); @@ -635,7 +612,7 @@ mac_proc_check_set_cs_info(proc_t curp, proc_t target, unsigned int op) if (!mac_proc_enforce) return 0; #endif - if (!mac_proc_check_enforce(curp, MAC_PROC_ENFORCE)) + if (!mac_proc_check_enforce(curp)) return 0; cred = kauth_cred_proc_ref(curp); diff --git a/security/mac_pty.c b/security/mac_pty.c index af8a8732b..bbd535272 100644 --- a/security/mac_pty.c +++ b/security/mac_pty.c @@ -1,3 +1,31 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + #include #include #include diff --git a/security/mac_skywalk.c b/security/mac_skywalk.c new file mode 100644 index 000000000..ba53dfe93 --- /dev/null +++ b/security/mac_skywalk.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include +#include +#include +#include +#include + +int +mac_skywalk_flow_check_connect(proc_t proc, void *flow, const struct sockaddr *addr, int type, int protocol) +{ + int error; + + MAC_CHECK(skywalk_flow_check_connect, proc_ucred(proc), flow, addr, type, protocol); + return (error); +} + +int +mac_skywalk_flow_check_listen(proc_t proc, void *flow, const struct sockaddr *addr, int type, int protocol) +{ + int error; + + MAC_CHECK(skywalk_flow_check_listen, proc_ucred(proc), flow, addr, type, protocol); + return (error); +} + diff --git a/security/mac_socket.c b/security/mac_socket.c index 2151c0915..e935f6f78 100644 --- a/security/mac_socket.c +++ b/security/mac_socket.c @@ -298,23 +298,6 @@ mac_socket_check_kqfilter(kauth_cred_t cred, struct knote *kn, } static int -mac_socket_check_label_update(kauth_cred_t cred, struct socket *so, - struct label *newlabel) -{ - int error; - -#if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_socket_enforce) - return 0; -#endif - - MAC_CHECK(socket_check_label_update, cred, - (socket_t)so, so->so_label, - newlabel); - return (error); -} - int mac_socket_check_select(kauth_cred_t cred, struct socket *so, int which) { @@ -331,8 +314,8 @@ mac_socket_check_select(kauth_cred_t cred, struct socket *so, int which) return (error); } -int -mac_socket_check_stat(kauth_cred_t cred, struct socket *so) +mac_socket_check_label_update(kauth_cred_t cred, struct socket *so, + struct label *newlabel) { int error; @@ -342,12 +325,12 @@ mac_socket_check_stat(kauth_cred_t cred, struct socket *so) return 0; #endif - MAC_CHECK(socket_check_stat, cred, - (socket_t)so, so->so_label); + MAC_CHECK(socket_check_label_update, cred, + (socket_t)so, so->so_label, + newlabel); return (error); } - int mac_socket_label_update(kauth_cred_t cred, struct socket *so, struct label *label) { @@ -484,6 +467,7 @@ mac_socketpeer_label_get(__unused kauth_cred_t cred, struct socket *so, return (error); } + #endif /* MAC_SOCKET */ int @@ -604,6 +588,39 @@ mac_socket_check_deliver(__unused struct socket *so, __unused struct mbuf *mbuf) } #endif +int +mac_socket_check_ioctl(kauth_cred_t cred, struct socket *so, + unsigned int cmd) +{ + int error; + +#if SECURITY_MAC_CHECK_ENFORCE + /* 21167099 - only check if we allow write */ + if (!mac_socket_enforce) + return 0; +#endif + + MAC_CHECK(socket_check_ioctl, cred, + (socket_t)so, cmd, so->so_label); + return (error); +} + +int +mac_socket_check_stat(kauth_cred_t cred, struct socket *so) +{ + int error; + +#if SECURITY_MAC_CHECK_ENFORCE + /* 21167099 - only check if we allow write */ + if (!mac_socket_enforce) + return 0; +#endif + + MAC_CHECK(socket_check_stat, cred, + (socket_t)so, so->so_label); + return (error); +} + int mac_socket_check_listen(kauth_cred_t cred, struct socket *so) { diff --git a/security/mac_vfs.c b/security/mac_vfs.c index 8a57039ce..0d55a1c27 100644 --- a/security/mac_vfs.c +++ b/security/mac_vfs.c @@ -389,10 +389,9 @@ mac_vnode_notify_create(vfs_context_t ctx, struct mount *mp, if (!mac_vnode_enforce) return (0); #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return (0); - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_notify_create, cred, mp, mp->mnt_mntlabel, dvp, dvp->v_label, vp, vp->v_label, cnp); @@ -410,10 +409,9 @@ mac_vnode_notify_rename(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_rename, cred, vp, vp->v_label, dvp, dvp->v_label, cnp); } @@ -428,10 +426,9 @@ mac_vnode_notify_open(vfs_context_t ctx, struct vnode *vp, int acc_flags) if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_open, cred, vp, vp->v_label, acc_flags); } @@ -446,10 +443,9 @@ mac_vnode_notify_link(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_link, cred, dvp, dvp->v_label, vp, vp->v_label, cnp); } @@ -463,10 +459,9 @@ mac_vnode_notify_deleteextattr(vfs_context_t ctx, struct vnode *vp, const char * if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_deleteextattr, cred, vp, vp->v_label, name); } @@ -480,10 +475,9 @@ mac_vnode_notify_setacl(vfs_context_t ctx, struct vnode *vp, struct kauth_acl *a if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_setacl, cred, vp, vp->v_label, acl); } @@ -497,10 +491,9 @@ mac_vnode_notify_setattrlist(vfs_context_t ctx, struct vnode *vp, struct attrlis if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_setattrlist, cred, vp, vp->v_label, alist); } @@ -514,10 +507,9 @@ mac_vnode_notify_setextattr(vfs_context_t ctx, struct vnode *vp, const char *nam if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_setextattr, cred, vp, vp->v_label, name, uio); } @@ -531,10 +523,9 @@ mac_vnode_notify_setflags(vfs_context_t ctx, struct vnode *vp, u_long flags) if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_setflags, cred, vp, vp->v_label, flags); } @@ -548,10 +539,9 @@ mac_vnode_notify_setmode(vfs_context_t ctx, struct vnode *vp, mode_t mode) if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_setmode, cred, vp, vp->v_label, mode); } @@ -565,10 +555,9 @@ mac_vnode_notify_setowner(vfs_context_t ctx, struct vnode *vp, uid_t uid, gid_t if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_setowner, cred, vp, vp->v_label, uid, gid); } @@ -582,10 +571,9 @@ mac_vnode_notify_setutimes(vfs_context_t ctx, struct vnode *vp, struct timespec if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_setutimes, cred, vp, vp->v_label, atime, mtime); } @@ -599,10 +587,9 @@ mac_vnode_notify_truncate(vfs_context_t ctx, kauth_cred_t file_cred, struct vnod if (!mac_vnode_enforce) return; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return; MAC_PERFORM(vnode_notify_truncate, cred, file_cred, vp, vp->v_label); } @@ -648,11 +635,12 @@ mac_vnode_label_store(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_label_vnodes || - !mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) + if (!mac_label_vnodes) return 0; cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_label_store, cred, vp, vp->v_label, intlabel); return (error); @@ -802,10 +790,9 @@ mac_vnode_check_access(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); /* Convert {R,W,X}_OK values to V{READ,WRITE,EXEC} for entry points */ mask = ACCESS_MODE_TO_VNODE_MASK(acc_mode); MAC_CHECK(vnode_check_access, cred, vp, vp->v_label, mask); @@ -823,10 +810,9 @@ mac_vnode_check_chdir(vfs_context_t ctx, struct vnode *dvp) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_chdir, cred, dvp, dvp->v_label); return (error); } @@ -843,10 +829,9 @@ mac_vnode_check_chroot(vfs_context_t ctx, struct vnode *dvp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_chroot, cred, dvp, dvp->v_label, cnp); return (error); } @@ -863,10 +848,9 @@ mac_vnode_check_clone(vfs_context_t ctx, struct vnode *dvp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_clone, cred, dvp, dvp->v_label, vp, vp->v_label, cnp); return (error); @@ -883,10 +867,9 @@ mac_vnode_check_create(vfs_context_t ctx, struct vnode *dvp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_create, cred, dvp, dvp->v_label, cnp, vap); return (error); } @@ -903,10 +886,9 @@ mac_vnode_check_unlink(vfs_context_t ctx, struct vnode *dvp, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_unlink, cred, dvp, dvp->v_label, vp, vp->v_label, cnp); return (error); @@ -924,10 +906,9 @@ mac_vnode_check_deleteacl(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_deleteacl, cred, vp, vp->v_label, type); return (error); } @@ -945,10 +926,9 @@ mac_vnode_check_deleteextattr(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_deleteextattr, cred, vp, vp->v_label, name); return (error); } @@ -964,10 +944,9 @@ mac_vnode_check_exchangedata(vfs_context_t ctx, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_exchangedata, cred, v1, v1->v_label, v2, v2->v_label); @@ -986,10 +965,9 @@ mac_vnode_check_getacl(vfs_context_t ctx, struct vnode *vp, acl_type_t type) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_getacl, cred, vp, vp->v_label, type); return (error); } @@ -1007,10 +985,9 @@ mac_vnode_check_getattr(vfs_context_t ctx, struct ucred *file_cred, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_getattr, cred, file_cred, vp, vp->v_label, va); return (error); } @@ -1027,10 +1004,9 @@ mac_vnode_check_getattrlist(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_getattrlist, cred, vp, vp->v_label, alist); /* Falsify results instead of returning error? */ @@ -1116,10 +1092,9 @@ mac_vnode_check_fsgetpath(vfs_context_t ctx, struct vnode *vp) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_fsgetpath, cred, vp, vp->v_label); return (error); } @@ -1127,7 +1102,8 @@ mac_vnode_check_fsgetpath(vfs_context_t ctx, struct vnode *vp) int mac_vnode_check_signature(struct vnode *vp, struct cs_blob *cs_blob, struct image_params *imgp, - unsigned int *cs_flags, int flags) + unsigned int *cs_flags, unsigned int *signer_type, + int flags) { int error; char *fatal_failure_desc = NULL; @@ -1144,7 +1120,7 @@ mac_vnode_check_signature(struct vnode *vp, struct cs_blob *cs_blob, #endif MAC_CHECK(vnode_check_signature, vp, vp->v_label, cs_blob, - cs_flags, flags, &fatal_failure_desc, &fatal_failure_desc_len); + cs_flags, signer_type, flags, &fatal_failure_desc, &fatal_failure_desc_len); if (fatal_failure_desc_len) { // A fatal code signature validation failure occured, formulate a crash @@ -1245,10 +1221,9 @@ mac_vnode_check_getacl(vfs_context_t ctx, struct vnode *vp, acl_type_t type) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_getacl, cred, vp, vp->v_label, type); return (error); } @@ -1266,10 +1241,9 @@ mac_vnode_check_getextattr(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_getextattr, cred, vp, vp->v_label, name, uio); return (error); @@ -1286,10 +1260,9 @@ mac_vnode_check_ioctl(vfs_context_t ctx, struct vnode *vp, u_int cmd) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_ioctl, cred, vp, vp->v_label, cmd); return (error); } @@ -1306,10 +1279,9 @@ mac_vnode_check_kqfilter(vfs_context_t ctx, kauth_cred_t file_cred, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_kqfilter, cred, file_cred, kn, vp, vp->v_label); @@ -1328,10 +1300,9 @@ mac_vnode_check_link(vfs_context_t ctx, struct vnode *dvp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_link, cred, dvp, dvp->v_label, vp, vp->v_label, cnp); return (error); @@ -1348,14 +1319,32 @@ mac_vnode_check_listextattr(vfs_context_t ctx, struct vnode *vp) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_listextattr, cred, vp, vp->v_label); return (error); } +int +mac_vnode_check_lookup_preflight(vfs_context_t ctx, struct vnode *dvp, + const char *path, size_t pathlen) +{ + kauth_cred_t cred; + int error; + +#if SECURITY_MAC_CHECK_ENFORCE + /* 21167099 - only check if we allow write */ + if (!mac_vnode_enforce) + return 0; +#endif + cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); + MAC_CHECK(vnode_check_lookup_preflight, cred, dvp, dvp->v_label, path, pathlen); + return (error); +} + int mac_vnode_check_lookup(vfs_context_t ctx, struct vnode *dvp, struct componentname *cnp) @@ -1368,10 +1357,9 @@ mac_vnode_check_lookup(vfs_context_t ctx, struct vnode *dvp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_lookup, cred, dvp, dvp->v_label, cnp); return (error); } @@ -1387,10 +1375,9 @@ mac_vnode_check_open(vfs_context_t ctx, struct vnode *vp, int acc_mode) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_open, cred, vp, vp->v_label, acc_mode); return (error); } @@ -1407,10 +1394,9 @@ mac_vnode_check_read(vfs_context_t ctx, struct ucred *file_cred, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_read, cred, file_cred, vp, vp->v_label); @@ -1428,10 +1414,9 @@ mac_vnode_check_readdir(vfs_context_t ctx, struct vnode *dvp) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_readdir, cred, dvp, dvp->v_label); return (error); } @@ -1447,10 +1432,9 @@ mac_vnode_check_readlink(vfs_context_t ctx, struct vnode *vp) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_readlink, cred, vp, vp->v_label); return (error); } @@ -1467,10 +1451,9 @@ mac_vnode_check_label_update(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_label_update, cred, vp, vp->v_label, newlabel); return (error); @@ -1489,10 +1472,9 @@ mac_vnode_check_rename(vfs_context_t ctx, struct vnode *dvp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_rename_from, cred, dvp, dvp->v_label, vp, vp->v_label, cnp); @@ -1521,10 +1503,9 @@ mac_vnode_check_revoke(vfs_context_t ctx, struct vnode *vp) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_revoke, cred, vp, vp->v_label); return (error); } @@ -1540,10 +1521,9 @@ mac_vnode_check_searchfs(vfs_context_t ctx, struct vnode *vp, struct attrlist *a if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_searchfs, cred, vp, vp->v_label, alist); return (error); } @@ -1559,10 +1539,9 @@ mac_vnode_check_select(vfs_context_t ctx, struct vnode *vp, int which) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_select, cred, vp, vp->v_label, which); return (error); } @@ -1579,10 +1558,9 @@ mac_vnode_check_setacl(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_setacl, cred, vp, vp->v_label, acl); return (error); } @@ -1599,10 +1577,9 @@ mac_vnode_check_setattrlist(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_setattrlist, cred, vp, vp->v_label, alist); return (error); } @@ -1619,10 +1596,9 @@ mac_vnode_check_setextattr(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_setextattr, cred, vp, vp->v_label, name, uio); return (error); @@ -1639,10 +1615,9 @@ mac_vnode_check_setflags(vfs_context_t ctx, struct vnode *vp, u_long flags) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_setflags, cred, vp, vp->v_label, flags); return (error); } @@ -1658,10 +1633,9 @@ mac_vnode_check_setmode(vfs_context_t ctx, struct vnode *vp, mode_t mode) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_setmode, cred, vp, vp->v_label, mode); return (error); } @@ -1678,10 +1652,9 @@ mac_vnode_check_setowner(vfs_context_t ctx, struct vnode *vp, uid_t uid, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_setowner, cred, vp, vp->v_label, uid, gid); return (error); } @@ -1698,10 +1671,9 @@ mac_vnode_check_setutimes(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_setutimes, cred, vp, vp->v_label, atime, mtime); return (error); @@ -1719,10 +1691,9 @@ mac_vnode_check_stat(vfs_context_t ctx, struct ucred *file_cred, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_stat, cred, file_cred, vp, vp->v_label); return (error); @@ -1740,10 +1711,9 @@ mac_vnode_check_truncate(vfs_context_t ctx, struct ucred *file_cred, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_truncate, cred, file_cred, vp, vp->v_label); @@ -1762,10 +1732,9 @@ mac_vnode_check_write(vfs_context_t ctx, struct ucred *file_cred, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_write, cred, file_cred, vp, vp->v_label); return (error); @@ -1783,10 +1752,9 @@ mac_vnode_check_uipc_bind(vfs_context_t ctx, struct vnode *dvp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_uipc_bind, cred, dvp, dvp->v_label, cnp, vap); return (error); } @@ -1802,10 +1770,9 @@ mac_vnode_check_uipc_connect(vfs_context_t ctx, struct vnode *vp, struct socket if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(vnode_check_uipc_connect, cred, vp, vp->v_label, (socket_t) so); return (error); } @@ -1911,10 +1878,9 @@ mac_mount_check_mount(vfs_context_t ctx, struct vnode *vp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_mount, cred, vp, vp->v_label, cnp, vfc_name); return (error); @@ -1932,10 +1898,9 @@ mac_mount_check_snapshot_create(vfs_context_t ctx, struct mount *mp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_snapshot_create, cred, mp, name); return (error); } @@ -1952,10 +1917,9 @@ mac_mount_check_snapshot_delete(vfs_context_t ctx, struct mount *mp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_snapshot_delete, cred, mp, name); return (error); } @@ -1972,10 +1936,9 @@ mac_mount_check_snapshot_revert(vfs_context_t ctx, struct mount *mp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_snapshot_revert, cred, mp, name); return (error); } @@ -1991,10 +1954,9 @@ mac_mount_check_remount(vfs_context_t ctx, struct mount *mp) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_remount, cred, mp, mp->mnt_mntlabel); return (error); @@ -2011,10 +1973,9 @@ mac_mount_check_umount(vfs_context_t ctx, struct mount *mp) if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_umount, cred, mp, mp->mnt_mntlabel); return (error); @@ -2032,10 +1993,9 @@ mac_mount_check_getattr(vfs_context_t ctx, struct mount *mp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_getattr, cred, mp, mp->mnt_mntlabel, vfa); return (error); } @@ -2052,10 +2012,9 @@ mac_mount_check_setattr(vfs_context_t ctx, struct mount *mp, if (!mac_vnode_enforce) return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_setattr, cred, mp, mp->mnt_mntlabel, vfa); return (error); } @@ -2067,14 +2026,13 @@ mac_mount_check_stat(vfs_context_t ctx, struct mount *mount) int error; #if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_vnode_enforce) - return 0; + /* 21167099 - only check if we allow write */ + if (!mac_vnode_enforce) + return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_stat, cred, mount, mount->mnt_mntlabel); return (error); @@ -2087,14 +2045,13 @@ mac_mount_check_label_update(vfs_context_t ctx, struct mount *mount) int error; #if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_vnode_enforce) - return 0; + /* 21167099 - only check if we allow write */ + if (!mac_vnode_enforce) + return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_label_update, cred, mount, mount->mnt_mntlabel); return (error); @@ -2107,14 +2064,13 @@ mac_mount_check_fsctl(vfs_context_t ctx, struct mount *mp, u_int cmd) int error; #if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_vnode_enforce) - return 0; + /* 21167099 - only check if we allow write */ + if (!mac_vnode_enforce) + return 0; #endif - if (!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE)) - return 0; - cred = vfs_context_ucred(ctx); + if (!mac_cred_check_enforce(cred)) + return (0); MAC_CHECK(mount_check_fsctl, cred, mp, mp->mnt_mntlabel, cmd); return (error); @@ -2125,9 +2081,9 @@ mac_devfs_label_associate_device(dev_t dev, struct devnode *de, const char *fullpath) { #if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_device_enforce) - return; + /* 21167099 - only check if we allow write */ + if (!mac_device_enforce) + return; #endif MAC_PERFORM(devfs_label_associate_device, dev, de, de->dn_label, @@ -2139,9 +2095,9 @@ mac_devfs_label_associate_directory(const char *dirname, int dirnamelen, struct devnode *de, const char *fullpath) { #if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_device_enforce) - return; + /* 21167099 - only check if we allow write */ + if (!mac_device_enforce) + return; #endif MAC_PERFORM(devfs_label_associate_directory, dirname, dirnamelen, de, @@ -2154,9 +2110,9 @@ vn_setlabel(struct vnode *vp, struct label *intlabel, vfs_context_t context) int error; #if SECURITY_MAC_CHECK_ENFORCE - /* 21167099 - only check if we allow write */ - if (!mac_vnode_enforce) - return 0; + /* 21167099 - only check if we allow write */ + if (!mac_vnode_enforce) + return 0; #endif if (!mac_label_vnodes) return (0); diff --git a/tools/lldbmacros/Makefile b/tools/lldbmacros/Makefile index 8a074e2d4..afb3d6b88 100644 --- a/tools/lldbmacros/Makefile +++ b/tools/lldbmacros/Makefile @@ -37,12 +37,14 @@ LLDBMACROS_PYTHON_FILES = $(LLDBMACROS_USERDEBUG_FILES) \ plugins/zprint_perf_log.py \ atm.py \ bank.py \ + kevent.py \ xnu.py \ xnudefines.py \ ktrace.py \ macho.py \ mbufdefines.py \ netdefines.py \ + ntstat.py \ routedefines.py \ ipc.py \ ipcimportancedetail.py \ @@ -61,6 +63,7 @@ LLDBMACROS_PYTHON_FILES = $(LLDBMACROS_USERDEBUG_FILES) \ pci.py \ misc.py \ apic.py \ + kasan.py \ kauth.py \ usertaskgdbserver.py \ waitq.py \ @@ -73,6 +76,7 @@ ifneq ($(PLATFORM),MacOSX) plugins/iosspeedtracer.sh endif + INSTALL_LLDBMACROS_PYTHON_FILES=$(addprefix $(LLDBMACROS_DEST), $(LLDBMACROS_PYTHON_FILES)) $(INSTALL_LLDBMACROS_PYTHON_FILES): $(LLDBMACROS_DEST)% : $(LLDBMACROS_SOURCE)% diff --git a/tools/lldbmacros/README.md b/tools/lldbmacros/README.md index 3446cbd90..53a03fdf5 100644 --- a/tools/lldbmacros/README.md +++ b/tools/lldbmacros/README.md @@ -170,6 +170,8 @@ Following is a step by step guideline on how to add a new command ( e.g showtask 6. If your function finds issue with the passed argument then you can `raise ArgumentError('error_message')` to notify the user. The framework will automatically catch this and show appropriate help using the function doc string. + 7. Please use "##" for commenting your code. This is important because single "#" based strings may be mistakenly considered in `unifdef` program. + Time for some code example? Try reading the code for function ShowTaskVmeHelper in memory.py. SPECIAL Note: Very often you will find yourself making changes to a file for some command/summary and would like to test it out in lldb. diff --git a/tools/lldbmacros/apic.py b/tools/lldbmacros/apic.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/atm.py b/tools/lldbmacros/atm.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/bank.py b/tools/lldbmacros/bank.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/core/__init__.py b/tools/lldbmacros/core/__init__.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/core/caching.py b/tools/lldbmacros/core/caching.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/core/configuration.py b/tools/lldbmacros/core/configuration.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/core/cvalue.py b/tools/lldbmacros/core/cvalue.py old mode 100644 new mode 100755 index 8d645e687..0941f7530 --- a/tools/lldbmacros/core/cvalue.py +++ b/tools/lldbmacros/core/cvalue.py @@ -43,14 +43,14 @@ class value(object): return self._sbval19k84obscure747.__str__() def __cmp__(self, other): - if type(other) is int: + if type(other) is int or type(other) is long: me = int(self) if type(me) is long: other = long(other) return me.__cmp__(other) if type(other) is value: return int(self).__cmp__(int(other)) - raise TypeError("Cannot compare value with this type") + raise TypeError("Cannot compare value with type {}".format(type(other))) def __str__(self): global _cstring_rex @@ -433,6 +433,14 @@ def gettype(target_type): if target_type in _value_types_cache: return _value_types_cache[target_type] + target_type = target_type.strip() + + requested_type_is_struct = False + m = re.match(r'\s*struct\s*(.*)$', target_type) + if m: + requested_type_is_struct = True + target_type = m.group(1) + tmp_type = None requested_type_is_pointer = False if target_type.endswith('*') : @@ -442,6 +450,9 @@ def gettype(target_type): search_type = target_type.rstrip('*').strip() type_arr = [t for t in LazyTarget.GetTarget().FindTypes(search_type)] + if requested_type_is_struct: + type_arr = [t for t in type_arr if t.type == lldb.eTypeClassStruct] + # After the sort, the struct type with more fields will be at index [0]. # This hueristic helps selecting struct type with more fields compared to ones with "opaque" members type_arr.sort(reverse=True, key=lambda x: x.GetNumberOfFields()) @@ -462,6 +473,7 @@ def gettype(target_type): def getfieldoffset(struct_type, field_name): """ Returns the byte offset of a field inside a given struct + Understands anonymous unions and field names in sub-structs params: struct_type - str or lldb.SBType, ex. 'struct ipc_port *' or port.gettype() field_name - str, name of the field inside the struct ex. 'ip_messages' @@ -470,13 +482,23 @@ def getfieldoffset(struct_type, field_name): raises: TypeError - - In case the struct_type has no field with the name field_name """ + if type(struct_type) == str: struct_type = gettype(struct_type) + + if '.' in field_name : + # Handle recursive fields in sub-structs + components = field_name.split('.', 1) + for field in struct_type.get_fields_array(): + if str(field.GetName()) == components[0]: + return getfieldoffset(struct_type, components[0]) + getfieldoffset(field.GetType(), components[1]) + raise TypeError('Field name "%s" not found in type "%s"' % (components[0], str(struct_type))) + offset = 0 for field in struct_type.get_fields_array(): if str(field.GetName()) == field_name: return field.GetOffsetInBytes() - + # Hack for anonymous unions - the compiler does this, so cvalue should too if field.GetName() is None and field.GetType().GetTypeClass() == lldb.eTypeClassUnion : for union_field in field.GetType().get_fields_array(): diff --git a/tools/lldbmacros/core/kernelcore.py b/tools/lldbmacros/core/kernelcore.py old mode 100644 new mode 100755 index 3c6e5802b..da145a437 --- a/tools/lldbmacros/core/kernelcore.py +++ b/tools/lldbmacros/core/kernelcore.py @@ -46,7 +46,25 @@ def IterateLinkedList(element, field_name): elt = elt.__getattr__(field_name) #end of while loop -def IterateListEntry(element, element_type, field_name): +def IterateSListEntry(element, element_type, field_name, slist_prefix=''): + """ iterate over a list as defined with SLIST_HEAD in bsd/sys/queue.h + params: + element - value : Value object for slh_first + element_type - str : Type of the next element + field_name - str : Name of the field in next element's structure + returns: + A generator does not return. It is used for iterating + value : an object thats of type (element_type) head->sle_next. Always a pointer object + """ + elt = element.__getattr__(slist_prefix + 'slh_first') + if type(element_type) == str: + element_type = gettype(element_type) + while unsigned(elt) != 0: + yield elt + next_el = elt.__getattr__(field_name).__getattr__(slist_prefix + 'sle_next') + elt = cast(next_el, element_type) + +def IterateListEntry(element, element_type, field_name, list_prefix=''): """ iterate over a list as defined with LIST_HEAD in bsd/sys/queue.h params: element - value : Value object for lh_first @@ -60,12 +78,12 @@ def IterateListEntry(element, element_type, field_name): for pp in IterateListEntry(headp, 'struct proc *', 'p_sibling'): print GetProcInfo(pp) """ - elt = element.lh_first + elt = element.__getattr__(list_prefix + 'lh_first') if type(element_type) == str: element_type = gettype(element_type) while unsigned(elt) != 0: yield elt - next_el = elt.__getattr__(field_name).le_next + next_el = elt.__getattr__(field_name).__getattr__(list_prefix + 'le_next') elt = cast(next_el, element_type) def IterateLinkageChain(queue_head, element_type, field_name, field_ofst=0): @@ -173,6 +191,7 @@ class KernelTarget(object): self._threads_list = [] self._tasks_list = [] self._coalitions_list = [] + self._thread_groups = [] self._allproc = [] self._terminated_tasks_list = [] self._zones_list = [] @@ -339,10 +358,10 @@ class KernelTarget(object): def PhysToKernelVirt(self, addr): if self.arch == 'x86_64': return (addr + unsigned(self.GetGlobalVariable('physmap_base'))) - elif self.arch == 'arm' or self.arch == 'arm64': + elif self.arch.startswith('arm'): return (addr - unsigned(self.GetGlobalVariable("gPhysBase")) + unsigned(self.GetGlobalVariable("gVirtBase"))) else: - raise ValueError("PhysToVirt does not support {0}".format(arch)) + raise ValueError("PhysToVirt does not support {0}".format(self.arch)) def GetNanotimeFromAbstime(self, abstime): """ convert absolute time (which is in MATUs) to nano seconds. @@ -408,6 +427,17 @@ class KernelTarget(object): caching.SaveDynamicCacheData("kern._coalitions_list", self._coalitions_list) return self._coalitions_list + if name == 'thread_groups' : + self._thread_groups_list = caching.GetDynamicCacheData("kern._thread_groups_list", []) + if len(self._thread_groups_list) > 0 : return self._thread_groups_list + thread_groups_queue_head = self.GetGlobalVariable('tg_queue') + thread_group_type = LazyTarget.GetTarget().FindFirstType('thread_group') + thread_groups_ptr_type = thread_group_type.GetPointerType() + for coal in IterateLinkageChain(addressof(thread_groups_queue_head), thread_groups_ptr_type, 'tg_queue_chain'): + self._thread_groups_list.append(coal) + caching.SaveDynamicCacheData("kern._thread_groups_list", self._thread_groups_list) + return self._thread_groups_list + if name == 'terminated_tasks' : self._terminated_tasks_list = caching.GetDynamicCacheData("kern._terminated_tasks_list", []) if len(self._terminated_tasks_list) > 0 : return self._terminated_tasks_list @@ -481,4 +511,18 @@ class KernelTarget(object): caching.SaveStaticCacheData("kern.ptrsize", self._ptrsize) return self._ptrsize + if name == 'VM_MIN_KERNEL_ADDRESS': + if self.arch == 'x86_64': + return unsigned(0xFFFFFF8000000000) + elif self.arch == 'arm64': + return unsigned(0xffffffe000000000) + else: + return unsigned(0x80000000) + + if name == 'VM_MIN_KERNEL_AND_KEXT_ADDRESS': + if self.arch == 'x86_64': + return self.VM_MIN_KERNEL_ADDRESS - 0x80000000 + else: + return self.VM_MIN_KERNEL_ADDRESS + return object.__getattribute__(self, name) diff --git a/tools/lldbmacros/core/lazytarget.py b/tools/lldbmacros/core/lazytarget.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/core/operating_system.py b/tools/lldbmacros/core/operating_system.py old mode 100644 new mode 100755 index fe6d71af2..2e7e21847 --- a/tools/lldbmacros/core/operating_system.py +++ b/tools/lldbmacros/core/operating_system.py @@ -670,7 +670,7 @@ def GetUniqueSessionID(process_obj): return hash(session_key_str) -(archX86_64, archARMv7_family, archI386, archARMv8) = ("x86_64", ("armv7", "armv7s", "armv7k") , "i386", "arm64") +(archX86_64, archARMv7, archI386, archARMv8) = ("x86_64", "armv7", "i386", "arm64") class OperatingSystemPlugIn(object): """Class that provides data for an instance of a LLDB 'OperatingSystemPython' plug-in class""" @@ -712,11 +712,12 @@ class OperatingSystemPlugIn(object): print "Target arch: x86_64" self.register_set = X86_64RegisterSet() self.kernel_context_size = self._target.FindFirstType('x86_kernel_state').GetByteSize() - elif arch in archARMv7_family : + self.kernel_thread_state_size = self._target.FindFirstType('struct thread_kernel_state').GetByteSize() + elif arch.startswith(archARMv7) : self.target_arch = arch print "Target arch: " + self.target_arch self.register_set = Armv7_RegisterSet() - elif arch == archARMv8: + elif arch.startswith(archARMv8): self.target_arch = arch print "Target arch: " + self.target_arch self.register_set = Armv8_RegisterSet() @@ -734,7 +735,7 @@ class OperatingSystemPlugIn(object): print "Instantiating threads completely from saved state in memory." def create_thread(self, tid, context): - # if tid is deadbeef means its a custom thread which kernel does not know of. + # tid == deadbeef means its a custom thread which kernel does not know of. if tid == 0xdeadbeef : # tid manipulation should be the same as in "switchtoregs" code in lldbmacros/process.py . tid = 0xdead0000 | (context & ~0xffff0000) @@ -872,21 +873,21 @@ class OperatingSystemPlugIn(object): if int(PluginValue(thobj).GetChildMemberWithName('kernel_stack').GetValueAsUnsigned()) != 0 : if self.target_arch == archX86_64 : # we do have a stack so lets get register information - saved_state_addr = PluginValue(thobj).GetChildMemberWithName('kernel_stack').GetValueAsUnsigned() + self.kernel_stack_size - self.kernel_context_size + saved_state_addr = PluginValue(thobj).GetChildMemberWithName('kernel_stack').GetValueAsUnsigned() + self.kernel_stack_size - self.kernel_thread_state_size regs.ReadRegisterDataFromKernelStack(saved_state_addr, self.version) return regs.GetPackedRegisterState() - elif self.target_arch in archARMv7_family and int(PluginValue(thobj).GetChildMemberWithName('machine').GetChildMemberWithName('kstackptr').GetValueAsUnsigned()) != 0: + elif self.target_arch.startswith(archARMv7) and int(PluginValue(thobj).GetChildMemberWithName('machine').GetChildMemberWithName('kstackptr').GetValueAsUnsigned()) != 0: #we have stack on the machine.kstackptr. saved_state_addr = PluginValue(thobj).GetChildMemberWithName('machine').GetChildMemberWithName('kstackptr').GetValueAsUnsigned() regs.ReadRegisterDataFromKernelStack(saved_state_addr, self.version) return regs.GetPackedRegisterState() - elif self.target_arch == archARMv8 and int(PluginValue(thobj).GetChildMemberWithName('machine').GetChildMemberWithName('kstackptr').GetValueAsUnsigned()) != 0: + elif self.target_arch.startswith(archARMv8) and int(PluginValue(thobj).GetChildMemberWithName('machine').GetChildMemberWithName('kstackptr').GetValueAsUnsigned()) != 0: saved_state_addr = PluginValue(thobj).GetChildMemberWithName('machine').GetChildMemberWithName('kstackptr').GetValueAsUnsigned() arm_ctx = PluginValue(self.version.CreateValueFromExpression(None, '(struct arm_context *) ' + str(saved_state_addr))) ss_64_addr = arm_ctx.GetChildMemberWithName('ss').GetChildMemberWithName('uss').GetChildMemberWithName('ss_64').GetLoadAddress() regs.ReadRegisterDataFromKernelStack(ss_64_addr, self.version) return regs.GetPackedRegisterState() - elif self.target_arch == archX86_64 or self.target_arch in archARMv7_family or self.target_arch == archARMv8: + elif self.target_arch == archX86_64 or self.target_arch.startswith(archARMv7) or self.target_arch.startswith(archARMv8): regs.ReadRegisterDataFromContinuation( PluginValue(thobj).GetChildMemberWithName('continuation').GetValueAsUnsigned()) return regs.GetPackedRegisterState() #incase we failed very miserably diff --git a/tools/lldbmacros/core/standard.py b/tools/lldbmacros/core/standard.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/core/xnu_lldb_init.py b/tools/lldbmacros/core/xnu_lldb_init.py old mode 100644 new mode 100755 index 41dd202b9..c7f49ea18 --- a/tools/lldbmacros/core/xnu_lldb_init.py +++ b/tools/lldbmacros/core/xnu_lldb_init.py @@ -77,6 +77,7 @@ def __lldb_init_module(debugger, internal_dict): whitelist_trap_cmd = "settings set target.trap-handler-names %s %s" % (' '.join(intel_whitelist), ' '.join(arm_whitelist)) xnu_debug_path = base_dir_name + "/lldbmacros/xnu.py" xnu_load_cmd = "command script import \"%s\"" % xnu_debug_path + disable_optimization_warnings_cmd = "settings set target.process.optimization-warnings false" source_map_cmd = "" try: @@ -97,6 +98,8 @@ def __lldb_init_module(debugger, internal_dict): debugger.HandleCommand(whitelist_trap_cmd) print xnu_load_cmd debugger.HandleCommand(xnu_load_cmd) + print disable_optimization_warnings_cmd + debugger.HandleCommand(disable_optimization_warnings_cmd) if source_map_cmd: print source_map_cmd debugger.HandleCommand(source_map_cmd) diff --git a/tools/lldbmacros/ioreg.py b/tools/lldbmacros/ioreg.py old mode 100644 new mode 100755 index a97b1d96d..74d2e3baa --- a/tools/lldbmacros/ioreg.py +++ b/tools/lldbmacros/ioreg.py @@ -898,10 +898,10 @@ def showinterruptstats(cmd_args=None): Workloop Time: Total time spent running the kernel context handler """ - header_format = "{0: <20s} {1: >5s} {2: >20s} {3: >20s} {4: >20s} {5: >20s} {6: >20s}" - content_format = "{0: <20s} {1: >5d} {2: >20d} {3: >20d} {4: >20d} {5: >20d} {6: >20d}" + header_format = "{0: <20s} {1: >5s} {2: >20s} {3: >20s} {4: >20s} {5: >20s} {6: >20s} {7: >20s} {8: >20s} {9: >20s}" + content_format = "{0: <20s} {1: >5d} {2: >20d} {3: >20d} {4: >20d} {5: >20d} {6: >20d} {7: >20d} {8: >20d} {9: >#20x}" - print header_format.format("Name", "Index", "Interrupt Count", "Interrupt Time", "Workloop Count", "Workloop CPU Time", "Workloop Time") + print header_format.format("Name", "Index", "Interrupt Count", "Interrupt Time", "Avg Interrupt Time", "Workloop Count", "Workloop CPU Time", "Workloop Time", "Avg Workloop Time", "Owner") for i in kern.interrupt_stats: owner = CastIOKitClass(i.owner, 'IOInterruptEventSource *') @@ -934,7 +934,16 @@ def showinterruptstats(cmd_args=None): second_level_cpu_time = i.interruptStatistics[3] second_level_system_time = i.interruptStatistics[4] - print content_format.format(nub_name, interrupt_index, first_level_count, first_level_time, second_level_count, second_level_cpu_time, second_level_system_time) + avg_first_level_time = 0 + if first_level_count != 0: + avg_first_level_time = first_level_time / first_level_count + + avg_second_level_time = 0 + if second_level_count != 0: + avg_second_level_time = second_level_system_time / second_level_count + + print content_format.format(nub_name, interrupt_index, first_level_count, first_level_time, avg_first_level_time, + second_level_count, second_level_cpu_time, second_level_system_time, avg_second_level_time, owner) return True diff --git a/tools/lldbmacros/ipc.py b/tools/lldbmacros/ipc.py old mode 100644 new mode 100755 index 6a71e6e3a..9e5c48215 --- a/tools/lldbmacros/ipc.py +++ b/tools/lldbmacros/ipc.py @@ -250,6 +250,28 @@ def PrintPortSetMembers(space, setid, show_kmsg_summary): idx += 1 return +def FindEntryName(obj, space): + """ Routine to locate a port/ipc_object in an ipc_space + and return the name within that space. + """ + if space == 0: + return 0 + + num_entries = int(space.is_table_size) + is_tableval = space.is_table + idx = 0 + while idx < num_entries: + entry_val = GetObjectAtIndexFromArray(is_tableval, idx) + entry_bits= unsigned(entry_val.ie_bits) + entry_obj = 0 + if (int(entry_bits) & 0x001f0000) != 0: ## it's a valid entry + entry_obj = unsigned(entry_val.ie_object) + if entry_obj == unsigned(obj): + nm = (idx << 8) | (entry_bits >> 24) + return nm + idx += 1 + return 0 + @header("{0: <20s} {1: <28s} {2: <12s} {3: <6s} {4: <6s} {5: <20s} {6: <7s}\n".format( "portset", "waitqueue", "recvname", "flags", "refs", "recvname", "process")) @@ -265,21 +287,22 @@ def PrintPortSetSummary(pset, space = 0): if config['verbosity'] > vHUMAN : show_kmsg_summary = True + local_name = FindEntryName(pset, space) setid = 0 if pset.ips_object.io_bits & 0x80000000: setid = pset.ips_messages.data.pset.setq.wqset_id out_str += "{0: #019x} {1: #019x} {2: <7s} {3: #011x} {4: <4s} {5: >6d} {6: #019x} ".format( unsigned(pset), addressof(pset.ips_messages), ' '*7, - pset.ips_messages.data.pset.local_name, "ASet", + local_name, "ASet", pset.ips_object.io_references, - pset.ips_messages.data.pset.local_name) + local_name) else: out_str += "{0: #019x} {1: #019x} {2: <7s} {3: #011x} {4: <4s} {5: >6d} {6: #019x} ".format( unsigned(pset), addressof(pset.ips_messages), ' '*7, - pset.ips_messages.data.pset.local_name, "DSet", + local_name, "DSet", pset.ips_object.io_references, - pset.ips_messages.data.pset.local_name) + local_name) print out_str if setid != 0 and space != 0: diff --git a/tools/lldbmacros/ipcimportancedetail.py b/tools/lldbmacros/ipcimportancedetail.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/kasan.py b/tools/lldbmacros/kasan.py new file mode 100755 index 000000000..2142fa7ac --- /dev/null +++ b/tools/lldbmacros/kasan.py @@ -0,0 +1,345 @@ +from xnu import * +from utils import * +from core.configuration import * + +shift = None + +shadow_strings = { + 0x00: 'VALID', + 0x01: 'PARTIAL1', + 0x02: 'PARTIAL2', + 0x03: 'PARTIAL3', + 0x04: 'PARTIAL4', + 0x05: 'PARTIAL5', + 0x06: 'PARTIAL6', + 0x07: 'PARTIAL7', + 0xac: 'ARRAY_COOKIE', + 0xf0: 'STACK_RZ', + 0xf1: 'STACK_LEFT_RZ', + 0xf2: 'STACK_MID_RZ', + 0xf3: 'STACK_RIGHT_RZ', + 0xf5: 'STACK_FREED', + 0xf8: 'STACK_OOSCOPE', + 0xf9: 'GLOBAL_RZ', + 0xe9: 'HEAP_RZ', + 0xfa: 'HEAP_LEFT_RZ', + 0xfb: 'HEAP_RIGHT_RZ', + 0xfd: 'HEAP_FREED' +} + +def is_kasan_build(): + try: + enable = kern.globals.kasan_enabled + return True + except ValueError, e: + return False + +def shadow_for_address(addr, shift): + return ((addr >> 3) + shift) + +def address_for_shadow(addr, shift): + return ((addr - shift) << 3) + +def get_shadow_byte(shadow_addr): + return unsigned(kern.GetValueFromAddress(shadow_addr, 'uint8_t *')[0]) + +def print_legend(): + for (k,v) in shadow_strings.iteritems(): + print " {:02x}: {}".format(k,v) + +def print_shadow_context(addr, context): + addr = shadow_for_address(addr, shift) + base = (addr & ~0xf) - 16 * context + shadow = kern.GetValueFromAddress(unsigned(base), "uint8_t *") + + print " "*17 + " 0 1 2 3 4 5 6 7 8 9 a b c d e f" + for x in range(0, 2*context+1): + vals = "" + l = " " + for y in xrange(x*16, (x+1)*16): + r = " " + if base+y == addr: + l = "[" + r = "]" + elif base+y+1 == addr: + r = "" + sh = shadow[y] + vals += "{}{:02x}{}".format(l, sh, r) + l = "" + print("{:x}:{}".format(base + 16*x, vals)) + +kasan_guard_size = 16 +def print_alloc_free_entry(addr, orig_ptr): + h = kern.GetValueFromAddress(addr, 'struct freelist_entry *') + asz = unsigned(h.size) + usz = unsigned(h.user_size) + pgsz = unsigned(kern.globals.page_size) + + if h.zone: + zone = h.zone + if str(zone.zone_name).startswith("fakestack"): + alloc_type = "fakestack" + leftrz = 16 + else: + alloc_type = "zone" + leftrz = unsigned(zone.kasan_redzone) + else: + alloc_type = "kalloc" + if asz - usz >= 2*pgsz: + leftrz = pgsz + else: + leftrz = kasan_guard_size + + rightrz = asz - usz - leftrz + + print "Freed {} object".format(alloc_type) + print "Valid range: 0x{:x} -- 0x{:x} ({} bytes)".format(addr + leftrz, addr + leftrz + usz - 1, usz) + print "Total range: 0x{:x} -- 0x{:x} ({} bytes)".format(addr, addr + asz - 1, asz) + print "Offset: {} bytes".format(orig_ptr - addr - leftrz) + print "Redzone: {} / {} bytes".format(leftrz, rightrz) + if h.zone: + print "Zone: 0x{:x} <{:s}>".format(unsigned(zone), zone.zone_name) + + btframes = unsigned(h.frames) + if btframes > 0: + print "", + print "Free site backtrace ({} frames):".format(btframes) + for i in xrange(0, btframes): + fr = unsigned(kern.globals.vm_kernel_slid_base) + unsigned(h.backtrace[i]) + print " #{:}: {}".format(btframes-i-1, GetSourceInformationForAddress(fr)) + + print "", + print_hexdump(addr, asz, 0) + +alloc_header_sz = 16 + +def print_alloc_info(_addr): + addr = (_addr & ~0x7) + + _shp = shadow_for_address(_addr, shift) + _shbyte = get_shadow_byte(_shp) + _shstr = shadow_byte_to_string(_shbyte) + + # If we're in a left redzone, scan to the start of the real allocation, where + # the header should live + shbyte = _shbyte + while shbyte == 0xfa: + addr += 8 + shbyte = get_shadow_byte(shadow_for_address(addr, shift)) + + # Search backwards for an allocation + searchbytes = 0 + while searchbytes < 8*4096: + + shp = shadow_for_address(addr, shift) + shbyte = get_shadow_byte(shp) + shstr = shadow_byte_to_string(shbyte) + + headerp = addr - alloc_header_sz + liveh = kern.GetValueFromAddress(headerp, 'struct kasan_alloc_header *') + freeh = kern.GetValueFromAddress(addr, 'struct freelist_entry *') + + # heap allocations should only ever have these shadow values + if shbyte not in (0,1,2,3,4,5,6,7, 0xfa, 0xfb, 0xfd, 0xf5): + print "No allocation found at 0x{:x} (found shadow {:x})".format(_addr, shbyte) + return + + live_magic = (addr & 0xffffffff) ^ 0xA110C8ED + free_magic = (addr & 0xffffffff) ^ 0xF23333D + + if live_magic == unsigned(liveh.magic): + usz = unsigned(liveh.user_size) + asz = unsigned(liveh.alloc_size) + leftrz = unsigned(liveh.left_rz) + base = headerp + alloc_header_sz - leftrz + + if _addr >= base and _addr < base + asz: + footer = kern.GetValueFromAddress(addr + usz, 'struct kasan_alloc_footer *') + rightrz = asz - usz - leftrz + + print "Live heap object" + print "Valid range: 0x{:x} -- 0x{:x} ({} bytes)".format(addr, addr + usz - 1, usz) + print "Total range: 0x{:x} -- 0x{:x} ({} bytes)".format(base, base + asz - 1, asz) + print "Offset: {} bytes (shadow: 0x{:02x} {})".format(_addr - addr, _shbyte, _shstr) + print "Redzone: {} / {} bytes".format(leftrz, rightrz) + + btframes = unsigned(liveh.frames) + print "", + print "Alloc site backtrace ({} frames):".format(btframes) + for i in xrange(0, btframes): + fr = unsigned(kern.globals.vm_kernel_slid_base) + unsigned(footer.backtrace[i]) + print " #{:}: {}".format(btframes-i-1, GetSourceInformationForAddress(fr)) + + print "", + print_hexdump(base, asz, 0) + return + + elif free_magic == unsigned(freeh.magic): + asz = unsigned(freeh.size) + if _addr >= addr and _addr < addr + asz: + print_alloc_free_entry(addr, _addr) + return + + searchbytes += 8 + addr -= 8 + + print "No allocation found at 0x{:x}".format(_addr) + +def shadow_byte_to_string(sb): + return shadow_strings.get(sb, '??') + +def print_whatis(_addr, ctx): + addr = _addr & ~0x7 + total_size = 0 + base = None + leftrz = None + rightrz = None + extra = "Live" + + shbyte = get_shadow_byte(shadow_for_address(addr, shift)) + maxsearch = 4096 * 2 + + if shbyte in [0xfa, 0xfb, 0xfd, 0xf5]: + print_alloc_info(_addr) + return + + if shbyte not in [0,1,2,3,4,5,6,7,0xf8]: + print "Poisoned memory, shadow {:x} [{}]".format(shbyte, shadow_byte_to_string(shbyte)) + return + + if shbyte is 0xf8: + extra = "Out-of-scope" + + # look for the base of the object + while shbyte in [0,1,2,3,4,5,6,7,0xf8]: + sz = 8 - shbyte + if shbyte is 0xf8: + sz = 8 + total_size += sz + addr -= 8 + shbyte = get_shadow_byte(shadow_for_address(addr, shift)) + maxsearch -= 8 + if maxsearch <= 0: + print "No object found" + return + base = addr + 8 + leftrz = shbyte + + # If we did not find a left/mid redzone, we aren't in an object + if leftrz not in [0xf1, 0xf2, 0xfa, 0xf9]: + print "No object found" + return + + # now size the object + addr = (_addr & ~0x7) + 8 + shbyte = get_shadow_byte(shadow_for_address(addr, shift)) + while shbyte in [0,1,2,3,4,5,6,7,0xf8]: + sz = 8 - shbyte + if shbyte is 0xf8: + sz = 8 + total_size += sz + addr += 8 + shbyte = get_shadow_byte(shadow_for_address(addr, shift)) + maxsearch -= 8 + if maxsearch <= 0: + print "No object found" + return + rightrz = shbyte + + # work out the type of the object from its redzone + objtype = "Unknown" + if leftrz == 0xf1 or leftrz == 0xf2: + objtype = "stack" + elif leftrz == 0xf9 and rightrz == 0xf9: + objtype = "global" + elif leftrz == 0xfa and rightrz == 0xfb: + print_alloc_info(_addr) + return + + print "{} {} object".format(extra, objtype) + print "Valid range: 0x{:x} -- 0x{:x} ({} bytes)".format(base, base+total_size-1, total_size) + print "Offset: {} bytes".format(_addr - base) + print "", + print_hexdump(base, total_size, 0) + +def print_hexdump(base, size, ctx): + start = base - 16*ctx + size += size % 16 + size = min(size + 16*2*ctx, 256) + + try: + data_array = kern.GetValueFromAddress(start, "uint8_t *") + print_hex_data(data_array[0:size], start, "Hexdump") + except: + pass + +def kasan_subcommand(cmd, args, opts): + addr = None + if len(args) > 0: + addr = long(args[0], 0) + + if cmd in ['a2s', 'toshadow', 'fromaddr', 'fromaddress']: + print "0x{:016x}".format(shadow_for_address(addr, shift)) + elif cmd in ['s2a', 'toaddr', 'toaddress', 'fromshadow']: + print "0x{:016x}".format(address_for_shadow(addr, shift)) + elif cmd == 'shadow': + shadow = shadow_for_address(addr, shift) + sb = get_shadow_byte(shadow) + print("0x{:02x} @ 0x{:016x} [{}]\n\n".format(sb, shadow, shadow_byte_to_string(sb))) + ctx = long(opts.get("-C", 5)) + print_shadow_context(addr, ctx) + elif cmd == 'legend': + print_legend() + elif cmd == 'info': + pages_used = unsigned(kern.globals.shadow_pages_used) + pages_total = unsigned(kern.globals.shadow_pages_total) + nkexts = unsigned(kern.globals.kexts_loaded) + print "Offset: 0x{:016x}".format(shift) + print "Shadow used: {} / {} ({:.1f}%)".format(pages_used, pages_total, 100.0*pages_used/pages_total) + print "Kexts loaded: {}".format(nkexts) + elif cmd == 'whatis': + ctx = long(opts.get("-C", 1)) + print_whatis(addr, ctx) + elif cmd == 'alloc' or cmd == 'heap': + print_alloc_info(addr) + +@lldb_command('kasan', 'C:') +def Kasan(cmd_args=None, cmd_options={}): + """kasan [opts..] + + Commands: + + info basic KASan information + shadow print shadow around 'addr' + heap show info about heap object at 'addr' + whatis print whatever KASan knows about address + toshadow convert address to shadow pointer + toaddr convert shadow pointer to address + legend print a shadow byte table + + -C : num lines of context to show""" + + if not is_kasan_build(): + print "KASan not enabled in build" + return + + if len(cmd_args) == 0: + print Kasan.__doc__ + return + + global shift + shift = unsigned(kern.globals.__asan_shadow_memory_dynamic_address) + + # Since the VM is not aware of the KASan shadow mapping, accesses to it will + # fail. Setting kdp_read_io=1 avoids this check. + if GetConnectionProtocol() == "kdp" and unsigned(kern.globals.kdp_read_io) == 0: + print "Setting kdp_read_io=1 to allow KASan shadow reads" + if sizeof(kern.globals.kdp_read_io) == 4: + WriteInt32ToMemoryAddress(1, addressof(kern.globals.kdp_read_io)) + elif sizeof(kern.globals.kdp_read_io) == 8: + WriteInt64ToMemoryAddress(1, addressof(kern.globals.kdp_read_io)) + readio = unsigned(kern.globals.kdp_read_io) + assert readio == 1 + + return kasan_subcommand(cmd_args[0], cmd_args[1:], cmd_options) + diff --git a/tools/lldbmacros/kauth.py b/tools/lldbmacros/kauth.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/kcdata.py b/tools/lldbmacros/kcdata.py index 4ef6a7597..ecc899a01 100755 --- a/tools/lldbmacros/kcdata.py +++ b/tools/lldbmacros/kcdata.py @@ -15,6 +15,12 @@ import logging import contextlib import base64 +class Globals(object): + pass +G = Globals() +G.accept_incomplete_data = False +G.data_was_incomplete = False + kcdata_type_def = { 'KCDATA_TYPE_INVALID': 0x0, 'KCDATA_TYPE_STRING_DESC': 0x1, @@ -82,6 +88,14 @@ kcdata_type_def = { 'STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS': 0x91b, 'STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO': 0x91c, 'STACKSHOT_KCTYPE_THREAD_WAITINFO' : 0x91d, + 'STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT' : 0x91e, + 'STACKSHOT_KCTYPE_THREAD_GROUP' : 0x91f, + 'STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT' : 0x920, + 'STACKSHOT_KCTYPE_JETSAM_COALITION' : 0x921, + 'STACKSHOT_KCTYPE_INSTRS_CYCLES' : 0x923, + + 'STACKSHOT_KCTYPE_THREAD_POLICY_VERSION': 0x922, + 'KCDATA_TYPE_BUFFER_END': 0xF19158ED, @@ -120,6 +134,8 @@ kcdata_type_def = { 'EXIT_REASON_USER_DESC': 0x1002, 'EXIT_REASON_USER_PAYLOAD': 0x1003, 'EXIT_REASON_CODESIGNING_INFO': 0x1004, + 'EXIT_REASON_WORKLOOP_ID': 0x1005, + 'EXIT_REASON_DISPATCH_QUEUE_NO': 0x1006, 'KCDATA_BUFFER_BEGIN_CRASHINFO': 0xDEADF157, 'KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT': 0xDE17A59A, 'KCDATA_BUFFER_BEGIN_STACKSHOT': 0x59a25807, @@ -239,8 +255,7 @@ class KCSubTypeElement(object): if ord(_v) == 0: break str_arr.append(self.GetValueAsString(base_data, i)) - - return '"' + ''.join(str_arr) + '"' + return json.dumps(''.join(str_arr)) count = self.count if count > len(base_data)/self.size: @@ -397,6 +412,9 @@ class KCObject(object): self.InitAfterParse() + def __str__(self): + return "" % self.offset + def InitAfterParse(self): pass @@ -605,11 +623,19 @@ class KCContainerObject(KCObject): if self.IsEndMarker(o): found_end = True break + if o.IsBufferEnd(): + break if isinstance(o, KCContainerObject): o.ReadItems(iterator) self.AddObject(o) if not found_end: - raise Exception, self.no_end_message + if G.accept_incomplete_data: + if not G.data_was_incomplete: + print >>sys.stderr, "kcdata.py WARNING: data is incomplete!" + G.data_was_incomplete = True + else: + raise Exception, self.no_end_message + class KCBufferObject(KCContainerObject): @@ -778,6 +804,8 @@ KNOWN_TYPES_COLLECTION[0x906] = KCTypeDescription(0x906, ( KCSubTypeElement.FromBasicCtype('ths_rqos_override', KCSUBTYPE_TYPE.KC_ST_UINT8, 102), KCSubTypeElement.FromBasicCtype('ths_io_tier', KCSUBTYPE_TYPE.KC_ST_UINT8, 103), KCSubTypeElement.FromBasicCtype('ths_thread_t', KCSUBTYPE_TYPE.KC_ST_UINT64, 104), + KCSubTypeElement.FromBasicCtype('ths_requested_policy', KCSUBTYPE_TYPE.KC_ST_UINT64, 112), + KCSubTypeElement.FromBasicCtype('ths_effective_policy', KCSUBTYPE_TYPE.KC_ST_UINT64, 120), ), 'thread_snapshot', legacy_size = 0x68 @@ -923,6 +951,8 @@ KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_BOOTARGS')] = KCSubTypeE KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERN_PAGE_SIZE')] = KCSubTypeElement('kernel_page_size', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 0, 0, KCSubTypeElement._get_naked_element_value) +KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_POLICY_VERSION')] = KCSubTypeElement('thread_policy_version', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 0, 0, KCSubTypeElement._get_naked_element_value) + KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_JETSAM_LEVEL')] = KCSubTypeElement('jetsam_level', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 0, 0, KCSubTypeElement._get_naked_element_value) KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_DELTA_SINCE_TIMESTAMP')] = KCSubTypeElement("stackshot_delta_since_timestamp", KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value) @@ -945,6 +975,34 @@ KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_WAITINFO')] = KCT ), 'thread_waitinfo') +KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP'), + ( + KCSubTypeElement.FromBasicCtype('tgs_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 0), + KCSubTypeElement('tgs_name', KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(16, 1), 8, 1) + ), + 'thread_group_snapshot') + + +KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP')] = KCSubTypeElement('thread_group', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value) + +KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT'), + ( + KCSubTypeElement.FromBasicCtype('jcs_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 0), + KCSubTypeElement.FromBasicCtype('jcs_flags', KCSUBTYPE_TYPE.KC_ST_UINT64, 8), + KCSubTypeElement.FromBasicCtype('jcs_thread_group', KCSUBTYPE_TYPE.KC_ST_UINT64, 16), + KCSubTypeElement.FromBasicCtype('jcs_leader_task_uniqueid', KCSUBTYPE_TYPE.KC_ST_UINT64, 24) + ), + 'jetsam_coalition_snapshot') + +KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_JETSAM_COALITION')] = KCSubTypeElement('jetsam_coalition', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value) + +KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_INSTRS_CYCLES')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_INSTRS_CYCLES'), + ( + KCSubTypeElement.FromBasicCtype('ics_instructions', KCSUBTYPE_TYPE.KC_ST_UINT64, 0), + KCSubTypeElement.FromBasicCtype('ics_cycles', KCSUBTYPE_TYPE.KC_ST_UINT64, 8) + ), + 'instrs_cycles_snapshot') + #KNOWN_TYPES_COLLECTION[0x907] = KCSubTypeElement('donating_pids', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 0, 0, KCSubTypeElement._get_naked_element_value) KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PID')] = KCSubTypeElement('pid', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0) KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PPID')] = KCSubTypeElement('ppid', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0) @@ -1075,6 +1133,11 @@ KNOWN_TYPES_COLLECTION[GetTypeForName('EXIT_REASON_CODESIGNING_INFO')] = KCTypeD KCSubTypeElement.FromBasicCtype('ceri_page_shadow_depth', KCSUBTYPE_TYPE.KC_ST_UINT32, 2104), ), 'exit_reason_codesigning_info') +KNOWN_TYPES_COLLECTION[GetTypeForName('EXIT_REASON_WORKLOOP_ID')] = ( + KCSubTypeElement('exit_reason_workloop_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value)) + +KNOWN_TYPES_COLLECTION[GetTypeForName('EXIT_REASON_DISPATCH_QUEUE_NO')] = ( + KCSubTypeElement('exit_reason_dispatch_queue_no', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value)) def GetSecondsFromMATime(mat, tb): return (float(mat) * tb['numer']) / tb['denom'] @@ -1141,7 +1204,127 @@ def GetStateDescription(s): def format_uuid(elementValues): return ''.join("%02x" % i for i in elementValues) -def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr): +kThreadWaitNone = 0x00 +kThreadWaitKernelMutex = 0x01 +kThreadWaitPortReceive = 0x02 +kThreadWaitPortSetReceive = 0x03 +kThreadWaitPortSend = 0x04 +kThreadWaitPortSendInTransit = 0x05 +kThreadWaitSemaphore = 0x06 +kThreadWaitKernelRWLockRead = 0x07 +kThreadWaitKernelRWLockWrite = 0x08 +kThreadWaitKernelRWLockUpgrade = 0x09 +kThreadWaitUserLock = 0x0a +kThreadWaitPThreadMutex = 0x0b +kThreadWaitPThreadRWLockRead = 0x0c +kThreadWaitPThreadRWLockWrite = 0x0d +kThreadWaitPThreadCondVar = 0x0e +kThreadWaitParkedWorkQueue = 0x0f +kThreadWaitWorkloopSyncWait = 0x10 + + +UINT64_MAX = 0xffffffffffffffff +STACKSHOT_WAITOWNER_KERNEL = (UINT64_MAX - 1) +STACKSHOT_WAITOWNER_PORT_LOCKED = (UINT64_MAX - 2) +STACKSHOT_WAITOWNER_PSET_LOCKED = (UINT64_MAX - 3) +STACKSHOT_WAITOWNER_INTRANSIT = (UINT64_MAX - 4) +STACKSHOT_WAITOWNER_MTXSPIN = (UINT64_MAX - 5) +STACKSHOT_WAITOWNER_THREQUESTED = (UINT64_MAX - 6) +STACKSHOT_WAITOWNER_SUSPENDED = (UINT64_MAX - 7) + +def formatWaitInfo(info): + s = 'thread %d: ' % info['waiter']; + type = info['wait_type'] + context = info['context'] + owner = info['owner'] + if type == kThreadWaitKernelMutex: + s += 'kernel mutex %x' % context + if owner == STACKSHOT_WAITOWNER_MTXSPIN: + s += " in spin mode" + elif owner: + s += " owned by thread %u" % owner + else: + s += "with unknown owner" + elif type == kThreadWaitPortReceive: + s += "mach_msg receive on " + if owner == STACKSHOT_WAITOWNER_PORT_LOCKED: + s += "locked port %x" % context + elif owner == STACKSHOT_WAITOWNER_INTRANSIT: + s += "intransit port %x" % context + elif owner: + s += "port %x name %x" % (context, owner) + else: + s += "port %x" % context + elif type == kThreadWaitPortSetReceive: + if owner == STACKSHOT_WAITOWNER_PSET_LOCKED: + s += "mach_msg receive on locked port set %x" % context + else: + s += "mach_msg receive on port set %x" % context + elif type == kThreadWaitPortSend: + s += "mach_msg send on " + if owner == STACKSHOT_WAITOWNER_PORT_LOCKED: + s += "locked port %x" % context + elif owner == STACKSHOT_WAITOWNER_INTRANSIT: + s += "intransit port %x" % context + elif owner == STACKSHOT_WAITOWNER_KERNEL: + s += "port %x owned by kernel" % context + elif owner: + s += "port %x owned by pid %d" % (context, owner) + else: + s += "port %x with unknown owner" % context + elif type == kThreadWaitPortSendInTransit: + s += "mach_msg send on port %x in transit to " % context + if owner: + s += "port %x" % owner + else: + s += "unknown port" + elif type == kThreadWaitSemaphore: + s += "semaphore port %x" % context + if owner: + s += "owned by pid %d" % owner + else: + s += "with unknown owner" + elif type == kThreadWaitKernelRWLockRead: + s += "krwlock %x for reading" % context + elif type == kThreadWaitKernelRWLockWrite: + s += "krwlock %x for writing" % context + elif type == kThreadWaitKernelRWLockUpgrade: + s += "krwlock %x for upgrading" % context + elif type == kThreadWaitUserLock: + if owner: + s += "unfair lock %x owned by pid %d" % (context, owner) + else: + s += "spin lock %x" % context + elif type == kThreadWaitPThreadMutex: + s += "pthread mutex %x" % context + if owner: + s += " owned by pid %d" % owner + else: + s += " with unknown owner" + elif type == kThreadWaitPThreadRWLockRead: + s += "pthread rwlock %x for reading" % context + elif type == kThreadWaitPThreadRWLockWrite: + s += "pthread rwlock %x for writing" % context + elif type == kThreadWaitPThreadCondVar: + s += "pthread condvar %x" % context + elif type == kThreadWaitWorkloopSyncWait: + s += "workloop sync wait" + if owner == STACKSHOT_WAITOWNER_SUSPENDED: + s += ", suspended" + elif owner == STACKSHOT_WAITOWNER_THREQUESTED: + s += ", thread requested" + elif owner != 0: + s += ", owned by thread %u" % owner + else: + s += ", unknown owner" + s += ", workloop id %x" % context + else: + s += "unknown type %d (owner %d, context %x)" % (type, owner, context) + + return s + + +def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr, incomplete): import time from operator import itemgetter, attrgetter ss = j.get('kcdata_stackshot') @@ -1198,6 +1381,11 @@ def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr): obj["frontmostPids"] = [0] obj["exception"] = "0xDEADF157" obj["processByPid"] = {} + + if incomplete: + obj["reason"] = "!!!INCOMPLETE!!! kernel panic stackshot" + obj["notes"] = "This stackshot report generated from incomplete data! Some information is missing! " + processByPid = obj["processByPid"] ssplist = ss.get('task_snapshots', {}) kern_load_info = [] @@ -1242,6 +1430,8 @@ def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr): pr_libs.sort(key=itemgetter(1)) + if 'task_snapshot' not in piddata: + continue tasksnap = piddata['task_snapshot'] tsnap["pid"] = tasksnap["ts_pid"] tsnap["residentMemoryBytes"] = tasksnap["ts_task_size"] @@ -1288,8 +1478,14 @@ def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr): if threadsnap['ths_wait_event']: thsnap["waitEvent"] = GetSymbolInfoForFrame(AllImageCatalog, pr_libs, threadsnap['ths_wait_event']) + if 'thread_waitinfo' in piddata: + tsnap['waitInfo'] = map(formatWaitInfo, piddata['thread_waitinfo']) + obj['binaryImages'] = AllImageCatalog - fh = open(outfile_name, "w") + if outfile_name == '-': + fh = sys.stdout + else: + fh = open(outfile_name, "w") fh.write('{"bug_type":"288", "timestamp":"'+ timestamp +'", "os_version":"'+ os_version +'"}\n') fh.write(json.dumps(obj, sort_keys=False, indent=2, separators=(',', ': '))) fh.close() @@ -1364,6 +1560,8 @@ parser.add_argument("-U", "--uuid", required=False, default="", help="UUID of dy parser.add_argument("-L", "--layout", required=False, type=argparse.FileType("r"), help="Path to layout file for DyldSharedCache. You can generate one by doing \n\tbash$xcrun -sdk dyld_shared_cache_util -text_info ", dest="layout") parser.add_argument("-S", "--sdk", required=False, default="", help="sdk property passed to xcrun command to find the required tools. Default is empty string.", dest="sdk") parser.add_argument("-D", "--dyld_shared_cache", required=False, default="", help="Path to dyld_shared_cache built by B&I", dest="dsc") +parser.add_argument("--pretty", default=False, action='store_true', help="make the output a little more human readable") +parser.add_argument("--incomplete", action='store_true', help="accept incomplete data") parser.add_argument("kcdata_file", type=argparse.FileType('r'), help="Path to a kcdata binary file.") class VerboseAction(argparse.Action): @@ -1419,6 +1617,26 @@ def iterate_kcdatas(kcdata_file): yield kcdata_buffer +def prettify(data): + if isinstance(data, list): + return map(prettify, data); + + elif isinstance(data, dict): + newdata = dict() + for key, value in data.items(): + if 'uuid' in key.lower() and isinstance(value, list) and len(value) == 16: + value = '%02X%02X%02X%02X-%02X%02X-%02X%02X-%02X%02X-%02X%02X%02X%02X%02X%02X' % tuple(value) + elif 'address' in key.lower() and isinstance(value, (int, long)): + value = '0x%X' % value + else: + value = prettify(value); + newdata[key] = value + + return newdata + + else: + return data + if __name__ == '__main__': args = parser.parse_args() @@ -1431,6 +1649,9 @@ if __name__ == '__main__': print "%d : %s " % (n, str(t)) sys.exit(1) + if args.incomplete or args.stackshot_file: + G.accept_incomplete_data = True + for i,kcdata_buffer in enumerate(iterate_kcdatas(args.kcdata_file)): if i > 0 and not args.multiple: break @@ -1446,6 +1667,9 @@ if __name__ == '__main__': print textwrap.fill(str_data, 100) raise + if args.pretty: + json_obj = prettify(json_obj) + dsc_uuid = None dsc_libs_arr = [] libs_re = re.compile("^\s*(0x[a-fA-F0-9]+)\s->\s(0x[a-fA-F0-9]+)\s+<([a-fA-F0-9\-]+)>\s+.*$", re.MULTILINE) @@ -1460,7 +1684,7 @@ if __name__ == '__main__': dsc_libs_arr = libs_re.findall(_ret[1]) if args.stackshot_file: - SaveStackshotReport(json_obj, args.stackshot_file, dsc_uuid, dsc_libs_arr) + SaveStackshotReport(json_obj, args.stackshot_file, dsc_uuid, dsc_libs_arr, G.data_was_incomplete) elif args.plist: import Foundation plist = Foundation.NSPropertyListSerialization.dataWithPropertyList_format_options_error_( diff --git a/tools/lldbmacros/kdp.py b/tools/lldbmacros/kdp.py old mode 100644 new mode 100755 index 7c31630ea..e8bc324b1 --- a/tools/lldbmacros/kdp.py +++ b/tools/lldbmacros/kdp.py @@ -174,7 +174,7 @@ def KDPResumeON(cmd_args=None): return retval @lldb_command('resume_off') -def KDPResumeON(cmd_args=None): +def KDPResumeOFF(cmd_args=None): """ The target system will not resume when detaching or exiting from lldb. """ subcmd = GetEnumValue('kdp_dumpinfo_t::KDP_DUMPINFO_SETINFO') | GetEnumValue('kdp_dumpinfo_t::KDP_DUMPINFO_NORESUME') diff --git a/tools/lldbmacros/kevent.py b/tools/lldbmacros/kevent.py new file mode 100755 index 000000000..8f9f7c951 --- /dev/null +++ b/tools/lldbmacros/kevent.py @@ -0,0 +1,381 @@ +from xnu import * + +def IterateProcKqueues(proc): + """ Iterate through all kqueues in the given process + + params: + proc - the proc object + returns: nothing, this is meant to be used as a generator function + kq - yields each kqueue in the process + """ + for kqf in IterateProcKqfiles(proc): + yield kern.GetValueFromAddress(int(kqf), 'struct kqueue *') + if int(proc.p_fd.fd_wqkqueue) != 0: + yield kern.GetValueFromAddress(int(proc.p_fd.fd_wqkqueue), 'struct kqueue *') + for kqwl in IterateProcKqworkloops(proc): + yield kern.GetValueFromAddress(int(kqwl), 'struct kqueue *') + +def IterateProcKqfiles(proc): + """ Iterate through all kqfiles in the given process + + params: + proc - the proc object + returns: nothing, this is meant to be used as a generator function + kqf - yields each kqfile in the process + """ + filetype_KQUEUE = 5 + + proc_filedesc = proc.p_fd + proc_lastfile = unsigned(proc_filedesc.fd_lastfile) + proc_ofiles = proc_filedesc.fd_ofiles + queues = list() + count = 0 + + if unsigned(proc_ofiles) == 0: + return + + while count <= proc_lastfile: + if unsigned(proc_ofiles[count]) != 0: + proc_fd_flags = proc_ofiles[count].f_flags + proc_fd_fglob = proc_ofiles[count].f_fglob + proc_fd_ftype = unsigned(proc_fd_fglob.fg_ops.fo_type) + if proc_fd_ftype == xnudefines.DTYPE_KQUEUE: + yield kern.GetValueFromAddress(int(proc_fd_fglob.fg_data), 'struct kqfile *') + count += 1 + +def IterateProcKqworkloops(proc): + """ Iterate through all kqworkloops in the given process + + params: + proc - the proc object + returns: nothing, this is meant to be used as a generator function + kqwl - yields each kqworkloop in the process + """ + proc_filedesc = proc.p_fd + if int(proc_filedesc.fd_kqhash) == 0: + return + + hash_mask = proc_filedesc.fd_kqhashmask + for i in xrange(hash_mask + 1): + for kqwl in IterateListEntry(proc_filedesc.fd_kqhash[i], 'struct kqworkloop *', 'kqwl_hashlink', list_prefix='s'): + yield kqwl + +def IterateAllKqueues(): + """ Iterate through all kqueues in the system + + returns: nothing, this is meant to be used as a generator function + kq - yields each kqueue in the system + """ + for t in kern.tasks: + if unsigned(t.bsd_info) == 0: + continue + proc = kern.GetValueFromAddress(t.bsd_info, 'proc_t') + for kq in IterateProcKqueues(proc): + yield kq + +def IterateProcKnotes(proc): + """ Iterate through all knotes in the given process + + params: + proc - the proc object + returns: nothing, this is meant to be used as a generator function + kn - yields each knote in the process + """ + proc_filedesc = proc.p_fd + + if int(proc.p_fd.fd_knlist) != 0: + for i in xrange(proc.p_fd.fd_knlistsize): + for kn in IterateListEntry(proc.p_fd.fd_knlist[i], 'struct knote *', 'kn_link', list_prefix='s'): + yield kn + if int(proc.p_fd.fd_knhash) != 0: + for i in xrange(proc.p_fd.fd_knhashmask + 1): + for kn in IterateListEntry(proc.p_fd.fd_knhash[i], 'struct knote *', 'kn_link', list_prefix='s'): + yield kn + +def GetKnoteKqueue(kn): + """ Get the kqueue corresponding to a given knote + + params: + kn - the knote object + returns: kq - the kqueue corresponding to the knote + """ + return kern.GetValueFromAddress(kn.kn_kq_packed + kern.VM_MIN_KERNEL_AND_KEXT_ADDRESS, 'struct kqueue *') + +@lldb_type_summary(['knote *']) +@header('{:<20s} {:<20s} {:<10s} {:<20s} {:<20s} {:<30s} {:<10} {:<10} {:<10} {:<30s}'.format('knote', 'ident', 'kev_flags', 'kqueue', 'udata', 'filtops', 'qos_use', 'qos_req', 'qos_ovr', 'status')) +def GetKnoteSummary(kn): + """ Summarizes a knote and related information + + returns: str - summary of knote + """ + format_string = '{o: <#020x} {o.kn_kevent.ident: <#020x} {o.kn_kevent.flags: <#010x} {kq_ptr: <#020x} {o.kn_kevent.udata: <#020x} {ops_str: <30s} {qos_use: <10s} {qos_req: <10s} {qos_ovr: <10s} {st_str: <30s}' + state = unsigned(kn.kn_status) + fops_str = kern.Symbolicate(kern.globals.sysfilt_ops[unsigned(kn.kn_filtid)]) + return format_string.format( + o=kn, + qos_use=xnudefines.thread_qos_short_strings[int(kn.kn_qos_index)], + qos_req=xnudefines.thread_qos_short_strings[int(kn.kn_req_index)], + qos_ovr=xnudefines.thread_qos_short_strings[int(kn.kn_qos_override)], + st_str=xnudefines.GetStateString(xnudefines.kn_state_strings, state), + kq_ptr=int(GetKnoteKqueue(kn)), + ops_str=fops_str) + +@lldb_command('showknote') +def ShowKnote(cmd_args=None): + """ Show information about a knote + + usage: showknote + """ + if not cmd_args: + raise ArgumentError('missing struct knote * argument') + + kn = kern.GetValueFromAddress(cmd_args[0], 'struct knote *') + print GetKnoteSummary.header + print GetKnoteSummary(kn) + +def IterateKqueueKnotes(kq): + """ Iterate through all knotes of a given kqueue + + params: + kq - the kqueue to iterate the knotes of + returns: nothing, this is meant to be used as a generator function + kn - yields each knote in the kqueue + """ + proc = kq.kq_p + for kn in IterateProcKnotes(proc): + if unsigned(GetKnoteKqueue(kn)) != unsigned(addressof(kq)): + continue + yield kn + +@lldb_type_summary(['struct kqrequest *']) +@header('{:<20s} {:<20s} {:<5s} {:<5s} {:<5s} {:<5s} {:s}'.format('kqrequest', 'thread', 'qos', 'ovr_qos', 'w_qos', 'sa_qos', 'state')) +def GetKqrequestSummary(kqr): + """ Summarize kqrequest information + + params: + kqr - the kqrequest object + returns: str - summary of kqrequest + """ + fmt = '{kqrp: <#020x} {kqr.kqr_bound.kqrb_thread: <#020x} {qos: <5s} {ovr_qos: <5s} {w_qos: <5s} {sa_qos: <5s} {state_str: + """ + if len(cmd_args) < 1: + raise ArgumentError('missing struct kqrequest * argument') + kqr = kern.GetValueFromAddress(cmd_args[0], 'struct kqrequest *') + print GetKqrequestSummary.header + print GetKqrequestSummary(kqr) + print GetKnoteSummary.header + for kn in IterateTAILQ_HEAD(kqr.kqr_suppressed, 'kn_tqe'): + print GetKnoteSummary(kn) + +kqueue_summary_fmt = '{ptr: <#020x} {o.kq_p: <#020x} {dyn_id: <#020x} {servicer: <#20x} {owner: <#20x} {o.kq_count: <6d} {wqs: <#020x} {kqr_state: <30s} {st_str: <10s}' + +@lldb_type_summary(['struct kqueue *']) +@header('{: <20s} {: <20s} {: <20s} {: <20s} {: <20s} {: <6s} {: <20s} {: <30s} {: <10s}'.format('kqueue', 'process', 'dynamic_id', 'servicer', 'owner', '#evts', 'wqs', 'request', 'state')) +def GetKqueueSummary(kq): + """ Summarize kqueue information + + params: + kq - the kqueue object + returns: str - summary of kqueue + """ + if kq.kq_state & xnudefines.KQ_WORKLOOP: + return GetKqworkloopSummary(kern.GetValueFromAddress(int(kq), 'struct kqworkloop *')) + elif kq.kq_state & xnudefines.KQ_WORKQ: + return GetKqworkqSummary(kern.GetValueFromAddress(int(kq), 'struct kqworkq *')) + else: + return GetKqfileSummary(kern.GetValueFromAddress(int(kq), 'struct kqfile *')) + +@lldb_type_summary(['struct kqfile *']) +@header(GetKqueueSummary.header) +def GetKqfileSummary(kqf): + kq = kern.GetValueFromAddress(int(kqf), 'struct kqueue *') + state = int(kq.kq_state) + return kqueue_summary_fmt.format( + o=kq, + ptr=int(kq), + wqs=int(kq.kq_wqs), + kqr_state='', + dyn_id=0, + st_str=xnudefines.GetStateString(xnudefines.kq_state_strings, state), + servicer=0, + owner=0) + +@lldb_command('showkqfile') +def ShowKqfile(cmd_args=None): + """ Display information about a kqfile object. + + usage: showkqfile + """ + if len(cmd_args) < 1: + raise ArgumentError('missing struct kqfile * argument') + + kqf = kern.GetValueFromAddress(cmd_args[0], 'kqfile *') + + print GetKqfileSummary.header + print GetKqfileSummary(kqf) + print GetKnoteSummary.header + for kn in IterateKqueueKnotes(kqf.kqf_kqueue): + print GetKnoteSummary(kn) + for kn in IterateTAILQ_HEAD(kqf.kqf_suppressed, 'kn_tqe'): + print GetKnoteSummary(kn) + +@lldb_type_summary(['struct kqworkq *']) +@header(GetKqueueSummary.header) +def GetKqworkqSummary(kqwq): + """ Summarize workqueue kqueue information + + params: + kqwq - the kqworkq object + returns: str - summary of workqueue kqueue + """ + return GetKqfileSummary(kern.GetValueFromAddress(int(kqwq), 'struct kqfile *')) + +@lldb_command('showkqworkq') +def ShowKqworkq(cmd_args=None): + """ Display summary and knote information about a kqworkq. + + usage: showkqworkq + """ + if len(cmd_args) < 1: + raise ArgumentError('missing struct kqworkq * argument') + + kqwq = kern.GetValueFromAddress(cmd_args[0], 'struct kqworkq *') + kq = kqwq.kqwq_kqueue + print GetKqueueSummary.header + print GetKqworkqSummary(kqwq) + print GetKnoteSummary.header + for kn in IterateKqueueKnotes(kq): + print GetKnoteSummary(kn) + for i in xrange(0, xnudefines.KQWQ_NBUCKETS): + for kn in IterateTAILQ_HEAD(kq.kq_queue[i], 'kn_tqe'): + print GetKnoteSummary(kn) + +@lldb_type_summary(['struct kqworkloop *']) +@header(GetKqueueSummary.header) +def GetKqworkloopSummary(kqwl): + """ Summarize workloop kqueue information + + params: + kqwl - the kqworkloop object + returns: str - summary of workloop kqueue + """ + state = int(kqwl.kqwl_kqueue.kq_state) + return kqueue_summary_fmt.format( + ptr=int(kqwl), + o=kqwl.kqwl_kqueue, + wqs=int(kqwl.kqwl_kqueue.kq_wqs), + dyn_id=kqwl.kqwl_dynamicid, + kqr_state=xnudefines.GetStateString(xnudefines.kqrequest_state_strings, kqwl.kqwl_request.kqr_state), + st_str=xnudefines.GetStateString(xnudefines.kq_state_strings, state), + servicer=int(kqwl.kqwl_request.kqr_bound.kqrb_thread), + owner=int(kqwl.kqwl_owner) + ) + +@lldb_command('showkqworkloop') +def ShowKqworkloop(cmd_args=None): + """ Display information about a kqworkloop. + + usage: showkqworkloop + """ + if len(cmd_args) < 1: + raise ArgumentError('missing struct kqworkloop * argument') + + kqwl = kern.GetValueFromAddress(cmd_args[0], 'struct kqworkloop *') + + print GetKqworkloopSummary.header + print GetKqworkloopSummary(kqwl) + + print GetKqrequestSummary.header + kqr = kern.GetValueFromAddress(unsigned(addressof(kqwl.kqwl_request)), 'struct kqrequest *') + print GetKqrequestSummary(kqr) + + print GetKnoteSummary.header + for kn in IterateKqueueKnotes(kqwl.kqwl_kqueue): + print GetKnoteSummary(kn) + +@lldb_command('showkqueue') +def ShowKqueue(cmd_args=None): + """ Given a struct kqueue pointer, display the summary of the kqueue + + usage: showkqueue + """ + if not cmd_args: + raise ArgumentError('missing struct kqueue * argument') + + kq = kern.GetValueFromAddress(cmd_args[0], 'struct kqueue *') + if int(kq.kq_state) & xnudefines.KQ_WORKQ: + ShowKqworkq(cmd_args=[str(int(kq))]) + elif int(kq.kq_state) & xnudefines.KQ_WORKLOOP: + ShowKqworkloop(cmd_args=[str(int(kq))]) + else: + print GetKqueueSummary.header + print GetKqueueSummary(kq) + print GetKnoteSummary.header + for kn in IterateKqueueKnotes(kq): + print GetKnoteSummary(kn) + +@lldb_command('showprocworkqkqueue') +def ShowProcWorkqKqueue(cmd_args=None): + """ Show the workqueue kqueue for a given process. + + usage: showworkqkqueue + """ + if not cmd_args: + raise ArgumentError('missing struct proc * argument') + + proc = kern.GetValueFromAddress(cmd_args[0], 'proc_t') + ShowKqworkq(cmd_args=[str(int(proc.p_fd.fd_wqkqueue))]) + +@lldb_command('showprockqueues') +def ShowProcKqueues(cmd_args=None): + """ Show the kqueues for a given process. + + usage: showprockqueues + """ + if not cmd_args: + raise ArgumentError('missing struct proc * argument') + + proc = kern.GetValueFromAddress(cmd_args[0], 'proc_t') + + print GetKqueueSummary.header + for kq in IterateProcKqueues(proc): + print GetKqueueSummary(kq) + +@lldb_command('showprocknotes') +def ShowProcKnotes(cmd_args=None): + """ Show the knotes for a given process. + + usage: showprocknotes + """ + + if not cmd_args: + raise ArgumentError('missing struct proc * argument') + + proc = kern.GetValueFromAddress(cmd_args[0], 'proc_t') + + print GetKnoteSummary.header + for kn in IterateProcKnotes(proc): + print GetKnoteSummary(kn) + +@lldb_command('showallkqueues') +def ShowAllKqueues(cmd_args=[], cmd_options={}): + """ Display a summary of all the kqueues in the system + + usage: showallkqueues + """ + print GetKqueueSummary.header + for kq in IterateAllKqueues(): + print GetKqueueSummary(kq) diff --git a/tools/lldbmacros/ktrace.py b/tools/lldbmacros/ktrace.py old mode 100644 new mode 100755 index 05f538e3d..ac6ddbe34 --- a/tools/lldbmacros/ktrace.py +++ b/tools/lldbmacros/ktrace.py @@ -72,7 +72,7 @@ def GetKdebugTypefilter(typefilter): if print_class: subclasses[j] = element - # if any of the bits were set in a class, print the entire class + ## if any of the bits were set in a class, print the entire class if print_class: out_str += '{:<20s}'.format(GetKdebugClassName(i)) for element in subclasses: @@ -119,7 +119,7 @@ def GetKdebugStatus(): kdebug_flags = kern.globals.kd_ctrl_page.kdebug_flags out += 'kdebug flags: {}\n'.format(xnudefines.GetStateString(xnudefines.kdebug_flags_strings, kdebug_flags)) events = kern.globals.nkdbufs - buf_mb = events * (64 if kern.arch == 'x86_64' or kern.arch == 'arm64' else 32) / 1000000 + buf_mb = events * (64 if kern.arch == 'x86_64' or kern.arch.startswith('arm64') else 32) / 1000000 out += 'events allocated: {: ledger_peak): - ledger_peak = phys_footprint_entry._le.le_peaks[0].le_max - if (now - phys_footprint_entry._le.le_peaks[1].le_time <= 1) and (phys_footprint_entry._le.le_peaks[1].le_max > ledger_peak): - ledger_peak = phys_footprint_entry._le.le_peaks[1].le_max + if (now - phys_footprint_entry._le.le_maxtracking.le_peaks[0].le_time <= 1) and (phys_footprint_entry._le.le_maxtracking.le_peaks[0].le_max > ledger_peak): + ledger_peak = phys_footprint_entry._le.le_maxtracking.le_peaks[0].le_max return ledger_peak -@header("{: >8s} {: >22s} {: >22s} {: >11s} {: >11s} {: >12s} {: >10s} {: >13s} {: ^10s} {: >8s} {: <20s}\n".format( -'pid', 'effective priority', 'requested priority', 'state', 'user_data', 'physical', 'iokit', 'footprint', -'spike', 'limit', 'command')) +@header("{: >8s} {: >12s} {: >12s} {: >10s} {: >12s} {: >14s} {: >10s} {: >12s} {: >10s} {: >10s} {: >10s} {: <20s}\n".format( +'pid', 'effective', 'requested', 'state', 'user_data', 'physical', 'iokit', 'footprint', +'spike', 'lifemax', 'limit', 'command')) def GetMemoryStatusNode(proc_val): """ Internal function to get memorystatus information from the given proc params: proc - value representing struct proc * @@ -82,24 +80,27 @@ def GetMemoryStatusNode(proc_val): task_iokit_footprint_ledger_entry = task_ledgerp.l_entries[kern.globals.task_ledgers.iokit_mapped] task_phys_footprint_ledger_entry = task_ledgerp.l_entries[kern.globals.task_ledgers.phys_footprint] page_size = kern.globals.page_size - + phys_mem_footprint = (task_physmem_footprint_ledger_entry.le_credit - task_physmem_footprint_ledger_entry.le_debit) / page_size iokit_footprint = (task_iokit_footprint_ledger_entry.le_credit - task_iokit_footprint_ledger_entry.le_debit) / page_size phys_footprint = (task_phys_footprint_ledger_entry.le_credit - task_phys_footprint_ledger_entry.le_debit) / page_size phys_footprint_limit = task_phys_footprint_ledger_entry.le_limit / page_size ledger_peak = CalculateLedgerPeak(task_phys_footprint_ledger_entry) phys_footprint_spike = ledger_peak / page_size + phys_footprint_lifetime_max = task_phys_footprint_ledger_entry._le.le_maxtracking.le_lifetime_max / page_size - format_string = '{0: >8d} {1: >22d} {2: >22d} {3: #011x} {4: #011x} {5: >12d} {6: >10d} {7: >13d}' + format_string = '{0: >8d} {1: >12d} {2: >12d} {3: #011x} {4: #011x} {5: >12d} {6: >10d} {7: >13d}' out_str += format_string.format(proc_val.p_pid, proc_val.p_memstat_effectivepriority, proc_val.p_memstat_requestedpriority, proc_val.p_memstat_state, proc_val.p_memstat_userdata, phys_mem_footprint, iokit_footprint, phys_footprint) if phys_footprint != phys_footprint_spike: - out_str += "{: ^12d}".format(phys_footprint_spike) + out_str += "{: >12d}".format(phys_footprint_spike) else: - out_str += "{: ^12s}".format('-') - out_str += "{: 8d} {: <20s}\n".format(phys_footprint_limit, proc_val.p_comm) - return out_str + out_str += "{: >12s}".format('-') + + out_str += "{: >10d} ".format(phys_footprint_lifetime_max) + out_str += "{: >10d} {: <20s}\n".format(phys_footprint_limit, proc_val.p_comm) + return out_str @lldb_command('showmemorystatus') def ShowMemoryStatus(cmd_args=None): @@ -109,8 +110,8 @@ def ShowMemoryStatus(cmd_args=None): bucket_index = 0 bucket_count = 20 print GetMemoryStatusNode.header - print "{: >91s} {: >10s} {: >13s} {: ^10s} {: >8s}\n".format("(pages)", "(pages)", "(pages)", - "(pages)", "(pages)") + print "{: >21s} {: >12s} {: >38s} {: >10s} {: >12s} {: >10s} {: >10s}\n".format("priority", "priority", "(pages)", "(pages)", "(pages)", + "(pages)", "(pages)", "(pages)") while bucket_index < bucket_count: current_bucket = kern.globals.memstat_bucket[bucket_index] current_list = current_bucket.list @@ -121,14 +122,14 @@ def ShowMemoryStatus(cmd_args=None): bucket_index += 1 print "\n\n" Memstats() - + # EndMacro: showmemorystatus def GetRealMetadata(meta): """ Get real metadata for a given metadata pointer """ try: - if unsigned(meta.zindex) != 255: + if unsigned(meta.zindex) != 0x03FF: return meta else: return kern.GetValueFromAddress(unsigned(meta) - unsigned(meta.real_metadata_offset), "struct zone_page_metadata *") @@ -194,7 +195,7 @@ def WhatIs(addr): metadata_offset = (unsigned(addr) - unsigned(zone_metadata_region_min)) % sizeof('struct zone_page_metadata') page_offset_str = "{:d}/{:d}".format((unsigned(addr) - (unsigned(addr) & ~(pagesize - 1))), pagesize) out_str += WhatIs.header + '\n' - out_str += "{:#018x} {:>18s} {:>18s} {:#018x}\n\n".format(unsigned(addr), "Metadata", page_offset_str, unsigned(addr) - metadata_offset) + out_str += "{:#018x} {:>18s} {:>18s} {:#018x}\n\n".format(unsigned(addr), "Metadata", page_offset_str, unsigned(addr) - metadata_offset) out_str += GetZoneMetadataSummary((unsigned(addr) - metadata_offset)) + '\n\n' else: page_index = ((unsigned(addr) & ~(pagesize - 1)) - unsigned(zone_map_min_address)) / pagesize @@ -238,18 +239,18 @@ def WhatIsHelper(cmd_args=None): 'ZONE', 'TOT_SZ', 'PAGE_COUNT', 'ALLOC_ELTS', 'FREE_ELTS', 'FREE_SZ', 'ALL_FREE_PGS', 'ELT_SZ', 'ALLOC', 'ELTS', 'PGS', 'WASTE', 'FLAGS', 'NAME')) def GetZoneSummary(zone): """ Summarize a zone with important information. See help zprint for description of each field - params: + params: zone: value - obj representing a zone in kernel - returns: + returns: str - summary of the zone """ out_string = "" format_string = '{:#018x} {:10d} {:10d} {:10d} {:10d} {:10d} {:10d} {:10d} {:6d} {:6d} {:6d} {markings} {name:s} ' pagesize = kern.globals.page_size - + free_elements = zone.countfree free_size = free_elements * zone.elem_size - + alloc_pages = zone.alloc_size / pagesize alloc_count = zone.alloc_size / zone.elem_size alloc_waste = zone.alloc_size % zone.elem_size @@ -271,21 +272,23 @@ def GetZoneSummary(zone): if kern.arch == 'x86_64': marks.append(["gzalloc_exempt", "M"]) marks.append(["alignment_required", "N"]) - + markings="" + if not zone.__getattr__("zone_valid") : + markings+="I" for mark in marks: if zone.__getattr__(mark[0]) : markings+=mark[1] else: markings+=" " out_string += format_string.format(zone, zone.cur_size, zone.page_count, - zone.count, free_elements, free_size, zone.count_all_free_pages, + zone.count, free_elements, free_size, zone.count_all_free_pages, zone.elem_size, zone.alloc_size, alloc_count, alloc_pages, alloc_waste, name = zone.zone_name, markings=markings) - + if zone.exhaustible : out_string += "(max: {:d})".format(zone.max_size) - + return out_string @lldb_command('zprint') @@ -308,6 +311,7 @@ def Zprint(cmd_args=None): W - another thread is waiting for more memory L - zone is being monitored by zleaks G - currently running GC + I - zone was destroyed and is no longer valid """ global kern print GetZoneSummary.header @@ -317,9 +321,9 @@ def Zprint(cmd_args=None): @xnudebug_test('test_zprint') def TestZprint(kernel_target, config, lldb_obj, isConnected ): """ Test the functionality of zprint command - returns + returns - False on failure - - True on success + - True on success """ if not isConnected: print "Target is not connected. Cannot test memstats" @@ -329,7 +333,7 @@ def TestZprint(kernel_target, config, lldb_obj, isConnected ): result = res.GetOutput() if len(result.split("\n")) > 2: return True - else: + else: return False @@ -345,7 +349,7 @@ def ShowZfreeListHeader(zone): returns: None """ - + scaled_factor = (unsigned(kern.globals.zp_factor) + (unsigned(zone.elem_size) >> unsigned(kern.globals.zp_scale))) @@ -429,7 +433,7 @@ def ShowZfreeList(cmd_args=None): zfirst = kern.GetValueFromAddress(GetFreeList(free_page_meta), 'void *') if unsigned(zfirst) != 0: ShowZfreeListChain(zone, zfirst, zlimit) - + if ShowZfreeList.elts_found == zlimit: print "Stopped at {0: When the kernel panics due to a corrupted zone element, get the @@ -597,7 +601,7 @@ def ZStackFindElem(cmd_args=None): if int(kern.globals.log_records) == 0 or unsigned(kern.globals.corruption_debug_flag) == 0: print "Zone logging with corruption detection not enabled. Add '-zc zlog=' to boot-args." return - + btlog_ptr = kern.GetValueFromAddress(cmd_args[0], 'btlog_t *') target_element = unsigned(kern.GetValueFromAddress(cmd_args[1], 'void *')) @@ -638,6 +642,43 @@ def ZStackFindElem(cmd_args=None): # EndMacro: zstack_findelem +@lldb_command('zstack_findtop', 'N:') +def ShowZstackTop(cmd_args=None, cmd_options={}): + """ Zone leak debugging: search the log and print the stacks with the most active references + in the stack trace. + + Usage: zstack_findtop [-N ] + """ + + if not cmd_args: + raise ArgumentError('Missing required btlog address argument') + + n = 5 + if '-N' in cmd_options: + n = int(cmd_options['-N']) + + btlog_ptr = kern.GetValueFromAddress(cmd_args[0], 'btlog_t *') + btrecord_size = unsigned(btlog_ptr.btrecord_size) + btrecords = unsigned(btlog_ptr.btrecords) + + cpcs_index = unsigned(btlog_ptr.head) + depth = unsigned(btlog_ptr.btrecord_btdepth) + + records = [] + while cpcs_index != 0xffffff: + cpcs_record_offset = cpcs_index * btrecord_size + cpcs_record = kern.GetValueFromAddress(btrecords + cpcs_record_offset, 'btlog_record_t *') + cpcs_record.index = cpcs_index + records.append(cpcs_record) + cpcs_index = cpcs_record.next + + recs = sorted(records, key=lambda x: x.ref_count, reverse=True) + + for rec in recs[:n]: + ShowZStackRecord(rec, rec.index, depth, unsigned(btlog_ptr.active_element_count)) + +# EndMacro: zstack_findtop + # Macro: btlog_find @lldb_command('btlog_find', "AS") @@ -722,14 +763,14 @@ def ShowZstackTrace(cmd_args=None): if len(cmd_args) >= 2: trace_size = ArgumentStringToInt(cmd_args[1]) ShowZstackTraceHelper(trace, trace_size) - + #EndMacro: showzstacktrace def ShowZstackTraceHelper(stack, depth): """ Helper routine for printing a zstack. params: stack: void *[] - An array of pointers representing the Zstack - depth: int - The depth of the ztrace stack + depth: int - The depth of the ztrace stack returns: None """ @@ -748,7 +789,7 @@ def ShowZstackTraceHelper(stack, depth): @lldb_command('showtopztrace') def ShowTopZtrace(cmd_args=None): - """ Shows the ztrace with the biggest size. + """ Shows the ztrace with the biggest size. (According to top_ztrace, not by iterating through the hash table) """ top_trace = kern.globals.top_ztrace @@ -767,7 +808,7 @@ def ShowZallocs(cmd_args=None): if unsigned(kern.globals.zallocations) == 0: print "zallocations array not initialized!" return - print '{0: <5s} {1: <18s} {2: <5s} {3: <15s}'.format('INDEX','ADDRESS','TRACE','SIZE') + print '{0: <5s} {1: <18s} {2: <5s} {3: <15s}'.format('INDEX','ADDRESS','TRACE','SIZE') current_index = 0 max_zallocation = unsigned(kern.globals.zleak_alloc_buckets) allocation_count = 0 @@ -791,7 +832,7 @@ def ShowZallocsForTrace(cmd_args=None): if not cmd_args: print ShowZallocsForTrace.__doc__ return - print '{0: <5s} {1: <18s} {2: <15s}'.format('INDEX','ADDRESS','SIZE') + print '{0: <5s} {1: <18s} {2: <15s}'.format('INDEX','ADDRESS','SIZE') target_index = ArgumentStringToInt(cmd_args[0]) current_index = 0 max_zallocation = unsigned(kern.globals.zleak_alloc_buckets) @@ -904,7 +945,7 @@ def GetBtlogBacktrace(depth, zstack_record): frame = 0 if not zstack_record: return "Zstack record none!" - + depth_val = unsigned(depth) while frame < depth_val: frame_pc = zstack_record.bt[frame] @@ -948,9 +989,9 @@ def ShowIOAllocations(cmd_args=None): print "Container allocation = {0: <#0x} = {1: d}K".format(kern.globals.debug_container_malloc_size, (kern.globals.debug_container_malloc_size / 1024)) print "IOMalloc allocation = {0: <#0x} = {1: d}K".format(kern.globals.debug_iomalloc_size, (kern.globals.debug_iomalloc_size / 1024)) print "Container allocation = {0: <#0x} = {1: d}K".format(kern.globals.debug_iomallocpageable_size, (kern.globals.debug_iomallocpageable_size / 1024)) - - -# EndMacro: showioalloc + + +# EndMacro: showioalloc # Macro: showselectmem @@ -996,8 +1037,8 @@ def ShowSelectMem(cmd_args=None, cmd_options={}): print '-'*40 print "Total: {:d} bytes ({:d} kbytes)".format(selmem, selmem/1024) # Endmacro: showselectmem - - + + # Macro: showtaskvme @lldb_command('showtaskvme', "PS") def ShowTaskVmeHelper(cmd_args=None, cmd_options={}): @@ -1112,11 +1153,11 @@ def ShowAllVMStats(cmd_args=None): vmstats.error += '*' print entry_format.format(p=proc, m=vmmap, vsize=(unsigned(vmmap.size) / page_size), t=task, s=vmstats) - + def ShowTaskVMEntries(task, show_pager_info, show_all_shadows): """ Routine to print out a summary listing of all the entries in a vm_map - params: + params: task - core.value : a object of type 'task *' returns: None @@ -1190,7 +1231,7 @@ def GetVMEntrySummary(vme): vme_protection = int(vme.protection) vme_max_protection = int(vme.max_protection) vme_extra_info_str ="SC-Ds"[int(vme.inheritance)] - if int(vme.is_sub_map) != 0 : + if int(vme.is_sub_map) != 0 : vme_extra_info_str +="s" elif int(vme.needs_copy) != 0 : vme_extra_info_str +="n" @@ -1212,7 +1253,7 @@ def ShowMapWired(cmd_args=None): @lldb_type_summary(['kmod_info_t *']) @header("{0: <20s} {1: <20s} {2: <20s} {3: >3s} {4: >5s} {5: <20s} {6: <20s} {7: >20s} {8: <30s}".format('kmod_info', 'address', 'size', 'id', 'refs', 'TEXT exec', 'size', 'version', 'name')) def GetKextSummary(kmod): - """ returns a string representation of kext information + """ returns a string representation of kext information """ out_string = "" format_string = "{0: <#020x} {1: <#020x} {2: <#020x} {3: >3d} {4: >5d} {5: <#020x} {6: <#020x} {7: >20s} {8: <30s}" @@ -1224,7 +1265,7 @@ def GetKextSummary(kmod): return out_string @lldb_type_summary(['uuid_t']) -@header("") +@header("") def GetUUIDSummary(uuid): """ returns a string representation like CA50DA4C-CA10-3246-B8DC-93542489AA26 """ @@ -1281,9 +1322,9 @@ def GetAllSegmentsAndSectionsFromDataInMemory(address, size): if address == 0 or size == 0: return ([defval], [defval]) - # if int(kern.globals.gLoadedKextSummaries.version) <= 2: + ## if int(kern.globals.gLoadedKextSummaries.version) <= 2: # until we have separate version. we will pay penalty only on arm64 devices - if kern.arch not in ('arm64',): + if not kern.arch.startswith('arm64'): return ([defval], [defval]) restrict_size_to_read = 1536 @@ -1324,7 +1365,7 @@ def GetKextLoadInformation(addr=0, show_progress=False): 'addr of macho header', [macho.MachOSegment,..], [MachoSection,...], kext, kmod_obj) """ cached_result = caching.GetDynamicCacheData("kern.kexts.loadinformation", []) - # if specific addr is provided then ignore caching + ## if specific addr is provided then ignore caching if cached_result and not addr: return cached_result @@ -1369,28 +1410,28 @@ def GetOSKextVersion(version_num): return "invalid" (MAJ_MULT, MIN_MULT, REV_MULT,STAGE_MULT) = (100000000, 1000000, 10000, 1000) version = version_num - + vers_major = version / MAJ_MULT version = version - (vers_major * MAJ_MULT) - + vers_minor = version / MIN_MULT version = version - (vers_minor * MIN_MULT) - + vers_revision = version / REV_MULT version = version - (vers_revision * REV_MULT) - + vers_stage = version / STAGE_MULT version = version - (vers_stage * STAGE_MULT) - - vers_stage_level = version - + + vers_stage_level = version + out_str = "%d.%d" % (vers_major, vers_minor) if vers_revision > 0: out_str += ".%d" % vers_revision if vers_stage == 1 : out_str += "d%d" % vers_stage_level if vers_stage == 3 : out_str += "a%d" % vers_stage_level if vers_stage == 5 : out_str += "b%d" % vers_stage_level if vers_stage == 6 : out_str += "fc%d" % vers_stage_level - + return out_str @lldb_command('showallknownkmods') @@ -1422,6 +1463,47 @@ def ShowAllKnownKexts(cmd_args=None): return +def FindKmodNameForAddr(addr): + """ Given an address, return the name of the kext containing that address + """ + addr = unsigned(addr) + all_kexts_info = GetKextLoadInformation() + for kinfo in all_kexts_info: + segment = macho.get_segment_with_addr(kinfo[4], addr) + if segment: + return kinfo[7].name + return None + + +@lldb_command('addkextaddr') +def AddKextAddr(cmd_args=[]): + """ Given an address, load the kext which contains that address + Syntax: (lldb) addkextaddr + """ + if len(cmd_args) < 1: + raise ArgumentError("Insufficient arguments") + + addr = ArgumentStringToInt(cmd_args[0]) + all_kexts_info = GetKextLoadInformation() + found_kinfo = None + found_segment = None + for kinfo in all_kexts_info: + segment = macho.get_segment_with_addr(kinfo[4], addr) + if segment: + print GetKextSummary.header + print GetKextSummary(kinfo[7]) + " segment: {} offset = {:#0x}".format(segment.name, (addr - segment.vmaddr)) + cur_uuid = kinfo[0].lower() + print "Fetching dSYM for %s" % cur_uuid + info = dsymForUUID(cur_uuid) + if info and 'DBGSymbolRichExecutable' in info: + print "Adding dSYM (%s) for %s" % (cur_uuid, info['DBGSymbolRichExecutable']) + addDSYM(cur_uuid, info) + loadDSYM(cur_uuid, int(kinfo[1],16), kinfo[4]) + else: + print "Failed to get symbol info for %s" % cur_uuid + return + + @lldb_command('showkmodaddr') def ShowKmodAddr(cmd_args=[]): """ Given an address, print the offset and name for the kmod containing it @@ -1472,6 +1554,7 @@ def AddKextSyms(cmd_args=[], cmd_options={}): \nNote: LLDB does not support adding kext based on directory paths like gdb used to.".format(exec_path)) slide_value = None + sections = None if cmd_args: slide_value = cmd_args[0] debuglog("loading slide value from user input %s" % cmd_args[0]) @@ -1489,12 +1572,24 @@ def AddKextSyms(cmd_args=[], cmd_options={}): debuglog(k[0]) if k[0].lower() == uuid_str.lower(): slide_value = k[1] + sections = k[4] debuglog("found the slide %s for uuid %s" % (k[1], k[0])) if slide_value is None: raise ArgumentError("Unable to find load address for module described at %s " % exec_full_path) - load_cmd = "target modules load --file %s --slide %s" % (exec_full_path, str(slide_value)) - print load_cmd - print lldb_run_command(load_cmd) + + if not sections: + cmd_str = "target modules load --file %s --slide %s" % ( exec_full_path, str(slide_value)) + debuglog(cmd_str) + else: + cmd_str = "target modules load --file {} ".format(exec_full_path) + sections_str = "" + for s in sections: + sections_str += " {} {:#0x} ".format(s.name, s.vmaddr) + cmd_str += sections_str + debuglog(cmd_str) + + lldb.debugger.HandleCommand(cmd_str) + kern.symbolicator = None return True @@ -1561,7 +1656,7 @@ lldb_alias('showkextaddr', 'showkmodaddr') @lldb_type_summary(['mount *']) @header("{0: <20s} {1: <20s} {2: <20s} {3: <12s} {4: <12s} {5: <12s} {6: >6s} {7: <30s} {8: <35s} {9: <30s}".format('volume(mp)', 'mnt_data', 'mnt_devvp', 'flag', 'kern_flag', 'lflag', 'type', 'mnton', 'mntfrom', 'iosched supported')) def GetMountSummary(mount): - """ Display a summary of mount on the system + """ Display a summary of mount on the system """ out_string = ("{mnt: <#020x} {mnt.mnt_data: <#020x} {mnt.mnt_devvp: <#020x} {mnt.mnt_flag: <#012x} " + "{mnt.mnt_kern_flag: <#012x} {mnt.mnt_lflag: <#012x} {vfs.f_fstypename: >6s} " + @@ -1628,7 +1723,7 @@ def _GetVnodePathName(vnode, vnodename): _GetVnodePathName(vnode.v_mount.mnt_vnodecovered, str(vnode.v_mount.mnt_vnodecovered.v_name) ) else: _GetVnodePathName(vnode.v_parent, str(vnode.v_parent.v_name)) - _GetVnodePathName.output += "/%s" % vnodename + _GetVnodePathName.output += "/%s" % vnodename def GetVnodePath(vnode): """ Get string representation of the vnode @@ -1676,7 +1771,7 @@ def GetVnodeDevInfo(vnode): devnode_major = (devnode_dev >> 24) & 0xff devnode_minor = devnode_dev & 0x00ffffff - # boilerplate device information for a vnode + # boilerplate device information for a vnode vnodedev_output += "Device Info:\n\t vnode:\t\t{:#x}".format(vnode) vnodedev_output += "\n\t type:\t\t" if (vnode.v_type == vblk_type): @@ -1772,7 +1867,7 @@ def GetVnodeLock(lockf): vnode_lock_output += ("PID {: <18d}").format(lockf_proc.p_pid) else: vnode_lock_output += ("ID {: <#019x}").format(int(lockf.lf_id)) - + # lock type if lockf_type == 1: vnode_lock_output += ("{: <12s}").format('shared') @@ -1784,7 +1879,7 @@ def GetVnodeLock(lockf): vnode_lock_output += ("{: <12s}").format('unlock') else: vnode_lock_output += ("{: <12s}").format('unknown') - + # start and stop values vnode_lock_output += ("{: #018x} ..").format(lockf.lf_start) vnode_lock_output += ("{: #018x}\n").format(lockf.lf_end) @@ -1806,7 +1901,7 @@ def GetVnodeLocksSummary(vnode): while lockf_blocker: out_str += ("{: <4s}").format('>') out_str += GetVnodeLock(lockf_blocker) - lockf_blocker = lockf_blocker.lf_block.tqe_next + lockf_blocker = lockf_blocker.lf_block.tqe_next return out_str @lldb_command('showvnodelocks') @@ -1828,7 +1923,7 @@ def ShowVnodeLocks(cmd_args=None): # EndMacro: showvnodelocks # Macro: showproclocks - + @lldb_command('showproclocks') def ShowProcLocks(cmd_args=None): """ Routine to display list of advisory record locks for the given process @@ -1900,7 +1995,7 @@ def GetVnodeSummary(vnode): csblob_version = '-' if (vtype == 1) and (vnode.v_un.vu_ubcinfo != 0): csblob_version = '{: <6d}'.format(vnode.v_un.vu_ubcinfo.cs_add_gen) - # Check to see if vnode is mapped/unmapped + # Check to see if vnode is mapped/unmapped if (vnode.v_un.vu_ubcinfo.ui_flags & 0x8) != 0: mapped = '1' else: @@ -1930,7 +2025,7 @@ def ShowVnode(cmd_args=None): vnodeval = kern.GetValueFromAddress(cmd_args[0],'vnode *') print GetVnodeSummary.header print GetVnodeSummary(vnodeval) - + @lldb_command('showvolvnodes') def ShowVolVnodes(cmd_args=None): """ Display info about all vnodes of a given mount_t @@ -2022,7 +2117,7 @@ def ShowProcVnodes(cmd_args=None): if int(fdptr.fd_rdir) != 0: print '{0: <25s}\n{1: llb fails to cast addresses to double pointers fpptr = Cast(fdptr.fd_ofiles, 'fileproc *') while count < fdptr.fd_nfiles: @@ -2055,9 +2150,9 @@ def ShowAllProcVnodes(cmd_args=None): @xnudebug_test('test_vnode') def TestShowAllVnodes(kernel_target, config, lldb_obj, isConnected ): """ Test the functionality of vnode related commands - returns + returns - False on failure - - True on success + - True on success """ if not isConnected: print "Target is not connected. Cannot test memstats" @@ -2067,7 +2162,7 @@ def TestShowAllVnodes(kernel_target, config, lldb_obj, isConnected ): result = res.GetOutput() if len(result.split("\n")) > 2 and result.find('VREG') != -1 and len(result.splitlines()[2].split()) > 5: return True - else: + else: return False # Macro: showallmtx @@ -2101,13 +2196,13 @@ def ShowAllMtx(cmd_args=None): hdr_format = '{:<18s} {:>10s} {:>10s} {:>10s} {:>10s} {:<30s} ' else: hdr_format = '{:<10s} {:>10s} {:>10s} {:>10s} {:>10s} {:<30s} ' - - print hdr_format.format('LCK GROUP', 'CNT', 'UTIL', 'MISS', 'WAIT', 'NAME') + + print hdr_format.format('LCK GROUP', 'CNT', 'UTIL', 'MISS', 'WAIT', 'NAME') mtxgrp_queue_head = kern.globals.lck_grp_queue - mtxgrp_ptr_type = GetType('_lck_grp_ *') - - for mtxgrp_ptr in IterateQueue(mtxgrp_queue_head, mtxgrp_ptr_type, "lck_grp_link"): + mtxgrp_ptr_type = GetType('_lck_grp_ *') + + for mtxgrp_ptr in IterateQueue(mtxgrp_queue_head, mtxgrp_ptr_type, "lck_grp_link"): print GetMutexEntry(mtxgrp_ptr) return # EndMacro: showallmtx @@ -2224,14 +2319,14 @@ def ShowLock(cmd_args=None, cmd_options={}): return summary_str = "" - lock = kern.GetValueFromAddress(cmd_args[0], 'uintptr_t*') + lock = kern.GetValueFromAddress(cmd_args[0], 'uintptr_t *') if kern.arch == "x86_64" and lock: if "-M" in cmd_options: - lock_mtx = Cast(lock, 'lck_mtx_t *') + lock_mtx = kern.GetValueFromAddress(lock, 'lck_mtx_t *') summary_str = GetMutexLockSummary(lock_mtx) elif "-S" in cmd_options: - lock_spin = Cast(lock, 'lck_spin_t *') + lock_spin = kern.GetValueFromAddress(lock, 'lck_spin_t *') summary_str = GetSpinLockSummary(lock_spin) else: summary_str = "Please specify supported lock option(-M/-S)" @@ -2278,12 +2373,10 @@ def ShowBooterMemoryMap(cmd_args=None): """ Prints out the phys memory map from kernelBootArgs Supported only on x86_64 """ - if kern.arch == 'x86_64': - voffset = unsigned(0xFFFFFF8000000000) - else: + if kern.arch != 'x86_64': print "showbootermemorymap not supported on this architecture" return - + out_string = "" # Memory type map @@ -2312,7 +2405,7 @@ def ShowBooterMemoryMap(cmd_args=None): i = 0 while i < mcount: - mptr = kern.GetValueFromAddress(unsigned(boot_args.MemoryMap) + voffset + unsigned(i*msize), 'EfiMemoryRange *') + mptr = kern.GetValueFromAddress(unsigned(boot_args.MemoryMap) + kern.VM_MIN_KERNEL_ADDRESS + unsigned(i*msize), 'EfiMemoryRange *') mtype = unsigned(mptr.Type) if mtype in memtype_dict: out_string += "{0: <12s}".format(memtype_dict[mtype]) @@ -2479,9 +2572,9 @@ def ShowPurgeableVolatileVmObject(object, idx, volatile_total): returns: None """ -# if int(object.vo_un2.vou_purgeable_owner) != int(object.vo_purgeable_volatilizer): +## if int(object.vo_un2.vou_purgeable_owner) != int(object.vo_purgeable_volatilizer): # diff=" !=" -# else: +## else: # diff=" " page_size = kern.globals.page_size if object.purgable == 0: @@ -2519,29 +2612,31 @@ def GetCompressedPagesForObject(obj): """ pager = Cast(obj.pager, 'compressor_pager_t') return pager.cpgr_num_slots_occupied -# if pager.cpgr_num_slots > 128: -# slots_arr = pager.cpgr_slots.cpgr_islots -# num_indirect_slot_ptr = (pager.cpgr_num_slots + 127) / 128 -# index = 0 -# compressor_slot = 0 -# compressed_pages = 0 -# while index < num_indirect_slot_ptr: -# compressor_slot = 0 -# if slots_arr[index]: -# while compressor_slot < 128: -# if slots_arr[index][compressor_slot]: -# compressed_pages += 1 -# compressor_slot += 1 -# index += 1 -# else: -# slots_arr = pager.cpgr_slots.cpgr_dslots -# compressor_slot = 0 -# compressed_pages = 0 -# while compressor_slot < pager.cpgr_num_slots: -# if slots_arr[compressor_slot]: -# compressed_pages += 1 -# compressor_slot += 1 -# return compressed_pages + """ # commented code below + if pager.cpgr_num_slots > 128: + slots_arr = pager.cpgr_slots.cpgr_islots + num_indirect_slot_ptr = (pager.cpgr_num_slots + 127) / 128 + index = 0 + compressor_slot = 0 + compressed_pages = 0 + while index < num_indirect_slot_ptr: + compressor_slot = 0 + if slots_arr[index]: + while compressor_slot < 128: + if slots_arr[index][compressor_slot]: + compressed_pages += 1 + compressor_slot += 1 + index += 1 + else: + slots_arr = pager.cpgr_slots.cpgr_dslots + compressor_slot = 0 + compressed_pages = 0 + while compressor_slot < pager.cpgr_num_slots: + if slots_arr[compressor_slot]: + compressed_pages += 1 + compressor_slot += 1 + return compressed_pages + """ def ShowTaskVMEntries(task, show_pager_info, show_all_shadows): """ Routine to print out a summary listing of all the entries in a vm_map @@ -2644,7 +2739,7 @@ def showmapvme(map, show_pager_info, show_all_shadows): tagstr = "" if map.pmap == kern.globals.kernel_pmap: xsite = Cast(kern.globals.vm_allocation_sites[tag],'OSKextAccount *') - if xsite and xsite.site.flags & 2: + if xsite and xsite.site.flags & 0x0200: tagstr = ".{:<3d}".format(xsite.loadTag) print "{:#018x} {:#018x}:{:#018x} {:>10d} {:>3d}{:<4s} {:1d}{:1d}{:<8s} {:<18s} {:<#18x}".format(vme,vme.links.start,vme.links.end,(unsigned(vme.links.end)-unsigned(vme.links.start))/page_size,tag,tagstr,vme.protection,vme.max_protection,vme_flags,object_str,offset) if (show_pager_info or show_all_shadows) and vme.is_sub_map == 0 and vme.vme_object.vmo_object != 0: @@ -2744,57 +2839,40 @@ def GetKmodIDName(kmod_id): return "{:<50s}".format(kmod.name) return "??" -def GetVMKernName(tag): - if 1 == tag: - return "VM_KERN_MEMORY_OSFMK" - elif 2 == tag: - return "VM_KERN_MEMORY_BSD" - elif 3 == tag: - return "VM_KERN_MEMORY_IOKIT" - elif 4 == tag: - return "VM_KERN_MEMORY_LIBKERN" - elif 5 == tag: - return "VM_KERN_MEMORY_OSKEXT" - elif 6 == tag: - return "VM_KERN_MEMORY_KEXT" - elif 7 == tag: - return "VM_KERN_MEMORY_IPC" - elif 8 == tag: - return "VM_KERN_MEMORY_STACK" - elif 9 == tag: - return "VM_KERN_MEMORY_CPU" - elif 10 == tag: - return "VM_KERN_MEMORY_PMAP" - elif 11 == tag: - return "VM_KERN_MEMORY_PTE" - elif 12 == tag: - return "VM_KERN_MEMORY_ZONE" - elif 13 == tag: - return "VM_KERN_MEMORY_KALLOC" - elif 14 == tag: - return "VM_KERN_MEMORY_COMPRESSOR" - elif 15 == tag: - return "VM_KERN_MEMORY_COMPRESSED_DATA" - elif 16 == tag: - return "VM_KERN_MEMORY_PHANTOM_CACHE" - elif 17 == tag: - return "VM_KERN_MEMORY_WAITQ" - elif 18 == tag: - return "VM_KERN_MEMORY_DIAG" - elif 19 == tag: - return "VM_KERN_MEMORY_LOG" - elif 20 == tag: - return "VM_KERN_MEMORY_FILE" - elif 21 == tag: - return "VM_KERN_MEMORY_MBUF" - elif 22 == tag: - return "VM_KERN_MEMORY_UBC" - elif 23 == tag: - return "VM_KERN_MEMORY_SECURITY" - elif 24 == tag: - return "VM_KERN_MEMORY_MLOCK" - return "??" +FixedTags = { + 0: "VM_KERN_MEMORY_NONE", + 1: "VM_KERN_MEMORY_OSFMK", + 2: "VM_KERN_MEMORY_BSD", + 3: "VM_KERN_MEMORY_IOKIT", + 4: "VM_KERN_MEMORY_LIBKERN", + 5: "VM_KERN_MEMORY_OSKEXT", + 6: "VM_KERN_MEMORY_KEXT", + 7: "VM_KERN_MEMORY_IPC", + 8: "VM_KERN_MEMORY_STACK", + 9: "VM_KERN_MEMORY_CPU", + 10: "VM_KERN_MEMORY_PMAP", + 11: "VM_KERN_MEMORY_PTE", + 12: "VM_KERN_MEMORY_ZONE", + 13: "VM_KERN_MEMORY_KALLOC", + 14: "VM_KERN_MEMORY_COMPRESSOR", + 15: "VM_KERN_MEMORY_COMPRESSED_DATA", + 16: "VM_KERN_MEMORY_PHANTOM_CACHE", + 17: "VM_KERN_MEMORY_WAITQ", + 18: "VM_KERN_MEMORY_DIAG", + 19: "VM_KERN_MEMORY_LOG", + 20: "VM_KERN_MEMORY_FILE", + 21: "VM_KERN_MEMORY_MBUF", + 22: "VM_KERN_MEMORY_UBC", + 23: "VM_KERN_MEMORY_SECURITY", + 24: "VM_KERN_MEMORY_MLOCK", + 25: "VM_KERN_MEMORY_REASON", + 26: "VM_KERN_MEMORY_SKYWALK", + 27: "VM_KERN_MEMORY_LTABLE", + 255:"VM_KERN_MEMORY_ANY", +} +def GetVMKernName(tag): + return FixedTags[tag] @lldb_command("showvmtags", "S") def showvmtags(cmd_args=None, cmd_options={}): @@ -2809,50 +2887,64 @@ def showvmtags(cmd_args=None, cmd_options={}): slow = True page_size = unsigned(kern.globals.page_size) tagcounts = [] + tagpeaks = [] for tag in range(256): tagcounts.append(0) + for tag in range(256): + tagpeaks.append(0) + + if kern.globals.vm_tag_active_update: + for tag in range(256): + site = kern.globals.vm_allocation_sites[tag] + if site: + tagcounts[unsigned(tag)] = unsigned(site.total) + tagpeaks[unsigned(tag)] = unsigned(site.peak) + else: + queue_head = kern.globals.vm_objects_wired + for object in IterateQueue(queue_head, 'struct vm_object *', 'objq'): + if object != kern.globals.kernel_object: + CountWiredObject(object, tagcounts) - queue_head = kern.globals.vm_objects_wired - for object in IterateQueue(queue_head, 'struct vm_object *', 'objq'): - if object != kern.globals.kernel_object: + queue_head = kern.globals.purgeable_nonvolatile_queue + for object in IterateQueue(queue_head, 'struct vm_object *', 'objq'): CountWiredObject(object, tagcounts) - queue_head = kern.globals.purgeable_nonvolatile_queue - for object in IterateQueue(queue_head, 'struct vm_object *', 'objq'): - CountWiredObject(object, tagcounts) - - purgeable_queues = kern.globals.purgeable_queues - CountWiredPurgeableQueue(purgeable_queues[0], tagcounts) - CountWiredPurgeableQueue(purgeable_queues[1], tagcounts) - CountWiredPurgeableQueue(purgeable_queues[2], tagcounts) + purgeable_queues = kern.globals.purgeable_queues + CountWiredPurgeableQueue(purgeable_queues[0], tagcounts) + CountWiredPurgeableQueue(purgeable_queues[1], tagcounts) + CountWiredPurgeableQueue(purgeable_queues[2], tagcounts) - CountMapTags(kern.globals.kernel_map, tagcounts, slow) + CountMapTags(kern.globals.kernel_map, tagcounts, slow) total = 0 - print " {:<8s} {:>7s} {:<50s}".format("tag.kmod","size","name") + print " {:<7s} {:>7s} {:>7s} {:<50s}".format("tag.kmod","peak","size","name") for tag in range(256): if tagcounts[tag]: total += tagcounts[tag] tagstr = "" sitestr = "" - if (tag <= 24): + if ((tag <= 27) or (tag == 255)): sitestr = GetVMKernName(tag) else: site = kern.globals.vm_allocation_sites[tag] if site: - if site.flags & 2: - xsite = Cast(site,'OSKextAccount *') - tagstr = ".{:<3d}".format(xsite.loadTag) - sitestr = GetKmodIDName(xsite.loadTag) + if site.flags & 0x007F: + cstr = addressof(site.subtotals[site.subtotalscount]) + sitestr = "{:<50s}".format(str(Cast(cstr, 'char *'))) else: - sitestr = kern.Symbolicate(site) - print " {:>3d}{:<4s} {:>7d}K {:<50s}".format(tag,tagstr,tagcounts[tag]*page_size / 1024,sitestr) - print "Total: {:>7d}K".format(total*page_size / 1024) + if site.flags & 0x0200: + xsite = Cast(site,'OSKextAccount *') + tagstr = ".{:<3d}".format(xsite.loadTag) + sitestr = GetKmodIDName(xsite.loadTag) + else: + sitestr = kern.Symbolicate(site) + print " {:>3d}{:<4s} {:>7d}K {:>7d}K {:<50s}".format(tag,tagstr,tagpeaks[tag] / 1024, tagcounts[tag] / 1024,sitestr) + print "Total: {:>7d}K".format(total / 1024) return None def FindVMEntriesForVnode(task, vn): - """ returns an array of vme that have the vnode set to defined vnode + """ returns an array of vme that have the vnode set to defined vnode each entry in array is of format (vme, start_addr, end_address, protection) """ retval = [] @@ -2878,7 +2970,7 @@ def FindVMEntriesForVnode(task, vn): pass else: vn_pager = Cast(obj.pager, 'vnode_pager *') - if unsigned(vn_pager.pager_ops) == pager_ops_addr and unsigned(vn_pager.vnode_handle) == unsigned(vn): + if unsigned(vn_pager.vn_pgr_hdr.mo_pager_ops) == pager_ops_addr and unsigned(vn_pager.vnode_handle) == unsigned(vn): retval.append((vme, unsigned(vme.links.start), unsigned(vme.links.end), unsigned(vme.protection))) obj = obj.shadow return retval @@ -3107,7 +3199,7 @@ def VMObjectWalkPages(cmd_args=None, cmd_options={}): out_string += second_bitfield_format_string.format(vmp.busy, vmp.wanted, vmp.tabled, vmp.hashed, vmp.fictitious, vmp.clustered, vmp.pmapped, vmp.xpmapped, vmp.wpmapped, vmp.free_when_done, vmp.absent, vmp.error, vmp.dirty, vmp.cleaning, vmp.precious, vmp.overwriting, - vmp.restart, vmp.unusual, vmp.encrypted, vmp.encrypted_cleaning, + vmp.restart, vmp.unusual, 0, 0, vmp.cs_validated, vmp.cs_tainted, vmp.cs_nx, vmp.reusable, vmp.lopage, vmp.slid, vmp.written_by_kernel) @@ -3229,9 +3321,6 @@ def ShowJetsamSnapshot(cmd_args=None, cmd_options={}): # Not shown are uuid, user_data, cpu_time global kern - if kern.arch == 'x86_64': - print "Snapshots are not supported.\n" - return show_footprint_details = False show_all_entries = False diff --git a/tools/lldbmacros/misc.py b/tools/lldbmacros/misc.py old mode 100644 new mode 100755 index a9b7bafda..88b1d7673 --- a/tools/lldbmacros/misc.py +++ b/tools/lldbmacros/misc.py @@ -5,6 +5,8 @@ Miscellaneous (Intel) platform-specific commands. from xnu import * import xnudefines +from scheduler import * + @lldb_command('showmcastate') def showMCAstate(cmd_args=None): """ @@ -62,14 +64,17 @@ def dumpTimerList(anchor): while entry != addressof(anchor): timer_call = Cast(entry, 'timer_call_t') call_entry = Cast(entry, 'struct call_entry *') - debugger_entry = kern.globals.debugger_entry_time - if (debugger_entry < call_entry.deadline): + recent_timestamp = GetRecentTimestamp() + if (recent_timestamp < call_entry.deadline): delta_sign = ' ' - timer_fire = call_entry.deadline - debugger_entry + timer_fire = call_entry.deadline - recent_timestamp else: delta_sign = '-' - timer_fire = debugger_entry - call_entry.deadline - tval = ' {:#018x}: {:16d} {:16d} {:s}{:3d}.{:09d} ({:#018x})({:#018x},{:#018x})' + timer_fire = recent_timestamp - call_entry.deadline + + func_name = kern.Symbolicate(call_entry.func) + + tval = ' {:#018x}: {:16d} {:16d} {:s}{:3d}.{:09d} ({:#018x})({:#018x},{:#018x}) ({:s})' print tval.format(entry, call_entry.deadline, timer_call.soft_deadline, @@ -78,17 +83,30 @@ def dumpTimerList(anchor): timer_fire%1000000000, call_entry.func, call_entry.param0, - call_entry.param1) + call_entry.param1, + func_name) entry = entry.next +def GetCpuDataForCpuID(cpu_id): + """ + Find struct cpu_data for a CPU + ARM is complicated + """ + if kern.arch == 'x86_64': + cpu_data = kern.globals.cpu_data_ptr[cpu_id] + return cpu_data + elif kern.arch in ['arm', 'arm64'] : + data_entries_addr = kern.GetLoadAddressForSymbol('CpuDataEntries') + data_entries = kern.GetValueFromAddress(data_entries_addr, 'cpu_data_entry_t *') + data_entry = data_entries[cpu_id]; + cpu_data_addr = data_entry.cpu_data_vaddr + return Cast(cpu_data_addr, 'cpu_data_t*') + @lldb_command('longtermtimers') def longtermTimers(cmd_args=None): """ Print details of long-term timers and stats. """ - if kern.arch != 'x86_64': - print "Not available for current architecture." - return lt = kern.globals.timer_longterm ltt = lt.threshold @@ -139,7 +157,8 @@ def processorTimers(cmd_args=None): p = kern.globals.processor_list while p: cpu = p.cpu_id - rt_timer = kern.globals.cpu_data_ptr[cpu].rtclock_timer + cpu_data = GetCpuDataForCpuID(cpu) + rt_timer = cpu_data.rtclock_timer diff = p.last_dispatch - rt_timer.deadline tmr = 'Processor {:d}: {:#018x} {:#018x} {:#018x} {:#018x} {:s}' print tmr.format(cpu, @@ -150,7 +169,7 @@ def processorTimers(cmd_args=None): ['probably BAD', '(ok)'][int(diff < 0)]) if kern.arch == 'x86_64': print 'Next deadline set at: {:#018x}. Timer call list:'.format(rt_timer.when_set) - dumpTimerList(rt_timer.queue) + dumpTimerList(rt_timer.queue) p = p.processor_list longtermTimers() @@ -178,8 +197,8 @@ def showTimerWakeupStats(cmd_args=None): tot_wakes = 0 #task.task_interrupt_wakeups tot_platform_wakes = 0 #task.task_platform_idle_wakeups for thread in IterateQueue(task.threads, 'thread_t', 'task_threads'): -# if thread.thread_interrupt_wakeups == 0: -# continue +## if thread.thread_interrupt_wakeups == 0: +## continue print '\tThread ID 0x{:x}, Tag 0x{:x}, timer wakeups: {:d} {:d} {:d} {:d} <2ms: {:d}, <5ms: {:d} UT: {:d} ST: {:d}'.format( thread.thread_id, thread.thread_tag, @@ -394,7 +413,7 @@ def GetKernelDebugBufferEntry(kdbg_entry): kdebug_arg3 = kdebug_entry.arg3 kdebug_arg4 = kdebug_entry.arg4 - if kern.arch in ('x86_64', 'arm64'): + if kern.arch == 'x86_64' or kern.arch.startswith('arm64'): kdebug_cpu = kdebug_entry.cpuid ts_hi = (kdebug_entry.timestamp >> 32) & 0xFFFFFFFF ts_lo = kdebug_entry.timestamp & 0xFFFFFFFF @@ -648,3 +667,376 @@ def ShowKernelDebugBuffer(cmd_args=None): cpu_num += 1 else: print "Trace buffer not enabled\n" + +@lldb_command('dumprawtracefile','U:') +def DumpRawTraceFile(cmd_args=[], cmd_options={}): + """ + support for ktrace(1) + + NB: trace is not wordsize flexible, so use ktrace(1) compiled for the compatible model, + e.g. if you dump from __LP64__ system, you will need to run ktrace(1) compiled __LP64__ to process the raw data file. + + read the kernel's debug trace buffer, and dump to a "raw" ktrace(1) file + Usage: dumprawtracefile + -U : specify system uptime in nsec, obtained e.g. from paniclog + Be patient, it is teh slow. + + cf. kdbg_read()\bsd/kern/kdebug.c + """ + + # Check if KDBG_BFINIT (0x80000000) is set in kdebug_flags + if (kern.globals.kd_ctrl_page.kdebug_flags & xnudefines.KDBG_BFINIT) == 0 : + print "Trace buffer not enabled\n" + return + + if ((kern.arch == "x86_64") or (kern.arch == "arm64")) : + lp64 = True + elif kern.arch == "arm" : + lp64 = False + else : + print "unknown kern.arch {:s}\n".format(kern.arch) + return + + # Various kern.globals are hashed by address, to + # a) avoid redundant kdp fetch from, and + # b) avoid all stores to + # the target system kernel structures. + # Stores to hashed structures remain strictly local to the lldb host, + # they are never written back to the target. + htab = {} + + if lp64 : + KDBG_TIMESTAMP_MASK = 0xffffffffffffffff + else : + KDBG_TIMESTAMP_MASK = 0x00ffffffffffffff + KDBG_CPU_SHIFT = 56 + + barrier_min = 0 + barrier_max = 0 + out_of_events = False + lostevents = False + lostevent_timestamp = 0 + lostevent_debugid = (((xnudefines.DBG_TRACE & 0xff) << 24) | ((xnudefines.DBG_TRACE_INFO & 0xff) << 16) | ((2 & 0x3fff) << 2)) # 0x01020008 + events_count_lost = 0 + events_count_found = 0 + + opt_verbose = config['verbosity'] + opt_progress = (opt_verbose > vHUMAN) and (opt_verbose < vDETAIL) + progress_count = 0 + progress_stride = 32 + + output_filename = str(cmd_args[0]) + if opt_verbose > vHUMAN : + print "output file : {:s}".format(output_filename) + wfd = open(output_filename, "wb") + + uptime = long(-1) + if "-U" in cmd_options: + uptime = long(cmd_options["-U"]) + if opt_verbose > vHUMAN : + print "uptime : {:d}".format(uptime) + + nkdbufs = kern.globals.nkdbufs + + kd_ctrl_page = kern.globals.kd_ctrl_page + if not kd_ctrl_page in htab : + htab[kd_ctrl_page] = kern.globals.kd_ctrl_page + + if opt_verbose > vHUMAN : + print "nkdbufs {0:#x}, enabled {1:#x}, flags {2:#x}, cpus {3:#x}".format(nkdbufs, htab[kd_ctrl_page].enabled, htab[kd_ctrl_page].kdebug_flags, htab[kd_ctrl_page].kdebug_cpus) + + if nkdbufs == 0 : + print "0 nkdbufs, nothing extracted" + return + + if htab[kd_ctrl_page].enabled != 0 : + barrier_max = uptime & KDBG_TIMESTAMP_MASK + + f = htab[kd_ctrl_page].kdebug_flags + wrapped = f & xnudefines.KDBG_WRAPPED + if wrapped != 0 : + barrier_min = htab[kd_ctrl_page].oldest_time + htab[kd_ctrl_page].kdebug_flags = htab[kd_ctrl_page].kdebug_flags & ~xnudefines.KDBG_WRAPPED + htab[kd_ctrl_page].oldest_time = 0 + + for cpu in range(htab[kd_ctrl_page].kdebug_cpus) : + kdbp = unsigned(addressof(kern.globals.kdbip[cpu])) + if not kdbp in htab : + htab[kdbp] = kern.globals.kdbip[cpu] + + kdsp = htab[kdbp].kd_list_head.raw + if kdsp == xnudefines.KDS_PTR_NULL : + continue + + ix = htab[kdbp].kd_list_head.buffer_index + off = htab[kdbp].kd_list_head.offset + kdsp_actual = unsigned(addressof(kern.globals.kd_bufs[ix].kdsb_addr[off])) + if not kdsp_actual in htab : + htab[kdsp_actual] = kern.globals.kd_bufs[ix].kdsb_addr[off] + htab[kdsp_actual].kds_lostevents = False + + + # generate trace file header; threadmap is stubbed/TBD + version_no = xnudefines.RAW_VERSION1 + thread_count = 0 + TOD_secs = uptime + TOD_usecs = 0 + header = struct.pack('IIqI', version_no, thread_count, TOD_secs, TOD_usecs) + pad_bytes = 4096 - (len(header) & 4095) + header += "\x00" * pad_bytes + wfd.write(buffer(header)) + + count = nkdbufs + while count != 0 : + tempbuf = "" + tempbuf_number = 0 + tempbuf_count = min(count, xnudefines.KDCOPYBUF_COUNT) + + # while space + while tempbuf_count != 0 : + + if opt_progress == True : + progress_count += 1 + if (progress_count % progress_stride) == 0 : + sys.stderr.write('.') + sys.stderr.flush() + + earliest_time = 0xffffffffffffffff + min_kdbp = None + min_cpu = 0 + + # Check all CPUs + for cpu in range(htab[kd_ctrl_page].kdebug_cpus) : + + kdbp = unsigned(addressof(kern.globals.kdbip[cpu])) + if not kdbp in htab : + htab[kdbp] = kern.globals.kdbip[cpu] + + # Skip CPUs without data. + kdsp = htab[kdbp].kd_list_head + if kdsp.raw == xnudefines.KDS_PTR_NULL : + continue + + kdsp_shadow = kdsp + + # Get from cpu data to buffer header to buffer + ix = kdsp.buffer_index + off = kdsp.offset + kdsp_actual = unsigned(addressof(kern.globals.kd_bufs[ix].kdsb_addr[off])) + if not kdsp_actual in htab : + htab[kdsp_actual] = kern.globals.kd_bufs[ix].kdsb_addr[off] + + kdsp_actual_shadow = kdsp_actual + + # Skip buffer if there are no events left. + rcursor = htab[kdsp_actual].kds_readlast + if rcursor == htab[kdsp_actual].kds_bufindx : + continue + + t = htab[kdsp_actual].kds_records[rcursor].timestamp & KDBG_TIMESTAMP_MASK + + # Ignore events that have aged out due to wrapping. + goto_next_cpu = False; + while (t < unsigned(barrier_min)) : + r = htab[kdsp_actual].kds_readlast + htab[kdsp_actual].kds_readlast = r + 1 + rcursor = r + 1 + + if rcursor >= xnudefines.EVENTS_PER_STORAGE_UNIT : + + kdsp = htab[kdbp].kd_list_head + if kdsp.raw == xnudefines.KDS_PTR_NULL : + goto_next_cpu = True + break + + kdsp_shadow = kdsp; + + ix = kdsp.buffer_index + off = kdsp.offset + kdsp_actual = unsigned(addressof(kern.globals.kd_bufs[ix].kdsb_addr[off])) + + kdsp_actual_shadow = kdsp_actual; + rcursor = htab[kdsp_actual].kds_readlast; + + t = htab[kdsp_actual].kds_records[rcursor].timestamp & KDBG_TIMESTAMP_MASK + + if goto_next_cpu == True : + continue + + if (t > barrier_max) and (barrier_max > 0) : + # Need to flush IOPs again before we + # can sort any more data from the + # buffers. + out_of_events = True + break + + if t < (htab[kdsp_actual].kds_timestamp & KDBG_TIMESTAMP_MASK) : + # indicates we've not yet completed filling + # in this event... + # this should only occur when we're looking + # at the buf that the record head is utilizing + # we'll pick these events up on the next + # call to kdbg_read + # we bail at this point so that we don't + # get an out-of-order timestream by continuing + # to read events from the other CPUs' timestream(s) + out_of_events = True + break + + if t < earliest_time : + earliest_time = t + min_kdbp = kdbp + min_cpu = cpu + + + if (min_kdbp is None) or (out_of_events == True) : + # all buffers ran empty + out_of_events = True + break + + kdsp = htab[min_kdbp].kd_list_head + + ix = kdsp.buffer_index + off = kdsp.offset + kdsp_actual = unsigned(addressof(kern.globals.kd_bufs[ix].kdsb_addr[off])) + if not kdsp_actual in htab : + htab[kdsp_actual] = kern.globals.kd_bufs[ix].kdsb_addr[off] + + # Copy earliest event into merged events scratch buffer. + r = htab[kdsp_actual].kds_readlast + htab[kdsp_actual].kds_readlast = r + 1 + e = htab[kdsp_actual].kds_records[r] + + # Concatenate event into buffer + # XXX condition here is on __LP64__ + if lp64 : + tempbuf += struct.pack('QQQQQQIIQ', + e.timestamp, e.arg1, e.arg2, e.arg3, e.arg4, e.arg5, e.debugid, e.cpuid, e.unused) + else : + tempbuf += struct.pack('QIIIIII', + e.timestamp, e.arg1, e.arg2, e.arg3, e.arg4, e.arg5, e.debugid) + + # Watch for out of order timestamps + if earliest_time < (htab[min_kdbp].kd_prev_timebase & KDBG_TIMESTAMP_MASK) : + ## if so, use the previous timestamp + 1 cycle + htab[min_kdbp].kd_prev_timebase += 1 + + e.timestamp = htab[min_kdbp].kd_prev_timebase & KDBG_TIMESTAMP_MASK + e.timestamp |= (min_cpu << KDBG_CPU_SHIFT) + else : + htab[min_kdbp].kd_prev_timebase = earliest_time + + if opt_verbose >= vDETAIL : + print "{0:#018x} {1:#018x} {2:#018x} {3:#018x} {4:#018x} {5:#018x} {6:#010x} {7:#010x} {8:#018x}".format( + e.timestamp, e.arg1, e.arg2, e.arg3, e.arg4, e.arg5, e.debugid, e.cpuid, e.unused) + + events_count_found += 1 + + # nextevent: + tempbuf_count -= 1 + tempbuf_number += 1 + + if opt_progress == True : + sys.stderr.write('\n') + sys.stderr.flush() + + if opt_verbose > vHUMAN : + print "events_count_lost {0:#x}, events_count_found {1:#x}, progress_count {2:#x}".format(events_count_lost, events_count_found, progress_count) + + # write trace events to output file + if tempbuf_number != 0 : + count -= tempbuf_number + wfd.write(buffer(tempbuf)) + + if out_of_events == True : + # all trace buffers are empty + if opt_verbose > vHUMAN : + print "out of events" + break + + wfd.close() + + return + + +def PrintIteratedElem(i, elem, elem_type, do_summary, summary, regex): + try: + if do_summary and summary: + s = summary(elem) + if regex: + if regex.match(s): + print "[{:d}] {:s}".format(i, s) + else: + print "[{:d}] {:s}".format(i, s) + else: + if regex: + if regex.match(str(elem)): + print "[{:4d}] ({:s}){:#x}".format(i, elem_type, unsigned(elem)) + else: + print "[{:4d}] ({:s}){:#x}".format(i, elem_type, unsigned(elem)) + except: + print "Exception while looking at elem {:#x}".format(unsigned(elem)) + return + +@lldb_command('q_iterate', "LQSG:") +def QIterate(cmd_args=None, cmd_options={}): + """ Iterate over a LinkageChain or Queue (osfmk/kern/queue.h method 1 or 2 respectively) + This is equivalent to the qe_foreach_element() macro + usage: + iterate [options] {queue_head_ptr} {element_type} {field_name} + option: + -L iterate over a linkage chain (method 1) [default] + -Q iterate over a queue (method 2) + + -S auto-summarize known types + -G regex to filter the output + e.g. + iterate_linkage `&coalitions_q` 'coalition *' coalitions + """ + if not cmd_args: + raise ArgumentError("usage: iterate_linkage {queue_head_ptr} {element_type} {field_name}") + + qhead = kern.GetValueFromAddress(cmd_args[0], 'struct queue_entry *') + if not qhead: + raise ArgumentError("Unknown queue_head pointer: %r" % cmd_args) + elem_type = cmd_args[1] + field_name = cmd_args[2] + if not elem_type or not field_name: + raise ArgumentError("usage: iterate_linkage {queue_head_ptr} {element_type} {field_name}") + + do_queue_iterate = False + do_linkage_iterate = True + if "-Q" in cmd_options: + do_queue_iterate = True + do_linkage_iterate = False + if "-L" in cmd_options: + do_queue_iterate = False + do_linkage_iterate = True + + do_summary = False + if "-S" in cmd_options: + do_summary = True + regex = None + if "-G" in cmd_options: + regex = re.compile(".*{:s}.*".format(cmd_options["-G"])) + print "Looking for: {:s}".format(regex.pattern) + + global lldb_summary_definitions + summary = None + if elem_type in lldb_summary_definitions: + summary = lldb_summary_definitions[elem_type] + if do_summary: + print summary.header + + try: + i = 0 + if do_linkage_iterate: + for elem in IterateLinkageChain(qhead, elem_type, field_name): + PrintIteratedElem(i, elem, elem_type, do_summary, summary, regex) + i = i + 1 + elif do_queue_iterate: + for elem in IterateQueue(qhead, elem_type, field_name): + PrintIteratedElem(i, elem, elem_type, do_summary, summary, regex) + i = i + 1 + except: + print "Exception while looking at queue_head: {:#x}".format(unsigned(qhead)) diff --git a/tools/lldbmacros/net.py b/tools/lldbmacros/net.py old mode 100644 new mode 100755 index ed39b0c2c..0c17e7b3b --- a/tools/lldbmacros/net.py +++ b/tools/lldbmacros/net.py @@ -1872,8 +1872,6 @@ def ShowDomains(cmd_args=None): out_string += GetSourceInformationForAddress(pru.pru_disconnect) + "\n" out_string += "\t listen:\t" out_string += GetSourceInformationForAddress(pru.pru_listen) + "\n" - out_string += "\t peeloff:\t" - out_string += GetSourceInformationForAddress(pru.pru_peeloff) + "\n" out_string += "\t peeraddr:\t" out_string += GetSourceInformationForAddress(pru.pru_peeraddr) + "\n" out_string += "\t rcvd:\t" diff --git a/tools/lldbmacros/netdefines.py b/tools/lldbmacros/netdefines.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/ntstat.py b/tools/lldbmacros/ntstat.py new file mode 100755 index 000000000..9cbc378d2 --- /dev/null +++ b/tools/lldbmacros/ntstat.py @@ -0,0 +1,175 @@ +""" Please make sure you read the README COMPLETELY BEFORE reading anything below. + It is very critical that you read coding guidelines in Section E in README file. +""" + +from xnu import * +from utils import * +from string import * +from socket import * + +import xnudefines +from netdefines import * +from routedefines import * + +def ShowNstatTUShadow(inshadow): + """ Display summary for an nstat_tu_shadow struct + params: + inshadow : cvalue object which points to 'struct nstat_tu_shadow *' + """ + shad = Cast(inshadow, 'struct nstat_tu_shadow *') + procdetails = shad.shad_procdetails + out_string = "" + if shad : + format_string = "nstat_tu_shadow {0: ] argument is heavily discouraged. """ - if kern.arch != 'arm' and kern.arch != 'arm64': + if not kern.arch.startswith('arm'): raise NotImplementedError("showallmappings does not support {0}".format(kern.arch)) targetPmap = None if len(cmd_args) > 0: @@ -1245,7 +1246,7 @@ def PmapsForLedger(cmd_args=None): """ if cmd_args == None or len(cmd_args) < 1: raise ArgumentError("Too few arguments to pmapsforledger.") - if kern.arch != 'arm' and kern.arch != 'arm64': + if not kern.arch.startswith('arm'): raise NotImplementedError("pmapsforledger does not support {0}".format(kern.arch)) ledger = kern.GetValueFromAddress(cmd_args[0], 'ledger_t') for pmap in IterateQueue(kern.globals.map_pmap_list, 'pmap_t', 'pmaps'): diff --git a/tools/lldbmacros/process.py b/tools/lldbmacros/process.py old mode 100644 new mode 100755 index 906d47a7f..1010c6eb3 --- a/tools/lldbmacros/process.py +++ b/tools/lldbmacros/process.py @@ -9,6 +9,7 @@ from utils import * from core.lazytarget import * import time import xnudefines +import memory def GetProcNameForTask(task): """ returns a string name of the process. if proc is not valid "unknown" is returned @@ -177,15 +178,19 @@ def GetASTSummary(ast): T - AST_TELEMETRY_KERNEL T - AST_TELEMETRY_WINDOWED S - AST_SFI + D - AST_DTRACE + I - AST_TELEMETRY_IO + E - AST_KEVENT """ out_string = "" state = int(ast) thread_state_chars = {0x0:'', 0x1:'P', 0x2:'Q', 0x4:'U', 0x8:'H', 0x10:'Y', 0x20:'A', 0x40:'L', 0x80:'B', 0x100:'K', 0x200:'M', 0x400:'C', 0x800:'C', - 0x1000:'G', 0x2000:'T', 0x4000:'T', 0x8000:'T', 0x10000:'S'} + 0x1000:'G', 0x2000:'T', 0x4000:'T', 0x8000:'T', 0x10000:'S', + 0x20000: 'D', 0x40000: 'I', 0x80000: 'E'} state_str = '' mask = 0x1 - while mask <= 0x10000: + while mask <= 0x80000: state_str += thread_state_chars[int(state & mask)] mask = mask << 1 @@ -216,6 +221,8 @@ def GetTaskSummary(task, showcorpse=False): task_flags = '' if hasattr(task, "suppression_generation") and (int(task.suppression_generation) & 0x1) == 0x1: task_flags += 'P' + if hasattr(task, "effective_policy") and int(task.effective_policy.tep_sup_active) == 1: + task_flags += 'N' if hasattr(task, "suspend_count") and int(task.suspend_count) > 0: task_flags += 'S' if hasattr(task, 'task_imp_base') and unsigned(task.task_imp_base): @@ -380,6 +387,8 @@ def GetCoalitionFlagString(coal): flags.append('reaped') if (coal.notified): flags.append('notified') + if (coal.efficient): + flags.append('efficient') return "|".join(flags) def GetCoalitionTasks(queue, coal_type, thread_details=False): @@ -441,6 +450,7 @@ def GetResourceCoalitionSummary(coal, verbose=False): out_string += "\t bytesread {0: vHUMAN: verbose = True if not cmd_args: - print "No arguments passed" - print ShowCoalitionInfo.__doc__ - return False + raise ArgumentError("No arguments passed") coal = kern.GetValueFromAddress(cmd_args[0], 'coalition *') if not coal: print "unknown arguments:", str(cmd_args) @@ -536,6 +545,34 @@ def ShowAllCoalitions(cmd_args=None): # EndMacro: showallcoalitions +# Macro: showallthreadgroups + +@lldb_type_summary(['thread_group_t', 'thread_group *']) +@header("{0: <20s} {1: <5s} {2: <16s} {3: <5s} {4: <8s} {5: <20s}".format("thread_group", "id", "name", "refc", "flags", "recommendation")) +def GetThreadGroupSummary(tg): + if unsigned(tg) == 0: + return '{0: <#020x} {1: <5d} {2: <16s} {3: <5d} {4: <8s} {5: <20d}'.format(0, -1, "", -1, "", -1) + out_string = "" + format_string = '{0: <#020x} {1: <5d} {2: <16s} {3: <5d} {4: <8s} {5: <20d}' + tg_flags = '' + if (tg.tg_flags & 0x1): + tg_flags += 'E' + if (tg.tg_flags & 0x2): + tg_flags += 'U' + out_string += format_string.format(tg, tg.tg_id, tg.tg_name, tg.tg_refcount, tg_flags, tg.tg_recommendation) + return out_string + +@lldb_command('showallthreadgroups') +def ShowAllThreadGroups(cmd_args=None): + """ Print a summary listing of all thread groups + """ + global kern + print GetThreadGroupSummary.header + for tg in kern.thread_groups: + print GetThreadGroupSummary(tg) + +# EndMacro: showallthreadgroups + # Macro: showtaskcoalitions @lldb_command('showtaskcoalitions', 'F:') @@ -645,47 +682,6 @@ def GetTTYDevSummary(tty_dev): out_string += format_string.format(tty_dev, tty_dev.master, tty_dev.slave, open_fn, free_fn, name_fn, revoke_fn) return out_string -@lldb_type_summary(['kqueue *']) -@header("{: <20s} {: <20s} {: <6s} {: <20s} {: <10s}".format('kqueue', 'process', '#events', 'wqs', 'state')) -def GetKQueueSummary(kq): - """ summarizes kqueue information - returns: str - summary of kqueue - """ - out_string = "" - format_string = "{o: <#020x} {o.kq_p: <#020x} {o.kq_count: <6d} {wqs: <#020x} {st_str: <10s}" - state = int(kq.kq_state) - state_str = '' - mask = 0x1 - while mask <= 0x80 : - if int(state & mask): - state_str += ' ' + xnudefines.kq_state_strings[int(state & mask)] - mask = mask << 1 - out_string += format_string.format(o=kq, wqs=addressof(kq.kq_wqs), st_str=state_str) - out_string += "\n" + GetKnoteSummary.header - for kn in IterateTAILQ_HEAD(kq.kq_head, 'kn_tqe'): - out_string += "\n" + GetKnoteSummary(kn) - return out_string - -@lldb_type_summary(['knote *']) -@header("{0: <20s} {1: <10s} {2: <10s} {3: <20s} {4: <20s} {5: <30s}".format('knote', 'ident', 'kev_flags', 'kn_kq', 'filtops', ' status')) -def GetKnoteSummary(kn): - """ Summarizes a knote and related information - returns: str - summary of knote - """ - out_string = "" - format_string = "{o: <#020x} {o.kn_kevent.ident: <#010X} {o.kn_kevent.flags: <#010X} {o.kn_kq: <#020X} {ops_str: <20s} {st_str: <30s}" - state = unsigned(kn.kn_status) - fops_str = kern.Symbolicate(unsigned(kn.kn_fop)) - mask = 0x1 - status_desc = '' - while mask <= 0x40: - if state & mask: - status_desc += ' ' + xnudefines.kn_state_strings[int(state & mask)] - mask = mask << 1 - - out_string += format_string.format(o=kn, st_str=status_desc, ops_str=fops_str) - return out_string - # Macro: showtask @lldb_command('showtask', 'F:') @@ -721,9 +717,7 @@ def ShowPid(cmd_args=None): Usage: showpid """ if not cmd_args: - print "No arguments passed" - print ShowPid.__doc__ - return False + raise ArgumentError("No arguments passed") pidval = ArgumentStringToInt(cmd_args[0]) for t in kern.tasks: pval = Cast(t.bsd_info, 'proc *') @@ -742,9 +736,7 @@ def ShowProc(cmd_args=None): Usage: showproc
""" if not cmd_args: - print "No arguments passed" - print ShowProc.__doc__ - return False + raise ArgumentError("No arguments passed") pval = kern.GetValueFromAddress(cmd_args[0], 'proc *') if not pval: print "unknown arguments:", str(cmd_args) @@ -764,9 +756,7 @@ def ShowProcInfo(cmd_args=None): Usage: showprocinfo
""" if not cmd_args: - print "No arguments passed" - print ShowProcInfo.__doc__ - return False + raise ArgumentError("No arguments passed") pval = kern.GetValueFromAddress(cmd_args[0], 'proc *') if not pval: print "unknown arguments:", str(cmd_args) @@ -796,17 +786,6 @@ def ShowProcFiles(cmd_args=None): print "{0:-<5s} {0:-<18s} {0:-<10s} {0:-<8s} {0:-<18s} {0:-<64s}".format("") count = 0 - # Filetype map - filetype_dict = { - 1: 'VNODE', - 2: 'SOCKET', - 3: 'PSXSHM', - 4: 'PSXSEM', - 5: 'KQUEUE', - 6: 'PIPE', - 7: 'FSEVENTS' - } - while count <= proc_lastfile: if unsigned(proc_ofiles[count]) != 0: out_str = '' @@ -816,8 +795,8 @@ def ShowProcFiles(cmd_args=None): out_str += "{0: <#18x} ".format(unsigned(proc_fd_fglob)) out_str += "0x{0:0>8x} ".format(unsigned(proc_fd_flags)) proc_fd_ftype = unsigned(proc_fd_fglob.fg_ops.fo_type) - if proc_fd_ftype in filetype_dict: - out_str += "{0: <8s} ".format(filetype_dict[proc_fd_ftype]) + if proc_fd_ftype in xnudefines.filetype_strings: + out_str += "{0: <8s} ".format(xnudefines.filetype_strings[proc_fd_ftype]) else: out_str += "?: {0: <5d} ".format(proc_fd_ftype) out_str += "{0: <#18x} ".format(unsigned(proc_fd_fglob.fg_data)) @@ -830,66 +809,6 @@ def ShowProcFiles(cmd_args=None): #EndMacro: showprocfiles - -def GetProcKqueues(proc): - filetype_KQUEUE = 5 - - proc_filedesc = proc.p_fd - proc_lastfile = unsigned(proc_filedesc.fd_lastfile) - proc_ofiles = proc_filedesc.fd_ofiles - - queues = list() - - if unsigned(proc_ofiles) == 0: - return queues - - count = 0 - - while count <= proc_lastfile: - if unsigned(proc_ofiles[count]) != 0: - proc_fd_flags = proc_ofiles[count].f_flags - proc_fd_fglob = proc_ofiles[count].f_fglob - proc_fd_ftype = unsigned(proc_fd_fglob.fg_ops.fo_type) - if proc_fd_ftype == filetype_KQUEUE: - q = Cast(proc_fd_fglob.fg_data, 'struct kqueue *') - queues.append(q) - count += 1 - - return queues - -def GetAllKqueues(): - for t in kern.tasks: - if unsigned(t.bsd_info) == 0: - continue - pval = Cast(t.bsd_info, 'proc *') - for kq in GetProcKqueues(pval): - yield kq - -#Macro: showallkqueues -@lldb_command('showallkqueues' ,'') -def ShowAllKqueues(cmd_args=[], cmd_options={}): - """ Display a summary of all the kqueues in the system """ - for kq in GetAllKqueues(): - print GetKQueueSummary.header - print GetKQueueSummary(kq) - print "\n\n" -#EndMacro: showallkqueues - -#Macro: showkqueue -@lldb_command('showkqueue' ,'') -def ShowKQueue(cmd_args=[], cmd_options={}): - """ Given a struct kqueue pointer, display the summary of the kqueue - Usage: (lldb) showkqueue - """ - if not cmd_args: - raise ArgumentError('Invalid arguments') - - kq = kern.GetValueFromAddress(cmd_args[0], 'struct kqueue *') - print GetKQueueSummary.header - print GetKQueueSummary(kq) - -#EndMacro: showkqueue - #Macro: showtty @lldb_command('showtty') @@ -1259,10 +1178,8 @@ def ShowAct(cmd_args=None): """ Routine to print out the state of a specific thread. usage: showact """ - if cmd_args == None or len(cmd_args) < 1: - print "No arguments passed" - print ShowAct.__doc__ - return False + if not cmd_args: + raise ArgumentError("No arguments passed") threadval = kern.GetValueFromAddress(cmd_args[0], 'thread *') print GetThreadSummary.header print GetThreadSummary(threadval) @@ -1272,10 +1189,8 @@ def ShowActStack(cmd_args=None): """ Routine to print out the stack of a specific thread. usage: showactstack """ - if cmd_args == None or len(cmd_args) < 1: - print "No arguments passed" - print ShowAct.__doc__.strip() - return False + if not cmd_args: + raise ArgumentError("No arguments passed") threadval = kern.GetValueFromAddress(cmd_args[0], 'thread *') print GetThreadSummary.header print GetThreadSummary(threadval) @@ -1291,10 +1206,8 @@ def SwitchToAct(cmd_args=None): Before resuming execution, issue a "resetctx" command, to return to the original execution context. """ - if cmd_args == None or len(cmd_args) < 1: - print "No arguments passed" - print SwitchToAct.__doc__.strip() - return False + if cmd_args is None or len(cmd_args) < 1: + raise ArgumentError("No arguments passed") thval = kern.GetValueFromAddress(cmd_args[0], 'thread *') lldbthread = GetLLDBThreadForKernelThread(thval) print GetThreadSummary.header @@ -1402,7 +1315,7 @@ def GetFullBackTrace(frame_addr, verbosity = vHUMAN, prefix = ""): # lldb unable to find symbol for _mh_execute_header mh_execute_addr = int(lldb_run_command('p/x (uintptr_t *)&_mh_execute_header').split('=')[-1].strip(), 16) while frame_ptr and frame_ptr != previous_frame_ptr and bt_count < 128: - if (kern.arch not in ('arm', 'arm64') and frame_ptr < mh_execute_addr) or (kern.arch in ('arm', 'arm64') and frame_ptr > mh_execute_addr): + if (not kern.arch.startswith('arm') and frame_ptr < mh_execute_addr) or (kern.arch.startswith('arm') and frame_ptr > mh_execute_addr): break pc_val = kern.GetValueFromAddress(frame_ptr + kern.ptrsize,'uintptr_t *') pc_val = unsigned(dereference(pc_val)) @@ -1598,11 +1511,14 @@ def GetLedgerEntrySummary(ledger_template, ledger, i): out_str += "{: >32s} {:<2d}:".format(ledger_template.lt_entries[i].et_key, i) out_str += "{: >15d} ".format(unsigned(ledger.le_credit) - unsigned(ledger.le_debit)) if (ledger.le_flags & lf_tracking_max): - out_str += "{:9d} {:5d} ".format(ledger._le.le_peaks[0].le_max, now - unsigned(ledger._le.le_peaks[0].le_time)) - out_str += "{:9d} {:4d} ".format(ledger._le.le_peaks[1].le_max, now - unsigned(ledger._le.le_peaks[1].le_time)) + out_str += "{:9d} {:5d} ".format(ledger._le.le_maxtracking.le_peaks[0].le_max, now - unsigned(ledger._le.le_maxtracking.le_peaks[0].le_time)) else: - out_str += " - - - - " - + out_str += " - -" + + if (ledger.le_flags & lf_tracking_max): + out_str += "{:12d} ".format(ledger._le.le_maxtracking.le_lifetime_max) + else: + out_str += " -" out_str += "{:12d} {:12d} ".format(unsigned(ledger.le_credit), unsigned(ledger.le_debit)) if (unsigned(ledger.le_limit) != ledger_limit_infinity): out_str += "{:12d} ".format(unsigned(ledger.le_limit)) @@ -1647,9 +1563,9 @@ def GetThreadLedgerSummary(thread_val): i = i + 1 return out_str -@header("{0: <15s} {1: >16s} {2: <2s} {3: >15s} {4: >9s} {5: >6s} {6: >8s} {7: <10s} {8: <9s} \ - {9: <12s} {10: <7s} {11: <15s} {12: <8s} {13: <9s} {14: <6s} {15: >6s}".format( - "task [thread]", "entry", "#", "balance", "peakA", "(age)", "peakB", "(age)", "credit", +@header("{0: <15s} {1: >16s} {2: <2s} {3: >15s} {4: >9s} {5: >6s} {6: >12s} {7: >11s} \ + {8: >7s} {9: >13s} {10: <15s} {11: <8s} {12: <9s} {13: <6s} {14: >6s}".format( + "task [thread]", "entry", "#", "balance", "peakA", "(age)", "lifemax", "credit", "debit", "limit", "refill period", "lim pct", "warn pct", "over?", "flags")) def GetTaskLedgers(task_val): """ Internal function to get summary of ledger entries from the task and its threads @@ -2151,61 +2067,3 @@ def Showstackafterthread(cmd_args = None): print '\n' return -def FindVMEntriesForVnode(task, vn): - """ returns an array of vme that have the vnode set to defined vnode - each entry in array is of format (vme, start_addr, end_address, protection) - """ - retval = [] - vmmap = task.map - pmap = vmmap.pmap - pager_ops_addr = unsigned(addressof(kern.globals.vnode_pager_ops)) - debuglog("pager_ops_addr %s" % hex(pager_ops_addr)) - - if unsigned(pmap) == 0: - return retval - vme_list_head = vmmap.hdr.links - vme_ptr_type = gettype('vm_map_entry *') - for vme in IterateQueue(vme_list_head, vme_ptr_type, 'links'): - #print vme - if unsigned(vme.is_sub_map) == 0 and unsigned(vme.object.vm_object) != 0: - obj = vme.object.vm_object - else: - continue - - while obj != 0: - if obj.pager != 0: - if obj.internal: - pass - else: - vn_pager = Cast(obj.pager, 'vnode_pager *') - if unsigned(vn_pager.pager_ops) == pager_ops_addr and unsigned(vn_pager.vnode_handle) == unsigned(vn): - retval.append((vme, unsigned(vme.links.start), unsigned(vme.links.end), unsigned(vme.protection))) - obj = obj.shadow - return retval - -@lldb_command('showtaskloadinfo') -def ShowTaskLoadInfo(cmd_args=None, cmd_options={}): - """ Print the load address and uuid for the process - Usage: (lldb)showtaskloadinfo - """ - if not cmd_args: - raise ArgumentError("Insufficient arguments") - t = kern.GetValueFromAddress(cmd_args[0], 'struct task *') - print_format = "0x{0:x} - 0x{1:x} {2: <50s} (??? - ???) <{3: <36s}> {4: <50s}" - p = Cast(t.bsd_info, 'struct proc *') - uuid = p.p_uuid - uuid_out_string = "{a[0]:02X}{a[1]:02X}{a[2]:02X}{a[3]:02X}-{a[4]:02X}{a[5]:02X}-{a[6]:02X}{a[7]:02X}-{a[8]:02X}{a[9]:02X}-{a[10]:02X}{a[11]:02X}{a[12]:02X}{a[13]:02X}{a[14]:02X}{a[15]:02X}".format(a=uuid) - filepath = GetVnodePath(p.p_textvp) - libname = filepath.split('/')[-1] - #print "uuid: %s file: %s" % (uuid_out_string, filepath) - mappings = FindVMEntriesForVnode(t, p.p_textvp) - load_addr = 0 - end_addr = 0 - for m in mappings: - if m[3] == 5: - load_addr = m[1] - end_addr = m[2] - #print "Load address: %s" % hex(m[1]) - print print_format.format(load_addr, end_addr, libname, uuid_out_string, filepath) - return None - diff --git a/tools/lldbmacros/routedefines.py b/tools/lldbmacros/routedefines.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/scheduler.py b/tools/lldbmacros/scheduler.py old mode 100644 new mode 100755 index cf828c2db..d60dd0e4e --- a/tools/lldbmacros/scheduler.py +++ b/tools/lldbmacros/scheduler.py @@ -1,6 +1,8 @@ from xnu import * from utils import * from process import * +from misc import * +from memory import * # TODO: write scheduler related macros here @@ -15,7 +17,7 @@ def ShowAllProcRunQCount(cmd_args=None): while processor_itr: out_str += "{:d}\t\t{:d}\n".format(processor_itr.cpu_id, processor_itr.runq.count) processor_itr = processor_itr.processor_list - out_str += "RT:\t\t{:d}\n".format(kern.globals.rt_runq.count) + # out_str += "RT:\t\t{:d}\n".format(kern.globals.rt_runq.count) print out_str # EndMacro: showallprocrunqcount @@ -27,18 +29,24 @@ def ShowInterrupts(cmd_args=None): """ Prints IRQ, IPI and TMR counts for each CPU """ - if kern.arch not in ('arm', 'arm64'): + if not kern.arch.startswith('arm'): print "showinterrupts is only supported on arm/arm64" return base_address = kern.GetLoadAddressForSymbol('CpuDataEntries') - struct_size = 16 - for x in xrange (0, unsigned(kern.globals.machine_info.physical_cpu)): - element = kern.GetValueFromAddress(base_address + (x * struct_size), 'uintptr_t *')[1] - cpu_data_entry = Cast(element, 'cpu_data_t *') - print "CPU {} IRQ: {:d}\n".format(x, cpu_data_entry.cpu_stat.irq_ex_cnt) - print "CPU {} IPI: {:d}\n".format(x, cpu_data_entry.cpu_stat.ipi_cnt) - print "CPU {} TMR: {:d}\n".format(x, cpu_data_entry.cpu_stat.timer_cnt) + struct_size = 16 + x = 0 + y = 0 + while x < unsigned(kern.globals.machine_info.physical_cpu): + element = kern.GetValueFromAddress(base_address + (y * struct_size), 'uintptr_t *')[1] + if element: + cpu_data_entry = Cast(element, 'cpu_data_t *') + print "CPU {} IRQ: {:d}\n".format(y, cpu_data_entry.cpu_stat.irq_ex_cnt) + print "CPU {} IPI: {:d}\n".format(y, cpu_data_entry.cpu_stat.ipi_cnt) + print "CPU {} TMR: {:d}\n".format(y, cpu_data_entry.cpu_stat.timer_cnt) + x = x + 1 + y = y + 1 + # EndMacro: showinterrupts # Macro: showactiveinterrupts @@ -91,10 +99,15 @@ def ShowIrqByIpiTimerRatio(cmd_args=None): out_str = "IRQ-IT Ratio: " base_address = kern.GetLoadAddressForSymbol('CpuDataEntries') struct_size = 16 - for x in range (0, unsigned(kern.globals.machine_info.physical_cpu)): - element = kern.GetValueFromAddress(base_address + (x * struct_size), 'uintptr_t *')[1] - cpu_data_entry = Cast(element, 'cpu_data_t *') - out_str += " CPU {} [{:.2f}]".format(x, float(cpu_data_entry.cpu_stat.irq_ex_cnt)/(cpu_data_entry.cpu_stat.ipi_cnt + cpu_data_entry.cpu_stat.timer_cnt)) + x = 0 + y = 0 + while x < unsigned(kern.globals.machine_info.physical_cpu): + element = kern.GetValueFromAddress(base_address + (y * struct_size), 'uintptr_t *')[1] + if element: + cpu_data_entry = Cast(element, 'cpu_data_t *') + out_str += " CPU {} [{:.2f}]".format(y, float(cpu_data_entry.cpu_stat.irq_ex_cnt)/(cpu_data_entry.cpu_stat.ipi_cnt + cpu_data_entry.cpu_stat.timer_cnt)) + x = x + 1 + y = y + 1 print out_str # EndMacro: showirqbyipitimerratio @@ -178,6 +191,16 @@ def ShowAbstimeToNanoTime(cmd_args=[]): # Macro: showschedhistory +def GetRecentTimestamp(): + """ + Return a recent timestamp. + TODO: on x86, if not in the debugger, then look at the scheduler + """ + if kern.arch == 'x86_64': + return kern.globals.debugger_entry_time + else : + return GetSchedMostRecentDispatch(False) + def GetSchedMostRecentDispatch(show_processor_details=False): """ Return the most recent dispatch on the system, printing processor details if argument is true. @@ -219,7 +242,7 @@ def GetSchedMostRecentDispatch(show_processor_details=False): return most_recent_dispatch -@header("{:<18s} {:<10s} {:>16s} {:>16s} {:>16s} {:>18s} {:>16s} {:>16s} {:>16s} {:2s} {:2s} {:2s} {:>2s} {:<19s} {:<9s} {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} {:>11s} {:>8s}".format("thread", "id", "on-core", "off-core", "runnable", "last-duration (us)", "since-off (us)", "since-on (us)", "pending (us)", "BP", "SP", "TP", "MP", "sched-mode", "state", "cpu-usage", "delta", "sch-usage", "stamp", "shift", "task", "thread-name")) +@header("{:<18s} {:<10s} {:>16s} {:>16s} {:>16s} {:>16s} {:>18s} {:>16s} {:>16s} {:>16s} {:>16s} {:2s} {:2s} {:2s} {:>2s} {:<19s} {:<9s} {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} {:>11s} {:>8s}".format("thread", "id", "on-core", "off-core", "runnable", "prichange", "last-duration (us)", "since-off (us)", "since-on (us)", "pending (us)", "pri-change (us)", "BP", "SP", "TP", "MP", "sched-mode", "state", "cpu-usage", "delta", "sch-usage", "stamp", "shift", "task", "thread-name")) def ShowThreadSchedHistory(thread, most_recent_dispatch): """ Given a thread and the most recent dispatch time of a thread on the system, print out details about scheduler history for the thread. @@ -268,50 +291,87 @@ def ShowThreadSchedHistory(thread, most_recent_dispatch): last_on = thread.computation_epoch last_off = thread.last_run_time last_runnable = thread.last_made_runnable_time - + last_prichange = thread.last_basepri_change_time + if int(last_runnable) == 18446744073709551615 : last_runnable = 0 + if int(last_prichange) == 18446744073709551615 : + last_prichange = 0 + time_on_abs = unsigned(last_off - last_on) time_on_us = kern.GetNanotimeFromAbstime(time_on_abs) / 1000.0 time_pending_abs = unsigned(most_recent_dispatch - last_runnable) time_pending_us = kern.GetNanotimeFromAbstime(time_pending_abs) / 1000.0 - + if int(last_runnable) == 0 : time_pending_us = 0 + last_prichange_abs = unsigned(most_recent_dispatch - last_prichange) + last_prichange_us = kern.GetNanotimeFromAbstime(last_prichange_abs) / 1000.0 + + if int(last_prichange) == 0 : + last_prichange_us = 0 + time_since_off_abs = unsigned(most_recent_dispatch - last_off) time_since_off_us = kern.GetNanotimeFromAbstime(time_since_off_abs) / 1000.0 time_since_on_abs = unsigned(most_recent_dispatch - last_on) time_since_on_us = kern.GetNanotimeFromAbstime(time_since_on_abs) / 1000.0 - fmt = "0x{t:<16x} 0x{t.thread_id:<8x} {t.computation_epoch:16d} {t.last_run_time:16d} {last_runnable:16d} {time_on_us:18.3f} {time_since_off_us:16.3f} {time_since_on_us:16.3f} {time_pending_us:16.3f}" + fmt = "0x{t:<16x} 0x{t.thread_id:<8x} {t.computation_epoch:16d} {t.last_run_time:16d} {last_runnable:16d} {last_prichange:16d} {time_on_us:18.3f} {time_since_off_us:16.3f} {time_since_on_us:16.3f} {time_pending_us:16.3f} {last_prichange_us:16.3f}" fmt2 = " {t.base_pri:2d} {t.sched_pri:2d} {t.task_priority:2d} {t.max_priority:2d} {sched_mode:19s}" fmt3 = " {state:9s} {t.cpu_usage:10d} {t.cpu_delta:10d} {t.sched_usage:10d} {t.sched_stamp:10d} {t.pri_shift:10d} {name:s} {thread_name:s}" - out_str = fmt.format(t=thread, time_on_us=time_on_us, time_since_off_us=time_since_off_us, time_since_on_us=time_since_on_us, last_runnable=last_runnable, time_pending_us=time_pending_us) + out_str = fmt.format(t=thread, time_on_us=time_on_us, time_since_off_us=time_since_off_us, time_since_on_us=time_since_on_us, last_runnable=last_runnable, time_pending_us=time_pending_us, last_prichange=last_prichange, last_prichange_us=last_prichange_us) out_str += fmt2.format(t=thread, sched_mode=sched_mode) out_str += fmt3.format(t=thread, state=state_str, name=task_name, thread_name=thread_name) - + print out_str -@lldb_command('showschedhistory') -def ShowSchedHistory(cmd_args=None): - """ Routine to print out thread scheduling history - Usage: showschedhistory [ ...] +def SortThreads(threads, column): + if column != 'on-core' and column != 'off-core' and column != 'last-duration': + raise ArgumentError("unsupported sort column") + if column == 'on-core': + threads.sort(key=lambda t: t.computation_epoch) + elif column == 'off-core': + threads.sort(key=lambda t: t.last_run_time) + else: + threads.sort(key=lambda t: t.last_run_time - t.computation_epoch) + +@lldb_command('showschedhistory', 'S:') +def ShowSchedHistory(cmd_args=None, cmd_options=None): + """ Routine to print out thread scheduling history, optionally sorted by a + column. + + Usage: showschedhistory [-S on-core|off-core|last-duration] [ ...] """ + sort_column = None + if '-S' in cmd_options: + sort_column = cmd_options['-S'] + if cmd_args: most_recent_dispatch = GetSchedMostRecentDispatch(False) print ShowThreadSchedHistory.header - for thread_ptr in cmd_args: - thread = kern.GetValueFromAddress(ArgumentStringToInt(thread_ptr), 'thread *') - ShowThreadSchedHistory(thread, most_recent_dispatch) + + if sort_column: + threads = [] + for thread_ptr in cmd_args: + threads.append(kern.GetValueFromAddress(ArgumentStringToInt(thread_ptr), 'thread *')) + + SortThreads(threads, sort_column) + + for thread in threads: + ShowThreadSchedHistory(thread, most_recent_dispatch) + else: + for thread_ptr in cmd_args: + thread = kern.GetValueFromAddress(ArgumentStringToInt(thread_ptr), 'thread *') + ShowThreadSchedHistory(thread, most_recent_dispatch) return - + run_buckets = kern.globals.sched_run_buckets run_count = run_buckets[GetEnumValue('sched_bucket_t::TH_BUCKET_RUN')] @@ -340,12 +400,24 @@ def ShowSchedHistory(cmd_args=None): print "Most recent dispatch: " + str(most_recent_dispatch) print ShowThreadSchedHistory.header - for thread in IterateQueue(kern.globals.threads, 'thread *', 'threads'): - ShowThreadSchedHistory(thread, most_recent_dispatch) + + if sort_column: + threads = [t for t in IterateQueue(kern.globals.threads, 'thread *', 'threads')] + + SortThreads(threads, sort_column) + + for thread in threads: + ShowThreadSchedHistory(thread, most_recent_dispatch) + else: + for thread in IterateQueue(kern.globals.threads, 'thread *', 'threads'): + ShowThreadSchedHistory(thread, most_recent_dispatch) # EndMacro: showschedhistory +def int32(n): + n = n & 0xffffffff + return (n ^ 0x80000000) - 0x80000000 # Macro: showallprocessors @@ -353,7 +425,8 @@ def ShowGroupSetSummary(runq, task_map): """ Internal function to print summary of group run queue params: runq - value representing struct run_queue * """ - print " runq: count {: <10d} highq: {: <10d} urgency {: <10d}\n".format(runq.count, runq.highq, runq.urgency) + + print " runq: count {: <10d} highq: {: <10d} urgency {: <10d}\n".format(runq.count, int32(runq.highq), runq.urgency) runq_queue_i = 0 runq_queue_count = sizeof(runq.queues)/sizeof(runq.queues[0]) @@ -395,7 +468,8 @@ def ShowRunQSummary(runq): """ Internal function to print summary of run_queue params: runq - value representing struct run_queue * """ - print " runq: count {: <10d} highq: {: <10d} urgency {: <10d}\n".format(runq.count, runq.highq, runq.urgency) + + print " runq: count {: <10d} highq: {: <10d} urgency {: <10d}\n".format(runq.count, int32(runq.highq), runq.urgency) runq_queue_i = 0 runq_queue_count = sizeof(runq.queues)/sizeof(runq.queues[0]) @@ -417,6 +491,12 @@ def ShowRunQSummary(runq): if config['verbosity'] > vHUMAN : print "\t" + GetThreadBackTrace(thread, prefix="\t\t") + "\n" +def ShowRTRunQSummary(rt_runq): + print " Realtime Queue ({:<#012x}) Count {:d}\n".format(addressof(rt_runq.queue), rt_runq.count) + if rt_runq.count != 0: + print "\t" + GetThreadSummary.header + "\n" + for rt_runq_thread in ParanoidIterateLinkageChain(rt_runq.queue, "thread_t", "runq_links"): + print "\t" + GetThreadSummary(rt_runq_thread) + "\n" def ShowGrrrSummary(grrr_runq): """ Internal function to print summary of grrr_run_queue @@ -455,12 +535,15 @@ def ShowScheduler(cmd_args=None): """ Routine to print information of all psets and processors Usage: showscheduler """ - pset = addressof(kern.globals.pset0) + node = addressof(kern.globals.pset_node0) show_grrr = 0 show_priority_runq = 0 show_priority_pset_runq = 0 show_group_pset_runq = 0 - sched_string = str(kern.globals.sched_current_dispatch.sched_name) + if unsigned(kern.globals.sched_current_dispatch) != 0 : + sched_string = str(kern.globals.sched_current_dispatch.sched_name) + else : + sched_string = str(kern.globals.sched_string) if sched_string == "traditional": show_priority_runq = 1 @@ -474,11 +557,15 @@ def ShowScheduler(cmd_args=None): elif sched_string == "dualq": show_priority_pset_runq = 1 show_priority_runq = 1 + elif sched_string == "amp": + show_priority_pset_runq = 1 + show_priority_runq = 1 else : print "Unknown sched_string {:s}".format(sched_string) - print "Scheduler: {:s} ({:s})\n".format(sched_string, - kern.Symbolicate(unsigned(kern.globals.sched_current_dispatch))) + if unsigned(kern.globals.sched_current_dispatch) != 0 : + print "Scheduler: {:s} ({:s})\n".format(sched_string, + kern.Symbolicate(unsigned(kern.globals.sched_current_dispatch))) run_buckets = kern.globals.sched_run_buckets @@ -492,7 +579,10 @@ def ShowScheduler(cmd_args=None): print "FG Timeshare threads: {:d} UT Timeshare threads: {:d} BG Timeshare threads: {:d}\n".format(share_fg_count, share_ut_count, share_bg_count) if show_group_pset_runq: - print "multiq scheduler config: deep-drain {g.deep_drain:d}, ceiling {g.drain_ceiling:d}, depth limit {g.drain_depth_limit:d}, band limit {g.drain_band_limit:d}, sanity check {g.multiq_sanity_check:d}\n".format(g=kern.globals) + if hasattr(kern.globals, "multiq_sanity_check"): + print "multiq scheduler config: deep-drain {g.deep_drain:d}, ceiling {g.drain_ceiling:d}, depth limit {g.drain_depth_limit:d}, band limit {g.drain_band_limit:d}, sanity check {g.multiq_sanity_check:d}\n".format(g=kern.globals) + else: + print "multiq scheduler config: deep-drain {g.deep_drain:d}, ceiling {g.drain_ceiling:d}, depth limit {g.drain_depth_limit:d}, band limit {g.drain_band_limit:d}\n".format(g=kern.globals) # Create a group->task mapping task_map = {} @@ -503,71 +593,74 @@ def ShowScheduler(cmd_args=None): print " \n" - while unsigned(pset) != 0: - print "Processor Set {: <#012x} Count {:d} (cpu_id {:<#x}-{:<#x})\n".format(pset, - pset.cpu_set_count, pset.cpu_set_low, pset.cpu_set_hi) - - if show_priority_pset_runq: - runq = pset.pset_runq - ShowRunQSummary(runq) - - if show_group_pset_runq: - print "Main Runq:\n" - runq = pset.pset_runq - ShowGroupSetSummary(runq, task_map) - print "All Groups:\n" - # TODO: Possibly output task header for each group - for group in IterateQueue(kern.globals.sched_groups, "sched_group_t", "sched_groups"): - if (group.runq.count != 0) : - task = task_map.get(unsigned(group), "Unknown task!") - print "Group {: <#012x} Task {: <#012x}\n".format(unsigned(group), unsigned(task)) - ShowRunQSummary(group.runq) - print " \n" - - print "Active Processors:\n" - for processor in ParanoidIterateLinkageChain(pset.active_queue, "processor_t", "processor_queue"): - print " " + GetProcessorSummary(processor) - ShowActiveThread(processor) - ShowNextThread(processor) + while node != 0: + pset = node.psets + pset = kern.GetValueFromAddress(unsigned(pset), 'struct processor_set *') + + while pset != 0: + print "Processor Set {: <#012x} Count {:d} (cpu_id {:<#x}-{:<#x})\n".format(pset, + unsigned(pset.cpu_set_count), pset.cpu_set_low, pset.cpu_set_hi) + + rt_runq = kern.GetValueFromAddress(unsigned(addressof(pset.rt_runq)), 'struct rt_queue *') + ShowRTRunQSummary(rt_runq) - if show_priority_runq: - runq = processor.runq + if show_priority_pset_runq: + runq = kern.GetValueFromAddress(unsigned(addressof(pset.pset_runq)), 'struct run_queue *') ShowRunQSummary(runq) - if show_grrr: - grrr_runq = processor.grrr_runq - ShowGrrrSummary(grrr_runq) - print " \n" + if show_group_pset_runq: + print "Main Runq:\n" + runq = kern.GetValueFromAddress(unsigned(addressof(pset.pset_runq)), 'struct run_queue *') + ShowGroupSetSummary(runq, task_map) + print "All Groups:\n" + # TODO: Possibly output task header for each group + for group in IterateQueue(kern.globals.sched_groups, "sched_group_t", "sched_groups"): + if (group.runq.count != 0) : + task = task_map.get(unsigned(group), "Unknown task!") + print "Group {: <#012x} Task {: <#012x}\n".format(unsigned(group), unsigned(task)) + ShowRunQSummary(group.runq) + print " \n" - print "Idle Processors:\n" - for processor in ParanoidIterateLinkageChain(pset.idle_queue, "processor_t", "processor_queue"): - print " " + GetProcessorSummary(processor) - ShowActiveThread(processor) - ShowNextThread(processor) + print "Active Processors:\n" + for processor in ParanoidIterateLinkageChain(pset.active_queue, "processor_t", "processor_queue"): + print " " + GetProcessorSummary(processor) + ShowActiveThread(processor) + ShowNextThread(processor) - if show_priority_runq: - ShowRunQSummary(processor.runq) - print " \n" + if show_priority_runq: + runq = processor.runq + ShowRunQSummary(runq) + if show_grrr: + grrr_runq = processor.grrr_runq + ShowGrrrSummary(grrr_runq) + print " \n" - print "Idle Secondary Processors:\n" - for processor in ParanoidIterateLinkageChain(pset.idle_secondary_queue, "processor_t", "processor_queue"): - print " " + GetProcessorSummary(processor) - ShowActiveThread(processor) - ShowNextThread(processor) + print "Idle Processors:\n" + for processor in ParanoidIterateLinkageChain(pset.idle_queue, "processor_t", "processor_queue"): + print " " + GetProcessorSummary(processor) + ShowActiveThread(processor) + ShowNextThread(processor) - if show_priority_runq: - print ShowRunQSummary(processor.runq) - print " \n" + if show_priority_runq: + ShowRunQSummary(processor.runq) + print " \n" - pset = pset.pset_list + print "Idle Secondary Processors:\n" + for processor in ParanoidIterateLinkageChain(pset.idle_secondary_queue, "processor_t", "processor_queue"): + print " " + GetProcessorSummary(processor) + ShowActiveThread(processor) + ShowNextThread(processor) - print "\nRealtime Queue ({:<#012x}) Count {:d}\n".format(addressof(kern.globals.rt_runq.queue), kern.globals.rt_runq.count) - if kern.globals.rt_runq.count != 0: - print "\t" + GetThreadSummary.header + "\n" - for rt_runq_thread in ParanoidIterateLinkageChain(kern.globals.rt_runq.queue, "thread_t", "runq_links"): - print "\t" + GetThreadSummary(rt_runq_thread) + "\n" + if show_priority_runq: + print ShowRunQSummary(processor.runq) + print " \n" + + + pset = pset.pset_list + + node = node.node_list print "\nTerminate Queue: ({:<#012x})\n".format(addressof(kern.globals.thread_terminate_queue)) first = False @@ -640,9 +733,11 @@ def ParanoidIterateLinkageChain(queue_head, element_type, field_name, field_ofst return if element_type.IsPointerType(): - elem_ofst = getfieldoffset(element_type.GetPointeeType(), field_name) + field_ofst + struct_type = element_type.GetPointeeType() else: - elem_ofst = getfieldoffset(element_type, field_name) + field_ofst + struct_type = element_type + + elem_ofst = getfieldoffset(struct_type, field_name) + field_ofst try: link = queue_head.next @@ -697,30 +792,77 @@ ParanoidIterateLinkageChain.enable_paranoia = True ParanoidIterateLinkageChain.enable_debug = False # Macro: showallcallouts + +def ShowThreadCall(prefix, call): + """ + Print a description of a thread_call_t and its relationship to its expected fire time + """ + func = call.tc_call.func + param0 = call.tc_call.param0 + param1 = call.tc_call.param1 + + iotes_desc = "" + iotes_callout = kern.GetLoadAddressForSymbol("_ZN18IOTimerEventSource17timeoutAndReleaseEPvS0_") + iotes_callout2 = kern.GetLoadAddressForSymbol("_ZN18IOTimerEventSource15timeoutSignaledEPvS0_") + + if (unsigned(func) == unsigned(iotes_callout) or + unsigned(func) == unsigned(iotes_callout2)) : + iotes = Cast(call.tc_call.param0, 'IOTimerEventSource*') + func = iotes.action + param0 = iotes.owner + param1 = unsigned(iotes) + + func_name = kern.Symbolicate(func) + if (func_name == "") : + func_name = FindKmodNameForAddr(func) + + call_entry = call.tc_call + + recent_timestamp = GetRecentTimestamp() + + # THREAD_CALL_CONTINUOUS 0x100 + kern.globals.mach_absolutetime_asleep + if (call.tc_flags & 0x100) : + timer_fire = call_entry.deadline - (recent_timestamp + kern.globals.mach_absolutetime_asleep) + else : + timer_fire = call_entry.deadline - recent_timestamp + + timer_fire_s = kern.GetNanotimeFromAbstime(timer_fire) / 1000000000.0 + + ttd_s = kern.GetNanotimeFromAbstime(call.tc_ttd) / 1000000000.0 + + print "{:s}{:#018x}: {:18d} {:18d} {:03.06f} {:03.06f} {:#018x}({:#018x},{:#018x}) ({:s})".format(prefix, + unsigned(call), call_entry.deadline, call.tc_soft_deadline, ttd_s, timer_fire_s, + func, param0, param1, func_name) + @lldb_command('showallcallouts') def ShowAllCallouts(cmd_args=None): - """ Prints out the pending and delayed thread calls for high priority thread call group + """ Prints out the pending and delayed thread calls for the thread call groups """ - # Get the high priority thread's call group - g = addressof(kern.globals.thread_call_groups[0]) - pq = addressof(g.pending_queue) - dq = addressof(g.delayed_queue) - - print "Active threads: {:d}\n".format(g.active_count) - print "Idle threads: {:d}\n".format(g.idle_count) - print "Pending threads: {:d}\n".format(g.pending_count) - - call = Cast(pq.next, 'thread_call_t') - while unsigned(call) != unsigned(pq): - print "Callout: " + kern.Symbolicate([unsigned(call.tc_call.func)]) + "\n" - call = Cast(call.tc_call.q_link.next, 'thread_call_t') - - print "\nDelayed:\n" - call = Cast(dq.next, 'thread_call_t') - while unsigned(call) != unsigned(dq): - out_str = "Deadline: {:>22d}. Callout: {:#x} <".format(call.tc_call.deadline, unsigned(call.tc_call.func)) - print out_str + kern.Symbolicate(unsigned(call.tc_call.func)) + ">\n" - call = Cast(call.tc_call.q_link.next, 'thread_call_t') + + index_max = GetEnumValue('thread_call_index_t::THREAD_CALL_INDEX_MAX') + + for i in range (0, index_max) : + group = kern.globals.thread_call_groups[i] + + print "Group {i:d}: {g.tcg_name:s} ({:>#18x})".format(addressof(group), i=i, g=group) + print "\t" +"Active: {g.active_count:d} Idle: {g.idle_count:d}\n".format(g=group) + print "\t" +"Blocked: {g.blocked_count:d} Pending: {g.pending_count:d}\n".format(g=group) + print "\t" +"Target: {g.target_thread_count:d}\n".format(g=group) + + print "\t" +"Pending Queue: ({:>#18x})\n".format(addressof(group.pending_queue)) + for call in ParanoidIterateLinkageChain(group.pending_queue, "thread_call_t", "tc_call.q_link"): + ShowThreadCall("\t\t", call) + + print "\t" +"Delayed Queue (Absolute Time): ({:>#18x}) timer: ({:>#18x})\n".format( + addressof(group.delayed_queues[0]), addressof(group.delayed_timers[0])) + for call in ParanoidIterateLinkageChain(group.delayed_queues[0], "thread_call_t", "tc_call.q_link"): + ShowThreadCall("\t\t", call) + + print "\t" +"Delayed Queue (Continuous Time): ({:>#18x}) timer: ({:>#18x})\n".format( + addressof(group.delayed_queues[1]), addressof(group.delayed_timers[1])) + for call in ParanoidIterateLinkageChain(group.delayed_queues[1], "thread_call_t", "tc_call.q_link"): + ShowThreadCall("\t\t", call) # EndMacro: showallcallouts diff --git a/tools/lldbmacros/structanalyze.py b/tools/lldbmacros/structanalyze.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/userspace.py b/tools/lldbmacros/userspace.py old mode 100644 new mode 100755 index 4450c248f..88a8858fc --- a/tools/lldbmacros/userspace.py +++ b/tools/lldbmacros/userspace.py @@ -140,7 +140,7 @@ def ShowThreadUserStack(cmd_args=None): ShowX86UserStack(thread) elif kern.arch == "arm": ShowARMUserStack(thread) - elif kern.arch == "arm64": + elif kern.arch.startswith("arm64"): ShowARM64UserStack(thread) return True @@ -273,11 +273,14 @@ Synthetic crash log generated from Kernel userstacks """ user_lib_rex = re.compile("([0-9a-fx]+)\s-\s([0-9a-fx]+)\s+(.*?)\s", re.IGNORECASE|re.MULTILINE) from datetime import datetime - ts = datetime.fromtimestamp(int(pval.p_start.tv_sec)) - date_string = ts.strftime('%Y-%m-%d %H:%M:%S') - is_64 = False - if pval.p_flag & 0x4 : - is_64 = True + if pval: + ts = datetime.fromtimestamp(int(pval.p_start.tv_sec)) + date_string = ts.strftime('%Y-%m-%d %H:%M:%S') + else: + date_string = "none" + is_64 = True + if pval and (pval.p_flag & 0x4) == 0 : + is_64 = False parch_s = "" if kern.arch == "x86_64" or kern.arch == "i386": @@ -289,15 +292,25 @@ Synthetic crash log generated from Kernel userstacks parch_s = kern.arch osversion = "iOS" osversion += " ({:s})".format(kern.globals.osversion) - print crash_report_format_string.format(pid = pval.p_pid, - pname = pval.p_comm, - path = pval.p_comm, - ppid = pval.p_ppid, - ppname = GetProcNameForPid(pval.p_ppid), + if pval: + pid = pval.p_pid + pname = pval.p_comm + path = pval.p_comm + ppid = pval.p_ppid + else: + pid = 0 + pname = "unknown" + path = "unknown" + ppid = 0 + + print crash_report_format_string.format(pid = pid, + pname = pname, + path = path, + ppid = ppid, + ppname = GetProcNameForPid(ppid), timest = date_string, parch = parch_s, osversion = osversion - ) print "Binary Images:" ShowTaskUserLibraries([hex(task)]) @@ -313,7 +326,7 @@ Synthetic crash log generated from Kernel userstacks printthread_user_stack_ptr = ShowX86UserStack if kern.arch == "arm": printthread_user_stack_ptr = ShowARMUserStack - elif kern.arch =="arm64": + elif kern.arch.startswith("arm64"): printthread_user_stack_ptr = ShowARM64UserStack counter = 0 @@ -383,7 +396,7 @@ def GetUserDataAsString(task, addr, size): if not WriteInt64ToMemoryAddress(0, kdp_pmap_addr): debuglog("Failed to reset in kdp_pmap from GetUserDataAsString.") return "" - elif kern.arch in ['arm', 'arm64', 'x86_64'] and long(size) < (2 * kern.globals.page_size): + elif (kern.arch == 'x86_64' or kern.arch.startswith('arm')) and (long(size) < (2 * kern.globals.page_size)): # Without the benefit of a KDP stub on the target, try to # find the user task's physical mapping and memcpy the data. # If it straddles a page boundary, copy in two passes @@ -881,6 +894,9 @@ def SaveKCDataToFile(cmd_args=None, cmd_options={}): memory_data = GetUserDataAsString(task, memory_begin_address, memory_size) else: data_ptr = kern.GetValueFromAddress(memory_begin_address, 'uint8_t *') + if data_ptr == 0: + print "Kcdata descriptor is NULL" + return False memory_data = [] for i in range(memory_size): memory_data.append(chr(data_ptr[i])) diff --git a/tools/lldbmacros/usertaskdebugging/__init__.py b/tools/lldbmacros/usertaskdebugging/__init__.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/usertaskdebugging/gdbserver.py b/tools/lldbmacros/usertaskdebugging/gdbserver.py old mode 100644 new mode 100755 index 19b871adb..53f788b14 --- a/tools/lldbmacros/usertaskdebugging/gdbserver.py +++ b/tools/lldbmacros/usertaskdebugging/gdbserver.py @@ -39,7 +39,7 @@ class GDBServer(object): p_end = readBytes.find('#') if p_begin >= 0 and p_end >= 0 and p_end > p_begin: break - #if empty message or acks just ignore + # ignore if empty or ack messages if readBytes in ('', '+'): logging.debug('ignoring message: %s' % readBytes) continue diff --git a/tools/lldbmacros/usertaskdebugging/interface.py b/tools/lldbmacros/usertaskdebugging/interface.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/usertaskdebugging/rsprotocol.py b/tools/lldbmacros/usertaskdebugging/rsprotocol.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/usertaskdebugging/target.py b/tools/lldbmacros/usertaskdebugging/target.py old mode 100644 new mode 100755 index 9a2059268..36f8ed8ca --- a/tools/lldbmacros/usertaskdebugging/target.py +++ b/tools/lldbmacros/usertaskdebugging/target.py @@ -25,7 +25,7 @@ class Process(object): 'default_packet_timeout': '10', 'distribution_id': None } - # if cputype is arm assume its ios + ## if cputype is arm assume its ios if (cputype & 0xc) != 0xc: self.hinfo['ostype'] = 'ios' self.ptrsize = ptrsize diff --git a/tools/lldbmacros/usertaskdebugging/userprocess.py b/tools/lldbmacros/usertaskdebugging/userprocess.py old mode 100644 new mode 100755 index c0698a25a..5a5079e7b --- a/tools/lldbmacros/usertaskdebugging/userprocess.py +++ b/tools/lldbmacros/usertaskdebugging/userprocess.py @@ -5,6 +5,7 @@ import struct from xnu import * from core.operating_system import Armv8_RegisterSet, Armv7_RegisterSet, I386_RegisterSet, X86_64RegisterSet +""" these defines should come from an authoritative header file """ CPU_TYPE_I386 = 0x00000007 CPU_TYPE_X86_64 = 0x01000007 CPU_TYPE_ARM = 0x0000000c @@ -20,14 +21,16 @@ CPU_SUBTYPE_ARM_V7K = 12 def GetRegisterSetForCPU(cputype, subtype): - retval = X86_64RegisterSet - if cputype in (CPU_TYPE_ARM, CPU_TYPE_ARM64): - if subtype == CPU_SUBTYPE_ARMV8: - retval = Armv8_RegisterSet - else: - retval = Armv7_RegisterSet + if cputype == CPU_TYPE_ARM64: + retval = Armv8_RegisterSet + elif cputype == CPU_TYPE_ARM: + retval = Armv7_RegisterSet elif cputype == CPU_TYPE_I386: retval = I386_RegisterSet + elif cputype == CPU_TYPE_X86_64: + retval = X86_64RegisterSet + + """ crash if unknown cputype """ return retval.register_info['registers'] @@ -110,6 +113,9 @@ class UserProcess(target.Process): cputype = CPU_TYPE_X86_64 cpusubtype = CPU_SUBTYPE_X86_64_ALL + + """ these computations should come out of the macho header i think """ + """ where does kern.arch come from? what's kern.arch == armv8?? """ if kern.arch in ('arm'): cputype = CPU_TYPE_ARM cpusubtype = CPU_SUBTYPE_ARM_V7 diff --git a/tools/lldbmacros/usertaskgdbserver.py b/tools/lldbmacros/usertaskgdbserver.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/utils.py b/tools/lldbmacros/utils.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/waitq.py b/tools/lldbmacros/waitq.py old mode 100644 new mode 100755 diff --git a/tools/lldbmacros/xnu.py b/tools/lldbmacros/xnu.py old mode 100644 new mode 100755 index b8bcc6bed..e9ccfe7f3 --- a/tools/lldbmacros/xnu.py +++ b/tools/lldbmacros/xnu.py @@ -1,4 +1,4 @@ -import sys, subprocess, os, re, time, getopt, shlex +import sys, subprocess, os, re, time, getopt, shlex, xnudefines import lldb from functools import wraps from ctypes import c_ulonglong as uint64_t @@ -12,15 +12,24 @@ from core.kernelcore import * from utils import * from core.lazytarget import * -MODULE_NAME=__name__ +MODULE_NAME=__name__ """ Kernel Debugging macros for lldb. Please make sure you read the README COMPLETELY BEFORE reading anything below. - It is very critical that you read coding guidelines in Section E in README file. + It is very critical that you read coding guidelines in Section E in README file. """ +COMMON_HELP_STRING = """ + -h Show the help string for the command. + -o The output of this command execution will be saved to file. Parser information or errors will + not be sent to file though. eg /tmp/output.txt + -s The "filter_string" param is parsed to python regex expression and each line of output + will be printed/saved only if it matches the expression. + -v [-v...] Each additional -v will increase the verbosity of the command. + -p Send the output of the command to plugin. Please see README for usage of plugins. +""" # End Utility functions -# Debugging specific utility functions +# Debugging specific utility functions #decorators. Not to be called directly. @@ -147,6 +156,7 @@ However, it is recommended that you report the exception to lldb/kernel debuggin if not obj.__doc__ : print "ERROR: Cannot register command({:s}) without documentation".format(cmd_name) return obj + obj.__doc__ += "\n" + COMMON_HELP_STRING command_function.__doc__ = obj.__doc__ global lldb_command_documentation if cmd_name in lldb_command_documentation: @@ -298,7 +308,7 @@ def GetKextSymbolInfo(load_addr): symbol_name = "None" symbol_offset = load_addr kmod_val = kern.globals.kmod - if kern.arch not in ('arm64',): + if not kern.arch.startswith('arm64'): for kval in IterateLinkedList(kmod_val, 'next'): if load_addr >= unsigned(kval.address) and \ load_addr <= (unsigned(kval.address) + unsigned(kval.size)): @@ -450,18 +460,9 @@ def KernelDebugCommandsHelp(cmd_args=None): print " {0: <20s} - {1}".format(cmd , lldb_command_documentation[cmd][1].split("\n")[0].strip()) else: print " {0: <20s} - {1}".format(cmd , "No help string found.") - print """ - Each of the functions listed here accept the following common options. - -h Show the help string for the command. - -o The output of this command execution will be saved to file. Parser information or errors will - not be sent to file though. eg /tmp/output.txt - -s The "filter_string" param is parsed to python regex expression and each line of output - will be printed/saved only if it matches the expression. - -v [-v...] Each additional -v will increase the verbosity of the command. - -p Send the output of the command to plugin. Please see README for usage of plugins. - - Additionally, each command implementation may have more options. "(lldb) help " will show these options. - """ + print 'Each of the functions listed here accept the following common options. ' + print COMMON_HELP_STRING + print 'Additionally, each command implementation may have more options. "(lldb) help " will show these options.' return None @@ -572,65 +573,91 @@ def ShowPanicLog(cmd_args=None, cmd_options={}): -v : increase verbosity -S : parse stackshot data (if panic stackshot available) """ - binary_data_bytes_to_skip = 0 - if hasattr(kern.globals, "kc_panic_data"): - binary_data_bytes_to_skip = unsigned(kern.globals.kc_panic_data.kcd_addr_end) - unsigned(kern.globals.kc_panic_data.kcd_addr_begin) - if binary_data_bytes_to_skip > 0: - binary_data_bytes_to_skip += sizeof("struct kcdata_item") - else: - binary_data_bytes_to_skip = 0 if "-S" in cmd_options: if hasattr(kern.globals, "kc_panic_data"): - kc_data = unsigned(addressof(kern.globals.kc_panic_data)) - ts = int(time.time()) - ss_binfile = "/tmp/panic_%d.bin" % ts - ss_ipsfile = "/tmp/stacks_%d.ips" % ts - print "savekcdata 0x%x -O %s" % (kc_data, ss_binfile) - SaveKCDataToFile(["0x%x" % kc_data], {"-O":ss_binfile}) - self_path = str(__file__) - base_dir_name = self_path[:self_path.rfind("/")] - print "python %s/kcdata.py %s -s %s" % (base_dir_name, ss_binfile, ss_ipsfile) - (c,so,se) = RunShellCommand("python %s/kcdata.py %s -s %s" % (base_dir_name, ss_binfile, ss_ipsfile)) - if c == 0: - print "Saved ips stackshot file as %s" % ss_ipsfile + stackshot_saved = False + if kern.arch == 'x86_64': + if kern.globals.panic_stackshot_len != 0: + stackshot_saved = True + else: + print "No panic stackshot available" else: - print "Failed to run command: exit code: %d, SO: %s SE: %s" % (c, so, se) + if unsigned(kern.globals.panic_info.eph_panic_flags) & xnudefines.EMBEDDED_PANIC_STACKSHOT_SUCCEEDED_FLAG: + stackshot_saved = True + else: + print "No panic stackshot available" + if stackshot_saved: + kc_data = unsigned(addressof(kern.globals.kc_panic_data)) + ts = int(time.time()) + ss_binfile = "/tmp/panic_%d.bin" % ts + ss_ipsfile = "/tmp/stacks_%d.ips" % ts + print "savekcdata 0x%x -O %s" % (kc_data, ss_binfile) + SaveKCDataToFile(["0x%x" % kc_data], {"-O":ss_binfile}) + self_path = str(__file__) + base_dir_name = self_path[:self_path.rfind("/")] + print "python %s/kcdata.py %s -s %s" % (base_dir_name, ss_binfile, ss_ipsfile) + (c,so,se) = RunShellCommand("python %s/kcdata.py %s -s %s" % (base_dir_name, ss_binfile, ss_ipsfile)) + if c == 0: + print "Saved ips stackshot file as %s" % ss_ipsfile + else: + print "Failed to run command: exit code: %d, SO: %s SE: %s" % (c, so, se) else: print "kc_panic_data is unavailable for this kernel config." - panic_buf = kern.globals.debug_buf_addr - panic_buf_start = unsigned(panic_buf) - panic_buf_end = unsigned(kern.globals.debug_buf_ptr) - num_bytes = panic_buf_end - panic_buf_start - if num_bytes == 0 : - return out_str = "" warn_str = "" - num_print_bytes = 0 - in_binary_data_region = False - pos = 0 - while pos < num_bytes: - p_char = str(panic_buf[pos]) - out_str += p_char - if p_char == '\n': - if not in_binary_data_region: - num_print_bytes += 1 - print out_str[:-1] - if (out_str.find("Data: BEGIN>>") >= 0): - in_binary_data_region = True - pos += binary_data_bytes_to_skip - 1 - if (out_str.find("<= 0): - in_binary_data_region = False - out_str = "" - if num_print_bytes > 4096 and config['verbosity'] == vHUMAN: - warn_str = "LLDBMacro Warning: The paniclog is too large. Trimming to 4096 bytes." - warn_str += " If you wish to see entire log please use '-v' argument." - break - pos += 1 + + if kern.arch == 'x86_64': + panic_buf = kern.globals.debug_buf_base + panic_buf_start = unsigned(panic_buf) + panic_buf_end = unsigned(kern.globals.debug_buf_ptr) + num_bytes = panic_buf_end - panic_buf_start + if num_bytes == 0: + return + num_print_bytes = 0 + pos = 0 + while pos < num_bytes: + p_char = str(panic_buf[pos]) + out_str += p_char + pos += 1 + else: + panic_buf = Cast(kern.globals.panic_info, 'char *') + panic_log_magic = unsigned(kern.globals.panic_info.eph_magic) + panic_log_begin_offset = unsigned(kern.globals.panic_info.eph_panic_log_offset) + panic_log_len = unsigned(kern.globals.panic_info.eph_panic_log_len) + other_log_begin_offset = unsigned(kern.globals.panic_info.eph_other_log_offset) + other_log_len = unsigned(kern.globals.panic_info.eph_other_log_len) + + if panic_log_begin_offset == 0: + return + + if panic_log_magic != 0 and panic_log_magic != xnudefines.EMBEDDED_PANIC_MAGIC: + warn_str += "BAD MAGIC! Found 0x%x expected 0x%x".format(panic_log_magic, + xnudefines.EMBEDDED_PANIC_MAGIC) + + if panic_log_begin_offset == 0: + if warn_str: + print "\n %s" % warn_str + return + + panic_log_curindex = 0 + while panic_log_curindex < panic_log_len: + p_char = str(panic_buf[(panic_log_begin_offset + panic_log_curindex)]) + out_str += p_char + panic_log_curindex += 1 + + if other_log_begin_offset != 0: + other_log_curindex = 0 + while other_log_curindex < other_log_len: + p_char = str(panic_buf[(other_log_begin_offset + other_log_curindex)]) + out_str += p_char + other_log_curindex += 1 + + print out_str if warn_str: - print warn_str + print "\n %s" % warn_str return @@ -784,9 +811,13 @@ from atm import * from structanalyze import * from ipcimportancedetail import * from bank import * +from kasan import * from kauth import * from waitq import * from usertaskgdbserver import * from ktrace import * from pgtrace import * from xnutriage import * +from kevent import * +from ntstat import * + diff --git a/tools/lldbmacros/xnudefines.py b/tools/lldbmacros/xnudefines.py old mode 100644 new mode 100755 index 9aa59c68f..38be2b393 --- a/tools/lldbmacros/xnudefines.py +++ b/tools/lldbmacros/xnudefines.py @@ -27,12 +27,15 @@ def GetStateString(strings_dict, state): mask = 0x1 while mask <= max_mask: bit = int(state & mask) - if bit and bit in strings_dict: - if not first: - output += ' ' + if bit: + if bit in strings_dict: + if not first: + output += ' ' + else: + first = False + output += strings_dict[int(state & mask)] else: - first = False - output += strings_dict[int(state & mask)] + output += '{:#x}'.format(mask) mask = mask << 1 return output @@ -66,9 +69,77 @@ arm_level2_access_strings = [ " noaccess", " supervisor(readonly) user(readonly)", " " ] -kq_state_strings = {0:"", 1:"SEL", 2:"SLEEP", 4:"PROCWAIT", 8:"KEV32", 16:"KEV64", 32:"QOS", 64:"WORKQ", 128:"PROCESS", 256: "DRAIN"} -kn_state_strings = {0:"", 1:"ACTIVE", 2:"QUEUED", 4:"DISABLED", 8:"DROPPING", 16:"USERWAIT", 32:"ATTACHING", 64:"STAYQUED", 128:"DEFERDROP"} +kq_state_strings = { 0x000: '', + 0x001: 'SEL', + 0x002: 'SLEEP', + 0x004: 'PROCWAIT', + 0x008: 'KEV32', + 0x010: 'KEV64', + 0x020: 'KEVQOS', + 0x040: 'WORKQ', + 0x080: 'WORKLOOP', + 0x100: 'PROCESS', + 0x200: 'DRAIN', + 0x400: 'WAKEUP' } + +kn_state_strings = { 0x0000: '', + 0x0001: 'ACTIVE', + 0x0002: 'QUEUED', + 0x0004: 'DISABLED', + 0x0008: 'DROPPING', + 0x0010: 'USERWAIT', + 0x0020: 'ATTACHING', + 0x0040: 'STAYACTIVE', + 0x0080: 'DEFERDROP', + 0x0100: 'ATTACHED', + 0x0200: 'DISPATCH', + 0x0400: 'UDATASPEC', + 0x0800: 'SUPPRESS', + 0x1000: 'STOLENDROP', + 0x2000: 'REQVANISH', + 0x4000: 'VANISHED' } + +kqrequest_state_strings = { 0x01: 'PROCESSING', + 0x02: 'THREQUESTED', + 0x04: 'WAKEUP', + 0x08: 'BOUND', + 0x20: 'THOVERCOMMIT', + 0x40: 'DRAIN' } + +thread_qos_short_strings = { 0: '--', + 1: 'MT', + 2: 'BG', + 3: 'UT', + 4: 'DF', + 5: 'IN', + 6: 'UI', + 7: 'MG' } + +KQ_WORKQ = 0x40 +KQ_WORKLOOP = 0x80 +KQWQ_NBUCKETS = 22 +KQWL_NBUCKETS = 8 + +DTYPE_VNODE = 1 +DTYPE_SOCKET = 2 +DTYPE_PSXSHM = 3 +DTYPE_PSXSEM = 4 +DTYPE_KQUEUE = 5 +DTYPE_PIPE = 6 +DTYPE_FSEVENTS = 7 +DTYPE_ATALK = 8 +DTYPE_NETPOLICY = 9 +filetype_strings = { DTYPE_VNODE: 'VNODE', + DTYPE_SOCKET: 'SOCKET', + DTYPE_PSXSHM: 'PSXSHM', + DTYPE_PSXSEM: 'PSXSEM', + DTYPE_KQUEUE: 'KQUEUE', + DTYPE_PIPE: 'PIPE', + DTYPE_FSEVENTS: 'FSEVENTS', + DTYPE_ATALK: 'APLTALK', + DTYPE_NETPOLICY: 'NETPOLI' + } mach_msg_type_descriptor_strings = {0: "PORT", 1: "OOLDESC", 2: "OOLPORTS", 3: "OOLVOLATILE"} @@ -111,7 +182,7 @@ proc_flag_explain_strings = ["!0x00000004 - process is 32 bit", #only exception # string representations for Kobject types kobject_types = ['', 'THREAD', 'TASK', 'HOST', 'HOST_PRIV', 'PROCESSOR', 'PSET', 'PSET_NAME', 'TIMER', 'PAGER_REQ', 'DEVICE', 'XMM_OBJECT', 'XMM_PAGER', 'XMM_KERNEL', 'XMM_REPLY', 'NOTDEF 15', 'NOTDEF 16', 'HOST_SEC', 'LEDGER', 'MASTER_DEV', 'TASK_NAME', 'SUBSYTEM', 'IO_DONE_QUE', 'SEMAPHORE', 'LOCK_SET', 'CLOCK', 'CLOCK_CTRL' , 'IOKIT_SPARE', - 'NAMED_MEM', 'IOKIT_CON', 'IOKIT_OBJ', 'UPL', 'MEM_OBJ_CONTROL', 'AU_SESSIONPORT', 'FILEPORT', 'LABELH', 'TASK_RESUME', 'VOUCHER', 'VOUCHER_ATTR_CONTROL'] + 'NAMED_MEM', 'IOKIT_CON', 'IOKIT_OBJ', 'UPL', 'MEM_OBJ_CONTROL', 'AU_SESSIONPORT', 'FILEPORT', 'LABELH', 'TASK_RESUME', 'VOUCHER', 'VOUCHER_ATTR_CONTROL', 'IKOT_WORK_INTERVAL'] def populate_kobject_types(xnu_dir_path): """ Function to read data from header file xnu/osfmk/kern/ipc_kobject.h @@ -125,6 +196,19 @@ def populate_kobject_types(xnu_dir_path): kobject_found_types.append(v[0]) return kobject_found_types +KDBG_BFINIT = 0x80000000 +KDBG_WRAPPED = 0x008 +KDCOPYBUF_COUNT = 8192 +KDS_PTR_NULL = 0xffffffff + +DBG_TRACE = 1 +DBG_TRACE_INFO = 2 +RAW_VERSION1 = 0x55aa0101 +EVENTS_PER_STORAGE_UNIT = 2048 + +EMBEDDED_PANIC_MAGIC = 0x46554E4B +EMBEDDED_PANIC_STACKSHOT_SUCCEEDED_FLAG = 0x02 + if __name__ == "__main__": populate_kobject_types("../../") diff --git a/tools/lldbmacros/xnutriage.py b/tools/lldbmacros/xnutriage.py old mode 100644 new mode 100755 index ae4050faf..0783ebea0 --- a/tools/lldbmacros/xnutriage.py +++ b/tools/lldbmacros/xnutriage.py @@ -56,7 +56,7 @@ def NewBt(cmd_args=None): return False a = ArgumentStringToInt(cmd_args[0]) while a != 0: - if kern.arch == "x86_64" or kern.arch == "arm64": + if kern.arch == "x86_64" or kern.arch.startswith("arm64"): offset = 8 else: offset = 4 diff --git a/tools/stackshot/Makefile b/tools/stackshot/Makefile deleted file mode 100644 index 97bdca25f..000000000 --- a/tools/stackshot/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -ifndef SDKROOT -SDKROOT := macosx.internal -endif - -SDKPATH := $(shell xcrun -sdk $(SDKROOT) -show-sdk-path) - -ifndef SRCROOT - SRCROOT := $(CURDIR)/../.. -endif - -ifndef OBJROOT - OBJROOT:=$(CURDIR) -endif - -ARCHES := $(shell file $(SDKPATH)/usr/lib/libc.dylib | perl -lne 'print "-arch $$1" if /for architecture ([\w_]+)/') - -CFLAGS := $(ARCHES) -I $(SDKPATH)/System/Library/Frameworks/System.framework/PrivateHeaders - -# -I $(SRCROOT)/kcdata -iprefix kern/ -iwithprefix $(SRCROOT)/osfmk/kern -#-I $(SRCROOT)/bsd -# - -$(OBJROOT)/stackshot: stackshot.c - xcrun -sdk $(SDKROOT) clang $(CFLAGS) $< -o $@ diff --git a/tools/stackshot/stackshot.c b/tools/stackshot/stackshot.c deleted file mode 100644 index 854e00a7d..000000000 --- a/tools/stackshot/stackshot.c +++ /dev/null @@ -1,192 +0,0 @@ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#define STACKSHOT_TAILSPIN (0x80000) - -uint64_t -stackshot_get_mach_absolute_time(void *buffer, uint32_t size) -{ - kcdata_iter_t iter = kcdata_iter_find_type(kcdata_iter(buffer, size), KCDATA_TYPE_MACH_ABSOLUTE_TIME); - if (!kcdata_iter_valid(iter) || kcdata_iter_size(iter) < sizeof(uint64_t)) { - fprintf(stderr, "bad kcdata\n"); - exit(1); - } - return *(uint64_t *)kcdata_iter_payload(iter); -} - -static void usage(char **argv) -{ - fprintf (stderr, "usage: %s [-d] [-t] >file\n", argv[0]); - fprintf (stderr, " -d : take delta stackshot\n"); - fprintf (stderr, " -b : get bootprofile\n"); - fprintf (stderr, " -t : enable tailspin mode\n"); - fprintf (stderr, " -s : fork a sleep process\n"); - fprintf (stderr, " -L : disable loadinfo\n"); - fprintf (stderr, " -k : active kernel threads only\n"); - fprintf (stderr, " -I : disable io statistics\n"); - fprintf (stderr, " -p PID : target a pid\n"); - exit(1); -} - -void forksleep() { - pid_t pid = fork(); - if (pid < 0) { - perror("fork"); - exit(1); - } - - if (pid == 0) { - execlp("sleep", "sleep", "30", NULL); - perror("execlp"); - exit(1); - } -} - - -int main(int argc, char **argv) { - - uint32_t iostats = 0; - uint32_t active_kernel_threads_only = 0; - uint32_t tailspin = 0; - uint32_t bootprofile = 0; - uint32_t loadinfo = STACKSHOT_SAVE_LOADINFO | STACKSHOT_SAVE_KEXT_LOADINFO; - boolean_t delta = FALSE; - boolean_t sleep = FALSE; - pid_t pid = -1; - int c; - - while ((c = getopt(argc, argv, "IkbLdtsp:")) != EOF) { - switch(c) { - case 'I': - iostats |= STACKSHOT_NO_IO_STATS; - break; - case 'k': - active_kernel_threads_only |= STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY; - loadinfo &= ~STACKSHOT_SAVE_LOADINFO; - break; - case 'b': - bootprofile |= STACKSHOT_GET_BOOT_PROFILE; - break; - case 'L': - loadinfo = 0; - break; - case 't': - tailspin |= STACKSHOT_TAILSPIN; - break; - case 'd': - delta = TRUE; - break; - case 's': - sleep = TRUE; - break; - case 'p': - pid = atoi(optarg); - break; - case '?': - case 'h': - default: - usage(argv); - break; - } - } - - if (optind < argc) - { - usage(argv); - } - - void * config = stackshot_config_create(); - if (!config) { - perror("stackshot_config_create"); - return 1; - } - uint32_t flags = loadinfo | STACKSHOT_SAVE_IMP_DONATION_PIDS | STACKSHOT_GET_DQ | STACKSHOT_KCDATA_FORMAT | - tailspin | bootprofile | active_kernel_threads_only | iostats; - - int err = stackshot_config_set_flags(config, flags); - if (err != 0) { - perror("stackshot_config_set_flags"); - return 1; - } - - if (pid != -1) { - int err = stackshot_config_set_pid(config, pid); - if (err != 0) { - perror("stackshot_config_set_flags"); - return 1; - } - } - - err = stackshot_capture_with_config(config); - if (err != 0) { - perror("stackshot_capture_with_config"); - return 1; - } - - void *buf = stackshot_config_get_stackshot_buffer(config); - if (!buf) { - perror("stackshot_config_get_stackshot_buffer"); - return 1; - } - - uint32_t size = stackshot_config_get_stackshot_size(config); - - if (delta) { - // output the original somewhere? - - uint64_t time = stackshot_get_mach_absolute_time(buf, size); - - err = stackshot_config_dealloc_buffer(config); - assert(!err); - - flags |= STACKSHOT_COLLECT_DELTA_SNAPSHOT; - int err = stackshot_config_set_flags(config, flags); - if (err != 0) { - perror("stackshot_config_set_flags"); - return 1; - } - - err = stackshot_config_set_delta_timestamp(config, time); - if (err != 0) { - perror("stackshot_config_delta_timestamp"); - return 1; - } - - if (sleep) { - forksleep(); - } - usleep(10000); - - err = stackshot_capture_with_config(config); - if (err != 0) { - perror("stackshot_capture_with_config"); - return 1; - } - - buf = stackshot_config_get_stackshot_buffer(config); - if (!buf) { - perror("stackshot_config_get_stackshot_buffer"); - return 1; - } - - size = stackshot_config_get_stackshot_size(config); - - } - - fwrite(buf, size, 1, stdout); -} diff --git a/tools/tests/MPMMTest/MPMMtest_run.sh b/tools/tests/MPMMTest/MPMMtest_run.sh index 95f4fb2a7..8b3f8b899 100755 --- a/tools/tests/MPMMTest/MPMMtest_run.sh +++ b/tools/tests/MPMMTest/MPMMtest_run.sh @@ -20,8 +20,16 @@ IS_64BIT_BOOTED_OS=$? if [ -e $MPMMTEST ] && [ -x $MPMMTEST ] then - echo ""; echo " Running $MPMMTEST"; - $MPMMTEST -perf || { x=$?; echo "$MPMMTEST failed $x "; exit $x; } + # Tentatively test for 32-bit support + $MPMMTEST -count 0 -servers 0 -clients 0 &> /dev/null + + if [ $? == 0 ] || [ $IS_64BIT_BOOTED_OS == 0 ] + then + # If 32-bit support appears to be present OR if this is not + # a 64-bit environment, run the test. + echo ""; echo " Running $MPMMTEST"; + $MPMMTEST -perf || { x=$?; echo "$MPMMTEST failed $x "; exit $x; } + fi fi if [ -e $MPMMTEST_64 ] && [ -x $MPMMTEST_64 ] && [ $IS_64BIT_BOOTED_OS == 1 ] @@ -32,8 +40,16 @@ fi if [ -e $KQMPMMTEST ] && [ -x $KQMPMMTEST ] then - echo ""; echo " Running $KQMPMMTEST" - $KQMPMMTEST -perf || { x=$?; echo "$KQMPMMTEST failed $x"; exit $x; } + # Tentatively test for 32-bit support + $KQMPMMTEST -count 0 -servers 0 -clients 0 &> /dev/null + + if [ $? == 0 ] || [ $IS_64BIT_BOOTED_OS == 0 ] + then + # If 32-bit support appears to be present OR if this is not + # a 64-bit environment, run the test. + echo ""; echo " Running $KQMPMMTEST" + $KQMPMMTEST -perf || { x=$?; echo "$KQMPMMTEST failed $x"; exit $x; } + fi fi if [ -e $KQMPMMTEST_64 ] && [ -x $KQMPMMTEST_64 ] && [ $IS_64BIT_BOOTED_OS == 1 ] diff --git a/tools/tests/Makefile b/tools/tests/Makefile index ce8f8ec54..0cc99e0dc 100644 --- a/tools/tests/Makefile +++ b/tools/tests/Makefile @@ -26,7 +26,6 @@ COMMON_TARGETS = unit_tests \ packetdrill \ affinity \ execperf \ - kqueue_tests \ superpages \ zero-to-n \ jitter \ @@ -65,9 +64,10 @@ $(DSTSUBPATH)/%: always $(_v)mkdir -p $(OBJROOT)/$(notdir $@) $(_v)mkdir -p $(SYMROOT)/$(notdir $@) $(_v)$(MAKE) -C $(SRCROOT)/$(notdir $@) SRCROOT=$(SRCROOT)/$(notdir $@) DSTROOT=$@ OBJROOT=$(OBJROOT)/$(notdir $@) SYMROOT=$(SYMROOT)/$(notdir $@) SDKROOT=$(SDKROOT) BASEDSTROOT=$(DSTROOT) + $(_v)find $@ -type d -name "*.dSYM" -exec cp -a {} $(SYMROOT)/ ';' $(BATS_TARGET): $(TARGETS) $(_v)mkdir -p $@ $(_v)mkdir -p $(OBJROOT)/$(notdir $@) $(_v)mkdir -p $(SYMROOT) - $(_v)$(MAKE) -C $(SRCROOT)/$(notdir $@) SRCROOT=$(SRCROOT)/$(notdir $@) DSTROOT=$@ OBJROOT=$(OBJROOT)/$(notdir $@) SDKROOT=$(SDKROOT) BASEDSTROOT=$(DSTROOT) + $(_v)$(MAKE) -C $(SRCROOT)/$(notdir $@) SRCROOT=$(SRCROOT)/$(notdir $@) DSTROOT=$@ OBJROOT=$(OBJROOT)/$(notdir $@) SDKROOT=$(SDKROOT) BASEDSTROOT=$(DSTROOT) BASEDSTSUBPATH=$(DSTSUBPATH) diff --git a/tools/tests/Makefile.common b/tools/tests/Makefile.common index 06b7e36f1..1d80fb11e 100644 --- a/tools/tests/Makefile.common +++ b/tools/tests/Makefile.common @@ -23,7 +23,7 @@ ifeq ($(PLATFORM),watchOS) PLATFORM := WatchOS endif -SUPPORTED_EMBEDDED_PLATFORMS := iPhoneOS iPhoneOSNano tvOS AppleTVOS WatchOS +SUPPORTED_EMBEDDED_PLATFORMS := iPhoneOS iPhoneOSNano tvOS AppleTVOS WatchOS BridgeOS Embedded = $(if $(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),YES,NO) # @@ -37,6 +37,8 @@ else ifeq ($(PLATFORM),tvOS) DEPLOYMENT_TARGET_FLAGS = -mtvos-version-min=$(SDKVERSION) else ifeq ($(PLATFORM),AppleTVOS) DEPLOYMENT_TARGET_FLAGS = -mtvos-version-min=$(SDKVERSION) +else ifeq ($(PLATFORM),BridgeOS) + DEPLOYMENT_TARGET_FLAGS = -mbridgeos-version-min=$(SDKVERSION) else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) DEPLOYMENT_TARGET_FLAGS = -miphoneos-version-min=$(SDKVERSION) else ifneq ($(filter $(SUPPORTED_SIMULATOR_PLATFORMS),$(PLATFORM)),) diff --git a/tools/tests/darwintests/Makefile b/tools/tests/darwintests/Makefile index 760e51545..a6f4012ab 100644 --- a/tools/tests/darwintests/Makefile +++ b/tools/tests/darwintests/Makefile @@ -6,6 +6,14 @@ ifdef BASEDSTROOT override DSTROOT = $(BASEDSTROOT) endif +ENABLE_LTE_TESTS=YES + +OTHER_LTE_INCLUDE_FILES += \ + /System/Library/PrivateFrameworks/LoggingSupport.framework, \ + /System/Library/PrivateFrameworks/MobileKeyBag.framework, \ + /usr/local/lib/libdarwintest_utils.dylib, \ + /usr/lib/libapple_crypto.dylib, + DEVELOPER_DIR ?= /Applications/Xcode.app/Contents/Developer/ # the xnu build system will only ever call us with the default target @@ -15,9 +23,10 @@ include $(DEVELOPER_DIR)/AppleInternal/Makefiles/darwintest/Makefile.common OTHER_CFLAGS = -Weverything -Wno-gnu-union-cast -Wno-missing-field-initializers -Wno-partial-availability OTHER_CFLAGS += -Wno-missing-noreturn -Wno-vla -Wno-reserved-id-macro -Wno-documentation-unknown-command -OTHER_CFLAGS += -Wno-padded -Wno-used-but-marked-unused -Wno-covered-switch-default +OTHER_CFLAGS += -Wno-padded -Wno-used-but-marked-unused -Wno-covered-switch-default -Wno-nullability-extension OTHER_CFLAGS += --std=gnu11 -isystem $(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders -OTHER_CFLAGS += -DT_NAMESPACE=xnu +OTHER_CFLAGS += -DT_NAMESPACE_PREFIX=xnu +OTHER_CFLAGS += -F $(SDKROOT)/System/Library/PrivateFrameworks CODESIGN:=$(shell xcrun -sdk "$(TARGETSDK)" -find codesign) CODESIGN_ALLOCATE:=$(shell xcrun -sdk "$(TARGETSDK)" -find codesign_allocate) @@ -25,15 +34,25 @@ CODESIGN_ALLOCATE:=$(shell xcrun -sdk "$(TARGETSDK)" -find codesign_allocate) # to have custom compiler flags to # target: OTHER_CFLAGS += -backtracing: OTHER_CFLAGS += -F $(SDKROOT)/System/Library/PrivateFrameworks +avx: INVALID_ARCHS = i386 +avx: OTHER_CFLAGS += -mavx512f -mavx512bw -mavx512vl +avx: OTHER_CFLAGS += -I$(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders +avx: CONFIG_FLAGS := $(filter-out -O%,$(CONFIG_FLAGS)) +# Level 2 optimization must be used to prevent compiler from generating +# invalid instructions when compiling with AVX-512 flags. +avx: CONFIG_FLAGS += -O2 +ifneq (osx,$(TARGET_NAME)) +EXCLUDED_SOURCES += avx.c +endif + backtracing: OTHER_LDFLAGS += -framework CoreSymbolication data_protection: OTHER_LDFLAGS += -framework IOKit kdebug: INVALID_ARCHS = i386 -kdebug: OTHER_LDFLAGS = -lktrace +kdebug: OTHER_LDFLAGS = -framework ktrace -EXCLUDED_SOURCES += kperf_helpers.c +EXCLUDED_SOURCES += kperf_helpers.c xnu_quick_test_helpers.c ifeq ($(PLATFORM),iPhoneOS) CONFIG_FREEZE_DEFINE:= -DCONFIG_FREEZE @@ -43,22 +62,43 @@ EXCLUDED_SOURCES += jumbo_va_spaces_28530648.c endif perf_compressor: OTHER_CFLAGS += $(CONFIG_FREEZE_DEFINE) +perf_compressor: CODE_SIGN_ENTITLEMENTS=./private_entitlement.plist + +stackshot: OTHER_LDFLAGS += -lkdd -framework Foundation +stackshot: INVALID_ARCHS = i386 + +memorystatus_zone_test: INVALID_ARCHS = i386 +memorystatus_zone_test: OTHER_CFLAGS += -isystem $(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders +memorystatus_zone_test: OTHER_LDFLAGS += -framework ktrace +memorystatus_zone_test: OTHER_LDFLAGS += -ldarwintest_utils + +kpc: OTHER_LDFLAGS += -framework kperf kperf: INVALID_ARCHS = i386 kperf: OTHER_CFLAGS += kperf_helpers.c -kperf: OTHER_CFLAGS += -F $(SDKROOT)/System/Library/PrivateFrameworks -kperf: OTHER_LDFLAGS += -framework kperf -framework kperfdata -lktrace +kperf: OTHER_LDFLAGS += -framework kperf -framework kperfdata -framework ktrace -ldarwintest_utils kperf_backtracing: INVALID_ARCHS = i386 kperf_backtracing: OTHER_CFLAGS += kperf_helpers.c -kperf_backtracing: OTHER_CFLAGS += -F $(SDKROOT)/System/Library/PrivateFrameworks -kperf_backtracing: OTHER_LDFLAGS += -framework kperf -framework kperfdata -lktrace +kperf_backtracing: OTHER_LDFLAGS += -framework kperf -framework kperfdata -framework ktrace kperf_backtracing: OTHER_LDFLAGS += -framework CoreSymbolication +kevent_qos: OTHER_CFLAGS += -Wno-unused-macros +kevent_qos: OTHER_CFLAGS += -I $(OBJROOT)/ + mach_get_times: OTHER_LDFLAGS += -ldarwintest_utils -perf_exit: OTHER_LDFLAGS = -lktrace +monotonic_core: OTHER_LDFLAGS += -framework ktrace +monotonic_core: INVALID_ARCHS = i386 + +perf_exit: OTHER_LDFLAGS = -framework ktrace perf_exit: INVALID_ARCHS = i386 +perf_exit: CODE_SIGN_ENTITLEMENTS=./private_entitlement.plist + +perf_spawn_fork: CODE_SIGN_ENTITLEMENTS=./private_entitlement.plist + +os_thread_self_restrict: os_thread_self_restrict.c os_thread_self_restrict-entitlements.plist +os_thread_self_restrict: CODE_SIGN_ENTITLEMENTS=os_thread_self_restrict-entitlements.plist task_inspect: CODE_SIGN_ENTITLEMENTS = task_inspect.entitlements task_inspect: OTHER_CFLAGS += -DENTITLED=1 @@ -80,6 +120,8 @@ stackshot_idle_25570396: OTHER_LDFLAGS += -lkdd -framework Foundation stackshot_block_owner_14362384: INVALID_ARCHS = i386 stackshot_block_owner_14362384: OTHER_LDFLAGS += -framework Foundation -lpthread -lkdd +xnu_quick_test: OTHER_CFLAGS += xnu_quick_test_helpers.c + ifeq ($(PLATFORM),iPhoneOS) OTHER_TEST_TARGETS += jumbo_va_spaces_28530648_unentitled jumbo_va_spaces_28530648: CODE_SIGN_ENTITLEMENTS = jumbo_va_spaces_28530648.entitlements @@ -91,4 +133,25 @@ jumbo_va_spaces_28530648_unentitled: jumbo_va_spaces_28530648.c $(CC) $(DT_CFLAGS) $(OTHER_CFLAGS) $(CFLAGS) $(DT_LDFLAGS) $(OTHER_LDFLAGS) $(LDFLAGS) $< -o $(SYMROOT)/$@ endif +task_info_28439149: CODE_SIGN_ENTITLEMENTS = ./task_for_pid_entitlement.plist + +disk_mount_conditioner: disk_mount_conditioner* +disk_mount_conditioner: CODE_SIGN_ENTITLEMENTS=./disk_mount_conditioner-entitlements.plist +disk_mount_conditioner: OTHER_LDFLAGS += -ldarwintest_utils + +OTHER_TEST_TARGETS += disk_mount_conditioner_unentitled +disk_mount_conditioner_unentitled: OTHER_CFLAGS += -DTEST_UNENTITLED +disk_mount_conditioner_unentitled: OTHER_LDFLAGS += -ldarwintest_utils +disk_mount_conditioner_unentitled: disk_mount_conditioner.c + $(CC) $(DT_CFLAGS) $(OTHER_CFLAGS) $(CFLAGS) $(DT_LDFLAGS) $(OTHER_LDFLAGS) $(LDFLAGS) $< -o $(SYMROOT)/$@ + +work_interval_test: CODE_SIGN_ENTITLEMENTS = work_interval_test.entitlements +work_interval_test: OTHER_CFLAGS += -DENTITLED=1 + +settimeofday_29193041_entitled: CODE_SIGN_ENTITLEMENTS = settimeofday_29193041.entitlements +thread_group_set_32261625: OTHER_LDFLAGS = -framework ktrace +thread_group_set_32261625: INVALID_ARCHS = i386 + +task_info: CODE_SIGN_ENTITLEMENTS = task_for_pid_entitlement.plist + include $(DEVELOPER_DIR)/AppleInternal/Makefiles/darwintest/Makefile.targets diff --git a/tools/tests/darwintests/avx.c b/tools/tests/darwintests/avx.c new file mode 100644 index 000000000..0041e999a --- /dev/null +++ b/tools/tests/darwintests/avx.c @@ -0,0 +1,736 @@ +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +T_GLOBAL_META( + T_META_NAMESPACE("xnu.intel"), + T_META_CHECK_LEAKS(false) +); + +#define NORMAL_RUN_TIME (10) +#define LONG_RUN_TIME (10*60) +#define TIMEOUT_OVERHEAD (10) + +volatile boolean_t checking = true; +char vec_str_buf[8196]; +char karray_str_buf[1024]; + +/* + * ymm defines/globals/prototypes + */ +#define STOP_COOKIE_256 0x01234567 +#if defined(__x86_64__) +#define YMM_MAX 16 +#define X86_AVX_STATE_T x86_avx_state64_t +#define X86_AVX_STATE_COUNT x86_AVX_STATE64_COUNT +#define X86_AVX_STATE_FLAVOR x86_AVX_STATE64 +#define MCONTEXT_SIZE_256 sizeof(struct __darwin_mcontext_avx64) +#else +#define YMM_MAX 8 +#define X86_AVX_STATE_T x86_avx_state32_t +#define X86_AVX_STATE_COUNT x86_AVX_STATE32_COUNT +#define X86_AVX_STATE_FLAVOR x86_AVX_STATE32 +#define MCONTEXT_SIZE_256 sizeof(struct __darwin_mcontext_avx32) +#endif +#define VECTOR256 __m256 +#define VEC256ALIGN __attribute ((aligned(32))) +static inline void populate_ymm(void); +static inline void check_ymm(void); +VECTOR256 vec256array0[YMM_MAX] VEC256ALIGN; +VECTOR256 vec256array1[YMM_MAX] VEC256ALIGN; +VECTOR256 vec256array2[YMM_MAX] VEC256ALIGN; +VECTOR256 vec256array3[YMM_MAX] VEC256ALIGN; + +/* + * zmm defines/globals/prototypes + */ +#define STOP_COOKIE_512 0x0123456789abcdefULL +#if defined(__x86_64__) +#define ZMM_MAX 32 +#define X86_AVX512_STATE_T x86_avx512_state64_t +#define X86_AVX512_STATE_COUNT x86_AVX512_STATE64_COUNT +#define X86_AVX512_STATE_FLAVOR x86_AVX512_STATE64 +#define MCONTEXT_SIZE_512 sizeof(struct __darwin_mcontext_avx512_64) +#else +#define ZMM_MAX 8 +#define X86_AVX512_STATE_T x86_avx512_state32_t +#define X86_AVX512_STATE_COUNT x86_AVX512_STATE32_COUNT +#define X86_AVX512_STATE_FLAVOR x86_AVX512_STATE32 +#define MCONTEXT_SIZE_512 sizeof(struct __darwin_mcontext_avx512_32) +#endif +#define VECTOR512 __m512 +#define VEC512ALIGN __attribute ((aligned(64))) +#define OPMASK uint64_t +#define KARRAY_MAX 8 +static inline void populate_zmm(void); +static inline void populate_opmask(void); +static inline void check_zmm(void); +VECTOR512 vec512array0[ZMM_MAX] VEC512ALIGN; +VECTOR512 vec512array1[ZMM_MAX] VEC512ALIGN; +VECTOR512 vec512array2[ZMM_MAX] VEC512ALIGN; +VECTOR512 vec512array3[ZMM_MAX] VEC512ALIGN; +OPMASK karray0[8]; +OPMASK karray1[8]; +OPMASK karray2[8]; +OPMASK karray3[8]; + + +/* + * Common functions + */ + +int +memcmp_unoptimized(const void *s1, const void *s2, size_t n) { + if (n != 0) { + const unsigned char *p1 = s1, *p2 = s2; + do { + if (*p1++ != *p2++) + return (*--p1 - *--p2); + } while (--n != 0); + } + return (0); +} + +void +start_timer(int seconds, void (*handler)(int, siginfo_t *, void *)) { + struct sigaction sigalrm_action = { + .sa_sigaction = handler, + .sa_flags = SA_RESTART, + .sa_mask = 0 + }; + struct itimerval timer = { + .it_value.tv_sec = seconds, + .it_value.tv_usec = 0, + .it_interval.tv_sec = 0, + .it_interval.tv_usec = 0 + }; + T_QUIET; T_WITH_ERRNO; + T_ASSERT_NE(sigaction(SIGALRM, &sigalrm_action, NULL), -1, NULL); + T_QUIET; T_WITH_ERRNO; + T_ASSERT_NE(setitimer(ITIMER_REAL, &timer, NULL), -1, NULL); +} + +void +require_avx(void) { + if((_get_cpu_capabilities() & kHasAVX1_0) != kHasAVX1_0) { + T_SKIP("AVX not supported on this system"); + } +} + +void +require_avx512(void) { + if((_get_cpu_capabilities() & kHasAVX512F) != kHasAVX512F) { + T_SKIP("AVX-512 not supported on this system"); + } +} + +/* + * ymm functions + */ + +static inline void +store_ymm(VECTOR256 *vec256array) { + int i = 0; + __asm__ volatile("vmovaps %%ymm0, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm1, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm2, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm3, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm4, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm5, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm6, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm7, %0" :"=m" (vec256array[i])); +#if defined(__x86_64__) + i++;__asm__ volatile("vmovaps %%ymm8, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm9, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm10, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm11, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm12, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm13, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm14, %0" :"=m" (vec256array[i])); + i++;__asm__ volatile("vmovaps %%ymm15, %0" :"=m" (vec256array[i])); +#endif +} + +static inline void +populate_ymm(void) { + int j; + uint32_t p[8] VEC256ALIGN; + + for (j = 0; j < (int) (sizeof(p)/sizeof(p[0])); j++) + p[j] = getpid(); + + p[0] = 0x22222222; + p[7] = 0x77777777; + __asm__ volatile("vmovaps %0, %%ymm0" :: "m" (*(__m256i*)p) : "ymm0"); + __asm__ volatile("vmovaps %0, %%ymm1" :: "m" (*(__m256i*)p) : "ymm1"); + __asm__ volatile("vmovaps %0, %%ymm2" :: "m" (*(__m256i*)p) : "ymm2"); + __asm__ volatile("vmovaps %0, %%ymm3" :: "m" (*(__m256i*)p) : "ymm3"); + + p[0] = 0x44444444; + p[7] = 0xEEEEEEEE; + __asm__ volatile("vmovaps %0, %%ymm4" :: "m" (*(__m256i*)p) : "ymm4"); + __asm__ volatile("vmovaps %0, %%ymm5" :: "m" (*(__m256i*)p) : "ymm5"); + __asm__ volatile("vmovaps %0, %%ymm6" :: "m" (*(__m256i*)p) : "ymm6"); + __asm__ volatile("vmovaps %0, %%ymm7" :: "m" (*(__m256i*)p) : "ymm7"); + +#if defined(__x86_64__) + p[0] = 0x88888888; + p[7] = 0xAAAAAAAA; + __asm__ volatile("vmovaps %0, %%ymm8" :: "m" (*(__m256i*)p) : "ymm8"); + __asm__ volatile("vmovaps %0, %%ymm9" :: "m" (*(__m256i*)p) : "ymm9"); + __asm__ volatile("vmovaps %0, %%ymm10" :: "m" (*(__m256i*)p) : "ymm10"); + __asm__ volatile("vmovaps %0, %%ymm11" :: "m" (*(__m256i*)p) : "ymm11"); + + p[0] = 0xBBBBBBBB; + p[7] = 0xCCCCCCCC; + __asm__ volatile("vmovaps %0, %%ymm12" :: "m" (*(__m256i*)p) : "ymm12"); + __asm__ volatile("vmovaps %0, %%ymm13" :: "m" (*(__m256i*)p) : "ymm13"); + __asm__ volatile("vmovaps %0, %%ymm14" :: "m" (*(__m256i*)p) : "ymm14"); + __asm__ volatile("vmovaps %0, %%ymm15" :: "m" (*(__m256i*)p) : "ymm15"); +#endif + + store_ymm(vec256array0); +} + +void +vec256_to_string(VECTOR256 *vec, char *buf) { + unsigned int vec_idx = 0; + unsigned int buf_idx = 0; + int ret = 0; + + for (vec_idx = 0; vec_idx < YMM_MAX; vec_idx++) { + uint64_t a[4]; + bcopy(&vec[vec_idx], &a[0], sizeof(a)); + ret = sprintf( + buf + buf_idx, + "0x%016llx:%016llx:%016llx:%016llx\n", + a[0], a[1], a[2], a[3] + ); + T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sprintf()"); + buf_idx += ret; + } +} + +void +assert_ymm_eq(void *a, void *b, int c) { + if(memcmp_unoptimized(a, b, c)) { + vec256_to_string(a, vec_str_buf); + T_LOG("Compare failed, vector A:\n%s", vec_str_buf); + vec256_to_string(b, vec_str_buf); + T_LOG("Compare failed, vector B:\n%s", vec_str_buf); + T_ASSERT_FAIL("vectors not equal"); + } +} + +void +check_ymm(void) { + uint32_t *p = (uint32_t *) &vec256array1[7]; + store_ymm(vec256array1); + if (p[0] == STOP_COOKIE_256) { + return; + } + assert_ymm_eq(vec256array0, vec256array1, sizeof(vec256array0)); +} + +static void +copy_ymm_state_to_vector(X86_AVX_STATE_T *sp, VECTOR256 *vp) { + int i; + struct __darwin_xmm_reg *xmm = &sp->__fpu_xmm0; + struct __darwin_xmm_reg *ymmh = &sp->__fpu_ymmh0; + + for (i = 0; i < YMM_MAX; i++ ) { + bcopy(&xmm[i], &vp[i], sizeof(*xmm)); + bcopy(&ymmh[i], (void *) ((uint64_t)&vp[i] + sizeof(*ymmh)), sizeof(*ymmh)); + } +} + +static void +ymm_sigalrm_handler(int signum __unused, siginfo_t *info __unused, void *ctx) +{ + ucontext_t *contextp = (ucontext_t *) ctx; + mcontext_t mcontext = contextp->uc_mcontext; + X86_AVX_STATE_T *avx_state = (X86_AVX_STATE_T *) &mcontext->__fs; + uint32_t *xp = (uint32_t *) &avx_state->__fpu_xmm7; + uint32_t *yp = (uint32_t *) &avx_state->__fpu_ymmh7; + + T_LOG("Got SIGALRM"); + + /* Check for AVX state */ + T_QUIET; + T_ASSERT_GE(contextp->uc_mcsize, MCONTEXT_SIZE_256, "check context size"); + + /* Check that the state in the context is what's set and expected */ + copy_ymm_state_to_vector(avx_state, vec256array3); + assert_ymm_eq(vec256array3, vec256array0, sizeof(vec256array1)); + + /* Change the context and break the main loop */ + xp[0] = STOP_COOKIE_256; + yp[0] = STOP_COOKIE_256; + checking = FALSE; +} + +void +ymm_integrity(int time) { + mach_msg_type_number_t avx_count = X86_AVX_STATE_COUNT; + kern_return_t kret; + X86_AVX_STATE_T avx_state, avx_state2; + mach_port_t ts = mach_thread_self(); + + bzero(&avx_state, sizeof(avx_state)); + bzero(&avx_state2, sizeof(avx_state)); + + kret = thread_get_state( + ts, X86_AVX_STATE_FLAVOR, (thread_state_t)&avx_state, &avx_count + ); + + store_ymm(vec256array2); + + T_QUIET; T_ASSERT_MACH_SUCCESS(kret, "thread_get_state()"); + vec256_to_string(vec256array2, vec_str_buf); + T_LOG("Initial state:\n%s", vec_str_buf); + + copy_ymm_state_to_vector(&avx_state, vec256array1); + assert_ymm_eq(vec256array2, vec256array1, sizeof(vec256array1)); + + populate_ymm(); + + kret = thread_get_state( + ts, X86_AVX_STATE_FLAVOR, (thread_state_t)&avx_state2, &avx_count + ); + + store_ymm(vec256array2); + + T_QUIET; T_ASSERT_MACH_SUCCESS(kret, "thread_get_state()"); + vec256_to_string(vec256array2, vec_str_buf); + T_LOG("Populated state:\n%s", vec_str_buf); + + copy_ymm_state_to_vector(&avx_state2, vec256array1); + assert_ymm_eq(vec256array2, vec256array1, sizeof(vec256array0)); + + T_LOG("Running for %ds…", time); + start_timer(time, ymm_sigalrm_handler); + + /* re-populate because printing mucks up XMMs */ + populate_ymm(); + + /* Check state until timer fires */ + while(checking) { + check_ymm(); + } + + /* Check that the sig handler changed out AVX state */ + store_ymm(vec256array1); + + uint32_t *p = (uint32_t *) &vec256array1[7]; + if (p[0] != STOP_COOKIE_256 || + p[4] != STOP_COOKIE_256) { + vec256_to_string(vec256array1, vec_str_buf); + T_ASSERT_FAIL("sigreturn failed to stick"); + T_LOG("State:\n%s", vec_str_buf); + } + + T_LOG("Ran for %ds", time); + T_PASS("No ymm register corruption occurred"); +} + +/* + * zmm functions + */ + +static inline void +store_opmask(OPMASK k[]) { + __asm__ volatile("kmovq %%k0, %0" :"=m" (k[0])); + __asm__ volatile("kmovq %%k1, %0" :"=m" (k[1])); + __asm__ volatile("kmovq %%k2, %0" :"=m" (k[2])); + __asm__ volatile("kmovq %%k3, %0" :"=m" (k[3])); + __asm__ volatile("kmovq %%k4, %0" :"=m" (k[4])); + __asm__ volatile("kmovq %%k5, %0" :"=m" (k[5])); + __asm__ volatile("kmovq %%k6, %0" :"=m" (k[6])); + __asm__ volatile("kmovq %%k7, %0" :"=m" (k[7])); +} + +static inline void +store_zmm(VECTOR512 *vecarray) { + int i = 0; + __asm__ volatile("vmovaps %%zmm0, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm1, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm2, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm3, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm4, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm5, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm6, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm7, %0" :"=m" (vecarray[i])); +#if defined(__x86_64__) + i++;__asm__ volatile("vmovaps %%zmm8, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm9, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm10, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm11, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm12, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm13, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm14, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm15, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm16, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm17, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm18, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm19, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm20, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm21, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm22, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm23, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm24, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm25, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm26, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm27, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm28, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm29, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm30, %0" :"=m" (vecarray[i])); + i++;__asm__ volatile("vmovaps %%zmm31, %0" :"=m" (vecarray[i])); +#endif +} + +static inline void +populate_opmask(void) { + uint64_t k[8]; + + for (int j = 0; j < 8; j++) + k[j] = ((uint64_t) getpid() << 32) + (0x11111111 * j); + + __asm__ volatile("kmovq %0, %%k0" : :"m" (k[0])); + __asm__ volatile("kmovq %0, %%k1" : :"m" (k[1])); + __asm__ volatile("kmovq %0, %%k2" : :"m" (k[2])); + __asm__ volatile("kmovq %0, %%k3" : :"m" (k[3])); + __asm__ volatile("kmovq %0, %%k4" : :"m" (k[4])); + __asm__ volatile("kmovq %0, %%k5" : :"m" (k[5])); + __asm__ volatile("kmovq %0, %%k6" : :"m" (k[6])); + __asm__ volatile("kmovq %0, %%k7" : :"m" (k[7])); + + store_opmask(karray0); +} + +static inline void +populate_zmm(void) { + int j; + uint64_t p[8] VEC512ALIGN; + + for (j = 0; j < (int) (sizeof(p)/sizeof(p[0])); j++) + p[j] = ((uint64_t) getpid() << 32) + getpid(); + + p[0] = 0x0000000000000000ULL; + p[2] = 0x4444444444444444ULL; + p[4] = 0x8888888888888888ULL; + p[7] = 0xCCCCCCCCCCCCCCCCULL; + __asm__ volatile("vmovaps %0, %%zmm0" :: "m" (*(__m256i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm1" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm2" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm3" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm4" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm5" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm6" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm7" :: "m" (*(__m512i*)p) ); + +#if defined(__x86_64__) + p[0] = 0x1111111111111111ULL; + p[2] = 0x5555555555555555ULL; + p[4] = 0x9999999999999999ULL; + p[7] = 0xDDDDDDDDDDDDDDDDULL; + __asm__ volatile("vmovaps %0, %%zmm8" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm9" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm10" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm11" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm12" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm13" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm14" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm15" :: "m" (*(__m512i*)p) ); + + p[0] = 0x2222222222222222ULL; + p[2] = 0x6666666666666666ULL; + p[4] = 0xAAAAAAAAAAAAAAAAULL; + p[7] = 0xEEEEEEEEEEEEEEEEULL; + __asm__ volatile("vmovaps %0, %%zmm16" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm17" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm18" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm19" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm20" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm21" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm22" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm23" :: "m" (*(__m512i*)p) ); + + p[0] = 0x3333333333333333ULL; + p[2] = 0x7777777777777777ULL; + p[4] = 0xBBBBBBBBBBBBBBBBULL; + p[7] = 0xFFFFFFFFFFFFFFFFULL; + __asm__ volatile("vmovaps %0, %%zmm24" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm25" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm26" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm27" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm28" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm29" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm30" :: "m" (*(__m512i*)p) ); + __asm__ volatile("vmovaps %0, %%zmm31" :: "m" (*(__m512i*)p) ); +#endif + + store_zmm(vec512array0); +} + +void +vec512_to_string(VECTOR512 *vec, char *buf) { + unsigned int vec_idx = 0; + unsigned int buf_idx = 0; + int ret = 0; + + for (vec_idx = 0; vec_idx < ZMM_MAX; vec_idx++) { + uint64_t a[8]; + bcopy(&vec[vec_idx], &a[0], sizeof(a)); + ret = sprintf( + buf + buf_idx, + "0x%016llx:%016llx:%016llx:%016llx:" + "%016llx:%016llx:%016llx:%016llx%s", + a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], + vec_idx < ZMM_MAX - 1 ? "\n" : "" + ); + T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sprintf()"); + buf_idx += ret; + } +} + +void +opmask_to_string(OPMASK *karray, char *buf) { + unsigned int karray_idx = 0; + unsigned int buf_idx = 0; + int ret = 0; + + for(karray_idx = 0; karray_idx < KARRAY_MAX; karray_idx++) { + ret = sprintf( + buf + buf_idx, + "k%d: 0x%016llx%s", + karray_idx, karray[karray_idx], + karray_idx < KARRAY_MAX ? "\n" : "" + ); + T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sprintf()"); + buf_idx += ret; + } +} + +static void +assert_zmm_eq(void *a, void *b, int c) { + if(memcmp_unoptimized(a, b, c)) { + vec512_to_string(a, vec_str_buf); + T_LOG("Compare failed, vector A:\n%s", vec_str_buf); + vec512_to_string(b, vec_str_buf); + T_LOG("Compare failed, vector B:\n%s", vec_str_buf); + T_ASSERT_FAIL("Vectors not equal"); + } +} + +static void +assert_opmask_eq(OPMASK *a, OPMASK *b) { + for (int i = 0; i < KARRAY_MAX; i++) { + if (a[i] != b[i]) { + opmask_to_string(a, karray_str_buf); + T_LOG("Compare failed, opmask A:\n%s", karray_str_buf); + opmask_to_string(b, karray_str_buf); + T_LOG("Compare failed, opmask B:\n%s", karray_str_buf); + T_ASSERT_FAIL("opmasks not equal"); + } + } +} + +void +check_zmm(void) { + uint64_t *p = (uint64_t *) &vec512array1[7]; + store_opmask(karray1); + store_zmm(vec512array1); + if (p[0] == STOP_COOKIE_512) { + return; + } + + assert_zmm_eq(vec512array0, vec512array1, sizeof(vec512array0)); + assert_opmask_eq(karray0, karray1); +} + +static void copy_state_to_opmask(X86_AVX512_STATE_T *sp, OPMASK *op) { + OPMASK *k = (OPMASK *) &sp->__fpu_k0; + for (int i = 0; i < KARRAY_MAX; i++) { + bcopy(&k[i], &op[i], sizeof(*op)); + } +} + +static void copy_zmm_state_to_vector(X86_AVX512_STATE_T *sp, VECTOR512 *vp) { + int i; + struct __darwin_xmm_reg *xmm = &sp->__fpu_xmm0; + struct __darwin_xmm_reg *ymmh = &sp->__fpu_ymmh0; + struct __darwin_ymm_reg *zmmh = &sp->__fpu_zmmh0; +#if defined(__x86_64__) + struct __darwin_zmm_reg *zmm = &sp->__fpu_zmm16; + + for (i = 0; i < ZMM_MAX/2; i++ ) { + bcopy(&xmm[i], &vp[i], sizeof(*xmm)); + bcopy(&ymmh[i], (void *) ((uint64_t)&vp[i] + sizeof(*ymmh)), sizeof(*ymmh)); + bcopy(&zmmh[i], (void *) ((uint64_t)&vp[i] + sizeof(*zmmh)), sizeof(*zmmh)); + bcopy(&zmm[i], &vp[(ZMM_MAX/2)+i], sizeof(*zmm)); + } +#else + for (i = 0; i < ZMM_MAX; i++ ) { + bcopy(&xmm[i], &vp[i], sizeof(*xmm)); + bcopy(&ymmh[i], (void *) ((uint64_t)&vp[i] + sizeof(*ymmh)), sizeof(*ymmh)); + bcopy(&zmmh[i], (void *) ((uint64_t)&vp[i] + sizeof(*zmmh)), sizeof(*zmmh)); + } +#endif +} + +static void +zmm_sigalrm_handler(int signum __unused, siginfo_t *info __unused, void *ctx) +{ + ucontext_t *contextp = (ucontext_t *) ctx; + mcontext_t mcontext = contextp->uc_mcontext; + X86_AVX512_STATE_T *avx_state = (X86_AVX512_STATE_T *) &mcontext->__fs; + uint64_t *xp = (uint64_t *) &avx_state->__fpu_xmm7; + uint64_t *yp = (uint64_t *) &avx_state->__fpu_ymmh7; + uint64_t *zp = (uint64_t *) &avx_state->__fpu_zmmh7; + uint64_t *kp = (uint64_t *) &avx_state->__fpu_k0; + + /* Check for AVX512 state */ + T_QUIET; + T_ASSERT_GE(contextp->uc_mcsize, MCONTEXT_SIZE_512, "check context size"); + + /* Check that the state in the context is what's set and expected */ + copy_zmm_state_to_vector(avx_state, vec512array3); + assert_zmm_eq(vec512array3, vec512array0, sizeof(vec512array1)); + copy_state_to_opmask(avx_state, karray3); + assert_opmask_eq(karray3, karray0); + + /* Change the context and break the main loop */ + xp[0] = STOP_COOKIE_512; + yp[0] = STOP_COOKIE_512; + zp[0] = STOP_COOKIE_512; + kp[7] = STOP_COOKIE_512; + checking = FALSE; +} + +void +zmm_integrity(int time) { + mach_msg_type_number_t avx_count = X86_AVX512_STATE_COUNT; + kern_return_t kret; + X86_AVX512_STATE_T avx_state, avx_state2; + mach_port_t ts = mach_thread_self(); + + bzero(&avx_state, sizeof(avx_state)); + bzero(&avx_state2, sizeof(avx_state)); + + store_zmm(vec512array2); + store_opmask(karray2); + + kret = thread_get_state( + ts, X86_AVX512_STATE_FLAVOR, (thread_state_t)&avx_state, &avx_count + ); + + T_QUIET; T_ASSERT_MACH_SUCCESS(kret, "thread_get_state()"); + vec512_to_string(vec512array2, vec_str_buf); + opmask_to_string(karray2, karray_str_buf); + T_LOG("Initial state:\n%s\n%s", vec_str_buf, karray_str_buf); + + copy_zmm_state_to_vector(&avx_state, vec512array1); + assert_zmm_eq(vec512array2, vec512array1, sizeof(vec512array1)); + copy_state_to_opmask(&avx_state, karray1); + assert_opmask_eq(karray2, karray1); + + populate_zmm(); + populate_opmask(); + + kret = thread_get_state( + ts, X86_AVX512_STATE_FLAVOR, (thread_state_t)&avx_state2, &avx_count + ); + + store_zmm(vec512array2); + store_opmask(karray2); + + T_QUIET; T_ASSERT_MACH_SUCCESS(kret, "thread_get_state()"); + vec512_to_string(vec512array2, vec_str_buf); + opmask_to_string(karray2, karray_str_buf); + T_LOG("Populated state:\n%s\n%s", vec_str_buf, karray_str_buf); + + copy_zmm_state_to_vector(&avx_state2, vec512array1); + assert_zmm_eq(vec512array2, vec512array1, sizeof(vec512array1)); + copy_state_to_opmask(&avx_state2, karray1); + assert_opmask_eq(karray2, karray1); + + T_LOG("Running for %ds…", time); + start_timer(time, zmm_sigalrm_handler); + + /* re-populate because printing mucks up XMMs */ + populate_zmm(); + populate_opmask(); + + /* Check state until timer fires */ + while(checking) { + check_zmm(); + } + + /* Check that the sig handler changed our AVX state */ + store_zmm(vec512array1); + store_opmask(karray1); + + uint64_t *p = (uint64_t *) &vec512array1[7]; + if (p[0] != STOP_COOKIE_512 || + p[2] != STOP_COOKIE_512 || + p[4] != STOP_COOKIE_512 || + karray1[7] != STOP_COOKIE_512) { + vec512_to_string(vec512array1, vec_str_buf); + opmask_to_string(karray1, karray_str_buf); + T_ASSERT_FAIL("sigreturn failed to stick"); + T_LOG("State:\n%s\n%s", vec_str_buf, karray_str_buf); + } + + T_LOG("Ran for %ds", time); + T_PASS("No zmm register corruption occurred"); +} + +/* + * Main test declarations + */ +T_DECL(ymm_integrity, + "Quick soak test to verify that AVX " + "register state is maintained correctly", + T_META_TIMEOUT(NORMAL_RUN_TIME + TIMEOUT_OVERHEAD)) { + require_avx(); + ymm_integrity(NORMAL_RUN_TIME); +} + +T_DECL(ymm_integrity_stress, + "Extended soak test to verify that AVX " + "register state is maintained correctly", + T_META_TIMEOUT(LONG_RUN_TIME + TIMEOUT_OVERHEAD), + T_META_ENABLED(false)) { + require_avx(); + ymm_integrity(LONG_RUN_TIME); +} + +T_DECL(zmm_integrity, + "Quick soak test to verify that AVX-512 " + "register state is maintained correctly", + T_META_TIMEOUT(LONG_RUN_TIME + TIMEOUT_OVERHEAD)) { + require_avx512(); + zmm_integrity(NORMAL_RUN_TIME); +} + +T_DECL(zmm_integrity_stress, + "Extended soak test to verify that AVX-512 " + "register state is maintained correctly", + T_META_TIMEOUT(NORMAL_RUN_TIME + TIMEOUT_OVERHEAD), + T_META_ENABLED(false)) { + require_avx512(); + zmm_integrity(LONG_RUN_TIME); +} + diff --git a/tools/tests/darwintests/contextswitch.c b/tools/tests/darwintests/contextswitch.c new file mode 100644 index 000000000..b059be9a3 --- /dev/null +++ b/tools/tests/darwintests/contextswitch.c @@ -0,0 +1,285 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif + +#include +#include + +#define MAX_THREADS 32 +#define SPIN_SECS 6 +#define THR_SPINNER_PRI 63 +#define THR_MANAGER_PRI 62 +#define WARMUP_ITERATIONS 100 +#define POWERCTRL_SUCCESS_STR "Factor1: 1.000000" + +static mach_timebase_info_data_t timebase_info; +static semaphore_t semaphore; +static semaphore_t worker_sem; +static uint32_t g_numcpus; +static _Atomic uint32_t keep_going = 1; +static dt_stat_time_t s; + +static struct { + pthread_t thread; + bool measure_thread; +} threads[MAX_THREADS]; + +static uint64_t +nanos_to_abs(uint64_t nanos) +{ + return nanos * timebase_info.denom / timebase_info.numer; +} + +extern char **environ; + +static void +csw_perf_test_init(void) +{ + int spawn_ret, pid; + char *const clpcctrl_args[] = {"/usr/local/bin/clpcctrl", "-f", "5000", NULL}; + spawn_ret = posix_spawn(&pid, clpcctrl_args[0], NULL, NULL, clpcctrl_args, environ); + waitpid(pid, &spawn_ret, 0); +} + +static void +csw_perf_test_cleanup(void) +{ + int spawn_ret, pid; + char *const clpcctrl_args[] = {"/usr/local/bin/clpcctrl", "-d", NULL}; + spawn_ret = posix_spawn(&pid, clpcctrl_args[0], NULL, NULL, clpcctrl_args, environ); + waitpid(pid, &spawn_ret, 0); +} + +static pthread_t +create_thread(uint32_t thread_id, uint32_t priority, bool fixpri, + void *(*start_routine)(void *)) +{ + int rv; + pthread_t new_thread; + struct sched_param param = { .sched_priority = (int)priority }; + pthread_attr_t attr; + + T_ASSERT_POSIX_ZERO(pthread_attr_init(&attr), "pthread_attr_init"); + + T_ASSERT_POSIX_ZERO(pthread_attr_setschedparam(&attr, ¶m), + "pthread_attr_setschedparam"); + + if (fixpri) { + T_ASSERT_POSIX_ZERO(pthread_attr_setschedpolicy(&attr, SCHED_RR), + "pthread_attr_setschedpolicy"); + } + + T_ASSERT_POSIX_ZERO(pthread_create(&new_thread, &attr, start_routine, + (void*)(uintptr_t)thread_id), "pthread_create"); + + T_ASSERT_POSIX_ZERO(pthread_attr_destroy(&attr), "pthread_attr_destroy"); + + threads[thread_id].thread = new_thread; + + return new_thread; +} + +/* Spin until a specified number of seconds elapses */ +static void +spin_for_duration(uint32_t seconds) +{ + uint64_t duration = nanos_to_abs((uint64_t)seconds * NSEC_PER_SEC); + uint64_t current_time = mach_absolute_time(); + uint64_t timeout = duration + current_time; + + uint64_t spin_count = 0; + + while (mach_absolute_time() < timeout && atomic_load_explicit(&keep_going, + memory_order_relaxed)) { + spin_count++; + } +} + +static void * +spin_thread(void *arg) +{ + uint32_t thread_id = (uint32_t) arg; + char name[30] = ""; + + snprintf(name, sizeof(name), "spin thread %2d", thread_id); + pthread_setname_np(name); + T_ASSERT_MACH_SUCCESS(semaphore_wait_signal(semaphore, worker_sem), + "semaphore_wait_signal"); + spin_for_duration(SPIN_SECS); + return NULL; +} + +static void * +thread(void *arg) +{ + uint32_t thread_id = (uint32_t) arg; + char name[30] = ""; + + snprintf(name, sizeof(name), "thread %2d", thread_id); + pthread_setname_np(name); + T_ASSERT_MACH_SUCCESS(semaphore_wait_signal(semaphore, worker_sem), "semaphore_wait"); + + if (threads[thread_id].measure_thread) { + for (int i = 0; i < WARMUP_ITERATIONS; i++) { + thread_switch(THREAD_NULL, SWITCH_OPTION_NONE, 0); + } + T_STAT_MEASURE_LOOP(s) { + if(thread_switch(THREAD_NULL, SWITCH_OPTION_NONE, 0)) + T_ASSERT_FAIL("thread_switch"); + } + atomic_store_explicit(&keep_going, 0, memory_order_relaxed); + } else { + while (atomic_load_explicit(&keep_going, memory_order_relaxed)) { + if (thread_switch(THREAD_NULL, SWITCH_OPTION_NONE, 0)) + T_ASSERT_FAIL("thread_switch"); + } + } + return NULL; +} + +void check_device_temperature(void) +{ + char buffer[256]; + FILE *pipe = popen("powerctrl Factor1", "r"); + + if (pipe == NULL) { + T_FAIL("Failed to check device temperature"); + T_END; + } + + fgets(buffer, sizeof(buffer), pipe); + + if (strncmp(POWERCTRL_SUCCESS_STR, buffer, strlen(POWERCTRL_SUCCESS_STR))) { + T_PERF("temperature", 0.0, "factor", "device temperature"); + } else { + T_PASS("Device temperature check pass"); + T_PERF("temperature", 1.0, "factor", "device temperature"); + } + pclose(pipe); +} + +void record_perfcontrol_stats(const char *sysctlname, const char *units, const char *info) +{ + int data = 0; + size_t data_size = sizeof(data); + T_ASSERT_POSIX_ZERO(sysctlbyname(sysctlname, + &data, &data_size, NULL, 0), + "%s", sysctlname); + T_PERF(info, data, units, info); +} + + +T_GLOBAL_META(T_META_NAMESPACE("xnu.scheduler")); + +/* Disable the test on MacOS for now */ +T_DECL(perf_csw, "context switch performance", T_META_TYPE_PERF, T_META_CHECK_LEAKS(NO), T_META_ASROOT(YES)) +{ + +#if !CONFIG_EMBEDDED + T_SKIP("Not supported on MacOS"); + return; +#endif /* CONFIG_EMBEDDED */ + check_device_temperature(); + + T_ATEND(csw_perf_test_cleanup); + + csw_perf_test_init(); + pthread_setname_np("main thread"); + + T_ASSERT_MACH_SUCCESS(mach_timebase_info(&timebase_info), "mach_timebase_info"); + + struct sched_param param = {.sched_priority = 48}; + + T_ASSERT_POSIX_ZERO(pthread_setschedparam(pthread_self(), SCHED_FIFO, ¶m), + "pthread_setschedparam"); + + T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &semaphore, + SYNC_POLICY_FIFO, 0), "semaphore_create"); + + T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &worker_sem, + SYNC_POLICY_FIFO, 0), "semaphore_create"); + + size_t ncpu_size = sizeof(g_numcpus); + T_ASSERT_POSIX_ZERO(sysctlbyname("hw.ncpu", &g_numcpus, &ncpu_size, NULL, 0), + "sysctlbyname hw.ncpu"); + + printf("hw.ncpu: %d\n", g_numcpus); + uint32_t n_spinners = g_numcpus - 1; + + int mt_supported = 0; + size_t mt_supported_size = sizeof(mt_supported); + T_ASSERT_POSIX_ZERO(sysctlbyname("kern.monotonic.supported", &mt_supported, + &mt_supported_size, NULL, 0), "sysctlbyname kern.monotonic.supported"); + + for (uint32_t thread_id = 0; thread_id < n_spinners; thread_id++) { + threads[thread_id].thread = create_thread(thread_id, THR_SPINNER_PRI, + true, &spin_thread); + } + + s = dt_stat_time_create("context switch time"); + + create_thread(n_spinners, THR_MANAGER_PRI, true, &thread); + threads[n_spinners].measure_thread = true; + create_thread(n_spinners + 1, THR_MANAGER_PRI, true, &thread); + + /* Allow the context switch threads to get into sem_wait() */ + for (uint32_t thread_id = 0; thread_id < n_spinners + 2; thread_id++) { + T_ASSERT_MACH_SUCCESS(semaphore_wait(worker_sem), "semaphore_wait"); + } + + int enable_callout_stats = 1; + size_t enable_size = sizeof(enable_callout_stats); + + if (mt_supported) { + /* Enable callout stat collection */ + T_ASSERT_POSIX_ZERO(sysctlbyname("kern.perfcontrol_callout.stats_enabled", + NULL, 0, &enable_callout_stats, enable_size), + "sysctlbyname kern.perfcontrol_callout.stats_enabled"); + } + + T_ASSERT_MACH_SUCCESS(semaphore_signal_all(semaphore), "semaphore_signal"); + + + for (uint32_t thread_id = 0; thread_id < n_spinners + 2; thread_id++) { + T_ASSERT_POSIX_ZERO(pthread_join(threads[thread_id].thread, NULL), + "pthread_join %d", thread_id); + } + + if (mt_supported) { + record_perfcontrol_stats("kern.perfcontrol_callout.oncore_instr", + "instructions", "oncore.instructions"); + record_perfcontrol_stats("kern.perfcontrol_callout.offcore_instr", + "instructions", "offcore.instructions"); + record_perfcontrol_stats("kern.perfcontrol_callout.oncore_cycles", + "cycles", "oncore.cycles"); + record_perfcontrol_stats("kern.perfcontrol_callout.offcore_cycles", + "cycles", "offcore.cycles"); + + /* Disable callout stat collection */ + enable_callout_stats = 0; + T_ASSERT_POSIX_ZERO(sysctlbyname("kern.perfcontrol_callout.stats_enabled", + NULL, 0, &enable_callout_stats, enable_size), + "sysctlbyname kern.perfcontrol_callout.stats_enabled"); + } + + check_device_temperature(); + dt_stat_finalize(s); +} diff --git a/tools/tests/darwintests/cpucount.c b/tools/tests/darwintests/cpucount.c new file mode 100644 index 000000000..bd0548a96 --- /dev/null +++ b/tools/tests/darwintests/cpucount.c @@ -0,0 +1,266 @@ +/* + * Test to validate that we can schedule threads on all hw.ncpus cores according to _os_cpu_number + * + * + * +xcrun -sdk macosx.internal clang -o cpucount cpucount.c -ldarwintest -g -Weverything +xcrun -sdk iphoneos.internal clang -arch arm64 -o cpucount-ios cpucount.c -ldarwintest -g -Weverything + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include /* private header for _os_cpu_number */ + +/* const variables aren't constants, but enums are */ +enum { max_threads = 40 }; + +#define CACHE_ALIGNED __attribute__((aligned(128))) + +static _Atomic CACHE_ALIGNED uint64_t g_ready_threads = 0; + +static _Atomic CACHE_ALIGNED bool g_cpu_seen[max_threads]; + +static _Atomic CACHE_ALIGNED bool g_bail = false; + +static uint32_t g_threads; /* set by sysctl hw.ncpu */ + +static uint64_t g_spin_ms = 50; /* it takes ~50ms of spinning for CLPC to deign to give us all cores */ + +/* + * sometimes pageout scan can eat all of CPU 0 long enough to fail the test, + * so we run the test at RT priority + */ +static uint32_t g_thread_pri = 97; + +/* + * add in some extra low-pri threads to convince the amp scheduler to use E-cores consistently + * works around + */ +static uint32_t g_spin_threads = 2; +static uint32_t g_spin_threads_pri = 20; + +static semaphore_t g_readysem, g_go_sem; + +static mach_timebase_info_data_t timebase_info; + +static uint64_t nanos_to_abs(uint64_t nanos) { return nanos * timebase_info.denom / timebase_info.numer; } + +static void set_realtime(pthread_t thread) { + kern_return_t kr; + thread_time_constraint_policy_data_t pol; + + mach_port_t target_thread = pthread_mach_thread_np(thread); + T_QUIET; T_ASSERT_NOTNULL(target_thread, "pthread_mach_thread_np"); + + /* 1s 100ms 10ms */ + pol.period = (uint32_t)nanos_to_abs(1000000000); + pol.constraint = (uint32_t)nanos_to_abs(100000000); + pol.computation = (uint32_t)nanos_to_abs(10000000); + + pol.preemptible = 0; /* Ignored by OS */ + kr = thread_policy_set(target_thread, THREAD_TIME_CONSTRAINT_POLICY, (thread_policy_t) &pol, + THREAD_TIME_CONSTRAINT_POLICY_COUNT); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_policy_set(THREAD_TIME_CONSTRAINT_POLICY)"); +} + +static pthread_t +create_thread(void *(*start_routine)(void *), uint32_t priority) +{ + int rv; + pthread_t new_thread; + pthread_attr_t attr; + + struct sched_param param = { .sched_priority = (int)priority }; + + rv = pthread_attr_init(&attr); + T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_attr_init"); + + rv = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_attr_setdetachstate"); + + rv = pthread_attr_setschedparam(&attr, ¶m); + T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_attr_setschedparam"); + + rv = pthread_create(&new_thread, &attr, start_routine, NULL); + T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_create"); + + if (priority == 97) + set_realtime(new_thread); + + rv = pthread_attr_destroy(&attr); + T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_attr_destroy"); + + return new_thread; +} + +static void * +thread_fn(__unused void *arg) +{ + T_QUIET; T_EXPECT_TRUE(true, "initialize darwintest on this thread"); + + kern_return_t kr; + + kr = semaphore_wait_signal(g_go_sem, g_readysem); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_wait_signal"); + + /* atomic inc to say hello */ + g_ready_threads++; + + uint64_t timeout = nanos_to_abs(g_spin_ms * NSEC_PER_MSEC) + mach_absolute_time(); + + /* + * spin to force the other threads to spread out across the cores + * may take some time if cores are masked and CLPC needs to warm up to unmask them + */ + while (g_ready_threads < g_threads && mach_absolute_time() < timeout); + + T_QUIET; T_ASSERT_GE(timeout, mach_absolute_time(), "waiting for all threads took too long"); + + timeout = nanos_to_abs(g_spin_ms * NSEC_PER_MSEC) + mach_absolute_time(); + + int iteration = 0; + uint32_t cpunum = 0; + + /* search for new CPUs for the duration */ + while (mach_absolute_time() < timeout) { + cpunum = _os_cpu_number(); + + assert(cpunum < max_threads); + + g_cpu_seen[cpunum] = true; + + if (iteration++ % 10000) { + uint32_t cpus_seen = 0; + + for (uint32_t i = 0 ; i < g_threads; i++) { + if (g_cpu_seen[i]) + cpus_seen++; + } + + /* bail out early if we saw all CPUs */ + if (cpus_seen == g_threads) + break; + } + } + + g_bail = true; + + printf("thread cpunum: %d\n", cpunum); + + kr = semaphore_wait_signal(g_go_sem, g_readysem); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_wait_signal"); + + return NULL; +} + +static void * +spin_fn(__unused void *arg) +{ + T_QUIET; T_EXPECT_TRUE(true, "initialize darwintest on this thread"); + + kern_return_t kr; + + kr = semaphore_wait_signal(g_go_sem, g_readysem); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_wait_signal"); + + uint64_t timeout = nanos_to_abs(g_spin_ms * NSEC_PER_MSEC * 2) + mach_absolute_time(); + + /* + * run and sleep a bit to force some scheduler churn to get all the cores active + * needed to work around bugs in the amp scheduler + */ + while (mach_absolute_time() < timeout && g_bail == false) { + usleep(500); + + uint64_t inner_timeout = nanos_to_abs(1 * NSEC_PER_MSEC) + mach_absolute_time(); + + while (mach_absolute_time() < inner_timeout && g_bail == false); + } + + kr = semaphore_wait_signal(g_go_sem, g_readysem); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_wait_signal"); + + return NULL; +} + + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wgnu-flexible-array-initializer" +T_DECL(count_cpus, "Tests we can schedule threads on all hw.ncpus cores according to _os_cpu_number", + T_META_CHECK_LEAKS(NO)) +#pragma clang diagnostic pop +{ + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + + int rv; + kern_return_t kr; + kr = mach_timebase_info(&timebase_info); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "mach_timebase_info"); + + kr = semaphore_create(mach_task_self(), &g_readysem, SYNC_POLICY_FIFO, 0); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_create"); + + kr = semaphore_create(mach_task_self(), &g_go_sem, SYNC_POLICY_FIFO, 0); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_create"); + + size_t ncpu_size = sizeof(g_threads); + rv = sysctlbyname("hw.ncpu", &g_threads, &ncpu_size, NULL, 0); + T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "sysctlbyname(hw.ncpu)"); + + printf("hw.ncpu: %2d\n", g_threads); + + assert(g_threads < max_threads); + + for (uint32_t i = 0; i < g_threads; i++) + create_thread(&thread_fn, g_thread_pri); + + for (uint32_t i = 0; i < g_spin_threads; i++) + create_thread(&spin_fn, g_spin_threads_pri); + + for (uint32_t i = 0 ; i < g_threads + g_spin_threads; i++) { + kr = semaphore_wait(g_readysem); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_wait"); + } + + uint64_t timeout = nanos_to_abs(g_spin_ms * NSEC_PER_MSEC) + mach_absolute_time(); + + /* spin to warm up CLPC :) */ + while (mach_absolute_time() < timeout); + + kr = semaphore_signal_all(g_go_sem); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_signal_all"); + + for (uint32_t i = 0 ; i < g_threads + g_spin_threads; i++) { + kr = semaphore_wait(g_readysem); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_wait"); + } + + uint32_t cpus_seen = 0; + + for (uint32_t i = 0 ; i < g_threads; i++) { + if (g_cpu_seen[i]) + cpus_seen++; + + printf("cpu %2d: %d\n", i, g_cpu_seen[i]); + } + + T_ASSERT_EQ(cpus_seen, g_threads, "test should have run threads on all CPUS"); +} + diff --git a/tools/tests/darwintests/data_protection.c b/tools/tests/darwintests/data_protection.c index f39fe0f66..331c4809f 100644 --- a/tools/tests/darwintests/data_protection.c +++ b/tools/tests/darwintests/data_protection.c @@ -48,7 +48,7 @@ int apple_key_store( uint32_t * output_count ); int spawn_proc(char * const command[]); -int supports_content_prot(); +int supports_content_prot(void); char* dp_class_num_to_string(int num); int lock_device(void); int unlock_device(char * passcode); diff --git a/tools/tests/darwintests/disk_mount_conditioner-entitlements.plist b/tools/tests/darwintests/disk_mount_conditioner-entitlements.plist new file mode 100644 index 000000000..95d21414e --- /dev/null +++ b/tools/tests/darwintests/disk_mount_conditioner-entitlements.plist @@ -0,0 +1,8 @@ + + + + + com.apple.private.dmc.set + + + diff --git a/tools/tests/darwintests/disk_mount_conditioner.c b/tools/tests/darwintests/disk_mount_conditioner.c new file mode 100644 index 000000000..5847149e0 --- /dev/null +++ b/tools/tests/darwintests/disk_mount_conditioner.c @@ -0,0 +1,388 @@ +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif +#include +#include + +#include +#include +#include +#include +#include + +static char *mktempdir(void); +static char *mktempmount(void); + +#ifndef TEST_UNENTITLED +static int system_legal(const char *command); +static char *mkramdisk(void); +static uint64_t time_for_read(int fd, const char *expected); +static void perf_setup(char **path, int *fd); + +#define READSIZE 1024L +#endif /* !TEST_UNENTITLED */ + +T_GLOBAL_META( + T_META_NAMESPACE("xnu.vfs.dmc"), + T_META_ASROOT(true) +); + +#pragma mark Entitled Tests + +#ifndef TEST_UNENTITLED +T_DECL(fsctl_get_uninitialized, + "Initial fsctl.get should return zeros", + T_META_ASROOT(false)) +{ + int err; + char *mount_path; + disk_conditioner_info info = {0}; + disk_conditioner_info expected_info = {0}; + + T_SETUPBEGIN; + mount_path = mktempmount(); + T_SETUPEND; + + info.enabled = true; + info.is_ssd = true; + err = fsctl(mount_path, DISK_CONDITIONER_IOC_GET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "fsctl(DISK_CONDITIONER_IOC_GET)"); + + err = memcmp(&info, &expected_info, sizeof(info)); + T_ASSERT_EQ_INT(0, err, "initial DMC info is zeroed"); +} + +T_DECL(fsctl_set, + "fsctl.set should succeed and fsctl.get should verify") +{ + int err; + char *mount_path; + disk_conditioner_info info = {0}; + disk_conditioner_info expected_info = {0}; + + T_SETUPBEGIN; + mount_path = mktempmount(); + T_SETUPEND; + + info.enabled = 1; + info.access_time_usec = 10; + info.read_throughput_mbps = 40; + info.write_throughput_mbps = 40; + info.is_ssd = 0; + expected_info = info; + + err = fsctl(mount_path, DISK_CONDITIONER_IOC_SET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "fsctl(DISK_CONDITIONER_IOC_SET)"); + + err = fsctl(mount_path, DISK_CONDITIONER_IOC_GET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "fsctl(DISK_CONDITIONER_IOC_GET) after SET"); + + err = memcmp(&info, &expected_info, sizeof(info)); + T_ASSERT_EQ_INT(0, err, "fsctl.get is the info configured by fsctl.set"); +} + +T_DECL(fsctl_get_nonroot, + "fsctl.get should not require root", + T_META_ASROOT(false)) +{ + int err; + char *mount_path; + disk_conditioner_info info; + + T_SETUPBEGIN; + // make sure we're not root + if (0 == geteuid()) { + seteuid(5000); + } + + mount_path = mktempmount(); + T_SETUPEND; + + err = fsctl(mount_path, DISK_CONDITIONER_IOC_GET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "fsctl.get without root"); +} + +T_DECL(fsctl_set_nonroot, + "fsctl.set should require root", + T_META_ASROOT(false)) +{ + int err; + char *mount_path; + disk_conditioner_info info = {0}; + disk_conditioner_info expected_info = {0}; + + T_SETUPBEGIN; + // make sure we're not root + if (0 == geteuid()) { + seteuid(5000); + } + + mount_path = mktempmount(); + T_SETUPEND; + + // save original info + err = fsctl(mount_path, DISK_CONDITIONER_IOC_GET, &expected_info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "Get original DMC info"); + + info.enabled = 1; + info.access_time_usec = 10; + err = fsctl(mount_path, DISK_CONDITIONER_IOC_SET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_NE_INT(0, err, "fsctl.set returns error without root"); + + err = fsctl(mount_path, DISK_CONDITIONER_IOC_GET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "fsctl.get after nonroot fsctl.set"); + + err = memcmp(&info, &expected_info, sizeof(info)); + T_ASSERT_EQ_INT(0, err, "fsctl.set should not change info without root"); +} + +T_DECL(fsctl_delays, + "Validate I/O delays when DMC is enabled") +{ + char *path; + int fd; + int err; + uint64_t elapsed_nsec, expected_nsec; + disk_conditioner_info info; + char buf[READSIZE]; + + T_SETUPBEGIN; + perf_setup(&path, &fd); + memset(buf, 0xFF, sizeof(buf)); + T_ASSERT_EQ_LONG((long)sizeof(buf), write(fd, buf, sizeof(buf)), "write random data to temp file"); + fcntl(fd, F_FULLFSYNC); + T_SETUPEND; + + expected_nsec = NSEC_PER_SEC / 2; + + // measure delay before setting parameters (should be none) + elapsed_nsec = time_for_read(fd, buf); + T_ASSERT_LT_ULLONG(elapsed_nsec, expected_nsec, "DMC disabled read(%ld) from %s is reasonably fast", READSIZE, path); + + // measure delay after setting parameters + info.enabled = 1; + info.access_time_usec = expected_nsec / NSEC_PER_USEC; + info.read_throughput_mbps = 40; + info.write_throughput_mbps = 40; + info.is_ssd = 1; // is_ssd will ensure we get constant access_time delays rather than scaled + err = fsctl(path, DISK_CONDITIONER_IOC_SET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "fsctl(DISK_CONDITIONER_IOC_SET) delay"); + + elapsed_nsec = time_for_read(fd, buf); + T_ASSERT_GT_ULLONG(elapsed_nsec, expected_nsec, "DMC enabled read(%ld) from %s is at least the expected delay", READSIZE, path); + T_ASSERT_LT_ULLONG(elapsed_nsec, 2 * expected_nsec, "DMC enabled read(%ld) from %s is no more than twice the expected delay", READSIZE, path); + + // measure delay after resetting parameters (should be none) + info.enabled = 0; + err = fsctl(path, DISK_CONDITIONER_IOC_SET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "fsctl(DISK_CONDITIONER_IOC_SET) reset delay"); + + usleep(USEC_PER_SEC / 2); // might still be other I/O inflight + elapsed_nsec = time_for_read(fd, buf); + T_ASSERT_LT_ULLONG(elapsed_nsec, expected_nsec, "After disabling DMC read(%ld) from %s is reasonably fast", READSIZE, path); +} + +#else /* TEST_UNENTITLED */ + +#pragma mark Unentitled Tests + +T_DECL(fsctl_get_unentitled, + "fsctl.get should not require entitlement") +{ + int err; + char *mount_path; + disk_conditioner_info info; + + T_SETUPBEGIN; + mount_path = mktempmount(); + T_SETUPEND; + + err = fsctl(mount_path, DISK_CONDITIONER_IOC_GET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "fsctl.get without entitlement"); +} + +T_DECL(fsctl_set_unentitled, + "fsctl.set should require entitlement") +{ + int err; + char *mount_path; + disk_conditioner_info info = {0}; + disk_conditioner_info expected_info = {0}; + + T_SETUPBEGIN; + mount_path = mktempmount(); + T_SETUPEND; + + // save original info + err = fsctl(mount_path, DISK_CONDITIONER_IOC_GET, &expected_info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "Get original DMC info"); + + info.enabled = 1; + info.access_time_usec = 10; + err = fsctl(mount_path, DISK_CONDITIONER_IOC_SET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_NE_INT(0, err, "fsctl.set returns error without entitlement"); + + err = fsctl(mount_path, DISK_CONDITIONER_IOC_GET, &info, 0); + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, err, "fsctl.get after unentitled fsctl.set"); + + err = memcmp(&info, &expected_info, sizeof(info)); + T_ASSERT_EQ_INT(0, err, "fsctl.set should not change info without entitlement"); +} + +#endif /* TEST_UNENTITLED */ + +#pragma mark Helpers + +static char *mktempdir(void) { + char *path = malloc(PATH_MAX); + strcpy(path, "/tmp/dmc.XXXXXXXX"); + atexit_b(^{ free(path); }); + + // create a temporary mount to run the fsctl on + T_WITH_ERRNO; + T_ASSERT_NOTNULL(mkdtemp(path), "Create temporary directory"); + atexit_b(^{ remove(path); }); + + return path; +} + +/* + * Return the path to a temporary mount + * with no usable filesystem but still + * can be configured by the disk conditioner + * + * Faster than creating a ram disk to test with + * when access to the filesystem is not necessary + */ +static char *mktempmount(void) { + char *mount_path = mktempdir(); + + T_WITH_ERRNO; + T_ASSERT_EQ_INT(0, mount("devfs", mount_path, MNT_RDONLY, NULL), "Create temporary devfs mount"); + atexit_b(^{ unmount(mount_path, MNT_FORCE); }); + + return mount_path; +} + +#ifndef TEST_UNENTITLED + +/* + * Wrapper around dt_launch_tool/dt_waitpid + * that works like libc:system() + */ +static int system_legal(const char *command) { + pid_t pid = -1; + int exit_status = 0; + const char *argv[] = { + _PATH_BSHELL, + "-c", + command, + NULL + }; + + int rc = dt_launch_tool(&pid, (char **)(void *)argv, false, NULL, NULL); + if (rc != 0) { + return -1; + } + if (!dt_waitpid(pid, &exit_status, NULL, 30)) { + if (exit_status != 0) { + return exit_status; + } + return -1; + } + + return exit_status; +} + +/* + * Return the path to a temporary mount + * that contains a usable HFS+ filesystem + * mounted via a ram disk + */ +static char *mkramdisk(void) { + char cmd[1024]; + char *mount_path = mktempdir(); + char *dev_disk_file = malloc(256); + atexit_b(^{ free(dev_disk_file); }); + strcpy(dev_disk_file, "/tmp/dmc.ramdisk.XXXXXXXX"); + + T_WITH_ERRNO; + T_ASSERT_NOTNULL(mktemp(dev_disk_file), "Create temporary file to store dev disk for ramdisk"); + atexit_b(^{ remove(dev_disk_file); }); + + // create the RAM disk device + snprintf(cmd, sizeof(cmd), "hdik -nomount ram://10000 > %s", dev_disk_file); + T_ASSERT_EQ_INT(0, system_legal(cmd), "Create ramdisk"); + + atexit_b(^{ + char eject_cmd[1024]; + unmount(mount_path, MNT_FORCE); + snprintf(eject_cmd, sizeof(eject_cmd), "hdik -e `cat %s`", dev_disk_file); + system_legal(eject_cmd); + remove(dev_disk_file); + }); + + // initialize as an HFS volume + snprintf(cmd, sizeof(cmd), "newfs_hfs `cat %s`", dev_disk_file); + T_ASSERT_EQ_INT(0, system_legal(cmd), "Initialize ramdisk as HFS"); + + // mount it + snprintf(cmd, sizeof(cmd), "mount -t hfs `cat %s` %s", dev_disk_file, mount_path); + T_ASSERT_EQ_INT(0, system_legal(cmd), "Mount ramdisk"); + + return mount_path; +} + +static uint64_t time_for_read(int fd, const char *expected) { + int err; + ssize_t ret; + char buf[READSIZE]; + uint64_t start, stop; + + bzero(buf, sizeof(buf)); + lseek(fd, 0, SEEK_SET); + + start = dt_nanoseconds(); + ret = read(fd, buf, READSIZE); + stop = dt_nanoseconds(); + + T_ASSERT_GE_LONG(ret, 0L, "read from temporary file"); + T_ASSERT_EQ_LONG(ret, READSIZE, "read %ld bytes from temporary file", READSIZE); + err = memcmp(buf, expected, sizeof(buf)); + T_ASSERT_EQ_INT(0, err, "read expected contents from temporary file"); + + return (stop - start); +} + +static void perf_setup(char **path, int *fd) { + int temp_fd; + char *temp_path; + + char *mount_path = mkramdisk(); + temp_path = *path = malloc(PATH_MAX); + snprintf(temp_path, PATH_MAX, "%s/dmc.XXXXXXXX", mount_path); + atexit_b(^{ free(temp_path); }); + + T_ASSERT_NOTNULL(mktemp(temp_path), "Create temporary file"); + atexit_b(^{ remove(temp_path); }); + + temp_fd = *fd = open(temp_path, O_RDWR | O_CREAT); + T_WITH_ERRNO; + T_ASSERT_GE_INT(temp_fd, 0, "Open temporary file for read/write"); + atexit_b(^{ close(temp_fd); }); + fcntl(temp_fd, F_NOCACHE, 1); +} +#endif /* !TEST_UNENTITLED */ diff --git a/tools/tests/darwintests/gettimeofday_29192647.c b/tools/tests/darwintests/gettimeofday_29192647.c new file mode 100644 index 000000000..bd7b66159 --- /dev/null +++ b/tools/tests/darwintests/gettimeofday_29192647.c @@ -0,0 +1,47 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +T_DECL(gettimeofday_tl, "gettimeofday performance in tight loop") { + { + struct timeval time; + dt_stat_time_t s = dt_stat_time_create("gettimeofday tight loop"); + T_STAT_MEASURE_LOOP(s){ + gettimeofday(&time, NULL); + } + dt_stat_finalize(s); + } +} + +extern int __gettimeofday(struct timeval *, struct timezone *); +T_DECL(__gettimeofday_tl, "__gettimeofday performance in tight loop") { + { + struct timeval time; + + dt_stat_time_t s = dt_stat_time_create("__gettimeofday tight loop"); + T_STAT_MEASURE_LOOP(s){ + __gettimeofday(&time, NULL); + } + dt_stat_finalize(s); + } +} + +T_DECL(gettimeofday_sl, "gettimeofday performance in loop with sleep") { + { + struct timeval time; + dt_stat_time_t s = dt_stat_time_create("gettimeofday loop with sleep"); + while (!dt_stat_stable(s)) { + T_STAT_MEASURE_BATCH(s){ + gettimeofday(&time, NULL); + } + sleep(1); + } + dt_stat_finalize(s); + } +} diff --git a/tools/tests/darwintests/ioperf.c b/tools/tests/darwintests/ioperf.c new file mode 100644 index 000000000..c2586ac53 --- /dev/null +++ b/tools/tests/darwintests/ioperf.c @@ -0,0 +1,256 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif + +#include +#include + +#define MAX_THREADS 32 +#define SPIN_SECS 6 +#define THR_SPINNER_PRI 63 +#define THR_MANAGER_PRI 62 +#define WARMUP_ITERATIONS 100 +#define FILE_SIZE (16384 * 4096) +#define IO_SIZE 4096 +#define IO_COUNT 2500 + +static mach_timebase_info_data_t timebase_info; +static semaphore_t semaphore; +static semaphore_t worker_sem; +static uint32_t g_numcpus; +static _Atomic uint32_t keep_going = 1; +int test_file_fd = 0; +char *data_buf = NULL; +extern char **environ; + +static struct { + pthread_t thread; +} threads[MAX_THREADS]; + +static uint64_t +nanos_to_abs(uint64_t nanos) +{ + return nanos * timebase_info.denom / timebase_info.numer; +} + +static void +io_perf_test_io_init(void) +{ + int spawn_ret, pid; + char *const mount_args[] = {"/usr/local/sbin/mount_nand.sh", NULL}; + spawn_ret = posix_spawn(&pid, mount_args[0], NULL, NULL, mount_args, environ); + if (spawn_ret < 0) { + T_SKIP("NAND mounting in LTE not possible on this device. Skipping test!"); + } + waitpid(pid, &spawn_ret, 0); + if (WIFEXITED(spawn_ret) && !WEXITSTATUS(spawn_ret)) { + T_PASS("NAND mounted successfully"); + } else { + T_SKIP("Unable to mount NAND. Skipping test!"); + } + + /* Mark the main thread as fixed priority */ + struct sched_param param = {.sched_priority = THR_MANAGER_PRI}; + T_ASSERT_POSIX_ZERO(pthread_setschedparam(pthread_self(), SCHED_FIFO, ¶m), + "pthread_setschedparam"); + + /* Set I/O Policy to Tier 0 */ + T_ASSERT_POSIX_ZERO(setiopolicy_np(IOPOL_TYPE_DISK, IOPOL_SCOPE_PROCESS, + IOPOL_IMPORTANT), "setiopolicy"); + + /* Create data buffer */ + data_buf = malloc(IO_SIZE * 16); + T_ASSERT_NOTNULL(data_buf, "Data buffer allocation"); + + int rndfd = open("/dev/urandom", O_RDONLY, S_IRUSR); + T_ASSERT_POSIX_SUCCESS(rndfd, "Open /dev/urandom"); + T_ASSERT_GE_INT((int)read(rndfd, data_buf, IO_SIZE * 16), 0, "read /dev/urandom"); + close(rndfd); + + /* Create test file */ + int fd = open("/mnt2/test", O_CREAT | O_WRONLY, S_IRUSR); + T_ASSERT_POSIX_SUCCESS(fd, 0, "Open /mnt2/test for writing!"); + + T_ASSERT_POSIX_ZERO(fcntl(fd, F_NOCACHE, 1), "fcntl F_NOCACHE enable"); + for (int size = 0; size < FILE_SIZE;) { + T_QUIET; + T_ASSERT_GE_INT((int)write(fd, data_buf, IO_SIZE * 16), 0, "write test file"); + size += (IO_SIZE * 16); + } + close(fd); + sync(); + +} + +static pthread_t +create_thread(uint32_t thread_id, uint32_t priority, bool fixpri, + void *(*start_routine)(void *)) +{ + int rv; + pthread_t new_thread; + struct sched_param param = { .sched_priority = (int)priority }; + pthread_attr_t attr; + + T_ASSERT_POSIX_ZERO(pthread_attr_init(&attr), "pthread_attr_init"); + + T_ASSERT_POSIX_ZERO(pthread_attr_setschedparam(&attr, ¶m), + "pthread_attr_setschedparam"); + + if (fixpri) { + T_ASSERT_POSIX_ZERO(pthread_attr_setschedpolicy(&attr, SCHED_RR), + "pthread_attr_setschedpolicy"); + } + + T_ASSERT_POSIX_ZERO(pthread_create(&new_thread, &attr, start_routine, + (void*)(uintptr_t)thread_id), "pthread_create"); + + T_ASSERT_POSIX_ZERO(pthread_attr_destroy(&attr), "pthread_attr_destroy"); + + threads[thread_id].thread = new_thread; + + return new_thread; +} + +/* Spin until a specified number of seconds elapses */ +static void +spin_for_duration(uint32_t seconds) +{ + uint64_t duration = nanos_to_abs((uint64_t)seconds * NSEC_PER_SEC); + uint64_t current_time = mach_absolute_time(); + uint64_t timeout = duration + current_time; + + uint64_t spin_count = 0; + + while (mach_absolute_time() < timeout && atomic_load_explicit(&keep_going, + memory_order_relaxed)) { + spin_count++; + } +} + +static void * +spin_thread(void *arg) +{ + uint32_t thread_id = (uint32_t) arg; + char name[30] = ""; + + snprintf(name, sizeof(name), "spin thread %2d", thread_id); + pthread_setname_np(name); + T_ASSERT_MACH_SUCCESS(semaphore_wait_signal(semaphore, worker_sem), + "semaphore_wait_signal"); + spin_for_duration(SPIN_SECS); + return NULL; +} + +void +perform_io(dt_stat_time_t stat) +{ + /* Open the test data file */ + int test_file_fd = open("/mnt2/test", O_RDONLY); + T_WITH_ERRNO; + T_ASSERT_POSIX_SUCCESS(test_file_fd, "Open test data file"); + + /* Disable caching and read-ahead for the file */ + T_ASSERT_POSIX_ZERO(fcntl(test_file_fd, F_NOCACHE, 1), "fcntl F_NOCACHE enable"); + T_ASSERT_POSIX_ZERO(fcntl(test_file_fd, F_RDAHEAD, 0), "fcntl F_RDAHEAD disable"); + + uint32_t count = 0; + int ret; + + for (int i=0; i < WARMUP_ITERATIONS; i++) { + /* Warmup loop */ + read(test_file_fd, data_buf, IO_SIZE); + } + + do { + T_STAT_MEASURE(stat) { + ret = read(test_file_fd, data_buf, IO_SIZE); + } + if (ret == 0) { + T_QUIET; + T_ASSERT_POSIX_SUCCESS(lseek(test_file_fd, 0, SEEK_SET), "lseek begin"); + } else if (ret < 0) { + T_FAIL("read failure"); + T_END; + } + count++; + } while(count < IO_COUNT); + close(test_file_fd); +} + +T_GLOBAL_META(T_META_NAMESPACE("xnu.io")); + +/* Disable the test on MacOS for now */ +T_DECL(read_perf, "Sequential Uncached Read Performance", T_META_TYPE_PERF, T_META_CHECK_LEAKS(NO), T_META_ASROOT(YES), T_META_LTEPHASE(LTE_POSTINIT)) +{ + +#if !CONFIG_EMBEDDED + T_SKIP("Not supported on MacOS"); +#endif /* !CONFIG_EMBEDDED */ + + io_perf_test_io_init(); + pthread_setname_np("main thread"); + + T_ASSERT_MACH_SUCCESS(mach_timebase_info(&timebase_info), "mach_timebase_info"); + + dt_stat_time_t seq_noload = dt_stat_time_create("sequential read latency (CPU idle)"); + perform_io(seq_noload); + dt_stat_finalize(seq_noload); + + /* + * We create spinner threads for this test so that all other cores are + * busy. That way the I/O issue thread has to context switch to the + * IOWorkLoop thread and back for the I/O. + */ + T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &semaphore, + SYNC_POLICY_FIFO, 0), "semaphore_create"); + + T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &worker_sem, + SYNC_POLICY_FIFO, 0), "semaphore_create"); + + size_t ncpu_size = sizeof(g_numcpus); + T_ASSERT_POSIX_SUCCESS(sysctlbyname("hw.ncpu", &g_numcpus, &ncpu_size, NULL, 0), + "sysctlbyname(hw.ncpu)"); + + T_LOG("hw.ncpu: %d\n", g_numcpus); + uint32_t n_spinners = g_numcpus - 1; + + for (uint32_t thread_id = 0; thread_id < n_spinners; thread_id++) { + threads[thread_id].thread = create_thread(thread_id, THR_SPINNER_PRI, + true, &spin_thread); + } + + for (uint32_t thread_id = 0; thread_id < n_spinners; thread_id++) { + T_ASSERT_MACH_SUCCESS(semaphore_wait(worker_sem), "semaphore_wait"); + } + + T_ASSERT_MACH_SUCCESS(semaphore_signal_all(semaphore), "semaphore_signal"); + + dt_stat_time_t seq_load = dt_stat_time_create("sequential read latency (Single CPU)"); + perform_io(seq_load); + dt_stat_finalize(seq_load); + + atomic_store_explicit(&keep_going, 0, memory_order_relaxed); + for (uint32_t thread_id = 0; thread_id < n_spinners; thread_id++) { + T_ASSERT_POSIX_ZERO(pthread_join(threads[thread_id].thread, NULL), + "pthread_join %d", thread_id); + } +} diff --git a/tools/tests/darwintests/kdebug.c b/tools/tests/darwintests/kdebug.c index 5eb0ff6c2..3cc0e2200 100644 --- a/tools/tests/darwintests/kdebug.c +++ b/tools/tests/darwintests/kdebug.c @@ -1,8 +1,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include @@ -13,7 +13,20 @@ #include #include -#define KTRACE_WAIT_TIMEOUT_S (10) +#define KDBG_TEST_MACROS 1 +#define KDBG_TEST_OLD_TIMES 2 + +static void +assert_kdebug_test(unsigned int flavor) +{ + size_t size = flavor; + int mib[] = { CTL_KERN, KERN_KDEBUG, KERN_KDTEST }; + T_ASSERT_POSIX_SUCCESS( + sysctl(mib, sizeof(mib) / sizeof(mib[0]), NULL, &size, NULL, 0), + "KERN_KDTEST sysctl"); +} + +#pragma mark kdebug syscalls #define TRACE_DEBUGID (0xfedfed00U) @@ -21,7 +34,6 @@ T_DECL(kdebug_trace_syscall, "test that kdebug_trace(2) emits correct events", T_META_ASROOT(true)) { ktrace_session_t s; - dispatch_time_t timeout; __block int events_seen = 0; s = ktrace_session_create(); @@ -65,7 +77,6 @@ T_DECL(kdebug_signpost_syscall, ktrace_session_t s; __block int single_seen = 0; __block int paired_seen = 0; - dispatch_time_t timeout; s = ktrace_session_create(); T_ASSERT_NOTNULL(s, NULL); @@ -134,11 +145,13 @@ T_DECL(kdebug_signpost_syscall, dispatch_main(); } +#pragma mark kdebug behaviors + #define WRAPPING_EVENTS_COUNT (150000) #define TRACE_ITERATIONS (5000) #define WRAPPING_EVENTS_THRESHOLD (100) -T_DECL(kdebug_wrapping, +T_DECL(wrapping, "ensure that wrapping traces lost events and no events prior to the wrap", T_META_ASROOT(true), T_META_CHECK_LEAKS(false)) { @@ -218,6 +231,91 @@ T_DECL(kdebug_wrapping, dispatch_main(); } +T_DECL(reject_old_events, + "ensure that kdebug rejects events from before tracing began", + T_META_ASROOT(true), T_META_CHECK_LEAKS(false)) +{ + __block uint64_t event_horizon_ts; + __block int events = 0; + + ktrace_session_t s = ktrace_session_create(); + T_QUIET; T_ASSERT_NOTNULL(s, "ktrace_session_create"); + + ktrace_events_range(s, KDBG_EVENTID(DBG_BSD, DBG_BSD_KDEBUG_TEST, 0), + KDBG_EVENTID(DBG_BSD + 1, 0, 0), + ^(struct trace_point *tp) + { + events++; + T_EXPECT_GT(tp->timestamp, event_horizon_ts, + "events in trace should be from after tracing began"); + }); + + ktrace_set_completion_handler(s, ^{ + T_EXPECT_EQ(events, 2, "should see only two events"); + ktrace_session_destroy(s); + T_END; + }); + + event_horizon_ts = mach_absolute_time(); + + T_ASSERT_POSIX_ZERO(ktrace_start(s, dispatch_get_main_queue()), NULL); + /* first, try an old event at the beginning of trace */ + assert_kdebug_test(KDBG_TEST_OLD_TIMES); + /* after a good event has been traced, old events should be rejected */ + assert_kdebug_test(KDBG_TEST_OLD_TIMES); + ktrace_end(s, 0); + + dispatch_main(); +} + +#define ORDERING_TIMEOUT_SEC 5 + +T_DECL(ascending_time_order, + "ensure that kdebug events are in ascending order based on time", + T_META_ASROOT(true), T_META_CHECK_LEAKS(false)) +{ + __block uint64_t prev_ts = 0; + __block uint32_t prev_debugid = 0; + __block unsigned int prev_cpu = 0; + __block bool in_order = true; + + ktrace_session_t s = ktrace_session_create(); + T_QUIET; T_ASSERT_NOTNULL(s, "ktrace_session_create"); + + ktrace_events_all(s, ^(struct trace_point *tp) { + if (tp->timestamp < prev_ts) { + in_order = false; + T_FAIL("found timestamps out of order"); + T_LOG("%" PRIu64 ": %#" PRIx32 " (cpu %d)", + prev_ts, prev_debugid, prev_cpu); + T_LOG("%" PRIu64 ": %#" PRIx32 " (cpu %d)", + tp->timestamp, tp->debugid, tp->cpuid); + } + }); + + ktrace_set_completion_handler(s, ^{ + ktrace_session_destroy(s); + T_EXPECT_TRUE(in_order, "event timestamps were in-order"); + T_END; + }); + + T_ASSERT_POSIX_ZERO(ktrace_start(s, dispatch_get_main_queue()), NULL); + + /* try to inject old timestamps into trace */ + assert_kdebug_test(KDBG_TEST_OLD_TIMES); + + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, ORDERING_TIMEOUT_SEC * NSEC_PER_SEC), + dispatch_get_main_queue(), ^{ + T_LOG("ending test after timeout"); + ktrace_end(s, 1); + }); + + dispatch_main(); + +} + +#pragma mark dyld tracing + __attribute__((aligned(8))) static const char map_uuid[16] = "map UUID"; @@ -404,6 +502,8 @@ T_DECL(dyld_events, "test that dyld registering libraries emits events", dispatch_main(); } +#pragma mark kdebug kernel macros + #define EXP_KERNEL_EVENTS 5U static const uint32_t dev_evts[EXP_KERNEL_EVENTS] = { @@ -449,15 +549,6 @@ is_development_kernel(void) return is_development; } -static void -assert_kdebug_test(void) -{ - int mib[] = { CTL_KERN, KERN_KDEBUG, KERN_KDTEST }; - T_ASSERT_POSIX_SUCCESS( - sysctl(mib, sizeof(mib) / sizeof(mib[0]), NULL, NULL, NULL, 0), - "KERN_KDTEST"); -} - static void expect_event(struct trace_point *tp, unsigned int *events, const uint32_t *event_ids, size_t event_ids_len) @@ -533,13 +624,17 @@ T_DECL(kernel_events, "ensure kernel macros work", * Development-only events are only filtered if running on an embedded * OS. */ - unsigned dev_exp; + unsigned int dev_exp; +#if TARGET_OS_EMBEDDED + dev_exp = is_development_kernel() ? EXP_KERNEL_EVENTS : 0U; +#else dev_exp = EXP_KERNEL_EVENTS; +#endif T_EXPECT_EQ(rel_seen, EXP_KERNEL_EVENTS, "release and development events seen"); T_EXPECT_EQ(dev_seen, dev_exp, "development-only events seen/not seen"); - T_EXPECT_EQ(filt_seen, EXP_KERNEL_EVENTS, "filter-only events seen"); + T_EXPECT_EQ(filt_seen, dev_exp, "filter-only events seen"); ktrace_session_destroy(s); T_END; }); @@ -547,7 +642,7 @@ T_DECL(kernel_events, "ensure kernel macros work", ktrace_filter_pid(s, getpid()); T_ASSERT_POSIX_ZERO(ktrace_start(s, dispatch_get_main_queue()), NULL); - assert_kdebug_test(); + assert_kdebug_test(KDBG_TEST_MACROS); ktrace_end(s, 0); @@ -576,16 +671,20 @@ T_DECL(kernel_events_filtered, "ensure that the filtered kernel macros work", ktrace_session_destroy(s); T_EXPECT_EQ(rel_seen, EXP_KERNEL_EVENTS, NULL); +#if defined(__arm__) || defined(__arm64__) + T_EXPECT_EQ(dev_seen, is_development_kernel() ? EXP_KERNEL_EVENTS : 0U, + NULL); +#else T_EXPECT_EQ(dev_seen, EXP_KERNEL_EVENTS, NULL); +#endif /* defined(__arm__) || defined(__arm64__) */ T_EXPECT_EQ(filt_seen, 0U, NULL); T_END; }); T_ASSERT_POSIX_ZERO(ktrace_start(s, dispatch_get_main_queue()), NULL); - assert_kdebug_test(); + assert_kdebug_test(KDBG_TEST_MACROS); ktrace_end(s, 0); dispatch_main(); } - diff --git a/tools/tests/darwintests/kevent_pty.c b/tools/tests/darwintests/kevent_pty.c new file mode 100644 index 000000000..a64c48dfa --- /dev/null +++ b/tools/tests/darwintests/kevent_pty.c @@ -0,0 +1,259 @@ +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif /* T_NAMESPACE */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +T_GLOBAL_META( + T_META_NAMESPACE("xnu.kevent"), + T_META_CHECK_LEAKS(false)); + +#define TIMEOUT_SECS 10 + +static int child_ready[2]; + +static void +child_tty_client(void) +{ + dispatch_source_t src; + char buf[16] = ""; + ssize_t bytes_wr; + + src = dispatch_source_create(DISPATCH_SOURCE_TYPE_READ, + (uintptr_t)STDIN_FILENO, 0, NULL); + if (!src) { + exit(1); + } + dispatch_source_set_event_handler(src, ^{}); + + dispatch_activate(src); + + close(child_ready[0]); + snprintf(buf, sizeof(buf), "%ds", getpid()); + bytes_wr = write(child_ready[1], buf, strlen(buf)); + if (bytes_wr < 0) { + exit(1); + } + + dispatch_main(); +} + +static void +pty_master(void) +{ + pid_t child_pid; + int ret; + + child_pid = fork(); + if (child_pid == 0) { + child_tty_client(); + } + ret = setpgid(child_pid, child_pid); + if (ret < 0) { + exit(1); + } + ret = tcsetpgrp(STDIN_FILENO, child_pid); + if (ret < 0) { + exit(1); + } + + sleep(TIMEOUT_SECS); + exit(1); +} + +T_DECL(pty_master_teardown, + "try removing a TTY master out from under a PTY slave holding a kevent", + T_META_ASROOT(true)) +{ + __block pid_t master_pid; + char buf[16] = ""; + char *end; + ssize_t bytes_rd; + size_t buf_len = 0; + unsigned long slave_pid; + int master_fd; + char pty_filename[PATH_MAX]; + int status; + + T_SETUPBEGIN; + T_ASSERT_POSIX_SUCCESS(pipe(child_ready), NULL); + + master_pid = forkpty(&master_fd, pty_filename, NULL, NULL); + if (master_pid == 0) { + pty_master(); + __builtin_unreachable(); + } + T_ASSERT_POSIX_SUCCESS(master_pid, + "forked child master PTY with pid %d, at pty %s", master_pid, + pty_filename); + + close(child_ready[1]); + + end = buf; + do { + bytes_rd = read(child_ready[0], end, sizeof(buf) - buf_len); + T_ASSERT_POSIX_SUCCESS(bytes_rd, "read on pipe between master and runner"); + buf_len += (size_t)bytes_rd; + T_LOG("runner read %zd bytes", bytes_rd); + end += bytes_rd; + } while (bytes_rd != 0 && *(end - 1) != 's'); + + slave_pid = strtoul(buf, &end, 0); + if (buf == end) { + T_ASSERT_FAIL("could not parse child PID from master pipe"); + } + + T_LOG("got pid %lu for slave process from master", slave_pid); + T_SETUPEND; + + T_LOG("sending fatal signal to master"); + T_ASSERT_POSIX_SUCCESS(kill(master_pid, SIGKILL), NULL); + + T_LOG("sending fatal signal to slave"); + (void)kill((int)slave_pid, SIGKILL); + + T_ASSERT_POSIX_SUCCESS(waitpid(master_pid, &status, 0), NULL); + T_ASSERT_TRUE(WIFSIGNALED(status), "master PID was signaled"); + (void)waitpid((int)slave_pid, &status, 0); +} + +volatile static bool writing = true; + +static void * +reader_thread(void *arg) +{ + int fd = (int)arg; + char c; + + T_SETUPBEGIN; + T_QUIET; + T_ASSERT_GT(fd, 0, "reader thread received valid fd"); + T_SETUPEND; + + for (;;) { + ssize_t rdsize = read(fd, &c, sizeof(c)); + if (rdsize == -1) { + if (errno == EINTR) { + continue; + } else if (errno == EBADF) { + T_LOG("reader got an error (%s), shutting down", strerror(errno)); + return NULL; + } else { + T_ASSERT_POSIX_SUCCESS(rdsize, "read on PTY"); + } + } else if (rdsize == 0) { + return NULL; + } + } + + return NULL; +} + +static void * +writer_thread(void *arg) +{ + int fd = (int)arg; + char c[4096]; + + T_SETUPBEGIN; + T_QUIET; + T_ASSERT_GT(fd, 0, "writer thread received valid fd"); + memset(c, 'a', sizeof(c)); + T_SETUPEND; + + while (writing) { + ssize_t wrsize = write(fd, c, sizeof(c)); + if (wrsize == -1) { + if (errno == EINTR) { + continue; + } else { + T_LOG("writer got an error (%s), shutting down", strerror(errno)); + return NULL; + } + } + } + + return NULL; +} + +#define ATTACH_ITERATIONS 10000 + +static int master, slave; +static pthread_t reader, writer; + +static void +join_threads(void) +{ + close(slave); + close(master); + writing = false; + pthread_join(reader, NULL); + pthread_join(writer, NULL); +} + +static void +redispatch(dispatch_group_t grp, dispatch_source_type_t type, int fd) +{ + __block int iters = 0; + + __block void (^redispatch_blk)(void) = Block_copy(^{ + if (iters++ > ATTACH_ITERATIONS) { + return; + } else if (iters == ATTACH_ITERATIONS) { + dispatch_group_leave(grp); + T_PASS("created %d %s sources on busy PTY", iters, + type == DISPATCH_SOURCE_TYPE_READ ? "read" : "write"); + } + + dispatch_source_t src = dispatch_source_create( + type, (uintptr_t)fd, 0, + dispatch_get_main_queue()); + + dispatch_source_set_event_handler(src, ^{ + dispatch_cancel(src); + }); + + dispatch_source_set_cancel_handler(src, redispatch_blk); + + dispatch_activate(src); + }); + + dispatch_group_enter(grp); + dispatch_async(dispatch_get_main_queue(), redispatch_blk); +} + +T_DECL(attach_while_tty_wakeups, + "try to attach knotes while a TTY is getting wakeups") +{ + dispatch_group_t grp = dispatch_group_create(); + + T_SETUPBEGIN; + T_ASSERT_POSIX_SUCCESS(openpty(&master, &slave, NULL, NULL, NULL), NULL); + + T_ASSERT_POSIX_ZERO(pthread_create(&reader, NULL, reader_thread, + (void *)(uintptr_t)master), NULL); + T_ASSERT_POSIX_ZERO(pthread_create(&writer, NULL, writer_thread, + (void *)(uintptr_t)slave), NULL); + T_ATEND(join_threads); + T_SETUPEND; + + redispatch(grp, DISPATCH_SOURCE_TYPE_READ, master); + redispatch(grp, DISPATCH_SOURCE_TYPE_WRITE, slave); + + dispatch_group_notify(grp, dispatch_get_main_queue(), ^{ + T_LOG("both reader and writer sources cleaned up"); + T_END; + }); + + dispatch_main(); +} diff --git a/tools/tests/darwintests/kevent_qos.c b/tools/tests/darwintests/kevent_qos.c new file mode 100644 index 000000000..823bf1a93 --- /dev/null +++ b/tools/tests/darwintests/kevent_qos.c @@ -0,0 +1,908 @@ +/* + * kevent_qos: Tests Synchronous IPC QOS override. + */ + +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +T_GLOBAL_META(T_META_NAMESPACE("xnu.kevent_qos")); + +#define ARRAYLEN(arr) (sizeof(arr) / sizeof(arr[0])) + +#define RECV_TIMEOUT_SECS (4) +#define SEND_TIMEOUT_SECS (6) +#define HELPER_TIMEOUT_SECS (15) + +#define ENV_VAR_QOS (3) +static const char *qos_env[ENV_VAR_QOS] = {"XNU_TEST_QOS_BO", "XNU_TEST_QOS_QO", "XNU_TEST_QOS_AO"}; +static const char *qos_name_env[ENV_VAR_QOS] = {"XNU_TEST_QOS_NAME_BO", "XNU_TEST_QOS_NAME_QO", "XNU_TEST_QOS_NAME_AO"}; + +#define ENV_VAR_FUNCTION (1) +static const char *wl_function_name = "XNU_TEST_WL_FUNCTION"; + +static qos_class_t g_expected_qos[ENV_VAR_QOS]; +static const char *g_expected_qos_name[ENV_VAR_QOS]; + +#define ENV_QOS_BEFORE_OVERRIDE (0) +#define ENV_QOS_QUEUE_OVERRIDE (1) +#define ENV_QOS_AFTER_OVERRIDE (2) + +#pragma mark pthread callbacks + +static void +worker_cb(pthread_priority_t __unused priority) +{ + T_FAIL("a worker thread was created"); +} + +static void +event_cb(void ** __unused events, int * __unused nevents) +{ + T_FAIL("a kevent routine was called instead of workloop"); +} + +/* + * Basic WL handler callback, it sleeps for n seconds and then checks the + * effective Qos of the servicer thread. + */ +static void +workloop_cb_test_intransit(uint64_t *workloop_id __unused, void **eventslist __unused, int *events) +{ + T_LOG("Workloop handler workloop_cb_test_intransit called. " + "Will wait for %d seconds to make sure client enqueues the sync msg \n", + 2 * RECV_TIMEOUT_SECS); + + /* Wait for the client to send the high priority message to override the qos */ + sleep(2 * RECV_TIMEOUT_SECS); + + /* Skip the test if we can't check Qos */ + if (geteuid() != 0) { + T_SKIP("kevent_qos test requires root privileges to run."); + } + + /* The effective Qos should be the one expected after override */ + T_EXPECT_EFFECTIVE_QOS_EQ(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], + "dispatch_source event handler QoS should be %s", g_expected_qos_name[ENV_QOS_AFTER_OVERRIDE]); + + T_END; + *events = 0; +} + +/* + * WL handler which checks if the servicer thread has correct Qos. + */ +static void +workloop_cb_test_sync_send(uint64_t *workloop_id __unused, void **eventslist __unused, int *events) +{ + T_LOG("Workloop handler workloop_cb_test_sync_send called"); + + if (geteuid() != 0) { + T_SKIP("kevent_qos test requires root privileges to run."); + } + + /* The effective Qos should be the one expected after override */ + T_EXPECT_EFFECTIVE_QOS_EQ(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], + "dispatch_source event handler QoS should be %s", g_expected_qos_name[ENV_QOS_AFTER_OVERRIDE]); + + T_END; + *events = 0; +} + +/* + * WL handler which checks the overridden Qos and then enables the knote and checks + * for the Qos again if that dropped the sync ipc override. + */ +static void +workloop_cb_test_sync_send_and_enable(uint64_t *workloop_id, struct kevent_qos_s **eventslist, int *events) +{ + int r; + T_LOG("Workloop handler workloop_cb_test_sync_send_and_enable called"); + + if (geteuid() != 0) { + T_SKIP("kevent_qos test requires root privileges to run."); + } + + /* The effective Qos should be the one expected after override */ + T_EXPECT_EFFECTIVE_QOS_EQ(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], + "dispatch_source event handler QoS should be %s", g_expected_qos_name[ENV_QOS_AFTER_OVERRIDE]); + + /* Enable the knote */ + struct kevent_qos_s *kev = *eventslist; + kev->flags = EV_ADD | EV_ENABLE | EV_UDATA_SPECIFIC | EV_DISPATCH | EV_VANISHED; + struct kevent_qos_s kev_err[] = {{ 0 }}; + + r = kevent_id(*workloop_id, kev, 1, kev_err, 1, NULL, + NULL, KEVENT_FLAG_WORKLOOP | KEVENT_FLAG_ERROR_EVENTS | KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST); + T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "kevent_id"); + + /* Sync override should have been removed */ + T_EXPECT_EFFECTIVE_QOS_EQ(g_expected_qos[ENV_QOS_BEFORE_OVERRIDE], + "dispatch_source event handler QoS should be %s", g_expected_qos_name[ENV_QOS_BEFORE_OVERRIDE]); + + T_END; + *events = 0; +} + +/* + * WL handler receives the first message and checks sync ipc override, then enables the knote + * and receives 2nd message and checks it sync ipc override. + */ +static int send_two_sync_handler_called = 0; +static void +workloop_cb_test_send_two_sync(uint64_t *workloop_id __unused, struct kevent_qos_s **eventslist, int *events) +{ + T_LOG("Workloop handler workloop_cb_test_send_two_sync called for %d time", send_two_sync_handler_called + 1); + + if (geteuid() != 0) { + T_SKIP("kevent_qos test requires root privileges to run."); + } + + T_LOG("Number of events received is %d\n", *events); + + if (send_two_sync_handler_called == 0) { + /* The effective Qos should be the one expected after override */ + T_EXPECT_EFFECTIVE_QOS_EQ(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], + "dispatch_source event handler QoS should be %s", g_expected_qos_name[ENV_QOS_AFTER_OVERRIDE]); + + /* Enable the knote to get 2nd message */ + struct kevent_qos_s *kev = *eventslist; + kev->flags = EV_ADD | EV_ENABLE | EV_UDATA_SPECIFIC | EV_DISPATCH | EV_VANISHED; + kev->fflags = (MACH_RCV_MSG | MACH_RCV_LARGE | MACH_RCV_LARGE_IDENTITY | + MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_CTX) | + MACH_RCV_TRAILER_TYPE(MACH_MSG_TRAILER_FORMAT_0) | + MACH_RCV_VOUCHER); + *events = 1; + } else { + T_EXPECT_EFFECTIVE_QOS_EQ(g_expected_qos[ENV_QOS_BEFORE_OVERRIDE], + "dispatch_source event handler QoS should be %s", g_expected_qos_name[ENV_QOS_BEFORE_OVERRIDE]); + T_END; + *events = 0; + } + send_two_sync_handler_called++; +} + +/* + * Checks the sync ipc override and then waits for client to destroy the + * special reply port and checks if that removes the sync ipc override. + */ +static boolean_t two_send_and_destroy_test_passed = FALSE; +static int two_send_and_destroy_handler = 0; +static void +workloop_cb_test_two_send_and_destroy(uint64_t *workloop_id __unused, struct kevent_qos_s **eventslist __unused, int *events) +{ + T_LOG("Workloop handler workloop_cb_test_two_send_and_destroy called %d times", two_send_and_destroy_handler + 1); + + if (geteuid() != 0) { + T_SKIP("kevent_qos test requires root privileges to run."); + } + + if (two_send_and_destroy_handler == 0) { + /* The effective Qos should be the one expected after override */ + T_EXPECT_EFFECTIVE_QOS_EQ(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], + "dispatch_source event handler QoS should be %s", g_expected_qos_name[ENV_QOS_AFTER_OVERRIDE]); + + sleep(2 * RECV_TIMEOUT_SECS); + + /* Special reply port should have been destroyed, check Qos again */ + T_EXPECT_EFFECTIVE_QOS_EQ(g_expected_qos[ENV_QOS_BEFORE_OVERRIDE], + "dispatch_source event handler QoS should be %s", g_expected_qos_name[ENV_QOS_BEFORE_OVERRIDE]); + + two_send_and_destroy_test_passed = TRUE; + } else { + if (two_send_and_destroy_test_passed) { + T_END; + } + } + + /* Enable the knote to get next message */ + struct kevent_qos_s *kev = *eventslist; + kev->flags = EV_ADD | EV_ENABLE | EV_UDATA_SPECIFIC | EV_DISPATCH | EV_VANISHED; + kev->fflags = (MACH_RCV_MSG | MACH_RCV_LARGE | MACH_RCV_LARGE_IDENTITY | + MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_CTX) | + MACH_RCV_TRAILER_TYPE(MACH_MSG_TRAILER_FORMAT_0) | + MACH_RCV_VOUCHER); + *events = 1; + two_send_and_destroy_handler++; + T_LOG("Handler returning \n"); +} + +#pragma mark Mach receive + +#define KEVENT_QOS_SERVICE_NAME "com.apple.xnu.test.kevent_qos" + +static mach_port_t +get_server_port(void) +{ + mach_port_t port; + kern_return_t kr = bootstrap_check_in(bootstrap_port, + KEVENT_QOS_SERVICE_NAME, &port); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "server bootstrap_check_in"); + return port; +} + +static void +env_set_qos(char **env, qos_class_t qos[], const char *qos_name[], const char *wl_function) +{ + int i; + char *qos_str, *qos_name_str; + for (i = 0; i < ENV_VAR_QOS; i++) { + T_QUIET; T_ASSERT_POSIX_SUCCESS(asprintf(&qos_str, "%s=%d", qos_env[i] , qos[i]), + NULL); + T_QUIET; T_ASSERT_POSIX_SUCCESS( + asprintf(&qos_name_str, "%s=%s", qos_name_env[i], qos_name[i]), NULL); + env[2 * i] = qos_str; + env[2 * i + 1] = qos_name_str; + } + T_QUIET; T_ASSERT_POSIX_SUCCESS(asprintf(&env[2 * i], "%s=%s", wl_function_name, wl_function), + NULL); + env[2 * i + 1] = NULL; +} + +static void +environ_get_qos(qos_class_t qos[], const char *qos_name[], const char **wl_function) +{ + char *qos_str; + char *qos_end; + int i; + + for (i = 0; i < ENV_VAR_QOS; i++) { + qos_str = getenv(qos_env[i]); + T_QUIET; T_ASSERT_NOTNULL(qos_str, "getenv(%s)", qos_env[i]); + + unsigned long qos_l = strtoul(qos_str, &qos_end, 10); + T_QUIET; T_ASSERT_EQ(*qos_end, '\0', "getenv(%s) = '%s' should be an " + "integer", qos_env[i], qos_str); + + T_QUIET; T_ASSERT_LT(qos_l, (unsigned long)100, "getenv(%s) = '%s' should " + "be less than 100", qos_env[i], qos_str); + + qos[i] = (qos_class_t)qos_l; + qos_name[i] = getenv(qos_name_env[i]); + T_QUIET; T_ASSERT_NOTNULL(qos_name[i], "getenv(%s)", qos_name_env[i]); + } + *wl_function = getenv(wl_function_name); + T_QUIET; T_ASSERT_NOTNULL(*wl_function, "getenv(%s)", wl_function_name); +} + +static mach_voucher_t +create_pthpriority_voucher(mach_msg_priority_t qos) +{ + char voucher_buf[sizeof(mach_voucher_attr_recipe_data_t) + sizeof(ipc_pthread_priority_value_t)]; + + mach_voucher_t voucher = MACH_PORT_NULL; + kern_return_t ret; + ipc_pthread_priority_value_t ipc_pthread_priority_value = + (ipc_pthread_priority_value_t)qos; + + mach_voucher_attr_raw_recipe_array_t recipes; + mach_voucher_attr_raw_recipe_size_t recipe_size = 0; + mach_voucher_attr_recipe_t recipe = + (mach_voucher_attr_recipe_t)&voucher_buf[recipe_size]; + + recipe->key = MACH_VOUCHER_ATTR_KEY_PTHPRIORITY; + recipe->command = MACH_VOUCHER_ATTR_PTHPRIORITY_CREATE; + recipe->previous_voucher = MACH_VOUCHER_NULL; + memcpy((char *)&recipe->content[0], &ipc_pthread_priority_value, sizeof(ipc_pthread_priority_value)); + recipe->content_size = sizeof(ipc_pthread_priority_value_t); + recipe_size += sizeof(mach_voucher_attr_recipe_data_t) + recipe->content_size; + + recipes = (mach_voucher_attr_raw_recipe_array_t)&voucher_buf[0]; + + ret = host_create_mach_voucher(mach_host_self(), + recipes, + recipe_size, + &voucher); + + T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "client host_create_mach_voucher"); + return voucher; +} + +static void +send( + mach_port_t send_port, + mach_port_t reply_port, + mach_port_t msg_port, + mach_msg_priority_t qos) +{ + kern_return_t ret = 0; + + struct { + mach_msg_header_t header; + mach_msg_body_t body; + mach_msg_port_descriptor_t port_descriptor; + } send_msg = { + .header = + { + .msgh_remote_port = send_port, + .msgh_local_port = reply_port, + .msgh_bits = MACH_MSGH_BITS_SET(MACH_MSG_TYPE_COPY_SEND, + reply_port ? MACH_MSG_TYPE_MAKE_SEND_ONCE : 0, + MACH_MSG_TYPE_MOVE_SEND, + MACH_MSGH_BITS_COMPLEX), + .msgh_id = 0x100, + .msgh_size = sizeof(send_msg), + .msgh_voucher_port = create_pthpriority_voucher(qos), + }, + .body = + { + .msgh_descriptor_count = 1, + }, + .port_descriptor = + { + .name = msg_port, .disposition = MACH_MSG_TYPE_MOVE_RECEIVE, .type = MACH_MSG_PORT_DESCRIPTOR, + }, + }; + + if (msg_port == MACH_PORT_NULL) { + send_msg.body.msgh_descriptor_count = 0; + } + + ret = mach_msg(&(send_msg.header), + MACH_SEND_MSG | + MACH_SEND_TIMEOUT | + MACH_SEND_OVERRIDE| + (reply_port ? MACH_SEND_SYNC_OVERRIDE : 0) , + send_msg.header.msgh_size, + 0, + MACH_PORT_NULL, + 0, + 0); + + T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "client mach_msg"); +} + +static void +receive( + mach_port_t rcv_port, + mach_port_t notify_port) +{ + kern_return_t ret = 0; + + struct { + mach_msg_header_t header; + mach_msg_body_t body; + mach_msg_port_descriptor_t port_descriptor; + } rcv_msg = { + .header = + { + .msgh_remote_port = MACH_PORT_NULL, + .msgh_local_port = rcv_port, + .msgh_size = sizeof(rcv_msg), + }, + }; + + T_LOG("Client: Starting sync receive\n"); + + ret = mach_msg(&(rcv_msg.header), + MACH_RCV_MSG | + MACH_RCV_TIMEOUT | + MACH_RCV_SYNC_WAIT, + 0, + rcv_msg.header.msgh_size, + rcv_port, + SEND_TIMEOUT_SECS * 1000, + notify_port); + + if (!(ret == MACH_RCV_TIMED_OUT || ret == MACH_MSG_SUCCESS)) { + T_ASSERT_FAIL("Sync rcv failed \n"); + } +} + +T_HELPER_DECL(qos_get_special_reply_port, + "Test get_special_reply_port and it's corner cases.") +{ + mach_port_t special_reply_port; + mach_port_t new_special_reply_port; + + special_reply_port = thread_get_special_reply_port(); + T_QUIET; T_ASSERT_NOTNULL(special_reply_port , "get_thread_special_reply_port"); + + new_special_reply_port = thread_get_special_reply_port(); + T_QUIET; T_ASSERT_NOTNULL(new_special_reply_port , "get_thread_special_reply_port"); + + mach_port_destroy(mach_task_self(), special_reply_port); + mach_port_destroy(mach_task_self(), new_special_reply_port); + + new_special_reply_port = thread_get_special_reply_port(); + T_QUIET; T_ASSERT_NOTNULL(new_special_reply_port , "get_thread_special_reply_port"); + + T_END; +} + +T_HELPER_DECL(qos_client_send_to_intransit, + "Send synchronous messages to an intransit port") +{ + mach_port_t qos_send_port; + mach_port_t msg_port; + mach_port_t special_reply_port; + + kern_return_t kr = bootstrap_look_up(bootstrap_port, + KEVENT_QOS_SERVICE_NAME, &qos_send_port); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client bootstrap_look_up"); + + special_reply_port = thread_get_special_reply_port(); + T_QUIET; T_ASSERT_NOTNULL(special_reply_port , "get_thread_special_reply_port"); + + /* Create a rcv right to send in a msg */ + kr = mach_port_allocate(mach_task_self(), + MACH_PORT_RIGHT_RECEIVE, + &msg_port); + + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client mach_port_allocate"); + + kr = mach_port_insert_right(mach_task_self(), + msg_port, + msg_port, + MACH_MSG_TYPE_MAKE_SEND); + + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client mach_port_insert_right"); + + /* Send an empty msg on the port to fire the WL thread */ + send(qos_send_port, MACH_PORT_NULL, MACH_PORT_NULL, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_BEFORE_OVERRIDE], 0, 0)); + + sleep(SEND_TIMEOUT_SECS); + + /* Send the message with msg port as in-transit port, this msg will not be dequeued */ + send(qos_send_port, MACH_PORT_NULL, msg_port, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_BEFORE_OVERRIDE], 0, 0)); + + /* Send the message to the in-transit port, it should override the rcv's workloop */ + send(msg_port, special_reply_port, MACH_PORT_NULL, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], 0, 0)); + T_LOG("Client done sending messages, now waiting for server to end the test"); + sleep(2 * SEND_TIMEOUT_SECS); + + T_ASSERT_FAIL("client timed out"); +} + +T_HELPER_DECL(qos_client_send_sync_and_enqueue_rcv, + "Send synchronous messages and enqueue the rcv right") +{ + mach_port_t qos_send_port; + mach_port_t msg_port; + mach_port_t special_reply_port; + + kern_return_t kr = bootstrap_look_up(bootstrap_port, + KEVENT_QOS_SERVICE_NAME, &qos_send_port); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client bootstrap_look_up"); + + special_reply_port = thread_get_special_reply_port(); + T_QUIET; T_ASSERT_NOTNULL(special_reply_port , "get_thread_special_reply_port"); + + /* Create a rcv right to send in a msg */ + kr = mach_port_allocate(mach_task_self(), + MACH_PORT_RIGHT_RECEIVE, + &msg_port); + + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client mach_port_allocate"); + + kr = mach_port_insert_right(mach_task_self(), + msg_port, + msg_port, + MACH_MSG_TYPE_MAKE_SEND); + + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client mach_port_insert_right"); + + /* Send the message to msg port */ + send(msg_port, special_reply_port, MACH_PORT_NULL, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], 0, 0)); + + /* Send the message with msg port as in-transit port, copyin of in-transit will cause sync override */ + send(qos_send_port, MACH_PORT_NULL, msg_port, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_BEFORE_OVERRIDE], 0, 0)); + + T_LOG("Client done sending messages, now waiting for server to end the test"); + sleep(3 * SEND_TIMEOUT_SECS); + + T_ASSERT_FAIL("client timed out"); +} + +static void +thread_create_at_qos(qos_class_t qos, void * (*function)(void *)) +{ + qos_class_t qos_thread; + pthread_t thread; + pthread_attr_t attr; + int ret; + + ret = setpriority(PRIO_DARWIN_ROLE, 0, PRIO_DARWIN_ROLE_UI_FOCAL); + if (ret != 0) { + T_LOG("set priority failed\n"); + } + + pthread_attr_init(&attr); + pthread_attr_set_qos_class_np(&attr, qos, 0); + pthread_create(&thread, &attr, function, NULL); + + T_LOG("pthread created\n"); + pthread_get_qos_class_np(thread, &qos_thread, NULL); + T_EXPECT_EQ(qos_thread, (qos_class_t)qos, NULL); +} + +static void * +qos_send_and_sync_rcv(void *arg __unused) +{ + mach_port_t qos_send_port; + mach_port_t special_reply_port; + + T_LOG("Client: from created thread\n"); + + T_EXPECT_EFFECTIVE_QOS_EQ(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], + "pthread QoS should be %s", g_expected_qos_name[ENV_QOS_AFTER_OVERRIDE]); + + kern_return_t kr = bootstrap_look_up(bootstrap_port, + KEVENT_QOS_SERVICE_NAME, &qos_send_port); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client bootstrap_look_up"); + + special_reply_port = thread_get_special_reply_port(); + T_QUIET; T_ASSERT_NOTNULL(special_reply_port , "get_thread_special_reply_port"); + + /* enqueue two messages to make sure that mqueue is not empty */ + send(qos_send_port, MACH_PORT_NULL, MACH_PORT_NULL, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_QUEUE_OVERRIDE], 0, 0)); + + send(qos_send_port, MACH_PORT_NULL, MACH_PORT_NULL, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_QUEUE_OVERRIDE], 0, 0)); + + sleep(SEND_TIMEOUT_SECS); + + /* sync wait on msg port */ + receive(special_reply_port, qos_send_port); + + T_LOG("Client done doing sync rcv, now waiting for server to end the test"); + sleep(SEND_TIMEOUT_SECS); + + T_ASSERT_FAIL("client timed out"); + return 0; +} + +T_HELPER_DECL(qos_client_send_sync_and_sync_rcv, + "Send messages and syncronously wait for rcv") +{ + thread_create_at_qos(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], qos_send_and_sync_rcv); + sleep(HELPER_TIMEOUT_SECS); +} + +T_HELPER_DECL(qos_client_send_sync_msg, + "Send synchronous messages") +{ + mach_port_t qos_send_port; + mach_port_t special_reply_port; + + kern_return_t kr = bootstrap_look_up(bootstrap_port, + KEVENT_QOS_SERVICE_NAME, &qos_send_port); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client bootstrap_look_up"); + + special_reply_port = thread_get_special_reply_port(); + T_QUIET; T_ASSERT_NOTNULL(special_reply_port , "get_thread_special_reply_port"); + + /* Send the message to msg port */ + send(qos_send_port, special_reply_port, MACH_PORT_NULL, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], 0, 0)); + + T_LOG("Client done sending messages, now waiting for server to end the test"); + sleep(2 * SEND_TIMEOUT_SECS); + + T_ASSERT_FAIL("client timed out"); +} + +T_HELPER_DECL(qos_client_send_two_sync_msg, + "Send two synchronous messages at different qos") +{ + mach_port_t qos_send_port; + mach_port_t special_reply_port; + + kern_return_t kr = bootstrap_look_up(bootstrap_port, + KEVENT_QOS_SERVICE_NAME, &qos_send_port); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client bootstrap_look_up"); + + special_reply_port = thread_get_special_reply_port(); + T_QUIET; T_ASSERT_NOTNULL(special_reply_port , "get_thread_special_reply_port"); + + /* Send the message to msg port */ + send(qos_send_port, special_reply_port, MACH_PORT_NULL, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], 0, 0)); + + /* Send the message to msg port */ + send(qos_send_port, special_reply_port, MACH_PORT_NULL, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_BEFORE_OVERRIDE], 0, 0)); + + T_LOG("Client done sending messages, now waiting for server to end the test"); + sleep(SEND_TIMEOUT_SECS); + + T_ASSERT_FAIL("client timed out"); +} + +T_HELPER_DECL(qos_client_send_two_msg_and_destroy, + "Send two messages with 2nd one as sync and then destory the special reply port") +{ + mach_port_t qos_send_port; + mach_port_t special_reply_port; + + kern_return_t kr = bootstrap_look_up(bootstrap_port, + KEVENT_QOS_SERVICE_NAME, &qos_send_port); + T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client bootstrap_look_up"); + + special_reply_port = thread_get_special_reply_port(); + T_QUIET; T_ASSERT_NOTNULL(special_reply_port , "get_thread_special_reply_port"); + + /* Send an async message to msg port */ + send(qos_send_port, MACH_PORT_NULL, MACH_PORT_NULL, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], 0, 0)); + + /* Send the message to msg port */ + send(qos_send_port, special_reply_port, MACH_PORT_NULL, + (uint32_t)_pthread_qos_class_encode(g_expected_qos[ENV_QOS_AFTER_OVERRIDE], 0, 0)); + + T_LOG("Client done sending messages, waiting for destroy the special reply_port"); + sleep(SEND_TIMEOUT_SECS); + + mach_port_destroy(mach_task_self(), special_reply_port); + sleep(SEND_TIMEOUT_SECS); + + T_ASSERT_FAIL("client timed out"); +} + +static void +run_client_server(const char *server_name, const char *client_name, qos_class_t qos[], + const char *qos_name[], const char *wl_function) +{ + char *env[2 * ENV_VAR_QOS + ENV_VAR_FUNCTION + 1]; + env_set_qos(env, qos, qos_name, wl_function); + + for (int i = 0; i < ENV_VAR_QOS; i++) { + g_expected_qos[i] = qos[i]; + g_expected_qos_name[i] = qos_name[i]; + } + + dt_helper_t helpers[] = { + dt_launchd_helper_env("com.apple.xnu.test.kevent_qos.plist", + server_name, env), + dt_fork_helper(client_name) + }; + dt_run_helpers(helpers, 2, HELPER_TIMEOUT_SECS); +} + +#pragma mark Mach receive - kevent_qos + + +static void +expect_kevent_id_recv(mach_port_t port, qos_class_t qos[], const char *qos_name[], const char *wl_function) +{ + int r; + + /* Qos expected by workloop thread */ + for (int i = 0; i < ENV_VAR_QOS; i++) { + g_expected_qos[i] = qos[i]; + g_expected_qos_name[i] = qos_name[i]; + } + + if (strcmp(wl_function, "workloop_cb_test_intransit") == 0) { + T_QUIET; T_ASSERT_POSIX_ZERO(_pthread_workqueue_init_with_workloop( + worker_cb, event_cb, + (pthread_workqueue_function_workloop_t)workloop_cb_test_intransit, 0, 0), NULL); + } else if (strcmp(wl_function, "workloop_cb_test_sync_send") == 0) { + T_QUIET; T_ASSERT_POSIX_ZERO(_pthread_workqueue_init_with_workloop( + worker_cb, event_cb, + (pthread_workqueue_function_workloop_t)workloop_cb_test_sync_send, 0, 0), NULL); + } else if (strcmp(wl_function, "workloop_cb_test_sync_send_and_enable") == 0) { + T_QUIET; T_ASSERT_POSIX_ZERO(_pthread_workqueue_init_with_workloop( + worker_cb, event_cb, + (pthread_workqueue_function_workloop_t)workloop_cb_test_sync_send_and_enable, 0, 0), NULL); + } else if (strcmp(wl_function, "workloop_cb_test_send_two_sync") == 0) { + T_QUIET; T_ASSERT_POSIX_ZERO(_pthread_workqueue_init_with_workloop( + worker_cb, event_cb, + (pthread_workqueue_function_workloop_t)workloop_cb_test_send_two_sync, 0, 0), NULL); + } else if (strcmp(wl_function, "workloop_cb_test_two_send_and_destroy") == 0) { + T_QUIET; T_ASSERT_POSIX_ZERO(_pthread_workqueue_init_with_workloop( + worker_cb, event_cb, + (pthread_workqueue_function_workloop_t)workloop_cb_test_two_send_and_destroy, 0, 0), NULL); + } else { + T_ASSERT_FAIL("no workloop function specified \n"); + } + + struct kevent_qos_s kev[] = {{ + .ident = port, + .filter = EVFILT_MACHPORT, + .flags = EV_ADD | EV_UDATA_SPECIFIC | EV_DISPATCH | EV_VANISHED, + .fflags = (MACH_RCV_MSG | MACH_RCV_LARGE | MACH_RCV_LARGE_IDENTITY | + MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_CTX) | + MACH_RCV_TRAILER_TYPE(MACH_MSG_TRAILER_FORMAT_0) | + MACH_RCV_VOUCHER), + .data = 1, + .qos = (int32_t)_pthread_qos_class_encode(qos[ENV_QOS_QUEUE_OVERRIDE], 0, 0) + }}; + + struct kevent_qos_s kev_err[] = {{ 0 }}; + + /* Setup workloop for mach msg rcv */ + r = kevent_id(25, kev, 1, kev_err, 1, NULL, + NULL, KEVENT_FLAG_WORKLOOP | KEVENT_FLAG_ERROR_EVENTS); + + T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "kevent_id"); + T_QUIET; T_ASSERT_EQ(r, 0, "no errors returned from kevent_id"); + sleep(HELPER_TIMEOUT_SECS); +} + +T_HELPER_DECL(server_kevent_id, + "Reply with the QoS that a dispatch source event handler ran with") +{ + qos_class_t qos[ENV_VAR_QOS]; + const char *qos_name[ENV_VAR_QOS]; + const char *wl_function; + environ_get_qos(qos, qos_name, &wl_function); + + expect_kevent_id_recv(get_server_port(), qos, qos_name, wl_function); + sleep(HELPER_TIMEOUT_SECS); + T_ASSERT_FAIL("should receive a message within %d seconds", + RECV_TIMEOUT_SECS); +} + +#define TEST_QOS(server_name, client_name, name, wl_function_name, qos_bo, qos_bo_name, qos_qo, qos_qo_name, qos_ao, qos_ao_name) \ + T_DECL(server_kevent_id_##name, \ + "Event delivery at " qos_ao_name " QoS using a kevent_id", \ + T_META_ASROOT(YES)) \ + { \ + qos_class_t qos_array[ENV_VAR_QOS] = {qos_bo, qos_qo, qos_ao}; \ + const char *qos_name_array[ENV_VAR_QOS] = {qos_bo_name, qos_qo_name, qos_ao_name}; \ + run_client_server(server_name, client_name, qos_array, qos_name_array, wl_function_name); \ + } + +/* + * Test 1: Test special reply port SPI + * + * Create thread special reply port and check any subsequent calls to + * the same should return MACH_PORT_NULL, unless the reply port is destroyed. + */ +TEST_QOS("server_kevent_id", "qos_get_special_reply_port", special_reply_port, "workloop_cb_test_intransit", + QOS_CLASS_DEFAULT, "default", + QOS_CLASS_DEFAULT, "default", + QOS_CLASS_DEFAULT, "default") + +/* + * Test 2: Test sync ipc send to an in-transit port + * + * Send a sync ipc message (at IN qos) to an in-transit port enqueued in a port + * attached to a workloop. Test that the servicer of the workloop gets + * sync ipc override. + */ +TEST_QOS("server_kevent_id", "qos_client_send_to_intransit", transit_IN, "workloop_cb_test_intransit", + QOS_CLASS_DEFAULT, "default", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INITIATED, "user initiated") + +/* + * Test 3: Test sync ipc send to an in-transit port + * + * Send a sync ipc message (at UI qos) to an in-transit port enqueued in a port + * attached to a workloop. Test that the servicer of the workloop gets + * sync ipc override. + */ +TEST_QOS("server_kevent_id", "qos_client_send_to_intransit", transit_UI, "workloop_cb_test_intransit", + QOS_CLASS_USER_INITIATED, "user initiated", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INTERACTIVE, "user interactive") + +/* + * Test 4: Test enqueue of a receive right having sync ipc override + * + * Enqueue a receive right which has a sync ipc override (at IN qos) + * and test that servicer of the workloop on other side gets sync ipc + * override. + */ +TEST_QOS("server_kevent_id", "qos_client_send_sync_and_enqueue_rcv", enqueue_IN, "workloop_cb_test_intransit", + QOS_CLASS_DEFAULT, "default", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INITIATED, "user initiated") + +/* + * Test 5: Test enqueue of a receive right having sync ipc override + * + * Enqueue a receive right which has a sync ipc override (at UI qos) + * and test that servicer of the workloop on other side gets sync ipc + * override. + */ +TEST_QOS("server_kevent_id", "qos_client_send_sync_and_enqueue_rcv", enqueue_UI, "workloop_cb_test_intransit", + QOS_CLASS_DEFAULT, "default", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INTERACTIVE, "user interactive") + +/* + * Test 6: Test starting a sync rcv overrides the servicer + * + * Send an async message to a port and then start waiting on + * the port in mach msg rcv (at IN qos) with sync wait and test if the + * servicer of the workloop gets sync ipc override. + */ +TEST_QOS("server_kevent_id", "qos_client_send_sync_and_sync_rcv", rcv_IN, "workloop_cb_test_intransit", + QOS_CLASS_DEFAULT, "default", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INITIATED, "user initiated") + +/* + * Test 7: Test starting a sync rcv overrides the servicer + * + * Send an async message to a port and then start waiting on + * the port in mach msg rcv (at UI qos) with sync wait and test if the + * servicer of the workloop gets sync ipc override. + */ +TEST_QOS("server_kevent_id", "qos_client_send_sync_and_sync_rcv", rcv_UI, "workloop_cb_test_intransit", + QOS_CLASS_DEFAULT, "default", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INTERACTIVE, "user interactive") + +/* + * Test 8: test sending sync ipc message (at IN qos) to port will override the servicer + * + * Send a message with sync ipc override to a port and check if the servicer + * of the workloop on other side gets sync ipc override. + */ +TEST_QOS("server_kevent_id", "qos_client_send_sync_msg", send_sync_IN, "workloop_cb_test_sync_send", + QOS_CLASS_DEFAULT, "default", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INITIATED, "user initiated") + +/* + * Test 9: test sending sync ipc message (at UI qos) to port will override the servicer + * + * Send a message with sync ipc override to a port and check if the servicer + * of the workloop on other side gets sync ipc override. + */ +TEST_QOS("server_kevent_id", "qos_client_send_sync_msg", send_sync_UI, "workloop_cb_test_sync_send", + QOS_CLASS_USER_INITIATED, "user initiated", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INTERACTIVE, "user interactive") + +/* + * Test 10: test enabling a knote in workloop handler will drop the sync ipc override of delivered message + * + * Send a sync ipc message to port and check the servicer of the workloop + * on other side gets sync ipc override and once the handler enables the knote, + * that sync ipc override is dropped. + */ +TEST_QOS("server_kevent_id", "qos_client_send_sync_msg", send_sync_UI_and_enable, "workloop_cb_test_sync_send_and_enable", + QOS_CLASS_USER_INITIATED, "user initiated", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INTERACTIVE, "user interactive") + +/* + * Test 11: test returning to begin processing drops sync ipc override of delivered message + * + * Send a sync ipc message and check if enabling the knote clears the override of + * the delivered message, but should still have the override of an enqueued message. + */ +TEST_QOS("server_kevent_id", "qos_client_send_two_sync_msg", send_two_sync_UI, "workloop_cb_test_send_two_sync", + QOS_CLASS_USER_INITIATED, "user initiated", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INTERACTIVE, "user interactive") + +/* + * Test 12: test destorying the special reply port drops the override + * + * Send two async messages and a sync ipc message, the workloop handler + * should get a sync ipc override, now test if destroying the special + * reply port drops the sync ipc override on the servicer. + */ +TEST_QOS("server_kevent_id", "qos_client_send_two_msg_and_destroy", send_two_UI_and_destroy, "workloop_cb_test_two_send_and_destroy", + QOS_CLASS_USER_INITIATED, "user initiated", + QOS_CLASS_MAINTENANCE, "maintenance", + QOS_CLASS_USER_INTERACTIVE, "user interactive") diff --git a/tools/tests/darwintests/kpc.c b/tools/tests/darwintests/kpc.c new file mode 100644 index 000000000..52009508a --- /dev/null +++ b/tools/tests/darwintests/kpc.c @@ -0,0 +1,68 @@ +#include +#include +#include + +#include + +T_DECL(fixed_counters, + "test that fixed counters return monotonically increasing values", + T_META_ASROOT(YES)) +{ + T_SKIP("unimplemented"); +} + +T_DECL(fixed_thread_counters, + "test that fixed thread counters return monotonically increasing values", + T_META_ASROOT(YES)) +{ + int err; + uint32_t ctrs_cnt; + uint64_t *ctrs_a; + uint64_t *ctrs_b; + + T_SETUPBEGIN; + + ctrs_cnt = kpc_get_counter_count(KPC_CLASS_FIXED_MASK); + if (ctrs_cnt == 0) { + T_SKIP("no fixed counters available"); + } + T_LOG("device has %" PRIu32 " fixed counters", ctrs_cnt); + + T_QUIET; T_ASSERT_POSIX_SUCCESS(kpc_force_all_ctrs_set(1), NULL); + T_ASSERT_POSIX_SUCCESS(kpc_set_counting(KPC_CLASS_FIXED_MASK), + "kpc_set_counting"); + T_ASSERT_POSIX_SUCCESS(kpc_set_thread_counting(KPC_CLASS_FIXED_MASK), + "kpc_set_thread_counting"); + + T_SETUPEND; + + ctrs_a = malloc(ctrs_cnt * sizeof(uint64_t)); + T_QUIET; T_ASSERT_NOTNULL(ctrs_a, NULL); + + err = kpc_get_thread_counters(0, ctrs_cnt, ctrs_a); + T_ASSERT_POSIX_SUCCESS(err, "kpc_get_thread_counters"); + + for (uint32_t i = 0; i < ctrs_cnt; i++) { + T_LOG("checking counter %d with value %" PRIu64 " > 0", i, ctrs_a[i]); + T_QUIET; + T_EXPECT_GT(ctrs_a[i], UINT64_C(0), "counter %d is non-zero", i); + } + + ctrs_b = malloc(ctrs_cnt * sizeof(uint64_t)); + T_QUIET; T_ASSERT_NOTNULL(ctrs_b, NULL); + + err = kpc_get_thread_counters(0, ctrs_cnt, ctrs_b); + T_ASSERT_POSIX_SUCCESS(err, "kpc_get_thread_counters"); + + for (uint32_t i = 0; i < ctrs_cnt; i++) { + T_LOG("checking counter %d with value %" PRIu64 + " > previous value %" PRIu64, i, ctrs_b[i], ctrs_a[i]); + T_QUIET; + T_EXPECT_GT(ctrs_b[i], UINT64_C(0), "counter %d is non-zero", i); + T_QUIET; T_EXPECT_LT(ctrs_a[i], ctrs_b[i], + "counter %d is increasing", i); + } + + free(ctrs_a); + free(ctrs_b); +} diff --git a/tools/tests/darwintests/kperf.c b/tools/tests/darwintests/kperf.c index a37ed39e2..81e3e4db9 100644 --- a/tools/tests/darwintests/kperf.c +++ b/tools/tests/darwintests/kperf.c @@ -1,8 +1,14 @@ +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif /* defined(T_NAMESPACE) */ + #include +#include #include #include -#include -#include +#include +#include +#include #include #include #include @@ -11,10 +17,281 @@ #include "kperf_helpers.h" +T_GLOBAL_META( + T_META_NAMESPACE("xnu.kperf"), + T_META_CHECK_LEAKS(false)); + +#define MAX_CPUS 64 +#define MAX_THREADS 64 + +volatile static bool running_threads = true; + +static void * +spinning_thread(void *semp) +{ + T_QUIET; + T_ASSERT_NOTNULL(semp, "semaphore passed to thread should not be NULL"); + dispatch_semaphore_signal(*(dispatch_semaphore_t *)semp); + + while (running_threads); + return NULL; +} + #define PERF_STK_KHDR UINT32_C(0x25020014) #define PERF_STK_UHDR UINT32_C(0x25020018) +#define PERF_TMR_FIRE KDBG_EVENTID(DBG_PERF, 3, 0) +#define PERF_TMR_HNDLR KDBG_EVENTID(DBG_PERF, 3, 2) +#define PERF_TMR_PEND KDBG_EVENTID(DBG_PERF, 3, 3) +#define PERF_TMR_SKIP KDBG_EVENTID(DBG_PERF, 3, 4) + +#define SCHED_HANDOFF KDBG_EVENTID(DBG_MACH, DBG_MACH_SCHED, \ + MACH_STACK_HANDOFF) +#define SCHED_SWITCH KDBG_EVENTID(DBG_MACH, DBG_MACH_SCHED, MACH_SCHED) +#define SCHED_IDLE KDBG_EVENTID(DBG_MACH, DBG_MACH_SCHED, MACH_IDLE) + +#define MP_CPUS_CALL UINT32_C(0x1900004) + +#define DISPATCH_AFTER_EVENT UINT32_C(0xfefffffc) +#define TIMEOUT_SECS 10 + +#define TIMER_PERIOD_NS (1 * NSEC_PER_MSEC) + +static void +reset_ktrace(void) +{ + kperf_reset(); +} + +/* + * Ensure that kperf is correctly IPIing CPUs that are actively scheduling by + * bringing up threads and ensuring that threads on-core are sampled by each + * timer fire. + */ + +T_DECL(ipi_active_cpus, + "make sure that kperf IPIs all active CPUs", + T_META_ASROOT(true)) +{ + int ncpus = dt_ncpu(); + T_QUIET; + T_ASSERT_LT(ncpus, MAX_CPUS, + "only supports up to %d CPUs", MAX_CPUS); + T_LOG("found %d CPUs", ncpus); -/* KDEBUG TRIGGER */ + int nthreads = ncpus - 1; + T_QUIET; + T_ASSERT_LT(nthreads, MAX_THREADS, + "only supports up to %d threads", MAX_THREADS); + + static pthread_t threads[MAX_THREADS]; + + /* + * TODO options to write this to a file and reinterpret a file... + */ + + /* + * Create threads to bring up all of the CPUs. + */ + + dispatch_semaphore_t thread_spinning = dispatch_semaphore_create(0); + + for (int i = 0; i < nthreads; i++) { + T_QUIET; + T_ASSERT_POSIX_ZERO( + pthread_create(&threads[i], NULL, &spinning_thread, + &thread_spinning), NULL); + dispatch_semaphore_wait(thread_spinning, DISPATCH_TIME_FOREVER); + } + + T_LOG("spun up %d thread%s", nthreads, nthreads == 1 ? "" : "s"); + + ktrace_session_t s = ktrace_session_create(); + T_WITH_ERRNO; T_ASSERT_NOTNULL(s, "ktrace_session_create"); + + dispatch_queue_t q = dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0); + + /* + * Only set the timeout after we've seen an event that was traced by us. + * This helps set a reasonable timeout after we're guaranteed to get a + * few events. + */ + + ktrace_events_single(s, DISPATCH_AFTER_EVENT, + ^(__unused struct trace_point *tp) + { + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, + TIMEOUT_SECS * NSEC_PER_SEC), q, ^{ + ktrace_end(s, 0); + }); + }); + + __block uint64_t nfires = 0; + __block uint64_t nsamples = 0; + static uint64_t idle_tids[MAX_CPUS] = { 0 }; + __block int nidles = 0; + + ktrace_set_completion_handler(s, ^{ + T_LOG("stopping threads"); + + running_threads = false; + + for (int i = 0; i < nthreads; i++) { + T_QUIET; + T_ASSERT_POSIX_ZERO(pthread_join(threads[i], NULL), NULL); + } + + for (int i = 0; i < nidles; i++) { + T_LOG("CPU %d idle thread: %#" PRIx64, i, idle_tids[i]); + } + + T_LOG("saw %" PRIu64 " timer fires, %" PRIu64 " samples, " + "%g samples/fire", nfires, nsamples, + (double)nsamples / (double)nfires); + + T_END; + }); + + /* + * Track which threads are running on each CPU. + */ + + static uint64_t tids_on_cpu[MAX_CPUS] = { 0 }; + + void (^switch_cb)(struct trace_point *) = ^(struct trace_point *tp) { + uint64_t new_thread = tp->arg2; + // uint64_t old_thread = tp->threadid; + + for (int i = 0; i < nidles; i++) { + if (idle_tids[i] == new_thread) { + return; + } + } + + tids_on_cpu[tp->cpuid] = new_thread; + }; + + ktrace_events_single(s, SCHED_SWITCH, switch_cb); + ktrace_events_single(s, SCHED_HANDOFF, switch_cb); + + /* + * Determine the thread IDs of the idle threads on each CPU. + */ + + ktrace_events_single(s, SCHED_IDLE, ^(struct trace_point *tp) { + uint64_t idle_thread = tp->threadid; + + tids_on_cpu[tp->cpuid] = 0; + + for (int i = 0; i < nidles; i++) { + if (idle_tids[i] == idle_thread) { + return; + } + } + + idle_tids[nidles++] = idle_thread; + }); + + /* + * On each timer fire, go through all the cores and mark any threads + * that should be sampled. + */ + + __block int last_fire_cpu = -1; + __block uint64_t sample_missing = 0; + static uint64_t tids_snap[MAX_CPUS] = { 0 }; + __block int nexpected = 0; +#if defined(__x86_64__) + __block int xcall_from_cpu = -1; +#endif /* defined(__x86_64__) */ + __block uint64_t xcall_mask = 0; + + ktrace_events_single(s, PERF_TMR_FIRE, ^(struct trace_point *tp) { + int last_expected = nexpected; + nfires++; + + nexpected = 0; + for (int i = 0; i < ncpus; i++) { + uint64_t i_bit = UINT64_C(1) << i; + if (sample_missing & i_bit) { + T_LOG("missed sample on CPU %d for thread %#llx from timer on CPU %d (xcall mask = %llx, expected %d samples)", + tp->cpuid, tids_snap[i], last_fire_cpu, + xcall_mask, last_expected); + sample_missing &= ~i_bit; + } + + if (tids_on_cpu[i] != 0) { + tids_snap[i] = tids_on_cpu[i]; + sample_missing |= i_bit; + nexpected++; + } + } + + T_QUIET; + T_ASSERT_LT((int)tp->cpuid, ncpus, "timer fire should not occur on an IOP"); + last_fire_cpu = (int)tp->cpuid; +#if defined(__x86_64__) + xcall_from_cpu = (int)tp->cpuid; +#endif /* defined(__x86_64__) */ + }); + +#if defined(__x86_64__) + /* + * Watch for the cross-call on Intel, make sure they match what kperf + * should be doing. + */ + + ktrace_events_single(s, MP_CPUS_CALL, ^(struct trace_point *tp) { + if (xcall_from_cpu != (int)tp->cpuid) { + return; + } + + xcall_mask = tp->arg1; + xcall_from_cpu = -1; + }); +#endif /* defined(__x86_64__) */ + + /* + * On the timer handler for each CPU, unset the missing sample bitmap. + */ + + ktrace_events_single(s, PERF_TMR_HNDLR, ^(struct trace_point *tp) { + nsamples++; + if ((int)tp->cpuid > ncpus) { + /* skip IOPs; they're not scheduling our threads */ + return; + } + + sample_missing &= ~(UINT64_C(1) << tp->cpuid); + }); + + /* + * Configure kperf and ktrace. + */ + + (void)kperf_action_count_set(1); + T_QUIET; + T_ASSERT_POSIX_SUCCESS(kperf_action_samplers_set(1, KPERF_SAMPLER_KSTACK), + NULL); + (void)kperf_timer_count_set(1); + T_QUIET; + T_ASSERT_POSIX_SUCCESS(kperf_timer_period_set(0, + kperf_ns_to_ticks(TIMER_PERIOD_NS)), NULL); + T_QUIET; + T_ASSERT_POSIX_SUCCESS(kperf_timer_action_set(0, 1), NULL); + + T_ASSERT_POSIX_SUCCESS(kperf_sample_set(1), "start kperf sampling"); + T_ATEND(reset_ktrace); + + T_ASSERT_POSIX_ZERO(ktrace_start(s, + dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0)), + "start ktrace"); + + kdebug_trace(DISPATCH_AFTER_EVENT, 0, 0, 0, 0); + + dispatch_main(); +} + +#pragma mark kdebug triggers #define KDEBUG_TRIGGER_TIMEOUT_NS (10 * NSEC_PER_SEC) @@ -23,87 +300,88 @@ #define NON_TRIGGER_CODE UINT8_C(0xff) #define NON_TRIGGER_EVENT \ - (KDBG_EVENTID(NON_TRIGGER_CLASS, NON_TRIGGER_SUBCLASS, NON_TRIGGER_CODE)) + (KDBG_EVENTID(NON_TRIGGER_CLASS, NON_TRIGGER_SUBCLASS, \ + NON_TRIGGER_CODE)) static void expect_kdebug_trigger(const char *filter_desc, const uint32_t *debugids, - unsigned int n_debugids) + unsigned int n_debugids) { - __block int missing_kernel_stacks = 0; - __block int missing_user_stacks = 0; - ktrace_session_t s; - kperf_kdebug_filter_t filter; - - s = ktrace_session_create(); - T_QUIET; T_ASSERT_NOTNULL(s, NULL); - - ktrace_events_single(s, PERF_STK_KHDR, ^(struct trace_point *tp) { - missing_kernel_stacks--; - T_LOG("saw kernel stack with %lu frames, flags = %#lx", tp->arg2, - tp->arg1); - }); - ktrace_events_single(s, PERF_STK_UHDR, ^(struct trace_point *tp) { - missing_user_stacks--; - T_LOG("saw user stack with %lu frames, flags = %#lx", tp->arg2, - tp->arg1); - }); - - for (unsigned int i = 0; i < n_debugids; i++) { - ktrace_events_single(s, debugids[i], ^(struct trace_point *tp) { - missing_kernel_stacks++; - missing_user_stacks++; - T_LOG("saw event with debugid 0x%" PRIx32, tp->debugid); - }); - } - - ktrace_events_single(s, NON_TRIGGER_EVENT, - ^(__unused struct trace_point *tp) - { - ktrace_end(s, 0); - }); - - ktrace_set_completion_handler(s, ^{ - T_EXPECT_LE(missing_kernel_stacks, 0, NULL); - T_EXPECT_LE(missing_user_stacks, 0, NULL); - - ktrace_session_destroy(s); - T_END; - }); - - /* configure kperf */ - - kperf_reset(); - - (void)kperf_action_count_set(1); - T_ASSERT_POSIX_SUCCESS(kperf_action_samplers_set(1, - KPERF_SAMPLER_KSTACK | KPERF_SAMPLER_USTACK), NULL); - - filter = kperf_kdebug_filter_create(); - T_ASSERT_NOTNULL(filter, NULL); - - T_ASSERT_POSIX_SUCCESS(kperf_kdebug_action_set(1), NULL); - T_ASSERT_POSIX_SUCCESS(kperf_kdebug_filter_add_desc(filter, filter_desc), - NULL); - T_ASSERT_POSIX_SUCCESS(kperf_kdebug_filter_set(filter), NULL); - kperf_kdebug_filter_destroy(filter); - - T_ASSERT_POSIX_SUCCESS(kperf_sample_set(1), NULL); - - T_ASSERT_POSIX_ZERO(ktrace_start(s, dispatch_get_main_queue()), NULL); - - /* trace the triggering debugids */ - - for (unsigned int i = 0; i < n_debugids; i++) { - T_ASSERT_POSIX_SUCCESS(kdebug_trace(debugids[i], 0, 0, 0, 0), NULL); - } - - T_ASSERT_POSIX_SUCCESS(kdebug_trace(NON_TRIGGER_EVENT, 0, 0, 0, 0), NULL); - - dispatch_after(dispatch_time(DISPATCH_TIME_NOW, KDEBUG_TRIGGER_TIMEOUT_NS), - dispatch_get_main_queue(), ^(void) - { - ktrace_end(s, 1); - }); + __block int missing_kernel_stacks = 0; + __block int missing_user_stacks = 0; + ktrace_session_t s; + kperf_kdebug_filter_t filter; + + s = ktrace_session_create(); + T_QUIET; T_ASSERT_NOTNULL(s, NULL); + + ktrace_events_single(s, PERF_STK_KHDR, ^(struct trace_point *tp) { + missing_kernel_stacks--; + T_LOG("saw kernel stack with %lu frames, flags = %#lx", tp->arg2, + tp->arg1); + }); + ktrace_events_single(s, PERF_STK_UHDR, ^(struct trace_point *tp) { + missing_user_stacks--; + T_LOG("saw user stack with %lu frames, flags = %#lx", tp->arg2, + tp->arg1); + }); + + for (unsigned int i = 0; i < n_debugids; i++) { + ktrace_events_single(s, debugids[i], ^(struct trace_point *tp) { + missing_kernel_stacks++; + missing_user_stacks++; + T_LOG("saw event with debugid 0x%" PRIx32, tp->debugid); + }); + } + + ktrace_events_single(s, NON_TRIGGER_EVENT, + ^(__unused struct trace_point *tp) + { + ktrace_end(s, 0); + }); + + ktrace_set_completion_handler(s, ^{ + T_EXPECT_LE(missing_kernel_stacks, 0, NULL); + T_EXPECT_LE(missing_user_stacks, 0, NULL); + + ktrace_session_destroy(s); + T_END; + }); + + /* configure kperf */ + + kperf_reset(); + + (void)kperf_action_count_set(1); + T_ASSERT_POSIX_SUCCESS(kperf_action_samplers_set(1, + KPERF_SAMPLER_KSTACK | KPERF_SAMPLER_USTACK), NULL); + + filter = kperf_kdebug_filter_create(); + T_ASSERT_NOTNULL(filter, NULL); + + T_ASSERT_POSIX_SUCCESS(kperf_kdebug_action_set(1), NULL); + T_ASSERT_POSIX_SUCCESS(kperf_kdebug_filter_add_desc(filter, filter_desc), + NULL); + T_ASSERT_POSIX_SUCCESS(kperf_kdebug_filter_set(filter), NULL); + kperf_kdebug_filter_destroy(filter); + + T_ASSERT_POSIX_SUCCESS(kperf_sample_set(1), NULL); + + T_ASSERT_POSIX_ZERO(ktrace_start(s, dispatch_get_main_queue()), NULL); + + /* trace the triggering debugids */ + + for (unsigned int i = 0; i < n_debugids; i++) { + T_ASSERT_POSIX_SUCCESS(kdebug_trace(debugids[i], 0, 0, 0, 0), NULL); + } + + T_ASSERT_POSIX_SUCCESS(kdebug_trace(NON_TRIGGER_EVENT, 0, 0, 0, 0), NULL); + + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, KDEBUG_TRIGGER_TIMEOUT_NS), + dispatch_get_main_queue(), ^(void) + { + ktrace_end(s, 1); + }); } #define TRIGGER_CLASS UINT8_C(0xfe) @@ -111,49 +389,51 @@ expect_kdebug_trigger(const char *filter_desc, const uint32_t *debugids, #define TRIGGER_SUBCLASS UINT8_C(0xff) #define TRIGGER_CODE UINT8_C(0) #define TRIGGER_DEBUGID \ - (KDBG_EVENTID(TRIGGER_CLASS, TRIGGER_SUBCLASS, TRIGGER_CODE)) + (KDBG_EVENTID(TRIGGER_CLASS, TRIGGER_SUBCLASS, TRIGGER_CODE)) -T_DECL(kdebug_trigger_classes, "test that kdebug trigger samples on classes", - T_META_ASROOT(true)) +T_DECL(kdebug_trigger_classes, + "test that kdebug trigger samples on classes", + T_META_ASROOT(true)) { - const uint32_t class_debugids[] = { - KDBG_EVENTID(TRIGGER_CLASS, 1, 1), - KDBG_EVENTID(TRIGGER_CLASS, 2, 1), - KDBG_EVENTID(TRIGGER_CLASS_END, 1, 1) | DBG_FUNC_END, - KDBG_EVENTID(TRIGGER_CLASS_END, 2, 1) | DBG_FUNC_END, - }; - - expect_kdebug_trigger("C0xfe,C0xfdr", class_debugids, - sizeof(class_debugids) / sizeof(class_debugids[0])); - dispatch_main(); + const uint32_t class_debugids[] = { + KDBG_EVENTID(TRIGGER_CLASS, 1, 1), + KDBG_EVENTID(TRIGGER_CLASS, 2, 1), + KDBG_EVENTID(TRIGGER_CLASS_END, 1, 1) | DBG_FUNC_END, + KDBG_EVENTID(TRIGGER_CLASS_END, 2, 1) | DBG_FUNC_END, + }; + + expect_kdebug_trigger("C0xfe,C0xfdr", class_debugids, + sizeof(class_debugids) / sizeof(class_debugids[0])); + dispatch_main(); } T_DECL(kdebug_trigger_subclasses, - "test that kdebug trigger samples on subclasses", - T_META_ASROOT(true)) + "test that kdebug trigger samples on subclasses", + T_META_ASROOT(true)) { - const uint32_t subclass_debugids[] = { - KDBG_EVENTID(TRIGGER_CLASS, TRIGGER_SUBCLASS, 0), - KDBG_EVENTID(TRIGGER_CLASS, TRIGGER_SUBCLASS, 1), - KDBG_EVENTID(TRIGGER_CLASS_END, TRIGGER_SUBCLASS, 0) | DBG_FUNC_END, - KDBG_EVENTID(TRIGGER_CLASS_END, TRIGGER_SUBCLASS, 1) | DBG_FUNC_END - }; - - expect_kdebug_trigger("S0xfeff,S0xfdffr", subclass_debugids, - sizeof(subclass_debugids) / sizeof(subclass_debugids[0])); - dispatch_main(); + const uint32_t subclass_debugids[] = { + KDBG_EVENTID(TRIGGER_CLASS, TRIGGER_SUBCLASS, 0), + KDBG_EVENTID(TRIGGER_CLASS, TRIGGER_SUBCLASS, 1), + KDBG_EVENTID(TRIGGER_CLASS_END, TRIGGER_SUBCLASS, 0) | DBG_FUNC_END, + KDBG_EVENTID(TRIGGER_CLASS_END, TRIGGER_SUBCLASS, 1) | DBG_FUNC_END + }; + + expect_kdebug_trigger("S0xfeff,S0xfdffr", subclass_debugids, + sizeof(subclass_debugids) / sizeof(subclass_debugids[0])); + dispatch_main(); } -T_DECL(kdebug_trigger_debugids, "test that kdebug trigger samples on debugids", - T_META_ASROOT(true)) +T_DECL(kdebug_trigger_debugids, + "test that kdebug trigger samples on debugids", + T_META_ASROOT(true)) { - const uint32_t debugids[] = { - TRIGGER_DEBUGID - }; + const uint32_t debugids[] = { + TRIGGER_DEBUGID + }; - expect_kdebug_trigger("D0xfeff0000", debugids, - sizeof(debugids) / sizeof(debugids[0])); - dispatch_main(); + expect_kdebug_trigger("D0xfeff0000", debugids, + sizeof(debugids) / sizeof(debugids[0])); + dispatch_main(); } /* @@ -161,159 +441,118 @@ T_DECL(kdebug_trigger_debugids, "test that kdebug trigger samples on debugids", * events from that class. */ -T_DECL(kdbg_callstacks, "test that the kdbg_callstacks samples on syscalls", - T_META_ASROOT(true)) +T_DECL(kdbg_callstacks, + "test that the kdbg_callstacks samples on syscalls", + T_META_ASROOT(true)) { - ktrace_session_t s; - __block bool saw_user_stack = false; - - s = ktrace_session_create(); - T_ASSERT_NOTNULL(s, NULL); - - /* - * Make sure BSD events are traced in order to trigger samples on syscalls. - */ - ktrace_events_class(s, DBG_BSD, - ^void(__unused struct trace_point *tp) {}); - - ktrace_events_single(s, PERF_STK_UHDR, ^(__unused struct trace_point *tp) { - saw_user_stack = true; - ktrace_end(s, 1); - }); + ktrace_session_t s; + __block bool saw_user_stack = false; - ktrace_set_completion_handler(s, ^{ - ktrace_session_destroy(s); + s = ktrace_session_create(); + T_ASSERT_NOTNULL(s, NULL); - T_EXPECT_TRUE(saw_user_stack, - "saw user stack after configuring kdbg_callstacks"); + /* + * Make sure BSD events are traced in order to trigger samples on syscalls. + */ + ktrace_events_class(s, DBG_BSD, ^void(__unused struct trace_point *tp) {}); - /* - * Ensure user stacks are not sampled after resetting kdbg_callstacks. - */ - ktrace_session_t s_after = ktrace_session_create(); - T_ASSERT_NOTNULL(s_after, NULL); + ktrace_events_single(s, PERF_STK_UHDR, ^(__unused struct trace_point *tp) { + saw_user_stack = true; + ktrace_end(s, 1); + }); -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wdeprecated-declarations" - T_ASSERT_POSIX_SUCCESS(kperf_kdbg_callstacks_set(0), NULL); -#pragma clang diagnostic pop - - ktrace_events_class(s_after, DBG_BSD, - ^void(__unused struct trace_point *tp) {}); - - __block bool saw_extra_stack = false; + ktrace_set_completion_handler(s, ^{ + ktrace_session_destroy(s); - ktrace_events_single(s_after, PERF_STK_UHDR, - ^(__unused struct trace_point *tp) - { - saw_extra_stack = true; - ktrace_end(s_after, 1); - }); - - ktrace_set_completion_handler(s_after, ^(void) { - ktrace_session_destroy(s_after); - T_EXPECT_FALSE(saw_extra_stack, - "saw user stack after disabling kdbg_callstacks)"); - kperf_reset(); - T_END; - }); - - T_ASSERT_POSIX_ZERO(ktrace_start(s_after, dispatch_get_main_queue()), - NULL); - - dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 1 * NSEC_PER_SEC), - dispatch_get_main_queue(), ^(void) - { - ktrace_end(s_after, 1); - }); - }); + T_EXPECT_TRUE(saw_user_stack, + "saw user stack after configuring kdbg_callstacks"); + T_END; + }); #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" - T_ASSERT_POSIX_SUCCESS(kperf_kdbg_callstacks_set(1), NULL); + T_ASSERT_POSIX_SUCCESS(kperf_kdbg_callstacks_set(1), NULL); #pragma clang diagnostic pop + T_ATEND(kperf_reset); - T_ASSERT_POSIX_ZERO(ktrace_start(s, dispatch_get_main_queue()), NULL); + T_ASSERT_POSIX_ZERO(ktrace_start(s, dispatch_get_main_queue()), NULL); - dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 10 * NSEC_PER_SEC), - dispatch_get_main_queue(), ^(void) - { - ktrace_end(s, 1); - }); + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 10 * NSEC_PER_SEC), + dispatch_get_main_queue(), ^(void) { + ktrace_end(s, 1); + }); - dispatch_main(); + dispatch_main(); } -/* - * PET mode - */ +#pragma mark PET #define STACKS_WAIT_DURATION_NS (3 * NSEC_PER_SEC) static void expect_stacks_traced(void (^cb)(void)) { - ktrace_session_t s; - - s = ktrace_session_create(); - T_QUIET; T_ASSERT_NOTNULL(s, "ktrace_session_create"); - - __block unsigned int user_stacks = 0; - __block unsigned int kernel_stacks = 0; - - ktrace_events_single(s, PERF_STK_UHDR, ^(__unused struct trace_point *tp) { - user_stacks++; - }); - ktrace_events_single(s, PERF_STK_KHDR, ^(__unused struct trace_point *tp) { - kernel_stacks++; - }); - - ktrace_set_completion_handler(s, ^(void) { - ktrace_session_destroy(s); - T_EXPECT_GT(user_stacks, 0U, NULL); - T_EXPECT_GT(kernel_stacks, 0U, NULL); - cb(); - }); - - T_QUIET; T_ASSERT_POSIX_SUCCESS(kperf_sample_set(1), NULL); - - T_ASSERT_POSIX_ZERO(ktrace_start(s, dispatch_get_main_queue()), NULL); - - dispatch_after(dispatch_time(DISPATCH_TIME_NOW, STACKS_WAIT_DURATION_NS), - dispatch_get_main_queue(), ^(void) - { - kperf_reset(); - ktrace_end(s, 0); - }); + ktrace_session_t s; + + s = ktrace_session_create(); + T_QUIET; T_ASSERT_NOTNULL(s, "ktrace_session_create"); + + __block unsigned int user_stacks = 0; + __block unsigned int kernel_stacks = 0; + + ktrace_events_single(s, PERF_STK_UHDR, ^(__unused struct trace_point *tp) { + user_stacks++; + }); + ktrace_events_single(s, PERF_STK_KHDR, ^(__unused struct trace_point *tp) { + kernel_stacks++; + }); + + ktrace_set_completion_handler(s, ^(void) { + ktrace_session_destroy(s); + T_EXPECT_GT(user_stacks, 0U, NULL); + T_EXPECT_GT(kernel_stacks, 0U, NULL); + cb(); + }); + + T_QUIET; T_ASSERT_POSIX_SUCCESS(kperf_sample_set(1), NULL); + + T_ASSERT_POSIX_ZERO(ktrace_start(s, dispatch_get_main_queue()), NULL); + + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, STACKS_WAIT_DURATION_NS), + dispatch_get_main_queue(), ^(void) + { + kperf_reset(); + ktrace_end(s, 0); + }); } T_DECL(pet, "test that PET mode samples kernel and user stacks", - T_META_ASROOT(true)) + T_META_ASROOT(true)) { - configure_kperf_stacks_timer(-1, 10); - T_ASSERT_POSIX_SUCCESS(kperf_timer_pet_set(0), NULL); + configure_kperf_stacks_timer(-1, 10); + T_ASSERT_POSIX_SUCCESS(kperf_timer_pet_set(0), NULL); - expect_stacks_traced(^(void) { - T_END; - }); + expect_stacks_traced(^(void) { + T_END; + }); - dispatch_main(); + dispatch_main(); } T_DECL(lightweight_pet, - "test that lightweight PET mode samples kernel and user stacks", - T_META_ASROOT(true)) + "test that lightweight PET mode samples kernel and user stacks", + T_META_ASROOT(true)) { - int set = 1; + int set = 1; - configure_kperf_stacks_timer(-1, 10); - T_ASSERT_POSIX_SUCCESS(sysctlbyname("kperf.lightweight_pet", NULL, NULL, - &set, sizeof(set)), NULL); - T_ASSERT_POSIX_SUCCESS(kperf_timer_pet_set(0), NULL); + configure_kperf_stacks_timer(-1, 10); + T_ASSERT_POSIX_SUCCESS(sysctlbyname("kperf.lightweight_pet", NULL, NULL, + &set, sizeof(set)), NULL); + T_ASSERT_POSIX_SUCCESS(kperf_timer_pet_set(0), NULL); - expect_stacks_traced(^(void) { - T_END; - }); + expect_stacks_traced(^(void) { + T_END; + }); - dispatch_main(); + dispatch_main(); } diff --git a/tools/tests/darwintests/kperf_backtracing.c b/tools/tests/darwintests/kperf_backtracing.c index f48931af2..1d3d46d08 100644 --- a/tools/tests/darwintests/kperf_backtracing.c +++ b/tools/tests/darwintests/kperf_backtracing.c @@ -1,8 +1,13 @@ +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif + #include #include #include #include -#include +#include +#include #include #include "kperf_helpers.h" @@ -12,6 +17,10 @@ #define PERF_STK_KDATA UINT32_C(0x2502000c) #define PERF_STK_UDATA UINT32_C(0x25020010) +T_GLOBAL_META( + T_META_NAMESPACE("xnu.kperf"), + T_META_CHECK_LEAKS(false)); + static void expect_frame(const char **bt, unsigned int bt_len, CSSymbolRef symbol, unsigned long addr, unsigned int bt_idx, unsigned int max_frames) @@ -218,7 +227,21 @@ static const char *user_bt[USER_FRAMES] = { NULL }; -#if defined(__x86_64__) +#if defined(__arm__) + +#define KERNEL_FRAMES (2) +static const char *kernel_bt[KERNEL_FRAMES] = { + "unix_syscall", "kdebug_trace64" +}; + +#elif defined(__arm64__) + +#define KERNEL_FRAMES (4) +static const char *kernel_bt[KERNEL_FRAMES] = { + "fleh_synchronous", "sleh_synchronous", "unix_syscall", "kdebug_trace64" +}; + +#elif defined(__x86_64__) #define KERNEL_FRAMES (2) static const char *kernel_bt[KERNEL_FRAMES] = { @@ -310,9 +333,13 @@ start_backtrace_thread(void) dispatch_semaphore_signal(backtrace_go); } +#if TARGET_OS_WATCH +#define TEST_TIMEOUT_NS (30 * NSEC_PER_SEC) +#else /* TARGET_OS_WATCH */ #define TEST_TIMEOUT_NS (5 * NSEC_PER_SEC) +#endif /* !TARGET_OS_WATCH */ -T_DECL(kdebug_trigger_backtraces, +T_DECL(backtraces_kdebug_trigger, "test that backtraces from kdebug trigger are correct", T_META_ASROOT(true)) { @@ -372,7 +399,7 @@ T_DECL(kdebug_trigger_backtraces, dispatch_main(); } -T_DECL(user_backtraces_timer, +T_DECL(backtraces_user_timer, "test that user backtraces on a timer are correct", T_META_ASROOT(true)) { diff --git a/tools/tests/darwintests/kqueue_add_and_trigger.c b/tools/tests/darwintests/kqueue_add_and_trigger.c new file mode 100644 index 000000000..15243a789 --- /dev/null +++ b/tools/tests/darwintests/kqueue_add_and_trigger.c @@ -0,0 +1,37 @@ +#include +#include +#include +#include + +/* EVFILT_USER doesn't properly support add&fire atomic combination + * + * Chek that using EV_ADD and EV_TRIGGER on a EV_USER actually trigger the event just added. + * + */ + +T_DECL(kqueue_add_and_trigger_evfilt_user, "Add and trigger EVFILT_USER events with kevent ") +{ + int kq_fd, ret; + struct kevent ret_kev; + const struct kevent kev = { + .ident = 1, + .filter = EVFILT_USER, + .flags = EV_ADD|EV_CLEAR, + .fflags = NOTE_TRIGGER, + }; + const struct timespec timeout = { + .tv_sec = 1, + .tv_nsec = 0, + }; + + T_ASSERT_POSIX_SUCCESS((kq_fd = kqueue()), NULL); + ret = kevent(kq_fd, &kev, 1, &ret_kev, 1, &timeout); + + T_ASSERT_POSIX_SUCCESS(ret, "kevent"); + + T_ASSERT_EQ(ret, 1, "kevent with add and trigger, ret"); + T_ASSERT_EQ(ret_kev.ident, 1, "kevent with add and trigger, ident"); + T_ASSERT_EQ(ret_kev.filter, EVFILT_USER, "kevent with add and trigger, filter"); + +} + diff --git a/tools/tests/darwintests/kqueue_close.c b/tools/tests/darwintests/kqueue_close.c new file mode 100644 index 000000000..3682d91e3 --- /dev/null +++ b/tools/tests/darwintests/kqueue_close.c @@ -0,0 +1,77 @@ +#include +#include +#include + +#include + +#include + +/* + * close() of kqueue FD races with kqueue_scan park + * + * When close concurrent with poll goes wrong, the close hangs + * and the kevent never gets any more events. + */ + +/* Both events should fire at about the same time */ +static uint32_t timeout_ms = 10; + +static void * +poll_kqueue(void *arg) +{ + int fd = (int)arg; + + struct kevent kev = { + .filter = EVFILT_TIMER, + .flags = EV_ADD, + .data = timeout_ms, + }; + + int rv = kevent(fd, &kev, 1, NULL, 0, NULL); + + if (rv == -1 && errno == EBADF) { + /* The close may race with this thread spawning */ + T_LOG("kqueue already closed?"); + return NULL; + } else { + T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "kevent"); + } + + while ((rv = kevent(fd, NULL, 0, &kev, 1, NULL)) == 1) { + T_LOG("poll\n"); + } + + if (rv != -1 || errno != EBADF) { + T_ASSERT_POSIX_SUCCESS(rv, "fd should be closed"); + } + + return NULL; +} + +static void +run_test() +{ + int fd = kqueue(); + T_QUIET; T_ASSERT_POSIX_SUCCESS(fd, "kqueue"); + + pthread_t thread; + int rv = pthread_create(&thread, NULL, poll_kqueue, + (void *)(uintptr_t)fd); + T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_create"); + + usleep(timeout_ms * 1000); + + rv = close(fd); + T_ASSERT_POSIX_SUCCESS(rv, "close"); + + rv = pthread_join(thread, NULL); + T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_join"); +} + +T_DECL(kqueue_close_race, "Races kqueue close with kqueue process", + T_META_LTEPHASE(LTE_POSTINIT), T_META_TIMEOUT(5)) +{ + for (uint32_t i = 1 ; i < 100 ; i++) { + run_test(); + } +} diff --git a/tools/tests/darwintests/kqueue_fifo_18776047.c b/tools/tests/darwintests/kqueue_fifo_18776047.c index a2e30174a..fe45758fd 100644 --- a/tools/tests/darwintests/kqueue_fifo_18776047.c +++ b/tools/tests/darwintests/kqueue_fifo_18776047.c @@ -11,11 +11,13 @@ #include #include +#include + #define TMP_FILE_PATH "/tmp/test_kqueue_fifo_18776047" #define READ_BUFFER_LEN 256 -#if defined(PLATFORM_WatchOS) +#if TARGET_OS_WATCH #define TOTAL_ITERATIONS 5000 #else #define TOTAL_ITERATIONS 10000 diff --git a/tools/tests/kqueue_tests/kqueue_file_tests.c b/tools/tests/darwintests/kqueue_file_tests.c similarity index 84% rename from tools/tests/kqueue_tests/kqueue_file_tests.c rename to tools/tests/darwintests/kqueue_file_tests.c index 9602fc861..dcd2c4793 100644 --- a/tools/tests/kqueue_tests/kqueue_file_tests.c +++ b/tools/tests/darwintests/kqueue_file_tests.c @@ -13,14 +13,24 @@ #include #include #include +#include +#include #include #include -#define DIR1 "/tmp/dir1" -#define DOTDOT ".." -#define DIR2 "/tmp/dir2" -#define FILE1 "/tmp/file1" -#define FILE2 "/tmp/file2" +#include +#include + +T_GLOBAL_META( + T_META_NAMESPACE("xnu.kevent") + ); + +#define PDIR "/tmp" +#define DIR1 PDIR "/dir1" +#define DOTDOT ".." +#define DIR2 PDIR "/dir2" +#define FILE1 PDIR "/file1" +#define FILE2 PDIR "/file2" #define KEY "somekey" #define VAL "someval" @@ -36,7 +46,7 @@ #define TEST_STRING "Some text!!! Yes indeed, some of that very structure which has passed on man's knowledge for generations." #define HELLO_WORLD "Hello, World!" -#define SLEEP_TIME 2 +#define USLEEP_TIME 5000 #define WAIT_TIME (4l) #define LENGTHEN_SIZE 500 #define FIFO_SPACE 8192 /* FIFOS have 8K of buffer space */ @@ -72,6 +82,12 @@ typedef struct _action { typedef struct _test { char *t_testname; + /* Is this test an expected failure? */ + int t_known_failure; + + /* Is this test behaving non-deterministically? */ + int t_nondeterministic; + /* Test kevent() or poll() */ int t_is_poll_test; @@ -108,25 +124,6 @@ typedef struct _test { int t_extra_sleep_hack; /* Sleep before waiting, to let a fifo fill up with data */ } test_t; -/* - * Extra logging infrastructure so we can filter some out - */ -void LOG(int level, FILE *f, const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - if (level >= OUTPUT_LEVEL) { - /* Indent for ease of reading */ - if (level < RESULT_LEVEL) { - for (int i = RESULT_LEVEL - level; i>0; i--) { - fprintf(f, "\t"); - } - } - vfprintf(f, fmt, ap); - } - - va_end(ap); -} - char * get_action_name(action_id_t a) { @@ -216,7 +213,9 @@ init_action(action_t *act, int sleep, action_id_t call, int nargs, ...) void * open_fifo_readside(void *arg) { - fifo_read_fd = open((char*)arg, O_RDONLY); + if ((fifo_read_fd = open((char*)arg, O_RDONLY)) == -1) { + T_LOG("open(%s, O_RDONLY) failed: %d (%s)\n", arg, errno, strerror(errno)); + } return (&fifo_read_fd); } @@ -235,7 +234,10 @@ open_fifo(const char *path, int *readfd, int *writefd) fifo_read_fd = -1; res = pthread_create(&thread, 0, open_fifo_readside, (void*)path); if (res == 0) { - tmpwritefd = open(path, O_WRONLY); + if ((tmpwritefd = open(path, O_WRONLY)) == -1) { + T_LOG("open(%s, O_WRONLY) failed: %d (%s)\n", path, errno, strerror(errno)); + return (-1); + } waitres = pthread_join(thread, (void**) &tmpreadfd); fcntl(tmpwritefd, F_SETFL, O_WRONLY | O_NONBLOCK); @@ -288,47 +290,52 @@ execute_action(void *actionptr) struct timeval tv; struct stat sstat; - LOG(1, stderr, "Beginning action of type %d: %s\n", act->act_id, get_action_name(act->act_id)); + T_LOG("Beginning action of type %d: %s\n", act->act_id, get_action_name(act->act_id)); /* Let other thread get into kevent() sleep */ if(SLEEP == act->act_dosleep) { - sleep(SLEEP_TIME); + usleep(USLEEP_TIME); } switch(act->act_id) { case NOTHING: res = 0; break; case CREAT: - tmpfd = creat((char*)args[0], 0755); - ftruncate(tmpfd, 1); /* So that mmap() doesn't fool us */ - if (tmpfd >= 0) { - close(tmpfd); - res = 0; + if ((tmpfd = creat((char*)args[0], 0755)) == -1) { + T_LOG("creat() failed on \"%s\": %d (%s)\n", args[0], errno, strerror(errno)); + res = -1; + break; } + ftruncate(tmpfd, 1); /* So that mmap() doesn't fool us */ + close(tmpfd); + res = 0; break; case MKDIR: res = mkdir((char*)args[0], 0755); break; case READ: - tmpfd = open((char*)args[0], O_RDONLY); - if (tmpfd >= 0) { - res = read(tmpfd, &c, 1); - res = (res == 1 ? 0 : -1); + if ((tmpfd = open((char*)args[0], O_RDONLY)) == -1) { + T_LOG("open(%s, O_RDONLY) failed: %d (%s)\n", args[0], errno, strerror(errno)); + res = -1; + break; } + res = read(tmpfd, &c, 1); + res = (res == 1 ? 0 : -1); close(tmpfd); break; case WRITE: - tmpfd = open((char*)args[0], O_RDWR); - if (tmpfd >= 0) { - res = write(tmpfd, TEST_STRING, strlen(TEST_STRING)); - if (res == strlen(TEST_STRING)) { - res = 0; - } else { - res = -1; - } - - close(tmpfd); + if ((tmpfd = open((char*)args[0], O_RDWR)) == -1) { + T_LOG("open(%s, O_RDWR) failed: %d (%s)\n", args[0], errno, strerror(errno)); + res = -1; + break; } + res = write(tmpfd, TEST_STRING, strlen(TEST_STRING)); + if (res == strlen(TEST_STRING)) { + res = 0; + } else { + res = -1; + } + close(tmpfd); break; case WRITEFD: res = write((int)act->act_fd, TEST_STRING, strlen(TEST_STRING)); @@ -378,12 +385,20 @@ execute_action(void *actionptr) res = rename((char*)args[0], (char*)args[1]); break; case OPEN: - tmpfd = open((char*)args[0], O_RDONLY | O_CREAT); + if ((tmpfd = open((char*)args[0], O_RDONLY | O_CREAT)) == -1) { + T_LOG("open(%s, O_RDONLY | O_CREAT) failed: %d (%s)\n", args[0], errno, strerror(errno)); + res = -1; + break; + } res = close(tmpfd); break; case MMAP: /* It had best already exist with nonzero size */ - tmpfd = open((char*)args[0], O_RDWR); + if ((tmpfd = open((char*)args[0], O_RDWR)) == -1) { + T_LOG("open(%s, O_RDWR) failed: %d (%s)\n", args[0], errno, strerror(errno)); + res = -1; + break; + } addr = mmap(0, 20, PROT_WRITE | PROT_READ, MAP_FILE | MAP_SHARED, tmpfd, 0); if (addr != ((void*)-1)) { res = 0; @@ -411,18 +426,31 @@ execute_action(void *actionptr) res = link((char*)args[0], (char*)args[1]); break; case REVOKE: - tmpfd = open((char*)args[0], O_RDONLY); + if ((tmpfd = open((char*)args[0], O_RDONLY)) == -1) { + T_LOG("open(%s, O_RDONLY) failed: %d (%s)\n", args[0], errno, strerror(errno)); + res = -1; + break; + } res = revoke((char*)args[0]); close(tmpfd); break; case FUNLOCK: - tmpfd = open((char*)args[0], O_RDONLY); - if (tmpfd != -1) { - res = flock(tmpfd, LOCK_EX); - if (res != -1) - res = flock(tmpfd, LOCK_UN); - (void)close(tmpfd); + if ((tmpfd = open((char*)args[0], O_RDONLY)) == -1) { + T_LOG("open(%s, O_RDONLY) failed: %d (%s)\n", args[0], errno, strerror(errno)); + res = -1; + break; + } + if ((res = flock(tmpfd, LOCK_EX)) == -1) { + T_LOG("flock() LOCK_EX failed: %d (%s)\n", errno, strerror(errno)); + close(tmpfd); + break; + } + if ((res = flock(tmpfd, LOCK_UN)) == -1) { + T_LOG("flock() LOCK_UN failed: %d (%s)\n", errno, strerror(errno)); + close(tmpfd); + break; } + close(tmpfd); break; default: res = -1; @@ -452,15 +480,16 @@ execute_action_list(action_t *actions, int nactions, int failout) { int i, res; for (i = 0, res = 0; (0 == res || (!failout)) && (i < nactions); i++) { - LOG(1, stderr, "Starting prep action %d\n", i); + T_LOG("Starting prep action %d\n", i); res = *((int *) execute_action(&(actions[i]))); if(res != 0) { - LOG(2, stderr, "Action list failed on step %d. res = %d\n", i, res); + T_LOG("Action list failed on step %d. res = %d errno = %d (%s)\n", i, res, + errno, strerror(errno)); } else { - LOG(1, stderr, "Action list work succeeded on step %d.\n", i); + T_LOG("Action list work succeeded on step %d.\n", i); } } - + return res; } @@ -479,8 +508,9 @@ execute_test(test_t *test) memset(&evlist, 0, sizeof(evlist)); - LOG(1, stderr, "Test %s starting.\n", test->t_testname); - LOG(1, stderr, test->t_want_event ? "Expecting an event.\n" : "Not expecting events.\n"); + T_LOG("[BEGIN] %s\n", test->t_testname); + + T_LOG(test->t_want_event ? "Expecting an event.\n" : "Not expecting events.\n"); res = execute_action_list(test->t_prep_actions, test->t_n_prep_actions, 1); @@ -488,22 +518,26 @@ execute_test(test_t *test) if (0 == res) { /* Create kqueue for kqueue tests*/ if (!test->t_is_poll_test) { - kqfd = kqueue(); + if ((kqfd = kqueue()) == -1) { + T_LOG("kqueue() failed: %d (%s)\n", errno, strerror(errno)); + } } if ((test->t_is_poll_test) || kqfd >= 0) { - LOG(1, stderr, "Opened kqueue.\n"); /* Open the file we're to monitor. Fifos get special handling */ if (test->t_file_is_fifo) { filefd = -1; open_fifo(test->t_watchfile, &filefd, &writefd); } else { - filefd = open(test->t_watchfile, O_RDONLY | O_SYMLINK); + if ((filefd = open(test->t_watchfile, O_RDONLY | O_SYMLINK)) == -1) { + T_LOG("open() of watchfile %s failed: %d (%s)\n", test->t_watchfile, + errno, strerror(errno)); + } } if (filefd >= 0) { - LOG(1, stderr, "Opened file to monitor.\n"); + T_LOG("Opened file to monitor.\n"); /* * Fill in the fd to monitor once you know it @@ -524,11 +558,11 @@ execute_test(test_t *test) thread_status = 0; res = pthread_create(&thr, NULL, execute_action, (void*) &test->t_helpthreadact); if (0 == res) { - LOG(1, stderr, "Created helper thread.\n"); + T_LOG("Created helper thread.\n"); /* This is ugly business to hack on filling up a FIFO */ if (test->t_extra_sleep_hack) { - sleep(5); + usleep(USLEEP_TIME); } if (test->t_is_poll_test) { @@ -536,23 +570,22 @@ execute_test(test_t *test) pl.fd = filefd; pl.events = test->t_union.tu_pollevents; cnt = poll(&pl, 1, WAIT_TIME); - LOG(1, stderr, "Finished poll() call.\n"); - + T_LOG("Finished poll() call.\n"); if ((cnt < 0)) { - LOG(2, stderr, "error is in errno, %s\n", strerror(errno)); + T_LOG("error is in errno, %s\n", strerror(errno)); res = cnt; } } else { test->t_union.tu_kev.ident = filefd; cnt = kevent(kqfd, &test->t_union.tu_kev, 1, &evlist, 1, &ts); - LOG(1, stderr, "Finished kevent() call.\n"); + T_LOG("Finished kevent() call.\n"); if ((cnt < 0) || (evlist.flags & EV_ERROR)) { - LOG(2, stderr, "kevent() call failed.\n"); + T_LOG("kevent() call failed.\n"); if (cnt < 0) { - LOG(2, stderr, "error is in errno, %s\n", strerror(errno)); + T_LOG("error is in errno, %s\n", strerror(errno)); } else { - LOG(2, stderr, "error is in data, %s\n", strerror(evlist.data)); + T_LOG("error is in data, %s\n", strerror(evlist.data)); } res = cnt; } @@ -561,14 +594,15 @@ execute_test(test_t *test) /* Success only if you've succeeded to this point AND joined AND other thread is happy*/ status = NULL; res2 = pthread_join(thr, (void **)&status); - if (res2 < 0) { - LOG(2, stderr, "Couldn't join helper thread.\n"); + if (res2 != 0) { + T_LOG("Couldn't join helper thread: %d (%s).\n", res2, + strerror(res2)); } else if (*status) { - LOG(2, stderr, "Helper action had result %d\n", *status); + T_LOG("Helper action had result %d\n", *status); } res = ((res == 0) && (res2 == 0) && (*status == 0)) ? 0 : -1; } else { - LOG(2, stderr, "Couldn't start thread.\n"); + T_LOG("Couldn't start thread: %d (%s).\n", res, strerror(res)); } close(filefd); @@ -576,12 +610,14 @@ execute_test(test_t *test) close(writefd); } } else { - LOG(2, stderr, "Couldn't open test file %s to monitor.\n", test->t_watchfile); + T_LOG("Couldn't open test file %s to monitor: %d (%s)\n", test->t_watchfile); res = -1; } - close(kqfd); + if (!test->t_is_poll_test) { + close(kqfd); + } } else { - LOG(2, stderr, "Couldn't open kqueue.\n"); + T_LOG("Couldn't open kqueue.\n"); res = -1; } } @@ -593,34 +629,55 @@ execute_test(test_t *test) * as expected */ if (0 == res) { - LOG(1, stderr, cnt > 0 ? "Got an event.\n" : "Did not get an event.\n"); + T_LOG(cnt > 0 ? "Got an event.\n" : "Did not get an event.\n"); if (((cnt > 0) && (test->t_want_event)) || ((cnt == 0) && (!test->t_want_event))) { if ((!test->t_is_poll_test) && (test->t_union.tu_kev.filter == EVFILT_READ || test->t_union.tu_kev.filter == EVFILT_WRITE) && (test->t_nbytes) && (test->t_nbytes != evlist.data)) { - LOG(2, stderr, "Read wrong number of bytes available. Wanted %d, got %d\n", test->t_nbytes, evlist.data); + T_LOG("Read wrong number of bytes available. Wanted %d, got %d\n", test->t_nbytes, evlist.data); retval = -1; } else { retval = 0; } } else { - LOG(2, stderr, "Got unexpected event or lack thereof.\n"); + T_LOG("Got unexpected event or lack thereof.\n"); retval = -1; } } else { - LOG(2, stderr, "Failed to execute test. res = %d\n", res); + T_LOG("Failed to execute test. res = %d\n", res); retval = -1; } - - LOG(3, stdout, "Test %s done with result %d.\n", test->t_testname, retval); + + if (test->t_nondeterministic) { + T_LOG("XXX non-deterministic test result = %d (%s)\n", retval, + (retval == 0) ? "pass" : "fail"); + T_MAYFAIL; + } else { + if (test->t_known_failure) { + // Signal to harness that this test is expected to fail. + T_EXPECTFAIL; + } + } + + if (retval == 0) { + T_PASS("%s", test->t_testname); + } else { + T_FAIL("%s", test->t_testname); + } + + T_LOG("Test %s done with result %d.\n", test->t_testname, retval); return (retval); } + + void init_test_common(test_t *tst, char *testname, char *watchfile, int nprep, int nclean, int event, int want, int ispoll) { memset(tst, 0, sizeof(test_t)); tst->t_testname = testname; + tst->t_known_failure = 0; + tst->t_nondeterministic = 0; tst->t_watchfile = watchfile; tst->t_n_prep_actions = nprep; tst->t_n_cleanup_actions = nclean; @@ -668,16 +725,19 @@ run_note_delete_tests() test_t test; init_test(&test, "1.1.2: unlink a file", FILE1, 1, 0, NOTE_DELETE, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, UNLINK, 2, (void*)FILE1, NULL); execute_test(&test); init_test(&test, "1.1.3: rmdir a dir", DIR1, 1, 0, NOTE_DELETE, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RMDIR, 2, (void*)DIR1, NULL); execute_test(&test); init_test(&test, "1.1.4: rename one file over another", FILE2, 2, 1, NOTE_DELETE, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)FILE2, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)FILE1, (void*)FILE2); @@ -685,6 +745,7 @@ run_note_delete_tests() execute_test(&test); init_test(&test, "1.1.5: rename one dir over another", DIR2, 2, 1, NOTE_DELETE, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, MKDIR, 2, (void*)DIR2, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)DIR1, (void*)DIR2); @@ -699,6 +760,7 @@ run_note_delete_tests() execute_test(&test); init_test(&test, "1.1.7: rename a file over a fifo", FILE1, 2, 1, NOTE_DELETE, YES_EVENT); + test.t_nondeterministic = 1; test.t_file_is_fifo = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKFIFO, 2, (void*)FILE1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)FILE2, (void*)NULL); @@ -707,6 +769,7 @@ run_note_delete_tests() execute_test(&test); init_test(&test, "1.1.8: unlink a symlink to a file", FILE2, 2, 1, NOTE_DELETE, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, SYMLINK, 2, (void*)FILE1, (void*)FILE2); init_action(&test.t_helpthreadact, SLEEP, UNLINK, 2, (void*)FILE2, NULL); @@ -747,6 +810,14 @@ run_note_delete_tests() init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR2, NULL); } +static bool +path_on_apfs(const char *path) +{ + struct statfs sfs = {}; + T_QUIET; T_ASSERT_POSIX_SUCCESS(statfs(path, &sfs), NULL); + return (memcmp(&sfs.f_fstypename[0], "apfs", strlen("apfs")) == 0); +} + void run_note_write_tests() { @@ -764,6 +835,7 @@ run_note_write_tests() makepath(pathbuf, DIR1, FILE1); init_test(&test, "2.1.2: creat() file inside a dir", DIR1, 1, 2, NOTE_WRITE, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, CREAT, 2, (void*)pathbuf, NULL); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)pathbuf, (void*)NULL); @@ -772,6 +844,7 @@ run_note_write_tests() makepath(pathbuf, DIR1, FILE1); init_test(&test, "2.1.3: open() file inside a dir", DIR1, 1, 2, NOTE_WRITE, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, OPEN, 2, (void*)pathbuf, NULL); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)pathbuf, (void*)NULL); @@ -779,7 +852,8 @@ run_note_write_tests() execute_test(&test); makepath(pathbuf, DIR1, FILE1); - init_test(&test, "2.1.3: unlink a file from a dir", DIR1, 2, 1, NOTE_WRITE, YES_EVENT); + init_test(&test, "2.1.4: unlink a file from a dir", DIR1, 2, 1, NOTE_WRITE, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)pathbuf, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, UNLINK, 2, (void*)pathbuf, NULL); @@ -789,6 +863,7 @@ run_note_write_tests() makepath(pathbuf, DIR1, FILE1); makepath(otherpathbuf, DIR1, FILE2); init_test(&test, "2.1.5: rename a file in a dir", DIR1, 2, 2, NOTE_WRITE, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)pathbuf, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)pathbuf, (void*)otherpathbuf); @@ -798,6 +873,7 @@ run_note_write_tests() makepath(pathbuf, DIR1, FILE1); init_test(&test, "2.1.6: rename a file to outside of a dir", DIR1, 2, 2, NOTE_WRITE, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)pathbuf, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)pathbuf, (void*)FILE1); @@ -807,6 +883,7 @@ run_note_write_tests() makepath(pathbuf, DIR1, FILE1); init_test(&test, "2.1.7: rename a file into a dir", DIR1, 2, 2, NOTE_WRITE, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)FILE1, (void*)pathbuf); @@ -816,6 +893,7 @@ run_note_write_tests() makepath(pathbuf, DIR1, FILE1); init_test(&test, "2.1.9: unlink a fifo from a dir", DIR1, 2, 1, NOTE_WRITE, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, MKFIFO, 2, (void*)pathbuf, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, UNLINK, 2, (void*)pathbuf, NULL); @@ -824,6 +902,7 @@ run_note_write_tests() makepath(pathbuf, DIR1, FILE1); init_test(&test, "2.1.10: make symlink in a dir", DIR1, 1, 2, NOTE_WRITE, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, SYMLINK, 2, (void*)DOTDOT, (void*)pathbuf); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)pathbuf, (void*)NULL); @@ -831,6 +910,7 @@ run_note_write_tests() execute_test(&test); init_test(&test, "2.1.12: write to a FIFO", FILE1, 1, 1, NOTE_WRITE, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKFIFO, 2, (void*)FILE1, (void*)NULL); test.t_file_is_fifo = 1; init_action(&test.t_helpthreadact, SLEEP, WRITEFD, 0); @@ -840,26 +920,30 @@ run_note_write_tests() makepath(pathbuf, DIR1, FILE1); init_test(&test, "2.1.13: delete a symlink in a dir", DIR1, 2, 1, NOTE_WRITE, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, SYMLINK, 2, (void*)DOTDOT, (void*)pathbuf); init_action(&test.t_helpthreadact, SLEEP, UNLINK, 2, (void*)pathbuf, (void*)FILE1); init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); execute_test(&test); - - /* This actually should not generate an event, though it's in this section */ - makepath(pathbuf, DIR1, FILE1); - makepath(otherpathbuf, DIR1, FILE2); - init_test(&test, "2.1.14: exchangedata two files in a dir", DIR1, 3, 3, NOTE_WRITE, NO_EVENT); - init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); - init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)pathbuf, (void*)NULL); - init_action(&(test.t_prep_actions[2]), NOSLEEP, CREAT, 2, (void*)otherpathbuf, (void*)NULL); - init_action(&test.t_helpthreadact, SLEEP, EXCHANGEDATA, 2, (void*)pathbuf, (void*)otherpathbuf); - init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)pathbuf, (void*)NULL); - init_action(&test.t_cleanup_actions[1], NOSLEEP, UNLINK, 2, (void*)otherpathbuf, (void*)NULL); - init_action(&test.t_cleanup_actions[2], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); - execute_test(&test); - - LOG(1, stderr, "MMAP test should fail on HFS.\n"); + + /* exchangedata is not supported on APFS volumes */ + if (!path_on_apfs(PDIR)) { + /* This actually should not generate an event, though it's in this section */ + makepath(pathbuf, DIR1, FILE1); + makepath(otherpathbuf, DIR1, FILE2); + init_test(&test, "2.1.14: exchangedata two files in a dir", DIR1, 3, 3, NOTE_WRITE, NO_EVENT); + test.t_known_failure = 1; + init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); + init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)pathbuf, (void*)NULL); + init_action(&(test.t_prep_actions[2]), NOSLEEP, CREAT, 2, (void*)otherpathbuf, (void*)NULL); + init_action(&test.t_helpthreadact, SLEEP, EXCHANGEDATA, 2, (void*)pathbuf, (void*)otherpathbuf); + init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)pathbuf, (void*)NULL); + init_action(&test.t_cleanup_actions[1], NOSLEEP, UNLINK, 2, (void*)otherpathbuf, (void*)NULL); + init_action(&test.t_cleanup_actions[2], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); + execute_test(&test); + } + init_test(&test, "2.1.15: Change a file with mmap()", FILE1, 1, 1, NOTE_WRITE, YES_EVENT); init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, MMAP, 2, (void*)FILE1, (void*)1); /* 1 -> "modify it"*/ @@ -919,16 +1003,15 @@ run_note_write_tests() execute_test(&test); struct passwd *pwd = getpwnam("local"); - int uid = pwd->pw_uid; - int gid = pwd->pw_gid; - - init_test(&test, "2.2.11: chown a file", FILE1, 2, 1, NOTE_WRITE, NO_EVENT); - init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); - init_action(&test.t_prep_actions[1], NOSLEEP, CHOWN, 3, (void*)FILE1, (void*)uid, (void*)gid); - init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, (void*)FILE1, (void*)getuid(), (void*)getgid()); - init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); - execute_test(&test); - + + if (pwd != NULL) { + init_test(&test, "2.2.11: chown a file", FILE1, 2, 1, NOTE_WRITE, NO_EVENT); + init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); + init_action(&test.t_prep_actions[1], NOSLEEP, CHOWN, 3, (void*)FILE1, (void*)pwd->pw_uid, (void*)pwd->pw_gid); + init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, (void*)FILE1, (void*)getuid(), (void*)getgid()); + init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); + execute_test(&test); + } init_test(&test, "2.2.12: chmod a dir", DIR1, 1, 1, NOTE_WRITE, NO_EVENT); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); @@ -936,16 +1019,16 @@ run_note_write_tests() init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); execute_test(&test); - init_test(&test, "2.2.13: chown a dir", DIR1, 2, 1, NOTE_WRITE, NO_EVENT); - init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); - init_action(&test.t_prep_actions[1], NOSLEEP, CHOWN, 3, (void*)DIR1, (void*)uid, (void*)gid); - init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, (void*)DIR1, (void*)getuid(), (void*)getgid()); - init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); - execute_test(&test); - - + if (pwd != NULL) { + init_test(&test, "2.2.13: chown a dir", DIR1, 2, 1, NOTE_WRITE, NO_EVENT); + init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); + init_action(&test.t_prep_actions[1], NOSLEEP, CHOWN, 3, (void*)DIR1, (void*)pwd->pw_uid, (void*)pwd->pw_gid); + init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, (void*)DIR1, (void*)getuid(), (void*)getgid()); + init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); + execute_test(&test); + } - LOG(1, stderr, "MMAP will never give a notification on HFS.\n"); + T_LOG("MMAP will never give a notification on HFS.\n"); init_test(&test, "2.1.14: mmap() a file but do not change it", FILE1, 1, 1, NOTE_WRITE, NO_EVENT); init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, MMAP, 2, (void*)FILE1, (void*)0); @@ -959,9 +1042,10 @@ run_note_extend_tests() test_t test; char pathbuf[50]; - LOG(1, stderr, "THESE TESTS WILL FAIL ON HFS!\n"); + T_LOG("THESE TESTS MAY FAIL ON HFS\n"); init_test(&test, "3.1.1: write beyond the end of a file", FILE1, 1, 1, NOTE_EXTEND, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, WRITE, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); @@ -1008,6 +1092,7 @@ run_note_extend_tests() execute_test(&test); */ init_test(&test, "3.1.7: lengthen a file with truncate()", FILE1, 1, 1, NOTE_EXTEND, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, LENGTHEN, 2, FILE1, (void*)NULL); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); @@ -1028,20 +1113,16 @@ run_note_extend_tests() execute_test(&test); struct passwd *pwd = getpwnam("local"); - if (!pwd) { - LOG(2, stderr, "Couldn't getpwnam for local.\n"); - exit(1); - } - int uid = pwd->pw_uid; - int gid = pwd->pw_gid; - - init_test(&test, "3.2.3: chown a file", FILE1, 2, 1, NOTE_EXTEND, NO_EVENT); - init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); - init_action(&test.t_prep_actions[1], NOSLEEP, CHOWN, 3, (void*)FILE1, (void*)uid, (void*)gid); - init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, (void*)FILE1, (void*)getuid(), (void*)getgid()); - init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); - execute_test(&test); - + if (pwd != NULL) { + init_test(&test, "3.2.3: chown a file", FILE1, 2, 1, NOTE_EXTEND, NO_EVENT); + init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); + init_action(&test.t_prep_actions[1], NOSLEEP, CHOWN, 3, (void*)FILE1, (void*)pwd->pw_uid, (void*)pwd->pw_gid); + init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, (void*)FILE1, (void*)getuid(), (void*)getgid()); + init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); + execute_test(&test); + } else { + T_LOG("Couldn't getpwnam for user \"local\"\n"); + } init_test(&test, "3.2.4: chmod a dir", DIR1, 1, 1, NOTE_EXTEND, NO_EVENT); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); @@ -1049,12 +1130,14 @@ run_note_extend_tests() init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); execute_test(&test); - init_test(&test, "3.2.5: chown a dir", DIR1, 2, 1, NOTE_EXTEND, NO_EVENT); - init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); - init_action(&test.t_prep_actions[1], NOSLEEP, CHOWN, 3, (void*)DIR1, (void*)uid, (void*)gid); - init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, (void*)DIR1, (void*)getuid(), (void*)getgid()); - init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); - execute_test(&test); + if (pwd != NULL) { + init_test(&test, "3.2.5: chown a dir", DIR1, 2, 1, NOTE_EXTEND, NO_EVENT); + init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); + init_action(&test.t_prep_actions[1], NOSLEEP, CHOWN, 3, (void*)DIR1, (void*)pwd->pw_uid, (void*)pwd->pw_gid); + init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, (void*)DIR1, (void*)getuid(), (void*)getgid()); + init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); + execute_test(&test); + } init_test(&test, "3.2.6: TRUNC a file with truncate()", FILE1, 1, 1, NOTE_EXTEND, NO_EVENT); init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); @@ -1070,58 +1153,64 @@ run_note_attrib_tests() char pathbuf[50]; init_test(&test, "4.1.1: chmod a file", FILE1, 1, 1, NOTE_ATTRIB, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, CHMOD, 2, FILE1, (void*)0700); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); execute_test(&test); struct passwd *pwd = getpwnam("local"); - int uid = pwd->pw_uid; - int gid = pwd->pw_gid; - - init_test(&test, "4.1.2: chown a file", FILE1, 2, 1, NOTE_ATTRIB, YES_EVENT); - init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); - init_action(&(test.t_prep_actions[1]), NOSLEEP, CHOWN, 3, (void*)FILE1, (void*)uid, (void*)gid); - init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, FILE1, (void*)getuid(), (void*)gid); - init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); - execute_test(&test); - + if (pwd != NULL) { + init_test(&test, "4.1.2: chown a file", FILE1, 2, 1, NOTE_ATTRIB, YES_EVENT); + init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); + init_action(&(test.t_prep_actions[1]), NOSLEEP, CHOWN, 3, (void*)FILE1, (void*)pwd->pw_uid, (void*)pwd->pw_gid); + init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, FILE1, (void*)getuid(), (void*)pwd->pw_gid); + init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); + execute_test(&test); + } + init_test(&test, "4.1.3: chmod a dir", DIR1, 1, 1, NOTE_ATTRIB, YES_EVENT); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_helpthreadact), SLEEP, CHMOD, 2, (void*)DIR1, (void*)0700); init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); execute_test(&test); - init_test(&test, "4.1.4: chown a dir", DIR1, 2, 1, NOTE_ATTRIB, YES_EVENT); - init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); - init_action(&(test.t_prep_actions[1]), NOSLEEP, CHOWN, 3, (void*)DIR1, (void*) uid, (void*)gid); - init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, DIR1, (void*)getuid(), (void*)getgid()); - init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); - execute_test(&test); + if (pwd != NULL) { + init_test(&test, "4.1.4: chown a dir", DIR1, 2, 1, NOTE_ATTRIB, YES_EVENT); + init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); + init_action(&(test.t_prep_actions[1]), NOSLEEP, CHOWN, 3, (void*)DIR1, (void*) pwd->pw_uid, (void*)pwd->pw_gid); + init_action(&test.t_helpthreadact, SLEEP, CHOWN, 3, DIR1, (void*)getuid(), (void*)getgid()); + init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); + execute_test(&test); + } init_test(&test, "4.1.5: setxattr on a file", FILE1, 1, 1, NOTE_ATTRIB, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, SETXATTR, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); execute_test(&test); init_test(&test, "4.1.6: setxattr on a dir", DIR1, 1, 1, NOTE_ATTRIB, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, SETXATTR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); execute_test(&test); - - - init_test(&test, "4.1.7: exchangedata", FILE1, 2, 2, NOTE_ATTRIB, YES_EVENT); - init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); - init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)FILE2, (void*)NULL); - init_action(&test.t_helpthreadact, SLEEP, EXCHANGEDATA, 2, (void*)FILE1, (void*)FILE2); - init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); - init_action(&test.t_cleanup_actions[1], NOSLEEP, UNLINK, 2, (void*)FILE2, (void*)NULL); - execute_test(&test); - - + + /* exchangedata is not supported on APFS volumes */ + if (!path_on_apfs(PDIR)) { + init_test(&test, "4.1.7: exchangedata", FILE1, 2, 2, NOTE_ATTRIB, YES_EVENT); + init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); + init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)FILE2, (void*)NULL); + init_action(&test.t_helpthreadact, SLEEP, EXCHANGEDATA, 2, (void*)FILE1, (void*)FILE2); + init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); + init_action(&test.t_cleanup_actions[1], NOSLEEP, UNLINK, 2, (void*)FILE2, (void*)NULL); + execute_test(&test); + } + init_test(&test, "4.1.8: utimes on a file", FILE1, 1, 1, NOTE_ATTRIB, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, UTIMES, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); @@ -1165,8 +1254,9 @@ run_note_attrib_tests() init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); execute_test(&test); - LOG(1, stderr, "EXPECT SPURIOUS NOTE_ATTRIB EVENTS FROM DIRECTORY OPERATIONS on HFS.\n"); + T_LOG("EXPECT SPURIOUS NOTE_ATTRIB EVENTS FROM DIRECTORY OPERATIONS on HFS.\n"); init_test(&test, "4.2.6: add a file to a directory with creat()", DIR1, 1, 2, NOTE_ATTRIB, NO_EVENT); + test.t_known_failure = 1; makepath(pathbuf, DIR1, FILE1); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, CREAT, 2, (void*)pathbuf, (void*)NULL); @@ -1175,6 +1265,7 @@ run_note_attrib_tests() execute_test(&test); init_test(&test, "4.2.7: mkdir in a dir", DIR1, 1, 2, NOTE_ATTRIB, NO_EVENT); + test.t_known_failure = 1; makepath(pathbuf, DIR1, DIR2); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, MKDIR, 2, (void*)pathbuf, (void*)NULL); @@ -1183,6 +1274,7 @@ run_note_attrib_tests() execute_test(&test); init_test(&test, "4.2.8: add a symlink to a directory", DIR1, 1, 2, NOTE_ATTRIB, NO_EVENT); + test.t_known_failure = 1; makepath(pathbuf, DIR1, FILE1); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, SYMLINK, 2, (void*)DOTDOT, (void*)pathbuf); @@ -1191,6 +1283,7 @@ run_note_attrib_tests() execute_test(&test); init_test(&test, "4.2.9: rename into a dir()", DIR1, 2, 2, NOTE_ATTRIB, NO_EVENT); + test.t_known_failure = 1; makepath(pathbuf, DIR1, FILE1); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); @@ -1200,6 +1293,7 @@ run_note_attrib_tests() execute_test(&test); init_test(&test, "4.2.10: unlink() file from dir", DIR1, 2, 1, NOTE_ATTRIB, NO_EVENT); + test.t_known_failure = 1; makepath(pathbuf, DIR1, FILE1); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)pathbuf, (void*)NULL); @@ -1208,6 +1302,7 @@ run_note_attrib_tests() execute_test(&test); init_test(&test, "4.2.11: mkfifo in a directory", DIR1, 1, 2, NOTE_ATTRIB, NO_EVENT); + test.t_known_failure = 1; makepath(pathbuf, DIR1, FILE1); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, MKFIFO, 1, (void*)pathbuf); @@ -1226,8 +1321,9 @@ run_note_link_tests() char pathbuf[50]; char otherpathbuf[50]; - LOG(1, stderr, "HFS DOES NOT HANDLE UNLINK CORRECTLY...\n"); + T_LOG("HFS DOES NOT HANDLE UNLINK CORRECTLY...\n"); init_test(&test, "5.1.1: unlink() a file", FILE1, 1, 0, NOTE_LINK, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, UNLINK, 2, (void*)FILE1, (void*)NULL); execute_test(&test); @@ -1241,6 +1337,9 @@ run_note_link_tests() execute_test(&test); init_test(&test, "5.1.2: link() to a file", FILE1, 1, 2, NOTE_LINK, YES_EVENT); +#if TARGET_OS_WATCH + test.t_nondeterministic = 1; +#endif init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, HARDLINK, 2, (void*)FILE1, (void*)FILE2); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, NULL); @@ -1249,6 +1348,7 @@ run_note_link_tests() makepath(pathbuf, DIR1, DIR2); init_test(&test, "5.1.3: make one dir in another", DIR1, 1, 2, NOTE_LINK, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, MKDIR, 2, (void*)pathbuf, (void*)NULL); init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)pathbuf, NULL); @@ -1257,6 +1357,7 @@ run_note_link_tests() makepath(pathbuf, DIR1, DIR2); init_test(&test, "5.1.4: rmdir a dir from within another", DIR1, 2, 1, NOTE_LINK, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, MKDIR, 2, (void*)pathbuf, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RMDIR, 2, (void*)pathbuf, (void*)NULL); @@ -1266,6 +1367,7 @@ run_note_link_tests() makepath(pathbuf, DIR1, DIR2); makepath(otherpathbuf, DIR1, DIR1); init_test(&test, "5.1.5: rename dir A over dir B inside dir C", DIR1, 3, 2, NOTE_LINK, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, MKDIR, 2, (void*)pathbuf, (void*)NULL); init_action(&(test.t_prep_actions[2]), NOSLEEP, MKDIR, 2, (void*)otherpathbuf, (void*)NULL); @@ -1274,9 +1376,10 @@ run_note_link_tests() init_action(&test.t_cleanup_actions[1], NOSLEEP, RMDIR, 2, (void*)DIR1, NULL); execute_test(&test); - LOG(1, stderr, "HFS bypasses hfs_makenode to create in target, so misses knote.\n"); + T_LOG("HFS bypasses hfs_makenode to create in target, so misses knote.\n"); makepath(pathbuf, DIR1, DIR2); init_test(&test, "5.1.6: rename one dir into another", DIR1, 2, 2, NOTE_LINK, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, MKDIR, 2, (void*)DIR2, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)DIR2, (void*)pathbuf); @@ -1284,9 +1387,10 @@ run_note_link_tests() init_action(&test.t_cleanup_actions[1], NOSLEEP, RMDIR, 2, (void*)DIR1, NULL); execute_test(&test); - LOG(1, stderr, "HFS bypasses hfs_removedir to remove from source, so misses knote.\n"); + T_LOG("HFS bypasses hfs_removedir to remove from source, so misses knote.\n"); makepath(pathbuf, DIR1, DIR2); init_test(&test, "5.1.7: rename one dir out of another", DIR1, 2, 2, NOTE_LINK, YES_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, MKDIR, 2, (void*)pathbuf, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)pathbuf, (void*)DIR2); @@ -1295,6 +1399,7 @@ run_note_link_tests() execute_test(&test); init_test(&test, "5.1.8: rmdir a dir", DIR1, 1, 0, NOTE_LINK, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RMDIR, 2, (void*)DIR1, (void*)NULL); execute_test(&test); @@ -1302,6 +1407,7 @@ run_note_link_tests() /* ============= NO EVENT SECTION ============== */ makepath(pathbuf, DIR1, FILE1); init_test(&test, "5.2.1: make a file in a dir", DIR1, 1, 2, NOTE_LINK, NO_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, CREAT, 2, (void*)pathbuf, (void*)NULL); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)pathbuf, NULL); @@ -1310,6 +1416,7 @@ run_note_link_tests() makepath(pathbuf, DIR1, FILE1); init_test(&test, "5.2.2: unlink a file in a dir", DIR1, 2, 1, NOTE_LINK, NO_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)pathbuf, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, UNLINK, 2, (void*)pathbuf, (void*)NULL); @@ -1319,6 +1426,7 @@ run_note_link_tests() makepath(pathbuf, DIR1, FILE1); makepath(otherpathbuf, DIR1, FILE2); init_test(&test, "5.2.3: rename a file within a dir", DIR1, 2, 2, NOTE_LINK, NO_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)pathbuf, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)pathbuf, (void*)otherpathbuf); @@ -1328,6 +1436,7 @@ run_note_link_tests() makepath(pathbuf, DIR1, FILE1); init_test(&test, "5.2.4: rename a file into a dir", DIR1, 2, 2, NOTE_LINK, NO_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)FILE1, (void*)pathbuf); @@ -1337,6 +1446,7 @@ run_note_link_tests() makepath(pathbuf, DIR1, FILE1); init_test(&test, "5.2.5: make a symlink in a dir", DIR1, 1, 2, NOTE_LINK, NO_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, SYMLINK, 2, (void*)DOTDOT, (void*)pathbuf); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)pathbuf, NULL); @@ -1344,6 +1454,7 @@ run_note_link_tests() execute_test(&test); init_test(&test, "5.2.6: make a symlink to a dir", DIR1, 1, 2, NOTE_LINK, NO_EVENT); + test.t_known_failure = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, SYMLINK, 2, (void*)DIR1, (void*)FILE1); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, NULL); @@ -1364,25 +1475,28 @@ run_note_rename_tests() test_t test; init_test(&test, "6.1.1: rename a file", FILE1, 1, 1, NOTE_RENAME, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)FILE1, (void*)FILE2); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE2, NULL); execute_test(&test); init_test(&test, "6.1.2: rename a dir", DIR1, 1, 1, NOTE_RENAME, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)DIR1, (void*)DIR2); init_action(&test.t_cleanup_actions[0], NOSLEEP, RMDIR, 2, (void*)DIR2, NULL); execute_test(&test); - init_test(&test, "6.1.2: rename one file over another", FILE1, 2, 1, NOTE_RENAME, YES_EVENT); + init_test(&test, "6.1.3: rename one file over another", FILE1, 2, 1, NOTE_RENAME, YES_EVENT); init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, CREAT, 2, (void*)FILE2, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)FILE1, (void*)FILE2); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE2, NULL); execute_test(&test); - init_test(&test, "6.1.3: rename one dir over another", DIR1, 2, 1, NOTE_RENAME, YES_EVENT); + init_test(&test, "6.1.4: rename one dir over another", DIR1, 2, 1, NOTE_RENAME, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, MKDIR, 2, (void*)DIR1, (void*)NULL); init_action(&(test.t_prep_actions[1]), NOSLEEP, MKDIR, 2, (void*)DIR2, (void*)NULL); init_action(&test.t_helpthreadact, SLEEP, RENAME, 2, (void*)DIR1, (void*)DIR2); @@ -1541,7 +1655,7 @@ read_from_fd(void *arg) { char buf[50]; int fd = (int) arg; - sleep(2); + usleep(USLEEP_TIME); return (void*) read(fd, buf, sizeof(buf)); } @@ -1550,7 +1664,7 @@ write_to_fd(void *arg) { char buf[50]; int fd = (int) arg; - sleep(2); + usleep(USLEEP_TIME); return (void*) write(fd, buf, sizeof(buf)); } @@ -1579,6 +1693,7 @@ run_evfilt_write_tests() init_test(&test, "9.2.1: how much space in a full fifo?", FILE1, 1, 1, EVFILT_WRITE, 0); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKFIFO, 1, (void*)FILE1, (void*)NULL); + test.t_nondeterministic = 1; test.t_file_is_fifo = 1; test.t_extra_sleep_hack = 1; init_action(&(test.t_helpthreadact), NOSLEEP, FILLFD, 1, (void*)FILE1, (void*)NULL); @@ -1636,10 +1751,12 @@ run_poll_tests() init_poll_test(&test, "10.2.3: does poll say I can write a full FIFO?", FILE1, 1, 1, POLLWRNORM, 0); init_action(&(test.t_prep_actions[0]), NOSLEEP, MKFIFO, 1, (void*)FILE1, (void*)NULL); + test.t_nondeterministic = 1; test.t_file_is_fifo = 1; test.t_extra_sleep_hack = 1; init_action(&(test.t_helpthreadact), NOSLEEP, FILLFD, 1, (void*)FILE1, (void*)NULL); init_action(&test.t_cleanup_actions[0], NOSLEEP, UNLINK, 2, (void*)FILE1, NULL); + test.t_known_failure = 1; execute_test(&test); } @@ -1648,6 +1765,7 @@ run_note_funlock_tests() { test_t test; init_test(&test, "11.1.1: unlock file", FILE1, 1, 1, NOTE_FUNLOCK, YES_EVENT); + test.t_nondeterministic = 1; init_action(&(test.t_prep_actions[0]), NOSLEEP, CREAT, 2, (void*)FILE1, (void *)NULL); init_action(&test.t_helpthreadact, SLEEP, FUNLOCK, 2, (void*)FILE1, (void *)NULL); init_action(&(test.t_cleanup_actions[0]), NOSLEEP, UNLINK, 2, (void*)FILE1, (void *)NULL); @@ -1672,14 +1790,19 @@ run_all_tests() run_note_funlock_tests(); } -int -main(int argc, char **argv) + T_DECL(kqueue_file_tests, + "Tests assorted kqueue operations for file-related events") { char *which = NULL; if (argc > 1) { which = argv[1]; } + T_SETUPBEGIN; + rmdir(DIR1); + rmdir(DIR2); + T_SETUPEND; + if ((!which) || (strcmp(which, "all") == 0)) run_all_tests(); else if (strcmp(which, "delete") == 0) @@ -1710,6 +1833,5 @@ main(int argc, char **argv) "fifo, all, evfiltwrite, funlock\n"); exit(1); } - return 0; } diff --git a/tools/tests/darwintests/kqueue_timer_tests.c b/tools/tests/darwintests/kqueue_timer_tests.c new file mode 100644 index 000000000..e02deb400 --- /dev/null +++ b/tools/tests/darwintests/kqueue_timer_tests.c @@ -0,0 +1,437 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifndef NOTE_MACHTIME +#define NOTE_MACHTIME 0x00000100 +#endif + +static mach_timebase_info_data_t timebase_info; + +static uint64_t nanos_to_abs(uint64_t nanos) { return nanos * timebase_info.denom / timebase_info.numer; } +static uint64_t abs_to_nanos(uint64_t abs) { return abs * timebase_info.numer / timebase_info.denom; } + +static int kq, passed, failed; + +static struct timespec failure_timeout = { .tv_sec = 10, .tv_nsec = 0 }; + +/* + * Wait for given kevent, which should return in 'expected' usecs. + */ +static int +do_simple_kevent(struct kevent64_s *kev, uint64_t expected) +{ + int ret; + int64_t elapsed_usecs; + uint64_t delta_usecs; + struct timespec timeout; + struct timeval before, after; + + /* time out after 1 sec extra delay */ + timeout.tv_sec = (expected / USEC_PER_SEC) + 1; + timeout.tv_nsec = (expected % USEC_PER_SEC) * 1000; + + T_SETUPBEGIN; + + /* measure time for the kevent */ + gettimeofday(&before, NULL); + ret = kevent64(kq, kev, 1, kev, 1, 0, &timeout); + gettimeofday(&after, NULL); + + if (ret < 1 || (kev->flags & EV_ERROR)) { + T_LOG("%s() failure: kevent returned %d, error %d\n", __func__, ret, + (ret == -1 ? errno : (int) kev->data)); + return 0; + } + + T_SETUPEND; + + /* did it work? */ + elapsed_usecs = (after.tv_sec - before.tv_sec) * (int64_t)USEC_PER_SEC + + (after.tv_usec - before.tv_usec); + delta_usecs = (uint64_t)llabs(elapsed_usecs - ((int64_t)expected)); + + /* failure if we're 30% off, or 50 mics late */ + if (delta_usecs > (30 * expected / 100.0) && delta_usecs > 50) { + T_LOG("\tfailure: expected %lld usec, measured %lld usec.\n", + expected, elapsed_usecs); + return 0; + } else { + T_LOG("\tsuccess, measured %lld usec.\n", elapsed_usecs); + return 1; + } +} + +static void +test_absolute_kevent(int time, int scale) +{ + struct timeval tv; + struct kevent64_s kev; + uint64_t nowus, expected, timescale = 0; + int ret; + int64_t deadline; + + gettimeofday(&tv, NULL); + nowus = (uint64_t)tv.tv_sec * USEC_PER_SEC + (uint64_t)tv.tv_usec; + + T_SETUPBEGIN; + + switch (scale) { + case NOTE_MACHTIME: + T_LOG("Testing %d MATUs absolute timer...\n", time); + break; + case NOTE_SECONDS: + T_LOG("Testing %d sec absolute timer...\n", time); + timescale = USEC_PER_SEC; + break; + case NOTE_USECONDS: + T_LOG("Testing %d usec absolute timer...\n", time); + timescale = 1; + break; + case 0: + T_LOG("Testing %d msec absolute timer...\n", time); + timescale = 1000; + break; + default: + T_FAIL("Failure: scale 0x%x not recognized.\n", scale); + return; + } + + T_SETUPEND; + + if (scale == NOTE_MACHTIME) { + expected = abs_to_nanos((uint64_t)time) / NSEC_PER_USEC; + deadline = (int64_t)mach_absolute_time() + time; + } else { + expected = (uint64_t)time * timescale; + deadline = (int64_t)(nowus / timescale) + time; + } + + /* deadlines in the past should fire immediately */ + if (time < 0) + expected = 0; + + EV_SET64(&kev, 1, EVFILT_TIMER, EV_ADD, + NOTE_ABSOLUTE | scale, deadline, 0,0,0); + ret = do_simple_kevent(&kev, expected); + + if (ret) { + passed++; + T_PASS("%s time:%d, scale:0x%x", __func__, time, scale); + } else { + failed++; + T_FAIL("%s time:%d, scale:0x%x", __func__, time, scale); + } +} + +static void +test_oneshot_kevent(int time, int scale) +{ + int ret; + uint64_t expected = 0; + struct kevent64_s kev; + + T_SETUPBEGIN; + + switch (scale) { + case NOTE_MACHTIME: + T_LOG("Testing %d MATUs interval timer...\n", time); + expected = abs_to_nanos((uint64_t)time) / NSEC_PER_USEC; + break; + case NOTE_SECONDS: + T_LOG("Testing %d sec interval timer...\n", time); + expected = (uint64_t)time * USEC_PER_SEC; + break; + case NOTE_USECONDS: + T_LOG("Testing %d usec interval timer...\n", time); + expected = (uint64_t)time; + break; + case NOTE_NSECONDS: + T_LOG("Testing %d nsec interval timer...\n", time); + expected = (uint64_t)time / 1000; + break; + case 0: + T_LOG("Testing %d msec interval timer...\n", time); + expected = (uint64_t)time * 1000; + break; + default: + T_FAIL("Failure: scale 0x%x not recognized.\n", scale); + return; + } + + T_SETUPEND; + + /* deadlines in the past should fire immediately */ + if (time < 0) + expected = 0; + + EV_SET64(&kev, 2, EVFILT_TIMER, EV_ADD | EV_ONESHOT, scale, time, + 0, 0, 0); + ret = do_simple_kevent(&kev, expected); + + if (ret) { + passed++; + T_PASS("%s time:%d, scale:0x%x", __func__, time, scale); + } else { + failed++; + T_FAIL("%s time:%d, scale:0x%x", __func__, time, scale); + } +} + +/* Test that the timer goes ding multiple times */ +static void +test_interval_kevent(int usec) +{ + struct kevent64_s kev; + int ret; + + T_SETUPBEGIN; + + uint64_t test_duration_us = USEC_PER_SEC; /* 1 second */ + uint64_t expected_pops; + + if (usec < 0) + expected_pops = 1; /* TODO: test 'and only once' */ + else + expected_pops = test_duration_us / (uint64_t)usec; + + T_LOG("Testing interval kevent at %d usec intervals (%lld pops/second)...\n", + usec, expected_pops); + + EV_SET64(&kev, 3, EVFILT_TIMER, EV_ADD, NOTE_USECONDS, usec, 0, 0, 0); + ret = kevent64(kq, &kev, 1, NULL, 0, 0, NULL); + if (ret != 0 || (kev.flags & EV_ERROR)) { + T_FAIL("%s() setup failure: kevent64 returned %d\n", __func__, ret); + failed++; + return; + } + + T_SETUPEND; + + struct timeval before, after; + uint64_t elapsed_usecs; + + gettimeofday(&before, NULL); + + uint64_t pops = 0; + + for (uint32_t i = 0; i < expected_pops; i++) { + ret = kevent64(kq, NULL, 0, &kev, 1, 0, &failure_timeout); + if (ret != 1) { + T_FAIL("%s() failure: kevent64 returned %d\n", __func__, ret); + failed++; + return; + } + + //T_LOG("\t ding: %lld\n", kev.data); + + pops += (uint64_t)kev.data; + gettimeofday(&after, NULL); + elapsed_usecs = (uint64_t)((after.tv_sec - before.tv_sec) * (int64_t)USEC_PER_SEC + + (after.tv_usec - before.tv_usec)); + + if (elapsed_usecs > test_duration_us) + break; + } + + /* check how many times the timer fired: within 5%? */ + if (pops > expected_pops + (expected_pops / 20) || + pops < expected_pops - (expected_pops / 20)) { + T_FAIL("%s() usec:%d (saw %lld of %lld expected pops)", __func__, usec, pops, expected_pops); + failed++; + } else { + T_PASS("%s() usec:%d (saw %lld pops)", __func__, usec, pops); + passed++; + } + + EV_SET64(&kev, 3, EVFILT_TIMER, EV_DELETE, 0, 0, 0, 0, 0); + ret = kevent64(kq, &kev, 1, NULL, 0, 0, NULL); + if (ret != 0) { + T_LOG("\tfailed to stop repeating timer: %d\n", ret); + } +} + +/* Test that the repeating timer repeats even while not polling in kqueue */ +static void +test_repeating_kevent(int usec) +{ + struct kevent64_s kev; + int ret; + + T_SETUPBEGIN; + + uint64_t test_duration_us = USEC_PER_SEC; /* 1 second */ + + uint64_t expected_pops = test_duration_us / (uint64_t)usec; + T_LOG("Testing repeating kevent at %d usec intervals (%lld pops/second)...\n", + usec, expected_pops); + + EV_SET64(&kev, 4, EVFILT_TIMER, EV_ADD, NOTE_USECONDS, usec, 0, 0, 0); + ret = kevent64(kq, &kev, 1, NULL, 0, 0, NULL); + if (ret != 0) { + T_FAIL("%s() setup failure: kevent64 returned %d\n", __func__, ret); + failed++; + return; + } + + usleep((useconds_t)test_duration_us); + + ret = kevent64(kq, NULL, 0, &kev, 1, 0, &failure_timeout); + if (ret != 1 || (kev.flags & EV_ERROR)) { + T_FAIL("%s() setup failure: kevent64 returned %d\n", __func__, ret); + failed++; + return; + } + + T_SETUPEND; + + uint64_t pops = (uint64_t) kev.data; + + /* check how many times the timer fired: within 5%? */ + if (pops > expected_pops + (expected_pops / 20) || + pops < expected_pops - (expected_pops / 20)) { + T_FAIL("%s() usec:%d (saw %lld of %lld expected pops)", __func__, usec, pops, expected_pops); + failed++; + } else { + T_PASS("%s() usec:%d (saw %lld pops)", __func__, usec, pops); + passed++; + } + + EV_SET64(&kev, 4, EVFILT_TIMER, EV_DELETE, 0, 0, 0, 0, 0); + ret = kevent64(kq, &kev, 1, NULL, 0, 0, NULL); + if (ret != 0) { + T_LOG("\tfailed to stop repeating timer: %d\n", ret); + } +} + + +static void +test_updated_kevent(int first, int second) +{ + struct kevent64_s kev; + int ret; + + T_LOG("Testing update from %d to %d msecs...\n", first, second); + + T_SETUPBEGIN; + + EV_SET64(&kev, 4, EVFILT_TIMER, EV_ADD|EV_ONESHOT, 0, first, 0, 0, 0); + ret = kevent64(kq, &kev, 1, NULL, 0, 0, NULL); + if (ret != 0) { + T_FAIL("%s() failure: initial kevent returned %d\n", __func__, ret); + failed++; + return; + } + + T_SETUPEND; + + EV_SET64(&kev, 4, EVFILT_TIMER, EV_ONESHOT, 0, second, 0, 0, 0); + + uint64_t expected_us = (uint64_t)second * 1000; + + if (second < 0) + expected_us = 0; + + ret = do_simple_kevent(&kev, expected_us); + + if (ret) { + passed++; + T_PASS("%s() %d, %d", __func__, first, second); + } else { + failed++; + T_FAIL("%s() %d, %d", __func__, first, second); + } +} + +static void +disable_timer_coalescing(void) +{ + struct task_qos_policy qosinfo; + kern_return_t kr; + + T_SETUPBEGIN; + + qosinfo.task_latency_qos_tier = LATENCY_QOS_TIER_0; + qosinfo.task_throughput_qos_tier = THROUGHPUT_QOS_TIER_0; + + kr = task_policy_set(mach_task_self(), TASK_OVERRIDE_QOS_POLICY, (task_policy_t)&qosinfo, + TASK_QOS_POLICY_COUNT); + if (kr != KERN_SUCCESS) { + T_FAIL("task_policy_set(... TASK_OVERRIDE_QOS_POLICY ...) failed: %d (%s)", kr, mach_error_string(kr)); + } + + T_SETUPEND; +} + +T_DECL(kqueue_timer_tests, + "Tests assorted kqueue operations for timer-related events") +{ + /* + * Since we're trying to test timers here, disable timer coalescing + * to improve the accuracy of timer fires for this process. + */ + disable_timer_coalescing(); + + mach_timebase_info(&timebase_info); + + kq = kqueue(); + assert(kq > 0); + passed = 0; + failed = 0; + + test_absolute_kevent(100, 0); + test_absolute_kevent(200, 0); + test_absolute_kevent(300, 0); + test_absolute_kevent(1000, 0); + T_MAYFAIL; + test_absolute_kevent(500, NOTE_USECONDS); + T_MAYFAIL; + test_absolute_kevent(100, NOTE_USECONDS); + T_MAYFAIL; + test_absolute_kevent(2, NOTE_SECONDS); + T_MAYFAIL; + test_absolute_kevent(-1000, 0); + + T_MAYFAIL; + test_absolute_kevent((int)nanos_to_abs(10 * NSEC_PER_MSEC), NOTE_MACHTIME); + + test_oneshot_kevent(1, NOTE_SECONDS); + T_MAYFAIL; + test_oneshot_kevent(10, 0); + T_MAYFAIL; + test_oneshot_kevent(200, NOTE_USECONDS); + T_MAYFAIL; + test_oneshot_kevent(300000, NOTE_NSECONDS); + T_MAYFAIL; + test_oneshot_kevent(-1, NOTE_SECONDS); + + T_MAYFAIL; + test_oneshot_kevent((int)nanos_to_abs(10 * NSEC_PER_MSEC), NOTE_MACHTIME); + + test_interval_kevent(250 * 1000); + T_MAYFAIL; + test_interval_kevent(5 * 1000); + T_MAYFAIL; + test_interval_kevent(200); + T_MAYFAIL; + test_interval_kevent(50); + + test_interval_kevent(-1000); + + test_repeating_kevent(10000); /* 10ms */ + + test_updated_kevent(1000, 2000); + test_updated_kevent(2000, 1000); + test_updated_kevent(1000, -1); + +} diff --git a/tools/tests/darwintests/launchd_plists/com.apple.xnu.test.kevent_qos.plist b/tools/tests/darwintests/launchd_plists/com.apple.xnu.test.kevent_qos.plist new file mode 100644 index 000000000..bf3c2f4a9 --- /dev/null +++ b/tools/tests/darwintests/launchd_plists/com.apple.xnu.test.kevent_qos.plist @@ -0,0 +1,24 @@ + + + + + Label + com.apple.xnu.test.kevent_qos + MachServices + + com.apple.xnu.test.kevent_qos + + + ThrottleInterval + 1 + UserName + root + ProcessType + Adaptive + EnvironmentVariables + + MallocNanoZone + 1 + + + diff --git a/tools/tests/darwintests/mach_continuous_time.c b/tools/tests/darwintests/mach_continuous_time.c index d83ed377a..a7d773bfb 100644 --- a/tools/tests/darwintests/mach_continuous_time.c +++ b/tools/tests/darwintests/mach_continuous_time.c @@ -12,11 +12,19 @@ #include +#if (defined(__arm__) || defined(__arm64__)) +#define HAS_KERNEL_TIME_TRAPS + +extern uint64_t mach_absolute_time_kernel(void); +extern uint64_t mach_continuous_time_kernel(void); + +#endif + extern char **environ; static const int64_t one_mil = 1000*1000; -#define to_ns(ticks) ((ticks * tb_info.numer) / (tb_info.denom)) +#define to_ns(ticks) (((ticks) * tb_info.numer) / (tb_info.denom)) #define to_ms(ticks) (to_ns(ticks)/one_mil) static mach_timebase_info_data_t tb_info; @@ -30,14 +38,60 @@ T_DECL(mct_monotonic, "Testing mach_continuous_time returns sane, monotonic valu T_META_ALL_VALID_ARCHS(true)) { mach_timebase_info(&tb_info); +#ifdef HAS_KERNEL_TIME_TRAPS + bool kernel = false; +#endif volatile uint64_t multiple_test = to_ms(mach_continuous_time()); - for(int i = 0; i < 10; i++) { - uint64_t tmp = to_ms(mach_continuous_time()); - T_ASSERT_GE(tmp, multiple_test, "mach_continuous_time must be monotonic"); + for(int i = 0; i < 20; i++) { + uint64_t tmp; + const char *test_type = "user"; +#ifdef HAS_KERNEL_TIME_TRAPS + if (kernel) { + test_type = "kernel"; + tmp = mach_continuous_time_kernel(); + } else + tmp = mach_continuous_time(); + kernel = !kernel; +#else + tmp = mach_continuous_time(); +#endif + tmp = to_ms(tmp); + T_ASSERT_GE(tmp, multiple_test, "mach_continuous_time (%s) must be monotonic", test_type); + + // each successive call shouldn't be more than 100ms in the future + T_ASSERT_LE(tmp - multiple_test, 100ULL, "mach_continuous_time (%s) should not jump forward too fast", test_type); + + multiple_test = tmp; + } +} + +T_DECL(mat_monotonic, "Testing mach_absolute_time returns sane, monotonic values", + T_META_ALL_VALID_ARCHS(true)) +{ + mach_timebase_info(&tb_info); +#ifdef HAS_KERNEL_TIME_TRAPS + bool kernel = false; +#endif + + volatile uint64_t multiple_test = to_ms(mach_absolute_time()); + for(int i = 0; i < 20; i++) { + uint64_t tmp; + const char *test_type = "user"; +#ifdef HAS_KERNEL_TIME_TRAPS + if (kernel) { + test_type = "kernel"; + tmp = mach_absolute_time_kernel(); + } else + tmp = mach_absolute_time(); + kernel = !kernel; +#endif + tmp = mach_absolute_time(); + tmp = to_ms(tmp); + T_ASSERT_GE(tmp, multiple_test, "mach_absolute_time (%s) must be monotonic", test_type); - // each successive call shouldn't be more than 50ms in the future - T_ASSERT_LE(tmp - multiple_test, 50ULL, "mach_continuous_time should not jump forward too fast"); + // each successive call shouldn't be more than 100ms in the future + T_ASSERT_LE(tmp - multiple_test, 100ULL, "mach_absolute_time (%s) should not jump forward too fast", test_type); multiple_test = tmp; } @@ -62,6 +116,42 @@ T_DECL(mct_pause, "Testing mach_continuous_time and mach_absolute_time don't div T_ASSERT_LE(abs(after_diff - before_diff), 1, "mach_continuous_time and mach_absolute_time should not diverge"); } +#ifdef HAS_KERNEL_TIME_TRAPS +static void update_kern(uint64_t *abs, uint64_t *cont) +{ + uint64_t abs1, abs2, cont1, cont2; + do { + abs1 = mach_absolute_time_kernel(); + cont1 = mach_continuous_time_kernel(); + abs2 = mach_absolute_time_kernel(); + cont2 = mach_continuous_time_kernel(); + } while (to_ms(abs2 - abs1) || to_ms(cont2 - cont1)); + *abs = abs2; + *cont = cont2; +} +#endif + +#ifdef HAS_KERNEL_TIME_TRAPS +T_DECL(mct_pause_kern, "Testing kernel mach_continuous_time and mach_absolute_time don't diverge") +{ + mach_timebase_info(&tb_info); + + uint64_t abs_now; + uint64_t cnt_now; + int before_diff, after_diff; + + update_kern(&abs_now, &cnt_now); + before_diff = (int)(to_ms(cnt_now) - to_ms(abs_now)); + + sleep(1); + + update_kern(&abs_now, &cnt_now); + after_diff = (int)(to_ms(cnt_now) - to_ms(abs_now)); + + T_ASSERT_LE(abs(after_diff - before_diff), 1, "mach_continuous_time_kernel and mach_absolute_time_kernel should not diverge"); +} +#endif + T_DECL(mct_sleep, "Testing mach_continuous_time behavior over system sleep"){ #ifndef MCT_SLEEP_TEST T_SKIP("Skipping test that sleeps the device; compile with MCT_SLEEP_TEST define to enable."); @@ -158,6 +248,34 @@ T_DECL(mct_settimeofday, "Testing mach_continuous_time behavior over settimeofda T_ASSERT_LT(abs(before - after), 1000, "mach_continuous_time should not jump more than 1s"); } +#ifdef HAS_KERNEL_TIME_TRAPS +T_DECL(mct_settimeofday_kern, "Testing kernel mach_continuous_time behavior over settimeofday"){ + if (geteuid() != 0){ + T_SKIP("The settimeofday() test requires root privileges to run."); + } + mach_timebase_info(&tb_info); + + struct timeval saved_tv; + struct timezone saved_tz; + int before, after; + + T_ASSERT_POSIX_ZERO(gettimeofday(&saved_tv, &saved_tz), NULL); + + struct timeval forward_tv = saved_tv; + // move time forward by two minutes, ensure mach_continuous_time keeps + // chugging along with mach_absolute_time + forward_tv.tv_sec += 2*60; + + before = (int)to_ms(mach_continuous_time_kernel()); + T_ASSERT_POSIX_ZERO(settimeofday(&forward_tv, &saved_tz), NULL); + + after = (int)to_ms(mach_continuous_time_kernel()); + T_ASSERT_POSIX_ZERO(settimeofday(&saved_tv, &saved_tz), NULL); + + T_ASSERT_LT(abs(before - after), 1000, "mach_continuous_time_kernel should not jump more than 1s"); +} +#endif + T_DECL(mct_aproximate, "Testing mach_continuous_approximate_time()", T_META_ALL_VALID_ARCHS(true)) { @@ -168,3 +286,82 @@ T_DECL(mct_aproximate, "Testing mach_continuous_approximate_time()", T_EXPECT_LE(llabs((long long)absolute - (long long)approximate), (long long)(25*NSEC_PER_MSEC), NULL); } + +T_DECL(mach_time_perf, "mach_time performance") { + { + dt_stat_time_t s = dt_stat_time_create("mach_absolute_time"); + T_STAT_MEASURE_LOOP(s) { + uint64_t t; + t = mach_absolute_time(); + } + dt_stat_finalize(s); + } + { + dt_stat_time_t s = dt_stat_time_create("mach_continuous_time"); + T_STAT_MEASURE_LOOP(s) { + uint64_t t; + t = mach_continuous_time(); + } + dt_stat_finalize(s); + } +} + +T_DECL(mach_time_perf_instructions, "instructions retired for mach_time", T_META_TYPE_PERF, T_META_ASROOT(YES)) { + { + dt_stat_thread_instructions_t s = dt_stat_thread_instructions_create("mach_absolute_time"); + T_STAT_MEASURE_LOOP(s) { + uint64_t t; + t = mach_absolute_time(); + } + dt_stat_finalize(s); + } + { + dt_stat_thread_instructions_t s = dt_stat_thread_instructions_create("mach_continuous_time"); + T_STAT_MEASURE_LOOP(s) { + uint64_t t; + t = mach_continuous_time(); + } + dt_stat_finalize(s); + } +} + +#ifdef HAS_KERNEL_TIME_TRAPS +T_DECL(mach_time_perf_kern, "kernel mach_time performance") { + { + dt_stat_time_t s = dt_stat_time_create("mach_absolute_time_kernel"); + T_STAT_MEASURE_LOOP(s) { + uint64_t t; + t = mach_absolute_time_kernel(); + } + dt_stat_finalize(s); + } + { + dt_stat_time_t s = dt_stat_time_create("mach_continuous_time_kernel"); + T_STAT_MEASURE_LOOP(s) { + uint64_t t; + t = mach_continuous_time_kernel(); + } + dt_stat_finalize(s); + } +} + +T_DECL(mach_time_perf_instructions_kern, "instructions retired for kernel mach_time", T_META_TYPE_PERF, T_META_ASROOT(YES)) { + { + dt_stat_thread_instructions_t s = dt_stat_thread_instructions_create("mach_absolute_time_kernel"); + T_STAT_MEASURE_LOOP(s) { + uint64_t t; + t = mach_absolute_time_kernel(); + } + dt_stat_finalize(s); + } + { + dt_stat_thread_instructions_t s = dt_stat_thread_instructions_create("mach_continuous_time_kernel"); + T_STAT_MEASURE_LOOP(s) { + uint64_t t; + t = mach_continuous_time_kernel(); + } + dt_stat_finalize(s); + } +} +#endif + diff --git a/tools/tests/darwintests/mach_port_deallocate_21692215.c b/tools/tests/darwintests/mach_port_deallocate_21692215.c new file mode 100644 index 000000000..4b84428f6 --- /dev/null +++ b/tools/tests/darwintests/mach_port_deallocate_21692215.c @@ -0,0 +1,38 @@ +#define T_NAMESPACE "xnu.ipc" +#include +#include +#include +#include + +#define NR_PORTS 4 + +T_DECL(mach_port_deallocate, "mach_port_deallocate deallocates also PORT_SET"){ + mach_port_t port_set; + mach_port_t port[NR_PORTS]; + int i,ret; + + ret= mach_port_allocate(mach_task_self(), MACH_PORT_RIGHT_PORT_SET, &port_set); + T_ASSERT_MACH_SUCCESS(ret, "mach_port_allocate MACH_PORT_RIGHT_PORT_SET"); + + for(i=0;i +#include +#include +#include + + +T_DECL(mach_port_mod_refs, "mach_port_mod_refs"){ + mach_port_t port_set; + mach_port_t port; + int ret; + + ret = mach_port_allocate(mach_task_self(), MACH_PORT_RIGHT_PORT_SET, &port_set); + T_ASSERT_MACH_SUCCESS(ret, "mach_port_allocate MACH_PORT_RIGHT_PORT_SET"); + + ret = mach_port_allocate(mach_task_self(), MACH_PORT_RIGHT_RECEIVE, &port); + T_ASSERT_MACH_SUCCESS(ret, "mach_port_allocate MACH_PORT_RIGHT_RECEIVE"); + + + /* + * Test all known variants of port rights on each type of port + */ + + /* can't subtract a send right if it doesn't exist */ + ret = mach_port_mod_refs(mach_task_self(), port, MACH_PORT_RIGHT_SEND, -1); + T_ASSERT_EQ(ret, KERN_INVALID_RIGHT, "mach_port_mod_refs SEND: -1 on a RECV right"); + + /* can't subtract a send once right if it doesn't exist */ + ret = mach_port_mod_refs(mach_task_self(), port, MACH_PORT_RIGHT_SEND_ONCE, -1); + T_ASSERT_EQ(ret, KERN_INVALID_RIGHT, "mach_port_mod_refs SEND_ONCE: -1 on a RECV right"); + + /* can't subtract a PORT SET right if it's not a port set */ + ret = mach_port_mod_refs(mach_task_self(), port, MACH_PORT_RIGHT_PORT_SET, -1); + T_ASSERT_EQ(ret, KERN_INVALID_RIGHT, "mach_port_mod_refs PORT_SET: -1 on a RECV right"); + + /* can't subtract a dead name right if it doesn't exist */ + ret = mach_port_mod_refs(mach_task_self(), port, MACH_PORT_RIGHT_DEAD_NAME, -1); + T_ASSERT_EQ(ret, KERN_INVALID_RIGHT, "mach_port_mod_refs DEAD_NAME: -1 on a RECV right"); + + /* can't subtract a LABELH right if it doesn't exist */ + ret = mach_port_mod_refs(mach_task_self(), port, MACH_PORT_RIGHT_LABELH, -1); + T_ASSERT_EQ(ret, KERN_INVALID_RIGHT, "mach_port_mod_refs LABELH: -1 on a RECV right"); + + /* can't subtract an invalid right-type */ + ret = mach_port_mod_refs(mach_task_self(), port, MACH_PORT_RIGHT_NUMBER, -1); + T_ASSERT_EQ(ret, KERN_INVALID_VALUE, "mach_port_mod_refs NUMBER: -1 on a RECV right"); + + /* can't subtract an invalid right-type */ + ret = mach_port_mod_refs(mach_task_self(), port, MACH_PORT_RIGHT_NUMBER + 1, -1); + T_ASSERT_EQ(ret, KERN_INVALID_VALUE, "mach_port_mod_refs NUMBER+1: -1 on a RECV right"); + + + /* can't subtract a send right if it doesn't exist */ + ret = mach_port_mod_refs(mach_task_self(), port_set, MACH_PORT_RIGHT_SEND, -1); + T_ASSERT_EQ(ret, KERN_INVALID_RIGHT, "mach_port_mod_refs SEND: -1 on a PORT_SET right"); + + /* can't subtract a send once right if it doesn't exist */ + ret = mach_port_mod_refs(mach_task_self(), port_set, MACH_PORT_RIGHT_SEND_ONCE, -1); + T_ASSERT_EQ(ret, KERN_INVALID_RIGHT, "mach_port_mod_refs SEND_ONCE: -1 on a PORT_SET right"); + + /* can't subtract a receive right if it's a port set */ + ret = mach_port_mod_refs(mach_task_self(), port_set, MACH_PORT_RIGHT_RECEIVE, -1); + T_ASSERT_EQ(ret, KERN_INVALID_RIGHT, "mach_port_mod_refs RECV: -1 on a PORT_SET right"); + + /* can't subtract a dead name right if it doesn't exist */ + ret = mach_port_mod_refs(mach_task_self(), port_set, MACH_PORT_RIGHT_DEAD_NAME, -1); + T_ASSERT_EQ(ret, KERN_INVALID_RIGHT, "mach_port_mod_refs DEAD_NAME: -1 on a PORT_SET right"); + + /* can't subtract a LABELH right if it doesn't exist */ + ret = mach_port_mod_refs(mach_task_self(), port_set, MACH_PORT_RIGHT_LABELH, -1); + T_ASSERT_EQ(ret, KERN_INVALID_RIGHT, "mach_port_mod_refs LABELH: -1 on a PORT_SET right"); + + /* can't subtract an invalid right-type */ + ret = mach_port_mod_refs(mach_task_self(), port_set, MACH_PORT_RIGHT_NUMBER, -1); + T_ASSERT_EQ(ret, KERN_INVALID_VALUE, "mach_port_mod_refs NUMBER: -1 on a PORT_SET right"); + + /* can't subtract an invalid right-type */ + ret = mach_port_mod_refs(mach_task_self(), port_set, MACH_PORT_RIGHT_NUMBER + 1, -1); + T_ASSERT_EQ(ret, KERN_INVALID_VALUE, "mach_port_mod_refs NUMBER+1: -1 on a PORT_SET right"); + + /* + * deallocate the ports/sets + */ + ret= mach_port_mod_refs(mach_task_self(), port_set, MACH_PORT_RIGHT_PORT_SET, -1); + T_ASSERT_MACH_SUCCESS(ret, "mach_port_mod_refs(PORT_SET, -1)"); + + ret= mach_port_mod_refs(mach_task_self(), port, MACH_PORT_RIGHT_RECEIVE, -1); + T_ASSERT_MACH_SUCCESS(ret, "mach_port_mod_refs(RECV_RIGHT, -1)"); +} diff --git a/tools/tests/darwintests/memorystatus_zone_test.c b/tools/tests/darwintests/memorystatus_zone_test.c new file mode 100644 index 000000000..1d0223a15 --- /dev/null +++ b/tools/tests/darwintests/memorystatus_zone_test.c @@ -0,0 +1,393 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif +#include +#include + +T_GLOBAL_META( + T_META_NAMESPACE("xnu.vm"), + T_META_CHECK_LEAKS(false) +); + +#define TIMEOUT_SECS 1500 + +#if TARGET_OS_EMBEDDED +#define ALLOCATION_SIZE_VM_REGION (16*1024) /* 16 KB */ +#define ALLOCATION_SIZE_VM_OBJECT ALLOCATION_SIZE_VM_REGION +#else +#define ALLOCATION_SIZE_VM_REGION (1024*1024*100) /* 100 MB */ +#define ALLOCATION_SIZE_VM_OBJECT (16*1024) /* 16 KB */ +#endif +#define MAX_CHILD_PROCS 100 + +#define ZONEMAP_JETSAM_LIMIT_SYSCTL "kern.zone_map_jetsam_limit=60" + +#define VME_ZONE_TEST_OPT "allocate_vm_regions" +#define VM_OBJECTS_ZONE_TEST_OPT "allocate_vm_objects" +#define GENERIC_ZONE_TEST_OPT "allocate_from_generic_zone" + +#define VM_TAG1 100 +#define VM_TAG2 101 + +enum { + VME_ZONE_TEST = 0, + VM_OBJECTS_ZONE_TEST, + GENERIC_ZONE_TEST, +}; + +static int current_test_index = 0; +static int num_children = 0; +static bool test_ending = false; +static bool within_dispatch_source_handler = false; +static dispatch_source_t ds_signal = NULL; +static ktrace_session_t session = NULL; + +static char testpath[PATH_MAX]; +static pid_t child_pids[MAX_CHILD_PROCS]; +static pthread_mutex_t test_ending_mtx; + +static void allocate_vm_regions(void); +static void allocate_vm_objects(void); +static void allocate_from_generic_zone(void); +static void cleanup_and_end_test(void); +static void setup_ktrace_session(void); +static void spawn_child_process(void); +static void run_test_for_zone(int index); + +extern void mach_zone_force_gc(host_t host); + +static void allocate_vm_regions(void) +{ + uint64_t alloc_size = ALLOCATION_SIZE_VM_REGION, i = 0; + + printf("[%d] Allocating VM regions, each of size %lld KB\n", getpid(), (alloc_size>>10)); + for (i = 0; ; i++) { + mach_vm_address_t addr = (mach_vm_address_t)NULL; + + /* Alternate VM tags between consecutive regions to prevent coalescing */ + int flags = VM_MAKE_TAG((i % 2)? VM_TAG1: VM_TAG2) | VM_FLAGS_ANYWHERE; + + if ((mach_vm_allocate(mach_task_self(), &addr, (mach_vm_size_t)alloc_size, flags)) != KERN_SUCCESS) { + break; + } + } + printf("[%d] Number of allocations: %lld\n", getpid(), i); + + /* Signal to the parent that we're done allocating */ + kill(getppid(), SIGUSR1); + + while (1) { + pause(); + } +} + +static void allocate_vm_objects(void) +{ + uint64_t alloc_size = ALLOCATION_SIZE_VM_OBJECT, i = 0; + + printf("[%d] Allocating VM regions, each of size %lld KB, each backed by a VM object\n", getpid(), (alloc_size>>10)); + for (i = 0; ; i++) { + mach_vm_address_t addr = (mach_vm_address_t)NULL; + + /* Alternate VM tags between consecutive regions to prevent coalescing */ + int flags = VM_MAKE_TAG((i % 2)? VM_TAG1: VM_TAG2) | VM_FLAGS_ANYWHERE; + + if ((mach_vm_allocate(mach_task_self(), &addr, (mach_vm_size_t)alloc_size, flags)) != KERN_SUCCESS) { + break; + } + /* Touch the region so the VM object can actually be created */ + *((int *)addr) = 0; + /* OK to free this page. Keeps us from holding a lot of dirty pages */ + madvise((void *)addr, (size_t)alloc_size, MADV_FREE); + } + printf("[%d] Number of allocations: %lld\n", getpid(), i); + + /* Signal to the parent that we're done allocating */ + kill(getppid(), SIGUSR1); + + while (1) { + pause(); + } +} + +static void allocate_from_generic_zone(void) +{ + uint64_t i = 0; + + printf("[%d] Allocating mach_ports\n", getpid()); + for (i = 0; ; i++) { + mach_port_t port; + + if ((mach_port_allocate(mach_task_self(), MACH_PORT_RIGHT_RECEIVE, &port)) != KERN_SUCCESS) { + break; + } + } + printf("[%d] Number of allocations: %lld\n", getpid(), i); + + /* Signal to the parent that we're done allocating */ + kill(getppid(), SIGUSR1); + + while (1) { + pause(); + } +} + +static void cleanup_and_end_test(void) +{ + int i; + + /* + * The atend handler executes on a different dispatch queue. + * We want to do the cleanup only once. + */ + pthread_mutex_lock(&test_ending_mtx); + if (test_ending) { + pthread_mutex_unlock(&test_ending_mtx); + return; + } + test_ending = true; + pthread_mutex_unlock(&test_ending_mtx); + + T_LOG("Number of processes spawned: %d", num_children); + T_LOG("Cleaning up..."); + + /* Disable signal handler that spawns child processes, only if we're not in the event handler's context */ + if (ds_signal != NULL && !within_dispatch_source_handler) { + dispatch_source_cancel_and_wait(ds_signal); + } + + /* Kill all the child processes that were spawned */ + for (i = 0; i < num_children; i++) { + kill(child_pids[i], SIGKILL); + } + for (i = 0; i < num_children; i++) { + int status = 0; + if (waitpid(child_pids[i], &status, 0) < 0) { + T_LOG("waitpid returned status %d", status); + } + } + sleep(1); + + /* Force zone_gc before starting test for another zone or exiting */ + mach_zone_force_gc(mach_host_self()); + + /* End ktrace session */ + if (session != NULL) { + ktrace_end(session, 1); + } +} + +static void setup_ktrace_session(void) +{ + int ret = 0; + + T_LOG("Setting up ktrace session..."); + session = ktrace_session_create(); + T_QUIET; T_ASSERT_NOTNULL(session, "ktrace_session_create"); + + ktrace_set_completion_handler(session, ^{ + ktrace_session_destroy(session); + T_END; + }); + + /* Listen for memorystatus_do_kill trace events */ + ret = ktrace_events_single(session, (BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DO_KILL)) | DBG_FUNC_END, ^(ktrace_event_t event) { + int i; + bool received_jetsam_event = false; + + /* We don't care about jetsams for any other reason except zone-map-exhaustion */ + if (event->arg2 == kMemorystatusKilledZoneMapExhaustion) { + T_LOG("[memorystatus_do_kill] jetsam reason: zone-map-exhaustion, pid: %lu", event->arg1); + if (current_test_index == VME_ZONE_TEST || current_test_index == VM_OBJECTS_ZONE_TEST) { + /* + * For the VM map entries zone we try to kill the leaking process. + * Verify that we jetsammed one of the processes we spawned. + */ + for (i = 0; i < num_children; i++) { + if (child_pids[i] == (pid_t)event->arg1) { + received_jetsam_event = true; + break; + } + } + } else { + received_jetsam_event = true; + } + + T_ASSERT_TRUE(received_jetsam_event, "Received jetsam event as expected"); + cleanup_and_end_test(); + } + }); + T_QUIET; T_ASSERT_POSIX_ZERO(ret, "ktrace_events_single"); + + ret = ktrace_start(session, dispatch_get_main_queue()); + T_QUIET; T_ASSERT_POSIX_ZERO(ret, "ktrace_start"); +} + +static void spawn_child_process(void) +{ + pid_t pid = -1; + char *launch_tool_args[4]; + within_dispatch_source_handler = true; + + T_QUIET; T_ASSERT_LT(num_children, MAX_CHILD_PROCS, "Spawned %d children. Timing out...", MAX_CHILD_PROCS); + + launch_tool_args[0] = testpath; + launch_tool_args[1] = "-n"; + launch_tool_args[3] = NULL; + + if (current_test_index == VME_ZONE_TEST) { + launch_tool_args[2] = VME_ZONE_TEST_OPT; + } else if (current_test_index == VM_OBJECTS_ZONE_TEST) { + launch_tool_args[2] = VM_OBJECTS_ZONE_TEST_OPT; + } else if (current_test_index == GENERIC_ZONE_TEST) { + launch_tool_args[2] = GENERIC_ZONE_TEST_OPT; + } + + /* Spawn the child process */ + int rc = dt_launch_tool(&pid, launch_tool_args, false, NULL, NULL); + if (rc != 0) { + T_LOG("dt_launch tool returned %d with error code %d", rc, errno); + } + T_QUIET; T_ASSERT_POSIX_SUCCESS(pid, "dt_launch_tool"); + + child_pids[num_children++] = pid; + within_dispatch_source_handler = false; +} + +static void run_test_for_zone(int index) +{ + int ret, dev; + size_t dev_size = sizeof(dev); + uint32_t testpath_buf_size = sizeof(testpath); + + T_ATEND(cleanup_and_end_test); + T_SETUPBEGIN; + + current_test_index = index; + + ret = sysctlbyname("kern.development", &dev, &dev_size, NULL, 0); + T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.development failed"); + if (dev == 0) { + T_SKIP("Skipping test on release kernel"); + } + + ret = _NSGetExecutablePath(testpath, &testpath_buf_size); + T_QUIET; T_ASSERT_POSIX_ZERO(ret, "_NSGetExecutablePath"); + T_LOG("Executable path: %s", testpath); + + /* + * If the timeout specified by T_META_TIMEOUT is hit, the atend handler does not get called. + * So we're queueing a dispatch block to fire after TIMEOUT_SECS seconds, so we can exit cleanly. + */ + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, TIMEOUT_SECS * NSEC_PER_SEC), dispatch_get_main_queue(), ^{ + T_ASSERT_FAIL("Timed out after %d seconds", TIMEOUT_SECS); + }); + + /* + * Create a dispatch source for the signal SIGUSR1. When a child is done allocating zone memory, it + * sends SIGUSR1 to the parent. Only then does the parent spawn another child. This prevents us from + * spawning many children at once and creating a lot of memory pressure. + */ + signal(SIGUSR1, SIG_IGN); + ds_signal = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue()); + T_QUIET; T_ASSERT_NOTNULL(ds_signal, "dispatch_source_create"); + + dispatch_source_set_event_handler(ds_signal, ^{ + /* Wait a few seconds before spawning another child. Keeps us from allocating too aggressively */ + sleep(5); + spawn_child_process(); + }); + dispatch_activate(ds_signal); + + /* Set up a ktrace session to listen for jetsam events */ + setup_ktrace_session(); + + T_SETUPEND; + + /* Spawn the first child process */ + T_LOG("Spawning child processes to allocate zone memory...\n\n"); + spawn_child_process(); + + dispatch_main(); +} + +T_HELPER_DECL(allocate_vm_regions, "allocates VM regions") +{ + allocate_vm_regions(); +} + +T_HELPER_DECL(allocate_vm_objects, "allocates VM objects and VM regions") +{ + allocate_vm_objects(); +} + +T_HELPER_DECL(allocate_from_generic_zone, "allocates from a generic zone") +{ + memorystatus_priority_properties_t props; + + /* + * We want to move the processes we spawn into the idle band, so that jetsam can target them first. + * This prevents other important BATS tasks from getting killed, specially in LTE where we have very few + * processes running. + */ + props.priority = JETSAM_PRIORITY_IDLE; + props.user_data = 0; + + if (memorystatus_control(MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES, getpid(), 0, &props, sizeof(props))) { + printf("memorystatus call to change jetsam priority failed\n"); + exit(-1); + } + + allocate_from_generic_zone(); +} + +/* + * T_META_SYSCTL_INT(ZONEMAP_JETSAM_LIMIT_SYSCTL) changes the zone_map_jetsam_limit to a + * lower value, so that the test can complete faster. + * The test allocates zone memory pretty aggressively which can cause the system to panic + * if the jetsam limit is quite high; a lower value keeps us from panicking. + */ +T_DECL( memorystatus_vme_zone_test, + "allocates elements from the VM map entries zone, verifies zone-map-exhaustion jetsams", + T_META_ASROOT(true), + T_META_TIMEOUT(1800), +/* T_META_LTEPHASE(LTE_POSTINIT), + */ + T_META_SYSCTL_INT(ZONEMAP_JETSAM_LIMIT_SYSCTL)) +{ + run_test_for_zone(VME_ZONE_TEST); +} + +T_DECL( memorystatus_vm_objects_zone_test, + "allocates elements from the VM objects and the VM map entries zones, verifies zone-map-exhaustion jetsams", + T_META_ASROOT(true), + T_META_TIMEOUT(1800), +/* T_META_LTEPHASE(LTE_POSTINIT), + */ + T_META_SYSCTL_INT(ZONEMAP_JETSAM_LIMIT_SYSCTL)) +{ + run_test_for_zone(VM_OBJECTS_ZONE_TEST); +} + +T_DECL( memorystatus_generic_zone_test, + "allocates elements from a zone that doesn't have an optimized jetsam path, verifies zone-map-exhaustion jetsams", + T_META_ASROOT(true), + T_META_TIMEOUT(1800), +/* T_META_LTEPHASE(LTE_POSTINIT), + */ + T_META_SYSCTL_INT(ZONEMAP_JETSAM_LIMIT_SYSCTL)) +{ + run_test_for_zone(GENERIC_ZONE_TEST); +} diff --git a/tools/tests/darwintests/monotonic_core.c b/tools/tests/darwintests/monotonic_core.c new file mode 100644 index 000000000..66bcc3185 --- /dev/null +++ b/tools/tests/darwintests/monotonic_core.c @@ -0,0 +1,236 @@ +/* + * Must come before including darwintest.h + */ +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif /* defined(T_NAMESPACE) */ + +#include +#include +#include +#ifndef PRIVATE +/* + * Need new CPU families. + */ +#define PRIVATE +#include +#undef PRIVATE +#else /* !defined(PRIVATE) */ +#include +#endif /* defined(PRIVATE) */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +T_GLOBAL_META( + T_META_NAMESPACE("xnu.monotonic"), + T_META_CHECK_LEAKS(false) +); + +static void +skip_if_unsupported(void) +{ + int r; + int supported = 0; + size_t supported_size = sizeof(supported); + + r = sysctlbyname("kern.monotonic.supported", &supported, &supported_size, + NULL, 0); + if (r < 0) { + T_WITH_ERRNO; + T_SKIP("could not find \"kern.monotonic.supported\" sysctl"); + } + + if (!supported) { + T_SKIP("monotonic is not supported on this platform"); + } +} + +static void +check_fixed_counts(uint64_t counts[2][2]) +{ + T_QUIET; + T_EXPECT_GT(counts[0][0], UINT64_C(0), "instructions are larger than 0"); + T_QUIET; + T_EXPECT_GT(counts[0][1], UINT64_C(0), "cycles are larger than 0"); + + T_EXPECT_GT(counts[1][0], counts[0][0], "instructions increase monotonically"); + T_EXPECT_GT(counts[1][1], counts[0][1], "cycles increase monotonically"); +} + +T_DECL(core_fixed_thread_self, "check the current thread's fixed counters", + T_META_ASROOT(true)) +{ + int err; + extern int thread_selfcounts(int type, void *buf, size_t nbytes); + uint64_t counts[2][2]; + + T_SETUPBEGIN; + skip_if_unsupported(); + T_SETUPEND; + + err = thread_selfcounts(1, &counts[0], sizeof(counts[0])); + T_ASSERT_POSIX_ZERO(err, "thread_selfcounts"); + err = thread_selfcounts(1, &counts[1], sizeof(counts[1])); + T_ASSERT_POSIX_ZERO(err, "thread_selfcounts"); + + check_fixed_counts(counts); +} + +T_DECL(core_fixed_task, "check that task counting is working", + T_META_ASROOT(true)) +{ + task_t task = mach_task_self(); + kern_return_t kr; + mach_msg_type_number_t size = TASK_INSPECT_BASIC_COUNTS_COUNT; + uint64_t counts[2][2]; + + skip_if_unsupported(); + + kr = task_inspect(task, TASK_INSPECT_BASIC_COUNTS, + (task_inspect_info_t)&counts[0], &size); + T_ASSERT_MACH_SUCCESS(kr, + "task_inspect(... TASK_INSPECT_BASIC_COUNTS ...)"); + + size = TASK_INSPECT_BASIC_COUNTS_COUNT; + kr = task_inspect(task, TASK_INSPECT_BASIC_COUNTS, + (task_inspect_info_t)&counts[1], &size); + T_ASSERT_MACH_SUCCESS(kr, + "task_inspect(... TASK_INSPECT_BASIC_COUNTS ...)"); + + check_fixed_counts(counts); +} + +T_DECL(core_fixed_kdebug, "check that the kdebug macros for monotonic work", + T_META_ASROOT(true)) +{ + __block bool saw_events = false; + ktrace_session_t s; + int r; + int set = 1; + + T_SETUPBEGIN; + skip_if_unsupported(); + + s = ktrace_session_create(); + T_QUIET; T_ASSERT_NOTNULL(s, "ktrace_session_create"); + + ktrace_events_single_paired(s, + KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_TMPCPU, 0x3fff), + ^(struct trace_point *start, struct trace_point *end) + { + uint64_t counts[2][2]; + + saw_events = true; + + counts[0][0] = start->arg1; + counts[0][1] = start->arg2; + counts[1][0] = end->arg1; + counts[1][1] = end->arg2; + + check_fixed_counts(counts); + }); + + ktrace_set_completion_handler(s, ^{ + T_ASSERT_TRUE(saw_events, "should see monotonic kdebug events"); + T_END; + }); + T_SETUPEND; + + T_ASSERT_POSIX_ZERO(ktrace_start(s, + dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0)), NULL); + + r = sysctlbyname("kern.monotonic.kdebug_test", NULL, NULL, &set, + sizeof(set)); + T_ASSERT_POSIX_SUCCESS(r, + "sysctlbyname(\"kern.monotonic.kdebug_test\", ...)"); + + ktrace_end(s, 0); + dispatch_main(); +} + +static void +perf_sysctl_deltas(const char *sysctl_name, const char *stat_name) +{ + uint64_t deltas[2]; + size_t deltas_size; + int r; + + T_SETUPBEGIN; + skip_if_unsupported(); + + dt_stat_t instrs = dt_stat_create("instructions", "%s_instrs", + stat_name); + dt_stat_t cycles = dt_stat_create("cycles", "%s_cycles", stat_name); + T_SETUPEND; + + while (!dt_stat_stable(instrs) || !dt_stat_stable(cycles)) { + deltas_size = sizeof(deltas); + r = sysctlbyname(sysctl_name, deltas, &deltas_size, NULL, 0); + T_QUIET; + T_ASSERT_POSIX_SUCCESS(r, "sysctlbyname(\"%s\", ...)", sysctl_name); + dt_stat_add(instrs, (double)deltas[0]); + dt_stat_add(cycles, (double)deltas[1]); + } + + dt_stat_finalize(instrs); + dt_stat_finalize(cycles); +} + +T_DECL(perf_core_fixed_cpu, "test the performance of fixed CPU counter access", + T_META_ASROOT(true)) +{ + perf_sysctl_deltas("kern.monotonic.fixed_cpu_perf", "fixed_cpu_counters"); +} + +T_DECL(perf_core_fixed_thread, "test the performance of fixed thread counter access", + T_META_ASROOT(true)) +{ + perf_sysctl_deltas("kern.monotonic.fixed_thread_perf", + "fixed_thread_counters"); +} + +T_DECL(perf_core_fixed_task, "test the performance of fixed task counter access", + T_META_ASROOT(true)) +{ + perf_sysctl_deltas("kern.monotonic.fixed_task_perf", "fixed_task_counters"); +} + +T_DECL(perf_core_fixed_thread_self, "test the performance of thread self counts") +{ + extern int thread_selfcounts(int type, void *buf, size_t nbytes); + uint64_t counts[2][2]; + + T_SETUPBEGIN; + dt_stat_t instrs = dt_stat_create("fixed_thread_self_instrs", "instructions"); + dt_stat_t cycles = dt_stat_create("fixed_thread_self_cycles", "cycles"); + + skip_if_unsupported(); + T_SETUPEND; + + while (!dt_stat_stable(instrs) || !dt_stat_stable(cycles)) { + int r1, r2; + + r1 = thread_selfcounts(1, &counts[0], sizeof(counts[0])); + r2 = thread_selfcounts(1, &counts[1], sizeof(counts[1])); + T_QUIET; T_ASSERT_POSIX_ZERO(r1, "__thread_selfcounts"); + T_QUIET; T_ASSERT_POSIX_ZERO(r2, "__thread_selfcounts"); + + T_QUIET; T_ASSERT_GT(counts[1][0], counts[0][0], + "instructions increase monotonically"); + dt_stat_add(instrs, counts[1][0] - counts[0][0]); + + T_QUIET; T_ASSERT_GT(counts[1][1], counts[0][1], + "cycles increase monotonically"); + dt_stat_add(cycles, counts[1][1] - counts[0][1]); + } + + dt_stat_finalize(instrs); + dt_stat_finalize(cycles); +} diff --git a/tools/tests/darwintests/netbsd_utimensat.c b/tools/tests/darwintests/netbsd_utimensat.c new file mode 100644 index 000000000..87f0657d1 --- /dev/null +++ b/tools/tests/darwintests/netbsd_utimensat.c @@ -0,0 +1,191 @@ +/* $NetBSD: t_utimensat.c,v 1.6 2017/01/10 15:13:56 christos Exp $ */ + +/*- + * Copyright (c) 2012 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Emmanuel Dreyfus. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +__RCSID("$NetBSD: t_utimensat.c,v 1.6 2017/01/10 15:13:56 christos Exp $"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define DIR "dir" +#define FILE "dir/utimensat" +#define BASEFILE "utimensat" +#define LINK "dir/symlink" +#define BASELINK "symlink" +#define FILEERR "dir/symlink" + +static const struct timespec tptr[] = { + { 0x12345678, 987654321 }, + { 0x15263748, 123456789 }, +}; + +static void chtmpdir(void) +{ + T_SETUPBEGIN; + T_ASSERT_POSIX_ZERO(chdir(dt_tmpdir()), NULL); + + // dt_tmpdir() should guarantee a clean directory for each run + unlink(FILE); + unlink(LINK); + rmdir(DIR); + + T_SETUPEND; +} + +T_DECL(netbsd_utimensat_fd, "See that utimensat works with fd") +{ + chtmpdir(); + + int dfd; + int fd; + struct stat st; + + T_ASSERT_POSIX_ZERO(mkdir(DIR, 0755), NULL); + T_ASSERT_POSIX_SUCCESS((fd = open(FILE, O_CREAT|O_RDWR, 0644)), NULL); + T_ASSERT_POSIX_ZERO(close(fd), NULL); + + T_ASSERT_POSIX_SUCCESS((dfd = open(DIR, O_RDONLY, 0)), NULL); + T_ASSERT_POSIX_ZERO(utimensat(dfd, BASEFILE, tptr, 0), NULL); + T_ASSERT_POSIX_ZERO(close(dfd), NULL); + + T_ASSERT_POSIX_ZERO(stat(FILE, &st), NULL); + T_ASSERT_EQ(st.st_atimespec.tv_sec, tptr[0].tv_sec, NULL); + T_ASSERT_EQ(st.st_atimespec.tv_nsec, tptr[0].tv_nsec, NULL); + T_ASSERT_EQ(st.st_mtimespec.tv_sec, tptr[1].tv_sec, NULL); + T_ASSERT_EQ(st.st_mtimespec.tv_nsec, tptr[1].tv_nsec, NULL); +} + +T_DECL(netbsd_utimensat_fdcwd, "See that utimensat works with fd as AT_FDCWD") +{ + chtmpdir(); + + int fd; + struct stat st; + + T_ASSERT_POSIX_ZERO(mkdir(DIR, 0755), NULL); + T_ASSERT_POSIX_SUCCESS((fd = open(FILE, O_CREAT|O_RDWR, 0644)), NULL); + T_ASSERT_POSIX_ZERO(close(fd), NULL); + + T_ASSERT_POSIX_ZERO(chdir(DIR), NULL); + T_ASSERT_POSIX_ZERO(utimensat(AT_FDCWD, BASEFILE, tptr, 0), NULL); + + T_ASSERT_POSIX_ZERO(stat(BASEFILE, &st), NULL); + T_ASSERT_EQ(st.st_atimespec.tv_sec, tptr[0].tv_sec, NULL); + T_ASSERT_EQ(st.st_atimespec.tv_nsec, tptr[0].tv_nsec, NULL); + T_ASSERT_EQ(st.st_mtimespec.tv_sec, tptr[1].tv_sec, NULL); + T_ASSERT_EQ(st.st_mtimespec.tv_nsec, tptr[1].tv_nsec, NULL); +} + +T_DECL(netbsd_utimensat_fdcwderr, "See that utimensat fails with fd as AT_FDCWD and bad path") +{ + chtmpdir(); + + T_ASSERT_POSIX_ZERO(mkdir(DIR, 0755), NULL); + T_ASSERT_EQ(utimensat(AT_FDCWD, FILEERR, tptr, 0), -1, NULL); +} + +T_DECL(netbsd_utimensat_fderr1, "See that utimensat fail with bad path") +{ + chtmpdir(); + + int dfd; + + T_ASSERT_POSIX_ZERO(mkdir(DIR, 0755), NULL); + T_ASSERT_POSIX_SUCCESS((dfd = open(DIR, O_RDONLY, 0)), NULL); + T_ASSERT_EQ(utimensat(dfd, FILEERR, tptr, 0), -1, NULL); + T_ASSERT_POSIX_ZERO(close(dfd), NULL); +} + +T_DECL(netbsd_utimensat_fderr2, "See that utimensat fails with bad fdat") +{ + chtmpdir(); + + int dfd; + int fd; + char cwd[MAXPATHLEN]; + + T_ASSERT_POSIX_ZERO(mkdir(DIR, 0755), NULL); + T_ASSERT_POSIX_SUCCESS((fd = open(FILE, O_CREAT|O_RDWR, 0644)), NULL); + T_ASSERT_POSIX_ZERO(close(fd), NULL); + + T_ASSERT_POSIX_SUCCESS((dfd = open(getcwd(cwd, MAXPATHLEN), O_RDONLY, 0)), NULL); + T_ASSERT_EQ(utimensat(dfd, BASEFILE, tptr, 0), -1, NULL); + T_ASSERT_POSIX_ZERO(close(dfd), NULL); +} + +T_DECL(netbsd_utimensat_fderr3, "See that utimensat fails with fd as -1") +{ + chtmpdir(); + + int fd; + + T_ASSERT_POSIX_ZERO(mkdir(DIR, 0755), NULL); + T_ASSERT_POSIX_SUCCESS((fd = open(FILE, O_CREAT|O_RDWR, 0644)), NULL); + T_ASSERT_POSIX_ZERO(close(fd), NULL); + + T_ASSERT_EQ(utimensat(-1, FILE, tptr, 0), -1, NULL); +} + +T_DECL(netbsd_utimensat_fdlink, "See that utimensat works on symlink") +{ + chtmpdir(); + + int dfd; + struct stat st; + + T_ASSERT_POSIX_ZERO(mkdir(DIR, 0755), NULL); + T_ASSERT_POSIX_ZERO(symlink(FILE, LINK), NULL); /* NB: FILE does not exists */ + + T_ASSERT_POSIX_SUCCESS((dfd = open(DIR, O_RDONLY, 0)), NULL); + + T_ASSERT_EQ(utimensat(dfd, BASELINK, tptr, 0), -1, NULL); + T_ASSERT_EQ(errno, ENOENT, NULL); + + T_ASSERT_POSIX_ZERO(utimensat(dfd, BASELINK, tptr, AT_SYMLINK_NOFOLLOW), NULL); + + T_ASSERT_POSIX_ZERO(close(dfd), NULL); + + T_ASSERT_POSIX_ZERO(lstat(LINK, &st), NULL); + T_ASSERT_EQ(st.st_atimespec.tv_sec, tptr[0].tv_sec, NULL); + T_ASSERT_EQ(st.st_atimespec.tv_nsec, tptr[0].tv_nsec, NULL); + T_ASSERT_EQ(st.st_mtimespec.tv_sec, tptr[1].tv_sec, NULL); + T_ASSERT_EQ(st.st_mtimespec.tv_nsec, tptr[1].tv_nsec, NULL); +} diff --git a/tools/tests/darwintests/ntp_adjtime_29192647.c b/tools/tests/darwintests/ntp_adjtime_29192647.c new file mode 100644 index 000000000..28663859e --- /dev/null +++ b/tools/tests/darwintests/ntp_adjtime_29192647.c @@ -0,0 +1,371 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define DAY 86400 /*1 day in sec*/ +#define ERROR 2 /*2 us of error tolerance*/ + +T_DECL(settimeofday_29192647, + "Verify that the syscall settimeofday is effective", + T_META_ASROOT(true), T_META_CHECK_LEAKS(NO), T_META_LTEPHASE(LTE_POSTINIT)) +{ + struct timeval time; + long new_time; + + if (geteuid() != 0){ + T_SKIP("settimeofday_29192647 test requires root privileges to run."); + } + + T_QUIET; + T_ASSERT_POSIX_ZERO(gettimeofday(&time, NULL), NULL); + + /* increment the time of one day */ + new_time = time.tv_sec + DAY; + + time.tv_sec = new_time; + time.tv_usec = 0; + + T_LOG("Attemping to set the time one day after."); + + T_WITH_ERRNO; + T_ASSERT_POSIX_ZERO(settimeofday(&time, NULL), NULL); + + T_QUIET; + T_ASSERT_POSIX_ZERO(gettimeofday(&time, NULL), NULL); + + /* expext to be past new_time */ + T_EXPECT_GE_LONG(time.tv_sec, new_time, "Time successfully changed"); + + /* set the time back to previous value */ + if (time.tv_sec >= new_time) { + time.tv_sec = time.tv_sec - DAY; + time.tv_usec = 0; + + T_WITH_ERRNO; + T_ASSERT_POSIX_ZERO(settimeofday(&time, NULL), NULL); + } +} + +static void get_abs_to_us_scale_factor(uint64_t* numer, uint64_t* denom){ + struct timespec time; + uint64_t old_abstime, new_abstime; + uint64_t old_time_usec, new_time_usec; + uint64_t time_conv1, diff; + mach_timebase_info_data_t timebaseInfo = { 0, 0 }; + + T_QUIET; T_ASSERT_EQ(mach_get_times(&old_abstime, NULL, &time), KERN_SUCCESS, NULL); + + old_time_usec = (uint64_t)time.tv_sec * USEC_PER_SEC + (uint64_t)time.tv_nsec/1000; + + sleep(1); + + T_QUIET; T_ASSERT_EQ(mach_get_times(&new_abstime, NULL, &time), KERN_SUCCESS, NULL); + + new_time_usec = (uint64_t)time.tv_sec * USEC_PER_SEC + (uint64_t)time.tv_nsec/1000; + + /* this is conversion factors from abs to nanos */ + T_ASSERT_EQ(mach_timebase_info(&timebaseInfo), KERN_SUCCESS, NULL); + + new_time_usec -= old_time_usec; + new_abstime -= old_abstime; + + time_conv1 = new_abstime; + time_conv1 *= timebaseInfo.numer; + time_conv1 /= timebaseInfo.denom * 1000; + + if (time_conv1 > new_time_usec) + diff = time_conv1 - new_time_usec; + else + diff = new_time_usec - time_conv1; + + T_EXPECT_LE_ULLONG(diff, (unsigned long long)ERROR, "Check scale factor time base (%u/%u) delta read usec %llu delta converted %llu delta abs %llu", timebaseInfo.numer, timebaseInfo.denom, time_conv1, new_time_usec, new_abstime); + + *numer = (uint64_t)timebaseInfo.numer; + *denom = (uint64_t)timebaseInfo.denom * 1000; +} + + +#define ADJSTMENT 3333 /*3333 us*/ +#define ADJTIME_OFFSET_PER_SEC 500 + +T_DECL(adjtime_29192647, + "Verify that the syscall adjtime is effective", + T_META_CHECK_LEAKS(NO), T_META_LTEPHASE(LTE_POSTINIT), T_META_ASROOT(true)) +{ + struct timespec time; + struct timeval adj; + uint64_t old_abstime, new_abstime, abs_delta; + uint64_t old_time_usec, new_time_usec, us_delta, num, den; + unsigned int sleep_time; + long diff; + const char * lterdos_env = NULL; + +#if defined(__i386__) || defined(__x86_64__) + T_SKIP("adjtime_29192647 test requires LTE to run."); +#endif + + if (geteuid() != 0) { + T_SKIP("adjtime_29192647 test requires root privileges to run."); + } + + lterdos_env = getenv("LTERDOS"); + + if (lterdos_env != NULL){ + if (!(strcmp(lterdos_env, "YES") == 0)) { + T_SKIP("adjtime_29192647 test requires LTE to run."); + } + } + else { + T_SKIP("adjtime_29192647 test requires LTE to run."); + } + + /* + * Calibrate scale factor for converting from abs time to usec + */ + get_abs_to_us_scale_factor(&num, &den); + + T_QUIET; T_ASSERT_EQ(mach_get_times(&old_abstime, NULL, &time), KERN_SUCCESS, NULL); + + old_time_usec = (uint64_t)time.tv_sec * USEC_PER_SEC + (uint64_t)time.tv_nsec/1000; + + adj.tv_sec = 0; + adj.tv_usec = ADJSTMENT; + + T_LOG("Attemping to adjust the time of %d", ADJSTMENT); + + /* + * If more than one second of adjustment + * the system slews at a rate of 5ms/s otherwise 500us/s + * until the last second is slewed the final < 500 usecs. + */ + T_WITH_ERRNO; + T_ASSERT_POSIX_ZERO(adjtime(&adj, NULL),NULL); + + /* + * Wait that the full adjustment is applied. + * Note, add 2 more secs for take into account division error + * and that the last block of adj is fully elapsed. + */ + sleep_time = (ADJSTMENT)/(ADJTIME_OFFSET_PER_SEC)+2; + + T_LOG("Waiting for %u sec\n", sleep_time); + sleep(sleep_time); + + T_QUIET; T_ASSERT_EQ(mach_get_times(&new_abstime, NULL, &time), KERN_SUCCESS, NULL); + + new_time_usec = (uint64_t)time.tv_sec * USEC_PER_SEC + (uint64_t)time.tv_nsec/1000; + + us_delta = new_time_usec - old_time_usec; + us_delta -= ADJSTMENT; + + /* abs time is not affected by adjtime */ + abs_delta = new_abstime - old_abstime; + + abs_delta *= num; + abs_delta /= den; + + diff = (long) us_delta - (long) abs_delta; + + /* expext that us_delta == abs_delta */ + T_EXPECT_LE_LONG(diff, (long) ERROR, "Check abs time vs calendar time"); + + T_EXPECT_GE_LONG(diff, (long) -ERROR, "Check abs time vs calendar time"); + +} + +#define FREQ_PPM 222 /*222 PPM(us/s)*/ +#define SHIFT_PLL 4 +#define OFFSET_US 123 /*123us*/ + +T_DECL(ntp_adjtime_29192647, + "Verify that the syscall ntp_adjtime is effective", + T_META_CHECK_LEAKS(NO), T_META_LTEPHASE(LTE_POSTINIT), T_META_ASROOT(true)) +{ + struct timespec time; + struct timex ntptime; + uint64_t abstime1, abstime2, abs_delta, num, den, time_delta; + uint64_t time1_usec, time2_usec, time_conv, us_delta, app; + int64_t offset; + long diff, freq; + unsigned int sleep_time; + const char * lterdos_env = NULL; + +#if defined(__i386__) || defined(__x86_64__) + T_SKIP("ntp_adjtime_29192647 test requires LTE to run."); +#endif + + if (geteuid() != 0){ + T_SKIP("ntp_adjtime_29192647 test requires root privileges to run."); + } + + lterdos_env = getenv("LTERDOS"); + + if (lterdos_env != NULL){ + if (!(strcmp(lterdos_env, "YES") == 0)) { + T_SKIP("adjtime_29192647 test requires LTE to run."); + } + } + else { + T_SKIP("adjtime_29192647 test requires LTE to run."); + } + + /* + * Calibrate scale factor for converting from abs time to usec + */ + get_abs_to_us_scale_factor(&num, &den); + + /* + * scale frequency using ntp_adjtime; + */ + memset(&ntptime, 0, sizeof(ntptime)); + + ntptime.modes = MOD_STATUS; + ntptime.status = TIME_OK; + /* ntp input freq is in ppm (us/s) * 2^16, max freq is 500 ppm */ + freq = (FREQ_PPM) * 65536; + ntptime.modes |= MOD_FREQUENCY; + ntptime.freq = freq; + + T_LOG("Attemping to change calendar frequency of %d ppm", FREQ_PPM); + + T_WITH_ERRNO; + T_ASSERT_EQ(ntp_adjtime(&ntptime), TIME_OK, NULL); + + T_WITH_ERRNO; + T_ASSERT_EQ(ntptime.freq, freq, NULL); + + sleep(2); + + T_QUIET; T_ASSERT_EQ(mach_get_times(&abstime1, NULL, &time), KERN_SUCCESS, NULL); + + time1_usec = (uint64_t)time.tv_sec * USEC_PER_SEC + (uint64_t)time.tv_nsec/1000; + + sleep(1); + + T_QUIET; T_ASSERT_EQ(mach_get_times(&abstime2, NULL, &time), KERN_SUCCESS, NULL); + + time2_usec = (uint64_t)time.tv_sec * USEC_PER_SEC + (uint64_t)time.tv_nsec/1000; + + abs_delta = abstime2 - abstime1; + us_delta = time2_usec - time1_usec; + + time_conv = abs_delta; + time_conv *= num; + time_conv /= den; + + app = time_conv/USEC_PER_SEC; //sec elapsed + + time_delta = time_conv; + time_delta += app * (FREQ_PPM); + + app = time_conv%USEC_PER_SEC; + + time_delta += (app*(FREQ_PPM))/USEC_PER_SEC; + + diff = (long) us_delta - (long) time_delta; + + /* expext that us_delta == time_delta */ + T_EXPECT_LE_LONG(diff, (long) ERROR, "Check abs time vs calendar time"); + + T_EXPECT_GE_LONG(diff, (long) -ERROR, "Check abs time vs calendar time"); + + memset(&ntptime, 0, sizeof(ntptime)); + + /* reset freq to zero */ + freq = 0; + ntptime.modes = MOD_STATUS; + ntptime.status = TIME_OK; + ntptime.modes |= MOD_FREQUENCY; + ntptime.freq = freq; + + T_WITH_ERRNO; + T_ASSERT_EQ(ntp_adjtime(&ntptime), TIME_OK, NULL); + + T_WITH_ERRNO; + T_ASSERT_EQ(ntptime.freq, freq, NULL); + + sleep(1); + + /* + * adjust the phase using ntp_adjtime; + */ + memset(&ntptime, 0, sizeof(ntptime)); + ntptime.modes |= MOD_STATUS; + ntptime.status = TIME_OK; + ntptime.status |= STA_PLL|STA_FREQHOLD; + + /* ntp input phase can be both ns or us (MOD_MICRO), max offset is 500 ms */ + ntptime.offset = OFFSET_US; + ntptime.modes |= MOD_OFFSET|MOD_MICRO; + + /* + * The system will slew each sec of: + * slew = ntp.offset >> (SHIFT_PLL + time_constant); + * ntp.offset -= slew; + */ + offset= (OFFSET_US) * 1000; + sleep_time = 2; + + while((offset>>SHIFT_PLL)>0){ + offset -= offset >> SHIFT_PLL; + sleep_time++; + } + + T_QUIET; T_ASSERT_EQ(mach_get_times(&abstime1, NULL, &time), KERN_SUCCESS, NULL); + + time1_usec = (uint64_t)time.tv_sec * USEC_PER_SEC + (uint64_t)time.tv_nsec/1000; + + T_LOG("Attemping to change calendar phase of %d us", OFFSET_US); + + T_WITH_ERRNO; + T_ASSERT_EQ(ntp_adjtime(&ntptime), TIME_OK, NULL); + + T_WITH_ERRNO; + T_ASSERT_EQ(ntptime.offset, (long) OFFSET_US, NULL); + + T_LOG("Waiting for %u sec\n", sleep_time); + sleep(sleep_time); + + T_QUIET; T_ASSERT_EQ(mach_get_times(&abstime2, NULL, &time), KERN_SUCCESS, NULL); + + time2_usec = (uint64_t)time.tv_sec * USEC_PER_SEC + (uint64_t)time.tv_nsec/1000; + + abs_delta = abstime2 - abstime1; + us_delta = time2_usec - time1_usec; + + abs_delta *= num; + abs_delta /= den; + + us_delta -= OFFSET_US; + + diff = (long) us_delta - (long) abs_delta; + + /* expext that us_delta == abs_delta */ + T_EXPECT_LE_LONG(diff, (long) ERROR, "Check abs time vs calendar time"); + + T_EXPECT_GE_LONG(diff, (long) -ERROR, "Check abs time vs calendar time"); + + memset(&ntptime, 0, sizeof(ntptime)); + ntptime.modes = MOD_STATUS; + ntptime.status = TIME_OK; + ntptime.modes |= MOD_FREQUENCY; + ntptime.freq = 0; + + ntptime.status |= STA_PLL; + ntptime.offset = 0; + ntptime.modes |= MOD_OFFSET; + + T_WITH_ERRNO; + T_ASSERT_EQ(ntp_adjtime(&ntptime), TIME_OK, NULL); + +} + + diff --git a/tools/tests/darwintests/perf_compressor.c b/tools/tests/darwintests/perf_compressor.c index b404a97c7..b0c6fa112 100644 --- a/tools/tests/darwintests/perf_compressor.c +++ b/tools/tests/darwintests/perf_compressor.c @@ -50,17 +50,12 @@ void allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize) { } void allocate_random_pages(char **buf, int num_pages, int vmpgsize) { - int fd, i; - - fd = open("/dev/random", O_RDONLY); - T_QUIET; T_ASSERT_POSIX_SUCCESS(fd, "open /dev/random failed [%s]\n", strerror(errno)); + int i; for (i = 0; i < num_pages; i++) { buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char)); - T_QUIET; T_ASSERT_POSIX_SUCCESS(read(fd, buf[i], (size_t)vmpgsize), - "read from /dev/random failed [%s]\n", strerror(errno)); + arc4random_buf((void*)buf[i], (size_t)vmpgsize); } - close(fd); } // Gives us the compression ratio we see in the typical case (~2.7) diff --git a/tools/tests/darwintests/perf_exit.c b/tools/tests/darwintests/perf_exit.c index 052b942c3..0caafdad5 100644 --- a/tools/tests/darwintests/perf_exit.c +++ b/tools/tests/darwintests/perf_exit.c @@ -4,10 +4,11 @@ #include #include -#include +#include #include #include #include +#include T_GLOBAL_META( T_META_NAMESPACE("xnu.perf.exit"), @@ -27,69 +28,72 @@ T_GLOBAL_META( #define EXIT_BINARY "perf_exit_proc" #define EXIT_BINARY_PATH "./" EXIT_BINARY +static ktrace_session_t session; +static dispatch_queue_t spawn_queue; +static uint64_t *begin_ts; +static dt_stat_time_t s; +static bool started_tracing = false; + void run_exit_test(int proc_wired_mem, int thread_priority, int nthreads); +static void cleanup(void) { + free(begin_ts); + dt_stat_finalize(s); + dispatch_release(spawn_queue); + if (started_tracing) { + ktrace_end(session, 1); + } +} + void run_exit_test(int proc_wired_mem, int thread_priority, int nthreads) { - _Atomic static int ended = 0; - dispatch_queue_t spawn_queue; + static atomic_bool ended = false; - dt_stat_time_t s = dt_stat_time_create("time"); + s = dt_stat_time_create("time"); + T_QUIET; T_ASSERT_NOTNULL(s, "created time statistic"); - uint64_t *begin_ts = malloc(sizeof(uint64_t) * PID_MAX); - if (begin_ts == NULL) { - T_FAIL("Error allocating timestamp array"); - } + begin_ts = malloc(sizeof(uint64_t) * PID_MAX); + T_QUIET; T_ASSERT_NOTNULL(begin_ts, "created pid array"); + + T_ATEND(cleanup); - ktrace_session_t session; session = ktrace_session_create(); - if (session == NULL) { - T_FAIL("Error creating ktrace session"); - } + T_QUIET; T_ASSERT_NOTNULL(session, "created a trace session"); spawn_queue = dispatch_queue_create("spawn_queue", NULL); ktrace_set_completion_handler(session, ^{ - free(begin_ts); - dt_stat_finalize(s); - dispatch_release(spawn_queue); + ktrace_session_destroy(session); T_END; }); ktrace_set_signal_handler(session); + ktrace_set_execnames_enabled(session, KTRACE_FEATURE_ENABLED); - // We are only interested by the process we launched + // We are only interested in the process we launched ktrace_filter_process(session, EXIT_BINARY); ktrace_events_single(session, (BSDDBG_CODE(DBG_BSD_EXCP_SC, 1) | DBG_FUNC_START), ^(ktrace_event_t e) { - pid_t pid = ktrace_get_pid_for_thread(session, e->threadid); - if (pid > PID_MAX) { - T_FAIL("Invalid pid returned by ktrace_get_pid_for_thread: %d\n", pid); - } - begin_ts[pid] = e->timestamp; - + T_QUIET; T_ASSERT_LE(e->pid, PID_MAX, "valid pid for tracepoint"); + begin_ts[e->pid] = e->timestamp; }); ktrace_events_single(session, (BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXIT) | DBG_FUNC_END), ^(ktrace_event_t e) { - pid_t pid = ktrace_get_pid_for_thread(session, e->threadid); - if (pid > PID_MAX) { - T_FAIL("Invalid pid returned by ktrace_get_pid_for_thread: %d\n", pid); - } - if (begin_ts[pid] == 0) { + T_QUIET; T_ASSERT_LE(e->pid, PID_MAX, "valid pid for tracepoint"); + + if (begin_ts[e->pid] == 0) { return; } - uint64_t delta = e->timestamp - begin_ts[pid]; - if (!dt_stat_stable(s)) { - dt_stat_mach_time_add(s, delta); - } - else { - ended = 1; + T_QUIET; T_ASSERT_LE(begin_ts[e->pid], e->timestamp, "timestamps are monotonically increasing"); + dt_stat_mach_time_add(s, e->timestamp - begin_ts[e->pid]); + + if (dt_stat_stable(s)) { + ended = true; ktrace_end(session, 1); } }); int ret = ktrace_start(session, dispatch_get_main_queue()); - if (ret != 0) { - T_FAIL("Error starting ktrace"); - } + T_ASSERT_POSIX_ZERO(ret, "starting trace"); + started_tracing = true; // Spawn processes continuously until the test is over dispatch_async(spawn_queue, ^(void) { @@ -103,13 +107,14 @@ void run_exit_test(int proc_wired_mem, int thread_priority, int nthreads) { int status; while (!ended) { pid_t pid; - int err = posix_spawn(&pid, args[0], NULL, NULL, args, NULL); - if (err) - T_FAIL("posix_spawn returned %d", err); + int bret = posix_spawn(&pid, args[0], NULL, NULL, args, NULL); + T_QUIET; T_ASSERT_POSIX_ZERO(bret, "spawned process '%s'", args[0]); + + bret = waitpid(pid, &status, 0); + T_QUIET; T_ASSERT_POSIX_SUCCESS(bret, "waited for process %d\n", pid); - waitpid(pid, &status, 0); if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) - T_FAIL("Child process of posix_spawn failed to run"); + T_ASSERT_FAIL("child process failed to run"); // Avoid saturating the CPU with new processes usleep(1); @@ -140,7 +145,6 @@ T_DECL(exit_10_threads, "exit(2) time with 10 threads") { run_exit_test(0, BASEPRI_FOREGROUND, 10); } - T_DECL(exit_1mb, "exit(2) time with 1MB of wired memory") { run_exit_test(10000000, BASEPRI_FOREGROUND, 0); } @@ -149,17 +153,14 @@ T_DECL(exit_10mb, "exit(2) time with 10MB of wired memory") { run_exit_test(10000000, BASEPRI_FOREGROUND, 0); } -/* -T_DECL(exit_100_threads, "exit(2) time with 100 threads", T_META_TIMEOUT(1800)) { +T_DECL(exit_100_threads, "exit(2) time with 100 threads", T_META_ENABLED(false), T_META_TIMEOUT(1800)) { run_exit_test(0, BASEPRI_FOREGROUND, 100); } -T_DECL(exit_1000_threads, "exit(2) time with 1000 threads", T_META_TIMEOUT(1800)) { +T_DECL(exit_1000_threads, "exit(2) time with 1000 threads", T_META_ENABLED(false), T_META_TIMEOUT(1800)) { run_exit_test(0, BASEPRI_FOREGROUND, 1000); } -T_DECL(exit_100mb, "exit(2) time with 100MB of wired memory", T_META_TIMEOUT(1800)) { +T_DECL(exit_100mb, "exit(2) time with 100MB of wired memory", T_META_ENABLED(false), T_META_TIMEOUT(1800)) { run_exit_test(100000000, BASEPRI_FOREGROUND, 0); } -*/ - diff --git a/tools/tests/darwintests/perf_kdebug.c b/tools/tests/darwintests/perf_kdebug.c index 1dc98d802..f0f058fbd 100644 --- a/tools/tests/darwintests/perf_kdebug.c +++ b/tools/tests/darwintests/perf_kdebug.c @@ -117,7 +117,12 @@ static void loop_getppid(dt_stat_time_t s) { } while (!dt_stat_stable(s)); } +static void reset_kdebug_trace(void) { + _sysctl_reset(); +} + static void test(const char* test_name, void (^pretest_setup)(void), void (*test)(dt_stat_time_t s)) { + T_ATEND(reset_kdebug_trace); _sysctl_reset(); _sysctl_setbuf(1000000); _sysctl_nowrap(false); @@ -129,7 +134,6 @@ static void test(const char* test_name, void (^pretest_setup)(void), void (*test test(s); - _sysctl_reset(); dt_stat_finalize(s); } diff --git a/tools/tests/darwintests/poll_select_kevent_paired_fds.c b/tools/tests/darwintests/poll_select_kevent_paired_fds.c index 732e00a12..169c698c7 100644 --- a/tools/tests/darwintests/poll_select_kevent_paired_fds.c +++ b/tools/tests/darwintests/poll_select_kevent_paired_fds.c @@ -1,11 +1,14 @@ #ifdef T_NAMESPACE #undef T_NAMESPACE #endif + #include +#include #include #include #include +#include #include #include #include @@ -26,7 +29,10 @@ #include #include /* kevent_qos */ -T_GLOBAL_META(T_META_NAMESPACE("xnu.poll_select_kevent_paired_fds")); +T_GLOBAL_META( + T_META_NAMESPACE("xnu.kevent"), + T_META_CHECK_LEAKS(false), + T_META_LTEPHASE(LTE_POSTINIT)); /* * Test to validate that monitoring a PTY device, FIFO, pipe, or socket pair in @@ -48,9 +54,13 @@ T_GLOBAL_META(T_META_NAMESPACE("xnu.poll_select_kevent_paired_fds")); #define READ_SETUP_TIMEOUT_SECS 2 #define WRITE_TIMEOUT_SECS 4 -#define READ_TIMEOUT_SECS 2 +#define READ_TIMEOUT_SECS 4 #define INCREMENTAL_WRITE_SLEEP_USECS 50 +static mach_timespec_t READ_SETUP_timeout = {.tv_sec = READ_SETUP_TIMEOUT_SECS, .tv_nsec = 0}; +static mach_timespec_t READ_timeout = {.tv_sec = READ_TIMEOUT_SECS, .tv_nsec = 0}; +static mach_timespec_t WRITE_timeout = {.tv_sec = WRITE_TIMEOUT_SECS, .tv_nsec = 0}; + enum fd_pair { PTY_PAIR, FIFO_PAIR, @@ -95,14 +105,14 @@ static struct { PROCESS_WRITER /* fd */ } wr_kind; union { - dispatch_semaphore_t sem; + semaphore_t sem; struct { int in_fd; int out_fd; }; } wr_wait; - dispatch_semaphore_t wr_finished; - dispatch_semaphore_t rd_finished; + semaphore_t wr_finished; + semaphore_t rd_finished; } shared; static bool handle_reading(enum fd_pair fd_pair, int fd); @@ -119,7 +129,8 @@ wake_writer(void) switch (shared.wr_kind) { case THREAD_WRITER: - dispatch_semaphore_signal(shared.wr_wait.sem); + T_LOG("signal shared.wr_wait.sem"); + semaphore_signal(shared.wr_wait.sem); break; case PROCESS_WRITER: { char tmp = 'a'; @@ -136,12 +147,16 @@ writer_wait(void) { switch (shared.wr_kind) { case THREAD_WRITER: - T_QUIET; T_ASSERT_EQ(dispatch_semaphore_wait( - shared.wr_wait.sem, - dispatch_time(DISPATCH_TIME_NOW, - READ_SETUP_TIMEOUT_SECS * NSEC_PER_SEC)), 0L, - NULL); + T_LOG("wait shared.wr_wait.sem"); + kern_return_t kret = semaphore_timedwait(shared.wr_wait.sem, READ_SETUP_timeout); + + if (kret == KERN_OPERATION_TIMED_OUT) { + T_ASSERT_FAIL("THREAD_WRITER semaphore timedout after %d seconds", READ_SETUP_timeout.tv_sec); + } + T_QUIET; + T_ASSERT_MACH_SUCCESS(kret, "semaphore_timedwait shared.wr_wait.sem"); break; + case PROCESS_WRITER: { char tmp; close(shared.wr_wait.in_fd); @@ -193,7 +208,8 @@ workqueue_write_fn(void ** __unused buf, int * __unused count) // "writer thread should be woken up at correct QoS"); if (!handle_writing(shared.fd_pair, shared.wr_fd)) { /* finished handling the fd, tear down the source */ - dispatch_semaphore_signal(shared.wr_finished); + T_LOG("signal shared.wr_finished"); + semaphore_signal(shared.wr_finished); return; } @@ -309,10 +325,6 @@ drive_kq(bool reading, union mode mode, enum fd_pair fd_pair, int fd) continue; } T_QUIET; T_ASSERT_POSIX_SUCCESS(kev, "kevent"); - /* */ - if (shared.fd_pair == PTY_PAIR) { - T_MAYFAIL; - } T_QUIET; T_ASSERT_NE(kev, 0, "kevent timed out"); if (reading) { @@ -368,14 +380,18 @@ write_to_fd(void * __unused ctx) } case WORKQ_INCREMENTAL_WRITE: { + // prohibit ourselves from going multi-threaded see:rdar://33296008 + _dispatch_prohibit_transition_to_multithreaded(true); int changes = 1; - shared.wr_finished = dispatch_semaphore_create(0); - T_QUIET; T_ASSERT_NOTNULL(shared.wr_finished, - "dispatch_semaphore_create"); + T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &shared.wr_finished, SYNC_POLICY_FIFO, 0), + "semaphore_create shared.wr_finished"); - T_QUIET; T_ASSERT_POSIX_ZERO(_pthread_workqueue_init_with_kevent( - workqueue_fn, workqueue_write_fn, 0, 0), NULL); + T_QUIET; + T_ASSERT_NE_UINT(shared.wr_finished, (unsigned)MACH_PORT_NULL, "wr_finished semaphore_create"); + + T_QUIET; + T_ASSERT_POSIX_ZERO(_pthread_workqueue_init_with_kevent(workqueue_fn, workqueue_write_fn, 0, 0), NULL); struct kevent_qos_s events[] = {{ .ident = (uint64_t)shared.wr_fd, @@ -406,9 +422,11 @@ write_to_fd(void * __unused ctx) case DISPATCH_INCREMENTAL_WRITE: { dispatch_source_t write_src; - shared.wr_finished = dispatch_semaphore_create(0); - T_QUIET; T_ASSERT_NOTNULL(shared.wr_finished, - "dispatch_semaphore_create"); + T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &shared.wr_finished, SYNC_POLICY_FIFO, 0), + "semaphore_create shared.wr_finished"); + + T_QUIET; + T_ASSERT_NE_UINT(shared.wr_finished, (unsigned)MACH_PORT_NULL, "semaphore_create"); write_src = dispatch_source_create(DISPATCH_SOURCE_TYPE_WRITE, (uintptr_t)shared.wr_fd, 0, NULL); @@ -424,7 +442,8 @@ write_to_fd(void * __unused ctx) /* finished handling the fd, tear down the source */ dispatch_source_cancel(write_src); dispatch_release(write_src); - dispatch_semaphore_signal(shared.wr_finished); + T_LOG("signal shared.wr_finished"); + semaphore_signal(shared.wr_finished); } }); @@ -440,17 +459,14 @@ write_to_fd(void * __unused ctx) } if (shared.wr_finished) { - long sem_timed_out = dispatch_semaphore_wait(shared.wr_finished, - dispatch_time(DISPATCH_TIME_NOW, - WRITE_TIMEOUT_SECS * NSEC_PER_SEC)); - dispatch_release(shared.wr_finished); - /* */ - if (shared.fd_pair == PTY_PAIR) { - T_MAYFAIL; + T_LOG("wait shared.wr_finished"); + kern_return_t kret = semaphore_timedwait(shared.wr_finished, WRITE_timeout); + if (kret == KERN_OPERATION_TIMED_OUT) { + T_ASSERT_FAIL("write side semaphore timedout after %d seconds", WRITE_timeout.tv_sec); } - T_QUIET; T_ASSERT_EQ(sem_timed_out, 0L, - "write side semaphore timed out after %d seconds", - WRITE_TIMEOUT_SECS); + T_QUIET; + T_ASSERT_MACH_SUCCESS(kret, "semaphore_timedwait shared.wr_finished"); + semaphore_destroy(mach_task_self(), shared.wr_finished); } T_LOG("writer finished, closing fd"); @@ -482,6 +498,8 @@ handle_reading(enum fd_pair fd_pair, int fd) bytes_rd = read(fd, read_buf, sizeof(read_buf) - 1); } while (bytes_rd == -1 && errno == EINTR); + // T_LOG("read %zd bytes: '%s'", bytes_rd, read_buf); + T_QUIET; T_ASSERT_POSIX_SUCCESS(bytes_rd, "reading from file"); T_QUIET; T_ASSERT_LE(bytes_rd, (ssize_t)EXPECTED_LEN, "read too much from file"); @@ -496,8 +514,6 @@ handle_reading(enum fd_pair fd_pair, int fd) sizeof(final_string) - final_length); final_length += (size_t)bytes_rd; - // T_LOG("read %zd bytes: '%s'", bytes_rd, read_buf); - T_QUIET; T_ASSERT_LE(final_length, EXPECTED_LEN, "should not read more from file than what can be sent"); @@ -519,7 +535,8 @@ workqueue_read_fn(void ** __unused buf, int * __unused count) // T_QUIET; T_ASSERT_EFFECTIVE_QOS_EQ(EXPECTED_QOS, // "reader thread should be requested at correct QoS"); if (!handle_reading(shared.fd_pair, shared.rd_fd)) { - dispatch_semaphore_signal(shared.rd_finished); + T_LOG("signal shared.rd_finished"); + semaphore_signal(shared.rd_finished); } reenable_workq(shared.rd_fd, EVFILT_READ); @@ -547,20 +564,19 @@ read_from_fd(int fd, enum fd_pair fd_pair, enum read_mode mode) case POLL_READ: { struct pollfd fds[] = { { .fd = fd, .events = POLLIN } }; wake_writer(); + for (;;) { fds[0].revents = 0; int pol = poll(fds, 1, READ_TIMEOUT_SECS * 1000); T_QUIET; T_ASSERT_POSIX_SUCCESS(pol, "poll"); - /* */ - if (shared.fd_pair == PTY_PAIR) { - T_MAYFAIL; - } T_QUIET; T_ASSERT_NE(pol, 0, "poll should not time out after %d seconds, read %zd out " "of %zu bytes", READ_TIMEOUT_SECS, final_length, strlen(EXPECTED_STRING)); T_QUIET; T_ASSERT_FALSE(fds[0].revents & POLLERR, "should not see an error on the device"); + T_QUIET; T_ASSERT_FALSE(fds[0].revents & POLLNVAL, + "should not set up an invalid poll"); if (!handle_reading(fd_pair, fd)) { break; @@ -591,10 +607,6 @@ read_from_fd(int fd, enum fd_pair fd_pair, enum read_mode mode) T_QUIET; T_ASSERT_POSIX_SUCCESS(sel, "select"); - /* */ - if (shared.fd_pair == PTY_PAIR) { - T_MAYFAIL; - } T_QUIET; T_ASSERT_NE(sel, 0, "select waited for %d seconds and timed out", READ_TIMEOUT_SECS); @@ -624,12 +636,16 @@ read_from_fd(int fd, enum fd_pair fd_pair, enum read_mode mode) } case WORKQ_READ: { - T_QUIET; T_ASSERT_POSIX_ZERO(_pthread_workqueue_init_with_kevent( + // prohibit ourselves from going multi-threaded see:rdar://33296008 + _dispatch_prohibit_transition_to_multithreaded(true); + T_ASSERT_POSIX_ZERO(_pthread_workqueue_init_with_kevent( workqueue_fn, workqueue_read_fn, 0, 0), NULL); - shared.rd_finished = dispatch_semaphore_create(0); - T_QUIET; T_ASSERT_NOTNULL(shared.rd_finished, - "dispatch_semaphore_create"); + T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &shared.rd_finished, SYNC_POLICY_FIFO, 0), + "semaphore_create shared.rd_finished"); + + T_QUIET; + T_ASSERT_NE_UINT(shared.rd_finished, (unsigned)MACH_PORT_NULL, "semaphore_create"); int changes = 1; struct kevent_qos_s events[] = {{ @@ -663,9 +679,11 @@ read_from_fd(int fd, enum fd_pair fd_pair, enum read_mode mode) case DISPATCH_READ: { dispatch_source_t read_src; - shared.rd_finished = dispatch_semaphore_create(0); - T_QUIET; T_ASSERT_NOTNULL(shared.rd_finished, - "dispatch_semaphore_create"); + T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &shared.rd_finished, SYNC_POLICY_FIFO, 0), + "semaphore_create shared.rd_finished"); + + T_QUIET; + T_ASSERT_NE_UINT(shared.rd_finished, (unsigned)MACH_PORT_NULL, "semaphore_create"); read_src = dispatch_source_create(DISPATCH_SOURCE_TYPE_READ, (uintptr_t)fd, 0, NULL); @@ -682,7 +700,8 @@ read_from_fd(int fd, enum fd_pair fd_pair, enum read_mode mode) /* finished handling the fd, tear down the source */ dispatch_source_cancel(read_src); dispatch_release(read_src); - dispatch_semaphore_signal(shared.rd_finished); + T_LOG("signal shared.rd_finished"); + semaphore_signal(shared.rd_finished); } }); @@ -699,16 +718,13 @@ read_from_fd(int fd, enum fd_pair fd_pair, enum read_mode mode) } if (shared.rd_finished) { - long timed_out = dispatch_semaphore_wait(shared.rd_finished, - dispatch_time(DISPATCH_TIME_NOW, - READ_TIMEOUT_SECS * NSEC_PER_SEC)); - /* */ - if (shared.fd_pair == PTY_PAIR) { - T_MAYFAIL; + T_LOG("wait shared.rd_finished"); + kern_return_t kret = semaphore_timedwait(shared.rd_finished, READ_timeout); + if (kret == KERN_OPERATION_TIMED_OUT) { + T_ASSERT_FAIL("reading timed out after %d seconds", READ_timeout.tv_sec); } - T_QUIET; T_ASSERT_EQ(timed_out, 0L, - "reading timed out after %d seconds", READ_TIMEOUT_SECS); - + T_QUIET; + T_ASSERT_MACH_SUCCESS(kret, "semaphore_timedwait shared.rd_finished"); } T_EXPECT_EQ_STR(final_string, EXPECTED_STRING, @@ -784,7 +800,8 @@ drive_threads(enum fd_pair fd_pair, enum read_mode rd_mode, fd_pair_init(fd_pair, &(shared.rd_fd), &(shared.wr_fd)); shared.wr_kind = THREAD_WRITER; - shared.wr_wait.sem = dispatch_semaphore_create(0); + T_ASSERT_MACH_SUCCESS(semaphore_create(mach_task_self(), &shared.wr_wait.sem, SYNC_POLICY_FIFO, 0), + "semaphore_create shared.wr_wait.sem"); T_QUIET; T_ASSERT_POSIX_ZERO(pthread_create(&thread, NULL, write_to_fd, NULL), @@ -792,6 +809,9 @@ drive_threads(enum fd_pair fd_pair, enum read_mode rd_mode, T_LOG("created writer thread"); read_from_fd(shared.rd_fd, fd_pair, rd_mode); + + T_ASSERT_POSIX_ZERO(pthread_join(thread, NULL), NULL); + T_END; } @@ -840,7 +860,7 @@ T_HELPER_DECL(writer_helper, "Write asynchronously") #define WR_DECL_PROCESSES(desc_name, fd_pair, write_name, write_str, \ write_mode, read_name, read_mode) \ - T_DECL(processes_##desc_name##_##read_name##_##write_name, "read changes to a " \ + T_DECL(desc_name##_r##read_name##_w##write_name##_procs, "read changes to a " \ #desc_name " with " #read_name " and writing " #write_str \ " across two processes") \ { \ @@ -848,7 +868,7 @@ T_HELPER_DECL(writer_helper, "Write asynchronously") } #define WR_DECL_THREADS(desc_name, fd_pair, write_name, write_str, \ write_mode, read_name, read_mode) \ - T_DECL(threads_##desc_name##_##read_name##_##write_name, "read changes to a " \ + T_DECL(desc_name##_r##read_name##_w##write_name##_thds, "read changes to a " \ #desc_name " with " #read_name " and writing " #write_str) \ { \ drive_threads(fd_pair, read_mode, write_mode); \ @@ -864,17 +884,17 @@ T_HELPER_DECL(writer_helper, "Write asynchronously") #define RD_DECL_SAFE(desc_name, fd_pair, read_name, read_mode) \ WR_DECL(desc_name, fd_pair, full, "the full string", FULL_WRITE, \ read_name, read_mode) \ - WR_DECL(desc_name, fd_pair, incremental, "incrementally", \ + WR_DECL(desc_name, fd_pair, inc, "incrementally", \ INCREMENTAL_WRITE, read_name, read_mode) #define RD_DECL_DISPATCH_ONLY(suffix, desc_name, fd_pair, read_name, \ read_mode) \ - WR_DECL##suffix(desc_name, fd_pair, incremental_dispatch, \ + WR_DECL##suffix(desc_name, fd_pair, inc_dispatch, \ "incrementally with a dispatch source", \ DISPATCH_INCREMENTAL_WRITE, read_name, read_mode) #define RD_DECL_WORKQ_ONLY(suffix, desc_name, fd_pair, read_name, \ read_mode) \ - WR_DECL##suffix(desc_name, fd_pair, incremental_workq, \ + WR_DECL##suffix(desc_name, fd_pair, inc_workq, \ "incrementally with the workqueue", \ WORKQ_INCREMENTAL_WRITE, read_name, read_mode) diff --git a/tools/tests/darwintests/private_entitlement.plist b/tools/tests/darwintests/private_entitlement.plist new file mode 100644 index 000000000..6f5ceceb2 --- /dev/null +++ b/tools/tests/darwintests/private_entitlement.plist @@ -0,0 +1,8 @@ + + + + + com.apple.private.entitlement-1 + something + + diff --git a/tools/tests/darwintests/proc_info.c b/tools/tests/darwintests/proc_info.c new file mode 100644 index 000000000..11b042d0d --- /dev/null +++ b/tools/tests/darwintests/proc_info.c @@ -0,0 +1,322 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PRIVATE +#include +#include +#include +#undef PRIVATE + +T_GLOBAL_META(T_META_NAMESPACE("xnu.all")); + +#pragma mark proc_list_uptrs + +#define NUPTRS 4 +static uint64_t uptrs[NUPTRS] = { + 0x1122334455667788ULL, + 0x99aabbccddeeff00ULL, + 0xaabbaaddccaaffeeULL, + 0xcc000011ccaa7755ULL +}; + +static const char *uptr_names[NUPTRS]; + +static void +print_uptrs(int argc, char * const *argv) +{ + for (int i = 0; i < argc; i++) { + char *end; + unsigned long pid = strtoul(argv[i], &end, 0); + if (pid > INT_MAX) { + printf("error: pid '%lu' would overflow an integer\n", pid); + } + if (end == argv[i]) { + printf("error: could not parse '%s' as a pid\n", argv[i]); + continue; + } + int uptrs_count = proc_list_uptrs((int)pid, NULL, 0); + if (uptrs_count == 0) { + printf("no uptrs for process %d\n", (int)pid); + return; + } + + /* extra space */ + unsigned int uptrs_len = (unsigned int)uptrs_count + 32; + + uint64_t *uptrs_alloc = malloc(sizeof(uint64_t) * uptrs_len); + os_assert(uptrs_alloc != NULL); + + uptrs_count = proc_list_uptrs((int)pid, uptrs_alloc, + (uint32_t)(sizeof(uint64_t) * uptrs_len)); + printf("process %d has %d uptrs:\n", (int)pid, uptrs_count); + if (uptrs_count > (int)uptrs_len) { + uptrs_count = (int)uptrs_len; + } + for (int j = 0; j < uptrs_count; j++) { + printf("%#17" PRIx64 "\n", uptrs_alloc[j]); + } + } +} + +T_DECL(proc_list_uptrs, + "the kernel should return any up-pointers it knows about", + T_META_ALL_VALID_ARCHS(YES)) +{ + if (argc > 0) { + print_uptrs(argc, argv); + T_SKIP("command line invocation of tool, not test"); + } + + unsigned int cur_uptr = 0; + + int kq = kqueue(); + T_QUIET; T_ASSERT_POSIX_SUCCESS(kq, "kqueue"); + + /* + * Should find uptrs on file-type knotes and generic knotes (two + * different search locations, internally). + */ + struct kevent64_s events[2]; + memset(events, 0, sizeof(events)); + + uptr_names[cur_uptr] = "kqueue file-backed knote"; + events[0].filter = EVFILT_WRITE; + events[0].ident = STDOUT_FILENO; + events[0].flags = EV_ADD; + events[0].udata = uptrs[cur_uptr++]; + + uptr_names[cur_uptr] = "kqueue non-file-backed knote"; + events[1].filter = EVFILT_USER; + events[1].ident = 1; + events[1].flags = EV_ADD; + events[1].udata = uptrs[cur_uptr++]; + + int kev_err = kevent64(kq, events, sizeof(events) / sizeof(events[0]), NULL, + 0, KEVENT_FLAG_IMMEDIATE, NULL); + T_ASSERT_POSIX_SUCCESS(kev_err, "register events with kevent64"); + + /* + * Should find uptrs both on a kevent_id kqueue and in a workloop + * kqueue's knote's udata field. + */ + uptr_names[cur_uptr] = "dynamic kqueue non-file-backed knote"; + struct kevent_qos_s events_id[] = {{ + .filter = EVFILT_USER, + .ident = 1, + .flags = EV_ADD, + .udata = uptrs[cur_uptr++] + }}; + + uptr_names[cur_uptr] = "dynamic kqueue ID"; + kev_err = kevent_id(uptrs[cur_uptr++], events_id, 1, NULL, 0, NULL, NULL, + KEVENT_FLAG_WORKLOOP | KEVENT_FLAG_IMMEDIATE); + T_ASSERT_POSIX_SUCCESS(kev_err, "register event with kevent_id"); + + errno = 0; + int uptrs_count = proc_list_uptrs(getpid(), NULL, 0); + T_QUIET; T_ASSERT_POSIX_SUCCESS(uptrs_count, "proc_list_uptrs"); + T_QUIET; T_EXPECT_EQ(uptrs_count, NUPTRS, + "should see correct number of up-pointers"); + + uint64_t uptrs_obs[NUPTRS] = { 0 }; + uptrs_count = proc_list_uptrs(getpid(), uptrs_obs, sizeof(uptrs_obs)); + T_QUIET; T_ASSERT_POSIX_SUCCESS(uptrs_count, "proc_list_uptrs"); + + for (int i = 0; i < uptrs_count; i++) { + int found = -1; + for (int j = 0; j < NUPTRS; j++) { + if (uptrs_obs[i] == uptrs[j]) { + found = j; + goto next; + } + } + T_FAIL("unexpected up-pointer found: %#" PRIx64, uptrs_obs[i]); +next:; + if (found != -1) { + T_PASS("found up-pointer for %s", uptr_names[found]); + } + } +} + +#pragma mark dynamic kqueue info + +#define EXPECTED_ID UINT64_C(0x1122334455667788) +#define EXPECTED_UDATA UINT64_C(0x99aabbccddeeff00) +#ifndef KQ_WORKLOOP +#define KQ_WORKLOOP 0x80 +#endif + +static void +setup_kevent_id(kqueue_id_t id) +{ + struct kevent_qos_s events_id[] = {{ + .filter = EVFILT_USER, + .ident = 1, + .flags = EV_ADD, + .udata = EXPECTED_UDATA + }}; + + int err = kevent_id(id, events_id, 1, NULL, 0, NULL, NULL, + KEVENT_FLAG_WORKLOOP | KEVENT_FLAG_IMMEDIATE); + T_ASSERT_POSIX_SUCCESS(err, "register event with kevent_id"); +} + +static kqueue_id_t * +list_kqids(pid_t pid, int *nkqids_out) +{ + int kqids_len = 256; + int nkqids; + kqueue_id_t *kqids = NULL; + uint32_t kqids_size; + +retry: + if (os_mul_overflow(sizeof(kqueue_id_t), kqids_len, &kqids_size)) { + T_QUIET; T_ASSERT_GT(kqids_len, PROC_PIDDYNKQUEUES_MAX, NULL); + kqids_len = PROC_PIDDYNKQUEUES_MAX; + goto retry; + } + if (!kqids) { + kqids = malloc(kqids_size); + T_QUIET; T_ASSERT_NOTNULL(kqids, "malloc(%" PRIu32 ")", kqids_size); + } + + nkqids = proc_list_dynkqueueids(pid, kqids, kqids_size); + if (nkqids > kqids_len && kqids_len < PROC_PIDDYNKQUEUES_MAX) { + kqids_len *= 2; + if (kqids_len > PROC_PIDDYNKQUEUES_MAX) { + kqids_len = PROC_PIDDYNKQUEUES_MAX; + } + free(kqids); + kqids = NULL; + goto retry; + } + + *nkqids_out = nkqids; + return kqids; +} + +T_DECL(list_dynamic_kqueues, + "the kernel should list IDs of dynamic kqueues", + T_META_ALL_VALID_ARCHS(true)) +{ + int nkqids; + bool found = false; + + setup_kevent_id(EXPECTED_ID); + kqueue_id_t *kqids = list_kqids(getpid(), &nkqids); + T_ASSERT_GE(nkqids, 1, "at least one dynamic kqueue is listed"); + for (int i = 0; i < nkqids; i++) { + if (kqids[i] == EXPECTED_ID) { + found = true; + T_PASS("found expected dynamic kqueue ID"); + } else { + T_LOG("found another dynamic kqueue with ID %#" PRIx64, kqids[i]); + } + } + + if (!found) { + T_FAIL("could not find dynamic ID of kqueue created"); + } + + free(kqids); +} + +T_DECL(dynamic_kqueue_basic_info, + "the kernel should report valid basic dynamic kqueue info", + T_META_ALL_VALID_ARCHS(true)) +{ + struct kqueue_info kqinfo; + int ret; + + setup_kevent_id(EXPECTED_ID); + ret = proc_piddynkqueueinfo(getpid(), PROC_PIDDYNKQUEUE_INFO, EXPECTED_ID, + &kqinfo, sizeof(kqinfo)); + T_ASSERT_POSIX_SUCCESS(ret, + "proc_piddynkqueueinfo(... PROC_PIDDYNKQUEUE_INFO ...)"); + T_QUIET; T_ASSERT_GE(ret, (int)sizeof(kqinfo), + "PROC_PIDDYNKQUEUE_INFO should return the right size"); + + T_EXPECT_NE(kqinfo.kq_state & KQ_WORKLOOP, 0U, + "kqueue info should be for a workloop kqueue"); + T_EXPECT_EQ(kqinfo.kq_stat.vst_ino, EXPECTED_ID, + "inode field should be the kqueue's ID"); +} + +T_DECL(dynamic_kqueue_extended_info, + "the kernel should report valid extended dynamic kqueue info", + T_META_ALL_VALID_ARCHS(true)) +{ + struct kevent_extinfo kqextinfo[1]; + int ret; + + setup_kevent_id(EXPECTED_ID); + ret = proc_piddynkqueueinfo(getpid(), PROC_PIDDYNKQUEUE_EXTINFO, + EXPECTED_ID, kqextinfo, sizeof(kqextinfo)); + T_ASSERT_POSIX_SUCCESS(ret, + "proc_piddynkqueueinfo(... PROC_PIDDYNKQUEUE_EXTINFO ...)"); + T_QUIET; T_ASSERT_EQ(ret, 1, + "PROC_PIDDYNKQUEUE_EXTINFO should return a single knote"); + + T_EXPECT_EQ(kqextinfo[0].kqext_kev.ident, 1ULL, + "kevent identifier matches what was configured"); + T_EXPECT_EQ(kqextinfo[0].kqext_kev.filter, (short)EVFILT_USER, + "kevent filter matches what was configured"); + T_EXPECT_EQ(kqextinfo[0].kqext_kev.udata, EXPECTED_UDATA, + "kevent udata matches what was configured"); +} + +#pragma mark proc_listpids + +T_DECL(list_kdebug_pids, + "the kernel should report processes that are filtered by kdebug", + T_META_ASROOT(YES)) +{ + int mib[4] = { CTL_KERN, KERN_KDEBUG }; + int npids; + int pids[1]; + int ret; + kd_regtype reg = {}; + size_t regsize = sizeof(reg); + + mib[2] = KERN_KDREMOVE; + ret = sysctl(mib, 3, NULL, NULL, NULL, 0); + T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "KERN_KDREMOVE sysctl"); + + mib[2] = KERN_KDSETBUF; mib[3] = 100000; + ret = sysctl(mib, 4, NULL, NULL, NULL, 0); + T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "KERN_KDSETBUF sysctl"); + + mib[2] = KERN_KDSETUP; + ret = sysctl(mib, 3, NULL, NULL, NULL, 0); + T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "KERN_KDSETUP sysctl"); + + npids = proc_listpids(PROC_KDBG_ONLY, 0, pids, sizeof(pids)); + T_EXPECT_EQ(npids, 0, "no processes should be filtered initially"); + + reg.type = KDBG_TYPENONE; + reg.value1 = getpid(); + reg.value2 = 1; /* set the pid in the filter */ + mib[2] = KERN_KDPIDTR; + ret = sysctl(mib, 3, ®, ®size, NULL, 0); + T_ASSERT_POSIX_SUCCESS(ret, + "KERN_KDPIDTR sysctl to set a pid in the filter"); + + npids = proc_listpids(PROC_KDBG_ONLY, 0, pids, sizeof(pids)); + npids /= 4; + T_EXPECT_EQ(npids, 1, "a process should be filtered"); + T_EXPECT_EQ(pids[0], getpid(), + "process filtered should be the one that was set"); + + mib[2] = KERN_KDREMOVE; + ret = sysctl(mib, 3, NULL, NULL, NULL, 0); + T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "KERN_KDREMOVE sysctl"); +} diff --git a/tools/tests/darwintests/settimeofday_29193041.c b/tools/tests/darwintests/settimeofday_29193041.c new file mode 100644 index 000000000..ae6d68b88 --- /dev/null +++ b/tools/tests/darwintests/settimeofday_29193041.c @@ -0,0 +1,229 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if CONFIG_EMBEDDED +#include +#include +#include +#endif + +#define EXIT_FAIL() exit((__LINE__ % 255) + 1) + +/* + * This test expects the entitlement or root privileges for a process to + * set the time using settimeofday syscall. + */ + +#define DAY 86400 //1 day in sec + +/* + * To run without root privileges + * libdarwintest should run leaks even without root + */ +static void drop_priv(void){ + /* determine the less-privileged UID and GID */ + + unsigned long lower_uid = 0; + unsigned long lower_gid = 0; + +#if CONFIG_EMBEDDED + struct passwd *pw = getpwnam("mobile"); + if (!pw) { + printf("child: error: get_pwname(\"mobile\") failed %d: %s\n", errno, strerror(errno)); + EXIT_FAIL(); + } + + lower_uid = pw->pw_uid; + lower_gid = pw->pw_gid; +#else + char *sudo_gid_str = getenv("SUDO_GID"); + if (!sudo_gid_str) { + printf("child: error: SUDO_GID environment variable unset (not run under sudo)\n"); + EXIT_FAIL(); + } + + char *sudo_uid_str = getenv("SUDO_UID"); + if (!sudo_uid_str) { + printf("child: error: SUDO_UID environment variable unset (not run under sudo)\n"); + EXIT_FAIL(); + } + + char *end = sudo_gid_str; + lower_gid = strtoul(sudo_gid_str, &end, 10); + if (sudo_gid_str == end && sudo_gid_str[0] != '\0') { + printf("child: error: SUDO_GID (%s) could not be converted to an integer\n", sudo_gid_str); + EXIT_FAIL(); + } + if (lower_gid == 0) { + printf("child: error: less-privileged GID invalid\n"); + EXIT_FAIL(); + } + + end = sudo_uid_str; + lower_uid = strtoul(sudo_uid_str, &end, 10); + if (sudo_uid_str == end && sudo_uid_str[0] != '\0') { + printf("child: error: SUDO_UID (%s) could not be converted to an integer\n", sudo_uid_str); + EXIT_FAIL(); + } + if (lower_gid == 0) { + printf("child: error: less-privileged UID invalid\n"); + EXIT_FAIL(); + } +#endif + + if (setgid(lower_gid) == -1) { + printf("child: error: could not change group to %lu\n", lower_gid); + EXIT_FAIL(); + } + if (setuid(lower_uid) == -1) { + printf("child: error: could not change user to %lu\n", lower_uid); + EXIT_FAIL(); + } +} + +T_DECL(settime_32089962_not_entitled_root, + "Verify that root privileges can allow to change the time", + T_META_ASROOT(true), T_META_CHECK_LEAKS(NO)) +{ + struct timeval settimeofdaytime; + struct timeval adj_time; + struct timex ntptime; + + if (geteuid() != 0){ + T_SKIP("settimeofday_root_29193041 test requires root privileges to run."); + } + + /* test settimeofday */ + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&settimeofdaytime, NULL), NULL); + T_ASSERT_POSIX_ZERO(settimeofday(&settimeofdaytime, NULL), NULL); + + /* test adjtime */ + adj_time.tv_sec = 1; + adj_time.tv_usec = 0; + T_ASSERT_POSIX_ZERO(adjtime(&adj_time, NULL),NULL); + + /* test ntp_adjtime */ + memset(&ntptime, 0, sizeof(ntptime)); + ntptime.modes |= MOD_STATUS; + ntptime.status = TIME_OK; + + T_ASSERT_EQ(ntp_adjtime(&ntptime), TIME_OK, NULL); +} + +T_DECL(settime_32089962_not_entitled_not_root, + "Verify that the \"com.apple.settime\" entitlement can allow to change the time", + T_META_ASROOT(false), T_META_CHECK_LEAKS(NO)) +{ + struct timeval settimeofdaytime; + struct timeval adj_time; + struct timex ntptime; + int res; + + drop_priv(); + + if (geteuid() == 0){ + T_SKIP("settimeofday_29193041 test requires no root privileges to run."); + } + + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&settimeofdaytime, NULL), NULL); + + /* test settimeofday */ +#if TARGET_OS_EMBEDDED + T_ASSERT_POSIX_ZERO(settimeofday(&settimeofdaytime, NULL), NULL); +#else + res = settimeofday(&settimeofdaytime, NULL); + T_ASSERT_EQ(res, -1, NULL); +#endif + + /* test adjtime */ + adj_time.tv_sec = 1; + adj_time.tv_usec = 0; + res = adjtime(&adj_time, NULL); + T_ASSERT_EQ(res, -1, NULL); + + /* test ntp_adjtime */ + memset(&ntptime, 0, sizeof(ntptime)); + ntptime.modes |= MOD_STATUS; + ntptime.status = TIME_OK; + res = ntp_adjtime(&ntptime); + T_ASSERT_EQ(res, -1, NULL); +} + +T_DECL(settimeofday_29193041_not_entitled_root, + "Verify that root privileges can allow to change the time", + T_META_ASROOT(true), T_META_CHECK_LEAKS(NO)) +{ + struct timeval time; + long new_time; + + if (geteuid() != 0){ + T_SKIP("settimeofday_root_29193041 test requires root privileges to run."); + } + + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&time, NULL), NULL); + + /* increment the time of one day */ + new_time = time.tv_sec + DAY; + + time.tv_sec = new_time; + time.tv_usec = 0; + + T_ASSERT_POSIX_ZERO(settimeofday(&time, NULL), NULL); + + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&time, NULL), NULL); + + /* expext to be past new_time */ + T_EXPECT_GE_LONG(time.tv_sec, new_time, "Time changed with root and without entitlement"); + + time.tv_sec -= DAY; + T_QUIET;T_ASSERT_POSIX_ZERO(settimeofday(&time, NULL), NULL); +} + +T_DECL(settimeofday_29193041_not_entitled_not_root, + "Verify that the \"com.apple.settime\" entitlement can allow to change the time", + T_META_ASROOT(false), T_META_CHECK_LEAKS(NO)) +{ + struct timeval time; + long new_time; + + drop_priv(); + + if (geteuid() == 0){ + T_SKIP("settimeofday_29193041 test requires no root privileges to run."); + } + + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&time, NULL), NULL); + + /* increment the time of one day */ + new_time = time.tv_sec + DAY; + + time.tv_sec = new_time; + time.tv_usec = 0; + +#if TARGET_OS_EMBEDDED + T_ASSERT_POSIX_ZERO(settimeofday(&time, NULL), NULL); +#else + int res = settimeofday(&time, NULL); + T_ASSERT_EQ(res, -1, NULL); +#endif + + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&time, NULL), NULL); + +#if TARGET_OS_EMBEDDED + /* expext to be past new_time */ + T_EXPECT_GE_LONG(time.tv_sec, new_time, "Time successfully changed without root and without entitlement"); + time.tv_sec -= DAY; + T_QUIET; T_ASSERT_POSIX_ZERO(settimeofday(&time, NULL), NULL); +#else + T_EXPECT_LT_LONG(time.tv_sec, new_time, "Not permitted to change time without root and without entitlement"); +#endif + +} diff --git a/tools/tests/darwintests/settimeofday_29193041.entitlements b/tools/tests/darwintests/settimeofday_29193041.entitlements new file mode 100644 index 000000000..fafc6c9f2 --- /dev/null +++ b/tools/tests/darwintests/settimeofday_29193041.entitlements @@ -0,0 +1,8 @@ + + + + + com.apple.private.settime + + + diff --git a/tools/tests/darwintests/settimeofday_29193041_entitled.c b/tools/tests/darwintests/settimeofday_29193041_entitled.c new file mode 100644 index 000000000..a68c6cac8 --- /dev/null +++ b/tools/tests/darwintests/settimeofday_29193041_entitled.c @@ -0,0 +1,214 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if CONFIG_EMBEDDED +#include +#include +#include +#endif + +#define EXIT_FAIL() exit((__LINE__ % 255) + 1) + +/* + * This test expects the entitlement or root privileges for a process to + * set the time using settimeofday syscall. + */ + +#define DAY 86400 //1 day in sec + +/* + * To run without root privileges + * libdarwintest should run leaks even without root + */ +static void drop_priv(void){ + /* determine the less-privileged UID and GID */ + + unsigned long lower_uid = 0; + unsigned long lower_gid = 0; + +#if CONFIG_EMBEDDED + struct passwd *pw = getpwnam("mobile"); + if (!pw) { + printf("child: error: get_pwname(\"mobile\") failed %d: %s\n", errno, strerror(errno)); + EXIT_FAIL(); + } + + lower_uid = pw->pw_uid; + lower_gid = pw->pw_gid; +#else + char *sudo_gid_str = getenv("SUDO_GID"); + if (!sudo_gid_str) { + printf("child: error: SUDO_GID environment variable unset (not run under sudo)\n"); + EXIT_FAIL(); + } + + char *sudo_uid_str = getenv("SUDO_UID"); + if (!sudo_uid_str) { + printf("child: error: SUDO_UID environment variable unset (not run under sudo)\n"); + EXIT_FAIL(); + } + + char *end = sudo_gid_str; + lower_gid = strtoul(sudo_gid_str, &end, 10); + if (sudo_gid_str == end && sudo_gid_str[0] != '\0') { + printf("child: error: SUDO_GID (%s) could not be converted to an integer\n", sudo_gid_str); + EXIT_FAIL(); + } + if (lower_gid == 0) { + printf("child: error: less-privileged GID invalid\n"); + EXIT_FAIL(); + } + + end = sudo_uid_str; + lower_uid = strtoul(sudo_uid_str, &end, 10); + if (sudo_uid_str == end && sudo_uid_str[0] != '\0') { + printf("child: error: SUDO_UID (%s) could not be converted to an integer\n", sudo_uid_str); + EXIT_FAIL(); + } + if (lower_gid == 0) { + printf("child: error: less-privileged UID invalid\n"); + EXIT_FAIL(); + } +#endif + + if (setgid(lower_gid) == -1) { + printf("child: error: could not change group to %lu\n", lower_gid); + EXIT_FAIL(); + } + if (setuid(lower_uid) == -1) { + printf("child: error: could not change user to %lu\n", lower_uid); + EXIT_FAIL(); + } +} + +T_DECL(settime_32089962_entitled_root, + "Verify that root privileges can allow to change the time", + T_META_ASROOT(true), T_META_CHECK_LEAKS(NO)) +{ + struct timeval settimeofdaytime; + struct timeval adj_time; + struct timex ntptime; + + if (geteuid() != 0){ + T_SKIP("settime_32089962_entitled_root test requires root privileges to run."); + } + + /* test settimeofday */ + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&settimeofdaytime, NULL), NULL); + T_ASSERT_POSIX_ZERO(settimeofday(&settimeofdaytime, NULL), NULL); + + /* test adjtime */ + adj_time.tv_sec = 1; + adj_time.tv_usec = 0; + T_ASSERT_POSIX_ZERO(adjtime(&adj_time, NULL),NULL); + + /* test ntp_adjtime */ + memset(&ntptime, 0, sizeof(ntptime)); + ntptime.modes |= MOD_STATUS; + ntptime.status = TIME_OK; + + T_ASSERT_EQ(ntp_adjtime(&ntptime), TIME_OK, NULL); +} + +T_DECL(settime_32089962_entitled_not_root, + "Verify that the \"com.apple.settime\" entitlement can allow to change the time", + T_META_ASROOT(false), T_META_CHECK_LEAKS(NO)) +{ + + struct timeval settimeofdaytime; + struct timeval adj_time; + struct timex ntptime; + + drop_priv(); + + if (geteuid() == 0){ + T_SKIP("settime_32089962_entitled_root test requires no root privileges to run."); + } + + /* test settimeofday */ + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&settimeofdaytime, NULL), NULL); + T_ASSERT_POSIX_ZERO(settimeofday(&settimeofdaytime, NULL), NULL); + + /* test adjtime */ + adj_time.tv_sec = 1; + adj_time.tv_usec = 0; + T_ASSERT_POSIX_ZERO(adjtime(&adj_time, NULL),NULL); + + /* test ntp_adjtime */ + memset(&ntptime, 0, sizeof(ntptime)); + ntptime.modes |= MOD_STATUS; + ntptime.status = TIME_OK; + + T_ASSERT_EQ(ntp_adjtime(&ntptime), TIME_OK, NULL); + +} + +T_DECL(settimeofday_29193041_entitled_root, + "Verify that root privileges can allow to change the time", + T_META_ASROOT(true), T_META_CHECK_LEAKS(NO)) +{ + struct timeval time; + long new_time; + + if (geteuid() != 0){ + T_SKIP("settimeofday_root_29193041 test requires root privileges to run."); + } + + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&time, NULL), NULL); + + /* increment the time of one day */ + new_time = time.tv_sec + DAY; + + time.tv_sec = new_time; + time.tv_usec = 0; + + T_ASSERT_POSIX_ZERO(settimeofday(&time, NULL), NULL); + + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&time, NULL), NULL); + + /* expext to be past new_time */ + T_EXPECT_GE_LONG(time.tv_sec, new_time, "Time changed with root and entitlement"); + + time.tv_sec -= DAY; + T_QUIET;T_ASSERT_POSIX_ZERO(settimeofday(&time, NULL), NULL); +} + +T_DECL(settimeofday_29193041_entitled_not_root, + "Verify that the \"com.apple.settime\" entitlement can allow to change the time", + T_META_ASROOT(false), T_META_CHECK_LEAKS(NO)) +{ + struct timeval time; + long new_time; + + drop_priv(); + + if (geteuid() == 0){ + T_SKIP("settimeofday_29193041 test requires no root privileges to run."); + } + + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&time, NULL), NULL); + + /* increment the time of one day */ + new_time = time.tv_sec + DAY; + + time.tv_sec = new_time; + time.tv_usec = 0; + + T_ASSERT_POSIX_ZERO(settimeofday(&time, NULL), NULL); + + T_QUIET; T_ASSERT_POSIX_ZERO(gettimeofday(&time, NULL), NULL); + + /* expext to be past new_time */ + T_EXPECT_GE_LONG(time.tv_sec, new_time, "Time successfully changed without root and with entitlement"); + + time.tv_sec -= DAY; + T_QUIET; T_ASSERT_POSIX_ZERO(settimeofday(&time, NULL), NULL); +} diff --git a/tools/tests/darwintests/sigchld_return.c b/tools/tests/darwintests/sigchld_return.c new file mode 100644 index 000000000..6a3cc6bcf --- /dev/null +++ b/tools/tests/darwintests/sigchld_return.c @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include + +#include + + +static int exitcode = 0x6789BEEF; +int should_exit = 0; + +void handler (int sig, siginfo_t *sip, __unused void *uconp) +{ + /* Should handle the SIGCHLD signal */ + T_ASSERT_EQ_INT(sig, SIGCHLD, "Captured signal returns 0x%x, expected SIGCHLD (0x%x).", sig, SIGCHLD); + T_QUIET; T_ASSERT_NOTNULL(sip, "siginfo_t returned NULL but should have returned data."); + T_ASSERT_EQ_INT(sip->si_code, CLD_EXITED, "si_code returns 0x%x, expected CLD_EXITED (0x%x).", sip->si_code, CLD_EXITED); + T_ASSERT_EQ_INT(sip->si_status, exitcode, "si_status returns 0x%08X, expected the child's exit code (0x%08X).", sip->si_status, exitcode); + should_exit = 1; +} + + +T_DECL(sigchldreturn, "checks that a child process exited with an exitcode returns correctly to parent", T_META_CHECK_LEAKS(false)) +{ + struct sigaction act; + int pid; + + act.sa_sigaction = handler; + act.sa_flags = SA_SIGINFO; + + /* Set action for signal */ + T_QUIET; T_ASSERT_POSIX_SUCCESS(sigaction (SIGCHLD, &act, NULL), "Calling sigaction() failed for SIGCHLD"); + + /* Now fork a child that just exits */ + pid = fork(); + T_QUIET; T_ASSERT_NE_INT(pid, -1, "fork() failed!"); + + if (pid == 0) { + /* Child process! */ + exit (exitcode); + } + + /* Main program that did the fork */ + /* We should process the signal, then exit */ + while (!should_exit) { + sleep(1); + } +} + diff --git a/tools/tests/darwintests/sigcont_return.c b/tools/tests/darwintests/sigcont_return.c new file mode 100644 index 000000000..606caa910 --- /dev/null +++ b/tools/tests/darwintests/sigcont_return.c @@ -0,0 +1,28 @@ +#include +#include +#include +#include +#include + +#include + +T_DECL(sigcontreturn, "checks that a call to waitid() for a child that is stopped and then continued returns correctly") +{ + pid_t pid; + siginfo_t siginfo; + pid = fork(); + T_QUIET; T_ASSERT_NE_INT(pid, -1, "fork() failed!"); + + if (pid == 0) { + while(1){} + } + + kill(pid, SIGSTOP); + kill(pid, SIGCONT); + sleep(1); + + T_QUIET; T_ASSERT_POSIX_SUCCESS(waitid(P_PID, pid, &siginfo, WCONTINUED), "Calling waitid() failed for pid %d", pid); + + T_ASSERT_EQ_INT(siginfo.si_status, SIGCONT, "A call to waitid() for stopped and continued child returns 0x%x, expected SIGCONT (0x%x)", siginfo.si_status, SIGCONT ); + kill(pid, SIGKILL); +} diff --git a/tools/tests/darwintests/stackshot.m b/tools/tests/darwintests/stackshot.m new file mode 100644 index 000000000..eb17dc1c5 --- /dev/null +++ b/tools/tests/darwintests/stackshot.m @@ -0,0 +1,422 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +T_GLOBAL_META( + T_META_NAMESPACE("xnu.stackshot"), + T_META_CHECK_LEAKS(false), + T_META_ASROOT(true) + ); + +static const char *current_process_name(void); +static void parse_stackshot(bool delta, void *ssbuf, size_t sslen); +static uint64_t stackshot_timestamp(void *ssbuf, size_t sslen); +static void initialize_thread(void); + +#define DEFAULT_STACKSHOT_BUFFER_SIZE (1024 * 1024) +#define MAX_STACKSHOT_BUFFER_SIZE (6 * 1024 * 1024) + +T_DECL(microstackshots, "test the microstackshot syscall") +{ + void *buf = NULL; + unsigned int size = DEFAULT_STACKSHOT_BUFFER_SIZE; + + while (1) { + buf = malloc(size); + T_QUIET; T_ASSERT_NOTNULL(buf, "allocated stackshot buffer"); + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + int len = syscall(SYS_microstackshot, buf, size, + STACKSHOT_GET_MICROSTACKSHOT); +#pragma clang diagnostic pop + if (len == ENOSYS) { + T_SKIP("microstackshot syscall failed, likely not compiled with CONFIG_TELEMETRY"); + } + if (len == -1 && errno == ENOSPC) { + /* syscall failed because buffer wasn't large enough, try again */ + free(buf); + buf = NULL; + size *= 2; + T_ASSERT_LE(size, (unsigned int)MAX_STACKSHOT_BUFFER_SIZE, + "growing stackshot buffer to sane size"); + continue; + } + T_ASSERT_POSIX_SUCCESS(len, "called microstackshot syscall"); + break; + } + + T_EXPECT_EQ(*(uint32_t *)buf, + (uint32_t)STACKSHOT_MICRO_SNAPSHOT_MAGIC, + "magic value for microstackshot matches"); + + free(buf); +} + +struct scenario { + uint32_t flags; + bool should_fail; + pid_t target_pid; + uint64_t since_timestamp; + uint32_t size_hint; + dt_stat_time_t timer; +}; + +static void +quiet(struct scenario *scenario) +{ + if (scenario->timer) { + T_QUIET; + } +} + +static void +take_stackshot(struct scenario *scenario, void (^cb)(void *buf, size_t size)) +{ + void *config = stackshot_config_create(); + quiet(scenario); + T_ASSERT_NOTNULL(config, "created stackshot config"); + + int ret = stackshot_config_set_flags(config, scenario->flags); + quiet(scenario); + T_ASSERT_POSIX_ZERO(ret, "set flags %#x on stackshot config", scenario->flags); + + if (scenario->size_hint > 0) { + ret = stackshot_config_set_size_hint(config, scenario->size_hint); + quiet(scenario); + T_ASSERT_POSIX_ZERO(ret, "set size hint %" PRIu32 " on stackshot config", + scenario->size_hint); + } + + if (scenario->target_pid > 0) { + ret = stackshot_config_set_pid(config, scenario->target_pid); + quiet(scenario); + T_ASSERT_POSIX_ZERO(ret, "set target pid %d on stackshot config", + scenario->target_pid); + } + + if (scenario->since_timestamp > 0) { + ret = stackshot_config_set_delta_timestamp(config, scenario->since_timestamp); + quiet(scenario); + T_ASSERT_POSIX_ZERO(ret, "set since timestamp %" PRIu64 " on stackshot config", + scenario->since_timestamp); + } + + int retries_remaining = 5; + +retry: ; + uint64_t start_time = mach_absolute_time(); + ret = stackshot_capture_with_config(config); + uint64_t end_time = mach_absolute_time(); + + if (scenario->should_fail) { + T_EXPECTFAIL; + T_ASSERT_POSIX_ZERO(ret, "called stackshot_capture_with_config"); + return; + } + + if (ret == EBUSY || ret == ETIMEDOUT) { + if (retries_remaining > 0) { + if (!scenario->timer) { + T_LOG("stackshot_capture_with_config failed with %s (%d), retrying", + strerror(ret), ret); + } + + retries_remaining--; + goto retry; + } else { + T_ASSERT_POSIX_ZERO(ret, + "called stackshot_capture_with_config (no retries remaining)"); + } + } else { + quiet(scenario); + T_ASSERT_POSIX_ZERO(ret, "called stackshot_capture_with_config"); + } + + if (scenario->timer) { + dt_stat_mach_time_add(scenario->timer, end_time - start_time); + } + cb(stackshot_config_get_stackshot_buffer(config), stackshot_config_get_stackshot_size(config)); + + ret = stackshot_config_dealloc(config); + T_QUIET; T_EXPECT_POSIX_ZERO(ret, "deallocated stackshot config"); +} + +T_DECL(kcdata, "test that kcdata stackshots can be taken and parsed") +{ + struct scenario scenario = { + .flags = (STACKSHOT_SAVE_LOADINFO | STACKSHOT_GET_GLOBAL_MEM_STATS | + STACKSHOT_SAVE_IMP_DONATION_PIDS | STACKSHOT_KCDATA_FORMAT) + }; + + initialize_thread(); + T_LOG("taking kcdata stackshot"); + take_stackshot(&scenario, ^(void *ssbuf, size_t sslen) { + parse_stackshot(false, ssbuf, sslen); + }); +} + +T_DECL(kcdata_faulting, "test that kcdata stackshots while faulting can be taken and parsed") +{ + struct scenario scenario = { + .flags = (STACKSHOT_SAVE_LOADINFO | STACKSHOT_GET_GLOBAL_MEM_STATS + | STACKSHOT_SAVE_IMP_DONATION_PIDS | STACKSHOT_KCDATA_FORMAT + | STACKSHOT_ENABLE_BT_FAULTING | STACKSHOT_ENABLE_UUID_FAULTING), + }; + + initialize_thread(); + T_LOG("taking faulting stackshot"); + take_stackshot(&scenario, ^(void *ssbuf, size_t sslen) { + parse_stackshot(false, ssbuf, sslen); + }); +} + +T_DECL(bad_flags, "test a poorly-formed stackshot syscall") +{ + struct scenario scenario = { + .flags = STACKSHOT_SAVE_IN_KERNEL_BUFFER /* not allowed from user space */, + .should_fail = true + }; + + T_LOG("attempting to take stackshot with kernel-only flag"); + take_stackshot(&scenario, ^(__unused void *ssbuf, __unused size_t sslen) { + T_ASSERT_FAIL("stackshot data callback called"); + }); +} + +T_DECL(delta, "test delta stackshots") +{ + struct scenario scenario = { + .flags = (STACKSHOT_SAVE_LOADINFO | STACKSHOT_GET_GLOBAL_MEM_STATS + | STACKSHOT_SAVE_IMP_DONATION_PIDS | STACKSHOT_KCDATA_FORMAT) + }; + + initialize_thread(); + T_LOG("taking full stackshot"); + take_stackshot(&scenario, ^(void *ssbuf, size_t sslen) { + uint64_t stackshot_time = stackshot_timestamp(ssbuf, sslen); + + T_LOG("taking delta stackshot since time %" PRIu64, stackshot_time); + + parse_stackshot(false, ssbuf, sslen); + + struct scenario delta_scenario = { + .flags = (STACKSHOT_SAVE_LOADINFO | STACKSHOT_GET_GLOBAL_MEM_STATS + | STACKSHOT_SAVE_IMP_DONATION_PIDS | STACKSHOT_KCDATA_FORMAT + | STACKSHOT_COLLECT_DELTA_SNAPSHOT), + .since_timestamp = stackshot_time + }; + + take_stackshot(&delta_scenario, ^(void *dssbuf, size_t dsslen) { + parse_stackshot(true, dssbuf, dsslen); + }); + }); +} + +T_DECL(instrs_cycles, "test a getting instructions and cycles in stackshot") +{ + struct scenario scenario = { + .flags = (STACKSHOT_SAVE_LOADINFO | STACKSHOT_INSTRS_CYCLES + | STACKSHOT_KCDATA_FORMAT) + }; + + T_LOG("attempting to take stackshot with kernel-only flag"); + take_stackshot(&scenario, ^(void *ssbuf, size_t sslen) { + parse_stackshot(false, ssbuf, sslen); + }); +} + +#pragma mark performance tests + +#define SHOULD_REUSE_SIZE_HINT 0x01 +#define SHOULD_USE_DELTA 0x02 +#define SHOULD_TARGET_SELF 0x04 + +static void +stackshot_perf(unsigned int options) +{ + struct scenario scenario = { + .flags = (STACKSHOT_SAVE_LOADINFO | STACKSHOT_GET_GLOBAL_MEM_STATS + | STACKSHOT_SAVE_IMP_DONATION_PIDS | STACKSHOT_KCDATA_FORMAT), + }; + + dt_stat_t size = dt_stat_create("bytes", "size"); + dt_stat_time_t duration = dt_stat_time_create("duration"); + scenario.timer = duration; + + if (options & SHOULD_TARGET_SELF) { + scenario.target_pid = getpid(); + } + + while (!dt_stat_stable(duration) || !dt_stat_stable(size)) { + __block uint64_t last_time = 0; + __block uint32_t size_hint = 0; + take_stackshot(&scenario, ^(void *ssbuf, size_t sslen) { + dt_stat_add(size, (double)sslen); + last_time = stackshot_timestamp(ssbuf, sslen); + size_hint = (uint32_t)sslen; + }); + if (options & SHOULD_USE_DELTA) { + scenario.since_timestamp = last_time; + scenario.flags |= STACKSHOT_COLLECT_DELTA_SNAPSHOT; + } + if (options & SHOULD_REUSE_SIZE_HINT) { + scenario.size_hint = size_hint; + } + } + + dt_stat_finalize(duration); + dt_stat_finalize(size); +} + +T_DECL(perf_no_size_hint, "test stackshot performance with no size hint") +{ + stackshot_perf(0); +} + +T_DECL(perf_size_hint, "test stackshot performance with size hint") +{ + stackshot_perf(SHOULD_REUSE_SIZE_HINT); +} + +T_DECL(perf_process, "test stackshot performance targeted at process") +{ + stackshot_perf(SHOULD_REUSE_SIZE_HINT | SHOULD_TARGET_SELF); +} + +T_DECL(perf_delta, "test delta stackshot performance") +{ + stackshot_perf(SHOULD_REUSE_SIZE_HINT | SHOULD_USE_DELTA); +} + +T_DECL(perf_delta_process, "test delta stackshot performance targeted at a process") +{ + stackshot_perf(SHOULD_REUSE_SIZE_HINT | SHOULD_USE_DELTA | SHOULD_TARGET_SELF); +} + +static uint64_t +stackshot_timestamp(void *ssbuf, size_t sslen) +{ + kcdata_iter_t iter = kcdata_iter(ssbuf, sslen); + + uint32_t type = kcdata_iter_type(iter); + if (type != KCDATA_BUFFER_BEGIN_STACKSHOT && type != KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT) { + T_ASSERT_FAIL("invalid kcdata type %u", kcdata_iter_type(iter)); + } + + iter = kcdata_iter_find_type(iter, KCDATA_TYPE_MACH_ABSOLUTE_TIME); + T_QUIET; + T_ASSERT_TRUE(kcdata_iter_valid(iter), "timestamp found in stackshot"); + + return *(uint64_t *)kcdata_iter_payload(iter); +} + +#define TEST_THREAD_NAME "stackshot_test_thread" + +static void +parse_stackshot(bool delta, void *ssbuf, size_t sslen) +{ + kcdata_iter_t iter = kcdata_iter(ssbuf, sslen); + if (delta) { + T_ASSERT_EQ(kcdata_iter_type(iter), KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT, + "buffer provided is a delta stackshot"); + } else { + T_ASSERT_EQ(kcdata_iter_type(iter), KCDATA_BUFFER_BEGIN_STACKSHOT, + "buffer provided is a stackshot"); + } + + iter = kcdata_iter_next(iter); + KCDATA_ITER_FOREACH(iter) { + NSError *error = nil; + + switch (kcdata_iter_type(iter)) { + case KCDATA_TYPE_ARRAY: { + T_QUIET; + T_ASSERT_TRUE(kcdata_iter_array_valid(iter), + "checked that array is valid"); + + NSMutableDictionary *array = parseKCDataArray(iter, &error); + T_QUIET; T_ASSERT_NOTNULL(array, "parsed array from stackshot"); + T_QUIET; T_ASSERT_NULL(error, "error unset after parsing array"); + break; + } + + case KCDATA_TYPE_CONTAINER_BEGIN: { + T_QUIET; + T_ASSERT_TRUE(kcdata_iter_container_valid(iter), + "checked that container is valid"); + + NSDictionary *container = parseKCDataContainer(&iter, &error); + T_QUIET; T_ASSERT_NOTNULL(container, "parsed container from stackshot"); + T_QUIET; T_ASSERT_NULL(error, "error unset after parsing container"); + + if (kcdata_iter_container_type(iter) != STACKSHOT_KCCONTAINER_TASK) { + break; + } + int pid = [container[@"task_snapshots"][@"task_snapshot"][@"ts_pid"] intValue]; + if (pid != getpid()) { + break; + } + + T_EXPECT_EQ_STR(current_process_name(), + [container[@"task_snapshots"][@"task_snapshot"][@"ts_p_comm"] UTF8String], + "current process name matches in stackshot"); + + T_QUIET; + T_EXPECT_LE(pid, [container[@"task_snapshots"][@"task_snapshot"][@"ts_unique_pid"] intValue], + "unique pid is greater than pid"); + + bool found_main_thread = 0; + for (id thread_key in container[@"task_snapshots"][@"thread_snapshots"]) { + NSMutableDictionary *thread = container[@"task_snapshots"][@"thread_snapshots"][thread_key]; + NSDictionary *thread_snap = thread[@"thread_snapshot"]; + + T_QUIET; T_EXPECT_GT([thread_snap[@"ths_thread_id"] intValue], 0, + "thread ID of thread in current task is valid"); + T_QUIET; T_EXPECT_GT([thread_snap[@"ths_total_syscalls"] intValue], 0, + "total syscalls of thread in current task is valid"); + T_QUIET; T_EXPECT_GT([thread_snap[@"ths_base_priority"] intValue], 0, + "base priority of thread in current task is valid"); + T_QUIET; T_EXPECT_GT([thread_snap[@"ths_sched_priority"] intValue], 0, + "scheduling priority of thread in current task is valid"); + + NSString *pth_name = thread_snap[@"pth_name"]; + if (pth_name != nil && [pth_name isEqualToString:@TEST_THREAD_NAME]) { + found_main_thread = true; + } + } + T_EXPECT_TRUE(found_main_thread, "found main thread for current task in stackshot"); + break; + } + } + } + + T_ASSERT_FALSE(KCDATA_ITER_FOREACH_FAILED(iter), "successfully iterated kcdata"); +} + +static const char * +current_process_name(void) +{ + static char name[64]; + + if (!name[0]) { + int ret = proc_name(getpid(), name, sizeof(name)); + T_QUIET; + T_ASSERT_POSIX_ZERO(ret, "proc_pidname failed for current process"); + } + + return name; +} + +static void +initialize_thread(void) +{ + int ret = pthread_setname_np(TEST_THREAD_NAME); + T_QUIET; + T_ASSERT_POSIX_ZERO(ret, "set thread name to %s", TEST_THREAD_NAME); +} diff --git a/tools/tests/darwintests/task_for_pid_entitlement.plist b/tools/tests/darwintests/task_for_pid_entitlement.plist new file mode 100644 index 000000000..2398d67f5 --- /dev/null +++ b/tools/tests/darwintests/task_for_pid_entitlement.plist @@ -0,0 +1,10 @@ + + + + + com.apple.system-task-ports + + task_for_pid-allow + + + diff --git a/tools/tests/darwintests/task_info.c b/tools/tests/darwintests/task_info.c new file mode 100644 index 000000000..516bf4a6b --- /dev/null +++ b/tools/tests/darwintests/task_info.c @@ -0,0 +1,907 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* ************************************************************************************* + * Test the task_info API. + * + * This is a functional test of the following APIs: + * TASK_BASIC_INFO_32 + * TASK_BASIC2_INFO_32 + * TASK_BASIC_INFO_64 + * TASK_BASIC_INFO_64_2 + * TASK_POWER_INFO_V2 + * TASK_FLAGS_INFO + * TASK_AFFINITY_TAG_INFO + * TASK_THREAD_TIMES_INFO + * TASK_ABSOLUTE_TIME_INFO + * Add tests to increase code coverage for the task_info API + * ************************************************************************************* + */ +#define TESTPHYSFOOTPRINTVAL 5 +#define CANARY 0x0f0f0f0f0f0f0f0fULL +#if !defined(CONFIG_EMBEDDED) +#define ABSOLUTE_MIN_USER_TIME_DIFF 150 +#define ABSOLUTE_MIN_SYSTEM_TIME_DIFF 300 +#endif + +enum info_kind { INFO_32, INFO_64, INFO_32_2, INFO_64_2, INFO_MACH, INFO_MAX }; + +enum info_get { GET_SUSPEND_COUNT, GET_RESIDENT_SIZE, GET_VIRTUAL_SIZE, GET_USER_TIME, GET_SYS_TIME, GET_POLICY, GET_MAX_RES }; + +/* + * This function uses CPU cycles by doing a factorial computation. + */ +static void do_factorial_task(void); + +void test_task_basic_info_32(void); +void test_task_basic_info_64(void); +void task_basic_info_32_debug(void); +void task_basic2_info_32_warmup(void); +static int is_development_kernel(void); +void test_task_basic_info(enum info_kind kind); +uint64_t info_get(enum info_kind kind, enum info_get get, void * data); + +T_DECL(task_vm_info, "tests task vm info", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + kern_return_t err; + task_vm_info_data_t vm_info; + + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + + err = task_info(mach_task_self(), TASK_VM_INFO_PURGEABLE, (task_info_t)&vm_info, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + T_EXPECT_NE(vm_info.virtual_size, 0ULL, "task_info return value !=0 for virtual_size\n"); + + T_EXPECT_NE(vm_info.phys_footprint, 0ULL, "task_info return value !=0 for phys_footprint\n"); + + /* + * Test the REV0 version of TASK_VM_INFO. It should not change the value of phys_footprint. + */ + + count = TASK_VM_INFO_REV0_COUNT; + vm_info.phys_footprint = TESTPHYSFOOTPRINTVAL; + vm_info.min_address = CANARY; + vm_info.max_address = CANARY; + + err = task_info(mach_task_self(), TASK_VM_INFO_PURGEABLE, (task_info_t)&vm_info, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + T_EXPECT_EQ(count, TASK_VM_INFO_REV0_COUNT, "task_info count(%d) is equal to TASK_VM_INFO_REV0_COUNT", count); + + T_EXPECT_NE(vm_info.virtual_size, 0ULL, "task_info --rev0 call does not return 0 for virtual_size"); + + T_EXPECT_EQ(vm_info.phys_footprint, (unsigned long long)TESTPHYSFOOTPRINTVAL, + "task_info --rev0 call returned value %llu for vm_info.phys_footprint. Expected %u since this value should not be " + "modified by rev0", + vm_info.phys_footprint, TESTPHYSFOOTPRINTVAL); + + T_EXPECT_EQ(vm_info.min_address, CANARY, + "task_info --rev0 call returned value 0x%llx for vm_info.min_address. Expected 0x%llx since this value should not " + "be modified by rev0", + vm_info.min_address, CANARY); + + T_EXPECT_EQ(vm_info.max_address, CANARY, + "task_info --rev0 call returned value 0x%llx for vm_info.max_address. Expected 0x%llx since this value should not " + "be modified by rev0", + vm_info.max_address, CANARY); + + /* + * Test the REV1 version of TASK_VM_INFO. + */ + + count = TASK_VM_INFO_REV1_COUNT; + vm_info.phys_footprint = TESTPHYSFOOTPRINTVAL; + vm_info.min_address = CANARY; + vm_info.max_address = CANARY; + + err = task_info(mach_task_self(), TASK_VM_INFO_PURGEABLE, (task_info_t)&vm_info, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + T_EXPECT_EQ(count, TASK_VM_INFO_REV1_COUNT, "task_info count(%d) is equal to TASK_VM_INFO_REV1_COUNT", count); + + T_EXPECT_NE(vm_info.virtual_size, 0ULL, "task_info --rev1 call does not return 0 for virtual_size"); + + T_EXPECT_NE(vm_info.phys_footprint, (unsigned long long)TESTPHYSFOOTPRINTVAL, + "task_info --rev1 call returned value %llu for vm_info.phys_footprint. Expected value is anything other than %u " + "since this value should not be modified by rev1", + vm_info.phys_footprint, TESTPHYSFOOTPRINTVAL); + + T_EXPECT_EQ(vm_info.min_address, CANARY, + "task_info --rev1 call returned value 0x%llx for vm_info.min_address. Expected 0x%llx since this value should not " + "be modified by rev1", + vm_info.min_address, CANARY); + + T_EXPECT_EQ(vm_info.max_address, CANARY, + "task_info --rev1 call returned value 0x%llx for vm_info.max_address. Expected 0x%llx since this value should not " + "be modified by rev1", + vm_info.max_address, CANARY); + + /* + * Test the REV2 version of TASK_VM_INFO. + */ + + count = TASK_VM_INFO_REV2_COUNT; + vm_info.phys_footprint = TESTPHYSFOOTPRINTVAL; + vm_info.min_address = CANARY; + vm_info.max_address = CANARY; + + err = task_info(mach_task_self(), TASK_VM_INFO_PURGEABLE, (task_info_t)&vm_info, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + T_EXPECT_EQ(count, TASK_VM_INFO_REV2_COUNT, "task_info count(%d) is equal to TASK_VM_INFO_REV2_COUNT\n", count); + + T_EXPECT_NE(vm_info.virtual_size, 0ULL, "task_info --rev2 call does not return 0 for virtual_size\n"); + + T_EXPECT_NE(vm_info.phys_footprint, (unsigned long long)TESTPHYSFOOTPRINTVAL, + "task_info --rev2 call returned value %llu for vm_info.phys_footprint. Expected anything other than %u since this " + "value should be modified by rev2", + vm_info.phys_footprint, TESTPHYSFOOTPRINTVAL); + + T_EXPECT_NE(vm_info.min_address, CANARY, + "task_info --rev2 call returned value 0x%llx for vm_info.min_address. Expected anything other than 0x%llx since " + "this value should be modified by rev2", + vm_info.min_address, CANARY); + + T_EXPECT_NE(vm_info.max_address, CANARY, + "task_info --rev2 call returned value 0x%llx for vm_info.max_address. Expected anything other than 0x%llx since " + "this value should be modified by rev2", + vm_info.max_address, CANARY); +} + +T_DECL(host_debug_info, "tests host debug info", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + T_SETUPBEGIN; + int is_dev = is_development_kernel(); + T_QUIET; + T_ASSERT_TRUE(is_dev, "verify development kernel is running"); + T_SETUPEND; + + kern_return_t err; + mach_port_t host; + host_debug_info_internal_data_t debug_info; + mach_msg_type_number_t count = HOST_DEBUG_INFO_INTERNAL_COUNT; + host = mach_host_self(); + err = host_info(host, HOST_DEBUG_INFO_INTERNAL, (host_info_t)&debug_info, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify host_info call succeeded"); +} + +T_DECL(task_debug_info, "tests task debug info", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + T_SETUPBEGIN; + int is_dev = is_development_kernel(); + T_QUIET; + T_ASSERT_TRUE(is_dev, "verify development kernel is running"); + T_SETUPEND; + + kern_return_t err; + task_debug_info_internal_data_t debug_info; + + mach_msg_type_number_t count = TASK_DEBUG_INFO_INTERNAL_COUNT; + + err = task_info(mach_task_self(), TASK_DEBUG_INFO_INTERNAL, (task_info_t)&debug_info, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); +} + +T_DECL(thread_debug_info, "tests thread debug info", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + T_SETUPBEGIN; + int is_dev = is_development_kernel(); + T_QUIET; + T_ASSERT_TRUE(is_dev, "verify development kernel is running"); + T_SETUPEND; + + kern_return_t err; + thread_debug_info_internal_data_t debug_info; + + mach_msg_type_number_t count = THREAD_DEBUG_INFO_INTERNAL_COUNT; + + err = thread_info(mach_thread_self(), THREAD_DEBUG_INFO_INTERNAL, (thread_info_t)&debug_info, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); +} + +static void +do_factorial_task() +{ + int number = 20; + int factorial = 1; + int i; + for (i = 1; i <= number; i++) { + factorial *= i; + } + + return; +} + +T_DECL(task_thread_times_info, "tests task thread times info", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + T_SETUPBEGIN; + int is_dev = is_development_kernel(); + T_QUIET; + T_ASSERT_TRUE(is_dev, "verify development kernel is running"); + T_SETUPEND; + + kern_return_t err; + task_thread_times_info_data_t thread_times_info_data; + task_thread_times_info_data_t thread_times_info_data_new; + mach_msg_type_number_t count = TASK_THREAD_TIMES_INFO_COUNT; + + err = task_info(mach_task_self(), TASK_THREAD_TIMES_INFO, (task_info_t)&thread_times_info_data, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + do_factorial_task(); + + err = task_info(mach_task_self(), TASK_THREAD_TIMES_INFO, (task_info_t)&thread_times_info_data_new, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + /* + * The difference is observed to be less than 30 microseconds for user_time + * and less than 50 microseconds for system_time. This observation was done for over + * 1000 runs. + */ + + T_EXPECT_FALSE((thread_times_info_data_new.user_time.seconds - thread_times_info_data.user_time.seconds) != 0 || + (thread_times_info_data_new.system_time.seconds - thread_times_info_data.system_time.seconds) != 0, + "Tests whether the difference between thread times is greater than the allowed limit"); + + /* + * This is a negative case. + */ + + count--; + err = task_info(mach_task_self(), TASK_THREAD_TIMES_INFO, (task_info_t)&thread_times_info_data, &count); + T_ASSERT_MACH_ERROR(err, KERN_INVALID_ARGUMENT, + "Negative test case: task_info should verify that count is at least equal to what is defined in API."); +} + +T_DECL(task_absolutetime_info, "tests task absolute time info", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + T_SETUPBEGIN; + int is_dev = is_development_kernel(); + T_QUIET; + T_ASSERT_TRUE(is_dev, "verify development kernel is running"); + T_SETUPEND; + + kern_return_t err; + uint64_t user_time_diff, system_time_diff; + task_absolutetime_info_data_t absolute_time_info_data; + task_absolutetime_info_data_t absolute_time_info_data_new; + mach_msg_type_number_t count = TASK_ABSOLUTETIME_INFO_COUNT; + + err = task_info(mach_task_self(), TASK_ABSOLUTETIME_INFO, (task_info_t)&absolute_time_info_data, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + do_factorial_task(); + + err = task_info(mach_task_self(), TASK_ABSOLUTETIME_INFO, (task_info_t)&absolute_time_info_data_new, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + user_time_diff = absolute_time_info_data_new.total_user - absolute_time_info_data.total_user; + system_time_diff = absolute_time_info_data_new.total_system - absolute_time_info_data.total_system; + +#if !(defined(__arm__) || defined(__arm64__)) + /* + * On embedded devices the difference is always zero. + * On non-embedded devices the difference occurs in this range. This was observed over ~10000 runs. + */ + + T_EXPECT_FALSE(user_time_diff < ABSOLUTE_MIN_USER_TIME_DIFF || system_time_diff < ABSOLUTE_MIN_SYSTEM_TIME_DIFF, + "Tests whether the difference between thread times is greater than the expected range"); +#endif + + /* + * There is no way of estimating the exact number of threads, hence checking the counter to be non-zero for now. + */ + + T_EXPECT_NE(absolute_time_info_data.threads_user, 0ULL, "task_info should return non-zero number of user threads"); + +#if !(defined(__arm__) || defined(__arm64__)) + /* + * On iOS, system threads are always zero. On OS X this value can be some large positive number. + * There is no real way to estimate the exact amount. + */ + T_EXPECT_NE(absolute_time_info_data.threads_system, 0ULL, "task_info should return non-zero number of system threads"); +#endif + + /* + * This is a negative case. + */ + count--; + err = task_info(mach_task_self(), TASK_ABSOLUTETIME_INFO, (task_info_t)&absolute_time_info_data_new, &count); + T_ASSERT_MACH_ERROR(err, KERN_INVALID_ARGUMENT, + "Negative test case: task_info should verify that count is at least equal to what is defined in API."); +} + +T_DECL(task_affinity_tag_info, "tests task_affinity_tag_info", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + T_SETUPBEGIN; + int is_dev = is_development_kernel(); + T_QUIET; + T_ASSERT_TRUE(is_dev, "verify development kernel is running"); + T_SETUPEND; + + kern_return_t err; + task_affinity_tag_info_data_t affinity_tag_info_data; + mach_msg_type_number_t count = TASK_AFFINITY_TAG_INFO_COUNT; + + err = task_info(mach_task_self(), TASK_AFFINITY_TAG_INFO, (task_info_t)&affinity_tag_info_data, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + /* + * The affinity is not set by default, hence expecting a zero value. + */ + T_ASSERT_FALSE(affinity_tag_info_data.min != 0 || affinity_tag_info_data.max != 0, + "task_info call returns non-zero min or max value"); + + /* + * This is a negative case. + */ + count--; + err = task_info(mach_task_self(), TASK_AFFINITY_TAG_INFO, (task_info_t)&affinity_tag_info_data, &count); + T_ASSERT_MACH_ERROR(err, KERN_INVALID_ARGUMENT, + "Negative test case: task_info should verify that count is at least equal to what is defined in API."); +} + +T_DECL(task_flags_info, "tests task_flags_info", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + T_SETUPBEGIN; + int is_dev = is_development_kernel(); + T_QUIET; + T_ASSERT_TRUE(is_dev, "verify development kernel is running"); + T_SETUPEND; + + kern_return_t err; + task_flags_info_data_t flags_info_data; + mach_msg_type_number_t count = TASK_FLAGS_INFO_COUNT; + + err = task_info(mach_task_self(), TASK_FLAGS_INFO, (task_info_t)&flags_info_data, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + /* Change for 32-bit arch possibility?*/ + T_ASSERT_EQ((flags_info_data.flags & (unsigned int)(~TF_LP64)), 0U, "task_info should only give out 64-bit addr flag"); + + /* + * This is a negative case. + */ + + count--; + err = task_info(mach_task_self(), TASK_FLAGS_INFO, (task_info_t)&flags_info_data, &count); + T_ASSERT_MACH_ERROR(err, KERN_INVALID_ARGUMENT, + "Negative test case: task_info should verify that count is at least equal to what is defined in API."); +} + +T_DECL(task_power_info_v2, "tests task_power_info_v2", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + T_SETUPBEGIN; + int is_dev = is_development_kernel(); + T_QUIET; + T_ASSERT_TRUE(is_dev, "verify development kernel is running"); + T_SETUPEND; + + kern_return_t err; + task_power_info_v2_data_t power_info_data_v2; + task_power_info_v2_data_t power_info_data_v2_new; + mach_msg_type_number_t count = TASK_POWER_INFO_V2_COUNT; + + sleep(1); + + err = task_info(mach_task_self(), TASK_POWER_INFO_V2, (task_info_t)&power_info_data_v2, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + T_ASSERT_LE(power_info_data_v2.gpu_energy.task_gpu_utilisation, 0ULL, + "verified task_info call shows zero GPU utilization for non-GPU task"); + + do_factorial_task(); + + /* + * Verify the cpu_energy parameters. + */ + err = task_info(mach_task_self(), TASK_POWER_INFO_V2, (task_info_t)&power_info_data_v2_new, &count); + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + +#if !(defined(__arm__) || defined(__arm64__)) + /* + * iOS does not have system_time. + */ + T_ASSERT_GT(power_info_data_v2_new.cpu_energy.total_user, power_info_data_v2.cpu_energy.total_user, + "task_info call returns valid user time"); + T_ASSERT_GT(power_info_data_v2_new.cpu_energy.total_system, power_info_data_v2.cpu_energy.total_system, + "task_info call returns valid system time"); +#endif + + T_ASSERT_GE(power_info_data_v2.cpu_energy.task_interrupt_wakeups, 1ULL, + "verify task_info call returns non-zero value for interrupt_wakeup (ret value = %llu)", + power_info_data_v2.cpu_energy.task_interrupt_wakeups); + +#if !(defined(__arm__) || defined(__arm64__)) + if (power_info_data_v2.cpu_energy.task_platform_idle_wakeups != 0) { + T_LOG("task_info call returned %llu for platform_idle_wakeup", power_info_data_v2.cpu_energy.task_platform_idle_wakeups); + } +#endif + + count = TASK_POWER_INFO_V2_COUNT_OLD; + err = task_info(mach_task_self(), TASK_POWER_INFO_V2, (task_info_t)&power_info_data_v2, &count); + + T_ASSERT_MACH_SUCCESS(err, "verify task_info call succeeded"); + + /* + * This is a negative case. + */ + count--; + err = task_info(mach_task_self(), TASK_POWER_INFO_V2, (task_info_t)&power_info_data_v2, &count); + + T_ASSERT_MACH_ERROR(err, KERN_INVALID_ARGUMENT, + "Negative test case: task_info should verify that count is at least equal to what is defined in API. Call " + "returns errno %d:%s", + err, mach_error_string(err)); +} + +T_DECL(test_task_basic_info_32, "tests TASK_BASIC_INFO_32", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + test_task_basic_info(INFO_32); +} + +T_DECL(test_task_basic_info_32_2, "tests TASK_BASIC_INFO_32_2", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + test_task_basic_info(INFO_32_2); +} + +#if defined(__arm__) || defined(__arm64__) +T_DECL(test_task_basic_info_64i_2, "tests TASK_BASIC_INFO_64_2", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + test_task_basic_info(INFO_64_2); +} +#else +T_DECL(test_task_basic_info_64, "tests TASK_BASIC_INFO_64", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + test_task_basic_info(INFO_64); +} +#endif /* defined(__arm__) || defined(__arm64__) */ + +T_DECL(test_mach_task_basic_info, "tests MACH_TASK_BASIC_INFO", T_META_ASROOT(true), T_META_LTEPHASE(LTE_POSTINIT)) +{ + test_task_basic_info(INFO_MACH); +} + +void +test_task_basic_info(enum info_kind kind) +{ +#define BEFORE 0 +#define AFTER 1 + + T_SETUPBEGIN; + int is_dev = is_development_kernel(); + T_QUIET; + T_ASSERT_TRUE(is_dev, "verify development kernel is running"); + T_SETUPEND; + + task_info_t info_data[2]; + task_basic_info_32_data_t basic_info_32_data[2]; +#if defined(__arm__) || defined(__arm64__) + task_basic_info_64_2_data_t basic_info_64_2_data[2]; +#else + task_basic_info_64_data_t basic_info_64_data[2]; +#endif /* defined(__arm__) || defined(__arm64__) */ + mach_task_basic_info_data_t mach_basic_info_data[2]; + + kern_return_t kr; + mach_msg_type_number_t count; + task_flavor_t flavor = 0; + integer_t suspend_count; + uint64_t resident_size_diff; + uint64_t virtual_size_diff; + + void * tmp_map = NULL; + pid_t child_pid; + mach_port_name_t child_task; + /*for dt_waitpid*/ + int timeout = 10; // change to max timeout + int exit_status = 0; + + switch (kind) { + case INFO_32: + case INFO_32_2: + info_data[BEFORE] = (task_info_t)&basic_info_32_data[BEFORE]; + info_data[AFTER] = (task_info_t)&basic_info_32_data[AFTER]; + count = TASK_BASIC_INFO_32_COUNT; + flavor = TASK_BASIC_INFO_32; + + if (kind == INFO_32_2) { + flavor = TASK_BASIC2_INFO_32; + } + + break; +#if defined(__arm__) || defined(__arm64__) + case INFO_64: + T_ASSERT_FAIL("invalid basic info kind"); + break; + + case INFO_64_2: + info_data[BEFORE] = (task_info_t)&basic_info_64_2_data[BEFORE]; + info_data[AFTER] = (task_info_t)&basic_info_64_2_data[AFTER]; + count = TASK_BASIC_INFO_64_2_COUNT; + flavor = TASK_BASIC_INFO_64_2; + break; + +#else + case INFO_64: + info_data[BEFORE] = (task_info_t)&basic_info_64_data[BEFORE]; + info_data[AFTER] = (task_info_t)&basic_info_64_data[AFTER]; + count = TASK_BASIC_INFO_64_COUNT; + flavor = TASK_BASIC_INFO_64; + break; + + case INFO_64_2: + T_ASSERT_FAIL("invalid basic info kind"); + break; +#endif /* defined(__arm__) || defined(__arm64__) */ + case INFO_MACH: + info_data[BEFORE] = (task_info_t)&mach_basic_info_data[BEFORE]; + info_data[AFTER] = (task_info_t)&mach_basic_info_data[AFTER]; + count = MACH_TASK_BASIC_INFO_COUNT; + flavor = MACH_TASK_BASIC_INFO; + break; + case INFO_MAX: + default: + T_ASSERT_FAIL("invalid basic info kind"); + break; + } + + kr = task_info(mach_task_self(), flavor, info_data[BEFORE], &count); + + T_ASSERT_MACH_SUCCESS(kr, "verify task_info succeeded"); + + do_factorial_task(); + + /* + * Allocate virtual and resident memory. + */ + tmp_map = mmap(0, PAGE_SIZE, PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); + + T_WITH_ERRNO; + T_EXPECT_NE(tmp_map, MAP_FAILED, "verify mmap call is successful"); + + memset(tmp_map, 'm', PAGE_SIZE); + + child_pid = fork(); + + T_ASSERT_POSIX_SUCCESS(child_pid, "verify process can be forked"); + + if (child_pid == 0) { + /* + * This will suspend the child process. + */ + kr = task_suspend(mach_task_self()); + exit(kr); + } + + /* + * Wait for the child process to suspend itself. + */ + sleep(1); + + kr = task_for_pid(mach_task_self(), child_pid, &child_task); + T_ASSERT_MACH_SUCCESS(kr, "verify task_for_pid succeeded. check sudo if failed"); + + /* + * Verify the suspend_count for child and resume it. + */ + + kr = task_info(child_task, flavor, info_data[AFTER], &count); + T_ASSERT_MACH_SUCCESS(kr, "verify task_info call succeeded"); + + suspend_count = (integer_t)(info_get(kind, GET_SUSPEND_COUNT, info_data[AFTER])); + T_ASSERT_EQ(suspend_count, 1, "verify task_info shows correct suspend_count"); + + kr = task_resume(child_task); + T_ASSERT_MACH_SUCCESS(kr, "verify task_resume succeeded"); + + /* + * reap kr from task_suspend call in child + */ + if (dt_waitpid(child_pid, &exit_status, NULL, timeout)) { + T_ASSERT_MACH_SUCCESS(exit_status, "verify child task_suspend is successful"); + } else { + T_FAIL("dt_waitpid failed"); + } + + kr = task_info(mach_task_self(), flavor, info_data[AFTER], &count); + T_ASSERT_MACH_SUCCESS(kr, "verify task_info call succeeded"); + + resident_size_diff = info_get(kind, GET_RESIDENT_SIZE, info_data[AFTER]) - info_get(kind, GET_RESIDENT_SIZE, info_data[BEFORE]); + virtual_size_diff = info_get(kind, GET_VIRTUAL_SIZE, info_data[AFTER]) - info_get(kind, GET_VIRTUAL_SIZE, info_data[BEFORE]); + + /* + * INFO_32_2 gets the max resident size instead of the current resident size + * 32 KB tolerance built into test. The returned value is generally between 0 and 16384 + * + * max resident size is a discrete field in INFO_MACH, so it's handled differently + */ + if (kind == INFO_32_2) { + T_EXPECT_EQ(resident_size_diff % 4096, 0ULL, "verify task_info returns valid max resident_size"); + T_EXPECT_GE(resident_size_diff, 0ULL, "verify task_info returns non-negative max resident_size"); + T_EXPECT_GE(virtual_size_diff, (unsigned long long)PAGE_SIZE, "verify task_info returns valid virtual_size"); + } else { + T_EXPECT_GE(resident_size_diff, (unsigned long long)PAGE_SIZE, "task_info returns valid resident_size"); + T_EXPECT_GE(virtual_size_diff, (unsigned long long)PAGE_SIZE, "task_info returns valid virtual_size"); + } + + if (kind == INFO_MACH) { + resident_size_diff = info_get(kind, GET_MAX_RES, info_data[AFTER]) - info_get(kind, GET_MAX_RES, info_data[BEFORE]); + T_EXPECT_EQ(resident_size_diff % 4096, 0ULL, "verify task_info returns valid max resident_size"); + T_EXPECT_GE(resident_size_diff, 0ULL, "verify task_info returns non-negative max resident_size"); + T_EXPECT_GE(info_get(kind, GET_MAX_RES, info_data[AFTER]), info_get(kind, GET_RESIDENT_SIZE, info_data[AFTER]), + "verify max resident size is greater than or equal to curr resident size"); + } + + do_factorial_task(); + + /* + * These counters give time for threads that have terminated. We dont have any, so checking for zero. + */ + + time_value_t * user_tv = (time_value_t *)(info_get(kind, GET_USER_TIME, info_data[BEFORE])); + T_EXPECT_EQ((user_tv->seconds + user_tv->microseconds / 1000000), 0, "verify task_info shows valid user time"); + + time_value_t * sys_tv = (time_value_t *)(info_get(kind, GET_SYS_TIME, info_data[BEFORE])); + T_EXPECT_EQ(sys_tv->seconds + (sys_tv->microseconds / 1000000), 0, "verify task_info shows valid system time"); + + /* + * The default value for non-kernel tasks is TIMESHARE. + */ + + policy_t pt = (policy_t)info_get(kind, GET_POLICY, info_data[BEFORE]); + + T_EXPECT_EQ(pt, POLICY_TIMESHARE, "verify task_info shows valid policy"); + + /* + * This is a negative case. + */ + + count--; + kr = task_info(mach_task_self(), flavor, info_data[AFTER], &count); + + T_ASSERT_MACH_ERROR(kr, KERN_INVALID_ARGUMENT, + "Negative test case: task_info should verify that count is at least equal to what is defined in API"); + + /* + * deallocate memory + */ + munmap(tmp_map, PAGE_SIZE); + + return; + +#undef BEFORE +#undef AFTER +} + +uint64_t +info_get(enum info_kind kind, enum info_get get, void * data) +{ + switch (get) { + case GET_SUSPEND_COUNT: + switch (kind) { + case INFO_32: + case INFO_32_2: + return (uint64_t)(((task_basic_info_32_t)data)->suspend_count); +#if defined(__arm__) || defined(__arm64__) + case INFO_64: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; + + case INFO_64_2: + return (uint64_t)(((task_basic_info_64_2_t)data)->suspend_count); +#else + case INFO_64: + return (uint64_t)(((task_basic_info_64_t)data)->suspend_count); + + case INFO_64_2: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; +#endif /* defined(__arm__) || defined(__arm64__) */ + case INFO_MACH: + return (uint64_t)(((mach_task_basic_info_t)data)->suspend_count); + case INFO_MAX: + default: + T_ASSERT_FAIL("unhandled info_get %d %d", kind, get); + } + case GET_RESIDENT_SIZE: + switch (kind) { + case INFO_32: + case INFO_32_2: + return (uint64_t)(((task_basic_info_32_t)data)->resident_size); +#if defined(__arm__) || defined(__arm64__) + case INFO_64: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; + + case INFO_64_2: + return (uint64_t)(((task_basic_info_64_2_t)data)->resident_size); +#else + case INFO_64: + return (uint64_t)(((task_basic_info_64_t)data)->resident_size); + + case INFO_64_2: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; +#endif /* defined(__arm__) || defined(__arm64__) */ + case INFO_MACH: + return (uint64_t)(((mach_task_basic_info_t)data)->resident_size); + case INFO_MAX: + default: + T_ASSERT_FAIL("unhandled info_get %d %d", kind, get); + } + case GET_VIRTUAL_SIZE: + switch (kind) { + case INFO_32: + case INFO_32_2: + return (uint64_t)(((task_basic_info_32_t)data)->virtual_size); +#if defined(__arm__) || defined(__arm64__) + case INFO_64: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; + + case INFO_64_2: + return (uint64_t)(((task_basic_info_64_2_t)data)->virtual_size); +#else + case INFO_64: + return (uint64_t)(((task_basic_info_64_t)data)->virtual_size); + + case INFO_64_2: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; +#endif /* defined(__arm__) || defined(__arm64__) */ + case INFO_MACH: + return (uint64_t)(((mach_task_basic_info_t)data)->virtual_size); + + case INFO_MAX: + default: + T_ASSERT_FAIL("unhandled info_get %d %d", kind, get); + } + case GET_USER_TIME: + switch (kind) { + case INFO_32: + case INFO_32_2: + return (uint64_t) & (((task_basic_info_32_t)data)->user_time); +#if defined(__arm__) || defined(__arm64__) + case INFO_64: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; + + case INFO_64_2: + return (uint64_t) & (((task_basic_info_64_2_t)data)->user_time); +#else + case INFO_64: + return (uint64_t) & (((task_basic_info_64_t)data)->user_time); + + case INFO_64_2: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; +#endif /* defined(__arm__) || defined(__arm64__) */ + case INFO_MACH: + return (uint64_t) & (((mach_task_basic_info_t)data)->user_time); + + case INFO_MAX: + default: + T_ASSERT_FAIL("unhandled info_get %d %d", kind, get); + } + case GET_SYS_TIME: + switch (kind) { + case INFO_32: + case INFO_32_2: + return (uint64_t) & (((task_basic_info_32_t)data)->system_time); +#if defined(__arm__) || defined(__arm64__) + case INFO_64: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; + + case INFO_64_2: + return (uint64_t) & (((task_basic_info_64_2_t)data)->system_time); +#else + case INFO_64: + return (uint64_t) & (((task_basic_info_64_t)data)->system_time); + + case INFO_64_2: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; +#endif /* defined(__arm__) || defined(__arm64__) */ + case INFO_MACH: + return (uint64_t) & (((mach_task_basic_info_t)data)->user_time); + case INFO_MAX: + default: + T_ASSERT_FAIL("unhandled info_get %d %d", kind, get); + } + case GET_POLICY: + switch (kind) { + case INFO_32: + case INFO_32_2: + return (uint64_t)(((task_basic_info_32_t)data)->policy); +#if defined(__arm__) || defined(__arm64__) + case INFO_64: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; + + case INFO_64_2: + return (uint64_t)(((task_basic_info_64_2_t)data)->policy); +#else + case INFO_64: + return (uint64_t)(((task_basic_info_64_t)data)->policy); + + case INFO_64_2: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + break; +#endif /* defined(__arm__) || defined(__arm64__) */ + case INFO_MACH: + return (uint64_t)(((mach_task_basic_info_t)data)->policy); + + case INFO_MAX: + default: + T_ASSERT_FAIL("unhandled info_get %d %d", kind, get); + } + case GET_MAX_RES: + switch (kind) { + case INFO_32: + case INFO_32_2: + case INFO_64: + case INFO_64_2: + T_ASSERT_FAIL("illegal info_get %d %d", kind, get); + case INFO_MACH: + return (uint64_t)(((mach_task_basic_info_t)data)->resident_size_max); + case INFO_MAX: + default: + T_ASSERT_FAIL("unhandled info_get %d %d", kind, get); + } + } + + __builtin_unreachable(); +} + +/* + * Determines whether we're running on a development kernel + */ +static int +is_development_kernel(void) +{ +#define NOTSET -1 + + static int is_dev = NOTSET; + + if (is_dev == NOTSET) { + int dev; + size_t dev_size = sizeof(dev); + + T_QUIET; + T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.development", &dev, &dev_size, NULL, 0), NULL); + is_dev = (dev != 0); + + return is_dev; + } else { + return is_dev; + } +#undef NOTSET +} diff --git a/tools/tests/darwintests/task_info_28439149.c b/tools/tests/darwintests/task_info_28439149.c new file mode 100644 index 000000000..9102ba600 --- /dev/null +++ b/tools/tests/darwintests/task_info_28439149.c @@ -0,0 +1,81 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void do_child(int *pipefd){ + int exit = 0; + + close(pipefd[1]); + read(pipefd[0], &exit, sizeof(int)); + T_QUIET; T_EXPECT_EQ_INT(exit, 1, "exit"); + close(pipefd[0]); +} + +T_DECL(task_info_28439149, "ensure that task_info has the correct permission", + T_META_CHECK_LEAKS(false), T_META_ASROOT(true)) +{ + int pipefd[2]; + + T_QUIET; T_ASSERT_POSIX_SUCCESS(pipe(pipefd), "pipe"); + + int pid = fork(); + T_QUIET; T_ASSERT_POSIX_SUCCESS(pid, "fork"); + + if (pid == 0) { + do_child(pipefd); + return; + } + + close(pipefd[0]); + + int exit; + mach_msg_type_number_t count; + struct task_basic_info_64 ti; + task_dyld_info_data_t di; + + task_t self = mach_task_self(); + task_t other_name; + task_t other; + int ret; + + T_EXPECT_MACH_SUCCESS(task_for_pid(self, pid, &other), NULL); + T_EXPECT_MACH_SUCCESS(task_name_for_pid(self, pid, &other_name), NULL); + + count = TASK_BASIC_INFO_64_COUNT; + T_EXPECT_MACH_SUCCESS(task_info(self, TASK_BASIC_INFO_64, (task_info_t)&ti, + &count), "task_info(self, TASK_BASIC_INFO_64 ...)"); + count = TASK_BASIC_INFO_64_COUNT; + T_EXPECT_MACH_SUCCESS(task_info(other, TASK_BASIC_INFO_64, (task_info_t)&ti, + &count), "task_info(other_name, TASK_BASIC_INFO_64 ...)"); + count = TASK_BASIC_INFO_64_COUNT; + T_EXPECT_MACH_SUCCESS(task_info(other_name, TASK_BASIC_INFO_64, (task_info_t)&ti, + &count), "task_info(other_name, TASK_BASIC_INFO_64 ...)"); + + + count = TASK_DYLD_INFO_COUNT; + T_EXPECT_MACH_SUCCESS(task_info(self, TASK_DYLD_INFO, (task_info_t)&di, + &count), "task_info(self, TASK_DYLD_INFO ...)"); + count = TASK_DYLD_INFO_COUNT; + T_EXPECT_MACH_SUCCESS(task_info(other, TASK_DYLD_INFO, (task_info_t)&di, + &count), "task_info(other_name, TASK_DYLD_INFO ...)"); + count = TASK_DYLD_INFO_COUNT; + ret = task_info(other_name, TASK_DYLD_INFO, (task_info_t)&di, &count); + T_EXPECT_EQ_INT(ret, KERN_INVALID_ARGUMENT, "task info TASK_DYLD_INFO should fail with mach_port_name"); + + exit = 1; + write(pipefd[1], &exit, sizeof(int)); + close(pipefd[1]); + + wait(NULL); +} + diff --git a/tools/tests/darwintests/task_inspect.c b/tools/tests/darwintests/task_inspect.c index 468ae8e2b..f16064a1f 100644 --- a/tools/tests/darwintests/task_inspect.c +++ b/tools/tests/darwintests/task_inspect.c @@ -1,3 +1,7 @@ +#ifdef T_NAMESPACE +#undef T_NAMESPACE +#endif + #include #include @@ -55,11 +59,20 @@ attempt_kernel_inspection(task_t task) T_EXPECT_MACH_SUCCESS(task_threads(task, &threads, &thcnt), "task_threads"); T_LOG("Found %d kernel threads.", thcnt); for (i = 0; i < thcnt; i++) { + kern_return_t kr; thread_basic_info_data_t basic_info; mach_msg_type_number_t bi_count = THREAD_BASIC_INFO_COUNT; - T_EXPECT_MACH_SUCCESS(thread_info(threads[i], THREAD_BASIC_INFO, - (thread_info_t)&basic_info, &bi_count), - "thread_info(... THREAD_BASIC_INFO ...)"); + + kr = thread_info(threads[i], THREAD_BASIC_INFO, + (thread_info_t)&basic_info, &bi_count); + /* + * Ignore threads that have gone away. + */ + if (kr == MACH_SEND_INVALID_DEST) { + T_LOG("ignoring thread that has been destroyed"); + continue; + } + T_EXPECT_MACH_SUCCESS(kr, "thread_info(... THREAD_BASIC_INFO ...)"); (void)mach_port_deallocate(mach_task_self(), threads[i]); } mach_vm_deallocate(mach_task_self(), diff --git a/tools/tests/darwintests/thread_group_set_32261625.c b/tools/tests/darwintests/thread_group_set_32261625.c new file mode 100644 index 000000000..cebd042d0 --- /dev/null +++ b/tools/tests/darwintests/thread_group_set_32261625.c @@ -0,0 +1,62 @@ +#include +#include +#include + +#define TEST_EVENTID (0xfedcbb00) + +static void* +newthread(void *arg) +{ +#pragma unused(arg) + while (1) { + kdebug_trace(TEST_EVENTID, 0, 0, 0, 0); + sleep(1); + } +} + +#define TEST_TIMEOUT (15 * NSEC_PER_SEC) + +T_DECL(thread_group_set, "Checks that new threads get a THREAD_GROUP_SET tracepoint with a non-zero tid") { + pthread_t thread; + __block int seen_new_thread = 0, __block seen_thread_group_set = 0; + + ktrace_machine_t machine = ktrace_machine_create_current(); + T_ASSERT_NOTNULL(machine, "ktrace_get_machine"); + + bool has_tg = false; + if (ktrace_machine_has_thread_groups(machine, &has_tg) || !has_tg) { + T_SKIP("thread groups not supported on this system"); + } + + ktrace_session_t session = ktrace_session_create(); + T_ASSERT_NOTNULL(session, "ktrace_session_create"); + + ktrace_set_interactive(session); + + ktrace_set_completion_handler(session, ^{ + T_ASSERT_TRUE(seen_new_thread, "seen new thread tracepoint"); + T_END; + }); + + T_EXPECT_POSIX_SUCCESS(pthread_create(&thread, NULL, newthread, NULL), "pthread_create"); + T_EXPECT_POSIX_SUCCESS(pthread_detach(thread), "pthread_detach"); + + ktrace_events_single(session, TEST_EVENTID, ^(__unused ktrace_event_t e) { + T_EXPECT_TRUE(seen_thread_group_set, "seen THREAD_GROUP_SET tracepoint"); + seen_new_thread = 1; + ktrace_end(session, 1); + }); + + ktrace_events_single(session, MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_SET), ^(ktrace_event_t e) { + T_EXPECT_GT(e->arg3, 0, "tid on THREAD_GROUP_SET"); + seen_thread_group_set = 1; + }); + + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, TEST_TIMEOUT), dispatch_get_main_queue(), ^{ + ktrace_end(session, 0); + }); + + T_ASSERT_POSIX_SUCCESS(ktrace_start(session, dispatch_get_main_queue()), "ktrace_start"); + + dispatch_main(); +} diff --git a/tools/tests/darwintests/utimensat.c b/tools/tests/darwintests/utimensat.c new file mode 100644 index 000000000..bcda28f9e --- /dev/null +++ b/tools/tests/darwintests/utimensat.c @@ -0,0 +1,77 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define FILENAME "utimensat" + +static const struct timespec tptr[][2] = { + { { 0x12345678, 987654321 }, { 0x15263748, 123456789 }, }, + + { { 0, UTIME_NOW }, { 0x15263748, 123456789 }, }, + { { 0x12345678, 987654321 }, { 0, UTIME_NOW }, }, + { { 0, UTIME_NOW }, { 0, UTIME_NOW }, }, + + { { 0, UTIME_OMIT }, { 0x15263748, 123456789 }, }, + { { 0x12345678, 987654321 }, { 0, UTIME_OMIT }, }, + { { 0, UTIME_OMIT }, { 0, UTIME_OMIT }, }, + + { { 0, UTIME_NOW }, { 0, UTIME_OMIT }, }, + { { 0, UTIME_OMIT }, { 0, UTIME_NOW }, }, +}; + +T_DECL(utimensat, "Try various versions of utimensat") +{ + T_SETUPBEGIN; + T_ASSERT_POSIX_ZERO(chdir(dt_tmpdir()), NULL); + T_SETUPEND; + + struct stat pre_st, post_st; + int fd; + + T_ASSERT_POSIX_SUCCESS((fd = open(FILENAME, O_CREAT|O_RDWR, 0644)), NULL); + T_ASSERT_POSIX_ZERO(close(fd), NULL); + + for (size_t i = 0; i < sizeof(tptr)/sizeof(tptr[0]); i++) { + T_LOG("=== {%ld, %ld} {%ld, %ld} ===", + tptr[i][0].tv_sec, tptr[i][0].tv_nsec, + tptr[i][1].tv_sec, tptr[i][1].tv_nsec); + + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + + T_ASSERT_POSIX_ZERO(stat(FILENAME, &pre_st), NULL); + T_ASSERT_POSIX_ZERO(utimensat(AT_FDCWD, FILENAME, tptr[i], 0), NULL); + T_ASSERT_POSIX_ZERO(stat(FILENAME, &post_st), NULL); + + if (tptr[i][0].tv_nsec == UTIME_NOW) { + T_ASSERT_GE(post_st.st_atimespec.tv_sec, now.tv_sec, NULL); + } else if (tptr[i][0].tv_nsec == UTIME_OMIT) { + T_ASSERT_EQ(post_st.st_atimespec.tv_sec, pre_st.st_atimespec.tv_sec, NULL); + T_ASSERT_EQ(post_st.st_atimespec.tv_nsec, pre_st.st_atimespec.tv_nsec, NULL); + } else { + T_ASSERT_EQ(post_st.st_atimespec.tv_sec, tptr[i][0].tv_sec, NULL); + T_ASSERT_EQ(post_st.st_atimespec.tv_nsec, tptr[i][0].tv_nsec, NULL); + } + + if (tptr[i][1].tv_nsec == UTIME_NOW) { + T_ASSERT_GE(post_st.st_mtimespec.tv_sec, now.tv_sec, NULL); + } else if (tptr[i][1].tv_nsec == UTIME_OMIT) { + T_ASSERT_EQ(post_st.st_mtimespec.tv_sec, pre_st.st_mtimespec.tv_sec, NULL); + T_ASSERT_EQ(post_st.st_mtimespec.tv_nsec, pre_st.st_mtimespec.tv_nsec, NULL); + } else { + T_ASSERT_EQ(post_st.st_mtimespec.tv_sec, tptr[i][1].tv_sec, NULL); + T_ASSERT_EQ(post_st.st_mtimespec.tv_nsec, tptr[i][1].tv_nsec, NULL); + } + } +} diff --git a/tools/tests/darwintests/work_interval_test.c b/tools/tests/darwintests/work_interval_test.c new file mode 100644 index 000000000..cc6925056 --- /dev/null +++ b/tools/tests/darwintests/work_interval_test.c @@ -0,0 +1,122 @@ + +/* test that the header doesn't implicitly depend on others */ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +T_GLOBAL_META(T_META_NAMESPACE("xnu.scheduler")); + +static mach_port_t port = MACH_PORT_NULL; + +static void * +joining_thread_fn(__unused void *arg) +{ + int ret = 0; + kern_return_t kr = KERN_SUCCESS; + + ret = work_interval_join_port(port); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_join_port, another thread"); + + kr = mach_port_deallocate(mach_task_self(), port); + T_ASSERT_MACH_SUCCESS(kr, "mach_port_deallocate of port, another thread"); + + /* deliberately exit with joined work interval */ + return NULL; +} + +T_DECL(work_interval, "work interval interface") +{ + int ret = 0; + work_interval_t handle = NULL; + uint64_t now = mach_absolute_time(); + kern_return_t kr = KERN_SUCCESS; + + ret = work_interval_create(NULL, 0); + T_ASSERT_EQ(errno, EINVAL, "create with null errno EINVAL"); + T_ASSERT_EQ(ret, -1, "create with null returns -1"); + + /* Binary must be entitled for this to succeed */ + ret = work_interval_create(&handle, 0); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_create, no flags"); + + ret = work_interval_copy_port(handle, &port); + T_ASSERT_EQ(errno, EINVAL, "work_interval_copy_port on non-joinable interval errno EINVAL"); + T_ASSERT_EQ(ret, -1, "work_interval_copy_port on non-joinable interval returns -1"); + + ret = work_interval_notify(handle, now - 1000, now, now + 1000, now + 2000, 0); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_notify, no flags"); + + ret = work_interval_destroy(handle); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_destroy, no flags"); + + uint32_t flags[] = { + WORK_INTERVAL_FLAG_JOINABLE, + WORK_INTERVAL_FLAG_JOINABLE | WORK_INTERVAL_FLAG_GROUP, + }; + + for (uint32_t i = 0 ; i < sizeof(flags) / sizeof(flags[0]) ; i++) { + ret = work_interval_create(&handle, flags[i]); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_create, joinable"); + + ret = work_interval_copy_port(handle, &port); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_copy_port, joinable"); + + ret = work_interval_notify(handle, now - 1000, now, now + 1000, now + 2000, 0); + T_ASSERT_EQ(ret, -1, "work_interval_notify on non-joined thread returns -1"); + T_ASSERT_EQ(errno, EINVAL, "work_interval_copy_port on non-joined thread errno EINVAL"); + + ret = work_interval_join_port(port); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_join_port, joinable"); + + ret = work_interval_notify(handle, now - 1000, now, now + 1000, now + 2000, 0); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_notify, on joined thread"); + + ret = work_interval_join_port(port); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_join_port, join the same interval after destroy"); + + kr = mach_port_deallocate(mach_task_self(), port); + T_ASSERT_MACH_SUCCESS(kr, "mach_port_deallocate of port"); + + ret = work_interval_notify(handle, now - 1000, now, now + 1000, now + 2000, 0); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_notify, on joined thread after destroy"); + + ret = work_interval_destroy(handle); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_destroy, joinable, on joined thread"); + + ret = work_interval_leave(); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_leave, on destroyed work interval"); + } + + ret = work_interval_create(&handle, WORK_INTERVAL_FLAG_JOINABLE | WORK_INTERVAL_FLAG_GROUP); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_create, joinable"); + + ret = work_interval_copy_port(handle, &port); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_copy_port, joinable"); + + ret = work_interval_join_port(port); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_join_port, join before handing to another thread"); + + pthread_t joining_thread; + + T_ASSERT_POSIX_ZERO(pthread_create(&joining_thread, NULL, joining_thread_fn, NULL), "pthread_create"); + + T_ASSERT_POSIX_ZERO(pthread_join(joining_thread, NULL), "pthread_join"); + + ret = work_interval_leave(); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_leave"); + + ret = work_interval_destroy(handle); + T_ASSERT_POSIX_SUCCESS(ret, "work_interval_destroy"); + +} + diff --git a/tools/tests/darwintests/work_interval_test.entitlements b/tools/tests/darwintests/work_interval_test.entitlements new file mode 100644 index 000000000..5726ec2c6 --- /dev/null +++ b/tools/tests/darwintests/work_interval_test.entitlements @@ -0,0 +1,8 @@ + + + + + com.apple.private.kernel.work-interval + + + diff --git a/tools/tests/darwintests/xnu_quick_test.c b/tools/tests/darwintests/xnu_quick_test.c index 5e2e66701..7698b3fc3 100644 --- a/tools/tests/darwintests/xnu_quick_test.c +++ b/tools/tests/darwintests/xnu_quick_test.c @@ -1,23 +1,75 @@ -#define T_NAMESPACE xnu.quicktest - #include +#include "xnu_quick_test_helpers.h" +#include #include #include +#include #include +#include +#include #include +T_GLOBAL_META (T_META_NAMESPACE("xnu.quicktest"), T_META_CHECK_LEAKS(false)); +char g_target_path[ PATH_MAX ]; + +/* ************************************************************************************************************** + * Test the syscall system call. + * ************************************************************************************************************** + */ +T_DECL(syscall, + "xnu_quick_test for syscall", T_META_CHECK_LEAKS(NO)) +{ + int my_fd = -1; + char * my_pathp; + kern_return_t my_kr; + + T_SETUPBEGIN; + + create_target_directory(TEST_DIRECTORY); + + T_SETUPEND; + + my_kr = vm_allocate((vm_map_t) mach_task_self(), (vm_address_t*)&my_pathp, + PATH_MAX, VM_FLAGS_ANYWHERE); + T_ASSERT_MACH_SUCCESS(my_kr, "Allocating vm to path %s", my_pathp); + + *my_pathp = 0x00; + strcpy( my_pathp, &g_target_path[0] ); + strcat( my_pathp, "/" ); + + /* create a test file */ + + T_ASSERT_MACH_SUCCESS( create_random_name( my_pathp, 1), "Create random test file" ); + /* use an indirect system call to open our test file. + * I picked open since it uses a path pointer which grows to 64 bits in an LP64 environment. + */ + T_EXPECT_NE(my_fd = syscall( SYS_open, my_pathp, (O_RDWR | O_EXCL), 0 ), + -1, "Attempt to open file using indirect syscall %s", my_pathp); + + if (my_fd != -1) + close(my_fd); + + if (my_pathp != NULL) { + remove(my_pathp); + vm_deallocate(mach_task_self(), (vm_address_t)my_pathp, PATH_MAX); + } + + T_ATEND(remove_target_directory); +} + /* ************************************************************************************************************** * Test fork wait4, and exit system calls. * ************************************************************************************************************** */ -T_DECL(fork_wait4_exit_test, +T_DECL(fork_wait4_exit, "Tests forking off a process and waiting for the child to exit", T_META_CHECK_LEAKS(false)) { int my_err, my_status; pid_t my_pid, my_wait_pid; struct rusage my_usage; - char * g_target_path="/"; + + strncpy(g_target_path, "/", 2); /* spin off another process */ T_ASSERT_NE(my_pid = fork(), -1, "Fork off a process"); @@ -50,3 +102,17 @@ T_DECL(fork_wait4_exit_test, T_ASSERT_TRUE(( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) == 44 ), "check if wait4 returns right exit status"); } + +T_DECL (getrusage, "Sanity check of getrusage") +{ + struct rusage my_rusage; + + T_WITH_ERRNO; + T_ASSERT_EQ(getrusage( RUSAGE_SELF, &my_rusage ), 0, NULL); + T_LOG("Checking that getrusage returned sane values"); + T_EXPECT_LT(my_rusage.ru_msgrcv, 1000, NULL); + T_EXPECT_GE(my_rusage.ru_msgrcv, 0, NULL); + T_EXPECT_LT(my_rusage.ru_nsignals, 1000, NULL); + T_EXPECT_GE(my_rusage.ru_nsignals, 0, NULL); +} + diff --git a/tools/tests/darwintests/xnu_quick_test_helpers.c b/tools/tests/darwintests/xnu_quick_test_helpers.c new file mode 100644 index 000000000..08670d831 --- /dev/null +++ b/tools/tests/darwintests/xnu_quick_test_helpers.c @@ -0,0 +1,114 @@ +#include + +#include "xnu_quick_test_helpers.h" + +#include +#include + +void create_target_directory( const char * the_targetp ) +{ + int err; + const char * my_targetp; + + my_targetp = getenv("TMPDIR"); + if ( my_targetp == NULL ) + my_targetp = "/tmp"; + + T_ASSERT_LT( strlen( the_targetp ), (unsigned long)( PATH_MAX - 1 ), + "check target path too long - \"%s\"", the_targetp ); + + for ( ;; ) { + int my_rand; + char my_name[64]; + + my_rand = rand( ); + sprintf( &my_name[0], "xnu_quick_test-%d", my_rand ); + T_ASSERT_LT( strlen( &my_name[0] ) + strlen( the_targetp ) + 2, (unsigned long)PATH_MAX, + "check target path plus our test directory name is too long: " + "target path - \"%s\" test directory name - \"%s\"", + the_targetp, &my_name[0] ); + + /* append generated directory name onto our path */ + g_target_path[0] = 0x00; + strcat( &g_target_path[0], the_targetp ); + if ( g_target_path[ (strlen(the_targetp) - 1) ] != '/' ) { + strcat( &g_target_path[0], "/" ); + } + strcat( &g_target_path[0], &my_name[0] ); + + /* try to create the test directory */ + err = mkdir( &g_target_path[0], (S_IRWXU | S_IRWXG | S_IROTH) ); + if ( err == 0 ) { + break; + } + err = errno; + if ( EEXIST != err ) { + T_ASSERT_FAIL( "test directory creation failed - \"%s\" \n" + "mkdir call failed with error %d - \"%s\"", + &g_target_path[0], errno, strerror( err) ); + } + } + +} /* create_target_directory */ + +/* + * create_random_name - creates a file with a random / unique name in the given directory. + * when do_open is true we create a file else we generaate a name that does not exist in the + * given directory (we do not create anything when do_open is 0). + * WARNING - caller provides enough space in path buffer for longest possible name. + * WARNING - assumes caller has appended a trailing '/' on the path passed to us. + * RAND_MAX is currently 2147483647 (ten characters plus one for a slash) + */ +int create_random_name( char *the_pathp, int do_open ) { + int i, my_err; + int my_fd = -1; + + for ( i = 0; i < 1; i++ ) { + int my_rand; + char *myp; + char my_name[32]; + + my_rand = rand( ); + sprintf( &my_name[0], "%d", my_rand ); + T_ASSERT_LT_ULONG((strlen( &my_name[0] ) + strlen( the_pathp ) + 2), (unsigned long)PATH_MAX, + "check if path to test file is less than PATH_MAX"); + + // append generated file name onto our path + myp = strrchr( the_pathp, '/' ); + *(myp + 1) = 0x00; + strcat( the_pathp, &my_name[0] ); + if ( do_open ) { + /* create a file with this name */ + my_fd = open( the_pathp, (O_RDWR | O_CREAT | O_EXCL), + (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) ); + T_EXPECT_TRUE((my_fd != -1 || errno == EEXIST), "open file with name %s", the_pathp); + + if( errno == EEXIST ) + continue; + } + else { + /* make sure the name is unique */ + struct stat my_sb; + my_err = stat( the_pathp, &my_sb ); + T_EXPECT_TRUE((my_err == 0 || errno == ENOENT), "make sure the name is unique"); + + if(errno == ENOENT) break; + /* name already exists, try another */ + i--; + continue; + } + } + + if ( my_fd != -1 ) + close( my_fd ); + + if(do_open && my_fd == -1) + return 1; + + return 0; +} /* create_random_name */ + +void remove_target_directory() { + rmdir(&g_target_path[0]); +} + diff --git a/tools/tests/darwintests/xnu_quick_test_helpers.h b/tools/tests/darwintests/xnu_quick_test_helpers.h new file mode 100644 index 000000000..b6a25ed9a --- /dev/null +++ b/tools/tests/darwintests/xnu_quick_test_helpers.h @@ -0,0 +1,16 @@ +#ifndef XNU_QUICK_TEST_HELPERS_H +#define XNU_QUICK_TEST_HELPERS_H + +#include +#include +#include + +#define TEST_DIRECTORY "/tmp" + +extern char g_target_path[ PATH_MAX ]; + +int create_random_name( char *the_pathp, int do_open ); +void create_target_directory( const char * the_targetp ); +void remove_target_directory( void ); + +#endif diff --git a/tools/tests/execperf/exit-asm.S b/tools/tests/execperf/exit-asm.S index 2b65a52c9..ed92a1323 100644 --- a/tools/tests/execperf/exit-asm.S +++ b/tools/tests/execperf/exit-asm.S @@ -38,6 +38,24 @@ _sysenter_trap: mov %esp, %ecx sysenter nop +#elif defined(__arm__) + push { lr } + mov r0, #42 + mov r12, #1 + svc 0x00000080 + bcc 1f +1: + nop + nop + nop + nop +#elif defined(__arm64__) + movz x0, #42 + movz x16, #1 + svc #0x80 + b.cc 1f +1: + nop #else #error Unsupported architecture #endif diff --git a/tools/tests/execperf/exit.c b/tools/tests/execperf/exit.c index 301679fcb..e58cbfebb 100644 --- a/tools/tests/execperf/exit.c +++ b/tools/tests/execperf/exit.c @@ -5,6 +5,8 @@ int main(int artc, char *argv[]) { asm volatile ("andq $0xfffffffffffffff0, %rsp\n"); #elif defined(__i386__) asm volatile ("andl $0xfffffff0, %esp\n"); +#elif defined(__arm__) || defined(__arm64__) + asm volatile (""); #else #error Unsupported architecture #endif diff --git a/tools/tests/jitter/Makefile b/tools/tests/jitter/Makefile index f78950c1a..901814c6e 100644 --- a/tools/tests/jitter/Makefile +++ b/tools/tests/jitter/Makefile @@ -1,5 +1,7 @@ include ../Makefile.common +DSTROOT?=$(shell /bin/pwd) +SYMROOT?=$(shell /bin/pwd) OBJROOT?=$(shell /bin/pwd) CC:=$(shell xcrun -sdk "$(SDKROOT)" -find cc) @@ -14,20 +16,14 @@ ifdef RC_ARCHS endif endif -DSTROOT?=$(shell /bin/pwd) CFLAGS:=$(patsubst %, -arch %,$(ARCHS)) -g -Wall -Os -isysroot $(SDKROOT) -I$(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders all: $(DSTROOT)/jitter -$(OBJROOT)/timer_jitter.o: timer_jitter.c - $(CC) -c -o $@ $< $(CFLAGS) - -$(OBJROOT)/cpu_number.o: cpu_number.s - $(CC) -c -o $@ $< $(CFLAGS) - -$(DSTROOT)/jitter: $(OBJROOT)/timer_jitter.o $(OBJROOT)/cpu_number.o +$(DSTROOT)/jitter: timer_jitter.c $(CC) -o $@ $^ $(CFLAGS) clean: rm -f $(DSTROOT)/jitter $(OBJROOT)/*.o + rm -rf $(SYMROOT)/*.dSYM diff --git a/tools/tests/jitter/cpu_number.s b/tools/tests/jitter/cpu_number.s deleted file mode 100644 index 77c95875c..000000000 --- a/tools/tests/jitter/cpu_number.s +++ /dev/null @@ -1,33 +0,0 @@ -.text -/* - * Taken from Libc - */ -.globl _cpu_number -_cpu_number: -#if defined(__x86_64__) - push %rbp - mov %rsp,%rbp - sub $16,%rsp // space to read IDTR - - sidt (%rsp) // store limit:base on stack - movw (%rsp), %ax // get limit - and $0xfff, %rax // mask off lower 12 bits to return - - mov %rbp,%rsp - pop %rbp - ret -#elif defined(__i386__) - push %ebp - mov %esp,%ebp - sub $8, %esp // space to read IDTR - - sidt (%esp) // store limit:base on stack - movw (%esp), %ax // get limit - and $0xfff, %eax // mask off lower 12 bits to return - - mov %ebp,%esp - pop %ebp - ret -#else -#error Unsupported architecture -#endif diff --git a/tools/tests/jitter/timer_jitter.c b/tools/tests/jitter/timer_jitter.c index 7e0c9a0c1..e6b4dc5d7 100644 --- a/tools/tests/jitter/timer_jitter.c +++ b/tools/tests/jitter/timer_jitter.c @@ -49,6 +49,8 @@ #include +#include /* private header for _os_cpu_number */ + typedef enum my_policy_type { MY_POLICY_REALTIME, MY_POLICY_TIMESHARE, MY_POLICY_FIXEDPRI } my_policy_type_t; #define DEFAULT_MAX_SLEEP_NS 2000000000ll /* Two seconds */ @@ -71,8 +73,6 @@ struct second_thread_args { volatile int cpuno; }; -extern int cpu_number(void); - void * second_thread(void *args); @@ -390,7 +390,7 @@ main(int argc, char **argv) if (wakeup_second_thread) { secargs.last_poke_time = mach_absolute_time(); - secargs.cpuno = cpu_number(); + secargs.cpuno = _os_cpu_number(); OSMemoryBarrier(); kret = semaphore_signal(wakeup_semaphore); if (kret != KERN_SUCCESS) { @@ -465,7 +465,7 @@ second_thread(void *args) } wake_time = mach_absolute_time(); - cpuno = cpu_number(); + cpuno = _os_cpu_number(); if (wake_time < secargs->last_poke_time) { /* Woke in past, unsynchronized mach_absolute_time()? */ diff --git a/tools/tests/kqueue_tests/Makefile b/tools/tests/kqueue_tests/Makefile deleted file mode 100755 index 0a3c7daa1..000000000 --- a/tools/tests/kqueue_tests/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -include ../Makefile.common - -CC:=$(shell xcrun -sdk "$(SDKROOT)" -find cc) - -ifdef RC_ARCHS - ARCHS:=$(RC_ARCHS) - else - ifeq "$(Embedded)" "YES" - ARCHS:=armv7 armv7s arm64 - else - ARCHS:=x86_64 i386 - endif -endif - -CFLAGS :=-g $(patsubst %, -arch %,$(ARCHS)) -isysroot $(SDKROOT) - -DSTROOT?=$(shell /bin/pwd) -SYMROOT?=$(shell /bin/pwd) - -all: $(addprefix $(DSTROOT)/, file_tests timer_tests) - -$(DSTROOT)/file_tests: kqueue_file_tests.c - $(CC) $(CFLAGS) -o $(SYMROOT)/file_tests kqueue_file_tests.c - ditto $(SYMROOT)/file_tests $(DSTROOT)/file_tests - -$(DSTROOT)/timer_tests: kqueue_timer_tests.c - $(CC) $(CFLAGS) -o $(SYMROOT)/timer_tests kqueue_timer_tests.c - ditto $(SYMROOT)/timer_tests $(DSTROOT)/timer_tests - -clean: - rm -rf $(DSTROOT)/file_tests $(DSTROOT)/timer_tests $(SYMROOT)/*.dSYM $(SYMROOT)/file_tests $(SYMROOT)/timer_tests diff --git a/tools/tests/kqueue_tests/kqueue_timer_tests.c b/tools/tests/kqueue_tests/kqueue_timer_tests.c deleted file mode 100644 index 9a4db010e..000000000 --- a/tools/tests/kqueue_tests/kqueue_timer_tests.c +++ /dev/null @@ -1,255 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -int kq, passed, failed; - -/* - * Wait for given kevent, which should return in 'expected' usecs. - */ -int -do_simple_kevent(struct kevent64_s *kev, uint64_t expected) -{ - int ret; - uint64_t elapsed_usecs, delta_usecs; - struct timespec timeout; - struct timeval before, after; - - /* time out after 1 sec extra delay */ - timeout.tv_sec = (expected / (1000 * 1000)) + 1; - timeout.tv_nsec = (expected % (1000 * 1000)) * 1000; - - /* measure time for the kevent */ - gettimeofday(&before, NULL); - ret = kevent64(kq, kev, 1, kev, 1, 0, &timeout); - gettimeofday(&after, NULL); - - if (ret < 1 || (kev->flags & EV_ERROR)) { - printf("\tfailure: kevent returned %d, error %d\n", ret, - (ret == -1 ? errno : (int) kev->data)); - return 0; - } - - /* did it work? */ - elapsed_usecs = (after.tv_sec - before.tv_sec) * (1000 * 1000) + - (after.tv_usec - before.tv_usec); - delta_usecs = abs(elapsed_usecs - (expected)); - - /* failure if we're 30% off, or 50 mics late */ - if (delta_usecs > (30 * expected / 100.0) && delta_usecs > 50) { - printf("\tfailure: expected %lld usec, measured %lld usec.\n", - expected, elapsed_usecs); - return 0; - } else { - printf("\tsuccess.\n"); - return 1; - } -} - -void -test_absolute_kevent(int time, int scale) -{ - struct timeval tv; - struct kevent64_s kev; - uint64_t nowus, expected, deadline; - int ret; - int timescale = 0; - - gettimeofday(&tv, NULL); - nowus = tv.tv_sec * (1000 * 1000LL) + tv.tv_usec; - - switch (scale) { - case NOTE_SECONDS: - printf("Testing %d sec absolute timer...\n", time); - timescale = 1000 * 1000; - break; - case NOTE_USECONDS: - printf("Testing %d usec absolute timer...\n", time); - timescale = 1; - break; - case 0: - printf("Testing %d msec absolute timer...\n", time); - timescale = 1000; - break; - default: - printf("Failure: scale 0x%x not recognized.\n", scale); - return; - } - - expected = time * timescale; - deadline = nowus / timescale + time; - - /* deadlines in the past should fire immediately */ - if (time < 0) - expected = 0; - - EV_SET64(&kev, 1, EVFILT_TIMER, EV_ADD, - NOTE_ABSOLUTE | scale, deadline, 0,0,0); - ret = do_simple_kevent(&kev, expected); - - if (ret) - passed++; - else - failed++; -} - -void -test_oneshot_kevent(int time, int scale) -{ - int ret; - uint64_t expected = 0; - struct kevent64_s kev; - - switch (scale) { - case NOTE_SECONDS: - printf("Testing %d sec interval timer...\n", time); - expected = time * (1000 * 1000); - break; - case NOTE_USECONDS: - printf("Testing %d usec interval timer...\n", time); - expected = time; - break; - case NOTE_NSECONDS: - printf("Testing %d nsec interval timer...\n", time); - expected = time / 1000; - break; - case 0: - printf("Testing %d msec interval timer...\n", time); - expected = time * 1000; - break; - default: - printf("Failure: scale 0x%x not recognized.\n", scale); - return; - } - - /* deadlines in the past should fire immediately */ - if (time < 0) - expected = 0; - - EV_SET64(&kev, 2, EVFILT_TIMER, EV_ADD | EV_ONESHOT, scale, time, - 0, 0, 0); - ret = do_simple_kevent(&kev, expected); - - if (ret) - passed++; - else - failed++; - -} - -void -test_repeating_kevent(int usec) -{ - struct kevent64_s kev; - int expected_pops, ret; - - expected_pops = 1000 * 1000 / usec; - printf("Testing repeating kevent for %d pops in a second...\n", - expected_pops); - - EV_SET64(&kev, 3, EVFILT_TIMER, EV_ADD, NOTE_USECONDS, usec, 0, 0, 0); - ret = kevent64(kq, &kev, 1, NULL, 0, 0, NULL); - if (ret != 0) { - printf("\tfailure: kevent64 returned %d\n", ret); - failed++; - return; - } - - /* sleep 1 second */ - usleep(1000 * 1000); - ret = kevent64(kq, NULL, 0, &kev, 1, 0, NULL); - if (ret != 1 || (kev.flags & EV_ERROR)) { - printf("\tfailure: kevent64 returned %d\n", ret); - failed++; - return; - } - - /* check how many times the timer fired: within 5%? */ - if (kev.data > expected_pops + (expected_pops / 20) || - kev.data < expected_pops - (expected_pops / 20)) { - printf("\tfailure: saw %lld pops.\n", kev.data); - failed++; - } else { - printf("\tsuccess: saw %lld pops.\n", kev.data); - passed++; - } - - EV_SET64(&kev, 3, EVFILT_TIMER, EV_DELETE, 0, 0, 0, 0, 0); - ret = kevent64(kq, &kev, 1, NULL, 0, 0, NULL); - if (ret != 0) { - printf("\tfailed to stop repeating timer: %d\n", ret); - } -} - -void -test_updated_kevent(int first, int second) -{ - struct kevent64_s kev; - int ret; - - printf("Testing update from %d to %d msecs...\n", first, second); - - EV_SET64(&kev, 4, EVFILT_TIMER, EV_ADD|EV_ONESHOT, 0, first, 0, 0, 0); - ret = kevent64(kq, &kev, 1, NULL, 0, 0, NULL); - if (ret != 0) { - printf("\tfailure: initial kevent returned %d\n", ret); - failed++; - return; - } - - EV_SET64(&kev, 4, EVFILT_TIMER, EV_ONESHOT, 0, second, 0, 0, 0); - if (second < 0) - second = 0; - ret = do_simple_kevent(&kev, second * 1000); - if (ret) - passed++; - else - failed++; -} - -int -main(void) -{ - struct timeval tv; - struct kevent64_s kev; - uint64_t nowms, deadline; - - kq = kqueue(); - assert(kq > 0); - passed = 0; - failed = 0; - - test_absolute_kevent(100, 0); - test_absolute_kevent(200, 0); - test_absolute_kevent(300, 0); - test_absolute_kevent(1000, 0); - test_absolute_kevent(500, NOTE_USECONDS); - test_absolute_kevent(100, NOTE_USECONDS); - test_absolute_kevent(5, NOTE_SECONDS); - test_absolute_kevent(-1000, 0); - - test_oneshot_kevent(1, NOTE_SECONDS); - test_oneshot_kevent(10, 0); - test_oneshot_kevent(200, NOTE_USECONDS); - test_oneshot_kevent(300000, NOTE_NSECONDS); - test_oneshot_kevent(-1, NOTE_SECONDS); - - test_repeating_kevent(100 * 1000); - test_repeating_kevent(5 * 1000); - test_repeating_kevent(200); - test_repeating_kevent(50); - test_repeating_kevent(10); - - test_updated_kevent(1000, 2000); - test_updated_kevent(2000, 1000); - test_updated_kevent(1000, -1); - - printf("\nFinished: %d tests passed, %d failed.\n", passed, failed); - - exit(EXIT_SUCCESS); -} diff --git a/tools/tests/libMicro/Makefile.com.Darwin b/tools/tests/libMicro/Makefile.com.Darwin old mode 100755 new mode 100644 diff --git a/tools/tests/perf_index/PerfIndex_COPS_Module/PITest.h b/tools/tests/perf_index/PerfIndex_COPS_Module/PITest.h index 6449307c1..7722852d9 100644 --- a/tools/tests/perf_index/PerfIndex_COPS_Module/PITest.h +++ b/tools/tests/perf_index/PerfIndex_COPS_Module/PITest.h @@ -33,4 +33,4 @@ - (void)cleanup; -@end \ No newline at end of file +@end diff --git a/tools/tests/perf_index/PerfIndex_COPS_Module/PerfIndex.xcodeproj/project.pbxproj b/tools/tests/perf_index/PerfIndex_COPS_Module/PerfIndex.xcodeproj/project.pbxproj index 7c0cd67b1..a1dbc0edc 100644 --- a/tools/tests/perf_index/PerfIndex_COPS_Module/PerfIndex.xcodeproj/project.pbxproj +++ b/tools/tests/perf_index/PerfIndex_COPS_Module/PerfIndex.xcodeproj/project.pbxproj @@ -202,7 +202,6 @@ CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; CLANG_CXX_LIBRARY = "libc++"; CLANG_ENABLE_MODULES = YES; - CLANG_ENABLE_OBJC_ARC = YES; CLANG_WARN_BOOL_CONVERSION = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; @@ -251,7 +250,6 @@ CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; CLANG_CXX_LIBRARY = "libc++"; CLANG_ENABLE_MODULES = YES; - CLANG_ENABLE_OBJC_ARC = YES; CLANG_WARN_BOOL_CONVERSION = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; @@ -293,7 +291,6 @@ CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; CLANG_CXX_LIBRARY = "libc++"; CLANG_ENABLE_MODULES = YES; - CLANG_ENABLE_OBJC_ARC = YES; CLANG_WARN_BOOL_CONVERSION = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; @@ -336,7 +333,6 @@ CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; CLANG_CXX_LIBRARY = "libc++"; CLANG_ENABLE_MODULES = YES; - CLANG_ENABLE_OBJC_ARC = YES; CLANG_WARN_BOOL_CONVERSION = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; diff --git a/tools/tests/perf_index/test_controller.py b/tools/tests/perf_index/test_controller.py old mode 100644 new mode 100755 diff --git a/tools/tests/perf_index/test_fault_helper.c b/tools/tests/perf_index/test_fault_helper.c index 94127979c..439757774 100644 --- a/tools/tests/perf_index/test_fault_helper.c +++ b/tools/tests/perf_index/test_fault_helper.c @@ -6,7 +6,11 @@ #include #include +#if TARGET_OS_EMBEDDED +#define MEMSIZE (1L<<28) +#else #define MEMSIZE (1L<<30) +#endif static char* memblock; diff --git a/tools/tests/zero-to-n/Makefile b/tools/tests/zero-to-n/Makefile index 76f53d169..63e5484c9 100644 --- a/tools/tests/zero-to-n/Makefile +++ b/tools/tests/zero-to-n/Makefile @@ -12,7 +12,7 @@ ifdef RC_ARCHS endif endif -CFLAGS := -g $(patsubst %, -arch %, $(ARCHS)) -isysroot $(SDKROOT) -isystem $(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders +CFLAGS := -Os -g $(patsubst %, -arch %, $(ARCHS)) -isysroot $(SDKROOT) -isystem $(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders DSTROOT?=$(shell /bin/pwd) SYMROOT?=$(shell /bin/pwd) diff --git a/tools/tests/zero-to-n/zero-to-n.c b/tools/tests/zero-to-n/zero-to-n.c index 87ce83bb7..f31b58df5 100644 --- a/tools/tests/zero-to-n/zero-to-n.c +++ b/tools/tests/zero-to-n/zero-to-n.c @@ -44,8 +44,6 @@ #include #include -#include - #include #include #include @@ -55,6 +53,8 @@ #include +#include + typedef enum wake_type { WAKE_BROADCAST_ONESEM, WAKE_BROADCAST_PERTHREAD, WAKE_CHAIN, WAKE_HOP } wake_type_t; typedef enum my_policy_type { MY_POLICY_REALTIME, MY_POLICY_TIMESHARE, MY_POLICY_FIXEDPRI } my_policy_type_t; @@ -66,6 +66,8 @@ typedef enum my_policy_type { MY_POLICY_REALTIME, MY_POLICY_TIMESHARE, MY_POLICY #define COMPUTATION_NANOS (10000000ll) /* 10 ms */ #define TRACEWORTHY_NANOS (10000000ll) /* 10 ms */ +#define DEBUG 0 + #if DEBUG #define debug_log(args...) printf(args) #else @@ -80,6 +82,10 @@ static my_policy_type_t parse_thread_policy(const char *str); static void selfexec_with_apptype(int argc, char *argv[]); static void parse_args(int argc, char *argv[]); +static __attribute__((aligned(128))) _Atomic uint32_t g_done_threads; +static __attribute__((aligned(128))) _Atomic boolean_t g_churn_stop = FALSE; +static __attribute__((aligned(128))) _Atomic uint64_t g_churn_stopped_at = 0; + /* Global variables (general) */ static uint32_t g_numcpus; static uint32_t g_numthreads; @@ -89,7 +95,6 @@ static uint32_t g_iterations; static struct mach_timebase_info g_mti; static semaphore_t g_main_sem; static uint64_t *g_thread_endtimes_abs; -static volatile uint32_t g_done_threads; static boolean_t g_verbose = FALSE; static boolean_t g_do_affinity = FALSE; static uint64_t g_starttime_abs; @@ -97,8 +102,6 @@ static uint32_t g_iteration_sleeptime_us = 0; static uint32_t g_priority = 0; static uint32_t g_churn_pri = 0; static uint32_t g_churn_count = 0; -static uint64_t g_churn_stopped_at = 0; -static boolean_t g_churn_stop = FALSE; static pthread_t* g_churn_threads = NULL; @@ -152,7 +155,9 @@ nanos_to_abs(uint64_t ns) inline static void yield(void) { -#if defined(__x86_64__) || defined(__i386__) +#if defined(__arm__) || defined(__arm64__) + asm volatile("yield"); +#elif defined(__x86_64__) || defined(__i386__) asm volatile("pause"); #else #error Unrecognized architecture @@ -176,7 +181,7 @@ churn_thread(__unused void *arg) } /* This is totally racy, but only here to detect if anyone stops early */ - g_churn_stopped_at += spin_count; + atomic_fetch_add_explicit(&g_churn_stopped_at, spin_count, memory_order_relaxed); return NULL; } @@ -220,13 +225,11 @@ create_churn_threads() static void join_churn_threads(void) { - if (g_churn_stopped_at != 0) + if (atomic_load_explicit(&g_churn_stopped_at, memory_order_seq_cst) != 0) printf("Warning: Some of the churn threads may have stopped early: %lld\n", g_churn_stopped_at); - OSMemoryBarrier(); - - g_churn_stop = TRUE; + atomic_store_explicit(&g_churn_stop, TRUE, memory_order_seq_cst); /* Rejoin churn threads */ for (uint32_t i = 0; i < g_churn_count; i++) { @@ -376,7 +379,7 @@ worker_thread(void *arg) debug_log("%d Leader thread go\n", i); - assert_zero_t(my_id, g_done_threads); + assert_zero_t(my_id, atomic_load_explicit(&g_done_threads, memory_order_relaxed)); switch (g_waketype) { case WAKE_BROADCAST_ONESEM: @@ -476,10 +479,10 @@ worker_thread(void *arg) } } - int32_t new = OSAtomicIncrement32((volatile int32_t *)&g_done_threads); - (void)new; + uint32_t done_threads; + done_threads = atomic_fetch_add_explicit(&g_done_threads, 1, memory_order_relaxed) + 1; - debug_log("Thread %p new value is %d, iteration %d\n", pthread_self(), new, i); + debug_log("Thread %p new value is %d, iteration %d\n", pthread_self(), done_threads, i); if (g_drop_priority) { /* Drop priority to BG momentarily */ @@ -490,7 +493,7 @@ worker_thread(void *arg) if (g_do_all_spin) { /* Everyone spins until the last thread checks in. */ - while (g_done_threads < g_numthreads) { + while (atomic_load_explicit(&g_done_threads, memory_order_relaxed) < g_numthreads) { y = y + 1.5 + x; x = sqrt(y); } @@ -673,8 +676,9 @@ main(int argc, char **argv) kr = semaphore_create(mach_task_self(), &g_readysem, SYNC_POLICY_FIFO, 0); mach_assert_zero(kr); + atomic_store_explicit(&g_done_threads, 0, memory_order_relaxed); + /* Create the threads */ - g_done_threads = 0; for (uint32_t i = 0; i < g_numthreads; i++) { ret = pthread_create(&threads[i], NULL, worker_thread, (void*)(uintptr_t)i); if (ret) errc(EX_OSERR, ret, "pthread_create %d", i); @@ -708,8 +712,7 @@ main(int argc, char **argv) debug_log("%d Main thread reset\n", i); - g_done_threads = 0; - OSMemoryBarrier(); + atomic_store_explicit(&g_done_threads, 0, memory_order_seq_cst); g_starttime_abs = mach_absolute_time(); @@ -719,6 +722,8 @@ main(int argc, char **argv) debug_log("%d Main thread return\n", i); + assert(atomic_load_explicit(&g_done_threads, memory_order_relaxed) == g_numthreads); + /* * We report the worst latencies relative to start time * and relative to the lead worker thread. diff --git a/tools/trace/kqtrace.lua b/tools/trace/kqtrace.lua new file mode 100755 index 000000000..bb5b9545f --- /dev/null +++ b/tools/trace/kqtrace.lua @@ -0,0 +1,339 @@ +#!/usr/local/bin/luatrace -s + +trace_eventname = function(codename, callback) + local debugid = trace.debugid(codename) + if debugid ~= 0 then + trace.single(debugid,callback) + else + printf("WARNING: Cannot locate debugid for '%s'\n", codename) + end +end + +initial_timestamp = 0 + +function event_prefix_string(buf, workq) + if initial_timestamp == 0 then + initial_timestamp = buf.timestamp + end + local secs = trace.convert_timestamp_to_nanoseconds(buf.timestamp - initial_timestamp) / 1000000000 + + local type + if trace.debugid_is_start(buf.debugid) then + type = "→" + elseif trace.debugid_is_end(buf.debugid) then + type = "←" + else + type = "↔" + end + + proc = buf.command + + local prefix = string.format("%s %6.9f %-17s [%05d.%06x] %-28s\t", + type, secs, proc, buf.pid, buf.threadid, buf.debugname) + if not workq then + prefix = prefix .. string.format(" 0x%16x", buf.arg1) + end + + return prefix +end + +function qos_string(qos) + if qos == 0 then + return "--" + elseif qos == 1 then + return "MT" + elseif qos == 2 then + return "BG" + elseif qos == 3 then + return "UT" + elseif qos == 4 then + return "DF" + elseif qos == 5 then + return "IN" + elseif qos == 6 then + return "UI" + elseif qos == 7 then + return "MG" + else + return string.format("??[0x%x]", qos) + end +end + +function state_string(strings, state) + local str = '' + local first = true + for name, bit in pairs(strings) do + if (state & bit) == bit then + if not first then + str = str .. ' ' + end + str = str .. name + first = false + end + end + return str +end + +kqrequest_state_strings = { + ['PROCESSING'] = 0x1, + ['THREQUESTED'] = 0x2, + ['WAKEUP'] = 0x4 +} + +kqueue_state_strings = { + ['SEL'] = 0x001, + ['SLEEP'] = 0x002, + ['PROCWAIT'] = 0x004, + ['KEV32'] = 0x008, + ['KEV64'] = 0x010, + ['KEV_QOS'] = 0x020, + ['WORKQ'] = 0x040, + ['WORKLOOP'] = 0x080, + ['PROCESSING'] = 0x100, + ['DRAIN'] = 0x200, + ['WAKEUP'] = 0x400, + ['DYNAMIC'] = 0x800, +} + +knote_state_strings = { + ['ACTIVE'] = 0x0001, + ['QUEUED'] = 0x0002, + ['DISABLED'] = 0x0004, + ['DROPPING'] = 0x0008, + ['USEWAIT'] = 0x0010, + ['ATTACHING'] = 0x0020, + ['STAYACTIVE'] = 0x0040, + ['DEFERDELETE'] = 0x0080, + ['ATTACHED'] = 0x0100, + ['DISPATCH'] = 0x0200, + ['UDATA_SPECIFIC'] = 0x0400, + ['SUPPRESSED'] = 0x0800, + ['STOLENDROP'] = 0x1000, + ['REQVANISH'] = 0x2000, + ['VANISHED'] = 0x4000, +} +knote_state_strings = { + ['ACTIVE'] = 0x0001, + ['QUEUED'] = 0x0002, + ['DISABLED'] = 0x0004, + ['DROPPING'] = 0x0008, + ['USEWAIT'] = 0x0010, + ['ATTACHING'] = 0x0020, + ['STAYACTIVE'] = 0x0040, + ['DEFERDELETE'] = 0x0080, + ['ATTACHED'] = 0x0100, + ['DISPATCH'] = 0x0200, + ['UDATA_SPECIFIC'] = 0x0400, + ['SUPPRESSED'] = 0x0800, + ['STOLENDROP'] = 0x1000, + ['REQVANISH'] = 0x2000, + ['VANISHED'] = 0x4000, +} + + +kevent_flags_strings = { + ['ADD'] = 0x0001, + ['DELETE'] = 0x0002, + ['ENABLE'] = 0x0004, + ['DISABLE'] = 0x0008, + ['ONESHOT'] = 0x0010, + ['CLEAR'] = 0x0020, + ['RECEIPT'] = 0x0040, + ['DISPATCH'] = 0x0080, + ['UDATA_SPECIFIC'] = 0x0100, + ['VANISHED'] = 0x0200, + ['FLAG0'] = 0x1000, + ['FLAG1'] = 0x2000, + ['EOF'] = 0x8000, + ['ERROR'] = 0x4000, +} + +function kevent_filter_string(filt) + if filt == -1 then + return 'READ' + elseif filt == -2 then + return 'WRITE' + elseif filt == -3 then + return 'AIO' + elseif filt == -4 then + return 'VNODE' + elseif filt == -5 then + return 'PROC' + elseif filt == -6 then + return 'SIGNAL' + elseif filt == -7 then + return 'TIMER' + elseif filt == -8 then + return 'MACHPORT' + elseif filt == -9 then + return 'FS' + elseif filt == -10 then + return 'USER' + -- -11 unused + elseif filt == -12 then + return 'VM' + elseif filt == -13 then + return 'SOCK' + elseif filt == -14 then + return 'MEMORYSTATUS' + elseif filt == 15 then + return 'KQREAD' + elseif filt == 16 then + return 'PIPE_R' + elseif filt == 17 then + return 'PIPE_W' + elseif filt == 18 then + return 'PTSD' + elseif filt == 19 then + return 'SOWRITE' + elseif filt == 20 then + return 'SOEXCEPT' + elseif filt == 21 then + return 'SPEC' + elseif filt == 22 then + return 'BPFREAD' + elseif filt == 23 then + return 'NECP_FD' + elseif filt == 24 then + return 'SKYWALK_CHANNEL_W' + elseif filt == 25 then + return 'SKYWALK_CHANNEL_R' + elseif filt == 26 then + return 'FSEVENT' + elseif filt == 27 then + return 'VN' + elseif filt == 28 then + return 'SKYWALK_CHANNEL_E' + elseif filt == 29 then + return 'TTY' + else + return string.format('[%d]', filt) + end +end + +-- kqueue lifecycle + +function processing_begin(workq) + return function(buf) + local prefix = event_prefix_string(buf, workq) + if trace.debugid_is_start(buf.debugid) then + local qos + if workq then + qos = buf.arg2 + else + qos = buf.arg3 + end + printf("%s QoS = %s\n", prefix, qos_string(qos)) + else + printf("%s request thread = 0x%x, kqrequest state = %s\n", prefix, + buf.arg1, state_string(kqrequest_state_strings, buf.arg2)) + end + end +end + +trace_eventname("KEVENT_kq_processing_begin", processing_begin(false)) +trace_eventname("KEVENT_kqwq_processing_begin", processing_begin(true)) +trace_eventname("KEVENT_kqwl_processing_begin", processing_begin(false)) + +function processing_end(workq) + return function(buf) + local qos + if workq then + qos = buf.arg2 + else + qos = buf.arg3 + end + printf("%s QoS = %s\n", event_prefix_string(buf, workq), qos_string(qos)) + end +end + +trace_eventname("KEVENT_kq_processing_end", processing_end(false)) +trace_eventname("KEVENT_kqwq_processing_end", processing_end(true)) +trace_eventname("KEVENT_kqwl_processing_end", processing_end(false)) + +trace_eventname("KEVENT_kqwq_bind", function(buf) + printf("%s thread = 0x%x, QoS = %s, kqrequest state = %s\n", + event_prefix_string(buf, true), buf.arg1, qos_string(buf.arg3), + state_string(kqrequest_state_strings, buf.arg4)) +end) + +trace_eventname("KEVENT_kqwq_unbind", function(buf) + printf("%s thread = 0x%x, QoS = %s\n", event_prefix_string(buf, true), + buf.arg1, qos_string(buf.arg3)) +end) + +trace_eventname("KEVENT_kqwl_bind", function(buf) + qos = buf.arg3 & 0xff + duplicate = buf.arg3 & (1 << 8) + kqr_override_qos_delta = buf.arg4 >> 8 + kqr_state = buf.arg4 & 0xff + + printf("%s thread = 0x%x, QoS = %s, override QoS delta = %d, kqrequest state = %s%s\n", + event_prefix_string(buf, false), buf.arg2, qos_string(qos), + kqr_override_qos_delta, + state_string(kqrequest_state_strings, kqr_state), + duplicate ? ", duplicate" : "") +end) + +trace_eventname("KEVENT_kqwl_unbind", function(buf) + flags = buf.arg3 + qos = buf.arg4 + + printf("%s thread = 0x%x, QoS = %s, flags = 0x%x\n", event_prefix_string(buf, false), + buf.arg2, qos_string(qos), flags) +end) + +function thread_request(workq) + return function(buf) + printf("%s QoS = %s, kqrequest state = %s, override QoS delta = %d\n", + event_prefix_string(buf, workq), qos_string(buf.arg2), + state_string(kqrequest_state_strings, buf.arg3), buf.arg3 >> 8) + end +end + +function thread_adjust(buf) + tid = buf.arg2 + kqr_qos = buf.arg3 >> 8 + new_qos = buf.arg3 & 0xff + kqr_qos_override = buf.arg4 >> 8 + kqr_state = buf.arg4 & 0xff + + printf("%s thread = 0x%x, old/new QoS = %s/%s, old/new override QoS delta = %d/%d, kqrequest state = %s\n", + event_prefix_string(buf, false), + tid, + qos_string(kqr_qos), + qos_string(new_qos), + kqr_qos_override, + new_qos - kqr_qos, + state_string(kqrequest_state_strings, kqr_state)) +end + +trace_eventname("KEVENT_kqwq_thread_request", thread_request(true)) +trace_eventname("KEVENT_kqwl_thread_request", thread_request(false)) +trace_eventname("KEVENT_kqwl_thread_adjust", thread_adjust) + +function kevent_register(workq) + return function(buf) + printf("%s kevent udata = 0x%x, kevent filter = %s, kevent flags = %s\n", + event_prefix_string(buf, workq), buf.arg2, + kevent_filter_string(buf.arg4), + state_string(kevent_flags_strings, buf.arg3)) + end +end + +trace_eventname("KEVENT_kq_register", kevent_register(false)) +trace_eventname("KEVENT_kqwq_register", kevent_register(true)) +trace_eventname("KEVENT_kqwl_register", kevent_register(false)) + +function kevent_process(workq) + return function(buf) + printf("%s kevent ident = 0x%x, udata = 0x%x, kevent filter = %s, knote status = %s\n", + event_prefix_string(buf, workq), buf.arg3 >> 32, buf.arg2, + kevent_filter_string(buf.arg4), + state_string(knote_state_strings, buf.arg3 & 0xffffffff)) + end +end + +trace_eventname("KEVENT_kq_process", kevent_process(false)) +trace_eventname("KEVENT_kqwq_process", kevent_process(true)) +trace_eventname("KEVENT_kqwl_process", kevent_process(false)) diff --git a/tools/trace/parse_ipc_trace.py b/tools/trace/parse_ipc_trace.py old mode 100644 new mode 100755 index 19d9a1401..0ffa386c5 --- a/tools/trace/parse_ipc_trace.py +++ b/tools/trace/parse_ipc_trace.py @@ -441,7 +441,7 @@ class IPCGraph: def print_dot_edge(self, nm, edge, ofile): #weight = 100 * edge.dweight / self.maxdweight - #if weight < 1: + ##if weight < 1: # weight = 1 weight = edge.dweight penwidth = edge.weight / 512 @@ -464,18 +464,20 @@ class IPCGraph: if edge.data('vcpy') > (edge.data('ool') + edge.data('std')): attrs += ',style="dotted"' - #ltype = [] - #if edge.flags & (edge.F_DST_NDFLTQ | edge.F_SRC_NDFLTQ): - # ltype.append('dotted') - #if edge.flags & edge.F_APP_SRC: - # ltype.append('bold') - #if len(ltype) > 0: - # attrs += ',style="' + reduce(lambda a, v: a + ',' + v, ltype) + '"' - # - #if edge.data('ool') > (edge.data('std') + edge.data('vcpy')): - # attrs += ",color=blue" - #if edge.data('vcpy') > (edge.data('ool') + edge.data('std')): - # attrs += ",color=green" + """ # block comment + ltype = [] + if edge.flags & (edge.F_DST_NDFLTQ | edge.F_SRC_NDFLTQ): + ltype.append('dotted') + if edge.flags & edge.F_APP_SRC: + ltype.append('bold') + if len(ltype) > 0: + attrs += ',style="' + reduce(lambda a, v: a + ',' + v, ltype) + '"' + + if edge.data('ool') > (edge.data('std') + edge.data('vcpy')): + attrs += ",color=blue" + if edge.data('vcpy') > (edge.data('ool') + edge.data('std')): + attrs += ",color=green" + """ ofile.write("\t{:s} [{:s}];\n".format(nm, attrs)) -- 2.45.2